@objy/server 0.0.0-stage → 0.1.0

package/src/references.js

@@ -0,0 +1,349 @@
+import { compareSortValues, getPathValue } from "./query.js";
+
+const MAX_REF_DEPTH = 8;
+
+export function isReference(value) {
+  return Boolean(
+    isPlainObject(value) &&
+      value.type === "reference" &&
+      typeof value.collection === "string" &&
+      value.collection.length > 0 &&
+      value.value !== undefined &&
+      value.value !== null &&
+      String(value.value).length > 0,
+  );
+}
+
+export function referenceTarget(ref, fallbackDatabase) {
+  if (!isReference(ref)) return null;
+  return {
+    database: ref.database !== undefined && ref.database !== null ? String(ref.database) : fallbackDatabase,
+    collection: String(ref.collection),
+    id: String(ref.value),
+  };
+}
+
+export function normalizeRefDepth(value, fallback = 1) {
+  if (value === undefined || value === null || value === "") return fallback;
+  const n = Number(value);
+  if (!Number.isFinite(n) || n < 0) return fallback;
+  return Math.min(MAX_REF_DEPTH, Math.floor(n));
+}
+
+export function stripReferenceObjects(value) {
+  return stripReferenceObjectsInner(value).value;
+}
+
+export function prepareReferenceValue(value) {
+  const prepared = stripReferenceObjectsInner(value);
+  return {
+    value: prepared.value,
+    hasReference: prepared.hasReference,
+  };
+}
+
+export function expandReferencesSync(value, {
+  database,
+  depth = 1,
+  getObject,
+  seen = new Set(),
+} = {}) {
+  const refDepth = normalizeRefDepth(depth, 1);
+  const prepared = prepareReferenceValue(value);
+  if (refDepth <= 0 || !prepared.hasReference) return prepared.value;
+  return expandReferencesSyncInner(prepared.value, {
+    database,
+    depth: refDepth,
+    getObject,
+    seen,
+  });
+}
+
+export async function expandReferences(value, {
+  database,
+  depth = 1,
+  getObject,
+  seen = new Set(),
+} = {}) {
+  const refDepth = normalizeRefDepth(depth, 1);
+  const prepared = prepareReferenceValue(value);
+  if (refDepth <= 0 || !prepared.hasReference) return prepared.value;
+  return expandReferencesInner(prepared.value, {
+    database,
+    depth: refDepth,
+    getObject,
+    seen,
+  });
+}
+
+export function referenceDepthForQuery(query = {}) {
+  return referenceDepthForCriteria(query);
+}
+
+export function referenceDepthForAggregateSpec(spec = {}) {
+  if (!isPlainObject(spec)) return 0;
+  let depth = 0;
+  for (const path of asArray(spec.groupBy)) depth = Math.max(depth, referenceDepthForPath(path));
+  for (const metric of ["sum", "avg", "min", "max"]) {
+    for (const path of asArray(spec[metric])) depth = Math.max(depth, referenceDepthForPath(path));
+  }
+  if (isPlainObject(spec.fields)) {
+    for (const definition of Object.values(spec.fields)) {
+      if (!isPlainObject(definition)) continue;
+      for (const [operator, operand] of Object.entries(definition)) {
+        if (operator === "$count" && operand === true) continue;
+        depth = Math.max(depth, referenceDepthForPath(operand));
+      }
+    }
+  }
+  return depth;
+}
+
+export function pathHasReferenceObject(path) {
+  return referenceDepthForPath(path) > 0;
+}
+
+export function referenceReadPlan(criteria = {}, options = {}) {
+  const queryDepth = referenceDepthForQuery(criteria);
+  const sortDepth = pathHasReferenceObject(options.sortBy) ? 1 : 0;
+  const matchDepth = Math.max(queryDepth, sortDepth);
+  const responseDepth = normalizeRefDepth(options.refDepth, matchDepth > 0 ? 1 : 0);
+  return {
+    queryDepth,
+    sortDepth,
+    matchDepth,
+    responseDepth,
+    expansionDepth: Math.max(matchDepth, normalizeRefDepth(options.refDepth, 0)),
+  };
+}
+
+export function sortReferencePairs(pairs, options = {}) {
+  if (!options.sortBy) return pairs;
+  const direction = options.sort === "desc" ? -1 : 1;
+  return [...pairs].sort((left, right) => {
+    return compareSortValues(
+      getPathValue(left.matchObject, options.sortBy),
+      getPathValue(right.matchObject, options.sortBy),
+    ) * direction;
+  });
+}
+
+export function paginateReferencePairs(pairs, options = {}) {
+  const offset = options.offset || 0;
+  const limit = options.limit ?? Infinity;
+  if (limit <= 0) return [];
+  return pairs.slice(offset, offset + limit);
+}
+
+export function baseCriteriaForReferenceQuery(query = {}) {
+  if (!isPlainObject(query)) return {};
+  const out = {};
+
+  for (const [path, expected] of Object.entries(query)) {
+    if (path === "$and" && Array.isArray(expected)) {
+      const children = expected.map(baseCriteriaForReferenceQuery).filter((child) => Object.keys(child).length > 0);
+      if (children.length) out.$and = children;
+      continue;
+    }
+    if (path.startsWith("$")) continue;
+    if (pathHasReferenceObject(path) || valueHasReferencePath(expected)) continue;
+    out[path] = expected;
+  }
+
+  return out;
+}
+
+function stripReferenceObjectsInner(value) {
+  if (Array.isArray(value)) {
+    let changed = false;
+    let hasReference = false;
+    const out = value.map((item) => {
+      const stripped = stripReferenceObjectsInner(item);
+      changed = changed || stripped.changed;
+      hasReference = hasReference || stripped.hasReference;
+      return stripped.value;
+    });
+    return changed
+      ? { value: out, changed: true, hasReference }
+      : { value, changed: false, hasReference };
+  }
+
+  if (!isPlainObject(value)) return { value, changed: false, hasReference: false };
+
+  let changed = false;
+  let hasReference = false;
+  const out = {};
+  const reference = isReference(value);
+  if (reference) hasReference = true;
+  for (const [key, item] of Object.entries(value)) {
+    if (reference && key === "object") {
+      changed = true;
+      continue;
+    }
+    const stripped = stripReferenceObjectsInner(item);
+    changed = changed || stripped.changed;
+    hasReference = hasReference || stripped.hasReference;
+    out[key] = stripped.value;
+  }
+  return changed
+    ? { value: out, changed: true, hasReference }
+    : { value, changed: false, hasReference };
+}
+
+function expandReferencesSyncInner(value, context) {
+  if (Array.isArray(value)) return value.map((item) => expandReferencesSyncInner(item, context));
+
+  if (isReference(value)) return expandReferenceSync(value, context);
+
+  if (isPlainObject(value)) {
+    return Object.fromEntries(
+      Object.entries(value).map(([key, item]) => [key, expandReferencesSyncInner(item, context)]),
+    );
+  }
+
+  return value;
+}
+
+function expandReferenceSync(ref, context) {
+  const clean = cloneReferenceDescriptor(ref);
+  if (context.depth <= 0 || typeof context.getObject !== "function") return clean;
+
+  const target = referenceTarget(ref, context.database);
+  if (!target?.database) {
+    clean.object = null;
+    return clean;
+  }
+
+  const key = referenceKey(target);
+  if (context.seen.has(key)) {
+    clean.object = null;
+    return clean;
+  }
+
+  const nextSeen = new Set(context.seen);
+  nextSeen.add(key);
+  const object = context.getObject(target.database, target.collection, target.id);
+  clean.object = object
+    ? expandReferencesSyncInner(object, {
+      ...context,
+      database: target.database,
+      depth: context.depth - 1,
+      seen: nextSeen,
+    })
+    : null;
+  return clean;
+}
+
+async function expandReferencesInner(value, context) {
+  if (Array.isArray(value)) {
+    const out = [];
+    for (const item of value) out.push(await expandReferencesInner(item, context));
+    return out;
+  }
+
+  if (isReference(value)) return expandReference(value, context);
+
+  if (isPlainObject(value)) {
+    const out = {};
+    for (const [key, item] of Object.entries(value)) out[key] = await expandReferencesInner(item, context);
+    return out;
+  }
+
+  return value;
+}
+
+async function expandReference(ref, context) {
+  const clean = cloneReferenceDescriptor(ref);
+  if (context.depth <= 0 || typeof context.getObject !== "function") return clean;
+
+  const target = referenceTarget(ref, context.database);
+  if (!target?.database) {
+    clean.object = null;
+    return clean;
+  }
+
+  const key = referenceKey(target);
+  if (context.seen.has(key)) {
+    clean.object = null;
+    return clean;
+  }
+
+  const nextSeen = new Set(context.seen);
+  nextSeen.add(key);
+  const object = await context.getObject(target.database, target.collection, target.id);
+  clean.object = object
+    ? await expandReferencesInner(object, {
+      ...context,
+      database: target.database,
+      depth: context.depth - 1,
+      seen: nextSeen,
+    })
+    : null;
+  return clean;
+}
+
+function cloneReferenceDescriptor(ref) {
+  const out = {};
+  for (const [key, value] of Object.entries(ref || {})) {
+    if (key === "object") continue;
+    out[key] = stripReferenceObjects(value);
+  }
+  return out;
+}
+
+function referenceDepthForCriteria(criteria) {
+  if (!isPlainObject(criteria)) return 0;
+  let depth = 0;
+
+  for (const [path, expected] of Object.entries(criteria)) {
+    if ((path === "$and" || path === "$or" || path === "$nor") && Array.isArray(expected)) {
+      for (const child of expected) depth = Math.max(depth, referenceDepthForCriteria(child));
+      continue;
+    }
+    if (path === "$not") {
+      depth = Math.max(depth, referenceDepthForCriteria(expected));
+      continue;
+    }
+    if (path === "$count" && isPlainObject(expected) && Array.isArray(expected.$constraints)) {
+      for (const child of expected.$constraints) depth = Math.max(depth, referenceDepthForCriteria(child));
+      continue;
+    }
+    if (!path.startsWith("$")) depth = Math.max(depth, referenceDepthForPath(path));
+    depth = Math.max(depth, referenceDepthForOperatorValue(expected));
+  }
+
+  return depth;
+}
+
+function referenceDepthForOperatorValue(value) {
+  if (Array.isArray(value)) return Math.max(0, ...value.map(referenceDepthForOperatorValue));
+  if (!isPlainObject(value)) return 0;
+  let depth = 0;
+  for (const [key, item] of Object.entries(value)) {
+    if (key === "$elemMatch") depth = Math.max(depth, referenceDepthForCriteria(item));
+    else depth = Math.max(depth, referenceDepthForOperatorValue(item));
+  }
+  return depth;
+}
+
+function valueHasReferencePath(value) {
+  return referenceDepthForOperatorValue(value) > 0;
+}
+
+function referenceDepthForPath(path) {
+  if (typeof path !== "string" || path.length === 0) return 0;
+  return path.split(".").filter((part) => part === "object").length;
+}
+
+function referenceKey({ database, collection, id }) {
+  return `${database}\0${collection}\0${id}`;
+}
+
+function asArray(value) {
+  if (value === undefined || value === null) return [];
+  return Array.isArray(value) ? value : [value];
+}
+
+function isPlainObject(value) {
+  return value !== null && typeof value === "object" && !Array.isArray(value);
+}

package/src/replication-scheduler.js

@@ -0,0 +1,303 @@
+import { GLOBAL_COLLECTION, GLOBAL_DATABASE, peerKey } from "./shard-map.js";
+
+const DEFAULT_DEBOUNCE_MS = 25;
+const DEFAULT_MAX_SCOPES_PER_HINT = 128;
+
+export function createReplicationScheduler(options = {}) {
+  return new ReplicationScheduler(options);
+}
+
+export class ReplicationScheduler {
+  constructor({
+    node,
+    enabled = true,
+    hintsEnabled = true,
+    antiEntropyMs = 0,
+    debounceMs = DEFAULT_DEBOUNCE_MS,
+    maxScopesPerHint = DEFAULT_MAX_SCOPES_PER_HINT,
+  } = {}) {
+    if (!node) throw new Error("node is required");
+    this.node = node;
+    this.enabled = enabled !== false;
+    this.hintsEnabled = hintsEnabled !== false;
+    this.antiEntropyMs = positiveInt(antiEntropyMs, 0);
+    this.debounceMs = positiveInt(debounceMs, DEFAULT_DEBOUNCE_MS);
+    this.maxScopesPerHint = positiveInt(maxScopesPerHint, DEFAULT_MAX_SCOPES_PER_HINT);
+    this.pendingHintScopes = new Map();
+    this.pullQueue = new Map();
+    this.inflightPulls = new Set();
+    this.hintTimer = null;
+    this.antiEntropyTimer = null;
+    this.syncingAll = false;
+    this.stats = {
+      hintsSent: 0,
+      hintsReceived: 0,
+      hintScopesSent: 0,
+      hintScopesReceived: 0,
+      pullSyncs: 0,
+      antiEntropySyncs: 0,
+      lastHintSentAt: null,
+      lastHintReceivedAt: null,
+      lastSyncAt: null,
+      lastPartialSyncAt: null,
+      lastError: null,
+      lastSyncFailures: [],
+    };
+    this.unsubscribe = this.node.onChange((change) => this._recordLocalChange(change));
+  }
+
+  start() {
+    if (!this.enabled || this.antiEntropyTimer || !this.antiEntropyMs) return;
+    this.antiEntropyTimer = setInterval(() => {
+      this.syncAll("anti-entropy");
+    }, this.antiEntropyMs);
+    this.antiEntropyTimer.unref?.();
+  }
+
+  async stop() {
+    if (this.hintTimer) clearTimeout(this.hintTimer);
+    if (this.antiEntropyTimer) clearInterval(this.antiEntropyTimer);
+    this.hintTimer = null;
+    this.antiEntropyTimer = null;
+    this.unsubscribe?.();
+  }
+
+  afterLocalWrite() {
+    if (!this.enabled || !this.hintsEnabled || !this.pendingHintScopes.size) return;
+    if (this.hintTimer) return;
+    this.hintTimer = setTimeout(() => {
+      this.hintTimer = null;
+      this.flushHints();
+    }, this.debounceMs);
+    this.hintTimer.unref?.();
+  }
+
+  async flushHints() {
+    if (!this.enabled || !this.hintsEnabled || !this.pendingHintScopes.size) return { peers: 0, scopes: 0 };
+    const scopes = [...this.pendingHintScopes.values()];
+    this.pendingHintScopes.clear();
+    const byPeer = this._hintTargets(scopes);
+    const sends = [];
+    let sentPeers = 0;
+    let sentScopes = 0;
+
+    for (const [key, { peer, scopes: peerScopes }] of byPeer) {
+      for (const chunk of chunks(peerScopes, this.maxScopesPerHint)) {
+        sentPeers++;
+        sentScopes += chunk.length;
+        sends.push(this._sendHint(peer, chunk, key));
+      }
+    }
+    await Promise.all(sends);
+    return { peers: sentPeers, scopes: sentScopes };
+  }
+
+  receiveHint(hint = {}) {
+    if (!this.enabled || !this.hintsEnabled) return { accepted: 0, matchedPeer: false };
+    const scopes = normalizeHintScopes(hint.scopes);
+    let accepted = 0;
+    let matchedPeer = false;
+
+    for (const scope of scopes) {
+      const peer = this._peerForHint(scope.shard, hint);
+      if (!peer) continue;
+      matchedPeer = true;
+      if (!this.node.ownsShard(scope.shard)) continue;
+      this.enqueuePull(peer, this.node.syncScopeForShard(scope.shard));
+      accepted++;
+    }
+
+    this.stats.hintsReceived++;
+    this.stats.hintScopesReceived += accepted;
+    this.stats.lastHintReceivedAt = new Date().toISOString();
+    return { accepted, matchedPeer };
+  }
+
+  enqueuePull(peer, scope) {
+    const key = pullKey(peer, scope);
+    this.pullQueue.set(key, { peer, scope });
+    this._drainPullQueue();
+  }
+
+  async syncAll(reason = "manual") {
+    if (!this.enabled || this.syncingAll) return 0;
+    this.syncingAll = true;
+    try {
+      const applied = await this.node.syncAll();
+      if (reason === "anti-entropy") this.stats.antiEntropySyncs++;
+      const failures = Array.isArray(this.node.lastSyncFailures) ? this.node.lastSyncFailures : [];
+      this.stats.lastSyncFailures = failures;
+      if (failures.length) {
+        this.stats.lastPartialSyncAt = new Date().toISOString();
+        this._rememberError(new Error(`${failures.length} peer sync(s) failed`), "sync");
+      } else {
+        this.stats.lastSyncAt = new Date().toISOString();
+        this._rememberSuccess();
+      }
+      return applied;
+    } catch (err) {
+      this._rememberError(err);
+      return 0;
+    } finally {
+      this.syncingAll = false;
+    }
+  }
+
+  status() {
+    return {
+      mode: "hinted-pull",
+      enabled: this.enabled,
+      hintsEnabled: this.hintsEnabled,
+      antiEntropyMs: this.antiEntropyMs,
+      debounceMs: this.debounceMs,
+      maxScopesPerHint: this.maxScopesPerHint,
+      pendingHintScopes: this.pendingHintScopes.size,
+      queuedPulls: this.pullQueue.size,
+      inflightPulls: this.inflightPulls.size,
+      syncingAll: this.syncingAll,
+      ...this.stats,
+    };
+  }
+
+  _recordLocalChange(change) {
+    if (!this.enabled || !this.hintsEnabled || change?.kind !== "ops") return;
+    for (const op of change.ops || []) {
+      if (op?.from !== this.node.id || !op.shard) continue;
+      const scope = explicitShardScope(op.shard, op.seq);
+      const key = scopeKey(scope);
+      const pending = this.pendingHintScopes.get(key);
+      if (!pending || scope.lastSeq > pending.lastSeq) this.pendingHintScopes.set(key, scope);
+    }
+  }
+
+  _hintTargets(scopes) {
+    const byPeer = new Map();
+    for (const scope of scopes) {
+      for (const peer of this.node.peersForShard(scope.shard)) {
+        if (!peer.url || isSelfPeer(this.node, peer)) continue;
+        const key = peerKey(peer);
+        const entry = byPeer.get(key) || { peer, scopes: [] };
+        entry.scopes.push(scope);
+        byPeer.set(key, entry);
+      }
+    }
+    return byPeer;
+  }
+
+  async _sendHint(peer, scopes, key = peerKey(peer)) {
+    if (!peer.hint) return;
+    try {
+      await peer.hint({
+        node: this.node.id,
+        url: this.node.url || undefined,
+        scopes,
+      });
+      this.stats.hintsSent++;
+      this.stats.hintScopesSent += scopes.length;
+      this.stats.lastHintSentAt = new Date().toISOString();
+      this._rememberSuccess(`hint ${key}`);
+    } catch (err) {
+      this._rememberError(err, `hint ${key}`);
+    }
+  }
+
+  _peerForHint(shard, hint) {
+    return this.node.peersForShard(shard).find((peer) => (
+      !isSelfPeer(this.node, peer) &&
+      (
+        (hint.url && normalizeUrl(peer.url) === normalizeUrl(hint.url)) ||
+        (hint.node && (peer.nodeId === hint.node || peer.key === hint.node))
+      )
+    ));
+  }
+
+  _drainPullQueue() {
+    for (const [key, item] of this.pullQueue) {
+      if (this.inflightPulls.has(key)) continue;
+      this.pullQueue.delete(key);
+      this._runPull(key, item.peer, item.scope);
+    }
+  }
+
+  async _runPull(key, peer, scope) {
+    this.inflightPulls.add(key);
+    try {
+      await this.node.syncPeer(peer, scope);
+      this.stats.pullSyncs++;
+      this.stats.lastSyncAt = new Date().toISOString();
+      this._rememberSuccess(`pull ${key}`);
+    } catch (err) {
+      this._rememberError(err, `pull ${key}`);
+    } finally {
+      this.inflightPulls.delete(key);
+      if (this.pullQueue.has(key)) setImmediate(() => this._drainPullQueue());
+    }
+  }
+
+  _rememberError(err, prefix = "") {
+    const message = err?.message || String(err);
+    this.stats.lastError = prefix ? `${prefix}: ${message}` : message;
+  }
+
+  _rememberSuccess(prefix = "") {
+    if (!this.stats.lastError) return;
+    if (!prefix || this.stats.lastError.startsWith(`${prefix}:`)) this.stats.lastError = null;
+  }
+}
+
+function explicitShardScope(shard, lastSeq = 0) {
+  return {
+    database: GLOBAL_DATABASE,
+    collection: GLOBAL_COLLECTION,
+    shard,
+    lastSeq: Number(lastSeq) || 0,
+  };
+}
+
+function normalizeHintScopes(scopes) {
+  if (!Array.isArray(scopes) || !scopes.length) {
+    const err = new Error("body.scopes must be a non-empty array");
+    err.status = 400;
+    throw err;
+  }
+  return scopes.map((scope, index) => {
+    if (!scope || typeof scope !== "object" || typeof scope.shard !== "string" || !scope.shard) {
+      const err = new Error(`scope at index ${index} must include shard`);
+      err.status = 400;
+      throw err;
+    }
+    return explicitShardScope(scope.shard, scope.lastSeq);
+  });
+}
+
+function chunks(values, size) {
+  const n = Math.max(1, size);
+  const out = [];
+  for (let i = 0; i < values.length; i += n) out.push(values.slice(i, i + n));
+  return out;
+}
+
+function pullKey(peer, scope) {
+  return `${peerKey(peer)}|${scopeKey(scope)}`;
+}
+
+function scopeKey(scope) {
+  return scope ? `${scope.database}/${scope.collection}/${scope.shard}` : "*";
+}
+
+function normalizeUrl(url) {
+  return url ? String(url).replace(/\/+$/, "") : null;
+}
+
+function isSelfPeer(node, peer) {
+  return (
+    (peer.url && normalizeUrl(peer.url) === node.url) ||
+    peer.nodeId === node.id ||
+    peer.key === node.id
+  );
+}
+
+function positiveInt(value, fallback) {
+  const n = Number(value);
+  return Number.isFinite(n) && n >= 0 ? Math.floor(n) : fallback;
+}