@objy/server 0.0.0-stage → 0.1.0

package/src/hnsw.js

@@ -0,0 +1,269 @@
+class Heap {
+  constructor(compare) {
+    this.items = [];
+    this.compare = compare;
+  }
+
+  get size() {
+    return this.items.length;
+  }
+
+  peek() {
+    return this.items[0];
+  }
+
+  push(item) {
+    this.items.push(item);
+    this.up(this.items.length - 1);
+  }
+
+  pop() {
+    const top = this.items[0];
+    const last = this.items.pop();
+    if (this.items.length > 0) {
+      this.items[0] = last;
+      this.down(0);
+    }
+    return top;
+  }
+
+  up(index) {
+    while (index > 0) {
+      const parent = (index - 1) >> 1;
+      if (this.compare(this.items[index], this.items[parent]) >= 0) break;
+      [this.items[index], this.items[parent]] = [this.items[parent], this.items[index]];
+      index = parent;
+    }
+  }
+
+  down(index) {
+    for (;;) {
+      let best = index;
+      const left = index * 2 + 1;
+      const right = left + 1;
+      if (left < this.items.length && this.compare(this.items[left], this.items[best]) < 0) best = left;
+      if (right < this.items.length && this.compare(this.items[right], this.items[best]) < 0) best = right;
+      if (best === index) break;
+      [this.items[index], this.items[best]] = [this.items[best], this.items[index]];
+      index = best;
+    }
+  }
+}
+
+function minHeap() {
+  return new Heap((left, right) => left.dist - right.dist);
+}
+
+function maxHeap() {
+  return new Heap((left, right) => right.dist - left.dist);
+}
+
+export function angularDistance(a, b) {
+  if (!Array.isArray(a) || !Array.isArray(b) || a.length === 0 || a.length !== b.length) return 1;
+  let dot = 0;
+  let na = 0;
+  let nb = 0;
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * b[i];
+    na += a[i] * a[i];
+    nb += b[i] * b[i];
+  }
+  if (na === 0 || nb === 0) return 1;
+  return 1 - dot / (Math.sqrt(na) * Math.sqrt(nb));
+}
+
+export class HNSWIndex {
+  constructor({ M = 16, efConstruction = 200, getVec } = {}) {
+    if (typeof getVec !== "function") {
+      throw new TypeError("HNSWIndex requires getVec(id)");
+    }
+    this.M = positiveInt(M, 16);
+    this.M0 = this.M * 2;
+    this.ef = positiveInt(efConstruction, 200);
+    this.mL = 1 / Math.log(this.M);
+    this.getVec = getVec;
+    this.nodes = new Map();
+    this.entryPoint = null;
+    this.maxLayer = -1;
+  }
+
+  get size() {
+    return this.nodes.size;
+  }
+
+  insert(id, vec) {
+    const key = String(id);
+    if (!Array.isArray(vec)) return;
+    if (this.nodes.has(key)) this._unlinkExisting(key);
+
+    const level = Math.floor(-Math.log(Math.random()) * this.mL);
+    const node = { neighbors: Array.from({ length: level + 1 }, () => []) };
+    this.nodes.set(key, node);
+
+    if (this.entryPoint === null) {
+      this.entryPoint = key;
+      this.maxLayer = level;
+      return;
+    }
+
+    const epVec = this.getVec(this.entryPoint);
+    let entry = epVec ? [{ id: this.entryPoint, dist: angularDistance(vec, epVec) }] : [];
+
+    for (let layer = this.maxLayer; layer > level; layer--) {
+      const found = this._searchLayer(entry, vec, 1, layer);
+      if (found.length) entry = found.slice(0, 1);
+    }
+
+    for (let layer = Math.min(level, this.maxLayer); layer >= 0; layer--) {
+      const maxNeighbors = layer === 0 ? this.M0 : this.M;
+      const candidates = this._searchLayer(entry, vec, this.ef, layer);
+      const neighbors = this._selectNeighbors(candidates, maxNeighbors);
+      node.neighbors[layer] = neighbors.map((neighbor) => neighbor.id);
+
+      for (const neighbor of neighbors) {
+        const other = this.nodes.get(neighbor.id);
+        if (!other) continue;
+        if (!other.neighbors[layer]) other.neighbors[layer] = [];
+        other.neighbors[layer].push(key);
+        if (other.neighbors[layer].length > maxNeighbors) {
+          const otherVec = this.getVec(neighbor.id);
+          if (!otherVec) continue;
+          const pruned = other.neighbors[layer]
+            .map((id) => {
+              const candidateVec = this.getVec(id);
+              return candidateVec ? { id, dist: angularDistance(otherVec, candidateVec) } : null;
+            })
+            .filter(Boolean)
+            .sort((left, right) => left.dist - right.dist);
+          other.neighbors[layer] = this._selectNeighbors(pruned, maxNeighbors).map((item) => item.id);
+        }
+      }
+
+      entry = candidates;
+    }
+
+    if (level > this.maxLayer) {
+      this.maxLayer = level;
+      this.entryPoint = key;
+    }
+  }
+
+  remove(id) {
+    const key = String(id);
+    if (!this.nodes.has(key)) return;
+    this._unlinkExisting(key);
+  }
+
+  search(queryVec, k, ef = Math.max(k, 20)) {
+    if (!this.entryPoint) return [];
+    const epVec = this.getVec(this.entryPoint);
+    if (!epVec) return [];
+
+    let entry = [{ id: this.entryPoint, dist: angularDistance(queryVec, epVec) }];
+    for (let layer = this.maxLayer; layer > 0; layer--) {
+      const found = this._searchLayer(entry, queryVec, 1, layer);
+      if (found.length) entry = found.slice(0, 1);
+    }
+
+    return this._searchLayer(entry, queryVec, Math.max(positiveInt(ef, 20), k), 0).slice(0, k);
+  }
+
+  clear() {
+    this.nodes.clear();
+    this.entryPoint = null;
+    this.maxLayer = -1;
+  }
+
+  _searchLayer(entryPoints, queryVec, ef, layer) {
+    const visited = new Set(entryPoints.map((entry) => entry.id));
+    const candidates = minHeap();
+    const result = maxHeap();
+
+    for (const entry of entryPoints) {
+      candidates.push(entry);
+      result.push(entry);
+    }
+
+    while (candidates.size > 0) {
+      const current = candidates.pop();
+      if (result.size >= ef && current.dist > result.peek().dist) break;
+
+      const node = this.nodes.get(current.id);
+      if (!node) continue;
+      for (const neighborId of node.neighbors[layer] || []) {
+        if (visited.has(neighborId)) continue;
+        visited.add(neighborId);
+        const neighborVec = this.getVec(neighborId);
+        if (!neighborVec) continue;
+        const dist = angularDistance(queryVec, neighborVec);
+        if (result.size < ef || dist < result.peek().dist) {
+          candidates.push({ id: neighborId, dist });
+          result.push({ id: neighborId, dist });
+          if (result.size > ef) result.pop();
+        }
+      }
+    }
+
+    const out = [];
+    while (result.size > 0) out.push(result.pop());
+    return out.reverse();
+  }
+
+  _selectNeighbors(candidates, maxNeighbors) {
+    const selected = [];
+    const discarded = [];
+    for (const candidate of candidates) {
+      if (selected.length >= maxNeighbors) break;
+      const candidateVec = this.getVec(candidate.id);
+      if (!candidateVec) continue;
+      let good = true;
+      for (const current of selected) {
+        const currentVec = this.getVec(current.id);
+        if (currentVec && angularDistance(candidateVec, currentVec) < candidate.dist) {
+          good = false;
+          break;
+        }
+      }
+      if (good) selected.push(candidate);
+      else discarded.push(candidate);
+    }
+    for (const candidate of discarded) {
+      if (selected.length >= maxNeighbors) break;
+      selected.push(candidate);
+    }
+    return selected;
+  }
+
+  _unlinkExisting(id) {
+    const old = this.nodes.get(id);
+    for (let layer = 0; layer < old.neighbors.length; layer++) {
+      for (const neighborId of old.neighbors[layer] || []) {
+        const neighbor = this.nodes.get(neighborId);
+        if (neighbor?.neighbors[layer]) {
+          neighbor.neighbors[layer] = neighbor.neighbors[layer].filter((entry) => entry !== id);
+        }
+      }
+    }
+    this.nodes.delete(id);
+    if (this.entryPoint === id) this._resetEntryPoint();
+  }
+
+  _resetEntryPoint() {
+    let bestId = null;
+    let bestLayer = -1;
+    for (const [id, node] of this.nodes) {
+      const layer = node.neighbors.length - 1;
+      if (layer > bestLayer) {
+        bestId = id;
+        bestLayer = layer;
+      }
+    }
+    this.entryPoint = bestId;
+    this.maxLayer = bestLayer;
+  }
+}
+
+function positiveInt(value, fallback) {
+  const n = Number(value);
+  return Number.isInteger(n) && n > 0 ? n : fallback;
+}

package/src/http-cors.js

@@ -0,0 +1,31 @@
+/** Apply CORS before routing, including error responses and preflight requests. */
+export function createCorsHandler(config = false) {
+  if (config === false || config == null) return () => false;
+  const origins = config.origins;
+  if (!Array.isArray(origins) || !origins.length || origins.some(origin => typeof origin !== 'string')) {
+    throw new TypeError('cors.origins must be a non-empty array of origins');
+  }
+  const allowed = new Set(origins);
+  const methods = 'GET, POST, PUT, PATCH, DELETE, HEAD, OPTIONS';
+  const headers = ['authorization', 'content-type', 'prefer', ...(config.headers || []).map(name => name.toLowerCase())];
+  return (req, res) => {
+    const origin = req.headers.origin;
+    if (!origin) return false;
+    res.setHeader('vary', 'Origin');
+    const accepted = allowed.has('*') || allowed.has(origin);
+    if (accepted) {
+      res.setHeader('access-control-allow-origin', allowed.has('*') ? '*' : origin);
+      res.setHeader('access-control-expose-headers', 'ETag, Link, Retry-After, Content-Disposition');
+    }
+    if (req.method !== 'OPTIONS') return false;
+    const requested = String(req.headers['access-control-request-headers'] || '').toLowerCase().split(',').map(name => name.trim()).filter(Boolean);
+    const method = req.headers['access-control-request-method'];
+    if (!accepted || !methods.split(', ').includes(method) || requested.some(name => !headers.includes(name))) {
+      res.writeHead(403); res.end(); return true;
+    }
+    res.setHeader('access-control-allow-methods', methods);
+    res.setHeader('access-control-allow-headers', headers.join(', '));
+    res.setHeader('access-control-max-age', '600');
+    res.writeHead(204); res.end(); return true;
+  };
+}

package/src/http-router.js

@@ -0,0 +1,162 @@
+import { isNdjsonContentType, parseNdjsonStream } from "./ndjson.js";
+
+export const DEFAULT_MAX_BODY_BYTES = 64 * 1024 * 1024;
+
+export class HttpRouter {
+  constructor({ maxBodyBytes = DEFAULT_MAX_BODY_BYTES } = {}) {
+    this.routes = [];
+    this.maxBodyBytes = maxBodyBytes;
+  }
+
+  add(method, pattern, handler) {
+    this.routes.push({ method, pattern, handler, ...compilePattern(pattern) });
+    return this;
+  }
+
+  get(pattern, handler) {
+    return this.add("GET", pattern, handler);
+  }
+
+  post(pattern, handler) {
+    return this.add("POST", pattern, handler);
+  }
+
+  put(pattern, handler) {
+    return this.add("PUT", pattern, handler);
+  }
+
+  patch(pattern, handler) {
+    return this.add("PATCH", pattern, handler);
+  }
+
+  delete(pattern, handler) {
+    return this.add("DELETE", pattern, handler);
+  }
+
+  async handle(req, res) {
+    const url = new URL(req.url, "http://localhost");
+    req.path = url.pathname;
+    req.query = url.searchParams;
+    req.maxBodyBytes = this.maxBodyBytes;
+
+    for (const route of this.routes) {
+      if (route.method !== req.method) continue;
+      const match = url.pathname.match(route.re);
+      if (!match) continue;
+      req.params = Object.fromEntries(route.names.map((name, i) => [name, decodeURIComponent(match[i + 1])]));
+      await route.handler(req, res);
+      return true;
+    }
+    return false;
+  }
+}
+
+export function json(res, status, body, headers = {}) {
+  // A downstream auth rejection cannot undo work already begun by this node.
+  if (status >= 400 && res.operationStarted && body?.operationStarted === false) body = { ...body, operationStarted: true };
+  const payload = body === undefined ? "" : JSON.stringify(body);
+  res.writeHead(status, {
+    "content-type": "application/json; charset=utf-8",
+    "content-length": Buffer.byteLength(payload),
+    ...headers,
+  });
+  res.end(payload);
+}
+
+export async function readJson(req, fallback = {}) {
+  const raw = await readBody(req);
+  if (!raw) return fallback;
+  return JSON.parse(raw);
+}
+
+export function remoteHttpError(remote, fallback) {
+  const err = new Error(remote.body?.error || fallback);
+  err.status = remote.status;
+  err.body = remote.body;
+  return err;
+}
+
+export function isNdjsonRequest(req) {
+  return isNdjsonContentType(req.headers?.["content-type"]);
+}
+
+export async function* readNdjson(req, { maxObjectBytes = 0 } = {}) {
+  let totalBytes = 0;
+  const counting = (async function* () {
+    for await (const chunk of req) {
+      totalBytes += chunk.length;
+      req._bodyBytes = totalBytes;
+      yield chunk;
+    }
+  })();
+
+  for await (const entry of parseNdjsonStream(counting, { maxObjectBytes, label: "NDJSON" })) {
+    req._bodyBytes = totalBytes;
+    yield entry;
+  }
+}
+
+export function text(res, status, body, contentType = "text/plain; charset=utf-8") {
+  const payload = body || "";
+  res.writeHead(status, {
+    "content-type": contentType,
+    "content-length": Buffer.byteLength(payload),
+  });
+  res.end(payload);
+}
+
+async function readBody(req) {
+  if (req._bodyText !== undefined) return req._bodyText;
+  req._bodyText = await new Promise((resolve, reject) => {
+    const chunks = [];
+    let size = 0;
+    let settled = false;
+    const maxBodyBytes = req.maxBodyBytes ?? DEFAULT_MAX_BODY_BYTES;
+    req.on("data", (chunk) => {
+      size += chunk.length;
+      req._bodyBytes = size;
+      if (size > maxBodyBytes) {
+        if (settled) return;
+        settled = true;
+        const err = new Error("payload too large");
+        err.status = 413;
+        err.body = { error: err.message, maxBodyBytes };
+        reject(err);
+        return;
+      }
+      if (!settled) chunks.push(chunk);
+    });
+    req.on("end", () => {
+      if (settled) return;
+      settled = true;
+      resolve(Buffer.concat(chunks).toString("utf8"));
+    });
+    req.on("error", (err) => {
+      if (settled) return;
+      settled = true;
+      reject(err);
+    });
+  });
+  return req._bodyText;
+}
+
+function compilePattern(pattern) {
+  if (pattern === "/") return { re: /^\/$/, names: [] };
+  const names = [];
+  const source = pattern
+    .split("/")
+    .filter(Boolean)
+    .map((segment) => {
+      if (segment.startsWith(":")) {
+        names.push(segment.slice(1));
+        return "([^/]+)";
+      }
+      return escapeRegex(segment);
+    })
+    .join("/");
+  return { re: new RegExp(`^/${source}$`), names };
+}
+
+function escapeRegex(value) {
+  return String(value).replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}

package/src/index.js

@@ -0,0 +1,137 @@
+import { resolve } from "node:path";
+import { shardMapFromHttp, startRestoreStatusServer, startServer } from "./server.js";
+import { CRDTNode } from "./crdt.js";
+import { loadConfig } from "./config.js";
+import { createFileService } from "./files.js";
+import { createPersistence } from "./persistence.js";
+import { createReplicationScheduler } from "./replication-scheduler.js";
+import { createRestoreStatus } from "./restore-status.js";
+import { createShardMapStore } from "./shard-map-store.js";
+import { createObjectHookRegistry } from "./object-hooks.js";
+import { loadPlugins } from "./plugin-loader.js";
+import { resolveSystemSecrets } from "./system-secrets.js";
+import { isTopologyResharding } from "./shard-map.js";
+import { nodeCompression, readHeapUsageRatio } from "./node-runtime.js";
+
+const config = loadConfig();
+
+const node = new CRDTNode({
+  id: config.node.id,
+  url: config.node.url,
+  retainOps: config.node.retainOps,
+  syncBatchSize: config.node.syncBatchSize,
+  storage: { ...config.storage, compression: nodeCompression, readHeapUsageRatio },
+  vector: config.vector,
+});
+const shardMapStore = createShardMapStore(config.cluster.shardMapStore, config.persistence);
+const persistence = createPersistence(node, config.persistence);
+const sharedSecrets = config.persistence.mode === "s3"
+  ? { storage: persistence.storage, prefix: config.persistence.prefix }
+  : config.cluster.shardMapStore.mode === "s3"
+    ? { storage: shardMapStore.storage, prefix: config.cluster.shardMapStore.prefix }
+    : {};
+const secrets = await resolveSystemSecrets({
+  jwtSecret: config.auth.jwtSecret,
+  clusterSecret: config.cluster.secret,
+  ...sharedSecrets,
+  localPath: resolve(config.persistence.localWal.dir, "..", "system-secrets.json"),
+});
+config.auth = { ...config.auth, jwtSecret: secrets.jwtSecret };
+config.cluster.secret = secrets.clusterSecret;
+const storedShardMap = await shardMapStore?.load();
+const configuredShardMap = storedShardMap || config.cluster.shardMap;
+if (configuredShardMap) {
+  node.replaceShardMap(shardMapFromHttp(configuredShardMap, node.id, {
+    secret: config.cluster.secret,
+    timeoutMs: config.cluster.replicationTimeoutMs,
+  }));
+}
+if (isTopologyResharding(node.shardMapSnapshot()) && (!persistence?.durable || !shardMapStore?.durable)) {
+  throw new Error("resharding requires S3 persistence and an S3 shard map store");
+}
+const replication = createReplicationScheduler({
+  node,
+  hintsEnabled: config.cluster.hints.enabled,
+  antiEntropyMs: config.cluster.syncMs,
+  debounceMs: config.cluster.hints.debounceMs,
+  maxScopesPerHint: config.cluster.hints.maxScopesPerHint,
+});
+const files = createFileService(config.files, config.persistence);
+const restoreStatus = createRestoreStatus();
+const restoreServer = await startRestoreStatusServer({
+  port: config.server.port,
+  node,
+  restoreStatus,
+  replication,
+});
+
+if (persistence) {
+  node.setLogTruncatedHandler?.(({ scope }) => persistence.restoreScope(scope));
+  const restored = await persistence.restore({ progress: restoreStatus });
+  if (restored.reshardCheckpoint && shardMapStore?.durable && persistence.durable) {
+    const readiness = await persistence.reshardReadiness();
+    restored.reshardReadiness = readiness;
+    let activeMap;
+    if (readiness.ready) {
+      const finalized = await shardMapStore.finalize(readiness.topologyId);
+      activeMap = finalized.shardMap;
+      restored.reshardFinalized = true;
+    } else {
+      restoreStatus.waiting({ topologyId: readiness.topologyId });
+      activeMap = await shardMapStore.waitForActive(readiness.topologyId);
+    }
+    node.replaceShardMap(shardMapFromHttp(activeMap, node.id, {
+      secret: config.cluster.secret,
+      timeoutMs: config.cluster.replicationTimeoutMs,
+    }));
+    restoreStatus.done(restored);
+  }
+  persistence.start();
+  console.log(`persistence restored ${restored.snapshotObjects} snapshot objects and ${restored.journalOps} journal ops`);
+} else {
+  restoreStatus.done({ disabled: true });
+}
+await new Promise((resolve) => restoreServer.server.close(resolve));
+
+const objectHooks = createObjectHookRegistry();
+await loadPlugins(config.plugins, { configPath: config.configPath, node, objectHooks });
+
+const state = await startServer({
+  port: restoreServer.port,
+  maxJsonBytes: config.server.maxJsonBytes,
+  cors: config.server.cors,
+  node,
+  shardMapStore,
+  auth: config.auth,
+  persistence,
+  files,
+  restoreStatus,
+  memoryGuard: config.memoryGuard,
+  clusterSecret: config.cluster.secret,
+  forwardTimeoutMs: config.cluster.forwardTimeoutMs,
+  replicationTimeoutMs: config.cluster.replicationTimeoutMs,
+  replication,
+  imports: config.imports,
+  objectHooks,
+});
+replication.start();
+console.log(`OBJY DB node ${state.node.id} listening on http://localhost:${state.port}`);
+
+let shuttingDown = false;
+
+async function shutdown() {
+  if (shuttingDown) return;
+  shuttingDown = true;
+  await replication.stop();
+  await new Promise((resolve) => state.server.close(resolve));
+  try {
+    await persistence?.stop();
+  } catch (err) {
+    console.error(`persistence flush failed during shutdown: ${err.message}`);
+    process.exitCode = 1;
+  }
+  process.exit(process.exitCode || 0);
+}
+
+process.on("SIGINT", shutdown);
+process.on("SIGTERM", shutdown);