@objy/server 0.0.0-stage → 0.1.0

package/src/restore-status.js

@@ -0,0 +1,77 @@
+export function createRestoreStatus() {
+  const state = {
+    status: "idle",
+    phase: "idle",
+    startedAt: null,
+    finishedAt: null,
+    error: null,
+    currentScope: null,
+    scopesTotal: 0,
+    scopesDone: 0,
+    snapshotObjects: 0,
+    journalOps: 0,
+    skippedJournals: 0,
+    skippedJournalOps: 0,
+    localWalOps: 0,
+  };
+
+  return {
+    start(meta = {}) {
+      Object.assign(state, {
+        status: "restoring",
+        phase: "starting",
+        startedAt: new Date().toISOString(),
+        finishedAt: null,
+        error: null,
+        ...meta,
+      });
+    },
+    update(patch = {}) {
+      Object.assign(state, patch);
+    },
+    add(field, count = 1) {
+      state[field] = (Number(state[field]) || 0) + (Number(count) || 0);
+    },
+    scope(scope, index, total) {
+      state.phase = "scope";
+      state.currentScope = scope ? { ...scope } : null;
+      state.scopesDone = Math.max(0, Number(index) || 0);
+      state.scopesTotal = Math.max(state.scopesTotal, Number(total) || 0);
+    },
+    done(stats = {}) {
+      Object.assign(state, {
+        status: "ready",
+        phase: "ready",
+        finishedAt: new Date().toISOString(),
+        currentScope: null,
+        stats: { ...stats },
+      });
+    },
+    fail(error) {
+      Object.assign(state, {
+        status: "error",
+        phase: "error",
+        finishedAt: new Date().toISOString(),
+        error: error?.message || String(error),
+      });
+    },
+    waiting(patch = {}) {
+      Object.assign(state, {
+        status: "restoring",
+        phase: "waiting-topology",
+        finishedAt: null,
+        ...patch,
+      });
+    },
+    snapshot() {
+      return {
+        ...state,
+        currentScope: state.currentScope ? { ...state.currentScope } : null,
+        stats: state.stats ? { ...state.stats } : undefined,
+      };
+    },
+    get restoring() {
+      return state.status === "restoring";
+    },
+  };
+}

package/src/runtime.js

@@ -0,0 +1,47 @@
+const BASE64URL = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_";
+
+export function randomUUID() {
+  const crypto = webCrypto();
+  if (typeof crypto.randomUUID === "function") return crypto.randomUUID();
+
+  const bytes = randomBytes(16);
+  bytes[6] = (bytes[6] & 0x0f) | 0x40;
+  bytes[8] = (bytes[8] & 0x3f) | 0x80;
+  const hex = [...bytes].map((byte) => byte.toString(16).padStart(2, "0")).join("");
+  return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
+}
+
+export function randomBase64Url(size) {
+  const bytes = randomBytes(size);
+  let out = "";
+  for (let index = 0; index < bytes.length; index += 3) {
+    const left = bytes[index];
+    const middle = bytes[index + 1];
+    const right = bytes[index + 2];
+    out += BASE64URL[left >> 2];
+    out += BASE64URL[((left & 3) << 4) | ((middle ?? 0) >> 4)];
+    if (middle !== undefined) out += BASE64URL[((middle & 15) << 2) | ((right ?? 0) >> 6)];
+    if (right !== undefined) out += BASE64URL[right & 63];
+  }
+  return out;
+}
+
+export function yieldToEventLoop() {
+  return new Promise((resolve) => {
+    if (typeof globalThis.setImmediate === "function") globalThis.setImmediate(resolve);
+    else globalThis.setTimeout(resolve, 0);
+  });
+}
+
+function randomBytes(size) {
+  const length = Number(size);
+  if (!Number.isInteger(length) || length <= 0) throw new TypeError("random byte count must be a positive integer");
+  const bytes = new Uint8Array(length);
+  webCrypto().getRandomValues(bytes);
+  return bytes;
+}
+
+function webCrypto() {
+  if (globalThis.crypto?.getRandomValues) return globalThis.crypto;
+  throw new Error("Web Crypto is required");
+}

package/src/server.js

@@ -0,0 +1,242 @@
+import { createServer } from "node:http";
+import { createAuthService } from "./auth.js";
+import { localClusterStatus } from "./cluster-status.js";
+import { CRDTNode } from "./crdt.js";
+import { HttpRouter, json } from "./http-router.js";
+import { createAuthInterface } from "./interfaces/auth.js";
+import { createClusterInterface, httpPeer, shardMapFromHttp } from "./interfaces/cluster.js";
+import { createPublicObjectInterface } from "./interfaces/public-object.js";
+import { createUiInterface } from "./interfaces/ui.js";
+import { createCorsHandler } from "./http-cors.js";
+import { normalizeClusterSecret } from "./cluster-secret.js";
+import { createMemoryGuard } from "./memory-guard.js";
+import { createObjectHookRegistry, createObjectHookRunner } from "./object-hooks.js";
+import { createReplicationScheduler } from "./replication-scheduler.js";
+import { ShardRequestRouter } from "./shard-router.js";
+import { randomSecret } from "./system-secrets.js";
+import { nodeCompression, readHeapUsageRatio } from "./node-runtime.js";
+
+export { createObjectHookRegistry, httpPeer, shardMapFromHttp };
+
+export async function startRestoreStatusServer({
+  port = 0,
+  node,
+  memoryGuard,
+  replication,
+  restoreStatus,
+} = {}) {
+  const n = node || new CRDTNode({ storage: { compression: nodeCompression, readHeapUsageRatio } });
+  const server = createServer((req, res) => {
+    const url = new URL(req.url || "/", "http://localhost");
+    if (url.pathname === "/health") {
+      return json(res, 200, {
+        ok: true,
+        node: n.id,
+        restoring: restoreStatus?.restoring === true,
+        restore: restoreStatus?.snapshot?.() || null,
+      });
+    }
+    if (url.pathname === "/api/config" || url.pathname === "/cluster/status") {
+      return json(res, 200, {
+        ...localClusterStatus(n, memoryGuard, replication, restoreStatus, {
+          details: url.searchParams.get("details") === "1",
+        }),
+        local: true,
+        reachable: true,
+      });
+    }
+    return json(res, 503, {
+      error: "database is restoring",
+      restore: restoreStatus?.snapshot?.() || null,
+    }, { "retry-after": "5" });
+  });
+
+  await new Promise((resolve) => server.listen(port, resolve));
+  return { server, port: server.address().port };
+}
+
+export async function startServer({
+  port = 0,
+  peers = [],
+  shardMap,
+  node,
+  shardMapStore,
+  auth,
+  persistence,
+  durability,
+  files,
+  memoryGuard,
+  clusterSecret,
+  forwardTimeoutMs,
+  replicationTimeoutMs,
+  replication,
+  restoreStatus,
+  replicationHints,
+  maxJsonBytes,
+  imports,
+  objectHooks,
+  environment = process.env,
+  bootstrapLogger = console.log,
+  interfaces = [],
+  useDefaultInterfaces = true,
+  cors = false,
+} = {}) {
+  const handleCors = createCorsHandler(cors);
+  const n = node || new CRDTNode({ storage: { compression: nodeCompression, readHeapUsageRatio } });
+  const secret = normalizeClusterSecret(clusterSecret ?? auth?.clusterSecret ?? auth?.jwtSecret) || randomSecret();
+  const guard = createMemoryGuard(memoryGuard);
+  const hookRunner = createObjectHookRunner(n, objectHooks || createObjectHookRegistry());
+  const scheduler = replication || createReplicationScheduler({ node: n, ...normalizeReplicationHintOptions(replicationHints) });
+  if (persistence?.restoreScope) {
+    n.setLogTruncatedHandler?.(({ scope }) => persistence.restoreScope(scope));
+  }
+
+  if (shardMap) n.replaceShardMap(shardMapFromHttp(shardMap, n.id, { secret, timeoutMs: replicationTimeoutMs }));
+  for (const peer of peers) n.addPeer(httpPeer(peer, n.id, { secret, timeoutMs: replicationTimeoutMs }));
+  const authService = createAuthService(n, auth, { environment });
+  const state = { node: n, auth: authService, server: null, port, router: null, memoryGuard: guard, replication: scheduler, restoreStatus, objectHooks: hookRunner };
+
+  const router = new HttpRouter({ maxBodyBytes: maxJsonBytes });
+  const context = createInterfaceContext({
+    node: n,
+    shardMapStore,
+    auth: authService,
+    persistence,
+    durability,
+    files,
+    memoryGuard: guard,
+    restoreStatus,
+    clusterSecret: secret,
+    forwardTimeoutMs,
+    replicationScheduler: scheduler,
+    imports,
+    objectHooks: hookRunner,
+  });
+  const activeInterfaces = [
+    ...(useDefaultInterfaces ? defaultInterfaces() : []),
+    ...interfaces,
+  ];
+  for (const iface of activeInterfaces) await registerInterface(iface, router, context);
+
+  const server = createServer(async (req, res) => {
+    try {
+      if (handleCors(req, res)) return;
+      if (await router.handle(req, res)) return;
+      json(res, 404, { error: "not found" });
+    } catch (err) {
+      if (!res.writableEnded) json(res, err.status || 500, err.body || { error: err.message }, err.headers || {});
+    }
+  });
+  server.once("close", () => hookRunner.dispose());
+
+  await new Promise((resolve) => server.listen(port, resolve));
+  state.server = server;
+  state.port = server.address().port;
+  state.router = router;
+  const bootstrap = authService.consumeBootstrapNotice?.();
+  if (bootstrap) {
+    await persistence?.afterWrite?.();
+    await persistence?.flush?.();
+    scheduler.afterLocalWrite?.();
+    bootstrapLogger?.(bootstrapMessage(bootstrap));
+  }
+  n.startIndexWarmup?.();
+  n.startVectorIndexWarmup().catch((err) => {
+    console.error(`vector index warmup failed: ${err.message}`);
+  });
+  return state;
+}
+
+function bootstrapMessage({ username, password, supplied }) {
+  if (supplied) return `OBJY DB initialized global administrator ${username} from OBJYDB_ADMIN_PASSWORD`;
+  return [
+    "OBJY DB initialized global administrator",
+    `username: ${username}`,
+    `password: ${password}`,
+    "This password will not be displayed again.",
+  ].join("\n");
+}
+
+export function defaultInterfaces() {
+  return [
+    createClusterInterface(),
+    createAuthInterface(),
+    createPublicObjectInterface(),
+    createUiInterface(),
+  ];
+}
+
+export function createInterfaceContext({
+  node,
+  shardMapStore,
+  auth,
+  persistence,
+  files,
+  durability,
+  memoryGuard,
+  restoreStatus,
+  clusterSecret,
+  forwardTimeoutMs,
+  replicationScheduler,
+  imports,
+  objectHooks,
+}) {
+  const durableWrite = durability || {
+    afterWrite: () => persistence?.afterWrite?.() || Promise.resolve({ records: 0, ops: 0 }),
+    getStats: () => persistence?.getStats?.() || null,
+  };
+  return {
+    node,
+    shardMapStore,
+    persistence,
+    auth,
+    files,
+    imports,
+    clusterSecret,
+    memoryGuard,
+    restoreStatus,
+    routing: new ShardRequestRouter({ node, forwardTimeoutMs, clusterSecret }),
+    objectHooks,
+    replicationScheduler,
+    durability: {
+      ...durableWrite,
+      afterWrite: async () => {
+        const result = await durableWrite.afterWrite();
+        replicationScheduler?.afterLocalWrite?.();
+        return result;
+      },
+    },
+    data: {
+      node,
+      store: node.db,
+    },
+    replication: {
+      syncAll: () => node.syncAll(),
+      httpPeer,
+    },
+    cluster: {
+      shardMapStore,
+      shardMapFromHttp,
+    },
+  };
+}
+
+function normalizeReplicationHintOptions(options = {}) {
+  const { enabled, ...rest } = options || {};
+  return {
+    ...rest,
+    ...(enabled !== undefined ? { hintsEnabled: enabled } : {}),
+  };
+}
+
+async function registerInterface(iface, router, context) {
+  if (typeof iface === "function") {
+    await iface(router, context);
+    return;
+  }
+  if (iface && typeof iface.register === "function") {
+    await iface.register(router, context);
+    return;
+  }
+  throw new TypeError("interface must be a function or an object with register(router, context)");
+}

package/src/shape.js

@@ -0,0 +1,286 @@
+export function valueKind(value) {
+  if (value === null) return "null";
+  if (Array.isArray(value)) return "array";
+  return typeof value;
+}
+
+export function typeOf(value) {
+  return valueKind(value);
+}
+
+export function shapeId(fields) {
+  let h = 0x811c9dc5;
+  const str = JSON.stringify(fields);
+  for (let i = 0; i < str.length; i++) {
+    h ^= str.charCodeAt(i);
+    h = Math.imul(h, 0x01000193);
+  }
+  return (h >>> 0).toString(16).padStart(8, "0");
+}
+
+export function layoutOf(entries) {
+  return [...new Set(entries.map(([name]) => name))]
+    .sort((a, b) => (a < b ? -1 : a > b ? 1 : 0))
+    .map((name) => ({ name }));
+}
+
+function stableKey(value) {
+  return `${valueKind(value)}:${JSON.stringify(sortForKey(value))}`;
+}
+
+function sortForKey(value) {
+  if (Array.isArray(value)) return value.map(sortForKey);
+  if (value && typeof value === "object") {
+    return Object.fromEntries(
+      Object.entries(value)
+        .sort(([a], [b]) => a.localeCompare(b))
+        .map(([key, val]) => [key, sortForKey(val)]),
+    );
+  }
+  return value;
+}
+
+function cloneValue(value) {
+  if (value === null || typeof value !== "object") return value;
+  return structuredClone(value);
+}
+
+function estBytes(value) {
+  if (value === null || value === undefined) return 0;
+  if (typeof value === "number") return 8;
+  if (typeof value === "boolean") return 1;
+  if (typeof value === "string") return value.length * 2;
+  return JSON.stringify(value).length * 2;
+}
+
+class ValueColumn {
+  constructor(capacity = 8) {
+    this.codes = new Array(capacity).fill(0);
+    this.keyToCode = new Map();
+    this.values = [undefined];
+    this.refs = [0];
+    this.freeCodes = [];
+  }
+
+  ensure(row) {
+    if (row < this.codes.length) return;
+    let cap = this.codes.length || 8;
+    while (row >= cap) cap *= 2;
+    this.codes.length = cap;
+    this.codes.fill(0, row + 1);
+  }
+
+  set(row, value) {
+    this.ensure(row);
+    const oldCode = this.codes[row];
+    if (oldCode) this._release(oldCode);
+    const nextCode = this._codeFor(value);
+    this.refs[nextCode]++;
+    this.codes[row] = nextCode;
+  }
+
+  get(row) {
+    return cloneValue(this.values[this.codes[row]]);
+  }
+
+  getForQuery(row) {
+    return this.values[this.codes[row]];
+  }
+
+  clear(row) {
+    const code = this.codes[row];
+    if (!code) return;
+    this._release(code);
+    this.codes[row] = 0;
+  }
+
+  bytes() {
+    let total = this.codes.length * 8;
+    for (const value of this.values) {
+      if (value !== undefined) total += estBytes(value);
+    }
+    return total;
+  }
+
+  stats() {
+    return {
+      kind: "value",
+      capacity: this.codes.length,
+      values: this.keyToCode.size,
+      bytes: this.bytes(),
+    };
+  }
+
+  _codeFor(value) {
+    const key = stableKey(value);
+    const existing = this.keyToCode.get(key);
+    if (existing) return existing;
+
+    const code = this.freeCodes.pop() || this.values.length;
+    this.keyToCode.set(key, code);
+    this.values[code] = cloneValue(value);
+    this.refs[code] = 0;
+    return code;
+  }
+
+  _release(code) {
+    this.refs[code]--;
+    if (this.refs[code] > 0) return;
+
+    const key = stableKey(this.values[code]);
+    this.keyToCode.delete(key);
+    this.values[code] = undefined;
+    this.refs[code] = 0;
+    this.freeCodes.push(code);
+  }
+}
+
+export class Shape {
+  constructor(fields) {
+    this.id = shapeId(fields);
+    this.fields = fields;
+    this.byName = new Set(fields.map((f) => f.name));
+    this.columns = {};
+    for (const f of fields) this.columns[f.name] = new ValueColumn();
+    this.rows = new Map();
+    this.free = [];
+    this.length = 0;
+  }
+
+  hasField(name) {
+    return this.byName.has(name);
+  }
+
+  _ensure(row) {
+    for (const f of this.fields) this.columns[f.name].ensure(row);
+  }
+
+  push(id, values) {
+    const row = this.free.length ? this.free.pop() : this.length++;
+    this._ensure(row);
+    this.rows.set(id, row);
+    for (const f of this.fields) this.columns[f.name].set(row, values[f.name]);
+    return row;
+  }
+
+  set(id, name, value) {
+    const row = this.rows.get(id);
+    if (row === undefined || !this.hasField(name)) return false;
+    this.columns[name].set(row, value);
+    return true;
+  }
+
+  remove(id) {
+    const row = this.rows.get(id);
+    if (row === undefined) return;
+    for (const f of this.fields) this.columns[f.name].clear(row);
+    this.rows.delete(id);
+    this.free.push(row);
+  }
+
+  read(id) {
+    const row = this.rows.get(id);
+    if (row === undefined) return undefined;
+    const out = {};
+    for (const f of this.fields) out[f.name] = this.columns[f.name].get(row);
+    return out;
+  }
+
+  readForQuery(id) {
+    const row = this.rows.get(id);
+    if (row === undefined) return undefined;
+    const out = {};
+    for (const f of this.fields) out[f.name] = this.columns[f.name].getForQuery(row);
+    return out;
+  }
+
+  bytes() {
+    let total = 0;
+    for (const f of this.fields) total += this.columns[f.name].bytes();
+    return total;
+  }
+
+  columnStats() {
+    return Object.fromEntries(this.fields.map((f) => [f.name, this.columns[f.name].stats()]));
+  }
+}
+
+export class ShapeStore {
+  constructor() {
+    this.shapes = new Map();
+    this.loc = new Map();
+  }
+
+  ensureShape(layout) {
+    const id = shapeId(layout);
+    let shape = this.shapes.get(id);
+    if (!shape) {
+      shape = new Shape(layout);
+      this.shapes.set(id, shape);
+    }
+    return shape;
+  }
+
+  rebuild(id, live) {
+    const old = this.loc.get(id);
+    if (old) {
+      const shape = this.shapes.get(old.shapeId);
+      shape.remove(id);
+      if (shape.rows.size === 0) this.shapes.delete(old.shapeId);
+      this.loc.delete(id);
+    }
+    if (!live.length) return;
+    const layout = layoutOf(live);
+    const shape = this.ensureShape(layout);
+    shape.push(id, Object.fromEntries(live));
+    this.loc.set(id, { shapeId: shape.id });
+  }
+
+  writeField(id, field, value, live) {
+    const old = this.loc.get(id);
+    if (old) {
+      const shape = this.shapes.get(old.shapeId);
+      if (shape.hasField(field)) {
+        shape.set(id, field, value);
+        return;
+      }
+    }
+    this.rebuild(id, live);
+  }
+
+  removeObject(id) {
+    const old = this.loc.get(id);
+    if (!old) return;
+    const shape = this.shapes.get(old.shapeId);
+    shape.remove(id);
+    if (shape.rows.size === 0) this.shapes.delete(old.shapeId);
+    this.loc.delete(id);
+  }
+
+  clear() {
+    this.shapes.clear();
+    this.loc.clear();
+  }
+
+  read(id) {
+    const old = this.loc.get(id);
+    if (!old) return undefined;
+    return this.shapes.get(old.shapeId).read(id);
+  }
+
+  readForQuery(id) {
+    const old = this.loc.get(id);
+    if (!old) return undefined;
+    return this.shapes.get(old.shapeId).readForQuery(id);
+  }
+
+  shapeStats() {
+    return [...this.shapes.values()].map((s) => ({
+      shapeId: s.id,
+      fields: s.fields,
+      rows: s.rows.size,
+      bytes: s.bytes(),
+      columns: s.columnStats(),
+    }));
+  }
+}