graphlin 0.1.3 → 0.2.0

package/runtime/daemon/model-persistence.mjs

@@ -0,0 +1,220 @@
+import { constants } from 'node:fs';
+import { lstat, open, realpath, rename, rm } from 'node:fs/promises';
+import { randomUUID } from 'node:crypto';
+import path from 'node:path';
+import { readPrivateJSON } from './paths.mjs';
+
+export const MAX_MODEL_STATE_BYTES = 48 * 1024 * 1024;
+const MAX_NODES = 2_000_000, MAX_DEPTH = 64;
+const COLLECTIONS = ['entities', 'relations', 'interpretations', 'activity', 'sessions', 'checkpoints'];
+const ROOT_FIELDS = new Set(['schemaVersion', 'projectId', 'revision', 'sequence', 'coverage', 'storage', ...COLLECTIONS]);
+const plain = value => value !== null && typeof value === 'object' &&
+  (Object.getPrototypeOf(value) === Object.prototype || Object.getPrototypeOf(value) === null);
+const natural = value => Number.isSafeInteger(value) && value >= 0;
+const id = value => typeof value === 'string' && /^[A-Za-z0-9][A-Za-z0-9_.:-]{0,159}$/.test(value);
+const invalid = () => { throw new Error('invalid_model_state'); };
+
+function validateSnapshot(value, projectId, depth = 0) {
+  if (!plain(value) || depth > 8 || value.schemaVersion !== 2 || value.projectId !== projectId ||
+      !natural(value.revision) || !natural(value.sequence) || !plain(value.coverage) ||
+      Object.keys(value).some(key => !ROOT_FIELDS.has(key)) ||
+      COLLECTIONS.some(key => !Array.isArray(value[key])) ||
+      (value.storage !== undefined && !plain(value.storage))) invalid();
+  for (const checkpoint of value.checkpoints) {
+    if (!plain(checkpoint) || !id(checkpoint.id) ||
+        (checkpoint.projectId !== undefined && checkpoint.projectId !== projectId)) invalid();
+    if (checkpoint.state !== undefined) validateSnapshot(checkpoint.state, projectId, depth + 1);
+  }
+}
+
+// Count exact UTF-8 JSON bytes before allocating the complete encoded snapshot.
+// Reject cycles, getters and non-JSON values instead of executing toJSON hooks or
+// silently losing Map/BigInt values. Undefined object fields are omitted, as in
+// JSON.stringify: current policy uses them to withhold display/path metadata.
+function boundedJSON(value, limit) {
+  let size = 0, nodes = 0;
+  const ancestors = new Set();
+  const add = count => { size += count; if (size > limit) invalid(); };
+  function string(value) {
+    add(2);
+    for (let i = 0; i < value.length; i++) {
+      const code = value.charCodeAt(i);
+      if (code === 34 || code === 92) add(2);
+      else if (code < 32) add([8, 9, 10, 12, 13].includes(code) ? 2 : 6);
+      else if (code < 128) add(1);
+      else if (code < 2048) add(2);
+      else if (code >= 0xd800 && code <= 0xdbff &&
+          value.charCodeAt(i + 1) >= 0xdc00 && value.charCodeAt(i + 1) <= 0xdfff) { add(4); i++; }
+      else if (code >= 0xd800 && code <= 0xdfff) add(6);
+      else add(3);
+    }
+  }
+  function visit(value, depth) {
+    if (++nodes > MAX_NODES || depth > MAX_DEPTH) invalid();
+    if (value === null) { add(4); return; }
+    if (typeof value === 'string') { string(value); return; }
+    if (typeof value === 'boolean') { add(value ? 4 : 5); return; }
+    if (typeof value === 'number' && Number.isFinite(value)) { add(JSON.stringify(value).length); return; }
+    if ((!plain(value) && !Array.isArray(value)) || ancestors.has(value)) invalid();
+    ancestors.add(value);
+    const keys = Reflect.ownKeys(value);
+    if (keys.some(key => typeof key !== 'string')) invalid();
+    add(2);
+    if (Array.isArray(value)) {
+      if (value.length > MAX_NODES || keys.length !== value.length + 1) invalid();
+      for (let i = 0; i < value.length; i++) {
+        const descriptor = Object.getOwnPropertyDescriptor(value, String(i));
+        if (!descriptor || !Object.hasOwn(descriptor, 'value') || !descriptor.enumerable) invalid();
+        if (i) add(1);
+        visit(descriptor.value, depth + 1);
+      }
+    } else {
+      let emitted = 0;
+      for (let i = 0; i < keys.length; i++) {
+        const key = keys[i], descriptor = Object.getOwnPropertyDescriptor(value, key);
+        if (!Object.hasOwn(descriptor, 'value') || !descriptor.enumerable) invalid();
+        if (descriptor.value === undefined) continue;
+        if (emitted++) add(1);
+        string(key); add(1); visit(descriptor.value, depth + 1);
+      }
+    }
+    ancestors.delete(value);
+  }
+  visit(value, 0);
+  const result = JSON.stringify(value);
+  if (Buffer.byteLength(result) !== size || size > limit) invalid();
+  return result;
+}
+
+function owned(info) {
+  return process.getuid === undefined || info.uid === process.getuid();
+}
+async function safeDirectory(directory) {
+  const info = await lstat(directory);
+  if (!info.isDirectory() || info.isSymbolicLink() || !owned(info) ||
+      (info.mode & 0o077) !== 0 || await realpath(directory) !== directory) invalid();
+  return info;
+}
+async function safeDestination(filename) {
+  try {
+    const info = await lstat(filename);
+    if (!info.isFile() || info.isSymbolicLink() || info.nlink !== 1 || !owned(info) ||
+        (info.mode & 0o077) !== 0) invalid();
+  } catch (error) { if (error.code !== 'ENOENT') throw error; }
+}
+
+/**
+ * The parent MUST pass model.snapshot({ persistent: true }) under current policy.
+ * This storage layer preserves that JSON (including storage/checkpoint state);
+ * it is not an export projector, source reader, migration, or policy authority.
+ *
+ * Use a separate, canonical private path such as <project-data>/model-state.json.
+ * The parent retains its existing state.json writer and owns daemon locking.
+ * An omitted projectId binds to the first successfully loaded/scheduled model.
+ */
+export function createModelPersistence(filename, {
+  projectId, maxBytes = MAX_MODEL_STATE_BYTES, debounceMs = 100, now = Date.now,
+} = {}) {
+  if (typeof filename !== 'string' || !filename || filename.includes('\0') || !path.isAbsolute(filename) ||
+      path.resolve(filename) !== filename || path.basename(filename).toLowerCase() === 'state.json' ||
+      (projectId !== undefined && !id(projectId)) || !Number.isSafeInteger(maxBytes) || maxBytes < 1 || maxBytes > MAX_MODEL_STATE_BYTES ||
+      !Number.isInteger(debounceMs) || debounceMs < 0 || debounceMs > 1000 || typeof now !== 'function') {
+    throw new TypeError('invalid_model_persistence_options');
+  }
+  const directory = path.dirname(filename);
+  let pending = null, timer = null, running = null, closing = null, closed = false;
+  let failures = 0, writes = 0;
+
+  async function write(body) {
+    const parent = await safeDirectory(directory);
+    await safeDestination(filename);
+    const temporary = path.join(directory, `.${path.basename(filename)}.${randomUUID()}.tmp`);
+    let file, directoryHandle;
+    try {
+      directoryHandle = await open(directory, constants.O_RDONLY | (constants.O_DIRECTORY ?? 0) | (constants.O_NOFOLLOW ?? 0));
+      const opened = await directoryHandle.stat();
+      if (opened.dev !== parent.dev || opened.ino !== parent.ino) invalid();
+      file = await open(temporary, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL |
+        (constants.O_NOFOLLOW ?? 0), 0o600);
+      await file.writeFile(body, 'utf8');
+      await file.sync();
+      await file.close(); file = null;
+      const current = await safeDirectory(directory);
+      if (current.dev !== parent.dev || current.ino !== parent.ino) invalid();
+      await safeDestination(filename);
+      await rename(temporary, filename);
+      try { await directoryHandle.sync(); }
+      catch (error) { if (!['EINVAL', 'ENOTSUP'].includes(error.code)) throw error; }
+      writes++;
+    } finally {
+      await file?.close().catch(() => {});
+      await directoryHandle?.close().catch(() => {});
+      await rm(temporary, { force: true }).catch(() => {});
+    }
+  }
+  function drain() {
+    if (running) return running;
+    running = (async () => {
+      while (pending !== null) {
+        const body = pending; pending = null;
+        try { await write(body); } catch { failures++; }
+      }
+    })().finally(() => { running = null; });
+    return running;
+  }
+  async function flush() {
+    do {
+      clearTimeout(timer); timer = null;
+      await drain();
+    } while (pending !== null || running !== null);
+  }
+  async function load() {
+    try {
+      await safeDirectory(directory);
+      const envelope = await readPrivateJSON(filename, maxBytes);
+      if (!plain(envelope) || envelope.schemaVersion !== 2 || !id(envelope.projectId) ||
+          (projectId !== undefined && envelope.projectId !== projectId) ||
+          !natural(envelope.savedAt) || envelope.savedAt > now() + 60_000 ||
+          Object.keys(envelope).some(key => !['schemaVersion', 'savedAt', 'projectId', 'snapshot'].includes(key))) invalid();
+      // Revalidate the same depth, JSON and schema limits when reading a file
+      // made by an older daemon. Never delete an incompatible or invalid file.
+      boundedJSON(envelope, maxBytes);
+      validateSnapshot(envelope.snapshot, envelope.projectId);
+      projectId ??= envelope.projectId;
+      return envelope.snapshot;
+    } catch (error) {
+      if (error?.code !== 'ENOENT') failures++;
+      return undefined;
+    }
+  }
+  function schedule(snapshot) {
+    if (closed) return false;
+    try {
+      const savedAt = now();
+      if (!natural(savedAt)) invalid();
+      const candidateProjectId = projectId ?? (plain(snapshot)
+        ? Object.getOwnPropertyDescriptor(snapshot, 'projectId')?.value : undefined);
+      if (!id(candidateProjectId)) invalid();
+      // Bounded serialization runs first, before reading any schema properties
+      // that could otherwise invoke a caller-provided getter.
+      const body = boundedJSON({ schemaVersion: 2, savedAt, projectId: candidateProjectId, snapshot }, maxBytes);
+      validateSnapshot(snapshot, candidateProjectId);
+      projectId ??= candidateProjectId;
+      pending = body; // An immutable serialization, never a caller-owned object.
+      if (!timer && !running) {
+        timer = setTimeout(() => { timer = null; void drain(); }, debounceMs);
+        timer.unref?.();
+      }
+      return true;
+    } catch {
+      failures++;
+      return false;
+    }
+  }
+  function close() {
+    if (!closing) { closed = true; closing = flush(); }
+    return closing;
+  }
+  return { load, schedule, flush, close,
+    stats: () => ({ persistenceFailures: failures, writes, pending: pending !== null || running !== null, maxBytes }) };
+}

package/runtime/daemon/server.mjs

@@ -1,16 +1,24 @@
 import http from 'node:http';
 import net from 'node:net';
+import path from 'node:path';
 import { readFile, chmod, rm } from 'node:fs/promises';
 import { createPipeline } from '../pipeline.mjs';
 import { createPolicy, materializeBundle, buildRelationProposals } from '../core/index.mjs';
-import { createDecisionService } from '../jev/index.mjs';
+import { createDecisionService } from '../decisions/index.mjs';
+import { createJevProvider } from '../jev/provider.mjs';
+import { createAnalysisBroker } from '../decisions/broker.mjs';
 import { projectPaths, canonicalProjectRoot, MAX_IPC_BYTES, MAX_STATE_BYTES, PROTOCOL, runtimeError } from './paths.mjs';
 import { acquireLock } from './lock.mjs';
 import { createAuth } from './auth.mjs';
 import { createPersistence } from './persistence.mjs';
+import { createModelPersistence } from './model-persistence.mjs';
+import { createModelAPI } from './model-api.mjs';
+import { createExtensionAPI } from './extension-api.mjs';
+import { createExtensionRegistry } from '../extensions/index.mjs';
 import { exportSnapshot } from './export.mjs';
 import { createDiagnostics } from './diagnostics.mjs';
 import { createDashboardInfoProvider } from './dashboard-info.mjs';
+import { createLineageReader } from './lineage.mjs';
 
 const WEB = new URL('../web/', import.meta.url);
 const assets = new Map([
@@ -21,8 +29,17 @@ const assets = new Map([
   ['/sketch.js', ['sketch.js', 'text/javascript; charset=utf-8']],
   ['/style.css', ['style.css', 'text/css; charset=utf-8']],
 ]);
+for (const filename of ['platform.js', 'model-client.js', 'scene.js', 'extension-frame.js']) {
+  assets.set(`/${filename}`, [filename, 'text/javascript; charset=utf-8']);
+}
+for (const filename of [
+  'visualizers/index.mjs', 'visualizers/structure.mjs', 'visualizers/code.mjs',
+  'visualizers/blocks.mjs', 'visualizers/c4.mjs', 'visualizers/changes.mjs', 'visualizers/timeline.mjs',
+  'model/changes.mjs', 'extensions/scene.mjs', 'extensions/contracts.mjs',
+  'extensions/sdk.mjs', 'extensions/profiles.mjs',
+]) assets.set(`/${filename}`, [`../${filename}`, 'text/javascript; charset=utf-8']);
 const HOSTS = new Set(['claude', 'codex', 'kiro']);
-const CSP = "default-src 'none'; script-src 'self'; style-src 'self'; connect-src 'self'; img-src 'self' data:; base-uri 'none'; form-action 'none'; frame-ancestors 'none'; object-src 'none'";
+const CSP = "default-src 'none'; script-src 'self'; style-src 'self'; connect-src 'self'; frame-src 'self'; img-src 'self' data:; base-uri 'none'; form-action 'none'; frame-ancestors 'none'; object-src 'none'";
 
 function headers(res) {
   res.setHeader('Cache-Control', 'no-store');
@@ -53,25 +70,27 @@ async function bodyJSON(req) {
 }
 
 export async function startServer({ projectRoot, dataDir, policy: policyOptions,
-  decisionService, apiKey: configuredKey, mode = 'live', port = 0, dashboardInfoDependencies } = {}) {
+  decisionService, decisionProvider, apiKey: configuredKey, mode = 'live', port = 0, dashboardInfoDependencies } = {}) {
   if (!Number.isInteger(port) || port < 0 || port > 65535) throw runtimeError('invalid_port');
   if (!['live', 'demo'].includes(mode)) throw runtimeError('invalid_mode');
   const paths = await projectPaths(projectRoot, dataDir, { create: true });
+  const readLineage = createLineageReader({ projectRoot: paths.projectRoot, projectId: paths.projectId });
   const dashboardInfo = createDashboardInfoProvider({
     projectRoot: paths.projectRoot, dataDir: paths.dataDir, mode,
   }, dashboardInfoDependencies);
   const lock = await acquireLock(paths);
   let finished;
   const whenClosed = new Promise(resolve => { finished = resolve; });
-  let web, ipc, pipeline, auth, diagnostics, interval, ping, closing, reconciling = false;
+  let web, ipc, pipeline, auth, diagnostics, modelAPI, extensionAPI, modelFlush, interval, ping, closing, reconciling = false;
   let drops = 0, intake = 0;
   const receivedHooks = { claude: 0, codex: 0, kiro: 0 };
   const clients = new Set(), connections = new Set(), ipcConnections = new Set();
   const persistence = createPersistence(paths.state);
+  const modelPersistence = createModelPersistence(path.join(paths.directory, 'model-state.json'), { projectId: paths.projectId });
   const policy = createPolicy(policyOptions ?? {});
-  const apiKey = !decisionService && policy.transmitSource && mode === 'live'
+  const apiKey = !decisionService && !decisionProvider && policy.transmitSource && mode === 'live'
     ? configuredKey ?? process.env.TYPESAFE_API_KEY : undefined;
-  const missingKey = !decisionService && policy.transmitSource && mode === 'live' && !apiKey;
+  const missingKey = !decisionService && !decisionProvider && policy.transmitSource && mode === 'live' && !apiKey;
   function snapshot(persistent = false) {
     const state = pipeline.getState({ persistent });
     return { ...state, status: { ...state.status,
@@ -88,13 +107,25 @@ export async function startServer({ projectRoot, dataDir, policy: policyOptions,
   function notify() {
     if (!pipeline || closing) return;
     persistence.schedule(snapshot(true));
+    if (!modelFlush) {
+      // Coalesce model serialization outside the hook intake path.
+      modelFlush = setTimeout(() => {
+        modelFlush = null;
+        if (!closing) modelPersistence.schedule(pipeline.getModelState({ persistent: true }));
+      }, 100);
+      modelFlush.unref?.();
+    }
+    modelAPI?.notify();
     const state = snapshot();
     for (const res of clients) stream(res, state);
   }
   async function reconcile() {
     if (reconciling || closing || !pipeline) return;
     reconciling = true;
-    try { await pipeline.reconcile(); }
+    try {
+      await pipeline.observeLineage(await readLineage());
+      await pipeline.reconcile();
+    }
     catch { drops++; }
     finally { reconciling = false; }
   }
@@ -102,6 +133,8 @@ export async function startServer({ projectRoot, dataDir, policy: policyOptions,
     if (closing) return closing;
     closing = (async () => {
       clearInterval(interval); clearInterval(ping);
+      clearTimeout(modelFlush);
+      modelAPI?.close();
       auth?.clear();
       for (const res of clients) res.end();
       for (const socket of [...connections, ...ipcConnections]) socket.destroy();
@@ -109,8 +142,11 @@ export async function startServer({ projectRoot, dataDir, policy: policyOptions,
         new Promise(resolve => server.close(() => resolve()))));
       try {
         await pipeline?.close();
-        if (pipeline) persistence.schedule(snapshot(true));
-        await persistence.close();
+        if (pipeline) {
+          persistence.schedule(snapshot(true));
+          modelPersistence.schedule(pipeline.getModelState({ persistent: true }));
+        }
+        await Promise.all([persistence.close(), modelPersistence.close()]);
       } finally {
         try { await diagnostics?.close(); }
         finally {
@@ -127,19 +163,40 @@ export async function startServer({ projectRoot, dataDir, policy: policyOptions,
     // Reading the key is conditional on explicit source-transmission permission.
     // Test/demo services are injected; this module never logs request bodies.
     const service = decisionService ?? createDecisionService({
-      apiKey,
+      provider: decisionProvider ?? createJevProvider({ apiKey }),
       materializeBundle, buildRelationProposals,
       limits: { eventDeadlineMs: 5000 },
     });
     pipeline = createPipeline({ projectRoot: paths.projectRoot, policy, decisionService: service,
       classificationDeadlineMs: 5000,
-      mode, restoredState: await persistence.load(), onChange: notify, onDiagnostic: diagnostics.record });
+      mode, restoredState: await persistence.load(), restoredModel: await modelPersistence.load(),
+      onChange: notify, onDiagnostic: diagnostics.record });
+    modelAPI = createModelAPI({ projectId: paths.projectId, getSnapshot: pipeline.getModelState,
+      createCheckpoint: options => {
+        const marker = pipeline.createCheckpoint(options);
+        notify();
+        return marker;
+      } });
+    const registry = await createExtensionRegistry({ dataDir: paths.dataDir, projectId: paths.projectId });
+    const analyze = typeof service.evaluate === 'function'
+      ? createAnalysisBroker({ service, model: pipeline.model, policy, projectId: paths.projectId, registry })
+      : async () => ({ status: 'unavailable' });
+    extensionAPI = createExtensionAPI({ registry, projectId: paths.projectId, getSnapshot: pipeline.getModelState,
+      runAnalysis: async input => { const result = await analyze(input); notify(); return result; } });
     web = http.createServer({ maxHeaderSize: 8192, requestTimeout: 2000, headersTimeout: 2000 }, (req, res) => {
       headers(res);
       void (async () => {
+        if (req.url?.startsWith('/api/model/v1/')) {
+          if (!auth?.validTransport(req)) return json(res, 403, { error: 'forbidden_origin' });
+          await modelAPI.handle(req, res, {
+            viewerAuthorized: auth.validRequest(req, { mutation: req.method !== 'GET' }) && auth.authorized(req),
+          });
+          return;
+        }
         if (!auth || !auth.validRequest(req, { mutation: req.method !== 'GET' })) {
           return json(res, 403, { error: 'forbidden_origin' });
         }
+        if (await extensionAPI.handle(req, res, { viewerAuthorized: auth.authorized(req) })) return;
         if (typeof req.url !== 'string' || req.url.length > 1024 || req.url.includes('?')) {
           return json(res, 400, { error: 'invalid_route' });
         }
@@ -218,10 +275,11 @@ export async function startServer({ projectRoot, dataDir, policy: policyOptions,
     function describe() {
       return { ok: true, protocol: PROTOCOL, instanceId: lock.owner.instanceId,
         projectId: paths.projectId, pid: process.pid, port: actualPort, mode,
-        policy: { transmitSource: policy.transmitSource, displayEvidence: policy.displayEvidence,
+        policy: { readSource: policy.readSource, transmitSource: policy.transmitSource, displayEvidence: policy.displayEvidence,
           persistEvidence: policy.persistEvidence, version: policy.version },
         status: snapshot().status, ...persistence.stats(), captureDropped: drops,
         observations: { hooks: { ...receivedHooks }, shapes: pipeline.getState().graph.nodes.length },
+        model: pipeline.model.stats(), modelPersistence: modelPersistence.stats(),
         logPath: diagnostics.stats().logPath, diagnostics: diagnostics.stats() };
     }
     // Only the exclusive owner may remove a stale socket from a prior process.

package/runtime/daemon/settings.mjs

@@ -7,8 +7,11 @@ const LIMIT = 16 * 1024;
 const record = value => value !== null && typeof value === 'object' && !Array.isArray(value);
 const validKey = value => typeof value === 'string' && value.length > 0 &&
   value.length <= 4096 && !/[\s\u0000-\u001f\u007f]/u.test(value);
-const validPolicy = value => record(value) && Object.keys(value).length === 3 &&
-  ['allowSource', 'persistEvidence', 'displayEvidence'].every(key => typeof value[key] === 'boolean');
+const validPolicy = value => record(value) &&
+  Object.keys(value).every(key => ['allowSource', 'localSource', 'persistEvidence', 'displayEvidence'].includes(key)) &&
+  ['allowSource', 'persistEvidence', 'displayEvidence'].every(key => typeof value[key] === 'boolean') &&
+  (value.localSource === undefined || typeof value.localSource === 'boolean') &&
+  !(value.localSource && value.allowSource);
 const validHosts = value => Array.isArray(value) && value.length <= 2 &&
   new Set(value).size === value.length && value.every(host => ['claude', 'codex'].includes(host));
 const validInstallation = value => record(value) &&
@@ -89,14 +92,19 @@ export function savedPolicy(policy) {
     allowSource: policy.transmitSource,
     persistEvidence: policy.persistEvidence,
     displayEvidence: policy.displayEvidence,
+    ...(policy.readSource && !policy.transmitSource ? { localSource: true } : {}),
   } : {};
 }
 
 // An omitted field means reuse consent, while false is an explicit opt-out.
 export function resolvePolicy(options, { current, saved } = {}) {
   const fallback = current ? savedPolicy(current) : saved ?? {};
+  const allowSource = options.allowSource ?? fallback.allowSource ?? false;
+  const localSource = !allowSource && (options.localSource ??
+    (options.allowSource === false ? false : fallback.localSource) ?? false);
   return {
-    allowSource: options.allowSource ?? fallback.allowSource ?? false,
+    allowSource,
+    ...(localSource ? { localSource: true } : {}),
     persistEvidence: options.persistEvidence ?? fallback.persistEvidence ?? false,
     displayEvidence: options.displayEvidence ?? fallback.displayEvidence ?? true,
   };