amalgm 0.1.149 → 0.1.150

package/runtime/scripts/amalgm-mcp/state/docs.js

@@ -0,0 +1,391 @@
+'use strict';
+
+/**
+ * Live documents: Yjs-backed text files on the local-live rails (wave 2.5).
+ *
+ * A document is a text file (markdown, code — anything in the fs layer's
+ * text-extension set) whose contents are held in a Y.Doc while open. Edits
+ * travel as Yjs updates in the event log's `patch` slot; the file on disk
+ * remains the artifact (debounced write-back), so everything that reads the
+ * file — agents, exports, git — keeps working unchanged.
+ *
+ * Wire contract:
+ *   POST /state/docs/open    {path}                → {resource, path, state, seq}
+ *   POST /state/docs/update  {path, updates:[b64]} → {seq, applied}
+ *   events: {resource: "doc:<base64url(path)>", op: "update", id: path,
+ *            patch: {yjs: "<base64 update>"}}
+ *   snapshot read: {path, state, updatedAt} for OPEN docs only — the open
+ *   endpoint is the bootstrap; snapshots never load documents.
+ *
+ * Failure isolation is a hard requirement: this module is only ever reached
+ * through its own routes and the `doc:` registry read (which catches all
+ * errors). If yjs is unavailable, routes answer 501 and nothing else in the
+ * realtime layer is affected.
+ *
+ * External writers (agents editing the file directly) are folded in by a
+ * per-document directory watcher: on file change, disk text is diffed into
+ * the Y.Doc as a splice, which emits a normal update event. The cycle is
+ * loop-safe because reconcile only acts when disk content changed past the
+ * last text we read or wrote (entry.lastDiskText) — our own write-backs and
+ * stale watcher events can never splice, so they can never re-emit.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { openLocalDb } = require('./db');
+const { appendStateEvent, currentSeq } = require('./events');
+
+const DOC_RESOURCE_PREFIX = 'doc:';
+const TEXT_KEY = 'content';
+const MAX_DOC_BYTES = Number.parseInt(process.env.AMALGM_DOC_MAX_BYTES || '', 10) || 2 * 1024 * 1024;
+const MAX_OPEN_DOCS = 64;
+const DISK_WRITE_DEBOUNCE_MS = 300;
+const WATCH_RECONCILE_DEBOUNCE_MS = 200;
+const MAX_UPDATES_PER_CALL = 200;
+
+// path -> { doc, resource, watcher, reconcileTimer, diskWriteTimer,
+//           lastAccess, suppressDiskWriteOnce }
+const openDocs = new Map();
+
+function fsPrivate() {
+  return require('../fs/rest')._private;
+}
+
+function loadYjs() {
+  try {
+    return require('yjs');
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    throw Object.assign(
+      new Error(`live documents require the yjs package: ${message}`),
+      { statusCode: 501 },
+    );
+  }
+}
+
+function invalid(message, statusCode = 400) {
+  return Object.assign(new Error(message), { statusCode });
+}
+
+function ensureSchema(database) {
+  database.exec(`
+    CREATE TABLE IF NOT EXISTS doc_states (
+      path TEXT PRIMARY KEY,
+      state BLOB NOT NULL,
+      updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
+    );
+  `);
+}
+
+function docResourceName(absolutePath) {
+  return `${DOC_RESOURCE_PREFIX}${fsPrivate().base64UrlEncode(absolutePath)}`;
+}
+
+function pathFromDocResource(resource) {
+  if (typeof resource !== 'string' || !resource.startsWith(DOC_RESOURCE_PREFIX)) return null;
+  try {
+    return fsPrivate().base64UrlDecode(resource.slice(DOC_RESOURCE_PREFIX.length));
+  } catch {
+    return null;
+  }
+}
+
+function resolveDocPath(pathInput) {
+  const resolved = fsPrivate().resolveSafePathSync(pathInput);
+  const { isText } = fsPrivate().classifyFile(resolved);
+  if (!isText) {
+    throw invalid(`live documents support text files only (got ${path.extname(resolved) || 'no extension'})`);
+  }
+  const stats = fs.statSync(resolved);
+  if (!stats.isFile()) throw invalid('path is not a file');
+  if (stats.size > MAX_DOC_BYTES) {
+    throw invalid(`file exceeds the ${MAX_DOC_BYTES}-byte live document limit`, 413);
+  }
+  return resolved;
+}
+
+/**
+ * Splice `nextText` into ytext as a minimal edit (common prefix/suffix kept),
+ * backing off surrogate-pair boundaries so we never split a character.
+ */
+function spliceTextInto(Y, doc, ytext, nextText, origin) {
+  const current = ytext.toString();
+  if (current === nextText) return false;
+
+  let start = 0;
+  const maxStart = Math.min(current.length, nextText.length);
+  while (start < maxStart && current[start] === nextText[start]) start += 1;
+  if (start > 0 && isLowSurrogate(current.charCodeAt(start))) start -= 1;
+
+  let endCurrent = current.length;
+  let endNext = nextText.length;
+  while (endCurrent > start && endNext > start && current[endCurrent - 1] === nextText[endNext - 1]) {
+    endCurrent -= 1;
+    endNext -= 1;
+  }
+  if (endCurrent < current.length && isLowSurrogate(current.charCodeAt(endCurrent))) {
+    endCurrent += 1;
+    endNext += 1;
+  }
+
+  doc.transact(() => {
+    if (endCurrent > start) ytext.delete(start, endCurrent - start);
+    if (endNext > start) ytext.insert(start, nextText.slice(start, endNext));
+  }, origin);
+  return true;
+}
+
+function isLowSurrogate(code) {
+  return code >= 0xdc00 && code <= 0xdfff;
+}
+
+function persistDocState(Y, entry) {
+  const database = openLocalDb();
+  ensureSchema(database);
+  database.prepare(`
+    INSERT INTO doc_states (path, state, updated_at)
+    VALUES (?, ?, strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
+    ON CONFLICT(path) DO UPDATE SET
+      state = excluded.state,
+      updated_at = excluded.updated_at
+  `).run(entry.path, Buffer.from(Y.encodeStateAsUpdate(entry.doc)));
+}
+
+function scheduleDiskWrite(Y, entry) {
+  if (entry.diskWriteTimer) return;
+  entry.diskWriteTimer = setTimeout(() => {
+    entry.diskWriteTimer = null;
+    flushDocToDisk(Y, entry);
+  }, DISK_WRITE_DEBOUNCE_MS);
+  if (typeof entry.diskWriteTimer.unref === 'function') entry.diskWriteTimer.unref();
+}
+
+function flushDocToDisk(Y, entry) {
+  try {
+    const text = entry.doc.getText(TEXT_KEY).toString();
+    const onDisk = fs.readFileSync(entry.path, 'utf8');
+    if (onDisk !== text) fs.writeFileSync(entry.path, text, 'utf8');
+    entry.lastDiskText = text;
+    persistDocState(Y, entry);
+  } catch (error) {
+    console.warn(`[LiveDoc] Failed to flush "${entry.path}" to disk:`, error?.message || error);
+  }
+}
+
+function reconcileFromDisk(Y, entry) {
+  try {
+    const onDisk = fs.readFileSync(entry.path, 'utf8');
+    // Only fold in disk content that actually CHANGED since we last read or
+    // wrote it. Without this, a watcher event firing while the doc is ahead
+    // of disk (write-back still debounced) would "reconcile" the stale disk
+    // text back over live edits.
+    if (onDisk === entry.lastDiskText) return;
+    entry.lastDiskText = onDisk;
+    const ytext = entry.doc.getText(TEXT_KEY);
+    // origin 'disk' keeps the update handler from writing the same bytes back.
+    spliceTextInto(Y, entry.doc, ytext, onDisk, 'disk');
+  } catch (error) {
+    // File deleted or unreadable — the doc simply stops following the disk.
+    console.warn(`[LiveDoc] Reconcile from disk failed for "${entry.path}":`, error?.message || error);
+  }
+}
+
+function watchDocFile(Y, entry) {
+  const dir = path.dirname(entry.path);
+  const base = path.basename(entry.path);
+  try {
+    entry.watcher = fs.watch(dir, { persistent: false }, (eventType, filename) => {
+      // Some platforms omit filename; reconcile is content-compare based and
+      // idempotent, so over-firing is safe.
+      if (filename && filename !== base) return;
+      if (entry.reconcileTimer) return;
+      entry.reconcileTimer = setTimeout(() => {
+        entry.reconcileTimer = null;
+        reconcileFromDisk(Y, entry);
+      }, WATCH_RECONCILE_DEBOUNCE_MS);
+      if (typeof entry.reconcileTimer.unref === 'function') entry.reconcileTimer.unref();
+    });
+    entry.watcher.on('error', () => {
+      try { entry.watcher.close(); } catch {}
+      entry.watcher = null;
+    });
+    if (typeof entry.watcher.unref === 'function') entry.watcher.unref();
+  } catch {
+    // Watching is best-effort; external edits still reconcile on next open.
+  }
+}
+
+function closeDocEntry(Y, entry) {
+  if (entry.diskWriteTimer) {
+    clearTimeout(entry.diskWriteTimer);
+    entry.diskWriteTimer = null;
+  }
+  if (entry.reconcileTimer) {
+    clearTimeout(entry.reconcileTimer);
+    entry.reconcileTimer = null;
+  }
+  flushDocToDisk(Y, entry);
+  if (entry.watcher) {
+    try { entry.watcher.close(); } catch {}
+    entry.watcher = null;
+  }
+  entry.doc.destroy();
+  openDocs.delete(entry.path);
+}
+
+function evictIfNeeded(Y) {
+  while (openDocs.size > MAX_OPEN_DOCS) {
+    let oldest = null;
+    for (const entry of openDocs.values()) {
+      if (!oldest || entry.lastAccess < oldest.lastAccess) oldest = entry;
+    }
+    if (!oldest) return;
+    closeDocEntry(Y, oldest);
+  }
+}
+
+function loadDocEntry(pathInput) {
+  const Y = loadYjs();
+  const resolved = resolveDocPath(pathInput);
+  const existing = openDocs.get(resolved);
+  if (existing) {
+    existing.lastAccess = Date.now();
+    return existing;
+  }
+
+  const database = openLocalDb();
+  ensureSchema(database);
+
+  const doc = new Y.Doc();
+  const entry = {
+    path: resolved,
+    resource: docResourceName(resolved),
+    doc,
+    watcher: null,
+    reconcileTimer: null,
+    diskWriteTimer: null,
+    lastAccess: Date.now(),
+    // The disk text as of our last read/write; reconcile skips anything that
+    // hasn't changed past it (see reconcileFromDisk).
+    lastDiskText: null,
+  };
+
+  const persisted = database.prepare('SELECT state FROM doc_states WHERE path = ?').get(resolved);
+  if (persisted?.state) {
+    try {
+      Y.applyUpdate(doc, new Uint8Array(persisted.state), 'persisted');
+    } catch (error) {
+      console.warn(`[LiveDoc] Discarding corrupt persisted state for "${resolved}":`, error?.message || error);
+    }
+  }
+
+  // Every applied change — client update, disk reconcile, or seed — flows
+  // through this one pipeline: event out, then (except disk-origin) disk out.
+  doc.on('update', (update, origin) => {
+    appendStateEvent({
+      resource: entry.resource,
+      op: 'update',
+      id: entry.path,
+      patch: { yjs: Buffer.from(update).toString('base64') },
+      source: origin === 'disk' ? 'doc:disk' : 'doc:update',
+    });
+    if (origin !== 'disk') scheduleDiskWrite(Y, entry);
+    else persistDocState(Y, entry);
+  });
+
+  // The disk is the source of truth at open: seeds new docs, and folds in
+  // anything written while the doc was closed (agents, git, other tools).
+  reconcileFromDisk(Y, entry);
+
+  openDocs.set(resolved, entry);
+  watchDocFile(Y, entry);
+  evictIfNeeded(Y);
+  return entry;
+}
+
+function openDoc(pathInput) {
+  const Y = loadYjs();
+  const entry = loadDocEntry(pathInput);
+  return {
+    resource: entry.resource,
+    path: entry.path,
+    state: Buffer.from(Y.encodeStateAsUpdate(entry.doc)).toString('base64'),
+    seq: currentSeq(),
+  };
+}
+
+function applyDocUpdates(pathInput, updatesInput) {
+  const Y = loadYjs();
+  const inputs = Array.isArray(updatesInput) ? updatesInput : [updatesInput];
+  if (inputs.length === 0) return { applied: 0, seq: currentSeq() };
+  if (inputs.length > MAX_UPDATES_PER_CALL) {
+    throw invalid(`at most ${MAX_UPDATES_PER_CALL} updates per call (got ${inputs.length})`);
+  }
+  const updates = inputs.map((value, index) => {
+    if (typeof value !== 'string' || !value) {
+      throw invalid(`updates[${index}] must be a base64-encoded Yjs update`);
+    }
+    return Buffer.from(value, 'base64');
+  });
+
+  const entry = loadDocEntry(pathInput);
+  entry.lastAccess = Date.now();
+  for (const update of updates) {
+    try {
+      Y.applyUpdate(entry.doc, new Uint8Array(update), 'client');
+    } catch (error) {
+      throw invalid(`invalid Yjs update: ${error?.message || error}`);
+    }
+  }
+  return { applied: updates.length, seq: currentSeq() };
+}
+
+/**
+ * Registry read for `doc:` resources. Serves only documents that are already
+ * open — the open endpoint is the bootstrap — and never throws: a failure
+ * here must not take down snapshots of unrelated resources.
+ */
+function readDocResource(resource) {
+  try {
+    const target = pathFromDocResource(resource);
+    if (!target) return undefined;
+    const entry = openDocs.get(target);
+    if (!entry) return undefined;
+    const Y = loadYjs();
+    return {
+      path: entry.path,
+      state: Buffer.from(Y.encodeStateAsUpdate(entry.doc)).toString('base64'),
+      updatedAt: new Date(entry.lastAccess).toISOString(),
+    };
+  } catch (error) {
+    console.warn(`[LiveDoc] Snapshot read failed for "${resource}":`, error?.message || error);
+    return undefined;
+  }
+}
+
+function getDocText(pathInput) {
+  const entry = loadDocEntry(pathInput);
+  return entry.doc.getText(TEXT_KEY).toString();
+}
+
+/** Test/shutdown helper: flush and close every open document. */
+function closeAllDocs() {
+  let Y;
+  try {
+    Y = loadYjs();
+  } catch {
+    return;
+  }
+  for (const entry of Array.from(openDocs.values())) closeDocEntry(Y, entry);
+}
+
+module.exports = {
+  DOC_RESOURCE_PREFIX,
+  applyDocUpdates,
+  closeAllDocs,
+  docResourceName,
+  getDocText,
+  openDoc,
+  pathFromDocResource,
+  readDocResource,
+};

package/runtime/scripts/amalgm-mcp/state/registry.js

@@ -0,0 +1,105 @@
+'use strict';
+
+/**
+ * The resource registry: the sign-up sheet for the local-live realtime system.
+ *
+ * A resource is a named, subscribable list of rows. Registering one takes two
+ * things — a name and a read function — and buys everything the realtime
+ * layer does: inclusion in snapshots, live event delivery, reconnect replay,
+ * and reset semantics. The registry never sees the underlying storage; `read`
+ * is the only bridge.
+ *
+ *   registerResource('skills', {
+ *     read: () => require('../skills/store').listSkills(),
+ *     snapshotDefault: true,
+ *   });
+ *
+ * Spec fields:
+ *   read(ctx)        required. Returns the current rows (or any snapshot
+ *                    value). `ctx` is a per-snapshot-attempt cache object so
+ *                    resources sharing an expensive read (e.g. the automation
+ *                    graph) compute it once per attempt. Must not be reused
+ *                    across attempts — a retry exists because a write landed
+ *                    mid-read.
+ *   snapshotDefault  optional. Include this resource when a snapshot is
+ *                    requested with no explicit resource list. Defaults false.
+ *
+ * Prefix registrations serve families of dynamic names (e.g. `files:<path>`):
+ * the spec's read receives the full resource name as its second argument.
+ */
+
+const resources = new Map();
+const prefixes = new Map();
+
+function assertSpec(name, spec) {
+  if (!name || typeof name !== 'string') {
+    throw new Error('registerResource requires a non-empty string name');
+  }
+  if (!spec || typeof spec.read !== 'function') {
+    throw new Error(`resource "${name}" requires a read() function`);
+  }
+}
+
+function registerResource(name, spec) {
+  assertSpec(name, spec);
+  if (resources.has(name)) {
+    throw new Error(`resource "${name}" is already registered`);
+  }
+  resources.set(name, { snapshotDefault: false, ...spec });
+}
+
+function registerResourcePrefix(prefix, spec) {
+  assertSpec(prefix, spec);
+  if (prefixes.has(prefix)) {
+    throw new Error(`resource prefix "${prefix}" is already registered`);
+  }
+  prefixes.set(prefix, { snapshotDefault: false, ...spec });
+}
+
+function unregisterResource(name) {
+  return resources.delete(name);
+}
+
+function getResourceSpec(name) {
+  const direct = resources.get(name);
+  if (direct) return direct;
+  for (const [prefix, spec] of prefixes) {
+    if (name.startsWith(prefix)) return spec;
+  }
+  return undefined;
+}
+
+function hasResource(name) {
+  return getResourceSpec(String(name || '')) !== undefined;
+}
+
+function readRegisteredResource(name, ctx) {
+  const spec = getResourceSpec(name);
+  if (!spec) return undefined;
+  return spec.read(ctx, name);
+}
+
+// Registration order is preserved: the default snapshot lists resources in
+// the order they signed up, which keeps wire output stable across releases.
+function listDefaultResources() {
+  const names = [];
+  for (const [name, spec] of resources) {
+    if (spec.snapshotDefault) names.push(name);
+  }
+  return names;
+}
+
+function listRegisteredResources() {
+  return Array.from(resources.keys());
+}
+
+module.exports = {
+  getResourceSpec,
+  hasResource,
+  listDefaultResources,
+  listRegisteredResources,
+  readRegisteredResource,
+  registerResource,
+  registerResourcePrefix,
+  unregisterResource,
+};

package/runtime/scripts/amalgm-mcp/state/resources.js

@@ -0,0 +1,162 @@
+'use strict';
+
+/**
+ * Built-in resource registrations. These are the first names on the registry's
+ * sign-up sheet — each one was previously a hand-written case in a switch in
+ * snapshot.js. Requires stay inside read() so opening the registry never pulls
+ * in a store the caller doesn't ask for.
+ *
+ * Registration order defines the default snapshot's resource order — append
+ * new defaults at the end.
+ */
+
+const {
+  registerResource,
+  registerResourcePrefix,
+} = require('./registry');
+
+// The automation graph is one read shared by three resources; ctx keeps it
+// single-pass per snapshot attempt.
+function automationGraph(ctx) {
+  ctx.automationGraph ||= require('../automations/store').listPublicAutomationGraph();
+  return ctx.automationGraph;
+}
+
+function toolboxCatalog(ctx) {
+  ctx.toolbox ||= require('../toolbox/service').defaultToolboxService.readCatalog();
+  return ctx.toolbox;
+}
+
+function registerBuiltinResources() {
+  registerResource('automations', {
+    snapshotDefault: true,
+    read: (ctx) => automationGraph(ctx).automations,
+  });
+  registerResource('triggers', {
+    snapshotDefault: true,
+    read: (ctx) => automationGraph(ctx).triggers,
+  });
+  registerResource('workflows', {
+    snapshotDefault: true,
+    read: (ctx) => automationGraph(ctx).workflows,
+  });
+  registerResource('automation_runs', {
+    snapshotDefault: true,
+    read: () => require('../automations/store').listAutomationRuns({ limit: 300 }),
+  });
+  registerResource('workflow_cell_runs', {
+    snapshotDefault: true,
+    read: () => require('../automations/store').listWorkflowCellRuns({ limit: 300 }),
+  });
+  registerResource('agents', {
+    snapshotDefault: true,
+    read: () => {
+      const { agentWithConfig } = require('../agent-config/store');
+      return require('../agents/store').getAllAgentsWithBuiltins()
+        .map((agent) => agentWithConfig(agent))
+        .filter(Boolean);
+    },
+  });
+  registerResource('agent_configs', {
+    snapshotDefault: true,
+    read: () => require('../agent-config/store').listAgentConfigs(),
+  });
+  registerResource('skills', {
+    snapshotDefault: true,
+    read: () => require('../skills/store').listSkills(),
+  });
+  registerResource('apps', {
+    snapshotDefault: true,
+    read: () => require('../apps/store').loadApps().apps,
+  });
+  registerResource('ports', {
+    snapshotDefault: true,
+    read: () => require('./ports').getCurrentPorts(),
+  });
+  registerResource('toolbox', {
+    snapshotDefault: true,
+    read: (ctx) => toolboxCatalog(ctx),
+  });
+  registerResource('tools', {
+    snapshotDefault: true,
+    read: (ctx) => toolboxCatalog(ctx).tools,
+  });
+  registerResource('tool_actions', {
+    snapshotDefault: true,
+    read: (ctx) => toolboxCatalog(ctx).toolActions,
+  });
+  registerResource('hooks', {
+    snapshotDefault: true,
+    read: () => require('../agents/hooks').collectNativeHooks(),
+  });
+  registerResource('projects', {
+    snapshotDefault: true,
+    read: () => require('../workspace/store').listWorkspaces(),
+  });
+  registerResource('workspaces', {
+    snapshotDefault: true,
+    read: () => require('../workspace/store').listWorkspaces(),
+  });
+  registerResource('project_context', {
+    snapshotDefault: true,
+    read: () => require('../project-context/store').listProjectContexts(),
+  });
+  registerResource('user_preferences', {
+    snapshotDefault: true,
+    read: () => require('../lib/prefs').readUserPreferences(),
+  });
+  registerResource('chat_payloads', {
+    snapshotDefault: true,
+    read: () => require('../lib/chat-payloads').listChatPayloads(),
+  });
+  registerResource('mcp_connections', {
+    snapshotDefault: true,
+    read: () => require('../mcp-connections/rest').buildConnectionsSnapshot(),
+  });
+  registerResource('browser_profiles', {
+    snapshotDefault: true,
+    read: () => require('../browser/store').listBrowserProfiles(),
+  });
+  registerResource('browser_auth_bundles', {
+    snapshotDefault: true,
+    read: () => require('../browser/store').listBrowserAuthBundles(),
+  });
+  registerResource('browser_login_sessions', {
+    snapshotDefault: true,
+    read: () => require('../browser/store').listBrowserLoginSessions(),
+  });
+
+  // Subscribable but not part of the default snapshot set.
+  registerResource('tasks', {
+    read: () => require('../tasks/store').loadTasks().tasks,
+  });
+  registerResource('task_runs', {
+    read: () => require('../tasks/store').listTaskRuns({ limit: 100 }),
+  });
+  registerResource('event_runs', {
+    read: () => require('../events/store').listEventRuns({ limit: 100 }),
+  });
+  registerResource('event_triggers', {
+    read: () => require('../events/store').loadEventTriggers().triggers,
+  });
+  registerResource('uploads', {
+    read: () => require('../fs/rest')._private.buildUploadsResource(),
+  });
+  registerResource('browser_surfaces', {
+    // Surface-open requests are commands, not state: delivered via the
+    // event stream only, so snapshots always start empty.
+    read: () => [],
+  });
+
+  registerResourcePrefix('files:', {
+    read: (ctx, name) => require('../fs/rest')._private.buildFilesResource(name),
+  });
+
+  // Live documents (wave 2.5). readDocResource never throws and serves only
+  // already-open docs, so a broken document cannot take down a snapshot.
+  registerResourcePrefix('doc:', {
+    read: (ctx, name) => require('./docs').readDocResource(name),
+  });
+}
+
+module.exports = { registerBuiltinResources };