create-walle 0.10.2 → 0.10.3

package/template/coding-task-manager/lib/api/memory-sync-routes.js

@@ -0,0 +1,335 @@
+'use strict';
+
+// WP-3 (docs/shared-memory-every-machine-2026-10-31.html §5 D-9/D-12, §6.6): the box-side MEMBER
+// front routes. Every CTM serves them — only a paired member is ever asked — and each is a thin,
+// verbatim proxy to THIS host's loopback Wall-E (D-12: the hub reaches a box only through its CTM
+// URL; Wall-E's own v2 routes accept only the local CTM token).
+//
+//   GET  /api/sync/brain/v2/vector         -> GET  <local Wall-E>/api/walle/sync/v2/vector
+//   POST /api/sync/brain/v2/changes        -> POST /api/walle/sync/v2/changes
+//   POST /api/sync/brain/v2/apply          -> POST /api/walle/sync/v2/apply
+//   GET  /api/sync/brain/v2/digest?table=  -> GET  /api/walle/sync/v2/digest?table=<same>
+//
+// WP-4 (§5 D-6, §6.7) add the JOIN family. The hub ships its snapshot in 4 MiB chunks to the member
+// CTM, which assembles them under <dataDir>/sync-inbox — the member's own disk is the only place the
+// bytes can live, so these three are NOT proxies: `join/chunk` and `join/transfer` do real,
+// bounded local file I/O; `join/import` and `join/status` forward verbatim to the box's Wall-E
+// exactly like the four above.
+//
+//   POST /api/sync/brain/v2/join/chunk    { joinId, index, total, chunkBytes, fileBytes, fileSha256,
+//                                           dataBase64 } -> 200 { ok, received, complete:false }
+//                                                         | 200 { ok, complete:true, path }
+//   GET  /api/sync/brain/v2/join/transfer?joinId= -> 200 { ok, joinId, total, received, complete, path }
+//   POST /api/sync/brain/v2/join/import   -> POST /api/walle/sync/v2/join/import
+//   GET  /api/sync/brain/v2/join/status   -> GET  /api/walle/sync/v2/join/status
+//
+// The Wall-E status code and JSON body are returned VERBATIM (a 409 clock_seed_pending, a 503
+// brain_sync_disabled and a 413 too_many_entries all pass through unchanged). A Wall-E that cannot
+// be reached becomes 503 { ok:false, error:'walle_unreachable' }; a request body over the 8 MiB cap
+// becomes 413 before any forward.
+//
+// Like lib/api/machines.js this is NEW surface: the handler bodies live here, so there is no legacy
+// handleApi branch to delete and api-router-parity's no-dual-dispatch rule is satisfied by
+// construction. Auth is NOT evaluated here — lib/auth-rules.js carries the four rows (scope `sync`,
+// peerControl: true, so only the pool/sync bearer reaches them) — and a route with no rule is 403
+// auth_rule_missing by design, so the rule rows and these registrations must land together.
+
+const crypto = require('node:crypto');
+const fs = require('node:fs');
+const fsp = require('node:fs/promises');
+const path = require('node:path');
+
+const { registerApiRoutes } = require('../api-router');
+const { CHUNK_BYTES } = require('../memory-join');
+
+// §6.6: one apply page is at most 2000 entries; 8 MiB comfortably covers a full page and is the
+// route's own refusal point, independent of Wall-E's. §6.7's chunk route reuses the same cap: a
+// 4 MiB chunk is ~5.4 MiB of base64 inside JSON, so 8 MiB is the honest bound for both.
+const MAX_BODY_BYTES = 8 * 1024 * 1024;
+const BODY_TOO_LARGE = 'request_body_too_large';
+
+// §6.7: the ingest directory and its file names. A join is `<dataDir>/sync-inbox/join-<joinId>`,
+// with a `.part` while the chunks are arriving, a `.part.json` sidecar tracking `received[]`, and
+// `.db` once the whole-file sha256 matches.
+const INBOX_DIR_NAME = 'sync-inbox';
+const JOIN_ID_RE = /^[A-Za-z0-9._:-]{1,200}$/;
+const SHA256_HEX_RE = /^[0-9a-f]{64}$/;
+const SIDECAR_SUFFIX = '.part.json';
+const PART_SUFFIX = '.part';
+const FINAL_SUFFIX = '.db';
+
+let _deps = null;
+
+function configure({ sendJsonResponse, readJsonRequestBody, walleClient, dataDir, statfs } = {}) {
+  _deps = { sendJsonResponse, readJsonRequestBody, walleClient, dataDir, statfs };
+}
+
+function _walle() {
+  return (_deps && _deps.walleClient) || require('../walle-client');
+}
+
+// §6.7: the ingest directory. Resolved lazily so a test can inject a temp dir (`dataDir` seam) and
+// production keeps lib/data-dir.js's single source of truth; process.env is never read here.
+function _dataDir() {
+  if (_deps && _deps.dataDir) return String(_deps.dataDir);
+  return require('../data-dir').getDataDir();
+}
+
+function _joinFiles(joinId) {
+  const dir = path.join(_dataDir(), INBOX_DIR_NAME);
+  return {
+    dir,
+    part: path.join(dir, `join-${joinId}${PART_SUFFIX}`),
+    sidecar: path.join(dir, `join-${joinId}${SIDECAR_SUFFIX}`),
+    final: path.join(dir, `join-${joinId}${FINAL_SUFFIX}`),
+  };
+}
+
+// Every file call here is ASYNC (node:fs/promises): this module sits on the server's main thread,
+// where a synchronous multi-megabyte hash or write would stall every other request (../bin/main-thread-lint.js).
+
+async function _readSidecar(sidecarPath) {
+  try {
+    const parsed = JSON.parse(await fsp.readFile(sidecarPath, 'utf8'));
+    return {
+      total: Number(parsed.total) || 0,
+      fileBytes: Number(parsed.fileBytes) || 0,
+      fileSha256: String(parsed.fileSha256 || ''),
+      received: Array.isArray(parsed.received) ? parsed.received.map(Number).filter(Number.isInteger) : [],
+    };
+  } catch {
+    return null; // expected: no sidecar yet (or an unreadable one) — the caller rebuilds it
+  }
+}
+
+// Atomic sidecar write: a temp file in the same directory then a rename, so a crash mid-write can
+// never leave a half-parsed `received[]` that would make the hub re-send or skip a chunk.
+async function _writeSidecar(sidecarPath, value) {
+  const tmp = `${sidecarPath}.${process.pid}.tmp`;
+  await fsp.writeFile(tmp, JSON.stringify(value), { mode: 0o600 });
+  await fsp.rename(tmp, sidecarPath);
+}
+
+async function _sha256OfFile(filePath) {
+  const hash = crypto.createHash('sha256');
+  const handle = await fsp.open(filePath, 'r');
+  try {
+    const buffer = Buffer.alloc(CHUNK_BYTES);
+    let position = 0;
+    for (;;) {
+      const { bytesRead } = await handle.read(buffer, 0, buffer.length, position);
+      if (bytesRead <= 0) break;
+      hash.update(buffer.subarray(0, bytesRead));
+      position += bytesRead;
+    }
+  } finally {
+    try { await handle.close(); } catch { /* expected: a close fault must not mask the hash verdict */ }
+  }
+  return hash.digest('hex');
+}
+
+async function _unlinkQuiet(filePath) {
+  try { await fsp.unlink(filePath); } catch { /* expected: already gone — the delete is idempotent */ }
+}
+
+async function _exists(filePath) {
+  try { await fsp.stat(filePath); return true; } catch { /* expected: a missing path is the normal case */ return false; }
+}
+
+function _freeBytes(dir) {
+  try {
+    const statfs = (_deps && typeof _deps.statfs === 'function') ? _deps.statfs : fs.statfsSync;
+    const stats = statfs(dir);
+    return Number(stats.bavail) * Number(stats.bsize);
+  } catch {
+    // expected: an unsupported platform/filesystem — treat as "unknown" and let the write fail
+    return null;
+  }
+}
+
+
+function _send(res, status, payload) {
+  if (_deps && typeof _deps.sendJsonResponse === 'function') return _deps.sendJsonResponse(res, status, payload || {});
+  const body = JSON.stringify(payload || {});
+  res.writeHead(status, { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(body) });
+  res.end(body);
+}
+
+async function _readBody(req) {
+  const reader = (_deps && _deps.readJsonRequestBody);
+  if (typeof reader !== 'function') {
+    // No reader injected (route registrations run without configure()): an empty body is the honest
+    // answer, and the forward still returns Wall-E's verdict.
+    return { ok: true, body: undefined };
+  }
+  try {
+    return { ok: true, body: await reader(req, { limitBytes: MAX_BODY_BYTES }) };
+  } catch (err) {
+    const tooLarge = err && err.message === BODY_TOO_LARGE;
+    return { ok: false, tooLarge };
+  }
+}
+
+async function _forward(res, pathname, { method = 'GET', body, headers } = {}) {
+  let result;
+  try {
+    const opts = body === undefined ? { method } : { method, body };
+    if (headers && Object.keys(headers).length) opts.headers = headers;
+    result = await _walle().requestJson(pathname, opts);
+  } catch {
+    // expected: the local Wall-E daemon is unreachable (down / mid-restart) — a truthful 503 the hub
+    // retries, never a fabricated empty page.
+    return _send(res, 503, { ok: false, error: 'walle_unreachable' });
+  }
+  const status = Number(result && result.status) || 502;
+  const payload = result && result.json != null ? result.json : { ok: false, error: 'walle_bad_response' };
+  return _send(res, status, payload);
+}
+
+async function _handleVector(req, res) {
+  return _forward(res, '/api/walle/sync/v2/vector', { method: 'GET' });
+}
+
+async function _handleChanges(req, res) {
+  const read = await _readBody(req);
+  if (!read.ok) return _send(res, 413, { ok: false, error: 'payload_too_large' });
+  return _forward(res, '/api/walle/sync/v2/changes', { method: 'POST', body: read.body === undefined ? {} : read.body });
+}
+
+async function _handleApply(req, res) {
+  const read = await _readBody(req);
+  if (!read.ok) return _send(res, 413, { ok: false, error: 'payload_too_large' });
+  // shared-memory D-5: the calling home names itself; Wall-E records this node as its member.
+  const home = String((req && req.headers && req.headers['x-ctm-memory-sync-home']) || '').slice(0, 160).trim();
+  return _forward(res, '/api/walle/sync/v2/apply', {
+    method: 'POST', body: read.body === undefined ? {} : read.body, ...(home ? { headers: { 'X-CTM-Memory-Sync-Home': home } } : {}),
+  });
+}
+
+async function _handleDigest(req, res, url) {
+  const table = (url && url.searchParams && url.searchParams.get('table')) || '';
+  const query = table ? `?table=${encodeURIComponent(table)}` : '';
+  return _forward(res, `/api/walle/sync/v2/digest${query}`, { method: 'GET' });
+}
+
+// ------------------------------------------------------------------------------------------------
+// §6.7 join: the member CTM assembles the hub's snapshot on its own disk (A-10). A join lives in
+// <dataDir>/sync-inbox as join-<joinId>.part (+ .part.json) until every chunk is in and the
+// whole-file sha256 matches, then becomes join-<joinId>.db for the local Wall-E to import (WP-4a).
+
+// One received chunk: write it at its own offset, record the index, and — when that was the last
+// one — verify the whole file before renaming it into place. Re-sending an index overwrites it.
+async function _handleJoinChunk(req, res) {
+  const read = await _readBody(req);
+  // §6.7: 8 MiB, the same bound as every other v2 body (4 MiB raw -> ~5.4 MiB of base64 JSON).
+  if (!read.ok) return _send(res, 413, { ok: false, error: 'payload_too_large' });
+  const body = (read.body && typeof read.body === 'object') ? read.body : {};
+  const joinId = String(body.joinId || '');
+  if (!JOIN_ID_RE.test(joinId)) return _send(res, 400, { ok: false, error: 'invalid_join_id' });
+  const index = Number(body.index);
+  const total = Number(body.total);
+  const fileBytes = Number(body.fileBytes);
+  const chunkBytes = Number(body.chunkBytes);
+  if (!Number.isInteger(index) || index < 0 || !Number.isInteger(total) || total < 1 || index >= total) {
+    return _send(res, 400, { ok: false, error: 'index_out_of_range' });
+  }
+  if (!Number.isInteger(fileBytes) || fileBytes < 0) return _send(res, 400, { ok: false, error: 'invalid_file_bytes' });
+  if (!Number.isInteger(chunkBytes) || chunkBytes < 0 || chunkBytes > CHUNK_BYTES) {
+    return _send(res, 400, { ok: false, error: 'invalid_chunk_bytes' });
+  }
+  if (index * CHUNK_BYTES + chunkBytes > fileBytes) return _send(res, 400, { ok: false, error: 'chunk_out_of_file' });
+  const data = Buffer.from(String(body.dataBase64 || ''), 'base64');
+  if (data.length !== chunkBytes) return _send(res, 400, { ok: false, error: 'chunk_length_mismatch' });
+
+  const files = _joinFiles(joinId);
+  try {
+    await fsp.mkdir(files.dir, { recursive: true, mode: 0o700 });
+  } catch {
+    return _send(res, 500, { ok: false, error: 'join_inbox_unwritable' });
+  }
+  // §6.7: the free-space precondition is checked before index 0 lands — 2x the snapshot, so the
+  // member can hold the .part and the imported DB side by side.
+  if (index === 0 || !(await _exists(files.part))) {
+    const free = _freeBytes(files.dir);
+    if (free != null && free < 2 * fileBytes) return _send(res, 507, { ok: false, error: 'insufficient_storage' });
+  }
+  let handle;
+  try {
+    handle = await fsp.open(files.part, (await _exists(files.part)) ? 'r+' : 'w', 0o600);
+  } catch {
+    return _send(res, 500, { ok: false, error: 'join_part_open_failed' });
+  }
+  try {
+    await handle.write(data, 0, data.length, index * CHUNK_BYTES);
+  } catch {
+    return _send(res, 500, { ok: false, error: 'join_part_write_failed' });
+  } finally {
+    try { await handle.close(); } catch { /* expected: a close fault must not mask the write verdict */ }
+  }
+
+  const prior = (await _readSidecar(files.sidecar)) || { received: [] };
+  const received = Array.from(new Set([...prior.received, index])).sort((a, b) => a - b);
+  const sidecar = { total, fileBytes, fileSha256: String(body.fileSha256 || ''), received };
+  try {
+    await _writeSidecar(files.sidecar, sidecar);
+  } catch {
+    return _send(res, 500, { ok: false, error: 'join_sidecar_write_failed' });
+  }
+  if (received.length < total) return _send(res, 200, { ok: true, received: index, complete: false });
+
+  // Every index is in: verify the assembled file and either promote it or drop the whole transfer.
+  const actual = await _sha256OfFile(files.part);
+  if (!SHA256_HEX_RE.test(sidecar.fileSha256) || actual !== sidecar.fileSha256) {
+    await _unlinkQuiet(files.part);
+    await _unlinkQuiet(files.sidecar);
+    return _send(res, 422, { ok: false, error: 'sha256_mismatch' });
+  }
+  try {
+    await fsp.rename(files.part, files.final);
+  } catch {
+    return _send(res, 500, { ok: false, error: 'join_promote_failed' });
+  }
+  await _unlinkQuiet(files.sidecar);
+  return _send(res, 200, { ok: true, complete: true, path: files.final });
+}
+
+// What the hub has already managed to send for this join — the resume answer after either side
+// restarted (P-3, checkpoints on both sides).
+async function _handleJoinTransfer(req, res, url) {
+  const joinId = String((url && url.searchParams && url.searchParams.get('joinId')) || '');
+  if (!JOIN_ID_RE.test(joinId)) return _send(res, 400, { ok: false, error: 'invalid_join_id' });
+  const files = _joinFiles(joinId);
+  const sidecar = await _readSidecar(files.sidecar);
+  if (sidecar) {
+    return _send(res, 200, { ok: true, joinId, total: sidecar.total, received: sidecar.received, complete: false, path: null });
+  }
+  if (await _exists(files.final)) {
+    return _send(res, 200, { ok: true, joinId, total: 0, received: [], complete: true, path: files.final });
+  }
+  return _send(res, 200, { ok: true, joinId, total: 0, received: [], complete: false, path: null });
+}
+
+async function _handleJoinImport(req, res) {
+  const read = await _readBody(req);
+  if (!read.ok) return _send(res, 413, { ok: false, error: 'payload_too_large' });
+  return _forward(res, '/api/walle/sync/v2/join/import', { method: 'POST', body: read.body === undefined ? {} : read.body });
+}
+
+async function _handleJoinStatus(req, res) {
+  return _forward(res, '/api/walle/sync/v2/join/status', { method: 'GET' });
+}
+
+function register() {
+  registerApiRoutes('memory-sync', [
+    { method: 'GET', path: '/api/sync/brain/v2/vector', handler: (req, res, url) => _handleVector(req, res, url) },
+    { method: 'POST', path: '/api/sync/brain/v2/changes', handler: (req, res, url) => _handleChanges(req, res, url) },
+    { method: 'POST', path: '/api/sync/brain/v2/apply', handler: (req, res, url) => _handleApply(req, res, url) },
+    { method: 'GET', path: '/api/sync/brain/v2/digest', handler: (req, res, url) => _handleDigest(req, res, url) },
+    { method: 'POST', path: '/api/sync/brain/v2/join/chunk', handler: (req, res, url) => _handleJoinChunk(req, res, url) },
+    { method: 'GET', path: '/api/sync/brain/v2/join/transfer', handler: (req, res, url) => _handleJoinTransfer(req, res, url) },
+    { method: 'POST', path: '/api/sync/brain/v2/join/import', handler: (req, res, url) => _handleJoinImport(req, res, url) },
+    { method: 'GET', path: '/api/sync/brain/v2/join/status', handler: (req, res, url) => _handleJoinStatus(req, res, url) },
+  ]);
+}
+
+module.exports = { configure, register };

package/template/coding-task-manager/lib/api/misc-ops.js

@@ -33,6 +33,7 @@
 
 const { registerApiRoutes } = require('../api-router');
 const remoteMachineProxy = require('../remote-machine-proxy');
+const { noteCreateIntent } = require('../create-intent-lease');
 
 let _deps = null;
 
@@ -263,9 +264,13 @@ function register() {
       handler: (req, res, url) => handleWorkspaceSessionsDelete(req, res, url),
     },
     { method: 'DELETE', prefix: '/api/workspaces/', handler: (req, res, url) => handleWorkspaceDelete(req, res, url) },
-    { method: 'GET', path: '/api/projects', handler: _machineRouted('GET /api/projects', (req, res) => _deps.apiGetProjects(req, res)) },
-    { method: 'GET', path: '/api/path-suggest', handler: _machineRouted('GET /api/path-suggest', (req, res, url) => _deps.apiPathSuggest(req, res, url)) },
-    { method: 'POST', path: '/api/path-status', handler: _machineRouted('POST /api/path-status', (req, res) => _deps.apiPathStatus(req, res)) },
+    // WP-3 (D-4): the new-session modal's three picker calls note a create intent on the machine
+    // that ANSWERS them. The note sits in the `local` arrow, so the `_machineRouted` proxy branch
+    // (a request naming another box) never notes here -- the box that would restart is the box
+    // that notes, and a hub must not hold its own restart for a peer's create.
+    { method: 'GET', path: '/api/projects', handler: _machineRouted('GET /api/projects', (req, res) => { noteCreateIntent('GET /api/projects', req); return _deps.apiGetProjects(req, res); }) },
+    { method: 'GET', path: '/api/path-suggest', handler: _machineRouted('GET /api/path-suggest', (req, res, url) => { noteCreateIntent('GET /api/path-suggest', req); return _deps.apiPathSuggest(req, res, url); }) },
+    { method: 'POST', path: '/api/path-status', handler: _machineRouted('POST /api/path-status', (req, res) => { noteCreateIntent('POST /api/path-status', req); return _deps.apiPathStatus(req, res); }) },
     { method: 'GET', path: '/api/rules', handler: (req, res, url) => _deps.apiGetRules(req, res, url) },
     { method: 'PUT', path: '/api/rules', handler: (req, res) => _deps.apiPutRules(req, res) },
     { method: 'GET', path: '/api/rules/list', handler: (req, res) => _deps.apiListRules(req, res) },