@volter/twin 0.1.0

package/src/proxy.ts

@@ -0,0 +1,180 @@
+// Zero-edit browser injection: a dev proxy you put in front of your app so the
+// browser's vendor SDK calls share the SAME twin the backend uses — no app edits.
+//
+//   volter-twin proxy --target http://localhost:3000 \
+//     --map stripe=http://127.0.0.1:12111 --route stripe=/v1/ --loader-host stripe=https://api.stripe.com
+//
+// The browser loads your app through the proxy. Requests to a configured vendor's
+// browser API path (its `apiPathPrefix`, e.g. Stripe's /v1/…) are forwarded to that
+// vendor's twin; everything else passes through to your app. For vendors whose browser
+// SDK loads from a CDN and calls an absolute host (its `loaderHost`, e.g. Stripe.js →
+// api.stripe.com), the proxy rewrites the loader so its calls become same-origin and get
+// forwarded too. This kernel module is VENDOR-AGNOSTIC: the per-vendor routing values are
+// DATA supplied by the caller (sourced from each pack's `TwinPack.browserRouting`), never a
+// hardcoded vendor table.
+import http from 'node:http';
+import net from 'node:net';
+
+/** How one vendor's browser SDK is routed to its twin. Supplied by the caller. */
+export interface VendorRoute {
+  /** The vendor's twin origin, e.g. http://127.0.0.1:12111 */
+  origin: string;
+  /** Same-origin browser API path that belongs to this vendor, e.g. '/v1/' */
+  apiPathPrefix: string;
+  /** Absolute API host to strip from the vendor's browser loader so calls become
+   *  same-origin (e.g. 'https://api.stripe.com'). Omit if the SDK already calls same-origin. */
+  loaderHost?: string;
+}
+
+export interface TwinProxyOptions {
+  /** Your app's origin, e.g. http://localhost:3000 */
+  target: string;
+  /** vendor → routing. */
+  map: Record<string, VendorRoute>;
+  /** Extra local service origins whose redirects should be rewritten back to this proxy origin. */
+  redirectOrigins?: string[];
+  /** Listen port (0 = ephemeral). */
+  port?: number;
+}
+
+export interface TwinProxy {
+  port: number;
+  stop(): void;
+}
+
+function vendorForPath(path: string, map: Record<string, VendorRoute>): string | null {
+  for (const [vendor, route] of Object.entries(map)) {
+    if (path.startsWith(route.apiPathPrefix)) return vendor;
+  }
+  return null;
+}
+
+function escapeRegExp(value: string): string {
+  return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
+/**
+ * Strip `loaderHost` (a full origin, e.g. 'https://api.stripe.com') from `text` so its
+ * calls become same-origin — but ONLY where it appears as a complete host, not as a
+ * blind substring. A plain `split(loaderHost).join('')` (the prior implementation) is
+ * NOT position-aware: it removes the substring wherever it occurs, including as a
+ * PREFIX of a longer, different host — e.g. 'https://api.stripe.com.evil.example/x'
+ * contains 'https://api.stripe.com' as a literal substring, and a naive strip corrupts
+ * it into '.evil.example/x' (the true host is stripe.com.evil.example, not stripe.com;
+ * same substring bug would also mangle a subdomain like sandbox.api.stripe.com if the
+ * scheme weren't part of the match). Position-aware fix: only strip a match that is NOT
+ * immediately followed by a domain-continuing character (letter/digit/dot/hyphen) —
+ * that character means the matched text is a prefix of a longer host, not the whole
+ * host, so it is left untouched.
+ */
+function stripLoaderHost(text: string, loaderHost: string): string {
+  const pattern = new RegExp(`${escapeRegExp(loaderHost)}(?![A-Za-z0-9.-])`, 'g');
+  return text.replace(pattern, '');
+}
+
+function rewriteLocation(location: string | string[] | undefined, req: http.IncomingMessage, origins: string[]): string | string[] | undefined {
+  if (!location || Array.isArray(location)) return location;
+  const host = req.headers.host;
+  if (!host) return location;
+  const proxyOrigin = `http://${host}`;
+  for (const origin of origins) {
+    const normalized = origin.replace(/\/$/, '');
+    if (location === normalized) return proxyOrigin;
+    if (location.startsWith(`${normalized}/`)) return `${proxyOrigin}${location.slice(normalized.length)}`;
+  }
+  return location;
+}
+
+function forwardTo(
+  origin: string,
+  req: http.IncomingMessage,
+  res: http.ServerResponse,
+  rewrite?: (body: Buffer, headers: http.IncomingHttpHeaders) => Buffer,
+  redirectOrigins: string[] = [],
+): void {
+  const target = new URL(req.url || '/', origin);
+  const headers = { ...req.headers, host: target.host };
+  delete headers['accept-encoding']; // so we can rewrite uncompressed bodies
+  const upstream = http.request(target, { method: req.method, headers }, (up) => {
+    if ((up.statusCode ?? 0) >= 300 && (up.statusCode ?? 0) < 400) {
+      const responseHeaders = { ...up.headers };
+      // `location` is single-valued, so rewriteLocation returns string|undefined here (never an array).
+      responseHeaders.location = rewriteLocation(responseHeaders.location, req, [origin, ...redirectOrigins]) as string | undefined;
+      delete responseHeaders['transfer-encoding'];
+      delete responseHeaders['content-length'];
+      res.writeHead(up.statusCode || 302, responseHeaders);
+      res.end();
+      up.resume();
+      return;
+    }
+    const chunks: Buffer[] = [];
+    up.on('data', (c) => chunks.push(c as Buffer));
+    up.on('error', () => res.destroyed || res.end());
+    up.on('end', () => {
+      const body = Buffer.concat(chunks);
+      const out = rewrite ? rewrite(body, up.headers) : body;
+      const responseHeaders = { ...up.headers };
+      delete responseHeaders['content-length'];
+      delete responseHeaders['content-encoding'];
+      delete responseHeaders['transfer-encoding'];
+      responseHeaders['content-length'] = String(Buffer.byteLength(out));
+      res.writeHead(up.statusCode || 502, responseHeaders);
+      res.end(out);
+    });
+  });
+  upstream.on('error', (error) => {
+    if (res.headersSent) { res.destroy(error); return; }
+    res.writeHead(502, { 'content-type': 'text/plain' });
+    res.end(`twin proxy upstream error: ${error.message}`);
+  });
+  req.pipe(upstream);
+}
+
+export function createTwinProxy(opts: TwinProxyOptions): TwinProxy {
+  const map: Record<string, VendorRoute> = {};
+  for (const [vendor, route] of Object.entries(opts.map)) map[vendor] = { ...route, origin: route.origin.replace(/\/$/, '') };
+  const redirectOrigins = (opts.redirectOrigins ?? []).map((origin) => origin.replace(/\/$/, ''));
+
+  const server = http.createServer((req, res) => {
+    const path = req.url || '/';
+    const vendor = vendorForPath(path, map);
+    if (vendor) {
+      forwardTo(map[vendor].origin, req, res, undefined, redirectOrigins); // browser vendor API call → the twin
+      return;
+    }
+    // Otherwise pass through to the app, rewriting vendor loaders so their calls come back here.
+    forwardTo(opts.target, req, res, (body, headers) => {
+      const contentType = String(headers['content-type'] || '');
+      if (!contentType.includes('javascript') && !contentType.includes('html')) return body;
+      let text = body.toString('utf8');
+      for (const route of Object.values(map)) {
+        if (route.loaderHost) text = stripLoaderHost(text, route.loaderHost);
+      }
+      return Buffer.from(text, 'utf8');
+    }, redirectOrigins);
+  });
+
+  // WebSocket / HTTP upgrade passthrough to the app.
+  server.on('upgrade', (req, socket, head) => {
+    const target = new URL(req.url || '/', opts.target);
+    const upstream = net.connect(Number(target.port || 80), target.hostname, () => {
+      upstream.write(`${req.method} ${target.pathname}${target.search} HTTP/${req.httpVersion}\r\n`);
+      for (const [name, value] of Object.entries(req.headers)) upstream.write(`${name}: ${value}\r\n`);
+      upstream.write('\r\n');
+      upstream.write(head);
+      upstream.pipe(socket);
+      socket.pipe(upstream);
+    });
+    const killSocket = () => { if (!socket.destroyed) socket.destroy(); };
+    const killUpstream = () => { if (!upstream.destroyed) upstream.destroy(); };
+    upstream.on('error', killSocket);
+    socket.on('error', killUpstream);
+    socket.on('close', killUpstream);
+    upstream.on('close', killSocket);
+  });
+
+  server.listen(opts.port ?? 0, '127.0.0.1');
+  const address = server.address();
+  const port = address && typeof address === 'object' ? address.port : (opts.port ?? 0);
+  return { port, stop: () => server.close() };
+}

package/src/pushLedger.ts

@@ -0,0 +1,189 @@
+// Push ledger + phases (the twins architecture notes → "Push Ledger Packet").
+// Records replication of a local transaction commit to the real provider, separate
+// from observed events and the action log. A push walks explicit phases —
+//   intent → attempted → provider_accepted → observed_confirmed
+//          → projection_suppressed → confirmed   (or failed / abandoned)
+// `provider_accepted` (the API call returned) is NOT confirmation; confirmation is
+// when the real provider fact is committed as an observed event and linked back to
+// the action (via confirmAction), at which point the local projection is suppressed.
+// Real I/O enters only through the injected writeFn (auth boundary). Append-only;
+// current status = the latest row per push id. Idempotent: a confirmed push replays.
+//
+// Crash parity with egress.ts: a crash mid-flight (process dies while writeFn is in
+// flight) leaves the latest row `attempted` with nothing after it — we don't know
+// whether the provider call landed. Re-sending blind risks a silent double-apply
+// against a non-idempotent vendor operation, so pushTransaction refuses
+// (UnreconciledPushError) unless the caller verifies external state and passes
+// onUnreconciled: 'retry' (mirrors egress.ts's UnreconciledWriteIntentError).
+import { randomUUID } from 'node:crypto';
+import { existsSync, mkdirSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { confirmAction } from './actions.ts';
+import type { SubjectFields } from './shadow.ts';
+import { appendDurable, twinLog, withFileLock, worldPaths } from './storage.ts';
+
+export type PushStatus =
+  | 'intent'
+  | 'attempted'
+  | 'provider_accepted'
+  | 'succeeded'
+  | 'failed'
+  | 'observed_confirmed'
+  | 'projection_suppressed'
+  | 'confirmed'
+  | 'abandoned';
+
+export type WorldPushRecord = {
+  id: string;
+  service: string;
+  actionId: string;
+  provider: string;
+  operation: string;
+  status: PushStatus;
+  idempotencyKey: string;
+  createdAt: string;
+  external?: { id?: string; url?: string; rawRef?: string };
+  confirmedByEventId?: string;
+  error?: string;
+  data?: Record<string, unknown>;
+  /** Request-scoped correlation id (D3), inherited from the action being pushed —
+   *  every phase row for one push carries the SAME id as the action row it drives,
+   *  so an action row and its push-ledger rows join on this id alone (no reliance
+   *  on actionId, which is a different, id-shaped field with its own naming
+   *  convention). Falls back to a fresh id only if the action predates D3. */
+  correlationId?: string;
+};
+
+export type PushOutcome = { externalId: string; url?: string; data?: Record<string, unknown> };
+
+function ledgerPath(service: string, root?: string): string {
+  return join(worldPaths(service, root).dir, 'push-ledger.jsonl');
+}
+
+/** Lock guarding the push ledger's read-then-append critical sections (TWIN-56):
+ *  the same file-lock primitive events/actions use, so two callers racing to push
+ *  the same action can't both read "no prior attempt" and both append duplicate
+ *  attempt rows. */
+function ledgerLockPath(service: string, root?: string): string {
+  return `${ledgerPath(service, root)}.lock`;
+}
+
+export function appendPushRecord(record: WorldPushRecord, root?: string): WorldPushRecord {
+  const path = ledgerPath(record.service, root);
+  mkdirSync(join(path, '..'), { recursive: true });
+  appendDurable(path, `${JSON.stringify(record)}\n`);
+  twinLog('push.append', { service: record.service, id: record.id, actionId: record.actionId, status: record.status, correlationId: record.correlationId });
+  return record;
+}
+
+export function listPushLedger(service: string, root?: string): WorldPushRecord[] {
+  const path = ledgerPath(service, root);
+  if (!existsSync(path)) return [];
+  return readFileSync(path, 'utf8').split('\n').filter((l) => l.trim()).map((l) => JSON.parse(l) as WorldPushRecord);
+}
+
+/** Latest phase per push id (append order is the tiebreak). */
+export function latestPushByActionId(service: string, root?: string): Map<string, WorldPushRecord> {
+  const latest = new Map<string, WorldPushRecord>();
+  for (const r of listPushLedger(service, root)) latest.set(r.actionId, r); // later rows overwrite
+  return latest;
+}
+
+/** Pushes that returned provider_accepted but have not yet been observed_confirmed. */
+export function unconfirmedPushes(service: string, root?: string): WorldPushRecord[] {
+  return [...latestPushByActionId(service, root).values()].filter((r) => r.status === 'provider_accepted' || r.status === 'attempted');
+}
+
+const TERMINAL: PushStatus[] = ['confirmed', 'projection_suppressed', 'succeeded'];
+
+export class UnreconciledPushError extends Error {
+  readonly pushId: string;
+
+  constructor(pushId: string, message: string) {
+    super(message);
+    this.name = 'UnreconciledPushError';
+    this.pushId = pushId;
+  }
+}
+
+/**
+ * Drive one local transaction commit through the push phases against the real
+ * provider via the injected writeFn (the ONLY real I/O — kernel holds no creds).
+ * On success it confirms the action into an observed event (confirmAction), which
+ * suppresses the local projection so state isn't double-counted. Idempotent: a
+ * previously-confirmed push for the same id replays without calling writeFn.
+ *
+ * Crash parity with egress.ts (performExternalWrite): the "is there a prior row,
+ * and is it terminal / unreconciled" read plus the fresh intent+attempted append
+ * run under ONE file-lock acquisition (TWIN-56), so two callers racing to push the
+ * same action id can't both observe "no attempt yet" and both append attempt rows.
+ * By default (onUnreconciled: 'fail') a latest row of `attempted` with nothing
+ * after it — a crash mid-flight, or a racing caller that lost the lock — refuses
+ * with UnreconciledPushError instead of blindly re-invoking writeFn. Pass
+ * onUnreconciled: 'retry' only after verifying external state.
+ */
+export async function pushTransaction(
+  opts: {
+    service: string;
+    action: { id: string; subject: { type: string; id: string }; fields: SubjectFields; correlationId?: string };
+    provider: string;
+    operation: string;
+    idempotencyKey: string;
+    occurredAt: string;
+    root?: string;
+    onUnreconciled?: 'fail' | 'retry';
+  },
+  writeFn: () => Promise<PushOutcome>,
+): Promise<WorldPushRecord> {
+  const { service, action, provider, operation, idempotencyKey, occurredAt, root, onUnreconciled = 'fail' } = opts;
+  const pushId = `push:${service}:${action.id}`;
+  // Inherit the action's correlationId (D3) so every phase row this push writes joins
+  // back to the action row on that id alone. Actions appended after D3 always carry
+  // one (appendAction/appendActionIfAbsent stamp it); the fallback only covers an
+  // action object built by hand without going through those (e.g. an older ledger).
+  const correlationId = action.correlationId ?? randomUUID();
+  const base = { id: pushId, service, actionId: action.id, provider, operation, idempotencyKey, createdAt: occurredAt, correlationId };
+
+  // Read (prior status) + decide (terminal replay / unreconciled refusal) + the
+  // fresh intent/attempted append, all under one lock — the atomic critical
+  // section a cross-process race would otherwise be able to interleave (TWIN-56).
+  const early = withFileLock(ledgerLockPath(service, root), (): WorldPushRecord | null => {
+    const prior = latestPushByActionId(service, root).get(action.id);
+    if (prior && TERMINAL.includes(prior.status)) return prior; // idempotent replay
+    if (prior && prior.status === 'attempted' && onUnreconciled === 'fail') {
+      throw new UnreconciledPushError(
+        pushId,
+        `Push ${pushId} has an 'attempted' row with no result after it — the provider call may or may not have happened. ` +
+        `Verify external state, then retry with onUnreconciled: 'retry'.`,
+      );
+    }
+    appendPushRecord({ ...base, status: 'intent' }, root);
+    appendPushRecord({ ...base, status: 'attempted' }, root);
+    return null;
+  });
+  if (early) return early;
+
+  let outcome: PushOutcome;
+  try {
+    outcome = await writeFn();
+  } catch (error) {
+    return appendPushRecord({ ...base, status: 'failed', error: error instanceof Error ? error.message : String(error) }, root);
+  }
+  appendPushRecord({ ...base, status: 'provider_accepted', external: { id: outcome.externalId, ...(outcome.url ? { url: outcome.url } : {}) }, ...(outcome.data ? { data: outcome.data } : {}) }, root);
+
+  // Confirm: append the observed provider fact + map it back, suppressing the local projection.
+  const { observedEventId } = confirmAction({ service, actionId: action.id, subject: action.subject, fields: action.fields, occurredAt, root });
+  appendPushRecord({ ...base, status: 'observed_confirmed', external: { id: outcome.externalId }, confirmedByEventId: observedEventId }, root);
+  appendPushRecord({ ...base, status: 'projection_suppressed', confirmedByEventId: observedEventId }, root);
+  return appendPushRecord({ ...base, status: 'confirmed', external: { id: outcome.externalId }, confirmedByEventId: observedEventId }, root);
+}
+
+/** Mark a push intent abandoned (e.g. operator decided not to push). */
+export function abandonPush(service: string, actionId: string, opts: { root?: string; at?: string; reason?: string } = {}): WorldPushRecord {
+  const prior = latestPushByActionId(service, opts.root).get(actionId);
+  return appendPushRecord({
+    id: `push:${service}:${actionId}`, service, actionId, provider: prior?.provider ?? '', operation: prior?.operation ?? '',
+    idempotencyKey: prior?.idempotencyKey ?? `abandon:${actionId}`, createdAt: opts.at ?? new Date().toISOString(),
+    status: 'abandoned', ...(opts.reason ? { error: opts.reason } : {}),
+  }, opts.root);
+}

package/src/queueLifecycle.ts

@@ -0,0 +1,130 @@
+// Event queue lifecycle (the twins architecture notes → "Event Intake Queue").
+// Listener/webhook/broker deliveries land in the durable queue (events stay
+// NON-canonical until committed). This module gives queue rows the named
+// lifecycle the doc specifies — queued | committed | ignored | superseded |
+// poisoned — as an append-only status ledger over the existing queue, so the
+// queue file itself stays immutable delivery history. Current status = the latest
+// transition (default `queued`). Deterministic: caller may supply `at`.
+import { existsSync, mkdirSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { appendDurable, appendEvent, listQueuedEvents, rebuildGenericState, worldPaths } from './storage.ts';
+import type { AppendEventResult, QueuedWorldServiceEvent } from './types.ts';
+
+export type QueueRowStatus = 'queued' | 'committed' | 'ignored' | 'superseded' | 'poisoned';
+
+export type QueueStatusTransition = {
+  queueId: string;
+  status: QueueRowStatus;
+  at: string;
+  reason?: string;
+  /** for committed rows: the canonical event id it produced. */
+  eventId?: string;
+  /** for superseded rows: the row that replaced it. */
+  supersededBy?: string;
+};
+
+export type QueueRowWithStatus = {
+  row: QueuedWorldServiceEvent;
+  status: QueueRowStatus;
+  transition?: QueueStatusTransition;
+};
+
+function statusPath(service: string, root?: string): string {
+  return join(worldPaths(service, root).dir, 'event-queue-status.jsonl');
+}
+
+function readTransitions(service: string, root?: string): QueueStatusTransition[] {
+  const path = statusPath(service, root);
+  if (!existsSync(path)) return [];
+  return readFileSync(path, 'utf8').split('\n').filter((l) => l.trim()).map((l) => JSON.parse(l) as QueueStatusTransition);
+}
+
+/** Latest transition per queue row (append order is the tiebreak — last wins). */
+function latestByRow(service: string, root?: string): Map<string, QueueStatusTransition> {
+  const latest = new Map<string, QueueStatusTransition>();
+  for (const t of readTransitions(service, root)) latest.set(t.queueId, t); // later lines overwrite
+  return latest;
+}
+
+export function setQueueRowStatus(
+  service: string,
+  queueId: string,
+  status: QueueRowStatus,
+  opts: { root?: string; at?: string; reason?: string; eventId?: string; supersededBy?: string } = {},
+): QueueStatusTransition {
+  const path = statusPath(service, opts.root);
+  mkdirSync(join(path, '..'), { recursive: true });
+  const transition: QueueStatusTransition = {
+    queueId,
+    status,
+    at: opts.at ?? new Date().toISOString(),
+    ...(opts.reason ? { reason: opts.reason } : {}),
+    ...(opts.eventId ? { eventId: opts.eventId } : {}),
+    ...(opts.supersededBy ? { supersededBy: opts.supersededBy } : {}),
+  };
+  appendDurable(path, `${JSON.stringify(transition)}\n`);
+  return transition;
+}
+
+export function queueRowStatus(service: string, queueId: string, root?: string): QueueRowStatus {
+  return latestByRow(service, root).get(queueId)?.status ?? 'queued';
+}
+
+/** Every queue row with its current lifecycle status. */
+export function listQueueWithStatus(service: string, root?: string): QueueRowWithStatus[] {
+  const latest = latestByRow(service, root);
+  return listQueuedEvents(service, root).map((row) => {
+    const transition = latest.get(row.id);
+    return { row, status: transition?.status ?? 'queued', ...(transition ? { transition } : {}) };
+  });
+}
+
+/** Rows still awaiting a decision (status === 'queued'). */
+export function pendingQueueRows(service: string, root?: string): QueuedWorldServiceEvent[] {
+  return listQueueWithStatus(service, root).filter((r) => r.status === 'queued').map((r) => r.row);
+}
+
+export type QueueCounts = Record<QueueRowStatus, number>;
+export function queueCounts(service: string, root?: string): QueueCounts {
+  const counts: QueueCounts = { queued: 0, committed: 0, ignored: 0, superseded: 0, poisoned: 0 };
+  for (const r of listQueueWithStatus(service, root)) counts[r.status] += 1;
+  return counts;
+}
+
+/** Commit a single queued row into the canonical event log + stamp `committed`. */
+export function commitQueueRow(
+  service: string,
+  queueId: string,
+  opts: { root?: string; at?: string } = {},
+): { appended: boolean; eventId?: string } {
+  const status = queueRowStatus(service, queueId, opts.root);
+  if (status !== 'queued') throw new Error(`queue row ${queueId} is ${status}, not committable`);
+  const row = listQueuedEvents(service, opts.root).find((r) => r.id === queueId);
+  if (!row) throw new Error(`queue row not found: ${queueId}`);
+  const result: AppendEventResult = appendEvent(row.event, opts.root);
+  rebuildGenericState(service, opts.root);
+  setQueueRowStatus(service, queueId, 'committed', { ...opts, eventId: result.event.id });
+  return { appended: result.appended, eventId: result.event.id };
+}
+
+export function ignoreQueueRow(service: string, queueId: string, opts: { root?: string; at?: string; reason?: string } = {}): QueueStatusTransition {
+  return setQueueRowStatus(service, queueId, 'ignored', opts);
+}
+export function supersedeQueueRow(service: string, queueId: string, supersededBy: string, opts: { root?: string; at?: string; reason?: string } = {}): QueueStatusTransition {
+  return setQueueRowStatus(service, queueId, 'superseded', { ...opts, supersededBy });
+}
+export function poisonQueueRow(service: string, queueId: string, opts: { root?: string; at?: string; reason?: string } = {}): QueueStatusTransition {
+  return setQueueRowStatus(service, queueId, 'poisoned', opts);
+}
+
+/** Commit all rows currently `queued`, in deterministic provider-time order; stamps committed. */
+export function commitPendingQueue(service: string, opts: { root?: string; at?: string } = {}): { committed: number; eventIds: string[] } {
+  const pending = pendingQueueRows(service, opts.root)
+    .sort((a, b) => a.event.occurredAt.localeCompare(b.event.occurredAt) || a.receivedAt.localeCompare(b.receivedAt) || a.id.localeCompare(b.id));
+  const eventIds: string[] = [];
+  for (const row of pending) {
+    const { eventId } = commitQueueRow(service, row.id, opts);
+    if (eventId) eventIds.push(eventId);
+  }
+  return { committed: eventIds.length, eventIds };
+}