@volter/twin 0.1.0

package/src/sync.ts

@@ -0,0 +1,160 @@
+// Bidirectional sync for twins (the twins architecture notes §4).
+//
+// Two directions, both over the kernel (events + egress); the vendor-specific
+// part — the actual network call — is INJECTED by the caller, never bundled.
+// This is the auth-boundary principle (hard-problem #6): the kernel touches no
+// real credentials; the host supplies a fetch fn (pull) or write fn (push) that
+// uses the user's own auth. The twin stays runnable fully offline (inject fakes).
+//
+//   pull (real → twin): fold externally-observed resources into the event log
+//                       as deltas, so twin state converges to what was observed.
+//   push (twin → real): enact a reconcile plan's `toPush` by calling the injected
+//                       real write fn, idempotently, through the egress ledger.
+//
+// pull + reconcile + push compose into the round-trip the architecture calls for:
+//   pull(real) → fork → write-in-fork → reconcile(base, fork, freshPull) → push.
+//
+// push is gated exactly like applyPlan (TWIN-53 R-K2): approval is recomputed from
+// the ReconcilePlan's own conflicts/field resolutions (reconcileRequiresApproval),
+// never trusted off a stored bit — a ReconcilePlan carries no such bit at all, so
+// there is nothing to tamper — and a mandatory lease (acquire → push → release)
+// makes push single-writer the same way applyPlan is.
+import { performExternalWrite } from './egress.ts';
+import type { EgressWriteResult } from './egress.ts';
+import { acquireLease, releaseLease } from './lease.ts';
+import type { WorldRemoteRef } from './refs.ts';
+import { buildShadowState, recordObservedDelta } from './shadow.ts';
+import type { SubjectFields } from './shadow.ts';
+import { twinResources } from './serve.ts';
+import type { TwinResource } from './serve.ts';
+import { reconcileRequiresApproval } from './reconcile.ts';
+import type { ReconcilePlan } from './reconcile.ts';
+
+// A resource observed from (or destined for) the real vendor.
+export type SyncResource = { type: string; id: string; fields: SubjectFields };
+
+export type PullResult = { observed: number; deltasAppended: number; unchanged: number };
+
+/**
+ * Pull: fold a set of externally-observed resources into the twin's event log.
+ * The caller has already fetched them from the real vendor (injected I/O); this
+ * is the pure fold. Only changed fields produce a delta (recordObservedDelta is
+ * a no-op when nothing differs), so re-pulling identical state appends nothing.
+ *
+ * `redact` (TWIN-45 dev/01 — implemented, not just documented) is an optional
+ * transform applied to each resource BEFORE it is folded into the event log, so
+ * a sensitive field never lands on disk in the first place — a redact-on-pull
+ * hook point, not a redaction policy: this library makes zero default redaction
+ * decisions (no built-in "PII field" list, no defaults). The caller supplies the
+ * hook deliberately, shaped for their own vendor/fields. See docs/DATA_AT_REST.md
+ * for the full data-at-rest story (what's implemented here vs. left as
+ * guidance) and for why day-count retention defaults are NOT hardcoded here —
+ * that is a policy call for the human owner, not this library.
+ */
+export function syncPull(opts: {
+  service: string;
+  resources: SyncResource[];
+  occurredAt: string;
+  root?: string;
+  actor?: { kind: 'agent' | 'human' | 'bot' | 'system'; id?: string };
+  redact?: (resource: SyncResource) => SyncResource;
+}): PullResult {
+  const { service, resources, occurredAt, root } = opts;
+  let deltasAppended = 0;
+  let unchanged = 0;
+  // Rebuild the shadow once; recordObservedDelta appends to the log and the next
+  // build reflects it, so process sequentially.
+  for (const raw of resources) {
+    const res = opts.redact ? opts.redact(raw) : raw;
+    const shadow = buildShadowState(service, () => null, root);
+    const result = recordObservedDelta(
+      shadow,
+      { service, subject: { type: res.type, id: res.id }, observed: res.fields, occurredAt, ...(opts.actor ? { actor: opts.actor } : {}) },
+      root,
+    );
+    if (result.changed) deltasAppended += 1;
+    else unchanged += 1;
+  }
+  return { observed: resources.length, deltasAppended, unchanged };
+}
+
+export type PushItemResult = { id: string; type: string; status: EgressWriteResult['status']; externalId?: string };
+export type PushResult = { attempted: number; pushed: PushItemResult[] };
+
+// Stable hash of the intended field change so identical pushes dedupe (replay)
+// while a different change is a new intent.
+function stableStringify(value: unknown): string {
+  if (value === null || typeof value !== 'object') return JSON.stringify(value);
+  if (Array.isArray(value)) return `[${value.map(stableStringify).join(',')}]`;
+  const keys = Object.keys(value as Record<string, unknown>).sort();
+  return `{${keys.map((k) => `${JSON.stringify(k)}:${stableStringify((value as Record<string, unknown>)[k])}`).join(',')}}`;
+}
+
+/**
+ * Push: enact a reconcile plan's twin→real changes by calling the injected real
+ * write fn, idempotently via the egress ledger. `write` is the ONLY place a real
+ * vendor API is touched; in tests/offline it's a fake. Re-pushing the same plan
+ * replays (no double-apply) because the egress idempotency key encodes the change.
+ *
+ * Gated the same way `applyPlan` is (TWIN-53 R-K2): `approve` and the lease fields
+ * (`remoteRef`/`holder`/`leaseId`/`acquiredAt`/`expiresAt`) are mandatory options —
+ * there is no implicit-approval, no-lease path. Approval is required when
+ * `reconcileRequiresApproval(plan)` is true (unresolved conflicts, or a
+ * twin-wins push that overwrites a real-side change) and `approve` is not set;
+ * refusal happens before the lease is acquired or `write` is ever called — zero
+ * side effects. The lease is acquired for `remoteRef` before any push and released
+ * (even on error) after.
+ */
+export async function syncPush(opts: {
+  service: string;
+  plan: ReconcilePlan;
+  write: (item: SyncResource) => Promise<{ externalId: string; data?: Record<string, unknown> }>;
+  remoteRef: WorldRemoteRef;
+  holder: { kind: 'agent' | 'human' | 'system'; id: string };
+  leaseId: string;
+  acquiredAt: string;
+  expiresAt: string;
+  approve: boolean;
+  root?: string;
+  actor?: { kind: 'agent' | 'human' | 'bot' | 'system'; id?: string };
+}): Promise<PushResult> {
+  const { service, plan, write, root } = opts;
+  if (reconcileRequiresApproval(plan) && !opts.approve) {
+    throw new Error(`sync push for ${service} requires approval (unresolved conflicts, or a twin-wins push overwriting a real-side change); pass approve:true to push`);
+  }
+  const lease = acquireLease({
+    service, provider: opts.remoteRef.provider, remoteRef: opts.remoteRef, planId: `sync-push:${service}`,
+    holder: opts.holder, id: opts.leaseId, acquiredAt: opts.acquiredAt, expiresAt: opts.expiresAt, root,
+  });
+  const pushed: PushItemResult[] = [];
+  try {
+    for (const item of plan.toPush) {
+      const syncItem: SyncResource = { type: item.type, id: item.id, fields: item.fields };
+      const result = await performExternalWrite(
+        {
+          service,
+          operation: `sync.push.${item.type}`,
+          provider: service,
+          subject: { type: item.type, id: item.id },
+          idempotencyKey: `sync-push:${item.id}:${stableStringify(item.fields)}`,
+          ...(opts.actor ? { actor: opts.actor } : {}),
+          data: { fields: item.fields },
+        },
+        async () => {
+          const out = await write(syncItem);
+          return { externalId: out.externalId, data: out.data ?? { ...item.fields } };
+        },
+        { root },
+      );
+      pushed.push({ id: item.id, type: item.type, status: result.status, ...(result.outcome?.externalId ? { externalId: result.outcome.externalId } : {}) });
+    }
+  } finally {
+    releaseLease(service, lease.id, { at: opts.acquiredAt, root });
+  }
+  return { attempted: plan.toPush.length, pushed };
+}
+
+/** Convenience: the twin's current resources as the reconcile `fork`/state input. */
+export function currentResources(service: string, root?: string): TwinResource[] {
+  return twinResources(service, root);
+}

package/src/types.ts

@@ -0,0 +1,50 @@
+import type { z } from 'zod';
+import type {
+  GenericWorldStateSchema,
+  WorldServiceEventSchema,
+} from './schemas.ts';
+
+export type WorldServiceEvent = z.infer<typeof WorldServiceEventSchema>;
+export type GenericWorldState = z.infer<typeof GenericWorldStateSchema>;
+
+export type AppendEventResult = {
+  event: WorldServiceEvent;
+  appended: boolean;
+  duplicateOf?: string;
+};
+
+export type QueuedWorldServiceEvent = {
+  id: string;
+  service: string;
+  receivedAt: string;
+  source: 'listener' | 'webhook' | 'broker' | 'capture' | 'manual' | 'replay';
+  event: WorldServiceEvent;
+};
+
+export type EnqueueEventResult = {
+  queued: QueuedWorldServiceEvent;
+  appended: boolean;
+  duplicateOf?: string;
+};
+
+export type CommitQueuedEventsResult = {
+  service: string;
+  queued: number;
+  committed: number;
+  skipped: number;
+  eventIds: string[];
+};
+
+export type WorldPaths = {
+  root: string;
+  service: string;
+  dir: string;
+  events: string;
+  state: string;
+  resources: string;
+  cursors: string;
+  ingests: string;
+  eventQueue: string;
+};
+
+export type WorldReducer<State> = (state: State, event: WorldServiceEvent) => State;

package/src/validate.ts

@@ -0,0 +1,95 @@
+// World integrity validation — the TWIN's own invariants: egress write intents
+// reconcile (no write left without a recorded result past the grace window). NOTE:
+// annotation integrity (quotes resolve, events annotated, …) moved to
+// @volter/tracker/world-annotations — that's a verification (tracker) concern.
+import { existsSync, readdirSync } from 'node:fs';
+import { join, resolve } from 'node:path';
+import { listEgressLedger } from './egress.ts';
+import { worldStateRoot } from './storage.ts';
+
+export type WorldValidationFinding = {
+  level: 'error' | 'warning';
+  code: string;
+  message: string;
+  service?: string;
+  /** set by the tracker's annotation validator (reuses this finding shape). */
+  annotation?: string;
+  event?: string;
+  details?: Record<string, unknown>;
+};
+
+export type WorldValidationSummary = {
+  status: 'pass' | 'warn' | 'fail';
+  errors: number;
+  warnings: number;
+};
+
+export type WorldValidationReport = {
+  valid: boolean;
+  summary: WorldValidationSummary;
+  findings: WorldValidationFinding[];
+};
+
+function stringValue(value: unknown): string {
+  return typeof value === 'string' ? value.trim() : '';
+}
+
+export function discoverWorldServices(root: string): string[] {
+  const worldRoot = worldStateRoot(root);
+  if (!existsSync(worldRoot)) return [];
+  return readdirSync(worldRoot, { withFileTypes: true })
+    .filter((entry) => entry.isDirectory() && existsSync(join(worldRoot, entry.name, 'events.jsonl')))
+    .map((entry) => entry.name)
+    .sort();
+}
+
+export function validateWorldService(root: string, service: string): WorldValidationFinding[] {
+  // Twin integrity = egress reconciliation. (Annotation integrity moved to
+  // @volter/tracker/world-annotations.)
+  return validateEgressLedger(root, service);
+}
+
+const UNRECONCILED_INTENT_GRACE_MS = 10 * 60 * 1000;
+
+function validateEgressLedger(root: string, service: string): WorldValidationFinding[] {
+  const findings: WorldValidationFinding[] = [];
+  for (const entry of listEgressLedger(service, root)) {
+    if (entry.results.length > 0) continue;
+    const ageMs = Date.now() - Date.parse(entry.intent.occurredAt);
+    if (Number.isFinite(ageMs) && ageMs < UNRECONCILED_INTENT_GRACE_MS) continue;
+    findings.push({
+      level: 'warning',
+      code: 'write_intent_unreconciled',
+      message: 'Egress write intent has no result event — the external write may have happened without being recorded. Verify external state and reconcile before retrying.',
+      service,
+      event: entry.intent.id,
+      details: {
+        operation: stringValue(entry.intent.data.operation),
+        provider: stringValue(entry.intent.data.provider),
+        occurredAt: entry.intent.occurredAt,
+        subject: entry.intent.subject,
+      },
+    });
+  }
+  return findings;
+}
+
+export function summarizeWorldFindings(findings: WorldValidationFinding[]): WorldValidationReport {
+  const errors = findings.filter((finding) => finding.level === 'error').length;
+  const warnings = findings.filter((finding) => finding.level === 'warning').length;
+  return {
+    valid: errors === 0,
+    summary: {
+      status: errors > 0 ? 'fail' : warnings > 0 ? 'warn' : 'pass',
+      errors,
+      warnings,
+    },
+    findings,
+  };
+}
+
+export function validateWorld(options: { root?: string; services?: string[] } = {}): WorldValidationReport {
+  const root = resolve(options.root || process.env.PROJECT_ROOT || process.cwd());
+  const services = options.services && options.services.length > 0 ? options.services : discoverWorldServices(root);
+  return summarizeWorldFindings(services.flatMap((service) => validateWorldService(root, service)));
+}

package/src/visualizer.ts

@@ -0,0 +1,142 @@
+// Twin DEBUG INSPECTOR (NOT the mirror UI / scorecard R13).
+//
+// This renders a generic internal status page — resource/event/egress/divergence
+// tables read directly from kernel state. It's a developer inspector, useful for
+// any twin, but it is explicitly NOT the per-vendor "mirror UI" R13 calls for (a
+// faithful clone of the vendor's product interface — a Slack-like chat UI, a
+// Linear-like board — consuming the twin's API). The real mirror UI is a separate,
+// per-vendor frontend. Keeping this as the honest, narrow thing it is.
+// `renderTwinHtml` is pure (state → HTML string) so it's testable without a server.
+import { listEgressLedger } from './egress.ts';
+import { isFork, forkDivergence } from './fork.ts';
+import { twinResources } from './serve.ts';
+import { listEvents } from './storage.ts';
+import { listActions } from './actions.ts';
+
+function esc(value: unknown): string {
+  return String(value ?? '')
+    .replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;').replace(/"/g, '&quot;');
+}
+
+function fieldCell(value: unknown): string {
+  if (value == null) return '<span class="muted">—</span>';
+  if (Array.isArray(value)) return value.map((v) => `<span class="tag">${esc(v)}</span>`).join(' ');
+  return esc(value);
+}
+
+/** Pure render: the twin's current state → a self-contained HTML page. */
+export function renderTwinHtml(service: string, opts: { root?: string } = {}): string {
+  const { root } = opts;
+  const resources = twinResources(service, root).sort((a, b) => (a.id < b.id ? -1 : 1));
+  const events = listEvents(service, root);
+  const ledger = listEgressLedger(service, root);
+  const actions = listActions(service, root); // R18: local actions (separate from observed events)
+  const fork = isFork(service, root ?? '') ? forkDivergence(service, root ?? '') : null;
+
+  // Union of data field names across resources (excluding identity/meta).
+  const fieldNames = [...new Set(resources.flatMap((r) => Object.keys(r)))].filter((k) => !['id', 'type', 'updatedAt'].includes(k)).sort();
+
+  const resourceRows = resources.map((r) => `
+        <tr>
+          <td class="mono">${esc(r.id)}</td>
+          ${fieldNames.map((f) => `<td>${fieldCell((r as Record<string, unknown>)[f])}</td>`).join('')}
+          <td class="mono muted">${esc(r.updatedAt)}</td>
+        </tr>`).join('');
+
+  const timelineRows = events.slice(-25).reverse().map((e) => `
+        <tr>
+          <td class="mono muted">${esc(e.occurredAt)}</td>
+          <td class="mono">${esc(e.type)}</td>
+          <td class="mono">${esc(e.subject.type)}:${esc(e.subject.id)}</td>
+          <td class="mono muted">${esc(e.origin ?? '')}</td>
+        </tr>`).join('');
+
+  const ledgerRows = ledger.map((entry) => {
+    const ok = entry.results.some((r) => r.data?.status === 'success');
+    const op = entry.intent.data && typeof entry.intent.data === 'object' ? (entry.intent.data as Record<string, unknown>).operation : '';
+    return `
+        <tr>
+          <td class="mono">${esc(entry.intent.subject.type)}:${esc(entry.intent.subject.id)}</td>
+          <td class="mono">${esc(op || entry.intent.type)}</td>
+          <td>${entry.results.length}</td>
+          <td>${ok ? '<span class="ok">success</span>' : '<span class="warn">unreconciled</span>'}</td>
+        </tr>`;
+  }).join('');
+
+  const actionRows = actions.slice(-25).reverse().map((a) => `
+        <tr>
+          <td class="mono muted">${esc(a.occurredAt)}</td>
+          <td class="mono">${esc(a.op)}</td>
+          <td class="mono">${esc(a.subject.type)}:${esc(a.subject.id)}</td>
+          <td class="mono muted">${esc(a.fields ? Object.keys(a.fields).join(',') : (a.revertsActionId ?? a.confirmsActionId ?? ''))}</td>
+        </tr>`).join('');
+
+  const forkSection = fork ? `
+      <section>
+        <h2>Fork divergence <span class="muted">(${fork.changed.length} changed · ${fork.created.length} created)</span></h2>
+        ${fork.changed.length + fork.created.length === 0 ? '<p class="muted">No divergence from base.</p>' : ''}
+        ${fork.changed.map((c) => `<div class="diff"><span class="mono">${esc(c.id)}</span>: ${Object.entries(c.changed).map(([f, d]) => `${esc(f)} <span class="muted">${esc(JSON.stringify(d.before))}</span> → <span class="ok">${esc(JSON.stringify(d.after))}</span>`).join(', ')}</div>`).join('')}
+        ${fork.created.map((c) => `<div class="diff"><span class="ok">+ ${esc(c.id)}</span> (created in fork)</div>`).join('')}
+      </section>` : '';
+
+  return `<!doctype html>
+<html lang="en"><head><meta charset="utf-8"><title>${esc(service)} twin</title>
+<style>
+  :root { color-scheme: light dark; }
+  body { font: 14px/1.5 -apple-system, system-ui, sans-serif; margin: 0; padding: 2rem; max-width: 1100px; }
+  h1 { margin: 0 0 .25rem; } h2 { margin: 2rem 0 .5rem; font-size: 1.05rem; }
+  .badges span { display: inline-block; padding: .1rem .5rem; border-radius: 1rem; background: #8884; margin-right: .4rem; font-size: .8rem; }
+  table { border-collapse: collapse; width: 100%; }
+  th, td { text-align: left; padding: .35rem .6rem; border-bottom: 1px solid #8883; vertical-align: top; }
+  th { font-size: .75rem; text-transform: uppercase; letter-spacing: .03em; opacity: .7; }
+  .mono { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; font-size: .85em; }
+  .muted { opacity: .55; } .ok { color: #1a7f37; } .warn { color: #b35900; }
+  .tag { background: #8884; border-radius: .4rem; padding: 0 .35rem; font-size: .8em; }
+  .diff { padding: .2rem 0; font-size: .9em; }
+  section { margin-bottom: .5rem; }
+</style></head>
+<body>
+  <h1>${esc(service)} twin</h1>
+  <div class="badges">
+    <span>twin: ${esc(service)}</span>
+    <span>${resources.length} resources</span>
+    <span>${events.length} events</span>
+    <span>${ledger.length} egress</span>
+    ${fork ? '<span>fork</span>' : ''}
+  </div>
+  ${forkSection}
+  <section>
+    <h2>Resources</h2>
+    <table><thead><tr><th>id</th>${fieldNames.map((f) => `<th>${esc(f)}</th>`).join('')}<th>updatedAt</th></tr></thead>
+    <tbody>${resourceRows || '<tr><td class="muted">no resources</td></tr>'}</tbody></table>
+  </section>
+  <section>
+    <h2>Local actions <span class="muted">(${actions.length} — simulator/fork writes, not yet pushed)</span></h2>
+    <table><thead><tr><th>occurredAt</th><th>op</th><th>subject</th><th>fields</th></tr></thead>
+    <tbody>${actionRows || '<tr><td class="muted">no local actions</td></tr>'}</tbody></table>
+  </section>
+  <section>
+    <h2>Egress ledger <span class="muted">(pushes to the real vendor)</span></h2>
+    <table><thead><tr><th>subject</th><th>operation</th><th>results</th><th>status</th></tr></thead>
+    <tbody>${ledgerRows || '<tr><td class="muted">no egress</td></tr>'}</tbody></table>
+  </section>
+  <section>
+    <h2>Observed event timeline <span class="muted">(latest 25 of ${events.length})</span></h2>
+    <table><thead><tr><th>occurredAt</th><th>type</th><th>subject</th><th>origin</th></tr></thead>
+    <tbody>${timelineRows || '<tr><td class="muted">no events</td></tr>'}</tbody></table>
+  </section>
+</body></html>`;
+}
+
+/** Serve the visualizer over HTTP (re-rendered per request, so it tracks state). */
+export function createVisualizerServer(options: { service: string; root?: string; port?: number }): { port: number; stop: () => void } {
+  const server = Bun.serve({
+    port: options.port ?? 0,
+    idleTimeout: 60,
+    fetch() {
+      const html = renderTwinHtml(options.service, { ...(options.root !== undefined ? { root: options.root } : {}) });
+      return new Response(html, { headers: { 'content-type': 'text/html; charset=utf-8' } });
+    },
+  });
+  return { port: server.port ?? options.port ?? 0, stop: () => server.stop(true) };
+}

package/src/worldConfig.ts

@@ -0,0 +1,26 @@
+// World service config — generic per-service settings for the twin runtime
+// (`.volter/world/config.json`). Kept deliberately small + open: the index
+// signature lets a service carry arbitrary extra config that downstream tools
+// define. (The tracker's annotation adapter defines + reads `annotationPolicy` and
+// `browseUrlTemplate` on top of this — those are verification concerns, not the
+// twin's, so they are NOT typed here.)
+import { existsSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { worldStateRoot } from './storage.ts';
+
+export type WorldServiceConfig = {
+  provider?: string;
+  tool?: string;
+  pollLimit?: number;
+} & Record<string, unknown>;
+
+export type WorldConfig = {
+  services: Record<string, WorldServiceConfig>;
+} & Record<string, unknown>;
+
+export function loadWorldConfig(root?: string): WorldConfig {
+  const path = join(worldStateRoot(root), 'config.json');
+  if (!existsSync(path)) return { services: {} };
+  const parsed = JSON.parse(readFileSync(path, 'utf8')) as Partial<WorldConfig>;
+  return { ...parsed, services: parsed.services ?? {} };
+}