@volter/twin 0.1.0

package/dist/src/shadow.d.ts

@@ -0,0 +1,77 @@
+import type { AppendEventResult, WorldServiceEvent } from './types.js';
+export declare const DELTA_TYPE_SUFFIX = ".delta";
+export type SubjectFields = Record<string, unknown>;
+export type SubjectShadow = {
+    subject: {
+        type: string;
+        id: string;
+    };
+    fields: SubjectFields;
+    fieldHashes: Record<string, string>;
+    latestEventId: string;
+    updatedAt: string;
+};
+export type ShadowState = {
+    version: 1;
+    service: string;
+    rebuiltAt: string;
+    subjects: Record<string, SubjectShadow>;
+};
+/**
+ * Maps a world event to the subject fields it observes, or null when the
+ * event says nothing about remote subject state (comments, egress records, …).
+ * Extractors are provider-specific; the shadow engine is not.
+ */
+export type SubjectFieldExtractor = (event: WorldServiceEvent) => SubjectFields | null;
+export type FieldChange = {
+    before: unknown;
+    after: unknown;
+};
+export declare function hashFieldValue(value: unknown): string;
+/**
+ * Fold the service event log into per-subject materialized remote state.
+ * Delta events apply their `after` values directly; every other event goes
+ * through the extractor. The log is the source of truth — the shadow is
+ * always reproducible from a replay.
+ */
+export declare function buildShadowState(service: string, extractor: SubjectFieldExtractor, root?: string): ShadowState;
+/** Field-level diff of freshly observed subject fields against the shadow. */
+export declare function diffSubjectFields(shadow: SubjectShadow | undefined, observed: SubjectFields): Record<string, FieldChange>;
+export type DeltaObservation = {
+    service: string;
+    subject: {
+        type: string;
+        id: string;
+    };
+    /** Freshly fetched provider-side fields for the subject. */
+    observed: SubjectFields;
+    /** Provider timestamp of the observed state when available. */
+    occurredAt?: string;
+    external?: {
+        provider: string;
+        id?: string;
+        url?: string;
+        cursor?: string;
+    };
+    actor?: {
+        kind: 'human' | 'agent' | 'bot' | 'system';
+        id?: string;
+        name?: string;
+    };
+};
+export type DeltaResult = {
+    changed: false;
+} | {
+    changed: true;
+    event: WorldServiceEvent;
+    changes: Record<string, FieldChange>;
+    append: AppendEventResult;
+};
+/**
+ * Compare an observed provider snapshot against the shadow and append a
+ * single delta event carrying only the changed fields. An unchanged
+ * re-observation appends nothing — this is the fix for re-snapshot noise.
+ * The delta idempotencyKey is derived from the after-state hashes, so the
+ * same observed change never appends twice.
+ */
+export declare function recordObservedDelta(state: ShadowState, observation: DeltaObservation, root?: string): DeltaResult;

package/dist/src/shadow.js

@@ -0,0 +1,138 @@
+import { createHash } from 'node:crypto';
+import { isEgressEventType } from "./egress.js";
+import { appendEvent, createEvent, listEvents } from "./storage.js";
+export const DELTA_TYPE_SUFFIX = '.delta';
+function canonicalJson(value) {
+    if (Array.isArray(value))
+        return `[${value.map(canonicalJson).join(',')}]`;
+    if (value && typeof value === 'object') {
+        const entries = Object.entries(value)
+            .filter(([, item]) => item !== undefined)
+            .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))
+            .map(([key, item]) => `${JSON.stringify(key)}:${canonicalJson(item)}`);
+        return `{${entries.join(',')}}`;
+    }
+    return JSON.stringify(value) ?? 'null';
+}
+export function hashFieldValue(value) {
+    return createHash('sha256').update(canonicalJson(value)).digest('hex').slice(0, 16);
+}
+function subjectKey(subject) {
+    return `${subject.type}:${subject.id}`;
+}
+function isDeltaEvent(event) {
+    return event.type.endsWith(DELTA_TYPE_SUFFIX) && typeof event.data.changed === 'object';
+}
+function deltaAfterFields(event) {
+    const changed = event.data.changed;
+    const fields = {};
+    for (const [field, change] of Object.entries(changed))
+        fields[field] = change.after;
+    return fields;
+}
+function applyFields(state, event, fields) {
+    const key = subjectKey(event.subject);
+    const existing = state.subjects[key];
+    // A late-arriving observation of OLDER provider state must not regress the
+    // shadow. Comparable occurredAt timestamps win over log append order.
+    if (existing) {
+        const incoming = Date.parse(event.occurredAt);
+        const current = Date.parse(existing.updatedAt);
+        if (Number.isFinite(incoming) && Number.isFinite(current) && incoming < current)
+            return;
+    }
+    const target = existing ?? {
+        subject: { ...event.subject },
+        fields: {},
+        fieldHashes: {},
+        latestEventId: event.id,
+        updatedAt: event.occurredAt,
+    };
+    for (const [field, value] of Object.entries(fields)) {
+        if (value === undefined)
+            continue;
+        target.fields[field] = value;
+        target.fieldHashes[field] = hashFieldValue(value);
+    }
+    target.latestEventId = event.id;
+    target.updatedAt = event.occurredAt || target.updatedAt;
+    state.subjects[key] = target;
+}
+/**
+ * Fold the service event log into per-subject materialized remote state.
+ * Delta events apply their `after` values directly; every other event goes
+ * through the extractor. The log is the source of truth — the shadow is
+ * always reproducible from a replay.
+ */
+export function buildShadowState(service, extractor, root) {
+    const state = { version: 1, service, rebuiltAt: new Date().toISOString(), subjects: {} };
+    for (const event of listEvents(service, root)) {
+        if (isEgressEventType(event.type))
+            continue;
+        if (isDeltaEvent(event)) {
+            applyFields(state, event, deltaAfterFields(event));
+            continue;
+        }
+        const fields = extractor(event);
+        if (fields)
+            applyFields(state, event, fields);
+    }
+    return state;
+}
+/** Field-level diff of freshly observed subject fields against the shadow. */
+export function diffSubjectFields(shadow, observed) {
+    const changed = {};
+    for (const [field, after] of Object.entries(observed)) {
+        if (after === undefined)
+            continue;
+        const beforeHash = shadow?.fieldHashes[field];
+        if (beforeHash !== undefined && beforeHash === hashFieldValue(after))
+            continue;
+        changed[field] = { before: shadow?.fields[field], after };
+    }
+    return changed;
+}
+/**
+ * Compare an observed provider snapshot against the shadow and append a
+ * single delta event carrying only the changed fields. An unchanged
+ * re-observation appends nothing — this is the fix for re-snapshot noise.
+ * The delta idempotencyKey is derived from the after-state hashes, so the
+ * same observed change never appends twice.
+ */
+export function recordObservedDelta(state, observation, root) {
+    const shadow = state.subjects[subjectKey(observation.subject)];
+    const changes = diffSubjectFields(shadow, observation.observed);
+    if (Object.keys(changes).length === 0)
+        return { changed: false };
+    // The hash covers the provider timestamp as well as the after-state:
+    // A→B→A→B transitions revisit an after-state, and an id derived from
+    // content alone would collide with the earlier delta and make appendEvent
+    // throw (same id, different occurredAt). Re-observing the SAME provider
+    // state still dedupes — occurredAt comes from the provider, not the poll.
+    const afterHash = createHash('sha256')
+        .update(observation.occurredAt ?? '')
+        .update('\u001f')
+        .update(canonicalJson(Object.fromEntries(Object.entries(changes).map(([field, change]) => [field, change.after]))))
+        .digest('hex')
+        .slice(0, 16);
+    const type = `${observation.service}.${observation.subject.type}${DELTA_TYPE_SUFFIX}`;
+    const event = createEvent({
+        id: `${type}:${observation.subject.id}:${afterHash}`,
+        service: observation.service,
+        type,
+        idempotencyKey: `${observation.service}:delta:${observation.subject.type}:${observation.subject.id}:${afterHash}`,
+        occurredAt: observation.occurredAt ?? new Date().toISOString(),
+        origin: 'connector',
+        ...(observation.actor ? { actor: observation.actor } : {}),
+        subject: observation.subject,
+        ...(observation.external ? { external: observation.external } : {}),
+        data: {
+            changedFields: Object.keys(changes),
+            changed: changes,
+        },
+    });
+    const append = appendEvent(event, root);
+    if (append.appended)
+        applyFields(state, append.event, deltaAfterFields(append.event));
+    return { changed: true, event: append.event, changes, append };
+}

package/dist/src/status.d.ts

@@ -0,0 +1,31 @@
+import type { QueueCounts } from './queueLifecycle.js';
+import type { WorldRemoteRef } from './refs.js';
+export type WorldStatus = {
+    service: string;
+    remote: WorldRemoteRef | null;
+    queue: QueueCounts;
+    local: {
+        unpushed: number;
+        reverted: number;
+    };
+    push: {
+        unconfirmed: number;
+    };
+    drift: {
+        created: number;
+        changed: number;
+    } | null;
+    conflicts: Array<{
+        actionId: string;
+        reason: string;
+    }>;
+    staleBase: boolean | null;
+};
+export declare function worldStatus(service: string, opts?: {
+    root?: string;
+    forkId?: string;
+    provider?: string;
+    refName?: string;
+}): WorldStatus;
+/** Render the status as the operator text block the doc shows. */
+export declare function formatStatus(s: WorldStatus): string;

package/dist/src/status.js

@@ -0,0 +1,42 @@
+// world status (the twins architecture notes → "Status"). The operator safety
+// dashboard: it makes the four state classes impossible to confuse — queued
+// deliveries, canonical observed events, local transaction commits, and push/apply
+// records — plus drift, conflicts, and stale bases. Pure read over the kernel
+// ledgers; deterministic.
+import { listActions, pendingActions } from "./actions.js";
+import { isFork, forkDivergence } from "./fork.js";
+import { pendingConflicts } from "./plan.js";
+import { unconfirmedPushes } from "./pushLedger.js";
+import { queueCounts } from "./queueLifecycle.js";
+import { isBaseStale, readLocalRef, readRemoteRef } from "./refs.js";
+export function worldStatus(service, opts = {}) {
+    const { root, forkId, provider, refName = 'main' } = opts;
+    const actions = listActions(service, root);
+    const remote = provider ? readRemoteRef(service, provider, refName, root) : null;
+    const local = forkId ? readLocalRef(service, forkId, root) : null;
+    const drift = isFork(service, root ?? '') ? (() => { const d = forkDivergence(service, root ?? ''); return { created: d.created.length, changed: d.changed.length }; })() : null;
+    return {
+        service,
+        remote,
+        queue: queueCounts(service, root),
+        local: { unpushed: pendingActions(service, root).length, reverted: actions.filter((a) => a.op === 'revert').length },
+        push: { unconfirmed: unconfirmedPushes(service, root).length },
+        drift,
+        conflicts: pendingConflicts(service, root),
+        staleBase: local ? isBaseStale(local, root) : null,
+    };
+}
+/** Render the status as the operator text block the doc shows. */
+export function formatStatus(s) {
+    const lines = [];
+    lines.push(s.remote ? `remote/${s.remote.provider}/${s.remote.name} at ${s.remote.cursor ?? s.remote.eventId ?? '(no checkpoint)'}` : 'remote: (no ref recorded)');
+    lines.push(`queue: ${s.queue.queued} queued, ${s.queue.committed} committed, ${s.queue.ignored} ignored, ${s.queue.superseded} superseded, ${s.queue.poisoned} poisoned`);
+    lines.push(`local: ${s.local.unpushed} unpushed transactions, ${s.local.reverted} reverted`);
+    lines.push(`push: ${s.push.unconfirmed} provider_accepted/attempted awaiting observed confirmation`);
+    if (s.drift)
+        lines.push(`drift: ${s.drift.created} created, ${s.drift.changed} changed since fork base`);
+    lines.push(`conflicts: ${s.conflicts.length}${s.conflicts.length ? ` (${s.conflicts.map((c) => c.actionId).join(', ')})` : ''}`);
+    if (s.staleBase !== null)
+        lines.push(`base: ${s.staleBase ? 'STALE — rebase before push' : 'current'}`);
+    return lines.join('\n');
+}

package/dist/src/storage.d.ts

@@ -0,0 +1,119 @@
+import type { AppendEventResult, CommitQueuedEventsResult, EnqueueEventResult, GenericWorldState, QueuedWorldServiceEvent, WorldPaths, WorldReducer, WorldServiceEvent } from './types.js';
+/**
+ * Name of the per-project state directory holding world data
+ * (`<root>/<stateDir>/world/...`). Defaults to `.volter`; hosts that need a
+ * different directory set VOLTER_STATE_DIR. Every path in this package must
+ * go through worldStateRoot — never the literal.
+ */
+export declare function stateDirName(): string;
+export declare function worldStateRoot(root?: string): string;
+export declare function worldPaths(service: string, root?: string): WorldPaths;
+/** Read + parse a whole-file JSON sidecar, citing the path on a parse failure (mirrors
+ *  readJsonl's `${path}:...` error shape). Callers own existence semantics — this parses a
+ *  file that is expected to exist; guard with existsSync first when absence is allowed. */
+export declare function readJsonFile<T>(path: string): T;
+/**
+ * Append `data` to `path`. When VOLTER_DURABLE=1, `fsyncSync` the file after the write so
+ * the appended record survives an OS crash / power loss — not just a process crash.
+ *
+ * Default (VOLTER_DURABLE unset) is OFF: `appendFileSync` is a completed write syscall, so a
+ * *process* crash after it returns still leaves the record on the log (the OS owns the page
+ * cache). The window this leaves open is a **kernel-panic / power loss** between the write
+ * landing in the page cache and the fs flushing it to stable storage — a torn or lost tail
+ * record. That crash window matters more once real pulled staging data lives in these files
+ * (purpose 2), so hosts that need durability opt in with VOLTER_DURABLE=1 at the cost of an
+ * fsync per append. See ARCHITECTURE.md D1.
+ */
+export declare function appendDurable(path: string, data: string): void;
+/**
+ * Opt-in structured stderr logging for the audit trail (ARCHITECTURE-REVIEW-TODOS D3):
+ * one line per action-log append and per push-ledger row, so "who reviewed the change
+ * that caused this real write" is mechanically greppable from a single log stream via
+ * `correlationId`. Off by default — mirrors the VOLTER_DURABLE opt-in policy above: no
+ * always-on I/O, never on the default path. Set VOLTER_TWIN_LOG=1 to enable.
+ */
+export declare function twinLog(kind: string, details: Record<string, unknown>): void;
+/** Run `fn` holding an exclusive cross-process file lock (the same primitive the event log
+ *  uses). Used to make read-then-append critical sections atomic across processes. The
+ *  lockfile records `{pid, hostname, at}`; a contender that finds a stale lock (dead pid or
+ *  age > LOCK_STALE_MS) reclaims it by atomically renaming it aside — so a crashed holder
+ *  can't wedge the world forever. Reclaim is race-safe: only the process that wins the
+ *  rename clears the stale inode, and the exclusive `wx` create still decides the winner. */
+export declare function withFileLock<T>(lockPath: string, fn: () => T): T;
+export declare function listEvents(service: string, root?: string): WorldServiceEvent[];
+/** Exported so other modules that share a service's events log (e.g. egress.ts,
+ *  TWIN-58) can serialize their own critical sections on the SAME lock appendEvent
+ *  itself uses, instead of inventing a second, uncoordinated lockfile. */
+export declare function eventsLockPath(paths: WorldPaths): string;
+/**
+ * The append body, assuming the caller already holds `eventsLockPath(paths)`. Split
+ * out so `commitQueuedEvents` can run a whole batch of appends (and the rebuild that
+ * follows them) under ONE lock acquisition instead of nesting a fresh `withFileLock`
+ * per row — `withFileLock` is not reentrant, so re-acquiring it from inside an
+ * already-held lock in the same process would just spin to its own timeout. Also
+ * exported for egress.ts (TWIN-58): performExternalWrite's ledger-check +
+ * intent-append span holds `eventsLockPath` itself, so it must append through this
+ * already-locked path rather than the public `appendEvent` (which would try to
+ * re-acquire the same lock and spin to its own timeout).
+ */
+export declare function appendEventLocked(parsed: WorldServiceEvent, paths: WorldPaths): AppendEventResult;
+export declare function appendEvent(event: WorldServiceEvent, root?: string): AppendEventResult;
+export declare function listQueuedEvents(service: string, root?: string): QueuedWorldServiceEvent[];
+export declare function enqueueEvent(event: WorldServiceEvent, options?: {
+    root?: string;
+    source?: QueuedWorldServiceEvent['source'];
+    receivedAt?: string;
+    id?: string;
+}): EnqueueEventResult;
+/**
+ * Commit queued (webhook/listener) events into the canonical log, then rebuild the
+ * generic projection — spanning BOTH under the SAME events-lock acquisition (D7).
+ * Appending each event and rebuilding state.json are two separate durable writes;
+ * without a shared lock, a concurrent commit/append on another process could
+ * interleave between "this commit's last append" and "this commit's rebuild",
+ * so the rebuilt state.json would not correspond to exactly this commit's view of
+ * the log. Holding one lock across the whole sequence rules that out: no other
+ * appendEvent/commitQueuedEvents call for this service can run until both the
+ * appends AND the rebuild here have completed. (A single-process crash between the
+ * last durable append and the rebuild can still leave state.json one rebuild behind
+ * — that window is inherent to a two-file update with no WAL, and is unaffected by
+ * locking; it is closed by the next successful commit/rebuild, which always starts
+ * from the durably-appended log, so no event is ever lost, only state.json's
+ * projection is briefly stale.)
+ */
+export declare function commitQueuedEvents(service: string, options?: {
+    root?: string;
+    limit?: number;
+}): CommitQueuedEventsResult;
+export declare function genericWorldReducer(state: GenericWorldState, event: WorldServiceEvent): GenericWorldState;
+export declare function emptyGenericState(service: string): GenericWorldState;
+export declare function rebuildState<State>(service: string, initialState: State, reducer: WorldReducer<State>, root?: string): State;
+export declare function rebuildGenericState(service: string, root?: string): GenericWorldState;
+export declare function loadState<T = unknown>(service: string, root?: string): T | null;
+export declare function createEvent(input: Omit<WorldServiceEvent, 'schemaVersion' | 'observedAt'> & {
+    schemaVersion?: number;
+    observedAt?: string;
+}): WorldServiceEvent;
+export type ScrubResult = {
+    /** The directory scrub targeted (may not have existed). */
+    target: string;
+    /** Every file actually removed, path relative to `target`, in no particular order. */
+    removed: string[];
+    /** One-line human summary, safe to print as-is. */
+    message: string;
+};
+/** Delete one service's pulled data at rest: its event log, queued events,
+ *  rebuilt state.json, and any resources/cursors/ingests sidecars — everything
+ *  `worldPaths(service)` points at. Refuses (unless `force`) when the dir holds
+ *  anything storage.ts didn't put there. */
+export declare function scrubService(service: string, options?: {
+    root?: string;
+    force?: boolean;
+}): ScrubResult;
+/** Delete the ENTIRE world state dir (`<root>/<stateDir>/world`) — every
+ *  service's pulled data at once. Refuses (unless `force`) when any entry in it
+ *  isn't itself a recognizable per-service state dir. */
+export declare function scrubWorld(options?: {
+    root?: string;
+    force?: boolean;
+}): ScrubResult;