@volter/twin 0.1.0

package/dist/src/cli.js

@@ -0,0 +1,425 @@
+#!/usr/bin/env bun
+import { readFileSync } from 'node:fs';
+import { hasFlag, optionValue } from "./args.js";
+import { appendEvent, listEgressLedger, listEvents, listUnreconciledWriteIntents, loadState, performExternalWrite, rebuildGenericState, validateWorld, worldPaths, createTwinServer, resolveTwinRead, forkTwin, forkDivergence, auditForkNoRealWrites, createVisualizerServer, renderTwinHtml, scrubService, scrubWorld, syncPull, reconcile, worldStatus, formatStatus, listRemoteRefs, pendingActions, pendingConflicts, createTwinProxy, getPack, } from "./index.js";
+import { readForkMeta } from "./fork.js";
+import { twinResources } from "./serve.js";
+function jsonField(stdout, path) {
+    let parsed;
+    try {
+        parsed = JSON.parse(stdout);
+    }
+    catch {
+        throw new Error('world egress write: command output is not JSON; omit --id-field to use raw stdout');
+    }
+    // Comma-separated paths compose an id from multiple fields, joined with ':'
+    // (e.g. --id-field channel,ts → "C123:1718000.42").
+    const parts = path.split(',').map((segmentPath) => {
+        let value = parsed;
+        for (const segment of segmentPath.trim().split('.')) {
+            if (!value || typeof value !== 'object' || Array.isArray(value))
+                return '';
+            value = value[segment];
+        }
+        return typeof value === 'string' || typeof value === 'number' ? String(value) : '';
+    });
+    return parts.every(Boolean) ? parts.join(':') : '';
+}
+function readJsonArg(args) {
+    const file = optionValue(args, '--file');
+    if (file)
+        return JSON.parse(readFileSync(file, 'utf8'));
+    const json = optionValue(args, '--json');
+    if (json)
+        return JSON.parse(json);
+    throw new Error('Expected --file <path> or --json <json>');
+}
+function printHelp() {
+    process.stdout.write(`Usage: world <resource> <action> [args...]
+
+Resources:
+  events       append, list
+  state        get, rebuild
+  paths        show
+  egress       write, ledger, unreconciled
+  validate     (egress reconciliation; annotations are a tracker concern now)
+  status       <service>   plan <service>   refs <service>
+  scrub        <service> | --all   [--force] [--root <path>]   delete pulled data at rest
+
+Examples:
+  world events append --file /tmp/event.json
+  world events list chat --json
+  world state rebuild chat
+  world validate [--root <path>] [--services chat,github]
+  world egress write chat --operation message.send --provider slack \\
+    --subject-type channel --subject-id dev --key case:ENG-1:approval-request \\
+    --id-field id -- bash tools/external/slack/slack message send dev --user otto --text "..." --json
+  world egress unreconciled chat
+  world scrub chat                  # delete that service pulled event log + state (refuses on foreign contents)
+  world scrub --all --force         # delete the whole world state dir, even if it looks unfamiliar
+  See docs/DATA_AT_REST.md for the full data-at-rest story (redaction, retention, purge).
+`);
+}
+function print(value, asJson) {
+    if (asJson)
+        process.stdout.write(`${JSON.stringify(value, null, 2)}\n`);
+    else
+        process.stdout.write(`${JSON.stringify(value)}\n`);
+}
+async function main() {
+    const args = process.argv.slice(2);
+    if (args.length === 0 || args[0] === '--help' || args[0] === '-h') {
+        printHelp();
+        return;
+    }
+    const [resource, action, ...rest] = args;
+    const asJson = hasFlag(rest, '--json');
+    if (resource === 'events' && action === 'append') {
+        const event = readJsonArg(rest);
+        print(appendEvent(event), asJson);
+        return;
+    }
+    if (resource === 'events' && action === 'list') {
+        const service = rest[0];
+        if (!service)
+            throw new Error('world events list: missing service');
+        const events = listEvents(service);
+        if (asJson)
+            print(events, true);
+        else {
+            for (const event of events) {
+                process.stdout.write(`${event.id}\t${event.type}\t${event.subject.type}:${event.subject.id}\n`);
+            }
+        }
+        return;
+    }
+    if (resource === 'state' && action === 'get') {
+        const service = rest[0];
+        if (!service)
+            throw new Error('world state get: missing service');
+        print(loadState(service), true);
+        return;
+    }
+    if (resource === 'state' && action === 'rebuild') {
+        const service = rest[0];
+        if (!service)
+            throw new Error('world state rebuild: missing service');
+        print(rebuildGenericState(service), true);
+        return;
+    }
+    // Operator surface (R20): world status / plan / refs.
+    if (resource === 'status') {
+        const service = action;
+        if (!service)
+            throw new Error('world status: missing service (e.g. `world status linear`)');
+        const provider = optionValue(rest, '--provider') || undefined;
+        const forkId = optionValue(rest, '--fork') || undefined;
+        const status = worldStatus(service, { ...(provider ? { provider } : {}), ...(forkId ? { forkId } : {}) });
+        if (asJson)
+            print(status, true);
+        else
+            process.stdout.write(`${formatStatus(status)}\n`);
+        return;
+    }
+    if (resource === 'plan') {
+        // Kernel-level (transaction) plan: the pending local commits + any conflicts.
+        // The provider-call mapping is a vendor-pack concern (see buildApplyPlan).
+        const service = action;
+        if (!service)
+            throw new Error('world plan: missing service');
+        const transactions = pendingActions(service).map((a) => ({ id: a.id, operation: a.operation, subject: a.subject }));
+        const conflicts = pendingConflicts(service);
+        print({ service, transactions, conflicts, requiresApproval: conflicts.length > 0 }, true);
+        return;
+    }
+    if (resource === 'refs') {
+        const service = action;
+        if (!service)
+            throw new Error('world refs: missing service');
+        print(listRemoteRefs(service), true);
+        return;
+    }
+    // State INSPECTION (not the vendor API): a generic read server over a twin's mirrored
+    // state — `volter-twin inspect <service>` answers generic `/<type>/<id>` reads for
+    // scripting/debugging, and `--once <path>` resolves a single read and prints it.
+    // This is NOT the vendor SDK surface — point your real SDK at `world-<vendor> serve`.
+    if (resource === 'inspect') {
+        const service = action; // `volter-twin inspect <service> ...`
+        if (!service)
+            throw new Error('volter-twin inspect: missing service (e.g. `volter-twin inspect linear`). NOTE: this serves generic state reads, not the vendor API — use `world-<vendor> serve` for that.');
+        const readOnly = hasFlag(rest, '--read-only'); // a twin accepts writes unless started read-only
+        const port = Number(optionValue(rest, '--port', '0')) || undefined;
+        const once = optionValue(rest, '--once');
+        if (once) {
+            print(resolveTwinRead(service, once), true);
+            return;
+        }
+        const server = createTwinServer({ service, readOnly, ...(port ? { port } : {}) });
+        process.stdout.write(`state inspection for ${service}${readOnly ? ' (read-only)' : ''} at http://127.0.0.1:${server.port} (generic reads — NOT the vendor API)\n`);
+        await new Promise(() => { }); // serve until killed
+    }
+    // Browser zero-edit injection: a dev proxy in front of your app that forwards
+    // the browser's vendor SDK calls to a twin (so browser + backend share one twin).
+    // The proxy needs each vendor's browser routing (api path + loader host). It reads that
+    // from the pack's TwinPack.browserRouting when the pack is registered; otherwise pass it
+    // explicitly (the kernel itself knows nothing vendor-specific):
+    //   volter-twin proxy --target http://localhost:3000 \
+    //     --map stripe=http://127.0.0.1:12111 --route stripe=/v1/ --loader-host stripe=https://api.stripe.com
+    if (resource === 'proxy') {
+        const target = optionValue(rest, '--target');
+        if (!target)
+            throw new Error("world proxy: --target <app origin> is required (e.g. --target http://localhost:3000)");
+        // Collect repeated --map / --route / --loader-host vendor=value flags.
+        const collect = (flag) => {
+            const out = {};
+            for (let i = 0; i < rest.length; i += 1) {
+                if (rest[i] === flag && rest[i + 1]) {
+                    const eq = rest[i + 1].indexOf('=');
+                    if (eq > 0)
+                        out[rest[i + 1].slice(0, eq).trim()] = rest[i + 1].slice(eq + 1).trim();
+                }
+            }
+            return out;
+        };
+        const origins = collect('--map');
+        const routeFlags = collect('--route');
+        const loaderFlags = collect('--loader-host');
+        if (Object.keys(origins).length === 0)
+            throw new Error('world proxy: at least one --map <vendor>=<twin-url> is required');
+        const map = {};
+        for (const [vendor, origin] of Object.entries(origins)) {
+            const reg = getPack(vendor)?.browserRouting; // vendor knowledge lives in the pack, not the kernel
+            const apiPathPrefix = routeFlags[vendor] ?? reg?.apiPathPrefix;
+            const loaderHost = loaderFlags[vendor] ?? reg?.loaderHost;
+            if (!apiPathPrefix)
+                throw new Error(`world proxy: don't know ${vendor}'s browser API path. Pass --route ${vendor}=/<prefix> ` +
+                    `(and optionally --loader-host ${vendor}=https://<host>), or register the ${vendor} pack ` +
+                    `so its TwinPack.browserRouting is used.`);
+            map[vendor] = { origin, apiPathPrefix, ...(loaderHost ? { loaderHost } : {}) };
+        }
+        const port = Number(optionValue(rest, '--port', '0')) || undefined;
+        const proxy = createTwinProxy({ target, map, ...(port ? { port } : {}) });
+        const pairs = Object.entries(map).map(([v, r]) => `${v}(${r.apiPathPrefix})→${r.origin}`).join(', ');
+        process.stdout.write(`twin proxy on http://127.0.0.1:${proxy.port} → ${target} (forwarding ${pairs})\n`);
+        await new Promise(() => { }); // serve until killed
+    }
+    // NOTE: `world annotations` moved to the tracker — annotations are a tracker
+    // (verification) concern. Use @volter/tracker/world-annotations.
+    if (resource === 'egress' && action === 'write') {
+        const service = rest[0];
+        if (!service)
+            throw new Error('world egress write: missing service');
+        const splitIndex = rest.indexOf('--');
+        if (splitIndex < 0 || splitIndex === rest.length - 1) {
+            throw new Error('world egress write: provide the provider command after --');
+        }
+        const flags = rest.slice(0, splitIndex);
+        const command = rest.slice(splitIndex + 1);
+        const operation = optionValue(flags, '--operation');
+        const provider = optionValue(flags, '--provider');
+        const subjectType = optionValue(flags, '--subject-type');
+        const subjectId = optionValue(flags, '--subject-id');
+        const idempotencyKey = optionValue(flags, '--key');
+        if (!operation || !provider || !subjectType || !subjectId || !idempotencyKey) {
+            throw new Error('world egress write: --operation, --provider, --subject-type, --subject-id, and --key are required');
+        }
+        const idField = optionValue(flags, '--id-field');
+        // For operations whose provider output carries no id (e.g. reaction add),
+        // the caller supplies the deterministic external id up front.
+        const explicitExternalId = optionValue(flags, '--external-id');
+        const urlField = optionValue(flags, '--url-field');
+        const dataJson = optionValue(flags, '--data');
+        const onUnreconciled = optionValue(flags, '--unreconciled') === 'retry' ? 'retry' : 'fail';
+        // --passthrough makes the wrapper invisible to callers: the wrapped
+        // command's stdout is reproduced verbatim and the egress record goes to
+        // stderr. Replays reproduce the recorded stdout.
+        const passthrough = hasFlag(flags, '--passthrough');
+        // On a live write, passthrough must reproduce the wrapped command's
+        // stdout VERBATIM; the recorded copy is capped (replays of very large
+        // outputs come back truncated, which the egress note on stderr flags).
+        let liveStdout = null;
+        const result = await performExternalWrite({
+            service,
+            operation,
+            provider,
+            subject: { type: subjectType, id: subjectId },
+            idempotencyKey,
+            ...(dataJson ? { data: JSON.parse(dataJson) } : {}),
+        }, async () => {
+            const proc = Bun.spawnSync(command, { stdout: 'pipe', stderr: 'pipe' });
+            const stdout = proc.stdout.toString();
+            const stderr = proc.stderr.toString();
+            if (proc.exitCode !== 0) {
+                throw new Error(`${command[0]} exited with code ${proc.exitCode}: ${stderr.trim() || stdout.trim()}`);
+            }
+            const externalId = explicitExternalId
+                || (idField ? jsonField(stdout, idField) : stdout.trim().split('\n').at(-1)?.trim() ?? '');
+            if (!externalId)
+                throw new Error(`world egress write: could not extract external id from command output${idField ? ` (field ${idField})` : ''}`);
+            const url = urlField ? jsonField(stdout, urlField) : '';
+            liveStdout = stdout;
+            return {
+                externalId,
+                ...(url ? { url } : {}),
+                data: { stdout: stdout.trim().slice(0, 65536), stdoutTruncated: stdout.trim().length > 65536 },
+            };
+        }, { onUnreconciled });
+        if (passthrough) {
+            const stdout = liveStdout ?? (typeof result.outcome.data?.stdout === 'string' ? result.outcome.data.stdout : '');
+            if (stdout)
+                process.stdout.write(stdout.endsWith('\n') ? stdout : `${stdout}\n`);
+            process.stderr.write(`${JSON.stringify({ egress: { status: result.status, intentEventId: result.intentEventId, externalId: result.outcome.externalId, ...(result.outcome.data?.stdoutTruncated ? { replayStdoutTruncated: true } : {}) } })}\n`);
+        }
+        else {
+            print(result, true);
+        }
+        return;
+    }
+    if (resource === 'egress' && action === 'ledger') {
+        const service = rest[0];
+        if (!service)
+            throw new Error('world egress ledger: missing service');
+        print(listEgressLedger(service), true);
+        return;
+    }
+    if (resource === 'egress' && action === 'unreconciled') {
+        const service = rest[0];
+        if (!service)
+            throw new Error('world egress unreconciled: missing service');
+        const intents = listUnreconciledWriteIntents(service);
+        print(intents, true);
+        if (intents.length > 0)
+            process.exitCode = 2;
+        return;
+    }
+    if (resource === 'validate') {
+        const validateArgs = [action, ...rest].filter((arg) => typeof arg === 'string');
+        const rootOption = optionValue(validateArgs, '--root');
+        const servicesOption = optionValue(validateArgs, '--services');
+        const report = validateWorld({
+            ...(rootOption ? { root: rootOption } : {}),
+            ...(servicesOption ? { services: servicesOption.split(',').map((item) => item.trim()).filter(Boolean) } : {}),
+        });
+        print(report, true);
+        if (!report.valid)
+            process.exitCode = 1;
+        return;
+    }
+    if (resource === 'paths' && action === 'show') {
+        const service = rest[0];
+        if (!service)
+            throw new Error('world paths show: missing service');
+        print(worldPaths(service), true);
+        return;
+    }
+    // Scrub: delete pulled data at rest (TWIN-45 dev/02a). `world scrub <service>` removes just
+    // that service's event log/state; `world scrub --all` removes the whole world state dir.
+    // Both are plain `rm` (see docs/DATA_AT_REST.md) and both refuse unfamiliar contents unless
+    // --force is passed — never rm a path that doesn't look like our own state dir.
+    if (resource === 'scrub') {
+        const scrubArgs = [action, ...rest].filter((arg) => typeof arg === 'string');
+        const all = hasFlag(scrubArgs, '--all');
+        const force = hasFlag(scrubArgs, '--force');
+        const rootOption = optionValue(scrubArgs, '--root') || undefined;
+        if (all) {
+            print(scrubWorld({ ...(rootOption ? { root: rootOption } : {}), force }), true);
+            return;
+        }
+        const service = action;
+        if (!service || service === '--force' || service === '--root') {
+            throw new Error('world scrub: missing service (e.g. `world scrub chat`), or pass --all to scrub the whole world state dir');
+        }
+        print(scrubService(service, { ...(rootOption ? { root: rootOption } : {}), force }), true);
+        return;
+    }
+    // Sync: pull (real→twin) folds observed resources into the log; reconcile
+    // computes a three-way merge plan (base from fork-meta, fork from twin state,
+    // real from a provided snapshot). Push-to-real is library-only: it needs an
+    // injected real write fn carrying the user's own auth (auth-boundary), so it
+    // is intentionally NOT a CLI command.
+    if (resource === 'sync' && action === 'pull') {
+        const service = rest[0];
+        if (!service)
+            throw new Error('world sync pull: missing service');
+        const file = optionValue(rest, '--resources');
+        if (!file)
+            throw new Error('world sync pull: missing --resources <file.json> (array of {type,id,fields})');
+        const at = optionValue(rest, '--at') || new Date().toISOString();
+        const resources = JSON.parse(readFileSync(file, 'utf8'));
+        print(syncPull({ service, resources, occurredAt: at }), true);
+        return;
+    }
+    if (resource === 'reconcile') {
+        const service = action;
+        if (!service)
+            throw new Error('world reconcile: missing service');
+        const forkRoot = optionValue(rest, '--root');
+        const realFile = optionValue(rest, '--real');
+        if (!forkRoot || !realFile)
+            throw new Error('world reconcile <service> --root <forkRoot> --real <file> [--policy hub-wins|twin-wins|merge]');
+        const policy = (optionValue(rest, '--policy') || 'hub-wins');
+        const base = readForkMeta(service, forkRoot).baseline;
+        const fork = twinResources(service, forkRoot);
+        const realRaw = JSON.parse(readFileSync(realFile, 'utf8'));
+        const real = realRaw.map((r) => (r.fields ? { id: r.id, type: r.type, updatedAt: '', ...r.fields } : r));
+        print(reconcile({ policy, base, fork, real }), true);
+        return;
+    }
+    // Mirror UIs moved to the per-vendor twin packages (modular). Point users there.
+    if (resource === 'mirror') {
+        throw new Error(`world mirror: vendor mirror UIs live in their own packages now — run \`world-${action ?? '<vendor>'} mirror\` (e.g. world-stripe / world-linear / world-jira / world-github / world-slack).`);
+    }
+    // Visualizer (debug inspector): a generic status view of the twin. `world visualize <service>` serves it;
+    // `--once` prints the rendered HTML (scriptable).
+    if (resource === 'visualize') {
+        const service = action;
+        if (!service)
+            throw new Error('world visualize: missing service (e.g. `world visualize linear`)');
+        if (hasFlag(rest, '--once')) {
+            process.stdout.write(renderTwinHtml(service));
+            return;
+        }
+        const port = Number(optionValue(rest, '--port', '0')) || undefined;
+        const server = createVisualizerServer({ service, ...(port ? { port } : {}) });
+        process.stdout.write(`${service} twin visualizer at http://127.0.0.1:${server.port}\n`);
+        await new Promise(() => { });
+        return;
+    }
+    // Fork mode: `world fork create <service> --to <root> [--from <root>] --at <iso>`,
+    // `world fork divergence <service> --root <forkRoot>`,
+    // `world fork audit <service> --root <forkRoot>`.
+    if (resource === 'fork') {
+        const service = rest[0];
+        if (!service)
+            throw new Error('world fork: missing service');
+        if (action === 'create') {
+            const toRoot = optionValue(rest, '--to');
+            if (!toRoot)
+                throw new Error('world fork create: missing --to <root>');
+            const fromRoot = optionValue(rest, '--from') || undefined;
+            const at = optionValue(rest, '--at') || new Date().toISOString();
+            print(forkTwin({ service, toRoot, ...(fromRoot ? { fromRoot } : {}), occurredAt: at }), true);
+            return;
+        }
+        const forkRoot = optionValue(rest, '--root');
+        if (!forkRoot)
+            throw new Error(`world fork ${action}: missing --root <forkRoot>`);
+        if (action === 'divergence') {
+            print(forkDivergence(service, forkRoot), true);
+            return;
+        }
+        if (action === 'audit') {
+            const audit = auditForkNoRealWrites(service, forkRoot);
+            print(audit, true);
+            if (!audit.ok)
+                process.exitCode = 1;
+            return;
+        }
+        throw new Error(`Unknown world fork action: ${action ?? ''}`);
+    }
+    throw new Error(`Unknown world command: ${resource ?? ''} ${action ?? ''}`.trim());
+}
+main().catch((error) => {
+    process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
+    process.exit(1);
+});

package/dist/src/connector.d.ts

@@ -0,0 +1,106 @@
+import type { SubjectFields } from './shadow.js';
+import type { WorldServiceEvent } from './types.js';
+export type ConnectorObservation = {
+    subject: {
+        type: string;
+        id: string;
+    };
+    /** Current field values; the runner diffs them against the shadow state. */
+    observed: SubjectFields;
+    /** Provider update timestamp (ISO); drives cursor advancement. */
+    occurredAt: string;
+    external?: {
+        provider: string;
+        id: string;
+        url?: string;
+    };
+    /**
+     * Extra append-only events tied to this observation (comments, reviews,
+     * …). Appended idempotently — the connector encodes identity (including
+     * content hashes for mutable bodies) in id/idempotencyKey.
+     */
+    events?: WorldServiceEvent[];
+};
+export type WorldConnector = {
+    service: string;
+    /** Cursor filename under the service's cursors/ dir (default poll.json). */
+    cursorFile?: string;
+    /** Seed the shadow from historical event types (default: deltas only). */
+    shadowExtractor?: (event: WorldServiceEvent) => SubjectFields | null;
+    /**
+     * Fetch observations whose provider timestamp advanced past the cursor
+     * (empty cursor = from the beginning). `limit` is a runaway-fetch safety
+     * cap, not a page size: the connector reports `truncated: true` when the
+     * cap was hit so the runner holds the cursor.
+     */
+    fetchSince(input: {
+        cursor: string;
+        limit: number;
+        root: string;
+    }): Promise<{
+        observations: ConnectorObservation[];
+        truncated: boolean;
+    }>;
+};
+export type ConnectorPollResult = {
+    service: string;
+    cursorBefore: string | null;
+    cursorAfter: string | null;
+    truncated: boolean;
+    fetched: number;
+    deltasAppended: number;
+    eventsAppended: number;
+    changedSubjects: string[];
+};
+export declare function pollCursorPath(service: string, root?: string, cursorFile?: string): string;
+export declare function loadPollCursor(service: string, root?: string, cursorFile?: string): string;
+export declare function savePollCursor(service: string, after: string, root?: string, cursorFile?: string): void;
+export declare function runConnectorPoll(connector: WorldConnector, options?: {
+    root?: string;
+    cursor?: string;
+    limit?: number;
+}): Promise<ConnectorPollResult>;
+import type { ShadowState } from './shadow.js';
+export type SweepConnector = {
+    service: string;
+    shadowExtractor?: (event: WorldServiceEvent) => SubjectFields | null;
+    /** Which tracked subjects to re-observe this sweep. */
+    selectSubjects(input: {
+        subjects: Array<{
+            type: string;
+            id: string;
+        }>;
+        shadow: ShadowState;
+        sweepAll: boolean;
+    }): Array<{
+        type: string;
+        id: string;
+    }>;
+    /** Fetch the subject's current field state; null = fetch error. */
+    fetchSubject(subject: {
+        type: string;
+        id: string;
+    }): Promise<{
+        observed: SubjectFields;
+        occurredAt: string;
+        external?: {
+            provider: string;
+            id: string;
+            url?: string;
+        };
+    } | null>;
+};
+export type ConnectorSweepResult = {
+    service: string;
+    subjects: number;
+    polled: number;
+    deltasAppended: number;
+    errors: number;
+    changedSubjects: string[];
+    sweepAll: boolean;
+};
+export declare function runConnectorSweep(connector: SweepConnector, options?: {
+    root?: string;
+    limit?: number;
+    sweepAll?: boolean;
+}): Promise<ConnectorSweepResult>;

package/dist/src/connector.js

@@ -0,0 +1,129 @@
+// Connector contract + poll runner for cursor-based world pollers.
+//
+// A connector owns only the provider-specific parts: fetching observations
+// whose provider timestamp advanced past the cursor, mapping them to subject
+// fields, and constructing any extra append-only events (comments, …). The
+// runner owns the invariants every poller must respect:
+//  - shadow-diff recording: an unchanged re-observation appends nothing
+//  - extra events append idempotently (id/idempotencyKey carry identity)
+//  - the cursor only advances after every observation in the batch is
+//    recorded, and NEVER advances on a truncated batch — results arrive
+//    newest-first, so a capped batch cut off OLDER updates and advancing
+//    would skip them forever
+//  - the cursor saves 1ms behind the max observed timestamp so an update
+//    landing at exactly that timestamp after the poll is not excluded by a
+//    strictly-greater filter; the overlap re-fetch is free (unchanged
+//    observations append nothing)
+import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { buildShadowState, recordObservedDelta } from "./shadow.js";
+import { appendEvent, readJsonFile, rebuildGenericState, worldPaths } from "./storage.js";
+export function pollCursorPath(service, root, cursorFile = 'poll.json') {
+    return join(worldPaths(service, root).cursors, cursorFile);
+}
+export function loadPollCursor(service, root, cursorFile) {
+    const path = pollCursorPath(service, root, cursorFile);
+    if (!existsSync(path))
+        return '';
+    return readJsonFile(path).after ?? '';
+}
+export function savePollCursor(service, after, root, cursorFile) {
+    const path = pollCursorPath(service, root, cursorFile);
+    mkdirSync(dirname(path), { recursive: true });
+    writeFileSync(path, `${JSON.stringify({ after, updatedAt: new Date().toISOString() }, null, 2)}\n`);
+}
+export async function runConnectorPoll(connector, options = {}) {
+    const root = options.root;
+    const service = connector.service;
+    const limit = options.limit ?? 10000;
+    const cursorBefore = options.cursor ?? loadPollCursor(service, root, connector.cursorFile);
+    const { observations, truncated } = await connector.fetchSince({ cursor: cursorBefore, limit, root: root ?? process.env.PROJECT_ROOT ?? process.cwd() });
+    // Shadow folds from the event log; subject state arrives via deltas, so no
+    // snapshot extractor is needed.
+    const shadow = buildShadowState(service, connector.shadowExtractor ?? (() => null), root);
+    let deltasAppended = 0;
+    let eventsAppended = 0;
+    let maxOccurredAt = cursorBefore;
+    const changedSubjects = [];
+    for (const observation of observations) {
+        const result = recordObservedDelta(shadow, {
+            service,
+            subject: observation.subject,
+            observed: observation.observed,
+            occurredAt: observation.occurredAt,
+            ...(observation.external ? { external: observation.external } : {}),
+        }, root);
+        if (result.changed && result.append.appended) {
+            deltasAppended += 1;
+            changedSubjects.push(observation.subject.id);
+        }
+        for (const event of observation.events ?? []) {
+            if (appendEvent(event, root).appended)
+                eventsAppended += 1;
+        }
+        if (!maxOccurredAt || observation.occurredAt > maxOccurredAt)
+            maxOccurredAt = observation.occurredAt;
+    }
+    if (!truncated && maxOccurredAt && maxOccurredAt !== cursorBefore) {
+        savePollCursor(service, new Date(Date.parse(maxOccurredAt) - 1).toISOString(), root, connector.cursorFile);
+    }
+    rebuildGenericState(service, root);
+    return {
+        service,
+        cursorBefore: cursorBefore || null,
+        cursorAfter: truncated ? cursorBefore || null : (maxOccurredAt || null),
+        truncated,
+        fetched: observations.length,
+        deltasAppended,
+        eventsAppended,
+        changedSubjects,
+    };
+}
+// ---------------------------------------------------------------------------
+// Sweep mode: for providers polled by SUBJECT rather than by time cursor
+// (e.g. re-observing the current state of every tracked pull request). The
+// runner owns subject enumeration from the generic world state, the
+// shadow-diff invariant (unchanged re-observations append nothing), and
+// error accounting; the connector owns selection and the per-subject fetch
+// (null = fetch error, counted and reported, never silently dropped).
+import { loadState } from "./storage.js";
+export async function runConnectorSweep(connector, options = {}) {
+    const root = options.root;
+    const service = connector.service;
+    const limit = options.limit ?? 200;
+    const sweepAll = options.sweepAll ?? false;
+    const generic = loadState(service, root);
+    const allSubjects = Object.values(generic?.subjects ?? {}).map((subject) => {
+        const record = subject;
+        return { type: String(record.type ?? ''), id: String(record.id ?? '') };
+    });
+    const shadow = buildShadowState(service, connector.shadowExtractor ?? (() => null), root);
+    const selected = connector.selectSubjects({ subjects: allSubjects, shadow, sweepAll });
+    let polled = 0;
+    let deltasAppended = 0;
+    let errors = 0;
+    const changedSubjects = [];
+    for (const subject of selected) {
+        if (polled >= limit)
+            break;
+        const observation = await connector.fetchSubject(subject);
+        polled += 1;
+        if (!observation) {
+            errors += 1;
+            continue;
+        }
+        const result = recordObservedDelta(shadow, {
+            service,
+            subject,
+            observed: observation.observed,
+            occurredAt: observation.occurredAt,
+            ...(observation.external ? { external: observation.external } : {}),
+        }, root);
+        if (result.changed && result.append.appended) {
+            deltasAppended += 1;
+            changedSubjects.push(subject.id);
+        }
+    }
+    rebuildGenericState(service, root);
+    return { service, subjects: allSubjects.length, polled, deltasAppended, errors, changedSubjects, sweepAll };
+}