@volter/twin 0.1.0 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/README.md +16 -2
  2. package/inject.cjs +453 -59
  3. package/package.json +12 -22
  4. package/src/actions.ts +234 -49
  5. package/src/blob-store.ts +136 -0
  6. package/src/changeset.ts +807 -0
  7. package/src/cli.ts +60 -10
  8. package/src/connector.ts +30 -7
  9. package/src/control-plane.ts +17 -1
  10. package/src/emit.ts +242 -0
  11. package/src/fork.ts +19 -7
  12. package/src/index.ts +139 -6
  13. package/src/lease.ts +4 -6
  14. package/src/lifecycle.ts +8 -0
  15. package/src/packRegistry.ts +248 -2
  16. package/src/plan.ts +131 -23
  17. package/src/proxy.ts +5 -2
  18. package/src/pushLedger.ts +116 -11
  19. package/src/queueLifecycle.ts +3 -4
  20. package/src/rateBudget.ts +1115 -0
  21. package/src/refs.ts +9 -10
  22. package/src/remote-execute.ts +16 -0
  23. package/src/scenario.ts +387 -0
  24. package/src/serve.ts +397 -15
  25. package/src/shadow.ts +86 -7
  26. package/src/storage.ts +76 -147
  27. package/src/sync.ts +63 -17
  28. package/src/twin-fetch.ts +115 -0
  29. package/src/validate.ts +6 -5
  30. package/src/world-clock.ts +33 -0
  31. package/src/world-store.ts +482 -0
  32. package/src/worldConfig.ts +4 -3
  33. package/dist/src/actions.d.ts +0 -138
  34. package/dist/src/actions.js +0 -201
  35. package/dist/src/args.d.ts +0 -3
  36. package/dist/src/args.js +0 -12
  37. package/dist/src/cli.d.ts +0 -2
  38. package/dist/src/cli.js +0 -425
  39. package/dist/src/connector.d.ts +0 -106
  40. package/dist/src/connector.js +0 -129
  41. package/dist/src/control-plane.d.ts +0 -21
  42. package/dist/src/control-plane.js +0 -40
  43. package/dist/src/egress.d.ts +0 -93
  44. package/dist/src/egress.js +0 -264
  45. package/dist/src/fork.d.ts +0 -126
  46. package/dist/src/fork.js +0 -206
  47. package/dist/src/index.d.ts +0 -42
  48. package/dist/src/index.js +0 -52
  49. package/dist/src/lease.d.ts +0 -50
  50. package/dist/src/lease.js +0 -80
  51. package/dist/src/packRegistry.d.ts +0 -34
  52. package/dist/src/packRegistry.js +0 -22
  53. package/dist/src/plan.d.ts +0 -97
  54. package/dist/src/plan.js +0 -151
  55. package/dist/src/proxy.d.ts +0 -25
  56. package/dist/src/proxy.js +0 -152
  57. package/dist/src/pushLedger.d.ts +0 -81
  58. package/dist/src/pushLedger.js +0 -130
  59. package/dist/src/queueLifecycle.d.ts +0 -62
  60. package/dist/src/queueLifecycle.js +0 -95
  61. package/dist/src/reconcile.d.ts +0 -58
  62. package/dist/src/reconcile.js +0 -137
  63. package/dist/src/refs.d.ts +0 -29
  64. package/dist/src/refs.js +0 -68
  65. package/dist/src/schemas.d.ts +0 -78
  66. package/dist/src/schemas.js +0 -50
  67. package/dist/src/serve.d.ts +0 -44
  68. package/dist/src/serve.js +0 -93
  69. package/dist/src/shadow.d.ts +0 -77
  70. package/dist/src/shadow.js +0 -138
  71. package/dist/src/status.d.ts +0 -31
  72. package/dist/src/status.js +0 -42
  73. package/dist/src/storage.d.ts +0 -119
  74. package/dist/src/storage.js +0 -535
  75. package/dist/src/sync.d.ts +0 -91
  76. package/dist/src/sync.js +0 -121
  77. package/dist/src/types.d.ts +0 -40
  78. package/dist/src/types.js +0 -1
  79. package/dist/src/validate.d.ts +0 -27
  80. package/dist/src/validate.js +0 -68
  81. package/dist/src/visualizer.d.ts +0 -13
  82. package/dist/src/visualizer.js +0 -133
  83. package/dist/src/worldConfig.d.ts +0 -9
  84. package/dist/src/worldConfig.js +0 -16
@@ -1,106 +0,0 @@
1
- import type { SubjectFields } from './shadow.js';
2
- import type { WorldServiceEvent } from './types.js';
3
- export type ConnectorObservation = {
4
- subject: {
5
- type: string;
6
- id: string;
7
- };
8
- /** Current field values; the runner diffs them against the shadow state. */
9
- observed: SubjectFields;
10
- /** Provider update timestamp (ISO); drives cursor advancement. */
11
- occurredAt: string;
12
- external?: {
13
- provider: string;
14
- id: string;
15
- url?: string;
16
- };
17
- /**
18
- * Extra append-only events tied to this observation (comments, reviews,
19
- * …). Appended idempotently — the connector encodes identity (including
20
- * content hashes for mutable bodies) in id/idempotencyKey.
21
- */
22
- events?: WorldServiceEvent[];
23
- };
24
- export type WorldConnector = {
25
- service: string;
26
- /** Cursor filename under the service's cursors/ dir (default poll.json). */
27
- cursorFile?: string;
28
- /** Seed the shadow from historical event types (default: deltas only). */
29
- shadowExtractor?: (event: WorldServiceEvent) => SubjectFields | null;
30
- /**
31
- * Fetch observations whose provider timestamp advanced past the cursor
32
- * (empty cursor = from the beginning). `limit` is a runaway-fetch safety
33
- * cap, not a page size: the connector reports `truncated: true` when the
34
- * cap was hit so the runner holds the cursor.
35
- */
36
- fetchSince(input: {
37
- cursor: string;
38
- limit: number;
39
- root: string;
40
- }): Promise<{
41
- observations: ConnectorObservation[];
42
- truncated: boolean;
43
- }>;
44
- };
45
- export type ConnectorPollResult = {
46
- service: string;
47
- cursorBefore: string | null;
48
- cursorAfter: string | null;
49
- truncated: boolean;
50
- fetched: number;
51
- deltasAppended: number;
52
- eventsAppended: number;
53
- changedSubjects: string[];
54
- };
55
- export declare function pollCursorPath(service: string, root?: string, cursorFile?: string): string;
56
- export declare function loadPollCursor(service: string, root?: string, cursorFile?: string): string;
57
- export declare function savePollCursor(service: string, after: string, root?: string, cursorFile?: string): void;
58
- export declare function runConnectorPoll(connector: WorldConnector, options?: {
59
- root?: string;
60
- cursor?: string;
61
- limit?: number;
62
- }): Promise<ConnectorPollResult>;
63
- import type { ShadowState } from './shadow.js';
64
- export type SweepConnector = {
65
- service: string;
66
- shadowExtractor?: (event: WorldServiceEvent) => SubjectFields | null;
67
- /** Which tracked subjects to re-observe this sweep. */
68
- selectSubjects(input: {
69
- subjects: Array<{
70
- type: string;
71
- id: string;
72
- }>;
73
- shadow: ShadowState;
74
- sweepAll: boolean;
75
- }): Array<{
76
- type: string;
77
- id: string;
78
- }>;
79
- /** Fetch the subject's current field state; null = fetch error. */
80
- fetchSubject(subject: {
81
- type: string;
82
- id: string;
83
- }): Promise<{
84
- observed: SubjectFields;
85
- occurredAt: string;
86
- external?: {
87
- provider: string;
88
- id: string;
89
- url?: string;
90
- };
91
- } | null>;
92
- };
93
- export type ConnectorSweepResult = {
94
- service: string;
95
- subjects: number;
96
- polled: number;
97
- deltasAppended: number;
98
- errors: number;
99
- changedSubjects: string[];
100
- sweepAll: boolean;
101
- };
102
- export declare function runConnectorSweep(connector: SweepConnector, options?: {
103
- root?: string;
104
- limit?: number;
105
- sweepAll?: boolean;
106
- }): Promise<ConnectorSweepResult>;
@@ -1,129 +0,0 @@
1
- // Connector contract + poll runner for cursor-based world pollers.
2
- //
3
- // A connector owns only the provider-specific parts: fetching observations
4
- // whose provider timestamp advanced past the cursor, mapping them to subject
5
- // fields, and constructing any extra append-only events (comments, …). The
6
- // runner owns the invariants every poller must respect:
7
- // - shadow-diff recording: an unchanged re-observation appends nothing
8
- // - extra events append idempotently (id/idempotencyKey carry identity)
9
- // - the cursor only advances after every observation in the batch is
10
- // recorded, and NEVER advances on a truncated batch — results arrive
11
- // newest-first, so a capped batch cut off OLDER updates and advancing
12
- // would skip them forever
13
- // - the cursor saves 1ms behind the max observed timestamp so an update
14
- // landing at exactly that timestamp after the poll is not excluded by a
15
- // strictly-greater filter; the overlap re-fetch is free (unchanged
16
- // observations append nothing)
17
- import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
18
- import { dirname, join } from 'node:path';
19
- import { buildShadowState, recordObservedDelta } from "./shadow.js";
20
- import { appendEvent, readJsonFile, rebuildGenericState, worldPaths } from "./storage.js";
21
- export function pollCursorPath(service, root, cursorFile = 'poll.json') {
22
- return join(worldPaths(service, root).cursors, cursorFile);
23
- }
24
- export function loadPollCursor(service, root, cursorFile) {
25
- const path = pollCursorPath(service, root, cursorFile);
26
- if (!existsSync(path))
27
- return '';
28
- return readJsonFile(path).after ?? '';
29
- }
30
- export function savePollCursor(service, after, root, cursorFile) {
31
- const path = pollCursorPath(service, root, cursorFile);
32
- mkdirSync(dirname(path), { recursive: true });
33
- writeFileSync(path, `${JSON.stringify({ after, updatedAt: new Date().toISOString() }, null, 2)}\n`);
34
- }
35
- export async function runConnectorPoll(connector, options = {}) {
36
- const root = options.root;
37
- const service = connector.service;
38
- const limit = options.limit ?? 10000;
39
- const cursorBefore = options.cursor ?? loadPollCursor(service, root, connector.cursorFile);
40
- const { observations, truncated } = await connector.fetchSince({ cursor: cursorBefore, limit, root: root ?? process.env.PROJECT_ROOT ?? process.cwd() });
41
- // Shadow folds from the event log; subject state arrives via deltas, so no
42
- // snapshot extractor is needed.
43
- const shadow = buildShadowState(service, connector.shadowExtractor ?? (() => null), root);
44
- let deltasAppended = 0;
45
- let eventsAppended = 0;
46
- let maxOccurredAt = cursorBefore;
47
- const changedSubjects = [];
48
- for (const observation of observations) {
49
- const result = recordObservedDelta(shadow, {
50
- service,
51
- subject: observation.subject,
52
- observed: observation.observed,
53
- occurredAt: observation.occurredAt,
54
- ...(observation.external ? { external: observation.external } : {}),
55
- }, root);
56
- if (result.changed && result.append.appended) {
57
- deltasAppended += 1;
58
- changedSubjects.push(observation.subject.id);
59
- }
60
- for (const event of observation.events ?? []) {
61
- if (appendEvent(event, root).appended)
62
- eventsAppended += 1;
63
- }
64
- if (!maxOccurredAt || observation.occurredAt > maxOccurredAt)
65
- maxOccurredAt = observation.occurredAt;
66
- }
67
- if (!truncated && maxOccurredAt && maxOccurredAt !== cursorBefore) {
68
- savePollCursor(service, new Date(Date.parse(maxOccurredAt) - 1).toISOString(), root, connector.cursorFile);
69
- }
70
- rebuildGenericState(service, root);
71
- return {
72
- service,
73
- cursorBefore: cursorBefore || null,
74
- cursorAfter: truncated ? cursorBefore || null : (maxOccurredAt || null),
75
- truncated,
76
- fetched: observations.length,
77
- deltasAppended,
78
- eventsAppended,
79
- changedSubjects,
80
- };
81
- }
82
- // ---------------------------------------------------------------------------
83
- // Sweep mode: for providers polled by SUBJECT rather than by time cursor
84
- // (e.g. re-observing the current state of every tracked pull request). The
85
- // runner owns subject enumeration from the generic world state, the
86
- // shadow-diff invariant (unchanged re-observations append nothing), and
87
- // error accounting; the connector owns selection and the per-subject fetch
88
- // (null = fetch error, counted and reported, never silently dropped).
89
- import { loadState } from "./storage.js";
90
- export async function runConnectorSweep(connector, options = {}) {
91
- const root = options.root;
92
- const service = connector.service;
93
- const limit = options.limit ?? 200;
94
- const sweepAll = options.sweepAll ?? false;
95
- const generic = loadState(service, root);
96
- const allSubjects = Object.values(generic?.subjects ?? {}).map((subject) => {
97
- const record = subject;
98
- return { type: String(record.type ?? ''), id: String(record.id ?? '') };
99
- });
100
- const shadow = buildShadowState(service, connector.shadowExtractor ?? (() => null), root);
101
- const selected = connector.selectSubjects({ subjects: allSubjects, shadow, sweepAll });
102
- let polled = 0;
103
- let deltasAppended = 0;
104
- let errors = 0;
105
- const changedSubjects = [];
106
- for (const subject of selected) {
107
- if (polled >= limit)
108
- break;
109
- const observation = await connector.fetchSubject(subject);
110
- polled += 1;
111
- if (!observation) {
112
- errors += 1;
113
- continue;
114
- }
115
- const result = recordObservedDelta(shadow, {
116
- service,
117
- subject,
118
- observed: observation.observed,
119
- occurredAt: observation.occurredAt,
120
- ...(observation.external ? { external: observation.external } : {}),
121
- }, root);
122
- if (result.changed && result.append.appended) {
123
- deltasAppended += 1;
124
- changedSubjects.push(subject.id);
125
- }
126
- }
127
- rebuildGenericState(service, root);
128
- return { service, subjects: allSubjects.length, polled, deltasAppended, errors, changedSubjects, sweepAll };
129
- }
@@ -1,21 +0,0 @@
1
- export { applyTwinWrite, createTwinServer, resolveTwinRead, twinResources } from './serve.js';
2
- export type { TwinResource, TwinWriteResult } from './serve.js';
3
- export { formatStatus, worldStatus } from './status.js';
4
- export type { WorldStatus } from './status.js';
5
- export { applyPlan, buildApplyPlan, listPlans, pendingConflicts, planRequiresApproval, readPlan, writePlan } from './plan.js';
6
- export type { ActionMapper, ApplyResult, ProviderCall, WorldApplyPlan } from './plan.js';
7
- export { abandonPush, appendPushRecord, latestPushByActionId, listPushLedger, pushTransaction, unconfirmedPushes, UnreconciledPushError } from './pushLedger.js';
8
- export type { PushOutcome, PushStatus, WorldPushRecord } from './pushLedger.js';
9
- export { acquireLease, activeLease, isLeaseActive, LeaseHeldError, listLeases, releaseLease } from './lease.js';
10
- export type { WorldApplyLease } from './lease.js';
11
- export { isBaseStale, listRemoteRefs, readLocalRef, readRemoteRef, writeLocalRef, writeRemoteRef } from './refs.js';
12
- export type { WorldLocalRef, WorldRemoteRef } from './refs.js';
13
- export { commitPendingQueue, commitQueueRow, ignoreQueueRow, listQueueWithStatus, pendingQueueRows, poisonQueueRow, queueCounts, queueRowStatus, setQueueRowStatus, supersedeQueueRow, } from './queueLifecycle.js';
14
- export type { QueueCounts, QueueRowStatus, QueueRowWithStatus, QueueStatusTransition } from './queueLifecycle.js';
15
- export { confirmAction } from './actions.js';
16
- export { auditForkNoRealWrites, cherryPickActions, discardFork, forkDivergence, forkTwin, isFork, mergeForks, rebaseFork, readForkMeta, resetFork, } from './fork.js';
17
- export type { CopyResult, ForkAudit, ForkDivergence, ForkMeta, SubjectDivergence } from './fork.js';
18
- export { isCleanlyReconcilable, reconcile, reconcileRequiresApproval } from './reconcile.js';
19
- export type { FieldDecision, ReconcilePlan, ReconcilePolicy, SubjectReconcile } from './reconcile.js';
20
- export { currentResources, syncPull, syncPush } from './sync.js';
21
- export type { PullResult, PushItemResult, PushResult, SyncResource } from './sync.js';
@@ -1,40 +0,0 @@
1
- // The CONTROL PLANE — the legitimately-shared thing across twins.
2
- //
3
- // This is the uniform contract + operator surface every twin is *driven* through:
4
- // the same way to serve/read/write, switch modes, see `status`, build a `plan`,
5
- // `push`/apply (with explicit phases), hold a `lease`, manage the event-queue
6
- // lifecycle, track remote refs, and run Git-like fork operations. An operator or
7
- // agent learns ONE way to drive any twin.
8
- //
9
- // What this is NOT: a "kernel" that makes every twin *implement its world the same
10
- // way*. Twins are implemented per-vendor (their exact API + query semantics). The
11
- // data-plane mechanics they reuse (event log, projection, shadow/delta, egress,
12
- // connectors — exported from the package root) are **optional libraries**, like a
13
- // UI library a visualizer may pick — not a mandate. See
14
- // `the twins architecture notes` → "share the control plane, not the
15
- // implementation".
16
- //
17
- // Barrel only: re-exports the control-plane surface as one cohesive thing. Consume
18
- // it as a namespace via `import { controlPlane } from '@volter/twin'`.
19
- // serve/read/write surface (how a twin is driven): one twin, readable always,
20
- // writable unless started read-only (writes land as local actions)
21
- export { applyTwinWrite, createTwinServer, resolveTwinRead, twinResources } from "./serve.js";
22
- // status
23
- export { formatStatus, worldStatus } from "./status.js";
24
- // plan + apply orchestration
25
- export { applyPlan, buildApplyPlan, listPlans, pendingConflicts, planRequiresApproval, readPlan, writePlan } from "./plan.js";
26
- // push ledger + phases
27
- export { abandonPush, appendPushRecord, latestPushByActionId, listPushLedger, pushTransaction, unconfirmedPushes, UnreconciledPushError } from "./pushLedger.js";
28
- // apply leases (single-writer per checkpoint)
29
- export { acquireLease, activeLease, isLeaseActive, LeaseHeldError, listLeases, releaseLease } from "./lease.js";
30
- // remote refs / checkpoints + stale-base guard
31
- export { isBaseStale, listRemoteRefs, readLocalRef, readRemoteRef, writeLocalRef, writeRemoteRef } from "./refs.js";
32
- // event-queue lifecycle (queued/committed/ignored/superseded/poisoned)
33
- export { commitPendingQueue, commitQueueRow, ignoreQueueRow, listQueueWithStatus, pendingQueueRows, poisonQueueRow, queueCounts, queueRowStatus, setQueueRowStatus, supersedeQueueRow, } from "./queueLifecycle.js";
34
- // confirm/revert ops (transaction → observed-event mapping)
35
- export { confirmAction } from "./actions.js";
36
- // fork operations (rebase/merge/cherry-pick/reset/discard) + three-way reconcile
37
- export { auditForkNoRealWrites, cherryPickActions, discardFork, forkDivergence, forkTwin, isFork, mergeForks, rebaseFork, readForkMeta, resetFork, } from "./fork.js";
38
- export { isCleanlyReconcilable, reconcile, reconcileRequiresApproval } from "./reconcile.js";
39
- // sync direction (pull/push selected resources) — the bidirectional surface
40
- export { currentResources, syncPull, syncPush } from "./sync.js";
@@ -1,93 +0,0 @@
1
- import type { WorldServiceEvent } from './types.js';
2
- export declare const WRITE_INTENT_SUFFIX = ".write.intent";
3
- export declare const WRITE_RESULT_SUFFIX = ".write.result";
4
- export declare function isEgressEventType(type: string): boolean;
5
- export type EgressActor = {
6
- kind: 'human' | 'agent' | 'bot' | 'system';
7
- id?: string;
8
- name?: string;
9
- };
10
- export type EgressWriteRequest = {
11
- service: string;
12
- /** Provider operation name, e.g. 'chat.postMessage', 'issueCreate'. */
13
- operation: string;
14
- /** external.provider value for result events, e.g. 'slack', 'github', 'linear'. */
15
- provider: string;
16
- subject: {
17
- type: string;
18
- id: string;
19
- };
20
- /**
21
- * Stable key for the logical action. Two calls with the same key are the
22
- * same action: a completed write is replayed from the ledger, never re-sent.
23
- */
24
- idempotencyKey: string;
25
- actor?: EgressActor;
26
- /** Request payload summary recorded on the intent event. Never put secrets here. */
27
- data?: Record<string, unknown>;
28
- };
29
- export type EgressWriteOutcome = {
30
- externalId: string;
31
- url?: string;
32
- data?: Record<string, unknown>;
33
- };
34
- export type EgressWriteResult = {
35
- /** 'performed' = writeFn ran this call; 'replayed' = ledger already had a success. */
36
- status: 'performed' | 'replayed';
37
- intentEventId: string;
38
- resultEventId: string;
39
- attempt: number;
40
- outcome: EgressWriteOutcome;
41
- };
42
- export type EgressLedgerEntry = {
43
- intent: WorldServiceEvent;
44
- results: WorldServiceEvent[];
45
- succeeded: boolean;
46
- };
47
- export declare class UnreconciledWriteIntentError extends Error {
48
- readonly intentEventId: string;
49
- constructor(intentEventId: string, message: string);
50
- }
51
- export declare function recordWriteIntent(request: EgressWriteRequest, root?: string): WorldServiceEvent;
52
- /**
53
- * Perform an external write through the egress ledger.
54
- *
55
- * Ordering: a write.intent event is durably appended BEFORE the provider call,
56
- * and a write.result event is appended after it (success or failure). A crash
57
- * between the two leaves an intent without results; by default the next call
58
- * with the same idempotencyKey refuses to re-send (UnreconciledWriteIntentError)
59
- * because the provider may or may not have applied the write. After verifying
60
- * the external state, callers retry with onUnreconciled: 'retry'.
61
- *
62
- * A ledger entry with a success result is terminal: the call returns the
63
- * recorded outcome without invoking writeFn. Failed attempts may be retried;
64
- * each attempt appends its own numbered result event.
65
- *
66
- * Concurrency (TWIN-58): the ledger-check + intent-append span runs under
67
- * `withFileLock` on the SAME events lock `appendEvent` itself uses (egress.ts has
68
- * no lock of its own to invent — reusing the events lock is what makes this
69
- * atomic against a concurrent appendEvent/commitQueuedEvents too, not just a
70
- * concurrent performExternalWrite). Without this, two concurrent calls with the
71
- * same idempotencyKey could both read "no entry" and both invoke writeFn — the
72
- * vendor receiving the write twice despite this function's contract that a
73
- * completed write is replayed, never re-sent (see actions.ts:121-123 for the same
74
- * check-then-append-under-one-lock reasoning applied to the action log). The
75
- * `writeFn` call itself deliberately stays OUTSIDE the lock (it's arbitrary,
76
- * possibly slow, real I/O — holding a file lock across it would serialize every
77
- * write to the service, not just same-key ones); `inFlightWrites` covers the
78
- * in-process race for the span the lock can't (an async call can't hold a
79
- * synchronous lock across its own `await`).
80
- */
81
- export declare function performExternalWrite(request: EgressWriteRequest, writeFn: () => Promise<EgressWriteOutcome>, options?: {
82
- root?: string;
83
- onUnreconciled?: 'fail' | 'retry';
84
- }): Promise<EgressWriteResult>;
85
- /** Intents with no result event at all — crash candidates needing reconciliation. */
86
- export declare function listUnreconciledWriteIntents(service: string, root?: string): WorldServiceEvent[];
87
- /** Full intent/result ledger for a service, keyed by intent event id. */
88
- export declare function listEgressLedger(service: string, root?: string): EgressLedgerEntry[];
89
- /**
90
- * Echo lookup: did we produce this external object? Inbound events whose
91
- * external id matches a recorded successful write are our own writes coming back.
92
- */
93
- export declare function findWriteResultByExternal(service: string, provider: string, externalId: string, root?: string): WorldServiceEvent | null;
@@ -1,264 +0,0 @@
1
- import { createHash } from 'node:crypto';
2
- import { appendEvent, appendEventLocked, createEvent, eventsLockPath, listEvents, withFileLock, worldPaths } from "./storage.js";
3
- export const WRITE_INTENT_SUFFIX = '.write.intent';
4
- export const WRITE_RESULT_SUFFIX = '.write.result';
5
- export function isEgressEventType(type) {
6
- return type.endsWith(WRITE_INTENT_SUFFIX) || type.endsWith(WRITE_RESULT_SUFFIX);
7
- }
8
- export class UnreconciledWriteIntentError extends Error {
9
- intentEventId;
10
- constructor(intentEventId, message) {
11
- super(message);
12
- this.name = 'UnreconciledWriteIntentError';
13
- this.intentEventId = intentEventId;
14
- }
15
- }
16
- function egressHash(request) {
17
- return createHash('sha256')
18
- .update(`${request.service}\u001f${request.operation}\u001f${request.idempotencyKey}`)
19
- .digest('hex')
20
- .slice(0, 16);
21
- }
22
- function intentIdempotencyKey(request) {
23
- return `egress:${request.service}:${request.operation}:${request.idempotencyKey}:intent`;
24
- }
25
- function resultIdempotencyKey(request, attempt) {
26
- return `egress:${request.service}:${request.operation}:${request.idempotencyKey}:result:${attempt}`;
27
- }
28
- function defaultActor(actor) {
29
- return actor ?? { kind: 'agent' };
30
- }
31
- function ledgerFor(service, root) {
32
- const entries = new Map();
33
- const resultsByCausation = new Map();
34
- for (const event of listEvents(service, root)) {
35
- if (event.type.endsWith(WRITE_INTENT_SUFFIX)) {
36
- entries.set(event.id, { intent: event, results: [], succeeded: false });
37
- }
38
- else if (event.type.endsWith(WRITE_RESULT_SUFFIX) && event.causationId) {
39
- const bucket = resultsByCausation.get(event.causationId) ?? [];
40
- bucket.push(event);
41
- resultsByCausation.set(event.causationId, bucket);
42
- }
43
- }
44
- for (const [intentId, results] of resultsByCausation) {
45
- const entry = entries.get(intentId);
46
- if (!entry)
47
- continue;
48
- entry.results = results;
49
- entry.succeeded = results.some((result) => result.data.status === 'success');
50
- }
51
- return entries;
52
- }
53
- function buildIntentEvent(request) {
54
- const hash = egressHash(request);
55
- return createEvent({
56
- id: `${request.service}${WRITE_INTENT_SUFFIX}:${hash}`,
57
- service: request.service,
58
- type: `${request.service}${WRITE_INTENT_SUFFIX}`,
59
- idempotencyKey: intentIdempotencyKey(request),
60
- occurredAt: new Date().toISOString(),
61
- origin: 'agent',
62
- actor: defaultActor(request.actor),
63
- subject: request.subject,
64
- correlationId: `${request.service}${WRITE_INTENT_SUFFIX}:${hash}`,
65
- data: {
66
- operation: request.operation,
67
- provider: request.provider,
68
- egressKey: request.idempotencyKey,
69
- ...(request.data ?? {}),
70
- },
71
- });
72
- }
73
- export function recordWriteIntent(request, root) {
74
- return appendEvent(buildIntentEvent(request), root).event;
75
- }
76
- function recordWriteResult(input) {
77
- const { request, intentEventId, attempt, status, outcome, error, root } = input;
78
- const hash = egressHash(request);
79
- const result = createEvent({
80
- id: `${request.service}${WRITE_RESULT_SUFFIX}:${hash}:${attempt}`,
81
- service: request.service,
82
- type: `${request.service}${WRITE_RESULT_SUFFIX}`,
83
- idempotencyKey: resultIdempotencyKey(request, attempt),
84
- occurredAt: new Date().toISOString(),
85
- origin: 'agent',
86
- actor: defaultActor(request.actor),
87
- subject: request.subject,
88
- causationId: intentEventId,
89
- correlationId: intentEventId,
90
- ...(status === 'success' && outcome
91
- ? {
92
- external: {
93
- provider: request.provider,
94
- id: outcome.externalId,
95
- ...(outcome.url ? { url: outcome.url } : {}),
96
- },
97
- }
98
- : {}),
99
- data: {
100
- operation: request.operation,
101
- provider: request.provider,
102
- egressKey: request.idempotencyKey,
103
- status,
104
- attempt,
105
- ...(outcome?.data ?? {}),
106
- ...(error ? { error } : {}),
107
- },
108
- });
109
- return appendEvent(result, root).event;
110
- }
111
- function replayOutcome(result) {
112
- const { operation: _operation, provider: _provider, egressKey: _key, status: _status, attempt: _attempt, ...rest } = result.data;
113
- return {
114
- externalId: result.external?.id ?? '',
115
- ...(result.external?.url ? { url: result.external.url } : {}),
116
- data: rest,
117
- };
118
- }
119
- // TWIN-58: same-process in-flight de-dup, keyed by (root, service, intentId). Two
120
- // callers racing the SAME idempotencyKey must invoke writeFn exactly once between
121
- // them; the SECOND caller awaits the FIRST's outcome and replays it rather than
122
- // redoing (or erroring out of) its own ledger-check. Populated synchronously,
123
- // before the first `await` in the winning call, so a caller that starts executing
124
- // immediately after (same microtask turn or later) is guaranteed to observe it.
125
- const inFlightWrites = new Map();
126
- function inFlightKey(service, intentId, root) {
127
- return `${root ?? ''}:${service}:${intentId}`;
128
- }
129
- /**
130
- * Perform an external write through the egress ledger.
131
- *
132
- * Ordering: a write.intent event is durably appended BEFORE the provider call,
133
- * and a write.result event is appended after it (success or failure). A crash
134
- * between the two leaves an intent without results; by default the next call
135
- * with the same idempotencyKey refuses to re-send (UnreconciledWriteIntentError)
136
- * because the provider may or may not have applied the write. After verifying
137
- * the external state, callers retry with onUnreconciled: 'retry'.
138
- *
139
- * A ledger entry with a success result is terminal: the call returns the
140
- * recorded outcome without invoking writeFn. Failed attempts may be retried;
141
- * each attempt appends its own numbered result event.
142
- *
143
- * Concurrency (TWIN-58): the ledger-check + intent-append span runs under
144
- * `withFileLock` on the SAME events lock `appendEvent` itself uses (egress.ts has
145
- * no lock of its own to invent — reusing the events lock is what makes this
146
- * atomic against a concurrent appendEvent/commitQueuedEvents too, not just a
147
- * concurrent performExternalWrite). Without this, two concurrent calls with the
148
- * same idempotencyKey could both read "no entry" and both invoke writeFn — the
149
- * vendor receiving the write twice despite this function's contract that a
150
- * completed write is replayed, never re-sent (see actions.ts:121-123 for the same
151
- * check-then-append-under-one-lock reasoning applied to the action log). The
152
- * `writeFn` call itself deliberately stays OUTSIDE the lock (it's arbitrary,
153
- * possibly slow, real I/O — holding a file lock across it would serialize every
154
- * write to the service, not just same-key ones); `inFlightWrites` covers the
155
- * in-process race for the span the lock can't (an async call can't hold a
156
- * synchronous lock across its own `await`).
157
- */
158
- export async function performExternalWrite(request, writeFn, options = {}) {
159
- const { root, onUnreconciled = 'fail' } = options;
160
- const hash = egressHash(request);
161
- const intentId = `${request.service}${WRITE_INTENT_SUFFIX}:${hash}`;
162
- const flightKey = inFlightKey(request.service, intentId, root);
163
- // Another in-process call already owns this idempotencyKey's write — replay its
164
- // outcome (or its failure) instead of racing it.
165
- const inFlight = inFlightWrites.get(flightKey);
166
- if (inFlight) {
167
- const settled = await inFlight;
168
- return { ...settled, status: 'replayed' };
169
- }
170
- const paths = worldPaths(request.service, root);
171
- const claim = withFileLock(eventsLockPath(paths), () => {
172
- const ledger = ledgerFor(request.service, root);
173
- const existing = ledger.get(intentId);
174
- if (existing) {
175
- const success = existing.results.find((result) => result.data.status === 'success');
176
- if (success) {
177
- return {
178
- replay: {
179
- status: 'replayed',
180
- intentEventId: intentId,
181
- resultEventId: success.id,
182
- attempt: Number(success.data.attempt) || existing.results.length,
183
- outcome: replayOutcome(success),
184
- },
185
- };
186
- }
187
- if (existing.results.length === 0 && onUnreconciled === 'fail') {
188
- throw new UnreconciledWriteIntentError(intentId, `Write intent ${intentId} has no result event — the provider call may or may not have happened. ` +
189
- `Verify external state, then retry with onUnreconciled: 'retry'.`);
190
- }
191
- }
192
- const intent = existing?.intent ?? appendEventLocked(buildIntentEvent(request), paths).event;
193
- const attempt = (existing?.results.length ?? 0) + 1;
194
- return { intent, attempt };
195
- });
196
- if ('replay' in claim)
197
- return claim.replay;
198
- const { intent, attempt } = claim;
199
- // Register the in-flight promise BEFORE the first await below (synchronous from
200
- // here), so a same-process concurrent call for this key — however soon it runs —
201
- // finds it rather than re-entering the lock/ledger-check itself.
202
- let resolveFlight;
203
- let rejectFlight;
204
- const flight = new Promise((resolve, reject) => {
205
- resolveFlight = resolve;
206
- rejectFlight = reject;
207
- });
208
- // A rejection here is only ever "unhandled" when no concurrent caller happened to
209
- // be waiting on this exact key — the normal (non-concurrent) case. Attach a no-op
210
- // handler so that case doesn't surface as an unhandled rejection; a real waiter
211
- // (the `await inFlight` above) still observes the SAME rejection independently.
212
- flight.catch(() => { });
213
- inFlightWrites.set(flightKey, flight);
214
- try {
215
- let outcome;
216
- try {
217
- outcome = await writeFn();
218
- }
219
- catch (error) {
220
- const message = error instanceof Error ? error.message : String(error);
221
- recordWriteResult({ request, intentEventId: intent.id, attempt, status: 'failed', error: message, root });
222
- rejectFlight(error);
223
- throw error;
224
- }
225
- const result = recordWriteResult({ request, intentEventId: intent.id, attempt, status: 'success', outcome, root });
226
- const performed = {
227
- status: 'performed',
228
- intentEventId: intent.id,
229
- resultEventId: result.id,
230
- attempt,
231
- outcome,
232
- };
233
- resolveFlight(performed);
234
- return performed;
235
- }
236
- finally {
237
- inFlightWrites.delete(flightKey);
238
- }
239
- }
240
- /** Intents with no result event at all — crash candidates needing reconciliation. */
241
- export function listUnreconciledWriteIntents(service, root) {
242
- return Array.from(ledgerFor(service, root).values())
243
- .filter((entry) => entry.results.length === 0)
244
- .map((entry) => entry.intent);
245
- }
246
- /** Full intent/result ledger for a service, keyed by intent event id. */
247
- export function listEgressLedger(service, root) {
248
- return Array.from(ledgerFor(service, root).values());
249
- }
250
- /**
251
- * Echo lookup: did we produce this external object? Inbound events whose
252
- * external id matches a recorded successful write are our own writes coming back.
253
- */
254
- export function findWriteResultByExternal(service, provider, externalId, root) {
255
- for (const event of listEvents(service, root)) {
256
- if (!event.type.endsWith(WRITE_RESULT_SUFFIX))
257
- continue;
258
- if (event.data.status !== 'success')
259
- continue;
260
- if (event.external?.provider === provider && event.external.id === externalId)
261
- return event;
262
- }
263
- return null;
264
- }