@volter/twin 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -2
- package/inject.cjs +453 -59
- package/package.json +12 -22
- package/src/actions.ts +234 -49
- package/src/blob-store.ts +136 -0
- package/src/changeset.ts +807 -0
- package/src/cli.ts +60 -10
- package/src/connector.ts +30 -7
- package/src/control-plane.ts +17 -1
- package/src/emit.ts +242 -0
- package/src/fork.ts +19 -7
- package/src/index.ts +139 -6
- package/src/lease.ts +4 -6
- package/src/lifecycle.ts +8 -0
- package/src/packRegistry.ts +248 -2
- package/src/plan.ts +131 -23
- package/src/proxy.ts +5 -2
- package/src/pushLedger.ts +116 -11
- package/src/queueLifecycle.ts +3 -4
- package/src/rateBudget.ts +1115 -0
- package/src/refs.ts +9 -10
- package/src/remote-execute.ts +16 -0
- package/src/scenario.ts +387 -0
- package/src/serve.ts +397 -15
- package/src/shadow.ts +86 -7
- package/src/storage.ts +76 -147
- package/src/sync.ts +63 -17
- package/src/twin-fetch.ts +115 -0
- package/src/validate.ts +6 -5
- package/src/world-clock.ts +33 -0
- package/src/world-store.ts +482 -0
- package/src/worldConfig.ts +4 -3
- package/dist/src/actions.d.ts +0 -138
- package/dist/src/actions.js +0 -201
- package/dist/src/args.d.ts +0 -3
- package/dist/src/args.js +0 -12
- package/dist/src/cli.d.ts +0 -2
- package/dist/src/cli.js +0 -425
- package/dist/src/connector.d.ts +0 -106
- package/dist/src/connector.js +0 -129
- package/dist/src/control-plane.d.ts +0 -21
- package/dist/src/control-plane.js +0 -40
- package/dist/src/egress.d.ts +0 -93
- package/dist/src/egress.js +0 -264
- package/dist/src/fork.d.ts +0 -126
- package/dist/src/fork.js +0 -206
- package/dist/src/index.d.ts +0 -42
- package/dist/src/index.js +0 -52
- package/dist/src/lease.d.ts +0 -50
- package/dist/src/lease.js +0 -80
- package/dist/src/packRegistry.d.ts +0 -34
- package/dist/src/packRegistry.js +0 -22
- package/dist/src/plan.d.ts +0 -97
- package/dist/src/plan.js +0 -151
- package/dist/src/proxy.d.ts +0 -25
- package/dist/src/proxy.js +0 -152
- package/dist/src/pushLedger.d.ts +0 -81
- package/dist/src/pushLedger.js +0 -130
- package/dist/src/queueLifecycle.d.ts +0 -62
- package/dist/src/queueLifecycle.js +0 -95
- package/dist/src/reconcile.d.ts +0 -58
- package/dist/src/reconcile.js +0 -137
- package/dist/src/refs.d.ts +0 -29
- package/dist/src/refs.js +0 -68
- package/dist/src/schemas.d.ts +0 -78
- package/dist/src/schemas.js +0 -50
- package/dist/src/serve.d.ts +0 -44
- package/dist/src/serve.js +0 -93
- package/dist/src/shadow.d.ts +0 -77
- package/dist/src/shadow.js +0 -138
- package/dist/src/status.d.ts +0 -31
- package/dist/src/status.js +0 -42
- package/dist/src/storage.d.ts +0 -119
- package/dist/src/storage.js +0 -535
- package/dist/src/sync.d.ts +0 -91
- package/dist/src/sync.js +0 -121
- package/dist/src/types.d.ts +0 -40
- package/dist/src/types.js +0 -1
- package/dist/src/validate.d.ts +0 -27
- package/dist/src/validate.js +0 -68
- package/dist/src/visualizer.d.ts +0 -13
- package/dist/src/visualizer.js +0 -133
- package/dist/src/worldConfig.d.ts +0 -9
- package/dist/src/worldConfig.js +0 -16
package/dist/src/connector.d.ts
DELETED
|
@@ -1,106 +0,0 @@
|
|
|
1
|
-
import type { SubjectFields } from './shadow.js';
|
|
2
|
-
import type { WorldServiceEvent } from './types.js';
|
|
3
|
-
export type ConnectorObservation = {
|
|
4
|
-
subject: {
|
|
5
|
-
type: string;
|
|
6
|
-
id: string;
|
|
7
|
-
};
|
|
8
|
-
/** Current field values; the runner diffs them against the shadow state. */
|
|
9
|
-
observed: SubjectFields;
|
|
10
|
-
/** Provider update timestamp (ISO); drives cursor advancement. */
|
|
11
|
-
occurredAt: string;
|
|
12
|
-
external?: {
|
|
13
|
-
provider: string;
|
|
14
|
-
id: string;
|
|
15
|
-
url?: string;
|
|
16
|
-
};
|
|
17
|
-
/**
|
|
18
|
-
* Extra append-only events tied to this observation (comments, reviews,
|
|
19
|
-
* …). Appended idempotently — the connector encodes identity (including
|
|
20
|
-
* content hashes for mutable bodies) in id/idempotencyKey.
|
|
21
|
-
*/
|
|
22
|
-
events?: WorldServiceEvent[];
|
|
23
|
-
};
|
|
24
|
-
export type WorldConnector = {
|
|
25
|
-
service: string;
|
|
26
|
-
/** Cursor filename under the service's cursors/ dir (default poll.json). */
|
|
27
|
-
cursorFile?: string;
|
|
28
|
-
/** Seed the shadow from historical event types (default: deltas only). */
|
|
29
|
-
shadowExtractor?: (event: WorldServiceEvent) => SubjectFields | null;
|
|
30
|
-
/**
|
|
31
|
-
* Fetch observations whose provider timestamp advanced past the cursor
|
|
32
|
-
* (empty cursor = from the beginning). `limit` is a runaway-fetch safety
|
|
33
|
-
* cap, not a page size: the connector reports `truncated: true` when the
|
|
34
|
-
* cap was hit so the runner holds the cursor.
|
|
35
|
-
*/
|
|
36
|
-
fetchSince(input: {
|
|
37
|
-
cursor: string;
|
|
38
|
-
limit: number;
|
|
39
|
-
root: string;
|
|
40
|
-
}): Promise<{
|
|
41
|
-
observations: ConnectorObservation[];
|
|
42
|
-
truncated: boolean;
|
|
43
|
-
}>;
|
|
44
|
-
};
|
|
45
|
-
export type ConnectorPollResult = {
|
|
46
|
-
service: string;
|
|
47
|
-
cursorBefore: string | null;
|
|
48
|
-
cursorAfter: string | null;
|
|
49
|
-
truncated: boolean;
|
|
50
|
-
fetched: number;
|
|
51
|
-
deltasAppended: number;
|
|
52
|
-
eventsAppended: number;
|
|
53
|
-
changedSubjects: string[];
|
|
54
|
-
};
|
|
55
|
-
export declare function pollCursorPath(service: string, root?: string, cursorFile?: string): string;
|
|
56
|
-
export declare function loadPollCursor(service: string, root?: string, cursorFile?: string): string;
|
|
57
|
-
export declare function savePollCursor(service: string, after: string, root?: string, cursorFile?: string): void;
|
|
58
|
-
export declare function runConnectorPoll(connector: WorldConnector, options?: {
|
|
59
|
-
root?: string;
|
|
60
|
-
cursor?: string;
|
|
61
|
-
limit?: number;
|
|
62
|
-
}): Promise<ConnectorPollResult>;
|
|
63
|
-
import type { ShadowState } from './shadow.js';
|
|
64
|
-
export type SweepConnector = {
|
|
65
|
-
service: string;
|
|
66
|
-
shadowExtractor?: (event: WorldServiceEvent) => SubjectFields | null;
|
|
67
|
-
/** Which tracked subjects to re-observe this sweep. */
|
|
68
|
-
selectSubjects(input: {
|
|
69
|
-
subjects: Array<{
|
|
70
|
-
type: string;
|
|
71
|
-
id: string;
|
|
72
|
-
}>;
|
|
73
|
-
shadow: ShadowState;
|
|
74
|
-
sweepAll: boolean;
|
|
75
|
-
}): Array<{
|
|
76
|
-
type: string;
|
|
77
|
-
id: string;
|
|
78
|
-
}>;
|
|
79
|
-
/** Fetch the subject's current field state; null = fetch error. */
|
|
80
|
-
fetchSubject(subject: {
|
|
81
|
-
type: string;
|
|
82
|
-
id: string;
|
|
83
|
-
}): Promise<{
|
|
84
|
-
observed: SubjectFields;
|
|
85
|
-
occurredAt: string;
|
|
86
|
-
external?: {
|
|
87
|
-
provider: string;
|
|
88
|
-
id: string;
|
|
89
|
-
url?: string;
|
|
90
|
-
};
|
|
91
|
-
} | null>;
|
|
92
|
-
};
|
|
93
|
-
export type ConnectorSweepResult = {
|
|
94
|
-
service: string;
|
|
95
|
-
subjects: number;
|
|
96
|
-
polled: number;
|
|
97
|
-
deltasAppended: number;
|
|
98
|
-
errors: number;
|
|
99
|
-
changedSubjects: string[];
|
|
100
|
-
sweepAll: boolean;
|
|
101
|
-
};
|
|
102
|
-
export declare function runConnectorSweep(connector: SweepConnector, options?: {
|
|
103
|
-
root?: string;
|
|
104
|
-
limit?: number;
|
|
105
|
-
sweepAll?: boolean;
|
|
106
|
-
}): Promise<ConnectorSweepResult>;
|
package/dist/src/connector.js
DELETED
|
@@ -1,129 +0,0 @@
|
|
|
1
|
-
// Connector contract + poll runner for cursor-based world pollers.
|
|
2
|
-
//
|
|
3
|
-
// A connector owns only the provider-specific parts: fetching observations
|
|
4
|
-
// whose provider timestamp advanced past the cursor, mapping them to subject
|
|
5
|
-
// fields, and constructing any extra append-only events (comments, …). The
|
|
6
|
-
// runner owns the invariants every poller must respect:
|
|
7
|
-
// - shadow-diff recording: an unchanged re-observation appends nothing
|
|
8
|
-
// - extra events append idempotently (id/idempotencyKey carry identity)
|
|
9
|
-
// - the cursor only advances after every observation in the batch is
|
|
10
|
-
// recorded, and NEVER advances on a truncated batch — results arrive
|
|
11
|
-
// newest-first, so a capped batch cut off OLDER updates and advancing
|
|
12
|
-
// would skip them forever
|
|
13
|
-
// - the cursor saves 1ms behind the max observed timestamp so an update
|
|
14
|
-
// landing at exactly that timestamp after the poll is not excluded by a
|
|
15
|
-
// strictly-greater filter; the overlap re-fetch is free (unchanged
|
|
16
|
-
// observations append nothing)
|
|
17
|
-
import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
|
|
18
|
-
import { dirname, join } from 'node:path';
|
|
19
|
-
import { buildShadowState, recordObservedDelta } from "./shadow.js";
|
|
20
|
-
import { appendEvent, readJsonFile, rebuildGenericState, worldPaths } from "./storage.js";
|
|
21
|
-
export function pollCursorPath(service, root, cursorFile = 'poll.json') {
|
|
22
|
-
return join(worldPaths(service, root).cursors, cursorFile);
|
|
23
|
-
}
|
|
24
|
-
export function loadPollCursor(service, root, cursorFile) {
|
|
25
|
-
const path = pollCursorPath(service, root, cursorFile);
|
|
26
|
-
if (!existsSync(path))
|
|
27
|
-
return '';
|
|
28
|
-
return readJsonFile(path).after ?? '';
|
|
29
|
-
}
|
|
30
|
-
export function savePollCursor(service, after, root, cursorFile) {
|
|
31
|
-
const path = pollCursorPath(service, root, cursorFile);
|
|
32
|
-
mkdirSync(dirname(path), { recursive: true });
|
|
33
|
-
writeFileSync(path, `${JSON.stringify({ after, updatedAt: new Date().toISOString() }, null, 2)}\n`);
|
|
34
|
-
}
|
|
35
|
-
export async function runConnectorPoll(connector, options = {}) {
|
|
36
|
-
const root = options.root;
|
|
37
|
-
const service = connector.service;
|
|
38
|
-
const limit = options.limit ?? 10000;
|
|
39
|
-
const cursorBefore = options.cursor ?? loadPollCursor(service, root, connector.cursorFile);
|
|
40
|
-
const { observations, truncated } = await connector.fetchSince({ cursor: cursorBefore, limit, root: root ?? process.env.PROJECT_ROOT ?? process.cwd() });
|
|
41
|
-
// Shadow folds from the event log; subject state arrives via deltas, so no
|
|
42
|
-
// snapshot extractor is needed.
|
|
43
|
-
const shadow = buildShadowState(service, connector.shadowExtractor ?? (() => null), root);
|
|
44
|
-
let deltasAppended = 0;
|
|
45
|
-
let eventsAppended = 0;
|
|
46
|
-
let maxOccurredAt = cursorBefore;
|
|
47
|
-
const changedSubjects = [];
|
|
48
|
-
for (const observation of observations) {
|
|
49
|
-
const result = recordObservedDelta(shadow, {
|
|
50
|
-
service,
|
|
51
|
-
subject: observation.subject,
|
|
52
|
-
observed: observation.observed,
|
|
53
|
-
occurredAt: observation.occurredAt,
|
|
54
|
-
...(observation.external ? { external: observation.external } : {}),
|
|
55
|
-
}, root);
|
|
56
|
-
if (result.changed && result.append.appended) {
|
|
57
|
-
deltasAppended += 1;
|
|
58
|
-
changedSubjects.push(observation.subject.id);
|
|
59
|
-
}
|
|
60
|
-
for (const event of observation.events ?? []) {
|
|
61
|
-
if (appendEvent(event, root).appended)
|
|
62
|
-
eventsAppended += 1;
|
|
63
|
-
}
|
|
64
|
-
if (!maxOccurredAt || observation.occurredAt > maxOccurredAt)
|
|
65
|
-
maxOccurredAt = observation.occurredAt;
|
|
66
|
-
}
|
|
67
|
-
if (!truncated && maxOccurredAt && maxOccurredAt !== cursorBefore) {
|
|
68
|
-
savePollCursor(service, new Date(Date.parse(maxOccurredAt) - 1).toISOString(), root, connector.cursorFile);
|
|
69
|
-
}
|
|
70
|
-
rebuildGenericState(service, root);
|
|
71
|
-
return {
|
|
72
|
-
service,
|
|
73
|
-
cursorBefore: cursorBefore || null,
|
|
74
|
-
cursorAfter: truncated ? cursorBefore || null : (maxOccurredAt || null),
|
|
75
|
-
truncated,
|
|
76
|
-
fetched: observations.length,
|
|
77
|
-
deltasAppended,
|
|
78
|
-
eventsAppended,
|
|
79
|
-
changedSubjects,
|
|
80
|
-
};
|
|
81
|
-
}
|
|
82
|
-
// ---------------------------------------------------------------------------
|
|
83
|
-
// Sweep mode: for providers polled by SUBJECT rather than by time cursor
|
|
84
|
-
// (e.g. re-observing the current state of every tracked pull request). The
|
|
85
|
-
// runner owns subject enumeration from the generic world state, the
|
|
86
|
-
// shadow-diff invariant (unchanged re-observations append nothing), and
|
|
87
|
-
// error accounting; the connector owns selection and the per-subject fetch
|
|
88
|
-
// (null = fetch error, counted and reported, never silently dropped).
|
|
89
|
-
import { loadState } from "./storage.js";
|
|
90
|
-
export async function runConnectorSweep(connector, options = {}) {
|
|
91
|
-
const root = options.root;
|
|
92
|
-
const service = connector.service;
|
|
93
|
-
const limit = options.limit ?? 200;
|
|
94
|
-
const sweepAll = options.sweepAll ?? false;
|
|
95
|
-
const generic = loadState(service, root);
|
|
96
|
-
const allSubjects = Object.values(generic?.subjects ?? {}).map((subject) => {
|
|
97
|
-
const record = subject;
|
|
98
|
-
return { type: String(record.type ?? ''), id: String(record.id ?? '') };
|
|
99
|
-
});
|
|
100
|
-
const shadow = buildShadowState(service, connector.shadowExtractor ?? (() => null), root);
|
|
101
|
-
const selected = connector.selectSubjects({ subjects: allSubjects, shadow, sweepAll });
|
|
102
|
-
let polled = 0;
|
|
103
|
-
let deltasAppended = 0;
|
|
104
|
-
let errors = 0;
|
|
105
|
-
const changedSubjects = [];
|
|
106
|
-
for (const subject of selected) {
|
|
107
|
-
if (polled >= limit)
|
|
108
|
-
break;
|
|
109
|
-
const observation = await connector.fetchSubject(subject);
|
|
110
|
-
polled += 1;
|
|
111
|
-
if (!observation) {
|
|
112
|
-
errors += 1;
|
|
113
|
-
continue;
|
|
114
|
-
}
|
|
115
|
-
const result = recordObservedDelta(shadow, {
|
|
116
|
-
service,
|
|
117
|
-
subject,
|
|
118
|
-
observed: observation.observed,
|
|
119
|
-
occurredAt: observation.occurredAt,
|
|
120
|
-
...(observation.external ? { external: observation.external } : {}),
|
|
121
|
-
}, root);
|
|
122
|
-
if (result.changed && result.append.appended) {
|
|
123
|
-
deltasAppended += 1;
|
|
124
|
-
changedSubjects.push(subject.id);
|
|
125
|
-
}
|
|
126
|
-
}
|
|
127
|
-
rebuildGenericState(service, root);
|
|
128
|
-
return { service, subjects: allSubjects.length, polled, deltasAppended, errors, changedSubjects, sweepAll };
|
|
129
|
-
}
|
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
export { applyTwinWrite, createTwinServer, resolveTwinRead, twinResources } from './serve.js';
|
|
2
|
-
export type { TwinResource, TwinWriteResult } from './serve.js';
|
|
3
|
-
export { formatStatus, worldStatus } from './status.js';
|
|
4
|
-
export type { WorldStatus } from './status.js';
|
|
5
|
-
export { applyPlan, buildApplyPlan, listPlans, pendingConflicts, planRequiresApproval, readPlan, writePlan } from './plan.js';
|
|
6
|
-
export type { ActionMapper, ApplyResult, ProviderCall, WorldApplyPlan } from './plan.js';
|
|
7
|
-
export { abandonPush, appendPushRecord, latestPushByActionId, listPushLedger, pushTransaction, unconfirmedPushes, UnreconciledPushError } from './pushLedger.js';
|
|
8
|
-
export type { PushOutcome, PushStatus, WorldPushRecord } from './pushLedger.js';
|
|
9
|
-
export { acquireLease, activeLease, isLeaseActive, LeaseHeldError, listLeases, releaseLease } from './lease.js';
|
|
10
|
-
export type { WorldApplyLease } from './lease.js';
|
|
11
|
-
export { isBaseStale, listRemoteRefs, readLocalRef, readRemoteRef, writeLocalRef, writeRemoteRef } from './refs.js';
|
|
12
|
-
export type { WorldLocalRef, WorldRemoteRef } from './refs.js';
|
|
13
|
-
export { commitPendingQueue, commitQueueRow, ignoreQueueRow, listQueueWithStatus, pendingQueueRows, poisonQueueRow, queueCounts, queueRowStatus, setQueueRowStatus, supersedeQueueRow, } from './queueLifecycle.js';
|
|
14
|
-
export type { QueueCounts, QueueRowStatus, QueueRowWithStatus, QueueStatusTransition } from './queueLifecycle.js';
|
|
15
|
-
export { confirmAction } from './actions.js';
|
|
16
|
-
export { auditForkNoRealWrites, cherryPickActions, discardFork, forkDivergence, forkTwin, isFork, mergeForks, rebaseFork, readForkMeta, resetFork, } from './fork.js';
|
|
17
|
-
export type { CopyResult, ForkAudit, ForkDivergence, ForkMeta, SubjectDivergence } from './fork.js';
|
|
18
|
-
export { isCleanlyReconcilable, reconcile, reconcileRequiresApproval } from './reconcile.js';
|
|
19
|
-
export type { FieldDecision, ReconcilePlan, ReconcilePolicy, SubjectReconcile } from './reconcile.js';
|
|
20
|
-
export { currentResources, syncPull, syncPush } from './sync.js';
|
|
21
|
-
export type { PullResult, PushItemResult, PushResult, SyncResource } from './sync.js';
|
|
@@ -1,40 +0,0 @@
|
|
|
1
|
-
// The CONTROL PLANE — the legitimately-shared thing across twins.
|
|
2
|
-
//
|
|
3
|
-
// This is the uniform contract + operator surface every twin is *driven* through:
|
|
4
|
-
// the same way to serve/read/write, switch modes, see `status`, build a `plan`,
|
|
5
|
-
// `push`/apply (with explicit phases), hold a `lease`, manage the event-queue
|
|
6
|
-
// lifecycle, track remote refs, and run Git-like fork operations. An operator or
|
|
7
|
-
// agent learns ONE way to drive any twin.
|
|
8
|
-
//
|
|
9
|
-
// What this is NOT: a "kernel" that makes every twin *implement its world the same
|
|
10
|
-
// way*. Twins are implemented per-vendor (their exact API + query semantics). The
|
|
11
|
-
// data-plane mechanics they reuse (event log, projection, shadow/delta, egress,
|
|
12
|
-
// connectors — exported from the package root) are **optional libraries**, like a
|
|
13
|
-
// UI library a visualizer may pick — not a mandate. See
|
|
14
|
-
// `the twins architecture notes` → "share the control plane, not the
|
|
15
|
-
// implementation".
|
|
16
|
-
//
|
|
17
|
-
// Barrel only: re-exports the control-plane surface as one cohesive thing. Consume
|
|
18
|
-
// it as a namespace via `import { controlPlane } from '@volter/twin'`.
|
|
19
|
-
// serve/read/write surface (how a twin is driven): one twin, readable always,
|
|
20
|
-
// writable unless started read-only (writes land as local actions)
|
|
21
|
-
export { applyTwinWrite, createTwinServer, resolveTwinRead, twinResources } from "./serve.js";
|
|
22
|
-
// status
|
|
23
|
-
export { formatStatus, worldStatus } from "./status.js";
|
|
24
|
-
// plan + apply orchestration
|
|
25
|
-
export { applyPlan, buildApplyPlan, listPlans, pendingConflicts, planRequiresApproval, readPlan, writePlan } from "./plan.js";
|
|
26
|
-
// push ledger + phases
|
|
27
|
-
export { abandonPush, appendPushRecord, latestPushByActionId, listPushLedger, pushTransaction, unconfirmedPushes, UnreconciledPushError } from "./pushLedger.js";
|
|
28
|
-
// apply leases (single-writer per checkpoint)
|
|
29
|
-
export { acquireLease, activeLease, isLeaseActive, LeaseHeldError, listLeases, releaseLease } from "./lease.js";
|
|
30
|
-
// remote refs / checkpoints + stale-base guard
|
|
31
|
-
export { isBaseStale, listRemoteRefs, readLocalRef, readRemoteRef, writeLocalRef, writeRemoteRef } from "./refs.js";
|
|
32
|
-
// event-queue lifecycle (queued/committed/ignored/superseded/poisoned)
|
|
33
|
-
export { commitPendingQueue, commitQueueRow, ignoreQueueRow, listQueueWithStatus, pendingQueueRows, poisonQueueRow, queueCounts, queueRowStatus, setQueueRowStatus, supersedeQueueRow, } from "./queueLifecycle.js";
|
|
34
|
-
// confirm/revert ops (transaction → observed-event mapping)
|
|
35
|
-
export { confirmAction } from "./actions.js";
|
|
36
|
-
// fork operations (rebase/merge/cherry-pick/reset/discard) + three-way reconcile
|
|
37
|
-
export { auditForkNoRealWrites, cherryPickActions, discardFork, forkDivergence, forkTwin, isFork, mergeForks, rebaseFork, readForkMeta, resetFork, } from "./fork.js";
|
|
38
|
-
export { isCleanlyReconcilable, reconcile, reconcileRequiresApproval } from "./reconcile.js";
|
|
39
|
-
// sync direction (pull/push selected resources) — the bidirectional surface
|
|
40
|
-
export { currentResources, syncPull, syncPush } from "./sync.js";
|
package/dist/src/egress.d.ts
DELETED
|
@@ -1,93 +0,0 @@
|
|
|
1
|
-
import type { WorldServiceEvent } from './types.js';
|
|
2
|
-
export declare const WRITE_INTENT_SUFFIX = ".write.intent";
|
|
3
|
-
export declare const WRITE_RESULT_SUFFIX = ".write.result";
|
|
4
|
-
export declare function isEgressEventType(type: string): boolean;
|
|
5
|
-
export type EgressActor = {
|
|
6
|
-
kind: 'human' | 'agent' | 'bot' | 'system';
|
|
7
|
-
id?: string;
|
|
8
|
-
name?: string;
|
|
9
|
-
};
|
|
10
|
-
export type EgressWriteRequest = {
|
|
11
|
-
service: string;
|
|
12
|
-
/** Provider operation name, e.g. 'chat.postMessage', 'issueCreate'. */
|
|
13
|
-
operation: string;
|
|
14
|
-
/** external.provider value for result events, e.g. 'slack', 'github', 'linear'. */
|
|
15
|
-
provider: string;
|
|
16
|
-
subject: {
|
|
17
|
-
type: string;
|
|
18
|
-
id: string;
|
|
19
|
-
};
|
|
20
|
-
/**
|
|
21
|
-
* Stable key for the logical action. Two calls with the same key are the
|
|
22
|
-
* same action: a completed write is replayed from the ledger, never re-sent.
|
|
23
|
-
*/
|
|
24
|
-
idempotencyKey: string;
|
|
25
|
-
actor?: EgressActor;
|
|
26
|
-
/** Request payload summary recorded on the intent event. Never put secrets here. */
|
|
27
|
-
data?: Record<string, unknown>;
|
|
28
|
-
};
|
|
29
|
-
export type EgressWriteOutcome = {
|
|
30
|
-
externalId: string;
|
|
31
|
-
url?: string;
|
|
32
|
-
data?: Record<string, unknown>;
|
|
33
|
-
};
|
|
34
|
-
export type EgressWriteResult = {
|
|
35
|
-
/** 'performed' = writeFn ran this call; 'replayed' = ledger already had a success. */
|
|
36
|
-
status: 'performed' | 'replayed';
|
|
37
|
-
intentEventId: string;
|
|
38
|
-
resultEventId: string;
|
|
39
|
-
attempt: number;
|
|
40
|
-
outcome: EgressWriteOutcome;
|
|
41
|
-
};
|
|
42
|
-
export type EgressLedgerEntry = {
|
|
43
|
-
intent: WorldServiceEvent;
|
|
44
|
-
results: WorldServiceEvent[];
|
|
45
|
-
succeeded: boolean;
|
|
46
|
-
};
|
|
47
|
-
export declare class UnreconciledWriteIntentError extends Error {
|
|
48
|
-
readonly intentEventId: string;
|
|
49
|
-
constructor(intentEventId: string, message: string);
|
|
50
|
-
}
|
|
51
|
-
export declare function recordWriteIntent(request: EgressWriteRequest, root?: string): WorldServiceEvent;
|
|
52
|
-
/**
|
|
53
|
-
* Perform an external write through the egress ledger.
|
|
54
|
-
*
|
|
55
|
-
* Ordering: a write.intent event is durably appended BEFORE the provider call,
|
|
56
|
-
* and a write.result event is appended after it (success or failure). A crash
|
|
57
|
-
* between the two leaves an intent without results; by default the next call
|
|
58
|
-
* with the same idempotencyKey refuses to re-send (UnreconciledWriteIntentError)
|
|
59
|
-
* because the provider may or may not have applied the write. After verifying
|
|
60
|
-
* the external state, callers retry with onUnreconciled: 'retry'.
|
|
61
|
-
*
|
|
62
|
-
* A ledger entry with a success result is terminal: the call returns the
|
|
63
|
-
* recorded outcome without invoking writeFn. Failed attempts may be retried;
|
|
64
|
-
* each attempt appends its own numbered result event.
|
|
65
|
-
*
|
|
66
|
-
* Concurrency (TWIN-58): the ledger-check + intent-append span runs under
|
|
67
|
-
* `withFileLock` on the SAME events lock `appendEvent` itself uses (egress.ts has
|
|
68
|
-
* no lock of its own to invent — reusing the events lock is what makes this
|
|
69
|
-
* atomic against a concurrent appendEvent/commitQueuedEvents too, not just a
|
|
70
|
-
* concurrent performExternalWrite). Without this, two concurrent calls with the
|
|
71
|
-
* same idempotencyKey could both read "no entry" and both invoke writeFn — the
|
|
72
|
-
* vendor receiving the write twice despite this function's contract that a
|
|
73
|
-
* completed write is replayed, never re-sent (see actions.ts:121-123 for the same
|
|
74
|
-
* check-then-append-under-one-lock reasoning applied to the action log). The
|
|
75
|
-
* `writeFn` call itself deliberately stays OUTSIDE the lock (it's arbitrary,
|
|
76
|
-
* possibly slow, real I/O — holding a file lock across it would serialize every
|
|
77
|
-
* write to the service, not just same-key ones); `inFlightWrites` covers the
|
|
78
|
-
* in-process race for the span the lock can't (an async call can't hold a
|
|
79
|
-
* synchronous lock across its own `await`).
|
|
80
|
-
*/
|
|
81
|
-
export declare function performExternalWrite(request: EgressWriteRequest, writeFn: () => Promise<EgressWriteOutcome>, options?: {
|
|
82
|
-
root?: string;
|
|
83
|
-
onUnreconciled?: 'fail' | 'retry';
|
|
84
|
-
}): Promise<EgressWriteResult>;
|
|
85
|
-
/** Intents with no result event at all — crash candidates needing reconciliation. */
|
|
86
|
-
export declare function listUnreconciledWriteIntents(service: string, root?: string): WorldServiceEvent[];
|
|
87
|
-
/** Full intent/result ledger for a service, keyed by intent event id. */
|
|
88
|
-
export declare function listEgressLedger(service: string, root?: string): EgressLedgerEntry[];
|
|
89
|
-
/**
|
|
90
|
-
* Echo lookup: did we produce this external object? Inbound events whose
|
|
91
|
-
* external id matches a recorded successful write are our own writes coming back.
|
|
92
|
-
*/
|
|
93
|
-
export declare function findWriteResultByExternal(service: string, provider: string, externalId: string, root?: string): WorldServiceEvent | null;
|
package/dist/src/egress.js
DELETED
|
@@ -1,264 +0,0 @@
|
|
|
1
|
-
import { createHash } from 'node:crypto';
|
|
2
|
-
import { appendEvent, appendEventLocked, createEvent, eventsLockPath, listEvents, withFileLock, worldPaths } from "./storage.js";
|
|
3
|
-
export const WRITE_INTENT_SUFFIX = '.write.intent';
|
|
4
|
-
export const WRITE_RESULT_SUFFIX = '.write.result';
|
|
5
|
-
export function isEgressEventType(type) {
|
|
6
|
-
return type.endsWith(WRITE_INTENT_SUFFIX) || type.endsWith(WRITE_RESULT_SUFFIX);
|
|
7
|
-
}
|
|
8
|
-
export class UnreconciledWriteIntentError extends Error {
|
|
9
|
-
intentEventId;
|
|
10
|
-
constructor(intentEventId, message) {
|
|
11
|
-
super(message);
|
|
12
|
-
this.name = 'UnreconciledWriteIntentError';
|
|
13
|
-
this.intentEventId = intentEventId;
|
|
14
|
-
}
|
|
15
|
-
}
|
|
16
|
-
function egressHash(request) {
|
|
17
|
-
return createHash('sha256')
|
|
18
|
-
.update(`${request.service}\u001f${request.operation}\u001f${request.idempotencyKey}`)
|
|
19
|
-
.digest('hex')
|
|
20
|
-
.slice(0, 16);
|
|
21
|
-
}
|
|
22
|
-
function intentIdempotencyKey(request) {
|
|
23
|
-
return `egress:${request.service}:${request.operation}:${request.idempotencyKey}:intent`;
|
|
24
|
-
}
|
|
25
|
-
function resultIdempotencyKey(request, attempt) {
|
|
26
|
-
return `egress:${request.service}:${request.operation}:${request.idempotencyKey}:result:${attempt}`;
|
|
27
|
-
}
|
|
28
|
-
function defaultActor(actor) {
|
|
29
|
-
return actor ?? { kind: 'agent' };
|
|
30
|
-
}
|
|
31
|
-
function ledgerFor(service, root) {
|
|
32
|
-
const entries = new Map();
|
|
33
|
-
const resultsByCausation = new Map();
|
|
34
|
-
for (const event of listEvents(service, root)) {
|
|
35
|
-
if (event.type.endsWith(WRITE_INTENT_SUFFIX)) {
|
|
36
|
-
entries.set(event.id, { intent: event, results: [], succeeded: false });
|
|
37
|
-
}
|
|
38
|
-
else if (event.type.endsWith(WRITE_RESULT_SUFFIX) && event.causationId) {
|
|
39
|
-
const bucket = resultsByCausation.get(event.causationId) ?? [];
|
|
40
|
-
bucket.push(event);
|
|
41
|
-
resultsByCausation.set(event.causationId, bucket);
|
|
42
|
-
}
|
|
43
|
-
}
|
|
44
|
-
for (const [intentId, results] of resultsByCausation) {
|
|
45
|
-
const entry = entries.get(intentId);
|
|
46
|
-
if (!entry)
|
|
47
|
-
continue;
|
|
48
|
-
entry.results = results;
|
|
49
|
-
entry.succeeded = results.some((result) => result.data.status === 'success');
|
|
50
|
-
}
|
|
51
|
-
return entries;
|
|
52
|
-
}
|
|
53
|
-
function buildIntentEvent(request) {
|
|
54
|
-
const hash = egressHash(request);
|
|
55
|
-
return createEvent({
|
|
56
|
-
id: `${request.service}${WRITE_INTENT_SUFFIX}:${hash}`,
|
|
57
|
-
service: request.service,
|
|
58
|
-
type: `${request.service}${WRITE_INTENT_SUFFIX}`,
|
|
59
|
-
idempotencyKey: intentIdempotencyKey(request),
|
|
60
|
-
occurredAt: new Date().toISOString(),
|
|
61
|
-
origin: 'agent',
|
|
62
|
-
actor: defaultActor(request.actor),
|
|
63
|
-
subject: request.subject,
|
|
64
|
-
correlationId: `${request.service}${WRITE_INTENT_SUFFIX}:${hash}`,
|
|
65
|
-
data: {
|
|
66
|
-
operation: request.operation,
|
|
67
|
-
provider: request.provider,
|
|
68
|
-
egressKey: request.idempotencyKey,
|
|
69
|
-
...(request.data ?? {}),
|
|
70
|
-
},
|
|
71
|
-
});
|
|
72
|
-
}
|
|
73
|
-
export function recordWriteIntent(request, root) {
|
|
74
|
-
return appendEvent(buildIntentEvent(request), root).event;
|
|
75
|
-
}
|
|
76
|
-
function recordWriteResult(input) {
|
|
77
|
-
const { request, intentEventId, attempt, status, outcome, error, root } = input;
|
|
78
|
-
const hash = egressHash(request);
|
|
79
|
-
const result = createEvent({
|
|
80
|
-
id: `${request.service}${WRITE_RESULT_SUFFIX}:${hash}:${attempt}`,
|
|
81
|
-
service: request.service,
|
|
82
|
-
type: `${request.service}${WRITE_RESULT_SUFFIX}`,
|
|
83
|
-
idempotencyKey: resultIdempotencyKey(request, attempt),
|
|
84
|
-
occurredAt: new Date().toISOString(),
|
|
85
|
-
origin: 'agent',
|
|
86
|
-
actor: defaultActor(request.actor),
|
|
87
|
-
subject: request.subject,
|
|
88
|
-
causationId: intentEventId,
|
|
89
|
-
correlationId: intentEventId,
|
|
90
|
-
...(status === 'success' && outcome
|
|
91
|
-
? {
|
|
92
|
-
external: {
|
|
93
|
-
provider: request.provider,
|
|
94
|
-
id: outcome.externalId,
|
|
95
|
-
...(outcome.url ? { url: outcome.url } : {}),
|
|
96
|
-
},
|
|
97
|
-
}
|
|
98
|
-
: {}),
|
|
99
|
-
data: {
|
|
100
|
-
operation: request.operation,
|
|
101
|
-
provider: request.provider,
|
|
102
|
-
egressKey: request.idempotencyKey,
|
|
103
|
-
status,
|
|
104
|
-
attempt,
|
|
105
|
-
...(outcome?.data ?? {}),
|
|
106
|
-
...(error ? { error } : {}),
|
|
107
|
-
},
|
|
108
|
-
});
|
|
109
|
-
return appendEvent(result, root).event;
|
|
110
|
-
}
|
|
111
|
-
function replayOutcome(result) {
|
|
112
|
-
const { operation: _operation, provider: _provider, egressKey: _key, status: _status, attempt: _attempt, ...rest } = result.data;
|
|
113
|
-
return {
|
|
114
|
-
externalId: result.external?.id ?? '',
|
|
115
|
-
...(result.external?.url ? { url: result.external.url } : {}),
|
|
116
|
-
data: rest,
|
|
117
|
-
};
|
|
118
|
-
}
|
|
119
|
-
// TWIN-58: same-process in-flight de-dup, keyed by (root, service, intentId). Two
|
|
120
|
-
// callers racing the SAME idempotencyKey must invoke writeFn exactly once between
|
|
121
|
-
// them; the SECOND caller awaits the FIRST's outcome and replays it rather than
|
|
122
|
-
// redoing (or erroring out of) its own ledger-check. Populated synchronously,
|
|
123
|
-
// before the first `await` in the winning call, so a caller that starts executing
|
|
124
|
-
// immediately after (same microtask turn or later) is guaranteed to observe it.
|
|
125
|
-
const inFlightWrites = new Map();
|
|
126
|
-
function inFlightKey(service, intentId, root) {
|
|
127
|
-
return `${root ?? ''}:${service}:${intentId}`;
|
|
128
|
-
}
|
|
129
|
-
/**
|
|
130
|
-
* Perform an external write through the egress ledger.
|
|
131
|
-
*
|
|
132
|
-
* Ordering: a write.intent event is durably appended BEFORE the provider call,
|
|
133
|
-
* and a write.result event is appended after it (success or failure). A crash
|
|
134
|
-
* between the two leaves an intent without results; by default the next call
|
|
135
|
-
* with the same idempotencyKey refuses to re-send (UnreconciledWriteIntentError)
|
|
136
|
-
* because the provider may or may not have applied the write. After verifying
|
|
137
|
-
* the external state, callers retry with onUnreconciled: 'retry'.
|
|
138
|
-
*
|
|
139
|
-
* A ledger entry with a success result is terminal: the call returns the
|
|
140
|
-
* recorded outcome without invoking writeFn. Failed attempts may be retried;
|
|
141
|
-
* each attempt appends its own numbered result event.
|
|
142
|
-
*
|
|
143
|
-
* Concurrency (TWIN-58): the ledger-check + intent-append span runs under
|
|
144
|
-
* `withFileLock` on the SAME events lock `appendEvent` itself uses (egress.ts has
|
|
145
|
-
* no lock of its own to invent — reusing the events lock is what makes this
|
|
146
|
-
* atomic against a concurrent appendEvent/commitQueuedEvents too, not just a
|
|
147
|
-
* concurrent performExternalWrite). Without this, two concurrent calls with the
|
|
148
|
-
* same idempotencyKey could both read "no entry" and both invoke writeFn — the
|
|
149
|
-
* vendor receiving the write twice despite this function's contract that a
|
|
150
|
-
* completed write is replayed, never re-sent (see actions.ts:121-123 for the same
|
|
151
|
-
* check-then-append-under-one-lock reasoning applied to the action log). The
|
|
152
|
-
* `writeFn` call itself deliberately stays OUTSIDE the lock (it's arbitrary,
|
|
153
|
-
* possibly slow, real I/O — holding a file lock across it would serialize every
|
|
154
|
-
* write to the service, not just same-key ones); `inFlightWrites` covers the
|
|
155
|
-
* in-process race for the span the lock can't (an async call can't hold a
|
|
156
|
-
* synchronous lock across its own `await`).
|
|
157
|
-
*/
|
|
158
|
-
export async function performExternalWrite(request, writeFn, options = {}) {
|
|
159
|
-
const { root, onUnreconciled = 'fail' } = options;
|
|
160
|
-
const hash = egressHash(request);
|
|
161
|
-
const intentId = `${request.service}${WRITE_INTENT_SUFFIX}:${hash}`;
|
|
162
|
-
const flightKey = inFlightKey(request.service, intentId, root);
|
|
163
|
-
// Another in-process call already owns this idempotencyKey's write — replay its
|
|
164
|
-
// outcome (or its failure) instead of racing it.
|
|
165
|
-
const inFlight = inFlightWrites.get(flightKey);
|
|
166
|
-
if (inFlight) {
|
|
167
|
-
const settled = await inFlight;
|
|
168
|
-
return { ...settled, status: 'replayed' };
|
|
169
|
-
}
|
|
170
|
-
const paths = worldPaths(request.service, root);
|
|
171
|
-
const claim = withFileLock(eventsLockPath(paths), () => {
|
|
172
|
-
const ledger = ledgerFor(request.service, root);
|
|
173
|
-
const existing = ledger.get(intentId);
|
|
174
|
-
if (existing) {
|
|
175
|
-
const success = existing.results.find((result) => result.data.status === 'success');
|
|
176
|
-
if (success) {
|
|
177
|
-
return {
|
|
178
|
-
replay: {
|
|
179
|
-
status: 'replayed',
|
|
180
|
-
intentEventId: intentId,
|
|
181
|
-
resultEventId: success.id,
|
|
182
|
-
attempt: Number(success.data.attempt) || existing.results.length,
|
|
183
|
-
outcome: replayOutcome(success),
|
|
184
|
-
},
|
|
185
|
-
};
|
|
186
|
-
}
|
|
187
|
-
if (existing.results.length === 0 && onUnreconciled === 'fail') {
|
|
188
|
-
throw new UnreconciledWriteIntentError(intentId, `Write intent ${intentId} has no result event — the provider call may or may not have happened. ` +
|
|
189
|
-
`Verify external state, then retry with onUnreconciled: 'retry'.`);
|
|
190
|
-
}
|
|
191
|
-
}
|
|
192
|
-
const intent = existing?.intent ?? appendEventLocked(buildIntentEvent(request), paths).event;
|
|
193
|
-
const attempt = (existing?.results.length ?? 0) + 1;
|
|
194
|
-
return { intent, attempt };
|
|
195
|
-
});
|
|
196
|
-
if ('replay' in claim)
|
|
197
|
-
return claim.replay;
|
|
198
|
-
const { intent, attempt } = claim;
|
|
199
|
-
// Register the in-flight promise BEFORE the first await below (synchronous from
|
|
200
|
-
// here), so a same-process concurrent call for this key — however soon it runs —
|
|
201
|
-
// finds it rather than re-entering the lock/ledger-check itself.
|
|
202
|
-
let resolveFlight;
|
|
203
|
-
let rejectFlight;
|
|
204
|
-
const flight = new Promise((resolve, reject) => {
|
|
205
|
-
resolveFlight = resolve;
|
|
206
|
-
rejectFlight = reject;
|
|
207
|
-
});
|
|
208
|
-
// A rejection here is only ever "unhandled" when no concurrent caller happened to
|
|
209
|
-
// be waiting on this exact key — the normal (non-concurrent) case. Attach a no-op
|
|
210
|
-
// handler so that case doesn't surface as an unhandled rejection; a real waiter
|
|
211
|
-
// (the `await inFlight` above) still observes the SAME rejection independently.
|
|
212
|
-
flight.catch(() => { });
|
|
213
|
-
inFlightWrites.set(flightKey, flight);
|
|
214
|
-
try {
|
|
215
|
-
let outcome;
|
|
216
|
-
try {
|
|
217
|
-
outcome = await writeFn();
|
|
218
|
-
}
|
|
219
|
-
catch (error) {
|
|
220
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
221
|
-
recordWriteResult({ request, intentEventId: intent.id, attempt, status: 'failed', error: message, root });
|
|
222
|
-
rejectFlight(error);
|
|
223
|
-
throw error;
|
|
224
|
-
}
|
|
225
|
-
const result = recordWriteResult({ request, intentEventId: intent.id, attempt, status: 'success', outcome, root });
|
|
226
|
-
const performed = {
|
|
227
|
-
status: 'performed',
|
|
228
|
-
intentEventId: intent.id,
|
|
229
|
-
resultEventId: result.id,
|
|
230
|
-
attempt,
|
|
231
|
-
outcome,
|
|
232
|
-
};
|
|
233
|
-
resolveFlight(performed);
|
|
234
|
-
return performed;
|
|
235
|
-
}
|
|
236
|
-
finally {
|
|
237
|
-
inFlightWrites.delete(flightKey);
|
|
238
|
-
}
|
|
239
|
-
}
|
|
240
|
-
/** Intents with no result event at all — crash candidates needing reconciliation. */
|
|
241
|
-
export function listUnreconciledWriteIntents(service, root) {
|
|
242
|
-
return Array.from(ledgerFor(service, root).values())
|
|
243
|
-
.filter((entry) => entry.results.length === 0)
|
|
244
|
-
.map((entry) => entry.intent);
|
|
245
|
-
}
|
|
246
|
-
/** Full intent/result ledger for a service, keyed by intent event id. */
|
|
247
|
-
export function listEgressLedger(service, root) {
|
|
248
|
-
return Array.from(ledgerFor(service, root).values());
|
|
249
|
-
}
|
|
250
|
-
/**
|
|
251
|
-
* Echo lookup: did we produce this external object? Inbound events whose
|
|
252
|
-
* external id matches a recorded successful write are our own writes coming back.
|
|
253
|
-
*/
|
|
254
|
-
export function findWriteResultByExternal(service, provider, externalId, root) {
|
|
255
|
-
for (const event of listEvents(service, root)) {
|
|
256
|
-
if (!event.type.endsWith(WRITE_RESULT_SUFFIX))
|
|
257
|
-
continue;
|
|
258
|
-
if (event.data.status !== 'success')
|
|
259
|
-
continue;
|
|
260
|
-
if (event.external?.provider === provider && event.external.id === externalId)
|
|
261
|
-
return event;
|
|
262
|
-
}
|
|
263
|
-
return null;
|
|
264
|
-
}
|