@volter/twin 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +68 -0
- package/dist/src/actions.d.ts +138 -0
- package/dist/src/actions.js +201 -0
- package/dist/src/args.d.ts +3 -0
- package/dist/src/args.js +12 -0
- package/dist/src/cli.d.ts +2 -0
- package/dist/src/cli.js +425 -0
- package/dist/src/connector.d.ts +106 -0
- package/dist/src/connector.js +129 -0
- package/dist/src/control-plane.d.ts +21 -0
- package/dist/src/control-plane.js +40 -0
- package/dist/src/egress.d.ts +93 -0
- package/dist/src/egress.js +264 -0
- package/dist/src/fork.d.ts +126 -0
- package/dist/src/fork.js +206 -0
- package/dist/src/index.d.ts +42 -0
- package/dist/src/index.js +52 -0
- package/dist/src/lease.d.ts +50 -0
- package/dist/src/lease.js +80 -0
- package/dist/src/packRegistry.d.ts +34 -0
- package/dist/src/packRegistry.js +22 -0
- package/dist/src/plan.d.ts +97 -0
- package/dist/src/plan.js +151 -0
- package/dist/src/proxy.d.ts +25 -0
- package/dist/src/proxy.js +152 -0
- package/dist/src/pushLedger.d.ts +81 -0
- package/dist/src/pushLedger.js +130 -0
- package/dist/src/queueLifecycle.d.ts +62 -0
- package/dist/src/queueLifecycle.js +95 -0
- package/dist/src/reconcile.d.ts +58 -0
- package/dist/src/reconcile.js +137 -0
- package/dist/src/refs.d.ts +29 -0
- package/dist/src/refs.js +68 -0
- package/dist/src/schemas.d.ts +78 -0
- package/dist/src/schemas.js +50 -0
- package/dist/src/serve.d.ts +44 -0
- package/dist/src/serve.js +93 -0
- package/dist/src/shadow.d.ts +77 -0
- package/dist/src/shadow.js +138 -0
- package/dist/src/status.d.ts +31 -0
- package/dist/src/status.js +42 -0
- package/dist/src/storage.d.ts +119 -0
- package/dist/src/storage.js +535 -0
- package/dist/src/sync.d.ts +91 -0
- package/dist/src/sync.js +121 -0
- package/dist/src/types.d.ts +40 -0
- package/dist/src/types.js +1 -0
- package/dist/src/validate.d.ts +27 -0
- package/dist/src/validate.js +68 -0
- package/dist/src/visualizer.d.ts +13 -0
- package/dist/src/visualizer.js +133 -0
- package/dist/src/worldConfig.d.ts +9 -0
- package/dist/src/worldConfig.js +16 -0
- package/inject.cjs +429 -0
- package/package.json +81 -0
- package/src/actions.ts +285 -0
- package/src/args.ts +14 -0
- package/src/cli.ts +443 -0
- package/src/connector.ts +220 -0
- package/src/control-plane.ts +66 -0
- package/src/egress.ts +355 -0
- package/src/fork.ts +256 -0
- package/src/index.ts +222 -0
- package/src/lease.ts +97 -0
- package/src/packRegistry.ts +60 -0
- package/src/plan.ts +190 -0
- package/src/proxy.ts +180 -0
- package/src/pushLedger.ts +189 -0
- package/src/queueLifecycle.ts +130 -0
- package/src/reconcile.ts +192 -0
- package/src/refs.ts +91 -0
- package/src/schemas.ts +56 -0
- package/src/serve.ts +120 -0
- package/src/shadow.ts +192 -0
- package/src/status.ts +58 -0
- package/src/storage.ts +632 -0
- package/src/sync.ts +160 -0
- package/src/types.ts +50 -0
- package/src/validate.ts +95 -0
- package/src/visualizer.ts +142 -0
- package/src/worldConfig.ts +26 -0
|
@@ -0,0 +1,535 @@
|
|
|
1
|
+
import { appendFileSync, closeSync, existsSync, fsyncSync, mkdirSync, openSync, readdirSync, readFileSync, renameSync, rmSync, statSync, unlinkSync, writeFileSync } from 'node:fs';
|
|
2
|
+
import { hostname } from 'node:os';
|
|
3
|
+
import { dirname, join, resolve } from 'node:path';
|
|
4
|
+
import { GenericWorldStateSchema, WorldServiceEventSchema, } from "./schemas.js";
|
|
5
|
+
function nowIso() {
|
|
6
|
+
return new Date().toISOString();
|
|
7
|
+
}
|
|
8
|
+
function projectRoot(root) {
|
|
9
|
+
return resolve(root || process.env.PROJECT_ROOT || process.cwd());
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Name of the per-project state directory holding world data
|
|
13
|
+
* (`<root>/<stateDir>/world/...`). Defaults to `.volter`; hosts that need a
|
|
14
|
+
* different directory set VOLTER_STATE_DIR. Every path in this package must
|
|
15
|
+
* go through worldStateRoot — never the literal.
|
|
16
|
+
*/
|
|
17
|
+
export function stateDirName() {
|
|
18
|
+
return process.env.VOLTER_STATE_DIR || '.volter';
|
|
19
|
+
}
|
|
20
|
+
export function worldStateRoot(root) {
|
|
21
|
+
return join(projectRoot(root), stateDirName(), 'world');
|
|
22
|
+
}
|
|
23
|
+
function assertServiceName(service) {
|
|
24
|
+
if (!/^[A-Za-z0-9_-]+$/.test(service)) {
|
|
25
|
+
throw new Error(`Invalid world service name: ${service}`);
|
|
26
|
+
}
|
|
27
|
+
return service;
|
|
28
|
+
}
|
|
29
|
+
export function worldPaths(service, root) {
|
|
30
|
+
const safeService = assertServiceName(service);
|
|
31
|
+
const resolvedRoot = projectRoot(root);
|
|
32
|
+
const dir = join(worldStateRoot(root), safeService);
|
|
33
|
+
return {
|
|
34
|
+
root: resolvedRoot,
|
|
35
|
+
service: safeService,
|
|
36
|
+
dir,
|
|
37
|
+
events: join(dir, 'events.jsonl'),
|
|
38
|
+
state: join(dir, 'state.json'),
|
|
39
|
+
resources: join(dir, 'resources'),
|
|
40
|
+
cursors: join(dir, 'cursors'),
|
|
41
|
+
ingests: join(dir, 'ingests'),
|
|
42
|
+
eventQueue: join(dir, 'event-queue.jsonl'),
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
function readJsonl(path) {
|
|
46
|
+
if (!existsSync(path))
|
|
47
|
+
return [];
|
|
48
|
+
const rows = [];
|
|
49
|
+
for (const [index, line] of readFileSync(path, 'utf8').split('\n').entries()) {
|
|
50
|
+
if (!line.trim())
|
|
51
|
+
continue;
|
|
52
|
+
try {
|
|
53
|
+
rows.push(JSON.parse(line));
|
|
54
|
+
}
|
|
55
|
+
catch (error) {
|
|
56
|
+
throw new Error(`${path}:${index + 1}: invalid JSONL row: ${error.message}`);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
return rows;
|
|
60
|
+
}
|
|
61
|
+
/** Read + parse a whole-file JSON sidecar, citing the path on a parse failure (mirrors
|
|
62
|
+
* readJsonl's `${path}:...` error shape). Callers own existence semantics — this parses a
|
|
63
|
+
* file that is expected to exist; guard with existsSync first when absence is allowed. */
|
|
64
|
+
export function readJsonFile(path) {
|
|
65
|
+
try {
|
|
66
|
+
return JSON.parse(readFileSync(path, 'utf8'));
|
|
67
|
+
}
|
|
68
|
+
catch (error) {
|
|
69
|
+
throw new Error(`${path}: invalid JSON: ${error.message}`);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Append `data` to `path`. When VOLTER_DURABLE=1, `fsyncSync` the file after the write so
|
|
74
|
+
* the appended record survives an OS crash / power loss — not just a process crash.
|
|
75
|
+
*
|
|
76
|
+
* Default (VOLTER_DURABLE unset) is OFF: `appendFileSync` is a completed write syscall, so a
|
|
77
|
+
* *process* crash after it returns still leaves the record on the log (the OS owns the page
|
|
78
|
+
* cache). The window this leaves open is a **kernel-panic / power loss** between the write
|
|
79
|
+
* landing in the page cache and the fs flushing it to stable storage — a torn or lost tail
|
|
80
|
+
* record. That crash window matters more once real pulled staging data lives in these files
|
|
81
|
+
* (purpose 2), so hosts that need durability opt in with VOLTER_DURABLE=1 at the cost of an
|
|
82
|
+
* fsync per append. See ARCHITECTURE.md D1.
|
|
83
|
+
*/
|
|
84
|
+
export function appendDurable(path, data) {
|
|
85
|
+
const fd = openSync(path, 'a');
|
|
86
|
+
try {
|
|
87
|
+
appendFileSync(fd, data);
|
|
88
|
+
if (process.env.VOLTER_DURABLE === '1')
|
|
89
|
+
fsyncSync(fd);
|
|
90
|
+
}
|
|
91
|
+
finally {
|
|
92
|
+
closeSync(fd);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* Opt-in structured stderr logging for the audit trail (ARCHITECTURE-REVIEW-TODOS D3):
|
|
97
|
+
* one line per action-log append and per push-ledger row, so "who reviewed the change
|
|
98
|
+
* that caused this real write" is mechanically greppable from a single log stream via
|
|
99
|
+
* `correlationId`. Off by default — mirrors the VOLTER_DURABLE opt-in policy above: no
|
|
100
|
+
* always-on I/O, never on the default path. Set VOLTER_TWIN_LOG=1 to enable.
|
|
101
|
+
*/
|
|
102
|
+
export function twinLog(kind, details) {
|
|
103
|
+
if (process.env.VOLTER_TWIN_LOG !== '1')
|
|
104
|
+
return;
|
|
105
|
+
console.error(`[twin:${kind}]`, JSON.stringify(details));
|
|
106
|
+
}
|
|
107
|
+
function appendJsonl(path, value) {
|
|
108
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
109
|
+
appendDurable(path, `${JSON.stringify(value)}\n`);
|
|
110
|
+
}
|
|
111
|
+
function sleepSync(ms) {
|
|
112
|
+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
|
|
113
|
+
}
|
|
114
|
+
/** How long a lock may live before an acquirer treats it as abandoned. Above the 10s
|
|
115
|
+
* wait timeout so a live-but-slow holder is never reclaimed out from under itself. */
|
|
116
|
+
const LOCK_STALE_MS = 60_000;
|
|
117
|
+
function readLockHolder(lockPath) {
|
|
118
|
+
try {
|
|
119
|
+
return JSON.parse(readFileSync(lockPath, 'utf8'));
|
|
120
|
+
}
|
|
121
|
+
catch {
|
|
122
|
+
return null; // missing, empty (mid-write), or malformed
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
function pidAlive(pid) {
|
|
126
|
+
try {
|
|
127
|
+
process.kill(pid, 0);
|
|
128
|
+
return true;
|
|
129
|
+
}
|
|
130
|
+
catch (error) {
|
|
131
|
+
// ESRCH → no such process (dead); EPERM → exists but not ours to signal (alive)
|
|
132
|
+
return error.code === 'EPERM';
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
/** A lock is stale when its holder is provably gone (same host + dead pid) or it has
|
|
136
|
+
* outlived LOCK_STALE_MS. The age falls back to the lockfile's own mtime when the
|
|
137
|
+
* `at` record is unreadable — so a writer that crashed between creating the lock and
|
|
138
|
+
* recording itself is still eventually reclaimed, while a freshly-created (recent
|
|
139
|
+
* mtime) empty lock is left alone, avoiding a race with the live writer. */
|
|
140
|
+
function lockIsStale(lockPath) {
|
|
141
|
+
const holder = readLockHolder(lockPath);
|
|
142
|
+
if (holder && holder.hostname === hostname() && Number.isInteger(holder.pid) && !pidAlive(holder.pid)) {
|
|
143
|
+
return true;
|
|
144
|
+
}
|
|
145
|
+
const recordedAt = holder && typeof holder.at === 'string' ? Date.parse(holder.at) : NaN;
|
|
146
|
+
let stamp = recordedAt;
|
|
147
|
+
if (!Number.isFinite(stamp)) {
|
|
148
|
+
try {
|
|
149
|
+
stamp = statSync(lockPath).mtimeMs;
|
|
150
|
+
}
|
|
151
|
+
catch {
|
|
152
|
+
return false; // lock vanished — let the next openSync settle it
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
return Date.now() - stamp > LOCK_STALE_MS;
|
|
156
|
+
}
|
|
157
|
+
/** Run `fn` holding an exclusive cross-process file lock (the same primitive the event log
|
|
158
|
+
* uses). Used to make read-then-append critical sections atomic across processes. The
|
|
159
|
+
* lockfile records `{pid, hostname, at}`; a contender that finds a stale lock (dead pid or
|
|
160
|
+
* age > LOCK_STALE_MS) reclaims it by atomically renaming it aside — so a crashed holder
|
|
161
|
+
* can't wedge the world forever. Reclaim is race-safe: only the process that wins the
|
|
162
|
+
* rename clears the stale inode, and the exclusive `wx` create still decides the winner. */
|
|
163
|
+
export function withFileLock(lockPath, fn) {
|
|
164
|
+
mkdirSync(dirname(lockPath), { recursive: true });
|
|
165
|
+
const started = Date.now();
|
|
166
|
+
let fd = null;
|
|
167
|
+
while (fd === null) {
|
|
168
|
+
try {
|
|
169
|
+
fd = openSync(lockPath, 'wx');
|
|
170
|
+
}
|
|
171
|
+
catch (error) {
|
|
172
|
+
const code = error.code;
|
|
173
|
+
if (code !== 'EEXIST')
|
|
174
|
+
throw error;
|
|
175
|
+
if (lockIsStale(lockPath)) {
|
|
176
|
+
const holder = readLockHolder(lockPath);
|
|
177
|
+
const salvage = `${lockPath}.stale.${process.pid}.${Date.now()}`;
|
|
178
|
+
try {
|
|
179
|
+
renameSync(lockPath, salvage);
|
|
180
|
+
}
|
|
181
|
+
catch (renameError) {
|
|
182
|
+
if (renameError.code === 'ENOENT')
|
|
183
|
+
continue; // another contender reclaimed it
|
|
184
|
+
throw renameError;
|
|
185
|
+
}
|
|
186
|
+
console.warn(`[world] reclaiming stale storage lock ${lockPath} (held by pid ${holder?.pid ?? '?'} on ${holder?.hostname ?? '?'} since ${holder?.at ?? 'unknown'})`);
|
|
187
|
+
try {
|
|
188
|
+
unlinkSync(salvage);
|
|
189
|
+
}
|
|
190
|
+
catch { /* the moved-aside stale inode; safe to leave if unlink fails */ }
|
|
191
|
+
continue; // retry the exclusive create immediately
|
|
192
|
+
}
|
|
193
|
+
if (Date.now() - started > 10_000) {
|
|
194
|
+
throw new Error(`Timed out waiting for world storage lock: ${lockPath}`);
|
|
195
|
+
}
|
|
196
|
+
sleepSync(25);
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
// Record the holder so a later contender can detect staleness. Best-effort: a write
|
|
200
|
+
// failure here doesn't weaken exclusivity, only staleness diagnostics.
|
|
201
|
+
try {
|
|
202
|
+
writeFileSync(fd, `${JSON.stringify({ pid: process.pid, hostname: hostname(), at: new Date().toISOString() })}\n`);
|
|
203
|
+
}
|
|
204
|
+
catch { /* ignore */ }
|
|
205
|
+
try {
|
|
206
|
+
return fn();
|
|
207
|
+
}
|
|
208
|
+
finally {
|
|
209
|
+
closeSync(fd);
|
|
210
|
+
try {
|
|
211
|
+
unlinkSync(lockPath);
|
|
212
|
+
}
|
|
213
|
+
catch { /* already reclaimed by a stale-lock sweep */ }
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
function writeJsonAtomic(path, value) {
|
|
217
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
218
|
+
const tmp = `${path}.${process.pid}.${Date.now()}.tmp`;
|
|
219
|
+
writeFileSync(tmp, `${JSON.stringify(value, null, 2)}\n`);
|
|
220
|
+
renameSync(tmp, path);
|
|
221
|
+
}
|
|
222
|
+
export function listEvents(service, root) {
|
|
223
|
+
const paths = worldPaths(service, root);
|
|
224
|
+
return readJsonl(paths.events).map((row) => WorldServiceEventSchema.parse(row));
|
|
225
|
+
}
|
|
226
|
+
const eventsIndexCache = new Map();
|
|
227
|
+
function eventsFileStamp(eventsPath) {
|
|
228
|
+
if (!existsSync(eventsPath))
|
|
229
|
+
return { size: 0, mtimeMs: 0 };
|
|
230
|
+
const stat = statSync(eventsPath);
|
|
231
|
+
return { size: stat.size, mtimeMs: stat.mtimeMs };
|
|
232
|
+
}
|
|
233
|
+
function eventsIndexFor(eventsPath) {
|
|
234
|
+
const { size, mtimeMs } = eventsFileStamp(eventsPath);
|
|
235
|
+
const cached = eventsIndexCache.get(eventsPath);
|
|
236
|
+
if (cached && cached.size === size && cached.mtimeMs === mtimeMs)
|
|
237
|
+
return cached;
|
|
238
|
+
const index = { size, mtimeMs, ids: new Set(), keys: new Set() };
|
|
239
|
+
for (const row of readJsonl(eventsPath)) {
|
|
240
|
+
if (typeof row.id === 'string')
|
|
241
|
+
index.ids.add(row.id);
|
|
242
|
+
if (typeof row.idempotencyKey === 'string')
|
|
243
|
+
index.keys.add(row.idempotencyKey);
|
|
244
|
+
}
|
|
245
|
+
eventsIndexCache.set(eventsPath, index);
|
|
246
|
+
return index;
|
|
247
|
+
}
|
|
248
|
+
/** Targeted fetch of one event row by id or idempotencyKey (rare duplicate path). */
|
|
249
|
+
function findEventRow(eventsPath, id, idempotencyKey) {
|
|
250
|
+
for (const row of readJsonl(eventsPath)) {
|
|
251
|
+
if (row.id === id || row.idempotencyKey === idempotencyKey) {
|
|
252
|
+
return WorldServiceEventSchema.parse(row);
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
return null;
|
|
256
|
+
}
|
|
257
|
+
function ensureEventDirs(paths) {
|
|
258
|
+
mkdirSync(paths.dir, { recursive: true });
|
|
259
|
+
mkdirSync(paths.resources, { recursive: true });
|
|
260
|
+
mkdirSync(paths.cursors, { recursive: true });
|
|
261
|
+
mkdirSync(paths.ingests, { recursive: true });
|
|
262
|
+
}
|
|
263
|
+
/** Exported so other modules that share a service's events log (e.g. egress.ts,
|
|
264
|
+
* TWIN-58) can serialize their own critical sections on the SAME lock appendEvent
|
|
265
|
+
* itself uses, instead of inventing a second, uncoordinated lockfile. */
|
|
266
|
+
export function eventsLockPath(paths) {
|
|
267
|
+
return join(paths.dir, 'events.jsonl.lock');
|
|
268
|
+
}
|
|
269
|
+
/**
|
|
270
|
+
* The append body, assuming the caller already holds `eventsLockPath(paths)`. Split
|
|
271
|
+
* out so `commitQueuedEvents` can run a whole batch of appends (and the rebuild that
|
|
272
|
+
* follows them) under ONE lock acquisition instead of nesting a fresh `withFileLock`
|
|
273
|
+
* per row — `withFileLock` is not reentrant, so re-acquiring it from inside an
|
|
274
|
+
* already-held lock in the same process would just spin to its own timeout. Also
|
|
275
|
+
* exported for egress.ts (TWIN-58): performExternalWrite's ledger-check +
|
|
276
|
+
* intent-append span holds `eventsLockPath` itself, so it must append through this
|
|
277
|
+
* already-locked path rather than the public `appendEvent` (which would try to
|
|
278
|
+
* re-acquire the same lock and spin to its own timeout).
|
|
279
|
+
*/
|
|
280
|
+
export function appendEventLocked(parsed, paths) {
|
|
281
|
+
const index = eventsIndexFor(paths.events);
|
|
282
|
+
if (index.ids.has(parsed.id) || index.keys.has(parsed.idempotencyKey)) {
|
|
283
|
+
const duplicate = findEventRow(paths.events, parsed.id, parsed.idempotencyKey);
|
|
284
|
+
if (!duplicate)
|
|
285
|
+
throw new Error(`World events index out of sync for ${paths.events}; delete nothing — rerun to rebuild`);
|
|
286
|
+
// observedAt (delivery time), occurredAt (poll-based connectors use
|
|
287
|
+
// poll-cycle time, not the resource's own timestamp), and external.url
|
|
288
|
+
// (a convenience pointer — provider URLs embed mutable title slugs,
|
|
289
|
+
// e.g. Linear) are metadata, not event identity: re-observations
|
|
290
|
+
// differing only there are duplicates, not conflicts.
|
|
291
|
+
// external.provider/id ARE identity.
|
|
292
|
+
const stripMetadata = (event) => {
|
|
293
|
+
const { observedAt: _observedAt, occurredAt: _occurredAt, external, ...content } = event;
|
|
294
|
+
if (!external)
|
|
295
|
+
return content;
|
|
296
|
+
const { url: _url, ...externalIdentity } = external;
|
|
297
|
+
return { ...content, external: externalIdentity };
|
|
298
|
+
};
|
|
299
|
+
if (JSON.stringify(stripMetadata(duplicate)) !== JSON.stringify(stripMetadata(parsed))) {
|
|
300
|
+
throw new Error(`Conflicting duplicate ${parsed.service} event: ${parsed.id} / ${parsed.idempotencyKey} conflicts with ${duplicate.id}`);
|
|
301
|
+
}
|
|
302
|
+
return { event: duplicate, appended: false, duplicateOf: duplicate.id };
|
|
303
|
+
}
|
|
304
|
+
appendJsonl(paths.events, parsed);
|
|
305
|
+
const stamp = eventsFileStamp(paths.events);
|
|
306
|
+
index.ids.add(parsed.id);
|
|
307
|
+
index.keys.add(parsed.idempotencyKey);
|
|
308
|
+
index.size = stamp.size;
|
|
309
|
+
index.mtimeMs = stamp.mtimeMs;
|
|
310
|
+
return { event: parsed, appended: true };
|
|
311
|
+
}
|
|
312
|
+
export function appendEvent(event, root) {
|
|
313
|
+
const parsed = WorldServiceEventSchema.parse(event);
|
|
314
|
+
const paths = worldPaths(parsed.service, root);
|
|
315
|
+
ensureEventDirs(paths);
|
|
316
|
+
return withFileLock(eventsLockPath(paths), () => appendEventLocked(parsed, paths));
|
|
317
|
+
}
|
|
318
|
+
function queuedEventId(event) {
|
|
319
|
+
return `queue:${event.service}:${event.idempotencyKey}`;
|
|
320
|
+
}
|
|
321
|
+
export function listQueuedEvents(service, root) {
|
|
322
|
+
const paths = worldPaths(service, root);
|
|
323
|
+
return readJsonl(paths.eventQueue);
|
|
324
|
+
}
|
|
325
|
+
export function enqueueEvent(event, options = {}) {
|
|
326
|
+
const parsed = WorldServiceEventSchema.parse(event);
|
|
327
|
+
const paths = worldPaths(parsed.service, options.root);
|
|
328
|
+
mkdirSync(paths.dir, { recursive: true });
|
|
329
|
+
mkdirSync(paths.ingests, { recursive: true });
|
|
330
|
+
const queued = {
|
|
331
|
+
id: options.id ?? queuedEventId(parsed),
|
|
332
|
+
service: parsed.service,
|
|
333
|
+
receivedAt: options.receivedAt ?? nowIso(),
|
|
334
|
+
source: options.source ?? 'listener',
|
|
335
|
+
event: parsed,
|
|
336
|
+
};
|
|
337
|
+
return withFileLock(join(paths.dir, 'event-queue.jsonl.lock'), () => {
|
|
338
|
+
const existing = readJsonl(paths.eventQueue);
|
|
339
|
+
const duplicate = existing.find((candidate) => candidate.id === queued.id || candidate.event.idempotencyKey === queued.event.idempotencyKey);
|
|
340
|
+
if (duplicate)
|
|
341
|
+
return { queued: duplicate, appended: false, duplicateOf: duplicate.id };
|
|
342
|
+
appendJsonl(paths.eventQueue, queued);
|
|
343
|
+
return { queued, appended: true };
|
|
344
|
+
});
|
|
345
|
+
}
|
|
346
|
+
/**
|
|
347
|
+
* Commit queued (webhook/listener) events into the canonical log, then rebuild the
|
|
348
|
+
* generic projection — spanning BOTH under the SAME events-lock acquisition (D7).
|
|
349
|
+
* Appending each event and rebuilding state.json are two separate durable writes;
|
|
350
|
+
* without a shared lock, a concurrent commit/append on another process could
|
|
351
|
+
* interleave between "this commit's last append" and "this commit's rebuild",
|
|
352
|
+
* so the rebuilt state.json would not correspond to exactly this commit's view of
|
|
353
|
+
* the log. Holding one lock across the whole sequence rules that out: no other
|
|
354
|
+
* appendEvent/commitQueuedEvents call for this service can run until both the
|
|
355
|
+
* appends AND the rebuild here have completed. (A single-process crash between the
|
|
356
|
+
* last durable append and the rebuild can still leave state.json one rebuild behind
|
|
357
|
+
* — that window is inherent to a two-file update with no WAL, and is unaffected by
|
|
358
|
+
* locking; it is closed by the next successful commit/rebuild, which always starts
|
|
359
|
+
* from the durably-appended log, so no event is ever lost, only state.json's
|
|
360
|
+
* projection is briefly stale.)
|
|
361
|
+
*/
|
|
362
|
+
export function commitQueuedEvents(service, options = {}) {
|
|
363
|
+
const paths = worldPaths(service, options.root);
|
|
364
|
+
ensureEventDirs(paths);
|
|
365
|
+
const queued = listQueuedEvents(service, options.root).sort((a, b) => a.event.occurredAt.localeCompare(b.event.occurredAt) || a.receivedAt.localeCompare(b.receivedAt) || a.id.localeCompare(b.id));
|
|
366
|
+
const limit = options.limit ?? queued.length;
|
|
367
|
+
return withFileLock(eventsLockPath(paths), () => {
|
|
368
|
+
let committed = 0;
|
|
369
|
+
let skipped = 0;
|
|
370
|
+
const eventIds = [];
|
|
371
|
+
for (const row of queued.slice(0, limit)) {
|
|
372
|
+
const parsed = WorldServiceEventSchema.parse(row.event);
|
|
373
|
+
const result = appendEventLocked(parsed, paths);
|
|
374
|
+
if (result.appended) {
|
|
375
|
+
committed += 1;
|
|
376
|
+
eventIds.push(result.event.id);
|
|
377
|
+
}
|
|
378
|
+
else {
|
|
379
|
+
skipped += 1;
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
if (committed > 0)
|
|
383
|
+
rebuildGenericState(service, options.root);
|
|
384
|
+
return { service, queued: Math.min(queued.length, limit), committed, skipped, eventIds };
|
|
385
|
+
});
|
|
386
|
+
}
|
|
387
|
+
// NOTE: annotation read/write (listAnnotations/addAnnotation/createAnnotation) moved
|
|
388
|
+
// to @volter/tracker/world-annotations — annotations are a tracker concern.
|
|
389
|
+
export function genericWorldReducer(state, event) {
|
|
390
|
+
const key = `${event.subject.type}:${event.subject.id}`;
|
|
391
|
+
return GenericWorldStateSchema.parse({
|
|
392
|
+
...state,
|
|
393
|
+
eventCount: state.eventCount + 1,
|
|
394
|
+
latestEventId: event.id,
|
|
395
|
+
subjects: {
|
|
396
|
+
...state.subjects,
|
|
397
|
+
[key]: {
|
|
398
|
+
type: event.subject.type,
|
|
399
|
+
id: event.subject.id,
|
|
400
|
+
latestEventId: event.id,
|
|
401
|
+
latestType: event.type,
|
|
402
|
+
updatedAt: event.occurredAt || event.observedAt,
|
|
403
|
+
},
|
|
404
|
+
},
|
|
405
|
+
});
|
|
406
|
+
}
|
|
407
|
+
export function emptyGenericState(service) {
|
|
408
|
+
return {
|
|
409
|
+
version: 1,
|
|
410
|
+
service: assertServiceName(service),
|
|
411
|
+
rebuiltAt: nowIso(),
|
|
412
|
+
eventCount: 0,
|
|
413
|
+
subjects: {},
|
|
414
|
+
};
|
|
415
|
+
}
|
|
416
|
+
export function rebuildState(service, initialState, reducer, root) {
|
|
417
|
+
const paths = worldPaths(service, root);
|
|
418
|
+
const state = listEvents(service, root).reduce(reducer, initialState);
|
|
419
|
+
writeJsonAtomic(paths.state, state);
|
|
420
|
+
return state;
|
|
421
|
+
}
|
|
422
|
+
export function rebuildGenericState(service, root) {
|
|
423
|
+
return rebuildState(service, emptyGenericState(service), genericWorldReducer, root);
|
|
424
|
+
}
|
|
425
|
+
export function loadState(service, root) {
|
|
426
|
+
const paths = worldPaths(service, root);
|
|
427
|
+
if (!existsSync(paths.state))
|
|
428
|
+
return null;
|
|
429
|
+
return readJsonFile(paths.state);
|
|
430
|
+
}
|
|
431
|
+
export function createEvent(input) {
|
|
432
|
+
return WorldServiceEventSchema.parse({
|
|
433
|
+
schemaVersion: 1,
|
|
434
|
+
observedAt: nowIso(),
|
|
435
|
+
...input,
|
|
436
|
+
});
|
|
437
|
+
}
|
|
438
|
+
/** Filenames/dirnames ANY control-plane src module ever writes inside a single
|
|
439
|
+
* service's state dir (see worldPaths) — plus the lockfiles withFileLock creates
|
|
440
|
+
* next to a log and the `.tmp`/`.stale.*` sidecars its atomic-write/reclaim paths
|
|
441
|
+
* use. storage.ts itself only accounts for events.jsonl/event-queue.jsonl/
|
|
442
|
+
* state.json/resources/cursors/ingests; the rest are written by other modules
|
|
443
|
+
* that share the same service dir (actions.ts, pushLedger.ts, plan.ts, lease.ts,
|
|
444
|
+
* refs.ts, fork.ts, queueLifecycle.ts) — this set MUST stay in sync with every
|
|
445
|
+
* one of them so scrubService never demands --force for an ordinary twin. */
|
|
446
|
+
const KNOWN_SERVICE_ENTRIES = new Set([
|
|
447
|
+
'events.jsonl',
|
|
448
|
+
'events.jsonl.lock',
|
|
449
|
+
'event-queue.jsonl',
|
|
450
|
+
'event-queue.jsonl.lock',
|
|
451
|
+
'state.json',
|
|
452
|
+
'resources',
|
|
453
|
+
'cursors',
|
|
454
|
+
'ingests',
|
|
455
|
+
// actions.ts: the local transaction-commit log + its lock.
|
|
456
|
+
'actions.jsonl',
|
|
457
|
+
'actions.jsonl.lock',
|
|
458
|
+
// pushLedger.ts: the push-phase ledger for real-vendor replication.
|
|
459
|
+
'push-ledger.jsonl',
|
|
460
|
+
// queueLifecycle.ts: the append-only status ledger over event-queue.jsonl rows.
|
|
461
|
+
'event-queue-status.jsonl',
|
|
462
|
+
// fork.ts: fork metadata (base snapshot + divergence baseline) for a fork root.
|
|
463
|
+
'fork-meta.json',
|
|
464
|
+
// plan.ts: one JSON file per apply plan, under plans/<planId>.json.
|
|
465
|
+
'plans',
|
|
466
|
+
// lease.ts: one JSON file per apply lease, under leases/<leaseId>.json.
|
|
467
|
+
'leases',
|
|
468
|
+
// refs.ts: remote/local checkpoint refs, under refs/remote/<provider>/<name>.json
|
|
469
|
+
// and refs/local/<forkId>.json.
|
|
470
|
+
'refs',
|
|
471
|
+
]);
|
|
472
|
+
function isKnownServiceEntry(name) {
|
|
473
|
+
return KNOWN_SERVICE_ENTRIES.has(name) || name.endsWith('.tmp') || /\.stale\.\d+\.\d+$/.test(name);
|
|
474
|
+
}
|
|
475
|
+
/** A service dir "looks like" one of ours when every entry in it is something
|
|
476
|
+
* storage.ts is known to create. A directory that doesn't exist yet trivially
|
|
477
|
+
* looks fine (scrub will just no-op on it). */
|
|
478
|
+
function looksLikeServiceStateDir(dir) {
|
|
479
|
+
if (!existsSync(dir))
|
|
480
|
+
return true;
|
|
481
|
+
return readdirSync(dir).every((entry) => isKnownServiceEntry(entry));
|
|
482
|
+
}
|
|
483
|
+
/** The whole world dir "looks like" ours when every entry in it is itself a
|
|
484
|
+
* directory that looks like a service state dir (each service gets one subdir
|
|
485
|
+
* under worldStateRoot — see worldPaths). */
|
|
486
|
+
function looksLikeWorldStateDir(dir) {
|
|
487
|
+
if (!existsSync(dir))
|
|
488
|
+
return true;
|
|
489
|
+
return readdirSync(dir).every((entry) => {
|
|
490
|
+
const full = join(dir, entry);
|
|
491
|
+
return statSync(full).isDirectory() && looksLikeServiceStateDir(full);
|
|
492
|
+
});
|
|
493
|
+
}
|
|
494
|
+
/** Every file under `dir`, path relative to `dir`, depth-first. Used to report
|
|
495
|
+
* exactly what scrub is about to remove before it removes it. */
|
|
496
|
+
function listFilesRecursive(dir, base = dir) {
|
|
497
|
+
if (!existsSync(dir))
|
|
498
|
+
return [];
|
|
499
|
+
const out = [];
|
|
500
|
+
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
|
501
|
+
const full = join(dir, entry.name);
|
|
502
|
+
if (entry.isDirectory())
|
|
503
|
+
out.push(...listFilesRecursive(full, base));
|
|
504
|
+
else
|
|
505
|
+
out.push(full.slice(base.length + 1));
|
|
506
|
+
}
|
|
507
|
+
return out;
|
|
508
|
+
}
|
|
509
|
+
function scrubDir(dir, label, looksLike, options) {
|
|
510
|
+
if (!existsSync(dir)) {
|
|
511
|
+
return { target: dir, removed: [], message: `nothing to scrub: ${label} (${dir}) does not exist` };
|
|
512
|
+
}
|
|
513
|
+
if (!options.force && !looksLike(dir)) {
|
|
514
|
+
throw new Error(`world scrub: refusing to remove "${dir}" — it does not look like a world state directory ` +
|
|
515
|
+
`(unexpected contents). Pass --force (or { force: true }) to scrub it anyway.`);
|
|
516
|
+
}
|
|
517
|
+
const removed = listFilesRecursive(dir);
|
|
518
|
+
rmSync(dir, { recursive: true, force: true });
|
|
519
|
+
return { target: dir, removed, message: `scrubbed ${label}: removed ${removed.length} file(s) under ${dir}` };
|
|
520
|
+
}
|
|
521
|
+
/** Delete one service's pulled data at rest: its event log, queued events,
|
|
522
|
+
* rebuilt state.json, and any resources/cursors/ingests sidecars — everything
|
|
523
|
+
* `worldPaths(service)` points at. Refuses (unless `force`) when the dir holds
|
|
524
|
+
* anything storage.ts didn't put there. */
|
|
525
|
+
export function scrubService(service, options = {}) {
|
|
526
|
+
const paths = worldPaths(service, options.root);
|
|
527
|
+
return scrubDir(paths.dir, `service "${service}"`, looksLikeServiceStateDir, options);
|
|
528
|
+
}
|
|
529
|
+
/** Delete the ENTIRE world state dir (`<root>/<stateDir>/world`) — every
|
|
530
|
+
* service's pulled data at once. Refuses (unless `force`) when any entry in it
|
|
531
|
+
* isn't itself a recognizable per-service state dir. */
|
|
532
|
+
export function scrubWorld(options = {}) {
|
|
533
|
+
const dir = worldStateRoot(options.root);
|
|
534
|
+
return scrubDir(dir, 'entire world state dir', looksLikeWorldStateDir, options);
|
|
535
|
+
}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import type { EgressWriteResult } from './egress.js';
|
|
2
|
+
import type { WorldRemoteRef } from './refs.js';
|
|
3
|
+
import type { SubjectFields } from './shadow.js';
|
|
4
|
+
import type { TwinResource } from './serve.js';
|
|
5
|
+
import type { ReconcilePlan } from './reconcile.js';
|
|
6
|
+
export type SyncResource = {
|
|
7
|
+
type: string;
|
|
8
|
+
id: string;
|
|
9
|
+
fields: SubjectFields;
|
|
10
|
+
};
|
|
11
|
+
export type PullResult = {
|
|
12
|
+
observed: number;
|
|
13
|
+
deltasAppended: number;
|
|
14
|
+
unchanged: number;
|
|
15
|
+
};
|
|
16
|
+
/**
|
|
17
|
+
* Pull: fold a set of externally-observed resources into the twin's event log.
|
|
18
|
+
* The caller has already fetched them from the real vendor (injected I/O); this
|
|
19
|
+
* is the pure fold. Only changed fields produce a delta (recordObservedDelta is
|
|
20
|
+
* a no-op when nothing differs), so re-pulling identical state appends nothing.
|
|
21
|
+
*
|
|
22
|
+
* `redact` (TWIN-45 dev/01 — implemented, not just documented) is an optional
|
|
23
|
+
* transform applied to each resource BEFORE it is folded into the event log, so
|
|
24
|
+
* a sensitive field never lands on disk in the first place — a redact-on-pull
|
|
25
|
+
* hook point, not a redaction policy: this library makes zero default redaction
|
|
26
|
+
* decisions (no built-in "PII field" list, no defaults). The caller supplies the
|
|
27
|
+
* hook deliberately, shaped for their own vendor/fields. See docs/DATA_AT_REST.md
|
|
28
|
+
* for the full data-at-rest story (what's implemented here vs. left as
|
|
29
|
+
* guidance) and for why day-count retention defaults are NOT hardcoded here —
|
|
30
|
+
* that is a policy call for the human owner, not this library.
|
|
31
|
+
*/
|
|
32
|
+
export declare function syncPull(opts: {
|
|
33
|
+
service: string;
|
|
34
|
+
resources: SyncResource[];
|
|
35
|
+
occurredAt: string;
|
|
36
|
+
root?: string;
|
|
37
|
+
actor?: {
|
|
38
|
+
kind: 'agent' | 'human' | 'bot' | 'system';
|
|
39
|
+
id?: string;
|
|
40
|
+
};
|
|
41
|
+
redact?: (resource: SyncResource) => SyncResource;
|
|
42
|
+
}): PullResult;
|
|
43
|
+
export type PushItemResult = {
|
|
44
|
+
id: string;
|
|
45
|
+
type: string;
|
|
46
|
+
status: EgressWriteResult['status'];
|
|
47
|
+
externalId?: string;
|
|
48
|
+
};
|
|
49
|
+
export type PushResult = {
|
|
50
|
+
attempted: number;
|
|
51
|
+
pushed: PushItemResult[];
|
|
52
|
+
};
|
|
53
|
+
/**
|
|
54
|
+
* Push: enact a reconcile plan's twin→real changes by calling the injected real
|
|
55
|
+
* write fn, idempotently via the egress ledger. `write` is the ONLY place a real
|
|
56
|
+
* vendor API is touched; in tests/offline it's a fake. Re-pushing the same plan
|
|
57
|
+
* replays (no double-apply) because the egress idempotency key encodes the change.
|
|
58
|
+
*
|
|
59
|
+
* Gated the same way `applyPlan` is (TWIN-53 R-K2): `approve` and the lease fields
|
|
60
|
+
* (`remoteRef`/`holder`/`leaseId`/`acquiredAt`/`expiresAt`) are mandatory options —
|
|
61
|
+
* there is no implicit-approval, no-lease path. Approval is required when
|
|
62
|
+
* `reconcileRequiresApproval(plan)` is true (unresolved conflicts, or a
|
|
63
|
+
* twin-wins push that overwrites a real-side change) and `approve` is not set;
|
|
64
|
+
* refusal happens before the lease is acquired or `write` is ever called — zero
|
|
65
|
+
* side effects. The lease is acquired for `remoteRef` before any push and released
|
|
66
|
+
* (even on error) after.
|
|
67
|
+
*/
|
|
68
|
+
export declare function syncPush(opts: {
|
|
69
|
+
service: string;
|
|
70
|
+
plan: ReconcilePlan;
|
|
71
|
+
write: (item: SyncResource) => Promise<{
|
|
72
|
+
externalId: string;
|
|
73
|
+
data?: Record<string, unknown>;
|
|
74
|
+
}>;
|
|
75
|
+
remoteRef: WorldRemoteRef;
|
|
76
|
+
holder: {
|
|
77
|
+
kind: 'agent' | 'human' | 'system';
|
|
78
|
+
id: string;
|
|
79
|
+
};
|
|
80
|
+
leaseId: string;
|
|
81
|
+
acquiredAt: string;
|
|
82
|
+
expiresAt: string;
|
|
83
|
+
approve: boolean;
|
|
84
|
+
root?: string;
|
|
85
|
+
actor?: {
|
|
86
|
+
kind: 'agent' | 'human' | 'bot' | 'system';
|
|
87
|
+
id?: string;
|
|
88
|
+
};
|
|
89
|
+
}): Promise<PushResult>;
|
|
90
|
+
/** Convenience: the twin's current resources as the reconcile `fork`/state input. */
|
|
91
|
+
export declare function currentResources(service: string, root?: string): TwinResource[];
|