@volter/twin 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/LICENSE +202 -0
  2. package/README.md +68 -0
  3. package/dist/src/actions.d.ts +138 -0
  4. package/dist/src/actions.js +201 -0
  5. package/dist/src/args.d.ts +3 -0
  6. package/dist/src/args.js +12 -0
  7. package/dist/src/cli.d.ts +2 -0
  8. package/dist/src/cli.js +425 -0
  9. package/dist/src/connector.d.ts +106 -0
  10. package/dist/src/connector.js +129 -0
  11. package/dist/src/control-plane.d.ts +21 -0
  12. package/dist/src/control-plane.js +40 -0
  13. package/dist/src/egress.d.ts +93 -0
  14. package/dist/src/egress.js +264 -0
  15. package/dist/src/fork.d.ts +126 -0
  16. package/dist/src/fork.js +206 -0
  17. package/dist/src/index.d.ts +42 -0
  18. package/dist/src/index.js +52 -0
  19. package/dist/src/lease.d.ts +50 -0
  20. package/dist/src/lease.js +80 -0
  21. package/dist/src/packRegistry.d.ts +34 -0
  22. package/dist/src/packRegistry.js +22 -0
  23. package/dist/src/plan.d.ts +97 -0
  24. package/dist/src/plan.js +151 -0
  25. package/dist/src/proxy.d.ts +25 -0
  26. package/dist/src/proxy.js +152 -0
  27. package/dist/src/pushLedger.d.ts +81 -0
  28. package/dist/src/pushLedger.js +130 -0
  29. package/dist/src/queueLifecycle.d.ts +62 -0
  30. package/dist/src/queueLifecycle.js +95 -0
  31. package/dist/src/reconcile.d.ts +58 -0
  32. package/dist/src/reconcile.js +137 -0
  33. package/dist/src/refs.d.ts +29 -0
  34. package/dist/src/refs.js +68 -0
  35. package/dist/src/schemas.d.ts +78 -0
  36. package/dist/src/schemas.js +50 -0
  37. package/dist/src/serve.d.ts +44 -0
  38. package/dist/src/serve.js +93 -0
  39. package/dist/src/shadow.d.ts +77 -0
  40. package/dist/src/shadow.js +138 -0
  41. package/dist/src/status.d.ts +31 -0
  42. package/dist/src/status.js +42 -0
  43. package/dist/src/storage.d.ts +119 -0
  44. package/dist/src/storage.js +535 -0
  45. package/dist/src/sync.d.ts +91 -0
  46. package/dist/src/sync.js +121 -0
  47. package/dist/src/types.d.ts +40 -0
  48. package/dist/src/types.js +1 -0
  49. package/dist/src/validate.d.ts +27 -0
  50. package/dist/src/validate.js +68 -0
  51. package/dist/src/visualizer.d.ts +13 -0
  52. package/dist/src/visualizer.js +133 -0
  53. package/dist/src/worldConfig.d.ts +9 -0
  54. package/dist/src/worldConfig.js +16 -0
  55. package/inject.cjs +429 -0
  56. package/package.json +81 -0
  57. package/src/actions.ts +285 -0
  58. package/src/args.ts +14 -0
  59. package/src/cli.ts +443 -0
  60. package/src/connector.ts +220 -0
  61. package/src/control-plane.ts +66 -0
  62. package/src/egress.ts +355 -0
  63. package/src/fork.ts +256 -0
  64. package/src/index.ts +222 -0
  65. package/src/lease.ts +97 -0
  66. package/src/packRegistry.ts +60 -0
  67. package/src/plan.ts +190 -0
  68. package/src/proxy.ts +180 -0
  69. package/src/pushLedger.ts +189 -0
  70. package/src/queueLifecycle.ts +130 -0
  71. package/src/reconcile.ts +192 -0
  72. package/src/refs.ts +91 -0
  73. package/src/schemas.ts +56 -0
  74. package/src/serve.ts +120 -0
  75. package/src/shadow.ts +192 -0
  76. package/src/status.ts +58 -0
  77. package/src/storage.ts +632 -0
  78. package/src/sync.ts +160 -0
  79. package/src/types.ts +50 -0
  80. package/src/validate.ts +95 -0
  81. package/src/visualizer.ts +142 -0
  82. package/src/worldConfig.ts +26 -0
@@ -0,0 +1,535 @@
1
+ import { appendFileSync, closeSync, existsSync, fsyncSync, mkdirSync, openSync, readdirSync, readFileSync, renameSync, rmSync, statSync, unlinkSync, writeFileSync } from 'node:fs';
2
+ import { hostname } from 'node:os';
3
+ import { dirname, join, resolve } from 'node:path';
4
+ import { GenericWorldStateSchema, WorldServiceEventSchema, } from "./schemas.js";
5
+ function nowIso() {
6
+ return new Date().toISOString();
7
+ }
8
+ function projectRoot(root) {
9
+ return resolve(root || process.env.PROJECT_ROOT || process.cwd());
10
+ }
11
+ /**
12
+ * Name of the per-project state directory holding world data
13
+ * (`<root>/<stateDir>/world/...`). Defaults to `.volter`; hosts that need a
14
+ * different directory set VOLTER_STATE_DIR. Every path in this package must
15
+ * go through worldStateRoot — never the literal.
16
+ */
17
+ export function stateDirName() {
18
+ return process.env.VOLTER_STATE_DIR || '.volter';
19
+ }
20
+ export function worldStateRoot(root) {
21
+ return join(projectRoot(root), stateDirName(), 'world');
22
+ }
23
+ function assertServiceName(service) {
24
+ if (!/^[A-Za-z0-9_-]+$/.test(service)) {
25
+ throw new Error(`Invalid world service name: ${service}`);
26
+ }
27
+ return service;
28
+ }
29
+ export function worldPaths(service, root) {
30
+ const safeService = assertServiceName(service);
31
+ const resolvedRoot = projectRoot(root);
32
+ const dir = join(worldStateRoot(root), safeService);
33
+ return {
34
+ root: resolvedRoot,
35
+ service: safeService,
36
+ dir,
37
+ events: join(dir, 'events.jsonl'),
38
+ state: join(dir, 'state.json'),
39
+ resources: join(dir, 'resources'),
40
+ cursors: join(dir, 'cursors'),
41
+ ingests: join(dir, 'ingests'),
42
+ eventQueue: join(dir, 'event-queue.jsonl'),
43
+ };
44
+ }
45
+ function readJsonl(path) {
46
+ if (!existsSync(path))
47
+ return [];
48
+ const rows = [];
49
+ for (const [index, line] of readFileSync(path, 'utf8').split('\n').entries()) {
50
+ if (!line.trim())
51
+ continue;
52
+ try {
53
+ rows.push(JSON.parse(line));
54
+ }
55
+ catch (error) {
56
+ throw new Error(`${path}:${index + 1}: invalid JSONL row: ${error.message}`);
57
+ }
58
+ }
59
+ return rows;
60
+ }
61
+ /** Read + parse a whole-file JSON sidecar, citing the path on a parse failure (mirrors
62
+ * readJsonl's `${path}:...` error shape). Callers own existence semantics — this parses a
63
+ * file that is expected to exist; guard with existsSync first when absence is allowed. */
64
+ export function readJsonFile(path) {
65
+ try {
66
+ return JSON.parse(readFileSync(path, 'utf8'));
67
+ }
68
+ catch (error) {
69
+ throw new Error(`${path}: invalid JSON: ${error.message}`);
70
+ }
71
+ }
72
+ /**
73
+ * Append `data` to `path`. When VOLTER_DURABLE=1, `fsyncSync` the file after the write so
74
+ * the appended record survives an OS crash / power loss — not just a process crash.
75
+ *
76
+ * Default (VOLTER_DURABLE unset) is OFF: `appendFileSync` is a completed write syscall, so a
77
+ * *process* crash after it returns still leaves the record on the log (the OS owns the page
78
+ * cache). The window this leaves open is a **kernel-panic / power loss** between the write
79
+ * landing in the page cache and the fs flushing it to stable storage — a torn or lost tail
80
+ * record. That crash window matters more once real pulled staging data lives in these files
81
+ * (purpose 2), so hosts that need durability opt in with VOLTER_DURABLE=1 at the cost of an
82
+ * fsync per append. See ARCHITECTURE.md D1.
83
+ */
84
+ export function appendDurable(path, data) {
85
+ const fd = openSync(path, 'a');
86
+ try {
87
+ appendFileSync(fd, data);
88
+ if (process.env.VOLTER_DURABLE === '1')
89
+ fsyncSync(fd);
90
+ }
91
+ finally {
92
+ closeSync(fd);
93
+ }
94
+ }
95
+ /**
96
+ * Opt-in structured stderr logging for the audit trail (ARCHITECTURE-REVIEW-TODOS D3):
97
+ * one line per action-log append and per push-ledger row, so "who reviewed the change
98
+ * that caused this real write" is mechanically greppable from a single log stream via
99
+ * `correlationId`. Off by default — mirrors the VOLTER_DURABLE opt-in policy above: no
100
+ * always-on I/O, never on the default path. Set VOLTER_TWIN_LOG=1 to enable.
101
+ */
102
+ export function twinLog(kind, details) {
103
+ if (process.env.VOLTER_TWIN_LOG !== '1')
104
+ return;
105
+ console.error(`[twin:${kind}]`, JSON.stringify(details));
106
+ }
107
+ function appendJsonl(path, value) {
108
+ mkdirSync(dirname(path), { recursive: true });
109
+ appendDurable(path, `${JSON.stringify(value)}\n`);
110
+ }
111
+ function sleepSync(ms) {
112
+ Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
113
+ }
114
+ /** How long a lock may live before an acquirer treats it as abandoned. Above the 10s
115
+ * wait timeout so a live-but-slow holder is never reclaimed out from under itself. */
116
+ const LOCK_STALE_MS = 60_000;
117
+ function readLockHolder(lockPath) {
118
+ try {
119
+ return JSON.parse(readFileSync(lockPath, 'utf8'));
120
+ }
121
+ catch {
122
+ return null; // missing, empty (mid-write), or malformed
123
+ }
124
+ }
125
+ function pidAlive(pid) {
126
+ try {
127
+ process.kill(pid, 0);
128
+ return true;
129
+ }
130
+ catch (error) {
131
+ // ESRCH → no such process (dead); EPERM → exists but not ours to signal (alive)
132
+ return error.code === 'EPERM';
133
+ }
134
+ }
135
+ /** A lock is stale when its holder is provably gone (same host + dead pid) or it has
136
+ * outlived LOCK_STALE_MS. The age falls back to the lockfile's own mtime when the
137
+ * `at` record is unreadable — so a writer that crashed between creating the lock and
138
+ * recording itself is still eventually reclaimed, while a freshly-created (recent
139
+ * mtime) empty lock is left alone, avoiding a race with the live writer. */
140
+ function lockIsStale(lockPath) {
141
+ const holder = readLockHolder(lockPath);
142
+ if (holder && holder.hostname === hostname() && Number.isInteger(holder.pid) && !pidAlive(holder.pid)) {
143
+ return true;
144
+ }
145
+ const recordedAt = holder && typeof holder.at === 'string' ? Date.parse(holder.at) : NaN;
146
+ let stamp = recordedAt;
147
+ if (!Number.isFinite(stamp)) {
148
+ try {
149
+ stamp = statSync(lockPath).mtimeMs;
150
+ }
151
+ catch {
152
+ return false; // lock vanished — let the next openSync settle it
153
+ }
154
+ }
155
+ return Date.now() - stamp > LOCK_STALE_MS;
156
+ }
157
+ /** Run `fn` holding an exclusive cross-process file lock (the same primitive the event log
158
+ * uses). Used to make read-then-append critical sections atomic across processes. The
159
+ * lockfile records `{pid, hostname, at}`; a contender that finds a stale lock (dead pid or
160
+ * age > LOCK_STALE_MS) reclaims it by atomically renaming it aside — so a crashed holder
161
+ * can't wedge the world forever. Reclaim is race-safe: only the process that wins the
162
+ * rename clears the stale inode, and the exclusive `wx` create still decides the winner. */
163
+ export function withFileLock(lockPath, fn) {
164
+ mkdirSync(dirname(lockPath), { recursive: true });
165
+ const started = Date.now();
166
+ let fd = null;
167
+ while (fd === null) {
168
+ try {
169
+ fd = openSync(lockPath, 'wx');
170
+ }
171
+ catch (error) {
172
+ const code = error.code;
173
+ if (code !== 'EEXIST')
174
+ throw error;
175
+ if (lockIsStale(lockPath)) {
176
+ const holder = readLockHolder(lockPath);
177
+ const salvage = `${lockPath}.stale.${process.pid}.${Date.now()}`;
178
+ try {
179
+ renameSync(lockPath, salvage);
180
+ }
181
+ catch (renameError) {
182
+ if (renameError.code === 'ENOENT')
183
+ continue; // another contender reclaimed it
184
+ throw renameError;
185
+ }
186
+ console.warn(`[world] reclaiming stale storage lock ${lockPath} (held by pid ${holder?.pid ?? '?'} on ${holder?.hostname ?? '?'} since ${holder?.at ?? 'unknown'})`);
187
+ try {
188
+ unlinkSync(salvage);
189
+ }
190
+ catch { /* the moved-aside stale inode; safe to leave if unlink fails */ }
191
+ continue; // retry the exclusive create immediately
192
+ }
193
+ if (Date.now() - started > 10_000) {
194
+ throw new Error(`Timed out waiting for world storage lock: ${lockPath}`);
195
+ }
196
+ sleepSync(25);
197
+ }
198
+ }
199
+ // Record the holder so a later contender can detect staleness. Best-effort: a write
200
+ // failure here doesn't weaken exclusivity, only staleness diagnostics.
201
+ try {
202
+ writeFileSync(fd, `${JSON.stringify({ pid: process.pid, hostname: hostname(), at: new Date().toISOString() })}\n`);
203
+ }
204
+ catch { /* ignore */ }
205
+ try {
206
+ return fn();
207
+ }
208
+ finally {
209
+ closeSync(fd);
210
+ try {
211
+ unlinkSync(lockPath);
212
+ }
213
+ catch { /* already reclaimed by a stale-lock sweep */ }
214
+ }
215
+ }
216
+ function writeJsonAtomic(path, value) {
217
+ mkdirSync(dirname(path), { recursive: true });
218
+ const tmp = `${path}.${process.pid}.${Date.now()}.tmp`;
219
+ writeFileSync(tmp, `${JSON.stringify(value, null, 2)}\n`);
220
+ renameSync(tmp, path);
221
+ }
222
+ export function listEvents(service, root) {
223
+ const paths = worldPaths(service, root);
224
+ return readJsonl(paths.events).map((row) => WorldServiceEventSchema.parse(row));
225
+ }
226
+ const eventsIndexCache = new Map();
227
+ function eventsFileStamp(eventsPath) {
228
+ if (!existsSync(eventsPath))
229
+ return { size: 0, mtimeMs: 0 };
230
+ const stat = statSync(eventsPath);
231
+ return { size: stat.size, mtimeMs: stat.mtimeMs };
232
+ }
233
+ function eventsIndexFor(eventsPath) {
234
+ const { size, mtimeMs } = eventsFileStamp(eventsPath);
235
+ const cached = eventsIndexCache.get(eventsPath);
236
+ if (cached && cached.size === size && cached.mtimeMs === mtimeMs)
237
+ return cached;
238
+ const index = { size, mtimeMs, ids: new Set(), keys: new Set() };
239
+ for (const row of readJsonl(eventsPath)) {
240
+ if (typeof row.id === 'string')
241
+ index.ids.add(row.id);
242
+ if (typeof row.idempotencyKey === 'string')
243
+ index.keys.add(row.idempotencyKey);
244
+ }
245
+ eventsIndexCache.set(eventsPath, index);
246
+ return index;
247
+ }
248
+ /** Targeted fetch of one event row by id or idempotencyKey (rare duplicate path). */
249
+ function findEventRow(eventsPath, id, idempotencyKey) {
250
+ for (const row of readJsonl(eventsPath)) {
251
+ if (row.id === id || row.idempotencyKey === idempotencyKey) {
252
+ return WorldServiceEventSchema.parse(row);
253
+ }
254
+ }
255
+ return null;
256
+ }
257
+ function ensureEventDirs(paths) {
258
+ mkdirSync(paths.dir, { recursive: true });
259
+ mkdirSync(paths.resources, { recursive: true });
260
+ mkdirSync(paths.cursors, { recursive: true });
261
+ mkdirSync(paths.ingests, { recursive: true });
262
+ }
263
+ /** Exported so other modules that share a service's events log (e.g. egress.ts,
264
+ * TWIN-58) can serialize their own critical sections on the SAME lock appendEvent
265
+ * itself uses, instead of inventing a second, uncoordinated lockfile. */
266
+ export function eventsLockPath(paths) {
267
+ return join(paths.dir, 'events.jsonl.lock');
268
+ }
269
+ /**
270
+ * The append body, assuming the caller already holds `eventsLockPath(paths)`. Split
271
+ * out so `commitQueuedEvents` can run a whole batch of appends (and the rebuild that
272
+ * follows them) under ONE lock acquisition instead of nesting a fresh `withFileLock`
273
+ * per row — `withFileLock` is not reentrant, so re-acquiring it from inside an
274
+ * already-held lock in the same process would just spin to its own timeout. Also
275
+ * exported for egress.ts (TWIN-58): performExternalWrite's ledger-check +
276
+ * intent-append span holds `eventsLockPath` itself, so it must append through this
277
+ * already-locked path rather than the public `appendEvent` (which would try to
278
+ * re-acquire the same lock and spin to its own timeout).
279
+ */
280
+ export function appendEventLocked(parsed, paths) {
281
+ const index = eventsIndexFor(paths.events);
282
+ if (index.ids.has(parsed.id) || index.keys.has(parsed.idempotencyKey)) {
283
+ const duplicate = findEventRow(paths.events, parsed.id, parsed.idempotencyKey);
284
+ if (!duplicate)
285
+ throw new Error(`World events index out of sync for ${paths.events}; delete nothing — rerun to rebuild`);
286
+ // observedAt (delivery time), occurredAt (poll-based connectors use
287
+ // poll-cycle time, not the resource's own timestamp), and external.url
288
+ // (a convenience pointer — provider URLs embed mutable title slugs,
289
+ // e.g. Linear) are metadata, not event identity: re-observations
290
+ // differing only there are duplicates, not conflicts.
291
+ // external.provider/id ARE identity.
292
+ const stripMetadata = (event) => {
293
+ const { observedAt: _observedAt, occurredAt: _occurredAt, external, ...content } = event;
294
+ if (!external)
295
+ return content;
296
+ const { url: _url, ...externalIdentity } = external;
297
+ return { ...content, external: externalIdentity };
298
+ };
299
+ if (JSON.stringify(stripMetadata(duplicate)) !== JSON.stringify(stripMetadata(parsed))) {
300
+ throw new Error(`Conflicting duplicate ${parsed.service} event: ${parsed.id} / ${parsed.idempotencyKey} conflicts with ${duplicate.id}`);
301
+ }
302
+ return { event: duplicate, appended: false, duplicateOf: duplicate.id };
303
+ }
304
+ appendJsonl(paths.events, parsed);
305
+ const stamp = eventsFileStamp(paths.events);
306
+ index.ids.add(parsed.id);
307
+ index.keys.add(parsed.idempotencyKey);
308
+ index.size = stamp.size;
309
+ index.mtimeMs = stamp.mtimeMs;
310
+ return { event: parsed, appended: true };
311
+ }
312
+ export function appendEvent(event, root) {
313
+ const parsed = WorldServiceEventSchema.parse(event);
314
+ const paths = worldPaths(parsed.service, root);
315
+ ensureEventDirs(paths);
316
+ return withFileLock(eventsLockPath(paths), () => appendEventLocked(parsed, paths));
317
+ }
318
+ function queuedEventId(event) {
319
+ return `queue:${event.service}:${event.idempotencyKey}`;
320
+ }
321
+ export function listQueuedEvents(service, root) {
322
+ const paths = worldPaths(service, root);
323
+ return readJsonl(paths.eventQueue);
324
+ }
325
+ export function enqueueEvent(event, options = {}) {
326
+ const parsed = WorldServiceEventSchema.parse(event);
327
+ const paths = worldPaths(parsed.service, options.root);
328
+ mkdirSync(paths.dir, { recursive: true });
329
+ mkdirSync(paths.ingests, { recursive: true });
330
+ const queued = {
331
+ id: options.id ?? queuedEventId(parsed),
332
+ service: parsed.service,
333
+ receivedAt: options.receivedAt ?? nowIso(),
334
+ source: options.source ?? 'listener',
335
+ event: parsed,
336
+ };
337
+ return withFileLock(join(paths.dir, 'event-queue.jsonl.lock'), () => {
338
+ const existing = readJsonl(paths.eventQueue);
339
+ const duplicate = existing.find((candidate) => candidate.id === queued.id || candidate.event.idempotencyKey === queued.event.idempotencyKey);
340
+ if (duplicate)
341
+ return { queued: duplicate, appended: false, duplicateOf: duplicate.id };
342
+ appendJsonl(paths.eventQueue, queued);
343
+ return { queued, appended: true };
344
+ });
345
+ }
346
+ /**
347
+ * Commit queued (webhook/listener) events into the canonical log, then rebuild the
348
+ * generic projection — spanning BOTH under the SAME events-lock acquisition (D7).
349
+ * Appending each event and rebuilding state.json are two separate durable writes;
350
+ * without a shared lock, a concurrent commit/append on another process could
351
+ * interleave between "this commit's last append" and "this commit's rebuild",
352
+ * so the rebuilt state.json would not correspond to exactly this commit's view of
353
+ * the log. Holding one lock across the whole sequence rules that out: no other
354
+ * appendEvent/commitQueuedEvents call for this service can run until both the
355
+ * appends AND the rebuild here have completed. (A single-process crash between the
356
+ * last durable append and the rebuild can still leave state.json one rebuild behind
357
+ * — that window is inherent to a two-file update with no WAL, and is unaffected by
358
+ * locking; it is closed by the next successful commit/rebuild, which always starts
359
+ * from the durably-appended log, so no event is ever lost, only state.json's
360
+ * projection is briefly stale.)
361
+ */
362
+ export function commitQueuedEvents(service, options = {}) {
363
+ const paths = worldPaths(service, options.root);
364
+ ensureEventDirs(paths);
365
+ const queued = listQueuedEvents(service, options.root).sort((a, b) => a.event.occurredAt.localeCompare(b.event.occurredAt) || a.receivedAt.localeCompare(b.receivedAt) || a.id.localeCompare(b.id));
366
+ const limit = options.limit ?? queued.length;
367
+ return withFileLock(eventsLockPath(paths), () => {
368
+ let committed = 0;
369
+ let skipped = 0;
370
+ const eventIds = [];
371
+ for (const row of queued.slice(0, limit)) {
372
+ const parsed = WorldServiceEventSchema.parse(row.event);
373
+ const result = appendEventLocked(parsed, paths);
374
+ if (result.appended) {
375
+ committed += 1;
376
+ eventIds.push(result.event.id);
377
+ }
378
+ else {
379
+ skipped += 1;
380
+ }
381
+ }
382
+ if (committed > 0)
383
+ rebuildGenericState(service, options.root);
384
+ return { service, queued: Math.min(queued.length, limit), committed, skipped, eventIds };
385
+ });
386
+ }
387
+ // NOTE: annotation read/write (listAnnotations/addAnnotation/createAnnotation) moved
388
+ // to @volter/tracker/world-annotations — annotations are a tracker concern.
389
+ export function genericWorldReducer(state, event) {
390
+ const key = `${event.subject.type}:${event.subject.id}`;
391
+ return GenericWorldStateSchema.parse({
392
+ ...state,
393
+ eventCount: state.eventCount + 1,
394
+ latestEventId: event.id,
395
+ subjects: {
396
+ ...state.subjects,
397
+ [key]: {
398
+ type: event.subject.type,
399
+ id: event.subject.id,
400
+ latestEventId: event.id,
401
+ latestType: event.type,
402
+ updatedAt: event.occurredAt || event.observedAt,
403
+ },
404
+ },
405
+ });
406
+ }
407
+ export function emptyGenericState(service) {
408
+ return {
409
+ version: 1,
410
+ service: assertServiceName(service),
411
+ rebuiltAt: nowIso(),
412
+ eventCount: 0,
413
+ subjects: {},
414
+ };
415
+ }
416
+ export function rebuildState(service, initialState, reducer, root) {
417
+ const paths = worldPaths(service, root);
418
+ const state = listEvents(service, root).reduce(reducer, initialState);
419
+ writeJsonAtomic(paths.state, state);
420
+ return state;
421
+ }
422
+ export function rebuildGenericState(service, root) {
423
+ return rebuildState(service, emptyGenericState(service), genericWorldReducer, root);
424
+ }
425
+ export function loadState(service, root) {
426
+ const paths = worldPaths(service, root);
427
+ if (!existsSync(paths.state))
428
+ return null;
429
+ return readJsonFile(paths.state);
430
+ }
431
+ export function createEvent(input) {
432
+ return WorldServiceEventSchema.parse({
433
+ schemaVersion: 1,
434
+ observedAt: nowIso(),
435
+ ...input,
436
+ });
437
+ }
438
+ /** Filenames/dirnames ANY control-plane src module ever writes inside a single
439
+ * service's state dir (see worldPaths) — plus the lockfiles withFileLock creates
440
+ * next to a log and the `.tmp`/`.stale.*` sidecars its atomic-write/reclaim paths
441
+ * use. storage.ts itself only accounts for events.jsonl/event-queue.jsonl/
442
+ * state.json/resources/cursors/ingests; the rest are written by other modules
443
+ * that share the same service dir (actions.ts, pushLedger.ts, plan.ts, lease.ts,
444
+ * refs.ts, fork.ts, queueLifecycle.ts) — this set MUST stay in sync with every
445
+ * one of them so scrubService never demands --force for an ordinary twin. */
446
+ const KNOWN_SERVICE_ENTRIES = new Set([
447
+ 'events.jsonl',
448
+ 'events.jsonl.lock',
449
+ 'event-queue.jsonl',
450
+ 'event-queue.jsonl.lock',
451
+ 'state.json',
452
+ 'resources',
453
+ 'cursors',
454
+ 'ingests',
455
+ // actions.ts: the local transaction-commit log + its lock.
456
+ 'actions.jsonl',
457
+ 'actions.jsonl.lock',
458
+ // pushLedger.ts: the push-phase ledger for real-vendor replication.
459
+ 'push-ledger.jsonl',
460
+ // queueLifecycle.ts: the append-only status ledger over event-queue.jsonl rows.
461
+ 'event-queue-status.jsonl',
462
+ // fork.ts: fork metadata (base snapshot + divergence baseline) for a fork root.
463
+ 'fork-meta.json',
464
+ // plan.ts: one JSON file per apply plan, under plans/<planId>.json.
465
+ 'plans',
466
+ // lease.ts: one JSON file per apply lease, under leases/<leaseId>.json.
467
+ 'leases',
468
+ // refs.ts: remote/local checkpoint refs, under refs/remote/<provider>/<name>.json
469
+ // and refs/local/<forkId>.json.
470
+ 'refs',
471
+ ]);
472
+ function isKnownServiceEntry(name) {
473
+ return KNOWN_SERVICE_ENTRIES.has(name) || name.endsWith('.tmp') || /\.stale\.\d+\.\d+$/.test(name);
474
+ }
475
+ /** A service dir "looks like" one of ours when every entry in it is something
476
+ * storage.ts is known to create. A directory that doesn't exist yet trivially
477
+ * looks fine (scrub will just no-op on it). */
478
+ function looksLikeServiceStateDir(dir) {
479
+ if (!existsSync(dir))
480
+ return true;
481
+ return readdirSync(dir).every((entry) => isKnownServiceEntry(entry));
482
+ }
483
+ /** The whole world dir "looks like" ours when every entry in it is itself a
484
+ * directory that looks like a service state dir (each service gets one subdir
485
+ * under worldStateRoot — see worldPaths). */
486
+ function looksLikeWorldStateDir(dir) {
487
+ if (!existsSync(dir))
488
+ return true;
489
+ return readdirSync(dir).every((entry) => {
490
+ const full = join(dir, entry);
491
+ return statSync(full).isDirectory() && looksLikeServiceStateDir(full);
492
+ });
493
+ }
494
+ /** Every file under `dir`, path relative to `dir`, depth-first. Used to report
495
+ * exactly what scrub is about to remove before it removes it. */
496
+ function listFilesRecursive(dir, base = dir) {
497
+ if (!existsSync(dir))
498
+ return [];
499
+ const out = [];
500
+ for (const entry of readdirSync(dir, { withFileTypes: true })) {
501
+ const full = join(dir, entry.name);
502
+ if (entry.isDirectory())
503
+ out.push(...listFilesRecursive(full, base));
504
+ else
505
+ out.push(full.slice(base.length + 1));
506
+ }
507
+ return out;
508
+ }
509
+ function scrubDir(dir, label, looksLike, options) {
510
+ if (!existsSync(dir)) {
511
+ return { target: dir, removed: [], message: `nothing to scrub: ${label} (${dir}) does not exist` };
512
+ }
513
+ if (!options.force && !looksLike(dir)) {
514
+ throw new Error(`world scrub: refusing to remove "${dir}" — it does not look like a world state directory ` +
515
+ `(unexpected contents). Pass --force (or { force: true }) to scrub it anyway.`);
516
+ }
517
+ const removed = listFilesRecursive(dir);
518
+ rmSync(dir, { recursive: true, force: true });
519
+ return { target: dir, removed, message: `scrubbed ${label}: removed ${removed.length} file(s) under ${dir}` };
520
+ }
521
+ /** Delete one service's pulled data at rest: its event log, queued events,
522
+ * rebuilt state.json, and any resources/cursors/ingests sidecars — everything
523
+ * `worldPaths(service)` points at. Refuses (unless `force`) when the dir holds
524
+ * anything storage.ts didn't put there. */
525
+ export function scrubService(service, options = {}) {
526
+ const paths = worldPaths(service, options.root);
527
+ return scrubDir(paths.dir, `service "${service}"`, looksLikeServiceStateDir, options);
528
+ }
529
+ /** Delete the ENTIRE world state dir (`<root>/<stateDir>/world`) — every
530
+ * service's pulled data at once. Refuses (unless `force`) when any entry in it
531
+ * isn't itself a recognizable per-service state dir. */
532
+ export function scrubWorld(options = {}) {
533
+ const dir = worldStateRoot(options.root);
534
+ return scrubDir(dir, 'entire world state dir', looksLikeWorldStateDir, options);
535
+ }
@@ -0,0 +1,91 @@
1
+ import type { EgressWriteResult } from './egress.js';
2
+ import type { WorldRemoteRef } from './refs.js';
3
+ import type { SubjectFields } from './shadow.js';
4
+ import type { TwinResource } from './serve.js';
5
+ import type { ReconcilePlan } from './reconcile.js';
6
+ export type SyncResource = {
7
+ type: string;
8
+ id: string;
9
+ fields: SubjectFields;
10
+ };
11
+ export type PullResult = {
12
+ observed: number;
13
+ deltasAppended: number;
14
+ unchanged: number;
15
+ };
16
+ /**
17
+ * Pull: fold a set of externally-observed resources into the twin's event log.
18
+ * The caller has already fetched them from the real vendor (injected I/O); this
19
+ * is the pure fold. Only changed fields produce a delta (recordObservedDelta is
20
+ * a no-op when nothing differs), so re-pulling identical state appends nothing.
21
+ *
22
+ * `redact` (TWIN-45 dev/01 — implemented, not just documented) is an optional
23
+ * transform applied to each resource BEFORE it is folded into the event log, so
24
+ * a sensitive field never lands on disk in the first place — a redact-on-pull
25
+ * hook point, not a redaction policy: this library makes zero default redaction
26
+ * decisions (no built-in "PII field" list, no defaults). The caller supplies the
27
+ * hook deliberately, shaped for their own vendor/fields. See docs/DATA_AT_REST.md
28
+ * for the full data-at-rest story (what's implemented here vs. left as
29
+ * guidance) and for why day-count retention defaults are NOT hardcoded here —
30
+ * that is a policy call for the human owner, not this library.
31
+ */
32
+ export declare function syncPull(opts: {
33
+ service: string;
34
+ resources: SyncResource[];
35
+ occurredAt: string;
36
+ root?: string;
37
+ actor?: {
38
+ kind: 'agent' | 'human' | 'bot' | 'system';
39
+ id?: string;
40
+ };
41
+ redact?: (resource: SyncResource) => SyncResource;
42
+ }): PullResult;
43
+ export type PushItemResult = {
44
+ id: string;
45
+ type: string;
46
+ status: EgressWriteResult['status'];
47
+ externalId?: string;
48
+ };
49
+ export type PushResult = {
50
+ attempted: number;
51
+ pushed: PushItemResult[];
52
+ };
53
+ /**
54
+ * Push: enact a reconcile plan's twin→real changes by calling the injected real
55
+ * write fn, idempotently via the egress ledger. `write` is the ONLY place a real
56
+ * vendor API is touched; in tests/offline it's a fake. Re-pushing the same plan
57
+ * replays (no double-apply) because the egress idempotency key encodes the change.
58
+ *
59
+ * Gated the same way `applyPlan` is (TWIN-53 R-K2): `approve` and the lease fields
60
+ * (`remoteRef`/`holder`/`leaseId`/`acquiredAt`/`expiresAt`) are mandatory options —
61
+ * there is no implicit-approval, no-lease path. Approval is required when
62
+ * `reconcileRequiresApproval(plan)` is true (unresolved conflicts, or a
63
+ * twin-wins push that overwrites a real-side change) and `approve` is not set;
64
+ * refusal happens before the lease is acquired or `write` is ever called — zero
65
+ * side effects. The lease is acquired for `remoteRef` before any push and released
66
+ * (even on error) after.
67
+ */
68
+ export declare function syncPush(opts: {
69
+ service: string;
70
+ plan: ReconcilePlan;
71
+ write: (item: SyncResource) => Promise<{
72
+ externalId: string;
73
+ data?: Record<string, unknown>;
74
+ }>;
75
+ remoteRef: WorldRemoteRef;
76
+ holder: {
77
+ kind: 'agent' | 'human' | 'system';
78
+ id: string;
79
+ };
80
+ leaseId: string;
81
+ acquiredAt: string;
82
+ expiresAt: string;
83
+ approve: boolean;
84
+ root?: string;
85
+ actor?: {
86
+ kind: 'agent' | 'human' | 'bot' | 'system';
87
+ id?: string;
88
+ };
89
+ }): Promise<PushResult>;
90
+ /** Convenience: the twin's current resources as the reconcile `fork`/state input. */
91
+ export declare function currentResources(service: string, root?: string): TwinResource[];