@atolis-hq/wake 0.4.101 → 0.4.103

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/dist/src/activities/pr/policy.d.ts +2 -0
  2. package/dist/src/activities/pr/policy.js +9 -1
  3. package/dist/src/artifacts/application/operator-instructions-service.js +2 -2
  4. package/dist/src/bootstrap/artifact-mcp.js +3 -4
  5. package/dist/src/bootstrap/delegation-authority.js +1 -1
  6. package/dist/src/bootstrap/integration-runtime.js +1 -1
  7. package/dist/src/bootstrap/persistence-composition.js +1 -1
  8. package/dist/src/bootstrap/surface-api-delivery.d.ts +1 -1
  9. package/dist/src/bootstrap/surface-api-delivery.js +8 -4
  10. package/dist/src/bootstrap/version.js +1 -1
  11. package/dist/src/control-plane/application/activation-scheduler-ports.d.ts +2 -1
  12. package/dist/src/control-plane/application/activation-scheduler.js +2 -1
  13. package/dist/src/control-plane/application/advance-once-dispatch.d.ts +2 -4
  14. package/dist/src/control-plane/application/advance-once-dispatch.js +6 -4
  15. package/dist/src/control-plane/domain/delivery-capacity.d.ts +2 -5
  16. package/dist/src/control-plane/domain/delivery-capacity.js +3 -4
  17. package/dist/src/control-plane/domain/open-work-policy.d.ts +2 -10
  18. package/dist/src/control-plane/domain/open-work-policy.js +2 -3
  19. package/dist/src/execution/application/run-repository.d.ts +2 -2
  20. package/dist/src/execution/application/run-repository.js +3 -3
  21. package/dist/src/integrations/application/delegation-outputs.js +41 -5
  22. package/dist/src/node_modules/@atolis-hq/eventing/dist/runtime/cached-journal-view.d.ts +9 -1
  23. package/dist/src/node_modules/@atolis-hq/eventing/dist/runtime/cached-journal-view.d.ts.map +1 -1
  24. package/dist/src/node_modules/@atolis-hq/eventing/dist/runtime/cached-journal-view.js +62 -7
  25. package/dist/src/node_modules/@atolis-hq/eventing/dist/store/event-journal.d.ts +2 -5
  26. package/dist/src/node_modules/@atolis-hq/eventing/dist/store/event-journal.d.ts.map +1 -1
  27. package/dist/src/node_modules/@atolis-hq/eventing-filesystem/dist/file-event-journal.d.ts +12 -13
  28. package/dist/src/node_modules/@atolis-hq/eventing-filesystem/dist/file-event-journal.d.ts.map +1 -1
  29. package/dist/src/node_modules/@atolis-hq/eventing-filesystem/dist/file-event-journal.js +168 -109
  30. package/dist/src/orchestration/application/deferred-signals.d.ts +4 -2
  31. package/dist/src/orchestration/application/deferred-signals.js +47 -37
  32. package/dist/src/orchestration/application/delegated-work.d.ts +48 -2
  33. package/dist/src/orchestration/application/delegated-work.js +145 -27
  34. package/dist/src/orchestration/application/delegation-commands.d.ts +48 -1
  35. package/dist/src/orchestration/application/delegation-commands.js +5 -4
  36. package/dist/src/orchestration/application/delegation-gate.d.ts +16 -7
  37. package/dist/src/orchestration/application/delegation-gate.js +62 -14
  38. package/dist/src/orchestration/application/delegation-ledger.d.ts +2 -0
  39. package/dist/src/orchestration/application/delegation-ledger.js +8 -0
  40. package/dist/src/orchestration/application/delegation-prerequisites.d.ts +28 -0
  41. package/dist/src/orchestration/application/delegation-prerequisites.js +118 -0
  42. package/dist/src/orchestration/application/delegation-reservation.js +1 -1
  43. package/dist/src/orchestration/application/delegation-wait-graph.d.ts +7 -0
  44. package/dist/src/orchestration/application/delegation-wait-graph.js +122 -0
  45. package/dist/src/orchestration/application/orchestration-repository.d.ts +5 -0
  46. package/dist/src/orchestration/application/orchestration-repository.js +3 -0
  47. package/dist/src/orchestration/application/orchestration-service.d.ts +87 -0
  48. package/dist/src/orchestration/application/orchestration-service.js +13 -7
  49. package/dist/src/orchestration/application/prerequisite-readiness.d.ts +68 -0
  50. package/dist/src/orchestration/application/prerequisite-readiness.js +217 -0
  51. package/dist/src/orchestration/contracts/delegation-ports.d.ts +2 -1
  52. package/dist/src/orchestration/contracts/delegation.d.ts +88 -0
  53. package/dist/src/orchestration/contracts/delegation.js +36 -1
  54. package/dist/src/orchestration/contracts/event-decoder.js +8 -1
  55. package/dist/src/orchestration/contracts/event-factory.js +5 -4
  56. package/dist/src/orchestration/contracts/events.d.ts +13 -1
  57. package/dist/src/orchestration/contracts/events.js +3 -0
  58. package/dist/src/orchestration/contracts/vocabulary.d.ts +27 -0
  59. package/dist/src/orchestration/contracts/vocabulary.js +25 -0
  60. package/dist/src/orchestration/domain/workflow-instance-events.js +3 -0
  61. package/dist/src/orchestration/index.d.ts +2 -2
  62. package/dist/src/orchestration/index.js +1 -1
  63. package/dist/src/surfaces/mcp/delegation.js +18 -10
  64. package/dist/src/work/application/work-index.d.ts +21 -0
  65. package/dist/src/work/application/work-index.js +73 -0
  66. package/dist/src/work/application/work-repository.d.ts +2 -2
  67. package/dist/src/work/application/work-repository.js +2 -2
  68. package/dist/src/work/application/work-service.d.ts +5 -2
  69. package/dist/src/work/application/work-service.js +5 -2
  70. package/dist/src/work/{application → domain}/delivery-availability.d.ts +4 -7
  71. package/dist/src/work/{application → domain}/delivery-availability.js +3 -3
  72. package/dist/src/work/index.d.ts +2 -1
  73. package/dist/src/work/index.js +2 -1
  74. package/package.json +2 -1
@@ -1,4 +1,4 @@
1
- import type { EventData, EventEnvelope, EventingClock, EventJournal, JournalChangeSignal, JournalReadOptions, StreamRef } from '@atolis-hq/eventing';
1
+ import type { EventData, EventEnvelope, EventingClock, EventJournal, JournalChangeSignal, StreamRef } from '@atolis-hq/eventing';
2
2
  export interface FileEventJournalWatcher {
3
3
  unref(): void;
4
4
  close(): void;
@@ -7,13 +7,11 @@ export interface FileEventJournalWatcher {
7
7
  export type FileEventJournalWatcherFactory = (directory: string, notify: () => void) => Promise<FileEventJournalWatcher>;
8
8
  export interface FileEventJournalOptions {
9
9
  readonly watcherFactory?: FileEventJournalWatcherFactory;
10
- readonly validationNow?: () => number;
11
10
  }
12
11
  export declare class FileEventJournal implements EventJournal {
13
12
  private readonly root;
14
13
  private readonly clock;
15
14
  private readonly options;
16
- private readonly validationNow;
17
15
  constructor(root: string, clock: EventingClock, options?: FileEventJournalOptions);
18
16
  private readonly changeSignalSource;
19
17
  private changeWatcher;
@@ -21,13 +19,15 @@ export declare class FileEventJournal implements EventJournal {
21
19
  get changeSignal(): JournalChangeSignal;
22
20
  private cached;
23
21
  private inFlightScan;
24
- private scanGeneration;
25
- private validatedEntries;
26
- private inFlightEntryRefresh;
27
- private entryValidationGeneration;
22
+ private readonly knownSegments;
23
+ private latestEntries;
24
+ private listing;
25
+ private runningListing;
26
+ private queuedListing;
27
+ private temporariesReclaimed;
28
28
  appendToStream(stream: StreamRef, expectedSequence: number, drafts: readonly EventData[]): Promise<readonly EventEnvelope[]>;
29
29
  private nextSegmentFile;
30
- readStream(stream: StreamRef, options?: JournalReadOptions): Promise<readonly EventEnvelope<EventData<string, unknown>, StreamRef<string, string>>[]>;
30
+ readStream(stream: StreamRef): Promise<readonly EventEnvelope<EventData<string, unknown>, StreamRef<string, string>>[]>;
31
31
  /** Live readers bypass the full-history cache and stop at captured segment sizes. */
32
32
  scanAll(signal?: AbortSignal): AsyncIterable<EventEnvelope>;
33
33
  private scanSegment;
@@ -39,15 +39,14 @@ export declare class FileEventJournal implements EventJournal {
39
39
  private ensureChangeWatcher;
40
40
  private startChangeWatcher;
41
41
  private extendCache;
42
+ private adopt;
42
43
  private manifestPath;
43
- private cachedEventsForStream;
44
44
  private persistManifest;
45
45
  private loadManifest;
46
46
  private scan;
47
- private readCurrentEntries;
48
- private readEntriesForRead;
49
- private rememberValidatedEntries;
50
- private invalidateValidatedEntries;
47
+ private currentEntries;
48
+ private startListing;
49
+ private listSegments;
51
50
  private parseIndexedEntries;
52
51
  private decodeIndexedRecord;
53
52
  private scanUncoalesced;
@@ -1 +1 @@
1
- {"version":3,"file":"file-event-journal.d.ts","sourceRoot":"","sources":["../src/file-event-journal.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,SAAS,EACT,aAAa,EACb,aAAa,EACb,YAAY,EACZ,mBAAmB,EACnB,kBAAkB,EAClB,SAAS,EACV,MAAM,qBAAqB,CAAC;AAW7B,MAAM,WAAW,uBAAuB;IACtC,KAAK,IAAI,IAAI,CAAC;IACd,KAAK,IAAI,IAAI,CAAC;IACd,IAAI,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,IAAI,GAAG,OAAO,CAAC;CACrD;AAED,MAAM,MAAM,8BAA8B,GAAG,CAC3C,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,IAAI,KACf,OAAO,CAAC,uBAAuB,CAAC,CAAC;AAEtC,MAAM,WAAW,uBAAuB;IACtC,QAAQ,CAAC,cAAc,CAAC,EAAE,8BAA8B,CAAC;IACzD,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,MAAM,CAAC;CACvC;AAOD,qBAAa,gBAAiB,YAAW,YAAY;IAIjD,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,OAAO;IAL1B,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAe;gBAG1B,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,aAAa,EACpB,OAAO,GAAE,uBAA4B;IAKxD,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAsC;IACzE,OAAO,CAAC,aAAa,CAAsC;IAC3D,OAAO,CAAC,qBAAqB,CAA4B;IAEzD,IAAI,YAAY,IAAI,mBAAmB,CAEtC;IAED,OAAO,CAAC,MAAM,CAOA;IAWd,OAAO,CAAC,YAAY,CAC4E;IAEhG,OAAO,CAAC,cAAc,CAAK;IAC3B,OAAO,CAAC,gBAAgB,CAC4D;IAEpF,OAAO,CAAC,oBAAoB,CAA2C;IACvE,OAAO,CAAC,yBAAyB,CAAK;IAEhC,cAAc,CAClB,MAAM,EAAE,SAAS,EACjB,gBAAgB,EAAE,MAAM,EACxB,MAAM,EAAE,SAAS,SAAS,EAAE,GAC3B,OAAO,CAAC,SAAS,aAAa,EAAE,CAAC;YAwCtB,eAAe;IAsBvB,UAAU,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,GAAE,kBAAuB;IA6BpE,qFAAqF;IAC9E,OAAO,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,aAAa,CAAC,aAAa,CAAC;YAcnD,WAAW;IAgC1B,OAAO,CAAC,gBAAgB;IAoBlB,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,SAA2B;IAyBvD,UAAU,CAAC,oBAAoB,CAAC,EAAE,MAAM,EAAE,KAAK,SAA2B;IAW1E,oBAAoB,IAAI,OAAO,CAAC,MAAM,CAAC;IAKvC,kBAAkB,CACtB,mBAAmB,EAAE,MAAM,EAC3B,MAAM,EAAE,WAAW,EACnB,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,IAAI,CAAC;IAWhB,OAAO,CAAC,mBAAmB;YAUb,kBAAkB;YA4BlB,WAAW;IAkDzB,OAAO,CAAC,YAAY;IAIpB,OAAO,CAAC,qBAAqB;YAQf,eAAe;YAMf,YAAY;YAYZ,IAAI;YAYJ,kBAAkB;YAmBlB,kBAAkB;IAkBhC,OAAO,CAAC,wBAAwB;IAYhC,OAAO,CAAC,0BAA0B;YAQpB,mBAAmB;IA0BjC,OAAO,CAAC,mBAAmB;YAwBb,eAAe;CAkD9B"}
1
+ {"version":3,"file":"file-event-journal.d.ts","sourceRoot":"","sources":["../src/file-event-journal.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,SAAS,EACT,aAAa,EACb,aAAa,EACb,YAAY,EACZ,mBAAmB,EACnB,SAAS,EACV,MAAM,qBAAqB,CAAC;AAU7B,MAAM,WAAW,uBAAuB;IACtC,KAAK,IAAI,IAAI,CAAC;IACd,KAAK,IAAI,IAAI,CAAC;IACd,IAAI,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,IAAI,GAAG,OAAO,CAAC;CACrD;AAED,MAAM,MAAM,8BAA8B,GAAG,CAC3C,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,IAAI,KACf,OAAO,CAAC,uBAAuB,CAAC,CAAC;AAEtC,MAAM,WAAW,uBAAuB;IACtC,QAAQ,CAAC,cAAc,CAAC,EAAE,8BAA8B,CAAC;CAC1D;AAWD,qBAAa,gBAAiB,YAAW,YAAY;IAEjD,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,OAAO;gBAFP,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,aAAa,EACpB,OAAO,GAAE,uBAA4B;IAGxD,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAsC;IACzE,OAAO,CAAC,aAAa,CAAsC;IAC3D,OAAO,CAAC,qBAAqB,CAA4B;IAEzD,IAAI,YAAY,IAAI,mBAAmB,CAEtC;IAKD,OAAO,CAAC,MAAM,CAA8B;IAW5C,OAAO,CAAC,YAAY,CAC4E;IAQhG,OAAO,CAAC,QAAQ,CAAC,aAAa,CAA+B;IAC7D,OAAO,CAAC,aAAa,CAA2B;IAChD,OAAO,CAAC,OAAO,CAA+E;IAE9F,OAAO,CAAC,cAAc,CAA2C;IACjE,OAAO,CAAC,aAAa,CAA2C;IAChE,OAAO,CAAC,oBAAoB,CAAS;IAE/B,cAAc,CAClB,MAAM,EAAE,SAAS,EACjB,gBAAgB,EAAE,MAAM,EACxB,MAAM,EAAE,SAAS,SAAS,EAAE,GAC3B,OAAO,CAAC,SAAS,aAAa,EAAE,CAAC;YA4CtB,eAAe;IAsBvB,UAAU,CAAC,MAAM,EAAE,SAAS;IA2BlC,qFAAqF;IAC9E,OAAO,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,aAAa,CAAC,aAAa,CAAC;YAcnD,WAAW;IAgC1B,OAAO,CAAC,gBAAgB;IAoBlB,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,SAA2B;IA0BvD,UAAU,CAAC,oBAAoB,CAAC,EAAE,MAAM,EAAE,KAAK,SAA2B;IAW1E,oBAAoB,IAAI,OAAO,CAAC,MAAM,CAAC;IAKvC,kBAAkB,CACtB,mBAAmB,EAAE,MAAM,EAC3B,MAAM,EAAE,WAAW,EACnB,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,IAAI,CAAC;IAWhB,OAAO,CAAC,mBAAmB;YAUb,kBAAkB;YAyBlB,WAAW;IAuDzB,OAAO,CAAC,KAAK;IAKb,OAAO,CAAC,YAAY;YAQN,eAAe;YAKf,YAAY;YAYZ,IAAI;IAelB,OAAO,CAAC,cAAc;IActB,OAAO,CAAC,YAAY;YAQN,YAAY;YA4DZ,mBAAmB;IA0BjC,OAAO,CAAC,mBAAmB;YAwBb,eAAe;CAuD9B"}
@@ -3,22 +3,23 @@ import { InProcessJournalChangeSignal } from '@atolis-hq/eventing/memory';
3
3
  import { createReadStream, watch } from 'node:fs';
4
4
  import { mkdir, open, readdir, readFile, rm, stat } from 'node:fs/promises';
5
5
  import { join } from 'node:path';
6
- import { performance } from 'node:perf_hooks';
7
6
  import { writeFileAtomically } from './atomic-write.js';
8
7
  import { decodeEventRecord, encodeEventRecord } from './event-record-codec.js';
9
8
  import { withFileLock } from './file-lock.js';
10
9
  const createFileSystemWatcher = async (directory, notify) => watch(directory, { persistent: false }, notify);
11
- const entryValidationTtlMs = 30_000;
10
+ // A directory timestamp only proves "unchanged" once it is older than the
11
+ // filesystem's timestamp granularity, otherwise a second change inside the same
12
+ // tick would leave it identical. Younger stamps are re-listed, never trusted.
13
+ const unchangedStampMinAgeMs = 2_500;
14
+ const statBatchSize = 256;
12
15
  export class FileEventJournal {
13
16
  root;
14
17
  clock;
15
18
  options;
16
- validationNow;
17
19
  constructor(root, clock, options = {}) {
18
20
  this.root = root;
19
21
  this.clock = clock;
20
22
  this.options = options;
21
- this.validationNow = options.validationNow ?? (() => performance.now());
22
23
  }
23
24
  changeSignalSource = new InProcessJournalChangeSignal();
24
25
  changeWatcher;
@@ -26,6 +27,9 @@ export class FileEventJournal {
26
27
  get changeSignal() {
27
28
  return this.changeSignalSource;
28
29
  }
30
+ // The most complete snapshot seen so far. Readers never build on it after an
31
+ // await: a scan returns its own snapshot, because a slower scan of older
32
+ // entries may finish after a newer one and must not be mistaken for it.
29
33
  cached;
30
34
  // Every read (readStream/readAll/readLatest/latestGlobalPosition) reaches
31
35
  // scan(), and the resident host's tick loop calls those many times a
@@ -37,17 +41,29 @@ export class FileEventJournal {
37
41
  // only for the same segment fingerprints keeps that protection without
38
42
  // letting an append reuse a scan from before another process wrote.
39
43
  inFlightScan;
40
- scanGeneration = 0;
41
- validatedEntries;
42
- inFlightEntryRefresh;
43
- entryValidationGeneration = 0;
44
+ // Segments are immutable: each is published by atomic rename to a name that is
45
+ // never reused, and nothing rewrites or deletes one. A segment seen once is
46
+ // therefore never stat'ed again, and a listing costs a directory stat, one
47
+ // readdir and a stat of the names not seen before. There is no validity
48
+ // window: every read reflects every append committed before it was called,
49
+ // including appends by other processes sharing the directory.
50
+ knownSegments = new Map();
51
+ latestEntries = [];
52
+ listing;
53
+ runningListing;
54
+ queuedListing;
55
+ temporariesReclaimed = false;
44
56
  async appendToStream(stream, expectedSequence, drafts) {
45
57
  if (drafts.length === 0)
46
58
  throw new Error('appendToStream requires at least one event');
47
59
  return withFileLock(join(this.root, 'locks', 'event-journal.lock'), async () => {
48
- await reclaimStaleSegmentTemps(join(this.root, 'events'));
49
- const current = await this.scan();
50
- const streamEvents = this.cachedEventsForStream(stream);
60
+ if (!this.temporariesReclaimed) {
61
+ await reclaimStaleSegmentTemps(join(this.root, 'events'));
62
+ this.temporariesReclaimed = true;
63
+ }
64
+ const snapshot = await this.scan();
65
+ const current = snapshot.events;
66
+ const streamEvents = snapshot.eventsByStream.get(key(stream)) ?? [];
51
67
  if (streamEvents.length !== expectedSequence)
52
68
  throw new WrongExpectedSequenceError(`Expected sequence ${expectedSequence} for ${key(stream)}, actual ${streamEvents.length}`);
53
69
  const recordedAt = this.clock.now().toISOString();
@@ -63,11 +79,11 @@ export class FileEventJournal {
63
79
  await mkdir(directory, { recursive: true });
64
80
  const file = await this.nextSegmentFile(directory, recordedAt, current.length + 1);
65
81
  await writeSegmentAtomically(join(directory, file), finalizedEnvelopes.map(encodeEventRecord).join('\n') + '\n');
66
- await this.extendCache(file, current, finalizedEnvelopes);
82
+ const next = await this.extendCache(file, snapshot, finalizedEnvelopes);
67
83
  // The manifest is derived data. The event append is authoritative, so
68
84
  // an index-write failure must not turn a successfully recorded event
69
85
  // into a failed append.
70
- await this.persistManifest().catch(() => undefined);
86
+ await this.persistManifest(next).catch(() => undefined);
71
87
  this.changeSignalSource.notify();
72
88
  return finalizedEnvelopes;
73
89
  }, { waitMs: 5_000, retryIntervalMs: 25 });
@@ -91,16 +107,13 @@ export class FileEventJournal {
91
107
  // the entire history. A missing or stale manifest degrades to scan()'s
92
108
  // full parse rather than to incorrect data. The next successful append
93
109
  // rebuilds the derived manifest.
94
- async readStream(stream, options = {}) {
110
+ async readStream(stream) {
95
111
  const streamKey = key(stream);
96
- if (options.fresh === true)
97
- this.invalidateValidatedEntries();
98
- const entries = await this.readEntriesForRead();
99
- if (this.cached !== undefined) {
100
- if (sameEntries(this.cached.entries, entries))
101
- return this.cached.eventsByStream.get(streamKey) ?? [];
102
- await this.scan(entries);
103
- return this.cachedEventsForStream(stream);
112
+ const entries = await this.currentEntries();
113
+ const warm = this.cached;
114
+ if (warm !== undefined) {
115
+ const snapshot = sameEntries(warm.entries, entries) ? warm : await this.scan(entries);
116
+ return snapshot.eventsByStream.get(streamKey) ?? [];
104
117
  }
105
118
  const manifest = await this.loadManifest();
106
119
  if (manifest !== undefined && isCompleteIndex(manifest, entries)) {
@@ -115,12 +128,12 @@ export class FileEventJournal {
115
128
  // authority and retains the original read semantics.
116
129
  }
117
130
  }
118
- return (await this.scan(entries)).filter((event) => key(event.stream) === streamKey);
131
+ return (await this.scan(entries)).events.filter((event) => key(event.stream) === streamKey);
119
132
  }
120
133
  /** Live readers bypass the full-history cache and stop at captured segment sizes. */
121
134
  async *scanAll(signal) {
122
135
  signal?.throwIfAborted();
123
- const entries = await this.readCurrentEntries();
136
+ const entries = await this.currentEntries();
124
137
  let position = 0;
125
138
  for (const entry of entries) {
126
139
  let lineNumber = 0;
@@ -176,9 +189,10 @@ export class FileEventJournal {
176
189
  }
177
190
  }
178
191
  async readAll(after, limit = Number.POSITIVE_INFINITY) {
179
- const entries = await this.readEntriesForRead();
180
- if (this.cached !== undefined && sameEntries(this.cached.entries, entries))
181
- return this.cached.events.filter((event) => event.globalPosition > after).slice(0, limit);
192
+ const entries = await this.currentEntries();
193
+ const warm = this.cached;
194
+ if (warm !== undefined && sameEntries(warm.entries, entries))
195
+ return warm.events.filter((event) => event.globalPosition > after).slice(0, limit);
182
196
  const manifest = await this.loadManifest();
183
197
  if (manifest !== undefined && isCompleteIndex(manifest, entries)) {
184
198
  try {
@@ -192,7 +206,7 @@ export class FileEventJournal {
192
206
  // See readStream(): never let a derived index change journal reads.
193
207
  }
194
208
  }
195
- return (await this.scan(entries))
209
+ return (await this.scan(entries)).events
196
210
  .filter((event) => event.globalPosition > after)
197
211
  .slice(0, limit);
198
212
  }
@@ -201,7 +215,7 @@ export class FileEventJournal {
201
215
  return [];
202
216
  if (limit <= 0)
203
217
  return [];
204
- const events = await this.scan(await this.readEntriesForRead());
218
+ const { events } = await this.scan(await this.currentEntries());
205
219
  const before = beforeGlobalPosition ?? events.length + 1;
206
220
  return events
207
221
  .slice(0, before - 1)
@@ -209,7 +223,7 @@ export class FileEventJournal {
209
223
  .reverse();
210
224
  }
211
225
  async latestGlobalPosition() {
212
- const events = await this.scan(await this.readEntriesForRead());
226
+ const { events } = await this.scan(await this.currentEntries());
213
227
  return events.at(-1)?.globalPosition ?? 0;
214
228
  }
215
229
  async waitForEventsAfter(afterGlobalPosition, signal, fallbackMs) {
@@ -232,10 +246,7 @@ export class FileEventJournal {
232
246
  const directory = join(this.root, 'events');
233
247
  try {
234
248
  await mkdir(directory, { recursive: true });
235
- const watcher = await (this.options.watcherFactory ?? createFileSystemWatcher)(directory, () => {
236
- this.invalidateValidatedEntries();
237
- this.changeSignalSource.notify();
238
- });
249
+ const watcher = await (this.options.watcherFactory ?? createFileSystemWatcher)(directory, () => this.changeSignalSource.notify());
239
250
  watcher.unref();
240
251
  watcher.once('error', () => {
241
252
  if (this.changeWatcher === watcher)
@@ -254,14 +265,15 @@ export class FileEventJournal {
254
265
  // sharing this instance) to re-parse the entire on-disk history. Only a
255
266
  // change this instance didn't make itself — another process writing to the
256
267
  // same journal — falls through to scan()'s full re-read.
257
- async extendCache(file, priorEvents, newEnvelopes) {
268
+ async extendCache(file, prior, newEnvelopes) {
269
+ const priorEvents = prior.events;
258
270
  const info = await stat(join(this.root, 'events', file));
259
271
  const updatedEntry = { file, size: info.size, mtimeMs: info.mtimeMs };
260
- const priorEntries = this.cached?.entries ?? [];
272
+ const priorEntries = prior.entries;
261
273
  const entries = priorEntries.some((entry) => entry.file === file)
262
274
  ? priorEntries.map((entry) => (entry.file === file ? updatedEntry : entry))
263
275
  : [...priorEntries, updatedEntry];
264
- const priorSegments = this.cached?.segments ?? [];
276
+ const priorSegments = prior.segments;
265
277
  const existingSegment = priorSegments.find((segment) => segment.file === file);
266
278
  let offset = existingSegment?.size ?? 0;
267
279
  const indexedEvents = newEnvelopes.map((event) => {
@@ -294,23 +306,27 @@ export class FileEventJournal {
294
306
  },
295
307
  ];
296
308
  const events = [...priorEvents, ...newEnvelopes];
297
- this.cached = { entries, events, eventsByStream: indexEventsByStream(events), segments };
298
- this.rememberValidatedEntries(entries, true);
309
+ const next = { entries, events, eventsByStream: indexEventsByStream(events), segments };
310
+ this.adopt(next);
311
+ this.knownSegments.set(file, updatedEntry);
312
+ if (entries.length >= this.latestEntries.length)
313
+ this.latestEntries = entries;
314
+ return next;
315
+ }
316
+ // Journals only grow, so the snapshot with more events is the newer one.
317
+ adopt(snapshot) {
318
+ if (this.cached === undefined || snapshot.events.length >= this.cached.events.length)
319
+ this.cached = snapshot;
299
320
  }
300
321
  manifestPath() {
301
322
  return join(this.root, 'events', 'index-manifest.json');
302
323
  }
303
- cachedEventsForStream(stream) {
304
- return this.cached?.eventsByStream.get(key(stream)) ?? [];
305
- }
306
324
  // Persisted alongside the segments, so any reader of the same on-disk
307
325
  // journal — not just this instance — can skip straight to the segments
308
326
  // that can hold what it's looking for on a cold cache. appendToStream() extends its
309
327
  // contents from the concrete envelopes it has just written while locked.
310
- async persistManifest() {
311
- if (this.cached === undefined)
312
- return;
313
- const manifest = { segments: this.cached.segments };
328
+ async persistManifest(snapshot) {
329
+ const manifest = { segments: snapshot.segments };
314
330
  await writeFileAtomically(this.manifestPath(), JSON.stringify(manifest));
315
331
  }
316
332
  async loadManifest() {
@@ -326,69 +342,99 @@ export class FileEventJournal {
326
342
  }
327
343
  }
328
344
  async scan(precomputedEntries) {
329
- const entries = precomputedEntries ?? (await this.readCurrentEntries());
345
+ const entries = precomputedEntries ?? (await this.currentEntries());
330
346
  const inFlight = this.inFlightScan;
331
347
  if (inFlight !== undefined && sameEntries(inFlight.entries, entries))
332
348
  return inFlight.run;
333
- const generation = ++this.scanGeneration;
334
- const run = this.scanUncoalesced(entries, generation).finally(() => {
349
+ const run = this.scanUncoalesced(entries).finally(() => {
335
350
  if (this.inFlightScan?.run === run)
336
351
  this.inFlightScan = undefined;
337
352
  });
338
353
  this.inFlightScan = { entries, run };
339
354
  return run;
340
355
  }
341
- async readCurrentEntries() {
356
+ // Resolves with a listing that reflects every append committed before the
357
+ // call. A listing already running may have read the directory before the
358
+ // call, so it cannot answer it; every caller arriving while one runs shares a
359
+ // single follow-up listing. Any number of concurrent reads cost at most two.
360
+ currentEntries() {
361
+ if (this.queuedListing !== undefined)
362
+ return this.queuedListing;
363
+ const running = this.runningListing;
364
+ if (running === undefined)
365
+ return this.startListing();
366
+ const queued = running
367
+ .catch(() => undefined)
368
+ .then(() => {
369
+ this.queuedListing = undefined;
370
+ return this.startListing();
371
+ });
372
+ this.queuedListing = queued;
373
+ return queued;
374
+ }
375
+ startListing() {
376
+ const run = this.listSegments().finally(() => {
377
+ if (this.runningListing === run)
378
+ this.runningListing = undefined;
379
+ });
380
+ this.runningListing = run;
381
+ return run;
382
+ }
383
+ async listSegments() {
342
384
  const directory = join(this.root, 'events');
343
- let files;
385
+ const startedAtMs = Date.now();
386
+ let stamp;
344
387
  try {
345
- files = (await readdir(directory))
346
- .filter((file) => /^\d{4}-\d{2}-\d{2}(?:~\d{20})?\.jsonl$/.test(file))
347
- .sort();
388
+ const info = await stat(directory);
389
+ stamp = { mtimeMs: info.mtimeMs, ctimeMs: info.ctimeMs, ino: info.ino };
348
390
  }
349
391
  catch (error) {
350
- if (error.code === 'ENOENT')
351
- return [];
352
- throw error;
392
+ if (error.code !== 'ENOENT')
393
+ throw error;
394
+ this.knownSegments.clear();
395
+ this.listing = undefined;
396
+ this.latestEntries = [];
397
+ return [];
353
398
  }
354
- return Promise.all(files.map(async (file) => {
355
- const info = await stat(join(directory, file));
356
- return { file, size: info.size, mtimeMs: info.mtimeMs };
357
- }));
358
- }
359
- async readEntriesForRead() {
360
- const now = this.validationNow();
361
- if (this.validatedEntries !== undefined && this.validatedEntries.expiresAt > now)
362
- return this.validatedEntries.entries;
363
- if (this.inFlightEntryRefresh !== undefined)
364
- return this.inFlightEntryRefresh;
365
- const generation = this.entryValidationGeneration;
366
- const refresh = this.readCurrentEntries()
367
- .then((entries) => {
368
- if (generation === this.entryValidationGeneration)
369
- this.rememberValidatedEntries(entries);
370
- return entries;
371
- })
372
- .finally(() => {
373
- if (this.inFlightEntryRefresh === refresh)
374
- this.inFlightEntryRefresh = undefined;
375
- });
376
- this.inFlightEntryRefresh = refresh;
377
- return refresh;
378
- }
379
- rememberValidatedEntries(entries, replacesInFlightRefresh = false) {
380
- this.entryValidationGeneration += 1;
381
- this.validatedEntries = {
382
- entries,
383
- expiresAt: this.validationNow() + entryValidationTtlMs,
384
- };
385
- if (replacesInFlightRefresh)
386
- this.inFlightEntryRefresh = undefined;
387
- }
388
- invalidateValidatedEntries() {
389
- this.entryValidationGeneration += 1;
390
- this.validatedEntries = undefined;
391
- this.inFlightEntryRefresh = undefined;
399
+ const previous = this.listing;
400
+ if (previous !== undefined &&
401
+ sameStamp(previous.stamp, stamp) &&
402
+ previous.startedAtMs - Math.max(stamp.mtimeMs, stamp.ctimeMs) > unchangedStampMinAgeMs &&
403
+ this.latestEntries.length === this.knownSegments.size)
404
+ return this.latestEntries;
405
+ // The stamp is taken before the listing so a change during it cannot be hidden.
406
+ const names = (await readdir(directory)).filter((name) => this.knownSegments.has(name) || segmentName.test(name));
407
+ const present = new Set(names);
408
+ let removed = false;
409
+ for (const name of this.knownSegments.keys())
410
+ if (!present.has(name)) {
411
+ this.knownSegments.delete(name);
412
+ removed = true;
413
+ }
414
+ const unseen = names.filter((name) => !this.knownSegments.has(name)).sort();
415
+ for (let index = 0; index < unseen.length; index += statBatchSize)
416
+ await Promise.all(unseen.slice(index, index + statBatchSize).map(async (file) => {
417
+ try {
418
+ const info = await stat(join(directory, file));
419
+ this.knownSegments.set(file, { file, size: info.size, mtimeMs: info.mtimeMs });
420
+ }
421
+ catch (error) {
422
+ if (error.code !== 'ENOENT')
423
+ throw error;
424
+ }
425
+ }));
426
+ this.listing = { stamp, startedAtMs };
427
+ const added = unseen.flatMap((file) => this.knownSegments.get(file) ?? []);
428
+ const last = this.latestEntries.at(-1);
429
+ const consistent = this.latestEntries.length + added.length === this.knownSegments.size;
430
+ if (!removed && added.length === 0 && consistent)
431
+ return this.latestEntries;
432
+ // Anything inconsistent with what has been seen is rebuilt from it, never patched.
433
+ this.latestEntries =
434
+ !removed && consistent && (last === undefined || added[0].file > last.file)
435
+ ? [...this.latestEntries, ...added]
436
+ : [...this.knownSegments.values()].sort((a, b) => (a.file < b.file ? -1 : 1));
437
+ return this.latestEntries;
392
438
  }
393
439
  // Reads exactly the indexed JSONL records. Byte offsets make tail reads and
394
440
  // stream reads proportional to the matching events, not segment size.
@@ -433,13 +479,21 @@ export class FileEventJournal {
433
479
  throw new Error(`Corrupt indexed event at ${file}:${indexed.globalPosition}${context}: ${error.message}`, { cause: error });
434
480
  }
435
481
  }
436
- async scanUncoalesced(entries, generation) {
482
+ async scanUncoalesced(entries) {
437
483
  const directory = join(this.root, 'events');
438
- if (this.cached !== undefined && sameEntries(this.cached.entries, entries))
439
- return this.cached.events;
440
- const events = [];
441
- const segments = [];
442
- for (const entry of entries) {
484
+ const warm = this.cached;
485
+ if (warm !== undefined && sameEntries(warm.entries, entries))
486
+ return warm;
487
+ // Segments are immutable, so when the cached segments are an unchanged
488
+ // prefix of the listing only the segments after it need reading. A foreign
489
+ // append then costs one segment, not a re-parse of the whole history.
490
+ const prefix = warm;
491
+ const reuse = prefix !== undefined &&
492
+ prefix.entries.length <= entries.length &&
493
+ sameEntries(prefix.entries, entries.slice(0, prefix.entries.length));
494
+ const events = reuse ? [...prefix.events] : [];
495
+ const segments = reuse ? [...prefix.segments] : [];
496
+ for (const entry of reuse ? entries.slice(prefix.entries.length) : entries) {
443
497
  const { file } = entry;
444
498
  const raw = await readFile(join(directory, file), 'utf8');
445
499
  if (raw.length > 0 && !raw.endsWith('\n'))
@@ -474,11 +528,15 @@ export class FileEventJournal {
474
528
  events: indexedEvents,
475
529
  });
476
530
  }
477
- if (generation === this.scanGeneration)
478
- this.cached = { entries, events, eventsByStream: indexEventsByStream(events), segments };
479
- return events;
531
+ const snapshot = { entries, events, eventsByStream: indexEventsByStream(events), segments };
532
+ this.adopt(snapshot);
533
+ return snapshot;
480
534
  }
481
535
  }
536
+ const segmentName = /^\d{4}-\d{2}-\d{2}(?:~\d{20})?\.jsonl$/;
537
+ function sameStamp(a, b) {
538
+ return a.mtimeMs === b.mtimeMs && a.ctimeMs === b.ctimeMs && a.ino === b.ino;
539
+ }
482
540
  function isPersistedIndex(value) {
483
541
  return (typeof value === 'object' &&
484
542
  value !== null &&
@@ -528,11 +586,12 @@ function isCompleteIndex(index, entries) {
528
586
  });
529
587
  }
530
588
  function sameEntries(a, b) {
531
- return (a.length === b.length &&
532
- a.every((entry, index) => {
533
- const other = b[index];
534
- return (entry.file === other.file && entry.size === other.size && entry.mtimeMs === other.mtimeMs);
535
- }));
589
+ return (a === b ||
590
+ (a.length === b.length &&
591
+ a.every((entry, index) => {
592
+ const other = b[index];
593
+ return (entry.file === other.file && entry.size === other.size && entry.mtimeMs === other.mtimeMs);
594
+ })));
536
595
  }
537
596
  const key = (stream) => `${stream.kind}:${stream.id}`;
538
597
  function indexEventsByStream(events) {
@@ -9,13 +9,15 @@ export declare class DeferredSignals {
9
9
  private readonly repository;
10
10
  private readonly deliver;
11
11
  private readonly transition;
12
- private readonly fenced;
12
+ private readonly fencedAmong;
13
13
  private readonly ledger;
14
- constructor(repository: OrchestrationRepository, deliver: (id: WorkflowInstanceId, signal: OrchestrationSignal, context: CommandContext) => Promise<WorkflowInstanceView>, transition: (id: WorkflowInstanceId, target: TransitionTarget, evidenceId: string, context: CommandContext) => Promise<WorkflowInstanceView | null>, fenced: (id: WorkflowInstanceId) => Promise<boolean>);
14
+ constructor(repository: OrchestrationRepository, deliver: (id: WorkflowInstanceId, signal: OrchestrationSignal, context: CommandContext) => Promise<WorkflowInstanceView>, transition: (id: WorkflowInstanceId, target: TransitionTarget, evidenceId: string, context: CommandContext) => Promise<WorkflowInstanceView | null>, fencedAmong: (ids: readonly WorkflowInstanceId[]) => Promise<ReadonlySet<WorkflowInstanceId>>);
15
15
  accept(id: WorkflowInstanceId, signal: OrchestrationSignal, context: CommandContext): Promise<WorkflowInstanceView>;
16
16
  applyTransition(id: WorkflowInstanceId, target: TransitionTarget, evidenceId: string, context: CommandContext): Promise<WorkflowInstanceView | null>;
17
17
  private record;
18
18
  private hasDeferred;
19
19
  reconcile(context: CommandContext): Promise<void>;
20
20
  reconcileWorkflow(id: WorkflowInstanceId, context: CommandContext): Promise<WorkflowInstanceView>;
21
+ /** Replays the deferred signals of one unfenced workflow that are still awaited. */
22
+ private replay;
21
23
  }
@@ -8,13 +8,13 @@ export class DeferredSignals {
8
8
  repository;
9
9
  deliver;
10
10
  transition;
11
- fenced;
11
+ fencedAmong;
12
12
  ledger;
13
- constructor(repository, deliver, transition, fenced) {
13
+ constructor(repository, deliver, transition, fencedAmong) {
14
14
  this.repository = repository;
15
15
  this.deliver = deliver;
16
16
  this.transition = transition;
17
- this.fenced = fenced;
17
+ this.fencedAmong = fencedAmong;
18
18
  this.ledger = new DelegationLedger(repository);
19
19
  }
20
20
  async accept(id, signal, context) {
@@ -67,43 +67,53 @@ export class DeferredSignals {
67
67
  return (await this.repository.events()).some((entry) => entry.event.eventType === OrchestrationEventType.SignalDeferred);
68
68
  }
69
69
  async reconcile(context) {
70
- if (!(await this.hasDeferred()))
70
+ // Only instances holding a deferred signal have anything to replay.
71
+ const deferred = new Set();
72
+ for (const entry of await this.repository.events())
73
+ if (entry.event.eventType === OrchestrationEventType.SignalDeferred)
74
+ deferred.add(entry.stream.id);
75
+ const ids = [...deferred];
76
+ if (ids.length === 0)
71
77
  return;
72
- for (const loaded of await this.repository.list()) {
73
- if (loaded.view !== null)
74
- await this.reconcileWorkflow(loaded.view.workflowInstanceId, context);
75
- }
78
+ const fenced = await this.fencedAmong(ids);
79
+ for (const id of ids)
80
+ if (!fenced.has(id))
81
+ await this.replay(id, context);
76
82
  }
77
83
  async reconcileWorkflow(id, context) {
78
- if ((await this.hasDeferred()) && !(await this.fenced(id))) {
79
- const { events } = await this.repository.history(id);
80
- for (const event of events) {
81
- if (event?.event.eventType !== OrchestrationEventType.SignalDeferred)
82
- continue;
83
- const deferred = event.event.payload;
84
- const loaded = await this.repository.loadRequired(id);
85
- const view = loaded.view;
86
- if (!matchesDeferredWait(view, deferred))
87
- continue;
88
- const deliveryContext = {
89
- ...context,
90
- commandId: `replay:${event.event.eventId}`,
91
- expectedSequence: loaded.sequence,
92
- correlationId: event.event.correlationId,
93
- actor: event.event.actor,
94
- };
95
- try {
96
- if (deferred.target === undefined)
97
- await this.deliver(id, deferred.signal, deliveryContext);
98
- else
99
- await this.transition(id, deferred.target, deferred.signal.providerEventId, deliveryContext);
100
- }
101
- catch (error) {
102
- if (!(error instanceof DelegationPendingError) &&
103
- !(error instanceof WrongExpectedSequenceError))
104
- throw error;
105
- break;
106
- }
84
+ if ((await this.hasDeferred()) && !(await this.fencedAmong([id])).has(id))
85
+ return this.replay(id, context);
86
+ return (await this.repository.loadRequired(id)).view;
87
+ }
88
+ /** Replays the deferred signals of one unfenced workflow that are still awaited. */
89
+ async replay(id, context) {
90
+ const { events } = await this.repository.history(id);
91
+ for (const event of events) {
92
+ if (event?.event.eventType !== OrchestrationEventType.SignalDeferred)
93
+ continue;
94
+ const deferred = event.event.payload;
95
+ const loaded = await this.repository.loadRequired(id);
96
+ const view = loaded.view;
97
+ if (!matchesDeferredWait(view, deferred))
98
+ continue;
99
+ const deliveryContext = {
100
+ ...context,
101
+ commandId: `replay:${event.event.eventId}`,
102
+ expectedSequence: loaded.sequence,
103
+ correlationId: event.event.correlationId,
104
+ actor: event.event.actor,
105
+ };
106
+ try {
107
+ if (deferred.target === undefined)
108
+ await this.deliver(id, deferred.signal, deliveryContext);
109
+ else
110
+ await this.transition(id, deferred.target, deferred.signal.providerEventId, deliveryContext);
111
+ }
112
+ catch (error) {
113
+ if (!(error instanceof DelegationPendingError) &&
114
+ !(error instanceof WrongExpectedSequenceError))
115
+ throw error;
116
+ break;
107
117
  }
108
118
  }
109
119
  return (await this.repository.loadRequired(id)).view;