@toa.io/core 1.0.0-alpha.293 → 1.0.0-alpha.299

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/package.json +3 -3
  2. package/transpiled/call.js +12 -0
  3. package/transpiled/call.js.map +1 -1
  4. package/transpiled/component.js +31 -4
  5. package/transpiled/component.js.map +1 -1
  6. package/transpiled/context.d.ts +5 -0
  7. package/transpiled/context.js +6 -0
  8. package/transpiled/context.js.map +1 -1
  9. package/transpiled/emission.d.ts +4 -2
  10. package/transpiled/emission.js +4 -2
  11. package/transpiled/emission.js.map +1 -1
  12. package/transpiled/entities/entity.js +34 -6
  13. package/transpiled/entities/entity.js.map +1 -1
  14. package/transpiled/entities/factory.d.ts +5 -0
  15. package/transpiled/entities/factory.js +6 -2
  16. package/transpiled/entities/factory.js.map +1 -1
  17. package/transpiled/event.d.ts +2 -2
  18. package/transpiled/event.js +6 -1
  19. package/transpiled/event.js.map +1 -1
  20. package/transpiled/exceptions.d.ts +19 -0
  21. package/transpiled/exceptions.js +74 -2
  22. package/transpiled/exceptions.js.map +1 -1
  23. package/transpiled/index.d.ts +2 -1
  24. package/transpiled/index.js +2 -1
  25. package/transpiled/index.js.map +1 -1
  26. package/transpiled/{outbox/outbox.d.ts → outbox.d.ts} +6 -7
  27. package/transpiled/outbox.js +424 -0
  28. package/transpiled/outbox.js.map +1 -0
  29. package/transpiled/query/options.js +1 -1
  30. package/transpiled/query/options.js.map +1 -1
  31. package/transpiled/receiver.js +14 -1
  32. package/transpiled/receiver.js.map +1 -1
  33. package/transpiled/remote.js +2 -3
  34. package/transpiled/remote.js.map +1 -1
  35. package/transpiled/state.d.ts +1 -1
  36. package/transpiled/trail.d.ts +49 -0
  37. package/transpiled/trail.js +98 -0
  38. package/transpiled/trail.js.map +1 -0
  39. package/transpiled/types/bindings.d.ts +22 -0
  40. package/transpiled/types/extensions.d.ts +13 -2
  41. package/transpiled/types/message.d.ts +5 -0
  42. package/transpiled/types/outbox.d.ts +29 -3
  43. package/transpiled/types/request.d.ts +5 -0
  44. package/transpiled/types/storages.d.ts +20 -0
  45. package/transpiled/outbox/index.d.ts +0 -1
  46. package/transpiled/outbox/index.js +0 -2
  47. package/transpiled/outbox/index.js.map +0 -1
  48. package/transpiled/outbox/outbox.js +0 -268
  49. package/transpiled/outbox/outbox.js.map +0 -1
@@ -0,0 +1,98 @@
1
+ import { AsyncLocalStorage } from 'node:async_hooks';
2
+ import { environment } from '@toa.io/generic';
3
+ import { LoopException } from './exceptions.js';
4
+ /**
5
+ * The chain of hops the invocation running now passed through, and the rule that refuses one
6
+ * that has come back to where it had been.
7
+ *
8
+ * Ambient, like the trace context and for the same reason: an algorithm's `context` is built
9
+ * once per operation at boot and shared by every invocation of it, so nothing per-invocation
10
+ * can be handed to it. It is not the trace context's own store, which `console.span` replaces
11
+ * on every span and does not enter at all when the trace is unsampled — a chain that
12
+ * disappears under sampling is a breaker that stops breaking in production.
13
+ *
14
+ * One writer, `Component.invoke`, where the hop is appended; two readers, `Call.invoke`, which
15
+ * puts it on the request it is about to send, and `Outbox.row`, which writes it onto the row so
16
+ * it outlives the operation that caused it.
17
+ */
18
+ // as openspan holds its own: a process may carry two copies of this module, and a chain that
19
+ // is empty because the writer sat in the other one is a breaker that never fires
20
+ const KEY = Symbol.for('toa.core.trail');
21
+ const storage = (globalThis[KEY] ??= new AsyncLocalStorage());
22
+ /**
23
+ * What the environment says, read where a component is built rather than once for the process,
24
+ * as the outbox reads its own — so a composition booted after a variable was set sees it.
25
+ *
26
+ * `TOA_TRAIL_REPEATS=0` refuses nothing: the chain is still stamped, carried and bounded. It is
27
+ * the off switch, and it exists because this refuses calls an application may be making today —
28
+ * a handshake written as `a > b > a > b > a` is three occurrences of one hop. A breaker with no
29
+ * way to open it is itself the outage.
30
+ */
31
+ export function limits() {
32
+ return {
33
+ repeats: number('TOA_TRAIL_REPEATS', 3),
34
+ depth: number('TOA_TRAIL_DEPTH', 32)
35
+ };
36
+ }
37
+ /** The chain that led to the invocation running now, where there is one. */
38
+ export function current() {
39
+ return storage.getStore();
40
+ }
41
+ /** Runs `task` as the hop the chain now ends with. */
42
+ export async function follow(hops, task) {
43
+ return storage.run(hops, task);
44
+ }
45
+ /**
46
+ * The chain this hop makes, or a raise where it is one hop too many: a hop already taken
47
+ * `repeats` times is a cycle, and a chain past `depth` is one nothing meant to make. Both are
48
+ * permanent, so what hits one is set aside rather than tried again into the same loop.
49
+ *
50
+ * The chain is copied rather than appended to, which is what makes it a path down the call
51
+ * tree rather than a log of everything that happened — an operation calling one endpoint fifty
52
+ * times makes fifty chains of one hop, not one chain of fifty.
53
+ */
54
+ export function extend(inbound, hop, limits) {
55
+ const hops = clip(received(inbound), limits.depth);
56
+ const trail = [...hops, hop];
57
+ if (limits.repeats === 0)
58
+ return trail;
59
+ if (trail.length > limits.depth)
60
+ throw new LoopException(`Call chain is ${trail.length} hops deep`, trail);
61
+ let seen = 0;
62
+ for (const passed of hops)
63
+ if (passed === hop)
64
+ seen++;
65
+ if (seen + 1 >= limits.repeats)
66
+ throw new LoopException(`'${hop}' is hop ${seen + 1} of this chain`, trail);
67
+ return trail;
68
+ }
69
+ /**
70
+ * How an event is named in a chain. The sigil is what tells it from an operation: the two are
71
+ * the same shape, a component may declare one of each under a single name, and `sync` is both
72
+ * the event every component inherits and an ordinary name for an operation.
73
+ *
74
+ * What an operator rewires to break a cycle is the subscription rather than the operation, so a
75
+ * chain that named only operations would not say how a component was re-entered.
76
+ */
77
+ export function event(destination) {
78
+ return '~' + destination;
79
+ }
80
+ /**
81
+ * What came off the wire, as a chain and nothing else. Neither the request contract nor a
82
+ * message validates this — a request is not validated at all once it is `authentic` — so a
83
+ * malformed one would otherwise become a `TypeError` where a named exception was contracted
84
+ * for. Bounding it is `extend`'s, one hop later, which is the only place the limits are known.
85
+ */
86
+ export function received(value) {
87
+ return Array.isArray(value) ? value.filter((hop) => typeof hop === 'string') : [];
88
+ }
89
+ // with the hop being appended that is one past the cap, which is enough to be refused; the
90
+ // rest is a message buying memory, and where the rule is off it is what bounds the chain
91
+ function clip(hops, depth) {
92
+ return hops.length > depth ? hops.slice(0, depth) : hops;
93
+ }
94
+ function number(variable, fallback) {
95
+ const declared = Number(environment.get(variable));
96
+ return Number.isNaN(declared) || declared < 0 ? fallback : declared;
97
+ }
98
+ //# sourceMappingURL=trail.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"trail.js","sourceRoot":"","sources":["../source/trail.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAA;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AAE/C;;;;;;;;;;;;;GAaG;AAEH,6FAA6F;AAC7F,iFAAiF;AACjF,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAA;AAIxC,MAAM,OAAO,GAAG,CAAE,UAAoB,CAAC,GAAG,CAAC,KAAK,IAAI,iBAAiB,EAAY,CAAC,CAAA;AAYlF;;;;;;;;GAQG;AACH,MAAM,UAAU,MAAM;IACpB,OAAO;QACL,OAAO,EAAE,MAAM,CAAC,mBAAmB,EAAE,CAAC,CAAC;QACvC,KAAK,EAAE,MAAM,CAAC,iBAAiB,EAAE,EAAE,CAAC;KACrC,CAAA;AACH,CAAC;AAED,4EAA4E;AAC5E,MAAM,UAAU,OAAO;IACrB,OAAO,OAAO,CAAC,QAAQ,EAAE,CAAA;AAC3B,CAAC;AAED,sDAAsD;AACtD,MAAM,CAAC,KAAK,UAAU,MAAM,CAAI,IAAc,EAAE,IAAsB;IACpE,OAAO,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;AAChC,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,MAAM,CAAC,OAAgB,EAAE,GAAW,EAAE,MAAc;IAClE,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC,CAAA;IAClD,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,EAAE,GAAG,CAAC,CAAA;IAE5B,IAAI,MAAM,CAAC,OAAO,KAAK,CAAC;QAAE,OAAO,KAAK,CAAA;IAEtC,IAAI,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,KAAK;QAC7B,MAAM,IAAI,aAAa,CAAC,iBAAiB,KAAK,CAAC,MAAM,YAAY,EAAE,KAAK,CAAC,CAAA;IAE3E,IAAI,IAAI,GAAG,CAAC,CAAA;IAEZ,KAAK,MAAM,MAAM,IAAI,IAAI;QAAE,IAAI,MAAM,KAAK,GAAG;YAAE,IAAI,EAAE,CAAA;IAErD,IAAI,IAAI,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO;QAC5B,MAAM,IAAI,aAAa,CAAC,IAAI,GAAG,YAAY,IAAI,GAAG,CAAC,gBAAgB,EAAE,KAAK,CAAC,CAAA;IAE7E,OAAO,KAAK,CAAA;AACd,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,KAAK,CAAC,WAAmB;IACvC,OAAO,GAAG,GAAG,WAAW,CAAA;AAC1B,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,QAAQ,CAAC,KAAc;IACrC,OAAO,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;AACnF,CAAC;AAED,2FAA2F;AAC3F,yFAAyF;AACzF,SAAS,IAAI,CAAC,IAAc,EAAE,KAAa;IACzC,OAAO,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;AAC1D,CAAC;AAED,SAAS,MAAM,CAAC,QAAgB,EAAE,QAAgB;IAChD,MAAM,QAAQ,GAAG,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAA;IAElD,OAAO,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAA;AACrE,CAAC"}
@@ -39,4 +39,26 @@ export interface Factory {
39
39
  /** `group` is absent for an exclusive subscription */
40
40
  receiver?(locator: Locator, label: string, group: string | undefined, receiver: Receiver): Connector;
41
41
  broadcast?(name: string, group?: string): Broadcast;
42
+ /**
43
+ * A channel is a name the caller picks, from which the binding derives whatever its
44
+ * transport needs, and `uris` are the brokers to carry it over — so a second broker set
45
+ * costs the binding no configuration and no variable of its own.
46
+ */
47
+ outbound?(channel: string, uris: string[]): Outbound;
48
+ /** what arrives on `channel` under `label` */
49
+ inbound?(channel: string, uris: string[], label: string, sink: Inbound): Connector;
50
+ }
51
+ /**
52
+ * Publishes to a channel, addressed by label. It forwards messages and nothing else: what
53
+ * `send` is handed is what is published, with no envelope, no field and no header of the
54
+ * binding's own, and none stripped — because a message shape is often somebody else's
55
+ * contract, and an extension shipping changes into another system must be able to send
56
+ * exactly what that system accepts.
57
+ */
58
+ export interface Outbound extends Connector {
59
+ send(label: string, message: object): Promise<void>;
60
+ }
61
+ /** What a binding hands a message to. */
62
+ export interface Inbound {
63
+ accept(message: object): Promise<void>;
42
64
  }
@@ -5,9 +5,10 @@ import type { Remote } from '../remote.js';
5
5
  import type { Receiver } from './receiver.js';
6
6
  import type { Context } from '../context.js';
7
7
  import type { Storage } from './storages.js';
8
- import type { Broadcast, Emitter } from './bindings.js';
8
+ import type { Broadcast, Emitter, Inbound, Outbound } from './bindings.js';
9
9
  import type { Atom } from './atomicity.js';
10
10
  import type { Source } from './request.js';
11
+ import type { Destination } from './outbox.js';
11
12
  /**
12
13
  * What the process hosting an extension provides to it: the counterpart of a component's
13
14
  * context. What is returned is a connector the extension depends on.
@@ -23,6 +24,10 @@ export interface Host {
23
24
  receive(label: string, receiver: Receiver): Promise<Connector>;
24
25
  /** what the replicas of one group decide together */
25
26
  atom(group: string): Atom;
27
+ /** where this deployment publishes a channel, over the brokers named */
28
+ outbound(binding: string, channel: string, uris: string[]): Promise<Outbound>;
29
+ /** what arrives on a channel under one label */
30
+ inbound(binding: string, channel: string, uris: string[], label: string, sink: Inbound): Promise<Connector>;
26
31
  }
27
32
  /**
28
33
  * `Manifest` is a type parameter rather than an import: `@toa.io/norm` depends on core, so
@@ -31,12 +36,18 @@ export interface Host {
31
36
  export interface Factory<Manifest = unknown> {
32
37
  tenant?(locator: Locator, declaration: any, manifest: Manifest): Connector | Promise<Connector>;
33
38
  aspect?(locator: Locator, declaration: any): Aspect | Aspect[];
39
+ /**
40
+ * Where a committed state change of this component goes, beside its own events. Read before
41
+ * the storage is made, because one of these is what gives a component an outbox when it
42
+ * declares no event at all.
43
+ */
44
+ destination?(locator: Locator, declaration: any, manifest: Manifest): Destination | Promise<Destination> | undefined;
34
45
  /** what the extension runs as a process of its own; `null` where it is off here */
35
46
  service?(): Connector | null | Promise<Connector | null>;
36
47
  component?(component: Component): Component;
37
48
  context?(context: Context): Context;
38
49
  manage?(composition: Connector): Connector;
39
- storage?(storage: Storage): Storage;
50
+ storage?(storage: Storage, locator: Locator): Storage;
40
51
  emitter?(emitter: Emitter, label: string, locator: Locator): Emitter;
41
52
  receiver?(receiver: Receiver, locator: Locator): Receiver;
42
53
  }
@@ -2,4 +2,9 @@ export interface Message<T = any> {
2
2
  payload: T;
3
3
  /** W3C traceparent */
4
4
  telemetry?: string;
5
+ /**
6
+ * The hops that led to the state change this is about, so a receiver of it continues the
7
+ * chain rather than starting one. See `core/source/trail.ts`.
8
+ */
9
+ trail?: string[];
5
10
  }
@@ -1,3 +1,4 @@
1
+ import type { Connector } from '../connector.js';
1
2
  import type { Event } from './state.js';
2
3
  /**
3
4
  * The intent to publish, committed with the state change it belongs to. Everything about it
@@ -8,21 +9,46 @@ export interface Row {
8
9
  id: string;
9
10
  /** which replica pumps this row; carries no other meaning, and no ordering */
10
11
  lane: number;
12
+ /** settled for every destination */
11
13
  published: boolean;
12
14
  /** not before this */
13
15
  pending: number;
16
+ /** the destinations it has not been sent to yet, by name */
17
+ outstanding: string[];
18
+ /**
19
+ * The hops that led to the change. Written onto the row rather than left in scope, because
20
+ * the pump publishes off the operation's path — possibly in another replica, an hour later.
21
+ * Absent on a row written before this existed, which reads as a chain that starts there.
22
+ */
23
+ trail?: string[];
14
24
  /** an assignment's images are absent until the storage fills them in */
15
25
  event: Event;
16
26
  }
27
+ /**
28
+ * Somewhere a committed state change goes. A component's `Emission` is one, under the name
29
+ * `events`; an extension may contribute others, and each is published and settled on its own,
30
+ * so one that is down delays nothing but itself.
31
+ */
32
+ export interface Destination extends Connector {
33
+ /** what a row is outstanding for, as the row records it */
34
+ readonly name: string;
35
+ /** the row, not its event: what goes on the wire is the chain it carries too */
36
+ emit(row: Row): Promise<void>;
37
+ }
17
38
  /**
18
39
  * The read side of an outbox. What writes a row is the storage's own: it happens inside the
19
40
  * transaction the storage opened, which core never reaches into.
20
41
  */
21
42
  export interface Storage {
22
43
  /**
23
- * One page of what is due, still unpublished, and in one of the given lanes, in the order
24
- * the rows were written. `after` continues from the last id of the page before.
44
+ * One page of what is due, not settled for every destination, and in one of the given lanes,
45
+ * in the order the rows were written. `after` continues from the last id of the page before.
25
46
  */
26
47
  pending(lanes: number[], now: number, limit: number, after?: string): Promise<Row[]>;
27
- settle(ids: string[]): Promise<void>;
48
+ /**
49
+ * Takes `destinations` out of what those rows are outstanding for, and marks published the
50
+ * ones left outstanding for nothing. Several destinations at once, because the ordinary case
51
+ * is all of them landing in one window, and that is then one write.
52
+ */
53
+ settle(ids: string[], destinations: string[]): Promise<void>;
28
54
  }
@@ -38,6 +38,11 @@ export interface Request<Input = any, Entity = any> {
38
38
  /** W3C traceparent */
39
39
  telemetry?: string;
40
40
  source?: Source;
41
+ /**
42
+ * The hops this call passed through, oldest first. Stamped by the framework; a call that has
43
+ * been where it is going already is refused rather than made. See `core/source/trail.ts`.
44
+ */
45
+ trail?: string[];
41
46
  }
42
47
  /**
43
48
  * An error an operation declares and returns. A call resolves to it rather than throwing:
@@ -19,6 +19,8 @@ export interface Record {
19
19
  UPDATED?: number;
20
20
  /** a tombstone's timestamp; `null` on a live record */
21
21
  DELETED?: number | null;
22
+ /** the rank of the region that last wrote it; `0` where there is one region */
23
+ REGION?: number;
22
24
  [key: string]: any;
23
25
  }
24
26
  /** Everything a request query carried that was not a selector. */
@@ -81,6 +83,24 @@ export interface Storage extends Connector {
81
83
  * starting with a structure nothing has made.
82
84
  */
83
85
  readonly migrates?: boolean;
86
+ /**
87
+ * Writes `record` as it stands — its `VERSION`, its timestamps, its `REGION` and whatever
88
+ * else it carries — where what it would replace precedes it: a lower `VERSION`, or the same
89
+ * `VERSION` written by a region this one outranks. `false` where it does not: nothing is
90
+ * written, and that is not an error.
91
+ *
92
+ * Both sides of the comparison are on the two records, so this takes nothing else. What it
93
+ * is for is a record that was written somewhere else and has to land here as it was, which
94
+ * is neither a transition nor an assignment: it is not the writer's version to increment,
95
+ * nor its timestamps to set.
96
+ */
97
+ converge?(record: Record): Promise<boolean>;
98
+ /**
99
+ * Whether this storage converges. Absent is what a storage that does not says, and a
100
+ * component of a context that converges stands down rather than running where it would
101
+ * never take a record from another region.
102
+ */
103
+ readonly converges?: boolean;
84
104
  }
85
105
  /**
86
106
  * The subset of a component's entity declaration a storage reads. Structural, so that core
@@ -1 +0,0 @@
1
- export { Outbox, LANES } from './outbox.js';
@@ -1,2 +0,0 @@
1
- export { Outbox, LANES } from './outbox.js';
2
- //# sourceMappingURL=index.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../source/outbox/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,aAAa,CAAA"}
@@ -1,268 +0,0 @@
1
- import { console } from 'openspan';
2
- import { Connector } from '../connector.js';
3
- import { newid } from '../entities/newid.js';
4
- import { environment } from '@toa.io/generic';
5
- /**
6
- * Owns the intent to publish. A row is built before the write so that the storage can commit
7
- * it in the same transaction as the entity; publication then happens off the operation's path,
8
- * and anything that fails to publish is recovered from the row.
9
- *
10
- * The mechanism is a safety net: in a healthy system the row is written, published within
11
- * milliseconds by the same process, and marked published on that process's next tick.
12
- *
13
- * A storage that cannot commit a row atomically has no outbox, and this degrades to the
14
- * inline emission it replaces.
15
- */
16
- export class Outbox extends Connector {
17
- #emission;
18
- #storage;
19
- #atom;
20
- #gap;
21
- #interval;
22
- #batch;
23
- #defer;
24
- /** ids this process has published, held until a cycle marks them */
25
- #published = new Set();
26
- /** in-flight publications, awaited (with a bound) on close */
27
- #inflight = new Set();
28
- /** rows this replica is publishing right now, so a cycle does not pick them up again */
29
- #publishing = new Set();
30
- #timer;
31
- #off;
32
- #pumping = false;
33
- #closing = false;
34
- // eslint-disable-next-line max-params
35
- constructor(emission, storage, atom, options = {}) {
36
- super();
37
- this.#emission = emission;
38
- this.#storage = storage;
39
- this.#atom = atom;
40
- this.#interval = number('TOA_OUTBOX_INTERVAL', options.interval, INTERVAL);
41
- this.#batch = number('TOA_OUTBOX_BATCH', options.batch, BATCH);
42
- this.#gap = options.gap ?? this.#interval * K;
43
- this.#defer = environment.get('TOA_OUTBOX_DEFER') === '1';
44
- this.depends(emission);
45
- this.depends(atom);
46
- if (storage !== undefined)
47
- this.depends(storage);
48
- }
49
- /** whether the storage can commit a row atomically with the entity */
50
- get durable() {
51
- return this.#storage?.outbox !== undefined;
52
- }
53
- /**
54
- * An assignment's images are the write's own, so it hands over an event with neither, and
55
- * the storage fills them in.
56
- */
57
- row(event) {
58
- return {
59
- id: newid(),
60
- lane: this.#lane(),
61
- published: false,
62
- pending: Date.now() + this.#gap,
63
- event: event
64
- };
65
- }
66
- /**
67
- * Hands a committed row over. Awaited by the caller only on the legacy path — with an
68
- * outbox this returns at once and the broker leaves the operation's path.
69
- */
70
- publish(row) {
71
- // without a durable outbox this is the inline path, and the caller awaits the emission
72
- if (!this.durable)
73
- return this.#emission.emit(row.event);
74
- /*
75
- * A publication started while the pump is closing would outlive the emitters it needs,
76
- * and `comq` waits on a connection that is going rather than failing. The row is already
77
- * durable, so leaving it is exactly what it is for.
78
- */
79
- if (this.#closing ||
80
- this.#defer ||
81
- this.#inflight.size >= INFLIGHT ||
82
- this.#published.size >= PUBLISHED)
83
- return;
84
- void this.#publish(row);
85
- }
86
- async open() {
87
- if (!this.durable)
88
- return;
89
- if (this.#defer)
90
- console.warn('Outbox immediate publication is deferred; events are published by the pump only');
91
- this.#timer = setInterval(() => {
92
- this.#tick();
93
- }, this.#interval);
94
- this.#timer.unref();
95
- /*
96
- * A lane changing hands is exactly when rows stranded in it become this replica's to
97
- * publish, and the cycle would not notice for up to an interval. Being told costs a cycle
98
- * that finds nothing in the usual case, where the claim arrives once and never changes.
99
- */
100
- this.#off = this.#atom.onassigned(() => {
101
- this.#tick();
102
- });
103
- }
104
- async close() {
105
- this.#closing = true;
106
- this.#off?.();
107
- if (this.#timer !== undefined)
108
- clearInterval(this.#timer);
109
- await this.#drain();
110
- await this.#mark();
111
- }
112
- /**
113
- * Publishes one row and swallows the failure: the row stays unpublished and comes back on a
114
- * later cycle, which is the whole point of having written it.
115
- *
116
- * There is no timeout here on purpose. A publication is a confirmed write to a durable
117
- * exchange, and `comq` waits for the broker to come back rather than failing — abandoning
118
- * it would not stop it, it would only mean the row is published twice once it lands. What
119
- * bounds this instead is the in-flight cap and the drain on close.
120
- *
121
- */
122
- async #publish(row) {
123
- this.#publishing.add(row.id);
124
- const publishing = this.#emission.emit(row.event);
125
- this.#inflight.add(publishing);
126
- try {
127
- await publishing;
128
- this.#published.add(row.id);
129
- }
130
- catch (error) {
131
- console.warn('Event publication failed', { row: row.id, error });
132
- }
133
- finally {
134
- this.#inflight.delete(publishing);
135
- this.#publishing.delete(row.id);
136
- }
137
- }
138
- /**
139
- * `comq` retries a publish for as long as the broker is down rather than rejecting, so an
140
- * unbounded drain outlives any grace period.
141
- *
142
- */
143
- async #drain() {
144
- if (this.#inflight.size === 0)
145
- return;
146
- await Promise.race([Promise.allSettled([...this.#inflight]), delay(DRAIN)]);
147
- }
148
- /**
149
- * Reads what is due, publishes it, and marks everything this process has sent — what it just
150
- * published and what the immediate path published since the last cycle. One cycle at a time.
151
- *
152
- */
153
- #tick() {
154
- if (this.#pumping)
155
- return;
156
- this.#pumping = true;
157
- void this.#pump().finally(() => (this.#pumping = false));
158
- }
159
- async #pump() {
160
- let page;
161
- let after;
162
- do {
163
- page = await this.#read(after);
164
- if (page.length === 0)
165
- break;
166
- // so a page is never read twice
167
- after = page[page.length - 1]?.id;
168
- /*
169
- * A row is unpublished in the database until a cycle marks it, so a page includes what
170
- * this replica is sending right now and what a failed marking left behind. Only this
171
- * process knows either.
172
- */
173
- const rows = page.filter((row) => !this.#published.has(row.id) && !this.#publishing.has(row.id));
174
- if (rows.length > 0) {
175
- console.info('Outbox recovering unpublished events', { count: rows.length });
176
- // every row is given its chance; what the broker refused stays unpublished and comes
177
- // back on a later cycle
178
- await Promise.allSettled(rows.map(async (row) => this.#publish(row)));
179
- }
180
- // a full page is a page that may have been cut short
181
- } while (page.length === this.#batch);
182
- await this.#mark();
183
- }
184
- /**
185
- * One page of what is due. In a healthy system the first one is empty, every cycle — a row is
186
- * due only if the process that wrote it failed to publish or died before marking it.
187
- *
188
- * Reading is suspended, not stopped, while this replica does not know which lanes are its
189
- * own: the cycle keeps running and keeps marking, and reading resumes as soon as an
190
- * assignment arrives. Reading without an assignment would be a different guarantee, where
191
- * every replica publishes every stranded row.
192
- *
193
- * @param after the last id of the page before, so a page is never read twice
194
- */
195
- async #read(after) {
196
- const lanes = this.#atom.slots(LANES);
197
- if (lanes === null || lanes.length === 0)
198
- return [];
199
- return this.#storage
200
- .outbox.pending(lanes, Date.now(), this.#batch, after)
201
- .catch((error) => {
202
- console.warn('Outbox read failed', { error });
203
- return [];
204
- });
205
- }
206
- /**
207
- * One batched write for many events, which is why the ids are held in memory rather than
208
- * marked one by one. Ids that fail to be marked are kept and retried; a row that is never
209
- * marked is simply published again, which is within the contract.
210
- *
211
- */
212
- async #mark() {
213
- if (this.#published.size === 0)
214
- return;
215
- const ids = [...this.#published];
216
- try {
217
- await this.#storage.outbox.settle(ids);
218
- for (const id of ids)
219
- this.#published.delete(id);
220
- }
221
- catch (error) {
222
- console.warn('Outbox marking failed', { count: ids.length, error });
223
- }
224
- }
225
- /**
226
- * A lane this replica currently owns, so that in steady state it settles its own rows
227
- * before it ever reads them. Any lane at all when it owns none: the row still has to be
228
- * written, and whoever ends up owning that lane will pump it.
229
- *
230
- */
231
- #lane() {
232
- const owned = this.#atom.slots(LANES);
233
- return owned === null || owned.length === 0
234
- ? Math.floor(Math.random() * LANES)
235
- : owned[Math.floor(Math.random() * owned.length)];
236
- }
237
- }
238
- function number(variable, declared, fallback) {
239
- if (declared !== undefined)
240
- return declared;
241
- const value = Number(environment.get(variable));
242
- return Number.isNaN(value) || value <= 0 ? fallback : value;
243
- }
244
- async function delay(ms) {
245
- return new Promise((resolve) => {
246
- setTimeout(resolve, ms).unref();
247
- });
248
- }
249
- /**
250
- * Constant, never configuration: rows carry their lane, so lowering this would leave rows in
251
- * lanes nobody reads any more. It is also the ceiling on replicas of one component, and a
252
- * power of two so that the common replica counts divide evenly.
253
- */
254
- export const LANES = 128;
255
- /** one cycle reads, publishes and marks; in steady state it finds nothing to read */
256
- const INTERVAL = 5000;
257
- /**
258
- * `gap = interval * K`. Not a steady-state necessity — a replica writes into a lane it owns
259
- * and marks what it published — but a guard for when a lane changes hands between the write
260
- * and the settle. Two cycles of separation, plus one of margin.
261
- */
262
- const K = 3;
263
- /** how many rows one read brings back; the pump reads on while a page comes back full */
264
- const BATCH = 200;
265
- const DRAIN = 10_000;
266
- const INFLIGHT = 1000;
267
- const PUBLISHED = 10_000;
268
- //# sourceMappingURL=outbox.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"outbox.js","sourceRoot":"","sources":["../../source/outbox/outbox.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,UAAU,CAAA;AAClC,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAA;AAC3C,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAA;AAC5C,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;AAa7C;;;;;;;;;;GAUG;AACH,MAAM,OAAO,MAAO,SAAQ,SAAS;IAC1B,SAAS,CAAU;IACnB,QAAQ,CAAqB;IAC7B,KAAK,CAAM;IAEX,IAAI,CAAQ;IACZ,SAAS,CAAQ;IACjB,MAAM,CAAQ;IACd,MAAM,CAAS;IAExB,oEAAoE;IAC3D,UAAU,GAAG,IAAI,GAAG,EAAU,CAAA;IAEvC,8DAA8D;IACrD,SAAS,GAAG,IAAI,GAAG,EAAiB,CAAA;IAE7C,wFAAwF;IAC/E,WAAW,GAAG,IAAI,GAAG,EAAU,CAAA;IAExC,MAAM,CAA4B;IAClC,IAAI,CAA0B;IAC9B,QAAQ,GAAG,KAAK,CAAA;IAChB,QAAQ,GAAG,KAAK,CAAA;IAEhB,sCAAsC;IACtC,YACE,QAAkB,EAClB,OAA4B,EAC5B,IAAU,EACV,OAAO,GAAY,EAAE;QAErB,KAAK,EAAE,CAAA;QAEP,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAA;QACzB,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAA;QACvB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAA;QAEjB,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,qBAAqB,EAAE,OAAO,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAA;QAC1E,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,kBAAkB,EAAE,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAA;QAC9D,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,SAAS,GAAG,CAAC,CAAA;QAC7C,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,GAAG,CAAC,kBAAkB,CAAC,KAAK,GAAG,CAAA;QAEzD,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;QACtB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;QAElB,IAAI,OAAO,KAAK,SAAS;YAAE,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;IAClD,CAAC;IAED,sEAAsE;IACtE,IAAW,OAAO;QAChB,OAAO,IAAI,CAAC,QAAQ,EAAE,MAAM,KAAK,SAAS,CAAA;IAC5C,CAAC;IAED;;;OAGG;IACI,GAAG,CAAC,KAAqB;QAC9B,OAAO;YACL,EAAE,EAAE,KAAK,EAAE;YACX,IAAI,EAAE,IAAI,CAAC,KAAK,EAAE;YAClB,SAAS,EAAE,KAAK;YAChB,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,IAAI;YAC/B,KAAK,EAAE,KAAc;SACtB,CAAA;IACH,CAAC;IAED;;;OAGG;IACI,OAAO,CAAC,GAAQ;QACrB,uFAAuF;QACvF,IAAI,CAAC,IAAI,CAAC,OAAO;YAAE,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;QAExD;;;;WAIG;QACH,IACE,IAAI,CAAC,QAAQ;YACb,IAAI,CAAC,MAAM;YACX,IAAI,CAAC,SAAS,CAAC,IAAI,IAAI,QAAQ;YAC/B,IAAI,CAAC,UAAU,CAAC,IAAI,IAAI,SAAS;YAEjC,OAAM;QAER,KAAK,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAA;IACzB,CAAC;IAEkB,KAAK,CAAC,IAAI;QAC3B,IAAI,CAAC,IAAI,CAAC,OAAO;YAAE,OAAM;QAEzB,IAAI,IAAI,CAAC,MAAM;YACb,OAAO,CAAC,IAAI,CACV,iFAAiF,CAClF,CAAA;QAEH,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,GAAG,EAAE;YAC7B,IAAI,CAAC,KAAK,EAAE,CAAA;QACd,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,CAAA;QAClB,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAA;QAEnB;;;;WAIG;QACH,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE;YACrC,IAAI,CAAC,KAAK,EAAE,CAAA;QACd,CAAC,CAAC,CAAA;IACJ,CAAC;IAEkB,KAAK,CAAC,KAAK;QAC5B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAA;QAEpB,IAAI,CAAC,IAAI,EAAE,EAAE,CAAA;QAEb,IAAI,IAAI,CAAC,MAAM,KAAK,SAAS;YAAE,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QAEzD,MAAM,IAAI,CAAC,MAAM,EAAE,CAAA;QACnB,MAAM,IAAI,CAAC,KAAK,EAAE,CAAA;IACpB,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,QAAQ,CAAC,GAAQ;QACrB,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QAE5B,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;QAEjD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,UAAU,CAAC,CAAA;QAE9B,IAAI,CAAC;YACH,MAAM,UAAU,CAAA;YAEhB,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QAC7B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,0BAA0B,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAA;QAClE,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC,CAAA;YACjC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QACjC,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,MAAM;QACV,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC;YAAE,OAAM;QAErC,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;IAC7E,CAAC;IAED;;;;OAIG;IACH,KAAK;QACH,IAAI,IAAI,CAAC,QAAQ;YAAE,OAAM;QAEzB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAA;QAEpB,KAAK,IAAI,CAAC,KAAK,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAA;IAC1D,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,IAAW,CAAA;QACf,IAAI,KAAyB,CAAA;QAE7B,GAAG,CAAC;YACF,IAAI,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;YAE9B,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,MAAK;YAE5B,gCAAgC;YAChC,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,EAAE,CAAA;YAEjC;;;;eAIG;YACH,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CACtB,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CACvE,CAAA;YAED,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACpB,OAAO,CAAC,IAAI,CAAC,sCAAsC,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAA;gBAE5E,qFAAqF;gBACrF,wBAAwB;gBACxB,MAAM,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;YACvE,CAAC;YAED,qDAAqD;QACvD,CAAC,QAAQ,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,EAAC;QAErC,MAAM,IAAI,CAAC,KAAK,EAAE,CAAA;IACpB,CAAC;IAED;;;;;;;;;;OAUG;IACH,KAAK,CAAC,KAAK,CAAC,KAAc;QACxB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;QAErC,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAA;QAEnD,OAAO,IAAI,CAAC,QAAS;aAClB,MAAO,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC;aACtD,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;YACf,OAAO,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,KAAK,EAAE,CAAC,CAAA;YAE7C,OAAO,EAAE,CAAA;QACX,CAAC,CAAC,CAAA;IACN,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC;YAAE,OAAM;QAEtC,MAAM,GAAG,GAAG,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,CAAA;QAEhC,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,QAAS,CAAC,MAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;YAExC,KAAK,MAAM,EAAE,IAAI,GAAG;gBAAE,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;QAClD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,uBAAuB,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,CAAA;QACrE,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACH,KAAK;QACH,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;QAErC,OAAO,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YACzC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,KAAK,CAAC;YACnC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAA;IACrD,CAAC;CACF;AAED,SAAS,MAAM,CACb,QAAgB,EAChB,QAA4B,EAC5B,QAAgB;IAEhB,IAAI,QAAQ,KAAK,SAAS;QAAE,OAAO,QAAQ,CAAA;IAE3C,MAAM,KAAK,GAAG,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAA;IAE/C,OAAO,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAA;AAC7D,CAAC;AAED,KAAK,UAAU,KAAK,CAAC,EAAU;IAC7B,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,CAAA;IACjC,CAAC,CAAC,CAAA;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,MAAM,KAAK,GAAG,GAAG,CAAA;AAExB,qFAAqF;AACrF,MAAM,QAAQ,GAAG,IAAI,CAAA;AAErB;;;;GAIG;AACH,MAAM,CAAC,GAAG,CAAC,CAAA;AAEX,yFAAyF;AACzF,MAAM,KAAK,GAAG,GAAG,CAAA;AAEjB,MAAM,KAAK,GAAG,MAAM,CAAA;AACpB,MAAM,QAAQ,GAAG,IAAI,CAAA;AACrB,MAAM,SAAS,GAAG,MAAM,CAAA"}