@sgtbeatdown/pi-tasks 0.1.14 → 0.1.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -4
- package/docs/protocol-reference.md +18 -5
- package/package.json +1 -1
- package/src/abortable-sleep.ts +14 -5
- package/src/configured-task-core.ts +8 -4
- package/src/extension.ts +164 -64
- package/src/index.ts +3 -2
- package/src/task-core.ts +475 -137
- package/src/task-errors.ts +111 -0
- package/src/task-inbox.ts +190 -89
- package/src/task-protocol.ts +44 -13
- package/src/task-store.ts +181 -27
- package/src/task-work-scope.ts +98 -0
- package/src/volatile-task-session.ts +30 -8
package/src/task-store.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { TASK_WORK_LIMITS } from "./task-work-scope";
|
|
2
|
+
import { TASK_EVENT_TYPES, TERMINAL_INTENT_OPERATION, TaskProtocolError } from "./task-protocol";
|
|
2
3
|
import type { RelayTransportBinding, RelayEnvelope, TaskEndpoint, TaskEvent, TaskRecord, TaskSnapshot, TerminalDeliveryState, TerminalTaskIntentType } from "./task-protocol";
|
|
3
4
|
|
|
4
5
|
export interface TaskStoreOptions {
|
|
@@ -16,29 +17,64 @@ export interface TaskOperationRecord {
|
|
|
16
17
|
readonly taskId: string; readonly operation: string; readonly logicalId: string;
|
|
17
18
|
readonly logicalType: string; readonly envelopeIds: readonly string[];
|
|
18
19
|
}
|
|
20
|
+
export interface PresentationRecord { readonly taskId: string; readonly eventId: string; }
|
|
21
|
+
export interface PendingPresentation extends PresentationRecord { readonly order: number; }
|
|
19
22
|
export interface TaskStore {
|
|
20
23
|
transaction<T>(operation: () => T): T;
|
|
24
|
+
/** Synchronous hook after each top-level commit or non-transactional write; a throwing listener is isolated and logged. */
|
|
25
|
+
onChange(listener: () => void): () => void;
|
|
21
26
|
putTask(task: Omit<TaskRecord, "events">): void;
|
|
22
27
|
getTask(taskId: string): TaskSnapshot | undefined;
|
|
23
28
|
listTasks(): readonly TaskSnapshot[];
|
|
29
|
+
/** Whether any task of this lifetime is active; indexed, no snapshot is built. */
|
|
30
|
+
hasActiveTasks(): boolean;
|
|
24
31
|
setStatus(taskId: string, status: TaskRecord["status"]): void;
|
|
32
|
+
/** Follows a reassignment; existing envelopes keep their addresses. */
|
|
33
|
+
setTarget(taskId: string, target: TaskEndpoint): void;
|
|
25
34
|
appendEvent(event: TaskEvent): boolean;
|
|
26
35
|
putOutbox(envelope: RelayEnvelope): void;
|
|
27
36
|
outbox(state: OutboxRecord["state"]): readonly OutboxRecord[];
|
|
37
|
+
/** Indexed lookup, fair rotation and cooldown are required store capabilities. */
|
|
38
|
+
getOutboxRecord(envelopeId: string): OutboxRecord | undefined;
|
|
39
|
+
hasPendingOutbox(): boolean;
|
|
40
|
+
/** Rotates fairly but yields only each target's oldest pending envelope (per-target FIFO). */
|
|
41
|
+
selectPendingOutbox(limit: number, now: number, maxInspections?: number): readonly OutboxRecord[];
|
|
42
|
+
/** Older pending envelopes to the same target, oldest first; the envelope may be sent only once this is empty. */
|
|
43
|
+
pendingOutboxAhead(envelopeId: string): readonly string[];
|
|
44
|
+
outboxRetryAt(envelopeId: string): number;
|
|
45
|
+
postponeOutbox(envelopeId: string, until: number): void;
|
|
28
46
|
markOutboxAccepted(envelopeId: string): void;
|
|
29
47
|
quarantineOutbox(envelopeId: string, input: OutboxQuarantineInput): void;
|
|
30
48
|
quarantinedOutbox(): readonly OutboxQuarantineRecord[];
|
|
49
|
+
/** Indexed lookup of one quarantined envelope. */
|
|
50
|
+
getQuarantine(envelopeId: string): OutboxQuarantineRecord | undefined;
|
|
31
51
|
persistInbox(envelope: RelayEnvelope, cursor: string): boolean;
|
|
52
|
+
/** Store arrival order of a persisted inbox envelope; shares the presentation order sequence. */
|
|
53
|
+
inboxOrder(envelopeId: string): number | undefined;
|
|
54
|
+
inboxEnvelope(envelopeId: string): RelayEnvelope | undefined;
|
|
55
|
+
/** Origin-local canonical events awaiting Pi presentation; ids only, events stay in the task log. */
|
|
56
|
+
enqueuePresentation(details: PresentationRecord): void;
|
|
57
|
+
pendingPresentations(): readonly PendingPresentation[];
|
|
58
|
+
completePresentation(eventId: string): void;
|
|
32
59
|
getReceiveCursor(): string;
|
|
33
60
|
setReceiveCursor(cursor: string): void;
|
|
34
61
|
trackRelayDeliveries(endpoint: TaskEndpoint, deliveries: readonly { readonly cursor: string; readonly envelopeId: string }[]): void;
|
|
35
62
|
pendingRelayEnvelopeId(endpoint: TaskEndpoint, cursor: string): string | undefined;
|
|
36
|
-
|
|
37
|
-
|
|
63
|
+
/** Tracked, unacknowledged deliveries in this endpoint scope, oldest cursor first. */
|
|
64
|
+
pendingRelayDeliveries(endpoint: TaskEndpoint): readonly { readonly cursor: string; readonly envelopeId: string }[];
|
|
65
|
+
/** Highest tracked cursor in this endpoint scope; intake fetches after it. */
|
|
66
|
+
relayIntakeCursor(endpoint: TaskEndpoint): string;
|
|
67
|
+
requestRelayAcknowledgement(endpoint: TaskEndpoint, cursor: string, retryAt?: number): void;
|
|
68
|
+
requestedRelayAcknowledgements(endpoint: TaskEndpoint, limit?: number, now?: number): readonly string[];
|
|
69
|
+
relayAcknowledgementRetryAt(endpoint: TaskEndpoint, cursor: string): number;
|
|
38
70
|
acknowledgeRelayCursor(endpoint: TaskEndpoint, cursor: string): void;
|
|
39
71
|
putIntent(intentId: string, taskId: string, envelopeId: string): void;
|
|
72
|
+
/** Locally submitted intent only; retained through acceptance or quarantine in this RAM lifetime. */
|
|
73
|
+
getIntentEnvelope(intentId: string): RelayEnvelope | undefined;
|
|
40
74
|
putInsertionReceipt(taskId: string, eventId: string): boolean;
|
|
41
75
|
reserveTaskOperation(input: TaskOperationRecord): { readonly created: boolean; readonly record: TaskOperationRecord };
|
|
76
|
+
/** Withdraws a reservation whose envelopes were never offered to the relay, with its pending envelopes and intent row. */
|
|
77
|
+
discardUnsentOperation(taskId: string, operation: string): void;
|
|
42
78
|
getEndpointGeneration(): string | undefined;
|
|
43
79
|
setEndpointGeneration(generation: string): void;
|
|
44
80
|
getEndpointBinding(): TaskEndpoint | undefined;
|
|
@@ -50,9 +86,15 @@ export interface TaskStore {
|
|
|
50
86
|
}
|
|
51
87
|
|
|
52
88
|
const LIMITS = { entries: 16_384, bytes: 32 * 1024 * 1024 };
|
|
53
|
-
|
|
54
|
-
type
|
|
55
|
-
|
|
89
|
+
// `text` is the immutable source of truth and of the byte charge; `value` is its lazily parsed, deep-frozen form.
|
|
90
|
+
type Cell = { readonly text: string; readonly bytes: number; readonly order: number; value?: unknown };
|
|
91
|
+
type Undo = { readonly name: string; readonly table: Map<string, Cell>; readonly key: string; readonly prior: Cell | undefined };
|
|
92
|
+
interface Checkpoint { frontier: string; highWater: string; pending: Record<string, string>; requested: Record<string, number>; }
|
|
93
|
+
const freeze = <T>(value: T): T => {
|
|
94
|
+
if (value && typeof value === "object" && !Object.isFrozen(value)) { Object.freeze(value); for (const child of Object.values(value)) freeze(child); }
|
|
95
|
+
return value;
|
|
96
|
+
};
|
|
97
|
+
const read = <T>(row: Cell): T => (row.value ??= freeze(JSON.parse(row.text))) as T;
|
|
56
98
|
|
|
57
99
|
/** One endpoint lifetime. No SQLite, files, migrations, startup reads or replay. */
|
|
58
100
|
export function createTaskStore(options: TaskStoreOptions = {}): TaskStore {
|
|
@@ -61,29 +103,74 @@ export function createTaskStore(options: TaskStoreOptions = {}): TaskStore {
|
|
|
61
103
|
if (!Number.isInteger(maxEntries) || maxEntries < 1 || maxEntries > LIMITS.entries || !Number.isInteger(maxBytes) || maxBytes < 1 || maxBytes > LIMITS.bytes) throw new TypeError("invalid task memory limit");
|
|
62
104
|
const tables = new Map<string, Map<string, Cell>>();
|
|
63
105
|
const frames: Map<string, Undo>[] = [];
|
|
106
|
+
// ID-only retry rotation, bounded by the existing pending outbox; no payload copy.
|
|
107
|
+
const pendingOutbox = new Map<string, number>();
|
|
108
|
+
// Per-target FIFO: pending ids grouped by target; outbox cell order is creation order.
|
|
109
|
+
const pendingByTarget = new Map<string, Set<string>>(), pendingTarget = new Map<string, string>();
|
|
110
|
+
const syncPendingOutbox = (id: string) => {
|
|
111
|
+
const row = tables.get("outbox")?.get(id);
|
|
112
|
+
const record = row && read<OutboxRecord>(row);
|
|
113
|
+
if (record?.state === "pending") {
|
|
114
|
+
if (!pendingOutbox.has(id)) pendingOutbox.set(id, 0);
|
|
115
|
+
if (!pendingTarget.has(id)) {
|
|
116
|
+
const key = pair(record.envelope.target.relay, record.envelope.target.id);
|
|
117
|
+
pendingTarget.set(id, key);
|
|
118
|
+
const ids = pendingByTarget.get(key) ?? new Set<string>(); ids.add(id); pendingByTarget.set(key, ids);
|
|
119
|
+
}
|
|
120
|
+
} else {
|
|
121
|
+
pendingOutbox.delete(id);
|
|
122
|
+
const key = pendingTarget.get(id);
|
|
123
|
+
if (key !== undefined) { pendingTarget.delete(id); const ids = pendingByTarget.get(key)!; ids.delete(id); if (!ids.size) pendingByTarget.delete(key); }
|
|
124
|
+
}
|
|
125
|
+
};
|
|
126
|
+
const outboxOrder = (id: string) => tables.get("outbox")!.get(id)!.order;
|
|
127
|
+
const pendingAhead = (id: string): string[] => {
|
|
128
|
+
const key = pendingTarget.get(id); if (key === undefined) return [];
|
|
129
|
+
const order = outboxOrder(id);
|
|
130
|
+
return [...pendingByTarget.get(key)!].filter(other => outboxOrder(other) < order).sort((a, b) => outboxOrder(a) - outboxOrder(b));
|
|
131
|
+
};
|
|
132
|
+
// Cell-derived indexes, updated with every cell change including rollback; never a second copy of the data.
|
|
133
|
+
const eventsByTask = new Map<string, Set<string>>(), activeTasks = new Set<string>();
|
|
134
|
+
const reindex = (name: string, key: string, before: Cell | undefined, after: Cell | undefined) => {
|
|
135
|
+
if (name === "tasks") {
|
|
136
|
+
if (after && read<TaskRecord>(after).status === "active") activeTasks.add(key); else activeTasks.delete(key);
|
|
137
|
+
} else if (name === "events") {
|
|
138
|
+
if (before) { const taskId = read<TaskEvent>(before).taskId, ids = eventsByTask.get(taskId); ids?.delete(key); if (ids?.size === 0) eventsByTask.delete(taskId); }
|
|
139
|
+
if (after) { const taskId = read<TaskEvent>(after).taskId, ids = eventsByTask.get(taskId) ?? new Set<string>(); ids.add(key); eventsByTask.set(taskId, ids); }
|
|
140
|
+
}
|
|
141
|
+
};
|
|
64
142
|
let bytes = 0, entries = 0, sequence = 0, closed = false;
|
|
143
|
+
const listeners = new Set<() => void>();
|
|
144
|
+
// Listeners run after state is final; a failure is logged, not rethrown, because the write already committed.
|
|
145
|
+
const notify = () => { for (const listener of [...listeners]) { try { listener(); } catch (error) { console.error("pi-tasks: task store change listener failed", error); } } };
|
|
146
|
+
// Top-level writes refresh the pending-outbox index before notifying so listeners see committed state.
|
|
147
|
+
const committed = (name: string, key: string) => { if (frames.length) return; if (name === "outbox") syncPendingOutbox(key); notify(); };
|
|
65
148
|
const remember = (name: string, key: string, target: Map<string, Cell>, prior: Cell | undefined) => {
|
|
66
149
|
const frame = frames.at(-1), identity = JSON.stringify([name, key]);
|
|
67
|
-
if (frame && !frame.has(identity)) frame.set(identity, { table: target, key, prior });
|
|
150
|
+
if (frame && !frame.has(identity)) frame.set(identity, { name, table: target, key, prior });
|
|
68
151
|
};
|
|
69
152
|
const check = () => { if (closed) throw new TaskProtocolError("RELAY_CLOSED", "task state is closed", { retryable: false }); };
|
|
70
153
|
const table = (name: string) => { check(); let value = tables.get(name); if (!value) { value = new Map(); tables.set(name, value); } return value; };
|
|
71
|
-
const get = <T>(name: string, key: string): T | undefined => { const row = table(name).get(key); return row ?
|
|
72
|
-
const rows = <T>(name: string): T[] => [...table(name).values()].sort((a, b) => a.order - b.order).map(row =>
|
|
154
|
+
const get = <T>(name: string, key: string): T | undefined => { const row = table(name).get(key); return row ? read<T>(row) : undefined; };
|
|
155
|
+
const rows = <T>(name: string): T[] => [...table(name).values()].sort((a, b) => a.order - b.order).map(row => read<T>(row));
|
|
156
|
+
const taskEvents = (taskId: string): TaskEvent[] => { const events = table("events"); return [...eventsByTask.get(taskId) ?? []].map(id => read<TaskEvent>(events.get(id)!)); };
|
|
73
157
|
const put = (name: string, key: string, value: unknown, onlyNew = false): boolean => {
|
|
74
158
|
const target = table(name), prior = target.get(key);
|
|
75
159
|
if (onlyNew && prior) return false;
|
|
76
160
|
const text = JSON.stringify(value), size = Buffer.byteLength(text) + Buffer.byteLength(key) + 64;
|
|
77
161
|
if (bytes - (prior?.bytes ?? 0) + size > maxBytes || (!prior && entries >= maxEntries)) throw new TaskProtocolError("TASK_CAPACITY", "active task state is full", { retryable: false });
|
|
78
162
|
remember(name, key, target, prior);
|
|
79
|
-
|
|
163
|
+
const cell = { text, bytes: size, order: prior?.order ?? sequence++ };
|
|
164
|
+
target.set(key, cell); reindex(name, key, prior, cell);
|
|
80
165
|
bytes += size - (prior?.bytes ?? 0); if (!prior) entries++;
|
|
166
|
+
committed(name, key);
|
|
81
167
|
return true;
|
|
82
168
|
};
|
|
83
169
|
const remove = (name: string, key: string) => {
|
|
84
170
|
const target = table(name), prior = target.get(key); if (!prior) return;
|
|
85
171
|
remember(name, key, target, prior);
|
|
86
|
-
target.delete(key); bytes -= prior.bytes; entries--;
|
|
172
|
+
target.delete(key); reindex(name, key, prior, undefined); bytes -= prior.bytes; entries--;
|
|
173
|
+
committed(name, key);
|
|
87
174
|
};
|
|
88
175
|
const pair = (a: string, b: string) => JSON.stringify([a, b]);
|
|
89
176
|
const requireTask = (id: string) => { if (!get("tasks", id)) throw new TaskProtocolError("TASK_NOT_FOUND", "task is not active in this endpoint lifetime"); };
|
|
@@ -91,7 +178,8 @@ export function createTaskStore(options: TaskStoreOptions = {}): TaskStore {
|
|
|
91
178
|
const terminal = (task: Omit<TaskRecord, "events">): TerminalDeliveryState => {
|
|
92
179
|
const record = get<TaskOperationRecord>("operations", pair(task.taskId, TERMINAL_INTENT_OPERATION));
|
|
93
180
|
if (!record) return { state: "not_submitted" };
|
|
94
|
-
|
|
181
|
+
const type = TASK_EVENT_TYPES[record.logicalType];
|
|
182
|
+
if (!type?.terminal || !type.intent || record.envelopeIds.length !== 1) throw new Error("terminal operation is malformed");
|
|
95
183
|
const identity = { intentId: record.logicalId, intentType: record.logicalType as TerminalTaskIntentType, envelopeId: record.envelopeIds[0]!, origin: task.origin };
|
|
96
184
|
const outbox = get<OutboxRecord>("outbox", identity.envelopeId);
|
|
97
185
|
if (outbox) return { state: outbox.state, ...identity };
|
|
@@ -106,43 +194,84 @@ export function createTaskStore(options: TaskStoreOptions = {}): TaskStore {
|
|
|
106
194
|
if (bound && (bound.relay !== endpoint.relay || bound.id !== endpoint.id)) throw new TaskProtocolError("RELAY_RESET", "delivery checkpoint belongs to a retired endpoint", { retryable: false });
|
|
107
195
|
return JSON.stringify([endpoint.relay, endpoint.id, store.getRelayTransportBinding()?.epoch]);
|
|
108
196
|
};
|
|
109
|
-
|
|
197
|
+
// A mutable copy: callers edit it before writing it back, and the cached cell value is frozen.
|
|
198
|
+
const checkpoint = (endpoint: TaskEndpoint): Checkpoint => structuredClone(get<Checkpoint>("checkpoints", checkpointKey(endpoint))) ?? { frontier: store.getReceiveCursor(), highWater: store.getReceiveCursor(), pending: {}, requested: {} };
|
|
110
199
|
const writeCheckpoint = (endpoint: TaskEndpoint, value: Checkpoint) => { put("checkpoints", checkpointKey(endpoint), value); };
|
|
111
200
|
const store: TaskStore = {
|
|
112
201
|
transaction<T>(operation: () => T): T {
|
|
113
202
|
check(); const priorBytes = bytes, priorEntries = entries, priorSequence = sequence;
|
|
114
203
|
const frame = new Map<string, Undo>(); frames.push(frame);
|
|
204
|
+
let result: T, topLevelChange = false;
|
|
115
205
|
try {
|
|
116
|
-
|
|
206
|
+
result = operation();
|
|
117
207
|
if (result instanceof Promise) throw new TypeError("task transactions must be synchronous");
|
|
118
208
|
const parent = frames.at(-2);
|
|
119
|
-
if (parent)
|
|
120
|
-
|
|
209
|
+
if (parent) {
|
|
210
|
+
for (const [key, item] of frame) if (!parent.has(key)) parent.set(key, item);
|
|
211
|
+
} else {
|
|
212
|
+
for (const item of frame.values()) if (item.table === tables.get("outbox")) syncPendingOutbox(item.key);
|
|
213
|
+
topLevelChange = frame.size > 0;
|
|
214
|
+
}
|
|
121
215
|
} catch (error) {
|
|
122
|
-
for (const item of [...frame.values()].reverse()) {
|
|
216
|
+
for (const item of [...frame.values()].reverse()) {
|
|
217
|
+
const current = item.table.get(item.key);
|
|
218
|
+
if (item.prior) item.table.set(item.key, item.prior); else item.table.delete(item.key);
|
|
219
|
+
reindex(item.name, item.key, current, item.prior);
|
|
220
|
+
}
|
|
123
221
|
bytes = priorBytes; entries = priorEntries; sequence = priorSequence;
|
|
124
222
|
throw error;
|
|
125
223
|
} finally { frames.pop(); }
|
|
224
|
+
if (topLevelChange) notify();
|
|
225
|
+
return result;
|
|
126
226
|
},
|
|
227
|
+
onChange(listener) { const entry = () => listener(); listeners.add(entry); return () => { listeners.delete(entry); }; },
|
|
127
228
|
putTask(task) { put("tasks", task.taskId, task, true); },
|
|
128
|
-
getTask(id) { const task = get<Omit<TaskRecord, "events">>("tasks", id); return task ? snapshot(task,
|
|
229
|
+
getTask(id) { const task = get<Omit<TaskRecord, "events">>("tasks", id); return task ? snapshot(task, taskEvents(id)) : undefined; },
|
|
129
230
|
listTasks() {
|
|
130
|
-
|
|
131
|
-
for (const event of rows<TaskEvent>("events")) { const list = events.get(event.taskId) ?? []; list.push(event); events.set(event.taskId, list); }
|
|
132
|
-
return rows<Omit<TaskRecord, "events">>("tasks").sort((a, b) => a.createdAt - b.createdAt || a.taskId.localeCompare(b.taskId)).map(task => snapshot(task, events.get(task.taskId) ?? []));
|
|
231
|
+
return rows<Omit<TaskRecord, "events">>("tasks").sort((a, b) => a.createdAt - b.createdAt || a.taskId.localeCompare(b.taskId)).map(task => snapshot(task, taskEvents(task.taskId)));
|
|
133
232
|
},
|
|
233
|
+
hasActiveTasks() { check(); return activeTasks.size > 0; },
|
|
134
234
|
setStatus(id, status) { const task = get<Omit<TaskRecord, "events">>("tasks", id); if (task) put("tasks", id, { ...task, status }); },
|
|
235
|
+
setTarget(id, target) { const task = get<Omit<TaskRecord, "events">>("tasks", id); if (task) put("tasks", id, { ...task, target }); },
|
|
135
236
|
appendEvent(event) { requireTask(event.taskId); return put("events", event.eventId, event, true); },
|
|
136
|
-
putOutbox(envelope) { put("outbox", envelope.envelopeId, { envelope, state: "pending" }, true); },
|
|
237
|
+
putOutbox(envelope) { put("outbox", envelope.envelopeId, { envelope, state: "pending" }, true); if (!frames.length) syncPendingOutbox(envelope.envelopeId); },
|
|
137
238
|
outbox(status) { return rows<OutboxRecord>("outbox").filter(item => item.state === status); },
|
|
138
|
-
|
|
239
|
+
getOutboxRecord(id) { return get<OutboxRecord>("outbox", id); },
|
|
240
|
+
hasPendingOutbox() { check(); return pendingOutbox.size > 0; },
|
|
241
|
+
selectPendingOutbox(limit, now, maxInspections = TASK_WORK_LIMITS.retryInspections) {
|
|
242
|
+
check();
|
|
243
|
+
const selected: OutboxRecord[] = [];
|
|
244
|
+
// Also bound inspection when the queue is cooling down. Rotate even skipped IDs.
|
|
245
|
+
const inspections = Math.min(pendingOutbox.size, TASK_WORK_LIMITS.retryInspections, maxInspections);
|
|
246
|
+
for (let i = 0; i < inspections && selected.length < limit; i++) {
|
|
247
|
+
const [id, retryAt] = pendingOutbox.entries().next().value!;
|
|
248
|
+
pendingOutbox.delete(id); pendingOutbox.set(id, retryAt);
|
|
249
|
+
// Only a target's oldest pending envelope is sendable; inspecting any of its ids offers the head.
|
|
250
|
+
const head = pendingAhead(id)[0] ?? id;
|
|
251
|
+
if (pendingOutbox.get(head)! <= now && !selected.some(record => record.envelope.envelopeId === head)) {
|
|
252
|
+
const record = get<OutboxRecord>("outbox", head);
|
|
253
|
+
if (record?.state === "pending") selected.push(record);
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
return selected;
|
|
257
|
+
},
|
|
258
|
+
pendingOutboxAhead(id) { check(); return pendingAhead(id); },
|
|
259
|
+
outboxRetryAt(id) { check(); return pendingOutbox.get(id) ?? 0; },
|
|
260
|
+
postponeOutbox(id, until) { check(); if (pendingOutbox.has(id)) pendingOutbox.set(id, until); },
|
|
261
|
+
markOutboxAccepted(id) { const prior = get<OutboxRecord>("outbox", id); if (prior) put("outbox", id, { ...prior, state: "accepted" }); if (!frames.length) syncPendingOutbox(id); },
|
|
139
262
|
quarantineOutbox(id, input) {
|
|
140
263
|
store.transaction(() => { const prior = get<OutboxRecord>("outbox", id); if (!prior) return;
|
|
141
264
|
put("quarantine", id, { ...input, envelope: prior.envelope, priorState: prior.state }, true); remove("outbox", id);
|
|
142
265
|
});
|
|
143
266
|
},
|
|
144
267
|
quarantinedOutbox() { return rows<OutboxQuarantineRecord>("quarantine"); },
|
|
268
|
+
getQuarantine(id) { return get<OutboxQuarantineRecord>("quarantine", id); },
|
|
145
269
|
persistInbox(envelope, cursor) { return put("inbox", envelope.envelopeId, { envelope, cursor }, true); },
|
|
270
|
+
inboxOrder(id) { return table("inbox").get(id)?.order; },
|
|
271
|
+
inboxEnvelope(id) { return get<{ envelope: RelayEnvelope }>("inbox", id)?.envelope; },
|
|
272
|
+
enqueuePresentation(details) { requireTask(details.taskId); put("presentation", details.eventId, { taskId: details.taskId, eventId: details.eventId }, true); },
|
|
273
|
+
pendingPresentations() { return [...table("presentation").values()].sort((a, b) => a.order - b.order).map(row => ({ ...read<PresentationRecord>(row), order: row.order })); },
|
|
274
|
+
completePresentation(eventId) { remove("presentation", eventId); },
|
|
146
275
|
getReceiveCursor() { return state<string>("cursor") ?? "0"; },
|
|
147
276
|
setReceiveCursor(cursor) { put("state", "cursor", cursor); },
|
|
148
277
|
trackRelayDeliveries(endpoint, deliveries) {
|
|
@@ -156,12 +285,21 @@ export function createTaskStore(options: TaskStoreOptions = {}): TaskStore {
|
|
|
156
285
|
writeCheckpoint(endpoint, value);
|
|
157
286
|
},
|
|
158
287
|
pendingRelayEnvelopeId(endpoint, cursor) { return checkpoint(endpoint).pending[cursor]; },
|
|
159
|
-
|
|
288
|
+
pendingRelayDeliveries(endpoint) {
|
|
289
|
+
return Object.entries(checkpoint(endpoint).pending).map(([cursor, envelopeId]) => ({ cursor, envelopeId }))
|
|
290
|
+
.sort((left, right) => BigInt(left.cursor) < BigInt(right.cursor) ? -1 : 1);
|
|
291
|
+
},
|
|
292
|
+
relayIntakeCursor(endpoint) { return checkpoint(endpoint).highWater; },
|
|
293
|
+
requestRelayAcknowledgement(endpoint, cursor, retryAt = 0) {
|
|
160
294
|
const value = checkpoint(endpoint);
|
|
161
295
|
if (value.pending[cursor] === undefined) throw new TaskProtocolError("INVALID_CURSOR", "relay delivery is not pending in this endpoint scope");
|
|
162
|
-
value.requested[cursor] =
|
|
296
|
+
value.requested[cursor] = retryAt; writeCheckpoint(endpoint, value);
|
|
163
297
|
},
|
|
164
|
-
requestedRelayAcknowledgements(endpoint
|
|
298
|
+
requestedRelayAcknowledgements(endpoint, limit = Number.MAX_SAFE_INTEGER, now = Date.now()) {
|
|
299
|
+
const value = checkpoint(endpoint);
|
|
300
|
+
return Object.keys(value.requested).filter(cursor => value.requested[cursor]! <= now).slice(0, limit);
|
|
301
|
+
},
|
|
302
|
+
relayAcknowledgementRetryAt(endpoint, cursor) { return checkpoint(endpoint).requested[cursor] ?? 0; },
|
|
165
303
|
acknowledgeRelayCursor(endpoint, cursor) {
|
|
166
304
|
store.transaction(() => {
|
|
167
305
|
const value = checkpoint(endpoint); if (value.pending[cursor] === undefined) return;
|
|
@@ -173,15 +311,31 @@ export function createTaskStore(options: TaskStoreOptions = {}): TaskStore {
|
|
|
173
311
|
});
|
|
174
312
|
},
|
|
175
313
|
putIntent(intentId, taskId, envelopeId) { put("intents", intentId, { taskId, envelopeId }, true); },
|
|
314
|
+
getIntentEnvelope(intentId) {
|
|
315
|
+
const intent = get<{ taskId: string; envelopeId: string }>("intents", intentId);
|
|
316
|
+
if (!intent) return undefined;
|
|
317
|
+
const envelope = (get<OutboxRecord>("outbox", intent.envelopeId) ?? get<OutboxQuarantineRecord>("quarantine", intent.envelopeId))?.envelope;
|
|
318
|
+
return envelope?.taskId === intent.taskId ? envelope : undefined;
|
|
319
|
+
},
|
|
176
320
|
putInsertionReceipt(taskId, eventId) { requireTask(taskId); return put("insertions", pair(taskId, eventId), true, true); },
|
|
177
321
|
reserveTaskOperation(input) { requireTask(input.taskId); const key = pair(input.taskId, input.operation); const created = put("operations", key, input, true); return { created, record: get<TaskOperationRecord>("operations", key)! }; },
|
|
322
|
+
discardUnsentOperation(taskId, operation) {
|
|
323
|
+
store.transaction(() => {
|
|
324
|
+
const key = pair(taskId, operation), record = get<TaskOperationRecord>("operations", key); if (!record) return;
|
|
325
|
+
for (const id of record.envelopeIds) {
|
|
326
|
+
if (get<OutboxRecord>("outbox", id)?.state !== "pending") throw new Error("only a pending, never-sent operation can be discarded");
|
|
327
|
+
remove("outbox", id);
|
|
328
|
+
}
|
|
329
|
+
remove("intents", record.logicalId); remove("operations", key);
|
|
330
|
+
});
|
|
331
|
+
},
|
|
178
332
|
getEndpointGeneration() { return state<string>("generation"); },
|
|
179
333
|
setEndpointGeneration(value) { put("state", "generation", value); },
|
|
180
334
|
getEndpointBinding() { return state<TaskEndpoint>("endpoint"); },
|
|
181
335
|
setEndpointBinding(value) { put("state", "endpoint", value); },
|
|
182
336
|
getRelayTransportBinding() { return state<RelayTransportBinding>("transport"); },
|
|
183
337
|
setRelayTransportBinding(value) { put("state", "transport", value); },
|
|
184
|
-
clear() { check(); if (frames.length) throw new Error("cannot clear task state during a transaction"); tables.clear(); bytes = entries = sequence = 0; },
|
|
338
|
+
clear() { check(); if (frames.length) throw new Error("cannot clear task state during a transaction"); tables.clear(); eventsByTask.clear(); activeTasks.clear(); pendingOutbox.clear(); pendingByTarget.clear(); pendingTarget.clear(); bytes = entries = sequence = 0; },
|
|
185
339
|
close() { if (closed) return; store.clear(); closed = true; },
|
|
186
340
|
};
|
|
187
341
|
return store;
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import { TaskProtocolError } from "./task-protocol";
|
|
2
|
+
|
|
3
|
+
export const TASK_WORK_LIMITS = { scopedSends: 56, retries: 8, acknowledgements: 48, pageItems: 50,
|
|
4
|
+
deadlineMs: 2_000, retrySliceMs: 250, ackRepairSliceMs: 250, capacityCooldownMs: 5_000, retryInspections: 64,
|
|
5
|
+
// Fetched-but-unacknowledged relay deliveries; intake pauses at this bound until ACKs drain it.
|
|
6
|
+
pendingDeliveries: 200 } as const;
|
|
7
|
+
|
|
8
|
+
/** Time source for delivery budgets: the background deadline, admission slices and retry cooldowns. Tests inject virtual time. */
|
|
9
|
+
export interface TaskTimers {
|
|
10
|
+
/** Wall-clock milliseconds, comparable with stored retry times. */
|
|
11
|
+
now(): number;
|
|
12
|
+
/** Monotonic milliseconds for admission slices. */
|
|
13
|
+
elapsed(): number;
|
|
14
|
+
/** A signal that aborts once `ms` have passed. */
|
|
15
|
+
timeout(ms: number): AbortSignal;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export const SYSTEM_TASK_TIMERS: TaskTimers = { now: () => Date.now(), elapsed: () => performance.now(), timeout: ms => AbortSignal.timeout(ms) };
|
|
19
|
+
|
|
20
|
+
/** Required envelopes a phased pass persisted; its ship phase sends them in persist order. */
|
|
21
|
+
export interface TaskShipment {
|
|
22
|
+
readonly envelopeIds: readonly string[];
|
|
23
|
+
/** Throws the blocked-delivery error once the envelopes were offered. */
|
|
24
|
+
readonly verify?: () => void;
|
|
25
|
+
/** Runs once every envelope is accepted, e.g. staging the raw ACK that accepted fan-out authorizes. */
|
|
26
|
+
readonly accepted?: () => void;
|
|
27
|
+
/** Set by the ship phase, with the first send or blocked-delivery failure, if any. */
|
|
28
|
+
shipped: boolean;
|
|
29
|
+
failure?: unknown;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** Shared counts for nested receipts/fanout; background I/O also has one finite window. */
|
|
33
|
+
export interface TaskWorkScope {
|
|
34
|
+
readonly signal: AbortSignal;
|
|
35
|
+
readonly attemptedSends: Set<string>;
|
|
36
|
+
readonly attemptedAcknowledgements: Set<string>;
|
|
37
|
+
readonly blockedRetryTargets: Set<string>;
|
|
38
|
+
retryStarted: boolean;
|
|
39
|
+
/**
|
|
40
|
+
* Fixed at creation for a refresh pass: its driver runs intake, presentation, required sends,
|
|
41
|
+
* ACKs and old-work retry as explicit phases, so core operations never start old-work retry
|
|
42
|
+
* or ACK repair on their own.
|
|
43
|
+
*/
|
|
44
|
+
readonly phased: boolean;
|
|
45
|
+
/** Deferred required sends of a phased pass, in persist order. */
|
|
46
|
+
readonly shipments: TaskShipment[];
|
|
47
|
+
claimSend(envelopeId: string, retry: boolean): boolean;
|
|
48
|
+
claimAcknowledgement(cursor: string): boolean;
|
|
49
|
+
canAcknowledge(): boolean;
|
|
50
|
+
remainingAcknowledgements(): number;
|
|
51
|
+
canRetry(): boolean;
|
|
52
|
+
canRepairAcknowledgement(): boolean;
|
|
53
|
+
backgroundSignal(): AbortSignal;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export function createTaskWorkScope(signal?: AbortSignal, mode: "background" | "foreground" | "phased" = "background", timers: TaskTimers = SYSTEM_TASK_TIMERS): TaskWorkScope {
|
|
57
|
+
const caller = signal ?? new AbortController().signal;
|
|
58
|
+
let background: AbortSignal | undefined;
|
|
59
|
+
const backgroundSignal = () => background ??= AbortSignal.any([caller, timers.timeout(TASK_WORK_LIMITS.deadlineMs)]);
|
|
60
|
+
// Foreground requests retain the transport/caller timeout. Their speculative
|
|
61
|
+
// retries start a bounded background window only after required sends finish.
|
|
62
|
+
const active = mode === "foreground" ? caller : backgroundSignal();
|
|
63
|
+
const attemptedSends = new Set<string>(), attemptedAcknowledgements = new Set<string>(), blockedRetryTargets = new Set<string>();
|
|
64
|
+
let scopedSends = 0, retrySends = 0, acknowledgements = 0;
|
|
65
|
+
let retryUntil: number | undefined, repairUntil: number | undefined;
|
|
66
|
+
return {
|
|
67
|
+
signal: active, attemptedSends, attemptedAcknowledgements, blockedRetryTargets, retryStarted: false, phased: mode === "phased", shipments: [],
|
|
68
|
+
backgroundSignal,
|
|
69
|
+
claimSend(id, retry) {
|
|
70
|
+
if ((retry ? backgroundSignal() : active).aborted || attemptedSends.has(id) || (retry ? retrySends >= TASK_WORK_LIMITS.retries : scopedSends >= TASK_WORK_LIMITS.scopedSends)) return false;
|
|
71
|
+
attemptedSends.add(id);
|
|
72
|
+
if (retry) retrySends++; else scopedSends++;
|
|
73
|
+
return true;
|
|
74
|
+
},
|
|
75
|
+
claimAcknowledgement(cursor) {
|
|
76
|
+
if (active.aborted || attemptedAcknowledgements.has(cursor) || acknowledgements >= TASK_WORK_LIMITS.acknowledgements) return false;
|
|
77
|
+
attemptedAcknowledgements.add(cursor); acknowledgements++;
|
|
78
|
+
return true;
|
|
79
|
+
},
|
|
80
|
+
canAcknowledge() { return !active.aborted && acknowledgements < TASK_WORK_LIMITS.acknowledgements; },
|
|
81
|
+
remainingAcknowledgements() { return active.aborted ? 0 : TASK_WORK_LIMITS.acknowledgements - acknowledgements; },
|
|
82
|
+
// Slices stop admission, not completion: an admitted request gets the
|
|
83
|
+
// remaining background window, including cancellation and late fences.
|
|
84
|
+
canRetry() { return !backgroundSignal().aborted && timers.elapsed() < (retryUntil ??= timers.elapsed() + TASK_WORK_LIMITS.retrySliceMs); },
|
|
85
|
+
canRepairAcknowledgement() { return !backgroundSignal().aborted && timers.elapsed() < (repairUntil ??= timers.elapsed() + TASK_WORK_LIMITS.ackRepairSliceMs); },
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
export function deferredTaskDelivery(details?: Record<string, unknown>): TaskProtocolError {
|
|
90
|
+
return new TaskProtocolError("TASK_DELIVERY_DEFERRED", "task delivery deferred by the local work budget or retry cooldown", { retryable: true, details });
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/** Known admission backpressure gets at least one normal polling interval, without another timer. */
|
|
94
|
+
export function capacityRetryAt(error: unknown, now = Date.now()): number | undefined {
|
|
95
|
+
if (!(error instanceof TaskProtocolError) || error.code !== "RELAY_CAPACITY" || !error.retryable) return undefined;
|
|
96
|
+
const hint = error.details?.retryAfterMs;
|
|
97
|
+
return now + Math.max(TASK_WORK_LIMITS.capacityCooldownMs, typeof hint === "number" && Number.isFinite(hint) && hint >= 0 ? hint : 0);
|
|
98
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { createTaskCore } from "./task-core";
|
|
2
2
|
import type { TaskCore } from "./task-core";
|
|
3
|
+
import { RESET_CODES, RETRY_CODES, taskErrorMessage } from "./task-errors";
|
|
3
4
|
import { TASK_PROTOCOL_VERSION, TaskProtocolError } from "./task-protocol";
|
|
4
5
|
import type { RelayTransportBinding, TaskEndpoint, TaskRelay } from "./task-protocol";
|
|
5
6
|
import type { TaskStore } from "./task-store";
|
|
@@ -8,8 +9,6 @@ import { fromWolfpackEnvelope, toWolfpackEnvelope, WOLFPACK_TASK_RELAY_ID } from
|
|
|
8
9
|
export const VOLATILE_PROFILE = "volatile-v1";
|
|
9
10
|
const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
|
10
11
|
const CURSOR = /^(0|[1-9][0-9]{0,31})$/;
|
|
11
|
-
const RESET_CODES = new Set(["RELAY_RESET", "REGISTRATION_EXPIRED", "SOURCE_MISMATCH", "CALLER_DEAD", "CALLER_NOT_FOUND"]);
|
|
12
|
-
const RETRY_CODES = new Set(["RELAY_CAPACITY", "RELAY_UNAVAILABLE", "PEER_UNREACHABLE"]);
|
|
13
12
|
|
|
14
13
|
export interface VolatileTaskSessionOptions {
|
|
15
14
|
/** Explicit trusted endpoint ingress URL. No default route or installed server. */
|
|
@@ -123,6 +122,7 @@ class Transport {
|
|
|
123
122
|
private expiresAt = 0;
|
|
124
123
|
private renewing: Promise<RelayTransportBinding> | undefined;
|
|
125
124
|
private active = 0;
|
|
125
|
+
private readonly waiters: Array<{ reserved: boolean; grant(): void }> = [];
|
|
126
126
|
constructor(private url: string, private caller: string, private generation: string, prior: RelayTransportBinding | undefined,
|
|
127
127
|
private fetcher: typeof fetch, private timeoutMs: number, private onReset: (expected: RelayTransportBinding | undefined) => void) { this.binding = prior; }
|
|
128
128
|
stop(): void { this.stopSignal.abort(); }
|
|
@@ -202,13 +202,34 @@ class Transport {
|
|
|
202
202
|
},
|
|
203
203
|
};
|
|
204
204
|
}
|
|
205
|
+
// FIFO in-flight slots: 8 for ordinary requests, 2 more held for acknowledge/connect.
|
|
206
|
+
// A waiter has sent nothing, so its abort is a plain retryable ABORTED; RELAY_CAPACITY
|
|
207
|
+
// only ever comes from the server.
|
|
208
|
+
private fits(reserved: boolean): boolean { return this.active < (reserved ? 10 : 8); }
|
|
209
|
+
private acquire(reserved: boolean, signal?: AbortSignal): Promise<void> {
|
|
210
|
+
if (this.fits(reserved)) { this.active++; return Promise.resolve(); }
|
|
211
|
+
return new Promise((resolve, reject) => {
|
|
212
|
+
const cleanup = () => { signal?.removeEventListener("abort", cancel); this.stopSignal.signal.removeEventListener("abort", cancel); };
|
|
213
|
+
const waiter = { reserved, grant: () => { cleanup(); this.active++; resolve(); } };
|
|
214
|
+
const cancel = () => {
|
|
215
|
+
cleanup(); this.waiters.splice(this.waiters.indexOf(waiter), 1);
|
|
216
|
+
reject(this.stopSignal.signal.aborted ? failure("RELAY_RESET", false, { mayHaveBeenDelivered: true }) : failure("ABORTED", true));
|
|
217
|
+
};
|
|
218
|
+
this.waiters.push(waiter);
|
|
219
|
+
signal?.addEventListener("abort", cancel, { once: true }); this.stopSignal.signal.addEventListener("abort", cancel, { once: true });
|
|
220
|
+
if (signal?.aborted) cancel();
|
|
221
|
+
});
|
|
222
|
+
}
|
|
223
|
+
private release(): void {
|
|
224
|
+
this.active--;
|
|
225
|
+
for (const waiter of [...this.waiters]) if (this.fits(waiter.reserved)) { this.waiters.splice(this.waiters.indexOf(waiter), 1); waiter.grant(); }
|
|
226
|
+
}
|
|
205
227
|
private async request(body: Record<string, unknown>, signal?: AbortSignal): Promise<{ epoch: string; value: unknown }> {
|
|
206
228
|
this.assertActive();
|
|
207
229
|
const reserved = body.operation === "acknowledge" || body.operation === "connect";
|
|
208
|
-
if (this.active >= (reserved ? 10 : 8)) throw failure("RELAY_CAPACITY", true);
|
|
209
230
|
const encoded = JSON.stringify(body);
|
|
210
231
|
if (Buffer.byteLength(encoded) > 64 * 1024) throw failure("PAYLOAD_TOO_LARGE", false);
|
|
211
|
-
this.
|
|
232
|
+
await this.acquire(reserved, signal);
|
|
212
233
|
const controller = new AbortController();
|
|
213
234
|
let timedOut = false;
|
|
214
235
|
const abort = () => controller.abort();
|
|
@@ -268,14 +289,15 @@ class Transport {
|
|
|
268
289
|
if (!result.ok || data.ok !== true || !uuid(data.epoch)) throw failure("INVALID_RESPONSE", true);
|
|
269
290
|
return { epoch: data.epoch, value: data.value };
|
|
270
291
|
} catch (error) {
|
|
271
|
-
|
|
272
|
-
|
|
292
|
+
// Every failure of an admitted request names its relay operation for the status line.
|
|
293
|
+
const known = error instanceof TaskProtocolError ? error : failure("RELAY_UNAVAILABLE", true, { mayHaveBeenDelivered: true });
|
|
294
|
+
throw new TaskProtocolError(known.code, known.message, { retryable: known.retryable, details: { ...known.details, operation: body.operation } });
|
|
273
295
|
} finally {
|
|
274
|
-
clearTimeout(timer); this.
|
|
296
|
+
clearTimeout(timer); this.release(); signal?.removeEventListener("abort", abort); this.stopSignal.signal.removeEventListener("abort", abort); controller.signal.removeEventListener("abort", rejectAbort);
|
|
275
297
|
}
|
|
276
298
|
}
|
|
277
299
|
}
|
|
278
|
-
function failure(code: string, retryable: boolean, details?: Record<string, unknown>): TaskProtocolError { return new TaskProtocolError(code, code, { retryable, details }); }
|
|
300
|
+
function failure(code: string, retryable: boolean, details?: Record<string, unknown>): TaskProtocolError { return new TaskProtocolError(code, taskErrorMessage(code), { retryable, details }); }
|
|
279
301
|
function record(value: unknown): value is Record<string, unknown> { return value !== null && typeof value === "object" && !Array.isArray(value); }
|
|
280
302
|
function text(value: unknown): value is string { return typeof value === "string" && value.length > 0 && value.length <= 512; }
|
|
281
303
|
function uuid(value: unknown): value is string { return typeof value === "string" && UUID.test(value); }
|