@opengeni/worker-bundle 0.7.5 → 0.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunk-BJUW5LCM.js +325 -0
- package/dist/chunk-BJUW5LCM.js.map +1 -0
- package/dist/host-export.d.ts +70 -0
- package/dist/host-export.js +7 -0
- package/dist/host-export.js.map +1 -0
- package/dist/index.d.ts +98 -6
- package/dist/index.js +2812 -813
- package/dist/index.js.map +1 -1
- package/dist/workflow-bundle.js +12358 -0
- package/package.json +18 -12
- package/src/activities/agent-turn.ts +2045 -545
- package/src/activities/capabilities.ts +9 -2
- package/src/activities/codex-capacity.ts +6 -11
- package/src/activities/codex-rotation.ts +75 -86
- package/src/activities/context-compaction.ts +104 -24
- package/src/activities/environment.ts +367 -92
- package/src/activities/expiring-material-renewal.ts +143 -0
- package/src/activities/goals.ts +4 -0
- package/src/activities/mcp-credentials.ts +38 -0
- package/src/activities/packs.ts +115 -2
- package/src/activities/run-credential-renewal.ts +70 -0
- package/src/activities/run-credentials.ts +132 -0
- package/src/activities/run-input.ts +23 -0
- package/src/activities/sandbox-lease.ts +1 -1
- package/src/activities/scheduled-tasks.ts +21 -0
- package/src/activities/session-state.ts +72 -0
- package/src/activities/streaming.ts +1 -0
- package/src/activities/toolspace-token-renewal.ts +57 -0
- package/src/activities/types.ts +44 -6
- package/src/activities/workspace-capture.ts +113 -93
- package/src/activities.ts +10 -2
- package/src/host-export-pump.ts +416 -0
- package/src/host-export.ts +22 -0
- package/src/http.ts +77 -28
- package/src/index.ts +360 -156
- package/src/sandbox-resume.ts +47 -1
- package/src/sandbox-routing.ts +1 -1
- package/src/worker-service-lifecycle.ts +104 -0
- package/src/workflows/activities.ts +13 -7
- package/src/workflows/session.ts +219 -23
- package/src/workflows.ts +1 -0
|
@@ -0,0 +1,325 @@
|
|
|
1
|
+
// src/host-export-pump.ts
|
|
2
|
+
import {
|
|
3
|
+
acknowledgeHostExportBatch,
|
|
4
|
+
claimHostExportBatch,
|
|
5
|
+
disableHostExportConsumer,
|
|
6
|
+
failHostExportBatch,
|
|
7
|
+
getHostExportConsumerStatus,
|
|
8
|
+
pruneHostExportOutbox,
|
|
9
|
+
registerHostExportConsumer,
|
|
10
|
+
retireHostExportConsumer
|
|
11
|
+
} from "@opengeni/db";
|
|
12
|
+
function createHostExportPump(options) {
|
|
13
|
+
if (!options.eventSink && !options.usageSink) {
|
|
14
|
+
throw new Error("createHostExportPump requires at least one host sink");
|
|
15
|
+
}
|
|
16
|
+
const instanceId = validateInstanceId(
|
|
17
|
+
options.instanceId ?? `worker-${process.pid}-${crypto.randomUUID()}`
|
|
18
|
+
);
|
|
19
|
+
const pollIntervalMs = boundedInteger(options.pollIntervalMs, 1e3, 25, 6e4);
|
|
20
|
+
const leaseSeconds = boundedInteger(options.leaseSeconds, 60, 5, 300);
|
|
21
|
+
const batchLimit = boundedInteger(options.batchLimit, 100, 1, 256);
|
|
22
|
+
const batchMaxBytes = boundedInteger(options.batchMaxBytes, 1048576, 98304, 4194304);
|
|
23
|
+
const maxFailures = boundedInteger(options.maxFailures, 20, 1, 1e3);
|
|
24
|
+
const maxBatchesPerPoll = boundedInteger(options.maxBatchesPerPoll, 32, 1, 1e3);
|
|
25
|
+
const pruneGraceSeconds = boundedInteger(options.pruneGraceSeconds, 3600, 0, 604800);
|
|
26
|
+
const pruneLimit = boundedInteger(options.pruneLimit, 1e3, 1, 1e4);
|
|
27
|
+
let running = false;
|
|
28
|
+
let loopPromise = null;
|
|
29
|
+
let waitTimer = null;
|
|
30
|
+
let wakeWait = null;
|
|
31
|
+
let lifecycleTail = Promise.resolve();
|
|
32
|
+
let drainTail = Promise.resolve();
|
|
33
|
+
const retiringKinds = /* @__PURE__ */ new Set();
|
|
34
|
+
const configuredConsumer = (kind) => kind === "session_event" ? options.eventSink?.consumerId ?? null : options.usageSink?.consumerId ?? null;
|
|
35
|
+
const recordResult = (result, durationSeconds) => {
|
|
36
|
+
const labels = { kind: result.kind, outcome: result.outcome };
|
|
37
|
+
options.observability?.incrementCounter({
|
|
38
|
+
name: "opengeni_host_export_batches_total",
|
|
39
|
+
help: "Durable host export batch attempts by outcome.",
|
|
40
|
+
labels
|
|
41
|
+
});
|
|
42
|
+
options.observability?.observeHistogram({
|
|
43
|
+
name: "opengeni_host_export_batch_duration_seconds",
|
|
44
|
+
help: "Duration of durable host export batch attempts.",
|
|
45
|
+
labels: { kind: result.kind },
|
|
46
|
+
value: durationSeconds
|
|
47
|
+
});
|
|
48
|
+
if (result.outcome === "delivered") {
|
|
49
|
+
options.observability?.incrementCounter({
|
|
50
|
+
name: "opengeni_host_export_records_total",
|
|
51
|
+
help: "Durable host export records acknowledged by a host sink.",
|
|
52
|
+
labels: { kind: result.kind },
|
|
53
|
+
amount: result.count
|
|
54
|
+
});
|
|
55
|
+
options.observability?.setGauge({
|
|
56
|
+
name: "opengeni_host_export_consumer_blocked",
|
|
57
|
+
help: "Whether the durable host export consumer is blocked.",
|
|
58
|
+
labels: { kind: result.kind },
|
|
59
|
+
value: 0
|
|
60
|
+
});
|
|
61
|
+
} else if (result.outcome === "failed") {
|
|
62
|
+
options.observability?.setGauge({
|
|
63
|
+
name: "opengeni_host_export_consumer_blocked",
|
|
64
|
+
help: "Whether the durable host export consumer is blocked.",
|
|
65
|
+
labels: { kind: result.kind },
|
|
66
|
+
value: result.blocked ? 1 : 0
|
|
67
|
+
});
|
|
68
|
+
} else if (result.outcome === "blocked") {
|
|
69
|
+
options.observability?.setGauge({
|
|
70
|
+
name: "opengeni_host_export_consumer_blocked",
|
|
71
|
+
help: "Whether the durable host export consumer is blocked.",
|
|
72
|
+
labels: { kind: result.kind },
|
|
73
|
+
value: 1
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
};
|
|
77
|
+
const drainKind = async (kind) => {
|
|
78
|
+
if (retiringKinds.has(kind)) return { kind, outcome: "not_configured" };
|
|
79
|
+
const startedAt = performance.now();
|
|
80
|
+
const consumerId = configuredConsumer(kind);
|
|
81
|
+
if (!consumerId) {
|
|
82
|
+
const result2 = { kind, outcome: "not_configured" };
|
|
83
|
+
recordResult(result2, (performance.now() - startedAt) / 1e3);
|
|
84
|
+
return result2;
|
|
85
|
+
}
|
|
86
|
+
const leaseToken = crypto.randomUUID();
|
|
87
|
+
const common = {
|
|
88
|
+
consumerId,
|
|
89
|
+
leaseToken,
|
|
90
|
+
leaseHolderId: instanceId,
|
|
91
|
+
leaseSeconds,
|
|
92
|
+
limit: batchLimit,
|
|
93
|
+
maxBytes: batchMaxBytes
|
|
94
|
+
};
|
|
95
|
+
const settleFailure = async (error) => {
|
|
96
|
+
const message = safeErrorMessage(error);
|
|
97
|
+
let failures = 0;
|
|
98
|
+
try {
|
|
99
|
+
failures = await failHostExportBatch(options.db, {
|
|
100
|
+
kind,
|
|
101
|
+
consumerId,
|
|
102
|
+
leaseToken,
|
|
103
|
+
error: message,
|
|
104
|
+
maxFailures
|
|
105
|
+
});
|
|
106
|
+
} catch (settlementError) {
|
|
107
|
+
options.observability?.warn("host export failure settlement was stale", {
|
|
108
|
+
kind,
|
|
109
|
+
consumerId,
|
|
110
|
+
error: safeErrorMessage(settlementError)
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
const blocked = failures >= maxFailures;
|
|
114
|
+
options.observability?.warn("host export batch failed", {
|
|
115
|
+
kind,
|
|
116
|
+
consumerId,
|
|
117
|
+
failures,
|
|
118
|
+
blocked,
|
|
119
|
+
error: message
|
|
120
|
+
});
|
|
121
|
+
const result2 = {
|
|
122
|
+
kind,
|
|
123
|
+
outcome: "failed",
|
|
124
|
+
failures,
|
|
125
|
+
blocked
|
|
126
|
+
};
|
|
127
|
+
recordResult(result2, (performance.now() - startedAt) / 1e3);
|
|
128
|
+
return result2;
|
|
129
|
+
};
|
|
130
|
+
const deliverClaim = async (count, deliver) => {
|
|
131
|
+
await deliver();
|
|
132
|
+
const throughCursor = await acknowledgeHostExportBatch(options.db, {
|
|
133
|
+
kind,
|
|
134
|
+
consumerId,
|
|
135
|
+
leaseToken
|
|
136
|
+
});
|
|
137
|
+
try {
|
|
138
|
+
await pruneHostExportOutbox(options.db, {
|
|
139
|
+
kind,
|
|
140
|
+
graceSeconds: pruneGraceSeconds,
|
|
141
|
+
limit: pruneLimit
|
|
142
|
+
});
|
|
143
|
+
} catch (error) {
|
|
144
|
+
options.observability?.error("host export retention prune failed", {
|
|
145
|
+
kind,
|
|
146
|
+
consumerId,
|
|
147
|
+
error: safeErrorMessage(error)
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
const result2 = {
|
|
151
|
+
kind,
|
|
152
|
+
outcome: "delivered",
|
|
153
|
+
count,
|
|
154
|
+
throughCursor
|
|
155
|
+
};
|
|
156
|
+
recordResult(result2, (performance.now() - startedAt) / 1e3);
|
|
157
|
+
return result2;
|
|
158
|
+
};
|
|
159
|
+
try {
|
|
160
|
+
if (kind === "session_event") {
|
|
161
|
+
const sink = options.eventSink;
|
|
162
|
+
if (!sink) return { kind, outcome: "not_configured" };
|
|
163
|
+
const batch = await claimHostExportBatch(options.db, {
|
|
164
|
+
kind,
|
|
165
|
+
...common
|
|
166
|
+
});
|
|
167
|
+
if (batch) return await deliverClaim(batch.events.length, () => sink.deliverEvents(batch));
|
|
168
|
+
} else {
|
|
169
|
+
const sink = options.usageSink;
|
|
170
|
+
if (!sink) return { kind, outcome: "not_configured" };
|
|
171
|
+
const batch = await claimHostExportBatch(options.db, {
|
|
172
|
+
kind,
|
|
173
|
+
...common
|
|
174
|
+
});
|
|
175
|
+
if (batch) return await deliverClaim(batch.events.length, () => sink.deliverUsage(batch));
|
|
176
|
+
}
|
|
177
|
+
} catch (error) {
|
|
178
|
+
return await settleFailure(error);
|
|
179
|
+
}
|
|
180
|
+
const status = await getHostExportConsumerStatus(options.db, {
|
|
181
|
+
kind,
|
|
182
|
+
consumerId
|
|
183
|
+
});
|
|
184
|
+
if (status?.blockedAt) {
|
|
185
|
+
const result2 = {
|
|
186
|
+
kind,
|
|
187
|
+
outcome: "blocked",
|
|
188
|
+
failures: status.consecutiveFailures
|
|
189
|
+
};
|
|
190
|
+
recordResult(result2, (performance.now() - startedAt) / 1e3);
|
|
191
|
+
return result2;
|
|
192
|
+
}
|
|
193
|
+
const result = { kind, outcome: "idle" };
|
|
194
|
+
recordResult(result, (performance.now() - startedAt) / 1e3);
|
|
195
|
+
return result;
|
|
196
|
+
};
|
|
197
|
+
const drainConfiguredOnce = async () => {
|
|
198
|
+
const results = [];
|
|
199
|
+
if (options.eventSink) results.push(await drainKind("session_event"));
|
|
200
|
+
if (options.usageSink) results.push(await drainKind("usage_event"));
|
|
201
|
+
return results;
|
|
202
|
+
};
|
|
203
|
+
const drainOnce = () => {
|
|
204
|
+
const next = drainTail.then(drainConfiguredOnce);
|
|
205
|
+
drainTail = next.then(
|
|
206
|
+
() => void 0,
|
|
207
|
+
() => void 0
|
|
208
|
+
);
|
|
209
|
+
return next;
|
|
210
|
+
};
|
|
211
|
+
const serializeLifecycle = (operation) => {
|
|
212
|
+
const next = lifecycleTail.then(operation);
|
|
213
|
+
lifecycleTail = next.then(
|
|
214
|
+
() => void 0,
|
|
215
|
+
() => void 0
|
|
216
|
+
);
|
|
217
|
+
return next;
|
|
218
|
+
};
|
|
219
|
+
const waitForPoll = async () => {
|
|
220
|
+
await new Promise((resolve) => {
|
|
221
|
+
wakeWait = resolve;
|
|
222
|
+
waitTimer = setTimeout(resolve, pollIntervalMs);
|
|
223
|
+
waitTimer.unref?.();
|
|
224
|
+
});
|
|
225
|
+
wakeWait = null;
|
|
226
|
+
if (waitTimer) clearTimeout(waitTimer);
|
|
227
|
+
waitTimer = null;
|
|
228
|
+
};
|
|
229
|
+
return {
|
|
230
|
+
start: () => serializeLifecycle(async () => {
|
|
231
|
+
if (running) return;
|
|
232
|
+
await options.db.transaction(async (tx) => {
|
|
233
|
+
const scoped = tx;
|
|
234
|
+
if (options.eventSink) {
|
|
235
|
+
await registerHostExportConsumer(scoped, {
|
|
236
|
+
kind: "session_event",
|
|
237
|
+
consumerId: options.eventSink.consumerId
|
|
238
|
+
});
|
|
239
|
+
}
|
|
240
|
+
if (options.usageSink) {
|
|
241
|
+
await registerHostExportConsumer(scoped, {
|
|
242
|
+
kind: "usage_event",
|
|
243
|
+
consumerId: options.usageSink.consumerId
|
|
244
|
+
});
|
|
245
|
+
}
|
|
246
|
+
});
|
|
247
|
+
if (options.eventSink) retiringKinds.delete("session_event");
|
|
248
|
+
if (options.usageSink) retiringKinds.delete("usage_event");
|
|
249
|
+
running = true;
|
|
250
|
+
loopPromise = (async () => {
|
|
251
|
+
while (true) {
|
|
252
|
+
if (!running) break;
|
|
253
|
+
try {
|
|
254
|
+
for (let batch = 0; batch < maxBatchesPerPoll; batch += 1) {
|
|
255
|
+
if (!running) break;
|
|
256
|
+
const results = await drainOnce();
|
|
257
|
+
if (!results.some((result) => result.outcome === "delivered")) break;
|
|
258
|
+
}
|
|
259
|
+
} catch (error) {
|
|
260
|
+
options.observability?.error("host export pump iteration failed", {
|
|
261
|
+
error: safeErrorMessage(error)
|
|
262
|
+
});
|
|
263
|
+
}
|
|
264
|
+
if (running) await waitForPoll();
|
|
265
|
+
}
|
|
266
|
+
})();
|
|
267
|
+
}),
|
|
268
|
+
stop: () => serializeLifecycle(async () => {
|
|
269
|
+
running = false;
|
|
270
|
+
if (waitTimer) clearTimeout(waitTimer);
|
|
271
|
+
wakeWait?.();
|
|
272
|
+
await loopPromise;
|
|
273
|
+
loopPromise = null;
|
|
274
|
+
}),
|
|
275
|
+
drainOnce,
|
|
276
|
+
disable: async (kind) => {
|
|
277
|
+
const consumerId = configuredConsumer(kind);
|
|
278
|
+
if (!consumerId) throw new Error(`No ${kind} sink is configured`);
|
|
279
|
+
await disableHostExportConsumer(options.db, { kind, consumerId });
|
|
280
|
+
},
|
|
281
|
+
retire: (kind) => serializeLifecycle(async () => {
|
|
282
|
+
if (running) throw new Error("Stop the host export pump before retiring a consumer");
|
|
283
|
+
const consumerId = configuredConsumer(kind);
|
|
284
|
+
if (!consumerId) throw new Error(`No ${kind} sink is configured`);
|
|
285
|
+
retiringKinds.add(kind);
|
|
286
|
+
try {
|
|
287
|
+
await drainTail;
|
|
288
|
+
await retireHostExportConsumer(options.db, { kind, consumerId });
|
|
289
|
+
} catch (error) {
|
|
290
|
+
retiringKinds.delete(kind);
|
|
291
|
+
throw error;
|
|
292
|
+
}
|
|
293
|
+
}),
|
|
294
|
+
status: async (kind) => {
|
|
295
|
+
const consumerId = configuredConsumer(kind);
|
|
296
|
+
if (!consumerId) return null;
|
|
297
|
+
return await getHostExportConsumerStatus(options.db, {
|
|
298
|
+
kind,
|
|
299
|
+
consumerId
|
|
300
|
+
});
|
|
301
|
+
}
|
|
302
|
+
};
|
|
303
|
+
}
|
|
304
|
+
function boundedInteger(value, fallback, minimum, maximum) {
|
|
305
|
+
const candidate = value ?? fallback;
|
|
306
|
+
if (!Number.isInteger(candidate) || candidate < minimum || candidate > maximum) {
|
|
307
|
+
throw new Error(`Expected an integer between ${minimum} and ${maximum}`);
|
|
308
|
+
}
|
|
309
|
+
return candidate;
|
|
310
|
+
}
|
|
311
|
+
function validateInstanceId(value) {
|
|
312
|
+
if (value.length < 1 || value.length > 128) {
|
|
313
|
+
throw new Error("Host export instanceId must contain 1 to 128 characters");
|
|
314
|
+
}
|
|
315
|
+
return value;
|
|
316
|
+
}
|
|
317
|
+
function safeErrorMessage(error) {
|
|
318
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
319
|
+
return message.trim().slice(0, 500) || "host sink failed";
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
export {
|
|
323
|
+
createHostExportPump
|
|
324
|
+
};
|
|
325
|
+
//# sourceMappingURL=chunk-BJUW5LCM.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/host-export-pump.ts"],"sourcesContent":["import type { HostEventSink, HostUsageSink } from \"@opengeni/contracts\";\nimport {\n acknowledgeHostExportBatch,\n claimHostExportBatch,\n disableHostExportConsumer,\n failHostExportBatch,\n getHostExportConsumerStatus,\n pruneHostExportOutbox,\n registerHostExportConsumer,\n retireHostExportConsumer,\n type Database,\n type HostExportConsumerStatus,\n type HostExportKind,\n} from \"@opengeni/db\";\nimport type { Observability } from \"@opengeni/observability\";\n\nexport type HostExportDrainResult =\n | { kind: HostExportKind; outcome: \"not_configured\" | \"idle\" }\n | { kind: HostExportKind; outcome: \"blocked\"; failures: number }\n | {\n kind: HostExportKind;\n outcome: \"delivered\";\n count: number;\n throughCursor: string;\n }\n | {\n kind: HostExportKind;\n outcome: \"failed\";\n failures: number;\n blocked: boolean;\n };\n\nexport type HostExportPumpOptions = {\n /** A handle authenticated as the separately provisioned host-export role. */\n db: Database;\n eventSink?: HostEventSink;\n usageSink?: HostUsageSink;\n observability?: Observability;\n instanceId?: string;\n pollIntervalMs?: number;\n leaseSeconds?: number;\n batchLimit?: number;\n batchMaxBytes?: number;\n maxFailures?: number;\n /** Maximum catch-up batches per poll tick; bounds one iteration's work. */\n maxBatchesPerPoll?: number;\n /** Acked rows younger than this remain available for replay. */\n pruneGraceSeconds?: number;\n /** Maximum rows pruned after one successful acknowledgement. */\n pruneLimit?: number;\n};\n\nexport type HostExportPump = {\n /** Register configured consumers and start the non-overlapping poll loop. */\n start: () => Promise<void>;\n /** Stop polling and wait for the currently executing sink call, if any. */\n stop: () => Promise<void>;\n /** Execute at most one bounded batch for each configured sink. */\n drainOnce: () => Promise<HostExportDrainResult[]>;\n /**\n * Disable this named consumer while retaining its replay checkpoint. If it is\n * the last enabled consumer of the kind, capture stops and that interval is\n * intentionally unrecoverable. Normal process shutdown must not call this.\n */\n disable: (kind: HostExportKind) => Promise<void>;\n /** Permanently remove a stopped consumer and release its retention floor. */\n retire: (kind: HostExportKind) => Promise<void>;\n status: (kind: HostExportKind) => Promise<HostExportConsumerStatus | null>;\n};\n\n/**\n * Build the durable delivery loop used by embedded hosts.\n *\n * The source transaction writes the outbox; this pump only leases, delivers,\n * and checkpoints it. Delivery is intentionally at least once: a process death\n * after sink success but before acknowledgement repeats the same idempotency\n * keys. Sink implementations must therefore deduplicate those keys.\n */\nexport function createHostExportPump(options: HostExportPumpOptions): HostExportPump {\n if (!options.eventSink && !options.usageSink) {\n throw new Error(\"createHostExportPump requires at least one host sink\");\n }\n const instanceId = validateInstanceId(\n options.instanceId ?? `worker-${process.pid}-${crypto.randomUUID()}`,\n );\n const pollIntervalMs = boundedInteger(options.pollIntervalMs, 1_000, 25, 60_000);\n const leaseSeconds = boundedInteger(options.leaseSeconds, 60, 5, 300);\n const batchLimit = boundedInteger(options.batchLimit, 100, 1, 256);\n const batchMaxBytes = boundedInteger(options.batchMaxBytes, 1_048_576, 98_304, 4_194_304);\n const maxFailures = boundedInteger(options.maxFailures, 20, 1, 1_000);\n const maxBatchesPerPoll = boundedInteger(options.maxBatchesPerPoll, 32, 1, 1_000);\n const pruneGraceSeconds = boundedInteger(options.pruneGraceSeconds, 3_600, 0, 604_800);\n const pruneLimit = boundedInteger(options.pruneLimit, 1_000, 1, 10_000);\n let running = false;\n let loopPromise: Promise<void> | null = null;\n let waitTimer: ReturnType<typeof setTimeout> | null = null;\n let wakeWait: (() => void) | null = null;\n let lifecycleTail: Promise<void> = Promise.resolve();\n let drainTail: Promise<void> = Promise.resolve();\n const retiringKinds = new Set<HostExportKind>();\n\n const configuredConsumer = (kind: HostExportKind): string | null =>\n kind === \"session_event\"\n ? (options.eventSink?.consumerId ?? null)\n : (options.usageSink?.consumerId ?? null);\n\n const recordResult = (result: HostExportDrainResult, durationSeconds: number) => {\n const labels = { kind: result.kind, outcome: result.outcome };\n options.observability?.incrementCounter({\n name: \"opengeni_host_export_batches_total\",\n help: \"Durable host export batch attempts by outcome.\",\n labels,\n });\n options.observability?.observeHistogram({\n name: \"opengeni_host_export_batch_duration_seconds\",\n help: \"Duration of durable host export batch attempts.\",\n labels: { kind: result.kind },\n value: durationSeconds,\n });\n if (result.outcome === \"delivered\") {\n options.observability?.incrementCounter({\n name: \"opengeni_host_export_records_total\",\n help: \"Durable host export records acknowledged by a host sink.\",\n labels: { kind: result.kind },\n amount: result.count,\n });\n options.observability?.setGauge({\n name: \"opengeni_host_export_consumer_blocked\",\n help: \"Whether the durable host export consumer is blocked.\",\n labels: { kind: result.kind },\n value: 0,\n });\n } else if (result.outcome === \"failed\") {\n options.observability?.setGauge({\n name: \"opengeni_host_export_consumer_blocked\",\n help: \"Whether the durable host export consumer is blocked.\",\n labels: { kind: result.kind },\n value: result.blocked ? 1 : 0,\n });\n } else if (result.outcome === \"blocked\") {\n options.observability?.setGauge({\n name: \"opengeni_host_export_consumer_blocked\",\n help: \"Whether the durable host export consumer is blocked.\",\n labels: { kind: result.kind },\n value: 1,\n });\n }\n };\n\n const drainKind = async (kind: HostExportKind): Promise<HostExportDrainResult> => {\n if (retiringKinds.has(kind)) return { kind, outcome: \"not_configured\" };\n const startedAt = performance.now();\n const consumerId = configuredConsumer(kind);\n if (!consumerId) {\n const result: HostExportDrainResult = { kind, outcome: \"not_configured\" };\n recordResult(result, (performance.now() - startedAt) / 1_000);\n return result;\n }\n const leaseToken = crypto.randomUUID();\n const common = {\n consumerId,\n leaseToken,\n leaseHolderId: instanceId,\n leaseSeconds,\n limit: batchLimit,\n maxBytes: batchMaxBytes,\n };\n const settleFailure = async (error: unknown): Promise<HostExportDrainResult> => {\n const message = safeErrorMessage(error);\n let failures = 0;\n try {\n failures = await failHostExportBatch(options.db, {\n kind,\n consumerId,\n leaseToken,\n error: message,\n maxFailures,\n });\n } catch (settlementError) {\n options.observability?.warn(\"host export failure settlement was stale\", {\n kind,\n consumerId,\n error: safeErrorMessage(settlementError),\n });\n }\n const blocked = failures >= maxFailures;\n options.observability?.warn(\"host export batch failed\", {\n kind,\n consumerId,\n failures,\n blocked,\n error: message,\n });\n const result: HostExportDrainResult = {\n kind,\n outcome: \"failed\",\n failures,\n blocked,\n };\n recordResult(result, (performance.now() - startedAt) / 1_000);\n return result;\n };\n const deliverClaim = async (\n count: number,\n deliver: () => Promise<void>,\n ): Promise<HostExportDrainResult> => {\n await deliver();\n const throughCursor = await acknowledgeHostExportBatch(options.db, {\n kind,\n consumerId,\n leaseToken,\n });\n try {\n await pruneHostExportOutbox(options.db, {\n kind,\n graceSeconds: pruneGraceSeconds,\n limit: pruneLimit,\n });\n } catch (error) {\n // The sink and checkpoint are already committed. A retention failure\n // must be visible, but must never turn into a false delivery failure or\n // a duplicate settlement attempt for the now-stale lease.\n options.observability?.error(\"host export retention prune failed\", {\n kind,\n consumerId,\n error: safeErrorMessage(error),\n });\n }\n const result: HostExportDrainResult = {\n kind,\n outcome: \"delivered\",\n count,\n throughCursor,\n };\n recordResult(result, (performance.now() - startedAt) / 1_000);\n return result;\n };\n\n try {\n if (kind === \"session_event\") {\n const sink = options.eventSink;\n if (!sink) return { kind, outcome: \"not_configured\" };\n const batch = await claimHostExportBatch(options.db, {\n kind,\n ...common,\n });\n if (batch) return await deliverClaim(batch.events.length, () => sink.deliverEvents(batch));\n } else {\n const sink = options.usageSink;\n if (!sink) return { kind, outcome: \"not_configured\" };\n const batch = await claimHostExportBatch(options.db, {\n kind,\n ...common,\n });\n if (batch) return await deliverClaim(batch.events.length, () => sink.deliverUsage(batch));\n }\n } catch (error) {\n return await settleFailure(error);\n }\n const status = await getHostExportConsumerStatus(options.db, {\n kind,\n consumerId,\n });\n if (status?.blockedAt) {\n const result: HostExportDrainResult = {\n kind,\n outcome: \"blocked\",\n failures: status.consecutiveFailures,\n };\n recordResult(result, (performance.now() - startedAt) / 1_000);\n return result;\n }\n const result: HostExportDrainResult = { kind, outcome: \"idle\" };\n recordResult(result, (performance.now() - startedAt) / 1_000);\n return result;\n };\n\n const drainConfiguredOnce = async (): Promise<HostExportDrainResult[]> => {\n const results: HostExportDrainResult[] = [];\n if (options.eventSink) results.push(await drainKind(\"session_event\"));\n if (options.usageSink) results.push(await drainKind(\"usage_event\"));\n return results;\n };\n\n const drainOnce = (): Promise<HostExportDrainResult[]> => {\n const next = drainTail.then(drainConfiguredOnce);\n drainTail = next.then(\n () => undefined,\n () => undefined,\n );\n return next;\n };\n\n const serializeLifecycle = <T>(operation: () => Promise<T>): Promise<T> => {\n const next = lifecycleTail.then(operation);\n lifecycleTail = next.then(\n () => undefined,\n () => undefined,\n );\n return next;\n };\n\n const waitForPoll = async (): Promise<void> => {\n await new Promise<void>((resolve) => {\n wakeWait = resolve;\n waitTimer = setTimeout(resolve, pollIntervalMs);\n waitTimer.unref?.();\n });\n wakeWait = null;\n if (waitTimer) clearTimeout(waitTimer);\n waitTimer = null;\n };\n\n return {\n start: () =>\n serializeLifecycle(async () => {\n if (running) return;\n await options.db.transaction(async (tx) => {\n const scoped = tx as unknown as Database;\n if (options.eventSink) {\n await registerHostExportConsumer(scoped, {\n kind: \"session_event\",\n consumerId: options.eventSink.consumerId,\n });\n }\n if (options.usageSink) {\n await registerHostExportConsumer(scoped, {\n kind: \"usage_event\",\n consumerId: options.usageSink.consumerId,\n });\n }\n });\n if (options.eventSink) retiringKinds.delete(\"session_event\");\n if (options.usageSink) retiringKinds.delete(\"usage_event\");\n running = true;\n loopPromise = (async () => {\n while (true) {\n if (!running) break;\n try {\n for (let batch = 0; batch < maxBatchesPerPoll; batch += 1) {\n if (!running) break;\n const results = await drainOnce();\n if (!results.some((result) => result.outcome === \"delivered\")) break;\n }\n } catch (error) {\n options.observability?.error(\"host export pump iteration failed\", {\n error: safeErrorMessage(error),\n });\n }\n if (running) await waitForPoll();\n }\n })();\n }),\n stop: () =>\n serializeLifecycle(async () => {\n running = false;\n if (waitTimer) clearTimeout(waitTimer);\n wakeWait?.();\n await loopPromise;\n loopPromise = null;\n }),\n drainOnce,\n disable: async (kind) => {\n const consumerId = configuredConsumer(kind);\n if (!consumerId) throw new Error(`No ${kind} sink is configured`);\n await disableHostExportConsumer(options.db, { kind, consumerId });\n },\n retire: (kind) =>\n serializeLifecycle(async () => {\n if (running) throw new Error(\"Stop the host export pump before retiring a consumer\");\n const consumerId = configuredConsumer(kind);\n if (!consumerId) throw new Error(`No ${kind} sink is configured`);\n retiringKinds.add(kind);\n try {\n await drainTail;\n await retireHostExportConsumer(options.db, { kind, consumerId });\n } catch (error) {\n retiringKinds.delete(kind);\n throw error;\n }\n }),\n status: async (kind) => {\n const consumerId = configuredConsumer(kind);\n if (!consumerId) return null;\n return await getHostExportConsumerStatus(options.db, {\n kind,\n consumerId,\n });\n },\n };\n}\n\nfunction boundedInteger(\n value: number | undefined,\n fallback: number,\n minimum: number,\n maximum: number,\n): number {\n const candidate = value ?? fallback;\n if (!Number.isInteger(candidate) || candidate < minimum || candidate > maximum) {\n throw new Error(`Expected an integer between ${minimum} and ${maximum}`);\n }\n return candidate;\n}\n\nfunction validateInstanceId(value: string): string {\n if (value.length < 1 || value.length > 128) {\n throw new Error(\"Host export instanceId must contain 1 to 128 characters\");\n }\n return value;\n}\n\nfunction safeErrorMessage(error: unknown): string {\n const message = error instanceof Error ? error.message : String(error);\n return message.trim().slice(0, 500) || \"host sink failed\";\n}\n"],"mappings":";AACA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OAIK;AAiEA,SAAS,qBAAqB,SAAgD;AACnF,MAAI,CAAC,QAAQ,aAAa,CAAC,QAAQ,WAAW;AAC5C,UAAM,IAAI,MAAM,sDAAsD;AAAA,EACxE;AACA,QAAM,aAAa;AAAA,IACjB,QAAQ,cAAc,UAAU,QAAQ,GAAG,IAAI,OAAO,WAAW,CAAC;AAAA,EACpE;AACA,QAAM,iBAAiB,eAAe,QAAQ,gBAAgB,KAAO,IAAI,GAAM;AAC/E,QAAM,eAAe,eAAe,QAAQ,cAAc,IAAI,GAAG,GAAG;AACpE,QAAM,aAAa,eAAe,QAAQ,YAAY,KAAK,GAAG,GAAG;AACjE,QAAM,gBAAgB,eAAe,QAAQ,eAAe,SAAW,OAAQ,OAAS;AACxF,QAAM,cAAc,eAAe,QAAQ,aAAa,IAAI,GAAG,GAAK;AACpE,QAAM,oBAAoB,eAAe,QAAQ,mBAAmB,IAAI,GAAG,GAAK;AAChF,QAAM,oBAAoB,eAAe,QAAQ,mBAAmB,MAAO,GAAG,MAAO;AACrF,QAAM,aAAa,eAAe,QAAQ,YAAY,KAAO,GAAG,GAAM;AACtE,MAAI,UAAU;AACd,MAAI,cAAoC;AACxC,MAAI,YAAkD;AACtD,MAAI,WAAgC;AACpC,MAAI,gBAA+B,QAAQ,QAAQ;AACnD,MAAI,YAA2B,QAAQ,QAAQ;AAC/C,QAAM,gBAAgB,oBAAI,IAAoB;AAE9C,QAAM,qBAAqB,CAAC,SAC1B,SAAS,kBACJ,QAAQ,WAAW,cAAc,OACjC,QAAQ,WAAW,cAAc;AAExC,QAAM,eAAe,CAAC,QAA+B,oBAA4B;AAC/E,UAAM,SAAS,EAAE,MAAM,OAAO,MAAM,SAAS,OAAO,QAAQ;AAC5D,YAAQ,eAAe,iBAAiB;AAAA,MACtC,MAAM;AAAA,MACN,MAAM;AAAA,MACN;AAAA,IACF,CAAC;AACD,YAAQ,eAAe,iBAAiB;AAAA,MACtC,MAAM;AAAA,MACN,MAAM;AAAA,MACN,QAAQ,EAAE,MAAM,OAAO,KAAK;AAAA,MAC5B,OAAO;AAAA,IACT,CAAC;AACD,QAAI,OAAO,YAAY,aAAa;AAClC,cAAQ,eAAe,iBAAiB;AAAA,QACtC,MAAM;AAAA,QACN,MAAM;AAAA,QACN,QAAQ,EAAE,MAAM,OAAO,KAAK;AAAA,QAC5B,QAAQ,OAAO;AAAA,MACjB,CAAC;AACD,cAAQ,eAAe,SAAS;AAAA,QAC9B,MAAM;AAAA,QACN,MAAM;AAAA,QACN,QAAQ,EAAE,MAAM,OAAO,KAAK;AAAA,QAC5B,OAAO;AAAA,MACT,CAAC;AAAA,IACH,WAAW,OAAO,YAAY,UAAU;AACtC,cAAQ,eAAe,SAAS;AAAA,QAC9B,MAAM;AAAA,QACN,MAAM;AAAA,QACN,QAAQ,EAAE,MAAM,OAAO,KAAK;AAAA,QAC5B,OAAO,OAAO,UAAU,IAAI;AAAA,MAC9B,CAAC;AAAA,IACH,WAAW,OAAO,YAAY,WAAW;AACvC,cAAQ,eAAe,SAAS;AAAA,QAC9B,MAAM;AAAA,QACN,MAAM;AAAA,QACN,QAAQ,EAAE,MAAM,OAAO,KAAK;AAAA,QAC5B,OAAO;AAAA,MACT,CAAC;AAAA,IACH;AAAA,EACF;AAEA,QAAM,YAAY,OAAO,SAAyD;AAChF,QAAI,cAAc,IAAI,IAAI,EAAG,QAAO,EAAE,MAAM,SAAS,iBAAiB;AACtE,UAAM,YAAY,YAAY,IAAI;AAClC,UAAM,aAAa,mBAAmB,IAAI;AAC1C,QAAI,CAAC,YAAY;AACf,YAAMA,UAAgC,EAAE,MAAM,SAAS,iBAAiB;AACxE,mBAAaA,UAAS,YAAY,IAAI,IAAI,aAAa,GAAK;AAC5D,aAAOA;AAAA,IACT;AACA,UAAM,aAAa,OAAO,WAAW;AACrC,UAAM,SAAS;AAAA,MACb;AAAA,MACA;AAAA,MACA,eAAe;AAAA,MACf;AAAA,MACA,OAAO;AAAA,MACP,UAAU;AAAA,IACZ;AACA,UAAM,gBAAgB,OAAO,UAAmD;AAC9E,YAAM,UAAU,iBAAiB,KAAK;AACtC,UAAI,WAAW;AACf,UAAI;AACF,mBAAW,MAAM,oBAAoB,QAAQ,IAAI;AAAA,UAC/C;AAAA,UACA;AAAA,UACA;AAAA,UACA,OAAO;AAAA,UACP;AAAA,QACF,CAAC;AAAA,MACH,SAAS,iBAAiB;AACxB,gBAAQ,eAAe,KAAK,4CAA4C;AAAA,UACtE;AAAA,UACA;AAAA,UACA,OAAO,iBAAiB,eAAe;AAAA,QACzC,CAAC;AAAA,MACH;AACA,YAAM,UAAU,YAAY;AAC5B,cAAQ,eAAe,KAAK,4BAA4B;AAAA,QACtD;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA,OAAO;AAAA,MACT,CAAC;AACD,YAAMA,UAAgC;AAAA,QACpC;AAAA,QACA,SAAS;AAAA,QACT;AAAA,QACA;AAAA,MACF;AACA,mBAAaA,UAAS,YAAY,IAAI,IAAI,aAAa,GAAK;AAC5D,aAAOA;AAAA,IACT;AACA,UAAM,eAAe,OACnB,OACA,YACmC;AACnC,YAAM,QAAQ;AACd,YAAM,gBAAgB,MAAM,2BAA2B,QAAQ,IAAI;AAAA,QACjE;AAAA,QACA;AAAA,QACA;AAAA,MACF,CAAC;AACD,UAAI;AACF,cAAM,sBAAsB,QAAQ,IAAI;AAAA,UACtC;AAAA,UACA,cAAc;AAAA,UACd,OAAO;AAAA,QACT,CAAC;AAAA,MACH,SAAS,OAAO;AAId,gBAAQ,eAAe,MAAM,sCAAsC;AAAA,UACjE;AAAA,UACA;AAAA,UACA,OAAO,iBAAiB,KAAK;AAAA,QAC/B,CAAC;AAAA,MACH;AACA,YAAMA,UAAgC;AAAA,QACpC;AAAA,QACA,SAAS;AAAA,QACT;AAAA,QACA;AAAA,MACF;AACA,mBAAaA,UAAS,YAAY,IAAI,IAAI,aAAa,GAAK;AAC5D,aAAOA;AAAA,IACT;AAEA,QAAI;AACF,UAAI,SAAS,iBAAiB;AAC5B,cAAM,OAAO,QAAQ;AACrB,YAAI,CAAC,KAAM,QAAO,EAAE,MAAM,SAAS,iBAAiB;AACpD,cAAM,QAAQ,MAAM,qBAAqB,QAAQ,IAAI;AAAA,UACnD;AAAA,UACA,GAAG;AAAA,QACL,CAAC;AACD,YAAI,MAAO,QAAO,MAAM,aAAa,MAAM,OAAO,QAAQ,MAAM,KAAK,cAAc,KAAK,CAAC;AAAA,MAC3F,OAAO;AACL,cAAM,OAAO,QAAQ;AACrB,YAAI,CAAC,KAAM,QAAO,EAAE,MAAM,SAAS,iBAAiB;AACpD,cAAM,QAAQ,MAAM,qBAAqB,QAAQ,IAAI;AAAA,UACnD;AAAA,UACA,GAAG;AAAA,QACL,CAAC;AACD,YAAI,MAAO,QAAO,MAAM,aAAa,MAAM,OAAO,QAAQ,MAAM,KAAK,aAAa,KAAK,CAAC;AAAA,MAC1F;AAAA,IACF,SAAS,OAAO;AACd,aAAO,MAAM,cAAc,KAAK;AAAA,IAClC;AACA,UAAM,SAAS,MAAM,4BAA4B,QAAQ,IAAI;AAAA,MAC3D;AAAA,MACA;AAAA,IACF,CAAC;AACD,QAAI,QAAQ,WAAW;AACrB,YAAMA,UAAgC;AAAA,QACpC;AAAA,QACA,SAAS;AAAA,QACT,UAAU,OAAO;AAAA,MACnB;AACA,mBAAaA,UAAS,YAAY,IAAI,IAAI,aAAa,GAAK;AAC5D,aAAOA;AAAA,IACT;AACA,UAAM,SAAgC,EAAE,MAAM,SAAS,OAAO;AAC9D,iBAAa,SAAS,YAAY,IAAI,IAAI,aAAa,GAAK;AAC5D,WAAO;AAAA,EACT;AAEA,QAAM,sBAAsB,YAA8C;AACxE,UAAM,UAAmC,CAAC;AAC1C,QAAI,QAAQ,UAAW,SAAQ,KAAK,MAAM,UAAU,eAAe,CAAC;AACpE,QAAI,QAAQ,UAAW,SAAQ,KAAK,MAAM,UAAU,aAAa,CAAC;AAClE,WAAO;AAAA,EACT;AAEA,QAAM,YAAY,MAAwC;AACxD,UAAM,OAAO,UAAU,KAAK,mBAAmB;AAC/C,gBAAY,KAAK;AAAA,MACf,MAAM;AAAA,MACN,MAAM;AAAA,IACR;AACA,WAAO;AAAA,EACT;AAEA,QAAM,qBAAqB,CAAI,cAA4C;AACzE,UAAM,OAAO,cAAc,KAAK,SAAS;AACzC,oBAAgB,KAAK;AAAA,MACnB,MAAM;AAAA,MACN,MAAM;AAAA,IACR;AACA,WAAO;AAAA,EACT;AAEA,QAAM,cAAc,YAA2B;AAC7C,UAAM,IAAI,QAAc,CAAC,YAAY;AACnC,iBAAW;AACX,kBAAY,WAAW,SAAS,cAAc;AAC9C,gBAAU,QAAQ;AAAA,IACpB,CAAC;AACD,eAAW;AACX,QAAI,UAAW,cAAa,SAAS;AACrC,gBAAY;AAAA,EACd;AAEA,SAAO;AAAA,IACL,OAAO,MACL,mBAAmB,YAAY;AAC7B,UAAI,QAAS;AACb,YAAM,QAAQ,GAAG,YAAY,OAAO,OAAO;AACzC,cAAM,SAAS;AACf,YAAI,QAAQ,WAAW;AACrB,gBAAM,2BAA2B,QAAQ;AAAA,YACvC,MAAM;AAAA,YACN,YAAY,QAAQ,UAAU;AAAA,UAChC,CAAC;AAAA,QACH;AACA,YAAI,QAAQ,WAAW;AACrB,gBAAM,2BAA2B,QAAQ;AAAA,YACvC,MAAM;AAAA,YACN,YAAY,QAAQ,UAAU;AAAA,UAChC,CAAC;AAAA,QACH;AAAA,MACF,CAAC;AACD,UAAI,QAAQ,UAAW,eAAc,OAAO,eAAe;AAC3D,UAAI,QAAQ,UAAW,eAAc,OAAO,aAAa;AACzD,gBAAU;AACV,qBAAe,YAAY;AACzB,eAAO,MAAM;AACX,cAAI,CAAC,QAAS;AACd,cAAI;AACF,qBAAS,QAAQ,GAAG,QAAQ,mBAAmB,SAAS,GAAG;AACzD,kBAAI,CAAC,QAAS;AACd,oBAAM,UAAU,MAAM,UAAU;AAChC,kBAAI,CAAC,QAAQ,KAAK,CAAC,WAAW,OAAO,YAAY,WAAW,EAAG;AAAA,YACjE;AAAA,UACF,SAAS,OAAO;AACd,oBAAQ,eAAe,MAAM,qCAAqC;AAAA,cAChE,OAAO,iBAAiB,KAAK;AAAA,YAC/B,CAAC;AAAA,UACH;AACA,cAAI,QAAS,OAAM,YAAY;AAAA,QACjC;AAAA,MACF,GAAG;AAAA,IACL,CAAC;AAAA,IACH,MAAM,MACJ,mBAAmB,YAAY;AAC7B,gBAAU;AACV,UAAI,UAAW,cAAa,SAAS;AACrC,iBAAW;AACX,YAAM;AACN,oBAAc;AAAA,IAChB,CAAC;AAAA,IACH;AAAA,IACA,SAAS,OAAO,SAAS;AACvB,YAAM,aAAa,mBAAmB,IAAI;AAC1C,UAAI,CAAC,WAAY,OAAM,IAAI,MAAM,MAAM,IAAI,qBAAqB;AAChE,YAAM,0BAA0B,QAAQ,IAAI,EAAE,MAAM,WAAW,CAAC;AAAA,IAClE;AAAA,IACA,QAAQ,CAAC,SACP,mBAAmB,YAAY;AAC7B,UAAI,QAAS,OAAM,IAAI,MAAM,sDAAsD;AACnF,YAAM,aAAa,mBAAmB,IAAI;AAC1C,UAAI,CAAC,WAAY,OAAM,IAAI,MAAM,MAAM,IAAI,qBAAqB;AAChE,oBAAc,IAAI,IAAI;AACtB,UAAI;AACF,cAAM;AACN,cAAM,yBAAyB,QAAQ,IAAI,EAAE,MAAM,WAAW,CAAC;AAAA,MACjE,SAAS,OAAO;AACd,sBAAc,OAAO,IAAI;AACzB,cAAM;AAAA,MACR;AAAA,IACF,CAAC;AAAA,IACH,QAAQ,OAAO,SAAS;AACtB,YAAM,aAAa,mBAAmB,IAAI;AAC1C,UAAI,CAAC,WAAY,QAAO;AACxB,aAAO,MAAM,4BAA4B,QAAQ,IAAI;AAAA,QACnD;AAAA,QACA;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AACF;AAEA,SAAS,eACP,OACA,UACA,SACA,SACQ;AACR,QAAM,YAAY,SAAS;AAC3B,MAAI,CAAC,OAAO,UAAU,SAAS,KAAK,YAAY,WAAW,YAAY,SAAS;AAC9E,UAAM,IAAI,MAAM,+BAA+B,OAAO,QAAQ,OAAO,EAAE;AAAA,EACzE;AACA,SAAO;AACT;AAEA,SAAS,mBAAmB,OAAuB;AACjD,MAAI,MAAM,SAAS,KAAK,MAAM,SAAS,KAAK;AAC1C,UAAM,IAAI,MAAM,yDAAyD;AAAA,EAC3E;AACA,SAAO;AACT;AAEA,SAAS,iBAAiB,OAAwB;AAChD,QAAM,UAAU,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AACrE,SAAO,QAAQ,KAAK,EAAE,MAAM,GAAG,GAAG,KAAK;AACzC;","names":["result"]}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { HostEventSink, HostUsageSink } from '@opengeni/contracts';
|
|
2
|
+
export { HostEventExport, HostEventExportBatch, HostEventSink, HostUsageExport, HostUsageExportBatch, HostUsageSink } from '@opengeni/contracts';
|
|
3
|
+
import { HostExportKind, HostExportConsumerStatus, Database } from '@opengeni/db';
|
|
4
|
+
import { Observability } from '@opengeni/observability';
|
|
5
|
+
|
|
6
|
+
type HostExportDrainResult = {
|
|
7
|
+
kind: HostExportKind;
|
|
8
|
+
outcome: "not_configured" | "idle";
|
|
9
|
+
} | {
|
|
10
|
+
kind: HostExportKind;
|
|
11
|
+
outcome: "blocked";
|
|
12
|
+
failures: number;
|
|
13
|
+
} | {
|
|
14
|
+
kind: HostExportKind;
|
|
15
|
+
outcome: "delivered";
|
|
16
|
+
count: number;
|
|
17
|
+
throughCursor: string;
|
|
18
|
+
} | {
|
|
19
|
+
kind: HostExportKind;
|
|
20
|
+
outcome: "failed";
|
|
21
|
+
failures: number;
|
|
22
|
+
blocked: boolean;
|
|
23
|
+
};
|
|
24
|
+
type HostExportPumpOptions = {
|
|
25
|
+
/** A handle authenticated as the separately provisioned host-export role. */
|
|
26
|
+
db: Database;
|
|
27
|
+
eventSink?: HostEventSink;
|
|
28
|
+
usageSink?: HostUsageSink;
|
|
29
|
+
observability?: Observability;
|
|
30
|
+
instanceId?: string;
|
|
31
|
+
pollIntervalMs?: number;
|
|
32
|
+
leaseSeconds?: number;
|
|
33
|
+
batchLimit?: number;
|
|
34
|
+
batchMaxBytes?: number;
|
|
35
|
+
maxFailures?: number;
|
|
36
|
+
/** Maximum catch-up batches per poll tick; bounds one iteration's work. */
|
|
37
|
+
maxBatchesPerPoll?: number;
|
|
38
|
+
/** Acked rows younger than this remain available for replay. */
|
|
39
|
+
pruneGraceSeconds?: number;
|
|
40
|
+
/** Maximum rows pruned after one successful acknowledgement. */
|
|
41
|
+
pruneLimit?: number;
|
|
42
|
+
};
|
|
43
|
+
type HostExportPump = {
|
|
44
|
+
/** Register configured consumers and start the non-overlapping poll loop. */
|
|
45
|
+
start: () => Promise<void>;
|
|
46
|
+
/** Stop polling and wait for the currently executing sink call, if any. */
|
|
47
|
+
stop: () => Promise<void>;
|
|
48
|
+
/** Execute at most one bounded batch for each configured sink. */
|
|
49
|
+
drainOnce: () => Promise<HostExportDrainResult[]>;
|
|
50
|
+
/**
|
|
51
|
+
* Disable this named consumer while retaining its replay checkpoint. If it is
|
|
52
|
+
* the last enabled consumer of the kind, capture stops and that interval is
|
|
53
|
+
* intentionally unrecoverable. Normal process shutdown must not call this.
|
|
54
|
+
*/
|
|
55
|
+
disable: (kind: HostExportKind) => Promise<void>;
|
|
56
|
+
/** Permanently remove a stopped consumer and release its retention floor. */
|
|
57
|
+
retire: (kind: HostExportKind) => Promise<void>;
|
|
58
|
+
status: (kind: HostExportKind) => Promise<HostExportConsumerStatus | null>;
|
|
59
|
+
};
|
|
60
|
+
/**
|
|
61
|
+
* Build the durable delivery loop used by embedded hosts.
|
|
62
|
+
*
|
|
63
|
+
* The source transaction writes the outbox; this pump only leases, delivers,
|
|
64
|
+
* and checkpoints it. Delivery is intentionally at least once: a process death
|
|
65
|
+
* after sink success but before acknowledgement repeats the same idempotency
|
|
66
|
+
* keys. Sink implementations must therefore deduplicate those keys.
|
|
67
|
+
*/
|
|
68
|
+
declare function createHostExportPump(options: HostExportPumpOptions): HostExportPump;
|
|
69
|
+
|
|
70
|
+
export { type HostExportDrainResult, type HostExportPump, type HostExportPumpOptions, createHostExportPump };
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,13 +1,15 @@
|
|
|
1
1
|
import { Settings } from '@opengeni/config';
|
|
2
2
|
import * as _opengeni_db from '@opengeni/db';
|
|
3
3
|
import { Database } from '@opengeni/db';
|
|
4
|
+
import { EventBus } from '@opengeni/events';
|
|
4
5
|
import { Observability } from '@opengeni/observability';
|
|
5
|
-
import { Worker, NativeConnection } from '@temporalio/worker';
|
|
6
|
+
import { Worker, NativeConnection, WorkflowBundleOption } from '@temporalio/worker';
|
|
6
7
|
import { EntitlementsPort, ConnectionCredentialsPort, ScheduledTaskTriggerType } from '@opengeni/contracts';
|
|
8
|
+
export { HostEventExport, HostEventExportBatch, HostEventSink, HostUsageExport, HostUsageExportBatch, HostUsageSink } from '@opengeni/contracts';
|
|
7
9
|
import { DocumentServices } from '@opengeni/documents';
|
|
8
|
-
import { EventBus } from '@opengeni/events';
|
|
9
10
|
import { OpenGeniRuntime } from '@opengeni/runtime';
|
|
10
11
|
import { ObjectStorage } from '@opengeni/storage';
|
|
12
|
+
export { HostExportDrainResult, HostExportPump, HostExportPumpOptions, createHostExportPump } from './host-export.js';
|
|
11
13
|
|
|
12
14
|
type WakeSessionWorkflowSignal = (input: {
|
|
13
15
|
accountId: string;
|
|
@@ -24,6 +26,19 @@ type SignalCodexCapacityWorkflow = (input: {
|
|
|
24
26
|
workflowId: string;
|
|
25
27
|
wakeRevision: number;
|
|
26
28
|
}) => Promise<void>;
|
|
29
|
+
/** Exact activity-owned proof that the hard sandbox/tool fence physically
|
|
30
|
+
* drained. This is delivery evidence only: the workflow still validates the
|
|
31
|
+
* persisted attempt dispatch and commits the authoritative Postgres receipt. */
|
|
32
|
+
type SessionAttemptQuiescenceProof = {
|
|
33
|
+
accountId: string;
|
|
34
|
+
workspaceId: string;
|
|
35
|
+
sessionId: string;
|
|
36
|
+
attemptId: string;
|
|
37
|
+
workflowId: string;
|
|
38
|
+
workflowRunId: string;
|
|
39
|
+
activityId: string;
|
|
40
|
+
};
|
|
41
|
+
type SignalSessionAttemptQuiesced = (input: SessionAttemptQuiescenceProof) => Promise<void>;
|
|
27
42
|
type ActivityServices = {
|
|
28
43
|
settings: Settings;
|
|
29
44
|
db: Database;
|
|
@@ -33,6 +48,9 @@ type ActivityServices = {
|
|
|
33
48
|
documentServices: DocumentServices;
|
|
34
49
|
observability: Observability;
|
|
35
50
|
wakeSessionWorkflow: WakeSessionWorkflowSignal | null;
|
|
51
|
+
/** Durable signalWithStart fallback used only after the activity's direct
|
|
52
|
+
* physical-quiescence receipt write exhausts its bounded DB retries. */
|
|
53
|
+
signalSessionAttemptQuiesced: SignalSessionAttemptQuiesced | null;
|
|
36
54
|
/** Revision-carrying capacity nudge; generic outbox repair is also sufficient. */
|
|
37
55
|
signalCodexCapacityWorkflow?: SignalCodexCapacityWorkflow | null;
|
|
38
56
|
entitlements?: EntitlementsPort | null;
|
|
@@ -85,7 +103,14 @@ type SettleSessionInterruptionsInput = {
|
|
|
85
103
|
sessionId: string;
|
|
86
104
|
attemptId: string;
|
|
87
105
|
workflowId: string;
|
|
106
|
+
/**
|
|
107
|
+
* Replay-only compatibility for session workflow histories created before
|
|
108
|
+
* the receipt-gated cancellation v2 patch. New histories never send this
|
|
109
|
+
* phase; the exact activity writes the authoritative receipt itself.
|
|
110
|
+
*/
|
|
111
|
+
phase?: "logical" | "attempt_quiesced";
|
|
88
112
|
};
|
|
113
|
+
type PersistSessionAttemptQuiescenceInput = SessionAttemptQuiescenceProof;
|
|
89
114
|
type FailSessionAttemptInput = {
|
|
90
115
|
accountId: string;
|
|
91
116
|
workspaceId: string;
|
|
@@ -117,6 +142,15 @@ type PeekSessionWorkInput = {
|
|
|
117
142
|
workspaceId: string;
|
|
118
143
|
sessionId: string;
|
|
119
144
|
};
|
|
145
|
+
type ExpireSessionHumanInputInput = {
|
|
146
|
+
accountId: string;
|
|
147
|
+
workspaceId: string;
|
|
148
|
+
sessionId: string;
|
|
149
|
+
requestId: string;
|
|
150
|
+
};
|
|
151
|
+
type ExpireSessionHumanInputResult = {
|
|
152
|
+
action: "expired" | "stale" | "not_found";
|
|
153
|
+
};
|
|
120
154
|
type MarkSessionIdleInput = {
|
|
121
155
|
workspaceId: string;
|
|
122
156
|
sessionId: string;
|
|
@@ -256,20 +290,22 @@ declare function createControlActivities(dependencies?: ActivityDependencies): {
|
|
|
256
290
|
settleSessionInterruptions: (input: SettleSessionInterruptionsInput) => Promise<{
|
|
257
291
|
action: "paused" | "continue" | "stale";
|
|
258
292
|
}>;
|
|
293
|
+
persistSessionAttemptQuiescence: (input: PersistSessionAttemptQuiescenceInput) => Promise<void>;
|
|
259
294
|
recoverDispatch: (input: RecoverDispatchInput) => Promise<RecoverDispatchResult>;
|
|
260
295
|
peekSessionWork: (input: PeekSessionWorkInput) => Promise<_opengeni_db.SessionWorkPeek>;
|
|
296
|
+
expireSessionHumanInput: (input: ExpireSessionHumanInputInput) => Promise<ExpireSessionHumanInputResult>;
|
|
261
297
|
markSessionIdle: (input: MarkSessionIdleInput) => Promise<void>;
|
|
262
298
|
indexDocument: (input: IndexDocumentInput) => Promise<{
|
|
263
299
|
id: string;
|
|
264
300
|
workspaceId: string;
|
|
265
301
|
baseId: string;
|
|
266
302
|
fileId: string;
|
|
267
|
-
status: "
|
|
303
|
+
status: "ready" | "failed" | "queued" | "indexing";
|
|
268
304
|
title: string;
|
|
269
305
|
parser: string;
|
|
270
306
|
chunkCount: number;
|
|
271
307
|
error: string | null;
|
|
272
|
-
sourceKind: "
|
|
308
|
+
sourceKind: "repository" | "email" | "manual_upload" | "meeting_transcript" | "chat" | "document" | "web" | "other";
|
|
273
309
|
sourceUri: string | null;
|
|
274
310
|
sourceExternalId: string | null;
|
|
275
311
|
sourceTitle: string | null;
|
|
@@ -286,6 +322,8 @@ declare function createTurnActivities(dependencies?: ActivityDependencies): {
|
|
|
286
322
|
runAgentTurn: (input: RunAgentTurnInput) => Promise<RunAgentTurnResult>;
|
|
287
323
|
};
|
|
288
324
|
|
|
325
|
+
type WorkerLifecycleState = "starting" | "ready" | "draining" | "stopped" | "failed";
|
|
326
|
+
|
|
289
327
|
declare const FILE_UPLOAD_REAPER_SCHEDULE_ID = "opengeni-file-upload-reaper";
|
|
290
328
|
declare const FILE_UPLOAD_REAPER_PERIOD_MS: number;
|
|
291
329
|
type OpenGeniWorkerRole = "control" | "turn";
|
|
@@ -294,14 +332,28 @@ type WorkerOptions = {
|
|
|
294
332
|
settings?: Settings;
|
|
295
333
|
activities?: ReturnType<typeof createControlActivities> | ReturnType<typeof createTurnActivities>;
|
|
296
334
|
activityDependencies?: ActivityDependencies;
|
|
297
|
-
|
|
335
|
+
/** Override the release-coherent workflow artifact. Most hosts should omit this. */
|
|
336
|
+
workflowBundle?: WorkflowBundleOption;
|
|
337
|
+
};
|
|
338
|
+
type WorkerWorkflowDefinition = {
|
|
339
|
+
workflowBundle: WorkflowBundleOption;
|
|
340
|
+
} | {
|
|
341
|
+
workflowsPath: string;
|
|
298
342
|
};
|
|
343
|
+
/**
|
|
344
|
+
* Resolve the deterministic workflow graph without asking installed hosts to
|
|
345
|
+
* transpile or relocate package TypeScript. Monorepo source execution retains
|
|
346
|
+
* the source path for the local development loop; published dist execution
|
|
347
|
+
* fails closed unless the build-generated sibling artifact is present.
|
|
348
|
+
*/
|
|
349
|
+
declare function resolveOpenGeniWorkflowDefinition(moduleUrl?: string): WorkerWorkflowDefinition;
|
|
299
350
|
declare function createOpenGeniWorker(options: WorkerOptions): Promise<{
|
|
300
351
|
worker: Worker;
|
|
301
352
|
connection: NativeConnection;
|
|
302
353
|
}>;
|
|
303
354
|
declare function createWorkerWorkflowSignaler(settings: Settings, db: Database): Promise<{
|
|
304
355
|
wakeSessionWorkflow: WakeSessionWorkflowSignal;
|
|
356
|
+
signalSessionAttemptQuiesced: SignalSessionAttemptQuiesced;
|
|
305
357
|
signalCodexCapacityWorkflow: SignalCodexCapacityWorkflow;
|
|
306
358
|
check: () => Promise<void>;
|
|
307
359
|
close: () => Promise<void>;
|
|
@@ -346,6 +398,46 @@ declare function registerSessionWorkflowWakeDispatcherSchedule(settings: Setting
|
|
|
346
398
|
registered: boolean;
|
|
347
399
|
close: () => Promise<void>;
|
|
348
400
|
}>;
|
|
401
|
+
type OpenGeniWorkerServiceOptions = Omit<WorkerOptions, "activityDependencies"> & {
|
|
402
|
+
activityDependencies: ActivityDependencies & {
|
|
403
|
+
db: Database;
|
|
404
|
+
bus: EventBus;
|
|
405
|
+
};
|
|
406
|
+
/**
|
|
407
|
+
* `role-default` registers OpenGeni's internal maintenance schedules on a
|
|
408
|
+
* control worker only. These are engine maintenance schedules, not a host's
|
|
409
|
+
* product-level scheduled-agent jobs. Use `none` when another control worker
|
|
410
|
+
* in the same deployment owns them.
|
|
411
|
+
*/
|
|
412
|
+
internalSchedules?: "role-default" | "none";
|
|
413
|
+
/** Set false only when the host exposes equivalent lifecycle endpoints itself. */
|
|
414
|
+
http?: false | {
|
|
415
|
+
readinessTimeoutMs?: number;
|
|
416
|
+
};
|
|
417
|
+
};
|
|
418
|
+
type OpenGeniWorkerService = {
|
|
419
|
+
readonly role: OpenGeniWorkerRole;
|
|
420
|
+
readonly worker: Worker;
|
|
421
|
+
readonly connection: NativeConnection;
|
|
422
|
+
state(): WorkerLifecycleState;
|
|
423
|
+
run(): Promise<void>;
|
|
424
|
+
drain(reason?: string): void;
|
|
425
|
+
close(): Promise<void>;
|
|
426
|
+
};
|
|
427
|
+
declare function workerOwnsInternalSchedules(role: OpenGeniWorkerRole, policy?: OpenGeniWorkerServiceOptions["internalSchedules"]): boolean;
|
|
428
|
+
/**
|
|
429
|
+
* Construct one role-specific worker process around the lower-level Temporal
|
|
430
|
+
* worker factory. The service owns every Temporal client and HTTP listener it
|
|
431
|
+
* creates. The embedding host retains ownership of its injected DB and EventBus
|
|
432
|
+
* handles and closes those only after this service has drained.
|
|
433
|
+
*/
|
|
434
|
+
declare function createOpenGeniWorkerService(options: OpenGeniWorkerServiceOptions): Promise<OpenGeniWorkerService>;
|
|
435
|
+
type RunOpenGeniWorkerOptions = OpenGeniWorkerServiceOptions & {
|
|
436
|
+
/** Defaults to SIGTERM and SIGINT. Pass false when the host owns process signals. */
|
|
437
|
+
shutdownSignals?: false | ReadonlyArray<"SIGTERM" | "SIGINT">;
|
|
438
|
+
};
|
|
439
|
+
/** Start, drain, and close one embedded worker process. */
|
|
440
|
+
declare function runOpenGeniWorker(options: RunOpenGeniWorkerOptions): Promise<void>;
|
|
349
441
|
declare function startWorker(): Promise<void>;
|
|
350
442
|
|
|
351
|
-
export { FILE_UPLOAD_REAPER_PERIOD_MS, FILE_UPLOAD_REAPER_SCHEDULE_ID, type OpenGeniWorkerRole, type WorkerOptions, createOpenGeniWorker, createWorkerWorkflowSignaler, registerFileUploadReaperSchedule, registerSandboxReaperSchedule, registerSessionWorkflowWakeDispatcherSchedule, startWorker };
|
|
443
|
+
export { FILE_UPLOAD_REAPER_PERIOD_MS, FILE_UPLOAD_REAPER_SCHEDULE_ID, type OpenGeniWorkerRole, type OpenGeniWorkerService, type OpenGeniWorkerServiceOptions, type RunOpenGeniWorkerOptions, type WorkerOptions, createOpenGeniWorker, createOpenGeniWorkerService, createWorkerWorkflowSignaler, registerFileUploadReaperSchedule, registerSandboxReaperSchedule, registerSessionWorkflowWakeDispatcherSchedule, resolveOpenGeniWorkflowDefinition, runOpenGeniWorker, startWorker, workerOwnsInternalSchedules };
|