@intx/hub-sessions 0.2.2 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -5
- package/dist/agent-repo.d.ts +23 -7
- package/dist/agent-repo.js +19 -6
- package/dist/agent-state-kind.js +18 -63
- package/dist/asset-service.d.ts +1 -20
- package/dist/asset-service.js +19 -97
- package/dist/committed-source-tree.d.ts +10 -0
- package/dist/committed-source-tree.js +35 -0
- package/dist/credential-push.d.ts +52 -7
- package/dist/credential-push.js +178 -22
- package/dist/event-collector-registry.d.ts +3 -2
- package/dist/event-collector-registry.js +42 -13
- package/dist/event-collector.d.ts +12 -2
- package/dist/event-collector.js +45 -4
- package/dist/hub-session-lookups.d.ts +125 -7
- package/dist/hub-session-lookups.js +541 -86
- package/dist/hub-session-orchestrator.d.ts +2 -3
- package/dist/hub-session-orchestrator.js +22 -56
- package/dist/index.d.ts +19 -9
- package/dist/index.js +16 -7
- package/dist/reconciliation-scheduler.d.ts +14 -0
- package/dist/reconciliation-scheduler.js +55 -0
- package/dist/repo-store/index.d.ts +2 -1
- package/dist/repo-store/index.js +1 -0
- package/dist/repo-store/store.d.ts +1 -1
- package/dist/repo-store/store.js +138 -1
- package/dist/repo-store/subscribe-kind.d.ts +6 -3
- package/dist/repo-store/subscribe-kind.js +42 -77
- package/dist/repo-store/types.d.ts +94 -6
- package/dist/repo-store/user-principal-gate.d.ts +26 -0
- package/dist/repo-store/user-principal-gate.js +78 -0
- package/dist/session-service.d.ts +252 -126
- package/dist/session-service.js +851 -624
- package/dist/sidecar-allocation/capability-policy.d.ts +27 -0
- package/dist/sidecar-allocation/capability-policy.js +124 -0
- package/dist/sidecar-allocation/contracts.d.ts +101 -0
- package/dist/sidecar-allocation/contracts.js +26 -0
- package/dist/sidecar-allocation/index.d.ts +5 -0
- package/dist/sidecar-allocation/index.js +4 -0
- package/dist/sidecar-allocation/operation.d.ts +10 -0
- package/dist/sidecar-allocation/operation.js +54 -0
- package/dist/sidecar-allocation/plugin-registry.d.ts +24 -0
- package/dist/sidecar-allocation/plugin-registry.js +61 -0
- package/dist/sidecar-allocation/reconciler.d.ts +54 -0
- package/dist/sidecar-allocation/reconciler.js +825 -0
- package/dist/skill-kind.js +12 -62
- package/dist/substrate.d.ts +3 -3
- package/dist/substrate.js +1 -1
- package/dist/workflow-allocation-service.d.ts +64 -0
- package/dist/workflow-allocation-service.js +554 -0
- package/dist/workflow-closure-resolution.d.ts +106 -0
- package/dist/workflow-closure-resolution.js +123 -0
- package/dist/workflow-definition-ensure.d.ts +24 -0
- package/dist/workflow-definition-ensure.js +75 -0
- package/dist/workflow-dispatch-service.d.ts +42 -0
- package/dist/workflow-dispatch-service.js +209 -0
- package/dist/workflow-dispatch-settlement.d.ts +29 -0
- package/dist/workflow-dispatch-settlement.js +140 -0
- package/dist/workflow-kind.d.ts +29 -1
- package/dist/workflow-kind.js +143 -139
- package/dist/workflow-probe-gate.d.ts +286 -0
- package/dist/workflow-probe-gate.js +382 -0
- package/dist/workflow-run-kind.d.ts +239 -32
- package/dist/workflow-run-kind.js +874 -188
- package/dist/workflow-run-reader.d.ts +1 -1
- package/dist/workflow-run-reader.js +3 -7
- package/dist/workflow-run-restore.d.ts +16 -0
- package/dist/workflow-run-restore.js +30 -0
- package/dist/workflow-source-closure.d.ts +35 -0
- package/dist/workflow-source-closure.js +342 -0
- package/dist/workflow-source-pins.d.ts +8 -0
- package/dist/workflow-source-pins.js +14 -0
- package/dist/ws/index.d.ts +3 -3
- package/dist/ws/index.js +2 -2
- package/dist/ws/pending-tracker.d.ts +93 -0
- package/dist/ws/pending-tracker.js +132 -0
- package/dist/ws/sidecar-events.d.ts +139 -37
- package/dist/ws/sidecar-events.js +2 -2
- package/dist/ws/sidecar-handler.d.ts +218 -60
- package/dist/ws/sidecar-handler.js +1713 -732
- package/dist/ws/sidecar-handler.test-helpers.d.ts +38 -0
- package/dist/ws/sidecar-handler.test-helpers.js +95 -0
- package/dist/ws/sidecar-token-authenticator.d.ts +3 -1
- package/dist/ws/sidecar-token-authenticator.js +78 -7
- package/package.json +14 -13
- package/dist/available-skills-stanza.d.ts +0 -21
- package/dist/available-skills-stanza.js +0 -32
package/dist/credential-push.js
CHANGED
|
@@ -1,39 +1,63 @@
|
|
|
1
|
-
//
|
|
2
|
-
//
|
|
1
|
+
// Hub-side producers that push credential-material changes to running
|
|
2
|
+
// deployments over `credentials.update`.
|
|
3
3
|
//
|
|
4
|
-
//
|
|
5
|
-
//
|
|
6
|
-
|
|
4
|
+
// Two shapes, one per removal semantic:
|
|
5
|
+
// - Source re-resolve (`pushSourceUpdates`, `pushSourceUpdatesSubtree`): after
|
|
6
|
+
// a credential secret rotation or a catalog edit, re-resolve each running
|
|
7
|
+
// instance's inference sources and push the refreshed material followed by
|
|
8
|
+
// the `sources.update` that references it. An inference source references
|
|
9
|
+
// its credential by id only, so the rotated secret rides the cell.
|
|
10
|
+
// - Flat named revoke (`pushCredentialRevoke`): after a credential is deleted
|
|
11
|
+
// or deliberately revoked, broadcast a `revoke` naming that credentialId so
|
|
12
|
+
// every running deployment drops it. The removed id is NAMED by the actor,
|
|
13
|
+
// so this needs no diff against a prior delivery.
|
|
14
|
+
import { eq, and, inArray, isNull, isNotNull } from "drizzle-orm";
|
|
7
15
|
import { getLogger } from "@intx/log";
|
|
8
|
-
import {
|
|
9
|
-
import { resolveInstanceModelSources, getDescendantTenants } from "@intx/db";
|
|
16
|
+
import { workflowRun } from "@intx/db/schema";
|
|
17
|
+
import { resolveInstanceModelSources, getDescendantTenants, reresolveCurrentMaterials, } from "@intx/db";
|
|
10
18
|
const log = getLogger(["hub", "credentials"]);
|
|
11
19
|
/**
|
|
12
20
|
* Re-resolve a single running instance's inference sources from the catalog
|
|
13
|
-
* (the
|
|
14
|
-
* the instance) and push the ordered list to its sidecar. The head of the
|
|
21
|
+
* (the definition's model requirements plus the invoker preferences persisted
|
|
22
|
+
* on the instance) and push the ordered list to its sidecar. The head of the
|
|
15
23
|
* catalog-priority-ordered list is the active default; the tail is the
|
|
16
24
|
* failover chain.
|
|
17
25
|
*
|
|
18
26
|
* No-op when the instance resolves to no launchable source — the resolver's
|
|
19
27
|
* own logger is the signal for why.
|
|
20
28
|
*/
|
|
21
|
-
export async function pushInstanceSourceUpdate(db, sidecarRouter, instance) {
|
|
22
|
-
const resolution = await resolveInstanceModelSources(db, instance.tenantId, instance);
|
|
29
|
+
export async function pushInstanceSourceUpdate(db, sidecarRouter, instance, credentialCipher) {
|
|
30
|
+
const resolution = await resolveInstanceModelSources(db, instance.tenantId, instance, credentialCipher);
|
|
23
31
|
if (!resolution.ok)
|
|
24
32
|
return;
|
|
25
33
|
const [head] = resolution.sources;
|
|
26
34
|
if (head === undefined)
|
|
27
35
|
return;
|
|
36
|
+
// Push the credential material before the source list. A source references
|
|
37
|
+
// its credential by id, so the cell must hold the (possibly rotated) secret
|
|
38
|
+
// before the source list that points at it lands. Inference sources carry no
|
|
39
|
+
// binding descriptor. A failure here propagates and aborts the source push --
|
|
40
|
+
// never a stale secret paired with a fresh source list.
|
|
41
|
+
if (resolution.materials.length > 0) {
|
|
42
|
+
const delivery = {
|
|
43
|
+
bindings: [],
|
|
44
|
+
materials: resolution.materials,
|
|
45
|
+
};
|
|
46
|
+
await sidecarRouter.sendCredentialsUpdate(instance.address, delivery);
|
|
47
|
+
}
|
|
28
48
|
await sidecarRouter.sendSourcesUpdate(instance.address, resolution.sources, head.id);
|
|
29
49
|
}
|
|
30
50
|
/**
|
|
31
51
|
* Re-resolve every running instance in the given tenants against the catalog
|
|
32
|
-
* and push the updates to sidecars.
|
|
33
|
-
*
|
|
34
|
-
*
|
|
52
|
+
* and push the updates to sidecars. The running instances are the folded runs
|
|
53
|
+
* a launch produces: born running, with a routing address and no deployment.
|
|
54
|
+
* Deployment-anchor runs (which own a deployment id and a workflow-derived
|
|
55
|
+
* address) and address-less child runs route via the deployment, not this
|
|
56
|
+
* per-instance push, so they are excluded. Each instance re-resolves from its
|
|
57
|
+
* own tenant's context (its ancestor chain), so the rotated/edited upstream
|
|
58
|
+
* entry flows through. Errors are logged per-instance but do not propagate.
|
|
35
59
|
*/
|
|
36
|
-
async function pushSourceUpdatesToTenants(db, sidecarRouter, tenantIds) {
|
|
60
|
+
async function pushSourceUpdatesToTenants(db, sidecarRouter, tenantIds, credentialCipher) {
|
|
37
61
|
if (tenantIds.length === 0)
|
|
38
62
|
return;
|
|
39
63
|
// Callers fire this without awaiting, so it must never reject: a failure to
|
|
@@ -41,12 +65,34 @@ async function pushSourceUpdatesToTenants(db, sidecarRouter, tenantIds) {
|
|
|
41
65
|
// rejection. The push is best effort — the next mutation or a sidecar
|
|
42
66
|
// reconnect re-resolves sources.
|
|
43
67
|
try {
|
|
44
|
-
|
|
45
|
-
|
|
68
|
+
// The instance-shaped runs: running, addressable, and not anchored on a
|
|
69
|
+
// deployment. `anchorRunId IS NULL` excludes the deployment-anchor runs
|
|
70
|
+
// (which set it to their own id), so no deployment-anchor run address can
|
|
71
|
+
// reach the address-targeted push below. Mirrors the /me/workflows/runs and
|
|
72
|
+
// tenant run-list predicate.
|
|
73
|
+
const instances = await db.query.workflowRun.findMany({
|
|
74
|
+
where: and(inArray(workflowRun.tenantId, tenantIds), eq(workflowRun.status, "running"), isNull(workflowRun.anchorRunId), isNotNull(workflowRun.address)),
|
|
46
75
|
});
|
|
47
76
|
if (instances.length === 0)
|
|
48
77
|
return;
|
|
49
|
-
const results = await Promise.allSettled(instances.map((instance) =>
|
|
78
|
+
const results = await Promise.allSettled(instances.map(async (instance) => {
|
|
79
|
+
// The isNotNull(address) filter guarantees a value; a null here is a
|
|
80
|
+
// broken invariant. The callback is async, so this throw becomes a
|
|
81
|
+
// rejected promise captured per-instance by allSettled and logged
|
|
82
|
+
// below -- one bad row is surfaced, not fatal to the whole batch.
|
|
83
|
+
if (instance.address === null) {
|
|
84
|
+
throw new Error(`running run ${instance.id} matched the non-null-address filter but has a null address`);
|
|
85
|
+
}
|
|
86
|
+
if (instance.definitionId === null) {
|
|
87
|
+
throw new Error(`running run ${instance.id} has no workflow definition`);
|
|
88
|
+
}
|
|
89
|
+
return pushInstanceSourceUpdate(db, sidecarRouter, {
|
|
90
|
+
address: instance.address,
|
|
91
|
+
definitionId: instance.definitionId,
|
|
92
|
+
tenantId: instance.tenantId,
|
|
93
|
+
modelPreferences: instance.modelPreferences,
|
|
94
|
+
}, credentialCipher);
|
|
95
|
+
}));
|
|
50
96
|
for (const result of results) {
|
|
51
97
|
if (result.status === "rejected") {
|
|
52
98
|
log.warn `Failed to push source update: ${String(result.reason)}`;
|
|
@@ -63,8 +109,8 @@ async function pushSourceUpdatesToTenants(db, sidecarRouter, tenantIds) {
|
|
|
63
109
|
* through because resolution dereferences the provider's credential reference
|
|
64
110
|
* to the current secret.
|
|
65
111
|
*/
|
|
66
|
-
export async function pushSourceUpdates(db, sidecarRouter, tenantId) {
|
|
67
|
-
await pushSourceUpdatesToTenants(db, sidecarRouter, [tenantId]);
|
|
112
|
+
export async function pushSourceUpdates(db, sidecarRouter, tenantId, credentialCipher) {
|
|
113
|
+
await pushSourceUpdatesToTenants(db, sidecarRouter, [tenantId], credentialCipher);
|
|
68
114
|
}
|
|
69
115
|
/**
|
|
70
116
|
* After a catalog edit in a tenant, re-resolve and push to every running
|
|
@@ -72,7 +118,7 @@ export async function pushSourceUpdates(db, sidecarRouter, tenantId) {
|
|
|
72
118
|
* edited tenant's catalog, so a change there (a disabled provider, a new
|
|
73
119
|
* offering, a price update) alters their resolved sources too.
|
|
74
120
|
*/
|
|
75
|
-
export async function pushSourceUpdatesSubtree(db, sidecarRouter, tenantId) {
|
|
121
|
+
export async function pushSourceUpdatesSubtree(db, sidecarRouter, tenantId, credentialCipher) {
|
|
76
122
|
let tenants;
|
|
77
123
|
try {
|
|
78
124
|
tenants = await getDescendantTenants(db, tenantId);
|
|
@@ -81,5 +127,115 @@ export async function pushSourceUpdatesSubtree(db, sidecarRouter, tenantId) {
|
|
|
81
127
|
log.warn `Failed to enumerate descendants for source push: ${String(err)}`;
|
|
82
128
|
return;
|
|
83
129
|
}
|
|
84
|
-
await pushSourceUpdatesToTenants(db, sidecarRouter, tenants);
|
|
130
|
+
await pushSourceUpdatesToTenants(db, sidecarRouter, tenants, credentialCipher);
|
|
131
|
+
}
|
|
132
|
+
/**
|
|
133
|
+
* After a credential is deleted or deliberately revoked, evict it from every
|
|
134
|
+
* running deployment in the tenant AND its descendants. A descendant resolves
|
|
135
|
+
* an ancestor's tenant-owned credential through the tenant walk-up, so a
|
|
136
|
+
* revoke in one tenant can affect a descendant's run. The push is a flat named
|
|
137
|
+
* revocation: the child drops the credentialId's material and any binding that
|
|
138
|
+
* references it, and a run that never held it no-ops. Because a flat revoke is
|
|
139
|
+
* safe to broadcast, this needs no per-instance ledger of what was delivered.
|
|
140
|
+
*
|
|
141
|
+
* Callers fire this without awaiting, so it must never reject: a failure to
|
|
142
|
+
* enumerate or push is logged and dropped. This closes the ONLINE revocation
|
|
143
|
+
* window (a running deployment stops holding the credential now); the offline
|
|
144
|
+
* window -- a run whose sidecar was disconnected when the revoke fired -- is
|
|
145
|
+
* closed by the reconnect resync, not here.
|
|
146
|
+
*/
|
|
147
|
+
export async function pushCredentialRevoke(db, sidecarRouter, tenantId, credentialId) {
|
|
148
|
+
let tenants;
|
|
149
|
+
try {
|
|
150
|
+
tenants = await getDescendantTenants(db, tenantId);
|
|
151
|
+
}
|
|
152
|
+
catch (err) {
|
|
153
|
+
log.warn `Failed to enumerate descendants for credential revoke: ${String(err)}`;
|
|
154
|
+
return;
|
|
155
|
+
}
|
|
156
|
+
try {
|
|
157
|
+
// Every running, addressable run in the subtree. Unlike the source-update
|
|
158
|
+
// push this does NOT exclude deployment-anchor runs (`anchorRunId IS NULL`):
|
|
159
|
+
// a deployed workflow is the primary credential consumer, and a flat revoke
|
|
160
|
+
// is safe to deliver to any address -- a run that never held the credential
|
|
161
|
+
// no-ops on it.
|
|
162
|
+
const runs = await db.query.workflowRun.findMany({
|
|
163
|
+
where: and(inArray(workflowRun.tenantId, tenants), eq(workflowRun.status, "running"), isNotNull(workflowRun.address)),
|
|
164
|
+
columns: { id: true, address: true },
|
|
165
|
+
});
|
|
166
|
+
const addresses = new Set();
|
|
167
|
+
for (const run of runs) {
|
|
168
|
+
// The isNotNull(address) filter guarantees a value; a null here is a
|
|
169
|
+
// broken invariant, surfaced rather than silently skipped.
|
|
170
|
+
if (run.address === null) {
|
|
171
|
+
throw new Error(`running run ${run.id} matched the non-null-address filter but has a null address`);
|
|
172
|
+
}
|
|
173
|
+
addresses.add(run.address);
|
|
174
|
+
}
|
|
175
|
+
if (addresses.size === 0)
|
|
176
|
+
return;
|
|
177
|
+
const emptyDelivery = { bindings: [], materials: [] };
|
|
178
|
+
const results = await Promise.allSettled([...addresses].map((address) => sidecarRouter.sendCredentialsUpdate(address, emptyDelivery, [
|
|
179
|
+
credentialId,
|
|
180
|
+
])));
|
|
181
|
+
for (const result of results) {
|
|
182
|
+
if (result.status === "rejected") {
|
|
183
|
+
log.warn `Failed to push credential revoke: ${String(result.reason)}`;
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
catch (err) {
|
|
188
|
+
log.warn `Failed to push credential revoke: ${String(err)}`;
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
/**
|
|
192
|
+
* Reconcile a reconnecting deployment's credentials against its deploy-time set.
|
|
193
|
+
* Re-resolve the CURRENT material for every credentialId the deployment
|
|
194
|
+
* persisted at deploy (`workflow_run.credentialRefs`), then push a MERGE that
|
|
195
|
+
* upserts the survivors (picking up a same-id secret rotation) and REVOKES the
|
|
196
|
+
* deploy-time ids that no longer resolve (deleted or revoked while the sidecar
|
|
197
|
+
* was disconnected). Closes the OFFLINE revocation window (the online window is
|
|
198
|
+
* closed by `pushCredentialRevoke`).
|
|
199
|
+
*
|
|
200
|
+
* Merge, not wholesale-replace: `credentialRefs` is only the deploy-time id set,
|
|
201
|
+
* not the child's complete live set (a catalog re-point can deliver a new
|
|
202
|
+
* credential online), so a replace would evict online-added credentials. The
|
|
203
|
+
* merge upserts survivors and names the dead ids in `revoke`, leaving online
|
|
204
|
+
* credentials untouched. It does NOT handle an id-CHANGING rotation of a
|
|
205
|
+
* deploy-time source (the new id is not in `credentialRefs`); a later source
|
|
206
|
+
* push delivers that.
|
|
207
|
+
*
|
|
208
|
+
* No-op when the run persisted no credential refs (a folded run, or a
|
|
209
|
+
* deployment with no credentials). Fire-and-forget from the reconnect handler:
|
|
210
|
+
* it never rejects. A live-but-unresolvable credential (its provider vanished
|
|
211
|
+
* or has no API base URL) makes `reresolveCurrentMaterials` throw, which aborts
|
|
212
|
+
* the WHOLE reconcile (logged, not sent) so a partial set with a spurious
|
|
213
|
+
* revoke never lands. This is deliberately all-or-nothing: one misconfigured
|
|
214
|
+
* credential blocks this reconnect's revocation of the others too, trading
|
|
215
|
+
* revocation timeliness for never falsely evicting a live credential. The next
|
|
216
|
+
* reconnect (or an online revoke) retries.
|
|
217
|
+
*/
|
|
218
|
+
export async function pushCredentialReconcile(db, sidecarRouter, agentAddress, credentialCipher) {
|
|
219
|
+
try {
|
|
220
|
+
const run = await db.query.workflowRun.findFirst({
|
|
221
|
+
where: eq(workflowRun.address, agentAddress),
|
|
222
|
+
columns: { credentialRefs: true },
|
|
223
|
+
});
|
|
224
|
+
if (run === undefined)
|
|
225
|
+
return;
|
|
226
|
+
const refs = run.credentialRefs;
|
|
227
|
+
if (refs === null)
|
|
228
|
+
return;
|
|
229
|
+
const materials = await reresolveCurrentMaterials(db, refs.credentialIds, credentialCipher);
|
|
230
|
+
const resolvedIds = new Set(materials.map((material) => material.credentialId));
|
|
231
|
+
// Deploy-time ids that no longer resolve: deleted or revoked while offline.
|
|
232
|
+
const revoke = refs.credentialIds.filter((id) => !resolvedIds.has(id));
|
|
233
|
+
// A binding whose credential dropped out of the re-resolution goes with it.
|
|
234
|
+
const bindings = refs.bindings.filter((binding) => resolvedIds.has(binding.credentialId));
|
|
235
|
+
const delivery = { bindings, materials };
|
|
236
|
+
await sidecarRouter.sendCredentialsUpdate(agentAddress, delivery, revoke.length > 0 ? revoke : undefined);
|
|
237
|
+
}
|
|
238
|
+
catch (err) {
|
|
239
|
+
log.warn `Failed to reconcile credentials for ${agentAddress}: ${String(err)}`;
|
|
240
|
+
}
|
|
85
241
|
}
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import type { DB } from "@intx/db";
|
|
2
2
|
import type { InferenceEvent } from "@intx/types/runtime";
|
|
3
3
|
import type { SessionStatus } from "@intx/types";
|
|
4
|
-
import { type TurnFinalized } from "./event-collector.js";
|
|
4
|
+
import { type TurnFinalized, type TurnUsage } from "./event-collector.js";
|
|
5
5
|
export type EventCollectorRegistry = {
|
|
6
|
-
create(agentAddress: string, tenantId: string, sessionId: string,
|
|
6
|
+
create(agentAddress: string, tenantId: string, sessionId: string, runId: string): void;
|
|
7
7
|
dispatch(agentAddress: string, event: InferenceEvent): void;
|
|
8
8
|
abandon(agentAddress: string): void;
|
|
9
9
|
has(agentAddress: string): boolean;
|
|
@@ -15,6 +15,7 @@ export type EventCollectorRegistry = {
|
|
|
15
15
|
export type EventCollectorRegistryConfig = {
|
|
16
16
|
db: DB["db"];
|
|
17
17
|
onTurnFinalized?: (agentAddress: string, turn: TurnFinalized) => void;
|
|
18
|
+
onUsage?: (agentAddress: string, usage: TurnUsage) => void;
|
|
18
19
|
};
|
|
19
20
|
export declare function deriveStatus(event: InferenceEvent): SessionStatus | null;
|
|
20
21
|
export declare function createEventCollectorRegistry(config: EventCollectorRegistryConfig): EventCollectorRegistry;
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
// Registry of active event collectors, keyed by
|
|
1
|
+
// Registry of active event collectors, keyed by run address.
|
|
2
2
|
//
|
|
3
3
|
// The hub creates a collector when an instance starts and removes it when the
|
|
4
4
|
// instance ends or the sidecar disconnects. The hub session orchestrator's
|
|
5
|
-
// `agent.event` listener looks up the collector by
|
|
5
|
+
// `agent.event` listener looks up the collector by run address and
|
|
6
6
|
// dispatches the event.
|
|
7
7
|
import { getLogger } from "@intx/log";
|
|
8
8
|
import { createEventCollector, } from "./event-collector.js";
|
|
@@ -30,10 +30,13 @@ export function deriveStatus(event) {
|
|
|
30
30
|
}
|
|
31
31
|
}
|
|
32
32
|
export function createEventCollectorRegistry(config) {
|
|
33
|
-
const { db, onTurnFinalized } = config;
|
|
33
|
+
const { db, onTurnFinalized, onUsage } = config;
|
|
34
34
|
const collectors = new Map();
|
|
35
35
|
const statuses = new Map();
|
|
36
|
-
|
|
36
|
+
// Per-address tail promise: serializes onEvent/abandon work for one run
|
|
37
|
+
// address so their DB writes cannot interleave. Reaped when it drains.
|
|
38
|
+
const tails = new Map();
|
|
39
|
+
function create(agentAddress, tenantId, sessionId, runId) {
|
|
37
40
|
if (collectors.has(agentAddress)) {
|
|
38
41
|
log.warn `Collector already exists for ${agentAddress}, replacing`;
|
|
39
42
|
abandon(agentAddress);
|
|
@@ -41,13 +44,18 @@ export function createEventCollectorRegistry(config) {
|
|
|
41
44
|
const collector = createEventCollector({
|
|
42
45
|
db,
|
|
43
46
|
sessionId,
|
|
44
|
-
|
|
47
|
+
runId,
|
|
45
48
|
tenantId,
|
|
46
49
|
...(onTurnFinalized
|
|
47
50
|
? {
|
|
48
51
|
onTurnFinalized: (turn) => onTurnFinalized(agentAddress, turn),
|
|
49
52
|
}
|
|
50
53
|
: {}),
|
|
54
|
+
...(onUsage
|
|
55
|
+
? {
|
|
56
|
+
onUsage: (usage) => onUsage(agentAddress, usage),
|
|
57
|
+
}
|
|
58
|
+
: {}),
|
|
51
59
|
});
|
|
52
60
|
collectors.set(agentAddress, collector);
|
|
53
61
|
statuses.set(agentAddress, { status: "idle" });
|
|
@@ -56,6 +64,25 @@ export function createEventCollectorRegistry(config) {
|
|
|
56
64
|
collectors.delete(agentAddress);
|
|
57
65
|
statuses.delete(agentAddress);
|
|
58
66
|
}
|
|
67
|
+
// Chain `work` onto the address's tail so per-address work runs in order and
|
|
68
|
+
// never interleaves, while the caller stays non-blocking. `onError` swallows
|
|
69
|
+
// a failure so one bad event cannot wedge the chain; `onSettled` runs after
|
|
70
|
+
// the work settles. The tail entry is reaped once no later work is queued.
|
|
71
|
+
function enqueue(agentAddress, work, onError, onSettled) {
|
|
72
|
+
const prev = tails.get(agentAddress) ?? Promise.resolve();
|
|
73
|
+
const next = prev
|
|
74
|
+
.then(work)
|
|
75
|
+
.catch(onError)
|
|
76
|
+
.finally(() => {
|
|
77
|
+
if (onSettled !== undefined) {
|
|
78
|
+
onSettled();
|
|
79
|
+
}
|
|
80
|
+
if (tails.get(agentAddress) === next) {
|
|
81
|
+
tails.delete(agentAddress);
|
|
82
|
+
}
|
|
83
|
+
});
|
|
84
|
+
tails.set(agentAddress, next);
|
|
85
|
+
}
|
|
59
86
|
function dispatch(agentAddress, event) {
|
|
60
87
|
const collector = collectors.get(agentAddress);
|
|
61
88
|
if (collector === undefined) {
|
|
@@ -67,13 +94,10 @@ export function createEventCollectorRegistry(config) {
|
|
|
67
94
|
}
|
|
68
95
|
const isTerminal = event.type === "reactor.done" ||
|
|
69
96
|
(event.type === "reactor.error" && event.data.fatal);
|
|
70
|
-
collector
|
|
71
|
-
.onEvent(event)
|
|
72
|
-
.catch((err) => {
|
|
97
|
+
enqueue(agentAddress, () => collector.onEvent(event), (err) => {
|
|
73
98
|
log.warn `Failed to persist event ${event.type} seq=${String(event.seq)} for ${agentAddress}: ${err instanceof Error ? err.message : String(err)}`;
|
|
74
|
-
})
|
|
75
|
-
.
|
|
76
|
-
if (isTerminal) {
|
|
99
|
+
}, () => {
|
|
100
|
+
if (isTerminal && collectors.get(agentAddress) === collector) {
|
|
77
101
|
removeCollector(agentAddress);
|
|
78
102
|
}
|
|
79
103
|
});
|
|
@@ -82,10 +106,15 @@ export function createEventCollectorRegistry(config) {
|
|
|
82
106
|
const collector = collectors.get(agentAddress);
|
|
83
107
|
if (collector === undefined)
|
|
84
108
|
return;
|
|
85
|
-
|
|
109
|
+
// Stop NEW dispatches immediately; the queued closures keep their own
|
|
110
|
+
// reference so already-queued events still drain before the abandon runs.
|
|
111
|
+
removeCollector(agentAddress);
|
|
112
|
+
// Chain the abandon onto the tail so it runs AFTER any queued onEvents
|
|
113
|
+
// instead of racing them. Otherwise a queued beginTurn could create a
|
|
114
|
+
// fresh `running` turn row after the collector was finalized, orphaning it.
|
|
115
|
+
enqueue(agentAddress, () => collector.abandon(), (err) => {
|
|
86
116
|
log.warn `Failed to abandon collector for ${agentAddress}: ${err instanceof Error ? err.message : String(err)}`;
|
|
87
117
|
});
|
|
88
|
-
removeCollector(agentAddress);
|
|
89
118
|
}
|
|
90
119
|
function has(agentAddress) {
|
|
91
120
|
return collectors.has(agentAddress);
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { InferenceEvent } from "@intx/types/runtime";
|
|
1
|
+
import type { InferenceEvent, TokenUsage } from "@intx/types/runtime";
|
|
2
2
|
import { type DB } from "@intx/db";
|
|
3
3
|
export type TurnToolCall = {
|
|
4
4
|
name: string;
|
|
@@ -22,6 +22,15 @@ export type TurnFinalized = {
|
|
|
22
22
|
content: string;
|
|
23
23
|
}[];
|
|
24
24
|
};
|
|
25
|
+
export type TurnUsage = {
|
|
26
|
+
tenantId: string;
|
|
27
|
+
sessionId: string;
|
|
28
|
+
runId: string;
|
|
29
|
+
turnId: string;
|
|
30
|
+
provider: string;
|
|
31
|
+
model: string;
|
|
32
|
+
usage: TokenUsage;
|
|
33
|
+
};
|
|
25
34
|
export type EventCollector = {
|
|
26
35
|
onEvent(event: InferenceEvent): Promise<void>;
|
|
27
36
|
abandon(): Promise<void>;
|
|
@@ -32,8 +41,9 @@ export type EventCollector = {
|
|
|
32
41
|
export type EventCollectorConfig = {
|
|
33
42
|
db: DB["db"];
|
|
34
43
|
sessionId: string;
|
|
35
|
-
|
|
44
|
+
runId: string;
|
|
36
45
|
tenantId: string;
|
|
37
46
|
onTurnFinalized?: (turn: TurnFinalized) => void;
|
|
47
|
+
onUsage?: (usage: TurnUsage) => void;
|
|
38
48
|
};
|
|
39
49
|
export declare function createEventCollector(config: EventCollectorConfig): EventCollector;
|
package/dist/event-collector.js
CHANGED
|
@@ -10,7 +10,7 @@ import { parseTurnPartType } from "@intx/db";
|
|
|
10
10
|
import { generateId } from "@intx/hub-common";
|
|
11
11
|
const log = getLogger(["hub", "event-collector"]);
|
|
12
12
|
export function createEventCollector(config) {
|
|
13
|
-
const { db, sessionId,
|
|
13
|
+
const { db, sessionId, runId, tenantId, onTurnFinalized, onUsage } = config;
|
|
14
14
|
// Current inference turn being accumulated. A new turn is created on each
|
|
15
15
|
// inference.start. Finalized on connector.reply, reactor.done,
|
|
16
16
|
// reactor.error (fatal), or abandon. Null when no turn is active.
|
|
@@ -47,6 +47,12 @@ export function createEventCollector(config) {
|
|
|
47
47
|
let accumulatedToolCalls = [];
|
|
48
48
|
// Tool results that reported isError, accumulated for TurnFinalized.
|
|
49
49
|
let accumulatedToolErrors = [];
|
|
50
|
+
// Final cumulative token usage for the current turn. `inference.usage` events
|
|
51
|
+
// carry a running cumulative total and fire multiple times per step, so these
|
|
52
|
+
// are OVERWRITTEN (not summed); the last value before finalize is the
|
|
53
|
+
// authoritative per-turn total. Both reset on each new turn.
|
|
54
|
+
let turnUsage = null;
|
|
55
|
+
let turnSource = null;
|
|
50
56
|
async function onEvent(event) {
|
|
51
57
|
switch (event.type) {
|
|
52
58
|
case "inference.start":
|
|
@@ -59,6 +65,8 @@ export function createEventCollector(config) {
|
|
|
59
65
|
case "inference.done":
|
|
60
66
|
await handleInferenceDone(event.data.turn.content);
|
|
61
67
|
streamingText = "";
|
|
68
|
+
turnUsage = event.data.usage;
|
|
69
|
+
turnSource = event.data.source;
|
|
62
70
|
break;
|
|
63
71
|
case "tool.done": {
|
|
64
72
|
const callId = event.data.result.callId;
|
|
@@ -145,9 +153,15 @@ export function createEventCollector(config) {
|
|
|
145
153
|
});
|
|
146
154
|
}
|
|
147
155
|
break;
|
|
156
|
+
case "inference.usage":
|
|
157
|
+
// Cumulative running total that fires several times per step; overwrite
|
|
158
|
+
// so the last value before finalize is the authoritative per-turn
|
|
159
|
+
// usage, emitted once via onUsage from finalizeTurn.
|
|
160
|
+
turnUsage = event.data.usage;
|
|
161
|
+
turnSource = event.data.source;
|
|
162
|
+
break;
|
|
148
163
|
default:
|
|
149
|
-
// reactor.start, streaming deltas,
|
|
150
|
-
// not persisted.
|
|
164
|
+
// reactor.start, streaming deltas, and other events are not persisted.
|
|
151
165
|
break;
|
|
152
166
|
}
|
|
153
167
|
}
|
|
@@ -170,10 +184,12 @@ export function createEventCollector(config) {
|
|
|
170
184
|
callArgs.clear();
|
|
171
185
|
accumulatedToolCalls = [];
|
|
172
186
|
accumulatedToolErrors = [];
|
|
187
|
+
turnUsage = null;
|
|
188
|
+
turnSource = null;
|
|
173
189
|
await db.insert(inferenceTurn).values({
|
|
174
190
|
id: currentTurnId,
|
|
175
191
|
sessionId,
|
|
176
|
-
|
|
192
|
+
runId,
|
|
177
193
|
tenantId,
|
|
178
194
|
model,
|
|
179
195
|
status: "running",
|
|
@@ -208,6 +224,14 @@ export function createEventCollector(config) {
|
|
|
208
224
|
// declines differently from regular assistant output.
|
|
209
225
|
await insertPart("refusal", block.reason, null);
|
|
210
226
|
break;
|
|
227
|
+
case "safety_rating":
|
|
228
|
+
// Structured safety signals (e.g. Gemini
|
|
229
|
+
// promptFeedback.blockReason). Persist the reason under a
|
|
230
|
+
// dedicated part kind so a blocked turn is not empty in
|
|
231
|
+
// the audit trail. Content is the provider-native reason
|
|
232
|
+
// string (observed: PROHIBITED_CONTENT).
|
|
233
|
+
await insertPart("safety_rating", block.blockReason, null);
|
|
234
|
+
break;
|
|
211
235
|
case "tool_call":
|
|
212
236
|
callNames.set(block.id, block.name);
|
|
213
237
|
callArgs.set(block.id, block.arguments);
|
|
@@ -310,6 +334,23 @@ export function createEventCollector(config) {
|
|
|
310
334
|
toolErrors: [...accumulatedToolErrors],
|
|
311
335
|
});
|
|
312
336
|
}
|
|
337
|
+
// Report per-turn usage once, alongside the finalize notify. Gated on
|
|
338
|
+
// turnUsage being present so a turn that ran no inference emits nothing.
|
|
339
|
+
// NOTE: abandon() finalizes with notify=false, so a turn abandoned
|
|
340
|
+
// mid-step (e.g. a sidecar disconnect) reports no usage even if the
|
|
341
|
+
// provider already billed input tokens -- an accepted gap until durable
|
|
342
|
+
// usage persistence lands on the turn row.
|
|
343
|
+
if (notify && onUsage && turnUsage !== null && turnSource !== null) {
|
|
344
|
+
onUsage({
|
|
345
|
+
tenantId,
|
|
346
|
+
sessionId,
|
|
347
|
+
runId,
|
|
348
|
+
turnId,
|
|
349
|
+
provider: turnSource.provider,
|
|
350
|
+
model: turnSource.model,
|
|
351
|
+
usage: turnUsage,
|
|
352
|
+
});
|
|
353
|
+
}
|
|
313
354
|
currentTurnId = null;
|
|
314
355
|
}
|
|
315
356
|
async function abandon() {
|
|
@@ -5,13 +5,131 @@ export type HubSessionLookupsDeps = {
|
|
|
5
5
|
db: DB["db"];
|
|
6
6
|
agentRepoStore: AgentRepoStore;
|
|
7
7
|
};
|
|
8
|
-
export declare function createHubSessionLookups(deps: HubSessionLookupsDeps): Required<SidecarLookups
|
|
8
|
+
export declare function createHubSessionLookups(deps: HubSessionLookupsDeps): Required<Omit<SidecarLookups, "materializeMailTriggeredRunGrants" | "resyncCredentials" | "resolveSenderKey" | "resolveSenderKeyStrict">>;
|
|
9
9
|
/**
|
|
10
|
-
* Extract the
|
|
11
|
-
* Throws on any input the `@intx/types`-owned `
|
|
12
|
-
* rejects: missing or leading `@`, empty domain, or
|
|
13
|
-
* without the canonical `
|
|
10
|
+
* Extract the run id from an `<runId>@<domain>` run address.
|
|
11
|
+
* Throws on any input the `@intx/types`-owned `parseRunAddress`
|
|
12
|
+
* rejects: missing or leading `@`, empty domain, or a run id
|
|
13
|
+
* without the canonical `run_` prefix.
|
|
14
14
|
*/
|
|
15
15
|
export declare function parseAgentId(agentAddress: string): string;
|
|
16
|
-
|
|
17
|
-
|
|
16
|
+
/**
|
|
17
|
+
* A live routing endpoint backing a run address. Each address names one
|
|
18
|
+
* self-anchored `workflow_run`, so the endpoint is always that run.
|
|
19
|
+
*/
|
|
20
|
+
export interface RoutableEndpoint {
|
|
21
|
+
readonly id: string;
|
|
22
|
+
readonly tenantId: string;
|
|
23
|
+
readonly address: string;
|
|
24
|
+
readonly publicKey: string | null;
|
|
25
|
+
/**
|
|
26
|
+
* The endpoint's raw run status. Resolution is `endedAt`-filtered, so a
|
|
27
|
+
* resolved endpoint is not necessarily live: a leaked run is deliberately
|
|
28
|
+
* kept routable (terminal status, null `endedAt`) to stay reachable, and the
|
|
29
|
+
* reconnect reaction reads this to keep such an endpoint routable without
|
|
30
|
+
* restoring a collector.
|
|
31
|
+
*/
|
|
32
|
+
readonly status: string;
|
|
33
|
+
/**
|
|
34
|
+
* The live session backing this endpoint. A folded run has no session column,
|
|
35
|
+
* so this is the run's not-yet-ended `agent_session`, keyed by the run's
|
|
36
|
+
* principal. Transitional -- it retires when mail record-keeping moves off
|
|
37
|
+
* `agent_session`.
|
|
38
|
+
*/
|
|
39
|
+
readonly sessionId: string | null;
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Resolve a run address to the `workflow_run` endpoint backing it, keyed by
|
|
43
|
+
* the run's `address`. Every routable address names one self-anchored run --
|
|
44
|
+
* the deployment's anchor -- so this resolves the run's own address (the
|
|
45
|
+
* source `persistMail` depends on to record a triggered deployment's mail).
|
|
46
|
+
*/
|
|
47
|
+
export declare function resolveRoutableAddress(db: DB["db"], address: string): Promise<RoutableEndpoint | undefined>;
|
|
48
|
+
/**
|
|
49
|
+
* A folded run has no session column; its session is the `agent_session` keyed
|
|
50
|
+
* by the run's principal. By default this is the live (not-yet-ended) session,
|
|
51
|
+
* matching routing semantics; `includeEnded` also resolves a stopped run's
|
|
52
|
+
* ended session, which mail history needs. Returns null when the run has no
|
|
53
|
+
* principal or no matching session. Transitional, alongside
|
|
54
|
+
* `RoutableEndpoint.sessionId`.
|
|
55
|
+
*/
|
|
56
|
+
export declare function resolveRunSessionId(db: DB["db"], principalId: string | null, opts?: {
|
|
57
|
+
includeEnded?: boolean;
|
|
58
|
+
}): Promise<string | null>;
|
|
59
|
+
/**
|
|
60
|
+
* The folded run that owns a session, or null when the session belongs to no
|
|
61
|
+
* run. This is the inverse of `resolveRunSessionId`: a mail-read path holds a
|
|
62
|
+
* `sessionMail.sessionId` and no address, so it recovers the owning run by
|
|
63
|
+
* joining `workflow_run` to `agent_session` on their shared principal (a folded
|
|
64
|
+
* run, its session, and its launch all key on the same `instancePrincipalId`).
|
|
65
|
+
* Scoped to the tenant and routed through `workflow_run` so the returned id is
|
|
66
|
+
* proven to name a real run of this tenant -- callers key an authorization
|
|
67
|
+
* subject on it, so a session held by a non-run principal must fail closed to
|
|
68
|
+
* null rather than resolve to a fabricated subject.
|
|
69
|
+
*/
|
|
70
|
+
export declare function resolveRunIdForSession(db: DB["db"], sessionId: string, tenantId: string): Promise<string | null>;
|
|
71
|
+
/**
|
|
72
|
+
* A folded run resolved BY ID for the instance read/interact surface, shaped
|
|
73
|
+
* into one instance-shaped record. Unlike `resolveRoutableAddress` (keyed by
|
|
74
|
+
* address, live-only), this is keyed by the path id and does NOT filter
|
|
75
|
+
* terminated rows -- a stopped run's detail, mail history, and turns are still
|
|
76
|
+
* served. Keep the two separate: routing must never reach a dead endpoint,
|
|
77
|
+
* while the read surface must still render one.
|
|
78
|
+
*/
|
|
79
|
+
export interface RoutableRecord {
|
|
80
|
+
readonly id: string;
|
|
81
|
+
readonly tenantId: string;
|
|
82
|
+
/** The routing address. Non-null: a run resolves here only when it owns an
|
|
83
|
+
* address. */
|
|
84
|
+
readonly address: string;
|
|
85
|
+
readonly publicKey: string | null;
|
|
86
|
+
/** Raw run status. The wire mapping onto the instance status enum is a
|
|
87
|
+
* hub-api concern, done by the response shaper. */
|
|
88
|
+
readonly status: string;
|
|
89
|
+
readonly createdAt: Date;
|
|
90
|
+
/** A run has no `updatedAt` column, so it reports `endedAt ?? createdAt`. */
|
|
91
|
+
readonly updatedAt: Date;
|
|
92
|
+
readonly endedAt: Date | null;
|
|
93
|
+
/** The folded definition this run belongs to (`workflow_definition.id`). */
|
|
94
|
+
readonly definitionId: string;
|
|
95
|
+
readonly principalId: string | null;
|
|
96
|
+
readonly kernelId: string | null;
|
|
97
|
+
readonly sidecarId: string | null;
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Shape a run row and its already-resolved routing address into the run
|
|
101
|
+
* record. Callers decide whether the run resolves at all -- only a top-level
|
|
102
|
+
* run (`isTopLevelRun`) does -- and pass the address they have narrowed; this
|
|
103
|
+
* only maps the columns, including the run's `endedAt ?? createdAt` stand-in
|
|
104
|
+
* for the absent `updatedAt`.
|
|
105
|
+
*/
|
|
106
|
+
export declare function runRowToRoutableRecord(run: {
|
|
107
|
+
id: string;
|
|
108
|
+
tenantId: string;
|
|
109
|
+
publicKey: string | null;
|
|
110
|
+
status: string;
|
|
111
|
+
createdAt: Date;
|
|
112
|
+
endedAt: Date | null;
|
|
113
|
+
definitionId: string;
|
|
114
|
+
principalId: string | null;
|
|
115
|
+
kernelId: string | null;
|
|
116
|
+
sidecarId: string | null;
|
|
117
|
+
}, address: string): RoutableRecord;
|
|
118
|
+
/**
|
|
119
|
+
* A run is a top-level run -- the addressable head of a deployment -- when it
|
|
120
|
+
* owns a routing address AND self-anchors (`anchorRunId === id`). A lazy child
|
|
121
|
+
* park row anchors on its parent (`anchorRunId !== id`) and carries no address;
|
|
122
|
+
* either condition excludes it. This is the single predicate the run read
|
|
123
|
+
* surface classifies on, so the resolver and the run list cannot drift.
|
|
124
|
+
*/
|
|
125
|
+
export declare function isTopLevelRun(row: {
|
|
126
|
+
id: string;
|
|
127
|
+
address: string | null;
|
|
128
|
+
anchorRunId: string | null;
|
|
129
|
+
}): boolean;
|
|
130
|
+
/**
|
|
131
|
+
* Resolve a run id to its record. A run resolves only when it is a top-level
|
|
132
|
+
* run (`isTopLevelRun`): it owns a routing address and self-anchors. A child
|
|
133
|
+
* park row (address-null, anchored on its parent) is not served here.
|
|
134
|
+
*/
|
|
135
|
+
export declare function findRoutableById(db: DB["db"], id: string, tenantId: string): Promise<RoutableRecord | undefined>;
|