@intx/hub-sessions 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/dist/agent-repo.d.ts +14 -2
  2. package/dist/agent-repo.js +17 -4
  3. package/dist/agent-state-kind.js +14 -63
  4. package/dist/asset-service.js +14 -10
  5. package/dist/credential-push.d.ts +48 -4
  6. package/dist/credential-push.js +138 -6
  7. package/dist/event-collector-registry.d.ts +2 -1
  8. package/dist/event-collector-registry.js +38 -9
  9. package/dist/event-collector.d.ts +11 -1
  10. package/dist/event-collector.js +36 -3
  11. package/dist/hub-session-lookups.d.ts +1 -1
  12. package/dist/hub-session-lookups.js +68 -72
  13. package/dist/hub-session-orchestrator.d.ts +2 -3
  14. package/dist/hub-session-orchestrator.js +13 -12
  15. package/dist/index.d.ts +7 -6
  16. package/dist/index.js +7 -6
  17. package/dist/reconciliation-scheduler.d.ts +14 -0
  18. package/dist/reconciliation-scheduler.js +55 -0
  19. package/dist/repo-store/index.d.ts +1 -0
  20. package/dist/repo-store/index.js +1 -0
  21. package/dist/repo-store/user-principal-gate.d.ts +26 -0
  22. package/dist/repo-store/user-principal-gate.js +78 -0
  23. package/dist/session-service.d.ts +66 -121
  24. package/dist/session-service.js +444 -411
  25. package/dist/sidecar-allocation/capability-policy.d.ts +27 -0
  26. package/dist/sidecar-allocation/capability-policy.js +124 -0
  27. package/dist/sidecar-allocation/contracts.d.ts +29 -6
  28. package/dist/sidecar-allocation/contracts.js +7 -2
  29. package/dist/sidecar-allocation/index.d.ts +4 -3
  30. package/dist/sidecar-allocation/index.js +3 -2
  31. package/dist/sidecar-allocation/operation.d.ts +10 -0
  32. package/dist/sidecar-allocation/operation.js +54 -0
  33. package/dist/sidecar-allocation/plugin-registry.d.ts +16 -3
  34. package/dist/sidecar-allocation/plugin-registry.js +36 -12
  35. package/dist/sidecar-allocation/reconciler.d.ts +16 -4
  36. package/dist/sidecar-allocation/reconciler.js +486 -92
  37. package/dist/skill-kind.js +8 -62
  38. package/dist/substrate.d.ts +1 -1
  39. package/dist/substrate.js +1 -1
  40. package/dist/workflow-allocation-service.d.ts +21 -15
  41. package/dist/workflow-allocation-service.js +440 -125
  42. package/dist/workflow-dispatch-service.d.ts +4 -2
  43. package/dist/workflow-dispatch-service.js +89 -26
  44. package/dist/workflow-kind.d.ts +12 -0
  45. package/dist/workflow-kind.js +17 -60
  46. package/dist/workflow-probe-gate.d.ts +99 -27
  47. package/dist/workflow-probe-gate.js +196 -21
  48. package/dist/workflow-run-kind.d.ts +112 -19
  49. package/dist/workflow-run-kind.js +626 -210
  50. package/dist/workflow-run-restore.d.ts +1 -0
  51. package/dist/workflow-run-restore.js +5 -1
  52. package/dist/workflow-source-pins.d.ts +8 -0
  53. package/dist/workflow-source-pins.js +14 -0
  54. package/dist/ws/index.d.ts +1 -1
  55. package/dist/ws/index.js +1 -1
  56. package/dist/ws/pending-tracker.d.ts +93 -0
  57. package/dist/ws/pending-tracker.js +132 -0
  58. package/dist/ws/sidecar-events.d.ts +43 -29
  59. package/dist/ws/sidecar-events.js +0 -2
  60. package/dist/ws/sidecar-handler.d.ts +122 -85
  61. package/dist/ws/sidecar-handler.js +925 -878
  62. package/dist/ws/sidecar-handler.test-helpers.d.ts +38 -0
  63. package/dist/ws/sidecar-handler.test-helpers.js +95 -0
  64. package/dist/ws/sidecar-token-authenticator.js +37 -23
  65. package/package.json +13 -13
  66. package/dist/sidecar-allocation/placement-policy.d.ts +0 -11
  67. package/dist/sidecar-allocation/placement-policy.js +0 -21
@@ -66,8 +66,6 @@ export type AgentRepoStore = {
66
66
  * DB state without re-deriving terminal-ness.
67
67
  */
68
68
  receiveWorkflowRunPack(repoId: RepoId, pack: Uint8Array, ref: string, commitSha: string): Promise<NewlyTerminalRun[]>;
69
- /** Resolve the current deploy ref SHA, or null if no deploy exists. */
70
- getDeployRef(agentId: string): Promise<string | null>;
71
69
  /** Raw 32-byte Ed25519 public key used to sign deploy commits. */
72
70
  getSigningPublicKey(): Uint8Array;
73
71
  /**
@@ -78,6 +76,20 @@ export type AgentRepoStore = {
78
76
  */
79
77
  readonly repoStore: RepoStore;
80
78
  };
79
+ /**
80
+ * Repo kinds eligible for write-path object GC. Deliberately EXCLUDES
81
+ * "workflow-run". The warm-agent mailbox physically expunges `<uid>.eml`
82
+ * blobs from the live tree; the raw bytes then persist only through the
83
+ * parent commit in git history, and they survive a push ONLY because a
84
+ * `workflow-run` repo's objects are never GC'd. Adding "workflow-run"
85
+ * here -- above all with a retention other than "keep-history" -- would
86
+ * make an expunged message's bytes prunable and silently destroy the
87
+ * mailbox audit trail. The mailbox subtree contract in `workflow-run-kind`
88
+ * documents this dependency; `agent-repo.test.ts` pins it. Do not add
89
+ * "workflow-run" without replacing physical expunge with a tip-reachable
90
+ * retain-bytes scheme first.
91
+ */
92
+ export declare const DEFAULT_GC_KINDS: readonly ["agent-state"];
81
93
  export declare function createAgentRepoStore(config: {
82
94
  dataDir: string;
83
95
  signingKey: {
@@ -5,6 +5,20 @@ import { skillKindHandler, skillAuthorize } from "./skill-kind.js";
5
5
  import { packageRegistryKindHandler, packageRegistryAuthorize, } from "./package-registry-kind.js";
6
6
  import { workflowKindHandler, workflowAuthorize } from "./workflow-kind.js";
7
7
  import { workflowRunKindHandler, workflowRunAuthorize, } from "./workflow-run-kind.js";
8
+ /**
9
+ * Repo kinds eligible for write-path object GC. Deliberately EXCLUDES
10
+ * "workflow-run". The warm-agent mailbox physically expunges `<uid>.eml`
11
+ * blobs from the live tree; the raw bytes then persist only through the
12
+ * parent commit in git history, and they survive a push ONLY because a
13
+ * `workflow-run` repo's objects are never GC'd. Adding "workflow-run"
14
+ * here -- above all with a retention other than "keep-history" -- would
15
+ * make an expunged message's bytes prunable and silently destroy the
16
+ * mailbox audit trail. The mailbox subtree contract in `workflow-run-kind`
17
+ * documents this dependency; `agent-repo.test.ts` pins it. Do not add
18
+ * "workflow-run" without replacing physical expunge with a tip-reachable
19
+ * retain-bytes scheme first.
20
+ */
21
+ export const DEFAULT_GC_KINDS = ["agent-state"];
8
22
  export function createAgentRepoStore(config) {
9
23
  const { dataDir, signingKey, gc } = config;
10
24
  const authorize = (principal, incomingRepoId, ref, action) => {
@@ -47,7 +61,9 @@ export function createAgentRepoStore(config) {
47
61
  },
48
62
  authorize,
49
63
  signingCallback: () => signer,
50
- ...(gc === undefined ? {} : { gc: { kinds: ["agent-state"], ...gc } }),
64
+ ...(gc === undefined
65
+ ? {}
66
+ : { gc: { kinds: [...DEFAULT_GC_KINDS], ...gc } }),
51
67
  });
52
68
  const hub = { kind: "hub" };
53
69
  function repoId(agentId) {
@@ -98,9 +114,6 @@ export function createAgentRepoStore(config) {
98
114
  const expectedOldSha = await store.resolveRef(hub, incomingRepoId, ref);
99
115
  return store.receivePack(principal, incomingRepoId, ref, pack, commitSha, expectedOldSha);
100
116
  },
101
- async getDeployRef(agentId) {
102
- return store.resolveRef(hub, repoId(agentId), AGENT_STATE_DEPLOY_REF);
103
- },
104
117
  getSigningPublicKey() {
105
118
  return signingKey.publicKey;
106
119
  },
@@ -1,6 +1,5 @@
1
1
  import { type } from "arktype";
2
- import { glob, repoActionToGrantVerb } from "@intx/hub-common";
3
- import { UserPrincipal, } from "./repo-store/index.js";
2
+ import { authorizeUserPrincipal, } from "./repo-store/index.js";
4
3
  const SidecarPrincipal = type({
5
4
  kind: "'sidecar'",
6
5
  agentId: "string",
@@ -62,10 +61,9 @@ export const agentStateKindHandler = {
62
61
  };
63
62
  export const agentStateAuthorize = (principal, repoId, ref, action) => {
64
63
  if (principal.kind === "hub") {
65
- // Full access at this kind. Hub-side reads (getDeployRef,
66
- // createDeployPack) and writes (writeDeployTree) all flow through
67
- // here, so changing this branch tightens behavior in non-obvious
68
- // places.
64
+ // Full access at this kind. Hub-side reads (createDeployPack) and writes
65
+ // (writeDeployTree) all flow through here, so changing this branch tightens
66
+ // behavior in non-obvious places.
69
67
  return { allowed: true };
70
68
  }
71
69
  if (principal.kind === "sidecar") {
@@ -114,69 +112,22 @@ export const agentStateAuthorize = (principal, repoId, ref, action) => {
114
112
  }
115
113
  }
116
114
  if (principal.kind === "user") {
117
- // The route layer has already pre-resolved the grant verdict and
118
- // attached it as `authz`. The substrate does NOT re-query the
119
- // grant store here; it (a) checks the bearer-token's claims
120
- // bound the requested (ref, action) and have not expired, and
121
- // (b) sanity-checks that the pre-resolved verdict targets this
122
- // exact resource and grant verb. Both gates must pass before the
123
- // verdict's `effect` is honoured.
124
- const parsed = UserPrincipal(principal);
125
- if (parsed instanceof type.errors) {
126
- return {
127
- allowed: false,
128
- reason: `user principal is malformed: ${parsed.summary}`,
129
- };
130
- }
115
+ // `agentStateAuthorize` has no top-level repoId.kind gate (the
116
+ // other kind handlers reject a mismatched kind up front), so the
117
+ // kind check lives here inside the user branch.
131
118
  if (repoId.kind !== "agent-state") {
132
119
  return {
133
120
  allowed: false,
134
121
  reason: `user authorize received non-agent-state repo ${repoId.kind}/${repoId.id}`,
135
122
  };
136
123
  }
137
- if (!parsed.tokenClaims.actions.includes(action)) {
138
- return {
139
- allowed: false,
140
- reason: `token does not grant action ${action}`,
141
- };
142
- }
143
- // `ref === "*"` is the substrate's sentinel for the bulk read
144
- // performed by `listRefs`. Per-ref filtering is the advertise-refs
145
- // layer's responsibility, so the bulk read is gated on action and
146
- // expiry alone.
147
- if (ref !== "*" && !glob.match(parsed.tokenClaims.refPattern, ref)) {
148
- return {
149
- allowed: false,
150
- reason: `token refPattern ${parsed.tokenClaims.refPattern} does not match ${ref}`,
151
- };
152
- }
153
- if (Date.now() >= parsed.tokenClaims.expiresAt) {
154
- return {
155
- allowed: false,
156
- reason: `token expired at ${parsed.tokenClaims.expiresAt}`,
157
- };
158
- }
159
- const expectedResource = `agent-state:${repoId.id}`;
160
- if (parsed.authz.resource !== expectedResource) {
161
- return {
162
- allowed: false,
163
- reason: `authz verdict resource ${parsed.authz.resource} does not match ${expectedResource}`,
164
- };
165
- }
166
- const expectedGrantVerb = repoActionToGrantVerb(action);
167
- if (parsed.authz.grantVerb !== expectedGrantVerb) {
168
- return {
169
- allowed: false,
170
- reason: `authz verdict grantVerb ${parsed.authz.grantVerb} does not match ${expectedGrantVerb}`,
171
- };
172
- }
173
- if (parsed.authz.effect === "allow") {
174
- return { allowed: true };
175
- }
176
- return {
177
- allowed: false,
178
- reason: `authz verdict denied for ${expectedResource} ${expectedGrantVerb}`,
179
- };
124
+ return authorizeUserPrincipal({
125
+ principal,
126
+ repoId,
127
+ ref,
128
+ action,
129
+ resourcePrefix: "agent-state",
130
+ });
180
131
  }
181
132
  // Fail closed on any kind not handled above. The tenant-level
182
133
  // `workflow` principal kind (`@intx/types` principalKinds) is a
@@ -26,16 +26,20 @@ const logger = getLogger(["hub-sessions", "asset-service"]);
26
26
  * push to this ref so it carries the published-asset HEAD.
27
27
  */
28
28
  export const DEFAULT_ASSET_REF = "refs/heads/main";
29
- // Asset names become the default workspace mountpath segment at
30
- // session start (`skills/<asset.name>/`). The mountpath segment
31
- // validator in applyAssetPack rejects anything outside a safe
32
- // character set; validate at the createAsset boundary so a bad name
33
- // fails at creation time rather than at materialization time. Names
34
- // must be lowercase-kebab: lowercase letters, digits, hyphens, with
35
- // no leading or trailing hyphen. Exported so a caller deriving a name
36
- // (the agent-fold materializer) asserts the shape at its own boundary
37
- // rather than discovering a violation three layers in as a generic
38
- // `invalid_name`.
29
+ // An asset name becomes a workspace mountpath segment for the asset
30
+ // kinds a session actually mounts: a tool-package registry mounts at
31
+ // `package-registries/<asset.name>/`. A skill is an asset kind too,
32
+ // and its `SKILL.md` frontmatter is validated on push and indexed
33
+ // under the pushed ref, but nothing consumes that index and no
34
+ // session path mounts a skill asset into an agent workspace. A skill
35
+ // asset is therefore recorded and validated, not materialized and not
36
+ // executed. The mountpath segment validator in applyAssetPack
37
+ // rejects anything outside a safe character set; validate at the
38
+ // createAsset boundary so a bad name fails at creation time rather
39
+ // than at materialization time. Names must be lowercase-kebab:
40
+ // lowercase letters, digits, hyphens, with no leading or trailing
41
+ // hyphen. The sole enforcement site is the createAsset check below,
42
+ // which rejects a bad name as `invalid_name`.
39
43
  export const ASSET_NAME_PATTERN = /^[a-z0-9]+(?:-[a-z0-9]+)*$/;
40
44
  export class AssetServiceError extends Error {
41
45
  reason;
@@ -11,23 +11,67 @@ import type { SidecarRouter } from "./ws/sidecar-handler.js";
11
11
  * No-op when the instance resolves to no launchable source — the resolver's
12
12
  * own logger is the signal for why.
13
13
  */
14
- export declare function pushInstanceSourceUpdate(db: DB["db"], sidecarRouter: Pick<SidecarRouter, "sendSourcesUpdate">, instance: {
14
+ export declare function pushInstanceSourceUpdate(db: DB["db"], sidecarRouter: Pick<SidecarRouter, "sendSourcesUpdate" | "sendCredentialsUpdate">, instance: {
15
15
  address: string;
16
16
  definitionId: string;
17
17
  tenantId: string;
18
18
  modelPreferences: unknown;
19
- }, credentialCipher?: CredentialCipher): Promise<void>;
19
+ }, credentialCipher: CredentialCipher): Promise<void>;
20
20
  /**
21
21
  * After a credential secret is rotated, re-resolve every running instance in
22
22
  * the tenant against the catalog and push the updates. A rotated secret flows
23
23
  * through because resolution dereferences the provider's credential reference
24
24
  * to the current secret.
25
25
  */
26
- export declare function pushSourceUpdates(db: DB["db"], sidecarRouter: SidecarRouter, tenantId: string, credentialCipher?: CredentialCipher): Promise<void>;
26
+ export declare function pushSourceUpdates(db: DB["db"], sidecarRouter: SidecarRouter, tenantId: string, credentialCipher: CredentialCipher): Promise<void>;
27
27
  /**
28
28
  * After a catalog edit in a tenant, re-resolve and push to every running
29
29
  * instance in that tenant AND its descendants. Descendants inherit the
30
30
  * edited tenant's catalog, so a change there (a disabled provider, a new
31
31
  * offering, a price update) alters their resolved sources too.
32
32
  */
33
- export declare function pushSourceUpdatesSubtree(db: DB["db"], sidecarRouter: SidecarRouter, tenantId: string, credentialCipher?: CredentialCipher): Promise<void>;
33
+ export declare function pushSourceUpdatesSubtree(db: DB["db"], sidecarRouter: SidecarRouter, tenantId: string, credentialCipher: CredentialCipher): Promise<void>;
34
+ /**
35
+ * After a credential is deleted or deliberately revoked, evict it from every
36
+ * running deployment in the tenant AND its descendants. A descendant resolves
37
+ * an ancestor's tenant-owned credential through the tenant walk-up, so a
38
+ * revoke in one tenant can affect a descendant's run. The push is a flat named
39
+ * revocation: the child drops the credentialId's material and any binding that
40
+ * references it, and a run that never held it no-ops. Because a flat revoke is
41
+ * safe to broadcast, this needs no per-instance ledger of what was delivered.
42
+ *
43
+ * Callers fire this without awaiting, so it must never reject: a failure to
44
+ * enumerate or push is logged and dropped. This closes the ONLINE revocation
45
+ * window (a running deployment stops holding the credential now); the offline
46
+ * window -- a run whose sidecar was disconnected when the revoke fired -- is
47
+ * closed by the reconnect resync, not here.
48
+ */
49
+ export declare function pushCredentialRevoke(db: DB["db"], sidecarRouter: Pick<SidecarRouter, "sendCredentialsUpdate">, tenantId: string, credentialId: string): Promise<void>;
50
+ /**
51
+ * Reconcile a reconnecting deployment's credentials against its deploy-time set.
52
+ * Re-resolve the CURRENT material for every credentialId the deployment
53
+ * persisted at deploy (`workflow_run.credentialRefs`), then push a MERGE that
54
+ * upserts the survivors (picking up a same-id secret rotation) and REVOKES the
55
+ * deploy-time ids that no longer resolve (deleted or revoked while the sidecar
56
+ * was disconnected). Closes the OFFLINE revocation window (the online window is
57
+ * closed by `pushCredentialRevoke`).
58
+ *
59
+ * Merge, not wholesale-replace: `credentialRefs` is only the deploy-time id set,
60
+ * not the child's complete live set (a catalog re-point can deliver a new
61
+ * credential online), so a replace would evict online-added credentials. The
62
+ * merge upserts survivors and names the dead ids in `revoke`, leaving online
63
+ * credentials untouched. It does NOT handle an id-CHANGING rotation of a
64
+ * deploy-time source (the new id is not in `credentialRefs`); a later source
65
+ * push delivers that.
66
+ *
67
+ * No-op when the run persisted no credential refs (a folded run, or a
68
+ * deployment with no credentials). Fire-and-forget from the reconnect handler:
69
+ * it never rejects. A live-but-unresolvable credential (its provider vanished
70
+ * or has no API base URL) makes `reresolveCurrentMaterials` throw, which aborts
71
+ * the WHOLE reconcile (logged, not sent) so a partial set with a spurious
72
+ * revoke never lands. This is deliberately all-or-nothing: one misconfigured
73
+ * credential blocks this reconnect's revocation of the others too, trading
74
+ * revocation timeliness for never falsely evicting a live credential. The next
75
+ * reconnect (or an online revoke) retries.
76
+ */
77
+ export declare function pushCredentialReconcile(db: DB["db"], sidecarRouter: Pick<SidecarRouter, "sendCredentialsUpdate">, agentAddress: string, credentialCipher: CredentialCipher): Promise<void>;
@@ -1,13 +1,20 @@
1
- // Shared logic for re-resolving a running instance's inference sources from
2
- // the catalog and pushing the update to its sidecar.
1
+ // Hub-side producers that push credential-material changes to running
2
+ // deployments over `credentials.update`.
3
3
  //
4
- // Used after a credential secret rotation (a model provider's credential
5
- // changes the resolved source's apiKey) and after a catalog edit in a tenant
6
- // or its subtree.
4
+ // Two shapes, one per removal semantic:
5
+ // - Source re-resolve (`pushSourceUpdates`, `pushSourceUpdatesSubtree`): after
6
+ // a credential secret rotation or a catalog edit, re-resolve each running
7
+ // instance's inference sources and push the refreshed material followed by
8
+ // the `sources.update` that references it. An inference source references
9
+ // its credential by id only, so the rotated secret rides the cell.
10
+ // - Flat named revoke (`pushCredentialRevoke`): after a credential is deleted
11
+ // or deliberately revoked, broadcast a `revoke` naming that credentialId so
12
+ // every running deployment drops it. The removed id is NAMED by the actor,
13
+ // so this needs no diff against a prior delivery.
7
14
  import { eq, and, inArray, isNull, isNotNull } from "drizzle-orm";
8
15
  import { getLogger } from "@intx/log";
9
16
  import { workflowRun } from "@intx/db/schema";
10
- import { resolveInstanceModelSources, getDescendantTenants } from "@intx/db";
17
+ import { resolveInstanceModelSources, getDescendantTenants, reresolveCurrentMaterials, } from "@intx/db";
11
18
  const log = getLogger(["hub", "credentials"]);
12
19
  /**
13
20
  * Re-resolve a single running instance's inference sources from the catalog
@@ -26,6 +33,18 @@ export async function pushInstanceSourceUpdate(db, sidecarRouter, instance, cred
26
33
  const [head] = resolution.sources;
27
34
  if (head === undefined)
28
35
  return;
36
+ // Push the credential material before the source list. A source references
37
+ // its credential by id, so the cell must hold the (possibly rotated) secret
38
+ // before the source list that points at it lands. Inference sources carry no
39
+ // binding descriptor. A failure here propagates and aborts the source push --
40
+ // never a stale secret paired with a fresh source list.
41
+ if (resolution.materials.length > 0) {
42
+ const delivery = {
43
+ bindings: [],
44
+ materials: resolution.materials,
45
+ };
46
+ await sidecarRouter.sendCredentialsUpdate(instance.address, delivery);
47
+ }
29
48
  await sidecarRouter.sendSourcesUpdate(instance.address, resolution.sources, head.id);
30
49
  }
31
50
  /**
@@ -64,6 +83,9 @@ async function pushSourceUpdatesToTenants(db, sidecarRouter, tenantIds, credenti
64
83
  if (instance.address === null) {
65
84
  throw new Error(`running run ${instance.id} matched the non-null-address filter but has a null address`);
66
85
  }
86
+ if (instance.definitionId === null) {
87
+ throw new Error(`running run ${instance.id} has no workflow definition`);
88
+ }
67
89
  return pushInstanceSourceUpdate(db, sidecarRouter, {
68
90
  address: instance.address,
69
91
  definitionId: instance.definitionId,
@@ -107,3 +129,113 @@ export async function pushSourceUpdatesSubtree(db, sidecarRouter, tenantId, cred
107
129
  }
108
130
  await pushSourceUpdatesToTenants(db, sidecarRouter, tenants, credentialCipher);
109
131
  }
132
+ /**
133
+ * After a credential is deleted or deliberately revoked, evict it from every
134
+ * running deployment in the tenant AND its descendants. A descendant resolves
135
+ * an ancestor's tenant-owned credential through the tenant walk-up, so a
136
+ * revoke in one tenant can affect a descendant's run. The push is a flat named
137
+ * revocation: the child drops the credentialId's material and any binding that
138
+ * references it, and a run that never held it no-ops. Because a flat revoke is
139
+ * safe to broadcast, this needs no per-instance ledger of what was delivered.
140
+ *
141
+ * Callers fire this without awaiting, so it must never reject: a failure to
142
+ * enumerate or push is logged and dropped. This closes the ONLINE revocation
143
+ * window (a running deployment stops holding the credential now); the offline
144
+ * window -- a run whose sidecar was disconnected when the revoke fired -- is
145
+ * closed by the reconnect resync, not here.
146
+ */
147
+ export async function pushCredentialRevoke(db, sidecarRouter, tenantId, credentialId) {
148
+ let tenants;
149
+ try {
150
+ tenants = await getDescendantTenants(db, tenantId);
151
+ }
152
+ catch (err) {
153
+ log.warn `Failed to enumerate descendants for credential revoke: ${String(err)}`;
154
+ return;
155
+ }
156
+ try {
157
+ // Every running, addressable run in the subtree. Unlike the source-update
158
+ // push this does NOT exclude deployment-anchor runs (`anchorRunId IS NULL`):
159
+ // a deployed workflow is the primary credential consumer, and a flat revoke
160
+ // is safe to deliver to any address -- a run that never held the credential
161
+ // no-ops on it.
162
+ const runs = await db.query.workflowRun.findMany({
163
+ where: and(inArray(workflowRun.tenantId, tenants), eq(workflowRun.status, "running"), isNotNull(workflowRun.address)),
164
+ columns: { id: true, address: true },
165
+ });
166
+ const addresses = new Set();
167
+ for (const run of runs) {
168
+ // The isNotNull(address) filter guarantees a value; a null here is a
169
+ // broken invariant, surfaced rather than silently skipped.
170
+ if (run.address === null) {
171
+ throw new Error(`running run ${run.id} matched the non-null-address filter but has a null address`);
172
+ }
173
+ addresses.add(run.address);
174
+ }
175
+ if (addresses.size === 0)
176
+ return;
177
+ const emptyDelivery = { bindings: [], materials: [] };
178
+ const results = await Promise.allSettled([...addresses].map((address) => sidecarRouter.sendCredentialsUpdate(address, emptyDelivery, [
179
+ credentialId,
180
+ ])));
181
+ for (const result of results) {
182
+ if (result.status === "rejected") {
183
+ log.warn `Failed to push credential revoke: ${String(result.reason)}`;
184
+ }
185
+ }
186
+ }
187
+ catch (err) {
188
+ log.warn `Failed to push credential revoke: ${String(err)}`;
189
+ }
190
+ }
191
+ /**
192
+ * Reconcile a reconnecting deployment's credentials against its deploy-time set.
193
+ * Re-resolve the CURRENT material for every credentialId the deployment
194
+ * persisted at deploy (`workflow_run.credentialRefs`), then push a MERGE that
195
+ * upserts the survivors (picking up a same-id secret rotation) and REVOKES the
196
+ * deploy-time ids that no longer resolve (deleted or revoked while the sidecar
197
+ * was disconnected). Closes the OFFLINE revocation window (the online window is
198
+ * closed by `pushCredentialRevoke`).
199
+ *
200
+ * Merge, not wholesale-replace: `credentialRefs` is only the deploy-time id set,
201
+ * not the child's complete live set (a catalog re-point can deliver a new
202
+ * credential online), so a replace would evict online-added credentials. The
203
+ * merge upserts survivors and names the dead ids in `revoke`, leaving online
204
+ * credentials untouched. It does NOT handle an id-CHANGING rotation of a
205
+ * deploy-time source (the new id is not in `credentialRefs`); a later source
206
+ * push delivers that.
207
+ *
208
+ * No-op when the run persisted no credential refs (a folded run, or a
209
+ * deployment with no credentials). Fire-and-forget from the reconnect handler:
210
+ * it never rejects. A live-but-unresolvable credential (its provider vanished
211
+ * or has no API base URL) makes `reresolveCurrentMaterials` throw, which aborts
212
+ * the WHOLE reconcile (logged, not sent) so a partial set with a spurious
213
+ * revoke never lands. This is deliberately all-or-nothing: one misconfigured
214
+ * credential blocks this reconnect's revocation of the others too, trading
215
+ * revocation timeliness for never falsely evicting a live credential. The next
216
+ * reconnect (or an online revoke) retries.
217
+ */
218
+ export async function pushCredentialReconcile(db, sidecarRouter, agentAddress, credentialCipher) {
219
+ try {
220
+ const run = await db.query.workflowRun.findFirst({
221
+ where: eq(workflowRun.address, agentAddress),
222
+ columns: { credentialRefs: true },
223
+ });
224
+ if (run === undefined)
225
+ return;
226
+ const refs = run.credentialRefs;
227
+ if (refs === null)
228
+ return;
229
+ const materials = await reresolveCurrentMaterials(db, refs.credentialIds, credentialCipher);
230
+ const resolvedIds = new Set(materials.map((material) => material.credentialId));
231
+ // Deploy-time ids that no longer resolve: deleted or revoked while offline.
232
+ const revoke = refs.credentialIds.filter((id) => !resolvedIds.has(id));
233
+ // A binding whose credential dropped out of the re-resolution goes with it.
234
+ const bindings = refs.bindings.filter((binding) => resolvedIds.has(binding.credentialId));
235
+ const delivery = { bindings, materials };
236
+ await sidecarRouter.sendCredentialsUpdate(agentAddress, delivery, revoke.length > 0 ? revoke : undefined);
237
+ }
238
+ catch (err) {
239
+ log.warn `Failed to reconcile credentials for ${agentAddress}: ${String(err)}`;
240
+ }
241
+ }
@@ -1,7 +1,7 @@
1
1
  import type { DB } from "@intx/db";
2
2
  import type { InferenceEvent } from "@intx/types/runtime";
3
3
  import type { SessionStatus } from "@intx/types";
4
- import { type TurnFinalized } from "./event-collector.js";
4
+ import { type TurnFinalized, type TurnUsage } from "./event-collector.js";
5
5
  export type EventCollectorRegistry = {
6
6
  create(agentAddress: string, tenantId: string, sessionId: string, runId: string): void;
7
7
  dispatch(agentAddress: string, event: InferenceEvent): void;
@@ -15,6 +15,7 @@ export type EventCollectorRegistry = {
15
15
  export type EventCollectorRegistryConfig = {
16
16
  db: DB["db"];
17
17
  onTurnFinalized?: (agentAddress: string, turn: TurnFinalized) => void;
18
+ onUsage?: (agentAddress: string, usage: TurnUsage) => void;
18
19
  };
19
20
  export declare function deriveStatus(event: InferenceEvent): SessionStatus | null;
20
21
  export declare function createEventCollectorRegistry(config: EventCollectorRegistryConfig): EventCollectorRegistry;
@@ -30,9 +30,12 @@ export function deriveStatus(event) {
30
30
  }
31
31
  }
32
32
  export function createEventCollectorRegistry(config) {
33
- const { db, onTurnFinalized } = config;
33
+ const { db, onTurnFinalized, onUsage } = config;
34
34
  const collectors = new Map();
35
35
  const statuses = new Map();
36
+ // Per-address tail promise: serializes onEvent/abandon work for one run
37
+ // address so their DB writes cannot interleave. Reaped when it drains.
38
+ const tails = new Map();
36
39
  function create(agentAddress, tenantId, sessionId, runId) {
37
40
  if (collectors.has(agentAddress)) {
38
41
  log.warn `Collector already exists for ${agentAddress}, replacing`;
@@ -48,6 +51,11 @@ export function createEventCollectorRegistry(config) {
48
51
  onTurnFinalized: (turn) => onTurnFinalized(agentAddress, turn),
49
52
  }
50
53
  : {}),
54
+ ...(onUsage
55
+ ? {
56
+ onUsage: (usage) => onUsage(agentAddress, usage),
57
+ }
58
+ : {}),
51
59
  });
52
60
  collectors.set(agentAddress, collector);
53
61
  statuses.set(agentAddress, { status: "idle" });
@@ -56,6 +64,25 @@ export function createEventCollectorRegistry(config) {
56
64
  collectors.delete(agentAddress);
57
65
  statuses.delete(agentAddress);
58
66
  }
67
+ // Chain `work` onto the address's tail so per-address work runs in order and
68
+ // never interleaves, while the caller stays non-blocking. `onError` swallows
69
+ // a failure so one bad event cannot wedge the chain; `onSettled` runs after
70
+ // the work settles. The tail entry is reaped once no later work is queued.
71
+ function enqueue(agentAddress, work, onError, onSettled) {
72
+ const prev = tails.get(agentAddress) ?? Promise.resolve();
73
+ const next = prev
74
+ .then(work)
75
+ .catch(onError)
76
+ .finally(() => {
77
+ if (onSettled !== undefined) {
78
+ onSettled();
79
+ }
80
+ if (tails.get(agentAddress) === next) {
81
+ tails.delete(agentAddress);
82
+ }
83
+ });
84
+ tails.set(agentAddress, next);
85
+ }
59
86
  function dispatch(agentAddress, event) {
60
87
  const collector = collectors.get(agentAddress);
61
88
  if (collector === undefined) {
@@ -67,13 +94,10 @@ export function createEventCollectorRegistry(config) {
67
94
  }
68
95
  const isTerminal = event.type === "reactor.done" ||
69
96
  (event.type === "reactor.error" && event.data.fatal);
70
- collector
71
- .onEvent(event)
72
- .catch((err) => {
97
+ enqueue(agentAddress, () => collector.onEvent(event), (err) => {
73
98
  log.warn `Failed to persist event ${event.type} seq=${String(event.seq)} for ${agentAddress}: ${err instanceof Error ? err.message : String(err)}`;
74
- })
75
- .finally(() => {
76
- if (isTerminal) {
99
+ }, () => {
100
+ if (isTerminal && collectors.get(agentAddress) === collector) {
77
101
  removeCollector(agentAddress);
78
102
  }
79
103
  });
@@ -82,10 +106,15 @@ export function createEventCollectorRegistry(config) {
82
106
  const collector = collectors.get(agentAddress);
83
107
  if (collector === undefined)
84
108
  return;
85
- collector.abandon().catch((err) => {
109
+ // Stop NEW dispatches immediately; the queued closures keep their own
110
+ // reference so already-queued events still drain before the abandon runs.
111
+ removeCollector(agentAddress);
112
+ // Chain the abandon onto the tail so it runs AFTER any queued onEvents
113
+ // instead of racing them. Otherwise a queued beginTurn could create a
114
+ // fresh `running` turn row after the collector was finalized, orphaning it.
115
+ enqueue(agentAddress, () => collector.abandon(), (err) => {
86
116
  log.warn `Failed to abandon collector for ${agentAddress}: ${err instanceof Error ? err.message : String(err)}`;
87
117
  });
88
- removeCollector(agentAddress);
89
118
  }
90
119
  function has(agentAddress) {
91
120
  return collectors.has(agentAddress);
@@ -1,4 +1,4 @@
1
- import type { InferenceEvent } from "@intx/types/runtime";
1
+ import type { InferenceEvent, TokenUsage } from "@intx/types/runtime";
2
2
  import { type DB } from "@intx/db";
3
3
  export type TurnToolCall = {
4
4
  name: string;
@@ -22,6 +22,15 @@ export type TurnFinalized = {
22
22
  content: string;
23
23
  }[];
24
24
  };
25
+ export type TurnUsage = {
26
+ tenantId: string;
27
+ sessionId: string;
28
+ runId: string;
29
+ turnId: string;
30
+ provider: string;
31
+ model: string;
32
+ usage: TokenUsage;
33
+ };
25
34
  export type EventCollector = {
26
35
  onEvent(event: InferenceEvent): Promise<void>;
27
36
  abandon(): Promise<void>;
@@ -35,5 +44,6 @@ export type EventCollectorConfig = {
35
44
  runId: string;
36
45
  tenantId: string;
37
46
  onTurnFinalized?: (turn: TurnFinalized) => void;
47
+ onUsage?: (usage: TurnUsage) => void;
38
48
  };
39
49
  export declare function createEventCollector(config: EventCollectorConfig): EventCollector;