@intx/hub-sessions 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-repo.d.ts +14 -2
- package/dist/agent-repo.js +17 -4
- package/dist/agent-state-kind.js +14 -63
- package/dist/asset-service.js +14 -10
- package/dist/credential-push.d.ts +48 -4
- package/dist/credential-push.js +138 -6
- package/dist/event-collector-registry.d.ts +2 -1
- package/dist/event-collector-registry.js +38 -9
- package/dist/event-collector.d.ts +11 -1
- package/dist/event-collector.js +36 -3
- package/dist/hub-session-lookups.d.ts +1 -1
- package/dist/hub-session-lookups.js +68 -72
- package/dist/hub-session-orchestrator.d.ts +2 -3
- package/dist/hub-session-orchestrator.js +13 -12
- package/dist/index.d.ts +7 -6
- package/dist/index.js +7 -6
- package/dist/reconciliation-scheduler.d.ts +14 -0
- package/dist/reconciliation-scheduler.js +55 -0
- package/dist/repo-store/index.d.ts +1 -0
- package/dist/repo-store/index.js +1 -0
- package/dist/repo-store/user-principal-gate.d.ts +26 -0
- package/dist/repo-store/user-principal-gate.js +78 -0
- package/dist/session-service.d.ts +66 -121
- package/dist/session-service.js +444 -411
- package/dist/sidecar-allocation/capability-policy.d.ts +27 -0
- package/dist/sidecar-allocation/capability-policy.js +124 -0
- package/dist/sidecar-allocation/contracts.d.ts +29 -6
- package/dist/sidecar-allocation/contracts.js +7 -2
- package/dist/sidecar-allocation/index.d.ts +4 -3
- package/dist/sidecar-allocation/index.js +3 -2
- package/dist/sidecar-allocation/operation.d.ts +10 -0
- package/dist/sidecar-allocation/operation.js +54 -0
- package/dist/sidecar-allocation/plugin-registry.d.ts +16 -3
- package/dist/sidecar-allocation/plugin-registry.js +36 -12
- package/dist/sidecar-allocation/reconciler.d.ts +16 -4
- package/dist/sidecar-allocation/reconciler.js +486 -92
- package/dist/skill-kind.js +8 -62
- package/dist/substrate.d.ts +1 -1
- package/dist/substrate.js +1 -1
- package/dist/workflow-allocation-service.d.ts +21 -15
- package/dist/workflow-allocation-service.js +440 -125
- package/dist/workflow-dispatch-service.d.ts +4 -2
- package/dist/workflow-dispatch-service.js +89 -26
- package/dist/workflow-kind.d.ts +12 -0
- package/dist/workflow-kind.js +17 -60
- package/dist/workflow-probe-gate.d.ts +99 -27
- package/dist/workflow-probe-gate.js +196 -21
- package/dist/workflow-run-kind.d.ts +112 -19
- package/dist/workflow-run-kind.js +626 -210
- package/dist/workflow-run-restore.d.ts +1 -0
- package/dist/workflow-run-restore.js +5 -1
- package/dist/workflow-source-pins.d.ts +8 -0
- package/dist/workflow-source-pins.js +14 -0
- package/dist/ws/index.d.ts +1 -1
- package/dist/ws/index.js +1 -1
- package/dist/ws/pending-tracker.d.ts +93 -0
- package/dist/ws/pending-tracker.js +132 -0
- package/dist/ws/sidecar-events.d.ts +43 -29
- package/dist/ws/sidecar-events.js +0 -2
- package/dist/ws/sidecar-handler.d.ts +122 -85
- package/dist/ws/sidecar-handler.js +925 -878
- package/dist/ws/sidecar-handler.test-helpers.d.ts +38 -0
- package/dist/ws/sidecar-handler.test-helpers.js +95 -0
- package/dist/ws/sidecar-token-authenticator.js +37 -23
- package/package.json +13 -13
- package/dist/sidecar-allocation/placement-policy.d.ts +0 -11
- package/dist/sidecar-allocation/placement-policy.js +0 -21
package/dist/agent-repo.d.ts
CHANGED
|
@@ -66,8 +66,6 @@ export type AgentRepoStore = {
|
|
|
66
66
|
* DB state without re-deriving terminal-ness.
|
|
67
67
|
*/
|
|
68
68
|
receiveWorkflowRunPack(repoId: RepoId, pack: Uint8Array, ref: string, commitSha: string): Promise<NewlyTerminalRun[]>;
|
|
69
|
-
/** Resolve the current deploy ref SHA, or null if no deploy exists. */
|
|
70
|
-
getDeployRef(agentId: string): Promise<string | null>;
|
|
71
69
|
/** Raw 32-byte Ed25519 public key used to sign deploy commits. */
|
|
72
70
|
getSigningPublicKey(): Uint8Array;
|
|
73
71
|
/**
|
|
@@ -78,6 +76,20 @@ export type AgentRepoStore = {
|
|
|
78
76
|
*/
|
|
79
77
|
readonly repoStore: RepoStore;
|
|
80
78
|
};
|
|
79
|
+
/**
|
|
80
|
+
* Repo kinds eligible for write-path object GC. Deliberately EXCLUDES
|
|
81
|
+
* "workflow-run". The warm-agent mailbox physically expunges `<uid>.eml`
|
|
82
|
+
* blobs from the live tree; the raw bytes then persist only through the
|
|
83
|
+
* parent commit in git history, and they survive a push ONLY because a
|
|
84
|
+
* `workflow-run` repo's objects are never GC'd. Adding "workflow-run"
|
|
85
|
+
* here -- above all with a retention other than "keep-history" -- would
|
|
86
|
+
* make an expunged message's bytes prunable and silently destroy the
|
|
87
|
+
* mailbox audit trail. The mailbox subtree contract in `workflow-run-kind`
|
|
88
|
+
* documents this dependency; `agent-repo.test.ts` pins it. Do not add
|
|
89
|
+
* "workflow-run" without replacing physical expunge with a tip-reachable
|
|
90
|
+
* retain-bytes scheme first.
|
|
91
|
+
*/
|
|
92
|
+
export declare const DEFAULT_GC_KINDS: readonly ["agent-state"];
|
|
81
93
|
export declare function createAgentRepoStore(config: {
|
|
82
94
|
dataDir: string;
|
|
83
95
|
signingKey: {
|
package/dist/agent-repo.js
CHANGED
|
@@ -5,6 +5,20 @@ import { skillKindHandler, skillAuthorize } from "./skill-kind.js";
|
|
|
5
5
|
import { packageRegistryKindHandler, packageRegistryAuthorize, } from "./package-registry-kind.js";
|
|
6
6
|
import { workflowKindHandler, workflowAuthorize } from "./workflow-kind.js";
|
|
7
7
|
import { workflowRunKindHandler, workflowRunAuthorize, } from "./workflow-run-kind.js";
|
|
8
|
+
/**
|
|
9
|
+
* Repo kinds eligible for write-path object GC. Deliberately EXCLUDES
|
|
10
|
+
* "workflow-run". The warm-agent mailbox physically expunges `<uid>.eml`
|
|
11
|
+
* blobs from the live tree; the raw bytes then persist only through the
|
|
12
|
+
* parent commit in git history, and they survive a push ONLY because a
|
|
13
|
+
* `workflow-run` repo's objects are never GC'd. Adding "workflow-run"
|
|
14
|
+
* here -- above all with a retention other than "keep-history" -- would
|
|
15
|
+
* make an expunged message's bytes prunable and silently destroy the
|
|
16
|
+
* mailbox audit trail. The mailbox subtree contract in `workflow-run-kind`
|
|
17
|
+
* documents this dependency; `agent-repo.test.ts` pins it. Do not add
|
|
18
|
+
* "workflow-run" without replacing physical expunge with a tip-reachable
|
|
19
|
+
* retain-bytes scheme first.
|
|
20
|
+
*/
|
|
21
|
+
export const DEFAULT_GC_KINDS = ["agent-state"];
|
|
8
22
|
export function createAgentRepoStore(config) {
|
|
9
23
|
const { dataDir, signingKey, gc } = config;
|
|
10
24
|
const authorize = (principal, incomingRepoId, ref, action) => {
|
|
@@ -47,7 +61,9 @@ export function createAgentRepoStore(config) {
|
|
|
47
61
|
},
|
|
48
62
|
authorize,
|
|
49
63
|
signingCallback: () => signer,
|
|
50
|
-
...(gc === undefined
|
|
64
|
+
...(gc === undefined
|
|
65
|
+
? {}
|
|
66
|
+
: { gc: { kinds: [...DEFAULT_GC_KINDS], ...gc } }),
|
|
51
67
|
});
|
|
52
68
|
const hub = { kind: "hub" };
|
|
53
69
|
function repoId(agentId) {
|
|
@@ -98,9 +114,6 @@ export function createAgentRepoStore(config) {
|
|
|
98
114
|
const expectedOldSha = await store.resolveRef(hub, incomingRepoId, ref);
|
|
99
115
|
return store.receivePack(principal, incomingRepoId, ref, pack, commitSha, expectedOldSha);
|
|
100
116
|
},
|
|
101
|
-
async getDeployRef(agentId) {
|
|
102
|
-
return store.resolveRef(hub, repoId(agentId), AGENT_STATE_DEPLOY_REF);
|
|
103
|
-
},
|
|
104
117
|
getSigningPublicKey() {
|
|
105
118
|
return signingKey.publicKey;
|
|
106
119
|
},
|
package/dist/agent-state-kind.js
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { type } from "arktype";
|
|
2
|
-
import {
|
|
3
|
-
import { UserPrincipal, } from "./repo-store/index.js";
|
|
2
|
+
import { authorizeUserPrincipal, } from "./repo-store/index.js";
|
|
4
3
|
const SidecarPrincipal = type({
|
|
5
4
|
kind: "'sidecar'",
|
|
6
5
|
agentId: "string",
|
|
@@ -62,10 +61,9 @@ export const agentStateKindHandler = {
|
|
|
62
61
|
};
|
|
63
62
|
export const agentStateAuthorize = (principal, repoId, ref, action) => {
|
|
64
63
|
if (principal.kind === "hub") {
|
|
65
|
-
// Full access at this kind. Hub-side reads (
|
|
66
|
-
//
|
|
67
|
-
//
|
|
68
|
-
// places.
|
|
64
|
+
// Full access at this kind. Hub-side reads (createDeployPack) and writes
|
|
65
|
+
// (writeDeployTree) all flow through here, so changing this branch tightens
|
|
66
|
+
// behavior in non-obvious places.
|
|
69
67
|
return { allowed: true };
|
|
70
68
|
}
|
|
71
69
|
if (principal.kind === "sidecar") {
|
|
@@ -114,69 +112,22 @@ export const agentStateAuthorize = (principal, repoId, ref, action) => {
|
|
|
114
112
|
}
|
|
115
113
|
}
|
|
116
114
|
if (principal.kind === "user") {
|
|
117
|
-
//
|
|
118
|
-
//
|
|
119
|
-
//
|
|
120
|
-
// bound the requested (ref, action) and have not expired, and
|
|
121
|
-
// (b) sanity-checks that the pre-resolved verdict targets this
|
|
122
|
-
// exact resource and grant verb. Both gates must pass before the
|
|
123
|
-
// verdict's `effect` is honoured.
|
|
124
|
-
const parsed = UserPrincipal(principal);
|
|
125
|
-
if (parsed instanceof type.errors) {
|
|
126
|
-
return {
|
|
127
|
-
allowed: false,
|
|
128
|
-
reason: `user principal is malformed: ${parsed.summary}`,
|
|
129
|
-
};
|
|
130
|
-
}
|
|
115
|
+
// `agentStateAuthorize` has no top-level repoId.kind gate (the
|
|
116
|
+
// other kind handlers reject a mismatched kind up front), so the
|
|
117
|
+
// kind check lives here inside the user branch.
|
|
131
118
|
if (repoId.kind !== "agent-state") {
|
|
132
119
|
return {
|
|
133
120
|
allowed: false,
|
|
134
121
|
reason: `user authorize received non-agent-state repo ${repoId.kind}/${repoId.id}`,
|
|
135
122
|
};
|
|
136
123
|
}
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
// performed by `listRefs`. Per-ref filtering is the advertise-refs
|
|
145
|
-
// layer's responsibility, so the bulk read is gated on action and
|
|
146
|
-
// expiry alone.
|
|
147
|
-
if (ref !== "*" && !glob.match(parsed.tokenClaims.refPattern, ref)) {
|
|
148
|
-
return {
|
|
149
|
-
allowed: false,
|
|
150
|
-
reason: `token refPattern ${parsed.tokenClaims.refPattern} does not match ${ref}`,
|
|
151
|
-
};
|
|
152
|
-
}
|
|
153
|
-
if (Date.now() >= parsed.tokenClaims.expiresAt) {
|
|
154
|
-
return {
|
|
155
|
-
allowed: false,
|
|
156
|
-
reason: `token expired at ${parsed.tokenClaims.expiresAt}`,
|
|
157
|
-
};
|
|
158
|
-
}
|
|
159
|
-
const expectedResource = `agent-state:${repoId.id}`;
|
|
160
|
-
if (parsed.authz.resource !== expectedResource) {
|
|
161
|
-
return {
|
|
162
|
-
allowed: false,
|
|
163
|
-
reason: `authz verdict resource ${parsed.authz.resource} does not match ${expectedResource}`,
|
|
164
|
-
};
|
|
165
|
-
}
|
|
166
|
-
const expectedGrantVerb = repoActionToGrantVerb(action);
|
|
167
|
-
if (parsed.authz.grantVerb !== expectedGrantVerb) {
|
|
168
|
-
return {
|
|
169
|
-
allowed: false,
|
|
170
|
-
reason: `authz verdict grantVerb ${parsed.authz.grantVerb} does not match ${expectedGrantVerb}`,
|
|
171
|
-
};
|
|
172
|
-
}
|
|
173
|
-
if (parsed.authz.effect === "allow") {
|
|
174
|
-
return { allowed: true };
|
|
175
|
-
}
|
|
176
|
-
return {
|
|
177
|
-
allowed: false,
|
|
178
|
-
reason: `authz verdict denied for ${expectedResource} ${expectedGrantVerb}`,
|
|
179
|
-
};
|
|
124
|
+
return authorizeUserPrincipal({
|
|
125
|
+
principal,
|
|
126
|
+
repoId,
|
|
127
|
+
ref,
|
|
128
|
+
action,
|
|
129
|
+
resourcePrefix: "agent-state",
|
|
130
|
+
});
|
|
180
131
|
}
|
|
181
132
|
// Fail closed on any kind not handled above. The tenant-level
|
|
182
133
|
// `workflow` principal kind (`@intx/types` principalKinds) is a
|
package/dist/asset-service.js
CHANGED
|
@@ -26,16 +26,20 @@ const logger = getLogger(["hub-sessions", "asset-service"]);
|
|
|
26
26
|
* push to this ref so it carries the published-asset HEAD.
|
|
27
27
|
*/
|
|
28
28
|
export const DEFAULT_ASSET_REF = "refs/heads/main";
|
|
29
|
-
//
|
|
30
|
-
// session
|
|
31
|
-
//
|
|
32
|
-
//
|
|
33
|
-
//
|
|
34
|
-
//
|
|
35
|
-
//
|
|
36
|
-
//
|
|
37
|
-
//
|
|
38
|
-
//
|
|
29
|
+
// An asset name becomes a workspace mountpath segment for the asset
|
|
30
|
+
// kinds a session actually mounts: a tool-package registry mounts at
|
|
31
|
+
// `package-registries/<asset.name>/`. A skill is an asset kind too,
|
|
32
|
+
// and its `SKILL.md` frontmatter is validated on push and indexed
|
|
33
|
+
// under the pushed ref, but nothing consumes that index and no
|
|
34
|
+
// session path mounts a skill asset into an agent workspace. A skill
|
|
35
|
+
// asset is therefore recorded and validated, not materialized and not
|
|
36
|
+
// executed. The mountpath segment validator in applyAssetPack
|
|
37
|
+
// rejects anything outside a safe character set; validate at the
|
|
38
|
+
// createAsset boundary so a bad name fails at creation time rather
|
|
39
|
+
// than at materialization time. Names must be lowercase-kebab:
|
|
40
|
+
// lowercase letters, digits, hyphens, with no leading or trailing
|
|
41
|
+
// hyphen. The sole enforcement site is the createAsset check below,
|
|
42
|
+
// which rejects a bad name as `invalid_name`.
|
|
39
43
|
export const ASSET_NAME_PATTERN = /^[a-z0-9]+(?:-[a-z0-9]+)*$/;
|
|
40
44
|
export class AssetServiceError extends Error {
|
|
41
45
|
reason;
|
|
@@ -11,23 +11,67 @@ import type { SidecarRouter } from "./ws/sidecar-handler.js";
|
|
|
11
11
|
* No-op when the instance resolves to no launchable source — the resolver's
|
|
12
12
|
* own logger is the signal for why.
|
|
13
13
|
*/
|
|
14
|
-
export declare function pushInstanceSourceUpdate(db: DB["db"], sidecarRouter: Pick<SidecarRouter, "sendSourcesUpdate">, instance: {
|
|
14
|
+
export declare function pushInstanceSourceUpdate(db: DB["db"], sidecarRouter: Pick<SidecarRouter, "sendSourcesUpdate" | "sendCredentialsUpdate">, instance: {
|
|
15
15
|
address: string;
|
|
16
16
|
definitionId: string;
|
|
17
17
|
tenantId: string;
|
|
18
18
|
modelPreferences: unknown;
|
|
19
|
-
}, credentialCipher
|
|
19
|
+
}, credentialCipher: CredentialCipher): Promise<void>;
|
|
20
20
|
/**
|
|
21
21
|
* After a credential secret is rotated, re-resolve every running instance in
|
|
22
22
|
* the tenant against the catalog and push the updates. A rotated secret flows
|
|
23
23
|
* through because resolution dereferences the provider's credential reference
|
|
24
24
|
* to the current secret.
|
|
25
25
|
*/
|
|
26
|
-
export declare function pushSourceUpdates(db: DB["db"], sidecarRouter: SidecarRouter, tenantId: string, credentialCipher
|
|
26
|
+
export declare function pushSourceUpdates(db: DB["db"], sidecarRouter: SidecarRouter, tenantId: string, credentialCipher: CredentialCipher): Promise<void>;
|
|
27
27
|
/**
|
|
28
28
|
* After a catalog edit in a tenant, re-resolve and push to every running
|
|
29
29
|
* instance in that tenant AND its descendants. Descendants inherit the
|
|
30
30
|
* edited tenant's catalog, so a change there (a disabled provider, a new
|
|
31
31
|
* offering, a price update) alters their resolved sources too.
|
|
32
32
|
*/
|
|
33
|
-
export declare function pushSourceUpdatesSubtree(db: DB["db"], sidecarRouter: SidecarRouter, tenantId: string, credentialCipher
|
|
33
|
+
export declare function pushSourceUpdatesSubtree(db: DB["db"], sidecarRouter: SidecarRouter, tenantId: string, credentialCipher: CredentialCipher): Promise<void>;
|
|
34
|
+
/**
|
|
35
|
+
* After a credential is deleted or deliberately revoked, evict it from every
|
|
36
|
+
* running deployment in the tenant AND its descendants. A descendant resolves
|
|
37
|
+
* an ancestor's tenant-owned credential through the tenant walk-up, so a
|
|
38
|
+
* revoke in one tenant can affect a descendant's run. The push is a flat named
|
|
39
|
+
* revocation: the child drops the credentialId's material and any binding that
|
|
40
|
+
* references it, and a run that never held it no-ops. Because a flat revoke is
|
|
41
|
+
* safe to broadcast, this needs no per-instance ledger of what was delivered.
|
|
42
|
+
*
|
|
43
|
+
* Callers fire this without awaiting, so it must never reject: a failure to
|
|
44
|
+
* enumerate or push is logged and dropped. This closes the ONLINE revocation
|
|
45
|
+
* window (a running deployment stops holding the credential now); the offline
|
|
46
|
+
* window -- a run whose sidecar was disconnected when the revoke fired -- is
|
|
47
|
+
* closed by the reconnect resync, not here.
|
|
48
|
+
*/
|
|
49
|
+
export declare function pushCredentialRevoke(db: DB["db"], sidecarRouter: Pick<SidecarRouter, "sendCredentialsUpdate">, tenantId: string, credentialId: string): Promise<void>;
|
|
50
|
+
/**
|
|
51
|
+
* Reconcile a reconnecting deployment's credentials against its deploy-time set.
|
|
52
|
+
* Re-resolve the CURRENT material for every credentialId the deployment
|
|
53
|
+
* persisted at deploy (`workflow_run.credentialRefs`), then push a MERGE that
|
|
54
|
+
* upserts the survivors (picking up a same-id secret rotation) and REVOKES the
|
|
55
|
+
* deploy-time ids that no longer resolve (deleted or revoked while the sidecar
|
|
56
|
+
* was disconnected). Closes the OFFLINE revocation window (the online window is
|
|
57
|
+
* closed by `pushCredentialRevoke`).
|
|
58
|
+
*
|
|
59
|
+
* Merge, not wholesale-replace: `credentialRefs` is only the deploy-time id set,
|
|
60
|
+
* not the child's complete live set (a catalog re-point can deliver a new
|
|
61
|
+
* credential online), so a replace would evict online-added credentials. The
|
|
62
|
+
* merge upserts survivors and names the dead ids in `revoke`, leaving online
|
|
63
|
+
* credentials untouched. It does NOT handle an id-CHANGING rotation of a
|
|
64
|
+
* deploy-time source (the new id is not in `credentialRefs`); a later source
|
|
65
|
+
* push delivers that.
|
|
66
|
+
*
|
|
67
|
+
* No-op when the run persisted no credential refs (a folded run, or a
|
|
68
|
+
* deployment with no credentials). Fire-and-forget from the reconnect handler:
|
|
69
|
+
* it never rejects. A live-but-unresolvable credential (its provider vanished
|
|
70
|
+
* or has no API base URL) makes `reresolveCurrentMaterials` throw, which aborts
|
|
71
|
+
* the WHOLE reconcile (logged, not sent) so a partial set with a spurious
|
|
72
|
+
* revoke never lands. This is deliberately all-or-nothing: one misconfigured
|
|
73
|
+
* credential blocks this reconnect's revocation of the others too, trading
|
|
74
|
+
* revocation timeliness for never falsely evicting a live credential. The next
|
|
75
|
+
* reconnect (or an online revoke) retries.
|
|
76
|
+
*/
|
|
77
|
+
export declare function pushCredentialReconcile(db: DB["db"], sidecarRouter: Pick<SidecarRouter, "sendCredentialsUpdate">, agentAddress: string, credentialCipher: CredentialCipher): Promise<void>;
|
package/dist/credential-push.js
CHANGED
|
@@ -1,13 +1,20 @@
|
|
|
1
|
-
//
|
|
2
|
-
//
|
|
1
|
+
// Hub-side producers that push credential-material changes to running
|
|
2
|
+
// deployments over `credentials.update`.
|
|
3
3
|
//
|
|
4
|
-
//
|
|
5
|
-
//
|
|
6
|
-
// or
|
|
4
|
+
// Two shapes, one per removal semantic:
|
|
5
|
+
// - Source re-resolve (`pushSourceUpdates`, `pushSourceUpdatesSubtree`): after
|
|
6
|
+
// a credential secret rotation or a catalog edit, re-resolve each running
|
|
7
|
+
// instance's inference sources and push the refreshed material followed by
|
|
8
|
+
// the `sources.update` that references it. An inference source references
|
|
9
|
+
// its credential by id only, so the rotated secret rides the cell.
|
|
10
|
+
// - Flat named revoke (`pushCredentialRevoke`): after a credential is deleted
|
|
11
|
+
// or deliberately revoked, broadcast a `revoke` naming that credentialId so
|
|
12
|
+
// every running deployment drops it. The removed id is NAMED by the actor,
|
|
13
|
+
// so this needs no diff against a prior delivery.
|
|
7
14
|
import { eq, and, inArray, isNull, isNotNull } from "drizzle-orm";
|
|
8
15
|
import { getLogger } from "@intx/log";
|
|
9
16
|
import { workflowRun } from "@intx/db/schema";
|
|
10
|
-
import { resolveInstanceModelSources, getDescendantTenants } from "@intx/db";
|
|
17
|
+
import { resolveInstanceModelSources, getDescendantTenants, reresolveCurrentMaterials, } from "@intx/db";
|
|
11
18
|
const log = getLogger(["hub", "credentials"]);
|
|
12
19
|
/**
|
|
13
20
|
* Re-resolve a single running instance's inference sources from the catalog
|
|
@@ -26,6 +33,18 @@ export async function pushInstanceSourceUpdate(db, sidecarRouter, instance, cred
|
|
|
26
33
|
const [head] = resolution.sources;
|
|
27
34
|
if (head === undefined)
|
|
28
35
|
return;
|
|
36
|
+
// Push the credential material before the source list. A source references
|
|
37
|
+
// its credential by id, so the cell must hold the (possibly rotated) secret
|
|
38
|
+
// before the source list that points at it lands. Inference sources carry no
|
|
39
|
+
// binding descriptor. A failure here propagates and aborts the source push --
|
|
40
|
+
// never a stale secret paired with a fresh source list.
|
|
41
|
+
if (resolution.materials.length > 0) {
|
|
42
|
+
const delivery = {
|
|
43
|
+
bindings: [],
|
|
44
|
+
materials: resolution.materials,
|
|
45
|
+
};
|
|
46
|
+
await sidecarRouter.sendCredentialsUpdate(instance.address, delivery);
|
|
47
|
+
}
|
|
29
48
|
await sidecarRouter.sendSourcesUpdate(instance.address, resolution.sources, head.id);
|
|
30
49
|
}
|
|
31
50
|
/**
|
|
@@ -64,6 +83,9 @@ async function pushSourceUpdatesToTenants(db, sidecarRouter, tenantIds, credenti
|
|
|
64
83
|
if (instance.address === null) {
|
|
65
84
|
throw new Error(`running run ${instance.id} matched the non-null-address filter but has a null address`);
|
|
66
85
|
}
|
|
86
|
+
if (instance.definitionId === null) {
|
|
87
|
+
throw new Error(`running run ${instance.id} has no workflow definition`);
|
|
88
|
+
}
|
|
67
89
|
return pushInstanceSourceUpdate(db, sidecarRouter, {
|
|
68
90
|
address: instance.address,
|
|
69
91
|
definitionId: instance.definitionId,
|
|
@@ -107,3 +129,113 @@ export async function pushSourceUpdatesSubtree(db, sidecarRouter, tenantId, cred
|
|
|
107
129
|
}
|
|
108
130
|
await pushSourceUpdatesToTenants(db, sidecarRouter, tenants, credentialCipher);
|
|
109
131
|
}
|
|
132
|
+
/**
|
|
133
|
+
* After a credential is deleted or deliberately revoked, evict it from every
|
|
134
|
+
* running deployment in the tenant AND its descendants. A descendant resolves
|
|
135
|
+
* an ancestor's tenant-owned credential through the tenant walk-up, so a
|
|
136
|
+
* revoke in one tenant can affect a descendant's run. The push is a flat named
|
|
137
|
+
* revocation: the child drops the credentialId's material and any binding that
|
|
138
|
+
* references it, and a run that never held it no-ops. Because a flat revoke is
|
|
139
|
+
* safe to broadcast, this needs no per-instance ledger of what was delivered.
|
|
140
|
+
*
|
|
141
|
+
* Callers fire this without awaiting, so it must never reject: a failure to
|
|
142
|
+
* enumerate or push is logged and dropped. This closes the ONLINE revocation
|
|
143
|
+
* window (a running deployment stops holding the credential now); the offline
|
|
144
|
+
* window -- a run whose sidecar was disconnected when the revoke fired -- is
|
|
145
|
+
* closed by the reconnect resync, not here.
|
|
146
|
+
*/
|
|
147
|
+
export async function pushCredentialRevoke(db, sidecarRouter, tenantId, credentialId) {
|
|
148
|
+
let tenants;
|
|
149
|
+
try {
|
|
150
|
+
tenants = await getDescendantTenants(db, tenantId);
|
|
151
|
+
}
|
|
152
|
+
catch (err) {
|
|
153
|
+
log.warn `Failed to enumerate descendants for credential revoke: ${String(err)}`;
|
|
154
|
+
return;
|
|
155
|
+
}
|
|
156
|
+
try {
|
|
157
|
+
// Every running, addressable run in the subtree. Unlike the source-update
|
|
158
|
+
// push this does NOT exclude deployment-anchor runs (`anchorRunId IS NULL`):
|
|
159
|
+
// a deployed workflow is the primary credential consumer, and a flat revoke
|
|
160
|
+
// is safe to deliver to any address -- a run that never held the credential
|
|
161
|
+
// no-ops on it.
|
|
162
|
+
const runs = await db.query.workflowRun.findMany({
|
|
163
|
+
where: and(inArray(workflowRun.tenantId, tenants), eq(workflowRun.status, "running"), isNotNull(workflowRun.address)),
|
|
164
|
+
columns: { id: true, address: true },
|
|
165
|
+
});
|
|
166
|
+
const addresses = new Set();
|
|
167
|
+
for (const run of runs) {
|
|
168
|
+
// The isNotNull(address) filter guarantees a value; a null here is a
|
|
169
|
+
// broken invariant, surfaced rather than silently skipped.
|
|
170
|
+
if (run.address === null) {
|
|
171
|
+
throw new Error(`running run ${run.id} matched the non-null-address filter but has a null address`);
|
|
172
|
+
}
|
|
173
|
+
addresses.add(run.address);
|
|
174
|
+
}
|
|
175
|
+
if (addresses.size === 0)
|
|
176
|
+
return;
|
|
177
|
+
const emptyDelivery = { bindings: [], materials: [] };
|
|
178
|
+
const results = await Promise.allSettled([...addresses].map((address) => sidecarRouter.sendCredentialsUpdate(address, emptyDelivery, [
|
|
179
|
+
credentialId,
|
|
180
|
+
])));
|
|
181
|
+
for (const result of results) {
|
|
182
|
+
if (result.status === "rejected") {
|
|
183
|
+
log.warn `Failed to push credential revoke: ${String(result.reason)}`;
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
catch (err) {
|
|
188
|
+
log.warn `Failed to push credential revoke: ${String(err)}`;
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
/**
|
|
192
|
+
* Reconcile a reconnecting deployment's credentials against its deploy-time set.
|
|
193
|
+
* Re-resolve the CURRENT material for every credentialId the deployment
|
|
194
|
+
* persisted at deploy (`workflow_run.credentialRefs`), then push a MERGE that
|
|
195
|
+
* upserts the survivors (picking up a same-id secret rotation) and REVOKES the
|
|
196
|
+
* deploy-time ids that no longer resolve (deleted or revoked while the sidecar
|
|
197
|
+
* was disconnected). Closes the OFFLINE revocation window (the online window is
|
|
198
|
+
* closed by `pushCredentialRevoke`).
|
|
199
|
+
*
|
|
200
|
+
* Merge, not wholesale-replace: `credentialRefs` is only the deploy-time id set,
|
|
201
|
+
* not the child's complete live set (a catalog re-point can deliver a new
|
|
202
|
+
* credential online), so a replace would evict online-added credentials. The
|
|
203
|
+
* merge upserts survivors and names the dead ids in `revoke`, leaving online
|
|
204
|
+
* credentials untouched. It does NOT handle an id-CHANGING rotation of a
|
|
205
|
+
* deploy-time source (the new id is not in `credentialRefs`); a later source
|
|
206
|
+
* push delivers that.
|
|
207
|
+
*
|
|
208
|
+
* No-op when the run persisted no credential refs (a folded run, or a
|
|
209
|
+
* deployment with no credentials). Fire-and-forget from the reconnect handler:
|
|
210
|
+
* it never rejects. A live-but-unresolvable credential (its provider vanished
|
|
211
|
+
* or has no API base URL) makes `reresolveCurrentMaterials` throw, which aborts
|
|
212
|
+
* the WHOLE reconcile (logged, not sent) so a partial set with a spurious
|
|
213
|
+
* revoke never lands. This is deliberately all-or-nothing: one misconfigured
|
|
214
|
+
* credential blocks this reconnect's revocation of the others too, trading
|
|
215
|
+
* revocation timeliness for never falsely evicting a live credential. The next
|
|
216
|
+
* reconnect (or an online revoke) retries.
|
|
217
|
+
*/
|
|
218
|
+
export async function pushCredentialReconcile(db, sidecarRouter, agentAddress, credentialCipher) {
|
|
219
|
+
try {
|
|
220
|
+
const run = await db.query.workflowRun.findFirst({
|
|
221
|
+
where: eq(workflowRun.address, agentAddress),
|
|
222
|
+
columns: { credentialRefs: true },
|
|
223
|
+
});
|
|
224
|
+
if (run === undefined)
|
|
225
|
+
return;
|
|
226
|
+
const refs = run.credentialRefs;
|
|
227
|
+
if (refs === null)
|
|
228
|
+
return;
|
|
229
|
+
const materials = await reresolveCurrentMaterials(db, refs.credentialIds, credentialCipher);
|
|
230
|
+
const resolvedIds = new Set(materials.map((material) => material.credentialId));
|
|
231
|
+
// Deploy-time ids that no longer resolve: deleted or revoked while offline.
|
|
232
|
+
const revoke = refs.credentialIds.filter((id) => !resolvedIds.has(id));
|
|
233
|
+
// A binding whose credential dropped out of the re-resolution goes with it.
|
|
234
|
+
const bindings = refs.bindings.filter((binding) => resolvedIds.has(binding.credentialId));
|
|
235
|
+
const delivery = { bindings, materials };
|
|
236
|
+
await sidecarRouter.sendCredentialsUpdate(agentAddress, delivery, revoke.length > 0 ? revoke : undefined);
|
|
237
|
+
}
|
|
238
|
+
catch (err) {
|
|
239
|
+
log.warn `Failed to reconcile credentials for ${agentAddress}: ${String(err)}`;
|
|
240
|
+
}
|
|
241
|
+
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { DB } from "@intx/db";
|
|
2
2
|
import type { InferenceEvent } from "@intx/types/runtime";
|
|
3
3
|
import type { SessionStatus } from "@intx/types";
|
|
4
|
-
import { type TurnFinalized } from "./event-collector.js";
|
|
4
|
+
import { type TurnFinalized, type TurnUsage } from "./event-collector.js";
|
|
5
5
|
export type EventCollectorRegistry = {
|
|
6
6
|
create(agentAddress: string, tenantId: string, sessionId: string, runId: string): void;
|
|
7
7
|
dispatch(agentAddress: string, event: InferenceEvent): void;
|
|
@@ -15,6 +15,7 @@ export type EventCollectorRegistry = {
|
|
|
15
15
|
export type EventCollectorRegistryConfig = {
|
|
16
16
|
db: DB["db"];
|
|
17
17
|
onTurnFinalized?: (agentAddress: string, turn: TurnFinalized) => void;
|
|
18
|
+
onUsage?: (agentAddress: string, usage: TurnUsage) => void;
|
|
18
19
|
};
|
|
19
20
|
export declare function deriveStatus(event: InferenceEvent): SessionStatus | null;
|
|
20
21
|
export declare function createEventCollectorRegistry(config: EventCollectorRegistryConfig): EventCollectorRegistry;
|
|
@@ -30,9 +30,12 @@ export function deriveStatus(event) {
|
|
|
30
30
|
}
|
|
31
31
|
}
|
|
32
32
|
export function createEventCollectorRegistry(config) {
|
|
33
|
-
const { db, onTurnFinalized } = config;
|
|
33
|
+
const { db, onTurnFinalized, onUsage } = config;
|
|
34
34
|
const collectors = new Map();
|
|
35
35
|
const statuses = new Map();
|
|
36
|
+
// Per-address tail promise: serializes onEvent/abandon work for one run
|
|
37
|
+
// address so their DB writes cannot interleave. Reaped when it drains.
|
|
38
|
+
const tails = new Map();
|
|
36
39
|
function create(agentAddress, tenantId, sessionId, runId) {
|
|
37
40
|
if (collectors.has(agentAddress)) {
|
|
38
41
|
log.warn `Collector already exists for ${agentAddress}, replacing`;
|
|
@@ -48,6 +51,11 @@ export function createEventCollectorRegistry(config) {
|
|
|
48
51
|
onTurnFinalized: (turn) => onTurnFinalized(agentAddress, turn),
|
|
49
52
|
}
|
|
50
53
|
: {}),
|
|
54
|
+
...(onUsage
|
|
55
|
+
? {
|
|
56
|
+
onUsage: (usage) => onUsage(agentAddress, usage),
|
|
57
|
+
}
|
|
58
|
+
: {}),
|
|
51
59
|
});
|
|
52
60
|
collectors.set(agentAddress, collector);
|
|
53
61
|
statuses.set(agentAddress, { status: "idle" });
|
|
@@ -56,6 +64,25 @@ export function createEventCollectorRegistry(config) {
|
|
|
56
64
|
collectors.delete(agentAddress);
|
|
57
65
|
statuses.delete(agentAddress);
|
|
58
66
|
}
|
|
67
|
+
// Chain `work` onto the address's tail so per-address work runs in order and
|
|
68
|
+
// never interleaves, while the caller stays non-blocking. `onError` swallows
|
|
69
|
+
// a failure so one bad event cannot wedge the chain; `onSettled` runs after
|
|
70
|
+
// the work settles. The tail entry is reaped once no later work is queued.
|
|
71
|
+
function enqueue(agentAddress, work, onError, onSettled) {
|
|
72
|
+
const prev = tails.get(agentAddress) ?? Promise.resolve();
|
|
73
|
+
const next = prev
|
|
74
|
+
.then(work)
|
|
75
|
+
.catch(onError)
|
|
76
|
+
.finally(() => {
|
|
77
|
+
if (onSettled !== undefined) {
|
|
78
|
+
onSettled();
|
|
79
|
+
}
|
|
80
|
+
if (tails.get(agentAddress) === next) {
|
|
81
|
+
tails.delete(agentAddress);
|
|
82
|
+
}
|
|
83
|
+
});
|
|
84
|
+
tails.set(agentAddress, next);
|
|
85
|
+
}
|
|
59
86
|
function dispatch(agentAddress, event) {
|
|
60
87
|
const collector = collectors.get(agentAddress);
|
|
61
88
|
if (collector === undefined) {
|
|
@@ -67,13 +94,10 @@ export function createEventCollectorRegistry(config) {
|
|
|
67
94
|
}
|
|
68
95
|
const isTerminal = event.type === "reactor.done" ||
|
|
69
96
|
(event.type === "reactor.error" && event.data.fatal);
|
|
70
|
-
collector
|
|
71
|
-
.onEvent(event)
|
|
72
|
-
.catch((err) => {
|
|
97
|
+
enqueue(agentAddress, () => collector.onEvent(event), (err) => {
|
|
73
98
|
log.warn `Failed to persist event ${event.type} seq=${String(event.seq)} for ${agentAddress}: ${err instanceof Error ? err.message : String(err)}`;
|
|
74
|
-
})
|
|
75
|
-
.
|
|
76
|
-
if (isTerminal) {
|
|
99
|
+
}, () => {
|
|
100
|
+
if (isTerminal && collectors.get(agentAddress) === collector) {
|
|
77
101
|
removeCollector(agentAddress);
|
|
78
102
|
}
|
|
79
103
|
});
|
|
@@ -82,10 +106,15 @@ export function createEventCollectorRegistry(config) {
|
|
|
82
106
|
const collector = collectors.get(agentAddress);
|
|
83
107
|
if (collector === undefined)
|
|
84
108
|
return;
|
|
85
|
-
|
|
109
|
+
// Stop NEW dispatches immediately; the queued closures keep their own
|
|
110
|
+
// reference so already-queued events still drain before the abandon runs.
|
|
111
|
+
removeCollector(agentAddress);
|
|
112
|
+
// Chain the abandon onto the tail so it runs AFTER any queued onEvents
|
|
113
|
+
// instead of racing them. Otherwise a queued beginTurn could create a
|
|
114
|
+
// fresh `running` turn row after the collector was finalized, orphaning it.
|
|
115
|
+
enqueue(agentAddress, () => collector.abandon(), (err) => {
|
|
86
116
|
log.warn `Failed to abandon collector for ${agentAddress}: ${err instanceof Error ? err.message : String(err)}`;
|
|
87
117
|
});
|
|
88
|
-
removeCollector(agentAddress);
|
|
89
118
|
}
|
|
90
119
|
function has(agentAddress) {
|
|
91
120
|
return collectors.has(agentAddress);
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { InferenceEvent } from "@intx/types/runtime";
|
|
1
|
+
import type { InferenceEvent, TokenUsage } from "@intx/types/runtime";
|
|
2
2
|
import { type DB } from "@intx/db";
|
|
3
3
|
export type TurnToolCall = {
|
|
4
4
|
name: string;
|
|
@@ -22,6 +22,15 @@ export type TurnFinalized = {
|
|
|
22
22
|
content: string;
|
|
23
23
|
}[];
|
|
24
24
|
};
|
|
25
|
+
export type TurnUsage = {
|
|
26
|
+
tenantId: string;
|
|
27
|
+
sessionId: string;
|
|
28
|
+
runId: string;
|
|
29
|
+
turnId: string;
|
|
30
|
+
provider: string;
|
|
31
|
+
model: string;
|
|
32
|
+
usage: TokenUsage;
|
|
33
|
+
};
|
|
25
34
|
export type EventCollector = {
|
|
26
35
|
onEvent(event: InferenceEvent): Promise<void>;
|
|
27
36
|
abandon(): Promise<void>;
|
|
@@ -35,5 +44,6 @@ export type EventCollectorConfig = {
|
|
|
35
44
|
runId: string;
|
|
36
45
|
tenantId: string;
|
|
37
46
|
onTurnFinalized?: (turn: TurnFinalized) => void;
|
|
47
|
+
onUsage?: (usage: TurnUsage) => void;
|
|
38
48
|
};
|
|
39
49
|
export declare function createEventCollector(config: EventCollectorConfig): EventCollector;
|