@intx/hub-sessions 0.2.2 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -5
- package/dist/agent-repo.d.ts +23 -7
- package/dist/agent-repo.js +19 -6
- package/dist/agent-state-kind.js +18 -63
- package/dist/asset-service.d.ts +1 -20
- package/dist/asset-service.js +19 -97
- package/dist/committed-source-tree.d.ts +10 -0
- package/dist/committed-source-tree.js +35 -0
- package/dist/credential-push.d.ts +52 -7
- package/dist/credential-push.js +178 -22
- package/dist/event-collector-registry.d.ts +3 -2
- package/dist/event-collector-registry.js +42 -13
- package/dist/event-collector.d.ts +12 -2
- package/dist/event-collector.js +45 -4
- package/dist/hub-session-lookups.d.ts +125 -7
- package/dist/hub-session-lookups.js +541 -86
- package/dist/hub-session-orchestrator.d.ts +2 -3
- package/dist/hub-session-orchestrator.js +22 -56
- package/dist/index.d.ts +19 -9
- package/dist/index.js +16 -7
- package/dist/reconciliation-scheduler.d.ts +14 -0
- package/dist/reconciliation-scheduler.js +55 -0
- package/dist/repo-store/index.d.ts +2 -1
- package/dist/repo-store/index.js +1 -0
- package/dist/repo-store/store.d.ts +1 -1
- package/dist/repo-store/store.js +138 -1
- package/dist/repo-store/subscribe-kind.d.ts +6 -3
- package/dist/repo-store/subscribe-kind.js +42 -77
- package/dist/repo-store/types.d.ts +94 -6
- package/dist/repo-store/user-principal-gate.d.ts +26 -0
- package/dist/repo-store/user-principal-gate.js +78 -0
- package/dist/session-service.d.ts +252 -126
- package/dist/session-service.js +851 -624
- package/dist/sidecar-allocation/capability-policy.d.ts +27 -0
- package/dist/sidecar-allocation/capability-policy.js +124 -0
- package/dist/sidecar-allocation/contracts.d.ts +101 -0
- package/dist/sidecar-allocation/contracts.js +26 -0
- package/dist/sidecar-allocation/index.d.ts +5 -0
- package/dist/sidecar-allocation/index.js +4 -0
- package/dist/sidecar-allocation/operation.d.ts +10 -0
- package/dist/sidecar-allocation/operation.js +54 -0
- package/dist/sidecar-allocation/plugin-registry.d.ts +24 -0
- package/dist/sidecar-allocation/plugin-registry.js +61 -0
- package/dist/sidecar-allocation/reconciler.d.ts +54 -0
- package/dist/sidecar-allocation/reconciler.js +825 -0
- package/dist/skill-kind.js +12 -62
- package/dist/substrate.d.ts +3 -3
- package/dist/substrate.js +1 -1
- package/dist/workflow-allocation-service.d.ts +64 -0
- package/dist/workflow-allocation-service.js +554 -0
- package/dist/workflow-closure-resolution.d.ts +106 -0
- package/dist/workflow-closure-resolution.js +123 -0
- package/dist/workflow-definition-ensure.d.ts +24 -0
- package/dist/workflow-definition-ensure.js +75 -0
- package/dist/workflow-dispatch-service.d.ts +42 -0
- package/dist/workflow-dispatch-service.js +209 -0
- package/dist/workflow-dispatch-settlement.d.ts +29 -0
- package/dist/workflow-dispatch-settlement.js +140 -0
- package/dist/workflow-kind.d.ts +29 -1
- package/dist/workflow-kind.js +143 -139
- package/dist/workflow-probe-gate.d.ts +286 -0
- package/dist/workflow-probe-gate.js +382 -0
- package/dist/workflow-run-kind.d.ts +239 -32
- package/dist/workflow-run-kind.js +874 -188
- package/dist/workflow-run-reader.d.ts +1 -1
- package/dist/workflow-run-reader.js +3 -7
- package/dist/workflow-run-restore.d.ts +16 -0
- package/dist/workflow-run-restore.js +30 -0
- package/dist/workflow-source-closure.d.ts +35 -0
- package/dist/workflow-source-closure.js +342 -0
- package/dist/workflow-source-pins.d.ts +8 -0
- package/dist/workflow-source-pins.js +14 -0
- package/dist/ws/index.d.ts +3 -3
- package/dist/ws/index.js +2 -2
- package/dist/ws/pending-tracker.d.ts +93 -0
- package/dist/ws/pending-tracker.js +132 -0
- package/dist/ws/sidecar-events.d.ts +139 -37
- package/dist/ws/sidecar-events.js +2 -2
- package/dist/ws/sidecar-handler.d.ts +218 -60
- package/dist/ws/sidecar-handler.js +1713 -732
- package/dist/ws/sidecar-handler.test-helpers.d.ts +38 -0
- package/dist/ws/sidecar-handler.test-helpers.js +95 -0
- package/dist/ws/sidecar-token-authenticator.d.ts +3 -1
- package/dist/ws/sidecar-token-authenticator.js +78 -7
- package/package.json +14 -13
- package/dist/available-skills-stanza.d.ts +0 -21
- package/dist/available-skills-stanza.js +0 -32
|
@@ -5,92 +5,80 @@
|
|
|
5
5
|
// Each lookup is a stateless DB or repo call. They are gathered into a
|
|
6
6
|
// single struct that the hub app passes to `createSidecarRouter` as
|
|
7
7
|
// `lookups`.
|
|
8
|
-
import { eq, and, isNull } from "drizzle-orm";
|
|
9
|
-
import {
|
|
8
|
+
import { eq, and, asc, inArray, isNotNull, isNull } from "drizzle-orm";
|
|
9
|
+
import { createApprovalStore, createSignalCorrelationStore, createWorkflowRunDispatchStore, createWorkflowRunStore, } from "@intx/db";
|
|
10
|
+
import { agentSession, liveWorkflowRunStatuses, principal, sessionMail, sidecarAllocation, workflowRun, } from "@intx/db/schema";
|
|
10
11
|
import { getLogger } from "@intx/log";
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
12
|
+
import { parseRunAddress, signalName } from "@intx/types";
|
|
13
|
+
import { SignalDeliverFrame } from "@intx/types/sidecar";
|
|
14
|
+
import { deriveWorkflowRunRepoId } from "@intx/workflow-deploy";
|
|
13
15
|
import { generateId } from "@intx/hub-common";
|
|
16
|
+
import { listAcceptedWorkflowDispatches, listConsumedWorkflowDispatches, } from "./workflow-dispatch-settlement.js";
|
|
17
|
+
import { readCommittedWorkflowRunLifecycle } from "./workflow-run-kind.js";
|
|
14
18
|
const logger = getLogger(["hub", "lookups"]);
|
|
15
19
|
export function createHubSessionLookups(deps) {
|
|
16
20
|
const { db, agentRepoStore } = deps;
|
|
21
|
+
const signalCorrelationStore = createSignalCorrelationStore(db);
|
|
22
|
+
const approvalStore = createApprovalStore(db);
|
|
23
|
+
const workflowRunStore = createWorkflowRunStore(db);
|
|
24
|
+
const workflowRunDispatchStore = createWorkflowRunDispatchStore(db);
|
|
17
25
|
return {
|
|
18
|
-
async
|
|
19
|
-
//
|
|
20
|
-
//
|
|
21
|
-
//
|
|
22
|
-
|
|
23
|
-
// launched agent that is missing its instance row returns null and
|
|
24
|
-
// fails its challenge visibly, instead of silently resolving against
|
|
25
|
-
// the wrong table.
|
|
26
|
-
if (isWorkflowDerivedAddress(agentAddress)) {
|
|
27
|
-
// Filter to a live ("deployed") deployment so a torn-down
|
|
28
|
-
// deployment's key can no longer satisfy a challenge. A null
|
|
29
|
-
// publicKey (deployed but not yet acked, or pre-migration) or an
|
|
30
|
-
// absent row returns null -- the challenge fails closed and the
|
|
31
|
-
// address stays unrouted rather than routing without ownership
|
|
32
|
-
// proof.
|
|
33
|
-
const row = await db
|
|
34
|
-
.select({ publicKey: workflowDeployment.publicKey })
|
|
35
|
-
.from(workflowDeployment)
|
|
36
|
-
.where(and(eq(workflowDeployment.address, agentAddress), eq(workflowDeployment.status, "deployed")))
|
|
37
|
-
.limit(1)
|
|
38
|
-
.then((rows) => rows[0]);
|
|
39
|
-
return row?.publicKey ?? null;
|
|
40
|
-
}
|
|
41
|
-
const row = await db
|
|
42
|
-
.select({ publicKey: agentInstance.publicKey })
|
|
43
|
-
.from(agentInstance)
|
|
44
|
-
.where(and(eq(agentInstance.address, agentAddress), isNull(agentInstance.endedAt)))
|
|
45
|
-
.limit(1)
|
|
46
|
-
.then((rows) => rows[0]);
|
|
47
|
-
return row?.publicKey ?? null;
|
|
48
|
-
},
|
|
49
|
-
async lookupDeployRef(agentAddress) {
|
|
50
|
-
const agentId = parseAgentId(agentAddress);
|
|
51
|
-
return agentRepoStore.getDeployRef(agentId);
|
|
26
|
+
async lookupDeployRef() {
|
|
27
|
+
// A workflow run is a supervised workflow-process pinned forever like a
|
|
28
|
+
// native deployment: it keeps its deploy-time definition and never
|
|
29
|
+
// reconciles, so no address enrolls in the reconnect deploy-ref catch-up.
|
|
30
|
+
return null;
|
|
52
31
|
},
|
|
53
32
|
async persistMail({ senderAddress, recipients, raw }) {
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
33
|
+
// The sender and recipients are run addresses, each backed by its
|
|
34
|
+
// self-anchored workflow_run; resolve each through the resolver. A mail
|
|
35
|
+
// record's `runId` is always null for a run -- it keys on the run's
|
|
36
|
+
// session instead.
|
|
37
|
+
const sender = await resolveRoutableAddress(db, senderAddress);
|
|
38
|
+
if (sender === undefined) {
|
|
39
|
+
throw new Error(`No active endpoint found for sender address "${senderAddress}"`);
|
|
40
|
+
}
|
|
41
|
+
if (sender.sessionId === null) {
|
|
42
|
+
throw new Error(`Endpoint ${sender.id} has no session for address "${senderAddress}"`);
|
|
57
43
|
}
|
|
58
44
|
const createdAt = new Date();
|
|
45
|
+
const senderRunId = null;
|
|
59
46
|
// Outbound record on the sender's session.
|
|
60
47
|
const outboundId = generateId("sessionMail");
|
|
61
48
|
const outboundRecord = {
|
|
62
49
|
id: outboundId,
|
|
63
|
-
sessionId:
|
|
64
|
-
|
|
65
|
-
tenantId:
|
|
50
|
+
sessionId: sender.sessionId,
|
|
51
|
+
runId: senderRunId,
|
|
52
|
+
tenantId: sender.tenantId,
|
|
66
53
|
direction: "outbound",
|
|
67
54
|
status: "delivered",
|
|
68
55
|
raw,
|
|
69
56
|
createdAt,
|
|
70
57
|
};
|
|
71
|
-
// Inbound records for each recipient that
|
|
72
|
-
//
|
|
73
|
-
// user addresses) are skipped.
|
|
58
|
+
// Inbound records for each recipient that is a live endpoint.
|
|
59
|
+
// Recipients that are not (e.g. human user addresses) are skipped.
|
|
74
60
|
const recipientResults = await Promise.all(recipients.map(async (addr) => {
|
|
75
|
-
const
|
|
76
|
-
if (
|
|
61
|
+
const endpoint = await resolveRoutableAddress(db, addr);
|
|
62
|
+
if (endpoint === undefined) {
|
|
77
63
|
return null;
|
|
78
64
|
}
|
|
79
|
-
if (
|
|
80
|
-
logger.warn `Active
|
|
65
|
+
if (endpoint.sessionId === null) {
|
|
66
|
+
logger.warn `Active endpoint ${endpoint.id} for "${addr}" has no session; skipping inbound record`;
|
|
81
67
|
return null;
|
|
82
68
|
}
|
|
83
|
-
return { addr,
|
|
69
|
+
return { addr, endpoint, sessionId: endpoint.sessionId };
|
|
84
70
|
}));
|
|
85
|
-
const
|
|
86
|
-
const inboundEntries =
|
|
71
|
+
const recipientEndpoints = recipientResults.filter((r) => r !== null);
|
|
72
|
+
const inboundEntries = recipientEndpoints.map(({ addr, endpoint, sessionId }) => {
|
|
87
73
|
const id = generateId("sessionMail");
|
|
74
|
+
// A folded run is not an instance, so its mail records no runId.
|
|
75
|
+
const runId = null;
|
|
88
76
|
return {
|
|
89
77
|
record: {
|
|
90
78
|
id,
|
|
91
79
|
sessionId,
|
|
92
|
-
|
|
93
|
-
tenantId:
|
|
80
|
+
runId,
|
|
81
|
+
tenantId: endpoint.tenantId,
|
|
94
82
|
direction: "inbound",
|
|
95
83
|
status: "delivered",
|
|
96
84
|
raw,
|
|
@@ -99,7 +87,7 @@ export function createHubSessionLookups(deps) {
|
|
|
99
87
|
result: {
|
|
100
88
|
id,
|
|
101
89
|
direction: "inbound",
|
|
102
|
-
|
|
90
|
+
runId,
|
|
103
91
|
address: addr,
|
|
104
92
|
createdAt,
|
|
105
93
|
},
|
|
@@ -112,13 +100,111 @@ export function createHubSessionLookups(deps) {
|
|
|
112
100
|
{
|
|
113
101
|
id: outboundId,
|
|
114
102
|
direction: "outbound",
|
|
115
|
-
|
|
116
|
-
address:
|
|
103
|
+
runId: senderRunId,
|
|
104
|
+
address: sender.address,
|
|
117
105
|
createdAt,
|
|
118
106
|
},
|
|
119
107
|
...inboundEntries.map((e) => e.result),
|
|
120
108
|
];
|
|
121
109
|
},
|
|
110
|
+
async registerSignalCorrelation({ correlationId, runId, anchorRunId, agentAddress, kind, approvalSnapshot, }) {
|
|
111
|
+
// Resolve tenancy and co-write both rows in one transaction so a resolver
|
|
112
|
+
// never sees a correlation without its approval or vice versa. Both
|
|
113
|
+
// inserts are idempotent on their dedup key (the signal_correlation
|
|
114
|
+
// primary key and the approval's unique correlationId), so a redelivered
|
|
115
|
+
// frame -- sidecar reconnect, workflow-log replay, supervisor restart
|
|
116
|
+
// re-emitting -- is a no-op rather than a unique-violation. `timeoutAt` is
|
|
117
|
+
// null: an agent-step suspend holds indefinitely (`parkOnSignal` is called
|
|
118
|
+
// with no timeout), so no deadline reaches this co-write.
|
|
119
|
+
await db.transaction(async (tx) => {
|
|
120
|
+
// Resolve tenancy and the run's definition from the deployment's anchor
|
|
121
|
+
// run -- the workflow_run whose id is the deployment id, which the
|
|
122
|
+
// address names. The anchor is the tenancy origin every approval needs
|
|
123
|
+
// (an approval has no agent_instance/agent/principal referent). The
|
|
124
|
+
// lookup keys off `address` (the field the wire layer's ownership gate
|
|
125
|
+
// authorized), not the frame's `anchorRunId`: that is the workflow-run
|
|
126
|
+
// repo slug the supervisor derives from the address
|
|
127
|
+
// (`deriveWorkflowRunRepoId`), cross-checked below against the slug
|
|
128
|
+
// re-derived from `agentAddress` rather than against the row id. A
|
|
129
|
+
// mismatch fails loud instead of silently writing an inconsistent pair.
|
|
130
|
+
// The FK columns take the anchor run's id (= the deployment id), which
|
|
131
|
+
// is what `signal_correlation.anchor_run_id` and `approval.anchor_run_id`
|
|
132
|
+
// reference.
|
|
133
|
+
//
|
|
134
|
+
// The resolution takes a `FOR UPDATE` row lock and runs inside the
|
|
135
|
+
// co-write transaction, gated on a live anchor run ("deployed" or
|
|
136
|
+
// "running"), so the liveness check and the inserts are atomic against a
|
|
137
|
+
// concurrent teardown that flips the anchor run terminal. The lock order
|
|
138
|
+
// is workflow_run before signal_correlation and approval; a teardown
|
|
139
|
+
// path must take the anchor-run lock before touching those rows to keep
|
|
140
|
+
// the ordering acyclic.
|
|
141
|
+
const anchor = await tx
|
|
142
|
+
.select({
|
|
143
|
+
id: workflowRun.id,
|
|
144
|
+
tenantId: workflowRun.tenantId,
|
|
145
|
+
definitionId: workflowRun.definitionId,
|
|
146
|
+
})
|
|
147
|
+
.from(workflowRun)
|
|
148
|
+
.where(and(eq(workflowRun.address, agentAddress), inArray(workflowRun.status, [...liveWorkflowRunStatuses])))
|
|
149
|
+
.for("update")
|
|
150
|
+
.limit(1)
|
|
151
|
+
.then((rows) => rows[0]);
|
|
152
|
+
if (anchor === undefined) {
|
|
153
|
+
throw new Error(`No live workflow run for address "${agentAddress}"; cannot register signal correlation ${correlationId}`);
|
|
154
|
+
}
|
|
155
|
+
const addressSlug = deriveWorkflowRunRepoId(agentAddress);
|
|
156
|
+
if (addressSlug !== anchorRunId) {
|
|
157
|
+
throw new Error(`Anchor run id mismatch registering signal correlation ${correlationId}: frame claims "${anchorRunId}" but address "${agentAddress}" derives the workflow-run repo slug "${addressSlug}"`);
|
|
158
|
+
}
|
|
159
|
+
const tenantId = anchor.tenantId;
|
|
160
|
+
const definitionId = anchor.definitionId;
|
|
161
|
+
// Lazily anchor the run before its correlation and approval reference
|
|
162
|
+
// it. A workflow-spawned internal run never crosses the external
|
|
163
|
+
// trigger route that mints a run principal, so its run row would
|
|
164
|
+
// otherwise not exist; ensure it here so the co-written rows have a
|
|
165
|
+
// referent. The principal is null: an internal run inherits its
|
|
166
|
+
// deployment's grants and has no principal of its own. The insert is
|
|
167
|
+
// idempotent on the run id, so a redelivered register frame -- the same
|
|
168
|
+
// redelivery the co-writes below tolerate -- is a no-op.
|
|
169
|
+
await workflowRunStore.createIfAbsent({
|
|
170
|
+
id: runId,
|
|
171
|
+
anchorRunId: anchor.id,
|
|
172
|
+
definitionId,
|
|
173
|
+
tenantId,
|
|
174
|
+
principalId: null,
|
|
175
|
+
status: "running",
|
|
176
|
+
}, tx);
|
|
177
|
+
await signalCorrelationStore.registerIfAbsent({
|
|
178
|
+
correlationId,
|
|
179
|
+
tenantId,
|
|
180
|
+
anchorRunId: anchor.id,
|
|
181
|
+
agentAddress,
|
|
182
|
+
runId,
|
|
183
|
+
signalName: signalName(correlationId),
|
|
184
|
+
kind,
|
|
185
|
+
}, tx);
|
|
186
|
+
await approvalStore.createIfAbsent({
|
|
187
|
+
id: generateId("approval"),
|
|
188
|
+
tenantId,
|
|
189
|
+
anchorRunId: anchor.id,
|
|
190
|
+
runId,
|
|
191
|
+
agentAddress,
|
|
192
|
+
correlationId,
|
|
193
|
+
status: "pending",
|
|
194
|
+
// The register frame guarantees the snapshot (the ask rail is its
|
|
195
|
+
// only producer), so the approver-facing columns are always
|
|
196
|
+
// populated -- never null on this path.
|
|
197
|
+
toolDefinition: {
|
|
198
|
+
name: approvalSnapshot.name,
|
|
199
|
+
description: approvalSnapshot.description,
|
|
200
|
+
inputSchema: approvalSnapshot.inputSchema,
|
|
201
|
+
},
|
|
202
|
+
toolArguments: approvalSnapshot.arguments,
|
|
203
|
+
scope: null,
|
|
204
|
+
timeoutAt: null,
|
|
205
|
+
}, tx);
|
|
206
|
+
});
|
|
207
|
+
},
|
|
122
208
|
async receiveAgentStatePack(repoId, pack, ref, commitSha) {
|
|
123
209
|
if (repoId.kind !== "agent-state") {
|
|
124
210
|
throw new Error(`hub-session lookups receiveAgentStatePack received unsupported repo kind ${JSON.stringify(repoId.kind)}`);
|
|
@@ -150,55 +236,424 @@ export function createHubSessionLookups(deps) {
|
|
|
150
236
|
}
|
|
151
237
|
return { accepted: true };
|
|
152
238
|
},
|
|
153
|
-
async receiveWorkflowRunPack(repoId, pack, ref, commitSha) {
|
|
239
|
+
async receiveWorkflowRunPack(repoId, pack, ref, commitSha, source) {
|
|
154
240
|
if (repoId.kind !== "workflow-run") {
|
|
155
241
|
throw new Error(`hub-session lookups receiveWorkflowRunPack received unsupported repo kind ${JSON.stringify(repoId.kind)}`);
|
|
156
242
|
}
|
|
157
|
-
const
|
|
243
|
+
const workflowRunRepoId = repoId.id;
|
|
244
|
+
if (deriveWorkflowRunRepoId(source.agentAddress) !== workflowRunRepoId) {
|
|
245
|
+
logger.warn `Workflow-run pack rejected for ${workflowRunRepoId}: source address does not own the repository`;
|
|
246
|
+
return { accepted: false, reason: "path_violation" };
|
|
247
|
+
}
|
|
248
|
+
const [anchor] = await db
|
|
249
|
+
.select({
|
|
250
|
+
id: workflowRun.id,
|
|
251
|
+
address: workflowRun.address,
|
|
252
|
+
anchorRunId: workflowRun.anchorRunId,
|
|
253
|
+
tenantId: workflowRun.tenantId,
|
|
254
|
+
definitionId: workflowRun.definitionId,
|
|
255
|
+
})
|
|
256
|
+
.from(workflowRun)
|
|
257
|
+
.where(and(eq(workflowRun.address, source.agentAddress), inArray(workflowRun.status, [...liveWorkflowRunStatuses]), isNotNull(workflowRun.definitionId)))
|
|
258
|
+
.limit(1);
|
|
259
|
+
if (anchor === undefined ||
|
|
260
|
+
anchor.anchorRunId !== anchor.id ||
|
|
261
|
+
anchor.address === null) {
|
|
262
|
+
logger.warn `Workflow-run pack rejected for ${workflowRunRepoId}: source address has no live deployment anchor`;
|
|
263
|
+
return { accepted: false, reason: "path_violation" };
|
|
264
|
+
}
|
|
265
|
+
const anchorAddress = anchor.address;
|
|
266
|
+
let newlyTerminalRuns;
|
|
158
267
|
try {
|
|
159
|
-
await
|
|
268
|
+
newlyTerminalRuns = await db.transaction(async (tx) => {
|
|
269
|
+
const [allocation] = await tx
|
|
270
|
+
.select()
|
|
271
|
+
.from(sidecarAllocation)
|
|
272
|
+
.where(eq(sidecarAllocation.anchorRunId, anchor.id))
|
|
273
|
+
.limit(1)
|
|
274
|
+
.for("update");
|
|
275
|
+
if (allocation === undefined ||
|
|
276
|
+
allocation.id !== source.allocationId ||
|
|
277
|
+
allocation.anchorRunId !== source.anchorRunId ||
|
|
278
|
+
source.anchorRunId !== anchor.id ||
|
|
279
|
+
allocation.status !== "allocated" ||
|
|
280
|
+
allocation.generation !== source.generation ||
|
|
281
|
+
allocation.ensureAcceptedGeneration !== source.generation) {
|
|
282
|
+
return null;
|
|
283
|
+
}
|
|
284
|
+
// Replacement advances this same row. Keep its lock until the
|
|
285
|
+
// repository ref has advanced so ownership cannot change after
|
|
286
|
+
// validation but before the old worker's pack becomes authoritative.
|
|
287
|
+
return agentRepoStore.receiveWorkflowRunPack({ kind: "workflow-run", id: workflowRunRepoId }, pack, ref, commitSha);
|
|
288
|
+
});
|
|
289
|
+
if (newlyTerminalRuns === null) {
|
|
290
|
+
logger.warn `Workflow-run pack rejected for ${workflowRunRepoId}: source connection does not own the deployment's current allocation`;
|
|
291
|
+
return { accepted: false, reason: "path_violation" };
|
|
292
|
+
}
|
|
160
293
|
}
|
|
161
294
|
catch (err) {
|
|
162
295
|
const msg = err instanceof Error ? err.message : String(err);
|
|
163
296
|
if (msg.startsWith("path_violation")) {
|
|
164
|
-
logger.warn `Workflow-run pack rejected for ${
|
|
297
|
+
logger.warn `Workflow-run pack rejected for ${workflowRunRepoId}: ${msg}`;
|
|
165
298
|
return { accepted: false, reason: "path_violation" };
|
|
166
299
|
}
|
|
167
|
-
// Mirror the agent-state branch's catch-all: any other failure
|
|
168
|
-
//
|
|
169
|
-
// diagnostics surfaced as Error messages, etc.) becomes
|
|
170
|
-
// structured `corrupt` rejection so the sender can re-push,
|
|
171
|
-
//
|
|
172
|
-
//
|
|
173
|
-
logger.error `Workflow-run pack receive failed for ${
|
|
300
|
+
// Mirror the agent-state branch's catch-all: any other failure from
|
|
301
|
+
// the fenced receive (transaction failures, filesystem races,
|
|
302
|
+
// kind-handler diagnostics surfaced as Error messages, etc.) becomes
|
|
303
|
+
// a structured `corrupt` rejection so the sender can re-push, and the
|
|
304
|
+
// underlying error is logged so the cause stays traceable on the hub
|
|
305
|
+
// side.
|
|
306
|
+
logger.error `Workflow-run pack receive failed for ${workflowRunRepoId}: ${msg}`;
|
|
174
307
|
return { accepted: false, reason: "corrupt" };
|
|
175
308
|
}
|
|
309
|
+
// The substrate has already durably advanced the git ref by the time it
|
|
310
|
+
// returns, so the pack is accepted regardless of what happens below. The
|
|
311
|
+
// per-run status flip and principal deactivation are a best-effort
|
|
312
|
+
// downstream side effect of that durable advance, not part of accepting
|
|
313
|
+
// the pack. A failure here leaves the run "running" in the DB with its
|
|
314
|
+
// principal still active; there is no automatic re-fire, because a
|
|
315
|
+
// redelivery of the same durable tip produces no newly-terminal signal
|
|
316
|
+
// (the substrate's per-commit walk short-circuits on an already-present
|
|
317
|
+
// tip). The failure is therefore logged at ERROR as the only record that
|
|
318
|
+
// the row needs a manual flip, and the pack verdict stays accepted so the
|
|
319
|
+
// sidecar is acked and does not wedge re-pushing a pack that already
|
|
320
|
+
// landed.
|
|
321
|
+
const now = new Date();
|
|
322
|
+
for (const { runId, status } of newlyTerminalRuns) {
|
|
323
|
+
try {
|
|
324
|
+
await db.transaction(async (tx) => {
|
|
325
|
+
// Lazily anchor the run before settling it. An internal run that
|
|
326
|
+
// parks only on a plain signal gate never reaches
|
|
327
|
+
// `registerSignalCorrelation`, the sole other path that mints an
|
|
328
|
+
// internal run row, so its terminal event can be the first the hub
|
|
329
|
+
// sees of the run. A never-minted row is ordinary bookkeeping, not
|
|
330
|
+
// a deployment-boundary violation, so mint it here against this
|
|
331
|
+
// deployment's anchor rather than letting the ownership guard below
|
|
332
|
+
// mistake absence for foreignness. The insert no-ops when any row
|
|
333
|
+
// already exists, which keeps that guard authoritative for a row
|
|
334
|
+
// that exists and anchors elsewhere. The principal is null: an
|
|
335
|
+
// internal run inherits its deployment's grants and has none of its
|
|
336
|
+
// own.
|
|
337
|
+
//
|
|
338
|
+
// The mint necessarily precedes the ownership guard, so an id the
|
|
339
|
+
// hub has never seen is claimed under THIS anchor before anything
|
|
340
|
+
// establishes it belongs here. That ordering is required -- the
|
|
341
|
+
// guard reads the row the mint may have to create -- and it is
|
|
342
|
+
// bounded rather than unbounded: internal run ids are supplied by
|
|
343
|
+
// the sidecar and accepted verbatim, so the value is
|
|
344
|
+
// caller-influenced, but it is a different population from the
|
|
345
|
+
// anchor ids the hub mints itself, and nothing resolves an
|
|
346
|
+
// internal id without also constraining the anchor or the tenant.
|
|
347
|
+
// The insert cannot take a row away from another deployment; the
|
|
348
|
+
// worst it does is create one for an id that deployment would
|
|
349
|
+
// otherwise have created later.
|
|
350
|
+
await workflowRunStore.createIfAbsent({
|
|
351
|
+
id: runId,
|
|
352
|
+
anchorRunId: anchor.id,
|
|
353
|
+
definitionId: anchor.definitionId,
|
|
354
|
+
tenantId: anchor.tenantId,
|
|
355
|
+
principalId: null,
|
|
356
|
+
status: "running",
|
|
357
|
+
}, tx);
|
|
358
|
+
const [ownedRun] = await tx
|
|
359
|
+
.select({ anchorRunId: workflowRun.anchorRunId })
|
|
360
|
+
.from(workflowRun)
|
|
361
|
+
.where(eq(workflowRun.id, runId))
|
|
362
|
+
.limit(1);
|
|
363
|
+
if (ownedRun?.anchorRunId !== anchor.id) {
|
|
364
|
+
logger.error `Ignoring terminal event for run ${runId}: it does not belong to source deployment ${anchor.id}`;
|
|
365
|
+
return;
|
|
366
|
+
}
|
|
367
|
+
const won = await workflowRunStore.markTerminal(runId, status, now, tx);
|
|
368
|
+
if (won === null) {
|
|
369
|
+
// The row exists (the mint above guarantees it) and belongs to
|
|
370
|
+
// this deployment (the guard above), so no running row matched
|
|
371
|
+
// only because the run is already terminal -- a benign replay
|
|
372
|
+
// against an already-settled row. Leave its settled status and
|
|
373
|
+
// `endedAt` alone.
|
|
374
|
+
return;
|
|
375
|
+
}
|
|
376
|
+
// Deactivate the run's own principal, if it has one. Externally-
|
|
377
|
+
// triggered runs carry a principal; internal, workflow-spawned runs
|
|
378
|
+
// have `principalId = null` and inherit the deployment's grants, so
|
|
379
|
+
// there is nothing to deactivate. Deactivation is gated on winning
|
|
380
|
+
// the flip -- the single claim point -- not on the principal's own
|
|
381
|
+
// status.
|
|
382
|
+
if (won.principalId !== null) {
|
|
383
|
+
await tx
|
|
384
|
+
.update(principal)
|
|
385
|
+
.set({ status: "deactivated", updatedAt: now })
|
|
386
|
+
// The `refId` clause is a defensive mirror of the per-instance
|
|
387
|
+
// teardown in instances.ts: `won.principalId` is already this
|
|
388
|
+
// run's own principal, and `principal.id` is the primary key,
|
|
389
|
+
// so the `refId` match is belt-and-suspenders that the id we
|
|
390
|
+
// won belongs to this run.
|
|
391
|
+
.where(and(eq(principal.id, won.principalId), eq(principal.refId, runId)));
|
|
392
|
+
}
|
|
393
|
+
});
|
|
394
|
+
}
|
|
395
|
+
catch (err) {
|
|
396
|
+
// Per-run isolation: a failed flip for one run must not abort the
|
|
397
|
+
// rest of the batch, and must not throw out of this method -- a throw
|
|
398
|
+
// would leave the sidecar with neither an ack nor a reject for a pack
|
|
399
|
+
// the substrate already accepted. This ERROR is the only signal that
|
|
400
|
+
// the run is stuck "running" in the DB with its principal active, so
|
|
401
|
+
// it carries enough to find and flip the row by hand.
|
|
402
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
403
|
+
logger.error `Terminal DB flip failed for run ${runId} (deployment ${anchor.id}, target status ${status}); run left running in the DB: ${msg}`;
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
// A sidecar-local mail ack is only receipt. The raw trigger remains in
|
|
407
|
+
// workflow_run_dispatch until the accepted Git tip proves either that
|
|
408
|
+
// the run recorded it (RunStarted / SignalReceived), or that the
|
|
409
|
+
// supervisor consumed it with an explicit rejection. Rescan the bounded
|
|
410
|
+
// retained claim-check index after every accepted pack, and search the
|
|
411
|
+
// stable run log newest-first only for currently-unsettled ids. Settlement
|
|
412
|
+
// is idempotent, and a later pack naturally retries a transient database
|
|
413
|
+
// failure here.
|
|
414
|
+
let topLevelTerminalSettlementProjected = false;
|
|
415
|
+
try {
|
|
416
|
+
const reads = await agentRepoStore.repoStore.openCommittedReads({ kind: "hub" }, repoId, ref);
|
|
417
|
+
if (reads !== null) {
|
|
418
|
+
const unsettledDispatches = await workflowRunDispatchStore.listUnsettled(anchor.id);
|
|
419
|
+
const unsettledByMessageId = new Map(unsettledDispatches.map((dispatch) => [
|
|
420
|
+
dispatch.messageId,
|
|
421
|
+
dispatch,
|
|
422
|
+
]));
|
|
423
|
+
for (const consumed of await listConsumedWorkflowDispatches(reads)) {
|
|
424
|
+
if (consumed.address !== anchorAddress)
|
|
425
|
+
continue;
|
|
426
|
+
const persisted = unsettledByMessageId.get(consumed.messageId);
|
|
427
|
+
if (persisted?.kind !== "mail")
|
|
428
|
+
continue;
|
|
429
|
+
if (consumed.rejection === undefined) {
|
|
430
|
+
await workflowRunDispatchStore.settle(anchor.id, consumed.messageId, now);
|
|
431
|
+
}
|
|
432
|
+
else {
|
|
433
|
+
await workflowRunDispatchStore.fail({
|
|
434
|
+
anchorRunId: anchor.id,
|
|
435
|
+
messageId: consumed.messageId,
|
|
436
|
+
code: consumed.rejection.code,
|
|
437
|
+
message: consumed.rejection.message,
|
|
438
|
+
now,
|
|
439
|
+
});
|
|
440
|
+
}
|
|
441
|
+
unsettledByMessageId.delete(consumed.messageId);
|
|
442
|
+
}
|
|
443
|
+
// Mail is recorded on the stable deployment run, while a signal is
|
|
444
|
+
// recorded on the exact run named by its durable delivery frame.
|
|
445
|
+
// Group retained dispatches by that Git run before scanning so an
|
|
446
|
+
// internal run's SignalReceived evidence settles its own dispatch.
|
|
447
|
+
// The `runs/<runId>/` log keys on the run id (the address local
|
|
448
|
+
// part), NOT the full address: post-collapse the top-level run's id
|
|
449
|
+
// IS `anchor.id`, so mail keys on `anchor.id`; a signal keys on its
|
|
450
|
+
// frame's own run id. (The `addresses/<address>/` consumed subtree
|
|
451
|
+
// above keys on the full address -- a different subtree.)
|
|
452
|
+
const messageIdsByRun = new Map();
|
|
453
|
+
for (const dispatch of unsettledByMessageId.values()) {
|
|
454
|
+
const runId = dispatch.kind === "mail"
|
|
455
|
+
? anchor.id
|
|
456
|
+
: SignalDeliverFrame.assert(JSON.parse(new TextDecoder().decode(dispatch.rawMessage))).runId;
|
|
457
|
+
const messageIds = messageIdsByRun.get(runId) ?? new Set();
|
|
458
|
+
messageIds.add(dispatch.messageId);
|
|
459
|
+
messageIdsByRun.set(runId, messageIds);
|
|
460
|
+
}
|
|
461
|
+
for (const [runId, messageIds] of messageIdsByRun) {
|
|
462
|
+
for (const accepted of await listAcceptedWorkflowDispatches(reads, runId, messageIds)) {
|
|
463
|
+
const persisted = unsettledByMessageId.get(accepted.messageId);
|
|
464
|
+
if (persisted?.kind !== accepted.kind)
|
|
465
|
+
continue;
|
|
466
|
+
await workflowRunDispatchStore.settle(anchor.id, accepted.messageId, now);
|
|
467
|
+
unsettledByMessageId.delete(accepted.messageId);
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
topLevelTerminalSettlementProjected =
|
|
471
|
+
(await readCommittedWorkflowRunLifecycle(reads, anchor.id)) ===
|
|
472
|
+
"terminal";
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
catch (error) {
|
|
476
|
+
logger.error `Workflow dispatch settlement failed for ${anchor.id}; accepted Git state remains authoritative and the retained payload will be retried: ${error instanceof Error ? error.message : String(error)}`;
|
|
477
|
+
}
|
|
478
|
+
if (topLevelTerminalSettlementProjected) {
|
|
479
|
+
try {
|
|
480
|
+
await workflowRunDispatchStore.failUnsettled(anchor.id, "workflow_run_terminal", `Workflow run ${anchorAddress} is terminal and cannot accept this dispatch`, now);
|
|
481
|
+
}
|
|
482
|
+
catch (error) {
|
|
483
|
+
logger.error `Failed to close unsettled workflow dispatches for terminal run ${anchorAddress}: ${error instanceof Error ? error.message : String(error)}`;
|
|
484
|
+
}
|
|
485
|
+
}
|
|
176
486
|
return { accepted: true };
|
|
177
487
|
},
|
|
178
488
|
};
|
|
179
489
|
}
|
|
180
490
|
/**
|
|
181
|
-
* Extract the
|
|
182
|
-
* Throws on any input the `@intx/types`-owned `
|
|
183
|
-
* rejects: missing or leading `@`, empty domain, or
|
|
184
|
-
* without the canonical `
|
|
491
|
+
* Extract the run id from an `<runId>@<domain>` run address.
|
|
492
|
+
* Throws on any input the `@intx/types`-owned `parseRunAddress`
|
|
493
|
+
* rejects: missing or leading `@`, empty domain, or a run id
|
|
494
|
+
* without the canonical `run_` prefix.
|
|
185
495
|
*/
|
|
186
496
|
export function parseAgentId(agentAddress) {
|
|
187
|
-
const parsed =
|
|
497
|
+
const parsed = parseRunAddress(agentAddress);
|
|
188
498
|
if (parsed === null) {
|
|
189
|
-
throw new Error(`Invalid
|
|
499
|
+
throw new Error(`Invalid run address: "${agentAddress}"`);
|
|
190
500
|
}
|
|
191
|
-
return parsed.
|
|
501
|
+
return parsed.runId;
|
|
192
502
|
}
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
503
|
+
/**
|
|
504
|
+
* Resolve a run address to the `workflow_run` endpoint backing it, keyed by
|
|
505
|
+
* the run's `address`. Every routable address names one self-anchored run --
|
|
506
|
+
* the deployment's anchor -- so this resolves the run's own address (the
|
|
507
|
+
* source `persistMail` depends on to record a triggered deployment's mail).
|
|
508
|
+
*/
|
|
509
|
+
export async function resolveRoutableAddress(db, address) {
|
|
510
|
+
const runRow = await db
|
|
511
|
+
.select({
|
|
512
|
+
id: workflowRun.id,
|
|
513
|
+
tenantId: workflowRun.tenantId,
|
|
514
|
+
publicKey: workflowRun.publicKey,
|
|
515
|
+
status: workflowRun.status,
|
|
516
|
+
principalId: workflowRun.principalId,
|
|
517
|
+
})
|
|
518
|
+
.from(workflowRun)
|
|
519
|
+
.where(and(eq(workflowRun.address, address), isNull(workflowRun.endedAt)))
|
|
520
|
+
.limit(1)
|
|
521
|
+
.then((rows) => rows[0]);
|
|
522
|
+
if (runRow === undefined) {
|
|
523
|
+
return undefined;
|
|
524
|
+
}
|
|
525
|
+
return {
|
|
526
|
+
id: runRow.id,
|
|
527
|
+
tenantId: runRow.tenantId,
|
|
528
|
+
address,
|
|
529
|
+
publicKey: runRow.publicKey,
|
|
530
|
+
status: runRow.status,
|
|
531
|
+
sessionId: await resolveRunSessionId(db, runRow.principalId),
|
|
532
|
+
};
|
|
197
533
|
}
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
534
|
+
/**
|
|
535
|
+
* A folded run has no session column; its session is the `agent_session` keyed
|
|
536
|
+
* by the run's principal. By default this is the live (not-yet-ended) session,
|
|
537
|
+
* matching routing semantics; `includeEnded` also resolves a stopped run's
|
|
538
|
+
* ended session, which mail history needs. Returns null when the run has no
|
|
539
|
+
* principal or no matching session. Transitional, alongside
|
|
540
|
+
* `RoutableEndpoint.sessionId`.
|
|
541
|
+
*/
|
|
542
|
+
export async function resolveRunSessionId(db, principalId, opts = {}) {
|
|
543
|
+
if (principalId === null) {
|
|
544
|
+
return null;
|
|
545
|
+
}
|
|
546
|
+
// One session per run principal (invariant), so limit(1) returns the whole
|
|
547
|
+
// history; order deterministically so a hypothetical second row cannot make
|
|
548
|
+
// the pick flap. If a run ever grows multiple sessions per principal this
|
|
549
|
+
// becomes a union and limit(1) silently truncates.
|
|
550
|
+
const conditions = [eq(agentSession.principalId, principalId)];
|
|
551
|
+
if (opts.includeEnded !== true) {
|
|
552
|
+
conditions.push(isNull(agentSession.endedAt));
|
|
553
|
+
}
|
|
554
|
+
const row = await db
|
|
555
|
+
.select({ id: agentSession.id })
|
|
556
|
+
.from(agentSession)
|
|
557
|
+
.where(and(...conditions))
|
|
558
|
+
.orderBy(asc(agentSession.createdAt))
|
|
559
|
+
.limit(1)
|
|
560
|
+
.then((rows) => rows[0]);
|
|
561
|
+
return row?.id ?? null;
|
|
562
|
+
}
|
|
563
|
+
/**
|
|
564
|
+
* The folded run that owns a session, or null when the session belongs to no
|
|
565
|
+
* run. This is the inverse of `resolveRunSessionId`: a mail-read path holds a
|
|
566
|
+
* `sessionMail.sessionId` and no address, so it recovers the owning run by
|
|
567
|
+
* joining `workflow_run` to `agent_session` on their shared principal (a folded
|
|
568
|
+
* run, its session, and its launch all key on the same `instancePrincipalId`).
|
|
569
|
+
* Scoped to the tenant and routed through `workflow_run` so the returned id is
|
|
570
|
+
* proven to name a real run of this tenant -- callers key an authorization
|
|
571
|
+
* subject on it, so a session held by a non-run principal must fail closed to
|
|
572
|
+
* null rather than resolve to a fabricated subject.
|
|
573
|
+
*/
|
|
574
|
+
export async function resolveRunIdForSession(db, sessionId, tenantId) {
|
|
575
|
+
// No `endedAt` filter: a stopped run's mail must stay fetchable, so the
|
|
576
|
+
// session resolves whether or not it has ended. The run principal is minted
|
|
577
|
+
// per launch and shared 1:1 by the run and its session, so at most one row
|
|
578
|
+
// matches; order deterministically anyway so a hypothetical second row cannot
|
|
579
|
+
// make the pick flap, mirroring `resolveRunSessionId`.
|
|
580
|
+
const row = await db
|
|
581
|
+
.select({ id: workflowRun.id })
|
|
582
|
+
.from(workflowRun)
|
|
583
|
+
.innerJoin(agentSession, eq(agentSession.principalId, workflowRun.principalId))
|
|
584
|
+
.where(and(eq(agentSession.id, sessionId), eq(workflowRun.tenantId, tenantId)))
|
|
585
|
+
.orderBy(asc(workflowRun.createdAt))
|
|
586
|
+
.limit(1)
|
|
587
|
+
.then((rows) => rows[0]);
|
|
588
|
+
return row?.id ?? null;
|
|
589
|
+
}
|
|
590
|
+
/**
|
|
591
|
+
* Shape a run row and its already-resolved routing address into the run
|
|
592
|
+
* record. Callers decide whether the run resolves at all -- only a top-level
|
|
593
|
+
* run (`isTopLevelRun`) does -- and pass the address they have narrowed; this
|
|
594
|
+
* only maps the columns, including the run's `endedAt ?? createdAt` stand-in
|
|
595
|
+
* for the absent `updatedAt`.
|
|
596
|
+
*/
|
|
597
|
+
export function runRowToRoutableRecord(run, address) {
|
|
598
|
+
return {
|
|
599
|
+
id: run.id,
|
|
600
|
+
tenantId: run.tenantId,
|
|
601
|
+
address,
|
|
602
|
+
publicKey: run.publicKey,
|
|
603
|
+
status: run.status,
|
|
604
|
+
createdAt: run.createdAt,
|
|
605
|
+
updatedAt: run.endedAt ?? run.createdAt,
|
|
606
|
+
endedAt: run.endedAt,
|
|
607
|
+
definitionId: run.definitionId,
|
|
608
|
+
principalId: run.principalId,
|
|
609
|
+
kernelId: run.kernelId,
|
|
610
|
+
sidecarId: run.sidecarId,
|
|
611
|
+
};
|
|
612
|
+
}
|
|
613
|
+
/**
|
|
614
|
+
* A run is a top-level run -- the addressable head of a deployment -- when it
|
|
615
|
+
* owns a routing address AND self-anchors (`anchorRunId === id`). A lazy child
|
|
616
|
+
* park row anchors on its parent (`anchorRunId !== id`) and carries no address;
|
|
617
|
+
* either condition excludes it. This is the single predicate the run read
|
|
618
|
+
* surface classifies on, so the resolver and the run list cannot drift.
|
|
619
|
+
*/
|
|
620
|
+
export function isTopLevelRun(row) {
|
|
621
|
+
return row.address !== null && row.anchorRunId === row.id;
|
|
622
|
+
}
|
|
623
|
+
/**
|
|
624
|
+
* Resolve a run id to its record. A run resolves only when it is a top-level
|
|
625
|
+
* run (`isTopLevelRun`): it owns a routing address and self-anchors. A child
|
|
626
|
+
* park row (address-null, anchored on its parent) is not served here.
|
|
627
|
+
*/
|
|
628
|
+
export async function findRoutableById(db, id, tenantId) {
|
|
629
|
+
const runRow = await db
|
|
630
|
+
.select({
|
|
631
|
+
id: workflowRun.id,
|
|
632
|
+
tenantId: workflowRun.tenantId,
|
|
633
|
+
address: workflowRun.address,
|
|
634
|
+
anchorRunId: workflowRun.anchorRunId,
|
|
635
|
+
publicKey: workflowRun.publicKey,
|
|
636
|
+
status: workflowRun.status,
|
|
637
|
+
createdAt: workflowRun.createdAt,
|
|
638
|
+
endedAt: workflowRun.endedAt,
|
|
639
|
+
principalId: workflowRun.principalId,
|
|
640
|
+
kernelId: workflowRun.kernelId,
|
|
641
|
+
sidecarId: workflowRun.sidecarId,
|
|
642
|
+
definitionId: workflowRun.definitionId,
|
|
643
|
+
})
|
|
644
|
+
.from(workflowRun)
|
|
645
|
+
.where(and(eq(workflowRun.id, id), eq(workflowRun.tenantId, tenantId)))
|
|
646
|
+
.limit(1)
|
|
647
|
+
.then((rows) => rows[0]);
|
|
648
|
+
// The `address === null` arm is redundant with `isTopLevelRun` (which already
|
|
649
|
+
// requires a non-null address) but narrows `address` from `string | null` to
|
|
650
|
+
// `string` for `runRowToRoutableRecord`, which `isTopLevelRun`'s boolean
|
|
651
|
+
// return cannot do.
|
|
652
|
+
if (runRow === undefined ||
|
|
653
|
+
!isTopLevelRun(runRow) ||
|
|
654
|
+
runRow.address === null ||
|
|
655
|
+
runRow.definitionId === null) {
|
|
656
|
+
return undefined;
|
|
202
657
|
}
|
|
203
|
-
return
|
|
658
|
+
return runRowToRoutableRecord({ ...runRow, definitionId: runRow.definitionId }, runRow.address);
|
|
204
659
|
}
|