@superjackfruitlabs/salaam 0.0.1 → 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/README.md +25 -17
- package/dist/src/a2a-backend.js +225 -0
- package/dist/src/a2a-inbound.js +487 -0
- package/dist/src/a2a-peer.js +370 -0
- package/dist/src/a2a-protocol.js +250 -0
- package/dist/src/agent-profiles.js +289 -0
- package/dist/src/children.js +132 -26
- package/dist/src/claude-provider.js +2 -179
- package/dist/src/cli-help.js +107 -6
- package/dist/src/cli.js +477 -13
- package/dist/src/config.js +26 -3
- package/dist/src/credentials.js +7 -2
- package/dist/src/external-runtime.js +1 -0
- package/dist/src/external-work.js +583 -0
- package/dist/src/harness-backend.js +466 -0
- package/dist/src/harness-container.js +450 -0
- package/dist/src/harness-model-gateway.js +104 -0
- package/dist/src/harness-prepare.js +88 -0
- package/dist/src/harness-probe.js +111 -0
- package/dist/src/harness-protocol.js +325 -0
- package/dist/src/harness-readiness.js +50 -0
- package/dist/src/harness-rpc.js +206 -0
- package/dist/src/harness-tool-identity.js +87 -0
- package/dist/src/harness-wire-bridge.js +387 -0
- package/dist/src/host.js +6 -1
- package/dist/src/house-extension.js +147 -64
- package/dist/src/live-provider.js +301 -0
- package/dist/src/managed.js +33 -3
- package/dist/src/protocol.js +9 -5
- package/dist/src/provider-auth.js +300 -0
- package/dist/src/provider-config.js +65 -17
- package/dist/src/provider-model.js +78 -0
- package/dist/src/provider-payload.js +264 -0
- package/dist/src/provider-transport.js +36 -0
- package/dist/src/provider-usage.js +59 -0
- package/dist/src/repository-snapshots.js +219 -0
- package/dist/src/service.js +274 -9
- package/dist/src/store.js +44 -2
- package/dist/src/supervisor.js +246 -35
- package/dist/src/task-budget.js +17 -8
- package/dist/src/task-context.js +15 -0
- package/dist/src/task-driver.js +38 -14
- package/dist/src/task-policy.js +20 -1
- package/dist/src/task-verification.js +3 -0
- package/dist/src/task-work.js +31 -0
- package/dist/src/team-work.js +378 -0
- package/dist/src/tui-view.js +65 -10
- package/dist/src/tui.js +4 -2
- package/dist/src/usage.js +18 -0
- package/dist/src/worker-boundary.js +298 -0
- package/dist/src/worker-workspaces.js +274 -0
- package/dist/src/workspaces.js +6 -2
- package/harness-runtime/codex-pins.json +10 -0
- package/harness-runtime/package-lock.json +560 -0
- package/harness-runtime/package.json +9 -0
- package/npm-shrinkwrap.json +57 -2
- package/package.json +2 -1
- package/src/a2a-backend.ts +254 -0
- package/src/a2a-inbound.ts +585 -0
- package/src/a2a-peer.ts +441 -0
- package/src/a2a-protocol.ts +286 -0
- package/src/agent-profiles.ts +379 -0
- package/src/children.ts +168 -27
- package/src/claude-provider.ts +2 -189
- package/src/cli-help.ts +107 -6
- package/src/cli.ts +602 -12
- package/src/config.ts +34 -3
- package/src/credentials.ts +9 -2
- package/src/external-runtime.ts +54 -0
- package/src/external-work.ts +663 -0
- package/src/harness-backend.ts +557 -0
- package/src/harness-container.ts +543 -0
- package/src/harness-model-gateway.ts +138 -0
- package/src/harness-prepare.ts +113 -0
- package/src/harness-probe.ts +150 -0
- package/src/harness-protocol.ts +388 -0
- package/src/harness-readiness.ts +71 -0
- package/src/harness-rpc.ts +244 -0
- package/src/harness-tool-identity.ts +91 -0
- package/src/harness-wire-bridge.ts +418 -0
- package/src/host.ts +11 -6
- package/src/house-extension.ts +165 -66
- package/src/live-provider.ts +347 -0
- package/src/managed.ts +43 -4
- package/src/protocol.ts +36 -8
- package/src/provider-auth.ts +336 -0
- package/src/provider-config.ts +89 -18
- package/src/provider-model.ts +86 -0
- package/src/provider-payload.ts +263 -0
- package/src/provider-transport.ts +51 -0
- package/src/provider-usage.ts +64 -0
- package/src/repository-snapshots.ts +259 -0
- package/src/service.ts +446 -11
- package/src/store.ts +60 -4
- package/src/supervisor.ts +294 -41
- package/src/task-budget.ts +24 -11
- package/src/task-context.ts +17 -0
- package/src/task-driver.ts +51 -15
- package/src/task-policy.ts +23 -3
- package/src/task-verification.ts +2 -0
- package/src/task-work.ts +46 -1
- package/src/team-work.ts +554 -0
- package/src/tui-client.ts +5 -0
- package/src/tui-view.ts +164 -12
- package/src/tui.ts +5 -2
- package/src/usage.ts +17 -0
- package/src/worker-boundary.ts +395 -0
- package/src/worker-workspaces.ts +390 -0
- package/src/workspaces.ts +4 -2
|
@@ -0,0 +1,663 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import type { Store } from './store.js';
|
|
3
|
+
import type { Artifacts } from './artifacts.js';
|
|
4
|
+
import type { ChildWork } from './children.js';
|
|
5
|
+
import type { TaskBudget } from './task-budget.js';
|
|
6
|
+
import type { WorkTask } from './task-work.js';
|
|
7
|
+
import type { ManagedWork } from './managed.js';
|
|
8
|
+
import type { ExternalBackend, ExternalRuntime, ExternalCheckpoint } from './external-runtime.js';
|
|
9
|
+
import { externalCapabilities, type WorkerBinding } from './worker-boundary.js';
|
|
10
|
+
import { readTree, validateTree, type Workspaces, type WorkspaceTree } from './workspaces.js';
|
|
11
|
+
import { canonical, digest, ServiceError, type ChildTask } from './protocol.js';
|
|
12
|
+
import { POLICY_DIGEST } from './config.js';
|
|
13
|
+
|
|
14
|
+
/** External invocations extend existing child attempts and task-step claims, never a second scheduler. */
|
|
15
|
+
export class ExternalWork {
|
|
16
|
+
private paused = false;
|
|
17
|
+
private active = new Map<string, Promise<void>>();
|
|
18
|
+
private failed = new Set<string>();
|
|
19
|
+
private owned = new Map<string, { child: ChildTask; run: string; runtime: ExternalRuntime }>();
|
|
20
|
+
constructor(
|
|
21
|
+
readonly store: Store,
|
|
22
|
+
readonly artifacts: Artifacts,
|
|
23
|
+
readonly children: ChildWork,
|
|
24
|
+
readonly budget: TaskBudget,
|
|
25
|
+
readonly managed: ManagedWork,
|
|
26
|
+
readonly workspaces: Workspaces,
|
|
27
|
+
readonly backend?: ExternalBackend,
|
|
28
|
+
readonly admitModelRequest?: (
|
|
29
|
+
session: string,
|
|
30
|
+
attempt: string,
|
|
31
|
+
input: number,
|
|
32
|
+
output: number,
|
|
33
|
+
) => void,
|
|
34
|
+
) {}
|
|
35
|
+
private policy(task: WorkTask): void {
|
|
36
|
+
if (this.paused)
|
|
37
|
+
throw new ServiceError('host_control_unavailable', 'External worker control is unavailable');
|
|
38
|
+
if (!task.limits.invocations || task.limits.token_units !== null)
|
|
39
|
+
throw new ServiceError(
|
|
40
|
+
'worker_budget_scope',
|
|
41
|
+
'Opaque workers require explicit limits.invocations and token_units:null; internal model usage remains unknown',
|
|
42
|
+
);
|
|
43
|
+
}
|
|
44
|
+
runtimes(): ExternalRuntime[] {
|
|
45
|
+
return this.backend?.runtimes() ?? [];
|
|
46
|
+
}
|
|
47
|
+
status() {
|
|
48
|
+
return {
|
|
49
|
+
available: !this.paused,
|
|
50
|
+
active_runs: this.active.size,
|
|
51
|
+
failed_workers: this.failed.size,
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
assign(id: string, task: WorkTask, runtimeId: string, tools: readonly string[]): void {
|
|
55
|
+
this.policy(task);
|
|
56
|
+
const runtime = this.runtimes().find((r) => r.id === runtimeId);
|
|
57
|
+
if (!runtime)
|
|
58
|
+
throw new ServiceError(
|
|
59
|
+
'runtime_unavailable',
|
|
60
|
+
'No operator-configured runtime with this identity',
|
|
61
|
+
);
|
|
62
|
+
this.backend!.validate(runtime);
|
|
63
|
+
this.backend!.assertAuthority?.(runtime, tools);
|
|
64
|
+
this.store.db
|
|
65
|
+
.prepare('INSERT INTO external_assignments VALUES (?,?,?)')
|
|
66
|
+
.run(id, canonical(runtime), digest(runtime));
|
|
67
|
+
}
|
|
68
|
+
assignment(id: string): ExternalRuntime | undefined {
|
|
69
|
+
const row = this.store.db
|
|
70
|
+
.prepare('SELECT * FROM external_assignments WHERE assignment_id=?')
|
|
71
|
+
.get(id);
|
|
72
|
+
if (!row) return;
|
|
73
|
+
const runtime = JSON.parse(String(row.runtime)) as ExternalRuntime;
|
|
74
|
+
if (digest(runtime) !== row.digest)
|
|
75
|
+
throw new ServiceError('worker_binding_changed', 'External assignment changed');
|
|
76
|
+
return runtime;
|
|
77
|
+
}
|
|
78
|
+
validateAssignment(id: string, task: WorkTask): void {
|
|
79
|
+
const runtime = this.assignment(id);
|
|
80
|
+
if (!runtime) return;
|
|
81
|
+
this.policy(task);
|
|
82
|
+
if (!this.backend)
|
|
83
|
+
throw new ServiceError('runtime_unavailable', 'Runtime configuration is unavailable');
|
|
84
|
+
this.backend.validate(runtime);
|
|
85
|
+
}
|
|
86
|
+
row(child: string): Record<string, any> {
|
|
87
|
+
const row = this.store.db.prepare('SELECT * FROM external_runs WHERE child_id=?').get(child);
|
|
88
|
+
if (!row) throw new ServiceError('not_found', 'External worker not found', 404);
|
|
89
|
+
return row;
|
|
90
|
+
}
|
|
91
|
+
stopped(child: string): boolean {
|
|
92
|
+
return this.row(child).stopped === 1;
|
|
93
|
+
}
|
|
94
|
+
owns(session: string): boolean {
|
|
95
|
+
if ([...this.owned.values()].some((r) => r.child.child_session === session)) return true;
|
|
96
|
+
const child = this.children.owned(session);
|
|
97
|
+
return !!child && !!this.store.workerBoundary.view(child.id).binding.external;
|
|
98
|
+
}
|
|
99
|
+
failControl(session: string, error: Error): void {
|
|
100
|
+
const child =
|
|
101
|
+
[...this.owned.values()].find((r) => r.child.child_session === session)?.child ??
|
|
102
|
+
this.children.owned(session);
|
|
103
|
+
if (!child) return;
|
|
104
|
+
this.failed.add(child.id);
|
|
105
|
+
try {
|
|
106
|
+
this.store.transaction(() => {
|
|
107
|
+
this.store.db
|
|
108
|
+
.prepare("UPDATE external_runs SET state='unknown',error=? WHERE child_id=?")
|
|
109
|
+
.run(error.message, child.id);
|
|
110
|
+
this.store.db
|
|
111
|
+
.prepare("UPDATE sessions SET state='unhealthy',capabilities='[]',error=? WHERE id=?")
|
|
112
|
+
.run(error.message, session);
|
|
113
|
+
this.store.event(child.parent_session, 'external.control_failed', {
|
|
114
|
+
child_id: child.id,
|
|
115
|
+
error: error.message,
|
|
116
|
+
});
|
|
117
|
+
});
|
|
118
|
+
} finally {
|
|
119
|
+
void this.cancel(child).catch(() => {});
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
failAll(error: Error): void {
|
|
123
|
+
this.paused = true;
|
|
124
|
+
for (const { child } of this.owned.values()) {
|
|
125
|
+
this.failed.add(child.id);
|
|
126
|
+
try {
|
|
127
|
+
this.failControl(child.child_session, error);
|
|
128
|
+
} catch {}
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
private binding(child: ChildTask): WorkerBinding {
|
|
132
|
+
const v = this.store.workerBoundary.view(child.id),
|
|
133
|
+
b = v.binding;
|
|
134
|
+
if (
|
|
135
|
+
!b.external ||
|
|
136
|
+
!b.assignment ||
|
|
137
|
+
!b.agent ||
|
|
138
|
+
!b.isolation ||
|
|
139
|
+
!b.repository ||
|
|
140
|
+
b.schema_version !== 1 ||
|
|
141
|
+
child.budget !== 1 ||
|
|
142
|
+
b.agent.owner_session !== child.parent_session ||
|
|
143
|
+
!b.input_artifact ||
|
|
144
|
+
b.runtime !== b.external.kind ||
|
|
145
|
+
digest(b) !== v.binding_digest ||
|
|
146
|
+
b.capability_digest !== digest(b.capabilities) ||
|
|
147
|
+
b.capability_digest !== digest(externalCapabilities(b.external, b.tools)) ||
|
|
148
|
+
b.bounds.scope !== 'invocations' ||
|
|
149
|
+
b.bounds.invocations !== 1 ||
|
|
150
|
+
b.owner_session !== child.parent_session ||
|
|
151
|
+
b.session_id !== child.child_session ||
|
|
152
|
+
b.attempt_id !== child.child_attempt ||
|
|
153
|
+
b.input_digest !== child.input_digest ||
|
|
154
|
+
b.parent_attempt !== child.parent_attempt ||
|
|
155
|
+
b.step_key !== child.step_key ||
|
|
156
|
+
canonical(b.tools) !== child.authority ||
|
|
157
|
+
digest(this.assignment(b.assignment.id)) !== digest(b.external) ||
|
|
158
|
+
b.policy_digest !== POLICY_DIGEST ||
|
|
159
|
+
b.profile_digest !== digest(b.profile) ||
|
|
160
|
+
b.agent.profile_digest !== b.profile_digest ||
|
|
161
|
+
b.route !== null
|
|
162
|
+
)
|
|
163
|
+
throw new ServiceError(
|
|
164
|
+
'worker_binding_changed',
|
|
165
|
+
'External worker ownership or admission changed',
|
|
166
|
+
);
|
|
167
|
+
const task = this.budget.task(child.parent_session);
|
|
168
|
+
const input = this.store.db
|
|
169
|
+
.prepare('SELECT text FROM submissions WHERE attempt_id=?')
|
|
170
|
+
.get(child.child_attempt);
|
|
171
|
+
if (!input || this.artifacts.read(b.input_artifact!) !== input.text)
|
|
172
|
+
throw new ServiceError('worker_binding_changed', 'External prompt artifact changed');
|
|
173
|
+
if (
|
|
174
|
+
!task ||
|
|
175
|
+
task.id !== b.owner_task ||
|
|
176
|
+
['cancelled', 'succeeded', 'succeeded_with_exception'].includes(task.state)
|
|
177
|
+
)
|
|
178
|
+
throw new ServiceError('task_settled', 'External task no longer admits work');
|
|
179
|
+
this.policy(task);
|
|
180
|
+
this.store.workerBoundary.profiles!.validate(b.agent);
|
|
181
|
+
this.store.workerBoundary.assertAssignment!(b);
|
|
182
|
+
this.store.workerBoundary.assertIsolation!(b);
|
|
183
|
+
this.backend?.validate(b.external);
|
|
184
|
+
if (!this.backend)
|
|
185
|
+
throw new ServiceError('runtime_unavailable', 'Runtime configuration is unavailable');
|
|
186
|
+
this.budget.check(child.parent_session);
|
|
187
|
+
return b;
|
|
188
|
+
}
|
|
189
|
+
prepare(child: ChildTask): void {
|
|
190
|
+
// Called inside the same transaction as child ownership and the task-step claim.
|
|
191
|
+
const b = this.store.workerBoundary.view(child.id).binding,
|
|
192
|
+
task = this.budget.task(child.parent_session)!;
|
|
193
|
+
this.policy(task);
|
|
194
|
+
const spent = Number(
|
|
195
|
+
this.store.db
|
|
196
|
+
.prepare('SELECT count(*) AS n FROM task_invocations WHERE task_id=?')
|
|
197
|
+
.get(task.id)!.n,
|
|
198
|
+
);
|
|
199
|
+
if (spent >= task.limits.invocations!)
|
|
200
|
+
throw new ServiceError('invocation_budget_exhausted', 'Task invocation allowance exhausted');
|
|
201
|
+
this.store.db
|
|
202
|
+
.prepare('INSERT INTO task_invocations VALUES (?,?,?,?,?)')
|
|
203
|
+
.run(child.child_attempt, task.id, b.external!.id, child.input_digest, Date.now());
|
|
204
|
+
this.store.db
|
|
205
|
+
.prepare("INSERT INTO external_runs(child_id,run_id,state,epoch) VALUES (?,?,'prepared',1)")
|
|
206
|
+
.run(child.id, randomUUID());
|
|
207
|
+
this.store.event(child.parent_session, 'external.invocation_admitted', {
|
|
208
|
+
child_id: child.id,
|
|
209
|
+
runtime: b.external,
|
|
210
|
+
usage: 'unknown',
|
|
211
|
+
invocations_spent: spent + 1,
|
|
212
|
+
});
|
|
213
|
+
}
|
|
214
|
+
assertActive(session: string, epoch: number, attempt: string): void {
|
|
215
|
+
const child = this.children.owned(session);
|
|
216
|
+
if (!child || child.child_attempt !== attempt || this.paused || this.failed.has(child.id))
|
|
217
|
+
throw new ServiceError('invocation_inactive', 'External invocation is inactive');
|
|
218
|
+
const row = this.row(child.id);
|
|
219
|
+
const worker = this.store.workerBoundary.view(child.id);
|
|
220
|
+
const cp = row.checkpoint ? (JSON.parse(row.checkpoint) as ExternalCheckpoint) : null;
|
|
221
|
+
if (
|
|
222
|
+
row.epoch !== epoch ||
|
|
223
|
+
this.store.session(session).epoch !== epoch ||
|
|
224
|
+
worker.worker_epoch !== epoch ||
|
|
225
|
+
worker.generation !== 1 ||
|
|
226
|
+
row.stopped ||
|
|
227
|
+
!['starting', 'running'].includes(row.state) ||
|
|
228
|
+
child.state !== 'accepted' ||
|
|
229
|
+
this.store.cancelled(attempt) ||
|
|
230
|
+
(cp && ['terminal', 'unknown'].includes(cp.state))
|
|
231
|
+
)
|
|
232
|
+
throw new ServiceError('invocation_inactive', 'External invocation lost authority');
|
|
233
|
+
this.binding(child);
|
|
234
|
+
}
|
|
235
|
+
async start(child: ChildTask): Promise<void> {
|
|
236
|
+
const row = this.row(child.id);
|
|
237
|
+
if (row.state !== 'prepared' || this.paused) return;
|
|
238
|
+
const b = this.binding(child),
|
|
239
|
+
before = readTree(this.workspaces.get(child.child_session).root);
|
|
240
|
+
this.store.transaction(() => {
|
|
241
|
+
if (this.row(child.id).state !== 'prepared')
|
|
242
|
+
throw new ServiceError('harness_replay', 'Invocation has already crossed dispatch intent');
|
|
243
|
+
this.store.db
|
|
244
|
+
.prepare("UPDATE external_runs SET state='starting' WHERE child_id=?")
|
|
245
|
+
.run(child.id);
|
|
246
|
+
this.store.db
|
|
247
|
+
.prepare(
|
|
248
|
+
"UPDATE child_workers SET generation=1,worker_epoch=1,dispatch_state='dispatching' WHERE child_id=?",
|
|
249
|
+
)
|
|
250
|
+
.run(child.id);
|
|
251
|
+
this.store.db
|
|
252
|
+
.prepare("UPDATE sessions SET epoch=1,state='ready',capabilities='[]' WHERE id=?")
|
|
253
|
+
.run(child.child_session);
|
|
254
|
+
this.store.db
|
|
255
|
+
.prepare("UPDATE attempts SET state='dispatching',worker_epoch=1 WHERE id=?")
|
|
256
|
+
.run(child.child_attempt);
|
|
257
|
+
this.store.projectTaskAttempt?.(this.store.attempt(child.child_attempt));
|
|
258
|
+
this.store.event(child.parent_session, 'external.dispatch_intent', {
|
|
259
|
+
child_id: child.id,
|
|
260
|
+
run_id: row.run_id,
|
|
261
|
+
});
|
|
262
|
+
});
|
|
263
|
+
this.owned.set(child.id, { child, run: String(row.run_id), runtime: b.external! });
|
|
264
|
+
const operations = new Map<string, string>();
|
|
265
|
+
let result: unknown, failure: unknown;
|
|
266
|
+
const running = (async () => {
|
|
267
|
+
try {
|
|
268
|
+
const prompt = String(
|
|
269
|
+
this.store.db
|
|
270
|
+
.prepare('SELECT text FROM submissions WHERE attempt_id=?')
|
|
271
|
+
.get(child.child_attempt)!.text,
|
|
272
|
+
);
|
|
273
|
+
result = (
|
|
274
|
+
await this.backend!.execute(b, String(row.run_id), prompt, before, {
|
|
275
|
+
failed: (error) => this.failControl(child.child_session, error),
|
|
276
|
+
modelRequest: (id, bytes, output) => {
|
|
277
|
+
this.assertActive(child.child_session, 1, child.child_attempt);
|
|
278
|
+
const current = this.row(child.id);
|
|
279
|
+
const checkpoint = current.checkpoint
|
|
280
|
+
? (JSON.parse(current.checkpoint) as ExternalCheckpoint)
|
|
281
|
+
: null;
|
|
282
|
+
if (!checkpoint || !['dispatching', 'running'].includes(checkpoint.state))
|
|
283
|
+
throw new ServiceError(
|
|
284
|
+
'invocation_inactive',
|
|
285
|
+
'No durable prompt dispatch admits a model request',
|
|
286
|
+
);
|
|
287
|
+
if (
|
|
288
|
+
!Number.isSafeInteger(bytes) ||
|
|
289
|
+
bytes < 0 ||
|
|
290
|
+
bytes > 2 * 1024 * 1024 ||
|
|
291
|
+
!Number.isSafeInteger(output) ||
|
|
292
|
+
output < 1
|
|
293
|
+
)
|
|
294
|
+
throw new ServiceError('invalid_budget', 'Invalid external gateway bounds');
|
|
295
|
+
this.store.transaction(() => {
|
|
296
|
+
const n = Number(
|
|
297
|
+
this.store.db
|
|
298
|
+
.prepare('SELECT count(*) AS n FROM external_model_requests WHERE child_id=?')
|
|
299
|
+
.get(child.id)!.n,
|
|
300
|
+
);
|
|
301
|
+
if (
|
|
302
|
+
!b.external!.max_model_requests ||
|
|
303
|
+
n >= b.external!.max_model_requests ||
|
|
304
|
+
!b.external!.output_cap ||
|
|
305
|
+
output > b.external!.output_cap
|
|
306
|
+
)
|
|
307
|
+
throw Error('Frozen harness model gateway allowance exhausted');
|
|
308
|
+
this.admitModelRequest?.(child.child_session, child.child_attempt, bytes, output);
|
|
309
|
+
this.store.db
|
|
310
|
+
.prepare('INSERT INTO external_model_requests VALUES (?,?,?,?)')
|
|
311
|
+
.run(id, child.id, bytes, output);
|
|
312
|
+
this.store.event(child.parent_session, 'external.model_request_admitted', {
|
|
313
|
+
child_id: child.id,
|
|
314
|
+
id,
|
|
315
|
+
bytes,
|
|
316
|
+
output,
|
|
317
|
+
});
|
|
318
|
+
});
|
|
319
|
+
},
|
|
320
|
+
assertActive: () => this.assertActive(child.child_session, 1, child.child_attempt),
|
|
321
|
+
receipt: (value) => {
|
|
322
|
+
this.save(child, 'receipt', value);
|
|
323
|
+
return undefined;
|
|
324
|
+
},
|
|
325
|
+
checkpoint: (cp) => {
|
|
326
|
+
if (cp.worker_id !== b.worker_id || cp.attempt_id !== b.attempt_id)
|
|
327
|
+
throw new ServiceError('harness_identity', 'Checkpoint belongs to another worker');
|
|
328
|
+
if (cp.dialect === 'a2a' && cp.run_id !== row.run_id)
|
|
329
|
+
throw Error('Peer checkpoint belongs to another run');
|
|
330
|
+
this.save(child, 'checkpoint', cp);
|
|
331
|
+
return undefined;
|
|
332
|
+
},
|
|
333
|
+
permission: async (method, params) => {
|
|
334
|
+
this.assertActive(child.child_session, 1, child.child_attempt);
|
|
335
|
+
const current = this.row(child.id);
|
|
336
|
+
if (
|
|
337
|
+
!current.checkpoint ||
|
|
338
|
+
!['dispatching', 'running'].includes(JSON.parse(current.checkpoint).state)
|
|
339
|
+
)
|
|
340
|
+
return 'deny';
|
|
341
|
+
const scope = params.scope as
|
|
342
|
+
{ id?: unknown; kind?: unknown; arguments?: unknown } | undefined;
|
|
343
|
+
if (
|
|
344
|
+
!scope ||
|
|
345
|
+
typeof scope.id !== 'string' ||
|
|
346
|
+
!['write', 'command'].includes(String(scope.kind)) ||
|
|
347
|
+
!b.tools.includes(scope.kind === 'write' ? 'salaam_files' : 'salaam_command') ||
|
|
348
|
+
canonical(params).length > 16384
|
|
349
|
+
)
|
|
350
|
+
return 'deny';
|
|
351
|
+
const prepared = await this.managed.prepare(
|
|
352
|
+
child.child_session,
|
|
353
|
+
1,
|
|
354
|
+
child.child_attempt,
|
|
355
|
+
scope.id,
|
|
356
|
+
'harness',
|
|
357
|
+
canonical({ runtime: b.external!.id, method, scope }),
|
|
358
|
+
canonical(params),
|
|
359
|
+
);
|
|
360
|
+
operations.set(scope.id, prepared.operation.id);
|
|
361
|
+
for (;;) {
|
|
362
|
+
try {
|
|
363
|
+
this.assertActive(child.child_session, 1, child.child_attempt);
|
|
364
|
+
} catch {
|
|
365
|
+
return 'deny';
|
|
366
|
+
}
|
|
367
|
+
const approval = this.managed.approval(prepared.approval.id);
|
|
368
|
+
if (approval.state !== 'pending')
|
|
369
|
+
return this.managed.takePermission(prepared.operation.id) ? 'approve' : 'deny';
|
|
370
|
+
await new Promise((r) => setTimeout(r, 25));
|
|
371
|
+
}
|
|
372
|
+
},
|
|
373
|
+
toolResult: (id, success, receipt) => {
|
|
374
|
+
const op = operations.get(id);
|
|
375
|
+
if (op) this.managed.permissionResult(op, success, receipt);
|
|
376
|
+
},
|
|
377
|
+
})
|
|
378
|
+
).result;
|
|
379
|
+
} catch (error) {
|
|
380
|
+
failure = error;
|
|
381
|
+
}
|
|
382
|
+
try {
|
|
383
|
+
let receipt: unknown = null;
|
|
384
|
+
try {
|
|
385
|
+
receipt = this.cleanupReceipt(child, b.external!);
|
|
386
|
+
} catch (error) {
|
|
387
|
+
failure ??= error;
|
|
388
|
+
}
|
|
389
|
+
const stopped = await this.backend!.stop(String(row.run_id), b.external!, receipt);
|
|
390
|
+
this.recordStop(child, String(row.run_id), b.external!, stopped);
|
|
391
|
+
} catch (error) {
|
|
392
|
+
failure = error;
|
|
393
|
+
}
|
|
394
|
+
try {
|
|
395
|
+
if (this.failed.has(child.id)) failure ??= Error('Required external worker control failed');
|
|
396
|
+
if (!failure && !this.store.cancelled(child.child_attempt)) {
|
|
397
|
+
const cp = JSON.parse(String(this.row(child.id).checkpoint)) as ExternalCheckpoint;
|
|
398
|
+
if (!this.stopped(child.id) || cp.state !== 'terminal')
|
|
399
|
+
throw Error('No confirmed terminal invocation');
|
|
400
|
+
const after = this.backend!.collect(String(row.run_id), b.external!);
|
|
401
|
+
validateTree(after);
|
|
402
|
+
if (!b.tools.includes('salaam_files') && digest(after) !== digest(before))
|
|
403
|
+
throw Error('Read-only worker changed its draft');
|
|
404
|
+
const artifact = this.artifacts.prepare(child.child_session, canonical(after));
|
|
405
|
+
this.store.transaction(() => {
|
|
406
|
+
artifact.commit();
|
|
407
|
+
this.store.db
|
|
408
|
+
.prepare('UPDATE external_runs SET result=?,import_artifact=? WHERE child_id=?')
|
|
409
|
+
.run(canonical(result), artifact.id, child.id);
|
|
410
|
+
this.store.event(child.parent_session, 'external.import_prepared', {
|
|
411
|
+
child_id: child.id,
|
|
412
|
+
artifact_id: artifact.id,
|
|
413
|
+
digest: digest(after),
|
|
414
|
+
});
|
|
415
|
+
});
|
|
416
|
+
const w = this.workspaces.verify(child.child_session);
|
|
417
|
+
this.workspaces.replace(
|
|
418
|
+
child.child_session,
|
|
419
|
+
'external-import:' + row.run_id,
|
|
420
|
+
digest(after),
|
|
421
|
+
w.revision,
|
|
422
|
+
before,
|
|
423
|
+
after,
|
|
424
|
+
);
|
|
425
|
+
}
|
|
426
|
+
} catch (error) {
|
|
427
|
+
failure = error;
|
|
428
|
+
}
|
|
429
|
+
this.managed.invalidateWorker(child.child_session, 1);
|
|
430
|
+
this.store.transaction(() => {
|
|
431
|
+
const state =
|
|
432
|
+
this.store.cancelled(child.child_attempt) && this.stopped(child.id)
|
|
433
|
+
? 'cancelled'
|
|
434
|
+
: failure
|
|
435
|
+
? 'interrupted'
|
|
436
|
+
: 'completed';
|
|
437
|
+
this.store.db
|
|
438
|
+
.prepare('UPDATE external_runs SET state=?,error=? WHERE child_id=?')
|
|
439
|
+
.run(
|
|
440
|
+
state === 'completed' ? 'terminal' : 'unknown',
|
|
441
|
+
failure ? String(failure).slice(0, 4096) : null,
|
|
442
|
+
child.id,
|
|
443
|
+
);
|
|
444
|
+
this.store.db
|
|
445
|
+
.prepare('UPDATE sessions SET state=?,error=? WHERE id=?')
|
|
446
|
+
.run(
|
|
447
|
+
this.stopped(child.id) ? 'exited' : 'unhealthy',
|
|
448
|
+
failure ? String(failure).slice(0, 4096) : null,
|
|
449
|
+
child.child_session,
|
|
450
|
+
);
|
|
451
|
+
this.store.event(child.parent_session, 'external.settled', {
|
|
452
|
+
child_id: child.id,
|
|
453
|
+
state,
|
|
454
|
+
stopped: this.stopped(child.id),
|
|
455
|
+
verified_success: false,
|
|
456
|
+
});
|
|
457
|
+
});
|
|
458
|
+
this.store.finish(
|
|
459
|
+
child.child_attempt,
|
|
460
|
+
this.store.cancelled(child.child_attempt) && this.stopped(child.id)
|
|
461
|
+
? 'cancelled'
|
|
462
|
+
: failure
|
|
463
|
+
? 'interrupted'
|
|
464
|
+
: 'completed',
|
|
465
|
+
failure ? null : canonical({ runtime: b.external!.id, result, verified_success: false }),
|
|
466
|
+
failure ? String(failure).slice(0, 4096) : null,
|
|
467
|
+
);
|
|
468
|
+
})();
|
|
469
|
+
this.active.set(child.id, running);
|
|
470
|
+
try {
|
|
471
|
+
await running;
|
|
472
|
+
} finally {
|
|
473
|
+
this.active.delete(child.id);
|
|
474
|
+
this.owned.delete(child.id);
|
|
475
|
+
}
|
|
476
|
+
}
|
|
477
|
+
private save(child: ChildTask, field: 'checkpoint' | 'receipt', value: unknown): void {
|
|
478
|
+
const bytes = canonical(value);
|
|
479
|
+
if (bytes.length > 1048576) throw Error('External checkpoint exceeds bound');
|
|
480
|
+
this.store.transaction(() => {
|
|
481
|
+
if (
|
|
482
|
+
field === 'checkpoint' &&
|
|
483
|
+
['dispatching', 'running'].includes((value as ExternalCheckpoint).state)
|
|
484
|
+
) {
|
|
485
|
+
this.store.db
|
|
486
|
+
.prepare("UPDATE external_runs SET state='running' WHERE child_id=?")
|
|
487
|
+
.run(child.id);
|
|
488
|
+
this.store.db
|
|
489
|
+
.prepare("UPDATE child_workers SET dispatch_state='accepted' WHERE child_id=?")
|
|
490
|
+
.run(child.id);
|
|
491
|
+
this.store.db
|
|
492
|
+
.prepare("UPDATE attempts SET state='running' WHERE id=?")
|
|
493
|
+
.run(child.child_attempt);
|
|
494
|
+
}
|
|
495
|
+
this.store.db
|
|
496
|
+
.prepare(`UPDATE external_runs SET ${field}=? WHERE child_id=?`)
|
|
497
|
+
.run(bytes, child.id);
|
|
498
|
+
this.store.event(child.parent_session, 'external.' + field, {
|
|
499
|
+
child_id: child.id,
|
|
500
|
+
digest: digest(value),
|
|
501
|
+
});
|
|
502
|
+
});
|
|
503
|
+
}
|
|
504
|
+
async cancel(child: ChildTask): Promise<void> {
|
|
505
|
+
const cached = this.owned.get(child.id);
|
|
506
|
+
const runtime = cached?.runtime ?? this.store.workerBoundary.view(child.id).binding.external;
|
|
507
|
+
const run = cached?.run ?? String(this.row(child.id).run_id);
|
|
508
|
+
let failure: unknown;
|
|
509
|
+
try {
|
|
510
|
+
this.managed.invalidateWorker(child.child_session, 1);
|
|
511
|
+
} catch (error) {
|
|
512
|
+
failure = error;
|
|
513
|
+
}
|
|
514
|
+
if (this.backend && runtime) {
|
|
515
|
+
let receipt: unknown = null;
|
|
516
|
+
try {
|
|
517
|
+
receipt = this.cleanupReceipt(child, runtime);
|
|
518
|
+
} catch (error) {
|
|
519
|
+
failure ??= error;
|
|
520
|
+
}
|
|
521
|
+
const stopped = await this.backend.stop(run, runtime, receipt);
|
|
522
|
+
this.recordStop(child, run, runtime, stopped);
|
|
523
|
+
}
|
|
524
|
+
if (failure) throw failure;
|
|
525
|
+
}
|
|
526
|
+
private cleanupReceipt(child: ChildTask, runtime: ExternalRuntime): unknown {
|
|
527
|
+
const row = this.row(child.id);
|
|
528
|
+
if (runtime.kind === 'a2a' && row.checkpoint) {
|
|
529
|
+
const cp = JSON.parse(row.checkpoint),
|
|
530
|
+
b = this.store.workerBoundary.view(child.id).binding;
|
|
531
|
+
if (
|
|
532
|
+
cp.dialect !== 'a2a' ||
|
|
533
|
+
cp.worker_id !== b.worker_id ||
|
|
534
|
+
cp.attempt_id !== child.child_attempt ||
|
|
535
|
+
cp.run_id !== row.run_id
|
|
536
|
+
)
|
|
537
|
+
throw Error('Peer cleanup identity changed');
|
|
538
|
+
return cp;
|
|
539
|
+
}
|
|
540
|
+
return row.receipt ? JSON.parse(row.receipt) : null;
|
|
541
|
+
}
|
|
542
|
+
private recordStop(
|
|
543
|
+
child: ChildTask,
|
|
544
|
+
run: string,
|
|
545
|
+
runtime: ExternalRuntime,
|
|
546
|
+
evidence: unknown,
|
|
547
|
+
): void {
|
|
548
|
+
const bytes = evidence === undefined ? null : canonical(evidence);
|
|
549
|
+
if (bytes && bytes.length > 1048576) throw Error('Stop receipt exceeds bound');
|
|
550
|
+
const cp =
|
|
551
|
+
runtime.kind === 'a2a'
|
|
552
|
+
? (evidence as { checkpoint?: import('./a2a-protocol.js').A2ACheckpoint } | undefined)
|
|
553
|
+
?.checkpoint
|
|
554
|
+
: undefined;
|
|
555
|
+
if (cp) {
|
|
556
|
+
const b = this.store.workerBoundary.view(child.id).binding;
|
|
557
|
+
if (
|
|
558
|
+
cp.worker_id !== b.worker_id ||
|
|
559
|
+
cp.attempt_id !== child.child_attempt ||
|
|
560
|
+
cp.run_id !== run
|
|
561
|
+
)
|
|
562
|
+
throw Error('Stop receipt belongs to another worker');
|
|
563
|
+
}
|
|
564
|
+
this.store.transaction(() => {
|
|
565
|
+
this.store.db
|
|
566
|
+
.prepare(
|
|
567
|
+
'UPDATE external_runs SET stopped=1,receipt=coalesce(?,receipt),checkpoint=coalesce(?,checkpoint) WHERE child_id=? AND run_id=?',
|
|
568
|
+
)
|
|
569
|
+
.run(bytes, cp ? canonical(cp) : null, child.id, run);
|
|
570
|
+
this.store.event(child.parent_session, 'external.stop_confirmed', {
|
|
571
|
+
child_id: child.id,
|
|
572
|
+
run_id: run,
|
|
573
|
+
scope:
|
|
574
|
+
runtime.kind === 'a2a'
|
|
575
|
+
? ((evidence as { scope?: string } | undefined)?.scope ?? 'remote-terminal-receipt')
|
|
576
|
+
: 'owned-processes',
|
|
577
|
+
evidence_digest: bytes ? digest(evidence) : null,
|
|
578
|
+
});
|
|
579
|
+
});
|
|
580
|
+
}
|
|
581
|
+
async recover(): Promise<void> {
|
|
582
|
+
for (const r of this.store.db.prepare('SELECT * FROM external_runs WHERE stopped=0').all()) {
|
|
583
|
+
const child = this.store.db
|
|
584
|
+
.prepare('SELECT * FROM child_tasks WHERE id=?')
|
|
585
|
+
.get(r.child_id!) as unknown as ChildTask;
|
|
586
|
+
let error = 'Daemon restart interrupted external work; prompt will not be replayed';
|
|
587
|
+
try {
|
|
588
|
+
await this.cancel(child);
|
|
589
|
+
} catch (e) {
|
|
590
|
+
error = String(e);
|
|
591
|
+
}
|
|
592
|
+
this.store.transaction(() => {
|
|
593
|
+
this.store.db
|
|
594
|
+
.prepare("UPDATE external_runs SET state='unknown',error=? WHERE child_id=?")
|
|
595
|
+
.run(error, child.id);
|
|
596
|
+
this.store.event(child.parent_session, 'external.recovered_unknown', {
|
|
597
|
+
child_id: child.id,
|
|
598
|
+
stopped: this.stopped(child.id),
|
|
599
|
+
error,
|
|
600
|
+
});
|
|
601
|
+
});
|
|
602
|
+
}
|
|
603
|
+
}
|
|
604
|
+
async cancelRoot(session: string): Promise<void> {
|
|
605
|
+
for (const child of this.children.list(session))
|
|
606
|
+
if (child.worker?.binding.external && !this.stopped(child.id)) await this.cancel(child);
|
|
607
|
+
}
|
|
608
|
+
async reconcile(
|
|
609
|
+
child: ChildTask,
|
|
610
|
+
request: string,
|
|
611
|
+
runId: string,
|
|
612
|
+
authorize: () => void,
|
|
613
|
+
): Promise<unknown> {
|
|
614
|
+
let fresh = false;
|
|
615
|
+
this.store.request(
|
|
616
|
+
'local-operator',
|
|
617
|
+
request,
|
|
618
|
+
{ kind: 'external-reconcile', child: child.id, runId },
|
|
619
|
+
() => {
|
|
620
|
+
authorize();
|
|
621
|
+
const row = this.row(child.id);
|
|
622
|
+
if (row.run_id !== runId || row.state !== 'unknown' || this.active.has(child.id))
|
|
623
|
+
throw new ServiceError(
|
|
624
|
+
'external_reconcile_stale',
|
|
625
|
+
'Reconcile an inactive uncertain run with its exact identity',
|
|
626
|
+
);
|
|
627
|
+
this.store.event(child.parent_session, 'external.reconcile_intent', {
|
|
628
|
+
child_id: child.id,
|
|
629
|
+
run_id: runId,
|
|
630
|
+
});
|
|
631
|
+
fresh = true;
|
|
632
|
+
return { child_id: child.id, run_id: runId };
|
|
633
|
+
},
|
|
634
|
+
);
|
|
635
|
+
if (fresh) await this.cancel(child);
|
|
636
|
+
return this.view(child.id);
|
|
637
|
+
}
|
|
638
|
+
view(child: string): unknown {
|
|
639
|
+
const r = this.row(child);
|
|
640
|
+
return {
|
|
641
|
+
...r,
|
|
642
|
+
checkpoint: r.checkpoint ? JSON.parse(r.checkpoint) : null,
|
|
643
|
+
receipt: r.receipt ? JSON.parse(r.receipt) : null,
|
|
644
|
+
result: r.result ? JSON.parse(r.result) : null,
|
|
645
|
+
verified_success: false,
|
|
646
|
+
};
|
|
647
|
+
}
|
|
648
|
+
async close(): Promise<void> {
|
|
649
|
+
this.paused = true;
|
|
650
|
+
for (const id of this.active.keys()) {
|
|
651
|
+
const child =
|
|
652
|
+
this.owned.get(id)?.child ??
|
|
653
|
+
(this.store.db
|
|
654
|
+
.prepare('SELECT * FROM child_tasks WHERE id=?')
|
|
655
|
+
.get(id) as unknown as ChildTask);
|
|
656
|
+
try {
|
|
657
|
+
await this.cancel(child);
|
|
658
|
+
} catch {}
|
|
659
|
+
}
|
|
660
|
+
await Promise.allSettled(this.active.values());
|
|
661
|
+
await this.backend?.close?.();
|
|
662
|
+
}
|
|
663
|
+
}
|