@lobu/runtime-device 24.2.0 → 24.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/component.json +1 -1
- package/dist/dependencies.bun.lock +43 -129
- package/npm-shrinkwrap.json +206 -690
- package/package.json +1 -1
- package/vendor/cli/dist/bundled-skills/lobu/SKILL.md +21 -3
- package/vendor/cli/dist/bundled-skills/lobu-connect/SKILL.md +6 -0
- package/vendor/cli/dist/commands/chat.d.ts +0 -12
- package/vendor/cli/dist/commands/chat.js +1 -1
- package/vendor/cli/dist/commands/daemon.js +36 -19
- package/vendor/cli/dist/commands/doctor.d.ts +0 -6
- package/vendor/cli/dist/commands/doctor.js +1 -1
- package/vendor/cli/dist/commands/memory/_lib/seed-cmd.d.ts +0 -8
- package/vendor/cli/dist/commands/memory/_lib/seed-cmd.js +1 -1
- package/vendor/cli/dist/config/define.d.ts +1 -1
- package/vendor/cli/dist/connectors/gchat.ts +31 -1
- package/vendor/cli/dist/connectors/postgres.ts +9 -5
- package/vendor/cli/dist/internal/cli-update-check.d.ts +33 -0
- package/vendor/cli/dist/internal/cli-update-check.js +98 -0
- package/vendor/cli/dist/internal/credentials.js +10 -36
- package/vendor/cli/dist/internal/gateway-url.d.ts +0 -8
- package/vendor/cli/dist/internal/gateway-url.js +1 -1
- package/vendor/cli/dist/runtime-components.json +4 -4
- package/vendor/cli/package.json +1 -1
- package/vendor/connector-sdk/dist/connector-types.d.ts +29 -0
- package/vendor/connector-sdk/dist/index.d.ts +3 -1
- package/vendor/connector-sdk/dist/index.js +4 -0
- package/vendor/connector-sdk/dist/redact.d.ts +10 -0
- package/vendor/{connector-worker/dist/executor → connector-sdk/dist}/redact.js +0 -49
- package/vendor/connector-sdk/dist/types.d.ts +2 -1
- package/vendor/connector-sdk/package.json +6 -1
- package/vendor/connector-worker/dist/daemon/acp-adapters/THIRD_PARTY_LICENSES.txt +1 -1
- package/vendor/connector-worker/dist/daemon/acp-adapters/claude.js +7339 -2136
- package/vendor/connector-worker/dist/daemon/agent-turn.js +58 -3
- package/vendor/connector-worker/dist/daemon/automation.js +1 -1
- package/vendor/connector-worker/dist/daemon/client.d.ts +15 -1
- package/vendor/connector-worker/dist/daemon/client.js +57 -4
- package/vendor/connector-worker/dist/daemon/executor.d.ts +0 -28
- package/vendor/connector-worker/dist/daemon/executor.js +27 -12
- package/vendor/connector-worker/dist/daemon/index.d.ts +1 -1
- package/vendor/connector-worker/dist/daemon/index.js +1 -1
- package/vendor/connector-worker/dist/daemon/log.js +2 -1
- package/vendor/connector-worker/dist/daemon/poll-loop.js +32 -8
- package/vendor/connector-worker/dist/daemon/start.d.ts +0 -1
- package/vendor/connector-worker/dist/daemon/start.js +1 -1
- package/vendor/connector-worker/dist/daemon/terminal-delivery.d.ts +1 -0
- package/vendor/connector-worker/dist/daemon/terminal-delivery.js +10 -6
- package/vendor/connector-worker/dist/executor/isolate.js +1 -1
- package/vendor/connector-worker/dist/isolate/bridge.d.ts +6 -2
- package/vendor/connector-worker/dist/isolate/bridge.js +14 -8
- package/vendor/connector-worker/package.json +2 -2
- package/vendor/core/dist/connector-query-errors.d.ts +6 -0
- package/vendor/core/dist/connector-query-errors.js +4 -0
- package/vendor/core/dist/contracts/tools/manage-automations.js +1 -1
- package/vendor/core/dist/contracts/tools/manage-entity-schema.js +1 -1
- package/vendor/core/dist/contracts/tools/manage-entity.d.ts +49 -20
- package/vendor/core/dist/contracts/tools/manage-entity.js +23 -5
- package/vendor/core/dist/contracts/worker/device-automation.d.ts +1 -1
- package/vendor/core/dist/contracts/worker/device-automation.js +1 -1
- package/vendor/core/dist/errors.d.ts +0 -20
- package/vendor/core/dist/errors.js +1 -25
- package/vendor/core/dist/index.d.ts +1 -0
- package/vendor/core/dist/index.js +4 -1
- package/vendor/core/dist/logger.js +8 -2
- package/vendor/core/dist/otel.d.ts +13 -2
- package/vendor/core/dist/otel.js +98 -44
- package/vendor/core/dist/trace-context.d.ts +12 -0
- package/vendor/core/dist/trace-context.js +50 -0
- package/vendor/core/dist/utils/ajv.js +3 -1
- package/vendor/core/package.json +26 -5
- package/vendor/connector-worker/dist/executor/http-status.d.ts +0 -16
- package/vendor/connector-worker/dist/executor/http-status.js +0 -21
- package/vendor/connector-worker/dist/executor/redact.d.ts +0 -29
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
* What this module adds is only the envelope: the guest bundle, the allowlist,
|
|
8
8
|
* and reporting the result.
|
|
9
9
|
*/
|
|
10
|
+
import { createChildSpan, initTracing, runInSpanContext, SpanStatusCode, traceparentHeader } from '@lobu/core';
|
|
10
11
|
import { AGENT_TURN_BRIDGE_BYTES, AGENT_TURN_INPUT_MAX, TURN_DELTA_MAX_CHARS, TURN_TOOL_EVENT_QUEUE_MAX, } from '@lobu/core/contracts/worker/protocol';
|
|
11
12
|
import { agentGuestBundle } from '../agent-turn/bundle.js';
|
|
12
13
|
import { selectExecutor } from '../executor/select.js';
|
|
@@ -36,7 +37,60 @@ const TURN_DELTA_DRAIN_BATCHES = 4;
|
|
|
36
37
|
function isAgentTurnPayload(value) {
|
|
37
38
|
return !!value && typeof value === 'object' && 'turn' in value;
|
|
38
39
|
}
|
|
40
|
+
function ensureWorkerTracing() {
|
|
41
|
+
const otlpEndpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT?.trim();
|
|
42
|
+
if (!otlpEndpoint)
|
|
43
|
+
return;
|
|
44
|
+
initTracing({
|
|
45
|
+
serviceName: 'lobu-worker',
|
|
46
|
+
serviceVersion: process.env.APP_GIT_SHA,
|
|
47
|
+
otlpEndpoint,
|
|
48
|
+
});
|
|
49
|
+
}
|
|
39
50
|
export async function executeAgentTurnRun(client, job, env, cfg) {
|
|
51
|
+
let span = null;
|
|
52
|
+
try {
|
|
53
|
+
ensureWorkerTracing();
|
|
54
|
+
const traceparent = traceparentHeader(job.traceparent);
|
|
55
|
+
if (traceparent) {
|
|
56
|
+
span = createChildSpan('agent_turn', traceparent, {
|
|
57
|
+
'lobu.run_id': String(job.run_id ?? ''),
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
catch (error) {
|
|
62
|
+
// Telemetry is optional. A missing exporter must not fail a claimed turn.
|
|
63
|
+
log.info('[agent-turn] Tracing unavailable; continuing without a span:', error);
|
|
64
|
+
}
|
|
65
|
+
try {
|
|
66
|
+
const result = await runInSpanContext(span, () => runAgentTurn(client, job, env, cfg));
|
|
67
|
+
// The gateway's typed receipt is authoritative, including a cancellation
|
|
68
|
+
// that raced the worker's completion. Keep this internal to tracing.
|
|
69
|
+
const { terminalStatus, ...outcome } = result;
|
|
70
|
+
const status = terminalStatus ?? (outcome.error ? 'failed' : 'completed');
|
|
71
|
+
span?.setAttribute('lobu.run_status', status);
|
|
72
|
+
if (status !== 'cancelled') {
|
|
73
|
+
span?.setStatus({
|
|
74
|
+
code: status === 'failed' ? SpanStatusCode.ERROR : SpanStatusCode.OK,
|
|
75
|
+
...(outcome.error ? { message: outcome.error } : {}),
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
return outcome;
|
|
79
|
+
}
|
|
80
|
+
catch (error) {
|
|
81
|
+
span?.setAttribute('lobu.run_status', 'failed');
|
|
82
|
+
span?.setStatus({
|
|
83
|
+
code: SpanStatusCode.ERROR,
|
|
84
|
+
message: error instanceof Error ? error.message : String(error),
|
|
85
|
+
});
|
|
86
|
+
span?.recordException(error instanceof Error ? error : new Error(String(error)));
|
|
87
|
+
throw error;
|
|
88
|
+
}
|
|
89
|
+
finally {
|
|
90
|
+
span?.end();
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
async function runAgentTurn(client, job, env, cfg) {
|
|
40
94
|
const runId = job.run_id;
|
|
41
95
|
if (!runId)
|
|
42
96
|
return { itemsCollected: 0, error: 'agent turn run missing its run id' };
|
|
@@ -58,7 +112,7 @@ export async function executeAgentTurnRun(client, job, env, cfg) {
|
|
|
58
112
|
...(trailing?.length ? { turn_tool_events: trailing } : {}),
|
|
59
113
|
exit_reason: 'error_message',
|
|
60
114
|
});
|
|
61
|
-
return { itemsCollected: 0, ...(receipt.status === 'completed' ? {} : { error }) };
|
|
115
|
+
return { itemsCollected: 0, terminalStatus: receipt.status, ...(receipt.status === 'completed' ? {} : { error }) };
|
|
62
116
|
}
|
|
63
117
|
catch (deliveryError) {
|
|
64
118
|
// The outer executor must not replace this failure's evidence with a
|
|
@@ -609,8 +663,9 @@ export async function executeAgentTurnRun(client, job, env, cfg) {
|
|
|
609
663
|
...(result.turn.suggestions ? { suggestions: result.turn.suggestions } : {}),
|
|
610
664
|
exit_reason: 'ok',
|
|
611
665
|
});
|
|
612
|
-
if (receipt.status !== 'completed')
|
|
613
|
-
return { itemsCollected: 0, error: `agent turn ${receipt.status}
|
|
666
|
+
if (receipt.status !== 'completed') {
|
|
667
|
+
return { itemsCollected: 0, error: `agent turn ${receipt.status}`, terminalStatus: receipt.status };
|
|
668
|
+
}
|
|
614
669
|
log.info(`[agent-turn] run ${runId} completed after ${deltaSequence} delta batches and ${toolCalls} tool calls`);
|
|
615
670
|
return { itemsCollected: 0 };
|
|
616
671
|
}
|
|
@@ -29,7 +29,7 @@ import { tmpdir } from 'node:os';
|
|
|
29
29
|
import path from 'node:path';
|
|
30
30
|
import { PROVIDER_BALANCE_EXHAUSTED } from '@lobu/core';
|
|
31
31
|
import { buildDeviceAutomationPrompt, DEVICE_AGENT_SPECS_BY_KIND, } from '@lobu/core/contracts/worker/device-automation';
|
|
32
|
-
import { redactOutput } from '
|
|
32
|
+
import { redactOutput } from '@lobu/connector-sdk/redact';
|
|
33
33
|
import { locateBinary, searchDirs } from './agent-binaries.js';
|
|
34
34
|
import { releaseSupervisor, spawnSupervisedCli, terminateChild, waitForTargetExit, waitForTargetExitAfterTermination, } from './automation-process.js';
|
|
35
35
|
import { WorkerHttpError } from './client.js';
|
|
@@ -97,7 +97,21 @@ export declare class MutableWorkerAdvertisementProvider implements WorkerAdverti
|
|
|
97
97
|
export declare class WorkerHttpError extends Error {
|
|
98
98
|
readonly status: number;
|
|
99
99
|
readonly path: string;
|
|
100
|
-
|
|
100
|
+
/** The raw response body, for callers that read its structured error code. */
|
|
101
|
+
readonly responseBody?: string | undefined;
|
|
102
|
+
constructor(status: number, path: string, body: string,
|
|
103
|
+
/** The raw response body, for callers that read its structured error code. */
|
|
104
|
+
responseBody?: string | undefined);
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* The server refused this worker's poll with 409 `upgrade_required`: its
|
|
108
|
+
* reported version is below the server's MIN_CLIENT_VERSION floor. Retrying
|
|
109
|
+
* cannot help — the same binary sends the same version — so the poll loop
|
|
110
|
+
* treats this as terminal and the launcher tells the user how to update.
|
|
111
|
+
*/
|
|
112
|
+
export declare class WorkerUpgradeRequiredError extends WorkerHttpError {
|
|
113
|
+
readonly description: string;
|
|
114
|
+
constructor(description: string);
|
|
101
115
|
}
|
|
102
116
|
/**
|
|
103
117
|
* A 2xx body we could not read as a completion decision. The server answered,
|
|
@@ -10,6 +10,7 @@ export function trimTrailingSlashes(value) {
|
|
|
10
10
|
end--;
|
|
11
11
|
return end === value.length ? value : value.slice(0, end);
|
|
12
12
|
}
|
|
13
|
+
import { traceparentHeader } from "@lobu/core";
|
|
13
14
|
import { resolveRunnableAgentKinds } from "./agent-binaries.js";
|
|
14
15
|
export class MutableWorkerAdvertisementProvider {
|
|
15
16
|
current;
|
|
@@ -56,13 +57,51 @@ function snapshotFingerprint(snapshot) {
|
|
|
56
57
|
export class WorkerHttpError extends Error {
|
|
57
58
|
status;
|
|
58
59
|
path;
|
|
59
|
-
|
|
60
|
+
responseBody;
|
|
61
|
+
constructor(status, path, body,
|
|
62
|
+
/** The raw response body, for callers that read its structured error code. */
|
|
63
|
+
responseBody) {
|
|
60
64
|
super(`${path} failed: ${status} ${body.slice(0, 500)}`);
|
|
61
65
|
this.status = status;
|
|
62
66
|
this.path = path;
|
|
67
|
+
this.responseBody = responseBody;
|
|
63
68
|
this.name = 'WorkerHttpError';
|
|
64
69
|
}
|
|
65
70
|
}
|
|
71
|
+
/**
|
|
72
|
+
* The server refused this worker's poll with 409 `upgrade_required`: its
|
|
73
|
+
* reported version is below the server's MIN_CLIENT_VERSION floor. Retrying
|
|
74
|
+
* cannot help — the same binary sends the same version — so the poll loop
|
|
75
|
+
* treats this as terminal and the launcher tells the user how to update.
|
|
76
|
+
*/
|
|
77
|
+
export class WorkerUpgradeRequiredError extends WorkerHttpError {
|
|
78
|
+
description;
|
|
79
|
+
constructor(description) {
|
|
80
|
+
super(409, '/api/workers/poll', `upgrade_required: ${description}`);
|
|
81
|
+
this.description = description;
|
|
82
|
+
this.name = 'WorkerUpgradeRequiredError';
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
/** The poll refusal's description when a 409 carries `upgrade_required`, else null. */
|
|
86
|
+
function upgradeRequiredDescription(error) {
|
|
87
|
+
if (error.status !== 409 || !error.responseBody)
|
|
88
|
+
return null;
|
|
89
|
+
let body;
|
|
90
|
+
try {
|
|
91
|
+
body = JSON.parse(error.responseBody);
|
|
92
|
+
}
|
|
93
|
+
catch {
|
|
94
|
+
return null;
|
|
95
|
+
}
|
|
96
|
+
if (typeof body !== 'object' || body === null)
|
|
97
|
+
return null;
|
|
98
|
+
const { error: code, error_description: description } = body;
|
|
99
|
+
if (code !== 'upgrade_required')
|
|
100
|
+
return null;
|
|
101
|
+
return typeof description === 'string' && description.trim() !== ''
|
|
102
|
+
? description
|
|
103
|
+
: 'This server requires a newer worker version';
|
|
104
|
+
}
|
|
66
105
|
/**
|
|
67
106
|
* A 2xx body we could not read as a completion decision. The server answered,
|
|
68
107
|
* so re-sending would only mangle the same body — non-retriable.
|
|
@@ -155,7 +194,7 @@ export class WorkerClient {
|
|
|
155
194
|
if (!response.ok) {
|
|
156
195
|
const responseText = await response.text();
|
|
157
196
|
const detail = `${response.statusText} ${responseText}`.trim();
|
|
158
|
-
throw new WorkerHttpError(response.status, path, detail);
|
|
197
|
+
throw new WorkerHttpError(response.status, path, detail, responseText);
|
|
159
198
|
}
|
|
160
199
|
return response;
|
|
161
200
|
}
|
|
@@ -175,7 +214,7 @@ export class WorkerClient {
|
|
|
175
214
|
? { ...this.advertisedCapabilities(), ...advertisement.capabilities }
|
|
176
215
|
: this.advertisedCapabilities();
|
|
177
216
|
const manifests = advertisement?.manifests ?? this.manifests;
|
|
178
|
-
|
|
217
|
+
const body = {
|
|
179
218
|
worker_id: this.workerId,
|
|
180
219
|
...(options?.waitSeconds === undefined ? {} : { wait_seconds: options.waitSeconds }),
|
|
181
220
|
capabilities,
|
|
@@ -200,7 +239,21 @@ export class WorkerClient {
|
|
|
200
239
|
...(this.advertisementProvider || manifests.length > 0
|
|
201
240
|
? { connector_manifests: manifests }
|
|
202
241
|
: {}),
|
|
203
|
-
}
|
|
242
|
+
};
|
|
243
|
+
try {
|
|
244
|
+
const response = await this.post('/api/workers/poll', body, options?.signal);
|
|
245
|
+
const job = (await response.json());
|
|
246
|
+
const traceparent = traceparentHeader(response.headers.get('traceparent'));
|
|
247
|
+
if (traceparent)
|
|
248
|
+
job.traceparent = traceparent;
|
|
249
|
+
return job;
|
|
250
|
+
}
|
|
251
|
+
catch (error) {
|
|
252
|
+
const description = error instanceof WorkerHttpError ? upgradeRequiredDescription(error) : null;
|
|
253
|
+
if (description != null)
|
|
254
|
+
throw new WorkerUpgradeRequiredError(description);
|
|
255
|
+
throw error;
|
|
256
|
+
}
|
|
204
257
|
}
|
|
205
258
|
/**
|
|
206
259
|
* Send heartbeat for active run
|
|
@@ -8,33 +8,6 @@ import type { Env } from '@lobu/connector-sdk';
|
|
|
8
8
|
import type { AgentKind } from '@lobu/core/contracts/worker/device-automation';
|
|
9
9
|
import { type ExecutorClient, type PollResponse } from './client.js';
|
|
10
10
|
import type { SyncExecutor } from '../executor/interface.js';
|
|
11
|
-
/**
|
|
12
|
-
* Resolve the executable compiled code for a job.
|
|
13
|
-
*
|
|
14
|
-
* The gateway prefers omitting `compiled_code` for fleet workers and
|
|
15
|
-
* relying on this side to find + compile the source locally from
|
|
16
|
-
* `connector_key`. This saves the ~13 MB inline blob in poll responses
|
|
17
|
-
* (lobu#771 postmortem trail; lobu#772 perf fix). Device workers and
|
|
18
|
-
* DB-only user-uploaded connectors still receive `compiled_code`
|
|
19
|
-
* directly — they don't have the connector source on disk.
|
|
20
|
-
*
|
|
21
|
-
* Gateway and worker images have different paths to the bundled source,
|
|
22
|
-
* so the gateway sends only `connector_key` and each side resolves it
|
|
23
|
-
* against its own filesystem.
|
|
24
|
-
*
|
|
25
|
-
* Returns `{ code }` on success or `{ error }` on failure. Callers must
|
|
26
|
-
* surface the error to the gateway via `client.complete*` rather than
|
|
27
|
-
* throwing — the daemon-level catch only logs, leaving runs stuck
|
|
28
|
-
* `running` until stale-run reaping.
|
|
29
|
-
*/
|
|
30
|
-
type JobCodeResult = {
|
|
31
|
-
ok: true;
|
|
32
|
-
code: string;
|
|
33
|
-
} | {
|
|
34
|
-
ok: false;
|
|
35
|
-
error: string;
|
|
36
|
-
};
|
|
37
|
-
export declare function resolveJobCode(job: PollResponse): Promise<JobCodeResult>;
|
|
38
11
|
export interface ExecutorConfig {
|
|
39
12
|
heartbeatIntervalMs: number;
|
|
40
13
|
/** Test-only override; production deliberately uses the 15-second default. */
|
|
@@ -100,5 +73,4 @@ export declare function executeRun(client: ExecutorClient, job: PollResponse, wo
|
|
|
100
73
|
itemsCollected: number;
|
|
101
74
|
error?: string;
|
|
102
75
|
}>;
|
|
103
|
-
export {};
|
|
104
76
|
//# sourceMappingURL=executor.d.ts.map
|
|
@@ -13,10 +13,10 @@ import { executeAgentTurnRun } from './agent-turn.js';
|
|
|
13
13
|
import { attachedInteractiveSession, attachInteractiveSession } from './interactive-session.js';
|
|
14
14
|
import { log } from './log.js';
|
|
15
15
|
import { reportTerminalFailure } from './terminal-failure.js';
|
|
16
|
-
import { completeActionOnce } from './terminal-delivery.js';
|
|
16
|
+
import { completeActionOnce, completeSyncOnce } from './terminal-delivery.js';
|
|
17
17
|
import { envelopeToContentItem } from '../executor/runtime.js';
|
|
18
18
|
import { withoutDeploymentProviderKeys } from '../env.js';
|
|
19
|
-
|
|
19
|
+
async function resolveJobCode(job) {
|
|
20
20
|
if (job.compiled_code)
|
|
21
21
|
return { ok: true, code: job.compiled_code };
|
|
22
22
|
// Inline compiled jobs must not load the optional compiler/SDK graph.
|
|
@@ -247,18 +247,32 @@ async function executeSyncRun(client, job, env, cfg) {
|
|
|
247
247
|
if (!run_id || !connector_key) {
|
|
248
248
|
throw new Error('Invalid run: missing run_id or connector_key');
|
|
249
249
|
}
|
|
250
|
+
const deliver = async (payload, executionError) => {
|
|
251
|
+
try {
|
|
252
|
+
await completeSyncOnce(client, payload);
|
|
253
|
+
if (payload.status === 'success') {
|
|
254
|
+
log.info(`[executor] Sync run ${run_id} completed: ${payload.items_collected ?? 0} items`);
|
|
255
|
+
}
|
|
256
|
+
return { itemsCollected: payload.items_collected ?? 0, ...(executionError ? { error: executionError } : {}) };
|
|
257
|
+
}
|
|
258
|
+
catch (error) {
|
|
259
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
260
|
+
log.error(`[executor] Sync run ${run_id} completion delivery failed:`, error);
|
|
261
|
+
return { itemsCollected: payload.items_collected ?? 0,
|
|
262
|
+
error: executionError ? `${executionError}; completion delivery failed: ${message}` : `Completion delivery failed: ${message}` };
|
|
263
|
+
}
|
|
264
|
+
};
|
|
250
265
|
const execution = await resolveJobExecution(select, job, cfg.timeoutMs, cfg.executor);
|
|
251
266
|
if (!execution.ok) {
|
|
252
267
|
const errorMessage = `Run ${run_id} (${connector_key}): ${execution.error}`;
|
|
253
268
|
log.error('[executor]', errorMessage);
|
|
254
|
-
|
|
269
|
+
return deliver({
|
|
255
270
|
run_id,
|
|
256
271
|
worker_id: client.id,
|
|
257
272
|
status: 'failed',
|
|
258
273
|
error_message: errorMessage,
|
|
259
274
|
items_collected: 0,
|
|
260
|
-
});
|
|
261
|
-
return { itemsCollected: 0, error: errorMessage };
|
|
275
|
+
}, errorMessage);
|
|
262
276
|
}
|
|
263
277
|
const compiled_code = execution.code;
|
|
264
278
|
const laneExecutor = execution.executor;
|
|
@@ -285,6 +299,8 @@ async function executeSyncRun(client, job, env, cfg) {
|
|
|
285
299
|
}
|
|
286
300
|
};
|
|
287
301
|
startHeartbeat();
|
|
302
|
+
let terminalPayload;
|
|
303
|
+
let executionError;
|
|
288
304
|
try {
|
|
289
305
|
// The last checkpoint a commit made durable. Completion re-sends it so the
|
|
290
306
|
// feed's `source_ack`, which only a successful completion may advance,
|
|
@@ -346,7 +362,7 @@ async function executeSyncRun(client, job, env, cfg) {
|
|
|
346
362
|
throw new Error(`Expected sync result, got mode=${result.mode}`);
|
|
347
363
|
}
|
|
348
364
|
stopHeartbeat();
|
|
349
|
-
|
|
365
|
+
terminalPayload = {
|
|
350
366
|
run_id,
|
|
351
367
|
worker_id: client.id,
|
|
352
368
|
status: 'success',
|
|
@@ -355,25 +371,24 @@ async function executeSyncRun(client, job, env, cfg) {
|
|
|
355
371
|
...(result.status === 'more' && cursorAdvanced(checkpoint, committedCheckpoint) ? { more: true } : {}),
|
|
356
372
|
auth_update: result.auth_update ?? undefined,
|
|
357
373
|
error_message: partialFetchFailureMessage(result.metadata),
|
|
358
|
-
}
|
|
359
|
-
log.info(`[executor] Sync run ${run_id} completed: ${itemsCollectedSoFar} items`);
|
|
360
|
-
return { itemsCollected: itemsCollectedSoFar };
|
|
374
|
+
};
|
|
361
375
|
}
|
|
362
376
|
catch (error) {
|
|
363
377
|
stopHeartbeat();
|
|
364
378
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
365
379
|
log.error(`[executor] Sync run ${run_id} failed:`, error);
|
|
366
380
|
const diag = extractExecutionDiagnostics(error);
|
|
367
|
-
|
|
381
|
+
terminalPayload = {
|
|
368
382
|
run_id,
|
|
369
383
|
worker_id: client.id,
|
|
370
384
|
status: 'failed',
|
|
371
385
|
items_collected: itemsCollectedSoFar,
|
|
372
386
|
error_message: errorMessage,
|
|
373
387
|
...(diag ?? {}),
|
|
374
|
-
}
|
|
375
|
-
|
|
388
|
+
};
|
|
389
|
+
executionError = errorMessage;
|
|
376
390
|
}
|
|
391
|
+
return deliver(terminalPayload, executionError);
|
|
377
392
|
}
|
|
378
393
|
/**
|
|
379
394
|
* Connectors report per-source failures on an otherwise-successful sync via
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* Exports worker daemon, client, and executor.
|
|
5
5
|
*/
|
|
6
6
|
export type { CompleteRequest, ContentItem, PollResponse, StreamBatch, WorkerAdvertisementProvider, WorkerAdvertisementSnapshot, WorkerCapabilities, } from './client.js';
|
|
7
|
-
export { MutableWorkerAdvertisementProvider, WorkerClient } from './client.js';
|
|
7
|
+
export { MutableWorkerAdvertisementProvider, WorkerClient, WorkerUpgradeRequiredError, } from './client.js';
|
|
8
8
|
export { NATIVE_BRIDGE_PROTOCOL, NATIVE_BRIDGE_PROTOCOL_VERSION, NativeBridgeProtocolError, } from './native-bridge/protocol.js';
|
|
9
9
|
export { NativeBridgeClient } from './native-bridge/client.js';
|
|
10
10
|
export type { ExecutorConfig } from './executor.js';
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Exports worker daemon, client, and executor.
|
|
5
5
|
*/
|
|
6
|
-
export { MutableWorkerAdvertisementProvider, WorkerClient } from './client.js';
|
|
6
|
+
export { MutableWorkerAdvertisementProvider, WorkerClient, WorkerUpgradeRequiredError, } from './client.js';
|
|
7
7
|
export { NATIVE_BRIDGE_PROTOCOL, NATIVE_BRIDGE_PROTOCOL_VERSION, NativeBridgeProtocolError, } from './native-bridge/protocol.js';
|
|
8
8
|
export { NativeBridgeClient } from './native-bridge/client.js';
|
|
9
9
|
export { executeRun } from './executor.js';
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
* the run's start and its terminal result — plus startup/shutdown and hard
|
|
7
7
|
* failures. Pass `--debug` to enable the poll/heartbeat/retry chatter.
|
|
8
8
|
*/
|
|
9
|
-
import { scrubDiagnosticValue } from '@lobu/core';
|
|
9
|
+
import { currentTraceLogFields, scrubDiagnosticValue } from '@lobu/core';
|
|
10
10
|
let debugEnabled = false;
|
|
11
11
|
function write(level, parts) {
|
|
12
12
|
let line;
|
|
@@ -14,6 +14,7 @@ function write(level, parts) {
|
|
|
14
14
|
const record = scrubDiagnosticValue({
|
|
15
15
|
timestamp: new Date().toISOString(), level, service: 'lobu-worker',
|
|
16
16
|
release: process.env.APP_GIT_SHA,
|
|
17
|
+
...currentTraceLogFields(),
|
|
17
18
|
message: parts.filter((part) => typeof part === 'string').join(' '),
|
|
18
19
|
data: parts.filter((part) => typeof part !== 'string'),
|
|
19
20
|
});
|
|
@@ -5,8 +5,9 @@
|
|
|
5
5
|
* polling, concurrency, and graceful-shutdown lifecycle. Run execution stays
|
|
6
6
|
* injected so the device build does not import fleet connector machinery.
|
|
7
7
|
*/
|
|
8
|
+
import { shutdownTracing } from '@lobu/core';
|
|
8
9
|
import { WORKER_POLL_MAX_WAIT_SECONDS } from '@lobu/core/contracts/worker/protocol';
|
|
9
|
-
import { WorkerHttpError } from './client.js';
|
|
10
|
+
import { WorkerHttpError, WorkerUpgradeRequiredError, } from './client.js';
|
|
10
11
|
import { log } from './log.js';
|
|
11
12
|
/** Drain budget for a daemon nobody told how long it has (devices, local runs). */
|
|
12
13
|
const DEFAULT_SHUTDOWN_DRAIN_MS = 30_000;
|
|
@@ -16,17 +17,22 @@ const DEFAULT_SHUTDOWN_DRAIN_MS = 30_000;
|
|
|
16
17
|
*/
|
|
17
18
|
const SHUTDOWN_EXIT_MARGIN_MS = 15_000;
|
|
18
19
|
/**
|
|
19
|
-
*
|
|
20
|
+
* The pod's whole grace period after SIGTERM. A fleet pod is SIGKILLed
|
|
20
21
|
* `terminationGracePeriodSeconds` after SIGTERM; the chart passes that same
|
|
21
|
-
* value as `WORKER_SHUTDOWN_GRACE_SECONDS
|
|
22
|
-
*
|
|
22
|
+
* value as `WORKER_SHUTDOWN_GRACE_SECONDS`. Anything else keeps the fixed
|
|
23
|
+
* default drain plus the exit margin.
|
|
23
24
|
*/
|
|
24
|
-
function
|
|
25
|
+
function resolveWorkerShutdownGraceMs(env = process.env) {
|
|
25
26
|
const raw = env.WORKER_SHUTDOWN_GRACE_SECONDS?.trim();
|
|
26
27
|
const seconds = raw ? Number(raw) : Number.NaN;
|
|
27
|
-
if (!Number.isInteger(seconds) || seconds < 0)
|
|
28
|
-
return DEFAULT_SHUTDOWN_DRAIN_MS;
|
|
29
|
-
|
|
28
|
+
if (!Number.isInteger(seconds) || seconds < 0) {
|
|
29
|
+
return DEFAULT_SHUTDOWN_DRAIN_MS + SHUTDOWN_EXIT_MARGIN_MS;
|
|
30
|
+
}
|
|
31
|
+
return seconds * 1000;
|
|
32
|
+
}
|
|
33
|
+
/** How long SIGTERM waits for in-flight runs: the grace minus the exit margin. */
|
|
34
|
+
function resolveWorkerShutdownDrainMs(env = process.env) {
|
|
35
|
+
return Math.max(0, resolveWorkerShutdownGraceMs(env) - SHUTDOWN_EXIT_MARGIN_MS);
|
|
30
36
|
}
|
|
31
37
|
export function shouldHandleWorkerPollLoopStdinEof(options = {}) {
|
|
32
38
|
return options.stdinEof === true;
|
|
@@ -85,6 +91,11 @@ export class WorkerPollLoop {
|
|
|
85
91
|
break;
|
|
86
92
|
// Server errors retain their retry backoff even if a slot opens.
|
|
87
93
|
capacityVersion = undefined;
|
|
94
|
+
// Below the server's version floor, every retry sends the same version
|
|
95
|
+
// and is refused the same way: stop, so the launcher can say how to
|
|
96
|
+
// update instead of logging a poll error every interval forever.
|
|
97
|
+
if (err instanceof WorkerUpgradeRequiredError)
|
|
98
|
+
throw err;
|
|
88
99
|
if (this.failClosedOnPollAuthError &&
|
|
89
100
|
err instanceof WorkerHttpError &&
|
|
90
101
|
(err.status === 401 || err.status === 403)) {
|
|
@@ -195,6 +206,7 @@ export class WorkerPollLoop {
|
|
|
195
206
|
}
|
|
196
207
|
export function createWorkerPollLoopShutdownHandler(loop, onShutdown = () => loop.stop(), exit = process.exit) {
|
|
197
208
|
const drainTimeoutMs = resolveWorkerShutdownDrainMs();
|
|
209
|
+
const shutdownGraceMs = resolveWorkerShutdownGraceMs();
|
|
198
210
|
let shuttingDown = false;
|
|
199
211
|
return async (signal) => {
|
|
200
212
|
if (shuttingDown) {
|
|
@@ -203,6 +215,7 @@ export function createWorkerPollLoopShutdownHandler(loop, onShutdown = () => loo
|
|
|
203
215
|
return;
|
|
204
216
|
}
|
|
205
217
|
shuttingDown = true;
|
|
218
|
+
const shutdownDeadline = Date.now() + shutdownGraceMs;
|
|
206
219
|
log.info(`\n[daemon] Received ${signal}, shutting down...`);
|
|
207
220
|
let shutdownHookSucceeded = true;
|
|
208
221
|
try {
|
|
@@ -216,6 +229,17 @@ export function createWorkerPollLoopShutdownHandler(loop, onShutdown = () => loo
|
|
|
216
229
|
const allDone = await loop.waitForActiveJobs(drainTimeoutMs);
|
|
217
230
|
if (!allDone)
|
|
218
231
|
log.info('[daemon] Forcing exit with active jobs still running');
|
|
232
|
+
// Exporters can still be waiting on a collector. Spend only the grace
|
|
233
|
+
// left after hooks/draining, cap telemetry at 5s, and retain 1s to exit.
|
|
234
|
+
const tracingTimeoutMs = Math.min(5_000, Math.max(0, shutdownDeadline - Date.now() - 1_000));
|
|
235
|
+
if (tracingTimeoutMs > 0) {
|
|
236
|
+
try {
|
|
237
|
+
await shutdownTracing(tracingTimeoutMs);
|
|
238
|
+
}
|
|
239
|
+
catch (error) {
|
|
240
|
+
log.info('[daemon] Tracing shutdown failed:', error);
|
|
241
|
+
}
|
|
242
|
+
}
|
|
219
243
|
exit(allDone && shutdownHookSucceeded ? 0 : 1);
|
|
220
244
|
};
|
|
221
245
|
}
|
|
@@ -30,7 +30,6 @@ export declare function resolveDaemonLaunchContext(opts: Pick<DaemonStartOptions
|
|
|
30
30
|
platform: string | undefined;
|
|
31
31
|
interactiveSession: InteractiveSession | undefined;
|
|
32
32
|
};
|
|
33
|
-
export declare function resolveDaemonMaxConcurrentJobs(platform: string | undefined, env?: NodeJS.ProcessEnv, hostThreads?: number): number | undefined;
|
|
34
33
|
/**
|
|
35
34
|
* Validate the options and boot the daemon. Throws a user-facing `Error` on
|
|
36
35
|
* bad input; the caller prints it and exits non-zero.
|
|
@@ -45,7 +45,7 @@ export function resolveDaemonLaunchContext(opts, env = process.env, sessionsDir)
|
|
|
45
45
|
interactiveSession: detected,
|
|
46
46
|
};
|
|
47
47
|
}
|
|
48
|
-
|
|
48
|
+
function resolveDaemonMaxConcurrentJobs(platform, env = process.env, hostThreads) {
|
|
49
49
|
const configured = env.WORKER_MAX_CONCURRENT_JOBS
|
|
50
50
|
? Math.max(1, Number.parseInt(env.WORKER_MAX_CONCURRENT_JOBS, 10))
|
|
51
51
|
: undefined;
|
|
@@ -4,6 +4,7 @@ import type { CompleteAgentTurnRequest, CompleteAgentTurnResponse, ExecutorClien
|
|
|
4
4
|
* are retriable; an explicit decode failure or any other 4xx is not.
|
|
5
5
|
*/
|
|
6
6
|
export declare function isRetriableDeliveryFailure(err: unknown): boolean;
|
|
7
|
+
export declare function completeSyncOnce(client: ExecutorClient, payload: Parameters<ExecutorClient['complete']>[0]): Promise<void>;
|
|
7
8
|
export declare function completeActionOnce(client: ExecutorClient, payload: Parameters<ExecutorClient['completeAction']>[0]): Promise<void>;
|
|
8
9
|
export declare function completeAgentTurnOnce(client: ExecutorClient, payload: CompleteAgentTurnRequest): Promise<CompleteAgentTurnResponse>;
|
|
9
10
|
//# sourceMappingURL=terminal-delivery.d.ts.map
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Bounded delivery of an immutable terminal outcome. Execution and failure
|
|
3
|
-
* reporting share the retry loop; action and agent-turn budgets differ.
|
|
3
|
+
* reporting share the retry loop; sync/action and agent-turn budgets differ.
|
|
4
4
|
* WorkerDecodeError identifies an unreadable response, not an execution
|
|
5
5
|
* failure, and is surfaced without retrying.
|
|
6
6
|
*/
|
|
7
7
|
import { WorkerDecodeError, WorkerHttpError } from './client.js';
|
|
8
|
-
/**
|
|
8
|
+
/** Syncs/actions: two attempts in 15s, anything but an undecodable answer retried. */
|
|
9
9
|
const ACTION_DELIVERY = {
|
|
10
10
|
deadlineMs: 15_000,
|
|
11
11
|
attempts: 2,
|
|
@@ -57,10 +57,11 @@ async function withTerminalDeliveryTimeout(promise, timeoutMs) {
|
|
|
57
57
|
* commonest transient failure, and abandoning it there would leave the rest of
|
|
58
58
|
* the deadline unused. The timeout does not cancel the in-flight request, so a
|
|
59
59
|
* retry can reach a gateway that already accepted the first one -- that is
|
|
60
|
-
* safe because
|
|
61
|
-
* `status = 'running' AND claimed_by = worker_id` (a
|
|
62
|
-
*
|
|
63
|
-
* completion hash (an exact retry is acknowledged, a divergent
|
|
60
|
+
* safe because terminal writes are fenced: syncs and actions by
|
|
61
|
+
* `status = 'running' AND claimed_by = worker_id` (a duplicate cannot repeat
|
|
62
|
+
* the terminal write or its side effects), an agent turn's by the run row lock
|
|
63
|
+
* plus its stored completion hash (an exact retry is acknowledged, a divergent
|
|
64
|
+
* one 409s).
|
|
64
65
|
*/
|
|
65
66
|
async function deliverTerminal(send, policy) {
|
|
66
67
|
const deadline = Date.now() + policy.deadlineMs;
|
|
@@ -88,6 +89,9 @@ async function deliverTerminal(send, policy) {
|
|
|
88
89
|
}
|
|
89
90
|
throw lastError instanceof Error ? lastError : new Error(String(lastError));
|
|
90
91
|
}
|
|
92
|
+
export async function completeSyncOnce(client, payload) {
|
|
93
|
+
await deliverTerminal(() => client.complete(payload), ACTION_DELIVERY);
|
|
94
|
+
}
|
|
91
95
|
export async function completeActionOnce(client, payload) {
|
|
92
96
|
await deliverTerminal(() => client.completeAction(payload), ACTION_DELIVERY);
|
|
93
97
|
}
|
|
@@ -37,7 +37,7 @@ import { IsolateHost, IsolateHostError } from '../isolate/bridge.js';
|
|
|
37
37
|
import { assertIsolateEligible } from '../isolate/eligibility.js';
|
|
38
38
|
import { isolatedVmUnavailableReason, loadIsolatedVm } from '../isolate/load.js';
|
|
39
39
|
import { buildConnectorConfig } from './connector-config.js';
|
|
40
|
-
import { redactOutput } from '
|
|
40
|
+
import { redactOutput } from '@lobu/connector-sdk/redact';
|
|
41
41
|
import { RingBuffer, ConnectorExecutionError } from './interface.js';
|
|
42
42
|
const MIB = 1024 * 1024;
|
|
43
43
|
const DEFAULT_OPTIONS = {
|
|
@@ -19,8 +19,10 @@
|
|
|
19
19
|
* - `script.run({ timeout })` bounds only the synchronous part of the run; a
|
|
20
20
|
* loop entered after an `await` is not interrupted. The wall clock here
|
|
21
21
|
* disposes the isolate, which rejects the pending run.
|
|
22
|
-
* - Exceeding `memoryLimit` disposes the isolate automatically
|
|
23
|
-
*
|
|
22
|
+
* - Exceeding `memoryLimit` disposes the isolate automatically and rejects
|
|
23
|
+
* the run. That is the only disposal the host does not perform itself, so
|
|
24
|
+
* an isolate disposed without `dispose()` is how a memory kill is told
|
|
25
|
+
* apart from a guest error whose text merely mentions a memory limit.
|
|
24
26
|
* - `dispose()` on a disposed isolate throws; guard with `isDisposed`.
|
|
25
27
|
*/
|
|
26
28
|
import type { IsolatedVm } from './ivm-types.js';
|
|
@@ -65,6 +67,8 @@ export declare class IsolateHost {
|
|
|
65
67
|
private readonly sync;
|
|
66
68
|
private terminalState;
|
|
67
69
|
private timedOut;
|
|
70
|
+
/** Set before every host disposal; a disposed isolate without it was disposed by V8. */
|
|
71
|
+
private disposedByHost;
|
|
68
72
|
private constructor();
|
|
69
73
|
static create(options: IsolateHostOptions): Promise<IsolateHost>;
|
|
70
74
|
/** The state the host imposed, if it ended the run. */
|
|
@@ -19,8 +19,10 @@
|
|
|
19
19
|
* - `script.run({ timeout })` bounds only the synchronous part of the run; a
|
|
20
20
|
* loop entered after an `await` is not interrupted. The wall clock here
|
|
21
21
|
* disposes the isolate, which rejects the pending run.
|
|
22
|
-
* - Exceeding `memoryLimit` disposes the isolate automatically
|
|
23
|
-
*
|
|
22
|
+
* - Exceeding `memoryLimit` disposes the isolate automatically and rejects
|
|
23
|
+
* the run. That is the only disposal the host does not perform itself, so
|
|
24
|
+
* an isolate disposed without `dispose()` is how a memory kill is told
|
|
25
|
+
* apart from a guest error whose text merely mentions a memory limit.
|
|
24
26
|
* - `dispose()` on a disposed isolate throws; guard with `isDisposed`.
|
|
25
27
|
*/
|
|
26
28
|
import { createPreludeHostSync, GUEST_PRELUDE } from './prelude.js';
|
|
@@ -57,6 +59,8 @@ export class IsolateHost {
|
|
|
57
59
|
sync;
|
|
58
60
|
terminalState = null;
|
|
59
61
|
timedOut = false;
|
|
62
|
+
/** Set before every host disposal; a disposed isolate without it was disposed by V8. */
|
|
63
|
+
disposedByHost = false;
|
|
60
64
|
constructor(isolate, context, options) {
|
|
61
65
|
this.isolate = isolate;
|
|
62
66
|
this.context = context;
|
|
@@ -204,19 +208,21 @@ export class IsolateHost {
|
|
|
204
208
|
cause: error,
|
|
205
209
|
});
|
|
206
210
|
}
|
|
207
|
-
|
|
208
|
-
|
|
211
|
+
// Decided from the isolate's state, never the rejection's text: a guest's
|
|
212
|
+
// own error may say "memory limit" or "terminated" too.
|
|
213
|
+
if (!this.isolate.isDisposed)
|
|
214
|
+
return error;
|
|
215
|
+
if (!this.disposedByHost) {
|
|
209
216
|
this.terminalState = { name: 'MemoryLimitExceeded', message: `isolate exceeded its ${this.options.memoryMb} MB memory limit` };
|
|
210
217
|
return new IsolateHostError('memory', this.terminalState.message, this.terminalState, { cause: error });
|
|
211
218
|
}
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
}
|
|
215
|
-
return error;
|
|
219
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
220
|
+
return new IsolateHostError('crash', `isolate ended unexpectedly: ${message}`, null, { cause: error });
|
|
216
221
|
}
|
|
217
222
|
dispose() {
|
|
218
223
|
if (this.isolate.isDisposed)
|
|
219
224
|
return;
|
|
225
|
+
this.disposedByHost = true;
|
|
220
226
|
try {
|
|
221
227
|
this.isolate.dispose();
|
|
222
228
|
}
|