@lobu/runtime-device 24.2.0 → 24.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/dist/component.json +1 -1
  2. package/dist/dependencies.bun.lock +43 -129
  3. package/npm-shrinkwrap.json +206 -690
  4. package/package.json +1 -1
  5. package/vendor/cli/dist/bundled-skills/lobu/SKILL.md +21 -3
  6. package/vendor/cli/dist/bundled-skills/lobu-connect/SKILL.md +6 -0
  7. package/vendor/cli/dist/commands/chat.d.ts +0 -12
  8. package/vendor/cli/dist/commands/chat.js +1 -1
  9. package/vendor/cli/dist/commands/daemon.js +36 -19
  10. package/vendor/cli/dist/commands/doctor.d.ts +0 -6
  11. package/vendor/cli/dist/commands/doctor.js +1 -1
  12. package/vendor/cli/dist/commands/memory/_lib/seed-cmd.d.ts +0 -8
  13. package/vendor/cli/dist/commands/memory/_lib/seed-cmd.js +1 -1
  14. package/vendor/cli/dist/config/define.d.ts +1 -1
  15. package/vendor/cli/dist/connectors/gchat.ts +31 -1
  16. package/vendor/cli/dist/connectors/postgres.ts +9 -5
  17. package/vendor/cli/dist/internal/cli-update-check.d.ts +33 -0
  18. package/vendor/cli/dist/internal/cli-update-check.js +98 -0
  19. package/vendor/cli/dist/internal/credentials.js +10 -36
  20. package/vendor/cli/dist/internal/gateway-url.d.ts +0 -8
  21. package/vendor/cli/dist/internal/gateway-url.js +1 -1
  22. package/vendor/cli/dist/runtime-components.json +4 -4
  23. package/vendor/cli/package.json +1 -1
  24. package/vendor/connector-sdk/dist/connector-types.d.ts +29 -0
  25. package/vendor/connector-sdk/dist/index.d.ts +3 -1
  26. package/vendor/connector-sdk/dist/index.js +4 -0
  27. package/vendor/connector-sdk/dist/redact.d.ts +10 -0
  28. package/vendor/{connector-worker/dist/executor → connector-sdk/dist}/redact.js +0 -49
  29. package/vendor/connector-sdk/dist/types.d.ts +2 -1
  30. package/vendor/connector-sdk/package.json +6 -1
  31. package/vendor/connector-worker/dist/daemon/acp-adapters/THIRD_PARTY_LICENSES.txt +1 -1
  32. package/vendor/connector-worker/dist/daemon/acp-adapters/claude.js +7339 -2136
  33. package/vendor/connector-worker/dist/daemon/agent-turn.js +58 -3
  34. package/vendor/connector-worker/dist/daemon/automation.js +1 -1
  35. package/vendor/connector-worker/dist/daemon/client.d.ts +15 -1
  36. package/vendor/connector-worker/dist/daemon/client.js +57 -4
  37. package/vendor/connector-worker/dist/daemon/executor.d.ts +0 -28
  38. package/vendor/connector-worker/dist/daemon/executor.js +27 -12
  39. package/vendor/connector-worker/dist/daemon/index.d.ts +1 -1
  40. package/vendor/connector-worker/dist/daemon/index.js +1 -1
  41. package/vendor/connector-worker/dist/daemon/log.js +2 -1
  42. package/vendor/connector-worker/dist/daemon/poll-loop.js +32 -8
  43. package/vendor/connector-worker/dist/daemon/start.d.ts +0 -1
  44. package/vendor/connector-worker/dist/daemon/start.js +1 -1
  45. package/vendor/connector-worker/dist/daemon/terminal-delivery.d.ts +1 -0
  46. package/vendor/connector-worker/dist/daemon/terminal-delivery.js +10 -6
  47. package/vendor/connector-worker/dist/executor/isolate.js +1 -1
  48. package/vendor/connector-worker/dist/isolate/bridge.d.ts +6 -2
  49. package/vendor/connector-worker/dist/isolate/bridge.js +14 -8
  50. package/vendor/connector-worker/package.json +2 -2
  51. package/vendor/core/dist/connector-query-errors.d.ts +6 -0
  52. package/vendor/core/dist/connector-query-errors.js +4 -0
  53. package/vendor/core/dist/contracts/tools/manage-automations.js +1 -1
  54. package/vendor/core/dist/contracts/tools/manage-entity-schema.js +1 -1
  55. package/vendor/core/dist/contracts/tools/manage-entity.d.ts +49 -20
  56. package/vendor/core/dist/contracts/tools/manage-entity.js +23 -5
  57. package/vendor/core/dist/contracts/worker/device-automation.d.ts +1 -1
  58. package/vendor/core/dist/contracts/worker/device-automation.js +1 -1
  59. package/vendor/core/dist/errors.d.ts +0 -20
  60. package/vendor/core/dist/errors.js +1 -25
  61. package/vendor/core/dist/index.d.ts +1 -0
  62. package/vendor/core/dist/index.js +4 -1
  63. package/vendor/core/dist/logger.js +8 -2
  64. package/vendor/core/dist/otel.d.ts +13 -2
  65. package/vendor/core/dist/otel.js +98 -44
  66. package/vendor/core/dist/trace-context.d.ts +12 -0
  67. package/vendor/core/dist/trace-context.js +50 -0
  68. package/vendor/core/dist/utils/ajv.js +3 -1
  69. package/vendor/core/package.json +26 -5
  70. package/vendor/connector-worker/dist/executor/http-status.d.ts +0 -16
  71. package/vendor/connector-worker/dist/executor/http-status.js +0 -21
  72. package/vendor/connector-worker/dist/executor/redact.d.ts +0 -29
@@ -7,6 +7,7 @@
7
7
  * What this module adds is only the envelope: the guest bundle, the allowlist,
8
8
  * and reporting the result.
9
9
  */
10
+ import { createChildSpan, initTracing, runInSpanContext, SpanStatusCode, traceparentHeader } from '@lobu/core';
10
11
  import { AGENT_TURN_BRIDGE_BYTES, AGENT_TURN_INPUT_MAX, TURN_DELTA_MAX_CHARS, TURN_TOOL_EVENT_QUEUE_MAX, } from '@lobu/core/contracts/worker/protocol';
11
12
  import { agentGuestBundle } from '../agent-turn/bundle.js';
12
13
  import { selectExecutor } from '../executor/select.js';
@@ -36,7 +37,60 @@ const TURN_DELTA_DRAIN_BATCHES = 4;
36
37
  function isAgentTurnPayload(value) {
37
38
  return !!value && typeof value === 'object' && 'turn' in value;
38
39
  }
40
+ function ensureWorkerTracing() {
41
+ const otlpEndpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT?.trim();
42
+ if (!otlpEndpoint)
43
+ return;
44
+ initTracing({
45
+ serviceName: 'lobu-worker',
46
+ serviceVersion: process.env.APP_GIT_SHA,
47
+ otlpEndpoint,
48
+ });
49
+ }
39
50
  export async function executeAgentTurnRun(client, job, env, cfg) {
51
+ let span = null;
52
+ try {
53
+ ensureWorkerTracing();
54
+ const traceparent = traceparentHeader(job.traceparent);
55
+ if (traceparent) {
56
+ span = createChildSpan('agent_turn', traceparent, {
57
+ 'lobu.run_id': String(job.run_id ?? ''),
58
+ });
59
+ }
60
+ }
61
+ catch (error) {
62
+ // Telemetry is optional. A missing exporter must not fail a claimed turn.
63
+ log.info('[agent-turn] Tracing unavailable; continuing without a span:', error);
64
+ }
65
+ try {
66
+ const result = await runInSpanContext(span, () => runAgentTurn(client, job, env, cfg));
67
+ // The gateway's typed receipt is authoritative, including a cancellation
68
+ // that raced the worker's completion. Keep this internal to tracing.
69
+ const { terminalStatus, ...outcome } = result;
70
+ const status = terminalStatus ?? (outcome.error ? 'failed' : 'completed');
71
+ span?.setAttribute('lobu.run_status', status);
72
+ if (status !== 'cancelled') {
73
+ span?.setStatus({
74
+ code: status === 'failed' ? SpanStatusCode.ERROR : SpanStatusCode.OK,
75
+ ...(outcome.error ? { message: outcome.error } : {}),
76
+ });
77
+ }
78
+ return outcome;
79
+ }
80
+ catch (error) {
81
+ span?.setAttribute('lobu.run_status', 'failed');
82
+ span?.setStatus({
83
+ code: SpanStatusCode.ERROR,
84
+ message: error instanceof Error ? error.message : String(error),
85
+ });
86
+ span?.recordException(error instanceof Error ? error : new Error(String(error)));
87
+ throw error;
88
+ }
89
+ finally {
90
+ span?.end();
91
+ }
92
+ }
93
+ async function runAgentTurn(client, job, env, cfg) {
40
94
  const runId = job.run_id;
41
95
  if (!runId)
42
96
  return { itemsCollected: 0, error: 'agent turn run missing its run id' };
@@ -58,7 +112,7 @@ export async function executeAgentTurnRun(client, job, env, cfg) {
58
112
  ...(trailing?.length ? { turn_tool_events: trailing } : {}),
59
113
  exit_reason: 'error_message',
60
114
  });
61
- return { itemsCollected: 0, ...(receipt.status === 'completed' ? {} : { error }) };
115
+ return { itemsCollected: 0, terminalStatus: receipt.status, ...(receipt.status === 'completed' ? {} : { error }) };
62
116
  }
63
117
  catch (deliveryError) {
64
118
  // The outer executor must not replace this failure's evidence with a
@@ -609,8 +663,9 @@ export async function executeAgentTurnRun(client, job, env, cfg) {
609
663
  ...(result.turn.suggestions ? { suggestions: result.turn.suggestions } : {}),
610
664
  exit_reason: 'ok',
611
665
  });
612
- if (receipt.status !== 'completed')
613
- return { itemsCollected: 0, error: `agent turn ${receipt.status}` };
666
+ if (receipt.status !== 'completed') {
667
+ return { itemsCollected: 0, error: `agent turn ${receipt.status}`, terminalStatus: receipt.status };
668
+ }
614
669
  log.info(`[agent-turn] run ${runId} completed after ${deltaSequence} delta batches and ${toolCalls} tool calls`);
615
670
  return { itemsCollected: 0 };
616
671
  }
@@ -29,7 +29,7 @@ import { tmpdir } from 'node:os';
29
29
  import path from 'node:path';
30
30
  import { PROVIDER_BALANCE_EXHAUSTED } from '@lobu/core';
31
31
  import { buildDeviceAutomationPrompt, DEVICE_AGENT_SPECS_BY_KIND, } from '@lobu/core/contracts/worker/device-automation';
32
- import { redactOutput } from '../executor/redact.js';
32
+ import { redactOutput } from '@lobu/connector-sdk/redact';
33
33
  import { locateBinary, searchDirs } from './agent-binaries.js';
34
34
  import { releaseSupervisor, spawnSupervisedCli, terminateChild, waitForTargetExit, waitForTargetExitAfterTermination, } from './automation-process.js';
35
35
  import { WorkerHttpError } from './client.js';
@@ -97,7 +97,21 @@ export declare class MutableWorkerAdvertisementProvider implements WorkerAdverti
97
97
  export declare class WorkerHttpError extends Error {
98
98
  readonly status: number;
99
99
  readonly path: string;
100
- constructor(status: number, path: string, body: string);
100
+ /** The raw response body, for callers that read its structured error code. */
101
+ readonly responseBody?: string | undefined;
102
+ constructor(status: number, path: string, body: string,
103
+ /** The raw response body, for callers that read its structured error code. */
104
+ responseBody?: string | undefined);
105
+ }
106
+ /**
107
+ * The server refused this worker's poll with 409 `upgrade_required`: its
108
+ * reported version is below the server's MIN_CLIENT_VERSION floor. Retrying
109
+ * cannot help — the same binary sends the same version — so the poll loop
110
+ * treats this as terminal and the launcher tells the user how to update.
111
+ */
112
+ export declare class WorkerUpgradeRequiredError extends WorkerHttpError {
113
+ readonly description: string;
114
+ constructor(description: string);
101
115
  }
102
116
  /**
103
117
  * A 2xx body we could not read as a completion decision. The server answered,
@@ -10,6 +10,7 @@ export function trimTrailingSlashes(value) {
10
10
  end--;
11
11
  return end === value.length ? value : value.slice(0, end);
12
12
  }
13
+ import { traceparentHeader } from "@lobu/core";
13
14
  import { resolveRunnableAgentKinds } from "./agent-binaries.js";
14
15
  export class MutableWorkerAdvertisementProvider {
15
16
  current;
@@ -56,13 +57,51 @@ function snapshotFingerprint(snapshot) {
56
57
  export class WorkerHttpError extends Error {
57
58
  status;
58
59
  path;
59
- constructor(status, path, body) {
60
+ responseBody;
61
+ constructor(status, path, body,
62
+ /** The raw response body, for callers that read its structured error code. */
63
+ responseBody) {
60
64
  super(`${path} failed: ${status} ${body.slice(0, 500)}`);
61
65
  this.status = status;
62
66
  this.path = path;
67
+ this.responseBody = responseBody;
63
68
  this.name = 'WorkerHttpError';
64
69
  }
65
70
  }
71
+ /**
72
+ * The server refused this worker's poll with 409 `upgrade_required`: its
73
+ * reported version is below the server's MIN_CLIENT_VERSION floor. Retrying
74
+ * cannot help — the same binary sends the same version — so the poll loop
75
+ * treats this as terminal and the launcher tells the user how to update.
76
+ */
77
+ export class WorkerUpgradeRequiredError extends WorkerHttpError {
78
+ description;
79
+ constructor(description) {
80
+ super(409, '/api/workers/poll', `upgrade_required: ${description}`);
81
+ this.description = description;
82
+ this.name = 'WorkerUpgradeRequiredError';
83
+ }
84
+ }
85
+ /** The poll refusal's description when a 409 carries `upgrade_required`, else null. */
86
+ function upgradeRequiredDescription(error) {
87
+ if (error.status !== 409 || !error.responseBody)
88
+ return null;
89
+ let body;
90
+ try {
91
+ body = JSON.parse(error.responseBody);
92
+ }
93
+ catch {
94
+ return null;
95
+ }
96
+ if (typeof body !== 'object' || body === null)
97
+ return null;
98
+ const { error: code, error_description: description } = body;
99
+ if (code !== 'upgrade_required')
100
+ return null;
101
+ return typeof description === 'string' && description.trim() !== ''
102
+ ? description
103
+ : 'This server requires a newer worker version';
104
+ }
66
105
  /**
67
106
  * A 2xx body we could not read as a completion decision. The server answered,
68
107
  * so re-sending would only mangle the same body — non-retriable.
@@ -155,7 +194,7 @@ export class WorkerClient {
155
194
  if (!response.ok) {
156
195
  const responseText = await response.text();
157
196
  const detail = `${response.statusText} ${responseText}`.trim();
158
- throw new WorkerHttpError(response.status, path, detail);
197
+ throw new WorkerHttpError(response.status, path, detail, responseText);
159
198
  }
160
199
  return response;
161
200
  }
@@ -175,7 +214,7 @@ export class WorkerClient {
175
214
  ? { ...this.advertisedCapabilities(), ...advertisement.capabilities }
176
215
  : this.advertisedCapabilities();
177
216
  const manifests = advertisement?.manifests ?? this.manifests;
178
- return this.requestJson('/api/workers/poll', {
217
+ const body = {
179
218
  worker_id: this.workerId,
180
219
  ...(options?.waitSeconds === undefined ? {} : { wait_seconds: options.waitSeconds }),
181
220
  capabilities,
@@ -200,7 +239,21 @@ export class WorkerClient {
200
239
  ...(this.advertisementProvider || manifests.length > 0
201
240
  ? { connector_manifests: manifests }
202
241
  : {}),
203
- }, options?.signal);
242
+ };
243
+ try {
244
+ const response = await this.post('/api/workers/poll', body, options?.signal);
245
+ const job = (await response.json());
246
+ const traceparent = traceparentHeader(response.headers.get('traceparent'));
247
+ if (traceparent)
248
+ job.traceparent = traceparent;
249
+ return job;
250
+ }
251
+ catch (error) {
252
+ const description = error instanceof WorkerHttpError ? upgradeRequiredDescription(error) : null;
253
+ if (description != null)
254
+ throw new WorkerUpgradeRequiredError(description);
255
+ throw error;
256
+ }
204
257
  }
205
258
  /**
206
259
  * Send heartbeat for active run
@@ -8,33 +8,6 @@ import type { Env } from '@lobu/connector-sdk';
8
8
  import type { AgentKind } from '@lobu/core/contracts/worker/device-automation';
9
9
  import { type ExecutorClient, type PollResponse } from './client.js';
10
10
  import type { SyncExecutor } from '../executor/interface.js';
11
- /**
12
- * Resolve the executable compiled code for a job.
13
- *
14
- * The gateway prefers omitting `compiled_code` for fleet workers and
15
- * relying on this side to find + compile the source locally from
16
- * `connector_key`. This saves the ~13 MB inline blob in poll responses
17
- * (lobu#771 postmortem trail; lobu#772 perf fix). Device workers and
18
- * DB-only user-uploaded connectors still receive `compiled_code`
19
- * directly — they don't have the connector source on disk.
20
- *
21
- * Gateway and worker images have different paths to the bundled source,
22
- * so the gateway sends only `connector_key` and each side resolves it
23
- * against its own filesystem.
24
- *
25
- * Returns `{ code }` on success or `{ error }` on failure. Callers must
26
- * surface the error to the gateway via `client.complete*` rather than
27
- * throwing — the daemon-level catch only logs, leaving runs stuck
28
- * `running` until stale-run reaping.
29
- */
30
- type JobCodeResult = {
31
- ok: true;
32
- code: string;
33
- } | {
34
- ok: false;
35
- error: string;
36
- };
37
- export declare function resolveJobCode(job: PollResponse): Promise<JobCodeResult>;
38
11
  export interface ExecutorConfig {
39
12
  heartbeatIntervalMs: number;
40
13
  /** Test-only override; production deliberately uses the 15-second default. */
@@ -100,5 +73,4 @@ export declare function executeRun(client: ExecutorClient, job: PollResponse, wo
100
73
  itemsCollected: number;
101
74
  error?: string;
102
75
  }>;
103
- export {};
104
76
  //# sourceMappingURL=executor.d.ts.map
@@ -13,10 +13,10 @@ import { executeAgentTurnRun } from './agent-turn.js';
13
13
  import { attachedInteractiveSession, attachInteractiveSession } from './interactive-session.js';
14
14
  import { log } from './log.js';
15
15
  import { reportTerminalFailure } from './terminal-failure.js';
16
- import { completeActionOnce } from './terminal-delivery.js';
16
+ import { completeActionOnce, completeSyncOnce } from './terminal-delivery.js';
17
17
  import { envelopeToContentItem } from '../executor/runtime.js';
18
18
  import { withoutDeploymentProviderKeys } from '../env.js';
19
- export async function resolveJobCode(job) {
19
+ async function resolveJobCode(job) {
20
20
  if (job.compiled_code)
21
21
  return { ok: true, code: job.compiled_code };
22
22
  // Inline compiled jobs must not load the optional compiler/SDK graph.
@@ -247,18 +247,32 @@ async function executeSyncRun(client, job, env, cfg) {
247
247
  if (!run_id || !connector_key) {
248
248
  throw new Error('Invalid run: missing run_id or connector_key');
249
249
  }
250
+ const deliver = async (payload, executionError) => {
251
+ try {
252
+ await completeSyncOnce(client, payload);
253
+ if (payload.status === 'success') {
254
+ log.info(`[executor] Sync run ${run_id} completed: ${payload.items_collected ?? 0} items`);
255
+ }
256
+ return { itemsCollected: payload.items_collected ?? 0, ...(executionError ? { error: executionError } : {}) };
257
+ }
258
+ catch (error) {
259
+ const message = error instanceof Error ? error.message : String(error);
260
+ log.error(`[executor] Sync run ${run_id} completion delivery failed:`, error);
261
+ return { itemsCollected: payload.items_collected ?? 0,
262
+ error: executionError ? `${executionError}; completion delivery failed: ${message}` : `Completion delivery failed: ${message}` };
263
+ }
264
+ };
250
265
  const execution = await resolveJobExecution(select, job, cfg.timeoutMs, cfg.executor);
251
266
  if (!execution.ok) {
252
267
  const errorMessage = `Run ${run_id} (${connector_key}): ${execution.error}`;
253
268
  log.error('[executor]', errorMessage);
254
- await client.complete({
269
+ return deliver({
255
270
  run_id,
256
271
  worker_id: client.id,
257
272
  status: 'failed',
258
273
  error_message: errorMessage,
259
274
  items_collected: 0,
260
- });
261
- return { itemsCollected: 0, error: errorMessage };
275
+ }, errorMessage);
262
276
  }
263
277
  const compiled_code = execution.code;
264
278
  const laneExecutor = execution.executor;
@@ -285,6 +299,8 @@ async function executeSyncRun(client, job, env, cfg) {
285
299
  }
286
300
  };
287
301
  startHeartbeat();
302
+ let terminalPayload;
303
+ let executionError;
288
304
  try {
289
305
  // The last checkpoint a commit made durable. Completion re-sends it so the
290
306
  // feed's `source_ack`, which only a successful completion may advance,
@@ -346,7 +362,7 @@ async function executeSyncRun(client, job, env, cfg) {
346
362
  throw new Error(`Expected sync result, got mode=${result.mode}`);
347
363
  }
348
364
  stopHeartbeat();
349
- await client.complete({
365
+ terminalPayload = {
350
366
  run_id,
351
367
  worker_id: client.id,
352
368
  status: 'success',
@@ -355,25 +371,24 @@ async function executeSyncRun(client, job, env, cfg) {
355
371
  ...(result.status === 'more' && cursorAdvanced(checkpoint, committedCheckpoint) ? { more: true } : {}),
356
372
  auth_update: result.auth_update ?? undefined,
357
373
  error_message: partialFetchFailureMessage(result.metadata),
358
- });
359
- log.info(`[executor] Sync run ${run_id} completed: ${itemsCollectedSoFar} items`);
360
- return { itemsCollected: itemsCollectedSoFar };
374
+ };
361
375
  }
362
376
  catch (error) {
363
377
  stopHeartbeat();
364
378
  const errorMessage = error instanceof Error ? error.message : String(error);
365
379
  log.error(`[executor] Sync run ${run_id} failed:`, error);
366
380
  const diag = extractExecutionDiagnostics(error);
367
- await client.complete({
381
+ terminalPayload = {
368
382
  run_id,
369
383
  worker_id: client.id,
370
384
  status: 'failed',
371
385
  items_collected: itemsCollectedSoFar,
372
386
  error_message: errorMessage,
373
387
  ...(diag ?? {}),
374
- });
375
- return { itemsCollected: itemsCollectedSoFar, error: errorMessage };
388
+ };
389
+ executionError = errorMessage;
376
390
  }
391
+ return deliver(terminalPayload, executionError);
377
392
  }
378
393
  /**
379
394
  * Connectors report per-source failures on an otherwise-successful sync via
@@ -4,7 +4,7 @@
4
4
  * Exports worker daemon, client, and executor.
5
5
  */
6
6
  export type { CompleteRequest, ContentItem, PollResponse, StreamBatch, WorkerAdvertisementProvider, WorkerAdvertisementSnapshot, WorkerCapabilities, } from './client.js';
7
- export { MutableWorkerAdvertisementProvider, WorkerClient } from './client.js';
7
+ export { MutableWorkerAdvertisementProvider, WorkerClient, WorkerUpgradeRequiredError, } from './client.js';
8
8
  export { NATIVE_BRIDGE_PROTOCOL, NATIVE_BRIDGE_PROTOCOL_VERSION, NativeBridgeProtocolError, } from './native-bridge/protocol.js';
9
9
  export { NativeBridgeClient } from './native-bridge/client.js';
10
10
  export type { ExecutorConfig } from './executor.js';
@@ -3,7 +3,7 @@
3
3
  *
4
4
  * Exports worker daemon, client, and executor.
5
5
  */
6
- export { MutableWorkerAdvertisementProvider, WorkerClient } from './client.js';
6
+ export { MutableWorkerAdvertisementProvider, WorkerClient, WorkerUpgradeRequiredError, } from './client.js';
7
7
  export { NATIVE_BRIDGE_PROTOCOL, NATIVE_BRIDGE_PROTOCOL_VERSION, NativeBridgeProtocolError, } from './native-bridge/protocol.js';
8
8
  export { NativeBridgeClient } from './native-bridge/client.js';
9
9
  export { executeRun } from './executor.js';
@@ -6,7 +6,7 @@
6
6
  * the run's start and its terminal result — plus startup/shutdown and hard
7
7
  * failures. Pass `--debug` to enable the poll/heartbeat/retry chatter.
8
8
  */
9
- import { scrubDiagnosticValue } from '@lobu/core';
9
+ import { currentTraceLogFields, scrubDiagnosticValue } from '@lobu/core';
10
10
  let debugEnabled = false;
11
11
  function write(level, parts) {
12
12
  let line;
@@ -14,6 +14,7 @@ function write(level, parts) {
14
14
  const record = scrubDiagnosticValue({
15
15
  timestamp: new Date().toISOString(), level, service: 'lobu-worker',
16
16
  release: process.env.APP_GIT_SHA,
17
+ ...currentTraceLogFields(),
17
18
  message: parts.filter((part) => typeof part === 'string').join(' '),
18
19
  data: parts.filter((part) => typeof part !== 'string'),
19
20
  });
@@ -5,8 +5,9 @@
5
5
  * polling, concurrency, and graceful-shutdown lifecycle. Run execution stays
6
6
  * injected so the device build does not import fleet connector machinery.
7
7
  */
8
+ import { shutdownTracing } from '@lobu/core';
8
9
  import { WORKER_POLL_MAX_WAIT_SECONDS } from '@lobu/core/contracts/worker/protocol';
9
- import { WorkerHttpError } from './client.js';
10
+ import { WorkerHttpError, WorkerUpgradeRequiredError, } from './client.js';
10
11
  import { log } from './log.js';
11
12
  /** Drain budget for a daemon nobody told how long it has (devices, local runs). */
12
13
  const DEFAULT_SHUTDOWN_DRAIN_MS = 30_000;
@@ -16,17 +17,22 @@ const DEFAULT_SHUTDOWN_DRAIN_MS = 30_000;
16
17
  */
17
18
  const SHUTDOWN_EXIT_MARGIN_MS = 15_000;
18
19
  /**
19
- * How long SIGTERM waits for in-flight runs. A fleet pod is SIGKILLed
20
+ * The pod's whole grace period after SIGTERM. A fleet pod is SIGKILLed
20
21
  * `terminationGracePeriodSeconds` after SIGTERM; the chart passes that same
21
- * value as `WORKER_SHUTDOWN_GRACE_SECONDS`, so the drain uses all of it except
22
- * the exit margin. Anything else keeps the fixed default.
22
+ * value as `WORKER_SHUTDOWN_GRACE_SECONDS`. Anything else keeps the fixed
23
+ * default drain plus the exit margin.
23
24
  */
24
- function resolveWorkerShutdownDrainMs(env = process.env) {
25
+ function resolveWorkerShutdownGraceMs(env = process.env) {
25
26
  const raw = env.WORKER_SHUTDOWN_GRACE_SECONDS?.trim();
26
27
  const seconds = raw ? Number(raw) : Number.NaN;
27
- if (!Number.isInteger(seconds) || seconds < 0)
28
- return DEFAULT_SHUTDOWN_DRAIN_MS;
29
- return Math.max(0, seconds * 1000 - SHUTDOWN_EXIT_MARGIN_MS);
28
+ if (!Number.isInteger(seconds) || seconds < 0) {
29
+ return DEFAULT_SHUTDOWN_DRAIN_MS + SHUTDOWN_EXIT_MARGIN_MS;
30
+ }
31
+ return seconds * 1000;
32
+ }
33
+ /** How long SIGTERM waits for in-flight runs: the grace minus the exit margin. */
34
+ function resolveWorkerShutdownDrainMs(env = process.env) {
35
+ return Math.max(0, resolveWorkerShutdownGraceMs(env) - SHUTDOWN_EXIT_MARGIN_MS);
30
36
  }
31
37
  export function shouldHandleWorkerPollLoopStdinEof(options = {}) {
32
38
  return options.stdinEof === true;
@@ -85,6 +91,11 @@ export class WorkerPollLoop {
85
91
  break;
86
92
  // Server errors retain their retry backoff even if a slot opens.
87
93
  capacityVersion = undefined;
94
+ // Below the server's version floor, every retry sends the same version
95
+ // and is refused the same way: stop, so the launcher can say how to
96
+ // update instead of logging a poll error every interval forever.
97
+ if (err instanceof WorkerUpgradeRequiredError)
98
+ throw err;
88
99
  if (this.failClosedOnPollAuthError &&
89
100
  err instanceof WorkerHttpError &&
90
101
  (err.status === 401 || err.status === 403)) {
@@ -195,6 +206,7 @@ export class WorkerPollLoop {
195
206
  }
196
207
  export function createWorkerPollLoopShutdownHandler(loop, onShutdown = () => loop.stop(), exit = process.exit) {
197
208
  const drainTimeoutMs = resolveWorkerShutdownDrainMs();
209
+ const shutdownGraceMs = resolveWorkerShutdownGraceMs();
198
210
  let shuttingDown = false;
199
211
  return async (signal) => {
200
212
  if (shuttingDown) {
@@ -203,6 +215,7 @@ export function createWorkerPollLoopShutdownHandler(loop, onShutdown = () => loo
203
215
  return;
204
216
  }
205
217
  shuttingDown = true;
218
+ const shutdownDeadline = Date.now() + shutdownGraceMs;
206
219
  log.info(`\n[daemon] Received ${signal}, shutting down...`);
207
220
  let shutdownHookSucceeded = true;
208
221
  try {
@@ -216,6 +229,17 @@ export function createWorkerPollLoopShutdownHandler(loop, onShutdown = () => loo
216
229
  const allDone = await loop.waitForActiveJobs(drainTimeoutMs);
217
230
  if (!allDone)
218
231
  log.info('[daemon] Forcing exit with active jobs still running');
232
+ // Exporters can still be waiting on a collector. Spend only the grace
233
+ // left after hooks/draining, cap telemetry at 5s, and retain 1s to exit.
234
+ const tracingTimeoutMs = Math.min(5_000, Math.max(0, shutdownDeadline - Date.now() - 1_000));
235
+ if (tracingTimeoutMs > 0) {
236
+ try {
237
+ await shutdownTracing(tracingTimeoutMs);
238
+ }
239
+ catch (error) {
240
+ log.info('[daemon] Tracing shutdown failed:', error);
241
+ }
242
+ }
219
243
  exit(allDone && shutdownHookSucceeded ? 0 : 1);
220
244
  };
221
245
  }
@@ -30,7 +30,6 @@ export declare function resolveDaemonLaunchContext(opts: Pick<DaemonStartOptions
30
30
  platform: string | undefined;
31
31
  interactiveSession: InteractiveSession | undefined;
32
32
  };
33
- export declare function resolveDaemonMaxConcurrentJobs(platform: string | undefined, env?: NodeJS.ProcessEnv, hostThreads?: number): number | undefined;
34
33
  /**
35
34
  * Validate the options and boot the daemon. Throws a user-facing `Error` on
36
35
  * bad input; the caller prints it and exits non-zero.
@@ -45,7 +45,7 @@ export function resolveDaemonLaunchContext(opts, env = process.env, sessionsDir)
45
45
  interactiveSession: detected,
46
46
  };
47
47
  }
48
- export function resolveDaemonMaxConcurrentJobs(platform, env = process.env, hostThreads) {
48
+ function resolveDaemonMaxConcurrentJobs(platform, env = process.env, hostThreads) {
49
49
  const configured = env.WORKER_MAX_CONCURRENT_JOBS
50
50
  ? Math.max(1, Number.parseInt(env.WORKER_MAX_CONCURRENT_JOBS, 10))
51
51
  : undefined;
@@ -4,6 +4,7 @@ import type { CompleteAgentTurnRequest, CompleteAgentTurnResponse, ExecutorClien
4
4
  * are retriable; an explicit decode failure or any other 4xx is not.
5
5
  */
6
6
  export declare function isRetriableDeliveryFailure(err: unknown): boolean;
7
+ export declare function completeSyncOnce(client: ExecutorClient, payload: Parameters<ExecutorClient['complete']>[0]): Promise<void>;
7
8
  export declare function completeActionOnce(client: ExecutorClient, payload: Parameters<ExecutorClient['completeAction']>[0]): Promise<void>;
8
9
  export declare function completeAgentTurnOnce(client: ExecutorClient, payload: CompleteAgentTurnRequest): Promise<CompleteAgentTurnResponse>;
9
10
  //# sourceMappingURL=terminal-delivery.d.ts.map
@@ -1,11 +1,11 @@
1
1
  /**
2
2
  * Bounded delivery of an immutable terminal outcome. Execution and failure
3
- * reporting share the retry loop; action and agent-turn budgets differ.
3
+ * reporting share the retry loop; sync/action and agent-turn budgets differ.
4
4
  * WorkerDecodeError identifies an unreadable response, not an execution
5
5
  * failure, and is surfaced without retrying.
6
6
  */
7
7
  import { WorkerDecodeError, WorkerHttpError } from './client.js';
8
- /** Actions: two attempts in 15s, anything but an undecodable answer retried. */
8
+ /** Syncs/actions: two attempts in 15s, anything but an undecodable answer retried. */
9
9
  const ACTION_DELIVERY = {
10
10
  deadlineMs: 15_000,
11
11
  attempts: 2,
@@ -57,10 +57,11 @@ async function withTerminalDeliveryTimeout(promise, timeoutMs) {
57
57
  * commonest transient failure, and abandoning it there would leave the rest of
58
58
  * the deadline unused. The timeout does not cancel the in-flight request, so a
59
59
  * retry can reach a gateway that already accepted the first one -- that is
60
- * safe because both terminal writes are fenced: an action's by
61
- * `status = 'running' AND claimed_by = worker_id` (a second delivery answers
62
- * `already_finalized`), an agent turn's by the run row lock plus its stored
63
- * completion hash (an exact retry is acknowledged, a divergent one 409s).
60
+ * safe because terminal writes are fenced: syncs and actions by
61
+ * `status = 'running' AND claimed_by = worker_id` (a duplicate cannot repeat
62
+ * the terminal write or its side effects), an agent turn's by the run row lock
63
+ * plus its stored completion hash (an exact retry is acknowledged, a divergent
64
+ * one 409s).
64
65
  */
65
66
  async function deliverTerminal(send, policy) {
66
67
  const deadline = Date.now() + policy.deadlineMs;
@@ -88,6 +89,9 @@ async function deliverTerminal(send, policy) {
88
89
  }
89
90
  throw lastError instanceof Error ? lastError : new Error(String(lastError));
90
91
  }
92
+ export async function completeSyncOnce(client, payload) {
93
+ await deliverTerminal(() => client.complete(payload), ACTION_DELIVERY);
94
+ }
91
95
  export async function completeActionOnce(client, payload) {
92
96
  await deliverTerminal(() => client.completeAction(payload), ACTION_DELIVERY);
93
97
  }
@@ -37,7 +37,7 @@ import { IsolateHost, IsolateHostError } from '../isolate/bridge.js';
37
37
  import { assertIsolateEligible } from '../isolate/eligibility.js';
38
38
  import { isolatedVmUnavailableReason, loadIsolatedVm } from '../isolate/load.js';
39
39
  import { buildConnectorConfig } from './connector-config.js';
40
- import { redactOutput } from './redact.js';
40
+ import { redactOutput } from '@lobu/connector-sdk/redact';
41
41
  import { RingBuffer, ConnectorExecutionError } from './interface.js';
42
42
  const MIB = 1024 * 1024;
43
43
  const DEFAULT_OPTIONS = {
@@ -19,8 +19,10 @@
19
19
  * - `script.run({ timeout })` bounds only the synchronous part of the run; a
20
20
  * loop entered after an `await` is not interrupted. The wall clock here
21
21
  * disposes the isolate, which rejects the pending run.
22
- * - Exceeding `memoryLimit` disposes the isolate automatically; the run
23
- * rejects with "disposed during execution due to memory limit".
22
+ * - Exceeding `memoryLimit` disposes the isolate automatically and rejects
23
+ * the run. That is the only disposal the host does not perform itself, so
24
+ * an isolate disposed without `dispose()` is how a memory kill is told
25
+ * apart from a guest error whose text merely mentions a memory limit.
24
26
  * - `dispose()` on a disposed isolate throws; guard with `isDisposed`.
25
27
  */
26
28
  import type { IsolatedVm } from './ivm-types.js';
@@ -65,6 +67,8 @@ export declare class IsolateHost {
65
67
  private readonly sync;
66
68
  private terminalState;
67
69
  private timedOut;
70
+ /** Set before every host disposal; a disposed isolate without it was disposed by V8. */
71
+ private disposedByHost;
68
72
  private constructor();
69
73
  static create(options: IsolateHostOptions): Promise<IsolateHost>;
70
74
  /** The state the host imposed, if it ended the run. */
@@ -19,8 +19,10 @@
19
19
  * - `script.run({ timeout })` bounds only the synchronous part of the run; a
20
20
  * loop entered after an `await` is not interrupted. The wall clock here
21
21
  * disposes the isolate, which rejects the pending run.
22
- * - Exceeding `memoryLimit` disposes the isolate automatically; the run
23
- * rejects with "disposed during execution due to memory limit".
22
+ * - Exceeding `memoryLimit` disposes the isolate automatically and rejects
23
+ * the run. That is the only disposal the host does not perform itself, so
24
+ * an isolate disposed without `dispose()` is how a memory kill is told
25
+ * apart from a guest error whose text merely mentions a memory limit.
24
26
  * - `dispose()` on a disposed isolate throws; guard with `isDisposed`.
25
27
  */
26
28
  import { createPreludeHostSync, GUEST_PRELUDE } from './prelude.js';
@@ -57,6 +59,8 @@ export class IsolateHost {
57
59
  sync;
58
60
  terminalState = null;
59
61
  timedOut = false;
62
+ /** Set before every host disposal; a disposed isolate without it was disposed by V8. */
63
+ disposedByHost = false;
60
64
  constructor(isolate, context, options) {
61
65
  this.isolate = isolate;
62
66
  this.context = context;
@@ -204,19 +208,21 @@ export class IsolateHost {
204
208
  cause: error,
205
209
  });
206
210
  }
207
- const message = error instanceof Error ? error.message : String(error);
208
- if (/memory limit/i.test(message)) {
211
+ // Decided from the isolate's state, never the rejection's text: a guest's
212
+ // own error may say "memory limit" or "terminated" too.
213
+ if (!this.isolate.isDisposed)
214
+ return error;
215
+ if (!this.disposedByHost) {
209
216
  this.terminalState = { name: 'MemoryLimitExceeded', message: `isolate exceeded its ${this.options.memoryMb} MB memory limit` };
210
217
  return new IsolateHostError('memory', this.terminalState.message, this.terminalState, { cause: error });
211
218
  }
212
- if (/disposed|abandoned|terminated/i.test(message)) {
213
- return new IsolateHostError('crash', `isolate ended unexpectedly: ${message}`, null, { cause: error });
214
- }
215
- return error;
219
+ const message = error instanceof Error ? error.message : String(error);
220
+ return new IsolateHostError('crash', `isolate ended unexpectedly: ${message}`, null, { cause: error });
216
221
  }
217
222
  dispose() {
218
223
  if (this.isolate.isDisposed)
219
224
  return;
225
+ this.disposedByHost = true;
220
226
  try {
221
227
  this.isolate.dispose();
222
228
  }