@lobu/runtime-device 20.1.0 → 21.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (121) hide show
  1. package/dist/component.json +1 -1
  2. package/dist/dependencies.bun.lock +46 -40
  3. package/npm-shrinkwrap.json +283 -237
  4. package/package.json +1 -1
  5. package/vendor/cli/dist/commands/_lib/apply/apply-cmd.d.ts +7 -2
  6. package/vendor/cli/dist/commands/_lib/apply/apply-cmd.js +77 -75
  7. package/vendor/cli/dist/commands/_lib/apply/client.d.ts +40 -25
  8. package/vendor/cli/dist/commands/_lib/apply/client.js +31 -39
  9. package/vendor/cli/dist/commands/_lib/apply/deployment.d.ts +1 -1
  10. package/vendor/cli/dist/commands/_lib/apply/deployment.js +2 -2
  11. package/vendor/cli/dist/commands/_lib/apply/desired-state.d.ts +47 -11
  12. package/vendor/cli/dist/commands/_lib/apply/desired-state.js +106 -7
  13. package/vendor/cli/dist/commands/_lib/apply/diff.d.ts +14 -6
  14. package/vendor/cli/dist/commands/_lib/apply/diff.js +42 -23
  15. package/vendor/cli/dist/commands/_lib/apply/map-config.js +2 -3
  16. package/vendor/cli/dist/commands/_lib/apply/render.js +2 -0
  17. package/vendor/cli/dist/commands/_lib/apply/rollback-cmd.js +3 -3
  18. package/vendor/cli/dist/commands/_lib/connector-loader.d.ts +5 -1
  19. package/vendor/cli/dist/commands/_lib/connector-loader.js +1 -1
  20. package/vendor/cli/dist/commands/_lib/ensure-deps-installed.d.ts +15 -26
  21. package/vendor/cli/dist/commands/_lib/ensure-deps-installed.js +227 -136
  22. package/vendor/cli/dist/commands/_lib/view-bundler.d.ts +33 -0
  23. package/vendor/cli/dist/commands/_lib/view-bundler.js +207 -0
  24. package/vendor/cli/dist/commands/_lib/view-reapply.d.ts +60 -0
  25. package/vendor/cli/dist/commands/_lib/view-reapply.js +182 -0
  26. package/vendor/cli/dist/commands/dev.d.ts +14 -5
  27. package/vendor/cli/dist/commands/dev.js +56 -8
  28. package/vendor/cli/dist/commands/init.d.ts +1 -1
  29. package/vendor/cli/dist/commands/init.js +4 -4
  30. package/vendor/cli/dist/commands/login.js +3 -0
  31. package/vendor/cli/dist/config/define.d.ts +25 -8
  32. package/vendor/cli/dist/config/define.js +24 -0
  33. package/vendor/cli/dist/connectors/README.md +15 -16
  34. package/vendor/cli/dist/connectors/chat-user-identity.ts +41 -0
  35. package/vendor/cli/dist/connectors/github.ts +15 -9
  36. package/vendor/cli/dist/connectors/google-identity.ts +11 -0
  37. package/vendor/cli/dist/connectors/google_calendar.ts +21 -31
  38. package/vendor/cli/dist/connectors/google_drive.ts +106 -131
  39. package/vendor/cli/dist/connectors/google_gmail.ts +53 -30
  40. package/vendor/cli/dist/connectors/hackernews.ts +2 -5
  41. package/vendor/cli/dist/connectors/jira.ts +62 -27
  42. package/vendor/cli/dist/connectors/linear.ts +55 -21
  43. package/vendor/cli/dist/connectors/microsoft_outlook.ts +151 -110
  44. package/vendor/cli/dist/connectors/postgres.ts +19 -11
  45. package/vendor/cli/dist/connectors/producthunt.ts +8 -6
  46. package/vendor/cli/dist/connectors/reddit.ts +2 -2
  47. package/vendor/cli/dist/connectors/rss.ts +2 -2
  48. package/vendor/cli/dist/connectors/slack-identity.ts +19 -0
  49. package/vendor/cli/dist/connectors/whatsapp_web.ts +5 -4
  50. package/vendor/cli/dist/connectors/x.ts +32 -17
  51. package/vendor/cli/dist/connectors/youtube.ts +18 -5
  52. package/vendor/cli/dist/internal/connector-compiler.d.ts +1 -1
  53. package/vendor/cli/dist/internal/connector-compiler.js +216 -17
  54. package/vendor/cli/dist/internal/credentials.js +5 -1
  55. package/vendor/cli/dist/internal/oauth.d.ts +15 -1
  56. package/vendor/cli/dist/internal/oauth.js +9 -1
  57. package/vendor/cli/dist/runtime-components.json +4 -4
  58. package/vendor/cli/package.json +2 -1
  59. package/vendor/connector-sdk/dist/connector-runtime.d.ts +4 -6
  60. package/vendor/connector-sdk/dist/connector-runtime.js +3 -5
  61. package/vendor/connector-sdk/dist/connector-types.d.ts +28 -13
  62. package/vendor/connector-sdk/dist/define-connector.d.ts +4 -1
  63. package/vendor/connector-sdk/dist/define-connector.js +4 -1
  64. package/vendor/connector-sdk/dist/http-client.d.ts +18 -7
  65. package/vendor/connector-sdk/dist/http-client.js +15 -13
  66. package/vendor/connector-sdk/dist/index.d.ts +1 -1
  67. package/vendor/connector-sdk/dist/retry.d.ts +15 -3
  68. package/vendor/connector-sdk/dist/retry.js +40 -64
  69. package/vendor/connector-sdk/package.json +1 -1
  70. package/vendor/connector-worker/dist/agent-turn/guest-entry.js +122 -10
  71. package/vendor/connector-worker/dist/agent-turn/guest.bundle.js +78 -10
  72. package/vendor/connector-worker/dist/agent-turn/types.d.ts +34 -4
  73. package/vendor/connector-worker/dist/compile/index.d.ts +21 -2
  74. package/vendor/connector-worker/dist/compile/index.js +41 -8
  75. package/vendor/connector-worker/dist/compile/source-capture.d.ts +10 -0
  76. package/vendor/connector-worker/dist/compile/source-capture.js +108 -0
  77. package/vendor/connector-worker/dist/daemon/agent-turn.js +97 -10
  78. package/vendor/connector-worker/dist/daemon/automation-process.d.ts +32 -0
  79. package/vendor/connector-worker/dist/daemon/automation-process.js +102 -1
  80. package/vendor/connector-worker/dist/daemon/automation.d.ts +1 -1
  81. package/vendor/connector-worker/dist/daemon/builtins/index.js +8 -0
  82. package/vendor/connector-worker/dist/daemon/builtins/os-shell.d.ts +13 -1
  83. package/vendor/connector-worker/dist/daemon/builtins/os-shell.js +28 -2
  84. package/vendor/connector-worker/dist/daemon/executor.d.ts +0 -1
  85. package/vendor/connector-worker/dist/daemon/executor.js +42 -51
  86. package/vendor/connector-worker/dist/daemon/start.js +5 -1
  87. package/vendor/connector-worker/dist/executor/interface.d.ts +12 -8
  88. package/vendor/connector-worker/dist/executor/isolate.js +45 -28
  89. package/vendor/connector-worker/dist/self-check/index.js +15 -13
  90. package/vendor/connector-worker/package.json +1 -1
  91. package/vendor/core/dist/agent-policy.js +3 -0
  92. package/vendor/core/dist/connector-query-errors.d.ts +3 -3
  93. package/vendor/core/dist/connector-query-errors.js +8 -3
  94. package/vendor/core/dist/contracts/tools/manage-connections.d.ts +24 -0
  95. package/vendor/core/dist/contracts/tools/manage-connections.js +13 -0
  96. package/vendor/core/dist/contracts/tools/manage-entity-schema.d.ts +15 -25
  97. package/vendor/core/dist/contracts/tools/manage-entity-schema.js +3 -5
  98. package/vendor/core/dist/contracts/tools/manage-feeds.d.ts +11 -0
  99. package/vendor/core/dist/contracts/tools/manage-feeds.js +12 -1
  100. package/vendor/core/dist/contracts/tools/manage-operations.d.ts +47 -6
  101. package/vendor/core/dist/contracts/tools/manage-operations.js +36 -1
  102. package/vendor/core/dist/contracts/tools/manage-views.d.ts +293 -0
  103. package/vendor/core/dist/contracts/tools/manage-views.js +203 -0
  104. package/vendor/core/dist/contracts/tools/source-files-schema.d.ts +7 -0
  105. package/vendor/core/dist/contracts/tools/source-files-schema.js +19 -0
  106. package/vendor/core/dist/contracts/tools/source-files.d.ts +14 -0
  107. package/vendor/core/dist/contracts/tools/source-files.js +55 -0
  108. package/vendor/core/dist/contracts/tools/view-content-hash.d.ts +37 -0
  109. package/vendor/core/dist/contracts/tools/view-content-hash.js +58 -0
  110. package/vendor/core/dist/contracts/tools/view-path.d.ts +18 -0
  111. package/vendor/core/dist/contracts/tools/view-path.js +24 -0
  112. package/vendor/core/dist/contracts/worker/protocol.d.ts +73 -8
  113. package/vendor/core/dist/contracts/worker/protocol.js +58 -9
  114. package/vendor/core/dist/credentials.d.ts +16 -1
  115. package/vendor/core/dist/credentials.js +10 -1
  116. package/vendor/core/dist/reserved.d.ts +3 -3
  117. package/vendor/core/dist/reserved.js +9 -1
  118. package/vendor/core/dist/utils/secret-redaction.js +2 -0
  119. package/vendor/core/package.json +25 -5
  120. package/vendor/core/dist/contracts/tools/manage-view-templates.d.ts +0 -165
  121. package/vendor/core/dist/contracts/tools/manage-view-templates.js +0 -130
@@ -39,12 +39,16 @@ export async function executeAgentTurnRun(client, job, env, cfg) {
39
39
  const runId = job.run_id;
40
40
  if (!runId)
41
41
  return { itemsCollected: 0, error: 'agent turn run missing its run id' };
42
- const fail = async (error) => {
42
+ const fail = async (error, trailing) => {
43
43
  const receipt = await client.completeAgentTurn({
44
44
  run_id: runId,
45
45
  worker_id: client.id,
46
46
  status: 'failed',
47
47
  error,
48
+ // Whatever evidence was retained rides the failure, so the server holds
49
+ // the prefix that explains the turn. Bounded by construction (the queue
50
+ // below never exceeds the cap); the overflow error names what did not fit.
51
+ ...(trailing?.length ? { turn_tool_events: trailing } : {}),
48
52
  exit_reason: 'error_message',
49
53
  });
50
54
  return { itemsCollected: 0, ...(receipt.status === 'completed' ? {} : { error }) };
@@ -80,11 +84,26 @@ export async function executeAgentTurnRun(client, job, env, cfg) {
80
84
  let deltaSequence = 0;
81
85
  let sending = false;
82
86
  // Tool traces waiting for the next beat, oldest first.
87
+ //
88
+ // Bounded and fail-closed: a trace is mandatory external-effect evidence,
89
+ // not a view the turn may shed. The queue (plus the batch in flight) never
90
+ // exceeds `TURN_TOOL_EVENT_QUEUE_MAX`; a turn that would need more latches
91
+ // `toolReceiptOverflow` and fails honestly instead of silently dropping the
92
+ // oldest (the previous `shift()`) or the newest (the previous tail slice).
83
93
  let toolEvents = [];
84
94
  // Traces handed to a beat that has not answered yet. A beat is fire-and-
85
95
  // forget from the delta timer, so one can still be in flight when the turn
86
96
  // completes; these come back rather than vanishing in that window.
87
97
  let tracesInFlight = [];
98
+ /**
99
+ * Latched when the turn produces more unacknowledged tool traces than the
100
+ * retained receipt capacity holds. The first trace that did not fit is named
101
+ * (its call id and tool), with how many traces could not be retained after
102
+ * it. Once latched the turn cannot complete: the evidence prefix stays
103
+ * queued for the failure report, and the error states that external effects
104
+ * past it are unknown.
105
+ */
106
+ let toolReceiptOverflow = null;
88
107
  /** The delta timer's most recent beat, awaited before the turn completes. */
89
108
  let beatInFlight = Promise.resolve();
90
109
  /** Arguments of the calls still running, by call id, for the trace their end produces. */
@@ -139,8 +158,18 @@ export async function executeAgentTurnRun(client, job, env, cfg) {
139
158
  if (ack?.continue === false)
140
159
  stopTurn(ack.stop_reason);
141
160
  queueSteering(ack?.steer);
142
- // The beat was answered, so its traces are the server's now.
143
- tracesInFlight = [];
161
+ // The beat was answered, but its traces are the server's only when the
162
+ // server says so: `turn_tool_ack.received` names how many of the sent
163
+ // traces it durably stored. Anything unacknowledged stays queued for the
164
+ // next beat (or the completion below) instead of being dropped — a
165
+ // liveness-only beat carries no ack and retires nothing.
166
+ const received = ack?.turn_tool_ack?.received;
167
+ if (received !== undefined && received >= traces.length) {
168
+ tracesInFlight = [];
169
+ }
170
+ else if (received !== undefined && received > 0) {
171
+ tracesInFlight = traces.slice(received);
172
+ }
144
173
  }
145
174
  catch (err) {
146
175
  // The batch stays in flight and is re-sent under the same sequence, and
@@ -152,11 +181,17 @@ export async function executeAgentTurnRun(client, job, env, cfg) {
152
181
  finally {
153
182
  // Anything this beat could not retire goes back to the front of the
154
183
  // queue: it is older than whatever arrived while the beat was in flight.
184
+ // Never truncated silently: the push-time capacity check below keeps the
185
+ // retained total within the cap, so an excess here is a bookkeeping
186
+ // surprise — latch it as overflow (fail closed) rather than slicing it
187
+ // away.
155
188
  if (tracesInFlight.length) {
156
189
  toolEvents = [...tracesInFlight, ...toolEvents];
157
190
  tracesInFlight = [];
158
191
  if (toolEvents.length > TURN_TOOL_EVENT_QUEUE_MAX) {
159
- toolEvents = toolEvents.slice(-TURN_TOOL_EVENT_QUEUE_MAX);
192
+ const excess = toolEvents.length - TURN_TOOL_EVENT_QUEUE_MAX;
193
+ toolEvents = toolEvents.slice(0, TURN_TOOL_EVENT_QUEUE_MAX);
194
+ noteToolReceiptOverflow("(requeued)", "requeued tool traces", excess);
160
195
  }
161
196
  }
162
197
  sending = false;
@@ -244,6 +279,28 @@ export async function executeAgentTurnRun(client, job, env, cfg) {
244
279
  log.info(`[agent-turn] run ${runId} stopping: ${reason ?? 'the gateway said stop'}`);
245
280
  cancel.abort();
246
281
  };
282
+ /**
283
+ * Latch receipt-capacity overflow and stop the turn. The queue itself stays
284
+ * bounded (the trace that did not fit is NOT queued); its identity and the
285
+ * running dropped count are retained here so the failure report names the
286
+ * evidence that is missing instead of dropping it silently.
287
+ */
288
+ const noteToolReceiptOverflow = (toolCallId, name, dropped = 1) => {
289
+ if (toolReceiptOverflow)
290
+ toolReceiptOverflow.dropped += dropped;
291
+ else
292
+ toolReceiptOverflow = { toolCallId, name, dropped };
293
+ stopTurn('tool receipt capacity exceeded');
294
+ };
295
+ /**
296
+ * The honest failure for a turn that outgrew its retained receipt capacity:
297
+ * the retained prefix is the evidence, and the error states that external
298
+ * effects past it are unknown — never a completed turn with a hole in it.
299
+ */
300
+ const toolOverflowError = () => `agent turn exceeded its retained tool receipt capacity (${TURN_TOOL_EVENT_QUEUE_MAX} traces): ` +
301
+ `${toolReceiptOverflow?.dropped ?? 0} trace(s) could not be retained, starting with ` +
302
+ `${toolReceiptOverflow?.toolCallId ?? 'unknown'} (${toolReceiptOverflow?.name ?? 'unknown'}); ` +
303
+ `external effects past the retained prefix are unknown`;
247
304
  try {
248
305
  const guestCode = await agentGuestBundle();
249
306
  // Same executor seam every other lane uses (`resolveJobExecution`): an
@@ -443,21 +500,31 @@ export async function executeAgentTurnRun(client, job, env, cfg) {
443
500
  // The server turns it into the established `tool_use` custom event,
444
501
  // so the SPA, promptfoo provider and menubar keep one trace shape.
445
502
  //
446
- // Bounded, and the newest are the ones kept: a tool trace is a view
447
- // of the turn, not its answer, and a turn that spends its budget on
448
- // tool calls must not grow an unbounded queue in the worker.
503
+ // Bounded and fail-closed: the queue plus the batch in flight never
504
+ // exceed the cap. A trace that would not fit latches overflow and
505
+ // stops the turn instead of dropping evidence — the retained prefix
506
+ // rides the failure report, and the error names what is missing.
507
+ // Traces already latched as overflow are counted, not queued, so
508
+ // the queue stays bounded after the latch too.
449
509
  const input = toolArgs.get(event.toolCallId);
450
510
  toolArgs.delete(event.toolCallId);
511
+ if (toolReceiptOverflow) {
512
+ toolReceiptOverflow.dropped += 1;
513
+ return;
514
+ }
515
+ if (toolEvents.length + tracesInFlight.length >= TURN_TOOL_EVENT_QUEUE_MAX) {
516
+ noteToolReceiptOverflow(event.toolCallId, event.name);
517
+ return;
518
+ }
451
519
  toolEvents.push({
452
520
  tool_call_id: event.toolCallId,
453
521
  name: event.name,
522
+ ...(event.inputRunId !== undefined ? { input_run_id: event.inputRunId } : {}),
454
523
  ...(input !== undefined ? { input } : {}),
455
524
  is_error: event.isError,
456
525
  output: event.output,
457
526
  ...(event.resultSummary ? { result_summary: event.resultSummary } : {}),
458
527
  });
459
- if (toolEvents.length > TURN_TOOL_EVENT_QUEUE_MAX)
460
- toolEvents.shift();
461
528
  }
462
529
  },
463
530
  });
@@ -479,6 +546,16 @@ export async function executeAgentTurnRun(client, job, env, cfg) {
479
546
  clearInterval(heartbeat);
480
547
  await beatInFlight;
481
548
  await drainDeltas();
549
+ // A turn that outgrew its retained receipt capacity cannot complete: the
550
+ // retained prefix rides a FAILED completion whose error states that
551
+ // external effects past it are unknown. Completing it would deliver an
552
+ // answer with a hole in its evidence.
553
+ if (toolReceiptOverflow) {
554
+ const retained = [...tracesInFlight, ...toolEvents].slice(0, TURN_TOOL_EVENT_QUEUE_MAX);
555
+ tracesInFlight = [];
556
+ toolEvents = [];
557
+ return fail(toolOverflowError(), retained);
558
+ }
482
559
  // Whatever drainDeltas could not place rides the completion instead of
483
560
  // being dropped. A trace can remain when no text delta triggered a drain
484
561
  // or its beat could not deliver. Heartbeat publishing requires the run
@@ -497,8 +574,12 @@ export async function executeAgentTurnRun(client, job, env, cfg) {
497
574
  usage: result.turn.usage,
498
575
  session_jsonl: result.turn.sessionJsonl,
499
576
  tools_used: result.turn.toolsUsed,
577
+ ...(result.turn.firstError ? { first_error: result.turn.firstError } : {}),
500
578
  consumed_inputs: result.turn.consumedInputs.map((input) => ({
501
579
  run_id: input.runId, session_entry_id: input.sessionEntryId,
580
+ response_text: input.responseText,
581
+ tools_used: input.toolsUsed,
582
+ ...(input.firstError ? { first_error: input.firstError } : {}),
502
583
  })),
503
584
  ...(result.turn.repliedInBand ? { replied_in_band: true } : {}),
504
585
  exit_reason: 'ok',
@@ -509,7 +590,13 @@ export async function executeAgentTurnRun(client, job, env, cfg) {
509
590
  return { itemsCollected: 0 };
510
591
  }
511
592
  catch (error) {
512
- return fail(error instanceof Error ? error.message : String(error));
593
+ // Any failure reports the retained receipt prefix. A provider/runtime
594
+ // error after a tool finished must not erase mandatory external-effect
595
+ // evidence merely because the queue did not overflow.
596
+ const retained = [...tracesInFlight, ...toolEvents].slice(0, TURN_TOOL_EVENT_QUEUE_MAX);
597
+ if (toolReceiptOverflow)
598
+ return fail(toolOverflowError(), retained);
599
+ return fail(error instanceof Error ? error.message : String(error), retained);
513
600
  }
514
601
  finally {
515
602
  clearInterval(deltaTimer);
@@ -40,6 +40,38 @@ type ProcessGroupOwner = Pick<ChildProcess, 'pid' | 'exitCode' | 'signalCode'>;
40
40
  * Exported only so the PID-reuse safety invariant has a direct regression test.
41
41
  */
42
42
  export declare function signalOwnedPosixProcessGroup(owner: ProcessGroupOwner, signal: NodeJS.Signals, sendSignal?: typeof process.kill): boolean;
43
+ /**
44
+ * Count the members of an owned POSIX process group other than the supervisor
45
+ * itself. Used to tell "the command cleaned up after itself" apart from "the
46
+ * command left background work that the group SIGKILL is about to destroy".
47
+ *
48
+ * The supervisor is spawned `detached`, so its pgid equals its pid; every
49
+ * process the command started without deliberately leaving the group shares
50
+ * that pgid. Anything still listed under it once the target has exited is a
51
+ * descendant the caller backgrounded, so reaping it is a caller-visible event
52
+ * rather than routine cleanup (#3629).
53
+ *
54
+ * POSIX exposes no syscall to list a group, and kill(-pgid, 0) cannot answer
55
+ * this because the live supervisor is itself a member and always makes the
56
+ * probe succeed -- so the group has to be enumerated from the process table.
57
+ * On Linux that is read from `/proc`, because the images this daemon ships in
58
+ * are slim ones that carry no `procps`: shelling out to `ps` there would fail
59
+ * with ENOENT and silently report "nothing was reaped" on exactly the hosts
60
+ * this fix is for. `ps` remains the reader everywhere else (macOS dev hosts).
61
+ *
62
+ * Zombies are not survivors. The daemon is PID 1 in the worker image with no
63
+ * init in front of it, and libuv only reaps the children it spawned itself, so
64
+ * a grandchild the command backgrounded and that exited before the command
65
+ * returned is reparented to the daemon and stays a zombie -- still listed
66
+ * under the supervisor's pgid -- for the life of the container. Counting it
67
+ * would fail a run that left nothing running, and would later charge it to an
68
+ * unrelated run when the kernel reuses the pid as a new supervisor's pgid.
69
+ *
70
+ * A failure is reported as "no survivors": this drives a report field, never
71
+ * the cleanup itself, so an unreadable process table must never change what
72
+ * gets killed.
73
+ */
74
+ export declare function countOwnedGroupSurvivors(owner: ProcessGroupOwner, readProcessTable?: () => string): number;
43
75
  /** Ask the supervisor to release its non-reusable group identity and reap it. */
44
76
  export declare function releaseSupervisor(proc: ChildProcess, timeoutMs?: number): Promise<boolean>;
45
77
  /** Windows has no negative-pid process groups; taskkill supplies bounded tree cleanup. */
@@ -5,7 +5,8 @@
5
5
  * share a control-flow surface with POSIX group ownership and Windows tree
6
6
  * termination.
7
7
  */
8
- import { spawn } from 'node:child_process';
8
+ import { execFileSync, spawn } from 'node:child_process';
9
+ import { readFileSync, readdirSync } from 'node:fs';
9
10
  const SUPPORTS_PROCESS_GROUPS = process.platform !== 'win32';
10
11
  const TREE_TERM_GRACE_MS = 3000;
11
12
  const PROCESS_REAP_GRACE_MS = 5000;
@@ -240,6 +241,106 @@ export function signalOwnedPosixProcessGroup(owner, signal, sendSignal = process
240
241
  throw err;
241
242
  }
242
243
  }
244
+ /**
245
+ * The process table as `<pid> <pgid> <state>` lines, however this host will
246
+ * give it up. Linux reads `/proc` directly so the daemon depends on no packaged
247
+ * binary; in `/proc/<pid>/stat` the state and pgid are the 1st and 3rd fields
248
+ * after the `)` that closes comm, and they are read from there because a
249
+ * process name may itself contain spaces or brackets.
250
+ */
251
+ function defaultProcessTableReader() {
252
+ if (process.platform === 'linux') {
253
+ const lines = [];
254
+ for (const entry of readdirSync('/proc')) {
255
+ if (!/^\d+$/.test(entry))
256
+ continue;
257
+ let stat;
258
+ try {
259
+ stat = readFileSync(`/proc/${entry}/stat`, 'utf8');
260
+ }
261
+ catch {
262
+ // The process exited between listing and reading; it is not a survivor.
263
+ continue;
264
+ }
265
+ const afterComm = stat.slice(stat.lastIndexOf(') ') + 2).split(' ');
266
+ // afterComm[0] is state, then ppid, then pgid.
267
+ const state = afterComm[0];
268
+ const pgid = afterComm[2];
269
+ if (pgid)
270
+ lines.push(`${entry} ${pgid} ${state}`);
271
+ }
272
+ return lines.join('\n');
273
+ }
274
+ return execFileSync('ps', ['-A', '-o', 'pid=,pgid=,stat='], {
275
+ encoding: 'utf8',
276
+ stdio: ['ignore', 'pipe', 'ignore'],
277
+ // The default 1 MiB overflows on a busy host, and an overflow throws --
278
+ // which the caller degrades to "no survivors", the silent loss #3629 is
279
+ // here to remove. A line is ~20 bytes; this covers millions of processes.
280
+ maxBuffer: 64 * 1024 * 1024,
281
+ });
282
+ }
283
+ /**
284
+ * Count the members of an owned POSIX process group other than the supervisor
285
+ * itself. Used to tell "the command cleaned up after itself" apart from "the
286
+ * command left background work that the group SIGKILL is about to destroy".
287
+ *
288
+ * The supervisor is spawned `detached`, so its pgid equals its pid; every
289
+ * process the command started without deliberately leaving the group shares
290
+ * that pgid. Anything still listed under it once the target has exited is a
291
+ * descendant the caller backgrounded, so reaping it is a caller-visible event
292
+ * rather than routine cleanup (#3629).
293
+ *
294
+ * POSIX exposes no syscall to list a group, and kill(-pgid, 0) cannot answer
295
+ * this because the live supervisor is itself a member and always makes the
296
+ * probe succeed -- so the group has to be enumerated from the process table.
297
+ * On Linux that is read from `/proc`, because the images this daemon ships in
298
+ * are slim ones that carry no `procps`: shelling out to `ps` there would fail
299
+ * with ENOENT and silently report "nothing was reaped" on exactly the hosts
300
+ * this fix is for. `ps` remains the reader everywhere else (macOS dev hosts).
301
+ *
302
+ * Zombies are not survivors. The daemon is PID 1 in the worker image with no
303
+ * init in front of it, and libuv only reaps the children it spawned itself, so
304
+ * a grandchild the command backgrounded and that exited before the command
305
+ * returned is reparented to the daemon and stays a zombie -- still listed
306
+ * under the supervisor's pgid -- for the life of the container. Counting it
307
+ * would fail a run that left nothing running, and would later charge it to an
308
+ * unrelated run when the kernel reuses the pid as a new supervisor's pgid.
309
+ *
310
+ * A failure is reported as "no survivors": this drives a report field, never
311
+ * the cleanup itself, so an unreadable process table must never change what
312
+ * gets killed.
313
+ */
314
+ export function countOwnedGroupSurvivors(owner, readProcessTable = defaultProcessTableReader) {
315
+ if (!SUPPORTS_PROCESS_GROUPS ||
316
+ owner.pid == null ||
317
+ owner.exitCode !== null ||
318
+ owner.signalCode !== null) {
319
+ return 0;
320
+ }
321
+ const pgid = owner.pid;
322
+ let table;
323
+ try {
324
+ table = readProcessTable();
325
+ }
326
+ catch {
327
+ return 0;
328
+ }
329
+ let survivors = 0;
330
+ for (const line of table.split('\n')) {
331
+ const [pidField, pgidField, stateField] = line.trim().split(/\s+/);
332
+ if (Number(pgidField) !== pgid)
333
+ continue;
334
+ // The supervisor is the anchor, not a survivor; the daemon always reaps it.
335
+ if (Number(pidField) === pgid)
336
+ continue;
337
+ // `Z` from /proc and from `ps -o stat=` alike: already dead, nothing to reap.
338
+ if (stateField?.startsWith('Z'))
339
+ continue;
340
+ survivors += 1;
341
+ }
342
+ return survivors;
343
+ }
243
344
  /** Ask the supervisor to release its non-reusable group identity and reap it. */
244
345
  export async function releaseSupervisor(proc, timeoutMs = 1000) {
245
346
  if (proc.exitCode !== null || proc.signalCode !== null)
@@ -34,7 +34,7 @@ import { type AutomationAcpAdapters } from './automation-acp.js';
34
34
  *
35
35
  * Declared narrowly so a caller that only executes automations — the one-shot
36
36
  * `executeClaimedAutomationRun` entry point — does not have to fabricate
37
- * connector-sync fields (`batchSize`, `generateEmbeddings`)
37
+ * connector-sync fields (`generateEmbeddings`)
38
38
  * that this arm never touches. The daemon's full `ExecutorConfig` is
39
39
  * structurally assignable to it.
40
40
  */
@@ -21,6 +21,14 @@ function describeShellFailure(output) {
21
21
  if (output.exit_signal) {
22
22
  return `Shell command terminated by ${output.exit_signal}`;
23
23
  }
24
+ // After the failures that name their own cause -- a signal death that also
25
+ // left survivors is reported as the signal death, with `reaped_descendants`
26
+ // still on the output -- but before the exit-code default: a reaped run
27
+ // exits 0, so falling through would report "exited with code 0" as the
28
+ // reason it failed, the silent loss #3629 exists to remove.
29
+ if (output.reaped_descendants) {
30
+ return `Shell command exited with code ${output.exit_code} but left background processes running in its process group; they were killed when the command returned. A command's background work does not outlive the call -- hand a durable process to a host service manager instead.`;
31
+ }
24
32
  return `Shell command exited with code ${output.exit_code}`;
25
33
  }
26
34
  /**
@@ -14,7 +14,19 @@ export interface ShellRunOutput {
14
14
  exit_signal?: NodeJS.Signals;
15
15
  process_error?: string;
16
16
  process_error_code?: string;
17
- process_stage?: TargetExit['stage'] | 'timeout' | 'shutdown';
17
+ process_stage?: TargetExit['stage'] | 'timeout' | 'shutdown' | 'descendants_reaped';
18
+ /**
19
+ * Set when the command exited while background work it started was still
20
+ * running inside the owned process group, and that work was therefore killed
21
+ * during cleanup. Absent -- never `false` -- on the ordinary path, so the
22
+ * field only ever appears when there is something to report.
23
+ *
24
+ * Scoped to a run that ended by its own exit. A timeout or a shutdown kills
25
+ * the group while the command is still IN it, so the group cannot be told
26
+ * apart from the work it backgrounded there; those paths already fail with
27
+ * `timeout`/`shutdown` as the named cause.
28
+ */
29
+ reaped_descendants?: boolean;
18
30
  success: boolean;
19
31
  timed_out: boolean;
20
32
  duration_ms: number;
@@ -18,11 +18,22 @@
18
18
  * This runs in the daemon's own process, so it REPLACES the environment rather
19
19
  * than inheriting it: an inherited env would hand the command the daemon's
20
20
  * credentials.
21
+ *
22
+ * LIFETIME: a run owns a process group for the duration of the call and no
23
+ * longer. Everything still in that group when the command returns is killed,
24
+ * which is what keeps a daemon from leaking processes, and it means `&`,
25
+ * `nohup` and `setsid` do not buy a caller a surviving daemon. That cleanup is
26
+ * deliberate; what was a bug (#3629) is doing it QUIETLY -- a run used to
27
+ * report exit 0 while the poller it had just launched was being killed, so the
28
+ * loss surfaced as an unrelated failure much later. A run that leaves
29
+ * background work now reports `reaped_descendants` and fails. Callers that
30
+ * need a durable process hand it to a host service manager; this contract
31
+ * deliberately offers no detached-lifetime primitive.
21
32
  */
22
33
  import { existsSync } from 'node:fs';
23
34
  import { homedir } from 'node:os';
24
35
  import { isAbsolute } from 'node:path';
25
- import { releaseSupervisor, signalOwnedPosixProcessGroup, spawnSupervisedCli, terminateChild, } from '../automation-process.js';
36
+ import { countOwnedGroupSurvivors, releaseSupervisor, signalOwnedPosixProcessGroup, spawnSupervisedCli, terminateChild, } from '../automation-process.js';
26
37
  /**
27
38
  * A caller-supplied argument the builtin rejects before spawning anything.
28
39
  * Distinguishes an input fault from a runtime one so the caller can classify
@@ -111,6 +122,7 @@ export async function runShellBuiltin(input, shutdownSignal) {
111
122
  let stdout = '';
112
123
  let stderr = '';
113
124
  let timedOut = false;
125
+ let reapedDescendants = false;
114
126
  let settled = false;
115
127
  let finishing = false;
116
128
  let timeoutTimer;
@@ -145,12 +157,23 @@ export async function runShellBuiltin(input, shutdownSignal) {
145
157
  ...(outcome.errorCode
146
158
  ? { process_error_code: outcome.errorCode }
147
159
  : {}),
160
+ // A real lifecycle stage (a signal death, a spawn failure) wins over
161
+ // the reaped marker: the marker is only the stage of a run that would
162
+ // otherwise have reported none, and `reaped_descendants` carries the
163
+ // fact either way.
148
164
  ...(outcome.stage !== 'target_exit' ||
149
165
  outcome.signalCode ||
150
166
  outcome.error
151
167
  ? { process_stage: outcome.stage }
152
- : {}),
168
+ : reapedDescendants
169
+ ? { process_stage: 'descendants_reaped' }
170
+ : {}),
171
+ ...(reapedDescendants ? { reaped_descendants: true } : {}),
172
+ // Killing work the caller deliberately backgrounded is not a success,
173
+ // whatever the shell's own exit code said. Reporting it as one is the
174
+ // whole of #3629: the caller banked a launch that no longer exists.
153
175
  success: !timedOut &&
176
+ !reapedDescendants &&
154
177
  outcome.exitCode === 0 &&
155
178
  outcome.signalCode === null &&
156
179
  outcome.error === null,
@@ -231,6 +254,9 @@ export async function runShellBuiltin(input, shutdownSignal) {
231
254
  // created it: signalling a numeric PGID we no longer own risks hitting a
232
255
  // reused one, so ownership ending is the end of our reach.
233
256
  if (process.platform !== 'win32') {
257
+ // Ask BEFORE the group SIGKILL: afterwards there is nothing left to
258
+ // count, and the caller would be told a clean exit either way.
259
+ reapedDescendants = countOwnedGroupSurvivors(child) > 0;
234
260
  try {
235
261
  if (child.pid == null ||
236
262
  !signalOwnedPosixProcessGroup(child, 'SIGKILL')) {
@@ -36,7 +36,6 @@ type JobCodeResult = {
36
36
  };
37
37
  export declare function resolveJobCode(job: PollResponse): Promise<JobCodeResult>;
38
38
  export interface ExecutorConfig {
39
- batchSize: number;
40
39
  heartbeatIntervalMs: number;
41
40
  /** Test-only override; production deliberately uses the 15-second default. */
42
41
  terminalHeartbeatGraceMs?: number;
@@ -4,6 +4,7 @@
4
4
  * Executes sync and action runs in a V8 isolate from compiled connector code.
5
5
  * Generates embeddings and streams results.
6
6
  */
7
+ import { stableStringify } from '@lobu/core/contracts/tools/view-content-hash';
7
8
  import { executeAutomationRun } from './automation.js';
8
9
  import { executeDaemonBuiltin, hasDaemonBuiltin, hasDaemonBuiltinConnector, } from './builtins/index.js';
9
10
  import { executeAgentTurnRun } from './agent-turn.js';
@@ -90,7 +91,6 @@ async function resolveJobExecution(select, job, timeoutMs, customExecutor) {
90
91
  * copy of these numbers that is free to drift away from them.
91
92
  */
92
93
  export const DEFAULT_CONFIG = {
93
- batchSize: 10,
94
94
  heartbeatIntervalMs: 30000,
95
95
  generateEmbeddings: true,
96
96
  timeoutMs: 600000,
@@ -135,6 +135,24 @@ export function resolveEffectiveEnv(env, job) {
135
135
  };
136
136
  return effective === 'block-private' && job.compiled_code ? withoutDeploymentProviderKeys(merged) : merged;
137
137
  }
138
+ /**
139
+ * Whether a sync pass moved the feed's cursor. `more` asks for an immediate
140
+ * rerun, which is progress only when this pass committed a checkpoint other
141
+ * than the one it started from; a pass that re-committed its starting cursor
142
+ * would otherwise rerun forever. `source_ack` is the platform's record, not the
143
+ * connector's cursor, so it does not count.
144
+ */
145
+ function cursorAdvanced(started, committed) {
146
+ if (committed === null)
147
+ return false;
148
+ const cursor = (value) => {
149
+ if (!value || typeof value !== 'object')
150
+ return stableStringify(value);
151
+ const { source_ack: _ack, ...rest } = value;
152
+ return stableStringify(rest);
153
+ };
154
+ return cursor(committed) !== cursor(started);
155
+ }
138
156
  /**
139
157
  * Execute a run (sync, action, automation, embed_backfill, or auth).
140
158
  *
@@ -255,28 +273,10 @@ async function executeSyncRun(client, job, env, cfg) {
255
273
  };
256
274
  startHeartbeat();
257
275
  try {
258
- let batch = [];
259
- let lastCheckpoint = checkpoint;
260
- const flushBatch = async () => {
261
- if (batch.length === 0)
262
- return;
263
- try {
264
- await client.stream({
265
- type: 'batch',
266
- run_id,
267
- worker_id: client.id,
268
- items: batch,
269
- checkpoint: lastCheckpoint ?? undefined,
270
- });
271
- }
272
- catch (streamErr) {
273
- const batchIds = batch.map((b) => b.id);
274
- log.debug(`[executor] Stream batch failed for run ${run_id} (${batchIds.length} items lost: ${batchIds.join(', ')}):`, streamErr);
275
- const msg = streamErr instanceof Error ? streamErr.message : String(streamErr);
276
- throw new Error(`Stream batch failed: ${msg} (lost ${batchIds.length} items: ${batchIds.join(', ')})`);
277
- }
278
- batch = [];
279
- };
276
+ // The last checkpoint a commit made durable. Completion re-sends it so the
277
+ // feed's `source_ack`, which only a successful completion may advance,
278
+ // moves with it.
279
+ let committedCheckpoint = null;
280
280
  const result = await executeCompiledConnector({
281
281
  compiledCode: compiled_code,
282
282
  executor: laneExecutor,
@@ -292,32 +292,24 @@ async function executeSyncRun(client, job, env, cfg) {
292
292
  entityIds: job.entity_ids ?? [],
293
293
  },
294
294
  hooks: {
295
- onCheckpointUpdate: async (nextCheckpoint) => {
296
- lastCheckpoint = nextCheckpoint;
297
- if (!lastCheckpoint)
295
+ // One chunk of a `ctx.commit`, sent as one page: the gateway stores its
296
+ // events and checkpoint in one transaction or not at all. Any failure
297
+ // propagates and ends the run; the next run resumes from the last
298
+ // checkpoint that did land.
299
+ onCommit: async (events, nextCheckpoint) => {
300
+ const items = await processEventChunk(events, cfg.generateEmbeddings);
301
+ if (items.length === 0 && !nextCheckpoint)
298
302
  return;
299
- try {
300
- await client.stream({
301
- type: 'batch',
302
- run_id,
303
- worker_id: client.id,
304
- items: [],
305
- checkpoint: lastCheckpoint,
306
- });
307
- }
308
- catch (err) {
309
- log.debug('[executor] Checkpoint flush failed:', err);
310
- }
311
- },
312
- onEventChunk: async (events) => {
313
- const contentItems = await processEventChunk(events, cfg.generateEmbeddings);
314
- for (const contentItem of contentItems) {
315
- batch.push(contentItem);
316
- itemsCollectedSoFar++;
317
- if (batch.length >= cfg.batchSize) {
318
- await flushBatch();
319
- }
320
- }
303
+ await client.stream({
304
+ type: 'batch',
305
+ run_id,
306
+ worker_id: client.id,
307
+ items,
308
+ ...(nextCheckpoint ? { checkpoint: nextCheckpoint } : {}),
309
+ });
310
+ itemsCollectedSoFar += items.length;
311
+ if (nextCheckpoint)
312
+ committedCheckpoint = nextCheckpoint;
321
313
  },
322
314
  onChromeDispatch: async (actionKey, actionInput) => {
323
315
  // Forward to the gateway's dispatch endpoint. The endpoint
@@ -337,15 +329,14 @@ async function executeSyncRun(client, job, env, cfg) {
337
329
  if (result.mode !== 'sync') {
338
330
  throw new Error(`Expected sync result, got mode=${result.mode}`);
339
331
  }
340
- lastCheckpoint = result.checkpoint;
341
- await flushBatch();
342
332
  stopHeartbeat();
343
333
  await client.complete({
344
334
  run_id,
345
335
  worker_id: client.id,
346
336
  status: 'success',
347
337
  items_collected: itemsCollectedSoFar,
348
- checkpoint: lastCheckpoint ?? undefined,
338
+ checkpoint: committedCheckpoint ?? undefined,
339
+ ...(result.status === 'more' && cursorAdvanced(checkpoint, committedCheckpoint) ? { more: true } : {}),
349
340
  auth_update: result.auth_update ?? undefined,
350
341
  error_message: partialFetchFailureMessage(result.metadata),
351
342
  });
@@ -91,9 +91,13 @@ export async function startDaemonCommand(opts) {
91
91
  // new run type from handing one to a worker binary that predates the lane:
92
92
  // the two sides agree by string equality, so an old worker never matches and
93
93
  // its dispatch default (a connector sync) is never reached.
94
+ // `agent_turn_per_input_responses` marks per-input attribution (#3662): one
95
+ // terminal reply, tool ledger, and verbatim first error per consumed input.
96
+ // The server negotiates by receipt presence, not by this flag, so old claims
97
+ // keep draining under the single-reply contract they were admitted with.
94
98
  const workerCapabilities = platform
95
99
  ? Object.fromEntries(capabilities.map((name) => [name, true]))
96
- : { db_egress_hardening: true, agent_turn: true };
100
+ : { db_egress_hardening: true, agent_turn: true, agent_turn_per_input_responses: true };
97
101
  // Auto-discover device identity from the host when not passed: a device
98
102
  // worker defaults to `<platform>:<short-hostname>` and a hostname label. An
99
103
  // interactive daemon instead derives its id from the exact inherited