@north-light/crouter 0.3.300 → 0.3.301

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -83,7 +83,7 @@ function nodeNewParams() {
83
83
  { kind: 'flag', name: 'parent', type: 'string', required: false, constraint: 'Parent node id. Defaults to the calling node.' },
84
84
  { kind: 'flag', name: 'node-id', type: 'string', required: false, constraint: 'Exact id for the new node; lowercase letters, digits, and hyphens only, at most 128 characters. Rejected if already present.' },
85
85
  { kind: 'flag', name: 'root', type: 'bool', required: false, constraint: 'Spawn an independent resident root with no parent or subscription. Mutually exclusive with --worktree.' },
86
- { kind: 'flag', name: 'resident', type: 'bool', required: false, constraint: 'Birth the node as resident: it parks and stays wakeable without being forced to submit a final. Applies to managed children and roots; roots default to resident.' },
86
+ { kind: 'flag', name: 'resident', type: 'bool', required: false, constraint: 'Birth the node as resident: it parks and stays wakeable without being forced to submit a final. Pass it ONLY when the user will open this node’s viewer pane and work with it back and forth; a child that only other nodes message stays terminal. Applies to managed children and roots; roots default to resident.' },
87
87
  { kind: 'flag', name: 'worktree', type: 'string', required: false, constraint: 'Local base branch for a crouter-managed worktree for this child. Mutually exclusive with --root.' },
88
88
  { kind: 'flag', name: 'fork-from', type: 'string', required: false, constraint: 'Fork the new node from an existing node id, absolute session path, or partial pi session uuid.' },
89
89
  {
@@ -3,6 +3,8 @@ import { envNodeId } from '../../shared/env.js';
3
3
  import { defineBranch, defineLeaf } from '../../core/command.js';
4
4
  import { InputError } from '../../core/io.js';
5
5
  import { cliClient, getNodeOrNull, rethrowAsCliError } from '../api-client.js';
6
+ import { contextDir } from '../../core/canvas/paths.js';
7
+ import { DEFAULT_BASH_NICENESS, MAX_BASH_NICENESS, writeBashNiceness } from '../../core/bash-jobs.js';
6
8
  import { nodeReviveLeaf } from '../node-lifecycle-revive.js';
7
9
  import { assertKind, kindsStateBlock } from './create.js';
8
10
  import { MODEL_SPEC_FORMS } from '../../core/runtime/model-selection.js';
@@ -140,6 +142,7 @@ export const nodeConfig = defineLeaf({
140
142
  { kind: 'flag', name: 'kind', type: 'string', required: false, constraint: 'Persona kind. The <kinds> list below names every top-level installable kind and when to use each; a registered sub-kind is valid too by exact path but not listed here.' },
141
143
  { kind: 'flag', name: 'mode', type: 'enum', choices: ['base', 'orchestrator'], required: false, constraint: 'Set persona mode headlessly. base is hands-on; orchestrator holds a roadmap and delegates. orchestrator seeds a roadmap scaffold if absent.' },
142
144
  { kind: 'flag', name: 'name', type: 'string', required: false, constraint: 'Rename the node; if the node has a live window, also rename that viewer window.' },
145
+ { kind: 'flag', name: 'nice', type: 'int', required: false, constraint: `Scheduler niceness every command this node's bash tool starts at, 0–${MAX_BASH_NICENESS} (default ${DEFAULT_BASH_NICENESS}). ${DEFAULT_BASH_NICENESS} keeps the node's builds and test suites below the daemon, the brokers, the viewers, and the person's own shell; 0 lets them compete on equal terms with everything else on the host; ${MAX_BASH_NICENESS} makes them yield to it. Applies to the node's very next command, foreground and background alike.` },
143
146
  { kind: 'flag', name: 'cwd', type: 'string', required: false, constraint: 'Repair-only replacement launch directory. Must be a non-empty absolute existing directory and used alone; the daemon accepts it only for a dormant, non-forked node with no open managed worktree whose recorded cwd is gone. Clears the saved Pi session, so the next revive starts fresh while durable node work remains.' },
144
147
  ],
145
148
  output: [
@@ -149,6 +152,7 @@ export const nodeConfig = defineLeaf({
149
152
  { name: 'kind', type: 'string', required: false, constraint: 'The new kind, present only when --kind changed it.' },
150
153
  { name: 'mode', type: 'string', required: false, constraint: 'The new mode, present only when --mode changed it.' },
151
154
  { name: 'name', type: 'string', required: false, constraint: 'The new name, present only when --name changed it.' },
155
+ { name: 'nice', type: 'number', required: false, constraint: 'The new bash niceness, present only when --nice changed it.' },
152
156
  { name: 'cwd', type: 'string', required: false, constraint: 'The lexically resolved replacement directory, present only after a cwd repair.' },
153
157
  ],
154
158
  outputKind: 'object',
@@ -157,6 +161,7 @@ export const nodeConfig = defineLeaf({
157
161
  'Kind, mode, and lifecycle changes rebuild the launch spec from the fresh node meta; setting mode to orchestrator seeds a roadmap scaffold if absent.',
158
162
  'Name changes update the row and, when the node already has a live window, rename that viewer window to the new full name.',
159
163
  'A cwd repair writes only the lexically resolved cwd and clears saved Pi session pointers; the old transcript and durable node work remain untouched.',
164
+ 'A niceness change is stored with the node\'s own bash-job state, so it takes effect on the next command without a revive and survives one.',
160
165
  ],
161
166
  dynamicState: () => kindsStateBlock(),
162
167
  },
@@ -169,7 +174,7 @@ export const nodeConfig = defineLeaf({
169
174
  // clean listing error, and re-validated server-side against the TARGET
170
175
  // node's scope as a backstop.
171
176
  const cwdSpec = input['cwd'];
172
- if (cwdSpec !== undefined && ['model', 'kind', 'lifecycle', 'mode', 'name'].some((flag) => input[flag] !== undefined)) {
177
+ if (cwdSpec !== undefined && ['model', 'kind', 'lifecycle', 'mode', 'name', 'nice'].some((flag) => input[flag] !== undefined)) {
173
178
  throw new InputError({ error: 'cwd_not_exclusive', message: 'cwd repair cannot be combined with another config flag', field: 'cwd', next: 'Pass --cwd alone.' });
174
179
  }
175
180
  const modelSpec = input['model']?.trim();
@@ -180,6 +185,15 @@ export const nodeConfig = defineLeaf({
180
185
  const lifecycleSpec = input['lifecycle']?.trim();
181
186
  const modeSpec = input['mode']?.trim();
182
187
  const nameSpec = input['name'];
188
+ const niceSpec = input['nice'];
189
+ if (niceSpec !== undefined && (niceSpec < 0 || niceSpec > MAX_BASH_NICENESS)) {
190
+ throw new InputError({
191
+ error: 'bad_nice',
192
+ message: `niceness must be between 0 and ${MAX_BASH_NICENESS}: ${niceSpec}`,
193
+ field: 'nice',
194
+ next: `A negative niceness needs root, so pass 0–${MAX_BASH_NICENESS}.`,
195
+ });
196
+ }
183
197
  if (kindSpec !== undefined)
184
198
  assertKind(kindSpec);
185
199
  if (lifecycleSpec !== undefined && lifecycleSpec.toLowerCase() !== 'terminal' && lifecycleSpec.toLowerCase() !== 'resident') {
@@ -214,8 +228,17 @@ export const nodeConfig = defineLeaf({
214
228
  patch.cwd = cwdSpec;
215
229
  applied.push('cwd');
216
230
  }
231
+ // Niceness is node-local bash-lane state, not canvas state: it is stored
232
+ // beside this node's job dirs so the valve reads it when it starts the next
233
+ // command, rather than waiting for a launch recipe to be rebuilt.
234
+ if (niceSpec !== undefined) {
235
+ writeBashNiceness(contextDir(nodeId), niceSpec);
236
+ applied.push('nice');
237
+ if (applied.length === 1)
238
+ return { node_id: nodeId, nice: niceSpec };
239
+ }
217
240
  if (applied.length === 0) {
218
- throw new InputError({ error: 'no_change', message: 'no config changes requested', next: 'Pass at least one of --model, --lifecycle, --kind, --mode, --name, --cwd.' });
241
+ throw new InputError({ error: 'no_change', message: 'no config changes requested', next: 'Pass at least one of --model, --lifecycle, --kind, --mode, --name, --nice, --cwd.' });
219
242
  }
220
243
  const detail = await cliClient()
221
244
  .patchConfig(nodeId, patch)
@@ -233,6 +256,8 @@ export const nodeConfig = defineLeaf({
233
256
  result.mode = detail.mode;
234
257
  if (applied.includes('name'))
235
258
  result.name = detail.name;
259
+ if (applied.includes('nice'))
260
+ result.nice = niceSpec;
236
261
  return result;
237
262
  },
238
263
  render: (r) => {
@@ -249,6 +274,8 @@ export const nodeConfig = defineLeaf({
249
274
  bits.push(`mode=${r['mode']}`);
250
275
  if (r['name'] !== undefined)
251
276
  bits.push(`name=${r['name']}`);
277
+ if (r['nice'] !== undefined)
278
+ bits.push(`nice=${r['nice']}`);
252
279
  return `Configured ${r['node_id']} — ${bits.join(', ')}`;
253
280
  },
254
281
  });
@@ -5,7 +5,9 @@ import { tmpdir } from 'node:os';
5
5
  import { join } from 'node:path';
6
6
  import { closeDb } from '../canvas/db.js';
7
7
  import { claimActionDelivery, enqueueActionDelivery, readActionDelivery, } from '../canvas/human-deliveries.js';
8
- import { actionDeliveryBackoffMs, actionDeliverySettlementForOutcome, deliverAction, } from '../../daemon/human/deliver-action.js';
8
+ import { deliveryBackoffMs } from '../canvas/delivery-contract.js';
9
+ import { deliverySettlementForOutcome } from '../../daemon/delivery/run-delivery.js';
10
+ import { deliverAction } from '../../daemon/human/deliver-action.js';
9
11
  let home;
10
12
  const priorHome = process.env['CRTR_HOME'];
11
13
  beforeEach(() => {
@@ -25,14 +27,14 @@ after(() => {
25
27
  else
26
28
  process.env['CRTR_HOME'] = priorHome;
27
29
  });
28
- test('action delivery retries follow the durable backoff and exit boundary', () => {
29
- assert.equal(actionDeliveryBackoffMs(1), 5_000);
30
- assert.equal(actionDeliveryBackoffMs(2), 10_000);
31
- assert.equal(actionDeliveryBackoffMs(3), 20_000);
32
- assert.equal(actionDeliveryBackoffMs(11), 3_600_000);
33
- assert.equal(actionDeliveryBackoffMs(100), 3_600_000);
30
+ test('delivery retries follow the durable backoff and exit boundary', () => {
31
+ assert.equal(deliveryBackoffMs(1), 5_000);
32
+ assert.equal(deliveryBackoffMs(2), 10_000);
33
+ assert.equal(deliveryBackoffMs(3), 20_000);
34
+ assert.equal(deliveryBackoffMs(11), 3_600_000);
35
+ assert.equal(deliveryBackoffMs(100), 3_600_000);
34
36
  const now = 1_000_000;
35
- const outcome = (exitCode, signal = null) => actionDeliverySettlementForOutcome({
37
+ const outcome = (exitCode, signal = null) => deliverySettlementForOutcome({
36
38
  exitCode,
37
39
  signal,
38
40
  timedOut: false,
@@ -50,14 +52,14 @@ test('action delivery retries follow the durable backoff and exit boundary', ()
50
52
  });
51
53
  assert.equal(outcome(79).state, 'pending');
52
54
  assert.equal(outcome(null, 'SIGTERM').state, 'pending');
53
- assert.equal(actionDeliverySettlementForOutcome({
55
+ assert.equal(deliverySettlementForOutcome({
54
56
  exitCode: null,
55
57
  signal: null,
56
58
  timedOut: true,
57
59
  stderr: '',
58
60
  }, 1, now).state, 'pending');
59
61
  // The kill escalation's signal is the only way to tell the graceful term from the SIGKILL.
60
- assert.deepEqual(actionDeliverySettlementForOutcome({
62
+ assert.deepEqual(deliverySettlementForOutcome({
61
63
  exitCode: null,
62
64
  signal: 'SIGKILL',
63
65
  timedOut: true,
@@ -67,7 +69,7 @@ test('action delivery retries follow the durable backoff and exit boundary', ()
67
69
  nextAttemptAt: now + 5_000,
68
70
  failure: { kind: 'timeout', signal: 'SIGKILL', stderr: 'stalled' },
69
71
  });
70
- assert.equal(actionDeliverySettlementForOutcome({
72
+ assert.equal(deliverySettlementForOutcome({
71
73
  exitCode: null,
72
74
  signal: null,
73
75
  timedOut: false,
@@ -6,7 +6,7 @@ import { tmpdir } from 'node:os';
6
6
  import { join } from 'node:path';
7
7
  import { closeDb, MIGRATIONS } from '../canvas/db.js';
8
8
  import { createNode, getRow } from '../canvas/canvas.js';
9
- import { armOutcomeDelivery, claimOutcomeDelivery, outcomeDeliveryBackoffMs, readOutcomeDelivery, recoverRunningOutcomeDeliveries, settleOutcomeDelivery, } from '../canvas/node-outcome-deliveries.js';
9
+ import { armOutcomeDelivery, claimOutcomeDelivery, readOutcomeDelivery, recoverRunningOutcomeDeliveries, settleOutcomeDelivery, } from '../canvas/node-outcome-deliveries.js';
10
10
  import { transition } from '../runtime/lifecycle.js';
11
11
  import { composeNodeOutcomeDocument } from '../runtime/outcome-document.js';
12
12
  import { pushFinal } from '../feed/feed.js';
@@ -153,11 +153,6 @@ test('outcome delivery state is fenced, recoverable, and follows the durable bac
153
153
  nextAttemptAt: readOutcomeDelivery('recover').nextAttemptAt,
154
154
  claimOwner: readOutcomeDelivery('recover').claimOwner,
155
155
  }, { state: 'pending', nextAttemptAt: recoveredAt, claimOwner: null });
156
- assert.equal(outcomeDeliveryBackoffMs(1), 5_000);
157
- assert.equal(outcomeDeliveryBackoffMs(2), 10_000);
158
- assert.equal(outcomeDeliveryBackoffMs(3), 20_000);
159
- assert.equal(outcomeDeliveryBackoffMs(11), 3_600_000);
160
- assert.equal(outcomeDeliveryBackoffMs(100), 3_600_000);
161
156
  });
162
157
  test('v39 migrates a v38 database and backfills finalized and terminal rows', () => {
163
158
  const dbPath = join(home, 'v38.db');
@@ -36,6 +36,28 @@ export declare function readJobPgid(contextDir: string, jobId: string): number |
36
36
  /** The optional human-readable job label. Jobs created before labels existed
37
37
  * have no file and deliberately project null. */
38
38
  export declare function readJobPurpose(contextDir: string, jobId: string): string | null;
39
+ /** Scheduler niceness every command a node runs is started with. Agent work —
40
+ * builds, test suites, typechecks — sits below the daemon, the brokers, the
41
+ * viewers, and the person's own shell, so a fan-out of release builds cannot
42
+ * starve them: the scheduler only honors this under contention and it costs
43
+ * nothing on an idle host. */
44
+ export declare const DEFAULT_BASH_NICENESS = 10;
45
+ /** The politest priority the scheduler accepts. A NEGATIVE niceness (more
46
+ * priority than the broker itself) needs root: `nice` prints a permission error
47
+ * into the command's own log and runs it at the inherited priority anyway, so
48
+ * the override range stops at 0. */
49
+ export declare const MAX_BASH_NICENESS = 19;
50
+ /** Per-node override of the bash niceness, written by `crtr node config --nice`
51
+ * and read by the valve when it starts each command. Node-local like the job
52
+ * dirs beside it, so a change applies to the node's very next command without
53
+ * waiting for a revive. */
54
+ export declare function bashNicenessPath(contextDir: string): string;
55
+ /** The niceness this node's commands run at. Any missing, malformed, or
56
+ * out-of-range file reads as the default rather than an odd priority. */
57
+ export declare function readBashNiceness(contextDir: string): number;
58
+ /** Set the node's bash niceness. The caller owns validation; the store keeps
59
+ * one integer so the valve's read stays a single file read per command. */
60
+ export declare function writeBashNiceness(contextDir: string, niceness: number): void;
39
61
  /** The auto-background cancel deadline (epoch ms), or undefined for a job with
40
62
  * no deadline (a deliberate handoff, or a legacy job dir). A non-finite or
41
63
  * malformed file reads as no deadline rather than an instant expiry. */
@@ -3,7 +3,7 @@
3
3
  // job.bg, while foreground completion renames it to job.done. The detached
4
4
  // supervisor can therefore outlive pi and still decide whether to report exit.
5
5
  import { randomBytes } from 'node:crypto';
6
- import { closeSync, existsSync, openSync, readdirSync, readFileSync, readSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs';
6
+ import { closeSync, existsSync, mkdirSync, openSync, readdirSync, readFileSync, readSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs';
7
7
  import { join } from 'node:path';
8
8
  const MAX_BASH_JOB_PURPOSE_BYTES = 280;
9
9
  /** A purpose is display text, never shell input. Keep the file-backed control
@@ -66,6 +66,43 @@ export function readJobPurpose(contextDir, jobId) {
66
66
  return null;
67
67
  }
68
68
  }
69
+ /** Scheduler niceness every command a node runs is started with. Agent work —
70
+ * builds, test suites, typechecks — sits below the daemon, the brokers, the
71
+ * viewers, and the person's own shell, so a fan-out of release builds cannot
72
+ * starve them: the scheduler only honors this under contention and it costs
73
+ * nothing on an idle host. */
74
+ export const DEFAULT_BASH_NICENESS = 10;
75
+ /** The politest priority the scheduler accepts. A NEGATIVE niceness (more
76
+ * priority than the broker itself) needs root: `nice` prints a permission error
77
+ * into the command's own log and runs it at the inherited priority anyway, so
78
+ * the override range stops at 0. */
79
+ export const MAX_BASH_NICENESS = 19;
80
+ /** Per-node override of the bash niceness, written by `crtr node config --nice`
81
+ * and read by the valve when it starts each command. Node-local like the job
82
+ * dirs beside it, so a change applies to the node's very next command without
83
+ * waiting for a revive. */
84
+ export function bashNicenessPath(contextDir) {
85
+ return join(bashJobsDir(contextDir), 'niceness');
86
+ }
87
+ /** The niceness this node's commands run at. Any missing, malformed, or
88
+ * out-of-range file reads as the default rather than an odd priority. */
89
+ export function readBashNiceness(contextDir) {
90
+ try {
91
+ const n = Number(readFileSync(bashNicenessPath(contextDir), 'utf8').trim());
92
+ if (!Number.isInteger(n) || n < 0 || n > MAX_BASH_NICENESS)
93
+ return DEFAULT_BASH_NICENESS;
94
+ return n;
95
+ }
96
+ catch {
97
+ return DEFAULT_BASH_NICENESS;
98
+ }
99
+ }
100
+ /** Set the node's bash niceness. The caller owns validation; the store keeps
101
+ * one integer so the valve's read stays a single file read per command. */
102
+ export function writeBashNiceness(contextDir, niceness) {
103
+ mkdirSync(bashJobsDir(contextDir), { recursive: true });
104
+ writeFileSync(bashNicenessPath(contextDir), `${Math.trunc(niceness)}\n`);
105
+ }
69
106
  /** The auto-background cancel deadline (epoch ms), or undefined for a job with
70
107
  * no deadline (a deliberate handoff, or a legacy job dir). A non-finite or
71
108
  * malformed file reads as no deadline rather than an instant expiry. */
@@ -0,0 +1,19 @@
1
+ export interface DeliveryFailure {
2
+ kind: 'exit' | 'signal' | 'timeout' | 'spawn_error';
3
+ exitCode?: number;
4
+ signal?: string;
5
+ message?: string;
6
+ stderr?: string;
7
+ }
8
+ export type DeliverySettlement = {
9
+ state: 'accepted';
10
+ } | {
11
+ state: 'permanent_failed';
12
+ failure: DeliveryFailure;
13
+ } | {
14
+ state: 'pending';
15
+ nextAttemptAt: number;
16
+ failure: DeliveryFailure;
17
+ };
18
+ /** Durable exponential retry delay for the attempt that just failed. */
19
+ export declare function deliveryBackoffMs(attempt: number): number;
@@ -0,0 +1,8 @@
1
+ // The durable contract shared by every retried delivery: what a failed attempt
2
+ // records, how an attempt settles, and when the next attempt becomes due. The
3
+ // canvas modules own the rows; `daemon/delivery/run-delivery.ts` owns the
4
+ // process that produces these settlements.
5
+ /** Durable exponential retry delay for the attempt that just failed. */
6
+ export function deliveryBackoffMs(attempt) {
7
+ return Math.min(5_000 * 2 ** (attempt - 1), 3_600_000);
8
+ }
@@ -1,11 +1,6 @@
1
+ import type { DeliveryFailure, DeliverySettlement } from './delivery-contract.js';
1
2
  export type ActionDeliveryState = 'pending' | 'running' | 'accepted' | 'permanent_failed';
2
- export interface ActionDeliveryFailure {
3
- kind: 'exit' | 'signal' | 'timeout' | 'spawn_error';
4
- exitCode?: number;
5
- signal?: string;
6
- message?: string;
7
- stderr?: string;
8
- }
3
+ export type ActionDeliveryFailure = DeliveryFailure;
9
4
  export interface ActionDeliveryRecord {
10
5
  requestId: string;
11
6
  state: ActionDeliveryState;
@@ -37,16 +32,7 @@ export declare function readActionDelivery(requestId: string): ActionDeliveryRec
37
32
  export declare function dueActionDeliveries(now: number): ActionDeliveryRecord[];
38
33
  /** Atomically start one due delivery attempt. A null result lost the claim. */
39
34
  export declare function claimActionDelivery(requestId: string, now: number, claimOwner: string): ActionDeliveryRecord | null;
40
- export type ActionDeliverySettlement = {
41
- state: 'accepted';
42
- } | {
43
- state: 'permanent_failed';
44
- failure: ActionDeliveryFailure;
45
- } | {
46
- state: 'pending';
47
- nextAttemptAt: number;
48
- failure: ActionDeliveryFailure;
49
- };
35
+ export type ActionDeliverySettlement = DeliverySettlement;
50
36
  /** Settle only the still-owned running attempt, so a stale process cannot overwrite a newer claim. */
51
37
  export declare function settleActionDelivery(requestId: string, attempt: number, claimOwner: string, settlement: ActionDeliverySettlement, now: number): boolean;
52
38
  /** A new daemon deliberately redelivers interrupted completions at least once. */
@@ -1,14 +1,7 @@
1
+ import type { DeliveryFailure, DeliverySettlement } from './delivery-contract.js';
1
2
  import { type NodeOutcomeKind, type TerminalReason } from './types.js';
2
3
  export type OutcomeDeliveryState = 'armed' | 'pending' | 'running' | 'accepted' | 'permanent_failed';
3
- export interface OutcomeDeliveryFailure {
4
- kind: 'exit' | 'signal' | 'timeout' | 'spawn_error';
5
- exitCode?: number;
6
- signal?: string;
7
- message?: string;
8
- stderr?: string;
9
- }
10
- /** Durable exponential retry delay for the attempt that just failed. */
11
- export declare function outcomeDeliveryBackoffMs(attempt: number): number;
4
+ export type OutcomeDeliveryFailure = DeliveryFailure;
12
5
  export interface OutcomeDeliveryRecord {
13
6
  nodeId: string;
14
7
  state: OutcomeDeliveryState;
@@ -61,16 +54,7 @@ export declare function armOutcomeDelivery(row: {
61
54
  export declare function readOutcomeDelivery(nodeId: string): OutcomeDeliveryRecord | null;
62
55
  export declare function dueOutcomeDeliveries(now: number): OutcomeDeliveryRecord[];
63
56
  export declare function claimOutcomeDelivery(nodeId: string, now: number, claimOwner: string): OutcomeDeliveryRecord | null;
64
- export type OutcomeDeliverySettlement = {
65
- state: 'accepted';
66
- } | {
67
- state: 'permanent_failed';
68
- failure: OutcomeDeliveryFailure;
69
- } | {
70
- state: 'pending';
71
- nextAttemptAt: number;
72
- failure: OutcomeDeliveryFailure;
73
- };
57
+ export type OutcomeDeliverySettlement = DeliverySettlement;
74
58
  /** Settle only the still-owned running attempt, fencing stale delivery processes. */
75
59
  export declare function settleOutcomeDelivery(nodeId: string, attempt: number, claimOwner: string, settlement: OutcomeDeliverySettlement, now: number): boolean;
76
60
  export declare function recoverRunningOutcomeDeliveries(now: number): number;
@@ -3,10 +3,6 @@ import { openDb, withCanvasWrite } from './db.js';
3
3
  import { getNode } from './canvas.js';
4
4
  import { OUTCOME_PAYLOAD_MAX_BYTES } from './types.js';
5
5
  import { composeNodeOutcomeDocument } from '../runtime/outcome-document.js';
6
- /** Durable exponential retry delay for the attempt that just failed. */
7
- export function outcomeDeliveryBackoffMs(attempt) {
8
- return Math.min(5_000 * 2 ** (attempt - 1), 3_600_000);
9
- }
10
6
  function serializedPayload(payload) {
11
7
  if (payload === undefined)
12
8
  return null;
@@ -84,7 +84,7 @@ export const BINDING_CATALOG = Object.freeze([
84
84
  ...attachEntry('crtr.tmux.menu.attach.model-ladder.previous', 'crtr.attach.model-ladder.previous', 'Previous model', 'Move to the previous model ladder rung.', ['['], ['alt+shift+m']),
85
85
  ...attachEntry('crtr.tmux.menu.attach.command.inspect', 'crtr.attach.command.inspect', 'Inspect loaded command', 'Open the loaded slash command’s expanded prompt.', ['h'], ['alt+shift+h']),
86
86
  ...attachEntry('crtr.tmux.menu.attach.file-review', 'crtr.attach.file-review', 'Review a file', 'Open a line-anchored review of a file named in the transcript.', [], ['alt+shift+r']),
87
- ...attachEntry('crtr.tmux.menu.attach.profile-files', 'crtr.attach.profile-files', 'Search profile files', 'Fuzzy-match path segments across the node family’s working directories, profile projects and memory, and ancestor crouter memory stores; Ctrl+Y copies a selected path.', ['f'], ['ctrl+f']),
87
+ ...attachEntry('crtr.tmux.menu.attach.profile-files', 'crtr.attach.profile-files', 'Search profile files', 'Fuzzy-match path segments across the node family’s working directories, their context and reports directories, profile projects and memory, and ancestor crouter memory stores; Ctrl+Y copies a selected path.', ['f'], ['ctrl+f']),
88
88
  ...attachEntry('crtr.tmux.menu.attach.search', 'crtr.attach.search', 'Search transcript', 'Search the transcript as it is currently displayed — folded tool output is not searched until it is unfolded.', ['/'], ['alt+/']),
89
89
  ...attachEntry('crtr.tmux.menu.attach.whip', 'crtr.attach.whip', 'Whip agent', 'Interrupt the agent and send a playful command.', ['w'], []),
90
90
  b('crtr.attach.scroll.up', 'Attach', 'Scroll up', 'Scroll the transcript one line up.', ['shift+up'], 'terminal', ATTACH_CONTEXTS),
@@ -8,7 +8,8 @@ import { createNode } from '../../core/canvas/canvas.js';
8
8
  import { isPidAlive, killProcessGroup } from '../../core/canvas/pid.js';
9
9
  import { armOutcomeDelivery, claimOutcomeDelivery, readOutcomeDelivery } from '../../core/canvas/node-outcome-deliveries.js';
10
10
  import { transition } from '../../core/runtime/lifecycle.js';
11
- import { OUTCOME_DELIVERY_TIMEOUT_MS, deliverOutcome } from '../node-outcome/deliver-outcome.js';
11
+ import { DELIVERY_TIMEOUT_MS } from '../delivery/run-delivery.js';
12
+ import { deliverOutcome } from '../node-outcome/deliver-outcome.js';
12
13
  let home;
13
14
  let processGroupFiles;
14
15
  const priorHome = process.env.CRTR_HOME;
@@ -70,7 +71,7 @@ test('timeout SIGKILL escalation reaps a SIGTERM-ignoring descendant after its l
70
71
  processGroupFiles.push(processGroupPid);
71
72
  const delivery = claimScript('timeout', `#!/bin/sh\necho $$ > ${processGroupPid}\n(\n trap '' TERM\n while :; do sleep 30; done\n) &\necho $! > ${childPid}\ntrap 'exit 0' TERM\nwhile :; do sleep 30; done\n`);
72
73
  const originalSetTimeout = global.setTimeout;
73
- global.setTimeout = ((callback, delay, ...args) => originalSetTimeout(callback, delay === OUTCOME_DELIVERY_TIMEOUT_MS ? 1_000 : delay, ...args));
74
+ global.setTimeout = ((callback, delay, ...args) => originalSetTimeout(callback, delay === DELIVERY_TIMEOUT_MS ? 1_000 : delay, ...args));
74
75
  try {
75
76
  await deliverOutcome(delivery, 'test-daemon');
76
77
  }
@@ -0,0 +1,25 @@
1
+ import { type DeliverySettlement } from '../../core/canvas/delivery-contract.js';
2
+ export declare const DELIVERY_TIMEOUT_MS = 120000;
3
+ export declare const DELIVERY_STDERR_TAIL_BYTES: number;
4
+ export interface DeliveryProcessOutcome {
5
+ exitCode: number | null;
6
+ signal: string | null;
7
+ timedOut: boolean;
8
+ spawnError?: Error;
9
+ stderr: string;
10
+ }
11
+ /** Process contract mapping. stdout deliberately does not participate. */
12
+ export declare function deliverySettlementForOutcome(outcome: DeliveryProcessOutcome, attempt: number, now: number): DeliverySettlement;
13
+ /** One already-claimed attempt: where to deliver, what to deliver, and which
14
+ * attempt this is. The caller resolved argv and cwd when the target was
15
+ * registered, so a later config edit cannot redirect this delivery. */
16
+ export interface DeliveryJob {
17
+ argv: string[];
18
+ cwd: string;
19
+ /** The stored document; its `delivery.attempt` is stamped per attempt. */
20
+ document: unknown;
21
+ attempt: number;
22
+ }
23
+ /** Spawn and settle one already-claimed delivery. `settle` writes the outcome
24
+ * to whichever durable row owns this attempt. */
25
+ export declare function runDelivery(job: DeliveryJob, settle: (settlement: DeliverySettlement) => void): Promise<void>;
@@ -0,0 +1,164 @@
1
+ // One process contract for every durable delivery crouter retries: spawn the
2
+ // declared target, hand it the document on stdin, and settle the attempt from
3
+ // its exit code. Human-request completions and node outcomes differ only in the
4
+ // row they settle, so both call this and neither reimplements the protocol.
5
+ import { spawn } from 'node:child_process';
6
+ import { envHomeOverride } from '../../shared/env.js';
7
+ import { deliveryBackoffMs } from '../../core/canvas/delivery-contract.js';
8
+ import { killProcessGroup } from '../../core/canvas/pid.js';
9
+ import { buildOperationalEnvBase } from '../../core/runtime/spawn-env.js';
10
+ export const DELIVERY_TIMEOUT_MS = 120_000;
11
+ const DELIVERY_KILL_GRACE_MS = 2_000;
12
+ export const DELIVERY_STDERR_TAIL_BYTES = 4 * 1024;
13
+ /** Process contract mapping. stdout deliberately does not participate. */
14
+ export function deliverySettlementForOutcome(outcome, attempt, now) {
15
+ if (outcome.spawnError !== undefined) {
16
+ return {
17
+ state: 'pending',
18
+ nextAttemptAt: now + deliveryBackoffMs(attempt),
19
+ failure: failure('spawn_error', { message: outcome.spawnError.message, stderr: outcome.stderr }),
20
+ };
21
+ }
22
+ if (outcome.timedOut) {
23
+ return {
24
+ state: 'pending',
25
+ nextAttemptAt: now + deliveryBackoffMs(attempt),
26
+ // The kill escalation's own signal distinguishes the graceful term from the SIGKILL.
27
+ failure: failure('timeout', { ...(outcome.signal === null ? {} : { signal: outcome.signal }), stderr: outcome.stderr }),
28
+ };
29
+ }
30
+ if (outcome.signal !== null) {
31
+ return {
32
+ state: 'pending',
33
+ nextAttemptAt: now + deliveryBackoffMs(attempt),
34
+ failure: failure('signal', { signal: outcome.signal, stderr: outcome.stderr }),
35
+ };
36
+ }
37
+ if (outcome.exitCode === 0)
38
+ return { state: 'accepted' };
39
+ if (outcome.exitCode === 78) {
40
+ return { state: 'permanent_failed', failure: failure('exit', { exitCode: 78, stderr: outcome.stderr }) };
41
+ }
42
+ return {
43
+ state: 'pending',
44
+ nextAttemptAt: now + deliveryBackoffMs(attempt),
45
+ failure: failure('exit', { exitCode: outcome.exitCode ?? undefined, stderr: outcome.stderr }),
46
+ };
47
+ }
48
+ function failure(kind, fields) {
49
+ return { kind, ...fields };
50
+ }
51
+ function stderrTailCapture() {
52
+ let tail = Buffer.alloc(0);
53
+ return {
54
+ take(chunk) {
55
+ tail = chunk.length >= DELIVERY_STDERR_TAIL_BYTES
56
+ ? chunk.subarray(chunk.length - DELIVERY_STDERR_TAIL_BYTES)
57
+ : Buffer.concat([tail, chunk]).subarray(Math.max(0, tail.length + chunk.length - DELIVERY_STDERR_TAIL_BYTES));
58
+ },
59
+ decode() {
60
+ return tail.toString('utf8');
61
+ },
62
+ };
63
+ }
64
+ /** The stored document is immutable; attempt is its only per-delivery mutation. */
65
+ function documentForAttempt(document, attempt) {
66
+ const stored = document;
67
+ const delivery = stored.delivery;
68
+ return JSON.stringify({ ...stored, delivery: { ...delivery, attempt } });
69
+ }
70
+ /** The delivery child crosses the same consent boundary as every other
71
+ * crouter spawn: nothing of crtrd's own environment reaches it except what
72
+ * `spawnEnv.allow` and the operational base admit for the target's own cwd.
73
+ * No profile participates — a delivery target is declared in scope config alone. */
74
+ function deliveryEnv(cwd) {
75
+ const env = buildOperationalEnvBase({ targetCwd: cwd, targetProfileId: null });
76
+ // Runtime identity, not host state: a target that calls `crtr` must reach
77
+ // the daemon that delivered to it, exactly as a cron body does.
78
+ const canvasHome = envHomeOverride();
79
+ if (canvasHome !== undefined && canvasHome !== '')
80
+ env['CRTR_HOME'] = canvasHome;
81
+ return env;
82
+ }
83
+ /** Spawn and settle one already-claimed delivery. `settle` writes the outcome
84
+ * to whichever durable row owns this attempt. */
85
+ export async function runDelivery(job, settle) {
86
+ const stderr = stderrTailCapture();
87
+ let document;
88
+ try {
89
+ document = documentForAttempt(job.document, job.attempt);
90
+ }
91
+ catch (error) {
92
+ settle(deliverySettlementForOutcome({
93
+ exitCode: null,
94
+ signal: null,
95
+ timedOut: false,
96
+ spawnError: error,
97
+ stderr: stderr.decode(),
98
+ }, job.attempt, Date.now()));
99
+ return;
100
+ }
101
+ let child;
102
+ try {
103
+ child = spawn(job.argv[0], job.argv.slice(1), {
104
+ cwd: job.cwd,
105
+ env: deliveryEnv(job.cwd),
106
+ detached: true,
107
+ stdio: ['pipe', 'pipe', 'pipe'],
108
+ });
109
+ }
110
+ catch (error) {
111
+ settle(deliverySettlementForOutcome({
112
+ exitCode: null,
113
+ signal: null,
114
+ timedOut: false,
115
+ spawnError: error,
116
+ stderr: stderr.decode(),
117
+ }, job.attempt, Date.now()));
118
+ return;
119
+ }
120
+ await new Promise((resolve) => {
121
+ let settled = false;
122
+ let timedOut = false;
123
+ let killTimer;
124
+ let timeout;
125
+ const finish = (outcome) => {
126
+ if (settled)
127
+ return;
128
+ settled = true;
129
+ if (timeout !== undefined)
130
+ clearTimeout(timeout);
131
+ // A timed-out leader can exit while a SIGTERM-ignoring group member remains.
132
+ if (!outcome.timedOut && killTimer !== undefined)
133
+ clearTimeout(killTimer);
134
+ child.stdin?.destroy();
135
+ child.stdout?.destroy();
136
+ child.stderr?.destroy();
137
+ settle(deliverySettlementForOutcome(outcome, job.attempt, Date.now()));
138
+ resolve();
139
+ };
140
+ child.stdout?.on('data', () => { }); // Drain output; stdout is never a control channel.
141
+ child.stderr?.on('data', (chunk) => stderr.take(chunk));
142
+ child.stdin?.on('error', () => { }); // The child may exit before consuming stdin; its exit code is authoritative.
143
+ child.on('error', (error) => {
144
+ finish({ exitCode: null, signal: null, timedOut: false, spawnError: error, stderr: stderr.decode() });
145
+ });
146
+ child.on('exit', (exitCode, signal) => {
147
+ finish({ exitCode, signal, timedOut, stderr: stderr.decode() });
148
+ });
149
+ const killWithEscalation = () => {
150
+ if (child.pid === undefined)
151
+ return;
152
+ killProcessGroup(child.pid);
153
+ killTimer = setTimeout(() => killProcessGroup(child.pid, 'SIGKILL'), DELIVERY_KILL_GRACE_MS);
154
+ killTimer.unref?.();
155
+ };
156
+ // The deadline starts at spawn, not after stdin write or any output.
157
+ timeout = setTimeout(() => {
158
+ timedOut = true;
159
+ killWithEscalation();
160
+ }, DELIVERY_TIMEOUT_MS);
161
+ timeout.unref?.();
162
+ child.stdin?.end(document, 'utf8');
163
+ });
164
+ }
@@ -1,16 +1,3 @@
1
- import { type ActionDeliveryRecord, type ActionDeliverySettlement } from '../../core/canvas/human-deliveries.js';
2
- export declare const ACTION_DELIVERY_TIMEOUT_MS = 120000;
3
- export declare const ACTION_DELIVERY_STDERR_TAIL_BYTES: number;
4
- /** Durable exponential retry delay for the attempt that just failed. */
5
- export declare function actionDeliveryBackoffMs(attempt: number): number;
6
- export interface ActionDeliveryProcessOutcome {
7
- exitCode: number | null;
8
- signal: string | null;
9
- timedOut: boolean;
10
- spawnError?: Error;
11
- stderr: string;
12
- }
13
- /** Process contract mapping. stdout deliberately does not participate. */
14
- export declare function actionDeliverySettlementForOutcome(outcome: ActionDeliveryProcessOutcome, attempt: number, now: number): ActionDeliverySettlement;
1
+ import { type ActionDeliveryRecord } from '../../core/canvas/human-deliveries.js';
15
2
  /** Spawn and settle one already-claimed action delivery. */
16
3
  export declare function deliverAction(delivery: ActionDeliveryRecord, claimOwner: string): Promise<void>;