@north-light/crouter 0.3.304 → 0.3.306
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/__tests__/integration/provider-rotation.test.ts +87 -0
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.js +31 -26
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.ts +33 -26
- package/dist/clients/attach/viewer.js +581 -581
- package/dist/commands/node/create.js +7 -1
- package/dist/commands/node/wait.js +1 -1
- package/dist/commands/node-worktree.js +5 -1
- package/dist/core/__tests__/fixtures/fake-engine.js +14 -15
- package/dist/core/__tests__/integration/git-optional-locks.test.d.ts +1 -0
- package/dist/core/__tests__/integration/git-optional-locks.test.js +81 -0
- package/dist/core/__tests__/integration/tmux-surface.test.js +53 -0
- package/dist/core/__tests__/kickoff.test.js +40 -0
- package/dist/core/__tests__/seam/broker-provider-retry.test.js +25 -0
- package/dist/core/__tests__/seam/deadline-wake-delivery.test.d.ts +1 -0
- package/dist/core/__tests__/seam/deadline-wake-delivery.test.js +138 -0
- package/dist/core/__tests__/seam/reopen-invalidated-worktree.test.d.ts +1 -0
- package/dist/core/__tests__/seam/reopen-invalidated-worktree.test.js +60 -0
- package/dist/core/fault-classifier.d.ts +6 -0
- package/dist/core/fault-classifier.js +9 -4
- package/dist/core/git.js +16 -2
- package/dist/core/provider-failure.d.ts +30 -0
- package/dist/core/provider-failure.js +86 -0
- package/dist/core/runtime/bearings-render.js +13 -4
- package/dist/core/runtime/broker/fault-retry.js +8 -0
- package/dist/core/runtime/fault.d.ts +2 -1
- package/dist/core/runtime/fault.js +38 -0
- package/dist/core/runtime/kickoff.d.ts +3 -0
- package/dist/core/runtime/kickoff.js +24 -4
- package/dist/core/runtime/reopen.d.ts +2 -2
- package/dist/core/runtime/reopen.js +6 -4
- package/dist/core/runtime/revive.js +3 -3
- package/dist/core/runtime/tmux-bindings.js +14 -3
- package/dist/core/termrender/version.d.ts +1 -1
- package/dist/core/termrender/version.js +1 -1
- package/dist/core/worktree-close.js +6 -3
- package/dist/core/worktree-sweep.js +7 -3
- package/dist/core/worktree.js +6 -3
- package/dist/daemon/api/__tests__/reopen-delivery.test.js +31 -0
- package/dist/daemon/api/__tests__/seam/leaf-api-parity.test.js +36 -4
- package/dist/daemon/api/handlers/messages.js +26 -9
- package/dist/daemon/reconcilers/broker-supervision.js +10 -1
- package/package.json +1 -1
- package/runtime.lock.json +5 -5
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
export declare const PROVIDER_FAILURE_DIAGNOSTIC = "crtr:provider-failure";
|
|
2
|
+
export interface ProviderFailureCause {
|
|
3
|
+
message?: string;
|
|
4
|
+
status?: number;
|
|
5
|
+
code?: string | number;
|
|
6
|
+
errno?: string;
|
|
7
|
+
body?: string;
|
|
8
|
+
}
|
|
9
|
+
export interface ProviderFailureDetails {
|
|
10
|
+
status?: number;
|
|
11
|
+
code?: string | number;
|
|
12
|
+
errno?: string;
|
|
13
|
+
body?: string;
|
|
14
|
+
causes?: ProviderFailureCause[];
|
|
15
|
+
}
|
|
16
|
+
export interface NormalizedProviderFailure extends ProviderFailureDetails {
|
|
17
|
+
message: string;
|
|
18
|
+
}
|
|
19
|
+
/** Converts an arbitrary provider failure into serializable facts before an SDK stream flattens it to `errorMessage`. */
|
|
20
|
+
export declare function normalizeProviderFailure(error: unknown, observed?: {
|
|
21
|
+
status?: number;
|
|
22
|
+
}): NormalizedProviderFailure;
|
|
23
|
+
/** The stable diagnostic carrier for normalized provider error facts on an assistant error message. */
|
|
24
|
+
export declare function providerFailureDiagnostic(details: ProviderFailureDetails): {
|
|
25
|
+
type: string;
|
|
26
|
+
timestamp: number;
|
|
27
|
+
details: Record<string, unknown>;
|
|
28
|
+
};
|
|
29
|
+
/** Reads provider facts from the assistant-message diagnostic without trusting arbitrary nested error objects. */
|
|
30
|
+
export declare function extractProviderFailureDetails(message: unknown): ProviderFailureDetails | undefined;
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import { isRecord } from '../shared/predicates.js';
|
|
2
|
+
export const PROVIDER_FAILURE_DIAGNOSTIC = 'crtr:provider-failure';
|
|
3
|
+
const CAUSE_LIMIT = 8;
|
|
4
|
+
function text(value) {
|
|
5
|
+
return typeof value === 'string' && value.trim() !== '' ? value.trim() : undefined;
|
|
6
|
+
}
|
|
7
|
+
function status(value) {
|
|
8
|
+
return typeof value === 'number' && Number.isFinite(value) ? value : undefined;
|
|
9
|
+
}
|
|
10
|
+
function code(value) {
|
|
11
|
+
return typeof value === 'string' || typeof value === 'number' ? value : undefined;
|
|
12
|
+
}
|
|
13
|
+
function record(value) {
|
|
14
|
+
return isRecord(value) ? value : undefined;
|
|
15
|
+
}
|
|
16
|
+
function failureCause(value) {
|
|
17
|
+
const message = text(value['errorMessage']) ?? text(value['message']) ?? text(value['reason']) ?? text(value['statusText']);
|
|
18
|
+
const cause = {
|
|
19
|
+
...(message === undefined ? {} : { message }),
|
|
20
|
+
...(status(value['status']) === undefined ? {} : { status: status(value['status']) }),
|
|
21
|
+
...(code(value['code']) === undefined ? {} : { code: code(value['code']) }),
|
|
22
|
+
...(text(value['errno']) === undefined ? {} : { errno: text(value['errno']) }),
|
|
23
|
+
...(text(value['body']) === undefined ? {} : { body: text(value['body']) }),
|
|
24
|
+
};
|
|
25
|
+
return Object.keys(cause).length === 0 ? undefined : cause;
|
|
26
|
+
}
|
|
27
|
+
/** Converts an arbitrary provider failure into serializable facts before an SDK stream flattens it to `errorMessage`. */
|
|
28
|
+
export function normalizeProviderFailure(error, observed = {}) {
|
|
29
|
+
const root = record(error);
|
|
30
|
+
const rootCause = root === undefined ? undefined : failureCause(root);
|
|
31
|
+
const causes = [];
|
|
32
|
+
const seen = new Set([error]);
|
|
33
|
+
let current = root === undefined ? undefined : record(root['cause']);
|
|
34
|
+
while (current !== undefined && !seen.has(current) && causes.length < CAUSE_LIMIT) {
|
|
35
|
+
seen.add(current);
|
|
36
|
+
const cause = failureCause(current);
|
|
37
|
+
if (cause !== undefined)
|
|
38
|
+
causes.push(cause);
|
|
39
|
+
current = record(current['cause']);
|
|
40
|
+
}
|
|
41
|
+
const message = rootCause?.message ?? (error instanceof Error ? error.message : undefined) ?? String(error ?? 'Provider request failed');
|
|
42
|
+
return {
|
|
43
|
+
message,
|
|
44
|
+
...((observed.status ?? rootCause?.status) === undefined ? {} : { status: observed.status ?? rootCause?.status }),
|
|
45
|
+
...(rootCause?.code === undefined ? {} : { code: rootCause.code }),
|
|
46
|
+
...(rootCause?.errno === undefined ? {} : { errno: rootCause.errno }),
|
|
47
|
+
...(rootCause?.body === undefined ? {} : { body: rootCause.body }),
|
|
48
|
+
...(causes.length === 0 ? {} : { causes }),
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
/** The stable diagnostic carrier for normalized provider error facts on an assistant error message. */
|
|
52
|
+
export function providerFailureDiagnostic(details) {
|
|
53
|
+
return { type: PROVIDER_FAILURE_DIAGNOSTIC, timestamp: Date.now(), details: { ...details } };
|
|
54
|
+
}
|
|
55
|
+
function providerFailureDetails(value) {
|
|
56
|
+
const diagnostics = record(value)?.['diagnostics'];
|
|
57
|
+
if (!Array.isArray(diagnostics))
|
|
58
|
+
return undefined;
|
|
59
|
+
for (const diagnostic of diagnostics) {
|
|
60
|
+
const diagnosticRecord = record(diagnostic);
|
|
61
|
+
if (diagnosticRecord?.['type'] !== PROVIDER_FAILURE_DIAGNOSTIC)
|
|
62
|
+
continue;
|
|
63
|
+
const details = record(diagnosticRecord['details']);
|
|
64
|
+
if (details === undefined)
|
|
65
|
+
continue;
|
|
66
|
+
const causes = Array.isArray(details['causes'])
|
|
67
|
+
? details['causes'].flatMap((item) => {
|
|
68
|
+
const cause = record(item);
|
|
69
|
+
const normalized = cause === undefined ? undefined : failureCause(cause);
|
|
70
|
+
return normalized === undefined ? [] : [normalized];
|
|
71
|
+
})
|
|
72
|
+
: undefined;
|
|
73
|
+
return {
|
|
74
|
+
...(status(details['status']) === undefined ? {} : { status: status(details['status']) }),
|
|
75
|
+
...(code(details['code']) === undefined ? {} : { code: code(details['code']) }),
|
|
76
|
+
...(text(details['errno']) === undefined ? {} : { errno: text(details['errno']) }),
|
|
77
|
+
...(text(details['body']) === undefined ? {} : { body: text(details['body']) }),
|
|
78
|
+
...(causes === undefined || causes.length === 0 ? {} : { causes }),
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
return undefined;
|
|
82
|
+
}
|
|
83
|
+
/** Reads provider facts from the assistant-message diagnostic without trusting arbitrary nested error objects. */
|
|
84
|
+
export function extractProviderFailureDetails(message) {
|
|
85
|
+
return providerFailureDetails(message);
|
|
86
|
+
}
|
|
@@ -57,8 +57,8 @@ function directoryListingLines(cwd) {
|
|
|
57
57
|
return ['Directory listing: unavailable (cwd missing or unreadable).'];
|
|
58
58
|
}
|
|
59
59
|
}
|
|
60
|
-
function runCommand(cwd, cmd, args) {
|
|
61
|
-
const result = spawnSync(cmd, args, { cwd, encoding: 'utf8' });
|
|
60
|
+
function runCommand(cwd, cmd, args, env) {
|
|
61
|
+
const result = spawnSync(cmd, args, { cwd, encoding: 'utf8', ...(env === undefined ? {} : { env }) });
|
|
62
62
|
if (result.error !== undefined) {
|
|
63
63
|
return { ok: false, stdout: '', stderr: result.error.message };
|
|
64
64
|
}
|
|
@@ -68,8 +68,17 @@ function runCommand(cwd, cmd, args) {
|
|
|
68
68
|
stderr: (result.stderr ?? '').trim(),
|
|
69
69
|
};
|
|
70
70
|
}
|
|
71
|
+
/** This snapshot renders on every node boot and every revive, which makes it the
|
|
72
|
+
* most frequent git caller in the runtime — and boot is exactly when a process
|
|
73
|
+
* gets killed. `git status` would take `.git/index.lock` to refresh the stat
|
|
74
|
+
* cache; suppressing that optional lock keeps an interrupted render from
|
|
75
|
+
* orphaning a lock that then blocks every writer in the user's checkout.
|
|
76
|
+
* Only optional locks are affected, and the output is unchanged. */
|
|
77
|
+
function gitEnv() {
|
|
78
|
+
return { ...process.env, GIT_OPTIONAL_LOCKS: '0' };
|
|
79
|
+
}
|
|
71
80
|
function runGit(cwd, args) {
|
|
72
|
-
const result = runCommand(cwd, 'git', args);
|
|
81
|
+
const result = runCommand(cwd, 'git', args, gitEnv());
|
|
73
82
|
return result.ok ? result.stdout : '';
|
|
74
83
|
}
|
|
75
84
|
function parseWorktrees(lines) {
|
|
@@ -98,7 +107,7 @@ function parseWorktrees(lines) {
|
|
|
98
107
|
* benign state. Every other probe here is genuinely optional and already
|
|
99
108
|
* falls back to 'unknown'/'HEAD'/a zero count on failure, which stays as-is. */
|
|
100
109
|
function gitSnapshotLines(cwd) {
|
|
101
|
-
const probe = runCommand(cwd, 'git', ['rev-parse', '--is-inside-work-tree']);
|
|
110
|
+
const probe = runCommand(cwd, 'git', ['rev-parse', '--is-inside-work-tree'], gitEnv());
|
|
102
111
|
if (!probe.ok) {
|
|
103
112
|
if (/not a git repository/i.test(probe.stderr)) {
|
|
104
113
|
return ['Git: not a git repository.'];
|
|
@@ -4,6 +4,7 @@ import { isContextOverflow } from '@earendil-works/pi-ai';
|
|
|
4
4
|
import { classify } from '../../fault-classifier.js';
|
|
5
5
|
import { admitProviderRetryEpisode, clearFault, clearProviderRetryEpisode, invalidateProviderRetryEpisode, isModelNotFoundError, isProviderUnconfiguredError, readFault, readProviderRetryEpisode, recordFault, recordPendingProviderRetryFault, reopenProviderRetryEpisode, } from '../fault.js';
|
|
6
6
|
import { extractCoolingDeadline } from '../managed-provider-cooling.js';
|
|
7
|
+
import { extractProviderFailureDetails } from '../../provider-failure.js';
|
|
7
8
|
import { expandModelCandidates, modelRequestFromConfig, probeRouteAvailability } from '../../model-routes.js';
|
|
8
9
|
import { operationIdContext } from '../../events/operation-id.js';
|
|
9
10
|
import { emitEvent } from '../../events/emit.js';
|
|
@@ -131,6 +132,12 @@ export class FaultRetry {
|
|
|
131
132
|
const uncompactedOverflow = isContextOverflow(last, settledSession.model?.contextWindow);
|
|
132
133
|
const classified = classify('pi→provider', last);
|
|
133
134
|
const coolingDeadline = extractCoolingDeadline(last);
|
|
135
|
+
const failure = extractProviderFailureDetails(last);
|
|
136
|
+
const provider = typeof last.provider === 'string' && last.provider !== '' ? last.provider : settledSession.model?.provider;
|
|
137
|
+
const model = typeof last.model === 'string' && last.model !== '' ? last.model : settledSession.model?.id;
|
|
138
|
+
const providerError = failure === undefined || provider === undefined || model === undefined
|
|
139
|
+
? undefined
|
|
140
|
+
: { provider, model, ...failure };
|
|
134
141
|
if (uncompactedOverflow) {
|
|
135
142
|
clearFault(this.deps.nodeId, { link: 'pi→provider' });
|
|
136
143
|
clearProviderRetryEpisode(this.deps.nodeId);
|
|
@@ -151,6 +158,7 @@ export class FaultRetry {
|
|
|
151
158
|
anchorEntryId: !uncompactedOverflow && classified.disposition === 'auto' && episode?.anchorEntryId !== undefined
|
|
152
159
|
? episode.anchorEntryId
|
|
153
160
|
: settledSession.sessionManager.getLeafId?.() ?? undefined,
|
|
161
|
+
...(providerError === undefined ? {} : { providerError }),
|
|
154
162
|
};
|
|
155
163
|
if (faultInput.retry.disposition === 'auto')
|
|
156
164
|
this.recordPendingProviderFault(settledSession, faultInput);
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type Fault, type FaultKind, type FaultLink, type FaultRetry } from '../fault-classifier.js';
|
|
1
|
+
import { type Fault, type FaultKind, type FaultLink, type FaultProviderError, type FaultRetry } from '../fault-classifier.js';
|
|
2
2
|
export type { Fault, FaultKind, FaultLink, FaultRetry, FaultRetryDisposition, FaultRetryOwner } from '../fault-classifier.js';
|
|
3
3
|
export declare const FAULT_QUIET_MS: number;
|
|
4
4
|
export interface FaultRecordInput {
|
|
@@ -10,6 +10,7 @@ export interface FaultRecordInput {
|
|
|
10
10
|
operation_id?: string;
|
|
11
11
|
since?: string;
|
|
12
12
|
anchorEntryId?: string;
|
|
13
|
+
providerError?: FaultProviderError;
|
|
13
14
|
}
|
|
14
15
|
/** A provider-retry episode outlives its broker-local timer. Its transcript
|
|
15
16
|
* path and anchor are explicit coordinates, so replacement invalidation never
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { envNodeId } from '../../shared/env.js';
|
|
2
|
+
import { isRecord } from '../../shared/predicates.js';
|
|
2
3
|
import { existsSync, mkdirSync, readFileSync, realpathSync, rmSync, writeFileSync } from 'node:fs';
|
|
3
4
|
import { join } from 'node:path';
|
|
4
5
|
import { jobDir } from '../canvas/paths.js';
|
|
@@ -33,6 +34,37 @@ function faultPriority(fault) {
|
|
|
33
34
|
}
|
|
34
35
|
const faultLinks = ['pi→provider', 'viewer↔broker', 'relay↔broker', 'viewer↔crtrd', 'broker↔crtrd', 'daemon→node', 'crtr→pi'];
|
|
35
36
|
const faultKinds = ['rate-limit', 'overloaded', 'connection', 'auth', 'protocol', 'context-overflow', 'other', 'wedged', 'model-not-found'];
|
|
37
|
+
function parseProviderError(value) {
|
|
38
|
+
if (!isRecord(value) || typeof value.provider !== 'string' || value.provider === '' || typeof value.model !== 'string' || value.model === '')
|
|
39
|
+
return undefined;
|
|
40
|
+
const status = typeof value.status === 'number' && Number.isFinite(value.status) ? value.status : undefined;
|
|
41
|
+
const code = typeof value.code === 'string' || typeof value.code === 'number' ? value.code : undefined;
|
|
42
|
+
const errno = typeof value.errno === 'string' ? value.errno : undefined;
|
|
43
|
+
const body = typeof value.body === 'string' ? value.body : undefined;
|
|
44
|
+
const causes = Array.isArray(value.causes)
|
|
45
|
+
? value.causes.flatMap((item) => {
|
|
46
|
+
if (!isRecord(item))
|
|
47
|
+
return [];
|
|
48
|
+
const cause = {
|
|
49
|
+
...(typeof item.message === 'string' ? { message: item.message } : {}),
|
|
50
|
+
...(typeof item.status === 'number' && Number.isFinite(item.status) ? { status: item.status } : {}),
|
|
51
|
+
...(typeof item.code === 'string' || typeof item.code === 'number' ? { code: item.code } : {}),
|
|
52
|
+
...(typeof item.errno === 'string' ? { errno: item.errno } : {}),
|
|
53
|
+
...(typeof item.body === 'string' ? { body: item.body } : {}),
|
|
54
|
+
};
|
|
55
|
+
return Object.keys(cause).length === 0 ? [] : [cause];
|
|
56
|
+
})
|
|
57
|
+
: undefined;
|
|
58
|
+
return {
|
|
59
|
+
provider: value.provider,
|
|
60
|
+
model: value.model,
|
|
61
|
+
...(status === undefined ? {} : { status }),
|
|
62
|
+
...(code === undefined ? {} : { code }),
|
|
63
|
+
...(errno === undefined ? {} : { errno }),
|
|
64
|
+
...(body === undefined ? {} : { body }),
|
|
65
|
+
...(causes === undefined || causes.length === 0 ? {} : { causes }),
|
|
66
|
+
};
|
|
67
|
+
}
|
|
36
68
|
function parseFault(raw) {
|
|
37
69
|
try {
|
|
38
70
|
const parsed = JSON.parse(raw);
|
|
@@ -60,6 +92,9 @@ function parseFault(raw) {
|
|
|
60
92
|
return null;
|
|
61
93
|
if (parsed.anchorEntryId !== undefined && (typeof parsed.anchorEntryId !== 'string' || parsed.anchorEntryId === ''))
|
|
62
94
|
return null;
|
|
95
|
+
const providerError = parsed.providerError === undefined ? undefined : parseProviderError(parsed.providerError);
|
|
96
|
+
if (parsed.providerError !== undefined && providerError === undefined)
|
|
97
|
+
return null;
|
|
63
98
|
return {
|
|
64
99
|
link: parsed.link,
|
|
65
100
|
op: parsed.op,
|
|
@@ -75,6 +110,7 @@ function parseFault(raw) {
|
|
|
75
110
|
since: parsed.since,
|
|
76
111
|
operation_id: operationId,
|
|
77
112
|
...(parsed.anchorEntryId === undefined ? {} : { anchorEntryId: parsed.anchorEntryId }),
|
|
113
|
+
...(providerError === undefined ? {} : { providerError }),
|
|
78
114
|
};
|
|
79
115
|
}
|
|
80
116
|
catch {
|
|
@@ -166,6 +202,7 @@ function storedFault(input) {
|
|
|
166
202
|
since: input.since ?? new Date().toISOString(),
|
|
167
203
|
operation_id: resolveOperationId(input.operation_id),
|
|
168
204
|
...(typeof input.anchorEntryId === 'string' && input.anchorEntryId !== '' ? { anchorEntryId: input.anchorEntryId } : {}),
|
|
205
|
+
...(input.providerError === undefined ? {} : { providerError: input.providerError }),
|
|
169
206
|
};
|
|
170
207
|
}
|
|
171
208
|
function emitRecordedFault(nodeId, fault) {
|
|
@@ -182,6 +219,7 @@ function emitRecordedFault(nodeId, fault) {
|
|
|
182
219
|
retry: fault.retry,
|
|
183
220
|
message: fault.message,
|
|
184
221
|
...(fault.anchorEntryId === undefined ? {} : { anchor_entry_id: fault.anchorEntryId }),
|
|
222
|
+
...(fault.providerError === undefined ? {} : { provider_error: fault.providerError }),
|
|
185
223
|
},
|
|
186
224
|
});
|
|
187
225
|
}
|
|
@@ -26,6 +26,9 @@ export declare function readYieldMessage(nodeId: string): string | null;
|
|
|
26
26
|
export declare function clearYieldMessage(nodeId: string): void;
|
|
27
27
|
/** List the node's context/ dir (filenames, sorted). Empty when absent. */
|
|
28
28
|
export declare function listContextDir(nodeId: string): string[];
|
|
29
|
+
/** A retained report proves this node has already done work. It must restart
|
|
30
|
+
* from current context, never by replaying its birth mandate. */
|
|
31
|
+
export declare function hasRetainedReports(nodeId: string): boolean;
|
|
29
32
|
export interface ReviveBearings {
|
|
30
33
|
/** The one-shot yield note left by the prior self, pending boot confirmation. */
|
|
31
34
|
yieldMsg: string | null;
|
|
@@ -99,6 +99,12 @@ export function listContextDir(nodeId) {
|
|
|
99
99
|
return [];
|
|
100
100
|
return readdirSync(dir).sort();
|
|
101
101
|
}
|
|
102
|
+
/** A retained report proves this node has already done work. It must restart
|
|
103
|
+
* from current context, never by replaying its birth mandate. */
|
|
104
|
+
export function hasRetainedReports(nodeId) {
|
|
105
|
+
const dir = reportsDir(nodeId);
|
|
106
|
+
return existsSync(dir) && readdirSync(dir).some((file) => file.endsWith('.md'));
|
|
107
|
+
}
|
|
102
108
|
/** Capture revive bearings for `meta`: read the pending yield note, advance the
|
|
103
109
|
* feed cursor past unread reports, and capture external persona framing.
|
|
104
110
|
* Revive paths call it once, then pass the result to the pure kickoff builder. */
|
|
@@ -113,9 +119,23 @@ export function drainBearings(meta) {
|
|
|
113
119
|
let unreadDigest = null;
|
|
114
120
|
if (entries.length > 0) {
|
|
115
121
|
writeCursor(nodeId, entries.at(-1).entry_id);
|
|
116
|
-
// The launch
|
|
117
|
-
//
|
|
118
|
-
|
|
122
|
+
// The launch argv carries one message. Keep coalescing card-free runs, but
|
|
123
|
+
// start a new batch at each entry with cards. A batch's cards are therefore
|
|
124
|
+
// directly ahead of its coalesced text, so they cannot leap an earlier body.
|
|
125
|
+
const cardsFor = (entry) => {
|
|
126
|
+
const cards = entry.data?.['cards'];
|
|
127
|
+
return Array.isArray(cards) ? cards.filter((card) => typeof card === 'string') : [];
|
|
128
|
+
};
|
|
129
|
+
const segments = [];
|
|
130
|
+
for (const entry of entries) {
|
|
131
|
+
if (cardsFor(entry).length > 0 || segments.length === 0)
|
|
132
|
+
segments.push([]);
|
|
133
|
+
segments.at(-1).push(entry);
|
|
134
|
+
}
|
|
135
|
+
unreadDigest = segments.map((segment) => [
|
|
136
|
+
...cardsFor(segment[0]),
|
|
137
|
+
coalesce(segment).map((delivery) => delivery.text).join('\n\n'),
|
|
138
|
+
].join('\n\n')).join('\n\n');
|
|
119
139
|
}
|
|
120
140
|
const drift = personaDrift(nodeId);
|
|
121
141
|
const driftGuidance = drift?.guidance === undefined || drift.guidance === ''
|
|
@@ -261,7 +281,7 @@ export function buildReviveKickoff(meta, bearings, wakeReason) {
|
|
|
261
281
|
// (audit 2026-06-09, Bug 4: mq45b6ch-9ecc2f03), whose only mandate on disk IS
|
|
262
282
|
// the goal. Surface the goal then. When a roadmap exists it stays the sole
|
|
263
283
|
// source of truth (the goal is usually stale by comparison) — unchanged.
|
|
264
|
-
if (!hasRoadmapBody) {
|
|
284
|
+
if (!hasRoadmapBody && !hasRetainedReports(nodeId)) {
|
|
265
285
|
const goal = readGoal(nodeId);
|
|
266
286
|
if (goal !== null && goal.trim() !== '') {
|
|
267
287
|
parts.push(`<goal file="${goalPath(nodeId)}">\n${goal.trim()}\n</goal>`);
|
|
@@ -25,14 +25,14 @@ export declare function commitReopen(nodeId: string, expectedFinalReport: string
|
|
|
25
25
|
* unlatched node is valid here — it still becomes resident. Writing lifecycle
|
|
26
26
|
* first leaves a failed latch CAS with the canonical-final pointer intact.
|
|
27
27
|
* Both writes finish before either delivery channel begins. */
|
|
28
|
-
export declare function commitReopenResident(nodeId: string):
|
|
28
|
+
export declare function commitReopenResident(nodeId: string, expectedFinalReport?: string): string | null;
|
|
29
29
|
/** Convenience wrapper for a doorway with NO intervening fallible step between
|
|
30
30
|
* the gate check and the reopen clear (`node lifecycle revive` has nothing
|
|
31
31
|
* fallible between the gate and `reviveNode`). Validates and, when `reopen`,
|
|
32
32
|
* clears in one call.
|
|
33
33
|
*
|
|
34
34
|
* - `reopen` false + latched → throws `node_finalized`.
|
|
35
|
-
* - `reopen` true + latched → CAS-clears the latch (side effect) and returns.
|
|
35
|
+
* - `reopen` true + latched → makes the node resident and CAS-clears the latch (side effect) and returns.
|
|
36
36
|
* - `reopen` true + not latched → throws `not_finalized`.
|
|
37
37
|
* - `reopen` false + not latched → no-op (the common case). */
|
|
38
38
|
export declare function assertReopenGate(nodeId: string, reopen: boolean): void;
|
|
@@ -104,7 +104,7 @@ export function commitReopen(nodeId, expectedFinalReport) {
|
|
|
104
104
|
* unlatched node is valid here — it still becomes resident. Writing lifecycle
|
|
105
105
|
* first leaves a failed latch CAS with the canonical-final pointer intact.
|
|
106
106
|
* Both writes finish before either delivery channel begins. */
|
|
107
|
-
export function commitReopenResident(nodeId) {
|
|
107
|
+
export function commitReopenResident(nodeId, expectedFinalReport) {
|
|
108
108
|
const finalReport = currentFinalReport(nodeId);
|
|
109
109
|
updateNode(nodeId, { lifecycle: 'resident' });
|
|
110
110
|
// A reopened node that is still parked in a terminal status has to come back
|
|
@@ -116,7 +116,8 @@ export function commitReopenResident(nodeId) {
|
|
|
116
116
|
transition(nodeId, 'revive');
|
|
117
117
|
}
|
|
118
118
|
if (finalReport !== null)
|
|
119
|
-
commitReopen(nodeId, finalReport);
|
|
119
|
+
commitReopen(nodeId, expectedFinalReport ?? finalReport);
|
|
120
|
+
return finalReport;
|
|
120
121
|
}
|
|
121
122
|
/** Convenience wrapper for a doorway with NO intervening fallible step between
|
|
122
123
|
* the gate check and the reopen clear (`node lifecycle revive` has nothing
|
|
@@ -124,12 +125,13 @@ export function commitReopenResident(nodeId) {
|
|
|
124
125
|
* clears in one call.
|
|
125
126
|
*
|
|
126
127
|
* - `reopen` false + latched → throws `node_finalized`.
|
|
127
|
-
* - `reopen` true + latched → CAS-clears the latch (side effect) and returns.
|
|
128
|
+
* - `reopen` true + latched → makes the node resident and CAS-clears the latch (side effect) and returns.
|
|
128
129
|
* - `reopen` true + not latched → throws `not_finalized`.
|
|
129
130
|
* - `reopen` false + not latched → no-op (the common case). */
|
|
130
131
|
export function assertReopenGate(nodeId, reopen) {
|
|
131
132
|
if (reopen) {
|
|
132
|
-
|
|
133
|
+
const expectedFinalReport = assertFinalizedForReopen(nodeId);
|
|
134
|
+
commitReopenResident(nodeId, expectedFinalReport);
|
|
133
135
|
return;
|
|
134
136
|
}
|
|
135
137
|
assertNotFinalized(nodeId);
|
|
@@ -31,7 +31,7 @@ import { isParked, transition } from './lifecycle.js';
|
|
|
31
31
|
import { fanDoctrineWake } from './close.js';
|
|
32
32
|
import { InputError } from '../io.js';
|
|
33
33
|
import { buildPiArgv } from './launch.js';
|
|
34
|
-
import { buildReviveKickoff, drainBearings, readGoal, readYieldMessage, RUNTIME_RESTART_CONTINUATION } from './kickoff.js';
|
|
34
|
+
import { buildReviveKickoff, drainBearings, hasRetainedReports, readGoal, readYieldMessage, RUNTIME_RESTART_CONTINUATION } from './kickoff.js';
|
|
35
35
|
import { readRoadmap } from './roadmap.js';
|
|
36
36
|
import { encodeKickoffOrigin, KICKOFF_ORIGIN_ENV } from './stamp/protocol.js';
|
|
37
37
|
import { headlessBrokerHost } from './host.js';
|
|
@@ -105,7 +105,7 @@ function resumableSessionArgs(nodeId, meta, resume) {
|
|
|
105
105
|
}
|
|
106
106
|
// Return type
|
|
107
107
|
/** A relaunch that has NO transcript to resume or branch from, and no durable
|
|
108
|
-
* work state on disk (no roadmap,
|
|
108
|
+
* work state on disk (no roadmap, yield note, or retained report), is not a revive at all — the
|
|
109
109
|
* node never got a conversation off the ground, so there is no "previous
|
|
110
110
|
* in-memory context" that was lost. pi does not write a `.jsonl` until the
|
|
111
111
|
* first turn, so an idle node the user never messaged is the common case: it
|
|
@@ -116,7 +116,7 @@ function resumableSessionArgs(nodeId, meta, resume) {
|
|
|
116
116
|
* a BIRTH instead: replay the spawn mandate when it has one, or boot with no
|
|
117
117
|
* prompt at all and leave it sitting exactly where it was. */
|
|
118
118
|
function isUnstartedBirth(nodeId) {
|
|
119
|
-
return readRoadmap(nodeId) === null && readYieldMessage(nodeId) === null;
|
|
119
|
+
return readRoadmap(nodeId) === null && readYieldMessage(nodeId) === null && !hasRetainedReports(nodeId);
|
|
120
120
|
}
|
|
121
121
|
/** Whether a fresh context window would have anything on disk to stand on: a
|
|
122
122
|
* goal, a roadmap, or both. The negative is what makes a fresh cycle worse
|
|
@@ -586,14 +586,25 @@ function reconcileTmuxBindings(bindingSnapshot, passthroughSnapshot = []) {
|
|
|
586
586
|
}
|
|
587
587
|
}
|
|
588
588
|
// Restore every wrapper crouter still owns before the ordinary binding sweep.
|
|
589
|
-
// A live line that
|
|
590
|
-
// is protected from this and every later
|
|
591
|
-
// passthrough preference off and on again.
|
|
589
|
+
// A live line that matches neither the fingerprint nor the original crouter
|
|
590
|
+
// recorded is a user's later rebind: it is protected from this and every later
|
|
591
|
+
// reconciliation until they toggle the passthrough preference off and on again.
|
|
592
|
+
//
|
|
593
|
+
// A live line equal to the ORIGINAL is not a rebind. It is the user's own tmux
|
|
594
|
+
// config re-sourced over the wrapper, which every `source-file` and every fresh
|
|
595
|
+
// tmux server does. Relinquishing there disarmed passthrough silently and
|
|
596
|
+
// permanently for anyone whose config binds a key crouter wraps, so that case
|
|
597
|
+
// falls through with nothing to restore — the wrap pass below re-reads the live
|
|
598
|
+
// line as the original and wraps it again. It also releases a key already stuck
|
|
599
|
+
// relinquished: matching the recorded original is proof the user did not rebind
|
|
600
|
+
// it to something of their own.
|
|
592
601
|
const protectedRootKeys = new Set();
|
|
593
602
|
const wrappers = [];
|
|
594
603
|
for (const prior of manifest.wrappers) {
|
|
595
604
|
const desired = desiredWrappers.get(prior.key);
|
|
596
605
|
const live = bindingAt(ROOT_TABLE, prior.key);
|
|
606
|
+
if (live !== null && live === prior.original)
|
|
607
|
+
continue;
|
|
597
608
|
if (prior.state === 'relinquished' || live !== prior.wrapper) {
|
|
598
609
|
protectedRootKeys.add(prior.key);
|
|
599
610
|
if (desired !== undefined) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
export declare const TERMRENDER_VERSION = "4.12.
|
|
1
|
+
export declare const TERMRENDER_VERSION = "4.12.12";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
export const TERMRENDER_VERSION = '4.12.
|
|
1
|
+
export const TERMRENDER_VERSION = '4.12.12';
|
|
@@ -100,18 +100,21 @@ async function landIntoBaseCheckout(baseCheckout, branch, baseSha, landedSha) {
|
|
|
100
100
|
const stashSha = after.status === 0 ? after.stdout.trim() : null;
|
|
101
101
|
if (stashSha === null || stashSha === beforeSha)
|
|
102
102
|
return merge();
|
|
103
|
+
const recovery = `local changes from ${baseCheckout} are retained in stash ${stashSha}; recover them with \`git -C ${shellQuote(baseCheckout)} stash apply ${stashSha}\` and resolve any conflict.`;
|
|
103
104
|
const landed = await merge();
|
|
104
105
|
if (!landed.ok) {
|
|
105
|
-
if ((await gitAsync(['stash', 'apply', stashSha], baseCheckout)).status === 0)
|
|
106
|
+
if ((await gitAsync(['stash', 'apply', stashSha], baseCheckout)).status === 0) {
|
|
106
107
|
await dropStashIfStillTip(baseCheckout, stashSha);
|
|
107
|
-
|
|
108
|
+
return landed;
|
|
109
|
+
}
|
|
110
|
+
return { ok: false, detail: `${landed.detail}\n${recovery}` };
|
|
108
111
|
}
|
|
109
112
|
if ((await gitAsync(['stash', 'apply', stashSha], baseCheckout)).status === 0) {
|
|
110
113
|
await dropStashIfStillTip(baseCheckout, stashSha);
|
|
111
114
|
return { ok: true };
|
|
112
115
|
}
|
|
113
116
|
await gitAsync(['reset', '--hard', landedSha], baseCheckout);
|
|
114
|
-
return { ok: true, stash: `
|
|
117
|
+
return { ok: true, stash: `the shared base checkout was reset to landed commit ${landedSha} because local changes conflicted with it. ${recovery}` };
|
|
115
118
|
}
|
|
116
119
|
async function closeLocked(nodeId) {
|
|
117
120
|
const node = getNode(nodeId);
|
|
@@ -23,6 +23,7 @@ import { withExclusiveDirectoryLockAsync } from './exclusive-lock.js';
|
|
|
23
23
|
import { gitAsync } from './git.js';
|
|
24
24
|
import { proveBranchContained, upstreamTip } from './worktree-containment.js';
|
|
25
25
|
import { parseStatusPorcelain, planLandIntoCheckout, refDeleteTransactionStdin } from './worktree-landing.js';
|
|
26
|
+
import { shellQuote } from '../shared/shell-quote.js';
|
|
26
27
|
import { WORKTREE_LOCK_WAIT_MS, checkoutPathForBranchInList, persistClosedWorktreeState, samePath, worktreeLockPathForCommonDir } from './worktree.js';
|
|
27
28
|
/** Escalating recheck interval, in minutes, indexed by consecutive refusal. The
|
|
28
29
|
* last tier is the cap AND the quarantine threshold: at that point the record
|
|
@@ -196,18 +197,21 @@ async function landIntoBaseCheckoutAsync(baseCheckout, branch, baseSha, landedSh
|
|
|
196
197
|
const stashSha = await revParseRef(baseCheckout, 'refs/stash');
|
|
197
198
|
if (stashSha === null || stashSha === beforeSha)
|
|
198
199
|
return merge(); // nothing was saved
|
|
200
|
+
const recovery = `local changes from ${baseCheckout} are retained in stash ${stashSha}; recover them with \`git -C ${shellQuote(baseCheckout)} stash apply ${stashSha}\` and resolve any conflict.`;
|
|
199
201
|
const landed = await merge();
|
|
200
202
|
if (!landed.ok) {
|
|
201
|
-
if ((await gitAsync(['stash', 'apply', stashSha], baseCheckout)).status === 0)
|
|
203
|
+
if ((await gitAsync(['stash', 'apply', stashSha], baseCheckout)).status === 0) {
|
|
202
204
|
await dropStashIfStillTip(baseCheckout, stashSha);
|
|
203
|
-
|
|
205
|
+
return landed;
|
|
206
|
+
}
|
|
207
|
+
return { ok: false, detail: `${landed.detail}\n${recovery}` };
|
|
204
208
|
}
|
|
205
209
|
if ((await gitAsync(['stash', 'apply', stashSha], baseCheckout)).status === 0) {
|
|
206
210
|
await dropStashIfStillTip(baseCheckout, stashSha);
|
|
207
211
|
return { ok: true };
|
|
208
212
|
}
|
|
209
213
|
await gitAsync(['reset', '--hard', landedSha], baseCheckout);
|
|
210
|
-
return { ok: true, stash: `
|
|
214
|
+
return { ok: true, stash: `the shared base checkout was reset to landed commit ${landedSha} because local changes conflicted with it. ${recovery}` };
|
|
211
215
|
}
|
|
212
216
|
async function dropStashIfStillTip(cwd, stashSha) {
|
|
213
217
|
if ((await revParseRef(cwd, 'refs/stash')) === stashSha)
|
package/dist/core/worktree.js
CHANGED
|
@@ -412,13 +412,16 @@ function landIntoBaseCheckout(baseCheckout, branch, baseSha, landedSha) {
|
|
|
412
412
|
const stashSha = after.status === 0 ? after.stdout.trim() : null;
|
|
413
413
|
if (stashSha === null || stashSha === beforeSha)
|
|
414
414
|
return merge(); // nothing was saved
|
|
415
|
+
const recovery = `local changes from ${baseCheckout} are retained in stash ${stashSha}; recover them with \`git -C ${shellQuote(baseCheckout)} stash apply ${stashSha}\` and resolve any conflict.`;
|
|
415
416
|
const landed = merge();
|
|
416
417
|
if (!landed.ok) {
|
|
417
418
|
// The failed merge left the tree exactly as the stash found it, so this
|
|
418
419
|
// re-apply is clean; restore before surfacing the refusal.
|
|
419
|
-
if (gitSync(['stash', 'apply', stashSha], baseCheckout).status === 0)
|
|
420
|
+
if (gitSync(['stash', 'apply', stashSha], baseCheckout).status === 0) {
|
|
420
421
|
dropStashIfStillTip(baseCheckout, stashSha);
|
|
421
|
-
|
|
422
|
+
return landed;
|
|
423
|
+
}
|
|
424
|
+
return { ok: false, detail: `${landed.detail}\n${recovery}` };
|
|
422
425
|
}
|
|
423
426
|
if (gitSync(['stash', 'apply', stashSha], baseCheckout).status === 0) {
|
|
424
427
|
dropStashIfStillTip(baseCheckout, stashSha);
|
|
@@ -427,7 +430,7 @@ function landIntoBaseCheckout(baseCheckout, branch, baseSha, landedSha) {
|
|
|
427
430
|
gitSync(['reset', '--hard', landedSha], baseCheckout);
|
|
428
431
|
return {
|
|
429
432
|
ok: true,
|
|
430
|
-
stash: `
|
|
433
|
+
stash: `the shared base checkout was reset to landed commit ${landedSha} because local changes conflicted with it. ${recovery}`,
|
|
431
434
|
};
|
|
432
435
|
}
|
|
433
436
|
/** Drop a stash entry only while it is still the tip `stash@{0}` names, so a
|
|
@@ -8,7 +8,9 @@ import { closeDb } from '../../../core/canvas/db.js';
|
|
|
8
8
|
import { viewSocketPath } from '../../../core/canvas/paths.js';
|
|
9
9
|
import { readInboxSince } from '../../../core/feed/inbox.js';
|
|
10
10
|
import { headlessBrokerHost } from '../../../core/runtime/host.js';
|
|
11
|
+
import { bindTestFleet } from '../../../core/__tests__/helpers/fleet.js';
|
|
11
12
|
import { messageRoutes } from '../handlers/messages.js';
|
|
13
|
+
import { nodeRoutes } from '../handlers/nodes.js';
|
|
12
14
|
let home;
|
|
13
15
|
const priorHome = process.env['CRTR_HOME'];
|
|
14
16
|
before(() => {
|
|
@@ -20,6 +22,7 @@ before(() => {
|
|
|
20
22
|
beforeEach(() => {
|
|
21
23
|
closeDb();
|
|
22
24
|
rmSync(home, { recursive: true, force: true });
|
|
25
|
+
bindTestFleet();
|
|
23
26
|
});
|
|
24
27
|
after(() => {
|
|
25
28
|
closeDb();
|
|
@@ -57,6 +60,19 @@ async function sendMessage(id, body) {
|
|
|
57
60
|
};
|
|
58
61
|
return messageRoutes[0].handler(ctx);
|
|
59
62
|
}
|
|
63
|
+
async function revive(id, body) {
|
|
64
|
+
const ctx = {
|
|
65
|
+
method: 'POST',
|
|
66
|
+
path: `/v1/nodes/${id}/revive`,
|
|
67
|
+
params: { id },
|
|
68
|
+
query: new URLSearchParams(),
|
|
69
|
+
body,
|
|
70
|
+
};
|
|
71
|
+
const route = nodeRoutes.find((candidate) => candidate.pattern === '/v1/nodes/:id/revive');
|
|
72
|
+
if (route === undefined)
|
|
73
|
+
throw new Error('missing node revive route');
|
|
74
|
+
return route.handler(ctx);
|
|
75
|
+
}
|
|
60
76
|
async function startLiveBroker(id, failDelivery = false) {
|
|
61
77
|
let observed = null;
|
|
62
78
|
const server = createServer((socket) => {
|
|
@@ -170,6 +186,21 @@ test('reopen makes live terminal, finalized terminal, and parked targets residen
|
|
|
170
186
|
headlessBrokerHost.launch = originalLaunch;
|
|
171
187
|
}
|
|
172
188
|
});
|
|
189
|
+
test('standalone revive reopen restores the same resident lifecycle before booting', async () => {
|
|
190
|
+
const id = 'standalone-reopen';
|
|
191
|
+
createNode(node(id, 'terminal', 'done'));
|
|
192
|
+
finalize(id);
|
|
193
|
+
const originalLaunch = headlessBrokerHost.launch;
|
|
194
|
+
headlessBrokerHost.launch = (() => ({ pid: 0x7ffffffe, exited: Promise.resolve({ code: 0, signal: null }) }));
|
|
195
|
+
try {
|
|
196
|
+
const result = await revive(id, { reopen: true });
|
|
197
|
+
assert.equal(result.status, 200, 'standalone revive accepts an explicit reopen');
|
|
198
|
+
}
|
|
199
|
+
finally {
|
|
200
|
+
headlessBrokerHost.launch = originalLaunch;
|
|
201
|
+
}
|
|
202
|
+
assertReopened(id);
|
|
203
|
+
});
|
|
173
204
|
test('a reopen commit failure delivers nothing', async () => {
|
|
174
205
|
const id = 'commit-failure';
|
|
175
206
|
createNode(node(id, 'terminal', 'done'));
|