@deksden-com/dd-flow-cli 0.9.0-beta.145 → 0.9.0-beta.149
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +71 -0
- package/dist/build-info.json +7 -7
- package/dist/cli/command-inputs.js +6 -2
- package/dist/cli/command-metadata.js +18 -2
- package/dist/cli/diagnostic-invocation.js +46 -0
- package/dist/cli/diagnostic.js +120 -0
- package/dist/cli/help.js +35 -0
- package/dist/cli/memory-flow-admission.js +4 -2
- package/dist/cli/memory-init.js +5 -2
- package/dist/cli/run-cli.js +16 -15
- package/dist/cli.js +6 -0
- package/dist/domain/diagnostic-aspects.js +103 -0
- package/dist/domain/diagnostic-flows.js +8 -0
- package/dist/harness-runtime/lib/agy-native-inspection.mjs +216 -0
- package/dist/harness-runtime/lib/agy-native_pb.js +117 -0
- package/dist/harness-runtime/lib/daemon-operations.d.mts +1 -0
- package/dist/harness-runtime/lib/daemon-operations.mjs +84 -10
- package/dist/harness-runtime/lib/dd-agy-daemon.mjs +306 -34
- package/dist/harness-runtime/lib/dd-codex-daemon.mjs +27 -13
- package/dist/harness-runtime/lib/dd-codex.mjs +4 -0
- package/dist/harness-runtime/lib/dd-droid-daemon.mjs +4 -3
- package/dist/harness-runtime/lib/dd-droid.mjs +70 -28
- package/dist/harness-runtime/lib/dd-grok-daemon.mjs +3 -3
- package/dist/harness-runtime/lib/dd-opencode-daemon.mjs +66 -20
- package/dist/harness-runtime/lib/dd-opencode.mjs +30 -3
- package/dist/harness-runtime/lib/dd-zcode-daemon.mjs +7 -6
- package/dist/harness-runtime/lib/driver-recovery.mjs +9 -8
- package/dist/harness-runtime/lib/managed-daemon.d.mts +1 -1
- package/dist/harness-runtime/lib/managed-daemon.mjs +11 -3
- package/dist/harness-runtime/lib/native-children.d.mts +8 -1
- package/dist/harness-runtime/lib/native-children.mjs +114 -8
- package/dist/harness-runtime/lib/native-request-proof.mjs +1 -1
- package/dist/harness-runtime/lib/observation-clock.d.mts +2 -2
- package/dist/harness-runtime/lib/observation-clock.mjs +7 -4
- package/dist/harness-runtime/lib/provider-overload-policy.d.mts +1 -1
- package/dist/harness-runtime/lib/provider-overload-policy.mjs +1 -1
- package/dist/harness-runtime/lib/tool-observations.mjs +1 -1
- package/dist/schemas/diagnostic-stage-result.schema.json +354 -0
- package/dist/schemas/diagnostic-work-result.schema.json +296 -0
- package/dist/schemas/execution-bundle.schema.json +7 -1
- package/dist/schemas/project-flow-pack-manifest.schema.json +1 -1
- package/dist/services/check-environment.js +21 -0
- package/dist/services/check-runner-evidence.js +267 -15
- package/dist/services/code-checks.js +23 -15
- package/dist/services/controller-execution.js +10 -4
- package/dist/services/controller-fanout.js +13 -6
- package/dist/services/controller-memory-worker.js +3 -2
- package/dist/services/dashboard.js +2 -2
- package/dist/services/descriptor-run-settlement.js +5 -3
- package/dist/services/descriptor-run-start.js +22 -3
- package/dist/services/descriptor-stage-coordinator.js +37 -4
- package/dist/services/descriptor-stage-settlement.js +2 -1
- package/dist/services/diagnostic-evidence.js +73 -0
- package/dist/services/diagnostic-report-publication.js +72 -0
- package/dist/services/diagnostic-report.js +67 -0
- package/dist/services/diagnostic-run-bundle.js +74 -0
- package/dist/services/diagnostic-snapshot.js +159 -0
- package/dist/services/diagnostic-stage-handlers.js +108 -0
- package/dist/services/diagnostic-stage-results.js +132 -0
- package/dist/services/diagnostic-start.js +84 -0
- package/dist/services/diagnostic-work-batch.js +34 -0
- package/dist/services/diagnostic-work-collection.js +97 -0
- package/dist/services/diagnostic-work-registration.js +101 -0
- package/dist/services/diagnostic-work-results.js +93 -0
- package/dist/services/eval-snapshots.js +85 -29
- package/dist/services/execution-bundles.js +19 -7
- package/dist/services/flow-catalog.js +6 -1
- package/dist/services/frozen-engine.js +37 -2
- package/dist/services/hybrid-contracts.js +2 -0
- package/dist/services/installed-flow-source.js +25 -3
- package/dist/services/lifecycle-invocations.js +147 -34
- package/dist/services/memory-init-contexts.js +16 -0
- package/dist/services/memory-init-controller-start.js +12 -4
- package/dist/services/memory-init-lint-capability.js +5 -5
- package/dist/services/memory-init-native-capacity.js +4 -4
- package/dist/services/memory-init-preflight.js +10 -16
- package/dist/services/memory-init-worktrunk-capability.js +3 -7
- package/dist/services/memory-layer.js +4 -3
- package/dist/services/memory-tool-probe.js +25 -0
- package/dist/services/memory-verification-checks.js +15 -4
- package/dist/services/native-daemon-history.js +2 -1
- package/dist/services/provider-capacity.js +43 -38
- package/dist/services/release-cache.js +8 -0
- package/dist/services/run-control.js +2 -1
- package/dist/services/run-controller-adapter.js +2 -1
- package/dist/services/run-controller-process.js +2 -1
- package/dist/services/run-controller-recovery.js +2 -1
- package/dist/services/run-controller.js +85 -15
- package/dist/services/run-fork.js +1 -1
- package/dist/services/runs.js +6 -1
- package/dist/services/runtime-scope-resume.js +12 -6
- package/dist/services/runtime-scope-stop.js +3 -2
- package/dist/services/runtime-service.js +2 -1
- package/dist/services/stage-handlers.js +16 -3
- package/dist/services/stage-pause.js +4 -1
- package/dist/services/version-status.js +1 -1
- package/dist/services/vnext-code.js +2 -2
- package/dist/services/vnext-fanout.js +31 -1
- package/dist/services/work-registry.js +37 -6
- package/dist/services/workspace-bootstrap.js +2 -1
- package/package.json +6 -3
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import fs from "node:fs";
|
|
2
2
|
import path from "node:path";
|
|
3
|
-
import { spawnSync } from "node:child_process";
|
|
4
3
|
import { createHash } from "node:crypto";
|
|
5
4
|
import { AppError } from "../shared/errors.js";
|
|
6
5
|
import { hybridDigest } from "./hybrid-contracts.js";
|
|
7
6
|
import { readBoundedReleaseFile } from "./release-source.js";
|
|
8
7
|
import { defaultMemoryLintConfiguration, validateMemoryLintConfig } from "./memory-lint-config.js";
|
|
8
|
+
import { runMemoryToolProbe } from "./memory-tool-probe.js";
|
|
9
9
|
function refuse(reason, diagnostics) {
|
|
10
10
|
throw new AppError("memory_init_preflight_failed", "Memory init prerequisites are not admitted", 1, { reason, phase: "prepare", effect: "no_effect", next_action: reason === "existing_memory_bank" ? "audit_or_upgrade" : "repair_prerequisite_then_retry", ...(diagnostics ? { tool_probe: diagnostics } : {}) });
|
|
11
11
|
}
|
|
@@ -56,20 +56,14 @@ function probeTools(tools, env) {
|
|
|
56
56
|
const facts = Object.create(null);
|
|
57
57
|
const probeEnv = Object.fromEntries(Object.entries(env).filter(([key]) => !key.startsWith("GIT_")));
|
|
58
58
|
for (const name of Object.keys(tools)) {
|
|
59
|
-
const diagnostics = { name, executable: tools[name], args: ["--version"],
|
|
59
|
+
const diagnostics = { name, executable: tools[name], args: ["--version"], phase: "executable_read" };
|
|
60
60
|
try {
|
|
61
61
|
const digest = () => createHash("sha256").update(readBoundedReleaseFile(tools[name], 64 * 1024 * 1024)).digest("hex");
|
|
62
62
|
const executableSha256 = digest();
|
|
63
|
-
const
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
Object.assign(diagnostics, { phase: "version_probe"
|
|
67
|
-
error_code: result.error?.code ?? null,
|
|
68
|
-
timed_out: result.error?.code === "ETIMEDOUT",
|
|
69
|
-
stdout_bytes: result.stdout?.length ?? null, stderr_bytes: result.stderr?.length ?? null,
|
|
70
|
-
// Tool output can contain environment secrets; retain bounded custody
|
|
71
|
-
// hashes rather than echoing uncontrolled stderr into public receipts.
|
|
72
|
-
stderr_sha256: result.stderr ? createHash("sha256").update(result.stderr).digest("hex") : null });
|
|
63
|
+
const { result, diagnostics: observed } = runMemoryToolProbe(tools[name], ["--version"], {
|
|
64
|
+
cwd: path.dirname(tools[name]), env: probeEnv, maxBuffer: 64 * 1024
|
|
65
|
+
});
|
|
66
|
+
Object.assign(diagnostics, observed, { name, phase: "version_probe" });
|
|
73
67
|
if (result.error || result.signal || result.status !== 0 || !result.stdout || !result.stderr)
|
|
74
68
|
refuse(`tool_probe_failed:${name}`, diagnostics);
|
|
75
69
|
const output = new TextDecoder("utf-8", { fatal: true }).decode(result.stdout);
|
|
@@ -155,12 +149,12 @@ function inspect(input, env, ownedPublication = false, qualifyTools = true, obse
|
|
|
155
149
|
const toolProbes = qualifyTools ? probeTools(tools, env) : undefined;
|
|
156
150
|
const gitEnv = Object.fromEntries(Object.entries(env).filter(([key]) => !key.startsWith("GIT_")));
|
|
157
151
|
const git = (directory, args, optional = false) => {
|
|
158
|
-
const result =
|
|
159
|
-
env: { ...gitEnv, GIT_OPTIONAL_LOCKS: "0" },
|
|
152
|
+
const { result, diagnostics } = runMemoryToolProbe(tools.git, ["-c", "core.fsmonitor=false", "-c", "core.untrackedCache=false", "-C", directory, ...args], {
|
|
153
|
+
cwd: directory, env: { ...gitEnv, GIT_OPTIONAL_LOCKS: "0" }, maxBuffer: 1024 * 1024,
|
|
160
154
|
});
|
|
161
155
|
if (result.error || result.signal || result.status !== 0 && !optional)
|
|
162
|
-
refuse("git_observation_failed");
|
|
163
|
-
return result.status === 0 ? result.stdout : null;
|
|
156
|
+
refuse("git_observation_failed", { ...diagnostics, phase: "git_observation" });
|
|
157
|
+
return result.status === 0 ? result.stdout.toString("utf8") : null;
|
|
164
158
|
};
|
|
165
159
|
for (const directory of new Set([project, workspace])) {
|
|
166
160
|
if (git(directory, ["rev-parse", "--is-bare-repository"])?.trim() !== "false"
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import fs from "node:fs";
|
|
2
2
|
import os from "node:os";
|
|
3
3
|
import path from "node:path";
|
|
4
|
-
import { spawnSync } from "node:child_process";
|
|
5
4
|
import { createHash } from "node:crypto";
|
|
6
5
|
import { AppError } from "../shared/errors.js";
|
|
7
6
|
import { hybridDigest } from "./hybrid-contracts.js";
|
|
8
7
|
import { PreparedMemoryInitTarget } from "./memory-init-preflight.js";
|
|
9
8
|
import { readBoundedReleaseFile } from "./release-source.js";
|
|
9
|
+
import { runMemoryToolProbe } from "./memory-tool-probe.js";
|
|
10
10
|
const hash = (bytes) => createHash("sha256").update(bytes).digest("hex");
|
|
11
11
|
function refuse(reason, diagnostics) {
|
|
12
12
|
throw new AppError("memory_init_worktrunk_capability_failed", "Selected Worktrunk cannot create the required workspace", 1, { reason, effect: "no_effect", next_action: "repair_prerequisite_then_retry", ...(diagnostics ? { tool_probe: diagnostics } : {}) });
|
|
@@ -83,14 +83,10 @@ export class PreparedMemoryInitWorktrunkCapability {
|
|
|
83
83
|
const run = (file, args) => {
|
|
84
84
|
ownRoot();
|
|
85
85
|
unchanged();
|
|
86
|
-
const
|
|
87
|
-
const result = spawnSync(file, args, { cwd: repo, env: probeEnv, shell: false, windowsHide: true, timeout: 5000, maxBuffer: 256 * 1024 });
|
|
86
|
+
const { result, diagnostics } = runMemoryToolProbe(file, args, { cwd: repo, env: probeEnv });
|
|
88
87
|
if (result.error || result.signal || result.status !== 0)
|
|
89
88
|
refuse("worktrunk_probe_execution_failed", {
|
|
90
|
-
name: path.basename(file), phase: "workspace_probe",
|
|
91
|
-
status: result.status, signal: result.signal, error_code: result.error?.code ?? null,
|
|
92
|
-
timed_out: result.error?.code === "ETIMEDOUT",
|
|
93
|
-
stdout_bytes: result.stdout?.length ?? null, stderr_bytes: result.stderr?.length ?? null
|
|
89
|
+
...diagnostics, name: path.basename(file), phase: "workspace_probe", expected_status: 0
|
|
94
90
|
});
|
|
95
91
|
ownRoot();
|
|
96
92
|
unchanged();
|
|
@@ -11,12 +11,13 @@ import { normalizeFlowContract } from "../domain/flow-contract.js";
|
|
|
11
11
|
import { inspectReleaseArchive } from "./release-archive.js";
|
|
12
12
|
import { canonicalOnlyFlowNames } from "./version-status.js";
|
|
13
13
|
export function isCanonicalOnlyMemoryPath(logical) {
|
|
14
|
-
return
|
|
14
|
+
return logical.startsWith("dd-flow/project-review/")
|
|
15
|
+
|| canonicalOnlyFlowNames.some(name => logical === `dd-flow/${name}` || logical.startsWith(`dd-flow/${name.slice(0, -3)}/`));
|
|
15
16
|
}
|
|
16
17
|
const rootFiles = ["README.md", "index.md", "compatibility.json", "flow-contract.json", "go.md", "f.md", "prime.md", "protocol.md",
|
|
17
18
|
"protocol-implement.md", "plan.md", "code.md", "merge.md", "merge-start.md", "merge-stop.md", "release.md", "deploy.md", "publish.md",
|
|
18
|
-
"review
|
|
19
|
-
const supportDirectories = ["common", "prime", "mb-sdlc", "aspects", "def", "workers", "
|
|
19
|
+
"review-fix.md", "mb-fix.md", "mb-lint.md", "interactive.md", "finish.md", "fix-def.md"];
|
|
20
|
+
const supportDirectories = ["common", "prime", "mb-sdlc", "aspects", "def", "workers", "schemas", "vnext", "specify", "stage-reports", "dashboard", "plan", "code", "merge"];
|
|
20
21
|
const required = ["README.md", "index.md", "flow-contract.json", "compatibility.json", "prime.md", "go.md",
|
|
21
22
|
"common/style.md", "common/runtime-contract.md", "common/lifecycle-guards.md", "common/workspace-bootstrap.md", "common/worker-session.md",
|
|
22
23
|
"vnext/start.md", "vnext/specify.md", "vnext/protocolize.md", "vnext/plan.md", "vnext/plan-review.md", "vnext/code.md", "vnext/code-review.md", "vnext/merge.md",
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { spawnSync } from "node:child_process";
|
|
2
|
+
import { createHash } from "node:crypto";
|
|
3
|
+
import { SETUP_STARTUP_MS, setupReporter } from "../harness-runtime/lib/setup-progress.mjs";
|
|
4
|
+
/** Silent, local prerequisite tools are not agent Turns. Reuse the CLI setup
|
|
5
|
+
* startup budget; do not invent heartbeat progress or retry a failed probe. */
|
|
6
|
+
export function runMemoryToolProbe(executable, args, options) {
|
|
7
|
+
const started = performance.now();
|
|
8
|
+
const report = setupReporter(options.env.DD_EVAL_OPERATION_ID);
|
|
9
|
+
const step = createHash("sha256").update(JSON.stringify([executable, args])).digest("hex");
|
|
10
|
+
report("probe_started", step);
|
|
11
|
+
const result = spawnSync(executable, args, { ...options, shell: false, windowsHide: true,
|
|
12
|
+
timeout: SETUP_STARTUP_MS, maxBuffer: options.maxBuffer ?? 256 * 1024 });
|
|
13
|
+
report("probe_completed", step);
|
|
14
|
+
const error = result.error?.code ?? null;
|
|
15
|
+
const output = (bytes) => ({ bytes: bytes?.length ?? null,
|
|
16
|
+
sha256: bytes ? createHash("sha256").update(bytes).digest("hex") : null });
|
|
17
|
+
const stdout = output(result.stdout), stderr = output(result.stderr);
|
|
18
|
+
// Arbitrary tool output may contain credentials. Public errors retain typed
|
|
19
|
+
// outcomes and bounded custody facts, never the uncontrolled output itself.
|
|
20
|
+
return { result, diagnostics: { executable, args, timeout_ms: SETUP_STARTUP_MS,
|
|
21
|
+
elapsed_ms: Math.round(performance.now() - started), status: result.status,
|
|
22
|
+
signal: result.signal, error_code: error, timed_out: error === "ETIMEDOUT",
|
|
23
|
+
stdout_bytes: stdout.bytes, stdout_sha256: stdout.sha256,
|
|
24
|
+
stderr_bytes: stderr.bytes, stderr_sha256: stderr.sha256 } };
|
|
25
|
+
}
|
|
@@ -20,6 +20,7 @@ import { readMemoryUpgradeBootstrapReadiness } from "./memory-upgrade-bootstrap-
|
|
|
20
20
|
import { readUpgradeStorageReadiness } from "./memory-upgrade-storage-readiness.js";
|
|
21
21
|
import { PreparedAcceptedMemoryComposition } from "./memory-accepted-composition.js";
|
|
22
22
|
import { RetainedExecutionBundle } from "./execution-bundles.js";
|
|
23
|
+
import { retainedBrowserEnvironment, validateCheckEnvironment } from "./check-environment.js";
|
|
23
24
|
const token = Symbol("memory-verification-checks");
|
|
24
25
|
const hash = (bytes) => createHash("sha256").update(bytes).digest("hex");
|
|
25
26
|
const quote = (value) => `'${value.replaceAll("'", "'\\''")}'`;
|
|
@@ -40,9 +41,19 @@ export function assertRetainedMemoryVerificationPolicy(previous, current) {
|
|
|
40
41
|
/** Finite project checks use the original Codex home, never a temporary
|
|
41
42
|
* provider Session overlay. Keep every other application environment input. */
|
|
42
43
|
export function memoryCheckEnvironment(context, projectId, runId) {
|
|
44
|
+
let browser = retainedBrowserEnvironment(context.env);
|
|
43
45
|
if (!context.db.get("SELECT 1 FROM sqlite_schema WHERE type='table' AND name='run_controllers'"))
|
|
44
|
-
return
|
|
45
|
-
const controllers = context.db.all("SELECT state_json FROM run_controllers WHERE project_id=? AND run_id=?", [projectId, runId]);
|
|
46
|
+
return browser;
|
|
47
|
+
const controllers = context.db.all("SELECT state_json,manifest_json FROM run_controllers WHERE project_id=? AND run_id=?", [projectId, runId]);
|
|
48
|
+
const retained = new Set(controllers.flatMap(controller => {
|
|
49
|
+
const manifest = JSON.parse(controller.manifest_json);
|
|
50
|
+
const value = validateCheckEnvironment(manifest.check_environment)?.PLAYWRIGHT_BROWSERS_PATH;
|
|
51
|
+
return value === undefined ? [] : [value];
|
|
52
|
+
}));
|
|
53
|
+
if (retained.size > 1)
|
|
54
|
+
refuse("ambiguous_check_browser_resource");
|
|
55
|
+
if (retained.size)
|
|
56
|
+
browser = { PLAYWRIGHT_BROWSERS_PATH: [...retained][0] };
|
|
46
57
|
const targets = new Set();
|
|
47
58
|
for (const controller of controllers) {
|
|
48
59
|
const state = JSON.parse(controller.state_json);
|
|
@@ -51,7 +62,7 @@ export function memoryCheckEnvironment(context, projectId, runId) {
|
|
|
51
62
|
targets.add(session.codex_home);
|
|
52
63
|
}
|
|
53
64
|
if (!targets.size)
|
|
54
|
-
return
|
|
65
|
+
return browser;
|
|
55
66
|
const current = process.env.CODEX_HOME;
|
|
56
67
|
if (current && !targets.has(current) && context.db.get("SELECT 1 FROM sqlite_schema WHERE type='table' AND name='work_launches'")) {
|
|
57
68
|
const launches = context.db.all("SELECT l.state_dir FROM work_launches l JOIN works w ON w.work_id=l.work_id WHERE w.project_id=? AND w.run_id=?", [projectId, runId]);
|
|
@@ -74,7 +85,7 @@ export function memoryCheckEnvironment(context, projectId, runId) {
|
|
|
74
85
|
const base = [...sources][0];
|
|
75
86
|
if (current === undefined ? base !== path.join(os.homedir(), ".codex") : current !== base && !targets.has(current))
|
|
76
87
|
refuse("foreign_check_home");
|
|
77
|
-
return { CODEX_HOME: base };
|
|
88
|
+
return { ...browser, CODEX_HOME: base };
|
|
78
89
|
}
|
|
79
90
|
function lintConfig(workspace) {
|
|
80
91
|
const file = path.join(workspace, ".mb-lint.json");
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import crypto from "node:crypto";
|
|
2
|
+
import { nativeShutdownConfirmed } from "../harness-runtime/lib/native-children.mjs";
|
|
2
3
|
import fs from "node:fs";
|
|
3
4
|
import path from "node:path";
|
|
4
5
|
import { AppError } from "../shared/errors.js";
|
|
@@ -29,7 +30,7 @@ export function historicalDaemonSettlement(context, scopeId, record) {
|
|
|
29
30
|
if (!state.daemon_id || path.basename(file) !== `${state.daemon_id}.json` || !owner || owner.budget?.scope_id !== scopeId || owner.owner_id !== record.owner_id || owner.role !== metadata.role || owner.state_dir !== metadata.state_dir || owner.adapter_executable !== metadata.adapter_executable || owner.dd_flow_home !== metadata.dd_flow_home || !retainedResourceHomeMatches(context, owner.resource_home) || (owner.project_id ?? null) !== record.project_id || (owner.run_id ?? null) !== record.run_id || (owner.work_id ?? null) !== record.work_id || !record.operation_id?.startsWith(`${owner.operation_id}:`) || state.resource_process?.lease_token !== record.lease_token || state.pid !== record.pid)
|
|
30
31
|
throw new AppError("runtime_scope_owner_unknown", "Archived daemon does not match its registration", 1);
|
|
31
32
|
const closed = state.shutdown_state === "clean" && state.active_tree === false
|
|
32
|
-
? retainedDaemonOperations(metadata.state_dir).find(operation => operation.daemon_id === state.daemon_id && operation.operation === "daemon.stop" && operation.state === "completed" && operation.result?.stopped === true && operation.result
|
|
33
|
+
? retainedDaemonOperations(metadata.state_dir).find(operation => operation.daemon_id === state.daemon_id && operation.operation === "daemon.stop" && operation.state === "completed" && operation.result?.stopped === true && nativeShutdownConfirmed(operation.result, { daemonId: state.daemon_id ?? "" })) : undefined;
|
|
33
34
|
const fresh = managedProcessStatus(context).find(item => item.id === record.id);
|
|
34
35
|
if (!closed || !fresh || fresh.lease_token !== record.lease_token || fresh.pid !== record.pid || fresh.pid_started_at !== record.pid_started_at || fresh.metadata_json !== record.metadata_json || fresh.state !== "stopped" || fresh.termination_reason !== "daemon_stopped" || processTreeIsAlive(fresh) || fs.readFileSync(file, "utf8") !== bytes)
|
|
35
36
|
throw new AppError("runtime_scope_outcome_unknown", "Previous daemon lacks stable clean shutdown evidence", 1);
|
|
@@ -4,6 +4,8 @@ import { CAPACITY_POLICY, MAX_OVERLOAD_RETRIES, capacityBackoff, nativeItemsSett
|
|
|
4
4
|
import { AppError } from "../shared/errors.js";
|
|
5
5
|
import { isObservationLoss } from "../harness-runtime/lib/operation-errors.mjs";
|
|
6
6
|
import { parseLifecycleCommand, commandOption } from "./lifecycle-command.js";
|
|
7
|
+
import { nativeTreeReadiness } from "../harness-runtime/lib/native-children.mjs";
|
|
8
|
+
import { ObservationClock } from "../harness-runtime/lib/observation-clock.mjs";
|
|
7
9
|
/** Native output is evidence, not authority: match the committed current scope. */
|
|
8
10
|
export function ownedProviderBoundary(context, input, boundary) {
|
|
9
11
|
if (typeof boundary.native_hook_event_key !== "string" || typeof boundary.daemon_instance !== "string"
|
|
@@ -54,26 +56,9 @@ export function eligibleProviderChildOverload(child, parentSessionId) {
|
|
|
54
56
|
}
|
|
55
57
|
/** A terminal root is not a settled tree. Only the daemon's exact owned tree
|
|
56
58
|
* with a settled root and positively live descendants permits waiting. */
|
|
57
|
-
export function capacityReadiness(seen, sessionId, operationId, turnId, items, allowDescendants = false) {
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
const ids = Array.isArray(seen.tree_session_ids) ? seen.tree_session_ids : [];
|
|
61
|
-
if (!ids.length || ids[0] !== sessionId || ids.some(id => typeof id !== "string" || !id) || new Set(ids).size !== ids.length)
|
|
62
|
-
return "unproven";
|
|
63
|
-
const observations = Array.isArray(seen.tree_observations) ? seen.tree_observations : [];
|
|
64
|
-
const exactTree = observations.length === ids.length && ids.every(id => observations.filter(item => item.provider_session_id === id).length === 1);
|
|
65
|
-
const settlement = seen.settlement;
|
|
66
|
-
const budgetSettled = !settlement || settlement.state === "not_required" || settlement.state === "settled" && settlement.operation_ids?.includes(operationId) === true;
|
|
67
|
-
const treeSettled = allowDescendants ? exactTree && observations.every(item => item.settled === true) && Array.isArray(seen.unsettled_sessions) && seen.unsettled_sessions.length === 0 : ids.length === 1;
|
|
68
|
-
if (seen.settled === true && treeSettled && budgetSettled)
|
|
69
|
-
return "ready";
|
|
70
|
-
if (!allowDescendants || seen.settled !== false || !exactTree || ids.length < 2 || observations[0]?.provider_session_id !== sessionId || observations[0].settled !== true
|
|
71
|
-
|| settlement && settlement.state !== "not_required" && !(settlement.state === "pending" && settlement.reason === "tree_unsettled"))
|
|
72
|
-
return "unproven";
|
|
73
|
-
const live = observations.filter(item => item.settled === false);
|
|
74
|
-
const unsettled = seen.unsettled_sessions;
|
|
75
|
-
const knownLive = live.length > 0 && live.every(item => item.provider_session_id !== sessionId && ["active", "running", "inProgress"].includes(String(typeof item.status === "object" && item.status ? item.status.type : item.status)));
|
|
76
|
-
return knownLive && observations.every(item => typeof item.settled === "boolean") && Array.isArray(unsettled) && unsettled.length === live.length && live.every(item => unsettled.includes(item.provider_session_id)) ? "owned_children_live" : "unproven";
|
|
59
|
+
export function capacityReadiness(seen, sessionId, operationId, turnId, items, allowDescendants = false, requestProof) {
|
|
60
|
+
const readiness = nativeTreeReadiness(seen, { sessionId, operationId, turnId, allowDescendants, requestProof });
|
|
61
|
+
return readiness === "inspection_pending" ? readiness : nativeItemsSettled(items) ? readiness : "unproven";
|
|
77
62
|
}
|
|
78
63
|
/** Permit acquisition must still belong to the original task, not merely an
|
|
79
64
|
* open RUN. Controls/inspection and ordinary successor prompts are unaffected. */
|
|
@@ -318,6 +303,7 @@ export async function continueProviderCapacity(input) {
|
|
|
318
303
|
|| confirmed?.request_proof.kind === "stream_request" && confirmed.request_proof.operation_id !== operationId) {
|
|
319
304
|
throw blocked("Retained predecessor does not prove the exact terminal overloaded Turn");
|
|
320
305
|
}
|
|
306
|
+
let pendingClock;
|
|
321
307
|
const readiness = async () => {
|
|
322
308
|
input.signal?.throwIfAborted();
|
|
323
309
|
input.assertCurrent();
|
|
@@ -327,16 +313,48 @@ export async function continueProviderCapacity(input) {
|
|
|
327
313
|
// An unloaded exact-Turn snapshot is not conflicting tool evidence.
|
|
328
314
|
// Preserve the bound failure receipt, but never hide a pending item.
|
|
329
315
|
const freshItems = seen.native_turn_items;
|
|
330
|
-
const
|
|
316
|
+
const stream = confirmed?.request_proof.kind === "stream_request";
|
|
317
|
+
const streamMetadata = seen.terminal_metadata;
|
|
318
|
+
const exactFresh = stream ? streamRequestKey(streamMetadata?.request_proof) === turnId : seen.terminal_turn_id === turnId && seen.terminal_status === "failed";
|
|
319
|
+
const items = (exactFresh
|
|
331
320
|
&& !(freshItems?.observed === false && freshItems.pending !== true) ? freshItems : confirmed?.native_turn_items);
|
|
332
321
|
if (confirmed?.request_proof.kind === "stream_request") {
|
|
333
322
|
const metadata = seen.terminal_metadata;
|
|
334
323
|
if (streamRequestKey(metadata?.request_proof) !== turnId)
|
|
335
324
|
return "unproven";
|
|
336
325
|
}
|
|
337
|
-
|
|
326
|
+
const current = capacityReadiness(seen, input.sessionId, operationId, turnId, items, input.allowSettledDescendants, confirmed?.request_proof);
|
|
327
|
+
if (current === "inspection_pending") {
|
|
328
|
+
const saved = seen.native_settlement.observation_clock;
|
|
329
|
+
pendingClock ??= new ObservationClock({ timeoutMs: saved.timeout_ms, saved, snapshotIsLive: true });
|
|
330
|
+
if (pendingClock.sample(saved.cursor, saved.observed_at ?? undefined))
|
|
331
|
+
return "unproven";
|
|
332
|
+
}
|
|
333
|
+
return current;
|
|
334
|
+
};
|
|
335
|
+
// Pending is a bound observer, not a refusal or an intent to write. Resolve
|
|
336
|
+
// it before retaining a productive retry. Its producer owns the old clock.
|
|
337
|
+
const waitForReadiness = async (initial) => {
|
|
338
|
+
let current = initial;
|
|
339
|
+
while (current === "owned_children_live" || current === "inspection_pending") {
|
|
340
|
+
await delay(250, undefined, { signal: input.signal });
|
|
341
|
+
const boundary = await decide();
|
|
342
|
+
if (boundary)
|
|
343
|
+
return boundary;
|
|
344
|
+
current = await readiness();
|
|
345
|
+
}
|
|
346
|
+
if (current !== "ready")
|
|
347
|
+
throw blocked("Native predecessor readiness became unproven");
|
|
348
|
+
return null;
|
|
338
349
|
};
|
|
339
|
-
|
|
350
|
+
let ready = await readiness();
|
|
351
|
+
while (ready === "inspection_pending") {
|
|
352
|
+
await delay(250, undefined, { signal: input.signal });
|
|
353
|
+
const boundary = await decide();
|
|
354
|
+
if (boundary)
|
|
355
|
+
return boundary;
|
|
356
|
+
ready = await readiness();
|
|
357
|
+
}
|
|
340
358
|
if (ready === "unproven")
|
|
341
359
|
throw blocked("Native predecessor readiness is unproven; do not cancel or replay to manufacture proof");
|
|
342
360
|
if (ready === "ready") {
|
|
@@ -399,20 +417,7 @@ export async function continueProviderCapacity(input) {
|
|
|
399
417
|
}
|
|
400
418
|
// Intent/refusal are durable before the first wait; no permit or writer is
|
|
401
419
|
// held here, and live siblings are never cancelled as a settlement proof.
|
|
402
|
-
const
|
|
403
|
-
let current = initial;
|
|
404
|
-
while (current === "owned_children_live") {
|
|
405
|
-
await delay(250, undefined, { signal: input.signal });
|
|
406
|
-
const boundary = await decide();
|
|
407
|
-
if (boundary)
|
|
408
|
-
return boundary;
|
|
409
|
-
current = await readiness();
|
|
410
|
-
}
|
|
411
|
-
if (current !== "ready")
|
|
412
|
-
throw blocked("Owned native child readiness became unproven");
|
|
413
|
-
return null;
|
|
414
|
-
};
|
|
415
|
-
const childBoundary = await waitForOwnedChildren(ready);
|
|
420
|
+
const childBoundary = await waitForReadiness(ready);
|
|
416
421
|
if (childBoundary)
|
|
417
422
|
return childBoundary;
|
|
418
423
|
// Node timers truncate large delays. Bounded slices preserve cancellation
|
|
@@ -427,7 +432,7 @@ export async function continueProviderCapacity(input) {
|
|
|
427
432
|
const afterWait = await decide();
|
|
428
433
|
if (afterWait)
|
|
429
434
|
return afterWait;
|
|
430
|
-
const refreshedBoundary = await
|
|
435
|
+
const refreshedBoundary = await waitForReadiness(await readiness());
|
|
431
436
|
if (refreshedBoundary)
|
|
432
437
|
return refreshedBoundary;
|
|
433
438
|
const beforeDispatch = await decide();
|
|
@@ -6,7 +6,9 @@ import { inspectReleaseArchive } from "./release-archive.js";
|
|
|
6
6
|
import { verifyReleaseArchive } from "./release-manifest.js";
|
|
7
7
|
import { acquireInstallationLock } from "./installation-lock.js";
|
|
8
8
|
import { hybridDigest } from "./hybrid-contracts.js";
|
|
9
|
+
import { publishSetup } from "../harness-runtime/lib/setup-progress.mjs";
|
|
9
10
|
const digest = (bytes) => createHash("sha256").update(bytes).digest("hex");
|
|
11
|
+
let cacheVerificationPass = 0;
|
|
10
12
|
function refuse(reason) {
|
|
11
13
|
throw new AppError("cache_integrity", "Release cache cannot be published", 1, { reason, phase: "cache_publication", effect: "no_effect", help: "setup" });
|
|
12
14
|
}
|
|
@@ -33,6 +35,8 @@ function write(target, bytes, mode) {
|
|
|
33
35
|
}
|
|
34
36
|
}
|
|
35
37
|
function retained(target, archive, entries) {
|
|
38
|
+
const observation = `release-cache:${++cacheVerificationPass}:${target}`;
|
|
39
|
+
publishSetup("probe_started", observation);
|
|
36
40
|
if (!fs.lstatSync(target).isDirectory())
|
|
37
41
|
refuse("unsafe_retained_cache");
|
|
38
42
|
const archiveFile = path.join(target, "archive.tgz");
|
|
@@ -53,6 +57,7 @@ function retained(target, archive, entries) {
|
|
|
53
57
|
directories.add(parts.slice(0, index).join("/"));
|
|
54
58
|
}
|
|
55
59
|
const seen = new Set();
|
|
60
|
+
let checked = 0;
|
|
56
61
|
const visit = (root, relative) => {
|
|
57
62
|
if (!fs.lstatSync(root).isDirectory())
|
|
58
63
|
refuse("unsafe_retained_cache");
|
|
@@ -70,12 +75,15 @@ function retained(target, archive, entries) {
|
|
|
70
75
|
if (!stat.isFile() || entry?.kind !== "file" || stat.size !== entry.bytes.length || digest(fs.readFileSync(full)) !== digest(entry.bytes) || (stat.mode & 0o111) !== (entry.mode & 0o111))
|
|
71
76
|
refuse("retained_cache_changed");
|
|
72
77
|
seen.add(logical);
|
|
78
|
+
if (++checked % 64 === 0)
|
|
79
|
+
publishSetup("identity_digest_progress", `${observation}:${checked}`);
|
|
73
80
|
}
|
|
74
81
|
}
|
|
75
82
|
};
|
|
76
83
|
visit(payload, "");
|
|
77
84
|
if (seen.size !== entries.length)
|
|
78
85
|
refuse("retained_cache_changed");
|
|
86
|
+
publishSetup("identity_digest_completed", observation);
|
|
79
87
|
}
|
|
80
88
|
/** Verify retained compressed and expanded bytes without executing or activating them. */
|
|
81
89
|
export function verifyPublishedReleaseCache(target, archive) {
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { setFlowRunRecoveryPaused } from "./runs.js";
|
|
2
2
|
import crypto from "node:crypto";
|
|
3
|
+
import { nativeShutdownConfirmed } from "../harness-runtime/lib/native-children.mjs";
|
|
3
4
|
import fs from "node:fs";
|
|
4
5
|
import { ensureRunControlStorage } from "../storage/database.js";
|
|
5
6
|
import { canonicalPath, resolveProjectRoot } from "../storage/paths.js";
|
|
@@ -414,7 +415,7 @@ async function reconcileOwnedSessions(context, input, interrupt, boundary) {
|
|
|
414
415
|
const unfinished = receipts.filter(receipt => !["completed", "failed", "interrupted"].includes(String(receipt.state)));
|
|
415
416
|
if (unfinished.some(receipt => receipt.daemon_id !== daemonId || String(receipt.operation) !== "session.create" || receipt.session_id !== null))
|
|
416
417
|
throw new AppError("shutdown_operation_unsettled", "Closed daemon retains an unfinished non-create operation", 1);
|
|
417
|
-
const stopped = receipts.find(receipt => receipt.daemon_id === daemonId && receipt.operation === "daemon.stop" && receipt.state === "completed" && receipt.result?.stopped === true && receipt.result
|
|
418
|
+
const stopped = receipts.find(receipt => receipt.daemon_id === daemonId && receipt.operation === "daemon.stop" && receipt.state === "completed" && receipt.result?.stopped === true && nativeShutdownConfirmed(receipt.result, { daemonId }));
|
|
418
419
|
if (!stopped)
|
|
419
420
|
throw new AppError("shutdown_receipt_required", "Closed daemon has no clean native shutdown receipt for its current identity", 1);
|
|
420
421
|
closedDaemons.set(directory, { source: "retained_clean_shutdown", daemon_id: daemonId, operation_id: stopped.operation_id, session_ids: state.sessions.map(item => typeof item === "string" ? item : adapterSessionId(item)), stopped: true, settled: true, processes: records.map(record => ({ process_id: record.id, pid: record.pid, pid_started_at: record.pid_started_at })), result: stopped.result });
|
|
@@ -12,6 +12,7 @@ import { canonicalNativeOperation } from "../harness-runtime/lib/adapter-timeout
|
|
|
12
12
|
import { retainedDaemonReply } from "../harness-runtime/lib/daemon-observation.mjs";
|
|
13
13
|
import { readOperationRecord } from "../harness-runtime/lib/operation-progress.mjs";
|
|
14
14
|
import { readNativeDaemonState } from "./managed-daemon-binding.js";
|
|
15
|
+
import { validateCheckEnvironment } from "./check-environment.js";
|
|
15
16
|
/** A paused RUN admits only its controller's exact accepted answer Turn. */
|
|
16
17
|
export function controllerHitlAdmission(context, record, input) {
|
|
17
18
|
if (input.operation !== "session.prompt" || !input.operationId || !input.promptSha256)
|
|
@@ -76,7 +77,7 @@ export function sessionArguments(context, manifest, session) {
|
|
|
76
77
|
return ["--dd-flow-bin", manifest.flow_executable, "--dd-flow-home", context.ddFlowHome, "--project-root", manifest.request.project_root, "--state-dir", session.state_dir, "--journal", session.journal, "--cwd", session.cwd, "--provider", profile.provider, "--model", profile.model, "--reasoning", profile.reasoning, "--mode", profile.mode, "--permission", profile.permission, ...manifest.adapters[runtimeHarness(profile)].runtime_args, "--json"];
|
|
77
78
|
}
|
|
78
79
|
export function controllerEnvironment(context, manifest, session) {
|
|
79
|
-
const env = { ...context.env, DD_FLOW_HOME: context.ddFlowHome, DD_FLOW_BIN: manifest.flow_executable, DD_FLOW_RESOURCE_HOME: manifest.resource_home, DD_FLOW_RUNTIME_BUDGET: manifest.budget ? JSON.stringify(manifest.budget) : undefined };
|
|
80
|
+
const env = { ...context.env, ...validateCheckEnvironment(manifest.check_environment), DD_FLOW_HOME: context.ddFlowHome, DD_FLOW_BIN: manifest.flow_executable, DD_FLOW_RESOURCE_HOME: manifest.resource_home, DD_FLOW_RUNTIME_BUDGET: manifest.budget ? JSON.stringify(manifest.budget) : undefined };
|
|
80
81
|
for (const key of ["DD_FLOW_CONTROLLER_TOKEN", "DD_FLOW_CONTROL_TOKEN", "DD_FLOW_DAEMON_ID", "DD_FLOW_RUNTIME_OWNER", "DD_FLOW_WORK_ID", "DD_FLOW_SESSION_ID", "CODEX_THREAD_ID"])
|
|
81
82
|
delete env[key];
|
|
82
83
|
if (session?.codex_home)
|
|
@@ -3,6 +3,7 @@ import crypto from "node:crypto";
|
|
|
3
3
|
import path from "node:path";
|
|
4
4
|
import { spawn } from "node:child_process";
|
|
5
5
|
import { AppError } from "../shared/errors.js";
|
|
6
|
+
import { validateCheckEnvironment } from "./check-environment.js";
|
|
6
7
|
import { getResourceDatabase } from "../storage/database.js";
|
|
7
8
|
import { finishManagedProcess, managedProcessRuntimeHome, managedProcessStatus, processIsAlive, processStartedAt, processTreeIsAlive, registerManagedProcess } from "./managed-processes.js";
|
|
8
9
|
import { setFlowRunRecoveryPaused } from "./runs.js";
|
|
@@ -175,7 +176,7 @@ export async function spawnController(context, row) {
|
|
|
175
176
|
const entry = (await import("./engines.js")).boundRunEngineEntrypoint(context, { projectRoot: run.project_root, runId: run.id });
|
|
176
177
|
const output = fs.openSync(path.join(root, "controller.log"), "a", 0o600);
|
|
177
178
|
try {
|
|
178
|
-
const child = spawn(process.execPath, [entry, "run", "drive", "serve", "--run", run.id, "--project-root", run.project_root, "--controller-id", controllerId, "--json"], { cwd: run.workspace_root, detached: true, stdio: ["ignore", output, output], env: { ...context.env, DD_FLOW_HOME: context.ddFlowHome, DD_FLOW_RESOURCE_HOME: manifest.resource_home, DD_FLOW_ENGINE_MODE: "1", DD_FLOW_CONTROLLER_TOKEN: token } });
|
|
179
|
+
const child = spawn(process.execPath, [entry, "run", "drive", "serve", "--run", run.id, "--project-root", run.project_root, "--controller-id", controllerId, "--json"], { cwd: run.workspace_root, detached: true, stdio: ["ignore", output, output], env: { ...context.env, ...validateCheckEnvironment(manifest.check_environment), DD_FLOW_HOME: context.ddFlowHome, DD_FLOW_RESOURCE_HOME: manifest.resource_home, DD_FLOW_ENGINE_MODE: "1", DD_FLOW_CONTROLLER_TOKEN: token } });
|
|
179
180
|
await new Promise((resolve, reject) => { child.once("spawn", resolve); child.once("error", reject); });
|
|
180
181
|
context.db.run("UPDATE run_controller_operations SET receipt_json = json_set(receipt_json, '$.child', json(?)), updated_at = ? WHERE operation_id = ?", [JSON.stringify({ pid: child.pid, pid_started_at: child.pid ? processStartedAt(child.pid) : null }), context.now(), launchId(row)]);
|
|
181
182
|
child.unref();
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import crypto from "node:crypto";
|
|
2
|
+
import { nativeShutdownConfirmed } from "../harness-runtime/lib/native-children.mjs";
|
|
2
3
|
import fs from "node:fs";
|
|
3
4
|
import path from "node:path";
|
|
4
5
|
import { setTimeout as delay } from "node:timers/promises";
|
|
@@ -148,7 +149,7 @@ export async function resumeControllerSession(context, row, manifest, state, ass
|
|
|
148
149
|
const call = (args, operationId) => controllerAdapter(executionContext, row, manifest, session, args, assertRecoveryOwner, operationId);
|
|
149
150
|
if (processTreeIsAlive(record)) {
|
|
150
151
|
const stopped = await call(["daemon", "stop", "--state-dir", session.state_dir, "--json"]);
|
|
151
|
-
if (stopped.stopped !== true || stopped.
|
|
152
|
+
if (!retired.daemon_id || stopped.stopped !== true || !nativeShutdownConfirmed(stopped, { daemonId: retired.daemon_id }))
|
|
152
153
|
throw new AppError("recovery_native_shutdown_unproven", "Recovery requires a clean native close before replacing its daemon", 1);
|
|
153
154
|
const deadline = Date.now() + 30_000;
|
|
154
155
|
while (processTreeIsAlive(record) && Date.now() < deadline) {
|