@deksden-com/dd-flow-cli 0.9.0-beta.144 → 0.9.0-beta.149
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +130 -0
- package/README.md +13 -8
- package/dist/build-info.json +7 -7
- package/dist/cli/command-inputs.js +13 -3
- package/dist/cli/command-metadata.js +51 -16
- package/dist/cli/diagnostic-invocation.js +46 -0
- package/dist/cli/diagnostic.js +120 -0
- package/dist/cli/help.js +67 -1
- package/dist/cli/hook-ingress.js +27 -3
- package/dist/cli/memory-flow-admission.js +9 -3
- package/dist/cli/memory-init-invocation.js +10 -3
- package/dist/cli/memory-init.js +77 -68
- package/dist/cli/memory-storage-admission.js +3 -2
- package/dist/cli/memory-storage-qualification.js +105 -0
- package/dist/cli/memory-storage-transition.js +123 -0
- package/dist/cli/memory-upgrade.js +189 -0
- package/dist/cli/native-hook-ingress.js +14 -4
- package/dist/cli/native-hook-preparation.js +55 -0
- package/dist/cli/response-file.js +2 -1
- package/dist/cli/run-cli.js +79 -40
- package/dist/cli/setup.js +2 -1
- package/dist/cli.js +18 -0
- package/dist/domain/diagnostic-aspects.js +103 -0
- package/dist/domain/diagnostic-flows.js +8 -0
- package/dist/domain/stage-handler-identities.js +8 -0
- package/dist/harness-runtime/bin/dd-agy.mjs +3 -3
- package/dist/harness-runtime/bin/dd-grok.mjs +3 -3
- package/dist/harness-runtime/bin/dd-zcode.mjs +5 -5
- package/dist/harness-runtime/lib/agy-error-boundary.mjs +58 -0
- package/dist/harness-runtime/lib/agy-native-inspection.mjs +216 -0
- package/dist/harness-runtime/lib/agy-native_pb.js +117 -0
- package/dist/harness-runtime/lib/daemon-operations.d.mts +1 -0
- package/dist/harness-runtime/lib/daemon-operations.mjs +84 -10
- package/dist/harness-runtime/lib/dd-agy-daemon.mjs +366 -52
- package/dist/harness-runtime/lib/dd-agy.mjs +17 -3
- package/dist/harness-runtime/lib/dd-codex-daemon.mjs +27 -13
- package/dist/harness-runtime/lib/dd-codex.mjs +4 -0
- package/dist/harness-runtime/lib/dd-droid-daemon.mjs +4 -3
- package/dist/harness-runtime/lib/dd-droid.mjs +71 -29
- package/dist/harness-runtime/lib/dd-grok-daemon.mjs +73 -16
- package/dist/harness-runtime/lib/dd-grok.mjs +6 -3
- package/dist/harness-runtime/lib/dd-opencode-daemon.mjs +66 -20
- package/dist/harness-runtime/lib/dd-opencode.mjs +30 -3
- package/dist/harness-runtime/lib/dd-zcode-daemon.mjs +7 -6
- package/dist/harness-runtime/lib/dd-zcode.mjs +44 -14
- package/dist/harness-runtime/lib/delegation-instructions.mjs +1 -1
- package/dist/harness-runtime/lib/driver-recovery.mjs +9 -8
- package/dist/harness-runtime/lib/entity-references.d.mts +2 -0
- package/dist/harness-runtime/lib/entity-references.mjs +9 -0
- package/dist/harness-runtime/lib/managed-daemon.d.mts +1 -1
- package/dist/harness-runtime/lib/managed-daemon.mjs +24 -9
- package/dist/harness-runtime/lib/native-children.d.mts +8 -1
- package/dist/harness-runtime/lib/native-children.mjs +114 -8
- package/dist/harness-runtime/lib/native-hook-command.mjs +4 -4
- package/dist/harness-runtime/lib/native-request-proof.mjs +1 -1
- package/dist/harness-runtime/lib/observation-clock.d.mts +2 -2
- package/dist/harness-runtime/lib/observation-clock.mjs +7 -4
- package/dist/harness-runtime/lib/opencode-plugin.mjs +3 -3
- package/dist/harness-runtime/lib/operation-errors.mjs +1 -1
- package/dist/harness-runtime/lib/provider-overload-policy.d.mts +5 -3
- package/dist/harness-runtime/lib/provider-overload-policy.mjs +20 -7
- package/dist/harness-runtime/lib/runner-events.mjs +3 -1
- package/dist/harness-runtime/lib/setup-progress.mjs +8 -6
- package/dist/harness-runtime/lib/tool-observations.mjs +1 -1
- package/dist/schemas/diagnostic-stage-result.schema.json +354 -0
- package/dist/schemas/diagnostic-work-result.schema.json +296 -0
- package/dist/schemas/execution-bundle.schema.json +7 -1
- package/dist/schemas/mb-upgrade-stage-result.schema.json +863 -0
- package/dist/schemas/memory-repair-work-result.schema.json +9 -2
- package/dist/schemas/project-flow-pack-manifest.schema.json +1 -1
- package/dist/schemas/release-impact.schema.json +3 -3
- package/dist/schemas/stage-start-response.schema.json +4 -0
- package/dist/services/accepted-intent.js +13 -1
- package/dist/services/canon.js +43 -7
- package/dist/services/check-environment.js +21 -0
- package/dist/services/check-runner-evidence.js +267 -15
- package/dist/services/cli-operation-classifier.js +1 -1
- package/dist/services/cli-upgrade-allowlist.js +1 -1
- package/dist/services/code-checks.js +34 -19
- package/dist/services/compatibility-preflight.js +5 -2
- package/dist/services/controller-execution.js +10 -4
- package/dist/services/controller-fanout.js +34 -10
- package/dist/services/controller-memory-task.js +45 -0
- package/dist/services/controller-memory-worker.js +49 -0
- package/dist/services/dashboard.js +2 -2
- package/dist/services/descriptor-run-settlement.js +5 -3
- package/dist/services/descriptor-run-start.js +46 -20
- package/dist/services/descriptor-stage-coordinator.js +66 -13
- package/dist/services/descriptor-stage-settlement.js +2 -1
- package/dist/services/descriptor-stage-start.js +40 -4
- package/dist/services/diagnostic-evidence.js +73 -0
- package/dist/services/diagnostic-report-publication.js +72 -0
- package/dist/services/diagnostic-report.js +67 -0
- package/dist/services/diagnostic-run-bundle.js +74 -0
- package/dist/services/diagnostic-snapshot.js +159 -0
- package/dist/services/diagnostic-stage-handlers.js +108 -0
- package/dist/services/diagnostic-stage-results.js +132 -0
- package/dist/services/diagnostic-start.js +84 -0
- package/dist/services/diagnostic-work-batch.js +34 -0
- package/dist/services/diagnostic-work-collection.js +97 -0
- package/dist/services/diagnostic-work-registration.js +101 -0
- package/dist/services/diagnostic-work-results.js +93 -0
- package/dist/services/engines.js +64 -14
- package/dist/services/eval-snapshots.js +186 -119
- package/dist/services/execution-bundles.js +19 -7
- package/dist/services/external-work-launch.js +5 -3
- package/dist/services/flow-catalog.js +13 -4
- package/dist/services/frozen-engine.js +148 -11
- package/dist/services/hook-memory-routing.js +188 -0
- package/dist/services/hook-recovery-guard.js +14 -0
- package/dist/services/hook-session-observation.js +24 -0
- package/dist/services/hooks.js +27 -8
- package/dist/services/hybrid-contracts.js +4 -13
- package/dist/services/hybrid-digest.js +14 -0
- package/dist/services/installed-flow-source.js +47 -4
- package/dist/services/lifecycle-invocations.js +179 -38
- package/dist/services/memory-accepted-composition.js +5 -4
- package/dist/services/memory-completed-correction-wave.js +23 -15
- package/dist/services/memory-compose-handler.js +163 -21
- package/dist/services/memory-composition-acceptance.js +186 -10
- package/dist/services/memory-composition-graph.js +42 -11
- package/dist/services/memory-composition-registration.js +23 -8
- package/dist/services/memory-composition-work-custody.js +14 -9
- package/dist/services/memory-composition-work-results.js +15 -3
- package/dist/services/memory-corrected-review-batch.js +10 -3
- package/dist/services/memory-corrected-review-findings.js +4 -2
- package/dist/services/memory-corrected-review-inputs.js +10 -5
- package/dist/services/memory-corrected-review-receipt.js +17 -9
- package/dist/services/memory-corrected-review-registration.js +3 -2
- package/dist/services/memory-correction-check-receipt.js +20 -4
- package/dist/services/memory-custody-snapshot.js +36 -0
- package/dist/services/memory-finish-evidence.js +44 -22
- package/dist/services/memory-finish-handler.js +37 -23
- package/dist/services/memory-finish-integration.js +218 -48
- package/dist/services/memory-finish-readiness.js +66 -19
- package/dist/services/memory-git-integration.js +87 -18
- package/dist/services/memory-historical-corrected-review.js +21 -15
- package/dist/services/memory-historical-original-review.js +22 -9
- package/dist/services/memory-historical-repair-registration.js +14 -8
- package/dist/services/memory-historical-second-review.js +30 -11
- package/dist/services/memory-init-contexts.js +44 -2
- package/dist/services/memory-init-controller-start.js +27 -8
- package/dist/services/memory-init-git-policy.js +50 -21
- package/dist/services/memory-init-lint-capability.js +24 -6
- package/dist/services/memory-init-native-capacity.js +17 -4
- package/dist/services/memory-init-preflight.js +21 -17
- package/dist/services/memory-init-request-candidate.js +73 -17
- package/dist/services/memory-init-request-input.js +23 -7
- package/dist/services/memory-init-run-bundle.js +49 -12
- package/dist/services/memory-init-storage.js +17 -7
- package/dist/services/memory-init-worktrunk-capability.js +3 -7
- package/dist/services/memory-layer.js +71 -6
- package/dist/services/memory-lint-config.js +111 -2
- package/dist/services/memory-native-review-copy.js +5 -67
- package/dist/services/memory-normal-flow-entry.js +12 -0
- package/dist/services/memory-ordinary-defaults.js +22 -5
- package/dist/services/memory-preflight-handler.js +15 -0
- package/dist/services/memory-repair-current-view.js +1 -1
- package/dist/services/memory-repair-document-delta.js +70 -33
- package/dist/services/memory-repair-history.js +12 -7
- package/dist/services/memory-repair-physical-binding.js +1 -119
- package/dist/services/memory-repair-publication-plan.js +5 -3
- package/dist/services/memory-repair-publication-source.js +1 -1
- package/dist/services/memory-repair-publication.js +3 -2
- package/dist/services/memory-repair-registration.js +2 -1
- package/dist/services/memory-repair-work-baseline.js +4 -2
- package/dist/services/memory-repair-work-completion.js +32 -12
- package/dist/services/memory-repair-work-results.js +21 -2
- package/dist/services/memory-repair-workspace.js +11 -2
- package/dist/services/memory-repair-writer-copy.js +2 -1
- package/dist/services/memory-result-binding.js +78 -12
- package/dist/services/memory-retained-review-authority.js +17 -8
- package/dist/services/memory-retained-second-baseline.js +1 -1
- package/dist/services/memory-retained-second-repairs.js +12 -6
- package/dist/services/memory-review-correction-plan.js +47 -12
- package/dist/services/memory-review-handler.js +17 -17
- package/dist/services/memory-review-inputs.js +55 -9
- package/dist/services/memory-review-limits.js +10 -0
- package/dist/services/memory-review-receipt.js +2 -2
- package/dist/services/memory-review-registration.js +13 -4
- package/dist/services/memory-review-work-custody.js +3 -1
- package/dist/services/memory-review-work-input.js +26 -7
- package/dist/services/memory-review-work-results.js +26 -4
- package/dist/services/memory-second-corrected-review-inputs.js +5 -3
- package/dist/services/memory-second-correction-plan.js +2 -1
- package/dist/services/memory-second-repair-launch.js +4 -4
- package/dist/services/memory-second-repair-source.js +1 -1
- package/dist/services/memory-stage-dispatch.js +38 -1
- package/dist/services/memory-stage-results.js +113 -22
- package/dist/services/memory-tool-probe.js +25 -0
- package/dist/services/memory-upgrade-analysis-handler.js +243 -0
- package/dist/services/memory-upgrade-bootstrap-readiness.js +251 -0
- package/dist/services/memory-upgrade-evidence.js +100 -0
- package/dist/services/memory-upgrade-inventory.js +442 -0
- package/dist/services/memory-upgrade-publication-custody.js +73 -0
- package/dist/services/memory-upgrade-publication.js +413 -0
- package/dist/services/memory-upgrade-selection.js +430 -0
- package/dist/services/memory-upgrade-start.js +60 -0
- package/dist/services/memory-upgrade-storage-readiness.js +340 -0
- package/dist/services/memory-verification-acceptance.js +38 -17
- package/dist/services/memory-verification-checks.js +92 -27
- package/dist/services/memory-verification-receipt.js +6 -5
- package/dist/services/memory-verify-handler.js +15 -13
- package/dist/services/memory-work-result-json.js +12 -0
- package/dist/services/memory-work-result-preparation.js +5 -4
- package/dist/services/merge-server.js +4 -2
- package/dist/services/migrations.js +127 -98
- package/dist/services/native-daemon-history.js +2 -1
- package/dist/services/physical-startup.js +45 -0
- package/dist/services/project-registry.js +97 -0
- package/dist/services/project-summary.js +1 -1
- package/dist/services/projects.js +3 -93
- package/dist/services/prompt-delivery.js +71 -0
- package/dist/services/provider-capacity.js +112 -46
- package/dist/services/recovery-snapshot-database.js +32 -4
- package/dist/services/release-cache.js +8 -0
- package/dist/services/repair-intents.js +1 -12
- package/dist/services/repair-receipt.js +13 -0
- package/dist/services/run-control-worker.js +35 -18
- package/dist/services/run-control.js +2 -1
- package/dist/services/run-controller-adapter.js +2 -1
- package/dist/services/run-controller-process.js +4 -7
- package/dist/services/run-controller-recovery.js +8 -2
- package/dist/services/run-controller.js +215 -94
- package/dist/services/run-fork.js +1 -1
- package/dist/services/run-recovery-guard.js +30 -0
- package/dist/services/run-recovery.js +11 -44
- package/dist/services/run-start-custody.js +17 -5
- package/dist/services/runs.js +6 -1
- package/dist/services/runtime-scope-resume.js +12 -6
- package/dist/services/runtime-scope-stop.js +3 -2
- package/dist/services/runtime-scope-worker.js +46 -19
- package/dist/services/runtime-service.js +2 -1
- package/dist/services/sessions.js +3 -24
- package/dist/services/setup.js +63 -13
- package/dist/services/stage-handlers.js +22 -7
- package/dist/services/stage-pause.js +111 -44
- package/dist/services/version-status.js +1 -1
- package/dist/services/vnext-code.js +2 -2
- package/dist/services/vnext-fanout.js +51 -9
- package/dist/services/vnext-plan-review.js +3 -5
- package/dist/services/work-registry.js +117 -39
- package/dist/services/workspace-bootstrap.js +2 -1
- package/dist/services/workspace-files.js +1 -1
- package/dist/shared/entity-references.js +1 -8
- package/dist/storage/data-preservation.js +65 -0
- package/dist/storage/database.js +53 -19
- package/dist/storage/import-guard.js +84 -27
- package/dist/storage/memory-upgrade-schema-bootstrap.js +38 -0
- package/dist/storage/paired-writer-migration.js +424 -0
- package/dist/storage/writer-contract.js +11 -4
- package/dist/storage/writer-migration.js +313 -52
- package/dist/storage/writer-restore.js +217 -0
- package/package.json +13 -6
|
@@ -5,6 +5,7 @@ import { commandJson } from "./process-json.mjs";
|
|
|
5
5
|
import { errorRecord, reportedError, diagnosticJson } from "./operation-errors.mjs";
|
|
6
6
|
import { writeJsonAtomic } from "./runner-events.mjs";
|
|
7
7
|
import { runBoundedCommand } from "./native-hook-command.mjs";
|
|
8
|
+
import { processSnapshot } from "./process-snapshot.mjs";
|
|
8
9
|
import { createRenewalState, maintenanceRetryable, renewalConfirmed, renewalDelay, renewalFailure, renewalRemaining, RENEWAL_POLICY } from "./lease-renewal.mjs";
|
|
9
10
|
|
|
10
11
|
const leases = new Map();
|
|
@@ -409,21 +410,28 @@ export async function closeManagedBridge(config, state, bridge) {
|
|
|
409
410
|
await finishDaemonProcess(config, state.provider_process, "stopped", "daemon_stopped");
|
|
410
411
|
}
|
|
411
412
|
|
|
412
|
-
export function scheduleDaemonProcessHeartbeat(config, record) {
|
|
413
|
+
export function scheduleDaemonProcessHeartbeat(config, record, { signal } = {}) {
|
|
413
414
|
if (!record?.id || !record.lease_token) return;
|
|
414
415
|
const key = leaseKey(config, record);
|
|
415
|
-
if (leases.has(key))
|
|
416
|
-
|
|
416
|
+
if (leases.has(key)) {
|
|
417
|
+
if (signal && leases.get(key).signal !== signal) throw Object.assign(new Error("Heartbeat cancellation owner differs from the retained lease"), { code: "process_maintenance_signal_conflict" });
|
|
418
|
+
return;
|
|
419
|
+
}
|
|
420
|
+
const lease = { pending: null, error: null, timer: null, lost: false, closing: false, signal, expiresAt: Date.parse(record.lease_expires_at ?? ""), renewal: createRenewalState({ leaseExpiresAt: record.lease_expires_at }), stateDir: path.resolve(config.runtime_owner?.state_dir ?? config.stateDir ?? config.cwd), config, record };
|
|
417
421
|
leases.set(key, lease);
|
|
418
422
|
// Retry no-effect contention inside the same authority episode while native
|
|
419
423
|
// work is active; scheduling ticks are not ownership evidence.
|
|
420
424
|
lease.timer = setInterval(() => reconfirmLease(lease, () => {}).catch(() => {}), 60_000);
|
|
421
425
|
lease.timer.unref();
|
|
426
|
+
if (signal) {
|
|
427
|
+
signal.addEventListener("abort", () => clearInterval(lease.timer), { once: true });
|
|
428
|
+
if (signal.aborted) clearInterval(lease.timer);
|
|
429
|
+
}
|
|
422
430
|
}
|
|
423
431
|
|
|
424
|
-
export async function heartbeatDaemonProcess(config, record) {
|
|
432
|
+
export async function heartbeatDaemonProcess(config, record, { signal } = {}) {
|
|
425
433
|
if (!record?.id || !record.lease_token) return;
|
|
426
|
-
scheduleDaemonProcessHeartbeat(config, record);
|
|
434
|
+
scheduleDaemonProcessHeartbeat(config, record, { signal });
|
|
427
435
|
const lease = leases.get(leaseKey(config, record));
|
|
428
436
|
if (lease.closing) throw Object.assign(new Error("Managed process is closing"), { code: "process_closing" });
|
|
429
437
|
if (lease.lost) throw lease.error;
|
|
@@ -435,7 +443,7 @@ export async function heartbeatDaemonProcess(config, record) {
|
|
|
435
443
|
lease.pending = (async () => {
|
|
436
444
|
try {
|
|
437
445
|
lease.renewal.uncertainSince ??= performance.now();
|
|
438
|
-
const result = await runtimeProcess(config, "heartbeat", { id: record.id, "lease-token": record.lease_token, "lease-ms": 300_000 }, { timeoutMs: renewalRemaining(lease.renewal) });
|
|
446
|
+
const result = await runtimeProcess(config, "heartbeat", { id: record.id, "lease-token": record.lease_token, "lease-ms": 300_000 }, { timeoutMs: renewalRemaining(lease.renewal), signal: lease.signal });
|
|
439
447
|
if (result?.ok === false) {
|
|
440
448
|
lease.lost = true; clearInterval(lease.timer);
|
|
441
449
|
throw Object.assign(new Error("managed process lease is no longer owned; new productive operations are blocked"), { code: "process_lease_lost", details: { process_id: record.id } });
|
|
@@ -591,14 +599,14 @@ export async function closeBridgeProcess(bridge, drain, stopOptions = {}) {
|
|
|
591
599
|
finally { bridge.closePending = null; }
|
|
592
600
|
}
|
|
593
601
|
|
|
594
|
-
export async function stopProcessGroup(child, graceMs = 1_000, { detached = true, settlementMs = 10_000 } = {}) {
|
|
602
|
+
export async function stopProcessGroup(child, graceMs = 1_000, { detached = true, settlementMs = 10_000, started = child?.started ?? null } = {}) {
|
|
595
603
|
if (!child?.pid) return;
|
|
596
604
|
if (processStops.has(child)) return await processStops.get(child);
|
|
597
|
-
const pending = stopOwnedChild(child, graceMs, detached, settlementMs);
|
|
605
|
+
const pending = stopOwnedChild(child, graceMs, detached, settlementMs, started);
|
|
598
606
|
processStops.set(child, pending);
|
|
599
607
|
try { return await pending; } finally { processStops.delete(child); }
|
|
600
608
|
}
|
|
601
|
-
async function stopOwnedChild(child, graceMs, detached, settlementMs) {
|
|
609
|
+
async function stopOwnedChild(child, graceMs, detached, settlementMs, started) {
|
|
602
610
|
const target = process.platform === "win32" || !detached ? child.pid : -child.pid;
|
|
603
611
|
const details = signal => ({ pid: child.pid, pgid: target < 0 ? child.pid : null, signal, platform: process.platform, exit_code: child.exitCode ?? null, signal_code: child.signalCode ?? null, observed_at: new Date().toISOString() });
|
|
604
612
|
const alive = () => { try { process.kill(target, 0); return true; } catch (error) { if (error.code === "ESRCH") return false; if (error.code === "EPERM") return true; throw error; } };
|
|
@@ -608,6 +616,13 @@ async function stopOwnedChild(child, graceMs, detached, settlementMs) {
|
|
|
608
616
|
return !alive();
|
|
609
617
|
};
|
|
610
618
|
const signal = async name => {
|
|
619
|
+
// A retained raw PID is not a ChildProcess handle. Revalidate its birth at
|
|
620
|
+
// every signal, including escalation after the TERM grace period.
|
|
621
|
+
if (started !== null) {
|
|
622
|
+
if (typeof started !== 'string' || !started) throw Object.assign(new Error('process birth identity is invalid'), { code: 'process_identity_unproven' });
|
|
623
|
+
const observed = (await processSnapshot()).find(item => item.pid === child.pid);
|
|
624
|
+
if (!observed || observed.started !== started || observed.zombie) return false;
|
|
625
|
+
}
|
|
611
626
|
try { process.kill(target, name); return true; }
|
|
612
627
|
catch (error) {
|
|
613
628
|
if (error.code === "ESRCH") return false;
|
|
@@ -1,4 +1,11 @@
|
|
|
1
|
-
export const NATIVE_CHILD_CONTRACT_VERSION: "native-children@
|
|
1
|
+
export const NATIVE_CHILD_CONTRACT_VERSION: "native-children@2";
|
|
2
|
+
export const NATIVE_TREE_SETTLEMENT_CONTRACT: "native-tree-settlement@2";
|
|
3
|
+
export const NATIVE_SHUTDOWN_CONTRACT: "dd-flow/daemon-shutdown@2";
|
|
4
|
+
export const NATIVE_SHUTDOWN_PHASES: readonly string[];
|
|
5
|
+
export function nativeShutdownConfirmed(receipt: unknown, binding?: { daemonId?: string; generation?: number }): boolean;
|
|
6
|
+
export function nativeExecutionReceipt(receipt: Record<string, unknown>, options?: { daemonId?: string; generation?: number; revision?: number; resources?: Record<string, unknown> }): Record<string, unknown>;
|
|
7
|
+
export type NativeTreeReadiness = "ready" | "owned_children_live" | "inspection_pending" | "unproven";
|
|
8
|
+
export function nativeTreeReadiness(receipt: Record<string, unknown>, binding: { sessionId: string; operationId: string; turnId?: string; requestProof?: unknown; allowDescendants?: boolean }): NativeTreeReadiness;
|
|
2
9
|
export function settledNativeTree(root: string, children: Record<string, unknown>[], settled: boolean, rootSettled?: boolean): Record<string, unknown>;
|
|
3
10
|
export function safeNativeFailure(value: unknown): Record<string, unknown> | undefined;
|
|
4
11
|
export interface NativeChild {
|
|
@@ -1,9 +1,104 @@
|
|
|
1
|
-
export const NATIVE_CHILD_CONTRACT_VERSION = "native-children@
|
|
1
|
+
export const NATIVE_CHILD_CONTRACT_VERSION = "native-children@2";
|
|
2
|
+
export const NATIVE_TREE_SETTLEMENT_CONTRACT = "native-tree-settlement@2";
|
|
3
|
+
export const NATIVE_SHUTDOWN_CONTRACT = "dd-flow/daemon-shutdown@2";
|
|
4
|
+
export const NATIVE_SHUTDOWN_PHASES = Object.freeze(["execution", "owned_resources", "provider", "daemon_resource"]);
|
|
2
5
|
import { errorRecord } from "./operation-errors.mjs";
|
|
3
6
|
import { safeProviderError } from "./provider-overload-policy.mjs";
|
|
7
|
+
import { streamRequestKey } from "./provider-overload-policy.mjs";
|
|
8
|
+
const liveStatus = value => ["active", "running", "inprogress", "pending", "streaming", "busy"].includes(String(typeof value === "object" && value ? value.type : value).toLowerCase());
|
|
9
|
+
|
|
10
|
+
/** One current receipt only. Producers, not status ranks, establish freshness. */
|
|
11
|
+
export function nativeTreeReadiness(receipt, { sessionId, operationId, turnId, requestProof, allowDescendants = true } = {}) {
|
|
12
|
+
if (receipt?.native_settlement?.contract !== NATIVE_TREE_SETTLEMENT_CONTRACT || receipt.native_settlement.scope !== "required_execution") return "unproven";
|
|
13
|
+
if (!receipt || typeof sessionId !== "string" || !sessionId || typeof operationId !== "string" || !operationId || receipt.provider_session_id !== sessionId
|
|
14
|
+
|| turnId !== undefined && receipt.terminal_turn_id !== undefined && receipt.terminal_turn_id !== turnId) return "unproven";
|
|
15
|
+
const requestKey = requestProof?.kind === "stream_request" ? streamRequestKey(requestProof) : null;
|
|
16
|
+
if (requestProof?.kind === "stream_request" && (!requestKey || requestProof.operation_id !== operationId
|
|
17
|
+
|| streamRequestKey(receipt.terminal_metadata?.request_proof) !== requestKey)) return "unproven";
|
|
18
|
+
const pending = receipt.native_settlement;
|
|
19
|
+
if (pending?.state === "inspection_pending") {
|
|
20
|
+
const clock = pending.observation_clock;
|
|
21
|
+
return receipt.settled === false && pending.confirmed === false && pending.operation_id === operationId
|
|
22
|
+
&& pending.provider_session_id === sessionId && requestKey && pending.request_key === requestKey
|
|
23
|
+
&& clock?.policy === "operation-progress@1" && Number.isFinite(clock.started_at)
|
|
24
|
+
&& Number.isSafeInteger(clock.timeout_ms) && clock.timeout_ms > 0 && clock.timeout_ms <= 2147483647
|
|
25
|
+
&& Number.isFinite(clock.elapsed_ms) && clock.elapsed_ms >= 0 && clock.elapsed_ms < clock.timeout_ms
|
|
26
|
+
&& Number.isSafeInteger(clock.gaps) && clock.gaps >= 0
|
|
27
|
+
&& typeof clock.uncertain === "boolean"
|
|
28
|
+
&& Number.isFinite(clock.uncertainty_elapsed_ms) && clock.uncertainty_elapsed_ms >= 0
|
|
29
|
+
&& (clock.observed_at === null || Number.isFinite(clock.observed_at))
|
|
30
|
+
&& clock.observation_lost === false ? "inspection_pending" : "unproven";
|
|
31
|
+
}
|
|
32
|
+
if (receipt.native_turn_items?.pending === true) return "unproven";
|
|
33
|
+
const ids = receipt.tree_session_ids, observations = receipt.tree_observations, unsettled = receipt.unsettled_sessions;
|
|
34
|
+
if (!Array.isArray(ids) || !ids.length || ids[0] !== sessionId || ids.some(id => typeof id !== "string" || !id)
|
|
35
|
+
|| new Set(ids).size !== ids.length || !allowDescendants && ids.length !== 1
|
|
36
|
+
|| !Array.isArray(observations) || observations.length !== ids.length
|
|
37
|
+
|| !ids.every(id => observations.filter(item => item?.provider_session_id === id).length === 1)
|
|
38
|
+
|| observations.some(item => !item || typeof item !== "object" || Array.isArray(item) || typeof item.settled !== "boolean" || item.settled === true && liveStatus(item.status)) || !Array.isArray(unsettled)
|
|
39
|
+
|| new Set(unsettled).size !== unsettled.length) return "unproven";
|
|
40
|
+
// A producer's tree list cannot silently omit another positively observed
|
|
41
|
+
// owned child. Normalize the same receipt, not a second historical ledger.
|
|
42
|
+
try {
|
|
43
|
+
const children = normalizeNativeChildren(receipt, sessionId);
|
|
44
|
+
if (children.some(child => !ids.includes(child.session_id) || !ids.includes(child.parent_session_id) || child.parent_session_id === child.session_id)) return "unproven";
|
|
45
|
+
for (const child of children) {
|
|
46
|
+
let parent = child.parent_session_id;
|
|
47
|
+
const visited = new Set([child.session_id]);
|
|
48
|
+
while (parent !== sessionId) {
|
|
49
|
+
if (visited.has(parent)) return "unproven";
|
|
50
|
+
visited.add(parent); parent = children.find(item => item.session_id === parent)?.parent_session_id;
|
|
51
|
+
if (!parent) return "unproven";
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
} catch { return "unproven"; }
|
|
55
|
+
const live = observations.filter(item => !item.settled);
|
|
56
|
+
if (unsettled.length !== live.length || !live.every(item => unsettled.includes(item.provider_session_id))) return "unproven";
|
|
57
|
+
const settlement = receipt.settlement;
|
|
58
|
+
const budgetSettled = !settlement || settlement.state === "not_required"
|
|
59
|
+
|| settlement.state === "settled" && settlement.operation_ids?.includes(operationId) === true;
|
|
60
|
+
if (receipt.settled === true && receipt.native_settled !== false && pending?.confirmed !== false && !live.length && budgetSettled) return "ready";
|
|
61
|
+
const root = observations.find(item => item.provider_session_id === sessionId);
|
|
62
|
+
const knownLive = live.length > 0 && live.every(item => item.provider_session_id !== sessionId && liveStatus(item.status));
|
|
63
|
+
return allowDescendants && receipt.settled === false && root.settled === true && knownLive
|
|
64
|
+
&& (!settlement || settlement.state === "not_required" || settlement.state === "pending" && settlement.reason === "tree_unsettled")
|
|
65
|
+
? "owned_children_live" : "unproven";
|
|
66
|
+
}
|
|
4
67
|
|
|
5
68
|
function nativeFailure(code, message, exitCode, details) { return Object.assign(new Error(message), { code, exitCode, details }); }
|
|
6
69
|
|
|
70
|
+
/** Control settlement is not native settlement; explicit uncertainty wins. */
|
|
71
|
+
export function nativeShutdownConfirmed(receipt, { daemonId, generation } = {}) {
|
|
72
|
+
const proof = receipt?.schema_id === NATIVE_SHUTDOWN_CONTRACT ? receipt : receipt?.shutdown_proof;
|
|
73
|
+
const result = receipt?.schema_id === NATIVE_SHUTDOWN_CONTRACT ? receipt.result : receipt;
|
|
74
|
+
return Boolean(proof?.schema_id === NATIVE_SHUTDOWN_CONTRACT && typeof proof.daemon_id === "string" && proof.daemon_id
|
|
75
|
+
&& (daemonId === undefined || proof.daemon_id === daemonId) && Number.isSafeInteger(proof.generation) && proof.generation >= 0
|
|
76
|
+
&& (generation === undefined || proof.generation === generation)
|
|
77
|
+
&& Array.isArray(proof.required_phases) && new Set(proof.required_phases).size === proof.required_phases.length
|
|
78
|
+
&& NATIVE_SHUTDOWN_PHASES.every(phase => proof.required_phases.includes(phase))
|
|
79
|
+
&& proof.required_phases.every(phase => typeof phase === "string" && proof.phases?.[phase] === true)
|
|
80
|
+
&& result?.daemon_id === proof.daemon_id && result.shutdown_contract === NATIVE_SHUTDOWN_CONTRACT
|
|
81
|
+
&& result.clean === true && result.stopped === true && result.physical_stopped === true
|
|
82
|
+
&& result.native_settled === true && result.settled === true && result.native_settlement?.confirmed !== false
|
|
83
|
+
&& result.resource_retirement?.confirmed !== false);
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/** Scope metadata does not manufacture readiness or physical cleanup. */
|
|
87
|
+
export function nativeExecutionReceipt(receipt, { daemonId, generation = 0, revision = 0, resources } = {}) {
|
|
88
|
+
if (receipt.native_settlement?.contract !== undefined && receipt.native_settlement.contract !== NATIVE_TREE_SETTLEMENT_CONTRACT
|
|
89
|
+
|| receipt.native_settlement?.scope !== undefined && receipt.native_settlement.scope !== "required_execution")
|
|
90
|
+
throw Object.assign(new Error("historical native proof requires fresh execution observation"), { code: "native_contract_mismatch" });
|
|
91
|
+
if ((resources?.entries ?? []).some(item => item.owner_daemon_id !== undefined && item.owner_daemon_id !== daemonId
|
|
92
|
+
|| item.generation !== undefined && item.generation !== generation)) throw Object.assign(new Error("native resource belongs to another owner"), { code: "native_resource_owner_mismatch" });
|
|
93
|
+
const owner = { owner_daemon_id: daemonId ?? null, generation, revision };
|
|
94
|
+
return { ...receipt,
|
|
95
|
+
native_settlement: { ...receipt.native_settlement, contract: NATIVE_TREE_SETTLEMENT_CONTRACT, scope: "required_execution", ...owner },
|
|
96
|
+
native_resources: { observed_at: new Date().toISOString(), coverage: "unsupported", ...resources,
|
|
97
|
+
schema_id: "native-resources@1", ...owner,
|
|
98
|
+
entries: (resources?.entries ?? []).map(item => ({ ...item, owner_daemon_id: owner.owner_daemon_id, generation: owner.generation })) }
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
|
|
7
102
|
/** Project a positively settled native tree, never infer it from missing activity. */
|
|
8
103
|
export function settledNativeTree(root, children, settled, rootSettled = false) {
|
|
9
104
|
if (settled !== true && rootSettled !== true || typeof root !== "string" || !root || !Array.isArray(children)) return {};
|
|
@@ -12,9 +107,9 @@ export function settledNativeTree(root, children, settled, rootSettled = false)
|
|
|
12
107
|
const observations = [{ provider_session_id: root, status: "idle", settled: true }, ...children.map((child, index) => {
|
|
13
108
|
const status = String(child.status ?? child.state ?? "unknown").toLowerCase();
|
|
14
109
|
const terminal = settled === true || ["completed", "success", "succeeded", "failed", "cancelled", "canceled", "settled_by_root"].includes(status);
|
|
15
|
-
return { provider_session_id: ids[index], status: terminal ? "idle" :
|
|
110
|
+
return { provider_session_id: ids[index], status: terminal ? "idle" : liveStatus(status) ? "running" : "unknown", settled: terminal };
|
|
16
111
|
})];
|
|
17
|
-
return { tree_session_ids: [root, ...ids], unsettled_sessions: observations.filter(item => !item.settled).map(item => item.provider_session_id), tree_observations: observations };
|
|
112
|
+
return { native_settlement: { contract: NATIVE_TREE_SETTLEMENT_CONTRACT, scope: "required_execution" }, tree_session_ids: [root, ...ids], unsettled_sessions: observations.filter(item => !item.settled).map(item => item.provider_session_id), tree_observations: observations };
|
|
18
113
|
}
|
|
19
114
|
|
|
20
115
|
/** Retain typed failure facts, never provider headers, request bodies or free-form messages. */
|
|
@@ -51,6 +146,8 @@ export function normalizeNativeChildren(receipt, rootSessionId) {
|
|
|
51
146
|
.map(value => text(object(value), ["provider_session_id", "session_id"]))
|
|
52
147
|
.filter(id => id !== null)
|
|
53
148
|
);
|
|
149
|
+
const unsettledTree = new Set(list(receipt.tree_observations).filter(value => object(value).settled === false)
|
|
150
|
+
.map(value => text(object(value), ["provider_session_id", "session_id"])).filter(Boolean));
|
|
54
151
|
const add = (raw, source, scopedParent = null, fallback = "unknown") => {
|
|
55
152
|
const value = object(raw);
|
|
56
153
|
const id = text(value, ["provider_session_id", "session_id", "sessionId", "childSessionId", "conversation_id", "conversationId", "id", "subagentId"]);
|
|
@@ -66,9 +163,12 @@ export function normalizeNativeChildren(receipt, rootSessionId) {
|
|
|
66
163
|
// Provider idle means no current activity, not a Work result. Only an
|
|
67
164
|
// explicit terminal result or a separately proven root settlement may be
|
|
68
165
|
// terminal here; unknown states stay unknown for reconciliation.
|
|
69
|
-
const status = ["completed", "complete", "done", "success", "succeeded"].includes(state) ? "completed" : state === "settled_by_root" ? "settled_by_root" : ["failed", "error"].includes(state) ? "failed" : ["cancelled", "canceled", "interrupted"].includes(state) ? "cancelled" :
|
|
166
|
+
const status = ["completed", "complete", "done", "success", "succeeded"].includes(state) ? "completed" : state === "settled_by_root" ? "settled_by_root" : ["failed", "error"].includes(state) ? "failed" : ["cancelled", "canceled", "interrupted"].includes(state) ? "cancelled" : liveStatus(state) ? "running" : fallback;
|
|
70
167
|
const rootChild = parent === rootSessionId;
|
|
71
168
|
const settled = rootChild && status === "unknown" && settledTree.has(id) ? "settled_by_root" : status;
|
|
169
|
+
if ((unsettledTree.has(id) || value.native_turn_items?.pending === true) && ["completed", "failed", "cancelled", "settled_by_root"].includes(settled)
|
|
170
|
+
|| settledTree.has(id) && (unsettledTree.has(id) || status === "running" || value.native_turn_items?.pending === true))
|
|
171
|
+
throw nativeFailure("native_child_observation_conflict", "Current child outcome contradicts tree observation", 1, { session_id: id, source });
|
|
72
172
|
candidates.push({ session_id: id, parent_session_id: parent ?? "", status: settled, source, ...(rootChild ? {} : { provenance: "parent_mismatch" }),
|
|
73
173
|
...(text(value, ["terminal_turn_id"]) ? { terminal_turn_id: text(value, ["terminal_turn_id"]) } : {}),
|
|
74
174
|
...(text(value, ["terminal_observed_at"]) ? { terminal_observed_at: text(value, ["terminal_observed_at"]) } : {}),
|
|
@@ -87,9 +187,12 @@ export function normalizeNativeChildren(receipt, rootSessionId) {
|
|
|
87
187
|
add(value, "zcode/session/subagents", rootSessionId);
|
|
88
188
|
}
|
|
89
189
|
const unique = new Map();
|
|
90
|
-
const
|
|
190
|
+
const terminal = value => ["completed", "failed", "cancelled", "settled_by_root"].includes(value);
|
|
91
191
|
for (const child of candidates) {
|
|
92
192
|
const previous = unique.get(child.session_id);
|
|
193
|
+
if (previous && (terminal(previous.status) && child.native_turn_items?.pending === true
|
|
194
|
+
|| terminal(child.status) && previous.native_turn_items?.pending === true))
|
|
195
|
+
throw nativeFailure("native_child_observation_conflict", "Current child outcome contradicts pending native items", 1, { session_id: child.session_id, observations: [previous, child] });
|
|
93
196
|
if (previous && previous.parent_session_id !== child.parent_session_id) {
|
|
94
197
|
// Keep the contradictory parent visible to the existing provenance gate,
|
|
95
198
|
// independently of input order or which source happened to say success.
|
|
@@ -97,17 +200,20 @@ export function normalizeNativeChildren(receipt, rootSessionId) {
|
|
|
97
200
|
unique.set(child.session_id, { ...foreign, provenance: "parent_mismatch" });
|
|
98
201
|
continue;
|
|
99
202
|
}
|
|
100
|
-
if (previous &&
|
|
203
|
+
if (previous && terminal(previous.status) && terminal(child.status) && previous.status !== child.status && ![previous.status, child.status].includes("settled_by_root")) throw nativeFailure("native_child_outcome_conflict", "Native child has contradictory terminal outcomes", 1, { session_id: child.session_id, observations: [previous, child] });
|
|
204
|
+
if (previous && (previous.status === "running" && terminal(child.status) || child.status === "running" && terminal(previous.status)))
|
|
205
|
+
throw nativeFailure("native_child_observation_conflict", "Current child has both active and terminal facts", 1, { session_id: child.session_id, observations: [previous, child] });
|
|
101
206
|
if (previous && previous.status === child.status && previous.terminal_turn_id && child.terminal_turn_id && previous.terminal_turn_id !== child.terminal_turn_id)
|
|
102
207
|
throw nativeFailure("native_child_turn_conflict", "Native child has contradictory terminal Turn identities", 1, { session_id: child.session_id, observations: [previous, child] });
|
|
103
208
|
if (previous && previous.status === child.status) {
|
|
104
209
|
unique.set(child.session_id, { ...previous, ...child,
|
|
105
210
|
...(child.terminal_turn_id ?? previous.terminal_turn_id ? { terminal_turn_id: child.terminal_turn_id ?? previous.terminal_turn_id } : {}),
|
|
106
211
|
...(child.provider_error ?? previous.provider_error ? { provider_error: child.provider_error ?? previous.provider_error } : {}),
|
|
107
|
-
...(child.native_turn_items ?? previous.native_turn_items ? { native_turn_items:
|
|
212
|
+
...(child.native_turn_items ?? previous.native_turn_items ? { native_turn_items: { ...previous.native_turn_items, ...child.native_turn_items,
|
|
213
|
+
...([previous.native_turn_items, child.native_turn_items].some(items => items?.pending === true) ? { pending: true } : {}) } } : {}) });
|
|
108
214
|
continue;
|
|
109
215
|
}
|
|
110
|
-
if (!previous ||
|
|
216
|
+
if (!previous || previous.status === "unknown" || previous.status === "settled_by_root" && terminal(child.status)) unique.set(child.session_id, child);
|
|
111
217
|
}
|
|
112
218
|
return [...unique.values()].sort((left, right) => left.session_id.localeCompare(right.session_id));
|
|
113
219
|
}
|
|
@@ -37,7 +37,7 @@ export async function writeNativeHookFailure(stateDir, binding, error, budget) {
|
|
|
37
37
|
if (!binding?.request_id || !/^[A-Za-z0-9-]{1,128}$/.test(binding.request_id)) return;
|
|
38
38
|
await budget.run(() => writeJsonAtomic(path.join(stateDir, "hook-failures", `${binding.request_id}.json`), {
|
|
39
39
|
...errorRecord(error), request_id: binding.request_id, observed_at: new Date().toISOString(), native_hook_binding: binding,
|
|
40
|
-
}), "failure_receipt");
|
|
40
|
+
}, () => budget.remaining("failure_receipt_commit")), "failure_receipt");
|
|
41
41
|
}
|
|
42
42
|
|
|
43
43
|
/** Read only requests accepted for this exact live operation, never scan a mailbox. */
|
|
@@ -74,7 +74,7 @@ export function hookBudget(timeoutMs = 25_000, deadlineAt = process.env.DD_FLOW_
|
|
|
74
74
|
return { deadlineAt: Math.floor(Date.now() + duration), remaining, async run(action, phase) {
|
|
75
75
|
const timeoutMs = remaining(phase);
|
|
76
76
|
let timer;
|
|
77
|
-
try { return await Promise.race([Promise.resolve().then(action), new Promise((_, reject) => { timer = setTimeout(() => reject(Object.assign(new Error(`Native hook deadline exceeded during ${phase}`), { code: "native_hook_timeout", details: { phase, effect: "unknown" } })), timeoutMs); })]); }
|
|
77
|
+
try { return await Promise.race([Promise.resolve().then(() => { remaining(phase); return action(); }), new Promise((_, reject) => { timer = setTimeout(() => reject(Object.assign(new Error(`Native hook deadline exceeded during ${phase}`), { code: "native_hook_timeout", details: { phase, effect: "unknown" } })), timeoutMs); })]); }
|
|
78
78
|
finally { clearTimeout(timer); }
|
|
79
79
|
} };
|
|
80
80
|
}
|
|
@@ -241,8 +241,8 @@ function hookDiagnostics(stderr) {
|
|
|
241
241
|
}).slice(-16);
|
|
242
242
|
}
|
|
243
243
|
|
|
244
|
-
export async function invokeNativeHook({ bin, args, nodeBin, payload, home, cwd, env, timeoutMs =
|
|
245
|
-
const output = await runBoundedCommand({ bin, args, nodeBin, input: `${JSON.stringify(payload)}\n`, cwd, env: { ...env, DD_FLOW_HOME: home }, timeoutMs });
|
|
244
|
+
export async function invokeNativeHook({ bin, args, nodeBin, payload, home, cwd, env, timeoutMs = 25_000 }) {
|
|
245
|
+
const output = await runBoundedCommand({ bin, args, nodeBin, input: `${JSON.stringify(payload)}\n`, cwd, env: { ...env, DD_FLOW_HOME: home }, timeoutMs, cleanupWithinBudget: true });
|
|
246
246
|
let result;
|
|
247
247
|
try { result = JSON.parse(output); } catch { throw Object.assign(new Error("Invalid native hook JSON response"), { code: "native_hook_response_invalid" }); }
|
|
248
248
|
if (result?.ok === false && typeof result.error?.code === "string") throw Object.assign(new Error(result.error.message ?? "Native hook failed"), { code: result.error.code, details: result.error.details });
|
|
@@ -13,7 +13,7 @@ const fail = reason => { throw Object.assign(new Error('Native request proof doe
|
|
|
13
13
|
* the adapter, and recomputed by the adapter's shared request builder. */
|
|
14
14
|
export function nativeCommandFingerprint({ executable, args, cwd, env, operationId }) {
|
|
15
15
|
return hashJson({ executable, args, cwd, operation_id: operationId,
|
|
16
|
-
environment: Object.fromEntries(environmentKeys.map(key => [key, env[key] ?? null])) });
|
|
16
|
+
environment: Object.fromEntries([...environmentKeys, ...(env.PLAYWRIGHT_BROWSERS_PATH === undefined ? [] : ['PLAYWRIGHT_BROWSERS_PATH'])].map(key => [key, env[key] ?? null])) });
|
|
17
17
|
}
|
|
18
18
|
|
|
19
19
|
/** Called after the adapter's own argument preparation but before its RPC or
|
|
@@ -3,12 +3,12 @@ export function validateObservationDuration(value: number, label?: string): numb
|
|
|
3
3
|
export interface ObservationState {
|
|
4
4
|
policy: string; timeout_ms: number; started_at: number; cursor?: unknown;
|
|
5
5
|
observed_at: number | null; elapsed_ms: number; uncertainty_elapsed_ms: number;
|
|
6
|
-
gaps: number; observation_lost: boolean;
|
|
6
|
+
gaps: number; uncertain?: boolean; observation_lost: boolean;
|
|
7
7
|
}
|
|
8
8
|
export class ObservationClock {
|
|
9
9
|
constructor(options: { timeoutMs: number; gapMs?: number; uncertaintyMs?: number;
|
|
10
10
|
wall?: () => number; monotonic?: () => number; onGap?: (value: Record<string, unknown>) => void;
|
|
11
|
-
saved?: ObservationState | null });
|
|
11
|
+
saved?: ObservationState | null; snapshotIsLive?: boolean });
|
|
12
12
|
observationLost: boolean;
|
|
13
13
|
sample(progress?: unknown, progressAt?: number): boolean;
|
|
14
14
|
state(): ObservationState;
|
|
@@ -9,7 +9,7 @@ export function validateObservationDuration(value, label = "observation duration
|
|
|
9
9
|
* progress or proof of provider failure. Durable snapshots retain that episode. */
|
|
10
10
|
export class ObservationClock {
|
|
11
11
|
constructor({ timeoutMs, gapMs = 60_000, uncertaintyMs = 120_000, wall = Date.now,
|
|
12
|
-
monotonic = () => performance.now(), onGap = () => {}, saved = null }) {
|
|
12
|
+
monotonic = () => performance.now(), onGap = () => {}, saved = null, snapshotIsLive = false }) {
|
|
13
13
|
validateObservationDuration(timeoutMs); validateObservationDuration(gapMs, "gap duration");
|
|
14
14
|
validateObservationDuration(uncertaintyMs, "observation recovery duration");
|
|
15
15
|
Object.assign(this, { timeoutMs, gapMs, uncertaintyMs, wall, monotonic, onGap });
|
|
@@ -21,13 +21,16 @@ export class ObservationClock {
|
|
|
21
21
|
|| !Number.isFinite(saved.started_at) || !Number.isSafeInteger(saved.gaps) || saved.gaps < 0
|
|
22
22
|
|| saved.observed_at !== null && !Number.isFinite(saved.observed_at)
|
|
23
23
|
|| typeof saved.observation_lost !== "boolean"
|
|
24
|
+
|| saved.uncertain !== undefined && typeof saved.uncertain !== "boolean"
|
|
24
25
|
|| !Number.isFinite(saved.elapsed_ms) || saved.elapsed_ms < 0
|
|
25
26
|
|| !Number.isFinite(saved.uncertainty_elapsed_ms) || saved.uncertainty_elapsed_ms < 0)
|
|
26
27
|
throw new RangeError("invalid retained observation clock");
|
|
27
28
|
this.startedWall = saved.started_at; this.elapsed = saved.elapsed_ms;
|
|
28
29
|
this.progress = saved.cursor; this.progressAt = saved.observed_at;
|
|
29
|
-
//
|
|
30
|
-
|
|
30
|
+
// Durable reconstruction has an observation gap. Transferring a freshly
|
|
31
|
+
// observed producer clock is not a second restart of that same producer.
|
|
32
|
+
this.uncertain = snapshotIsLive ? saved.uncertain ?? true : true;
|
|
33
|
+
this.gaps = saved.gaps + (snapshotIsLive ? 0 : 1);
|
|
31
34
|
this.uncertaintyElapsed = saved.uncertainty_elapsed_ms;
|
|
32
35
|
this.observationLost = saved.observation_lost || this.gaps > 1 || this.uncertaintyElapsed >= uncertaintyMs;
|
|
33
36
|
}
|
|
@@ -77,7 +80,7 @@ export class ObservationClock {
|
|
|
77
80
|
return { policy: OBSERVATION_CLOCK_CONTRACT_VERSION, timeout_ms: this.timeoutMs,
|
|
78
81
|
started_at: this.startedWall, cursor: this.progress, observed_at: this.progressAt,
|
|
79
82
|
elapsed_ms: this.elapsed, uncertainty_elapsed_ms: this.uncertaintyElapsed,
|
|
80
|
-
gaps: this.gaps, observation_lost: this.observationLost };
|
|
83
|
+
gaps: this.gaps, uncertain: this.uncertain, observation_lost: this.observationLost };
|
|
81
84
|
}
|
|
82
85
|
}
|
|
83
86
|
|
|
@@ -33,16 +33,16 @@ export function makeOpenCodePlugin(binding, invoke = invokeNativeHook, resolve =
|
|
|
33
33
|
try {
|
|
34
34
|
const result = await invoke({ bin: binding.ddFlowBin, nodeBin: binding.nodeBin,
|
|
35
35
|
args: ["opencode", "event", "handle", "--project-root", binding.projectRoot, "--json"],
|
|
36
|
-
payload: { ...payload, native_hook_binding: identity.nativeHookBinding }, home: binding.ddFlowHome, timeoutMs: budget.remaining("admission"
|
|
36
|
+
payload: { ...payload, native_hook_binding: identity.nativeHookBinding }, home: binding.ddFlowHome, timeoutMs: budget.remaining("admission") });
|
|
37
37
|
await budget.run(() => resolve(binding.stateDir, "hook.verify", { binding: identity.nativeHookBinding }, budget.remaining("generation")), "generation");
|
|
38
38
|
if (result?.permissionDecision === "deny" || result?.hookSpecificOutput?.permissionDecision === "deny") throw Object.assign(new Error("dd-flow denied native shell admission"), { code: "native_hook_rejected" });
|
|
39
39
|
return result;
|
|
40
40
|
} catch (error) {
|
|
41
41
|
error.details = { ...error.details, native_hook_binding: identity.nativeHookBinding, ...(payload.phase === "after" ? { effect: "unknown", tool_outcome: payload.outcome } : {}) };
|
|
42
|
-
try { await writeNativeHookFailure(binding.stateDir, identity.nativeHookBinding, error,
|
|
42
|
+
try { await writeNativeHookFailure(binding.stateDir, identity.nativeHookBinding, error, budget); await resolve(binding.stateDir, "hook.failed", {}, budget.remaining("failure_notification")); }
|
|
43
43
|
catch (storage) {
|
|
44
44
|
error.details.receipt_error = { code: storage.code ?? "hook_receipt_write_failed", message: storage.message };
|
|
45
|
-
try { await resolve(binding.stateDir, "hook.failed", { binding: identity.nativeHookBinding, failure: errorRecord(error) },
|
|
45
|
+
try { await resolve(binding.stateDir, "hook.failed", { binding: identity.nativeHookBinding, failure: errorRecord(error) }, budget.remaining("failure_notification")); }
|
|
46
46
|
catch (notify) { error.details.receipt_error.notification_error = notify.code ?? "hook_notification_failed"; }
|
|
47
47
|
}
|
|
48
48
|
throw error;
|
|
@@ -35,7 +35,7 @@ function boundedText(value, budget, limit = 16000) {
|
|
|
35
35
|
// Lifecycle controls are not optional diagnostics. Reserve them before bulk
|
|
36
36
|
// messages/details, including nested typed causes, without copying authority
|
|
37
37
|
// that was truncated into a different command or continuation receipt.
|
|
38
|
-
const operationalDetailKeys = ["phase", "effect", "recoverable", "retryable_no_effect", "handoff_required", "storage_retry", "repeated_without_progress", "cleanup_unconfirmed", "native_dispatch_started", "disposition", "cli_admission_barrier", "native_pre_cli_refusal", "issued_invocation_id", "invocation_id", "event_key", "refusal_event_key", "hook_event_id", "work_id", "hook_retry_signature", "rejection_signature", "retry_command", "repair_command", "continuation", "retry_instruction", "recovery_instruction", "authority_truncated"];
|
|
38
|
+
const operationalDetailKeys = ["phase", "effect", "recoverable", "retryable_no_effect", "handoff_required", "storage_retry", "repeated_without_progress", "cleanup_unconfirmed", "native_dispatch_started", "disposition", "cli_admission_barrier", "native_pre_cli_refusal", "issued_invocation_id", "invocation_id", "event_key", "refusal_event_key", "hook_event_id", "work_id", "hook_retry_signature", "rejection_signature", "retry_command", "repair_command", "continuation", "retry_instruction", "correction_instruction", "recovery_instruction", "authority_truncated"];
|
|
39
39
|
function operationalDetails(value, budget) {
|
|
40
40
|
if (!value || typeof value !== "object" || Array.isArray(value)) return null;
|
|
41
41
|
const entries = [];
|
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
export const CAPACITY_POLICY: "provider-overload-episode@1";
|
|
2
|
-
export const MAX_OVERLOAD_RETRIES:
|
|
2
|
+
export const MAX_OVERLOAD_RETRIES: 3;
|
|
3
3
|
export const MAX_RETRY_AFTER_MS: 30000;
|
|
4
4
|
export function classifyProviderError(native: unknown): string;
|
|
5
5
|
export function safeProviderError(native: unknown): Record<string, unknown>;
|
|
6
|
-
export type
|
|
6
|
+
export type RequestProof = { kind: "native_turn"; session_id: string; turn_id: string } | { kind: "stream_request"; daemon_instance: string; connection_id: string; operation_id: string; turn_generation: number; session_id: string; outbound_position: number; terminal_position: number };
|
|
7
|
+
export function streamRequestKey(proof: unknown): string | null;
|
|
8
|
+
export type CapacityRefusal = { policy: string; ordinal: number; session_id: string; turn_id: string; request_key?: string; request_proof?: RequestProof; first_observed_at: string; native_terminal_at?: string; native_terminal_basis?: string };
|
|
7
9
|
export function parseCapacityOperation(id: string): { root: string; ordinal: number };
|
|
8
|
-
export function normalizeProviderFailure(error: unknown, sessionId?: string, turnId?: string): null | { code: string; session_id: string; turn_id: string; native_error: Record<string, unknown>; observed_at: unknown; terminal_at: unknown; native_turn_items: unknown; primary_error: { details?: Record<string, unknown> } };
|
|
10
|
+
export function normalizeProviderFailure(error: unknown, sessionId?: string, turnId?: string): null | { code: string; session_id: string; turn_id: string; request_key: string; request_proof: RequestProof; native_error: Record<string, unknown>; observed_at: unknown; terminal_at: unknown; native_turn_items: unknown; primary_error: { details?: Record<string, unknown> } };
|
|
9
11
|
export function terminalProviderOverload(error: unknown, sessionId?: string, turnId?: string): boolean;
|
|
10
12
|
export function nativeItemsSettled(items: unknown): boolean;
|
|
11
13
|
export function capacityBackoff(error: unknown, ordinal: number, now?: number, random?: () => number): number;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/** Pure policy shared by the controller and pinned-engine EVAL consumers. */
|
|
2
2
|
export const CAPACITY_POLICY = "provider-overload-episode@1";
|
|
3
|
-
export const MAX_OVERLOAD_RETRIES =
|
|
3
|
+
export const MAX_OVERLOAD_RETRIES = 3;
|
|
4
4
|
export const MAX_RETRY_AFTER_MS = 30_000;
|
|
5
5
|
|
|
6
6
|
function policyError(code, message, details = {}) { return Object.assign(new Error(message), { code, details }); }
|
|
@@ -11,10 +11,10 @@ export function classifyProviderError(native) {
|
|
|
11
11
|
const code = [native.code, native.type, native.codexErrorInfo?.code ?? native.codexErrorInfo].filter(value => typeof value === "string").join(" ");
|
|
12
12
|
const message = String(native.message ?? "");
|
|
13
13
|
const text = `${code} ${message}`;
|
|
14
|
-
const status = Number(native.status ?? native.statusCode ?? native.status_code ?? native.http_status);
|
|
14
|
+
const status = Number(native.status ?? native.statusCode ?? native.status_code ?? native.http_status ?? /\(code (\d{3})\)/i.exec(message)?.[1]);
|
|
15
15
|
if (status === 402 || /quota|usageLimitExceeded|sessionBudgetExceeded|billing|payment.required|balance.exhausted|insufficient.credit|resource_exhausted.*limit/i.test(text)) return "provider_quota_exhausted";
|
|
16
16
|
if (status === 401 || status === 403 || /provider_auth_failed|invalid.api.key|authentication|unauthorized|eligib|forbidden|access.denied|rejected.this.account/i.test(text)) return "provider_auth_failed";
|
|
17
|
-
if (/provider_capacity_transient|serverOverloaded|model.?capacity|at capacity|server.overload|model.overload|server.?busy/i.test(text) || /(?:^|\s)overloaded_error(?:\s|$)/i.test(code) || status === 503 || status === 529) return "provider_capacity_transient";
|
|
17
|
+
if (/provider_capacity_transient|serverOverloaded|model.?capacity|at capacity|no capacity available for model|server.overload|model.overload|server.?busy/i.test(text) || /(?:^|\s)overloaded_error(?:\s|$)/i.test(code) || status === 503 || status === 529) return "provider_capacity_transient";
|
|
18
18
|
if (/rateLimitExceeded|rate.limit/i.test(text) && (native.retryAfter !== undefined || native.retry_after !== undefined || Object.keys(native.headers ?? {}).some(key => key.toLowerCase() === "retry-after"))) return "provider_rate_limited";
|
|
19
19
|
return status === 429 || /provider_limit_unknown|\b429\b|rateLimitExceeded|rate.limit/i.test(text) ? "provider_limit_unknown" : "provider_failed";
|
|
20
20
|
}
|
|
@@ -45,6 +45,15 @@ export function parseCapacityOperation(id) {
|
|
|
45
45
|
return { root: match[1], ordinal };
|
|
46
46
|
}
|
|
47
47
|
|
|
48
|
+
/** Local ordered-stream identity is not a native Turn ID. */
|
|
49
|
+
export function streamRequestKey(proof) {
|
|
50
|
+
if (proof?.kind !== "stream_request" || ![proof.daemon_instance, proof.connection_id, proof.operation_id, proof.session_id].every(value => typeof value === "string" && value)
|
|
51
|
+
|| !Number.isSafeInteger(proof.turn_generation) || proof.turn_generation < 1
|
|
52
|
+
|| !Number.isSafeInteger(proof.outbound_position) || proof.outbound_position < 1
|
|
53
|
+
|| !Number.isSafeInteger(proof.terminal_position) || proof.terminal_position <= proof.outbound_position) return null;
|
|
54
|
+
return JSON.stringify(["stream_request", proof.daemon_instance, proof.connection_id, proof.operation_id, proof.session_id, proof.turn_generation, proof.outbound_position, proof.terminal_position]);
|
|
55
|
+
}
|
|
56
|
+
|
|
48
57
|
export function normalizeProviderFailure(error, expectedSessionId, expectedTurnId) {
|
|
49
58
|
let current = error;
|
|
50
59
|
for (let depth = 0; depth < 4 && current && typeof current === "object"; depth++) {
|
|
@@ -52,14 +61,17 @@ export function normalizeProviderFailure(error, expectedSessionId, expectedTurnI
|
|
|
52
61
|
// refusal. Never classify transport loss or an active native retry here.
|
|
53
62
|
if (current.code === "provider_capacity_transient" || current.code === "provider_rate_limited") {
|
|
54
63
|
const details = current.details ?? {};
|
|
64
|
+
const streamKey = details.native_source === "agy.result" && details.turn_id === undefined && details.native_turn_id === undefined ? streamRequestKey(details.request_proof) : null;
|
|
65
|
+
const turnKey = streamKey ?? details.turn_id;
|
|
55
66
|
if (!["opencode.v2.session.execution", "agy.result", "acp.error_response", "droid.agent_turn_outcome", "grok.subagent.event"].includes(details.native_source) || details.terminal_status !== "failed"
|
|
56
67
|
|| details.will_retry === true || details.native_retry_active === true || details.provider_error?.willRetry === true
|
|
57
68
|
|| details.provider_error && !["provider_capacity_transient", "provider_rate_limited"].includes(classifyProviderError(details.provider_error))
|
|
58
69
|
|| typeof details.provider_session_id !== "string" || !details.provider_session_id
|
|
59
|
-
|| typeof
|
|
70
|
+
|| typeof turnKey !== "string" || !turnKey || (streamKey ? details.request_proof.session_id !== details.provider_session_id || details.native_turn_id !== undefined : details.native_turn_id !== details.turn_id)
|
|
60
71
|
|| expectedSessionId !== undefined && details.provider_session_id !== expectedSessionId
|
|
61
|
-
|| expectedTurnId !== undefined &&
|
|
62
|
-
return { code: "serverOverloaded", session_id: details.provider_session_id, turn_id:
|
|
72
|
+
|| expectedTurnId !== undefined && turnKey !== expectedTurnId) return null;
|
|
73
|
+
return { code: "serverOverloaded", session_id: details.provider_session_id, turn_id: turnKey, request_key: turnKey,
|
|
74
|
+
request_proof: streamKey ? details.request_proof : { kind: "native_turn", session_id: details.provider_session_id, turn_id: details.turn_id },
|
|
63
75
|
native_error: details.provider_error ?? { type: details.native_error_type, status: details.http_status }, observed_at: details.observed_at ?? null,
|
|
64
76
|
terminal_at: null, terminal_basis: null, native_turn_items: details.native_turn_items ?? null, primary_error: current };
|
|
65
77
|
}
|
|
@@ -70,7 +82,7 @@ export function normalizeProviderFailure(error, expectedSessionId, expectedTurnI
|
|
|
70
82
|
|| expectedSessionId !== undefined && details.provider_session_id !== expectedSessionId
|
|
71
83
|
|| typeof details.turn_id !== "string" || !details.turn_id || details.native_turn_id !== details.turn_id
|
|
72
84
|
|| expectedTurnId !== undefined && details.turn_id !== expectedTurnId || typeof code !== "string") return null;
|
|
73
|
-
return { code, session_id: details.provider_session_id, turn_id: details.turn_id, native_error: native,
|
|
85
|
+
return { code, session_id: details.provider_session_id, turn_id: details.turn_id, request_key: details.turn_id, request_proof: { kind: "native_turn", session_id: details.provider_session_id, turn_id: details.turn_id }, native_error: native,
|
|
74
86
|
observed_at: details.observed_at ?? null, terminal_at: details.terminal_at ?? null, terminal_basis: details.terminal_basis ?? null,
|
|
75
87
|
native_turn_items: details.native_turn_items ?? null, primary_error: current };
|
|
76
88
|
}
|
|
@@ -118,6 +130,7 @@ export function refusalObservation(error, ordinal, firstObservedAt) {
|
|
|
118
130
|
throw policyError("capacity_continuation_conflict", "Native refusal observation time is invalid or changed");
|
|
119
131
|
const native = failure.terminal_at && Date.parse(failure.terminal_at);
|
|
120
132
|
return { policy: CAPACITY_POLICY, ordinal, session_id: failure.session_id, turn_id: failure.turn_id,
|
|
133
|
+
request_key: failure.request_key, request_proof: failure.request_proof,
|
|
121
134
|
first_observed_at: firstObservedAt, ...(Number.isFinite(native) && typeof failure.terminal_basis === "string" && failure.terminal_basis ? { native_terminal_at: failure.terminal_at, native_terminal_basis: failure.terminal_basis } : {}) };
|
|
122
135
|
}
|
|
123
136
|
|
|
@@ -22,13 +22,15 @@ export function sha256(value) {
|
|
|
22
22
|
|
|
23
23
|
export function hashJson(value) { return sha256(canonicalJson(value)); }
|
|
24
24
|
|
|
25
|
-
export async function writeJsonAtomic(file, value) {
|
|
25
|
+
export async function writeJsonAtomic(file, value, beforeCommit = () => {}) {
|
|
26
|
+
beforeCommit();
|
|
26
27
|
await mkdir(path.dirname(file), { recursive: true });
|
|
27
28
|
const temporary = `${file}.${process.pid}.${randomUUID()}.tmp`;
|
|
28
29
|
try {
|
|
29
30
|
const payload = await open(temporary, "wx", 0o600);
|
|
30
31
|
try { await payload.writeFile(`${JSON.stringify(value, null, 2)}\n`); await payload.sync(); }
|
|
31
32
|
finally { await payload.close(); }
|
|
33
|
+
beforeCommit();
|
|
32
34
|
await rename(temporary, file);
|
|
33
35
|
const directory = await open(path.dirname(file), "r");
|
|
34
36
|
try { await directory.sync(); }
|
|
@@ -1,19 +1,21 @@
|
|
|
1
1
|
import { readFile } from 'node:fs/promises';
|
|
2
2
|
import path from 'node:path';
|
|
3
|
+
import { randomUUID } from 'node:crypto';
|
|
3
4
|
export const SETUP_STARTUP_MS = 120_000;
|
|
4
5
|
export const SETUP_QUIET_MS = 30_000;
|
|
5
6
|
export const setupPhases = new Set(['ingress_entered', 'imports_ready', 'probe_started', 'probe_completed', 'dependency_check_completed', 'transfer_progress', 'identity_digest_progress', 'identity_digest_completed', 'native_initialized', 'ready']);
|
|
6
7
|
export function setupReporter(operationId = process.env.DD_EVAL_OPERATION_ID) {
|
|
7
8
|
if (!operationId) return () => {};
|
|
8
|
-
let sequence = 0;
|
|
9
|
+
let sequence = 0; const producerId = randomUUID();
|
|
9
10
|
return (phase, step = null) => {
|
|
10
11
|
if (!setupPhases.has(phase)) throw new Error('Invalid setup phase');
|
|
11
|
-
process.stderr.write(JSON.stringify({ kind: 'setup_phase', operation_id: operationId, phase, step, sequence: ++sequence, observed_at: new Date().toISOString() }) + '\n');
|
|
12
|
+
process.stderr.write(JSON.stringify({ kind: 'setup_phase', operation_id: operationId, producer_id: producerId, phase, step, sequence: ++sequence, observed_at: new Date().toISOString() }) + '\n');
|
|
12
13
|
};
|
|
13
14
|
}
|
|
14
15
|
export const publishSetup = setupReporter();
|
|
15
16
|
export function isSetupPhase(value) {
|
|
16
17
|
return value?.kind === 'setup_phase' && typeof value.operation_id === 'string' && value.operation_id.length > 0 && setupPhases.has(value.phase)
|
|
18
|
+
&& (!Object.hasOwn(value, 'producer_id') || typeof value.producer_id === 'string' && value.producer_id.length > 0 && value.producer_id.length <= 128)
|
|
17
19
|
&& Number.isSafeInteger(value.sequence) && value.sequence > 0 && value.sequence <= 1_000_000
|
|
18
20
|
&& (value.step === null || typeof value.step === 'string' && value.step.length <= 4096 || Number.isSafeInteger(value.step) && value.step >= 0)
|
|
19
21
|
&& typeof value.observed_at === 'string' && Number.isFinite(Date.parse(value.observed_at))
|
|
@@ -36,13 +38,13 @@ export function startupObserver(stateDir, daemonId) {
|
|
|
36
38
|
};
|
|
37
39
|
}
|
|
38
40
|
export function setupObserver(operationId) {
|
|
39
|
-
const startedAt = Date.now();
|
|
41
|
+
const startedAt = Date.now(); const sequences = new Map(), markers = new Set();
|
|
40
42
|
return value => {
|
|
41
43
|
if (typeof operationId !== 'string' || markers.size >= 4096 || !isSetupPhase(value) || value.operation_id !== operationId
|
|
42
|
-
|| value.sequence <=
|
|
43
|
-
const marker = JSON.stringify([value.phase, value.step]);
|
|
44
|
+
|| value.sequence <= (sequences.get(value.producer_id ?? null) ?? 0) || Date.parse(value.observed_at) < startedAt || Date.parse(value.observed_at) > Date.now()) return false;
|
|
45
|
+
const marker = JSON.stringify([value.producer_id ?? null, value.phase, value.step]);
|
|
44
46
|
if (markers.has(marker)) return false;
|
|
45
|
-
markers.add(marker);
|
|
47
|
+
markers.add(marker); sequences.set(value.producer_id ?? null, value.sequence); return true;
|
|
46
48
|
};
|
|
47
49
|
}
|
|
48
50
|
export function observationDetails(clock, phase, pid) {
|
|
@@ -156,7 +156,7 @@ export function agyToolObservation(step, rootSessionId) {
|
|
|
156
156
|
const sessionId = step.conversation_id ?? rootSessionId;
|
|
157
157
|
return { session: identity("antigravity-cli", sessionId), parent_session: sessionId !== rootSessionId ? identity("antigravity-cli", rootSessionId) : null,
|
|
158
158
|
call_id: step.step_id ?? step.stepId ?? step.id ?? step.tool_call_id ?? step.toolCallId ?? (Number.isInteger(step.step_index) ? String(step.step_index) : null),
|
|
159
|
-
name: step.tool_name ?? step.tool_info?.name, ...(step.tool_info?.error ? { status: "failed" } : nativeStatus(step.state)) };
|
|
159
|
+
name: step.tool_name ?? step.tool_info?.name, ...(step.tool_info?.error ? { status: "failed" } : nativeStatus(step.state === "ACTIVE" ? "running" : step.state)) };
|
|
160
160
|
}
|
|
161
161
|
|
|
162
162
|
/** Native AGY JSONL records have planner tool invocations, not matching live
|