@deksden-com/dd-flow-cli 0.9.0-beta.16 → 0.9.0-beta.161

package/dist/harness-runtime/lib/provider-overload-policy.d.mts

@@ -0,0 +1,23 @@
+export const CAPACITY_POLICY: "provider-overload-episode@1";
+export const RECOVERY_POLICY: "provider-recovery-episode@1";
+export function classifyRecoveryError(native: unknown): string;
+export function recoveryRetryAfter(native: unknown, now?: number): number;
+export const normalizeRecoveryFailure: typeof normalizeProviderFailure;
+export const terminalProviderRecovery: typeof terminalProviderOverload;
+export const recoveryBackoff: typeof capacityBackoff;
+export const recoveryRefusal: typeof refusalObservation;
+export const recoveryExhausted: typeof overloadExhausted;
+export const MAX_OVERLOAD_RETRIES: 3;
+export const MAX_RETRY_AFTER_MS: 30000;
+export function classifyProviderError(native: unknown): string;
+export function safeProviderError(native: unknown): Record<string, unknown>;
+export type RequestProof = { kind: "native_turn"; session_id: string; turn_id: string } | { kind: "stream_request"; daemon_instance: string; connection_id: string; operation_id: string; turn_generation: number; session_id: string; outbound_position: number; terminal_position: number };
+export function streamRequestKey(proof: unknown): string | null;
+export type CapacityRefusal = { policy: string; ordinal: number; session_id: string; turn_id: string; category?: string; request_key?: string; request_proof?: RequestProof; first_observed_at: string; native_terminal_at?: string; native_terminal_basis?: string };
+export function parseCapacityOperation(id: string): { root: string; ordinal: number };
+export function normalizeProviderFailure(error: unknown, sessionId?: string, turnId?: string): null | { code: string; session_id: string; turn_id: string; request_key: string; request_proof: RequestProof; native_error: Record<string, unknown>; observed_at: unknown; terminal_at: unknown; native_turn_items: unknown; primary_error: { details?: Record<string, unknown> } };
+export function terminalProviderOverload(error: unknown, sessionId?: string, turnId?: string): boolean;
+export function nativeItemsSettled(items: unknown): boolean;
+export function capacityBackoff(error: unknown, ordinal: number, now?: number, random?: () => number): number;
+export function refusalObservation(error: unknown, ordinal: number, firstObservedAt: string): CapacityRefusal;
+export function overloadExhausted(refusals: CapacityRefusal[], maxRetries?: number): null | { policy: string; reason: string; attempts: number; refusals: CapacityRefusal[] };

package/dist/harness-runtime/lib/provider-overload-policy.mjs

@@ -0,0 +1,242 @@
+/** Pure policy shared by the controller and pinned-engine EVAL consumers. */
+export const CAPACITY_POLICY = "provider-overload-episode@1";
+export const MAX_OVERLOAD_RETRIES = 3;
+export const MAX_RETRY_AFTER_MS = 30_000;
+export const RECOVERY_POLICY = "provider-recovery-episode@1";
+
+function policyError(code, message, details = {}) { return Object.assign(new Error(message), { code, details }); }
+
+/** Only call on a native error envelope, never prompt or tool output. */
+export function classifyProviderError(native) {
+  if (!native || typeof native !== "object") return "provider_failed";
+  const code = [native.code, native.type, native.codexErrorInfo?.code ?? native.codexErrorInfo].filter(value => typeof value === "string").join(" ");
+  const message = String(native.message ?? "");
+  const text = `${code} ${message}`;
+  const status = Number(native.status ?? native.statusCode ?? native.status_code ?? native.http_status ?? /\(code (\d{3})\)/i.exec(message)?.[1]);
+  if (status === 402 || /quota|usageLimitExceeded|sessionBudgetExceeded|billing|payment.required|balance.exhausted|insufficient.credit|resource_exhausted.*limit/i.test(text)) return "provider_quota_exhausted";
+  if (status === 401 || status === 403 || /provider_auth_failed|invalid.api.key|authentication|unauthorized|eligib|forbidden|access.denied|rejected.this.account/i.test(text)) return "provider_auth_failed";
+  if (/provider_capacity_transient|serverOverloaded|model.?capacity|at capacity|no capacity available for model|server.overload|model.overload|server.?busy/i.test(text) || /(?:^|\s)overloaded_error(?:\s|$)/i.test(code) || status === 503 || status === 529) return "provider_capacity_transient";
+  if (/rateLimitExceeded|rate.limit/i.test(text) && (native.retryAfter !== undefined || native.retry_after !== undefined || Object.keys(native.headers ?? {}).some(key => key.toLowerCase() === "retry-after"))) return "provider_rate_limited";
+  return status === 429 || /provider_limit_unknown|\b429\b|rateLimitExceeded|rate.limit/i.test(text) ? "provider_limit_unknown" : "provider_failed";
+}
+
+/** Public retry evidence excludes response bodies, credentials and other headers. */
+export function safeProviderError(native) {
+  const result = { code: classifyProviderError(native) };
+  if (!native || typeof native !== "object") return result;
+  const status = native.status ?? native.statusCode ?? native.status_code ?? native.http_status;
+  if (Number.isInteger(Number(status))) result.status = Number(status);
+  const retryHeaders = Object.entries(native.headers ?? {}).filter(([key]) => key.toLowerCase() === "retry-after");
+  const header = retryHeaders[0]?.[1];
+  const retry = native.retryAfter ?? native.retry_after ?? header;
+  if (retry !== undefined && retry !== null) result.retryAfter = typeof retry === "string" && retry.length <= 128 || typeof retry === "number" && Number.isFinite(retry) ? retry : "invalid-retry-after";
+  if (native.retry_after !== undefined && native.retry_after !== null) result.retry_after = typeof native.retry_after === "string" && native.retry_after.length <= 128 || typeof native.retry_after === "number" && Number.isFinite(native.retry_after) ? native.retry_after : "invalid-retry-after";
+  if (header !== undefined && header !== null && (native.retryAfter !== undefined || native.retry_after !== undefined || retryHeaders.length > 1)) result.headers = Object.fromEntries(retryHeaders.map(([key, value]) => [key, typeof value === "string" && value.length <= 128 || typeof value === "number" && Number.isFinite(value) ? value : "invalid-retry-after"]));
+  if (typeof native.willRetry === "boolean") result.willRetry = native.willRetry;
+  for (const key of ["type", "source", "reason", "errorPhase", "exceptionKind", "transport", "retryable", "retryAfterMs"]) {
+    const value = native[key];
+    if (typeof value === "boolean" || typeof value === "number" && Number.isFinite(value) || typeof value === "string" && value.length <= 128) result[key] = value;
+  }
+  if (!result.type && /^(model_request_failed|ECONNRESET|EPIPE|ETIMEDOUT|EAI_AGAIN|UND_ERR_SOCKET|UND_ERR_CONNECT_TIMEOUT|UND_ERR_HEADERS_TIMEOUT|UND_ERR_BODY_TIMEOUT)$/.test(native.code ?? "")) result.type = native.code;
+  if (Object.hasOwn(native, "retryAfterMs") && native.retryAfterMs !== null) result.retryAfterMs = typeof native.retryAfterMs === "number" && Number.isFinite(native.retryAfterMs) || typeof native.retryAfterMs === "string" && native.retryAfterMs.length <= 128 ? native.retryAfterMs : "invalid-retry-after";
+  if (native.attribution && typeof native.attribution === "object") {
+    const attribution = {};
+    for (const key of ["source", "reason", "errorPhase", "exceptionKind", "transport", "retryable", "retryAfterMs"]) {
+      const value = native.attribution[key];
+      if (typeof value === "boolean" || typeof value === "number" && Number.isFinite(value) || typeof value === "string" && value.length <= 128) attribution[key] = value;
+    }
+    if (Object.hasOwn(native.attribution, "retryAfterMs") && native.attribution.retryAfterMs !== null) attribution.retryAfterMs = typeof native.attribution.retryAfterMs === "number" && Number.isFinite(native.attribution.retryAfterMs) || typeof native.attribution.retryAfterMs === "string" && native.attribution.retryAfterMs.length <= 128 ? native.attribution.retryAfterMs : "invalid-retry-after";
+    if (Object.keys(attribution).length) result.attribution = attribution;
+  }
+  for (const key of ["reset_at", "resets_at", "resetAt", "resetsAt", "reset_time"]) {
+    const value = native[key];
+    if (typeof value === "string" && value.length <= 128 || typeof value === "number" && Number.isFinite(value)) result[key] = value;
+  }
+  return result;
+}
+
+/** New native-recovery ABI; the overload-only exports below retain their scope. */
+export function classifyRecoveryError(native) {
+  const legacy = classifyProviderError(native);
+  if (["provider_quota_exhausted", "provider_auth_failed"].includes(legacy)) return legacy;
+  if (!native || typeof native !== "object") return legacy;
+  const facts = [native, native.attribution].filter(value => value && typeof value === "object");
+  const text = [native.code, native.type, native.message, ...facts.flatMap(value => [value.reason, value.exceptionKind])].filter(value => typeof value === "string").join(" ");
+  if (facts.some(value => value.retryable === false) || /cancel|CERT_|certificate|permission|configuration/i.test(text)) return "provider_failed";
+  if (legacy === "provider_capacity_transient") return legacy;
+  if (["provider_rate_limited", "provider_limit_unknown"].includes(legacy) && facts.some(value => value.retryAfterMs !== undefined)) return "provider_rate_limited";
+  if (legacy === "provider_rate_limited") return legacy;
+  const structuredNetwork = [native.type, native.code].includes("model_request_failed") && facts.some(value => value.source === "network" && value.reason === "network_error" && value.errorPhase === "stream" && value.exceptionKind === "transport");
+  const networkSource = facts.some(value => ["network", "provider", "transport"].includes(value.source));
+  const code = [native.code, native.type, native.transport, ...facts.map(value => value.transport)].filter(value => typeof value === "string");
+  const knownCode = code.some(value => /^(ECONNRESET|EPIPE|ETIMEDOUT|EAI_AGAIN|UND_ERR_SOCKET|UND_ERR_CONNECT_TIMEOUT|UND_ERR_HEADERS_TIMEOUT|UND_ERR_BODY_TIMEOUT)$/.test(value));
+  const status = Number(native.status ?? native.statusCode ?? native.status_code ?? native.http_status);
+  return structuredNetwork || networkSource && (knownCode || [408, 502, 504].includes(status)) ? "provider_network_transient" : legacy;
+}
+
+/** All delay representations must agree; no units guessing and no moving deadline. */
+export function recoveryRetryAfter(native, now = Date.now()) {
+  if (!Number.isFinite(now)) throw policyError("capacity_retry_metadata_invalid", "Invalid recovery observation time");
+  if (!native || typeof native !== "object") return 0;
+  const values = [];
+  for (const source of [native, native.attribution].filter(value => value && typeof value === "object")) {
+    for (const key of ["retryAfterMs", "retryAfter", "retry_after"]) if (source[key] !== undefined && source[key] !== null) values.push([source[key], key === "retryAfterMs"]);
+    for (const [key, value] of Object.entries(source.headers ?? {})) if (key.toLowerCase() === "retry-after") values.push([value, false]);
+  }
+  const parsed = values.map(([value, milliseconds]) => {
+    const numeric = typeof value === "number" || typeof value === "string" && /^\d+(?:\.\d+)?$/.test(value.trim());
+    const duration = numeric ? Number(value) * (milliseconds ? 1 : 1000) : !milliseconds && typeof value === "string" && /^\w{3}, \d{2} \w{3} \d{4} \d{2}:\d{2}:\d{2} GMT$/.test(value.trim()) ? Math.max(0, Date.parse(value) - now) : NaN;
+    if (!numeric && Number.isFinite(duration) && new Date(Date.parse(value)).toUTCString() !== value.trim()) throw policyError("capacity_retry_metadata_invalid", "Invalid recovery HTTP date");
+    if (!Number.isFinite(duration) || duration < 0 || !Number.isFinite(now + duration) || now + duration > 8.64e15) throw policyError("capacity_retry_metadata_invalid", "Invalid recovery Retry-After metadata");
+    return duration;
+  });
+  if (parsed.some(value => value !== parsed[0])) throw policyError("capacity_retry_metadata_invalid", "Conflicting recovery Retry-After metadata");
+  return parsed[0] ?? 0;
+}
+
+export function normalizeRecoveryFailure(error, expectedSessionId, expectedTurnId) {
+  const normalized = normalizeNativeFailure(error, expectedSessionId, expectedTurnId, true);
+  if (!normalized) return null;
+  const category = classifyRecoveryError(normalized.native_error);
+  if (!["provider_capacity_transient", "provider_rate_limited", "provider_network_transient"].includes(category)) return null;
+  const nativeCode = typeof normalized.native_error.codexErrorInfo === "string" ? normalized.native_error.codexErrorInfo : normalized.native_error.codexErrorInfo?.code;
+  return { ...normalized, code: nativeCode ?? category, category };
+}
+
+export function terminalProviderRecovery(error, sessionId, turnId) { return normalizeRecoveryFailure(error, sessionId, turnId) !== null; }
+export function recoveryBackoff(error, ordinal, now = Date.now(), random = Math.random) {
+  const native = normalizeRecoveryFailure(error)?.native_error ?? error?.details?.provider_error ?? error;
+  try { return capacityBackoff({ retryAfter: recoveryRetryAfter(native, now) / 1000 }, ordinal, now, random); }
+  catch (failure) { if (failure.details?.policy === CAPACITY_POLICY) failure.details = { ...failure.details, policy: RECOVERY_POLICY }; throw failure; }
+}
+export function recoveryRefusal(error, ordinal, firstObservedAt) {
+  const failure = normalizeRecoveryFailure(error);
+  return { ...observedRefusal(failure, ordinal, firstObservedAt, RECOVERY_POLICY), category: failure.category };
+}
+export function recoveryExhausted(refusals, maxRetries = MAX_OVERLOAD_RETRIES) {
+  if (refusals.some(value => value.policy !== RECOVERY_POLICY)) throw policyError("capacity_policy_mismatch", "Retained episode uses another recovery policy");
+  const exhausted = overloadExhausted(refusals.map(value => ({ ...value, policy: CAPACITY_POLICY })), maxRetries);
+  return exhausted ? { ...exhausted, policy: RECOVERY_POLICY, reason: "provider_retries_exhausted", refusals } : null;
+}
+
+export function parseCapacityOperation(id) {
+  if (typeof id !== "string" || !id) throw policyError("capacity_continuation_conflict", "Missing capacity operation identity");
+  const match = /^(.*):capacity:(\d+):session\.prompt$/.exec(id);
+  if (!match) return { root: id, ordinal: 0 };
+  const ordinal = Number(match[2]);
+  if (!Number.isSafeInteger(ordinal) || ordinal < 1 || String(ordinal) !== match[2]) throw policyError("capacity_continuation_conflict", "Capacity ordinal is not a canonical positive safe integer", { operation_id: id });
+  return { root: match[1], ordinal };
+}
+
+/** Local ordered-stream identity is not a native Turn ID. */
+export function streamRequestKey(proof) {
+  if (proof?.kind !== "stream_request" || ![proof.daemon_instance, proof.connection_id, proof.operation_id, proof.session_id].every(value => typeof value === "string" && value)
+    || !Number.isSafeInteger(proof.turn_generation) || proof.turn_generation < 1
+    || !Number.isSafeInteger(proof.outbound_position) || proof.outbound_position < 1
+    || !Number.isSafeInteger(proof.terminal_position) || proof.terminal_position <= proof.outbound_position) return null;
+  return JSON.stringify(["stream_request", proof.daemon_instance, proof.connection_id, proof.operation_id, proof.session_id, proof.turn_generation, proof.outbound_position, proof.terminal_position]);
+}
+
+export function normalizeProviderFailure(error, expectedSessionId, expectedTurnId) {
+  return normalizeNativeFailure(error, expectedSessionId, expectedTurnId, false);
+}
+function normalizeNativeFailure(error, expectedSessionId, expectedTurnId, recovery) {
+  let current = error;
+  for (let depth = 0; depth < 4 && current && typeof current === "object"; depth++) {
+    // The same continuation authority applies to a proven v2 execution
+    // refusal. Never classify transport loss or an active native retry here.
+    if (current.code === "provider_capacity_transient" || current.code === "provider_rate_limited" || recovery && current.code === "provider_network_transient") {
+      const details = current.details ?? {};
+      const streamKey = details.native_source === "agy.result" && details.turn_id === undefined && details.native_turn_id === undefined ? streamRequestKey(details.request_proof) : null;
+      const turnKey = streamKey ?? details.turn_id;
+      const sources = ["opencode.v2.session.execution", "agy.result", "acp.error_response", "droid.agent_turn_outcome", "grok.subagent.event", ...(recovery ? ["zcode.retained-terminal"] : [])];
+      const categories = ["provider_capacity_transient", "provider_rate_limited", ...(recovery ? ["provider_network_transient"] : [])];
+      if (!sources.includes(details.native_source) || details.terminal_status !== "failed"
+        || details.will_retry === true || details.native_retry_active === true || details.provider_error?.willRetry === true
+        || details.provider_error && !categories.includes((recovery ? classifyRecoveryError : classifyProviderError)(details.provider_error))
+        || typeof details.provider_session_id !== "string" || !details.provider_session_id
+        || typeof turnKey !== "string" || !turnKey || (streamKey ? details.request_proof.session_id !== details.provider_session_id || details.native_turn_id !== undefined : details.native_turn_id !== details.turn_id)
+        || expectedSessionId !== undefined && details.provider_session_id !== expectedSessionId
+        || expectedTurnId !== undefined && turnKey !== expectedTurnId) return null;
+      return { code: "serverOverloaded", session_id: details.provider_session_id, turn_id: turnKey, request_key: turnKey,
+        request_proof: streamKey ? details.request_proof : { kind: "native_turn", session_id: details.provider_session_id, turn_id: details.turn_id },
+        native_error: details.provider_error ?? { type: details.native_error_type, status: details.http_status }, observed_at: details.observed_at ?? null,
+        terminal_at: null, terminal_basis: null, native_turn_items: details.native_turn_items ?? null, primary_error: current };
+    }
+    if (current.code === "turn_interrupted") {
+      const details = current.details ?? {}, native = details.provider_error;
+      const code = typeof native?.codexErrorInfo === "string" ? native.codexErrorInfo : native?.codexErrorInfo?.code;
+      if (native?.willRetry === true || details.terminal_status !== "failed" || typeof details.provider_session_id !== "string" || !details.provider_session_id
+        || expectedSessionId !== undefined && details.provider_session_id !== expectedSessionId
+        || typeof details.turn_id !== "string" || !details.turn_id || details.native_turn_id !== details.turn_id
+        || expectedTurnId !== undefined && details.turn_id !== expectedTurnId || typeof code !== "string") return null;
+      return { code, session_id: details.provider_session_id, turn_id: details.turn_id, request_key: details.turn_id, request_proof: { kind: "native_turn", session_id: details.provider_session_id, turn_id: details.turn_id }, native_error: native,
+        observed_at: details.observed_at ?? null, terminal_at: details.terminal_at ?? null, terminal_basis: details.terminal_basis ?? null,
+        native_turn_items: details.native_turn_items ?? null, primary_error: current };
+    }
+    // No arbitrary recursive cause search: an ownership/hook/lifecycle error wins.
+    if (current.code !== "harness_adapter_failed") return null;
+    current = current.details?.cause ?? current.cause;
+  }
+  return null;
+}
+
+export function terminalProviderOverload(error, expectedSessionId, expectedTurnId) {
+  const normalized = normalizeProviderFailure(error, expectedSessionId, expectedTurnId);
+  return normalized !== null && ["provider_capacity_transient", "provider_rate_limited"].includes(classifyProviderError(normalized.native_error));
+}
+
+export function nativeItemsSettled(items) {
+  return items?.observed === true && typeof items.possible_effects === "boolean" && items.pending !== true
+    && (items.possible_effects === false || items.pending === false);
+}
+
+export function capacityBackoff(error, ordinal, now = Date.now(), random = Math.random) {
+  if (!Number.isSafeInteger(ordinal) || ordinal < 1 || !Number.isFinite(now)) throw policyError("capacity_continuation_conflict", "Invalid capacity timing identity");
+  const native = normalizeProviderFailure(error)?.native_error ?? error?.details?.provider_error ?? error;
+  const header = Object.entries(native?.headers ?? {}).find(([key]) => key.toLowerCase() === "retry-after")?.[1];
+  const value = header ?? native?.retryAfter ?? native?.retry_after;
+  let retry = 0;
+  if (value !== undefined && value !== null) {
+    retry = typeof value === "number" || typeof value === "string" && /^\d+(\.\d+)?$/.test(value.trim()) ? Number(value) * 1000 : typeof value === "string" && /^\w{3}, \d{2} \w{3} \d{4} \d{2}:\d{2}:\d{2} GMT$/.test(value.trim()) ? Math.max(0, Date.parse(value) - now) : NaN;
+    if (!Number.isFinite(retry) || retry < 0 || !Number.isFinite(now + retry) || now + retry > 8.64e15)
+      throw policyError("capacity_retry_metadata_invalid", "Retry-After cannot be represented safely", { retry_after: value });
+  }
+  const jitter = random();
+  if (!Number.isFinite(jitter) || jitter < 0 || jitter >= 1) throw policyError("capacity_retry_metadata_invalid", "Invalid retry jitter");
+  const wait = Math.max((ordinal === 1 ? 5_000 : 15_000) + Math.floor(jitter * 251), retry);
+  if (wait > MAX_RETRY_AFTER_MS) throw policyError("provider_unavailable", "Provider requested a delay beyond the retry budget", { policy: CAPACITY_POLICY, reason: "retry_after_exceeds_budget", retry_after_ms: retry, suggested_retry_at: new Date(now + retry).toISOString() });
+  return wait;
+}
+
+export function refusalObservation(error, ordinal, firstObservedAt) {
+  const failure = normalizeProviderFailure(error);
+  return observedRefusal(failure, ordinal, firstObservedAt, CAPACITY_POLICY);
+}
+function observedRefusal(failure, ordinal, firstObservedAt, policy) {
+  const observed = Date.parse(firstObservedAt);
+  if (!failure || !Number.isSafeInteger(ordinal) || ordinal < 0 || !Number.isFinite(observed))
+    throw policyError("capacity_continuation_conflict", "Refusal lacks an immutable native identity/time");
+  if (failure.observed_at !== null && (typeof failure.observed_at !== "string" || !Number.isFinite(Date.parse(failure.observed_at)) || Date.parse(failure.observed_at) !== observed))
+    throw policyError("capacity_continuation_conflict", "Native refusal observation time is invalid or changed");
+  const native = failure.terminal_at && Date.parse(failure.terminal_at);
+  return { policy, ordinal, session_id: failure.session_id, turn_id: failure.turn_id,
+    request_key: failure.request_key, request_proof: failure.request_proof,
+    first_observed_at: firstObservedAt, ...(Number.isFinite(native) && typeof failure.terminal_basis === "string" && failure.terminal_basis ? { native_terminal_at: failure.terminal_at, native_terminal_basis: failure.terminal_basis } : {}) };
+}
+
+export function overloadExhausted(refusals, maxRetries = MAX_OVERLOAD_RETRIES) {
+  if (!Number.isSafeInteger(maxRetries) || maxRetries < 0 || maxRetries > MAX_OVERLOAD_RETRIES) throw policyError("capacity_continuation_conflict", "Invalid retry budget");
+  for (const [index, refusal] of refusals.entries()) {
+    const previous = refusals[index - 1];
+    if (typeof refusal.first_observed_at !== "string" || !Number.isFinite(Date.parse(refusal.first_observed_at))
+      || previous && Date.parse(refusal.first_observed_at) < Date.parse(previous.first_observed_at))
+      throw policyError("capacity_continuation_conflict", "Refusal observation clock reversed or is invalid");
+  }
+  for (const [index, refusal] of refusals.entries()) {
+    if (refusal.policy !== CAPACITY_POLICY) throw policyError("capacity_policy_mismatch", "Retained episode uses another retry policy");
+    const previous = refusals[index - 1];
+    if (refusal.ordinal !== index || previous && (refusal.session_id !== previous.session_id || refusals.slice(0, index).some(item => item.turn_id === refusal.turn_id))) throw policyError("capacity_continuation_conflict", "Refusal chain identity is inconsistent");
+  }
+  return refusals.length > maxRetries ? { policy: CAPACITY_POLICY, reason: "overload_retries_exhausted", attempts: refusals.length, refusals } : null;
+}

package/dist/harness-runtime/lib/runner-events.mjs

@@ -0,0 +1,202 @@
+import { createHash, randomUUID } from "node:crypto";
+import { mkdir, open, readFile, rename, unlink } from "node:fs/promises";
+import path from "node:path";
+import { withRunnerLock as withLock } from "./runner-lock.mjs";
+import { isObservationLoss, errorRecord } from "./operation-errors.mjs";
+import { operationContext } from "./operation-context.mjs";
+
+const appendQueues = new Map();
+
+export function canonicalJson(value) {
+  const normalize = (item) => {
+    if (Array.isArray(item)) return item.map(normalize);
+    if (item && typeof item === "object") return Object.fromEntries(Object.keys(item).sort().map((key) => [key, normalize(item[key])]));
+    return item;
+  };
+  return `${JSON.stringify(normalize(value))}\n`;
+}
+
+export function sha256(value) {
+  return createHash("sha256").update(value).digest("hex");
+}
+
+export function hashJson(value) { return sha256(canonicalJson(value)); }
+
+export async function writeJsonAtomic(file, value, beforeCommit = () => {}) {
+  beforeCommit();
+  await mkdir(path.dirname(file), { recursive: true });
+  const temporary = `${file}.${process.pid}.${randomUUID()}.tmp`;
+  try {
+    const payload = await open(temporary, "wx", 0o600);
+    try { await payload.writeFile(`${JSON.stringify(value, null, 2)}\n`); await payload.sync(); }
+    finally { await payload.close(); }
+    beforeCommit();
+    await rename(temporary, file);
+    const directory = await open(path.dirname(file), "r");
+    try { await directory.sync(); }
+    finally { await directory.close(); }
+  } finally { await unlink(temporary).catch(error => { if (error.code !== "ENOENT") throw error; }); }
+}
+
+export async function readJsonLines(file, label = "JSONL") {
+  try {
+    const source = await readFile(file, "utf8");
+    return source.split("\n").filter(Boolean).map((line, index) => {
+      try { return JSON.parse(line); }
+      catch { throw new Error(`invalid ${label} record at ${file}:${index + 1}`); }
+    });
+  } catch (error) {
+    if (error?.code === "ENOENT") return [];
+    throw error;
+  }
+}
+
+export async function readEvents(file) {
+  const records = await readJsonLines(file, "runner event");
+  return records.map((record, index) => {
+    try { return validateRunnerEvent(record); }
+    catch { throw new Error(`invalid runner event at ${file}:${index + 1}`); }
+  });
+}
+
+export function validateRunnerEvent(value) {
+  if (!value || typeof value !== "object" || Array.isArray(value) || value.specversion !== "1.0" || typeof value.id !== "string" || typeof value.source !== "string" || typeof value.type !== "string" || typeof value.time !== "string" || !value.data || typeof value.data !== "object" || !Number.isInteger(value.data.sequence) || value.data.sequence < 1) {
+    throw new Error("invalid normalized runner event");
+  }
+  return value;
+}
+
+export async function appendEvent(file, input) {
+  const prior = appendQueues.get(file) ?? Promise.resolve();
+  const current = prior.then(() => withLock(file, async () => {
+    const events = await readEvents(file);
+    // Internal callers can validate a dispatch or bind its generation under
+    // the same cross-process lock that orders cancellation and completion.
+    if (input.deduplicate && input.id) { const existing = events.find(event => event.id === input.id); if (existing) return existing; }
+    if (input.beforeAppend?.(events) === false) return null;
+    const sequence = events.length + 1;
+    const event = {
+      specversion: "1.0",
+      id: input.id ?? `EVT-${randomUUID()}`,
+      source: input.source,
+      type: input.type,
+      time: input.time ?? new Date().toISOString(),
+      subject: input.subject ?? "runner",
+      datacontenttype: "application/json",
+      runid: input.runId,
+      executionid: input.executionId,
+      traceid: input.traceId,
+      spanid: input.spanId ?? randomUUID(),
+      ...(input.parentSpanId ? { parentspanid: input.parentSpanId } : {}),
+      data: { sequence, ...input.data }
+    };
+    const context = operationContext.getStore();
+    if (context?.eventsFile === file && context.executionId === input.executionId && context.operationId && input.executionId) {
+      event.data.execution_operation_id ??= context.operationId;
+    }
+    validateRunnerEvent(event);
+    await mkdir(path.dirname(file), { recursive: true });
+    const handle = await open(file, "a");
+    try { await handle.write(`${JSON.stringify(event)}\n`); await handle.sync(); }
+    finally { await handle.close(); }
+    return event;
+  }));
+  const tail = current.catch(() => {}); appendQueues.set(file, tail);
+  try { return await current; }
+  finally { if (appendQueues.get(file) === tail) appendQueues.delete(file); }
+}
+
+export function reduceEvents(events) {
+  const operations = new Map();
+  let state = "planned";
+  for (const event of events) {
+    const data = event?.data ?? {};
+    if (["dev.dd.eval.planned", "dev.dd.eval.completed"].includes(event.type) && typeof data.state === "string") state = data.state;
+    if (typeof data.operation_id !== "string") continue;
+    const prior = operations.get(data.operation_id) ?? { id: data.operation_id, requested: false, started: false, terminal: null };
+    if (event.type.endsWith(".requested")) prior.requested = true;
+    if (event.type.endsWith(".started")) prior.started = true;
+    if (event.type.endsWith(".observation_lost")) prior.observation_lost = data.error ?? { code: "observation_lost" };
+    if (event.type.endsWith(".completed") || event.type.endsWith(".failed") || event.type.endsWith(".cancelled")) {
+      const terminal = event.type.split(".").at(-1);
+      const result = terminal === "completed" ? data.result : { error: data.error ?? null };
+      const resultHash = hashJson({ terminal, result });
+      if (prior.terminal && (prior.terminal !== terminal || prior.result_hash !== resultHash)) {
+        throw Object.assign(new Error(`runner operation ${data.operation_id} has conflicting terminal events`), { code: "journal_conflict", operation_id: data.operation_id });
+      }
+      prior.terminal = terminal;
+      prior.result = result;
+      prior.result_hash = resultHash;
+    }
+    operations.set(data.operation_id, prior);
+  }
+  return { state, operations: Object.fromEntries([...operations].map(([id, value]) => [id, value])) };
+}
+
+export async function recordOperation({ eventsFile, source, runId, executionId, traceId, operationId, operation, subject = "runner", action }) {
+  const existing = await withLock(`${eventsFile}.${operationId.replace(/[^a-zA-Z0-9._-]/g, "_")}`, async () => {
+    const current = reduceEvents(await readEvents(eventsFile)).operations[operationId];
+    if (current?.terminal === "completed") return current;
+    if (current?.terminal) throw Object.assign(new Error(`operation ${operationId} is already ${current.terminal}; use explicit recovery instead of replaying it`), { code: "operation_terminal", operation_id: operationId, terminal: current.terminal });
+    if (current?.started && !current.terminal) {
+      const error = new Error(current.observation_lost
+        ? `operation ${operationId} has no confirmed outcome; reconcile its provider state before retrying`
+        : `operation ${operationId} is already in progress; reconcile before retrying`);
+      error.code = current.observation_lost ? "operation_observation_lost" : "operation_in_progress";
+      error.operation_id = operationId;
+      throw error;
+    }
+    await appendEvent(eventsFile, { source, runId, executionId, traceId, subject, type: "dev.dd.eval.operation.requested", data: { operation_id: operationId, operation, status: "requested" } });
+    await appendEvent(eventsFile, { source, runId, executionId, traceId, subject, type: "dev.dd.eval.operation.started", data: { operation_id: operationId, operation, status: "started" } });
+    return null;
+  });
+  if (existing) return { reused: true, operation_id: operationId, result: existing.result };
+  try {
+    const result = await operationContext.run({ operationId, eventsFile, runId, executionId }, action);
+    await completeOperation({ eventsFile, source, runId, executionId, traceId, subject, operationId, operation, result });
+    return { reused: false, operation_id: operationId, result };
+  } catch (error) {
+    if (isObservationLoss(error)) await observationLost({ eventsFile, source, runId, executionId, traceId, subject, operationId, operation, error });
+    else await failOperation({ eventsFile, source, runId, executionId, traceId, subject, operationId, operation, error });
+    throw error;
+  }
+}
+
+export async function completeOperation({ eventsFile, source, runId, executionId, traceId, subject = "runner", operationId, operation, result }) {
+  return await settleOperation({ eventsFile, source, runId, executionId, traceId, subject, operationId, operation, terminal: "completed", result });
+}
+
+export async function failOperation({ eventsFile, source, runId, executionId, traceId, subject = "runner", operationId, operation, error }) {
+  return await settleOperation({ eventsFile, source, runId, executionId, traceId, subject, operationId, operation, terminal: "failed", result: { error: errorRecord(error) } });
+}
+
+/** A lost observer response is deliberately non-terminal: replay could duplicate a provider Turn. */
+export async function observationLost({ eventsFile, source, runId, executionId, traceId, subject = "runner", operationId, operation, error }) {
+  return await withLock(`${eventsFile}.${operationId.replace(/[^a-zA-Z0-9._-]/g, "_")}`, async () => {
+    const existing = reduceEvents(await readEvents(eventsFile)).operations[operationId];
+    if (existing?.terminal) return { reused: true, operation_id: operationId, result: existing.result };
+    if (!existing?.started) throw Object.assign(new Error(`operation ${operationId} has no started receipt`), { code: "operation_not_started" });
+    if (existing.observation_lost) return { reused: true, operation_id: operationId, observation_lost: existing.observation_lost };
+    const recorded = errorRecord(error);
+    await appendEvent(eventsFile, { source, runId, executionId, traceId, subject, type: "dev.dd.eval.operation.observation_lost", data: { operation_id: operationId, operation, status: "observation_lost", error: recorded } });
+    return { reused: false, operation_id: operationId, observation_lost: recorded };
+  });
+}
+
+async function settleOperation({ eventsFile, source, runId, executionId, traceId, subject, operationId, operation, terminal, result }) {
+  return await withLock(`${eventsFile}.${operationId.replace(/[^a-zA-Z0-9._-]/g, "_")}`, async () => {
+    const existing = reduceEvents(await readEvents(eventsFile)).operations[operationId];
+    const expectedHash = hashJson({ terminal, result });
+    if (existing?.terminal) {
+      if (existing.terminal === terminal && existing.result_hash === expectedHash) return { reused: true, operation_id: operationId, result: existing.result };
+      throw Object.assign(new Error(`runner operation ${operationId} has conflicting terminal result`), { code: "journal_conflict", operation_id: operationId });
+    }
+    if (!existing?.started) throw Object.assign(new Error(`operation ${operationId} has no started receipt`), { code: "operation_not_started" });
+    const type = terminal === "completed" ? "dev.dd.eval.operation.completed" : "dev.dd.eval.operation.failed";
+    const data = terminal === "completed"
+      ? { operation_id: operationId, operation, status: terminal, result }
+      : { operation_id: operationId, operation, status: terminal, error: result.error };
+    await appendEvent(eventsFile, { source, runId, executionId, traceId, subject, type, data });
+    return { reused: false, operation_id: operationId, result };
+  });
+}

package/dist/harness-runtime/lib/runner-lock.mjs

@@ -0,0 +1,42 @@
+import { randomUUID } from "node:crypto";
+import { mkdirSync, readdirSync, readFileSync, writeFileSync, unlinkSync, rmdirSync } from "node:fs";
+import path from "node:path";
+
+function release(lock, owner) {
+  try { unlinkSync(path.join(lock, owner)); } catch (error) { if (error.code === "ENOENT") return; throw error; }
+  // Never recursively remove a directory: another owner may already have acquired it.
+  try { rmdirSync(lock); } catch (error) { if (!["ENOENT", "ENOTEMPTY", "EEXIST"].includes(error.code)) throw error; }
+}
+
+function reclaimDeadOwner(lock) {
+  let entries;
+  try { entries = readdirSync(lock); } catch (error) { if (error.code === "ENOENT") return; throw error; }
+  if (entries.length !== 1 || !/^owner-.*\.json$/.test(entries[0])) return;
+  let owner;
+  try { owner = JSON.parse(readFileSync(path.join(lock, entries[0]), "utf8")); } catch { return; }
+  if (!Number.isInteger(owner.pid) || owner.pid <= 0) return;
+  try { process.kill(owner.pid, 0); }
+  catch (error) { if (error.code === "ESRCH") release(lock, entries[0]); }
+  // A live/reused PID or unknown owner is conservative: age never proves death.
+}
+
+export async function withRunnerLock(file, action, { timeoutMs = 30_000 } = {}) {
+  const lock = `${file}.lock`;
+  const owner = `owner-${process.pid}-${randomUUID()}.json`;
+  const deadline = performance.now() + timeoutMs;
+  for (;;) {
+    let acquired = false;
+    try { mkdirSync(lock); acquired = true; }
+    catch (error) { if (error.code !== "EEXIST") throw error; }
+    if (acquired) {
+      // No await between acquisition and identifying the owner.
+      try { writeFileSync(path.join(lock, owner), JSON.stringify({ pid: process.pid }), { flag: "wx" }); }
+      catch (error) { release(lock, owner); throw error; }
+      try { return await action(); }
+      finally { release(lock, owner); }
+    }
+    reclaimDeadOwner(lock);
+    if (performance.now() >= deadline) throw Object.assign(new Error(`runner lock has a live or unconfirmed owner: ${lock}; inspect its owner before recovery`), { code: "runner_lock_timeout" });
+    await new Promise((resolve) => setTimeout(resolve, 10));
+  }
+}

package/dist/harness-runtime/lib/session-settlement.mjs

@@ -0,0 +1,30 @@
+import { errorRecord } from "./operation-errors.mjs";
+
+/** Diagnostic failure never prevents cancellation of already owned Sessions.
+ * Only a fresh successful observation can establish settlement. */
+export async function waitForSettlement({ observe, cancel, knownSessions = () => [], timeoutMs = 5_000 }) {
+  const deadline = performance.now() + timeoutMs;
+  const cancelled = new Set(), errors = [];
+  if (cancel) for (const id of knownSessions()) {
+    cancelled.add(id);
+    try { await cancel(id); } catch (error) { errors.push({ phase: "cancel", session_id: id, ...errorRecord(error) }); }
+  }
+  for (;;) {
+    let state;
+    try { state = await observe(); }
+    catch (error) {
+      if (!cancel) throw error;
+      errors.push({ phase: "observe", ...errorRecord(error) });
+      state = { sessions: knownSessions(), active: true, observation_error: errorRecord(error) };
+    }
+    if (!state.active && state.sessions.length === 0) return { ...state, cleanup_errors: errors };
+    // The observation budget must not skip a newly discovered owned target.
+    // Attempt its cancellation once, then leave physical settlement pending.
+    for (const id of state.sessions) if (cancel && !cancelled.has(id)) {
+      cancelled.add(id);
+      try { await cancel(id); } catch (error) { errors.push({ phase: "cancel", session_id: id, ...errorRecord(error) }); }
+    }
+    if (!cancel || performance.now() >= deadline) throw Object.assign(new Error("daemon still owns an unsettled Session tree"), { code: "tree_not_settled", details: { ...state, cleanup_errors: errors } });
+    await new Promise(resolve => setTimeout(resolve, 50));
+  }
+}

package/dist/harness-runtime/lib/setup-progress.d.mts

@@ -0,0 +1,9 @@
+export const SETUP_STARTUP_MS: number;
+export const SETUP_QUIET_MS: number;
+export const setupPhases: Set<string>;
+export function setupReporter(operationId?: string): (phase: string, step?: string | number | null) => void;
+export const publishSetup: ReturnType<typeof setupReporter>;
+export function isSetupPhase(value: unknown): boolean;
+export function startupObserver(stateDir: string, daemonId: string): () => Promise<number>;
+export function setupObserver(operationId: string): (value: unknown) => boolean;
+export function observationDetails(clock: import('./observation-clock.mjs').ObservationClock, phase: string | null, pid: number | undefined): Record<string, unknown>;

package/dist/harness-runtime/lib/setup-progress.mjs

@@ -0,0 +1,53 @@
+import { readFile } from 'node:fs/promises';
+import path from 'node:path';
+import { randomUUID } from 'node:crypto';
+export const SETUP_STARTUP_MS = 120_000;
+export const SETUP_QUIET_MS = 30_000;
+export const setupPhases = new Set(['ingress_entered', 'imports_ready', 'probe_started', 'probe_completed', 'dependency_check_completed', 'transfer_progress', 'identity_digest_progress', 'identity_digest_completed', 'native_initialized', 'ready']);
+export function setupReporter(operationId = process.env.DD_EVAL_OPERATION_ID) {
+  if (!operationId) return () => {};
+  let sequence = 0; const producerId = randomUUID();
+  return (phase, step = null) => {
+    if (!setupPhases.has(phase)) throw new Error('Invalid setup phase');
+    process.stderr.write(JSON.stringify({ kind: 'setup_phase', operation_id: operationId, producer_id: producerId, phase, step, sequence: ++sequence, observed_at: new Date().toISOString() }) + '\n');
+  };
+}
+export const publishSetup = setupReporter();
+export function isSetupPhase(value) {
+  return value?.kind === 'setup_phase' && typeof value.operation_id === 'string' && value.operation_id.length > 0 && setupPhases.has(value.phase)
+    && (!Object.hasOwn(value, 'producer_id') || typeof value.producer_id === 'string' && value.producer_id.length > 0 && value.producer_id.length <= 128)
+    && Number.isSafeInteger(value.sequence) && value.sequence > 0 && value.sequence <= 1_000_000
+    && (value.step === null || typeof value.step === 'string' && value.step.length <= 4096 || Number.isSafeInteger(value.step) && value.step >= 0)
+    && typeof value.observed_at === 'string' && Number.isFinite(Date.parse(value.observed_at))
+    && !Object.hasOwn(value, 'error') && !Object.hasOwn(value, 'code');
+}
+export function startupObserver(stateDir, daemonId) {
+  const seen = new Set(); let sequence = 0, binding;
+  return async () => {
+    let state;
+    try { state = JSON.parse(await readFile(path.join(stateDir, 'daemon.json'), 'utf8')); }
+    catch (error) { if (error.code === 'ENOENT') return sequence; throw error; }
+    if (state.daemon_id !== daemonId) throw Object.assign(new Error('Startup daemon identity changed'), { code: 'operation_binding_conflict' });
+    const owner = state.config?.runtime_owner;
+    const identity = JSON.stringify([state.started_at ?? null, owner?.owner_id ?? null, owner?.operation_id ?? null, state.generation ?? null]);
+    binding ??= identity;
+    if (identity !== binding) throw Object.assign(new Error('Startup ownership binding changed'), { code: 'operation_binding_conflict' });
+    const marker = JSON.stringify([state.pid ?? null, state.provider_pid ?? null, state.shutdown_state ?? null, state.startup_phase ?? null]);
+    if (!seen.has(marker)) { seen.add(marker); sequence++; publishSetup(state.shutdown_state === 'running' ? 'native_initialized' : 'probe_started', `${daemonId}:${sequence}`); }
+    return sequence;
+  };
+}
+export function setupObserver(operationId) {
+  const startedAt = Date.now(); const sequences = new Map(), markers = new Set();
+  return value => {
+    if (typeof operationId !== 'string' || markers.size >= 4096 || !isSetupPhase(value) || value.operation_id !== operationId
+      || value.sequence <= (sequences.get(value.producer_id ?? null) ?? 0) || Date.parse(value.observed_at) < startedAt || Date.parse(value.observed_at) > Date.now()) return false;
+    const marker = JSON.stringify([value.producer_id ?? null, value.phase, value.step]);
+    if (markers.has(marker)) return false;
+    markers.add(marker); sequences.set(value.producer_id ?? null, value.sequence); return true;
+  };
+}
+export function observationDetails(clock, phase, pid) {
+  return { reason: clock.observationLost ? 'observation_gap' : 'inactivity', phase, pid,
+    timeout_ms: clock.timeoutMs, observation_lost: clock.observationLost, observation_clock: clock.state(), ...(clock.setupProgress ? { last_setup: clock.setupProgress } : {}), effect: 'unknown' };
+}