chati-dev 4.5.16 → 4.5.28

package/src/terminal/run-rail-rework.js

@@ -0,0 +1,325 @@
+#!/usr/bin/env node
+
+import { randomUUID } from 'node:crypto';
+import { fileURLToPath } from 'node:url';
+import { resolve } from 'node:path';
+import {
+  buildSpawnCommand, getTerminalCompletionAuthorityReceipt,
+  getTerminalLaunchAuthorityReceipt, isTransientFailure, killTerminal, spawnTerminal, waitForTerminal,
+} from './spawner.js';
+import { checkProviderReadiness } from './provider-preflight.js';
+import { getRateLimiter } from './rate-limiter.js';
+import { startClaimHeartbeat } from './run-rail-task.js';
+import {
+  acquireRailTaskSupervisor, listRailExecutionEvidence, recordRailExecutionEvidence,
+  recordRailExecutionDispatch, releaseRailReworkReservation, relinquishRailTask,
+  reserveRailRework, reserveRailReworkLaunch, resolveRailAttemptContext,
+} from '../orchestrator/rail-runtime.js';
+import { parseRailExecutionEnvelope } from '../orchestrator/rail-execution-evidence.js';
+import { listRailReviewEvidence } from '../orchestrator/review-runtime.js';
+import { buildRailReworkPrompt } from './rail-prompts.js';
+import { sha256 } from '@chati/core';
+import {
+  loadRuntimeInstallationV2, resolveRuntimeExecutableBindingV2,
+} from '../orchestrator/runtime-installation-v2.js';
+
+export { buildRailReworkPrompt } from './rail-prompts.js';
+
+function fail(code, message, details = {}) { const error = new Error(message); error.code = code; error.details = details; throw error; }
+
+function terminalEvidence(handle, runNumber) {
+  const stdout = (handle.stdout || []).join('');
+  const stderr = (handle.stderr || []).join('');
+  if (Buffer.byteLength(stdout) > 1_000_000 || Buffer.byteLength(stderr) > 100_000) {
+    fail('RAIL_EXECUTION_OUTPUT_TOO_LARGE', 'RAIL terminal output exceeds the journal-bound evidence limit');
+  }
+  return Object.freeze({
+    terminal_id: handle.id,
+    provider: handle.provider,
+    model: handle.model,
+    reasoning_configuration: handle.reasoningConfiguration,
+    exit_status: handle.exitCode,
+    retry_count: handle.retryCount || 0,
+    run_number: runNumber,
+    started_at: handle.startedAt,
+    stdout,
+    stderr,
+    terminal_authority_receipt: getTerminalCompletionAuthorityReceipt(handle),
+  });
+}
+
+function latestReworkAuthorization(context) {
+  const decisions = context.records.filter((record) => ['review_decided', 'adjudication_decided'].includes(record.type)
+    && record.payload?.attempt_id === context.attempt.attempt_id);
+  const latest = decisions.at(-1);
+  if (latest?.type === 'review_decided' && latest.payload?.decision?.decision === 'normal-rework') {
+    return Object.freeze({ source_decision_ref: latest.payload.review_ref, review_ref: latest.payload.review_ref });
+  }
+  if (latest?.type === 'adjudication_decided' && latest.payload?.decision?.outcome === 'bounded-correction') {
+    return Object.freeze({
+      source_decision_ref: latest.payload.adjudication_ref,
+      review_ref: latest.payload.review_ref,
+      source_adjudication: Object.freeze({
+        adjudication_ref: latest.payload.adjudication_ref,
+        review_ref: latest.payload.review_ref,
+        decision: latest.payload.decision,
+        evidence: latest.payload.evidence,
+      }),
+    });
+  }
+  return fail('RAIL_REWORK_NOT_AUTHORIZED', 'the latest canonical decision does not authorize bounded rework');
+}
+
+export async function executeRailRework({
+  projectDir,
+  handoff_id,
+  attempt_id,
+  timeout = 30 * 60 * 1000,
+  max_retries = 2,
+  lease_ttl_ms = 5 * 60 * 1000,
+} = {}) {
+  if (!projectDir || !handoff_id || !attempt_id) fail('RAIL_REWORK_INPUT_REQUIRED', 'projectDir, handoff_id and attempt_id are required');
+  let context = resolveRailAttemptContext({ projectDir, handoff_id, attempt_id });
+  if (context.state !== 'active') fail('RAIL_REWORK_ATTEMPT_INACTIVE', `attempt is ${context.state}`);
+  const authorization = latestReworkAuthorization(context);
+  const reviewEntry = listRailReviewEvidence({ projectDir, attempt_id })
+    .find((entry) => `rail-review://${handoff_id}/${attempt_id}/${entry.review?.review_id}` === authorization.review_ref);
+  if (!reviewEntry) fail('RAIL_REWORK_REVIEW_EVIDENCE_MISSING', 'the authorizing review evidence is unavailable or does not match the journal');
+  const executionHistory = listRailExecutionEvidence({ projectDir, attempt_id });
+  if (executionHistory.length === 0) fail('RAIL_EXECUTION_EVIDENCE_REQUIRED', 'rework requires prior immutable execution evidence');
+  if (executionHistory.some((entry) => typeof entry.result_commit_ref === 'string'
+    && entry.source_decision_ref === authorization.source_decision_ref)) {
+    fail('RAIL_REWORK_ALREADY_EXECUTED', 'this decision already has a successful rework execution');
+  }
+
+  const originalInvocation = executionHistory[0].invocation;
+  const binding = context.task.execution_binding;
+  const providerExecutableBinding = resolveRuntimeExecutableBindingV2({
+    artifact: loadRuntimeInstallationV2(projectDir),
+    provider_id: binding.provider_id,
+    harness_id: binding.harness_id,
+  });
+  if (originalInvocation.provider_id !== binding.provider_id
+    || originalInvocation.harness_id !== binding.harness_id
+    || originalInvocation.model_pin?.model_id !== binding.model_id
+    || originalInvocation.model_pin?.reasoning_configuration !== binding.reasoning_configuration) {
+    fail('RAIL_REWORK_BINDING_MISMATCH', 'original execution evidence does not match the sealed task binding');
+  }
+  const readiness = checkProviderReadiness(binding.harness_id, {
+    workingDir: projectDir, executableBinding: providerExecutableBinding,
+  });
+  if (!readiness.ready) fail(readiness.code || 'PROVIDER_NOT_READY', readiness.detail || `${binding.harness_id} is not ready`);
+  const base_commit_ref = executionHistory.filter((entry) => typeof entry.result_commit_ref === 'string').at(-1)?.result_commit_ref;
+  if (!base_commit_ref) fail('RAIL_REWORK_BASE_COMMIT_REQUIRED', 'latest immutable execution evidence has no result commit');
+  const ownership = acquireRailTaskSupervisor({ projectDir, handoff_id, attempt_id, lease_ttl_ms });
+  context = ownership.context;
+  const renewed = ownership.renewed;
+  const reservation_id = `rework-reservation-${randomUUID()}`;
+  let reservationAcquired = false;
+  let reservationReleased = false;
+  let dispatchAwaitingEvidence = false;
+  let reworkSucceeded = false;
+  let invocation;
+  let handle;
+  let heartbeat;
+  try {
+  reserveRailRework({
+    projectDir, handoff_id, attempt_id,
+    source_decision_ref: authorization.source_decision_ref,
+    reservation_id,
+  });
+  reservationAcquired = true;
+  heartbeat = startClaimHeartbeat({
+    projectDir, handoff_id, attempt_id, claim_id: context.claim.claim_id,
+    operator_id: ownership.operator_id, lease_ttl_ms, initial_lease: renewed.lease,
+    activeHandle: () => handle,
+  });
+  await getRateLimiter(binding.harness_id).waitForSlot();
+  if (heartbeat.evidence().failure) fail('RAIL_LEASE_HEARTBEAT_FAILED', 'rework lost its RAIL claim heartbeat');
+  const launch = async () => {
+    const promptExecutionHistory = listRailExecutionEvidence({ projectDir, attempt_id });
+    const invocationDraft = {
+      ...originalInvocation,
+      invocation_id: `rail-rework-${attempt_id}-${executionHistory.length + 1}-${randomUUID()}`,
+      attempt_id,
+      launch_nonce: randomUUID(),
+      prompt_kind: 'rework',
+      source_decision_ref: authorization.source_decision_ref,
+      rework_reservation_id: reservation_id,
+      language: context.language,
+    };
+    const prompt = buildRailReworkPrompt({
+      projectDir, context, sourceReview: reviewEntry.review, executionHistory: promptExecutionHistory,
+      sourceAdjudication: authorization.source_adjudication,
+      baseCommitRef: base_commit_ref, invocation: invocationDraft,
+    });
+    invocation = Object.freeze({ ...invocationDraft, prompt_digest: sha256(prompt) });
+    const config = {
+      agent: 'rail-rework', taskId: context.task.task_id,
+      provider: binding.harness_id, providerId: binding.provider_id,
+      model: binding.model_id, reasoningConfiguration: binding.reasoning_configuration,
+      catalogSnapshotRef: invocation.model_pin.catalog_snapshot_ref,
+      providerExecutableBinding,
+      prompt, workingDir: projectDir, timeout, writeScope: ['.'],
+      executionBaseCommitRef: base_commit_ref,
+      authorityContext: {
+        authority_kind: 'rail_execution', invocation_id: invocation.invocation_id,
+        launch_nonce: invocation.launch_nonce, provider_id: invocation.provider_id,
+      },
+      beforeAuthoritySpawn: () => {
+        reserveRailReworkLaunch({
+          projectDir, handoff_id, attempt_id,
+          source_decision_ref: authorization.source_decision_ref,
+          reservation_id,
+          invocation,
+        });
+        // From the final pre-spawn durable launch intent onward, absence of
+        // canonical completion evidence remains unresolved after a crash.
+        dispatchAwaitingEvidence = true;
+      },
+    };
+    const built = buildSpawnCommand(config);
+    if (built.actualProvider !== config.provider || built.actualModel !== config.model
+      || built.actualReasoningConfiguration !== config.reasoningConfiguration) {
+      fail('RAIL_REWORK_ROUTE_MISMATCH', 'terminal adapter changed the sealed rework binding');
+    }
+    handle = spawnTerminal(config);
+    try {
+      recordRailExecutionDispatch({
+        projectDir, handoff_id, attempt_id, invocation,
+        launch_receipt: getTerminalLaunchAuthorityReceipt(handle),
+        prompt, run_kind: 'rework',
+      });
+    } catch (error) {
+      await killTerminal(handle);
+      throw error;
+    }
+    return handle;
+  };
+  await launch();
+  let retries = 0;
+  const failedAttemptRefs = [];
+  let persisted;
+  let execution;
+  const recordFailedAttempt = () => {
+    handle.retryCount = retries;
+    const failedExecution = terminalEvidence(handle, executionHistory.length + failedAttemptRefs.length + 1);
+    persisted = recordRailExecutionEvidence({
+      projectDir, handoff_id, task_id: context.task.task_id, attempt_id,
+      invocation, execution: failedExecution, base_commit_ref, result_commit_ref: null,
+      run_kind: 'rework', source_decision_ref: authorization.source_decision_ref,
+    });
+    execution = failedExecution;
+    dispatchAwaitingEvidence = false;
+    failedAttemptRefs.push(persisted.evidence_ref);
+  };
+  try {
+    await waitForTerminal(handle);
+    while (handle.exitCode !== 0 && retries < max_retries && isTransientFailure(handle.exitCode, [...handle.stdout, ...handle.stderr])) {
+      if (heartbeat.evidence().failure) break;
+      recordFailedAttempt();
+      retries += 1;
+      await new Promise((done) => setTimeout(done, 2_000 * (2 ** (retries - 1))));
+      if (heartbeat.evidence().failure) break;
+      await getRateLimiter(binding.harness_id).waitForSlot();
+      if (heartbeat.evidence().failure) break;
+      await launch();
+      await waitForTerminal(handle);
+    }
+  } finally { heartbeat.stop(); }
+  const heartbeatEvidence = heartbeat.evidence();
+  let result_commit_ref = null;
+  if (dispatchAwaitingEvidence) {
+    handle.retryCount = retries;
+    execution = terminalEvidence(handle, executionHistory.length + failedAttemptRefs.length + 1);
+    const invalidOutput = execution.terminal_authority_receipt.execution_result_status === 'invalid_output';
+    const envelope = execution.exit_status === 0 && !invalidOutput
+      ? parseRailExecutionEnvelope((handle.stdout || []).join(''))
+      : null;
+    if (envelope && envelope.launch_nonce !== invocation.launch_nonce) {
+      fail('RAIL_EXECUTION_LAUNCH_NONCE_MISMATCH', 'rework output does not echo the canonical launch nonce');
+    }
+    result_commit_ref = envelope?.result_commit_ref ?? null;
+    persisted = recordRailExecutionEvidence({
+      projectDir, handoff_id, task_id: context.task.task_id, attempt_id,
+      invocation, execution, base_commit_ref, result_commit_ref,
+      run_kind: 'rework', source_decision_ref: authorization.source_decision_ref,
+    });
+    dispatchAwaitingEvidence = false;
+  }
+  if (heartbeatEvidence.failure) fail('RAIL_LEASE_HEARTBEAT_FAILED', 'rework lost its RAIL claim heartbeat');
+  reworkSucceeded = persisted.outcome === 'succeeded';
+  if (!reworkSucceeded && execution.exit_status === 0) {
+    fail(
+      persisted.failure_code || 'RAIL_EXECUTION_OUTPUT_INVALID',
+      'RAIL rework completed without a valid canonical result commit',
+      { execution_evidence_ref: persisted.evidence_ref },
+    );
+  }
+  return Object.freeze({
+    ok: reworkSucceeded,
+    action: reworkSucceeded ? 'rail_rework_executed' : 'rail_rework_failed',
+    handoff_id, task_id: context.task.task_id, attempt_id,
+    source_decision_ref: authorization.source_decision_ref,
+    ...(authorization.source_decision_ref === authorization.review_ref ? { source_review_ref: authorization.review_ref } : {}),
+    invocation, execution, base_commit_ref, result_commit_ref,
+    execution_evidence_ref: persisted.evidence_ref,
+    failed_execution_evidence_refs: Object.freeze(failedAttemptRefs),
+    retry_count: retries,
+    lease_heartbeat_count: heartbeatEvidence.heartbeat_count,
+    next_required_action: reworkSucceeded ? 'independent_review' : 'rail_retry_same_rework',
+  });
+  } finally {
+    try {
+      heartbeat?.stop();
+      if (handle) await killTerminal(handle);
+      if (reservationAcquired && !dispatchAwaitingEvidence) {
+        releaseRailReworkReservation({
+          projectDir, handoff_id, attempt_id,
+          source_decision_ref: authorization.source_decision_ref,
+          reservation_id,
+          reason: reworkSucceeded ? 'canonical_success_recorded' : 'canonical_failure_recorded',
+        });
+        reservationReleased = true;
+      }
+    } finally {
+      if ((reservationAcquired && reservationReleased) || (!reservationAcquired && ownership.recovered)) {
+        relinquishRailTask({
+          projectDir, handoff_id, attempt_id,
+          claim_id: context.claim.claim_id,
+          operator_id: ownership.operator_id,
+          reason: 'action_complete',
+        });
+      }
+    }
+  }
+}
+
+function parseArgs(argv) {
+  const args = {};
+  for (let index = 2; index < argv.length; index += 1) {
+    const token = argv[index];
+    if (!token.startsWith('--')) continue;
+    const value = argv[index + 1];
+    if (value && !value.startsWith('--')) { args[token.slice(2)] = value; index += 1; } else args[token.slice(2)] = true;
+  }
+  return args;
+}
+
+async function main() {
+  const args = parseArgs(process.argv);
+  try {
+    const result = await executeRailRework({
+      projectDir: resolve(args['project-dir'] || process.cwd()),
+      handoff_id: args['handoff-id'], attempt_id: args['attempt-id'],
+      ...(args.timeout ? { timeout: Number(args.timeout) } : {}),
+    });
+    process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
+    if (!result.ok) process.exitCode = 1;
+  } catch (error) {
+    process.stdout.write(`${JSON.stringify({ ok: false, action: 'error', code: error.code || 'RAIL_REWORK_FAILED', error: error.message })}\n`);
+    process.exitCode = 1;
+  }
+}
+
+if (process.argv[1] && resolve(process.argv[1]) === resolve(fileURLToPath(import.meta.url))) await main();