archal 0.9.19 → 0.10.0

package/dist/commands/autoloop-worker.d.cts

@@ -0,0 +1,102 @@
+import { A as AutoloopWorkerJob, C as CodexAuth, E as E2BAutoloopWorkerOptions, b as AutoloopWorker } from '../autoloop-worker-types-CHaclqtD.cjs';
+export { c as AutoloopGithubInstallationTokenContext, d as AutoloopWorkerError, a as AutoloopWorkerResult, e as AzureCodexAuth, O as OpenAICodexAuth, i as isAutoloopWorkerError, p as preflightGithubAutoloopPermissions } from '../autoloop-worker-types-CHaclqtD.cjs';
+import '../layer-1-ziaLpfLr.cjs';
+
+declare const PHASES: readonly [{
+    readonly id: "01-grade" | "02-seed" | "03-reproduce" | "04-fix-pr";
+    readonly title: "Grade trace";
+    readonly envKind: "grade";
+    readonly buildPrompt: typeof buildGradePrompt;
+}, {
+    readonly id: "01-grade" | "02-seed" | "03-reproduce" | "04-fix-pr";
+    readonly title: "Create seed and scenario";
+    readonly envKind: "codex";
+    readonly buildPrompt: typeof buildSeedPrompt;
+}, {
+    readonly id: "01-grade" | "02-seed" | "03-reproduce" | "04-fix-pr";
+    readonly title: "Reproduce generated scenario";
+    readonly envKind: "codex";
+    readonly buildPrompt: typeof buildReproducePrompt;
+}, {
+    readonly id: "01-grade" | "02-seed" | "03-reproduce" | "04-fix-pr";
+    readonly title: "Fix and verify PR";
+    readonly envKind: "fix";
+    readonly buildPrompt: typeof buildFixPrPrompt;
+}];
+type AutoloopPhaseId = (typeof PHASES)[number]['id'];
+declare function buildCodexPrompt(job: AutoloopWorkerJob): string;
+declare function buildGradePrompt(job: AutoloopWorkerJob): string;
+declare function buildSeedPrompt(job: AutoloopWorkerJob): string;
+declare function buildReproducePrompt(job: AutoloopWorkerJob): string;
+declare function buildFixPrPrompt(job: AutoloopWorkerJob): string;
+
+declare function resolveNativeHarnessRuntimeEnv(repoDir: string, env?: NodeJS.ProcessEnv): Record<string, string>;
+declare function buildSandboxPhaseEnvs(input: {
+    traceId: string;
+    contentHash?: string;
+    repoRemote: string;
+    gitRepository?: string;
+    codexAuth?: CodexAuth;
+    archalEnv: Record<string, string>;
+    harnessModelEnv?: Record<string, string>;
+    nativeHarnessRuntimeEnv?: Record<string, string>;
+    githubInstallationToken?: string;
+    githubSourceBranch?: string;
+    githubPublishBranch?: string;
+    githubBaseBranch?: string;
+    ragReplayPackagePath?: string;
+    ragAnswerBenchmarkPath?: string;
+    ragRetrieverReplayPath?: string;
+    controlPlaneCloneFree?: boolean;
+    controlPlaneHarnessCommand?: string;
+    controlPlaneHarnessArgs?: string[];
+    controlPlaneJudgePrompt?: string;
+    operatorEnv?: NodeJS.ProcessEnv;
+}): Record<'setup' | (typeof PHASES)[number]['id'], Record<string, string>>;
+declare function resolveArchalSandboxPackage(_cliVersion?: string, env?: NodeJS.ProcessEnv): string;
+declare function resolveArchalSandboxPackageArchivePath(env?: NodeJS.ProcessEnv): string | undefined;
+declare function resolveCodexReasoningEffort(env: NodeJS.ProcessEnv): string;
+
+declare function buildSetupShellCommand(): string;
+declare function buildFixPrPhaseShellCommand(phaseId: AutoloopPhaseId): string;
+declare function buildGradePhaseShellCommand(_phaseId: AutoloopPhaseId): string;
+declare function buildSeedTemplateReproducePhaseShellCommand(phaseId: AutoloopPhaseId): string;
+declare function buildSeedPhaseShellCommand(phaseId: AutoloopPhaseId): string;
+
+declare function redactAutoloopTraceContents(contents: string): string;
+declare function sanitizeAutoloopTraceContentsForSandbox(contents: string): string;
+
+type ReusableSeedBundle = {
+    restored: boolean;
+    files: Array<{
+        path: string;
+        data: string;
+    }>;
+};
+type PhaseResumeDecision = {
+    reused: boolean;
+    reason: string;
+};
+declare function computeAutoloopPhaseFingerprint(job: AutoloopWorkerJob, sandboxPackageArchivePath: string | undefined): string;
+declare function computeAutoloopSeedBundleFingerprint(job: AutoloopWorkerJob, sandboxPackageArchivePath: string | undefined): string;
+declare function restoreReusableSeedBundle(job: AutoloopWorkerJob, fingerprint: string): ReusableSeedBundle;
+declare function collectPhaseResumeArtifacts(job: AutoloopWorkerJob, fingerprint: string): Array<{
+    path: string;
+    data: string;
+}>;
+declare function phaseResumeDecision(job: AutoloopWorkerJob, phaseId: string, fingerprint: string, seedRestored: boolean): PhaseResumeDecision;
+declare function persistReusablePhaseArtifacts(job: AutoloopWorkerJob, fingerprint: string): void;
+declare function createE2BAutoloopWorker(options?: E2BAutoloopWorkerOptions): AutoloopWorker;
+declare function localRepoSnapshotPackage(job: AutoloopWorkerJob): string | undefined;
+declare function resolveSandboxPhaseCommandTimeoutMs(phaseId: string, phaseEnv: Record<string, string | undefined>): number;
+
+declare const __autoloopWorkerTestInternals: {
+    collectPhaseResumeArtifacts: typeof collectPhaseResumeArtifacts;
+    computeAutoloopPhaseFingerprint: typeof computeAutoloopPhaseFingerprint;
+    computeAutoloopSeedBundleFingerprint: typeof computeAutoloopSeedBundleFingerprint;
+    persistReusablePhaseArtifacts: typeof persistReusablePhaseArtifacts;
+    phaseResumeDecision: typeof phaseResumeDecision;
+    restoreReusableSeedBundle: typeof restoreReusableSeedBundle;
+};
+
+export { AutoloopWorker, AutoloopWorkerJob, CodexAuth, __autoloopWorkerTestInternals, buildCodexPrompt, buildFixPrPhaseShellCommand, buildGradePhaseShellCommand, buildSandboxPhaseEnvs, buildSeedPhaseShellCommand, buildSeedPrompt, buildSeedTemplateReproducePhaseShellCommand, buildSetupShellCommand, createE2BAutoloopWorker, localRepoSnapshotPackage, redactAutoloopTraceContents, resolveArchalSandboxPackage, resolveArchalSandboxPackageArchivePath, resolveCodexReasoningEffort, resolveNativeHarnessRuntimeEnv, resolveSandboxPhaseCommandTimeoutMs, sanitizeAutoloopTraceContentsForSandbox };

package/dist/eval-shim.mjs

@@ -0,0 +1,216 @@
+// Eval-file runner shim. `archal run path/to/file.eval.ts` spawns this file
+// in a subprocess (under `npx tsx` for TypeScript eval files, plain node
+// otherwise) so the eval file's `import 'archal'` resolves to the consumer's
+// own installed copy, not the CLI's bundled sources. It stays plain ESM with
+// node builtins only and is staged into dist/ untouched by tsup — bundling to
+// CJS would rewrite the dynamic import below into `require()`, which cannot
+// load ESM eval files.
+//
+// Protocol: one JSON event per line on stdout, prefixed with the sentinel.
+// Everything else on stdout is the eval file's own output and is passed
+// through by the parent. The event shapes mirror the EvalShimEvent union in
+// run-eval-file.ts. Exit code is 0 whenever the protocol completed (a `done`
+// event was emitted) — pass/fail policy belongs to the parent; only a load
+// failure exits non-zero.
+import { pathToFileURL } from 'node:url';
+
+// The parent passes a per-run random prefix so a line the eval file happens to
+// print cannot accidentally collide with the protocol. This is collision
+// avoidance, NOT a trust boundary: the eval file is imported into this same
+// process and could read this value or forge events — that is acceptable
+// because it is the user's own code running with the user's own credentials.
+// Fall back to the fixed literal (must match EVAL_SHIM_SENTINEL in
+// run-eval-file.ts) when invoked without it.
+const SENTINEL = process.env.ARCHAL_EVAL_SENTINEL || 'ARCHAL_EVAL_EVENT ';
+
+function emit(event) {
+  process.stdout.write(SENTINEL + JSON.stringify(event) + '\n');
+}
+
+// Duck-typed on the SDK Scenario shape ({ definition, run() }) instead of a
+// brand so eval files keep working across archal versions and across multiple
+// archal copies in one node_modules tree. `criteria` is intentionally NOT
+// required: the SDK marks it optional (a scenario with no criteria is scored
+// on exec success alone), so requiring it here would silently hide valid
+// scenarios.
+function isScenarioExport(value) {
+  return (
+    typeof value === 'object' &&
+    value !== null &&
+    typeof value.run === 'function' &&
+    typeof value.definition === 'object' &&
+    value.definition !== null &&
+    typeof value.definition.task === 'string'
+  );
+}
+
+function errorMessage(err) {
+  return err instanceof Error ? err.message : String(err);
+}
+
+// Mirrors SYNTHETIC_TRACE_TOOL_NAMES in @archal/evaluator's trace-activity.ts
+// (the shim stays dependency-free, so the three names are duplicated here).
+// Synthetic proxy/canary entries are bookkeeping, not agent action, and must
+// not make a do-nothing agent look like it exercised the scenario.
+const SYNTHETIC_TRACE_TOOL_NAMES = new Set([
+  'sandbox_proxy_intercept',
+  'sandbox_proxy_passthrough',
+  'sandbox_canary_scan',
+]);
+
+function countAgentToolCalls(trace) {
+  return trace.filter(
+    (entry) =>
+      entry !== null &&
+      typeof entry === 'object' &&
+      typeof entry.toolName === 'string' &&
+      !SYNTHETIC_TRACE_TOOL_NAMES.has(entry.toolName),
+  ).length;
+}
+
+function scenarioRunOptions(options) {
+  const runOptions = {};
+  if (typeof options.runs === 'number') {
+    runOptions.runs = options.runs;
+  }
+  if (typeof options.timeout === 'number') {
+    runOptions.timeout = options.timeout;
+  }
+  // The SDK exposes one run-time model option (`model`), advertised to the
+  // agent as AGENT_MODEL. The CLI's --agent-model maps onto it. --model
+  // selects the evaluator/judge model, which the SDK configures separately
+  // (there is no evaluator run-option seam yet), so it is intentionally not
+  // forwarded here rather than passed under a key scenario.run() ignores.
+  if (typeof options.agentModel === 'string') {
+    runOptions.model = options.agentModel;
+  }
+  if (
+    typeof options.agent === 'object' &&
+    options.agent !== null &&
+    typeof options.agent.command === 'string'
+  ) {
+    runOptions.agent = {
+      command: options.agent.command,
+      ...(Array.isArray(options.agent.args) ? { args: options.agent.args } : {}),
+      ...(typeof options.agent.cwd === 'string' ? { cwd: options.agent.cwd } : {}),
+      ...(typeof options.agent.env === 'object' && options.agent.env !== null
+        ? { env: options.agent.env }
+        : {}),
+    };
+  }
+  return runOptions;
+}
+
+async function main() {
+  const [evalPath, optionsJson] = process.argv.slice(2);
+  if (!evalPath) {
+    emit({ type: 'load-error', message: 'eval-shim: missing eval file path argument.' });
+    process.exitCode = 1;
+    return;
+  }
+
+  let options = {};
+  if (optionsJson) {
+    try {
+      options = JSON.parse(optionsJson);
+    } catch (err) {
+      emit({ type: 'load-error', message: `eval-shim: invalid options JSON: ${errorMessage(err)}` });
+      process.exitCode = 1;
+      return;
+    }
+  }
+
+  let evalModule;
+  try {
+    evalModule = await import(pathToFileURL(evalPath).href);
+  } catch (err) {
+    emit({ type: 'load-error', message: errorMessage(err) });
+    process.exitCode = 1;
+    return;
+  }
+
+  // Inspect each export inside its own try/catch: a throwing getter or a proxy
+  // trap on one export must not abort discovery of the remaining (valid)
+  // scenarios and turn into an opaque "exited unexpectedly".
+  const scenarios = [];
+  for (const [exportName, value] of Object.entries(evalModule)) {
+    try {
+      if (!isScenarioExport(value)) {
+        continue;
+      }
+      const definitionName = value.definition.name;
+      scenarios.push({
+        exportName,
+        // The SDK documents this fallback: a scenario's name defaults to its
+        // export name when omitted. An empty/whitespace name falls back too.
+        name:
+          typeof definitionName === 'string' && definitionName.trim()
+            ? definitionName
+            : exportName,
+        scenario: value,
+      });
+    } catch {
+      continue;
+    }
+  }
+
+  emit({
+    type: 'discovered',
+    scenarios: scenarios.map(({ exportName, name }) => ({ exportName, name })),
+  });
+
+  for (const { exportName, name, scenario } of scenarios) {
+    emit({ type: 'scenario-start', exportName, name });
+    if (options.requireDirectAgent === true && !options.agent && !scenario.definition.agent) {
+      emit({
+        type: 'scenario-error',
+        exportName,
+        name,
+        message:
+          'This eval scenario resolves to a harness default, but eval files require a direct agent command. ' +
+          'Set an "agent" block in .archal.json or declare the agent in scenario({ agent }).',
+      });
+      continue;
+    }
+    try {
+      const result = await scenario.run(scenarioRunOptions(options));
+      emit({
+        type: 'scenario-result',
+        exportName,
+        name,
+        satisfaction: result.satisfaction,
+        // judgment is absent when the scenario declares no criteria (the SDK
+        // scores those on exec success alone); fall back to the attempt's own
+        // satisfaction/passed and an empty criteria list instead of throwing.
+        runs: result.runs.map((run) => ({
+          score: run.judgment?.score ?? run.satisfaction,
+          // run.passed is the SDK's verdict and already folds in the
+          // zero-tool-call gate; the judgment's own passed is only a fallback
+          // for hand-rolled run shapes that never set a per-run passed.
+          passed: run.passed ?? run.judgment?.passed,
+          criteria: run.judgment?.criteria ?? [],
+          // The parent's pass policy needs this to fail runs whose agent
+          // never touched a clone, mirroring the markdown runner's
+          // no-tool-calls outcome. The SDK computes it on each attempt
+          // (ScenarioAttemptResult.toolCallCount) and owns the zero-tool-call
+          // policy; the trace-derived count survives only as a fallback for
+          // older installed SDK copies that predate the field. Absent both,
+          // activity is unknown, not zero, and must not fire the structural
+          // failure with a misleading zero-tool-call message.
+          ...(typeof run.toolCallCount === 'number'
+            ? { toolCallCount: run.toolCallCount }
+            : Array.isArray(run.outcome?.trace)
+              ? { toolCallCount: countAgentToolCalls(run.outcome.trace) }
+              : {}),
+          ...(run.exec ? { exec: run.exec } : {}),
+        })),
+      });
+    } catch (err) {
+      emit({ type: 'scenario-error', exportName, name, message: errorMessage(err) });
+    }
+  }
+
+  emit({ type: 'done' });
+}
+
+await main();

package/dist/index.cjs

@@ -1,7 +1,20 @@
 'use strict';
 
-const version = "0.9.19";
-const archal = { version };
+const sdk = require('./sdk/index.cjs');
+const version = "0.10.0";
+const archal = { ...sdk, version };
 
 archal.default = archal;
 module.exports = archal;
+
+// Keep Node CommonJS named-export detection explicit for ESM importers.
+module.exports.benchmark = sdk.benchmark;
+module.exports.check = sdk.check;
+module.exports.clones = sdk.clones;
+module.exports.criterion = sdk.criterion;
+module.exports.entity = sdk.entity;
+module.exports.judge = sdk.judge;
+module.exports.publish = sdk.publish;
+module.exports.reportAgentMetrics = sdk.reportAgentMetrics;
+module.exports.scenario = sdk.scenario;
+module.exports.version = version;

package/dist/index.d.cts

@@ -1,3 +1,5 @@
+export * from './sdk/index.cjs';
 export declare const version: string;
-declare const archal: { readonly version: string };
+import * as sdk from './sdk/index.cjs';
+declare const archal: typeof sdk & { readonly version: string };
 export default archal;

package/dist/layer-1-ziaLpfLr.d.cts

@@ -0,0 +1,62 @@
+declare const AUTOLOOP_PHASES: readonly ["queued", "grade", "seed", "reproduce", "fix_pr", "done"];
+declare const AUTOLOOP_ACTIVE_PHASES: readonly ["grade", "seed", "reproduce", "fix_pr"];
+declare const AUTOLOOP_PHASE_HEALTH_STATUSES: readonly ["running", "completed", "stream_reconnect_recovered", "stalled_no_artifact", "timeout", "failed"];
+declare const AUTOLOOP_TRACE_SOURCE_STATUSES: readonly ["active", "paused", "error", "disabled"];
+type AutoloopExecutionPolicy = 'observe' | 'grade' | 'seed' | 'reproduce' | 'fix';
+type AutoloopRunResultSubmissionPhaseResult = {
+    phase: 'grade';
+    status: 'passed' | 'failed' | 'flaky' | 'inconclusive' | 'infrastructure';
+    summary: string;
+    confidence?: number | null;
+    failureClass?: string | null;
+    payload?: unknown;
+} | {
+    phase: 'seed';
+    status: 'seeded' | 'blocked' | 'failed';
+    summary: string;
+    payload?: unknown;
+} | {
+    phase: 'reproduce';
+    status: 'reproduced' | 'not_reproduced' | 'flaky' | 'blocked';
+    summary: string;
+    reason?: string | null;
+    diagnostic?: string | null;
+    hasCloneEvidence?: boolean;
+    hasCriteriaEvidence?: boolean;
+    payload?: unknown;
+    replayPlan?: unknown;
+    regressionPack?: unknown;
+    assertionResult?: unknown;
+} | {
+    phase: 'fix_pr';
+    status: 'pr_open' | 'checks_passing' | 'blocked' | 'failed';
+    summary: string;
+    issueUrl?: string | null;
+    prUrl?: string | null;
+    branchName?: string | null;
+    baseBranch?: string | null;
+    latestCheckState?: 'unknown' | 'pending' | 'passing' | 'failing' | 'blocked' | 'no_checks' | null;
+    localValidationStatus?: string | null;
+    blockingReason?: string | null;
+    postFixReproduceStatus?: string | null;
+    postFixReproduceReason?: string | null;
+    postFixReproduceHasCloneEvidence?: boolean;
+    postFixReproduceHasCriteriaEvidence?: boolean;
+    postFixReproduceIdentityMatchesPreFix?: boolean;
+    postFixReproduceCommitSha?: string | null;
+    postFixReproduceFailureSummary?: string | null;
+    postFixReproduceCloneEvidenceExpected?: boolean;
+    reproductionReverified?: boolean;
+    payload?: unknown;
+};
+type AutoloopRunResultSubmissionBody = Record<string, unknown>;
+type AutoloopBenchmarkExportConfig = {
+    enabled: boolean;
+};
+
+type AutoloopTraceSourceStatus = (typeof AUTOLOOP_TRACE_SOURCE_STATUSES)[number];
+type AutoloopPhase = (typeof AUTOLOOP_PHASES)[number];
+type AutoloopActivePhase = (typeof AUTOLOOP_ACTIVE_PHASES)[number];
+type AutoloopPhaseHealthStatus = (typeof AUTOLOOP_PHASE_HEALTH_STATUSES)[number];
+
+export type { AutoloopRunResultSubmissionPhaseResult as A, AutoloopPhaseHealthStatus as a, AutoloopPhase as b, AutoloopExecutionPolicy as c, AutoloopBenchmarkExportConfig as d, AutoloopActivePhase as e, AutoloopTraceSourceStatus as f, AutoloopRunResultSubmissionBody as g };