@bjoernrochel/sevro 0.1.0-rc.2 → 0.1.0-rc.21

package/src/runtime/state.ts

@@ -0,0 +1,143 @@
+import { mkdir, realpath, stat } from "node:fs/promises";
+import { join } from "node:path";
+import {
+  requireRuntimeEnvironment,
+  runtimeRoleSeeds,
+  type RuntimeRole,
+  requireRuntimeSeedTargets,
+  type RuntimePolicy,
+} from "./config";
+import { materializeRuntimeSeed } from "./seeds";
+import { requireRuntimeReadRoots } from "./paths";
+import { hashJson } from "../identity";
+import {
+  linkSharedCaches,
+  sharedCacheRoots,
+  selectedSharedCaches,
+} from "./shared-cache";
+
+export type { RuntimeRole } from "./config";
+
+export function runtimeObservations(
+  policy: RuntimePolicy | undefined,
+  role: RuntimeRole = "candidate",
+) {
+  if (!policy) return [];
+  return [
+    {
+      id: "sevro.host.runtime",
+      completeness: "complete" as const,
+      data: {
+        method: "runner_configuration",
+        role,
+        policyDigest: runtimeDigest(policy),
+        environmentNames: Object.keys(policy.environment).sort(),
+        readOnlyRoots: policy.readOnlyRoots,
+        ...discoveryObservations(policy),
+        ...(policy.sharedCacheDirectories?.length
+          ? { sharedCacheDirectories: selectedSharedCaches(policy, role) }
+          : {}),
+        seeds: runtimeRoleSeeds(policy, role).map((seed) => ({
+          target: seed.target,
+          sha256: seed.sha256,
+        })),
+      },
+    },
+  ];
+}
+
+function discoveryObservations(policy: RuntimePolicy) {
+  return {
+    discovery: policy.discovery ?? [],
+    skippedOptionalReadOnlyRoots: policy.skippedOptionalReadOnlyRoots ?? [],
+  };
+}
+
+function runtimeDigest(policy: RuntimePolicy): string {
+  const hooks = policy.hooks ?? {};
+  return hashJson({
+    ...policy,
+    seeds: policy.seeds ?? [],
+    hooks: {
+      nativeGoal: hooks.nativeGoal ?? false,
+      plugins: hooks.plugins ?? [],
+    },
+  });
+}
+
+/** Private role state contains copied seeds and aliases to explicitly shared caches. */
+export async function prepareRuntimeState(
+  workspace: string,
+  policy: RuntimePolicy,
+  role: RuntimeRole,
+  privateRoot?: string,
+) {
+  requireRuntimeEnvironment(policy.environment);
+  requireRuntimeSeedTargets([
+    ...(policy.seeds ?? []),
+    ...(policy.sharedCacheDirectories ?? []),
+  ]);
+  const root = await runtimeRoot(workspace, role, privateRoot);
+  const writableCacheRoots = await sharedCacheRoots(policy, role, [
+    workspace,
+    root,
+  ]);
+  const home = join(root, "home"),
+    temp = join(root, "tmp");
+  await Promise.all(
+    [home, temp].map((path) => mkdir(path, { recursive: true, mode: 0o700 })),
+  );
+  for (const seed of runtimeRoleSeeds(policy, role))
+    await copyRuntimeSeed(seed, join(root, seed.target));
+  await linkSharedCaches(policy, role, root);
+  const environment = Object.fromEntries(
+    Object.entries(policy.environment).map(([name, value]) => [
+      name,
+      value.replaceAll("{{sevro.runtime}}", root),
+    ]),
+  );
+  const isolatedEnvironment: Record<string, string> = {
+    ...environment,
+    HOME: home,
+    TMPDIR: temp,
+    UV_OFFLINE: "1",
+    PYTHONDONTWRITEBYTECODE: "1",
+  };
+  return {
+    root,
+    home,
+    temp,
+    environment: isolatedEnvironment,
+    writableCacheRoots,
+  };
+}
+
+async function runtimeRoot(
+  workspace: string,
+  role: RuntimeRole,
+  privateRoot: string | undefined,
+): Promise<string> {
+  const root = privateRoot ?? join(workspace, ".git", "sevro-runtime", role);
+  await mkdir(root, { recursive: true, mode: 0o700 });
+  return realpath(root);
+}
+
+async function copyRuntimeSeed(
+  seed: NonNullable<RuntimePolicy["seeds"]>[number],
+  target: string,
+): Promise<void> {
+  const canonical = await realpath(seed.source);
+  requireRuntimeReadRoots([seed.source, canonical], []);
+  if (await seedTargetExists(target)) return;
+  await materializeRuntimeSeed(seed, target);
+}
+
+async function seedTargetExists(target: string): Promise<boolean> {
+  try {
+    if ((await stat(target)).isDirectory()) return true;
+    throw new Error("runtime seed target is not a directory");
+  } catch (cause) {
+    if ((cause as NodeJS.ErrnoException).code !== "ENOENT") throw cause;
+    return false;
+  }
+}

package/src/schema-types.ts

@@ -0,0 +1,198 @@
+import type { AdvisoryAssessment } from "./advisory";
+import type {
+  ArtifactReference,
+  Assessment,
+  CheckOutcome,
+  Diagnostic,
+  DomainOutcome,
+  EvidenceRoute,
+  ExecutionStatus,
+  ExitCode,
+  GradingStatus,
+  MetricEvidence,
+  ObservationCompleteness,
+  ObservationEvidence,
+  OutcomeStatus,
+  RawResultReference,
+  TaskVerdict,
+  UsageEvidence,
+} from "./contract-types";
+import type { IdentityDimensions } from "./identity";
+import type { InstrumentationRequest } from "./instrumentation";
+
+/** Wire assessments additionally allow optional serialized error codes. */
+export type WireAssessment = {
+  execution: Assessment["execution"] & { errorCode?: string };
+  grading: Assessment["grading"] & { errorCode?: string };
+  task: Assessment["task"];
+};
+
+/** Validated v1 wire data also represents failures before run allocation. */
+export type CliResultData = WireAssessment & {
+  format: "sevro.cli-result.v1";
+  runId: string | null;
+  exitCode: ExitCode;
+  evidencePath: string | null;
+  cases: Array<CliResultDataCase>;
+  diagnostic?: Diagnostic;
+};
+export type CliResultDataTrial = WireAssessment & {
+  trial: number;
+  checks: Array<CheckOutcome>;
+  domainOutcomes?: Array<DomainOutcome>;
+  artifactPath: string | null;
+};
+export type CliResultDataCase = WireAssessment & {
+  caseId: string;
+  trials: Array<CliResultDataTrial>;
+};
+
+/** Structural contracts for data validated by the versioned JSON schemas. */
+export type RunEvidenceData = {
+  format: "sevro.run-evidence.v1";
+  runId: string;
+  evaluationIdentity: {
+    algorithm: "sevro.identity.v1";
+    digest: RunEvidenceDataDigest;
+    dimensions: IdentityDimensions;
+  };
+  configuration: {
+    digest: RunEvidenceDataDigest;
+    redacted: Record<string, unknown>;
+    resolved?: Record<string, unknown>;
+  };
+  runner: RunEvidenceDataRunner;
+  project: {
+    root: RunEvidenceDataFileUrl;
+    revision: string | null;
+    dirtyPatchDigest: RunEvidenceDataDigest | null;
+  };
+  extension: null | RunEvidenceDataExtension;
+  condition: {
+    requested: "passive" | "enforced";
+    actual: "passive" | "enforced" | "unknown";
+    requestedInstrumentation: InstrumentationRequest[];
+    appliedInstrumentation: InstrumentationRequest[];
+  };
+  graders: {
+    active: Array<RunEvidenceDataGrader>;
+    replacedDefaults: Array<string>;
+  };
+  routes: Array<EvidenceRoute>;
+  result: CliResultData;
+  trials: Array<RunEvidenceDataTrialEvidence>;
+  diagnostic?: Diagnostic;
+};
+type RunEvidenceDataDigest = string;
+type RunEvidenceDataFileUrl = string;
+type RunEvidenceDataRunner =
+  | {
+      source: "package";
+      packageName: string;
+      version: string;
+      buildDigest: RunEvidenceDataDigest;
+    }
+  | {
+      source: "checkout";
+      root: RunEvidenceDataFileUrl;
+      revision: string;
+      dirtyPatchDigest: RunEvidenceDataDigest | null;
+      buildDigest: RunEvidenceDataDigest;
+    };
+type RunEvidenceDataExtension = {
+  id: string;
+  version: string;
+  sourceDigest: RunEvidenceDataDigest;
+  configurationDigest: RunEvidenceDataDigest;
+  protocol: "sevro.extension.v1";
+  capabilities: Array<string>;
+  replacements: { graders: Array<string>; taskVerdictPolicy: string | null };
+};
+type RunEvidenceDataGrader = {
+  id: string;
+  source: "builtin" | "extension";
+  version: string;
+};
+export type AdvisoryReviewData = {
+  status: "completed" | "failed" | "not_run";
+  assessment: AdvisoryAssessment | null;
+  usage: UsageEvidence;
+  rawResult: RawResultReference;
+};
+export type RunEvidenceDataTrialEvidence = {
+  caseId: string;
+  trial: number;
+  executionMode: "executed" | "dry" | "unknown";
+  candidateDurationMs?: number | null;
+  condition: {
+    requested: "passive" | "enforced";
+    actual: "passive" | "enforced" | "unknown";
+    appliedInstrumentation: InstrumentationRequest[];
+  };
+  observationCompleteness: ObservationCompleteness;
+  observations: Array<ObservationEvidence>;
+  metrics: Array<MetricEvidence>;
+  domainOutcomes?: Array<DomainOutcome>;
+  taskVerdictPolicy: null | {
+    id: string;
+    recommendation: TaskVerdict | null;
+  };
+  advisoryReview?: AdvisoryReviewData | null;
+  routes: Array<EvidenceRoute>;
+  usage: UsageEvidence;
+  rawResult: RawResultReference;
+  artifactRefs: Array<ArtifactReference>;
+};
+export type ReportData = {
+  format: "sevro.report.v1";
+  inputs: Array<ReportDataInput>;
+  rows: Array<ReportDataRow>;
+  summary: {
+    cases: number;
+    passed: number;
+    failed: number;
+    notAssessed: number;
+  };
+};
+type ReportDataPath = string;
+type ReportDataDigest = string;
+/** The report wire schema deliberately allows either state family in rows. */
+type ReportDataState = ExecutionStatus | GradingStatus;
+type ReportDataInput = {
+  resultFile: ReportDataPath;
+  execution: ExecutionStatus;
+  grading: GradingStatus;
+  task: TaskVerdict;
+  exitCode: ExitCode;
+  caseCount: number;
+};
+type ReportDataRoute = Omit<EvidenceRoute, "role">;
+type ReportDataOutcome = {
+  trial: number;
+  id: string;
+  status: OutcomeStatus;
+};
+type ReportDataRow = {
+  nativeTranscripts?: { retained: number; pruned: number };
+  resultFile: ReportDataPath;
+  evidencePath: string | null;
+  runId: string | null;
+  caseId: string;
+  execution: ReportDataState;
+  grading: ReportDataState;
+  task: TaskVerdict;
+  exitCode: ExitCode;
+  trialCount: number;
+  taskPassRate: number | null;
+  candidateDurationMs: number | null;
+  inputTokens: number | null;
+  outputTokens: number | null;
+  costUsd: number | null;
+  runnerBuildDigest: ReportDataDigest | null;
+  evaluationDigest: ReportDataDigest | null;
+  requestedCondition: "passive" | "enforced" | null;
+  actualCondition: "passive" | "enforced" | "unknown" | null;
+  candidateRoute: ReportDataRoute | null;
+  routes: Array<EvidenceRoute>;
+  domainOutcomes: Array<ReportDataOutcome>;
+};

package/src/schema.ts

@@ -1,28 +1,24 @@
-import Ajv2020 from "ajv/dist/2020.js";
-import cliSchema from "../schemas/cli-result-v1.schema.json";
-import runSchema from "../schemas/run-evidence-v1.schema.json";
-import reportSchema from "../schemas/report-v1.schema.json";
+import type {
+  CliResultData,
+  RunEvidenceData,
+  ReportData,
+} from "./schema-types";
+import cliValidator from "./generated/cli-result.cjs";
+import runValidator from "./generated/run-evidence.cjs";
+import reportValidator from "./generated/report.cjs";
 
-const ajv = new Ajv2020({
-  strict: true,
-  strictRequired: false,
-  strictTypes: false,
-});
-ajv.addSchema(cliSchema);
-ajv.addSchema(runSchema);
-ajv.addSchema(reportSchema);
-const cliValidator = ajv.getSchema("urn:sevro:schema:cli-result:v1")!;
-const runValidator = ajv.getSchema("urn:sevro:schema:run-evidence:v1")!;
-const reportValidator = ajv.getSchema("urn:sevro:schema:report:v1")!;
-
-export function assertCliResult(value: unknown): void {
+export function assertCliResult(
+  value: unknown,
+): asserts value is CliResultData {
   if (!cliValidator(value)) throw new Error("invalid Sevro CLI result");
 }
 
-export function assertRunEvidence(value: unknown): void {
+export function assertRunEvidence(
+  value: unknown,
+): asserts value is RunEvidenceData {
   if (!runValidator(value)) throw new Error("invalid Sevro retained evidence");
 }
 
-export function assertReport(value: unknown): void {
+export function assertReport(value: unknown): asserts value is ReportData {
   if (!reportValidator(value)) throw new Error("invalid Sevro report");
 }

package/src/transcript-retention.ts

@@ -0,0 +1,146 @@
+import { readFile, rm } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import type { RunEvidenceData } from "./schema-types";
+import { canonicalJson, sha256 } from "./identity";
+import type { ArtifactReference } from "./contract-types";
+import { retainedFile } from "./results-lock";
+import { atomicWriteJson } from "./storage";
+
+function isNativeTranscript(artifact: ArtifactReference) {
+  return /^(?:sevro\.(?:semantic(?:\.[a-f0-9]{64})?|advisory)\.)?sevro\.native-transcripts\.bundle$/.test(
+    artifact.id,
+  );
+}
+
+function transcriptArtifacts(evidence: RunEvidenceData) {
+  return evidence.trials.flatMap((trial) =>
+    trial.artifactRefs.filter(isNativeTranscript).map((artifact) => ({
+      caseId: trial.caseId,
+      trial: trial.trial,
+      ...artifact,
+    })),
+  );
+}
+
+function pruningRecord(evidence: RunEvidenceData) {
+  return {
+    format: "sevro.pruned-native-transcripts.v1",
+    runId: evidence.runId,
+    artifacts: transcriptArtifacts(evidence),
+  };
+}
+
+function pruningPath(evidence: RunEvidenceData) {
+  if (evidence.result.evidencePath === null)
+    throw new Error(
+      "native transcript retention requires a retained evidence path",
+    );
+  return join(
+    dirname(evidence.result.evidencePath),
+    "native-transcripts-pruned.json",
+  );
+}
+
+function transcriptPath(
+  evidence: RunEvidenceData,
+  artifact: ArtifactReference,
+) {
+  const path = fileURLToPath(artifact.path);
+  if (dirname(path) !== dirname(pruningPath(evidence)))
+    throw new Error(
+      "native transcript artifact is outside its retained result directory",
+    );
+  if (metadataPaths(evidence).includes(path))
+    throw new Error(
+      "native transcript artifact aliases retained result metadata",
+    );
+  if (otherArtifactPaths(evidence).includes(path))
+    throw new Error(
+      "native transcript artifact aliases other retained evidence",
+    );
+  return path;
+}
+
+function otherArtifactPaths(evidence: RunEvidenceData) {
+  return evidence.trials
+    .flatMap((trial) => [
+      trial.rawResult.path,
+      trial.advisoryReview?.rawResult.path,
+      ...trial.artifactRefs
+        .filter((ref) => !isNativeTranscript(ref))
+        .map((ref) => ref.path),
+    ])
+    .filter((path): path is string => path != null)
+    .map((path) => fileURLToPath(path));
+}
+
+function metadataPaths(evidence: RunEvidenceData) {
+  const receipt = pruningPath(evidence);
+  return [
+    receipt,
+    evidence.result.evidencePath,
+    join(dirname(receipt), "result.json"),
+    join(dirname(receipt), "latest.json"),
+    ...evidence.result.cases.flatMap((selected) =>
+      selected.trials.map((trial) => trial.artifactPath),
+    ),
+  ];
+}
+
+async function transcriptPresent(
+  evidence: RunEvidenceData,
+  artifact: ArtifactReference,
+) {
+  const path = transcriptPath(evidence, artifact);
+  if (!(await retainedFile(path))) return false;
+  if (sha256(await readFile(path)) !== artifact.sha256)
+    throw new Error("retained native transcript evidence hash differs");
+  return true;
+}
+
+/** Keep capture evidence immutable; a separate receipt records intentional removal. */
+export async function pruneNativeTranscripts(evidence: RunEvidenceData) {
+  const artifacts = transcriptArtifacts(evidence);
+  if (!artifacts.length) return 0;
+  await atomicWriteJson(pruningPath(evidence), pruningRecord(evidence));
+  let removed = 0;
+  for (const artifact of artifacts) {
+    const path = transcriptPath(evidence, artifact);
+    if (!(await retainedFile(path))) continue;
+    await rm(path);
+    removed++;
+  }
+  return removed;
+}
+
+/** Missing native bytes require an exact receipt bound to their original references. */
+async function hasPruningRecord(evidence: RunEvidenceData) {
+  const path = pruningPath(evidence);
+  const marked = await retainedFile(path);
+  if (marked) {
+    const receipt: unknown = JSON.parse(await readFile(path, "utf8"));
+    if (canonicalJson(receipt) !== canonicalJson(pruningRecord(evidence)))
+      throw new Error(
+        "native transcript pruning record differs from retained evidence",
+      );
+  }
+  return marked !== null;
+}
+
+export async function nativeTranscriptRetention(
+  evidence: RunEvidenceData | null,
+) {
+  if (!evidence) return [];
+  const marked = await hasPruningRecord(evidence);
+  const states = [];
+  for (const artifact of transcriptArtifacts(evidence)) {
+    const present = await transcriptPresent(evidence, artifact);
+    if (!present && !marked)
+      throw new Error(
+        "retained native transcript evidence is missing without a pruning record",
+      );
+    states.push({ caseId: artifact.caseId, pruned: !present });
+  }
+  return states;
+}

package/src/value-guards.ts

@@ -0,0 +1,20 @@
+/** Narrow JSON containers before accessing untrusted members. */
+export function isRecord(value: unknown): value is Record<string, unknown> {
+  return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+
+export function isUnknownArray(value: unknown): value is unknown[] {
+  return Array.isArray(value);
+}
+
+export function isStringArray(value: unknown): value is string[] {
+  return (
+    isUnknownArray(value) && value.every((item) => typeof item === "string")
+  );
+}
+
+export function isOptionalNonblankString(value: unknown): boolean {
+  return (
+    value === undefined || (typeof value === "string" && Boolean(value.trim()))
+  );
+}