@actuarial-ts/compliance 0.6.1 → 0.7.0

package/src/diagnosticReplayWriter.ts

@@ -0,0 +1,190 @@
+import { iterateDiagnosticIdentityJson } from "@actuarial-ts/core";
+import { getCompletedCompactDiagnosticRunInput } from "@actuarial-ts/data";
+import {
+  assertVerifiedCompactDiagnosticRunProvenance,
+  getCompactDiagnosticRunOwner,
+  type VerifiedCompactDiagnosticRunProvenance,
+} from "./diagnosticCompactRun.js";
+import type { DiagnosticArtifactDigest } from "./diagnosticRun.js";
+import { diagnosticByteView } from "./diagnosticByteView.js";
+import { consumeDiagnosticByteSource } from "./diagnosticByteSource.js";
+import {
+  encodeReplayFrames,
+  REPLAY_TEXT_UNITS,
+  type ReplayFrame,
+} from "./diagnosticReplayFrames.js";
+import { replayValueFrames } from "./diagnosticReplayValue.js";
+import { createSha256 } from "./sha256Stream.js";
+import {
+  ARTIFACT_CHUNK_BYTES,
+  REPLAY_FORMAT,
+  REPLAY_VERSION,
+  badReplay,
+  mismatch,
+  replayArray,
+  replayCancellation,
+  replayFrameBytes,
+  replayRecord,
+  replayToken,
+  encodeArtifactChunk,
+} from "./diagnosticReplayProtocol.js";
+
+export interface DiagnosticReplayStreamRun {
+  readonly id: string;
+  readonly provenance: VerifiedCompactDiagnosticRunProvenance;
+}
+export type DiagnosticReplayByteSource = Iterable<Uint8Array> | AsyncIterable<Uint8Array>;
+export interface WriteDiagnosticReplayStreamInput {
+  readonly runs: readonly DiagnosticReplayStreamRun[];
+  /** Fresh stream of at most 64 KiB per chunk, for each unique computed artifact. */
+  readonly openArtifact: (
+    artifact: Extract<DiagnosticArtifactDigest, { assurance: "sdk-computed" }>,
+  ) => DiagnosticReplayByteSource | Promise<DiagnosticReplayByteSource>;
+  readonly signal?: AbortSignal;
+}
+
+function replayInput(provenance: VerifiedCompactDiagnosticRunProvenance) {
+  const input = getCompletedCompactDiagnosticRunInput(getCompactDiagnosticRunOwner(provenance));
+  // Only a small envelope is new. All dataset/evidence arrays are the immutable
+  // originals owned by the validated gateway, including filtered-out inputs.
+  // Definition uses the normalized wire shape accepted by the core compiler,
+  // not a falsely typed authored definition with its optional-null defaults.
+  return {
+    definition: input.definition.definition,
+    losses: input.losses,
+    exposures: input.exposures,
+    ...(input.filter === null ? {} : { filter: input.filter }),
+    completePeriodCutoffs: input.completePeriodCutoffs,
+    ...(input.expectedCells === null ? {} : { expectedCells: input.expectedCells }),
+    reviewEvidence: input.reviewEvidence,
+    ...(input.runPresetId === null ? {} : { runPresetId: input.runPresetId }),
+    ...(input.datasetArtifactId === null ? {} : { datasetArtifactId: input.datasetArtifactId }),
+    groupMap: input.groupMap,
+    groupDimensions: input.groupDimensions,
+    policy: {
+      allowedReviewStatuses: input.policy.allowedReviewStatuses,
+      allowedMetricFindingSeverities: input.policy.allowedMetricFindingSeverities,
+      ...(input.policy.rationaleRef === null ? {} : { rationaleRef: input.policy.rationaleRef }),
+    },
+  };
+}
+
+/**
+ * Versioned large-data replay, NOT an interchange BundleDoc or the old JSON
+ * provenance DTO. Writes only from genuine immutable provenance owners. The
+ * host owns I/O and must publish a file only after the generator completes.
+ * All artifact bytes are rechecked against their authenticated digests while
+ * writing. Consumer cancellation closes producers; pending host I/O should use
+ * the same signal. No full candidate identity or whole archive string exists.
+ */
+export function writeDiagnosticReplayStream(
+  input: WriteDiagnosticReplayStreamInput,
+): AsyncGenerator<Uint8Array> {
+  // Snapshot at call time, before the first await/yield (not at first next()).
+  const options = replayRecord(input, ["runs", "openArtifact", "signal"]);
+  const runs = replayArray(options.runs).map((value) => {
+    const item = replayRecord(value, ["id", "provenance"]);
+    const id = replayToken(item.id);
+    assertVerifiedCompactDiagnosticRunProvenance(item.provenance);
+    return Object.freeze({ id, provenance: item.provenance });
+  });
+  if (!runs.length || new Set(runs.map((run) => run.id)).size !== runs.length)
+    badReplay("Replay must have at least one run and unique run IDs");
+  if (typeof options.openArtifact !== "function")
+    badReplay("Replay openArtifact must be a function");
+  const openArtifact = options.openArtifact as WriteDiagnosticReplayStreamInput["openArtifact"];
+  const checkCancellation = replayCancellation(options.signal);
+  checkCancellation();
+  const artifacts = new Map<string, DiagnosticArtifactDigest>();
+  for (const { provenance } of runs)
+    for (const artifact of [...provenance.inputArtifacts, ...provenance.preparationArtifacts]) {
+      const prior = artifacts.get(artifact.id);
+      if (
+        prior &&
+        (prior.scope !== artifact.scope ||
+          prior.assurance !== artifact.assurance ||
+          prior.algorithm !== artifact.algorithm ||
+          prior.value !== artifact.value ||
+          (prior.assurance === "sdk-computed" &&
+            artifact.assurance === "sdk-computed" &&
+            prior.byteLength !== artifact.byteLength))
+      )
+        badReplay("Shared replay artifact IDs have conflicting evidence");
+      artifacts.set(artifact.id, artifact);
+    }
+
+  async function* body(): AsyncGenerator<ReplayFrame> {
+    yield [REPLAY_FORMAT, REPLAY_VERSION];
+    for (const artifact of artifacts.values()) {
+      checkCancellation();
+      yield ["artifact"];
+      yield* replayValueFrames(artifact);
+      yield ["artifact-data"];
+      if (artifact.assurance === "sdk-computed") {
+        const hash = createSha256();
+        let length = 0;
+        for await (const raw of consumeDiagnosticByteSource(
+          await openArtifact(artifact),
+          checkCancellation,
+        )) {
+          checkCancellation();
+          const view = diagnosticByteView(raw);
+          if (view.byteLength > ARTIFACT_CHUNK_BYTES)
+            badReplay("Artifact read chunks must be at most 64 KiB");
+          // Snapshot the entire bounded delivered chunk before yielding.
+          const owned = new Uint8Array(view);
+          length += owned.length;
+          if (!Number.isSafeInteger(length) || length > artifact.byteLength)
+            mismatch("Artifact grew after its verified digest");
+          hash.update(owned);
+          if (owned.length) yield ["bytes", encodeArtifactChunk(owned)];
+        }
+        checkCancellation();
+        if (length !== artifact.byteLength || hash.digest() !== artifact.value)
+          mismatch("Artifact bytes changed after their verified digest");
+      }
+      yield ["artifact-end"];
+    }
+    for (const { id, provenance } of runs) {
+      checkCancellation();
+      yield ["run"];
+      yield* replayValueFrames({
+        id,
+        input: replayInput(provenance),
+        inputArtifactIds: provenance.inputArtifacts.map((item) => item.id),
+        preparationArtifactIds: provenance.preparationArtifacts.map((item) => item.id),
+        preparationLineage: provenance.preparationLineage,
+      });
+      yield ["input-end"];
+      for (const [channel, document] of [
+        ["manifest", provenance.manifestIdentityDocument],
+        ["result", provenance.resultIdentityDocument],
+      ] as const) {
+        for (const text of iterateDiagnosticIdentityJson(document))
+          for (let start = 0; start < text.length; start += REPLAY_TEXT_UNITS)
+            yield [channel, text.slice(start, start + REPLAY_TEXT_UNITS)];
+        yield [`${channel}-end`];
+      }
+      yield [
+        "run-end",
+        provenance.runFingerprint,
+        provenance.resultFingerprint,
+        provenance.runResultFingerprint,
+      ];
+    }
+  }
+  async function* sequenced(): AsyncGenerator<ReplayFrame> {
+    let sequence = 0;
+    const hash = createSha256();
+    for await (const event of body()) {
+      checkCancellation();
+      if (!Number.isSafeInteger(sequence + 1)) badReplay("Replay sequence exceeded safe integers");
+      const frame = [sequence++, ...event];
+      hash.update(replayFrameBytes(frame));
+      yield frame;
+    }
+    checkCancellation();
+    yield [sequence, "end", artifacts.size, runs.length, hash.digest()];
+  }
+  return encodeReplayFrames(sequenced());
+}

package/src/diagnosticRun.ts

@@ -29,6 +29,13 @@ import {
 } from "@actuarial-ts/data";
 import { ComplianceError } from "./errors.js";
 import { COMPLIANCE_PACKAGE_VERSION } from "./version.js";
+import {
+  token,
+  plain,
+  exactKeys,
+  snapshotLineage,
+  validateArtifactGraph,
+} from "./diagnosticRunEvidence.js";
 
 export interface DiagnosticArtifactDigestBase {
   readonly id: string;
@@ -141,36 +148,6 @@ export interface CreateDiagnosticRunIdentityInput {
 }
 
 const verified = new WeakMap<object, CompletedValidatedMetricDiagnosticsRun>();
-const token = (value: string, path: string) => {
-  if (
-    value.length === 0 ||
-    /^[\t-\r ]|[\t-\r ]$/.test(value) ||
-    value.includes("\0")
-  )
-    throw new ComplianceError(
-      "BAD_DIAGNOSTIC_RUN",
-      `${path} must be a nonempty token`,
-      path,
-    );
-  for (let index = 0; index < value.length; index++) {
-    const unit = value.charCodeAt(index);
-    if (unit >= 0xd800 && unit <= 0xdbff) {
-      const next = value.charCodeAt(index + 1);
-      if (!(next >= 0xdc00 && next <= 0xdfff))
-        throw new ComplianceError(
-          "BAD_DIAGNOSTIC_RUN",
-          `${path} contains malformed Unicode`,
-          path,
-        );
-      index++;
-    } else if (unit >= 0xdc00 && unit <= 0xdfff)
-      throw new ComplianceError(
-        "BAD_DIAGNOSTIC_RUN",
-        `${path} contains malformed Unicode`,
-        path,
-      );
-  }
-};
 function freeze<T>(
   value: T,
   seen = new WeakSet<object>(),
@@ -210,25 +187,6 @@ type SnapshottedArtifact =
       readonly assurance: "sdk-computed";
       readonly bytes: Uint8Array;
     };
-function plain(value: unknown): value is Record<string, unknown> {
-  if (value === null || typeof value !== "object" || Array.isArray(value))
-    return false;
-  const prototype = Object.getPrototypeOf(value);
-  return prototype === Object.prototype || prototype === null;
-}
-function exactKeys(
-  value: Record<string, unknown>,
-  allowed: readonly string[],
-  path: string,
-): void {
-  for (const key of Object.keys(value))
-    if (!allowed.includes(key))
-      throw new ComplianceError(
-        "BAD_DIAGNOSTIC_RUN",
-        `Unknown field ${key}`,
-        `${path}.${key}`,
-      );
-}
 function snapshotArtifacts(
   evidence: unknown,
   scope: "input" | "preparation",
@@ -348,330 +306,6 @@ async function digestArtifacts(
   );
 }
 
-function snapshotLineage(
-  lineage: unknown,
-): readonly DiagnosticPreparationLineage[] {
-  if (!Array.isArray(lineage))
-    throw new ComplianceError(
-      "BAD_DIAGNOSTIC_RUN",
-      "$.preparationLineage must be an array",
-      "$.preparationLineage",
-    );
-  return lineage
-    .map((raw, index) => {
-      const path = `$.preparationLineage[${index}]`;
-      if (!plain(raw))
-        throw new ComplianceError(
-          "BAD_DIAGNOSTIC_RUN",
-          "Lineage edge must be a plain object",
-          path,
-        );
-      exactKeys(
-        raw,
-        ["outputArtifactId", "inputArtifactIds", "transformationArtifactIds"],
-        path,
-      );
-      if (typeof raw.outputArtifactId !== "string")
-        throw new ComplianceError(
-          "BAD_DIAGNOSTIC_RUN",
-          "Lineage output must be a string",
-          `${path}.outputArtifactId`,
-        );
-      token(raw.outputArtifactId, `${path}.outputArtifactId`);
-      const readIds = (value: unknown, key: string) => {
-        if (!Array.isArray(value))
-          throw new ComplianceError(
-            "BAD_DIAGNOSTIC_RUN",
-            `Lineage ${key} must be an array`,
-            `${path}.${key}`,
-          );
-        const result = value.map((id, inputIndex) => {
-          if (typeof id !== "string")
-            throw new ComplianceError(
-              "BAD_DIAGNOSTIC_RUN",
-              "Lineage artifact id must be a string",
-              `${path}.${key}[${inputIndex}]`,
-            );
-          token(id, `${path}.${key}[${inputIndex}]`);
-          return id;
-        });
-        if (new Set(result).size !== result.length)
-          throw new ComplianceError(
-            "BAD_DIAGNOSTIC_RUN",
-            `Lineage ${key} must be unique`,
-            `${path}.${key}`,
-          );
-        return [...result].sort();
-      };
-      const inputs = readIds(raw.inputArtifactIds, "inputArtifactIds"),
-        transformations = readIds(
-          raw.transformationArtifactIds,
-          "transformationArtifactIds",
-        );
-      if (inputs.length + transformations.length === 0)
-        throw new ComplianceError(
-          "BAD_DIAGNOSTIC_RUN",
-          "Lineage edge must have at least one upstream artifact",
-          path,
-        );
-      return {
-        outputArtifactId: raw.outputArtifactId,
-        inputArtifactIds: inputs,
-        transformationArtifactIds: transformations,
-      };
-    })
-    .sort((a, b) =>
-      a.outputArtifactId < b.outputArtifactId
-        ? -1
-        : a.outputArtifactId > b.outputArtifactId
-          ? 1
-          : 0,
-    );
-}
-
-function validateArtifactGraph(
-  run: CompletedValidatedMetricDiagnosticsRun,
-  inputArtifacts: readonly Pick<
-    DiagnosticArtifactDigest,
-    "id" | "scope" | "assurance"
-  >[],
-  preparationArtifacts: readonly Pick<
-    DiagnosticArtifactDigest,
-    "id" | "scope" | "assurance"
-  >[],
-  lineage: readonly DiagnosticPreparationLineage[],
-): void {
-  const byId = new Map<
-    string,
-    Pick<DiagnosticArtifactDigest, "id" | "scope" | "assurance">
-  >();
-  for (const [index, item] of [
-    ...inputArtifacts,
-    ...preparationArtifacts,
-  ].entries()) {
-    if (byId.has(item.id))
-      throw new ComplianceError(
-        "BAD_DIAGNOSTIC_RUN",
-        `Duplicate artifact ID ${item.id}`,
-        `$.${index < inputArtifacts.length ? "inputArtifacts" : "preparationArtifacts"}[${index < inputArtifacts.length ? index : index - inputArtifacts.length}].id`,
-      );
-    byId.set(item.id, item);
-  }
-  const referenced = new Set<string>();
-  const requireArtifact = (
-    id: string,
-    scope: "input" | "preparation",
-    path: string,
-  ) => {
-    const artifact = byId.get(id);
-    if (!artifact)
-      throw new ComplianceError(
-        "BAD_DIAGNOSTIC_RUN",
-        `Artifact reference ${id} is unresolved`,
-        path,
-      );
-    if (artifact.scope !== scope)
-      throw new ComplianceError(
-        "BAD_DIAGNOSTIC_RUN",
-        `Artifact ${id} must have ${scope} scope`,
-        path,
-      );
-    referenced.add(id);
-  };
-  let unsourced = false;
-  for (const [index, item] of run.prepared.inputAudit.entries()) {
-    const source = item.record.source;
-    if (source)
-      requireArtifact(
-        source.artifactId,
-        "input",
-        `$.completedRun.prepared.inputAudit[${index}].record.source.artifactId`,
-      );
-    else unsourced = true;
-  }
-  const evidence = run.review.evidence;
-  if (evidence) {
-    for (const [index, item] of evidence.groupingAssignments.entries()) {
-      if (item.source)
-        requireArtifact(
-          item.source.artifactId,
-          "input",
-          `$.completedRun.review.evidence.groupingAssignments[${index}].source.artifactId`,
-        );
-      else unsourced = true;
-    }
-    for (const [index, item] of evidence.cachedFormulas.entries()) {
-      if (item.source)
-        requireArtifact(
-          item.source.artifactId,
-          "input",
-          `$.completedRun.review.evidence.cachedFormulas[${index}].source.artifactId`,
-        );
-      else unsourced = true;
-    }
-  }
-  if (unsourced) {
-    if (run.datasetArtifactId === null)
-      throw new ComplianceError(
-        "BAD_DIAGNOSTIC_RUN",
-        "Unsourced diagnostic input requires datasetArtifactId",
-        "$.completedRun.datasetArtifactId",
-      );
-    const fallback = byId.get(run.datasetArtifactId);
-    if (
-      !fallback ||
-      fallback.scope !== "input" ||
-      fallback.assurance !== "sdk-computed"
-    )
-      throw new ComplianceError(
-        "BAD_DIAGNOSTIC_RUN",
-        "datasetArtifactId must resolve to SDK-computed input evidence",
-        "$.completedRun.datasetArtifactId",
-      );
-    referenced.add(run.datasetArtifactId);
-  } else if (run.datasetArtifactId !== null) {
-    requireArtifact(
-      run.datasetArtifactId,
-      "input",
-      "$.completedRun.datasetArtifactId",
-    );
-    if (byId.get(run.datasetArtifactId)!.assurance !== "sdk-computed")
-      throw new ComplianceError(
-        "BAD_DIAGNOSTIC_RUN",
-        "datasetArtifactId must resolve to SDK-computed input evidence",
-        "$.completedRun.datasetArtifactId",
-      );
-  }
-  for (const [
-    basisIndex,
-    basis,
-  ] of run.prepared.definition.definition.amountBases.entries())
-    for (const [componentIndex, component] of basis.components.entries())
-      if (
-        component.limitation.kind !== "unlimited" &&
-        component.limitation.kind !== "unknown" &&
-        component.limitation.derivation.kind === "external"
-      )
-        requireArtifact(
-          component.limitation.derivation.transformationRef,
-          "preparation",
-          `$.definition.amountBases[${basisIndex}].components[${componentIndex}].limitation.derivation.transformationRef`,
-        );
-  for (const [
-    ruleIndex,
-    rule,
-  ] of run.prepared.definition.definition.reviewRules.entries())
-    if (
-      rule.kind === "layer-order" &&
-      rule.comparability.kind === "caller-asserted"
-    )
-      requireArtifact(
-        rule.comparability.rationaleArtifactId,
-        "preparation",
-        `$.definition.reviewRules[${ruleIndex}].comparability.rationaleArtifactId`,
-      );
-  if (run.gate.rationaleRef !== null)
-    requireArtifact(
-      run.gate.rationaleRef,
-      "preparation",
-      "$.completedRun.gate.rationaleRef",
-    );
-  const edges = new Map<string, readonly string[]>();
-  for (const [index, edge] of lineage.entries()) {
-    const path = `$.preparationLineage[${index}]`;
-    if (edges.has(edge.outputArtifactId))
-      throw new ComplianceError(
-        "BAD_DIAGNOSTIC_RUN",
-        "An artifact may have only one producing lineage edge",
-        `${path}.outputArtifactId`,
-      );
-    const downstream = byId.get(edge.outputArtifactId);
-    if (!downstream)
-      throw new ComplianceError(
-        "BAD_DIAGNOSTIC_RUN",
-        `Lineage artifact ${edge.outputArtifactId} is unresolved`,
-        `${path}.outputArtifactId`,
-      );
-    if (downstream.scope !== "input")
-      throw new ComplianceError(
-        "BAD_DIAGNOSTIC_RUN",
-        "Lineage output artifact must have input scope",
-        `${path}.outputArtifactId`,
-      );
-    for (const [inputIndex, id] of edge.inputArtifactIds.entries()) {
-      if (id === edge.outputArtifactId)
-        throw new ComplianceError(
-          "BAD_DIAGNOSTIC_RUN",
-          "Lineage may not reference itself",
-          `${path}.inputArtifactIds[${inputIndex}]`,
-        );
-      const artifact = byId.get(id);
-      if (!artifact || artifact.scope !== "input")
-        throw new ComplianceError(
-          "BAD_DIAGNOSTIC_RUN",
-          `Lineage input ${id} must resolve to input evidence`,
-          `${path}.inputArtifactIds[${inputIndex}]`,
-        );
-    }
-    for (const [
-      transformIndex,
-      id,
-    ] of edge.transformationArtifactIds.entries()) {
-      const artifact = byId.get(id);
-      if (!artifact || artifact.scope !== "preparation")
-        throw new ComplianceError(
-          "BAD_DIAGNOSTIC_RUN",
-          `Lineage transformation ${id} must resolve to preparation evidence`,
-          `${path}.transformationArtifactIds[${transformIndex}]`,
-        );
-    }
-    edges.set(edge.outputArtifactId, [
-      ...edge.inputArtifactIds,
-      ...edge.transformationArtifactIds,
-    ]);
-  }
-  const visiting = new Set<string>(),
-    visited = new Set<string>();
-  for (const root of [...referenced]) {
-    const stack = [{ id: root, exit: false }];
-    while (stack.length > 0) {
-      const { id, exit } = stack.pop()!;
-      if (exit) {
-        visiting.delete(id);
-        visited.add(id);
-        continue;
-      }
-      if (visiting.has(id))
-        throw new ComplianceError(
-          "BAD_DIAGNOSTIC_RUN",
-          "Artifact lineage contains a cycle",
-          "$.preparationLineage",
-        );
-      if (visited.has(id)) continue;
-      visiting.add(id);
-      stack.push({ id, exit: true });
-      for (const upstream of [...(edges.get(id) ?? [])].reverse()) {
-        referenced.add(upstream);
-        stack.push({ id: upstream, exit: false });
-      }
-    }
-  }
-  for (const [index, artifact] of inputArtifacts.entries())
-    if (!referenced.has(artifact.id))
-      throw new ComplianceError(
-        "BAD_DIAGNOSTIC_RUN",
-        `Artifact ${artifact.id} is orphaned`,
-        `$.inputArtifacts[${index}].id`,
-      );
-  for (const [index, artifact] of preparationArtifacts.entries())
-    if (!referenced.has(artifact.id))
-      throw new ComplianceError(
-        "BAD_DIAGNOSTIC_RUN",
-        `Artifact ${artifact.id} is orphaned`,
-        `$.preparationArtifacts[${index}].id`,
-      );
-}
-
 function rerunAndVerify(run: CompletedValidatedMetricDiagnosticsRun): {
   review: DiagnosticReviewReceipt;
   result: DiagnosticDeepReadonly<NormalizedDiagnosticResultIdentity>;