@actuarial-ts/data 0.7.0 → 0.8.0

package/src/customizationExternalState.ts

@@ -0,0 +1,208 @@
+import {
+  isDiagnosticToken,
+  isWellFormedDiagnosticString,
+  snapshotDiagnosticJson,
+  type CustomizationResourceLimits,
+  type JsonValue,
+} from "@actuarial-ts/core";
+import { parseCustomizationResourceLimits } from "./customizationContracts.js";
+
+export type CustomizationExecutionErrorCode =
+  | "CANCELLED"
+  | "RESOURCE_LIMIT"
+  | "EXTERNAL_STATE_FAILURE"
+  | "EXTERNAL_STATE_ORDER"
+  | "EXTERNAL_STATE_TRUNCATED";
+
+export class CustomizationExecutionError extends Error {
+  readonly code: CustomizationExecutionErrorCode;
+
+  constructor(code: CustomizationExecutionErrorCode, message: string) {
+    super(message);
+    this.name = "CustomizationExecutionError";
+    this.code = code;
+    Object.freeze(this);
+  }
+}
+
+export interface CustomizationExternalRecord<T extends JsonValue = JsonValue> {
+  readonly id: string;
+  readonly partitionKey: string;
+  readonly orderKey: string;
+  readonly value: T;
+}
+
+export interface CustomizationExternalStateSession<T extends JsonValue = JsonValue> {
+  /** Store one SDK-owned record. The adapter must not mutate it. */
+  readonly append: (record: CustomizationExternalRecord<T>) => void | Promise<void>;
+  /** Finish writes before reads begin. */
+  readonly seal: () => void | Promise<void>;
+  /** Reopen all records exactly once in partitionKey/orderKey/id order. */
+  readonly read: () => AsyncIterable<CustomizationExternalRecord<T>>;
+  /** Release the run's temporary namespace for every terminal outcome. */
+  readonly dispose: (outcome: "completed" | "failed" | "cancelled") => void | Promise<void>;
+}
+
+export interface CustomizationExternalStateAdapter {
+  readonly open: <T extends JsonValue>(request: {
+    readonly runId: string;
+    readonly signal?: AbortSignal;
+  }) => CustomizationExternalStateSession<T> | Promise<CustomizationExternalStateSession<T>>;
+}
+
+export interface ExternalSortCustomizationRecordsInput<T extends JsonValue = JsonValue> {
+  readonly runId: string;
+  readonly records: Iterable<CustomizationExternalRecord<T>> | AsyncIterable<CustomizationExternalRecord<T>>;
+  readonly adapter: CustomizationExternalStateAdapter;
+  readonly limits: CustomizationResourceLimits;
+  readonly signal?: AbortSignal;
+}
+
+const encoder = new TextEncoder();
+
+function cancelled(signal: AbortSignal | undefined): void {
+  if (signal?.aborted)
+    throw new CustomizationExecutionError("CANCELLED", "Customization execution was cancelled");
+}
+
+function validateRecord<T extends JsonValue>(
+  value: CustomizationExternalRecord<T>,
+  maximumRecordBytes: number,
+): CustomizationExternalRecord<T> {
+  if (!isDiagnosticToken(value.id))
+    throw new CustomizationExecutionError("EXTERNAL_STATE_FAILURE", "External record id is invalid");
+  if (
+    !isWellFormedDiagnosticString(value.partitionKey) ||
+    !isWellFormedDiagnosticString(value.orderKey)
+  )
+    throw new CustomizationExecutionError(
+      "EXTERNAL_STATE_FAILURE",
+      "External record sort keys must be valid strings without U+0000",
+    );
+  const owned = snapshotDiagnosticJson(value);
+  const bytes = encoder.encode(JSON.stringify(owned)).byteLength;
+  if (bytes > maximumRecordBytes)
+    throw new CustomizationExecutionError(
+      "RESOURCE_LIMIT",
+      `External record ${value.id} requires ${bytes} encoded bytes, exceeding maximumRecordBytes ${maximumRecordBytes}`,
+    );
+  return owned;
+}
+
+function compareRecords(
+  left: CustomizationExternalRecord,
+  right: CustomizationExternalRecord,
+): number {
+  for (const key of ["partitionKey", "orderKey", "id"] as const) {
+    const result = left[key] === right[key] ? 0 : left[key] < right[key] ? -1 : 1;
+    if (result !== 0) return result;
+  }
+  return 0;
+}
+
+/**
+ * Validates, owns, externally stages, and deterministically rereads records.
+ * The adapter supplies storage mechanics only; it cannot choose actuarial
+ * ordering or publish a completed run. Early consumer return is cancellation.
+ */
+export async function* externalSortCustomizationRecords<T extends JsonValue>(
+  input: ExternalSortCustomizationRecordsInput<T>,
+): AsyncGenerator<CustomizationExternalRecord<T>> {
+  const runId = input.runId;
+  const records = input.records;
+  const adapter = input.adapter;
+  const signal = input.signal;
+  let limits: CustomizationResourceLimits;
+  try {
+    limits = parseCustomizationResourceLimits(input.limits);
+  } catch (error) {
+    throw new CustomizationExecutionError(
+      "RESOURCE_LIMIT",
+      error instanceof Error ? error.message : "Invalid customization resource limits",
+    );
+  }
+  if (!isDiagnosticToken(runId))
+    throw new CustomizationExecutionError("EXTERNAL_STATE_FAILURE", "runId is invalid");
+  cancelled(signal);
+
+  let session: CustomizationExternalStateSession<T> | undefined;
+  let outcome: "completed" | "failed" | "cancelled" = "failed";
+  let primary: unknown;
+  try {
+    session = await adapter.open<T>({
+      runId,
+      ...(signal === undefined ? {} : { signal }),
+    });
+    let written = 0;
+    for await (const raw of records) {
+      cancelled(signal);
+      written += 1;
+      if (written > limits.maximumInputRecords)
+        throw new CustomizationExecutionError(
+          "RESOURCE_LIMIT",
+          `Input exceeds maximumInputRecords ${limits.maximumInputRecords}`,
+        );
+      await session.append(validateRecord(raw, limits.maximumRecordBytes));
+    }
+    cancelled(signal);
+    await session.seal();
+
+    let read = 0;
+    let partitionRecords = 0;
+    let prior: CustomizationExternalRecord<T> | undefined;
+    for await (const raw of session.read()) {
+      cancelled(signal);
+      const record = validateRecord(raw, limits.maximumRecordBytes);
+      if (prior !== undefined && compareRecords(prior, record) > 0)
+        throw new CustomizationExecutionError(
+          "EXTERNAL_STATE_ORDER",
+          "External records are not in nondecreasing partitionKey/orderKey/id order",
+        );
+      partitionRecords = prior?.partitionKey === record.partitionKey ? partitionRecords + 1 : 1;
+      if (partitionRecords > limits.maximumPartitionRecords)
+        throw new CustomizationExecutionError(
+          "RESOURCE_LIMIT",
+          `Partition ${record.partitionKey} exceeds maximumPartitionRecords ${limits.maximumPartitionRecords}`,
+        );
+      read += 1;
+      if (read > written)
+        throw new CustomizationExecutionError(
+          "EXTERNAL_STATE_FAILURE",
+          "External state returned more records than were written",
+        );
+      prior = record;
+      yield record;
+    }
+    if (read !== written)
+      throw new CustomizationExecutionError(
+        "EXTERNAL_STATE_TRUNCATED",
+        `External state returned ${read} of ${written} records`,
+      );
+    outcome = "completed";
+  } catch (error) {
+    primary = error;
+    outcome =
+      error instanceof CustomizationExecutionError && error.code === "CANCELLED"
+        ? "cancelled"
+        : "failed";
+    throw error;
+  } finally {
+    if (session !== undefined) {
+      try {
+        const terminal =
+          outcome === "completed"
+            ? "completed"
+            : primary === undefined
+              ? "cancelled"
+              : outcome;
+        await session.dispose(terminal);
+      } catch (error) {
+        if (primary === undefined)
+          throw new CustomizationExecutionError(
+            "EXTERNAL_STATE_FAILURE",
+            `External state cleanup failed: ${error instanceof Error ? error.message : String(error)}`,
+          );
+      }
+    }
+  }
+}

package/src/customizationHistoryStream.ts

@@ -0,0 +1,90 @@
+import {
+  canonicalJson,
+  fnv1a64,
+  prepareHistoricalDataset,
+  type CustomizationCapabilityFinding,
+  type CustomizationResourceLimits,
+  type HistoricalDatasetInput,
+  type HistoricalObservationRecord,
+  type HistoricalRecordDisposition,
+  type JsonValue,
+  type PreparedHistoricalObservation,
+} from "@actuarial-ts/core";
+import { parseHistoricalDatasetInput, parseHistoricalObservationRecord } from "./customizationContracts.js";
+import {
+  externalSortCustomizationRecords,
+  type CustomizationExternalRecord,
+  type CustomizationExternalStateAdapter,
+} from "./customizationExternalState.js";
+
+export type PreparedHistoryStreamEvent =
+  | { readonly kind: "observation"; readonly observation: PreparedHistoricalObservation }
+  | { readonly kind: "disposition"; readonly disposition: HistoricalRecordDisposition }
+  | { readonly kind: "finding"; readonly finding: CustomizationCapabilityFinding };
+
+function partitionKey(record: HistoricalObservationRecord): string {
+  return canonicalJson([record.sourceNamespace, record.claimId]);
+}
+
+function orderKey(record: HistoricalObservationRecord, precedence: HistoricalDatasetInput["revisionPrecedence"]): string {
+  const order = precedence === "sequence"
+    ? String(record.revision.sequence ?? -1).padStart(20, "0")
+    : record.revision.correctedAt ?? "";
+  return canonicalJson([record.recordId, record.evaluationDate ?? record.effectiveDate ?? "", order, record.revision.id]);
+}
+
+/**
+ * Sorts raw revisions in host-owned state and prepares one claim partition at
+ * a time. Retained SDK state is bounded by maximumPartitionRecords; oversized
+ * claims fail rather than silently spilling or dropping observations.
+ */
+export async function* streamPreparedHistoricalDataset(input: {
+  readonly runId: string;
+  readonly header: Omit<HistoricalDatasetInput, "records">;
+  readonly records: AsyncIterable<unknown> | Iterable<unknown>;
+  readonly adapter: CustomizationExternalStateAdapter;
+  readonly limits: CustomizationResourceLimits;
+  readonly signal?: AbortSignal;
+}): AsyncGenerator<PreparedHistoryStreamEvent> {
+  const header = parseHistoricalDatasetInput({ ...input.header, records: [] });
+  async function* externalRecords(): AsyncGenerator<CustomizationExternalRecord<JsonValue>> {
+    for await (const raw of input.records) {
+      const record = parseHistoricalObservationRecord(raw);
+      const stable = canonicalJson([record.sourceNamespace, record.claimId, record.recordId, record.revision.id]);
+      yield {
+        id: `record-${fnv1a64(stable)}`,
+        partitionKey: partitionKey(record),
+        orderKey: orderKey(record, header.revisionPrecedence),
+        value: record as unknown as JsonValue,
+      };
+    }
+  }
+  let currentPartition: string | undefined;
+  let current: HistoricalObservationRecord[] = [];
+  const flush = function* (): Generator<PreparedHistoryStreamEvent> {
+    if (current.length === 0) return;
+    const sourceNamespace = current[0]!.sourceNamespace;
+    const claimId = current[0]!.claimId;
+    const prepared = prepareHistoricalDataset({
+      ...header,
+      ...(header.openingBalances === undefined ? {} : { openingBalances: header.openingBalances.filter((balance) => balance.sourceNamespace === sourceNamespace && balance.claimId === claimId) }),
+      records: current,
+    });
+    for (const disposition of prepared.dispositions) yield { kind: "disposition", disposition };
+    for (const finding of prepared.findings) yield { kind: "finding", finding };
+    for (const observation of prepared.observations) yield { kind: "observation", observation };
+    current = [];
+  };
+  for await (const external of externalSortCustomizationRecords({
+    runId: input.runId,
+    records: externalRecords(),
+    adapter: input.adapter,
+    limits: input.limits,
+    ...(input.signal === undefined ? {} : { signal: input.signal }),
+  })) {
+    if (currentPartition !== undefined && external.partitionKey !== currentPartition) yield* flush();
+    currentPartition = external.partitionKey;
+    current.push(parseHistoricalObservationRecord(external.value));
+  }
+  yield* flush();
+}

package/src/diagnosticInput.ts

@@ -3,6 +3,10 @@ import {
   compileDiagnosticDefinition,
   prepareDiagnosticData,
   prepareDiagnosticDataCompact,
+  reselectCompactDiagnosticMetrics,
+  selectCompactDiagnosticSourceGroups,
+  selectCompactDiagnosticOrigins,
+  selectCompactDiagnosticCoordinates,
   runMetricDiagnostics,
   runMetricDiagnosticsCompact,
   validateDiagnosticGroupingConfiguration,
@@ -307,6 +311,38 @@ const compactPreparedByInput = new WeakMap<
   CompactValidatedDiagnosticRunInput,
   CompactPreparedDiagnosticData
 >();
+
+declare const queryContextBrand: unique symbol;
+/** SDK-owned source snapshot. Cloning or serializing this handle loses authority. */
+export interface DiagnosticQueryContext {
+  readonly [queryContextBrand]: true;
+}
+
+/**
+ * Omitted fields inherit the context; null clears filter/expected-cell selection.
+ * Group maps/dimensions must match the selected scope; empty records clear them.
+ */
+export interface DiagnosticQuery {
+  /** Per-query evidence labels; null clears the inherited label. */
+  readonly runPresetId?: string | null;
+  readonly datasetArtifactId?: string | null;
+  readonly filter?: DiagnosticsFilter | null;
+  readonly completePeriodCutoffs?: readonly DiagnosticCompletePeriodCutoff[];
+  readonly expectedCells?: readonly DiagnosticExpectedCell[] | null;
+  readonly groupMap?: Readonly<Record<string, string>>;
+  readonly groupDimensions?: Readonly<Record<string, JsonValue>>;
+}
+
+const querySchema = z.object({
+  runPresetId: tokenSchema.nullable().optional(),
+  datasetArtifactId: tokenSchema.nullable().optional(),
+  filter: filterSchema.nullable().optional(),
+  completePeriodCutoffs: z.array(cutoffSchema).optional(),
+  expectedCells: z.array(expectedSchema).nullable().optional(),
+  groupMap: recordSchema(tokenSchema).optional(),
+  groupDimensions: recordSchema(jsonSchema).optional(),
+}).strict();
+const queryContexts = new WeakMap<DiagnosticQueryContext, CompactValidatedDiagnosticRunInput>();
 const compactCompleted = new WeakSet<object>();
 // Retain the exact immutable validated input only while its completed run lives.
 // Reconstructing from the audit would lose the original optional/raw-value form.
@@ -509,6 +545,71 @@ export function validateDiagnosticRunInputCompact(
   return result;
 }
 
+/**
+ * Own and validate source rows and compile the definition once. Queries keep the
+ * definition, execution policy, source order and review evidence fixed.
+ * Metric selections and eligible clean aggregate source/origin/date/age queries
+ * share prepared cells; ineligible inputs still perform full core preparation.
+ * Source/origin queries reuse invariant owned review evidence. Date/age queries
+ * reuse cell-local checks but recompute monotonic comparisons and controls. Every
+ * query enforces review and metric gates, including unchanged evidence validation.
+ */
+export function createDiagnosticQueryContext(value: unknown): DiagnosticQueryContext {
+  const validated = validateDiagnosticRunInputCompact(value);
+  const handle = Object.freeze({}) as DiagnosticQueryContext;
+  queryContexts.set(handle, validated);
+  return handle;
+}
+
+/** Create an authentic per-query input without cloning/reparsing owned source rows. */
+export function validateDiagnosticQueryCompact(
+  context: DiagnosticQueryContext,
+  query: unknown = {},
+): CompactValidatedDiagnosticRunInput {
+  const base = queryContexts.get(context);
+  if (!base) throw new DiagnosticValidationError([{
+    domain: "input",
+    code: "invalid-input-relationship",
+    path: "$",
+    message: "Value is not an authentic diagnostic query context",
+  }]);
+  const undefinedIssues = explicitUndefinedIssues(query);
+  if (undefinedIssues.length) throw new DiagnosticValidationError(undefinedIssues);
+  const parsed = querySchema.safeParse(query);
+  if (!parsed.success) throw issues(parsed.error);
+  if (Object.keys(parsed.data).length === 0) return base;
+  const ownedQuery = freeze({
+    ...parsed.data,
+    ...(parsed.data.groupMap === undefined ? {} : { groupMap: sortedRecord<string>(parsed.data.groupMap) }),
+    ...(parsed.data.groupDimensions === undefined ? {} : { groupDimensions: sortedRecord<JsonValue>(parsed.data.groupDimensions) }),
+  });
+  // Only the new query fields require freezing. Every inherited graph belongs
+  // to the authentic validated base; never trust caller Object.freeze markers.
+  const result = Object.freeze({ ...base, ...ownedQuery }) as CompactValidatedDiagnosticRunInput;
+  const projected = Object.keys(ownedQuery).every((key) => ["filter", "groupMap", "groupDimensions", "runPresetId", "datasetArtifactId"].includes(key))
+    ? reselectCompactDiagnosticMetrics(compactPreparedByInput.get(base)!, result.filter ?? undefined) ??
+      selectCompactDiagnosticSourceGroups(compactPreparedByInput.get(base)!, result.filter ?? undefined) ??
+      selectCompactDiagnosticOrigins(compactPreparedByInput.get(base)!, result.filter ?? undefined) ??
+      selectCompactDiagnosticCoordinates(compactPreparedByInput.get(base)!, result.filter ?? undefined)
+    : undefined;
+  const prepared = projected ?? prepareDiagnosticDataCompact(preparationInput(result));
+  validateCompactDiagnosticGroupingConfiguration({
+    prepared,
+    groupMap: result.groupMap,
+    groupDimensions: result.groupDimensions,
+  });
+  compactPreparedByInput.set(result, prepared);
+  return result;
+}
+
+/** Execute all review and metric gates for this scope, even if another scope passed. */
+export function runDiagnosticQueryCompact(
+  context: DiagnosticQueryContext,
+  query: DiagnosticQuery = {},
+): CompactMetricDiagnosticsOutcome {
+  return runValidatedMetricDiagnosticsCompact(validateDiagnosticQueryCompact(context, query));
+}
+
 export function assertCompactValidatedDiagnosticRunInput(
   value: unknown,
 ): asserts value is CompactValidatedDiagnosticRunInput {

package/src/diagnosticPreparedReview.ts

@@ -747,6 +747,30 @@ const compactReceiptFingerprints = new WeakMap<
   CompactDiagnosticReviewReceipt,
   string
 >();
+
+// At most one review per live immutable cell graph. Weak ownership releases the
+// retained report with its preparation; changed evidence replaces, never grows,
+// the entry. Metric selection does not affect rule evaluation or structural
+// checks, but every source, definition and evidence dependency below does.
+const reusableCompactReviews = new WeakMap<object, {
+  prepared: CompactPreparedDiagnosticData;
+  evidenceKey: string;
+  scopeKey: string;
+  receipt: CompactDiagnosticReviewReceipt;
+}>();
+
+function reviewScopeKey(prepared: CompactPreparedDiagnosticData): string {
+  const { instanceIds: _instances, ...scope } = prepared.filter ?? {};
+  return canonicalJson(scope);
+}
+
+function sameReviewSourceGraph(left: CompactPreparedDiagnosticData, right: CompactPreparedDiagnosticData): boolean {
+  return left.definition === right.definition && left.cells === right.cells &&
+    left.inputAudit === right.inputAudit && left.exposures === right.exposures &&
+    left.completePeriodCutoffs === right.completePeriodCutoffs &&
+    left.expectedCellsProvided === right.expectedCellsProvided &&
+    left.expectedCells === right.expectedCells && left.findings === right.findings;
+}
 function compactError(message: string, path = "$"): never {
   throw new DiagnosticValidationError([
     { domain: "input", code: "invalid-input-relationship", path, message },
@@ -935,6 +959,24 @@ export function reviewPreparedDiagnosticDataCompact(
     input.evidence === null
       ? null
       : validateDiagnosticReviewEvidence(input.evidence);
+  // Validated evidence has a fixed JSON schema. Preserve signed zero in its
+  // numeric slots; identity JSON intentionally normalizes it, cache ownership
+  // must not silently substitute a different caller's raw evidence.
+  const evidenceKey = JSON.stringify(evidence, (_key, value: unknown) =>
+    typeof value === "number" && Object.is(value, -0) ? { negativeZero: true } : value,
+  );
+  const scopeKey = reviewScopeKey(input.prepared);
+  const previous = reusableCompactReviews.get(input.prepared.cells);
+  if (previous && previous.evidenceKey === evidenceKey && previous.scopeKey === scopeKey &&
+    sameReviewSourceGraph(previous.prepared, input.prepared)) {
+    // The content is genuinely evaluated evidence, not a copied passing flag.
+    // Bind the new receipt to this preparation so its exported fingerprint
+    // includes this query's actual metric selection and preparation identity.
+    const receipt = Object.freeze({ ...previous.receipt, evidence });
+    compactReceipts.add(receipt);
+    compactReceiptPreparations.set(receipt, input.prepared);
+    return receipt;
+  }
   const evaluations = evaluateDiagnosticReviewRulesCompact(input.prepared);
   const table = new CompactDiagnosticJson();
   const blocks: FindingBlock[] = [];
@@ -1136,6 +1178,12 @@ export function reviewPreparedDiagnosticDataCompact(
   });
   compactReceipts.add(receipt);
   compactReceiptPreparations.set(receipt, input.prepared);
+  // Very large supplied evidence remains fully reviewed, simply not cached.
+  if (evidenceKey.length <= 65_536 && scopeKey.length <= 65_536) {
+    reusableCompactReviews.set(input.prepared.cells, {
+      prepared: input.prepared, evidenceKey, scopeKey, receipt,
+    });
+  }
   return receipt;
 }