@actuarial-ts/data 0.9.0 → 0.11.0

package/src/customizationContracts.ts

@@ -250,7 +250,10 @@ export const historicalDatasetInputSchema = z
     sources: z
       .array(
         z
-          .object({ namespace: token, artifactId: token, mappingVersion: token })
+          .object({
+            namespace: token, artifactId: token, mappingVersion: token,
+            additionalArtifacts: z.array(z.object({ artifactId: token, mappingVersion: token }).strict()).min(1).optional(),
+          })
           .strict(),
       )
       .min(1),
@@ -273,7 +276,7 @@ export const historicalDatasetInputSchema = z
   .strict()
   .superRefine((input, context) => {
     const namespaces = new Set<string>();
-    const artifactByNamespace = new Map<string, string>();
+    const artifactByNamespace = new Map<string, Set<string>>();
     input.sources.forEach((source, index) => {
       if (namespaces.has(source.namespace))
         context.addIssue({
@@ -282,7 +285,13 @@ export const historicalDatasetInputSchema = z
           message: "Source namespace is duplicated",
         });
       namespaces.add(source.namespace);
-      artifactByNamespace.set(source.namespace, source.artifactId);
+      const artifacts = new Set([source.artifactId]);
+      source.additionalArtifacts?.forEach((artifact, position) => {
+        if (artifacts.has(artifact.artifactId))
+          context.addIssue({ code: z.ZodIssueCode.custom, path: ["sources", index, "additionalArtifacts", position, "artifactId"], message: "Artifact ID is duplicated within the source namespace" });
+        artifacts.add(artifact.artifactId);
+      });
+      artifactByNamespace.set(source.namespace, artifacts);
     });
     input.records.forEach((record, index) => {
       if (!namespaces.has(record.sourceNamespace))
@@ -293,7 +302,7 @@ export const historicalDatasetInputSchema = z
         });
       if (
         artifactByNamespace.has(record.sourceNamespace) &&
-        artifactByNamespace.get(record.sourceNamespace) !== record.source.artifactId
+        !artifactByNamespace.get(record.sourceNamespace)!.has(record.source.artifactId)
       )
         context.addIssue({
           code: z.ZodIssueCode.custom,
@@ -391,7 +400,7 @@ export const historicalDatasetInputSchema = z
         });
       if (
         artifactByNamespace.has(balance.sourceNamespace) &&
-        artifactByNamespace.get(balance.sourceNamespace) !== balance.source.artifactId
+        !artifactByNamespace.get(balance.sourceNamespace)!.has(balance.source.artifactId)
       )
         context.addIssue({
           code: z.ZodIssueCode.custom,
@@ -422,7 +431,7 @@ const statistic = z.discriminatedUnion("kind", [
       numeratorMeasureId: token,
       denominatorMeasureId: token,
       scale: finite,
-      denominatorRule: z.enum(["positive", "nonzero"]),
+      denominatorRule: z.literal("positive"),
     })
     .strict(),
   z

package/src/diagnosticInput.ts

@@ -350,12 +350,33 @@ const compactInputByCompletedRun = new WeakMap<
   CompletedCompactMetricDiagnosticsRun,
   CompactValidatedDiagnosticRunInput
 >();
-function freeze<T>(value: T, seen = new WeakSet<object>()): DiagnosticDeepReadonly<T> {
-  if (value === null || typeof value !== "object" || seen.has(value))
-    return value as DiagnosticDeepReadonly<T>;
-  seen.add(value);
-  for (const child of Object.values(value as Record<string, unknown>)) freeze(child, seen);
-  return Object.freeze(value) as DiagnosticDeepReadonly<T>;
+// Validated inputs are JSON-shaped trees (the schema copies every row), so a
+// shared node is a small leaf visited once per reference rather than tracked in
+// a table of every node. Ancestors are recorded only past this depth, which a
+// cycle in a caller-supplied graph must exceed before it is cut.
+const UNGUARDED_FREEZE_DEPTH = 64;
+function freeze<T>(value: T, skip?: WeakSet<object>): DiagnosticDeepReadonly<T> {
+  let ancestors: Set<object> | undefined;
+  const visit = (node: unknown, depth: number): void => {
+    if (node === null || typeof node !== "object" || skip?.has(node)) return;
+    const guarded = depth > UNGUARDED_FREEZE_DEPTH;
+    if (guarded) {
+      ancestors ??= new Set();
+      if (ancestors.has(node)) return;
+      ancestors.add(node);
+    }
+    if (Array.isArray(node)) {
+      for (let index = 0; index < node.length; index++) visit(node[index], depth + 1);
+    } else {
+      const record = node as Record<string, unknown>;
+      const keys = Object.keys(record);
+      for (let index = 0; index < keys.length; index++) visit(record[keys[index]!], depth + 1);
+    }
+    if (guarded) ancestors!.delete(node);
+    Object.freeze(node);
+  };
+  visit(value, 0);
+  return value as DiagnosticDeepReadonly<T>;
 }
 function issues(error: z.ZodError): DiagnosticValidationError {
   return new DiagnosticValidationError(
@@ -382,22 +403,39 @@ function sortedRecord<T>(value: Readonly<Record<string, T>>): Readonly<Record<st
   return result;
 }
 
+interface UndefinedScanFrame {
+  readonly value: unknown;
+  readonly parent: UndefinedScanFrame | null;
+  readonly key: string | null;
+}
+/** Renders the JSON path of a frame only when an issue must cite it. */
+function scanFramePath(frame: UndefinedScanFrame): string {
+  if (frame.parent === null || frame.key === null) return "$";
+  const parentPath = scanFramePath(frame.parent);
+  return Array.isArray(frame.parent.value)
+    ? `${parentPath}[${frame.key}]`
+    : /^[A-Za-z_$][\w$]*$/.test(frame.key)
+      ? `${parentPath}.${frame.key}`
+      : `${parentPath}[${JSON.stringify(frame.key)}]`;
+}
 function explicitUndefinedIssues(value: unknown): DiagnosticValidationIssue[] {
   const found: DiagnosticValidationIssue[] = [];
-  const stack: { readonly value: unknown; readonly path: string }[] = [{ value, path: "$" }];
+  const stack: UndefinedScanFrame[] = [{ value, parent: null, key: null }];
   const seen = new WeakSet<object>();
   while (stack.length > 0) {
     const current = stack.pop()!;
     if (current.value === null || typeof current.value !== "object" || seen.has(current.value))
       continue;
     seen.add(current.value);
-    for (const [key, child] of Object.entries(current.value)) {
-      const path = Array.isArray(current.value)
-        ? `${current.path}[${key}]`
-        : /^[A-Za-z_$][\w$]*$/.test(key)
-          ? `${current.path}.${key}`
-          : `${current.path}[${JSON.stringify(key)}]`;
-      if (child === undefined)
+    const record = current.value as Record<string, unknown>;
+    // Large inputs are ordinary: build no path text or entry pairs on the
+    // common path, only the frame links needed to render a path on demand.
+    const keys = Object.keys(record);
+    for (let index = 0; index < keys.length; index++) {
+      const key = keys[index]!;
+      const child = record[key];
+      if (child === undefined) {
+        const path = scanFramePath({ value: child, parent: current, key });
         found.push({
           domain: path.startsWith("$.definition")
             ? "definition"
@@ -410,21 +448,188 @@ function explicitUndefinedIssues(value: unknown): DiagnosticValidationIssue[] {
           path,
           message: "Explicit undefined is not allowed",
         });
-      else stack.push({ value: child, path });
+      } else if (child !== null && typeof child === "object")
+        stack.push({ value: child, parent: current, key });
     }
   }
   return found;
 }
 
+// ---------------------------------------------------------------------------
+// Large-input fast path for the two row arrays. Zod validates and copies every
+// row through generic machinery, which on 100,000-row inputs allocates more
+// than the preparation itself. Rows are accepted here only under exactly the
+// schema's rules (plain records, strict keys, token strings, boolean flags,
+// safe non-negative integer source rows, raw number-or-null measures) and are
+// copied in the schema's key order, so the owned rows are indistinguishable
+// from the schema's output. Anything the check does not accept, including any
+// non-plain prototype, is handed to the schema unchanged, so every rejection
+// and every error message still comes from zod.
+type FastLossRow = z.output<typeof lossSchema>;
+type FastExposureRow = z.output<typeof exposureSchema>;
+const isPlainRecord = (value: unknown): value is Record<string, unknown> => {
+  if (value === null || typeof value !== "object" || Array.isArray(value)) return false;
+  const prototype: unknown = Object.getPrototypeOf(value);
+  return prototype === Object.prototype || prototype === null;
+};
+const hasOwn = (value: object, key: string): boolean => Object.prototype.hasOwnProperty.call(value, key);
+function fastSource(value: unknown): z.output<typeof sourceSchema> | undefined {
+  if (!isPlainRecord(value)) return undefined;
+  const artifactId = value.artifactId;
+  if (!isDiagnosticToken(artifactId)) return undefined;
+  const out: Record<string, unknown> = { artifactId };
+  let matched = 1;
+  for (const field of ["sourceFile", "sourceSheet"] as const)
+    if (hasOwn(value, field)) {
+      const item = value[field];
+      if (!isDiagnosticToken(item)) return undefined;
+      out[field] = item;
+      matched++;
+    }
+  if (hasOwn(value, "sourceRow")) {
+    const item = value.sourceRow;
+    if (typeof item !== "number" || !Number.isSafeInteger(item) || item < 0) return undefined;
+    out.sourceRow = item;
+    matched++;
+  }
+  if (hasOwn(value, "sourceCell")) {
+    const item = value.sourceCell;
+    if (!isDiagnosticToken(item)) return undefined;
+    out.sourceCell = item;
+    matched++;
+  }
+  if (Object.keys(value).length !== matched) return undefined;
+  return out as z.output<typeof sourceSchema>;
+}
+function fastMeasures(value: unknown): Record<string, number | null> | undefined {
+  if (!isPlainRecord(value)) return undefined;
+  const out: Record<string, number | null> = {};
+  const keys = Object.keys(value);
+  for (let index = 0; index < keys.length; index++) {
+    const key = keys[index]!;
+    const item = value[key];
+    if (item !== null && typeof item !== "number") return undefined;
+    // The schema's key round trip keeps a literal "__proto__" as own data.
+    if (key === "__proto__")
+      Object.defineProperty(out, key, { value: item, enumerable: true, writable: true, configurable: true });
+    else out[key] = item;
+  }
+  return out;
+}
+function fastLossRow(value: unknown): FastLossRow | undefined {
+  if (!isPlainRecord(value)) return undefined;
+  const { recordId, sourceGroup, origin, valuation, complete, rowType } = value;
+  if (
+    !isDiagnosticToken(recordId) ||
+    !isDiagnosticToken(sourceGroup) ||
+    !isDiagnosticToken(origin) ||
+    !isDiagnosticToken(valuation) ||
+    typeof complete !== "boolean"
+  )
+    return undefined;
+  const out: Record<string, unknown> = { recordId, sourceGroup, origin, valuation, complete };
+  let expected = 7;
+  if (hasOwn(value, "source")) {
+    const source = fastSource(value.source);
+    if (source === undefined) return undefined;
+    out.source = source;
+    expected++;
+  }
+  const measures = fastMeasures(value.measures);
+  if (measures === undefined) return undefined;
+  out.measures = measures;
+  if (rowType === "claim") {
+    const claimId = value.claimId;
+    if (!hasOwn(value, "claimId") || !isDiagnosticToken(claimId)) return undefined;
+    out.rowType = "claim";
+    out.claimId = claimId;
+    expected++;
+  } else if (rowType === "aggregate") out.rowType = "aggregate";
+  else return undefined;
+  if (Object.keys(value).length !== expected) return undefined;
+  return out as FastLossRow;
+}
+function fastExposureRow(value: unknown): FastExposureRow | undefined {
+  if (!isPlainRecord(value)) return undefined;
+  const { key, sourceGroup, origin, measureId, complete } = value;
+  const measure = value.value;
+  if (
+    !isDiagnosticToken(key) ||
+    !isDiagnosticToken(sourceGroup) ||
+    !isDiagnosticToken(origin) ||
+    !isDiagnosticToken(measureId) ||
+    (measure !== null && typeof measure !== "number") ||
+    typeof complete !== "boolean"
+  )
+    return undefined;
+  const out: Record<string, unknown> = { key, sourceGroup, origin };
+  let expected = 6;
+  if (hasOwn(value, "valuation")) {
+    const valuation = value.valuation;
+    if (!isDiagnosticToken(valuation)) return undefined;
+    out.valuation = valuation;
+    expected++;
+  }
+  out.measureId = measureId;
+  out.value = measure;
+  out.complete = complete;
+  if (hasOwn(value, "source")) {
+    const source = fastSource(value.source);
+    if (source === undefined) return undefined;
+    out.source = source;
+    expected++;
+  }
+  if (Object.keys(value).length !== expected) return undefined;
+  return out as FastExposureRow;
+}
+/**
+ * Validates and copies the row arrays directly when every row is schema-shaped.
+ * Returns the owned rows plus a shell of the input with empty row arrays for
+ * the schema to validate everything else; undefined defers the whole input.
+ */
+function fastSourceRows(
+  value: unknown,
+): { shell: Record<string, unknown>; losses: FastLossRow[]; exposures: FastExposureRow[] | undefined } | undefined {
+  if (!isPlainRecord(value) || !Array.isArray(value.losses)) return undefined;
+  const losses: FastLossRow[] = [];
+  for (let index = 0; index < value.losses.length; index++) {
+    const row = fastLossRow(value.losses[index]);
+    if (row === undefined) return undefined;
+    losses.push(row);
+  }
+  let exposures: FastExposureRow[] | undefined;
+  if (hasOwn(value, "exposures")) {
+    if (!Array.isArray(value.exposures)) return undefined;
+    exposures = [];
+    for (let index = 0; index < value.exposures.length; index++) {
+      const row = fastExposureRow(value.exposures[index]);
+      if (row === undefined) return undefined;
+      exposures.push(row);
+    }
+  }
+  return {
+    shell: { ...value, losses: [], ...(exposures === undefined ? {} : { exposures: [] }) },
+    losses,
+    exposures,
+  };
+}
+
 // Both public gateways share the same full validation/ownership boundary.
 // Selecting compact storage never invokes the eager preparation first.
 function validateRunInputContent(value: unknown): DiagnosticRunInputContent {
-  const undefinedIssues = explicitUndefinedIssues(value);
+  // A row the fast check accepts holds no explicit undefined anywhere: every
+  // field it reads must be a token, boolean, or number-or-null, and any other
+  // value (including undefined) makes it decline the whole input. So when it
+  // accepts, scanning its row-free shell reports exactly the same issues.
+  const fast = fastSourceRows(value);
+  const undefinedIssues = explicitUndefinedIssues(fast === undefined ? value : fast.shell);
   if (undefinedIssues.length > 0) throw new DiagnosticValidationError(undefinedIssues);
-  const parsed = runSchema.safeParse(value);
+  const parsed = runSchema.safeParse(fast === undefined ? value : fast.shell);
   if (!parsed.success) throw issues(parsed.error);
+  const losses = fast === undefined ? parsed.data.losses : fast.losses;
+  const exposures = (fast === undefined ? parsed.data.exposures : fast.exposures) ?? [];
   const definition = compileDiagnosticDefinition(parsed.data.definition as DiagnosticDefinition);
-  const relationIssues: DiagnosticValidationIssue[] = parsed.data.losses.flatMap((row, index) =>
+  const relationIssues: DiagnosticValidationIssue[] = losses.flatMap((row, index) =>
     row.rowType === definition.definition.lossRowGrain
       ? []
       : [
@@ -436,7 +641,7 @@ function validateRunInputContent(value: unknown): DiagnosticRunInputContent {
           },
         ],
   );
-  for (const [index, row] of (parsed.data.exposures ?? []).entries()) {
+  for (const [index, row] of exposures.entries()) {
     const measure = definition.definition.measures.find((item) => item.id === row.measureId);
     if (measure?.exposureTiming === "valuation-specific" && row.valuation === undefined)
       relationIssues.push({
@@ -485,8 +690,8 @@ function validateRunInputContent(value: unknown): DiagnosticRunInputContent {
       ]);
   const result = freeze({
     definition,
-    losses: parsed.data.losses,
-    exposures: parsed.data.exposures ?? [],
+    losses,
+    exposures,
     filter: parsed.data.filter ?? null,
     completePeriodCutoffs: parsed.data.completePeriodCutoffs ?? [],
     expectedCells: parsed.data.expectedCells ?? null,

package/src/historyMapping.ts

@@ -1,5 +1,6 @@
 import {
   DiagnosticValidationError,
+  canonicalJson,
   diagnosticJsonPreflight,
   isDiagnosticToken,
   isRealIsoDate,
@@ -13,6 +14,12 @@ export interface HistoricalColumnSelector {
   readonly columns: readonly string[];
 }
 
+/** Identity parts are required, ordered strings; each part may declare column aliases. */
+export type HistoricalIdentitySelector = HistoricalColumnSelector | {
+  readonly kind: "composite";
+  readonly parts: readonly (HistoricalColumnSelector & { readonly format?: HistoricalDateSelector["format"] })[];
+};
+
 export interface HistoricalDateSelector extends HistoricalColumnSelector {
   readonly format: "iso-yyyy-mm-dd" | "mdy-slash" | "dmy-slash";
 }
@@ -40,8 +47,8 @@ export interface HistoricalSourceMapping {
   readonly grain: HistoricalObservationRecord["grain"];
   readonly completeness: HistoricalObservationRecord["completeness"];
   readonly fields: {
-    readonly recordId: HistoricalColumnSelector;
-    readonly claimId: HistoricalColumnSelector;
+    readonly recordId: HistoricalIdentitySelector;
+    readonly claimId: HistoricalIdentitySelector;
     readonly componentId?: HistoricalColumnSelector;
     readonly accidentDate?: HistoricalDateSelector;
     readonly reportDate?: HistoricalDateSelector;
@@ -58,7 +65,7 @@ export interface HistoricalSourceMapping {
     readonly coverageIds?: HistoricalColumnSelector & { readonly delimiter: string };
   };
   readonly revision: {
-    readonly id: HistoricalColumnSelector;
+    readonly id: HistoricalIdentitySelector;
     readonly action?: HistoricalColumnSelector;
     readonly sequence?: HistoricalNumberSelector;
     readonly correctedAt?: HistoricalColumnSelector;
@@ -98,6 +105,15 @@ export interface HistoricalMappingResult {
 
 const token = z.string().refine(isDiagnosticToken, "Expected a valid nonempty token");
 const columnSelector = z.object({ columns: z.array(token).min(1) }).strict();
+const identitySelector = z.union([
+  columnSelector,
+  z.object({
+    kind: z.literal("composite"),
+    parts: z.array(columnSelector.extend({
+      format: z.enum(["iso-yyyy-mm-dd", "mdy-slash", "dmy-slash"]).optional(),
+    }).strict()).min(2).max(8),
+  }).strict(),
+]);
 const dateSelector = columnSelector.extend({
   format: z.enum(["iso-yyyy-mm-dd", "mdy-slash", "dmy-slash"]),
 }).strict();
@@ -148,8 +164,8 @@ export const historicalSourceMappingSchema = z
     completeness: z.enum(["complete-snapshot", "partial-snapshot", "change-only"]),
     fields: z
       .object({
-        recordId: columnSelector,
-        claimId: columnSelector,
+        recordId: identitySelector,
+        claimId: identitySelector,
         componentId: columnSelector.optional(),
         accidentDate: dateSelector.optional(),
         reportDate: dateSelector.optional(),
@@ -171,7 +187,7 @@ export const historicalSourceMappingSchema = z
       .optional(),
     revision: z
       .object({
-        id: columnSelector,
+        id: identitySelector,
         action: columnSelector.optional(),
         sequence: numberSelector.optional(),
         correctedAt: columnSelector.optional(),
@@ -317,7 +333,11 @@ function splitIds(value: HistoricalScalar | undefined, delimiter: string): strin
 
 function selectorColumns(mapping: HistoricalSourceMapping): Set<string> {
   const result = new Set<string>();
-  const add = (selector: HistoricalColumnSelector | undefined) => selector?.columns.forEach((column) => result.add(column));
+  const add = (selector: HistoricalIdentitySelector | undefined) => {
+    if (selector === undefined) return;
+    for (const part of "parts" in selector ? selector.parts : [selector])
+      part.columns.forEach((column) => result.add(column));
+  };
   Object.values(mapping.fields).forEach(add);
   Object.values(mapping.measures).forEach(add);
   Object.values(mapping.dimensions ?? {}).forEach(add);
@@ -355,8 +375,19 @@ export function mapHistoricalSourceRows(
       issues.push({ path: "$.rowNumber", message: "rowNumber must be a positive safe integer" });
     const field = (selector: HistoricalColumnSelector | undefined, name: string, required = false) =>
       selector === undefined ? undefined : selectedValue(row, selector, `$.${name}`, issues, required);
-    const recordId = scalarText(field(mapping.fields.recordId, "recordId", true) ?? "");
-    const claimId = scalarText(field(mapping.fields.claimId, "claimId", true) ?? "");
+    const identity = (selector: HistoricalIdentitySelector, name: string) => {
+      if (!("parts" in selector)) return scalarText(field(selector, name, true) ?? "");
+      const parts = selector.parts.map((part, index) => {
+        const partName = `${name}.parts[${index}]`;
+        const value = field(part, partName, true);
+        return part.format === undefined
+          ? scalarText(value ?? "")
+          : parseDateValue(value, part.format, `$.${partName}`, issues) ?? "";
+      });
+      return `tuple/1:${canonicalJson(parts)}`;
+    };
+    const recordId = identity(mapping.fields.recordId, "recordId");
+    const claimId = identity(mapping.fields.claimId, "claimId");
     const componentIdValue = field(mapping.fields.componentId, "componentId", mapping.grain === "claim-component-snapshot");
     const statusValue = field(mapping.fields.status, "status");
     const mappedDate = (selector: HistoricalDateSelector | undefined, name: string, required = false) => {
@@ -423,7 +454,7 @@ export function mapHistoricalSourceRows(
     const coverageIds = relationship?.coverageIds
       ? splitIds(field(relationship.coverageIds, "relationships.coverageIds"), relationship.coverageIds.delimiter)
       : undefined;
-    const revisionId = scalarText(field(mapping.revision.id, "revision.id", true) ?? "");
+    const revisionId = identity(mapping.revision.id, "revision.id");
     const actionText = scalarText(field(mapping.revision.action, "revision.action") ?? "upsert").toLowerCase();
     if (actionText !== "upsert" && actionText !== "delete")
       issues.push({ path: "$.revision.action", message: "Revision action must be upsert or delete" });

package/src/version.ts

@@ -1 +1 @@
-export const DATA_PACKAGE_VERSION = "0.9.0";
+export const DATA_PACKAGE_VERSION = "0.11.0";