@actuarial-ts/core 0.6.1 → 0.7.0

package/src/canonical.ts

@@ -1,4 +1,5 @@
 import { ReservingError } from "./types.js";
+import { createFnvAccumulator } from "./fnvAccumulator.js";
 
 /**
  * Canonical JSON serialization and the FNV-1a integrity hash — the SDK's
@@ -34,18 +35,35 @@ function isPlainObject(value: unknown): value is Record<string, unknown> {
   return proto === Object.prototype || proto === null;
 }
 
-function canonicalize(value: unknown, path: string, seen: Set<object>): string {
-  if (value === null) return "null";
+interface CanonicalSink {
+  write(text: string): void;
+  quote(text: string): void;
+  startContainer(token: "[" | "{"): void;
+  endContainer(token: "]" | "}"): void;
+}
+
+/** One traversal owns validation and property access order for both outputs. */
+function canonicalize(value: unknown, path: string, seen: Set<object>, sink: CanonicalSink): void {
+  if (value === null) {
+    sink.write("null");
+    return;
+  }
   switch (typeof value) {
     case "string":
-      return JSON.stringify(value);
+      sink.quote(value);
+      return;
     case "boolean":
-      return value ? "true" : "false";
+      sink.write(value ? "true" : "false");
+      return;
     case "number": {
       if (!Number.isFinite(value)) {
-        throw new ReservingError("UNSUPPORTED_VALUE", `non-finite number (${String(value)}) at ${path}`);
+        throw new ReservingError(
+          "UNSUPPORTED_VALUE",
+          `non-finite number (${String(value)}) at ${path}`,
+        );
       }
-      return Object.is(value, -0) ? "0" : String(value);
+      sink.write(Object.is(value, -0) ? "0" : String(value));
+      return;
     }
     case "undefined":
       throw new ReservingError("UNSUPPORTED_VALUE", `undefined at ${path}`);
@@ -62,20 +80,24 @@ function canonicalize(value: unknown, path: string, seen: Set<object>): string {
     throw new ReservingError("UNSUPPORTED_VALUE", `circular reference at ${path}`);
   }
   seen.add(obj);
-  let out: string;
   if (Array.isArray(obj)) {
-    const parts: string[] = [];
+    sink.startContainer("[");
     for (let i = 0; i < obj.length; i++) {
-      parts.push(canonicalize(obj[i], `${path}[${i}]`, seen));
+      if (i > 0) sink.write(",");
+      canonicalize(obj[i], `${path}[${i}]`, seen, sink);
     }
-    out = `[${parts.join(",")}]`;
+    sink.endContainer("]");
   } else if (isPlainObject(obj)) {
     const keys = Object.keys(obj).sort();
-    const parts: string[] = [];
-    for (const key of keys) {
-      parts.push(`${JSON.stringify(key)}:${canonicalize(obj[key], `${path}.${key}`, seen)}`);
+    sink.startContainer("{");
+    for (let i = 0; i < keys.length; i++) {
+      if (i > 0) sink.write(",");
+      const key = keys[i]!;
+      sink.quote(key);
+      sink.write(":");
+      canonicalize(obj[key], `${path}.${key}`, seen, sink);
     }
-    out = `{${parts.join(",")}}`;
+    sink.endContainer("}");
   } else {
     const name = (obj.constructor as { name?: string } | undefined)?.name ?? "unknown";
     throw new ReservingError(
@@ -84,7 +106,6 @@ function canonicalize(value: unknown, path: string, seen: Set<object>): string {
     );
   }
   seen.delete(obj);
-  return out;
 }
 
 /**
@@ -95,12 +116,59 @@ function canonicalize(value: unknown, path: string, seen: Set<object>): string {
  * "$.rows[2].ultimate" — for any value JSON cannot faithfully represent.
  */
 export function canonicalJson(value: unknown): string {
-  return canonicalize(value, "$", new Set());
+  // Join finished subtrees as before, rather than retaining a flat token array
+  // for the entire input graph alongside the final serialized string.
+  const frames: string[][] = [[]];
+  const write = (text: string) => {
+    frames[frames.length - 1]!.push(text);
+  };
+  canonicalize(value, "$", new Set(), {
+    write,
+    quote(text) {
+      write(JSON.stringify(text));
+    },
+    startContainer(token) {
+      frames.push([token]);
+    },
+    endContainer(token) {
+      const parts = frames.pop()!;
+      parts.push(token);
+      write(parts.join(""));
+    },
+  });
+  return frames[0]!.join("");
 }
 
-const FNV_OFFSET_BASIS = 0xcbf29ce484222325n;
-const FNV_PRIME = 0x100000001b3n;
-const MASK_64 = 0xffffffffffffffffn;
+const UTF8_CHUNK_CODE_UNITS = 16_384;
+const QUOTED_CHUNK_CODE_UNITS = 4_096;
+
+/** A valid surrogate pair must reach native escaping/encoding together. */
+function chunkEnd(text: string, start: number, limit: number): number {
+  let end = Math.min(start + limit, text.length);
+  const before = text.charCodeAt(end - 1);
+  const after = text.charCodeAt(end);
+  if (before >= 0xd800 && before <= 0xdbff && after >= 0xdc00 && after <= 0xdfff) end -= 1;
+  return end;
+}
+
+/** Reuse at most 48 KiB for long strings instead of duplicating all UTF-8 bytes. */
+function* utf8Chunks(text: string): Generator<Uint8Array> {
+  const encoder = new TextEncoder();
+  // Retain TextEncoder's runtime coercion for untyped JavaScript callers.
+  if (typeof text !== "string" || text.length <= UTF8_CHUNK_CODE_UNITS) {
+    yield encoder.encode(text);
+    return;
+  }
+  // A UTF-16 code unit needs at most three UTF-8 bytes (a surrogate pair
+  // needs four bytes for two code units), so encodeInto always consumes a chunk.
+  const buffer = new Uint8Array(UTF8_CHUNK_CODE_UNITS * 3);
+  for (let start = 0; start < text.length; ) {
+    const end = chunkEnd(text, start, UTF8_CHUNK_CODE_UNITS);
+    const { written } = encoder.encodeInto(text.slice(start, end), buffer);
+    yield buffer.subarray(0, written);
+    start = end;
+  }
+}
 
 /**
  * FNV-1a 64-bit hash over the UTF-8 bytes of `text`, returned as a 16-hex-char
@@ -112,11 +180,61 @@ const MASK_64 = 0xffffffffffffffffn;
  * needing tamper evidence must sign or cryptographically hash the payload.
  */
 export function fnv1a64(text: string): string {
-  const bytes = new TextEncoder().encode(text);
-  let hash = FNV_OFFSET_BASIS;
-  for (const byte of bytes) {
-    hash ^= BigInt(byte);
-    hash = (hash * FNV_PRIME) & MASK_64;
-  }
-  return hash.toString(16).padStart(16, "0");
+  const hash = createFnvAccumulator();
+  for (const bytes of utf8Chunks(text)) hash.update(bytes);
+  return hash.digest();
+}
+
+/**
+ * The exact integrity tag `fnv1a64(canonicalJson(value))`, without constructing
+ * the complete canonical string or UTF-8 byte array. Validation, key ordering,
+ * escaping and unsupported-value error paths match `canonicalJson`.
+ *
+ * Text and byte buffers are bounded, including for a single large string/key;
+ * object-key sorting, ancestor tracking and paths still scale with input shape.
+ * The caller's input remains in memory. This is not a constant-memory dataset
+ * pipeline, a collision-resistant hash, or a substitute for exact equality.
+ */
+export function canonicalFnv1a64(value: unknown): string {
+  const hash = createFnvAccumulator();
+  const encoder = new TextEncoder();
+  const bytes = new Uint8Array(UTF8_CHUNK_CODE_UNITS * 3);
+  let buffered = "";
+  const flush = () => {
+    if (!buffered.length) return;
+    // Three bytes per UTF-16 code unit always fit; pairs need only four for two.
+    const { written } = encoder.encodeInto(buffered, bytes);
+    hash.update(bytes.subarray(0, written));
+    buffered = "";
+  };
+  const write = (text: string) => {
+    for (let start = 0; start < text.length; ) {
+      const end = chunkEnd(text, start, UTF8_CHUNK_CODE_UNITS - buffered.length);
+      if (end === start) {
+        flush();
+        continue;
+      }
+      buffered += text.slice(start, end);
+      start = end;
+      if (buffered.length >= UTF8_CHUNK_CODE_UNITS - 1) flush();
+    }
+  };
+  canonicalize(value, "$", new Set(), {
+    write,
+    startContainer: write,
+    endContainer: write,
+    quote(text) {
+      write('"');
+      for (let start = 0; start < text.length; ) {
+        const end = chunkEnd(text, start, QUOTED_CHUNK_CODE_UNITS);
+        // Bounded native escaping retains minimal JSON escape spelling and lone
+        // surrogate escapes without allocating one full escaped string atom.
+        write(JSON.stringify(text.slice(start, end)).slice(1, -1));
+        start = end;
+      }
+      write('"');
+    },
+  });
+  flush();
+  return hash.digest();
 }

package/src/diagnosticEvidenceIntern.ts

@@ -0,0 +1,151 @@
+/**
+ * Private, invocation-scoped sharing of small, newly owned evidence nodes.
+ * This is not a validation boundary: callers must finish normalizing/cloning
+ * every child first, and must never submit caller-owned objects or SDK brands.
+ */
+export interface DiagnosticEvidenceInternOptions {
+  readonly maxEntries?: number;
+  readonly maxSignatureCharacters?: number;
+  readonly maxCandidateProperties?: number;
+}
+
+type EvidenceMode = "plain" | "source" | "free";
+
+const DEFAULT_MAX_ENTRIES = 100_000;
+const DEFAULT_MAX_SIGNATURE_CHARACTERS = 16 * 1024 * 1024;
+// Wide result envelopes are usually unique; focus the budget on their small
+// repeated coordinates, sources, scopes, states, and short child lists.
+const DEFAULT_MAX_CANDIDATE_PROPERTIES = 10;
+const MAX_CANDIDATE_STRING_CHARACTERS = 4096;
+
+function sameOwnedNode(left: object, right: object): boolean {
+  if (Object.getPrototypeOf(left) !== Object.getPrototypeOf(right))
+    return false;
+  const leftKeys = Reflect.ownKeys(left);
+  const rightKeys = Reflect.ownKeys(right);
+  if (leftKeys.length !== rightKeys.length) return false;
+  return leftKeys.every((key, index) => {
+    if (key !== rightKeys[index]) return false;
+    const a = Object.getOwnPropertyDescriptor(left, key)!;
+    const b = Object.getOwnPropertyDescriptor(right, key)!;
+    return (
+      "value" in a &&
+      "value" in b &&
+      a.enumerable === b.enumerable &&
+      a.configurable === b.configurable &&
+      a.writable === b.writable &&
+      Object.is(a.value, b.value)
+    );
+  });
+}
+
+export function createDiagnosticEvidenceInterner(
+  options: DiagnosticEvidenceInternOptions = {},
+): {
+  internOwned<T extends object>(candidate: T, mode: EvidenceMode): T;
+} {
+  const maxEntries = options.maxEntries ?? DEFAULT_MAX_ENTRIES;
+  const maxSignatureCharacters =
+    options.maxSignatureCharacters ?? DEFAULT_MAX_SIGNATURE_CHARACTERS;
+  const maxCandidateProperties =
+    options.maxCandidateProperties ?? DEFAULT_MAX_CANDIDATE_PROPERTIES;
+  const pool = new Map<string, object>();
+  const childIds = new WeakMap<object, number>();
+  let nextChildId = 1;
+  let signatureCharacters = 0;
+
+  return {
+    internOwned<T extends object>(candidate: T, mode: EvidenceMode): T {
+      if (!Object.isFrozen(candidate)) return candidate;
+      const prototype = Object.getPrototypeOf(candidate);
+      const array = Array.isArray(candidate);
+      if (
+        array
+          ? prototype !== Array.prototype
+          : prototype !== null && prototype !== Object.prototype
+      )
+        return candidate;
+      if (array && candidate.length >= maxCandidateProperties) return candidate;
+      const keys = Reflect.ownKeys(candidate);
+      if (keys.length > maxCandidateProperties) return candidate;
+      const signatureParts: unknown[] = [
+        mode,
+        array ? "array" : prototype === null ? "null" : "object",
+      ];
+      // A declined candidate must not grow persistent bookkeeping. Stage at
+      // most this small candidate's children, including repeated references,
+      // and commit their IDs only if the candidate enters the bounded pool.
+      let pendingChildIds: Map<object, number> | undefined;
+      let stringCharacters = 0;
+      for (const key of keys) {
+        // Diagnostic JSON does not contain symbols. Declining unexpected
+        // shapes is harmless and never invokes an accessor or caller method.
+        if (typeof key !== "string") return candidate;
+        const descriptor = Object.getOwnPropertyDescriptor(candidate, key)!;
+        if (!("value" in descriptor)) return candidate;
+        const value: unknown = descriptor.value;
+        let tokenType: string;
+        let tokenValue: unknown;
+        if (value === null) {
+          tokenType = "null";
+          tokenValue = null;
+        } else if (typeof value === "object") {
+          if (!Object.isFrozen(value)) return candidate;
+          tokenType = "object";
+          const existingId = childIds.get(value);
+          if (existingId !== undefined) tokenValue = existingId;
+          else {
+            pendingChildIds ??= new Map();
+            let pendingId = pendingChildIds.get(value);
+            if (pendingId === undefined) {
+              pendingId = nextChildId + pendingChildIds.size;
+              pendingChildIds.set(value, pendingId);
+            }
+            tokenValue = pendingId;
+          }
+        } else if (typeof value === "number") {
+          if (!Number.isFinite(value)) return candidate;
+          tokenType = "number";
+          tokenValue = Object.is(value, -0) ? "-0" : value;
+        } else if (
+          typeof value === "string" ||
+          typeof value === "boolean" ||
+          value === undefined
+        ) {
+          tokenType = typeof value;
+          tokenValue = value;
+          if (typeof value === "string") stringCharacters += value.length;
+        } else return candidate;
+        stringCharacters += key.length;
+        if (stringCharacters > MAX_CANDIDATE_STRING_CHARACTERS)
+          return candidate;
+        // Frozen data properties are necessarily non-configurable and
+        // non-writable. Keep the varying enumerability bit (including array
+        // length) and a fixed four-slot typed tuple, without temporary nested
+        // arrays. The independent equality check still verifies every flag.
+        signatureParts.push(key, descriptor.enumerable, tokenType, tokenValue);
+      }
+      const signature = JSON.stringify(signatureParts);
+      // An unseen child cannot occur in an existing entry: every admitted
+      // entry committed all its child IDs. No equality hit is skipped here.
+      const existing = pendingChildIds ? undefined : pool.get(signature);
+      // Signatures encode exact children and descriptor/order information;
+      // independently check equality before sharing rather than treating a
+      // fingerprint as proof of equality.
+      if (existing !== undefined && sameOwnedNode(existing, candidate))
+        return existing as T;
+      if (
+        pool.size < maxEntries &&
+        signatureCharacters + signature.length <= maxSignatureCharacters
+      ) {
+        if (pendingChildIds) {
+          for (const [child, id] of pendingChildIds) childIds.set(child, id);
+          nextChildId += pendingChildIds.size;
+        }
+        pool.set(signature, candidate);
+        signatureCharacters += signature.length;
+      }
+      return candidate;
+    },
+  };
+}

package/src/diagnosticExposure.ts

@@ -14,12 +14,119 @@ import {
 } from "./types.js";
 import {
   diagnosticJsonPreflight,
+  diagnosticRecord,
   hasDiagnosticOwn,
   isDiagnosticPlainRecord,
   isDiagnosticToken,
   snapshotDiagnosticJson,
+  MAX_DIAGNOSTIC_JSON_DEPTH,
 } from "./diagnosticRuntime.js";
 
+// Bound the collection separately from each untrusted record. The generic
+// million-node JSON cap otherwise rejects legitimate files at ~71k–100k rows,
+// depending only on how much source provenance each observation carries.
+const MAX_EXPOSURE_OBSERVATIONS = 250_000;
+
+// JSON enumeration skips hidden properties, but the strict exposure schema
+// reads recognized own fields. Refuse hidden accessors before those reads.
+function hiddenExposureAccessorIssues(
+  value: unknown,
+  path: string,
+): DiagnosticValidationIssue[] {
+  if (!isDiagnosticPlainRecord(value)) return [];
+  const issues: DiagnosticValidationIssue[] = [];
+  for (const key of Object.getOwnPropertyNames(value)) {
+    const descriptor = Object.getOwnPropertyDescriptor(value, key)!;
+    if (!descriptor.enumerable && !("value" in descriptor))
+      issues.push({
+        domain: "input",
+        code: "invalid-json-value",
+        path: propertyPath(path, key),
+        message: "JSON objects may contain only data properties",
+      });
+  }
+  return issues;
+}
+
+function exposureCollectionPreflight(
+  value: unknown,
+): readonly DiagnosticValidationIssue[] {
+  if (!Array.isArray(value)) return diagnosticJsonPreflight(value, "input");
+  if (Object.getPrototypeOf(value) !== Array.prototype)
+    return [
+      {
+        domain: "input",
+        code: "invalid-json-value",
+        path: "$",
+        message: "Value must use a plain object or array prototype",
+      },
+    ];
+  if (value.length > MAX_EXPOSURE_OBSERVATIONS)
+    return [
+      {
+        domain: "input",
+        code: "expression-limit",
+        path: "$",
+        message: `Exposure observation count exceeds ${MAX_EXPOSURE_OBSERVATIONS}`,
+      },
+    ];
+  const issues: DiagnosticValidationIssue[] = [];
+  for (const key of Reflect.ownKeys(value)) {
+    if (key === "length") continue;
+    const index = typeof key === "string" ? Number(key) : Number.NaN;
+    if (
+      Number.isInteger(index) &&
+      index >= 0 &&
+      index < value.length &&
+      String(index) === key
+    )
+      continue;
+    issues.push({
+      domain: "input",
+      code: "invalid-json-value",
+      path: typeof key === "symbol" ? "$" : propertyPath("$", key),
+      message: "JSON arrays may contain only indexed data properties",
+    });
+  }
+  for (let index = 0; index < value.length; index++) {
+    const descriptor = Object.getOwnPropertyDescriptor(value, String(index));
+    if (!descriptor || !("value" in descriptor)) {
+      issues.push({
+        domain: "input",
+        code: "invalid-json-value",
+        path: `$[${index}]`,
+        message: "JSON arrays may contain only indexed data properties",
+      });
+      continue;
+    }
+    issues.push(
+      ...hiddenExposureAccessorIssues(descriptor.value, `$[${index}]`),
+    );
+    if (isDiagnosticPlainRecord(descriptor.value)) {
+      const source = Object.getOwnPropertyDescriptor(
+        descriptor.value,
+        "source",
+      );
+      if (source && "value" in source)
+        issues.push(
+          ...hiddenExposureAccessorIssues(source.value, `$[${index}].source`),
+        );
+    }
+    // The outer collection used to occupy depth 1: keep the exact depth
+    // contract, and retain the default million-node guard for each record.
+    for (const issue of diagnosticJsonPreflight(descriptor.value, "input", {
+      maxDepth: MAX_DIAGNOSTIC_JSON_DEPTH - 1,
+    }))
+      issues.push({ ...issue, path: `$[${index}]${issue.path.slice(1)}` });
+  }
+  return issues;
+}
+
+/** Only for new SDK-owned objects whose leaves have already been validated. */
+function freezeExposureRecord<T extends object>(value: T): T {
+  return Object.freeze(Object.assign(diagnosticRecord<unknown>(), value)) as T;
+}
+
 export interface DiagnosticExposureObservation {
   readonly key: string;
   readonly sourceGroup: string;
@@ -115,7 +222,7 @@ function validateExposureArguments(
   // Nonfinite exposure amounts are supported audited inputs. They are the
   // only non-JSON numbers permitted here; metadata still must be finite.
   const issues: DiagnosticValidationIssue[] = [
-    ...diagnosticJsonPreflight(observations, "input")
+    ...exposureCollectionPreflight(observations)
       .filter(
         (issue) =>
           !(
@@ -396,7 +503,7 @@ export function reconcileDiagnosticExposures(
     cohort.push(observation);
     cohorts.set(identity, cohort);
   }
-  return snapshotDiagnosticJson(
+  return Object.freeze(
     [...cohorts.values()]
       .map((cohort): ReconciledDiagnosticExposure => {
         const first = cohort[0]!;
@@ -404,18 +511,20 @@ export function reconcileDiagnosticExposures(
           ? timingByMeasure[first.measureId]
           : undefined;
         const audited = cohort
-          .map((item): DiagnosticExposureAuditObservation => ({
-            sourceGroup: item.sourceGroup,
-            origin: item.origin,
-            ...(item.valuation === undefined
-              ? {}
-              : { valuation: item.valuation }),
-            value: auditDiagnosticNumber(item.value),
-            complete: item.complete,
-            ...(item.source === undefined
-              ? {}
-              : { source: Object.freeze({ ...item.source }) }),
-          }))
+          .map((item): DiagnosticExposureAuditObservation =>
+            snapshotDiagnosticJson({
+              sourceGroup: item.sourceGroup,
+              origin: item.origin,
+              ...(item.valuation === undefined
+                ? {}
+                : { valuation: item.valuation }),
+              value: auditDiagnosticNumber(item.value),
+              complete: item.complete,
+              ...(item.source === undefined
+                ? {}
+                : { source: Object.freeze({ ...item.source }) }),
+            }),
+          )
           .sort(observationOrder);
         const issues: (
           "missing" | "incomplete" | "non-finite" | "duplicate" | "conflict"
@@ -432,7 +541,7 @@ export function reconcileDiagnosticExposures(
         const validStaticCopies =
           timing === "origin-static" && issues.length === 0;
         if (issues.length > 0)
-          return Object.freeze({
+          return freezeExposureRecord({
             measureId: first.measureId,
             key: first.key,
             status: "invalid",
@@ -445,8 +554,8 @@ export function reconcileDiagnosticExposures(
           throw new Error("unreachable invalid exposure state");
         const sources = normalizeDiagnosticSourceLocations(
           audited.map((item) => item.source),
-        );
-        return Object.freeze({
+        ).map((source) => snapshotDiagnosticJson(source));
+        return freezeExposureRecord({
           measureId: first.measureId,
           key: first.key,
           status: "valid",

package/src/diagnosticIdentity.ts

@@ -1,4 +1,5 @@
 import { canonicalJson, fnv1a64 } from "./canonical.js";
+import { createDiagnosticEvidenceInterner } from "./diagnosticEvidenceIntern.js";
 import type {
   AmountBasisDefinition,
   AmountLimitation,
@@ -181,6 +182,7 @@ export function projectDiagnosticIdentity<T>(
     maxNodes: Number.MAX_SAFE_INTEGER,
   });
   if (issues.length) throw new DiagnosticValidationError(issues);
+  const sharing = createDiagnosticEvidenceInterner();
   const clone = (
     item: unknown,
     sourceSlot = false,
@@ -189,11 +191,15 @@ export function projectDiagnosticIdentity<T>(
   ): unknown => {
     if (item === null || typeof item !== "object")
       return typeof item === "number" && Object.is(item, -0) ? 0 : item;
+    const mode = freeJson ? "free" : sourceSlot ? "source" : "plain";
     if (Array.isArray(item))
-      return Object.freeze(
-        item.map((child, index) =>
-          clone(child, sourceSlot, freeJson, `${path}[${index}]`),
+      return sharing.internOwned(
+        Object.freeze(
+          item.map((child, index) =>
+            clone(child, sourceSlot, freeJson, `${path}[${index}]`),
+          ),
         ),
+        mode,
       );
     const record = item as Record<string, unknown>;
     if (sourceSlot && typeof record.artifactId === "string") {
@@ -243,8 +249,11 @@ export function projectDiagnosticIdentity<T>(
       }
       if (sourceIssues.length)
         throw new DiagnosticValidationError(sourceIssues);
-      return normalizeDiagnosticSourceLocation(
-        record as unknown as DiagnosticSourceLocation,
+      return sharing.internOwned(
+        normalizeDiagnosticSourceLocation(
+          record as unknown as DiagnosticSourceLocation,
+        ),
+        mode,
       );
     }
     const result = Object.create(null) as Record<string, unknown>;
@@ -258,7 +267,7 @@ export function projectDiagnosticIdentity<T>(
         `${path}.${key}`,
       );
     }
-    return Object.freeze(result);
+    return sharing.internOwned(Object.freeze(result), mode);
   };
   return clone(value) as DiagnosticIdentityProjection<T>;
 }