@actuarial-ts/core 0.10.0 → 0.12.0

package/src/diagnosticReviewSources.ts

@@ -1,32 +1,52 @@
-import { canonicalJson } from "./canonical.js";
 import type { DiagnosticSourceLocation } from "./diagnosticDefinitions.js";
 import {
   compareDiagnosticSourceLocations,
+  diagnosticSourceLocationKey,
   normalizeDiagnosticSourceLocations,
 } from "./diagnosticSourceOrdering.js";
 
 /**
  * Private, invocation-owned evidence interner. Only the authenticated review
  * evaluator registers preparation sources: its preparation is deeply frozen.
- * Neither this pool nor its bookkeeping is retained on the result or a cell.
+ * Neither this pool nor its bookkeeping is retained on the result or a cell;
+ * the pool lives exactly as long as the evaluator invocation that created it.
  */
 export function createDiagnosticReviewSourcePool() {
-  type SourceEntry = { key: string; value: DiagnosticSourceLocation };
-  const preparedSources = new WeakMap<object, SourceEntry>();
-  const ownedSources = new WeakMap<object, SourceEntry>();
-  const ownedLists = new WeakSet<readonly DiagnosticSourceLocation[]>();
+  type SourceEntry = {
+    key: string;
+    value: DiagnosticSourceLocation;
+    /** The owned one-element list citing exactly this prepared source. */
+    singleton?: readonly DiagnosticSourceLocation[];
+    /** Owned two-element unions of this entry's singleton with another's. */
+    pairs?: Map<SourceEntry, readonly DiagnosticSourceLocation[]>;
+  };
+  const singletonOf = (entry: SourceEntry): readonly DiagnosticSourceLocation[] => {
+    if (entry.singleton === undefined) {
+      entry.singleton = Object.freeze([entry.value]);
+      ownedLists.add(entry.singleton);
+    }
+    return entry.singleton;
+  };
+  // Strong, invocation-scoped tables: the prepared inputs outlive the pool, so
+  // weak tables would only add per-lookup cost and collector pressure here.
+  const preparedSources = new Map<object, SourceEntry>();
+  const ownedSources = new Map<object, SourceEntry>();
+  const ownedLists = new Set<readonly DiagnosticSourceLocation[]>();
+  const emptyList: readonly DiagnosticSourceLocation[] = Object.freeze([]);
+  ownedLists.add(emptyList);
   const sourceEntry = (
     value: DiagnosticSourceLocation,
     fromPrepared: boolean,
   ): SourceEntry => {
-    const previous =
-      ownedSources.get(value) ??
-      (fromPrepared ? preparedSources.get(value) : undefined);
+    // A prepared source is almost always registered already; look there first.
+    const previous = fromPrepared
+      ? (preparedSources.get(value) ?? ownedSources.get(value))
+      : ownedSources.get(value);
     if (previous) return previous;
     // Preserve the authoritative full-value contract, including extra own
     // data. An artifact/file/sheet/row tuple is not an exact-value substitute.
     const normalized = normalizeDiagnosticSourceLocations([value])[0]!;
-    const key = canonicalJson(normalized);
+    const key = diagnosticSourceLocationKey(normalized);
     // Fresh/unrecognized expression evidence takes the value path every time.
     // Its children may be caller-owned even if the outer source was frozen.
     if (!fromPrepared) return { key, value: normalized };
@@ -34,7 +54,7 @@ export function createDiagnosticReviewSourcePool() {
     // Across subsets, equal canonical keys can still have different original
     // representatives (for example -0/+0 in extra own data). Keep the snapshot
     // associated with each prepared source so the first occurrence still wins.
-    const entry = { key, value: Object.freeze(normalized) };
+    const entry: SourceEntry = { key, value: Object.freeze(normalized) };
     ownedSources.set(entry.value, entry);
     preparedSources.set(value, entry);
     return entry;
@@ -58,13 +78,19 @@ export function createDiagnosticReviewSourcePool() {
         }
       },
       finish(): readonly DiagnosticSourceLocation[] {
-        const values = Object.freeze(
-          entries
-            ? [...entries.values()].sort(compareDiagnosticSourceLocations)
-            : first ? [first.value] : [],
-        );
-        if (owned) ownedLists.add(values);
-        return values;
+        if (entries) {
+          const values = Object.freeze(
+            [...entries.values()].sort(compareDiagnosticSourceLocations),
+          );
+          if (owned) ownedLists.add(values);
+          return values;
+        }
+        if (!first) return owned ? emptyList : Object.freeze([]);
+        // Equal owned singletons are one immutable list: every measure subset
+        // of a cell that cites the same prepared row reuses it, so the compact
+        // store also sees one list identity instead of one per subset.
+        if (owned) return singletonOf(first);
+        return Object.freeze([first.value]);
       },
     };
   };
@@ -80,6 +106,26 @@ export function createDiagnosticReviewSourcePool() {
       values.length === first.length && values.every((value, index) => value === first[index]))) {
       return first;
     }
+    // Valuation-pair scopes union two owned singletons for every monotonic
+    // rule; the union of the same two entries is one immutable owned list.
+    if (lists.length === 2 && first!.length === 1 && lists[1]!.length === 1 &&
+      ownedLists.has(first!) && ownedLists.has(lists[1]!)) {
+      const left = ownedSources.get(first![0]!);
+      const right = ownedSources.get(lists[1]![0]!);
+      if (left && right) {
+        if (left === right || left.key === right.key) return singletonOf(left);
+        const known = left.pairs?.get(right);
+        if (known) return known;
+        const pair = Object.freeze(
+          compareDiagnosticSourceLocations(left.value, right.value) <= 0
+            ? [left.value, right.value]
+            : [right.value, left.value],
+        );
+        ownedLists.add(pair);
+        (left.pairs ??= new Map()).set(right, pair);
+        return pair;
+      }
+    }
     const target = collection();
     for (const values of lists)
       for (const source of values) target.add(source, false);
@@ -96,6 +142,11 @@ export function createDiagnosticReviewSourcePool() {
     collection,
     union,
     forPreparedSources,
+    /** The owned empty list every empty owned collection resolves to. */
+    emptyList,
+    /** The owned singleton list for one prepared source, without a collection. */
+    singletonForPrepared: (value: DiagnosticSourceLocation) =>
+      singletonOf(sourceEntry(value, true)),
     // Ownership means this invocation's snapshots, not Object.isFrozen on an
     // arbitrary caller array/object. The builder uses these only while appending.
     sourceKey: (value: DiagnosticSourceLocation) => ownedSources.get(value)?.key,

package/src/diagnosticReviewStore.ts

@@ -305,9 +305,11 @@ export function createCompactReviewBuilder(
   const sourceIds = new Map<string, number>();
   // Only this evaluator invocation's snapshots may skip canonicalization.
   // Merely freezing a caller array/object does not prove deep snapshot ownership.
-  const ownedSourceIds = new WeakMap<object, number>();
-  const ownedListIds = new WeakMap<readonly DiagnosticSourceLocation[], number>();
+  // The builder is released with its invocation, so strong tables are safe.
+  const ownedSourceIds = new Map<object, number>();
+  const ownedListIds = new Map<readonly DiagnosticSourceLocation[], number>();
   const listIds = new Map<string, number>();
+  const singletonListIds = new Map<number, number>();
   const overflows = new Map<number, readonly EncodedOverflow[]>();
   const controls = new Map<number, Omit<Control, "sources">>();
   const ruleIds = new Map(rules.map((rule, index) => [rule.id, index]));
@@ -315,7 +317,28 @@ export function createCompactReviewBuilder(
   const counts = zeroCounts();
   let count = 0;
   let finished = false;
+  // Identity shortcut for coordinates the evaluator reuses across rules. A hit
+  // is honoured only when the object's current fields still equal the stored
+  // snapshot, so a mutated caller object is re-indexed exactly as before.
+  const coordinateIdByObject = new Map<DiagnosticReviewCoordinate, number>();
   const coordinateId = (value: DiagnosticReviewCoordinate): number => {
+    const cached = coordinateIdByObject.get(value);
+    if (cached !== undefined) {
+      const stored = coordinates[cached]!;
+      if (
+        stored.sourceGroup === value.sourceGroup &&
+        stored.origin === value.origin &&
+        stored.valuation === value.valuation &&
+        Object.is(stored.developmentAge, value.developmentAge) &&
+        stored.ageUnit === value.ageUnit
+      )
+        return cached;
+    }
+    const id = coordinateIdByFields(value);
+    coordinateIdByObject.set(value, id);
+    return id;
+  };
+  const coordinateIdByFields = (value: DiagnosticReviewCoordinate): number => {
     let index = coordinateIds;
     for (const key of [
       value.sourceGroup,
@@ -342,11 +365,13 @@ export function createCompactReviewBuilder(
   const sourceListId = (
     values: readonly DiagnosticSourceLocation[],
   ): number => {
+    // Only owned lists ever enter this table, so a hit already proves ownership
+    // and its capacity was checked when it was first indexed.
+    const knownList = ownedListIds.get(values);
+    if (knownList !== undefined) return knownList;
     assertCompactReviewCapacity(values.length, 0);
     const ownedList = sourceOwnership?.ownsList(values) ?? false;
-    const knownList = ownedList ? ownedListIds.get(values) : undefined;
-    if (knownList !== undefined) return knownList;
-    const ids = values.map((value) => {
+    const sourceIdOf = (value: DiagnosticSourceLocation): number => {
       const ownedKey = sourceOwnership?.sourceKey(value);
       const owned = ownedKey !== undefined;
       const known = owned ? ownedSourceIds.get(value) : undefined;
@@ -361,7 +386,24 @@ export function createCompactReviewBuilder(
       }
       if (owned) ownedSourceIds.set(value, id);
       return id;
-    });
+    };
+    // One-source lists are the common cell scope: index them by source id
+    // without building an id array or a joined text key. The text-keyed table
+    // stays consistent so longer lists can never collide with them.
+    if (values.length === 1) {
+      const sourceId = sourceIdOf(values[0]!);
+      let id = singletonListIds.get(sourceId);
+      if (id === undefined) {
+        assertCompactReviewCapacity(sourceLists.length);
+        id = sourceLists.length;
+        singletonListIds.set(sourceId, id);
+        listIds.set(String(sourceId), id);
+        sourceLists.push(Uint32Array.of(sourceId));
+      }
+      if (ownedList) ownedListIds.set(values, id);
+      return id;
+    }
+    const ids = values.map(sourceIdOf);
     // Do not construct a giant signature for high-fanout control-total evidence.
     const key = ids.length <= 64 ? ids.join(",") : undefined;
     let id = key === undefined ? undefined : listIds.get(key);

package/src/diagnosticRunner.ts

@@ -2,7 +2,6 @@ import type {
   DiagnosticDeepReadonly,
   DiagnosticMeasureDefinition,
   DiagnosticMetricPresentation,
-  DiagnosticsFilter,
   JsonValue,
 } from "./diagnosticDefinitions.js";
 import { getCompiledDiagnosticDefinitionInternals } from "./diagnosticDefinitions.js";
@@ -48,6 +47,12 @@ import { compareDiagnosticFindings } from "./diagnosticOrdering.js";
 import { normalizeDiagnosticPeriod } from "./diagnosticPeriods.js";
 import { normalizeDiagnosticSourceLocations } from "./diagnosticSourceOrdering.js";
 import { freezeDiagnosticGraph as deepFreeze } from "./diagnosticFreeze.js";
+import {
+  normalizeDiagnosticSourceLocationsMemoized,
+  type DiagnosticSourceLocationMemo,
+} from "./diagnosticSourceOrdering.js";
+import type { DiagnosticMeasureContribution } from "./diagnosticAggregation.js";
+import { getPreparedSourceGroups } from "./diagnosticPreparation.js";
 import {
   diagnosticJsonPreflight,
   hasDiagnosticOwn,
@@ -209,10 +214,32 @@ function expressionMeasureIds(
     ),
   ].sort(codeUnit);
 }
+const NO_RUNNER_SOURCES: readonly import("./diagnosticDefinitions.js").DiagnosticSourceLocation[] =
+  Object.freeze([]);
+
+/** One run cites each prepared source once per measure; a memo normalizes it once. */
 function unionSources(
   values: readonly import("./diagnosticDefinitions.js").DiagnosticSourceLocation[],
-): import("./diagnosticDefinitions.js").DiagnosticSourceLocation[] {
-  return normalizeDiagnosticSourceLocations(values);
+  memo?: DiagnosticSourceLocationMemo,
+): readonly import("./diagnosticDefinitions.js").DiagnosticSourceLocation[] {
+  return normalizeDiagnosticSourceLocationsMemoized(values, memo);
+}
+
+type SourceList = readonly import("./diagnosticDefinitions.js").DiagnosticSourceLocation[];
+
+/**
+ * Union of two lists that are already normalized owned unions of this run:
+ * an identical or empty operand contributes nothing new, so the other list is
+ * already the union and needs no re-normalization.
+ */
+function unionOfNormalized(
+  left: SourceList,
+  right: SourceList,
+  memo?: DiagnosticSourceLocationMemo,
+): SourceList {
+  if (left === right || right.length === 0) return left;
+  if (left.length === 0) return right;
+  return unionSources([...left, ...right], memo);
 }
 
 function mergeFindings(
@@ -271,16 +298,28 @@ function validateGroupingContent(
   }
   if (boundaryIssues.length > 0)
     throw new DiagnosticValidationError(boundaryIssues);
-  const auditedSourceGroups = input.prepared.inputAudit.flatMap((item) => {
-    return input.prepared.filter?.sourceGroups === undefined ||
-      input.prepared.filter.sourceGroups.includes(item.record.sourceGroup)
-      ? [item.record.sourceGroup]
-      : [];
-  });
-  const sourceGroups = new Set([
-    ...auditedSourceGroups,
-    ...input.prepared.cells.map((cell) => cell.sourceGroup),
-  ]);
+  const sourceGroups = new Set<string>();
+  const selectedGroups =
+    input.prepared.filter?.sourceGroups === undefined
+      ? undefined
+      : new Set(input.prepared.filter.sourceGroups);
+  // The audit holds every input row, so its source groups are exactly the
+  // groups the preparation recorded; rescanning the audit is the fallback.
+  const known = getPreparedSourceGroups(input.prepared);
+  if (known !== undefined) {
+    for (const group of known)
+      if (selectedGroups === undefined || selectedGroups.has(group))
+        sourceGroups.add(group);
+  } else {
+    const audit = input.prepared.inputAudit;
+    for (let index = 0; index < audit.length; index++) {
+      const group = audit[index]!.record.sourceGroup;
+      if (selectedGroups === undefined || selectedGroups.has(group))
+        sourceGroups.add(group);
+    }
+  }
+  const cells = input.prepared.cells;
+  for (let index = 0; index < cells.length; index++) sourceGroups.add(cells[index]!.sourceGroup);
   for (const [source, target] of Object.entries(input.groupMap ?? {})) {
     if (!sourceGroups.has(source))
       throw new DiagnosticValidationError([
@@ -343,18 +382,25 @@ function mergeStats(
   cells: readonly PreparedDiagnosticDataContent["cells"][number][],
   measure: { readonly id: string; readonly missing: "unknown" | "zero" },
 ): DiagnosticMeasureStats {
-  const contributions = cells.flatMap(
-    (cell) => cell.contributions[measure.id] ?? [],
-  );
-  const blockers = [
-    ...new Map(
-      cells
-        .flatMap((cell) => cell.structuralBlockers[measure.id] ?? [])
-        .map((blocker) => [canonicalJson(blocker), blocker]),
-    ).entries(),
-  ]
-    .sort(([left], [right]) => codeUnit(left, right))
-    .map(([, blocker]) => blocker);
+  const contributions: DiagnosticMeasureContribution[] = [];
+  for (let at = 0; at < cells.length; at++) {
+    const items = cells[at]!.contributions[measure.id];
+    if (items !== undefined)
+      for (let index = 0; index < items.length; index++) contributions.push(items[index]!);
+  }
+  const blockers = cells.some(
+    (cell) => (cell.structuralBlockers[measure.id]?.length ?? 0) > 0,
+  )
+    ? [
+        ...new Map(
+          cells
+            .flatMap((cell) => cell.structuralBlockers[measure.id] ?? [])
+            .map((blocker) => [canonicalJson(blocker), blocker]),
+        ).entries(),
+      ]
+        .sort(([left], [right]) => codeUnit(left, right))
+        .map(([, blocker]) => blocker)
+    : [];
   return finalizeDiagnosticContributions(
     contributions,
     measure.missing,
@@ -418,6 +464,7 @@ function evaluatePoint(
       readonly import("./diagnosticDefinitions.js").DiagnosticSourceLocation[]
     >
   >,
+  sourceMemo?: DiagnosticSourceLocationMemo,
 ): DiagnosticMetricEvaluation {
   const definition = prepared.definition;
   const internals = getCompiledDiagnosticDefinitionInternals(definition);
@@ -456,12 +503,16 @@ function evaluatePoint(
   }
   const expressionSources = (
     expression: import("./diagnosticExpressions.js").DiagnosticMeasureExpression,
-  ) =>
-    unionSources(
-      expressionMeasureIds(expression).flatMap(
-        (measureId) => sourcesByMeasure[measureId] ?? [],
-      ),
+  ) => {
+    const measureIds = expressionMeasureIds(expression);
+    // A single measure's citation list is already a normalized owned union.
+    if (measureIds.length === 1)
+      return sourcesByMeasure[measureIds[0]!] ?? NO_RUNNER_SOURCES;
+    return unionSources(
+      measureIds.flatMap((measureId) => sourcesByMeasure[measureId] ?? []),
+      sourceMemo,
     );
+  };
   const withOverflowSources = <
     T extends {
       readonly value: number | null;
@@ -498,10 +549,11 @@ function evaluatePoint(
     (
       internals.calculationDependenciesByInstanceId.get(instance.id) ?? []
     ).flatMap((measureId) => sourcesByMeasure[measureId] ?? []),
+    sourceMemo,
   );
   const roleExpressionSources = (
     expression: import("./diagnosticExpressions.js").DiagnosticRoleExpression,
-  ): import("./diagnosticDefinitions.js").DiagnosticSourceLocation[] => {
+  ): readonly import("./diagnosticDefinitions.js").DiagnosticSourceLocation[] => {
     if (expression.op === "role")
       return expressionSources(instance.bindings[expression.role]!);
     return unionSources(
@@ -509,6 +561,7 @@ function evaluatePoint(
         ? expression.terms
         : [expression.left, expression.right]
       ).flatMap(roleExpressionSources),
+      sourceMemo,
     );
   };
   const numeratorSources = roleExpressionSources(formula.numerator);
@@ -652,7 +705,7 @@ function evaluatePoint(
     );
     const left = operand(rule.when.left, pointer(rulePath, "left"));
     const right = operand(rule.when.right, pointer(rulePath, "right"));
-    const ruleSources = unionSources([...left.sources, ...right.sources]);
+    const ruleSources = unionOfNormalized(left.sources, right.sources, sourceMemo);
     const expressionOverflows = [
       ...new Map(
         [...left.overflows, ...right.overflows].map((overflow) => [
@@ -892,7 +945,10 @@ function calculateMetricDiagnostics(
       snapshotOwnedDimensions(value, dimensionSnapshots),
     ]),
   );
-  const buckets = new Map<string, typeof prepared.cells>();
+  const buckets = new Map<
+    string,
+    { group: string; origin: string; valuation: string; cells: (typeof prepared.cells)[number][] }
+  >();
   for (const cell of prepared.cells) {
     const group =
       input.groupMap !== undefined &&
@@ -904,41 +960,63 @@ function calculateMetricDiagnostics(
       !prepared.filter.outputGroups.includes(group)
     )
       continue;
-    const key = canonicalJson([group, cell.origin, cell.valuation]);
-    buckets.set(key, [...(buckets.get(key) ?? []), cell]);
+    // Grouping key only; emergence order is sorted afterwards. Output groups
+    // and period labels are validated tokens, so U+0000 cannot occur in them.
+    const key = `${group}${cell.origin}${cell.valuation}`;
+    const bucket = buckets.get(key);
+    if (bucket) bucket.cells.push(cell);
+    else buckets.set(key, { group, origin: cell.origin, valuation: cell.valuation, cells: [cell] });
   }
+  // Prepared source objects are immutable for this run; snapshot each once.
+  const sourceMemo: DiagnosticSourceLocationMemo = new Map();
   const selectedInstances = prepared.definition.definition.instances.filter(
     (instance) =>
       prepared.filter?.instanceIds === undefined ||
       prepared.filter.instanceIds.includes(instance.id),
   );
   const emergence: DiagnosticEmergencePoint[] = [];
-  for (const [key, cells] of buckets) {
-    const [group, origin, valuation] = JSON.parse(key) as [
-      string,
-      string,
-      string,
-    ];
+  for (const { group, origin, valuation, cells } of buckets.values()) {
     const components = Object.fromEntries(
       prepared.definition.definition.measures.map((measure) => [
         measure.id,
         mergeStats(cells, measure),
       ]),
     );
+    // Loss measures of one bucket cite exactly the same rows: when a measure's
+    // citation sequence is the previous measure's, by identity, its union is too.
+    let previousCited: import("./diagnosticDefinitions.js").DiagnosticSourceLocation[] | undefined;
+    let previousUnion: readonly import("./diagnosticDefinitions.js").DiagnosticSourceLocation[] | undefined;
     const sourcesByMeasure = Object.fromEntries(
-      prepared.definition.definition.measures.map((measure) => [
-        measure.id,
-        unionSources(
-          cells.flatMap((cell) => [
-            ...(cell.contributions[measure.id] ?? []).flatMap(
-              (item) => item.sources,
-            ),
-            ...(cell.structuralBlockers[measure.id] ?? []).flatMap(
-              (item) => item.sources,
-            ),
-          ]),
-        ),
-      ]),
+      prepared.definition.definition.measures.map((measure) => {
+        const cited: import("./diagnosticDefinitions.js").DiagnosticSourceLocation[] =
+          [];
+        for (let at = 0; at < cells.length; at++) {
+          const cell = cells[at]!;
+          const contributions = cell.contributions[measure.id];
+          if (contributions !== undefined)
+            for (let index = 0; index < contributions.length; index++) {
+              const sources = contributions[index]!.sources;
+              for (let position = 0; position < sources.length; position++)
+                cited.push(sources[position]!);
+            }
+          const blockers = cell.structuralBlockers[measure.id];
+          if (blockers !== undefined)
+            for (let index = 0; index < blockers.length; index++) {
+              const sources = blockers[index]!.sources;
+              for (let position = 0; position < sources.length; position++)
+                cited.push(sources[position]!);
+            }
+        }
+        const union: readonly import("./diagnosticDefinitions.js").DiagnosticSourceLocation[] =
+          previousUnion !== undefined &&
+          previousCited!.length === cited.length &&
+          cited.every((source, index) => source === previousCited![index])
+            ? previousUnion
+            : unionSources(cited, sourceMemo);
+        previousCited = cited;
+        previousUnion = union;
+        return [measure.id, union];
+      }),
     );
     const context = {
       group,
@@ -1017,6 +1095,7 @@ function calculateMetricDiagnostics(
           mappedFindings,
           blockerFindingsByMeasure,
           sourcesByMeasure,
+          sourceMemo,
         ),
       ]),
     );

package/src/diagnosticRuntime.ts

@@ -213,7 +213,17 @@ export function diagnosticJsonPreflight(
       }
       continue;
     }
-    for (const key of Object.keys(item).sort().reverse()) {
+    // Children are pushed in descending key order so they pop ascending; sort
+    // only when insertion order is not already sorted, and walk backwards
+    // rather than reversing a copy.
+    const keys = Object.keys(item);
+    for (let index = 1; index < keys.length; index++)
+      if (keys[index - 1]! > keys[index]!) {
+        keys.sort();
+        break;
+      }
+    for (let position = keys.length - 1; position >= 0; position--) {
+      const key = keys[position]!;
       const path = /^[A-Za-z_$][A-Za-z0-9_$]*$/.test(key)
         ? `${frame.path}.${key}`
         : `${frame.path}[${JSON.stringify(key)}]`;