logiparity 1.1.0 → 1.2.0

package/lib/core/dist/html.js

@@ -1,3 +1,4 @@
+import { DecimalSum, displayDecimal, negateDecimal, sumDecimal } from './aggregates.js';
 import { reportHeader, reportHero, findingDonut, findingsSection, reportDocument } from './html-layout.js';
 import { redactTree, secretValues } from './diagnostics.js';
 import { aggregateDisplay, rawAggregateDisplay, aggregateDifference, aggregateStatus } from './money-display.js';
@@ -147,50 +148,56 @@ function reconciliationCards(report, presentation) {
         return partitions.flatMap(({ currency, parts }) => {
             const actual = currency
                 ? (aggregate.valuesByCurrency?.[source]?.[currency] ?? 0n)
-                : aggregate.values[source], reference = currency
+                : (aggregate.exactValues?.[source] ?? aggregate.values[source]), reference = currency
                 ? (aggregate.valuesByCurrency?.[entity.reference]?.[currency] ?? 0n)
-                : aggregate.values[entity.reference];
+                : (aggregate.exactValues?.[entity.reference] ?? aggregate.values[entity.reference]);
             if (actual === undefined || reference === undefined || actual === reference)
                 return [];
             const definition = labels.entity(entity.entity)?.aggregates.find((item) => item.name === aggregate.name);
             const rule = definition?.type === 'sum' ? labels.entity(entity.entity)?.compare[definition.field] : undefined;
-            const difference = BigInt(actual) - BigInt(reference);
-            const amount = (value) => `${value > 0n ? '+' : ''}${(currency ?? rule?.currency) ? formatMoneyMajor(value, currency ?? rule?.currency ?? 'USD') : numberText(value)}`;
+            const difference = sumDecimal(actual, negateDecimal(String(reference)));
+            const amount = (value) => `${sumDecimal(value) !== '0' && !String(value).startsWith('-') ? '+' : ''}${(currency ?? rule?.currency) ? formatMoneyMajor(BigInt(value), currency ?? rule?.currency ?? 'USD') : displayDecimal(value)}`;
             const buckets = [
                 {
                     label: `Missing rows (${numberText(parts.missingRows.count)})`,
-                    value: BigInt(parts.missingRows.amount),
+                    value: parts.missingRows.amount,
                     color: 'page',
-                    show: BigInt(parts.missingRows.amount) !== 0n,
+                    show: sumDecimal(parts.missingRows.amount) !== '0',
                 },
                 ...[
                     { label: 'Extra rows', part: parts.extraRows },
                     { label: 'Value differences', part: parts.valueDifferences },
                 ].map(({ label, part }) => ({
                     label: `${label} (${numberText(part?.count ?? 0)})`,
-                    value: BigInt(part?.amount ?? 0),
+                    value: part?.amount ?? 0n,
                     color: 'round',
                     show: !!part?.count,
                 })),
                 ...parts.patterns.map((part) => ({
                     label: `${BigInt(report.groups?.[part.group]?.delta ?? 0) >= -3n && BigInt(report.groups?.[part.group]?.delta ?? 0) <= 3n ? 'Rounding' : 'Fixed differences'} (${numberText(part.count)})`,
-                    value: BigInt(part.amount),
+                    value: part.amount,
                     color: 'round',
-                    show: BigInt(part.amount) !== 0n,
+                    show: sumDecimal(part.amount) !== '0',
                 })),
                 {
                     label: `Uncertain lines${parts.uncertain?.keys.length ? ': ' + uncertainKeyText(parts.uncertain.keys) : ''}`,
-                    value: BigInt(parts.uncertain?.amount ?? 0),
+                    value: parts.uncertain?.amount ?? 0n,
                     color: 'tz',
                     show: !!parts.uncertain?.keys.length,
                 },
-                { label: 'Unexplained remainder', value: BigInt(parts.remainder), color: 'review', show: true },
+                { label: 'Unexplained remainder', value: parts.remainder, color: 'review', show: true },
             ].filter((part) => part.show);
-            const total = buckets.reduce((sum, part) => sum + (part.value < 0n ? -part.value : part.value), 0n);
-            const accounted = BigInt(parts.remainder) === 0n && !parts.uncertain?.keys.length;
+            const total = new DecimalSum();
+            for (const part of buckets)
+                total.add(String(part.value).startsWith('-') ? negateDecimal(part.value) : part.value);
+            const accounted = sumDecimal(parts.remainder) === '0' && !parts.uncertain?.keys.length;
             const bars = buckets
-                .filter((part) => part.value)
-                .map((part) => `<i style="width:${total ? Number(((part.value < 0n ? -part.value : part.value) * 10000n) / total) / 100 : 0}%;background:var(--c-${part.color})"></i>`)
+                .filter((part) => sumDecimal(part.value) !== '0')
+                .map((part) => {
+                const absolute = new DecimalSum();
+                absolute.add(String(part.value).startsWith('-') ? negateDecimal(part.value) : part.value);
+                return `<i style="width:${absolute.percentOf(total)}%;background:var(--c-${part.color})"></i>`;
+            })
                 .join('');
             const rows = buckets
                 .map((part) => `<div><span><i style="background:var(--c-${part.color})"></i>${escape(part.label)}</span><b>${escape(amount(part.value))}${part.label === 'Unexplained remainder' && accounted ? ' · accounted for' : ''}</b></div>`)
@@ -311,7 +318,7 @@ export function htmlReport(report, presentation = {}) {
     const causeCards = cards
         .map((card, index) => {
         const info = kindInfo[card.kind], group = groups[card.groups[0] ?? -1], words = group ? patternWords(group, presentation) : undefined;
-        return `<article class="card pat" style="--c:var(--c-${info.color});--soft:var(--c-${info.color}-soft)"><div class="ic">${icon(info.path)}</div><div class="kind">${escape(card.label)}</div><div class="delta">${escape(card.headline)}</div><p>${escape(card.hint)}</p><div class="viz" title="${escape(words?.sentence ?? '')}">${card.chart}</div><p class="next-step"><b>Next step:</b> ${escape(card.nextStep)}</p><button class="btn btn-soft" data-card="${index}" type="button" aria-pressed="false">Show these findings</button></article>`;
+        return `<article class="card pat" style="--c:var(--c-${info.color});--soft:var(--c-${info.color}-soft)"><div class="ic">${icon(info.path)}</div><div class="kind">${escape(card.label)}</div><div class="delta">${escape(card.headline)}</div><p${card.kind === 'formatting_difference' ? ' style="white-space:pre-wrap"' : ''}>${escape(card.hint)}</p><div class="viz" title="${escape(words?.sentence ?? '')}">${card.chart}</div><p class="next-step"><b>Next step:</b> ${escape(card.nextStep)}</p><button class="btn btn-soft" data-card="${index}" type="button" aria-pressed="false">Show these findings</button></article>`;
     })
         .join('');
     const options = (values) => values.map(([raw, label]) => `<option value="${escape(raw)}">${escape(label)}</option>`).join('');

package/lib/core/dist/human.js

@@ -145,12 +145,33 @@ export function patternWords(group, presentation = {}) {
         };
     }
     const value = (input) => presentationValue(input, group.entity, group.field, false, presentation);
+    const reference = labels.source(group.entity, Object.keys(labels.entity(group.entity)?.sources ?? {})[0] ?? 'reference');
+    if (group.kind === 'formatting_difference') {
+        const difference = group.cause === 'case_only'
+            ? 'letter case'
+            : group.cause === 'whitespace_only'
+                ? 'whitespace'
+                : 'letter case and whitespace';
+        return {
+            label: 'Formatting difference',
+            sentence: `${numberText(group.count)} ${field} values in ${source} differ from ${reference} only in ${difference} (${JSON.stringify(group.actual)} vs ${JSON.stringify(group.expected)}).`,
+            hint: 'Likely a formatting difference.',
+            nextStep: `Consider ${group.cause === 'whitespace_only' ? 'whitespaceFold: true' : group.cause === 'case_only' ? 'caseFold: true' : 'caseFold: true and whitespaceFold: true'} for this field after reviewing the raw values.`,
+        };
+    }
+    if (group.kind === 'number_change')
+        return {
+            label: 'Repeated numeric change',
+            sentence: `${numberText(group.count)} ${field} values in ${source} are ${value(group.actual)} where ${reference} has ${value(group.expected)}.`,
+            hint: 'Repeated numeric values differ between sources.',
+            nextStep: `Check how ${field} is derived or updated in each source.`,
+        };
     if (group.cause === 'stale_value')
         return {
             label: 'Stale or lagging values',
-            sentence: `${numberText(group.count)} of ${numberText(group.referenceCount ?? group.count)} ${value(group.expected)} ${labels.entity(group.entity)?.pluralLabel ?? 'records'} show ${value(group.actual)} in ${source}; ${numberText(group.matchingCount ?? 0)} match.`,
-            nextStep: `Check refresh timing and synchronization in ${source}; preserve the distinct ${field} values.`,
-            hint: `Likely a stale or lagging ${field}.`,
+            sentence: `${numberText(group.count)} of ${numberText(group.referenceCount ?? group.count)} ${labels.entity(group.entity)?.pluralLabel ?? 'records'} with ${field} ${value(group.expected)} in ${reference} show ${value(group.actual)} in ${source}; ${numberText(group.matchingCount ?? 0)} match.`,
+            nextStep: 'Compare refresh timing in both sources and the field definitions.',
+            hint: 'Likely a lagging update or a different definition of the field.',
         };
     return {
         label: 'Value mapping',
@@ -190,6 +211,12 @@ export function mergedPatternWords(groups, presentation = {}) {
         const words = patternWords(first, presentation);
         if (values.length === 1)
             return words;
+        if (first.kind === 'formatting_difference' || first.kind === 'number_change' || first.cause === 'stale_value')
+            return {
+                ...words,
+                sentence: values.map((group) => patternWords(group, presentation).sentence).join(' '),
+                nextStep: [...new Set(values.map((group) => patternWords(group, presentation).nextStep))].join(' '),
+            };
         const labels = reportLabels(presentation);
         return {
             ...words,

package/lib/core/dist/labels.d.ts

@@ -12,7 +12,9 @@ export declare function reportLabels(presentation: ReportPresentation): {
             fields: Record<string, string | {
                 path: string;
                 label?: string | undefined;
+                missing?: null | undefined;
             }>;
+            absentAsNull: boolean;
             items: string;
             headers: Record<string, string>;
             headersEnv: Record<string, string | {
@@ -73,6 +75,7 @@ export declare function reportLabels(presentation: ReportPresentation): {
                 group?: string | undefined;
             } | undefined;
             datetimeFormat?: string | undefined;
+            dateFormat?: string | undefined;
             label?: string | undefined;
             rollup?: {
                 by: string | string[];
@@ -109,6 +112,7 @@ export declare function reportLabels(presentation: ReportPresentation): {
                 group?: string | undefined;
             } | undefined;
             datetimeFormat?: string | undefined;
+            dateFormat?: string | undefined;
             label?: string | undefined;
             rollup?: {
                 by: string | string[];
@@ -143,6 +147,7 @@ export declare function reportLabels(presentation: ReportPresentation): {
                 group?: string | undefined;
             } | undefined;
             datetimeFormat?: string | undefined;
+            dateFormat?: string | undefined;
             label?: string | undefined;
             rollup?: {
                 by: string | string[];
@@ -183,6 +188,7 @@ export declare function reportLabels(presentation: ReportPresentation): {
                 group?: string | undefined;
             } | undefined;
             datetimeFormat?: string | undefined;
+            dateFormat?: string | undefined;
             label?: string | undefined;
             rollup?: {
                 by: string | string[];
@@ -219,6 +225,7 @@ export declare function reportLabels(presentation: ReportPresentation): {
                 group?: string | undefined;
             } | undefined;
             datetimeFormat?: string | undefined;
+            dateFormat?: string | undefined;
             label?: string | undefined;
             rollup?: {
                 by: string | string[];
@@ -233,11 +240,12 @@ export declare function reportLabels(presentation: ReportPresentation): {
             } | undefined;
         }>;
         compare: Record<string, {
-            type: "string" | "number" | "enum" | "money" | "datetime";
+            type: "string" | "number" | "date" | "enum" | "money" | "datetime";
             allowCurrencySymbol: boolean;
             negativeParentheses: boolean;
             allowNumber: boolean;
             caseFold: boolean;
+            whitespaceFold: boolean;
             emptyAsNull: boolean;
             tolerance: number;
             toleranceSeconds: number;
@@ -272,7 +280,7 @@ export declare function reportLabels(presentation: ReportPresentation): {
             tolerance: number;
             label?: string | undefined;
         })[];
-        failOn: ("invalid" | "missing" | "extra" | "mismatch" | "aggregate" | "duplicate")[] | Partial<Record<"invalid" | "missing" | "extra" | "mismatch" | "aggregate" | "duplicate", number>>;
+        failOn: ("missing" | "invalid" | "extra" | "mismatch" | "aggregate" | "duplicate")[] | Partial<Record<"missing" | "invalid" | "extra" | "mismatch" | "aggregate" | "duplicate", number>>;
         label?: string | undefined;
         pluralLabel?: string | undefined;
     } | undefined;
@@ -281,11 +289,12 @@ export declare function reportLabels(presentation: ReportPresentation): {
     all: (name: string) => string;
 };
 export declare function presentationRule(entity: EntityConfig | undefined, field: string, aggregate: boolean): {
-    type: "string" | "number" | "enum" | "money" | "datetime";
+    type: "string" | "number" | "date" | "enum" | "money" | "datetime";
     allowCurrencySymbol: boolean;
     negativeParentheses: boolean;
     allowNumber: boolean;
     caseFold: boolean;
+    whitespaceFold: boolean;
     emptyAsNull: boolean;
     tolerance: number;
     toleranceSeconds: number;

package/lib/core/dist/money-display.js

@@ -1,5 +1,6 @@
 import { humanize, presentationRule } from './labels.js';
 import { displayValue } from './report-utils.js';
+import { displayDecimal, negateDecimal, sumDecimal } from './aggregates.js';
 export function formatDatetimeUtc(epoch) {
     if (!Number.isSafeInteger(epoch))
         return undefined;
@@ -39,6 +40,8 @@ export function presentationValue(value, entityName, field, aggregate, presentat
         return displayValue(value);
     if (rule?.type === 'datetime' && typeof value === 'number')
         return readableDatetime(value, presentation.displayTimezone) ?? plain;
+    if (rule?.type === 'number' && typeof value === 'string' && /^-?\d+(?:\.\d+)?$/.test(value))
+        return displayDecimal(value);
     if (rule?.type === 'enum' && typeof value === 'string')
         return humanize(value);
     if (rule?.type !== 'money' || !rule.currency || integerValue(value) === undefined)
@@ -95,7 +98,7 @@ export function aggregateDisplay(aggregate, source, entity, presentation) {
             .sort(([a], [b]) => a.localeCompare(b))
             .map(([currency, value]) => formatMoneyMajor(BigInt(value), currency))
             .join('; ');
-    return presentationValue(aggregate.values[source], entity, aggregate.name, true, presentation);
+    return presentationValue(aggregate.exactValues?.[source] ?? aggregate.values[source], entity, aggregate.name, true, presentation);
 }
 export function rawAggregateDisplay(aggregate, source, entity, presentation) {
     if (aggregate.type === 'count')
@@ -126,9 +129,10 @@ export function presentationDifference(expected, actual, entityName, field, aggr
             return `${delta > 0n ? '+' : ''}${formatMoneyMajor(delta, rule.currency)}`;
         return `${delta > 0n ? '+' : ''}${delta.toLocaleString('en-US')}`.replace(/^-/, '−');
     }
-    if (typeof expected === 'number' && typeof actual === 'number') {
-        const delta = actual - expected;
-        return `${delta > 0 ? '+' : ''}${delta.toLocaleString('en-US', { maximumFractionDigits: 12 })}`.replace(/^-/, '−');
+    if ((typeof expected === 'number' && typeof actual === 'number') ||
+        (rule?.type === 'number' && expected !== undefined && actual !== undefined)) {
+        const delta = sumDecimal(String(actual), negateDecimal(String(expected)));
+        return `${delta !== '0' && !delta.startsWith('-') ? '+' : ''}${displayDecimal(delta)}`;
     }
     return '—';
 }
@@ -152,7 +156,7 @@ export function aggregateDifference(aggregate, source, reference, entity, presen
             return `${delta > 0n ? '+' : ''}${formatMoneyMajor(delta, currency)}`;
         })
             .join('; ');
-    return presentationDifference(aggregate.values[reference], aggregate.values[source], entity, aggregate.name, true, presentation);
+    return presentationDifference(aggregate.exactValues?.[reference] ?? aggregate.values[reference], aggregate.exactValues?.[source] ?? aggregate.values[source], entity, aggregate.name, true, presentation);
 }
 export function aggregateStatus(aggregate, source, reference) {
     if (aggregate.sourceStatus?.[source] === 'not_mapped')
@@ -170,7 +174,8 @@ export function aggregateStatus(aggregate, source, reference) {
     const matches = currencies.length
         ? currencies.every((currency) => BigInt(aggregate.valuesByCurrency?.[source]?.[currency] ?? 0) ===
             BigInt(aggregate.valuesByCurrency?.[reference]?.[currency] ?? 0))
-        : aggregate.values[source] === aggregate.values[reference];
+        : (aggregate.exactValues?.[source] ?? aggregate.values[source]) ===
+            (aggregate.exactValues?.[reference] ?? aggregate.values[reference]);
     if (matches)
         return 'Matches';
     const buckets = aggregate.reconciliationByCurrency?.[source];
@@ -179,7 +184,7 @@ export function aggregateStatus(aggregate, source, reference) {
         : aggregate.reconciliation?.[source]
             ? [aggregate.reconciliation[source]]
             : [];
-    return parts.length && parts.every((part) => BigInt(part.remainder) === 0n && !part.uncertain?.keys.length)
+    return parts.length && parts.every((part) => sumDecimal(part.remainder) === '0' && !part.uncertain?.keys.length)
         ? 'Accounted for'
         : 'Needs review';
 }

package/lib/core/dist/normalize.js

@@ -1,11 +1,14 @@
 import { DateTime } from 'luxon';
 import { wallClock } from './types.js';
+import { normalizeDate } from './calendar.js';
 export function sourceRule(rule, config, source = '') {
     return (rule && {
         ...rule,
         locale: rule.locale ?? config?.locale ?? 'en-US',
         numberFormat: rule.numberFormat ?? config?.numberFormat,
-        format: typeof rule.format === 'string' ? rule.format : (rule.format?.[source] ?? config?.datetimeFormat),
+        format: typeof rule.format === 'string'
+            ? rule.format
+            : (rule.format?.[source] ?? (rule.type === 'date' ? config?.dateFormat : config?.datetimeFormat)),
     });
 }
 const localeFormats = new Map();
@@ -178,6 +181,8 @@ export function normalize(value, rule, source) {
         return null;
     if (value instanceof Date) {
         const local = Reflect.get(value, wallClock);
+        if (rule?.type === 'date')
+            return normalizeDate(typeof local === 'string' ? local.slice(0, 10) : value.toISOString().slice(0, 10), undefined, rule.locale ?? 'en-US');
         if (typeof local === 'string' && rule?.type === 'datetime') {
             const isoRule = { ...rule, format: undefined };
             try {
@@ -193,7 +198,8 @@ export function normalize(value, rule, source) {
     }
     if (!['string', 'number', 'boolean', 'bigint'].includes(typeof value))
         throw new Error('Expected a scalar field; map a scalar JSONPath or file column.');
-    const text = String(value).trim();
+    const trimmed = String(value).trim();
+    const text = rule?.whitespaceFold && ['string', 'enum'].includes(rule.type) ? trimmed.replace(/\s+/gu, ' ') : trimmed;
     if (text === '' && (rule?.emptyAsNull ?? true))
         return null;
     if (!rule) {
@@ -229,6 +235,8 @@ export function normalize(value, rule, source) {
         }
         case 'datetime':
             return normalizeDatetime(text, rule, source);
+        case 'date':
+            return normalizeDate(text, typeof rule.format === 'string' ? rule.format : rule.format?.[source], rule.locale ?? 'en-US');
     }
 }
 export function equalValues(a, b, rule) {

package/lib/core/dist/reconciliation-display.js

@@ -1,5 +1,6 @@
 import { reportLabels, uncertainKeyText } from './labels.js';
 import { formatMoneyMajor } from './money-display.js';
+import { displayDecimal, sumDecimal } from './aggregates.js';
 export function reconciliationDetails(report, presentation = {}, onlyEntity) {
     const labels = reportLabels(presentation), lines = [];
     for (const entity of report.entities) {
@@ -11,20 +12,22 @@ export function reconciliationDetails(report, presentation = {}, onlyEntity) {
                     ? Object.entries(value).map(([currency, parts]) => ({ currency, parts: parts }))
                     : [{ currency: undefined, parts: value }];
                 for (const { currency, parts } of partitions) {
-                    const actual = currency ? (aggregate.valuesByCurrency?.[source]?.[currency] ?? 0) : aggregate.values[source];
+                    const actual = currency
+                        ? (aggregate.valuesByCurrency?.[source]?.[currency] ?? 0)
+                        : (aggregate.exactValues?.[source] ?? aggregate.values[source]);
                     const expected = currency
                         ? (aggregate.valuesByCurrency?.[entity.reference]?.[currency] ?? 0)
-                        : aggregate.values[entity.reference];
+                        : (aggregate.exactValues?.[entity.reference] ?? aggregate.values[entity.reference]);
                     if (actual === undefined ||
                         expected === undefined ||
-                        (BigInt(actual) === BigInt(expected) &&
+                        (sumDecimal(actual) === sumDecimal(expected) &&
                             !parts.uncertain?.keys.length &&
                             !parts.missingRows.count &&
                             !parts.extraRows?.count))
                         continue;
                     const amount = (value) => {
-                        const n = BigInt(value);
-                        return `${n > 0n ? '+' : ''}${currency ? formatMoneyMajor(n, currency) : n.toLocaleString('en-US').replace(/^-/, '−')}`;
+                        const n = sumDecimal(value);
+                        return `${n !== '0' && !n.startsWith('-') ? '+' : ''}${currency ? formatMoneyMajor(BigInt(n), currency) : displayDecimal(n)}`;
                     };
                     const components = [
                         `Missing rows (${parts.missingRows.count.toLocaleString('en-US')}): ${amount(parts.missingRows.amount)}`,

package/lib/core/dist/reconciliation.js

@@ -1,4 +1,7 @@
 import { matchesFilter } from './filters.js';
+import { negateDecimal, sumDecimal } from './aggregates.js';
+const sum = (...values) => values.every((v) => typeof v === 'bigint') ? values.reduce((a, b) => a + b, 0n) : sumDecimal(...values);
+const zero = (value) => sumDecimal(value) === '0';
 const ledgers = new WeakMap();
 function contributes(entity, definition, row, source) {
     return !!row && matchesFilter(row, definition.where, entity, source);
@@ -12,7 +15,7 @@ function contribution(entity, definition, state, key, source, currency) {
     if (currency && state.currencies.get(key)?.get(definition.field) !== currency)
         return 0n;
     const value = row[definition.field];
-    return typeof value === 'bigint' ? value : 0n;
+    return typeof value === 'bigint' ? value : typeof value === 'number' ? sumDecimal(value) : 0n;
 }
 function collectBucket(entity, definition, baseline, current, source, reference, currency) {
     const parts = {
@@ -43,25 +46,25 @@ function collectBucket(entity, definition, baseline, current, source, reference,
             if (codes.length && codes.every((code) => code !== currency))
                 continue;
         }
-        const difference = actual - expected;
+        const difference = sum(actual, negateDecimal(expected));
         if (current.excluded.get(definition.name)?.has(key) || baseline.excluded.get(definition.name)?.has(key)) {
             parts.uncertain?.keys.push(current.parts.get(key) ?? baseline.parts.get(key) ?? [key]);
             if (parts.uncertain)
-                parts.uncertain.amount = BigInt(parts.uncertain.amount) + difference;
+                parts.uncertain.amount = sum(parts.uncertain.amount, difference);
         }
         else if (!actualRow) {
-            parts.missingRows.amount = BigInt(parts.missingRows.amount) + difference;
-            if (expected !== 0n)
+            parts.missingRows.amount = sum(parts.missingRows.amount, difference);
+            if (!zero(expected))
                 parts.missingRows.count++;
         }
         else if (!expectedRow) {
             if (parts.extraRows) {
-                parts.extraRows.amount = BigInt(parts.extraRows.amount) + difference;
-                if (actual !== 0n)
+                parts.extraRows.amount = sum(parts.extraRows.amount, difference);
+                if (!zero(actual))
                     parts.extraRows.count++;
             }
         }
-        else if (difference !== 0n) {
+        else if (!zero(difference)) {
             const moved = contributes(entity, definition, expectedRow, reference) !== contributes(entity, definition, actualRow, source);
             const currencyField = definition.type === 'sum'
                 ? (entity.compare[definition.field]?.currencyField ??
@@ -92,8 +95,6 @@ function collectBucket(entity, definition, baseline, current, source, reference,
 }
 export function collectReconciliation(entity, baseline, current, source, reference, summary) {
     for (const definition of entity.aggregates) {
-        if (definition.type === 'sum' && entity.compare[definition.field]?.type !== 'money')
-            continue;
         const aggregate = summary.aggregates?.find((item) => item.name === definition.name);
         if (!aggregate || aggregate.values[source] === undefined || aggregate.values[reference] === undefined)
             continue;
@@ -133,30 +134,35 @@ function attributeDifferences(report, entity, parts, source) {
         if (group >= 0) {
             const part = patterns.get(group) ?? { group, count: 0, amount: 0n };
             part.count++;
-            part.amount += entry.amount;
+            part.amount = sum(part.amount, entry.amount);
             patterns.set(group, part);
         }
         else {
             parts.valueDifferences.count++;
-            parts.valueDifferences.amount = BigInt(parts.valueDifferences.amount) + entry.amount;
+            parts.valueDifferences.amount = sum(parts.valueDifferences.amount, entry.amount);
         }
     }
     parts.patterns = [...patterns.values()];
 }
 function reconcileBucket(report, entity, aggregate, source, reference, parts, currency) {
     attributeDifferences(report, entity, parts, source);
-    const actual = currency ? (aggregate.valuesByCurrency?.[source]?.[currency] ?? 0) : aggregate.values[source];
-    const expected = currency ? (aggregate.valuesByCurrency?.[reference]?.[currency] ?? 0) : aggregate.values[reference];
+    const actual = currency
+        ? (aggregate.valuesByCurrency?.[source]?.[currency] ?? 0)
+        : (aggregate.exactValues?.[source] ?? aggregate.values[source]);
+    const expected = currency
+        ? (aggregate.valuesByCurrency?.[reference]?.[currency] ?? 0)
+        : (aggregate.exactValues?.[reference] ?? aggregate.values[reference]);
     if (actual === undefined || expected === undefined)
         return;
-    parts.remainder =
-        BigInt(actual) -
-            BigInt(expected) -
-            BigInt(parts.missingRows.amount) -
-            BigInt(parts.extraRows?.amount ?? 0) -
-            BigInt(parts.valueDifferences?.amount ?? 0) -
-            BigInt(parts.uncertain?.amount ?? 0) -
-            parts.patterns.reduce((sum, item) => sum + BigInt(item.amount), 0n);
+    const numeric = entity.aggregates.some((d) => d.name === aggregate.name && d.type === 'sum' && entity.compare[d.field]?.type === 'number');
+    const remainder = sumDecimal(String(actual), negateDecimal(String(expected)), ...[
+        parts.missingRows.amount,
+        parts.extraRows?.amount ?? 0n,
+        parts.valueDifferences?.amount ?? 0n,
+        parts.uncertain?.amount ?? 0n,
+        ...parts.patterns.map((p) => p.amount),
+    ].map(negateDecimal));
+    parts.remainder = numeric ? remainder : BigInt(remainder);
     parts.causes = [
         ...(parts.missingRows.count ? ['missing rows'] : []),
         ...(parts.extraRows?.count ? ['extra rows'] : []),
@@ -172,7 +178,7 @@ function reconcileBucket(report, entity, aggregate, source, reference, parts, cu
         item.source === source &&
         item.field === aggregate.name &&
         item.currency === currency);
-    if (finding && BigInt(parts.remainder) === 0n && !parts.uncertain?.keys.length)
+    if (finding && zero(parts.remainder) && !parts.uncertain?.keys.length)
         finding.accounting = { causes: parts.causes, remainder: '0' };
 }
 export function reconcileAggregates(report, entities) {

package/lib/core/dist/source-state.d.ts

@@ -22,6 +22,7 @@ export interface SourceState {
     identities: Map<string, number>;
     sums: Map<string, Map<string, DecimalSum>>;
     measurements: Map<string, number | bigint | undefined>;
+    exactMeasurements: Map<string, string>;
     excluded: Map<string, Set<string>>;
     count: number;
     position: number;

package/lib/core/dist/source-state.js

@@ -190,7 +190,7 @@ function measureSource(entity, source, config, state) {
         const excluded = new Set([...state.uncertain].flatMap(([key, uncertainFields]) => required.some((field) => uncertainFields.has(field)) ? [key] : []));
         state.excluded.set(aggregate.name, excluded);
         try {
-            state.measurements.set(aggregate.name, measureAggregate(aggregate, entity, source, { ...config, fields }, new Map([...state.rows].filter(([key]) => !excluded.has(key)))));
+            state.measurements.set(aggregate.name, measureAggregate(aggregate, entity, source, { ...config, fields }, new Map([...state.rows].filter(([key]) => !excluded.has(key))), state.exactMeasurements));
         }
         catch (error) {
             throw new Error(`Aggregate ${aggregate.name}: ${errorText(error)}`);
@@ -261,6 +261,7 @@ export async function readSource(entity, source, config, read, secrets, report,
         identities: new Map(),
         sums: new Map(),
         measurements: new Map(),
+        exactMeasurements: new Map(),
         excluded: new Map(),
         count: 0,
         position: 0,

package/lib/core/dist/types.d.ts

@@ -96,6 +96,7 @@ export interface AggregateResult {
     name: string;
     type: 'count' | 'sum';
     values: Record<string, number | bigint>;
+    exactValues?: Record<string, string>;
     rawValues?: Record<string, number | bigint>;
     valuesByCurrency?: Record<string, Record<string, bigint | string>>;
     rawValuesByCurrency?: Record<string, Record<string, bigint | string>>;
@@ -126,10 +127,10 @@ export interface FindingGroup {
     field: string;
     count: number;
     currency?: string;
-    cause?: 'timezone' | 'pagination' | 'stale_value';
+    cause?: 'timezone' | 'pagination' | 'stale_value' | 'case_only' | 'whitespace_only' | 'case_whitespace';
     referenceCount?: number;
     matchingCount?: number;
-    kind: 'datetime_offset' | 'money_delta' | 'value_mapping' | 'missing_stride' | 'extra_stride';
+    kind: 'datetime_offset' | 'money_delta' | 'value_mapping' | 'formatting_difference' | 'number_change' | 'missing_stride' | 'extra_stride';
     delta?: string;
     stride?: string;
     keys?: string[];

package/lib/scenarios/dist/config.d.ts

@@ -143,7 +143,9 @@ export declare const targetSchema: z.ZodObject<{
         fields: z.ZodRecord<z.ZodString, z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
             path: z.ZodString;
             label: z.ZodOptional<z.ZodString>;
+            missing: z.ZodOptional<z.ZodNull>;
         }, z.core.$strict>]>>;
+        absentAsNull: z.ZodDefault<z.ZodBoolean>;
         items: z.ZodDefault<z.ZodString>;
         headers: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodString>>;
         headersEnv: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
@@ -199,6 +201,7 @@ export declare const targetSchema: z.ZodObject<{
             group: z.ZodOptional<z.ZodString>;
         }, z.core.$strict>>;
         datetimeFormat: z.ZodOptional<z.ZodString>;
+        dateFormat: z.ZodOptional<z.ZodString>;
         type: z.ZodLiteral<"rest">;
         label: z.ZodOptional<z.ZodString>;
         rollup: z.ZodOptional<z.ZodObject<{
@@ -239,6 +242,7 @@ export declare const targetSchema: z.ZodObject<{
             group: z.ZodOptional<z.ZodString>;
         }, z.core.$strict>>;
         datetimeFormat: z.ZodOptional<z.ZodString>;
+        dateFormat: z.ZodOptional<z.ZodString>;
         type: z.ZodLiteral<"csv">;
         label: z.ZodOptional<z.ZodString>;
         rollup: z.ZodOptional<z.ZodObject<{
@@ -273,6 +277,7 @@ export declare const targetSchema: z.ZodObject<{
             group: z.ZodOptional<z.ZodString>;
         }, z.core.$strict>>;
         datetimeFormat: z.ZodOptional<z.ZodString>;
+        dateFormat: z.ZodOptional<z.ZodString>;
         type: z.ZodLiteral<"xlsx">;
         label: z.ZodOptional<z.ZodString>;
         rollup: z.ZodOptional<z.ZodObject<{
@@ -313,6 +318,7 @@ export declare const targetSchema: z.ZodObject<{
             group: z.ZodOptional<z.ZodString>;
         }, z.core.$strict>>;
         datetimeFormat: z.ZodOptional<z.ZodString>;
+        dateFormat: z.ZodOptional<z.ZodString>;
         type: z.ZodLiteral<"ui">;
         label: z.ZodOptional<z.ZodString>;
         rollup: z.ZodOptional<z.ZodObject<{
@@ -349,6 +355,7 @@ export declare const targetSchema: z.ZodObject<{
             group: z.ZodOptional<z.ZodString>;
         }, z.core.$strict>>;
         datetimeFormat: z.ZodOptional<z.ZodString>;
+        dateFormat: z.ZodOptional<z.ZodString>;
         type: z.ZodLiteral<"sql">;
         label: z.ZodOptional<z.ZodString>;
         rollup: z.ZodOptional<z.ZodObject<{
@@ -448,7 +455,9 @@ export declare function loadTarget(file: string, env: NodeJS.ProcessEnv): Promis
             fields: Record<string, string | {
                 path: string;
                 label?: string | undefined;
+                missing?: null | undefined;
             }>;
+            absentAsNull: boolean;
             items: string;
             headers: Record<string, string>;
             headersEnv: Record<string, string | {
@@ -509,6 +518,7 @@ export declare function loadTarget(file: string, env: NodeJS.ProcessEnv): Promis
                 group?: string | undefined;
             } | undefined;
             datetimeFormat?: string | undefined;
+            dateFormat?: string | undefined;
             label?: string | undefined;
             rollup?: {
                 by: string | string[];
@@ -545,6 +555,7 @@ export declare function loadTarget(file: string, env: NodeJS.ProcessEnv): Promis
                 group?: string | undefined;
             } | undefined;
             datetimeFormat?: string | undefined;
+            dateFormat?: string | undefined;
             label?: string | undefined;
             rollup?: {
                 by: string | string[];
@@ -579,6 +590,7 @@ export declare function loadTarget(file: string, env: NodeJS.ProcessEnv): Promis
                 group?: string | undefined;
             } | undefined;
             datetimeFormat?: string | undefined;
+            dateFormat?: string | undefined;
             label?: string | undefined;
             rollup?: {
                 by: string | string[];
@@ -619,6 +631,7 @@ export declare function loadTarget(file: string, env: NodeJS.ProcessEnv): Promis
                 group?: string | undefined;
             } | undefined;
             datetimeFormat?: string | undefined;
+            dateFormat?: string | undefined;
             label?: string | undefined;
             rollup?: {
                 by: string | string[];
@@ -655,6 +668,7 @@ export declare function loadTarget(file: string, env: NodeJS.ProcessEnv): Promis
                 group?: string | undefined;
             } | undefined;
             datetimeFormat?: string | undefined;
+            dateFormat?: string | undefined;
             label?: string | undefined;
             rollup?: {
                 by: string | string[];