dsh-data-quality 0.1.3 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (96) hide show
  1. package/CHANGELOG.md +26 -0
  2. package/README.es.md +7 -1
  3. package/README.hi.md +7 -1
  4. package/README.md +22 -9
  5. package/README.pt.md +7 -1
  6. package/README.zh.md +7 -1
  7. package/cordis.patch.yml +11 -0
  8. package/lib/index.js +1405 -140
  9. package/lib/types/config.d.ts +12 -0
  10. package/lib/types/config.d.ts.map +1 -1
  11. package/lib/types/config.js +35 -0
  12. package/lib/types/config.js.map +1 -1
  13. package/lib/types/contract.d.ts +83 -0
  14. package/lib/types/contract.d.ts.map +1 -0
  15. package/lib/types/contract.js +144 -0
  16. package/lib/types/contract.js.map +1 -0
  17. package/lib/types/dataset.d.ts +50 -3
  18. package/lib/types/dataset.d.ts.map +1 -1
  19. package/lib/types/dataset.js +76 -21
  20. package/lib/types/dataset.js.map +1 -1
  21. package/lib/types/events.d.ts +2 -0
  22. package/lib/types/events.d.ts.map +1 -1
  23. package/lib/types/events.js.map +1 -1
  24. package/lib/types/index.d.ts +11 -8
  25. package/lib/types/index.d.ts.map +1 -1
  26. package/lib/types/index.js +15 -6
  27. package/lib/types/index.js.map +1 -1
  28. package/lib/types/presets.d.ts +28 -0
  29. package/lib/types/presets.d.ts.map +1 -0
  30. package/lib/types/presets.js +134 -0
  31. package/lib/types/presets.js.map +1 -0
  32. package/lib/types/profile.d.ts +38 -1
  33. package/lib/types/profile.d.ts.map +1 -1
  34. package/lib/types/profile.js +64 -10
  35. package/lib/types/profile.js.map +1 -1
  36. package/lib/types/provider-local.d.ts +7 -1
  37. package/lib/types/provider-local.d.ts.map +1 -1
  38. package/lib/types/provider-local.js +72 -2
  39. package/lib/types/provider-local.js.map +1 -1
  40. package/lib/types/scorecard.d.ts +50 -0
  41. package/lib/types/scorecard.d.ts.map +1 -0
  42. package/lib/types/scorecard.js +186 -0
  43. package/lib/types/scorecard.js.map +1 -0
  44. package/lib/types/service.d.ts +32 -2
  45. package/lib/types/service.d.ts.map +1 -1
  46. package/lib/types/service.js.map +1 -1
  47. package/lib/types/store.d.ts +21 -1
  48. package/lib/types/store.d.ts.map +1 -1
  49. package/lib/types/store.js +13 -1
  50. package/lib/types/store.js.map +1 -1
  51. package/lib/types/tools/clean.d.ts.map +1 -1
  52. package/lib/types/tools/clean.js +76 -1
  53. package/lib/types/tools/clean.js.map +1 -1
  54. package/lib/types/tools/profile-report-schema.d.ts +315 -0
  55. package/lib/types/tools/profile-report-schema.d.ts.map +1 -0
  56. package/lib/types/tools/profile-report-schema.js +95 -0
  57. package/lib/types/tools/profile-report-schema.js.map +1 -0
  58. package/lib/types/tools/profile.d.ts.map +1 -1
  59. package/lib/types/tools/profile.js +6 -53
  60. package/lib/types/tools/profile.js.map +1 -1
  61. package/lib/types/tools/report.d.ts +14 -0
  62. package/lib/types/tools/report.d.ts.map +1 -0
  63. package/lib/types/tools/report.js +86 -0
  64. package/lib/types/tools/report.js.map +1 -0
  65. package/lib/types/tools/shared.d.ts.map +1 -1
  66. package/lib/types/tools/shared.js +22 -2
  67. package/lib/types/tools/shared.js.map +1 -1
  68. package/lib/types/tools/verify.d.ts.map +1 -1
  69. package/lib/types/tools/verify.js +36 -1
  70. package/lib/types/tools/verify.js.map +1 -1
  71. package/lib/types/verify.d.ts +49 -3
  72. package/lib/types/verify.d.ts.map +1 -1
  73. package/lib/types/verify.js +111 -4
  74. package/lib/types/verify.js.map +1 -1
  75. package/lib/types/version.d.ts +1 -1
  76. package/lib/types/version.js +1 -1
  77. package/package.json +1 -1
  78. package/src/config.ts +52 -0
  79. package/src/contract.ts +190 -0
  80. package/src/dataset.ts +101 -21
  81. package/src/events.ts +2 -0
  82. package/src/index.ts +25 -8
  83. package/src/presets.ts +146 -0
  84. package/src/profile.ts +101 -11
  85. package/src/provider-local.ts +77 -3
  86. package/src/scorecard.ts +244 -0
  87. package/src/service.ts +34 -2
  88. package/src/store.ts +28 -2
  89. package/src/tools/clean.ts +76 -1
  90. package/src/tools/profile-report-schema.ts +97 -0
  91. package/src/tools/profile.ts +8 -56
  92. package/src/tools/report.ts +108 -0
  93. package/src/tools/shared.ts +21 -2
  94. package/src/tools/verify.ts +40 -2
  95. package/src/verify.ts +151 -5
  96. package/src/version.ts +1 -1
package/lib/index.js CHANGED
@@ -4,6 +4,7 @@ import { defineDomain, domainTable } from "@deepseek-ai/dsh-storage-domain";
4
4
  import { mkdir, open, stat, writeFile } from "node:fs/promises";
5
5
  import path from "node:path";
6
6
  import { createHash } from "node:crypto";
7
+ import { TextDecoder } from "node:util";
7
8
  import { KNOWN_SESSION_EVENT_TYPES } from "@deepseek-ai/dsh-session";
8
9
  import { Service } from "@deepseek-ai/cordis";
9
10
  import { defineTool } from "@deepseek-ai/dsh-tools";
@@ -14,6 +15,15 @@ import { defineTool } from "@deepseek-ai/dsh-tools";
14
15
  * step validates bounds so misconfiguration fails loud at mount.
15
16
  * @module dsh-data-quality/config
16
17
  */
18
+ /** The six scorecard dimension ids, in report order (kept in sync with `scorecard.ts`). */
19
+ const SCORECARD_DIMENSIONS = [
20
+ "completeness",
21
+ "uniqueness",
22
+ "validity",
23
+ "consistency",
24
+ "timeliness",
25
+ "accuracy"
26
+ ];
17
27
  /** Schemastery schema: the loader validates and fills defaults before `apply`. */
18
28
  const Config = z.object({
19
29
  enabled: z.boolean().default(true),
@@ -28,7 +38,22 @@ const Config = z.object({
28
38
  ".jsonl"
29
39
  ]),
30
40
  workspaceRoot: z.string().default(""),
31
- storeReports: z.boolean().default(true)
41
+ storeReports: z.boolean().default(true),
42
+ scorecardWeights: z.object({
43
+ completeness: z.number().default(1),
44
+ uniqueness: z.number().default(1),
45
+ validity: z.number().default(1),
46
+ consistency: z.number().default(1),
47
+ timeliness: z.number().default(1),
48
+ accuracy: z.number().default(1)
49
+ }).default({
50
+ completeness: 1,
51
+ uniqueness: 1,
52
+ validity: 1,
53
+ consistency: 1,
54
+ timeliness: 1,
55
+ accuracy: 1
56
+ })
32
57
  });
33
58
  /** Throw unless `value` is a positive safe integer. */
34
59
  function assertPositiveInt(name, value) {
@@ -63,6 +88,7 @@ function resolveConfig(config = {}) {
63
88
  return ext;
64
89
  });
65
90
  if (allowedExtensions.length === 0) throw new TypeError("allowedExtensions must not be empty");
91
+ const scorecardWeights = resolveScorecardWeights(config.scorecardWeights);
66
92
  return {
67
93
  enabled: config.enabled ?? true,
68
94
  maxRows,
@@ -71,9 +97,20 @@ function resolveConfig(config = {}) {
71
97
  evidenceRowLimit,
72
98
  allowedExtensions,
73
99
  workspaceRoot: config.workspaceRoot ?? "",
74
- storeReports: config.storeReports ?? true
100
+ storeReports: config.storeReports ?? true,
101
+ scorecardWeights
75
102
  };
76
103
  }
104
+ /** Resolve scorecard weights, filling defaults and rejecting non-negative violations loudly. */
105
+ function resolveScorecardWeights(weights = {}) {
106
+ const out = {};
107
+ for (const dimension of SCORECARD_DIMENSIONS) {
108
+ const value = weights[dimension] ?? 1;
109
+ if (typeof value !== "number" || !Number.isFinite(value) || value < 0) throw new TypeError(`scorecardWeights.${dimension} must be a non-negative finite number, got ${String(value)}`);
110
+ out[dimension] = value;
111
+ }
112
+ return out;
113
+ }
77
114
  //#endregion
78
115
  //#region src/dataset.ts
79
116
  /**
@@ -127,13 +164,40 @@ function throwIfAborted(signal) {
127
164
  if (signal?.aborted === true) throw signal.reason instanceof Error ? signal.reason : /* @__PURE__ */ new Error("operation aborted");
128
165
  }
129
166
  /**
130
- * Read a dataset file under the size cap.
167
+ * Detect a UTF-8 byte-order mark and validate the byte sequence. Invalid
168
+ * UTF-8 is a data-quality finding the profile reports (`validUtf8: false`)
169
+ * rather than a structural error that blocks the read — the decoded text
170
+ * keeps U+FFFD replacement characters so the profile can still run.
171
+ * @param buffer - raw file bytes.
172
+ * @returns the encoding metadata.
173
+ */
174
+ function detectEncoding(buffer) {
175
+ const bom = buffer.length >= 3 && buffer[0] === 239 && buffer[1] === 187 && buffer[2] === 191 ? "utf-8" : null;
176
+ const body = bom === "utf-8" ? buffer.subarray(3) : buffer;
177
+ let validUtf8 = true;
178
+ try {
179
+ new TextDecoder("utf-8", { fatal: true }).decode(body);
180
+ } catch {
181
+ validUtf8 = false;
182
+ }
183
+ return {
184
+ bom,
185
+ validUtf8
186
+ };
187
+ }
188
+ /** Decode file bytes, stripping a UTF-8 BOM and preserving replacement characters for invalid bytes. */
189
+ function decodeUtf8Text(buffer, encoding) {
190
+ const body = encoding.bom === "utf-8" ? buffer.subarray(3) : buffer;
191
+ return new TextDecoder("utf-8").decode(body);
192
+ }
193
+ /**
194
+ * Read a dataset file under the size cap, detecting its encoding.
131
195
  * @param absolutePath - normalized absolute path (from {@link resolveWorkspacePath}).
132
196
  * @param config - resolved config (size cap).
133
197
  * @param signal - optional abort signal honored around the read.
134
- * @returns the UTF-8 text.
198
+ * @returns the decoded UTF-8 text plus its encoding metadata.
135
199
  */
136
- async function readDatasetText(absolutePath, config, signal) {
200
+ async function readDatasetFile(absolutePath, config, signal) {
137
201
  throwIfAborted(signal);
138
202
  let info;
139
203
  try {
@@ -147,7 +211,12 @@ async function readDatasetText(absolutePath, config, signal) {
147
211
  const handle = await open(absolutePath, "r");
148
212
  try {
149
213
  throwIfAborted(signal);
150
- return await handle.readFile("utf8");
214
+ const buffer = await handle.readFile();
215
+ const encoding = detectEncoding(buffer);
216
+ return {
217
+ text: decodeUtf8Text(buffer, encoding),
218
+ encoding
219
+ };
151
220
  } finally {
152
221
  await handle.close();
153
222
  }
@@ -316,12 +385,21 @@ function parseJsonLines(text) {
316
385
  * @returns the parsed table.
317
386
  */
318
387
  async function loadTable(absolutePath, config, signal) {
319
- const text = await readDatasetText(absolutePath, config, signal);
388
+ const { text, encoding } = await readDatasetFile(absolutePath, config, signal);
320
389
  const ext = path.extname(absolutePath).toLowerCase();
321
390
  throwIfAborted(signal);
322
- if (ext === ".csv") return parseDelimited(text, ",", config, signal);
323
- if (ext === ".tsv") return parseDelimited(text, " ", config, signal);
324
- return parseJsonTable(text, ext, config, signal);
391
+ if (ext === ".csv") return {
392
+ ...parseDelimited(text, ",", config, signal),
393
+ encoding
394
+ };
395
+ if (ext === ".tsv") return {
396
+ ...parseDelimited(text, " ", config, signal),
397
+ encoding
398
+ };
399
+ return {
400
+ ...parseJsonTable(text, ext, config, signal),
401
+ encoding
402
+ };
325
403
  }
326
404
  /**
327
405
  * Load a dataset as a citation-checkable document root: CSV/TSV become
@@ -334,14 +412,20 @@ async function loadTable(absolutePath, config, signal) {
334
412
  */
335
413
  async function loadDocument(absolutePath, config, signal) {
336
414
  const ext = path.extname(absolutePath).toLowerCase();
337
- if (ext === ".json") return {
338
- kind: "json",
339
- value: parseJsonDocument(await readDatasetText(absolutePath, config, signal))
340
- };
341
- if (ext === ".jsonl") return {
342
- kind: "json",
343
- value: parseJsonLines(await readDatasetText(absolutePath, config, signal))
344
- };
415
+ if (ext === ".json") {
416
+ const { text } = await readDatasetFile(absolutePath, config, signal);
417
+ return {
418
+ kind: "json",
419
+ value: parseJsonDocument(text)
420
+ };
421
+ }
422
+ if (ext === ".jsonl") {
423
+ const { text } = await readDatasetFile(absolutePath, config, signal);
424
+ return {
425
+ kind: "json",
426
+ value: parseJsonLines(text)
427
+ };
428
+ }
345
429
  const table = await loadTable(absolutePath, config, signal);
346
430
  return {
347
431
  kind: "table",
@@ -390,18 +474,17 @@ const DATE_PATTERNS = [
390
474
  /^(\d{4})\/(\d{1,2})\/(\d{1,2})$/u,
391
475
  /^(\d{4})-(\d{1,2})-(\d{1,2})[ T](\d{1,2}):(\d{2})(?::(\d{2}))?(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?$/u
392
476
  ];
393
- /**
394
- * Deterministic date parse to epoch milliseconds. Accepts `YYYY-MM-DD`,
395
- * `YYYY/MM/DD`, and ISO-like datetimes (date-only forms read as UTC midnight).
396
- * Calendar-invalid dates (e.g. 2025-13-40) reject. Returns `undefined` when
397
- * the cell is not a recognized date.
398
- * @param cell - the cell to parse (`undefined` when the column is absent).
399
- * @returns epoch milliseconds, or `undefined`.
400
- */
401
- function parseDate(cell) {
477
+ /** Deterministic date-format labels in {@link DATE_PATTERNS} order. */
478
+ const DATE_FORMATS = [
479
+ "iso-date",
480
+ "slash-date",
481
+ "datetime"
482
+ ];
483
+ /** Parse one date cell to its epoch plus format; `undefined` when unrecognized. */
484
+ function parseDateCellInternal(cell) {
402
485
  if (typeof cell !== "string") return void 0;
403
486
  const text = cell.trim();
404
- for (const pattern of DATE_PATTERNS) {
487
+ for (const [index, pattern] of DATE_PATTERNS.entries()) {
405
488
  const match = pattern.exec(text);
406
489
  if (match === null) continue;
407
490
  const year = Number(match[1]);
@@ -414,9 +497,43 @@ function parseDate(cell) {
414
497
  const epoch = Date.UTC(year, month - 1, day, hour, minute, second);
415
498
  const check = new Date(epoch);
416
499
  if (check.getUTCMonth() !== month - 1 || check.getUTCDate() !== day) return void 0;
417
- return epoch;
500
+ const format = DATE_FORMATS[index];
501
+ if (format === void 0) return void 0;
502
+ return {
503
+ epoch,
504
+ format
505
+ };
418
506
  }
419
507
  }
508
+ /**
509
+ * Deterministic date parse to epoch milliseconds. Accepts `YYYY-MM-DD`,
510
+ * `YYYY/MM/DD`, and ISO-like datetimes (date-only forms read as UTC midnight).
511
+ * Calendar-invalid dates (e.g. 2025-13-40) reject. Returns `undefined` when
512
+ * the cell is not a recognized date.
513
+ * @param cell - the cell to parse (`undefined` when the column is absent).
514
+ * @returns epoch milliseconds, or `undefined`.
515
+ */
516
+ function parseDate(cell) {
517
+ return parseDateCellInternal(cell)?.epoch;
518
+ }
519
+ /**
520
+ * Parse one date cell to its epoch plus source format label.
521
+ * @param cell - the cell to parse (`undefined` when the column is absent).
522
+ * @returns the parsed date, or `undefined` when the cell is not a recognized date.
523
+ */
524
+ function parseDateCell(cell) {
525
+ return parseDateCellInternal(cell);
526
+ }
527
+ /**
528
+ * The format label of a recognized date cell (`iso-date` / `slash-date` /
529
+ * `datetime`); `undefined` when the cell is not a recognized date. Used to
530
+ * measure a date column's format consistency.
531
+ * @param cell - the cell to inspect (`undefined` when the column is absent).
532
+ * @returns the source format label, or `undefined`.
533
+ */
534
+ function dateFormatOf(cell) {
535
+ return parseDateCellInternal(cell)?.format;
536
+ }
420
537
  /** Boolean parse: true/false/yes/no/1/0, case-insensitive. */
421
538
  function parseBoolean(cell) {
422
539
  if (typeof cell === "boolean") return cell;
@@ -440,7 +557,8 @@ const reportRecordSchema = z$1.object({
440
557
  "profile",
441
558
  "clean",
442
559
  "verify",
443
- "citations"
560
+ "citations",
561
+ "clean-diff"
444
562
  ]),
445
563
  at: z$1.number().int().nonnegative(),
446
564
  dataset: z$1.string(),
@@ -452,6 +570,18 @@ const dataQualityDomainSpec = defineDomain({
452
570
  version: 1,
453
571
  tables: { reports: domainTable(reportRecordSchema) }
454
572
  });
573
+ /** Well-formed report-key shape: `<17-digit timestamp>-<kind>-<8-hex fingerprint>`. */
574
+ const REPORT_KEY_PATTERN = /^\d{17}-(?:profile|clean|verify|citations|clean-diff)-[0-9a-f]{8}$/u;
575
+ /**
576
+ * Whether `key` is a well-formed, path-safe storage report key. Rejects any
577
+ * key with separators, traversal, or unexpected characters before it can be
578
+ * handed to the storage backend.
579
+ * @param key - candidate report key.
580
+ * @returns whether the key matches the deterministic report-key format.
581
+ */
582
+ function isValidReportKey(key) {
583
+ return REPORT_KEY_PATTERN.test(key);
584
+ }
455
585
  /** Pad to two digits for the key timestamp. */
456
586
  function pad2(value) {
457
587
  return String(value).padStart(2, "0");
@@ -842,6 +972,199 @@ function serializeDelimited(columns, rows, delimiter) {
842
972
  return `${lines.join("\n")}\n`;
843
973
  }
844
974
  //#endregion
975
+ //#region src/scorecard.ts
976
+ /**
977
+ * DAMA-style six-dimension quality scorecard over a parsed {@link Table}.
978
+ * Pure and deterministic: every rate derives from one full-table pass, the
979
+ * only clock is the injected `now`, and no dimension fabricates a score it
980
+ * cannot defend — `accuracy` stays `null` (undetermined) without a declared
981
+ * schema and `timeliness` stays `null` without date cells.
982
+ * @module dsh-data-quality/scorecard
983
+ */
984
+ /** Round to 6 significant digits (mirrors the profile report's rounding). */
985
+ function round6$1(value) {
986
+ return Number(value.toPrecision(6));
987
+ }
988
+ /** Increment a format-label counter. */
989
+ function bump(formats, tag) {
990
+ formats.set(tag, (formats.get(tag) ?? 0) + 1);
991
+ }
992
+ /** Inferred column type from the full-table class counts (mirrors `profileColumn`). */
993
+ function inferType(stats) {
994
+ if (stats.present === 0) return "empty";
995
+ if (stats.number === stats.present) return "number";
996
+ if (stats.date === stats.present) return "date";
997
+ if (stats.boolean === stats.present) return "boolean";
998
+ if (stats.string === stats.present) return "string";
999
+ return "mixed";
1000
+ }
1001
+ /** Format the `value / total` ratio as a plain `note` fragment. */
1002
+ function ratioNote(numerator, denominator) {
1003
+ return `${numerator}/${denominator}`;
1004
+ }
1005
+ /** Build one determinable dimension with its ratio note. */
1006
+ function dimension(name, score, numerator, denominator) {
1007
+ return {
1008
+ name,
1009
+ score,
1010
+ note: ratioNote(numerator, denominator)
1011
+ };
1012
+ }
1013
+ /**
1014
+ * Compute the six-dimension scorecard over a full table:
1015
+ * - `completeness` — non-empty cell rate.
1016
+ * - `uniqueness` — unique full-row content rate (`1 - duplicateRows / rowCount`).
1017
+ * - `validity` — present cells conforming to their column's inferred type.
1018
+ * - `consistency` — present cells in their column's dominant format (a date
1019
+ * column's unified `YYYY-MM-DD` vs `YYYY/MM/DD` vs datetime ratio).
1020
+ * - `timeliness` — date cells not future-dated relative to the injected `now`.
1021
+ * - `accuracy` — declared-schema agreement (limited definition); `null`
1022
+ * (undetermined) without a declared schema — never fabricated.
1023
+ * @param table - the parsed dataset (full table, not the sampled cards).
1024
+ * @param options - injected clock, duplicate-row count, optional declared schema, optional weights, abort signal.
1025
+ * @returns the scorecard.
1026
+ */
1027
+ function computeScorecard(table, options) {
1028
+ throwIfAborted(options.signal);
1029
+ const stats = /* @__PURE__ */ new Map();
1030
+ for (const column of table.columns) stats.set(column, {
1031
+ present: 0,
1032
+ number: 0,
1033
+ date: 0,
1034
+ boolean: 0,
1035
+ string: 0,
1036
+ formats: /* @__PURE__ */ new Map()
1037
+ });
1038
+ let missingCells = 0;
1039
+ let dateCells = 0;
1040
+ let futureDateCells = 0;
1041
+ for (const [index, row] of table.rows.entries()) {
1042
+ if (index % 1024 === 0) throwIfAborted(options.signal);
1043
+ for (const column of table.columns) {
1044
+ const cell = row[column];
1045
+ const columnStats = stats.get(column);
1046
+ if (isMissing(cell)) {
1047
+ missingCells += 1;
1048
+ continue;
1049
+ }
1050
+ columnStats.present += 1;
1051
+ if (parseNumeric(cell) !== void 0) {
1052
+ columnStats.number += 1;
1053
+ bump(columnStats.formats, "number");
1054
+ continue;
1055
+ }
1056
+ const date = parseDateCell(cell);
1057
+ if (date !== void 0) {
1058
+ columnStats.date += 1;
1059
+ bump(columnStats.formats, date.format);
1060
+ dateCells += 1;
1061
+ if (date.epoch > options.now) futureDateCells += 1;
1062
+ continue;
1063
+ }
1064
+ if (parseBoolean(cell) !== void 0) {
1065
+ columnStats.boolean += 1;
1066
+ bump(columnStats.formats, "boolean");
1067
+ continue;
1068
+ }
1069
+ columnStats.string += 1;
1070
+ bump(columnStats.formats, typeof cell === "string" ? "string" : "json");
1071
+ }
1072
+ }
1073
+ const totalCells = table.rows.length * table.columns.length;
1074
+ let presentCells = 0;
1075
+ let invalidCells = 0;
1076
+ let consistentCells = 0;
1077
+ for (const [, columnStats] of stats) {
1078
+ presentCells += columnStats.present;
1079
+ const dominantClass = Math.max(columnStats.number, columnStats.date, columnStats.boolean, columnStats.string);
1080
+ invalidCells += columnStats.present - dominantClass;
1081
+ if (columnStats.present === 0) continue;
1082
+ let dominant = 0;
1083
+ for (const count of columnStats.formats.values()) if (count > dominant) dominant = count;
1084
+ consistentCells += dominant;
1085
+ }
1086
+ const dimensions = [
1087
+ totalCells === 0 ? {
1088
+ name: "completeness",
1089
+ score: null,
1090
+ note: "undetermined: no cells"
1091
+ } : dimension("completeness", round6$1(1 - missingCells / totalCells), presentCells, totalCells),
1092
+ table.rows.length === 0 ? {
1093
+ name: "uniqueness",
1094
+ score: null,
1095
+ note: "undetermined: no rows"
1096
+ } : dimension("uniqueness", round6$1(1 - options.duplicateRows / table.rows.length), table.rows.length - options.duplicateRows, table.rows.length),
1097
+ presentCells === 0 ? {
1098
+ name: "validity",
1099
+ score: null,
1100
+ note: "undetermined: no present cells"
1101
+ } : dimension("validity", round6$1(1 - invalidCells / presentCells), presentCells - invalidCells, presentCells),
1102
+ presentCells === 0 ? {
1103
+ name: "consistency",
1104
+ score: null,
1105
+ note: "undetermined: no present cells"
1106
+ } : dimension("consistency", round6$1(consistentCells / presentCells), consistentCells, presentCells),
1107
+ dateCells === 0 ? {
1108
+ name: "timeliness",
1109
+ score: null,
1110
+ note: "undetermined: no date cells"
1111
+ } : dimension("timeliness", round6$1(1 - futureDateCells / dateCells), dateCells - futureDateCells, dateCells),
1112
+ computeAccuracy(stats, table, options.declaredSchema)
1113
+ ];
1114
+ const scores = dimensions.map((entry) => entry.score).filter((score) => score !== null);
1115
+ return {
1116
+ dimensions,
1117
+ overall: scores.length === 0 ? null : round6$1(scores.reduce((sum, score) => sum + score, 0) / scores.length),
1118
+ weightedOverall: computeWeightedOverall(dimensions, options.weights)
1119
+ };
1120
+ }
1121
+ /** Weighted mean over determinable dimensions (weights default to 1; validated non-negative by config). */
1122
+ function computeWeightedOverall(dimensions, weights) {
1123
+ let weightedSum = 0;
1124
+ let weightTotal = 0;
1125
+ for (const entry of dimensions) {
1126
+ if (entry.score === null) continue;
1127
+ const weight = weights?.[entry.name] ?? 1;
1128
+ weightedSum += weight * entry.score;
1129
+ weightTotal += weight;
1130
+ }
1131
+ return weightTotal === 0 ? null : round6$1(weightedSum / weightTotal);
1132
+ }
1133
+ /**
1134
+ * Limited `accuracy` definition: agreement between the full-table inferred
1135
+ * type of each declared column and the declared type. Without a declared
1136
+ * schema (or an external truth source) the dimension is `null` (undetermined)
1137
+ * — accuracy is never fabricated.
1138
+ */
1139
+ function computeAccuracy(stats, table, declaredSchema) {
1140
+ if (declaredSchema === void 0 || Object.keys(declaredSchema).length === 0) return {
1141
+ name: "accuracy",
1142
+ score: null,
1143
+ note: "undetermined: no declared schema or external truth to compare against; accuracy is never fabricated"
1144
+ };
1145
+ let checked = 0;
1146
+ let matched = 0;
1147
+ const mismatches = [];
1148
+ for (const column of table.columns) {
1149
+ const declared = declaredSchema[column];
1150
+ if (declared === void 0) continue;
1151
+ checked += 1;
1152
+ const inferred = inferType(stats.get(column));
1153
+ if (inferred === declared) matched += 1;
1154
+ else mismatches.push(`${column}: declared ${declared}, inferred ${inferred}`);
1155
+ }
1156
+ if (checked === 0) return {
1157
+ name: "accuracy",
1158
+ score: null,
1159
+ note: "undetermined: declared schema covers none of the dataset columns"
1160
+ };
1161
+ return {
1162
+ name: "accuracy",
1163
+ score: round6$1(matched / checked),
1164
+ note: `${matched}/${checked} columns match their declared type` + (mismatches.length > 0 ? `; mismatches: ${mismatches.join(", ")}` : "")
1165
+ };
1166
+ }
1167
+ //#endregion
845
1168
  //#region src/profile.ts
846
1169
  /**
847
1170
  * Deterministic dataset profiling: per-column type inference, missingness,
@@ -850,6 +1173,8 @@ function serializeDelimited(columns, rows, delimiter) {
850
1173
  * no I/O; `generatedAt` is injected by the caller.
851
1174
  * @module dsh-data-quality/profile
852
1175
  */
1176
+ /** Fallback duplicate-sample cap for direct engine use; the provider always passes the configured `evidenceRowLimit`. */
1177
+ const DEFAULT_DUPLICATE_SAMPLE_LIMIT = 20;
853
1178
  /** Round to 6 significant digits for stable, readable report numbers. */
854
1179
  function round6(value) {
855
1180
  return Number(value.toPrecision(6));
@@ -874,6 +1199,8 @@ function numericProfile(values) {
874
1199
  const highFence = p75 + 1.5 * iqr;
875
1200
  const outliers = iqr === 0 ? 0 : sorted.filter((value) => value < lowFence || value > highFence).length;
876
1201
  return {
1202
+ count: sorted.length,
1203
+ distinct: new Set(sorted).size,
877
1204
  min: round6(sorted[0]),
878
1205
  max: round6(sorted[sorted.length - 1]),
879
1206
  mean: round6(sum / sorted.length),
@@ -883,17 +1210,43 @@ function numericProfile(values) {
883
1210
  outliers
884
1211
  };
885
1212
  }
886
- /** Count rows whose full content duplicates an earlier row (first occurrence is not counted). */
887
- function countDuplicateRows(table, signal) {
1213
+ /** Deterministic sha256 key of one row's full content (columns in table order). */
1214
+ function rowContentKey(table, row) {
1215
+ return createHash("sha256").update(JSON.stringify(table.columns.map((column) => row[column] ?? null))).digest("hex");
1216
+ }
1217
+ /**
1218
+ * Detect full-content duplicate rows with a bounded sample of their 0-based
1219
+ * indexes. The first occurrence of each content is never counted; later rows
1220
+ * with identical full content are duplicates.
1221
+ * @param table - the parsed dataset.
1222
+ * @param options - sample cap and optional abort signal.
1223
+ * @returns the duplicate count, rate, and capped sample indexes.
1224
+ */
1225
+ function detectDuplicateRows(table, options) {
1226
+ if (!Number.isSafeInteger(options.sampleLimit) || options.sampleLimit <= 0) throw new TypeError(`sampleLimit must be a positive safe integer, got ${String(options.sampleLimit)}`);
888
1227
  const seen = /* @__PURE__ */ new Set();
889
1228
  let duplicates = 0;
1229
+ const duplicateSampleRowIndexes = [];
890
1230
  for (const [index, row] of table.rows.entries()) {
891
- if (index % 1024 === 0) throwIfAborted(signal);
892
- const key = JSON.stringify(table.columns.map((column) => row[column] ?? null));
893
- if (seen.has(key)) duplicates += 1;
894
- else seen.add(key);
1231
+ if (index % 1024 === 0) throwIfAborted(options.signal);
1232
+ const key = rowContentKey(table, row);
1233
+ if (seen.has(key)) {
1234
+ duplicates += 1;
1235
+ if (duplicateSampleRowIndexes.length < options.sampleLimit) duplicateSampleRowIndexes.push(index);
1236
+ } else seen.add(key);
895
1237
  }
896
- return duplicates;
1238
+ return {
1239
+ duplicateRows: duplicates,
1240
+ duplicateRate: table.rows.length === 0 ? 0 : round6(duplicates / table.rows.length),
1241
+ duplicateSampleRowIndexes
1242
+ };
1243
+ }
1244
+ /** Count rows whose full content duplicates an earlier row (first occurrence is not counted). */
1245
+ function countDuplicateRows(table, signal) {
1246
+ return detectDuplicateRows(table, {
1247
+ sampleLimit: 1,
1248
+ signal
1249
+ }).duplicateRows;
897
1250
  }
898
1251
  /** Profile one column over the given rows. */
899
1252
  function profileColumn(rows, column, signal) {
@@ -980,14 +1333,28 @@ function profileTable(table, options) {
980
1333
  throwIfAborted(options.signal);
981
1334
  const profiled = options.sample === void 0 ? table.rows : sampleRows(table.rows, options.sample);
982
1335
  const columns = table.columns.map((column) => profileColumn(profiled, column, options.signal));
983
- const duplicateRows = countDuplicateRows(table, options.signal);
1336
+ const detection = detectDuplicateRows(table, {
1337
+ sampleLimit: options.duplicateSampleLimit ?? DEFAULT_DUPLICATE_SAMPLE_LIMIT,
1338
+ signal: options.signal
1339
+ });
1340
+ const scorecard = computeScorecard(table, {
1341
+ now: options.generatedAt,
1342
+ duplicateRows: detection.duplicateRows,
1343
+ declaredSchema: options.declaredSchema,
1344
+ weights: options.scorecardWeights,
1345
+ signal: options.signal
1346
+ });
984
1347
  return {
985
1348
  dataset: options.dataset,
986
1349
  rowCount: table.rows.length,
987
1350
  sampled: profiled.length !== table.rows.length,
988
1351
  profiledRows: profiled.length,
989
1352
  columnCount: table.columns.length,
990
- duplicateRows,
1353
+ duplicateRows: detection.duplicateRows,
1354
+ duplicateRate: detection.duplicateRate,
1355
+ duplicateSampleRowIndexes: detection.duplicateSampleRowIndexes,
1356
+ scorecard,
1357
+ ...table.encoding !== void 0 ? { encoding: table.encoding } : {},
991
1358
  columns,
992
1359
  generatedAt: options.generatedAt
993
1360
  };
@@ -996,19 +1363,167 @@ function profileTable(table, options) {
996
1363
  function renderProfileText(report) {
997
1364
  const lines = [];
998
1365
  lines.push(`Profile of ${report.dataset}: ${report.rowCount} rows x ${report.columnCount} columns` + (report.sampled ? ` (column cards over a systematic sample of ${report.profiledRows} rows)` : ""));
999
- if (report.duplicateRows > 0) lines.push(`Duplicate rows: ${report.duplicateRows}`);
1366
+ if (report.duplicateRows > 0) lines.push(`Duplicate rows: ${report.duplicateRows} (${(report.duplicateRate * 100).toFixed(1)}%)` + (report.duplicateSampleRowIndexes.length > 0 ? `; sample row indexes: ${report.duplicateSampleRowIndexes.join(", ")}` : ""));
1000
1367
  for (const column of report.columns) {
1001
1368
  const parts = [`${column.name}: ${column.inferredType}`];
1002
1369
  if (column.missing > 0) parts.push(`missing ${column.missing} (${(column.missingRate * 100).toFixed(1)}%)`);
1003
1370
  parts.push(`unique ${column.unique}`);
1004
- if (column.numeric !== void 0) parts.push(`min ${column.numeric.min}, p25 ${column.numeric.p25}, median ${column.numeric.median}, p75 ${column.numeric.p75}, max ${column.numeric.max}, mean ${column.numeric.mean}` + (column.numeric.outliers > 0 ? `, ${column.numeric.outliers} IQR outliers` : ""));
1371
+ if (column.numeric !== void 0) parts.push(`count ${column.numeric.count}, distinct ${column.numeric.distinct}, min ${column.numeric.min}, p25 ${column.numeric.p25}, median ${column.numeric.median}, p75 ${column.numeric.p75}, max ${column.numeric.max}, mean ${column.numeric.mean}` + (column.numeric.outliers > 0 ? `, ${column.numeric.outliers} IQR outliers` : ""));
1005
1372
  if (column.topValues !== void 0) parts.push(`top: ${column.topValues.map((entry) => `${JSON.stringify(entry.value)} x${entry.count}`).join(", ")}`);
1006
1373
  for (const note of column.notes) parts.push(`note: ${note}`);
1007
1374
  lines.push(`- ${parts.join("; ")}`);
1008
1375
  }
1376
+ if (report.encoding !== void 0) lines.push(`Encoding: UTF-8${report.encoding.bom === "utf-8" ? " (BOM)" : ""}${report.encoding.validUtf8 ? "" : " (INVALID UTF-8)"}`);
1377
+ const overall = report.scorecard.overall;
1378
+ const weighted = report.scorecard.weightedOverall;
1379
+ lines.push(`Scorecard (overall ${overall === null ? "undetermined" : `${(overall * 100).toFixed(1)}%`}, weighted ${weighted === null ? "undetermined" : `${(weighted * 100).toFixed(1)}%`}):`);
1380
+ for (const dimension of report.scorecard.dimensions) {
1381
+ const value = dimension.score === null ? "undetermined" : `${(dimension.score * 100).toFixed(1)}%`;
1382
+ lines.push(` ${dimension.name}: ${value} (${dimension.note})`);
1383
+ }
1009
1384
  return lines.join("\n");
1010
1385
  }
1011
1386
  //#endregion
1387
+ //#region src/contract.ts
1388
+ /**
1389
+ * Delivery contract for `data_clean`: a deterministic pre-delivery validation
1390
+ * summary (primary-key/type/dedupe row-count comparison plus uniqueness and
1391
+ * non-null regression) and the clean before/after profile diff report. Pure —
1392
+ * no I/O, no clock, no RNG; timestamps are injected.
1393
+ * @module dsh-data-quality/contract
1394
+ */
1395
+ /** Count missing cells in one column of the cleaned output. */
1396
+ function countMissing(rows, column, signal) {
1397
+ let count = 0;
1398
+ for (const [index, row] of rows.entries()) {
1399
+ if (index % 1024 === 0) throwIfAborted(signal);
1400
+ if (isMissing(row[column])) count += 1;
1401
+ }
1402
+ return count;
1403
+ }
1404
+ /** Count present cells in one column that still fail to parse as the requested type. */
1405
+ function countNonConforming(rows, column, to, signal) {
1406
+ let count = 0;
1407
+ for (const [index, row] of rows.entries()) {
1408
+ if (index % 1024 === 0) throwIfAborted(signal);
1409
+ const cell = row[column];
1410
+ if (isMissing(cell)) continue;
1411
+ if (!(to === "number" ? parseNumeric(cell) !== void 0 : to === "date" ? parseDate(cell) !== void 0 : parseBoolean(cell) !== void 0)) count += 1;
1412
+ }
1413
+ return count;
1414
+ }
1415
+ /**
1416
+ * Compute the pre-delivery contract summary from a cleaning run: dedupe
1417
+ * before/after row counts, the dedupe key's uniqueness regression, and the
1418
+ * non-null/type regressions over the columns the rules targeted.
1419
+ * @param result - the cleaning outcome (input/output rows plus cleaned rows).
1420
+ * @param rules - the ordered rules that produced `result`.
1421
+ * @param options - optional abort signal.
1422
+ * @returns the contract summary.
1423
+ */
1424
+ function computeCleanContract(result, rules, options = {}) {
1425
+ const dedupeRule = rules.find((rule) => rule.rule === "dedupe");
1426
+ const dedupeColumns = dedupeRule === void 0 ? null : [...dedupeRule.columns ?? result.columns];
1427
+ const keyColumns = dedupeColumns ?? result.columns;
1428
+ const seen = /* @__PURE__ */ new Set();
1429
+ let remainingDuplicateRows = 0;
1430
+ for (const [index, row] of result.rows.entries()) {
1431
+ if (index % 1024 === 0) throwIfAborted(options.signal);
1432
+ const key = JSON.stringify(keyColumns.map((column) => row[column] ?? null));
1433
+ if (seen.has(key)) remainingDuplicateRows += 1;
1434
+ else seen.add(key);
1435
+ }
1436
+ const remainingMissing = rules.filter((rule) => rule.rule === "fill-missing").map((rule) => {
1437
+ const fill = rule;
1438
+ return {
1439
+ column: fill.column,
1440
+ count: countMissing(result.rows, fill.column, options.signal)
1441
+ };
1442
+ });
1443
+ const typeConformance = rules.filter((rule) => rule.rule === "coerce-type").map((rule) => {
1444
+ const coerce = rule;
1445
+ return {
1446
+ column: coerce.column,
1447
+ to: coerce.to,
1448
+ invalidCount: countNonConforming(result.rows, coerce.column, coerce.to, options.signal)
1449
+ };
1450
+ });
1451
+ const columnDecisions = traceColumnDecisions(result, rules);
1452
+ return {
1453
+ inputRows: result.inputRows,
1454
+ outputRows: result.outputRows,
1455
+ removedRows: result.inputRows - result.outputRows,
1456
+ dedupeColumns,
1457
+ uniqueKeys: remainingDuplicateRows === 0,
1458
+ remainingDuplicateRows,
1459
+ remainingMissing,
1460
+ typeConformance,
1461
+ columnDecisions
1462
+ };
1463
+ }
1464
+ /** The columns a single rule targets (single-column rules target one column; trim/dedupe may target many). */
1465
+ function ruleColumns(rule, allColumns) {
1466
+ switch (rule.rule) {
1467
+ case "fill-missing":
1468
+ case "coerce-type":
1469
+ case "normalize-unit":
1470
+ case "map-values": return [rule.column];
1471
+ case "trim":
1472
+ case "dedupe": return [...rule.columns ?? allColumns];
1473
+ default: return [];
1474
+ }
1475
+ }
1476
+ /** Build the per-column decision trace: strategies + affected rows, in dataset column order. */
1477
+ function traceColumnDecisions(result, rules) {
1478
+ const byColumn = /* @__PURE__ */ new Map();
1479
+ for (const [index, rule] of rules.entries()) {
1480
+ const log = result.logs[index];
1481
+ if (log === void 0) continue;
1482
+ for (const column of ruleColumns(rule, result.columns)) {
1483
+ const list = byColumn.get(column) ?? [];
1484
+ list.push({
1485
+ strategy: rule.rule,
1486
+ affectedRows: log.affectedRows
1487
+ });
1488
+ byColumn.set(column, list);
1489
+ }
1490
+ }
1491
+ return result.columns.filter((column) => byColumn.has(column)).map((column) => ({
1492
+ column,
1493
+ decisions: byColumn.get(column)
1494
+ }));
1495
+ }
1496
+ /**
1497
+ * Build the clean before/after profile diff: two full {@link ProfileReport}
1498
+ * snapshots (input and cleaned output) under one envelope. Reuses the profile
1499
+ * engine so the scorecard and duplicate detection stay consistent across the
1500
+ * whole plugin.
1501
+ * @param input - the input dataset.
1502
+ * @param output - the cleaned dataset.
1503
+ * @param options - dataset label, injected timestamp, optional weights, optional abort signal.
1504
+ * @returns the diff report.
1505
+ */
1506
+ function computeCleanProfileDiff(input, output, options) {
1507
+ const before = profileTable(input, {
1508
+ dataset: options.dataset,
1509
+ generatedAt: options.generatedAt,
1510
+ scorecardWeights: options.scorecardWeights,
1511
+ signal: options.signal
1512
+ });
1513
+ const after = profileTable(output, {
1514
+ dataset: options.dataset,
1515
+ generatedAt: options.generatedAt,
1516
+ scorecardWeights: options.scorecardWeights,
1517
+ signal: options.signal
1518
+ });
1519
+ return {
1520
+ dataset: options.dataset,
1521
+ before,
1522
+ after,
1523
+ generatedAt: options.generatedAt
1524
+ };
1525
+ }
1526
+ //#endregion
1012
1527
  //#region src/verify.ts
1013
1528
  /**
1014
1529
  * Declarative verification rules over a parsed {@link Table} (not-null,
@@ -1031,6 +1546,19 @@ var VerifyRuleError = class extends Error {
1031
1546
  this.name = "VerifyRuleError";
1032
1547
  }
1033
1548
  };
1549
+ /** Raised for invalid expectations; `message` names the expectation index and reason. */
1550
+ var VerifyExpectationError = class extends Error {
1551
+ expectationIndex;
1552
+ /**
1553
+ * @param expectationIndex - index of the offending expectation in the request array.
1554
+ * @param message - actionable human-readable detail.
1555
+ */
1556
+ constructor(expectationIndex, message) {
1557
+ super(message);
1558
+ this.expectationIndex = expectationIndex;
1559
+ this.name = "VerifyExpectationError";
1560
+ }
1561
+ };
1034
1562
  /** Assert `column` exists. */
1035
1563
  function requireColumn(columns, column, ruleIndex) {
1036
1564
  if (!columns.includes(column)) throw new VerifyRuleError(ruleIndex, `rule ${ruleIndex}: unknown column ${JSON.stringify(column)} (columns: ${columns.join(", ")})`);
@@ -1076,13 +1604,90 @@ const CROSS_OPS = [
1076
1604
  ">=",
1077
1605
  ">"
1078
1606
  ];
1607
+ /** The metric ids, for validation and diagnostics. */
1608
+ const VERIFY_METRICS = [
1609
+ "rowCount",
1610
+ "columnSum",
1611
+ "columnMean",
1612
+ "uniqueCount",
1613
+ "nullCount"
1614
+ ];
1615
+ /** Throw unless the expectation is well-formed (metric/column/tolerance). */
1616
+ function validateExpectation(table, expectation, index) {
1617
+ if (!VERIFY_METRICS.includes(expectation.metric)) throw new VerifyExpectationError(index, `expectation ${index}: unknown metric ${JSON.stringify(expectation.metric)} (expected one of ${VERIFY_METRICS.join(", ")})`);
1618
+ if (expectation.metric !== "rowCount") {
1619
+ if (expectation.column === void 0 || expectation.column === "") throw new VerifyExpectationError(index, `expectation ${index}: metric ${expectation.metric} requires a column`);
1620
+ if (!table.columns.includes(expectation.column)) throw new VerifyExpectationError(index, `expectation ${index}: unknown column ${JSON.stringify(expectation.column)} (columns: ${table.columns.join(", ")})`);
1621
+ } else if (expectation.column !== void 0) throw new VerifyExpectationError(index, `expectation ${index}: metric rowCount takes no column, got ${JSON.stringify(expectation.column)}`);
1622
+ if (expectation.tolerance !== void 0 && (typeof expectation.tolerance !== "number" || !Number.isFinite(expectation.tolerance) || expectation.tolerance < 0 || expectation.tolerance > 1)) throw new VerifyExpectationError(index, `expectation ${index}: tolerance must be a finite number in [0, 1], got ${String(expectation.tolerance)}`);
1623
+ }
1624
+ /** Compute the deterministic actual value of one expectation's metric. */
1625
+ function metricValueOf(table, expectation) {
1626
+ switch (expectation.metric) {
1627
+ case "rowCount": return table.rows.length;
1628
+ case "nullCount": {
1629
+ let count = 0;
1630
+ for (const row of table.rows) if (isMissing(row[expectation.column])) count += 1;
1631
+ return count;
1632
+ }
1633
+ case "uniqueCount": {
1634
+ const distinct = /* @__PURE__ */ new Set();
1635
+ for (const row of table.rows) {
1636
+ const cell = row[expectation.column];
1637
+ if (isMissing(cell)) continue;
1638
+ distinct.add(typeof cell === "string" ? cell : JSON.stringify(cell));
1639
+ }
1640
+ return distinct.size;
1641
+ }
1642
+ case "columnSum":
1643
+ case "columnMean": {
1644
+ let sum = 0;
1645
+ let count = 0;
1646
+ for (const row of table.rows) {
1647
+ const value = parseNumeric(row[expectation.column]);
1648
+ if (value === void 0) continue;
1649
+ sum += value;
1650
+ count += 1;
1651
+ }
1652
+ return expectation.metric === "columnSum" ? sum : count === 0 ? 0 : sum / count;
1653
+ }
1654
+ }
1655
+ }
1656
+ /**
1657
+ * Reconcile each expectation against its deterministic actual value. A
1658
+ * mismatch is a normal `passed: false` result, never a thrown error; invalid
1659
+ * metrics, columns, and tolerances fail loud.
1660
+ * @param table - the parsed dataset.
1661
+ * @param expectations - the expectations to reconcile.
1662
+ * @param defaultTolerance - configured fallback relative tolerance.
1663
+ * @param signal - optional abort signal.
1664
+ * @returns one outcome per expectation.
1665
+ */
1666
+ function verifyExpectations(table, expectations, defaultTolerance, signal) {
1667
+ return expectations.map((expectation, index) => {
1668
+ throwIfAborted(signal);
1669
+ validateExpectation(table, expectation, index);
1670
+ const actual = metricValueOf(table, expectation);
1671
+ const tolerance = expectation.tolerance ?? defaultTolerance;
1672
+ const passed = numericClose(actual, expectation.expected, tolerance);
1673
+ return {
1674
+ metric: expectation.metric,
1675
+ ...expectation.column !== void 0 ? { column: expectation.column } : {},
1676
+ expected: expectation.expected,
1677
+ actual,
1678
+ tolerance,
1679
+ passed
1680
+ };
1681
+ });
1682
+ }
1079
1683
  /**
1080
1684
  * Apply verification rules over a parsed table. A missing cell fails every
1081
- * rule that reads it. The overall `passed` is the conjunction of rule passes;
1082
- * a failing dataset is a normal result, never a thrown error.
1685
+ * rule that reads it. The overall `passed` is the conjunction of rule passes
1686
+ * and expectation passes; a failing dataset is a normal result, never a
1687
+ * thrown error.
1083
1688
  * @param table - the parsed dataset.
1084
1689
  * @param rules - non-empty rule list.
1085
- * @param options - evidence cap, injected clock for `freshness`, abort signal.
1690
+ * @param options - evidence cap, injected clock for `freshness`, optional expectations, default tolerance, abort signal.
1086
1691
  * @returns the verify report (without the dataset label; the caller adds it).
1087
1692
  */
1088
1693
  function verifyTable(table, rules, options) {
@@ -1205,10 +1810,12 @@ function verifyTable(table, rules, options) {
1205
1810
  }
1206
1811
  results.push(result);
1207
1812
  }
1813
+ const expectations = verifyExpectations(table, options.expectations ?? [], options.defaultTolerance ?? 1e-9, options.signal);
1208
1814
  return {
1209
- passed: results.every((result) => result.passed),
1815
+ passed: results.every((result) => result.passed) && expectations.every((expectation) => expectation.passed),
1210
1816
  rowCount: table.rows.length,
1211
1817
  rules: results,
1818
+ expectations,
1212
1819
  generatedAt: options.now()
1213
1820
  };
1214
1821
  }
@@ -1224,6 +1831,10 @@ function renderVerifyText(report) {
1224
1831
  }
1225
1832
  if (rule.failedCount > rule.evidence.length) lines.push(` … and ${rule.failedCount - rule.evidence.length} more failing row(s)`);
1226
1833
  }
1834
+ for (const expectation of report.expectations) {
1835
+ const target = expectation.column !== void 0 ? `${expectation.metric}(${expectation.column})` : expectation.metric;
1836
+ lines.push(`- [${expectation.passed ? "pass" : "FAIL"}] expectation ${target}: actual ${expectation.actual} vs expected ${expectation.expected} (tolerance ${expectation.tolerance})`);
1837
+ }
1227
1838
  return lines.join("\n");
1228
1839
  }
1229
1840
  /**
@@ -1409,6 +2020,131 @@ function appendDataQualityEvent(session, type, data) {
1409
2020
  if (Function.prototype.toString.call(append).includes("ignorable")) append.call(session, type, data, { ignorable: true });
1410
2021
  }
1411
2022
  //#endregion
2023
+ //#region src/presets.ts
2024
+ /** The built-in industry preset registry. */
2025
+ const INDUSTRY_PRESETS = {
2026
+ retail: {
2027
+ id: "retail",
2028
+ label: "Retail",
2029
+ columns: {
2030
+ order_id: "string",
2031
+ sku: "string",
2032
+ quantity: "number",
2033
+ unit_price: "number",
2034
+ revenue: "number",
2035
+ order_date: "date",
2036
+ customer_id: "string"
2037
+ }
2038
+ },
2039
+ saas: {
2040
+ id: "saas",
2041
+ label: "SaaS",
2042
+ columns: {
2043
+ account_id: "string",
2044
+ plan: "string",
2045
+ seats: "number",
2046
+ mrr: "number",
2047
+ signup_date: "date",
2048
+ churned: "boolean"
2049
+ }
2050
+ },
2051
+ fund: {
2052
+ id: "fund",
2053
+ label: "Fund",
2054
+ columns: {
2055
+ fund_code: "string",
2056
+ fund_name: "string",
2057
+ nav: "number",
2058
+ nav_date: "date",
2059
+ holding_value: "number",
2060
+ currency: "string"
2061
+ }
2062
+ },
2063
+ "real-estate": {
2064
+ id: "real-estate",
2065
+ label: "Real estate",
2066
+ columns: {
2067
+ property_id: "string",
2068
+ listing_price: "number",
2069
+ area_sqm: "number",
2070
+ bedrooms: "number",
2071
+ list_date: "date",
2072
+ city: "string"
2073
+ }
2074
+ },
2075
+ "e-commerce": {
2076
+ id: "e-commerce",
2077
+ label: "E-commerce",
2078
+ columns: {
2079
+ order_id: "string",
2080
+ product_id: "string",
2081
+ quantity: "number",
2082
+ price: "number",
2083
+ order_date: "date",
2084
+ status: "string"
2085
+ }
2086
+ },
2087
+ healthcare: {
2088
+ id: "healthcare",
2089
+ label: "Healthcare",
2090
+ columns: {
2091
+ patient_id: "string",
2092
+ admission_date: "date",
2093
+ discharge_date: "date",
2094
+ age: "number",
2095
+ diagnosis: "string",
2096
+ cost: "number"
2097
+ }
2098
+ },
2099
+ logistics: {
2100
+ id: "logistics",
2101
+ label: "Logistics",
2102
+ columns: {
2103
+ shipment_id: "string",
2104
+ origin: "string",
2105
+ destination: "string",
2106
+ weight_kg: "number",
2107
+ shipped_date: "date",
2108
+ delivered: "boolean"
2109
+ }
2110
+ },
2111
+ manufacturing: {
2112
+ id: "manufacturing",
2113
+ label: "Manufacturing",
2114
+ columns: {
2115
+ part_id: "string",
2116
+ quantity: "number",
2117
+ unit_cost: "number",
2118
+ produced_date: "date",
2119
+ defect: "boolean"
2120
+ }
2121
+ },
2122
+ energy: {
2123
+ id: "energy",
2124
+ label: "Energy",
2125
+ columns: {
2126
+ meter_id: "string",
2127
+ reading: "number",
2128
+ unit: "string",
2129
+ reading_date: "date",
2130
+ consumption: "number"
2131
+ }
2132
+ }
2133
+ };
2134
+ /** The preset ids, for diagnostics and documentation. */
2135
+ const INDUSTRY_PRESET_IDS = Object.keys(INDUSTRY_PRESETS);
2136
+ /**
2137
+ * Resolve an industry preset id to its registry entry, failing loud on an
2138
+ * unknown id.
2139
+ * @param id - the requested preset id.
2140
+ * @returns the preset.
2141
+ */
2142
+ function resolveIndustryPreset(id) {
2143
+ const preset = INDUSTRY_PRESETS[id];
2144
+ if (preset === void 0) throw new Error(`unknown industryPreset ${JSON.stringify(id)}; choose one of: ${INDUSTRY_PRESET_IDS.join(", ")}`);
2145
+ return preset;
2146
+ }
2147
+ //#endregion
1412
2148
  //#region src/present.ts
1413
2149
  /** Maximum characters one cell contributes to a tool-facing row payload. */
1414
2150
  const MAX_CELL_TEXT = 120;
@@ -1514,11 +2250,16 @@ var LocalDataQualityService = class extends DataQualityService {
1514
2250
  /** @inheritdoc DataQualityService.profileDataset */
1515
2251
  async profileDataset(request) {
1516
2252
  throwIfAborted(request.signal);
1517
- const report = profileTable(await loadTable(resolveWorkspacePath(request.workspace, request.dataset, this.config), this.config, request.signal), {
2253
+ const table = await loadTable(resolveWorkspacePath(request.workspace, request.dataset, this.config), this.config, request.signal);
2254
+ const declaredSchema = request.industryPreset === void 0 ? void 0 : resolveIndustryPreset(request.industryPreset).columns;
2255
+ const report = profileTable(table, {
1518
2256
  dataset: request.dataset,
1519
2257
  sample: request.sample,
1520
2258
  generatedAt: this.deps.now(),
1521
- signal: request.signal
2259
+ signal: request.signal,
2260
+ duplicateSampleLimit: this.config.evidenceRowLimit,
2261
+ declaredSchema,
2262
+ scorecardWeights: this.config.scorecardWeights
1522
2263
  });
1523
2264
  const reportKey = await this.persist("profile", request.dataset, report);
1524
2265
  this.emitEvent(request.session, "profile", request.dataset, reportKey, {
@@ -1533,10 +2274,12 @@ var LocalDataQualityService = class extends DataQualityService {
1533
2274
  /** @inheritdoc DataQualityService.cleanDataset */
1534
2275
  async cleanDataset(request) {
1535
2276
  throwIfAborted(request.signal);
2277
+ const dryRun = request.dryRun === true;
1536
2278
  const absolute = resolveWorkspacePath(request.workspace, request.dataset, this.config);
1537
- const result = applyCleanRules(await loadTable(absolute, this.config, request.signal), request.rules, { signal: request.signal });
2279
+ const table = await loadTable(absolute, this.config, request.signal);
2280
+ const result = applyCleanRules(table, request.rules, { signal: request.signal });
1538
2281
  let writtenPath;
1539
- if (request.outputPath !== void 0) {
2282
+ if (!dryRun && request.outputPath !== void 0) {
1540
2283
  const outputAbsolute = resolveWorkspacePath(request.workspace, request.outputPath, this.config);
1541
2284
  if (outputAbsolute === absolute) throw new Error(`outputPath ${JSON.stringify(request.outputPath)} would overwrite the input dataset; choose a different path`);
1542
2285
  const ext = path.extname(outputAbsolute).toLowerCase();
@@ -1547,19 +2290,51 @@ var LocalDataQualityService = class extends DataQualityService {
1547
2290
  writtenPath = request.outputPath;
1548
2291
  }
1549
2292
  const generatedAt = this.deps.now();
2293
+ const contract = computeCleanContract(result, request.rules, { signal: request.signal });
2294
+ const diff = computeCleanProfileDiff(table, {
2295
+ columns: result.columns,
2296
+ rows: result.rows,
2297
+ ...table.encoding !== void 0 ? { encoding: table.encoding } : {}
2298
+ }, {
2299
+ dataset: request.dataset,
2300
+ generatedAt,
2301
+ scorecardWeights: this.config.scorecardWeights,
2302
+ signal: request.signal
2303
+ });
1550
2304
  const preview = {
1551
2305
  columns: result.columns,
1552
2306
  rows: result.rows.slice(0, this.config.evidenceRowLimit).map((row) => truncateRow(row))
1553
2307
  };
2308
+ if (dryRun) {
2309
+ this.emitEvent(request.session, "clean", request.dataset, void 0, {
2310
+ rows: result.outputRows,
2311
+ columns: result.columns.length,
2312
+ rules: result.logs.length
2313
+ });
2314
+ return {
2315
+ dataset: request.dataset,
2316
+ inputRows: result.inputRows,
2317
+ outputRows: result.outputRows,
2318
+ dryRun: true,
2319
+ logs: result.logs,
2320
+ contract,
2321
+ preview,
2322
+ diffPreview: diff,
2323
+ generatedAt
2324
+ };
2325
+ }
1554
2326
  const reportKey = await this.persist("clean", request.dataset, {
1555
2327
  dataset: request.dataset,
1556
2328
  inputRows: result.inputRows,
1557
2329
  outputRows: result.outputRows,
2330
+ dryRun: false,
1558
2331
  logs: result.logs,
2332
+ contract,
1559
2333
  preview,
1560
2334
  ...writtenPath !== void 0 ? { outputPath: writtenPath } : {},
1561
2335
  generatedAt
1562
2336
  });
2337
+ await this.persist("clean-diff", request.dataset, diff);
1563
2338
  this.emitEvent(request.session, "clean", request.dataset, reportKey, {
1564
2339
  rows: result.outputRows,
1565
2340
  columns: result.columns.length,
@@ -1569,32 +2344,57 @@ var LocalDataQualityService = class extends DataQualityService {
1569
2344
  dataset: request.dataset,
1570
2345
  inputRows: result.inputRows,
1571
2346
  outputRows: result.outputRows,
2347
+ dryRun: false,
1572
2348
  logs: result.logs,
2349
+ contract,
1573
2350
  preview,
1574
2351
  ...writtenPath !== void 0 ? { outputPath: writtenPath } : {},
1575
2352
  ...reportKey !== void 0 ? { reportKey } : {},
1576
2353
  generatedAt
1577
2354
  };
1578
2355
  }
2356
+ /** @inheritdoc DataQualityService.getReport */
2357
+ async getReport(key) {
2358
+ if (!isValidReportKey(key)) throw new Error(`invalid reportKey ${JSON.stringify(key)}: expected the deterministic <timestamp>-<kind>-<fingerprint> format`);
2359
+ const store = this.deps.store;
2360
+ if (store === void 0) throw new Error("report storage is disabled (storeReports is false); no persisted reports to read");
2361
+ const record = store.get(key);
2362
+ if (record === void 0) throw new Error(`no persisted report found for reportKey ${JSON.stringify(key)}`);
2363
+ return {
2364
+ key,
2365
+ ...record
2366
+ };
2367
+ }
2368
+ /** @inheritdoc DataQualityService.listReports */
2369
+ async listReports(kind) {
2370
+ const store = this.deps.store;
2371
+ if (store === void 0) throw new Error("report storage is disabled (storeReports is false); no persisted reports to read");
2372
+ return store.list(kind);
2373
+ }
1579
2374
  /** @inheritdoc DataQualityService.verifyDataset */
1580
2375
  async verifyDataset(request) {
1581
2376
  throwIfAborted(request.signal);
1582
2377
  const outcome = verifyTable(await loadTable(resolveWorkspacePath(request.workspace, request.dataset, this.config), this.config, request.signal), request.rules, {
1583
2378
  evidenceRowLimit: this.config.evidenceRowLimit,
1584
2379
  now: this.deps.now,
1585
- signal: request.signal
2380
+ signal: request.signal,
2381
+ expectations: request.expectations,
2382
+ defaultTolerance: this.config.defaultTolerance
1586
2383
  });
1587
2384
  const report = {
1588
2385
  dataset: request.dataset,
1589
2386
  ...outcome
1590
2387
  };
1591
2388
  const failedRules = report.rules.filter((rule) => !rule.passed).length;
2389
+ const failedExpectations = report.expectations.filter((expectation) => !expectation.passed).length;
1592
2390
  const reportKey = await this.persist("verify", request.dataset, report);
1593
2391
  this.emitEvent(request.session, "verify", request.dataset, reportKey, {
1594
2392
  rows: report.rowCount,
1595
2393
  rules: report.rules.length,
1596
2394
  failedRules,
1597
- passed: report.passed
2395
+ passed: report.passed,
2396
+ expectations: report.expectations.length,
2397
+ failedExpectations
1598
2398
  });
1599
2399
  return {
1600
2400
  ...report,
@@ -1622,48 +2422,22 @@ var LocalDataQualityService = class extends DataQualityService {
1622
2422
  }
1623
2423
  };
1624
2424
  //#endregion
1625
- //#region src/tools/shared.ts
1626
- /**
1627
- * Shared helpers for the three data-quality model tools: session workspace
1628
- * resolution and a compact cleaning-run text render. Kept presentation-only;
1629
- * computation lives in the engines and the provider.
1630
- * @module dsh-data-quality/tools-shared
1631
- */
2425
+ //#region src/tools/profile-report-schema.ts
1632
2426
  /**
1633
- * The absolute workspace root a tool call resolves dataset paths against: the
1634
- * calling agent's per-session cwd (mirroring the official fs tools). Non-agent
1635
- * calls fail loud there is no honest workspace to confine paths to.
1636
- * @param exec - the tool-execution context.
1637
- * @returns the absolute workspace root.
1638
- */
1639
- function workspaceOf(exec) {
1640
- const cwd = exec.agent?.session.header.cwd;
1641
- if (cwd === void 0) throw new Error("data-quality tools require an agent-owned session workspace");
1642
- return path.resolve(cwd);
1643
- }
1644
- /** Human-readable cleaning summary for the tool's Native render. */
1645
- function renderCleanText(report) {
1646
- const lines = [];
1647
- lines.push(`Cleaned ${report.dataset}: ${report.inputRows} -> ${report.outputRows} rows over ${report.logs.length} rule(s)`);
1648
- for (const log of report.logs) lines.push(`- rule ${log.ruleIndex} (${log.rule}): ${log.affectedRows} row(s) affected; ${log.detail}`);
1649
- if (report.outputPath !== void 0) lines.push(`Wrote cleaned dataset to ${report.outputPath}`);
1650
- else lines.push("No outputPath given: the source file was left untouched; preview below.");
1651
- if (report.preview.rows.length > 0) {
1652
- lines.push(`Preview (first ${report.preview.rows.length} row(s)):`);
1653
- for (const row of report.preview.rows) {
1654
- const cells = report.preview.columns.map((column) => `${column}=${JSON.stringify(row[column] ?? null)}`).join(", ");
1655
- lines.push(` ${cells}`);
1656
- }
1657
- }
1658
- return lines.join("\n");
1659
- }
1660
- //#endregion
1661
- //#region src/tools/profile.ts
1662
- /**
1663
- * The `data_profile` model tool: deterministic dataset profiling through
1664
- * `ctx.dataQuality.profileDataset` — never model arithmetic.
1665
- * @module dsh-data-quality/tools/profile
2427
+ * Shared JSON-Schema spec for the profile report's canonical value, reused by
2428
+ * `data_profile` (its output) and `data_clean` (the `diffPreview` before/after
2429
+ * snapshots). One source of truth so the two tools never drift.
2430
+ * @module dsh-data-quality/tools-profile-report-schema
1666
2431
  */
2432
+ const SCORECARD_DIMENSION_NAMES = [
2433
+ "completeness",
2434
+ "uniqueness",
2435
+ "validity",
2436
+ "consistency",
2437
+ "timeliness",
2438
+ "accuracy"
2439
+ ];
2440
+ /** One column card's schema. */
1667
2441
  const COLUMN_PROFILE_SCHEMA = {
1668
2442
  type: "object",
1669
2443
  properties: {
@@ -1698,6 +2472,14 @@ const COLUMN_PROFILE_SCHEMA = {
1698
2472
  numeric: {
1699
2473
  type: "object",
1700
2474
  properties: {
2475
+ count: {
2476
+ type: "number",
2477
+ required: true
2478
+ },
2479
+ distinct: {
2480
+ type: "number",
2481
+ required: true
2482
+ },
1701
2483
  min: {
1702
2484
  type: "number",
1703
2485
  required: true
@@ -1754,6 +2536,163 @@ const COLUMN_PROFILE_SCHEMA = {
1754
2536
  },
1755
2537
  additionalProperties: false
1756
2538
  };
2539
+ /** The full profile report's schema. */
2540
+ const PROFILE_REPORT_SCHEMA = {
2541
+ type: "object",
2542
+ properties: {
2543
+ dataset: {
2544
+ type: "string",
2545
+ required: true
2546
+ },
2547
+ rowCount: {
2548
+ type: "number",
2549
+ required: true
2550
+ },
2551
+ sampled: {
2552
+ type: "boolean",
2553
+ required: true
2554
+ },
2555
+ profiledRows: {
2556
+ type: "number",
2557
+ required: true
2558
+ },
2559
+ columnCount: {
2560
+ type: "number",
2561
+ required: true
2562
+ },
2563
+ duplicateRows: {
2564
+ type: "number",
2565
+ required: true
2566
+ },
2567
+ duplicateRate: {
2568
+ type: "number",
2569
+ required: true
2570
+ },
2571
+ duplicateSampleRowIndexes: {
2572
+ type: "array",
2573
+ items: { type: "number" },
2574
+ required: true
2575
+ },
2576
+ scorecard: {
2577
+ type: "object",
2578
+ properties: {
2579
+ overall: {
2580
+ oneOf: [{ type: "number" }, { type: "null" }],
2581
+ required: true
2582
+ },
2583
+ weightedOverall: {
2584
+ oneOf: [{ type: "number" }, { type: "null" }],
2585
+ required: true
2586
+ },
2587
+ dimensions: {
2588
+ type: "array",
2589
+ items: {
2590
+ type: "object",
2591
+ properties: {
2592
+ name: {
2593
+ type: "string",
2594
+ enum: [...SCORECARD_DIMENSION_NAMES],
2595
+ required: true
2596
+ },
2597
+ score: {
2598
+ oneOf: [{ type: "number" }, { type: "null" }],
2599
+ required: true
2600
+ },
2601
+ note: {
2602
+ type: "string",
2603
+ required: true
2604
+ }
2605
+ },
2606
+ additionalProperties: false
2607
+ },
2608
+ required: true
2609
+ }
2610
+ },
2611
+ additionalProperties: false,
2612
+ required: true
2613
+ },
2614
+ encoding: {
2615
+ type: "object",
2616
+ properties: {
2617
+ bom: {
2618
+ oneOf: [{ type: "string" }, { type: "null" }],
2619
+ required: true
2620
+ },
2621
+ validUtf8: {
2622
+ type: "boolean",
2623
+ required: true
2624
+ }
2625
+ },
2626
+ additionalProperties: false
2627
+ },
2628
+ generatedAt: {
2629
+ type: "number",
2630
+ required: true
2631
+ },
2632
+ reportKey: { type: "string" },
2633
+ columns: {
2634
+ type: "array",
2635
+ items: COLUMN_PROFILE_SCHEMA,
2636
+ required: true
2637
+ }
2638
+ },
2639
+ additionalProperties: false
2640
+ };
2641
+ //#endregion
2642
+ //#region src/tools/shared.ts
2643
+ /**
2644
+ * Shared helpers for the three data-quality model tools: session workspace
2645
+ * resolution and a compact cleaning-run text render. Kept presentation-only;
2646
+ * computation lives in the engines and the provider.
2647
+ * @module dsh-data-quality/tools-shared
2648
+ */
2649
+ /**
2650
+ * The absolute workspace root a tool call resolves dataset paths against: the
2651
+ * calling agent's per-session cwd (mirroring the official fs tools). Non-agent
2652
+ * calls fail loud — there is no honest workspace to confine paths to.
2653
+ * @param exec - the tool-execution context.
2654
+ * @returns the absolute workspace root.
2655
+ */
2656
+ function workspaceOf(exec) {
2657
+ const cwd = exec.agent?.session.header.cwd;
2658
+ if (cwd === void 0) throw new Error("data-quality tools require an agent-owned session workspace");
2659
+ return path.resolve(cwd);
2660
+ }
2661
+ /** Human-readable cleaning summary for the tool's Native render. */
2662
+ function renderCleanText(report) {
2663
+ const lines = [];
2664
+ const mode = report.dryRun ? "Dry-run plan for" : "Cleaned";
2665
+ lines.push(`${mode} ${report.dataset}: ${report.inputRows} -> ${report.outputRows} rows over ${report.logs.length} rule(s)`);
2666
+ for (const log of report.logs) lines.push(`- rule ${log.ruleIndex} (${log.rule}): ${log.affectedRows} row(s) affected; ${log.detail}`);
2667
+ const contract = report.contract;
2668
+ const keyLabel = contract.dedupeColumns === null ? "full rows" : `[${contract.dedupeColumns.join(", ")}]`;
2669
+ lines.push(`Contract: ${contract.inputRows} -> ${contract.outputRows} rows (${contract.removedRows} removed); uniqueness ${contract.uniqueKeys ? "OK" : "VIOLATED"} over ${keyLabel}${contract.remainingDuplicateRows > 0 ? ` (${contract.remainingDuplicateRows} duplicate row(s) remain)` : ""}`);
2670
+ for (const entry of contract.remainingMissing) lines.push(`- non-null regression: ${entry.column} still has ${entry.count} missing cell(s)`);
2671
+ for (const entry of contract.typeConformance) lines.push(`- type regression: ${entry.column} (${entry.to}) has ${entry.invalidCount} non-conforming cell(s)`);
2672
+ for (const entry of contract.columnDecisions) {
2673
+ const decisions = entry.decisions.map((decision) => `${decision.strategy} (${decision.affectedRows} row(s))`).join(", ");
2674
+ lines.push(`- column ${entry.column}: ${decisions}`);
2675
+ }
2676
+ if (report.diffPreview !== void 0) lines.push(`Diff preview: ${report.diffPreview.before.rowCount} -> ${report.diffPreview.after.rowCount} rows; duplicates ${report.diffPreview.before.duplicateRows} -> ${report.diffPreview.after.duplicateRows}`);
2677
+ if (report.dryRun) lines.push("Dry run: no output file written and no report persisted; plan/preview above.");
2678
+ else if (report.outputPath !== void 0) lines.push(`Wrote cleaned dataset to ${report.outputPath}`);
2679
+ else lines.push("No outputPath given: the source file was left untouched; preview below.");
2680
+ if (report.preview.rows.length > 0) {
2681
+ lines.push(`Preview (first ${report.preview.rows.length} row(s)):`);
2682
+ for (const row of report.preview.rows) {
2683
+ const cells = report.preview.columns.map((column) => `${column}=${JSON.stringify(row[column] ?? null)}`).join(", ");
2684
+ lines.push(` ${cells}`);
2685
+ }
2686
+ }
2687
+ return lines.join("\n");
2688
+ }
2689
+ //#endregion
2690
+ //#region src/tools/profile.ts
2691
+ /**
2692
+ * The `data_profile` model tool: deterministic dataset profiling through
2693
+ * `ctx.dataQuality.profileDataset` — never model arithmetic.
2694
+ * @module dsh-data-quality/tools/profile
2695
+ */
1757
2696
  /**
1758
2697
  * Build the `data_profile` tool definition against a mounted service.
1759
2698
  * @param service - the mounted ctx.dataQuality implementation.
@@ -1764,8 +2703,8 @@ function defineProfileTool(service) {
1764
2703
  name: "data_profile",
1765
2704
  description: [
1766
2705
  "Profile a workspace CSV/TSV/JSON/JSONL dataset with deterministic TypeScript computation (no mental math).",
1767
- "Returns row/column counts, inferred column types, missing rates, unique counts, numeric distributions (min/max/mean/median/p25/p75), IQR outlier counts, mixed-type suspicion notes, and duplicate-row counts.",
1768
- "Column cards cover every row by default; pass sample for a deterministic systematic sample on large files. Datasets above the configured row/size caps are rejected — use sample or raise the caps. The full report persists to the data_quality storage domain (reportKey in the result)."
2706
+ "Returns row/column counts, inferred column types, missing rates, unique counts, numeric distributions (count/distinct/min/max/mean/median/p25/p75), IQR outlier counts, mixed-type suspicion notes, sha256 duplicate-row detection (rate + sample indexes), file encoding (BOM/UTF-8 validity), and a weighted DAMA six-dimension scorecard.",
2707
+ "Pass industryPreset (retail/saas/fund/real-estate/e-commerce/healthcare/logistics/manufacturing/energy) to compare the dataset against that industry's expected columns, making the scorecard accuracy dimension determinable. Column cards cover every row by default; pass sample for a deterministic systematic sample on large files. Datasets above the configured row/size caps are rejected — use sample or raise the caps. The full report persists to the data_quality storage domain (reportKey in the result)."
1769
2708
  ].join("\n"),
1770
2709
  parameters: {
1771
2710
  path: {
@@ -1776,49 +2715,25 @@ function defineProfileTool(service) {
1776
2715
  sample: {
1777
2716
  type: "number",
1778
2717
  description: "Optional systematic sample size (every ceil(N/sample)-th row) for the column cards; row counts stay exact."
2718
+ },
2719
+ industryPreset: {
2720
+ type: "string",
2721
+ enum: [
2722
+ "retail",
2723
+ "saas",
2724
+ "fund",
2725
+ "real-estate",
2726
+ "e-commerce",
2727
+ "healthcare",
2728
+ "logistics",
2729
+ "manufacturing",
2730
+ "energy"
2731
+ ],
2732
+ description: "Optional industry preset id; its expected columns feed the scorecard accuracy dimension."
1779
2733
  }
1780
2734
  },
1781
2735
  output: {
1782
- schema: {
1783
- type: "object",
1784
- properties: {
1785
- dataset: {
1786
- type: "string",
1787
- required: true
1788
- },
1789
- rowCount: {
1790
- type: "number",
1791
- required: true
1792
- },
1793
- sampled: {
1794
- type: "boolean",
1795
- required: true
1796
- },
1797
- profiledRows: {
1798
- type: "number",
1799
- required: true
1800
- },
1801
- columnCount: {
1802
- type: "number",
1803
- required: true
1804
- },
1805
- duplicateRows: {
1806
- type: "number",
1807
- required: true
1808
- },
1809
- generatedAt: {
1810
- type: "number",
1811
- required: true
1812
- },
1813
- reportKey: { type: "string" },
1814
- columns: {
1815
- type: "array",
1816
- items: COLUMN_PROFILE_SCHEMA,
1817
- required: true
1818
- }
1819
- },
1820
- additionalProperties: false
1821
- },
2736
+ schema: PROFILE_REPORT_SCHEMA,
1822
2737
  render: (_args, value) => [{
1823
2738
  type: "text",
1824
2739
  text: renderProfileText(value)
@@ -1828,6 +2743,7 @@ function defineProfileTool(service) {
1828
2743
  return service.profileDataset({
1829
2744
  dataset: args.path,
1830
2745
  sample: args.sample,
2746
+ ...args.industryPreset !== void 0 ? { industryPreset: args.industryPreset } : {},
1831
2747
  workspace: workspaceOf(exec),
1832
2748
  session: exec.agent?.session,
1833
2749
  signal: exec.signal
@@ -1995,7 +2911,7 @@ function defineCleanTool(service) {
1995
2911
  description: [
1996
2912
  "Apply declarative cleaning rules to a workspace CSV/TSV/JSON/JSONL dataset with deterministic TypeScript computation (no mental math).",
1997
2913
  "Rules apply in array order: dedupe (by column group), fill-missing (constant/mean/median/forward), coerce-type (number/date/boolean; failures counted and set to missing), normalize-unit (e.g. 万/亿 suffixes to base units), trim (whitespace), map-values (enum mapping).",
1998
- "The source file is NEVER overwritten. Without outputPath the run is preview-only; with outputPath the cleaned dataset is written there (workspace-confined, .csv/.tsv/.json/.jsonl). Returns the per-rule audit log (affected rows per rule) plus a bounded preview. The full report persists to the data_quality storage domain (reportKey)."
2914
+ "The source file is NEVER overwritten. Without outputPath the run is preview-only; with outputPath the cleaned dataset is written there (workspace-confined, .csv/.tsv/.json/.jsonl). Returns the per-rule audit log, the pre-delivery contract summary (with per-column decision trace), and a bounded preview. Pass dryRun: true to skip the write and get the cleaning plan plus the expected contract/diff preview instead. The full report persists to the data_quality storage domain (reportKey)."
1999
2915
  ].join("\n"),
2000
2916
  parameters: {
2001
2917
  path: {
@@ -2010,6 +2926,10 @@ function defineCleanTool(service) {
2010
2926
  outputPath: {
2011
2927
  type: "string",
2012
2928
  description: "Optional workspace-relative output path for the cleaned dataset (must differ from path)."
2929
+ },
2930
+ dryRun: {
2931
+ type: "boolean",
2932
+ description: "When true, do not write any output file; return the cleaning plan and expected contract/diff preview instead (default false)."
2013
2933
  }
2014
2934
  },
2015
2935
  output: {
@@ -2028,12 +2948,38 @@ function defineCleanTool(service) {
2028
2948
  type: "number",
2029
2949
  required: true
2030
2950
  },
2951
+ dryRun: {
2952
+ type: "boolean",
2953
+ required: true
2954
+ },
2031
2955
  generatedAt: {
2032
2956
  type: "number",
2033
2957
  required: true
2034
2958
  },
2035
2959
  outputPath: { type: "string" },
2036
2960
  reportKey: { type: "string" },
2961
+ diffPreview: {
2962
+ type: "object",
2963
+ properties: {
2964
+ dataset: {
2965
+ type: "string",
2966
+ required: true
2967
+ },
2968
+ before: {
2969
+ ...PROFILE_REPORT_SCHEMA,
2970
+ required: true
2971
+ },
2972
+ after: {
2973
+ ...PROFILE_REPORT_SCHEMA,
2974
+ required: true
2975
+ },
2976
+ generatedAt: {
2977
+ type: "number",
2978
+ required: true
2979
+ }
2980
+ },
2981
+ additionalProperties: false
2982
+ },
2037
2983
  logs: {
2038
2984
  type: "array",
2039
2985
  items: {
@@ -2068,6 +3014,117 @@ function defineCleanTool(service) {
2068
3014
  },
2069
3015
  required: true
2070
3016
  },
3017
+ contract: {
3018
+ type: "object",
3019
+ properties: {
3020
+ inputRows: {
3021
+ type: "number",
3022
+ required: true
3023
+ },
3024
+ outputRows: {
3025
+ type: "number",
3026
+ required: true
3027
+ },
3028
+ removedRows: {
3029
+ type: "number",
3030
+ required: true
3031
+ },
3032
+ dedupeColumns: {
3033
+ oneOf: [{
3034
+ type: "array",
3035
+ items: { type: "string" }
3036
+ }, { type: "null" }],
3037
+ required: true
3038
+ },
3039
+ uniqueKeys: {
3040
+ type: "boolean",
3041
+ required: true
3042
+ },
3043
+ remainingDuplicateRows: {
3044
+ type: "number",
3045
+ required: true
3046
+ },
3047
+ remainingMissing: {
3048
+ type: "array",
3049
+ items: {
3050
+ type: "object",
3051
+ properties: {
3052
+ column: {
3053
+ type: "string",
3054
+ required: true
3055
+ },
3056
+ count: {
3057
+ type: "number",
3058
+ required: true
3059
+ }
3060
+ },
3061
+ additionalProperties: false
3062
+ },
3063
+ required: true
3064
+ },
3065
+ typeConformance: {
3066
+ type: "array",
3067
+ items: {
3068
+ type: "object",
3069
+ properties: {
3070
+ column: {
3071
+ type: "string",
3072
+ required: true
3073
+ },
3074
+ to: {
3075
+ type: "string",
3076
+ enum: [
3077
+ "number",
3078
+ "date",
3079
+ "boolean"
3080
+ ],
3081
+ required: true
3082
+ },
3083
+ invalidCount: {
3084
+ type: "number",
3085
+ required: true
3086
+ }
3087
+ },
3088
+ additionalProperties: false
3089
+ },
3090
+ required: true
3091
+ },
3092
+ columnDecisions: {
3093
+ type: "array",
3094
+ items: {
3095
+ type: "object",
3096
+ properties: {
3097
+ column: {
3098
+ type: "string",
3099
+ required: true
3100
+ },
3101
+ decisions: {
3102
+ type: "array",
3103
+ items: {
3104
+ type: "object",
3105
+ properties: {
3106
+ strategy: {
3107
+ type: "string",
3108
+ required: true
3109
+ },
3110
+ affectedRows: {
3111
+ type: "number",
3112
+ required: true
3113
+ }
3114
+ },
3115
+ additionalProperties: false
3116
+ },
3117
+ required: true
3118
+ }
3119
+ },
3120
+ additionalProperties: false
3121
+ },
3122
+ required: true
3123
+ }
3124
+ },
3125
+ additionalProperties: false,
3126
+ required: true
3127
+ },
2071
3128
  preview: {
2072
3129
  type: "object",
2073
3130
  properties: {
@@ -2098,6 +3155,7 @@ function defineCleanTool(service) {
2098
3155
  dataset: args.path,
2099
3156
  rules: args.rules,
2100
3157
  ...args.outputPath !== void 0 ? { outputPath: args.outputPath } : {},
3158
+ ...args.dryRun !== void 0 ? { dryRun: args.dryRun } : {},
2101
3159
  workspace: workspaceOf(exec),
2102
3160
  session: exec.agent?.session,
2103
3161
  signal: exec.signal
@@ -2113,6 +3171,74 @@ function defineCleanTool(service) {
2113
3171
  * (`passed: false` with evidence), never a tool error.
2114
3172
  * @module dsh-data-quality/tools/verify
2115
3173
  */
3174
+ const EXPECTATION_METRICS = [
3175
+ "rowCount",
3176
+ "columnSum",
3177
+ "columnMean",
3178
+ "uniqueCount",
3179
+ "nullCount"
3180
+ ];
3181
+ const EXPECTATION_SCHEMA = {
3182
+ type: "array",
3183
+ items: {
3184
+ type: "object",
3185
+ properties: {
3186
+ metric: {
3187
+ type: "string",
3188
+ enum: [...EXPECTATION_METRICS],
3189
+ required: true,
3190
+ description: "Metric to reconcile: rowCount/columnSum/columnMean/uniqueCount/nullCount."
3191
+ },
3192
+ column: {
3193
+ type: "string",
3194
+ description: "Required for every metric except rowCount."
3195
+ },
3196
+ expected: {
3197
+ type: "number",
3198
+ required: true,
3199
+ description: "The expected value to reconcile against."
3200
+ },
3201
+ tolerance: {
3202
+ type: "number",
3203
+ description: "Optional relative tolerance in [0, 1]; defaults to defaultTolerance."
3204
+ }
3205
+ },
3206
+ additionalProperties: false,
3207
+ description: "Reconcile a deterministic computed metric against an expected value with relative tolerance."
3208
+ },
3209
+ description: "Optional metric expectations; each yields passed true or passed false with actual/expected/tolerance detail."
3210
+ };
3211
+ const EXPECTATION_RESULT_SCHEMA = {
3212
+ type: "array",
3213
+ items: {
3214
+ type: "object",
3215
+ properties: {
3216
+ metric: {
3217
+ type: "string",
3218
+ enum: [...EXPECTATION_METRICS],
3219
+ required: true
3220
+ },
3221
+ column: { type: "string" },
3222
+ expected: {
3223
+ type: "number",
3224
+ required: true
3225
+ },
3226
+ actual: {
3227
+ type: "number",
3228
+ required: true
3229
+ },
3230
+ tolerance: {
3231
+ type: "number",
3232
+ required: true
3233
+ },
3234
+ passed: {
3235
+ type: "boolean",
3236
+ required: true
3237
+ }
3238
+ },
3239
+ additionalProperties: false
3240
+ }
3241
+ };
2116
3242
  const VERIFY_RULE_SCHEMA = {
2117
3243
  type: "array",
2118
3244
  items: { oneOf: [
@@ -2290,7 +3416,8 @@ function defineVerifyTool(service) {
2290
3416
  description: [
2291
3417
  "Verify a workspace CSV/TSV/JSON/JSONL dataset against declarative quality rules with deterministic TypeScript computation (no mental math).",
2292
3418
  "Rules: not-null, unique (column group), range (numeric bounds), regex, enum, cross-column (e.g. startDate < endDate), freshness (date column within N days of asOf). A missing cell fails every rule that reads it.",
2293
- "Returns per-rule pass/fail with capped failing-row evidence. Overall failure is a NORMAL result with passed: false not a tool error. The full report persists to the data_quality storage domain (reportKey)."
3419
+ "Optional expectations reconcile deterministic metrics (rowCount/columnSum/columnMean/uniqueCount/nullCount) against expected values with relative tolerance; a mismatch is a normal passed: false with actual/expected/tolerance detail, never a tool error.",
3420
+ "Returns per-rule pass/fail with capped failing-row evidence plus the expectation outcomes. Overall failure is a NORMAL result with passed: false — not a tool error. The full report persists to the data_quality storage domain (reportKey)."
2294
3421
  ].join("\n"),
2295
3422
  parameters: {
2296
3423
  path: {
@@ -2301,6 +3428,10 @@ function defineVerifyTool(service) {
2301
3428
  rules: {
2302
3429
  ...VERIFY_RULE_SCHEMA,
2303
3430
  required: true
3431
+ },
3432
+ expectations: {
3433
+ ...EXPECTATION_SCHEMA,
3434
+ description: "Optional metric expectations to reconcile (rowCount/columnSum/columnMean/uniqueCount/nullCount)."
2304
3435
  }
2305
3436
  },
2306
3437
  output: {
@@ -2380,6 +3511,10 @@ function defineVerifyTool(service) {
2380
3511
  additionalProperties: false
2381
3512
  },
2382
3513
  required: true
3514
+ },
3515
+ expectations: {
3516
+ ...EXPECTATION_RESULT_SCHEMA,
3517
+ required: true
2383
3518
  }
2384
3519
  },
2385
3520
  additionalProperties: false
@@ -2393,6 +3528,7 @@ function defineVerifyTool(service) {
2393
3528
  return service.verifyDataset({
2394
3529
  dataset: args.path,
2395
3530
  rules: args.rules,
3531
+ ...args.expectations !== void 0 ? { expectations: args.expectations } : {},
2396
3532
  workspace: workspaceOf(exec),
2397
3533
  session: exec.agent?.session,
2398
3534
  signal: exec.signal
@@ -2401,6 +3537,130 @@ function defineVerifyTool(service) {
2401
3537
  });
2402
3538
  }
2403
3539
  //#endregion
3540
+ //#region src/tools/report.ts
3541
+ /**
3542
+ * The `data_report` model tool: read persisted profile/clean-diff reports
3543
+ * back from the `data_quality` storage domain, by exact `reportKey` or by
3544
+ * `kind`. Deterministic read-only consumption — no model arithmetic.
3545
+ * @module dsh-data-quality/tools/report
3546
+ */
3547
+ /** The report kinds a caller may query. */
3548
+ const REPORT_KINDS = [
3549
+ "profile",
3550
+ "clean",
3551
+ "clean-diff",
3552
+ "verify",
3553
+ "citations"
3554
+ ];
3555
+ /** Project a stored report into the canonical value (the stored report is already lossless JSON). */
3556
+ function toView(record) {
3557
+ return {
3558
+ key: record.key,
3559
+ kind: record.kind,
3560
+ at: record.at,
3561
+ dataset: record.dataset,
3562
+ report: record.report
3563
+ };
3564
+ }
3565
+ /** Human-readable report summary for the tool's Native render. */
3566
+ function renderReportText(value) {
3567
+ const lines = [];
3568
+ if (value.key !== void 0) lines.push(value.records.length === 0 ? `No report for ${value.key}` : `Report ${value.key}`);
3569
+ else lines.push(`Reports of kind ${value.kind ?? ""} (${value.records.length})`);
3570
+ for (const record of value.records) lines.push(`- ${record.key} [${record.kind}] ${record.dataset} @ ${new Date(record.at).toISOString()}`);
3571
+ return lines.join("\n");
3572
+ }
3573
+ /**
3574
+ * Build the `data_report` tool definition against a mounted service.
3575
+ * @param service - the mounted ctx.dataQuality implementation.
3576
+ * @returns the tool definition to register.
3577
+ */
3578
+ function defineReportTool(service) {
3579
+ return defineTool({
3580
+ name: "data_report",
3581
+ description: [
3582
+ "Read persisted data-quality reports back from the data_quality storage domain (deterministic, read-only).",
3583
+ "Pass key (the exact reportKey a prior run returned) to fetch one report, or kind to list every persisted report of that kind, ordered chronologically. Exactly one of key/kind.",
3584
+ "Returns the report envelope(s): kind, dataset, timestamp, and the full stored report (profile/clean/clean-diff/verify/citations). Missing keys and unknown kinds fail loudly."
3585
+ ].join("\n"),
3586
+ parameters: {
3587
+ key: {
3588
+ type: "string",
3589
+ description: "Exact storage reportKey (e.g. 20260819000000000-profile-1a2b3c4d); fetches that one report."
3590
+ },
3591
+ kind: {
3592
+ type: "string",
3593
+ enum: [...REPORT_KINDS],
3594
+ description: "Report kind to list (profile/clean/clean-diff/verify/citations)."
3595
+ }
3596
+ },
3597
+ output: {
3598
+ schema: {
3599
+ type: "object",
3600
+ properties: {
3601
+ key: { type: "string" },
3602
+ kind: {
3603
+ type: "string",
3604
+ enum: [...REPORT_KINDS]
3605
+ },
3606
+ records: {
3607
+ type: "array",
3608
+ items: {
3609
+ type: "object",
3610
+ properties: {
3611
+ key: {
3612
+ type: "string",
3613
+ required: true
3614
+ },
3615
+ kind: {
3616
+ type: "string",
3617
+ enum: [...REPORT_KINDS],
3618
+ required: true
3619
+ },
3620
+ at: {
3621
+ type: "number",
3622
+ required: true
3623
+ },
3624
+ dataset: {
3625
+ type: "string",
3626
+ required: true
3627
+ },
3628
+ report: {
3629
+ type: "json",
3630
+ required: true
3631
+ }
3632
+ },
3633
+ additionalProperties: false
3634
+ },
3635
+ required: true
3636
+ }
3637
+ },
3638
+ additionalProperties: false
3639
+ },
3640
+ render: (_args, value) => [{
3641
+ type: "text",
3642
+ text: renderReportText(value)
3643
+ }]
3644
+ },
3645
+ async execute(args, _exec) {
3646
+ const hasKey = args.key !== void 0;
3647
+ if (hasKey === (args.kind !== void 0)) throw new Error("data_report needs exactly one of key/kind");
3648
+ if (hasKey) {
3649
+ const record = await service.getReport(args.key);
3650
+ return {
3651
+ key: args.key,
3652
+ records: [toView(record)]
3653
+ };
3654
+ }
3655
+ const records = await service.listReports(args.kind);
3656
+ return {
3657
+ kind: args.kind,
3658
+ records: records.map(toView)
3659
+ };
3660
+ }
3661
+ });
3662
+ }
3663
+ //#endregion
2404
3664
  //#region src/version.ts
2405
3665
  /**
2406
3666
  * Plugin version, kept in one place so `scripts/release.mjs` can stamp it and
@@ -2408,11 +3668,11 @@ function defineVerifyTool(service) {
2408
3668
  * @module dsh-data-quality/version
2409
3669
  */
2410
3670
  /** The package version reported in persisted reports. */
2411
- const VERSION = "0.1.3";
3671
+ const VERSION = "0.2.0";
2412
3672
  //#endregion
2413
3673
  //#region src/index.ts
2414
3674
  const name = "data-quality";
2415
- /** The three model tools and the durable report domain. */
3675
+ /** The four model tools and the durable report domain. */
2416
3676
  const inject = ["tools", "storageDomain"];
2417
3677
  /**
2418
3678
  * Mount the seam: resolve config (fail loud), open the report domain, publish
@@ -2439,7 +3699,11 @@ async function apply(ctx, config = {}) {
2439
3699
  await reports.put(key, record);
2440
3700
  return key;
2441
3701
  },
2442
- get: (key) => reports.get(key)
3702
+ get: (key) => reports.get(key),
3703
+ list: (kind) => [...reports.entries()].filter(([, record]) => record.kind === kind).sort(([keyA], [keyB]) => keyA < keyB ? -1 : keyA > keyB ? 1 : 0).map(([key, record]) => ({
3704
+ key,
3705
+ ...record
3706
+ }))
2443
3707
  };
2444
3708
  }
2445
3709
  const service = new LocalDataQualityService(ctx, resolved, {
@@ -2449,7 +3713,8 @@ async function apply(ctx, config = {}) {
2449
3713
  ctx.tools.register(defineProfileTool(service));
2450
3714
  ctx.tools.register(defineCleanTool(service));
2451
3715
  ctx.tools.register(defineVerifyTool(service));
2452
- logger.info(`dsh-data-quality ${VERSION} mounted: ctx.dataQuality + data_profile/data_clean/data_verify`);
3716
+ ctx.tools.register(defineReportTool(service));
3717
+ logger.info(`dsh-data-quality ${VERSION} mounted: ctx.dataQuality + data_profile/data_clean/data_verify/data_report`);
2453
3718
  if (domain !== void 0) {
2454
3719
  const handle = domain;
2455
3720
  ctx.effect(() => async () => {
@@ -2458,4 +3723,4 @@ async function apply(ctx, config = {}) {
2458
3723
  }
2459
3724
  }
2460
3725
  //#endregion
2461
- export { Config, DATA_QUALITY_EVENT_TYPES, DataQualityService, DatasetError, LocalDataQualityService, MAX_CELL_TEXT, VERSION, appendDataQualityEvent, apply, applyCleanRules, checkCitations, dataQualityDomainSpec, inject, isMissing, loadDocument, loadTable, name, parseBoolean, parseDate, parseDelimited, parseJsonTable, parseLocator, parseNumeric, profileTable, renderProfileText, renderVerifyText, reportKeyOf, reportRecordSchema, resolveConfig, resolveWorkspacePath, sampleRows, serializeDelimited, truncateCell, truncateRow, verifyTable };
3726
+ export { Config, DATA_QUALITY_EVENT_TYPES, DataQualityService, DatasetError, INDUSTRY_PRESETS, INDUSTRY_PRESET_IDS, LocalDataQualityService, MAX_CELL_TEXT, VERSION, VerifyExpectationError, appendDataQualityEvent, apply, applyCleanRules, checkCitations, computeCleanContract, computeCleanProfileDiff, computeScorecard, countDuplicateRows, dataQualityDomainSpec, dateFormatOf, detectDuplicateRows, detectEncoding, inject, isMissing, isValidReportKey, loadDocument, loadTable, name, numericProfile, parseBoolean, parseDate, parseDateCell, parseDelimited, parseJsonTable, parseLocator, parseNumeric, profileTable, readDatasetFile, renderProfileText, renderVerifyText, reportKeyOf, reportRecordSchema, resolveConfig, resolveIndustryPreset, resolveWorkspacePath, sampleRows, serializeDelimited, truncateCell, truncateRow, verifyExpectations, verifyTable };