dsh-data-quality 0.1.3 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/CHANGELOG.md +33 -0
  2. package/README.es.md +7 -1
  3. package/README.hi.md +7 -1
  4. package/README.md +22 -9
  5. package/README.pt.md +7 -1
  6. package/README.zh.md +7 -1
  7. package/cordis.patch.yml +11 -0
  8. package/lib/index.js +1609 -145
  9. package/lib/types/config.d.ts +12 -0
  10. package/lib/types/config.d.ts.map +1 -1
  11. package/lib/types/config.js +35 -0
  12. package/lib/types/config.js.map +1 -1
  13. package/lib/types/contract.d.ts +83 -0
  14. package/lib/types/contract.d.ts.map +1 -0
  15. package/lib/types/contract.js +144 -0
  16. package/lib/types/contract.js.map +1 -0
  17. package/lib/types/dataset.d.ts +50 -3
  18. package/lib/types/dataset.d.ts.map +1 -1
  19. package/lib/types/dataset.js +76 -21
  20. package/lib/types/dataset.js.map +1 -1
  21. package/lib/types/events.d.ts +2 -0
  22. package/lib/types/events.d.ts.map +1 -1
  23. package/lib/types/events.js.map +1 -1
  24. package/lib/types/index.d.ts +13 -8
  25. package/lib/types/index.d.ts.map +1 -1
  26. package/lib/types/index.js +17 -6
  27. package/lib/types/index.js.map +1 -1
  28. package/lib/types/presets.d.ts +28 -0
  29. package/lib/types/presets.d.ts.map +1 -0
  30. package/lib/types/presets.js +134 -0
  31. package/lib/types/presets.js.map +1 -0
  32. package/lib/types/profile.d.ts +40 -1
  33. package/lib/types/profile.d.ts.map +1 -1
  34. package/lib/types/profile.js +66 -10
  35. package/lib/types/profile.js.map +1 -1
  36. package/lib/types/provider-local.d.ts +7 -1
  37. package/lib/types/provider-local.d.ts.map +1 -1
  38. package/lib/types/provider-local.js +72 -2
  39. package/lib/types/provider-local.js.map +1 -1
  40. package/lib/types/report-html.d.ts +41 -0
  41. package/lib/types/report-html.d.ts.map +1 -0
  42. package/lib/types/report-html.js +192 -0
  43. package/lib/types/report-html.js.map +1 -0
  44. package/lib/types/scorecard.d.ts +50 -0
  45. package/lib/types/scorecard.d.ts.map +1 -0
  46. package/lib/types/scorecard.js +186 -0
  47. package/lib/types/scorecard.js.map +1 -0
  48. package/lib/types/service.d.ts +32 -2
  49. package/lib/types/service.d.ts.map +1 -1
  50. package/lib/types/service.js.map +1 -1
  51. package/lib/types/store.d.ts +21 -1
  52. package/lib/types/store.d.ts.map +1 -1
  53. package/lib/types/store.js +13 -1
  54. package/lib/types/store.js.map +1 -1
  55. package/lib/types/tools/clean.d.ts.map +1 -1
  56. package/lib/types/tools/clean.js +76 -1
  57. package/lib/types/tools/clean.js.map +1 -1
  58. package/lib/types/tools/profile-report-schema.d.ts +319 -0
  59. package/lib/types/tools/profile-report-schema.d.ts.map +1 -0
  60. package/lib/types/tools/profile-report-schema.js +96 -0
  61. package/lib/types/tools/profile-report-schema.js.map +1 -0
  62. package/lib/types/tools/profile.d.ts.map +1 -1
  63. package/lib/types/tools/profile.js +6 -53
  64. package/lib/types/tools/profile.js.map +1 -1
  65. package/lib/types/tools/report.d.ts +14 -0
  66. package/lib/types/tools/report.d.ts.map +1 -0
  67. package/lib/types/tools/report.js +112 -0
  68. package/lib/types/tools/report.js.map +1 -0
  69. package/lib/types/tools/shared.d.ts.map +1 -1
  70. package/lib/types/tools/shared.js +22 -2
  71. package/lib/types/tools/shared.js.map +1 -1
  72. package/lib/types/tools/verify.d.ts.map +1 -1
  73. package/lib/types/tools/verify.js +36 -1
  74. package/lib/types/tools/verify.js.map +1 -1
  75. package/lib/types/verify.d.ts +49 -3
  76. package/lib/types/verify.d.ts.map +1 -1
  77. package/lib/types/verify.js +111 -4
  78. package/lib/types/verify.js.map +1 -1
  79. package/lib/types/version.d.ts +8 -1
  80. package/lib/types/version.d.ts.map +1 -1
  81. package/lib/types/version.js +8 -1
  82. package/lib/types/version.js.map +1 -1
  83. package/package.json +1 -1
  84. package/src/config.ts +52 -0
  85. package/src/contract.ts +190 -0
  86. package/src/dataset.ts +101 -21
  87. package/src/events.ts +2 -0
  88. package/src/index.ts +27 -8
  89. package/src/presets.ts +146 -0
  90. package/src/profile.ts +105 -11
  91. package/src/provider-local.ts +77 -3
  92. package/src/report-html.ts +208 -0
  93. package/src/scorecard.ts +244 -0
  94. package/src/service.ts +34 -2
  95. package/src/store.ts +28 -2
  96. package/src/tools/clean.ts +76 -1
  97. package/src/tools/profile-report-schema.ts +98 -0
  98. package/src/tools/profile.ts +8 -56
  99. package/src/tools/report.ts +137 -0
  100. package/src/tools/shared.ts +21 -2
  101. package/src/tools/verify.ts +40 -2
  102. package/src/verify.ts +151 -5
  103. package/src/version.ts +9 -1
package/lib/index.js CHANGED
@@ -4,6 +4,7 @@ import { defineDomain, domainTable } from "@deepseek-ai/dsh-storage-domain";
4
4
  import { mkdir, open, stat, writeFile } from "node:fs/promises";
5
5
  import path from "node:path";
6
6
  import { createHash } from "node:crypto";
7
+ import { TextDecoder } from "node:util";
7
8
  import { KNOWN_SESSION_EVENT_TYPES } from "@deepseek-ai/dsh-session";
8
9
  import { Service } from "@deepseek-ai/cordis";
9
10
  import { defineTool } from "@deepseek-ai/dsh-tools";
@@ -14,6 +15,15 @@ import { defineTool } from "@deepseek-ai/dsh-tools";
14
15
  * step validates bounds so misconfiguration fails loud at mount.
15
16
  * @module dsh-data-quality/config
16
17
  */
18
+ /** The six scorecard dimension ids, in report order (kept in sync with `scorecard.ts`). */
19
+ const SCORECARD_DIMENSIONS = [
20
+ "completeness",
21
+ "uniqueness",
22
+ "validity",
23
+ "consistency",
24
+ "timeliness",
25
+ "accuracy"
26
+ ];
17
27
  /** Schemastery schema: the loader validates and fills defaults before `apply`. */
18
28
  const Config = z.object({
19
29
  enabled: z.boolean().default(true),
@@ -28,7 +38,22 @@ const Config = z.object({
28
38
  ".jsonl"
29
39
  ]),
30
40
  workspaceRoot: z.string().default(""),
31
- storeReports: z.boolean().default(true)
41
+ storeReports: z.boolean().default(true),
42
+ scorecardWeights: z.object({
43
+ completeness: z.number().default(1),
44
+ uniqueness: z.number().default(1),
45
+ validity: z.number().default(1),
46
+ consistency: z.number().default(1),
47
+ timeliness: z.number().default(1),
48
+ accuracy: z.number().default(1)
49
+ }).default({
50
+ completeness: 1,
51
+ uniqueness: 1,
52
+ validity: 1,
53
+ consistency: 1,
54
+ timeliness: 1,
55
+ accuracy: 1
56
+ })
32
57
  });
33
58
  /** Throw unless `value` is a positive safe integer. */
34
59
  function assertPositiveInt(name, value) {
@@ -63,6 +88,7 @@ function resolveConfig(config = {}) {
63
88
  return ext;
64
89
  });
65
90
  if (allowedExtensions.length === 0) throw new TypeError("allowedExtensions must not be empty");
91
+ const scorecardWeights = resolveScorecardWeights(config.scorecardWeights);
66
92
  return {
67
93
  enabled: config.enabled ?? true,
68
94
  maxRows,
@@ -71,9 +97,20 @@ function resolveConfig(config = {}) {
71
97
  evidenceRowLimit,
72
98
  allowedExtensions,
73
99
  workspaceRoot: config.workspaceRoot ?? "",
74
- storeReports: config.storeReports ?? true
100
+ storeReports: config.storeReports ?? true,
101
+ scorecardWeights
75
102
  };
76
103
  }
104
+ /** Resolve scorecard weights, filling defaults and rejecting non-negative violations loudly. */
105
+ function resolveScorecardWeights(weights = {}) {
106
+ const out = {};
107
+ for (const dimension of SCORECARD_DIMENSIONS) {
108
+ const value = weights[dimension] ?? 1;
109
+ if (typeof value !== "number" || !Number.isFinite(value) || value < 0) throw new TypeError(`scorecardWeights.${dimension} must be a non-negative finite number, got ${String(value)}`);
110
+ out[dimension] = value;
111
+ }
112
+ return out;
113
+ }
77
114
  //#endregion
78
115
  //#region src/dataset.ts
79
116
  /**
@@ -127,13 +164,40 @@ function throwIfAborted(signal) {
127
164
  if (signal?.aborted === true) throw signal.reason instanceof Error ? signal.reason : /* @__PURE__ */ new Error("operation aborted");
128
165
  }
129
166
  /**
130
- * Read a dataset file under the size cap.
167
+ * Detect a UTF-8 byte-order mark and validate the byte sequence. Invalid
168
+ * UTF-8 is a data-quality finding the profile reports (`validUtf8: false`)
169
+ * rather than a structural error that blocks the read — the decoded text
170
+ * keeps U+FFFD replacement characters so the profile can still run.
171
+ * @param buffer - raw file bytes.
172
+ * @returns the encoding metadata.
173
+ */
174
+ function detectEncoding(buffer) {
175
+ const bom = buffer.length >= 3 && buffer[0] === 239 && buffer[1] === 187 && buffer[2] === 191 ? "utf-8" : null;
176
+ const body = bom === "utf-8" ? buffer.subarray(3) : buffer;
177
+ let validUtf8 = true;
178
+ try {
179
+ new TextDecoder("utf-8", { fatal: true }).decode(body);
180
+ } catch {
181
+ validUtf8 = false;
182
+ }
183
+ return {
184
+ bom,
185
+ validUtf8
186
+ };
187
+ }
188
+ /** Decode file bytes, stripping a UTF-8 BOM and preserving replacement characters for invalid bytes. */
189
+ function decodeUtf8Text(buffer, encoding) {
190
+ const body = encoding.bom === "utf-8" ? buffer.subarray(3) : buffer;
191
+ return new TextDecoder("utf-8").decode(body);
192
+ }
193
+ /**
194
+ * Read a dataset file under the size cap, detecting its encoding.
131
195
  * @param absolutePath - normalized absolute path (from {@link resolveWorkspacePath}).
132
196
  * @param config - resolved config (size cap).
133
197
  * @param signal - optional abort signal honored around the read.
134
- * @returns the UTF-8 text.
198
+ * @returns the decoded UTF-8 text plus its encoding metadata.
135
199
  */
136
- async function readDatasetText(absolutePath, config, signal) {
200
+ async function readDatasetFile(absolutePath, config, signal) {
137
201
  throwIfAborted(signal);
138
202
  let info;
139
203
  try {
@@ -147,7 +211,12 @@ async function readDatasetText(absolutePath, config, signal) {
147
211
  const handle = await open(absolutePath, "r");
148
212
  try {
149
213
  throwIfAborted(signal);
150
- return await handle.readFile("utf8");
214
+ const buffer = await handle.readFile();
215
+ const encoding = detectEncoding(buffer);
216
+ return {
217
+ text: decodeUtf8Text(buffer, encoding),
218
+ encoding
219
+ };
151
220
  } finally {
152
221
  await handle.close();
153
222
  }
@@ -316,12 +385,21 @@ function parseJsonLines(text) {
316
385
  * @returns the parsed table.
317
386
  */
318
387
  async function loadTable(absolutePath, config, signal) {
319
- const text = await readDatasetText(absolutePath, config, signal);
388
+ const { text, encoding } = await readDatasetFile(absolutePath, config, signal);
320
389
  const ext = path.extname(absolutePath).toLowerCase();
321
390
  throwIfAborted(signal);
322
- if (ext === ".csv") return parseDelimited(text, ",", config, signal);
323
- if (ext === ".tsv") return parseDelimited(text, " ", config, signal);
324
- return parseJsonTable(text, ext, config, signal);
391
+ if (ext === ".csv") return {
392
+ ...parseDelimited(text, ",", config, signal),
393
+ encoding
394
+ };
395
+ if (ext === ".tsv") return {
396
+ ...parseDelimited(text, " ", config, signal),
397
+ encoding
398
+ };
399
+ return {
400
+ ...parseJsonTable(text, ext, config, signal),
401
+ encoding
402
+ };
325
403
  }
326
404
  /**
327
405
  * Load a dataset as a citation-checkable document root: CSV/TSV become
@@ -334,14 +412,20 @@ async function loadTable(absolutePath, config, signal) {
334
412
  */
335
413
  async function loadDocument(absolutePath, config, signal) {
336
414
  const ext = path.extname(absolutePath).toLowerCase();
337
- if (ext === ".json") return {
338
- kind: "json",
339
- value: parseJsonDocument(await readDatasetText(absolutePath, config, signal))
340
- };
341
- if (ext === ".jsonl") return {
342
- kind: "json",
343
- value: parseJsonLines(await readDatasetText(absolutePath, config, signal))
344
- };
415
+ if (ext === ".json") {
416
+ const { text } = await readDatasetFile(absolutePath, config, signal);
417
+ return {
418
+ kind: "json",
419
+ value: parseJsonDocument(text)
420
+ };
421
+ }
422
+ if (ext === ".jsonl") {
423
+ const { text } = await readDatasetFile(absolutePath, config, signal);
424
+ return {
425
+ kind: "json",
426
+ value: parseJsonLines(text)
427
+ };
428
+ }
345
429
  const table = await loadTable(absolutePath, config, signal);
346
430
  return {
347
431
  kind: "table",
@@ -390,18 +474,17 @@ const DATE_PATTERNS = [
390
474
  /^(\d{4})\/(\d{1,2})\/(\d{1,2})$/u,
391
475
  /^(\d{4})-(\d{1,2})-(\d{1,2})[ T](\d{1,2}):(\d{2})(?::(\d{2}))?(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?$/u
392
476
  ];
393
- /**
394
- * Deterministic date parse to epoch milliseconds. Accepts `YYYY-MM-DD`,
395
- * `YYYY/MM/DD`, and ISO-like datetimes (date-only forms read as UTC midnight).
396
- * Calendar-invalid dates (e.g. 2025-13-40) reject. Returns `undefined` when
397
- * the cell is not a recognized date.
398
- * @param cell - the cell to parse (`undefined` when the column is absent).
399
- * @returns epoch milliseconds, or `undefined`.
400
- */
401
- function parseDate(cell) {
477
+ /** Deterministic date-format labels in {@link DATE_PATTERNS} order. */
478
+ const DATE_FORMATS = [
479
+ "iso-date",
480
+ "slash-date",
481
+ "datetime"
482
+ ];
483
+ /** Parse one date cell to its epoch plus format; `undefined` when unrecognized. */
484
+ function parseDateCellInternal(cell) {
402
485
  if (typeof cell !== "string") return void 0;
403
486
  const text = cell.trim();
404
- for (const pattern of DATE_PATTERNS) {
487
+ for (const [index, pattern] of DATE_PATTERNS.entries()) {
405
488
  const match = pattern.exec(text);
406
489
  if (match === null) continue;
407
490
  const year = Number(match[1]);
@@ -414,9 +497,43 @@ function parseDate(cell) {
414
497
  const epoch = Date.UTC(year, month - 1, day, hour, minute, second);
415
498
  const check = new Date(epoch);
416
499
  if (check.getUTCMonth() !== month - 1 || check.getUTCDate() !== day) return void 0;
417
- return epoch;
500
+ const format = DATE_FORMATS[index];
501
+ if (format === void 0) return void 0;
502
+ return {
503
+ epoch,
504
+ format
505
+ };
418
506
  }
419
507
  }
508
+ /**
509
+ * Deterministic date parse to epoch milliseconds. Accepts `YYYY-MM-DD`,
510
+ * `YYYY/MM/DD`, and ISO-like datetimes (date-only forms read as UTC midnight).
511
+ * Calendar-invalid dates (e.g. 2025-13-40) reject. Returns `undefined` when
512
+ * the cell is not a recognized date.
513
+ * @param cell - the cell to parse (`undefined` when the column is absent).
514
+ * @returns epoch milliseconds, or `undefined`.
515
+ */
516
+ function parseDate(cell) {
517
+ return parseDateCellInternal(cell)?.epoch;
518
+ }
519
+ /**
520
+ * Parse one date cell to its epoch plus source format label.
521
+ * @param cell - the cell to parse (`undefined` when the column is absent).
522
+ * @returns the parsed date, or `undefined` when the cell is not a recognized date.
523
+ */
524
+ function parseDateCell(cell) {
525
+ return parseDateCellInternal(cell);
526
+ }
527
+ /**
528
+ * The format label of a recognized date cell (`iso-date` / `slash-date` /
529
+ * `datetime`); `undefined` when the cell is not a recognized date. Used to
530
+ * measure a date column's format consistency.
531
+ * @param cell - the cell to inspect (`undefined` when the column is absent).
532
+ * @returns the source format label, or `undefined`.
533
+ */
534
+ function dateFormatOf(cell) {
535
+ return parseDateCellInternal(cell)?.format;
536
+ }
420
537
  /** Boolean parse: true/false/yes/no/1/0, case-insensitive. */
421
538
  function parseBoolean(cell) {
422
539
  if (typeof cell === "boolean") return cell;
@@ -440,7 +557,8 @@ const reportRecordSchema = z$1.object({
440
557
  "profile",
441
558
  "clean",
442
559
  "verify",
443
- "citations"
560
+ "citations",
561
+ "clean-diff"
444
562
  ]),
445
563
  at: z$1.number().int().nonnegative(),
446
564
  dataset: z$1.string(),
@@ -452,6 +570,18 @@ const dataQualityDomainSpec = defineDomain({
452
570
  version: 1,
453
571
  tables: { reports: domainTable(reportRecordSchema) }
454
572
  });
573
+ /** Well-formed report-key shape: `<17-digit timestamp>-<kind>-<8-hex fingerprint>`. */
574
+ const REPORT_KEY_PATTERN = /^\d{17}-(?:profile|clean|verify|citations|clean-diff)-[0-9a-f]{8}$/u;
575
+ /**
576
+ * Whether `key` is a well-formed, path-safe storage report key. Rejects any
577
+ * key with separators, traversal, or unexpected characters before it can be
578
+ * handed to the storage backend.
579
+ * @param key - candidate report key.
580
+ * @returns whether the key matches the deterministic report-key format.
581
+ */
582
+ function isValidReportKey(key) {
583
+ return REPORT_KEY_PATTERN.test(key);
584
+ }
455
585
  /** Pad to two digits for the key timestamp. */
456
586
  function pad2(value) {
457
587
  return String(value).padStart(2, "0");
@@ -842,6 +972,215 @@ function serializeDelimited(columns, rows, delimiter) {
842
972
  return `${lines.join("\n")}\n`;
843
973
  }
844
974
  //#endregion
975
+ //#region src/scorecard.ts
976
+ /**
977
+ * DAMA-style six-dimension quality scorecard over a parsed {@link Table}.
978
+ * Pure and deterministic: every rate derives from one full-table pass, the
979
+ * only clock is the injected `now`, and no dimension fabricates a score it
980
+ * cannot defend — `accuracy` stays `null` (undetermined) without a declared
981
+ * schema and `timeliness` stays `null` without date cells.
982
+ * @module dsh-data-quality/scorecard
983
+ */
984
+ /** Round to 6 significant digits (mirrors the profile report's rounding). */
985
+ function round6$1(value) {
986
+ return Number(value.toPrecision(6));
987
+ }
988
+ /** Increment a format-label counter. */
989
+ function bump(formats, tag) {
990
+ formats.set(tag, (formats.get(tag) ?? 0) + 1);
991
+ }
992
+ /** Inferred column type from the full-table class counts (mirrors `profileColumn`). */
993
+ function inferType(stats) {
994
+ if (stats.present === 0) return "empty";
995
+ if (stats.number === stats.present) return "number";
996
+ if (stats.date === stats.present) return "date";
997
+ if (stats.boolean === stats.present) return "boolean";
998
+ if (stats.string === stats.present) return "string";
999
+ return "mixed";
1000
+ }
1001
+ /** Format the `value / total` ratio as a plain `note` fragment. */
1002
+ function ratioNote(numerator, denominator) {
1003
+ return `${numerator}/${denominator}`;
1004
+ }
1005
+ /** Build one determinable dimension with its ratio note. */
1006
+ function dimension(name, score, numerator, denominator) {
1007
+ return {
1008
+ name,
1009
+ score,
1010
+ note: ratioNote(numerator, denominator)
1011
+ };
1012
+ }
1013
+ /**
1014
+ * Compute the six-dimension scorecard over a full table:
1015
+ * - `completeness` — non-empty cell rate.
1016
+ * - `uniqueness` — unique full-row content rate (`1 - duplicateRows / rowCount`).
1017
+ * - `validity` — present cells conforming to their column's inferred type.
1018
+ * - `consistency` — present cells in their column's dominant format (a date
1019
+ * column's unified `YYYY-MM-DD` vs `YYYY/MM/DD` vs datetime ratio).
1020
+ * - `timeliness` — date cells not future-dated relative to the injected `now`.
1021
+ * - `accuracy` — declared-schema agreement (limited definition); `null`
1022
+ * (undetermined) without a declared schema — never fabricated.
1023
+ * @param table - the parsed dataset (full table, not the sampled cards).
1024
+ * @param options - injected clock, duplicate-row count, optional declared schema, optional weights, abort signal.
1025
+ * @returns the scorecard.
1026
+ */
1027
+ function computeScorecard(table, options) {
1028
+ throwIfAborted(options.signal);
1029
+ const stats = /* @__PURE__ */ new Map();
1030
+ for (const column of table.columns) stats.set(column, {
1031
+ present: 0,
1032
+ number: 0,
1033
+ date: 0,
1034
+ boolean: 0,
1035
+ string: 0,
1036
+ formats: /* @__PURE__ */ new Map()
1037
+ });
1038
+ let missingCells = 0;
1039
+ let dateCells = 0;
1040
+ let futureDateCells = 0;
1041
+ for (const [index, row] of table.rows.entries()) {
1042
+ if (index % 1024 === 0) throwIfAborted(options.signal);
1043
+ for (const column of table.columns) {
1044
+ const cell = row[column];
1045
+ const columnStats = stats.get(column);
1046
+ if (isMissing(cell)) {
1047
+ missingCells += 1;
1048
+ continue;
1049
+ }
1050
+ columnStats.present += 1;
1051
+ if (parseNumeric(cell) !== void 0) {
1052
+ columnStats.number += 1;
1053
+ bump(columnStats.formats, "number");
1054
+ continue;
1055
+ }
1056
+ const date = parseDateCell(cell);
1057
+ if (date !== void 0) {
1058
+ columnStats.date += 1;
1059
+ bump(columnStats.formats, date.format);
1060
+ dateCells += 1;
1061
+ if (date.epoch > options.now) futureDateCells += 1;
1062
+ continue;
1063
+ }
1064
+ if (parseBoolean(cell) !== void 0) {
1065
+ columnStats.boolean += 1;
1066
+ bump(columnStats.formats, "boolean");
1067
+ continue;
1068
+ }
1069
+ columnStats.string += 1;
1070
+ bump(columnStats.formats, typeof cell === "string" ? "string" : "json");
1071
+ }
1072
+ }
1073
+ const totalCells = table.rows.length * table.columns.length;
1074
+ let presentCells = 0;
1075
+ let invalidCells = 0;
1076
+ let consistentCells = 0;
1077
+ for (const [, columnStats] of stats) {
1078
+ presentCells += columnStats.present;
1079
+ const dominantClass = Math.max(columnStats.number, columnStats.date, columnStats.boolean, columnStats.string);
1080
+ invalidCells += columnStats.present - dominantClass;
1081
+ if (columnStats.present === 0) continue;
1082
+ let dominant = 0;
1083
+ for (const count of columnStats.formats.values()) if (count > dominant) dominant = count;
1084
+ consistentCells += dominant;
1085
+ }
1086
+ const dimensions = [
1087
+ totalCells === 0 ? {
1088
+ name: "completeness",
1089
+ score: null,
1090
+ note: "undetermined: no cells"
1091
+ } : dimension("completeness", round6$1(1 - missingCells / totalCells), presentCells, totalCells),
1092
+ table.rows.length === 0 ? {
1093
+ name: "uniqueness",
1094
+ score: null,
1095
+ note: "undetermined: no rows"
1096
+ } : dimension("uniqueness", round6$1(1 - options.duplicateRows / table.rows.length), table.rows.length - options.duplicateRows, table.rows.length),
1097
+ presentCells === 0 ? {
1098
+ name: "validity",
1099
+ score: null,
1100
+ note: "undetermined: no present cells"
1101
+ } : dimension("validity", round6$1(1 - invalidCells / presentCells), presentCells - invalidCells, presentCells),
1102
+ presentCells === 0 ? {
1103
+ name: "consistency",
1104
+ score: null,
1105
+ note: "undetermined: no present cells"
1106
+ } : dimension("consistency", round6$1(consistentCells / presentCells), consistentCells, presentCells),
1107
+ dateCells === 0 ? {
1108
+ name: "timeliness",
1109
+ score: null,
1110
+ note: "undetermined: no date cells"
1111
+ } : dimension("timeliness", round6$1(1 - futureDateCells / dateCells), dateCells - futureDateCells, dateCells),
1112
+ computeAccuracy(stats, table, options.declaredSchema)
1113
+ ];
1114
+ const scores = dimensions.map((entry) => entry.score).filter((score) => score !== null);
1115
+ return {
1116
+ dimensions,
1117
+ overall: scores.length === 0 ? null : round6$1(scores.reduce((sum, score) => sum + score, 0) / scores.length),
1118
+ weightedOverall: computeWeightedOverall(dimensions, options.weights)
1119
+ };
1120
+ }
1121
+ /** Weighted mean over determinable dimensions (weights default to 1; validated non-negative by config). */
1122
+ function computeWeightedOverall(dimensions, weights) {
1123
+ let weightedSum = 0;
1124
+ let weightTotal = 0;
1125
+ for (const entry of dimensions) {
1126
+ if (entry.score === null) continue;
1127
+ const weight = weights?.[entry.name] ?? 1;
1128
+ weightedSum += weight * entry.score;
1129
+ weightTotal += weight;
1130
+ }
1131
+ return weightTotal === 0 ? null : round6$1(weightedSum / weightTotal);
1132
+ }
1133
+ /**
1134
+ * Limited `accuracy` definition: agreement between the full-table inferred
1135
+ * type of each declared column and the declared type. Without a declared
1136
+ * schema (or an external truth source) the dimension is `null` (undetermined)
1137
+ * — accuracy is never fabricated.
1138
+ */
1139
+ function computeAccuracy(stats, table, declaredSchema) {
1140
+ if (declaredSchema === void 0 || Object.keys(declaredSchema).length === 0) return {
1141
+ name: "accuracy",
1142
+ score: null,
1143
+ note: "undetermined: no declared schema or external truth to compare against; accuracy is never fabricated"
1144
+ };
1145
+ let checked = 0;
1146
+ let matched = 0;
1147
+ const mismatches = [];
1148
+ for (const column of table.columns) {
1149
+ const declared = declaredSchema[column];
1150
+ if (declared === void 0) continue;
1151
+ checked += 1;
1152
+ const inferred = inferType(stats.get(column));
1153
+ if (inferred === declared) matched += 1;
1154
+ else mismatches.push(`${column}: declared ${declared}, inferred ${inferred}`);
1155
+ }
1156
+ if (checked === 0) return {
1157
+ name: "accuracy",
1158
+ score: null,
1159
+ note: "undetermined: declared schema covers none of the dataset columns"
1160
+ };
1161
+ return {
1162
+ name: "accuracy",
1163
+ score: round6$1(matched / checked),
1164
+ note: `${matched}/${checked} columns match their declared type` + (mismatches.length > 0 ? `; mismatches: ${mismatches.join(", ")}` : "")
1165
+ };
1166
+ }
1167
+ //#endregion
1168
+ //#region src/version.ts
1169
+ /**
1170
+ * Plugin version, kept in one place so `scripts/release.mjs` can stamp it and
1171
+ * reports can name their generator.
1172
+ * @module dsh-data-quality/version
1173
+ */
1174
+ /** The package version reported in persisted reports. */
1175
+ const VERSION = "0.3.0";
1176
+ /**
1177
+ * Version of the persisted report schema. Bump it whenever a report's
1178
+ * canonical shape changes in a way old consumers cannot read (the durable
1179
+ * `data_quality` records keep their own `schemaVersion` so a future reader
1180
+ * can detect and reject an incompatible record instead of misreading it).
1181
+ */
1182
+ const REPORT_SCHEMA_VERSION = 1;
1183
+ //#endregion
845
1184
  //#region src/profile.ts
846
1185
  /**
847
1186
  * Deterministic dataset profiling: per-column type inference, missingness,
@@ -850,6 +1189,8 @@ function serializeDelimited(columns, rows, delimiter) {
850
1189
  * no I/O; `generatedAt` is injected by the caller.
851
1190
  * @module dsh-data-quality/profile
852
1191
  */
1192
+ /** Fallback duplicate-sample cap for direct engine use; the provider always passes the configured `evidenceRowLimit`. */
1193
+ const DEFAULT_DUPLICATE_SAMPLE_LIMIT = 20;
853
1194
  /** Round to 6 significant digits for stable, readable report numbers. */
854
1195
  function round6(value) {
855
1196
  return Number(value.toPrecision(6));
@@ -874,6 +1215,8 @@ function numericProfile(values) {
874
1215
  const highFence = p75 + 1.5 * iqr;
875
1216
  const outliers = iqr === 0 ? 0 : sorted.filter((value) => value < lowFence || value > highFence).length;
876
1217
  return {
1218
+ count: sorted.length,
1219
+ distinct: new Set(sorted).size,
877
1220
  min: round6(sorted[0]),
878
1221
  max: round6(sorted[sorted.length - 1]),
879
1222
  mean: round6(sum / sorted.length),
@@ -883,17 +1226,43 @@ function numericProfile(values) {
883
1226
  outliers
884
1227
  };
885
1228
  }
886
- /** Count rows whose full content duplicates an earlier row (first occurrence is not counted). */
887
- function countDuplicateRows(table, signal) {
1229
+ /** Deterministic sha256 key of one row's full content (columns in table order). */
1230
+ function rowContentKey(table, row) {
1231
+ return createHash("sha256").update(JSON.stringify(table.columns.map((column) => row[column] ?? null))).digest("hex");
1232
+ }
1233
+ /**
1234
+ * Detect full-content duplicate rows with a bounded sample of their 0-based
1235
+ * indexes. The first occurrence of each content is never counted; later rows
1236
+ * with identical full content are duplicates.
1237
+ * @param table - the parsed dataset.
1238
+ * @param options - sample cap and optional abort signal.
1239
+ * @returns the duplicate count, rate, and capped sample indexes.
1240
+ */
1241
+ function detectDuplicateRows(table, options) {
1242
+ if (!Number.isSafeInteger(options.sampleLimit) || options.sampleLimit <= 0) throw new TypeError(`sampleLimit must be a positive safe integer, got ${String(options.sampleLimit)}`);
888
1243
  const seen = /* @__PURE__ */ new Set();
889
1244
  let duplicates = 0;
1245
+ const duplicateSampleRowIndexes = [];
890
1246
  for (const [index, row] of table.rows.entries()) {
891
- if (index % 1024 === 0) throwIfAborted(signal);
892
- const key = JSON.stringify(table.columns.map((column) => row[column] ?? null));
893
- if (seen.has(key)) duplicates += 1;
894
- else seen.add(key);
1247
+ if (index % 1024 === 0) throwIfAborted(options.signal);
1248
+ const key = rowContentKey(table, row);
1249
+ if (seen.has(key)) {
1250
+ duplicates += 1;
1251
+ if (duplicateSampleRowIndexes.length < options.sampleLimit) duplicateSampleRowIndexes.push(index);
1252
+ } else seen.add(key);
895
1253
  }
896
- return duplicates;
1254
+ return {
1255
+ duplicateRows: duplicates,
1256
+ duplicateRate: table.rows.length === 0 ? 0 : round6(duplicates / table.rows.length),
1257
+ duplicateSampleRowIndexes
1258
+ };
1259
+ }
1260
+ /** Count rows whose full content duplicates an earlier row (first occurrence is not counted). */
1261
+ function countDuplicateRows(table, signal) {
1262
+ return detectDuplicateRows(table, {
1263
+ sampleLimit: 1,
1264
+ signal
1265
+ }).duplicateRows;
897
1266
  }
898
1267
  /** Profile one column over the given rows. */
899
1268
  function profileColumn(rows, column, signal) {
@@ -980,14 +1349,29 @@ function profileTable(table, options) {
980
1349
  throwIfAborted(options.signal);
981
1350
  const profiled = options.sample === void 0 ? table.rows : sampleRows(table.rows, options.sample);
982
1351
  const columns = table.columns.map((column) => profileColumn(profiled, column, options.signal));
983
- const duplicateRows = countDuplicateRows(table, options.signal);
1352
+ const detection = detectDuplicateRows(table, {
1353
+ sampleLimit: options.duplicateSampleLimit ?? DEFAULT_DUPLICATE_SAMPLE_LIMIT,
1354
+ signal: options.signal
1355
+ });
1356
+ const scorecard = computeScorecard(table, {
1357
+ now: options.generatedAt,
1358
+ duplicateRows: detection.duplicateRows,
1359
+ declaredSchema: options.declaredSchema,
1360
+ weights: options.scorecardWeights,
1361
+ signal: options.signal
1362
+ });
984
1363
  return {
1364
+ schemaVersion: 1,
985
1365
  dataset: options.dataset,
986
1366
  rowCount: table.rows.length,
987
1367
  sampled: profiled.length !== table.rows.length,
988
1368
  profiledRows: profiled.length,
989
1369
  columnCount: table.columns.length,
990
- duplicateRows,
1370
+ duplicateRows: detection.duplicateRows,
1371
+ duplicateRate: detection.duplicateRate,
1372
+ duplicateSampleRowIndexes: detection.duplicateSampleRowIndexes,
1373
+ scorecard,
1374
+ ...table.encoding !== void 0 ? { encoding: table.encoding } : {},
991
1375
  columns,
992
1376
  generatedAt: options.generatedAt
993
1377
  };
@@ -996,19 +1380,167 @@ function profileTable(table, options) {
996
1380
  function renderProfileText(report) {
997
1381
  const lines = [];
998
1382
  lines.push(`Profile of ${report.dataset}: ${report.rowCount} rows x ${report.columnCount} columns` + (report.sampled ? ` (column cards over a systematic sample of ${report.profiledRows} rows)` : ""));
999
- if (report.duplicateRows > 0) lines.push(`Duplicate rows: ${report.duplicateRows}`);
1383
+ if (report.duplicateRows > 0) lines.push(`Duplicate rows: ${report.duplicateRows} (${(report.duplicateRate * 100).toFixed(1)}%)` + (report.duplicateSampleRowIndexes.length > 0 ? `; sample row indexes: ${report.duplicateSampleRowIndexes.join(", ")}` : ""));
1000
1384
  for (const column of report.columns) {
1001
1385
  const parts = [`${column.name}: ${column.inferredType}`];
1002
1386
  if (column.missing > 0) parts.push(`missing ${column.missing} (${(column.missingRate * 100).toFixed(1)}%)`);
1003
1387
  parts.push(`unique ${column.unique}`);
1004
- if (column.numeric !== void 0) parts.push(`min ${column.numeric.min}, p25 ${column.numeric.p25}, median ${column.numeric.median}, p75 ${column.numeric.p75}, max ${column.numeric.max}, mean ${column.numeric.mean}` + (column.numeric.outliers > 0 ? `, ${column.numeric.outliers} IQR outliers` : ""));
1388
+ if (column.numeric !== void 0) parts.push(`count ${column.numeric.count}, distinct ${column.numeric.distinct}, min ${column.numeric.min}, p25 ${column.numeric.p25}, median ${column.numeric.median}, p75 ${column.numeric.p75}, max ${column.numeric.max}, mean ${column.numeric.mean}` + (column.numeric.outliers > 0 ? `, ${column.numeric.outliers} IQR outliers` : ""));
1005
1389
  if (column.topValues !== void 0) parts.push(`top: ${column.topValues.map((entry) => `${JSON.stringify(entry.value)} x${entry.count}`).join(", ")}`);
1006
1390
  for (const note of column.notes) parts.push(`note: ${note}`);
1007
1391
  lines.push(`- ${parts.join("; ")}`);
1008
1392
  }
1393
+ if (report.encoding !== void 0) lines.push(`Encoding: UTF-8${report.encoding.bom === "utf-8" ? " (BOM)" : ""}${report.encoding.validUtf8 ? "" : " (INVALID UTF-8)"}`);
1394
+ const overall = report.scorecard.overall;
1395
+ const weighted = report.scorecard.weightedOverall;
1396
+ lines.push(`Scorecard (overall ${overall === null ? "undetermined" : `${(overall * 100).toFixed(1)}%`}, weighted ${weighted === null ? "undetermined" : `${(weighted * 100).toFixed(1)}%`}):`);
1397
+ for (const dimension of report.scorecard.dimensions) {
1398
+ const value = dimension.score === null ? "undetermined" : `${(dimension.score * 100).toFixed(1)}%`;
1399
+ lines.push(` ${dimension.name}: ${value} (${dimension.note})`);
1400
+ }
1009
1401
  return lines.join("\n");
1010
1402
  }
1011
1403
  //#endregion
1404
+ //#region src/contract.ts
1405
+ /**
1406
+ * Delivery contract for `data_clean`: a deterministic pre-delivery validation
1407
+ * summary (primary-key/type/dedupe row-count comparison plus uniqueness and
1408
+ * non-null regression) and the clean before/after profile diff report. Pure —
1409
+ * no I/O, no clock, no RNG; timestamps are injected.
1410
+ * @module dsh-data-quality/contract
1411
+ */
1412
+ /** Count missing cells in one column of the cleaned output. */
1413
+ function countMissing(rows, column, signal) {
1414
+ let count = 0;
1415
+ for (const [index, row] of rows.entries()) {
1416
+ if (index % 1024 === 0) throwIfAborted(signal);
1417
+ if (isMissing(row[column])) count += 1;
1418
+ }
1419
+ return count;
1420
+ }
1421
+ /** Count present cells in one column that still fail to parse as the requested type. */
1422
+ function countNonConforming(rows, column, to, signal) {
1423
+ let count = 0;
1424
+ for (const [index, row] of rows.entries()) {
1425
+ if (index % 1024 === 0) throwIfAborted(signal);
1426
+ const cell = row[column];
1427
+ if (isMissing(cell)) continue;
1428
+ if (!(to === "number" ? parseNumeric(cell) !== void 0 : to === "date" ? parseDate(cell) !== void 0 : parseBoolean(cell) !== void 0)) count += 1;
1429
+ }
1430
+ return count;
1431
+ }
1432
+ /**
1433
+ * Compute the pre-delivery contract summary from a cleaning run: dedupe
1434
+ * before/after row counts, the dedupe key's uniqueness regression, and the
1435
+ * non-null/type regressions over the columns the rules targeted.
1436
+ * @param result - the cleaning outcome (input/output rows plus cleaned rows).
1437
+ * @param rules - the ordered rules that produced `result`.
1438
+ * @param options - optional abort signal.
1439
+ * @returns the contract summary.
1440
+ */
1441
+ function computeCleanContract(result, rules, options = {}) {
1442
+ const dedupeRule = rules.find((rule) => rule.rule === "dedupe");
1443
+ const dedupeColumns = dedupeRule === void 0 ? null : [...dedupeRule.columns ?? result.columns];
1444
+ const keyColumns = dedupeColumns ?? result.columns;
1445
+ const seen = /* @__PURE__ */ new Set();
1446
+ let remainingDuplicateRows = 0;
1447
+ for (const [index, row] of result.rows.entries()) {
1448
+ if (index % 1024 === 0) throwIfAborted(options.signal);
1449
+ const key = JSON.stringify(keyColumns.map((column) => row[column] ?? null));
1450
+ if (seen.has(key)) remainingDuplicateRows += 1;
1451
+ else seen.add(key);
1452
+ }
1453
+ const remainingMissing = rules.filter((rule) => rule.rule === "fill-missing").map((rule) => {
1454
+ const fill = rule;
1455
+ return {
1456
+ column: fill.column,
1457
+ count: countMissing(result.rows, fill.column, options.signal)
1458
+ };
1459
+ });
1460
+ const typeConformance = rules.filter((rule) => rule.rule === "coerce-type").map((rule) => {
1461
+ const coerce = rule;
1462
+ return {
1463
+ column: coerce.column,
1464
+ to: coerce.to,
1465
+ invalidCount: countNonConforming(result.rows, coerce.column, coerce.to, options.signal)
1466
+ };
1467
+ });
1468
+ const columnDecisions = traceColumnDecisions(result, rules);
1469
+ return {
1470
+ inputRows: result.inputRows,
1471
+ outputRows: result.outputRows,
1472
+ removedRows: result.inputRows - result.outputRows,
1473
+ dedupeColumns,
1474
+ uniqueKeys: remainingDuplicateRows === 0,
1475
+ remainingDuplicateRows,
1476
+ remainingMissing,
1477
+ typeConformance,
1478
+ columnDecisions
1479
+ };
1480
+ }
1481
+ /** The columns a single rule targets (single-column rules target one column; trim/dedupe may target many). */
1482
+ function ruleColumns(rule, allColumns) {
1483
+ switch (rule.rule) {
1484
+ case "fill-missing":
1485
+ case "coerce-type":
1486
+ case "normalize-unit":
1487
+ case "map-values": return [rule.column];
1488
+ case "trim":
1489
+ case "dedupe": return [...rule.columns ?? allColumns];
1490
+ default: return [];
1491
+ }
1492
+ }
1493
+ /** Build the per-column decision trace: strategies + affected rows, in dataset column order. */
1494
+ function traceColumnDecisions(result, rules) {
1495
+ const byColumn = /* @__PURE__ */ new Map();
1496
+ for (const [index, rule] of rules.entries()) {
1497
+ const log = result.logs[index];
1498
+ if (log === void 0) continue;
1499
+ for (const column of ruleColumns(rule, result.columns)) {
1500
+ const list = byColumn.get(column) ?? [];
1501
+ list.push({
1502
+ strategy: rule.rule,
1503
+ affectedRows: log.affectedRows
1504
+ });
1505
+ byColumn.set(column, list);
1506
+ }
1507
+ }
1508
+ return result.columns.filter((column) => byColumn.has(column)).map((column) => ({
1509
+ column,
1510
+ decisions: byColumn.get(column)
1511
+ }));
1512
+ }
1513
+ /**
1514
+ * Build the clean before/after profile diff: two full {@link ProfileReport}
1515
+ * snapshots (input and cleaned output) under one envelope. Reuses the profile
1516
+ * engine so the scorecard and duplicate detection stay consistent across the
1517
+ * whole plugin.
1518
+ * @param input - the input dataset.
1519
+ * @param output - the cleaned dataset.
1520
+ * @param options - dataset label, injected timestamp, optional weights, optional abort signal.
1521
+ * @returns the diff report.
1522
+ */
1523
+ function computeCleanProfileDiff(input, output, options) {
1524
+ const before = profileTable(input, {
1525
+ dataset: options.dataset,
1526
+ generatedAt: options.generatedAt,
1527
+ scorecardWeights: options.scorecardWeights,
1528
+ signal: options.signal
1529
+ });
1530
+ const after = profileTable(output, {
1531
+ dataset: options.dataset,
1532
+ generatedAt: options.generatedAt,
1533
+ scorecardWeights: options.scorecardWeights,
1534
+ signal: options.signal
1535
+ });
1536
+ return {
1537
+ dataset: options.dataset,
1538
+ before,
1539
+ after,
1540
+ generatedAt: options.generatedAt
1541
+ };
1542
+ }
1543
+ //#endregion
1012
1544
  //#region src/verify.ts
1013
1545
  /**
1014
1546
  * Declarative verification rules over a parsed {@link Table} (not-null,
@@ -1031,6 +1563,19 @@ var VerifyRuleError = class extends Error {
1031
1563
  this.name = "VerifyRuleError";
1032
1564
  }
1033
1565
  };
1566
+ /** Raised for invalid expectations; `message` names the expectation index and reason. */
1567
+ var VerifyExpectationError = class extends Error {
1568
+ expectationIndex;
1569
+ /**
1570
+ * @param expectationIndex - index of the offending expectation in the request array.
1571
+ * @param message - actionable human-readable detail.
1572
+ */
1573
+ constructor(expectationIndex, message) {
1574
+ super(message);
1575
+ this.expectationIndex = expectationIndex;
1576
+ this.name = "VerifyExpectationError";
1577
+ }
1578
+ };
1034
1579
  /** Assert `column` exists. */
1035
1580
  function requireColumn(columns, column, ruleIndex) {
1036
1581
  if (!columns.includes(column)) throw new VerifyRuleError(ruleIndex, `rule ${ruleIndex}: unknown column ${JSON.stringify(column)} (columns: ${columns.join(", ")})`);
@@ -1076,13 +1621,90 @@ const CROSS_OPS = [
1076
1621
  ">=",
1077
1622
  ">"
1078
1623
  ];
1624
+ /** The metric ids, for validation and diagnostics. */
1625
+ const VERIFY_METRICS = [
1626
+ "rowCount",
1627
+ "columnSum",
1628
+ "columnMean",
1629
+ "uniqueCount",
1630
+ "nullCount"
1631
+ ];
1632
+ /** Throw unless the expectation is well-formed (metric/column/tolerance). */
1633
+ function validateExpectation(table, expectation, index) {
1634
+ if (!VERIFY_METRICS.includes(expectation.metric)) throw new VerifyExpectationError(index, `expectation ${index}: unknown metric ${JSON.stringify(expectation.metric)} (expected one of ${VERIFY_METRICS.join(", ")})`);
1635
+ if (expectation.metric !== "rowCount") {
1636
+ if (expectation.column === void 0 || expectation.column === "") throw new VerifyExpectationError(index, `expectation ${index}: metric ${expectation.metric} requires a column`);
1637
+ if (!table.columns.includes(expectation.column)) throw new VerifyExpectationError(index, `expectation ${index}: unknown column ${JSON.stringify(expectation.column)} (columns: ${table.columns.join(", ")})`);
1638
+ } else if (expectation.column !== void 0) throw new VerifyExpectationError(index, `expectation ${index}: metric rowCount takes no column, got ${JSON.stringify(expectation.column)}`);
1639
+ if (expectation.tolerance !== void 0 && (typeof expectation.tolerance !== "number" || !Number.isFinite(expectation.tolerance) || expectation.tolerance < 0 || expectation.tolerance > 1)) throw new VerifyExpectationError(index, `expectation ${index}: tolerance must be a finite number in [0, 1], got ${String(expectation.tolerance)}`);
1640
+ }
1641
+ /** Compute the deterministic actual value of one expectation's metric. */
1642
+ function metricValueOf(table, expectation) {
1643
+ switch (expectation.metric) {
1644
+ case "rowCount": return table.rows.length;
1645
+ case "nullCount": {
1646
+ let count = 0;
1647
+ for (const row of table.rows) if (isMissing(row[expectation.column])) count += 1;
1648
+ return count;
1649
+ }
1650
+ case "uniqueCount": {
1651
+ const distinct = /* @__PURE__ */ new Set();
1652
+ for (const row of table.rows) {
1653
+ const cell = row[expectation.column];
1654
+ if (isMissing(cell)) continue;
1655
+ distinct.add(typeof cell === "string" ? cell : JSON.stringify(cell));
1656
+ }
1657
+ return distinct.size;
1658
+ }
1659
+ case "columnSum":
1660
+ case "columnMean": {
1661
+ let sum = 0;
1662
+ let count = 0;
1663
+ for (const row of table.rows) {
1664
+ const value = parseNumeric(row[expectation.column]);
1665
+ if (value === void 0) continue;
1666
+ sum += value;
1667
+ count += 1;
1668
+ }
1669
+ return expectation.metric === "columnSum" ? sum : count === 0 ? 0 : sum / count;
1670
+ }
1671
+ }
1672
+ }
1673
+ /**
1674
+ * Reconcile each expectation against its deterministic actual value. A
1675
+ * mismatch is a normal `passed: false` result, never a thrown error; invalid
1676
+ * metrics, columns, and tolerances fail loud.
1677
+ * @param table - the parsed dataset.
1678
+ * @param expectations - the expectations to reconcile.
1679
+ * @param defaultTolerance - configured fallback relative tolerance.
1680
+ * @param signal - optional abort signal.
1681
+ * @returns one outcome per expectation.
1682
+ */
1683
+ function verifyExpectations(table, expectations, defaultTolerance, signal) {
1684
+ return expectations.map((expectation, index) => {
1685
+ throwIfAborted(signal);
1686
+ validateExpectation(table, expectation, index);
1687
+ const actual = metricValueOf(table, expectation);
1688
+ const tolerance = expectation.tolerance ?? defaultTolerance;
1689
+ const passed = numericClose(actual, expectation.expected, tolerance);
1690
+ return {
1691
+ metric: expectation.metric,
1692
+ ...expectation.column !== void 0 ? { column: expectation.column } : {},
1693
+ expected: expectation.expected,
1694
+ actual,
1695
+ tolerance,
1696
+ passed
1697
+ };
1698
+ });
1699
+ }
1079
1700
  /**
1080
1701
  * Apply verification rules over a parsed table. A missing cell fails every
1081
- * rule that reads it. The overall `passed` is the conjunction of rule passes;
1082
- * a failing dataset is a normal result, never a thrown error.
1702
+ * rule that reads it. The overall `passed` is the conjunction of rule passes
1703
+ * and expectation passes; a failing dataset is a normal result, never a
1704
+ * thrown error.
1083
1705
  * @param table - the parsed dataset.
1084
1706
  * @param rules - non-empty rule list.
1085
- * @param options - evidence cap, injected clock for `freshness`, abort signal.
1707
+ * @param options - evidence cap, injected clock for `freshness`, optional expectations, default tolerance, abort signal.
1086
1708
  * @returns the verify report (without the dataset label; the caller adds it).
1087
1709
  */
1088
1710
  function verifyTable(table, rules, options) {
@@ -1205,10 +1827,12 @@ function verifyTable(table, rules, options) {
1205
1827
  }
1206
1828
  results.push(result);
1207
1829
  }
1830
+ const expectations = verifyExpectations(table, options.expectations ?? [], options.defaultTolerance ?? 1e-9, options.signal);
1208
1831
  return {
1209
- passed: results.every((result) => result.passed),
1832
+ passed: results.every((result) => result.passed) && expectations.every((expectation) => expectation.passed),
1210
1833
  rowCount: table.rows.length,
1211
1834
  rules: results,
1835
+ expectations,
1212
1836
  generatedAt: options.now()
1213
1837
  };
1214
1838
  }
@@ -1224,6 +1848,10 @@ function renderVerifyText(report) {
1224
1848
  }
1225
1849
  if (rule.failedCount > rule.evidence.length) lines.push(` … and ${rule.failedCount - rule.evidence.length} more failing row(s)`);
1226
1850
  }
1851
+ for (const expectation of report.expectations) {
1852
+ const target = expectation.column !== void 0 ? `${expectation.metric}(${expectation.column})` : expectation.metric;
1853
+ lines.push(`- [${expectation.passed ? "pass" : "FAIL"}] expectation ${target}: actual ${expectation.actual} vs expected ${expectation.expected} (tolerance ${expectation.tolerance})`);
1854
+ }
1227
1855
  return lines.join("\n");
1228
1856
  }
1229
1857
  /**
@@ -1409,6 +2037,131 @@ function appendDataQualityEvent(session, type, data) {
1409
2037
  if (Function.prototype.toString.call(append).includes("ignorable")) append.call(session, type, data, { ignorable: true });
1410
2038
  }
1411
2039
  //#endregion
2040
+ //#region src/presets.ts
2041
+ /** The built-in industry preset registry. */
2042
+ const INDUSTRY_PRESETS = {
2043
+ retail: {
2044
+ id: "retail",
2045
+ label: "Retail",
2046
+ columns: {
2047
+ order_id: "string",
2048
+ sku: "string",
2049
+ quantity: "number",
2050
+ unit_price: "number",
2051
+ revenue: "number",
2052
+ order_date: "date",
2053
+ customer_id: "string"
2054
+ }
2055
+ },
2056
+ saas: {
2057
+ id: "saas",
2058
+ label: "SaaS",
2059
+ columns: {
2060
+ account_id: "string",
2061
+ plan: "string",
2062
+ seats: "number",
2063
+ mrr: "number",
2064
+ signup_date: "date",
2065
+ churned: "boolean"
2066
+ }
2067
+ },
2068
+ fund: {
2069
+ id: "fund",
2070
+ label: "Fund",
2071
+ columns: {
2072
+ fund_code: "string",
2073
+ fund_name: "string",
2074
+ nav: "number",
2075
+ nav_date: "date",
2076
+ holding_value: "number",
2077
+ currency: "string"
2078
+ }
2079
+ },
2080
+ "real-estate": {
2081
+ id: "real-estate",
2082
+ label: "Real estate",
2083
+ columns: {
2084
+ property_id: "string",
2085
+ listing_price: "number",
2086
+ area_sqm: "number",
2087
+ bedrooms: "number",
2088
+ list_date: "date",
2089
+ city: "string"
2090
+ }
2091
+ },
2092
+ "e-commerce": {
2093
+ id: "e-commerce",
2094
+ label: "E-commerce",
2095
+ columns: {
2096
+ order_id: "string",
2097
+ product_id: "string",
2098
+ quantity: "number",
2099
+ price: "number",
2100
+ order_date: "date",
2101
+ status: "string"
2102
+ }
2103
+ },
2104
+ healthcare: {
2105
+ id: "healthcare",
2106
+ label: "Healthcare",
2107
+ columns: {
2108
+ patient_id: "string",
2109
+ admission_date: "date",
2110
+ discharge_date: "date",
2111
+ age: "number",
2112
+ diagnosis: "string",
2113
+ cost: "number"
2114
+ }
2115
+ },
2116
+ logistics: {
2117
+ id: "logistics",
2118
+ label: "Logistics",
2119
+ columns: {
2120
+ shipment_id: "string",
2121
+ origin: "string",
2122
+ destination: "string",
2123
+ weight_kg: "number",
2124
+ shipped_date: "date",
2125
+ delivered: "boolean"
2126
+ }
2127
+ },
2128
+ manufacturing: {
2129
+ id: "manufacturing",
2130
+ label: "Manufacturing",
2131
+ columns: {
2132
+ part_id: "string",
2133
+ quantity: "number",
2134
+ unit_cost: "number",
2135
+ produced_date: "date",
2136
+ defect: "boolean"
2137
+ }
2138
+ },
2139
+ energy: {
2140
+ id: "energy",
2141
+ label: "Energy",
2142
+ columns: {
2143
+ meter_id: "string",
2144
+ reading: "number",
2145
+ unit: "string",
2146
+ reading_date: "date",
2147
+ consumption: "number"
2148
+ }
2149
+ }
2150
+ };
2151
+ /** The preset ids, for diagnostics and documentation. */
2152
+ const INDUSTRY_PRESET_IDS = Object.keys(INDUSTRY_PRESETS);
2153
+ /**
2154
+ * Resolve an industry preset id to its registry entry, failing loud on an
2155
+ * unknown id.
2156
+ * @param id - the requested preset id.
2157
+ * @returns the preset.
2158
+ */
2159
+ function resolveIndustryPreset(id) {
2160
+ const preset = INDUSTRY_PRESETS[id];
2161
+ if (preset === void 0) throw new Error(`unknown industryPreset ${JSON.stringify(id)}; choose one of: ${INDUSTRY_PRESET_IDS.join(", ")}`);
2162
+ return preset;
2163
+ }
2164
+ //#endregion
1412
2165
  //#region src/present.ts
1413
2166
  /** Maximum characters one cell contributes to a tool-facing row payload. */
1414
2167
  const MAX_CELL_TEXT = 120;
@@ -1514,11 +2267,16 @@ var LocalDataQualityService = class extends DataQualityService {
1514
2267
  /** @inheritdoc DataQualityService.profileDataset */
1515
2268
  async profileDataset(request) {
1516
2269
  throwIfAborted(request.signal);
1517
- const report = profileTable(await loadTable(resolveWorkspacePath(request.workspace, request.dataset, this.config), this.config, request.signal), {
2270
+ const table = await loadTable(resolveWorkspacePath(request.workspace, request.dataset, this.config), this.config, request.signal);
2271
+ const declaredSchema = request.industryPreset === void 0 ? void 0 : resolveIndustryPreset(request.industryPreset).columns;
2272
+ const report = profileTable(table, {
1518
2273
  dataset: request.dataset,
1519
2274
  sample: request.sample,
1520
2275
  generatedAt: this.deps.now(),
1521
- signal: request.signal
2276
+ signal: request.signal,
2277
+ duplicateSampleLimit: this.config.evidenceRowLimit,
2278
+ declaredSchema,
2279
+ scorecardWeights: this.config.scorecardWeights
1522
2280
  });
1523
2281
  const reportKey = await this.persist("profile", request.dataset, report);
1524
2282
  this.emitEvent(request.session, "profile", request.dataset, reportKey, {
@@ -1533,10 +2291,12 @@ var LocalDataQualityService = class extends DataQualityService {
1533
2291
  /** @inheritdoc DataQualityService.cleanDataset */
1534
2292
  async cleanDataset(request) {
1535
2293
  throwIfAborted(request.signal);
2294
+ const dryRun = request.dryRun === true;
1536
2295
  const absolute = resolveWorkspacePath(request.workspace, request.dataset, this.config);
1537
- const result = applyCleanRules(await loadTable(absolute, this.config, request.signal), request.rules, { signal: request.signal });
2296
+ const table = await loadTable(absolute, this.config, request.signal);
2297
+ const result = applyCleanRules(table, request.rules, { signal: request.signal });
1538
2298
  let writtenPath;
1539
- if (request.outputPath !== void 0) {
2299
+ if (!dryRun && request.outputPath !== void 0) {
1540
2300
  const outputAbsolute = resolveWorkspacePath(request.workspace, request.outputPath, this.config);
1541
2301
  if (outputAbsolute === absolute) throw new Error(`outputPath ${JSON.stringify(request.outputPath)} would overwrite the input dataset; choose a different path`);
1542
2302
  const ext = path.extname(outputAbsolute).toLowerCase();
@@ -1547,19 +2307,51 @@ var LocalDataQualityService = class extends DataQualityService {
1547
2307
  writtenPath = request.outputPath;
1548
2308
  }
1549
2309
  const generatedAt = this.deps.now();
2310
+ const contract = computeCleanContract(result, request.rules, { signal: request.signal });
2311
+ const diff = computeCleanProfileDiff(table, {
2312
+ columns: result.columns,
2313
+ rows: result.rows,
2314
+ ...table.encoding !== void 0 ? { encoding: table.encoding } : {}
2315
+ }, {
2316
+ dataset: request.dataset,
2317
+ generatedAt,
2318
+ scorecardWeights: this.config.scorecardWeights,
2319
+ signal: request.signal
2320
+ });
1550
2321
  const preview = {
1551
2322
  columns: result.columns,
1552
2323
  rows: result.rows.slice(0, this.config.evidenceRowLimit).map((row) => truncateRow(row))
1553
2324
  };
2325
+ if (dryRun) {
2326
+ this.emitEvent(request.session, "clean", request.dataset, void 0, {
2327
+ rows: result.outputRows,
2328
+ columns: result.columns.length,
2329
+ rules: result.logs.length
2330
+ });
2331
+ return {
2332
+ dataset: request.dataset,
2333
+ inputRows: result.inputRows,
2334
+ outputRows: result.outputRows,
2335
+ dryRun: true,
2336
+ logs: result.logs,
2337
+ contract,
2338
+ preview,
2339
+ diffPreview: diff,
2340
+ generatedAt
2341
+ };
2342
+ }
1554
2343
  const reportKey = await this.persist("clean", request.dataset, {
1555
2344
  dataset: request.dataset,
1556
2345
  inputRows: result.inputRows,
1557
2346
  outputRows: result.outputRows,
2347
+ dryRun: false,
1558
2348
  logs: result.logs,
2349
+ contract,
1559
2350
  preview,
1560
2351
  ...writtenPath !== void 0 ? { outputPath: writtenPath } : {},
1561
2352
  generatedAt
1562
2353
  });
2354
+ await this.persist("clean-diff", request.dataset, diff);
1563
2355
  this.emitEvent(request.session, "clean", request.dataset, reportKey, {
1564
2356
  rows: result.outputRows,
1565
2357
  columns: result.columns.length,
@@ -1569,32 +2361,57 @@ var LocalDataQualityService = class extends DataQualityService {
1569
2361
  dataset: request.dataset,
1570
2362
  inputRows: result.inputRows,
1571
2363
  outputRows: result.outputRows,
2364
+ dryRun: false,
1572
2365
  logs: result.logs,
2366
+ contract,
1573
2367
  preview,
1574
2368
  ...writtenPath !== void 0 ? { outputPath: writtenPath } : {},
1575
2369
  ...reportKey !== void 0 ? { reportKey } : {},
1576
2370
  generatedAt
1577
2371
  };
1578
2372
  }
2373
+ /** @inheritdoc DataQualityService.getReport */
2374
+ async getReport(key) {
2375
+ if (!isValidReportKey(key)) throw new Error(`invalid reportKey ${JSON.stringify(key)}: expected the deterministic <timestamp>-<kind>-<fingerprint> format`);
2376
+ const store = this.deps.store;
2377
+ if (store === void 0) throw new Error("report storage is disabled (storeReports is false); no persisted reports to read");
2378
+ const record = store.get(key);
2379
+ if (record === void 0) throw new Error(`no persisted report found for reportKey ${JSON.stringify(key)}`);
2380
+ return {
2381
+ key,
2382
+ ...record
2383
+ };
2384
+ }
2385
+ /** @inheritdoc DataQualityService.listReports */
2386
+ async listReports(kind) {
2387
+ const store = this.deps.store;
2388
+ if (store === void 0) throw new Error("report storage is disabled (storeReports is false); no persisted reports to read");
2389
+ return store.list(kind);
2390
+ }
1579
2391
  /** @inheritdoc DataQualityService.verifyDataset */
1580
2392
  async verifyDataset(request) {
1581
2393
  throwIfAborted(request.signal);
1582
2394
  const outcome = verifyTable(await loadTable(resolveWorkspacePath(request.workspace, request.dataset, this.config), this.config, request.signal), request.rules, {
1583
2395
  evidenceRowLimit: this.config.evidenceRowLimit,
1584
2396
  now: this.deps.now,
1585
- signal: request.signal
2397
+ signal: request.signal,
2398
+ expectations: request.expectations,
2399
+ defaultTolerance: this.config.defaultTolerance
1586
2400
  });
1587
2401
  const report = {
1588
2402
  dataset: request.dataset,
1589
2403
  ...outcome
1590
2404
  };
1591
2405
  const failedRules = report.rules.filter((rule) => !rule.passed).length;
2406
+ const failedExpectations = report.expectations.filter((expectation) => !expectation.passed).length;
1592
2407
  const reportKey = await this.persist("verify", request.dataset, report);
1593
2408
  this.emitEvent(request.session, "verify", request.dataset, reportKey, {
1594
2409
  rows: report.rowCount,
1595
2410
  rules: report.rules.length,
1596
2411
  failedRules,
1597
- passed: report.passed
2412
+ passed: report.passed,
2413
+ expectations: report.expectations.length,
2414
+ failedExpectations
1598
2415
  });
1599
2416
  return {
1600
2417
  ...report,
@@ -1622,48 +2439,22 @@ var LocalDataQualityService = class extends DataQualityService {
1622
2439
  }
1623
2440
  };
1624
2441
  //#endregion
1625
- //#region src/tools/shared.ts
1626
- /**
1627
- * Shared helpers for the three data-quality model tools: session workspace
1628
- * resolution and a compact cleaning-run text render. Kept presentation-only;
1629
- * computation lives in the engines and the provider.
1630
- * @module dsh-data-quality/tools-shared
1631
- */
2442
+ //#region src/tools/profile-report-schema.ts
1632
2443
  /**
1633
- * The absolute workspace root a tool call resolves dataset paths against: the
1634
- * calling agent's per-session cwd (mirroring the official fs tools). Non-agent
1635
- * calls fail loud there is no honest workspace to confine paths to.
1636
- * @param exec - the tool-execution context.
1637
- * @returns the absolute workspace root.
1638
- */
1639
- function workspaceOf(exec) {
1640
- const cwd = exec.agent?.session.header.cwd;
1641
- if (cwd === void 0) throw new Error("data-quality tools require an agent-owned session workspace");
1642
- return path.resolve(cwd);
1643
- }
1644
- /** Human-readable cleaning summary for the tool's Native render. */
1645
- function renderCleanText(report) {
1646
- const lines = [];
1647
- lines.push(`Cleaned ${report.dataset}: ${report.inputRows} -> ${report.outputRows} rows over ${report.logs.length} rule(s)`);
1648
- for (const log of report.logs) lines.push(`- rule ${log.ruleIndex} (${log.rule}): ${log.affectedRows} row(s) affected; ${log.detail}`);
1649
- if (report.outputPath !== void 0) lines.push(`Wrote cleaned dataset to ${report.outputPath}`);
1650
- else lines.push("No outputPath given: the source file was left untouched; preview below.");
1651
- if (report.preview.rows.length > 0) {
1652
- lines.push(`Preview (first ${report.preview.rows.length} row(s)):`);
1653
- for (const row of report.preview.rows) {
1654
- const cells = report.preview.columns.map((column) => `${column}=${JSON.stringify(row[column] ?? null)}`).join(", ");
1655
- lines.push(` ${cells}`);
1656
- }
1657
- }
1658
- return lines.join("\n");
1659
- }
1660
- //#endregion
1661
- //#region src/tools/profile.ts
1662
- /**
1663
- * The `data_profile` model tool: deterministic dataset profiling through
1664
- * `ctx.dataQuality.profileDataset` — never model arithmetic.
1665
- * @module dsh-data-quality/tools/profile
2444
+ * Shared JSON-Schema spec for the profile report's canonical value, reused by
2445
+ * `data_profile` (its output) and `data_clean` (the `diffPreview` before/after
2446
+ * snapshots). One source of truth so the two tools never drift.
2447
+ * @module dsh-data-quality/tools-profile-report-schema
1666
2448
  */
2449
+ const SCORECARD_DIMENSION_NAMES = [
2450
+ "completeness",
2451
+ "uniqueness",
2452
+ "validity",
2453
+ "consistency",
2454
+ "timeliness",
2455
+ "accuracy"
2456
+ ];
2457
+ /** One column card's schema. */
1667
2458
  const COLUMN_PROFILE_SCHEMA = {
1668
2459
  type: "object",
1669
2460
  properties: {
@@ -1698,6 +2489,14 @@ const COLUMN_PROFILE_SCHEMA = {
1698
2489
  numeric: {
1699
2490
  type: "object",
1700
2491
  properties: {
2492
+ count: {
2493
+ type: "number",
2494
+ required: true
2495
+ },
2496
+ distinct: {
2497
+ type: "number",
2498
+ required: true
2499
+ },
1701
2500
  min: {
1702
2501
  type: "number",
1703
2502
  required: true
@@ -1754,6 +2553,167 @@ const COLUMN_PROFILE_SCHEMA = {
1754
2553
  },
1755
2554
  additionalProperties: false
1756
2555
  };
2556
+ /** The full profile report's schema. */
2557
+ const PROFILE_REPORT_SCHEMA = {
2558
+ type: "object",
2559
+ properties: {
2560
+ schemaVersion: {
2561
+ type: "number",
2562
+ required: true
2563
+ },
2564
+ dataset: {
2565
+ type: "string",
2566
+ required: true
2567
+ },
2568
+ rowCount: {
2569
+ type: "number",
2570
+ required: true
2571
+ },
2572
+ sampled: {
2573
+ type: "boolean",
2574
+ required: true
2575
+ },
2576
+ profiledRows: {
2577
+ type: "number",
2578
+ required: true
2579
+ },
2580
+ columnCount: {
2581
+ type: "number",
2582
+ required: true
2583
+ },
2584
+ duplicateRows: {
2585
+ type: "number",
2586
+ required: true
2587
+ },
2588
+ duplicateRate: {
2589
+ type: "number",
2590
+ required: true
2591
+ },
2592
+ duplicateSampleRowIndexes: {
2593
+ type: "array",
2594
+ items: { type: "number" },
2595
+ required: true
2596
+ },
2597
+ scorecard: {
2598
+ type: "object",
2599
+ properties: {
2600
+ overall: {
2601
+ oneOf: [{ type: "number" }, { type: "null" }],
2602
+ required: true
2603
+ },
2604
+ weightedOverall: {
2605
+ oneOf: [{ type: "number" }, { type: "null" }],
2606
+ required: true
2607
+ },
2608
+ dimensions: {
2609
+ type: "array",
2610
+ items: {
2611
+ type: "object",
2612
+ properties: {
2613
+ name: {
2614
+ type: "string",
2615
+ enum: [...SCORECARD_DIMENSION_NAMES],
2616
+ required: true
2617
+ },
2618
+ score: {
2619
+ oneOf: [{ type: "number" }, { type: "null" }],
2620
+ required: true
2621
+ },
2622
+ note: {
2623
+ type: "string",
2624
+ required: true
2625
+ }
2626
+ },
2627
+ additionalProperties: false
2628
+ },
2629
+ required: true
2630
+ }
2631
+ },
2632
+ additionalProperties: false,
2633
+ required: true
2634
+ },
2635
+ encoding: {
2636
+ type: "object",
2637
+ properties: {
2638
+ bom: {
2639
+ oneOf: [{ type: "string" }, { type: "null" }],
2640
+ required: true
2641
+ },
2642
+ validUtf8: {
2643
+ type: "boolean",
2644
+ required: true
2645
+ }
2646
+ },
2647
+ additionalProperties: false
2648
+ },
2649
+ generatedAt: {
2650
+ type: "number",
2651
+ required: true
2652
+ },
2653
+ reportKey: { type: "string" },
2654
+ columns: {
2655
+ type: "array",
2656
+ items: COLUMN_PROFILE_SCHEMA,
2657
+ required: true
2658
+ }
2659
+ },
2660
+ additionalProperties: false
2661
+ };
2662
+ //#endregion
2663
+ //#region src/tools/shared.ts
2664
+ /**
2665
+ * Shared helpers for the three data-quality model tools: session workspace
2666
+ * resolution and a compact cleaning-run text render. Kept presentation-only;
2667
+ * computation lives in the engines and the provider.
2668
+ * @module dsh-data-quality/tools-shared
2669
+ */
2670
+ /**
2671
+ * The absolute workspace root a tool call resolves dataset paths against: the
2672
+ * calling agent's per-session cwd (mirroring the official fs tools). Non-agent
2673
+ * calls fail loud — there is no honest workspace to confine paths to.
2674
+ * @param exec - the tool-execution context.
2675
+ * @returns the absolute workspace root.
2676
+ */
2677
+ function workspaceOf(exec) {
2678
+ const cwd = exec.agent?.session.header.cwd;
2679
+ if (cwd === void 0) throw new Error("data-quality tools require an agent-owned session workspace");
2680
+ return path.resolve(cwd);
2681
+ }
2682
+ /** Human-readable cleaning summary for the tool's Native render. */
2683
+ function renderCleanText(report) {
2684
+ const lines = [];
2685
+ const mode = report.dryRun ? "Dry-run plan for" : "Cleaned";
2686
+ lines.push(`${mode} ${report.dataset}: ${report.inputRows} -> ${report.outputRows} rows over ${report.logs.length} rule(s)`);
2687
+ for (const log of report.logs) lines.push(`- rule ${log.ruleIndex} (${log.rule}): ${log.affectedRows} row(s) affected; ${log.detail}`);
2688
+ const contract = report.contract;
2689
+ const keyLabel = contract.dedupeColumns === null ? "full rows" : `[${contract.dedupeColumns.join(", ")}]`;
2690
+ lines.push(`Contract: ${contract.inputRows} -> ${contract.outputRows} rows (${contract.removedRows} removed); uniqueness ${contract.uniqueKeys ? "OK" : "VIOLATED"} over ${keyLabel}${contract.remainingDuplicateRows > 0 ? ` (${contract.remainingDuplicateRows} duplicate row(s) remain)` : ""}`);
2691
+ for (const entry of contract.remainingMissing) lines.push(`- non-null regression: ${entry.column} still has ${entry.count} missing cell(s)`);
2692
+ for (const entry of contract.typeConformance) lines.push(`- type regression: ${entry.column} (${entry.to}) has ${entry.invalidCount} non-conforming cell(s)`);
2693
+ for (const entry of contract.columnDecisions) {
2694
+ const decisions = entry.decisions.map((decision) => `${decision.strategy} (${decision.affectedRows} row(s))`).join(", ");
2695
+ lines.push(`- column ${entry.column}: ${decisions}`);
2696
+ }
2697
+ if (report.diffPreview !== void 0) lines.push(`Diff preview: ${report.diffPreview.before.rowCount} -> ${report.diffPreview.after.rowCount} rows; duplicates ${report.diffPreview.before.duplicateRows} -> ${report.diffPreview.after.duplicateRows}`);
2698
+ if (report.dryRun) lines.push("Dry run: no output file written and no report persisted; plan/preview above.");
2699
+ else if (report.outputPath !== void 0) lines.push(`Wrote cleaned dataset to ${report.outputPath}`);
2700
+ else lines.push("No outputPath given: the source file was left untouched; preview below.");
2701
+ if (report.preview.rows.length > 0) {
2702
+ lines.push(`Preview (first ${report.preview.rows.length} row(s)):`);
2703
+ for (const row of report.preview.rows) {
2704
+ const cells = report.preview.columns.map((column) => `${column}=${JSON.stringify(row[column] ?? null)}`).join(", ");
2705
+ lines.push(` ${cells}`);
2706
+ }
2707
+ }
2708
+ return lines.join("\n");
2709
+ }
2710
+ //#endregion
2711
+ //#region src/tools/profile.ts
2712
+ /**
2713
+ * The `data_profile` model tool: deterministic dataset profiling through
2714
+ * `ctx.dataQuality.profileDataset` — never model arithmetic.
2715
+ * @module dsh-data-quality/tools/profile
2716
+ */
1757
2717
  /**
1758
2718
  * Build the `data_profile` tool definition against a mounted service.
1759
2719
  * @param service - the mounted ctx.dataQuality implementation.
@@ -1764,8 +2724,8 @@ function defineProfileTool(service) {
1764
2724
  name: "data_profile",
1765
2725
  description: [
1766
2726
  "Profile a workspace CSV/TSV/JSON/JSONL dataset with deterministic TypeScript computation (no mental math).",
1767
- "Returns row/column counts, inferred column types, missing rates, unique counts, numeric distributions (min/max/mean/median/p25/p75), IQR outlier counts, mixed-type suspicion notes, and duplicate-row counts.",
1768
- "Column cards cover every row by default; pass sample for a deterministic systematic sample on large files. Datasets above the configured row/size caps are rejected — use sample or raise the caps. The full report persists to the data_quality storage domain (reportKey in the result)."
2727
+ "Returns row/column counts, inferred column types, missing rates, unique counts, numeric distributions (count/distinct/min/max/mean/median/p25/p75), IQR outlier counts, mixed-type suspicion notes, sha256 duplicate-row detection (rate + sample indexes), file encoding (BOM/UTF-8 validity), and a weighted DAMA six-dimension scorecard.",
2728
+ "Pass industryPreset (retail/saas/fund/real-estate/e-commerce/healthcare/logistics/manufacturing/energy) to compare the dataset against that industry's expected columns, making the scorecard accuracy dimension determinable. Column cards cover every row by default; pass sample for a deterministic systematic sample on large files. Datasets above the configured row/size caps are rejected — use sample or raise the caps. The full report persists to the data_quality storage domain (reportKey in the result)."
1769
2729
  ].join("\n"),
1770
2730
  parameters: {
1771
2731
  path: {
@@ -1776,49 +2736,25 @@ function defineProfileTool(service) {
1776
2736
  sample: {
1777
2737
  type: "number",
1778
2738
  description: "Optional systematic sample size (every ceil(N/sample)-th row) for the column cards; row counts stay exact."
2739
+ },
2740
+ industryPreset: {
2741
+ type: "string",
2742
+ enum: [
2743
+ "retail",
2744
+ "saas",
2745
+ "fund",
2746
+ "real-estate",
2747
+ "e-commerce",
2748
+ "healthcare",
2749
+ "logistics",
2750
+ "manufacturing",
2751
+ "energy"
2752
+ ],
2753
+ description: "Optional industry preset id; its expected columns feed the scorecard accuracy dimension."
1779
2754
  }
1780
2755
  },
1781
2756
  output: {
1782
- schema: {
1783
- type: "object",
1784
- properties: {
1785
- dataset: {
1786
- type: "string",
1787
- required: true
1788
- },
1789
- rowCount: {
1790
- type: "number",
1791
- required: true
1792
- },
1793
- sampled: {
1794
- type: "boolean",
1795
- required: true
1796
- },
1797
- profiledRows: {
1798
- type: "number",
1799
- required: true
1800
- },
1801
- columnCount: {
1802
- type: "number",
1803
- required: true
1804
- },
1805
- duplicateRows: {
1806
- type: "number",
1807
- required: true
1808
- },
1809
- generatedAt: {
1810
- type: "number",
1811
- required: true
1812
- },
1813
- reportKey: { type: "string" },
1814
- columns: {
1815
- type: "array",
1816
- items: COLUMN_PROFILE_SCHEMA,
1817
- required: true
1818
- }
1819
- },
1820
- additionalProperties: false
1821
- },
2757
+ schema: PROFILE_REPORT_SCHEMA,
1822
2758
  render: (_args, value) => [{
1823
2759
  type: "text",
1824
2760
  text: renderProfileText(value)
@@ -1828,6 +2764,7 @@ function defineProfileTool(service) {
1828
2764
  return service.profileDataset({
1829
2765
  dataset: args.path,
1830
2766
  sample: args.sample,
2767
+ ...args.industryPreset !== void 0 ? { industryPreset: args.industryPreset } : {},
1831
2768
  workspace: workspaceOf(exec),
1832
2769
  session: exec.agent?.session,
1833
2770
  signal: exec.signal
@@ -1995,7 +2932,7 @@ function defineCleanTool(service) {
1995
2932
  description: [
1996
2933
  "Apply declarative cleaning rules to a workspace CSV/TSV/JSON/JSONL dataset with deterministic TypeScript computation (no mental math).",
1997
2934
  "Rules apply in array order: dedupe (by column group), fill-missing (constant/mean/median/forward), coerce-type (number/date/boolean; failures counted and set to missing), normalize-unit (e.g. 万/亿 suffixes to base units), trim (whitespace), map-values (enum mapping).",
1998
- "The source file is NEVER overwritten. Without outputPath the run is preview-only; with outputPath the cleaned dataset is written there (workspace-confined, .csv/.tsv/.json/.jsonl). Returns the per-rule audit log (affected rows per rule) plus a bounded preview. The full report persists to the data_quality storage domain (reportKey)."
2935
+ "The source file is NEVER overwritten. Without outputPath the run is preview-only; with outputPath the cleaned dataset is written there (workspace-confined, .csv/.tsv/.json/.jsonl). Returns the per-rule audit log, the pre-delivery contract summary (with per-column decision trace), and a bounded preview. Pass dryRun: true to skip the write and get the cleaning plan plus the expected contract/diff preview instead. The full report persists to the data_quality storage domain (reportKey)."
1999
2936
  ].join("\n"),
2000
2937
  parameters: {
2001
2938
  path: {
@@ -2010,6 +2947,10 @@ function defineCleanTool(service) {
2010
2947
  outputPath: {
2011
2948
  type: "string",
2012
2949
  description: "Optional workspace-relative output path for the cleaned dataset (must differ from path)."
2950
+ },
2951
+ dryRun: {
2952
+ type: "boolean",
2953
+ description: "When true, do not write any output file; return the cleaning plan and expected contract/diff preview instead (default false)."
2013
2954
  }
2014
2955
  },
2015
2956
  output: {
@@ -2028,12 +2969,38 @@ function defineCleanTool(service) {
2028
2969
  type: "number",
2029
2970
  required: true
2030
2971
  },
2972
+ dryRun: {
2973
+ type: "boolean",
2974
+ required: true
2975
+ },
2031
2976
  generatedAt: {
2032
2977
  type: "number",
2033
2978
  required: true
2034
2979
  },
2035
2980
  outputPath: { type: "string" },
2036
2981
  reportKey: { type: "string" },
2982
+ diffPreview: {
2983
+ type: "object",
2984
+ properties: {
2985
+ dataset: {
2986
+ type: "string",
2987
+ required: true
2988
+ },
2989
+ before: {
2990
+ ...PROFILE_REPORT_SCHEMA,
2991
+ required: true
2992
+ },
2993
+ after: {
2994
+ ...PROFILE_REPORT_SCHEMA,
2995
+ required: true
2996
+ },
2997
+ generatedAt: {
2998
+ type: "number",
2999
+ required: true
3000
+ }
3001
+ },
3002
+ additionalProperties: false
3003
+ },
2037
3004
  logs: {
2038
3005
  type: "array",
2039
3006
  items: {
@@ -2068,6 +3035,117 @@ function defineCleanTool(service) {
2068
3035
  },
2069
3036
  required: true
2070
3037
  },
3038
+ contract: {
3039
+ type: "object",
3040
+ properties: {
3041
+ inputRows: {
3042
+ type: "number",
3043
+ required: true
3044
+ },
3045
+ outputRows: {
3046
+ type: "number",
3047
+ required: true
3048
+ },
3049
+ removedRows: {
3050
+ type: "number",
3051
+ required: true
3052
+ },
3053
+ dedupeColumns: {
3054
+ oneOf: [{
3055
+ type: "array",
3056
+ items: { type: "string" }
3057
+ }, { type: "null" }],
3058
+ required: true
3059
+ },
3060
+ uniqueKeys: {
3061
+ type: "boolean",
3062
+ required: true
3063
+ },
3064
+ remainingDuplicateRows: {
3065
+ type: "number",
3066
+ required: true
3067
+ },
3068
+ remainingMissing: {
3069
+ type: "array",
3070
+ items: {
3071
+ type: "object",
3072
+ properties: {
3073
+ column: {
3074
+ type: "string",
3075
+ required: true
3076
+ },
3077
+ count: {
3078
+ type: "number",
3079
+ required: true
3080
+ }
3081
+ },
3082
+ additionalProperties: false
3083
+ },
3084
+ required: true
3085
+ },
3086
+ typeConformance: {
3087
+ type: "array",
3088
+ items: {
3089
+ type: "object",
3090
+ properties: {
3091
+ column: {
3092
+ type: "string",
3093
+ required: true
3094
+ },
3095
+ to: {
3096
+ type: "string",
3097
+ enum: [
3098
+ "number",
3099
+ "date",
3100
+ "boolean"
3101
+ ],
3102
+ required: true
3103
+ },
3104
+ invalidCount: {
3105
+ type: "number",
3106
+ required: true
3107
+ }
3108
+ },
3109
+ additionalProperties: false
3110
+ },
3111
+ required: true
3112
+ },
3113
+ columnDecisions: {
3114
+ type: "array",
3115
+ items: {
3116
+ type: "object",
3117
+ properties: {
3118
+ column: {
3119
+ type: "string",
3120
+ required: true
3121
+ },
3122
+ decisions: {
3123
+ type: "array",
3124
+ items: {
3125
+ type: "object",
3126
+ properties: {
3127
+ strategy: {
3128
+ type: "string",
3129
+ required: true
3130
+ },
3131
+ affectedRows: {
3132
+ type: "number",
3133
+ required: true
3134
+ }
3135
+ },
3136
+ additionalProperties: false
3137
+ },
3138
+ required: true
3139
+ }
3140
+ },
3141
+ additionalProperties: false
3142
+ },
3143
+ required: true
3144
+ }
3145
+ },
3146
+ additionalProperties: false,
3147
+ required: true
3148
+ },
2071
3149
  preview: {
2072
3150
  type: "object",
2073
3151
  properties: {
@@ -2098,6 +3176,7 @@ function defineCleanTool(service) {
2098
3176
  dataset: args.path,
2099
3177
  rules: args.rules,
2100
3178
  ...args.outputPath !== void 0 ? { outputPath: args.outputPath } : {},
3179
+ ...args.dryRun !== void 0 ? { dryRun: args.dryRun } : {},
2101
3180
  workspace: workspaceOf(exec),
2102
3181
  session: exec.agent?.session,
2103
3182
  signal: exec.signal
@@ -2113,6 +3192,74 @@ function defineCleanTool(service) {
2113
3192
  * (`passed: false` with evidence), never a tool error.
2114
3193
  * @module dsh-data-quality/tools/verify
2115
3194
  */
3195
+ const EXPECTATION_METRICS = [
3196
+ "rowCount",
3197
+ "columnSum",
3198
+ "columnMean",
3199
+ "uniqueCount",
3200
+ "nullCount"
3201
+ ];
3202
+ const EXPECTATION_SCHEMA = {
3203
+ type: "array",
3204
+ items: {
3205
+ type: "object",
3206
+ properties: {
3207
+ metric: {
3208
+ type: "string",
3209
+ enum: [...EXPECTATION_METRICS],
3210
+ required: true,
3211
+ description: "Metric to reconcile: rowCount/columnSum/columnMean/uniqueCount/nullCount."
3212
+ },
3213
+ column: {
3214
+ type: "string",
3215
+ description: "Required for every metric except rowCount."
3216
+ },
3217
+ expected: {
3218
+ type: "number",
3219
+ required: true,
3220
+ description: "The expected value to reconcile against."
3221
+ },
3222
+ tolerance: {
3223
+ type: "number",
3224
+ description: "Optional relative tolerance in [0, 1]; defaults to defaultTolerance."
3225
+ }
3226
+ },
3227
+ additionalProperties: false,
3228
+ description: "Reconcile a deterministic computed metric against an expected value with relative tolerance."
3229
+ },
3230
+ description: "Optional metric expectations; each yields passed true or passed false with actual/expected/tolerance detail."
3231
+ };
3232
+ const EXPECTATION_RESULT_SCHEMA = {
3233
+ type: "array",
3234
+ items: {
3235
+ type: "object",
3236
+ properties: {
3237
+ metric: {
3238
+ type: "string",
3239
+ enum: [...EXPECTATION_METRICS],
3240
+ required: true
3241
+ },
3242
+ column: { type: "string" },
3243
+ expected: {
3244
+ type: "number",
3245
+ required: true
3246
+ },
3247
+ actual: {
3248
+ type: "number",
3249
+ required: true
3250
+ },
3251
+ tolerance: {
3252
+ type: "number",
3253
+ required: true
3254
+ },
3255
+ passed: {
3256
+ type: "boolean",
3257
+ required: true
3258
+ }
3259
+ },
3260
+ additionalProperties: false
3261
+ }
3262
+ };
2116
3263
  const VERIFY_RULE_SCHEMA = {
2117
3264
  type: "array",
2118
3265
  items: { oneOf: [
@@ -2290,7 +3437,8 @@ function defineVerifyTool(service) {
2290
3437
  description: [
2291
3438
  "Verify a workspace CSV/TSV/JSON/JSONL dataset against declarative quality rules with deterministic TypeScript computation (no mental math).",
2292
3439
  "Rules: not-null, unique (column group), range (numeric bounds), regex, enum, cross-column (e.g. startDate < endDate), freshness (date column within N days of asOf). A missing cell fails every rule that reads it.",
2293
- "Returns per-rule pass/fail with capped failing-row evidence. Overall failure is a NORMAL result with passed: false not a tool error. The full report persists to the data_quality storage domain (reportKey)."
3440
+ "Optional expectations reconcile deterministic metrics (rowCount/columnSum/columnMean/uniqueCount/nullCount) against expected values with relative tolerance; a mismatch is a normal passed: false with actual/expected/tolerance detail, never a tool error.",
3441
+ "Returns per-rule pass/fail with capped failing-row evidence plus the expectation outcomes. Overall failure is a NORMAL result with passed: false — not a tool error. The full report persists to the data_quality storage domain (reportKey)."
2294
3442
  ].join("\n"),
2295
3443
  parameters: {
2296
3444
  path: {
@@ -2301,6 +3449,10 @@ function defineVerifyTool(service) {
2301
3449
  rules: {
2302
3450
  ...VERIFY_RULE_SCHEMA,
2303
3451
  required: true
3452
+ },
3453
+ expectations: {
3454
+ ...EXPECTATION_SCHEMA,
3455
+ description: "Optional metric expectations to reconcile (rowCount/columnSum/columnMean/uniqueCount/nullCount)."
2304
3456
  }
2305
3457
  },
2306
3458
  output: {
@@ -2380,6 +3532,10 @@ function defineVerifyTool(service) {
2380
3532
  additionalProperties: false
2381
3533
  },
2382
3534
  required: true
3535
+ },
3536
+ expectations: {
3537
+ ...EXPECTATION_RESULT_SCHEMA,
3538
+ required: true
2383
3539
  }
2384
3540
  },
2385
3541
  additionalProperties: false
@@ -2393,6 +3549,7 @@ function defineVerifyTool(service) {
2393
3549
  return service.verifyDataset({
2394
3550
  dataset: args.path,
2395
3551
  rules: args.rules,
3552
+ ...args.expectations !== void 0 ? { expectations: args.expectations } : {},
2396
3553
  workspace: workspaceOf(exec),
2397
3554
  session: exec.agent?.session,
2398
3555
  signal: exec.signal
@@ -2401,18 +3558,320 @@ function defineVerifyTool(service) {
2401
3558
  });
2402
3559
  }
2403
3560
  //#endregion
2404
- //#region src/version.ts
3561
+ //#region src/report-html.ts
3562
+ /** Escape text for safe embedding in an HTML document. */
3563
+ function escapeHtml(text) {
3564
+ return text.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;").replace(/'/g, "&#39;");
3565
+ }
3566
+ /** Render a 0..1 rate as a percentage string, or `—` when undetermined. */
3567
+ function pct(score) {
3568
+ return score === null ? "—" : `${(score * 100).toFixed(1)}%`;
3569
+ }
3570
+ /** The shared document shell: inline CSS + inline JS, no external requests. */
3571
+ function shell(title, body, script) {
3572
+ return [
3573
+ "<!doctype html>",
3574
+ "<html lang=\"en\">",
3575
+ "<head>",
3576
+ "<meta charset=\"utf-8\">",
3577
+ `<title>${escapeHtml(title)}</title>`,
3578
+ "<style>",
3579
+ " :root { --ink: #1a1f2e; --muted: #6b7280; --line: #e5e7eb; --accent: #0f766e; --fail: #b91c1c; --warn: #b45309; --pass: #15803d; }",
3580
+ " * { box-sizing: border-box; }",
3581
+ " body { margin: 0; padding: 24px; font: 14px/1.5 ui-sans-serif, system-ui, -apple-system, \"Segoe UI\", sans-serif; color: var(--ink); background: #f8fafc; }",
3582
+ " header { margin-bottom: 20px; }",
3583
+ " h1 { font-size: 20px; margin: 0 0 4px; }",
3584
+ " .meta { color: var(--muted); font-size: 12px; }",
3585
+ " section { background: #fff; border: 1px solid var(--line); border-radius: 8px; padding: 16px 20px; margin-bottom: 16px; }",
3586
+ " h2 { font-size: 15px; margin: 0 0 12px; }",
3587
+ " table { border-collapse: collapse; width: 100%; font-size: 13px; }",
3588
+ " th, td { text-align: left; padding: 6px 10px; border-top: 1px solid var(--line); vertical-align: top; }",
3589
+ " th { color: var(--muted); font-weight: 600; }",
3590
+ " .score-cell { font-variant-numeric: tabular-nums; }",
3591
+ " .dim-fail { color: var(--fail); font-weight: 600; }",
3592
+ " .dim-warn { color: var(--warn); }",
3593
+ " .dim-pass { color: var(--pass); }",
3594
+ " .dim-undetermined { color: var(--muted); }",
3595
+ " button { cursor: pointer; font: inherit; }",
3596
+ "</style>",
3597
+ "</head>",
3598
+ "<body>",
3599
+ body,
3600
+ "<script>",
3601
+ script,
3602
+ "<\/script>",
3603
+ "</body>",
3604
+ "</html>",
3605
+ ""
3606
+ ].join("\n");
3607
+ }
3608
+ /** The DAMA six-dimension scorecard section. */
3609
+ function scorecardSection(report) {
3610
+ const rows = report.scorecard.dimensions.map((dimension) => {
3611
+ const cls = dimension.score === null ? "dim-undetermined" : dimension.score >= .9 ? "dim-pass" : dimension.score >= .7 ? "dim-warn" : "dim-fail";
3612
+ return `<tr><td>${escapeHtml(dimension.name)}</td><td class="score-cell ${cls}">${pct(dimension.score)}</td><td>${escapeHtml(dimension.note)}</td></tr>`;
3613
+ }).join("\n");
3614
+ const overall = report.scorecard.overall;
3615
+ const weighted = report.scorecard.weightedOverall;
3616
+ return [
3617
+ "<section>",
3618
+ "<h2>DAMA six-dimension quality scorecard</h2>",
3619
+ "<table>",
3620
+ "<thead><tr><th>Dimension</th><th>Score</th><th>Note</th></tr></thead>",
3621
+ "<tbody>",
3622
+ rows,
3623
+ "</tbody>",
3624
+ "</table>",
3625
+ `<p class="meta" id="summary-text">overall ${pct(overall)} · weighted ${pct(weighted)}</p>`,
3626
+ "</section>"
3627
+ ].join("\n");
3628
+ }
3629
+ /** The per-column profile summary table. */
3630
+ function columnsSection(report) {
3631
+ return [
3632
+ "<section>",
3633
+ "<h2>Column profile</h2>",
3634
+ "<table>",
3635
+ "<thead><tr><th>Column</th><th>Type</th><th>Missing</th><th>Missing rate</th><th>Unique</th><th>Numeric distribution</th><th>Top values</th><th>Notes</th></tr></thead>",
3636
+ "<tbody>",
3637
+ report.columns.map((column) => {
3638
+ const numeric = column.numeric;
3639
+ const numericCell = numeric === void 0 ? "—" : `min ${numeric.min} · p25 ${numeric.p25} · median ${numeric.median} · p75 ${numeric.p75} · max ${numeric.max} · mean ${numeric.mean}${numeric.outliers > 0 ? ` · ${numeric.outliers} IQR outliers` : ""}`;
3640
+ const top = column.topValues === void 0 ? "—" : column.topValues.map((entry) => `${escapeHtml(entry.value)} ×${entry.count}`).join(", ");
3641
+ const notes = column.notes.length === 0 ? "" : `<p class="meta">${column.notes.map(escapeHtml).join("; ")}</p>`;
3642
+ return [
3643
+ "<tr>",
3644
+ `<td>${escapeHtml(column.name)}</td>`,
3645
+ `<td>${escapeHtml(column.inferredType)}</td>`,
3646
+ `<td class="score-cell">${column.missing}</td>`,
3647
+ `<td class="score-cell">${(column.missingRate * 100).toFixed(1)}%</td>`,
3648
+ `<td class="score-cell">${column.unique}</td>`,
3649
+ `<td>${numericCell}</td>`,
3650
+ `<td>${top}</td>`,
3651
+ `<td>${notes}</td>`,
3652
+ "</tr>"
3653
+ ].join("");
3654
+ }).join("\n"),
3655
+ "</tbody>",
3656
+ "</table>",
3657
+ "</section>"
3658
+ ].join("\n");
3659
+ }
2405
3660
  /**
2406
- * Plugin version, kept in one place so `scripts/release.mjs` can stamp it and
2407
- * reports can name their generator.
2408
- * @module dsh-data-quality/version
3661
+ * Render a profile report as a self-contained offline HTML document.
3662
+ * @param report - the profile report (already persisted/returned by data_profile).
3663
+ * @returns the complete single-file HTML.
2409
3664
  */
2410
- /** The package version reported in persisted reports. */
2411
- const VERSION = "0.1.3";
3665
+ function renderProfileHtml(report) {
3666
+ const body = [
3667
+ "<header>",
3668
+ `<h1>Data profile: ${escapeHtml(report.dataset)}</h1>`,
3669
+ `<p class="meta">${report.rowCount} rows × ${report.columnCount} columns · generated ${new Date(report.generatedAt).toISOString()} · schema v1 · report ${report.reportKey ?? "(unpersisted)"}</p>`,
3670
+ "<button id=\"copy-summary\" type=\"button\">Copy summary</button>",
3671
+ "</header>",
3672
+ scorecardSection(report),
3673
+ columnsSection(report)
3674
+ ].join("\n");
3675
+ const script = [
3676
+ "const button = document.getElementById('copy-summary');",
3677
+ "if (button) {",
3678
+ " button.addEventListener(\"click\", () => {",
3679
+ " const text = document.getElementById(\"summary-text\");",
3680
+ " if (text && navigator.clipboard) navigator.clipboard.writeText(text.textContent || \"\");",
3681
+ " });",
3682
+ "}"
3683
+ ].join("\n");
3684
+ return shell(`Data profile: ${report.dataset}`, body, script);
3685
+ }
3686
+ /** The per-rule cleaning summary table (for clean/clean-diff reports). */
3687
+ function cleaningSection(report) {
3688
+ const rows = report.logs.map((log) => {
3689
+ return `<tr><td>${escapeHtml(log.rule)}</td><td class="score-cell">${log.affectedRows}</td><td>${escapeHtml(log.detail)}</td></tr>`;
3690
+ }).join("\n");
3691
+ const removed = report.inputRows - report.outputRows;
3692
+ return [
3693
+ "<section>",
3694
+ "<h2>Cleaning summary</h2>",
3695
+ `<p class="meta" id="summary-text">input ${report.inputRows} rows · output ${report.outputRows} rows · removed ${removed} rows</p>`,
3696
+ "<table>",
3697
+ "<thead><tr><th>Rule</th><th>Affected rows</th><th>Detail</th></tr></thead>",
3698
+ "<tbody>",
3699
+ rows,
3700
+ "</tbody>",
3701
+ "</table>",
3702
+ "</section>"
3703
+ ].join("\n");
3704
+ }
3705
+ /**
3706
+ * Render a clean report as a self-contained offline HTML document (the
3707
+ * per-rule cleaning summary table).
3708
+ * @param report - the clean report (logs + input/output row counts).
3709
+ * @param dataset - the dataset label for the document title.
3710
+ * @returns the complete single-file HTML.
3711
+ */
3712
+ function renderCleanHtml(report, dataset) {
3713
+ const body = [
3714
+ "<header>",
3715
+ `<h1>Cleaning report: ${escapeHtml(dataset)}</h1>`,
3716
+ `<p class="meta">schema v1</p>`,
3717
+ "</header>",
3718
+ cleaningSection(report)
3719
+ ].join("\n");
3720
+ return shell(`Cleaning report: ${dataset}`, body, "// no interactive behavior needed; everything renders without external requests\n");
3721
+ }
3722
+ //#endregion
3723
+ //#region src/tools/report.ts
3724
+ /**
3725
+ * The `data_report` model tool: read persisted profile/clean-diff reports
3726
+ * back from the `data_quality` storage domain, by exact `reportKey` or by
3727
+ * `kind`. Deterministic read-only consumption — no model arithmetic.
3728
+ * @module dsh-data-quality/tools/report
3729
+ */
3730
+ /** The report kinds a caller may query. */
3731
+ const REPORT_KINDS = [
3732
+ "profile",
3733
+ "clean",
3734
+ "clean-diff",
3735
+ "verify",
3736
+ "citations"
3737
+ ];
3738
+ /** Render one stored report as a self-contained HTML document (profile/clean only). */
3739
+ function renderRecordHtml(record) {
3740
+ if (record.kind === "profile") return renderProfileHtml(record.report);
3741
+ if (record.kind === "clean" || record.kind === "clean-diff") return renderCleanHtml(record.report, record.dataset);
3742
+ throw new Error(`data_report html format does not support kind "${record.kind}" (profile/clean only)`);
3743
+ }
3744
+ /** Project a stored report into the canonical value (the stored report is already lossless JSON). */
3745
+ function toView(record) {
3746
+ return {
3747
+ key: record.key,
3748
+ kind: record.kind,
3749
+ at: record.at,
3750
+ dataset: record.dataset,
3751
+ report: record.report
3752
+ };
3753
+ }
3754
+ /** Human-readable report summary for the tool's Native render. */
3755
+ function renderReportText(value) {
3756
+ const lines = [];
3757
+ if (value.key !== void 0) lines.push(value.records.length === 0 ? `No report for ${value.key}` : `Report ${value.key}`);
3758
+ else lines.push(`Reports of kind ${value.kind ?? ""} (${value.records.length})`);
3759
+ for (const record of value.records) lines.push(`- ${record.key} [${record.kind}] ${record.dataset} @ ${new Date(record.at).toISOString()}`);
3760
+ return lines.join("\n");
3761
+ }
3762
+ /**
3763
+ * Build the `data_report` tool definition against a mounted service.
3764
+ * @param service - the mounted ctx.dataQuality implementation.
3765
+ * @returns the tool definition to register.
3766
+ */
3767
+ function defineReportTool(service) {
3768
+ return defineTool({
3769
+ name: "data_report",
3770
+ description: [
3771
+ "Read persisted data-quality reports back from the data_quality storage domain (deterministic, read-only).",
3772
+ "Pass key (the exact reportKey a prior run returned) to fetch one report, or kind to list every persisted report of that kind, ordered chronologically. Exactly one of key/kind.",
3773
+ "Returns the report envelope(s): kind, dataset, timestamp, and the full stored report (profile/clean/clean-diff/verify/citations). Missing keys and unknown kinds fail loudly."
3774
+ ].join("\n"),
3775
+ parameters: {
3776
+ key: {
3777
+ type: "string",
3778
+ description: "Exact storage reportKey (e.g. 20260819000000000-profile-1a2b3c4d); fetches that one report."
3779
+ },
3780
+ kind: {
3781
+ type: "string",
3782
+ enum: [...REPORT_KINDS],
3783
+ description: "Report kind to list (profile/clean/clean-diff/verify/citations)."
3784
+ },
3785
+ format: {
3786
+ type: "string",
3787
+ enum: ["json", "html"],
3788
+ description: "Output format. json (default) returns the report envelope(s); html renders one report as a self-contained offline HTML document (requires key; profile/clean only)."
3789
+ }
3790
+ },
3791
+ output: {
3792
+ schema: {
3793
+ type: "object",
3794
+ properties: {
3795
+ key: { type: "string" },
3796
+ kind: {
3797
+ type: "string",
3798
+ enum: [...REPORT_KINDS]
3799
+ },
3800
+ records: {
3801
+ type: "array",
3802
+ items: {
3803
+ type: "object",
3804
+ properties: {
3805
+ key: {
3806
+ type: "string",
3807
+ required: true
3808
+ },
3809
+ kind: {
3810
+ type: "string",
3811
+ enum: [...REPORT_KINDS],
3812
+ required: true
3813
+ },
3814
+ at: {
3815
+ type: "number",
3816
+ required: true
3817
+ },
3818
+ dataset: {
3819
+ type: "string",
3820
+ required: true
3821
+ },
3822
+ report: {
3823
+ type: "json",
3824
+ required: true
3825
+ }
3826
+ },
3827
+ additionalProperties: false
3828
+ },
3829
+ required: true
3830
+ },
3831
+ html: {
3832
+ type: "string",
3833
+ description: "Self-contained offline HTML (present only when format: html)."
3834
+ }
3835
+ },
3836
+ additionalProperties: false
3837
+ },
3838
+ render: (_args, value) => {
3839
+ const view = value;
3840
+ if (view.html !== void 0) return [{
3841
+ type: "text",
3842
+ text: view.html
3843
+ }];
3844
+ return [{
3845
+ type: "text",
3846
+ text: renderReportText(view)
3847
+ }];
3848
+ }
3849
+ },
3850
+ async execute(args, _exec) {
3851
+ const hasKey = args.key !== void 0;
3852
+ if (hasKey === (args.kind !== void 0)) throw new Error("data_report needs exactly one of key/kind");
3853
+ const format = args.format ?? "json";
3854
+ if (hasKey) {
3855
+ const record = await service.getReport(args.key);
3856
+ return {
3857
+ key: args.key,
3858
+ records: [toView(record)],
3859
+ ...format === "html" ? { html: renderRecordHtml(record) } : {}
3860
+ };
3861
+ }
3862
+ if (format === "html") throw new Error("data_report html format requires key (exactly one report)");
3863
+ const records = await service.listReports(args.kind);
3864
+ return {
3865
+ kind: args.kind,
3866
+ records: records.map(toView)
3867
+ };
3868
+ }
3869
+ });
3870
+ }
2412
3871
  //#endregion
2413
3872
  //#region src/index.ts
2414
3873
  const name = "data-quality";
2415
- /** The three model tools and the durable report domain. */
3874
+ /** The four model tools and the durable report domain. */
2416
3875
  const inject = ["tools", "storageDomain"];
2417
3876
  /**
2418
3877
  * Mount the seam: resolve config (fail loud), open the report domain, publish
@@ -2439,7 +3898,11 @@ async function apply(ctx, config = {}) {
2439
3898
  await reports.put(key, record);
2440
3899
  return key;
2441
3900
  },
2442
- get: (key) => reports.get(key)
3901
+ get: (key) => reports.get(key),
3902
+ list: (kind) => [...reports.entries()].filter(([, record]) => record.kind === kind).sort(([keyA], [keyB]) => keyA < keyB ? -1 : keyA > keyB ? 1 : 0).map(([key, record]) => ({
3903
+ key,
3904
+ ...record
3905
+ }))
2443
3906
  };
2444
3907
  }
2445
3908
  const service = new LocalDataQualityService(ctx, resolved, {
@@ -2449,7 +3912,8 @@ async function apply(ctx, config = {}) {
2449
3912
  ctx.tools.register(defineProfileTool(service));
2450
3913
  ctx.tools.register(defineCleanTool(service));
2451
3914
  ctx.tools.register(defineVerifyTool(service));
2452
- logger.info(`dsh-data-quality ${VERSION} mounted: ctx.dataQuality + data_profile/data_clean/data_verify`);
3915
+ ctx.tools.register(defineReportTool(service));
3916
+ logger.info(`dsh-data-quality ${VERSION} mounted: ctx.dataQuality + data_profile/data_clean/data_verify/data_report`);
2453
3917
  if (domain !== void 0) {
2454
3918
  const handle = domain;
2455
3919
  ctx.effect(() => async () => {
@@ -2458,4 +3922,4 @@ async function apply(ctx, config = {}) {
2458
3922
  }
2459
3923
  }
2460
3924
  //#endregion
2461
- export { Config, DATA_QUALITY_EVENT_TYPES, DataQualityService, DatasetError, LocalDataQualityService, MAX_CELL_TEXT, VERSION, appendDataQualityEvent, apply, applyCleanRules, checkCitations, dataQualityDomainSpec, inject, isMissing, loadDocument, loadTable, name, parseBoolean, parseDate, parseDelimited, parseJsonTable, parseLocator, parseNumeric, profileTable, renderProfileText, renderVerifyText, reportKeyOf, reportRecordSchema, resolveConfig, resolveWorkspacePath, sampleRows, serializeDelimited, truncateCell, truncateRow, verifyTable };
3925
+ export { Config, DATA_QUALITY_EVENT_TYPES, DataQualityService, DatasetError, INDUSTRY_PRESETS, INDUSTRY_PRESET_IDS, LocalDataQualityService, MAX_CELL_TEXT, REPORT_SCHEMA_VERSION, VERSION, VerifyExpectationError, appendDataQualityEvent, apply, applyCleanRules, checkCitations, computeCleanContract, computeCleanProfileDiff, computeScorecard, countDuplicateRows, dataQualityDomainSpec, dateFormatOf, detectDuplicateRows, detectEncoding, inject, isMissing, isValidReportKey, loadDocument, loadTable, name, numericProfile, parseBoolean, parseDate, parseDateCell, parseDelimited, parseJsonTable, parseLocator, parseNumeric, profileTable, readDatasetFile, renderCleanHtml, renderProfileHtml, renderProfileText, renderVerifyText, reportKeyOf, reportRecordSchema, resolveConfig, resolveIndustryPreset, resolveWorkspacePath, sampleRows, serializeDelimited, truncateCell, truncateRow, verifyExpectations, verifyTable };