dsh-data-quality 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. package/CHANGELOG.md +16 -0
  2. package/LICENSE +201 -0
  3. package/README.es.md +181 -0
  4. package/README.hi.md +181 -0
  5. package/README.md +181 -0
  6. package/README.pt.md +181 -0
  7. package/README.zh.md +181 -0
  8. package/THIRD_PARTY_NOTICES.md +20 -0
  9. package/cordis.patch.yml +46 -0
  10. package/lib/index.js +2458 -0
  11. package/lib/types/clean.d.ts +82 -0
  12. package/lib/types/clean.d.ts.map +1 -0
  13. package/lib/types/clean.js +351 -0
  14. package/lib/types/clean.js.map +1 -0
  15. package/lib/types/config.d.ts +47 -0
  16. package/lib/types/config.d.ts.map +1 -0
  17. package/lib/types/config.js +66 -0
  18. package/lib/types/config.js.map +1 -0
  19. package/lib/types/dataset.d.ts +133 -0
  20. package/lib/types/dataset.d.ts.map +1 -0
  21. package/lib/types/dataset.js +404 -0
  22. package/lib/types/dataset.js.map +1 -0
  23. package/lib/types/events.d.ts +73 -0
  24. package/lib/types/events.d.ts.map +1 -0
  25. package/lib/types/events.js +41 -0
  26. package/lib/types/events.js.map +1 -0
  27. package/lib/types/index.d.ts +45 -0
  28. package/lib/types/index.d.ts.map +1 -0
  29. package/lib/types/index.js +78 -0
  30. package/lib/types/index.js.map +1 -0
  31. package/lib/types/present.d.ts +24 -0
  32. package/lib/types/present.d.ts.map +1 -0
  33. package/lib/types/present.js +34 -0
  34. package/lib/types/present.js.map +1 -0
  35. package/lib/types/profile.d.ts +79 -0
  36. package/lib/types/profile.d.ts.map +1 -0
  37. package/lib/types/profile.js +196 -0
  38. package/lib/types/profile.js.map +1 -0
  39. package/lib/types/provider-local.d.ts +56 -0
  40. package/lib/types/provider-local.d.ts.map +1 -0
  41. package/lib/types/provider-local.js +163 -0
  42. package/lib/types/provider-local.js.map +1 -0
  43. package/lib/types/service.d.ts +160 -0
  44. package/lib/types/service.d.ts.map +1 -0
  45. package/lib/types/service.js +25 -0
  46. package/lib/types/service.js.map +1 -0
  47. package/lib/types/store.d.ts +61 -0
  48. package/lib/types/store.d.ts.map +1 -0
  49. package/lib/types/store.js +42 -0
  50. package/lib/types/store.js.map +1 -0
  51. package/lib/types/tools/clean.d.ts +14 -0
  52. package/lib/types/tools/clean.d.ts.map +1 -0
  53. package/lib/types/tools/clean.js +146 -0
  54. package/lib/types/tools/clean.js.map +1 -0
  55. package/lib/types/tools/profile.d.ts +13 -0
  56. package/lib/types/tools/profile.d.ts.map +1 -0
  57. package/lib/types/tools/profile.js +91 -0
  58. package/lib/types/tools/profile.js.map +1 -0
  59. package/lib/types/tools/shared.d.ts +19 -0
  60. package/lib/types/tools/shared.d.ts.map +1 -0
  61. package/lib/types/tools/shared.js +44 -0
  62. package/lib/types/tools/shared.js.map +1 -0
  63. package/lib/types/tools/verify.d.ts +14 -0
  64. package/lib/types/tools/verify.d.ts.map +1 -0
  65. package/lib/types/tools/verify.js +160 -0
  66. package/lib/types/tools/verify.js.map +1 -0
  67. package/lib/types/verify.d.ts +124 -0
  68. package/lib/types/verify.d.ts.map +1 -0
  69. package/lib/types/verify.js +391 -0
  70. package/lib/types/verify.js.map +1 -0
  71. package/lib/types/version.d.ts +8 -0
  72. package/lib/types/version.d.ts.map +1 -0
  73. package/lib/types/version.js +8 -0
  74. package/lib/types/version.js.map +1 -0
  75. package/package.json +137 -0
  76. package/src/clean.ts +382 -0
  77. package/src/config.ts +104 -0
  78. package/src/dataset.ts +445 -0
  79. package/src/events.ts +90 -0
  80. package/src/index.ts +115 -0
  81. package/src/present.ts +38 -0
  82. package/src/profile.ts +250 -0
  83. package/src/provider-local.ts +194 -0
  84. package/src/service.ts +172 -0
  85. package/src/store.ts +74 -0
  86. package/src/tools/clean.ts +150 -0
  87. package/src/tools/profile.ts +94 -0
  88. package/src/tools/shared.ts +47 -0
  89. package/src/tools/verify.ts +163 -0
  90. package/src/verify.ts +496 -0
  91. package/src/version.ts +8 -0
package/lib/index.js ADDED
@@ -0,0 +1,2458 @@
1
+ import z from "@deepseek-ai/schemastery";
2
+ import z$1 from "zod";
3
+ import { defineDomain, domainTable } from "@deepseek-ai/dsh-storage-domain";
4
+ import { mkdir, open, stat, writeFile } from "node:fs/promises";
5
+ import path from "node:path";
6
+ import { createHash } from "node:crypto";
7
+ import { KNOWN_SESSION_EVENT_TYPES } from "@deepseek-ai/dsh-session";
8
+ import { Service } from "@deepseek-ai/cordis";
9
+ import { defineTool } from "@deepseek-ai/dsh-tools";
10
+ //#region src/config.ts
11
+ /**
12
+ * Config schema and resolution for `dsh-data-quality`. Every tunable is a
13
+ * validated {@link Config} field changeable from cordis.yml; the resolution
14
+ * step validates bounds so misconfiguration fails loud at mount.
15
+ * @module dsh-data-quality/config
16
+ */
17
+ /** Schemastery schema: the loader validates and fills defaults before `apply`. */
18
+ const Config = z.object({
19
+ enabled: z.boolean().default(true),
20
+ maxRows: z.number().default(2e5),
21
+ maxFileSizeMB: z.number().default(64),
22
+ defaultTolerance: z.number().default(1e-9),
23
+ evidenceRowLimit: z.number().default(20),
24
+ allowedExtensions: z.array(z.string()).default([
25
+ ".csv",
26
+ ".tsv",
27
+ ".json",
28
+ ".jsonl"
29
+ ]),
30
+ workspaceRoot: z.string().default(""),
31
+ storeReports: z.boolean().default(true)
32
+ });
33
+ /** Throw unless `value` is a positive safe integer. */
34
+ function assertPositiveInt(name, value) {
35
+ if (!Number.isSafeInteger(value) || value <= 0) throw new TypeError(`${name} must be a positive safe integer, got ${String(value)}`);
36
+ }
37
+ /** Throw unless `value` is a finite number in `[min, max]`. */
38
+ function assertFiniteRange(name, value, min, max) {
39
+ if (typeof value !== "number" || !Number.isFinite(value) || value < min || value > max) throw new TypeError(`${name} must be a finite number in [${min}, ${max}], got ${String(value)}`);
40
+ }
41
+ /**
42
+ * Validate raw values and fill explicit defaults. Invalid bounds throw here —
43
+ * misconfiguration fails loud at mount even without the Schemastery loader.
44
+ * @param config - raw (possibly partial) plugin config.
45
+ * @returns the fully resolved config.
46
+ */
47
+ function resolveConfig(config = {}) {
48
+ const maxRows = config.maxRows ?? 2e5;
49
+ assertPositiveInt("maxRows", maxRows);
50
+ const maxFileSizeMB = config.maxFileSizeMB ?? 64;
51
+ assertFiniteRange("maxFileSizeMB", maxFileSizeMB, Number.MIN_VALUE, 1024);
52
+ const defaultTolerance = config.defaultTolerance ?? 1e-9;
53
+ assertFiniteRange("defaultTolerance", defaultTolerance, 0, 1);
54
+ const evidenceRowLimit = config.evidenceRowLimit ?? 20;
55
+ assertPositiveInt("evidenceRowLimit", evidenceRowLimit);
56
+ const allowedExtensions = (config.allowedExtensions ?? [
57
+ ".csv",
58
+ ".tsv",
59
+ ".json",
60
+ ".jsonl"
61
+ ]).map((ext) => {
62
+ if (!/^\.[a-z0-9]+$/u.test(ext)) throw new TypeError(`allowedExtensions entries must be dot-prefixed lowercase extensions, got ${JSON.stringify(ext)}`);
63
+ return ext;
64
+ });
65
+ if (allowedExtensions.length === 0) throw new TypeError("allowedExtensions must not be empty");
66
+ return {
67
+ enabled: config.enabled ?? true,
68
+ maxRows,
69
+ maxFileSizeMB,
70
+ defaultTolerance,
71
+ evidenceRowLimit,
72
+ allowedExtensions,
73
+ workspaceRoot: config.workspaceRoot ?? "",
74
+ storeReports: config.storeReports ?? true
75
+ };
76
+ }
77
+ //#endregion
78
+ //#region src/dataset.ts
79
+ /**
80
+ * Dataset loading for `dsh-data-quality`: workspace-confined path resolution,
81
+ * size/row guards, and deterministic parsers for CSV/TSV/JSON/JSONL. All
82
+ * parsing is hand-rolled (no external runtime dependencies) and bounded by
83
+ * the resolved config; oversized inputs reject loudly with actionable hints.
84
+ * @module dsh-data-quality/dataset
85
+ */
86
+ /** Raised for every user-facing dataset failure; `code` is stable for tests and tooling. */
87
+ var DatasetError = class extends Error {
88
+ code;
89
+ /**
90
+ * @param code - stable machine-readable reason.
91
+ * @param message - actionable human-readable detail.
92
+ */
93
+ constructor(code, message) {
94
+ super(message);
95
+ this.code = code;
96
+ this.name = "DatasetError";
97
+ }
98
+ };
99
+ /**
100
+ * Resolve `requested` inside `root`, rejecting escapes and disallowed
101
+ * extensions. Both sides go through `path.resolve` before comparison so
102
+ * Windows backslash/forward-slash mixes never defeat the containment check.
103
+ * @param root - absolute workspace root.
104
+ * @param requested - the caller-supplied path (relative to root, or absolute inside root).
105
+ * @param config - resolved config (extension allowlist).
106
+ * @returns the normalized absolute path inside `root`.
107
+ */
108
+ function resolveWorkspacePath(root, requested, config) {
109
+ if (requested.trim() === "") throw new DatasetError("path-escape", "dataset path must not be empty");
110
+ const resolvedRoot = path.resolve(root);
111
+ const resolved = path.resolve(resolvedRoot, requested);
112
+ if (resolved !== resolvedRoot && !resolved.startsWith(resolvedRoot + path.sep)) throw new DatasetError("path-escape", `path ${JSON.stringify(requested)} resolves outside the workspace root ${JSON.stringify(resolvedRoot)}`);
113
+ const ext = path.extname(resolved).toLowerCase();
114
+ if (!config.allowedExtensions.includes(ext)) throw new DatasetError("extension-denied", `extension ${JSON.stringify(ext)} is not in allowedExtensions (${config.allowedExtensions.join(", ")})`);
115
+ return resolved;
116
+ }
117
+ /**
118
+ * Short deterministic fingerprint of a dataset path, used in storage keys.
119
+ * @param datasetPath - the workspace-relative dataset path as callers gave it.
120
+ * @returns an 8-hex-char fingerprint.
121
+ */
122
+ function pathFingerprint(datasetPath) {
123
+ return createHash("sha256").update(datasetPath).digest("hex").slice(0, 8);
124
+ }
125
+ /** Assert the abort signal has not fired; throws the signal reason otherwise. */
126
+ function throwIfAborted(signal) {
127
+ if (signal?.aborted === true) throw signal.reason instanceof Error ? signal.reason : /* @__PURE__ */ new Error("operation aborted");
128
+ }
129
+ /**
130
+ * Read a dataset file under the size cap.
131
+ * @param absolutePath - normalized absolute path (from {@link resolveWorkspacePath}).
132
+ * @param config - resolved config (size cap).
133
+ * @param signal - optional abort signal honored around the read.
134
+ * @returns the UTF-8 text.
135
+ */
136
+ async function readDatasetText(absolutePath, config, signal) {
137
+ throwIfAborted(signal);
138
+ let info;
139
+ try {
140
+ info = await stat(absolutePath);
141
+ } catch {
142
+ throw new DatasetError("not-found", `dataset file not found: ${absolutePath}`);
143
+ }
144
+ if (!info.isFile()) throw new DatasetError("not-found", `dataset path is not a file: ${absolutePath}`);
145
+ const capBytes = config.maxFileSizeMB * 1024 * 1024;
146
+ if (info.size > capBytes) throw new DatasetError("too-large", `dataset is ${(info.size / 1024 / 1024).toFixed(1)} MiB, above the ${config.maxFileSizeMB} MiB cap; split the file or raise maxFileSizeMB`);
147
+ const handle = await open(absolutePath, "r");
148
+ try {
149
+ throwIfAborted(signal);
150
+ return await handle.readFile("utf8");
151
+ } finally {
152
+ await handle.close();
153
+ }
154
+ }
155
+ /**
156
+ * Parse CSV/TSV text (RFC-4180-style: quoted fields, `""` escapes, CRLF/LF).
157
+ * The first record is the header; short records pad with `null`, long records
158
+ * fail loud. An empty field parses as `null` (missing).
159
+ * @param text - file text.
160
+ * @param delimiter - field delimiter (`,` or tab).
161
+ * @param config - resolved config (row cap).
162
+ * @param signal - optional abort signal, checked per 1024-record stride.
163
+ * @returns the parsed table.
164
+ */
165
+ function parseDelimited(text, delimiter, config, signal) {
166
+ const records = [];
167
+ let field = "";
168
+ let record = [];
169
+ let inQuotes = false;
170
+ const pushField = () => {
171
+ record.push(field);
172
+ field = "";
173
+ };
174
+ const pushRecord = () => {
175
+ pushField();
176
+ if (record.length === 1 && record[0] === "") {
177
+ record = [];
178
+ field = "";
179
+ return;
180
+ }
181
+ records.push(record);
182
+ record = [];
183
+ if (records.length % 1024 === 0) throwIfAborted(signal);
184
+ if (records.length - 1 > config.maxRows) throw new DatasetError("too-many-rows", `dataset exceeds the ${config.maxRows} row cap; use the sample parameter or raise maxRows`);
185
+ };
186
+ let index = 0;
187
+ while (index < text.length) {
188
+ const char = text[index];
189
+ if (inQuotes) {
190
+ if (char === "\"") {
191
+ if (text[index + 1] === "\"") {
192
+ field += "\"";
193
+ index += 2;
194
+ continue;
195
+ }
196
+ inQuotes = false;
197
+ index += 1;
198
+ continue;
199
+ }
200
+ field += char;
201
+ index += 1;
202
+ continue;
203
+ }
204
+ if (char === "\"") {
205
+ inQuotes = true;
206
+ index += 1;
207
+ continue;
208
+ }
209
+ if (char === delimiter) {
210
+ pushField();
211
+ index += 1;
212
+ continue;
213
+ }
214
+ if (char === "\r") {
215
+ index += 1;
216
+ continue;
217
+ }
218
+ if (char === "\n") {
219
+ pushRecord();
220
+ index += 1;
221
+ continue;
222
+ }
223
+ field += char;
224
+ index += 1;
225
+ }
226
+ if (field !== "" || record.length > 0) pushRecord();
227
+ const header = records[0];
228
+ if (header === void 0 || header.length === 1 && header[0] === "") throw new DatasetError("malformed", "dataset has no header row");
229
+ const columns = header.map((name) => name.trim());
230
+ if (columns.some((name) => name === "")) throw new DatasetError("malformed", "header contains an empty column name");
231
+ if (new Set(columns).size !== columns.length) throw new DatasetError("malformed", "header contains duplicate column names");
232
+ const rows = [];
233
+ for (let i = 1; i < records.length; i++) {
234
+ const cells = records[i];
235
+ if (cells.length > columns.length) throw new DatasetError("malformed", `row ${i + 1} has ${cells.length} fields but the header has ${columns.length}`);
236
+ const out = {};
237
+ for (let c = 0; c < columns.length; c++) {
238
+ const raw = cells[c];
239
+ out[columns[c]] = raw === void 0 || raw === "" ? null : raw;
240
+ }
241
+ rows.push(out);
242
+ }
243
+ return {
244
+ columns,
245
+ rows
246
+ };
247
+ }
248
+ /**
249
+ * Parse a JSON or JSONL text into a tabular dataset. JSON must be an array of
250
+ * flat objects; JSONL is one object per line. Non-scalar cell values are kept
251
+ * as-is (engines treat them as non-scalar).
252
+ * @param text - file text.
253
+ * @param extension - `.json` or `.jsonl`.
254
+ * @param config - resolved config (row cap).
255
+ * @param signal - optional abort signal.
256
+ * @returns the parsed table.
257
+ */
258
+ function parseJsonTable(text, extension, config, signal) {
259
+ const values = extension === ".jsonl" ? parseJsonLines(text) : parseJsonDocument(text);
260
+ const items = Array.isArray(values) ? values : [values];
261
+ if (items.length > config.maxRows) throw new DatasetError("too-many-rows", `dataset exceeds the ${config.maxRows} row cap; use the sample parameter or raise maxRows`);
262
+ const rows = [];
263
+ const columns = [];
264
+ const seen = /* @__PURE__ */ new Set();
265
+ for (const [index, item] of items.entries()) {
266
+ if (index % 1024 === 0) throwIfAborted(signal);
267
+ if (typeof item !== "object" || item === null || Array.isArray(item)) throw new DatasetError("not-tabular", `row ${index + 1} is not a flat object; profiling/cleaning/verification need a tabular dataset (array of objects)`);
268
+ const row = item;
269
+ for (const key of Object.keys(row)) if (!seen.has(key)) {
270
+ seen.add(key);
271
+ columns.push(key);
272
+ }
273
+ rows.push(row);
274
+ }
275
+ return {
276
+ columns,
277
+ rows: rows.map((row) => normalizeRow(row, columns))
278
+ };
279
+ }
280
+ /** Fill absent columns with `null` so every row carries every column. */
281
+ function normalizeRow(row, columns) {
282
+ const out = {};
283
+ for (const column of columns) {
284
+ const value = row[column];
285
+ out[column] = value === void 0 ? null : value;
286
+ }
287
+ return out;
288
+ }
289
+ /** Parse one JSON document; failures become a loud malformed error. */
290
+ function parseJsonDocument(text) {
291
+ try {
292
+ return JSON.parse(text);
293
+ } catch (error) {
294
+ throw new DatasetError("malformed", `invalid JSON: ${error instanceof Error ? error.message : String(error)}`);
295
+ }
296
+ }
297
+ /** Parse JSONL (one JSON value per non-empty line); failures name the line. */
298
+ function parseJsonLines(text) {
299
+ const out = [];
300
+ const lines = text.split(/\r?\n/u);
301
+ for (const [index, line] of lines.entries()) {
302
+ if (line.trim() === "") continue;
303
+ try {
304
+ out.push(JSON.parse(line));
305
+ } catch {
306
+ throw new DatasetError("malformed", `invalid JSON on line ${index + 1}`);
307
+ }
308
+ }
309
+ return out;
310
+ }
311
+ /**
312
+ * Load a tabular dataset from a file already resolved inside the workspace.
313
+ * @param absolutePath - normalized absolute dataset path.
314
+ * @param config - resolved config.
315
+ * @param signal - optional abort signal.
316
+ * @returns the parsed table.
317
+ */
318
+ async function loadTable(absolutePath, config, signal) {
319
+ const text = await readDatasetText(absolutePath, config, signal);
320
+ const ext = path.extname(absolutePath).toLowerCase();
321
+ throwIfAborted(signal);
322
+ if (ext === ".csv") return parseDelimited(text, ",", config, signal);
323
+ if (ext === ".tsv") return parseDelimited(text, " ", config, signal);
324
+ return parseJsonTable(text, ext, config, signal);
325
+ }
326
+ /**
327
+ * Load a dataset as a citation-checkable document root: CSV/TSV become
328
+ * `{ columns, rows }` (so `rows[3].nav` resolves), JSON is the parsed value,
329
+ * JSONL is the array of parsed lines.
330
+ * @param absolutePath - normalized absolute dataset path.
331
+ * @param config - resolved config.
332
+ * @param signal - optional abort signal.
333
+ * @returns the document root.
334
+ */
335
+ async function loadDocument(absolutePath, config, signal) {
336
+ const ext = path.extname(absolutePath).toLowerCase();
337
+ if (ext === ".json") return {
338
+ kind: "json",
339
+ value: parseJsonDocument(await readDatasetText(absolutePath, config, signal))
340
+ };
341
+ if (ext === ".jsonl") return {
342
+ kind: "json",
343
+ value: parseJsonLines(await readDatasetText(absolutePath, config, signal))
344
+ };
345
+ const table = await loadTable(absolutePath, config, signal);
346
+ return {
347
+ kind: "table",
348
+ columns: table.columns,
349
+ rows: table.rows
350
+ };
351
+ }
352
+ /**
353
+ * Deterministic systematic sample: every `ceil(total / count)`-th row from
354
+ * index 0. No clock, no RNG — the same input always yields the same sample.
355
+ * @param rows - full row list.
356
+ * @param count - requested sample size.
357
+ * @returns the sampled rows (identity when `count >= rows.length`).
358
+ */
359
+ function sampleRows(rows, count) {
360
+ if (!Number.isSafeInteger(count) || count <= 0) throw new DatasetError("malformed", `sample must be a positive integer, got ${String(count)}`);
361
+ if (count >= rows.length) return rows;
362
+ const stride = Math.ceil(rows.length / count);
363
+ const out = [];
364
+ for (let i = 0; i < rows.length; i += stride) out.push(rows[i]);
365
+ return out;
366
+ }
367
+ /**
368
+ * Missingness: `null`, `undefined`, or a string empty after trimming. A
369
+ * whitespace-only cell counts as missing even before any `trim` clean rule.
370
+ * @param cell - the cell to test (`undefined` when the column is absent).
371
+ * @returns whether the cell is missing.
372
+ */
373
+ function isMissing(cell) {
374
+ if (cell === null || cell === void 0) return true;
375
+ if (typeof cell === "string") return cell.trim() === "";
376
+ return false;
377
+ }
378
+ /** Strict full-string numeric parse (no thousands separators, no unit suffixes). */
379
+ function parseNumeric(cell) {
380
+ if (typeof cell === "number") return Number.isFinite(cell) ? cell : void 0;
381
+ if (typeof cell !== "string") return void 0;
382
+ const text = cell.trim();
383
+ if (text === "") return void 0;
384
+ if (!/^[+-]?(?:\d+\.?\d*|\.\d+)(?:[eE][+-]?\d+)?$/u.test(text)) return void 0;
385
+ const value = Number(text);
386
+ return Number.isFinite(value) ? value : void 0;
387
+ }
388
+ const DATE_PATTERNS = [
389
+ /^(\d{4})-(\d{1,2})-(\d{1,2})$/u,
390
+ /^(\d{4})\/(\d{1,2})\/(\d{1,2})$/u,
391
+ /^(\d{4})-(\d{1,2})-(\d{1,2})[ T](\d{1,2}):(\d{2})(?::(\d{2}))?(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?$/u
392
+ ];
393
+ /**
394
+ * Deterministic date parse to epoch milliseconds. Accepts `YYYY-MM-DD`,
395
+ * `YYYY/MM/DD`, and ISO-like datetimes (date-only forms read as UTC midnight).
396
+ * Calendar-invalid dates (e.g. 2025-13-40) reject. Returns `undefined` when
397
+ * the cell is not a recognized date.
398
+ * @param cell - the cell to parse (`undefined` when the column is absent).
399
+ * @returns epoch milliseconds, or `undefined`.
400
+ */
401
+ function parseDate(cell) {
402
+ if (typeof cell !== "string") return void 0;
403
+ const text = cell.trim();
404
+ for (const pattern of DATE_PATTERNS) {
405
+ const match = pattern.exec(text);
406
+ if (match === null) continue;
407
+ const year = Number(match[1]);
408
+ const month = Number(match[2]);
409
+ const day = Number(match[3]);
410
+ const hour = Number(match[4] ?? 0);
411
+ const minute = Number(match[5] ?? 0);
412
+ const second = Number(match[6] ?? 0);
413
+ if (month < 1 || month > 12 || day < 1 || day > 31 || hour > 23 || minute > 59 || second > 59) return void 0;
414
+ const epoch = Date.UTC(year, month - 1, day, hour, minute, second);
415
+ const check = new Date(epoch);
416
+ if (check.getUTCMonth() !== month - 1 || check.getUTCDate() !== day) return void 0;
417
+ return epoch;
418
+ }
419
+ }
420
+ /** Boolean parse: true/false/yes/no/1/0, case-insensitive. */
421
+ function parseBoolean(cell) {
422
+ if (typeof cell === "boolean") return cell;
423
+ if (typeof cell !== "string") return void 0;
424
+ const text = cell.trim().toLowerCase();
425
+ if (text === "true" || text === "yes" || text === "1") return true;
426
+ if (text === "false" || text === "no" || text === "0") return false;
427
+ }
428
+ //#endregion
429
+ //#region src/store.ts
430
+ /**
431
+ * Durable report storage over the harness storage domain. The `data_quality`
432
+ * domain keeps one record per profile/clean/verify/citations run, keyed by an
433
+ * injected timestamp plus a path fingerprint, so reports survive a restart
434
+ * and stay queryable without touching the session log.
435
+ * @module dsh-data-quality/store
436
+ */
437
+ /** Zod schema of one persisted report record (durable-boundary validation). */
438
+ const reportRecordSchema = z$1.object({
439
+ kind: z$1.enum([
440
+ "profile",
441
+ "clean",
442
+ "verify",
443
+ "citations"
444
+ ]),
445
+ at: z$1.number().int().nonnegative(),
446
+ dataset: z$1.string(),
447
+ report: z$1.record(z$1.string(), z$1.unknown())
448
+ });
449
+ /** The `dsh-data-quality` storage-domain declaration. */
450
+ const dataQualityDomainSpec = defineDomain({
451
+ name: "data_quality",
452
+ version: 1,
453
+ tables: { reports: domainTable(reportRecordSchema) }
454
+ });
455
+ /** Pad to two digits for the key timestamp. */
456
+ function pad2(value) {
457
+ return String(value).padStart(2, "0");
458
+ }
459
+ /**
460
+ * Build the storage key for one run: `YYYYMMDDHHmmssSSS-<kind>-<pathFp>`.
461
+ * UTC digits keep the key timezone-independent and filename-safe everywhere.
462
+ * @param record - the report envelope.
463
+ * @returns the deterministic key.
464
+ */
465
+ function reportKeyOf(record) {
466
+ const date = new Date(record.at);
467
+ return `${`${date.getUTCFullYear()}${pad2(date.getUTCMonth() + 1)}${pad2(date.getUTCDate())}${pad2(date.getUTCHours())}${pad2(date.getUTCMinutes())}${pad2(date.getUTCSeconds())}${String(date.getUTCMilliseconds()).padStart(3, "0")}`}-${record.kind}-${pathFingerprint(record.dataset)}`;
468
+ }
469
+ //#endregion
470
+ //#region src/clean.ts
471
+ /**
472
+ * Declarative cleaning rules over a parsed {@link Table}: dedupe, fill-missing,
473
+ * coerce-type, normalize-unit, trim, and map-values. Pure and deterministic —
474
+ * rules apply in array order, each seeing the previous rule's output; every
475
+ * rule logs how many rows it affected. Unknown columns and invalid rule
476
+ * parameters fail loud.
477
+ * @module dsh-data-quality/clean
478
+ */
479
+ /** Raised for invalid rules or unknown columns; `message` names the rule index and reason. */
480
+ var CleanRuleError = class extends Error {
481
+ ruleIndex;
482
+ /**
483
+ * @param ruleIndex - index of the offending rule in the request array.
484
+ * @param message - actionable human-readable detail.
485
+ */
486
+ constructor(ruleIndex, message) {
487
+ super(message);
488
+ this.ruleIndex = ruleIndex;
489
+ this.name = "CleanRuleError";
490
+ }
491
+ };
492
+ /** Assert `column` exists; every rule validates its columns before touching data. */
493
+ function requireColumn$1(columns, column, ruleIndex) {
494
+ if (!columns.includes(column)) throw new CleanRuleError(ruleIndex, `rule ${ruleIndex}: unknown column ${JSON.stringify(column)} (columns: ${columns.join(", ")})`);
495
+ }
496
+ /** Assert every selected column exists. */
497
+ function requireColumns(columns, selected, ruleIndex) {
498
+ for (const column of selected) requireColumn$1(columns, column, ruleIndex);
499
+ }
500
+ /** dedupe: keep the first row of each key group; later duplicates are removed. */
501
+ function applyDedupe(table, rule, ruleIndex, signal) {
502
+ const selected = rule.columns ?? table.columns;
503
+ requireColumns(table.columns, selected, ruleIndex);
504
+ const seen = /* @__PURE__ */ new Set();
505
+ const rows = [];
506
+ let removed = 0;
507
+ for (const [index, row] of table.rows.entries()) {
508
+ if (index % 1024 === 0) throwIfAborted(signal);
509
+ const key = JSON.stringify(selected.map((column) => row[column] ?? null));
510
+ if (seen.has(key)) {
511
+ removed += 1;
512
+ continue;
513
+ }
514
+ seen.add(key);
515
+ rows.push(row);
516
+ }
517
+ return {
518
+ table: {
519
+ columns: table.columns,
520
+ rows
521
+ },
522
+ log: {
523
+ ruleIndex,
524
+ rule: "dedupe",
525
+ affectedRows: removed,
526
+ detail: `removed ${removed} duplicate row(s) by [${selected.join(", ")}]`
527
+ }
528
+ };
529
+ }
530
+ /** fill-missing: constant / mean / median / forward fill of missing cells in one column. */
531
+ function applyFillMissing(table, rule, ruleIndex, signal) {
532
+ requireColumn$1(table.columns, rule.column, ruleIndex);
533
+ let fill;
534
+ let detail;
535
+ switch (rule.strategy) {
536
+ case "constant":
537
+ if (rule.value === void 0) throw new CleanRuleError(ruleIndex, `rule ${ruleIndex}: fill-missing strategy "constant" requires a value`);
538
+ fill = rule.value;
539
+ detail = `filled with constant ${JSON.stringify(rule.value)}`;
540
+ break;
541
+ case "mean":
542
+ case "median": {
543
+ const values = table.rows.map((row) => parseNumeric(row[rule.column])).filter((value) => value !== void 0).sort((a, b) => a - b);
544
+ if (values.length === 0) throw new CleanRuleError(ruleIndex, `rule ${ruleIndex}: fill-missing strategy "${rule.strategy}" needs at least one numeric value in column ${JSON.stringify(rule.column)}`);
545
+ const computed = rule.strategy === "mean" ? values.reduce((acc, value) => acc + value, 0) / values.length : values.length % 2 === 1 ? values[(values.length - 1) / 2] : (values[values.length / 2 - 1] + values[values.length / 2]) / 2;
546
+ fill = computed;
547
+ detail = `filled with ${rule.strategy} ${computed}`;
548
+ break;
549
+ }
550
+ case "forward":
551
+ fill = void 0;
552
+ detail = "filled with the previous non-missing value";
553
+ }
554
+ let filled = 0;
555
+ let previous;
556
+ const rows = table.rows.map((row, index) => {
557
+ if (index % 1024 === 0) throwIfAborted(signal);
558
+ const cell = row[rule.column];
559
+ if (!isMissing(cell)) {
560
+ previous = cell ?? null;
561
+ return row;
562
+ }
563
+ const replacement = rule.strategy === "forward" ? previous : fill;
564
+ if (replacement === void 0) return row;
565
+ filled += 1;
566
+ return {
567
+ ...row,
568
+ [rule.column]: replacement
569
+ };
570
+ });
571
+ return {
572
+ table: {
573
+ columns: table.columns,
574
+ rows
575
+ },
576
+ log: {
577
+ ruleIndex,
578
+ rule: "fill-missing",
579
+ affectedRows: filled,
580
+ detail: `${detail}; ${filled} row(s) filled`
581
+ }
582
+ };
583
+ }
584
+ /** coerce-type: rewrite one column to number / ISO date / boolean; failures become missing and are counted. */
585
+ function applyCoerceType(table, rule, ruleIndex, signal) {
586
+ requireColumn$1(table.columns, rule.column, ruleIndex);
587
+ let coerced = 0;
588
+ let failed = 0;
589
+ const failureExamples = [];
590
+ const rows = table.rows.map((row, index) => {
591
+ if (index % 1024 === 0) throwIfAborted(signal);
592
+ const cell = row[rule.column];
593
+ if (isMissing(cell)) return row;
594
+ let value;
595
+ switch (rule.to) {
596
+ case "number": {
597
+ const numeric = parseNumeric(cell);
598
+ value = numeric === void 0 ? void 0 : numeric;
599
+ break;
600
+ }
601
+ case "date": {
602
+ const epoch = parseDate(cell);
603
+ if (epoch === void 0) value = void 0;
604
+ else {
605
+ const date = new Date(epoch);
606
+ value = date.getUTCHours() === 0 && date.getUTCMinutes() === 0 && date.getUTCSeconds() === 0 ? date.toISOString().slice(0, 10) : date.toISOString();
607
+ }
608
+ break;
609
+ }
610
+ case "boolean": {
611
+ const bool = parseBoolean(cell);
612
+ value = bool === void 0 ? void 0 : String(bool);
613
+ break;
614
+ }
615
+ }
616
+ if (value === void 0) {
617
+ failed += 1;
618
+ if (failureExamples.length < 3) failureExamples.push(JSON.stringify(cell));
619
+ return {
620
+ ...row,
621
+ [rule.column]: null
622
+ };
623
+ }
624
+ coerced += 1;
625
+ return {
626
+ ...row,
627
+ [rule.column]: value
628
+ };
629
+ });
630
+ const detail = `${coerced} value(s) coerced to ${rule.to}` + (failed > 0 ? `; ${failed} failure(s) set to missing (e.g. ${failureExamples.join(", ")})` : "");
631
+ return {
632
+ table: {
633
+ columns: table.columns,
634
+ rows
635
+ },
636
+ log: {
637
+ ruleIndex,
638
+ rule: "coerce-type",
639
+ affectedRows: coerced,
640
+ detail
641
+ }
642
+ };
643
+ }
644
+ /** normalize-unit: strip a unit suffix and multiply by its factor (e.g. 万 -> 1e4). */
645
+ function applyNormalizeUnit(table, rule, ruleIndex, signal) {
646
+ requireColumn$1(table.columns, rule.column, ruleIndex);
647
+ const entries = Object.entries(rule.factors);
648
+ if (entries.length === 0) throw new CleanRuleError(ruleIndex, `rule ${ruleIndex}: normalize-unit requires a non-empty factors map`);
649
+ for (const [suffix, factor] of entries) {
650
+ if (suffix === "") throw new CleanRuleError(ruleIndex, `rule ${ruleIndex}: normalize-unit suffixes must not be empty`);
651
+ if (!Number.isFinite(factor) || factor <= 0) throw new CleanRuleError(ruleIndex, `rule ${ruleIndex}: factor for ${JSON.stringify(suffix)} must be a positive finite number, got ${String(factor)}`);
652
+ }
653
+ const ordered = [...entries].sort((a, b) => b[0].length - a[0].length);
654
+ let normalized = 0;
655
+ let skipped = 0;
656
+ const rows = table.rows.map((row, index) => {
657
+ if (index % 1024 === 0) throwIfAborted(signal);
658
+ const cell = row[rule.column];
659
+ if (isMissing(cell)) return row;
660
+ const plain = parseNumeric(cell);
661
+ if (plain !== void 0) {
662
+ normalized += 1;
663
+ return {
664
+ ...row,
665
+ [rule.column]: plain
666
+ };
667
+ }
668
+ if (typeof cell === "string") {
669
+ const text = cell.trim();
670
+ for (const [suffix, factor] of ordered) {
671
+ if (!text.endsWith(suffix)) continue;
672
+ const numeric = parseNumeric(text.slice(0, text.length - suffix.length));
673
+ if (numeric === void 0) continue;
674
+ normalized += 1;
675
+ return {
676
+ ...row,
677
+ [rule.column]: numeric * factor
678
+ };
679
+ }
680
+ }
681
+ skipped += 1;
682
+ return row;
683
+ });
684
+ const detail = `${normalized} value(s) normalized to base units` + (skipped > 0 ? `; ${skipped} unparseable value(s) left unchanged` : "");
685
+ return {
686
+ table: {
687
+ columns: table.columns,
688
+ rows
689
+ },
690
+ log: {
691
+ ruleIndex,
692
+ rule: "normalize-unit",
693
+ affectedRows: normalized,
694
+ detail
695
+ }
696
+ };
697
+ }
698
+ /** trim: strip leading/trailing whitespace of string cells in the selected columns. */
699
+ function applyTrim(table, rule, ruleIndex, signal) {
700
+ const selected = rule.columns ?? table.columns;
701
+ requireColumns(table.columns, selected, ruleIndex);
702
+ let trimmed = 0;
703
+ const rows = table.rows.map((row, index) => {
704
+ if (index % 1024 === 0) throwIfAborted(signal);
705
+ let changed = false;
706
+ const next = { ...row };
707
+ for (const column of selected) {
708
+ const cell = next[column];
709
+ if (typeof cell !== "string") continue;
710
+ const value = cell.trim();
711
+ if (value !== cell) {
712
+ next[column] = value;
713
+ changed = true;
714
+ }
715
+ }
716
+ if (changed) trimmed += 1;
717
+ return changed ? next : row;
718
+ });
719
+ return {
720
+ table: {
721
+ columns: table.columns,
722
+ rows
723
+ },
724
+ log: {
725
+ ruleIndex,
726
+ rule: "trim",
727
+ affectedRows: trimmed,
728
+ detail: `trimmed whitespace in [${selected.join(", ")}] for ${trimmed} row(s)`
729
+ }
730
+ };
731
+ }
732
+ /** map-values: exact-match enum mapping; unmapped values follow the `else` policy. */
733
+ function applyMapValues(table, rule, ruleIndex, signal) {
734
+ requireColumn$1(table.columns, rule.column, ruleIndex);
735
+ const elsePolicy = rule.else ?? "keep";
736
+ let mapped = 0;
737
+ let unmapped = 0;
738
+ const rows = table.rows.map((row, index) => {
739
+ if (index % 1024 === 0) throwIfAborted(signal);
740
+ const cell = row[rule.column];
741
+ if (isMissing(cell)) return row;
742
+ const key = typeof cell === "string" ? cell : typeof cell === "object" && cell !== null ? JSON.stringify(cell) : String(cell);
743
+ const replacement = rule.map[key];
744
+ if (replacement !== void 0) {
745
+ mapped += 1;
746
+ return {
747
+ ...row,
748
+ [rule.column]: replacement
749
+ };
750
+ }
751
+ if (elsePolicy === "missing") {
752
+ unmapped += 1;
753
+ return {
754
+ ...row,
755
+ [rule.column]: null
756
+ };
757
+ }
758
+ unmapped += 1;
759
+ return row;
760
+ });
761
+ const detail = `${mapped} value(s) mapped` + (unmapped > 0 ? `; ${unmapped} unmapped value(s) ${elsePolicy === "missing" ? "set to missing" : "kept"}` : "");
762
+ return {
763
+ table: {
764
+ columns: table.columns,
765
+ rows
766
+ },
767
+ log: {
768
+ ruleIndex,
769
+ rule: "map-values",
770
+ affectedRows: mapped,
771
+ detail
772
+ }
773
+ };
774
+ }
775
+ /**
776
+ * Apply cleaning rules in array order; each rule sees the previous output.
777
+ * @param table - the parsed dataset.
778
+ * @param rules - non-empty rule list.
779
+ * @param options - abort signal.
780
+ * @returns the cleaned table plus the per-rule audit log.
781
+ */
782
+ function applyCleanRules(table, rules, options = {}) {
783
+ if (rules.length === 0) throw new CleanRuleError(0, "rules must be a non-empty array");
784
+ let current = {
785
+ columns: [...table.columns],
786
+ rows: table.rows.map((row) => ({ ...row }))
787
+ };
788
+ const logs = [];
789
+ for (const [ruleIndex, rule] of rules.entries()) {
790
+ throwIfAborted(options.signal);
791
+ let step;
792
+ switch (rule.rule) {
793
+ case "dedupe":
794
+ step = applyDedupe(current, rule, ruleIndex, options.signal);
795
+ break;
796
+ case "fill-missing":
797
+ step = applyFillMissing(current, rule, ruleIndex, options.signal);
798
+ break;
799
+ case "coerce-type":
800
+ step = applyCoerceType(current, rule, ruleIndex, options.signal);
801
+ break;
802
+ case "normalize-unit":
803
+ step = applyNormalizeUnit(current, rule, ruleIndex, options.signal);
804
+ break;
805
+ case "trim":
806
+ step = applyTrim(current, rule, ruleIndex, options.signal);
807
+ break;
808
+ case "map-values":
809
+ step = applyMapValues(current, rule, ruleIndex, options.signal);
810
+ break;
811
+ default: throw new CleanRuleError(ruleIndex, `rule ${ruleIndex}: unknown rule type ${JSON.stringify(rule.rule)}`);
812
+ }
813
+ current = step.table;
814
+ logs.push(step.log);
815
+ }
816
+ return {
817
+ columns: current.columns,
818
+ rows: current.rows,
819
+ logs,
820
+ inputRows: table.rows.length,
821
+ outputRows: current.rows.length
822
+ };
823
+ }
824
+ /**
825
+ * Serialize a table back to CSV/TSV text with RFC-4180 quoting (used when a
826
+ * clean run writes an output file). Numbers and booleans render via `String`;
827
+ * missing cells render empty.
828
+ * @param columns - column order.
829
+ * @param rows - rows to serialize.
830
+ * @param delimiter - field delimiter.
831
+ * @returns the file text (LF newlines, trailing newline).
832
+ */
833
+ function serializeDelimited(columns, rows, delimiter) {
834
+ const encode = (cell) => {
835
+ if (isMissing(cell)) return "";
836
+ const text = typeof cell === "string" ? cell : typeof cell === "object" && cell !== null ? JSON.stringify(cell) : String(cell);
837
+ if (text.includes(delimiter) || text.includes("\"") || text.includes("\n") || text.includes("\r")) return `"${text.replace(/"/gu, "\"\"")}"`;
838
+ return text;
839
+ };
840
+ const lines = [columns.map((column) => encode(column)).join(delimiter)];
841
+ for (const row of rows) lines.push(columns.map((column) => encode(row[column])).join(delimiter));
842
+ return `${lines.join("\n")}\n`;
843
+ }
844
+ //#endregion
845
+ //#region src/profile.ts
846
+ /**
847
+ * Deterministic dataset profiling: per-column type inference, missingness,
848
+ * cardinality, numeric distribution, IQR outlier counts, and duplicate-row
849
+ * detection. Pure functions over a parsed {@link Table} — no clock, no RNG,
850
+ * no I/O; `generatedAt` is injected by the caller.
851
+ * @module dsh-data-quality/profile
852
+ */
853
+ /** Round to 6 significant digits for stable, readable report numbers. */
854
+ function round6(value) {
855
+ return Number(value.toPrecision(6));
856
+ }
857
+ /** Linear-interpolation quantile over an ascending-sorted array. */
858
+ function quantile(sorted, q) {
859
+ const index = (sorted.length - 1) * q;
860
+ const low = Math.floor(index);
861
+ const high = Math.ceil(index);
862
+ const lower = sorted[low];
863
+ return lower + (sorted[high] - lower) * (index - low);
864
+ }
865
+ /** Compute the numeric distribution of already-parsed values. */
866
+ function numericProfile(values) {
867
+ if (values.length === 0) return void 0;
868
+ const sorted = [...values].sort((a, b) => a - b);
869
+ const sum = sorted.reduce((acc, value) => acc + value, 0);
870
+ const p25 = quantile(sorted, .25);
871
+ const p75 = quantile(sorted, .75);
872
+ const iqr = p75 - p25;
873
+ const lowFence = p25 - 1.5 * iqr;
874
+ const highFence = p75 + 1.5 * iqr;
875
+ const outliers = iqr === 0 ? 0 : sorted.filter((value) => value < lowFence || value > highFence).length;
876
+ return {
877
+ min: round6(sorted[0]),
878
+ max: round6(sorted[sorted.length - 1]),
879
+ mean: round6(sum / sorted.length),
880
+ median: round6(quantile(sorted, .5)),
881
+ p25: round6(p25),
882
+ p75: round6(p75),
883
+ outliers
884
+ };
885
+ }
886
+ /** Count rows whose full content duplicates an earlier row (first occurrence is not counted). */
887
+ function countDuplicateRows(table, signal) {
888
+ const seen = /* @__PURE__ */ new Set();
889
+ let duplicates = 0;
890
+ for (const [index, row] of table.rows.entries()) {
891
+ if (index % 1024 === 0) throwIfAborted(signal);
892
+ const key = JSON.stringify(table.columns.map((column) => row[column] ?? null));
893
+ if (seen.has(key)) duplicates += 1;
894
+ else seen.add(key);
895
+ }
896
+ return duplicates;
897
+ }
898
+ /** Profile one column over the given rows. */
899
+ function profileColumn(rows, column, signal) {
900
+ let missing = 0;
901
+ let numbers = 0;
902
+ let dates = 0;
903
+ let booleans = 0;
904
+ let strings = 0;
905
+ const numericValues = [];
906
+ const distinct = /* @__PURE__ */ new Set();
907
+ const frequencies = /* @__PURE__ */ new Map();
908
+ for (const [index, row] of rows.entries()) {
909
+ if (index % 1024 === 0) throwIfAborted(signal);
910
+ const cell = row[column];
911
+ if (isMissing(cell)) {
912
+ missing += 1;
913
+ continue;
914
+ }
915
+ const text = typeof cell === "string" ? cell : JSON.stringify(cell);
916
+ distinct.add(text);
917
+ frequencies.set(text, (frequencies.get(text) ?? 0) + 1);
918
+ const numeric = parseNumeric(cell);
919
+ if (numeric !== void 0) {
920
+ numbers += 1;
921
+ numericValues.push(numeric);
922
+ continue;
923
+ }
924
+ if (parseDate(cell) !== void 0) {
925
+ dates += 1;
926
+ continue;
927
+ }
928
+ if (parseBoolean(cell) !== void 0) {
929
+ booleans += 1;
930
+ continue;
931
+ }
932
+ strings += 1;
933
+ }
934
+ const present = rows.length - missing;
935
+ const notes = [];
936
+ let inferredType;
937
+ if (present === 0) inferredType = "empty";
938
+ else if (numbers === present) inferredType = "number";
939
+ else if (dates === present) inferredType = "date";
940
+ else if (booleans === present) inferredType = "boolean";
941
+ else if (strings === present) inferredType = "string";
942
+ else {
943
+ inferredType = "mixed";
944
+ const parts = [
945
+ numbers > 0 ? `${numbers} numeric` : void 0,
946
+ dates > 0 ? `${dates} date` : void 0,
947
+ booleans > 0 ? `${booleans} boolean` : void 0,
948
+ strings > 0 ? `${strings} string` : void 0
949
+ ].filter((part) => part !== void 0);
950
+ notes.push(`mixed types among ${present} present values: ${parts.join(", ")}`);
951
+ }
952
+ const profile = {
953
+ name: column,
954
+ inferredType,
955
+ missing,
956
+ missingRate: rows.length === 0 ? 0 : round6(missing / rows.length),
957
+ unique: distinct.size,
958
+ notes
959
+ };
960
+ const numeric = inferredType === "number" ? numericProfile(numericValues) : void 0;
961
+ const topValues = (inferredType === "string" || inferredType === "boolean") && distinct.size > 0 && distinct.size <= 10 ? [...frequencies.entries()].sort((a, b) => b[1] - a[1] || (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0)).slice(0, 5).map(([value, count]) => ({
962
+ value,
963
+ count
964
+ })) : void 0;
965
+ return {
966
+ ...profile,
967
+ ...numeric !== void 0 ? { numeric } : {},
968
+ ...topValues !== void 0 ? { topValues } : {}
969
+ };
970
+ }
971
+ /**
972
+ * Profile a parsed table. Column cards are computed over a deterministic
973
+ * systematic sample when `sample` is given; row counts and duplicate counts
974
+ * always cover the full table.
975
+ * @param table - the parsed dataset.
976
+ * @param options - dataset label, optional sample size, injected timestamp, abort signal.
977
+ * @returns the profile report.
978
+ */
979
+ function profileTable(table, options) {
980
+ throwIfAborted(options.signal);
981
+ const profiled = options.sample === void 0 ? table.rows : sampleRows(table.rows, options.sample);
982
+ const columns = table.columns.map((column) => profileColumn(profiled, column, options.signal));
983
+ const duplicateRows = countDuplicateRows(table, options.signal);
984
+ return {
985
+ dataset: options.dataset,
986
+ rowCount: table.rows.length,
987
+ sampled: profiled.length !== table.rows.length,
988
+ profiledRows: profiled.length,
989
+ columnCount: table.columns.length,
990
+ duplicateRows,
991
+ columns,
992
+ generatedAt: options.generatedAt
993
+ };
994
+ }
995
+ /** Human-readable profile summary for the tool's Native render. */
996
+ function renderProfileText(report) {
997
+ const lines = [];
998
+ lines.push(`Profile of ${report.dataset}: ${report.rowCount} rows x ${report.columnCount} columns` + (report.sampled ? ` (column cards over a systematic sample of ${report.profiledRows} rows)` : ""));
999
+ if (report.duplicateRows > 0) lines.push(`Duplicate rows: ${report.duplicateRows}`);
1000
+ for (const column of report.columns) {
1001
+ const parts = [`${column.name}: ${column.inferredType}`];
1002
+ if (column.missing > 0) parts.push(`missing ${column.missing} (${(column.missingRate * 100).toFixed(1)}%)`);
1003
+ parts.push(`unique ${column.unique}`);
1004
+ if (column.numeric !== void 0) parts.push(`min ${column.numeric.min}, p25 ${column.numeric.p25}, median ${column.numeric.median}, p75 ${column.numeric.p75}, max ${column.numeric.max}, mean ${column.numeric.mean}` + (column.numeric.outliers > 0 ? `, ${column.numeric.outliers} IQR outliers` : ""));
1005
+ if (column.topValues !== void 0) parts.push(`top: ${column.topValues.map((entry) => `${JSON.stringify(entry.value)} x${entry.count}`).join(", ")}`);
1006
+ for (const note of column.notes) parts.push(`note: ${note}`);
1007
+ lines.push(`- ${parts.join("; ")}`);
1008
+ }
1009
+ return lines.join("\n");
1010
+ }
1011
+ //#endregion
1012
+ //#region src/verify.ts
1013
+ /**
1014
+ * Declarative verification rules over a parsed {@link Table} (not-null,
1015
+ * unique, range, regex, enum, cross-column, freshness) plus the frozen
1016
+ * citation-checking semantics for `ctx.dataQuality.verifyCitations`. Pure and
1017
+ * deterministic: the freshness clock is injected, numeric comparison never
1018
+ * uses raw `===`, and evidence rows are capped by config.
1019
+ * @module dsh-data-quality/verify
1020
+ */
1021
+ /** Raised for invalid rules; `message` names the rule index and reason. */
1022
+ var VerifyRuleError = class extends Error {
1023
+ ruleIndex;
1024
+ /**
1025
+ * @param ruleIndex - index of the offending rule in the request array.
1026
+ * @param message - actionable human-readable detail.
1027
+ */
1028
+ constructor(ruleIndex, message) {
1029
+ super(message);
1030
+ this.ruleIndex = ruleIndex;
1031
+ this.name = "VerifyRuleError";
1032
+ }
1033
+ };
1034
+ /** Assert `column` exists. */
1035
+ function requireColumn(columns, column, ruleIndex) {
1036
+ if (!columns.includes(column)) throw new VerifyRuleError(ruleIndex, `rule ${ruleIndex}: unknown column ${JSON.stringify(column)} (columns: ${columns.join(", ")})`);
1037
+ }
1038
+ /** The trimmed string form used for regex/enum/cross-column string comparison. */
1039
+ function textOf(cell) {
1040
+ if (typeof cell === "string") return cell.trim();
1041
+ if (cell === null || cell === void 0) return "";
1042
+ if (typeof cell === "object") return JSON.stringify(cell);
1043
+ return String(cell);
1044
+ }
1045
+ /** Run one per-row predicate and collect capped evidence. */
1046
+ function evaluate(table, ruleIndex, ruleName, columns, predicate, evidenceRowLimit, signal) {
1047
+ let failedCount = 0;
1048
+ const evidence = [];
1049
+ for (const [rowIndex, row] of table.rows.entries()) {
1050
+ if (rowIndex % 1024 === 0) throwIfAborted(signal);
1051
+ if (predicate(row)) continue;
1052
+ failedCount += 1;
1053
+ if (evidence.length < evidenceRowLimit) {
1054
+ const values = {};
1055
+ for (const column of columns) values[column] = row[column] ?? null;
1056
+ evidence.push({
1057
+ rowIndex,
1058
+ values
1059
+ });
1060
+ }
1061
+ }
1062
+ return {
1063
+ ruleIndex,
1064
+ rule: ruleName,
1065
+ passed: failedCount === 0,
1066
+ failedCount,
1067
+ checkedCount: table.rows.length,
1068
+ evidence
1069
+ };
1070
+ }
1071
+ const CROSS_OPS = [
1072
+ "<",
1073
+ "<=",
1074
+ "==",
1075
+ "!=",
1076
+ ">=",
1077
+ ">"
1078
+ ];
1079
+ /**
1080
+ * Apply verification rules over a parsed table. A missing cell fails every
1081
+ * rule that reads it. The overall `passed` is the conjunction of rule passes;
1082
+ * a failing dataset is a normal result, never a thrown error.
1083
+ * @param table - the parsed dataset.
1084
+ * @param rules - non-empty rule list.
1085
+ * @param options - evidence cap, injected clock for `freshness`, abort signal.
1086
+ * @returns the verify report (without the dataset label; the caller adds it).
1087
+ */
1088
+ function verifyTable(table, rules, options) {
1089
+ if (rules.length === 0) throw new VerifyRuleError(0, "rules must be a non-empty array");
1090
+ const results = [];
1091
+ for (const [ruleIndex, rule] of rules.entries()) {
1092
+ throwIfAborted(options.signal);
1093
+ let result;
1094
+ switch (rule.rule) {
1095
+ case "not-null":
1096
+ requireColumn(table.columns, rule.column, ruleIndex);
1097
+ result = evaluate(table, ruleIndex, rule.rule, [rule.column], (row) => !isMissing(row[rule.column]), options.evidenceRowLimit, options.signal);
1098
+ break;
1099
+ case "unique": {
1100
+ if (rule.columns.length === 0) throw new VerifyRuleError(ruleIndex, `rule ${ruleIndex}: unique requires at least one column`);
1101
+ for (const column of rule.columns) requireColumn(table.columns, column, ruleIndex);
1102
+ const counts = /* @__PURE__ */ new Map();
1103
+ for (const row of table.rows) {
1104
+ const key = JSON.stringify(rule.columns.map((column) => row[column] ?? null));
1105
+ counts.set(key, (counts.get(key) ?? 0) + 1);
1106
+ }
1107
+ result = evaluate(table, ruleIndex, rule.rule, rule.columns, (row) => (counts.get(JSON.stringify(rule.columns.map((column) => row[column] ?? null))) ?? 0) === 1, options.evidenceRowLimit, options.signal);
1108
+ break;
1109
+ }
1110
+ case "range":
1111
+ requireColumn(table.columns, rule.column, ruleIndex);
1112
+ if (rule.min === void 0 && rule.max === void 0) throw new VerifyRuleError(ruleIndex, `rule ${ruleIndex}: range requires at least one of min/max`);
1113
+ result = evaluate(table, ruleIndex, rule.rule, [rule.column], (row) => {
1114
+ const value = parseNumeric(row[rule.column]);
1115
+ if (value === void 0) return false;
1116
+ if (rule.min !== void 0 && value < rule.min) return false;
1117
+ if (rule.max !== void 0 && value > rule.max) return false;
1118
+ return true;
1119
+ }, options.evidenceRowLimit, options.signal);
1120
+ break;
1121
+ case "regex": {
1122
+ requireColumn(table.columns, rule.column, ruleIndex);
1123
+ let pattern;
1124
+ try {
1125
+ pattern = new RegExp(rule.pattern, rule.flags ?? "");
1126
+ } catch (error) {
1127
+ throw new VerifyRuleError(ruleIndex, `rule ${ruleIndex}: invalid regex ${JSON.stringify(rule.pattern)}: ${error instanceof Error ? error.message : String(error)}`);
1128
+ }
1129
+ result = evaluate(table, ruleIndex, rule.rule, [rule.column], (row) => {
1130
+ const cell = row[rule.column];
1131
+ if (isMissing(cell)) return false;
1132
+ pattern.lastIndex = 0;
1133
+ return pattern.test(textOf(cell));
1134
+ }, options.evidenceRowLimit, options.signal);
1135
+ break;
1136
+ }
1137
+ case "enum": {
1138
+ requireColumn(table.columns, rule.column, ruleIndex);
1139
+ if (rule.values.length === 0) throw new VerifyRuleError(ruleIndex, `rule ${ruleIndex}: enum requires a non-empty values array`);
1140
+ const allowed = new Set(rule.values);
1141
+ result = evaluate(table, ruleIndex, rule.rule, [rule.column], (row) => {
1142
+ const cell = row[rule.column];
1143
+ if (isMissing(cell)) return false;
1144
+ return allowed.has(textOf(cell));
1145
+ }, options.evidenceRowLimit, options.signal);
1146
+ break;
1147
+ }
1148
+ case "cross-column": {
1149
+ requireColumn(table.columns, rule.left, ruleIndex);
1150
+ if (!CROSS_OPS.includes(rule.op)) throw new VerifyRuleError(ruleIndex, `rule ${ruleIndex}: unsupported op ${JSON.stringify(rule.op)}`);
1151
+ const hasColumn = rule.rightColumn !== void 0;
1152
+ if (hasColumn === (rule.value !== void 0)) throw new VerifyRuleError(ruleIndex, `rule ${ruleIndex}: cross-column needs exactly one of rightColumn/value`);
1153
+ if (hasColumn) requireColumn(table.columns, rule.rightColumn, ruleIndex);
1154
+ const columns = hasColumn ? [rule.left, rule.rightColumn] : [rule.left];
1155
+ result = evaluate(table, ruleIndex, rule.rule, columns, (row) => {
1156
+ const left = row[rule.left];
1157
+ const right = hasColumn ? row[rule.rightColumn] : rule.value;
1158
+ if (isMissing(left) || isMissing(right)) return false;
1159
+ const leftNum = parseNumeric(left);
1160
+ const rightNum = parseNumeric(right);
1161
+ if (leftNum !== void 0 && rightNum !== void 0) switch (rule.op) {
1162
+ case "<": return leftNum < rightNum;
1163
+ case "<=": return leftNum <= rightNum;
1164
+ case ">": return leftNum > rightNum;
1165
+ case ">=": return leftNum >= rightNum;
1166
+ case "==": return Math.abs(leftNum - rightNum) <= 1e-12 * Math.max(Math.abs(leftNum), Math.abs(rightNum), 1);
1167
+ case "!=": return Math.abs(leftNum - rightNum) > 1e-12 * Math.max(Math.abs(leftNum), Math.abs(rightNum), 1);
1168
+ }
1169
+ const leftDate = parseDate(left);
1170
+ const rightDate = parseDate(right);
1171
+ if (leftDate !== void 0 && rightDate !== void 0) switch (rule.op) {
1172
+ case "<": return leftDate < rightDate;
1173
+ case "<=": return leftDate <= rightDate;
1174
+ case ">": return leftDate > rightDate;
1175
+ case ">=": return leftDate >= rightDate;
1176
+ case "==": return leftDate === rightDate;
1177
+ case "!=": return leftDate !== rightDate;
1178
+ }
1179
+ const leftText = textOf(left);
1180
+ const rightText = textOf(right);
1181
+ if (rule.op === "==") return leftText === rightText;
1182
+ if (rule.op === "!=") return leftText !== rightText;
1183
+ return false;
1184
+ }, options.evidenceRowLimit, options.signal);
1185
+ break;
1186
+ }
1187
+ case "freshness": {
1188
+ requireColumn(table.columns, rule.column, ruleIndex);
1189
+ if (!Number.isFinite(rule.maxAgeDays) || rule.maxAgeDays <= 0) throw new VerifyRuleError(ruleIndex, `rule ${ruleIndex}: freshness maxAgeDays must be a positive number, got ${String(rule.maxAgeDays)}`);
1190
+ let asOf;
1191
+ if (rule.asOf !== void 0) {
1192
+ const parsed = parseDate(rule.asOf);
1193
+ if (parsed === void 0) throw new VerifyRuleError(ruleIndex, `rule ${ruleIndex}: freshness asOf is not a recognized date: ${JSON.stringify(rule.asOf)}`);
1194
+ asOf = parsed;
1195
+ } else asOf = options.now();
1196
+ const maxAgeMs = rule.maxAgeDays * 864e5;
1197
+ result = evaluate(table, ruleIndex, rule.rule, [rule.column], (row) => {
1198
+ const epoch = parseDate(row[rule.column]);
1199
+ if (epoch === void 0) return false;
1200
+ return asOf - epoch <= maxAgeMs;
1201
+ }, options.evidenceRowLimit, options.signal);
1202
+ break;
1203
+ }
1204
+ default: throw new VerifyRuleError(ruleIndex, `rule ${ruleIndex}: unknown rule type ${JSON.stringify(rule.rule)}`);
1205
+ }
1206
+ results.push(result);
1207
+ }
1208
+ return {
1209
+ passed: results.every((result) => result.passed),
1210
+ rowCount: table.rows.length,
1211
+ rules: results,
1212
+ generatedAt: options.now()
1213
+ };
1214
+ }
1215
+ /** Human-readable verify summary for the tool's Native render. */
1216
+ function renderVerifyText(report) {
1217
+ const lines = [];
1218
+ lines.push(`Verification of ${report.dataset}: ${report.passed ? "PASSED" : "FAILED"} (${report.rowCount} rows, ${report.rules.length} rules)`);
1219
+ for (const rule of report.rules) {
1220
+ lines.push(`- [${rule.passed ? "pass" : "FAIL"}] rule ${rule.ruleIndex} (${rule.rule}): ${rule.failedCount}/${rule.checkedCount} row(s) failed`);
1221
+ for (const row of rule.evidence) {
1222
+ const cells = Object.entries(row.values).map(([column, value]) => `${column}=${JSON.stringify(value)}`).join(", ");
1223
+ lines.push(` row ${row.rowIndex}: ${cells}`);
1224
+ }
1225
+ if (rule.failedCount > rule.evidence.length) lines.push(` … and ${rule.failedCount - rule.evidence.length} more failing row(s)`);
1226
+ }
1227
+ return lines.join("\n");
1228
+ }
1229
+ /**
1230
+ * Parse a JSON-path-ish locator (`rows[3].nav`, `summary.annualReturn`,
1231
+ * `[0].value`) into walk segments. Throws on unparsable input.
1232
+ * @param locator - the citation's path string.
1233
+ * @returns ordered walk segments.
1234
+ */
1235
+ function parseLocator(locator) {
1236
+ if (locator.trim() === "") throw new Error("citation path must not be empty");
1237
+ const segments = [];
1238
+ for (const part of locator.split(".")) {
1239
+ const match = /^([A-Za-z0-9_$-]*)((?:\[\d+\])*)$/u.exec(part);
1240
+ if (match === null || match[1] === "" && match[2] === "") throw new Error(`unparsable citation path segment ${JSON.stringify(part)} in ${JSON.stringify(locator)}`);
1241
+ const indices = [...(match[2] ?? "").matchAll(/\[(\d+)\]/gu)].map((index) => Number(index[1]));
1242
+ segments.push({
1243
+ key: match[1] === "" ? void 0 : match[1],
1244
+ indices
1245
+ });
1246
+ }
1247
+ return segments;
1248
+ }
1249
+ /** Sentinel for a failed locator walk (distinguishable from an actual `undefined` value). */
1250
+ const NOT_FOUND = Symbol("not-found");
1251
+ /** Walk the document root along the parsed locator; returns {@link NOT_FOUND} when any step misses. */
1252
+ function walkLocator(root, segments) {
1253
+ let current = root.kind === "table" ? {
1254
+ columns: root.columns,
1255
+ rows: root.rows
1256
+ } : root.value;
1257
+ for (const segment of segments) {
1258
+ if (segment.key !== void 0) {
1259
+ if (typeof current !== "object" || current === null || Array.isArray(current)) return NOT_FOUND;
1260
+ if (!Object.prototype.hasOwnProperty.call(current, segment.key)) return NOT_FOUND;
1261
+ current = current[segment.key];
1262
+ }
1263
+ for (const index of segment.indices) {
1264
+ if (!Array.isArray(current) || index >= current.length) return NOT_FOUND;
1265
+ current = current[index];
1266
+ }
1267
+ }
1268
+ return current;
1269
+ }
1270
+ /** Relative-tolerance numeric comparison (never raw `===` on floats). */
1271
+ function numericClose(actual, cited, tolerance) {
1272
+ return Math.abs(actual - cited) <= tolerance * Math.max(Math.abs(actual), Math.abs(cited), Number.MIN_VALUE);
1273
+ }
1274
+ /** Display form of a located value for evidence notes (`undefined` when it cannot be rendered). */
1275
+ function displayActual(located) {
1276
+ if (typeof located === "string") return located;
1277
+ if (typeof located === "number" || typeof located === "boolean") return String(located);
1278
+ if (located === null) return "null";
1279
+ const rendered = JSON.stringify(located);
1280
+ return typeof rendered === "string" ? rendered : void 0;
1281
+ }
1282
+ /**
1283
+ * The frozen citation-checking semantics: locate each citation's `path` in the
1284
+ * dataset document, then compare. Numbers compare with relative `tolerance`
1285
+ * (falling back to `defaultTolerance`); strings compare exactly. A missing
1286
+ * location is `not-found`; an incomparable actual/cited type pair is
1287
+ * `unverifiable`.
1288
+ * @param root - the loaded dataset document.
1289
+ * @param citations - citations to check.
1290
+ * @param defaultTolerance - configured fallback relative tolerance.
1291
+ * @returns one result per citation, ids echoed.
1292
+ */
1293
+ function checkCitations(root, citations, defaultTolerance) {
1294
+ const results = [];
1295
+ for (const citation of citations) {
1296
+ let located;
1297
+ try {
1298
+ located = walkLocator(root, parseLocator(citation.path));
1299
+ } catch (error) {
1300
+ results.push({
1301
+ id: citation.id,
1302
+ status: "not-found",
1303
+ note: `path ${JSON.stringify(citation.path)} is not a valid locator: ${error instanceof Error ? error.message : String(error)}`
1304
+ });
1305
+ continue;
1306
+ }
1307
+ if (located === NOT_FOUND) {
1308
+ results.push({
1309
+ id: citation.id,
1310
+ status: "not-found",
1311
+ note: `nothing found at ${JSON.stringify(citation.path)}`
1312
+ });
1313
+ continue;
1314
+ }
1315
+ if (typeof citation.value === "number") {
1316
+ const actualNumber = typeof located === "number" ? located : parseNumeric(located);
1317
+ if (actualNumber === void 0) {
1318
+ const display = displayActual(located);
1319
+ results.push({
1320
+ id: citation.id,
1321
+ status: "unverifiable",
1322
+ ...display !== void 0 ? { actual: display } : {},
1323
+ note: `value at ${JSON.stringify(citation.path)} is not numeric and cannot be compared to ${citation.value}`
1324
+ });
1325
+ continue;
1326
+ }
1327
+ const tolerance = citation.tolerance ?? defaultTolerance;
1328
+ if (numericClose(actualNumber, citation.value, tolerance)) results.push({
1329
+ id: citation.id,
1330
+ status: "verified",
1331
+ actual: actualNumber,
1332
+ note: `within relative tolerance ${tolerance}`
1333
+ });
1334
+ else results.push({
1335
+ id: citation.id,
1336
+ status: "mismatch",
1337
+ actual: actualNumber,
1338
+ note: `cited ${citation.value} differs from actual ${actualNumber} beyond relative tolerance ${tolerance}`
1339
+ });
1340
+ continue;
1341
+ }
1342
+ if (typeof located === "string") {
1343
+ if (located === citation.value) results.push({
1344
+ id: citation.id,
1345
+ status: "verified",
1346
+ actual: located
1347
+ });
1348
+ else results.push({
1349
+ id: citation.id,
1350
+ status: "mismatch",
1351
+ actual: located,
1352
+ note: `cited ${JSON.stringify(citation.value)} != actual ${JSON.stringify(located)}`
1353
+ });
1354
+ continue;
1355
+ }
1356
+ const display = displayActual(located);
1357
+ results.push({
1358
+ id: citation.id,
1359
+ status: "unverifiable",
1360
+ ...display !== void 0 ? { actual: display } : {},
1361
+ note: `value at ${JSON.stringify(citation.path)} is ${typeof located}, not comparable to cited string`
1362
+ });
1363
+ }
1364
+ return { results };
1365
+ }
1366
+ //#endregion
1367
+ //#region src/events.ts
1368
+ /**
1369
+ * The `data-quality/*` session-event vocabulary and its adaptive append gate.
1370
+ * The vocabulary is declared on `SessionEventMap` for type-checked payloads;
1371
+ * the runtime gate decides per host whether appending is safe:
1372
+ *
1373
+ * - Hosts whose known-type set already covers the vocabulary (a future harness
1374
+ * that adopts these events) append plainly.
1375
+ * - Hosts with an `ignorable` append option (the master-build
1376
+ * `Session.append(type, data, { ignorable: true })` contract) append with the
1377
+ * marker, so builds that do not know the type skip the event on restore.
1378
+ * - 0.1.0-rc.6 hosts have neither a registration surface nor the `ignorable`
1379
+ * append flag; appending an unknown type there would make the persistence
1380
+ * coordinator refuse the session log on restore, so the append is skipped
1381
+ * and the storage-domain report remains the durable copy.
1382
+ * @module dsh-data-quality/events
1383
+ */
1384
+ /** The event type vocabulary this plugin appends. */
1385
+ const DATA_QUALITY_EVENT_TYPES = [
1386
+ "data-quality/profile",
1387
+ "data-quality/clean",
1388
+ "data-quality/verify"
1389
+ ];
1390
+ /**
1391
+ * Append one `data-quality/*` event when the host can carry it safely; skip
1392
+ * silently otherwise (the storage-domain report is always the durable copy).
1393
+ * The `ignorable` probe reads the UNBOUND method's source (a `.bind()` result
1394
+ * reports `[native code]`): the rc.6 build contains no `ignorable` handling
1395
+ * while the master build references the flag by name; property names survive
1396
+ * minification, so the probe fails safe (skips) rather than corrupting a log.
1397
+ * @param session - the calling session.
1398
+ * @param type - the event type.
1399
+ * @param data - the payload.
1400
+ */
1401
+ function appendDataQualityEvent(session, type, data) {
1402
+ if (KNOWN_SESSION_EVENT_TYPES.has(type)) {
1403
+ session.append(type, data);
1404
+ return;
1405
+ }
1406
+ const append = session.append;
1407
+ if (Function.prototype.toString.call(append).includes("ignorable")) append.call(session, type, data, { ignorable: true });
1408
+ }
1409
+ //#endregion
1410
+ //#region src/present.ts
1411
+ /** Maximum characters one cell contributes to a tool-facing row payload. */
1412
+ const MAX_CELL_TEXT = 120;
1413
+ /**
1414
+ * Truncate one cell for display: long strings are cut with an ellipsis
1415
+ * marker; non-strings pass through.
1416
+ * @param cell - the cell to bound.
1417
+ * @returns the display-safe cell.
1418
+ */
1419
+ function truncateCell(cell) {
1420
+ if (typeof cell === "string" && cell.length > 120) return `${cell.slice(0, 120)}…`;
1421
+ return cell;
1422
+ }
1423
+ /**
1424
+ * Truncate every string cell of one row.
1425
+ * @param row - the row to bound.
1426
+ * @returns a display-safe copy.
1427
+ */
1428
+ function truncateRow(row) {
1429
+ const out = {};
1430
+ for (const [key, value] of Object.entries(row)) out[key] = truncateCell(value);
1431
+ return out;
1432
+ }
1433
+ //#endregion
1434
+ //#region src/service.ts
1435
+ /**
1436
+ * Service Definition of the `dsh-data-quality` capability seam: the
1437
+ * `ctx.dataQuality` surface other plugins may optionally consume, plus the
1438
+ * internal request/report types the local Provider and the tool Consumers
1439
+ * share. The {@link CitationCheckRequest} / {@link CitationCheckResult}
1440
+ * citation contract is FROZEN for cross-plugin consumers — change it only
1441
+ * with a coordinated ecosystem migration.
1442
+ * @module dsh-data-quality/service
1443
+ */
1444
+ /**
1445
+ * The `ctx.dataQuality` service. The local Provider implements deterministic
1446
+ * TypeScript computation; Consumers (the three tools and third-party plugins)
1447
+ * never re-implement the math.
1448
+ */
1449
+ var DataQualityService = class extends Service {
1450
+ /**
1451
+ * Register as `dataQuality` on the context.
1452
+ * @param ctx - the plugin context.
1453
+ */
1454
+ constructor(ctx) {
1455
+ super(ctx, "dataQuality");
1456
+ }
1457
+ };
1458
+ //#endregion
1459
+ //#region src/provider-local.ts
1460
+ /**
1461
+ * The local deterministic Provider of the `ctx.dataQuality` seam: orchestrates
1462
+ * dataset loading, the pure engines, durable report persistence, and the
1463
+ * adaptive `data-quality/*` session events. All computation is TypeScript in
1464
+ * this process — no model arithmetic, no external processes.
1465
+ * @module dsh-data-quality/provider-local
1466
+ */
1467
+ /**
1468
+ * The local deterministic `dataQuality` implementation. Mounted by the plugin
1469
+ * entry; third-party plugins consume it through `ctx.dataQuality`.
1470
+ */
1471
+ var LocalDataQualityService = class extends DataQualityService {
1472
+ config;
1473
+ deps;
1474
+ /**
1475
+ * @param ctx - the plugin context.
1476
+ * @param config - the resolved plugin config.
1477
+ * @param deps - store handle plus the injected clock.
1478
+ */
1479
+ constructor(ctx, config, deps) {
1480
+ super(ctx);
1481
+ this.config = config;
1482
+ this.deps = deps;
1483
+ }
1484
+ /** The absolute root for service-level calls without a session workspace. */
1485
+ serviceRoot() {
1486
+ return this.config.workspaceRoot !== "" ? path.resolve(this.config.workspaceRoot) : process.cwd();
1487
+ }
1488
+ /** Persist one report when persistence is on; returns the storage key. */
1489
+ async persist(kind, dataset, report) {
1490
+ const store = this.deps.store;
1491
+ if (store === void 0) return void 0;
1492
+ const record = {
1493
+ kind,
1494
+ at: this.deps.now(),
1495
+ dataset,
1496
+ report
1497
+ };
1498
+ const key = reportKeyOf(record);
1499
+ await store.put({ ...record });
1500
+ return key;
1501
+ }
1502
+ /** Emit the adaptive session event when the call carries a session. */
1503
+ emitEvent(session, kind, dataset, reportKey, summary) {
1504
+ if (session === void 0) return;
1505
+ appendDataQualityEvent(session, `data-quality/${kind}`, {
1506
+ kind,
1507
+ dataset,
1508
+ ...reportKey !== void 0 ? { reportKey } : {},
1509
+ summary
1510
+ });
1511
+ }
1512
+ /** @inheritdoc DataQualityService.profileDataset */
1513
+ async profileDataset(request) {
1514
+ throwIfAborted(request.signal);
1515
+ const report = profileTable(await loadTable(resolveWorkspacePath(request.workspace, request.dataset, this.config), this.config, request.signal), {
1516
+ dataset: request.dataset,
1517
+ sample: request.sample,
1518
+ generatedAt: this.deps.now(),
1519
+ signal: request.signal
1520
+ });
1521
+ const reportKey = await this.persist("profile", request.dataset, report);
1522
+ this.emitEvent(request.session, "profile", request.dataset, reportKey, {
1523
+ rows: report.rowCount,
1524
+ columns: report.columnCount
1525
+ });
1526
+ return {
1527
+ ...report,
1528
+ ...reportKey !== void 0 ? { reportKey } : {}
1529
+ };
1530
+ }
1531
+ /** @inheritdoc DataQualityService.cleanDataset */
1532
+ async cleanDataset(request) {
1533
+ throwIfAborted(request.signal);
1534
+ const absolute = resolveWorkspacePath(request.workspace, request.dataset, this.config);
1535
+ const result = applyCleanRules(await loadTable(absolute, this.config, request.signal), request.rules, { signal: request.signal });
1536
+ let writtenPath;
1537
+ if (request.outputPath !== void 0) {
1538
+ const outputAbsolute = resolveWorkspacePath(request.workspace, request.outputPath, this.config);
1539
+ if (outputAbsolute === absolute) throw new Error(`outputPath ${JSON.stringify(request.outputPath)} would overwrite the input dataset; choose a different path`);
1540
+ const ext = path.extname(outputAbsolute).toLowerCase();
1541
+ const text = ext === ".csv" || ext === ".tsv" ? serializeDelimited(result.columns, result.rows, ext === ".csv" ? "," : " ") : ext === ".jsonl" ? `${result.rows.map((row) => JSON.stringify(row)).join("\n")}\n` : `${JSON.stringify(result.rows, null, 2)}\n`;
1542
+ await mkdir(path.dirname(outputAbsolute), { recursive: true });
1543
+ throwIfAborted(request.signal);
1544
+ await writeFile(outputAbsolute, text, "utf8");
1545
+ writtenPath = request.outputPath;
1546
+ }
1547
+ const generatedAt = this.deps.now();
1548
+ const reportKey = await this.persist("clean", request.dataset, {
1549
+ dataset: request.dataset,
1550
+ inputRows: result.inputRows,
1551
+ outputRows: result.outputRows,
1552
+ logs: result.logs,
1553
+ ...writtenPath !== void 0 ? { outputPath: writtenPath } : {},
1554
+ generatedAt
1555
+ });
1556
+ this.emitEvent(request.session, "clean", request.dataset, reportKey, {
1557
+ rows: result.outputRows,
1558
+ columns: result.columns.length,
1559
+ rules: result.logs.length
1560
+ });
1561
+ const previewRows = result.rows.slice(0, this.config.evidenceRowLimit).map((row) => truncateRow(row));
1562
+ return {
1563
+ dataset: request.dataset,
1564
+ inputRows: result.inputRows,
1565
+ outputRows: result.outputRows,
1566
+ logs: result.logs,
1567
+ preview: {
1568
+ columns: result.columns,
1569
+ rows: previewRows
1570
+ },
1571
+ ...writtenPath !== void 0 ? { outputPath: writtenPath } : {},
1572
+ ...reportKey !== void 0 ? { reportKey } : {},
1573
+ generatedAt
1574
+ };
1575
+ }
1576
+ /** @inheritdoc DataQualityService.verifyDataset */
1577
+ async verifyDataset(request) {
1578
+ throwIfAborted(request.signal);
1579
+ const outcome = verifyTable(await loadTable(resolveWorkspacePath(request.workspace, request.dataset, this.config), this.config, request.signal), request.rules, {
1580
+ evidenceRowLimit: this.config.evidenceRowLimit,
1581
+ now: this.deps.now,
1582
+ signal: request.signal
1583
+ });
1584
+ const report = {
1585
+ dataset: request.dataset,
1586
+ ...outcome
1587
+ };
1588
+ const failedRules = report.rules.filter((rule) => !rule.passed).length;
1589
+ const reportKey = await this.persist("verify", request.dataset, report);
1590
+ this.emitEvent(request.session, "verify", request.dataset, reportKey, {
1591
+ rows: report.rowCount,
1592
+ rules: report.rules.length,
1593
+ failedRules,
1594
+ passed: report.passed
1595
+ });
1596
+ return {
1597
+ ...report,
1598
+ ...reportKey !== void 0 ? { reportKey } : {}
1599
+ };
1600
+ }
1601
+ /** @inheritdoc DataQualityService.verifyCitations */
1602
+ async verifyCitations(request) {
1603
+ const result = checkCitations(await loadDocument(resolveWorkspacePath(this.serviceRoot(), request.dataset, this.config), this.config), request.citations, this.config.defaultTolerance);
1604
+ const verified = result.results.filter((entry) => entry.status === "verified").length;
1605
+ const mismatched = result.results.filter((entry) => entry.status === "mismatch").length;
1606
+ const notFound = result.results.filter((entry) => entry.status === "not-found").length;
1607
+ const unverifiable = result.results.filter((entry) => entry.status === "unverifiable").length;
1608
+ await this.persist("citations", request.dataset, {
1609
+ dataset: request.dataset,
1610
+ checked: result.results.length,
1611
+ verified,
1612
+ mismatched,
1613
+ notFound,
1614
+ unverifiable,
1615
+ results: result.results,
1616
+ generatedAt: this.deps.now()
1617
+ });
1618
+ return result;
1619
+ }
1620
+ };
1621
+ //#endregion
1622
+ //#region src/tools/shared.ts
1623
+ /**
1624
+ * Shared helpers for the three data-quality model tools: session workspace
1625
+ * resolution and a compact cleaning-run text render. Kept presentation-only;
1626
+ * computation lives in the engines and the provider.
1627
+ * @module dsh-data-quality/tools-shared
1628
+ */
1629
+ /**
1630
+ * The absolute workspace root a tool call resolves dataset paths against: the
1631
+ * calling agent's per-session cwd (mirroring the official fs tools). Non-agent
1632
+ * calls fail loud — there is no honest workspace to confine paths to.
1633
+ * @param exec - the tool-execution context.
1634
+ * @returns the absolute workspace root.
1635
+ */
1636
+ function workspaceOf(exec) {
1637
+ const cwd = exec.agent?.session.header.cwd;
1638
+ if (cwd === void 0) throw new Error("data-quality tools require an agent-owned session workspace");
1639
+ return path.resolve(cwd);
1640
+ }
1641
+ /** Human-readable cleaning summary for the tool's Native render. */
1642
+ function renderCleanText(report) {
1643
+ const lines = [];
1644
+ lines.push(`Cleaned ${report.dataset}: ${report.inputRows} -> ${report.outputRows} rows over ${report.logs.length} rule(s)`);
1645
+ for (const log of report.logs) lines.push(`- rule ${log.ruleIndex} (${log.rule}): ${log.affectedRows} row(s) affected; ${log.detail}`);
1646
+ if (report.outputPath !== void 0) lines.push(`Wrote cleaned dataset to ${report.outputPath}`);
1647
+ else lines.push("No outputPath given: the source file was left untouched; preview below.");
1648
+ if (report.preview.rows.length > 0) {
1649
+ lines.push(`Preview (first ${report.preview.rows.length} row(s)):`);
1650
+ for (const row of report.preview.rows) {
1651
+ const cells = report.preview.columns.map((column) => `${column}=${JSON.stringify(row[column] ?? null)}`).join(", ");
1652
+ lines.push(` ${cells}`);
1653
+ }
1654
+ }
1655
+ return lines.join("\n");
1656
+ }
1657
+ //#endregion
1658
+ //#region src/tools/profile.ts
1659
+ /**
1660
+ * The `data_profile` model tool: deterministic dataset profiling through
1661
+ * `ctx.dataQuality.profileDataset` — never model arithmetic.
1662
+ * @module dsh-data-quality/tools/profile
1663
+ */
1664
+ const COLUMN_PROFILE_SCHEMA = {
1665
+ type: "object",
1666
+ properties: {
1667
+ name: {
1668
+ type: "string",
1669
+ required: true
1670
+ },
1671
+ inferredType: {
1672
+ type: "string",
1673
+ enum: [
1674
+ "number",
1675
+ "date",
1676
+ "boolean",
1677
+ "string",
1678
+ "empty",
1679
+ "mixed"
1680
+ ],
1681
+ required: true
1682
+ },
1683
+ missing: {
1684
+ type: "number",
1685
+ required: true
1686
+ },
1687
+ missingRate: {
1688
+ type: "number",
1689
+ required: true
1690
+ },
1691
+ unique: {
1692
+ type: "number",
1693
+ required: true
1694
+ },
1695
+ numeric: {
1696
+ type: "object",
1697
+ properties: {
1698
+ min: {
1699
+ type: "number",
1700
+ required: true
1701
+ },
1702
+ max: {
1703
+ type: "number",
1704
+ required: true
1705
+ },
1706
+ mean: {
1707
+ type: "number",
1708
+ required: true
1709
+ },
1710
+ median: {
1711
+ type: "number",
1712
+ required: true
1713
+ },
1714
+ p25: {
1715
+ type: "number",
1716
+ required: true
1717
+ },
1718
+ p75: {
1719
+ type: "number",
1720
+ required: true
1721
+ },
1722
+ outliers: {
1723
+ type: "number",
1724
+ required: true
1725
+ }
1726
+ },
1727
+ additionalProperties: false
1728
+ },
1729
+ topValues: {
1730
+ type: "array",
1731
+ items: {
1732
+ type: "object",
1733
+ properties: {
1734
+ value: {
1735
+ type: "string",
1736
+ required: true
1737
+ },
1738
+ count: {
1739
+ type: "number",
1740
+ required: true
1741
+ }
1742
+ },
1743
+ additionalProperties: false
1744
+ }
1745
+ },
1746
+ notes: {
1747
+ type: "array",
1748
+ items: { type: "string" },
1749
+ required: true
1750
+ }
1751
+ },
1752
+ additionalProperties: false
1753
+ };
1754
+ /**
1755
+ * Build the `data_profile` tool definition against a mounted service.
1756
+ * @param service - the mounted ctx.dataQuality implementation.
1757
+ * @returns the tool definition to register.
1758
+ */
1759
+ function defineProfileTool(service) {
1760
+ return defineTool({
1761
+ name: "data_profile",
1762
+ description: [
1763
+ "Profile a workspace CSV/TSV/JSON/JSONL dataset with deterministic TypeScript computation (no mental math).",
1764
+ "Returns row/column counts, inferred column types, missing rates, unique counts, numeric distributions (min/max/mean/median/p25/p75), IQR outlier counts, mixed-type suspicion notes, and duplicate-row counts.",
1765
+ "Column cards cover every row by default; pass sample for a deterministic systematic sample on large files. Datasets above the configured row/size caps are rejected — use sample or raise the caps. The full report persists to the data_quality storage domain (reportKey in the result)."
1766
+ ].join("\n"),
1767
+ parameters: {
1768
+ path: {
1769
+ type: "string",
1770
+ required: true,
1771
+ description: "Workspace-relative dataset path (.csv/.tsv/.json/.jsonl). JSON datasets must be an array of flat objects."
1772
+ },
1773
+ sample: {
1774
+ type: "number",
1775
+ description: "Optional systematic sample size (every ceil(N/sample)-th row) for the column cards; row counts stay exact."
1776
+ }
1777
+ },
1778
+ output: {
1779
+ schema: {
1780
+ type: "object",
1781
+ properties: {
1782
+ dataset: {
1783
+ type: "string",
1784
+ required: true
1785
+ },
1786
+ rowCount: {
1787
+ type: "number",
1788
+ required: true
1789
+ },
1790
+ sampled: {
1791
+ type: "boolean",
1792
+ required: true
1793
+ },
1794
+ profiledRows: {
1795
+ type: "number",
1796
+ required: true
1797
+ },
1798
+ columnCount: {
1799
+ type: "number",
1800
+ required: true
1801
+ },
1802
+ duplicateRows: {
1803
+ type: "number",
1804
+ required: true
1805
+ },
1806
+ generatedAt: {
1807
+ type: "number",
1808
+ required: true
1809
+ },
1810
+ reportKey: { type: "string" },
1811
+ columns: {
1812
+ type: "array",
1813
+ items: COLUMN_PROFILE_SCHEMA,
1814
+ required: true
1815
+ }
1816
+ },
1817
+ additionalProperties: false
1818
+ },
1819
+ render: (_args, value) => [{
1820
+ type: "text",
1821
+ text: renderProfileText(value)
1822
+ }]
1823
+ },
1824
+ async execute(args, exec) {
1825
+ return service.profileDataset({
1826
+ dataset: args.path,
1827
+ sample: args.sample,
1828
+ workspace: workspaceOf(exec),
1829
+ session: exec.agent?.session,
1830
+ signal: exec.signal
1831
+ });
1832
+ }
1833
+ });
1834
+ }
1835
+ //#endregion
1836
+ //#region src/tools/clean.ts
1837
+ /**
1838
+ * The `data_clean` model tool: declarative cleaning rules through
1839
+ * `ctx.dataQuality.cleanDataset`. The source file is never overwritten;
1840
+ * without `outputPath` the run is preview-only.
1841
+ * @module dsh-data-quality/tools/clean
1842
+ */
1843
+ const STRING_MAP_SCHEMA = { type: "json" };
1844
+ const CLEAN_RULE_SCHEMA = {
1845
+ type: "array",
1846
+ items: { oneOf: [
1847
+ {
1848
+ type: "object",
1849
+ properties: {
1850
+ rule: {
1851
+ type: "string",
1852
+ const: "dedupe",
1853
+ required: true
1854
+ },
1855
+ columns: {
1856
+ type: "array",
1857
+ items: { type: "string" },
1858
+ description: "Key columns; all columns when omitted."
1859
+ }
1860
+ },
1861
+ additionalProperties: false,
1862
+ description: "Remove rows whose key-column values duplicate an earlier row (first occurrence kept)."
1863
+ },
1864
+ {
1865
+ type: "object",
1866
+ properties: {
1867
+ rule: {
1868
+ type: "string",
1869
+ const: "fill-missing",
1870
+ required: true
1871
+ },
1872
+ column: {
1873
+ type: "string",
1874
+ required: true
1875
+ },
1876
+ strategy: {
1877
+ type: "string",
1878
+ enum: [
1879
+ "constant",
1880
+ "mean",
1881
+ "median",
1882
+ "forward"
1883
+ ],
1884
+ required: true
1885
+ },
1886
+ value: {
1887
+ oneOf: [{ type: "string" }, { type: "number" }],
1888
+ description: "Required for strategy \"constant\"."
1889
+ }
1890
+ },
1891
+ additionalProperties: false,
1892
+ description: "Fill missing cells: a constant, the column mean/median (numeric columns), or the previous non-missing value."
1893
+ },
1894
+ {
1895
+ type: "object",
1896
+ properties: {
1897
+ rule: {
1898
+ type: "string",
1899
+ const: "coerce-type",
1900
+ required: true
1901
+ },
1902
+ column: {
1903
+ type: "string",
1904
+ required: true
1905
+ },
1906
+ to: {
1907
+ type: "string",
1908
+ enum: [
1909
+ "number",
1910
+ "date",
1911
+ "boolean"
1912
+ ],
1913
+ required: true
1914
+ }
1915
+ },
1916
+ additionalProperties: false,
1917
+ description: "Coerce cells to number / ISO date / boolean; failures become missing and are counted in the log."
1918
+ },
1919
+ {
1920
+ type: "object",
1921
+ properties: {
1922
+ rule: {
1923
+ type: "string",
1924
+ const: "normalize-unit",
1925
+ required: true
1926
+ },
1927
+ column: {
1928
+ type: "string",
1929
+ required: true
1930
+ },
1931
+ factors: {
1932
+ ...STRING_MAP_SCHEMA,
1933
+ description: "Unit suffix to multiplier, e.g. {\"万\": 10000, \"亿\": 100000000}."
1934
+ }
1935
+ },
1936
+ additionalProperties: false,
1937
+ description: "Strip a unit suffix and multiply by its factor; plain numeric values convert too."
1938
+ },
1939
+ {
1940
+ type: "object",
1941
+ properties: {
1942
+ rule: {
1943
+ type: "string",
1944
+ const: "trim",
1945
+ required: true
1946
+ },
1947
+ columns: {
1948
+ type: "array",
1949
+ items: { type: "string" },
1950
+ description: "Columns to trim; all columns when omitted."
1951
+ }
1952
+ },
1953
+ additionalProperties: false,
1954
+ description: "Trim leading/trailing whitespace of string cells."
1955
+ },
1956
+ {
1957
+ type: "object",
1958
+ properties: {
1959
+ rule: {
1960
+ type: "string",
1961
+ const: "map-values",
1962
+ required: true
1963
+ },
1964
+ column: {
1965
+ type: "string",
1966
+ required: true
1967
+ },
1968
+ map: {
1969
+ ...STRING_MAP_SCHEMA,
1970
+ description: "Exact-match value mapping, e.g. {\"Y\": \"yes\", \"N\": \"no\"}."
1971
+ },
1972
+ else: {
1973
+ type: "string",
1974
+ enum: ["keep", "missing"],
1975
+ description: "Policy for unmapped values (default keep)."
1976
+ }
1977
+ },
1978
+ additionalProperties: false,
1979
+ description: "Map enum values by exact match."
1980
+ }
1981
+ ] },
1982
+ description: "Ordered cleaning rules; each rule sees the previous rule's output."
1983
+ };
1984
+ /**
1985
+ * Build the `data_clean` tool definition against a mounted service.
1986
+ * @param service - the mounted ctx.dataQuality implementation.
1987
+ * @returns the tool definition to register.
1988
+ */
1989
+ function defineCleanTool(service) {
1990
+ return defineTool({
1991
+ name: "data_clean",
1992
+ description: [
1993
+ "Apply declarative cleaning rules to a workspace CSV/TSV/JSON/JSONL dataset with deterministic TypeScript computation (no mental math).",
1994
+ "Rules apply in array order: dedupe (by column group), fill-missing (constant/mean/median/forward), coerce-type (number/date/boolean; failures counted and set to missing), normalize-unit (e.g. 万/亿 suffixes to base units), trim (whitespace), map-values (enum mapping).",
1995
+ "The source file is NEVER overwritten. Without outputPath the run is preview-only; with outputPath the cleaned dataset is written there (workspace-confined, .csv/.tsv/.json/.jsonl). Returns the per-rule audit log (affected rows per rule) plus a bounded preview. The full report persists to the data_quality storage domain (reportKey)."
1996
+ ].join("\n"),
1997
+ parameters: {
1998
+ path: {
1999
+ type: "string",
2000
+ required: true,
2001
+ description: "Workspace-relative dataset path (.csv/.tsv/.json/.jsonl)."
2002
+ },
2003
+ rules: {
2004
+ ...CLEAN_RULE_SCHEMA,
2005
+ required: true
2006
+ },
2007
+ outputPath: {
2008
+ type: "string",
2009
+ description: "Optional workspace-relative output path for the cleaned dataset (must differ from path)."
2010
+ }
2011
+ },
2012
+ output: {
2013
+ schema: {
2014
+ type: "object",
2015
+ properties: {
2016
+ dataset: {
2017
+ type: "string",
2018
+ required: true
2019
+ },
2020
+ inputRows: {
2021
+ type: "number",
2022
+ required: true
2023
+ },
2024
+ outputRows: {
2025
+ type: "number",
2026
+ required: true
2027
+ },
2028
+ generatedAt: {
2029
+ type: "number",
2030
+ required: true
2031
+ },
2032
+ outputPath: { type: "string" },
2033
+ reportKey: { type: "string" },
2034
+ logs: {
2035
+ type: "array",
2036
+ items: {
2037
+ type: "object",
2038
+ properties: {
2039
+ ruleIndex: {
2040
+ type: "number",
2041
+ required: true
2042
+ },
2043
+ rule: {
2044
+ type: "string",
2045
+ enum: [
2046
+ "dedupe",
2047
+ "fill-missing",
2048
+ "coerce-type",
2049
+ "normalize-unit",
2050
+ "trim",
2051
+ "map-values"
2052
+ ],
2053
+ required: true
2054
+ },
2055
+ affectedRows: {
2056
+ type: "number",
2057
+ required: true
2058
+ },
2059
+ detail: {
2060
+ type: "string",
2061
+ required: true
2062
+ }
2063
+ },
2064
+ additionalProperties: false
2065
+ },
2066
+ required: true
2067
+ },
2068
+ preview: {
2069
+ type: "object",
2070
+ properties: {
2071
+ columns: {
2072
+ type: "array",
2073
+ items: { type: "string" },
2074
+ required: true
2075
+ },
2076
+ rows: {
2077
+ type: "array",
2078
+ items: { type: "json" },
2079
+ required: true
2080
+ }
2081
+ },
2082
+ additionalProperties: false,
2083
+ required: true
2084
+ }
2085
+ },
2086
+ additionalProperties: false
2087
+ },
2088
+ render: (_args, value) => [{
2089
+ type: "text",
2090
+ text: renderCleanText(value)
2091
+ }]
2092
+ },
2093
+ async execute(args, exec) {
2094
+ return service.cleanDataset({
2095
+ dataset: args.path,
2096
+ rules: args.rules,
2097
+ ...args.outputPath !== void 0 ? { outputPath: args.outputPath } : {},
2098
+ workspace: workspaceOf(exec),
2099
+ session: exec.agent?.session,
2100
+ signal: exec.signal
2101
+ });
2102
+ }
2103
+ });
2104
+ }
2105
+ //#endregion
2106
+ //#region src/tools/verify.ts
2107
+ /**
2108
+ * The `data_verify` model tool: declarative verification rules through
2109
+ * `ctx.dataQuality.verifyDataset`. A failing dataset is a normal result
2110
+ * (`passed: false` with evidence), never a tool error.
2111
+ * @module dsh-data-quality/tools/verify
2112
+ */
2113
+ const VERIFY_RULE_SCHEMA = {
2114
+ type: "array",
2115
+ items: { oneOf: [
2116
+ {
2117
+ type: "object",
2118
+ properties: {
2119
+ rule: {
2120
+ type: "string",
2121
+ const: "not-null",
2122
+ required: true
2123
+ },
2124
+ column: {
2125
+ type: "string",
2126
+ required: true
2127
+ }
2128
+ },
2129
+ additionalProperties: false,
2130
+ description: "Fail rows where the column is missing (null/empty/whitespace)."
2131
+ },
2132
+ {
2133
+ type: "object",
2134
+ properties: {
2135
+ rule: {
2136
+ type: "string",
2137
+ const: "unique",
2138
+ required: true
2139
+ },
2140
+ columns: {
2141
+ type: "array",
2142
+ items: { type: "string" },
2143
+ required: true,
2144
+ description: "Key columns; every row whose key repeats fails."
2145
+ }
2146
+ },
2147
+ additionalProperties: false,
2148
+ description: "Fail rows whose key-column combination appears more than once."
2149
+ },
2150
+ {
2151
+ type: "object",
2152
+ properties: {
2153
+ rule: {
2154
+ type: "string",
2155
+ const: "range",
2156
+ required: true
2157
+ },
2158
+ column: {
2159
+ type: "string",
2160
+ required: true
2161
+ },
2162
+ min: { type: "number" },
2163
+ max: { type: "number" }
2164
+ },
2165
+ additionalProperties: false,
2166
+ description: "Fail rows where the numeric cell is missing, unparseable, or outside [min, max] (inclusive; at least one bound required)."
2167
+ },
2168
+ {
2169
+ type: "object",
2170
+ properties: {
2171
+ rule: {
2172
+ type: "string",
2173
+ const: "regex",
2174
+ required: true
2175
+ },
2176
+ column: {
2177
+ type: "string",
2178
+ required: true
2179
+ },
2180
+ pattern: {
2181
+ type: "string",
2182
+ required: true
2183
+ },
2184
+ flags: {
2185
+ type: "string",
2186
+ description: "Optional JS regex flags, e.g. \"i\"."
2187
+ }
2188
+ },
2189
+ additionalProperties: false,
2190
+ description: "Fail rows where the cell is missing or does not match the pattern (full JS regex semantics)."
2191
+ },
2192
+ {
2193
+ type: "object",
2194
+ properties: {
2195
+ rule: {
2196
+ type: "string",
2197
+ const: "enum",
2198
+ required: true
2199
+ },
2200
+ column: {
2201
+ type: "string",
2202
+ required: true
2203
+ },
2204
+ values: {
2205
+ type: "array",
2206
+ items: { type: "string" },
2207
+ required: true
2208
+ }
2209
+ },
2210
+ additionalProperties: false,
2211
+ description: "Fail rows where the trimmed cell text is not one of the allowed values."
2212
+ },
2213
+ {
2214
+ type: "object",
2215
+ properties: {
2216
+ rule: {
2217
+ type: "string",
2218
+ const: "cross-column",
2219
+ required: true
2220
+ },
2221
+ left: {
2222
+ type: "string",
2223
+ required: true,
2224
+ description: "Left column name."
2225
+ },
2226
+ op: {
2227
+ type: "string",
2228
+ enum: [
2229
+ "<",
2230
+ "<=",
2231
+ "==",
2232
+ "!=",
2233
+ ">=",
2234
+ ">"
2235
+ ],
2236
+ required: true
2237
+ },
2238
+ rightColumn: {
2239
+ type: "string",
2240
+ description: "Compare against another column…"
2241
+ },
2242
+ value: {
2243
+ oneOf: [{ type: "number" }, { type: "string" }],
2244
+ description: "…or against this literal (exactly one of rightColumn/value)."
2245
+ }
2246
+ },
2247
+ additionalProperties: false,
2248
+ description: "Compare two columns (e.g. startDate < endDate) or a column against a literal. Numeric when both sides parse, string equality for ==/!= otherwise."
2249
+ },
2250
+ {
2251
+ type: "object",
2252
+ properties: {
2253
+ rule: {
2254
+ type: "string",
2255
+ const: "freshness",
2256
+ required: true
2257
+ },
2258
+ column: {
2259
+ type: "string",
2260
+ required: true,
2261
+ description: "Date column (YYYY-MM-DD, YYYY/MM/DD, or ISO datetime)."
2262
+ },
2263
+ maxAgeDays: {
2264
+ type: "number",
2265
+ required: true,
2266
+ description: "Fail rows whose date is older than this many days before asOf."
2267
+ },
2268
+ asOf: {
2269
+ type: "string",
2270
+ description: "Reference date; defaults to the current date."
2271
+ }
2272
+ },
2273
+ additionalProperties: false,
2274
+ description: "Fail rows whose date cell is missing, unparseable, or older than maxAgeDays before asOf."
2275
+ }
2276
+ ] },
2277
+ description: "Verification rules; a missing cell fails every rule that reads it."
2278
+ };
2279
+ /**
2280
+ * Build the `data_verify` tool definition against a mounted service.
2281
+ * @param service - the mounted ctx.dataQuality implementation.
2282
+ * @returns the tool definition to register.
2283
+ */
2284
+ function defineVerifyTool(service) {
2285
+ return defineTool({
2286
+ name: "data_verify",
2287
+ description: [
2288
+ "Verify a workspace CSV/TSV/JSON/JSONL dataset against declarative quality rules with deterministic TypeScript computation (no mental math).",
2289
+ "Rules: not-null, unique (column group), range (numeric bounds), regex, enum, cross-column (e.g. startDate < endDate), freshness (date column within N days of asOf). A missing cell fails every rule that reads it.",
2290
+ "Returns per-rule pass/fail with capped failing-row evidence. Overall failure is a NORMAL result with passed: false — not a tool error. The full report persists to the data_quality storage domain (reportKey)."
2291
+ ].join("\n"),
2292
+ parameters: {
2293
+ path: {
2294
+ type: "string",
2295
+ required: true,
2296
+ description: "Workspace-relative dataset path (.csv/.tsv/.json/.jsonl)."
2297
+ },
2298
+ rules: {
2299
+ ...VERIFY_RULE_SCHEMA,
2300
+ required: true
2301
+ }
2302
+ },
2303
+ output: {
2304
+ schema: {
2305
+ type: "object",
2306
+ properties: {
2307
+ dataset: {
2308
+ type: "string",
2309
+ required: true
2310
+ },
2311
+ passed: {
2312
+ type: "boolean",
2313
+ required: true
2314
+ },
2315
+ rowCount: {
2316
+ type: "number",
2317
+ required: true
2318
+ },
2319
+ generatedAt: {
2320
+ type: "number",
2321
+ required: true
2322
+ },
2323
+ reportKey: { type: "string" },
2324
+ rules: {
2325
+ type: "array",
2326
+ items: {
2327
+ type: "object",
2328
+ properties: {
2329
+ ruleIndex: {
2330
+ type: "number",
2331
+ required: true
2332
+ },
2333
+ rule: {
2334
+ type: "string",
2335
+ enum: [
2336
+ "not-null",
2337
+ "unique",
2338
+ "range",
2339
+ "regex",
2340
+ "enum",
2341
+ "cross-column",
2342
+ "freshness"
2343
+ ],
2344
+ required: true
2345
+ },
2346
+ passed: {
2347
+ type: "boolean",
2348
+ required: true
2349
+ },
2350
+ failedCount: {
2351
+ type: "number",
2352
+ required: true
2353
+ },
2354
+ checkedCount: {
2355
+ type: "number",
2356
+ required: true
2357
+ },
2358
+ evidence: {
2359
+ type: "array",
2360
+ items: {
2361
+ type: "object",
2362
+ properties: {
2363
+ rowIndex: {
2364
+ type: "number",
2365
+ required: true
2366
+ },
2367
+ values: {
2368
+ type: "json",
2369
+ required: true
2370
+ }
2371
+ },
2372
+ additionalProperties: false
2373
+ },
2374
+ required: true
2375
+ }
2376
+ },
2377
+ additionalProperties: false
2378
+ },
2379
+ required: true
2380
+ }
2381
+ },
2382
+ additionalProperties: false
2383
+ },
2384
+ render: (_args, value) => [{
2385
+ type: "text",
2386
+ text: renderVerifyText(value)
2387
+ }]
2388
+ },
2389
+ async execute(args, exec) {
2390
+ return service.verifyDataset({
2391
+ dataset: args.path,
2392
+ rules: args.rules,
2393
+ workspace: workspaceOf(exec),
2394
+ session: exec.agent?.session,
2395
+ signal: exec.signal
2396
+ });
2397
+ }
2398
+ });
2399
+ }
2400
+ //#endregion
2401
+ //#region src/version.ts
2402
+ /**
2403
+ * Plugin version, kept in one place so `scripts/release.mjs` can stamp it and
2404
+ * reports can name their generator.
2405
+ * @module dsh-data-quality/version
2406
+ */
2407
+ /** The package version reported in persisted reports. */
2408
+ const VERSION = "0.1.0";
2409
+ //#endregion
2410
+ //#region src/index.ts
2411
+ const name = "data-quality";
2412
+ /** The three model tools and the durable report domain. */
2413
+ const inject = ["tools", "storageDomain"];
2414
+ /**
2415
+ * Mount the seam: resolve config (fail loud), open the report domain, publish
2416
+ * `ctx.dataQuality`, and register the three tools. With `enabled: false` the
2417
+ * plugin registers nothing and stays inert.
2418
+ * @param ctx - the plugin context (host).
2419
+ * @param config - raw plugin config.
2420
+ */
2421
+ async function apply(ctx, config = {}) {
2422
+ const resolved = resolveConfig(config);
2423
+ const logger = ctx.logger("data-quality");
2424
+ if (!resolved.enabled) {
2425
+ logger.info("disabled: enabled is false — no service or tools are mounted");
2426
+ return;
2427
+ }
2428
+ let store;
2429
+ let domain;
2430
+ if (resolved.storeReports) {
2431
+ domain = await ctx.storageDomain.open(dataQualityDomainSpec);
2432
+ const reports = domain.table("reports");
2433
+ store = {
2434
+ put: async (record) => {
2435
+ const key = reportKeyOf(record);
2436
+ await reports.put(key, record);
2437
+ return key;
2438
+ },
2439
+ get: (key) => reports.get(key)
2440
+ };
2441
+ }
2442
+ const service = new LocalDataQualityService(ctx, resolved, {
2443
+ store,
2444
+ now: Date.now
2445
+ });
2446
+ ctx.tools.register(defineProfileTool(service));
2447
+ ctx.tools.register(defineCleanTool(service));
2448
+ ctx.tools.register(defineVerifyTool(service));
2449
+ logger.info(`dsh-data-quality ${VERSION} mounted: ctx.dataQuality + data_profile/data_clean/data_verify`);
2450
+ if (domain !== void 0) {
2451
+ const handle = domain;
2452
+ ctx.effect(() => async () => {
2453
+ await handle.close();
2454
+ });
2455
+ }
2456
+ }
2457
+ //#endregion
2458
+ export { Config, DATA_QUALITY_EVENT_TYPES, DataQualityService, DatasetError, LocalDataQualityService, MAX_CELL_TEXT, VERSION, appendDataQualityEvent, apply, applyCleanRules, checkCitations, dataQualityDomainSpec, inject, isMissing, loadDocument, loadTable, name, parseBoolean, parseDate, parseDelimited, parseJsonTable, parseLocator, parseNumeric, profileTable, renderProfileText, renderVerifyText, reportKeyOf, reportRecordSchema, resolveConfig, resolveWorkspacePath, sampleRows, serializeDelimited, truncateCell, truncateRow, verifyTable };