dsh-data-quality 0.1.3 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -0
- package/README.es.md +7 -1
- package/README.hi.md +7 -1
- package/README.md +22 -9
- package/README.pt.md +7 -1
- package/README.zh.md +7 -1
- package/cordis.patch.yml +11 -0
- package/lib/index.js +1609 -145
- package/lib/types/config.d.ts +12 -0
- package/lib/types/config.d.ts.map +1 -1
- package/lib/types/config.js +35 -0
- package/lib/types/config.js.map +1 -1
- package/lib/types/contract.d.ts +83 -0
- package/lib/types/contract.d.ts.map +1 -0
- package/lib/types/contract.js +144 -0
- package/lib/types/contract.js.map +1 -0
- package/lib/types/dataset.d.ts +50 -3
- package/lib/types/dataset.d.ts.map +1 -1
- package/lib/types/dataset.js +76 -21
- package/lib/types/dataset.js.map +1 -1
- package/lib/types/events.d.ts +2 -0
- package/lib/types/events.d.ts.map +1 -1
- package/lib/types/events.js.map +1 -1
- package/lib/types/index.d.ts +13 -8
- package/lib/types/index.d.ts.map +1 -1
- package/lib/types/index.js +17 -6
- package/lib/types/index.js.map +1 -1
- package/lib/types/presets.d.ts +28 -0
- package/lib/types/presets.d.ts.map +1 -0
- package/lib/types/presets.js +134 -0
- package/lib/types/presets.js.map +1 -0
- package/lib/types/profile.d.ts +40 -1
- package/lib/types/profile.d.ts.map +1 -1
- package/lib/types/profile.js +66 -10
- package/lib/types/profile.js.map +1 -1
- package/lib/types/provider-local.d.ts +7 -1
- package/lib/types/provider-local.d.ts.map +1 -1
- package/lib/types/provider-local.js +72 -2
- package/lib/types/provider-local.js.map +1 -1
- package/lib/types/report-html.d.ts +41 -0
- package/lib/types/report-html.d.ts.map +1 -0
- package/lib/types/report-html.js +192 -0
- package/lib/types/report-html.js.map +1 -0
- package/lib/types/scorecard.d.ts +50 -0
- package/lib/types/scorecard.d.ts.map +1 -0
- package/lib/types/scorecard.js +186 -0
- package/lib/types/scorecard.js.map +1 -0
- package/lib/types/service.d.ts +32 -2
- package/lib/types/service.d.ts.map +1 -1
- package/lib/types/service.js.map +1 -1
- package/lib/types/store.d.ts +21 -1
- package/lib/types/store.d.ts.map +1 -1
- package/lib/types/store.js +13 -1
- package/lib/types/store.js.map +1 -1
- package/lib/types/tools/clean.d.ts.map +1 -1
- package/lib/types/tools/clean.js +76 -1
- package/lib/types/tools/clean.js.map +1 -1
- package/lib/types/tools/profile-report-schema.d.ts +319 -0
- package/lib/types/tools/profile-report-schema.d.ts.map +1 -0
- package/lib/types/tools/profile-report-schema.js +96 -0
- package/lib/types/tools/profile-report-schema.js.map +1 -0
- package/lib/types/tools/profile.d.ts.map +1 -1
- package/lib/types/tools/profile.js +6 -53
- package/lib/types/tools/profile.js.map +1 -1
- package/lib/types/tools/report.d.ts +14 -0
- package/lib/types/tools/report.d.ts.map +1 -0
- package/lib/types/tools/report.js +112 -0
- package/lib/types/tools/report.js.map +1 -0
- package/lib/types/tools/shared.d.ts.map +1 -1
- package/lib/types/tools/shared.js +22 -2
- package/lib/types/tools/shared.js.map +1 -1
- package/lib/types/tools/verify.d.ts.map +1 -1
- package/lib/types/tools/verify.js +36 -1
- package/lib/types/tools/verify.js.map +1 -1
- package/lib/types/verify.d.ts +49 -3
- package/lib/types/verify.d.ts.map +1 -1
- package/lib/types/verify.js +111 -4
- package/lib/types/verify.js.map +1 -1
- package/lib/types/version.d.ts +8 -1
- package/lib/types/version.d.ts.map +1 -1
- package/lib/types/version.js +8 -1
- package/lib/types/version.js.map +1 -1
- package/package.json +1 -1
- package/src/config.ts +52 -0
- package/src/contract.ts +190 -0
- package/src/dataset.ts +101 -21
- package/src/events.ts +2 -0
- package/src/index.ts +27 -8
- package/src/presets.ts +146 -0
- package/src/profile.ts +105 -11
- package/src/provider-local.ts +77 -3
- package/src/report-html.ts +208 -0
- package/src/scorecard.ts +244 -0
- package/src/service.ts +34 -2
- package/src/store.ts +28 -2
- package/src/tools/clean.ts +76 -1
- package/src/tools/profile-report-schema.ts +98 -0
- package/src/tools/profile.ts +8 -56
- package/src/tools/report.ts +137 -0
- package/src/tools/shared.ts +21 -2
- package/src/tools/verify.ts +40 -2
- package/src/verify.ts +151 -5
- package/src/version.ts +9 -1
package/lib/index.js
CHANGED
|
@@ -4,6 +4,7 @@ import { defineDomain, domainTable } from "@deepseek-ai/dsh-storage-domain";
|
|
|
4
4
|
import { mkdir, open, stat, writeFile } from "node:fs/promises";
|
|
5
5
|
import path from "node:path";
|
|
6
6
|
import { createHash } from "node:crypto";
|
|
7
|
+
import { TextDecoder } from "node:util";
|
|
7
8
|
import { KNOWN_SESSION_EVENT_TYPES } from "@deepseek-ai/dsh-session";
|
|
8
9
|
import { Service } from "@deepseek-ai/cordis";
|
|
9
10
|
import { defineTool } from "@deepseek-ai/dsh-tools";
|
|
@@ -14,6 +15,15 @@ import { defineTool } from "@deepseek-ai/dsh-tools";
|
|
|
14
15
|
* step validates bounds so misconfiguration fails loud at mount.
|
|
15
16
|
* @module dsh-data-quality/config
|
|
16
17
|
*/
|
|
18
|
+
/** The six scorecard dimension ids, in report order (kept in sync with `scorecard.ts`). */
|
|
19
|
+
const SCORECARD_DIMENSIONS = [
|
|
20
|
+
"completeness",
|
|
21
|
+
"uniqueness",
|
|
22
|
+
"validity",
|
|
23
|
+
"consistency",
|
|
24
|
+
"timeliness",
|
|
25
|
+
"accuracy"
|
|
26
|
+
];
|
|
17
27
|
/** Schemastery schema: the loader validates and fills defaults before `apply`. */
|
|
18
28
|
const Config = z.object({
|
|
19
29
|
enabled: z.boolean().default(true),
|
|
@@ -28,7 +38,22 @@ const Config = z.object({
|
|
|
28
38
|
".jsonl"
|
|
29
39
|
]),
|
|
30
40
|
workspaceRoot: z.string().default(""),
|
|
31
|
-
storeReports: z.boolean().default(true)
|
|
41
|
+
storeReports: z.boolean().default(true),
|
|
42
|
+
scorecardWeights: z.object({
|
|
43
|
+
completeness: z.number().default(1),
|
|
44
|
+
uniqueness: z.number().default(1),
|
|
45
|
+
validity: z.number().default(1),
|
|
46
|
+
consistency: z.number().default(1),
|
|
47
|
+
timeliness: z.number().default(1),
|
|
48
|
+
accuracy: z.number().default(1)
|
|
49
|
+
}).default({
|
|
50
|
+
completeness: 1,
|
|
51
|
+
uniqueness: 1,
|
|
52
|
+
validity: 1,
|
|
53
|
+
consistency: 1,
|
|
54
|
+
timeliness: 1,
|
|
55
|
+
accuracy: 1
|
|
56
|
+
})
|
|
32
57
|
});
|
|
33
58
|
/** Throw unless `value` is a positive safe integer. */
|
|
34
59
|
function assertPositiveInt(name, value) {
|
|
@@ -63,6 +88,7 @@ function resolveConfig(config = {}) {
|
|
|
63
88
|
return ext;
|
|
64
89
|
});
|
|
65
90
|
if (allowedExtensions.length === 0) throw new TypeError("allowedExtensions must not be empty");
|
|
91
|
+
const scorecardWeights = resolveScorecardWeights(config.scorecardWeights);
|
|
66
92
|
return {
|
|
67
93
|
enabled: config.enabled ?? true,
|
|
68
94
|
maxRows,
|
|
@@ -71,9 +97,20 @@ function resolveConfig(config = {}) {
|
|
|
71
97
|
evidenceRowLimit,
|
|
72
98
|
allowedExtensions,
|
|
73
99
|
workspaceRoot: config.workspaceRoot ?? "",
|
|
74
|
-
storeReports: config.storeReports ?? true
|
|
100
|
+
storeReports: config.storeReports ?? true,
|
|
101
|
+
scorecardWeights
|
|
75
102
|
};
|
|
76
103
|
}
|
|
104
|
+
/** Resolve scorecard weights, filling defaults and rejecting non-negative violations loudly. */
|
|
105
|
+
function resolveScorecardWeights(weights = {}) {
|
|
106
|
+
const out = {};
|
|
107
|
+
for (const dimension of SCORECARD_DIMENSIONS) {
|
|
108
|
+
const value = weights[dimension] ?? 1;
|
|
109
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value < 0) throw new TypeError(`scorecardWeights.${dimension} must be a non-negative finite number, got ${String(value)}`);
|
|
110
|
+
out[dimension] = value;
|
|
111
|
+
}
|
|
112
|
+
return out;
|
|
113
|
+
}
|
|
77
114
|
//#endregion
|
|
78
115
|
//#region src/dataset.ts
|
|
79
116
|
/**
|
|
@@ -127,13 +164,40 @@ function throwIfAborted(signal) {
|
|
|
127
164
|
if (signal?.aborted === true) throw signal.reason instanceof Error ? signal.reason : /* @__PURE__ */ new Error("operation aborted");
|
|
128
165
|
}
|
|
129
166
|
/**
|
|
130
|
-
*
|
|
167
|
+
* Detect a UTF-8 byte-order mark and validate the byte sequence. Invalid
|
|
168
|
+
* UTF-8 is a data-quality finding the profile reports (`validUtf8: false`)
|
|
169
|
+
* rather than a structural error that blocks the read — the decoded text
|
|
170
|
+
* keeps U+FFFD replacement characters so the profile can still run.
|
|
171
|
+
* @param buffer - raw file bytes.
|
|
172
|
+
* @returns the encoding metadata.
|
|
173
|
+
*/
|
|
174
|
+
function detectEncoding(buffer) {
|
|
175
|
+
const bom = buffer.length >= 3 && buffer[0] === 239 && buffer[1] === 187 && buffer[2] === 191 ? "utf-8" : null;
|
|
176
|
+
const body = bom === "utf-8" ? buffer.subarray(3) : buffer;
|
|
177
|
+
let validUtf8 = true;
|
|
178
|
+
try {
|
|
179
|
+
new TextDecoder("utf-8", { fatal: true }).decode(body);
|
|
180
|
+
} catch {
|
|
181
|
+
validUtf8 = false;
|
|
182
|
+
}
|
|
183
|
+
return {
|
|
184
|
+
bom,
|
|
185
|
+
validUtf8
|
|
186
|
+
};
|
|
187
|
+
}
|
|
188
|
+
/** Decode file bytes, stripping a UTF-8 BOM and preserving replacement characters for invalid bytes. */
|
|
189
|
+
function decodeUtf8Text(buffer, encoding) {
|
|
190
|
+
const body = encoding.bom === "utf-8" ? buffer.subarray(3) : buffer;
|
|
191
|
+
return new TextDecoder("utf-8").decode(body);
|
|
192
|
+
}
|
|
193
|
+
/**
|
|
194
|
+
* Read a dataset file under the size cap, detecting its encoding.
|
|
131
195
|
* @param absolutePath - normalized absolute path (from {@link resolveWorkspacePath}).
|
|
132
196
|
* @param config - resolved config (size cap).
|
|
133
197
|
* @param signal - optional abort signal honored around the read.
|
|
134
|
-
* @returns the UTF-8 text.
|
|
198
|
+
* @returns the decoded UTF-8 text plus its encoding metadata.
|
|
135
199
|
*/
|
|
136
|
-
async function
|
|
200
|
+
async function readDatasetFile(absolutePath, config, signal) {
|
|
137
201
|
throwIfAborted(signal);
|
|
138
202
|
let info;
|
|
139
203
|
try {
|
|
@@ -147,7 +211,12 @@ async function readDatasetText(absolutePath, config, signal) {
|
|
|
147
211
|
const handle = await open(absolutePath, "r");
|
|
148
212
|
try {
|
|
149
213
|
throwIfAborted(signal);
|
|
150
|
-
|
|
214
|
+
const buffer = await handle.readFile();
|
|
215
|
+
const encoding = detectEncoding(buffer);
|
|
216
|
+
return {
|
|
217
|
+
text: decodeUtf8Text(buffer, encoding),
|
|
218
|
+
encoding
|
|
219
|
+
};
|
|
151
220
|
} finally {
|
|
152
221
|
await handle.close();
|
|
153
222
|
}
|
|
@@ -316,12 +385,21 @@ function parseJsonLines(text) {
|
|
|
316
385
|
* @returns the parsed table.
|
|
317
386
|
*/
|
|
318
387
|
async function loadTable(absolutePath, config, signal) {
|
|
319
|
-
const text = await
|
|
388
|
+
const { text, encoding } = await readDatasetFile(absolutePath, config, signal);
|
|
320
389
|
const ext = path.extname(absolutePath).toLowerCase();
|
|
321
390
|
throwIfAborted(signal);
|
|
322
|
-
if (ext === ".csv") return
|
|
323
|
-
|
|
324
|
-
|
|
391
|
+
if (ext === ".csv") return {
|
|
392
|
+
...parseDelimited(text, ",", config, signal),
|
|
393
|
+
encoding
|
|
394
|
+
};
|
|
395
|
+
if (ext === ".tsv") return {
|
|
396
|
+
...parseDelimited(text, " ", config, signal),
|
|
397
|
+
encoding
|
|
398
|
+
};
|
|
399
|
+
return {
|
|
400
|
+
...parseJsonTable(text, ext, config, signal),
|
|
401
|
+
encoding
|
|
402
|
+
};
|
|
325
403
|
}
|
|
326
404
|
/**
|
|
327
405
|
* Load a dataset as a citation-checkable document root: CSV/TSV become
|
|
@@ -334,14 +412,20 @@ async function loadTable(absolutePath, config, signal) {
|
|
|
334
412
|
*/
|
|
335
413
|
async function loadDocument(absolutePath, config, signal) {
|
|
336
414
|
const ext = path.extname(absolutePath).toLowerCase();
|
|
337
|
-
if (ext === ".json")
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
415
|
+
if (ext === ".json") {
|
|
416
|
+
const { text } = await readDatasetFile(absolutePath, config, signal);
|
|
417
|
+
return {
|
|
418
|
+
kind: "json",
|
|
419
|
+
value: parseJsonDocument(text)
|
|
420
|
+
};
|
|
421
|
+
}
|
|
422
|
+
if (ext === ".jsonl") {
|
|
423
|
+
const { text } = await readDatasetFile(absolutePath, config, signal);
|
|
424
|
+
return {
|
|
425
|
+
kind: "json",
|
|
426
|
+
value: parseJsonLines(text)
|
|
427
|
+
};
|
|
428
|
+
}
|
|
345
429
|
const table = await loadTable(absolutePath, config, signal);
|
|
346
430
|
return {
|
|
347
431
|
kind: "table",
|
|
@@ -390,18 +474,17 @@ const DATE_PATTERNS = [
|
|
|
390
474
|
/^(\d{4})\/(\d{1,2})\/(\d{1,2})$/u,
|
|
391
475
|
/^(\d{4})-(\d{1,2})-(\d{1,2})[ T](\d{1,2}):(\d{2})(?::(\d{2}))?(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?$/u
|
|
392
476
|
];
|
|
393
|
-
/**
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
function parseDate(cell) {
|
|
477
|
+
/** Deterministic date-format labels in {@link DATE_PATTERNS} order. */
|
|
478
|
+
const DATE_FORMATS = [
|
|
479
|
+
"iso-date",
|
|
480
|
+
"slash-date",
|
|
481
|
+
"datetime"
|
|
482
|
+
];
|
|
483
|
+
/** Parse one date cell to its epoch plus format; `undefined` when unrecognized. */
|
|
484
|
+
function parseDateCellInternal(cell) {
|
|
402
485
|
if (typeof cell !== "string") return void 0;
|
|
403
486
|
const text = cell.trim();
|
|
404
|
-
for (const pattern of DATE_PATTERNS) {
|
|
487
|
+
for (const [index, pattern] of DATE_PATTERNS.entries()) {
|
|
405
488
|
const match = pattern.exec(text);
|
|
406
489
|
if (match === null) continue;
|
|
407
490
|
const year = Number(match[1]);
|
|
@@ -414,9 +497,43 @@ function parseDate(cell) {
|
|
|
414
497
|
const epoch = Date.UTC(year, month - 1, day, hour, minute, second);
|
|
415
498
|
const check = new Date(epoch);
|
|
416
499
|
if (check.getUTCMonth() !== month - 1 || check.getUTCDate() !== day) return void 0;
|
|
417
|
-
|
|
500
|
+
const format = DATE_FORMATS[index];
|
|
501
|
+
if (format === void 0) return void 0;
|
|
502
|
+
return {
|
|
503
|
+
epoch,
|
|
504
|
+
format
|
|
505
|
+
};
|
|
418
506
|
}
|
|
419
507
|
}
|
|
508
|
+
/**
|
|
509
|
+
* Deterministic date parse to epoch milliseconds. Accepts `YYYY-MM-DD`,
|
|
510
|
+
* `YYYY/MM/DD`, and ISO-like datetimes (date-only forms read as UTC midnight).
|
|
511
|
+
* Calendar-invalid dates (e.g. 2025-13-40) reject. Returns `undefined` when
|
|
512
|
+
* the cell is not a recognized date.
|
|
513
|
+
* @param cell - the cell to parse (`undefined` when the column is absent).
|
|
514
|
+
* @returns epoch milliseconds, or `undefined`.
|
|
515
|
+
*/
|
|
516
|
+
function parseDate(cell) {
|
|
517
|
+
return parseDateCellInternal(cell)?.epoch;
|
|
518
|
+
}
|
|
519
|
+
/**
|
|
520
|
+
* Parse one date cell to its epoch plus source format label.
|
|
521
|
+
* @param cell - the cell to parse (`undefined` when the column is absent).
|
|
522
|
+
* @returns the parsed date, or `undefined` when the cell is not a recognized date.
|
|
523
|
+
*/
|
|
524
|
+
function parseDateCell(cell) {
|
|
525
|
+
return parseDateCellInternal(cell);
|
|
526
|
+
}
|
|
527
|
+
/**
|
|
528
|
+
* The format label of a recognized date cell (`iso-date` / `slash-date` /
|
|
529
|
+
* `datetime`); `undefined` when the cell is not a recognized date. Used to
|
|
530
|
+
* measure a date column's format consistency.
|
|
531
|
+
* @param cell - the cell to inspect (`undefined` when the column is absent).
|
|
532
|
+
* @returns the source format label, or `undefined`.
|
|
533
|
+
*/
|
|
534
|
+
function dateFormatOf(cell) {
|
|
535
|
+
return parseDateCellInternal(cell)?.format;
|
|
536
|
+
}
|
|
420
537
|
/** Boolean parse: true/false/yes/no/1/0, case-insensitive. */
|
|
421
538
|
function parseBoolean(cell) {
|
|
422
539
|
if (typeof cell === "boolean") return cell;
|
|
@@ -440,7 +557,8 @@ const reportRecordSchema = z$1.object({
|
|
|
440
557
|
"profile",
|
|
441
558
|
"clean",
|
|
442
559
|
"verify",
|
|
443
|
-
"citations"
|
|
560
|
+
"citations",
|
|
561
|
+
"clean-diff"
|
|
444
562
|
]),
|
|
445
563
|
at: z$1.number().int().nonnegative(),
|
|
446
564
|
dataset: z$1.string(),
|
|
@@ -452,6 +570,18 @@ const dataQualityDomainSpec = defineDomain({
|
|
|
452
570
|
version: 1,
|
|
453
571
|
tables: { reports: domainTable(reportRecordSchema) }
|
|
454
572
|
});
|
|
573
|
+
/** Well-formed report-key shape: `<17-digit timestamp>-<kind>-<8-hex fingerprint>`. */
|
|
574
|
+
const REPORT_KEY_PATTERN = /^\d{17}-(?:profile|clean|verify|citations|clean-diff)-[0-9a-f]{8}$/u;
|
|
575
|
+
/**
|
|
576
|
+
* Whether `key` is a well-formed, path-safe storage report key. Rejects any
|
|
577
|
+
* key with separators, traversal, or unexpected characters before it can be
|
|
578
|
+
* handed to the storage backend.
|
|
579
|
+
* @param key - candidate report key.
|
|
580
|
+
* @returns whether the key matches the deterministic report-key format.
|
|
581
|
+
*/
|
|
582
|
+
function isValidReportKey(key) {
|
|
583
|
+
return REPORT_KEY_PATTERN.test(key);
|
|
584
|
+
}
|
|
455
585
|
/** Pad to two digits for the key timestamp. */
|
|
456
586
|
function pad2(value) {
|
|
457
587
|
return String(value).padStart(2, "0");
|
|
@@ -842,6 +972,215 @@ function serializeDelimited(columns, rows, delimiter) {
|
|
|
842
972
|
return `${lines.join("\n")}\n`;
|
|
843
973
|
}
|
|
844
974
|
//#endregion
|
|
975
|
+
//#region src/scorecard.ts
|
|
976
|
+
/**
|
|
977
|
+
* DAMA-style six-dimension quality scorecard over a parsed {@link Table}.
|
|
978
|
+
* Pure and deterministic: every rate derives from one full-table pass, the
|
|
979
|
+
* only clock is the injected `now`, and no dimension fabricates a score it
|
|
980
|
+
* cannot defend — `accuracy` stays `null` (undetermined) without a declared
|
|
981
|
+
* schema and `timeliness` stays `null` without date cells.
|
|
982
|
+
* @module dsh-data-quality/scorecard
|
|
983
|
+
*/
|
|
984
|
+
/** Round to 6 significant digits (mirrors the profile report's rounding). */
|
|
985
|
+
function round6$1(value) {
|
|
986
|
+
return Number(value.toPrecision(6));
|
|
987
|
+
}
|
|
988
|
+
/** Increment a format-label counter. */
|
|
989
|
+
function bump(formats, tag) {
|
|
990
|
+
formats.set(tag, (formats.get(tag) ?? 0) + 1);
|
|
991
|
+
}
|
|
992
|
+
/** Inferred column type from the full-table class counts (mirrors `profileColumn`). */
|
|
993
|
+
function inferType(stats) {
|
|
994
|
+
if (stats.present === 0) return "empty";
|
|
995
|
+
if (stats.number === stats.present) return "number";
|
|
996
|
+
if (stats.date === stats.present) return "date";
|
|
997
|
+
if (stats.boolean === stats.present) return "boolean";
|
|
998
|
+
if (stats.string === stats.present) return "string";
|
|
999
|
+
return "mixed";
|
|
1000
|
+
}
|
|
1001
|
+
/** Format the `value / total` ratio as a plain `note` fragment. */
|
|
1002
|
+
function ratioNote(numerator, denominator) {
|
|
1003
|
+
return `${numerator}/${denominator}`;
|
|
1004
|
+
}
|
|
1005
|
+
/** Build one determinable dimension with its ratio note. */
|
|
1006
|
+
function dimension(name, score, numerator, denominator) {
|
|
1007
|
+
return {
|
|
1008
|
+
name,
|
|
1009
|
+
score,
|
|
1010
|
+
note: ratioNote(numerator, denominator)
|
|
1011
|
+
};
|
|
1012
|
+
}
|
|
1013
|
+
/**
|
|
1014
|
+
* Compute the six-dimension scorecard over a full table:
|
|
1015
|
+
* - `completeness` — non-empty cell rate.
|
|
1016
|
+
* - `uniqueness` — unique full-row content rate (`1 - duplicateRows / rowCount`).
|
|
1017
|
+
* - `validity` — present cells conforming to their column's inferred type.
|
|
1018
|
+
* - `consistency` — present cells in their column's dominant format (a date
|
|
1019
|
+
* column's unified `YYYY-MM-DD` vs `YYYY/MM/DD` vs datetime ratio).
|
|
1020
|
+
* - `timeliness` — date cells not future-dated relative to the injected `now`.
|
|
1021
|
+
* - `accuracy` — declared-schema agreement (limited definition); `null`
|
|
1022
|
+
* (undetermined) without a declared schema — never fabricated.
|
|
1023
|
+
* @param table - the parsed dataset (full table, not the sampled cards).
|
|
1024
|
+
* @param options - injected clock, duplicate-row count, optional declared schema, optional weights, abort signal.
|
|
1025
|
+
* @returns the scorecard.
|
|
1026
|
+
*/
|
|
1027
|
+
function computeScorecard(table, options) {
|
|
1028
|
+
throwIfAborted(options.signal);
|
|
1029
|
+
const stats = /* @__PURE__ */ new Map();
|
|
1030
|
+
for (const column of table.columns) stats.set(column, {
|
|
1031
|
+
present: 0,
|
|
1032
|
+
number: 0,
|
|
1033
|
+
date: 0,
|
|
1034
|
+
boolean: 0,
|
|
1035
|
+
string: 0,
|
|
1036
|
+
formats: /* @__PURE__ */ new Map()
|
|
1037
|
+
});
|
|
1038
|
+
let missingCells = 0;
|
|
1039
|
+
let dateCells = 0;
|
|
1040
|
+
let futureDateCells = 0;
|
|
1041
|
+
for (const [index, row] of table.rows.entries()) {
|
|
1042
|
+
if (index % 1024 === 0) throwIfAborted(options.signal);
|
|
1043
|
+
for (const column of table.columns) {
|
|
1044
|
+
const cell = row[column];
|
|
1045
|
+
const columnStats = stats.get(column);
|
|
1046
|
+
if (isMissing(cell)) {
|
|
1047
|
+
missingCells += 1;
|
|
1048
|
+
continue;
|
|
1049
|
+
}
|
|
1050
|
+
columnStats.present += 1;
|
|
1051
|
+
if (parseNumeric(cell) !== void 0) {
|
|
1052
|
+
columnStats.number += 1;
|
|
1053
|
+
bump(columnStats.formats, "number");
|
|
1054
|
+
continue;
|
|
1055
|
+
}
|
|
1056
|
+
const date = parseDateCell(cell);
|
|
1057
|
+
if (date !== void 0) {
|
|
1058
|
+
columnStats.date += 1;
|
|
1059
|
+
bump(columnStats.formats, date.format);
|
|
1060
|
+
dateCells += 1;
|
|
1061
|
+
if (date.epoch > options.now) futureDateCells += 1;
|
|
1062
|
+
continue;
|
|
1063
|
+
}
|
|
1064
|
+
if (parseBoolean(cell) !== void 0) {
|
|
1065
|
+
columnStats.boolean += 1;
|
|
1066
|
+
bump(columnStats.formats, "boolean");
|
|
1067
|
+
continue;
|
|
1068
|
+
}
|
|
1069
|
+
columnStats.string += 1;
|
|
1070
|
+
bump(columnStats.formats, typeof cell === "string" ? "string" : "json");
|
|
1071
|
+
}
|
|
1072
|
+
}
|
|
1073
|
+
const totalCells = table.rows.length * table.columns.length;
|
|
1074
|
+
let presentCells = 0;
|
|
1075
|
+
let invalidCells = 0;
|
|
1076
|
+
let consistentCells = 0;
|
|
1077
|
+
for (const [, columnStats] of stats) {
|
|
1078
|
+
presentCells += columnStats.present;
|
|
1079
|
+
const dominantClass = Math.max(columnStats.number, columnStats.date, columnStats.boolean, columnStats.string);
|
|
1080
|
+
invalidCells += columnStats.present - dominantClass;
|
|
1081
|
+
if (columnStats.present === 0) continue;
|
|
1082
|
+
let dominant = 0;
|
|
1083
|
+
for (const count of columnStats.formats.values()) if (count > dominant) dominant = count;
|
|
1084
|
+
consistentCells += dominant;
|
|
1085
|
+
}
|
|
1086
|
+
const dimensions = [
|
|
1087
|
+
totalCells === 0 ? {
|
|
1088
|
+
name: "completeness",
|
|
1089
|
+
score: null,
|
|
1090
|
+
note: "undetermined: no cells"
|
|
1091
|
+
} : dimension("completeness", round6$1(1 - missingCells / totalCells), presentCells, totalCells),
|
|
1092
|
+
table.rows.length === 0 ? {
|
|
1093
|
+
name: "uniqueness",
|
|
1094
|
+
score: null,
|
|
1095
|
+
note: "undetermined: no rows"
|
|
1096
|
+
} : dimension("uniqueness", round6$1(1 - options.duplicateRows / table.rows.length), table.rows.length - options.duplicateRows, table.rows.length),
|
|
1097
|
+
presentCells === 0 ? {
|
|
1098
|
+
name: "validity",
|
|
1099
|
+
score: null,
|
|
1100
|
+
note: "undetermined: no present cells"
|
|
1101
|
+
} : dimension("validity", round6$1(1 - invalidCells / presentCells), presentCells - invalidCells, presentCells),
|
|
1102
|
+
presentCells === 0 ? {
|
|
1103
|
+
name: "consistency",
|
|
1104
|
+
score: null,
|
|
1105
|
+
note: "undetermined: no present cells"
|
|
1106
|
+
} : dimension("consistency", round6$1(consistentCells / presentCells), consistentCells, presentCells),
|
|
1107
|
+
dateCells === 0 ? {
|
|
1108
|
+
name: "timeliness",
|
|
1109
|
+
score: null,
|
|
1110
|
+
note: "undetermined: no date cells"
|
|
1111
|
+
} : dimension("timeliness", round6$1(1 - futureDateCells / dateCells), dateCells - futureDateCells, dateCells),
|
|
1112
|
+
computeAccuracy(stats, table, options.declaredSchema)
|
|
1113
|
+
];
|
|
1114
|
+
const scores = dimensions.map((entry) => entry.score).filter((score) => score !== null);
|
|
1115
|
+
return {
|
|
1116
|
+
dimensions,
|
|
1117
|
+
overall: scores.length === 0 ? null : round6$1(scores.reduce((sum, score) => sum + score, 0) / scores.length),
|
|
1118
|
+
weightedOverall: computeWeightedOverall(dimensions, options.weights)
|
|
1119
|
+
};
|
|
1120
|
+
}
|
|
1121
|
+
/** Weighted mean over determinable dimensions (weights default to 1; validated non-negative by config). */
|
|
1122
|
+
function computeWeightedOverall(dimensions, weights) {
|
|
1123
|
+
let weightedSum = 0;
|
|
1124
|
+
let weightTotal = 0;
|
|
1125
|
+
for (const entry of dimensions) {
|
|
1126
|
+
if (entry.score === null) continue;
|
|
1127
|
+
const weight = weights?.[entry.name] ?? 1;
|
|
1128
|
+
weightedSum += weight * entry.score;
|
|
1129
|
+
weightTotal += weight;
|
|
1130
|
+
}
|
|
1131
|
+
return weightTotal === 0 ? null : round6$1(weightedSum / weightTotal);
|
|
1132
|
+
}
|
|
1133
|
+
/**
|
|
1134
|
+
* Limited `accuracy` definition: agreement between the full-table inferred
|
|
1135
|
+
* type of each declared column and the declared type. Without a declared
|
|
1136
|
+
* schema (or an external truth source) the dimension is `null` (undetermined)
|
|
1137
|
+
* — accuracy is never fabricated.
|
|
1138
|
+
*/
|
|
1139
|
+
function computeAccuracy(stats, table, declaredSchema) {
|
|
1140
|
+
if (declaredSchema === void 0 || Object.keys(declaredSchema).length === 0) return {
|
|
1141
|
+
name: "accuracy",
|
|
1142
|
+
score: null,
|
|
1143
|
+
note: "undetermined: no declared schema or external truth to compare against; accuracy is never fabricated"
|
|
1144
|
+
};
|
|
1145
|
+
let checked = 0;
|
|
1146
|
+
let matched = 0;
|
|
1147
|
+
const mismatches = [];
|
|
1148
|
+
for (const column of table.columns) {
|
|
1149
|
+
const declared = declaredSchema[column];
|
|
1150
|
+
if (declared === void 0) continue;
|
|
1151
|
+
checked += 1;
|
|
1152
|
+
const inferred = inferType(stats.get(column));
|
|
1153
|
+
if (inferred === declared) matched += 1;
|
|
1154
|
+
else mismatches.push(`${column}: declared ${declared}, inferred ${inferred}`);
|
|
1155
|
+
}
|
|
1156
|
+
if (checked === 0) return {
|
|
1157
|
+
name: "accuracy",
|
|
1158
|
+
score: null,
|
|
1159
|
+
note: "undetermined: declared schema covers none of the dataset columns"
|
|
1160
|
+
};
|
|
1161
|
+
return {
|
|
1162
|
+
name: "accuracy",
|
|
1163
|
+
score: round6$1(matched / checked),
|
|
1164
|
+
note: `${matched}/${checked} columns match their declared type` + (mismatches.length > 0 ? `; mismatches: ${mismatches.join(", ")}` : "")
|
|
1165
|
+
};
|
|
1166
|
+
}
|
|
1167
|
+
//#endregion
|
|
1168
|
+
//#region src/version.ts
|
|
1169
|
+
/**
|
|
1170
|
+
* Plugin version, kept in one place so `scripts/release.mjs` can stamp it and
|
|
1171
|
+
* reports can name their generator.
|
|
1172
|
+
* @module dsh-data-quality/version
|
|
1173
|
+
*/
|
|
1174
|
+
/** The package version reported in persisted reports. */
|
|
1175
|
+
const VERSION = "0.3.0";
|
|
1176
|
+
/**
|
|
1177
|
+
* Version of the persisted report schema. Bump it whenever a report's
|
|
1178
|
+
* canonical shape changes in a way old consumers cannot read (the durable
|
|
1179
|
+
* `data_quality` records keep their own `schemaVersion` so a future reader
|
|
1180
|
+
* can detect and reject an incompatible record instead of misreading it).
|
|
1181
|
+
*/
|
|
1182
|
+
const REPORT_SCHEMA_VERSION = 1;
|
|
1183
|
+
//#endregion
|
|
845
1184
|
//#region src/profile.ts
|
|
846
1185
|
/**
|
|
847
1186
|
* Deterministic dataset profiling: per-column type inference, missingness,
|
|
@@ -850,6 +1189,8 @@ function serializeDelimited(columns, rows, delimiter) {
|
|
|
850
1189
|
* no I/O; `generatedAt` is injected by the caller.
|
|
851
1190
|
* @module dsh-data-quality/profile
|
|
852
1191
|
*/
|
|
1192
|
+
/** Fallback duplicate-sample cap for direct engine use; the provider always passes the configured `evidenceRowLimit`. */
|
|
1193
|
+
const DEFAULT_DUPLICATE_SAMPLE_LIMIT = 20;
|
|
853
1194
|
/** Round to 6 significant digits for stable, readable report numbers. */
|
|
854
1195
|
function round6(value) {
|
|
855
1196
|
return Number(value.toPrecision(6));
|
|
@@ -874,6 +1215,8 @@ function numericProfile(values) {
|
|
|
874
1215
|
const highFence = p75 + 1.5 * iqr;
|
|
875
1216
|
const outliers = iqr === 0 ? 0 : sorted.filter((value) => value < lowFence || value > highFence).length;
|
|
876
1217
|
return {
|
|
1218
|
+
count: sorted.length,
|
|
1219
|
+
distinct: new Set(sorted).size,
|
|
877
1220
|
min: round6(sorted[0]),
|
|
878
1221
|
max: round6(sorted[sorted.length - 1]),
|
|
879
1222
|
mean: round6(sum / sorted.length),
|
|
@@ -883,17 +1226,43 @@ function numericProfile(values) {
|
|
|
883
1226
|
outliers
|
|
884
1227
|
};
|
|
885
1228
|
}
|
|
886
|
-
/**
|
|
887
|
-
function
|
|
1229
|
+
/** Deterministic sha256 key of one row's full content (columns in table order). */
|
|
1230
|
+
function rowContentKey(table, row) {
|
|
1231
|
+
return createHash("sha256").update(JSON.stringify(table.columns.map((column) => row[column] ?? null))).digest("hex");
|
|
1232
|
+
}
|
|
1233
|
+
/**
|
|
1234
|
+
* Detect full-content duplicate rows with a bounded sample of their 0-based
|
|
1235
|
+
* indexes. The first occurrence of each content is never counted; later rows
|
|
1236
|
+
* with identical full content are duplicates.
|
|
1237
|
+
* @param table - the parsed dataset.
|
|
1238
|
+
* @param options - sample cap and optional abort signal.
|
|
1239
|
+
* @returns the duplicate count, rate, and capped sample indexes.
|
|
1240
|
+
*/
|
|
1241
|
+
function detectDuplicateRows(table, options) {
|
|
1242
|
+
if (!Number.isSafeInteger(options.sampleLimit) || options.sampleLimit <= 0) throw new TypeError(`sampleLimit must be a positive safe integer, got ${String(options.sampleLimit)}`);
|
|
888
1243
|
const seen = /* @__PURE__ */ new Set();
|
|
889
1244
|
let duplicates = 0;
|
|
1245
|
+
const duplicateSampleRowIndexes = [];
|
|
890
1246
|
for (const [index, row] of table.rows.entries()) {
|
|
891
|
-
if (index % 1024 === 0) throwIfAborted(signal);
|
|
892
|
-
const key =
|
|
893
|
-
if (seen.has(key))
|
|
894
|
-
|
|
1247
|
+
if (index % 1024 === 0) throwIfAborted(options.signal);
|
|
1248
|
+
const key = rowContentKey(table, row);
|
|
1249
|
+
if (seen.has(key)) {
|
|
1250
|
+
duplicates += 1;
|
|
1251
|
+
if (duplicateSampleRowIndexes.length < options.sampleLimit) duplicateSampleRowIndexes.push(index);
|
|
1252
|
+
} else seen.add(key);
|
|
895
1253
|
}
|
|
896
|
-
return
|
|
1254
|
+
return {
|
|
1255
|
+
duplicateRows: duplicates,
|
|
1256
|
+
duplicateRate: table.rows.length === 0 ? 0 : round6(duplicates / table.rows.length),
|
|
1257
|
+
duplicateSampleRowIndexes
|
|
1258
|
+
};
|
|
1259
|
+
}
|
|
1260
|
+
/** Count rows whose full content duplicates an earlier row (first occurrence is not counted). */
|
|
1261
|
+
function countDuplicateRows(table, signal) {
|
|
1262
|
+
return detectDuplicateRows(table, {
|
|
1263
|
+
sampleLimit: 1,
|
|
1264
|
+
signal
|
|
1265
|
+
}).duplicateRows;
|
|
897
1266
|
}
|
|
898
1267
|
/** Profile one column over the given rows. */
|
|
899
1268
|
function profileColumn(rows, column, signal) {
|
|
@@ -980,14 +1349,29 @@ function profileTable(table, options) {
|
|
|
980
1349
|
throwIfAborted(options.signal);
|
|
981
1350
|
const profiled = options.sample === void 0 ? table.rows : sampleRows(table.rows, options.sample);
|
|
982
1351
|
const columns = table.columns.map((column) => profileColumn(profiled, column, options.signal));
|
|
983
|
-
const
|
|
1352
|
+
const detection = detectDuplicateRows(table, {
|
|
1353
|
+
sampleLimit: options.duplicateSampleLimit ?? DEFAULT_DUPLICATE_SAMPLE_LIMIT,
|
|
1354
|
+
signal: options.signal
|
|
1355
|
+
});
|
|
1356
|
+
const scorecard = computeScorecard(table, {
|
|
1357
|
+
now: options.generatedAt,
|
|
1358
|
+
duplicateRows: detection.duplicateRows,
|
|
1359
|
+
declaredSchema: options.declaredSchema,
|
|
1360
|
+
weights: options.scorecardWeights,
|
|
1361
|
+
signal: options.signal
|
|
1362
|
+
});
|
|
984
1363
|
return {
|
|
1364
|
+
schemaVersion: 1,
|
|
985
1365
|
dataset: options.dataset,
|
|
986
1366
|
rowCount: table.rows.length,
|
|
987
1367
|
sampled: profiled.length !== table.rows.length,
|
|
988
1368
|
profiledRows: profiled.length,
|
|
989
1369
|
columnCount: table.columns.length,
|
|
990
|
-
duplicateRows,
|
|
1370
|
+
duplicateRows: detection.duplicateRows,
|
|
1371
|
+
duplicateRate: detection.duplicateRate,
|
|
1372
|
+
duplicateSampleRowIndexes: detection.duplicateSampleRowIndexes,
|
|
1373
|
+
scorecard,
|
|
1374
|
+
...table.encoding !== void 0 ? { encoding: table.encoding } : {},
|
|
991
1375
|
columns,
|
|
992
1376
|
generatedAt: options.generatedAt
|
|
993
1377
|
};
|
|
@@ -996,19 +1380,167 @@ function profileTable(table, options) {
|
|
|
996
1380
|
function renderProfileText(report) {
|
|
997
1381
|
const lines = [];
|
|
998
1382
|
lines.push(`Profile of ${report.dataset}: ${report.rowCount} rows x ${report.columnCount} columns` + (report.sampled ? ` (column cards over a systematic sample of ${report.profiledRows} rows)` : ""));
|
|
999
|
-
if (report.duplicateRows > 0) lines.push(`Duplicate rows: ${report.duplicateRows}`);
|
|
1383
|
+
if (report.duplicateRows > 0) lines.push(`Duplicate rows: ${report.duplicateRows} (${(report.duplicateRate * 100).toFixed(1)}%)` + (report.duplicateSampleRowIndexes.length > 0 ? `; sample row indexes: ${report.duplicateSampleRowIndexes.join(", ")}` : ""));
|
|
1000
1384
|
for (const column of report.columns) {
|
|
1001
1385
|
const parts = [`${column.name}: ${column.inferredType}`];
|
|
1002
1386
|
if (column.missing > 0) parts.push(`missing ${column.missing} (${(column.missingRate * 100).toFixed(1)}%)`);
|
|
1003
1387
|
parts.push(`unique ${column.unique}`);
|
|
1004
|
-
if (column.numeric !== void 0) parts.push(`min ${column.numeric.min}, p25 ${column.numeric.p25}, median ${column.numeric.median}, p75 ${column.numeric.p75}, max ${column.numeric.max}, mean ${column.numeric.mean}` + (column.numeric.outliers > 0 ? `, ${column.numeric.outliers} IQR outliers` : ""));
|
|
1388
|
+
if (column.numeric !== void 0) parts.push(`count ${column.numeric.count}, distinct ${column.numeric.distinct}, min ${column.numeric.min}, p25 ${column.numeric.p25}, median ${column.numeric.median}, p75 ${column.numeric.p75}, max ${column.numeric.max}, mean ${column.numeric.mean}` + (column.numeric.outliers > 0 ? `, ${column.numeric.outliers} IQR outliers` : ""));
|
|
1005
1389
|
if (column.topValues !== void 0) parts.push(`top: ${column.topValues.map((entry) => `${JSON.stringify(entry.value)} x${entry.count}`).join(", ")}`);
|
|
1006
1390
|
for (const note of column.notes) parts.push(`note: ${note}`);
|
|
1007
1391
|
lines.push(`- ${parts.join("; ")}`);
|
|
1008
1392
|
}
|
|
1393
|
+
if (report.encoding !== void 0) lines.push(`Encoding: UTF-8${report.encoding.bom === "utf-8" ? " (BOM)" : ""}${report.encoding.validUtf8 ? "" : " (INVALID UTF-8)"}`);
|
|
1394
|
+
const overall = report.scorecard.overall;
|
|
1395
|
+
const weighted = report.scorecard.weightedOverall;
|
|
1396
|
+
lines.push(`Scorecard (overall ${overall === null ? "undetermined" : `${(overall * 100).toFixed(1)}%`}, weighted ${weighted === null ? "undetermined" : `${(weighted * 100).toFixed(1)}%`}):`);
|
|
1397
|
+
for (const dimension of report.scorecard.dimensions) {
|
|
1398
|
+
const value = dimension.score === null ? "undetermined" : `${(dimension.score * 100).toFixed(1)}%`;
|
|
1399
|
+
lines.push(` ${dimension.name}: ${value} (${dimension.note})`);
|
|
1400
|
+
}
|
|
1009
1401
|
return lines.join("\n");
|
|
1010
1402
|
}
|
|
1011
1403
|
//#endregion
|
|
1404
|
+
//#region src/contract.ts
|
|
1405
|
+
/**
|
|
1406
|
+
* Delivery contract for `data_clean`: a deterministic pre-delivery validation
|
|
1407
|
+
* summary (primary-key/type/dedupe row-count comparison plus uniqueness and
|
|
1408
|
+
* non-null regression) and the clean before/after profile diff report. Pure —
|
|
1409
|
+
* no I/O, no clock, no RNG; timestamps are injected.
|
|
1410
|
+
* @module dsh-data-quality/contract
|
|
1411
|
+
*/
|
|
1412
|
+
/** Count missing cells in one column of the cleaned output. */
|
|
1413
|
+
function countMissing(rows, column, signal) {
|
|
1414
|
+
let count = 0;
|
|
1415
|
+
for (const [index, row] of rows.entries()) {
|
|
1416
|
+
if (index % 1024 === 0) throwIfAborted(signal);
|
|
1417
|
+
if (isMissing(row[column])) count += 1;
|
|
1418
|
+
}
|
|
1419
|
+
return count;
|
|
1420
|
+
}
|
|
1421
|
+
/** Count present cells in one column that still fail to parse as the requested type. */
|
|
1422
|
+
function countNonConforming(rows, column, to, signal) {
|
|
1423
|
+
let count = 0;
|
|
1424
|
+
for (const [index, row] of rows.entries()) {
|
|
1425
|
+
if (index % 1024 === 0) throwIfAborted(signal);
|
|
1426
|
+
const cell = row[column];
|
|
1427
|
+
if (isMissing(cell)) continue;
|
|
1428
|
+
if (!(to === "number" ? parseNumeric(cell) !== void 0 : to === "date" ? parseDate(cell) !== void 0 : parseBoolean(cell) !== void 0)) count += 1;
|
|
1429
|
+
}
|
|
1430
|
+
return count;
|
|
1431
|
+
}
|
|
1432
|
+
/**
|
|
1433
|
+
* Compute the pre-delivery contract summary from a cleaning run: dedupe
|
|
1434
|
+
* before/after row counts, the dedupe key's uniqueness regression, and the
|
|
1435
|
+
* non-null/type regressions over the columns the rules targeted.
|
|
1436
|
+
* @param result - the cleaning outcome (input/output rows plus cleaned rows).
|
|
1437
|
+
* @param rules - the ordered rules that produced `result`.
|
|
1438
|
+
* @param options - optional abort signal.
|
|
1439
|
+
* @returns the contract summary.
|
|
1440
|
+
*/
|
|
1441
|
+
function computeCleanContract(result, rules, options = {}) {
|
|
1442
|
+
const dedupeRule = rules.find((rule) => rule.rule === "dedupe");
|
|
1443
|
+
const dedupeColumns = dedupeRule === void 0 ? null : [...dedupeRule.columns ?? result.columns];
|
|
1444
|
+
const keyColumns = dedupeColumns ?? result.columns;
|
|
1445
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1446
|
+
let remainingDuplicateRows = 0;
|
|
1447
|
+
for (const [index, row] of result.rows.entries()) {
|
|
1448
|
+
if (index % 1024 === 0) throwIfAborted(options.signal);
|
|
1449
|
+
const key = JSON.stringify(keyColumns.map((column) => row[column] ?? null));
|
|
1450
|
+
if (seen.has(key)) remainingDuplicateRows += 1;
|
|
1451
|
+
else seen.add(key);
|
|
1452
|
+
}
|
|
1453
|
+
const remainingMissing = rules.filter((rule) => rule.rule === "fill-missing").map((rule) => {
|
|
1454
|
+
const fill = rule;
|
|
1455
|
+
return {
|
|
1456
|
+
column: fill.column,
|
|
1457
|
+
count: countMissing(result.rows, fill.column, options.signal)
|
|
1458
|
+
};
|
|
1459
|
+
});
|
|
1460
|
+
const typeConformance = rules.filter((rule) => rule.rule === "coerce-type").map((rule) => {
|
|
1461
|
+
const coerce = rule;
|
|
1462
|
+
return {
|
|
1463
|
+
column: coerce.column,
|
|
1464
|
+
to: coerce.to,
|
|
1465
|
+
invalidCount: countNonConforming(result.rows, coerce.column, coerce.to, options.signal)
|
|
1466
|
+
};
|
|
1467
|
+
});
|
|
1468
|
+
const columnDecisions = traceColumnDecisions(result, rules);
|
|
1469
|
+
return {
|
|
1470
|
+
inputRows: result.inputRows,
|
|
1471
|
+
outputRows: result.outputRows,
|
|
1472
|
+
removedRows: result.inputRows - result.outputRows,
|
|
1473
|
+
dedupeColumns,
|
|
1474
|
+
uniqueKeys: remainingDuplicateRows === 0,
|
|
1475
|
+
remainingDuplicateRows,
|
|
1476
|
+
remainingMissing,
|
|
1477
|
+
typeConformance,
|
|
1478
|
+
columnDecisions
|
|
1479
|
+
};
|
|
1480
|
+
}
|
|
1481
|
+
/** The columns a single rule targets (single-column rules target one column; trim/dedupe may target many). */
|
|
1482
|
+
function ruleColumns(rule, allColumns) {
|
|
1483
|
+
switch (rule.rule) {
|
|
1484
|
+
case "fill-missing":
|
|
1485
|
+
case "coerce-type":
|
|
1486
|
+
case "normalize-unit":
|
|
1487
|
+
case "map-values": return [rule.column];
|
|
1488
|
+
case "trim":
|
|
1489
|
+
case "dedupe": return [...rule.columns ?? allColumns];
|
|
1490
|
+
default: return [];
|
|
1491
|
+
}
|
|
1492
|
+
}
|
|
1493
|
+
/** Build the per-column decision trace: strategies + affected rows, in dataset column order. */
|
|
1494
|
+
function traceColumnDecisions(result, rules) {
|
|
1495
|
+
const byColumn = /* @__PURE__ */ new Map();
|
|
1496
|
+
for (const [index, rule] of rules.entries()) {
|
|
1497
|
+
const log = result.logs[index];
|
|
1498
|
+
if (log === void 0) continue;
|
|
1499
|
+
for (const column of ruleColumns(rule, result.columns)) {
|
|
1500
|
+
const list = byColumn.get(column) ?? [];
|
|
1501
|
+
list.push({
|
|
1502
|
+
strategy: rule.rule,
|
|
1503
|
+
affectedRows: log.affectedRows
|
|
1504
|
+
});
|
|
1505
|
+
byColumn.set(column, list);
|
|
1506
|
+
}
|
|
1507
|
+
}
|
|
1508
|
+
return result.columns.filter((column) => byColumn.has(column)).map((column) => ({
|
|
1509
|
+
column,
|
|
1510
|
+
decisions: byColumn.get(column)
|
|
1511
|
+
}));
|
|
1512
|
+
}
|
|
1513
|
+
/**
|
|
1514
|
+
* Build the clean before/after profile diff: two full {@link ProfileReport}
|
|
1515
|
+
* snapshots (input and cleaned output) under one envelope. Reuses the profile
|
|
1516
|
+
* engine so the scorecard and duplicate detection stay consistent across the
|
|
1517
|
+
* whole plugin.
|
|
1518
|
+
* @param input - the input dataset.
|
|
1519
|
+
* @param output - the cleaned dataset.
|
|
1520
|
+
* @param options - dataset label, injected timestamp, optional weights, optional abort signal.
|
|
1521
|
+
* @returns the diff report.
|
|
1522
|
+
*/
|
|
1523
|
+
function computeCleanProfileDiff(input, output, options) {
|
|
1524
|
+
const before = profileTable(input, {
|
|
1525
|
+
dataset: options.dataset,
|
|
1526
|
+
generatedAt: options.generatedAt,
|
|
1527
|
+
scorecardWeights: options.scorecardWeights,
|
|
1528
|
+
signal: options.signal
|
|
1529
|
+
});
|
|
1530
|
+
const after = profileTable(output, {
|
|
1531
|
+
dataset: options.dataset,
|
|
1532
|
+
generatedAt: options.generatedAt,
|
|
1533
|
+
scorecardWeights: options.scorecardWeights,
|
|
1534
|
+
signal: options.signal
|
|
1535
|
+
});
|
|
1536
|
+
return {
|
|
1537
|
+
dataset: options.dataset,
|
|
1538
|
+
before,
|
|
1539
|
+
after,
|
|
1540
|
+
generatedAt: options.generatedAt
|
|
1541
|
+
};
|
|
1542
|
+
}
|
|
1543
|
+
//#endregion
|
|
1012
1544
|
//#region src/verify.ts
|
|
1013
1545
|
/**
|
|
1014
1546
|
* Declarative verification rules over a parsed {@link Table} (not-null,
|
|
@@ -1031,6 +1563,19 @@ var VerifyRuleError = class extends Error {
|
|
|
1031
1563
|
this.name = "VerifyRuleError";
|
|
1032
1564
|
}
|
|
1033
1565
|
};
|
|
1566
|
+
/** Raised for invalid expectations; `message` names the expectation index and reason. */
|
|
1567
|
+
var VerifyExpectationError = class extends Error {
|
|
1568
|
+
expectationIndex;
|
|
1569
|
+
/**
|
|
1570
|
+
* @param expectationIndex - index of the offending expectation in the request array.
|
|
1571
|
+
* @param message - actionable human-readable detail.
|
|
1572
|
+
*/
|
|
1573
|
+
constructor(expectationIndex, message) {
|
|
1574
|
+
super(message);
|
|
1575
|
+
this.expectationIndex = expectationIndex;
|
|
1576
|
+
this.name = "VerifyExpectationError";
|
|
1577
|
+
}
|
|
1578
|
+
};
|
|
1034
1579
|
/** Assert `column` exists. */
|
|
1035
1580
|
function requireColumn(columns, column, ruleIndex) {
|
|
1036
1581
|
if (!columns.includes(column)) throw new VerifyRuleError(ruleIndex, `rule ${ruleIndex}: unknown column ${JSON.stringify(column)} (columns: ${columns.join(", ")})`);
|
|
@@ -1076,13 +1621,90 @@ const CROSS_OPS = [
|
|
|
1076
1621
|
">=",
|
|
1077
1622
|
">"
|
|
1078
1623
|
];
|
|
1624
|
+
/** The metric ids, for validation and diagnostics. */
|
|
1625
|
+
const VERIFY_METRICS = [
|
|
1626
|
+
"rowCount",
|
|
1627
|
+
"columnSum",
|
|
1628
|
+
"columnMean",
|
|
1629
|
+
"uniqueCount",
|
|
1630
|
+
"nullCount"
|
|
1631
|
+
];
|
|
1632
|
+
/** Throw unless the expectation is well-formed (metric/column/tolerance). */
|
|
1633
|
+
function validateExpectation(table, expectation, index) {
|
|
1634
|
+
if (!VERIFY_METRICS.includes(expectation.metric)) throw new VerifyExpectationError(index, `expectation ${index}: unknown metric ${JSON.stringify(expectation.metric)} (expected one of ${VERIFY_METRICS.join(", ")})`);
|
|
1635
|
+
if (expectation.metric !== "rowCount") {
|
|
1636
|
+
if (expectation.column === void 0 || expectation.column === "") throw new VerifyExpectationError(index, `expectation ${index}: metric ${expectation.metric} requires a column`);
|
|
1637
|
+
if (!table.columns.includes(expectation.column)) throw new VerifyExpectationError(index, `expectation ${index}: unknown column ${JSON.stringify(expectation.column)} (columns: ${table.columns.join(", ")})`);
|
|
1638
|
+
} else if (expectation.column !== void 0) throw new VerifyExpectationError(index, `expectation ${index}: metric rowCount takes no column, got ${JSON.stringify(expectation.column)}`);
|
|
1639
|
+
if (expectation.tolerance !== void 0 && (typeof expectation.tolerance !== "number" || !Number.isFinite(expectation.tolerance) || expectation.tolerance < 0 || expectation.tolerance > 1)) throw new VerifyExpectationError(index, `expectation ${index}: tolerance must be a finite number in [0, 1], got ${String(expectation.tolerance)}`);
|
|
1640
|
+
}
|
|
1641
|
+
/** Compute the deterministic actual value of one expectation's metric. */
|
|
1642
|
+
function metricValueOf(table, expectation) {
|
|
1643
|
+
switch (expectation.metric) {
|
|
1644
|
+
case "rowCount": return table.rows.length;
|
|
1645
|
+
case "nullCount": {
|
|
1646
|
+
let count = 0;
|
|
1647
|
+
for (const row of table.rows) if (isMissing(row[expectation.column])) count += 1;
|
|
1648
|
+
return count;
|
|
1649
|
+
}
|
|
1650
|
+
case "uniqueCount": {
|
|
1651
|
+
const distinct = /* @__PURE__ */ new Set();
|
|
1652
|
+
for (const row of table.rows) {
|
|
1653
|
+
const cell = row[expectation.column];
|
|
1654
|
+
if (isMissing(cell)) continue;
|
|
1655
|
+
distinct.add(typeof cell === "string" ? cell : JSON.stringify(cell));
|
|
1656
|
+
}
|
|
1657
|
+
return distinct.size;
|
|
1658
|
+
}
|
|
1659
|
+
case "columnSum":
|
|
1660
|
+
case "columnMean": {
|
|
1661
|
+
let sum = 0;
|
|
1662
|
+
let count = 0;
|
|
1663
|
+
for (const row of table.rows) {
|
|
1664
|
+
const value = parseNumeric(row[expectation.column]);
|
|
1665
|
+
if (value === void 0) continue;
|
|
1666
|
+
sum += value;
|
|
1667
|
+
count += 1;
|
|
1668
|
+
}
|
|
1669
|
+
return expectation.metric === "columnSum" ? sum : count === 0 ? 0 : sum / count;
|
|
1670
|
+
}
|
|
1671
|
+
}
|
|
1672
|
+
}
|
|
1673
|
+
/**
|
|
1674
|
+
* Reconcile each expectation against its deterministic actual value. A
|
|
1675
|
+
* mismatch is a normal `passed: false` result, never a thrown error; invalid
|
|
1676
|
+
* metrics, columns, and tolerances fail loud.
|
|
1677
|
+
* @param table - the parsed dataset.
|
|
1678
|
+
* @param expectations - the expectations to reconcile.
|
|
1679
|
+
* @param defaultTolerance - configured fallback relative tolerance.
|
|
1680
|
+
* @param signal - optional abort signal.
|
|
1681
|
+
* @returns one outcome per expectation.
|
|
1682
|
+
*/
|
|
1683
|
+
function verifyExpectations(table, expectations, defaultTolerance, signal) {
|
|
1684
|
+
return expectations.map((expectation, index) => {
|
|
1685
|
+
throwIfAborted(signal);
|
|
1686
|
+
validateExpectation(table, expectation, index);
|
|
1687
|
+
const actual = metricValueOf(table, expectation);
|
|
1688
|
+
const tolerance = expectation.tolerance ?? defaultTolerance;
|
|
1689
|
+
const passed = numericClose(actual, expectation.expected, tolerance);
|
|
1690
|
+
return {
|
|
1691
|
+
metric: expectation.metric,
|
|
1692
|
+
...expectation.column !== void 0 ? { column: expectation.column } : {},
|
|
1693
|
+
expected: expectation.expected,
|
|
1694
|
+
actual,
|
|
1695
|
+
tolerance,
|
|
1696
|
+
passed
|
|
1697
|
+
};
|
|
1698
|
+
});
|
|
1699
|
+
}
|
|
1079
1700
|
/**
|
|
1080
1701
|
* Apply verification rules over a parsed table. A missing cell fails every
|
|
1081
|
-
* rule that reads it. The overall `passed` is the conjunction of rule passes
|
|
1082
|
-
* a failing dataset is a normal result, never a
|
|
1702
|
+
* rule that reads it. The overall `passed` is the conjunction of rule passes
|
|
1703
|
+
* and expectation passes; a failing dataset is a normal result, never a
|
|
1704
|
+
* thrown error.
|
|
1083
1705
|
* @param table - the parsed dataset.
|
|
1084
1706
|
* @param rules - non-empty rule list.
|
|
1085
|
-
* @param options - evidence cap, injected clock for `freshness`, abort signal.
|
|
1707
|
+
* @param options - evidence cap, injected clock for `freshness`, optional expectations, default tolerance, abort signal.
|
|
1086
1708
|
* @returns the verify report (without the dataset label; the caller adds it).
|
|
1087
1709
|
*/
|
|
1088
1710
|
function verifyTable(table, rules, options) {
|
|
@@ -1205,10 +1827,12 @@ function verifyTable(table, rules, options) {
|
|
|
1205
1827
|
}
|
|
1206
1828
|
results.push(result);
|
|
1207
1829
|
}
|
|
1830
|
+
const expectations = verifyExpectations(table, options.expectations ?? [], options.defaultTolerance ?? 1e-9, options.signal);
|
|
1208
1831
|
return {
|
|
1209
|
-
passed: results.every((result) => result.passed),
|
|
1832
|
+
passed: results.every((result) => result.passed) && expectations.every((expectation) => expectation.passed),
|
|
1210
1833
|
rowCount: table.rows.length,
|
|
1211
1834
|
rules: results,
|
|
1835
|
+
expectations,
|
|
1212
1836
|
generatedAt: options.now()
|
|
1213
1837
|
};
|
|
1214
1838
|
}
|
|
@@ -1224,6 +1848,10 @@ function renderVerifyText(report) {
|
|
|
1224
1848
|
}
|
|
1225
1849
|
if (rule.failedCount > rule.evidence.length) lines.push(` … and ${rule.failedCount - rule.evidence.length} more failing row(s)`);
|
|
1226
1850
|
}
|
|
1851
|
+
for (const expectation of report.expectations) {
|
|
1852
|
+
const target = expectation.column !== void 0 ? `${expectation.metric}(${expectation.column})` : expectation.metric;
|
|
1853
|
+
lines.push(`- [${expectation.passed ? "pass" : "FAIL"}] expectation ${target}: actual ${expectation.actual} vs expected ${expectation.expected} (tolerance ${expectation.tolerance})`);
|
|
1854
|
+
}
|
|
1227
1855
|
return lines.join("\n");
|
|
1228
1856
|
}
|
|
1229
1857
|
/**
|
|
@@ -1409,6 +2037,131 @@ function appendDataQualityEvent(session, type, data) {
|
|
|
1409
2037
|
if (Function.prototype.toString.call(append).includes("ignorable")) append.call(session, type, data, { ignorable: true });
|
|
1410
2038
|
}
|
|
1411
2039
|
//#endregion
|
|
2040
|
+
//#region src/presets.ts
|
|
2041
|
+
/** The built-in industry preset registry. */
|
|
2042
|
+
const INDUSTRY_PRESETS = {
|
|
2043
|
+
retail: {
|
|
2044
|
+
id: "retail",
|
|
2045
|
+
label: "Retail",
|
|
2046
|
+
columns: {
|
|
2047
|
+
order_id: "string",
|
|
2048
|
+
sku: "string",
|
|
2049
|
+
quantity: "number",
|
|
2050
|
+
unit_price: "number",
|
|
2051
|
+
revenue: "number",
|
|
2052
|
+
order_date: "date",
|
|
2053
|
+
customer_id: "string"
|
|
2054
|
+
}
|
|
2055
|
+
},
|
|
2056
|
+
saas: {
|
|
2057
|
+
id: "saas",
|
|
2058
|
+
label: "SaaS",
|
|
2059
|
+
columns: {
|
|
2060
|
+
account_id: "string",
|
|
2061
|
+
plan: "string",
|
|
2062
|
+
seats: "number",
|
|
2063
|
+
mrr: "number",
|
|
2064
|
+
signup_date: "date",
|
|
2065
|
+
churned: "boolean"
|
|
2066
|
+
}
|
|
2067
|
+
},
|
|
2068
|
+
fund: {
|
|
2069
|
+
id: "fund",
|
|
2070
|
+
label: "Fund",
|
|
2071
|
+
columns: {
|
|
2072
|
+
fund_code: "string",
|
|
2073
|
+
fund_name: "string",
|
|
2074
|
+
nav: "number",
|
|
2075
|
+
nav_date: "date",
|
|
2076
|
+
holding_value: "number",
|
|
2077
|
+
currency: "string"
|
|
2078
|
+
}
|
|
2079
|
+
},
|
|
2080
|
+
"real-estate": {
|
|
2081
|
+
id: "real-estate",
|
|
2082
|
+
label: "Real estate",
|
|
2083
|
+
columns: {
|
|
2084
|
+
property_id: "string",
|
|
2085
|
+
listing_price: "number",
|
|
2086
|
+
area_sqm: "number",
|
|
2087
|
+
bedrooms: "number",
|
|
2088
|
+
list_date: "date",
|
|
2089
|
+
city: "string"
|
|
2090
|
+
}
|
|
2091
|
+
},
|
|
2092
|
+
"e-commerce": {
|
|
2093
|
+
id: "e-commerce",
|
|
2094
|
+
label: "E-commerce",
|
|
2095
|
+
columns: {
|
|
2096
|
+
order_id: "string",
|
|
2097
|
+
product_id: "string",
|
|
2098
|
+
quantity: "number",
|
|
2099
|
+
price: "number",
|
|
2100
|
+
order_date: "date",
|
|
2101
|
+
status: "string"
|
|
2102
|
+
}
|
|
2103
|
+
},
|
|
2104
|
+
healthcare: {
|
|
2105
|
+
id: "healthcare",
|
|
2106
|
+
label: "Healthcare",
|
|
2107
|
+
columns: {
|
|
2108
|
+
patient_id: "string",
|
|
2109
|
+
admission_date: "date",
|
|
2110
|
+
discharge_date: "date",
|
|
2111
|
+
age: "number",
|
|
2112
|
+
diagnosis: "string",
|
|
2113
|
+
cost: "number"
|
|
2114
|
+
}
|
|
2115
|
+
},
|
|
2116
|
+
logistics: {
|
|
2117
|
+
id: "logistics",
|
|
2118
|
+
label: "Logistics",
|
|
2119
|
+
columns: {
|
|
2120
|
+
shipment_id: "string",
|
|
2121
|
+
origin: "string",
|
|
2122
|
+
destination: "string",
|
|
2123
|
+
weight_kg: "number",
|
|
2124
|
+
shipped_date: "date",
|
|
2125
|
+
delivered: "boolean"
|
|
2126
|
+
}
|
|
2127
|
+
},
|
|
2128
|
+
manufacturing: {
|
|
2129
|
+
id: "manufacturing",
|
|
2130
|
+
label: "Manufacturing",
|
|
2131
|
+
columns: {
|
|
2132
|
+
part_id: "string",
|
|
2133
|
+
quantity: "number",
|
|
2134
|
+
unit_cost: "number",
|
|
2135
|
+
produced_date: "date",
|
|
2136
|
+
defect: "boolean"
|
|
2137
|
+
}
|
|
2138
|
+
},
|
|
2139
|
+
energy: {
|
|
2140
|
+
id: "energy",
|
|
2141
|
+
label: "Energy",
|
|
2142
|
+
columns: {
|
|
2143
|
+
meter_id: "string",
|
|
2144
|
+
reading: "number",
|
|
2145
|
+
unit: "string",
|
|
2146
|
+
reading_date: "date",
|
|
2147
|
+
consumption: "number"
|
|
2148
|
+
}
|
|
2149
|
+
}
|
|
2150
|
+
};
|
|
2151
|
+
/** The preset ids, for diagnostics and documentation. */
|
|
2152
|
+
const INDUSTRY_PRESET_IDS = Object.keys(INDUSTRY_PRESETS);
|
|
2153
|
+
/**
|
|
2154
|
+
* Resolve an industry preset id to its registry entry, failing loud on an
|
|
2155
|
+
* unknown id.
|
|
2156
|
+
* @param id - the requested preset id.
|
|
2157
|
+
* @returns the preset.
|
|
2158
|
+
*/
|
|
2159
|
+
function resolveIndustryPreset(id) {
|
|
2160
|
+
const preset = INDUSTRY_PRESETS[id];
|
|
2161
|
+
if (preset === void 0) throw new Error(`unknown industryPreset ${JSON.stringify(id)}; choose one of: ${INDUSTRY_PRESET_IDS.join(", ")}`);
|
|
2162
|
+
return preset;
|
|
2163
|
+
}
|
|
2164
|
+
//#endregion
|
|
1412
2165
|
//#region src/present.ts
|
|
1413
2166
|
/** Maximum characters one cell contributes to a tool-facing row payload. */
|
|
1414
2167
|
const MAX_CELL_TEXT = 120;
|
|
@@ -1514,11 +2267,16 @@ var LocalDataQualityService = class extends DataQualityService {
|
|
|
1514
2267
|
/** @inheritdoc DataQualityService.profileDataset */
|
|
1515
2268
|
async profileDataset(request) {
|
|
1516
2269
|
throwIfAborted(request.signal);
|
|
1517
|
-
const
|
|
2270
|
+
const table = await loadTable(resolveWorkspacePath(request.workspace, request.dataset, this.config), this.config, request.signal);
|
|
2271
|
+
const declaredSchema = request.industryPreset === void 0 ? void 0 : resolveIndustryPreset(request.industryPreset).columns;
|
|
2272
|
+
const report = profileTable(table, {
|
|
1518
2273
|
dataset: request.dataset,
|
|
1519
2274
|
sample: request.sample,
|
|
1520
2275
|
generatedAt: this.deps.now(),
|
|
1521
|
-
signal: request.signal
|
|
2276
|
+
signal: request.signal,
|
|
2277
|
+
duplicateSampleLimit: this.config.evidenceRowLimit,
|
|
2278
|
+
declaredSchema,
|
|
2279
|
+
scorecardWeights: this.config.scorecardWeights
|
|
1522
2280
|
});
|
|
1523
2281
|
const reportKey = await this.persist("profile", request.dataset, report);
|
|
1524
2282
|
this.emitEvent(request.session, "profile", request.dataset, reportKey, {
|
|
@@ -1533,10 +2291,12 @@ var LocalDataQualityService = class extends DataQualityService {
|
|
|
1533
2291
|
/** @inheritdoc DataQualityService.cleanDataset */
|
|
1534
2292
|
async cleanDataset(request) {
|
|
1535
2293
|
throwIfAborted(request.signal);
|
|
2294
|
+
const dryRun = request.dryRun === true;
|
|
1536
2295
|
const absolute = resolveWorkspacePath(request.workspace, request.dataset, this.config);
|
|
1537
|
-
const
|
|
2296
|
+
const table = await loadTable(absolute, this.config, request.signal);
|
|
2297
|
+
const result = applyCleanRules(table, request.rules, { signal: request.signal });
|
|
1538
2298
|
let writtenPath;
|
|
1539
|
-
if (request.outputPath !== void 0) {
|
|
2299
|
+
if (!dryRun && request.outputPath !== void 0) {
|
|
1540
2300
|
const outputAbsolute = resolveWorkspacePath(request.workspace, request.outputPath, this.config);
|
|
1541
2301
|
if (outputAbsolute === absolute) throw new Error(`outputPath ${JSON.stringify(request.outputPath)} would overwrite the input dataset; choose a different path`);
|
|
1542
2302
|
const ext = path.extname(outputAbsolute).toLowerCase();
|
|
@@ -1547,19 +2307,51 @@ var LocalDataQualityService = class extends DataQualityService {
|
|
|
1547
2307
|
writtenPath = request.outputPath;
|
|
1548
2308
|
}
|
|
1549
2309
|
const generatedAt = this.deps.now();
|
|
2310
|
+
const contract = computeCleanContract(result, request.rules, { signal: request.signal });
|
|
2311
|
+
const diff = computeCleanProfileDiff(table, {
|
|
2312
|
+
columns: result.columns,
|
|
2313
|
+
rows: result.rows,
|
|
2314
|
+
...table.encoding !== void 0 ? { encoding: table.encoding } : {}
|
|
2315
|
+
}, {
|
|
2316
|
+
dataset: request.dataset,
|
|
2317
|
+
generatedAt,
|
|
2318
|
+
scorecardWeights: this.config.scorecardWeights,
|
|
2319
|
+
signal: request.signal
|
|
2320
|
+
});
|
|
1550
2321
|
const preview = {
|
|
1551
2322
|
columns: result.columns,
|
|
1552
2323
|
rows: result.rows.slice(0, this.config.evidenceRowLimit).map((row) => truncateRow(row))
|
|
1553
2324
|
};
|
|
2325
|
+
if (dryRun) {
|
|
2326
|
+
this.emitEvent(request.session, "clean", request.dataset, void 0, {
|
|
2327
|
+
rows: result.outputRows,
|
|
2328
|
+
columns: result.columns.length,
|
|
2329
|
+
rules: result.logs.length
|
|
2330
|
+
});
|
|
2331
|
+
return {
|
|
2332
|
+
dataset: request.dataset,
|
|
2333
|
+
inputRows: result.inputRows,
|
|
2334
|
+
outputRows: result.outputRows,
|
|
2335
|
+
dryRun: true,
|
|
2336
|
+
logs: result.logs,
|
|
2337
|
+
contract,
|
|
2338
|
+
preview,
|
|
2339
|
+
diffPreview: diff,
|
|
2340
|
+
generatedAt
|
|
2341
|
+
};
|
|
2342
|
+
}
|
|
1554
2343
|
const reportKey = await this.persist("clean", request.dataset, {
|
|
1555
2344
|
dataset: request.dataset,
|
|
1556
2345
|
inputRows: result.inputRows,
|
|
1557
2346
|
outputRows: result.outputRows,
|
|
2347
|
+
dryRun: false,
|
|
1558
2348
|
logs: result.logs,
|
|
2349
|
+
contract,
|
|
1559
2350
|
preview,
|
|
1560
2351
|
...writtenPath !== void 0 ? { outputPath: writtenPath } : {},
|
|
1561
2352
|
generatedAt
|
|
1562
2353
|
});
|
|
2354
|
+
await this.persist("clean-diff", request.dataset, diff);
|
|
1563
2355
|
this.emitEvent(request.session, "clean", request.dataset, reportKey, {
|
|
1564
2356
|
rows: result.outputRows,
|
|
1565
2357
|
columns: result.columns.length,
|
|
@@ -1569,32 +2361,57 @@ var LocalDataQualityService = class extends DataQualityService {
|
|
|
1569
2361
|
dataset: request.dataset,
|
|
1570
2362
|
inputRows: result.inputRows,
|
|
1571
2363
|
outputRows: result.outputRows,
|
|
2364
|
+
dryRun: false,
|
|
1572
2365
|
logs: result.logs,
|
|
2366
|
+
contract,
|
|
1573
2367
|
preview,
|
|
1574
2368
|
...writtenPath !== void 0 ? { outputPath: writtenPath } : {},
|
|
1575
2369
|
...reportKey !== void 0 ? { reportKey } : {},
|
|
1576
2370
|
generatedAt
|
|
1577
2371
|
};
|
|
1578
2372
|
}
|
|
2373
|
+
/** @inheritdoc DataQualityService.getReport */
|
|
2374
|
+
async getReport(key) {
|
|
2375
|
+
if (!isValidReportKey(key)) throw new Error(`invalid reportKey ${JSON.stringify(key)}: expected the deterministic <timestamp>-<kind>-<fingerprint> format`);
|
|
2376
|
+
const store = this.deps.store;
|
|
2377
|
+
if (store === void 0) throw new Error("report storage is disabled (storeReports is false); no persisted reports to read");
|
|
2378
|
+
const record = store.get(key);
|
|
2379
|
+
if (record === void 0) throw new Error(`no persisted report found for reportKey ${JSON.stringify(key)}`);
|
|
2380
|
+
return {
|
|
2381
|
+
key,
|
|
2382
|
+
...record
|
|
2383
|
+
};
|
|
2384
|
+
}
|
|
2385
|
+
/** @inheritdoc DataQualityService.listReports */
|
|
2386
|
+
async listReports(kind) {
|
|
2387
|
+
const store = this.deps.store;
|
|
2388
|
+
if (store === void 0) throw new Error("report storage is disabled (storeReports is false); no persisted reports to read");
|
|
2389
|
+
return store.list(kind);
|
|
2390
|
+
}
|
|
1579
2391
|
/** @inheritdoc DataQualityService.verifyDataset */
|
|
1580
2392
|
async verifyDataset(request) {
|
|
1581
2393
|
throwIfAborted(request.signal);
|
|
1582
2394
|
const outcome = verifyTable(await loadTable(resolveWorkspacePath(request.workspace, request.dataset, this.config), this.config, request.signal), request.rules, {
|
|
1583
2395
|
evidenceRowLimit: this.config.evidenceRowLimit,
|
|
1584
2396
|
now: this.deps.now,
|
|
1585
|
-
signal: request.signal
|
|
2397
|
+
signal: request.signal,
|
|
2398
|
+
expectations: request.expectations,
|
|
2399
|
+
defaultTolerance: this.config.defaultTolerance
|
|
1586
2400
|
});
|
|
1587
2401
|
const report = {
|
|
1588
2402
|
dataset: request.dataset,
|
|
1589
2403
|
...outcome
|
|
1590
2404
|
};
|
|
1591
2405
|
const failedRules = report.rules.filter((rule) => !rule.passed).length;
|
|
2406
|
+
const failedExpectations = report.expectations.filter((expectation) => !expectation.passed).length;
|
|
1592
2407
|
const reportKey = await this.persist("verify", request.dataset, report);
|
|
1593
2408
|
this.emitEvent(request.session, "verify", request.dataset, reportKey, {
|
|
1594
2409
|
rows: report.rowCount,
|
|
1595
2410
|
rules: report.rules.length,
|
|
1596
2411
|
failedRules,
|
|
1597
|
-
passed: report.passed
|
|
2412
|
+
passed: report.passed,
|
|
2413
|
+
expectations: report.expectations.length,
|
|
2414
|
+
failedExpectations
|
|
1598
2415
|
});
|
|
1599
2416
|
return {
|
|
1600
2417
|
...report,
|
|
@@ -1622,48 +2439,22 @@ var LocalDataQualityService = class extends DataQualityService {
|
|
|
1622
2439
|
}
|
|
1623
2440
|
};
|
|
1624
2441
|
//#endregion
|
|
1625
|
-
//#region src/tools/
|
|
1626
|
-
/**
|
|
1627
|
-
* Shared helpers for the three data-quality model tools: session workspace
|
|
1628
|
-
* resolution and a compact cleaning-run text render. Kept presentation-only;
|
|
1629
|
-
* computation lives in the engines and the provider.
|
|
1630
|
-
* @module dsh-data-quality/tools-shared
|
|
1631
|
-
*/
|
|
2442
|
+
//#region src/tools/profile-report-schema.ts
|
|
1632
2443
|
/**
|
|
1633
|
-
*
|
|
1634
|
-
*
|
|
1635
|
-
*
|
|
1636
|
-
* @
|
|
1637
|
-
* @returns the absolute workspace root.
|
|
1638
|
-
*/
|
|
1639
|
-
function workspaceOf(exec) {
|
|
1640
|
-
const cwd = exec.agent?.session.header.cwd;
|
|
1641
|
-
if (cwd === void 0) throw new Error("data-quality tools require an agent-owned session workspace");
|
|
1642
|
-
return path.resolve(cwd);
|
|
1643
|
-
}
|
|
1644
|
-
/** Human-readable cleaning summary for the tool's Native render. */
|
|
1645
|
-
function renderCleanText(report) {
|
|
1646
|
-
const lines = [];
|
|
1647
|
-
lines.push(`Cleaned ${report.dataset}: ${report.inputRows} -> ${report.outputRows} rows over ${report.logs.length} rule(s)`);
|
|
1648
|
-
for (const log of report.logs) lines.push(`- rule ${log.ruleIndex} (${log.rule}): ${log.affectedRows} row(s) affected; ${log.detail}`);
|
|
1649
|
-
if (report.outputPath !== void 0) lines.push(`Wrote cleaned dataset to ${report.outputPath}`);
|
|
1650
|
-
else lines.push("No outputPath given: the source file was left untouched; preview below.");
|
|
1651
|
-
if (report.preview.rows.length > 0) {
|
|
1652
|
-
lines.push(`Preview (first ${report.preview.rows.length} row(s)):`);
|
|
1653
|
-
for (const row of report.preview.rows) {
|
|
1654
|
-
const cells = report.preview.columns.map((column) => `${column}=${JSON.stringify(row[column] ?? null)}`).join(", ");
|
|
1655
|
-
lines.push(` ${cells}`);
|
|
1656
|
-
}
|
|
1657
|
-
}
|
|
1658
|
-
return lines.join("\n");
|
|
1659
|
-
}
|
|
1660
|
-
//#endregion
|
|
1661
|
-
//#region src/tools/profile.ts
|
|
1662
|
-
/**
|
|
1663
|
-
* The `data_profile` model tool: deterministic dataset profiling through
|
|
1664
|
-
* `ctx.dataQuality.profileDataset` — never model arithmetic.
|
|
1665
|
-
* @module dsh-data-quality/tools/profile
|
|
2444
|
+
* Shared JSON-Schema spec for the profile report's canonical value, reused by
|
|
2445
|
+
* `data_profile` (its output) and `data_clean` (the `diffPreview` before/after
|
|
2446
|
+
* snapshots). One source of truth so the two tools never drift.
|
|
2447
|
+
* @module dsh-data-quality/tools-profile-report-schema
|
|
1666
2448
|
*/
|
|
2449
|
+
const SCORECARD_DIMENSION_NAMES = [
|
|
2450
|
+
"completeness",
|
|
2451
|
+
"uniqueness",
|
|
2452
|
+
"validity",
|
|
2453
|
+
"consistency",
|
|
2454
|
+
"timeliness",
|
|
2455
|
+
"accuracy"
|
|
2456
|
+
];
|
|
2457
|
+
/** One column card's schema. */
|
|
1667
2458
|
const COLUMN_PROFILE_SCHEMA = {
|
|
1668
2459
|
type: "object",
|
|
1669
2460
|
properties: {
|
|
@@ -1698,6 +2489,14 @@ const COLUMN_PROFILE_SCHEMA = {
|
|
|
1698
2489
|
numeric: {
|
|
1699
2490
|
type: "object",
|
|
1700
2491
|
properties: {
|
|
2492
|
+
count: {
|
|
2493
|
+
type: "number",
|
|
2494
|
+
required: true
|
|
2495
|
+
},
|
|
2496
|
+
distinct: {
|
|
2497
|
+
type: "number",
|
|
2498
|
+
required: true
|
|
2499
|
+
},
|
|
1701
2500
|
min: {
|
|
1702
2501
|
type: "number",
|
|
1703
2502
|
required: true
|
|
@@ -1754,6 +2553,167 @@ const COLUMN_PROFILE_SCHEMA = {
|
|
|
1754
2553
|
},
|
|
1755
2554
|
additionalProperties: false
|
|
1756
2555
|
};
|
|
2556
|
+
/** The full profile report's schema. */
|
|
2557
|
+
const PROFILE_REPORT_SCHEMA = {
|
|
2558
|
+
type: "object",
|
|
2559
|
+
properties: {
|
|
2560
|
+
schemaVersion: {
|
|
2561
|
+
type: "number",
|
|
2562
|
+
required: true
|
|
2563
|
+
},
|
|
2564
|
+
dataset: {
|
|
2565
|
+
type: "string",
|
|
2566
|
+
required: true
|
|
2567
|
+
},
|
|
2568
|
+
rowCount: {
|
|
2569
|
+
type: "number",
|
|
2570
|
+
required: true
|
|
2571
|
+
},
|
|
2572
|
+
sampled: {
|
|
2573
|
+
type: "boolean",
|
|
2574
|
+
required: true
|
|
2575
|
+
},
|
|
2576
|
+
profiledRows: {
|
|
2577
|
+
type: "number",
|
|
2578
|
+
required: true
|
|
2579
|
+
},
|
|
2580
|
+
columnCount: {
|
|
2581
|
+
type: "number",
|
|
2582
|
+
required: true
|
|
2583
|
+
},
|
|
2584
|
+
duplicateRows: {
|
|
2585
|
+
type: "number",
|
|
2586
|
+
required: true
|
|
2587
|
+
},
|
|
2588
|
+
duplicateRate: {
|
|
2589
|
+
type: "number",
|
|
2590
|
+
required: true
|
|
2591
|
+
},
|
|
2592
|
+
duplicateSampleRowIndexes: {
|
|
2593
|
+
type: "array",
|
|
2594
|
+
items: { type: "number" },
|
|
2595
|
+
required: true
|
|
2596
|
+
},
|
|
2597
|
+
scorecard: {
|
|
2598
|
+
type: "object",
|
|
2599
|
+
properties: {
|
|
2600
|
+
overall: {
|
|
2601
|
+
oneOf: [{ type: "number" }, { type: "null" }],
|
|
2602
|
+
required: true
|
|
2603
|
+
},
|
|
2604
|
+
weightedOverall: {
|
|
2605
|
+
oneOf: [{ type: "number" }, { type: "null" }],
|
|
2606
|
+
required: true
|
|
2607
|
+
},
|
|
2608
|
+
dimensions: {
|
|
2609
|
+
type: "array",
|
|
2610
|
+
items: {
|
|
2611
|
+
type: "object",
|
|
2612
|
+
properties: {
|
|
2613
|
+
name: {
|
|
2614
|
+
type: "string",
|
|
2615
|
+
enum: [...SCORECARD_DIMENSION_NAMES],
|
|
2616
|
+
required: true
|
|
2617
|
+
},
|
|
2618
|
+
score: {
|
|
2619
|
+
oneOf: [{ type: "number" }, { type: "null" }],
|
|
2620
|
+
required: true
|
|
2621
|
+
},
|
|
2622
|
+
note: {
|
|
2623
|
+
type: "string",
|
|
2624
|
+
required: true
|
|
2625
|
+
}
|
|
2626
|
+
},
|
|
2627
|
+
additionalProperties: false
|
|
2628
|
+
},
|
|
2629
|
+
required: true
|
|
2630
|
+
}
|
|
2631
|
+
},
|
|
2632
|
+
additionalProperties: false,
|
|
2633
|
+
required: true
|
|
2634
|
+
},
|
|
2635
|
+
encoding: {
|
|
2636
|
+
type: "object",
|
|
2637
|
+
properties: {
|
|
2638
|
+
bom: {
|
|
2639
|
+
oneOf: [{ type: "string" }, { type: "null" }],
|
|
2640
|
+
required: true
|
|
2641
|
+
},
|
|
2642
|
+
validUtf8: {
|
|
2643
|
+
type: "boolean",
|
|
2644
|
+
required: true
|
|
2645
|
+
}
|
|
2646
|
+
},
|
|
2647
|
+
additionalProperties: false
|
|
2648
|
+
},
|
|
2649
|
+
generatedAt: {
|
|
2650
|
+
type: "number",
|
|
2651
|
+
required: true
|
|
2652
|
+
},
|
|
2653
|
+
reportKey: { type: "string" },
|
|
2654
|
+
columns: {
|
|
2655
|
+
type: "array",
|
|
2656
|
+
items: COLUMN_PROFILE_SCHEMA,
|
|
2657
|
+
required: true
|
|
2658
|
+
}
|
|
2659
|
+
},
|
|
2660
|
+
additionalProperties: false
|
|
2661
|
+
};
|
|
2662
|
+
//#endregion
|
|
2663
|
+
//#region src/tools/shared.ts
|
|
2664
|
+
/**
|
|
2665
|
+
* Shared helpers for the three data-quality model tools: session workspace
|
|
2666
|
+
* resolution and a compact cleaning-run text render. Kept presentation-only;
|
|
2667
|
+
* computation lives in the engines and the provider.
|
|
2668
|
+
* @module dsh-data-quality/tools-shared
|
|
2669
|
+
*/
|
|
2670
|
+
/**
|
|
2671
|
+
* The absolute workspace root a tool call resolves dataset paths against: the
|
|
2672
|
+
* calling agent's per-session cwd (mirroring the official fs tools). Non-agent
|
|
2673
|
+
* calls fail loud — there is no honest workspace to confine paths to.
|
|
2674
|
+
* @param exec - the tool-execution context.
|
|
2675
|
+
* @returns the absolute workspace root.
|
|
2676
|
+
*/
|
|
2677
|
+
function workspaceOf(exec) {
|
|
2678
|
+
const cwd = exec.agent?.session.header.cwd;
|
|
2679
|
+
if (cwd === void 0) throw new Error("data-quality tools require an agent-owned session workspace");
|
|
2680
|
+
return path.resolve(cwd);
|
|
2681
|
+
}
|
|
2682
|
+
/** Human-readable cleaning summary for the tool's Native render. */
|
|
2683
|
+
function renderCleanText(report) {
|
|
2684
|
+
const lines = [];
|
|
2685
|
+
const mode = report.dryRun ? "Dry-run plan for" : "Cleaned";
|
|
2686
|
+
lines.push(`${mode} ${report.dataset}: ${report.inputRows} -> ${report.outputRows} rows over ${report.logs.length} rule(s)`);
|
|
2687
|
+
for (const log of report.logs) lines.push(`- rule ${log.ruleIndex} (${log.rule}): ${log.affectedRows} row(s) affected; ${log.detail}`);
|
|
2688
|
+
const contract = report.contract;
|
|
2689
|
+
const keyLabel = contract.dedupeColumns === null ? "full rows" : `[${contract.dedupeColumns.join(", ")}]`;
|
|
2690
|
+
lines.push(`Contract: ${contract.inputRows} -> ${contract.outputRows} rows (${contract.removedRows} removed); uniqueness ${contract.uniqueKeys ? "OK" : "VIOLATED"} over ${keyLabel}${contract.remainingDuplicateRows > 0 ? ` (${contract.remainingDuplicateRows} duplicate row(s) remain)` : ""}`);
|
|
2691
|
+
for (const entry of contract.remainingMissing) lines.push(`- non-null regression: ${entry.column} still has ${entry.count} missing cell(s)`);
|
|
2692
|
+
for (const entry of contract.typeConformance) lines.push(`- type regression: ${entry.column} (${entry.to}) has ${entry.invalidCount} non-conforming cell(s)`);
|
|
2693
|
+
for (const entry of contract.columnDecisions) {
|
|
2694
|
+
const decisions = entry.decisions.map((decision) => `${decision.strategy} (${decision.affectedRows} row(s))`).join(", ");
|
|
2695
|
+
lines.push(`- column ${entry.column}: ${decisions}`);
|
|
2696
|
+
}
|
|
2697
|
+
if (report.diffPreview !== void 0) lines.push(`Diff preview: ${report.diffPreview.before.rowCount} -> ${report.diffPreview.after.rowCount} rows; duplicates ${report.diffPreview.before.duplicateRows} -> ${report.diffPreview.after.duplicateRows}`);
|
|
2698
|
+
if (report.dryRun) lines.push("Dry run: no output file written and no report persisted; plan/preview above.");
|
|
2699
|
+
else if (report.outputPath !== void 0) lines.push(`Wrote cleaned dataset to ${report.outputPath}`);
|
|
2700
|
+
else lines.push("No outputPath given: the source file was left untouched; preview below.");
|
|
2701
|
+
if (report.preview.rows.length > 0) {
|
|
2702
|
+
lines.push(`Preview (first ${report.preview.rows.length} row(s)):`);
|
|
2703
|
+
for (const row of report.preview.rows) {
|
|
2704
|
+
const cells = report.preview.columns.map((column) => `${column}=${JSON.stringify(row[column] ?? null)}`).join(", ");
|
|
2705
|
+
lines.push(` ${cells}`);
|
|
2706
|
+
}
|
|
2707
|
+
}
|
|
2708
|
+
return lines.join("\n");
|
|
2709
|
+
}
|
|
2710
|
+
//#endregion
|
|
2711
|
+
//#region src/tools/profile.ts
|
|
2712
|
+
/**
|
|
2713
|
+
* The `data_profile` model tool: deterministic dataset profiling through
|
|
2714
|
+
* `ctx.dataQuality.profileDataset` — never model arithmetic.
|
|
2715
|
+
* @module dsh-data-quality/tools/profile
|
|
2716
|
+
*/
|
|
1757
2717
|
/**
|
|
1758
2718
|
* Build the `data_profile` tool definition against a mounted service.
|
|
1759
2719
|
* @param service - the mounted ctx.dataQuality implementation.
|
|
@@ -1764,8 +2724,8 @@ function defineProfileTool(service) {
|
|
|
1764
2724
|
name: "data_profile",
|
|
1765
2725
|
description: [
|
|
1766
2726
|
"Profile a workspace CSV/TSV/JSON/JSONL dataset with deterministic TypeScript computation (no mental math).",
|
|
1767
|
-
"Returns row/column counts, inferred column types, missing rates, unique counts, numeric distributions (min/max/mean/median/p25/p75), IQR outlier counts, mixed-type suspicion notes,
|
|
1768
|
-
"Column cards cover every row by default; pass sample for a deterministic systematic sample on large files. Datasets above the configured row/size caps are rejected — use sample or raise the caps. The full report persists to the data_quality storage domain (reportKey in the result)."
|
|
2727
|
+
"Returns row/column counts, inferred column types, missing rates, unique counts, numeric distributions (count/distinct/min/max/mean/median/p25/p75), IQR outlier counts, mixed-type suspicion notes, sha256 duplicate-row detection (rate + sample indexes), file encoding (BOM/UTF-8 validity), and a weighted DAMA six-dimension scorecard.",
|
|
2728
|
+
"Pass industryPreset (retail/saas/fund/real-estate/e-commerce/healthcare/logistics/manufacturing/energy) to compare the dataset against that industry's expected columns, making the scorecard accuracy dimension determinable. Column cards cover every row by default; pass sample for a deterministic systematic sample on large files. Datasets above the configured row/size caps are rejected — use sample or raise the caps. The full report persists to the data_quality storage domain (reportKey in the result)."
|
|
1769
2729
|
].join("\n"),
|
|
1770
2730
|
parameters: {
|
|
1771
2731
|
path: {
|
|
@@ -1776,49 +2736,25 @@ function defineProfileTool(service) {
|
|
|
1776
2736
|
sample: {
|
|
1777
2737
|
type: "number",
|
|
1778
2738
|
description: "Optional systematic sample size (every ceil(N/sample)-th row) for the column cards; row counts stay exact."
|
|
2739
|
+
},
|
|
2740
|
+
industryPreset: {
|
|
2741
|
+
type: "string",
|
|
2742
|
+
enum: [
|
|
2743
|
+
"retail",
|
|
2744
|
+
"saas",
|
|
2745
|
+
"fund",
|
|
2746
|
+
"real-estate",
|
|
2747
|
+
"e-commerce",
|
|
2748
|
+
"healthcare",
|
|
2749
|
+
"logistics",
|
|
2750
|
+
"manufacturing",
|
|
2751
|
+
"energy"
|
|
2752
|
+
],
|
|
2753
|
+
description: "Optional industry preset id; its expected columns feed the scorecard accuracy dimension."
|
|
1779
2754
|
}
|
|
1780
2755
|
},
|
|
1781
2756
|
output: {
|
|
1782
|
-
schema:
|
|
1783
|
-
type: "object",
|
|
1784
|
-
properties: {
|
|
1785
|
-
dataset: {
|
|
1786
|
-
type: "string",
|
|
1787
|
-
required: true
|
|
1788
|
-
},
|
|
1789
|
-
rowCount: {
|
|
1790
|
-
type: "number",
|
|
1791
|
-
required: true
|
|
1792
|
-
},
|
|
1793
|
-
sampled: {
|
|
1794
|
-
type: "boolean",
|
|
1795
|
-
required: true
|
|
1796
|
-
},
|
|
1797
|
-
profiledRows: {
|
|
1798
|
-
type: "number",
|
|
1799
|
-
required: true
|
|
1800
|
-
},
|
|
1801
|
-
columnCount: {
|
|
1802
|
-
type: "number",
|
|
1803
|
-
required: true
|
|
1804
|
-
},
|
|
1805
|
-
duplicateRows: {
|
|
1806
|
-
type: "number",
|
|
1807
|
-
required: true
|
|
1808
|
-
},
|
|
1809
|
-
generatedAt: {
|
|
1810
|
-
type: "number",
|
|
1811
|
-
required: true
|
|
1812
|
-
},
|
|
1813
|
-
reportKey: { type: "string" },
|
|
1814
|
-
columns: {
|
|
1815
|
-
type: "array",
|
|
1816
|
-
items: COLUMN_PROFILE_SCHEMA,
|
|
1817
|
-
required: true
|
|
1818
|
-
}
|
|
1819
|
-
},
|
|
1820
|
-
additionalProperties: false
|
|
1821
|
-
},
|
|
2757
|
+
schema: PROFILE_REPORT_SCHEMA,
|
|
1822
2758
|
render: (_args, value) => [{
|
|
1823
2759
|
type: "text",
|
|
1824
2760
|
text: renderProfileText(value)
|
|
@@ -1828,6 +2764,7 @@ function defineProfileTool(service) {
|
|
|
1828
2764
|
return service.profileDataset({
|
|
1829
2765
|
dataset: args.path,
|
|
1830
2766
|
sample: args.sample,
|
|
2767
|
+
...args.industryPreset !== void 0 ? { industryPreset: args.industryPreset } : {},
|
|
1831
2768
|
workspace: workspaceOf(exec),
|
|
1832
2769
|
session: exec.agent?.session,
|
|
1833
2770
|
signal: exec.signal
|
|
@@ -1995,7 +2932,7 @@ function defineCleanTool(service) {
|
|
|
1995
2932
|
description: [
|
|
1996
2933
|
"Apply declarative cleaning rules to a workspace CSV/TSV/JSON/JSONL dataset with deterministic TypeScript computation (no mental math).",
|
|
1997
2934
|
"Rules apply in array order: dedupe (by column group), fill-missing (constant/mean/median/forward), coerce-type (number/date/boolean; failures counted and set to missing), normalize-unit (e.g. 万/亿 suffixes to base units), trim (whitespace), map-values (enum mapping).",
|
|
1998
|
-
"The source file is NEVER overwritten. Without outputPath the run is preview-only; with outputPath the cleaned dataset is written there (workspace-confined, .csv/.tsv/.json/.jsonl). Returns the per-rule audit log (
|
|
2935
|
+
"The source file is NEVER overwritten. Without outputPath the run is preview-only; with outputPath the cleaned dataset is written there (workspace-confined, .csv/.tsv/.json/.jsonl). Returns the per-rule audit log, the pre-delivery contract summary (with per-column decision trace), and a bounded preview. Pass dryRun: true to skip the write and get the cleaning plan plus the expected contract/diff preview instead. The full report persists to the data_quality storage domain (reportKey)."
|
|
1999
2936
|
].join("\n"),
|
|
2000
2937
|
parameters: {
|
|
2001
2938
|
path: {
|
|
@@ -2010,6 +2947,10 @@ function defineCleanTool(service) {
|
|
|
2010
2947
|
outputPath: {
|
|
2011
2948
|
type: "string",
|
|
2012
2949
|
description: "Optional workspace-relative output path for the cleaned dataset (must differ from path)."
|
|
2950
|
+
},
|
|
2951
|
+
dryRun: {
|
|
2952
|
+
type: "boolean",
|
|
2953
|
+
description: "When true, do not write any output file; return the cleaning plan and expected contract/diff preview instead (default false)."
|
|
2013
2954
|
}
|
|
2014
2955
|
},
|
|
2015
2956
|
output: {
|
|
@@ -2028,12 +2969,38 @@ function defineCleanTool(service) {
|
|
|
2028
2969
|
type: "number",
|
|
2029
2970
|
required: true
|
|
2030
2971
|
},
|
|
2972
|
+
dryRun: {
|
|
2973
|
+
type: "boolean",
|
|
2974
|
+
required: true
|
|
2975
|
+
},
|
|
2031
2976
|
generatedAt: {
|
|
2032
2977
|
type: "number",
|
|
2033
2978
|
required: true
|
|
2034
2979
|
},
|
|
2035
2980
|
outputPath: { type: "string" },
|
|
2036
2981
|
reportKey: { type: "string" },
|
|
2982
|
+
diffPreview: {
|
|
2983
|
+
type: "object",
|
|
2984
|
+
properties: {
|
|
2985
|
+
dataset: {
|
|
2986
|
+
type: "string",
|
|
2987
|
+
required: true
|
|
2988
|
+
},
|
|
2989
|
+
before: {
|
|
2990
|
+
...PROFILE_REPORT_SCHEMA,
|
|
2991
|
+
required: true
|
|
2992
|
+
},
|
|
2993
|
+
after: {
|
|
2994
|
+
...PROFILE_REPORT_SCHEMA,
|
|
2995
|
+
required: true
|
|
2996
|
+
},
|
|
2997
|
+
generatedAt: {
|
|
2998
|
+
type: "number",
|
|
2999
|
+
required: true
|
|
3000
|
+
}
|
|
3001
|
+
},
|
|
3002
|
+
additionalProperties: false
|
|
3003
|
+
},
|
|
2037
3004
|
logs: {
|
|
2038
3005
|
type: "array",
|
|
2039
3006
|
items: {
|
|
@@ -2068,6 +3035,117 @@ function defineCleanTool(service) {
|
|
|
2068
3035
|
},
|
|
2069
3036
|
required: true
|
|
2070
3037
|
},
|
|
3038
|
+
contract: {
|
|
3039
|
+
type: "object",
|
|
3040
|
+
properties: {
|
|
3041
|
+
inputRows: {
|
|
3042
|
+
type: "number",
|
|
3043
|
+
required: true
|
|
3044
|
+
},
|
|
3045
|
+
outputRows: {
|
|
3046
|
+
type: "number",
|
|
3047
|
+
required: true
|
|
3048
|
+
},
|
|
3049
|
+
removedRows: {
|
|
3050
|
+
type: "number",
|
|
3051
|
+
required: true
|
|
3052
|
+
},
|
|
3053
|
+
dedupeColumns: {
|
|
3054
|
+
oneOf: [{
|
|
3055
|
+
type: "array",
|
|
3056
|
+
items: { type: "string" }
|
|
3057
|
+
}, { type: "null" }],
|
|
3058
|
+
required: true
|
|
3059
|
+
},
|
|
3060
|
+
uniqueKeys: {
|
|
3061
|
+
type: "boolean",
|
|
3062
|
+
required: true
|
|
3063
|
+
},
|
|
3064
|
+
remainingDuplicateRows: {
|
|
3065
|
+
type: "number",
|
|
3066
|
+
required: true
|
|
3067
|
+
},
|
|
3068
|
+
remainingMissing: {
|
|
3069
|
+
type: "array",
|
|
3070
|
+
items: {
|
|
3071
|
+
type: "object",
|
|
3072
|
+
properties: {
|
|
3073
|
+
column: {
|
|
3074
|
+
type: "string",
|
|
3075
|
+
required: true
|
|
3076
|
+
},
|
|
3077
|
+
count: {
|
|
3078
|
+
type: "number",
|
|
3079
|
+
required: true
|
|
3080
|
+
}
|
|
3081
|
+
},
|
|
3082
|
+
additionalProperties: false
|
|
3083
|
+
},
|
|
3084
|
+
required: true
|
|
3085
|
+
},
|
|
3086
|
+
typeConformance: {
|
|
3087
|
+
type: "array",
|
|
3088
|
+
items: {
|
|
3089
|
+
type: "object",
|
|
3090
|
+
properties: {
|
|
3091
|
+
column: {
|
|
3092
|
+
type: "string",
|
|
3093
|
+
required: true
|
|
3094
|
+
},
|
|
3095
|
+
to: {
|
|
3096
|
+
type: "string",
|
|
3097
|
+
enum: [
|
|
3098
|
+
"number",
|
|
3099
|
+
"date",
|
|
3100
|
+
"boolean"
|
|
3101
|
+
],
|
|
3102
|
+
required: true
|
|
3103
|
+
},
|
|
3104
|
+
invalidCount: {
|
|
3105
|
+
type: "number",
|
|
3106
|
+
required: true
|
|
3107
|
+
}
|
|
3108
|
+
},
|
|
3109
|
+
additionalProperties: false
|
|
3110
|
+
},
|
|
3111
|
+
required: true
|
|
3112
|
+
},
|
|
3113
|
+
columnDecisions: {
|
|
3114
|
+
type: "array",
|
|
3115
|
+
items: {
|
|
3116
|
+
type: "object",
|
|
3117
|
+
properties: {
|
|
3118
|
+
column: {
|
|
3119
|
+
type: "string",
|
|
3120
|
+
required: true
|
|
3121
|
+
},
|
|
3122
|
+
decisions: {
|
|
3123
|
+
type: "array",
|
|
3124
|
+
items: {
|
|
3125
|
+
type: "object",
|
|
3126
|
+
properties: {
|
|
3127
|
+
strategy: {
|
|
3128
|
+
type: "string",
|
|
3129
|
+
required: true
|
|
3130
|
+
},
|
|
3131
|
+
affectedRows: {
|
|
3132
|
+
type: "number",
|
|
3133
|
+
required: true
|
|
3134
|
+
}
|
|
3135
|
+
},
|
|
3136
|
+
additionalProperties: false
|
|
3137
|
+
},
|
|
3138
|
+
required: true
|
|
3139
|
+
}
|
|
3140
|
+
},
|
|
3141
|
+
additionalProperties: false
|
|
3142
|
+
},
|
|
3143
|
+
required: true
|
|
3144
|
+
}
|
|
3145
|
+
},
|
|
3146
|
+
additionalProperties: false,
|
|
3147
|
+
required: true
|
|
3148
|
+
},
|
|
2071
3149
|
preview: {
|
|
2072
3150
|
type: "object",
|
|
2073
3151
|
properties: {
|
|
@@ -2098,6 +3176,7 @@ function defineCleanTool(service) {
|
|
|
2098
3176
|
dataset: args.path,
|
|
2099
3177
|
rules: args.rules,
|
|
2100
3178
|
...args.outputPath !== void 0 ? { outputPath: args.outputPath } : {},
|
|
3179
|
+
...args.dryRun !== void 0 ? { dryRun: args.dryRun } : {},
|
|
2101
3180
|
workspace: workspaceOf(exec),
|
|
2102
3181
|
session: exec.agent?.session,
|
|
2103
3182
|
signal: exec.signal
|
|
@@ -2113,6 +3192,74 @@ function defineCleanTool(service) {
|
|
|
2113
3192
|
* (`passed: false` with evidence), never a tool error.
|
|
2114
3193
|
* @module dsh-data-quality/tools/verify
|
|
2115
3194
|
*/
|
|
3195
|
+
const EXPECTATION_METRICS = [
|
|
3196
|
+
"rowCount",
|
|
3197
|
+
"columnSum",
|
|
3198
|
+
"columnMean",
|
|
3199
|
+
"uniqueCount",
|
|
3200
|
+
"nullCount"
|
|
3201
|
+
];
|
|
3202
|
+
const EXPECTATION_SCHEMA = {
|
|
3203
|
+
type: "array",
|
|
3204
|
+
items: {
|
|
3205
|
+
type: "object",
|
|
3206
|
+
properties: {
|
|
3207
|
+
metric: {
|
|
3208
|
+
type: "string",
|
|
3209
|
+
enum: [...EXPECTATION_METRICS],
|
|
3210
|
+
required: true,
|
|
3211
|
+
description: "Metric to reconcile: rowCount/columnSum/columnMean/uniqueCount/nullCount."
|
|
3212
|
+
},
|
|
3213
|
+
column: {
|
|
3214
|
+
type: "string",
|
|
3215
|
+
description: "Required for every metric except rowCount."
|
|
3216
|
+
},
|
|
3217
|
+
expected: {
|
|
3218
|
+
type: "number",
|
|
3219
|
+
required: true,
|
|
3220
|
+
description: "The expected value to reconcile against."
|
|
3221
|
+
},
|
|
3222
|
+
tolerance: {
|
|
3223
|
+
type: "number",
|
|
3224
|
+
description: "Optional relative tolerance in [0, 1]; defaults to defaultTolerance."
|
|
3225
|
+
}
|
|
3226
|
+
},
|
|
3227
|
+
additionalProperties: false,
|
|
3228
|
+
description: "Reconcile a deterministic computed metric against an expected value with relative tolerance."
|
|
3229
|
+
},
|
|
3230
|
+
description: "Optional metric expectations; each yields passed true or passed false with actual/expected/tolerance detail."
|
|
3231
|
+
};
|
|
3232
|
+
const EXPECTATION_RESULT_SCHEMA = {
|
|
3233
|
+
type: "array",
|
|
3234
|
+
items: {
|
|
3235
|
+
type: "object",
|
|
3236
|
+
properties: {
|
|
3237
|
+
metric: {
|
|
3238
|
+
type: "string",
|
|
3239
|
+
enum: [...EXPECTATION_METRICS],
|
|
3240
|
+
required: true
|
|
3241
|
+
},
|
|
3242
|
+
column: { type: "string" },
|
|
3243
|
+
expected: {
|
|
3244
|
+
type: "number",
|
|
3245
|
+
required: true
|
|
3246
|
+
},
|
|
3247
|
+
actual: {
|
|
3248
|
+
type: "number",
|
|
3249
|
+
required: true
|
|
3250
|
+
},
|
|
3251
|
+
tolerance: {
|
|
3252
|
+
type: "number",
|
|
3253
|
+
required: true
|
|
3254
|
+
},
|
|
3255
|
+
passed: {
|
|
3256
|
+
type: "boolean",
|
|
3257
|
+
required: true
|
|
3258
|
+
}
|
|
3259
|
+
},
|
|
3260
|
+
additionalProperties: false
|
|
3261
|
+
}
|
|
3262
|
+
};
|
|
2116
3263
|
const VERIFY_RULE_SCHEMA = {
|
|
2117
3264
|
type: "array",
|
|
2118
3265
|
items: { oneOf: [
|
|
@@ -2290,7 +3437,8 @@ function defineVerifyTool(service) {
|
|
|
2290
3437
|
description: [
|
|
2291
3438
|
"Verify a workspace CSV/TSV/JSON/JSONL dataset against declarative quality rules with deterministic TypeScript computation (no mental math).",
|
|
2292
3439
|
"Rules: not-null, unique (column group), range (numeric bounds), regex, enum, cross-column (e.g. startDate < endDate), freshness (date column within N days of asOf). A missing cell fails every rule that reads it.",
|
|
2293
|
-
"
|
|
3440
|
+
"Optional expectations reconcile deterministic metrics (rowCount/columnSum/columnMean/uniqueCount/nullCount) against expected values with relative tolerance; a mismatch is a normal passed: false with actual/expected/tolerance detail, never a tool error.",
|
|
3441
|
+
"Returns per-rule pass/fail with capped failing-row evidence plus the expectation outcomes. Overall failure is a NORMAL result with passed: false — not a tool error. The full report persists to the data_quality storage domain (reportKey)."
|
|
2294
3442
|
].join("\n"),
|
|
2295
3443
|
parameters: {
|
|
2296
3444
|
path: {
|
|
@@ -2301,6 +3449,10 @@ function defineVerifyTool(service) {
|
|
|
2301
3449
|
rules: {
|
|
2302
3450
|
...VERIFY_RULE_SCHEMA,
|
|
2303
3451
|
required: true
|
|
3452
|
+
},
|
|
3453
|
+
expectations: {
|
|
3454
|
+
...EXPECTATION_SCHEMA,
|
|
3455
|
+
description: "Optional metric expectations to reconcile (rowCount/columnSum/columnMean/uniqueCount/nullCount)."
|
|
2304
3456
|
}
|
|
2305
3457
|
},
|
|
2306
3458
|
output: {
|
|
@@ -2380,6 +3532,10 @@ function defineVerifyTool(service) {
|
|
|
2380
3532
|
additionalProperties: false
|
|
2381
3533
|
},
|
|
2382
3534
|
required: true
|
|
3535
|
+
},
|
|
3536
|
+
expectations: {
|
|
3537
|
+
...EXPECTATION_RESULT_SCHEMA,
|
|
3538
|
+
required: true
|
|
2383
3539
|
}
|
|
2384
3540
|
},
|
|
2385
3541
|
additionalProperties: false
|
|
@@ -2393,6 +3549,7 @@ function defineVerifyTool(service) {
|
|
|
2393
3549
|
return service.verifyDataset({
|
|
2394
3550
|
dataset: args.path,
|
|
2395
3551
|
rules: args.rules,
|
|
3552
|
+
...args.expectations !== void 0 ? { expectations: args.expectations } : {},
|
|
2396
3553
|
workspace: workspaceOf(exec),
|
|
2397
3554
|
session: exec.agent?.session,
|
|
2398
3555
|
signal: exec.signal
|
|
@@ -2401,18 +3558,320 @@ function defineVerifyTool(service) {
|
|
|
2401
3558
|
});
|
|
2402
3559
|
}
|
|
2403
3560
|
//#endregion
|
|
2404
|
-
//#region src/
|
|
3561
|
+
//#region src/report-html.ts
|
|
3562
|
+
/** Escape text for safe embedding in an HTML document. */
|
|
3563
|
+
function escapeHtml(text) {
|
|
3564
|
+
return text.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'");
|
|
3565
|
+
}
|
|
3566
|
+
/** Render a 0..1 rate as a percentage string, or `—` when undetermined. */
|
|
3567
|
+
function pct(score) {
|
|
3568
|
+
return score === null ? "—" : `${(score * 100).toFixed(1)}%`;
|
|
3569
|
+
}
|
|
3570
|
+
/** The shared document shell: inline CSS + inline JS, no external requests. */
|
|
3571
|
+
function shell(title, body, script) {
|
|
3572
|
+
return [
|
|
3573
|
+
"<!doctype html>",
|
|
3574
|
+
"<html lang=\"en\">",
|
|
3575
|
+
"<head>",
|
|
3576
|
+
"<meta charset=\"utf-8\">",
|
|
3577
|
+
`<title>${escapeHtml(title)}</title>`,
|
|
3578
|
+
"<style>",
|
|
3579
|
+
" :root { --ink: #1a1f2e; --muted: #6b7280; --line: #e5e7eb; --accent: #0f766e; --fail: #b91c1c; --warn: #b45309; --pass: #15803d; }",
|
|
3580
|
+
" * { box-sizing: border-box; }",
|
|
3581
|
+
" body { margin: 0; padding: 24px; font: 14px/1.5 ui-sans-serif, system-ui, -apple-system, \"Segoe UI\", sans-serif; color: var(--ink); background: #f8fafc; }",
|
|
3582
|
+
" header { margin-bottom: 20px; }",
|
|
3583
|
+
" h1 { font-size: 20px; margin: 0 0 4px; }",
|
|
3584
|
+
" .meta { color: var(--muted); font-size: 12px; }",
|
|
3585
|
+
" section { background: #fff; border: 1px solid var(--line); border-radius: 8px; padding: 16px 20px; margin-bottom: 16px; }",
|
|
3586
|
+
" h2 { font-size: 15px; margin: 0 0 12px; }",
|
|
3587
|
+
" table { border-collapse: collapse; width: 100%; font-size: 13px; }",
|
|
3588
|
+
" th, td { text-align: left; padding: 6px 10px; border-top: 1px solid var(--line); vertical-align: top; }",
|
|
3589
|
+
" th { color: var(--muted); font-weight: 600; }",
|
|
3590
|
+
" .score-cell { font-variant-numeric: tabular-nums; }",
|
|
3591
|
+
" .dim-fail { color: var(--fail); font-weight: 600; }",
|
|
3592
|
+
" .dim-warn { color: var(--warn); }",
|
|
3593
|
+
" .dim-pass { color: var(--pass); }",
|
|
3594
|
+
" .dim-undetermined { color: var(--muted); }",
|
|
3595
|
+
" button { cursor: pointer; font: inherit; }",
|
|
3596
|
+
"</style>",
|
|
3597
|
+
"</head>",
|
|
3598
|
+
"<body>",
|
|
3599
|
+
body,
|
|
3600
|
+
"<script>",
|
|
3601
|
+
script,
|
|
3602
|
+
"<\/script>",
|
|
3603
|
+
"</body>",
|
|
3604
|
+
"</html>",
|
|
3605
|
+
""
|
|
3606
|
+
].join("\n");
|
|
3607
|
+
}
|
|
3608
|
+
/** The DAMA six-dimension scorecard section. */
|
|
3609
|
+
function scorecardSection(report) {
|
|
3610
|
+
const rows = report.scorecard.dimensions.map((dimension) => {
|
|
3611
|
+
const cls = dimension.score === null ? "dim-undetermined" : dimension.score >= .9 ? "dim-pass" : dimension.score >= .7 ? "dim-warn" : "dim-fail";
|
|
3612
|
+
return `<tr><td>${escapeHtml(dimension.name)}</td><td class="score-cell ${cls}">${pct(dimension.score)}</td><td>${escapeHtml(dimension.note)}</td></tr>`;
|
|
3613
|
+
}).join("\n");
|
|
3614
|
+
const overall = report.scorecard.overall;
|
|
3615
|
+
const weighted = report.scorecard.weightedOverall;
|
|
3616
|
+
return [
|
|
3617
|
+
"<section>",
|
|
3618
|
+
"<h2>DAMA six-dimension quality scorecard</h2>",
|
|
3619
|
+
"<table>",
|
|
3620
|
+
"<thead><tr><th>Dimension</th><th>Score</th><th>Note</th></tr></thead>",
|
|
3621
|
+
"<tbody>",
|
|
3622
|
+
rows,
|
|
3623
|
+
"</tbody>",
|
|
3624
|
+
"</table>",
|
|
3625
|
+
`<p class="meta" id="summary-text">overall ${pct(overall)} · weighted ${pct(weighted)}</p>`,
|
|
3626
|
+
"</section>"
|
|
3627
|
+
].join("\n");
|
|
3628
|
+
}
|
|
3629
|
+
/** The per-column profile summary table. */
|
|
3630
|
+
function columnsSection(report) {
|
|
3631
|
+
return [
|
|
3632
|
+
"<section>",
|
|
3633
|
+
"<h2>Column profile</h2>",
|
|
3634
|
+
"<table>",
|
|
3635
|
+
"<thead><tr><th>Column</th><th>Type</th><th>Missing</th><th>Missing rate</th><th>Unique</th><th>Numeric distribution</th><th>Top values</th><th>Notes</th></tr></thead>",
|
|
3636
|
+
"<tbody>",
|
|
3637
|
+
report.columns.map((column) => {
|
|
3638
|
+
const numeric = column.numeric;
|
|
3639
|
+
const numericCell = numeric === void 0 ? "—" : `min ${numeric.min} · p25 ${numeric.p25} · median ${numeric.median} · p75 ${numeric.p75} · max ${numeric.max} · mean ${numeric.mean}${numeric.outliers > 0 ? ` · ${numeric.outliers} IQR outliers` : ""}`;
|
|
3640
|
+
const top = column.topValues === void 0 ? "—" : column.topValues.map((entry) => `${escapeHtml(entry.value)} ×${entry.count}`).join(", ");
|
|
3641
|
+
const notes = column.notes.length === 0 ? "" : `<p class="meta">${column.notes.map(escapeHtml).join("; ")}</p>`;
|
|
3642
|
+
return [
|
|
3643
|
+
"<tr>",
|
|
3644
|
+
`<td>${escapeHtml(column.name)}</td>`,
|
|
3645
|
+
`<td>${escapeHtml(column.inferredType)}</td>`,
|
|
3646
|
+
`<td class="score-cell">${column.missing}</td>`,
|
|
3647
|
+
`<td class="score-cell">${(column.missingRate * 100).toFixed(1)}%</td>`,
|
|
3648
|
+
`<td class="score-cell">${column.unique}</td>`,
|
|
3649
|
+
`<td>${numericCell}</td>`,
|
|
3650
|
+
`<td>${top}</td>`,
|
|
3651
|
+
`<td>${notes}</td>`,
|
|
3652
|
+
"</tr>"
|
|
3653
|
+
].join("");
|
|
3654
|
+
}).join("\n"),
|
|
3655
|
+
"</tbody>",
|
|
3656
|
+
"</table>",
|
|
3657
|
+
"</section>"
|
|
3658
|
+
].join("\n");
|
|
3659
|
+
}
|
|
2405
3660
|
/**
|
|
2406
|
-
*
|
|
2407
|
-
*
|
|
2408
|
-
* @
|
|
3661
|
+
* Render a profile report as a self-contained offline HTML document.
|
|
3662
|
+
* @param report - the profile report (already persisted/returned by data_profile).
|
|
3663
|
+
* @returns the complete single-file HTML.
|
|
2409
3664
|
*/
|
|
2410
|
-
|
|
2411
|
-
const
|
|
3665
|
+
function renderProfileHtml(report) {
|
|
3666
|
+
const body = [
|
|
3667
|
+
"<header>",
|
|
3668
|
+
`<h1>Data profile: ${escapeHtml(report.dataset)}</h1>`,
|
|
3669
|
+
`<p class="meta">${report.rowCount} rows × ${report.columnCount} columns · generated ${new Date(report.generatedAt).toISOString()} · schema v1 · report ${report.reportKey ?? "(unpersisted)"}</p>`,
|
|
3670
|
+
"<button id=\"copy-summary\" type=\"button\">Copy summary</button>",
|
|
3671
|
+
"</header>",
|
|
3672
|
+
scorecardSection(report),
|
|
3673
|
+
columnsSection(report)
|
|
3674
|
+
].join("\n");
|
|
3675
|
+
const script = [
|
|
3676
|
+
"const button = document.getElementById('copy-summary');",
|
|
3677
|
+
"if (button) {",
|
|
3678
|
+
" button.addEventListener(\"click\", () => {",
|
|
3679
|
+
" const text = document.getElementById(\"summary-text\");",
|
|
3680
|
+
" if (text && navigator.clipboard) navigator.clipboard.writeText(text.textContent || \"\");",
|
|
3681
|
+
" });",
|
|
3682
|
+
"}"
|
|
3683
|
+
].join("\n");
|
|
3684
|
+
return shell(`Data profile: ${report.dataset}`, body, script);
|
|
3685
|
+
}
|
|
3686
|
+
/** The per-rule cleaning summary table (for clean/clean-diff reports). */
|
|
3687
|
+
function cleaningSection(report) {
|
|
3688
|
+
const rows = report.logs.map((log) => {
|
|
3689
|
+
return `<tr><td>${escapeHtml(log.rule)}</td><td class="score-cell">${log.affectedRows}</td><td>${escapeHtml(log.detail)}</td></tr>`;
|
|
3690
|
+
}).join("\n");
|
|
3691
|
+
const removed = report.inputRows - report.outputRows;
|
|
3692
|
+
return [
|
|
3693
|
+
"<section>",
|
|
3694
|
+
"<h2>Cleaning summary</h2>",
|
|
3695
|
+
`<p class="meta" id="summary-text">input ${report.inputRows} rows · output ${report.outputRows} rows · removed ${removed} rows</p>`,
|
|
3696
|
+
"<table>",
|
|
3697
|
+
"<thead><tr><th>Rule</th><th>Affected rows</th><th>Detail</th></tr></thead>",
|
|
3698
|
+
"<tbody>",
|
|
3699
|
+
rows,
|
|
3700
|
+
"</tbody>",
|
|
3701
|
+
"</table>",
|
|
3702
|
+
"</section>"
|
|
3703
|
+
].join("\n");
|
|
3704
|
+
}
|
|
3705
|
+
/**
|
|
3706
|
+
* Render a clean report as a self-contained offline HTML document (the
|
|
3707
|
+
* per-rule cleaning summary table).
|
|
3708
|
+
* @param report - the clean report (logs + input/output row counts).
|
|
3709
|
+
* @param dataset - the dataset label for the document title.
|
|
3710
|
+
* @returns the complete single-file HTML.
|
|
3711
|
+
*/
|
|
3712
|
+
function renderCleanHtml(report, dataset) {
|
|
3713
|
+
const body = [
|
|
3714
|
+
"<header>",
|
|
3715
|
+
`<h1>Cleaning report: ${escapeHtml(dataset)}</h1>`,
|
|
3716
|
+
`<p class="meta">schema v1</p>`,
|
|
3717
|
+
"</header>",
|
|
3718
|
+
cleaningSection(report)
|
|
3719
|
+
].join("\n");
|
|
3720
|
+
return shell(`Cleaning report: ${dataset}`, body, "// no interactive behavior needed; everything renders without external requests\n");
|
|
3721
|
+
}
|
|
3722
|
+
//#endregion
|
|
3723
|
+
//#region src/tools/report.ts
|
|
3724
|
+
/**
|
|
3725
|
+
* The `data_report` model tool: read persisted profile/clean-diff reports
|
|
3726
|
+
* back from the `data_quality` storage domain, by exact `reportKey` or by
|
|
3727
|
+
* `kind`. Deterministic read-only consumption — no model arithmetic.
|
|
3728
|
+
* @module dsh-data-quality/tools/report
|
|
3729
|
+
*/
|
|
3730
|
+
/** The report kinds a caller may query. */
|
|
3731
|
+
const REPORT_KINDS = [
|
|
3732
|
+
"profile",
|
|
3733
|
+
"clean",
|
|
3734
|
+
"clean-diff",
|
|
3735
|
+
"verify",
|
|
3736
|
+
"citations"
|
|
3737
|
+
];
|
|
3738
|
+
/** Render one stored report as a self-contained HTML document (profile/clean only). */
|
|
3739
|
+
function renderRecordHtml(record) {
|
|
3740
|
+
if (record.kind === "profile") return renderProfileHtml(record.report);
|
|
3741
|
+
if (record.kind === "clean" || record.kind === "clean-diff") return renderCleanHtml(record.report, record.dataset);
|
|
3742
|
+
throw new Error(`data_report html format does not support kind "${record.kind}" (profile/clean only)`);
|
|
3743
|
+
}
|
|
3744
|
+
/** Project a stored report into the canonical value (the stored report is already lossless JSON). */
|
|
3745
|
+
function toView(record) {
|
|
3746
|
+
return {
|
|
3747
|
+
key: record.key,
|
|
3748
|
+
kind: record.kind,
|
|
3749
|
+
at: record.at,
|
|
3750
|
+
dataset: record.dataset,
|
|
3751
|
+
report: record.report
|
|
3752
|
+
};
|
|
3753
|
+
}
|
|
3754
|
+
/** Human-readable report summary for the tool's Native render. */
|
|
3755
|
+
function renderReportText(value) {
|
|
3756
|
+
const lines = [];
|
|
3757
|
+
if (value.key !== void 0) lines.push(value.records.length === 0 ? `No report for ${value.key}` : `Report ${value.key}`);
|
|
3758
|
+
else lines.push(`Reports of kind ${value.kind ?? ""} (${value.records.length})`);
|
|
3759
|
+
for (const record of value.records) lines.push(`- ${record.key} [${record.kind}] ${record.dataset} @ ${new Date(record.at).toISOString()}`);
|
|
3760
|
+
return lines.join("\n");
|
|
3761
|
+
}
|
|
3762
|
+
/**
|
|
3763
|
+
* Build the `data_report` tool definition against a mounted service.
|
|
3764
|
+
* @param service - the mounted ctx.dataQuality implementation.
|
|
3765
|
+
* @returns the tool definition to register.
|
|
3766
|
+
*/
|
|
3767
|
+
function defineReportTool(service) {
|
|
3768
|
+
return defineTool({
|
|
3769
|
+
name: "data_report",
|
|
3770
|
+
description: [
|
|
3771
|
+
"Read persisted data-quality reports back from the data_quality storage domain (deterministic, read-only).",
|
|
3772
|
+
"Pass key (the exact reportKey a prior run returned) to fetch one report, or kind to list every persisted report of that kind, ordered chronologically. Exactly one of key/kind.",
|
|
3773
|
+
"Returns the report envelope(s): kind, dataset, timestamp, and the full stored report (profile/clean/clean-diff/verify/citations). Missing keys and unknown kinds fail loudly."
|
|
3774
|
+
].join("\n"),
|
|
3775
|
+
parameters: {
|
|
3776
|
+
key: {
|
|
3777
|
+
type: "string",
|
|
3778
|
+
description: "Exact storage reportKey (e.g. 20260819000000000-profile-1a2b3c4d); fetches that one report."
|
|
3779
|
+
},
|
|
3780
|
+
kind: {
|
|
3781
|
+
type: "string",
|
|
3782
|
+
enum: [...REPORT_KINDS],
|
|
3783
|
+
description: "Report kind to list (profile/clean/clean-diff/verify/citations)."
|
|
3784
|
+
},
|
|
3785
|
+
format: {
|
|
3786
|
+
type: "string",
|
|
3787
|
+
enum: ["json", "html"],
|
|
3788
|
+
description: "Output format. json (default) returns the report envelope(s); html renders one report as a self-contained offline HTML document (requires key; profile/clean only)."
|
|
3789
|
+
}
|
|
3790
|
+
},
|
|
3791
|
+
output: {
|
|
3792
|
+
schema: {
|
|
3793
|
+
type: "object",
|
|
3794
|
+
properties: {
|
|
3795
|
+
key: { type: "string" },
|
|
3796
|
+
kind: {
|
|
3797
|
+
type: "string",
|
|
3798
|
+
enum: [...REPORT_KINDS]
|
|
3799
|
+
},
|
|
3800
|
+
records: {
|
|
3801
|
+
type: "array",
|
|
3802
|
+
items: {
|
|
3803
|
+
type: "object",
|
|
3804
|
+
properties: {
|
|
3805
|
+
key: {
|
|
3806
|
+
type: "string",
|
|
3807
|
+
required: true
|
|
3808
|
+
},
|
|
3809
|
+
kind: {
|
|
3810
|
+
type: "string",
|
|
3811
|
+
enum: [...REPORT_KINDS],
|
|
3812
|
+
required: true
|
|
3813
|
+
},
|
|
3814
|
+
at: {
|
|
3815
|
+
type: "number",
|
|
3816
|
+
required: true
|
|
3817
|
+
},
|
|
3818
|
+
dataset: {
|
|
3819
|
+
type: "string",
|
|
3820
|
+
required: true
|
|
3821
|
+
},
|
|
3822
|
+
report: {
|
|
3823
|
+
type: "json",
|
|
3824
|
+
required: true
|
|
3825
|
+
}
|
|
3826
|
+
},
|
|
3827
|
+
additionalProperties: false
|
|
3828
|
+
},
|
|
3829
|
+
required: true
|
|
3830
|
+
},
|
|
3831
|
+
html: {
|
|
3832
|
+
type: "string",
|
|
3833
|
+
description: "Self-contained offline HTML (present only when format: html)."
|
|
3834
|
+
}
|
|
3835
|
+
},
|
|
3836
|
+
additionalProperties: false
|
|
3837
|
+
},
|
|
3838
|
+
render: (_args, value) => {
|
|
3839
|
+
const view = value;
|
|
3840
|
+
if (view.html !== void 0) return [{
|
|
3841
|
+
type: "text",
|
|
3842
|
+
text: view.html
|
|
3843
|
+
}];
|
|
3844
|
+
return [{
|
|
3845
|
+
type: "text",
|
|
3846
|
+
text: renderReportText(view)
|
|
3847
|
+
}];
|
|
3848
|
+
}
|
|
3849
|
+
},
|
|
3850
|
+
async execute(args, _exec) {
|
|
3851
|
+
const hasKey = args.key !== void 0;
|
|
3852
|
+
if (hasKey === (args.kind !== void 0)) throw new Error("data_report needs exactly one of key/kind");
|
|
3853
|
+
const format = args.format ?? "json";
|
|
3854
|
+
if (hasKey) {
|
|
3855
|
+
const record = await service.getReport(args.key);
|
|
3856
|
+
return {
|
|
3857
|
+
key: args.key,
|
|
3858
|
+
records: [toView(record)],
|
|
3859
|
+
...format === "html" ? { html: renderRecordHtml(record) } : {}
|
|
3860
|
+
};
|
|
3861
|
+
}
|
|
3862
|
+
if (format === "html") throw new Error("data_report html format requires key (exactly one report)");
|
|
3863
|
+
const records = await service.listReports(args.kind);
|
|
3864
|
+
return {
|
|
3865
|
+
kind: args.kind,
|
|
3866
|
+
records: records.map(toView)
|
|
3867
|
+
};
|
|
3868
|
+
}
|
|
3869
|
+
});
|
|
3870
|
+
}
|
|
2412
3871
|
//#endregion
|
|
2413
3872
|
//#region src/index.ts
|
|
2414
3873
|
const name = "data-quality";
|
|
2415
|
-
/** The
|
|
3874
|
+
/** The four model tools and the durable report domain. */
|
|
2416
3875
|
const inject = ["tools", "storageDomain"];
|
|
2417
3876
|
/**
|
|
2418
3877
|
* Mount the seam: resolve config (fail loud), open the report domain, publish
|
|
@@ -2439,7 +3898,11 @@ async function apply(ctx, config = {}) {
|
|
|
2439
3898
|
await reports.put(key, record);
|
|
2440
3899
|
return key;
|
|
2441
3900
|
},
|
|
2442
|
-
get: (key) => reports.get(key)
|
|
3901
|
+
get: (key) => reports.get(key),
|
|
3902
|
+
list: (kind) => [...reports.entries()].filter(([, record]) => record.kind === kind).sort(([keyA], [keyB]) => keyA < keyB ? -1 : keyA > keyB ? 1 : 0).map(([key, record]) => ({
|
|
3903
|
+
key,
|
|
3904
|
+
...record
|
|
3905
|
+
}))
|
|
2443
3906
|
};
|
|
2444
3907
|
}
|
|
2445
3908
|
const service = new LocalDataQualityService(ctx, resolved, {
|
|
@@ -2449,7 +3912,8 @@ async function apply(ctx, config = {}) {
|
|
|
2449
3912
|
ctx.tools.register(defineProfileTool(service));
|
|
2450
3913
|
ctx.tools.register(defineCleanTool(service));
|
|
2451
3914
|
ctx.tools.register(defineVerifyTool(service));
|
|
2452
|
-
|
|
3915
|
+
ctx.tools.register(defineReportTool(service));
|
|
3916
|
+
logger.info(`dsh-data-quality ${VERSION} mounted: ctx.dataQuality + data_profile/data_clean/data_verify/data_report`);
|
|
2453
3917
|
if (domain !== void 0) {
|
|
2454
3918
|
const handle = domain;
|
|
2455
3919
|
ctx.effect(() => async () => {
|
|
@@ -2458,4 +3922,4 @@ async function apply(ctx, config = {}) {
|
|
|
2458
3922
|
}
|
|
2459
3923
|
}
|
|
2460
3924
|
//#endregion
|
|
2461
|
-
export { Config, DATA_QUALITY_EVENT_TYPES, DataQualityService, DatasetError, LocalDataQualityService, MAX_CELL_TEXT, VERSION, appendDataQualityEvent, apply, applyCleanRules, checkCitations, dataQualityDomainSpec, inject, isMissing, loadDocument, loadTable, name, parseBoolean, parseDate, parseDelimited, parseJsonTable, parseLocator, parseNumeric, profileTable, renderProfileText, renderVerifyText, reportKeyOf, reportRecordSchema, resolveConfig, resolveWorkspacePath, sampleRows, serializeDelimited, truncateCell, truncateRow, verifyTable };
|
|
3925
|
+
export { Config, DATA_QUALITY_EVENT_TYPES, DataQualityService, DatasetError, INDUSTRY_PRESETS, INDUSTRY_PRESET_IDS, LocalDataQualityService, MAX_CELL_TEXT, REPORT_SCHEMA_VERSION, VERSION, VerifyExpectationError, appendDataQualityEvent, apply, applyCleanRules, checkCitations, computeCleanContract, computeCleanProfileDiff, computeScorecard, countDuplicateRows, dataQualityDomainSpec, dateFormatOf, detectDuplicateRows, detectEncoding, inject, isMissing, isValidReportKey, loadDocument, loadTable, name, numericProfile, parseBoolean, parseDate, parseDateCell, parseDelimited, parseJsonTable, parseLocator, parseNumeric, profileTable, readDatasetFile, renderCleanHtml, renderProfileHtml, renderProfileText, renderVerifyText, reportKeyOf, reportRecordSchema, resolveConfig, resolveIndustryPreset, resolveWorkspacePath, sampleRows, serializeDelimited, truncateCell, truncateRow, verifyExpectations, verifyTable };
|