dsh-data-quality 0.1.3 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -0
- package/README.es.md +7 -1
- package/README.hi.md +7 -1
- package/README.md +22 -9
- package/README.pt.md +7 -1
- package/README.zh.md +7 -1
- package/cordis.patch.yml +11 -0
- package/lib/index.js +1405 -140
- package/lib/types/config.d.ts +12 -0
- package/lib/types/config.d.ts.map +1 -1
- package/lib/types/config.js +35 -0
- package/lib/types/config.js.map +1 -1
- package/lib/types/contract.d.ts +83 -0
- package/lib/types/contract.d.ts.map +1 -0
- package/lib/types/contract.js +144 -0
- package/lib/types/contract.js.map +1 -0
- package/lib/types/dataset.d.ts +50 -3
- package/lib/types/dataset.d.ts.map +1 -1
- package/lib/types/dataset.js +76 -21
- package/lib/types/dataset.js.map +1 -1
- package/lib/types/events.d.ts +2 -0
- package/lib/types/events.d.ts.map +1 -1
- package/lib/types/events.js.map +1 -1
- package/lib/types/index.d.ts +11 -8
- package/lib/types/index.d.ts.map +1 -1
- package/lib/types/index.js +15 -6
- package/lib/types/index.js.map +1 -1
- package/lib/types/presets.d.ts +28 -0
- package/lib/types/presets.d.ts.map +1 -0
- package/lib/types/presets.js +134 -0
- package/lib/types/presets.js.map +1 -0
- package/lib/types/profile.d.ts +38 -1
- package/lib/types/profile.d.ts.map +1 -1
- package/lib/types/profile.js +64 -10
- package/lib/types/profile.js.map +1 -1
- package/lib/types/provider-local.d.ts +7 -1
- package/lib/types/provider-local.d.ts.map +1 -1
- package/lib/types/provider-local.js +72 -2
- package/lib/types/provider-local.js.map +1 -1
- package/lib/types/scorecard.d.ts +50 -0
- package/lib/types/scorecard.d.ts.map +1 -0
- package/lib/types/scorecard.js +186 -0
- package/lib/types/scorecard.js.map +1 -0
- package/lib/types/service.d.ts +32 -2
- package/lib/types/service.d.ts.map +1 -1
- package/lib/types/service.js.map +1 -1
- package/lib/types/store.d.ts +21 -1
- package/lib/types/store.d.ts.map +1 -1
- package/lib/types/store.js +13 -1
- package/lib/types/store.js.map +1 -1
- package/lib/types/tools/clean.d.ts.map +1 -1
- package/lib/types/tools/clean.js +76 -1
- package/lib/types/tools/clean.js.map +1 -1
- package/lib/types/tools/profile-report-schema.d.ts +315 -0
- package/lib/types/tools/profile-report-schema.d.ts.map +1 -0
- package/lib/types/tools/profile-report-schema.js +95 -0
- package/lib/types/tools/profile-report-schema.js.map +1 -0
- package/lib/types/tools/profile.d.ts.map +1 -1
- package/lib/types/tools/profile.js +6 -53
- package/lib/types/tools/profile.js.map +1 -1
- package/lib/types/tools/report.d.ts +14 -0
- package/lib/types/tools/report.d.ts.map +1 -0
- package/lib/types/tools/report.js +86 -0
- package/lib/types/tools/report.js.map +1 -0
- package/lib/types/tools/shared.d.ts.map +1 -1
- package/lib/types/tools/shared.js +22 -2
- package/lib/types/tools/shared.js.map +1 -1
- package/lib/types/tools/verify.d.ts.map +1 -1
- package/lib/types/tools/verify.js +36 -1
- package/lib/types/tools/verify.js.map +1 -1
- package/lib/types/verify.d.ts +49 -3
- package/lib/types/verify.d.ts.map +1 -1
- package/lib/types/verify.js +111 -4
- package/lib/types/verify.js.map +1 -1
- package/lib/types/version.d.ts +1 -1
- package/lib/types/version.js +1 -1
- package/package.json +1 -1
- package/src/config.ts +52 -0
- package/src/contract.ts +190 -0
- package/src/dataset.ts +101 -21
- package/src/events.ts +2 -0
- package/src/index.ts +25 -8
- package/src/presets.ts +146 -0
- package/src/profile.ts +101 -11
- package/src/provider-local.ts +77 -3
- package/src/scorecard.ts +244 -0
- package/src/service.ts +34 -2
- package/src/store.ts +28 -2
- package/src/tools/clean.ts +76 -1
- package/src/tools/profile-report-schema.ts +97 -0
- package/src/tools/profile.ts +8 -56
- package/src/tools/report.ts +108 -0
- package/src/tools/shared.ts +21 -2
- package/src/tools/verify.ts +40 -2
- package/src/verify.ts +151 -5
- package/src/version.ts +1 -1
package/lib/index.js
CHANGED
|
@@ -4,6 +4,7 @@ import { defineDomain, domainTable } from "@deepseek-ai/dsh-storage-domain";
|
|
|
4
4
|
import { mkdir, open, stat, writeFile } from "node:fs/promises";
|
|
5
5
|
import path from "node:path";
|
|
6
6
|
import { createHash } from "node:crypto";
|
|
7
|
+
import { TextDecoder } from "node:util";
|
|
7
8
|
import { KNOWN_SESSION_EVENT_TYPES } from "@deepseek-ai/dsh-session";
|
|
8
9
|
import { Service } from "@deepseek-ai/cordis";
|
|
9
10
|
import { defineTool } from "@deepseek-ai/dsh-tools";
|
|
@@ -14,6 +15,15 @@ import { defineTool } from "@deepseek-ai/dsh-tools";
|
|
|
14
15
|
* step validates bounds so misconfiguration fails loud at mount.
|
|
15
16
|
* @module dsh-data-quality/config
|
|
16
17
|
*/
|
|
18
|
+
/** The six scorecard dimension ids, in report order (kept in sync with `scorecard.ts`). */
|
|
19
|
+
const SCORECARD_DIMENSIONS = [
|
|
20
|
+
"completeness",
|
|
21
|
+
"uniqueness",
|
|
22
|
+
"validity",
|
|
23
|
+
"consistency",
|
|
24
|
+
"timeliness",
|
|
25
|
+
"accuracy"
|
|
26
|
+
];
|
|
17
27
|
/** Schemastery schema: the loader validates and fills defaults before `apply`. */
|
|
18
28
|
const Config = z.object({
|
|
19
29
|
enabled: z.boolean().default(true),
|
|
@@ -28,7 +38,22 @@ const Config = z.object({
|
|
|
28
38
|
".jsonl"
|
|
29
39
|
]),
|
|
30
40
|
workspaceRoot: z.string().default(""),
|
|
31
|
-
storeReports: z.boolean().default(true)
|
|
41
|
+
storeReports: z.boolean().default(true),
|
|
42
|
+
scorecardWeights: z.object({
|
|
43
|
+
completeness: z.number().default(1),
|
|
44
|
+
uniqueness: z.number().default(1),
|
|
45
|
+
validity: z.number().default(1),
|
|
46
|
+
consistency: z.number().default(1),
|
|
47
|
+
timeliness: z.number().default(1),
|
|
48
|
+
accuracy: z.number().default(1)
|
|
49
|
+
}).default({
|
|
50
|
+
completeness: 1,
|
|
51
|
+
uniqueness: 1,
|
|
52
|
+
validity: 1,
|
|
53
|
+
consistency: 1,
|
|
54
|
+
timeliness: 1,
|
|
55
|
+
accuracy: 1
|
|
56
|
+
})
|
|
32
57
|
});
|
|
33
58
|
/** Throw unless `value` is a positive safe integer. */
|
|
34
59
|
function assertPositiveInt(name, value) {
|
|
@@ -63,6 +88,7 @@ function resolveConfig(config = {}) {
|
|
|
63
88
|
return ext;
|
|
64
89
|
});
|
|
65
90
|
if (allowedExtensions.length === 0) throw new TypeError("allowedExtensions must not be empty");
|
|
91
|
+
const scorecardWeights = resolveScorecardWeights(config.scorecardWeights);
|
|
66
92
|
return {
|
|
67
93
|
enabled: config.enabled ?? true,
|
|
68
94
|
maxRows,
|
|
@@ -71,9 +97,20 @@ function resolveConfig(config = {}) {
|
|
|
71
97
|
evidenceRowLimit,
|
|
72
98
|
allowedExtensions,
|
|
73
99
|
workspaceRoot: config.workspaceRoot ?? "",
|
|
74
|
-
storeReports: config.storeReports ?? true
|
|
100
|
+
storeReports: config.storeReports ?? true,
|
|
101
|
+
scorecardWeights
|
|
75
102
|
};
|
|
76
103
|
}
|
|
104
|
+
/** Resolve scorecard weights, filling defaults and rejecting non-negative violations loudly. */
|
|
105
|
+
function resolveScorecardWeights(weights = {}) {
|
|
106
|
+
const out = {};
|
|
107
|
+
for (const dimension of SCORECARD_DIMENSIONS) {
|
|
108
|
+
const value = weights[dimension] ?? 1;
|
|
109
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value < 0) throw new TypeError(`scorecardWeights.${dimension} must be a non-negative finite number, got ${String(value)}`);
|
|
110
|
+
out[dimension] = value;
|
|
111
|
+
}
|
|
112
|
+
return out;
|
|
113
|
+
}
|
|
77
114
|
//#endregion
|
|
78
115
|
//#region src/dataset.ts
|
|
79
116
|
/**
|
|
@@ -127,13 +164,40 @@ function throwIfAborted(signal) {
|
|
|
127
164
|
if (signal?.aborted === true) throw signal.reason instanceof Error ? signal.reason : /* @__PURE__ */ new Error("operation aborted");
|
|
128
165
|
}
|
|
129
166
|
/**
|
|
130
|
-
*
|
|
167
|
+
* Detect a UTF-8 byte-order mark and validate the byte sequence. Invalid
|
|
168
|
+
* UTF-8 is a data-quality finding the profile reports (`validUtf8: false`)
|
|
169
|
+
* rather than a structural error that blocks the read — the decoded text
|
|
170
|
+
* keeps U+FFFD replacement characters so the profile can still run.
|
|
171
|
+
* @param buffer - raw file bytes.
|
|
172
|
+
* @returns the encoding metadata.
|
|
173
|
+
*/
|
|
174
|
+
function detectEncoding(buffer) {
|
|
175
|
+
const bom = buffer.length >= 3 && buffer[0] === 239 && buffer[1] === 187 && buffer[2] === 191 ? "utf-8" : null;
|
|
176
|
+
const body = bom === "utf-8" ? buffer.subarray(3) : buffer;
|
|
177
|
+
let validUtf8 = true;
|
|
178
|
+
try {
|
|
179
|
+
new TextDecoder("utf-8", { fatal: true }).decode(body);
|
|
180
|
+
} catch {
|
|
181
|
+
validUtf8 = false;
|
|
182
|
+
}
|
|
183
|
+
return {
|
|
184
|
+
bom,
|
|
185
|
+
validUtf8
|
|
186
|
+
};
|
|
187
|
+
}
|
|
188
|
+
/** Decode file bytes, stripping a UTF-8 BOM and preserving replacement characters for invalid bytes. */
|
|
189
|
+
function decodeUtf8Text(buffer, encoding) {
|
|
190
|
+
const body = encoding.bom === "utf-8" ? buffer.subarray(3) : buffer;
|
|
191
|
+
return new TextDecoder("utf-8").decode(body);
|
|
192
|
+
}
|
|
193
|
+
/**
|
|
194
|
+
* Read a dataset file under the size cap, detecting its encoding.
|
|
131
195
|
* @param absolutePath - normalized absolute path (from {@link resolveWorkspacePath}).
|
|
132
196
|
* @param config - resolved config (size cap).
|
|
133
197
|
* @param signal - optional abort signal honored around the read.
|
|
134
|
-
* @returns the UTF-8 text.
|
|
198
|
+
* @returns the decoded UTF-8 text plus its encoding metadata.
|
|
135
199
|
*/
|
|
136
|
-
async function
|
|
200
|
+
async function readDatasetFile(absolutePath, config, signal) {
|
|
137
201
|
throwIfAborted(signal);
|
|
138
202
|
let info;
|
|
139
203
|
try {
|
|
@@ -147,7 +211,12 @@ async function readDatasetText(absolutePath, config, signal) {
|
|
|
147
211
|
const handle = await open(absolutePath, "r");
|
|
148
212
|
try {
|
|
149
213
|
throwIfAborted(signal);
|
|
150
|
-
|
|
214
|
+
const buffer = await handle.readFile();
|
|
215
|
+
const encoding = detectEncoding(buffer);
|
|
216
|
+
return {
|
|
217
|
+
text: decodeUtf8Text(buffer, encoding),
|
|
218
|
+
encoding
|
|
219
|
+
};
|
|
151
220
|
} finally {
|
|
152
221
|
await handle.close();
|
|
153
222
|
}
|
|
@@ -316,12 +385,21 @@ function parseJsonLines(text) {
|
|
|
316
385
|
* @returns the parsed table.
|
|
317
386
|
*/
|
|
318
387
|
async function loadTable(absolutePath, config, signal) {
|
|
319
|
-
const text = await
|
|
388
|
+
const { text, encoding } = await readDatasetFile(absolutePath, config, signal);
|
|
320
389
|
const ext = path.extname(absolutePath).toLowerCase();
|
|
321
390
|
throwIfAborted(signal);
|
|
322
|
-
if (ext === ".csv") return
|
|
323
|
-
|
|
324
|
-
|
|
391
|
+
if (ext === ".csv") return {
|
|
392
|
+
...parseDelimited(text, ",", config, signal),
|
|
393
|
+
encoding
|
|
394
|
+
};
|
|
395
|
+
if (ext === ".tsv") return {
|
|
396
|
+
...parseDelimited(text, " ", config, signal),
|
|
397
|
+
encoding
|
|
398
|
+
};
|
|
399
|
+
return {
|
|
400
|
+
...parseJsonTable(text, ext, config, signal),
|
|
401
|
+
encoding
|
|
402
|
+
};
|
|
325
403
|
}
|
|
326
404
|
/**
|
|
327
405
|
* Load a dataset as a citation-checkable document root: CSV/TSV become
|
|
@@ -334,14 +412,20 @@ async function loadTable(absolutePath, config, signal) {
|
|
|
334
412
|
*/
|
|
335
413
|
async function loadDocument(absolutePath, config, signal) {
|
|
336
414
|
const ext = path.extname(absolutePath).toLowerCase();
|
|
337
|
-
if (ext === ".json")
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
415
|
+
if (ext === ".json") {
|
|
416
|
+
const { text } = await readDatasetFile(absolutePath, config, signal);
|
|
417
|
+
return {
|
|
418
|
+
kind: "json",
|
|
419
|
+
value: parseJsonDocument(text)
|
|
420
|
+
};
|
|
421
|
+
}
|
|
422
|
+
if (ext === ".jsonl") {
|
|
423
|
+
const { text } = await readDatasetFile(absolutePath, config, signal);
|
|
424
|
+
return {
|
|
425
|
+
kind: "json",
|
|
426
|
+
value: parseJsonLines(text)
|
|
427
|
+
};
|
|
428
|
+
}
|
|
345
429
|
const table = await loadTable(absolutePath, config, signal);
|
|
346
430
|
return {
|
|
347
431
|
kind: "table",
|
|
@@ -390,18 +474,17 @@ const DATE_PATTERNS = [
|
|
|
390
474
|
/^(\d{4})\/(\d{1,2})\/(\d{1,2})$/u,
|
|
391
475
|
/^(\d{4})-(\d{1,2})-(\d{1,2})[ T](\d{1,2}):(\d{2})(?::(\d{2}))?(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?$/u
|
|
392
476
|
];
|
|
393
|
-
/**
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
function parseDate(cell) {
|
|
477
|
+
/** Deterministic date-format labels in {@link DATE_PATTERNS} order. */
|
|
478
|
+
const DATE_FORMATS = [
|
|
479
|
+
"iso-date",
|
|
480
|
+
"slash-date",
|
|
481
|
+
"datetime"
|
|
482
|
+
];
|
|
483
|
+
/** Parse one date cell to its epoch plus format; `undefined` when unrecognized. */
|
|
484
|
+
function parseDateCellInternal(cell) {
|
|
402
485
|
if (typeof cell !== "string") return void 0;
|
|
403
486
|
const text = cell.trim();
|
|
404
|
-
for (const pattern of DATE_PATTERNS) {
|
|
487
|
+
for (const [index, pattern] of DATE_PATTERNS.entries()) {
|
|
405
488
|
const match = pattern.exec(text);
|
|
406
489
|
if (match === null) continue;
|
|
407
490
|
const year = Number(match[1]);
|
|
@@ -414,9 +497,43 @@ function parseDate(cell) {
|
|
|
414
497
|
const epoch = Date.UTC(year, month - 1, day, hour, minute, second);
|
|
415
498
|
const check = new Date(epoch);
|
|
416
499
|
if (check.getUTCMonth() !== month - 1 || check.getUTCDate() !== day) return void 0;
|
|
417
|
-
|
|
500
|
+
const format = DATE_FORMATS[index];
|
|
501
|
+
if (format === void 0) return void 0;
|
|
502
|
+
return {
|
|
503
|
+
epoch,
|
|
504
|
+
format
|
|
505
|
+
};
|
|
418
506
|
}
|
|
419
507
|
}
|
|
508
|
+
/**
|
|
509
|
+
* Deterministic date parse to epoch milliseconds. Accepts `YYYY-MM-DD`,
|
|
510
|
+
* `YYYY/MM/DD`, and ISO-like datetimes (date-only forms read as UTC midnight).
|
|
511
|
+
* Calendar-invalid dates (e.g. 2025-13-40) reject. Returns `undefined` when
|
|
512
|
+
* the cell is not a recognized date.
|
|
513
|
+
* @param cell - the cell to parse (`undefined` when the column is absent).
|
|
514
|
+
* @returns epoch milliseconds, or `undefined`.
|
|
515
|
+
*/
|
|
516
|
+
function parseDate(cell) {
|
|
517
|
+
return parseDateCellInternal(cell)?.epoch;
|
|
518
|
+
}
|
|
519
|
+
/**
|
|
520
|
+
* Parse one date cell to its epoch plus source format label.
|
|
521
|
+
* @param cell - the cell to parse (`undefined` when the column is absent).
|
|
522
|
+
* @returns the parsed date, or `undefined` when the cell is not a recognized date.
|
|
523
|
+
*/
|
|
524
|
+
function parseDateCell(cell) {
|
|
525
|
+
return parseDateCellInternal(cell);
|
|
526
|
+
}
|
|
527
|
+
/**
|
|
528
|
+
* The format label of a recognized date cell (`iso-date` / `slash-date` /
|
|
529
|
+
* `datetime`); `undefined` when the cell is not a recognized date. Used to
|
|
530
|
+
* measure a date column's format consistency.
|
|
531
|
+
* @param cell - the cell to inspect (`undefined` when the column is absent).
|
|
532
|
+
* @returns the source format label, or `undefined`.
|
|
533
|
+
*/
|
|
534
|
+
function dateFormatOf(cell) {
|
|
535
|
+
return parseDateCellInternal(cell)?.format;
|
|
536
|
+
}
|
|
420
537
|
/** Boolean parse: true/false/yes/no/1/0, case-insensitive. */
|
|
421
538
|
function parseBoolean(cell) {
|
|
422
539
|
if (typeof cell === "boolean") return cell;
|
|
@@ -440,7 +557,8 @@ const reportRecordSchema = z$1.object({
|
|
|
440
557
|
"profile",
|
|
441
558
|
"clean",
|
|
442
559
|
"verify",
|
|
443
|
-
"citations"
|
|
560
|
+
"citations",
|
|
561
|
+
"clean-diff"
|
|
444
562
|
]),
|
|
445
563
|
at: z$1.number().int().nonnegative(),
|
|
446
564
|
dataset: z$1.string(),
|
|
@@ -452,6 +570,18 @@ const dataQualityDomainSpec = defineDomain({
|
|
|
452
570
|
version: 1,
|
|
453
571
|
tables: { reports: domainTable(reportRecordSchema) }
|
|
454
572
|
});
|
|
573
|
+
/** Well-formed report-key shape: `<17-digit timestamp>-<kind>-<8-hex fingerprint>`. */
|
|
574
|
+
const REPORT_KEY_PATTERN = /^\d{17}-(?:profile|clean|verify|citations|clean-diff)-[0-9a-f]{8}$/u;
|
|
575
|
+
/**
|
|
576
|
+
* Whether `key` is a well-formed, path-safe storage report key. Rejects any
|
|
577
|
+
* key with separators, traversal, or unexpected characters before it can be
|
|
578
|
+
* handed to the storage backend.
|
|
579
|
+
* @param key - candidate report key.
|
|
580
|
+
* @returns whether the key matches the deterministic report-key format.
|
|
581
|
+
*/
|
|
582
|
+
function isValidReportKey(key) {
|
|
583
|
+
return REPORT_KEY_PATTERN.test(key);
|
|
584
|
+
}
|
|
455
585
|
/** Pad to two digits for the key timestamp. */
|
|
456
586
|
function pad2(value) {
|
|
457
587
|
return String(value).padStart(2, "0");
|
|
@@ -842,6 +972,199 @@ function serializeDelimited(columns, rows, delimiter) {
|
|
|
842
972
|
return `${lines.join("\n")}\n`;
|
|
843
973
|
}
|
|
844
974
|
//#endregion
|
|
975
|
+
//#region src/scorecard.ts
|
|
976
|
+
/**
|
|
977
|
+
* DAMA-style six-dimension quality scorecard over a parsed {@link Table}.
|
|
978
|
+
* Pure and deterministic: every rate derives from one full-table pass, the
|
|
979
|
+
* only clock is the injected `now`, and no dimension fabricates a score it
|
|
980
|
+
* cannot defend — `accuracy` stays `null` (undetermined) without a declared
|
|
981
|
+
* schema and `timeliness` stays `null` without date cells.
|
|
982
|
+
* @module dsh-data-quality/scorecard
|
|
983
|
+
*/
|
|
984
|
+
/** Round to 6 significant digits (mirrors the profile report's rounding). */
|
|
985
|
+
function round6$1(value) {
|
|
986
|
+
return Number(value.toPrecision(6));
|
|
987
|
+
}
|
|
988
|
+
/** Increment a format-label counter. */
|
|
989
|
+
function bump(formats, tag) {
|
|
990
|
+
formats.set(tag, (formats.get(tag) ?? 0) + 1);
|
|
991
|
+
}
|
|
992
|
+
/** Inferred column type from the full-table class counts (mirrors `profileColumn`). */
|
|
993
|
+
function inferType(stats) {
|
|
994
|
+
if (stats.present === 0) return "empty";
|
|
995
|
+
if (stats.number === stats.present) return "number";
|
|
996
|
+
if (stats.date === stats.present) return "date";
|
|
997
|
+
if (stats.boolean === stats.present) return "boolean";
|
|
998
|
+
if (stats.string === stats.present) return "string";
|
|
999
|
+
return "mixed";
|
|
1000
|
+
}
|
|
1001
|
+
/** Format the `value / total` ratio as a plain `note` fragment. */
|
|
1002
|
+
function ratioNote(numerator, denominator) {
|
|
1003
|
+
return `${numerator}/${denominator}`;
|
|
1004
|
+
}
|
|
1005
|
+
/** Build one determinable dimension with its ratio note. */
|
|
1006
|
+
function dimension(name, score, numerator, denominator) {
|
|
1007
|
+
return {
|
|
1008
|
+
name,
|
|
1009
|
+
score,
|
|
1010
|
+
note: ratioNote(numerator, denominator)
|
|
1011
|
+
};
|
|
1012
|
+
}
|
|
1013
|
+
/**
|
|
1014
|
+
* Compute the six-dimension scorecard over a full table:
|
|
1015
|
+
* - `completeness` — non-empty cell rate.
|
|
1016
|
+
* - `uniqueness` — unique full-row content rate (`1 - duplicateRows / rowCount`).
|
|
1017
|
+
* - `validity` — present cells conforming to their column's inferred type.
|
|
1018
|
+
* - `consistency` — present cells in their column's dominant format (a date
|
|
1019
|
+
* column's unified `YYYY-MM-DD` vs `YYYY/MM/DD` vs datetime ratio).
|
|
1020
|
+
* - `timeliness` — date cells not future-dated relative to the injected `now`.
|
|
1021
|
+
* - `accuracy` — declared-schema agreement (limited definition); `null`
|
|
1022
|
+
* (undetermined) without a declared schema — never fabricated.
|
|
1023
|
+
* @param table - the parsed dataset (full table, not the sampled cards).
|
|
1024
|
+
* @param options - injected clock, duplicate-row count, optional declared schema, optional weights, abort signal.
|
|
1025
|
+
* @returns the scorecard.
|
|
1026
|
+
*/
|
|
1027
|
+
function computeScorecard(table, options) {
|
|
1028
|
+
throwIfAborted(options.signal);
|
|
1029
|
+
const stats = /* @__PURE__ */ new Map();
|
|
1030
|
+
for (const column of table.columns) stats.set(column, {
|
|
1031
|
+
present: 0,
|
|
1032
|
+
number: 0,
|
|
1033
|
+
date: 0,
|
|
1034
|
+
boolean: 0,
|
|
1035
|
+
string: 0,
|
|
1036
|
+
formats: /* @__PURE__ */ new Map()
|
|
1037
|
+
});
|
|
1038
|
+
let missingCells = 0;
|
|
1039
|
+
let dateCells = 0;
|
|
1040
|
+
let futureDateCells = 0;
|
|
1041
|
+
for (const [index, row] of table.rows.entries()) {
|
|
1042
|
+
if (index % 1024 === 0) throwIfAborted(options.signal);
|
|
1043
|
+
for (const column of table.columns) {
|
|
1044
|
+
const cell = row[column];
|
|
1045
|
+
const columnStats = stats.get(column);
|
|
1046
|
+
if (isMissing(cell)) {
|
|
1047
|
+
missingCells += 1;
|
|
1048
|
+
continue;
|
|
1049
|
+
}
|
|
1050
|
+
columnStats.present += 1;
|
|
1051
|
+
if (parseNumeric(cell) !== void 0) {
|
|
1052
|
+
columnStats.number += 1;
|
|
1053
|
+
bump(columnStats.formats, "number");
|
|
1054
|
+
continue;
|
|
1055
|
+
}
|
|
1056
|
+
const date = parseDateCell(cell);
|
|
1057
|
+
if (date !== void 0) {
|
|
1058
|
+
columnStats.date += 1;
|
|
1059
|
+
bump(columnStats.formats, date.format);
|
|
1060
|
+
dateCells += 1;
|
|
1061
|
+
if (date.epoch > options.now) futureDateCells += 1;
|
|
1062
|
+
continue;
|
|
1063
|
+
}
|
|
1064
|
+
if (parseBoolean(cell) !== void 0) {
|
|
1065
|
+
columnStats.boolean += 1;
|
|
1066
|
+
bump(columnStats.formats, "boolean");
|
|
1067
|
+
continue;
|
|
1068
|
+
}
|
|
1069
|
+
columnStats.string += 1;
|
|
1070
|
+
bump(columnStats.formats, typeof cell === "string" ? "string" : "json");
|
|
1071
|
+
}
|
|
1072
|
+
}
|
|
1073
|
+
const totalCells = table.rows.length * table.columns.length;
|
|
1074
|
+
let presentCells = 0;
|
|
1075
|
+
let invalidCells = 0;
|
|
1076
|
+
let consistentCells = 0;
|
|
1077
|
+
for (const [, columnStats] of stats) {
|
|
1078
|
+
presentCells += columnStats.present;
|
|
1079
|
+
const dominantClass = Math.max(columnStats.number, columnStats.date, columnStats.boolean, columnStats.string);
|
|
1080
|
+
invalidCells += columnStats.present - dominantClass;
|
|
1081
|
+
if (columnStats.present === 0) continue;
|
|
1082
|
+
let dominant = 0;
|
|
1083
|
+
for (const count of columnStats.formats.values()) if (count > dominant) dominant = count;
|
|
1084
|
+
consistentCells += dominant;
|
|
1085
|
+
}
|
|
1086
|
+
const dimensions = [
|
|
1087
|
+
totalCells === 0 ? {
|
|
1088
|
+
name: "completeness",
|
|
1089
|
+
score: null,
|
|
1090
|
+
note: "undetermined: no cells"
|
|
1091
|
+
} : dimension("completeness", round6$1(1 - missingCells / totalCells), presentCells, totalCells),
|
|
1092
|
+
table.rows.length === 0 ? {
|
|
1093
|
+
name: "uniqueness",
|
|
1094
|
+
score: null,
|
|
1095
|
+
note: "undetermined: no rows"
|
|
1096
|
+
} : dimension("uniqueness", round6$1(1 - options.duplicateRows / table.rows.length), table.rows.length - options.duplicateRows, table.rows.length),
|
|
1097
|
+
presentCells === 0 ? {
|
|
1098
|
+
name: "validity",
|
|
1099
|
+
score: null,
|
|
1100
|
+
note: "undetermined: no present cells"
|
|
1101
|
+
} : dimension("validity", round6$1(1 - invalidCells / presentCells), presentCells - invalidCells, presentCells),
|
|
1102
|
+
presentCells === 0 ? {
|
|
1103
|
+
name: "consistency",
|
|
1104
|
+
score: null,
|
|
1105
|
+
note: "undetermined: no present cells"
|
|
1106
|
+
} : dimension("consistency", round6$1(consistentCells / presentCells), consistentCells, presentCells),
|
|
1107
|
+
dateCells === 0 ? {
|
|
1108
|
+
name: "timeliness",
|
|
1109
|
+
score: null,
|
|
1110
|
+
note: "undetermined: no date cells"
|
|
1111
|
+
} : dimension("timeliness", round6$1(1 - futureDateCells / dateCells), dateCells - futureDateCells, dateCells),
|
|
1112
|
+
computeAccuracy(stats, table, options.declaredSchema)
|
|
1113
|
+
];
|
|
1114
|
+
const scores = dimensions.map((entry) => entry.score).filter((score) => score !== null);
|
|
1115
|
+
return {
|
|
1116
|
+
dimensions,
|
|
1117
|
+
overall: scores.length === 0 ? null : round6$1(scores.reduce((sum, score) => sum + score, 0) / scores.length),
|
|
1118
|
+
weightedOverall: computeWeightedOverall(dimensions, options.weights)
|
|
1119
|
+
};
|
|
1120
|
+
}
|
|
1121
|
+
/** Weighted mean over determinable dimensions (weights default to 1; validated non-negative by config). */
|
|
1122
|
+
function computeWeightedOverall(dimensions, weights) {
|
|
1123
|
+
let weightedSum = 0;
|
|
1124
|
+
let weightTotal = 0;
|
|
1125
|
+
for (const entry of dimensions) {
|
|
1126
|
+
if (entry.score === null) continue;
|
|
1127
|
+
const weight = weights?.[entry.name] ?? 1;
|
|
1128
|
+
weightedSum += weight * entry.score;
|
|
1129
|
+
weightTotal += weight;
|
|
1130
|
+
}
|
|
1131
|
+
return weightTotal === 0 ? null : round6$1(weightedSum / weightTotal);
|
|
1132
|
+
}
|
|
1133
|
+
/**
|
|
1134
|
+
* Limited `accuracy` definition: agreement between the full-table inferred
|
|
1135
|
+
* type of each declared column and the declared type. Without a declared
|
|
1136
|
+
* schema (or an external truth source) the dimension is `null` (undetermined)
|
|
1137
|
+
* — accuracy is never fabricated.
|
|
1138
|
+
*/
|
|
1139
|
+
function computeAccuracy(stats, table, declaredSchema) {
|
|
1140
|
+
if (declaredSchema === void 0 || Object.keys(declaredSchema).length === 0) return {
|
|
1141
|
+
name: "accuracy",
|
|
1142
|
+
score: null,
|
|
1143
|
+
note: "undetermined: no declared schema or external truth to compare against; accuracy is never fabricated"
|
|
1144
|
+
};
|
|
1145
|
+
let checked = 0;
|
|
1146
|
+
let matched = 0;
|
|
1147
|
+
const mismatches = [];
|
|
1148
|
+
for (const column of table.columns) {
|
|
1149
|
+
const declared = declaredSchema[column];
|
|
1150
|
+
if (declared === void 0) continue;
|
|
1151
|
+
checked += 1;
|
|
1152
|
+
const inferred = inferType(stats.get(column));
|
|
1153
|
+
if (inferred === declared) matched += 1;
|
|
1154
|
+
else mismatches.push(`${column}: declared ${declared}, inferred ${inferred}`);
|
|
1155
|
+
}
|
|
1156
|
+
if (checked === 0) return {
|
|
1157
|
+
name: "accuracy",
|
|
1158
|
+
score: null,
|
|
1159
|
+
note: "undetermined: declared schema covers none of the dataset columns"
|
|
1160
|
+
};
|
|
1161
|
+
return {
|
|
1162
|
+
name: "accuracy",
|
|
1163
|
+
score: round6$1(matched / checked),
|
|
1164
|
+
note: `${matched}/${checked} columns match their declared type` + (mismatches.length > 0 ? `; mismatches: ${mismatches.join(", ")}` : "")
|
|
1165
|
+
};
|
|
1166
|
+
}
|
|
1167
|
+
//#endregion
|
|
845
1168
|
//#region src/profile.ts
|
|
846
1169
|
/**
|
|
847
1170
|
* Deterministic dataset profiling: per-column type inference, missingness,
|
|
@@ -850,6 +1173,8 @@ function serializeDelimited(columns, rows, delimiter) {
|
|
|
850
1173
|
* no I/O; `generatedAt` is injected by the caller.
|
|
851
1174
|
* @module dsh-data-quality/profile
|
|
852
1175
|
*/
|
|
1176
|
+
/** Fallback duplicate-sample cap for direct engine use; the provider always passes the configured `evidenceRowLimit`. */
|
|
1177
|
+
const DEFAULT_DUPLICATE_SAMPLE_LIMIT = 20;
|
|
853
1178
|
/** Round to 6 significant digits for stable, readable report numbers. */
|
|
854
1179
|
function round6(value) {
|
|
855
1180
|
return Number(value.toPrecision(6));
|
|
@@ -874,6 +1199,8 @@ function numericProfile(values) {
|
|
|
874
1199
|
const highFence = p75 + 1.5 * iqr;
|
|
875
1200
|
const outliers = iqr === 0 ? 0 : sorted.filter((value) => value < lowFence || value > highFence).length;
|
|
876
1201
|
return {
|
|
1202
|
+
count: sorted.length,
|
|
1203
|
+
distinct: new Set(sorted).size,
|
|
877
1204
|
min: round6(sorted[0]),
|
|
878
1205
|
max: round6(sorted[sorted.length - 1]),
|
|
879
1206
|
mean: round6(sum / sorted.length),
|
|
@@ -883,17 +1210,43 @@ function numericProfile(values) {
|
|
|
883
1210
|
outliers
|
|
884
1211
|
};
|
|
885
1212
|
}
|
|
886
|
-
/**
|
|
887
|
-
function
|
|
1213
|
+
/** Deterministic sha256 key of one row's full content (columns in table order). */
|
|
1214
|
+
function rowContentKey(table, row) {
|
|
1215
|
+
return createHash("sha256").update(JSON.stringify(table.columns.map((column) => row[column] ?? null))).digest("hex");
|
|
1216
|
+
}
|
|
1217
|
+
/**
|
|
1218
|
+
* Detect full-content duplicate rows with a bounded sample of their 0-based
|
|
1219
|
+
* indexes. The first occurrence of each content is never counted; later rows
|
|
1220
|
+
* with identical full content are duplicates.
|
|
1221
|
+
* @param table - the parsed dataset.
|
|
1222
|
+
* @param options - sample cap and optional abort signal.
|
|
1223
|
+
* @returns the duplicate count, rate, and capped sample indexes.
|
|
1224
|
+
*/
|
|
1225
|
+
function detectDuplicateRows(table, options) {
|
|
1226
|
+
if (!Number.isSafeInteger(options.sampleLimit) || options.sampleLimit <= 0) throw new TypeError(`sampleLimit must be a positive safe integer, got ${String(options.sampleLimit)}`);
|
|
888
1227
|
const seen = /* @__PURE__ */ new Set();
|
|
889
1228
|
let duplicates = 0;
|
|
1229
|
+
const duplicateSampleRowIndexes = [];
|
|
890
1230
|
for (const [index, row] of table.rows.entries()) {
|
|
891
|
-
if (index % 1024 === 0) throwIfAborted(signal);
|
|
892
|
-
const key =
|
|
893
|
-
if (seen.has(key))
|
|
894
|
-
|
|
1231
|
+
if (index % 1024 === 0) throwIfAborted(options.signal);
|
|
1232
|
+
const key = rowContentKey(table, row);
|
|
1233
|
+
if (seen.has(key)) {
|
|
1234
|
+
duplicates += 1;
|
|
1235
|
+
if (duplicateSampleRowIndexes.length < options.sampleLimit) duplicateSampleRowIndexes.push(index);
|
|
1236
|
+
} else seen.add(key);
|
|
895
1237
|
}
|
|
896
|
-
return
|
|
1238
|
+
return {
|
|
1239
|
+
duplicateRows: duplicates,
|
|
1240
|
+
duplicateRate: table.rows.length === 0 ? 0 : round6(duplicates / table.rows.length),
|
|
1241
|
+
duplicateSampleRowIndexes
|
|
1242
|
+
};
|
|
1243
|
+
}
|
|
1244
|
+
/** Count rows whose full content duplicates an earlier row (first occurrence is not counted). */
|
|
1245
|
+
function countDuplicateRows(table, signal) {
|
|
1246
|
+
return detectDuplicateRows(table, {
|
|
1247
|
+
sampleLimit: 1,
|
|
1248
|
+
signal
|
|
1249
|
+
}).duplicateRows;
|
|
897
1250
|
}
|
|
898
1251
|
/** Profile one column over the given rows. */
|
|
899
1252
|
function profileColumn(rows, column, signal) {
|
|
@@ -980,14 +1333,28 @@ function profileTable(table, options) {
|
|
|
980
1333
|
throwIfAborted(options.signal);
|
|
981
1334
|
const profiled = options.sample === void 0 ? table.rows : sampleRows(table.rows, options.sample);
|
|
982
1335
|
const columns = table.columns.map((column) => profileColumn(profiled, column, options.signal));
|
|
983
|
-
const
|
|
1336
|
+
const detection = detectDuplicateRows(table, {
|
|
1337
|
+
sampleLimit: options.duplicateSampleLimit ?? DEFAULT_DUPLICATE_SAMPLE_LIMIT,
|
|
1338
|
+
signal: options.signal
|
|
1339
|
+
});
|
|
1340
|
+
const scorecard = computeScorecard(table, {
|
|
1341
|
+
now: options.generatedAt,
|
|
1342
|
+
duplicateRows: detection.duplicateRows,
|
|
1343
|
+
declaredSchema: options.declaredSchema,
|
|
1344
|
+
weights: options.scorecardWeights,
|
|
1345
|
+
signal: options.signal
|
|
1346
|
+
});
|
|
984
1347
|
return {
|
|
985
1348
|
dataset: options.dataset,
|
|
986
1349
|
rowCount: table.rows.length,
|
|
987
1350
|
sampled: profiled.length !== table.rows.length,
|
|
988
1351
|
profiledRows: profiled.length,
|
|
989
1352
|
columnCount: table.columns.length,
|
|
990
|
-
duplicateRows,
|
|
1353
|
+
duplicateRows: detection.duplicateRows,
|
|
1354
|
+
duplicateRate: detection.duplicateRate,
|
|
1355
|
+
duplicateSampleRowIndexes: detection.duplicateSampleRowIndexes,
|
|
1356
|
+
scorecard,
|
|
1357
|
+
...table.encoding !== void 0 ? { encoding: table.encoding } : {},
|
|
991
1358
|
columns,
|
|
992
1359
|
generatedAt: options.generatedAt
|
|
993
1360
|
};
|
|
@@ -996,19 +1363,167 @@ function profileTable(table, options) {
|
|
|
996
1363
|
function renderProfileText(report) {
|
|
997
1364
|
const lines = [];
|
|
998
1365
|
lines.push(`Profile of ${report.dataset}: ${report.rowCount} rows x ${report.columnCount} columns` + (report.sampled ? ` (column cards over a systematic sample of ${report.profiledRows} rows)` : ""));
|
|
999
|
-
if (report.duplicateRows > 0) lines.push(`Duplicate rows: ${report.duplicateRows}`);
|
|
1366
|
+
if (report.duplicateRows > 0) lines.push(`Duplicate rows: ${report.duplicateRows} (${(report.duplicateRate * 100).toFixed(1)}%)` + (report.duplicateSampleRowIndexes.length > 0 ? `; sample row indexes: ${report.duplicateSampleRowIndexes.join(", ")}` : ""));
|
|
1000
1367
|
for (const column of report.columns) {
|
|
1001
1368
|
const parts = [`${column.name}: ${column.inferredType}`];
|
|
1002
1369
|
if (column.missing > 0) parts.push(`missing ${column.missing} (${(column.missingRate * 100).toFixed(1)}%)`);
|
|
1003
1370
|
parts.push(`unique ${column.unique}`);
|
|
1004
|
-
if (column.numeric !== void 0) parts.push(`min ${column.numeric.min}, p25 ${column.numeric.p25}, median ${column.numeric.median}, p75 ${column.numeric.p75}, max ${column.numeric.max}, mean ${column.numeric.mean}` + (column.numeric.outliers > 0 ? `, ${column.numeric.outliers} IQR outliers` : ""));
|
|
1371
|
+
if (column.numeric !== void 0) parts.push(`count ${column.numeric.count}, distinct ${column.numeric.distinct}, min ${column.numeric.min}, p25 ${column.numeric.p25}, median ${column.numeric.median}, p75 ${column.numeric.p75}, max ${column.numeric.max}, mean ${column.numeric.mean}` + (column.numeric.outliers > 0 ? `, ${column.numeric.outliers} IQR outliers` : ""));
|
|
1005
1372
|
if (column.topValues !== void 0) parts.push(`top: ${column.topValues.map((entry) => `${JSON.stringify(entry.value)} x${entry.count}`).join(", ")}`);
|
|
1006
1373
|
for (const note of column.notes) parts.push(`note: ${note}`);
|
|
1007
1374
|
lines.push(`- ${parts.join("; ")}`);
|
|
1008
1375
|
}
|
|
1376
|
+
if (report.encoding !== void 0) lines.push(`Encoding: UTF-8${report.encoding.bom === "utf-8" ? " (BOM)" : ""}${report.encoding.validUtf8 ? "" : " (INVALID UTF-8)"}`);
|
|
1377
|
+
const overall = report.scorecard.overall;
|
|
1378
|
+
const weighted = report.scorecard.weightedOverall;
|
|
1379
|
+
lines.push(`Scorecard (overall ${overall === null ? "undetermined" : `${(overall * 100).toFixed(1)}%`}, weighted ${weighted === null ? "undetermined" : `${(weighted * 100).toFixed(1)}%`}):`);
|
|
1380
|
+
for (const dimension of report.scorecard.dimensions) {
|
|
1381
|
+
const value = dimension.score === null ? "undetermined" : `${(dimension.score * 100).toFixed(1)}%`;
|
|
1382
|
+
lines.push(` ${dimension.name}: ${value} (${dimension.note})`);
|
|
1383
|
+
}
|
|
1009
1384
|
return lines.join("\n");
|
|
1010
1385
|
}
|
|
1011
1386
|
//#endregion
|
|
1387
|
+
//#region src/contract.ts
|
|
1388
|
+
/**
|
|
1389
|
+
* Delivery contract for `data_clean`: a deterministic pre-delivery validation
|
|
1390
|
+
* summary (primary-key/type/dedupe row-count comparison plus uniqueness and
|
|
1391
|
+
* non-null regression) and the clean before/after profile diff report. Pure —
|
|
1392
|
+
* no I/O, no clock, no RNG; timestamps are injected.
|
|
1393
|
+
* @module dsh-data-quality/contract
|
|
1394
|
+
*/
|
|
1395
|
+
/** Count missing cells in one column of the cleaned output. */
|
|
1396
|
+
function countMissing(rows, column, signal) {
|
|
1397
|
+
let count = 0;
|
|
1398
|
+
for (const [index, row] of rows.entries()) {
|
|
1399
|
+
if (index % 1024 === 0) throwIfAborted(signal);
|
|
1400
|
+
if (isMissing(row[column])) count += 1;
|
|
1401
|
+
}
|
|
1402
|
+
return count;
|
|
1403
|
+
}
|
|
1404
|
+
/** Count present cells in one column that still fail to parse as the requested type. */
|
|
1405
|
+
function countNonConforming(rows, column, to, signal) {
|
|
1406
|
+
let count = 0;
|
|
1407
|
+
for (const [index, row] of rows.entries()) {
|
|
1408
|
+
if (index % 1024 === 0) throwIfAborted(signal);
|
|
1409
|
+
const cell = row[column];
|
|
1410
|
+
if (isMissing(cell)) continue;
|
|
1411
|
+
if (!(to === "number" ? parseNumeric(cell) !== void 0 : to === "date" ? parseDate(cell) !== void 0 : parseBoolean(cell) !== void 0)) count += 1;
|
|
1412
|
+
}
|
|
1413
|
+
return count;
|
|
1414
|
+
}
|
|
1415
|
+
/**
|
|
1416
|
+
* Compute the pre-delivery contract summary from a cleaning run: dedupe
|
|
1417
|
+
* before/after row counts, the dedupe key's uniqueness regression, and the
|
|
1418
|
+
* non-null/type regressions over the columns the rules targeted.
|
|
1419
|
+
* @param result - the cleaning outcome (input/output rows plus cleaned rows).
|
|
1420
|
+
* @param rules - the ordered rules that produced `result`.
|
|
1421
|
+
* @param options - optional abort signal.
|
|
1422
|
+
* @returns the contract summary.
|
|
1423
|
+
*/
|
|
1424
|
+
function computeCleanContract(result, rules, options = {}) {
|
|
1425
|
+
const dedupeRule = rules.find((rule) => rule.rule === "dedupe");
|
|
1426
|
+
const dedupeColumns = dedupeRule === void 0 ? null : [...dedupeRule.columns ?? result.columns];
|
|
1427
|
+
const keyColumns = dedupeColumns ?? result.columns;
|
|
1428
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1429
|
+
let remainingDuplicateRows = 0;
|
|
1430
|
+
for (const [index, row] of result.rows.entries()) {
|
|
1431
|
+
if (index % 1024 === 0) throwIfAborted(options.signal);
|
|
1432
|
+
const key = JSON.stringify(keyColumns.map((column) => row[column] ?? null));
|
|
1433
|
+
if (seen.has(key)) remainingDuplicateRows += 1;
|
|
1434
|
+
else seen.add(key);
|
|
1435
|
+
}
|
|
1436
|
+
const remainingMissing = rules.filter((rule) => rule.rule === "fill-missing").map((rule) => {
|
|
1437
|
+
const fill = rule;
|
|
1438
|
+
return {
|
|
1439
|
+
column: fill.column,
|
|
1440
|
+
count: countMissing(result.rows, fill.column, options.signal)
|
|
1441
|
+
};
|
|
1442
|
+
});
|
|
1443
|
+
const typeConformance = rules.filter((rule) => rule.rule === "coerce-type").map((rule) => {
|
|
1444
|
+
const coerce = rule;
|
|
1445
|
+
return {
|
|
1446
|
+
column: coerce.column,
|
|
1447
|
+
to: coerce.to,
|
|
1448
|
+
invalidCount: countNonConforming(result.rows, coerce.column, coerce.to, options.signal)
|
|
1449
|
+
};
|
|
1450
|
+
});
|
|
1451
|
+
const columnDecisions = traceColumnDecisions(result, rules);
|
|
1452
|
+
return {
|
|
1453
|
+
inputRows: result.inputRows,
|
|
1454
|
+
outputRows: result.outputRows,
|
|
1455
|
+
removedRows: result.inputRows - result.outputRows,
|
|
1456
|
+
dedupeColumns,
|
|
1457
|
+
uniqueKeys: remainingDuplicateRows === 0,
|
|
1458
|
+
remainingDuplicateRows,
|
|
1459
|
+
remainingMissing,
|
|
1460
|
+
typeConformance,
|
|
1461
|
+
columnDecisions
|
|
1462
|
+
};
|
|
1463
|
+
}
|
|
1464
|
+
/** The columns a single rule targets (single-column rules target one column; trim/dedupe may target many). */
|
|
1465
|
+
function ruleColumns(rule, allColumns) {
|
|
1466
|
+
switch (rule.rule) {
|
|
1467
|
+
case "fill-missing":
|
|
1468
|
+
case "coerce-type":
|
|
1469
|
+
case "normalize-unit":
|
|
1470
|
+
case "map-values": return [rule.column];
|
|
1471
|
+
case "trim":
|
|
1472
|
+
case "dedupe": return [...rule.columns ?? allColumns];
|
|
1473
|
+
default: return [];
|
|
1474
|
+
}
|
|
1475
|
+
}
|
|
1476
|
+
/** Build the per-column decision trace: strategies + affected rows, in dataset column order. */
|
|
1477
|
+
function traceColumnDecisions(result, rules) {
|
|
1478
|
+
const byColumn = /* @__PURE__ */ new Map();
|
|
1479
|
+
for (const [index, rule] of rules.entries()) {
|
|
1480
|
+
const log = result.logs[index];
|
|
1481
|
+
if (log === void 0) continue;
|
|
1482
|
+
for (const column of ruleColumns(rule, result.columns)) {
|
|
1483
|
+
const list = byColumn.get(column) ?? [];
|
|
1484
|
+
list.push({
|
|
1485
|
+
strategy: rule.rule,
|
|
1486
|
+
affectedRows: log.affectedRows
|
|
1487
|
+
});
|
|
1488
|
+
byColumn.set(column, list);
|
|
1489
|
+
}
|
|
1490
|
+
}
|
|
1491
|
+
return result.columns.filter((column) => byColumn.has(column)).map((column) => ({
|
|
1492
|
+
column,
|
|
1493
|
+
decisions: byColumn.get(column)
|
|
1494
|
+
}));
|
|
1495
|
+
}
|
|
1496
|
+
/**
|
|
1497
|
+
* Build the clean before/after profile diff: two full {@link ProfileReport}
|
|
1498
|
+
* snapshots (input and cleaned output) under one envelope. Reuses the profile
|
|
1499
|
+
* engine so the scorecard and duplicate detection stay consistent across the
|
|
1500
|
+
* whole plugin.
|
|
1501
|
+
* @param input - the input dataset.
|
|
1502
|
+
* @param output - the cleaned dataset.
|
|
1503
|
+
* @param options - dataset label, injected timestamp, optional weights, optional abort signal.
|
|
1504
|
+
* @returns the diff report.
|
|
1505
|
+
*/
|
|
1506
|
+
function computeCleanProfileDiff(input, output, options) {
|
|
1507
|
+
const before = profileTable(input, {
|
|
1508
|
+
dataset: options.dataset,
|
|
1509
|
+
generatedAt: options.generatedAt,
|
|
1510
|
+
scorecardWeights: options.scorecardWeights,
|
|
1511
|
+
signal: options.signal
|
|
1512
|
+
});
|
|
1513
|
+
const after = profileTable(output, {
|
|
1514
|
+
dataset: options.dataset,
|
|
1515
|
+
generatedAt: options.generatedAt,
|
|
1516
|
+
scorecardWeights: options.scorecardWeights,
|
|
1517
|
+
signal: options.signal
|
|
1518
|
+
});
|
|
1519
|
+
return {
|
|
1520
|
+
dataset: options.dataset,
|
|
1521
|
+
before,
|
|
1522
|
+
after,
|
|
1523
|
+
generatedAt: options.generatedAt
|
|
1524
|
+
};
|
|
1525
|
+
}
|
|
1526
|
+
//#endregion
|
|
1012
1527
|
//#region src/verify.ts
|
|
1013
1528
|
/**
|
|
1014
1529
|
* Declarative verification rules over a parsed {@link Table} (not-null,
|
|
@@ -1031,6 +1546,19 @@ var VerifyRuleError = class extends Error {
|
|
|
1031
1546
|
this.name = "VerifyRuleError";
|
|
1032
1547
|
}
|
|
1033
1548
|
};
|
|
1549
|
+
/** Raised for invalid expectations; `message` names the expectation index and reason. */
|
|
1550
|
+
var VerifyExpectationError = class extends Error {
|
|
1551
|
+
expectationIndex;
|
|
1552
|
+
/**
|
|
1553
|
+
* @param expectationIndex - index of the offending expectation in the request array.
|
|
1554
|
+
* @param message - actionable human-readable detail.
|
|
1555
|
+
*/
|
|
1556
|
+
constructor(expectationIndex, message) {
|
|
1557
|
+
super(message);
|
|
1558
|
+
this.expectationIndex = expectationIndex;
|
|
1559
|
+
this.name = "VerifyExpectationError";
|
|
1560
|
+
}
|
|
1561
|
+
};
|
|
1034
1562
|
/** Assert `column` exists. */
|
|
1035
1563
|
function requireColumn(columns, column, ruleIndex) {
|
|
1036
1564
|
if (!columns.includes(column)) throw new VerifyRuleError(ruleIndex, `rule ${ruleIndex}: unknown column ${JSON.stringify(column)} (columns: ${columns.join(", ")})`);
|
|
@@ -1076,13 +1604,90 @@ const CROSS_OPS = [
|
|
|
1076
1604
|
">=",
|
|
1077
1605
|
">"
|
|
1078
1606
|
];
|
|
1607
|
+
/** The metric ids, for validation and diagnostics. */
|
|
1608
|
+
const VERIFY_METRICS = [
|
|
1609
|
+
"rowCount",
|
|
1610
|
+
"columnSum",
|
|
1611
|
+
"columnMean",
|
|
1612
|
+
"uniqueCount",
|
|
1613
|
+
"nullCount"
|
|
1614
|
+
];
|
|
1615
|
+
/** Throw unless the expectation is well-formed (metric/column/tolerance). */
|
|
1616
|
+
function validateExpectation(table, expectation, index) {
|
|
1617
|
+
if (!VERIFY_METRICS.includes(expectation.metric)) throw new VerifyExpectationError(index, `expectation ${index}: unknown metric ${JSON.stringify(expectation.metric)} (expected one of ${VERIFY_METRICS.join(", ")})`);
|
|
1618
|
+
if (expectation.metric !== "rowCount") {
|
|
1619
|
+
if (expectation.column === void 0 || expectation.column === "") throw new VerifyExpectationError(index, `expectation ${index}: metric ${expectation.metric} requires a column`);
|
|
1620
|
+
if (!table.columns.includes(expectation.column)) throw new VerifyExpectationError(index, `expectation ${index}: unknown column ${JSON.stringify(expectation.column)} (columns: ${table.columns.join(", ")})`);
|
|
1621
|
+
} else if (expectation.column !== void 0) throw new VerifyExpectationError(index, `expectation ${index}: metric rowCount takes no column, got ${JSON.stringify(expectation.column)}`);
|
|
1622
|
+
if (expectation.tolerance !== void 0 && (typeof expectation.tolerance !== "number" || !Number.isFinite(expectation.tolerance) || expectation.tolerance < 0 || expectation.tolerance > 1)) throw new VerifyExpectationError(index, `expectation ${index}: tolerance must be a finite number in [0, 1], got ${String(expectation.tolerance)}`);
|
|
1623
|
+
}
|
|
1624
|
+
/** Compute the deterministic actual value of one expectation's metric. */
|
|
1625
|
+
function metricValueOf(table, expectation) {
|
|
1626
|
+
switch (expectation.metric) {
|
|
1627
|
+
case "rowCount": return table.rows.length;
|
|
1628
|
+
case "nullCount": {
|
|
1629
|
+
let count = 0;
|
|
1630
|
+
for (const row of table.rows) if (isMissing(row[expectation.column])) count += 1;
|
|
1631
|
+
return count;
|
|
1632
|
+
}
|
|
1633
|
+
case "uniqueCount": {
|
|
1634
|
+
const distinct = /* @__PURE__ */ new Set();
|
|
1635
|
+
for (const row of table.rows) {
|
|
1636
|
+
const cell = row[expectation.column];
|
|
1637
|
+
if (isMissing(cell)) continue;
|
|
1638
|
+
distinct.add(typeof cell === "string" ? cell : JSON.stringify(cell));
|
|
1639
|
+
}
|
|
1640
|
+
return distinct.size;
|
|
1641
|
+
}
|
|
1642
|
+
case "columnSum":
|
|
1643
|
+
case "columnMean": {
|
|
1644
|
+
let sum = 0;
|
|
1645
|
+
let count = 0;
|
|
1646
|
+
for (const row of table.rows) {
|
|
1647
|
+
const value = parseNumeric(row[expectation.column]);
|
|
1648
|
+
if (value === void 0) continue;
|
|
1649
|
+
sum += value;
|
|
1650
|
+
count += 1;
|
|
1651
|
+
}
|
|
1652
|
+
return expectation.metric === "columnSum" ? sum : count === 0 ? 0 : sum / count;
|
|
1653
|
+
}
|
|
1654
|
+
}
|
|
1655
|
+
}
|
|
1656
|
+
/**
|
|
1657
|
+
* Reconcile each expectation against its deterministic actual value. A
|
|
1658
|
+
* mismatch is a normal `passed: false` result, never a thrown error; invalid
|
|
1659
|
+
* metrics, columns, and tolerances fail loud.
|
|
1660
|
+
* @param table - the parsed dataset.
|
|
1661
|
+
* @param expectations - the expectations to reconcile.
|
|
1662
|
+
* @param defaultTolerance - configured fallback relative tolerance.
|
|
1663
|
+
* @param signal - optional abort signal.
|
|
1664
|
+
* @returns one outcome per expectation.
|
|
1665
|
+
*/
|
|
1666
|
+
function verifyExpectations(table, expectations, defaultTolerance, signal) {
|
|
1667
|
+
return expectations.map((expectation, index) => {
|
|
1668
|
+
throwIfAborted(signal);
|
|
1669
|
+
validateExpectation(table, expectation, index);
|
|
1670
|
+
const actual = metricValueOf(table, expectation);
|
|
1671
|
+
const tolerance = expectation.tolerance ?? defaultTolerance;
|
|
1672
|
+
const passed = numericClose(actual, expectation.expected, tolerance);
|
|
1673
|
+
return {
|
|
1674
|
+
metric: expectation.metric,
|
|
1675
|
+
...expectation.column !== void 0 ? { column: expectation.column } : {},
|
|
1676
|
+
expected: expectation.expected,
|
|
1677
|
+
actual,
|
|
1678
|
+
tolerance,
|
|
1679
|
+
passed
|
|
1680
|
+
};
|
|
1681
|
+
});
|
|
1682
|
+
}
|
|
1079
1683
|
/**
|
|
1080
1684
|
* Apply verification rules over a parsed table. A missing cell fails every
|
|
1081
|
-
* rule that reads it. The overall `passed` is the conjunction of rule passes
|
|
1082
|
-
* a failing dataset is a normal result, never a
|
|
1685
|
+
* rule that reads it. The overall `passed` is the conjunction of rule passes
|
|
1686
|
+
* and expectation passes; a failing dataset is a normal result, never a
|
|
1687
|
+
* thrown error.
|
|
1083
1688
|
* @param table - the parsed dataset.
|
|
1084
1689
|
* @param rules - non-empty rule list.
|
|
1085
|
-
* @param options - evidence cap, injected clock for `freshness`, abort signal.
|
|
1690
|
+
* @param options - evidence cap, injected clock for `freshness`, optional expectations, default tolerance, abort signal.
|
|
1086
1691
|
* @returns the verify report (without the dataset label; the caller adds it).
|
|
1087
1692
|
*/
|
|
1088
1693
|
function verifyTable(table, rules, options) {
|
|
@@ -1205,10 +1810,12 @@ function verifyTable(table, rules, options) {
|
|
|
1205
1810
|
}
|
|
1206
1811
|
results.push(result);
|
|
1207
1812
|
}
|
|
1813
|
+
const expectations = verifyExpectations(table, options.expectations ?? [], options.defaultTolerance ?? 1e-9, options.signal);
|
|
1208
1814
|
return {
|
|
1209
|
-
passed: results.every((result) => result.passed),
|
|
1815
|
+
passed: results.every((result) => result.passed) && expectations.every((expectation) => expectation.passed),
|
|
1210
1816
|
rowCount: table.rows.length,
|
|
1211
1817
|
rules: results,
|
|
1818
|
+
expectations,
|
|
1212
1819
|
generatedAt: options.now()
|
|
1213
1820
|
};
|
|
1214
1821
|
}
|
|
@@ -1224,6 +1831,10 @@ function renderVerifyText(report) {
|
|
|
1224
1831
|
}
|
|
1225
1832
|
if (rule.failedCount > rule.evidence.length) lines.push(` … and ${rule.failedCount - rule.evidence.length} more failing row(s)`);
|
|
1226
1833
|
}
|
|
1834
|
+
for (const expectation of report.expectations) {
|
|
1835
|
+
const target = expectation.column !== void 0 ? `${expectation.metric}(${expectation.column})` : expectation.metric;
|
|
1836
|
+
lines.push(`- [${expectation.passed ? "pass" : "FAIL"}] expectation ${target}: actual ${expectation.actual} vs expected ${expectation.expected} (tolerance ${expectation.tolerance})`);
|
|
1837
|
+
}
|
|
1227
1838
|
return lines.join("\n");
|
|
1228
1839
|
}
|
|
1229
1840
|
/**
|
|
@@ -1409,6 +2020,131 @@ function appendDataQualityEvent(session, type, data) {
|
|
|
1409
2020
|
if (Function.prototype.toString.call(append).includes("ignorable")) append.call(session, type, data, { ignorable: true });
|
|
1410
2021
|
}
|
|
1411
2022
|
//#endregion
|
|
2023
|
+
//#region src/presets.ts
|
|
2024
|
+
/** The built-in industry preset registry. */
|
|
2025
|
+
const INDUSTRY_PRESETS = {
|
|
2026
|
+
retail: {
|
|
2027
|
+
id: "retail",
|
|
2028
|
+
label: "Retail",
|
|
2029
|
+
columns: {
|
|
2030
|
+
order_id: "string",
|
|
2031
|
+
sku: "string",
|
|
2032
|
+
quantity: "number",
|
|
2033
|
+
unit_price: "number",
|
|
2034
|
+
revenue: "number",
|
|
2035
|
+
order_date: "date",
|
|
2036
|
+
customer_id: "string"
|
|
2037
|
+
}
|
|
2038
|
+
},
|
|
2039
|
+
saas: {
|
|
2040
|
+
id: "saas",
|
|
2041
|
+
label: "SaaS",
|
|
2042
|
+
columns: {
|
|
2043
|
+
account_id: "string",
|
|
2044
|
+
plan: "string",
|
|
2045
|
+
seats: "number",
|
|
2046
|
+
mrr: "number",
|
|
2047
|
+
signup_date: "date",
|
|
2048
|
+
churned: "boolean"
|
|
2049
|
+
}
|
|
2050
|
+
},
|
|
2051
|
+
fund: {
|
|
2052
|
+
id: "fund",
|
|
2053
|
+
label: "Fund",
|
|
2054
|
+
columns: {
|
|
2055
|
+
fund_code: "string",
|
|
2056
|
+
fund_name: "string",
|
|
2057
|
+
nav: "number",
|
|
2058
|
+
nav_date: "date",
|
|
2059
|
+
holding_value: "number",
|
|
2060
|
+
currency: "string"
|
|
2061
|
+
}
|
|
2062
|
+
},
|
|
2063
|
+
"real-estate": {
|
|
2064
|
+
id: "real-estate",
|
|
2065
|
+
label: "Real estate",
|
|
2066
|
+
columns: {
|
|
2067
|
+
property_id: "string",
|
|
2068
|
+
listing_price: "number",
|
|
2069
|
+
area_sqm: "number",
|
|
2070
|
+
bedrooms: "number",
|
|
2071
|
+
list_date: "date",
|
|
2072
|
+
city: "string"
|
|
2073
|
+
}
|
|
2074
|
+
},
|
|
2075
|
+
"e-commerce": {
|
|
2076
|
+
id: "e-commerce",
|
|
2077
|
+
label: "E-commerce",
|
|
2078
|
+
columns: {
|
|
2079
|
+
order_id: "string",
|
|
2080
|
+
product_id: "string",
|
|
2081
|
+
quantity: "number",
|
|
2082
|
+
price: "number",
|
|
2083
|
+
order_date: "date",
|
|
2084
|
+
status: "string"
|
|
2085
|
+
}
|
|
2086
|
+
},
|
|
2087
|
+
healthcare: {
|
|
2088
|
+
id: "healthcare",
|
|
2089
|
+
label: "Healthcare",
|
|
2090
|
+
columns: {
|
|
2091
|
+
patient_id: "string",
|
|
2092
|
+
admission_date: "date",
|
|
2093
|
+
discharge_date: "date",
|
|
2094
|
+
age: "number",
|
|
2095
|
+
diagnosis: "string",
|
|
2096
|
+
cost: "number"
|
|
2097
|
+
}
|
|
2098
|
+
},
|
|
2099
|
+
logistics: {
|
|
2100
|
+
id: "logistics",
|
|
2101
|
+
label: "Logistics",
|
|
2102
|
+
columns: {
|
|
2103
|
+
shipment_id: "string",
|
|
2104
|
+
origin: "string",
|
|
2105
|
+
destination: "string",
|
|
2106
|
+
weight_kg: "number",
|
|
2107
|
+
shipped_date: "date",
|
|
2108
|
+
delivered: "boolean"
|
|
2109
|
+
}
|
|
2110
|
+
},
|
|
2111
|
+
manufacturing: {
|
|
2112
|
+
id: "manufacturing",
|
|
2113
|
+
label: "Manufacturing",
|
|
2114
|
+
columns: {
|
|
2115
|
+
part_id: "string",
|
|
2116
|
+
quantity: "number",
|
|
2117
|
+
unit_cost: "number",
|
|
2118
|
+
produced_date: "date",
|
|
2119
|
+
defect: "boolean"
|
|
2120
|
+
}
|
|
2121
|
+
},
|
|
2122
|
+
energy: {
|
|
2123
|
+
id: "energy",
|
|
2124
|
+
label: "Energy",
|
|
2125
|
+
columns: {
|
|
2126
|
+
meter_id: "string",
|
|
2127
|
+
reading: "number",
|
|
2128
|
+
unit: "string",
|
|
2129
|
+
reading_date: "date",
|
|
2130
|
+
consumption: "number"
|
|
2131
|
+
}
|
|
2132
|
+
}
|
|
2133
|
+
};
|
|
2134
|
+
/** The preset ids, for diagnostics and documentation. */
|
|
2135
|
+
const INDUSTRY_PRESET_IDS = Object.keys(INDUSTRY_PRESETS);
|
|
2136
|
+
/**
|
|
2137
|
+
* Resolve an industry preset id to its registry entry, failing loud on an
|
|
2138
|
+
* unknown id.
|
|
2139
|
+
* @param id - the requested preset id.
|
|
2140
|
+
* @returns the preset.
|
|
2141
|
+
*/
|
|
2142
|
+
function resolveIndustryPreset(id) {
|
|
2143
|
+
const preset = INDUSTRY_PRESETS[id];
|
|
2144
|
+
if (preset === void 0) throw new Error(`unknown industryPreset ${JSON.stringify(id)}; choose one of: ${INDUSTRY_PRESET_IDS.join(", ")}`);
|
|
2145
|
+
return preset;
|
|
2146
|
+
}
|
|
2147
|
+
//#endregion
|
|
1412
2148
|
//#region src/present.ts
|
|
1413
2149
|
/** Maximum characters one cell contributes to a tool-facing row payload. */
|
|
1414
2150
|
const MAX_CELL_TEXT = 120;
|
|
@@ -1514,11 +2250,16 @@ var LocalDataQualityService = class extends DataQualityService {
|
|
|
1514
2250
|
/** @inheritdoc DataQualityService.profileDataset */
|
|
1515
2251
|
async profileDataset(request) {
|
|
1516
2252
|
throwIfAborted(request.signal);
|
|
1517
|
-
const
|
|
2253
|
+
const table = await loadTable(resolveWorkspacePath(request.workspace, request.dataset, this.config), this.config, request.signal);
|
|
2254
|
+
const declaredSchema = request.industryPreset === void 0 ? void 0 : resolveIndustryPreset(request.industryPreset).columns;
|
|
2255
|
+
const report = profileTable(table, {
|
|
1518
2256
|
dataset: request.dataset,
|
|
1519
2257
|
sample: request.sample,
|
|
1520
2258
|
generatedAt: this.deps.now(),
|
|
1521
|
-
signal: request.signal
|
|
2259
|
+
signal: request.signal,
|
|
2260
|
+
duplicateSampleLimit: this.config.evidenceRowLimit,
|
|
2261
|
+
declaredSchema,
|
|
2262
|
+
scorecardWeights: this.config.scorecardWeights
|
|
1522
2263
|
});
|
|
1523
2264
|
const reportKey = await this.persist("profile", request.dataset, report);
|
|
1524
2265
|
this.emitEvent(request.session, "profile", request.dataset, reportKey, {
|
|
@@ -1533,10 +2274,12 @@ var LocalDataQualityService = class extends DataQualityService {
|
|
|
1533
2274
|
/** @inheritdoc DataQualityService.cleanDataset */
|
|
1534
2275
|
async cleanDataset(request) {
|
|
1535
2276
|
throwIfAborted(request.signal);
|
|
2277
|
+
const dryRun = request.dryRun === true;
|
|
1536
2278
|
const absolute = resolveWorkspacePath(request.workspace, request.dataset, this.config);
|
|
1537
|
-
const
|
|
2279
|
+
const table = await loadTable(absolute, this.config, request.signal);
|
|
2280
|
+
const result = applyCleanRules(table, request.rules, { signal: request.signal });
|
|
1538
2281
|
let writtenPath;
|
|
1539
|
-
if (request.outputPath !== void 0) {
|
|
2282
|
+
if (!dryRun && request.outputPath !== void 0) {
|
|
1540
2283
|
const outputAbsolute = resolveWorkspacePath(request.workspace, request.outputPath, this.config);
|
|
1541
2284
|
if (outputAbsolute === absolute) throw new Error(`outputPath ${JSON.stringify(request.outputPath)} would overwrite the input dataset; choose a different path`);
|
|
1542
2285
|
const ext = path.extname(outputAbsolute).toLowerCase();
|
|
@@ -1547,19 +2290,51 @@ var LocalDataQualityService = class extends DataQualityService {
|
|
|
1547
2290
|
writtenPath = request.outputPath;
|
|
1548
2291
|
}
|
|
1549
2292
|
const generatedAt = this.deps.now();
|
|
2293
|
+
const contract = computeCleanContract(result, request.rules, { signal: request.signal });
|
|
2294
|
+
const diff = computeCleanProfileDiff(table, {
|
|
2295
|
+
columns: result.columns,
|
|
2296
|
+
rows: result.rows,
|
|
2297
|
+
...table.encoding !== void 0 ? { encoding: table.encoding } : {}
|
|
2298
|
+
}, {
|
|
2299
|
+
dataset: request.dataset,
|
|
2300
|
+
generatedAt,
|
|
2301
|
+
scorecardWeights: this.config.scorecardWeights,
|
|
2302
|
+
signal: request.signal
|
|
2303
|
+
});
|
|
1550
2304
|
const preview = {
|
|
1551
2305
|
columns: result.columns,
|
|
1552
2306
|
rows: result.rows.slice(0, this.config.evidenceRowLimit).map((row) => truncateRow(row))
|
|
1553
2307
|
};
|
|
2308
|
+
if (dryRun) {
|
|
2309
|
+
this.emitEvent(request.session, "clean", request.dataset, void 0, {
|
|
2310
|
+
rows: result.outputRows,
|
|
2311
|
+
columns: result.columns.length,
|
|
2312
|
+
rules: result.logs.length
|
|
2313
|
+
});
|
|
2314
|
+
return {
|
|
2315
|
+
dataset: request.dataset,
|
|
2316
|
+
inputRows: result.inputRows,
|
|
2317
|
+
outputRows: result.outputRows,
|
|
2318
|
+
dryRun: true,
|
|
2319
|
+
logs: result.logs,
|
|
2320
|
+
contract,
|
|
2321
|
+
preview,
|
|
2322
|
+
diffPreview: diff,
|
|
2323
|
+
generatedAt
|
|
2324
|
+
};
|
|
2325
|
+
}
|
|
1554
2326
|
const reportKey = await this.persist("clean", request.dataset, {
|
|
1555
2327
|
dataset: request.dataset,
|
|
1556
2328
|
inputRows: result.inputRows,
|
|
1557
2329
|
outputRows: result.outputRows,
|
|
2330
|
+
dryRun: false,
|
|
1558
2331
|
logs: result.logs,
|
|
2332
|
+
contract,
|
|
1559
2333
|
preview,
|
|
1560
2334
|
...writtenPath !== void 0 ? { outputPath: writtenPath } : {},
|
|
1561
2335
|
generatedAt
|
|
1562
2336
|
});
|
|
2337
|
+
await this.persist("clean-diff", request.dataset, diff);
|
|
1563
2338
|
this.emitEvent(request.session, "clean", request.dataset, reportKey, {
|
|
1564
2339
|
rows: result.outputRows,
|
|
1565
2340
|
columns: result.columns.length,
|
|
@@ -1569,32 +2344,57 @@ var LocalDataQualityService = class extends DataQualityService {
|
|
|
1569
2344
|
dataset: request.dataset,
|
|
1570
2345
|
inputRows: result.inputRows,
|
|
1571
2346
|
outputRows: result.outputRows,
|
|
2347
|
+
dryRun: false,
|
|
1572
2348
|
logs: result.logs,
|
|
2349
|
+
contract,
|
|
1573
2350
|
preview,
|
|
1574
2351
|
...writtenPath !== void 0 ? { outputPath: writtenPath } : {},
|
|
1575
2352
|
...reportKey !== void 0 ? { reportKey } : {},
|
|
1576
2353
|
generatedAt
|
|
1577
2354
|
};
|
|
1578
2355
|
}
|
|
2356
|
+
/** @inheritdoc DataQualityService.getReport */
|
|
2357
|
+
async getReport(key) {
|
|
2358
|
+
if (!isValidReportKey(key)) throw new Error(`invalid reportKey ${JSON.stringify(key)}: expected the deterministic <timestamp>-<kind>-<fingerprint> format`);
|
|
2359
|
+
const store = this.deps.store;
|
|
2360
|
+
if (store === void 0) throw new Error("report storage is disabled (storeReports is false); no persisted reports to read");
|
|
2361
|
+
const record = store.get(key);
|
|
2362
|
+
if (record === void 0) throw new Error(`no persisted report found for reportKey ${JSON.stringify(key)}`);
|
|
2363
|
+
return {
|
|
2364
|
+
key,
|
|
2365
|
+
...record
|
|
2366
|
+
};
|
|
2367
|
+
}
|
|
2368
|
+
/** @inheritdoc DataQualityService.listReports */
|
|
2369
|
+
async listReports(kind) {
|
|
2370
|
+
const store = this.deps.store;
|
|
2371
|
+
if (store === void 0) throw new Error("report storage is disabled (storeReports is false); no persisted reports to read");
|
|
2372
|
+
return store.list(kind);
|
|
2373
|
+
}
|
|
1579
2374
|
/** @inheritdoc DataQualityService.verifyDataset */
|
|
1580
2375
|
async verifyDataset(request) {
|
|
1581
2376
|
throwIfAborted(request.signal);
|
|
1582
2377
|
const outcome = verifyTable(await loadTable(resolveWorkspacePath(request.workspace, request.dataset, this.config), this.config, request.signal), request.rules, {
|
|
1583
2378
|
evidenceRowLimit: this.config.evidenceRowLimit,
|
|
1584
2379
|
now: this.deps.now,
|
|
1585
|
-
signal: request.signal
|
|
2380
|
+
signal: request.signal,
|
|
2381
|
+
expectations: request.expectations,
|
|
2382
|
+
defaultTolerance: this.config.defaultTolerance
|
|
1586
2383
|
});
|
|
1587
2384
|
const report = {
|
|
1588
2385
|
dataset: request.dataset,
|
|
1589
2386
|
...outcome
|
|
1590
2387
|
};
|
|
1591
2388
|
const failedRules = report.rules.filter((rule) => !rule.passed).length;
|
|
2389
|
+
const failedExpectations = report.expectations.filter((expectation) => !expectation.passed).length;
|
|
1592
2390
|
const reportKey = await this.persist("verify", request.dataset, report);
|
|
1593
2391
|
this.emitEvent(request.session, "verify", request.dataset, reportKey, {
|
|
1594
2392
|
rows: report.rowCount,
|
|
1595
2393
|
rules: report.rules.length,
|
|
1596
2394
|
failedRules,
|
|
1597
|
-
passed: report.passed
|
|
2395
|
+
passed: report.passed,
|
|
2396
|
+
expectations: report.expectations.length,
|
|
2397
|
+
failedExpectations
|
|
1598
2398
|
});
|
|
1599
2399
|
return {
|
|
1600
2400
|
...report,
|
|
@@ -1622,48 +2422,22 @@ var LocalDataQualityService = class extends DataQualityService {
|
|
|
1622
2422
|
}
|
|
1623
2423
|
};
|
|
1624
2424
|
//#endregion
|
|
1625
|
-
//#region src/tools/
|
|
1626
|
-
/**
|
|
1627
|
-
* Shared helpers for the three data-quality model tools: session workspace
|
|
1628
|
-
* resolution and a compact cleaning-run text render. Kept presentation-only;
|
|
1629
|
-
* computation lives in the engines and the provider.
|
|
1630
|
-
* @module dsh-data-quality/tools-shared
|
|
1631
|
-
*/
|
|
2425
|
+
//#region src/tools/profile-report-schema.ts
|
|
1632
2426
|
/**
|
|
1633
|
-
*
|
|
1634
|
-
*
|
|
1635
|
-
*
|
|
1636
|
-
* @
|
|
1637
|
-
* @returns the absolute workspace root.
|
|
1638
|
-
*/
|
|
1639
|
-
function workspaceOf(exec) {
|
|
1640
|
-
const cwd = exec.agent?.session.header.cwd;
|
|
1641
|
-
if (cwd === void 0) throw new Error("data-quality tools require an agent-owned session workspace");
|
|
1642
|
-
return path.resolve(cwd);
|
|
1643
|
-
}
|
|
1644
|
-
/** Human-readable cleaning summary for the tool's Native render. */
|
|
1645
|
-
function renderCleanText(report) {
|
|
1646
|
-
const lines = [];
|
|
1647
|
-
lines.push(`Cleaned ${report.dataset}: ${report.inputRows} -> ${report.outputRows} rows over ${report.logs.length} rule(s)`);
|
|
1648
|
-
for (const log of report.logs) lines.push(`- rule ${log.ruleIndex} (${log.rule}): ${log.affectedRows} row(s) affected; ${log.detail}`);
|
|
1649
|
-
if (report.outputPath !== void 0) lines.push(`Wrote cleaned dataset to ${report.outputPath}`);
|
|
1650
|
-
else lines.push("No outputPath given: the source file was left untouched; preview below.");
|
|
1651
|
-
if (report.preview.rows.length > 0) {
|
|
1652
|
-
lines.push(`Preview (first ${report.preview.rows.length} row(s)):`);
|
|
1653
|
-
for (const row of report.preview.rows) {
|
|
1654
|
-
const cells = report.preview.columns.map((column) => `${column}=${JSON.stringify(row[column] ?? null)}`).join(", ");
|
|
1655
|
-
lines.push(` ${cells}`);
|
|
1656
|
-
}
|
|
1657
|
-
}
|
|
1658
|
-
return lines.join("\n");
|
|
1659
|
-
}
|
|
1660
|
-
//#endregion
|
|
1661
|
-
//#region src/tools/profile.ts
|
|
1662
|
-
/**
|
|
1663
|
-
* The `data_profile` model tool: deterministic dataset profiling through
|
|
1664
|
-
* `ctx.dataQuality.profileDataset` — never model arithmetic.
|
|
1665
|
-
* @module dsh-data-quality/tools/profile
|
|
2427
|
+
* Shared JSON-Schema spec for the profile report's canonical value, reused by
|
|
2428
|
+
* `data_profile` (its output) and `data_clean` (the `diffPreview` before/after
|
|
2429
|
+
* snapshots). One source of truth so the two tools never drift.
|
|
2430
|
+
* @module dsh-data-quality/tools-profile-report-schema
|
|
1666
2431
|
*/
|
|
2432
|
+
const SCORECARD_DIMENSION_NAMES = [
|
|
2433
|
+
"completeness",
|
|
2434
|
+
"uniqueness",
|
|
2435
|
+
"validity",
|
|
2436
|
+
"consistency",
|
|
2437
|
+
"timeliness",
|
|
2438
|
+
"accuracy"
|
|
2439
|
+
];
|
|
2440
|
+
/** One column card's schema. */
|
|
1667
2441
|
const COLUMN_PROFILE_SCHEMA = {
|
|
1668
2442
|
type: "object",
|
|
1669
2443
|
properties: {
|
|
@@ -1698,6 +2472,14 @@ const COLUMN_PROFILE_SCHEMA = {
|
|
|
1698
2472
|
numeric: {
|
|
1699
2473
|
type: "object",
|
|
1700
2474
|
properties: {
|
|
2475
|
+
count: {
|
|
2476
|
+
type: "number",
|
|
2477
|
+
required: true
|
|
2478
|
+
},
|
|
2479
|
+
distinct: {
|
|
2480
|
+
type: "number",
|
|
2481
|
+
required: true
|
|
2482
|
+
},
|
|
1701
2483
|
min: {
|
|
1702
2484
|
type: "number",
|
|
1703
2485
|
required: true
|
|
@@ -1754,6 +2536,163 @@ const COLUMN_PROFILE_SCHEMA = {
|
|
|
1754
2536
|
},
|
|
1755
2537
|
additionalProperties: false
|
|
1756
2538
|
};
|
|
2539
|
+
/** The full profile report's schema. */
|
|
2540
|
+
const PROFILE_REPORT_SCHEMA = {
|
|
2541
|
+
type: "object",
|
|
2542
|
+
properties: {
|
|
2543
|
+
dataset: {
|
|
2544
|
+
type: "string",
|
|
2545
|
+
required: true
|
|
2546
|
+
},
|
|
2547
|
+
rowCount: {
|
|
2548
|
+
type: "number",
|
|
2549
|
+
required: true
|
|
2550
|
+
},
|
|
2551
|
+
sampled: {
|
|
2552
|
+
type: "boolean",
|
|
2553
|
+
required: true
|
|
2554
|
+
},
|
|
2555
|
+
profiledRows: {
|
|
2556
|
+
type: "number",
|
|
2557
|
+
required: true
|
|
2558
|
+
},
|
|
2559
|
+
columnCount: {
|
|
2560
|
+
type: "number",
|
|
2561
|
+
required: true
|
|
2562
|
+
},
|
|
2563
|
+
duplicateRows: {
|
|
2564
|
+
type: "number",
|
|
2565
|
+
required: true
|
|
2566
|
+
},
|
|
2567
|
+
duplicateRate: {
|
|
2568
|
+
type: "number",
|
|
2569
|
+
required: true
|
|
2570
|
+
},
|
|
2571
|
+
duplicateSampleRowIndexes: {
|
|
2572
|
+
type: "array",
|
|
2573
|
+
items: { type: "number" },
|
|
2574
|
+
required: true
|
|
2575
|
+
},
|
|
2576
|
+
scorecard: {
|
|
2577
|
+
type: "object",
|
|
2578
|
+
properties: {
|
|
2579
|
+
overall: {
|
|
2580
|
+
oneOf: [{ type: "number" }, { type: "null" }],
|
|
2581
|
+
required: true
|
|
2582
|
+
},
|
|
2583
|
+
weightedOverall: {
|
|
2584
|
+
oneOf: [{ type: "number" }, { type: "null" }],
|
|
2585
|
+
required: true
|
|
2586
|
+
},
|
|
2587
|
+
dimensions: {
|
|
2588
|
+
type: "array",
|
|
2589
|
+
items: {
|
|
2590
|
+
type: "object",
|
|
2591
|
+
properties: {
|
|
2592
|
+
name: {
|
|
2593
|
+
type: "string",
|
|
2594
|
+
enum: [...SCORECARD_DIMENSION_NAMES],
|
|
2595
|
+
required: true
|
|
2596
|
+
},
|
|
2597
|
+
score: {
|
|
2598
|
+
oneOf: [{ type: "number" }, { type: "null" }],
|
|
2599
|
+
required: true
|
|
2600
|
+
},
|
|
2601
|
+
note: {
|
|
2602
|
+
type: "string",
|
|
2603
|
+
required: true
|
|
2604
|
+
}
|
|
2605
|
+
},
|
|
2606
|
+
additionalProperties: false
|
|
2607
|
+
},
|
|
2608
|
+
required: true
|
|
2609
|
+
}
|
|
2610
|
+
},
|
|
2611
|
+
additionalProperties: false,
|
|
2612
|
+
required: true
|
|
2613
|
+
},
|
|
2614
|
+
encoding: {
|
|
2615
|
+
type: "object",
|
|
2616
|
+
properties: {
|
|
2617
|
+
bom: {
|
|
2618
|
+
oneOf: [{ type: "string" }, { type: "null" }],
|
|
2619
|
+
required: true
|
|
2620
|
+
},
|
|
2621
|
+
validUtf8: {
|
|
2622
|
+
type: "boolean",
|
|
2623
|
+
required: true
|
|
2624
|
+
}
|
|
2625
|
+
},
|
|
2626
|
+
additionalProperties: false
|
|
2627
|
+
},
|
|
2628
|
+
generatedAt: {
|
|
2629
|
+
type: "number",
|
|
2630
|
+
required: true
|
|
2631
|
+
},
|
|
2632
|
+
reportKey: { type: "string" },
|
|
2633
|
+
columns: {
|
|
2634
|
+
type: "array",
|
|
2635
|
+
items: COLUMN_PROFILE_SCHEMA,
|
|
2636
|
+
required: true
|
|
2637
|
+
}
|
|
2638
|
+
},
|
|
2639
|
+
additionalProperties: false
|
|
2640
|
+
};
|
|
2641
|
+
//#endregion
|
|
2642
|
+
//#region src/tools/shared.ts
|
|
2643
|
+
/**
|
|
2644
|
+
* Shared helpers for the three data-quality model tools: session workspace
|
|
2645
|
+
* resolution and a compact cleaning-run text render. Kept presentation-only;
|
|
2646
|
+
* computation lives in the engines and the provider.
|
|
2647
|
+
* @module dsh-data-quality/tools-shared
|
|
2648
|
+
*/
|
|
2649
|
+
/**
|
|
2650
|
+
* The absolute workspace root a tool call resolves dataset paths against: the
|
|
2651
|
+
* calling agent's per-session cwd (mirroring the official fs tools). Non-agent
|
|
2652
|
+
* calls fail loud — there is no honest workspace to confine paths to.
|
|
2653
|
+
* @param exec - the tool-execution context.
|
|
2654
|
+
* @returns the absolute workspace root.
|
|
2655
|
+
*/
|
|
2656
|
+
function workspaceOf(exec) {
|
|
2657
|
+
const cwd = exec.agent?.session.header.cwd;
|
|
2658
|
+
if (cwd === void 0) throw new Error("data-quality tools require an agent-owned session workspace");
|
|
2659
|
+
return path.resolve(cwd);
|
|
2660
|
+
}
|
|
2661
|
+
/** Human-readable cleaning summary for the tool's Native render. */
|
|
2662
|
+
function renderCleanText(report) {
|
|
2663
|
+
const lines = [];
|
|
2664
|
+
const mode = report.dryRun ? "Dry-run plan for" : "Cleaned";
|
|
2665
|
+
lines.push(`${mode} ${report.dataset}: ${report.inputRows} -> ${report.outputRows} rows over ${report.logs.length} rule(s)`);
|
|
2666
|
+
for (const log of report.logs) lines.push(`- rule ${log.ruleIndex} (${log.rule}): ${log.affectedRows} row(s) affected; ${log.detail}`);
|
|
2667
|
+
const contract = report.contract;
|
|
2668
|
+
const keyLabel = contract.dedupeColumns === null ? "full rows" : `[${contract.dedupeColumns.join(", ")}]`;
|
|
2669
|
+
lines.push(`Contract: ${contract.inputRows} -> ${contract.outputRows} rows (${contract.removedRows} removed); uniqueness ${contract.uniqueKeys ? "OK" : "VIOLATED"} over ${keyLabel}${contract.remainingDuplicateRows > 0 ? ` (${contract.remainingDuplicateRows} duplicate row(s) remain)` : ""}`);
|
|
2670
|
+
for (const entry of contract.remainingMissing) lines.push(`- non-null regression: ${entry.column} still has ${entry.count} missing cell(s)`);
|
|
2671
|
+
for (const entry of contract.typeConformance) lines.push(`- type regression: ${entry.column} (${entry.to}) has ${entry.invalidCount} non-conforming cell(s)`);
|
|
2672
|
+
for (const entry of contract.columnDecisions) {
|
|
2673
|
+
const decisions = entry.decisions.map((decision) => `${decision.strategy} (${decision.affectedRows} row(s))`).join(", ");
|
|
2674
|
+
lines.push(`- column ${entry.column}: ${decisions}`);
|
|
2675
|
+
}
|
|
2676
|
+
if (report.diffPreview !== void 0) lines.push(`Diff preview: ${report.diffPreview.before.rowCount} -> ${report.diffPreview.after.rowCount} rows; duplicates ${report.diffPreview.before.duplicateRows} -> ${report.diffPreview.after.duplicateRows}`);
|
|
2677
|
+
if (report.dryRun) lines.push("Dry run: no output file written and no report persisted; plan/preview above.");
|
|
2678
|
+
else if (report.outputPath !== void 0) lines.push(`Wrote cleaned dataset to ${report.outputPath}`);
|
|
2679
|
+
else lines.push("No outputPath given: the source file was left untouched; preview below.");
|
|
2680
|
+
if (report.preview.rows.length > 0) {
|
|
2681
|
+
lines.push(`Preview (first ${report.preview.rows.length} row(s)):`);
|
|
2682
|
+
for (const row of report.preview.rows) {
|
|
2683
|
+
const cells = report.preview.columns.map((column) => `${column}=${JSON.stringify(row[column] ?? null)}`).join(", ");
|
|
2684
|
+
lines.push(` ${cells}`);
|
|
2685
|
+
}
|
|
2686
|
+
}
|
|
2687
|
+
return lines.join("\n");
|
|
2688
|
+
}
|
|
2689
|
+
//#endregion
|
|
2690
|
+
//#region src/tools/profile.ts
|
|
2691
|
+
/**
|
|
2692
|
+
* The `data_profile` model tool: deterministic dataset profiling through
|
|
2693
|
+
* `ctx.dataQuality.profileDataset` — never model arithmetic.
|
|
2694
|
+
* @module dsh-data-quality/tools/profile
|
|
2695
|
+
*/
|
|
1757
2696
|
/**
|
|
1758
2697
|
* Build the `data_profile` tool definition against a mounted service.
|
|
1759
2698
|
* @param service - the mounted ctx.dataQuality implementation.
|
|
@@ -1764,8 +2703,8 @@ function defineProfileTool(service) {
|
|
|
1764
2703
|
name: "data_profile",
|
|
1765
2704
|
description: [
|
|
1766
2705
|
"Profile a workspace CSV/TSV/JSON/JSONL dataset with deterministic TypeScript computation (no mental math).",
|
|
1767
|
-
"Returns row/column counts, inferred column types, missing rates, unique counts, numeric distributions (min/max/mean/median/p25/p75), IQR outlier counts, mixed-type suspicion notes,
|
|
1768
|
-
"Column cards cover every row by default; pass sample for a deterministic systematic sample on large files. Datasets above the configured row/size caps are rejected — use sample or raise the caps. The full report persists to the data_quality storage domain (reportKey in the result)."
|
|
2706
|
+
"Returns row/column counts, inferred column types, missing rates, unique counts, numeric distributions (count/distinct/min/max/mean/median/p25/p75), IQR outlier counts, mixed-type suspicion notes, sha256 duplicate-row detection (rate + sample indexes), file encoding (BOM/UTF-8 validity), and a weighted DAMA six-dimension scorecard.",
|
|
2707
|
+
"Pass industryPreset (retail/saas/fund/real-estate/e-commerce/healthcare/logistics/manufacturing/energy) to compare the dataset against that industry's expected columns, making the scorecard accuracy dimension determinable. Column cards cover every row by default; pass sample for a deterministic systematic sample on large files. Datasets above the configured row/size caps are rejected — use sample or raise the caps. The full report persists to the data_quality storage domain (reportKey in the result)."
|
|
1769
2708
|
].join("\n"),
|
|
1770
2709
|
parameters: {
|
|
1771
2710
|
path: {
|
|
@@ -1776,49 +2715,25 @@ function defineProfileTool(service) {
|
|
|
1776
2715
|
sample: {
|
|
1777
2716
|
type: "number",
|
|
1778
2717
|
description: "Optional systematic sample size (every ceil(N/sample)-th row) for the column cards; row counts stay exact."
|
|
2718
|
+
},
|
|
2719
|
+
industryPreset: {
|
|
2720
|
+
type: "string",
|
|
2721
|
+
enum: [
|
|
2722
|
+
"retail",
|
|
2723
|
+
"saas",
|
|
2724
|
+
"fund",
|
|
2725
|
+
"real-estate",
|
|
2726
|
+
"e-commerce",
|
|
2727
|
+
"healthcare",
|
|
2728
|
+
"logistics",
|
|
2729
|
+
"manufacturing",
|
|
2730
|
+
"energy"
|
|
2731
|
+
],
|
|
2732
|
+
description: "Optional industry preset id; its expected columns feed the scorecard accuracy dimension."
|
|
1779
2733
|
}
|
|
1780
2734
|
},
|
|
1781
2735
|
output: {
|
|
1782
|
-
schema:
|
|
1783
|
-
type: "object",
|
|
1784
|
-
properties: {
|
|
1785
|
-
dataset: {
|
|
1786
|
-
type: "string",
|
|
1787
|
-
required: true
|
|
1788
|
-
},
|
|
1789
|
-
rowCount: {
|
|
1790
|
-
type: "number",
|
|
1791
|
-
required: true
|
|
1792
|
-
},
|
|
1793
|
-
sampled: {
|
|
1794
|
-
type: "boolean",
|
|
1795
|
-
required: true
|
|
1796
|
-
},
|
|
1797
|
-
profiledRows: {
|
|
1798
|
-
type: "number",
|
|
1799
|
-
required: true
|
|
1800
|
-
},
|
|
1801
|
-
columnCount: {
|
|
1802
|
-
type: "number",
|
|
1803
|
-
required: true
|
|
1804
|
-
},
|
|
1805
|
-
duplicateRows: {
|
|
1806
|
-
type: "number",
|
|
1807
|
-
required: true
|
|
1808
|
-
},
|
|
1809
|
-
generatedAt: {
|
|
1810
|
-
type: "number",
|
|
1811
|
-
required: true
|
|
1812
|
-
},
|
|
1813
|
-
reportKey: { type: "string" },
|
|
1814
|
-
columns: {
|
|
1815
|
-
type: "array",
|
|
1816
|
-
items: COLUMN_PROFILE_SCHEMA,
|
|
1817
|
-
required: true
|
|
1818
|
-
}
|
|
1819
|
-
},
|
|
1820
|
-
additionalProperties: false
|
|
1821
|
-
},
|
|
2736
|
+
schema: PROFILE_REPORT_SCHEMA,
|
|
1822
2737
|
render: (_args, value) => [{
|
|
1823
2738
|
type: "text",
|
|
1824
2739
|
text: renderProfileText(value)
|
|
@@ -1828,6 +2743,7 @@ function defineProfileTool(service) {
|
|
|
1828
2743
|
return service.profileDataset({
|
|
1829
2744
|
dataset: args.path,
|
|
1830
2745
|
sample: args.sample,
|
|
2746
|
+
...args.industryPreset !== void 0 ? { industryPreset: args.industryPreset } : {},
|
|
1831
2747
|
workspace: workspaceOf(exec),
|
|
1832
2748
|
session: exec.agent?.session,
|
|
1833
2749
|
signal: exec.signal
|
|
@@ -1995,7 +2911,7 @@ function defineCleanTool(service) {
|
|
|
1995
2911
|
description: [
|
|
1996
2912
|
"Apply declarative cleaning rules to a workspace CSV/TSV/JSON/JSONL dataset with deterministic TypeScript computation (no mental math).",
|
|
1997
2913
|
"Rules apply in array order: dedupe (by column group), fill-missing (constant/mean/median/forward), coerce-type (number/date/boolean; failures counted and set to missing), normalize-unit (e.g. 万/亿 suffixes to base units), trim (whitespace), map-values (enum mapping).",
|
|
1998
|
-
"The source file is NEVER overwritten. Without outputPath the run is preview-only; with outputPath the cleaned dataset is written there (workspace-confined, .csv/.tsv/.json/.jsonl). Returns the per-rule audit log (
|
|
2914
|
+
"The source file is NEVER overwritten. Without outputPath the run is preview-only; with outputPath the cleaned dataset is written there (workspace-confined, .csv/.tsv/.json/.jsonl). Returns the per-rule audit log, the pre-delivery contract summary (with per-column decision trace), and a bounded preview. Pass dryRun: true to skip the write and get the cleaning plan plus the expected contract/diff preview instead. The full report persists to the data_quality storage domain (reportKey)."
|
|
1999
2915
|
].join("\n"),
|
|
2000
2916
|
parameters: {
|
|
2001
2917
|
path: {
|
|
@@ -2010,6 +2926,10 @@ function defineCleanTool(service) {
|
|
|
2010
2926
|
outputPath: {
|
|
2011
2927
|
type: "string",
|
|
2012
2928
|
description: "Optional workspace-relative output path for the cleaned dataset (must differ from path)."
|
|
2929
|
+
},
|
|
2930
|
+
dryRun: {
|
|
2931
|
+
type: "boolean",
|
|
2932
|
+
description: "When true, do not write any output file; return the cleaning plan and expected contract/diff preview instead (default false)."
|
|
2013
2933
|
}
|
|
2014
2934
|
},
|
|
2015
2935
|
output: {
|
|
@@ -2028,12 +2948,38 @@ function defineCleanTool(service) {
|
|
|
2028
2948
|
type: "number",
|
|
2029
2949
|
required: true
|
|
2030
2950
|
},
|
|
2951
|
+
dryRun: {
|
|
2952
|
+
type: "boolean",
|
|
2953
|
+
required: true
|
|
2954
|
+
},
|
|
2031
2955
|
generatedAt: {
|
|
2032
2956
|
type: "number",
|
|
2033
2957
|
required: true
|
|
2034
2958
|
},
|
|
2035
2959
|
outputPath: { type: "string" },
|
|
2036
2960
|
reportKey: { type: "string" },
|
|
2961
|
+
diffPreview: {
|
|
2962
|
+
type: "object",
|
|
2963
|
+
properties: {
|
|
2964
|
+
dataset: {
|
|
2965
|
+
type: "string",
|
|
2966
|
+
required: true
|
|
2967
|
+
},
|
|
2968
|
+
before: {
|
|
2969
|
+
...PROFILE_REPORT_SCHEMA,
|
|
2970
|
+
required: true
|
|
2971
|
+
},
|
|
2972
|
+
after: {
|
|
2973
|
+
...PROFILE_REPORT_SCHEMA,
|
|
2974
|
+
required: true
|
|
2975
|
+
},
|
|
2976
|
+
generatedAt: {
|
|
2977
|
+
type: "number",
|
|
2978
|
+
required: true
|
|
2979
|
+
}
|
|
2980
|
+
},
|
|
2981
|
+
additionalProperties: false
|
|
2982
|
+
},
|
|
2037
2983
|
logs: {
|
|
2038
2984
|
type: "array",
|
|
2039
2985
|
items: {
|
|
@@ -2068,6 +3014,117 @@ function defineCleanTool(service) {
|
|
|
2068
3014
|
},
|
|
2069
3015
|
required: true
|
|
2070
3016
|
},
|
|
3017
|
+
contract: {
|
|
3018
|
+
type: "object",
|
|
3019
|
+
properties: {
|
|
3020
|
+
inputRows: {
|
|
3021
|
+
type: "number",
|
|
3022
|
+
required: true
|
|
3023
|
+
},
|
|
3024
|
+
outputRows: {
|
|
3025
|
+
type: "number",
|
|
3026
|
+
required: true
|
|
3027
|
+
},
|
|
3028
|
+
removedRows: {
|
|
3029
|
+
type: "number",
|
|
3030
|
+
required: true
|
|
3031
|
+
},
|
|
3032
|
+
dedupeColumns: {
|
|
3033
|
+
oneOf: [{
|
|
3034
|
+
type: "array",
|
|
3035
|
+
items: { type: "string" }
|
|
3036
|
+
}, { type: "null" }],
|
|
3037
|
+
required: true
|
|
3038
|
+
},
|
|
3039
|
+
uniqueKeys: {
|
|
3040
|
+
type: "boolean",
|
|
3041
|
+
required: true
|
|
3042
|
+
},
|
|
3043
|
+
remainingDuplicateRows: {
|
|
3044
|
+
type: "number",
|
|
3045
|
+
required: true
|
|
3046
|
+
},
|
|
3047
|
+
remainingMissing: {
|
|
3048
|
+
type: "array",
|
|
3049
|
+
items: {
|
|
3050
|
+
type: "object",
|
|
3051
|
+
properties: {
|
|
3052
|
+
column: {
|
|
3053
|
+
type: "string",
|
|
3054
|
+
required: true
|
|
3055
|
+
},
|
|
3056
|
+
count: {
|
|
3057
|
+
type: "number",
|
|
3058
|
+
required: true
|
|
3059
|
+
}
|
|
3060
|
+
},
|
|
3061
|
+
additionalProperties: false
|
|
3062
|
+
},
|
|
3063
|
+
required: true
|
|
3064
|
+
},
|
|
3065
|
+
typeConformance: {
|
|
3066
|
+
type: "array",
|
|
3067
|
+
items: {
|
|
3068
|
+
type: "object",
|
|
3069
|
+
properties: {
|
|
3070
|
+
column: {
|
|
3071
|
+
type: "string",
|
|
3072
|
+
required: true
|
|
3073
|
+
},
|
|
3074
|
+
to: {
|
|
3075
|
+
type: "string",
|
|
3076
|
+
enum: [
|
|
3077
|
+
"number",
|
|
3078
|
+
"date",
|
|
3079
|
+
"boolean"
|
|
3080
|
+
],
|
|
3081
|
+
required: true
|
|
3082
|
+
},
|
|
3083
|
+
invalidCount: {
|
|
3084
|
+
type: "number",
|
|
3085
|
+
required: true
|
|
3086
|
+
}
|
|
3087
|
+
},
|
|
3088
|
+
additionalProperties: false
|
|
3089
|
+
},
|
|
3090
|
+
required: true
|
|
3091
|
+
},
|
|
3092
|
+
columnDecisions: {
|
|
3093
|
+
type: "array",
|
|
3094
|
+
items: {
|
|
3095
|
+
type: "object",
|
|
3096
|
+
properties: {
|
|
3097
|
+
column: {
|
|
3098
|
+
type: "string",
|
|
3099
|
+
required: true
|
|
3100
|
+
},
|
|
3101
|
+
decisions: {
|
|
3102
|
+
type: "array",
|
|
3103
|
+
items: {
|
|
3104
|
+
type: "object",
|
|
3105
|
+
properties: {
|
|
3106
|
+
strategy: {
|
|
3107
|
+
type: "string",
|
|
3108
|
+
required: true
|
|
3109
|
+
},
|
|
3110
|
+
affectedRows: {
|
|
3111
|
+
type: "number",
|
|
3112
|
+
required: true
|
|
3113
|
+
}
|
|
3114
|
+
},
|
|
3115
|
+
additionalProperties: false
|
|
3116
|
+
},
|
|
3117
|
+
required: true
|
|
3118
|
+
}
|
|
3119
|
+
},
|
|
3120
|
+
additionalProperties: false
|
|
3121
|
+
},
|
|
3122
|
+
required: true
|
|
3123
|
+
}
|
|
3124
|
+
},
|
|
3125
|
+
additionalProperties: false,
|
|
3126
|
+
required: true
|
|
3127
|
+
},
|
|
2071
3128
|
preview: {
|
|
2072
3129
|
type: "object",
|
|
2073
3130
|
properties: {
|
|
@@ -2098,6 +3155,7 @@ function defineCleanTool(service) {
|
|
|
2098
3155
|
dataset: args.path,
|
|
2099
3156
|
rules: args.rules,
|
|
2100
3157
|
...args.outputPath !== void 0 ? { outputPath: args.outputPath } : {},
|
|
3158
|
+
...args.dryRun !== void 0 ? { dryRun: args.dryRun } : {},
|
|
2101
3159
|
workspace: workspaceOf(exec),
|
|
2102
3160
|
session: exec.agent?.session,
|
|
2103
3161
|
signal: exec.signal
|
|
@@ -2113,6 +3171,74 @@ function defineCleanTool(service) {
|
|
|
2113
3171
|
* (`passed: false` with evidence), never a tool error.
|
|
2114
3172
|
* @module dsh-data-quality/tools/verify
|
|
2115
3173
|
*/
|
|
3174
|
+
const EXPECTATION_METRICS = [
|
|
3175
|
+
"rowCount",
|
|
3176
|
+
"columnSum",
|
|
3177
|
+
"columnMean",
|
|
3178
|
+
"uniqueCount",
|
|
3179
|
+
"nullCount"
|
|
3180
|
+
];
|
|
3181
|
+
const EXPECTATION_SCHEMA = {
|
|
3182
|
+
type: "array",
|
|
3183
|
+
items: {
|
|
3184
|
+
type: "object",
|
|
3185
|
+
properties: {
|
|
3186
|
+
metric: {
|
|
3187
|
+
type: "string",
|
|
3188
|
+
enum: [...EXPECTATION_METRICS],
|
|
3189
|
+
required: true,
|
|
3190
|
+
description: "Metric to reconcile: rowCount/columnSum/columnMean/uniqueCount/nullCount."
|
|
3191
|
+
},
|
|
3192
|
+
column: {
|
|
3193
|
+
type: "string",
|
|
3194
|
+
description: "Required for every metric except rowCount."
|
|
3195
|
+
},
|
|
3196
|
+
expected: {
|
|
3197
|
+
type: "number",
|
|
3198
|
+
required: true,
|
|
3199
|
+
description: "The expected value to reconcile against."
|
|
3200
|
+
},
|
|
3201
|
+
tolerance: {
|
|
3202
|
+
type: "number",
|
|
3203
|
+
description: "Optional relative tolerance in [0, 1]; defaults to defaultTolerance."
|
|
3204
|
+
}
|
|
3205
|
+
},
|
|
3206
|
+
additionalProperties: false,
|
|
3207
|
+
description: "Reconcile a deterministic computed metric against an expected value with relative tolerance."
|
|
3208
|
+
},
|
|
3209
|
+
description: "Optional metric expectations; each yields passed true or passed false with actual/expected/tolerance detail."
|
|
3210
|
+
};
|
|
3211
|
+
const EXPECTATION_RESULT_SCHEMA = {
|
|
3212
|
+
type: "array",
|
|
3213
|
+
items: {
|
|
3214
|
+
type: "object",
|
|
3215
|
+
properties: {
|
|
3216
|
+
metric: {
|
|
3217
|
+
type: "string",
|
|
3218
|
+
enum: [...EXPECTATION_METRICS],
|
|
3219
|
+
required: true
|
|
3220
|
+
},
|
|
3221
|
+
column: { type: "string" },
|
|
3222
|
+
expected: {
|
|
3223
|
+
type: "number",
|
|
3224
|
+
required: true
|
|
3225
|
+
},
|
|
3226
|
+
actual: {
|
|
3227
|
+
type: "number",
|
|
3228
|
+
required: true
|
|
3229
|
+
},
|
|
3230
|
+
tolerance: {
|
|
3231
|
+
type: "number",
|
|
3232
|
+
required: true
|
|
3233
|
+
},
|
|
3234
|
+
passed: {
|
|
3235
|
+
type: "boolean",
|
|
3236
|
+
required: true
|
|
3237
|
+
}
|
|
3238
|
+
},
|
|
3239
|
+
additionalProperties: false
|
|
3240
|
+
}
|
|
3241
|
+
};
|
|
2116
3242
|
const VERIFY_RULE_SCHEMA = {
|
|
2117
3243
|
type: "array",
|
|
2118
3244
|
items: { oneOf: [
|
|
@@ -2290,7 +3416,8 @@ function defineVerifyTool(service) {
|
|
|
2290
3416
|
description: [
|
|
2291
3417
|
"Verify a workspace CSV/TSV/JSON/JSONL dataset against declarative quality rules with deterministic TypeScript computation (no mental math).",
|
|
2292
3418
|
"Rules: not-null, unique (column group), range (numeric bounds), regex, enum, cross-column (e.g. startDate < endDate), freshness (date column within N days of asOf). A missing cell fails every rule that reads it.",
|
|
2293
|
-
"
|
|
3419
|
+
"Optional expectations reconcile deterministic metrics (rowCount/columnSum/columnMean/uniqueCount/nullCount) against expected values with relative tolerance; a mismatch is a normal passed: false with actual/expected/tolerance detail, never a tool error.",
|
|
3420
|
+
"Returns per-rule pass/fail with capped failing-row evidence plus the expectation outcomes. Overall failure is a NORMAL result with passed: false — not a tool error. The full report persists to the data_quality storage domain (reportKey)."
|
|
2294
3421
|
].join("\n"),
|
|
2295
3422
|
parameters: {
|
|
2296
3423
|
path: {
|
|
@@ -2301,6 +3428,10 @@ function defineVerifyTool(service) {
|
|
|
2301
3428
|
rules: {
|
|
2302
3429
|
...VERIFY_RULE_SCHEMA,
|
|
2303
3430
|
required: true
|
|
3431
|
+
},
|
|
3432
|
+
expectations: {
|
|
3433
|
+
...EXPECTATION_SCHEMA,
|
|
3434
|
+
description: "Optional metric expectations to reconcile (rowCount/columnSum/columnMean/uniqueCount/nullCount)."
|
|
2304
3435
|
}
|
|
2305
3436
|
},
|
|
2306
3437
|
output: {
|
|
@@ -2380,6 +3511,10 @@ function defineVerifyTool(service) {
|
|
|
2380
3511
|
additionalProperties: false
|
|
2381
3512
|
},
|
|
2382
3513
|
required: true
|
|
3514
|
+
},
|
|
3515
|
+
expectations: {
|
|
3516
|
+
...EXPECTATION_RESULT_SCHEMA,
|
|
3517
|
+
required: true
|
|
2383
3518
|
}
|
|
2384
3519
|
},
|
|
2385
3520
|
additionalProperties: false
|
|
@@ -2393,6 +3528,7 @@ function defineVerifyTool(service) {
|
|
|
2393
3528
|
return service.verifyDataset({
|
|
2394
3529
|
dataset: args.path,
|
|
2395
3530
|
rules: args.rules,
|
|
3531
|
+
...args.expectations !== void 0 ? { expectations: args.expectations } : {},
|
|
2396
3532
|
workspace: workspaceOf(exec),
|
|
2397
3533
|
session: exec.agent?.session,
|
|
2398
3534
|
signal: exec.signal
|
|
@@ -2401,6 +3537,130 @@ function defineVerifyTool(service) {
|
|
|
2401
3537
|
});
|
|
2402
3538
|
}
|
|
2403
3539
|
//#endregion
|
|
3540
|
+
//#region src/tools/report.ts
|
|
3541
|
+
/**
|
|
3542
|
+
* The `data_report` model tool: read persisted profile/clean-diff reports
|
|
3543
|
+
* back from the `data_quality` storage domain, by exact `reportKey` or by
|
|
3544
|
+
* `kind`. Deterministic read-only consumption — no model arithmetic.
|
|
3545
|
+
* @module dsh-data-quality/tools/report
|
|
3546
|
+
*/
|
|
3547
|
+
/** The report kinds a caller may query. */
|
|
3548
|
+
const REPORT_KINDS = [
|
|
3549
|
+
"profile",
|
|
3550
|
+
"clean",
|
|
3551
|
+
"clean-diff",
|
|
3552
|
+
"verify",
|
|
3553
|
+
"citations"
|
|
3554
|
+
];
|
|
3555
|
+
/** Project a stored report into the canonical value (the stored report is already lossless JSON). */
|
|
3556
|
+
function toView(record) {
|
|
3557
|
+
return {
|
|
3558
|
+
key: record.key,
|
|
3559
|
+
kind: record.kind,
|
|
3560
|
+
at: record.at,
|
|
3561
|
+
dataset: record.dataset,
|
|
3562
|
+
report: record.report
|
|
3563
|
+
};
|
|
3564
|
+
}
|
|
3565
|
+
/** Human-readable report summary for the tool's Native render. */
|
|
3566
|
+
function renderReportText(value) {
|
|
3567
|
+
const lines = [];
|
|
3568
|
+
if (value.key !== void 0) lines.push(value.records.length === 0 ? `No report for ${value.key}` : `Report ${value.key}`);
|
|
3569
|
+
else lines.push(`Reports of kind ${value.kind ?? ""} (${value.records.length})`);
|
|
3570
|
+
for (const record of value.records) lines.push(`- ${record.key} [${record.kind}] ${record.dataset} @ ${new Date(record.at).toISOString()}`);
|
|
3571
|
+
return lines.join("\n");
|
|
3572
|
+
}
|
|
3573
|
+
/**
|
|
3574
|
+
* Build the `data_report` tool definition against a mounted service.
|
|
3575
|
+
* @param service - the mounted ctx.dataQuality implementation.
|
|
3576
|
+
* @returns the tool definition to register.
|
|
3577
|
+
*/
|
|
3578
|
+
function defineReportTool(service) {
|
|
3579
|
+
return defineTool({
|
|
3580
|
+
name: "data_report",
|
|
3581
|
+
description: [
|
|
3582
|
+
"Read persisted data-quality reports back from the data_quality storage domain (deterministic, read-only).",
|
|
3583
|
+
"Pass key (the exact reportKey a prior run returned) to fetch one report, or kind to list every persisted report of that kind, ordered chronologically. Exactly one of key/kind.",
|
|
3584
|
+
"Returns the report envelope(s): kind, dataset, timestamp, and the full stored report (profile/clean/clean-diff/verify/citations). Missing keys and unknown kinds fail loudly."
|
|
3585
|
+
].join("\n"),
|
|
3586
|
+
parameters: {
|
|
3587
|
+
key: {
|
|
3588
|
+
type: "string",
|
|
3589
|
+
description: "Exact storage reportKey (e.g. 20260819000000000-profile-1a2b3c4d); fetches that one report."
|
|
3590
|
+
},
|
|
3591
|
+
kind: {
|
|
3592
|
+
type: "string",
|
|
3593
|
+
enum: [...REPORT_KINDS],
|
|
3594
|
+
description: "Report kind to list (profile/clean/clean-diff/verify/citations)."
|
|
3595
|
+
}
|
|
3596
|
+
},
|
|
3597
|
+
output: {
|
|
3598
|
+
schema: {
|
|
3599
|
+
type: "object",
|
|
3600
|
+
properties: {
|
|
3601
|
+
key: { type: "string" },
|
|
3602
|
+
kind: {
|
|
3603
|
+
type: "string",
|
|
3604
|
+
enum: [...REPORT_KINDS]
|
|
3605
|
+
},
|
|
3606
|
+
records: {
|
|
3607
|
+
type: "array",
|
|
3608
|
+
items: {
|
|
3609
|
+
type: "object",
|
|
3610
|
+
properties: {
|
|
3611
|
+
key: {
|
|
3612
|
+
type: "string",
|
|
3613
|
+
required: true
|
|
3614
|
+
},
|
|
3615
|
+
kind: {
|
|
3616
|
+
type: "string",
|
|
3617
|
+
enum: [...REPORT_KINDS],
|
|
3618
|
+
required: true
|
|
3619
|
+
},
|
|
3620
|
+
at: {
|
|
3621
|
+
type: "number",
|
|
3622
|
+
required: true
|
|
3623
|
+
},
|
|
3624
|
+
dataset: {
|
|
3625
|
+
type: "string",
|
|
3626
|
+
required: true
|
|
3627
|
+
},
|
|
3628
|
+
report: {
|
|
3629
|
+
type: "json",
|
|
3630
|
+
required: true
|
|
3631
|
+
}
|
|
3632
|
+
},
|
|
3633
|
+
additionalProperties: false
|
|
3634
|
+
},
|
|
3635
|
+
required: true
|
|
3636
|
+
}
|
|
3637
|
+
},
|
|
3638
|
+
additionalProperties: false
|
|
3639
|
+
},
|
|
3640
|
+
render: (_args, value) => [{
|
|
3641
|
+
type: "text",
|
|
3642
|
+
text: renderReportText(value)
|
|
3643
|
+
}]
|
|
3644
|
+
},
|
|
3645
|
+
async execute(args, _exec) {
|
|
3646
|
+
const hasKey = args.key !== void 0;
|
|
3647
|
+
if (hasKey === (args.kind !== void 0)) throw new Error("data_report needs exactly one of key/kind");
|
|
3648
|
+
if (hasKey) {
|
|
3649
|
+
const record = await service.getReport(args.key);
|
|
3650
|
+
return {
|
|
3651
|
+
key: args.key,
|
|
3652
|
+
records: [toView(record)]
|
|
3653
|
+
};
|
|
3654
|
+
}
|
|
3655
|
+
const records = await service.listReports(args.kind);
|
|
3656
|
+
return {
|
|
3657
|
+
kind: args.kind,
|
|
3658
|
+
records: records.map(toView)
|
|
3659
|
+
};
|
|
3660
|
+
}
|
|
3661
|
+
});
|
|
3662
|
+
}
|
|
3663
|
+
//#endregion
|
|
2404
3664
|
//#region src/version.ts
|
|
2405
3665
|
/**
|
|
2406
3666
|
* Plugin version, kept in one place so `scripts/release.mjs` can stamp it and
|
|
@@ -2408,11 +3668,11 @@ function defineVerifyTool(service) {
|
|
|
2408
3668
|
* @module dsh-data-quality/version
|
|
2409
3669
|
*/
|
|
2410
3670
|
/** The package version reported in persisted reports. */
|
|
2411
|
-
const VERSION = "0.
|
|
3671
|
+
const VERSION = "0.2.0";
|
|
2412
3672
|
//#endregion
|
|
2413
3673
|
//#region src/index.ts
|
|
2414
3674
|
const name = "data-quality";
|
|
2415
|
-
/** The
|
|
3675
|
+
/** The four model tools and the durable report domain. */
|
|
2416
3676
|
const inject = ["tools", "storageDomain"];
|
|
2417
3677
|
/**
|
|
2418
3678
|
* Mount the seam: resolve config (fail loud), open the report domain, publish
|
|
@@ -2439,7 +3699,11 @@ async function apply(ctx, config = {}) {
|
|
|
2439
3699
|
await reports.put(key, record);
|
|
2440
3700
|
return key;
|
|
2441
3701
|
},
|
|
2442
|
-
get: (key) => reports.get(key)
|
|
3702
|
+
get: (key) => reports.get(key),
|
|
3703
|
+
list: (kind) => [...reports.entries()].filter(([, record]) => record.kind === kind).sort(([keyA], [keyB]) => keyA < keyB ? -1 : keyA > keyB ? 1 : 0).map(([key, record]) => ({
|
|
3704
|
+
key,
|
|
3705
|
+
...record
|
|
3706
|
+
}))
|
|
2443
3707
|
};
|
|
2444
3708
|
}
|
|
2445
3709
|
const service = new LocalDataQualityService(ctx, resolved, {
|
|
@@ -2449,7 +3713,8 @@ async function apply(ctx, config = {}) {
|
|
|
2449
3713
|
ctx.tools.register(defineProfileTool(service));
|
|
2450
3714
|
ctx.tools.register(defineCleanTool(service));
|
|
2451
3715
|
ctx.tools.register(defineVerifyTool(service));
|
|
2452
|
-
|
|
3716
|
+
ctx.tools.register(defineReportTool(service));
|
|
3717
|
+
logger.info(`dsh-data-quality ${VERSION} mounted: ctx.dataQuality + data_profile/data_clean/data_verify/data_report`);
|
|
2453
3718
|
if (domain !== void 0) {
|
|
2454
3719
|
const handle = domain;
|
|
2455
3720
|
ctx.effect(() => async () => {
|
|
@@ -2458,4 +3723,4 @@ async function apply(ctx, config = {}) {
|
|
|
2458
3723
|
}
|
|
2459
3724
|
}
|
|
2460
3725
|
//#endregion
|
|
2461
|
-
export { Config, DATA_QUALITY_EVENT_TYPES, DataQualityService, DatasetError, LocalDataQualityService, MAX_CELL_TEXT, VERSION, appendDataQualityEvent, apply, applyCleanRules, checkCitations, dataQualityDomainSpec, inject, isMissing, loadDocument, loadTable, name, parseBoolean, parseDate, parseDelimited, parseJsonTable, parseLocator, parseNumeric, profileTable, renderProfileText, renderVerifyText, reportKeyOf, reportRecordSchema, resolveConfig, resolveWorkspacePath, sampleRows, serializeDelimited, truncateCell, truncateRow, verifyTable };
|
|
3726
|
+
export { Config, DATA_QUALITY_EVENT_TYPES, DataQualityService, DatasetError, INDUSTRY_PRESETS, INDUSTRY_PRESET_IDS, LocalDataQualityService, MAX_CELL_TEXT, VERSION, VerifyExpectationError, appendDataQualityEvent, apply, applyCleanRules, checkCitations, computeCleanContract, computeCleanProfileDiff, computeScorecard, countDuplicateRows, dataQualityDomainSpec, dateFormatOf, detectDuplicateRows, detectEncoding, inject, isMissing, isValidReportKey, loadDocument, loadTable, name, numericProfile, parseBoolean, parseDate, parseDateCell, parseDelimited, parseJsonTable, parseLocator, parseNumeric, profileTable, readDatasetFile, renderProfileText, renderVerifyText, reportKeyOf, reportRecordSchema, resolveConfig, resolveIndustryPreset, resolveWorkspacePath, sampleRows, serializeDelimited, truncateCell, truncateRow, verifyExpectations, verifyTable };
|