dsh-data-quality 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/LICENSE +201 -0
- package/README.es.md +181 -0
- package/README.hi.md +181 -0
- package/README.md +181 -0
- package/README.pt.md +181 -0
- package/README.zh.md +181 -0
- package/THIRD_PARTY_NOTICES.md +20 -0
- package/cordis.patch.yml +46 -0
- package/lib/index.js +2458 -0
- package/lib/types/clean.d.ts +82 -0
- package/lib/types/clean.d.ts.map +1 -0
- package/lib/types/clean.js +351 -0
- package/lib/types/clean.js.map +1 -0
- package/lib/types/config.d.ts +47 -0
- package/lib/types/config.d.ts.map +1 -0
- package/lib/types/config.js +66 -0
- package/lib/types/config.js.map +1 -0
- package/lib/types/dataset.d.ts +133 -0
- package/lib/types/dataset.d.ts.map +1 -0
- package/lib/types/dataset.js +404 -0
- package/lib/types/dataset.js.map +1 -0
- package/lib/types/events.d.ts +73 -0
- package/lib/types/events.d.ts.map +1 -0
- package/lib/types/events.js +41 -0
- package/lib/types/events.js.map +1 -0
- package/lib/types/index.d.ts +45 -0
- package/lib/types/index.d.ts.map +1 -0
- package/lib/types/index.js +78 -0
- package/lib/types/index.js.map +1 -0
- package/lib/types/present.d.ts +24 -0
- package/lib/types/present.d.ts.map +1 -0
- package/lib/types/present.js +34 -0
- package/lib/types/present.js.map +1 -0
- package/lib/types/profile.d.ts +79 -0
- package/lib/types/profile.d.ts.map +1 -0
- package/lib/types/profile.js +196 -0
- package/lib/types/profile.js.map +1 -0
- package/lib/types/provider-local.d.ts +56 -0
- package/lib/types/provider-local.d.ts.map +1 -0
- package/lib/types/provider-local.js +163 -0
- package/lib/types/provider-local.js.map +1 -0
- package/lib/types/service.d.ts +160 -0
- package/lib/types/service.d.ts.map +1 -0
- package/lib/types/service.js +25 -0
- package/lib/types/service.js.map +1 -0
- package/lib/types/store.d.ts +61 -0
- package/lib/types/store.d.ts.map +1 -0
- package/lib/types/store.js +42 -0
- package/lib/types/store.js.map +1 -0
- package/lib/types/tools/clean.d.ts +14 -0
- package/lib/types/tools/clean.d.ts.map +1 -0
- package/lib/types/tools/clean.js +146 -0
- package/lib/types/tools/clean.js.map +1 -0
- package/lib/types/tools/profile.d.ts +13 -0
- package/lib/types/tools/profile.d.ts.map +1 -0
- package/lib/types/tools/profile.js +91 -0
- package/lib/types/tools/profile.js.map +1 -0
- package/lib/types/tools/shared.d.ts +19 -0
- package/lib/types/tools/shared.d.ts.map +1 -0
- package/lib/types/tools/shared.js +44 -0
- package/lib/types/tools/shared.js.map +1 -0
- package/lib/types/tools/verify.d.ts +14 -0
- package/lib/types/tools/verify.d.ts.map +1 -0
- package/lib/types/tools/verify.js +160 -0
- package/lib/types/tools/verify.js.map +1 -0
- package/lib/types/verify.d.ts +124 -0
- package/lib/types/verify.d.ts.map +1 -0
- package/lib/types/verify.js +391 -0
- package/lib/types/verify.js.map +1 -0
- package/lib/types/version.d.ts +8 -0
- package/lib/types/version.d.ts.map +1 -0
- package/lib/types/version.js +8 -0
- package/lib/types/version.js.map +1 -0
- package/package.json +137 -0
- package/src/clean.ts +382 -0
- package/src/config.ts +104 -0
- package/src/dataset.ts +445 -0
- package/src/events.ts +90 -0
- package/src/index.ts +115 -0
- package/src/present.ts +38 -0
- package/src/profile.ts +250 -0
- package/src/provider-local.ts +194 -0
- package/src/service.ts +172 -0
- package/src/store.ts +74 -0
- package/src/tools/clean.ts +150 -0
- package/src/tools/profile.ts +94 -0
- package/src/tools/shared.ts +47 -0
- package/src/tools/verify.ts +163 -0
- package/src/verify.ts +496 -0
- package/src/version.ts +8 -0
package/lib/index.js
ADDED
|
@@ -0,0 +1,2458 @@
|
|
|
1
|
+
import z from "@deepseek-ai/schemastery";
|
|
2
|
+
import z$1 from "zod";
|
|
3
|
+
import { defineDomain, domainTable } from "@deepseek-ai/dsh-storage-domain";
|
|
4
|
+
import { mkdir, open, stat, writeFile } from "node:fs/promises";
|
|
5
|
+
import path from "node:path";
|
|
6
|
+
import { createHash } from "node:crypto";
|
|
7
|
+
import { KNOWN_SESSION_EVENT_TYPES } from "@deepseek-ai/dsh-session";
|
|
8
|
+
import { Service } from "@deepseek-ai/cordis";
|
|
9
|
+
import { defineTool } from "@deepseek-ai/dsh-tools";
|
|
10
|
+
//#region src/config.ts
|
|
11
|
+
/**
|
|
12
|
+
* Config schema and resolution for `dsh-data-quality`. Every tunable is a
|
|
13
|
+
* validated {@link Config} field changeable from cordis.yml; the resolution
|
|
14
|
+
* step validates bounds so misconfiguration fails loud at mount.
|
|
15
|
+
* @module dsh-data-quality/config
|
|
16
|
+
*/
|
|
17
|
+
/** Schemastery schema: the loader validates and fills defaults before `apply`. */
|
|
18
|
+
const Config = z.object({
|
|
19
|
+
enabled: z.boolean().default(true),
|
|
20
|
+
maxRows: z.number().default(2e5),
|
|
21
|
+
maxFileSizeMB: z.number().default(64),
|
|
22
|
+
defaultTolerance: z.number().default(1e-9),
|
|
23
|
+
evidenceRowLimit: z.number().default(20),
|
|
24
|
+
allowedExtensions: z.array(z.string()).default([
|
|
25
|
+
".csv",
|
|
26
|
+
".tsv",
|
|
27
|
+
".json",
|
|
28
|
+
".jsonl"
|
|
29
|
+
]),
|
|
30
|
+
workspaceRoot: z.string().default(""),
|
|
31
|
+
storeReports: z.boolean().default(true)
|
|
32
|
+
});
|
|
33
|
+
/** Throw unless `value` is a positive safe integer. */
|
|
34
|
+
function assertPositiveInt(name, value) {
|
|
35
|
+
if (!Number.isSafeInteger(value) || value <= 0) throw new TypeError(`${name} must be a positive safe integer, got ${String(value)}`);
|
|
36
|
+
}
|
|
37
|
+
/** Throw unless `value` is a finite number in `[min, max]`. */
|
|
38
|
+
function assertFiniteRange(name, value, min, max) {
|
|
39
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value < min || value > max) throw new TypeError(`${name} must be a finite number in [${min}, ${max}], got ${String(value)}`);
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Validate raw values and fill explicit defaults. Invalid bounds throw here —
|
|
43
|
+
* misconfiguration fails loud at mount even without the Schemastery loader.
|
|
44
|
+
* @param config - raw (possibly partial) plugin config.
|
|
45
|
+
* @returns the fully resolved config.
|
|
46
|
+
*/
|
|
47
|
+
function resolveConfig(config = {}) {
|
|
48
|
+
const maxRows = config.maxRows ?? 2e5;
|
|
49
|
+
assertPositiveInt("maxRows", maxRows);
|
|
50
|
+
const maxFileSizeMB = config.maxFileSizeMB ?? 64;
|
|
51
|
+
assertFiniteRange("maxFileSizeMB", maxFileSizeMB, Number.MIN_VALUE, 1024);
|
|
52
|
+
const defaultTolerance = config.defaultTolerance ?? 1e-9;
|
|
53
|
+
assertFiniteRange("defaultTolerance", defaultTolerance, 0, 1);
|
|
54
|
+
const evidenceRowLimit = config.evidenceRowLimit ?? 20;
|
|
55
|
+
assertPositiveInt("evidenceRowLimit", evidenceRowLimit);
|
|
56
|
+
const allowedExtensions = (config.allowedExtensions ?? [
|
|
57
|
+
".csv",
|
|
58
|
+
".tsv",
|
|
59
|
+
".json",
|
|
60
|
+
".jsonl"
|
|
61
|
+
]).map((ext) => {
|
|
62
|
+
if (!/^\.[a-z0-9]+$/u.test(ext)) throw new TypeError(`allowedExtensions entries must be dot-prefixed lowercase extensions, got ${JSON.stringify(ext)}`);
|
|
63
|
+
return ext;
|
|
64
|
+
});
|
|
65
|
+
if (allowedExtensions.length === 0) throw new TypeError("allowedExtensions must not be empty");
|
|
66
|
+
return {
|
|
67
|
+
enabled: config.enabled ?? true,
|
|
68
|
+
maxRows,
|
|
69
|
+
maxFileSizeMB,
|
|
70
|
+
defaultTolerance,
|
|
71
|
+
evidenceRowLimit,
|
|
72
|
+
allowedExtensions,
|
|
73
|
+
workspaceRoot: config.workspaceRoot ?? "",
|
|
74
|
+
storeReports: config.storeReports ?? true
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
//#endregion
|
|
78
|
+
//#region src/dataset.ts
|
|
79
|
+
/**
|
|
80
|
+
* Dataset loading for `dsh-data-quality`: workspace-confined path resolution,
|
|
81
|
+
* size/row guards, and deterministic parsers for CSV/TSV/JSON/JSONL. All
|
|
82
|
+
* parsing is hand-rolled (no external runtime dependencies) and bounded by
|
|
83
|
+
* the resolved config; oversized inputs reject loudly with actionable hints.
|
|
84
|
+
* @module dsh-data-quality/dataset
|
|
85
|
+
*/
|
|
86
|
+
/** Raised for every user-facing dataset failure; `code` is stable for tests and tooling. */
|
|
87
|
+
var DatasetError = class extends Error {
|
|
88
|
+
code;
|
|
89
|
+
/**
|
|
90
|
+
* @param code - stable machine-readable reason.
|
|
91
|
+
* @param message - actionable human-readable detail.
|
|
92
|
+
*/
|
|
93
|
+
constructor(code, message) {
|
|
94
|
+
super(message);
|
|
95
|
+
this.code = code;
|
|
96
|
+
this.name = "DatasetError";
|
|
97
|
+
}
|
|
98
|
+
};
|
|
99
|
+
/**
|
|
100
|
+
* Resolve `requested` inside `root`, rejecting escapes and disallowed
|
|
101
|
+
* extensions. Both sides go through `path.resolve` before comparison so
|
|
102
|
+
* Windows backslash/forward-slash mixes never defeat the containment check.
|
|
103
|
+
* @param root - absolute workspace root.
|
|
104
|
+
* @param requested - the caller-supplied path (relative to root, or absolute inside root).
|
|
105
|
+
* @param config - resolved config (extension allowlist).
|
|
106
|
+
* @returns the normalized absolute path inside `root`.
|
|
107
|
+
*/
|
|
108
|
+
function resolveWorkspacePath(root, requested, config) {
|
|
109
|
+
if (requested.trim() === "") throw new DatasetError("path-escape", "dataset path must not be empty");
|
|
110
|
+
const resolvedRoot = path.resolve(root);
|
|
111
|
+
const resolved = path.resolve(resolvedRoot, requested);
|
|
112
|
+
if (resolved !== resolvedRoot && !resolved.startsWith(resolvedRoot + path.sep)) throw new DatasetError("path-escape", `path ${JSON.stringify(requested)} resolves outside the workspace root ${JSON.stringify(resolvedRoot)}`);
|
|
113
|
+
const ext = path.extname(resolved).toLowerCase();
|
|
114
|
+
if (!config.allowedExtensions.includes(ext)) throw new DatasetError("extension-denied", `extension ${JSON.stringify(ext)} is not in allowedExtensions (${config.allowedExtensions.join(", ")})`);
|
|
115
|
+
return resolved;
|
|
116
|
+
}
|
|
117
|
+
/**
|
|
118
|
+
* Short deterministic fingerprint of a dataset path, used in storage keys.
|
|
119
|
+
* @param datasetPath - the workspace-relative dataset path as callers gave it.
|
|
120
|
+
* @returns an 8-hex-char fingerprint.
|
|
121
|
+
*/
|
|
122
|
+
function pathFingerprint(datasetPath) {
|
|
123
|
+
return createHash("sha256").update(datasetPath).digest("hex").slice(0, 8);
|
|
124
|
+
}
|
|
125
|
+
/** Assert the abort signal has not fired; throws the signal reason otherwise. */
|
|
126
|
+
function throwIfAborted(signal) {
|
|
127
|
+
if (signal?.aborted === true) throw signal.reason instanceof Error ? signal.reason : /* @__PURE__ */ new Error("operation aborted");
|
|
128
|
+
}
|
|
129
|
+
/**
|
|
130
|
+
* Read a dataset file under the size cap.
|
|
131
|
+
* @param absolutePath - normalized absolute path (from {@link resolveWorkspacePath}).
|
|
132
|
+
* @param config - resolved config (size cap).
|
|
133
|
+
* @param signal - optional abort signal honored around the read.
|
|
134
|
+
* @returns the UTF-8 text.
|
|
135
|
+
*/
|
|
136
|
+
async function readDatasetText(absolutePath, config, signal) {
|
|
137
|
+
throwIfAborted(signal);
|
|
138
|
+
let info;
|
|
139
|
+
try {
|
|
140
|
+
info = await stat(absolutePath);
|
|
141
|
+
} catch {
|
|
142
|
+
throw new DatasetError("not-found", `dataset file not found: ${absolutePath}`);
|
|
143
|
+
}
|
|
144
|
+
if (!info.isFile()) throw new DatasetError("not-found", `dataset path is not a file: ${absolutePath}`);
|
|
145
|
+
const capBytes = config.maxFileSizeMB * 1024 * 1024;
|
|
146
|
+
if (info.size > capBytes) throw new DatasetError("too-large", `dataset is ${(info.size / 1024 / 1024).toFixed(1)} MiB, above the ${config.maxFileSizeMB} MiB cap; split the file or raise maxFileSizeMB`);
|
|
147
|
+
const handle = await open(absolutePath, "r");
|
|
148
|
+
try {
|
|
149
|
+
throwIfAborted(signal);
|
|
150
|
+
return await handle.readFile("utf8");
|
|
151
|
+
} finally {
|
|
152
|
+
await handle.close();
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
/**
|
|
156
|
+
* Parse CSV/TSV text (RFC-4180-style: quoted fields, `""` escapes, CRLF/LF).
|
|
157
|
+
* The first record is the header; short records pad with `null`, long records
|
|
158
|
+
* fail loud. An empty field parses as `null` (missing).
|
|
159
|
+
* @param text - file text.
|
|
160
|
+
* @param delimiter - field delimiter (`,` or tab).
|
|
161
|
+
* @param config - resolved config (row cap).
|
|
162
|
+
* @param signal - optional abort signal, checked per 1024-record stride.
|
|
163
|
+
* @returns the parsed table.
|
|
164
|
+
*/
|
|
165
|
+
function parseDelimited(text, delimiter, config, signal) {
|
|
166
|
+
const records = [];
|
|
167
|
+
let field = "";
|
|
168
|
+
let record = [];
|
|
169
|
+
let inQuotes = false;
|
|
170
|
+
const pushField = () => {
|
|
171
|
+
record.push(field);
|
|
172
|
+
field = "";
|
|
173
|
+
};
|
|
174
|
+
const pushRecord = () => {
|
|
175
|
+
pushField();
|
|
176
|
+
if (record.length === 1 && record[0] === "") {
|
|
177
|
+
record = [];
|
|
178
|
+
field = "";
|
|
179
|
+
return;
|
|
180
|
+
}
|
|
181
|
+
records.push(record);
|
|
182
|
+
record = [];
|
|
183
|
+
if (records.length % 1024 === 0) throwIfAborted(signal);
|
|
184
|
+
if (records.length - 1 > config.maxRows) throw new DatasetError("too-many-rows", `dataset exceeds the ${config.maxRows} row cap; use the sample parameter or raise maxRows`);
|
|
185
|
+
};
|
|
186
|
+
let index = 0;
|
|
187
|
+
while (index < text.length) {
|
|
188
|
+
const char = text[index];
|
|
189
|
+
if (inQuotes) {
|
|
190
|
+
if (char === "\"") {
|
|
191
|
+
if (text[index + 1] === "\"") {
|
|
192
|
+
field += "\"";
|
|
193
|
+
index += 2;
|
|
194
|
+
continue;
|
|
195
|
+
}
|
|
196
|
+
inQuotes = false;
|
|
197
|
+
index += 1;
|
|
198
|
+
continue;
|
|
199
|
+
}
|
|
200
|
+
field += char;
|
|
201
|
+
index += 1;
|
|
202
|
+
continue;
|
|
203
|
+
}
|
|
204
|
+
if (char === "\"") {
|
|
205
|
+
inQuotes = true;
|
|
206
|
+
index += 1;
|
|
207
|
+
continue;
|
|
208
|
+
}
|
|
209
|
+
if (char === delimiter) {
|
|
210
|
+
pushField();
|
|
211
|
+
index += 1;
|
|
212
|
+
continue;
|
|
213
|
+
}
|
|
214
|
+
if (char === "\r") {
|
|
215
|
+
index += 1;
|
|
216
|
+
continue;
|
|
217
|
+
}
|
|
218
|
+
if (char === "\n") {
|
|
219
|
+
pushRecord();
|
|
220
|
+
index += 1;
|
|
221
|
+
continue;
|
|
222
|
+
}
|
|
223
|
+
field += char;
|
|
224
|
+
index += 1;
|
|
225
|
+
}
|
|
226
|
+
if (field !== "" || record.length > 0) pushRecord();
|
|
227
|
+
const header = records[0];
|
|
228
|
+
if (header === void 0 || header.length === 1 && header[0] === "") throw new DatasetError("malformed", "dataset has no header row");
|
|
229
|
+
const columns = header.map((name) => name.trim());
|
|
230
|
+
if (columns.some((name) => name === "")) throw new DatasetError("malformed", "header contains an empty column name");
|
|
231
|
+
if (new Set(columns).size !== columns.length) throw new DatasetError("malformed", "header contains duplicate column names");
|
|
232
|
+
const rows = [];
|
|
233
|
+
for (let i = 1; i < records.length; i++) {
|
|
234
|
+
const cells = records[i];
|
|
235
|
+
if (cells.length > columns.length) throw new DatasetError("malformed", `row ${i + 1} has ${cells.length} fields but the header has ${columns.length}`);
|
|
236
|
+
const out = {};
|
|
237
|
+
for (let c = 0; c < columns.length; c++) {
|
|
238
|
+
const raw = cells[c];
|
|
239
|
+
out[columns[c]] = raw === void 0 || raw === "" ? null : raw;
|
|
240
|
+
}
|
|
241
|
+
rows.push(out);
|
|
242
|
+
}
|
|
243
|
+
return {
|
|
244
|
+
columns,
|
|
245
|
+
rows
|
|
246
|
+
};
|
|
247
|
+
}
|
|
248
|
+
/**
|
|
249
|
+
* Parse a JSON or JSONL text into a tabular dataset. JSON must be an array of
|
|
250
|
+
* flat objects; JSONL is one object per line. Non-scalar cell values are kept
|
|
251
|
+
* as-is (engines treat them as non-scalar).
|
|
252
|
+
* @param text - file text.
|
|
253
|
+
* @param extension - `.json` or `.jsonl`.
|
|
254
|
+
* @param config - resolved config (row cap).
|
|
255
|
+
* @param signal - optional abort signal.
|
|
256
|
+
* @returns the parsed table.
|
|
257
|
+
*/
|
|
258
|
+
function parseJsonTable(text, extension, config, signal) {
|
|
259
|
+
const values = extension === ".jsonl" ? parseJsonLines(text) : parseJsonDocument(text);
|
|
260
|
+
const items = Array.isArray(values) ? values : [values];
|
|
261
|
+
if (items.length > config.maxRows) throw new DatasetError("too-many-rows", `dataset exceeds the ${config.maxRows} row cap; use the sample parameter or raise maxRows`);
|
|
262
|
+
const rows = [];
|
|
263
|
+
const columns = [];
|
|
264
|
+
const seen = /* @__PURE__ */ new Set();
|
|
265
|
+
for (const [index, item] of items.entries()) {
|
|
266
|
+
if (index % 1024 === 0) throwIfAborted(signal);
|
|
267
|
+
if (typeof item !== "object" || item === null || Array.isArray(item)) throw new DatasetError("not-tabular", `row ${index + 1} is not a flat object; profiling/cleaning/verification need a tabular dataset (array of objects)`);
|
|
268
|
+
const row = item;
|
|
269
|
+
for (const key of Object.keys(row)) if (!seen.has(key)) {
|
|
270
|
+
seen.add(key);
|
|
271
|
+
columns.push(key);
|
|
272
|
+
}
|
|
273
|
+
rows.push(row);
|
|
274
|
+
}
|
|
275
|
+
return {
|
|
276
|
+
columns,
|
|
277
|
+
rows: rows.map((row) => normalizeRow(row, columns))
|
|
278
|
+
};
|
|
279
|
+
}
|
|
280
|
+
/** Fill absent columns with `null` so every row carries every column. */
|
|
281
|
+
function normalizeRow(row, columns) {
|
|
282
|
+
const out = {};
|
|
283
|
+
for (const column of columns) {
|
|
284
|
+
const value = row[column];
|
|
285
|
+
out[column] = value === void 0 ? null : value;
|
|
286
|
+
}
|
|
287
|
+
return out;
|
|
288
|
+
}
|
|
289
|
+
/** Parse one JSON document; failures become a loud malformed error. */
|
|
290
|
+
function parseJsonDocument(text) {
|
|
291
|
+
try {
|
|
292
|
+
return JSON.parse(text);
|
|
293
|
+
} catch (error) {
|
|
294
|
+
throw new DatasetError("malformed", `invalid JSON: ${error instanceof Error ? error.message : String(error)}`);
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
/** Parse JSONL (one JSON value per non-empty line); failures name the line. */
|
|
298
|
+
function parseJsonLines(text) {
|
|
299
|
+
const out = [];
|
|
300
|
+
const lines = text.split(/\r?\n/u);
|
|
301
|
+
for (const [index, line] of lines.entries()) {
|
|
302
|
+
if (line.trim() === "") continue;
|
|
303
|
+
try {
|
|
304
|
+
out.push(JSON.parse(line));
|
|
305
|
+
} catch {
|
|
306
|
+
throw new DatasetError("malformed", `invalid JSON on line ${index + 1}`);
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
return out;
|
|
310
|
+
}
|
|
311
|
+
/**
|
|
312
|
+
* Load a tabular dataset from a file already resolved inside the workspace.
|
|
313
|
+
* @param absolutePath - normalized absolute dataset path.
|
|
314
|
+
* @param config - resolved config.
|
|
315
|
+
* @param signal - optional abort signal.
|
|
316
|
+
* @returns the parsed table.
|
|
317
|
+
*/
|
|
318
|
+
async function loadTable(absolutePath, config, signal) {
|
|
319
|
+
const text = await readDatasetText(absolutePath, config, signal);
|
|
320
|
+
const ext = path.extname(absolutePath).toLowerCase();
|
|
321
|
+
throwIfAborted(signal);
|
|
322
|
+
if (ext === ".csv") return parseDelimited(text, ",", config, signal);
|
|
323
|
+
if (ext === ".tsv") return parseDelimited(text, " ", config, signal);
|
|
324
|
+
return parseJsonTable(text, ext, config, signal);
|
|
325
|
+
}
|
|
326
|
+
/**
|
|
327
|
+
* Load a dataset as a citation-checkable document root: CSV/TSV become
|
|
328
|
+
* `{ columns, rows }` (so `rows[3].nav` resolves), JSON is the parsed value,
|
|
329
|
+
* JSONL is the array of parsed lines.
|
|
330
|
+
* @param absolutePath - normalized absolute dataset path.
|
|
331
|
+
* @param config - resolved config.
|
|
332
|
+
* @param signal - optional abort signal.
|
|
333
|
+
* @returns the document root.
|
|
334
|
+
*/
|
|
335
|
+
async function loadDocument(absolutePath, config, signal) {
|
|
336
|
+
const ext = path.extname(absolutePath).toLowerCase();
|
|
337
|
+
if (ext === ".json") return {
|
|
338
|
+
kind: "json",
|
|
339
|
+
value: parseJsonDocument(await readDatasetText(absolutePath, config, signal))
|
|
340
|
+
};
|
|
341
|
+
if (ext === ".jsonl") return {
|
|
342
|
+
kind: "json",
|
|
343
|
+
value: parseJsonLines(await readDatasetText(absolutePath, config, signal))
|
|
344
|
+
};
|
|
345
|
+
const table = await loadTable(absolutePath, config, signal);
|
|
346
|
+
return {
|
|
347
|
+
kind: "table",
|
|
348
|
+
columns: table.columns,
|
|
349
|
+
rows: table.rows
|
|
350
|
+
};
|
|
351
|
+
}
|
|
352
|
+
/**
|
|
353
|
+
* Deterministic systematic sample: every `ceil(total / count)`-th row from
|
|
354
|
+
* index 0. No clock, no RNG — the same input always yields the same sample.
|
|
355
|
+
* @param rows - full row list.
|
|
356
|
+
* @param count - requested sample size.
|
|
357
|
+
* @returns the sampled rows (identity when `count >= rows.length`).
|
|
358
|
+
*/
|
|
359
|
+
function sampleRows(rows, count) {
|
|
360
|
+
if (!Number.isSafeInteger(count) || count <= 0) throw new DatasetError("malformed", `sample must be a positive integer, got ${String(count)}`);
|
|
361
|
+
if (count >= rows.length) return rows;
|
|
362
|
+
const stride = Math.ceil(rows.length / count);
|
|
363
|
+
const out = [];
|
|
364
|
+
for (let i = 0; i < rows.length; i += stride) out.push(rows[i]);
|
|
365
|
+
return out;
|
|
366
|
+
}
|
|
367
|
+
/**
|
|
368
|
+
* Missingness: `null`, `undefined`, or a string empty after trimming. A
|
|
369
|
+
* whitespace-only cell counts as missing even before any `trim` clean rule.
|
|
370
|
+
* @param cell - the cell to test (`undefined` when the column is absent).
|
|
371
|
+
* @returns whether the cell is missing.
|
|
372
|
+
*/
|
|
373
|
+
function isMissing(cell) {
|
|
374
|
+
if (cell === null || cell === void 0) return true;
|
|
375
|
+
if (typeof cell === "string") return cell.trim() === "";
|
|
376
|
+
return false;
|
|
377
|
+
}
|
|
378
|
+
/** Strict full-string numeric parse (no thousands separators, no unit suffixes). */
|
|
379
|
+
function parseNumeric(cell) {
|
|
380
|
+
if (typeof cell === "number") return Number.isFinite(cell) ? cell : void 0;
|
|
381
|
+
if (typeof cell !== "string") return void 0;
|
|
382
|
+
const text = cell.trim();
|
|
383
|
+
if (text === "") return void 0;
|
|
384
|
+
if (!/^[+-]?(?:\d+\.?\d*|\.\d+)(?:[eE][+-]?\d+)?$/u.test(text)) return void 0;
|
|
385
|
+
const value = Number(text);
|
|
386
|
+
return Number.isFinite(value) ? value : void 0;
|
|
387
|
+
}
|
|
388
|
+
const DATE_PATTERNS = [
|
|
389
|
+
/^(\d{4})-(\d{1,2})-(\d{1,2})$/u,
|
|
390
|
+
/^(\d{4})\/(\d{1,2})\/(\d{1,2})$/u,
|
|
391
|
+
/^(\d{4})-(\d{1,2})-(\d{1,2})[ T](\d{1,2}):(\d{2})(?::(\d{2}))?(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?$/u
|
|
392
|
+
];
|
|
393
|
+
/**
|
|
394
|
+
* Deterministic date parse to epoch milliseconds. Accepts `YYYY-MM-DD`,
|
|
395
|
+
* `YYYY/MM/DD`, and ISO-like datetimes (date-only forms read as UTC midnight).
|
|
396
|
+
* Calendar-invalid dates (e.g. 2025-13-40) reject. Returns `undefined` when
|
|
397
|
+
* the cell is not a recognized date.
|
|
398
|
+
* @param cell - the cell to parse (`undefined` when the column is absent).
|
|
399
|
+
* @returns epoch milliseconds, or `undefined`.
|
|
400
|
+
*/
|
|
401
|
+
function parseDate(cell) {
|
|
402
|
+
if (typeof cell !== "string") return void 0;
|
|
403
|
+
const text = cell.trim();
|
|
404
|
+
for (const pattern of DATE_PATTERNS) {
|
|
405
|
+
const match = pattern.exec(text);
|
|
406
|
+
if (match === null) continue;
|
|
407
|
+
const year = Number(match[1]);
|
|
408
|
+
const month = Number(match[2]);
|
|
409
|
+
const day = Number(match[3]);
|
|
410
|
+
const hour = Number(match[4] ?? 0);
|
|
411
|
+
const minute = Number(match[5] ?? 0);
|
|
412
|
+
const second = Number(match[6] ?? 0);
|
|
413
|
+
if (month < 1 || month > 12 || day < 1 || day > 31 || hour > 23 || minute > 59 || second > 59) return void 0;
|
|
414
|
+
const epoch = Date.UTC(year, month - 1, day, hour, minute, second);
|
|
415
|
+
const check = new Date(epoch);
|
|
416
|
+
if (check.getUTCMonth() !== month - 1 || check.getUTCDate() !== day) return void 0;
|
|
417
|
+
return epoch;
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
/** Boolean parse: true/false/yes/no/1/0, case-insensitive. */
|
|
421
|
+
function parseBoolean(cell) {
|
|
422
|
+
if (typeof cell === "boolean") return cell;
|
|
423
|
+
if (typeof cell !== "string") return void 0;
|
|
424
|
+
const text = cell.trim().toLowerCase();
|
|
425
|
+
if (text === "true" || text === "yes" || text === "1") return true;
|
|
426
|
+
if (text === "false" || text === "no" || text === "0") return false;
|
|
427
|
+
}
|
|
428
|
+
//#endregion
|
|
429
|
+
//#region src/store.ts
|
|
430
|
+
/**
|
|
431
|
+
* Durable report storage over the harness storage domain. The `data_quality`
|
|
432
|
+
* domain keeps one record per profile/clean/verify/citations run, keyed by an
|
|
433
|
+
* injected timestamp plus a path fingerprint, so reports survive a restart
|
|
434
|
+
* and stay queryable without touching the session log.
|
|
435
|
+
* @module dsh-data-quality/store
|
|
436
|
+
*/
|
|
437
|
+
/** Zod schema of one persisted report record (durable-boundary validation). */
|
|
438
|
+
const reportRecordSchema = z$1.object({
|
|
439
|
+
kind: z$1.enum([
|
|
440
|
+
"profile",
|
|
441
|
+
"clean",
|
|
442
|
+
"verify",
|
|
443
|
+
"citations"
|
|
444
|
+
]),
|
|
445
|
+
at: z$1.number().int().nonnegative(),
|
|
446
|
+
dataset: z$1.string(),
|
|
447
|
+
report: z$1.record(z$1.string(), z$1.unknown())
|
|
448
|
+
});
|
|
449
|
+
/** The `dsh-data-quality` storage-domain declaration. */
|
|
450
|
+
const dataQualityDomainSpec = defineDomain({
|
|
451
|
+
name: "data_quality",
|
|
452
|
+
version: 1,
|
|
453
|
+
tables: { reports: domainTable(reportRecordSchema) }
|
|
454
|
+
});
|
|
455
|
+
/** Pad to two digits for the key timestamp. */
|
|
456
|
+
function pad2(value) {
|
|
457
|
+
return String(value).padStart(2, "0");
|
|
458
|
+
}
|
|
459
|
+
/**
|
|
460
|
+
* Build the storage key for one run: `YYYYMMDDHHmmssSSS-<kind>-<pathFp>`.
|
|
461
|
+
* UTC digits keep the key timezone-independent and filename-safe everywhere.
|
|
462
|
+
* @param record - the report envelope.
|
|
463
|
+
* @returns the deterministic key.
|
|
464
|
+
*/
|
|
465
|
+
function reportKeyOf(record) {
|
|
466
|
+
const date = new Date(record.at);
|
|
467
|
+
return `${`${date.getUTCFullYear()}${pad2(date.getUTCMonth() + 1)}${pad2(date.getUTCDate())}${pad2(date.getUTCHours())}${pad2(date.getUTCMinutes())}${pad2(date.getUTCSeconds())}${String(date.getUTCMilliseconds()).padStart(3, "0")}`}-${record.kind}-${pathFingerprint(record.dataset)}`;
|
|
468
|
+
}
|
|
469
|
+
//#endregion
|
|
470
|
+
//#region src/clean.ts
|
|
471
|
+
/**
|
|
472
|
+
* Declarative cleaning rules over a parsed {@link Table}: dedupe, fill-missing,
|
|
473
|
+
* coerce-type, normalize-unit, trim, and map-values. Pure and deterministic —
|
|
474
|
+
* rules apply in array order, each seeing the previous rule's output; every
|
|
475
|
+
* rule logs how many rows it affected. Unknown columns and invalid rule
|
|
476
|
+
* parameters fail loud.
|
|
477
|
+
* @module dsh-data-quality/clean
|
|
478
|
+
*/
|
|
479
|
+
/** Raised for invalid rules or unknown columns; `message` names the rule index and reason. */
|
|
480
|
+
var CleanRuleError = class extends Error {
|
|
481
|
+
ruleIndex;
|
|
482
|
+
/**
|
|
483
|
+
* @param ruleIndex - index of the offending rule in the request array.
|
|
484
|
+
* @param message - actionable human-readable detail.
|
|
485
|
+
*/
|
|
486
|
+
constructor(ruleIndex, message) {
|
|
487
|
+
super(message);
|
|
488
|
+
this.ruleIndex = ruleIndex;
|
|
489
|
+
this.name = "CleanRuleError";
|
|
490
|
+
}
|
|
491
|
+
};
|
|
492
|
+
/** Assert `column` exists; every rule validates its columns before touching data. */
|
|
493
|
+
function requireColumn$1(columns, column, ruleIndex) {
|
|
494
|
+
if (!columns.includes(column)) throw new CleanRuleError(ruleIndex, `rule ${ruleIndex}: unknown column ${JSON.stringify(column)} (columns: ${columns.join(", ")})`);
|
|
495
|
+
}
|
|
496
|
+
/** Assert every selected column exists. */
|
|
497
|
+
function requireColumns(columns, selected, ruleIndex) {
|
|
498
|
+
for (const column of selected) requireColumn$1(columns, column, ruleIndex);
|
|
499
|
+
}
|
|
500
|
+
/** dedupe: keep the first row of each key group; later duplicates are removed. */
|
|
501
|
+
function applyDedupe(table, rule, ruleIndex, signal) {
|
|
502
|
+
const selected = rule.columns ?? table.columns;
|
|
503
|
+
requireColumns(table.columns, selected, ruleIndex);
|
|
504
|
+
const seen = /* @__PURE__ */ new Set();
|
|
505
|
+
const rows = [];
|
|
506
|
+
let removed = 0;
|
|
507
|
+
for (const [index, row] of table.rows.entries()) {
|
|
508
|
+
if (index % 1024 === 0) throwIfAborted(signal);
|
|
509
|
+
const key = JSON.stringify(selected.map((column) => row[column] ?? null));
|
|
510
|
+
if (seen.has(key)) {
|
|
511
|
+
removed += 1;
|
|
512
|
+
continue;
|
|
513
|
+
}
|
|
514
|
+
seen.add(key);
|
|
515
|
+
rows.push(row);
|
|
516
|
+
}
|
|
517
|
+
return {
|
|
518
|
+
table: {
|
|
519
|
+
columns: table.columns,
|
|
520
|
+
rows
|
|
521
|
+
},
|
|
522
|
+
log: {
|
|
523
|
+
ruleIndex,
|
|
524
|
+
rule: "dedupe",
|
|
525
|
+
affectedRows: removed,
|
|
526
|
+
detail: `removed ${removed} duplicate row(s) by [${selected.join(", ")}]`
|
|
527
|
+
}
|
|
528
|
+
};
|
|
529
|
+
}
|
|
530
|
+
/** fill-missing: constant / mean / median / forward fill of missing cells in one column. */
|
|
531
|
+
function applyFillMissing(table, rule, ruleIndex, signal) {
|
|
532
|
+
requireColumn$1(table.columns, rule.column, ruleIndex);
|
|
533
|
+
let fill;
|
|
534
|
+
let detail;
|
|
535
|
+
switch (rule.strategy) {
|
|
536
|
+
case "constant":
|
|
537
|
+
if (rule.value === void 0) throw new CleanRuleError(ruleIndex, `rule ${ruleIndex}: fill-missing strategy "constant" requires a value`);
|
|
538
|
+
fill = rule.value;
|
|
539
|
+
detail = `filled with constant ${JSON.stringify(rule.value)}`;
|
|
540
|
+
break;
|
|
541
|
+
case "mean":
|
|
542
|
+
case "median": {
|
|
543
|
+
const values = table.rows.map((row) => parseNumeric(row[rule.column])).filter((value) => value !== void 0).sort((a, b) => a - b);
|
|
544
|
+
if (values.length === 0) throw new CleanRuleError(ruleIndex, `rule ${ruleIndex}: fill-missing strategy "${rule.strategy}" needs at least one numeric value in column ${JSON.stringify(rule.column)}`);
|
|
545
|
+
const computed = rule.strategy === "mean" ? values.reduce((acc, value) => acc + value, 0) / values.length : values.length % 2 === 1 ? values[(values.length - 1) / 2] : (values[values.length / 2 - 1] + values[values.length / 2]) / 2;
|
|
546
|
+
fill = computed;
|
|
547
|
+
detail = `filled with ${rule.strategy} ${computed}`;
|
|
548
|
+
break;
|
|
549
|
+
}
|
|
550
|
+
case "forward":
|
|
551
|
+
fill = void 0;
|
|
552
|
+
detail = "filled with the previous non-missing value";
|
|
553
|
+
}
|
|
554
|
+
let filled = 0;
|
|
555
|
+
let previous;
|
|
556
|
+
const rows = table.rows.map((row, index) => {
|
|
557
|
+
if (index % 1024 === 0) throwIfAborted(signal);
|
|
558
|
+
const cell = row[rule.column];
|
|
559
|
+
if (!isMissing(cell)) {
|
|
560
|
+
previous = cell ?? null;
|
|
561
|
+
return row;
|
|
562
|
+
}
|
|
563
|
+
const replacement = rule.strategy === "forward" ? previous : fill;
|
|
564
|
+
if (replacement === void 0) return row;
|
|
565
|
+
filled += 1;
|
|
566
|
+
return {
|
|
567
|
+
...row,
|
|
568
|
+
[rule.column]: replacement
|
|
569
|
+
};
|
|
570
|
+
});
|
|
571
|
+
return {
|
|
572
|
+
table: {
|
|
573
|
+
columns: table.columns,
|
|
574
|
+
rows
|
|
575
|
+
},
|
|
576
|
+
log: {
|
|
577
|
+
ruleIndex,
|
|
578
|
+
rule: "fill-missing",
|
|
579
|
+
affectedRows: filled,
|
|
580
|
+
detail: `${detail}; ${filled} row(s) filled`
|
|
581
|
+
}
|
|
582
|
+
};
|
|
583
|
+
}
|
|
584
|
+
/** coerce-type: rewrite one column to number / ISO date / boolean; failures become missing and are counted. */
|
|
585
|
+
function applyCoerceType(table, rule, ruleIndex, signal) {
|
|
586
|
+
requireColumn$1(table.columns, rule.column, ruleIndex);
|
|
587
|
+
let coerced = 0;
|
|
588
|
+
let failed = 0;
|
|
589
|
+
const failureExamples = [];
|
|
590
|
+
const rows = table.rows.map((row, index) => {
|
|
591
|
+
if (index % 1024 === 0) throwIfAborted(signal);
|
|
592
|
+
const cell = row[rule.column];
|
|
593
|
+
if (isMissing(cell)) return row;
|
|
594
|
+
let value;
|
|
595
|
+
switch (rule.to) {
|
|
596
|
+
case "number": {
|
|
597
|
+
const numeric = parseNumeric(cell);
|
|
598
|
+
value = numeric === void 0 ? void 0 : numeric;
|
|
599
|
+
break;
|
|
600
|
+
}
|
|
601
|
+
case "date": {
|
|
602
|
+
const epoch = parseDate(cell);
|
|
603
|
+
if (epoch === void 0) value = void 0;
|
|
604
|
+
else {
|
|
605
|
+
const date = new Date(epoch);
|
|
606
|
+
value = date.getUTCHours() === 0 && date.getUTCMinutes() === 0 && date.getUTCSeconds() === 0 ? date.toISOString().slice(0, 10) : date.toISOString();
|
|
607
|
+
}
|
|
608
|
+
break;
|
|
609
|
+
}
|
|
610
|
+
case "boolean": {
|
|
611
|
+
const bool = parseBoolean(cell);
|
|
612
|
+
value = bool === void 0 ? void 0 : String(bool);
|
|
613
|
+
break;
|
|
614
|
+
}
|
|
615
|
+
}
|
|
616
|
+
if (value === void 0) {
|
|
617
|
+
failed += 1;
|
|
618
|
+
if (failureExamples.length < 3) failureExamples.push(JSON.stringify(cell));
|
|
619
|
+
return {
|
|
620
|
+
...row,
|
|
621
|
+
[rule.column]: null
|
|
622
|
+
};
|
|
623
|
+
}
|
|
624
|
+
coerced += 1;
|
|
625
|
+
return {
|
|
626
|
+
...row,
|
|
627
|
+
[rule.column]: value
|
|
628
|
+
};
|
|
629
|
+
});
|
|
630
|
+
const detail = `${coerced} value(s) coerced to ${rule.to}` + (failed > 0 ? `; ${failed} failure(s) set to missing (e.g. ${failureExamples.join(", ")})` : "");
|
|
631
|
+
return {
|
|
632
|
+
table: {
|
|
633
|
+
columns: table.columns,
|
|
634
|
+
rows
|
|
635
|
+
},
|
|
636
|
+
log: {
|
|
637
|
+
ruleIndex,
|
|
638
|
+
rule: "coerce-type",
|
|
639
|
+
affectedRows: coerced,
|
|
640
|
+
detail
|
|
641
|
+
}
|
|
642
|
+
};
|
|
643
|
+
}
|
|
644
|
+
/** normalize-unit: strip a unit suffix and multiply by its factor (e.g. 万 -> 1e4). */
|
|
645
|
+
function applyNormalizeUnit(table, rule, ruleIndex, signal) {
|
|
646
|
+
requireColumn$1(table.columns, rule.column, ruleIndex);
|
|
647
|
+
const entries = Object.entries(rule.factors);
|
|
648
|
+
if (entries.length === 0) throw new CleanRuleError(ruleIndex, `rule ${ruleIndex}: normalize-unit requires a non-empty factors map`);
|
|
649
|
+
for (const [suffix, factor] of entries) {
|
|
650
|
+
if (suffix === "") throw new CleanRuleError(ruleIndex, `rule ${ruleIndex}: normalize-unit suffixes must not be empty`);
|
|
651
|
+
if (!Number.isFinite(factor) || factor <= 0) throw new CleanRuleError(ruleIndex, `rule ${ruleIndex}: factor for ${JSON.stringify(suffix)} must be a positive finite number, got ${String(factor)}`);
|
|
652
|
+
}
|
|
653
|
+
const ordered = [...entries].sort((a, b) => b[0].length - a[0].length);
|
|
654
|
+
let normalized = 0;
|
|
655
|
+
let skipped = 0;
|
|
656
|
+
const rows = table.rows.map((row, index) => {
|
|
657
|
+
if (index % 1024 === 0) throwIfAborted(signal);
|
|
658
|
+
const cell = row[rule.column];
|
|
659
|
+
if (isMissing(cell)) return row;
|
|
660
|
+
const plain = parseNumeric(cell);
|
|
661
|
+
if (plain !== void 0) {
|
|
662
|
+
normalized += 1;
|
|
663
|
+
return {
|
|
664
|
+
...row,
|
|
665
|
+
[rule.column]: plain
|
|
666
|
+
};
|
|
667
|
+
}
|
|
668
|
+
if (typeof cell === "string") {
|
|
669
|
+
const text = cell.trim();
|
|
670
|
+
for (const [suffix, factor] of ordered) {
|
|
671
|
+
if (!text.endsWith(suffix)) continue;
|
|
672
|
+
const numeric = parseNumeric(text.slice(0, text.length - suffix.length));
|
|
673
|
+
if (numeric === void 0) continue;
|
|
674
|
+
normalized += 1;
|
|
675
|
+
return {
|
|
676
|
+
...row,
|
|
677
|
+
[rule.column]: numeric * factor
|
|
678
|
+
};
|
|
679
|
+
}
|
|
680
|
+
}
|
|
681
|
+
skipped += 1;
|
|
682
|
+
return row;
|
|
683
|
+
});
|
|
684
|
+
const detail = `${normalized} value(s) normalized to base units` + (skipped > 0 ? `; ${skipped} unparseable value(s) left unchanged` : "");
|
|
685
|
+
return {
|
|
686
|
+
table: {
|
|
687
|
+
columns: table.columns,
|
|
688
|
+
rows
|
|
689
|
+
},
|
|
690
|
+
log: {
|
|
691
|
+
ruleIndex,
|
|
692
|
+
rule: "normalize-unit",
|
|
693
|
+
affectedRows: normalized,
|
|
694
|
+
detail
|
|
695
|
+
}
|
|
696
|
+
};
|
|
697
|
+
}
|
|
698
|
+
/** trim: strip leading/trailing whitespace of string cells in the selected columns. */
|
|
699
|
+
function applyTrim(table, rule, ruleIndex, signal) {
|
|
700
|
+
const selected = rule.columns ?? table.columns;
|
|
701
|
+
requireColumns(table.columns, selected, ruleIndex);
|
|
702
|
+
let trimmed = 0;
|
|
703
|
+
const rows = table.rows.map((row, index) => {
|
|
704
|
+
if (index % 1024 === 0) throwIfAborted(signal);
|
|
705
|
+
let changed = false;
|
|
706
|
+
const next = { ...row };
|
|
707
|
+
for (const column of selected) {
|
|
708
|
+
const cell = next[column];
|
|
709
|
+
if (typeof cell !== "string") continue;
|
|
710
|
+
const value = cell.trim();
|
|
711
|
+
if (value !== cell) {
|
|
712
|
+
next[column] = value;
|
|
713
|
+
changed = true;
|
|
714
|
+
}
|
|
715
|
+
}
|
|
716
|
+
if (changed) trimmed += 1;
|
|
717
|
+
return changed ? next : row;
|
|
718
|
+
});
|
|
719
|
+
return {
|
|
720
|
+
table: {
|
|
721
|
+
columns: table.columns,
|
|
722
|
+
rows
|
|
723
|
+
},
|
|
724
|
+
log: {
|
|
725
|
+
ruleIndex,
|
|
726
|
+
rule: "trim",
|
|
727
|
+
affectedRows: trimmed,
|
|
728
|
+
detail: `trimmed whitespace in [${selected.join(", ")}] for ${trimmed} row(s)`
|
|
729
|
+
}
|
|
730
|
+
};
|
|
731
|
+
}
|
|
732
|
+
/** map-values: exact-match enum mapping; unmapped values follow the `else` policy. */
|
|
733
|
+
function applyMapValues(table, rule, ruleIndex, signal) {
|
|
734
|
+
requireColumn$1(table.columns, rule.column, ruleIndex);
|
|
735
|
+
const elsePolicy = rule.else ?? "keep";
|
|
736
|
+
let mapped = 0;
|
|
737
|
+
let unmapped = 0;
|
|
738
|
+
const rows = table.rows.map((row, index) => {
|
|
739
|
+
if (index % 1024 === 0) throwIfAborted(signal);
|
|
740
|
+
const cell = row[rule.column];
|
|
741
|
+
if (isMissing(cell)) return row;
|
|
742
|
+
const key = typeof cell === "string" ? cell : typeof cell === "object" && cell !== null ? JSON.stringify(cell) : String(cell);
|
|
743
|
+
const replacement = rule.map[key];
|
|
744
|
+
if (replacement !== void 0) {
|
|
745
|
+
mapped += 1;
|
|
746
|
+
return {
|
|
747
|
+
...row,
|
|
748
|
+
[rule.column]: replacement
|
|
749
|
+
};
|
|
750
|
+
}
|
|
751
|
+
if (elsePolicy === "missing") {
|
|
752
|
+
unmapped += 1;
|
|
753
|
+
return {
|
|
754
|
+
...row,
|
|
755
|
+
[rule.column]: null
|
|
756
|
+
};
|
|
757
|
+
}
|
|
758
|
+
unmapped += 1;
|
|
759
|
+
return row;
|
|
760
|
+
});
|
|
761
|
+
const detail = `${mapped} value(s) mapped` + (unmapped > 0 ? `; ${unmapped} unmapped value(s) ${elsePolicy === "missing" ? "set to missing" : "kept"}` : "");
|
|
762
|
+
return {
|
|
763
|
+
table: {
|
|
764
|
+
columns: table.columns,
|
|
765
|
+
rows
|
|
766
|
+
},
|
|
767
|
+
log: {
|
|
768
|
+
ruleIndex,
|
|
769
|
+
rule: "map-values",
|
|
770
|
+
affectedRows: mapped,
|
|
771
|
+
detail
|
|
772
|
+
}
|
|
773
|
+
};
|
|
774
|
+
}
|
|
775
|
+
/**
|
|
776
|
+
* Apply cleaning rules in array order; each rule sees the previous output.
|
|
777
|
+
* @param table - the parsed dataset.
|
|
778
|
+
* @param rules - non-empty rule list.
|
|
779
|
+
* @param options - abort signal.
|
|
780
|
+
* @returns the cleaned table plus the per-rule audit log.
|
|
781
|
+
*/
|
|
782
|
+
function applyCleanRules(table, rules, options = {}) {
|
|
783
|
+
if (rules.length === 0) throw new CleanRuleError(0, "rules must be a non-empty array");
|
|
784
|
+
let current = {
|
|
785
|
+
columns: [...table.columns],
|
|
786
|
+
rows: table.rows.map((row) => ({ ...row }))
|
|
787
|
+
};
|
|
788
|
+
const logs = [];
|
|
789
|
+
for (const [ruleIndex, rule] of rules.entries()) {
|
|
790
|
+
throwIfAborted(options.signal);
|
|
791
|
+
let step;
|
|
792
|
+
switch (rule.rule) {
|
|
793
|
+
case "dedupe":
|
|
794
|
+
step = applyDedupe(current, rule, ruleIndex, options.signal);
|
|
795
|
+
break;
|
|
796
|
+
case "fill-missing":
|
|
797
|
+
step = applyFillMissing(current, rule, ruleIndex, options.signal);
|
|
798
|
+
break;
|
|
799
|
+
case "coerce-type":
|
|
800
|
+
step = applyCoerceType(current, rule, ruleIndex, options.signal);
|
|
801
|
+
break;
|
|
802
|
+
case "normalize-unit":
|
|
803
|
+
step = applyNormalizeUnit(current, rule, ruleIndex, options.signal);
|
|
804
|
+
break;
|
|
805
|
+
case "trim":
|
|
806
|
+
step = applyTrim(current, rule, ruleIndex, options.signal);
|
|
807
|
+
break;
|
|
808
|
+
case "map-values":
|
|
809
|
+
step = applyMapValues(current, rule, ruleIndex, options.signal);
|
|
810
|
+
break;
|
|
811
|
+
default: throw new CleanRuleError(ruleIndex, `rule ${ruleIndex}: unknown rule type ${JSON.stringify(rule.rule)}`);
|
|
812
|
+
}
|
|
813
|
+
current = step.table;
|
|
814
|
+
logs.push(step.log);
|
|
815
|
+
}
|
|
816
|
+
return {
|
|
817
|
+
columns: current.columns,
|
|
818
|
+
rows: current.rows,
|
|
819
|
+
logs,
|
|
820
|
+
inputRows: table.rows.length,
|
|
821
|
+
outputRows: current.rows.length
|
|
822
|
+
};
|
|
823
|
+
}
|
|
824
|
+
/**
|
|
825
|
+
* Serialize a table back to CSV/TSV text with RFC-4180 quoting (used when a
|
|
826
|
+
* clean run writes an output file). Numbers and booleans render via `String`;
|
|
827
|
+
* missing cells render empty.
|
|
828
|
+
* @param columns - column order.
|
|
829
|
+
* @param rows - rows to serialize.
|
|
830
|
+
* @param delimiter - field delimiter.
|
|
831
|
+
* @returns the file text (LF newlines, trailing newline).
|
|
832
|
+
*/
|
|
833
|
+
function serializeDelimited(columns, rows, delimiter) {
|
|
834
|
+
const encode = (cell) => {
|
|
835
|
+
if (isMissing(cell)) return "";
|
|
836
|
+
const text = typeof cell === "string" ? cell : typeof cell === "object" && cell !== null ? JSON.stringify(cell) : String(cell);
|
|
837
|
+
if (text.includes(delimiter) || text.includes("\"") || text.includes("\n") || text.includes("\r")) return `"${text.replace(/"/gu, "\"\"")}"`;
|
|
838
|
+
return text;
|
|
839
|
+
};
|
|
840
|
+
const lines = [columns.map((column) => encode(column)).join(delimiter)];
|
|
841
|
+
for (const row of rows) lines.push(columns.map((column) => encode(row[column])).join(delimiter));
|
|
842
|
+
return `${lines.join("\n")}\n`;
|
|
843
|
+
}
|
|
844
|
+
//#endregion
|
|
845
|
+
//#region src/profile.ts
|
|
846
|
+
/**
|
|
847
|
+
* Deterministic dataset profiling: per-column type inference, missingness,
|
|
848
|
+
* cardinality, numeric distribution, IQR outlier counts, and duplicate-row
|
|
849
|
+
* detection. Pure functions over a parsed {@link Table} — no clock, no RNG,
|
|
850
|
+
* no I/O; `generatedAt` is injected by the caller.
|
|
851
|
+
* @module dsh-data-quality/profile
|
|
852
|
+
*/
|
|
853
|
+
/** Round to 6 significant digits for stable, readable report numbers. */
|
|
854
|
+
function round6(value) {
|
|
855
|
+
return Number(value.toPrecision(6));
|
|
856
|
+
}
|
|
857
|
+
/** Linear-interpolation quantile over an ascending-sorted array. */
|
|
858
|
+
function quantile(sorted, q) {
|
|
859
|
+
const index = (sorted.length - 1) * q;
|
|
860
|
+
const low = Math.floor(index);
|
|
861
|
+
const high = Math.ceil(index);
|
|
862
|
+
const lower = sorted[low];
|
|
863
|
+
return lower + (sorted[high] - lower) * (index - low);
|
|
864
|
+
}
|
|
865
|
+
/** Compute the numeric distribution of already-parsed values. */
|
|
866
|
+
function numericProfile(values) {
|
|
867
|
+
if (values.length === 0) return void 0;
|
|
868
|
+
const sorted = [...values].sort((a, b) => a - b);
|
|
869
|
+
const sum = sorted.reduce((acc, value) => acc + value, 0);
|
|
870
|
+
const p25 = quantile(sorted, .25);
|
|
871
|
+
const p75 = quantile(sorted, .75);
|
|
872
|
+
const iqr = p75 - p25;
|
|
873
|
+
const lowFence = p25 - 1.5 * iqr;
|
|
874
|
+
const highFence = p75 + 1.5 * iqr;
|
|
875
|
+
const outliers = iqr === 0 ? 0 : sorted.filter((value) => value < lowFence || value > highFence).length;
|
|
876
|
+
return {
|
|
877
|
+
min: round6(sorted[0]),
|
|
878
|
+
max: round6(sorted[sorted.length - 1]),
|
|
879
|
+
mean: round6(sum / sorted.length),
|
|
880
|
+
median: round6(quantile(sorted, .5)),
|
|
881
|
+
p25: round6(p25),
|
|
882
|
+
p75: round6(p75),
|
|
883
|
+
outliers
|
|
884
|
+
};
|
|
885
|
+
}
|
|
886
|
+
/** Count rows whose full content duplicates an earlier row (first occurrence is not counted). */
|
|
887
|
+
function countDuplicateRows(table, signal) {
|
|
888
|
+
const seen = /* @__PURE__ */ new Set();
|
|
889
|
+
let duplicates = 0;
|
|
890
|
+
for (const [index, row] of table.rows.entries()) {
|
|
891
|
+
if (index % 1024 === 0) throwIfAborted(signal);
|
|
892
|
+
const key = JSON.stringify(table.columns.map((column) => row[column] ?? null));
|
|
893
|
+
if (seen.has(key)) duplicates += 1;
|
|
894
|
+
else seen.add(key);
|
|
895
|
+
}
|
|
896
|
+
return duplicates;
|
|
897
|
+
}
|
|
898
|
+
/** Profile one column over the given rows. */
|
|
899
|
+
function profileColumn(rows, column, signal) {
|
|
900
|
+
let missing = 0;
|
|
901
|
+
let numbers = 0;
|
|
902
|
+
let dates = 0;
|
|
903
|
+
let booleans = 0;
|
|
904
|
+
let strings = 0;
|
|
905
|
+
const numericValues = [];
|
|
906
|
+
const distinct = /* @__PURE__ */ new Set();
|
|
907
|
+
const frequencies = /* @__PURE__ */ new Map();
|
|
908
|
+
for (const [index, row] of rows.entries()) {
|
|
909
|
+
if (index % 1024 === 0) throwIfAborted(signal);
|
|
910
|
+
const cell = row[column];
|
|
911
|
+
if (isMissing(cell)) {
|
|
912
|
+
missing += 1;
|
|
913
|
+
continue;
|
|
914
|
+
}
|
|
915
|
+
const text = typeof cell === "string" ? cell : JSON.stringify(cell);
|
|
916
|
+
distinct.add(text);
|
|
917
|
+
frequencies.set(text, (frequencies.get(text) ?? 0) + 1);
|
|
918
|
+
const numeric = parseNumeric(cell);
|
|
919
|
+
if (numeric !== void 0) {
|
|
920
|
+
numbers += 1;
|
|
921
|
+
numericValues.push(numeric);
|
|
922
|
+
continue;
|
|
923
|
+
}
|
|
924
|
+
if (parseDate(cell) !== void 0) {
|
|
925
|
+
dates += 1;
|
|
926
|
+
continue;
|
|
927
|
+
}
|
|
928
|
+
if (parseBoolean(cell) !== void 0) {
|
|
929
|
+
booleans += 1;
|
|
930
|
+
continue;
|
|
931
|
+
}
|
|
932
|
+
strings += 1;
|
|
933
|
+
}
|
|
934
|
+
const present = rows.length - missing;
|
|
935
|
+
const notes = [];
|
|
936
|
+
let inferredType;
|
|
937
|
+
if (present === 0) inferredType = "empty";
|
|
938
|
+
else if (numbers === present) inferredType = "number";
|
|
939
|
+
else if (dates === present) inferredType = "date";
|
|
940
|
+
else if (booleans === present) inferredType = "boolean";
|
|
941
|
+
else if (strings === present) inferredType = "string";
|
|
942
|
+
else {
|
|
943
|
+
inferredType = "mixed";
|
|
944
|
+
const parts = [
|
|
945
|
+
numbers > 0 ? `${numbers} numeric` : void 0,
|
|
946
|
+
dates > 0 ? `${dates} date` : void 0,
|
|
947
|
+
booleans > 0 ? `${booleans} boolean` : void 0,
|
|
948
|
+
strings > 0 ? `${strings} string` : void 0
|
|
949
|
+
].filter((part) => part !== void 0);
|
|
950
|
+
notes.push(`mixed types among ${present} present values: ${parts.join(", ")}`);
|
|
951
|
+
}
|
|
952
|
+
const profile = {
|
|
953
|
+
name: column,
|
|
954
|
+
inferredType,
|
|
955
|
+
missing,
|
|
956
|
+
missingRate: rows.length === 0 ? 0 : round6(missing / rows.length),
|
|
957
|
+
unique: distinct.size,
|
|
958
|
+
notes
|
|
959
|
+
};
|
|
960
|
+
const numeric = inferredType === "number" ? numericProfile(numericValues) : void 0;
|
|
961
|
+
const topValues = (inferredType === "string" || inferredType === "boolean") && distinct.size > 0 && distinct.size <= 10 ? [...frequencies.entries()].sort((a, b) => b[1] - a[1] || (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0)).slice(0, 5).map(([value, count]) => ({
|
|
962
|
+
value,
|
|
963
|
+
count
|
|
964
|
+
})) : void 0;
|
|
965
|
+
return {
|
|
966
|
+
...profile,
|
|
967
|
+
...numeric !== void 0 ? { numeric } : {},
|
|
968
|
+
...topValues !== void 0 ? { topValues } : {}
|
|
969
|
+
};
|
|
970
|
+
}
|
|
971
|
+
/**
|
|
972
|
+
* Profile a parsed table. Column cards are computed over a deterministic
|
|
973
|
+
* systematic sample when `sample` is given; row counts and duplicate counts
|
|
974
|
+
* always cover the full table.
|
|
975
|
+
* @param table - the parsed dataset.
|
|
976
|
+
* @param options - dataset label, optional sample size, injected timestamp, abort signal.
|
|
977
|
+
* @returns the profile report.
|
|
978
|
+
*/
|
|
979
|
+
function profileTable(table, options) {
|
|
980
|
+
throwIfAborted(options.signal);
|
|
981
|
+
const profiled = options.sample === void 0 ? table.rows : sampleRows(table.rows, options.sample);
|
|
982
|
+
const columns = table.columns.map((column) => profileColumn(profiled, column, options.signal));
|
|
983
|
+
const duplicateRows = countDuplicateRows(table, options.signal);
|
|
984
|
+
return {
|
|
985
|
+
dataset: options.dataset,
|
|
986
|
+
rowCount: table.rows.length,
|
|
987
|
+
sampled: profiled.length !== table.rows.length,
|
|
988
|
+
profiledRows: profiled.length,
|
|
989
|
+
columnCount: table.columns.length,
|
|
990
|
+
duplicateRows,
|
|
991
|
+
columns,
|
|
992
|
+
generatedAt: options.generatedAt
|
|
993
|
+
};
|
|
994
|
+
}
|
|
995
|
+
/** Human-readable profile summary for the tool's Native render. */
|
|
996
|
+
function renderProfileText(report) {
|
|
997
|
+
const lines = [];
|
|
998
|
+
lines.push(`Profile of ${report.dataset}: ${report.rowCount} rows x ${report.columnCount} columns` + (report.sampled ? ` (column cards over a systematic sample of ${report.profiledRows} rows)` : ""));
|
|
999
|
+
if (report.duplicateRows > 0) lines.push(`Duplicate rows: ${report.duplicateRows}`);
|
|
1000
|
+
for (const column of report.columns) {
|
|
1001
|
+
const parts = [`${column.name}: ${column.inferredType}`];
|
|
1002
|
+
if (column.missing > 0) parts.push(`missing ${column.missing} (${(column.missingRate * 100).toFixed(1)}%)`);
|
|
1003
|
+
parts.push(`unique ${column.unique}`);
|
|
1004
|
+
if (column.numeric !== void 0) parts.push(`min ${column.numeric.min}, p25 ${column.numeric.p25}, median ${column.numeric.median}, p75 ${column.numeric.p75}, max ${column.numeric.max}, mean ${column.numeric.mean}` + (column.numeric.outliers > 0 ? `, ${column.numeric.outliers} IQR outliers` : ""));
|
|
1005
|
+
if (column.topValues !== void 0) parts.push(`top: ${column.topValues.map((entry) => `${JSON.stringify(entry.value)} x${entry.count}`).join(", ")}`);
|
|
1006
|
+
for (const note of column.notes) parts.push(`note: ${note}`);
|
|
1007
|
+
lines.push(`- ${parts.join("; ")}`);
|
|
1008
|
+
}
|
|
1009
|
+
return lines.join("\n");
|
|
1010
|
+
}
|
|
1011
|
+
//#endregion
|
|
1012
|
+
//#region src/verify.ts
|
|
1013
|
+
/**
|
|
1014
|
+
* Declarative verification rules over a parsed {@link Table} (not-null,
|
|
1015
|
+
* unique, range, regex, enum, cross-column, freshness) plus the frozen
|
|
1016
|
+
* citation-checking semantics for `ctx.dataQuality.verifyCitations`. Pure and
|
|
1017
|
+
* deterministic: the freshness clock is injected, numeric comparison never
|
|
1018
|
+
* uses raw `===`, and evidence rows are capped by config.
|
|
1019
|
+
* @module dsh-data-quality/verify
|
|
1020
|
+
*/
|
|
1021
|
+
/** Raised for invalid rules; `message` names the rule index and reason. */
|
|
1022
|
+
var VerifyRuleError = class extends Error {
|
|
1023
|
+
ruleIndex;
|
|
1024
|
+
/**
|
|
1025
|
+
* @param ruleIndex - index of the offending rule in the request array.
|
|
1026
|
+
* @param message - actionable human-readable detail.
|
|
1027
|
+
*/
|
|
1028
|
+
constructor(ruleIndex, message) {
|
|
1029
|
+
super(message);
|
|
1030
|
+
this.ruleIndex = ruleIndex;
|
|
1031
|
+
this.name = "VerifyRuleError";
|
|
1032
|
+
}
|
|
1033
|
+
};
|
|
1034
|
+
/** Assert `column` exists. */
|
|
1035
|
+
function requireColumn(columns, column, ruleIndex) {
|
|
1036
|
+
if (!columns.includes(column)) throw new VerifyRuleError(ruleIndex, `rule ${ruleIndex}: unknown column ${JSON.stringify(column)} (columns: ${columns.join(", ")})`);
|
|
1037
|
+
}
|
|
1038
|
+
/** The trimmed string form used for regex/enum/cross-column string comparison. */
|
|
1039
|
+
function textOf(cell) {
|
|
1040
|
+
if (typeof cell === "string") return cell.trim();
|
|
1041
|
+
if (cell === null || cell === void 0) return "";
|
|
1042
|
+
if (typeof cell === "object") return JSON.stringify(cell);
|
|
1043
|
+
return String(cell);
|
|
1044
|
+
}
|
|
1045
|
+
/** Run one per-row predicate and collect capped evidence. */
|
|
1046
|
+
function evaluate(table, ruleIndex, ruleName, columns, predicate, evidenceRowLimit, signal) {
|
|
1047
|
+
let failedCount = 0;
|
|
1048
|
+
const evidence = [];
|
|
1049
|
+
for (const [rowIndex, row] of table.rows.entries()) {
|
|
1050
|
+
if (rowIndex % 1024 === 0) throwIfAborted(signal);
|
|
1051
|
+
if (predicate(row)) continue;
|
|
1052
|
+
failedCount += 1;
|
|
1053
|
+
if (evidence.length < evidenceRowLimit) {
|
|
1054
|
+
const values = {};
|
|
1055
|
+
for (const column of columns) values[column] = row[column] ?? null;
|
|
1056
|
+
evidence.push({
|
|
1057
|
+
rowIndex,
|
|
1058
|
+
values
|
|
1059
|
+
});
|
|
1060
|
+
}
|
|
1061
|
+
}
|
|
1062
|
+
return {
|
|
1063
|
+
ruleIndex,
|
|
1064
|
+
rule: ruleName,
|
|
1065
|
+
passed: failedCount === 0,
|
|
1066
|
+
failedCount,
|
|
1067
|
+
checkedCount: table.rows.length,
|
|
1068
|
+
evidence
|
|
1069
|
+
};
|
|
1070
|
+
}
|
|
1071
|
+
const CROSS_OPS = [
|
|
1072
|
+
"<",
|
|
1073
|
+
"<=",
|
|
1074
|
+
"==",
|
|
1075
|
+
"!=",
|
|
1076
|
+
">=",
|
|
1077
|
+
">"
|
|
1078
|
+
];
|
|
1079
|
+
/**
|
|
1080
|
+
* Apply verification rules over a parsed table. A missing cell fails every
|
|
1081
|
+
* rule that reads it. The overall `passed` is the conjunction of rule passes;
|
|
1082
|
+
* a failing dataset is a normal result, never a thrown error.
|
|
1083
|
+
* @param table - the parsed dataset.
|
|
1084
|
+
* @param rules - non-empty rule list.
|
|
1085
|
+
* @param options - evidence cap, injected clock for `freshness`, abort signal.
|
|
1086
|
+
* @returns the verify report (without the dataset label; the caller adds it).
|
|
1087
|
+
*/
|
|
1088
|
+
function verifyTable(table, rules, options) {
|
|
1089
|
+
if (rules.length === 0) throw new VerifyRuleError(0, "rules must be a non-empty array");
|
|
1090
|
+
const results = [];
|
|
1091
|
+
for (const [ruleIndex, rule] of rules.entries()) {
|
|
1092
|
+
throwIfAborted(options.signal);
|
|
1093
|
+
let result;
|
|
1094
|
+
switch (rule.rule) {
|
|
1095
|
+
case "not-null":
|
|
1096
|
+
requireColumn(table.columns, rule.column, ruleIndex);
|
|
1097
|
+
result = evaluate(table, ruleIndex, rule.rule, [rule.column], (row) => !isMissing(row[rule.column]), options.evidenceRowLimit, options.signal);
|
|
1098
|
+
break;
|
|
1099
|
+
case "unique": {
|
|
1100
|
+
if (rule.columns.length === 0) throw new VerifyRuleError(ruleIndex, `rule ${ruleIndex}: unique requires at least one column`);
|
|
1101
|
+
for (const column of rule.columns) requireColumn(table.columns, column, ruleIndex);
|
|
1102
|
+
const counts = /* @__PURE__ */ new Map();
|
|
1103
|
+
for (const row of table.rows) {
|
|
1104
|
+
const key = JSON.stringify(rule.columns.map((column) => row[column] ?? null));
|
|
1105
|
+
counts.set(key, (counts.get(key) ?? 0) + 1);
|
|
1106
|
+
}
|
|
1107
|
+
result = evaluate(table, ruleIndex, rule.rule, rule.columns, (row) => (counts.get(JSON.stringify(rule.columns.map((column) => row[column] ?? null))) ?? 0) === 1, options.evidenceRowLimit, options.signal);
|
|
1108
|
+
break;
|
|
1109
|
+
}
|
|
1110
|
+
case "range":
|
|
1111
|
+
requireColumn(table.columns, rule.column, ruleIndex);
|
|
1112
|
+
if (rule.min === void 0 && rule.max === void 0) throw new VerifyRuleError(ruleIndex, `rule ${ruleIndex}: range requires at least one of min/max`);
|
|
1113
|
+
result = evaluate(table, ruleIndex, rule.rule, [rule.column], (row) => {
|
|
1114
|
+
const value = parseNumeric(row[rule.column]);
|
|
1115
|
+
if (value === void 0) return false;
|
|
1116
|
+
if (rule.min !== void 0 && value < rule.min) return false;
|
|
1117
|
+
if (rule.max !== void 0 && value > rule.max) return false;
|
|
1118
|
+
return true;
|
|
1119
|
+
}, options.evidenceRowLimit, options.signal);
|
|
1120
|
+
break;
|
|
1121
|
+
case "regex": {
|
|
1122
|
+
requireColumn(table.columns, rule.column, ruleIndex);
|
|
1123
|
+
let pattern;
|
|
1124
|
+
try {
|
|
1125
|
+
pattern = new RegExp(rule.pattern, rule.flags ?? "");
|
|
1126
|
+
} catch (error) {
|
|
1127
|
+
throw new VerifyRuleError(ruleIndex, `rule ${ruleIndex}: invalid regex ${JSON.stringify(rule.pattern)}: ${error instanceof Error ? error.message : String(error)}`);
|
|
1128
|
+
}
|
|
1129
|
+
result = evaluate(table, ruleIndex, rule.rule, [rule.column], (row) => {
|
|
1130
|
+
const cell = row[rule.column];
|
|
1131
|
+
if (isMissing(cell)) return false;
|
|
1132
|
+
pattern.lastIndex = 0;
|
|
1133
|
+
return pattern.test(textOf(cell));
|
|
1134
|
+
}, options.evidenceRowLimit, options.signal);
|
|
1135
|
+
break;
|
|
1136
|
+
}
|
|
1137
|
+
case "enum": {
|
|
1138
|
+
requireColumn(table.columns, rule.column, ruleIndex);
|
|
1139
|
+
if (rule.values.length === 0) throw new VerifyRuleError(ruleIndex, `rule ${ruleIndex}: enum requires a non-empty values array`);
|
|
1140
|
+
const allowed = new Set(rule.values);
|
|
1141
|
+
result = evaluate(table, ruleIndex, rule.rule, [rule.column], (row) => {
|
|
1142
|
+
const cell = row[rule.column];
|
|
1143
|
+
if (isMissing(cell)) return false;
|
|
1144
|
+
return allowed.has(textOf(cell));
|
|
1145
|
+
}, options.evidenceRowLimit, options.signal);
|
|
1146
|
+
break;
|
|
1147
|
+
}
|
|
1148
|
+
case "cross-column": {
|
|
1149
|
+
requireColumn(table.columns, rule.left, ruleIndex);
|
|
1150
|
+
if (!CROSS_OPS.includes(rule.op)) throw new VerifyRuleError(ruleIndex, `rule ${ruleIndex}: unsupported op ${JSON.stringify(rule.op)}`);
|
|
1151
|
+
const hasColumn = rule.rightColumn !== void 0;
|
|
1152
|
+
if (hasColumn === (rule.value !== void 0)) throw new VerifyRuleError(ruleIndex, `rule ${ruleIndex}: cross-column needs exactly one of rightColumn/value`);
|
|
1153
|
+
if (hasColumn) requireColumn(table.columns, rule.rightColumn, ruleIndex);
|
|
1154
|
+
const columns = hasColumn ? [rule.left, rule.rightColumn] : [rule.left];
|
|
1155
|
+
result = evaluate(table, ruleIndex, rule.rule, columns, (row) => {
|
|
1156
|
+
const left = row[rule.left];
|
|
1157
|
+
const right = hasColumn ? row[rule.rightColumn] : rule.value;
|
|
1158
|
+
if (isMissing(left) || isMissing(right)) return false;
|
|
1159
|
+
const leftNum = parseNumeric(left);
|
|
1160
|
+
const rightNum = parseNumeric(right);
|
|
1161
|
+
if (leftNum !== void 0 && rightNum !== void 0) switch (rule.op) {
|
|
1162
|
+
case "<": return leftNum < rightNum;
|
|
1163
|
+
case "<=": return leftNum <= rightNum;
|
|
1164
|
+
case ">": return leftNum > rightNum;
|
|
1165
|
+
case ">=": return leftNum >= rightNum;
|
|
1166
|
+
case "==": return Math.abs(leftNum - rightNum) <= 1e-12 * Math.max(Math.abs(leftNum), Math.abs(rightNum), 1);
|
|
1167
|
+
case "!=": return Math.abs(leftNum - rightNum) > 1e-12 * Math.max(Math.abs(leftNum), Math.abs(rightNum), 1);
|
|
1168
|
+
}
|
|
1169
|
+
const leftDate = parseDate(left);
|
|
1170
|
+
const rightDate = parseDate(right);
|
|
1171
|
+
if (leftDate !== void 0 && rightDate !== void 0) switch (rule.op) {
|
|
1172
|
+
case "<": return leftDate < rightDate;
|
|
1173
|
+
case "<=": return leftDate <= rightDate;
|
|
1174
|
+
case ">": return leftDate > rightDate;
|
|
1175
|
+
case ">=": return leftDate >= rightDate;
|
|
1176
|
+
case "==": return leftDate === rightDate;
|
|
1177
|
+
case "!=": return leftDate !== rightDate;
|
|
1178
|
+
}
|
|
1179
|
+
const leftText = textOf(left);
|
|
1180
|
+
const rightText = textOf(right);
|
|
1181
|
+
if (rule.op === "==") return leftText === rightText;
|
|
1182
|
+
if (rule.op === "!=") return leftText !== rightText;
|
|
1183
|
+
return false;
|
|
1184
|
+
}, options.evidenceRowLimit, options.signal);
|
|
1185
|
+
break;
|
|
1186
|
+
}
|
|
1187
|
+
case "freshness": {
|
|
1188
|
+
requireColumn(table.columns, rule.column, ruleIndex);
|
|
1189
|
+
if (!Number.isFinite(rule.maxAgeDays) || rule.maxAgeDays <= 0) throw new VerifyRuleError(ruleIndex, `rule ${ruleIndex}: freshness maxAgeDays must be a positive number, got ${String(rule.maxAgeDays)}`);
|
|
1190
|
+
let asOf;
|
|
1191
|
+
if (rule.asOf !== void 0) {
|
|
1192
|
+
const parsed = parseDate(rule.asOf);
|
|
1193
|
+
if (parsed === void 0) throw new VerifyRuleError(ruleIndex, `rule ${ruleIndex}: freshness asOf is not a recognized date: ${JSON.stringify(rule.asOf)}`);
|
|
1194
|
+
asOf = parsed;
|
|
1195
|
+
} else asOf = options.now();
|
|
1196
|
+
const maxAgeMs = rule.maxAgeDays * 864e5;
|
|
1197
|
+
result = evaluate(table, ruleIndex, rule.rule, [rule.column], (row) => {
|
|
1198
|
+
const epoch = parseDate(row[rule.column]);
|
|
1199
|
+
if (epoch === void 0) return false;
|
|
1200
|
+
return asOf - epoch <= maxAgeMs;
|
|
1201
|
+
}, options.evidenceRowLimit, options.signal);
|
|
1202
|
+
break;
|
|
1203
|
+
}
|
|
1204
|
+
default: throw new VerifyRuleError(ruleIndex, `rule ${ruleIndex}: unknown rule type ${JSON.stringify(rule.rule)}`);
|
|
1205
|
+
}
|
|
1206
|
+
results.push(result);
|
|
1207
|
+
}
|
|
1208
|
+
return {
|
|
1209
|
+
passed: results.every((result) => result.passed),
|
|
1210
|
+
rowCount: table.rows.length,
|
|
1211
|
+
rules: results,
|
|
1212
|
+
generatedAt: options.now()
|
|
1213
|
+
};
|
|
1214
|
+
}
|
|
1215
|
+
/** Human-readable verify summary for the tool's Native render. */
|
|
1216
|
+
function renderVerifyText(report) {
|
|
1217
|
+
const lines = [];
|
|
1218
|
+
lines.push(`Verification of ${report.dataset}: ${report.passed ? "PASSED" : "FAILED"} (${report.rowCount} rows, ${report.rules.length} rules)`);
|
|
1219
|
+
for (const rule of report.rules) {
|
|
1220
|
+
lines.push(`- [${rule.passed ? "pass" : "FAIL"}] rule ${rule.ruleIndex} (${rule.rule}): ${rule.failedCount}/${rule.checkedCount} row(s) failed`);
|
|
1221
|
+
for (const row of rule.evidence) {
|
|
1222
|
+
const cells = Object.entries(row.values).map(([column, value]) => `${column}=${JSON.stringify(value)}`).join(", ");
|
|
1223
|
+
lines.push(` row ${row.rowIndex}: ${cells}`);
|
|
1224
|
+
}
|
|
1225
|
+
if (rule.failedCount > rule.evidence.length) lines.push(` … and ${rule.failedCount - rule.evidence.length} more failing row(s)`);
|
|
1226
|
+
}
|
|
1227
|
+
return lines.join("\n");
|
|
1228
|
+
}
|
|
1229
|
+
/**
|
|
1230
|
+
* Parse a JSON-path-ish locator (`rows[3].nav`, `summary.annualReturn`,
|
|
1231
|
+
* `[0].value`) into walk segments. Throws on unparsable input.
|
|
1232
|
+
* @param locator - the citation's path string.
|
|
1233
|
+
* @returns ordered walk segments.
|
|
1234
|
+
*/
|
|
1235
|
+
function parseLocator(locator) {
|
|
1236
|
+
if (locator.trim() === "") throw new Error("citation path must not be empty");
|
|
1237
|
+
const segments = [];
|
|
1238
|
+
for (const part of locator.split(".")) {
|
|
1239
|
+
const match = /^([A-Za-z0-9_$-]*)((?:\[\d+\])*)$/u.exec(part);
|
|
1240
|
+
if (match === null || match[1] === "" && match[2] === "") throw new Error(`unparsable citation path segment ${JSON.stringify(part)} in ${JSON.stringify(locator)}`);
|
|
1241
|
+
const indices = [...(match[2] ?? "").matchAll(/\[(\d+)\]/gu)].map((index) => Number(index[1]));
|
|
1242
|
+
segments.push({
|
|
1243
|
+
key: match[1] === "" ? void 0 : match[1],
|
|
1244
|
+
indices
|
|
1245
|
+
});
|
|
1246
|
+
}
|
|
1247
|
+
return segments;
|
|
1248
|
+
}
|
|
1249
|
+
/** Sentinel for a failed locator walk (distinguishable from an actual `undefined` value). */
|
|
1250
|
+
const NOT_FOUND = Symbol("not-found");
|
|
1251
|
+
/** Walk the document root along the parsed locator; returns {@link NOT_FOUND} when any step misses. */
|
|
1252
|
+
function walkLocator(root, segments) {
|
|
1253
|
+
let current = root.kind === "table" ? {
|
|
1254
|
+
columns: root.columns,
|
|
1255
|
+
rows: root.rows
|
|
1256
|
+
} : root.value;
|
|
1257
|
+
for (const segment of segments) {
|
|
1258
|
+
if (segment.key !== void 0) {
|
|
1259
|
+
if (typeof current !== "object" || current === null || Array.isArray(current)) return NOT_FOUND;
|
|
1260
|
+
if (!Object.prototype.hasOwnProperty.call(current, segment.key)) return NOT_FOUND;
|
|
1261
|
+
current = current[segment.key];
|
|
1262
|
+
}
|
|
1263
|
+
for (const index of segment.indices) {
|
|
1264
|
+
if (!Array.isArray(current) || index >= current.length) return NOT_FOUND;
|
|
1265
|
+
current = current[index];
|
|
1266
|
+
}
|
|
1267
|
+
}
|
|
1268
|
+
return current;
|
|
1269
|
+
}
|
|
1270
|
+
/** Relative-tolerance numeric comparison (never raw `===` on floats). */
|
|
1271
|
+
function numericClose(actual, cited, tolerance) {
|
|
1272
|
+
return Math.abs(actual - cited) <= tolerance * Math.max(Math.abs(actual), Math.abs(cited), Number.MIN_VALUE);
|
|
1273
|
+
}
|
|
1274
|
+
/** Display form of a located value for evidence notes (`undefined` when it cannot be rendered). */
|
|
1275
|
+
function displayActual(located) {
|
|
1276
|
+
if (typeof located === "string") return located;
|
|
1277
|
+
if (typeof located === "number" || typeof located === "boolean") return String(located);
|
|
1278
|
+
if (located === null) return "null";
|
|
1279
|
+
const rendered = JSON.stringify(located);
|
|
1280
|
+
return typeof rendered === "string" ? rendered : void 0;
|
|
1281
|
+
}
|
|
1282
|
+
/**
|
|
1283
|
+
* The frozen citation-checking semantics: locate each citation's `path` in the
|
|
1284
|
+
* dataset document, then compare. Numbers compare with relative `tolerance`
|
|
1285
|
+
* (falling back to `defaultTolerance`); strings compare exactly. A missing
|
|
1286
|
+
* location is `not-found`; an incomparable actual/cited type pair is
|
|
1287
|
+
* `unverifiable`.
|
|
1288
|
+
* @param root - the loaded dataset document.
|
|
1289
|
+
* @param citations - citations to check.
|
|
1290
|
+
* @param defaultTolerance - configured fallback relative tolerance.
|
|
1291
|
+
* @returns one result per citation, ids echoed.
|
|
1292
|
+
*/
|
|
1293
|
+
function checkCitations(root, citations, defaultTolerance) {
|
|
1294
|
+
const results = [];
|
|
1295
|
+
for (const citation of citations) {
|
|
1296
|
+
let located;
|
|
1297
|
+
try {
|
|
1298
|
+
located = walkLocator(root, parseLocator(citation.path));
|
|
1299
|
+
} catch (error) {
|
|
1300
|
+
results.push({
|
|
1301
|
+
id: citation.id,
|
|
1302
|
+
status: "not-found",
|
|
1303
|
+
note: `path ${JSON.stringify(citation.path)} is not a valid locator: ${error instanceof Error ? error.message : String(error)}`
|
|
1304
|
+
});
|
|
1305
|
+
continue;
|
|
1306
|
+
}
|
|
1307
|
+
if (located === NOT_FOUND) {
|
|
1308
|
+
results.push({
|
|
1309
|
+
id: citation.id,
|
|
1310
|
+
status: "not-found",
|
|
1311
|
+
note: `nothing found at ${JSON.stringify(citation.path)}`
|
|
1312
|
+
});
|
|
1313
|
+
continue;
|
|
1314
|
+
}
|
|
1315
|
+
if (typeof citation.value === "number") {
|
|
1316
|
+
const actualNumber = typeof located === "number" ? located : parseNumeric(located);
|
|
1317
|
+
if (actualNumber === void 0) {
|
|
1318
|
+
const display = displayActual(located);
|
|
1319
|
+
results.push({
|
|
1320
|
+
id: citation.id,
|
|
1321
|
+
status: "unverifiable",
|
|
1322
|
+
...display !== void 0 ? { actual: display } : {},
|
|
1323
|
+
note: `value at ${JSON.stringify(citation.path)} is not numeric and cannot be compared to ${citation.value}`
|
|
1324
|
+
});
|
|
1325
|
+
continue;
|
|
1326
|
+
}
|
|
1327
|
+
const tolerance = citation.tolerance ?? defaultTolerance;
|
|
1328
|
+
if (numericClose(actualNumber, citation.value, tolerance)) results.push({
|
|
1329
|
+
id: citation.id,
|
|
1330
|
+
status: "verified",
|
|
1331
|
+
actual: actualNumber,
|
|
1332
|
+
note: `within relative tolerance ${tolerance}`
|
|
1333
|
+
});
|
|
1334
|
+
else results.push({
|
|
1335
|
+
id: citation.id,
|
|
1336
|
+
status: "mismatch",
|
|
1337
|
+
actual: actualNumber,
|
|
1338
|
+
note: `cited ${citation.value} differs from actual ${actualNumber} beyond relative tolerance ${tolerance}`
|
|
1339
|
+
});
|
|
1340
|
+
continue;
|
|
1341
|
+
}
|
|
1342
|
+
if (typeof located === "string") {
|
|
1343
|
+
if (located === citation.value) results.push({
|
|
1344
|
+
id: citation.id,
|
|
1345
|
+
status: "verified",
|
|
1346
|
+
actual: located
|
|
1347
|
+
});
|
|
1348
|
+
else results.push({
|
|
1349
|
+
id: citation.id,
|
|
1350
|
+
status: "mismatch",
|
|
1351
|
+
actual: located,
|
|
1352
|
+
note: `cited ${JSON.stringify(citation.value)} != actual ${JSON.stringify(located)}`
|
|
1353
|
+
});
|
|
1354
|
+
continue;
|
|
1355
|
+
}
|
|
1356
|
+
const display = displayActual(located);
|
|
1357
|
+
results.push({
|
|
1358
|
+
id: citation.id,
|
|
1359
|
+
status: "unverifiable",
|
|
1360
|
+
...display !== void 0 ? { actual: display } : {},
|
|
1361
|
+
note: `value at ${JSON.stringify(citation.path)} is ${typeof located}, not comparable to cited string`
|
|
1362
|
+
});
|
|
1363
|
+
}
|
|
1364
|
+
return { results };
|
|
1365
|
+
}
|
|
1366
|
+
//#endregion
|
|
1367
|
+
//#region src/events.ts
|
|
1368
|
+
/**
|
|
1369
|
+
* The `data-quality/*` session-event vocabulary and its adaptive append gate.
|
|
1370
|
+
* The vocabulary is declared on `SessionEventMap` for type-checked payloads;
|
|
1371
|
+
* the runtime gate decides per host whether appending is safe:
|
|
1372
|
+
*
|
|
1373
|
+
* - Hosts whose known-type set already covers the vocabulary (a future harness
|
|
1374
|
+
* that adopts these events) append plainly.
|
|
1375
|
+
* - Hosts with an `ignorable` append option (the master-build
|
|
1376
|
+
* `Session.append(type, data, { ignorable: true })` contract) append with the
|
|
1377
|
+
* marker, so builds that do not know the type skip the event on restore.
|
|
1378
|
+
* - 0.1.0-rc.6 hosts have neither a registration surface nor the `ignorable`
|
|
1379
|
+
* append flag; appending an unknown type there would make the persistence
|
|
1380
|
+
* coordinator refuse the session log on restore, so the append is skipped
|
|
1381
|
+
* and the storage-domain report remains the durable copy.
|
|
1382
|
+
* @module dsh-data-quality/events
|
|
1383
|
+
*/
|
|
1384
|
+
/** The event type vocabulary this plugin appends. */
|
|
1385
|
+
const DATA_QUALITY_EVENT_TYPES = [
|
|
1386
|
+
"data-quality/profile",
|
|
1387
|
+
"data-quality/clean",
|
|
1388
|
+
"data-quality/verify"
|
|
1389
|
+
];
|
|
1390
|
+
/**
|
|
1391
|
+
* Append one `data-quality/*` event when the host can carry it safely; skip
|
|
1392
|
+
* silently otherwise (the storage-domain report is always the durable copy).
|
|
1393
|
+
* The `ignorable` probe reads the UNBOUND method's source (a `.bind()` result
|
|
1394
|
+
* reports `[native code]`): the rc.6 build contains no `ignorable` handling
|
|
1395
|
+
* while the master build references the flag by name; property names survive
|
|
1396
|
+
* minification, so the probe fails safe (skips) rather than corrupting a log.
|
|
1397
|
+
* @param session - the calling session.
|
|
1398
|
+
* @param type - the event type.
|
|
1399
|
+
* @param data - the payload.
|
|
1400
|
+
*/
|
|
1401
|
+
function appendDataQualityEvent(session, type, data) {
|
|
1402
|
+
if (KNOWN_SESSION_EVENT_TYPES.has(type)) {
|
|
1403
|
+
session.append(type, data);
|
|
1404
|
+
return;
|
|
1405
|
+
}
|
|
1406
|
+
const append = session.append;
|
|
1407
|
+
if (Function.prototype.toString.call(append).includes("ignorable")) append.call(session, type, data, { ignorable: true });
|
|
1408
|
+
}
|
|
1409
|
+
//#endregion
|
|
1410
|
+
//#region src/present.ts
|
|
1411
|
+
/** Maximum characters one cell contributes to a tool-facing row payload. */
|
|
1412
|
+
const MAX_CELL_TEXT = 120;
|
|
1413
|
+
/**
|
|
1414
|
+
* Truncate one cell for display: long strings are cut with an ellipsis
|
|
1415
|
+
* marker; non-strings pass through.
|
|
1416
|
+
* @param cell - the cell to bound.
|
|
1417
|
+
* @returns the display-safe cell.
|
|
1418
|
+
*/
|
|
1419
|
+
function truncateCell(cell) {
|
|
1420
|
+
if (typeof cell === "string" && cell.length > 120) return `${cell.slice(0, 120)}…`;
|
|
1421
|
+
return cell;
|
|
1422
|
+
}
|
|
1423
|
+
/**
|
|
1424
|
+
* Truncate every string cell of one row.
|
|
1425
|
+
* @param row - the row to bound.
|
|
1426
|
+
* @returns a display-safe copy.
|
|
1427
|
+
*/
|
|
1428
|
+
function truncateRow(row) {
|
|
1429
|
+
const out = {};
|
|
1430
|
+
for (const [key, value] of Object.entries(row)) out[key] = truncateCell(value);
|
|
1431
|
+
return out;
|
|
1432
|
+
}
|
|
1433
|
+
//#endregion
|
|
1434
|
+
//#region src/service.ts
|
|
1435
|
+
/**
|
|
1436
|
+
* Service Definition of the `dsh-data-quality` capability seam: the
|
|
1437
|
+
* `ctx.dataQuality` surface other plugins may optionally consume, plus the
|
|
1438
|
+
* internal request/report types the local Provider and the tool Consumers
|
|
1439
|
+
* share. The {@link CitationCheckRequest} / {@link CitationCheckResult}
|
|
1440
|
+
* citation contract is FROZEN for cross-plugin consumers — change it only
|
|
1441
|
+
* with a coordinated ecosystem migration.
|
|
1442
|
+
* @module dsh-data-quality/service
|
|
1443
|
+
*/
|
|
1444
|
+
/**
|
|
1445
|
+
* The `ctx.dataQuality` service. The local Provider implements deterministic
|
|
1446
|
+
* TypeScript computation; Consumers (the three tools and third-party plugins)
|
|
1447
|
+
* never re-implement the math.
|
|
1448
|
+
*/
|
|
1449
|
+
var DataQualityService = class extends Service {
|
|
1450
|
+
/**
|
|
1451
|
+
* Register as `dataQuality` on the context.
|
|
1452
|
+
* @param ctx - the plugin context.
|
|
1453
|
+
*/
|
|
1454
|
+
constructor(ctx) {
|
|
1455
|
+
super(ctx, "dataQuality");
|
|
1456
|
+
}
|
|
1457
|
+
};
|
|
1458
|
+
//#endregion
|
|
1459
|
+
//#region src/provider-local.ts
|
|
1460
|
+
/**
|
|
1461
|
+
* The local deterministic Provider of the `ctx.dataQuality` seam: orchestrates
|
|
1462
|
+
* dataset loading, the pure engines, durable report persistence, and the
|
|
1463
|
+
* adaptive `data-quality/*` session events. All computation is TypeScript in
|
|
1464
|
+
* this process — no model arithmetic, no external processes.
|
|
1465
|
+
* @module dsh-data-quality/provider-local
|
|
1466
|
+
*/
|
|
1467
|
+
/**
|
|
1468
|
+
* The local deterministic `dataQuality` implementation. Mounted by the plugin
|
|
1469
|
+
* entry; third-party plugins consume it through `ctx.dataQuality`.
|
|
1470
|
+
*/
|
|
1471
|
+
var LocalDataQualityService = class extends DataQualityService {
|
|
1472
|
+
config;
|
|
1473
|
+
deps;
|
|
1474
|
+
/**
|
|
1475
|
+
* @param ctx - the plugin context.
|
|
1476
|
+
* @param config - the resolved plugin config.
|
|
1477
|
+
* @param deps - store handle plus the injected clock.
|
|
1478
|
+
*/
|
|
1479
|
+
constructor(ctx, config, deps) {
|
|
1480
|
+
super(ctx);
|
|
1481
|
+
this.config = config;
|
|
1482
|
+
this.deps = deps;
|
|
1483
|
+
}
|
|
1484
|
+
/** The absolute root for service-level calls without a session workspace. */
|
|
1485
|
+
serviceRoot() {
|
|
1486
|
+
return this.config.workspaceRoot !== "" ? path.resolve(this.config.workspaceRoot) : process.cwd();
|
|
1487
|
+
}
|
|
1488
|
+
/** Persist one report when persistence is on; returns the storage key. */
|
|
1489
|
+
async persist(kind, dataset, report) {
|
|
1490
|
+
const store = this.deps.store;
|
|
1491
|
+
if (store === void 0) return void 0;
|
|
1492
|
+
const record = {
|
|
1493
|
+
kind,
|
|
1494
|
+
at: this.deps.now(),
|
|
1495
|
+
dataset,
|
|
1496
|
+
report
|
|
1497
|
+
};
|
|
1498
|
+
const key = reportKeyOf(record);
|
|
1499
|
+
await store.put({ ...record });
|
|
1500
|
+
return key;
|
|
1501
|
+
}
|
|
1502
|
+
/** Emit the adaptive session event when the call carries a session. */
|
|
1503
|
+
emitEvent(session, kind, dataset, reportKey, summary) {
|
|
1504
|
+
if (session === void 0) return;
|
|
1505
|
+
appendDataQualityEvent(session, `data-quality/${kind}`, {
|
|
1506
|
+
kind,
|
|
1507
|
+
dataset,
|
|
1508
|
+
...reportKey !== void 0 ? { reportKey } : {},
|
|
1509
|
+
summary
|
|
1510
|
+
});
|
|
1511
|
+
}
|
|
1512
|
+
/** @inheritdoc DataQualityService.profileDataset */
|
|
1513
|
+
async profileDataset(request) {
|
|
1514
|
+
throwIfAborted(request.signal);
|
|
1515
|
+
const report = profileTable(await loadTable(resolveWorkspacePath(request.workspace, request.dataset, this.config), this.config, request.signal), {
|
|
1516
|
+
dataset: request.dataset,
|
|
1517
|
+
sample: request.sample,
|
|
1518
|
+
generatedAt: this.deps.now(),
|
|
1519
|
+
signal: request.signal
|
|
1520
|
+
});
|
|
1521
|
+
const reportKey = await this.persist("profile", request.dataset, report);
|
|
1522
|
+
this.emitEvent(request.session, "profile", request.dataset, reportKey, {
|
|
1523
|
+
rows: report.rowCount,
|
|
1524
|
+
columns: report.columnCount
|
|
1525
|
+
});
|
|
1526
|
+
return {
|
|
1527
|
+
...report,
|
|
1528
|
+
...reportKey !== void 0 ? { reportKey } : {}
|
|
1529
|
+
};
|
|
1530
|
+
}
|
|
1531
|
+
/** @inheritdoc DataQualityService.cleanDataset */
|
|
1532
|
+
async cleanDataset(request) {
|
|
1533
|
+
throwIfAborted(request.signal);
|
|
1534
|
+
const absolute = resolveWorkspacePath(request.workspace, request.dataset, this.config);
|
|
1535
|
+
const result = applyCleanRules(await loadTable(absolute, this.config, request.signal), request.rules, { signal: request.signal });
|
|
1536
|
+
let writtenPath;
|
|
1537
|
+
if (request.outputPath !== void 0) {
|
|
1538
|
+
const outputAbsolute = resolveWorkspacePath(request.workspace, request.outputPath, this.config);
|
|
1539
|
+
if (outputAbsolute === absolute) throw new Error(`outputPath ${JSON.stringify(request.outputPath)} would overwrite the input dataset; choose a different path`);
|
|
1540
|
+
const ext = path.extname(outputAbsolute).toLowerCase();
|
|
1541
|
+
const text = ext === ".csv" || ext === ".tsv" ? serializeDelimited(result.columns, result.rows, ext === ".csv" ? "," : " ") : ext === ".jsonl" ? `${result.rows.map((row) => JSON.stringify(row)).join("\n")}\n` : `${JSON.stringify(result.rows, null, 2)}\n`;
|
|
1542
|
+
await mkdir(path.dirname(outputAbsolute), { recursive: true });
|
|
1543
|
+
throwIfAborted(request.signal);
|
|
1544
|
+
await writeFile(outputAbsolute, text, "utf8");
|
|
1545
|
+
writtenPath = request.outputPath;
|
|
1546
|
+
}
|
|
1547
|
+
const generatedAt = this.deps.now();
|
|
1548
|
+
const reportKey = await this.persist("clean", request.dataset, {
|
|
1549
|
+
dataset: request.dataset,
|
|
1550
|
+
inputRows: result.inputRows,
|
|
1551
|
+
outputRows: result.outputRows,
|
|
1552
|
+
logs: result.logs,
|
|
1553
|
+
...writtenPath !== void 0 ? { outputPath: writtenPath } : {},
|
|
1554
|
+
generatedAt
|
|
1555
|
+
});
|
|
1556
|
+
this.emitEvent(request.session, "clean", request.dataset, reportKey, {
|
|
1557
|
+
rows: result.outputRows,
|
|
1558
|
+
columns: result.columns.length,
|
|
1559
|
+
rules: result.logs.length
|
|
1560
|
+
});
|
|
1561
|
+
const previewRows = result.rows.slice(0, this.config.evidenceRowLimit).map((row) => truncateRow(row));
|
|
1562
|
+
return {
|
|
1563
|
+
dataset: request.dataset,
|
|
1564
|
+
inputRows: result.inputRows,
|
|
1565
|
+
outputRows: result.outputRows,
|
|
1566
|
+
logs: result.logs,
|
|
1567
|
+
preview: {
|
|
1568
|
+
columns: result.columns,
|
|
1569
|
+
rows: previewRows
|
|
1570
|
+
},
|
|
1571
|
+
...writtenPath !== void 0 ? { outputPath: writtenPath } : {},
|
|
1572
|
+
...reportKey !== void 0 ? { reportKey } : {},
|
|
1573
|
+
generatedAt
|
|
1574
|
+
};
|
|
1575
|
+
}
|
|
1576
|
+
/** @inheritdoc DataQualityService.verifyDataset */
|
|
1577
|
+
async verifyDataset(request) {
|
|
1578
|
+
throwIfAborted(request.signal);
|
|
1579
|
+
const outcome = verifyTable(await loadTable(resolveWorkspacePath(request.workspace, request.dataset, this.config), this.config, request.signal), request.rules, {
|
|
1580
|
+
evidenceRowLimit: this.config.evidenceRowLimit,
|
|
1581
|
+
now: this.deps.now,
|
|
1582
|
+
signal: request.signal
|
|
1583
|
+
});
|
|
1584
|
+
const report = {
|
|
1585
|
+
dataset: request.dataset,
|
|
1586
|
+
...outcome
|
|
1587
|
+
};
|
|
1588
|
+
const failedRules = report.rules.filter((rule) => !rule.passed).length;
|
|
1589
|
+
const reportKey = await this.persist("verify", request.dataset, report);
|
|
1590
|
+
this.emitEvent(request.session, "verify", request.dataset, reportKey, {
|
|
1591
|
+
rows: report.rowCount,
|
|
1592
|
+
rules: report.rules.length,
|
|
1593
|
+
failedRules,
|
|
1594
|
+
passed: report.passed
|
|
1595
|
+
});
|
|
1596
|
+
return {
|
|
1597
|
+
...report,
|
|
1598
|
+
...reportKey !== void 0 ? { reportKey } : {}
|
|
1599
|
+
};
|
|
1600
|
+
}
|
|
1601
|
+
/** @inheritdoc DataQualityService.verifyCitations */
|
|
1602
|
+
async verifyCitations(request) {
|
|
1603
|
+
const result = checkCitations(await loadDocument(resolveWorkspacePath(this.serviceRoot(), request.dataset, this.config), this.config), request.citations, this.config.defaultTolerance);
|
|
1604
|
+
const verified = result.results.filter((entry) => entry.status === "verified").length;
|
|
1605
|
+
const mismatched = result.results.filter((entry) => entry.status === "mismatch").length;
|
|
1606
|
+
const notFound = result.results.filter((entry) => entry.status === "not-found").length;
|
|
1607
|
+
const unverifiable = result.results.filter((entry) => entry.status === "unverifiable").length;
|
|
1608
|
+
await this.persist("citations", request.dataset, {
|
|
1609
|
+
dataset: request.dataset,
|
|
1610
|
+
checked: result.results.length,
|
|
1611
|
+
verified,
|
|
1612
|
+
mismatched,
|
|
1613
|
+
notFound,
|
|
1614
|
+
unverifiable,
|
|
1615
|
+
results: result.results,
|
|
1616
|
+
generatedAt: this.deps.now()
|
|
1617
|
+
});
|
|
1618
|
+
return result;
|
|
1619
|
+
}
|
|
1620
|
+
};
|
|
1621
|
+
//#endregion
|
|
1622
|
+
//#region src/tools/shared.ts
|
|
1623
|
+
/**
|
|
1624
|
+
* Shared helpers for the three data-quality model tools: session workspace
|
|
1625
|
+
* resolution and a compact cleaning-run text render. Kept presentation-only;
|
|
1626
|
+
* computation lives in the engines and the provider.
|
|
1627
|
+
* @module dsh-data-quality/tools-shared
|
|
1628
|
+
*/
|
|
1629
|
+
/**
|
|
1630
|
+
* The absolute workspace root a tool call resolves dataset paths against: the
|
|
1631
|
+
* calling agent's per-session cwd (mirroring the official fs tools). Non-agent
|
|
1632
|
+
* calls fail loud — there is no honest workspace to confine paths to.
|
|
1633
|
+
* @param exec - the tool-execution context.
|
|
1634
|
+
* @returns the absolute workspace root.
|
|
1635
|
+
*/
|
|
1636
|
+
function workspaceOf(exec) {
|
|
1637
|
+
const cwd = exec.agent?.session.header.cwd;
|
|
1638
|
+
if (cwd === void 0) throw new Error("data-quality tools require an agent-owned session workspace");
|
|
1639
|
+
return path.resolve(cwd);
|
|
1640
|
+
}
|
|
1641
|
+
/** Human-readable cleaning summary for the tool's Native render. */
|
|
1642
|
+
function renderCleanText(report) {
|
|
1643
|
+
const lines = [];
|
|
1644
|
+
lines.push(`Cleaned ${report.dataset}: ${report.inputRows} -> ${report.outputRows} rows over ${report.logs.length} rule(s)`);
|
|
1645
|
+
for (const log of report.logs) lines.push(`- rule ${log.ruleIndex} (${log.rule}): ${log.affectedRows} row(s) affected; ${log.detail}`);
|
|
1646
|
+
if (report.outputPath !== void 0) lines.push(`Wrote cleaned dataset to ${report.outputPath}`);
|
|
1647
|
+
else lines.push("No outputPath given: the source file was left untouched; preview below.");
|
|
1648
|
+
if (report.preview.rows.length > 0) {
|
|
1649
|
+
lines.push(`Preview (first ${report.preview.rows.length} row(s)):`);
|
|
1650
|
+
for (const row of report.preview.rows) {
|
|
1651
|
+
const cells = report.preview.columns.map((column) => `${column}=${JSON.stringify(row[column] ?? null)}`).join(", ");
|
|
1652
|
+
lines.push(` ${cells}`);
|
|
1653
|
+
}
|
|
1654
|
+
}
|
|
1655
|
+
return lines.join("\n");
|
|
1656
|
+
}
|
|
1657
|
+
//#endregion
|
|
1658
|
+
//#region src/tools/profile.ts
|
|
1659
|
+
/**
|
|
1660
|
+
* The `data_profile` model tool: deterministic dataset profiling through
|
|
1661
|
+
* `ctx.dataQuality.profileDataset` — never model arithmetic.
|
|
1662
|
+
* @module dsh-data-quality/tools/profile
|
|
1663
|
+
*/
|
|
1664
|
+
const COLUMN_PROFILE_SCHEMA = {
|
|
1665
|
+
type: "object",
|
|
1666
|
+
properties: {
|
|
1667
|
+
name: {
|
|
1668
|
+
type: "string",
|
|
1669
|
+
required: true
|
|
1670
|
+
},
|
|
1671
|
+
inferredType: {
|
|
1672
|
+
type: "string",
|
|
1673
|
+
enum: [
|
|
1674
|
+
"number",
|
|
1675
|
+
"date",
|
|
1676
|
+
"boolean",
|
|
1677
|
+
"string",
|
|
1678
|
+
"empty",
|
|
1679
|
+
"mixed"
|
|
1680
|
+
],
|
|
1681
|
+
required: true
|
|
1682
|
+
},
|
|
1683
|
+
missing: {
|
|
1684
|
+
type: "number",
|
|
1685
|
+
required: true
|
|
1686
|
+
},
|
|
1687
|
+
missingRate: {
|
|
1688
|
+
type: "number",
|
|
1689
|
+
required: true
|
|
1690
|
+
},
|
|
1691
|
+
unique: {
|
|
1692
|
+
type: "number",
|
|
1693
|
+
required: true
|
|
1694
|
+
},
|
|
1695
|
+
numeric: {
|
|
1696
|
+
type: "object",
|
|
1697
|
+
properties: {
|
|
1698
|
+
min: {
|
|
1699
|
+
type: "number",
|
|
1700
|
+
required: true
|
|
1701
|
+
},
|
|
1702
|
+
max: {
|
|
1703
|
+
type: "number",
|
|
1704
|
+
required: true
|
|
1705
|
+
},
|
|
1706
|
+
mean: {
|
|
1707
|
+
type: "number",
|
|
1708
|
+
required: true
|
|
1709
|
+
},
|
|
1710
|
+
median: {
|
|
1711
|
+
type: "number",
|
|
1712
|
+
required: true
|
|
1713
|
+
},
|
|
1714
|
+
p25: {
|
|
1715
|
+
type: "number",
|
|
1716
|
+
required: true
|
|
1717
|
+
},
|
|
1718
|
+
p75: {
|
|
1719
|
+
type: "number",
|
|
1720
|
+
required: true
|
|
1721
|
+
},
|
|
1722
|
+
outliers: {
|
|
1723
|
+
type: "number",
|
|
1724
|
+
required: true
|
|
1725
|
+
}
|
|
1726
|
+
},
|
|
1727
|
+
additionalProperties: false
|
|
1728
|
+
},
|
|
1729
|
+
topValues: {
|
|
1730
|
+
type: "array",
|
|
1731
|
+
items: {
|
|
1732
|
+
type: "object",
|
|
1733
|
+
properties: {
|
|
1734
|
+
value: {
|
|
1735
|
+
type: "string",
|
|
1736
|
+
required: true
|
|
1737
|
+
},
|
|
1738
|
+
count: {
|
|
1739
|
+
type: "number",
|
|
1740
|
+
required: true
|
|
1741
|
+
}
|
|
1742
|
+
},
|
|
1743
|
+
additionalProperties: false
|
|
1744
|
+
}
|
|
1745
|
+
},
|
|
1746
|
+
notes: {
|
|
1747
|
+
type: "array",
|
|
1748
|
+
items: { type: "string" },
|
|
1749
|
+
required: true
|
|
1750
|
+
}
|
|
1751
|
+
},
|
|
1752
|
+
additionalProperties: false
|
|
1753
|
+
};
|
|
1754
|
+
/**
|
|
1755
|
+
* Build the `data_profile` tool definition against a mounted service.
|
|
1756
|
+
* @param service - the mounted ctx.dataQuality implementation.
|
|
1757
|
+
* @returns the tool definition to register.
|
|
1758
|
+
*/
|
|
1759
|
+
function defineProfileTool(service) {
|
|
1760
|
+
return defineTool({
|
|
1761
|
+
name: "data_profile",
|
|
1762
|
+
description: [
|
|
1763
|
+
"Profile a workspace CSV/TSV/JSON/JSONL dataset with deterministic TypeScript computation (no mental math).",
|
|
1764
|
+
"Returns row/column counts, inferred column types, missing rates, unique counts, numeric distributions (min/max/mean/median/p25/p75), IQR outlier counts, mixed-type suspicion notes, and duplicate-row counts.",
|
|
1765
|
+
"Column cards cover every row by default; pass sample for a deterministic systematic sample on large files. Datasets above the configured row/size caps are rejected — use sample or raise the caps. The full report persists to the data_quality storage domain (reportKey in the result)."
|
|
1766
|
+
].join("\n"),
|
|
1767
|
+
parameters: {
|
|
1768
|
+
path: {
|
|
1769
|
+
type: "string",
|
|
1770
|
+
required: true,
|
|
1771
|
+
description: "Workspace-relative dataset path (.csv/.tsv/.json/.jsonl). JSON datasets must be an array of flat objects."
|
|
1772
|
+
},
|
|
1773
|
+
sample: {
|
|
1774
|
+
type: "number",
|
|
1775
|
+
description: "Optional systematic sample size (every ceil(N/sample)-th row) for the column cards; row counts stay exact."
|
|
1776
|
+
}
|
|
1777
|
+
},
|
|
1778
|
+
output: {
|
|
1779
|
+
schema: {
|
|
1780
|
+
type: "object",
|
|
1781
|
+
properties: {
|
|
1782
|
+
dataset: {
|
|
1783
|
+
type: "string",
|
|
1784
|
+
required: true
|
|
1785
|
+
},
|
|
1786
|
+
rowCount: {
|
|
1787
|
+
type: "number",
|
|
1788
|
+
required: true
|
|
1789
|
+
},
|
|
1790
|
+
sampled: {
|
|
1791
|
+
type: "boolean",
|
|
1792
|
+
required: true
|
|
1793
|
+
},
|
|
1794
|
+
profiledRows: {
|
|
1795
|
+
type: "number",
|
|
1796
|
+
required: true
|
|
1797
|
+
},
|
|
1798
|
+
columnCount: {
|
|
1799
|
+
type: "number",
|
|
1800
|
+
required: true
|
|
1801
|
+
},
|
|
1802
|
+
duplicateRows: {
|
|
1803
|
+
type: "number",
|
|
1804
|
+
required: true
|
|
1805
|
+
},
|
|
1806
|
+
generatedAt: {
|
|
1807
|
+
type: "number",
|
|
1808
|
+
required: true
|
|
1809
|
+
},
|
|
1810
|
+
reportKey: { type: "string" },
|
|
1811
|
+
columns: {
|
|
1812
|
+
type: "array",
|
|
1813
|
+
items: COLUMN_PROFILE_SCHEMA,
|
|
1814
|
+
required: true
|
|
1815
|
+
}
|
|
1816
|
+
},
|
|
1817
|
+
additionalProperties: false
|
|
1818
|
+
},
|
|
1819
|
+
render: (_args, value) => [{
|
|
1820
|
+
type: "text",
|
|
1821
|
+
text: renderProfileText(value)
|
|
1822
|
+
}]
|
|
1823
|
+
},
|
|
1824
|
+
async execute(args, exec) {
|
|
1825
|
+
return service.profileDataset({
|
|
1826
|
+
dataset: args.path,
|
|
1827
|
+
sample: args.sample,
|
|
1828
|
+
workspace: workspaceOf(exec),
|
|
1829
|
+
session: exec.agent?.session,
|
|
1830
|
+
signal: exec.signal
|
|
1831
|
+
});
|
|
1832
|
+
}
|
|
1833
|
+
});
|
|
1834
|
+
}
|
|
1835
|
+
//#endregion
|
|
1836
|
+
//#region src/tools/clean.ts
|
|
1837
|
+
/**
|
|
1838
|
+
* The `data_clean` model tool: declarative cleaning rules through
|
|
1839
|
+
* `ctx.dataQuality.cleanDataset`. The source file is never overwritten;
|
|
1840
|
+
* without `outputPath` the run is preview-only.
|
|
1841
|
+
* @module dsh-data-quality/tools/clean
|
|
1842
|
+
*/
|
|
1843
|
+
const STRING_MAP_SCHEMA = { type: "json" };
|
|
1844
|
+
const CLEAN_RULE_SCHEMA = {
|
|
1845
|
+
type: "array",
|
|
1846
|
+
items: { oneOf: [
|
|
1847
|
+
{
|
|
1848
|
+
type: "object",
|
|
1849
|
+
properties: {
|
|
1850
|
+
rule: {
|
|
1851
|
+
type: "string",
|
|
1852
|
+
const: "dedupe",
|
|
1853
|
+
required: true
|
|
1854
|
+
},
|
|
1855
|
+
columns: {
|
|
1856
|
+
type: "array",
|
|
1857
|
+
items: { type: "string" },
|
|
1858
|
+
description: "Key columns; all columns when omitted."
|
|
1859
|
+
}
|
|
1860
|
+
},
|
|
1861
|
+
additionalProperties: false,
|
|
1862
|
+
description: "Remove rows whose key-column values duplicate an earlier row (first occurrence kept)."
|
|
1863
|
+
},
|
|
1864
|
+
{
|
|
1865
|
+
type: "object",
|
|
1866
|
+
properties: {
|
|
1867
|
+
rule: {
|
|
1868
|
+
type: "string",
|
|
1869
|
+
const: "fill-missing",
|
|
1870
|
+
required: true
|
|
1871
|
+
},
|
|
1872
|
+
column: {
|
|
1873
|
+
type: "string",
|
|
1874
|
+
required: true
|
|
1875
|
+
},
|
|
1876
|
+
strategy: {
|
|
1877
|
+
type: "string",
|
|
1878
|
+
enum: [
|
|
1879
|
+
"constant",
|
|
1880
|
+
"mean",
|
|
1881
|
+
"median",
|
|
1882
|
+
"forward"
|
|
1883
|
+
],
|
|
1884
|
+
required: true
|
|
1885
|
+
},
|
|
1886
|
+
value: {
|
|
1887
|
+
oneOf: [{ type: "string" }, { type: "number" }],
|
|
1888
|
+
description: "Required for strategy \"constant\"."
|
|
1889
|
+
}
|
|
1890
|
+
},
|
|
1891
|
+
additionalProperties: false,
|
|
1892
|
+
description: "Fill missing cells: a constant, the column mean/median (numeric columns), or the previous non-missing value."
|
|
1893
|
+
},
|
|
1894
|
+
{
|
|
1895
|
+
type: "object",
|
|
1896
|
+
properties: {
|
|
1897
|
+
rule: {
|
|
1898
|
+
type: "string",
|
|
1899
|
+
const: "coerce-type",
|
|
1900
|
+
required: true
|
|
1901
|
+
},
|
|
1902
|
+
column: {
|
|
1903
|
+
type: "string",
|
|
1904
|
+
required: true
|
|
1905
|
+
},
|
|
1906
|
+
to: {
|
|
1907
|
+
type: "string",
|
|
1908
|
+
enum: [
|
|
1909
|
+
"number",
|
|
1910
|
+
"date",
|
|
1911
|
+
"boolean"
|
|
1912
|
+
],
|
|
1913
|
+
required: true
|
|
1914
|
+
}
|
|
1915
|
+
},
|
|
1916
|
+
additionalProperties: false,
|
|
1917
|
+
description: "Coerce cells to number / ISO date / boolean; failures become missing and are counted in the log."
|
|
1918
|
+
},
|
|
1919
|
+
{
|
|
1920
|
+
type: "object",
|
|
1921
|
+
properties: {
|
|
1922
|
+
rule: {
|
|
1923
|
+
type: "string",
|
|
1924
|
+
const: "normalize-unit",
|
|
1925
|
+
required: true
|
|
1926
|
+
},
|
|
1927
|
+
column: {
|
|
1928
|
+
type: "string",
|
|
1929
|
+
required: true
|
|
1930
|
+
},
|
|
1931
|
+
factors: {
|
|
1932
|
+
...STRING_MAP_SCHEMA,
|
|
1933
|
+
description: "Unit suffix to multiplier, e.g. {\"万\": 10000, \"亿\": 100000000}."
|
|
1934
|
+
}
|
|
1935
|
+
},
|
|
1936
|
+
additionalProperties: false,
|
|
1937
|
+
description: "Strip a unit suffix and multiply by its factor; plain numeric values convert too."
|
|
1938
|
+
},
|
|
1939
|
+
{
|
|
1940
|
+
type: "object",
|
|
1941
|
+
properties: {
|
|
1942
|
+
rule: {
|
|
1943
|
+
type: "string",
|
|
1944
|
+
const: "trim",
|
|
1945
|
+
required: true
|
|
1946
|
+
},
|
|
1947
|
+
columns: {
|
|
1948
|
+
type: "array",
|
|
1949
|
+
items: { type: "string" },
|
|
1950
|
+
description: "Columns to trim; all columns when omitted."
|
|
1951
|
+
}
|
|
1952
|
+
},
|
|
1953
|
+
additionalProperties: false,
|
|
1954
|
+
description: "Trim leading/trailing whitespace of string cells."
|
|
1955
|
+
},
|
|
1956
|
+
{
|
|
1957
|
+
type: "object",
|
|
1958
|
+
properties: {
|
|
1959
|
+
rule: {
|
|
1960
|
+
type: "string",
|
|
1961
|
+
const: "map-values",
|
|
1962
|
+
required: true
|
|
1963
|
+
},
|
|
1964
|
+
column: {
|
|
1965
|
+
type: "string",
|
|
1966
|
+
required: true
|
|
1967
|
+
},
|
|
1968
|
+
map: {
|
|
1969
|
+
...STRING_MAP_SCHEMA,
|
|
1970
|
+
description: "Exact-match value mapping, e.g. {\"Y\": \"yes\", \"N\": \"no\"}."
|
|
1971
|
+
},
|
|
1972
|
+
else: {
|
|
1973
|
+
type: "string",
|
|
1974
|
+
enum: ["keep", "missing"],
|
|
1975
|
+
description: "Policy for unmapped values (default keep)."
|
|
1976
|
+
}
|
|
1977
|
+
},
|
|
1978
|
+
additionalProperties: false,
|
|
1979
|
+
description: "Map enum values by exact match."
|
|
1980
|
+
}
|
|
1981
|
+
] },
|
|
1982
|
+
description: "Ordered cleaning rules; each rule sees the previous rule's output."
|
|
1983
|
+
};
|
|
1984
|
+
/**
|
|
1985
|
+
* Build the `data_clean` tool definition against a mounted service.
|
|
1986
|
+
* @param service - the mounted ctx.dataQuality implementation.
|
|
1987
|
+
* @returns the tool definition to register.
|
|
1988
|
+
*/
|
|
1989
|
+
function defineCleanTool(service) {
|
|
1990
|
+
return defineTool({
|
|
1991
|
+
name: "data_clean",
|
|
1992
|
+
description: [
|
|
1993
|
+
"Apply declarative cleaning rules to a workspace CSV/TSV/JSON/JSONL dataset with deterministic TypeScript computation (no mental math).",
|
|
1994
|
+
"Rules apply in array order: dedupe (by column group), fill-missing (constant/mean/median/forward), coerce-type (number/date/boolean; failures counted and set to missing), normalize-unit (e.g. 万/亿 suffixes to base units), trim (whitespace), map-values (enum mapping).",
|
|
1995
|
+
"The source file is NEVER overwritten. Without outputPath the run is preview-only; with outputPath the cleaned dataset is written there (workspace-confined, .csv/.tsv/.json/.jsonl). Returns the per-rule audit log (affected rows per rule) plus a bounded preview. The full report persists to the data_quality storage domain (reportKey)."
|
|
1996
|
+
].join("\n"),
|
|
1997
|
+
parameters: {
|
|
1998
|
+
path: {
|
|
1999
|
+
type: "string",
|
|
2000
|
+
required: true,
|
|
2001
|
+
description: "Workspace-relative dataset path (.csv/.tsv/.json/.jsonl)."
|
|
2002
|
+
},
|
|
2003
|
+
rules: {
|
|
2004
|
+
...CLEAN_RULE_SCHEMA,
|
|
2005
|
+
required: true
|
|
2006
|
+
},
|
|
2007
|
+
outputPath: {
|
|
2008
|
+
type: "string",
|
|
2009
|
+
description: "Optional workspace-relative output path for the cleaned dataset (must differ from path)."
|
|
2010
|
+
}
|
|
2011
|
+
},
|
|
2012
|
+
output: {
|
|
2013
|
+
schema: {
|
|
2014
|
+
type: "object",
|
|
2015
|
+
properties: {
|
|
2016
|
+
dataset: {
|
|
2017
|
+
type: "string",
|
|
2018
|
+
required: true
|
|
2019
|
+
},
|
|
2020
|
+
inputRows: {
|
|
2021
|
+
type: "number",
|
|
2022
|
+
required: true
|
|
2023
|
+
},
|
|
2024
|
+
outputRows: {
|
|
2025
|
+
type: "number",
|
|
2026
|
+
required: true
|
|
2027
|
+
},
|
|
2028
|
+
generatedAt: {
|
|
2029
|
+
type: "number",
|
|
2030
|
+
required: true
|
|
2031
|
+
},
|
|
2032
|
+
outputPath: { type: "string" },
|
|
2033
|
+
reportKey: { type: "string" },
|
|
2034
|
+
logs: {
|
|
2035
|
+
type: "array",
|
|
2036
|
+
items: {
|
|
2037
|
+
type: "object",
|
|
2038
|
+
properties: {
|
|
2039
|
+
ruleIndex: {
|
|
2040
|
+
type: "number",
|
|
2041
|
+
required: true
|
|
2042
|
+
},
|
|
2043
|
+
rule: {
|
|
2044
|
+
type: "string",
|
|
2045
|
+
enum: [
|
|
2046
|
+
"dedupe",
|
|
2047
|
+
"fill-missing",
|
|
2048
|
+
"coerce-type",
|
|
2049
|
+
"normalize-unit",
|
|
2050
|
+
"trim",
|
|
2051
|
+
"map-values"
|
|
2052
|
+
],
|
|
2053
|
+
required: true
|
|
2054
|
+
},
|
|
2055
|
+
affectedRows: {
|
|
2056
|
+
type: "number",
|
|
2057
|
+
required: true
|
|
2058
|
+
},
|
|
2059
|
+
detail: {
|
|
2060
|
+
type: "string",
|
|
2061
|
+
required: true
|
|
2062
|
+
}
|
|
2063
|
+
},
|
|
2064
|
+
additionalProperties: false
|
|
2065
|
+
},
|
|
2066
|
+
required: true
|
|
2067
|
+
},
|
|
2068
|
+
preview: {
|
|
2069
|
+
type: "object",
|
|
2070
|
+
properties: {
|
|
2071
|
+
columns: {
|
|
2072
|
+
type: "array",
|
|
2073
|
+
items: { type: "string" },
|
|
2074
|
+
required: true
|
|
2075
|
+
},
|
|
2076
|
+
rows: {
|
|
2077
|
+
type: "array",
|
|
2078
|
+
items: { type: "json" },
|
|
2079
|
+
required: true
|
|
2080
|
+
}
|
|
2081
|
+
},
|
|
2082
|
+
additionalProperties: false,
|
|
2083
|
+
required: true
|
|
2084
|
+
}
|
|
2085
|
+
},
|
|
2086
|
+
additionalProperties: false
|
|
2087
|
+
},
|
|
2088
|
+
render: (_args, value) => [{
|
|
2089
|
+
type: "text",
|
|
2090
|
+
text: renderCleanText(value)
|
|
2091
|
+
}]
|
|
2092
|
+
},
|
|
2093
|
+
async execute(args, exec) {
|
|
2094
|
+
return service.cleanDataset({
|
|
2095
|
+
dataset: args.path,
|
|
2096
|
+
rules: args.rules,
|
|
2097
|
+
...args.outputPath !== void 0 ? { outputPath: args.outputPath } : {},
|
|
2098
|
+
workspace: workspaceOf(exec),
|
|
2099
|
+
session: exec.agent?.session,
|
|
2100
|
+
signal: exec.signal
|
|
2101
|
+
});
|
|
2102
|
+
}
|
|
2103
|
+
});
|
|
2104
|
+
}
|
|
2105
|
+
//#endregion
|
|
2106
|
+
//#region src/tools/verify.ts
|
|
2107
|
+
/**
|
|
2108
|
+
* The `data_verify` model tool: declarative verification rules through
|
|
2109
|
+
* `ctx.dataQuality.verifyDataset`. A failing dataset is a normal result
|
|
2110
|
+
* (`passed: false` with evidence), never a tool error.
|
|
2111
|
+
* @module dsh-data-quality/tools/verify
|
|
2112
|
+
*/
|
|
2113
|
+
const VERIFY_RULE_SCHEMA = {
|
|
2114
|
+
type: "array",
|
|
2115
|
+
items: { oneOf: [
|
|
2116
|
+
{
|
|
2117
|
+
type: "object",
|
|
2118
|
+
properties: {
|
|
2119
|
+
rule: {
|
|
2120
|
+
type: "string",
|
|
2121
|
+
const: "not-null",
|
|
2122
|
+
required: true
|
|
2123
|
+
},
|
|
2124
|
+
column: {
|
|
2125
|
+
type: "string",
|
|
2126
|
+
required: true
|
|
2127
|
+
}
|
|
2128
|
+
},
|
|
2129
|
+
additionalProperties: false,
|
|
2130
|
+
description: "Fail rows where the column is missing (null/empty/whitespace)."
|
|
2131
|
+
},
|
|
2132
|
+
{
|
|
2133
|
+
type: "object",
|
|
2134
|
+
properties: {
|
|
2135
|
+
rule: {
|
|
2136
|
+
type: "string",
|
|
2137
|
+
const: "unique",
|
|
2138
|
+
required: true
|
|
2139
|
+
},
|
|
2140
|
+
columns: {
|
|
2141
|
+
type: "array",
|
|
2142
|
+
items: { type: "string" },
|
|
2143
|
+
required: true,
|
|
2144
|
+
description: "Key columns; every row whose key repeats fails."
|
|
2145
|
+
}
|
|
2146
|
+
},
|
|
2147
|
+
additionalProperties: false,
|
|
2148
|
+
description: "Fail rows whose key-column combination appears more than once."
|
|
2149
|
+
},
|
|
2150
|
+
{
|
|
2151
|
+
type: "object",
|
|
2152
|
+
properties: {
|
|
2153
|
+
rule: {
|
|
2154
|
+
type: "string",
|
|
2155
|
+
const: "range",
|
|
2156
|
+
required: true
|
|
2157
|
+
},
|
|
2158
|
+
column: {
|
|
2159
|
+
type: "string",
|
|
2160
|
+
required: true
|
|
2161
|
+
},
|
|
2162
|
+
min: { type: "number" },
|
|
2163
|
+
max: { type: "number" }
|
|
2164
|
+
},
|
|
2165
|
+
additionalProperties: false,
|
|
2166
|
+
description: "Fail rows where the numeric cell is missing, unparseable, or outside [min, max] (inclusive; at least one bound required)."
|
|
2167
|
+
},
|
|
2168
|
+
{
|
|
2169
|
+
type: "object",
|
|
2170
|
+
properties: {
|
|
2171
|
+
rule: {
|
|
2172
|
+
type: "string",
|
|
2173
|
+
const: "regex",
|
|
2174
|
+
required: true
|
|
2175
|
+
},
|
|
2176
|
+
column: {
|
|
2177
|
+
type: "string",
|
|
2178
|
+
required: true
|
|
2179
|
+
},
|
|
2180
|
+
pattern: {
|
|
2181
|
+
type: "string",
|
|
2182
|
+
required: true
|
|
2183
|
+
},
|
|
2184
|
+
flags: {
|
|
2185
|
+
type: "string",
|
|
2186
|
+
description: "Optional JS regex flags, e.g. \"i\"."
|
|
2187
|
+
}
|
|
2188
|
+
},
|
|
2189
|
+
additionalProperties: false,
|
|
2190
|
+
description: "Fail rows where the cell is missing or does not match the pattern (full JS regex semantics)."
|
|
2191
|
+
},
|
|
2192
|
+
{
|
|
2193
|
+
type: "object",
|
|
2194
|
+
properties: {
|
|
2195
|
+
rule: {
|
|
2196
|
+
type: "string",
|
|
2197
|
+
const: "enum",
|
|
2198
|
+
required: true
|
|
2199
|
+
},
|
|
2200
|
+
column: {
|
|
2201
|
+
type: "string",
|
|
2202
|
+
required: true
|
|
2203
|
+
},
|
|
2204
|
+
values: {
|
|
2205
|
+
type: "array",
|
|
2206
|
+
items: { type: "string" },
|
|
2207
|
+
required: true
|
|
2208
|
+
}
|
|
2209
|
+
},
|
|
2210
|
+
additionalProperties: false,
|
|
2211
|
+
description: "Fail rows where the trimmed cell text is not one of the allowed values."
|
|
2212
|
+
},
|
|
2213
|
+
{
|
|
2214
|
+
type: "object",
|
|
2215
|
+
properties: {
|
|
2216
|
+
rule: {
|
|
2217
|
+
type: "string",
|
|
2218
|
+
const: "cross-column",
|
|
2219
|
+
required: true
|
|
2220
|
+
},
|
|
2221
|
+
left: {
|
|
2222
|
+
type: "string",
|
|
2223
|
+
required: true,
|
|
2224
|
+
description: "Left column name."
|
|
2225
|
+
},
|
|
2226
|
+
op: {
|
|
2227
|
+
type: "string",
|
|
2228
|
+
enum: [
|
|
2229
|
+
"<",
|
|
2230
|
+
"<=",
|
|
2231
|
+
"==",
|
|
2232
|
+
"!=",
|
|
2233
|
+
">=",
|
|
2234
|
+
">"
|
|
2235
|
+
],
|
|
2236
|
+
required: true
|
|
2237
|
+
},
|
|
2238
|
+
rightColumn: {
|
|
2239
|
+
type: "string",
|
|
2240
|
+
description: "Compare against another column…"
|
|
2241
|
+
},
|
|
2242
|
+
value: {
|
|
2243
|
+
oneOf: [{ type: "number" }, { type: "string" }],
|
|
2244
|
+
description: "…or against this literal (exactly one of rightColumn/value)."
|
|
2245
|
+
}
|
|
2246
|
+
},
|
|
2247
|
+
additionalProperties: false,
|
|
2248
|
+
description: "Compare two columns (e.g. startDate < endDate) or a column against a literal. Numeric when both sides parse, string equality for ==/!= otherwise."
|
|
2249
|
+
},
|
|
2250
|
+
{
|
|
2251
|
+
type: "object",
|
|
2252
|
+
properties: {
|
|
2253
|
+
rule: {
|
|
2254
|
+
type: "string",
|
|
2255
|
+
const: "freshness",
|
|
2256
|
+
required: true
|
|
2257
|
+
},
|
|
2258
|
+
column: {
|
|
2259
|
+
type: "string",
|
|
2260
|
+
required: true,
|
|
2261
|
+
description: "Date column (YYYY-MM-DD, YYYY/MM/DD, or ISO datetime)."
|
|
2262
|
+
},
|
|
2263
|
+
maxAgeDays: {
|
|
2264
|
+
type: "number",
|
|
2265
|
+
required: true,
|
|
2266
|
+
description: "Fail rows whose date is older than this many days before asOf."
|
|
2267
|
+
},
|
|
2268
|
+
asOf: {
|
|
2269
|
+
type: "string",
|
|
2270
|
+
description: "Reference date; defaults to the current date."
|
|
2271
|
+
}
|
|
2272
|
+
},
|
|
2273
|
+
additionalProperties: false,
|
|
2274
|
+
description: "Fail rows whose date cell is missing, unparseable, or older than maxAgeDays before asOf."
|
|
2275
|
+
}
|
|
2276
|
+
] },
|
|
2277
|
+
description: "Verification rules; a missing cell fails every rule that reads it."
|
|
2278
|
+
};
|
|
2279
|
+
/**
|
|
2280
|
+
* Build the `data_verify` tool definition against a mounted service.
|
|
2281
|
+
* @param service - the mounted ctx.dataQuality implementation.
|
|
2282
|
+
* @returns the tool definition to register.
|
|
2283
|
+
*/
|
|
2284
|
+
function defineVerifyTool(service) {
|
|
2285
|
+
return defineTool({
|
|
2286
|
+
name: "data_verify",
|
|
2287
|
+
description: [
|
|
2288
|
+
"Verify a workspace CSV/TSV/JSON/JSONL dataset against declarative quality rules with deterministic TypeScript computation (no mental math).",
|
|
2289
|
+
"Rules: not-null, unique (column group), range (numeric bounds), regex, enum, cross-column (e.g. startDate < endDate), freshness (date column within N days of asOf). A missing cell fails every rule that reads it.",
|
|
2290
|
+
"Returns per-rule pass/fail with capped failing-row evidence. Overall failure is a NORMAL result with passed: false — not a tool error. The full report persists to the data_quality storage domain (reportKey)."
|
|
2291
|
+
].join("\n"),
|
|
2292
|
+
parameters: {
|
|
2293
|
+
path: {
|
|
2294
|
+
type: "string",
|
|
2295
|
+
required: true,
|
|
2296
|
+
description: "Workspace-relative dataset path (.csv/.tsv/.json/.jsonl)."
|
|
2297
|
+
},
|
|
2298
|
+
rules: {
|
|
2299
|
+
...VERIFY_RULE_SCHEMA,
|
|
2300
|
+
required: true
|
|
2301
|
+
}
|
|
2302
|
+
},
|
|
2303
|
+
output: {
|
|
2304
|
+
schema: {
|
|
2305
|
+
type: "object",
|
|
2306
|
+
properties: {
|
|
2307
|
+
dataset: {
|
|
2308
|
+
type: "string",
|
|
2309
|
+
required: true
|
|
2310
|
+
},
|
|
2311
|
+
passed: {
|
|
2312
|
+
type: "boolean",
|
|
2313
|
+
required: true
|
|
2314
|
+
},
|
|
2315
|
+
rowCount: {
|
|
2316
|
+
type: "number",
|
|
2317
|
+
required: true
|
|
2318
|
+
},
|
|
2319
|
+
generatedAt: {
|
|
2320
|
+
type: "number",
|
|
2321
|
+
required: true
|
|
2322
|
+
},
|
|
2323
|
+
reportKey: { type: "string" },
|
|
2324
|
+
rules: {
|
|
2325
|
+
type: "array",
|
|
2326
|
+
items: {
|
|
2327
|
+
type: "object",
|
|
2328
|
+
properties: {
|
|
2329
|
+
ruleIndex: {
|
|
2330
|
+
type: "number",
|
|
2331
|
+
required: true
|
|
2332
|
+
},
|
|
2333
|
+
rule: {
|
|
2334
|
+
type: "string",
|
|
2335
|
+
enum: [
|
|
2336
|
+
"not-null",
|
|
2337
|
+
"unique",
|
|
2338
|
+
"range",
|
|
2339
|
+
"regex",
|
|
2340
|
+
"enum",
|
|
2341
|
+
"cross-column",
|
|
2342
|
+
"freshness"
|
|
2343
|
+
],
|
|
2344
|
+
required: true
|
|
2345
|
+
},
|
|
2346
|
+
passed: {
|
|
2347
|
+
type: "boolean",
|
|
2348
|
+
required: true
|
|
2349
|
+
},
|
|
2350
|
+
failedCount: {
|
|
2351
|
+
type: "number",
|
|
2352
|
+
required: true
|
|
2353
|
+
},
|
|
2354
|
+
checkedCount: {
|
|
2355
|
+
type: "number",
|
|
2356
|
+
required: true
|
|
2357
|
+
},
|
|
2358
|
+
evidence: {
|
|
2359
|
+
type: "array",
|
|
2360
|
+
items: {
|
|
2361
|
+
type: "object",
|
|
2362
|
+
properties: {
|
|
2363
|
+
rowIndex: {
|
|
2364
|
+
type: "number",
|
|
2365
|
+
required: true
|
|
2366
|
+
},
|
|
2367
|
+
values: {
|
|
2368
|
+
type: "json",
|
|
2369
|
+
required: true
|
|
2370
|
+
}
|
|
2371
|
+
},
|
|
2372
|
+
additionalProperties: false
|
|
2373
|
+
},
|
|
2374
|
+
required: true
|
|
2375
|
+
}
|
|
2376
|
+
},
|
|
2377
|
+
additionalProperties: false
|
|
2378
|
+
},
|
|
2379
|
+
required: true
|
|
2380
|
+
}
|
|
2381
|
+
},
|
|
2382
|
+
additionalProperties: false
|
|
2383
|
+
},
|
|
2384
|
+
render: (_args, value) => [{
|
|
2385
|
+
type: "text",
|
|
2386
|
+
text: renderVerifyText(value)
|
|
2387
|
+
}]
|
|
2388
|
+
},
|
|
2389
|
+
async execute(args, exec) {
|
|
2390
|
+
return service.verifyDataset({
|
|
2391
|
+
dataset: args.path,
|
|
2392
|
+
rules: args.rules,
|
|
2393
|
+
workspace: workspaceOf(exec),
|
|
2394
|
+
session: exec.agent?.session,
|
|
2395
|
+
signal: exec.signal
|
|
2396
|
+
});
|
|
2397
|
+
}
|
|
2398
|
+
});
|
|
2399
|
+
}
|
|
2400
|
+
//#endregion
|
|
2401
|
+
//#region src/version.ts
|
|
2402
|
+
/**
|
|
2403
|
+
* Plugin version, kept in one place so `scripts/release.mjs` can stamp it and
|
|
2404
|
+
* reports can name their generator.
|
|
2405
|
+
* @module dsh-data-quality/version
|
|
2406
|
+
*/
|
|
2407
|
+
/** The package version reported in persisted reports. */
|
|
2408
|
+
const VERSION = "0.1.0";
|
|
2409
|
+
//#endregion
|
|
2410
|
+
//#region src/index.ts
|
|
2411
|
+
const name = "data-quality";
|
|
2412
|
+
/** The three model tools and the durable report domain. */
|
|
2413
|
+
const inject = ["tools", "storageDomain"];
|
|
2414
|
+
/**
|
|
2415
|
+
* Mount the seam: resolve config (fail loud), open the report domain, publish
|
|
2416
|
+
* `ctx.dataQuality`, and register the three tools. With `enabled: false` the
|
|
2417
|
+
* plugin registers nothing and stays inert.
|
|
2418
|
+
* @param ctx - the plugin context (host).
|
|
2419
|
+
* @param config - raw plugin config.
|
|
2420
|
+
*/
|
|
2421
|
+
async function apply(ctx, config = {}) {
|
|
2422
|
+
const resolved = resolveConfig(config);
|
|
2423
|
+
const logger = ctx.logger("data-quality");
|
|
2424
|
+
if (!resolved.enabled) {
|
|
2425
|
+
logger.info("disabled: enabled is false — no service or tools are mounted");
|
|
2426
|
+
return;
|
|
2427
|
+
}
|
|
2428
|
+
let store;
|
|
2429
|
+
let domain;
|
|
2430
|
+
if (resolved.storeReports) {
|
|
2431
|
+
domain = await ctx.storageDomain.open(dataQualityDomainSpec);
|
|
2432
|
+
const reports = domain.table("reports");
|
|
2433
|
+
store = {
|
|
2434
|
+
put: async (record) => {
|
|
2435
|
+
const key = reportKeyOf(record);
|
|
2436
|
+
await reports.put(key, record);
|
|
2437
|
+
return key;
|
|
2438
|
+
},
|
|
2439
|
+
get: (key) => reports.get(key)
|
|
2440
|
+
};
|
|
2441
|
+
}
|
|
2442
|
+
const service = new LocalDataQualityService(ctx, resolved, {
|
|
2443
|
+
store,
|
|
2444
|
+
now: Date.now
|
|
2445
|
+
});
|
|
2446
|
+
ctx.tools.register(defineProfileTool(service));
|
|
2447
|
+
ctx.tools.register(defineCleanTool(service));
|
|
2448
|
+
ctx.tools.register(defineVerifyTool(service));
|
|
2449
|
+
logger.info(`dsh-data-quality ${VERSION} mounted: ctx.dataQuality + data_profile/data_clean/data_verify`);
|
|
2450
|
+
if (domain !== void 0) {
|
|
2451
|
+
const handle = domain;
|
|
2452
|
+
ctx.effect(() => async () => {
|
|
2453
|
+
await handle.close();
|
|
2454
|
+
});
|
|
2455
|
+
}
|
|
2456
|
+
}
|
|
2457
|
+
//#endregion
|
|
2458
|
+
export { Config, DATA_QUALITY_EVENT_TYPES, DataQualityService, DatasetError, LocalDataQualityService, MAX_CELL_TEXT, VERSION, appendDataQualityEvent, apply, applyCleanRules, checkCitations, dataQualityDomainSpec, inject, isMissing, loadDocument, loadTable, name, parseBoolean, parseDate, parseDelimited, parseJsonTable, parseLocator, parseNumeric, profileTable, renderProfileText, renderVerifyText, reportKeyOf, reportRecordSchema, resolveConfig, resolveWorkspacePath, sampleRows, serializeDelimited, truncateCell, truncateRow, verifyTable };
|