dsh-data-quality 0.1.3 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -0
- package/README.es.md +7 -1
- package/README.hi.md +7 -1
- package/README.md +22 -9
- package/README.pt.md +7 -1
- package/README.zh.md +7 -1
- package/cordis.patch.yml +11 -0
- package/lib/index.js +1609 -145
- package/lib/types/config.d.ts +12 -0
- package/lib/types/config.d.ts.map +1 -1
- package/lib/types/config.js +35 -0
- package/lib/types/config.js.map +1 -1
- package/lib/types/contract.d.ts +83 -0
- package/lib/types/contract.d.ts.map +1 -0
- package/lib/types/contract.js +144 -0
- package/lib/types/contract.js.map +1 -0
- package/lib/types/dataset.d.ts +50 -3
- package/lib/types/dataset.d.ts.map +1 -1
- package/lib/types/dataset.js +76 -21
- package/lib/types/dataset.js.map +1 -1
- package/lib/types/events.d.ts +2 -0
- package/lib/types/events.d.ts.map +1 -1
- package/lib/types/events.js.map +1 -1
- package/lib/types/index.d.ts +13 -8
- package/lib/types/index.d.ts.map +1 -1
- package/lib/types/index.js +17 -6
- package/lib/types/index.js.map +1 -1
- package/lib/types/presets.d.ts +28 -0
- package/lib/types/presets.d.ts.map +1 -0
- package/lib/types/presets.js +134 -0
- package/lib/types/presets.js.map +1 -0
- package/lib/types/profile.d.ts +40 -1
- package/lib/types/profile.d.ts.map +1 -1
- package/lib/types/profile.js +66 -10
- package/lib/types/profile.js.map +1 -1
- package/lib/types/provider-local.d.ts +7 -1
- package/lib/types/provider-local.d.ts.map +1 -1
- package/lib/types/provider-local.js +72 -2
- package/lib/types/provider-local.js.map +1 -1
- package/lib/types/report-html.d.ts +41 -0
- package/lib/types/report-html.d.ts.map +1 -0
- package/lib/types/report-html.js +192 -0
- package/lib/types/report-html.js.map +1 -0
- package/lib/types/scorecard.d.ts +50 -0
- package/lib/types/scorecard.d.ts.map +1 -0
- package/lib/types/scorecard.js +186 -0
- package/lib/types/scorecard.js.map +1 -0
- package/lib/types/service.d.ts +32 -2
- package/lib/types/service.d.ts.map +1 -1
- package/lib/types/service.js.map +1 -1
- package/lib/types/store.d.ts +21 -1
- package/lib/types/store.d.ts.map +1 -1
- package/lib/types/store.js +13 -1
- package/lib/types/store.js.map +1 -1
- package/lib/types/tools/clean.d.ts.map +1 -1
- package/lib/types/tools/clean.js +76 -1
- package/lib/types/tools/clean.js.map +1 -1
- package/lib/types/tools/profile-report-schema.d.ts +319 -0
- package/lib/types/tools/profile-report-schema.d.ts.map +1 -0
- package/lib/types/tools/profile-report-schema.js +96 -0
- package/lib/types/tools/profile-report-schema.js.map +1 -0
- package/lib/types/tools/profile.d.ts.map +1 -1
- package/lib/types/tools/profile.js +6 -53
- package/lib/types/tools/profile.js.map +1 -1
- package/lib/types/tools/report.d.ts +14 -0
- package/lib/types/tools/report.d.ts.map +1 -0
- package/lib/types/tools/report.js +112 -0
- package/lib/types/tools/report.js.map +1 -0
- package/lib/types/tools/shared.d.ts.map +1 -1
- package/lib/types/tools/shared.js +22 -2
- package/lib/types/tools/shared.js.map +1 -1
- package/lib/types/tools/verify.d.ts.map +1 -1
- package/lib/types/tools/verify.js +36 -1
- package/lib/types/tools/verify.js.map +1 -1
- package/lib/types/verify.d.ts +49 -3
- package/lib/types/verify.d.ts.map +1 -1
- package/lib/types/verify.js +111 -4
- package/lib/types/verify.js.map +1 -1
- package/lib/types/version.d.ts +8 -1
- package/lib/types/version.d.ts.map +1 -1
- package/lib/types/version.js +8 -1
- package/lib/types/version.js.map +1 -1
- package/package.json +1 -1
- package/src/config.ts +52 -0
- package/src/contract.ts +190 -0
- package/src/dataset.ts +101 -21
- package/src/events.ts +2 -0
- package/src/index.ts +27 -8
- package/src/presets.ts +146 -0
- package/src/profile.ts +105 -11
- package/src/provider-local.ts +77 -3
- package/src/report-html.ts +208 -0
- package/src/scorecard.ts +244 -0
- package/src/service.ts +34 -2
- package/src/store.ts +28 -2
- package/src/tools/clean.ts +76 -1
- package/src/tools/profile-report-schema.ts +98 -0
- package/src/tools/profile.ts +8 -56
- package/src/tools/report.ts +137 -0
- package/src/tools/shared.ts +21 -2
- package/src/tools/verify.ts +40 -2
- package/src/verify.ts +151 -5
- package/src/version.ts +9 -1
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* DAMA-style six-dimension quality scorecard over a parsed {@link Table}.
|
|
3
|
+
* Pure and deterministic: every rate derives from one full-table pass, the
|
|
4
|
+
* only clock is the injected `now`, and no dimension fabricates a score it
|
|
5
|
+
* cannot defend — `accuracy` stays `null` (undetermined) without a declared
|
|
6
|
+
* schema and `timeliness` stays `null` without date cells.
|
|
7
|
+
* @module dsh-data-quality/scorecard
|
|
8
|
+
*/
|
|
9
|
+
import { isMissing, parseBoolean, parseDateCell, parseNumeric, throwIfAborted } from "./dataset.js";
|
|
10
|
+
/** Round to 6 significant digits (mirrors the profile report's rounding). */
|
|
11
|
+
function round6(value) {
|
|
12
|
+
return Number(value.toPrecision(6));
|
|
13
|
+
}
|
|
14
|
+
/** Increment a format-label counter. */
|
|
15
|
+
function bump(formats, tag) {
|
|
16
|
+
formats.set(tag, (formats.get(tag) ?? 0) + 1);
|
|
17
|
+
}
|
|
18
|
+
/** Inferred column type from the full-table class counts (mirrors `profileColumn`). */
|
|
19
|
+
function inferType(stats) {
|
|
20
|
+
if (stats.present === 0)
|
|
21
|
+
return 'empty';
|
|
22
|
+
if (stats.number === stats.present)
|
|
23
|
+
return 'number';
|
|
24
|
+
if (stats.date === stats.present)
|
|
25
|
+
return 'date';
|
|
26
|
+
if (stats.boolean === stats.present)
|
|
27
|
+
return 'boolean';
|
|
28
|
+
if (stats.string === stats.present)
|
|
29
|
+
return 'string';
|
|
30
|
+
return 'mixed';
|
|
31
|
+
}
|
|
32
|
+
/** Format the `value / total` ratio as a plain `note` fragment. */
|
|
33
|
+
function ratioNote(numerator, denominator) {
|
|
34
|
+
return `${numerator}/${denominator}`;
|
|
35
|
+
}
|
|
36
|
+
/** Build one determinable dimension with its ratio note. */
|
|
37
|
+
function dimension(name, score, numerator, denominator) {
|
|
38
|
+
return { name, score, note: ratioNote(numerator, denominator) };
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Compute the six-dimension scorecard over a full table:
|
|
42
|
+
* - `completeness` — non-empty cell rate.
|
|
43
|
+
* - `uniqueness` — unique full-row content rate (`1 - duplicateRows / rowCount`).
|
|
44
|
+
* - `validity` — present cells conforming to their column's inferred type.
|
|
45
|
+
* - `consistency` — present cells in their column's dominant format (a date
|
|
46
|
+
* column's unified `YYYY-MM-DD` vs `YYYY/MM/DD` vs datetime ratio).
|
|
47
|
+
* - `timeliness` — date cells not future-dated relative to the injected `now`.
|
|
48
|
+
* - `accuracy` — declared-schema agreement (limited definition); `null`
|
|
49
|
+
* (undetermined) without a declared schema — never fabricated.
|
|
50
|
+
* @param table - the parsed dataset (full table, not the sampled cards).
|
|
51
|
+
* @param options - injected clock, duplicate-row count, optional declared schema, optional weights, abort signal.
|
|
52
|
+
* @returns the scorecard.
|
|
53
|
+
*/
|
|
54
|
+
export function computeScorecard(table, options) {
|
|
55
|
+
throwIfAborted(options.signal);
|
|
56
|
+
const stats = new Map();
|
|
57
|
+
for (const column of table.columns) {
|
|
58
|
+
stats.set(column, { present: 0, number: 0, date: 0, boolean: 0, string: 0, formats: new Map() });
|
|
59
|
+
}
|
|
60
|
+
let missingCells = 0;
|
|
61
|
+
let dateCells = 0;
|
|
62
|
+
let futureDateCells = 0;
|
|
63
|
+
for (const [index, row] of table.rows.entries()) {
|
|
64
|
+
if (index % 1024 === 0)
|
|
65
|
+
throwIfAborted(options.signal);
|
|
66
|
+
for (const column of table.columns) {
|
|
67
|
+
const cell = row[column];
|
|
68
|
+
const columnStats = stats.get(column);
|
|
69
|
+
if (isMissing(cell)) {
|
|
70
|
+
missingCells += 1;
|
|
71
|
+
continue;
|
|
72
|
+
}
|
|
73
|
+
columnStats.present += 1;
|
|
74
|
+
const numeric = parseNumeric(cell);
|
|
75
|
+
if (numeric !== undefined) {
|
|
76
|
+
columnStats.number += 1;
|
|
77
|
+
bump(columnStats.formats, 'number');
|
|
78
|
+
continue;
|
|
79
|
+
}
|
|
80
|
+
const date = parseDateCell(cell);
|
|
81
|
+
if (date !== undefined) {
|
|
82
|
+
columnStats.date += 1;
|
|
83
|
+
bump(columnStats.formats, date.format);
|
|
84
|
+
dateCells += 1;
|
|
85
|
+
if (date.epoch > options.now)
|
|
86
|
+
futureDateCells += 1;
|
|
87
|
+
continue;
|
|
88
|
+
}
|
|
89
|
+
if (parseBoolean(cell) !== undefined) {
|
|
90
|
+
columnStats.boolean += 1;
|
|
91
|
+
bump(columnStats.formats, 'boolean');
|
|
92
|
+
continue;
|
|
93
|
+
}
|
|
94
|
+
columnStats.string += 1;
|
|
95
|
+
bump(columnStats.formats, typeof cell === 'string' ? 'string' : 'json');
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
const totalCells = table.rows.length * table.columns.length;
|
|
99
|
+
let presentCells = 0;
|
|
100
|
+
let invalidCells = 0;
|
|
101
|
+
let consistentCells = 0;
|
|
102
|
+
for (const [, columnStats] of stats) {
|
|
103
|
+
presentCells += columnStats.present;
|
|
104
|
+
// Validity = conformance to the column's dominant TYPE class; consistency
|
|
105
|
+
// below measures dominant FORMAT (so a date column with mixed YYYY-MM-DD /
|
|
106
|
+
// YYYY/MM/DD formats stays valid but inconsistent).
|
|
107
|
+
const dominantClass = Math.max(columnStats.number, columnStats.date, columnStats.boolean, columnStats.string);
|
|
108
|
+
invalidCells += columnStats.present - dominantClass;
|
|
109
|
+
if (columnStats.present === 0)
|
|
110
|
+
continue;
|
|
111
|
+
let dominant = 0;
|
|
112
|
+
for (const count of columnStats.formats.values()) {
|
|
113
|
+
if (count > dominant)
|
|
114
|
+
dominant = count;
|
|
115
|
+
}
|
|
116
|
+
consistentCells += dominant;
|
|
117
|
+
}
|
|
118
|
+
const completeness = totalCells === 0
|
|
119
|
+
? { name: 'completeness', score: null, note: 'undetermined: no cells' }
|
|
120
|
+
: dimension('completeness', round6(1 - missingCells / totalCells), presentCells, totalCells);
|
|
121
|
+
const uniqueness = table.rows.length === 0
|
|
122
|
+
? { name: 'uniqueness', score: null, note: 'undetermined: no rows' }
|
|
123
|
+
: dimension('uniqueness', round6(1 - options.duplicateRows / table.rows.length), table.rows.length - options.duplicateRows, table.rows.length);
|
|
124
|
+
const validity = presentCells === 0
|
|
125
|
+
? { name: 'validity', score: null, note: 'undetermined: no present cells' }
|
|
126
|
+
: dimension('validity', round6(1 - invalidCells / presentCells), presentCells - invalidCells, presentCells);
|
|
127
|
+
const consistency = presentCells === 0
|
|
128
|
+
? { name: 'consistency', score: null, note: 'undetermined: no present cells' }
|
|
129
|
+
: dimension('consistency', round6(consistentCells / presentCells), consistentCells, presentCells);
|
|
130
|
+
const timeliness = dateCells === 0
|
|
131
|
+
? { name: 'timeliness', score: null, note: 'undetermined: no date cells' }
|
|
132
|
+
: dimension('timeliness', round6(1 - futureDateCells / dateCells), dateCells - futureDateCells, dateCells);
|
|
133
|
+
const accuracy = computeAccuracy(stats, table, options.declaredSchema);
|
|
134
|
+
const dimensions = [completeness, uniqueness, validity, consistency, timeliness, accuracy];
|
|
135
|
+
const scores = dimensions.map((entry) => entry.score).filter((score) => score !== null);
|
|
136
|
+
const overall = scores.length === 0 ? null : round6(scores.reduce((sum, score) => sum + score, 0) / scores.length);
|
|
137
|
+
const weightedOverall = computeWeightedOverall(dimensions, options.weights);
|
|
138
|
+
return { dimensions, overall, weightedOverall };
|
|
139
|
+
}
|
|
140
|
+
/** Weighted mean over determinable dimensions (weights default to 1; validated non-negative by config). */
|
|
141
|
+
function computeWeightedOverall(dimensions, weights) {
|
|
142
|
+
let weightedSum = 0;
|
|
143
|
+
let weightTotal = 0;
|
|
144
|
+
for (const entry of dimensions) {
|
|
145
|
+
if (entry.score === null)
|
|
146
|
+
continue;
|
|
147
|
+
const weight = weights?.[entry.name] ?? 1;
|
|
148
|
+
weightedSum += weight * entry.score;
|
|
149
|
+
weightTotal += weight;
|
|
150
|
+
}
|
|
151
|
+
return weightTotal === 0 ? null : round6(weightedSum / weightTotal);
|
|
152
|
+
}
|
|
153
|
+
/**
|
|
154
|
+
* Limited `accuracy` definition: agreement between the full-table inferred
|
|
155
|
+
* type of each declared column and the declared type. Without a declared
|
|
156
|
+
* schema (or an external truth source) the dimension is `null` (undetermined)
|
|
157
|
+
* — accuracy is never fabricated.
|
|
158
|
+
*/
|
|
159
|
+
function computeAccuracy(stats, table, declaredSchema) {
|
|
160
|
+
if (declaredSchema === undefined || Object.keys(declaredSchema).length === 0) {
|
|
161
|
+
return { name: 'accuracy', score: null, note: 'undetermined: no declared schema or external truth to compare against; accuracy is never fabricated' };
|
|
162
|
+
}
|
|
163
|
+
let checked = 0;
|
|
164
|
+
let matched = 0;
|
|
165
|
+
const mismatches = [];
|
|
166
|
+
for (const column of table.columns) {
|
|
167
|
+
const declared = declaredSchema[column];
|
|
168
|
+
if (declared === undefined)
|
|
169
|
+
continue;
|
|
170
|
+
checked += 1;
|
|
171
|
+
const inferred = inferType(stats.get(column));
|
|
172
|
+
if (inferred === declared)
|
|
173
|
+
matched += 1;
|
|
174
|
+
else
|
|
175
|
+
mismatches.push(`${column}: declared ${declared}, inferred ${inferred}`);
|
|
176
|
+
}
|
|
177
|
+
if (checked === 0) {
|
|
178
|
+
return { name: 'accuracy', score: null, note: 'undetermined: declared schema covers none of the dataset columns' };
|
|
179
|
+
}
|
|
180
|
+
return {
|
|
181
|
+
name: 'accuracy',
|
|
182
|
+
score: round6(matched / checked),
|
|
183
|
+
note: `${matched}/${checked} columns match their declared type` + (mismatches.length > 0 ? `; mismatches: ${mismatches.join(', ')}` : ''),
|
|
184
|
+
};
|
|
185
|
+
}
|
|
186
|
+
//# sourceMappingURL=scorecard.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scorecard.js","sourceRoot":"","sources":["../../src/scorecard.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,aAAa,EAAE,YAAY,EAAE,cAAc,EAAc,MAAM,cAAc,CAAA;AAmC/G,6EAA6E;AAC7E,SAAS,MAAM,CAAC,KAAa;IAC3B,OAAO,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAA;AACrC,CAAC;AAED,wCAAwC;AACxC,SAAS,IAAI,CAAC,OAA4B,EAAE,GAAW;IACrD,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;AAC/C,CAAC;AAED,uFAAuF;AACvF,SAAS,SAAS,CAAC,KAAkB;IACnC,IAAI,KAAK,CAAC,OAAO,KAAK,CAAC;QAAE,OAAO,OAAO,CAAA;IACvC,IAAI,KAAK,CAAC,MAAM,KAAK,KAAK,CAAC,OAAO;QAAE,OAAO,QAAQ,CAAA;IACnD,IAAI,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,OAAO;QAAE,OAAO,MAAM,CAAA;IAC/C,IAAI,KAAK,CAAC,OAAO,KAAK,KAAK,CAAC,OAAO;QAAE,OAAO,SAAS,CAAA;IACrD,IAAI,KAAK,CAAC,MAAM,KAAK,KAAK,CAAC,OAAO;QAAE,OAAO,QAAQ,CAAA;IACnD,OAAO,OAAO,CAAA;AAChB,CAAC;AAED,mEAAmE;AACnE,SAAS,SAAS,CAAC,SAAiB,EAAE,WAAmB;IACvD,OAAO,GAAG,SAAS,IAAI,WAAW,EAAE,CAAA;AACtC,CAAC;AAED,4DAA4D;AAC5D,SAAS,SAAS,CAAC,IAA4B,EAAE,KAAa,EAAE,SAAiB,EAAE,WAAmB;IACpG,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,CAAC,SAAS,EAAE,WAAW,CAAC,EAAE,CAAA;AACjE,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,gBAAgB,CAC9B,KAAY,EACZ,OAMC;IAED,cAAc,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;IAC9B,MAAM,KAAK,GAAG,IAAI,GAAG,EAAuB,CAAA;IAC5C,KAAK,MAAM,MAAM,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;QACnC,KAAK,CAAC,GAAG,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,GAAG,EAAE,EAAE,CAAC,CAAA;IAClG,CAAC;IACD,IAAI,YAAY,GAAG,CAAC,CAAA;IACpB,IAAI,SAAS,GAAG,CAAC,CAAA;IACjB,IAAI,eAAe,GAAG,CAAC,CAAA;IAEvB,KAAK,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;QAChD,IAAI,KAAK,GAAG,IAAI,KAAK,CAAC;YAAE,cAAc,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;QACtD,KAAK,MAAM,MAAM,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;YACnC,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,CAAA;YACxB,MAAM,WAAW,GAAG,KAAK,CAAC,GAAG,CAAC,MAAM,CAAgB,CAAA;YACpD,IAAI,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;gBACpB,YAAY,IAAI,CAAC,CAAA;gBACjB,SAAQ;YACV,CAAC;YACD,WAAW,CAAC,OAAO,IAAI,CAAC,CAAA;YACxB,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,CAAA;YAClC,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;gBAC1B,WAAW,CAAC,MAAM,IAAI,CAAC,CAAA;gBACvB,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAA;gBACnC,SAAQ;YACV,CAAC;YACD,MAAM,IAAI,GAAG,aAAa,CAAC,IAAI,CAAC,CAAA;YAChC,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;gBACvB,WAAW,CAAC,IAAI,IAAI,CAAC,CAAA;gBACrB,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;gBACtC,SAAS,IAAI,CAAC,CAAA;gBACd,IAAI,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,GAAG;oBAAE,eAAe,IAAI,CAAC,CAAA;gBAClD,SAAQ;YACV,CAAC;YACD,IAAI,YAAY,CAAC,IAAI,CAAC,KAAK,SAAS,EAAE,CAAC;gBACrC,WAAW,CAAC,OAAO,IAAI,CAAC,CAAA;gBACxB,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,SAAS,CAAC,CAAA;gBACpC,SAAQ;YACV,CAAC;YACD,WAAW,CAAC,MAAM,IAAI,CAAC,CAAA;YACvB,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAA;QACzE,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAA;IAC3D,IAAI,YAAY,GAAG,CAAC,CAAA;IACpB,IAAI,YAAY,GAAG,CAAC,CAAA;IACpB,IAAI,eAAe,GAAG,CAAC,CAAA;IACvB,KAAK,MAAM,CAAC,EAAE,WAAW,CAAC,IAAI,KAAK,EAAE,CAAC;QACpC,YAAY,IAAI,WAAW,CAAC,OAAO,CAAA;QACnC,0EAA0E;QAC1E,2EAA2E;QAC3E,oDAAoD;QACpD,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,MAAM,EAAE,WAAW,CAAC,IAAI,EAAE,WAAW,CAAC,OAAO,EAAE,WAAW,CAAC,MAAM,CAAC,CAAA;QAC7G,YAAY,IAAI,WAAW,CAAC,OAAO,GAAG,aAAa,CAAA;QACnD,IAAI,WAAW,CAAC,OAAO,KAAK,CAAC;YAAE,SAAQ;QACvC,IAAI,QAAQ,GAAG,CAAC,CAAA;QAChB,KAAK,MAAM,KAAK,IAAI,WAAW,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;YACjD,IAAI,KAAK,GAAG,QAAQ;gBAAE,QAAQ,GAAG,KAAK,CAAA;QACxC,CAAC;QACD,eAAe,IAAI,QAAQ,CAAA;IAC7B,CAAC;IAED,MAAM,YAAY,GAChB,UAAU,KAAK,CAAC;QACd,CAAC,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,wBAAwB,EAAE;QACvE,CAAC,CAAC,SAAS,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC,GAAG,YAAY,GAAG,UAAU,CAAC,EAAE,YAAY,EAAE,UAAU,CAAC,CAAA;IAEhG,MAAM,UAAU,GACd,KAAK,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC;QACrB,CAAC,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,uBAAuB,EAAE;QACpE,CAAC,CAAC,SAAS,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,GAAG,OAAO,CAAC,aAAa,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,aAAa,EAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;IAElJ,MAAM,QAAQ,GACZ,YAAY,KAAK,CAAC;QAChB,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,gCAAgC,EAAE;QAC3E,CAAC,CAAC,SAAS,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,GAAG,YAAY,GAAG,YAAY,CAAC,EAAE,YAAY,GAAG,YAAY,EAAE,YAAY,CAAC,CAAA;IAE/G,MAAM,WAAW,GACf,YAAY,KAAK,CAAC;QAChB,CAAC,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,gCAAgC,EAAE;QAC9E,CAAC,CAAC,SAAS,CAAC,aAAa,EAAE,MAAM,CAAC,eAAe,GAAG,YAAY,CAAC,EAAE,eAAe,EAAE,YAAY,CAAC,CAAA;IAErG,MAAM,UAAU,GACd,SAAS,KAAK,CAAC;QACb,CAAC,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,6BAA6B,EAAE;QAC1E,CAAC,CAAC,SAAS,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,GAAG,eAAe,GAAG,SAAS,CAAC,EAAE,SAAS,GAAG,eAAe,EAAE,SAAS,CAAC,CAAA;IAE9G,MAAM,QAAQ,GAAG,eAAe,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,cAAc,CAAC,CAAA;IAEtE,MAAM,UAAU,GAAG,CAAC,YAAY,EAAE,UAAU,EAAE,QAAQ,EAAE,WAAW,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAA;IAC1F,MAAM,MAAM,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAmB,EAAE,CAAC,KAAK,KAAK,IAAI,CAAC,CAAA;IACxG,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAA;IAClH,MAAM,eAAe,GAAG,sBAAsB,CAAC,UAAU,EAAE,OAAO,CAAC,OAAO,CAAC,CAAA;IAC3E,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,eAAe,EAAE,CAAA;AACjD,CAAC;AAED,2GAA2G;AAC3G,SAAS,sBAAsB,CAC7B,UAAyC,EACzC,OAAqE;IAErE,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;QAC/B,IAAI,KAAK,CAAC,KAAK,KAAK,IAAI;YAAE,SAAQ;QAClC,MAAM,MAAM,GAAG,OAAO,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACzC,WAAW,IAAI,MAAM,GAAG,KAAK,CAAC,KAAK,CAAA;QACnC,WAAW,IAAI,MAAM,CAAA;IACvB,CAAC;IACD,OAAO,WAAW,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,GAAG,WAAW,CAAC,CAAA;AACrE,CAAC;AAED;;;;;GAKG;AACH,SAAS,eAAe,CACtB,KAAuC,EACvC,KAAY,EACZ,cAAkE;IAElE,IAAI,cAAc,KAAK,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7E,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,qGAAqG,EAAE,CAAA;IACvJ,CAAC;IACD,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,MAAM,UAAU,GAAa,EAAE,CAAA;IAC/B,KAAK,MAAM,MAAM,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;QACnC,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,CAAC,CAAA;QACvC,IAAI,QAAQ,KAAK,SAAS;YAAE,SAAQ;QACpC,OAAO,IAAI,CAAC,CAAA;QACZ,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAgB,CAAC,CAAA;QAC5D,IAAI,QAAQ,KAAK,QAAQ;YAAE,OAAO,IAAI,CAAC,CAAA;;YAClC,UAAU,CAAC,IAAI,CAAC,GAAG,MAAM,cAAc,QAAQ,cAAc,QAAQ,EAAE,CAAC,CAAA;IAC/E,CAAC;IACD,IAAI,OAAO,KAAK,CAAC,EAAE,CAAC;QAClB,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,kEAAkE,EAAE,CAAA;IACpH,CAAC;IACD,OAAO;QACL,IAAI,EAAE,UAAU;QAChB,KAAK,EAAE,MAAM,CAAC,OAAO,GAAG,OAAO,CAAC;QAChC,IAAI,EAAE,GAAG,OAAO,IAAI,OAAO,oCAAoC,GAAG,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,iBAAiB,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC1I,CAAA;AACH,CAAC"}
|
package/lib/types/service.d.ts
CHANGED
|
@@ -10,8 +10,10 @@
|
|
|
10
10
|
import { Service, type Context } from '@deepseek-ai/cordis';
|
|
11
11
|
import type { JsonValue, Session } from '@deepseek-ai/dsh-session';
|
|
12
12
|
import type { CleanRule, CleanRuleLog } from './clean.js';
|
|
13
|
+
import type { CleanContractSummary, CleanProfileDiff } from './contract.js';
|
|
13
14
|
import type { ProfileReport } from './profile.js';
|
|
14
|
-
import type { VerifyReport, VerifyRule } from './verify.js';
|
|
15
|
+
import type { VerifyExpectation, VerifyReport, VerifyRule } from './verify.js';
|
|
16
|
+
import type { ReportRecord, StoredReport } from './store.js';
|
|
15
17
|
export interface CitationCheckRequest {
|
|
16
18
|
/** Workspace-relative path of the source dataset snapshot (CSV/JSON). */
|
|
17
19
|
dataset: string;
|
|
@@ -43,6 +45,8 @@ export interface ProfileRequest {
|
|
|
43
45
|
readonly dataset: string;
|
|
44
46
|
/** Optional deterministic systematic sample size for column cards. */
|
|
45
47
|
readonly sample?: number | undefined;
|
|
48
|
+
/** Optional industry preset id; its expected columns feed the scorecard's declared schema. */
|
|
49
|
+
readonly industryPreset?: string | undefined;
|
|
46
50
|
/** Absolute workspace root the dataset resolves inside. */
|
|
47
51
|
readonly workspace: string;
|
|
48
52
|
/** Calling session (receives the `data-quality/profile` event), when any. */
|
|
@@ -58,6 +62,8 @@ export interface CleanRequest {
|
|
|
58
62
|
readonly rules: readonly CleanRule[];
|
|
59
63
|
/** Workspace-relative output path; omitted = no disk write, preview only. */
|
|
60
64
|
readonly outputPath?: string | undefined;
|
|
65
|
+
/** Dry run: no file written, no report persisted; returns the plan + expected contract/diff preview. */
|
|
66
|
+
readonly dryRun?: boolean | undefined;
|
|
61
67
|
/** Absolute workspace root the dataset resolves inside. */
|
|
62
68
|
readonly workspace: string;
|
|
63
69
|
/** Calling session (receives the `data-quality/clean` event), when any. */
|
|
@@ -71,6 +77,8 @@ export interface VerifyRequest {
|
|
|
71
77
|
readonly dataset: string;
|
|
72
78
|
/** Non-empty declarative verification rule list. */
|
|
73
79
|
readonly rules: readonly VerifyRule[];
|
|
80
|
+
/** Optional metric expectations to reconcile (rowCount/columnSum/columnMean/uniqueCount/nullCount). */
|
|
81
|
+
readonly expectations?: readonly VerifyExpectation[] | undefined;
|
|
74
82
|
/** Absolute workspace root the dataset resolves inside. */
|
|
75
83
|
readonly workspace: string;
|
|
76
84
|
/** Calling session (receives the `data-quality/verify` event), when any. */
|
|
@@ -83,12 +91,18 @@ export interface CleanRunReport {
|
|
|
83
91
|
readonly dataset: string;
|
|
84
92
|
readonly inputRows: number;
|
|
85
93
|
readonly outputRows: number;
|
|
94
|
+
/** Whether this was a dry run (no file written, no report persisted). */
|
|
95
|
+
readonly dryRun: boolean;
|
|
86
96
|
readonly logs: CleanRuleLog[];
|
|
97
|
+
/** Pre-delivery contract validation summary (dedupe/uniqueness/non-null/type/decision regressions). */
|
|
98
|
+
readonly contract: CleanContractSummary;
|
|
87
99
|
/** First `evidenceRowLimit` cleaned rows for inspection (display-truncated). */
|
|
88
100
|
readonly preview: {
|
|
89
101
|
readonly columns: string[];
|
|
90
102
|
readonly rows: Array<Record<string, JsonValue>>;
|
|
91
103
|
};
|
|
104
|
+
/** Expected before/after profile diff preview; present only for dry runs. */
|
|
105
|
+
readonly diffPreview?: CleanProfileDiff;
|
|
92
106
|
/** Workspace-relative output path when the run wrote a file. */
|
|
93
107
|
readonly outputPath?: string;
|
|
94
108
|
/** Storage-domain key of the persisted report, when persistence is on. */
|
|
@@ -146,6 +160,20 @@ export declare abstract class DataQualityService extends Service {
|
|
|
146
160
|
* @returns the verify report.
|
|
147
161
|
*/
|
|
148
162
|
abstract verifyDataset(request: VerifyRequest): Promise<VerifyReport>;
|
|
163
|
+
/**
|
|
164
|
+
* Read one persisted report by its storage key. The key is validated
|
|
165
|
+
* against the deterministic report-key format (path-safe) and a missing
|
|
166
|
+
* record fails loud.
|
|
167
|
+
* @param key - the storage report key.
|
|
168
|
+
* @returns the stored report (with its key).
|
|
169
|
+
*/
|
|
170
|
+
abstract getReport(key: string): Promise<StoredReport>;
|
|
171
|
+
/**
|
|
172
|
+
* List every persisted report of one kind, ordered by key (chronological).
|
|
173
|
+
* @param kind - the report kind.
|
|
174
|
+
* @returns the stored reports (empty when none match).
|
|
175
|
+
*/
|
|
176
|
+
abstract listReports(kind: ReportRecord['kind']): Promise<StoredReport[]>;
|
|
149
177
|
}
|
|
150
178
|
declare module '@deepseek-ai/cordis' {
|
|
151
179
|
interface Context {
|
|
@@ -155,6 +183,8 @@ declare module '@deepseek-ai/cordis' {
|
|
|
155
183
|
}
|
|
156
184
|
/** Re-exports so consumers pull the whole seam vocabulary from one module. */
|
|
157
185
|
export type { CleanResult, CleanRule, CleanRuleLog } from './clean.js';
|
|
186
|
+
export type { CleanContractSummary, CleanProfileDiff } from './contract.js';
|
|
158
187
|
export type { ProfileReport } from './profile.js';
|
|
159
|
-
export type { VerifyReport, VerifyRule } from './verify.js';
|
|
188
|
+
export type { VerifyReport, VerifyRule, VerifyExpectation, VerifyExpectationResult, VerifyMetric } from './verify.js';
|
|
189
|
+
export type { ReportRecord, StoredReport } from './store.js';
|
|
160
190
|
//# sourceMappingURL=service.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"service.d.ts","sourceRoot":"","sources":["../../src/service.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,OAAO,EAAE,KAAK,OAAO,EAAE,MAAM,qBAAqB,CAAA;AAC3D,OAAO,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAA;AAClE,OAAO,KAAK,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AACzD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,cAAc,CAAA;AACjD,OAAO,KAAK,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;
|
|
1
|
+
{"version":3,"file":"service.d.ts","sourceRoot":"","sources":["../../src/service.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,OAAO,EAAE,KAAK,OAAO,EAAE,MAAM,qBAAqB,CAAA;AAC3D,OAAO,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAA;AAClE,OAAO,KAAK,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AACzD,OAAO,KAAK,EAAE,oBAAoB,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAA;AAC3E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,cAAc,CAAA;AACjD,OAAO,KAAK,EAAE,iBAAiB,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AAC9E,OAAO,KAAK,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AAE5D,MAAM,WAAW,oBAAoB;IACnC,yEAAyE;IACzE,OAAO,EAAE,MAAM,CAAA;IACf,+CAA+C;IAC/C,SAAS,EAAE,KAAK,CAAC;QACf,8DAA8D;QAC9D,EAAE,EAAE,MAAM,CAAA;QACV,2EAA2E;QAC3E,IAAI,EAAE,MAAM,CAAA;QACZ,0CAA0C;QAC1C,KAAK,EAAE,MAAM,GAAG,MAAM,CAAA;QACtB,0EAA0E;QAC1E,SAAS,CAAC,EAAE,MAAM,CAAA;KACnB,CAAC,CAAA;CACH;AACD,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,KAAK,CAAC;QACb,EAAE,EAAE,MAAM,CAAA;QACV,MAAM,EAAE,UAAU,GAAG,UAAU,GAAG,WAAW,GAAG,cAAc,CAAA;QAC9D,8CAA8C;QAC9C,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,CAAA;QACxB,oCAAoC;QACpC,IAAI,CAAC,EAAE,MAAM,CAAA;KACd,CAAC,CAAA;CACH;AAED,oFAAoF;AACpF,MAAM,WAAW,cAAc;IAC7B,uCAAuC;IACvC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,sEAAsE;IACtE,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACpC,8FAA8F;IAC9F,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IAC5C,2DAA2D;IAC3D,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,6EAA6E;IAC7E,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,GAAG,SAAS,CAAA;IACtC,uCAAuC;IACvC,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,SAAS,CAAA;CAC1C;AAED,+CAA+C;AAC/C,MAAM,WAAW,YAAY;IAC3B,uCAAuC;IACvC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,kEAAkE;IAClE,QAAQ,CAAC,KAAK,EAAE,SAAS,SAAS,EAAE,CAAA;IACpC,6EAA6E;IAC7E,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IACxC,wGAAwG;IACxG,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,GAAG,SAAS,CAAA;IACrC,2DAA2D;IAC3D,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,2EAA2E;IAC3E,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,GAAG,SAAS,CAAA;IACtC,uCAAuC;IACvC,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,SAAS,CAAA;CAC1C;AAED,gDAAgD;AAChD,MAAM,WAAW,aAAa;IAC5B,uCAAuC;IACvC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,oDAAoD;IACpD,QAAQ,CAAC,KAAK,EAAE,SAAS,UAAU,EAAE,CAAA;IACrC,uGAAuG;IACvG,QAAQ,CAAC,YAAY,CAAC,EAAE,SAAS,iBAAiB,EAAE,GAAG,SAAS,CAAA;IAChE,2DAA2D;IAC3D,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,4EAA4E;IAC5E,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,GAAG,SAAS,CAAA;IACtC,uCAAuC;IACvC,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,SAAS,CAAA;CAC1C;AAED,+EAA+E;AAC/E,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,yEAAyE;IACzE,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAA;IACxB,QAAQ,CAAC,IAAI,EAAE,YAAY,EAAE,CAAA;IAC7B,uGAAuG;IACvG,QAAQ,CAAC,QAAQ,EAAE,oBAAoB,CAAA;IACvC,gFAAgF;IAChF,QAAQ,CAAC,OAAO,EAAE;QAAE,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;QAAC,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC,CAAA;KAAE,CAAA;IACjG,6EAA6E;IAC7E,QAAQ,CAAC,WAAW,CAAC,EAAE,gBAAgB,CAAA;IACvC,gEAAgE;IAChE,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAA;IAC5B,0EAA0E;IAC1E,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;CAC7B;AAED,mFAAmF;AACnF,MAAM,WAAW,sBAAsB;IACrC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;IACzB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAA;IAC7B,QAAQ,CAAC,OAAO,EAAE,mBAAmB,CAAC,SAAS,CAAC,CAAA;IAChD,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;CAC7B;AAED;;;;GAIG;AACH,8BAAsB,kBAAmB,SAAQ,OAAO;IACtD;;;OAGG;gBACS,GAAG,EAAE,OAAO;IAIxB;;;;;;OAMG;IACH,QAAQ,CAAC,eAAe,CAAC,OAAO,EAAE,oBAAoB,GAAG,OAAO,CAAC,mBAAmB,CAAC;IAErF;;;;;OAKG;IACH,QAAQ,CAAC,cAAc,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC;IAExE;;;;;OAKG;IACH,QAAQ,CAAC,YAAY,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,cAAc,CAAC;IAErE;;;;;OAKG;IACH,QAAQ,CAAC,aAAa,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC;IAErE;;;;;;OAMG;IACH,QAAQ,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IAEtD;;;;OAIG;IACH,QAAQ,CAAC,WAAW,CAAC,IAAI,EAAE,YAAY,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;CAC1E;AAED,OAAO,QAAQ,qBAAqB,CAAC;IACnC,UAAU,OAAO;QACf,2GAA2G;QAC3G,WAAW,EAAE,kBAAkB,CAAA;KAChC;CACF;AAED,8EAA8E;AAC9E,YAAY,EAAE,WAAW,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AACtE,YAAY,EAAE,oBAAoB,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAA;AAC3E,YAAY,EAAE,aAAa,EAAE,MAAM,cAAc,CAAA;AACjD,YAAY,EAAE,YAAY,EAAE,UAAU,EAAE,iBAAiB,EAAE,uBAAuB,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AACrH,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA"}
|
package/lib/types/service.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"service.js","sourceRoot":"","sources":["../../src/service.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,OAAO,EAAgB,MAAM,qBAAqB,CAAA;
|
|
1
|
+
{"version":3,"file":"service.js","sourceRoot":"","sources":["../../src/service.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,OAAO,EAAgB,MAAM,qBAAqB,CAAA;AAqH3D;;;;GAIG;AACH,MAAM,OAAgB,kBAAmB,SAAQ,OAAO;IACtD;;;OAGG;IACH,YAAY,GAAY;QACtB,KAAK,CAAC,GAAG,EAAE,aAAa,CAAC,CAAA;IAC3B,CAAC;CAkDF"}
|
package/lib/types/store.d.ts
CHANGED
|
@@ -13,6 +13,7 @@ export declare const reportRecordSchema: z.ZodObject<{
|
|
|
13
13
|
clean: "clean";
|
|
14
14
|
verify: "verify";
|
|
15
15
|
citations: "citations";
|
|
16
|
+
"clean-diff": "clean-diff";
|
|
16
17
|
}>;
|
|
17
18
|
at: z.ZodNumber;
|
|
18
19
|
dataset: z.ZodString;
|
|
@@ -20,7 +21,7 @@ export declare const reportRecordSchema: z.ZodObject<{
|
|
|
20
21
|
}, z.core.$strip>;
|
|
21
22
|
/** One persisted report record. */
|
|
22
23
|
export interface ReportRecord {
|
|
23
|
-
readonly kind: 'profile' | 'clean' | 'verify' | 'citations';
|
|
24
|
+
readonly kind: 'profile' | 'clean' | 'verify' | 'citations' | 'clean-diff';
|
|
24
25
|
/** Injected run timestamp (epoch ms). */
|
|
25
26
|
readonly at: number;
|
|
26
27
|
/** Workspace-relative dataset path as the caller gave it. */
|
|
@@ -28,6 +29,11 @@ export interface ReportRecord {
|
|
|
28
29
|
/** The full run report (profile/clean/verify/citations shape by `kind`). */
|
|
29
30
|
readonly report: Record<string, unknown>;
|
|
30
31
|
}
|
|
32
|
+
/** A persisted report plus its storage key (the query surface's return value). */
|
|
33
|
+
export interface StoredReport extends ReportRecord {
|
|
34
|
+
/** The storage key the record was written under. */
|
|
35
|
+
readonly key: string;
|
|
36
|
+
}
|
|
31
37
|
/** The `dsh-data-quality` storage-domain declaration. */
|
|
32
38
|
export declare const dataQualityDomainSpec: {
|
|
33
39
|
name: string;
|
|
@@ -50,7 +56,21 @@ export interface ReportStore {
|
|
|
50
56
|
* @returns the record, or `undefined` when absent.
|
|
51
57
|
*/
|
|
52
58
|
get(key: string): ReportRecord | undefined;
|
|
59
|
+
/**
|
|
60
|
+
* Snapshot of every record of one kind, ordered by key (chronological).
|
|
61
|
+
* @param kind - the report kind.
|
|
62
|
+
* @returns the stored reports (empty when none match).
|
|
63
|
+
*/
|
|
64
|
+
list(kind: ReportRecord['kind']): StoredReport[];
|
|
53
65
|
}
|
|
66
|
+
/**
|
|
67
|
+
* Whether `key` is a well-formed, path-safe storage report key. Rejects any
|
|
68
|
+
* key with separators, traversal, or unexpected characters before it can be
|
|
69
|
+
* handed to the storage backend.
|
|
70
|
+
* @param key - candidate report key.
|
|
71
|
+
* @returns whether the key matches the deterministic report-key format.
|
|
72
|
+
*/
|
|
73
|
+
export declare function isValidReportKey(key: string): boolean;
|
|
54
74
|
/**
|
|
55
75
|
* Build the storage key for one run: `YYYYMMDDHHmmssSSS-<kind>-<pathFp>`.
|
|
56
76
|
* UTC digits keep the key timezone-independent and filename-safe everywhere.
|
package/lib/types/store.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../src/store.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,CAAC,MAAM,KAAK,CAAA;AAInB,+EAA+E;AAC/E,eAAO,MAAM,kBAAkB
|
|
1
|
+
{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../src/store.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,CAAC,MAAM,KAAK,CAAA;AAInB,+EAA+E;AAC/E,eAAO,MAAM,kBAAkB;;;;;;;;;;;iBAK7B,CAAA;AAEF,mCAAmC;AACnC,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EAAE,SAAS,GAAG,OAAO,GAAG,QAAQ,GAAG,WAAW,GAAG,YAAY,CAAA;IAC1E,yCAAyC;IACzC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;IACnB,6DAA6D;IAC7D,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,4EAA4E;IAC5E,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACzC;AAED,kFAAkF;AAClF,MAAM,WAAW,YAAa,SAAQ,YAAY;IAChD,oDAAoD;IACpD,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;CACrB;AAED,yDAAyD;AACzD,eAAO,MAAM,qBAAqB;;;;;;CAMhC,CAAA;AAEF,yEAAyE;AACzE,MAAM,WAAW,WAAW;IAC1B;;;;OAIG;IACH,GAAG,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAAA;IAC1C;;;;OAIG;IACH,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,YAAY,GAAG,SAAS,CAAA;IAC1C;;;;OAIG;IACH,IAAI,CAAC,IAAI,EAAE,YAAY,CAAC,MAAM,CAAC,GAAG,YAAY,EAAE,CAAA;CACjD;AAKD;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAErD;AAOD;;;;;GAKG;AACH,wBAAgB,WAAW,CAAC,MAAM,EAAE,YAAY,GAAG,MAAM,CAMxD"}
|
package/lib/types/store.js
CHANGED
|
@@ -10,7 +10,7 @@ import { defineDomain, domainTable } from '@deepseek-ai/dsh-storage-domain';
|
|
|
10
10
|
import { pathFingerprint } from "./dataset.js";
|
|
11
11
|
/** Zod schema of one persisted report record (durable-boundary validation). */
|
|
12
12
|
export const reportRecordSchema = z.object({
|
|
13
|
-
kind: z.enum(['profile', 'clean', 'verify', 'citations']),
|
|
13
|
+
kind: z.enum(['profile', 'clean', 'verify', 'citations', 'clean-diff']),
|
|
14
14
|
at: z.number().int().nonnegative(),
|
|
15
15
|
dataset: z.string(),
|
|
16
16
|
report: z.record(z.string(), z.unknown()),
|
|
@@ -23,6 +23,18 @@ export const dataQualityDomainSpec = defineDomain({
|
|
|
23
23
|
reports: domainTable(reportRecordSchema),
|
|
24
24
|
},
|
|
25
25
|
});
|
|
26
|
+
/** Well-formed report-key shape: `<17-digit timestamp>-<kind>-<8-hex fingerprint>`. */
|
|
27
|
+
const REPORT_KEY_PATTERN = /^\d{17}-(?:profile|clean|verify|citations|clean-diff)-[0-9a-f]{8}$/u;
|
|
28
|
+
/**
|
|
29
|
+
* Whether `key` is a well-formed, path-safe storage report key. Rejects any
|
|
30
|
+
* key with separators, traversal, or unexpected characters before it can be
|
|
31
|
+
* handed to the storage backend.
|
|
32
|
+
* @param key - candidate report key.
|
|
33
|
+
* @returns whether the key matches the deterministic report-key format.
|
|
34
|
+
*/
|
|
35
|
+
export function isValidReportKey(key) {
|
|
36
|
+
return REPORT_KEY_PATTERN.test(key);
|
|
37
|
+
}
|
|
26
38
|
/** Pad to two digits for the key timestamp. */
|
|
27
39
|
function pad2(value) {
|
|
28
40
|
return String(value).padStart(2, '0');
|
package/lib/types/store.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"store.js","sourceRoot":"","sources":["../../src/store.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,CAAC,MAAM,KAAK,CAAA;AACnB,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAA;AAC3E,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAE9C,+EAA+E;AAC/E,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IACzC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,OAAO,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"store.js","sourceRoot":"","sources":["../../src/store.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,CAAC,MAAM,KAAK,CAAA;AACnB,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAA;AAC3E,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAE9C,+EAA+E;AAC/E,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IACzC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,YAAY,CAAC,CAAC;IACvE,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;IAClC,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;IACnB,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;CAC1C,CAAC,CAAA;AAmBF,yDAAyD;AACzD,MAAM,CAAC,MAAM,qBAAqB,GAAG,YAAY,CAAC;IAChD,IAAI,EAAE,cAAc;IACpB,OAAO,EAAE,CAAC;IACV,MAAM,EAAE;QACN,OAAO,EAAE,WAAW,CAAuB,kBAAkB,CAAC;KAC/D;CACF,CAAC,CAAA;AAwBF,uFAAuF;AACvF,MAAM,kBAAkB,GAAG,qEAAqE,CAAA;AAEhG;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAAC,GAAW;IAC1C,OAAO,kBAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;AACrC,CAAC;AAED,+CAA+C;AAC/C,SAAS,IAAI,CAAC,KAAa;IACzB,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAA;AACvC,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,WAAW,CAAC,MAAoB;IAC9C,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;IAChC,MAAM,KAAK,GACT,GAAG,IAAI,CAAC,cAAc,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE;QACnF,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,kBAAkB,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAA;IAC9I,OAAO,GAAG,KAAK,IAAI,MAAM,CAAC,IAAI,IAAI,eAAe,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAA;AACrE,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"clean.d.ts","sourceRoot":"","sources":["../../../src/tools/clean.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,KAAK,EAA6B,kBAAkB,EAAE,MAAM,eAAe,CAAA;
|
|
1
|
+
{"version":3,"file":"clean.d.ts","sourceRoot":"","sources":["../../../src/tools/clean.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,KAAK,EAA6B,kBAAkB,EAAE,MAAM,eAAe,CAAA;AA2ElF;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,kBAAkB,mDAwI1D"}
|
package/lib/types/tools/clean.js
CHANGED
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
* @module dsh-data-quality/tools/clean
|
|
6
6
|
*/
|
|
7
7
|
import { defineTool } from '@deepseek-ai/dsh-tools';
|
|
8
|
+
import { PROFILE_REPORT_SCHEMA } from "./profile-report-schema.js";
|
|
8
9
|
import { renderCleanText, workspaceOf } from "./shared.js";
|
|
9
10
|
const STRING_MAP_SCHEMA = { type: 'json' };
|
|
10
11
|
const CLEAN_RULE_SCHEMA = {
|
|
@@ -86,12 +87,13 @@ export function defineCleanTool(service) {
|
|
|
86
87
|
description: [
|
|
87
88
|
'Apply declarative cleaning rules to a workspace CSV/TSV/JSON/JSONL dataset with deterministic TypeScript computation (no mental math).',
|
|
88
89
|
'Rules apply in array order: dedupe (by column group), fill-missing (constant/mean/median/forward), coerce-type (number/date/boolean; failures counted and set to missing), normalize-unit (e.g. 万/亿 suffixes to base units), trim (whitespace), map-values (enum mapping).',
|
|
89
|
-
'The source file is NEVER overwritten. Without outputPath the run is preview-only; with outputPath the cleaned dataset is written there (workspace-confined, .csv/.tsv/.json/.jsonl). Returns the per-rule audit log (
|
|
90
|
+
'The source file is NEVER overwritten. Without outputPath the run is preview-only; with outputPath the cleaned dataset is written there (workspace-confined, .csv/.tsv/.json/.jsonl). Returns the per-rule audit log, the pre-delivery contract summary (with per-column decision trace), and a bounded preview. Pass dryRun: true to skip the write and get the cleaning plan plus the expected contract/diff preview instead. The full report persists to the data_quality storage domain (reportKey).',
|
|
90
91
|
].join('\n'),
|
|
91
92
|
parameters: {
|
|
92
93
|
path: { type: 'string', required: true, description: 'Workspace-relative dataset path (.csv/.tsv/.json/.jsonl).' },
|
|
93
94
|
rules: { ...CLEAN_RULE_SCHEMA, required: true },
|
|
94
95
|
outputPath: { type: 'string', description: 'Optional workspace-relative output path for the cleaned dataset (must differ from path).' },
|
|
96
|
+
dryRun: { type: 'boolean', description: 'When true, do not write any output file; return the cleaning plan and expected contract/diff preview instead (default false).' },
|
|
95
97
|
},
|
|
96
98
|
output: {
|
|
97
99
|
schema: {
|
|
@@ -100,9 +102,20 @@ export function defineCleanTool(service) {
|
|
|
100
102
|
dataset: { type: 'string', required: true },
|
|
101
103
|
inputRows: { type: 'number', required: true },
|
|
102
104
|
outputRows: { type: 'number', required: true },
|
|
105
|
+
dryRun: { type: 'boolean', required: true },
|
|
103
106
|
generatedAt: { type: 'number', required: true },
|
|
104
107
|
outputPath: { type: 'string' },
|
|
105
108
|
reportKey: { type: 'string' },
|
|
109
|
+
diffPreview: {
|
|
110
|
+
type: 'object',
|
|
111
|
+
properties: {
|
|
112
|
+
dataset: { type: 'string', required: true },
|
|
113
|
+
before: { ...PROFILE_REPORT_SCHEMA, required: true },
|
|
114
|
+
after: { ...PROFILE_REPORT_SCHEMA, required: true },
|
|
115
|
+
generatedAt: { type: 'number', required: true },
|
|
116
|
+
},
|
|
117
|
+
additionalProperties: false,
|
|
118
|
+
},
|
|
106
119
|
logs: {
|
|
107
120
|
type: 'array',
|
|
108
121
|
items: {
|
|
@@ -117,6 +130,67 @@ export function defineCleanTool(service) {
|
|
|
117
130
|
},
|
|
118
131
|
required: true,
|
|
119
132
|
},
|
|
133
|
+
contract: {
|
|
134
|
+
type: 'object',
|
|
135
|
+
properties: {
|
|
136
|
+
inputRows: { type: 'number', required: true },
|
|
137
|
+
outputRows: { type: 'number', required: true },
|
|
138
|
+
removedRows: { type: 'number', required: true },
|
|
139
|
+
dedupeColumns: { oneOf: [{ type: 'array', items: { type: 'string' } }, { type: 'null' }], required: true },
|
|
140
|
+
uniqueKeys: { type: 'boolean', required: true },
|
|
141
|
+
remainingDuplicateRows: { type: 'number', required: true },
|
|
142
|
+
remainingMissing: {
|
|
143
|
+
type: 'array',
|
|
144
|
+
items: {
|
|
145
|
+
type: 'object',
|
|
146
|
+
properties: {
|
|
147
|
+
column: { type: 'string', required: true },
|
|
148
|
+
count: { type: 'number', required: true },
|
|
149
|
+
},
|
|
150
|
+
additionalProperties: false,
|
|
151
|
+
},
|
|
152
|
+
required: true,
|
|
153
|
+
},
|
|
154
|
+
typeConformance: {
|
|
155
|
+
type: 'array',
|
|
156
|
+
items: {
|
|
157
|
+
type: 'object',
|
|
158
|
+
properties: {
|
|
159
|
+
column: { type: 'string', required: true },
|
|
160
|
+
to: { type: 'string', enum: ['number', 'date', 'boolean'], required: true },
|
|
161
|
+
invalidCount: { type: 'number', required: true },
|
|
162
|
+
},
|
|
163
|
+
additionalProperties: false,
|
|
164
|
+
},
|
|
165
|
+
required: true,
|
|
166
|
+
},
|
|
167
|
+
columnDecisions: {
|
|
168
|
+
type: 'array',
|
|
169
|
+
items: {
|
|
170
|
+
type: 'object',
|
|
171
|
+
properties: {
|
|
172
|
+
column: { type: 'string', required: true },
|
|
173
|
+
decisions: {
|
|
174
|
+
type: 'array',
|
|
175
|
+
items: {
|
|
176
|
+
type: 'object',
|
|
177
|
+
properties: {
|
|
178
|
+
strategy: { type: 'string', required: true },
|
|
179
|
+
affectedRows: { type: 'number', required: true },
|
|
180
|
+
},
|
|
181
|
+
additionalProperties: false,
|
|
182
|
+
},
|
|
183
|
+
required: true,
|
|
184
|
+
},
|
|
185
|
+
},
|
|
186
|
+
additionalProperties: false,
|
|
187
|
+
},
|
|
188
|
+
required: true,
|
|
189
|
+
},
|
|
190
|
+
},
|
|
191
|
+
additionalProperties: false,
|
|
192
|
+
required: true,
|
|
193
|
+
},
|
|
120
194
|
preview: {
|
|
121
195
|
type: 'object',
|
|
122
196
|
properties: {
|
|
@@ -136,6 +210,7 @@ export function defineCleanTool(service) {
|
|
|
136
210
|
dataset: args.path,
|
|
137
211
|
rules: args.rules,
|
|
138
212
|
...(args.outputPath !== undefined ? { outputPath: args.outputPath } : {}),
|
|
213
|
+
...(args.dryRun !== undefined ? { dryRun: args.dryRun } : {}),
|
|
139
214
|
workspace: workspaceOf(exec),
|
|
140
215
|
session: exec.agent?.session,
|
|
141
216
|
signal: exec.signal,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"clean.js","sourceRoot":"","sources":["../../../src/tools/clean.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAA;AAEnD,OAAO,EAAE,eAAe,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AAE1D,MAAM,iBAAiB,GAAG,EAAE,IAAI,EAAE,MAAM,EAAW,CAAA;AAEnD,MAAM,iBAAiB,GAAG;IACxB,IAAI,EAAE,OAAO;IACb,KAAK,EAAE;QACL,KAAK,EAAE;YACL;gBACE,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE;oBACzD,OAAO,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,WAAW,EAAE,wCAAwC,EAAE;iBAC7G;gBACD,oBAAoB,EAAE,KAAK;gBAC3B,WAAW,EAAE,uFAAuF;aACrG;YACD;gBACE,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,cAAc,EAAE,QAAQ,EAAE,IAAI,EAAE;oBAC/D,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE;oBAC1C,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,UAAU,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE;oBAC7F,KAAK,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,EAAE,WAAW,EAAE,mCAAmC,EAAE;iBAC7G;gBACD,oBAAoB,EAAE,KAAK;gBAC3B,WAAW,EAAE,8GAA8G;aAC5H;YACD;gBACE,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,aAAa,EAAE,QAAQ,EAAE,IAAI,EAAE;oBAC9D,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE;oBAC1C,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE;iBAC5E;gBACD,oBAAoB,EAAE,KAAK;gBAC3B,WAAW,EAAE,kGAAkG;aAChH;YACD;gBACE,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,gBAAgB,EAAE,QAAQ,EAAE,IAAI,EAAE;oBACjE,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE;oBAC1C,OAAO,EAAE,EAAE,GAAG,iBAAiB,EAAE,WAAW,EAAE,+DAA+D,EAAE;iBAChH;gBACD,oBAAoB,EAAE,KAAK;gBAC3B,WAAW,EAAE,mFAAmF;aACjG;YACD;gBACE,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE;oBACvD,OAAO,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,WAAW,EAAE,4CAA4C,EAAE;iBACjH;gBACD,oBAAoB,EAAE,KAAK;gBAC3B,WAAW,EAAE,mDAAmD;aACjE;YACD;gBACE,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,YAAY,EAAE,QAAQ,EAAE,IAAI,EAAE;oBAC7D,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE;oBAC1C,GAAG,EAAE,EAAE,GAAG,iBAAiB,EAAE,WAAW,EAAE,0DAA0D,EAAE;oBACtG,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,MAAM,EAAE,SAAS,CAAC,EAAE,WAAW,EAAE,4CAA4C,EAAE;iBAC/G;gBACD,oBAAoB,EAAE,KAAK;gBAC3B,WAAW,EAAE,iCAAiC;aAC/C;SACF;KACF;IACD,WAAW,EAAE,qEAAqE;CAC1E,CAAA;AAEV;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,OAA2B;IACzD,OAAO,UAAU,CAAC;QAChB,IAAI,EAAE,YAAY;QAClB,WAAW,EAAE;YACX,wIAAwI;YACxI,4QAA4Q;YAC5Q
|
|
1
|
+
{"version":3,"file":"clean.js","sourceRoot":"","sources":["../../../src/tools/clean.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAA;AAEnD,OAAO,EAAE,qBAAqB,EAAE,MAAM,4BAA4B,CAAA;AAClE,OAAO,EAAE,eAAe,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AAE1D,MAAM,iBAAiB,GAAG,EAAE,IAAI,EAAE,MAAM,EAAW,CAAA;AAEnD,MAAM,iBAAiB,GAAG;IACxB,IAAI,EAAE,OAAO;IACb,KAAK,EAAE;QACL,KAAK,EAAE;YACL;gBACE,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE;oBACzD,OAAO,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,WAAW,EAAE,wCAAwC,EAAE;iBAC7G;gBACD,oBAAoB,EAAE,KAAK;gBAC3B,WAAW,EAAE,uFAAuF;aACrG;YACD;gBACE,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,cAAc,EAAE,QAAQ,EAAE,IAAI,EAAE;oBAC/D,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE;oBAC1C,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,UAAU,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE;oBAC7F,KAAK,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,EAAE,WAAW,EAAE,mCAAmC,EAAE;iBAC7G;gBACD,oBAAoB,EAAE,KAAK;gBAC3B,WAAW,EAAE,8GAA8G;aAC5H;YACD;gBACE,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,aAAa,EAAE,QAAQ,EAAE,IAAI,EAAE;oBAC9D,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE;oBAC1C,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE;iBAC5E;gBACD,oBAAoB,EAAE,KAAK;gBAC3B,WAAW,EAAE,kGAAkG;aAChH;YACD;gBACE,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,gBAAgB,EAAE,QAAQ,EAAE,IAAI,EAAE;oBACjE,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE;oBAC1C,OAAO,EAAE,EAAE,GAAG,iBAAiB,EAAE,WAAW,EAAE,+DAA+D,EAAE;iBAChH;gBACD,oBAAoB,EAAE,KAAK;gBAC3B,WAAW,EAAE,mFAAmF;aACjG;YACD;gBACE,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE;oBACvD,OAAO,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,WAAW,EAAE,4CAA4C,EAAE;iBACjH;gBACD,oBAAoB,EAAE,KAAK;gBAC3B,WAAW,EAAE,mDAAmD;aACjE;YACD;gBACE,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,YAAY,EAAE,QAAQ,EAAE,IAAI,EAAE;oBAC7D,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE;oBAC1C,GAAG,EAAE,EAAE,GAAG,iBAAiB,EAAE,WAAW,EAAE,0DAA0D,EAAE;oBACtG,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,MAAM,EAAE,SAAS,CAAC,EAAE,WAAW,EAAE,4CAA4C,EAAE;iBAC/G;gBACD,oBAAoB,EAAE,KAAK;gBAC3B,WAAW,EAAE,iCAAiC;aAC/C;SACF;KACF;IACD,WAAW,EAAE,qEAAqE;CAC1E,CAAA;AAEV;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,OAA2B;IACzD,OAAO,UAAU,CAAC;QAChB,IAAI,EAAE,YAAY;QAClB,WAAW,EAAE;YACX,wIAAwI;YACxI,4QAA4Q;YAC5Q,yeAAye;SAC1e,CAAC,IAAI,CAAC,IAAI,CAAC;QACZ,UAAU,EAAE;YACV,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,2DAA2D,EAAE;YAClH,KAAK,EAAE,EAAE,GAAG,iBAAiB,EAAE,QAAQ,EAAE,IAAI,EAAE;YAC/C,UAAU,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,0FAA0F,EAAE;YACvI,MAAM,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,+HAA+H,EAAE;SAC1K;QACD,MAAM,EAAE;YACN,MAAM,EAAE;gBACN,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE;oBAC3C,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE;oBAC7C,UAAU,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE;oBAC9C,MAAM,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,IAAI,EAAE;oBAC3C,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE;oBAC/C,UAAU,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;oBAC9B,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;oBAC7B,WAAW,EAAE;wBACX,IAAI,EAAE,QAAQ;wBACd,UAAU,EAAE;4BACV,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE;4BAC3C,MAAM,EAAE,EAAE,GAAG,qBAAqB,EAAE,QAAQ,EAAE,IAAI,EAAE;4BACpD,KAAK,EAAE,EAAE,GAAG,qBAAqB,EAAE,QAAQ,EAAE,IAAI,EAAE;4BACnD,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE;yBAChD;wBACD,oBAAoB,EAAE,KAAK;qBAC5B;oBACD,IAAI,EAAE;wBACJ,IAAI,EAAE,OAAO;wBACb,KAAK,EAAE;4BACL,IAAI,EAAE,QAAQ;4BACd,UAAU,EAAE;gCACV,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE;gCAC7C,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,QAAQ,EAAE,cAAc,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE;gCACjI,YAAY,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE;gCAChD,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE;6BAC3C;4BACD,oBAAoB,EAAE,KAAK;yBAC5B;wBACD,QAAQ,EAAE,IAAI;qBACf;oBACD,QAAQ,EAAE;wBACR,IAAI,EAAE,QAAQ;wBACd,UAAU,EAAE;4BACV,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE;4BAC7C,UAAU,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE;4BAC9C,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE;4BAC/C,aAAa,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE;4BAC1G,UAAU,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,IAAI,EAAE;4BAC/C,sBAAsB,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE;4BAC1D,gBAAgB,EAAE;gCAChB,IAAI,EAAE,OAAO;gCACb,KAAK,EAAE;oCACL,IAAI,EAAE,QAAQ;oCACd,UAAU,EAAE;wCACV,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE;wCAC1C,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE;qCAC1C;oCACD,oBAAoB,EAAE,KAAK;iCAC5B;gCACD,QAAQ,EAAE,IAAI;6BACf;4BACD,eAAe,EAAE;gCACf,IAAI,EAAE,OAAO;gCACb,KAAK,EAAE;oCACL,IAAI,EAAE,QAAQ;oCACd,UAAU,EAAE;wCACV,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE;wCAC1C,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE;wCAC3E,YAAY,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE;qCACjD;oCACD,oBAAoB,EAAE,KAAK;iCAC5B;gCACD,QAAQ,EAAE,IAAI;6BACf;4BACD,eAAe,EAAE;gCACf,IAAI,EAAE,OAAO;gCACb,KAAK,EAAE;oCACL,IAAI,EAAE,QAAQ;oCACd,UAAU,EAAE;wCACV,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE;wCAC1C,SAAS,EAAE;4CACT,IAAI,EAAE,OAAO;4CACb,KAAK,EAAE;gDACL,IAAI,EAAE,QAAQ;gDACd,UAAU,EAAE;oDACV,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE;oDAC5C,YAAY,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE;iDACjD;gDACD,oBAAoB,EAAE,KAAK;6CAC5B;4CACD,QAAQ,EAAE,IAAI;yCACf;qCACF;oCACD,oBAAoB,EAAE,KAAK;iCAC5B;gCACD,QAAQ,EAAE,IAAI;6BACf;yBACF;wBACD,oBAAoB,EAAE,KAAK;wBAC3B,QAAQ,EAAE,IAAI;qBACf;oBACD,OAAO,EAAE;wBACP,IAAI,EAAE,QAAQ;wBACd,UAAU,EAAE;4BACV,OAAO,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE;4BACrE,IAAI,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE;yBACjE;wBACD,oBAAoB,EAAE,KAAK;wBAC3B,QAAQ,EAAE,IAAI;qBACf;iBACF;gBACD,oBAAoB,EAAE,KAAK;aAC5B;YACD,MAAM,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,eAAe,CAAC,KAAkC,CAAC,EAAE,CAAC;SACxG;QACD,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI;YACtB,OAAO,OAAO,CAAC,YAAY,CAAC;gBAC1B,OAAO,EAAE,IAAI,CAAC,IAAI;gBAClB,KAAK,EAAE,IAAI,CAAC,KAAwC;gBACpD,GAAG,CAAC,IAAI,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACzE,GAAG,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC7D,SAAS,EAAE,WAAW,CAAC,IAAI,CAAC;gBAC5B,OAAO,EAAE,IAAI,CAAC,KAAK,EAAE,OAAO;gBAC5B,MAAM,EAAE,IAAI,CAAC,MAAM;aACpB,CAAC,CAAA;QACJ,CAAC;KACF,CAAC,CAAA;AACJ,CAAC"}
|