dsh-data-quality 0.1.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +32 -0
- package/README.es.md +8 -2
- package/README.hi.md +8 -2
- package/README.md +22 -9
- package/README.pt.md +8 -2
- package/README.zh.md +8 -2
- package/cordis.patch.yml +11 -0
- package/lib/index.js +1411 -145
- package/lib/types/config.d.ts +12 -0
- package/lib/types/config.d.ts.map +1 -1
- package/lib/types/config.js +35 -0
- package/lib/types/config.js.map +1 -1
- package/lib/types/contract.d.ts +83 -0
- package/lib/types/contract.d.ts.map +1 -0
- package/lib/types/contract.js +144 -0
- package/lib/types/contract.js.map +1 -0
- package/lib/types/dataset.d.ts +50 -3
- package/lib/types/dataset.d.ts.map +1 -1
- package/lib/types/dataset.js +76 -21
- package/lib/types/dataset.js.map +1 -1
- package/lib/types/events.d.ts +2 -0
- package/lib/types/events.d.ts.map +1 -1
- package/lib/types/events.js.map +1 -1
- package/lib/types/index.d.ts +11 -8
- package/lib/types/index.d.ts.map +1 -1
- package/lib/types/index.js +15 -6
- package/lib/types/index.js.map +1 -1
- package/lib/types/presets.d.ts +28 -0
- package/lib/types/presets.d.ts.map +1 -0
- package/lib/types/presets.js +134 -0
- package/lib/types/presets.js.map +1 -0
- package/lib/types/profile.d.ts +38 -1
- package/lib/types/profile.d.ts.map +1 -1
- package/lib/types/profile.js +64 -10
- package/lib/types/profile.js.map +1 -1
- package/lib/types/provider-local.d.ts +7 -1
- package/lib/types/provider-local.d.ts.map +1 -1
- package/lib/types/provider-local.js +78 -4
- package/lib/types/provider-local.js.map +1 -1
- package/lib/types/scorecard.d.ts +50 -0
- package/lib/types/scorecard.d.ts.map +1 -0
- package/lib/types/scorecard.js +186 -0
- package/lib/types/scorecard.js.map +1 -0
- package/lib/types/service.d.ts +32 -2
- package/lib/types/service.d.ts.map +1 -1
- package/lib/types/service.js.map +1 -1
- package/lib/types/store.d.ts +21 -1
- package/lib/types/store.d.ts.map +1 -1
- package/lib/types/store.js +13 -1
- package/lib/types/store.js.map +1 -1
- package/lib/types/tools/clean.d.ts.map +1 -1
- package/lib/types/tools/clean.js +76 -1
- package/lib/types/tools/clean.js.map +1 -1
- package/lib/types/tools/profile-report-schema.d.ts +315 -0
- package/lib/types/tools/profile-report-schema.d.ts.map +1 -0
- package/lib/types/tools/profile-report-schema.js +95 -0
- package/lib/types/tools/profile-report-schema.js.map +1 -0
- package/lib/types/tools/profile.d.ts.map +1 -1
- package/lib/types/tools/profile.js +6 -53
- package/lib/types/tools/profile.js.map +1 -1
- package/lib/types/tools/report.d.ts +14 -0
- package/lib/types/tools/report.d.ts.map +1 -0
- package/lib/types/tools/report.js +86 -0
- package/lib/types/tools/report.js.map +1 -0
- package/lib/types/tools/shared.d.ts.map +1 -1
- package/lib/types/tools/shared.js +22 -2
- package/lib/types/tools/shared.js.map +1 -1
- package/lib/types/tools/verify.d.ts.map +1 -1
- package/lib/types/tools/verify.js +36 -1
- package/lib/types/tools/verify.js.map +1 -1
- package/lib/types/verify.d.ts +49 -3
- package/lib/types/verify.d.ts.map +1 -1
- package/lib/types/verify.js +111 -4
- package/lib/types/verify.js.map +1 -1
- package/lib/types/version.d.ts +1 -1
- package/lib/types/version.js +1 -1
- package/package.json +1 -1
- package/src/config.ts +52 -0
- package/src/contract.ts +190 -0
- package/src/dataset.ts +101 -21
- package/src/events.ts +2 -0
- package/src/index.ts +25 -8
- package/src/presets.ts +146 -0
- package/src/profile.ts +101 -11
- package/src/provider-local.ts +83 -5
- package/src/scorecard.ts +244 -0
- package/src/service.ts +34 -2
- package/src/store.ts +28 -2
- package/src/tools/clean.ts +76 -1
- package/src/tools/profile-report-schema.ts +97 -0
- package/src/tools/profile.ts +8 -56
- package/src/tools/report.ts +108 -0
- package/src/tools/shared.ts +21 -2
- package/src/tools/verify.ts +40 -2
- package/src/verify.ts +151 -5
- package/src/version.ts +1 -1
package/lib/types/config.d.ts
CHANGED
|
@@ -5,6 +5,15 @@
|
|
|
5
5
|
* @module dsh-data-quality/config
|
|
6
6
|
*/
|
|
7
7
|
import z from '@deepseek-ai/schemastery';
|
|
8
|
+
/** Per-dimension scorecard weights (all optional; defaults fill to `1` = equal). */
|
|
9
|
+
export interface ScorecardWeights {
|
|
10
|
+
completeness?: number;
|
|
11
|
+
uniqueness?: number;
|
|
12
|
+
validity?: number;
|
|
13
|
+
consistency?: number;
|
|
14
|
+
timeliness?: number;
|
|
15
|
+
accuracy?: number;
|
|
16
|
+
}
|
|
8
17
|
/** Raw plugin config — every field optional; {@link resolveConfig} supplies the defaults. */
|
|
9
18
|
export interface Config {
|
|
10
19
|
/** Master switch; `false` mounts nothing. */
|
|
@@ -23,6 +32,8 @@ export interface Config {
|
|
|
23
32
|
workspaceRoot?: string;
|
|
24
33
|
/** Persist reports to the `data_quality` storage domain. */
|
|
25
34
|
storeReports?: boolean;
|
|
35
|
+
/** Per-dimension weights for the scorecard's weighted overall total (non-negative). */
|
|
36
|
+
scorecardWeights?: ScorecardWeights;
|
|
26
37
|
}
|
|
27
38
|
/** Fully resolved config handed to the runtime. */
|
|
28
39
|
export interface ResolvedConfig {
|
|
@@ -34,6 +45,7 @@ export interface ResolvedConfig {
|
|
|
34
45
|
readonly allowedExtensions: readonly string[];
|
|
35
46
|
readonly workspaceRoot: string;
|
|
36
47
|
readonly storeReports: boolean;
|
|
48
|
+
readonly scorecardWeights: Required<ScorecardWeights>;
|
|
37
49
|
}
|
|
38
50
|
/** Schemastery schema: the loader validates and fills defaults before `apply`. */
|
|
39
51
|
export declare const Config: z<Config>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/config.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,CAAC,MAAM,0BAA0B,CAAA;AAExC,6FAA6F;AAC7F,MAAM,WAAW,MAAM;IACrB,6CAA6C;IAC7C,OAAO,CAAC,EAAE,OAAO,CAAA;IACjB,qCAAqC;IACrC,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,kDAAkD;IAClD,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,kEAAkE;IAClE,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,4EAA4E;IAC5E,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,iEAAiE;IACjE,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAA;IAC5B,2GAA2G;IAC3G,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,4DAA4D;IAC5D,YAAY,CAAC,EAAE,OAAO,CAAA;
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/config.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,CAAC,MAAM,0BAA0B,CAAA;AAExC,oFAAoF;AACpF,MAAM,WAAW,gBAAgB;IAC/B,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAA;CAClB;AAeD,6FAA6F;AAC7F,MAAM,WAAW,MAAM;IACrB,6CAA6C;IAC7C,OAAO,CAAC,EAAE,OAAO,CAAA;IACjB,qCAAqC;IACrC,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,kDAAkD;IAClD,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,kEAAkE;IAClE,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,4EAA4E;IAC5E,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,iEAAiE;IACjE,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAA;IAC5B,2GAA2G;IAC3G,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,4DAA4D;IAC5D,YAAY,CAAC,EAAE,OAAO,CAAA;IACtB,uFAAuF;IACvF,gBAAgB,CAAC,EAAE,gBAAgB,CAAA;CACpC;AAED,mDAAmD;AACnD,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAA;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAA;IAC9B,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAA;IACjC,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAA;IACjC,QAAQ,CAAC,iBAAiB,EAAE,SAAS,MAAM,EAAE,CAAA;IAC7C,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAA;IAC9B,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAA;IAC9B,QAAQ,CAAC,gBAAgB,EAAE,QAAQ,CAAC,gBAAgB,CAAC,CAAA;CACtD;AAED,kFAAkF;AAClF,eAAO,MAAM,MAAM,EAAE,CAAC,CAAC,MAAM,CAmB3B,CAAA;AAgBF;;;;;GAKG;AACH,wBAAgB,aAAa,CAAC,MAAM,GAAE,MAAW,GAAG,cAAc,CAiCjE"}
|
package/lib/types/config.js
CHANGED
|
@@ -5,6 +5,17 @@
|
|
|
5
5
|
* @module dsh-data-quality/config
|
|
6
6
|
*/
|
|
7
7
|
import z from '@deepseek-ai/schemastery';
|
|
8
|
+
/** The six scorecard dimension ids, in report order (kept in sync with `scorecard.ts`). */
|
|
9
|
+
const SCORECARD_DIMENSIONS = ['completeness', 'uniqueness', 'validity', 'consistency', 'timeliness', 'accuracy'];
|
|
10
|
+
/** Equal-weight defaults for {@link ScorecardWeights}. */
|
|
11
|
+
const EQUAL_WEIGHTS = {
|
|
12
|
+
completeness: 1,
|
|
13
|
+
uniqueness: 1,
|
|
14
|
+
validity: 1,
|
|
15
|
+
consistency: 1,
|
|
16
|
+
timeliness: 1,
|
|
17
|
+
accuracy: 1,
|
|
18
|
+
};
|
|
8
19
|
/** Schemastery schema: the loader validates and fills defaults before `apply`. */
|
|
9
20
|
export const Config = z.object({
|
|
10
21
|
enabled: z.boolean().default(true),
|
|
@@ -15,6 +26,16 @@ export const Config = z.object({
|
|
|
15
26
|
allowedExtensions: z.array(z.string()).default(['.csv', '.tsv', '.json', '.jsonl']),
|
|
16
27
|
workspaceRoot: z.string().default(''),
|
|
17
28
|
storeReports: z.boolean().default(true),
|
|
29
|
+
scorecardWeights: z
|
|
30
|
+
.object({
|
|
31
|
+
completeness: z.number().default(1),
|
|
32
|
+
uniqueness: z.number().default(1),
|
|
33
|
+
validity: z.number().default(1),
|
|
34
|
+
consistency: z.number().default(1),
|
|
35
|
+
timeliness: z.number().default(1),
|
|
36
|
+
accuracy: z.number().default(1),
|
|
37
|
+
})
|
|
38
|
+
.default(EQUAL_WEIGHTS),
|
|
18
39
|
});
|
|
19
40
|
/** Throw unless `value` is a positive safe integer. */
|
|
20
41
|
function assertPositiveInt(name, value) {
|
|
@@ -52,6 +73,7 @@ export function resolveConfig(config = {}) {
|
|
|
52
73
|
if (allowedExtensions.length === 0) {
|
|
53
74
|
throw new TypeError('allowedExtensions must not be empty');
|
|
54
75
|
}
|
|
76
|
+
const scorecardWeights = resolveScorecardWeights(config.scorecardWeights);
|
|
55
77
|
return {
|
|
56
78
|
enabled: config.enabled ?? true,
|
|
57
79
|
maxRows,
|
|
@@ -61,6 +83,19 @@ export function resolveConfig(config = {}) {
|
|
|
61
83
|
allowedExtensions,
|
|
62
84
|
workspaceRoot: config.workspaceRoot ?? '',
|
|
63
85
|
storeReports: config.storeReports ?? true,
|
|
86
|
+
scorecardWeights,
|
|
64
87
|
};
|
|
65
88
|
}
|
|
89
|
+
/** Resolve scorecard weights, filling defaults and rejecting non-negative violations loudly. */
|
|
90
|
+
function resolveScorecardWeights(weights = {}) {
|
|
91
|
+
const out = {};
|
|
92
|
+
for (const dimension of SCORECARD_DIMENSIONS) {
|
|
93
|
+
const value = weights[dimension] ?? 1;
|
|
94
|
+
if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) {
|
|
95
|
+
throw new TypeError(`scorecardWeights.${dimension} must be a non-negative finite number, got ${String(value)}`);
|
|
96
|
+
}
|
|
97
|
+
out[dimension] = value;
|
|
98
|
+
}
|
|
99
|
+
return out;
|
|
100
|
+
}
|
|
66
101
|
//# sourceMappingURL=config.js.map
|
package/lib/types/config.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/config.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,CAAC,MAAM,0BAA0B,CAAA;
|
|
1
|
+
{"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/config.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,CAAC,MAAM,0BAA0B,CAAA;AAYxC,2FAA2F;AAC3F,MAAM,oBAAoB,GAAG,CAAC,cAAc,EAAE,YAAY,EAAE,UAAU,EAAE,aAAa,EAAE,YAAY,EAAE,UAAU,CAAU,CAAA;AAEzH,0DAA0D;AAC1D,MAAM,aAAa,GAA+B;IAChD,YAAY,EAAE,CAAC;IACf,UAAU,EAAE,CAAC;IACb,QAAQ,EAAE,CAAC;IACX,WAAW,EAAE,CAAC;IACd,UAAU,EAAE,CAAC;IACb,QAAQ,EAAE,CAAC;CACZ,CAAA;AAqCD,kFAAkF;AAClF,MAAM,CAAC,MAAM,MAAM,GAAc,CAAC,CAAC,MAAM,CAAC;IACxC,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;IAClC,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC;IACpC,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IACrC,gBAAgB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;IAC1C,gBAAgB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IACxC,iBAAiB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;IACnF,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IACrC,YAAY,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;IACvC,gBAAgB,EAAE,CAAC;SAChB,MAAM,CAAC;QACN,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;QACnC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;QACjC,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;QAC/B,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;QAClC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;QACjC,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;KAChC,CAAC;SACD,OAAO,CAAC,aAAa,CAAC;CAC1B,CAAC,CAAA;AAEF,uDAAuD;AACvD,SAAS,iBAAiB,CAAC,IAAY,EAAE,KAAa;IACpD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;QAC/C,MAAM,IAAI,SAAS,CAAC,GAAG,IAAI,yCAAyC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;IACtF,CAAC;AACH,CAAC;AAED,+DAA+D;AAC/D,SAAS,iBAAiB,CAAC,IAAY,EAAE,KAAa,EAAE,GAAW,EAAE,GAAW;IAC9E,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,GAAG,IAAI,KAAK,GAAG,GAAG,EAAE,CAAC;QACvF,MAAM,IAAI,SAAS,CAAC,GAAG,IAAI,gCAAgC,GAAG,KAAK,GAAG,UAAU,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;IAClG,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,aAAa,CAAC,SAAiB,EAAE;IAC/C,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,OAAO,CAAA;IACzC,iBAAiB,CAAC,SAAS,EAAE,OAAO,CAAC,CAAA;IACrC,MAAM,aAAa,GAAG,MAAM,CAAC,aAAa,IAAI,EAAE,CAAA;IAChD,iBAAiB,CAAC,eAAe,EAAE,aAAa,EAAE,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,CAAA;IACzE,MAAM,gBAAgB,GAAG,MAAM,CAAC,gBAAgB,IAAI,IAAI,CAAA;IACxD,iBAAiB,CAAC,kBAAkB,EAAE,gBAAgB,EAAE,CAAC,EAAE,CAAC,CAAC,CAAA;IAC7D,MAAM,gBAAgB,GAAG,MAAM,CAAC,gBAAgB,IAAI,EAAE,CAAA;IACtD,iBAAiB,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,CAAA;IAEvD,MAAM,iBAAiB,GAAG,CAAC,MAAM,CAAC,iBAAiB,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QACtG,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YAChC,MAAM,IAAI,SAAS,CAAC,4EAA4E,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QACxH,CAAC;QACD,OAAO,GAAG,CAAA;IACZ,CAAC,CAAC,CAAA;IACF,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnC,MAAM,IAAI,SAAS,CAAC,qCAAqC,CAAC,CAAA;IAC5D,CAAC;IAED,MAAM,gBAAgB,GAAG,uBAAuB,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAA;IAEzE,OAAO;QACL,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,IAAI;QAC/B,OAAO;QACP,aAAa;QACb,gBAAgB;QAChB,gBAAgB;QAChB,iBAAiB;QACjB,aAAa,EAAE,MAAM,CAAC,aAAa,IAAI,EAAE;QACzC,YAAY,EAAE,MAAM,CAAC,YAAY,IAAI,IAAI;QACzC,gBAAgB;KACjB,CAAA;AACH,CAAC;AAED,gGAAgG;AAChG,SAAS,uBAAuB,CAAC,UAA4B,EAAE;IAC7D,MAAM,GAAG,GAAG,EAAgC,CAAA;IAC5C,KAAK,MAAM,SAAS,IAAI,oBAAoB,EAAE,CAAC;QAC7C,MAAM,KAAK,GAAG,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;QACrC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YACtE,MAAM,IAAI,SAAS,CAAC,oBAAoB,SAAS,8CAA8C,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;QACjH,CAAC;QACD,GAAG,CAAC,SAAS,CAAC,GAAG,KAAK,CAAA;IACxB,CAAC;IACD,OAAO,GAAG,CAAA;AACZ,CAAC"}
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Delivery contract for `data_clean`: a deterministic pre-delivery validation
|
|
3
|
+
* summary (primary-key/type/dedupe row-count comparison plus uniqueness and
|
|
4
|
+
* non-null regression) and the clean before/after profile diff report. Pure —
|
|
5
|
+
* no I/O, no clock, no RNG; timestamps are injected.
|
|
6
|
+
* @module dsh-data-quality/contract
|
|
7
|
+
*/
|
|
8
|
+
import { type Table } from './dataset.js';
|
|
9
|
+
import type { CleanResult, CleanRule } from './clean.js';
|
|
10
|
+
import { type ProfileReport } from './profile.js';
|
|
11
|
+
import type { ScorecardDimensionName } from './scorecard.js';
|
|
12
|
+
/** The pre-delivery contract summary attached to a clean run. */
|
|
13
|
+
export interface CleanContractSummary {
|
|
14
|
+
/** Row count before cleaning. */
|
|
15
|
+
readonly inputRows: number;
|
|
16
|
+
/** Row count after cleaning. */
|
|
17
|
+
readonly outputRows: number;
|
|
18
|
+
/** Rows removed across all rules. */
|
|
19
|
+
readonly removedRows: number;
|
|
20
|
+
/** Key columns of the first `dedupe` rule; `null` when no dedupe rule ran (uniqueness then covers full rows). */
|
|
21
|
+
readonly dedupeColumns: string[] | null;
|
|
22
|
+
/** Whether the uniqueness key is free of duplicates in the cleaned output. */
|
|
23
|
+
readonly uniqueKeys: boolean;
|
|
24
|
+
/** Remaining duplicate rows over the uniqueness key in the cleaned output. */
|
|
25
|
+
readonly remainingDuplicateRows: number;
|
|
26
|
+
/** Non-null regression: remaining missing cells in columns a `fill-missing` rule targeted. */
|
|
27
|
+
readonly remainingMissing: Array<{
|
|
28
|
+
readonly column: string;
|
|
29
|
+
readonly count: number;
|
|
30
|
+
}>;
|
|
31
|
+
/** Type regression: remaining non-conforming cells in columns a `coerce-type` rule targeted. */
|
|
32
|
+
readonly typeConformance: Array<{
|
|
33
|
+
readonly column: string;
|
|
34
|
+
readonly to: 'number' | 'date' | 'boolean';
|
|
35
|
+
readonly invalidCount: number;
|
|
36
|
+
}>;
|
|
37
|
+
/** Per-column decision trace: which strategies touched each column and how many rows they affected. */
|
|
38
|
+
readonly columnDecisions: Array<{
|
|
39
|
+
readonly column: string;
|
|
40
|
+
readonly decisions: Array<{
|
|
41
|
+
readonly strategy: string;
|
|
42
|
+
readonly affectedRows: number;
|
|
43
|
+
}>;
|
|
44
|
+
}>;
|
|
45
|
+
}
|
|
46
|
+
/** The clean before/after profile diff report (persisted under the `clean-diff` report kind). */
|
|
47
|
+
export interface CleanProfileDiff {
|
|
48
|
+
readonly dataset: string;
|
|
49
|
+
/** Full profile of the input dataset. */
|
|
50
|
+
readonly before: ProfileReport;
|
|
51
|
+
/** Full profile of the cleaned output (including its scorecard). */
|
|
52
|
+
readonly after: ProfileReport;
|
|
53
|
+
readonly generatedAt: number;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Compute the pre-delivery contract summary from a cleaning run: dedupe
|
|
57
|
+
* before/after row counts, the dedupe key's uniqueness regression, and the
|
|
58
|
+
* non-null/type regressions over the columns the rules targeted.
|
|
59
|
+
* @param result - the cleaning outcome (input/output rows plus cleaned rows).
|
|
60
|
+
* @param rules - the ordered rules that produced `result`.
|
|
61
|
+
* @param options - optional abort signal.
|
|
62
|
+
* @returns the contract summary.
|
|
63
|
+
*/
|
|
64
|
+
export declare function computeCleanContract(result: CleanResult, rules: readonly CleanRule[], options?: {
|
|
65
|
+
signal?: AbortSignal | undefined;
|
|
66
|
+
}): CleanContractSummary;
|
|
67
|
+
/**
|
|
68
|
+
* Build the clean before/after profile diff: two full {@link ProfileReport}
|
|
69
|
+
* snapshots (input and cleaned output) under one envelope. Reuses the profile
|
|
70
|
+
* engine so the scorecard and duplicate detection stay consistent across the
|
|
71
|
+
* whole plugin.
|
|
72
|
+
* @param input - the input dataset.
|
|
73
|
+
* @param output - the cleaned dataset.
|
|
74
|
+
* @param options - dataset label, injected timestamp, optional weights, optional abort signal.
|
|
75
|
+
* @returns the diff report.
|
|
76
|
+
*/
|
|
77
|
+
export declare function computeCleanProfileDiff(input: Table, output: Table, options: {
|
|
78
|
+
dataset: string;
|
|
79
|
+
generatedAt: number;
|
|
80
|
+
scorecardWeights?: Readonly<Record<ScorecardDimensionName, number>> | undefined;
|
|
81
|
+
signal?: AbortSignal | undefined;
|
|
82
|
+
}): CleanProfileDiff;
|
|
83
|
+
//# sourceMappingURL=contract.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"contract.d.ts","sourceRoot":"","sources":["../../src/contract.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAA8E,KAAK,KAAK,EAAE,MAAM,cAAc,CAAA;AACrH,OAAO,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AACxD,OAAO,EAAgB,KAAK,aAAa,EAAE,MAAM,cAAc,CAAA;AAC/D,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAA;AAE5D,iEAAiE;AACjE,MAAM,WAAW,oBAAoB;IACnC,iCAAiC;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,gCAAgC;IAChC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,qCAAqC;IACrC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IAC5B,iHAAiH;IACjH,QAAQ,CAAC,aAAa,EAAE,MAAM,EAAE,GAAG,IAAI,CAAA;IACvC,8EAA8E;IAC9E,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAA;IAC5B,8EAA8E;IAC9E,QAAQ,CAAC,sBAAsB,EAAE,MAAM,CAAA;IACvC,8FAA8F;IAC9F,QAAQ,CAAC,gBAAgB,EAAE,KAAK,CAAC;QAAE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;IACrF,gGAAgG;IAChG,QAAQ,CAAC,eAAe,EAAE,KAAK,CAAC;QAAE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,EAAE,QAAQ,GAAG,MAAM,GAAG,SAAS,CAAC;QAAC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;IACvI,uGAAuG;IACvG,QAAQ,CAAC,eAAe,EAAE,KAAK,CAAC;QAC9B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;QACvB,QAAQ,CAAC,SAAS,EAAE,KAAK,CAAC;YAAE,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;YAAC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KACxF,CAAC,CAAA;CACH;AAED,iGAAiG;AACjG,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,yCAAyC;IACzC,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAA;IAC9B,oEAAoE;IACpE,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAA;IAC7B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;CAC7B;AAyBD;;;;;;;;GAQG;AACH,wBAAgB,oBAAoB,CAClC,MAAM,EAAE,WAAW,EACnB,KAAK,EAAE,SAAS,SAAS,EAAE,EAC3B,OAAO,GAAE;IAAE,MAAM,CAAC,EAAE,WAAW,GAAG,SAAS,CAAA;CAAO,GACjD,oBAAoB,CAuCtB;AAsCD;;;;;;;;;GASG;AACH,wBAAgB,uBAAuB,CACrC,KAAK,EAAE,KAAK,EACZ,MAAM,EAAE,KAAK,EACb,OAAO,EAAE;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,MAAM,CAAC;IAAC,gBAAgB,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,sBAAsB,EAAE,MAAM,CAAC,CAAC,GAAG,SAAS,CAAC;IAAC,MAAM,CAAC,EAAE,WAAW,GAAG,SAAS,CAAA;CAAE,GACnK,gBAAgB,CAclB"}
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Delivery contract for `data_clean`: a deterministic pre-delivery validation
|
|
3
|
+
* summary (primary-key/type/dedupe row-count comparison plus uniqueness and
|
|
4
|
+
* non-null regression) and the clean before/after profile diff report. Pure —
|
|
5
|
+
* no I/O, no clock, no RNG; timestamps are injected.
|
|
6
|
+
* @module dsh-data-quality/contract
|
|
7
|
+
*/
|
|
8
|
+
import { isMissing, parseBoolean, parseDate, parseNumeric, throwIfAborted } from "./dataset.js";
|
|
9
|
+
import { profileTable } from "./profile.js";
|
|
10
|
+
/** Count missing cells in one column of the cleaned output. */
|
|
11
|
+
function countMissing(rows, column, signal) {
|
|
12
|
+
let count = 0;
|
|
13
|
+
for (const [index, row] of rows.entries()) {
|
|
14
|
+
if (index % 1024 === 0)
|
|
15
|
+
throwIfAborted(signal);
|
|
16
|
+
if (isMissing(row[column]))
|
|
17
|
+
count += 1;
|
|
18
|
+
}
|
|
19
|
+
return count;
|
|
20
|
+
}
|
|
21
|
+
/** Count present cells in one column that still fail to parse as the requested type. */
|
|
22
|
+
function countNonConforming(rows, column, to, signal) {
|
|
23
|
+
let count = 0;
|
|
24
|
+
for (const [index, row] of rows.entries()) {
|
|
25
|
+
if (index % 1024 === 0)
|
|
26
|
+
throwIfAborted(signal);
|
|
27
|
+
const cell = row[column];
|
|
28
|
+
if (isMissing(cell))
|
|
29
|
+
continue;
|
|
30
|
+
const conforms = to === 'number' ? parseNumeric(cell) !== undefined : to === 'date' ? parseDate(cell) !== undefined : parseBoolean(cell) !== undefined;
|
|
31
|
+
if (!conforms)
|
|
32
|
+
count += 1;
|
|
33
|
+
}
|
|
34
|
+
return count;
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Compute the pre-delivery contract summary from a cleaning run: dedupe
|
|
38
|
+
* before/after row counts, the dedupe key's uniqueness regression, and the
|
|
39
|
+
* non-null/type regressions over the columns the rules targeted.
|
|
40
|
+
* @param result - the cleaning outcome (input/output rows plus cleaned rows).
|
|
41
|
+
* @param rules - the ordered rules that produced `result`.
|
|
42
|
+
* @param options - optional abort signal.
|
|
43
|
+
* @returns the contract summary.
|
|
44
|
+
*/
|
|
45
|
+
export function computeCleanContract(result, rules, options = {}) {
|
|
46
|
+
const dedupeRule = rules.find((rule) => rule.rule === 'dedupe');
|
|
47
|
+
const dedupeColumns = dedupeRule === undefined ? null : [...(dedupeRule.columns ?? result.columns)];
|
|
48
|
+
const keyColumns = dedupeColumns ?? result.columns;
|
|
49
|
+
const seen = new Set();
|
|
50
|
+
let remainingDuplicateRows = 0;
|
|
51
|
+
for (const [index, row] of result.rows.entries()) {
|
|
52
|
+
if (index % 1024 === 0)
|
|
53
|
+
throwIfAborted(options.signal);
|
|
54
|
+
const key = JSON.stringify(keyColumns.map((column) => row[column] ?? null));
|
|
55
|
+
if (seen.has(key)) {
|
|
56
|
+
remainingDuplicateRows += 1;
|
|
57
|
+
}
|
|
58
|
+
else {
|
|
59
|
+
seen.add(key);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
const remainingMissing = rules
|
|
63
|
+
.filter((rule) => rule.rule === 'fill-missing')
|
|
64
|
+
.map((rule) => {
|
|
65
|
+
const fill = rule;
|
|
66
|
+
return { column: fill.column, count: countMissing(result.rows, fill.column, options.signal) };
|
|
67
|
+
});
|
|
68
|
+
const typeConformance = rules
|
|
69
|
+
.filter((rule) => rule.rule === 'coerce-type')
|
|
70
|
+
.map((rule) => {
|
|
71
|
+
const coerce = rule;
|
|
72
|
+
return { column: coerce.column, to: coerce.to, invalidCount: countNonConforming(result.rows, coerce.column, coerce.to, options.signal) };
|
|
73
|
+
});
|
|
74
|
+
const columnDecisions = traceColumnDecisions(result, rules);
|
|
75
|
+
return {
|
|
76
|
+
inputRows: result.inputRows,
|
|
77
|
+
outputRows: result.outputRows,
|
|
78
|
+
removedRows: result.inputRows - result.outputRows,
|
|
79
|
+
dedupeColumns,
|
|
80
|
+
uniqueKeys: remainingDuplicateRows === 0,
|
|
81
|
+
remainingDuplicateRows,
|
|
82
|
+
remainingMissing,
|
|
83
|
+
typeConformance,
|
|
84
|
+
columnDecisions,
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
/** The columns a single rule targets (single-column rules target one column; trim/dedupe may target many). */
|
|
88
|
+
function ruleColumns(rule, allColumns) {
|
|
89
|
+
switch (rule.rule) {
|
|
90
|
+
case 'fill-missing':
|
|
91
|
+
case 'coerce-type':
|
|
92
|
+
case 'normalize-unit':
|
|
93
|
+
case 'map-values':
|
|
94
|
+
return [rule.column];
|
|
95
|
+
case 'trim':
|
|
96
|
+
case 'dedupe':
|
|
97
|
+
return [...(rule.columns ?? allColumns)];
|
|
98
|
+
default:
|
|
99
|
+
return [];
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
/** Build the per-column decision trace: strategies + affected rows, in dataset column order. */
|
|
103
|
+
function traceColumnDecisions(result, rules) {
|
|
104
|
+
const byColumn = new Map();
|
|
105
|
+
for (const [index, rule] of rules.entries()) {
|
|
106
|
+
const log = result.logs[index];
|
|
107
|
+
if (log === undefined)
|
|
108
|
+
continue;
|
|
109
|
+
for (const column of ruleColumns(rule, result.columns)) {
|
|
110
|
+
const list = byColumn.get(column) ?? [];
|
|
111
|
+
list.push({ strategy: rule.rule, affectedRows: log.affectedRows });
|
|
112
|
+
byColumn.set(column, list);
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
return result.columns
|
|
116
|
+
.filter((column) => byColumn.has(column))
|
|
117
|
+
.map((column) => ({ column, decisions: byColumn.get(column) }));
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* Build the clean before/after profile diff: two full {@link ProfileReport}
|
|
121
|
+
* snapshots (input and cleaned output) under one envelope. Reuses the profile
|
|
122
|
+
* engine so the scorecard and duplicate detection stay consistent across the
|
|
123
|
+
* whole plugin.
|
|
124
|
+
* @param input - the input dataset.
|
|
125
|
+
* @param output - the cleaned dataset.
|
|
126
|
+
* @param options - dataset label, injected timestamp, optional weights, optional abort signal.
|
|
127
|
+
* @returns the diff report.
|
|
128
|
+
*/
|
|
129
|
+
export function computeCleanProfileDiff(input, output, options) {
|
|
130
|
+
const before = profileTable(input, {
|
|
131
|
+
dataset: options.dataset,
|
|
132
|
+
generatedAt: options.generatedAt,
|
|
133
|
+
scorecardWeights: options.scorecardWeights,
|
|
134
|
+
signal: options.signal,
|
|
135
|
+
});
|
|
136
|
+
const after = profileTable(output, {
|
|
137
|
+
dataset: options.dataset,
|
|
138
|
+
generatedAt: options.generatedAt,
|
|
139
|
+
scorecardWeights: options.scorecardWeights,
|
|
140
|
+
signal: options.signal,
|
|
141
|
+
});
|
|
142
|
+
return { dataset: options.dataset, before, after, generatedAt: options.generatedAt };
|
|
143
|
+
}
|
|
144
|
+
//# sourceMappingURL=contract.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"contract.js","sourceRoot":"","sources":["../../src/contract.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,SAAS,EAAE,YAAY,EAAE,cAAc,EAAwB,MAAM,cAAc,CAAA;AAErH,OAAO,EAAE,YAAY,EAAsB,MAAM,cAAc,CAAA;AAsC/D,+DAA+D;AAC/D,SAAS,YAAY,CAAC,IAAoB,EAAE,MAAc,EAAE,MAAoB;IAC9E,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,KAAK,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;QAC1C,IAAI,KAAK,GAAG,IAAI,KAAK,CAAC;YAAE,cAAc,CAAC,MAAM,CAAC,CAAA;QAC9C,IAAI,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAAE,KAAK,IAAI,CAAC,CAAA;IACxC,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,wFAAwF;AACxF,SAAS,kBAAkB,CAAC,IAAoB,EAAE,MAAc,EAAE,EAAiC,EAAE,MAAoB;IACvH,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,KAAK,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;QAC1C,IAAI,KAAK,GAAG,IAAI,KAAK,CAAC;YAAE,cAAc,CAAC,MAAM,CAAC,CAAA;QAC9C,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,CAAA;QACxB,IAAI,SAAS,CAAC,IAAI,CAAC;YAAE,SAAQ;QAC7B,MAAM,QAAQ,GAAG,EAAE,KAAK,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,SAAS,CAAA;QACtJ,IAAI,CAAC,QAAQ;YAAE,KAAK,IAAI,CAAC,CAAA;IAC3B,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,oBAAoB,CAClC,MAAmB,EACnB,KAA2B,EAC3B,UAAgD,EAAE;IAElD,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAuD,CAAA;IACrH,MAAM,aAAa,GAAG,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAA;IACnG,MAAM,UAAU,GAAG,aAAa,IAAI,MAAM,CAAC,OAAO,CAAA;IAClD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAC9B,IAAI,sBAAsB,GAAG,CAAC,CAAA;IAC9B,KAAK,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;QACjD,IAAI,KAAK,GAAG,IAAI,KAAK,CAAC;YAAE,cAAc,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;QACtD,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,CAAC,CAAA;QAC3E,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAClB,sBAAsB,IAAI,CAAC,CAAA;QAC7B,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QACf,CAAC;IACH,CAAC;IACD,MAAM,gBAAgB,GAAG,KAAK;SAC3B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,cAAc,CAAC;SAC9C,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACZ,MAAM,IAAI,GAAG,IAAoD,CAAA;QACjE,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,EAAE,CAAA;IAC/F,CAAC,CAAC,CAAA;IACJ,MAAM,eAAe,GAAG,KAAK;SAC1B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,aAAa,CAAC;SAC7C,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACZ,MAAM,MAAM,GAAG,IAAmD,CAAA;QAClE,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,EAAE,EAAE,MAAM,CAAC,EAAE,EAAE,YAAY,EAAE,kBAAkB,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,EAAE,OAAO,CAAC,MAAM,CAAC,EAAE,CAAA;IAC1I,CAAC,CAAC,CAAA;IACJ,MAAM,eAAe,GAAG,oBAAoB,CAAC,MAAM,EAAE,KAAK,CAAC,CAAA;IAC3D,OAAO;QACL,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,WAAW,EAAE,MAAM,CAAC,SAAS,GAAG,MAAM,CAAC,UAAU;QACjD,aAAa;QACb,UAAU,EAAE,sBAAsB,KAAK,CAAC;QACxC,sBAAsB;QACtB,gBAAgB;QAChB,eAAe;QACf,eAAe;KAChB,CAAA;AACH,CAAC;AAED,8GAA8G;AAC9G,SAAS,WAAW,CAAC,IAAe,EAAE,UAA6B;IACjE,QAAQ,IAAI,CAAC,IAAI,EAAE,CAAC;QAClB,KAAK,cAAc,CAAC;QACpB,KAAK,aAAa,CAAC;QACnB,KAAK,gBAAgB,CAAC;QACtB,KAAK,YAAY;YACf,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QACtB,KAAK,MAAM,CAAC;QACZ,KAAK,QAAQ;YACX,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,UAAU,CAAC,CAAC,CAAA;QAC1C;YACE,OAAO,EAAE,CAAA;IACb,CAAC;AACH,CAAC;AAED,gGAAgG;AAChG,SAAS,oBAAoB,CAC3B,MAAmB,EACnB,KAA2B;IAE3B,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA6D,CAAA;IACrF,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;QAC5C,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAC9B,IAAI,GAAG,KAAK,SAAS;YAAE,SAAQ;QAC/B,KAAK,MAAM,MAAM,IAAI,WAAW,CAAC,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;YACvD,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAA;YACvC,IAAI,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,IAAI,EAAE,YAAY,EAAE,GAAG,CAAC,YAAY,EAAE,CAAC,CAAA;YAClE,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;QAC5B,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC,OAAO;SAClB,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;SACxC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAsD,EAAE,CAAC,CAAC,CAAA;AACxH,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,uBAAuB,CACrC,KAAY,EACZ,MAAa,EACb,OAAoK;IAEpK,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,EAAE;QACjC,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;QAC1C,MAAM,EAAE,OAAO,CAAC,MAAM;KACvB,CAAC,CAAA;IACF,MAAM,KAAK,GAAG,YAAY,CAAC,MAAM,EAAE;QACjC,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;QAC1C,MAAM,EAAE,OAAO,CAAC,MAAM;KACvB,CAAC,CAAA;IACF,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,CAAC,WAAW,EAAE,CAAA;AACtF,CAAC"}
|
package/lib/types/dataset.d.ts
CHANGED
|
@@ -11,12 +11,21 @@ import type { ResolvedConfig } from './config.js';
|
|
|
11
11
|
export type Cell = JsonValue;
|
|
12
12
|
/** One dataset row keyed by column name. */
|
|
13
13
|
export type Row = Record<string, Cell>;
|
|
14
|
+
/** Detected file encoding metadata (BOM presence + UTF-8 validity). */
|
|
15
|
+
export interface EncodingInfo {
|
|
16
|
+
/** `'utf-8'` when a UTF-8 byte-order mark was present, else `null`. */
|
|
17
|
+
readonly bom: 'utf-8' | null;
|
|
18
|
+
/** Whether the bytes decode as valid UTF-8 (no replacement characters forced). */
|
|
19
|
+
readonly validUtf8: boolean;
|
|
20
|
+
}
|
|
14
21
|
/** A tabular dataset: ordered columns plus rows. */
|
|
15
22
|
export interface Table {
|
|
16
23
|
/** Column names in file order. */
|
|
17
24
|
readonly columns: string[];
|
|
18
25
|
/** Rows, each carrying every declared column (missing cells are `null`). */
|
|
19
26
|
readonly rows: Row[];
|
|
27
|
+
/** Detected file encoding; present only for tables loaded from a file. */
|
|
28
|
+
readonly encoding?: EncodingInfo;
|
|
20
29
|
}
|
|
21
30
|
/** The root form a document load returns (citation checking walks this). */
|
|
22
31
|
export type DocumentRoot = {
|
|
@@ -55,13 +64,25 @@ export declare function pathFingerprint(datasetPath: string): string;
|
|
|
55
64
|
/** Assert the abort signal has not fired; throws the signal reason otherwise. */
|
|
56
65
|
export declare function throwIfAborted(signal: AbortSignal | undefined): void;
|
|
57
66
|
/**
|
|
58
|
-
*
|
|
67
|
+
* Detect a UTF-8 byte-order mark and validate the byte sequence. Invalid
|
|
68
|
+
* UTF-8 is a data-quality finding the profile reports (`validUtf8: false`)
|
|
69
|
+
* rather than a structural error that blocks the read — the decoded text
|
|
70
|
+
* keeps U+FFFD replacement characters so the profile can still run.
|
|
71
|
+
* @param buffer - raw file bytes.
|
|
72
|
+
* @returns the encoding metadata.
|
|
73
|
+
*/
|
|
74
|
+
export declare function detectEncoding(buffer: Uint8Array): EncodingInfo;
|
|
75
|
+
/**
|
|
76
|
+
* Read a dataset file under the size cap, detecting its encoding.
|
|
59
77
|
* @param absolutePath - normalized absolute path (from {@link resolveWorkspacePath}).
|
|
60
78
|
* @param config - resolved config (size cap).
|
|
61
79
|
* @param signal - optional abort signal honored around the read.
|
|
62
|
-
* @returns the UTF-8 text.
|
|
80
|
+
* @returns the decoded UTF-8 text plus its encoding metadata.
|
|
63
81
|
*/
|
|
64
|
-
export declare function
|
|
82
|
+
export declare function readDatasetFile(absolutePath: string, config: ResolvedConfig, signal?: AbortSignal): Promise<{
|
|
83
|
+
text: string;
|
|
84
|
+
encoding: EncodingInfo;
|
|
85
|
+
}>;
|
|
65
86
|
/**
|
|
66
87
|
* Parse CSV/TSV text (RFC-4180-style: quoted fields, `""` escapes, CRLF/LF).
|
|
67
88
|
* The first record is the header; short records pad with `null`, long records
|
|
@@ -119,6 +140,17 @@ export declare function sampleRows(rows: Row[], count: number): Row[];
|
|
|
119
140
|
export declare function isMissing(cell: Cell | undefined): boolean;
|
|
120
141
|
/** Strict full-string numeric parse (no thousands separators, no unit suffixes). */
|
|
121
142
|
export declare function parseNumeric(cell: Cell | undefined): number | undefined;
|
|
143
|
+
/** Deterministic date-format labels in {@link DATE_PATTERNS} order. */
|
|
144
|
+
declare const DATE_FORMATS: readonly ["iso-date", "slash-date", "datetime"];
|
|
145
|
+
/** The format label of a recognized date cell (source pattern, not the epoch). */
|
|
146
|
+
export type DateFormat = (typeof DATE_FORMATS)[number];
|
|
147
|
+
/** A recognized date cell: its parsed epoch plus the source format label. */
|
|
148
|
+
export interface DateParse {
|
|
149
|
+
/** Epoch milliseconds (UTC). */
|
|
150
|
+
readonly epoch: number;
|
|
151
|
+
/** Which {@link DATE_PATTERNS} entry matched. */
|
|
152
|
+
readonly format: DateFormat;
|
|
153
|
+
}
|
|
122
154
|
/**
|
|
123
155
|
* Deterministic date parse to epoch milliseconds. Accepts `YYYY-MM-DD`,
|
|
124
156
|
* `YYYY/MM/DD`, and ISO-like datetimes (date-only forms read as UTC midnight).
|
|
@@ -128,6 +160,21 @@ export declare function parseNumeric(cell: Cell | undefined): number | undefined
|
|
|
128
160
|
* @returns epoch milliseconds, or `undefined`.
|
|
129
161
|
*/
|
|
130
162
|
export declare function parseDate(cell: Cell | undefined): number | undefined;
|
|
163
|
+
/**
|
|
164
|
+
* Parse one date cell to its epoch plus source format label.
|
|
165
|
+
* @param cell - the cell to parse (`undefined` when the column is absent).
|
|
166
|
+
* @returns the parsed date, or `undefined` when the cell is not a recognized date.
|
|
167
|
+
*/
|
|
168
|
+
export declare function parseDateCell(cell: Cell | undefined): DateParse | undefined;
|
|
169
|
+
/**
|
|
170
|
+
* The format label of a recognized date cell (`iso-date` / `slash-date` /
|
|
171
|
+
* `datetime`); `undefined` when the cell is not a recognized date. Used to
|
|
172
|
+
* measure a date column's format consistency.
|
|
173
|
+
* @param cell - the cell to inspect (`undefined` when the column is absent).
|
|
174
|
+
* @returns the source format label, or `undefined`.
|
|
175
|
+
*/
|
|
176
|
+
export declare function dateFormatOf(cell: Cell | undefined): DateFormat | undefined;
|
|
131
177
|
/** Boolean parse: true/false/yes/no/1/0, case-insensitive. */
|
|
132
178
|
export declare function parseBoolean(cell: Cell | undefined): boolean | undefined;
|
|
179
|
+
export {};
|
|
133
180
|
//# sourceMappingURL=dataset.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dataset.d.ts","sourceRoot":"","sources":["../../src/dataset.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;
|
|
1
|
+
{"version":3,"file":"dataset.d.ts","sourceRoot":"","sources":["../../src/dataset.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAMH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAA;AACzD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAEjD,gHAAgH;AAChH,MAAM,MAAM,IAAI,GAAG,SAAS,CAAA;AAE5B,4CAA4C;AAC5C,MAAM,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;AAEtC,uEAAuE;AACvE,MAAM,WAAW,YAAY;IAC3B,uEAAuE;IACvE,QAAQ,CAAC,GAAG,EAAE,OAAO,GAAG,IAAI,CAAA;IAC5B,kFAAkF;IAClF,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAA;CAC5B;AAED,oDAAoD;AACpD,MAAM,WAAW,KAAK;IACpB,kCAAkC;IAClC,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,CAAA;IAC1B,4EAA4E;IAC5E,QAAQ,CAAC,IAAI,EAAE,GAAG,EAAE,CAAA;IACpB,0EAA0E;IAC1E,QAAQ,CAAC,QAAQ,CAAC,EAAE,YAAY,CAAA;CACjC;AAED,4EAA4E;AAC5E,MAAM,MAAM,YAAY,GACpB;IAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,GAAG,EAAE,CAAA;CAAE,GAC5E;IAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAA;CAAE,CAAA;AAEtD,4FAA4F;AAC5F,qBAAa,YAAa,SAAQ,KAAK;IAMnC,QAAQ,CAAC,IAAI,EACT,aAAa,GACb,kBAAkB,GAClB,WAAW,GACX,WAAW,GACX,eAAe,GACf,WAAW,GACX,aAAa;IAZnB;;;OAGG;gBAEQ,IAAI,EACT,aAAa,GACb,kBAAkB,GAClB,WAAW,GACX,WAAW,GACX,eAAe,GACf,WAAW,GACX,aAAa,EACjB,OAAO,EAAE,MAAM;CAKlB;AAED;;;;;;;;GAQG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,cAAc,GAAG,MAAM,CAoBpG;AAED;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM,CAE3D;AAED,iFAAiF;AACjF,wBAAgB,cAAc,CAAC,MAAM,EAAE,WAAW,GAAG,SAAS,GAAG,IAAI,CAIpE;AAED;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,UAAU,GAAG,YAAY,CAU/D;AAQD;;;;;;GAMG;AACH,wBAAsB,eAAe,CAAC,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,YAAY,CAAA;CAAE,CAAC,CA2B3J;AAED;;;;;;;;;GASG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,KAAK,CA+FnH;AAED;;;;;;;;;GASG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,KAAK,CA8BnH;AAoCD;;;;;;GAMG;AACH,wBAAsB,SAAS,CAAC,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,CAOlH;AAED;;;;;;;;GAQG;AACH,wBAAsB,YAAY,CAAC,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC,CAY5H;AAED;;;;;;GAMG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,MAAM,GAAG,GAAG,EAAE,CAW5D;AAED;;;;;GAKG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,IAAI,GAAG,SAAS,GAAG,OAAO,CAIzD;AAED,oFAAoF;AACpF,wBAAgB,YAAY,CAAC,IAAI,EAAE,IAAI,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CAQvE;AAQD,uEAAuE;AACvE,QAAA,MAAM,YAAY,iDAAkD,CAAA;AAEpE,kFAAkF;AAClF,MAAM,MAAM,UAAU,GAAG,CAAC,OAAO,YAAY,CAAC,CAAC,MAAM,CAAC,CAAA;AAEtD,6EAA6E;AAC7E,MAAM,WAAW,SAAS;IACxB,gCAAgC;IAChC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,iDAAiD;IACjD,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAA;CAC5B;AA0BD;;;;;;;GAOG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,IAAI,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CAEpE;AAED;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,IAAI,GAAG,SAAS,GAAG,SAAS,GAAG,SAAS,CAE3E;AAED;;;;;;GAMG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,IAAI,GAAG,SAAS,GAAG,UAAU,GAAG,SAAS,CAE3E;AAED,8DAA8D;AAC9D,wBAAgB,YAAY,CAAC,IAAI,EAAE,IAAI,GAAG,SAAS,GAAG,OAAO,GAAG,SAAS,CAOxE"}
|
package/lib/types/dataset.js
CHANGED
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
import { open, stat } from 'node:fs/promises';
|
|
9
9
|
import path from 'node:path';
|
|
10
10
|
import { createHash } from 'node:crypto';
|
|
11
|
+
import { TextDecoder } from 'node:util';
|
|
11
12
|
/** Raised for every user-facing dataset failure; `code` is stable for tests and tooling. */
|
|
12
13
|
export class DatasetError extends Error {
|
|
13
14
|
code;
|
|
@@ -60,13 +61,38 @@ export function throwIfAborted(signal) {
|
|
|
60
61
|
}
|
|
61
62
|
}
|
|
62
63
|
/**
|
|
63
|
-
*
|
|
64
|
+
* Detect a UTF-8 byte-order mark and validate the byte sequence. Invalid
|
|
65
|
+
* UTF-8 is a data-quality finding the profile reports (`validUtf8: false`)
|
|
66
|
+
* rather than a structural error that blocks the read — the decoded text
|
|
67
|
+
* keeps U+FFFD replacement characters so the profile can still run.
|
|
68
|
+
* @param buffer - raw file bytes.
|
|
69
|
+
* @returns the encoding metadata.
|
|
70
|
+
*/
|
|
71
|
+
export function detectEncoding(buffer) {
|
|
72
|
+
const bom = buffer.length >= 3 && buffer[0] === 0xef && buffer[1] === 0xbb && buffer[2] === 0xbf ? 'utf-8' : null;
|
|
73
|
+
const body = bom === 'utf-8' ? buffer.subarray(3) : buffer;
|
|
74
|
+
let validUtf8 = true;
|
|
75
|
+
try {
|
|
76
|
+
new TextDecoder('utf-8', { fatal: true }).decode(body);
|
|
77
|
+
}
|
|
78
|
+
catch {
|
|
79
|
+
validUtf8 = false;
|
|
80
|
+
}
|
|
81
|
+
return { bom, validUtf8 };
|
|
82
|
+
}
|
|
83
|
+
/** Decode file bytes, stripping a UTF-8 BOM and preserving replacement characters for invalid bytes. */
|
|
84
|
+
function decodeUtf8Text(buffer, encoding) {
|
|
85
|
+
const body = encoding.bom === 'utf-8' ? buffer.subarray(3) : buffer;
|
|
86
|
+
return new TextDecoder('utf-8').decode(body);
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Read a dataset file under the size cap, detecting its encoding.
|
|
64
90
|
* @param absolutePath - normalized absolute path (from {@link resolveWorkspacePath}).
|
|
65
91
|
* @param config - resolved config (size cap).
|
|
66
92
|
* @param signal - optional abort signal honored around the read.
|
|
67
|
-
* @returns the UTF-8 text.
|
|
93
|
+
* @returns the decoded UTF-8 text plus its encoding metadata.
|
|
68
94
|
*/
|
|
69
|
-
export async function
|
|
95
|
+
export async function readDatasetFile(absolutePath, config, signal) {
|
|
70
96
|
throwIfAborted(signal);
|
|
71
97
|
let info;
|
|
72
98
|
try {
|
|
@@ -85,7 +111,9 @@ export async function readDatasetText(absolutePath, config, signal) {
|
|
|
85
111
|
const handle = await open(absolutePath, 'r');
|
|
86
112
|
try {
|
|
87
113
|
throwIfAborted(signal);
|
|
88
|
-
|
|
114
|
+
const buffer = await handle.readFile();
|
|
115
|
+
const encoding = detectEncoding(buffer);
|
|
116
|
+
return { text: decodeUtf8Text(buffer, encoding), encoding };
|
|
89
117
|
}
|
|
90
118
|
finally {
|
|
91
119
|
await handle.close();
|
|
@@ -273,14 +301,14 @@ function parseJsonLines(text) {
|
|
|
273
301
|
* @returns the parsed table.
|
|
274
302
|
*/
|
|
275
303
|
export async function loadTable(absolutePath, config, signal) {
|
|
276
|
-
const text = await
|
|
304
|
+
const { text, encoding } = await readDatasetFile(absolutePath, config, signal);
|
|
277
305
|
const ext = path.extname(absolutePath).toLowerCase();
|
|
278
306
|
throwIfAborted(signal);
|
|
279
307
|
if (ext === '.csv')
|
|
280
|
-
return parseDelimited(text, ',', config, signal);
|
|
308
|
+
return { ...parseDelimited(text, ',', config, signal), encoding };
|
|
281
309
|
if (ext === '.tsv')
|
|
282
|
-
return parseDelimited(text, '\t', config, signal);
|
|
283
|
-
return parseJsonTable(text, ext, config, signal);
|
|
310
|
+
return { ...parseDelimited(text, '\t', config, signal), encoding };
|
|
311
|
+
return { ...parseJsonTable(text, ext, config, signal), encoding };
|
|
284
312
|
}
|
|
285
313
|
/**
|
|
286
314
|
* Load a dataset as a citation-checkable document root: CSV/TSV become
|
|
@@ -294,11 +322,11 @@ export async function loadTable(absolutePath, config, signal) {
|
|
|
294
322
|
export async function loadDocument(absolutePath, config, signal) {
|
|
295
323
|
const ext = path.extname(absolutePath).toLowerCase();
|
|
296
324
|
if (ext === '.json') {
|
|
297
|
-
const text = await
|
|
325
|
+
const { text } = await readDatasetFile(absolutePath, config, signal);
|
|
298
326
|
return { kind: 'json', value: parseJsonDocument(text) };
|
|
299
327
|
}
|
|
300
328
|
if (ext === '.jsonl') {
|
|
301
|
-
const text = await
|
|
329
|
+
const { text } = await readDatasetFile(absolutePath, config, signal);
|
|
302
330
|
return { kind: 'json', value: parseJsonLines(text) };
|
|
303
331
|
}
|
|
304
332
|
const table = await loadTable(absolutePath, config, signal);
|
|
@@ -356,19 +384,14 @@ const DATE_PATTERNS = [
|
|
|
356
384
|
/^(\d{4})\/(\d{1,2})\/(\d{1,2})$/u,
|
|
357
385
|
/^(\d{4})-(\d{1,2})-(\d{1,2})[ T](\d{1,2}):(\d{2})(?::(\d{2}))?(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?$/u,
|
|
358
386
|
];
|
|
359
|
-
/**
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
* the cell is not a recognized date.
|
|
364
|
-
* @param cell - the cell to parse (`undefined` when the column is absent).
|
|
365
|
-
* @returns epoch milliseconds, or `undefined`.
|
|
366
|
-
*/
|
|
367
|
-
export function parseDate(cell) {
|
|
387
|
+
/** Deterministic date-format labels in {@link DATE_PATTERNS} order. */
|
|
388
|
+
const DATE_FORMATS = ['iso-date', 'slash-date', 'datetime'];
|
|
389
|
+
/** Parse one date cell to its epoch plus format; `undefined` when unrecognized. */
|
|
390
|
+
function parseDateCellInternal(cell) {
|
|
368
391
|
if (typeof cell !== 'string')
|
|
369
392
|
return undefined;
|
|
370
393
|
const text = cell.trim();
|
|
371
|
-
for (const pattern of DATE_PATTERNS) {
|
|
394
|
+
for (const [index, pattern] of DATE_PATTERNS.entries()) {
|
|
372
395
|
const match = pattern.exec(text);
|
|
373
396
|
if (match === null)
|
|
374
397
|
continue;
|
|
@@ -384,10 +407,42 @@ export function parseDate(cell) {
|
|
|
384
407
|
const check = new Date(epoch);
|
|
385
408
|
if (check.getUTCMonth() !== month - 1 || check.getUTCDate() !== day)
|
|
386
409
|
return undefined;
|
|
387
|
-
|
|
410
|
+
const format = DATE_FORMATS[index];
|
|
411
|
+
if (format === undefined)
|
|
412
|
+
return undefined;
|
|
413
|
+
return { epoch, format };
|
|
388
414
|
}
|
|
389
415
|
return undefined;
|
|
390
416
|
}
|
|
417
|
+
/**
|
|
418
|
+
* Deterministic date parse to epoch milliseconds. Accepts `YYYY-MM-DD`,
|
|
419
|
+
* `YYYY/MM/DD`, and ISO-like datetimes (date-only forms read as UTC midnight).
|
|
420
|
+
* Calendar-invalid dates (e.g. 2025-13-40) reject. Returns `undefined` when
|
|
421
|
+
* the cell is not a recognized date.
|
|
422
|
+
* @param cell - the cell to parse (`undefined` when the column is absent).
|
|
423
|
+
* @returns epoch milliseconds, or `undefined`.
|
|
424
|
+
*/
|
|
425
|
+
export function parseDate(cell) {
|
|
426
|
+
return parseDateCellInternal(cell)?.epoch;
|
|
427
|
+
}
|
|
428
|
+
/**
|
|
429
|
+
* Parse one date cell to its epoch plus source format label.
|
|
430
|
+
* @param cell - the cell to parse (`undefined` when the column is absent).
|
|
431
|
+
* @returns the parsed date, or `undefined` when the cell is not a recognized date.
|
|
432
|
+
*/
|
|
433
|
+
export function parseDateCell(cell) {
|
|
434
|
+
return parseDateCellInternal(cell);
|
|
435
|
+
}
|
|
436
|
+
/**
|
|
437
|
+
* The format label of a recognized date cell (`iso-date` / `slash-date` /
|
|
438
|
+
* `datetime`); `undefined` when the cell is not a recognized date. Used to
|
|
439
|
+
* measure a date column's format consistency.
|
|
440
|
+
* @param cell - the cell to inspect (`undefined` when the column is absent).
|
|
441
|
+
* @returns the source format label, or `undefined`.
|
|
442
|
+
*/
|
|
443
|
+
export function dateFormatOf(cell) {
|
|
444
|
+
return parseDateCellInternal(cell)?.format;
|
|
445
|
+
}
|
|
391
446
|
/** Boolean parse: true/false/yes/no/1/0, case-insensitive. */
|
|
392
447
|
export function parseBoolean(cell) {
|
|
393
448
|
if (typeof cell === 'boolean')
|