dsh-data-quality 0.1.2 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (96) hide show
  1. package/CHANGELOG.md +32 -0
  2. package/README.es.md +8 -2
  3. package/README.hi.md +8 -2
  4. package/README.md +22 -9
  5. package/README.pt.md +8 -2
  6. package/README.zh.md +8 -2
  7. package/cordis.patch.yml +11 -0
  8. package/lib/index.js +1411 -145
  9. package/lib/types/config.d.ts +12 -0
  10. package/lib/types/config.d.ts.map +1 -1
  11. package/lib/types/config.js +35 -0
  12. package/lib/types/config.js.map +1 -1
  13. package/lib/types/contract.d.ts +83 -0
  14. package/lib/types/contract.d.ts.map +1 -0
  15. package/lib/types/contract.js +144 -0
  16. package/lib/types/contract.js.map +1 -0
  17. package/lib/types/dataset.d.ts +50 -3
  18. package/lib/types/dataset.d.ts.map +1 -1
  19. package/lib/types/dataset.js +76 -21
  20. package/lib/types/dataset.js.map +1 -1
  21. package/lib/types/events.d.ts +2 -0
  22. package/lib/types/events.d.ts.map +1 -1
  23. package/lib/types/events.js.map +1 -1
  24. package/lib/types/index.d.ts +11 -8
  25. package/lib/types/index.d.ts.map +1 -1
  26. package/lib/types/index.js +15 -6
  27. package/lib/types/index.js.map +1 -1
  28. package/lib/types/presets.d.ts +28 -0
  29. package/lib/types/presets.d.ts.map +1 -0
  30. package/lib/types/presets.js +134 -0
  31. package/lib/types/presets.js.map +1 -0
  32. package/lib/types/profile.d.ts +38 -1
  33. package/lib/types/profile.d.ts.map +1 -1
  34. package/lib/types/profile.js +64 -10
  35. package/lib/types/profile.js.map +1 -1
  36. package/lib/types/provider-local.d.ts +7 -1
  37. package/lib/types/provider-local.d.ts.map +1 -1
  38. package/lib/types/provider-local.js +78 -4
  39. package/lib/types/provider-local.js.map +1 -1
  40. package/lib/types/scorecard.d.ts +50 -0
  41. package/lib/types/scorecard.d.ts.map +1 -0
  42. package/lib/types/scorecard.js +186 -0
  43. package/lib/types/scorecard.js.map +1 -0
  44. package/lib/types/service.d.ts +32 -2
  45. package/lib/types/service.d.ts.map +1 -1
  46. package/lib/types/service.js.map +1 -1
  47. package/lib/types/store.d.ts +21 -1
  48. package/lib/types/store.d.ts.map +1 -1
  49. package/lib/types/store.js +13 -1
  50. package/lib/types/store.js.map +1 -1
  51. package/lib/types/tools/clean.d.ts.map +1 -1
  52. package/lib/types/tools/clean.js +76 -1
  53. package/lib/types/tools/clean.js.map +1 -1
  54. package/lib/types/tools/profile-report-schema.d.ts +315 -0
  55. package/lib/types/tools/profile-report-schema.d.ts.map +1 -0
  56. package/lib/types/tools/profile-report-schema.js +95 -0
  57. package/lib/types/tools/profile-report-schema.js.map +1 -0
  58. package/lib/types/tools/profile.d.ts.map +1 -1
  59. package/lib/types/tools/profile.js +6 -53
  60. package/lib/types/tools/profile.js.map +1 -1
  61. package/lib/types/tools/report.d.ts +14 -0
  62. package/lib/types/tools/report.d.ts.map +1 -0
  63. package/lib/types/tools/report.js +86 -0
  64. package/lib/types/tools/report.js.map +1 -0
  65. package/lib/types/tools/shared.d.ts.map +1 -1
  66. package/lib/types/tools/shared.js +22 -2
  67. package/lib/types/tools/shared.js.map +1 -1
  68. package/lib/types/tools/verify.d.ts.map +1 -1
  69. package/lib/types/tools/verify.js +36 -1
  70. package/lib/types/tools/verify.js.map +1 -1
  71. package/lib/types/verify.d.ts +49 -3
  72. package/lib/types/verify.d.ts.map +1 -1
  73. package/lib/types/verify.js +111 -4
  74. package/lib/types/verify.js.map +1 -1
  75. package/lib/types/version.d.ts +1 -1
  76. package/lib/types/version.js +1 -1
  77. package/package.json +1 -1
  78. package/src/config.ts +52 -0
  79. package/src/contract.ts +190 -0
  80. package/src/dataset.ts +101 -21
  81. package/src/events.ts +2 -0
  82. package/src/index.ts +25 -8
  83. package/src/presets.ts +146 -0
  84. package/src/profile.ts +101 -11
  85. package/src/provider-local.ts +83 -5
  86. package/src/scorecard.ts +244 -0
  87. package/src/service.ts +34 -2
  88. package/src/store.ts +28 -2
  89. package/src/tools/clean.ts +76 -1
  90. package/src/tools/profile-report-schema.ts +97 -0
  91. package/src/tools/profile.ts +8 -56
  92. package/src/tools/report.ts +108 -0
  93. package/src/tools/shared.ts +21 -2
  94. package/src/tools/verify.ts +40 -2
  95. package/src/verify.ts +151 -5
  96. package/src/version.ts +1 -1
@@ -5,6 +5,15 @@
5
5
  * @module dsh-data-quality/config
6
6
  */
7
7
  import z from '@deepseek-ai/schemastery';
8
+ /** Per-dimension scorecard weights (all optional; defaults fill to `1` = equal). */
9
+ export interface ScorecardWeights {
10
+ completeness?: number;
11
+ uniqueness?: number;
12
+ validity?: number;
13
+ consistency?: number;
14
+ timeliness?: number;
15
+ accuracy?: number;
16
+ }
8
17
  /** Raw plugin config — every field optional; {@link resolveConfig} supplies the defaults. */
9
18
  export interface Config {
10
19
  /** Master switch; `false` mounts nothing. */
@@ -23,6 +32,8 @@ export interface Config {
23
32
  workspaceRoot?: string;
24
33
  /** Persist reports to the `data_quality` storage domain. */
25
34
  storeReports?: boolean;
35
+ /** Per-dimension weights for the scorecard's weighted overall total (non-negative). */
36
+ scorecardWeights?: ScorecardWeights;
26
37
  }
27
38
  /** Fully resolved config handed to the runtime. */
28
39
  export interface ResolvedConfig {
@@ -34,6 +45,7 @@ export interface ResolvedConfig {
34
45
  readonly allowedExtensions: readonly string[];
35
46
  readonly workspaceRoot: string;
36
47
  readonly storeReports: boolean;
48
+ readonly scorecardWeights: Required<ScorecardWeights>;
37
49
  }
38
50
  /** Schemastery schema: the loader validates and fills defaults before `apply`. */
39
51
  export declare const Config: z<Config>;
@@ -1 +1 @@
1
- {"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/config.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,CAAC,MAAM,0BAA0B,CAAA;AAExC,6FAA6F;AAC7F,MAAM,WAAW,MAAM;IACrB,6CAA6C;IAC7C,OAAO,CAAC,EAAE,OAAO,CAAA;IACjB,qCAAqC;IACrC,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,kDAAkD;IAClD,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,kEAAkE;IAClE,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,4EAA4E;IAC5E,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,iEAAiE;IACjE,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAA;IAC5B,2GAA2G;IAC3G,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,4DAA4D;IAC5D,YAAY,CAAC,EAAE,OAAO,CAAA;CACvB;AAED,mDAAmD;AACnD,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAA;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAA;IAC9B,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAA;IACjC,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAA;IACjC,QAAQ,CAAC,iBAAiB,EAAE,SAAS,MAAM,EAAE,CAAA;IAC7C,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAA;IAC9B,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAA;CAC/B;AAED,kFAAkF;AAClF,eAAO,MAAM,MAAM,EAAE,CAAC,CAAC,MAAM,CAS3B,CAAA;AAgBF;;;;;GAKG;AACH,wBAAgB,aAAa,CAAC,MAAM,GAAE,MAAW,GAAG,cAAc,CA8BjE"}
1
+ {"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/config.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,CAAC,MAAM,0BAA0B,CAAA;AAExC,oFAAoF;AACpF,MAAM,WAAW,gBAAgB;IAC/B,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAA;CAClB;AAeD,6FAA6F;AAC7F,MAAM,WAAW,MAAM;IACrB,6CAA6C;IAC7C,OAAO,CAAC,EAAE,OAAO,CAAA;IACjB,qCAAqC;IACrC,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,kDAAkD;IAClD,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,kEAAkE;IAClE,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,4EAA4E;IAC5E,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,iEAAiE;IACjE,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAA;IAC5B,2GAA2G;IAC3G,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,4DAA4D;IAC5D,YAAY,CAAC,EAAE,OAAO,CAAA;IACtB,uFAAuF;IACvF,gBAAgB,CAAC,EAAE,gBAAgB,CAAA;CACpC;AAED,mDAAmD;AACnD,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAA;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAA;IAC9B,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAA;IACjC,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAA;IACjC,QAAQ,CAAC,iBAAiB,EAAE,SAAS,MAAM,EAAE,CAAA;IAC7C,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAA;IAC9B,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAA;IAC9B,QAAQ,CAAC,gBAAgB,EAAE,QAAQ,CAAC,gBAAgB,CAAC,CAAA;CACtD;AAED,kFAAkF;AAClF,eAAO,MAAM,MAAM,EAAE,CAAC,CAAC,MAAM,CAmB3B,CAAA;AAgBF;;;;;GAKG;AACH,wBAAgB,aAAa,CAAC,MAAM,GAAE,MAAW,GAAG,cAAc,CAiCjE"}
@@ -5,6 +5,17 @@
5
5
  * @module dsh-data-quality/config
6
6
  */
7
7
  import z from '@deepseek-ai/schemastery';
8
+ /** The six scorecard dimension ids, in report order (kept in sync with `scorecard.ts`). */
9
+ const SCORECARD_DIMENSIONS = ['completeness', 'uniqueness', 'validity', 'consistency', 'timeliness', 'accuracy'];
10
+ /** Equal-weight defaults for {@link ScorecardWeights}. */
11
+ const EQUAL_WEIGHTS = {
12
+ completeness: 1,
13
+ uniqueness: 1,
14
+ validity: 1,
15
+ consistency: 1,
16
+ timeliness: 1,
17
+ accuracy: 1,
18
+ };
8
19
  /** Schemastery schema: the loader validates and fills defaults before `apply`. */
9
20
  export const Config = z.object({
10
21
  enabled: z.boolean().default(true),
@@ -15,6 +26,16 @@ export const Config = z.object({
15
26
  allowedExtensions: z.array(z.string()).default(['.csv', '.tsv', '.json', '.jsonl']),
16
27
  workspaceRoot: z.string().default(''),
17
28
  storeReports: z.boolean().default(true),
29
+ scorecardWeights: z
30
+ .object({
31
+ completeness: z.number().default(1),
32
+ uniqueness: z.number().default(1),
33
+ validity: z.number().default(1),
34
+ consistency: z.number().default(1),
35
+ timeliness: z.number().default(1),
36
+ accuracy: z.number().default(1),
37
+ })
38
+ .default(EQUAL_WEIGHTS),
18
39
  });
19
40
  /** Throw unless `value` is a positive safe integer. */
20
41
  function assertPositiveInt(name, value) {
@@ -52,6 +73,7 @@ export function resolveConfig(config = {}) {
52
73
  if (allowedExtensions.length === 0) {
53
74
  throw new TypeError('allowedExtensions must not be empty');
54
75
  }
76
+ const scorecardWeights = resolveScorecardWeights(config.scorecardWeights);
55
77
  return {
56
78
  enabled: config.enabled ?? true,
57
79
  maxRows,
@@ -61,6 +83,19 @@ export function resolveConfig(config = {}) {
61
83
  allowedExtensions,
62
84
  workspaceRoot: config.workspaceRoot ?? '',
63
85
  storeReports: config.storeReports ?? true,
86
+ scorecardWeights,
64
87
  };
65
88
  }
89
+ /** Resolve scorecard weights, filling defaults and rejecting non-negative violations loudly. */
90
+ function resolveScorecardWeights(weights = {}) {
91
+ const out = {};
92
+ for (const dimension of SCORECARD_DIMENSIONS) {
93
+ const value = weights[dimension] ?? 1;
94
+ if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) {
95
+ throw new TypeError(`scorecardWeights.${dimension} must be a non-negative finite number, got ${String(value)}`);
96
+ }
97
+ out[dimension] = value;
98
+ }
99
+ return out;
100
+ }
66
101
  //# sourceMappingURL=config.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/config.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,CAAC,MAAM,0BAA0B,CAAA;AAkCxC,kFAAkF;AAClF,MAAM,CAAC,MAAM,MAAM,GAAc,CAAC,CAAC,MAAM,CAAC;IACxC,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;IAClC,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC;IACpC,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IACrC,gBAAgB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;IAC1C,gBAAgB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IACxC,iBAAiB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;IACnF,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IACrC,YAAY,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;CACxC,CAAC,CAAA;AAEF,uDAAuD;AACvD,SAAS,iBAAiB,CAAC,IAAY,EAAE,KAAa;IACpD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;QAC/C,MAAM,IAAI,SAAS,CAAC,GAAG,IAAI,yCAAyC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;IACtF,CAAC;AACH,CAAC;AAED,+DAA+D;AAC/D,SAAS,iBAAiB,CAAC,IAAY,EAAE,KAAa,EAAE,GAAW,EAAE,GAAW;IAC9E,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,GAAG,IAAI,KAAK,GAAG,GAAG,EAAE,CAAC;QACvF,MAAM,IAAI,SAAS,CAAC,GAAG,IAAI,gCAAgC,GAAG,KAAK,GAAG,UAAU,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;IAClG,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,aAAa,CAAC,SAAiB,EAAE;IAC/C,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,OAAO,CAAA;IACzC,iBAAiB,CAAC,SAAS,EAAE,OAAO,CAAC,CAAA;IACrC,MAAM,aAAa,GAAG,MAAM,CAAC,aAAa,IAAI,EAAE,CAAA;IAChD,iBAAiB,CAAC,eAAe,EAAE,aAAa,EAAE,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,CAAA;IACzE,MAAM,gBAAgB,GAAG,MAAM,CAAC,gBAAgB,IAAI,IAAI,CAAA;IACxD,iBAAiB,CAAC,kBAAkB,EAAE,gBAAgB,EAAE,CAAC,EAAE,CAAC,CAAC,CAAA;IAC7D,MAAM,gBAAgB,GAAG,MAAM,CAAC,gBAAgB,IAAI,EAAE,CAAA;IACtD,iBAAiB,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,CAAA;IAEvD,MAAM,iBAAiB,GAAG,CAAC,MAAM,CAAC,iBAAiB,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QACtG,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YAChC,MAAM,IAAI,SAAS,CAAC,4EAA4E,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QACxH,CAAC;QACD,OAAO,GAAG,CAAA;IACZ,CAAC,CAAC,CAAA;IACF,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnC,MAAM,IAAI,SAAS,CAAC,qCAAqC,CAAC,CAAA;IAC5D,CAAC;IAED,OAAO;QACL,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,IAAI;QAC/B,OAAO;QACP,aAAa;QACb,gBAAgB;QAChB,gBAAgB;QAChB,iBAAiB;QACjB,aAAa,EAAE,MAAM,CAAC,aAAa,IAAI,EAAE;QACzC,YAAY,EAAE,MAAM,CAAC,YAAY,IAAI,IAAI;KAC1C,CAAA;AACH,CAAC"}
1
+ {"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/config.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,CAAC,MAAM,0BAA0B,CAAA;AAYxC,2FAA2F;AAC3F,MAAM,oBAAoB,GAAG,CAAC,cAAc,EAAE,YAAY,EAAE,UAAU,EAAE,aAAa,EAAE,YAAY,EAAE,UAAU,CAAU,CAAA;AAEzH,0DAA0D;AAC1D,MAAM,aAAa,GAA+B;IAChD,YAAY,EAAE,CAAC;IACf,UAAU,EAAE,CAAC;IACb,QAAQ,EAAE,CAAC;IACX,WAAW,EAAE,CAAC;IACd,UAAU,EAAE,CAAC;IACb,QAAQ,EAAE,CAAC;CACZ,CAAA;AAqCD,kFAAkF;AAClF,MAAM,CAAC,MAAM,MAAM,GAAc,CAAC,CAAC,MAAM,CAAC;IACxC,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;IAClC,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC;IACpC,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IACrC,gBAAgB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;IAC1C,gBAAgB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IACxC,iBAAiB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;IACnF,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IACrC,YAAY,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;IACvC,gBAAgB,EAAE,CAAC;SAChB,MAAM,CAAC;QACN,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;QACnC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;QACjC,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;QAC/B,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;QAClC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;QACjC,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;KAChC,CAAC;SACD,OAAO,CAAC,aAAa,CAAC;CAC1B,CAAC,CAAA;AAEF,uDAAuD;AACvD,SAAS,iBAAiB,CAAC,IAAY,EAAE,KAAa;IACpD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;QAC/C,MAAM,IAAI,SAAS,CAAC,GAAG,IAAI,yCAAyC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;IACtF,CAAC;AACH,CAAC;AAED,+DAA+D;AAC/D,SAAS,iBAAiB,CAAC,IAAY,EAAE,KAAa,EAAE,GAAW,EAAE,GAAW;IAC9E,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,GAAG,IAAI,KAAK,GAAG,GAAG,EAAE,CAAC;QACvF,MAAM,IAAI,SAAS,CAAC,GAAG,IAAI,gCAAgC,GAAG,KAAK,GAAG,UAAU,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;IAClG,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,aAAa,CAAC,SAAiB,EAAE;IAC/C,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,OAAO,CAAA;IACzC,iBAAiB,CAAC,SAAS,EAAE,OAAO,CAAC,CAAA;IACrC,MAAM,aAAa,GAAG,MAAM,CAAC,aAAa,IAAI,EAAE,CAAA;IAChD,iBAAiB,CAAC,eAAe,EAAE,aAAa,EAAE,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,CAAA;IACzE,MAAM,gBAAgB,GAAG,MAAM,CAAC,gBAAgB,IAAI,IAAI,CAAA;IACxD,iBAAiB,CAAC,kBAAkB,EAAE,gBAAgB,EAAE,CAAC,EAAE,CAAC,CAAC,CAAA;IAC7D,MAAM,gBAAgB,GAAG,MAAM,CAAC,gBAAgB,IAAI,EAAE,CAAA;IACtD,iBAAiB,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,CAAA;IAEvD,MAAM,iBAAiB,GAAG,CAAC,MAAM,CAAC,iBAAiB,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QACtG,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YAChC,MAAM,IAAI,SAAS,CAAC,4EAA4E,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QACxH,CAAC;QACD,OAAO,GAAG,CAAA;IACZ,CAAC,CAAC,CAAA;IACF,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnC,MAAM,IAAI,SAAS,CAAC,qCAAqC,CAAC,CAAA;IAC5D,CAAC;IAED,MAAM,gBAAgB,GAAG,uBAAuB,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAA;IAEzE,OAAO;QACL,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,IAAI;QAC/B,OAAO;QACP,aAAa;QACb,gBAAgB;QAChB,gBAAgB;QAChB,iBAAiB;QACjB,aAAa,EAAE,MAAM,CAAC,aAAa,IAAI,EAAE;QACzC,YAAY,EAAE,MAAM,CAAC,YAAY,IAAI,IAAI;QACzC,gBAAgB;KACjB,CAAA;AACH,CAAC;AAED,gGAAgG;AAChG,SAAS,uBAAuB,CAAC,UAA4B,EAAE;IAC7D,MAAM,GAAG,GAAG,EAAgC,CAAA;IAC5C,KAAK,MAAM,SAAS,IAAI,oBAAoB,EAAE,CAAC;QAC7C,MAAM,KAAK,GAAG,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;QACrC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YACtE,MAAM,IAAI,SAAS,CAAC,oBAAoB,SAAS,8CAA8C,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;QACjH,CAAC;QACD,GAAG,CAAC,SAAS,CAAC,GAAG,KAAK,CAAA;IACxB,CAAC;IACD,OAAO,GAAG,CAAA;AACZ,CAAC"}
@@ -0,0 +1,83 @@
1
+ /**
2
+ * Delivery contract for `data_clean`: a deterministic pre-delivery validation
3
+ * summary (primary-key/type/dedupe row-count comparison plus uniqueness and
4
+ * non-null regression) and the clean before/after profile diff report. Pure —
5
+ * no I/O, no clock, no RNG; timestamps are injected.
6
+ * @module dsh-data-quality/contract
7
+ */
8
+ import { type Table } from './dataset.js';
9
+ import type { CleanResult, CleanRule } from './clean.js';
10
+ import { type ProfileReport } from './profile.js';
11
+ import type { ScorecardDimensionName } from './scorecard.js';
12
+ /** The pre-delivery contract summary attached to a clean run. */
13
+ export interface CleanContractSummary {
14
+ /** Row count before cleaning. */
15
+ readonly inputRows: number;
16
+ /** Row count after cleaning. */
17
+ readonly outputRows: number;
18
+ /** Rows removed across all rules. */
19
+ readonly removedRows: number;
20
+ /** Key columns of the first `dedupe` rule; `null` when no dedupe rule ran (uniqueness then covers full rows). */
21
+ readonly dedupeColumns: string[] | null;
22
+ /** Whether the uniqueness key is free of duplicates in the cleaned output. */
23
+ readonly uniqueKeys: boolean;
24
+ /** Remaining duplicate rows over the uniqueness key in the cleaned output. */
25
+ readonly remainingDuplicateRows: number;
26
+ /** Non-null regression: remaining missing cells in columns a `fill-missing` rule targeted. */
27
+ readonly remainingMissing: Array<{
28
+ readonly column: string;
29
+ readonly count: number;
30
+ }>;
31
+ /** Type regression: remaining non-conforming cells in columns a `coerce-type` rule targeted. */
32
+ readonly typeConformance: Array<{
33
+ readonly column: string;
34
+ readonly to: 'number' | 'date' | 'boolean';
35
+ readonly invalidCount: number;
36
+ }>;
37
+ /** Per-column decision trace: which strategies touched each column and how many rows they affected. */
38
+ readonly columnDecisions: Array<{
39
+ readonly column: string;
40
+ readonly decisions: Array<{
41
+ readonly strategy: string;
42
+ readonly affectedRows: number;
43
+ }>;
44
+ }>;
45
+ }
46
+ /** The clean before/after profile diff report (persisted under the `clean-diff` report kind). */
47
+ export interface CleanProfileDiff {
48
+ readonly dataset: string;
49
+ /** Full profile of the input dataset. */
50
+ readonly before: ProfileReport;
51
+ /** Full profile of the cleaned output (including its scorecard). */
52
+ readonly after: ProfileReport;
53
+ readonly generatedAt: number;
54
+ }
55
+ /**
56
+ * Compute the pre-delivery contract summary from a cleaning run: dedupe
57
+ * before/after row counts, the dedupe key's uniqueness regression, and the
58
+ * non-null/type regressions over the columns the rules targeted.
59
+ * @param result - the cleaning outcome (input/output rows plus cleaned rows).
60
+ * @param rules - the ordered rules that produced `result`.
61
+ * @param options - optional abort signal.
62
+ * @returns the contract summary.
63
+ */
64
+ export declare function computeCleanContract(result: CleanResult, rules: readonly CleanRule[], options?: {
65
+ signal?: AbortSignal | undefined;
66
+ }): CleanContractSummary;
67
+ /**
68
+ * Build the clean before/after profile diff: two full {@link ProfileReport}
69
+ * snapshots (input and cleaned output) under one envelope. Reuses the profile
70
+ * engine so the scorecard and duplicate detection stay consistent across the
71
+ * whole plugin.
72
+ * @param input - the input dataset.
73
+ * @param output - the cleaned dataset.
74
+ * @param options - dataset label, injected timestamp, optional weights, optional abort signal.
75
+ * @returns the diff report.
76
+ */
77
+ export declare function computeCleanProfileDiff(input: Table, output: Table, options: {
78
+ dataset: string;
79
+ generatedAt: number;
80
+ scorecardWeights?: Readonly<Record<ScorecardDimensionName, number>> | undefined;
81
+ signal?: AbortSignal | undefined;
82
+ }): CleanProfileDiff;
83
+ //# sourceMappingURL=contract.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"contract.d.ts","sourceRoot":"","sources":["../../src/contract.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAA8E,KAAK,KAAK,EAAE,MAAM,cAAc,CAAA;AACrH,OAAO,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AACxD,OAAO,EAAgB,KAAK,aAAa,EAAE,MAAM,cAAc,CAAA;AAC/D,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAA;AAE5D,iEAAiE;AACjE,MAAM,WAAW,oBAAoB;IACnC,iCAAiC;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,gCAAgC;IAChC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,qCAAqC;IACrC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IAC5B,iHAAiH;IACjH,QAAQ,CAAC,aAAa,EAAE,MAAM,EAAE,GAAG,IAAI,CAAA;IACvC,8EAA8E;IAC9E,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAA;IAC5B,8EAA8E;IAC9E,QAAQ,CAAC,sBAAsB,EAAE,MAAM,CAAA;IACvC,8FAA8F;IAC9F,QAAQ,CAAC,gBAAgB,EAAE,KAAK,CAAC;QAAE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;IACrF,gGAAgG;IAChG,QAAQ,CAAC,eAAe,EAAE,KAAK,CAAC;QAAE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,EAAE,QAAQ,GAAG,MAAM,GAAG,SAAS,CAAC;QAAC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;IACvI,uGAAuG;IACvG,QAAQ,CAAC,eAAe,EAAE,KAAK,CAAC;QAC9B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;QACvB,QAAQ,CAAC,SAAS,EAAE,KAAK,CAAC;YAAE,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;YAAC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KACxF,CAAC,CAAA;CACH;AAED,iGAAiG;AACjG,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,yCAAyC;IACzC,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAA;IAC9B,oEAAoE;IACpE,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAA;IAC7B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;CAC7B;AAyBD;;;;;;;;GAQG;AACH,wBAAgB,oBAAoB,CAClC,MAAM,EAAE,WAAW,EACnB,KAAK,EAAE,SAAS,SAAS,EAAE,EAC3B,OAAO,GAAE;IAAE,MAAM,CAAC,EAAE,WAAW,GAAG,SAAS,CAAA;CAAO,GACjD,oBAAoB,CAuCtB;AAsCD;;;;;;;;;GASG;AACH,wBAAgB,uBAAuB,CACrC,KAAK,EAAE,KAAK,EACZ,MAAM,EAAE,KAAK,EACb,OAAO,EAAE;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,MAAM,CAAC;IAAC,gBAAgB,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,sBAAsB,EAAE,MAAM,CAAC,CAAC,GAAG,SAAS,CAAC;IAAC,MAAM,CAAC,EAAE,WAAW,GAAG,SAAS,CAAA;CAAE,GACnK,gBAAgB,CAclB"}
@@ -0,0 +1,144 @@
1
+ /**
2
+ * Delivery contract for `data_clean`: a deterministic pre-delivery validation
3
+ * summary (primary-key/type/dedupe row-count comparison plus uniqueness and
4
+ * non-null regression) and the clean before/after profile diff report. Pure —
5
+ * no I/O, no clock, no RNG; timestamps are injected.
6
+ * @module dsh-data-quality/contract
7
+ */
8
+ import { isMissing, parseBoolean, parseDate, parseNumeric, throwIfAborted } from "./dataset.js";
9
+ import { profileTable } from "./profile.js";
10
+ /** Count missing cells in one column of the cleaned output. */
11
+ function countMissing(rows, column, signal) {
12
+ let count = 0;
13
+ for (const [index, row] of rows.entries()) {
14
+ if (index % 1024 === 0)
15
+ throwIfAborted(signal);
16
+ if (isMissing(row[column]))
17
+ count += 1;
18
+ }
19
+ return count;
20
+ }
21
+ /** Count present cells in one column that still fail to parse as the requested type. */
22
+ function countNonConforming(rows, column, to, signal) {
23
+ let count = 0;
24
+ for (const [index, row] of rows.entries()) {
25
+ if (index % 1024 === 0)
26
+ throwIfAborted(signal);
27
+ const cell = row[column];
28
+ if (isMissing(cell))
29
+ continue;
30
+ const conforms = to === 'number' ? parseNumeric(cell) !== undefined : to === 'date' ? parseDate(cell) !== undefined : parseBoolean(cell) !== undefined;
31
+ if (!conforms)
32
+ count += 1;
33
+ }
34
+ return count;
35
+ }
36
+ /**
37
+ * Compute the pre-delivery contract summary from a cleaning run: dedupe
38
+ * before/after row counts, the dedupe key's uniqueness regression, and the
39
+ * non-null/type regressions over the columns the rules targeted.
40
+ * @param result - the cleaning outcome (input/output rows plus cleaned rows).
41
+ * @param rules - the ordered rules that produced `result`.
42
+ * @param options - optional abort signal.
43
+ * @returns the contract summary.
44
+ */
45
+ export function computeCleanContract(result, rules, options = {}) {
46
+ const dedupeRule = rules.find((rule) => rule.rule === 'dedupe');
47
+ const dedupeColumns = dedupeRule === undefined ? null : [...(dedupeRule.columns ?? result.columns)];
48
+ const keyColumns = dedupeColumns ?? result.columns;
49
+ const seen = new Set();
50
+ let remainingDuplicateRows = 0;
51
+ for (const [index, row] of result.rows.entries()) {
52
+ if (index % 1024 === 0)
53
+ throwIfAborted(options.signal);
54
+ const key = JSON.stringify(keyColumns.map((column) => row[column] ?? null));
55
+ if (seen.has(key)) {
56
+ remainingDuplicateRows += 1;
57
+ }
58
+ else {
59
+ seen.add(key);
60
+ }
61
+ }
62
+ const remainingMissing = rules
63
+ .filter((rule) => rule.rule === 'fill-missing')
64
+ .map((rule) => {
65
+ const fill = rule;
66
+ return { column: fill.column, count: countMissing(result.rows, fill.column, options.signal) };
67
+ });
68
+ const typeConformance = rules
69
+ .filter((rule) => rule.rule === 'coerce-type')
70
+ .map((rule) => {
71
+ const coerce = rule;
72
+ return { column: coerce.column, to: coerce.to, invalidCount: countNonConforming(result.rows, coerce.column, coerce.to, options.signal) };
73
+ });
74
+ const columnDecisions = traceColumnDecisions(result, rules);
75
+ return {
76
+ inputRows: result.inputRows,
77
+ outputRows: result.outputRows,
78
+ removedRows: result.inputRows - result.outputRows,
79
+ dedupeColumns,
80
+ uniqueKeys: remainingDuplicateRows === 0,
81
+ remainingDuplicateRows,
82
+ remainingMissing,
83
+ typeConformance,
84
+ columnDecisions,
85
+ };
86
+ }
87
+ /** The columns a single rule targets (single-column rules target one column; trim/dedupe may target many). */
88
+ function ruleColumns(rule, allColumns) {
89
+ switch (rule.rule) {
90
+ case 'fill-missing':
91
+ case 'coerce-type':
92
+ case 'normalize-unit':
93
+ case 'map-values':
94
+ return [rule.column];
95
+ case 'trim':
96
+ case 'dedupe':
97
+ return [...(rule.columns ?? allColumns)];
98
+ default:
99
+ return [];
100
+ }
101
+ }
102
+ /** Build the per-column decision trace: strategies + affected rows, in dataset column order. */
103
+ function traceColumnDecisions(result, rules) {
104
+ const byColumn = new Map();
105
+ for (const [index, rule] of rules.entries()) {
106
+ const log = result.logs[index];
107
+ if (log === undefined)
108
+ continue;
109
+ for (const column of ruleColumns(rule, result.columns)) {
110
+ const list = byColumn.get(column) ?? [];
111
+ list.push({ strategy: rule.rule, affectedRows: log.affectedRows });
112
+ byColumn.set(column, list);
113
+ }
114
+ }
115
+ return result.columns
116
+ .filter((column) => byColumn.has(column))
117
+ .map((column) => ({ column, decisions: byColumn.get(column) }));
118
+ }
119
+ /**
120
+ * Build the clean before/after profile diff: two full {@link ProfileReport}
121
+ * snapshots (input and cleaned output) under one envelope. Reuses the profile
122
+ * engine so the scorecard and duplicate detection stay consistent across the
123
+ * whole plugin.
124
+ * @param input - the input dataset.
125
+ * @param output - the cleaned dataset.
126
+ * @param options - dataset label, injected timestamp, optional weights, optional abort signal.
127
+ * @returns the diff report.
128
+ */
129
+ export function computeCleanProfileDiff(input, output, options) {
130
+ const before = profileTable(input, {
131
+ dataset: options.dataset,
132
+ generatedAt: options.generatedAt,
133
+ scorecardWeights: options.scorecardWeights,
134
+ signal: options.signal,
135
+ });
136
+ const after = profileTable(output, {
137
+ dataset: options.dataset,
138
+ generatedAt: options.generatedAt,
139
+ scorecardWeights: options.scorecardWeights,
140
+ signal: options.signal,
141
+ });
142
+ return { dataset: options.dataset, before, after, generatedAt: options.generatedAt };
143
+ }
144
+ //# sourceMappingURL=contract.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"contract.js","sourceRoot":"","sources":["../../src/contract.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,SAAS,EAAE,YAAY,EAAE,cAAc,EAAwB,MAAM,cAAc,CAAA;AAErH,OAAO,EAAE,YAAY,EAAsB,MAAM,cAAc,CAAA;AAsC/D,+DAA+D;AAC/D,SAAS,YAAY,CAAC,IAAoB,EAAE,MAAc,EAAE,MAAoB;IAC9E,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,KAAK,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;QAC1C,IAAI,KAAK,GAAG,IAAI,KAAK,CAAC;YAAE,cAAc,CAAC,MAAM,CAAC,CAAA;QAC9C,IAAI,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAAE,KAAK,IAAI,CAAC,CAAA;IACxC,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,wFAAwF;AACxF,SAAS,kBAAkB,CAAC,IAAoB,EAAE,MAAc,EAAE,EAAiC,EAAE,MAAoB;IACvH,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,KAAK,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;QAC1C,IAAI,KAAK,GAAG,IAAI,KAAK,CAAC;YAAE,cAAc,CAAC,MAAM,CAAC,CAAA;QAC9C,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,CAAA;QACxB,IAAI,SAAS,CAAC,IAAI,CAAC;YAAE,SAAQ;QAC7B,MAAM,QAAQ,GAAG,EAAE,KAAK,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,SAAS,CAAA;QACtJ,IAAI,CAAC,QAAQ;YAAE,KAAK,IAAI,CAAC,CAAA;IAC3B,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,oBAAoB,CAClC,MAAmB,EACnB,KAA2B,EAC3B,UAAgD,EAAE;IAElD,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAuD,CAAA;IACrH,MAAM,aAAa,GAAG,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAA;IACnG,MAAM,UAAU,GAAG,aAAa,IAAI,MAAM,CAAC,OAAO,CAAA;IAClD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAC9B,IAAI,sBAAsB,GAAG,CAAC,CAAA;IAC9B,KAAK,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;QACjD,IAAI,KAAK,GAAG,IAAI,KAAK,CAAC;YAAE,cAAc,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;QACtD,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,CAAC,CAAA;QAC3E,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAClB,sBAAsB,IAAI,CAAC,CAAA;QAC7B,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QACf,CAAC;IACH,CAAC;IACD,MAAM,gBAAgB,GAAG,KAAK;SAC3B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,cAAc,CAAC;SAC9C,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACZ,MAAM,IAAI,GAAG,IAAoD,CAAA;QACjE,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,EAAE,CAAA;IAC/F,CAAC,CAAC,CAAA;IACJ,MAAM,eAAe,GAAG,KAAK;SAC1B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,aAAa,CAAC;SAC7C,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACZ,MAAM,MAAM,GAAG,IAAmD,CAAA;QAClE,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,EAAE,EAAE,MAAM,CAAC,EAAE,EAAE,YAAY,EAAE,kBAAkB,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,EAAE,OAAO,CAAC,MAAM,CAAC,EAAE,CAAA;IAC1I,CAAC,CAAC,CAAA;IACJ,MAAM,eAAe,GAAG,oBAAoB,CAAC,MAAM,EAAE,KAAK,CAAC,CAAA;IAC3D,OAAO;QACL,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,WAAW,EAAE,MAAM,CAAC,SAAS,GAAG,MAAM,CAAC,UAAU;QACjD,aAAa;QACb,UAAU,EAAE,sBAAsB,KAAK,CAAC;QACxC,sBAAsB;QACtB,gBAAgB;QAChB,eAAe;QACf,eAAe;KAChB,CAAA;AACH,CAAC;AAED,8GAA8G;AAC9G,SAAS,WAAW,CAAC,IAAe,EAAE,UAA6B;IACjE,QAAQ,IAAI,CAAC,IAAI,EAAE,CAAC;QAClB,KAAK,cAAc,CAAC;QACpB,KAAK,aAAa,CAAC;QACnB,KAAK,gBAAgB,CAAC;QACtB,KAAK,YAAY;YACf,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QACtB,KAAK,MAAM,CAAC;QACZ,KAAK,QAAQ;YACX,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,UAAU,CAAC,CAAC,CAAA;QAC1C;YACE,OAAO,EAAE,CAAA;IACb,CAAC;AACH,CAAC;AAED,gGAAgG;AAChG,SAAS,oBAAoB,CAC3B,MAAmB,EACnB,KAA2B;IAE3B,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA6D,CAAA;IACrF,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;QAC5C,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAC9B,IAAI,GAAG,KAAK,SAAS;YAAE,SAAQ;QAC/B,KAAK,MAAM,MAAM,IAAI,WAAW,CAAC,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;YACvD,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CAAA;YACvC,IAAI,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,IAAI,EAAE,YAAY,EAAE,GAAG,CAAC,YAAY,EAAE,CAAC,CAAA;YAClE,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;QAC5B,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC,OAAO;SAClB,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;SACxC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAsD,EAAE,CAAC,CAAC,CAAA;AACxH,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,uBAAuB,CACrC,KAAY,EACZ,MAAa,EACb,OAAoK;IAEpK,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,EAAE;QACjC,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;QAC1C,MAAM,EAAE,OAAO,CAAC,MAAM;KACvB,CAAC,CAAA;IACF,MAAM,KAAK,GAAG,YAAY,CAAC,MAAM,EAAE;QACjC,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;QAC1C,MAAM,EAAE,OAAO,CAAC,MAAM;KACvB,CAAC,CAAA;IACF,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,CAAC,WAAW,EAAE,CAAA;AACtF,CAAC"}
@@ -11,12 +11,21 @@ import type { ResolvedConfig } from './config.js';
11
11
  export type Cell = JsonValue;
12
12
  /** One dataset row keyed by column name. */
13
13
  export type Row = Record<string, Cell>;
14
+ /** Detected file encoding metadata (BOM presence + UTF-8 validity). */
15
+ export interface EncodingInfo {
16
+ /** `'utf-8'` when a UTF-8 byte-order mark was present, else `null`. */
17
+ readonly bom: 'utf-8' | null;
18
+ /** Whether the bytes decode as valid UTF-8 (no replacement characters forced). */
19
+ readonly validUtf8: boolean;
20
+ }
14
21
  /** A tabular dataset: ordered columns plus rows. */
15
22
  export interface Table {
16
23
  /** Column names in file order. */
17
24
  readonly columns: string[];
18
25
  /** Rows, each carrying every declared column (missing cells are `null`). */
19
26
  readonly rows: Row[];
27
+ /** Detected file encoding; present only for tables loaded from a file. */
28
+ readonly encoding?: EncodingInfo;
20
29
  }
21
30
  /** The root form a document load returns (citation checking walks this). */
22
31
  export type DocumentRoot = {
@@ -55,13 +64,25 @@ export declare function pathFingerprint(datasetPath: string): string;
55
64
  /** Assert the abort signal has not fired; throws the signal reason otherwise. */
56
65
  export declare function throwIfAborted(signal: AbortSignal | undefined): void;
57
66
  /**
58
- * Read a dataset file under the size cap.
67
+ * Detect a UTF-8 byte-order mark and validate the byte sequence. Invalid
68
+ * UTF-8 is a data-quality finding the profile reports (`validUtf8: false`)
69
+ * rather than a structural error that blocks the read — the decoded text
70
+ * keeps U+FFFD replacement characters so the profile can still run.
71
+ * @param buffer - raw file bytes.
72
+ * @returns the encoding metadata.
73
+ */
74
+ export declare function detectEncoding(buffer: Uint8Array): EncodingInfo;
75
+ /**
76
+ * Read a dataset file under the size cap, detecting its encoding.
59
77
  * @param absolutePath - normalized absolute path (from {@link resolveWorkspacePath}).
60
78
  * @param config - resolved config (size cap).
61
79
  * @param signal - optional abort signal honored around the read.
62
- * @returns the UTF-8 text.
80
+ * @returns the decoded UTF-8 text plus its encoding metadata.
63
81
  */
64
- export declare function readDatasetText(absolutePath: string, config: ResolvedConfig, signal?: AbortSignal): Promise<string>;
82
+ export declare function readDatasetFile(absolutePath: string, config: ResolvedConfig, signal?: AbortSignal): Promise<{
83
+ text: string;
84
+ encoding: EncodingInfo;
85
+ }>;
65
86
  /**
66
87
  * Parse CSV/TSV text (RFC-4180-style: quoted fields, `""` escapes, CRLF/LF).
67
88
  * The first record is the header; short records pad with `null`, long records
@@ -119,6 +140,17 @@ export declare function sampleRows(rows: Row[], count: number): Row[];
119
140
  export declare function isMissing(cell: Cell | undefined): boolean;
120
141
  /** Strict full-string numeric parse (no thousands separators, no unit suffixes). */
121
142
  export declare function parseNumeric(cell: Cell | undefined): number | undefined;
143
+ /** Deterministic date-format labels in {@link DATE_PATTERNS} order. */
144
+ declare const DATE_FORMATS: readonly ["iso-date", "slash-date", "datetime"];
145
+ /** The format label of a recognized date cell (source pattern, not the epoch). */
146
+ export type DateFormat = (typeof DATE_FORMATS)[number];
147
+ /** A recognized date cell: its parsed epoch plus the source format label. */
148
+ export interface DateParse {
149
+ /** Epoch milliseconds (UTC). */
150
+ readonly epoch: number;
151
+ /** Which {@link DATE_PATTERNS} entry matched. */
152
+ readonly format: DateFormat;
153
+ }
122
154
  /**
123
155
  * Deterministic date parse to epoch milliseconds. Accepts `YYYY-MM-DD`,
124
156
  * `YYYY/MM/DD`, and ISO-like datetimes (date-only forms read as UTC midnight).
@@ -128,6 +160,21 @@ export declare function parseNumeric(cell: Cell | undefined): number | undefined
128
160
  * @returns epoch milliseconds, or `undefined`.
129
161
  */
130
162
  export declare function parseDate(cell: Cell | undefined): number | undefined;
163
+ /**
164
+ * Parse one date cell to its epoch plus source format label.
165
+ * @param cell - the cell to parse (`undefined` when the column is absent).
166
+ * @returns the parsed date, or `undefined` when the cell is not a recognized date.
167
+ */
168
+ export declare function parseDateCell(cell: Cell | undefined): DateParse | undefined;
169
+ /**
170
+ * The format label of a recognized date cell (`iso-date` / `slash-date` /
171
+ * `datetime`); `undefined` when the cell is not a recognized date. Used to
172
+ * measure a date column's format consistency.
173
+ * @param cell - the cell to inspect (`undefined` when the column is absent).
174
+ * @returns the source format label, or `undefined`.
175
+ */
176
+ export declare function dateFormatOf(cell: Cell | undefined): DateFormat | undefined;
131
177
  /** Boolean parse: true/false/yes/no/1/0, case-insensitive. */
132
178
  export declare function parseBoolean(cell: Cell | undefined): boolean | undefined;
179
+ export {};
133
180
  //# sourceMappingURL=dataset.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"dataset.d.ts","sourceRoot":"","sources":["../../src/dataset.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAKH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAA;AACzD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAEjD,gHAAgH;AAChH,MAAM,MAAM,IAAI,GAAG,SAAS,CAAA;AAE5B,4CAA4C;AAC5C,MAAM,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;AAEtC,oDAAoD;AACpD,MAAM,WAAW,KAAK;IACpB,kCAAkC;IAClC,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,CAAA;IAC1B,4EAA4E;IAC5E,QAAQ,CAAC,IAAI,EAAE,GAAG,EAAE,CAAA;CACrB;AAED,4EAA4E;AAC5E,MAAM,MAAM,YAAY,GACpB;IAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,GAAG,EAAE,CAAA;CAAE,GAC5E;IAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAA;CAAE,CAAA;AAEtD,4FAA4F;AAC5F,qBAAa,YAAa,SAAQ,KAAK;IAMnC,QAAQ,CAAC,IAAI,EACT,aAAa,GACb,kBAAkB,GAClB,WAAW,GACX,WAAW,GACX,eAAe,GACf,WAAW,GACX,aAAa;IAZnB;;;OAGG;gBAEQ,IAAI,EACT,aAAa,GACb,kBAAkB,GAClB,WAAW,GACX,WAAW,GACX,eAAe,GACf,WAAW,GACX,aAAa,EACjB,OAAO,EAAE,MAAM;CAKlB;AAED;;;;;;;;GAQG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,cAAc,GAAG,MAAM,CAoBpG;AAED;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM,CAE3D;AAED,iFAAiF;AACjF,wBAAgB,cAAc,CAAC,MAAM,EAAE,WAAW,GAAG,SAAS,GAAG,IAAI,CAIpE;AAED;;;;;;GAMG;AACH,wBAAsB,eAAe,CAAC,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAyBzH;AAED;;;;;;;;;GASG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,KAAK,CA+FnH;AAED;;;;;;;;;GASG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,KAAK,CA8BnH;AAoCD;;;;;;GAMG;AACH,wBAAsB,SAAS,CAAC,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,CAOlH;AAED;;;;;;;;GAQG;AACH,wBAAsB,YAAY,CAAC,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC,CAY5H;AAED;;;;;;GAMG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,MAAM,GAAG,GAAG,EAAE,CAW5D;AAED;;;;;GAKG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,IAAI,GAAG,SAAS,GAAG,OAAO,CAIzD;AAED,oFAAoF;AACpF,wBAAgB,YAAY,CAAC,IAAI,EAAE,IAAI,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CAQvE;AAQD;;;;;;;GAOG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,IAAI,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CAmBpE;AAED,8DAA8D;AAC9D,wBAAgB,YAAY,CAAC,IAAI,EAAE,IAAI,GAAG,SAAS,GAAG,OAAO,GAAG,SAAS,CAOxE"}
1
+ {"version":3,"file":"dataset.d.ts","sourceRoot":"","sources":["../../src/dataset.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAMH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAA;AACzD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAEjD,gHAAgH;AAChH,MAAM,MAAM,IAAI,GAAG,SAAS,CAAA;AAE5B,4CAA4C;AAC5C,MAAM,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;AAEtC,uEAAuE;AACvE,MAAM,WAAW,YAAY;IAC3B,uEAAuE;IACvE,QAAQ,CAAC,GAAG,EAAE,OAAO,GAAG,IAAI,CAAA;IAC5B,kFAAkF;IAClF,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAA;CAC5B;AAED,oDAAoD;AACpD,MAAM,WAAW,KAAK;IACpB,kCAAkC;IAClC,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,CAAA;IAC1B,4EAA4E;IAC5E,QAAQ,CAAC,IAAI,EAAE,GAAG,EAAE,CAAA;IACpB,0EAA0E;IAC1E,QAAQ,CAAC,QAAQ,CAAC,EAAE,YAAY,CAAA;CACjC;AAED,4EAA4E;AAC5E,MAAM,MAAM,YAAY,GACpB;IAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,GAAG,EAAE,CAAA;CAAE,GAC5E;IAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAA;CAAE,CAAA;AAEtD,4FAA4F;AAC5F,qBAAa,YAAa,SAAQ,KAAK;IAMnC,QAAQ,CAAC,IAAI,EACT,aAAa,GACb,kBAAkB,GAClB,WAAW,GACX,WAAW,GACX,eAAe,GACf,WAAW,GACX,aAAa;IAZnB;;;OAGG;gBAEQ,IAAI,EACT,aAAa,GACb,kBAAkB,GAClB,WAAW,GACX,WAAW,GACX,eAAe,GACf,WAAW,GACX,aAAa,EACjB,OAAO,EAAE,MAAM;CAKlB;AAED;;;;;;;;GAQG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,cAAc,GAAG,MAAM,CAoBpG;AAED;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM,CAE3D;AAED,iFAAiF;AACjF,wBAAgB,cAAc,CAAC,MAAM,EAAE,WAAW,GAAG,SAAS,GAAG,IAAI,CAIpE;AAED;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,UAAU,GAAG,YAAY,CAU/D;AAQD;;;;;;GAMG;AACH,wBAAsB,eAAe,CAAC,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,YAAY,CAAA;CAAE,CAAC,CA2B3J;AAED;;;;;;;;;GASG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,KAAK,CA+FnH;AAED;;;;;;;;;GASG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,KAAK,CA8BnH;AAoCD;;;;;;GAMG;AACH,wBAAsB,SAAS,CAAC,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,CAOlH;AAED;;;;;;;;GAQG;AACH,wBAAsB,YAAY,CAAC,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC,CAY5H;AAED;;;;;;GAMG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,MAAM,GAAG,GAAG,EAAE,CAW5D;AAED;;;;;GAKG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,IAAI,GAAG,SAAS,GAAG,OAAO,CAIzD;AAED,oFAAoF;AACpF,wBAAgB,YAAY,CAAC,IAAI,EAAE,IAAI,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CAQvE;AAQD,uEAAuE;AACvE,QAAA,MAAM,YAAY,iDAAkD,CAAA;AAEpE,kFAAkF;AAClF,MAAM,MAAM,UAAU,GAAG,CAAC,OAAO,YAAY,CAAC,CAAC,MAAM,CAAC,CAAA;AAEtD,6EAA6E;AAC7E,MAAM,WAAW,SAAS;IACxB,gCAAgC;IAChC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,iDAAiD;IACjD,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAA;CAC5B;AA0BD;;;;;;;GAOG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,IAAI,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CAEpE;AAED;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,IAAI,GAAG,SAAS,GAAG,SAAS,GAAG,SAAS,CAE3E;AAED;;;;;;GAMG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,IAAI,GAAG,SAAS,GAAG,UAAU,GAAG,SAAS,CAE3E;AAED,8DAA8D;AAC9D,wBAAgB,YAAY,CAAC,IAAI,EAAE,IAAI,GAAG,SAAS,GAAG,OAAO,GAAG,SAAS,CAOxE"}
@@ -8,6 +8,7 @@
8
8
  import { open, stat } from 'node:fs/promises';
9
9
  import path from 'node:path';
10
10
  import { createHash } from 'node:crypto';
11
+ import { TextDecoder } from 'node:util';
11
12
  /** Raised for every user-facing dataset failure; `code` is stable for tests and tooling. */
12
13
  export class DatasetError extends Error {
13
14
  code;
@@ -60,13 +61,38 @@ export function throwIfAborted(signal) {
60
61
  }
61
62
  }
62
63
  /**
63
- * Read a dataset file under the size cap.
64
+ * Detect a UTF-8 byte-order mark and validate the byte sequence. Invalid
65
+ * UTF-8 is a data-quality finding the profile reports (`validUtf8: false`)
66
+ * rather than a structural error that blocks the read — the decoded text
67
+ * keeps U+FFFD replacement characters so the profile can still run.
68
+ * @param buffer - raw file bytes.
69
+ * @returns the encoding metadata.
70
+ */
71
+ export function detectEncoding(buffer) {
72
+ const bom = buffer.length >= 3 && buffer[0] === 0xef && buffer[1] === 0xbb && buffer[2] === 0xbf ? 'utf-8' : null;
73
+ const body = bom === 'utf-8' ? buffer.subarray(3) : buffer;
74
+ let validUtf8 = true;
75
+ try {
76
+ new TextDecoder('utf-8', { fatal: true }).decode(body);
77
+ }
78
+ catch {
79
+ validUtf8 = false;
80
+ }
81
+ return { bom, validUtf8 };
82
+ }
83
+ /** Decode file bytes, stripping a UTF-8 BOM and preserving replacement characters for invalid bytes. */
84
+ function decodeUtf8Text(buffer, encoding) {
85
+ const body = encoding.bom === 'utf-8' ? buffer.subarray(3) : buffer;
86
+ return new TextDecoder('utf-8').decode(body);
87
+ }
88
+ /**
89
+ * Read a dataset file under the size cap, detecting its encoding.
64
90
  * @param absolutePath - normalized absolute path (from {@link resolveWorkspacePath}).
65
91
  * @param config - resolved config (size cap).
66
92
  * @param signal - optional abort signal honored around the read.
67
- * @returns the UTF-8 text.
93
+ * @returns the decoded UTF-8 text plus its encoding metadata.
68
94
  */
69
- export async function readDatasetText(absolutePath, config, signal) {
95
+ export async function readDatasetFile(absolutePath, config, signal) {
70
96
  throwIfAborted(signal);
71
97
  let info;
72
98
  try {
@@ -85,7 +111,9 @@ export async function readDatasetText(absolutePath, config, signal) {
85
111
  const handle = await open(absolutePath, 'r');
86
112
  try {
87
113
  throwIfAborted(signal);
88
- return await handle.readFile('utf8');
114
+ const buffer = await handle.readFile();
115
+ const encoding = detectEncoding(buffer);
116
+ return { text: decodeUtf8Text(buffer, encoding), encoding };
89
117
  }
90
118
  finally {
91
119
  await handle.close();
@@ -273,14 +301,14 @@ function parseJsonLines(text) {
273
301
  * @returns the parsed table.
274
302
  */
275
303
  export async function loadTable(absolutePath, config, signal) {
276
- const text = await readDatasetText(absolutePath, config, signal);
304
+ const { text, encoding } = await readDatasetFile(absolutePath, config, signal);
277
305
  const ext = path.extname(absolutePath).toLowerCase();
278
306
  throwIfAborted(signal);
279
307
  if (ext === '.csv')
280
- return parseDelimited(text, ',', config, signal);
308
+ return { ...parseDelimited(text, ',', config, signal), encoding };
281
309
  if (ext === '.tsv')
282
- return parseDelimited(text, '\t', config, signal);
283
- return parseJsonTable(text, ext, config, signal);
310
+ return { ...parseDelimited(text, '\t', config, signal), encoding };
311
+ return { ...parseJsonTable(text, ext, config, signal), encoding };
284
312
  }
285
313
  /**
286
314
  * Load a dataset as a citation-checkable document root: CSV/TSV become
@@ -294,11 +322,11 @@ export async function loadTable(absolutePath, config, signal) {
294
322
  export async function loadDocument(absolutePath, config, signal) {
295
323
  const ext = path.extname(absolutePath).toLowerCase();
296
324
  if (ext === '.json') {
297
- const text = await readDatasetText(absolutePath, config, signal);
325
+ const { text } = await readDatasetFile(absolutePath, config, signal);
298
326
  return { kind: 'json', value: parseJsonDocument(text) };
299
327
  }
300
328
  if (ext === '.jsonl') {
301
- const text = await readDatasetText(absolutePath, config, signal);
329
+ const { text } = await readDatasetFile(absolutePath, config, signal);
302
330
  return { kind: 'json', value: parseJsonLines(text) };
303
331
  }
304
332
  const table = await loadTable(absolutePath, config, signal);
@@ -356,19 +384,14 @@ const DATE_PATTERNS = [
356
384
  /^(\d{4})\/(\d{1,2})\/(\d{1,2})$/u,
357
385
  /^(\d{4})-(\d{1,2})-(\d{1,2})[ T](\d{1,2}):(\d{2})(?::(\d{2}))?(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?$/u,
358
386
  ];
359
- /**
360
- * Deterministic date parse to epoch milliseconds. Accepts `YYYY-MM-DD`,
361
- * `YYYY/MM/DD`, and ISO-like datetimes (date-only forms read as UTC midnight).
362
- * Calendar-invalid dates (e.g. 2025-13-40) reject. Returns `undefined` when
363
- * the cell is not a recognized date.
364
- * @param cell - the cell to parse (`undefined` when the column is absent).
365
- * @returns epoch milliseconds, or `undefined`.
366
- */
367
- export function parseDate(cell) {
387
+ /** Deterministic date-format labels in {@link DATE_PATTERNS} order. */
388
+ const DATE_FORMATS = ['iso-date', 'slash-date', 'datetime'];
389
+ /** Parse one date cell to its epoch plus format; `undefined` when unrecognized. */
390
+ function parseDateCellInternal(cell) {
368
391
  if (typeof cell !== 'string')
369
392
  return undefined;
370
393
  const text = cell.trim();
371
- for (const pattern of DATE_PATTERNS) {
394
+ for (const [index, pattern] of DATE_PATTERNS.entries()) {
372
395
  const match = pattern.exec(text);
373
396
  if (match === null)
374
397
  continue;
@@ -384,10 +407,42 @@ export function parseDate(cell) {
384
407
  const check = new Date(epoch);
385
408
  if (check.getUTCMonth() !== month - 1 || check.getUTCDate() !== day)
386
409
  return undefined;
387
- return epoch;
410
+ const format = DATE_FORMATS[index];
411
+ if (format === undefined)
412
+ return undefined;
413
+ return { epoch, format };
388
414
  }
389
415
  return undefined;
390
416
  }
417
+ /**
418
+ * Deterministic date parse to epoch milliseconds. Accepts `YYYY-MM-DD`,
419
+ * `YYYY/MM/DD`, and ISO-like datetimes (date-only forms read as UTC midnight).
420
+ * Calendar-invalid dates (e.g. 2025-13-40) reject. Returns `undefined` when
421
+ * the cell is not a recognized date.
422
+ * @param cell - the cell to parse (`undefined` when the column is absent).
423
+ * @returns epoch milliseconds, or `undefined`.
424
+ */
425
+ export function parseDate(cell) {
426
+ return parseDateCellInternal(cell)?.epoch;
427
+ }
428
+ /**
429
+ * Parse one date cell to its epoch plus source format label.
430
+ * @param cell - the cell to parse (`undefined` when the column is absent).
431
+ * @returns the parsed date, or `undefined` when the cell is not a recognized date.
432
+ */
433
+ export function parseDateCell(cell) {
434
+ return parseDateCellInternal(cell);
435
+ }
436
+ /**
437
+ * The format label of a recognized date cell (`iso-date` / `slash-date` /
438
+ * `datetime`); `undefined` when the cell is not a recognized date. Used to
439
+ * measure a date column's format consistency.
440
+ * @param cell - the cell to inspect (`undefined` when the column is absent).
441
+ * @returns the source format label, or `undefined`.
442
+ */
443
+ export function dateFormatOf(cell) {
444
+ return parseDateCellInternal(cell)?.format;
445
+ }
391
446
  /** Boolean parse: true/false/yes/no/1/0, case-insensitive. */
392
447
  export function parseBoolean(cell) {
393
448
  if (typeof cell === 'boolean')