dsh-data-quality 0.1.3 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/CHANGELOG.md +33 -0
  2. package/README.es.md +7 -1
  3. package/README.hi.md +7 -1
  4. package/README.md +22 -9
  5. package/README.pt.md +7 -1
  6. package/README.zh.md +7 -1
  7. package/cordis.patch.yml +11 -0
  8. package/lib/index.js +1609 -145
  9. package/lib/types/config.d.ts +12 -0
  10. package/lib/types/config.d.ts.map +1 -1
  11. package/lib/types/config.js +35 -0
  12. package/lib/types/config.js.map +1 -1
  13. package/lib/types/contract.d.ts +83 -0
  14. package/lib/types/contract.d.ts.map +1 -0
  15. package/lib/types/contract.js +144 -0
  16. package/lib/types/contract.js.map +1 -0
  17. package/lib/types/dataset.d.ts +50 -3
  18. package/lib/types/dataset.d.ts.map +1 -1
  19. package/lib/types/dataset.js +76 -21
  20. package/lib/types/dataset.js.map +1 -1
  21. package/lib/types/events.d.ts +2 -0
  22. package/lib/types/events.d.ts.map +1 -1
  23. package/lib/types/events.js.map +1 -1
  24. package/lib/types/index.d.ts +13 -8
  25. package/lib/types/index.d.ts.map +1 -1
  26. package/lib/types/index.js +17 -6
  27. package/lib/types/index.js.map +1 -1
  28. package/lib/types/presets.d.ts +28 -0
  29. package/lib/types/presets.d.ts.map +1 -0
  30. package/lib/types/presets.js +134 -0
  31. package/lib/types/presets.js.map +1 -0
  32. package/lib/types/profile.d.ts +40 -1
  33. package/lib/types/profile.d.ts.map +1 -1
  34. package/lib/types/profile.js +66 -10
  35. package/lib/types/profile.js.map +1 -1
  36. package/lib/types/provider-local.d.ts +7 -1
  37. package/lib/types/provider-local.d.ts.map +1 -1
  38. package/lib/types/provider-local.js +72 -2
  39. package/lib/types/provider-local.js.map +1 -1
  40. package/lib/types/report-html.d.ts +41 -0
  41. package/lib/types/report-html.d.ts.map +1 -0
  42. package/lib/types/report-html.js +192 -0
  43. package/lib/types/report-html.js.map +1 -0
  44. package/lib/types/scorecard.d.ts +50 -0
  45. package/lib/types/scorecard.d.ts.map +1 -0
  46. package/lib/types/scorecard.js +186 -0
  47. package/lib/types/scorecard.js.map +1 -0
  48. package/lib/types/service.d.ts +32 -2
  49. package/lib/types/service.d.ts.map +1 -1
  50. package/lib/types/service.js.map +1 -1
  51. package/lib/types/store.d.ts +21 -1
  52. package/lib/types/store.d.ts.map +1 -1
  53. package/lib/types/store.js +13 -1
  54. package/lib/types/store.js.map +1 -1
  55. package/lib/types/tools/clean.d.ts.map +1 -1
  56. package/lib/types/tools/clean.js +76 -1
  57. package/lib/types/tools/clean.js.map +1 -1
  58. package/lib/types/tools/profile-report-schema.d.ts +319 -0
  59. package/lib/types/tools/profile-report-schema.d.ts.map +1 -0
  60. package/lib/types/tools/profile-report-schema.js +96 -0
  61. package/lib/types/tools/profile-report-schema.js.map +1 -0
  62. package/lib/types/tools/profile.d.ts.map +1 -1
  63. package/lib/types/tools/profile.js +6 -53
  64. package/lib/types/tools/profile.js.map +1 -1
  65. package/lib/types/tools/report.d.ts +14 -0
  66. package/lib/types/tools/report.d.ts.map +1 -0
  67. package/lib/types/tools/report.js +112 -0
  68. package/lib/types/tools/report.js.map +1 -0
  69. package/lib/types/tools/shared.d.ts.map +1 -1
  70. package/lib/types/tools/shared.js +22 -2
  71. package/lib/types/tools/shared.js.map +1 -1
  72. package/lib/types/tools/verify.d.ts.map +1 -1
  73. package/lib/types/tools/verify.js +36 -1
  74. package/lib/types/tools/verify.js.map +1 -1
  75. package/lib/types/verify.d.ts +49 -3
  76. package/lib/types/verify.d.ts.map +1 -1
  77. package/lib/types/verify.js +111 -4
  78. package/lib/types/verify.js.map +1 -1
  79. package/lib/types/version.d.ts +8 -1
  80. package/lib/types/version.d.ts.map +1 -1
  81. package/lib/types/version.js +8 -1
  82. package/lib/types/version.js.map +1 -1
  83. package/package.json +1 -1
  84. package/src/config.ts +52 -0
  85. package/src/contract.ts +190 -0
  86. package/src/dataset.ts +101 -21
  87. package/src/events.ts +2 -0
  88. package/src/index.ts +27 -8
  89. package/src/presets.ts +146 -0
  90. package/src/profile.ts +105 -11
  91. package/src/provider-local.ts +77 -3
  92. package/src/report-html.ts +208 -0
  93. package/src/scorecard.ts +244 -0
  94. package/src/service.ts +34 -2
  95. package/src/store.ts +28 -2
  96. package/src/tools/clean.ts +76 -1
  97. package/src/tools/profile-report-schema.ts +98 -0
  98. package/src/tools/profile.ts +8 -56
  99. package/src/tools/report.ts +137 -0
  100. package/src/tools/shared.ts +21 -2
  101. package/src/tools/verify.ts +40 -2
  102. package/src/verify.ts +151 -5
  103. package/src/version.ts +9 -1
package/src/config.ts CHANGED
@@ -7,6 +7,29 @@
7
7
 
8
8
  import z from '@deepseek-ai/schemastery'
9
9
 
10
+ /** Per-dimension scorecard weights (all optional; defaults fill to `1` = equal). */
11
+ export interface ScorecardWeights {
12
+ completeness?: number
13
+ uniqueness?: number
14
+ validity?: number
15
+ consistency?: number
16
+ timeliness?: number
17
+ accuracy?: number
18
+ }
19
+
20
+ /** The six scorecard dimension ids, in report order (kept in sync with `scorecard.ts`). */
21
+ const SCORECARD_DIMENSIONS = ['completeness', 'uniqueness', 'validity', 'consistency', 'timeliness', 'accuracy'] as const
22
+
23
+ /** Equal-weight defaults for {@link ScorecardWeights}. */
24
+ const EQUAL_WEIGHTS: Required<ScorecardWeights> = {
25
+ completeness: 1,
26
+ uniqueness: 1,
27
+ validity: 1,
28
+ consistency: 1,
29
+ timeliness: 1,
30
+ accuracy: 1,
31
+ }
32
+
10
33
  /** Raw plugin config — every field optional; {@link resolveConfig} supplies the defaults. */
11
34
  export interface Config {
12
35
  /** Master switch; `false` mounts nothing. */
@@ -25,6 +48,8 @@ export interface Config {
25
48
  workspaceRoot?: string
26
49
  /** Persist reports to the `data_quality` storage domain. */
27
50
  storeReports?: boolean
51
+ /** Per-dimension weights for the scorecard's weighted overall total (non-negative). */
52
+ scorecardWeights?: ScorecardWeights
28
53
  }
29
54
 
30
55
  /** Fully resolved config handed to the runtime. */
@@ -37,6 +62,7 @@ export interface ResolvedConfig {
37
62
  readonly allowedExtensions: readonly string[]
38
63
  readonly workspaceRoot: string
39
64
  readonly storeReports: boolean
65
+ readonly scorecardWeights: Required<ScorecardWeights>
40
66
  }
41
67
 
42
68
  /** Schemastery schema: the loader validates and fills defaults before `apply`. */
@@ -49,6 +75,16 @@ export const Config: z<Config> = z.object({
49
75
  allowedExtensions: z.array(z.string()).default(['.csv', '.tsv', '.json', '.jsonl']),
50
76
  workspaceRoot: z.string().default(''),
51
77
  storeReports: z.boolean().default(true),
78
+ scorecardWeights: z
79
+ .object({
80
+ completeness: z.number().default(1),
81
+ uniqueness: z.number().default(1),
82
+ validity: z.number().default(1),
83
+ consistency: z.number().default(1),
84
+ timeliness: z.number().default(1),
85
+ accuracy: z.number().default(1),
86
+ })
87
+ .default(EQUAL_WEIGHTS),
52
88
  })
53
89
 
54
90
  /** Throw unless `value` is a positive safe integer. */
@@ -91,6 +127,8 @@ export function resolveConfig(config: Config = {}): ResolvedConfig {
91
127
  throw new TypeError('allowedExtensions must not be empty')
92
128
  }
93
129
 
130
+ const scorecardWeights = resolveScorecardWeights(config.scorecardWeights)
131
+
94
132
  return {
95
133
  enabled: config.enabled ?? true,
96
134
  maxRows,
@@ -100,5 +138,19 @@ export function resolveConfig(config: Config = {}): ResolvedConfig {
100
138
  allowedExtensions,
101
139
  workspaceRoot: config.workspaceRoot ?? '',
102
140
  storeReports: config.storeReports ?? true,
141
+ scorecardWeights,
142
+ }
143
+ }
144
+
145
+ /** Resolve scorecard weights, filling defaults and rejecting non-negative violations loudly. */
146
+ function resolveScorecardWeights(weights: ScorecardWeights = {}): Required<ScorecardWeights> {
147
+ const out = {} as Required<ScorecardWeights>
148
+ for (const dimension of SCORECARD_DIMENSIONS) {
149
+ const value = weights[dimension] ?? 1
150
+ if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) {
151
+ throw new TypeError(`scorecardWeights.${dimension} must be a non-negative finite number, got ${String(value)}`)
152
+ }
153
+ out[dimension] = value
103
154
  }
155
+ return out
104
156
  }
@@ -0,0 +1,190 @@
1
+ /**
2
+ * Delivery contract for `data_clean`: a deterministic pre-delivery validation
3
+ * summary (primary-key/type/dedupe row-count comparison plus uniqueness and
4
+ * non-null regression) and the clean before/after profile diff report. Pure —
5
+ * no I/O, no clock, no RNG; timestamps are injected.
6
+ * @module dsh-data-quality/contract
7
+ */
8
+
9
+ import { isMissing, parseBoolean, parseDate, parseNumeric, throwIfAborted, type Row, type Table } from './dataset.ts'
10
+ import type { CleanResult, CleanRule } from './clean.ts'
11
+ import { profileTable, type ProfileReport } from './profile.ts'
12
+ import type { ScorecardDimensionName } from './scorecard.ts'
13
+
14
+ /** The pre-delivery contract summary attached to a clean run. */
15
+ export interface CleanContractSummary {
16
+ /** Row count before cleaning. */
17
+ readonly inputRows: number
18
+ /** Row count after cleaning. */
19
+ readonly outputRows: number
20
+ /** Rows removed across all rules. */
21
+ readonly removedRows: number
22
+ /** Key columns of the first `dedupe` rule; `null` when no dedupe rule ran (uniqueness then covers full rows). */
23
+ readonly dedupeColumns: string[] | null
24
+ /** Whether the uniqueness key is free of duplicates in the cleaned output. */
25
+ readonly uniqueKeys: boolean
26
+ /** Remaining duplicate rows over the uniqueness key in the cleaned output. */
27
+ readonly remainingDuplicateRows: number
28
+ /** Non-null regression: remaining missing cells in columns a `fill-missing` rule targeted. */
29
+ readonly remainingMissing: Array<{ readonly column: string; readonly count: number }>
30
+ /** Type regression: remaining non-conforming cells in columns a `coerce-type` rule targeted. */
31
+ readonly typeConformance: Array<{ readonly column: string; readonly to: 'number' | 'date' | 'boolean'; readonly invalidCount: number }>
32
+ /** Per-column decision trace: which strategies touched each column and how many rows they affected. */
33
+ readonly columnDecisions: Array<{
34
+ readonly column: string
35
+ readonly decisions: Array<{ readonly strategy: string; readonly affectedRows: number }>
36
+ }>
37
+ }
38
+
39
+ /** The clean before/after profile diff report (persisted under the `clean-diff` report kind). */
40
+ export interface CleanProfileDiff {
41
+ readonly dataset: string
42
+ /** Full profile of the input dataset. */
43
+ readonly before: ProfileReport
44
+ /** Full profile of the cleaned output (including its scorecard). */
45
+ readonly after: ProfileReport
46
+ readonly generatedAt: number
47
+ }
48
+
49
+ /** Count missing cells in one column of the cleaned output. */
50
+ function countMissing(rows: readonly Row[], column: string, signal?: AbortSignal): number {
51
+ let count = 0
52
+ for (const [index, row] of rows.entries()) {
53
+ if (index % 1024 === 0) throwIfAborted(signal)
54
+ if (isMissing(row[column])) count += 1
55
+ }
56
+ return count
57
+ }
58
+
59
+ /** Count present cells in one column that still fail to parse as the requested type. */
60
+ function countNonConforming(rows: readonly Row[], column: string, to: 'number' | 'date' | 'boolean', signal?: AbortSignal): number {
61
+ let count = 0
62
+ for (const [index, row] of rows.entries()) {
63
+ if (index % 1024 === 0) throwIfAborted(signal)
64
+ const cell = row[column]
65
+ if (isMissing(cell)) continue
66
+ const conforms = to === 'number' ? parseNumeric(cell) !== undefined : to === 'date' ? parseDate(cell) !== undefined : parseBoolean(cell) !== undefined
67
+ if (!conforms) count += 1
68
+ }
69
+ return count
70
+ }
71
+
72
+ /**
73
+ * Compute the pre-delivery contract summary from a cleaning run: dedupe
74
+ * before/after row counts, the dedupe key's uniqueness regression, and the
75
+ * non-null/type regressions over the columns the rules targeted.
76
+ * @param result - the cleaning outcome (input/output rows plus cleaned rows).
77
+ * @param rules - the ordered rules that produced `result`.
78
+ * @param options - optional abort signal.
79
+ * @returns the contract summary.
80
+ */
81
+ export function computeCleanContract(
82
+ result: CleanResult,
83
+ rules: readonly CleanRule[],
84
+ options: { signal?: AbortSignal | undefined } = {},
85
+ ): CleanContractSummary {
86
+ const dedupeRule = rules.find((rule) => rule.rule === 'dedupe') as Extract<CleanRule, { rule: 'dedupe' }> | undefined
87
+ const dedupeColumns = dedupeRule === undefined ? null : [...(dedupeRule.columns ?? result.columns)]
88
+ const keyColumns = dedupeColumns ?? result.columns
89
+ const seen = new Set<string>()
90
+ let remainingDuplicateRows = 0
91
+ for (const [index, row] of result.rows.entries()) {
92
+ if (index % 1024 === 0) throwIfAborted(options.signal)
93
+ const key = JSON.stringify(keyColumns.map((column) => row[column] ?? null))
94
+ if (seen.has(key)) {
95
+ remainingDuplicateRows += 1
96
+ } else {
97
+ seen.add(key)
98
+ }
99
+ }
100
+ const remainingMissing = rules
101
+ .filter((rule) => rule.rule === 'fill-missing')
102
+ .map((rule) => {
103
+ const fill = rule as Extract<CleanRule, { rule: 'fill-missing' }>
104
+ return { column: fill.column, count: countMissing(result.rows, fill.column, options.signal) }
105
+ })
106
+ const typeConformance = rules
107
+ .filter((rule) => rule.rule === 'coerce-type')
108
+ .map((rule) => {
109
+ const coerce = rule as Extract<CleanRule, { rule: 'coerce-type' }>
110
+ return { column: coerce.column, to: coerce.to, invalidCount: countNonConforming(result.rows, coerce.column, coerce.to, options.signal) }
111
+ })
112
+ const columnDecisions = traceColumnDecisions(result, rules)
113
+ return {
114
+ inputRows: result.inputRows,
115
+ outputRows: result.outputRows,
116
+ removedRows: result.inputRows - result.outputRows,
117
+ dedupeColumns,
118
+ uniqueKeys: remainingDuplicateRows === 0,
119
+ remainingDuplicateRows,
120
+ remainingMissing,
121
+ typeConformance,
122
+ columnDecisions,
123
+ }
124
+ }
125
+
126
+ /** The columns a single rule targets (single-column rules target one column; trim/dedupe may target many). */
127
+ function ruleColumns(rule: CleanRule, allColumns: readonly string[]): string[] {
128
+ switch (rule.rule) {
129
+ case 'fill-missing':
130
+ case 'coerce-type':
131
+ case 'normalize-unit':
132
+ case 'map-values':
133
+ return [rule.column]
134
+ case 'trim':
135
+ case 'dedupe':
136
+ return [...(rule.columns ?? allColumns)]
137
+ default:
138
+ return []
139
+ }
140
+ }
141
+
142
+ /** Build the per-column decision trace: strategies + affected rows, in dataset column order. */
143
+ function traceColumnDecisions(
144
+ result: CleanResult,
145
+ rules: readonly CleanRule[],
146
+ ): CleanContractSummary['columnDecisions'] {
147
+ const byColumn = new Map<string, Array<{ strategy: string; affectedRows: number }>>()
148
+ for (const [index, rule] of rules.entries()) {
149
+ const log = result.logs[index]
150
+ if (log === undefined) continue
151
+ for (const column of ruleColumns(rule, result.columns)) {
152
+ const list = byColumn.get(column) ?? []
153
+ list.push({ strategy: rule.rule, affectedRows: log.affectedRows })
154
+ byColumn.set(column, list)
155
+ }
156
+ }
157
+ return result.columns
158
+ .filter((column) => byColumn.has(column))
159
+ .map((column) => ({ column, decisions: byColumn.get(column) as Array<{ strategy: string; affectedRows: number }> }))
160
+ }
161
+
162
+ /**
163
+ * Build the clean before/after profile diff: two full {@link ProfileReport}
164
+ * snapshots (input and cleaned output) under one envelope. Reuses the profile
165
+ * engine so the scorecard and duplicate detection stay consistent across the
166
+ * whole plugin.
167
+ * @param input - the input dataset.
168
+ * @param output - the cleaned dataset.
169
+ * @param options - dataset label, injected timestamp, optional weights, optional abort signal.
170
+ * @returns the diff report.
171
+ */
172
+ export function computeCleanProfileDiff(
173
+ input: Table,
174
+ output: Table,
175
+ options: { dataset: string; generatedAt: number; scorecardWeights?: Readonly<Record<ScorecardDimensionName, number>> | undefined; signal?: AbortSignal | undefined },
176
+ ): CleanProfileDiff {
177
+ const before = profileTable(input, {
178
+ dataset: options.dataset,
179
+ generatedAt: options.generatedAt,
180
+ scorecardWeights: options.scorecardWeights,
181
+ signal: options.signal,
182
+ })
183
+ const after = profileTable(output, {
184
+ dataset: options.dataset,
185
+ generatedAt: options.generatedAt,
186
+ scorecardWeights: options.scorecardWeights,
187
+ signal: options.signal,
188
+ })
189
+ return { dataset: options.dataset, before, after, generatedAt: options.generatedAt }
190
+ }
package/src/dataset.ts CHANGED
@@ -9,6 +9,7 @@
9
9
  import { open, stat } from 'node:fs/promises'
10
10
  import path from 'node:path'
11
11
  import { createHash } from 'node:crypto'
12
+ import { TextDecoder } from 'node:util'
12
13
  import type { JsonValue } from '@deepseek-ai/dsh-session'
13
14
  import type { ResolvedConfig } from './config.ts'
14
15
 
@@ -18,12 +19,22 @@ export type Cell = JsonValue
18
19
  /** One dataset row keyed by column name. */
19
20
  export type Row = Record<string, Cell>
20
21
 
22
+ /** Detected file encoding metadata (BOM presence + UTF-8 validity). */
23
+ export interface EncodingInfo {
24
+ /** `'utf-8'` when a UTF-8 byte-order mark was present, else `null`. */
25
+ readonly bom: 'utf-8' | null
26
+ /** Whether the bytes decode as valid UTF-8 (no replacement characters forced). */
27
+ readonly validUtf8: boolean
28
+ }
29
+
21
30
  /** A tabular dataset: ordered columns plus rows. */
22
31
  export interface Table {
23
32
  /** Column names in file order. */
24
33
  readonly columns: string[]
25
34
  /** Rows, each carrying every declared column (missing cells are `null`). */
26
35
  readonly rows: Row[]
36
+ /** Detected file encoding; present only for tables loaded from a file. */
37
+ readonly encoding?: EncodingInfo
27
38
  }
28
39
 
29
40
  /** The root form a document load returns (citation checking walks this). */
@@ -101,13 +112,39 @@ export function throwIfAborted(signal: AbortSignal | undefined): void {
101
112
  }
102
113
 
103
114
  /**
104
- * Read a dataset file under the size cap.
115
+ * Detect a UTF-8 byte-order mark and validate the byte sequence. Invalid
116
+ * UTF-8 is a data-quality finding the profile reports (`validUtf8: false`)
117
+ * rather than a structural error that blocks the read — the decoded text
118
+ * keeps U+FFFD replacement characters so the profile can still run.
119
+ * @param buffer - raw file bytes.
120
+ * @returns the encoding metadata.
121
+ */
122
+ export function detectEncoding(buffer: Uint8Array): EncodingInfo {
123
+ const bom = buffer.length >= 3 && buffer[0] === 0xef && buffer[1] === 0xbb && buffer[2] === 0xbf ? 'utf-8' : null
124
+ const body = bom === 'utf-8' ? buffer.subarray(3) : buffer
125
+ let validUtf8 = true
126
+ try {
127
+ new TextDecoder('utf-8', { fatal: true }).decode(body)
128
+ } catch {
129
+ validUtf8 = false
130
+ }
131
+ return { bom, validUtf8 }
132
+ }
133
+
134
+ /** Decode file bytes, stripping a UTF-8 BOM and preserving replacement characters for invalid bytes. */
135
+ function decodeUtf8Text(buffer: Uint8Array, encoding: EncodingInfo): string {
136
+ const body = encoding.bom === 'utf-8' ? buffer.subarray(3) : buffer
137
+ return new TextDecoder('utf-8').decode(body)
138
+ }
139
+
140
+ /**
141
+ * Read a dataset file under the size cap, detecting its encoding.
105
142
  * @param absolutePath - normalized absolute path (from {@link resolveWorkspacePath}).
106
143
  * @param config - resolved config (size cap).
107
144
  * @param signal - optional abort signal honored around the read.
108
- * @returns the UTF-8 text.
145
+ * @returns the decoded UTF-8 text plus its encoding metadata.
109
146
  */
110
- export async function readDatasetText(absolutePath: string, config: ResolvedConfig, signal?: AbortSignal): Promise<string> {
147
+ export async function readDatasetFile(absolutePath: string, config: ResolvedConfig, signal?: AbortSignal): Promise<{ text: string; encoding: EncodingInfo }> {
111
148
  throwIfAborted(signal)
112
149
  let info
113
150
  try {
@@ -128,7 +165,9 @@ export async function readDatasetText(absolutePath: string, config: ResolvedConf
128
165
  const handle = await open(absolutePath, 'r')
129
166
  try {
130
167
  throwIfAborted(signal)
131
- return await handle.readFile('utf8')
168
+ const buffer = await handle.readFile()
169
+ const encoding = detectEncoding(buffer)
170
+ return { text: decodeUtf8Text(buffer, encoding), encoding }
132
171
  } finally {
133
172
  await handle.close()
134
173
  }
@@ -325,12 +364,12 @@ function parseJsonLines(text: string): unknown[] {
325
364
  * @returns the parsed table.
326
365
  */
327
366
  export async function loadTable(absolutePath: string, config: ResolvedConfig, signal?: AbortSignal): Promise<Table> {
328
- const text = await readDatasetText(absolutePath, config, signal)
367
+ const { text, encoding } = await readDatasetFile(absolutePath, config, signal)
329
368
  const ext = path.extname(absolutePath).toLowerCase()
330
369
  throwIfAborted(signal)
331
- if (ext === '.csv') return parseDelimited(text, ',', config, signal)
332
- if (ext === '.tsv') return parseDelimited(text, '\t', config, signal)
333
- return parseJsonTable(text, ext, config, signal)
370
+ if (ext === '.csv') return { ...parseDelimited(text, ',', config, signal), encoding }
371
+ if (ext === '.tsv') return { ...parseDelimited(text, '\t', config, signal), encoding }
372
+ return { ...parseJsonTable(text, ext, config, signal), encoding }
334
373
  }
335
374
 
336
375
  /**
@@ -345,11 +384,11 @@ export async function loadTable(absolutePath: string, config: ResolvedConfig, si
345
384
  export async function loadDocument(absolutePath: string, config: ResolvedConfig, signal?: AbortSignal): Promise<DocumentRoot> {
346
385
  const ext = path.extname(absolutePath).toLowerCase()
347
386
  if (ext === '.json') {
348
- const text = await readDatasetText(absolutePath, config, signal)
387
+ const { text } = await readDatasetFile(absolutePath, config, signal)
349
388
  return { kind: 'json', value: parseJsonDocument(text) }
350
389
  }
351
390
  if (ext === '.jsonl') {
352
- const text = await readDatasetText(absolutePath, config, signal)
391
+ const { text } = await readDatasetFile(absolutePath, config, signal)
353
392
  return { kind: 'json', value: parseJsonLines(text) }
354
393
  }
355
394
  const table = await loadTable(absolutePath, config, signal)
@@ -405,18 +444,25 @@ const DATE_PATTERNS: readonly RegExp[] = [
405
444
  /^(\d{4})-(\d{1,2})-(\d{1,2})[ T](\d{1,2}):(\d{2})(?::(\d{2}))?(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?$/u,
406
445
  ]
407
446
 
408
- /**
409
- * Deterministic date parse to epoch milliseconds. Accepts `YYYY-MM-DD`,
410
- * `YYYY/MM/DD`, and ISO-like datetimes (date-only forms read as UTC midnight).
411
- * Calendar-invalid dates (e.g. 2025-13-40) reject. Returns `undefined` when
412
- * the cell is not a recognized date.
413
- * @param cell - the cell to parse (`undefined` when the column is absent).
414
- * @returns epoch milliseconds, or `undefined`.
415
- */
416
- export function parseDate(cell: Cell | undefined): number | undefined {
447
+ /** Deterministic date-format labels in {@link DATE_PATTERNS} order. */
448
+ const DATE_FORMATS = ['iso-date', 'slash-date', 'datetime'] as const
449
+
450
+ /** The format label of a recognized date cell (source pattern, not the epoch). */
451
+ export type DateFormat = (typeof DATE_FORMATS)[number]
452
+
453
+ /** A recognized date cell: its parsed epoch plus the source format label. */
454
+ export interface DateParse {
455
+ /** Epoch milliseconds (UTC). */
456
+ readonly epoch: number
457
+ /** Which {@link DATE_PATTERNS} entry matched. */
458
+ readonly format: DateFormat
459
+ }
460
+
461
+ /** Parse one date cell to its epoch plus format; `undefined` when unrecognized. */
462
+ function parseDateCellInternal(cell: Cell | undefined): DateParse | undefined {
417
463
  if (typeof cell !== 'string') return undefined
418
464
  const text = cell.trim()
419
- for (const pattern of DATE_PATTERNS) {
465
+ for (const [index, pattern] of DATE_PATTERNS.entries()) {
420
466
  const match = pattern.exec(text)
421
467
  if (match === null) continue
422
468
  const year = Number(match[1])
@@ -429,11 +475,45 @@ export function parseDate(cell: Cell | undefined): number | undefined {
429
475
  const epoch = Date.UTC(year, month - 1, day, hour, minute, second)
430
476
  const check = new Date(epoch)
431
477
  if (check.getUTCMonth() !== month - 1 || check.getUTCDate() !== day) return undefined
432
- return epoch
478
+ const format = DATE_FORMATS[index]
479
+ if (format === undefined) return undefined
480
+ return { epoch, format }
433
481
  }
434
482
  return undefined
435
483
  }
436
484
 
485
+ /**
486
+ * Deterministic date parse to epoch milliseconds. Accepts `YYYY-MM-DD`,
487
+ * `YYYY/MM/DD`, and ISO-like datetimes (date-only forms read as UTC midnight).
488
+ * Calendar-invalid dates (e.g. 2025-13-40) reject. Returns `undefined` when
489
+ * the cell is not a recognized date.
490
+ * @param cell - the cell to parse (`undefined` when the column is absent).
491
+ * @returns epoch milliseconds, or `undefined`.
492
+ */
493
+ export function parseDate(cell: Cell | undefined): number | undefined {
494
+ return parseDateCellInternal(cell)?.epoch
495
+ }
496
+
497
+ /**
498
+ * Parse one date cell to its epoch plus source format label.
499
+ * @param cell - the cell to parse (`undefined` when the column is absent).
500
+ * @returns the parsed date, or `undefined` when the cell is not a recognized date.
501
+ */
502
+ export function parseDateCell(cell: Cell | undefined): DateParse | undefined {
503
+ return parseDateCellInternal(cell)
504
+ }
505
+
506
+ /**
507
+ * The format label of a recognized date cell (`iso-date` / `slash-date` /
508
+ * `datetime`); `undefined` when the cell is not a recognized date. Used to
509
+ * measure a date column's format consistency.
510
+ * @param cell - the cell to inspect (`undefined` when the column is absent).
511
+ * @returns the source format label, or `undefined`.
512
+ */
513
+ export function dateFormatOf(cell: Cell | undefined): DateFormat | undefined {
514
+ return parseDateCellInternal(cell)?.format
515
+ }
516
+
437
517
  /** Boolean parse: true/false/yes/no/1/0, case-insensitive. */
438
518
  export function parseBoolean(cell: Cell | undefined): boolean | undefined {
439
519
  if (typeof cell === 'boolean') return cell
package/src/events.ts CHANGED
@@ -33,6 +33,8 @@ export interface DataQualityEventData {
33
33
  readonly rules?: number
34
34
  readonly failedRules?: number
35
35
  readonly passed?: boolean
36
+ readonly expectations?: number
37
+ readonly failedExpectations?: number
36
38
  }
37
39
  }
38
40
 
package/src/index.ts CHANGED
@@ -23,14 +23,15 @@ import { LocalDataQualityService } from './provider-local.ts'
23
23
  import { defineProfileTool } from './tools/profile.ts'
24
24
  import { defineCleanTool } from './tools/clean.ts'
25
25
  import { defineVerifyTool } from './tools/verify.ts'
26
+ import { defineReportTool } from './tools/report.ts'
26
27
  import { VERSION } from './version.ts'
27
28
 
28
29
  export const name = 'data-quality'
29
- /** The three model tools and the durable report domain. */
30
+ /** The four model tools and the durable report domain. */
30
31
  export const inject = ['tools', 'storageDomain']
31
32
 
32
33
  export { Config, resolveConfig } from './config.ts'
33
- export type { Config as DataQualityConfig, ResolvedConfig } from './config.ts'
34
+ export type { Config as DataQualityConfig, ResolvedConfig, ScorecardWeights } from './config.ts'
34
35
  export { VERSION } from './version.ts'
35
36
  export {
36
37
  DataQualityService,
@@ -43,16 +44,21 @@ export {
43
44
  } from './service.ts'
44
45
  export { LocalDataQualityService, type ProviderDeps } from './provider-local.ts'
45
46
  export type { CleanRule, CleanRuleLog, CleanResult } from './clean.ts'
46
- export type { ProfileReport, ColumnProfile, NumericProfile } from './profile.ts'
47
- export type { VerifyRule, VerifyReport, VerifyRuleResult, VerifyEvidenceRow } from './verify.ts'
48
- export { parseLocator, checkCitations } from './verify.ts'
49
- export { profileTable, renderProfileText } from './profile.ts'
47
+ export { computeCleanContract, computeCleanProfileDiff, type CleanContractSummary, type CleanProfileDiff } from './contract.ts'
48
+ export type { ProfileReport, ColumnProfile, NumericProfile, DuplicateDetection, InferredType } from './profile.ts'
49
+ export type { VerifyRule, VerifyReport, VerifyRuleResult, VerifyEvidenceRow, VerifyExpectation, VerifyExpectationResult, VerifyMetric } from './verify.ts'
50
+ export { parseLocator, checkCitations, verifyExpectations, VerifyExpectationError } from './verify.ts'
51
+ export { profileTable, renderProfileText, detectDuplicateRows, countDuplicateRows, numericProfile } from './profile.ts'
52
+ export { computeScorecard, type DataQualityScorecard, type ScorecardDimension, type ScorecardDimensionName } from './scorecard.ts'
53
+ export { INDUSTRY_PRESETS, INDUSTRY_PRESET_IDS, resolveIndustryPreset, type IndustryPreset } from './presets.ts'
50
54
  export { applyCleanRules, serializeDelimited } from './clean.ts'
51
55
  export { verifyTable, renderVerifyText } from './verify.ts'
52
56
  export {
53
57
  DatasetError,
54
58
  loadTable,
55
59
  loadDocument,
60
+ readDatasetFile,
61
+ detectEncoding,
56
62
  parseDelimited,
57
63
  parseJsonTable,
58
64
  resolveWorkspacePath,
@@ -60,15 +66,22 @@ export {
60
66
  isMissing,
61
67
  parseNumeric,
62
68
  parseDate,
69
+ parseDateCell,
70
+ dateFormatOf,
63
71
  parseBoolean,
64
72
  type Table,
65
73
  type Row,
66
74
  type Cell,
67
75
  type DocumentRoot,
76
+ type DateParse,
77
+ type DateFormat,
78
+ type EncodingInfo,
68
79
  } from './dataset.ts'
69
- export { dataQualityDomainSpec, reportKeyOf, reportRecordSchema, type ReportRecord, type ReportStore } from './store.ts'
80
+ export { dataQualityDomainSpec, reportKeyOf, reportRecordSchema, isValidReportKey, type ReportRecord, type ReportStore, type StoredReport } from './store.ts'
70
81
  export { DATA_QUALITY_EVENT_TYPES, appendDataQualityEvent, type DataQualityEventData, type DataQualityEventType } from './events.ts'
71
82
  export { MAX_CELL_TEXT, truncateCell, truncateRow } from './present.ts'
83
+ export { renderProfileHtml, renderCleanHtml, type CleanReportHtml } from './report-html.ts'
84
+ export { REPORT_SCHEMA_VERSION } from './version.ts'
72
85
 
73
86
  /**
74
87
  * Mount the seam: resolve config (fail loud), open the report domain, publish
@@ -97,6 +110,11 @@ export async function apply(ctx: Context, config: Config = {}): Promise<void> {
97
110
  return key
98
111
  },
99
112
  get: (key) => reports.get(key),
113
+ list: (kind) =>
114
+ [...reports.entries()]
115
+ .filter(([, record]) => record.kind === kind)
116
+ .sort(([keyA], [keyB]) => (keyA < keyB ? -1 : keyA > keyB ? 1 : 0))
117
+ .map(([key, record]) => ({ key, ...record })),
100
118
  }
101
119
  }
102
120
 
@@ -104,7 +122,8 @@ export async function apply(ctx: Context, config: Config = {}): Promise<void> {
104
122
  ctx.tools.register(defineProfileTool(service))
105
123
  ctx.tools.register(defineCleanTool(service))
106
124
  ctx.tools.register(defineVerifyTool(service))
107
- logger.info(`dsh-data-quality ${VERSION} mounted: ctx.dataQuality + data_profile/data_clean/data_verify`)
125
+ ctx.tools.register(defineReportTool(service))
126
+ logger.info(`dsh-data-quality ${VERSION} mounted: ctx.dataQuality + data_profile/data_clean/data_verify/data_report`)
108
127
 
109
128
  if (domain !== undefined) {
110
129
  const handle = domain