dsh-data-quality 0.1.3 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -0
- package/README.es.md +7 -1
- package/README.hi.md +7 -1
- package/README.md +22 -9
- package/README.pt.md +7 -1
- package/README.zh.md +7 -1
- package/cordis.patch.yml +11 -0
- package/lib/index.js +1609 -145
- package/lib/types/config.d.ts +12 -0
- package/lib/types/config.d.ts.map +1 -1
- package/lib/types/config.js +35 -0
- package/lib/types/config.js.map +1 -1
- package/lib/types/contract.d.ts +83 -0
- package/lib/types/contract.d.ts.map +1 -0
- package/lib/types/contract.js +144 -0
- package/lib/types/contract.js.map +1 -0
- package/lib/types/dataset.d.ts +50 -3
- package/lib/types/dataset.d.ts.map +1 -1
- package/lib/types/dataset.js +76 -21
- package/lib/types/dataset.js.map +1 -1
- package/lib/types/events.d.ts +2 -0
- package/lib/types/events.d.ts.map +1 -1
- package/lib/types/events.js.map +1 -1
- package/lib/types/index.d.ts +13 -8
- package/lib/types/index.d.ts.map +1 -1
- package/lib/types/index.js +17 -6
- package/lib/types/index.js.map +1 -1
- package/lib/types/presets.d.ts +28 -0
- package/lib/types/presets.d.ts.map +1 -0
- package/lib/types/presets.js +134 -0
- package/lib/types/presets.js.map +1 -0
- package/lib/types/profile.d.ts +40 -1
- package/lib/types/profile.d.ts.map +1 -1
- package/lib/types/profile.js +66 -10
- package/lib/types/profile.js.map +1 -1
- package/lib/types/provider-local.d.ts +7 -1
- package/lib/types/provider-local.d.ts.map +1 -1
- package/lib/types/provider-local.js +72 -2
- package/lib/types/provider-local.js.map +1 -1
- package/lib/types/report-html.d.ts +41 -0
- package/lib/types/report-html.d.ts.map +1 -0
- package/lib/types/report-html.js +192 -0
- package/lib/types/report-html.js.map +1 -0
- package/lib/types/scorecard.d.ts +50 -0
- package/lib/types/scorecard.d.ts.map +1 -0
- package/lib/types/scorecard.js +186 -0
- package/lib/types/scorecard.js.map +1 -0
- package/lib/types/service.d.ts +32 -2
- package/lib/types/service.d.ts.map +1 -1
- package/lib/types/service.js.map +1 -1
- package/lib/types/store.d.ts +21 -1
- package/lib/types/store.d.ts.map +1 -1
- package/lib/types/store.js +13 -1
- package/lib/types/store.js.map +1 -1
- package/lib/types/tools/clean.d.ts.map +1 -1
- package/lib/types/tools/clean.js +76 -1
- package/lib/types/tools/clean.js.map +1 -1
- package/lib/types/tools/profile-report-schema.d.ts +319 -0
- package/lib/types/tools/profile-report-schema.d.ts.map +1 -0
- package/lib/types/tools/profile-report-schema.js +96 -0
- package/lib/types/tools/profile-report-schema.js.map +1 -0
- package/lib/types/tools/profile.d.ts.map +1 -1
- package/lib/types/tools/profile.js +6 -53
- package/lib/types/tools/profile.js.map +1 -1
- package/lib/types/tools/report.d.ts +14 -0
- package/lib/types/tools/report.d.ts.map +1 -0
- package/lib/types/tools/report.js +112 -0
- package/lib/types/tools/report.js.map +1 -0
- package/lib/types/tools/shared.d.ts.map +1 -1
- package/lib/types/tools/shared.js +22 -2
- package/lib/types/tools/shared.js.map +1 -1
- package/lib/types/tools/verify.d.ts.map +1 -1
- package/lib/types/tools/verify.js +36 -1
- package/lib/types/tools/verify.js.map +1 -1
- package/lib/types/verify.d.ts +49 -3
- package/lib/types/verify.d.ts.map +1 -1
- package/lib/types/verify.js +111 -4
- package/lib/types/verify.js.map +1 -1
- package/lib/types/version.d.ts +8 -1
- package/lib/types/version.d.ts.map +1 -1
- package/lib/types/version.js +8 -1
- package/lib/types/version.js.map +1 -1
- package/package.json +1 -1
- package/src/config.ts +52 -0
- package/src/contract.ts +190 -0
- package/src/dataset.ts +101 -21
- package/src/events.ts +2 -0
- package/src/index.ts +27 -8
- package/src/presets.ts +146 -0
- package/src/profile.ts +105 -11
- package/src/provider-local.ts +77 -3
- package/src/report-html.ts +208 -0
- package/src/scorecard.ts +244 -0
- package/src/service.ts +34 -2
- package/src/store.ts +28 -2
- package/src/tools/clean.ts +76 -1
- package/src/tools/profile-report-schema.ts +98 -0
- package/src/tools/profile.ts +8 -56
- package/src/tools/report.ts +137 -0
- package/src/tools/shared.ts +21 -2
- package/src/tools/verify.ts +40 -2
- package/src/verify.ts +151 -5
- package/src/version.ts +9 -1
package/src/config.ts
CHANGED
|
@@ -7,6 +7,29 @@
|
|
|
7
7
|
|
|
8
8
|
import z from '@deepseek-ai/schemastery'
|
|
9
9
|
|
|
10
|
+
/** Per-dimension scorecard weights (all optional; defaults fill to `1` = equal). */
|
|
11
|
+
export interface ScorecardWeights {
|
|
12
|
+
completeness?: number
|
|
13
|
+
uniqueness?: number
|
|
14
|
+
validity?: number
|
|
15
|
+
consistency?: number
|
|
16
|
+
timeliness?: number
|
|
17
|
+
accuracy?: number
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/** The six scorecard dimension ids, in report order (kept in sync with `scorecard.ts`). */
|
|
21
|
+
const SCORECARD_DIMENSIONS = ['completeness', 'uniqueness', 'validity', 'consistency', 'timeliness', 'accuracy'] as const
|
|
22
|
+
|
|
23
|
+
/** Equal-weight defaults for {@link ScorecardWeights}. */
|
|
24
|
+
const EQUAL_WEIGHTS: Required<ScorecardWeights> = {
|
|
25
|
+
completeness: 1,
|
|
26
|
+
uniqueness: 1,
|
|
27
|
+
validity: 1,
|
|
28
|
+
consistency: 1,
|
|
29
|
+
timeliness: 1,
|
|
30
|
+
accuracy: 1,
|
|
31
|
+
}
|
|
32
|
+
|
|
10
33
|
/** Raw plugin config — every field optional; {@link resolveConfig} supplies the defaults. */
|
|
11
34
|
export interface Config {
|
|
12
35
|
/** Master switch; `false` mounts nothing. */
|
|
@@ -25,6 +48,8 @@ export interface Config {
|
|
|
25
48
|
workspaceRoot?: string
|
|
26
49
|
/** Persist reports to the `data_quality` storage domain. */
|
|
27
50
|
storeReports?: boolean
|
|
51
|
+
/** Per-dimension weights for the scorecard's weighted overall total (non-negative). */
|
|
52
|
+
scorecardWeights?: ScorecardWeights
|
|
28
53
|
}
|
|
29
54
|
|
|
30
55
|
/** Fully resolved config handed to the runtime. */
|
|
@@ -37,6 +62,7 @@ export interface ResolvedConfig {
|
|
|
37
62
|
readonly allowedExtensions: readonly string[]
|
|
38
63
|
readonly workspaceRoot: string
|
|
39
64
|
readonly storeReports: boolean
|
|
65
|
+
readonly scorecardWeights: Required<ScorecardWeights>
|
|
40
66
|
}
|
|
41
67
|
|
|
42
68
|
/** Schemastery schema: the loader validates and fills defaults before `apply`. */
|
|
@@ -49,6 +75,16 @@ export const Config: z<Config> = z.object({
|
|
|
49
75
|
allowedExtensions: z.array(z.string()).default(['.csv', '.tsv', '.json', '.jsonl']),
|
|
50
76
|
workspaceRoot: z.string().default(''),
|
|
51
77
|
storeReports: z.boolean().default(true),
|
|
78
|
+
scorecardWeights: z
|
|
79
|
+
.object({
|
|
80
|
+
completeness: z.number().default(1),
|
|
81
|
+
uniqueness: z.number().default(1),
|
|
82
|
+
validity: z.number().default(1),
|
|
83
|
+
consistency: z.number().default(1),
|
|
84
|
+
timeliness: z.number().default(1),
|
|
85
|
+
accuracy: z.number().default(1),
|
|
86
|
+
})
|
|
87
|
+
.default(EQUAL_WEIGHTS),
|
|
52
88
|
})
|
|
53
89
|
|
|
54
90
|
/** Throw unless `value` is a positive safe integer. */
|
|
@@ -91,6 +127,8 @@ export function resolveConfig(config: Config = {}): ResolvedConfig {
|
|
|
91
127
|
throw new TypeError('allowedExtensions must not be empty')
|
|
92
128
|
}
|
|
93
129
|
|
|
130
|
+
const scorecardWeights = resolveScorecardWeights(config.scorecardWeights)
|
|
131
|
+
|
|
94
132
|
return {
|
|
95
133
|
enabled: config.enabled ?? true,
|
|
96
134
|
maxRows,
|
|
@@ -100,5 +138,19 @@ export function resolveConfig(config: Config = {}): ResolvedConfig {
|
|
|
100
138
|
allowedExtensions,
|
|
101
139
|
workspaceRoot: config.workspaceRoot ?? '',
|
|
102
140
|
storeReports: config.storeReports ?? true,
|
|
141
|
+
scorecardWeights,
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/** Resolve scorecard weights, filling defaults and rejecting non-negative violations loudly. */
|
|
146
|
+
function resolveScorecardWeights(weights: ScorecardWeights = {}): Required<ScorecardWeights> {
|
|
147
|
+
const out = {} as Required<ScorecardWeights>
|
|
148
|
+
for (const dimension of SCORECARD_DIMENSIONS) {
|
|
149
|
+
const value = weights[dimension] ?? 1
|
|
150
|
+
if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) {
|
|
151
|
+
throw new TypeError(`scorecardWeights.${dimension} must be a non-negative finite number, got ${String(value)}`)
|
|
152
|
+
}
|
|
153
|
+
out[dimension] = value
|
|
103
154
|
}
|
|
155
|
+
return out
|
|
104
156
|
}
|
package/src/contract.ts
ADDED
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Delivery contract for `data_clean`: a deterministic pre-delivery validation
|
|
3
|
+
* summary (primary-key/type/dedupe row-count comparison plus uniqueness and
|
|
4
|
+
* non-null regression) and the clean before/after profile diff report. Pure —
|
|
5
|
+
* no I/O, no clock, no RNG; timestamps are injected.
|
|
6
|
+
* @module dsh-data-quality/contract
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { isMissing, parseBoolean, parseDate, parseNumeric, throwIfAborted, type Row, type Table } from './dataset.ts'
|
|
10
|
+
import type { CleanResult, CleanRule } from './clean.ts'
|
|
11
|
+
import { profileTable, type ProfileReport } from './profile.ts'
|
|
12
|
+
import type { ScorecardDimensionName } from './scorecard.ts'
|
|
13
|
+
|
|
14
|
+
/** The pre-delivery contract summary attached to a clean run. */
|
|
15
|
+
export interface CleanContractSummary {
|
|
16
|
+
/** Row count before cleaning. */
|
|
17
|
+
readonly inputRows: number
|
|
18
|
+
/** Row count after cleaning. */
|
|
19
|
+
readonly outputRows: number
|
|
20
|
+
/** Rows removed across all rules. */
|
|
21
|
+
readonly removedRows: number
|
|
22
|
+
/** Key columns of the first `dedupe` rule; `null` when no dedupe rule ran (uniqueness then covers full rows). */
|
|
23
|
+
readonly dedupeColumns: string[] | null
|
|
24
|
+
/** Whether the uniqueness key is free of duplicates in the cleaned output. */
|
|
25
|
+
readonly uniqueKeys: boolean
|
|
26
|
+
/** Remaining duplicate rows over the uniqueness key in the cleaned output. */
|
|
27
|
+
readonly remainingDuplicateRows: number
|
|
28
|
+
/** Non-null regression: remaining missing cells in columns a `fill-missing` rule targeted. */
|
|
29
|
+
readonly remainingMissing: Array<{ readonly column: string; readonly count: number }>
|
|
30
|
+
/** Type regression: remaining non-conforming cells in columns a `coerce-type` rule targeted. */
|
|
31
|
+
readonly typeConformance: Array<{ readonly column: string; readonly to: 'number' | 'date' | 'boolean'; readonly invalidCount: number }>
|
|
32
|
+
/** Per-column decision trace: which strategies touched each column and how many rows they affected. */
|
|
33
|
+
readonly columnDecisions: Array<{
|
|
34
|
+
readonly column: string
|
|
35
|
+
readonly decisions: Array<{ readonly strategy: string; readonly affectedRows: number }>
|
|
36
|
+
}>
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** The clean before/after profile diff report (persisted under the `clean-diff` report kind). */
|
|
40
|
+
export interface CleanProfileDiff {
|
|
41
|
+
readonly dataset: string
|
|
42
|
+
/** Full profile of the input dataset. */
|
|
43
|
+
readonly before: ProfileReport
|
|
44
|
+
/** Full profile of the cleaned output (including its scorecard). */
|
|
45
|
+
readonly after: ProfileReport
|
|
46
|
+
readonly generatedAt: number
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** Count missing cells in one column of the cleaned output. */
|
|
50
|
+
function countMissing(rows: readonly Row[], column: string, signal?: AbortSignal): number {
|
|
51
|
+
let count = 0
|
|
52
|
+
for (const [index, row] of rows.entries()) {
|
|
53
|
+
if (index % 1024 === 0) throwIfAborted(signal)
|
|
54
|
+
if (isMissing(row[column])) count += 1
|
|
55
|
+
}
|
|
56
|
+
return count
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** Count present cells in one column that still fail to parse as the requested type. */
|
|
60
|
+
function countNonConforming(rows: readonly Row[], column: string, to: 'number' | 'date' | 'boolean', signal?: AbortSignal): number {
|
|
61
|
+
let count = 0
|
|
62
|
+
for (const [index, row] of rows.entries()) {
|
|
63
|
+
if (index % 1024 === 0) throwIfAborted(signal)
|
|
64
|
+
const cell = row[column]
|
|
65
|
+
if (isMissing(cell)) continue
|
|
66
|
+
const conforms = to === 'number' ? parseNumeric(cell) !== undefined : to === 'date' ? parseDate(cell) !== undefined : parseBoolean(cell) !== undefined
|
|
67
|
+
if (!conforms) count += 1
|
|
68
|
+
}
|
|
69
|
+
return count
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Compute the pre-delivery contract summary from a cleaning run: dedupe
|
|
74
|
+
* before/after row counts, the dedupe key's uniqueness regression, and the
|
|
75
|
+
* non-null/type regressions over the columns the rules targeted.
|
|
76
|
+
* @param result - the cleaning outcome (input/output rows plus cleaned rows).
|
|
77
|
+
* @param rules - the ordered rules that produced `result`.
|
|
78
|
+
* @param options - optional abort signal.
|
|
79
|
+
* @returns the contract summary.
|
|
80
|
+
*/
|
|
81
|
+
export function computeCleanContract(
|
|
82
|
+
result: CleanResult,
|
|
83
|
+
rules: readonly CleanRule[],
|
|
84
|
+
options: { signal?: AbortSignal | undefined } = {},
|
|
85
|
+
): CleanContractSummary {
|
|
86
|
+
const dedupeRule = rules.find((rule) => rule.rule === 'dedupe') as Extract<CleanRule, { rule: 'dedupe' }> | undefined
|
|
87
|
+
const dedupeColumns = dedupeRule === undefined ? null : [...(dedupeRule.columns ?? result.columns)]
|
|
88
|
+
const keyColumns = dedupeColumns ?? result.columns
|
|
89
|
+
const seen = new Set<string>()
|
|
90
|
+
let remainingDuplicateRows = 0
|
|
91
|
+
for (const [index, row] of result.rows.entries()) {
|
|
92
|
+
if (index % 1024 === 0) throwIfAborted(options.signal)
|
|
93
|
+
const key = JSON.stringify(keyColumns.map((column) => row[column] ?? null))
|
|
94
|
+
if (seen.has(key)) {
|
|
95
|
+
remainingDuplicateRows += 1
|
|
96
|
+
} else {
|
|
97
|
+
seen.add(key)
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
const remainingMissing = rules
|
|
101
|
+
.filter((rule) => rule.rule === 'fill-missing')
|
|
102
|
+
.map((rule) => {
|
|
103
|
+
const fill = rule as Extract<CleanRule, { rule: 'fill-missing' }>
|
|
104
|
+
return { column: fill.column, count: countMissing(result.rows, fill.column, options.signal) }
|
|
105
|
+
})
|
|
106
|
+
const typeConformance = rules
|
|
107
|
+
.filter((rule) => rule.rule === 'coerce-type')
|
|
108
|
+
.map((rule) => {
|
|
109
|
+
const coerce = rule as Extract<CleanRule, { rule: 'coerce-type' }>
|
|
110
|
+
return { column: coerce.column, to: coerce.to, invalidCount: countNonConforming(result.rows, coerce.column, coerce.to, options.signal) }
|
|
111
|
+
})
|
|
112
|
+
const columnDecisions = traceColumnDecisions(result, rules)
|
|
113
|
+
return {
|
|
114
|
+
inputRows: result.inputRows,
|
|
115
|
+
outputRows: result.outputRows,
|
|
116
|
+
removedRows: result.inputRows - result.outputRows,
|
|
117
|
+
dedupeColumns,
|
|
118
|
+
uniqueKeys: remainingDuplicateRows === 0,
|
|
119
|
+
remainingDuplicateRows,
|
|
120
|
+
remainingMissing,
|
|
121
|
+
typeConformance,
|
|
122
|
+
columnDecisions,
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/** The columns a single rule targets (single-column rules target one column; trim/dedupe may target many). */
|
|
127
|
+
function ruleColumns(rule: CleanRule, allColumns: readonly string[]): string[] {
|
|
128
|
+
switch (rule.rule) {
|
|
129
|
+
case 'fill-missing':
|
|
130
|
+
case 'coerce-type':
|
|
131
|
+
case 'normalize-unit':
|
|
132
|
+
case 'map-values':
|
|
133
|
+
return [rule.column]
|
|
134
|
+
case 'trim':
|
|
135
|
+
case 'dedupe':
|
|
136
|
+
return [...(rule.columns ?? allColumns)]
|
|
137
|
+
default:
|
|
138
|
+
return []
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/** Build the per-column decision trace: strategies + affected rows, in dataset column order. */
|
|
143
|
+
function traceColumnDecisions(
|
|
144
|
+
result: CleanResult,
|
|
145
|
+
rules: readonly CleanRule[],
|
|
146
|
+
): CleanContractSummary['columnDecisions'] {
|
|
147
|
+
const byColumn = new Map<string, Array<{ strategy: string; affectedRows: number }>>()
|
|
148
|
+
for (const [index, rule] of rules.entries()) {
|
|
149
|
+
const log = result.logs[index]
|
|
150
|
+
if (log === undefined) continue
|
|
151
|
+
for (const column of ruleColumns(rule, result.columns)) {
|
|
152
|
+
const list = byColumn.get(column) ?? []
|
|
153
|
+
list.push({ strategy: rule.rule, affectedRows: log.affectedRows })
|
|
154
|
+
byColumn.set(column, list)
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
return result.columns
|
|
158
|
+
.filter((column) => byColumn.has(column))
|
|
159
|
+
.map((column) => ({ column, decisions: byColumn.get(column) as Array<{ strategy: string; affectedRows: number }> }))
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* Build the clean before/after profile diff: two full {@link ProfileReport}
|
|
164
|
+
* snapshots (input and cleaned output) under one envelope. Reuses the profile
|
|
165
|
+
* engine so the scorecard and duplicate detection stay consistent across the
|
|
166
|
+
* whole plugin.
|
|
167
|
+
* @param input - the input dataset.
|
|
168
|
+
* @param output - the cleaned dataset.
|
|
169
|
+
* @param options - dataset label, injected timestamp, optional weights, optional abort signal.
|
|
170
|
+
* @returns the diff report.
|
|
171
|
+
*/
|
|
172
|
+
export function computeCleanProfileDiff(
|
|
173
|
+
input: Table,
|
|
174
|
+
output: Table,
|
|
175
|
+
options: { dataset: string; generatedAt: number; scorecardWeights?: Readonly<Record<ScorecardDimensionName, number>> | undefined; signal?: AbortSignal | undefined },
|
|
176
|
+
): CleanProfileDiff {
|
|
177
|
+
const before = profileTable(input, {
|
|
178
|
+
dataset: options.dataset,
|
|
179
|
+
generatedAt: options.generatedAt,
|
|
180
|
+
scorecardWeights: options.scorecardWeights,
|
|
181
|
+
signal: options.signal,
|
|
182
|
+
})
|
|
183
|
+
const after = profileTable(output, {
|
|
184
|
+
dataset: options.dataset,
|
|
185
|
+
generatedAt: options.generatedAt,
|
|
186
|
+
scorecardWeights: options.scorecardWeights,
|
|
187
|
+
signal: options.signal,
|
|
188
|
+
})
|
|
189
|
+
return { dataset: options.dataset, before, after, generatedAt: options.generatedAt }
|
|
190
|
+
}
|
package/src/dataset.ts
CHANGED
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
import { open, stat } from 'node:fs/promises'
|
|
10
10
|
import path from 'node:path'
|
|
11
11
|
import { createHash } from 'node:crypto'
|
|
12
|
+
import { TextDecoder } from 'node:util'
|
|
12
13
|
import type { JsonValue } from '@deepseek-ai/dsh-session'
|
|
13
14
|
import type { ResolvedConfig } from './config.ts'
|
|
14
15
|
|
|
@@ -18,12 +19,22 @@ export type Cell = JsonValue
|
|
|
18
19
|
/** One dataset row keyed by column name. */
|
|
19
20
|
export type Row = Record<string, Cell>
|
|
20
21
|
|
|
22
|
+
/** Detected file encoding metadata (BOM presence + UTF-8 validity). */
|
|
23
|
+
export interface EncodingInfo {
|
|
24
|
+
/** `'utf-8'` when a UTF-8 byte-order mark was present, else `null`. */
|
|
25
|
+
readonly bom: 'utf-8' | null
|
|
26
|
+
/** Whether the bytes decode as valid UTF-8 (no replacement characters forced). */
|
|
27
|
+
readonly validUtf8: boolean
|
|
28
|
+
}
|
|
29
|
+
|
|
21
30
|
/** A tabular dataset: ordered columns plus rows. */
|
|
22
31
|
export interface Table {
|
|
23
32
|
/** Column names in file order. */
|
|
24
33
|
readonly columns: string[]
|
|
25
34
|
/** Rows, each carrying every declared column (missing cells are `null`). */
|
|
26
35
|
readonly rows: Row[]
|
|
36
|
+
/** Detected file encoding; present only for tables loaded from a file. */
|
|
37
|
+
readonly encoding?: EncodingInfo
|
|
27
38
|
}
|
|
28
39
|
|
|
29
40
|
/** The root form a document load returns (citation checking walks this). */
|
|
@@ -101,13 +112,39 @@ export function throwIfAborted(signal: AbortSignal | undefined): void {
|
|
|
101
112
|
}
|
|
102
113
|
|
|
103
114
|
/**
|
|
104
|
-
*
|
|
115
|
+
* Detect a UTF-8 byte-order mark and validate the byte sequence. Invalid
|
|
116
|
+
* UTF-8 is a data-quality finding the profile reports (`validUtf8: false`)
|
|
117
|
+
* rather than a structural error that blocks the read — the decoded text
|
|
118
|
+
* keeps U+FFFD replacement characters so the profile can still run.
|
|
119
|
+
* @param buffer - raw file bytes.
|
|
120
|
+
* @returns the encoding metadata.
|
|
121
|
+
*/
|
|
122
|
+
export function detectEncoding(buffer: Uint8Array): EncodingInfo {
|
|
123
|
+
const bom = buffer.length >= 3 && buffer[0] === 0xef && buffer[1] === 0xbb && buffer[2] === 0xbf ? 'utf-8' : null
|
|
124
|
+
const body = bom === 'utf-8' ? buffer.subarray(3) : buffer
|
|
125
|
+
let validUtf8 = true
|
|
126
|
+
try {
|
|
127
|
+
new TextDecoder('utf-8', { fatal: true }).decode(body)
|
|
128
|
+
} catch {
|
|
129
|
+
validUtf8 = false
|
|
130
|
+
}
|
|
131
|
+
return { bom, validUtf8 }
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/** Decode file bytes, stripping a UTF-8 BOM and preserving replacement characters for invalid bytes. */
|
|
135
|
+
function decodeUtf8Text(buffer: Uint8Array, encoding: EncodingInfo): string {
|
|
136
|
+
const body = encoding.bom === 'utf-8' ? buffer.subarray(3) : buffer
|
|
137
|
+
return new TextDecoder('utf-8').decode(body)
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* Read a dataset file under the size cap, detecting its encoding.
|
|
105
142
|
* @param absolutePath - normalized absolute path (from {@link resolveWorkspacePath}).
|
|
106
143
|
* @param config - resolved config (size cap).
|
|
107
144
|
* @param signal - optional abort signal honored around the read.
|
|
108
|
-
* @returns the UTF-8 text.
|
|
145
|
+
* @returns the decoded UTF-8 text plus its encoding metadata.
|
|
109
146
|
*/
|
|
110
|
-
export async function
|
|
147
|
+
export async function readDatasetFile(absolutePath: string, config: ResolvedConfig, signal?: AbortSignal): Promise<{ text: string; encoding: EncodingInfo }> {
|
|
111
148
|
throwIfAborted(signal)
|
|
112
149
|
let info
|
|
113
150
|
try {
|
|
@@ -128,7 +165,9 @@ export async function readDatasetText(absolutePath: string, config: ResolvedConf
|
|
|
128
165
|
const handle = await open(absolutePath, 'r')
|
|
129
166
|
try {
|
|
130
167
|
throwIfAborted(signal)
|
|
131
|
-
|
|
168
|
+
const buffer = await handle.readFile()
|
|
169
|
+
const encoding = detectEncoding(buffer)
|
|
170
|
+
return { text: decodeUtf8Text(buffer, encoding), encoding }
|
|
132
171
|
} finally {
|
|
133
172
|
await handle.close()
|
|
134
173
|
}
|
|
@@ -325,12 +364,12 @@ function parseJsonLines(text: string): unknown[] {
|
|
|
325
364
|
* @returns the parsed table.
|
|
326
365
|
*/
|
|
327
366
|
export async function loadTable(absolutePath: string, config: ResolvedConfig, signal?: AbortSignal): Promise<Table> {
|
|
328
|
-
const text = await
|
|
367
|
+
const { text, encoding } = await readDatasetFile(absolutePath, config, signal)
|
|
329
368
|
const ext = path.extname(absolutePath).toLowerCase()
|
|
330
369
|
throwIfAborted(signal)
|
|
331
|
-
if (ext === '.csv') return parseDelimited(text, ',', config, signal)
|
|
332
|
-
if (ext === '.tsv') return parseDelimited(text, '\t', config, signal)
|
|
333
|
-
return parseJsonTable(text, ext, config, signal)
|
|
370
|
+
if (ext === '.csv') return { ...parseDelimited(text, ',', config, signal), encoding }
|
|
371
|
+
if (ext === '.tsv') return { ...parseDelimited(text, '\t', config, signal), encoding }
|
|
372
|
+
return { ...parseJsonTable(text, ext, config, signal), encoding }
|
|
334
373
|
}
|
|
335
374
|
|
|
336
375
|
/**
|
|
@@ -345,11 +384,11 @@ export async function loadTable(absolutePath: string, config: ResolvedConfig, si
|
|
|
345
384
|
export async function loadDocument(absolutePath: string, config: ResolvedConfig, signal?: AbortSignal): Promise<DocumentRoot> {
|
|
346
385
|
const ext = path.extname(absolutePath).toLowerCase()
|
|
347
386
|
if (ext === '.json') {
|
|
348
|
-
const text = await
|
|
387
|
+
const { text } = await readDatasetFile(absolutePath, config, signal)
|
|
349
388
|
return { kind: 'json', value: parseJsonDocument(text) }
|
|
350
389
|
}
|
|
351
390
|
if (ext === '.jsonl') {
|
|
352
|
-
const text = await
|
|
391
|
+
const { text } = await readDatasetFile(absolutePath, config, signal)
|
|
353
392
|
return { kind: 'json', value: parseJsonLines(text) }
|
|
354
393
|
}
|
|
355
394
|
const table = await loadTable(absolutePath, config, signal)
|
|
@@ -405,18 +444,25 @@ const DATE_PATTERNS: readonly RegExp[] = [
|
|
|
405
444
|
/^(\d{4})-(\d{1,2})-(\d{1,2})[ T](\d{1,2}):(\d{2})(?::(\d{2}))?(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?$/u,
|
|
406
445
|
]
|
|
407
446
|
|
|
408
|
-
/**
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
447
|
+
/** Deterministic date-format labels in {@link DATE_PATTERNS} order. */
|
|
448
|
+
const DATE_FORMATS = ['iso-date', 'slash-date', 'datetime'] as const
|
|
449
|
+
|
|
450
|
+
/** The format label of a recognized date cell (source pattern, not the epoch). */
|
|
451
|
+
export type DateFormat = (typeof DATE_FORMATS)[number]
|
|
452
|
+
|
|
453
|
+
/** A recognized date cell: its parsed epoch plus the source format label. */
|
|
454
|
+
export interface DateParse {
|
|
455
|
+
/** Epoch milliseconds (UTC). */
|
|
456
|
+
readonly epoch: number
|
|
457
|
+
/** Which {@link DATE_PATTERNS} entry matched. */
|
|
458
|
+
readonly format: DateFormat
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
/** Parse one date cell to its epoch plus format; `undefined` when unrecognized. */
|
|
462
|
+
function parseDateCellInternal(cell: Cell | undefined): DateParse | undefined {
|
|
417
463
|
if (typeof cell !== 'string') return undefined
|
|
418
464
|
const text = cell.trim()
|
|
419
|
-
for (const pattern of DATE_PATTERNS) {
|
|
465
|
+
for (const [index, pattern] of DATE_PATTERNS.entries()) {
|
|
420
466
|
const match = pattern.exec(text)
|
|
421
467
|
if (match === null) continue
|
|
422
468
|
const year = Number(match[1])
|
|
@@ -429,11 +475,45 @@ export function parseDate(cell: Cell | undefined): number | undefined {
|
|
|
429
475
|
const epoch = Date.UTC(year, month - 1, day, hour, minute, second)
|
|
430
476
|
const check = new Date(epoch)
|
|
431
477
|
if (check.getUTCMonth() !== month - 1 || check.getUTCDate() !== day) return undefined
|
|
432
|
-
|
|
478
|
+
const format = DATE_FORMATS[index]
|
|
479
|
+
if (format === undefined) return undefined
|
|
480
|
+
return { epoch, format }
|
|
433
481
|
}
|
|
434
482
|
return undefined
|
|
435
483
|
}
|
|
436
484
|
|
|
485
|
+
/**
|
|
486
|
+
* Deterministic date parse to epoch milliseconds. Accepts `YYYY-MM-DD`,
|
|
487
|
+
* `YYYY/MM/DD`, and ISO-like datetimes (date-only forms read as UTC midnight).
|
|
488
|
+
* Calendar-invalid dates (e.g. 2025-13-40) reject. Returns `undefined` when
|
|
489
|
+
* the cell is not a recognized date.
|
|
490
|
+
* @param cell - the cell to parse (`undefined` when the column is absent).
|
|
491
|
+
* @returns epoch milliseconds, or `undefined`.
|
|
492
|
+
*/
|
|
493
|
+
export function parseDate(cell: Cell | undefined): number | undefined {
|
|
494
|
+
return parseDateCellInternal(cell)?.epoch
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
/**
|
|
498
|
+
* Parse one date cell to its epoch plus source format label.
|
|
499
|
+
* @param cell - the cell to parse (`undefined` when the column is absent).
|
|
500
|
+
* @returns the parsed date, or `undefined` when the cell is not a recognized date.
|
|
501
|
+
*/
|
|
502
|
+
export function parseDateCell(cell: Cell | undefined): DateParse | undefined {
|
|
503
|
+
return parseDateCellInternal(cell)
|
|
504
|
+
}
|
|
505
|
+
|
|
506
|
+
/**
|
|
507
|
+
* The format label of a recognized date cell (`iso-date` / `slash-date` /
|
|
508
|
+
* `datetime`); `undefined` when the cell is not a recognized date. Used to
|
|
509
|
+
* measure a date column's format consistency.
|
|
510
|
+
* @param cell - the cell to inspect (`undefined` when the column is absent).
|
|
511
|
+
* @returns the source format label, or `undefined`.
|
|
512
|
+
*/
|
|
513
|
+
export function dateFormatOf(cell: Cell | undefined): DateFormat | undefined {
|
|
514
|
+
return parseDateCellInternal(cell)?.format
|
|
515
|
+
}
|
|
516
|
+
|
|
437
517
|
/** Boolean parse: true/false/yes/no/1/0, case-insensitive. */
|
|
438
518
|
export function parseBoolean(cell: Cell | undefined): boolean | undefined {
|
|
439
519
|
if (typeof cell === 'boolean') return cell
|
package/src/events.ts
CHANGED
package/src/index.ts
CHANGED
|
@@ -23,14 +23,15 @@ import { LocalDataQualityService } from './provider-local.ts'
|
|
|
23
23
|
import { defineProfileTool } from './tools/profile.ts'
|
|
24
24
|
import { defineCleanTool } from './tools/clean.ts'
|
|
25
25
|
import { defineVerifyTool } from './tools/verify.ts'
|
|
26
|
+
import { defineReportTool } from './tools/report.ts'
|
|
26
27
|
import { VERSION } from './version.ts'
|
|
27
28
|
|
|
28
29
|
export const name = 'data-quality'
|
|
29
|
-
/** The
|
|
30
|
+
/** The four model tools and the durable report domain. */
|
|
30
31
|
export const inject = ['tools', 'storageDomain']
|
|
31
32
|
|
|
32
33
|
export { Config, resolveConfig } from './config.ts'
|
|
33
|
-
export type { Config as DataQualityConfig, ResolvedConfig } from './config.ts'
|
|
34
|
+
export type { Config as DataQualityConfig, ResolvedConfig, ScorecardWeights } from './config.ts'
|
|
34
35
|
export { VERSION } from './version.ts'
|
|
35
36
|
export {
|
|
36
37
|
DataQualityService,
|
|
@@ -43,16 +44,21 @@ export {
|
|
|
43
44
|
} from './service.ts'
|
|
44
45
|
export { LocalDataQualityService, type ProviderDeps } from './provider-local.ts'
|
|
45
46
|
export type { CleanRule, CleanRuleLog, CleanResult } from './clean.ts'
|
|
46
|
-
export
|
|
47
|
-
export type {
|
|
48
|
-
export {
|
|
49
|
-
export {
|
|
47
|
+
export { computeCleanContract, computeCleanProfileDiff, type CleanContractSummary, type CleanProfileDiff } from './contract.ts'
|
|
48
|
+
export type { ProfileReport, ColumnProfile, NumericProfile, DuplicateDetection, InferredType } from './profile.ts'
|
|
49
|
+
export type { VerifyRule, VerifyReport, VerifyRuleResult, VerifyEvidenceRow, VerifyExpectation, VerifyExpectationResult, VerifyMetric } from './verify.ts'
|
|
50
|
+
export { parseLocator, checkCitations, verifyExpectations, VerifyExpectationError } from './verify.ts'
|
|
51
|
+
export { profileTable, renderProfileText, detectDuplicateRows, countDuplicateRows, numericProfile } from './profile.ts'
|
|
52
|
+
export { computeScorecard, type DataQualityScorecard, type ScorecardDimension, type ScorecardDimensionName } from './scorecard.ts'
|
|
53
|
+
export { INDUSTRY_PRESETS, INDUSTRY_PRESET_IDS, resolveIndustryPreset, type IndustryPreset } from './presets.ts'
|
|
50
54
|
export { applyCleanRules, serializeDelimited } from './clean.ts'
|
|
51
55
|
export { verifyTable, renderVerifyText } from './verify.ts'
|
|
52
56
|
export {
|
|
53
57
|
DatasetError,
|
|
54
58
|
loadTable,
|
|
55
59
|
loadDocument,
|
|
60
|
+
readDatasetFile,
|
|
61
|
+
detectEncoding,
|
|
56
62
|
parseDelimited,
|
|
57
63
|
parseJsonTable,
|
|
58
64
|
resolveWorkspacePath,
|
|
@@ -60,15 +66,22 @@ export {
|
|
|
60
66
|
isMissing,
|
|
61
67
|
parseNumeric,
|
|
62
68
|
parseDate,
|
|
69
|
+
parseDateCell,
|
|
70
|
+
dateFormatOf,
|
|
63
71
|
parseBoolean,
|
|
64
72
|
type Table,
|
|
65
73
|
type Row,
|
|
66
74
|
type Cell,
|
|
67
75
|
type DocumentRoot,
|
|
76
|
+
type DateParse,
|
|
77
|
+
type DateFormat,
|
|
78
|
+
type EncodingInfo,
|
|
68
79
|
} from './dataset.ts'
|
|
69
|
-
export { dataQualityDomainSpec, reportKeyOf, reportRecordSchema, type ReportRecord, type ReportStore } from './store.ts'
|
|
80
|
+
export { dataQualityDomainSpec, reportKeyOf, reportRecordSchema, isValidReportKey, type ReportRecord, type ReportStore, type StoredReport } from './store.ts'
|
|
70
81
|
export { DATA_QUALITY_EVENT_TYPES, appendDataQualityEvent, type DataQualityEventData, type DataQualityEventType } from './events.ts'
|
|
71
82
|
export { MAX_CELL_TEXT, truncateCell, truncateRow } from './present.ts'
|
|
83
|
+
export { renderProfileHtml, renderCleanHtml, type CleanReportHtml } from './report-html.ts'
|
|
84
|
+
export { REPORT_SCHEMA_VERSION } from './version.ts'
|
|
72
85
|
|
|
73
86
|
/**
|
|
74
87
|
* Mount the seam: resolve config (fail loud), open the report domain, publish
|
|
@@ -97,6 +110,11 @@ export async function apply(ctx: Context, config: Config = {}): Promise<void> {
|
|
|
97
110
|
return key
|
|
98
111
|
},
|
|
99
112
|
get: (key) => reports.get(key),
|
|
113
|
+
list: (kind) =>
|
|
114
|
+
[...reports.entries()]
|
|
115
|
+
.filter(([, record]) => record.kind === kind)
|
|
116
|
+
.sort(([keyA], [keyB]) => (keyA < keyB ? -1 : keyA > keyB ? 1 : 0))
|
|
117
|
+
.map(([key, record]) => ({ key, ...record })),
|
|
100
118
|
}
|
|
101
119
|
}
|
|
102
120
|
|
|
@@ -104,7 +122,8 @@ export async function apply(ctx: Context, config: Config = {}): Promise<void> {
|
|
|
104
122
|
ctx.tools.register(defineProfileTool(service))
|
|
105
123
|
ctx.tools.register(defineCleanTool(service))
|
|
106
124
|
ctx.tools.register(defineVerifyTool(service))
|
|
107
|
-
|
|
125
|
+
ctx.tools.register(defineReportTool(service))
|
|
126
|
+
logger.info(`dsh-data-quality ${VERSION} mounted: ctx.dataQuality + data_profile/data_clean/data_verify/data_report`)
|
|
108
127
|
|
|
109
128
|
if (domain !== undefined) {
|
|
110
129
|
const handle = domain
|