dsh-data-quality 0.1.3 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/CHANGELOG.md +33 -0
  2. package/README.es.md +7 -1
  3. package/README.hi.md +7 -1
  4. package/README.md +22 -9
  5. package/README.pt.md +7 -1
  6. package/README.zh.md +7 -1
  7. package/cordis.patch.yml +11 -0
  8. package/lib/index.js +1609 -145
  9. package/lib/types/config.d.ts +12 -0
  10. package/lib/types/config.d.ts.map +1 -1
  11. package/lib/types/config.js +35 -0
  12. package/lib/types/config.js.map +1 -1
  13. package/lib/types/contract.d.ts +83 -0
  14. package/lib/types/contract.d.ts.map +1 -0
  15. package/lib/types/contract.js +144 -0
  16. package/lib/types/contract.js.map +1 -0
  17. package/lib/types/dataset.d.ts +50 -3
  18. package/lib/types/dataset.d.ts.map +1 -1
  19. package/lib/types/dataset.js +76 -21
  20. package/lib/types/dataset.js.map +1 -1
  21. package/lib/types/events.d.ts +2 -0
  22. package/lib/types/events.d.ts.map +1 -1
  23. package/lib/types/events.js.map +1 -1
  24. package/lib/types/index.d.ts +13 -8
  25. package/lib/types/index.d.ts.map +1 -1
  26. package/lib/types/index.js +17 -6
  27. package/lib/types/index.js.map +1 -1
  28. package/lib/types/presets.d.ts +28 -0
  29. package/lib/types/presets.d.ts.map +1 -0
  30. package/lib/types/presets.js +134 -0
  31. package/lib/types/presets.js.map +1 -0
  32. package/lib/types/profile.d.ts +40 -1
  33. package/lib/types/profile.d.ts.map +1 -1
  34. package/lib/types/profile.js +66 -10
  35. package/lib/types/profile.js.map +1 -1
  36. package/lib/types/provider-local.d.ts +7 -1
  37. package/lib/types/provider-local.d.ts.map +1 -1
  38. package/lib/types/provider-local.js +72 -2
  39. package/lib/types/provider-local.js.map +1 -1
  40. package/lib/types/report-html.d.ts +41 -0
  41. package/lib/types/report-html.d.ts.map +1 -0
  42. package/lib/types/report-html.js +192 -0
  43. package/lib/types/report-html.js.map +1 -0
  44. package/lib/types/scorecard.d.ts +50 -0
  45. package/lib/types/scorecard.d.ts.map +1 -0
  46. package/lib/types/scorecard.js +186 -0
  47. package/lib/types/scorecard.js.map +1 -0
  48. package/lib/types/service.d.ts +32 -2
  49. package/lib/types/service.d.ts.map +1 -1
  50. package/lib/types/service.js.map +1 -1
  51. package/lib/types/store.d.ts +21 -1
  52. package/lib/types/store.d.ts.map +1 -1
  53. package/lib/types/store.js +13 -1
  54. package/lib/types/store.js.map +1 -1
  55. package/lib/types/tools/clean.d.ts.map +1 -1
  56. package/lib/types/tools/clean.js +76 -1
  57. package/lib/types/tools/clean.js.map +1 -1
  58. package/lib/types/tools/profile-report-schema.d.ts +319 -0
  59. package/lib/types/tools/profile-report-schema.d.ts.map +1 -0
  60. package/lib/types/tools/profile-report-schema.js +96 -0
  61. package/lib/types/tools/profile-report-schema.js.map +1 -0
  62. package/lib/types/tools/profile.d.ts.map +1 -1
  63. package/lib/types/tools/profile.js +6 -53
  64. package/lib/types/tools/profile.js.map +1 -1
  65. package/lib/types/tools/report.d.ts +14 -0
  66. package/lib/types/tools/report.d.ts.map +1 -0
  67. package/lib/types/tools/report.js +112 -0
  68. package/lib/types/tools/report.js.map +1 -0
  69. package/lib/types/tools/shared.d.ts.map +1 -1
  70. package/lib/types/tools/shared.js +22 -2
  71. package/lib/types/tools/shared.js.map +1 -1
  72. package/lib/types/tools/verify.d.ts.map +1 -1
  73. package/lib/types/tools/verify.js +36 -1
  74. package/lib/types/tools/verify.js.map +1 -1
  75. package/lib/types/verify.d.ts +49 -3
  76. package/lib/types/verify.d.ts.map +1 -1
  77. package/lib/types/verify.js +111 -4
  78. package/lib/types/verify.js.map +1 -1
  79. package/lib/types/version.d.ts +8 -1
  80. package/lib/types/version.d.ts.map +1 -1
  81. package/lib/types/version.js +8 -1
  82. package/lib/types/version.js.map +1 -1
  83. package/package.json +1 -1
  84. package/src/config.ts +52 -0
  85. package/src/contract.ts +190 -0
  86. package/src/dataset.ts +101 -21
  87. package/src/events.ts +2 -0
  88. package/src/index.ts +27 -8
  89. package/src/presets.ts +146 -0
  90. package/src/profile.ts +105 -11
  91. package/src/provider-local.ts +77 -3
  92. package/src/report-html.ts +208 -0
  93. package/src/scorecard.ts +244 -0
  94. package/src/service.ts +34 -2
  95. package/src/store.ts +28 -2
  96. package/src/tools/clean.ts +76 -1
  97. package/src/tools/profile-report-schema.ts +98 -0
  98. package/src/tools/profile.ts +8 -56
  99. package/src/tools/report.ts +137 -0
  100. package/src/tools/shared.ts +21 -2
  101. package/src/tools/verify.ts +40 -2
  102. package/src/verify.ts +151 -5
  103. package/src/version.ts +9 -1
package/src/report-html.ts ADDED
@@ -0,0 +1,208 @@
1
+ /**
2
+ * Self-contained offline HTML renderers for persisted data-quality reports.
3
+ *
4
+ * `data_report` (with `format: html`) turns a profile or clean report into a
5
+ * single `.html` file that opens offline: every style rule and every script is
6
+ * inlined, there are no external requests (no CDN, no `<link>`, no `<script
7
+ * src>`), and all dataset values are HTML-escaped before they reach the
8
+ * markup. The document carries the DAMA six-dimension scorecard, the
9
+ * per-column profile summary, and (for clean reports) the per-rule cleaning
10
+ * summary table.
11
+ *
12
+ * Pure and deterministic: the only clock is the report's own `generatedAt`;
13
+ * nothing here reads the filesystem or the network.
14
+ * @module dsh-data-quality/report-html
15
+ */
16
+
17
+ import type { ProfileReport } from './profile.ts'
18
+ import { REPORT_SCHEMA_VERSION } from './version.ts'
19
+
20
+ /** Minimal structural view of a clean report's per-rule audit + row contract. */
21
+ export interface CleanReportHtml {
22
+ readonly inputRows: number
23
+ readonly outputRows: number
24
+ readonly logs: ReadonlyArray<{ readonly rule: string; readonly affectedRows: number; readonly detail: string }>
25
+ }
26
+
27
+ /** Escape text for safe embedding in an HTML document. */
28
+ function escapeHtml(text: string): string {
29
+ return text
30
+ .replace(/&/g, '&amp;')
31
+ .replace(/</g, '&lt;')
32
+ .replace(/>/g, '&gt;')
33
+ .replace(/"/g, '&quot;')
34
+ .replace(/'/g, '&#39;')
35
+ }
36
+
37
+ /** Render a 0..1 rate as a percentage string, or `—` when undetermined. */
38
+ function pct(score: number | null): string {
39
+ return score === null ? '—' : `${(score * 100).toFixed(1)}%`
40
+ }
41
+
42
+ /** The shared document shell: inline CSS + inline JS, no external requests. */
43
+ function shell(title: string, body: string, script: string): string {
44
+ return [
45
+ '<!doctype html>',
46
+ '<html lang="en">',
47
+ '<head>',
48
+ '<meta charset="utf-8">',
49
+ `<title>${escapeHtml(title)}</title>`,
50
+ '<style>',
51
+ ' :root { --ink: #1a1f2e; --muted: #6b7280; --line: #e5e7eb; --accent: #0f766e; --fail: #b91c1c; --warn: #b45309; --pass: #15803d; }',
52
+ ' * { box-sizing: border-box; }',
53
+ ' body { margin: 0; padding: 24px; font: 14px/1.5 ui-sans-serif, system-ui, -apple-system, "Segoe UI", sans-serif; color: var(--ink); background: #f8fafc; }',
54
+ ' header { margin-bottom: 20px; }',
55
+ ' h1 { font-size: 20px; margin: 0 0 4px; }',
56
+ ' .meta { color: var(--muted); font-size: 12px; }',
57
+ ' section { background: #fff; border: 1px solid var(--line); border-radius: 8px; padding: 16px 20px; margin-bottom: 16px; }',
58
+ ' h2 { font-size: 15px; margin: 0 0 12px; }',
59
+ ' table { border-collapse: collapse; width: 100%; font-size: 13px; }',
60
+ ' th, td { text-align: left; padding: 6px 10px; border-top: 1px solid var(--line); vertical-align: top; }',
61
+ ' th { color: var(--muted); font-weight: 600; }',
62
+ ' .score-cell { font-variant-numeric: tabular-nums; }',
63
+ ' .dim-fail { color: var(--fail); font-weight: 600; }',
64
+ ' .dim-warn { color: var(--warn); }',
65
+ ' .dim-pass { color: var(--pass); }',
66
+ ' .dim-undetermined { color: var(--muted); }',
67
+ ' button { cursor: pointer; font: inherit; }',
68
+ '</style>',
69
+ '</head>',
70
+ '<body>',
71
+ body,
72
+ '<script>',
73
+ script,
74
+ '</script>',
75
+ '</body>',
76
+ '</html>',
77
+ '',
78
+ ].join('\n')
79
+ }
80
+
81
+ /** The DAMA six-dimension scorecard section. */
82
+ function scorecardSection(report: ProfileReport): string {
83
+ const rows = report.scorecard.dimensions.map((dimension) => {
84
+ const cls = dimension.score === null
85
+ ? 'dim-undetermined'
86
+ : dimension.score >= 0.9 ? 'dim-pass'
87
+ : dimension.score >= 0.7 ? 'dim-warn'
88
+ : 'dim-fail'
89
+ return `<tr><td>${escapeHtml(dimension.name)}</td><td class="score-cell ${cls}">${pct(dimension.score)}</td><td>${escapeHtml(dimension.note)}</td></tr>`
90
+ }).join('\n')
91
+ const overall = report.scorecard.overall
92
+ const weighted = report.scorecard.weightedOverall
93
+ return [
94
+ '<section>',
95
+ '<h2>DAMA six-dimension quality scorecard</h2>',
96
+ '<table>',
97
+ '<thead><tr><th>Dimension</th><th>Score</th><th>Note</th></tr></thead>',
98
+ '<tbody>',
99
+ rows,
100
+ '</tbody>',
101
+ '</table>',
102
+ `<p class="meta" id="summary-text">overall ${pct(overall)} · weighted ${pct(weighted)}</p>`,
103
+ '</section>',
104
+ ].join('\n')
105
+ }
106
+
107
+ /** The per-column profile summary table. */
108
+ function columnsSection(report: ProfileReport): string {
109
+ const rows = report.columns.map((column) => {
110
+ const numeric = column.numeric
111
+ const numericCell = numeric === undefined
112
+ ? '—'
113
+ : `min ${numeric.min} · p25 ${numeric.p25} · median ${numeric.median} · p75 ${numeric.p75} · max ${numeric.max} · mean ${numeric.mean}${numeric.outliers > 0 ? ` · ${numeric.outliers} IQR outliers` : ''}`
114
+ const top = column.topValues === undefined
115
+ ? '—'
116
+ : column.topValues.map((entry) => `${escapeHtml(entry.value)} ×${entry.count}`).join(', ')
117
+ const notes = column.notes.length === 0 ? '' : `<p class="meta">${column.notes.map(escapeHtml).join('; ')}</p>`
118
+ return [
119
+ '<tr>',
120
+ `<td>${escapeHtml(column.name)}</td>`,
121
+ `<td>${escapeHtml(column.inferredType)}</td>`,
122
+ `<td class="score-cell">${column.missing}</td>`,
123
+ `<td class="score-cell">${(column.missingRate * 100).toFixed(1)}%</td>`,
124
+ `<td class="score-cell">${column.unique}</td>`,
125
+ `<td>${numericCell}</td>`,
126
+ `<td>${top}</td>`,
127
+ `<td>${notes}</td>`,
128
+ '</tr>',
129
+ ].join('')
130
+ }).join('\n')
131
+ return [
132
+ '<section>',
133
+ '<h2>Column profile</h2>',
134
+ '<table>',
135
+ '<thead><tr><th>Column</th><th>Type</th><th>Missing</th><th>Missing rate</th><th>Unique</th><th>Numeric distribution</th><th>Top values</th><th>Notes</th></tr></thead>',
136
+ '<tbody>',
137
+ rows,
138
+ '</tbody>',
139
+ '</table>',
140
+ '</section>',
141
+ ].join('\n')
142
+ }
143
+
144
+ /**
145
+ * Render a profile report as a self-contained offline HTML document.
146
+ * @param report - the profile report (already persisted/returned by data_profile).
147
+ * @returns the complete single-file HTML.
148
+ */
149
+ export function renderProfileHtml(report: ProfileReport): string {
150
+ const body = [
151
+ '<header>',
152
+ `<h1>Data profile: ${escapeHtml(report.dataset)}</h1>`,
153
+ `<p class="meta">${report.rowCount} rows × ${report.columnCount} columns · generated ${new Date(report.generatedAt).toISOString()} · schema v${REPORT_SCHEMA_VERSION} · report ${report.reportKey ?? '(unpersisted)'}</p>`,
154
+ '<button id="copy-summary" type="button">Copy summary</button>',
155
+ '</header>',
156
+ scorecardSection(report),
157
+ columnsSection(report),
158
+ ].join('\n')
159
+ const script = [
160
+ "const button = document.getElementById('copy-summary');",
161
+ 'if (button) {',
162
+ ' button.addEventListener("click", () => {',
163
+ ' const text = document.getElementById("summary-text");',
164
+ ' if (text && navigator.clipboard) navigator.clipboard.writeText(text.textContent || "");',
165
+ ' });',
166
+ '}',
167
+ ].join('\n')
168
+ return shell(`Data profile: ${report.dataset}`, body, script)
169
+ }
170
+
171
+ /** The per-rule cleaning summary table (for clean/clean-diff reports). */
172
+ function cleaningSection(report: CleanReportHtml): string {
173
+ const rows = report.logs.map((log) => {
174
+ return `<tr><td>${escapeHtml(log.rule)}</td><td class="score-cell">${log.affectedRows}</td><td>${escapeHtml(log.detail)}</td></tr>`
175
+ }).join('\n')
176
+ const removed = report.inputRows - report.outputRows
177
+ return [
178
+ '<section>',
179
+ '<h2>Cleaning summary</h2>',
180
+ `<p class="meta" id="summary-text">input ${report.inputRows} rows · output ${report.outputRows} rows · removed ${removed} rows</p>`,
181
+ '<table>',
182
+ '<thead><tr><th>Rule</th><th>Affected rows</th><th>Detail</th></tr></thead>',
183
+ '<tbody>',
184
+ rows,
185
+ '</tbody>',
186
+ '</table>',
187
+ '</section>',
188
+ ].join('\n')
189
+ }
190
+
191
+ /**
192
+ * Render a clean report as a self-contained offline HTML document (the
193
+ * per-rule cleaning summary table).
194
+ * @param report - the clean report (logs + input/output row counts).
195
+ * @param dataset - the dataset label for the document title.
196
+ * @returns the complete single-file HTML.
197
+ */
198
+ export function renderCleanHtml(report: CleanReportHtml, dataset: string): string {
199
+ const body = [
200
+ '<header>',
201
+ `<h1>Cleaning report: ${escapeHtml(dataset)}</h1>`,
202
+ `<p class="meta">schema v${REPORT_SCHEMA_VERSION}</p>`,
203
+ '</header>',
204
+ cleaningSection(report),
205
+ ].join('\n')
206
+ const script = "// no interactive behavior needed; everything renders without external requests\n"
207
+ return shell(`Cleaning report: ${dataset}`, body, script)
208
+ }
package/src/scorecard.ts ADDED
@@ -0,0 +1,244 @@
1
+ /**
2
+ * DAMA-style six-dimension quality scorecard over a parsed {@link Table}.
3
+ * Pure and deterministic: every rate derives from one full-table pass, the
4
+ * only clock is the injected `now`, and no dimension fabricates a score it
5
+ * cannot defend — `accuracy` stays `null` (undetermined) without a declared
6
+ * schema and `timeliness` stays `null` without date cells.
7
+ * @module dsh-data-quality/scorecard
8
+ */
9
+
10
+ import { isMissing, parseBoolean, parseDateCell, parseNumeric, throwIfAborted, type Table } from './dataset.ts'
11
+ import type { InferredType } from './profile.ts'
12
+
13
+ /** The six scorecard dimension ids, in report order. */
14
+ export type ScorecardDimensionName = 'completeness' | 'uniqueness' | 'validity' | 'consistency' | 'timeliness' | 'accuracy'
15
+
16
+ /** One scorecard dimension: a 0..1 rate, or `null` when it cannot be determined. */
17
+ export interface ScorecardDimension {
18
+ readonly name: ScorecardDimensionName
19
+ /** Rate in [0, 1] rounded to 6 significant digits; `null` = cannot be determined (never fabricated). */
20
+ readonly score: number | null
21
+ /** Human-readable definition and verdict. */
22
+ readonly note: string
23
+ }
24
+
25
+ /** The six-dimension quality scorecard (also the profile report's `scorecard` field). */
26
+ export interface DataQualityScorecard {
27
+ readonly dimensions: ScorecardDimension[]
28
+ /** Unweighted mean of the determinable (non-null) dimension scores; `null` when none are determinable. */
29
+ readonly overall: number | null
30
+ /** Weighted mean over the determinable dimensions; `null` when none are determinable or all their weights are 0. */
31
+ readonly weightedOverall: number | null
32
+ }
33
+
34
+ /** Per-column aggregation for one full-table pass. */
35
+ interface ColumnStats {
36
+ present: number
37
+ number: number
38
+ date: number
39
+ boolean: number
40
+ string: number
41
+ /** Format-label frequency for consistency. */
42
+ formats: Map<string, number>
43
+ }
44
+
45
+ /** Round to 6 significant digits (mirrors the profile report's rounding). */
46
+ function round6(value: number): number {
47
+ return Number(value.toPrecision(6))
48
+ }
49
+
50
+ /** Increment a format-label counter. */
51
+ function bump(formats: Map<string, number>, tag: string): void {
52
+ formats.set(tag, (formats.get(tag) ?? 0) + 1)
53
+ }
54
+
55
+ /** Inferred column type from the full-table class counts (mirrors `profileColumn`). */
56
+ function inferType(stats: ColumnStats): InferredType {
57
+ if (stats.present === 0) return 'empty'
58
+ if (stats.number === stats.present) return 'number'
59
+ if (stats.date === stats.present) return 'date'
60
+ if (stats.boolean === stats.present) return 'boolean'
61
+ if (stats.string === stats.present) return 'string'
62
+ return 'mixed'
63
+ }
64
+
65
+ /** Format the `value / total` ratio as a plain `note` fragment. */
66
+ function ratioNote(numerator: number, denominator: number): string {
67
+ return `${numerator}/${denominator}`
68
+ }
69
+
70
+ /** Build one determinable dimension with its ratio note. */
71
+ function dimension(name: ScorecardDimensionName, score: number, numerator: number, denominator: number): ScorecardDimension {
72
+ return { name, score, note: ratioNote(numerator, denominator) }
73
+ }
74
+
75
+ /**
76
+ * Compute the six-dimension scorecard over a full table:
77
+ * - `completeness` — non-empty cell rate.
78
+ * - `uniqueness` — unique full-row content rate (`1 - duplicateRows / rowCount`).
79
+ * - `validity` — present cells conforming to their column's inferred type.
80
+ * - `consistency` — present cells in their column's dominant format (a date
81
+ * column's unified `YYYY-MM-DD` vs `YYYY/MM/DD` vs datetime ratio).
82
+ * - `timeliness` — date cells not future-dated relative to the injected `now`.
83
+ * - `accuracy` — declared-schema agreement (limited definition); `null`
84
+ * (undetermined) without a declared schema — never fabricated.
85
+ * @param table - the parsed dataset (full table, not the sampled cards).
86
+ * @param options - injected clock, duplicate-row count, optional declared schema, optional weights, abort signal.
87
+ * @returns the scorecard.
88
+ */
89
+ export function computeScorecard(
90
+ table: Table,
91
+ options: {
92
+ now: number
93
+ duplicateRows: number
94
+ declaredSchema?: Readonly<Record<string, InferredType>> | undefined
95
+ weights?: Readonly<Record<ScorecardDimensionName, number>> | undefined
96
+ signal?: AbortSignal | undefined
97
+ },
98
+ ): DataQualityScorecard {
99
+ throwIfAborted(options.signal)
100
+ const stats = new Map<string, ColumnStats>()
101
+ for (const column of table.columns) {
102
+ stats.set(column, { present: 0, number: 0, date: 0, boolean: 0, string: 0, formats: new Map() })
103
+ }
104
+ let missingCells = 0
105
+ let dateCells = 0
106
+ let futureDateCells = 0
107
+
108
+ for (const [index, row] of table.rows.entries()) {
109
+ if (index % 1024 === 0) throwIfAborted(options.signal)
110
+ for (const column of table.columns) {
111
+ const cell = row[column]
112
+ const columnStats = stats.get(column) as ColumnStats
113
+ if (isMissing(cell)) {
114
+ missingCells += 1
115
+ continue
116
+ }
117
+ columnStats.present += 1
118
+ const numeric = parseNumeric(cell)
119
+ if (numeric !== undefined) {
120
+ columnStats.number += 1
121
+ bump(columnStats.formats, 'number')
122
+ continue
123
+ }
124
+ const date = parseDateCell(cell)
125
+ if (date !== undefined) {
126
+ columnStats.date += 1
127
+ bump(columnStats.formats, date.format)
128
+ dateCells += 1
129
+ if (date.epoch > options.now) futureDateCells += 1
130
+ continue
131
+ }
132
+ if (parseBoolean(cell) !== undefined) {
133
+ columnStats.boolean += 1
134
+ bump(columnStats.formats, 'boolean')
135
+ continue
136
+ }
137
+ columnStats.string += 1
138
+ bump(columnStats.formats, typeof cell === 'string' ? 'string' : 'json')
139
+ }
140
+ }
141
+
142
+ const totalCells = table.rows.length * table.columns.length
143
+ let presentCells = 0
144
+ let invalidCells = 0
145
+ let consistentCells = 0
146
+ for (const [, columnStats] of stats) {
147
+ presentCells += columnStats.present
148
+ // Validity = conformance to the column's dominant TYPE class; consistency
149
+ // below measures dominant FORMAT (so a date column with mixed YYYY-MM-DD /
150
+ // YYYY/MM/DD formats stays valid but inconsistent).
151
+ const dominantClass = Math.max(columnStats.number, columnStats.date, columnStats.boolean, columnStats.string)
152
+ invalidCells += columnStats.present - dominantClass
153
+ if (columnStats.present === 0) continue
154
+ let dominant = 0
155
+ for (const count of columnStats.formats.values()) {
156
+ if (count > dominant) dominant = count
157
+ }
158
+ consistentCells += dominant
159
+ }
160
+
161
+ const completeness: ScorecardDimension =
162
+ totalCells === 0
163
+ ? { name: 'completeness', score: null, note: 'undetermined: no cells' }
164
+ : dimension('completeness', round6(1 - missingCells / totalCells), presentCells, totalCells)
165
+
166
+ const uniqueness: ScorecardDimension =
167
+ table.rows.length === 0
168
+ ? { name: 'uniqueness', score: null, note: 'undetermined: no rows' }
169
+ : dimension('uniqueness', round6(1 - options.duplicateRows / table.rows.length), table.rows.length - options.duplicateRows, table.rows.length)
170
+
171
+ const validity: ScorecardDimension =
172
+ presentCells === 0
173
+ ? { name: 'validity', score: null, note: 'undetermined: no present cells' }
174
+ : dimension('validity', round6(1 - invalidCells / presentCells), presentCells - invalidCells, presentCells)
175
+
176
+ const consistency: ScorecardDimension =
177
+ presentCells === 0
178
+ ? { name: 'consistency', score: null, note: 'undetermined: no present cells' }
179
+ : dimension('consistency', round6(consistentCells / presentCells), consistentCells, presentCells)
180
+
181
+ const timeliness: ScorecardDimension =
182
+ dateCells === 0
183
+ ? { name: 'timeliness', score: null, note: 'undetermined: no date cells' }
184
+ : dimension('timeliness', round6(1 - futureDateCells / dateCells), dateCells - futureDateCells, dateCells)
185
+
186
+ const accuracy = computeAccuracy(stats, table, options.declaredSchema)
187
+
188
+ const dimensions = [completeness, uniqueness, validity, consistency, timeliness, accuracy]
189
+ const scores = dimensions.map((entry) => entry.score).filter((score): score is number => score !== null)
190
+ const overall = scores.length === 0 ? null : round6(scores.reduce((sum, score) => sum + score, 0) / scores.length)
191
+ const weightedOverall = computeWeightedOverall(dimensions, options.weights)
192
+ return { dimensions, overall, weightedOverall }
193
+ }
194
+
195
+ /** Weighted mean over determinable dimensions (weights default to 1; validated non-negative by config). */
196
+ function computeWeightedOverall(
197
+ dimensions: readonly ScorecardDimension[],
198
+ weights: Readonly<Record<ScorecardDimensionName, number>> | undefined,
199
+ ): number | null {
200
+ let weightedSum = 0
201
+ let weightTotal = 0
202
+ for (const entry of dimensions) {
203
+ if (entry.score === null) continue
204
+ const weight = weights?.[entry.name] ?? 1
205
+ weightedSum += weight * entry.score
206
+ weightTotal += weight
207
+ }
208
+ return weightTotal === 0 ? null : round6(weightedSum / weightTotal)
209
+ }
210
+
211
+ /**
212
+ * Limited `accuracy` definition: agreement between the full-table inferred
213
+ * type of each declared column and the declared type. Without a declared
214
+ * schema (or an external truth source) the dimension is `null` (undetermined)
215
+ * — accuracy is never fabricated.
216
+ */
217
+ function computeAccuracy(
218
+ stats: ReadonlyMap<string, ColumnStats>,
219
+ table: Table,
220
+ declaredSchema: Readonly<Record<string, InferredType>> | undefined,
221
+ ): ScorecardDimension {
222
+ if (declaredSchema === undefined || Object.keys(declaredSchema).length === 0) {
223
+ return { name: 'accuracy', score: null, note: 'undetermined: no declared schema or external truth to compare against; accuracy is never fabricated' }
224
+ }
225
+ let checked = 0
226
+ let matched = 0
227
+ const mismatches: string[] = []
228
+ for (const column of table.columns) {
229
+ const declared = declaredSchema[column]
230
+ if (declared === undefined) continue
231
+ checked += 1
232
+ const inferred = inferType(stats.get(column) as ColumnStats)
233
+ if (inferred === declared) matched += 1
234
+ else mismatches.push(`${column}: declared ${declared}, inferred ${inferred}`)
235
+ }
236
+ if (checked === 0) {
237
+ return { name: 'accuracy', score: null, note: 'undetermined: declared schema covers none of the dataset columns' }
238
+ }
239
+ return {
240
+ name: 'accuracy',
241
+ score: round6(matched / checked),
242
+ note: `${matched}/${checked} columns match their declared type` + (mismatches.length > 0 ? `; mismatches: ${mismatches.join(', ')}` : ''),
243
+ }
244
+ }
package/src/service.ts CHANGED
@@ -11,8 +11,10 @@
11
11
  import { Service, type Context } from '@deepseek-ai/cordis'
12
12
  import type { JsonValue, Session } from '@deepseek-ai/dsh-session'
13
13
  import type { CleanRule, CleanRuleLog } from './clean.ts'
14
+ import type { CleanContractSummary, CleanProfileDiff } from './contract.ts'
14
15
  import type { ProfileReport } from './profile.ts'
15
- import type { VerifyReport, VerifyRule } from './verify.ts'
16
+ import type { VerifyExpectation, VerifyReport, VerifyRule } from './verify.ts'
17
+ import type { ReportRecord, StoredReport } from './store.ts'
16
18
 
17
19
  export interface CitationCheckRequest {
18
20
  /** Workspace-relative path of the source dataset snapshot (CSV/JSON). */
@@ -46,6 +48,8 @@ export interface ProfileRequest {
46
48
  readonly dataset: string
47
49
  /** Optional deterministic systematic sample size for column cards. */
48
50
  readonly sample?: number | undefined
51
+ /** Optional industry preset id; its expected columns feed the scorecard's declared schema. */
52
+ readonly industryPreset?: string | undefined
49
53
  /** Absolute workspace root the dataset resolves inside. */
50
54
  readonly workspace: string
51
55
  /** Calling session (receives the `data-quality/profile` event), when any. */
@@ -62,6 +66,8 @@ export interface CleanRequest {
62
66
  readonly rules: readonly CleanRule[]
63
67
  /** Workspace-relative output path; omitted = no disk write, preview only. */
64
68
  readonly outputPath?: string | undefined
69
+ /** Dry run: no file written, no report persisted; returns the plan + expected contract/diff preview. */
70
+ readonly dryRun?: boolean | undefined
65
71
  /** Absolute workspace root the dataset resolves inside. */
66
72
  readonly workspace: string
67
73
  /** Calling session (receives the `data-quality/clean` event), when any. */
@@ -76,6 +82,8 @@ export interface VerifyRequest {
76
82
  readonly dataset: string
77
83
  /** Non-empty declarative verification rule list. */
78
84
  readonly rules: readonly VerifyRule[]
85
+ /** Optional metric expectations to reconcile (rowCount/columnSum/columnMean/uniqueCount/nullCount). */
86
+ readonly expectations?: readonly VerifyExpectation[] | undefined
79
87
  /** Absolute workspace root the dataset resolves inside. */
80
88
  readonly workspace: string
81
89
  /** Calling session (receives the `data-quality/verify` event), when any. */
@@ -89,9 +97,15 @@ export interface CleanRunReport {
89
97
  readonly dataset: string
90
98
  readonly inputRows: number
91
99
  readonly outputRows: number
100
+ /** Whether this was a dry run (no file written, no report persisted). */
101
+ readonly dryRun: boolean
92
102
  readonly logs: CleanRuleLog[]
103
+ /** Pre-delivery contract validation summary (dedupe/uniqueness/non-null/type/decision regressions). */
104
+ readonly contract: CleanContractSummary
93
105
  /** First `evidenceRowLimit` cleaned rows for inspection (display-truncated). */
94
106
  readonly preview: { readonly columns: string[]; readonly rows: Array<Record<string, JsonValue>> }
107
+ /** Expected before/after profile diff preview; present only for dry runs. */
108
+ readonly diffPreview?: CleanProfileDiff
95
109
  /** Workspace-relative output path when the run wrote a file. */
96
110
  readonly outputPath?: string
97
111
  /** Storage-domain key of the persisted report, when persistence is on. */
@@ -157,6 +171,22 @@ export abstract class DataQualityService extends Service {
157
171
  * @returns the verify report.
158
172
  */
159
173
  abstract verifyDataset(request: VerifyRequest): Promise<VerifyReport>
174
+
175
+ /**
176
+ * Read one persisted report by its storage key. The key is validated
177
+ * against the deterministic report-key format (path-safe) and a missing
178
+ * record fails loud.
179
+ * @param key - the storage report key.
180
+ * @returns the stored report (with its key).
181
+ */
182
+ abstract getReport(key: string): Promise<StoredReport>
183
+
184
+ /**
185
+ * List every persisted report of one kind, ordered by key (chronological).
186
+ * @param kind - the report kind.
187
+ * @returns the stored reports (empty when none match).
188
+ */
189
+ abstract listReports(kind: ReportRecord['kind']): Promise<StoredReport[]>
160
190
  }
161
191
 
162
192
  declare module '@deepseek-ai/cordis' {
@@ -168,5 +198,7 @@ declare module '@deepseek-ai/cordis' {
168
198
 
169
199
  /** Re-exports so consumers pull the whole seam vocabulary from one module. */
170
200
  export type { CleanResult, CleanRule, CleanRuleLog } from './clean.ts'
201
+ export type { CleanContractSummary, CleanProfileDiff } from './contract.ts'
171
202
  export type { ProfileReport } from './profile.ts'
172
- export type { VerifyReport, VerifyRule } from './verify.ts'
203
+ export type { VerifyReport, VerifyRule, VerifyExpectation, VerifyExpectationResult, VerifyMetric } from './verify.ts'
204
+ export type { ReportRecord, StoredReport } from './store.ts'
package/src/store.ts CHANGED
@@ -12,7 +12,7 @@ import { pathFingerprint } from './dataset.ts'
12
12
 
13
13
  /** Zod schema of one persisted report record (durable-boundary validation). */
14
14
  export const reportRecordSchema = z.object({
15
- kind: z.enum(['profile', 'clean', 'verify', 'citations']),
15
+ kind: z.enum(['profile', 'clean', 'verify', 'citations', 'clean-diff']),
16
16
  at: z.number().int().nonnegative(),
17
17
  dataset: z.string(),
18
18
  report: z.record(z.string(), z.unknown()),
@@ -20,7 +20,7 @@ export const reportRecordSchema = z.object({
20
20
 
21
21
  /** One persisted report record. */
22
22
  export interface ReportRecord {
23
- readonly kind: 'profile' | 'clean' | 'verify' | 'citations'
23
+ readonly kind: 'profile' | 'clean' | 'verify' | 'citations' | 'clean-diff'
24
24
  /** Injected run timestamp (epoch ms). */
25
25
  readonly at: number
26
26
  /** Workspace-relative dataset path as the caller gave it. */
@@ -29,6 +29,12 @@ export interface ReportRecord {
29
29
  readonly report: Record<string, unknown>
30
30
  }
31
31
 
32
+ /** A persisted report plus its storage key (the query surface's return value). */
33
+ export interface StoredReport extends ReportRecord {
34
+ /** The storage key the record was written under. */
35
+ readonly key: string
36
+ }
37
+
32
38
  /** The `dsh-data-quality` storage-domain declaration. */
33
39
  export const dataQualityDomainSpec = defineDomain({
34
40
  name: 'data_quality',
@@ -52,6 +58,26 @@ export interface ReportStore {
52
58
  * @returns the record, or `undefined` when absent.
53
59
  */
54
60
  get(key: string): ReportRecord | undefined
61
+ /**
62
+ * Snapshot of every record of one kind, ordered by key (chronological).
63
+ * @param kind - the report kind.
64
+ * @returns the stored reports (empty when none match).
65
+ */
66
+ list(kind: ReportRecord['kind']): StoredReport[]
67
+ }
68
+
69
+ /** Well-formed report-key shape: `<17-digit timestamp>-<kind>-<8-hex fingerprint>`. */
70
+ const REPORT_KEY_PATTERN = /^\d{17}-(?:profile|clean|verify|citations|clean-diff)-[0-9a-f]{8}$/u
71
+
72
+ /**
73
+ * Whether `key` is a well-formed, path-safe storage report key. Rejects any
74
+ * key with separators, traversal, or unexpected characters before it can be
75
+ * handed to the storage backend.
76
+ * @param key - candidate report key.
77
+ * @returns whether the key matches the deterministic report-key format.
78
+ */
79
+ export function isValidReportKey(key: string): boolean {
80
+ return REPORT_KEY_PATTERN.test(key)
55
81
  }
56
82
 
57
83
  /** Pad to two digits for the key timestamp. */