dsh-data-quality 0.1.3 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -0
- package/README.es.md +7 -1
- package/README.hi.md +7 -1
- package/README.md +22 -9
- package/README.pt.md +7 -1
- package/README.zh.md +7 -1
- package/cordis.patch.yml +11 -0
- package/lib/index.js +1609 -145
- package/lib/types/config.d.ts +12 -0
- package/lib/types/config.d.ts.map +1 -1
- package/lib/types/config.js +35 -0
- package/lib/types/config.js.map +1 -1
- package/lib/types/contract.d.ts +83 -0
- package/lib/types/contract.d.ts.map +1 -0
- package/lib/types/contract.js +144 -0
- package/lib/types/contract.js.map +1 -0
- package/lib/types/dataset.d.ts +50 -3
- package/lib/types/dataset.d.ts.map +1 -1
- package/lib/types/dataset.js +76 -21
- package/lib/types/dataset.js.map +1 -1
- package/lib/types/events.d.ts +2 -0
- package/lib/types/events.d.ts.map +1 -1
- package/lib/types/events.js.map +1 -1
- package/lib/types/index.d.ts +13 -8
- package/lib/types/index.d.ts.map +1 -1
- package/lib/types/index.js +17 -6
- package/lib/types/index.js.map +1 -1
- package/lib/types/presets.d.ts +28 -0
- package/lib/types/presets.d.ts.map +1 -0
- package/lib/types/presets.js +134 -0
- package/lib/types/presets.js.map +1 -0
- package/lib/types/profile.d.ts +40 -1
- package/lib/types/profile.d.ts.map +1 -1
- package/lib/types/profile.js +66 -10
- package/lib/types/profile.js.map +1 -1
- package/lib/types/provider-local.d.ts +7 -1
- package/lib/types/provider-local.d.ts.map +1 -1
- package/lib/types/provider-local.js +72 -2
- package/lib/types/provider-local.js.map +1 -1
- package/lib/types/report-html.d.ts +41 -0
- package/lib/types/report-html.d.ts.map +1 -0
- package/lib/types/report-html.js +192 -0
- package/lib/types/report-html.js.map +1 -0
- package/lib/types/scorecard.d.ts +50 -0
- package/lib/types/scorecard.d.ts.map +1 -0
- package/lib/types/scorecard.js +186 -0
- package/lib/types/scorecard.js.map +1 -0
- package/lib/types/service.d.ts +32 -2
- package/lib/types/service.d.ts.map +1 -1
- package/lib/types/service.js.map +1 -1
- package/lib/types/store.d.ts +21 -1
- package/lib/types/store.d.ts.map +1 -1
- package/lib/types/store.js +13 -1
- package/lib/types/store.js.map +1 -1
- package/lib/types/tools/clean.d.ts.map +1 -1
- package/lib/types/tools/clean.js +76 -1
- package/lib/types/tools/clean.js.map +1 -1
- package/lib/types/tools/profile-report-schema.d.ts +319 -0
- package/lib/types/tools/profile-report-schema.d.ts.map +1 -0
- package/lib/types/tools/profile-report-schema.js +96 -0
- package/lib/types/tools/profile-report-schema.js.map +1 -0
- package/lib/types/tools/profile.d.ts.map +1 -1
- package/lib/types/tools/profile.js +6 -53
- package/lib/types/tools/profile.js.map +1 -1
- package/lib/types/tools/report.d.ts +14 -0
- package/lib/types/tools/report.d.ts.map +1 -0
- package/lib/types/tools/report.js +112 -0
- package/lib/types/tools/report.js.map +1 -0
- package/lib/types/tools/shared.d.ts.map +1 -1
- package/lib/types/tools/shared.js +22 -2
- package/lib/types/tools/shared.js.map +1 -1
- package/lib/types/tools/verify.d.ts.map +1 -1
- package/lib/types/tools/verify.js +36 -1
- package/lib/types/tools/verify.js.map +1 -1
- package/lib/types/verify.d.ts +49 -3
- package/lib/types/verify.d.ts.map +1 -1
- package/lib/types/verify.js +111 -4
- package/lib/types/verify.js.map +1 -1
- package/lib/types/version.d.ts +8 -1
- package/lib/types/version.d.ts.map +1 -1
- package/lib/types/version.js +8 -1
- package/lib/types/version.js.map +1 -1
- package/package.json +1 -1
- package/src/config.ts +52 -0
- package/src/contract.ts +190 -0
- package/src/dataset.ts +101 -21
- package/src/events.ts +2 -0
- package/src/index.ts +27 -8
- package/src/presets.ts +146 -0
- package/src/profile.ts +105 -11
- package/src/provider-local.ts +77 -3
- package/src/report-html.ts +208 -0
- package/src/scorecard.ts +244 -0
- package/src/service.ts +34 -2
- package/src/store.ts +28 -2
- package/src/tools/clean.ts +76 -1
- package/src/tools/profile-report-schema.ts +98 -0
- package/src/tools/profile.ts +8 -56
- package/src/tools/report.ts +137 -0
- package/src/tools/shared.ts +21 -2
- package/src/tools/verify.ts +40 -2
- package/src/verify.ts +151 -5
- package/src/version.ts +9 -1
package/src/tools/clean.ts
CHANGED
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
|
|
8
8
|
import { defineTool } from '@deepseek-ai/dsh-tools'
|
|
9
9
|
import type { CleanRule, CleanRunReport, DataQualityService } from '../service.ts'
|
|
10
|
+
import { PROFILE_REPORT_SCHEMA } from './profile-report-schema.ts'
|
|
10
11
|
import { renderCleanText, workspaceOf } from './shared.ts'
|
|
11
12
|
|
|
12
13
|
const STRING_MAP_SCHEMA = { type: 'json' } as const
|
|
@@ -91,12 +92,13 @@ export function defineCleanTool(service: DataQualityService) {
|
|
|
91
92
|
description: [
|
|
92
93
|
'Apply declarative cleaning rules to a workspace CSV/TSV/JSON/JSONL dataset with deterministic TypeScript computation (no mental math).',
|
|
93
94
|
'Rules apply in array order: dedupe (by column group), fill-missing (constant/mean/median/forward), coerce-type (number/date/boolean; failures counted and set to missing), normalize-unit (e.g. 万/亿 suffixes to base units), trim (whitespace), map-values (enum mapping).',
|
|
94
|
-
'The source file is NEVER overwritten. Without outputPath the run is preview-only; with outputPath the cleaned dataset is written there (workspace-confined, .csv/.tsv/.json/.jsonl). Returns the per-rule audit log (
|
|
95
|
+
'The source file is NEVER overwritten. Without outputPath the run is preview-only; with outputPath the cleaned dataset is written there (workspace-confined, .csv/.tsv/.json/.jsonl). Returns the per-rule audit log, the pre-delivery contract summary (with per-column decision trace), and a bounded preview. Pass dryRun: true to skip the write and get the cleaning plan plus the expected contract/diff preview instead. The full report persists to the data_quality storage domain (reportKey).',
|
|
95
96
|
].join('\n'),
|
|
96
97
|
parameters: {
|
|
97
98
|
path: { type: 'string', required: true, description: 'Workspace-relative dataset path (.csv/.tsv/.json/.jsonl).' },
|
|
98
99
|
rules: { ...CLEAN_RULE_SCHEMA, required: true },
|
|
99
100
|
outputPath: { type: 'string', description: 'Optional workspace-relative output path for the cleaned dataset (must differ from path).' },
|
|
101
|
+
dryRun: { type: 'boolean', description: 'When true, do not write any output file; return the cleaning plan and expected contract/diff preview instead (default false).' },
|
|
100
102
|
},
|
|
101
103
|
output: {
|
|
102
104
|
schema: {
|
|
@@ -105,9 +107,20 @@ export function defineCleanTool(service: DataQualityService) {
|
|
|
105
107
|
dataset: { type: 'string', required: true },
|
|
106
108
|
inputRows: { type: 'number', required: true },
|
|
107
109
|
outputRows: { type: 'number', required: true },
|
|
110
|
+
dryRun: { type: 'boolean', required: true },
|
|
108
111
|
generatedAt: { type: 'number', required: true },
|
|
109
112
|
outputPath: { type: 'string' },
|
|
110
113
|
reportKey: { type: 'string' },
|
|
114
|
+
diffPreview: {
|
|
115
|
+
type: 'object',
|
|
116
|
+
properties: {
|
|
117
|
+
dataset: { type: 'string', required: true },
|
|
118
|
+
before: { ...PROFILE_REPORT_SCHEMA, required: true },
|
|
119
|
+
after: { ...PROFILE_REPORT_SCHEMA, required: true },
|
|
120
|
+
generatedAt: { type: 'number', required: true },
|
|
121
|
+
},
|
|
122
|
+
additionalProperties: false,
|
|
123
|
+
},
|
|
111
124
|
logs: {
|
|
112
125
|
type: 'array',
|
|
113
126
|
items: {
|
|
@@ -122,6 +135,67 @@ export function defineCleanTool(service: DataQualityService) {
|
|
|
122
135
|
},
|
|
123
136
|
required: true,
|
|
124
137
|
},
|
|
138
|
+
contract: {
|
|
139
|
+
type: 'object',
|
|
140
|
+
properties: {
|
|
141
|
+
inputRows: { type: 'number', required: true },
|
|
142
|
+
outputRows: { type: 'number', required: true },
|
|
143
|
+
removedRows: { type: 'number', required: true },
|
|
144
|
+
dedupeColumns: { oneOf: [{ type: 'array', items: { type: 'string' } }, { type: 'null' }], required: true },
|
|
145
|
+
uniqueKeys: { type: 'boolean', required: true },
|
|
146
|
+
remainingDuplicateRows: { type: 'number', required: true },
|
|
147
|
+
remainingMissing: {
|
|
148
|
+
type: 'array',
|
|
149
|
+
items: {
|
|
150
|
+
type: 'object',
|
|
151
|
+
properties: {
|
|
152
|
+
column: { type: 'string', required: true },
|
|
153
|
+
count: { type: 'number', required: true },
|
|
154
|
+
},
|
|
155
|
+
additionalProperties: false,
|
|
156
|
+
},
|
|
157
|
+
required: true,
|
|
158
|
+
},
|
|
159
|
+
typeConformance: {
|
|
160
|
+
type: 'array',
|
|
161
|
+
items: {
|
|
162
|
+
type: 'object',
|
|
163
|
+
properties: {
|
|
164
|
+
column: { type: 'string', required: true },
|
|
165
|
+
to: { type: 'string', enum: ['number', 'date', 'boolean'], required: true },
|
|
166
|
+
invalidCount: { type: 'number', required: true },
|
|
167
|
+
},
|
|
168
|
+
additionalProperties: false,
|
|
169
|
+
},
|
|
170
|
+
required: true,
|
|
171
|
+
},
|
|
172
|
+
columnDecisions: {
|
|
173
|
+
type: 'array',
|
|
174
|
+
items: {
|
|
175
|
+
type: 'object',
|
|
176
|
+
properties: {
|
|
177
|
+
column: { type: 'string', required: true },
|
|
178
|
+
decisions: {
|
|
179
|
+
type: 'array',
|
|
180
|
+
items: {
|
|
181
|
+
type: 'object',
|
|
182
|
+
properties: {
|
|
183
|
+
strategy: { type: 'string', required: true },
|
|
184
|
+
affectedRows: { type: 'number', required: true },
|
|
185
|
+
},
|
|
186
|
+
additionalProperties: false,
|
|
187
|
+
},
|
|
188
|
+
required: true,
|
|
189
|
+
},
|
|
190
|
+
},
|
|
191
|
+
additionalProperties: false,
|
|
192
|
+
},
|
|
193
|
+
required: true,
|
|
194
|
+
},
|
|
195
|
+
},
|
|
196
|
+
additionalProperties: false,
|
|
197
|
+
required: true,
|
|
198
|
+
},
|
|
125
199
|
preview: {
|
|
126
200
|
type: 'object',
|
|
127
201
|
properties: {
|
|
@@ -141,6 +215,7 @@ export function defineCleanTool(service: DataQualityService) {
|
|
|
141
215
|
dataset: args.path,
|
|
142
216
|
rules: args.rules as unknown as readonly CleanRule[],
|
|
143
217
|
...(args.outputPath !== undefined ? { outputPath: args.outputPath } : {}),
|
|
218
|
+
...(args.dryRun !== undefined ? { dryRun: args.dryRun } : {}),
|
|
144
219
|
workspace: workspaceOf(exec),
|
|
145
220
|
session: exec.agent?.session,
|
|
146
221
|
signal: exec.signal,
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared JSON-Schema spec for the profile report's canonical value, reused by
|
|
3
|
+
* `data_profile` (its output) and `data_clean` (the `diffPreview` before/after
|
|
4
|
+
* snapshots). One source of truth so the two tools never drift.
|
|
5
|
+
* @module dsh-data-quality/tools-profile-report-schema
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
const SCORECARD_DIMENSION_NAMES = ['completeness', 'uniqueness', 'validity', 'consistency', 'timeliness', 'accuracy'] as const
|
|
9
|
+
|
|
10
|
+
/** One column card's schema. */
|
|
11
|
+
export const COLUMN_PROFILE_SCHEMA = {
|
|
12
|
+
type: 'object',
|
|
13
|
+
properties: {
|
|
14
|
+
name: { type: 'string', required: true },
|
|
15
|
+
inferredType: { type: 'string', enum: ['number', 'date', 'boolean', 'string', 'empty', 'mixed'], required: true },
|
|
16
|
+
missing: { type: 'number', required: true },
|
|
17
|
+
missingRate: { type: 'number', required: true },
|
|
18
|
+
unique: { type: 'number', required: true },
|
|
19
|
+
numeric: {
|
|
20
|
+
type: 'object',
|
|
21
|
+
properties: {
|
|
22
|
+
count: { type: 'number', required: true },
|
|
23
|
+
distinct: { type: 'number', required: true },
|
|
24
|
+
min: { type: 'number', required: true },
|
|
25
|
+
max: { type: 'number', required: true },
|
|
26
|
+
mean: { type: 'number', required: true },
|
|
27
|
+
median: { type: 'number', required: true },
|
|
28
|
+
p25: { type: 'number', required: true },
|
|
29
|
+
p75: { type: 'number', required: true },
|
|
30
|
+
outliers: { type: 'number', required: true },
|
|
31
|
+
},
|
|
32
|
+
additionalProperties: false,
|
|
33
|
+
},
|
|
34
|
+
topValues: {
|
|
35
|
+
type: 'array',
|
|
36
|
+
items: {
|
|
37
|
+
type: 'object',
|
|
38
|
+
properties: {
|
|
39
|
+
value: { type: 'string', required: true },
|
|
40
|
+
count: { type: 'number', required: true },
|
|
41
|
+
},
|
|
42
|
+
additionalProperties: false,
|
|
43
|
+
},
|
|
44
|
+
},
|
|
45
|
+
notes: { type: 'array', items: { type: 'string' }, required: true },
|
|
46
|
+
},
|
|
47
|
+
additionalProperties: false,
|
|
48
|
+
} as const
|
|
49
|
+
|
|
50
|
+
/** The full profile report's schema. */
|
|
51
|
+
export const PROFILE_REPORT_SCHEMA = {
|
|
52
|
+
type: 'object',
|
|
53
|
+
properties: {
|
|
54
|
+
schemaVersion: { type: 'number', required: true },
|
|
55
|
+
dataset: { type: 'string', required: true },
|
|
56
|
+
rowCount: { type: 'number', required: true },
|
|
57
|
+
sampled: { type: 'boolean', required: true },
|
|
58
|
+
profiledRows: { type: 'number', required: true },
|
|
59
|
+
columnCount: { type: 'number', required: true },
|
|
60
|
+
duplicateRows: { type: 'number', required: true },
|
|
61
|
+
duplicateRate: { type: 'number', required: true },
|
|
62
|
+
duplicateSampleRowIndexes: { type: 'array', items: { type: 'number' }, required: true },
|
|
63
|
+
scorecard: {
|
|
64
|
+
type: 'object',
|
|
65
|
+
properties: {
|
|
66
|
+
overall: { oneOf: [{ type: 'number' }, { type: 'null' }], required: true },
|
|
67
|
+
weightedOverall: { oneOf: [{ type: 'number' }, { type: 'null' }], required: true },
|
|
68
|
+
dimensions: {
|
|
69
|
+
type: 'array',
|
|
70
|
+
items: {
|
|
71
|
+
type: 'object',
|
|
72
|
+
properties: {
|
|
73
|
+
name: { type: 'string', enum: [...SCORECARD_DIMENSION_NAMES], required: true },
|
|
74
|
+
score: { oneOf: [{ type: 'number' }, { type: 'null' }], required: true },
|
|
75
|
+
note: { type: 'string', required: true },
|
|
76
|
+
},
|
|
77
|
+
additionalProperties: false,
|
|
78
|
+
},
|
|
79
|
+
required: true,
|
|
80
|
+
},
|
|
81
|
+
},
|
|
82
|
+
additionalProperties: false,
|
|
83
|
+
required: true,
|
|
84
|
+
},
|
|
85
|
+
encoding: {
|
|
86
|
+
type: 'object',
|
|
87
|
+
properties: {
|
|
88
|
+
bom: { oneOf: [{ type: 'string' }, { type: 'null' }], required: true },
|
|
89
|
+
validUtf8: { type: 'boolean', required: true },
|
|
90
|
+
},
|
|
91
|
+
additionalProperties: false,
|
|
92
|
+
},
|
|
93
|
+
generatedAt: { type: 'number', required: true },
|
|
94
|
+
reportKey: { type: 'string' },
|
|
95
|
+
columns: { type: 'array', items: COLUMN_PROFILE_SCHEMA, required: true },
|
|
96
|
+
},
|
|
97
|
+
additionalProperties: false,
|
|
98
|
+
} as const
|
package/src/tools/profile.ts
CHANGED
|
@@ -6,46 +6,10 @@
|
|
|
6
6
|
|
|
7
7
|
import { defineTool } from '@deepseek-ai/dsh-tools'
|
|
8
8
|
import type { DataQualityService } from '../service.ts'
|
|
9
|
-
import { renderProfileText } from '../profile.ts'
|
|
9
|
+
import { renderProfileText, type ProfileReport } from '../profile.ts'
|
|
10
|
+
import { PROFILE_REPORT_SCHEMA } from './profile-report-schema.ts'
|
|
10
11
|
import { workspaceOf } from './shared.ts'
|
|
11
12
|
|
|
12
|
-
const COLUMN_PROFILE_SCHEMA = {
|
|
13
|
-
type: 'object',
|
|
14
|
-
properties: {
|
|
15
|
-
name: { type: 'string', required: true },
|
|
16
|
-
inferredType: { type: 'string', enum: ['number', 'date', 'boolean', 'string', 'empty', 'mixed'], required: true },
|
|
17
|
-
missing: { type: 'number', required: true },
|
|
18
|
-
missingRate: { type: 'number', required: true },
|
|
19
|
-
unique: { type: 'number', required: true },
|
|
20
|
-
numeric: {
|
|
21
|
-
type: 'object',
|
|
22
|
-
properties: {
|
|
23
|
-
min: { type: 'number', required: true },
|
|
24
|
-
max: { type: 'number', required: true },
|
|
25
|
-
mean: { type: 'number', required: true },
|
|
26
|
-
median: { type: 'number', required: true },
|
|
27
|
-
p25: { type: 'number', required: true },
|
|
28
|
-
p75: { type: 'number', required: true },
|
|
29
|
-
outliers: { type: 'number', required: true },
|
|
30
|
-
},
|
|
31
|
-
additionalProperties: false,
|
|
32
|
-
},
|
|
33
|
-
topValues: {
|
|
34
|
-
type: 'array',
|
|
35
|
-
items: {
|
|
36
|
-
type: 'object',
|
|
37
|
-
properties: {
|
|
38
|
-
value: { type: 'string', required: true },
|
|
39
|
-
count: { type: 'number', required: true },
|
|
40
|
-
},
|
|
41
|
-
additionalProperties: false,
|
|
42
|
-
},
|
|
43
|
-
},
|
|
44
|
-
notes: { type: 'array', items: { type: 'string' }, required: true },
|
|
45
|
-
},
|
|
46
|
-
additionalProperties: false,
|
|
47
|
-
} as const
|
|
48
|
-
|
|
49
13
|
/**
|
|
50
14
|
* Build the `data_profile` tool definition against a mounted service.
|
|
51
15
|
* @param service - the mounted ctx.dataQuality implementation.
|
|
@@ -56,35 +20,23 @@ export function defineProfileTool(service: DataQualityService) {
|
|
|
56
20
|
name: 'data_profile',
|
|
57
21
|
description: [
|
|
58
22
|
'Profile a workspace CSV/TSV/JSON/JSONL dataset with deterministic TypeScript computation (no mental math).',
|
|
59
|
-
'Returns row/column counts, inferred column types, missing rates, unique counts, numeric distributions (min/max/mean/median/p25/p75), IQR outlier counts, mixed-type suspicion notes,
|
|
60
|
-
'Column cards cover every row by default; pass sample for a deterministic systematic sample on large files. Datasets above the configured row/size caps are rejected — use sample or raise the caps. The full report persists to the data_quality storage domain (reportKey in the result).',
|
|
23
|
+
'Returns row/column counts, inferred column types, missing rates, unique counts, numeric distributions (count/distinct/min/max/mean/median/p25/p75), IQR outlier counts, mixed-type suspicion notes, sha256 duplicate-row detection (rate + sample indexes), file encoding (BOM/UTF-8 validity), and a weighted DAMA six-dimension scorecard.',
|
|
24
|
+
'Pass industryPreset (retail/saas/fund/real-estate/e-commerce/healthcare/logistics/manufacturing/energy) to compare the dataset against that industry\'s expected columns, making the scorecard accuracy dimension determinable. Column cards cover every row by default; pass sample for a deterministic systematic sample on large files. Datasets above the configured row/size caps are rejected — use sample or raise the caps. The full report persists to the data_quality storage domain (reportKey in the result).',
|
|
61
25
|
].join('\n'),
|
|
62
26
|
parameters: {
|
|
63
27
|
path: { type: 'string', required: true, description: 'Workspace-relative dataset path (.csv/.tsv/.json/.jsonl). JSON datasets must be an array of flat objects.' },
|
|
64
28
|
sample: { type: 'number', description: 'Optional systematic sample size (every ceil(N/sample)-th row) for the column cards; row counts stay exact.' },
|
|
29
|
+
industryPreset: { type: 'string', enum: ['retail', 'saas', 'fund', 'real-estate', 'e-commerce', 'healthcare', 'logistics', 'manufacturing', 'energy'], description: 'Optional industry preset id; its expected columns feed the scorecard accuracy dimension.' },
|
|
65
30
|
},
|
|
66
31
|
output: {
|
|
67
|
-
schema:
|
|
68
|
-
|
|
69
|
-
properties: {
|
|
70
|
-
dataset: { type: 'string', required: true },
|
|
71
|
-
rowCount: { type: 'number', required: true },
|
|
72
|
-
sampled: { type: 'boolean', required: true },
|
|
73
|
-
profiledRows: { type: 'number', required: true },
|
|
74
|
-
columnCount: { type: 'number', required: true },
|
|
75
|
-
duplicateRows: { type: 'number', required: true },
|
|
76
|
-
generatedAt: { type: 'number', required: true },
|
|
77
|
-
reportKey: { type: 'string' },
|
|
78
|
-
columns: { type: 'array', items: COLUMN_PROFILE_SCHEMA, required: true },
|
|
79
|
-
},
|
|
80
|
-
additionalProperties: false,
|
|
81
|
-
},
|
|
82
|
-
render: (_args, value) => [{ type: 'text', text: renderProfileText(value) }],
|
|
32
|
+
schema: PROFILE_REPORT_SCHEMA,
|
|
33
|
+
render: (_args, value) => [{ type: 'text', text: renderProfileText(value as unknown as ProfileReport) }],
|
|
83
34
|
},
|
|
84
35
|
async execute(args, exec) {
|
|
85
36
|
return service.profileDataset({
|
|
86
37
|
dataset: args.path,
|
|
87
38
|
sample: args.sample,
|
|
39
|
+
...(args.industryPreset !== undefined ? { industryPreset: args.industryPreset } : {}),
|
|
88
40
|
workspace: workspaceOf(exec),
|
|
89
41
|
session: exec.agent?.session,
|
|
90
42
|
signal: exec.signal,
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The `data_report` model tool: read persisted profile/clean-diff reports
|
|
3
|
+
* back from the `data_quality` storage domain, by exact `reportKey` or by
|
|
4
|
+
* `kind`. Deterministic read-only consumption — no model arithmetic.
|
|
5
|
+
* @module dsh-data-quality/tools/report
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import { defineTool } from '@deepseek-ai/dsh-tools'
|
|
9
|
+
import type { JsonValue } from '@deepseek-ai/dsh-session'
|
|
10
|
+
import type { ProfileReport } from '../profile.ts'
|
|
11
|
+
import { renderCleanHtml, renderProfileHtml, type CleanReportHtml } from '../report-html.ts'
|
|
12
|
+
import type { DataQualityService } from '../service.ts'
|
|
13
|
+
import type { ReportRecord, StoredReport } from '../store.ts'
|
|
14
|
+
|
|
15
|
+
/** The report kinds a caller may query. */
|
|
16
|
+
const REPORT_KINDS = ['profile', 'clean', 'clean-diff', 'verify', 'citations'] as const
|
|
17
|
+
|
|
18
|
+
/** One stored report projected into the model-facing canonical value (report as lossless JSON). */
|
|
19
|
+
interface ReportView {
|
|
20
|
+
readonly key: string
|
|
21
|
+
readonly kind: ReportRecord['kind']
|
|
22
|
+
readonly at: number
|
|
23
|
+
readonly dataset: string
|
|
24
|
+
readonly report: JsonValue
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/** The canonical value the tool returns. */
|
|
28
|
+
interface DataReportValue {
|
|
29
|
+
readonly key?: string
|
|
30
|
+
readonly kind?: ReportRecord['kind']
|
|
31
|
+
readonly records: ReportView[]
|
|
32
|
+
/** Self-contained offline HTML (present only when `format: html`). */
|
|
33
|
+
readonly html?: string
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** Render one stored report as a self-contained HTML document (profile/clean only). */
|
|
37
|
+
function renderRecordHtml(record: StoredReport): string {
|
|
38
|
+
if (record.kind === 'profile') {
|
|
39
|
+
return renderProfileHtml(record.report as unknown as ProfileReport)
|
|
40
|
+
}
|
|
41
|
+
if (record.kind === 'clean' || record.kind === 'clean-diff') {
|
|
42
|
+
return renderCleanHtml(record.report as unknown as CleanReportHtml, record.dataset)
|
|
43
|
+
}
|
|
44
|
+
throw new Error(`data_report html format does not support kind "${record.kind}" (profile/clean only)`)
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** Project a stored report into the canonical value (the stored report is already lossless JSON). */
|
|
48
|
+
function toView(record: StoredReport): ReportView {
|
|
49
|
+
return { key: record.key, kind: record.kind, at: record.at, dataset: record.dataset, report: record.report as unknown as JsonValue }
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** Human-readable report summary for the tool's Native render. */
|
|
53
|
+
function renderReportText(value: DataReportValue): string {
|
|
54
|
+
const lines: string[] = []
|
|
55
|
+
if (value.key !== undefined) {
|
|
56
|
+
lines.push(value.records.length === 0 ? `No report for ${value.key}` : `Report ${value.key}`)
|
|
57
|
+
} else {
|
|
58
|
+
lines.push(`Reports of kind ${value.kind ?? ''} (${value.records.length})`)
|
|
59
|
+
}
|
|
60
|
+
for (const record of value.records) {
|
|
61
|
+
lines.push(`- ${record.key} [${record.kind}] ${record.dataset} @ ${new Date(record.at).toISOString()}`)
|
|
62
|
+
}
|
|
63
|
+
return lines.join('\n')
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Build the `data_report` tool definition against a mounted service.
|
|
68
|
+
* @param service - the mounted ctx.dataQuality implementation.
|
|
69
|
+
* @returns the tool definition to register.
|
|
70
|
+
*/
|
|
71
|
+
export function defineReportTool(service: DataQualityService) {
|
|
72
|
+
return defineTool({
|
|
73
|
+
name: 'data_report',
|
|
74
|
+
description: [
|
|
75
|
+
'Read persisted data-quality reports back from the data_quality storage domain (deterministic, read-only).',
|
|
76
|
+
'Pass key (the exact reportKey a prior run returned) to fetch one report, or kind to list every persisted report of that kind, ordered chronologically. Exactly one of key/kind.',
|
|
77
|
+
'Returns the report envelope(s): kind, dataset, timestamp, and the full stored report (profile/clean/clean-diff/verify/citations). Missing keys and unknown kinds fail loudly.',
|
|
78
|
+
].join('\n'),
|
|
79
|
+
parameters: {
|
|
80
|
+
key: { type: 'string', description: 'Exact storage reportKey (e.g. 20260819000000000-profile-1a2b3c4d); fetches that one report.' },
|
|
81
|
+
kind: { type: 'string', enum: [...REPORT_KINDS], description: 'Report kind to list (profile/clean/clean-diff/verify/citations).' },
|
|
82
|
+
format: { type: 'string', enum: ['json', 'html'], description: 'Output format. json (default) returns the report envelope(s); html renders one report as a self-contained offline HTML document (requires key; profile/clean only).' },
|
|
83
|
+
},
|
|
84
|
+
output: {
|
|
85
|
+
schema: {
|
|
86
|
+
type: 'object',
|
|
87
|
+
properties: {
|
|
88
|
+
key: { type: 'string' },
|
|
89
|
+
kind: { type: 'string', enum: [...REPORT_KINDS] },
|
|
90
|
+
records: {
|
|
91
|
+
type: 'array',
|
|
92
|
+
items: {
|
|
93
|
+
type: 'object',
|
|
94
|
+
properties: {
|
|
95
|
+
key: { type: 'string', required: true },
|
|
96
|
+
kind: { type: 'string', enum: [...REPORT_KINDS], required: true },
|
|
97
|
+
at: { type: 'number', required: true },
|
|
98
|
+
dataset: { type: 'string', required: true },
|
|
99
|
+
report: { type: 'json', required: true },
|
|
100
|
+
},
|
|
101
|
+
additionalProperties: false,
|
|
102
|
+
},
|
|
103
|
+
required: true,
|
|
104
|
+
},
|
|
105
|
+
html: { type: 'string', description: 'Self-contained offline HTML (present only when format: html).' },
|
|
106
|
+
},
|
|
107
|
+
additionalProperties: false,
|
|
108
|
+
},
|
|
109
|
+
render: (_args, value) => {
|
|
110
|
+
const view = value as unknown as DataReportValue
|
|
111
|
+
if (view.html !== undefined) return [{ type: 'text', text: view.html }]
|
|
112
|
+
return [{ type: 'text', text: renderReportText(view) }]
|
|
113
|
+
},
|
|
114
|
+
},
|
|
115
|
+
async execute(args, _exec): Promise<DataReportValue> {
|
|
116
|
+
const hasKey = args.key !== undefined
|
|
117
|
+
const hasKind = args.kind !== undefined
|
|
118
|
+
if (hasKey === hasKind) {
|
|
119
|
+
throw new Error('data_report needs exactly one of key/kind')
|
|
120
|
+
}
|
|
121
|
+
const format = (args.format ?? 'json') as 'json' | 'html'
|
|
122
|
+
if (hasKey) {
|
|
123
|
+
const record = await service.getReport(args.key as string)
|
|
124
|
+
return {
|
|
125
|
+
key: args.key as string,
|
|
126
|
+
records: [toView(record)],
|
|
127
|
+
...(format === 'html' ? { html: renderRecordHtml(record) } : {}),
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
if (format === 'html') {
|
|
131
|
+
throw new Error('data_report html format requires key (exactly one report)')
|
|
132
|
+
}
|
|
133
|
+
const records = await service.listReports(args.kind as ReportRecord['kind'])
|
|
134
|
+
return { kind: args.kind as ReportRecord['kind'], records: records.map(toView) }
|
|
135
|
+
},
|
|
136
|
+
})
|
|
137
|
+
}
|
package/src/tools/shared.ts
CHANGED
|
@@ -27,11 +27,30 @@ export function workspaceOf(exec: ToolExecution): string {
|
|
|
27
27
|
/** Human-readable cleaning summary for the tool's Native render. */
|
|
28
28
|
export function renderCleanText(report: CleanRunReport): string {
|
|
29
29
|
const lines: string[] = []
|
|
30
|
-
|
|
30
|
+
const mode = report.dryRun ? 'Dry-run plan for' : 'Cleaned'
|
|
31
|
+
lines.push(`${mode} ${report.dataset}: ${report.inputRows} -> ${report.outputRows} rows over ${report.logs.length} rule(s)`)
|
|
31
32
|
for (const log of report.logs) {
|
|
32
33
|
lines.push(`- rule ${log.ruleIndex} (${log.rule}): ${log.affectedRows} row(s) affected; ${log.detail}`)
|
|
33
34
|
}
|
|
34
|
-
|
|
35
|
+
const contract = report.contract
|
|
36
|
+
const keyLabel = contract.dedupeColumns === null ? 'full rows' : `[${contract.dedupeColumns.join(', ')}]`
|
|
37
|
+
lines.push(`Contract: ${contract.inputRows} -> ${contract.outputRows} rows (${contract.removedRows} removed); uniqueness ${contract.uniqueKeys ? 'OK' : 'VIOLATED'} over ${keyLabel}${contract.remainingDuplicateRows > 0 ? ` (${contract.remainingDuplicateRows} duplicate row(s) remain)` : ''}`)
|
|
38
|
+
for (const entry of contract.remainingMissing) {
|
|
39
|
+
lines.push(`- non-null regression: ${entry.column} still has ${entry.count} missing cell(s)`)
|
|
40
|
+
}
|
|
41
|
+
for (const entry of contract.typeConformance) {
|
|
42
|
+
lines.push(`- type regression: ${entry.column} (${entry.to}) has ${entry.invalidCount} non-conforming cell(s)`)
|
|
43
|
+
}
|
|
44
|
+
for (const entry of contract.columnDecisions) {
|
|
45
|
+
const decisions = entry.decisions.map((decision) => `${decision.strategy} (${decision.affectedRows} row(s))`).join(', ')
|
|
46
|
+
lines.push(`- column ${entry.column}: ${decisions}`)
|
|
47
|
+
}
|
|
48
|
+
if (report.diffPreview !== undefined) {
|
|
49
|
+
lines.push(`Diff preview: ${report.diffPreview.before.rowCount} -> ${report.diffPreview.after.rowCount} rows; duplicates ${report.diffPreview.before.duplicateRows} -> ${report.diffPreview.after.duplicateRows}`)
|
|
50
|
+
}
|
|
51
|
+
if (report.dryRun) {
|
|
52
|
+
lines.push('Dry run: no output file written and no report persisted; plan/preview above.')
|
|
53
|
+
} else if (report.outputPath !== undefined) {
|
|
35
54
|
lines.push(`Wrote cleaned dataset to ${report.outputPath}`)
|
|
36
55
|
} else {
|
|
37
56
|
lines.push('No outputPath given: the source file was left untouched; preview below.')
|
package/src/tools/verify.ts
CHANGED
|
@@ -6,10 +6,44 @@
|
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
8
|
import { defineTool } from '@deepseek-ai/dsh-tools'
|
|
9
|
-
import type { DataQualityService, VerifyRule } from '../service.ts'
|
|
9
|
+
import type { DataQualityService, VerifyExpectation, VerifyRule } from '../service.ts'
|
|
10
10
|
import { renderVerifyText, type VerifyReport } from '../verify.ts'
|
|
11
11
|
import { workspaceOf } from './shared.ts'
|
|
12
12
|
|
|
13
|
+
const EXPECTATION_METRICS = ['rowCount', 'columnSum', 'columnMean', 'uniqueCount', 'nullCount'] as const
|
|
14
|
+
|
|
15
|
+
const EXPECTATION_SCHEMA = {
|
|
16
|
+
type: 'array',
|
|
17
|
+
items: {
|
|
18
|
+
type: 'object',
|
|
19
|
+
properties: {
|
|
20
|
+
metric: { type: 'string', enum: [...EXPECTATION_METRICS], required: true, description: 'Metric to reconcile: rowCount/columnSum/columnMean/uniqueCount/nullCount.' },
|
|
21
|
+
column: { type: 'string', description: 'Required for every metric except rowCount.' },
|
|
22
|
+
expected: { type: 'number', required: true, description: 'The expected value to reconcile against.' },
|
|
23
|
+
tolerance: { type: 'number', description: 'Optional relative tolerance in [0, 1]; defaults to defaultTolerance.' },
|
|
24
|
+
},
|
|
25
|
+
additionalProperties: false,
|
|
26
|
+
description: 'Reconcile a deterministic computed metric against an expected value with relative tolerance.',
|
|
27
|
+
},
|
|
28
|
+
description: 'Optional metric expectations; each yields passed true or passed false with actual/expected/tolerance detail.',
|
|
29
|
+
} as const
|
|
30
|
+
|
|
31
|
+
const EXPECTATION_RESULT_SCHEMA = {
|
|
32
|
+
type: 'array',
|
|
33
|
+
items: {
|
|
34
|
+
type: 'object',
|
|
35
|
+
properties: {
|
|
36
|
+
metric: { type: 'string', enum: [...EXPECTATION_METRICS], required: true },
|
|
37
|
+
column: { type: 'string' },
|
|
38
|
+
expected: { type: 'number', required: true },
|
|
39
|
+
actual: { type: 'number', required: true },
|
|
40
|
+
tolerance: { type: 'number', required: true },
|
|
41
|
+
passed: { type: 'boolean', required: true },
|
|
42
|
+
},
|
|
43
|
+
additionalProperties: false,
|
|
44
|
+
},
|
|
45
|
+
} as const
|
|
46
|
+
|
|
13
47
|
const VERIFY_RULE_SCHEMA = {
|
|
14
48
|
type: 'array',
|
|
15
49
|
items: {
|
|
@@ -103,11 +137,13 @@ export function defineVerifyTool(service: DataQualityService) {
|
|
|
103
137
|
description: [
|
|
104
138
|
'Verify a workspace CSV/TSV/JSON/JSONL dataset against declarative quality rules with deterministic TypeScript computation (no mental math).',
|
|
105
139
|
'Rules: not-null, unique (column group), range (numeric bounds), regex, enum, cross-column (e.g. startDate < endDate), freshness (date column within N days of asOf). A missing cell fails every rule that reads it.',
|
|
106
|
-
'
|
|
140
|
+
'Optional expectations reconcile deterministic metrics (rowCount/columnSum/columnMean/uniqueCount/nullCount) against expected values with relative tolerance; a mismatch is a normal passed: false with actual/expected/tolerance detail, never a tool error.',
|
|
141
|
+
'Returns per-rule pass/fail with capped failing-row evidence plus the expectation outcomes. Overall failure is a NORMAL result with passed: false — not a tool error. The full report persists to the data_quality storage domain (reportKey).',
|
|
107
142
|
].join('\n'),
|
|
108
143
|
parameters: {
|
|
109
144
|
path: { type: 'string', required: true, description: 'Workspace-relative dataset path (.csv/.tsv/.json/.jsonl).' },
|
|
110
145
|
rules: { ...VERIFY_RULE_SCHEMA, required: true },
|
|
146
|
+
expectations: { ...EXPECTATION_SCHEMA, description: 'Optional metric expectations to reconcile (rowCount/columnSum/columnMean/uniqueCount/nullCount).' },
|
|
111
147
|
},
|
|
112
148
|
output: {
|
|
113
149
|
schema: {
|
|
@@ -145,6 +181,7 @@ export function defineVerifyTool(service: DataQualityService) {
|
|
|
145
181
|
},
|
|
146
182
|
required: true,
|
|
147
183
|
},
|
|
184
|
+
expectations: { ...EXPECTATION_RESULT_SCHEMA, required: true },
|
|
148
185
|
},
|
|
149
186
|
additionalProperties: false,
|
|
150
187
|
},
|
|
@@ -154,6 +191,7 @@ export function defineVerifyTool(service: DataQualityService) {
|
|
|
154
191
|
return service.verifyDataset({
|
|
155
192
|
dataset: args.path,
|
|
156
193
|
rules: args.rules as unknown as readonly VerifyRule[],
|
|
194
|
+
...(args.expectations !== undefined ? { expectations: args.expectations as unknown as readonly VerifyExpectation[] } : {}),
|
|
157
195
|
workspace: workspaceOf(exec),
|
|
158
196
|
session: exec.agent?.session,
|
|
159
197
|
signal: exec.signal,
|