dsh-data-quality 0.1.3 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/CHANGELOG.md +33 -0
  2. package/README.es.md +7 -1
  3. package/README.hi.md +7 -1
  4. package/README.md +22 -9
  5. package/README.pt.md +7 -1
  6. package/README.zh.md +7 -1
  7. package/cordis.patch.yml +11 -0
  8. package/lib/index.js +1609 -145
  9. package/lib/types/config.d.ts +12 -0
  10. package/lib/types/config.d.ts.map +1 -1
  11. package/lib/types/config.js +35 -0
  12. package/lib/types/config.js.map +1 -1
  13. package/lib/types/contract.d.ts +83 -0
  14. package/lib/types/contract.d.ts.map +1 -0
  15. package/lib/types/contract.js +144 -0
  16. package/lib/types/contract.js.map +1 -0
  17. package/lib/types/dataset.d.ts +50 -3
  18. package/lib/types/dataset.d.ts.map +1 -1
  19. package/lib/types/dataset.js +76 -21
  20. package/lib/types/dataset.js.map +1 -1
  21. package/lib/types/events.d.ts +2 -0
  22. package/lib/types/events.d.ts.map +1 -1
  23. package/lib/types/events.js.map +1 -1
  24. package/lib/types/index.d.ts +13 -8
  25. package/lib/types/index.d.ts.map +1 -1
  26. package/lib/types/index.js +17 -6
  27. package/lib/types/index.js.map +1 -1
  28. package/lib/types/presets.d.ts +28 -0
  29. package/lib/types/presets.d.ts.map +1 -0
  30. package/lib/types/presets.js +134 -0
  31. package/lib/types/presets.js.map +1 -0
  32. package/lib/types/profile.d.ts +40 -1
  33. package/lib/types/profile.d.ts.map +1 -1
  34. package/lib/types/profile.js +66 -10
  35. package/lib/types/profile.js.map +1 -1
  36. package/lib/types/provider-local.d.ts +7 -1
  37. package/lib/types/provider-local.d.ts.map +1 -1
  38. package/lib/types/provider-local.js +72 -2
  39. package/lib/types/provider-local.js.map +1 -1
  40. package/lib/types/report-html.d.ts +41 -0
  41. package/lib/types/report-html.d.ts.map +1 -0
  42. package/lib/types/report-html.js +192 -0
  43. package/lib/types/report-html.js.map +1 -0
  44. package/lib/types/scorecard.d.ts +50 -0
  45. package/lib/types/scorecard.d.ts.map +1 -0
  46. package/lib/types/scorecard.js +186 -0
  47. package/lib/types/scorecard.js.map +1 -0
  48. package/lib/types/service.d.ts +32 -2
  49. package/lib/types/service.d.ts.map +1 -1
  50. package/lib/types/service.js.map +1 -1
  51. package/lib/types/store.d.ts +21 -1
  52. package/lib/types/store.d.ts.map +1 -1
  53. package/lib/types/store.js +13 -1
  54. package/lib/types/store.js.map +1 -1
  55. package/lib/types/tools/clean.d.ts.map +1 -1
  56. package/lib/types/tools/clean.js +76 -1
  57. package/lib/types/tools/clean.js.map +1 -1
  58. package/lib/types/tools/profile-report-schema.d.ts +319 -0
  59. package/lib/types/tools/profile-report-schema.d.ts.map +1 -0
  60. package/lib/types/tools/profile-report-schema.js +96 -0
  61. package/lib/types/tools/profile-report-schema.js.map +1 -0
  62. package/lib/types/tools/profile.d.ts.map +1 -1
  63. package/lib/types/tools/profile.js +6 -53
  64. package/lib/types/tools/profile.js.map +1 -1
  65. package/lib/types/tools/report.d.ts +14 -0
  66. package/lib/types/tools/report.d.ts.map +1 -0
  67. package/lib/types/tools/report.js +112 -0
  68. package/lib/types/tools/report.js.map +1 -0
  69. package/lib/types/tools/shared.d.ts.map +1 -1
  70. package/lib/types/tools/shared.js +22 -2
  71. package/lib/types/tools/shared.js.map +1 -1
  72. package/lib/types/tools/verify.d.ts.map +1 -1
  73. package/lib/types/tools/verify.js +36 -1
  74. package/lib/types/tools/verify.js.map +1 -1
  75. package/lib/types/verify.d.ts +49 -3
  76. package/lib/types/verify.d.ts.map +1 -1
  77. package/lib/types/verify.js +111 -4
  78. package/lib/types/verify.js.map +1 -1
  79. package/lib/types/version.d.ts +8 -1
  80. package/lib/types/version.d.ts.map +1 -1
  81. package/lib/types/version.js +8 -1
  82. package/lib/types/version.js.map +1 -1
  83. package/package.json +1 -1
  84. package/src/config.ts +52 -0
  85. package/src/contract.ts +190 -0
  86. package/src/dataset.ts +101 -21
  87. package/src/events.ts +2 -0
  88. package/src/index.ts +27 -8
  89. package/src/presets.ts +146 -0
  90. package/src/profile.ts +105 -11
  91. package/src/provider-local.ts +77 -3
  92. package/src/report-html.ts +208 -0
  93. package/src/scorecard.ts +244 -0
  94. package/src/service.ts +34 -2
  95. package/src/store.ts +28 -2
  96. package/src/tools/clean.ts +76 -1
  97. package/src/tools/profile-report-schema.ts +98 -0
  98. package/src/tools/profile.ts +8 -56
  99. package/src/tools/report.ts +137 -0
  100. package/src/tools/shared.ts +21 -2
  101. package/src/tools/verify.ts +40 -2
  102. package/src/verify.ts +151 -5
  103. package/src/version.ts +9 -1
@@ -7,6 +7,7 @@
7
7
 
8
8
  import { defineTool } from '@deepseek-ai/dsh-tools'
9
9
  import type { CleanRule, CleanRunReport, DataQualityService } from '../service.ts'
10
+ import { PROFILE_REPORT_SCHEMA } from './profile-report-schema.ts'
10
11
  import { renderCleanText, workspaceOf } from './shared.ts'
11
12
 
12
13
  const STRING_MAP_SCHEMA = { type: 'json' } as const
@@ -91,12 +92,13 @@ export function defineCleanTool(service: DataQualityService) {
91
92
  description: [
92
93
  'Apply declarative cleaning rules to a workspace CSV/TSV/JSON/JSONL dataset with deterministic TypeScript computation (no mental math).',
93
94
  'Rules apply in array order: dedupe (by column group), fill-missing (constant/mean/median/forward), coerce-type (number/date/boolean; failures counted and set to missing), normalize-unit (e.g. 万/亿 suffixes to base units), trim (whitespace), map-values (enum mapping).',
94
- 'The source file is NEVER overwritten. Without outputPath the run is preview-only; with outputPath the cleaned dataset is written there (workspace-confined, .csv/.tsv/.json/.jsonl). Returns the per-rule audit log (affected rows per rule) plus a bounded preview. The full report persists to the data_quality storage domain (reportKey).',
95
+ 'The source file is NEVER overwritten. Without outputPath the run is preview-only; with outputPath the cleaned dataset is written there (workspace-confined, .csv/.tsv/.json/.jsonl). Returns the per-rule audit log, the pre-delivery contract summary (with per-column decision trace), and a bounded preview. Pass dryRun: true to skip the write and get the cleaning plan plus the expected contract/diff preview instead. The full report persists to the data_quality storage domain (reportKey).',
95
96
  ].join('\n'),
96
97
  parameters: {
97
98
  path: { type: 'string', required: true, description: 'Workspace-relative dataset path (.csv/.tsv/.json/.jsonl).' },
98
99
  rules: { ...CLEAN_RULE_SCHEMA, required: true },
99
100
  outputPath: { type: 'string', description: 'Optional workspace-relative output path for the cleaned dataset (must differ from path).' },
101
+ dryRun: { type: 'boolean', description: 'When true, do not write any output file; return the cleaning plan and expected contract/diff preview instead (default false).' },
100
102
  },
101
103
  output: {
102
104
  schema: {
@@ -105,9 +107,20 @@ export function defineCleanTool(service: DataQualityService) {
105
107
  dataset: { type: 'string', required: true },
106
108
  inputRows: { type: 'number', required: true },
107
109
  outputRows: { type: 'number', required: true },
110
+ dryRun: { type: 'boolean', required: true },
108
111
  generatedAt: { type: 'number', required: true },
109
112
  outputPath: { type: 'string' },
110
113
  reportKey: { type: 'string' },
114
+ diffPreview: {
115
+ type: 'object',
116
+ properties: {
117
+ dataset: { type: 'string', required: true },
118
+ before: { ...PROFILE_REPORT_SCHEMA, required: true },
119
+ after: { ...PROFILE_REPORT_SCHEMA, required: true },
120
+ generatedAt: { type: 'number', required: true },
121
+ },
122
+ additionalProperties: false,
123
+ },
111
124
  logs: {
112
125
  type: 'array',
113
126
  items: {
@@ -122,6 +135,67 @@ export function defineCleanTool(service: DataQualityService) {
122
135
  },
123
136
  required: true,
124
137
  },
138
+ contract: {
139
+ type: 'object',
140
+ properties: {
141
+ inputRows: { type: 'number', required: true },
142
+ outputRows: { type: 'number', required: true },
143
+ removedRows: { type: 'number', required: true },
144
+ dedupeColumns: { oneOf: [{ type: 'array', items: { type: 'string' } }, { type: 'null' }], required: true },
145
+ uniqueKeys: { type: 'boolean', required: true },
146
+ remainingDuplicateRows: { type: 'number', required: true },
147
+ remainingMissing: {
148
+ type: 'array',
149
+ items: {
150
+ type: 'object',
151
+ properties: {
152
+ column: { type: 'string', required: true },
153
+ count: { type: 'number', required: true },
154
+ },
155
+ additionalProperties: false,
156
+ },
157
+ required: true,
158
+ },
159
+ typeConformance: {
160
+ type: 'array',
161
+ items: {
162
+ type: 'object',
163
+ properties: {
164
+ column: { type: 'string', required: true },
165
+ to: { type: 'string', enum: ['number', 'date', 'boolean'], required: true },
166
+ invalidCount: { type: 'number', required: true },
167
+ },
168
+ additionalProperties: false,
169
+ },
170
+ required: true,
171
+ },
172
+ columnDecisions: {
173
+ type: 'array',
174
+ items: {
175
+ type: 'object',
176
+ properties: {
177
+ column: { type: 'string', required: true },
178
+ decisions: {
179
+ type: 'array',
180
+ items: {
181
+ type: 'object',
182
+ properties: {
183
+ strategy: { type: 'string', required: true },
184
+ affectedRows: { type: 'number', required: true },
185
+ },
186
+ additionalProperties: false,
187
+ },
188
+ required: true,
189
+ },
190
+ },
191
+ additionalProperties: false,
192
+ },
193
+ required: true,
194
+ },
195
+ },
196
+ additionalProperties: false,
197
+ required: true,
198
+ },
125
199
  preview: {
126
200
  type: 'object',
127
201
  properties: {
@@ -141,6 +215,7 @@ export function defineCleanTool(service: DataQualityService) {
141
215
  dataset: args.path,
142
216
  rules: args.rules as unknown as readonly CleanRule[],
143
217
  ...(args.outputPath !== undefined ? { outputPath: args.outputPath } : {}),
218
+ ...(args.dryRun !== undefined ? { dryRun: args.dryRun } : {}),
144
219
  workspace: workspaceOf(exec),
145
220
  session: exec.agent?.session,
146
221
  signal: exec.signal,
@@ -0,0 +1,98 @@
1
+ /**
2
+ * Shared JSON-Schema spec for the profile report's canonical value, reused by
3
+ * `data_profile` (its output) and `data_clean` (the `diffPreview` before/after
4
+ * snapshots). One source of truth so the two tools never drift.
5
+ * @module dsh-data-quality/tools-profile-report-schema
6
+ */
7
+
8
+ const SCORECARD_DIMENSION_NAMES = ['completeness', 'uniqueness', 'validity', 'consistency', 'timeliness', 'accuracy'] as const
9
+
10
+ /** One column card's schema. */
11
+ export const COLUMN_PROFILE_SCHEMA = {
12
+ type: 'object',
13
+ properties: {
14
+ name: { type: 'string', required: true },
15
+ inferredType: { type: 'string', enum: ['number', 'date', 'boolean', 'string', 'empty', 'mixed'], required: true },
16
+ missing: { type: 'number', required: true },
17
+ missingRate: { type: 'number', required: true },
18
+ unique: { type: 'number', required: true },
19
+ numeric: {
20
+ type: 'object',
21
+ properties: {
22
+ count: { type: 'number', required: true },
23
+ distinct: { type: 'number', required: true },
24
+ min: { type: 'number', required: true },
25
+ max: { type: 'number', required: true },
26
+ mean: { type: 'number', required: true },
27
+ median: { type: 'number', required: true },
28
+ p25: { type: 'number', required: true },
29
+ p75: { type: 'number', required: true },
30
+ outliers: { type: 'number', required: true },
31
+ },
32
+ additionalProperties: false,
33
+ },
34
+ topValues: {
35
+ type: 'array',
36
+ items: {
37
+ type: 'object',
38
+ properties: {
39
+ value: { type: 'string', required: true },
40
+ count: { type: 'number', required: true },
41
+ },
42
+ additionalProperties: false,
43
+ },
44
+ },
45
+ notes: { type: 'array', items: { type: 'string' }, required: true },
46
+ },
47
+ additionalProperties: false,
48
+ } as const
49
+
50
+ /** The full profile report's schema. */
51
+ export const PROFILE_REPORT_SCHEMA = {
52
+ type: 'object',
53
+ properties: {
54
+ schemaVersion: { type: 'number', required: true },
55
+ dataset: { type: 'string', required: true },
56
+ rowCount: { type: 'number', required: true },
57
+ sampled: { type: 'boolean', required: true },
58
+ profiledRows: { type: 'number', required: true },
59
+ columnCount: { type: 'number', required: true },
60
+ duplicateRows: { type: 'number', required: true },
61
+ duplicateRate: { type: 'number', required: true },
62
+ duplicateSampleRowIndexes: { type: 'array', items: { type: 'number' }, required: true },
63
+ scorecard: {
64
+ type: 'object',
65
+ properties: {
66
+ overall: { oneOf: [{ type: 'number' }, { type: 'null' }], required: true },
67
+ weightedOverall: { oneOf: [{ type: 'number' }, { type: 'null' }], required: true },
68
+ dimensions: {
69
+ type: 'array',
70
+ items: {
71
+ type: 'object',
72
+ properties: {
73
+ name: { type: 'string', enum: [...SCORECARD_DIMENSION_NAMES], required: true },
74
+ score: { oneOf: [{ type: 'number' }, { type: 'null' }], required: true },
75
+ note: { type: 'string', required: true },
76
+ },
77
+ additionalProperties: false,
78
+ },
79
+ required: true,
80
+ },
81
+ },
82
+ additionalProperties: false,
83
+ required: true,
84
+ },
85
+ encoding: {
86
+ type: 'object',
87
+ properties: {
88
+ bom: { oneOf: [{ type: 'string' }, { type: 'null' }], required: true },
89
+ validUtf8: { type: 'boolean', required: true },
90
+ },
91
+ additionalProperties: false,
92
+ },
93
+ generatedAt: { type: 'number', required: true },
94
+ reportKey: { type: 'string' },
95
+ columns: { type: 'array', items: COLUMN_PROFILE_SCHEMA, required: true },
96
+ },
97
+ additionalProperties: false,
98
+ } as const
@@ -6,46 +6,10 @@
6
6
 
7
7
  import { defineTool } from '@deepseek-ai/dsh-tools'
8
8
  import type { DataQualityService } from '../service.ts'
9
- import { renderProfileText } from '../profile.ts'
9
+ import { renderProfileText, type ProfileReport } from '../profile.ts'
10
+ import { PROFILE_REPORT_SCHEMA } from './profile-report-schema.ts'
10
11
  import { workspaceOf } from './shared.ts'
11
12
 
12
- const COLUMN_PROFILE_SCHEMA = {
13
- type: 'object',
14
- properties: {
15
- name: { type: 'string', required: true },
16
- inferredType: { type: 'string', enum: ['number', 'date', 'boolean', 'string', 'empty', 'mixed'], required: true },
17
- missing: { type: 'number', required: true },
18
- missingRate: { type: 'number', required: true },
19
- unique: { type: 'number', required: true },
20
- numeric: {
21
- type: 'object',
22
- properties: {
23
- min: { type: 'number', required: true },
24
- max: { type: 'number', required: true },
25
- mean: { type: 'number', required: true },
26
- median: { type: 'number', required: true },
27
- p25: { type: 'number', required: true },
28
- p75: { type: 'number', required: true },
29
- outliers: { type: 'number', required: true },
30
- },
31
- additionalProperties: false,
32
- },
33
- topValues: {
34
- type: 'array',
35
- items: {
36
- type: 'object',
37
- properties: {
38
- value: { type: 'string', required: true },
39
- count: { type: 'number', required: true },
40
- },
41
- additionalProperties: false,
42
- },
43
- },
44
- notes: { type: 'array', items: { type: 'string' }, required: true },
45
- },
46
- additionalProperties: false,
47
- } as const
48
-
49
13
  /**
50
14
  * Build the `data_profile` tool definition against a mounted service.
51
15
  * @param service - the mounted ctx.dataQuality implementation.
@@ -56,35 +20,23 @@ export function defineProfileTool(service: DataQualityService) {
56
20
  name: 'data_profile',
57
21
  description: [
58
22
  'Profile a workspace CSV/TSV/JSON/JSONL dataset with deterministic TypeScript computation (no mental math).',
59
- 'Returns row/column counts, inferred column types, missing rates, unique counts, numeric distributions (min/max/mean/median/p25/p75), IQR outlier counts, mixed-type suspicion notes, and duplicate-row counts.',
60
- 'Column cards cover every row by default; pass sample for a deterministic systematic sample on large files. Datasets above the configured row/size caps are rejected — use sample or raise the caps. The full report persists to the data_quality storage domain (reportKey in the result).',
23
+ 'Returns row/column counts, inferred column types, missing rates, unique counts, numeric distributions (count/distinct/min/max/mean/median/p25/p75), IQR outlier counts, mixed-type suspicion notes, sha256 duplicate-row detection (rate + sample indexes), file encoding (BOM/UTF-8 validity), and a weighted DAMA six-dimension scorecard.',
24
+ 'Pass industryPreset (retail/saas/fund/real-estate/e-commerce/healthcare/logistics/manufacturing/energy) to compare the dataset against that industry\'s expected columns, making the scorecard accuracy dimension determinable. Column cards cover every row by default; pass sample for a deterministic systematic sample on large files. Datasets above the configured row/size caps are rejected — use sample or raise the caps. The full report persists to the data_quality storage domain (reportKey in the result).',
61
25
  ].join('\n'),
62
26
  parameters: {
63
27
  path: { type: 'string', required: true, description: 'Workspace-relative dataset path (.csv/.tsv/.json/.jsonl). JSON datasets must be an array of flat objects.' },
64
28
  sample: { type: 'number', description: 'Optional systematic sample size (every ceil(N/sample)-th row) for the column cards; row counts stay exact.' },
29
+ industryPreset: { type: 'string', enum: ['retail', 'saas', 'fund', 'real-estate', 'e-commerce', 'healthcare', 'logistics', 'manufacturing', 'energy'], description: 'Optional industry preset id; its expected columns feed the scorecard accuracy dimension.' },
65
30
  },
66
31
  output: {
67
- schema: {
68
- type: 'object',
69
- properties: {
70
- dataset: { type: 'string', required: true },
71
- rowCount: { type: 'number', required: true },
72
- sampled: { type: 'boolean', required: true },
73
- profiledRows: { type: 'number', required: true },
74
- columnCount: { type: 'number', required: true },
75
- duplicateRows: { type: 'number', required: true },
76
- generatedAt: { type: 'number', required: true },
77
- reportKey: { type: 'string' },
78
- columns: { type: 'array', items: COLUMN_PROFILE_SCHEMA, required: true },
79
- },
80
- additionalProperties: false,
81
- },
82
- render: (_args, value) => [{ type: 'text', text: renderProfileText(value) }],
32
+ schema: PROFILE_REPORT_SCHEMA,
33
+ render: (_args, value) => [{ type: 'text', text: renderProfileText(value as unknown as ProfileReport) }],
83
34
  },
84
35
  async execute(args, exec) {
85
36
  return service.profileDataset({
86
37
  dataset: args.path,
87
38
  sample: args.sample,
39
+ ...(args.industryPreset !== undefined ? { industryPreset: args.industryPreset } : {}),
88
40
  workspace: workspaceOf(exec),
89
41
  session: exec.agent?.session,
90
42
  signal: exec.signal,
@@ -0,0 +1,137 @@
1
+ /**
2
+ * The `data_report` model tool: read persisted profile/clean-diff reports
3
+ * back from the `data_quality` storage domain, by exact `reportKey` or by
4
+ * `kind`. Deterministic read-only consumption — no model arithmetic.
5
+ * @module dsh-data-quality/tools/report
6
+ */
7
+
8
+ import { defineTool } from '@deepseek-ai/dsh-tools'
9
+ import type { JsonValue } from '@deepseek-ai/dsh-session'
10
+ import type { ProfileReport } from '../profile.ts'
11
+ import { renderCleanHtml, renderProfileHtml, type CleanReportHtml } from '../report-html.ts'
12
+ import type { DataQualityService } from '../service.ts'
13
+ import type { ReportRecord, StoredReport } from '../store.ts'
14
+
15
+ /** The report kinds a caller may query. */
16
+ const REPORT_KINDS = ['profile', 'clean', 'clean-diff', 'verify', 'citations'] as const
17
+
18
+ /** One stored report projected into the model-facing canonical value (report as lossless JSON). */
19
+ interface ReportView {
20
+ readonly key: string
21
+ readonly kind: ReportRecord['kind']
22
+ readonly at: number
23
+ readonly dataset: string
24
+ readonly report: JsonValue
25
+ }
26
+
27
+ /** The canonical value the tool returns. */
28
+ interface DataReportValue {
29
+ readonly key?: string
30
+ readonly kind?: ReportRecord['kind']
31
+ readonly records: ReportView[]
32
+ /** Self-contained offline HTML (present only when `format: html`). */
33
+ readonly html?: string
34
+ }
35
+
36
+ /** Render one stored report as a self-contained HTML document (profile/clean only). */
37
+ function renderRecordHtml(record: StoredReport): string {
38
+ if (record.kind === 'profile') {
39
+ return renderProfileHtml(record.report as unknown as ProfileReport)
40
+ }
41
+ if (record.kind === 'clean' || record.kind === 'clean-diff') {
42
+ return renderCleanHtml(record.report as unknown as CleanReportHtml, record.dataset)
43
+ }
44
+ throw new Error(`data_report html format does not support kind "${record.kind}" (profile/clean only)`)
45
+ }
46
+
47
+ /** Project a stored report into the canonical value (the stored report is already lossless JSON). */
48
+ function toView(record: StoredReport): ReportView {
49
+ return { key: record.key, kind: record.kind, at: record.at, dataset: record.dataset, report: record.report as unknown as JsonValue }
50
+ }
51
+
52
+ /** Human-readable report summary for the tool's Native render. */
53
+ function renderReportText(value: DataReportValue): string {
54
+ const lines: string[] = []
55
+ if (value.key !== undefined) {
56
+ lines.push(value.records.length === 0 ? `No report for ${value.key}` : `Report ${value.key}`)
57
+ } else {
58
+ lines.push(`Reports of kind ${value.kind ?? ''} (${value.records.length})`)
59
+ }
60
+ for (const record of value.records) {
61
+ lines.push(`- ${record.key} [${record.kind}] ${record.dataset} @ ${new Date(record.at).toISOString()}`)
62
+ }
63
+ return lines.join('\n')
64
+ }
65
+
66
+ /**
67
+ * Build the `data_report` tool definition against a mounted service.
68
+ * @param service - the mounted ctx.dataQuality implementation.
69
+ * @returns the tool definition to register.
70
+ */
71
+ export function defineReportTool(service: DataQualityService) {
72
+ return defineTool({
73
+ name: 'data_report',
74
+ description: [
75
+ 'Read persisted data-quality reports back from the data_quality storage domain (deterministic, read-only).',
76
+ 'Pass key (the exact reportKey a prior run returned) to fetch one report, or kind to list every persisted report of that kind, ordered chronologically. Exactly one of key/kind.',
77
+ 'Returns the report envelope(s): kind, dataset, timestamp, and the full stored report (profile/clean/clean-diff/verify/citations). Missing keys and unknown kinds fail loudly.',
78
+ ].join('\n'),
79
+ parameters: {
80
+ key: { type: 'string', description: 'Exact storage reportKey (e.g. 20260819000000000-profile-1a2b3c4d); fetches that one report.' },
81
+ kind: { type: 'string', enum: [...REPORT_KINDS], description: 'Report kind to list (profile/clean/clean-diff/verify/citations).' },
82
+ format: { type: 'string', enum: ['json', 'html'], description: 'Output format. json (default) returns the report envelope(s); html renders one report as a self-contained offline HTML document (requires key; profile/clean only).' },
83
+ },
84
+ output: {
85
+ schema: {
86
+ type: 'object',
87
+ properties: {
88
+ key: { type: 'string' },
89
+ kind: { type: 'string', enum: [...REPORT_KINDS] },
90
+ records: {
91
+ type: 'array',
92
+ items: {
93
+ type: 'object',
94
+ properties: {
95
+ key: { type: 'string', required: true },
96
+ kind: { type: 'string', enum: [...REPORT_KINDS], required: true },
97
+ at: { type: 'number', required: true },
98
+ dataset: { type: 'string', required: true },
99
+ report: { type: 'json', required: true },
100
+ },
101
+ additionalProperties: false,
102
+ },
103
+ required: true,
104
+ },
105
+ html: { type: 'string', description: 'Self-contained offline HTML (present only when format: html).' },
106
+ },
107
+ additionalProperties: false,
108
+ },
109
+ render: (_args, value) => {
110
+ const view = value as unknown as DataReportValue
111
+ if (view.html !== undefined) return [{ type: 'text', text: view.html }]
112
+ return [{ type: 'text', text: renderReportText(view) }]
113
+ },
114
+ },
115
+ async execute(args, _exec): Promise<DataReportValue> {
116
+ const hasKey = args.key !== undefined
117
+ const hasKind = args.kind !== undefined
118
+ if (hasKey === hasKind) {
119
+ throw new Error('data_report needs exactly one of key/kind')
120
+ }
121
+ const format = (args.format ?? 'json') as 'json' | 'html'
122
+ if (hasKey) {
123
+ const record = await service.getReport(args.key as string)
124
+ return {
125
+ key: args.key as string,
126
+ records: [toView(record)],
127
+ ...(format === 'html' ? { html: renderRecordHtml(record) } : {}),
128
+ }
129
+ }
130
+ if (format === 'html') {
131
+ throw new Error('data_report html format requires key (exactly one report)')
132
+ }
133
+ const records = await service.listReports(args.kind as ReportRecord['kind'])
134
+ return { kind: args.kind as ReportRecord['kind'], records: records.map(toView) }
135
+ },
136
+ })
137
+ }
@@ -27,11 +27,30 @@ export function workspaceOf(exec: ToolExecution): string {
27
27
  /** Human-readable cleaning summary for the tool's Native render. */
28
28
  export function renderCleanText(report: CleanRunReport): string {
29
29
  const lines: string[] = []
30
- lines.push(`Cleaned ${report.dataset}: ${report.inputRows} -> ${report.outputRows} rows over ${report.logs.length} rule(s)`)
30
+ const mode = report.dryRun ? 'Dry-run plan for' : 'Cleaned'
31
+ lines.push(`${mode} ${report.dataset}: ${report.inputRows} -> ${report.outputRows} rows over ${report.logs.length} rule(s)`)
31
32
  for (const log of report.logs) {
32
33
  lines.push(`- rule ${log.ruleIndex} (${log.rule}): ${log.affectedRows} row(s) affected; ${log.detail}`)
33
34
  }
34
- if (report.outputPath !== undefined) {
35
+ const contract = report.contract
36
+ const keyLabel = contract.dedupeColumns === null ? 'full rows' : `[${contract.dedupeColumns.join(', ')}]`
37
+ lines.push(`Contract: ${contract.inputRows} -> ${contract.outputRows} rows (${contract.removedRows} removed); uniqueness ${contract.uniqueKeys ? 'OK' : 'VIOLATED'} over ${keyLabel}${contract.remainingDuplicateRows > 0 ? ` (${contract.remainingDuplicateRows} duplicate row(s) remain)` : ''}`)
38
+ for (const entry of contract.remainingMissing) {
39
+ lines.push(`- non-null regression: ${entry.column} still has ${entry.count} missing cell(s)`)
40
+ }
41
+ for (const entry of contract.typeConformance) {
42
+ lines.push(`- type regression: ${entry.column} (${entry.to}) has ${entry.invalidCount} non-conforming cell(s)`)
43
+ }
44
+ for (const entry of contract.columnDecisions) {
45
+ const decisions = entry.decisions.map((decision) => `${decision.strategy} (${decision.affectedRows} row(s))`).join(', ')
46
+ lines.push(`- column ${entry.column}: ${decisions}`)
47
+ }
48
+ if (report.diffPreview !== undefined) {
49
+ lines.push(`Diff preview: ${report.diffPreview.before.rowCount} -> ${report.diffPreview.after.rowCount} rows; duplicates ${report.diffPreview.before.duplicateRows} -> ${report.diffPreview.after.duplicateRows}`)
50
+ }
51
+ if (report.dryRun) {
52
+ lines.push('Dry run: no output file written and no report persisted; plan/preview above.')
53
+ } else if (report.outputPath !== undefined) {
35
54
  lines.push(`Wrote cleaned dataset to ${report.outputPath}`)
36
55
  } else {
37
56
  lines.push('No outputPath given: the source file was left untouched; preview below.')
@@ -6,10 +6,44 @@
6
6
  */
7
7
 
8
8
  import { defineTool } from '@deepseek-ai/dsh-tools'
9
- import type { DataQualityService, VerifyRule } from '../service.ts'
9
+ import type { DataQualityService, VerifyExpectation, VerifyRule } from '../service.ts'
10
10
  import { renderVerifyText, type VerifyReport } from '../verify.ts'
11
11
  import { workspaceOf } from './shared.ts'
12
12
 
13
+ const EXPECTATION_METRICS = ['rowCount', 'columnSum', 'columnMean', 'uniqueCount', 'nullCount'] as const
14
+
15
+ const EXPECTATION_SCHEMA = {
16
+ type: 'array',
17
+ items: {
18
+ type: 'object',
19
+ properties: {
20
+ metric: { type: 'string', enum: [...EXPECTATION_METRICS], required: true, description: 'Metric to reconcile: rowCount/columnSum/columnMean/uniqueCount/nullCount.' },
21
+ column: { type: 'string', description: 'Required for every metric except rowCount.' },
22
+ expected: { type: 'number', required: true, description: 'The expected value to reconcile against.' },
23
+ tolerance: { type: 'number', description: 'Optional relative tolerance in [0, 1]; defaults to defaultTolerance.' },
24
+ },
25
+ additionalProperties: false,
26
+ description: 'Reconcile a deterministic computed metric against an expected value with relative tolerance.',
27
+ },
28
+ description: 'Optional metric expectations; each yields passed true or passed false with actual/expected/tolerance detail.',
29
+ } as const
30
+
31
+ const EXPECTATION_RESULT_SCHEMA = {
32
+ type: 'array',
33
+ items: {
34
+ type: 'object',
35
+ properties: {
36
+ metric: { type: 'string', enum: [...EXPECTATION_METRICS], required: true },
37
+ column: { type: 'string' },
38
+ expected: { type: 'number', required: true },
39
+ actual: { type: 'number', required: true },
40
+ tolerance: { type: 'number', required: true },
41
+ passed: { type: 'boolean', required: true },
42
+ },
43
+ additionalProperties: false,
44
+ },
45
+ } as const
46
+
13
47
  const VERIFY_RULE_SCHEMA = {
14
48
  type: 'array',
15
49
  items: {
@@ -103,11 +137,13 @@ export function defineVerifyTool(service: DataQualityService) {
103
137
  description: [
104
138
  'Verify a workspace CSV/TSV/JSON/JSONL dataset against declarative quality rules with deterministic TypeScript computation (no mental math).',
105
139
  'Rules: not-null, unique (column group), range (numeric bounds), regex, enum, cross-column (e.g. startDate < endDate), freshness (date column within N days of asOf). A missing cell fails every rule that reads it.',
106
- 'Returns per-rule pass/fail with capped failing-row evidence. Overall failure is a NORMAL result with passed: false not a tool error. The full report persists to the data_quality storage domain (reportKey).',
140
+ 'Optional expectations reconcile deterministic metrics (rowCount/columnSum/columnMean/uniqueCount/nullCount) against expected values with relative tolerance; a mismatch is a normal passed: false with actual/expected/tolerance detail, never a tool error.',
141
+ 'Returns per-rule pass/fail with capped failing-row evidence plus the expectation outcomes. Overall failure is a NORMAL result with passed: false — not a tool error. The full report persists to the data_quality storage domain (reportKey).',
107
142
  ].join('\n'),
108
143
  parameters: {
109
144
  path: { type: 'string', required: true, description: 'Workspace-relative dataset path (.csv/.tsv/.json/.jsonl).' },
110
145
  rules: { ...VERIFY_RULE_SCHEMA, required: true },
146
+ expectations: { ...EXPECTATION_SCHEMA, description: 'Optional metric expectations to reconcile (rowCount/columnSum/columnMean/uniqueCount/nullCount).' },
111
147
  },
112
148
  output: {
113
149
  schema: {
@@ -145,6 +181,7 @@ export function defineVerifyTool(service: DataQualityService) {
145
181
  },
146
182
  required: true,
147
183
  },
184
+ expectations: { ...EXPECTATION_RESULT_SCHEMA, required: true },
148
185
  },
149
186
  additionalProperties: false,
150
187
  },
@@ -154,6 +191,7 @@ export function defineVerifyTool(service: DataQualityService) {
154
191
  return service.verifyDataset({
155
192
  dataset: args.path,
156
193
  rules: args.rules as unknown as readonly VerifyRule[],
194
+ ...(args.expectations !== undefined ? { expectations: args.expectations as unknown as readonly VerifyExpectation[] } : {}),
157
195
  workspace: workspaceOf(exec),
158
196
  session: exec.agent?.session,
159
197
  signal: exec.signal,