dsh-data-quality 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. package/CHANGELOG.md +16 -0
  2. package/LICENSE +201 -0
  3. package/README.es.md +181 -0
  4. package/README.hi.md +181 -0
  5. package/README.md +181 -0
  6. package/README.pt.md +181 -0
  7. package/README.zh.md +181 -0
  8. package/THIRD_PARTY_NOTICES.md +20 -0
  9. package/cordis.patch.yml +46 -0
  10. package/lib/index.js +2458 -0
  11. package/lib/types/clean.d.ts +82 -0
  12. package/lib/types/clean.d.ts.map +1 -0
  13. package/lib/types/clean.js +351 -0
  14. package/lib/types/clean.js.map +1 -0
  15. package/lib/types/config.d.ts +47 -0
  16. package/lib/types/config.d.ts.map +1 -0
  17. package/lib/types/config.js +66 -0
  18. package/lib/types/config.js.map +1 -0
  19. package/lib/types/dataset.d.ts +133 -0
  20. package/lib/types/dataset.d.ts.map +1 -0
  21. package/lib/types/dataset.js +404 -0
  22. package/lib/types/dataset.js.map +1 -0
  23. package/lib/types/events.d.ts +73 -0
  24. package/lib/types/events.d.ts.map +1 -0
  25. package/lib/types/events.js +41 -0
  26. package/lib/types/events.js.map +1 -0
  27. package/lib/types/index.d.ts +45 -0
  28. package/lib/types/index.d.ts.map +1 -0
  29. package/lib/types/index.js +78 -0
  30. package/lib/types/index.js.map +1 -0
  31. package/lib/types/present.d.ts +24 -0
  32. package/lib/types/present.d.ts.map +1 -0
  33. package/lib/types/present.js +34 -0
  34. package/lib/types/present.js.map +1 -0
  35. package/lib/types/profile.d.ts +79 -0
  36. package/lib/types/profile.d.ts.map +1 -0
  37. package/lib/types/profile.js +196 -0
  38. package/lib/types/profile.js.map +1 -0
  39. package/lib/types/provider-local.d.ts +56 -0
  40. package/lib/types/provider-local.d.ts.map +1 -0
  41. package/lib/types/provider-local.js +163 -0
  42. package/lib/types/provider-local.js.map +1 -0
  43. package/lib/types/service.d.ts +160 -0
  44. package/lib/types/service.d.ts.map +1 -0
  45. package/lib/types/service.js +25 -0
  46. package/lib/types/service.js.map +1 -0
  47. package/lib/types/store.d.ts +61 -0
  48. package/lib/types/store.d.ts.map +1 -0
  49. package/lib/types/store.js +42 -0
  50. package/lib/types/store.js.map +1 -0
  51. package/lib/types/tools/clean.d.ts +14 -0
  52. package/lib/types/tools/clean.d.ts.map +1 -0
  53. package/lib/types/tools/clean.js +146 -0
  54. package/lib/types/tools/clean.js.map +1 -0
  55. package/lib/types/tools/profile.d.ts +13 -0
  56. package/lib/types/tools/profile.d.ts.map +1 -0
  57. package/lib/types/tools/profile.js +91 -0
  58. package/lib/types/tools/profile.js.map +1 -0
  59. package/lib/types/tools/shared.d.ts +19 -0
  60. package/lib/types/tools/shared.d.ts.map +1 -0
  61. package/lib/types/tools/shared.js +44 -0
  62. package/lib/types/tools/shared.js.map +1 -0
  63. package/lib/types/tools/verify.d.ts +14 -0
  64. package/lib/types/tools/verify.d.ts.map +1 -0
  65. package/lib/types/tools/verify.js +160 -0
  66. package/lib/types/tools/verify.js.map +1 -0
  67. package/lib/types/verify.d.ts +124 -0
  68. package/lib/types/verify.d.ts.map +1 -0
  69. package/lib/types/verify.js +391 -0
  70. package/lib/types/verify.js.map +1 -0
  71. package/lib/types/version.d.ts +8 -0
  72. package/lib/types/version.d.ts.map +1 -0
  73. package/lib/types/version.js +8 -0
  74. package/lib/types/version.js.map +1 -0
  75. package/package.json +137 -0
  76. package/src/clean.ts +382 -0
  77. package/src/config.ts +104 -0
  78. package/src/dataset.ts +445 -0
  79. package/src/events.ts +90 -0
  80. package/src/index.ts +115 -0
  81. package/src/present.ts +38 -0
  82. package/src/profile.ts +250 -0
  83. package/src/provider-local.ts +194 -0
  84. package/src/service.ts +172 -0
  85. package/src/store.ts +74 -0
  86. package/src/tools/clean.ts +150 -0
  87. package/src/tools/profile.ts +94 -0
  88. package/src/tools/shared.ts +47 -0
  89. package/src/tools/verify.ts +163 -0
  90. package/src/verify.ts +496 -0
  91. package/src/version.ts +8 -0
package/src/dataset.ts ADDED
@@ -0,0 +1,445 @@
1
+ /**
2
+ * Dataset loading for `dsh-data-quality`: workspace-confined path resolution,
3
+ * size/row guards, and deterministic parsers for CSV/TSV/JSON/JSONL. All
4
+ * parsing is hand-rolled (no external runtime dependencies) and bounded by
5
+ * the resolved config; oversized inputs reject loudly with actionable hints.
6
+ * @module dsh-data-quality/dataset
7
+ */
8
+
9
+ import { open, stat } from 'node:fs/promises'
10
+ import path from 'node:path'
11
+ import { createHash } from 'node:crypto'
12
+ import type { JsonValue } from '@deepseek-ai/dsh-session'
13
+ import type { ResolvedConfig } from './config.ts'
14
+
15
+ /** One cell of a tabular dataset: a JSON scalar, `null` for missing, or a nested JSON value (document rows). */
16
+ export type Cell = JsonValue
17
+
18
+ /** One dataset row keyed by column name. */
19
+ export type Row = Record<string, Cell>
20
+
21
+ /** A tabular dataset: ordered columns plus rows. */
22
+ export interface Table {
23
+ /** Column names in file order. */
24
+ readonly columns: string[]
25
+ /** Rows, each carrying every declared column (missing cells are `null`). */
26
+ readonly rows: Row[]
27
+ }
28
+
29
+ /** The root form a document load returns (citation checking walks this). */
30
+ export type DocumentRoot =
31
+ | { readonly kind: 'table'; readonly columns: string[]; readonly rows: Row[] }
32
+ | { readonly kind: 'json'; readonly value: unknown }
33
+
34
+ /** Raised for every user-facing dataset failure; `code` is stable for tests and tooling. */
35
+ export class DatasetError extends Error {
36
+ /**
37
+ * @param code - stable machine-readable reason.
38
+ * @param message - actionable human-readable detail.
39
+ */
40
+ constructor(
41
+ readonly code:
42
+ | 'path-escape'
43
+ | 'extension-denied'
44
+ | 'not-found'
45
+ | 'too-large'
46
+ | 'too-many-rows'
47
+ | 'malformed'
48
+ | 'not-tabular',
49
+ message: string,
50
+ ) {
51
+ super(message)
52
+ this.name = 'DatasetError'
53
+ }
54
+ }
55
+
56
+ /**
57
+ * Resolve `requested` inside `root`, rejecting escapes and disallowed
58
+ * extensions. Both sides go through `path.resolve` before comparison so
59
+ * Windows backslash/forward-slash mixes never defeat the containment check.
60
+ * @param root - absolute workspace root.
61
+ * @param requested - the caller-supplied path (relative to root, or absolute inside root).
62
+ * @param config - resolved config (extension allowlist).
63
+ * @returns the normalized absolute path inside `root`.
64
+ */
65
+ export function resolveWorkspacePath(root: string, requested: string, config: ResolvedConfig): string {
66
+ if (requested.trim() === '') {
67
+ throw new DatasetError('path-escape', 'dataset path must not be empty')
68
+ }
69
+ const resolvedRoot = path.resolve(root)
70
+ const resolved = path.resolve(resolvedRoot, requested)
71
+ if (resolved !== resolvedRoot && !resolved.startsWith(resolvedRoot + path.sep)) {
72
+ throw new DatasetError(
73
+ 'path-escape',
74
+ `path ${JSON.stringify(requested)} resolves outside the workspace root ${JSON.stringify(resolvedRoot)}`,
75
+ )
76
+ }
77
+ const ext = path.extname(resolved).toLowerCase()
78
+ if (!config.allowedExtensions.includes(ext)) {
79
+ throw new DatasetError(
80
+ 'extension-denied',
81
+ `extension ${JSON.stringify(ext)} is not in allowedExtensions (${config.allowedExtensions.join(', ')})`,
82
+ )
83
+ }
84
+ return resolved
85
+ }
86
+
87
+ /**
88
+ * Short deterministic fingerprint of a dataset path, used in storage keys.
89
+ * @param datasetPath - the workspace-relative dataset path as callers gave it.
90
+ * @returns an 8-hex-char fingerprint.
91
+ */
92
+ export function pathFingerprint(datasetPath: string): string {
93
+ return createHash('sha256').update(datasetPath).digest('hex').slice(0, 8)
94
+ }
95
+
96
+ /** Assert the abort signal has not fired; throws the signal reason otherwise. */
97
+ export function throwIfAborted(signal: AbortSignal | undefined): void {
98
+ if (signal?.aborted === true) {
99
+ throw signal.reason instanceof Error ? signal.reason : new Error('operation aborted')
100
+ }
101
+ }
102
+
103
+ /**
104
+ * Read a dataset file under the size cap.
105
+ * @param absolutePath - normalized absolute path (from {@link resolveWorkspacePath}).
106
+ * @param config - resolved config (size cap).
107
+ * @param signal - optional abort signal honored around the read.
108
+ * @returns the UTF-8 text.
109
+ */
110
+ export async function readDatasetText(absolutePath: string, config: ResolvedConfig, signal?: AbortSignal): Promise<string> {
111
+ throwIfAborted(signal)
112
+ let info
113
+ try {
114
+ info = await stat(absolutePath)
115
+ } catch {
116
+ throw new DatasetError('not-found', `dataset file not found: ${absolutePath}`)
117
+ }
118
+ if (!info.isFile()) {
119
+ throw new DatasetError('not-found', `dataset path is not a file: ${absolutePath}`)
120
+ }
121
+ const capBytes = config.maxFileSizeMB * 1024 * 1024
122
+ if (info.size > capBytes) {
123
+ throw new DatasetError(
124
+ 'too-large',
125
+ `dataset is ${(info.size / 1024 / 1024).toFixed(1)} MiB, above the ${config.maxFileSizeMB} MiB cap; split the file or raise maxFileSizeMB`,
126
+ )
127
+ }
128
+ const handle = await open(absolutePath, 'r')
129
+ try {
130
+ throwIfAborted(signal)
131
+ return await handle.readFile('utf8')
132
+ } finally {
133
+ await handle.close()
134
+ }
135
+ }
136
+
137
+ /**
138
+ * Parse CSV/TSV text (RFC-4180-style: quoted fields, `""` escapes, CRLF/LF).
139
+ * The first record is the header; short records pad with `null`, long records
140
+ * fail loud. An empty field parses as `null` (missing).
141
+ * @param text - file text.
142
+ * @param delimiter - field delimiter (`,` or tab).
143
+ * @param config - resolved config (row cap).
144
+ * @param signal - optional abort signal, checked per 1024-record stride.
145
+ * @returns the parsed table.
146
+ */
147
+ export function parseDelimited(text: string, delimiter: string, config: ResolvedConfig, signal?: AbortSignal): Table {
148
+ const records: string[][] = []
149
+ let field = ''
150
+ let record: string[] = []
151
+ let inQuotes = false
152
+ const pushField = (): void => {
153
+ record.push(field)
154
+ field = ''
155
+ }
156
+ const pushRecord = (): void => {
157
+ pushField()
158
+ // Skip truly blank lines (a record of one empty field), as RFC-4180
159
+ // writers commonly emit a trailing empty line.
160
+ if (record.length === 1 && record[0] === '') {
161
+ record = []
162
+ field = ''
163
+ return
164
+ }
165
+ records.push(record)
166
+ record = []
167
+ if (records.length % 1024 === 0) throwIfAborted(signal)
168
+ if (records.length - 1 > config.maxRows) {
169
+ throw new DatasetError(
170
+ 'too-many-rows',
171
+ `dataset exceeds the ${config.maxRows} row cap; use the sample parameter or raise maxRows`,
172
+ )
173
+ }
174
+ }
175
+ let index = 0
176
+ while (index < text.length) {
177
+ const char = text[index] as string
178
+ if (inQuotes) {
179
+ if (char === '"') {
180
+ if (text[index + 1] === '"') {
181
+ field += '"'
182
+ index += 2
183
+ continue
184
+ }
185
+ inQuotes = false
186
+ index += 1
187
+ continue
188
+ }
189
+ field += char
190
+ index += 1
191
+ continue
192
+ }
193
+ if (char === '"') {
194
+ inQuotes = true
195
+ index += 1
196
+ continue
197
+ }
198
+ if (char === delimiter) {
199
+ pushField()
200
+ index += 1
201
+ continue
202
+ }
203
+ if (char === '\r') {
204
+ index += 1
205
+ continue
206
+ }
207
+ if (char === '\n') {
208
+ pushRecord()
209
+ index += 1
210
+ continue
211
+ }
212
+ field += char
213
+ index += 1
214
+ }
215
+ if (field !== '' || record.length > 0) pushRecord()
216
+
217
+ const header = records[0]
218
+ if (header === undefined || (header.length === 1 && header[0] === '')) {
219
+ throw new DatasetError('malformed', 'dataset has no header row')
220
+ }
221
+ const columns = header.map((name) => name.trim())
222
+ if (columns.some((name) => name === '')) {
223
+ throw new DatasetError('malformed', 'header contains an empty column name')
224
+ }
225
+ if (new Set(columns).size !== columns.length) {
226
+ throw new DatasetError('malformed', 'header contains duplicate column names')
227
+ }
228
+ const rows: Row[] = []
229
+ for (let i = 1; i < records.length; i++) {
230
+ const cells = records[i] as string[]
231
+ if (cells.length > columns.length) {
232
+ throw new DatasetError('malformed', `row ${i + 1} has ${cells.length} fields but the header has ${columns.length}`)
233
+ }
234
+ const out: Row = {}
235
+ for (let c = 0; c < columns.length; c++) {
236
+ const raw = cells[c]
237
+ out[columns[c] as string] = raw === undefined || raw === '' ? null : raw
238
+ }
239
+ rows.push(out)
240
+ }
241
+ return { columns, rows }
242
+ }
243
+
244
+ /**
245
+ * Parse a JSON or JSONL text into a tabular dataset. JSON must be an array of
246
+ * flat objects; JSONL is one object per line. Non-scalar cell values are kept
247
+ * as-is (engines treat them as non-scalar).
248
+ * @param text - file text.
249
+ * @param extension - `.json` or `.jsonl`.
250
+ * @param config - resolved config (row cap).
251
+ * @param signal - optional abort signal.
252
+ * @returns the parsed table.
253
+ */
254
+ export function parseJsonTable(text: string, extension: string, config: ResolvedConfig, signal?: AbortSignal): Table {
255
+ const values = extension === '.jsonl' ? parseJsonLines(text) : parseJsonDocument(text)
256
+ const items = Array.isArray(values) ? values : [values]
257
+ if (items.length > config.maxRows) {
258
+ throw new DatasetError(
259
+ 'too-many-rows',
260
+ `dataset exceeds the ${config.maxRows} row cap; use the sample parameter or raise maxRows`,
261
+ )
262
+ }
263
+ const rows: Row[] = []
264
+ const columns: string[] = []
265
+ const seen = new Set<string>()
266
+ for (const [index, item] of items.entries()) {
267
+ if (index % 1024 === 0) throwIfAborted(signal)
268
+ if (typeof item !== 'object' || item === null || Array.isArray(item)) {
269
+ throw new DatasetError(
270
+ 'not-tabular',
271
+ `row ${index + 1} is not a flat object; profiling/cleaning/verification need a tabular dataset (array of objects)`,
272
+ )
273
+ }
274
+ const row = item as Row
275
+ for (const key of Object.keys(row)) {
276
+ if (!seen.has(key)) {
277
+ seen.add(key)
278
+ columns.push(key)
279
+ }
280
+ }
281
+ rows.push(row)
282
+ }
283
+ return { columns, rows: rows.map((row) => normalizeRow(row, columns)) }
284
+ }
285
+
286
+ /** Fill absent columns with `null` so every row carries every column. */
287
+ function normalizeRow(row: Row, columns: string[]): Row {
288
+ const out: Row = {}
289
+ for (const column of columns) {
290
+ const value = row[column]
291
+ out[column] = value === undefined ? null : value
292
+ }
293
+ return out
294
+ }
295
+
296
+ /** Parse one JSON document; failures become a loud malformed error. */
297
+ function parseJsonDocument(text: string): unknown {
298
+ try {
299
+ return JSON.parse(text) as unknown
300
+ } catch (error) {
301
+ throw new DatasetError('malformed', `invalid JSON: ${error instanceof Error ? error.message : String(error)}`)
302
+ }
303
+ }
304
+
305
+ /** Parse JSONL (one JSON value per non-empty line); failures name the line. */
306
+ function parseJsonLines(text: string): unknown[] {
307
+ const out: unknown[] = []
308
+ const lines = text.split(/\r?\n/u)
309
+ for (const [index, line] of lines.entries()) {
310
+ if (line.trim() === '') continue
311
+ try {
312
+ out.push(JSON.parse(line))
313
+ } catch {
314
+ throw new DatasetError('malformed', `invalid JSON on line ${index + 1}`)
315
+ }
316
+ }
317
+ return out
318
+ }
319
+
320
+ /**
321
+ * Load a tabular dataset from a file already resolved inside the workspace.
322
+ * @param absolutePath - normalized absolute dataset path.
323
+ * @param config - resolved config.
324
+ * @param signal - optional abort signal.
325
+ * @returns the parsed table.
326
+ */
327
+ export async function loadTable(absolutePath: string, config: ResolvedConfig, signal?: AbortSignal): Promise<Table> {
328
+ const text = await readDatasetText(absolutePath, config, signal)
329
+ const ext = path.extname(absolutePath).toLowerCase()
330
+ throwIfAborted(signal)
331
+ if (ext === '.csv') return parseDelimited(text, ',', config, signal)
332
+ if (ext === '.tsv') return parseDelimited(text, '\t', config, signal)
333
+ return parseJsonTable(text, ext, config, signal)
334
+ }
335
+
336
+ /**
337
+ * Load a dataset as a citation-checkable document root: CSV/TSV become
338
+ * `{ columns, rows }` (so `rows[3].nav` resolves), JSON is the parsed value,
339
+ * JSONL is the array of parsed lines.
340
+ * @param absolutePath - normalized absolute dataset path.
341
+ * @param config - resolved config.
342
+ * @param signal - optional abort signal.
343
+ * @returns the document root.
344
+ */
345
+ export async function loadDocument(absolutePath: string, config: ResolvedConfig, signal?: AbortSignal): Promise<DocumentRoot> {
346
+ const ext = path.extname(absolutePath).toLowerCase()
347
+ if (ext === '.json') {
348
+ const text = await readDatasetText(absolutePath, config, signal)
349
+ return { kind: 'json', value: parseJsonDocument(text) }
350
+ }
351
+ if (ext === '.jsonl') {
352
+ const text = await readDatasetText(absolutePath, config, signal)
353
+ return { kind: 'json', value: parseJsonLines(text) }
354
+ }
355
+ const table = await loadTable(absolutePath, config, signal)
356
+ return { kind: 'table', columns: table.columns, rows: table.rows }
357
+ }
358
+
359
+ /**
360
+ * Deterministic systematic sample: every `ceil(total / count)`-th row from
361
+ * index 0. No clock, no RNG — the same input always yields the same sample.
362
+ * @param rows - full row list.
363
+ * @param count - requested sample size.
364
+ * @returns the sampled rows (identity when `count >= rows.length`).
365
+ */
366
+ export function sampleRows(rows: Row[], count: number): Row[] {
367
+ if (!Number.isSafeInteger(count) || count <= 0) {
368
+ throw new DatasetError('malformed', `sample must be a positive integer, got ${String(count)}`)
369
+ }
370
+ if (count >= rows.length) return rows
371
+ const stride = Math.ceil(rows.length / count)
372
+ const out: Row[] = []
373
+ for (let i = 0; i < rows.length; i += stride) {
374
+ out.push(rows[i] as Row)
375
+ }
376
+ return out
377
+ }
378
+
379
+ /**
380
+ * Missingness: `null`, `undefined`, or a string empty after trimming. A
381
+ * whitespace-only cell counts as missing even before any `trim` clean rule.
382
+ * @param cell - the cell to test (`undefined` when the column is absent).
383
+ * @returns whether the cell is missing.
384
+ */
385
+ export function isMissing(cell: Cell | undefined): boolean {
386
+ if (cell === null || cell === undefined) return true
387
+ if (typeof cell === 'string') return cell.trim() === ''
388
+ return false
389
+ }
390
+
391
+ /** Strict full-string numeric parse (no thousands separators, no unit suffixes). */
392
+ export function parseNumeric(cell: Cell | undefined): number | undefined {
393
+ if (typeof cell === 'number') return Number.isFinite(cell) ? cell : undefined
394
+ if (typeof cell !== 'string') return undefined
395
+ const text = cell.trim()
396
+ if (text === '') return undefined
397
+ if (!/^[+-]?(?:\d+\.?\d*|\.\d+)(?:[eE][+-]?\d+)?$/u.test(text)) return undefined
398
+ const value = Number(text)
399
+ return Number.isFinite(value) ? value : undefined
400
+ }
401
+
402
+ const DATE_PATTERNS: readonly RegExp[] = [
403
+ /^(\d{4})-(\d{1,2})-(\d{1,2})$/u,
404
+ /^(\d{4})\/(\d{1,2})\/(\d{1,2})$/u,
405
+ /^(\d{4})-(\d{1,2})-(\d{1,2})[ T](\d{1,2}):(\d{2})(?::(\d{2}))?(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?$/u,
406
+ ]
407
+
408
+ /**
409
+ * Deterministic date parse to epoch milliseconds. Accepts `YYYY-MM-DD`,
410
+ * `YYYY/MM/DD`, and ISO-like datetimes (date-only forms read as UTC midnight).
411
+ * Calendar-invalid dates (e.g. 2025-13-40) reject. Returns `undefined` when
412
+ * the cell is not a recognized date.
413
+ * @param cell - the cell to parse (`undefined` when the column is absent).
414
+ * @returns epoch milliseconds, or `undefined`.
415
+ */
416
+ export function parseDate(cell: Cell | undefined): number | undefined {
417
+ if (typeof cell !== 'string') return undefined
418
+ const text = cell.trim()
419
+ for (const pattern of DATE_PATTERNS) {
420
+ const match = pattern.exec(text)
421
+ if (match === null) continue
422
+ const year = Number(match[1])
423
+ const month = Number(match[2])
424
+ const day = Number(match[3])
425
+ const hour = Number(match[4] ?? 0)
426
+ const minute = Number(match[5] ?? 0)
427
+ const second = Number(match[6] ?? 0)
428
+ if (month < 1 || month > 12 || day < 1 || day > 31 || hour > 23 || minute > 59 || second > 59) return undefined
429
+ const epoch = Date.UTC(year, month - 1, day, hour, minute, second)
430
+ const check = new Date(epoch)
431
+ if (check.getUTCMonth() !== month - 1 || check.getUTCDate() !== day) return undefined
432
+ return epoch
433
+ }
434
+ return undefined
435
+ }
436
+
437
+ /** Boolean parse: true/false/yes/no/1/0, case-insensitive. */
438
+ export function parseBoolean(cell: Cell | undefined): boolean | undefined {
439
+ if (typeof cell === 'boolean') return cell
440
+ if (typeof cell !== 'string') return undefined
441
+ const text = cell.trim().toLowerCase()
442
+ if (text === 'true' || text === 'yes' || text === '1') return true
443
+ if (text === 'false' || text === 'no' || text === '0') return false
444
+ return undefined
445
+ }
package/src/events.ts ADDED
@@ -0,0 +1,90 @@
1
+ /**
2
+ * The `data-quality/*` session-event vocabulary and its adaptive append gate.
3
+ * The vocabulary is declared on `SessionEventMap` for type-checked payloads;
4
+ * the runtime gate decides per host whether appending is safe:
5
+ *
6
+ * - Hosts whose known-type set already covers the vocabulary (a future harness
7
+ * that adopts these events) append plainly.
8
+ * - Hosts with an `ignorable` append option (the master-build
9
+ * `Session.append(type, data, { ignorable: true })` contract) append with the
10
+ * marker, so builds that do not know the type skip the event on restore.
11
+ * - 0.1.0-rc.6 hosts have neither a registration surface nor the `ignorable`
12
+ * append flag; appending an unknown type there would make the persistence
13
+ * coordinator refuse the session log on restore, so the append is skipped
14
+ * and the storage-domain report remains the durable copy.
15
+ * @module dsh-data-quality/events
16
+ */
17
+
18
+ import { KNOWN_SESSION_EVENT_TYPES, type Session } from '@deepseek-ai/dsh-session'
19
+
20
+ /** Payload shared by every `data-quality/*` session event. */
21
+ export interface DataQualityEventData {
22
+ /** Which run produced the event. */
23
+ readonly kind: 'profile' | 'clean' | 'verify'
24
+ /** Workspace-relative dataset path as the caller gave it. */
25
+ readonly dataset: string
26
+ /** Storage-domain key of the full persisted report, when persistence is on. */
27
+ readonly reportKey?: string
28
+ /** Compact run summary (small integers/booleans only). */
29
+ readonly summary: {
30
+ readonly rows: number
31
+ readonly columns?: number
32
+ readonly rules?: number
33
+ readonly failedRules?: number
34
+ readonly passed?: boolean
35
+ }
36
+ }
37
+
38
+ declare module '@deepseek-ai/dsh-session/types' {
39
+ interface SessionEventMap {
40
+ /**
41
+ * A `data_profile` run completed.
42
+ * @mode emit
43
+ * @param data - run kind, dataset, optional report key, and compact summary.
44
+ */
45
+ 'data-quality/profile': DataQualityEventData
46
+ /**
47
+ * A `data_clean` run completed.
48
+ * @mode emit
49
+ * @param data - run kind, dataset, optional report key, and compact summary.
50
+ */
51
+ 'data-quality/clean': DataQualityEventData
52
+ /**
53
+ * A `data_verify` run completed.
54
+ * @mode emit
55
+ * @param data - run kind, dataset, optional report key, and compact summary.
56
+ */
57
+ 'data-quality/verify': DataQualityEventData
58
+ }
59
+ }
60
+
61
+ /** The event type vocabulary this plugin appends. */
62
+ export const DATA_QUALITY_EVENT_TYPES = ['data-quality/profile', 'data-quality/clean', 'data-quality/verify'] as const
63
+
64
+ /** Union of the event types this plugin appends. */
65
+ export type DataQualityEventType = (typeof DATA_QUALITY_EVENT_TYPES)[number]
66
+
67
+ /** Loose append shape probed at runtime (rc.6 takes no options; master takes `ignorable`). */
68
+ type AppendProbe = (type: string, data: unknown, options?: { ignorable: true }) => unknown
69
+
70
+ /**
71
+ * Append one `data-quality/*` event when the host can carry it safely; skip
72
+ * silently otherwise (the storage-domain report is always the durable copy).
73
+ * The `ignorable` probe reads the UNBOUND method's source (a `.bind()` result
74
+ * reports `[native code]`): the rc.6 build contains no `ignorable` handling
75
+ * while the master build references the flag by name; property names survive
76
+ * minification, so the probe fails safe (skips) rather than corrupting a log.
77
+ * @param session - the calling session.
78
+ * @param type - the event type.
79
+ * @param data - the payload.
80
+ */
81
+ export function appendDataQualityEvent(session: Session, type: DataQualityEventType, data: DataQualityEventData): void {
82
+ if (KNOWN_SESSION_EVENT_TYPES.has(type)) {
83
+ session.append(type, data)
84
+ return
85
+ }
86
+ const append = session.append as AppendProbe
87
+ if (Function.prototype.toString.call(append).includes('ignorable')) {
88
+ append.call(session, type, data, { ignorable: true })
89
+ }
90
+ }
package/src/index.ts ADDED
@@ -0,0 +1,115 @@
1
+ /**
2
+ * `dsh-data-quality` — deterministic data profiling, cleaning, and
3
+ * verification for DeepSeek Harness. Mounts the `ctx.dataQuality` capability
4
+ * seam (Service Definition in `service.ts`, local Provider in
5
+ * `provider-local.ts`), registers the `data_profile` / `data_clean` /
6
+ * `data_verify` model tools (Consumers — all computation goes through the
7
+ * service layer), persists run reports to the `data_quality` storage domain,
8
+ * and appends adaptive `data-quality/*` session events on hosts that can
9
+ * carry them.
10
+ *
11
+ * Function plugin — no default export (the Loader unwraps
12
+ * `exports.default ?? exports`, and a stray default would discard
13
+ * `name`/`inject`/`Config`/`apply`).
14
+ * @module dsh-data-quality
15
+ */
16
+
17
+ import type { Context } from '@deepseek-ai/cordis'
18
+ // Type-only: registers the `ctx.storageDomain` Context merge for the inject.
19
+ import type { Domain } from '@deepseek-ai/dsh-storage-domain'
20
+ import { Config, resolveConfig } from './config.ts'
21
+ import { dataQualityDomainSpec, reportKeyOf, type ReportStore } from './store.ts'
22
+ import { LocalDataQualityService } from './provider-local.ts'
23
+ import { defineProfileTool } from './tools/profile.ts'
24
+ import { defineCleanTool } from './tools/clean.ts'
25
+ import { defineVerifyTool } from './tools/verify.ts'
26
+ import { VERSION } from './version.ts'
27
+
28
+ export const name = 'data-quality'
29
+ /** The three model tools and the durable report domain. */
30
+ export const inject = ['tools', 'storageDomain']
31
+
32
+ export { Config, resolveConfig } from './config.ts'
33
+ export type { Config as DataQualityConfig, ResolvedConfig } from './config.ts'
34
+ export { VERSION } from './version.ts'
35
+ export {
36
+ DataQualityService,
37
+ type CitationCheckRequest,
38
+ type CitationCheckResult,
39
+ type ProfileRequest,
40
+ type CleanRequest,
41
+ type CleanRunReport,
42
+ type VerifyRequest,
43
+ } from './service.ts'
44
+ export { LocalDataQualityService, type ProviderDeps } from './provider-local.ts'
45
+ export type { CleanRule, CleanRuleLog, CleanResult } from './clean.ts'
46
+ export type { ProfileReport, ColumnProfile, NumericProfile } from './profile.ts'
47
+ export type { VerifyRule, VerifyReport, VerifyRuleResult, VerifyEvidenceRow } from './verify.ts'
48
+ export { parseLocator, checkCitations } from './verify.ts'
49
+ export { profileTable, renderProfileText } from './profile.ts'
50
+ export { applyCleanRules, serializeDelimited } from './clean.ts'
51
+ export { verifyTable, renderVerifyText } from './verify.ts'
52
+ export {
53
+ DatasetError,
54
+ loadTable,
55
+ loadDocument,
56
+ parseDelimited,
57
+ parseJsonTable,
58
+ resolveWorkspacePath,
59
+ sampleRows,
60
+ isMissing,
61
+ parseNumeric,
62
+ parseDate,
63
+ parseBoolean,
64
+ type Table,
65
+ type Row,
66
+ type Cell,
67
+ type DocumentRoot,
68
+ } from './dataset.ts'
69
+ export { dataQualityDomainSpec, reportKeyOf, reportRecordSchema, type ReportRecord, type ReportStore } from './store.ts'
70
+ export { DATA_QUALITY_EVENT_TYPES, appendDataQualityEvent, type DataQualityEventData, type DataQualityEventType } from './events.ts'
71
+ export { MAX_CELL_TEXT, truncateCell, truncateRow } from './present.ts'
72
+
73
+ /**
74
+ * Mount the seam: resolve config (fail loud), open the report domain, publish
75
+ * `ctx.dataQuality`, and register the three tools. With `enabled: false` the
76
+ * plugin registers nothing and stays inert.
77
+ * @param ctx - the plugin context (host).
78
+ * @param config - raw plugin config.
79
+ */
80
+ export async function apply(ctx: Context, config: Config = {}): Promise<void> {
81
+ const resolved = resolveConfig(config)
82
+ const logger = ctx.logger('data-quality')
83
+ if (!resolved.enabled) {
84
+ logger.info('disabled: enabled is false — no service or tools are mounted')
85
+ return
86
+ }
87
+
88
+ let store: ReportStore | undefined
89
+ let domain: Domain<typeof dataQualityDomainSpec> | undefined
90
+ if (resolved.storeReports) {
91
+ domain = await ctx.storageDomain.open(dataQualityDomainSpec)
92
+ const reports = domain.table('reports')
93
+ store = {
94
+ put: async (record) => {
95
+ const key = reportKeyOf(record)
96
+ await reports.put(key, record)
97
+ return key
98
+ },
99
+ get: (key) => reports.get(key),
100
+ }
101
+ }
102
+
103
+ const service = new LocalDataQualityService(ctx, resolved, { store, now: Date.now })
104
+ ctx.tools.register(defineProfileTool(service))
105
+ ctx.tools.register(defineCleanTool(service))
106
+ ctx.tools.register(defineVerifyTool(service))
107
+ logger.info(`dsh-data-quality ${VERSION} mounted: ctx.dataQuality + data_profile/data_clean/data_verify`)
108
+
109
+ if (domain !== undefined) {
110
+ const handle = domain
111
+ ctx.effect(() => async () => {
112
+ await handle.close()
113
+ })
114
+ }
115
+ }
package/src/present.ts ADDED
@@ -0,0 +1,38 @@
1
+ /**
2
+ * Presentation bounds for tool-facing canonical values: cell text is truncated
3
+ * so one oversized cell cannot flood the model context. This is a display
4
+ * safety invariant, not a deployment tunable — the durable storage-domain
5
+ * report always keeps full fidelity.
6
+ * @module dsh-data-quality/present
7
+ */
8
+
9
+ import type { JsonValue } from '@deepseek-ai/dsh-session'
10
+
11
+ /** Maximum characters one cell contributes to a tool-facing row payload. */
12
+ export const MAX_CELL_TEXT = 120
13
+
14
+ /**
15
+ * Truncate one cell for display: long strings are cut with an ellipsis
16
+ * marker; non-strings pass through.
17
+ * @param cell - the cell to bound.
18
+ * @returns the display-safe cell.
19
+ */
20
+ export function truncateCell(cell: JsonValue): JsonValue {
21
+ if (typeof cell === 'string' && cell.length > MAX_CELL_TEXT) {
22
+ return `${cell.slice(0, MAX_CELL_TEXT)}…`
23
+ }
24
+ return cell
25
+ }
26
+
27
+ /**
28
+ * Truncate every string cell of one row.
29
+ * @param row - the row to bound.
30
+ * @returns a display-safe copy.
31
+ */
32
+ export function truncateRow(row: Record<string, JsonValue>): Record<string, JsonValue> {
33
+ const out: Record<string, JsonValue> = {}
34
+ for (const [key, value] of Object.entries(row)) {
35
+ out[key] = truncateCell(value)
36
+ }
37
+ return out
38
+ }