dsh-data-quality 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/LICENSE +201 -0
- package/README.es.md +181 -0
- package/README.hi.md +181 -0
- package/README.md +181 -0
- package/README.pt.md +181 -0
- package/README.zh.md +181 -0
- package/THIRD_PARTY_NOTICES.md +20 -0
- package/cordis.patch.yml +46 -0
- package/lib/index.js +2458 -0
- package/lib/types/clean.d.ts +82 -0
- package/lib/types/clean.d.ts.map +1 -0
- package/lib/types/clean.js +351 -0
- package/lib/types/clean.js.map +1 -0
- package/lib/types/config.d.ts +47 -0
- package/lib/types/config.d.ts.map +1 -0
- package/lib/types/config.js +66 -0
- package/lib/types/config.js.map +1 -0
- package/lib/types/dataset.d.ts +133 -0
- package/lib/types/dataset.d.ts.map +1 -0
- package/lib/types/dataset.js +404 -0
- package/lib/types/dataset.js.map +1 -0
- package/lib/types/events.d.ts +73 -0
- package/lib/types/events.d.ts.map +1 -0
- package/lib/types/events.js +41 -0
- package/lib/types/events.js.map +1 -0
- package/lib/types/index.d.ts +45 -0
- package/lib/types/index.d.ts.map +1 -0
- package/lib/types/index.js +78 -0
- package/lib/types/index.js.map +1 -0
- package/lib/types/present.d.ts +24 -0
- package/lib/types/present.d.ts.map +1 -0
- package/lib/types/present.js +34 -0
- package/lib/types/present.js.map +1 -0
- package/lib/types/profile.d.ts +79 -0
- package/lib/types/profile.d.ts.map +1 -0
- package/lib/types/profile.js +196 -0
- package/lib/types/profile.js.map +1 -0
- package/lib/types/provider-local.d.ts +56 -0
- package/lib/types/provider-local.d.ts.map +1 -0
- package/lib/types/provider-local.js +163 -0
- package/lib/types/provider-local.js.map +1 -0
- package/lib/types/service.d.ts +160 -0
- package/lib/types/service.d.ts.map +1 -0
- package/lib/types/service.js +25 -0
- package/lib/types/service.js.map +1 -0
- package/lib/types/store.d.ts +61 -0
- package/lib/types/store.d.ts.map +1 -0
- package/lib/types/store.js +42 -0
- package/lib/types/store.js.map +1 -0
- package/lib/types/tools/clean.d.ts +14 -0
- package/lib/types/tools/clean.d.ts.map +1 -0
- package/lib/types/tools/clean.js +146 -0
- package/lib/types/tools/clean.js.map +1 -0
- package/lib/types/tools/profile.d.ts +13 -0
- package/lib/types/tools/profile.d.ts.map +1 -0
- package/lib/types/tools/profile.js +91 -0
- package/lib/types/tools/profile.js.map +1 -0
- package/lib/types/tools/shared.d.ts +19 -0
- package/lib/types/tools/shared.d.ts.map +1 -0
- package/lib/types/tools/shared.js +44 -0
- package/lib/types/tools/shared.js.map +1 -0
- package/lib/types/tools/verify.d.ts +14 -0
- package/lib/types/tools/verify.d.ts.map +1 -0
- package/lib/types/tools/verify.js +160 -0
- package/lib/types/tools/verify.js.map +1 -0
- package/lib/types/verify.d.ts +124 -0
- package/lib/types/verify.d.ts.map +1 -0
- package/lib/types/verify.js +391 -0
- package/lib/types/verify.js.map +1 -0
- package/lib/types/version.d.ts +8 -0
- package/lib/types/version.d.ts.map +1 -0
- package/lib/types/version.js +8 -0
- package/lib/types/version.js.map +1 -0
- package/package.json +137 -0
- package/src/clean.ts +382 -0
- package/src/config.ts +104 -0
- package/src/dataset.ts +445 -0
- package/src/events.ts +90 -0
- package/src/index.ts +115 -0
- package/src/present.ts +38 -0
- package/src/profile.ts +250 -0
- package/src/provider-local.ts +194 -0
- package/src/service.ts +172 -0
- package/src/store.ts +74 -0
- package/src/tools/clean.ts +150 -0
- package/src/tools/profile.ts +94 -0
- package/src/tools/shared.ts +47 -0
- package/src/tools/verify.ts +163 -0
- package/src/verify.ts +496 -0
- package/src/version.ts +8 -0
package/src/dataset.ts
ADDED
|
@@ -0,0 +1,445 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Dataset loading for `dsh-data-quality`: workspace-confined path resolution,
|
|
3
|
+
* size/row guards, and deterministic parsers for CSV/TSV/JSON/JSONL. All
|
|
4
|
+
* parsing is hand-rolled (no external runtime dependencies) and bounded by
|
|
5
|
+
* the resolved config; oversized inputs reject loudly with actionable hints.
|
|
6
|
+
* @module dsh-data-quality/dataset
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { open, stat } from 'node:fs/promises'
|
|
10
|
+
import path from 'node:path'
|
|
11
|
+
import { createHash } from 'node:crypto'
|
|
12
|
+
import type { JsonValue } from '@deepseek-ai/dsh-session'
|
|
13
|
+
import type { ResolvedConfig } from './config.ts'
|
|
14
|
+
|
|
15
|
+
/** One cell of a tabular dataset: a JSON scalar, `null` for missing, or a nested JSON value (document rows). */
|
|
16
|
+
export type Cell = JsonValue
|
|
17
|
+
|
|
18
|
+
/** One dataset row keyed by column name. */
|
|
19
|
+
export type Row = Record<string, Cell>
|
|
20
|
+
|
|
21
|
+
/** A tabular dataset: ordered columns plus rows. */
|
|
22
|
+
export interface Table {
|
|
23
|
+
/** Column names in file order. */
|
|
24
|
+
readonly columns: string[]
|
|
25
|
+
/** Rows, each carrying every declared column (missing cells are `null`). */
|
|
26
|
+
readonly rows: Row[]
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** The root form a document load returns (citation checking walks this). */
|
|
30
|
+
export type DocumentRoot =
|
|
31
|
+
| { readonly kind: 'table'; readonly columns: string[]; readonly rows: Row[] }
|
|
32
|
+
| { readonly kind: 'json'; readonly value: unknown }
|
|
33
|
+
|
|
34
|
+
/** Raised for every user-facing dataset failure; `code` is stable for tests and tooling. */
|
|
35
|
+
export class DatasetError extends Error {
|
|
36
|
+
/**
|
|
37
|
+
* @param code - stable machine-readable reason.
|
|
38
|
+
* @param message - actionable human-readable detail.
|
|
39
|
+
*/
|
|
40
|
+
constructor(
|
|
41
|
+
readonly code:
|
|
42
|
+
| 'path-escape'
|
|
43
|
+
| 'extension-denied'
|
|
44
|
+
| 'not-found'
|
|
45
|
+
| 'too-large'
|
|
46
|
+
| 'too-many-rows'
|
|
47
|
+
| 'malformed'
|
|
48
|
+
| 'not-tabular',
|
|
49
|
+
message: string,
|
|
50
|
+
) {
|
|
51
|
+
super(message)
|
|
52
|
+
this.name = 'DatasetError'
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Resolve `requested` inside `root`, rejecting escapes and disallowed
|
|
58
|
+
* extensions. Both sides go through `path.resolve` before comparison so
|
|
59
|
+
* Windows backslash/forward-slash mixes never defeat the containment check.
|
|
60
|
+
* @param root - absolute workspace root.
|
|
61
|
+
* @param requested - the caller-supplied path (relative to root, or absolute inside root).
|
|
62
|
+
* @param config - resolved config (extension allowlist).
|
|
63
|
+
* @returns the normalized absolute path inside `root`.
|
|
64
|
+
*/
|
|
65
|
+
export function resolveWorkspacePath(root: string, requested: string, config: ResolvedConfig): string {
|
|
66
|
+
if (requested.trim() === '') {
|
|
67
|
+
throw new DatasetError('path-escape', 'dataset path must not be empty')
|
|
68
|
+
}
|
|
69
|
+
const resolvedRoot = path.resolve(root)
|
|
70
|
+
const resolved = path.resolve(resolvedRoot, requested)
|
|
71
|
+
if (resolved !== resolvedRoot && !resolved.startsWith(resolvedRoot + path.sep)) {
|
|
72
|
+
throw new DatasetError(
|
|
73
|
+
'path-escape',
|
|
74
|
+
`path ${JSON.stringify(requested)} resolves outside the workspace root ${JSON.stringify(resolvedRoot)}`,
|
|
75
|
+
)
|
|
76
|
+
}
|
|
77
|
+
const ext = path.extname(resolved).toLowerCase()
|
|
78
|
+
if (!config.allowedExtensions.includes(ext)) {
|
|
79
|
+
throw new DatasetError(
|
|
80
|
+
'extension-denied',
|
|
81
|
+
`extension ${JSON.stringify(ext)} is not in allowedExtensions (${config.allowedExtensions.join(', ')})`,
|
|
82
|
+
)
|
|
83
|
+
}
|
|
84
|
+
return resolved
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Short deterministic fingerprint of a dataset path, used in storage keys.
|
|
89
|
+
* @param datasetPath - the workspace-relative dataset path as callers gave it.
|
|
90
|
+
* @returns an 8-hex-char fingerprint.
|
|
91
|
+
*/
|
|
92
|
+
export function pathFingerprint(datasetPath: string): string {
|
|
93
|
+
return createHash('sha256').update(datasetPath).digest('hex').slice(0, 8)
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** Assert the abort signal has not fired; throws the signal reason otherwise. */
|
|
97
|
+
export function throwIfAborted(signal: AbortSignal | undefined): void {
|
|
98
|
+
if (signal?.aborted === true) {
|
|
99
|
+
throw signal.reason instanceof Error ? signal.reason : new Error('operation aborted')
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Read a dataset file under the size cap.
|
|
105
|
+
* @param absolutePath - normalized absolute path (from {@link resolveWorkspacePath}).
|
|
106
|
+
* @param config - resolved config (size cap).
|
|
107
|
+
* @param signal - optional abort signal honored around the read.
|
|
108
|
+
* @returns the UTF-8 text.
|
|
109
|
+
*/
|
|
110
|
+
export async function readDatasetText(absolutePath: string, config: ResolvedConfig, signal?: AbortSignal): Promise<string> {
|
|
111
|
+
throwIfAborted(signal)
|
|
112
|
+
let info
|
|
113
|
+
try {
|
|
114
|
+
info = await stat(absolutePath)
|
|
115
|
+
} catch {
|
|
116
|
+
throw new DatasetError('not-found', `dataset file not found: ${absolutePath}`)
|
|
117
|
+
}
|
|
118
|
+
if (!info.isFile()) {
|
|
119
|
+
throw new DatasetError('not-found', `dataset path is not a file: ${absolutePath}`)
|
|
120
|
+
}
|
|
121
|
+
const capBytes = config.maxFileSizeMB * 1024 * 1024
|
|
122
|
+
if (info.size > capBytes) {
|
|
123
|
+
throw new DatasetError(
|
|
124
|
+
'too-large',
|
|
125
|
+
`dataset is ${(info.size / 1024 / 1024).toFixed(1)} MiB, above the ${config.maxFileSizeMB} MiB cap; split the file or raise maxFileSizeMB`,
|
|
126
|
+
)
|
|
127
|
+
}
|
|
128
|
+
const handle = await open(absolutePath, 'r')
|
|
129
|
+
try {
|
|
130
|
+
throwIfAborted(signal)
|
|
131
|
+
return await handle.readFile('utf8')
|
|
132
|
+
} finally {
|
|
133
|
+
await handle.close()
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* Parse CSV/TSV text (RFC-4180-style: quoted fields, `""` escapes, CRLF/LF).
|
|
139
|
+
* The first record is the header; short records pad with `null`, long records
|
|
140
|
+
* fail loud. An empty field parses as `null` (missing).
|
|
141
|
+
* @param text - file text.
|
|
142
|
+
* @param delimiter - field delimiter (`,` or tab).
|
|
143
|
+
* @param config - resolved config (row cap).
|
|
144
|
+
* @param signal - optional abort signal, checked per 1024-record stride.
|
|
145
|
+
* @returns the parsed table.
|
|
146
|
+
*/
|
|
147
|
+
export function parseDelimited(text: string, delimiter: string, config: ResolvedConfig, signal?: AbortSignal): Table {
|
|
148
|
+
const records: string[][] = []
|
|
149
|
+
let field = ''
|
|
150
|
+
let record: string[] = []
|
|
151
|
+
let inQuotes = false
|
|
152
|
+
const pushField = (): void => {
|
|
153
|
+
record.push(field)
|
|
154
|
+
field = ''
|
|
155
|
+
}
|
|
156
|
+
const pushRecord = (): void => {
|
|
157
|
+
pushField()
|
|
158
|
+
// Skip truly blank lines (a record of one empty field), as RFC-4180
|
|
159
|
+
// writers commonly emit a trailing empty line.
|
|
160
|
+
if (record.length === 1 && record[0] === '') {
|
|
161
|
+
record = []
|
|
162
|
+
field = ''
|
|
163
|
+
return
|
|
164
|
+
}
|
|
165
|
+
records.push(record)
|
|
166
|
+
record = []
|
|
167
|
+
if (records.length % 1024 === 0) throwIfAborted(signal)
|
|
168
|
+
if (records.length - 1 > config.maxRows) {
|
|
169
|
+
throw new DatasetError(
|
|
170
|
+
'too-many-rows',
|
|
171
|
+
`dataset exceeds the ${config.maxRows} row cap; use the sample parameter or raise maxRows`,
|
|
172
|
+
)
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
let index = 0
|
|
176
|
+
while (index < text.length) {
|
|
177
|
+
const char = text[index] as string
|
|
178
|
+
if (inQuotes) {
|
|
179
|
+
if (char === '"') {
|
|
180
|
+
if (text[index + 1] === '"') {
|
|
181
|
+
field += '"'
|
|
182
|
+
index += 2
|
|
183
|
+
continue
|
|
184
|
+
}
|
|
185
|
+
inQuotes = false
|
|
186
|
+
index += 1
|
|
187
|
+
continue
|
|
188
|
+
}
|
|
189
|
+
field += char
|
|
190
|
+
index += 1
|
|
191
|
+
continue
|
|
192
|
+
}
|
|
193
|
+
if (char === '"') {
|
|
194
|
+
inQuotes = true
|
|
195
|
+
index += 1
|
|
196
|
+
continue
|
|
197
|
+
}
|
|
198
|
+
if (char === delimiter) {
|
|
199
|
+
pushField()
|
|
200
|
+
index += 1
|
|
201
|
+
continue
|
|
202
|
+
}
|
|
203
|
+
if (char === '\r') {
|
|
204
|
+
index += 1
|
|
205
|
+
continue
|
|
206
|
+
}
|
|
207
|
+
if (char === '\n') {
|
|
208
|
+
pushRecord()
|
|
209
|
+
index += 1
|
|
210
|
+
continue
|
|
211
|
+
}
|
|
212
|
+
field += char
|
|
213
|
+
index += 1
|
|
214
|
+
}
|
|
215
|
+
if (field !== '' || record.length > 0) pushRecord()
|
|
216
|
+
|
|
217
|
+
const header = records[0]
|
|
218
|
+
if (header === undefined || (header.length === 1 && header[0] === '')) {
|
|
219
|
+
throw new DatasetError('malformed', 'dataset has no header row')
|
|
220
|
+
}
|
|
221
|
+
const columns = header.map((name) => name.trim())
|
|
222
|
+
if (columns.some((name) => name === '')) {
|
|
223
|
+
throw new DatasetError('malformed', 'header contains an empty column name')
|
|
224
|
+
}
|
|
225
|
+
if (new Set(columns).size !== columns.length) {
|
|
226
|
+
throw new DatasetError('malformed', 'header contains duplicate column names')
|
|
227
|
+
}
|
|
228
|
+
const rows: Row[] = []
|
|
229
|
+
for (let i = 1; i < records.length; i++) {
|
|
230
|
+
const cells = records[i] as string[]
|
|
231
|
+
if (cells.length > columns.length) {
|
|
232
|
+
throw new DatasetError('malformed', `row ${i + 1} has ${cells.length} fields but the header has ${columns.length}`)
|
|
233
|
+
}
|
|
234
|
+
const out: Row = {}
|
|
235
|
+
for (let c = 0; c < columns.length; c++) {
|
|
236
|
+
const raw = cells[c]
|
|
237
|
+
out[columns[c] as string] = raw === undefined || raw === '' ? null : raw
|
|
238
|
+
}
|
|
239
|
+
rows.push(out)
|
|
240
|
+
}
|
|
241
|
+
return { columns, rows }
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
/**
|
|
245
|
+
* Parse a JSON or JSONL text into a tabular dataset. JSON must be an array of
|
|
246
|
+
* flat objects; JSONL is one object per line. Non-scalar cell values are kept
|
|
247
|
+
* as-is (engines treat them as non-scalar).
|
|
248
|
+
* @param text - file text.
|
|
249
|
+
* @param extension - `.json` or `.jsonl`.
|
|
250
|
+
* @param config - resolved config (row cap).
|
|
251
|
+
* @param signal - optional abort signal.
|
|
252
|
+
* @returns the parsed table.
|
|
253
|
+
*/
|
|
254
|
+
export function parseJsonTable(text: string, extension: string, config: ResolvedConfig, signal?: AbortSignal): Table {
|
|
255
|
+
const values = extension === '.jsonl' ? parseJsonLines(text) : parseJsonDocument(text)
|
|
256
|
+
const items = Array.isArray(values) ? values : [values]
|
|
257
|
+
if (items.length > config.maxRows) {
|
|
258
|
+
throw new DatasetError(
|
|
259
|
+
'too-many-rows',
|
|
260
|
+
`dataset exceeds the ${config.maxRows} row cap; use the sample parameter or raise maxRows`,
|
|
261
|
+
)
|
|
262
|
+
}
|
|
263
|
+
const rows: Row[] = []
|
|
264
|
+
const columns: string[] = []
|
|
265
|
+
const seen = new Set<string>()
|
|
266
|
+
for (const [index, item] of items.entries()) {
|
|
267
|
+
if (index % 1024 === 0) throwIfAborted(signal)
|
|
268
|
+
if (typeof item !== 'object' || item === null || Array.isArray(item)) {
|
|
269
|
+
throw new DatasetError(
|
|
270
|
+
'not-tabular',
|
|
271
|
+
`row ${index + 1} is not a flat object; profiling/cleaning/verification need a tabular dataset (array of objects)`,
|
|
272
|
+
)
|
|
273
|
+
}
|
|
274
|
+
const row = item as Row
|
|
275
|
+
for (const key of Object.keys(row)) {
|
|
276
|
+
if (!seen.has(key)) {
|
|
277
|
+
seen.add(key)
|
|
278
|
+
columns.push(key)
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
rows.push(row)
|
|
282
|
+
}
|
|
283
|
+
return { columns, rows: rows.map((row) => normalizeRow(row, columns)) }
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
/** Fill absent columns with `null` so every row carries every column. */
|
|
287
|
+
function normalizeRow(row: Row, columns: string[]): Row {
|
|
288
|
+
const out: Row = {}
|
|
289
|
+
for (const column of columns) {
|
|
290
|
+
const value = row[column]
|
|
291
|
+
out[column] = value === undefined ? null : value
|
|
292
|
+
}
|
|
293
|
+
return out
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
/** Parse one JSON document; failures become a loud malformed error. */
|
|
297
|
+
function parseJsonDocument(text: string): unknown {
|
|
298
|
+
try {
|
|
299
|
+
return JSON.parse(text) as unknown
|
|
300
|
+
} catch (error) {
|
|
301
|
+
throw new DatasetError('malformed', `invalid JSON: ${error instanceof Error ? error.message : String(error)}`)
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
/** Parse JSONL (one JSON value per non-empty line); failures name the line. */
|
|
306
|
+
function parseJsonLines(text: string): unknown[] {
|
|
307
|
+
const out: unknown[] = []
|
|
308
|
+
const lines = text.split(/\r?\n/u)
|
|
309
|
+
for (const [index, line] of lines.entries()) {
|
|
310
|
+
if (line.trim() === '') continue
|
|
311
|
+
try {
|
|
312
|
+
out.push(JSON.parse(line))
|
|
313
|
+
} catch {
|
|
314
|
+
throw new DatasetError('malformed', `invalid JSON on line ${index + 1}`)
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
return out
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
/**
|
|
321
|
+
* Load a tabular dataset from a file already resolved inside the workspace.
|
|
322
|
+
* @param absolutePath - normalized absolute dataset path.
|
|
323
|
+
* @param config - resolved config.
|
|
324
|
+
* @param signal - optional abort signal.
|
|
325
|
+
* @returns the parsed table.
|
|
326
|
+
*/
|
|
327
|
+
export async function loadTable(absolutePath: string, config: ResolvedConfig, signal?: AbortSignal): Promise<Table> {
|
|
328
|
+
const text = await readDatasetText(absolutePath, config, signal)
|
|
329
|
+
const ext = path.extname(absolutePath).toLowerCase()
|
|
330
|
+
throwIfAborted(signal)
|
|
331
|
+
if (ext === '.csv') return parseDelimited(text, ',', config, signal)
|
|
332
|
+
if (ext === '.tsv') return parseDelimited(text, '\t', config, signal)
|
|
333
|
+
return parseJsonTable(text, ext, config, signal)
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
/**
|
|
337
|
+
* Load a dataset as a citation-checkable document root: CSV/TSV become
|
|
338
|
+
* `{ columns, rows }` (so `rows[3].nav` resolves), JSON is the parsed value,
|
|
339
|
+
* JSONL is the array of parsed lines.
|
|
340
|
+
* @param absolutePath - normalized absolute dataset path.
|
|
341
|
+
* @param config - resolved config.
|
|
342
|
+
* @param signal - optional abort signal.
|
|
343
|
+
* @returns the document root.
|
|
344
|
+
*/
|
|
345
|
+
export async function loadDocument(absolutePath: string, config: ResolvedConfig, signal?: AbortSignal): Promise<DocumentRoot> {
|
|
346
|
+
const ext = path.extname(absolutePath).toLowerCase()
|
|
347
|
+
if (ext === '.json') {
|
|
348
|
+
const text = await readDatasetText(absolutePath, config, signal)
|
|
349
|
+
return { kind: 'json', value: parseJsonDocument(text) }
|
|
350
|
+
}
|
|
351
|
+
if (ext === '.jsonl') {
|
|
352
|
+
const text = await readDatasetText(absolutePath, config, signal)
|
|
353
|
+
return { kind: 'json', value: parseJsonLines(text) }
|
|
354
|
+
}
|
|
355
|
+
const table = await loadTable(absolutePath, config, signal)
|
|
356
|
+
return { kind: 'table', columns: table.columns, rows: table.rows }
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
/**
|
|
360
|
+
* Deterministic systematic sample: every `ceil(total / count)`-th row from
|
|
361
|
+
* index 0. No clock, no RNG — the same input always yields the same sample.
|
|
362
|
+
* @param rows - full row list.
|
|
363
|
+
* @param count - requested sample size.
|
|
364
|
+
* @returns the sampled rows (identity when `count >= rows.length`).
|
|
365
|
+
*/
|
|
366
|
+
export function sampleRows(rows: Row[], count: number): Row[] {
|
|
367
|
+
if (!Number.isSafeInteger(count) || count <= 0) {
|
|
368
|
+
throw new DatasetError('malformed', `sample must be a positive integer, got ${String(count)}`)
|
|
369
|
+
}
|
|
370
|
+
if (count >= rows.length) return rows
|
|
371
|
+
const stride = Math.ceil(rows.length / count)
|
|
372
|
+
const out: Row[] = []
|
|
373
|
+
for (let i = 0; i < rows.length; i += stride) {
|
|
374
|
+
out.push(rows[i] as Row)
|
|
375
|
+
}
|
|
376
|
+
return out
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
/**
|
|
380
|
+
* Missingness: `null`, `undefined`, or a string empty after trimming. A
|
|
381
|
+
* whitespace-only cell counts as missing even before any `trim` clean rule.
|
|
382
|
+
* @param cell - the cell to test (`undefined` when the column is absent).
|
|
383
|
+
* @returns whether the cell is missing.
|
|
384
|
+
*/
|
|
385
|
+
export function isMissing(cell: Cell | undefined): boolean {
|
|
386
|
+
if (cell === null || cell === undefined) return true
|
|
387
|
+
if (typeof cell === 'string') return cell.trim() === ''
|
|
388
|
+
return false
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
/** Strict full-string numeric parse (no thousands separators, no unit suffixes). */
|
|
392
|
+
export function parseNumeric(cell: Cell | undefined): number | undefined {
|
|
393
|
+
if (typeof cell === 'number') return Number.isFinite(cell) ? cell : undefined
|
|
394
|
+
if (typeof cell !== 'string') return undefined
|
|
395
|
+
const text = cell.trim()
|
|
396
|
+
if (text === '') return undefined
|
|
397
|
+
if (!/^[+-]?(?:\d+\.?\d*|\.\d+)(?:[eE][+-]?\d+)?$/u.test(text)) return undefined
|
|
398
|
+
const value = Number(text)
|
|
399
|
+
return Number.isFinite(value) ? value : undefined
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
const DATE_PATTERNS: readonly RegExp[] = [
|
|
403
|
+
/^(\d{4})-(\d{1,2})-(\d{1,2})$/u,
|
|
404
|
+
/^(\d{4})\/(\d{1,2})\/(\d{1,2})$/u,
|
|
405
|
+
/^(\d{4})-(\d{1,2})-(\d{1,2})[ T](\d{1,2}):(\d{2})(?::(\d{2}))?(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?$/u,
|
|
406
|
+
]
|
|
407
|
+
|
|
408
|
+
/**
|
|
409
|
+
* Deterministic date parse to epoch milliseconds. Accepts `YYYY-MM-DD`,
|
|
410
|
+
* `YYYY/MM/DD`, and ISO-like datetimes (date-only forms read as UTC midnight).
|
|
411
|
+
* Calendar-invalid dates (e.g. 2025-13-40) reject. Returns `undefined` when
|
|
412
|
+
* the cell is not a recognized date.
|
|
413
|
+
* @param cell - the cell to parse (`undefined` when the column is absent).
|
|
414
|
+
* @returns epoch milliseconds, or `undefined`.
|
|
415
|
+
*/
|
|
416
|
+
export function parseDate(cell: Cell | undefined): number | undefined {
|
|
417
|
+
if (typeof cell !== 'string') return undefined
|
|
418
|
+
const text = cell.trim()
|
|
419
|
+
for (const pattern of DATE_PATTERNS) {
|
|
420
|
+
const match = pattern.exec(text)
|
|
421
|
+
if (match === null) continue
|
|
422
|
+
const year = Number(match[1])
|
|
423
|
+
const month = Number(match[2])
|
|
424
|
+
const day = Number(match[3])
|
|
425
|
+
const hour = Number(match[4] ?? 0)
|
|
426
|
+
const minute = Number(match[5] ?? 0)
|
|
427
|
+
const second = Number(match[6] ?? 0)
|
|
428
|
+
if (month < 1 || month > 12 || day < 1 || day > 31 || hour > 23 || minute > 59 || second > 59) return undefined
|
|
429
|
+
const epoch = Date.UTC(year, month - 1, day, hour, minute, second)
|
|
430
|
+
const check = new Date(epoch)
|
|
431
|
+
if (check.getUTCMonth() !== month - 1 || check.getUTCDate() !== day) return undefined
|
|
432
|
+
return epoch
|
|
433
|
+
}
|
|
434
|
+
return undefined
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
/** Boolean parse: true/false/yes/no/1/0, case-insensitive. */
|
|
438
|
+
export function parseBoolean(cell: Cell | undefined): boolean | undefined {
|
|
439
|
+
if (typeof cell === 'boolean') return cell
|
|
440
|
+
if (typeof cell !== 'string') return undefined
|
|
441
|
+
const text = cell.trim().toLowerCase()
|
|
442
|
+
if (text === 'true' || text === 'yes' || text === '1') return true
|
|
443
|
+
if (text === 'false' || text === 'no' || text === '0') return false
|
|
444
|
+
return undefined
|
|
445
|
+
}
|
package/src/events.ts
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The `data-quality/*` session-event vocabulary and its adaptive append gate.
|
|
3
|
+
* The vocabulary is declared on `SessionEventMap` for type-checked payloads;
|
|
4
|
+
* the runtime gate decides per host whether appending is safe:
|
|
5
|
+
*
|
|
6
|
+
* - Hosts whose known-type set already covers the vocabulary (a future harness
|
|
7
|
+
* that adopts these events) append plainly.
|
|
8
|
+
* - Hosts with an `ignorable` append option (the master-build
|
|
9
|
+
* `Session.append(type, data, { ignorable: true })` contract) append with the
|
|
10
|
+
* marker, so builds that do not know the type skip the event on restore.
|
|
11
|
+
* - 0.1.0-rc.6 hosts have neither a registration surface nor the `ignorable`
|
|
12
|
+
* append flag; appending an unknown type there would make the persistence
|
|
13
|
+
* coordinator refuse the session log on restore, so the append is skipped
|
|
14
|
+
* and the storage-domain report remains the durable copy.
|
|
15
|
+
* @module dsh-data-quality/events
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
import { KNOWN_SESSION_EVENT_TYPES, type Session } from '@deepseek-ai/dsh-session'
|
|
19
|
+
|
|
20
|
+
/** Payload shared by every `data-quality/*` session event. */
|
|
21
|
+
export interface DataQualityEventData {
|
|
22
|
+
/** Which run produced the event. */
|
|
23
|
+
readonly kind: 'profile' | 'clean' | 'verify'
|
|
24
|
+
/** Workspace-relative dataset path as the caller gave it. */
|
|
25
|
+
readonly dataset: string
|
|
26
|
+
/** Storage-domain key of the full persisted report, when persistence is on. */
|
|
27
|
+
readonly reportKey?: string
|
|
28
|
+
/** Compact run summary (small integers/booleans only). */
|
|
29
|
+
readonly summary: {
|
|
30
|
+
readonly rows: number
|
|
31
|
+
readonly columns?: number
|
|
32
|
+
readonly rules?: number
|
|
33
|
+
readonly failedRules?: number
|
|
34
|
+
readonly passed?: boolean
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
declare module '@deepseek-ai/dsh-session/types' {
|
|
39
|
+
interface SessionEventMap {
|
|
40
|
+
/**
|
|
41
|
+
* A `data_profile` run completed.
|
|
42
|
+
* @mode emit
|
|
43
|
+
* @param data - run kind, dataset, optional report key, and compact summary.
|
|
44
|
+
*/
|
|
45
|
+
'data-quality/profile': DataQualityEventData
|
|
46
|
+
/**
|
|
47
|
+
* A `data_clean` run completed.
|
|
48
|
+
* @mode emit
|
|
49
|
+
* @param data - run kind, dataset, optional report key, and compact summary.
|
|
50
|
+
*/
|
|
51
|
+
'data-quality/clean': DataQualityEventData
|
|
52
|
+
/**
|
|
53
|
+
* A `data_verify` run completed.
|
|
54
|
+
* @mode emit
|
|
55
|
+
* @param data - run kind, dataset, optional report key, and compact summary.
|
|
56
|
+
*/
|
|
57
|
+
'data-quality/verify': DataQualityEventData
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** The event type vocabulary this plugin appends. */
|
|
62
|
+
export const DATA_QUALITY_EVENT_TYPES = ['data-quality/profile', 'data-quality/clean', 'data-quality/verify'] as const
|
|
63
|
+
|
|
64
|
+
/** Union of the event types this plugin appends. */
|
|
65
|
+
export type DataQualityEventType = (typeof DATA_QUALITY_EVENT_TYPES)[number]
|
|
66
|
+
|
|
67
|
+
/** Loose append shape probed at runtime (rc.6 takes no options; master takes `ignorable`). */
|
|
68
|
+
type AppendProbe = (type: string, data: unknown, options?: { ignorable: true }) => unknown
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Append one `data-quality/*` event when the host can carry it safely; skip
|
|
72
|
+
* silently otherwise (the storage-domain report is always the durable copy).
|
|
73
|
+
* The `ignorable` probe reads the UNBOUND method's source (a `.bind()` result
|
|
74
|
+
* reports `[native code]`): the rc.6 build contains no `ignorable` handling
|
|
75
|
+
* while the master build references the flag by name; property names survive
|
|
76
|
+
* minification, so the probe fails safe (skips) rather than corrupting a log.
|
|
77
|
+
* @param session - the calling session.
|
|
78
|
+
* @param type - the event type.
|
|
79
|
+
* @param data - the payload.
|
|
80
|
+
*/
|
|
81
|
+
export function appendDataQualityEvent(session: Session, type: DataQualityEventType, data: DataQualityEventData): void {
|
|
82
|
+
if (KNOWN_SESSION_EVENT_TYPES.has(type)) {
|
|
83
|
+
session.append(type, data)
|
|
84
|
+
return
|
|
85
|
+
}
|
|
86
|
+
const append = session.append as AppendProbe
|
|
87
|
+
if (Function.prototype.toString.call(append).includes('ignorable')) {
|
|
88
|
+
append.call(session, type, data, { ignorable: true })
|
|
89
|
+
}
|
|
90
|
+
}
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `dsh-data-quality` — deterministic data profiling, cleaning, and
|
|
3
|
+
* verification for DeepSeek Harness. Mounts the `ctx.dataQuality` capability
|
|
4
|
+
* seam (Service Definition in `service.ts`, local Provider in
|
|
5
|
+
* `provider-local.ts`), registers the `data_profile` / `data_clean` /
|
|
6
|
+
* `data_verify` model tools (Consumers — all computation goes through the
|
|
7
|
+
* service layer), persists run reports to the `data_quality` storage domain,
|
|
8
|
+
* and appends adaptive `data-quality/*` session events on hosts that can
|
|
9
|
+
* carry them.
|
|
10
|
+
*
|
|
11
|
+
* Function plugin — no default export (the Loader unwraps
|
|
12
|
+
* `exports.default ?? exports`, and a stray default would discard
|
|
13
|
+
* `name`/`inject`/`Config`/`apply`).
|
|
14
|
+
* @module dsh-data-quality
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import type { Context } from '@deepseek-ai/cordis'
|
|
18
|
+
// Type-only: registers the `ctx.storageDomain` Context merge for the inject.
|
|
19
|
+
import type { Domain } from '@deepseek-ai/dsh-storage-domain'
|
|
20
|
+
import { Config, resolveConfig } from './config.ts'
|
|
21
|
+
import { dataQualityDomainSpec, reportKeyOf, type ReportStore } from './store.ts'
|
|
22
|
+
import { LocalDataQualityService } from './provider-local.ts'
|
|
23
|
+
import { defineProfileTool } from './tools/profile.ts'
|
|
24
|
+
import { defineCleanTool } from './tools/clean.ts'
|
|
25
|
+
import { defineVerifyTool } from './tools/verify.ts'
|
|
26
|
+
import { VERSION } from './version.ts'
|
|
27
|
+
|
|
28
|
+
export const name = 'data-quality'
|
|
29
|
+
/** The three model tools and the durable report domain. */
|
|
30
|
+
export const inject = ['tools', 'storageDomain']
|
|
31
|
+
|
|
32
|
+
export { Config, resolveConfig } from './config.ts'
|
|
33
|
+
export type { Config as DataQualityConfig, ResolvedConfig } from './config.ts'
|
|
34
|
+
export { VERSION } from './version.ts'
|
|
35
|
+
export {
|
|
36
|
+
DataQualityService,
|
|
37
|
+
type CitationCheckRequest,
|
|
38
|
+
type CitationCheckResult,
|
|
39
|
+
type ProfileRequest,
|
|
40
|
+
type CleanRequest,
|
|
41
|
+
type CleanRunReport,
|
|
42
|
+
type VerifyRequest,
|
|
43
|
+
} from './service.ts'
|
|
44
|
+
export { LocalDataQualityService, type ProviderDeps } from './provider-local.ts'
|
|
45
|
+
export type { CleanRule, CleanRuleLog, CleanResult } from './clean.ts'
|
|
46
|
+
export type { ProfileReport, ColumnProfile, NumericProfile } from './profile.ts'
|
|
47
|
+
export type { VerifyRule, VerifyReport, VerifyRuleResult, VerifyEvidenceRow } from './verify.ts'
|
|
48
|
+
export { parseLocator, checkCitations } from './verify.ts'
|
|
49
|
+
export { profileTable, renderProfileText } from './profile.ts'
|
|
50
|
+
export { applyCleanRules, serializeDelimited } from './clean.ts'
|
|
51
|
+
export { verifyTable, renderVerifyText } from './verify.ts'
|
|
52
|
+
export {
|
|
53
|
+
DatasetError,
|
|
54
|
+
loadTable,
|
|
55
|
+
loadDocument,
|
|
56
|
+
parseDelimited,
|
|
57
|
+
parseJsonTable,
|
|
58
|
+
resolveWorkspacePath,
|
|
59
|
+
sampleRows,
|
|
60
|
+
isMissing,
|
|
61
|
+
parseNumeric,
|
|
62
|
+
parseDate,
|
|
63
|
+
parseBoolean,
|
|
64
|
+
type Table,
|
|
65
|
+
type Row,
|
|
66
|
+
type Cell,
|
|
67
|
+
type DocumentRoot,
|
|
68
|
+
} from './dataset.ts'
|
|
69
|
+
export { dataQualityDomainSpec, reportKeyOf, reportRecordSchema, type ReportRecord, type ReportStore } from './store.ts'
|
|
70
|
+
export { DATA_QUALITY_EVENT_TYPES, appendDataQualityEvent, type DataQualityEventData, type DataQualityEventType } from './events.ts'
|
|
71
|
+
export { MAX_CELL_TEXT, truncateCell, truncateRow } from './present.ts'
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Mount the seam: resolve config (fail loud), open the report domain, publish
|
|
75
|
+
* `ctx.dataQuality`, and register the three tools. With `enabled: false` the
|
|
76
|
+
* plugin registers nothing and stays inert.
|
|
77
|
+
* @param ctx - the plugin context (host).
|
|
78
|
+
* @param config - raw plugin config.
|
|
79
|
+
*/
|
|
80
|
+
export async function apply(ctx: Context, config: Config = {}): Promise<void> {
|
|
81
|
+
const resolved = resolveConfig(config)
|
|
82
|
+
const logger = ctx.logger('data-quality')
|
|
83
|
+
if (!resolved.enabled) {
|
|
84
|
+
logger.info('disabled: enabled is false — no service or tools are mounted')
|
|
85
|
+
return
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
let store: ReportStore | undefined
|
|
89
|
+
let domain: Domain<typeof dataQualityDomainSpec> | undefined
|
|
90
|
+
if (resolved.storeReports) {
|
|
91
|
+
domain = await ctx.storageDomain.open(dataQualityDomainSpec)
|
|
92
|
+
const reports = domain.table('reports')
|
|
93
|
+
store = {
|
|
94
|
+
put: async (record) => {
|
|
95
|
+
const key = reportKeyOf(record)
|
|
96
|
+
await reports.put(key, record)
|
|
97
|
+
return key
|
|
98
|
+
},
|
|
99
|
+
get: (key) => reports.get(key),
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
const service = new LocalDataQualityService(ctx, resolved, { store, now: Date.now })
|
|
104
|
+
ctx.tools.register(defineProfileTool(service))
|
|
105
|
+
ctx.tools.register(defineCleanTool(service))
|
|
106
|
+
ctx.tools.register(defineVerifyTool(service))
|
|
107
|
+
logger.info(`dsh-data-quality ${VERSION} mounted: ctx.dataQuality + data_profile/data_clean/data_verify`)
|
|
108
|
+
|
|
109
|
+
if (domain !== undefined) {
|
|
110
|
+
const handle = domain
|
|
111
|
+
ctx.effect(() => async () => {
|
|
112
|
+
await handle.close()
|
|
113
|
+
})
|
|
114
|
+
}
|
|
115
|
+
}
|
package/src/present.ts
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Presentation bounds for tool-facing canonical values: cell text is truncated
|
|
3
|
+
* so one oversized cell cannot flood the model context. This is a display
|
|
4
|
+
* safety invariant, not a deployment tunable — the durable storage-domain
|
|
5
|
+
* report always keeps full fidelity.
|
|
6
|
+
* @module dsh-data-quality/present
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import type { JsonValue } from '@deepseek-ai/dsh-session'
|
|
10
|
+
|
|
11
|
+
/** Maximum characters one cell contributes to a tool-facing row payload. */
|
|
12
|
+
export const MAX_CELL_TEXT = 120
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Truncate one cell for display: long strings are cut with an ellipsis
|
|
16
|
+
* marker; non-strings pass through.
|
|
17
|
+
* @param cell - the cell to bound.
|
|
18
|
+
* @returns the display-safe cell.
|
|
19
|
+
*/
|
|
20
|
+
export function truncateCell(cell: JsonValue): JsonValue {
|
|
21
|
+
if (typeof cell === 'string' && cell.length > MAX_CELL_TEXT) {
|
|
22
|
+
return `${cell.slice(0, MAX_CELL_TEXT)}…`
|
|
23
|
+
}
|
|
24
|
+
return cell
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Truncate every string cell of one row.
|
|
29
|
+
* @param row - the row to bound.
|
|
30
|
+
* @returns a display-safe copy.
|
|
31
|
+
*/
|
|
32
|
+
export function truncateRow(row: Record<string, JsonValue>): Record<string, JsonValue> {
|
|
33
|
+
const out: Record<string, JsonValue> = {}
|
|
34
|
+
for (const [key, value] of Object.entries(row)) {
|
|
35
|
+
out[key] = truncateCell(value)
|
|
36
|
+
}
|
|
37
|
+
return out
|
|
38
|
+
}
|