dsh-data-quality 0.1.3 → 0.3.0

package/src/presets.ts

@@ -0,0 +1,146 @@
+/**
+ * Built-in industry profile presets: a deterministic registry of expected
+ * column lists (name -> declared type) per industry. When `data_profile` is
+ * called with `industryPreset`, the preset's columns feed the scorecard's
+ * `declaredSchema` so the `accuracy` dimension becomes determinable instead
+ * of undetermined. Unknown preset ids fail loud.
+ * @module dsh-data-quality/presets
+ */
+
+import type { InferredType } from './profile.ts'
+
+/** One industry preset: an id, a human label, and the expected columns with their declared types. */
+export interface IndustryPreset {
+  readonly id: string
+  readonly label: string
+  /** Expected column name -> declared type (fed to the scorecard as `declaredSchema`). */
+  readonly columns: Readonly<Record<string, InferredType>>
+}
+
+/** The built-in industry preset registry. */
+export const INDUSTRY_PRESETS: Readonly<Record<string, IndustryPreset>> = {
+  retail: {
+    id: 'retail',
+    label: 'Retail',
+    columns: {
+      order_id: 'string',
+      sku: 'string',
+      quantity: 'number',
+      unit_price: 'number',
+      revenue: 'number',
+      order_date: 'date',
+      customer_id: 'string',
+    },
+  },
+  saas: {
+    id: 'saas',
+    label: 'SaaS',
+    columns: {
+      account_id: 'string',
+      plan: 'string',
+      seats: 'number',
+      mrr: 'number',
+      signup_date: 'date',
+      churned: 'boolean',
+    },
+  },
+  fund: {
+    id: 'fund',
+    label: 'Fund',
+    columns: {
+      fund_code: 'string',
+      fund_name: 'string',
+      nav: 'number',
+      nav_date: 'date',
+      holding_value: 'number',
+      currency: 'string',
+    },
+  },
+  'real-estate': {
+    id: 'real-estate',
+    label: 'Real estate',
+    columns: {
+      property_id: 'string',
+      listing_price: 'number',
+      area_sqm: 'number',
+      bedrooms: 'number',
+      list_date: 'date',
+      city: 'string',
+    },
+  },
+  'e-commerce': {
+    id: 'e-commerce',
+    label: 'E-commerce',
+    columns: {
+      order_id: 'string',
+      product_id: 'string',
+      quantity: 'number',
+      price: 'number',
+      order_date: 'date',
+      status: 'string',
+    },
+  },
+  healthcare: {
+    id: 'healthcare',
+    label: 'Healthcare',
+    columns: {
+      patient_id: 'string',
+      admission_date: 'date',
+      discharge_date: 'date',
+      age: 'number',
+      diagnosis: 'string',
+      cost: 'number',
+    },
+  },
+  logistics: {
+    id: 'logistics',
+    label: 'Logistics',
+    columns: {
+      shipment_id: 'string',
+      origin: 'string',
+      destination: 'string',
+      weight_kg: 'number',
+      shipped_date: 'date',
+      delivered: 'boolean',
+    },
+  },
+  manufacturing: {
+    id: 'manufacturing',
+    label: 'Manufacturing',
+    columns: {
+      part_id: 'string',
+      quantity: 'number',
+      unit_cost: 'number',
+      produced_date: 'date',
+      defect: 'boolean',
+    },
+  },
+  energy: {
+    id: 'energy',
+    label: 'Energy',
+    columns: {
+      meter_id: 'string',
+      reading: 'number',
+      unit: 'string',
+      reading_date: 'date',
+      consumption: 'number',
+    },
+  },
+}
+
+/** The preset ids, for diagnostics and documentation. */
+export const INDUSTRY_PRESET_IDS = Object.keys(INDUSTRY_PRESETS) as readonly string[]
+
+/**
+ * Resolve an industry preset id to its registry entry, failing loud on an
+ * unknown id.
+ * @param id - the requested preset id.
+ * @returns the preset.
+ */
+export function resolveIndustryPreset(id: string): IndustryPreset {
+  const preset = INDUSTRY_PRESETS[id]
+  if (preset === undefined) {
+    throw new Error(`unknown industryPreset ${JSON.stringify(id)}; choose one of: ${INDUSTRY_PRESET_IDS.join(', ')}`)
+  }
+  return preset
+}

package/src/profile.ts

@@ -6,13 +6,20 @@
  * @module dsh-data-quality/profile
  */
 
-import { isMissing, parseBoolean, parseDate, parseNumeric, sampleRows, throwIfAborted, type Row, type Table } from './dataset.ts'
+import { createHash } from 'node:crypto'
+import { isMissing, parseBoolean, parseDate, parseNumeric, sampleRows, throwIfAborted, type EncodingInfo, type Row, type Table } from './dataset.ts'
+import { computeScorecard, type DataQualityScorecard, type ScorecardDimensionName } from './scorecard.ts'
+import { REPORT_SCHEMA_VERSION } from './version.ts'
 
 /** Inferred column type from parsed cell classes. */
 export type InferredType = 'number' | 'date' | 'boolean' | 'string' | 'empty' | 'mixed'
 
 /** Numeric distribution of one numeric column. */
 export interface NumericProfile {
+  /** Number of numeric values the distribution covers. */
+  readonly count: number
+  /** Distinct numeric values among the profiled rows. */
+  readonly distinct: number
   readonly min: number
   readonly max: number
   readonly mean: number
@@ -41,8 +48,20 @@ export interface ColumnProfile {
   notes: string[]
 }
 
+/** Full-content duplicate detection over ALL rows. */
+export interface DuplicateDetection {
+  /** Rows whose full sha256 content duplicates an earlier row. */
+  readonly duplicateRows: number
+  /** `duplicateRows / rowCount` (0 when the table has no rows). */
+  readonly duplicateRate: number
+  /** 0-based indexes of duplicated rows, capped by the sample limit. */
+  readonly duplicateSampleRowIndexes: number[]
+}
+
 /** The full profile report (also the persisted and tool-returned value). */
 export interface ProfileReport {
+  /** The persisted-report schema version (see {@link REPORT_SCHEMA_VERSION}). */
+  readonly schemaVersion: number
   readonly dataset: string
   readonly rowCount: number
   /** Whether column cards describe a systematic sample rather than every row. */
@@ -52,6 +71,14 @@ export interface ProfileReport {
   readonly columnCount: number
   /** Rows whose full content duplicates an earlier row (over ALL rows). */
   readonly duplicateRows: number
+  /** `duplicateRows / rowCount` (0 when the table has no rows). */
+  readonly duplicateRate: number
+  /** 0-based indexes of duplicated rows (capped by the configured sample limit). */
+  readonly duplicateSampleRowIndexes: number[]
+  /** DAMA six-dimension quality scorecard. */
+  readonly scorecard: DataQualityScorecard
+  /** Detected file encoding (BOM/UTF-8 validity); present only for file-loaded tables. */
+  readonly encoding?: EncodingInfo
   columns: ColumnProfile[]
   /** Storage-domain key of the persisted report, when persistence is on (set by the provider). */
   readonly reportKey?: string
@@ -59,6 +86,9 @@ export interface ProfileReport {
   readonly generatedAt: number
 }
 
+/** Fallback duplicate-sample cap for direct engine use; the provider always passes the configured `evidenceRowLimit`. */
+const DEFAULT_DUPLICATE_SAMPLE_LIMIT = 20
+
 /** Round to 6 significant digits for stable, readable report numbers. */
 function round6(value: number): number {
   return Number(value.toPrecision(6))
@@ -86,6 +116,8 @@ export function numericProfile(values: readonly number[]): NumericProfile | unde
   const highFence = p75 + 1.5 * iqr
   const outliers = iqr === 0 ? 0 : sorted.filter((value) => value < lowFence || value > highFence).length
   return {
+    count: sorted.length,
+    distinct: new Set(sorted).size,
     min: round6(sorted[0] as number),
     max: round6(sorted[sorted.length - 1] as number),
     mean: round6(sum / sorted.length),
@@ -96,20 +128,49 @@ export function numericProfile(values: readonly number[]): NumericProfile | unde
   }
 }
 
-/** Count rows whose full content duplicates an earlier row (first occurrence is not counted). */
-export function countDuplicateRows(table: Table, signal?: AbortSignal): number {
+/** Deterministic sha256 key of one row's full content (columns in table order). */
+function rowContentKey(table: Table, row: Row): string {
+  return createHash('sha256').update(JSON.stringify(table.columns.map((column) => row[column] ?? null))).digest('hex')
+}
+
+/**
+ * Detect full-content duplicate rows with a bounded sample of their 0-based
+ * indexes. The first occurrence of each content is never counted; later rows
+ * with identical full content are duplicates.
+ * @param table - the parsed dataset.
+ * @param options - sample cap and optional abort signal.
+ * @returns the duplicate count, rate, and capped sample indexes.
+ */
+export function detectDuplicateRows(
+  table: Table,
+  options: { sampleLimit: number; signal?: AbortSignal | undefined },
+): DuplicateDetection {
+  if (!Number.isSafeInteger(options.sampleLimit) || options.sampleLimit <= 0) {
+    throw new TypeError(`sampleLimit must be a positive safe integer, got ${String(options.sampleLimit)}`)
+  }
   const seen = new Set<string>()
   let duplicates = 0
+  const duplicateSampleRowIndexes: number[] = []
   for (const [index, row] of table.rows.entries()) {
-    if (index % 1024 === 0) throwIfAborted(signal)
-    const key = JSON.stringify(table.columns.map((column) => row[column] ?? null))
+    if (index % 1024 === 0) throwIfAborted(options.signal)
+    const key = rowContentKey(table, row)
     if (seen.has(key)) {
       duplicates += 1
+      if (duplicateSampleRowIndexes.length < options.sampleLimit) duplicateSampleRowIndexes.push(index)
     } else {
       seen.add(key)
     }
   }
-  return duplicates
+  return {
+    duplicateRows: duplicates,
+    duplicateRate: table.rows.length === 0 ? 0 : round6(duplicates / table.rows.length),
+    duplicateSampleRowIndexes,
+  }
+}
+
+/** Count rows whose full content duplicates an earlier row (first occurrence is not counted). */
+export function countDuplicateRows(table: Table, signal?: AbortSignal): number {
+  return detectDuplicateRows(table, { sampleLimit: 1, signal }).duplicateRows
 }
 
 /** Profile one column over the given rows. */
@@ -206,19 +267,39 @@ function profileColumn(rows: readonly Row[], column: string, signal?: AbortSigna
  */
 export function profileTable(
   table: Table,
-  options: { dataset: string; sample?: number | undefined; generatedAt: number; signal?: AbortSignal | undefined },
+  options: {
+    dataset: string
+    sample?: number | undefined
+    generatedAt: number
+    signal?: AbortSignal | undefined
+    duplicateSampleLimit?: number | undefined
+    declaredSchema?: Readonly<Record<string, InferredType>> | undefined
+    scorecardWeights?: Readonly<Record<ScorecardDimensionName, number>> | undefined
+  },
 ): ProfileReport {
   throwIfAborted(options.signal)
   const profiled = options.sample === undefined ? table.rows : sampleRows(table.rows, options.sample)
   const columns = table.columns.map((column) => profileColumn(profiled, column, options.signal))
-  const duplicateRows = countDuplicateRows(table, options.signal)
+  const detection = detectDuplicateRows(table, { sampleLimit: options.duplicateSampleLimit ?? DEFAULT_DUPLICATE_SAMPLE_LIMIT, signal: options.signal })
+  const scorecard = computeScorecard(table, {
+    now: options.generatedAt,
+    duplicateRows: detection.duplicateRows,
+    declaredSchema: options.declaredSchema,
+    weights: options.scorecardWeights,
+    signal: options.signal,
+  })
   return {
+    schemaVersion: REPORT_SCHEMA_VERSION,
     dataset: options.dataset,
     rowCount: table.rows.length,
     sampled: profiled.length !== table.rows.length,
     profiledRows: profiled.length,
     columnCount: table.columns.length,
-    duplicateRows,
+    duplicateRows: detection.duplicateRows,
+    duplicateRate: detection.duplicateRate,
+    duplicateSampleRowIndexes: detection.duplicateSampleRowIndexes,
+    scorecard,
+    ...(table.encoding !== undefined ? { encoding: table.encoding } : {}),
     columns,
     generatedAt: options.generatedAt,
   }
@@ -229,14 +310,17 @@ export function renderProfileText(report: ProfileReport): string {
   const lines: string[] = []
   lines.push(`Profile of ${report.dataset}: ${report.rowCount} rows x ${report.columnCount} columns` +
     (report.sampled ? ` (column cards over a systematic sample of ${report.profiledRows} rows)` : ''))
-  if (report.duplicateRows > 0) lines.push(`Duplicate rows: ${report.duplicateRows}`)
+  if (report.duplicateRows > 0) {
+    lines.push(`Duplicate rows: ${report.duplicateRows} (${(report.duplicateRate * 100).toFixed(1)}%)` +
+      (report.duplicateSampleRowIndexes.length > 0 ? `; sample row indexes: ${report.duplicateSampleRowIndexes.join(', ')}` : ''))
+  }
   for (const column of report.columns) {
     const parts = [`${column.name}: ${column.inferredType}`]
     if (column.missing > 0) parts.push(`missing ${column.missing} (${(column.missingRate * 100).toFixed(1)}%)`)
     parts.push(`unique ${column.unique}`)
     if (column.numeric !== undefined) {
       parts.push(
-        `min ${column.numeric.min}, p25 ${column.numeric.p25}, median ${column.numeric.median}, p75 ${column.numeric.p75}, max ${column.numeric.max}, mean ${column.numeric.mean}` +
+        `count ${column.numeric.count}, distinct ${column.numeric.distinct}, min ${column.numeric.min}, p25 ${column.numeric.p25}, median ${column.numeric.median}, p75 ${column.numeric.p75}, max ${column.numeric.max}, mean ${column.numeric.mean}` +
           (column.numeric.outliers > 0 ? `, ${column.numeric.outliers} IQR outliers` : ''),
       )
     }
@@ -246,5 +330,15 @@ export function renderProfileText(report: ProfileReport): string {
     for (const note of column.notes) parts.push(`note: ${note}`)
     lines.push(`- ${parts.join('; ')}`)
   }
+  if (report.encoding !== undefined) {
+    lines.push(`Encoding: UTF-8${report.encoding.bom === 'utf-8' ? ' (BOM)' : ''}${report.encoding.validUtf8 ? '' : ' (INVALID UTF-8)'}`)
+  }
+  const overall = report.scorecard.overall
+  const weighted = report.scorecard.weightedOverall
+  lines.push(`Scorecard (overall ${overall === null ? 'undetermined' : `${(overall * 100).toFixed(1)}%`}, weighted ${weighted === null ? 'undetermined' : `${(weighted * 100).toFixed(1)}%`}):`)
+  for (const dimension of report.scorecard.dimensions) {
+    const value = dimension.score === null ? 'undetermined' : `${(dimension.score * 100).toFixed(1)}%`
+    lines.push(`  ${dimension.name}: ${value} (${dimension.note})`)
+  }
   return lines.join('\n')
 }

package/src/provider-local.ts

@@ -12,10 +12,12 @@ import type { Context } from '@deepseek-ai/cordis'
 import type { ResolvedConfig } from './config.ts'
 import { loadDocument, loadTable, resolveWorkspacePath, throwIfAborted } from './dataset.ts'
 import { applyCleanRules, serializeDelimited } from './clean.ts'
+import { computeCleanContract, computeCleanProfileDiff } from './contract.ts'
 import { profileTable } from './profile.ts'
 import { checkCitations, verifyTable } from './verify.ts'
 import { appendDataQualityEvent } from './events.ts'
-import { reportKeyOf, type ReportRecord, type ReportStore } from './store.ts'
+import { isValidReportKey, reportKeyOf, type ReportRecord, type ReportStore, type StoredReport } from './store.ts'
+import { resolveIndustryPreset } from './presets.ts'
 import { truncateRow } from './present.ts'
 import {
   DataQualityService,
@@ -73,7 +75,7 @@ export class LocalDataQualityService extends DataQualityService {
   }
 
   /** Emit the adaptive session event when the call carries a session. */
-  emitEvent(session: Session | undefined, kind: 'profile' | 'clean' | 'verify', dataset: string, reportKey: string | undefined, summary: { rows: number; columns?: number; rules?: number; failedRules?: number; passed?: boolean }): void {
+  emitEvent(session: Session | undefined, kind: 'profile' | 'clean' | 'verify', dataset: string, reportKey: string | undefined, summary: { rows: number; columns?: number; rules?: number; failedRules?: number; passed?: boolean; expectations?: number; failedExpectations?: number }): void {
     if (session === undefined) return
     appendDataQualityEvent(session, `data-quality/${kind}`, {
       kind,
@@ -88,11 +90,15 @@ export class LocalDataQualityService extends DataQualityService {
     throwIfAborted(request.signal)
     const absolute = resolveWorkspacePath(request.workspace, request.dataset, this.config)
     const table = await loadTable(absolute, this.config, request.signal)
+    const declaredSchema = request.industryPreset === undefined ? undefined : resolveIndustryPreset(request.industryPreset).columns
     const report = profileTable(table, {
       dataset: request.dataset,
       sample: request.sample,
       generatedAt: this.deps.now(),
       signal: request.signal,
+      duplicateSampleLimit: this.config.evidenceRowLimit,
+      declaredSchema,
+      scorecardWeights: this.config.scorecardWeights,
     })
     const reportKey = await this.persist('profile', request.dataset, report as unknown as Record<string, unknown>)
     this.emitEvent(request.session, 'profile', request.dataset, reportKey, { rows: report.rowCount, columns: report.columnCount })
@@ -102,12 +108,13 @@ export class LocalDataQualityService extends DataQualityService {
   /** @inheritdoc DataQualityService.cleanDataset */
   override async cleanDataset(request: CleanRequest): Promise<CleanRunReport> {
     throwIfAborted(request.signal)
+    const dryRun = request.dryRun === true
     const absolute = resolveWorkspacePath(request.workspace, request.dataset, this.config)
     const table = await loadTable(absolute, this.config, request.signal)
     const result = applyCleanRules(table, request.rules, { signal: request.signal })
 
     let writtenPath: string | undefined
-    if (request.outputPath !== undefined) {
+    if (!dryRun && request.outputPath !== undefined) {
       const outputAbsolute = resolveWorkspacePath(request.workspace, request.outputPath, this.config)
       if (outputAbsolute === absolute) {
         throw new Error(`outputPath ${JSON.stringify(request.outputPath)} would overwrite the input dataset; choose a different path`)
@@ -125,26 +132,63 @@ export class LocalDataQualityService extends DataQualityService {
     }
 
     const generatedAt = this.deps.now()
+    // The contract summary is part of the model-visible canonical value, so it
+    // must also be part of the durable report (model-visible ⟺ logged).
+    const contract = computeCleanContract(result, request.rules, { signal: request.signal })
+    const outputTable = {
+      columns: result.columns,
+      rows: result.rows,
+      ...(table.encoding !== undefined ? { encoding: table.encoding } : {}),
+    }
+    // The before/after profile diff preview is the expected diff in a dry run
+    // and the durable audit artifact (separate `clean-diff` record) otherwise.
+    const diff = computeCleanProfileDiff(table, outputTable, {
+      dataset: request.dataset,
+      generatedAt,
+      scorecardWeights: this.config.scorecardWeights,
+      signal: request.signal,
+    })
     // The preview is part of the model-visible canonical value, so it must
     // also be part of the durable report: otherwise the clean result's preview
     // could not be reconstructed from the session log (model-visible ⟺ logged).
     const preview = { columns: result.columns, rows: result.rows.slice(0, this.config.evidenceRowLimit).map((row) => truncateRow(row)) }
+
+    if (dryRun) {
+      this.emitEvent(request.session, 'clean', request.dataset, undefined, { rows: result.outputRows, columns: result.columns.length, rules: result.logs.length })
+      return {
+        dataset: request.dataset,
+        inputRows: result.inputRows,
+        outputRows: result.outputRows,
+        dryRun: true,
+        logs: result.logs,
+        contract,
+        preview,
+        diffPreview: diff,
+        generatedAt,
+      }
+    }
+
     const reportKey = await this.persist('clean', request.dataset, {
       dataset: request.dataset,
       inputRows: result.inputRows,
       outputRows: result.outputRows,
+      dryRun: false,
       logs: result.logs,
+      contract,
       preview,
       ...(writtenPath !== undefined ? { outputPath: writtenPath } : {}),
       generatedAt,
     } as unknown as Record<string, unknown>)
+    await this.persist('clean-diff', request.dataset, diff as unknown as Record<string, unknown>)
     this.emitEvent(request.session, 'clean', request.dataset, reportKey, { rows: result.outputRows, columns: result.columns.length, rules: result.logs.length })
 
     return {
       dataset: request.dataset,
       inputRows: result.inputRows,
       outputRows: result.outputRows,
+      dryRun: false,
       logs: result.logs,
+      contract,
       preview,
       ...(writtenPath !== undefined ? { outputPath: writtenPath } : {}),
       ...(reportKey !== undefined ? { reportKey } : {}),
@@ -152,6 +196,31 @@ export class LocalDataQualityService extends DataQualityService {
     }
   }
 
+  /** @inheritdoc DataQualityService.getReport */
+  override async getReport(key: string): Promise<StoredReport> {
+    if (!isValidReportKey(key)) {
+      throw new Error(`invalid reportKey ${JSON.stringify(key)}: expected the deterministic <timestamp>-<kind>-<fingerprint> format`)
+    }
+    const store = this.deps.store
+    if (store === undefined) {
+      throw new Error('report storage is disabled (storeReports is false); no persisted reports to read')
+    }
+    const record = store.get(key)
+    if (record === undefined) {
+      throw new Error(`no persisted report found for reportKey ${JSON.stringify(key)}`)
+    }
+    return { key, ...record }
+  }
+
+  /** @inheritdoc DataQualityService.listReports */
+  override async listReports(kind: ReportRecord['kind']): Promise<StoredReport[]> {
+    const store = this.deps.store
+    if (store === undefined) {
+      throw new Error('report storage is disabled (storeReports is false); no persisted reports to read')
+    }
+    return store.list(kind)
+  }
+
   /** @inheritdoc DataQualityService.verifyDataset */
   override async verifyDataset(request: VerifyRequest): Promise<VerifyReport> {
     throwIfAborted(request.signal)
@@ -161,15 +230,20 @@ export class LocalDataQualityService extends DataQualityService {
       evidenceRowLimit: this.config.evidenceRowLimit,
       now: this.deps.now,
       signal: request.signal,
+      expectations: request.expectations,
+      defaultTolerance: this.config.defaultTolerance,
     })
     const report: VerifyReport = { dataset: request.dataset, ...outcome }
     const failedRules = report.rules.filter((rule) => !rule.passed).length
+    const failedExpectations = report.expectations.filter((expectation) => !expectation.passed).length
     const reportKey = await this.persist('verify', request.dataset, report as unknown as Record<string, unknown>)
     this.emitEvent(request.session, 'verify', request.dataset, reportKey, {
       rows: report.rowCount,
       rules: report.rules.length,
       failedRules,
       passed: report.passed,
+      expectations: report.expectations.length,
+      failedExpectations,
     })
     return { ...report, ...(reportKey !== undefined ? { reportKey } : {}) }
   }