dsh-data-quality 0.1.3 → 0.2.0

package/src/dataset.ts

@@ -9,6 +9,7 @@
 import { open, stat } from 'node:fs/promises'
 import path from 'node:path'
 import { createHash } from 'node:crypto'
+import { TextDecoder } from 'node:util'
 import type { JsonValue } from '@deepseek-ai/dsh-session'
 import type { ResolvedConfig } from './config.ts'
 
@@ -18,12 +19,22 @@ export type Cell = JsonValue
 /** One dataset row keyed by column name. */
 export type Row = Record<string, Cell>
 
+/** Detected file encoding metadata (BOM presence + UTF-8 validity). */
+export interface EncodingInfo {
+  /** `'utf-8'` when a UTF-8 byte-order mark was present, else `null`. */
+  readonly bom: 'utf-8' | null
+  /** Whether the bytes decode as valid UTF-8 (no replacement characters forced). */
+  readonly validUtf8: boolean
+}
+
 /** A tabular dataset: ordered columns plus rows. */
 export interface Table {
   /** Column names in file order. */
   readonly columns: string[]
   /** Rows, each carrying every declared column (missing cells are `null`). */
   readonly rows: Row[]
+  /** Detected file encoding; present only for tables loaded from a file. */
+  readonly encoding?: EncodingInfo
 }
 
 /** The root form a document load returns (citation checking walks this). */
@@ -101,13 +112,39 @@ export function throwIfAborted(signal: AbortSignal | undefined): void {
 }
 
 /**
- * Read a dataset file under the size cap.
+ * Detect a UTF-8 byte-order mark and validate the byte sequence. Invalid
+ * UTF-8 is a data-quality finding the profile reports (`validUtf8: false`)
+ * rather than a structural error that blocks the read — the decoded text
+ * keeps U+FFFD replacement characters so the profile can still run.
+ * @param buffer - raw file bytes.
+ * @returns the encoding metadata.
+ */
+export function detectEncoding(buffer: Uint8Array): EncodingInfo {
+  const bom = buffer.length >= 3 && buffer[0] === 0xef && buffer[1] === 0xbb && buffer[2] === 0xbf ? 'utf-8' : null
+  const body = bom === 'utf-8' ? buffer.subarray(3) : buffer
+  let validUtf8 = true
+  try {
+    new TextDecoder('utf-8', { fatal: true }).decode(body)
+  } catch {
+    validUtf8 = false
+  }
+  return { bom, validUtf8 }
+}
+
+/** Decode file bytes, stripping a UTF-8 BOM and preserving replacement characters for invalid bytes. */
+function decodeUtf8Text(buffer: Uint8Array, encoding: EncodingInfo): string {
+  const body = encoding.bom === 'utf-8' ? buffer.subarray(3) : buffer
+  return new TextDecoder('utf-8').decode(body)
+}
+
+/**
+ * Read a dataset file under the size cap, detecting its encoding.
  * @param absolutePath - normalized absolute path (from {@link resolveWorkspacePath}).
  * @param config - resolved config (size cap).
  * @param signal - optional abort signal honored around the read.
- * @returns the UTF-8 text.
+ * @returns the decoded UTF-8 text plus its encoding metadata.
  */
-export async function readDatasetText(absolutePath: string, config: ResolvedConfig, signal?: AbortSignal): Promise<string> {
+export async function readDatasetFile(absolutePath: string, config: ResolvedConfig, signal?: AbortSignal): Promise<{ text: string; encoding: EncodingInfo }> {
   throwIfAborted(signal)
   let info
   try {
@@ -128,7 +165,9 @@ export async function readDatasetText(absolutePath: string, config: ResolvedConf
   const handle = await open(absolutePath, 'r')
   try {
     throwIfAborted(signal)
-    return await handle.readFile('utf8')
+    const buffer = await handle.readFile()
+    const encoding = detectEncoding(buffer)
+    return { text: decodeUtf8Text(buffer, encoding), encoding }
   } finally {
     await handle.close()
   }
@@ -325,12 +364,12 @@ function parseJsonLines(text: string): unknown[] {
  * @returns the parsed table.
  */
 export async function loadTable(absolutePath: string, config: ResolvedConfig, signal?: AbortSignal): Promise<Table> {
-  const text = await readDatasetText(absolutePath, config, signal)
+  const { text, encoding } = await readDatasetFile(absolutePath, config, signal)
   const ext = path.extname(absolutePath).toLowerCase()
   throwIfAborted(signal)
-  if (ext === '.csv') return parseDelimited(text, ',', config, signal)
-  if (ext === '.tsv') return parseDelimited(text, '\t', config, signal)
-  return parseJsonTable(text, ext, config, signal)
+  if (ext === '.csv') return { ...parseDelimited(text, ',', config, signal), encoding }
+  if (ext === '.tsv') return { ...parseDelimited(text, '\t', config, signal), encoding }
+  return { ...parseJsonTable(text, ext, config, signal), encoding }
 }
 
 /**
@@ -345,11 +384,11 @@ export async function loadTable(absolutePath: string, config: ResolvedConfig, si
 export async function loadDocument(absolutePath: string, config: ResolvedConfig, signal?: AbortSignal): Promise<DocumentRoot> {
   const ext = path.extname(absolutePath).toLowerCase()
   if (ext === '.json') {
-    const text = await readDatasetText(absolutePath, config, signal)
+    const { text } = await readDatasetFile(absolutePath, config, signal)
     return { kind: 'json', value: parseJsonDocument(text) }
   }
   if (ext === '.jsonl') {
-    const text = await readDatasetText(absolutePath, config, signal)
+    const { text } = await readDatasetFile(absolutePath, config, signal)
     return { kind: 'json', value: parseJsonLines(text) }
   }
   const table = await loadTable(absolutePath, config, signal)
@@ -405,18 +444,25 @@ const DATE_PATTERNS: readonly RegExp[] = [
   /^(\d{4})-(\d{1,2})-(\d{1,2})[ T](\d{1,2}):(\d{2})(?::(\d{2}))?(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?$/u,
 ]
 
-/**
- * Deterministic date parse to epoch milliseconds. Accepts `YYYY-MM-DD`,
- * `YYYY/MM/DD`, and ISO-like datetimes (date-only forms read as UTC midnight).
- * Calendar-invalid dates (e.g. 2025-13-40) reject. Returns `undefined` when
- * the cell is not a recognized date.
- * @param cell - the cell to parse (`undefined` when the column is absent).
- * @returns epoch milliseconds, or `undefined`.
- */
-export function parseDate(cell: Cell | undefined): number | undefined {
+/** Deterministic date-format labels in {@link DATE_PATTERNS} order. */
+const DATE_FORMATS = ['iso-date', 'slash-date', 'datetime'] as const
+
+/** The format label of a recognized date cell (source pattern, not the epoch). */
+export type DateFormat = (typeof DATE_FORMATS)[number]
+
+/** A recognized date cell: its parsed epoch plus the source format label. */
+export interface DateParse {
+  /** Epoch milliseconds (UTC). */
+  readonly epoch: number
+  /** Which {@link DATE_PATTERNS} entry matched. */
+  readonly format: DateFormat
+}
+
+/** Parse one date cell to its epoch plus format; `undefined` when unrecognized. */
+function parseDateCellInternal(cell: Cell | undefined): DateParse | undefined {
   if (typeof cell !== 'string') return undefined
   const text = cell.trim()
-  for (const pattern of DATE_PATTERNS) {
+  for (const [index, pattern] of DATE_PATTERNS.entries()) {
     const match = pattern.exec(text)
     if (match === null) continue
     const year = Number(match[1])
@@ -429,11 +475,45 @@ export function parseDate(cell: Cell | undefined): number | undefined {
     const epoch = Date.UTC(year, month - 1, day, hour, minute, second)
     const check = new Date(epoch)
     if (check.getUTCMonth() !== month - 1 || check.getUTCDate() !== day) return undefined
-    return epoch
+    const format = DATE_FORMATS[index]
+    if (format === undefined) return undefined
+    return { epoch, format }
   }
   return undefined
 }
 
+/**
+ * Deterministic date parse to epoch milliseconds. Accepts `YYYY-MM-DD`,
+ * `YYYY/MM/DD`, and ISO-like datetimes (date-only forms read as UTC midnight).
+ * Calendar-invalid dates (e.g. 2025-13-40) reject. Returns `undefined` when
+ * the cell is not a recognized date.
+ * @param cell - the cell to parse (`undefined` when the column is absent).
+ * @returns epoch milliseconds, or `undefined`.
+ */
+export function parseDate(cell: Cell | undefined): number | undefined {
+  return parseDateCellInternal(cell)?.epoch
+}
+
+/**
+ * Parse one date cell to its epoch plus source format label.
+ * @param cell - the cell to parse (`undefined` when the column is absent).
+ * @returns the parsed date, or `undefined` when the cell is not a recognized date.
+ */
+export function parseDateCell(cell: Cell | undefined): DateParse | undefined {
+  return parseDateCellInternal(cell)
+}
+
+/**
+ * The format label of a recognized date cell (`iso-date` / `slash-date` /
+ * `datetime`); `undefined` when the cell is not a recognized date. Used to
+ * measure a date column's format consistency.
+ * @param cell - the cell to inspect (`undefined` when the column is absent).
+ * @returns the source format label, or `undefined`.
+ */
+export function dateFormatOf(cell: Cell | undefined): DateFormat | undefined {
+  return parseDateCellInternal(cell)?.format
+}
+
 /** Boolean parse: true/false/yes/no/1/0, case-insensitive. */
 export function parseBoolean(cell: Cell | undefined): boolean | undefined {
   if (typeof cell === 'boolean') return cell

package/src/events.ts

@@ -33,6 +33,8 @@ export interface DataQualityEventData {
     readonly rules?: number
     readonly failedRules?: number
     readonly passed?: boolean
+    readonly expectations?: number
+    readonly failedExpectations?: number
   }
 }
 

package/src/index.ts

@@ -23,14 +23,15 @@ import { LocalDataQualityService } from './provider-local.ts'
 import { defineProfileTool } from './tools/profile.ts'
 import { defineCleanTool } from './tools/clean.ts'
 import { defineVerifyTool } from './tools/verify.ts'
+import { defineReportTool } from './tools/report.ts'
 import { VERSION } from './version.ts'
 
 export const name = 'data-quality'
-/** The three model tools and the durable report domain. */
+/** The four model tools and the durable report domain. */
 export const inject = ['tools', 'storageDomain']
 
 export { Config, resolveConfig } from './config.ts'
-export type { Config as DataQualityConfig, ResolvedConfig } from './config.ts'
+export type { Config as DataQualityConfig, ResolvedConfig, ScorecardWeights } from './config.ts'
 export { VERSION } from './version.ts'
 export {
   DataQualityService,
@@ -43,16 +44,21 @@ export {
 } from './service.ts'
 export { LocalDataQualityService, type ProviderDeps } from './provider-local.ts'
 export type { CleanRule, CleanRuleLog, CleanResult } from './clean.ts'
-export type { ProfileReport, ColumnProfile, NumericProfile } from './profile.ts'
-export type { VerifyRule, VerifyReport, VerifyRuleResult, VerifyEvidenceRow } from './verify.ts'
-export { parseLocator, checkCitations } from './verify.ts'
-export { profileTable, renderProfileText } from './profile.ts'
+export { computeCleanContract, computeCleanProfileDiff, type CleanContractSummary, type CleanProfileDiff } from './contract.ts'
+export type { ProfileReport, ColumnProfile, NumericProfile, DuplicateDetection, InferredType } from './profile.ts'
+export type { VerifyRule, VerifyReport, VerifyRuleResult, VerifyEvidenceRow, VerifyExpectation, VerifyExpectationResult, VerifyMetric } from './verify.ts'
+export { parseLocator, checkCitations, verifyExpectations, VerifyExpectationError } from './verify.ts'
+export { profileTable, renderProfileText, detectDuplicateRows, countDuplicateRows, numericProfile } from './profile.ts'
+export { computeScorecard, type DataQualityScorecard, type ScorecardDimension, type ScorecardDimensionName } from './scorecard.ts'
+export { INDUSTRY_PRESETS, INDUSTRY_PRESET_IDS, resolveIndustryPreset, type IndustryPreset } from './presets.ts'
 export { applyCleanRules, serializeDelimited } from './clean.ts'
 export { verifyTable, renderVerifyText } from './verify.ts'
 export {
   DatasetError,
   loadTable,
   loadDocument,
+  readDatasetFile,
+  detectEncoding,
   parseDelimited,
   parseJsonTable,
   resolveWorkspacePath,
@@ -60,13 +66,18 @@ export {
   isMissing,
   parseNumeric,
   parseDate,
+  parseDateCell,
+  dateFormatOf,
   parseBoolean,
   type Table,
   type Row,
   type Cell,
   type DocumentRoot,
+  type DateParse,
+  type DateFormat,
+  type EncodingInfo,
 } from './dataset.ts'
-export { dataQualityDomainSpec, reportKeyOf, reportRecordSchema, type ReportRecord, type ReportStore } from './store.ts'
+export { dataQualityDomainSpec, reportKeyOf, reportRecordSchema, isValidReportKey, type ReportRecord, type ReportStore, type StoredReport } from './store.ts'
 export { DATA_QUALITY_EVENT_TYPES, appendDataQualityEvent, type DataQualityEventData, type DataQualityEventType } from './events.ts'
 export { MAX_CELL_TEXT, truncateCell, truncateRow } from './present.ts'
 
@@ -97,6 +108,11 @@ export async function apply(ctx: Context, config: Config = {}): Promise<void> {
         return key
       },
       get: (key) => reports.get(key),
+      list: (kind) =>
+        [...reports.entries()]
+          .filter(([, record]) => record.kind === kind)
+          .sort(([keyA], [keyB]) => (keyA < keyB ? -1 : keyA > keyB ? 1 : 0))
+          .map(([key, record]) => ({ key, ...record })),
     }
   }
 
@@ -104,7 +120,8 @@ export async function apply(ctx: Context, config: Config = {}): Promise<void> {
   ctx.tools.register(defineProfileTool(service))
   ctx.tools.register(defineCleanTool(service))
   ctx.tools.register(defineVerifyTool(service))
-  logger.info(`dsh-data-quality ${VERSION} mounted: ctx.dataQuality + data_profile/data_clean/data_verify`)
+  ctx.tools.register(defineReportTool(service))
+  logger.info(`dsh-data-quality ${VERSION} mounted: ctx.dataQuality + data_profile/data_clean/data_verify/data_report`)
 
   if (domain !== undefined) {
     const handle = domain

package/src/presets.ts

@@ -0,0 +1,146 @@
+/**
+ * Built-in industry profile presets: a deterministic registry of expected
+ * column lists (name -> declared type) per industry. When `data_profile` is
+ * called with `industryPreset`, the preset's columns feed the scorecard's
+ * `declaredSchema` so the `accuracy` dimension becomes determinable instead
+ * of undetermined. Unknown preset ids fail loud.
+ * @module dsh-data-quality/presets
+ */
+
+import type { InferredType } from './profile.ts'
+
+/** One industry preset: an id, a human label, and the expected columns with their declared types. */
+export interface IndustryPreset {
+  readonly id: string
+  readonly label: string
+  /** Expected column name -> declared type (fed to the scorecard as `declaredSchema`). */
+  readonly columns: Readonly<Record<string, InferredType>>
+}
+
+/** The built-in industry preset registry. */
+export const INDUSTRY_PRESETS: Readonly<Record<string, IndustryPreset>> = {
+  retail: {
+    id: 'retail',
+    label: 'Retail',
+    columns: {
+      order_id: 'string',
+      sku: 'string',
+      quantity: 'number',
+      unit_price: 'number',
+      revenue: 'number',
+      order_date: 'date',
+      customer_id: 'string',
+    },
+  },
+  saas: {
+    id: 'saas',
+    label: 'SaaS',
+    columns: {
+      account_id: 'string',
+      plan: 'string',
+      seats: 'number',
+      mrr: 'number',
+      signup_date: 'date',
+      churned: 'boolean',
+    },
+  },
+  fund: {
+    id: 'fund',
+    label: 'Fund',
+    columns: {
+      fund_code: 'string',
+      fund_name: 'string',
+      nav: 'number',
+      nav_date: 'date',
+      holding_value: 'number',
+      currency: 'string',
+    },
+  },
+  'real-estate': {
+    id: 'real-estate',
+    label: 'Real estate',
+    columns: {
+      property_id: 'string',
+      listing_price: 'number',
+      area_sqm: 'number',
+      bedrooms: 'number',
+      list_date: 'date',
+      city: 'string',
+    },
+  },
+  'e-commerce': {
+    id: 'e-commerce',
+    label: 'E-commerce',
+    columns: {
+      order_id: 'string',
+      product_id: 'string',
+      quantity: 'number',
+      price: 'number',
+      order_date: 'date',
+      status: 'string',
+    },
+  },
+  healthcare: {
+    id: 'healthcare',
+    label: 'Healthcare',
+    columns: {
+      patient_id: 'string',
+      admission_date: 'date',
+      discharge_date: 'date',
+      age: 'number',
+      diagnosis: 'string',
+      cost: 'number',
+    },
+  },
+  logistics: {
+    id: 'logistics',
+    label: 'Logistics',
+    columns: {
+      shipment_id: 'string',
+      origin: 'string',
+      destination: 'string',
+      weight_kg: 'number',
+      shipped_date: 'date',
+      delivered: 'boolean',
+    },
+  },
+  manufacturing: {
+    id: 'manufacturing',
+    label: 'Manufacturing',
+    columns: {
+      part_id: 'string',
+      quantity: 'number',
+      unit_cost: 'number',
+      produced_date: 'date',
+      defect: 'boolean',
+    },
+  },
+  energy: {
+    id: 'energy',
+    label: 'Energy',
+    columns: {
+      meter_id: 'string',
+      reading: 'number',
+      unit: 'string',
+      reading_date: 'date',
+      consumption: 'number',
+    },
+  },
+}
+
+/** The preset ids, for diagnostics and documentation. */
+export const INDUSTRY_PRESET_IDS = Object.keys(INDUSTRY_PRESETS) as readonly string[]
+
+/**
+ * Resolve an industry preset id to its registry entry, failing loud on an
+ * unknown id.
+ * @param id - the requested preset id.
+ * @returns the preset.
+ */
+export function resolveIndustryPreset(id: string): IndustryPreset {
+  const preset = INDUSTRY_PRESETS[id]
+  if (preset === undefined) {
+    throw new Error(`unknown industryPreset ${JSON.stringify(id)}; choose one of: ${INDUSTRY_PRESET_IDS.join(', ')}`)
+  }
+  return preset
+}

package/src/profile.ts

@@ -6,13 +6,19 @@
  * @module dsh-data-quality/profile
  */
 
-import { isMissing, parseBoolean, parseDate, parseNumeric, sampleRows, throwIfAborted, type Row, type Table } from './dataset.ts'
+import { createHash } from 'node:crypto'
+import { isMissing, parseBoolean, parseDate, parseNumeric, sampleRows, throwIfAborted, type EncodingInfo, type Row, type Table } from './dataset.ts'
+import { computeScorecard, type DataQualityScorecard, type ScorecardDimensionName } from './scorecard.ts'
 
 /** Inferred column type from parsed cell classes. */
 export type InferredType = 'number' | 'date' | 'boolean' | 'string' | 'empty' | 'mixed'
 
 /** Numeric distribution of one numeric column. */
 export interface NumericProfile {
+  /** Number of numeric values the distribution covers. */
+  readonly count: number
+  /** Distinct numeric values among the profiled rows. */
+  readonly distinct: number
   readonly min: number
   readonly max: number
   readonly mean: number
@@ -41,6 +47,16 @@ export interface ColumnProfile {
   notes: string[]
 }
 
+/** Full-content duplicate detection over ALL rows. */
+export interface DuplicateDetection {
+  /** Rows whose full sha256 content duplicates an earlier row. */
+  readonly duplicateRows: number
+  /** `duplicateRows / rowCount` (0 when the table has no rows). */
+  readonly duplicateRate: number
+  /** 0-based indexes of duplicated rows, capped by the sample limit. */
+  readonly duplicateSampleRowIndexes: number[]
+}
+
 /** The full profile report (also the persisted and tool-returned value). */
 export interface ProfileReport {
   readonly dataset: string
@@ -52,6 +68,14 @@ export interface ProfileReport {
   readonly columnCount: number
   /** Rows whose full content duplicates an earlier row (over ALL rows). */
   readonly duplicateRows: number
+  /** `duplicateRows / rowCount` (0 when the table has no rows). */
+  readonly duplicateRate: number
+  /** 0-based indexes of duplicated rows (capped by the configured sample limit). */
+  readonly duplicateSampleRowIndexes: number[]
+  /** DAMA six-dimension quality scorecard. */
+  readonly scorecard: DataQualityScorecard
+  /** Detected file encoding (BOM/UTF-8 validity); present only for file-loaded tables. */
+  readonly encoding?: EncodingInfo
   columns: ColumnProfile[]
   /** Storage-domain key of the persisted report, when persistence is on (set by the provider). */
   readonly reportKey?: string
@@ -59,6 +83,9 @@ export interface ProfileReport {
   readonly generatedAt: number
 }
 
+/** Fallback duplicate-sample cap for direct engine use; the provider always passes the configured `evidenceRowLimit`. */
+const DEFAULT_DUPLICATE_SAMPLE_LIMIT = 20
+
 /** Round to 6 significant digits for stable, readable report numbers. */
 function round6(value: number): number {
   return Number(value.toPrecision(6))
@@ -86,6 +113,8 @@ export function numericProfile(values: readonly number[]): NumericProfile | unde
   const highFence = p75 + 1.5 * iqr
   const outliers = iqr === 0 ? 0 : sorted.filter((value) => value < lowFence || value > highFence).length
   return {
+    count: sorted.length,
+    distinct: new Set(sorted).size,
     min: round6(sorted[0] as number),
     max: round6(sorted[sorted.length - 1] as number),
     mean: round6(sum / sorted.length),
@@ -96,20 +125,49 @@ export function numericProfile(values: readonly number[]): NumericProfile | unde
   }
 }
 
-/** Count rows whose full content duplicates an earlier row (first occurrence is not counted). */
-export function countDuplicateRows(table: Table, signal?: AbortSignal): number {
+/** Deterministic sha256 key of one row's full content (columns in table order). */
+function rowContentKey(table: Table, row: Row): string {
+  return createHash('sha256').update(JSON.stringify(table.columns.map((column) => row[column] ?? null))).digest('hex')
+}
+
+/**
+ * Detect full-content duplicate rows with a bounded sample of their 0-based
+ * indexes. The first occurrence of each content is never counted; later rows
+ * with identical full content are duplicates.
+ * @param table - the parsed dataset.
+ * @param options - sample cap and optional abort signal.
+ * @returns the duplicate count, rate, and capped sample indexes.
+ */
+export function detectDuplicateRows(
+  table: Table,
+  options: { sampleLimit: number; signal?: AbortSignal | undefined },
+): DuplicateDetection {
+  if (!Number.isSafeInteger(options.sampleLimit) || options.sampleLimit <= 0) {
+    throw new TypeError(`sampleLimit must be a positive safe integer, got ${String(options.sampleLimit)}`)
+  }
   const seen = new Set<string>()
   let duplicates = 0
+  const duplicateSampleRowIndexes: number[] = []
   for (const [index, row] of table.rows.entries()) {
-    if (index % 1024 === 0) throwIfAborted(signal)
-    const key = JSON.stringify(table.columns.map((column) => row[column] ?? null))
+    if (index % 1024 === 0) throwIfAborted(options.signal)
+    const key = rowContentKey(table, row)
     if (seen.has(key)) {
       duplicates += 1
+      if (duplicateSampleRowIndexes.length < options.sampleLimit) duplicateSampleRowIndexes.push(index)
     } else {
       seen.add(key)
     }
   }
-  return duplicates
+  return {
+    duplicateRows: duplicates,
+    duplicateRate: table.rows.length === 0 ? 0 : round6(duplicates / table.rows.length),
+    duplicateSampleRowIndexes,
+  }
+}
+
+/** Count rows whose full content duplicates an earlier row (first occurrence is not counted). */
+export function countDuplicateRows(table: Table, signal?: AbortSignal): number {
+  return detectDuplicateRows(table, { sampleLimit: 1, signal }).duplicateRows
 }
 
 /** Profile one column over the given rows. */
@@ -206,19 +264,38 @@ function profileColumn(rows: readonly Row[], column: string, signal?: AbortSigna
  */
 export function profileTable(
   table: Table,
-  options: { dataset: string; sample?: number | undefined; generatedAt: number; signal?: AbortSignal | undefined },
+  options: {
+    dataset: string
+    sample?: number | undefined
+    generatedAt: number
+    signal?: AbortSignal | undefined
+    duplicateSampleLimit?: number | undefined
+    declaredSchema?: Readonly<Record<string, InferredType>> | undefined
+    scorecardWeights?: Readonly<Record<ScorecardDimensionName, number>> | undefined
+  },
 ): ProfileReport {
   throwIfAborted(options.signal)
   const profiled = options.sample === undefined ? table.rows : sampleRows(table.rows, options.sample)
   const columns = table.columns.map((column) => profileColumn(profiled, column, options.signal))
-  const duplicateRows = countDuplicateRows(table, options.signal)
+  const detection = detectDuplicateRows(table, { sampleLimit: options.duplicateSampleLimit ?? DEFAULT_DUPLICATE_SAMPLE_LIMIT, signal: options.signal })
+  const scorecard = computeScorecard(table, {
+    now: options.generatedAt,
+    duplicateRows: detection.duplicateRows,
+    declaredSchema: options.declaredSchema,
+    weights: options.scorecardWeights,
+    signal: options.signal,
+  })
   return {
     dataset: options.dataset,
     rowCount: table.rows.length,
     sampled: profiled.length !== table.rows.length,
     profiledRows: profiled.length,
     columnCount: table.columns.length,
-    duplicateRows,
+    duplicateRows: detection.duplicateRows,
+    duplicateRate: detection.duplicateRate,
+    duplicateSampleRowIndexes: detection.duplicateSampleRowIndexes,
+    scorecard,
+    ...(table.encoding !== undefined ? { encoding: table.encoding } : {}),
     columns,
     generatedAt: options.generatedAt,
   }
@@ -229,14 +306,17 @@ export function renderProfileText(report: ProfileReport): string {
   const lines: string[] = []
   lines.push(`Profile of ${report.dataset}: ${report.rowCount} rows x ${report.columnCount} columns` +
     (report.sampled ? ` (column cards over a systematic sample of ${report.profiledRows} rows)` : ''))
-  if (report.duplicateRows > 0) lines.push(`Duplicate rows: ${report.duplicateRows}`)
+  if (report.duplicateRows > 0) {
+    lines.push(`Duplicate rows: ${report.duplicateRows} (${(report.duplicateRate * 100).toFixed(1)}%)` +
+      (report.duplicateSampleRowIndexes.length > 0 ? `; sample row indexes: ${report.duplicateSampleRowIndexes.join(', ')}` : ''))
+  }
   for (const column of report.columns) {
     const parts = [`${column.name}: ${column.inferredType}`]
     if (column.missing > 0) parts.push(`missing ${column.missing} (${(column.missingRate * 100).toFixed(1)}%)`)
     parts.push(`unique ${column.unique}`)
     if (column.numeric !== undefined) {
       parts.push(
-        `min ${column.numeric.min}, p25 ${column.numeric.p25}, median ${column.numeric.median}, p75 ${column.numeric.p75}, max ${column.numeric.max}, mean ${column.numeric.mean}` +
+        `count ${column.numeric.count}, distinct ${column.numeric.distinct}, min ${column.numeric.min}, p25 ${column.numeric.p25}, median ${column.numeric.median}, p75 ${column.numeric.p75}, max ${column.numeric.max}, mean ${column.numeric.mean}` +
           (column.numeric.outliers > 0 ? `, ${column.numeric.outliers} IQR outliers` : ''),
       )
     }
@@ -246,5 +326,15 @@ export function renderProfileText(report: ProfileReport): string {
     for (const note of column.notes) parts.push(`note: ${note}`)
     lines.push(`- ${parts.join('; ')}`)
   }
+  if (report.encoding !== undefined) {
+    lines.push(`Encoding: UTF-8${report.encoding.bom === 'utf-8' ? ' (BOM)' : ''}${report.encoding.validUtf8 ? '' : ' (INVALID UTF-8)'}`)
+  }
+  const overall = report.scorecard.overall
+  const weighted = report.scorecard.weightedOverall
+  lines.push(`Scorecard (overall ${overall === null ? 'undetermined' : `${(overall * 100).toFixed(1)}%`}, weighted ${weighted === null ? 'undetermined' : `${(weighted * 100).toFixed(1)}%`}):`)
+  for (const dimension of report.scorecard.dimensions) {
+    const value = dimension.score === null ? 'undetermined' : `${(dimension.score * 100).toFixed(1)}%`
+    lines.push(`  ${dimension.name}: ${value} (${dimension.note})`)
+  }
   return lines.join('\n')
 }