dsh-data-cleaning-agent 0.5.3 → 0.6.0

package/docs/USER-GUIDE.md

@@ -30,10 +30,11 @@ bash <(curl -fsSL https://raw.githubusercontent.com/duhu2000/dsh-data-cleaning-a
 中央业务首页，右侧工作台保持关闭；输入框下方的五个入口分别定位到上传清洗、质量体检、匹配核验、
 字段补全和任务历史。处理步骤为：
 
-1. **上传与映射**：粘贴或上传 CSV / XLSX / JSON，确认企业名称字段；
-2. **数据体检**：查看列概览、缺失和金额分布；
-3. **匹配核验**：先做本地清洗，再选择风险/知产/经营域；多候选由人工确认；
-4. **补全与导出**：恢复任务、重试可重试失败，下载结果 CSV 和复核 CSV。
+1. **上传数据**：粘贴或上传 CSV / XLSX / JSON，预览列和行；
+2. **规则确认**：映射企业名称/统一社会信用代码/注册号，选择清洗目标与补全字段；
+3. **数据匹配**：运行质量体检与主体匹配，多候选由人工确认；
+4. **清洗补全**：执行本地确定性清洗；需要 QCC 时先零调用估算，再由当前用户确认使用自己的账号额度；
+5. **下载数据**：生成结果 CSV/XLSX 与异常清单 CSV/XLSX，后续可从任务历史恢复下载。
 
 输入框左上角的「提示词生成」提供三种名单录入方式：
 
@@ -52,7 +53,8 @@ bash <(curl -fsSL https://raw.githubusercontent.com/duhu2000/dsh-data-cleaning-a
 ### 2.3 web 界面
 
 打开 DeepSeek Harness 后访问插件的同源界面（`/data-cleaning/`），可上传 CSV/XLSX/JSON，
-执行解析、清洗、补全、导出 CSV。web 界面的后端路由前缀为 `/data-cleaning/api/mvp/*`。
+执行解析、清洗、补全与导出。旧 MVP 路由前缀为 `/data-cleaning/api/mvp/*`；0.6.0 五步任务和
+耐久制品使用 `/data-cleaning/api/workflow/*`。
 
 ## 3. 能力说明
 
@@ -134,7 +136,7 @@ bash <(curl -fsSL https://raw.githubusercontent.com/duhu2000/dsh-data-cleaning-a
 4. 点击“估算调用量”。估算为上界且不执行 QCC 工具；
 5. 只有在核对企业数、工具数、估算调用量和 `maxCalls` 后，才勾选“确认使用当前用户的企查查账号额度”；
 6. 多候选逐项人工选择；失败项只在明确点击重试时重放；
-7. 下载补全结果 CSV，必要时下载候选复核 CSV。
+7. 生成并下载结果/异常清单的 CSV 或 XLSX；需要重新核验的行会进入异常清单。
 
 同源 API 为 `/data-cleaning/api/phase3/*`。单批最多 100 行，并发上限 4，硬调用上限 2000；
 默认调用上限 500。`enrich` / `resolve` / `retry` 均要求 `confirmPaidCalls:true` 和唯一幂等键。
@@ -164,5 +166,6 @@ bash <(curl -fsSL https://raw.githubusercontent.com/duhu2000/dsh-data-cleaning-a
   [QCC-ENRICHMENT-DESIGN.md](QCC-ENRICHMENT-DESIGN.md)。
 - **Q：可以在后台批量补全吗？** A：可以。0.4.0 G5 工商批量已发布并完成真实 OAuth/token 验收；
   0.5.0 风险/知产/经营三域已发布，先用 estimate 核对调用上限，再显式确认使用当前用户自己的 QCC 账号额度。
-- **Q：刷新页面或重启后还能恢复吗？** A：同一 Host 进程内可用 `runId` 恢复，默认保留 30 分钟；
-  Host 重启后内存 run 失效，但已下载 CSV 和本地源文件不受影响。
+- **Q：刷新页面或重启后还能恢复吗？** A：五步任务元数据与已生成制品可按 taskId 跨 Host 重启恢复；
+  尚未导出的浏览器原始行不会持久化，需要重新上传。QCC 的临时 `runId` 仍只在同一 Host 进程保留
+  30 分钟，但完成任务的 CSV/XLSX 不依赖该内存 run。

package/lib/artifacts.js

@@ -0,0 +1,239 @@
+/**
+ * Host 耐久导出制品。
+ *
+ * DSH rc.2 / alpha.2 已验证的 fs seam 只提供原子 writeText 与有界
+ * readBytes，没有稳定的二进制写接口。因此 CSV 以 UTF-8 文本保存，XLSX
+ * 先生成真实工作簿，再以 Base64 文本保存，下载时还原为原始字节。
+ */
+import { createHash, randomUUID } from 'node:crypto';
+import XLSX from 'xlsx';
+
+const ROOT = '.dsh-data-cleaning-artifacts/v1';
+const MAX_ARTIFACT_BYTES = 32 * 1024 * 1024;
+// Base64 最坏会把二进制扩大到 4/3；readBytes 必须允许读取完整编码文本，
+// 再对解码后的真实制品执行 MAX_ARTIFACT_BYTES 限制。
+const MAX_STORED_BYTES = Math.ceil(MAX_ARTIFACT_BYTES / 3) * 4 + 4;
+const SAFE_ID = /^(?:dcw|dca)-[a-zA-Z0-9-]{8,80}$/;
+
+export class ArtifactError extends Error {
+  constructor(code, message, status = 400) {
+    super(message);
+    this.name = 'ArtifactError';
+    this.code = code;
+    this.status = status;
+  }
+}
+
+function safeFilePart(value, fallback) {
+  const text = String(value ?? '').trim()
+    .replace(/[\\/:*?"<>|\u0000-\u001f]/g, '-')
+    .replace(/\s+/g, '-')
+    .replace(/-+/g, '-')
+    .replace(/^[-.]+|[-.]+$/g, '')
+    .slice(0, 80);
+  return text || fallback;
+}
+
+function assertId(value, label) {
+  const id = String(value ?? '');
+  if (!SAFE_ID.test(id)) throw new ArtifactError('DC_ARTIFACT_ID_INVALID', `${label} is invalid.`, 400);
+  return id;
+}
+
+function sha256(buffer) {
+  return createHash('sha256').update(buffer).digest('hex');
+}
+
+function normalizeHeaders(headers, rows) {
+  const fromInput = Array.isArray(headers) ? headers.map((item) => String(item ?? '').trim()).filter(Boolean) : [];
+  const discovered = [];
+  const seen = new Set(fromInput);
+  for (const row of rows) {
+    for (const key of Object.keys(row ?? {})) {
+      if (!seen.has(key)) {
+        seen.add(key);
+        discovered.push(key);
+      }
+    }
+  }
+  return [...fromInput, ...discovered].slice(0, 256);
+}
+
+function normalizeRows(value) {
+  if (!Array.isArray(value)) throw new ArtifactError('DC_ARTIFACT_ROWS_REQUIRED', 'Export rows must be an array.', 400);
+  if (value.length > 100_000) throw new ArtifactError('DC_ARTIFACT_ROWS_TOO_MANY', 'Export is limited to 100,000 rows.', 413);
+  const normalizeCell = (cell) => {
+    if (cell === null || cell === undefined) return '';
+    if (cell instanceof Date) return cell.toISOString();
+    if (typeof cell === 'number' || typeof cell === 'boolean') return cell;
+    if (typeof cell === 'string') return cell.slice(0, 32_767);
+    try {
+      const serialized = JSON.stringify(cell);
+      return String(serialized ?? cell).slice(0, 32_767);
+    } catch {
+      return String(cell).slice(0, 32_767);
+    }
+  };
+  return value.map((row) => {
+    if (!row || typeof row !== 'object' || Array.isArray(row)) return { value: normalizeCell(row) };
+    return Object.fromEntries(Object.entries(row).slice(0, 256).map(([key, cell]) => [String(key), normalizeCell(cell)]));
+  });
+}
+
+function exceptionReason(row) {
+  const status = String(
+    row?.qcc_match_status
+      ?? row?.match_status
+      ?? row?.['匹配状态']
+      ?? '',
+  ).trim().toLowerCase();
+  const error = row?.qcc_error ?? row?.error ?? row?.['错误原因'];
+  if (error) return String(error).slice(0, 500);
+  if (['candidate', 'ambiguous', 'review_required'].includes(status)) return '存在多个候选主体，需人工核验';
+  if (['unresolved', 'not_found'].includes(status)) return '未匹配到可验证主体';
+  if (['failed', 'error', 'partial'].includes(status)) return '匹配或补全未完成';
+  return '';
+}
+
+export function deriveExceptionRows(rows) {
+  return rows.flatMap((row) => {
+    const reason = exceptionReason(row);
+    return reason ? [{ ...row, _exception_reason: reason }] : [];
+  });
+}
+
+function workbookBytes(rows, headers, sheetName) {
+  const workbook = XLSX.utils.book_new();
+  const worksheet = XLSX.utils.json_to_sheet(rows, { header: headers, skipHeader: false });
+  worksheet['!cols'] = headers.map((header) => ({ wch: Math.min(42, Math.max(12, String(header).length * 2 + 4)) }));
+  XLSX.utils.book_append_sheet(workbook, worksheet, sheetName.slice(0, 31));
+  workbook.Props = {
+    Title: sheetName,
+    Subject: 'DeepSeek Harness 数据清洗补全智能体导出',
+    Author: 'dsh-data-cleaning-agent',
+    Company: 'QCC',
+  };
+  return Buffer.from(XLSX.write(workbook, {
+    type: 'buffer',
+    bookType: 'xlsx',
+    compression: true,
+    cellDates: true,
+  }));
+}
+
+function csvBytes(rows, headers) {
+  const escapeCell = (value) => {
+    let text = String(value ?? '');
+    // 防止 Excel / LibreOffice 将外部数据解释为公式。数字类型不经过此前缀；
+    // 以危险字符开头的文本保留原值但加前导单引号。
+    if (/^[\u0009\u000d\u000a ]*[=+\-@]/.test(text)) text = `'${text}`;
+    return /[",\r\n]/.test(text) ? `"${text.replaceAll('"', '""')}"` : text;
+  };
+  const lines = [
+    headers.map(escapeCell).join(','),
+    ...rows.map((row) => headers.map((header) => escapeCell(row?.[header])).join(',')),
+  ];
+  return Buffer.from(`\uFEFF${lines.join('\r\n')}`, 'utf8');
+}
+
+function artifactDescriptor({ id, kind, format, fileName, rowCount, bytes, createdAt }) {
+  return {
+    id,
+    kind,
+    format,
+    fileName,
+    rowCount,
+    sizeBytes: bytes.length,
+    checksum: `sha256:${sha256(bytes)}`,
+    mediaType: format === 'xlsx'
+      ? 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
+      : 'text/csv; charset=utf-8',
+    createdAt,
+  };
+}
+
+export class WorkflowArtifactStore {
+  constructor({ fs, nowFn = () => new Date().toISOString(), idFactory = () => `dca-${randomUUID()}` }) {
+    if (!fs) throw new ArtifactError('DC_ARTIFACT_UNAVAILABLE', 'DSH fs service unavailable.', 503);
+    this.fs = fs;
+    this.nowFn = nowFn;
+    this.idFactory = idFactory;
+  }
+
+  pathFor(taskId, artifact) {
+    const safeTaskId = assertId(taskId, 'taskId');
+    const artifactId = assertId(artifact.id, 'artifactId');
+    const suffix = artifact.format === 'xlsx' ? 'xlsx.b64' : 'csv';
+    return `${ROOT}/${safeTaskId}/${artifactId}.${suffix}`;
+  }
+
+  async write(taskId, descriptor, bytes) {
+    if (bytes.length > MAX_ARTIFACT_BYTES) {
+      throw new ArtifactError('DC_ARTIFACT_TOO_LARGE', 'Generated artifact exceeds the 32 MiB limit.', 413);
+    }
+    const target = await this.fs.resolve(this.pathFor(taskId, descriptor));
+    const content = descriptor.format === 'xlsx' ? bytes.toString('base64') : bytes.toString('utf8');
+    await this.fs.writeText(target, content);
+    return descriptor;
+  }
+
+  async createBundle(taskId, input = {}) {
+    assertId(taskId, 'taskId');
+    const rows = normalizeRows(input.rows);
+    const headers = normalizeHeaders(input.headers, rows);
+    if (!headers.length) throw new ArtifactError('DC_ARTIFACT_HEADERS_REQUIRED', 'At least one export column is required.', 400);
+    const exceptions = input.exceptionRows === undefined
+      ? deriveExceptionRows(rows)
+      : normalizeRows(input.exceptionRows);
+    const exceptionHeaders = normalizeHeaders([...headers, '_exception_reason'], exceptions);
+    const baseName = safeFilePart(input.baseName, '数据清洗补全结果');
+    const timestamp = this.nowFn();
+    const definitions = [
+      { kind: 'complete', format: 'csv', fileName: `${baseName}.csv`, rows, headers, sheet: '清洗补全结果' },
+      { kind: 'complete', format: 'xlsx', fileName: `${baseName}.xlsx`, rows, headers, sheet: '清洗补全结果' },
+      { kind: 'review', format: 'csv', fileName: `${baseName}-异常清单.csv`, rows: exceptions, headers: exceptionHeaders, sheet: '异常清单' },
+      { kind: 'review', format: 'xlsx', fileName: `${baseName}-异常清单.xlsx`, rows: exceptions, headers: exceptionHeaders, sheet: '异常清单' },
+    ];
+    const artifacts = [];
+    for (const definition of definitions) {
+      const bytes = definition.format === 'xlsx'
+        ? workbookBytes(definition.rows, definition.headers, definition.sheet)
+        : csvBytes(definition.rows, definition.headers);
+      const descriptor = artifactDescriptor({
+        id: this.idFactory(),
+        kind: definition.kind,
+        format: definition.format,
+        fileName: definition.fileName,
+        rowCount: definition.rows.length,
+        bytes,
+        createdAt: timestamp,
+      });
+      await this.write(taskId, descriptor, bytes);
+      artifacts.push(descriptor);
+    }
+    return artifacts;
+  }
+
+  async read(taskId, artifact) {
+    const target = await this.fs.resolve(this.pathFor(taskId, artifact));
+    const storedLimit = artifact.format === 'xlsx' ? MAX_STORED_BYTES : MAX_ARTIFACT_BYTES;
+    const stored = Buffer.from(await this.fs.readBytes(target, undefined, storedLimit));
+    const bytes = artifact.format === 'xlsx'
+      ? Buffer.from(stored.toString('utf8'), 'base64')
+      : stored;
+    if (bytes.length > MAX_ARTIFACT_BYTES) {
+      throw new ArtifactError('DC_ARTIFACT_TOO_LARGE', 'Stored artifact exceeds the 32 MiB limit.', 413);
+    }
+    const actual = `sha256:${sha256(bytes)}`;
+    if (artifact.checksum && actual !== artifact.checksum) {
+      throw new ArtifactError('DC_ARTIFACT_CHECKSUM', 'Stored artifact checksum verification failed.', 409);
+    }
+    return bytes;
+  }
+}
+
+export const ARTIFACT_STORAGE = Object.freeze({
+  root: ROOT,
+  maxBytes: MAX_ARTIFACT_BYTES,
+  maxStoredBytes: MAX_STORED_BYTES,
+});