dsh-data-cleaning-agent 0.9.13 → 0.9.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -0
- package/README.en.md +4 -2
- package/README.md +5 -3
- package/docs/COMPATIBILITY.md +7 -0
- package/docs/RELEASE-0.9.14.md +12 -0
- package/docs/RELEASE-0.9.15.md +13 -0
- package/docs/UNIFIED-HOME-1.5.4-ADOPTION.md +22 -1
- package/lib/client.js +1321 -298
- package/lib/image-workflow.js +7 -1
- package/lib/qcc-command.js +8 -0
- package/lib/qcc-runs.js +9 -0
- package/lib/skill-enrich.js +1 -0
- package/lib/skill.js +1 -0
- package/lib/web.js +98 -42
- package/lib/workflow-contract.js +125 -7
- package/lib/workflow-execution.js +121 -20
- package/lib/workflow.js +170 -14
- package/package.json +3 -1
package/lib/image-workflow.js
CHANGED
|
@@ -13,7 +13,13 @@ export function createImageWorkflow({ getWorkflow, execution, commands }) {
|
|
|
13
13
|
task = await store.recordUpload(task.id, { expectedRevision: task.revision,
|
|
14
14
|
source: { type: 'image', fileName: record.fileName, sizeBytes: record.sizeBytes,
|
|
15
15
|
rowCount: rows.length, columnCount: 1, headers: ['主体标识'] } });
|
|
16
|
-
task = await store.updateDraft(task.id, {
|
|
16
|
+
task = await store.updateDraft(task.id, {
|
|
17
|
+
expectedRevision: task.revision,
|
|
18
|
+
mappings,
|
|
19
|
+
objectives: task.objectives.includes('complete_fields')
|
|
20
|
+
? task.objectives
|
|
21
|
+
: [...task.objectives, 'complete_fields'],
|
|
22
|
+
});
|
|
17
23
|
if (result.truncated || result.needsReview || rows.length > 100) {
|
|
18
24
|
return { taskId: task.id, deliveryState: 'review_required', artifacts: [],
|
|
19
25
|
reviewReason: rows.length > 100 || result.truncated ? '识别名单超出单批范围,请核验并拆分。' : '识别文本含模糊字符或待确认提示,请核验名单。' };
|
package/lib/qcc-command.js
CHANGED
|
@@ -198,6 +198,14 @@ export class QccCommandStore {
|
|
|
198
198
|
return output;
|
|
199
199
|
}
|
|
200
200
|
|
|
201
|
+
hasActiveForTask(taskId) {
|
|
202
|
+
this.cleanup();
|
|
203
|
+
const id = String(taskId ?? '');
|
|
204
|
+
return [...this.commands.values()].some((command) => (
|
|
205
|
+
command.taskId === id && ['prepared', 'running'].includes(command.state)
|
|
206
|
+
));
|
|
207
|
+
}
|
|
208
|
+
|
|
201
209
|
async run(commandId, execution) {
|
|
202
210
|
const record = this.require(commandId);
|
|
203
211
|
if (!execution?.agent || !execution?.token) {
|
package/lib/qcc-runs.js
CHANGED
|
@@ -191,6 +191,15 @@ export class G5RunStore {
|
|
|
191
191
|
return this.snapshot(this.requireRun(id));
|
|
192
192
|
}
|
|
193
193
|
|
|
194
|
+
capabilities(id) {
|
|
195
|
+
this.cleanup();
|
|
196
|
+
const record = this.runs.get(String(id ?? ''));
|
|
197
|
+
return {
|
|
198
|
+
liveRunAvailable: Boolean(record),
|
|
199
|
+
retryableFailuresAvailable: Boolean(record?.errors.some((item) => item.error?.retryable)),
|
|
200
|
+
};
|
|
201
|
+
}
|
|
202
|
+
|
|
194
203
|
snapshot(record) {
|
|
195
204
|
return clone({
|
|
196
205
|
runId: record.id,
|
package/lib/skill-enrich.js
CHANGED
|
@@ -52,6 +52,7 @@ export function registerEnrichSkill(skills) {
|
|
|
52
52
|
'When the user gives company names and asks to enrich / complete them with Qichacha (企查查), including registration fields, company panorama, ownership penetration, governance, historical changes, risk tags, intellectual property, or operational / bidding dimensions.',
|
|
53
53
|
source: 'dsh-data-cleaning-agent',
|
|
54
54
|
content: [
|
|
55
|
+
'UX-49 最高优先级:先检查本业务请求的信息是否充分,再考虑以下任何工具或执行规则。若仍含未补全的【】占位符,或没有实际企业名单/Excel/名单图片与待补字段,先用中文要求提供名单或文件及待补字段;零工具调用,包括 OAuth 检查、体检、OCR、MCP、创建 workflow。例子、说明、【企业名单】均不是企业。用户改过的初始模板按普通内容理解,不得整体忽略。有真实暂存来源和已选字段的安全任务凭证可提供这些信息,不要求重复提供。此规则只作用于数据清洗补全业务,不得全局拦截 DSH 发送或其它产品。',
|
|
55
56
|
'You are an enterprise-list enrichment assistant. You fill company lists with Qichacha (QCC) data by calling QCC MCP tools. Never invent, pad, or fabricate any field.',
|
|
56
57
|
'',
|
|
57
58
|
'Workbench execution request (highest priority):',
|
package/lib/skill.js
CHANGED
|
@@ -15,6 +15,7 @@ export function registerSkill(skills) {
|
|
|
15
15
|
'When the user asks to clean, complete, validate, de-duplicate, or summarize a batch of raw data rows, CSV records, or table-like data.',
|
|
16
16
|
source: 'dsh-data-cleaning-agent',
|
|
17
17
|
content: [
|
|
18
|
+
'UX-49 — Highest priority, before ANY tool: within this data-cleaning business request, if the message still contains an unfilled 【】 placeholder, or lacks actual rows/list/file and requested completion fields, ask the user to provide the actual enterprise list or Excel/list image and the fields to complete. Make ZERO tool calls (including profile, OAuth checks, OCR, MCP and workflow creation). Do not treat examples, instructions or 【企业名单】 as enterprises. Do not intercept other DSH products or the global send action. A valid staged-source task credential can supply the list/field scope; do not demand the same data twice. An edited initial template is ordinary user content, never discard it wholesale.',
|
|
18
19
|
'You are a data cleaning and completion assistant. Work only on the rows the user actually provided; never invent, pad, or fabricate extra rows.',
|
|
19
20
|
'',
|
|
20
21
|
'Workflow:',
|
package/lib/web.js
CHANGED
|
@@ -7,13 +7,19 @@
|
|
|
7
7
|
* - 上传体大小上限(parseBody)。
|
|
8
8
|
* - 同步接口返回明细行仅供「已授权同源 UI」下载,不面向模型。
|
|
9
9
|
*/
|
|
10
|
+
import { createHash, randomBytes } from 'node:crypto';
|
|
10
11
|
import { parseCsv, parseXlsx, parseJson, detectFormat, toCsv } from './engine.js';
|
|
11
12
|
import { runSync, DataCleaningJobs } from './jobs.js';
|
|
12
13
|
import { QccBridgeError, QccHostBridge } from './qcc.js';
|
|
13
14
|
import { QccCommandStore, registerQccCommandTool, TOOL_QCC_COMMAND } from './qcc-command.js';
|
|
14
15
|
import { fingerprintRequest, G5RunStore } from './qcc-runs.js';
|
|
15
16
|
import { PHASE3_BATCH_LIMITS, Phase3BatchService, Phase3RunStore } from './qcc-phase3-batch.js';
|
|
16
|
-
import {
|
|
17
|
+
import {
|
|
18
|
+
WORKFLOW_ACTIONS,
|
|
19
|
+
allowedWorkflowActions,
|
|
20
|
+
deriveWorkflowFlow,
|
|
21
|
+
publicWorkflowContract,
|
|
22
|
+
} from './workflow-contract.js';
|
|
17
23
|
import { DataCleaningWorkflowStore, WorkflowError } from './workflow.js';
|
|
18
24
|
import { ArtifactError, WorkflowArtifactStore } from './artifacts.js';
|
|
19
25
|
import { renderArtifactPreview } from './artifact-preview.js';
|
|
@@ -141,6 +147,23 @@ function writeImageError(res, error) {
|
|
|
141
147
|
}
|
|
142
148
|
|
|
143
149
|
/** 从 JSON 协议解析上传:{ filename, content }。content 为字符串;xlsx 时为 base64。 */
|
|
150
|
+
function canonicalize(value) {
|
|
151
|
+
if (Array.isArray(value)) return value.map(canonicalize);
|
|
152
|
+
if (value === null || typeof value !== 'object') return value;
|
|
153
|
+
return Object.fromEntries(Object.keys(value).sort().map((key) => [key, canonicalize(value[key])]));
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
function sourceVerifier(headers, rows, previous = '') {
|
|
157
|
+
const match = /^sha256-canonical-rows-v1:([a-f0-9]{32}):[a-f0-9]{64}$/.exec(String(previous));
|
|
158
|
+
const salt = match?.[1] || randomBytes(16).toString('hex');
|
|
159
|
+
const canonical = JSON.stringify({
|
|
160
|
+
headers: (Array.isArray(headers) ? headers : []).map(String),
|
|
161
|
+
rows: canonicalize(Array.isArray(rows) ? rows : []),
|
|
162
|
+
});
|
|
163
|
+
const digest = createHash('sha256').update(salt).update('\0').update(canonical).digest('hex');
|
|
164
|
+
return `sha256-canonical-rows-v1:${salt}:${digest}`;
|
|
165
|
+
}
|
|
166
|
+
|
|
144
167
|
async function parseUpload(body) {
|
|
145
168
|
let payload;
|
|
146
169
|
try {
|
|
@@ -153,12 +176,20 @@ async function parseUpload(body) {
|
|
|
153
176
|
const filename = String(payload?.filename ?? 'data.csv');
|
|
154
177
|
const content = payload?.content ?? '';
|
|
155
178
|
const fmt = detectFormat(filename);
|
|
179
|
+
let parsed;
|
|
156
180
|
if (fmt === 'xlsx') {
|
|
157
181
|
const buf = Buffer.from(String(content), 'base64');
|
|
158
|
-
|
|
182
|
+
parsed = await parseXlsx(buf);
|
|
183
|
+
} else if (fmt === 'json') {
|
|
184
|
+
parsed = parseJson(String(content));
|
|
185
|
+
} else {
|
|
186
|
+
parsed = parseCsv(String(content));
|
|
159
187
|
}
|
|
160
|
-
|
|
161
|
-
|
|
188
|
+
return {
|
|
189
|
+
fmt,
|
|
190
|
+
...parsed,
|
|
191
|
+
checksum: sourceVerifier(parsed.headers, parsed.rows, payload?.previousChecksum),
|
|
192
|
+
};
|
|
162
193
|
}
|
|
163
194
|
|
|
164
195
|
const UI_HTML = `<!doctype html>
|
|
@@ -307,7 +338,11 @@ export function mountWebRoutes(wctx, { logger, report, TOOL_NAME, SKILL_NAME })
|
|
|
307
338
|
const disposers = [];
|
|
308
339
|
const qccBridge = new QccHostBridge({ tools, logger });
|
|
309
340
|
const g5Runs = new G5RunStore();
|
|
310
|
-
const execution = createWorkflowExecution({
|
|
341
|
+
const execution = createWorkflowExecution({
|
|
342
|
+
getWorkflow: () => getWorkflow(),
|
|
343
|
+
artifacts: wctx.fs ? new WorkflowArtifactStore({ fs: wctx.fs }) : null,
|
|
344
|
+
getRun: (runId) => g5Runs.get(runId),
|
|
345
|
+
});
|
|
311
346
|
const qccCommands = new QccCommandStore({ bridge: qccBridge, runs: g5Runs, lifecycle: execution });
|
|
312
347
|
const imageIntake = new ImageIntakeStore({ tools,
|
|
313
348
|
continueWorkflow: createImageWorkflow({ getWorkflow: () => getWorkflow(), execution, commands: qccCommands }) });
|
|
@@ -353,6 +388,22 @@ export function mountWebRoutes(wctx, { logger, report, TOOL_NAME, SKILL_NAME })
|
|
|
353
388
|
return workflowReady;
|
|
354
389
|
};
|
|
355
390
|
|
|
391
|
+
const workflowView = (task) => ({
|
|
392
|
+
...task,
|
|
393
|
+
flow: deriveWorkflowFlow(task),
|
|
394
|
+
runtimeCapabilities: {
|
|
395
|
+
activeCommand: qccCommands.hasActiveForTask(task.id),
|
|
396
|
+
...g5Runs.capabilities(task.qccRunId),
|
|
397
|
+
},
|
|
398
|
+
});
|
|
399
|
+
|
|
400
|
+
const requireWorkflowAction = (task, action) => {
|
|
401
|
+
if (!allowedWorkflowActions(task).includes(action)) {
|
|
402
|
+
const current = deriveWorkflowFlow(task).currentStage;
|
|
403
|
+
throw new WorkflowError('DC_WORKFLOW_ACTION_LOCKED', `当前任务位于“${current}”,不能执行该操作。`, 409);
|
|
404
|
+
}
|
|
405
|
+
};
|
|
406
|
+
|
|
356
407
|
register('/data-cleaning/', (req, res) => {
|
|
357
408
|
if (!isTrusted(req)) { res.writeHead(403); return res.end('untrusted origin'); }
|
|
358
409
|
res.writeHead(200, { 'content-type': 'text/html; charset=utf-8', 'cache-control': 'no-store' });
|
|
@@ -490,7 +541,7 @@ export function mountWebRoutes(wctx, { logger, report, TOOL_NAME, SKILL_NAME })
|
|
|
490
541
|
return writeJson(res, 200, {
|
|
491
542
|
ok: true,
|
|
492
543
|
marker: 'data-cleaning-workflow-v2',
|
|
493
|
-
tasks: await workflow.list(),
|
|
544
|
+
tasks: (await workflow.list()).map(workflowView),
|
|
494
545
|
});
|
|
495
546
|
}
|
|
496
547
|
if (req.method === 'POST') {
|
|
@@ -499,7 +550,7 @@ export function mountWebRoutes(wctx, { logger, report, TOOL_NAME, SKILL_NAME })
|
|
|
499
550
|
return writeJson(res, 201, {
|
|
500
551
|
ok: true,
|
|
501
552
|
marker: 'data-cleaning-workflow-v2',
|
|
502
|
-
task: await workflow.create(payload),
|
|
553
|
+
task: workflowView(await workflow.create(payload)),
|
|
503
554
|
});
|
|
504
555
|
}
|
|
505
556
|
return writeJson(res, 405, { ok: false, code: 'DC_METHOD', message: 'GET or POST required' });
|
|
@@ -511,17 +562,18 @@ export function mountWebRoutes(wctx, { logger, report, TOOL_NAME, SKILL_NAME })
|
|
|
511
562
|
return writeJson(res, 200, {
|
|
512
563
|
ok: true,
|
|
513
564
|
marker: 'data-cleaning-workflow-v2',
|
|
514
|
-
task: await workflow.require(taskId),
|
|
565
|
+
task: workflowView(await workflow.require(taskId)),
|
|
515
566
|
});
|
|
516
567
|
}
|
|
517
568
|
if (req.method === 'PATCH') {
|
|
518
569
|
const body = await readBody(req);
|
|
519
570
|
const payload = body.length ? JSON.parse(body.toString('utf8')) : {};
|
|
520
|
-
await workflow.assertRequestOrigin(taskId, payload);
|
|
571
|
+
const current = await workflow.assertRequestOrigin(taskId, payload);
|
|
572
|
+
requireWorkflowAction(current, WORKFLOW_ACTIONS.EDIT_RULES);
|
|
521
573
|
return writeJson(res, 200, {
|
|
522
574
|
ok: true,
|
|
523
575
|
marker: 'data-cleaning-workflow-v2',
|
|
524
|
-
task: await workflow.updateDraft(taskId, payload),
|
|
576
|
+
task: workflowView(await workflow.updateDraft(taskId, payload)),
|
|
525
577
|
});
|
|
526
578
|
}
|
|
527
579
|
return writeJson(res, 405, { ok: false, code: 'DC_METHOD', message: 'GET or PATCH required' });
|
|
@@ -540,7 +592,7 @@ export function mountWebRoutes(wctx, { logger, report, TOOL_NAME, SKILL_NAME })
|
|
|
540
592
|
return writeJson(res, 200, {
|
|
541
593
|
ok: true,
|
|
542
594
|
marker: 'data-cleaning-artifacts-v1',
|
|
543
|
-
task: current,
|
|
595
|
+
task: workflowView(current),
|
|
544
596
|
artifacts: current.artifacts,
|
|
545
597
|
});
|
|
546
598
|
}
|
|
@@ -551,32 +603,24 @@ export function mountWebRoutes(wctx, { logger, report, TOOL_NAME, SKILL_NAME })
|
|
|
551
603
|
if (Number(payload.expectedRevision) !== current.revision) {
|
|
552
604
|
throw new WorkflowError('DC_WORKFLOW_REVISION_CONFLICT', 'Workflow task was updated by another session.', 409);
|
|
553
605
|
}
|
|
554
|
-
|
|
555
|
-
if (['rules_confirmed', 'diagnosed'].includes(readyTask.state)) {
|
|
556
|
-
readyTask = await workflow.prepareLocalExport(taskId, {
|
|
557
|
-
expectedRevision: readyTask.revision,
|
|
558
|
-
summary: payload.summary,
|
|
559
|
-
});
|
|
560
|
-
}
|
|
561
|
-
if (!['export_ready', 'partial'].includes(readyTask.state)) {
|
|
562
|
-
throw new WorkflowError('DC_WORKFLOW_EXPORT_STATE', 'Output is not ready for durable export.', 409);
|
|
563
|
-
}
|
|
606
|
+
requireWorkflowAction(current, WORKFLOW_ACTIONS.CREATE_LOCAL_ARTIFACTS);
|
|
564
607
|
const artifacts = await artifactStore.createBundle(taskId, {
|
|
565
608
|
rows: payload.rows,
|
|
566
609
|
headers: payload.headers,
|
|
567
|
-
fieldSelection:
|
|
568
|
-
mappings:
|
|
610
|
+
fieldSelection: current.fieldSelection,
|
|
611
|
+
mappings: current.mappings,
|
|
569
612
|
exceptionRows: payload.exceptionRows,
|
|
570
|
-
baseName: payload.baseName ||
|
|
613
|
+
baseName: payload.baseName || current.title,
|
|
571
614
|
});
|
|
572
|
-
const completed = await workflow.
|
|
573
|
-
expectedRevision:
|
|
615
|
+
const completed = await workflow.completeLocalDelivery(taskId, {
|
|
616
|
+
expectedRevision: current.revision,
|
|
617
|
+
summary: payload.summary,
|
|
574
618
|
artifacts,
|
|
575
619
|
});
|
|
576
620
|
return writeJson(res, 201, {
|
|
577
621
|
ok: true,
|
|
578
622
|
marker: 'data-cleaning-artifacts-v1',
|
|
579
|
-
task: completed,
|
|
623
|
+
task: workflowView(completed),
|
|
580
624
|
artifacts: completed.artifacts,
|
|
581
625
|
});
|
|
582
626
|
}
|
|
@@ -609,6 +653,22 @@ export function mountWebRoutes(wctx, { logger, report, TOOL_NAME, SKILL_NAME })
|
|
|
609
653
|
return writeJson(res, 405, { ok: false, code: 'DC_METHOD', message: 'Artifact route requires GET or POST.' });
|
|
610
654
|
}
|
|
611
655
|
|
|
656
|
+
if (rest.length === 2 && rest[1] === 'verify-source') {
|
|
657
|
+
if (req.method !== 'POST') {
|
|
658
|
+
return writeJson(res, 405, { ok: false, code: 'DC_METHOD', message: 'Source verification requires POST.' });
|
|
659
|
+
}
|
|
660
|
+
const body = await readBody(req);
|
|
661
|
+
const payload = body.length ? JSON.parse(body.toString('utf8')) : {};
|
|
662
|
+
const current = await workflow.assertRequestOrigin(taskId, payload);
|
|
663
|
+
const verified = Boolean(current.source?.checksum)
|
|
664
|
+
&& sourceVerifier(payload.headers, payload.rows, current.source.checksum) === current.source.checksum;
|
|
665
|
+
return writeJson(res, 200, {
|
|
666
|
+
ok: true,
|
|
667
|
+
marker: 'data-cleaning-source-verifier-v1',
|
|
668
|
+
verified,
|
|
669
|
+
});
|
|
670
|
+
}
|
|
671
|
+
|
|
612
672
|
if (rest.length !== 3 || rest[1] !== 'actions' || req.method !== 'POST') {
|
|
613
673
|
return writeJson(res, 404, { ok: false, code: 'DC_WORKFLOW_ROUTE', message: 'Workflow route not found.' });
|
|
614
674
|
}
|
|
@@ -617,27 +677,22 @@ export function mountWebRoutes(wctx, { logger, report, TOOL_NAME, SKILL_NAME })
|
|
|
617
677
|
const action = rest[2];
|
|
618
678
|
await workflow.assertRequestOrigin(taskId, payload);
|
|
619
679
|
const actions = {
|
|
620
|
-
upload: () => workflow.recordUpload(taskId, payload),
|
|
621
|
-
rules: () => workflow.confirmRules(taskId, payload),
|
|
622
|
-
quality: () => workflow.recordQuality(taskId, payload),
|
|
623
|
-
'
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
enrichment: () => workflow.recordEnrichment(taskId, payload),
|
|
627
|
-
'local-export-ready': () => workflow.prepareLocalExport(taskId, payload),
|
|
628
|
-
export: () => workflow.recordExport(taskId, payload),
|
|
629
|
-
'parse-failed': () => workflow.recordParseFailure(taskId, payload),
|
|
630
|
-
'authorization-required': () => workflow.requireAuthorization(taskId, payload),
|
|
631
|
-
fail: () => workflow.recordFailure(taskId, payload),
|
|
632
|
-
cancel: () => workflow.cancel(taskId, payload),
|
|
680
|
+
upload: { action: WORKFLOW_ACTIONS.IMPORT_DATA, run: () => workflow.recordUpload(taskId, payload) },
|
|
681
|
+
rules: { action: WORKFLOW_ACTIONS.CONFIRM_RULES, run: () => workflow.confirmRules(taskId, payload) },
|
|
682
|
+
quality: { action: WORKFLOW_ACTIONS.RUN_QUALITY, run: () => workflow.recordQuality(taskId, payload) },
|
|
683
|
+
'retry-delivery': { action: WORKFLOW_ACTIONS.RETRY_DELIVERY, run: async () => (
|
|
684
|
+
await execution.retryDelivery({ taskId, expectedRevision: payload.expectedRevision })
|
|
685
|
+
).task },
|
|
633
686
|
};
|
|
634
687
|
if (!actions[action]) {
|
|
635
688
|
return writeJson(res, 404, { ok: false, code: 'DC_WORKFLOW_ACTION', message: `Unknown workflow action: ${action}` });
|
|
636
689
|
}
|
|
690
|
+
const current = await workflow.require(taskId);
|
|
691
|
+
requireWorkflowAction(current, actions[action].action);
|
|
637
692
|
return writeJson(res, 200, {
|
|
638
693
|
ok: true,
|
|
639
694
|
marker: 'data-cleaning-workflow-v2',
|
|
640
|
-
task: await actions[action](),
|
|
695
|
+
task: workflowView(await actions[action].run()),
|
|
641
696
|
});
|
|
642
697
|
} catch (error) {
|
|
643
698
|
if (error instanceof SyntaxError) {
|
|
@@ -984,7 +1039,7 @@ export function mountWebRoutes(wctx, { logger, report, TOOL_NAME, SKILL_NAME })
|
|
|
984
1039
|
if (!isTrusted(req)) return writeJson(res, 403, { ok: false, error: 'untrusted origin' });
|
|
985
1040
|
try {
|
|
986
1041
|
const body = await readBody(req);
|
|
987
|
-
const { fmt, headers, rows } = await parseUpload(body);
|
|
1042
|
+
const { fmt, headers, rows, checksum } = await parseUpload(body);
|
|
988
1043
|
writeJson(res, 200, {
|
|
989
1044
|
ok: true,
|
|
990
1045
|
fmt,
|
|
@@ -992,6 +1047,7 @@ export function mountWebRoutes(wctx, { logger, report, TOOL_NAME, SKILL_NAME })
|
|
|
992
1047
|
rowCount: rows.length,
|
|
993
1048
|
preview: rows.slice(0, 5),
|
|
994
1049
|
rows,
|
|
1050
|
+
checksum,
|
|
995
1051
|
});
|
|
996
1052
|
} catch (error) {
|
|
997
1053
|
writeJson(res, 400, { ok: false, code: error?.code ?? 'DC_PARSE', message: error instanceof Error ? error.message : String(error) });
|
package/lib/workflow-contract.js
CHANGED
|
@@ -10,6 +10,27 @@
|
|
|
10
10
|
import { QCC_FIELD_CATALOG } from './qcc-field-catalog.js';
|
|
11
11
|
|
|
12
12
|
export const WORKFLOW_SCHEMA_VERSION = 2;
|
|
13
|
+
export const FLOW_VERSION = 1;
|
|
14
|
+
|
|
15
|
+
export const WORKFLOW_STAGE_ACCESS = Object.freeze({
|
|
16
|
+
CURRENT: 'current',
|
|
17
|
+
READ: 'read',
|
|
18
|
+
LOCKED: 'locked',
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
export const WORKFLOW_ACTIONS = Object.freeze({
|
|
22
|
+
IMPORT_DATA: 'import-data',
|
|
23
|
+
EDIT_RULES: 'edit-rules',
|
|
24
|
+
CONFIRM_RULES: 'confirm-rules',
|
|
25
|
+
RUN_QUALITY: 'run-quality',
|
|
26
|
+
PREPARE_QCC_COMMAND: 'prepare-qcc-command',
|
|
27
|
+
RUN_LOCAL_CLEAN: 'run-local-clean',
|
|
28
|
+
RUN_LOCAL_COMPLETE: 'run-local-complete',
|
|
29
|
+
CREATE_LOCAL_ARTIFACTS: 'create-local-artifacts',
|
|
30
|
+
RESOLVE_CANDIDATE: 'resolve-candidate',
|
|
31
|
+
RETRY_FAILED: 'retry-failed',
|
|
32
|
+
RETRY_DELIVERY: 'retry-delivery',
|
|
33
|
+
});
|
|
13
34
|
|
|
14
35
|
export const WORKFLOW_STAGES = Object.freeze([
|
|
15
36
|
Object.freeze({ id: 'upload', label: '上传数据', order: 1 }),
|
|
@@ -79,17 +100,17 @@ const STATE_IDS = new Set(WORKFLOW_STATES);
|
|
|
79
100
|
const TRANSITIONS = Object.freeze({
|
|
80
101
|
draft: ['uploaded', 'parse_failed', 'cancelled'],
|
|
81
102
|
uploaded: ['draft', 'rules_confirmed', 'parse_failed', 'cancelled'],
|
|
82
|
-
rules_confirmed: ['diagnosed', '
|
|
83
|
-
diagnosed: ['matching', '
|
|
84
|
-
matching: ['matched', 'review_required', 'authorization_required', 'partial', 'failed', 'cancelled'],
|
|
85
|
-
review_required: ['matching', 'matched', 'authorization_required', 'partial', 'failed', 'cancelled'],
|
|
103
|
+
rules_confirmed: ['diagnosed', 'failed', 'cancelled'],
|
|
104
|
+
diagnosed: ['matching', 'export_ready', 'completed', 'authorization_required', 'failed', 'cancelled'],
|
|
105
|
+
matching: ['matched', 'review_required', 'authorization_required', 'partial', 'completed', 'failed', 'cancelled'],
|
|
106
|
+
review_required: ['matching', 'matched', 'authorization_required', 'partial', 'completed', 'failed', 'cancelled'],
|
|
86
107
|
matched: ['enriching', 'export_ready', 'authorization_required', 'partial', 'failed', 'cancelled'],
|
|
87
108
|
enriching: ['export_ready', 'review_required', 'authorization_required', 'partial', 'failed', 'cancelled'],
|
|
88
|
-
export_ready: ['enriching', 'completed', 'failed', 'cancelled'],
|
|
109
|
+
export_ready: ['enriching', 'partial', 'completed', 'failed', 'cancelled'],
|
|
89
110
|
parse_failed: ['draft', 'uploaded', 'cancelled'],
|
|
90
111
|
authorization_required: ['matching', 'enriching', 'failed', 'cancelled'],
|
|
91
|
-
partial: ['
|
|
92
|
-
failed: ['
|
|
112
|
+
partial: ['review_required', 'authorization_required', 'completed', 'failed', 'cancelled'],
|
|
113
|
+
failed: ['partial', 'completed', 'cancelled'],
|
|
93
114
|
completed: [],
|
|
94
115
|
cancelled: [],
|
|
95
116
|
});
|
|
@@ -173,6 +194,100 @@ export function normalizeWorkflowDraft(value = {}) {
|
|
|
173
194
|
};
|
|
174
195
|
}
|
|
175
196
|
|
|
197
|
+
export function requiresQcc(task) {
|
|
198
|
+
return Array.isArray(task?.objectives)
|
|
199
|
+
&& task.objectives.some((objective) => ['validate_identity', 'complete_fields'].includes(objective));
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
function currentStageFor(task) {
|
|
203
|
+
switch (task?.state) {
|
|
204
|
+
case 'draft':
|
|
205
|
+
case 'parse_failed':
|
|
206
|
+
return 'upload';
|
|
207
|
+
case 'uploaded':
|
|
208
|
+
case 'rules_confirmed':
|
|
209
|
+
return 'rules';
|
|
210
|
+
case 'diagnosed':
|
|
211
|
+
return requiresQcc(task) ? 'match' : 'enrich';
|
|
212
|
+
case 'matching':
|
|
213
|
+
case 'review_required':
|
|
214
|
+
return 'match';
|
|
215
|
+
case 'matched':
|
|
216
|
+
case 'enriching':
|
|
217
|
+
return 'enrich';
|
|
218
|
+
case 'export_ready':
|
|
219
|
+
case 'partial':
|
|
220
|
+
case 'completed':
|
|
221
|
+
return 'download';
|
|
222
|
+
case 'authorization_required':
|
|
223
|
+
case 'failed':
|
|
224
|
+
case 'cancelled':
|
|
225
|
+
return STAGE_IDS.has(task?.stage) ? task.stage : 'upload';
|
|
226
|
+
default:
|
|
227
|
+
return STAGE_IDS.has(task?.stage) ? task.stage : 'upload';
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
export function allowedWorkflowActions(task) {
|
|
232
|
+
const state = task?.state;
|
|
233
|
+
if (state === 'draft' || state === 'parse_failed') {
|
|
234
|
+
return [WORKFLOW_ACTIONS.IMPORT_DATA, WORKFLOW_ACTIONS.EDIT_RULES];
|
|
235
|
+
}
|
|
236
|
+
if (state === 'uploaded') {
|
|
237
|
+
return [WORKFLOW_ACTIONS.IMPORT_DATA, WORKFLOW_ACTIONS.EDIT_RULES, WORKFLOW_ACTIONS.CONFIRM_RULES];
|
|
238
|
+
}
|
|
239
|
+
if (state === 'rules_confirmed') return [WORKFLOW_ACTIONS.RUN_QUALITY];
|
|
240
|
+
if (state === 'diagnosed') {
|
|
241
|
+
return requiresQcc(task)
|
|
242
|
+
? [WORKFLOW_ACTIONS.PREPARE_QCC_COMMAND]
|
|
243
|
+
: [
|
|
244
|
+
WORKFLOW_ACTIONS.RUN_LOCAL_CLEAN,
|
|
245
|
+
WORKFLOW_ACTIONS.RUN_LOCAL_COMPLETE,
|
|
246
|
+
WORKFLOW_ACTIONS.CREATE_LOCAL_ARTIFACTS,
|
|
247
|
+
];
|
|
248
|
+
}
|
|
249
|
+
if (state === 'review_required') return [WORKFLOW_ACTIONS.RESOLVE_CANDIDATE];
|
|
250
|
+
if (state === 'export_ready') {
|
|
251
|
+
return requiresQcc(task)
|
|
252
|
+
? [WORKFLOW_ACTIONS.RETRY_DELIVERY]
|
|
253
|
+
: [WORKFLOW_ACTIONS.CREATE_LOCAL_ARTIFACTS];
|
|
254
|
+
}
|
|
255
|
+
if (state === 'partial') {
|
|
256
|
+
return [
|
|
257
|
+
WORKFLOW_ACTIONS.RETRY_FAILED,
|
|
258
|
+
...(task?.error?.code === 'DC_DELIVERY_FAILED' ? [WORKFLOW_ACTIONS.RETRY_DELIVERY] : []),
|
|
259
|
+
];
|
|
260
|
+
}
|
|
261
|
+
if (state === 'failed' && task?.error?.code === 'DC_DELIVERY_FAILED' && task?.qccRunId) {
|
|
262
|
+
return [WORKFLOW_ACTIONS.RETRY_DELIVERY];
|
|
263
|
+
}
|
|
264
|
+
return [];
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
export function deriveWorkflowFlow(task) {
|
|
268
|
+
const currentStage = currentStageFor(task);
|
|
269
|
+
const stageAccess = Object.fromEntries(WORKFLOW_STAGES.map(({ id }) => [id, WORKFLOW_STAGE_ACCESS.LOCKED]));
|
|
270
|
+
stageAccess[currentStage] = WORKFLOW_STAGE_ACCESS.CURRENT;
|
|
271
|
+
|
|
272
|
+
const makeReadable = (stage, condition) => {
|
|
273
|
+
if (condition && stage !== currentStage) stageAccess[stage] = WORKFLOW_STAGE_ACCESS.READ;
|
|
274
|
+
};
|
|
275
|
+
makeReadable('upload', Boolean(task?.source));
|
|
276
|
+
makeReadable('rules', !['draft', 'uploaded', 'parse_failed'].includes(task?.state));
|
|
277
|
+
makeReadable('match', requiresQcc(task) && Boolean(task?.matchSummary || task?.qccRunId));
|
|
278
|
+
makeReadable('enrich', Boolean(task?.enrichmentSummary));
|
|
279
|
+
makeReadable('download', Array.isArray(task?.artifacts) && task.artifacts.length > 0);
|
|
280
|
+
|
|
281
|
+
const allowedActions = allowedWorkflowActions(task);
|
|
282
|
+
return {
|
|
283
|
+
flowVersion: FLOW_VERSION,
|
|
284
|
+
currentStage,
|
|
285
|
+
stageAccess,
|
|
286
|
+
allowedActions,
|
|
287
|
+
nextAction: allowedActions[0] ?? null,
|
|
288
|
+
};
|
|
289
|
+
}
|
|
290
|
+
|
|
176
291
|
export function canTransition(from, to) {
|
|
177
292
|
if (!STATE_IDS.has(from) || !STATE_IDS.has(to)) return false;
|
|
178
293
|
return TRANSITIONS[from].includes(to);
|
|
@@ -187,7 +302,10 @@ export function assertWorkflowTransition(from, to) {
|
|
|
187
302
|
export function publicWorkflowContract() {
|
|
188
303
|
return {
|
|
189
304
|
schemaVersion: WORKFLOW_SCHEMA_VERSION,
|
|
305
|
+
flowVersion: FLOW_VERSION,
|
|
190
306
|
stages: WORKFLOW_STAGES,
|
|
307
|
+
stageAccessModes: Object.values(WORKFLOW_STAGE_ACCESS),
|
|
308
|
+
actions: Object.values(WORKFLOW_ACTIONS),
|
|
191
309
|
states: WORKFLOW_STATES,
|
|
192
310
|
terminalStates: TERMINAL_WORKFLOW_STATES,
|
|
193
311
|
sourceTypes: SOURCE_TYPES,
|