dsh-data-cleaning-agent 0.9.13 → 0.9.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/README.en.md +2 -2
- package/README.md +3 -3
- package/docs/RELEASE-0.9.14.md +12 -0
- package/docs/RELEASE-0.9.15.md +13 -0
- package/lib/client.js +1006 -202
- package/lib/image-workflow.js +7 -1
- package/lib/qcc-command.js +8 -0
- package/lib/qcc-runs.js +9 -0
- package/lib/web.js +82 -42
- package/lib/workflow-contract.js +125 -7
- package/lib/workflow-execution.js +121 -20
- package/lib/workflow.js +170 -14
- package/package.json +3 -1
package/lib/image-workflow.js
CHANGED
|
@@ -13,7 +13,13 @@ export function createImageWorkflow({ getWorkflow, execution, commands }) {
|
|
|
13
13
|
task = await store.recordUpload(task.id, { expectedRevision: task.revision,
|
|
14
14
|
source: { type: 'image', fileName: record.fileName, sizeBytes: record.sizeBytes,
|
|
15
15
|
rowCount: rows.length, columnCount: 1, headers: ['主体标识'] } });
|
|
16
|
-
task = await store.updateDraft(task.id, {
|
|
16
|
+
task = await store.updateDraft(task.id, {
|
|
17
|
+
expectedRevision: task.revision,
|
|
18
|
+
mappings,
|
|
19
|
+
objectives: task.objectives.includes('complete_fields')
|
|
20
|
+
? task.objectives
|
|
21
|
+
: [...task.objectives, 'complete_fields'],
|
|
22
|
+
});
|
|
17
23
|
if (result.truncated || result.needsReview || rows.length > 100) {
|
|
18
24
|
return { taskId: task.id, deliveryState: 'review_required', artifacts: [],
|
|
19
25
|
reviewReason: rows.length > 100 || result.truncated ? '识别名单超出单批范围,请核验并拆分。' : '识别文本含模糊字符或待确认提示,请核验名单。' };
|
package/lib/qcc-command.js
CHANGED
|
@@ -198,6 +198,14 @@ export class QccCommandStore {
|
|
|
198
198
|
return output;
|
|
199
199
|
}
|
|
200
200
|
|
|
201
|
+
hasActiveForTask(taskId) {
|
|
202
|
+
this.cleanup();
|
|
203
|
+
const id = String(taskId ?? '');
|
|
204
|
+
return [...this.commands.values()].some((command) => (
|
|
205
|
+
command.taskId === id && ['prepared', 'running'].includes(command.state)
|
|
206
|
+
));
|
|
207
|
+
}
|
|
208
|
+
|
|
201
209
|
async run(commandId, execution) {
|
|
202
210
|
const record = this.require(commandId);
|
|
203
211
|
if (!execution?.agent || !execution?.token) {
|
package/lib/qcc-runs.js
CHANGED
|
@@ -191,6 +191,15 @@ export class G5RunStore {
|
|
|
191
191
|
return this.snapshot(this.requireRun(id));
|
|
192
192
|
}
|
|
193
193
|
|
|
194
|
+
capabilities(id) {
|
|
195
|
+
this.cleanup();
|
|
196
|
+
const record = this.runs.get(String(id ?? ''));
|
|
197
|
+
return {
|
|
198
|
+
liveRunAvailable: Boolean(record),
|
|
199
|
+
retryableFailuresAvailable: Boolean(record?.errors.some((item) => item.error?.retryable)),
|
|
200
|
+
};
|
|
201
|
+
}
|
|
202
|
+
|
|
194
203
|
snapshot(record) {
|
|
195
204
|
return clone({
|
|
196
205
|
runId: record.id,
|
package/lib/web.js
CHANGED
|
@@ -7,13 +7,19 @@
|
|
|
7
7
|
* - 上传体大小上限(parseBody)。
|
|
8
8
|
* - 同步接口返回明细行仅供「已授权同源 UI」下载,不面向模型。
|
|
9
9
|
*/
|
|
10
|
+
import { createHash, randomBytes } from 'node:crypto';
|
|
10
11
|
import { parseCsv, parseXlsx, parseJson, detectFormat, toCsv } from './engine.js';
|
|
11
12
|
import { runSync, DataCleaningJobs } from './jobs.js';
|
|
12
13
|
import { QccBridgeError, QccHostBridge } from './qcc.js';
|
|
13
14
|
import { QccCommandStore, registerQccCommandTool, TOOL_QCC_COMMAND } from './qcc-command.js';
|
|
14
15
|
import { fingerprintRequest, G5RunStore } from './qcc-runs.js';
|
|
15
16
|
import { PHASE3_BATCH_LIMITS, Phase3BatchService, Phase3RunStore } from './qcc-phase3-batch.js';
|
|
16
|
-
import {
|
|
17
|
+
import {
|
|
18
|
+
WORKFLOW_ACTIONS,
|
|
19
|
+
allowedWorkflowActions,
|
|
20
|
+
deriveWorkflowFlow,
|
|
21
|
+
publicWorkflowContract,
|
|
22
|
+
} from './workflow-contract.js';
|
|
17
23
|
import { DataCleaningWorkflowStore, WorkflowError } from './workflow.js';
|
|
18
24
|
import { ArtifactError, WorkflowArtifactStore } from './artifacts.js';
|
|
19
25
|
import { renderArtifactPreview } from './artifact-preview.js';
|
|
@@ -141,6 +147,23 @@ function writeImageError(res, error) {
|
|
|
141
147
|
}
|
|
142
148
|
|
|
143
149
|
/** 从 JSON 协议解析上传:{ filename, content }。content 为字符串;xlsx 时为 base64。 */
|
|
150
|
+
function canonicalize(value) {
|
|
151
|
+
if (Array.isArray(value)) return value.map(canonicalize);
|
|
152
|
+
if (value === null || typeof value !== 'object') return value;
|
|
153
|
+
return Object.fromEntries(Object.keys(value).sort().map((key) => [key, canonicalize(value[key])]));
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
function sourceVerifier(headers, rows, previous = '') {
|
|
157
|
+
const match = /^sha256-canonical-rows-v1:([a-f0-9]{32}):[a-f0-9]{64}$/.exec(String(previous));
|
|
158
|
+
const salt = match?.[1] || randomBytes(16).toString('hex');
|
|
159
|
+
const canonical = JSON.stringify({
|
|
160
|
+
headers: (Array.isArray(headers) ? headers : []).map(String),
|
|
161
|
+
rows: canonicalize(Array.isArray(rows) ? rows : []),
|
|
162
|
+
});
|
|
163
|
+
const digest = createHash('sha256').update(salt).update('\0').update(canonical).digest('hex');
|
|
164
|
+
return `sha256-canonical-rows-v1:${salt}:${digest}`;
|
|
165
|
+
}
|
|
166
|
+
|
|
144
167
|
async function parseUpload(body) {
|
|
145
168
|
let payload;
|
|
146
169
|
try {
|
|
@@ -153,12 +176,20 @@ async function parseUpload(body) {
|
|
|
153
176
|
const filename = String(payload?.filename ?? 'data.csv');
|
|
154
177
|
const content = payload?.content ?? '';
|
|
155
178
|
const fmt = detectFormat(filename);
|
|
179
|
+
let parsed;
|
|
156
180
|
if (fmt === 'xlsx') {
|
|
157
181
|
const buf = Buffer.from(String(content), 'base64');
|
|
158
|
-
|
|
182
|
+
parsed = await parseXlsx(buf);
|
|
183
|
+
} else if (fmt === 'json') {
|
|
184
|
+
parsed = parseJson(String(content));
|
|
185
|
+
} else {
|
|
186
|
+
parsed = parseCsv(String(content));
|
|
159
187
|
}
|
|
160
|
-
|
|
161
|
-
|
|
188
|
+
return {
|
|
189
|
+
fmt,
|
|
190
|
+
...parsed,
|
|
191
|
+
checksum: sourceVerifier(parsed.headers, parsed.rows, payload?.previousChecksum),
|
|
192
|
+
};
|
|
162
193
|
}
|
|
163
194
|
|
|
164
195
|
const UI_HTML = `<!doctype html>
|
|
@@ -307,7 +338,11 @@ export function mountWebRoutes(wctx, { logger, report, TOOL_NAME, SKILL_NAME })
|
|
|
307
338
|
const disposers = [];
|
|
308
339
|
const qccBridge = new QccHostBridge({ tools, logger });
|
|
309
340
|
const g5Runs = new G5RunStore();
|
|
310
|
-
const execution = createWorkflowExecution({
|
|
341
|
+
const execution = createWorkflowExecution({
|
|
342
|
+
getWorkflow: () => getWorkflow(),
|
|
343
|
+
artifacts: wctx.fs ? new WorkflowArtifactStore({ fs: wctx.fs }) : null,
|
|
344
|
+
getRun: (runId) => g5Runs.get(runId),
|
|
345
|
+
});
|
|
311
346
|
const qccCommands = new QccCommandStore({ bridge: qccBridge, runs: g5Runs, lifecycle: execution });
|
|
312
347
|
const imageIntake = new ImageIntakeStore({ tools,
|
|
313
348
|
continueWorkflow: createImageWorkflow({ getWorkflow: () => getWorkflow(), execution, commands: qccCommands }) });
|
|
@@ -353,6 +388,22 @@ export function mountWebRoutes(wctx, { logger, report, TOOL_NAME, SKILL_NAME })
|
|
|
353
388
|
return workflowReady;
|
|
354
389
|
};
|
|
355
390
|
|
|
391
|
+
const workflowView = (task) => ({
|
|
392
|
+
...task,
|
|
393
|
+
flow: deriveWorkflowFlow(task),
|
|
394
|
+
runtimeCapabilities: {
|
|
395
|
+
activeCommand: qccCommands.hasActiveForTask(task.id),
|
|
396
|
+
...g5Runs.capabilities(task.qccRunId),
|
|
397
|
+
},
|
|
398
|
+
});
|
|
399
|
+
|
|
400
|
+
const requireWorkflowAction = (task, action) => {
|
|
401
|
+
if (!allowedWorkflowActions(task).includes(action)) {
|
|
402
|
+
const current = deriveWorkflowFlow(task).currentStage;
|
|
403
|
+
throw new WorkflowError('DC_WORKFLOW_ACTION_LOCKED', `当前任务位于“${current}”,不能执行该操作。`, 409);
|
|
404
|
+
}
|
|
405
|
+
};
|
|
406
|
+
|
|
356
407
|
register('/data-cleaning/', (req, res) => {
|
|
357
408
|
if (!isTrusted(req)) { res.writeHead(403); return res.end('untrusted origin'); }
|
|
358
409
|
res.writeHead(200, { 'content-type': 'text/html; charset=utf-8', 'cache-control': 'no-store' });
|
|
@@ -490,7 +541,7 @@ export function mountWebRoutes(wctx, { logger, report, TOOL_NAME, SKILL_NAME })
|
|
|
490
541
|
return writeJson(res, 200, {
|
|
491
542
|
ok: true,
|
|
492
543
|
marker: 'data-cleaning-workflow-v2',
|
|
493
|
-
tasks: await workflow.list(),
|
|
544
|
+
tasks: (await workflow.list()).map(workflowView),
|
|
494
545
|
});
|
|
495
546
|
}
|
|
496
547
|
if (req.method === 'POST') {
|
|
@@ -499,7 +550,7 @@ export function mountWebRoutes(wctx, { logger, report, TOOL_NAME, SKILL_NAME })
|
|
|
499
550
|
return writeJson(res, 201, {
|
|
500
551
|
ok: true,
|
|
501
552
|
marker: 'data-cleaning-workflow-v2',
|
|
502
|
-
task: await workflow.create(payload),
|
|
553
|
+
task: workflowView(await workflow.create(payload)),
|
|
503
554
|
});
|
|
504
555
|
}
|
|
505
556
|
return writeJson(res, 405, { ok: false, code: 'DC_METHOD', message: 'GET or POST required' });
|
|
@@ -511,17 +562,18 @@ export function mountWebRoutes(wctx, { logger, report, TOOL_NAME, SKILL_NAME })
|
|
|
511
562
|
return writeJson(res, 200, {
|
|
512
563
|
ok: true,
|
|
513
564
|
marker: 'data-cleaning-workflow-v2',
|
|
514
|
-
task: await workflow.require(taskId),
|
|
565
|
+
task: workflowView(await workflow.require(taskId)),
|
|
515
566
|
});
|
|
516
567
|
}
|
|
517
568
|
if (req.method === 'PATCH') {
|
|
518
569
|
const body = await readBody(req);
|
|
519
570
|
const payload = body.length ? JSON.parse(body.toString('utf8')) : {};
|
|
520
|
-
await workflow.assertRequestOrigin(taskId, payload);
|
|
571
|
+
const current = await workflow.assertRequestOrigin(taskId, payload);
|
|
572
|
+
requireWorkflowAction(current, WORKFLOW_ACTIONS.EDIT_RULES);
|
|
521
573
|
return writeJson(res, 200, {
|
|
522
574
|
ok: true,
|
|
523
575
|
marker: 'data-cleaning-workflow-v2',
|
|
524
|
-
task: await workflow.updateDraft(taskId, payload),
|
|
576
|
+
task: workflowView(await workflow.updateDraft(taskId, payload)),
|
|
525
577
|
});
|
|
526
578
|
}
|
|
527
579
|
return writeJson(res, 405, { ok: false, code: 'DC_METHOD', message: 'GET or PATCH required' });
|
|
@@ -540,7 +592,7 @@ export function mountWebRoutes(wctx, { logger, report, TOOL_NAME, SKILL_NAME })
|
|
|
540
592
|
return writeJson(res, 200, {
|
|
541
593
|
ok: true,
|
|
542
594
|
marker: 'data-cleaning-artifacts-v1',
|
|
543
|
-
task: current,
|
|
595
|
+
task: workflowView(current),
|
|
544
596
|
artifacts: current.artifacts,
|
|
545
597
|
});
|
|
546
598
|
}
|
|
@@ -551,32 +603,24 @@ export function mountWebRoutes(wctx, { logger, report, TOOL_NAME, SKILL_NAME })
|
|
|
551
603
|
if (Number(payload.expectedRevision) !== current.revision) {
|
|
552
604
|
throw new WorkflowError('DC_WORKFLOW_REVISION_CONFLICT', 'Workflow task was updated by another session.', 409);
|
|
553
605
|
}
|
|
554
|
-
|
|
555
|
-
if (['rules_confirmed', 'diagnosed'].includes(readyTask.state)) {
|
|
556
|
-
readyTask = await workflow.prepareLocalExport(taskId, {
|
|
557
|
-
expectedRevision: readyTask.revision,
|
|
558
|
-
summary: payload.summary,
|
|
559
|
-
});
|
|
560
|
-
}
|
|
561
|
-
if (!['export_ready', 'partial'].includes(readyTask.state)) {
|
|
562
|
-
throw new WorkflowError('DC_WORKFLOW_EXPORT_STATE', 'Output is not ready for durable export.', 409);
|
|
563
|
-
}
|
|
606
|
+
requireWorkflowAction(current, WORKFLOW_ACTIONS.CREATE_LOCAL_ARTIFACTS);
|
|
564
607
|
const artifacts = await artifactStore.createBundle(taskId, {
|
|
565
608
|
rows: payload.rows,
|
|
566
609
|
headers: payload.headers,
|
|
567
|
-
fieldSelection:
|
|
568
|
-
mappings:
|
|
610
|
+
fieldSelection: current.fieldSelection,
|
|
611
|
+
mappings: current.mappings,
|
|
569
612
|
exceptionRows: payload.exceptionRows,
|
|
570
|
-
baseName: payload.baseName ||
|
|
613
|
+
baseName: payload.baseName || current.title,
|
|
571
614
|
});
|
|
572
|
-
const completed = await workflow.
|
|
573
|
-
expectedRevision:
|
|
615
|
+
const completed = await workflow.completeLocalDelivery(taskId, {
|
|
616
|
+
expectedRevision: current.revision,
|
|
617
|
+
summary: payload.summary,
|
|
574
618
|
artifacts,
|
|
575
619
|
});
|
|
576
620
|
return writeJson(res, 201, {
|
|
577
621
|
ok: true,
|
|
578
622
|
marker: 'data-cleaning-artifacts-v1',
|
|
579
|
-
task: completed,
|
|
623
|
+
task: workflowView(completed),
|
|
580
624
|
artifacts: completed.artifacts,
|
|
581
625
|
});
|
|
582
626
|
}
|
|
@@ -617,27 +661,22 @@ export function mountWebRoutes(wctx, { logger, report, TOOL_NAME, SKILL_NAME })
|
|
|
617
661
|
const action = rest[2];
|
|
618
662
|
await workflow.assertRequestOrigin(taskId, payload);
|
|
619
663
|
const actions = {
|
|
620
|
-
upload: () => workflow.recordUpload(taskId, payload),
|
|
621
|
-
rules: () => workflow.confirmRules(taskId, payload),
|
|
622
|
-
quality: () => workflow.recordQuality(taskId, payload),
|
|
623
|
-
'
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
enrichment: () => workflow.recordEnrichment(taskId, payload),
|
|
627
|
-
'local-export-ready': () => workflow.prepareLocalExport(taskId, payload),
|
|
628
|
-
export: () => workflow.recordExport(taskId, payload),
|
|
629
|
-
'parse-failed': () => workflow.recordParseFailure(taskId, payload),
|
|
630
|
-
'authorization-required': () => workflow.requireAuthorization(taskId, payload),
|
|
631
|
-
fail: () => workflow.recordFailure(taskId, payload),
|
|
632
|
-
cancel: () => workflow.cancel(taskId, payload),
|
|
664
|
+
upload: { action: WORKFLOW_ACTIONS.IMPORT_DATA, run: () => workflow.recordUpload(taskId, payload) },
|
|
665
|
+
rules: { action: WORKFLOW_ACTIONS.CONFIRM_RULES, run: () => workflow.confirmRules(taskId, payload) },
|
|
666
|
+
quality: { action: WORKFLOW_ACTIONS.RUN_QUALITY, run: () => workflow.recordQuality(taskId, payload) },
|
|
667
|
+
'retry-delivery': { action: WORKFLOW_ACTIONS.RETRY_DELIVERY, run: async () => (
|
|
668
|
+
await execution.retryDelivery({ taskId, expectedRevision: payload.expectedRevision })
|
|
669
|
+
).task },
|
|
633
670
|
};
|
|
634
671
|
if (!actions[action]) {
|
|
635
672
|
return writeJson(res, 404, { ok: false, code: 'DC_WORKFLOW_ACTION', message: `Unknown workflow action: ${action}` });
|
|
636
673
|
}
|
|
674
|
+
const current = await workflow.require(taskId);
|
|
675
|
+
requireWorkflowAction(current, actions[action].action);
|
|
637
676
|
return writeJson(res, 200, {
|
|
638
677
|
ok: true,
|
|
639
678
|
marker: 'data-cleaning-workflow-v2',
|
|
640
|
-
task: await actions[action](),
|
|
679
|
+
task: workflowView(await actions[action].run()),
|
|
641
680
|
});
|
|
642
681
|
} catch (error) {
|
|
643
682
|
if (error instanceof SyntaxError) {
|
|
@@ -984,7 +1023,7 @@ export function mountWebRoutes(wctx, { logger, report, TOOL_NAME, SKILL_NAME })
|
|
|
984
1023
|
if (!isTrusted(req)) return writeJson(res, 403, { ok: false, error: 'untrusted origin' });
|
|
985
1024
|
try {
|
|
986
1025
|
const body = await readBody(req);
|
|
987
|
-
const { fmt, headers, rows } = await parseUpload(body);
|
|
1026
|
+
const { fmt, headers, rows, checksum } = await parseUpload(body);
|
|
988
1027
|
writeJson(res, 200, {
|
|
989
1028
|
ok: true,
|
|
990
1029
|
fmt,
|
|
@@ -992,6 +1031,7 @@ export function mountWebRoutes(wctx, { logger, report, TOOL_NAME, SKILL_NAME })
|
|
|
992
1031
|
rowCount: rows.length,
|
|
993
1032
|
preview: rows.slice(0, 5),
|
|
994
1033
|
rows,
|
|
1034
|
+
checksum,
|
|
995
1035
|
});
|
|
996
1036
|
} catch (error) {
|
|
997
1037
|
writeJson(res, 400, { ok: false, code: error?.code ?? 'DC_PARSE', message: error instanceof Error ? error.message : String(error) });
|
package/lib/workflow-contract.js
CHANGED
|
@@ -10,6 +10,27 @@
|
|
|
10
10
|
import { QCC_FIELD_CATALOG } from './qcc-field-catalog.js';
|
|
11
11
|
|
|
12
12
|
export const WORKFLOW_SCHEMA_VERSION = 2;
|
|
13
|
+
export const FLOW_VERSION = 1;
|
|
14
|
+
|
|
15
|
+
export const WORKFLOW_STAGE_ACCESS = Object.freeze({
|
|
16
|
+
CURRENT: 'current',
|
|
17
|
+
READ: 'read',
|
|
18
|
+
LOCKED: 'locked',
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
export const WORKFLOW_ACTIONS = Object.freeze({
|
|
22
|
+
IMPORT_DATA: 'import-data',
|
|
23
|
+
EDIT_RULES: 'edit-rules',
|
|
24
|
+
CONFIRM_RULES: 'confirm-rules',
|
|
25
|
+
RUN_QUALITY: 'run-quality',
|
|
26
|
+
PREPARE_QCC_COMMAND: 'prepare-qcc-command',
|
|
27
|
+
RUN_LOCAL_CLEAN: 'run-local-clean',
|
|
28
|
+
RUN_LOCAL_COMPLETE: 'run-local-complete',
|
|
29
|
+
CREATE_LOCAL_ARTIFACTS: 'create-local-artifacts',
|
|
30
|
+
RESOLVE_CANDIDATE: 'resolve-candidate',
|
|
31
|
+
RETRY_FAILED: 'retry-failed',
|
|
32
|
+
RETRY_DELIVERY: 'retry-delivery',
|
|
33
|
+
});
|
|
13
34
|
|
|
14
35
|
export const WORKFLOW_STAGES = Object.freeze([
|
|
15
36
|
Object.freeze({ id: 'upload', label: '上传数据', order: 1 }),
|
|
@@ -79,17 +100,17 @@ const STATE_IDS = new Set(WORKFLOW_STATES);
|
|
|
79
100
|
const TRANSITIONS = Object.freeze({
|
|
80
101
|
draft: ['uploaded', 'parse_failed', 'cancelled'],
|
|
81
102
|
uploaded: ['draft', 'rules_confirmed', 'parse_failed', 'cancelled'],
|
|
82
|
-
rules_confirmed: ['diagnosed', '
|
|
83
|
-
diagnosed: ['matching', '
|
|
84
|
-
matching: ['matched', 'review_required', 'authorization_required', 'partial', 'failed', 'cancelled'],
|
|
85
|
-
review_required: ['matching', 'matched', 'authorization_required', 'partial', 'failed', 'cancelled'],
|
|
103
|
+
rules_confirmed: ['diagnosed', 'failed', 'cancelled'],
|
|
104
|
+
diagnosed: ['matching', 'export_ready', 'completed', 'authorization_required', 'failed', 'cancelled'],
|
|
105
|
+
matching: ['matched', 'review_required', 'authorization_required', 'partial', 'completed', 'failed', 'cancelled'],
|
|
106
|
+
review_required: ['matching', 'matched', 'authorization_required', 'partial', 'completed', 'failed', 'cancelled'],
|
|
86
107
|
matched: ['enriching', 'export_ready', 'authorization_required', 'partial', 'failed', 'cancelled'],
|
|
87
108
|
enriching: ['export_ready', 'review_required', 'authorization_required', 'partial', 'failed', 'cancelled'],
|
|
88
|
-
export_ready: ['enriching', 'completed', 'failed', 'cancelled'],
|
|
109
|
+
export_ready: ['enriching', 'partial', 'completed', 'failed', 'cancelled'],
|
|
89
110
|
parse_failed: ['draft', 'uploaded', 'cancelled'],
|
|
90
111
|
authorization_required: ['matching', 'enriching', 'failed', 'cancelled'],
|
|
91
|
-
partial: ['
|
|
92
|
-
failed: ['
|
|
112
|
+
partial: ['review_required', 'authorization_required', 'completed', 'failed', 'cancelled'],
|
|
113
|
+
failed: ['partial', 'completed', 'cancelled'],
|
|
93
114
|
completed: [],
|
|
94
115
|
cancelled: [],
|
|
95
116
|
});
|
|
@@ -173,6 +194,100 @@ export function normalizeWorkflowDraft(value = {}) {
|
|
|
173
194
|
};
|
|
174
195
|
}
|
|
175
196
|
|
|
197
|
+
export function requiresQcc(task) {
|
|
198
|
+
return Array.isArray(task?.objectives)
|
|
199
|
+
&& task.objectives.some((objective) => ['validate_identity', 'complete_fields'].includes(objective));
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
function currentStageFor(task) {
|
|
203
|
+
switch (task?.state) {
|
|
204
|
+
case 'draft':
|
|
205
|
+
case 'parse_failed':
|
|
206
|
+
return 'upload';
|
|
207
|
+
case 'uploaded':
|
|
208
|
+
case 'rules_confirmed':
|
|
209
|
+
return 'rules';
|
|
210
|
+
case 'diagnosed':
|
|
211
|
+
return requiresQcc(task) ? 'match' : 'enrich';
|
|
212
|
+
case 'matching':
|
|
213
|
+
case 'review_required':
|
|
214
|
+
return 'match';
|
|
215
|
+
case 'matched':
|
|
216
|
+
case 'enriching':
|
|
217
|
+
return 'enrich';
|
|
218
|
+
case 'export_ready':
|
|
219
|
+
case 'partial':
|
|
220
|
+
case 'completed':
|
|
221
|
+
return 'download';
|
|
222
|
+
case 'authorization_required':
|
|
223
|
+
case 'failed':
|
|
224
|
+
case 'cancelled':
|
|
225
|
+
return STAGE_IDS.has(task?.stage) ? task.stage : 'upload';
|
|
226
|
+
default:
|
|
227
|
+
return STAGE_IDS.has(task?.stage) ? task.stage : 'upload';
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
export function allowedWorkflowActions(task) {
|
|
232
|
+
const state = task?.state;
|
|
233
|
+
if (state === 'draft' || state === 'parse_failed') {
|
|
234
|
+
return [WORKFLOW_ACTIONS.IMPORT_DATA, WORKFLOW_ACTIONS.EDIT_RULES];
|
|
235
|
+
}
|
|
236
|
+
if (state === 'uploaded') {
|
|
237
|
+
return [WORKFLOW_ACTIONS.IMPORT_DATA, WORKFLOW_ACTIONS.EDIT_RULES, WORKFLOW_ACTIONS.CONFIRM_RULES];
|
|
238
|
+
}
|
|
239
|
+
if (state === 'rules_confirmed') return [WORKFLOW_ACTIONS.RUN_QUALITY];
|
|
240
|
+
if (state === 'diagnosed') {
|
|
241
|
+
return requiresQcc(task)
|
|
242
|
+
? [WORKFLOW_ACTIONS.PREPARE_QCC_COMMAND]
|
|
243
|
+
: [
|
|
244
|
+
WORKFLOW_ACTIONS.RUN_LOCAL_CLEAN,
|
|
245
|
+
WORKFLOW_ACTIONS.RUN_LOCAL_COMPLETE,
|
|
246
|
+
WORKFLOW_ACTIONS.CREATE_LOCAL_ARTIFACTS,
|
|
247
|
+
];
|
|
248
|
+
}
|
|
249
|
+
if (state === 'review_required') return [WORKFLOW_ACTIONS.RESOLVE_CANDIDATE];
|
|
250
|
+
if (state === 'export_ready') {
|
|
251
|
+
return requiresQcc(task)
|
|
252
|
+
? [WORKFLOW_ACTIONS.RETRY_DELIVERY]
|
|
253
|
+
: [WORKFLOW_ACTIONS.CREATE_LOCAL_ARTIFACTS];
|
|
254
|
+
}
|
|
255
|
+
if (state === 'partial') {
|
|
256
|
+
return [
|
|
257
|
+
WORKFLOW_ACTIONS.RETRY_FAILED,
|
|
258
|
+
...(task?.error?.code === 'DC_DELIVERY_FAILED' ? [WORKFLOW_ACTIONS.RETRY_DELIVERY] : []),
|
|
259
|
+
];
|
|
260
|
+
}
|
|
261
|
+
if (state === 'failed' && task?.error?.code === 'DC_DELIVERY_FAILED' && task?.qccRunId) {
|
|
262
|
+
return [WORKFLOW_ACTIONS.RETRY_DELIVERY];
|
|
263
|
+
}
|
|
264
|
+
return [];
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
export function deriveWorkflowFlow(task) {
|
|
268
|
+
const currentStage = currentStageFor(task);
|
|
269
|
+
const stageAccess = Object.fromEntries(WORKFLOW_STAGES.map(({ id }) => [id, WORKFLOW_STAGE_ACCESS.LOCKED]));
|
|
270
|
+
stageAccess[currentStage] = WORKFLOW_STAGE_ACCESS.CURRENT;
|
|
271
|
+
|
|
272
|
+
const makeReadable = (stage, condition) => {
|
|
273
|
+
if (condition && stage !== currentStage) stageAccess[stage] = WORKFLOW_STAGE_ACCESS.READ;
|
|
274
|
+
};
|
|
275
|
+
makeReadable('upload', Boolean(task?.source));
|
|
276
|
+
makeReadable('rules', !['draft', 'uploaded', 'parse_failed'].includes(task?.state));
|
|
277
|
+
makeReadable('match', requiresQcc(task) && Boolean(task?.matchSummary || task?.qccRunId));
|
|
278
|
+
makeReadable('enrich', Boolean(task?.enrichmentSummary));
|
|
279
|
+
makeReadable('download', Array.isArray(task?.artifacts) && task.artifacts.length > 0);
|
|
280
|
+
|
|
281
|
+
const allowedActions = allowedWorkflowActions(task);
|
|
282
|
+
return {
|
|
283
|
+
flowVersion: FLOW_VERSION,
|
|
284
|
+
currentStage,
|
|
285
|
+
stageAccess,
|
|
286
|
+
allowedActions,
|
|
287
|
+
nextAction: allowedActions[0] ?? null,
|
|
288
|
+
};
|
|
289
|
+
}
|
|
290
|
+
|
|
176
291
|
export function canTransition(from, to) {
|
|
177
292
|
if (!STATE_IDS.has(from) || !STATE_IDS.has(to)) return false;
|
|
178
293
|
return TRANSITIONS[from].includes(to);
|
|
@@ -187,7 +302,10 @@ export function assertWorkflowTransition(from, to) {
|
|
|
187
302
|
export function publicWorkflowContract() {
|
|
188
303
|
return {
|
|
189
304
|
schemaVersion: WORKFLOW_SCHEMA_VERSION,
|
|
305
|
+
flowVersion: FLOW_VERSION,
|
|
190
306
|
stages: WORKFLOW_STAGES,
|
|
307
|
+
stageAccessModes: Object.values(WORKFLOW_STAGE_ACCESS),
|
|
308
|
+
actions: Object.values(WORKFLOW_ACTIONS),
|
|
191
309
|
states: WORKFLOW_STATES,
|
|
192
310
|
terminalStates: TERMINAL_WORKFLOW_STATES,
|
|
193
311
|
sourceTypes: SOURCE_TYPES,
|
|
@@ -1,7 +1,34 @@
|
|
|
1
1
|
import { WorkflowError } from './workflow.js';
|
|
2
|
+
import { WORKFLOW_ACTIONS, allowedWorkflowActions } from './workflow-contract.js';
|
|
3
|
+
|
|
4
|
+
function summariesFor(run) {
|
|
5
|
+
const summary = run.summary || {};
|
|
6
|
+
const total = Number(summary.totalRows ?? run.rows.length);
|
|
7
|
+
const reviewRequired = Number(summary.ambiguous ?? 0);
|
|
8
|
+
return {
|
|
9
|
+
reviewRequired,
|
|
10
|
+
matchSummary: {
|
|
11
|
+
total,
|
|
12
|
+
exact: Number(summary.enriched || 0) + Number(summary.fieldReview || 0),
|
|
13
|
+
candidate: reviewRequired,
|
|
14
|
+
confirmed: 0,
|
|
15
|
+
unresolved: Number(summary.unresolved || 0) + Number(summary.missingName || 0),
|
|
16
|
+
failed: Number(summary.failed || 0),
|
|
17
|
+
reviewRequired,
|
|
18
|
+
},
|
|
19
|
+
enrichmentSummary: {
|
|
20
|
+
total,
|
|
21
|
+
completed: Number(summary.enriched || 0),
|
|
22
|
+
unchanged: Number(summary.unresolved || 0) + Number(summary.missingName || 0),
|
|
23
|
+
failed: Number(summary.failed || 0),
|
|
24
|
+
reviewRequired: Number(summary.fieldReview || 0),
|
|
25
|
+
callsUsed: run.audit?.length || 0,
|
|
26
|
+
},
|
|
27
|
+
};
|
|
28
|
+
}
|
|
2
29
|
|
|
3
30
|
// Execution owns finalization: closing a browser must not interrupt result delivery.
|
|
4
|
-
export function createWorkflowExecution({ getWorkflow, artifacts }) {
|
|
31
|
+
export function createWorkflowExecution({ getWorkflow, artifacts, getRun }) {
|
|
5
32
|
const requireStore = async () => {
|
|
6
33
|
const store = await getWorkflow();
|
|
7
34
|
if (!store || !artifacts) throw new WorkflowError('DC_DELIVERY_UNAVAILABLE', 'Host 任务存储或下载服务不可用,请升级 DSH。', 503);
|
|
@@ -11,15 +38,22 @@ export function createWorkflowExecution({ getWorkflow, artifacts }) {
|
|
|
11
38
|
async prepare(payload) {
|
|
12
39
|
const store = await requireStore();
|
|
13
40
|
const task = await store.assertRequestOrigin(payload.taskId, payload);
|
|
14
|
-
|
|
41
|
+
const kind = String(payload.kind || 'enrich');
|
|
42
|
+
const expectedState = kind === 'resolve' ? 'review_required' : kind === 'retry' ? 'partial' : 'diagnosed';
|
|
43
|
+
if (task.state !== expectedState) {
|
|
15
44
|
throw new WorkflowError('DC_EXECUTION_STATE', '请先在工作台确认规则;进行中或已完成任务不能重复启动。', 409);
|
|
16
45
|
}
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
46
|
+
const action = kind === 'resolve'
|
|
47
|
+
? WORKFLOW_ACTIONS.RESOLVE_CANDIDATE
|
|
48
|
+
: kind === 'retry'
|
|
49
|
+
? WORKFLOW_ACTIONS.RETRY_FAILED
|
|
50
|
+
: WORKFLOW_ACTIONS.PREPARE_QCC_COMMAND;
|
|
51
|
+
if (!allowedWorkflowActions(task).includes(action)) {
|
|
52
|
+
throw new WorkflowError('DC_WORKFLOW_ACTION_LOCKED', '当前任务状态不允许生成该执行命令。', 409);
|
|
20
53
|
}
|
|
21
|
-
if (payload.
|
|
22
|
-
if (
|
|
54
|
+
if (Number(payload.expectedRevision) !== task.revision) throw new WorkflowError('DC_WORKFLOW_REVISION_CONFLICT', '任务已更新,请重新生成说明。', 409);
|
|
55
|
+
if (kind !== 'enrich' && payload.runId !== task.qccRunId) throw new WorkflowError('DC_RUN_MISMATCH', '执行结果不属于当前任务。', 409);
|
|
56
|
+
if (kind === 'enrich') {
|
|
23
57
|
if (!Array.isArray(payload.rows) || payload.rows.length !== task.source?.rowCount) throw new WorkflowError('DC_SOURCE_MISMATCH', '任务名单数量不一致,请重新载入。', 409);
|
|
24
58
|
}
|
|
25
59
|
if (!task.fieldSelection.length) throw new WorkflowError('DC_FIELDS_REQUIRED', '未选择外部补全字段,请使用本地清洗与导出。', 400);
|
|
@@ -35,7 +69,9 @@ export function createWorkflowExecution({ getWorkflow, artifacts }) {
|
|
|
35
69
|
const store = await requireStore();
|
|
36
70
|
const task = await store.require(record.taskId);
|
|
37
71
|
if (task.revision !== record.input.workflowRevision) throw new WorkflowError('DC_WORKFLOW_REVISION_CONFLICT', '说明已失效,请返回工作台重新生成;未执行查询。', 409);
|
|
38
|
-
|
|
72
|
+
if (record.kind !== 'retry') {
|
|
73
|
+
await store.startMatch(task.id, { expectedRevision: task.revision });
|
|
74
|
+
}
|
|
39
75
|
record.workflowStarted = true;
|
|
40
76
|
},
|
|
41
77
|
async failed(record, error) {
|
|
@@ -43,6 +79,22 @@ export function createWorkflowExecution({ getWorkflow, artifacts }) {
|
|
|
43
79
|
const store = await requireStore();
|
|
44
80
|
const task = await store.require(record.taskId);
|
|
45
81
|
if (['completed', 'cancelled', 'failed'].includes(task.state)) return;
|
|
82
|
+
if (record.kind === 'retry' && task.state === 'partial') {
|
|
83
|
+
await store.recordRetryFailure(task.id, {
|
|
84
|
+
expectedRevision: task.revision,
|
|
85
|
+
code: record.deliveryDraft ? 'DC_DELIVERY_FAILED' : error.code,
|
|
86
|
+
});
|
|
87
|
+
return;
|
|
88
|
+
}
|
|
89
|
+
if (record.deliveryDraft) {
|
|
90
|
+
await store.recordDeliveryFailure(task.id, {
|
|
91
|
+
expectedRevision: task.revision,
|
|
92
|
+
qccRunId: record.deliveryDraft.qccRunId,
|
|
93
|
+
matchSummary: record.deliveryDraft.matchSummary,
|
|
94
|
+
enrichmentSummary: record.deliveryDraft.enrichmentSummary,
|
|
95
|
+
});
|
|
96
|
+
return;
|
|
97
|
+
}
|
|
46
98
|
if (['QCC_AUTH_REQUIRED', 'QCC_NOT_CONNECTED', 'QCC_TOOL_UNAVAILABLE'].includes(error.code)) {
|
|
47
99
|
await store.requireAuthorization(task.id, { expectedRevision: task.revision });
|
|
48
100
|
} else {
|
|
@@ -53,22 +105,71 @@ export function createWorkflowExecution({ getWorkflow, artifacts }) {
|
|
|
53
105
|
if (!record.input.workflowOwned) return;
|
|
54
106
|
const store = await requireStore();
|
|
55
107
|
let task = await store.require(record.taskId);
|
|
56
|
-
const
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
108
|
+
const summaries = summariesFor(run);
|
|
109
|
+
if (summaries.reviewRequired) {
|
|
110
|
+
task = record.kind === 'retry'
|
|
111
|
+
? await store.recordRetryReview(task.id, {
|
|
112
|
+
expectedRevision: task.revision,
|
|
113
|
+
qccRunId: run.runId,
|
|
114
|
+
summary: summaries.matchSummary,
|
|
115
|
+
})
|
|
116
|
+
: await store.recordMatch(task.id, {
|
|
117
|
+
expectedRevision: task.revision,
|
|
118
|
+
qccRunId: run.runId,
|
|
119
|
+
summary: summaries.matchSummary,
|
|
120
|
+
});
|
|
121
|
+
return { state: task.state, artifacts: task.artifacts, taskId: task.id };
|
|
122
|
+
}
|
|
123
|
+
record.deliveryDraft = { ...summaries, qccRunId: run.runId };
|
|
67
124
|
const bundle = await artifacts.createBundle(task.id, { rows: run.rows, headers: task.source?.headers || run.headers,
|
|
68
125
|
mappings: task.mappings, fieldSelection: task.fieldSelection,
|
|
69
126
|
baseName: (task.source?.fileName || task.title).replace(/\.[^.]+$/, '') + '-清洗补全结果' });
|
|
70
|
-
task =
|
|
127
|
+
task = record.kind === 'retry'
|
|
128
|
+
? await store.completeRetryDelivery(task.id, {
|
|
129
|
+
expectedRevision: task.revision,
|
|
130
|
+
qccRunId: run.runId,
|
|
131
|
+
matchSummary: summaries.matchSummary,
|
|
132
|
+
enrichmentSummary: summaries.enrichmentSummary,
|
|
133
|
+
artifacts: bundle,
|
|
134
|
+
})
|
|
135
|
+
: await store.completeQccDelivery(task.id, {
|
|
136
|
+
expectedRevision: task.revision,
|
|
137
|
+
qccRunId: run.runId,
|
|
138
|
+
matchSummary: summaries.matchSummary,
|
|
139
|
+
enrichmentSummary: summaries.enrichmentSummary,
|
|
140
|
+
artifacts: bundle,
|
|
141
|
+
});
|
|
71
142
|
return { state: task.state, artifacts: task.artifacts, taskId: task.id };
|
|
72
143
|
},
|
|
144
|
+
async retryDelivery(payload) {
|
|
145
|
+
const store = await requireStore();
|
|
146
|
+
const task = await store.require(payload.taskId);
|
|
147
|
+
if (Number(payload.expectedRevision) !== task.revision) {
|
|
148
|
+
throw new WorkflowError('DC_WORKFLOW_REVISION_CONFLICT', '任务已更新,请刷新后重试交付。', 409);
|
|
149
|
+
}
|
|
150
|
+
if (!task.qccRunId || typeof getRun !== 'function') {
|
|
151
|
+
throw new WorkflowError('DC_DELIVERY_RUNTIME_EXPIRED', '企查查运行结果已失效,无法仅重试文件交付。', 409);
|
|
152
|
+
}
|
|
153
|
+
const run = getRun(task.qccRunId);
|
|
154
|
+
const summaries = summariesFor(run);
|
|
155
|
+
if (summaries.reviewRequired) {
|
|
156
|
+
throw new WorkflowError('DC_WORKFLOW_REVIEW_REQUIRED', '请先完成企业主体候选核验。', 409);
|
|
157
|
+
}
|
|
158
|
+
const bundle = await artifacts.createBundle(task.id, {
|
|
159
|
+
rows: run.rows,
|
|
160
|
+
headers: task.source?.headers || run.headers,
|
|
161
|
+
mappings: task.mappings,
|
|
162
|
+
fieldSelection: task.fieldSelection,
|
|
163
|
+
baseName: (task.source?.fileName || task.title).replace(/\.[^.]+$/, '') + '-清洗补全结果',
|
|
164
|
+
});
|
|
165
|
+
const completed = await store.completeRetryDelivery(task.id, {
|
|
166
|
+
expectedRevision: task.revision,
|
|
167
|
+
qccRunId: run.runId,
|
|
168
|
+
matchSummary: summaries.matchSummary,
|
|
169
|
+
enrichmentSummary: summaries.enrichmentSummary,
|
|
170
|
+
artifacts: bundle,
|
|
171
|
+
});
|
|
172
|
+
return { task: completed, artifacts: completed.artifacts };
|
|
173
|
+
},
|
|
73
174
|
};
|
|
74
175
|
}
|