dsh-data-cleaning-agent 0.5.3 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +66 -0
- package/README.en.md +17 -8
- package/README.md +15 -7
- package/docs/COMPATIBILITY.md +28 -7
- package/docs/G5-HOST-BRIDGE.md +34 -8
- package/docs/QCC-ENRICHMENT-DESIGN.md +4 -2
- package/docs/QCC-PHASES-ROADMAP.md +3 -2
- package/docs/RELEASE-0.5.3.md +10 -3
- package/docs/RELEASE-0.6.0.md +57 -0
- package/docs/RELEASE-0.6.1.md +50 -0
- package/docs/UI-WORKFLOW-V2-ACCEPTANCE.md +84 -0
- package/docs/UI-WORKFLOW-V2-MIGRATION.md +62 -0
- package/docs/UI-WORKFLOW-V2.md +173 -0
- package/docs/USER-GUIDE.md +11 -8
- package/lib/artifacts.js +239 -0
- package/lib/client.js +1298 -306
- package/lib/engine.js +1 -1
- package/lib/index.js +10 -5
- package/lib/jobs.js +1 -1
- package/lib/qcc-command.js +276 -0
- package/lib/qcc.js +39 -1
- package/lib/skill-enrich.js +7 -2
- package/lib/tools.js +1 -1
- package/lib/web.js +268 -1
- package/lib/workflow-contract.js +263 -0
- package/lib/workflow.js +452 -0
- package/package.json +7 -2
package/lib/web.js
CHANGED
|
@@ -10,8 +10,12 @@
|
|
|
10
10
|
import { parseCsv, parseXlsx, parseJson, detectFormat, toCsv } from './engine.js';
|
|
11
11
|
import { runSync, DataCleaningJobs } from './jobs.js';
|
|
12
12
|
import { QccBridgeError, QccHostBridge } from './qcc.js';
|
|
13
|
+
import { QccCommandStore, registerQccCommandTool, TOOL_QCC_COMMAND } from './qcc-command.js';
|
|
13
14
|
import { fingerprintRequest, G5RunStore } from './qcc-runs.js';
|
|
14
15
|
import { PHASE3_BATCH_LIMITS, Phase3BatchService, Phase3RunStore } from './qcc-phase3-batch.js';
|
|
16
|
+
import { publicWorkflowContract } from './workflow-contract.js';
|
|
17
|
+
import { DataCleaningWorkflowStore, WorkflowError } from './workflow.js';
|
|
18
|
+
import { ArtifactError, WorkflowArtifactStore } from './artifacts.js';
|
|
15
19
|
|
|
16
20
|
const MAX_BODY = 16 * 1024 * 1024; // 16 MiB 上传上限(MVP)
|
|
17
21
|
|
|
@@ -65,7 +69,7 @@ function requirePaidConfirmation(payload) {
|
|
|
65
69
|
}
|
|
66
70
|
|
|
67
71
|
function qccHttpStatus(code) {
|
|
68
|
-
if (code === 'QCC_RUN_NOT_FOUND') return 404;
|
|
72
|
+
if (code === 'QCC_RUN_NOT_FOUND' || code === 'QCC_COMMAND_NOT_FOUND') return 404;
|
|
69
73
|
if (code === 'QCC_AUTH_REQUIRED') return 401;
|
|
70
74
|
if (code === 'QCC_PERMISSION_DENIED') return 403;
|
|
71
75
|
if (code === 'QCC_QUOTA_EXHAUSTED') return 402;
|
|
@@ -76,6 +80,8 @@ function qccHttpStatus(code) {
|
|
|
76
80
|
|| code === 'QCC_TOOL_UNAVAILABLE'
|
|
77
81
|
|| code === 'QCC_UPSTREAM_UNAVAILABLE'
|
|
78
82
|
|| code === 'QCC_IDEMPOTENCY_CAPACITY'
|
|
83
|
+
|| code === 'QCC_COMMAND_CAPACITY'
|
|
84
|
+
|| code === 'QCC_AGENT_COMMAND_UNAVAILABLE'
|
|
79
85
|
) return 503;
|
|
80
86
|
if (
|
|
81
87
|
code === 'QCC_CONFIRM_REQUIRED'
|
|
@@ -86,6 +92,7 @@ function qccHttpStatus(code) {
|
|
|
86
92
|
|| code === 'QCC_OPERATION_IN_PROGRESS'
|
|
87
93
|
|| code === 'QCC_RETRY_NOT_FAILED'
|
|
88
94
|
|| code === 'QCC_RETRY_NOT_ALLOWED'
|
|
95
|
+
|| code === 'QCC_AGENT_EXECUTION_REQUIRED'
|
|
89
96
|
) return 409;
|
|
90
97
|
return 400;
|
|
91
98
|
}
|
|
@@ -98,6 +105,20 @@ function writeQccError(res, error) {
|
|
|
98
105
|
writeJson(res, qccHttpStatus(code), { ok: false, ...payload });
|
|
99
106
|
}
|
|
100
107
|
|
|
108
|
+
function writeWorkflowError(res, error) {
|
|
109
|
+
if (error instanceof ArtifactError) {
|
|
110
|
+
return writeJson(res, error.status, { ok: false, code: error.code, message: error.message });
|
|
111
|
+
}
|
|
112
|
+
if (error instanceof WorkflowError) {
|
|
113
|
+
return writeJson(res, error.status, { ok: false, code: error.code, message: error.message });
|
|
114
|
+
}
|
|
115
|
+
return writeJson(res, 500, {
|
|
116
|
+
ok: false,
|
|
117
|
+
code: 'DC_WORKFLOW_INTERNAL',
|
|
118
|
+
message: 'Workflow request failed.',
|
|
119
|
+
});
|
|
120
|
+
}
|
|
121
|
+
|
|
101
122
|
/** 从 JSON 协议解析上传:{ filename, content }。content 为字符串;xlsx 时为 base64。 */
|
|
102
123
|
async function parseUpload(body) {
|
|
103
124
|
let payload;
|
|
@@ -265,11 +286,21 @@ export function mountWebRoutes(wctx, { logger, report, TOOL_NAME, SKILL_NAME })
|
|
|
265
286
|
const disposers = [];
|
|
266
287
|
const qccBridge = new QccHostBridge({ tools, logger });
|
|
267
288
|
const g5Runs = new G5RunStore();
|
|
289
|
+
const qccCommands = new QccCommandStore({ bridge: qccBridge, runs: g5Runs });
|
|
290
|
+
if (typeof tools?.register === 'function') {
|
|
291
|
+
disposers.push(registerQccCommandTool(tools, qccCommands));
|
|
292
|
+
report.qccCommandToolRegistered = true;
|
|
293
|
+
} else {
|
|
294
|
+
report.qccCommandToolRegistered = false;
|
|
295
|
+
}
|
|
268
296
|
const phase3Service = new Phase3BatchService(qccBridge);
|
|
269
297
|
const phase3Runs = new Phase3RunStore();
|
|
270
298
|
report.qccBridgeMounted = true;
|
|
271
299
|
let state = null; // DataCleaningJobs,惰性初始化
|
|
272
300
|
let stateReady = null;
|
|
301
|
+
let workflow = null; // v2 工作流元数据,惰性初始化
|
|
302
|
+
let workflowReady = null;
|
|
303
|
+
const artifactStore = wctx.fs ? new WorkflowArtifactStore({ fs: wctx.fs }) : null;
|
|
273
304
|
|
|
274
305
|
const register = (path, handler) => {
|
|
275
306
|
disposers.push(server.register({ kind: 'prefix', path, handler }));
|
|
@@ -286,6 +317,15 @@ export function mountWebRoutes(wctx, { logger, report, TOOL_NAME, SKILL_NAME })
|
|
|
286
317
|
return null;
|
|
287
318
|
};
|
|
288
319
|
|
|
320
|
+
const getWorkflow = () => {
|
|
321
|
+
if (!wctx.storageDomain) return null;
|
|
322
|
+
if (!workflowReady) {
|
|
323
|
+
workflow = new DataCleaningWorkflowStore({ storageDomain: wctx.storageDomain, logger });
|
|
324
|
+
workflowReady = workflow.init();
|
|
325
|
+
}
|
|
326
|
+
return workflowReady;
|
|
327
|
+
};
|
|
328
|
+
|
|
289
329
|
register('/data-cleaning/', (req, res) => {
|
|
290
330
|
if (!isTrusted(req)) { res.writeHead(403); return res.end('untrusted origin'); }
|
|
291
331
|
res.writeHead(200, { 'content-type': 'text/html; charset=utf-8', 'cache-control': 'no-store' });
|
|
@@ -304,11 +344,193 @@ export function mountWebRoutes(wctx, { logger, report, TOOL_NAME, SKILL_NAME })
|
|
|
304
344
|
skillListed: null,
|
|
305
345
|
jobs: Boolean(wctx.jobs),
|
|
306
346
|
storageDomain: Boolean(wctx.storageDomain),
|
|
347
|
+
workflowV2: Boolean(wctx.storageDomain),
|
|
348
|
+
durableArtifacts: Boolean(artifactStore),
|
|
349
|
+
artifactBinaryStrategy: artifactStore ? 'xlsx-base64-over-writeText' : 'unavailable',
|
|
307
350
|
qccBridge: qccBridge.capabilities(),
|
|
308
351
|
},
|
|
309
352
|
});
|
|
310
353
|
});
|
|
311
354
|
|
|
355
|
+
register('/data-cleaning/api/workflow/contract', (req, res) => {
|
|
356
|
+
if (!isTrusted(req)) return writeJson(res, 403, { ok: false, error: 'untrusted origin' });
|
|
357
|
+
if (req.method !== 'GET') {
|
|
358
|
+
return writeJson(res, 405, { ok: false, code: 'DC_METHOD', message: 'GET required' });
|
|
359
|
+
}
|
|
360
|
+
writeJson(res, 200, {
|
|
361
|
+
ok: true,
|
|
362
|
+
marker: 'data-cleaning-workflow-v2',
|
|
363
|
+
contract: publicWorkflowContract(),
|
|
364
|
+
executesTools: false,
|
|
365
|
+
paidCalls: false,
|
|
366
|
+
});
|
|
367
|
+
});
|
|
368
|
+
|
|
369
|
+
register('/data-cleaning/api/workflow/tasks', async (req, res) => {
|
|
370
|
+
if (!isTrusted(req)) return writeJson(res, 403, { ok: false, error: 'untrusted origin' });
|
|
371
|
+
try {
|
|
372
|
+
const ready = getWorkflow();
|
|
373
|
+
if (!ready) {
|
|
374
|
+
return writeJson(res, 503, {
|
|
375
|
+
ok: false,
|
|
376
|
+
code: 'DC_WORKFLOW_UNAVAILABLE',
|
|
377
|
+
message: 'storageDomain unavailable in this composition',
|
|
378
|
+
});
|
|
379
|
+
}
|
|
380
|
+
await ready;
|
|
381
|
+
const pathname = new URL(req.url ?? '/data-cleaning/api/workflow/tasks', 'http://127.0.0.1').pathname;
|
|
382
|
+
const segments = pathname.split('/').filter(Boolean);
|
|
383
|
+
const tasksIndex = segments.indexOf('tasks');
|
|
384
|
+
const rest = tasksIndex >= 0 ? segments.slice(tasksIndex + 1) : [];
|
|
385
|
+
|
|
386
|
+
if (rest.length === 0) {
|
|
387
|
+
if (req.method === 'GET') {
|
|
388
|
+
return writeJson(res, 200, {
|
|
389
|
+
ok: true,
|
|
390
|
+
marker: 'data-cleaning-workflow-v2',
|
|
391
|
+
tasks: await workflow.list(),
|
|
392
|
+
});
|
|
393
|
+
}
|
|
394
|
+
if (req.method === 'POST') {
|
|
395
|
+
const body = await readBody(req);
|
|
396
|
+
const payload = body.length ? JSON.parse(body.toString('utf8')) : {};
|
|
397
|
+
return writeJson(res, 201, {
|
|
398
|
+
ok: true,
|
|
399
|
+
marker: 'data-cleaning-workflow-v2',
|
|
400
|
+
task: await workflow.create(payload),
|
|
401
|
+
});
|
|
402
|
+
}
|
|
403
|
+
return writeJson(res, 405, { ok: false, code: 'DC_METHOD', message: 'GET or POST required' });
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
const taskId = decodeURIComponent(rest[0]);
|
|
407
|
+
if (rest.length === 1) {
|
|
408
|
+
if (req.method === 'GET') {
|
|
409
|
+
return writeJson(res, 200, {
|
|
410
|
+
ok: true,
|
|
411
|
+
marker: 'data-cleaning-workflow-v2',
|
|
412
|
+
task: await workflow.require(taskId),
|
|
413
|
+
});
|
|
414
|
+
}
|
|
415
|
+
if (req.method === 'PATCH') {
|
|
416
|
+
const body = await readBody(req);
|
|
417
|
+
const payload = body.length ? JSON.parse(body.toString('utf8')) : {};
|
|
418
|
+
return writeJson(res, 200, {
|
|
419
|
+
ok: true,
|
|
420
|
+
marker: 'data-cleaning-workflow-v2',
|
|
421
|
+
task: await workflow.updateDraft(taskId, payload),
|
|
422
|
+
});
|
|
423
|
+
}
|
|
424
|
+
return writeJson(res, 405, { ok: false, code: 'DC_METHOD', message: 'GET or PATCH required' });
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
if (rest[1] === 'artifacts') {
|
|
428
|
+
if (!artifactStore) {
|
|
429
|
+
return writeJson(res, 503, {
|
|
430
|
+
ok: false,
|
|
431
|
+
code: 'DC_ARTIFACT_UNAVAILABLE',
|
|
432
|
+
message: 'DSH fs service unavailable in this composition.',
|
|
433
|
+
});
|
|
434
|
+
}
|
|
435
|
+
const current = await workflow.require(taskId);
|
|
436
|
+
if (rest.length === 2 && req.method === 'GET') {
|
|
437
|
+
return writeJson(res, 200, {
|
|
438
|
+
ok: true,
|
|
439
|
+
marker: 'data-cleaning-artifacts-v1',
|
|
440
|
+
task: current,
|
|
441
|
+
artifacts: current.artifacts,
|
|
442
|
+
});
|
|
443
|
+
}
|
|
444
|
+
if (rest.length === 2 && req.method === 'POST') {
|
|
445
|
+
const body = await readBody(req);
|
|
446
|
+
const payload = body.length ? JSON.parse(body.toString('utf8')) : {};
|
|
447
|
+
if (Number(payload.expectedRevision) !== current.revision) {
|
|
448
|
+
throw new WorkflowError('DC_WORKFLOW_REVISION_CONFLICT', 'Workflow task was updated by another session.', 409);
|
|
449
|
+
}
|
|
450
|
+
let readyTask = current;
|
|
451
|
+
if (['rules_confirmed', 'diagnosed'].includes(readyTask.state)) {
|
|
452
|
+
readyTask = await workflow.prepareLocalExport(taskId, {
|
|
453
|
+
expectedRevision: readyTask.revision,
|
|
454
|
+
summary: payload.summary,
|
|
455
|
+
});
|
|
456
|
+
}
|
|
457
|
+
if (!['export_ready', 'partial'].includes(readyTask.state)) {
|
|
458
|
+
throw new WorkflowError('DC_WORKFLOW_EXPORT_STATE', 'Output is not ready for durable export.', 409);
|
|
459
|
+
}
|
|
460
|
+
const artifacts = await artifactStore.createBundle(taskId, {
|
|
461
|
+
rows: payload.rows,
|
|
462
|
+
headers: payload.headers,
|
|
463
|
+
exceptionRows: payload.exceptionRows,
|
|
464
|
+
baseName: payload.baseName || readyTask.title,
|
|
465
|
+
});
|
|
466
|
+
const completed = await workflow.recordExport(taskId, {
|
|
467
|
+
expectedRevision: readyTask.revision,
|
|
468
|
+
artifacts,
|
|
469
|
+
});
|
|
470
|
+
return writeJson(res, 201, {
|
|
471
|
+
ok: true,
|
|
472
|
+
marker: 'data-cleaning-artifacts-v1',
|
|
473
|
+
task: completed,
|
|
474
|
+
artifacts: completed.artifacts,
|
|
475
|
+
});
|
|
476
|
+
}
|
|
477
|
+
if (rest.length === 3 && req.method === 'GET') {
|
|
478
|
+
const artifactId = decodeURIComponent(rest[2]);
|
|
479
|
+
const artifact = current.artifacts.find((item) => item.id === artifactId);
|
|
480
|
+
if (!artifact) throw new ArtifactError('DC_ARTIFACT_NOT_FOUND', 'Artifact not found for this task.', 404);
|
|
481
|
+
const bytes = await artifactStore.read(taskId, artifact);
|
|
482
|
+
const fallback = artifact.format === 'xlsx' ? 'export.xlsx' : 'export.csv';
|
|
483
|
+
const asciiName = String(artifact.fileName || fallback).replace(/[^\x20-\x7e]/g, '_').replace(/["\\]/g, '_');
|
|
484
|
+
res.writeHead(200, {
|
|
485
|
+
'content-type': artifact.mediaType || 'application/octet-stream',
|
|
486
|
+
'content-length': String(bytes.length),
|
|
487
|
+
'content-disposition': `attachment; filename="${asciiName}"; filename*=UTF-8''${encodeURIComponent(artifact.fileName || fallback)}`,
|
|
488
|
+
'cache-control': 'no-store',
|
|
489
|
+
'x-content-type-options': 'nosniff',
|
|
490
|
+
'referrer-policy': 'no-referrer',
|
|
491
|
+
});
|
|
492
|
+
return res.end(bytes);
|
|
493
|
+
}
|
|
494
|
+
return writeJson(res, 405, { ok: false, code: 'DC_METHOD', message: 'Artifact route requires GET or POST.' });
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
if (rest.length !== 3 || rest[1] !== 'actions' || req.method !== 'POST') {
|
|
498
|
+
return writeJson(res, 404, { ok: false, code: 'DC_WORKFLOW_ROUTE', message: 'Workflow route not found.' });
|
|
499
|
+
}
|
|
500
|
+
const body = await readBody(req);
|
|
501
|
+
const payload = body.length ? JSON.parse(body.toString('utf8')) : {};
|
|
502
|
+
const action = rest[2];
|
|
503
|
+
const actions = {
|
|
504
|
+
upload: () => workflow.recordUpload(taskId, payload),
|
|
505
|
+
rules: () => workflow.confirmRules(taskId, payload),
|
|
506
|
+
quality: () => workflow.recordQuality(taskId, payload),
|
|
507
|
+
'match-start': () => workflow.startMatch(taskId, payload),
|
|
508
|
+
match: () => workflow.recordMatch(taskId, payload),
|
|
509
|
+
'enrich-start': () => workflow.startEnrichment(taskId, payload),
|
|
510
|
+
enrichment: () => workflow.recordEnrichment(taskId, payload),
|
|
511
|
+
'local-export-ready': () => workflow.prepareLocalExport(taskId, payload),
|
|
512
|
+
export: () => workflow.recordExport(taskId, payload),
|
|
513
|
+
'parse-failed': () => workflow.recordParseFailure(taskId, payload),
|
|
514
|
+
'authorization-required': () => workflow.requireAuthorization(taskId, payload),
|
|
515
|
+
fail: () => workflow.recordFailure(taskId, payload),
|
|
516
|
+
cancel: () => workflow.cancel(taskId, payload),
|
|
517
|
+
};
|
|
518
|
+
if (!actions[action]) {
|
|
519
|
+
return writeJson(res, 404, { ok: false, code: 'DC_WORKFLOW_ACTION', message: `Unknown workflow action: ${action}` });
|
|
520
|
+
}
|
|
521
|
+
return writeJson(res, 200, {
|
|
522
|
+
ok: true,
|
|
523
|
+
marker: 'data-cleaning-workflow-v2',
|
|
524
|
+
task: await actions[action](),
|
|
525
|
+
});
|
|
526
|
+
} catch (error) {
|
|
527
|
+
if (error instanceof SyntaxError) {
|
|
528
|
+
return writeJson(res, 400, { ok: false, code: 'DC_BAD_JSON', message: 'Request body must be valid JSON.' });
|
|
529
|
+
}
|
|
530
|
+
return writeWorkflowError(res, error);
|
|
531
|
+
}
|
|
532
|
+
});
|
|
533
|
+
|
|
312
534
|
register('/data-cleaning/api/g5/capabilities', (req, res) => {
|
|
313
535
|
if (!isTrusted(req)) return writeJson(res, 403, { ok: false, error: 'untrusted origin' });
|
|
314
536
|
writeJson(res, 200, {
|
|
@@ -320,10 +542,52 @@ export function mountWebRoutes(wctx, { logger, report, TOOL_NAME, SKILL_NAME })
|
|
|
320
542
|
idempotencyRequired: true,
|
|
321
543
|
candidateResume: true,
|
|
322
544
|
manualRetry: true,
|
|
545
|
+
agentCommandTool: TOOL_QCC_COMMAND,
|
|
546
|
+
agentCommandToolRegistered: report.qccCommandToolRegistered,
|
|
547
|
+
agentOwnedExecutionRequired: true,
|
|
323
548
|
runPersistence: 'host-memory',
|
|
324
549
|
});
|
|
325
550
|
});
|
|
326
551
|
|
|
552
|
+
register('/data-cleaning/api/g5/commands', async (req, res) => {
|
|
553
|
+
if (!isTrusted(req)) return writeJson(res, 403, { ok: false, error: 'untrusted origin' });
|
|
554
|
+
try {
|
|
555
|
+
const pathname = new URL(req.url ?? '/data-cleaning/api/g5/commands', 'http://127.0.0.1').pathname;
|
|
556
|
+
const segments = pathname.split('/').filter(Boolean);
|
|
557
|
+
const commandsIndex = segments.indexOf('commands');
|
|
558
|
+
const rest = commandsIndex >= 0 ? segments.slice(commandsIndex + 1) : [];
|
|
559
|
+
if (rest.length === 0 && req.method === 'POST') {
|
|
560
|
+
if (!report.qccCommandToolRegistered) {
|
|
561
|
+
throw new QccBridgeError(
|
|
562
|
+
'QCC_AGENT_COMMAND_UNAVAILABLE',
|
|
563
|
+
'The current DSH Host does not expose the Agent command registration API.',
|
|
564
|
+
);
|
|
565
|
+
}
|
|
566
|
+
const payload = JSON.parse((await readBody(req)).toString('utf8'));
|
|
567
|
+
requirePaidConfirmation(payload);
|
|
568
|
+
return writeJson(res, 201, {
|
|
569
|
+
ok: true,
|
|
570
|
+
marker: 'g5-agent-command',
|
|
571
|
+
paidCalls: false,
|
|
572
|
+
command: qccCommands.prepare(payload),
|
|
573
|
+
});
|
|
574
|
+
}
|
|
575
|
+
if (rest.length === 1 && req.method === 'GET') {
|
|
576
|
+
return writeJson(res, 200, {
|
|
577
|
+
ok: true,
|
|
578
|
+
marker: 'g5-agent-command',
|
|
579
|
+
command: qccCommands.status(decodeURIComponent(rest[0])),
|
|
580
|
+
});
|
|
581
|
+
}
|
|
582
|
+
return writeJson(res, 405, { ok: false, code: 'DC_METHOD', message: 'POST a command or GET one command id' });
|
|
583
|
+
} catch (error) {
|
|
584
|
+
if (error instanceof SyntaxError) {
|
|
585
|
+
return writeJson(res, 400, { ok: false, code: 'DC_BAD_JSON', message: 'Request body must be valid JSON.' });
|
|
586
|
+
}
|
|
587
|
+
return writeQccError(res, error);
|
|
588
|
+
}
|
|
589
|
+
});
|
|
590
|
+
|
|
327
591
|
register('/data-cleaning/api/phase2/capabilities', (req, res) => {
|
|
328
592
|
if (!isTrusted(req)) return writeJson(res, 403, { ok: false, error: 'untrusted origin' });
|
|
329
593
|
if (req.method !== 'GET') {
|
|
@@ -633,6 +897,8 @@ export function mountWebRoutes(wctx, { logger, report, TOOL_NAME, SKILL_NAME })
|
|
|
633
897
|
kind,
|
|
634
898
|
summary: result.summary,
|
|
635
899
|
rowCount: result.rows.length,
|
|
900
|
+
headers: headers.length ? headers : result.rows[0] ? Object.keys(result.rows[0]) : [],
|
|
901
|
+
rows: result.rows,
|
|
636
902
|
csv,
|
|
637
903
|
downloadName: kind === 'clean' ? 'cleaned.csv' : kind === 'complete' ? 'completed.csv' : null,
|
|
638
904
|
});
|
|
@@ -687,6 +953,7 @@ export function mountWebRoutes(wctx, { logger, report, TOOL_NAME, SKILL_NAME })
|
|
|
687
953
|
|
|
688
954
|
return () => {
|
|
689
955
|
if (state) { state.dispose().catch(() => {}); }
|
|
956
|
+
if (workflow) { workflow.dispose().catch(() => {}); }
|
|
690
957
|
for (const dispose of disposers) dispose();
|
|
691
958
|
};
|
|
692
959
|
}
|
|
@@ -0,0 +1,263 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 数据清洗补全智能体 v2 工作流契约。
|
|
3
|
+
*
|
|
4
|
+
* 设计原则:
|
|
5
|
+
* - 业务主流程固定为五步;质量体检与任务历史是横向能力。
|
|
6
|
+
* - 匹配结果只表达状态与可审计依据,不生成无法验证的“置信度”。
|
|
7
|
+
* - 历史、人员、招投标三域不属于当前版本字段目录。
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
export const WORKFLOW_SCHEMA_VERSION = 2;
|
|
11
|
+
|
|
12
|
+
export const WORKFLOW_STAGES = Object.freeze([
|
|
13
|
+
Object.freeze({ id: 'upload', label: '上传数据', order: 1 }),
|
|
14
|
+
Object.freeze({ id: 'rules', label: '规则确认', order: 2 }),
|
|
15
|
+
Object.freeze({ id: 'match', label: '数据匹配', order: 3 }),
|
|
16
|
+
Object.freeze({ id: 'enrich', label: '清洗补全', order: 4 }),
|
|
17
|
+
Object.freeze({ id: 'download', label: '下载数据', order: 5 }),
|
|
18
|
+
]);
|
|
19
|
+
|
|
20
|
+
export const WORKFLOW_STATES = Object.freeze([
|
|
21
|
+
'draft',
|
|
22
|
+
'uploaded',
|
|
23
|
+
'rules_confirmed',
|
|
24
|
+
'diagnosed',
|
|
25
|
+
'matching',
|
|
26
|
+
'review_required',
|
|
27
|
+
'matched',
|
|
28
|
+
'enriching',
|
|
29
|
+
'export_ready',
|
|
30
|
+
'completed',
|
|
31
|
+
'parse_failed',
|
|
32
|
+
'authorization_required',
|
|
33
|
+
'partial',
|
|
34
|
+
'failed',
|
|
35
|
+
'cancelled',
|
|
36
|
+
]);
|
|
37
|
+
|
|
38
|
+
export const TERMINAL_WORKFLOW_STATES = Object.freeze(['completed', 'cancelled']);
|
|
39
|
+
|
|
40
|
+
export const SOURCE_TYPES = Object.freeze(['text', 'csv', 'xlsx', 'json', 'image']);
|
|
41
|
+
|
|
42
|
+
export const MATCH_STATUSES = Object.freeze([
|
|
43
|
+
'exact',
|
|
44
|
+
'candidate',
|
|
45
|
+
'confirmed',
|
|
46
|
+
'unresolved',
|
|
47
|
+
'failed',
|
|
48
|
+
]);
|
|
49
|
+
|
|
50
|
+
export const MATCH_ANCHORS = Object.freeze(['company_name', 'credit_no', 'reg_no']);
|
|
51
|
+
|
|
52
|
+
export const FIELD_CATALOG = Object.freeze([
|
|
53
|
+
Object.freeze({
|
|
54
|
+
id: 'identity',
|
|
55
|
+
label: '基础工商信息',
|
|
56
|
+
fields: Object.freeze([
|
|
57
|
+
Object.freeze({ id: 'company_name', label: '企业名称', inputAnchor: true, defaultSelected: true }),
|
|
58
|
+
Object.freeze({ id: 'credit_no', label: '统一社会信用代码', inputAnchor: true, defaultSelected: true }),
|
|
59
|
+
Object.freeze({ id: 'reg_no', label: '注册号', inputAnchor: true }),
|
|
60
|
+
Object.freeze({ id: 'org_no', label: '组织机构代码' }),
|
|
61
|
+
Object.freeze({ id: 'reg_status', label: '登记状态', defaultSelected: true }),
|
|
62
|
+
Object.freeze({ id: 'legal_rep', label: '法定代表人', defaultSelected: true }),
|
|
63
|
+
Object.freeze({ id: 'reg_capital', label: '注册资本', defaultSelected: true }),
|
|
64
|
+
Object.freeze({ id: 'paid_capital', label: '实缴资本' }),
|
|
65
|
+
Object.freeze({ id: 'establish_date', label: '成立日期', defaultSelected: true }),
|
|
66
|
+
Object.freeze({ id: 'company_type', label: '企业类型' }),
|
|
67
|
+
Object.freeze({ id: 'registration_authority', label: '登记机关' }),
|
|
68
|
+
Object.freeze({ id: 'former_name', label: '曾用名' }),
|
|
69
|
+
Object.freeze({ id: 'english_name', label: '英文名' }),
|
|
70
|
+
]),
|
|
71
|
+
}),
|
|
72
|
+
Object.freeze({
|
|
73
|
+
id: 'contact',
|
|
74
|
+
label: '地址与联系方式',
|
|
75
|
+
fields: Object.freeze([
|
|
76
|
+
Object.freeze({ id: 'registered_address', label: '注册地址', defaultSelected: true }),
|
|
77
|
+
Object.freeze({ id: 'province', label: '省份地区', matchAuxiliary: true }),
|
|
78
|
+
Object.freeze({ id: 'city', label: '城市', matchAuxiliary: true }),
|
|
79
|
+
Object.freeze({ id: 'district', label: '区县' }),
|
|
80
|
+
Object.freeze({ id: 'phone', label: '电话', matchAuxiliary: true }),
|
|
81
|
+
Object.freeze({ id: 'email', label: '邮箱' }),
|
|
82
|
+
Object.freeze({ id: 'website', label: '官网' }),
|
|
83
|
+
]),
|
|
84
|
+
}),
|
|
85
|
+
Object.freeze({
|
|
86
|
+
id: 'operation',
|
|
87
|
+
label: '经营信息',
|
|
88
|
+
fields: Object.freeze([
|
|
89
|
+
Object.freeze({ id: 'business_scope', label: '经营范围' }),
|
|
90
|
+
Object.freeze({ id: 'industry_category', label: '国标行业' }),
|
|
91
|
+
Object.freeze({ id: 'industry_large', label: '一级行业' }),
|
|
92
|
+
Object.freeze({ id: 'industry_middle', label: '二级行业' }),
|
|
93
|
+
Object.freeze({ id: 'operating_period', label: '营业期限' }),
|
|
94
|
+
Object.freeze({ id: 'company_size', label: '企业规模' }),
|
|
95
|
+
Object.freeze({ id: 'company_profile', label: '企业简介' }),
|
|
96
|
+
]),
|
|
97
|
+
}),
|
|
98
|
+
Object.freeze({
|
|
99
|
+
id: 'risk',
|
|
100
|
+
label: '风险摘要',
|
|
101
|
+
capability: 'qcc.risk',
|
|
102
|
+
fields: Object.freeze([
|
|
103
|
+
Object.freeze({ id: 'risk_summary', label: '风险摘要', capability: 'qcc.risk' }),
|
|
104
|
+
Object.freeze({ id: 'operating_exception', label: '经营异常摘要', capability: 'qcc.risk' }),
|
|
105
|
+
Object.freeze({ id: 'administrative_penalty', label: '行政处罚摘要', capability: 'qcc.risk' }),
|
|
106
|
+
]),
|
|
107
|
+
}),
|
|
108
|
+
Object.freeze({
|
|
109
|
+
id: 'ipr',
|
|
110
|
+
label: '知识产权摘要',
|
|
111
|
+
capability: 'qcc.ipr',
|
|
112
|
+
fields: Object.freeze([
|
|
113
|
+
Object.freeze({ id: 'trademark_summary', label: '商标摘要', capability: 'qcc.ipr' }),
|
|
114
|
+
Object.freeze({ id: 'patent_summary', label: '专利摘要', capability: 'qcc.ipr' }),
|
|
115
|
+
Object.freeze({ id: 'software_copyright_summary', label: '软件著作权摘要', capability: 'qcc.ipr' }),
|
|
116
|
+
]),
|
|
117
|
+
}),
|
|
118
|
+
]);
|
|
119
|
+
|
|
120
|
+
const FIELD_IDS = new Set(FIELD_CATALOG.flatMap((group) => group.fields.map((field) => field.id)));
|
|
121
|
+
const STAGE_IDS = new Set(WORKFLOW_STAGES.map((stage) => stage.id));
|
|
122
|
+
const STATE_IDS = new Set(WORKFLOW_STATES);
|
|
123
|
+
|
|
124
|
+
const TRANSITIONS = Object.freeze({
|
|
125
|
+
draft: ['uploaded', 'parse_failed', 'cancelled'],
|
|
126
|
+
uploaded: ['draft', 'rules_confirmed', 'parse_failed', 'cancelled'],
|
|
127
|
+
rules_confirmed: ['diagnosed', 'matching', 'matched', 'review_required', 'export_ready', 'authorization_required', 'failed', 'cancelled'],
|
|
128
|
+
diagnosed: ['matching', 'matched', 'review_required', 'export_ready', 'authorization_required', 'failed', 'cancelled'],
|
|
129
|
+
matching: ['matched', 'review_required', 'authorization_required', 'partial', 'failed', 'cancelled'],
|
|
130
|
+
review_required: ['matching', 'matched', 'authorization_required', 'partial', 'failed', 'cancelled'],
|
|
131
|
+
matched: ['enriching', 'export_ready', 'authorization_required', 'partial', 'failed', 'cancelled'],
|
|
132
|
+
enriching: ['export_ready', 'review_required', 'authorization_required', 'partial', 'failed', 'cancelled'],
|
|
133
|
+
export_ready: ['enriching', 'completed', 'failed', 'cancelled'],
|
|
134
|
+
parse_failed: ['draft', 'uploaded', 'cancelled'],
|
|
135
|
+
authorization_required: ['matching', 'enriching', 'failed', 'cancelled'],
|
|
136
|
+
partial: ['review_required', 'enriching', 'export_ready', 'authorization_required', 'completed', 'failed', 'cancelled'],
|
|
137
|
+
failed: ['draft', 'uploaded', 'rules_confirmed', 'diagnosed', 'matching', 'enriching', 'cancelled'],
|
|
138
|
+
completed: [],
|
|
139
|
+
cancelled: [],
|
|
140
|
+
});
|
|
141
|
+
|
|
142
|
+
export class WorkflowContractError extends Error {
|
|
143
|
+
constructor(code, message) {
|
|
144
|
+
super(message);
|
|
145
|
+
this.name = 'WorkflowContractError';
|
|
146
|
+
this.code = code;
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
function text(value, max = 160) {
|
|
151
|
+
return String(value ?? '').trim().slice(0, max);
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
function uniqueStrings(values, max = 64) {
|
|
155
|
+
if (!Array.isArray(values)) return [];
|
|
156
|
+
return [...new Set(values.map((value) => text(value)).filter(Boolean))].slice(0, max);
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
export function normalizeMappings(value) {
|
|
160
|
+
if (!Array.isArray(value)) return [];
|
|
161
|
+
return value.slice(0, 128).map((mapping) => ({
|
|
162
|
+
sourceField: text(mapping?.sourceField),
|
|
163
|
+
targetField: text(mapping?.targetField),
|
|
164
|
+
})).filter((mapping) => mapping.sourceField && FIELD_IDS.has(mapping.targetField));
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
export function validateMappings(value) {
|
|
168
|
+
if (!Array.isArray(value) || value.length > 128) {
|
|
169
|
+
throw new WorkflowContractError('DC_WORKFLOW_MAPPING_INVALID', 'Field mappings must be an array with at most 128 entries.');
|
|
170
|
+
}
|
|
171
|
+
const mappings = normalizeMappings(value);
|
|
172
|
+
if (mappings.length !== value.length) {
|
|
173
|
+
throw new WorkflowContractError('DC_WORKFLOW_MAPPING_INVALID', 'Every mapping needs a source field and a supported target field.');
|
|
174
|
+
}
|
|
175
|
+
if (mappings.length === 0) {
|
|
176
|
+
throw new WorkflowContractError('DC_WORKFLOW_MAPPING_REQUIRED', 'At least one valid field mapping is required.');
|
|
177
|
+
}
|
|
178
|
+
if (!mappings.some((mapping) => MATCH_ANCHORS.includes(mapping.targetField))) {
|
|
179
|
+
throw new WorkflowContractError(
|
|
180
|
+
'DC_WORKFLOW_ANCHOR_REQUIRED',
|
|
181
|
+
'Map at least one enterprise identity anchor: company name, unified social credit code, or registration number.',
|
|
182
|
+
);
|
|
183
|
+
}
|
|
184
|
+
const targets = new Set();
|
|
185
|
+
const sources = new Set();
|
|
186
|
+
for (const mapping of mappings) {
|
|
187
|
+
if (targets.has(mapping.targetField)) {
|
|
188
|
+
throw new WorkflowContractError('DC_WORKFLOW_DUPLICATE_MAPPING', `Duplicate target mapping: ${mapping.targetField}`);
|
|
189
|
+
}
|
|
190
|
+
if (sources.has(mapping.sourceField)) {
|
|
191
|
+
throw new WorkflowContractError('DC_WORKFLOW_DUPLICATE_MAPPING', `Duplicate source mapping: ${mapping.sourceField}`);
|
|
192
|
+
}
|
|
193
|
+
targets.add(mapping.targetField);
|
|
194
|
+
sources.add(mapping.sourceField);
|
|
195
|
+
}
|
|
196
|
+
return mappings;
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
export function normalizeFieldSelection(value) {
|
|
200
|
+
return uniqueStrings(value).filter((field) => FIELD_IDS.has(field));
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
export function normalizeWorkflowDraft(value = {}) {
|
|
204
|
+
const objectives = uniqueStrings(value.objectives, 16).filter((item) => (
|
|
205
|
+
['clean_name', 'deduplicate', 'validate_identity', 'complete_fields'].includes(item)
|
|
206
|
+
));
|
|
207
|
+
return {
|
|
208
|
+
title: text(value.title, 120) || '未命名数据清洗补全任务',
|
|
209
|
+
objectives,
|
|
210
|
+
fieldSelection: normalizeFieldSelection(value.fieldSelection),
|
|
211
|
+
mappings: normalizeMappings(value.mappings),
|
|
212
|
+
matchRules: {
|
|
213
|
+
normalizeNames: value.matchRules?.normalizeNames !== false,
|
|
214
|
+
preferCreditNo: value.matchRules?.preferCreditNo !== false,
|
|
215
|
+
deduplicate: value.matchRules?.deduplicate !== false,
|
|
216
|
+
manualReviewAmbiguous: value.matchRules?.manualReviewAmbiguous !== false,
|
|
217
|
+
},
|
|
218
|
+
};
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
export function canTransition(from, to) {
|
|
222
|
+
if (!STATE_IDS.has(from) || !STATE_IDS.has(to)) return false;
|
|
223
|
+
return TRANSITIONS[from].includes(to);
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
export function assertWorkflowTransition(from, to) {
|
|
227
|
+
if (!canTransition(from, to)) {
|
|
228
|
+
throw new WorkflowContractError('DC_WORKFLOW_TRANSITION', `Invalid workflow transition: ${from} -> ${to}`);
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
export function publicWorkflowContract() {
|
|
233
|
+
return {
|
|
234
|
+
schemaVersion: WORKFLOW_SCHEMA_VERSION,
|
|
235
|
+
stages: WORKFLOW_STAGES,
|
|
236
|
+
states: WORKFLOW_STATES,
|
|
237
|
+
terminalStates: TERMINAL_WORKFLOW_STATES,
|
|
238
|
+
sourceTypes: SOURCE_TYPES,
|
|
239
|
+
matchStatuses: MATCH_STATUSES,
|
|
240
|
+
matchAnchors: MATCH_ANCHORS,
|
|
241
|
+
fieldCatalog: FIELD_CATALOG,
|
|
242
|
+
crossCuttingCapabilities: [
|
|
243
|
+
{ id: 'prompt', label: '任务设置' },
|
|
244
|
+
{ id: 'profile', label: '质量体检' },
|
|
245
|
+
{ id: 'history', label: '任务历史' },
|
|
246
|
+
],
|
|
247
|
+
privacy: {
|
|
248
|
+
persisted: ['task metadata', 'numeric summaries', 'artifact references'],
|
|
249
|
+
notPersisted: ['raw rows and enterprise lists', 'QCC response payloads', 'candidate details'],
|
|
250
|
+
},
|
|
251
|
+
deferredDomains: ['history', 'person', 'tender'],
|
|
252
|
+
};
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
export function assertWorkflowRecordShape(record) {
|
|
256
|
+
if (!record || record.schemaVersion !== WORKFLOW_SCHEMA_VERSION) {
|
|
257
|
+
throw new WorkflowContractError('DC_WORKFLOW_SCHEMA', 'Unsupported workflow record schema.');
|
|
258
|
+
}
|
|
259
|
+
if (!STAGE_IDS.has(record.stage) || !STATE_IDS.has(record.state)) {
|
|
260
|
+
throw new WorkflowContractError('DC_WORKFLOW_SCHEMA', 'Workflow record contains an invalid stage or state.');
|
|
261
|
+
}
|
|
262
|
+
return record;
|
|
263
|
+
}
|