dsh-data-cleaning-agent 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +71 -0
- package/README.en.md +48 -7
- package/README.md +44 -6
- package/docs/COMPATIBILITY.md +54 -2
- package/docs/G5-E2E-RUNBOOK.md +110 -0
- package/docs/G5-HOST-BRIDGE.md +136 -0
- package/docs/PHASE2-ACCEPTANCE.md +133 -0
- package/docs/PHASE3-ACCEPTANCE.md +129 -0
- package/docs/QCC-ENRICHMENT-DESIGN.md +5 -2
- package/docs/QCC-PHASES-ROADMAP.md +33 -6
- package/docs/RELEASE-0.4.0.md +66 -0
- package/docs/RELEASE-0.5.0.md +84 -0
- package/docs/USER-GUIDE.md +85 -6
- package/lib/client.js +1132 -16
- package/lib/index.js +2 -0
- package/lib/qcc-phase2-acceptance.js +191 -0
- package/lib/qcc-phase2.js +99 -0
- package/lib/qcc-phase3-batch.js +437 -0
- package/lib/qcc-phase3.js +224 -0
- package/lib/qcc-runs.js +322 -0
- package/lib/qcc-safety.js +77 -0
- package/lib/qcc.js +735 -0
- package/lib/skill-enrich.js +54 -9
- package/lib/web.js +351 -2
- package/package.json +23 -4
package/lib/skill-enrich.js
CHANGED
|
@@ -7,36 +7,81 @@
|
|
|
7
7
|
* (依赖 `qcc-dsh-mcp-oauth` 已连接;本 Skill 不重造 OAuth)。
|
|
8
8
|
*
|
|
9
9
|
* 方案 A(模型中介式):模型亲自调用 `mcp__qcc-company__*` / `mcp__qcc-risk__*`
|
|
10
|
-
* 完成消歧 → 工商详情 →
|
|
10
|
+
* / `mcp__qcc-ipr__*` / `mcp__qcc-operation__*` 完成消歧 → 工商详情 → 各域维度,
|
|
11
|
+
* 再组装结果。本插件零后端改动(Host Bridge 批量是独立于本 Skill 的方案 B)。
|
|
11
12
|
*/
|
|
13
|
+
import {
|
|
14
|
+
QCC_PHASE2_COMPANY_TOOLS,
|
|
15
|
+
QCC_PHASE2_DIMENSION_GROUPS,
|
|
16
|
+
QCC_PHASE2_HISTORY_TOOLS,
|
|
17
|
+
} from './qcc-phase2.js';
|
|
18
|
+
import {
|
|
19
|
+
QCC_PHASE3_DOMAIN_META,
|
|
20
|
+
QCC_PHASE3_TOOL_NAMES,
|
|
21
|
+
canonicalPhase3ToolName,
|
|
22
|
+
} from './qcc-phase3.js';
|
|
23
|
+
|
|
12
24
|
export const ENRICH_SKILL_NAME = 'enterprise-enrichment';
|
|
13
25
|
|
|
26
|
+
const companyTools = QCC_PHASE2_COMPANY_TOOLS;
|
|
27
|
+
const historyTools = QCC_PHASE2_HISTORY_TOOLS;
|
|
28
|
+
|
|
29
|
+
const groupSummary = Object.entries(QCC_PHASE2_DIMENSION_GROUPS)
|
|
30
|
+
.map(([id, group]) => ` - \`${id}\`(${group.label}): ${group.tools.join(', ')}`)
|
|
31
|
+
.join('\n');
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* 0.5.0 三大域(风险/知产/经营)的用户可选域组说明。
|
|
35
|
+
* 每一域均为 basic 授权 + 按次计费 + 调用前必须确认;工具名一律用规范名。
|
|
36
|
+
*/
|
|
37
|
+
const phase3GroupSummary = Object.keys(QCC_PHASE3_TOOL_NAMES)
|
|
38
|
+
.map((domain) => {
|
|
39
|
+
const meta = QCC_PHASE3_DOMAIN_META[domain];
|
|
40
|
+
const tools = QCC_PHASE3_TOOL_NAMES[domain].map((name) => canonicalPhase3ToolName(domain, name));
|
|
41
|
+
return ` - \`${domain}\`(${meta.label} · basic · 按次计费 · 调用前须确认): ${tools.join(', ')}`;
|
|
42
|
+
})
|
|
43
|
+
.join('\n');
|
|
44
|
+
|
|
14
45
|
export function registerEnrichSkill(skills) {
|
|
15
46
|
return skills.register({
|
|
16
47
|
name: ENRICH_SKILL_NAME,
|
|
17
48
|
description:
|
|
18
|
-
'Enrich
|
|
49
|
+
'Enrich company lists with Qichacha (QCC) registration, company panorama, ownership, governance, risk, intellectual-property, operations, and optional historical-business dimensions.',
|
|
19
50
|
whenToUse:
|
|
20
|
-
'When the user gives
|
|
51
|
+
'When the user gives company names and asks to enrich / complete them with Qichacha (企查查), including registration fields, company panorama, ownership penetration, governance, historical changes, risk tags, intellectual property, or operational / bidding dimensions.',
|
|
21
52
|
source: 'dsh-data-cleaning-agent',
|
|
22
53
|
content: [
|
|
23
|
-
'You are an enterprise-list enrichment assistant. You fill
|
|
54
|
+
'You are an enterprise-list enrichment assistant. You fill company lists with Qichacha (QCC) data by calling QCC MCP tools. Never invent, pad, or fabricate any field.',
|
|
24
55
|
'',
|
|
25
56
|
'Workflow:',
|
|
26
57
|
'1. Check QCC availability first: run `qcc_oauth_status`. If not connected, tell the user to run `qcc_oauth_connect` first and stop. If the token is expired, guide the user to `qcc_oauth_connect` (it reuses the grant and refreshes without a new authorization page).',
|
|
27
58
|
'2. Parse the company-name list from what the user gave (pasted text / CSV / JSON / inline list). Keep only the distinct company-name column.',
|
|
28
|
-
|
|
59
|
+
`3. For EACH name: run \`${companyTools.resolveEntity}\`.`,
|
|
29
60
|
' - Unique exact match → lock that entity and keep its credit code.',
|
|
30
61
|
' - Multiple candidates → DO NOT auto-pick the first. List the candidates (name + region + credit code) and ask the user which one to use.',
|
|
31
62
|
' - No match → mark that row as `unresolved` and continue.',
|
|
32
|
-
|
|
33
|
-
'5.
|
|
34
|
-
|
|
63
|
+
`4. For each locked entity, always run \`${companyTools.registration}\`. Run \`${companyTools.verifyIdentity}\` when the input includes a credit code or the user asks for identity verification.`,
|
|
64
|
+
'5. Determine requested dimension groups from the user request. If it is not explicit, ask the user to choose among the 0.4.0 groups (`panorama`, `ownership`, `governance`, `history`) and the 0.5.0 domains (`risk`, `ipr`, `operation`); do not invoke every tool by default. The verified 0.4.0 group contract is:',
|
|
65
|
+
groupSummary,
|
|
66
|
+
'6. Call only the tools required by the selected groups. Process large lists in explicit batches, announce the next batch before paid calls, preserve input row order, and never drop a row silently.',
|
|
67
|
+
`7. The \`history\` group requires an enterprise-certified account. Only call ${Object.values(historyTools).map((tool) => `\`${tool}\``).join(', ')} when the user requested history and the account is eligible. If a history tool is unavailable or returns a permission error, mark the group \`permission_required\` or \`not_available\`, continue current-data groups, and never replace history with guessed values.`,
|
|
68
|
+
'8. The 0.5.0 domains are risk / intellectual-property / operations. Each is basic access, per-call paid, and requires explicit user confirmation BEFORE any call; do not call them implicitly and do not call any tool before the user confirms the domain and the batch. The verified 0.5.0 domain contract is:',
|
|
69
|
+
phase3GroupSummary,
|
|
70
|
+
'9. Within a confirmed 0.5.0 domain, call only that domain\'s tools, and only the ones that answer the requested dimensions. For risk, prefer `mcp__qcc-risk__get_company_risk_scan` (35-factor hit counts) as the entry scan and `mcp__qcc-risk__get_company_related_risk_scan` for related-party risk; `mcp__qcc-risk__get_judicial_document_detail` additionally requires `documentId`, so call it only with a document ID returned by `mcp__qcc-risk__get_judicial_documents`. For intellectual property, prefer `mcp__qcc-ipr__get_patent_info`, `mcp__qcc-ipr__get_trademark_info`, `mcp__qcc-ipr__get_software_copyright_info` as entry points. For operations, prefer `mcp__qcc-operation__get_bidding_info` (bid role), `mcp__qcc-operation__get_financing_records`, `mcp__qcc-operation__get_qualifications` as entry points. Use hit dimensions + returned counts only; do not derive risk conclusions or ratings beyond what the tool returned.',
|
|
71
|
+
'10. Degrade explicitly on the three failure classes and never fabricate a fallback:',
|
|
72
|
+
' - permission / authorization → mark `permission_required` and continue other confirmed domains;',
|
|
73
|
+
' - no data / empty result → mark `not_available`; an absent field never means "none" or zero;',
|
|
74
|
+
' - rate limit / quota / throttling → mark `rate_limited` and STOP that domain for this batch; do not retry in a loop or switch to an unconfirmed domain.',
|
|
75
|
+
'11. Assemble each requested dimension with `value`, `status`, and `sourceTool`. Missing values are `unresolved`; an absent field never means "none" or zero.',
|
|
76
|
+
'12. Report enriched / unresolved / ambiguous / permission-required / rate-limited counts and a small requested preview. Do not paste a full sensitive list into chat. Use a same-origin Host download or artifact when that capability is available; otherwise say that no downloadable artifact was created instead of pretending one exists.',
|
|
35
77
|
'',
|
|
36
78
|
'Safety rules:',
|
|
37
79
|
'- Never fabricate a credit code, legal representative, capital, amount, ratio, or status.',
|
|
38
80
|
'- Never auto-select among ambiguous candidates — always confirm with the user.',
|
|
39
|
-
'- Quote amounts / ratios / counts exactly as the QCC tool returned them; never recompute or estimate.',
|
|
81
|
+
'- Quote amounts / ratios / counts exactly as the QCC tool returned them; never recompute, multiply ownership chains, aggregate, or estimate.',
|
|
82
|
+
'- Preserve provenance: every populated dimension must identify the QCC source tool that returned it (`sourceTool`).',
|
|
83
|
+
'- Do not continue a paid batch after cancellation, authorization failure, or an unresolved ambiguity that affects entity identity.',
|
|
84
|
+
'- Make zero QCC calls before the user confirms the domain and batch; idempotent replay must not re-bill.',
|
|
40
85
|
'- Never expose QCC tokens or credentials.',
|
|
41
86
|
].join('\n'),
|
|
42
87
|
});
|
package/lib/web.js
CHANGED
|
@@ -9,6 +9,9 @@
|
|
|
9
9
|
*/
|
|
10
10
|
import { parseCsv, parseXlsx, parseJson, detectFormat, toCsv } from './engine.js';
|
|
11
11
|
import { runSync, DataCleaningJobs } from './jobs.js';
|
|
12
|
+
import { QccBridgeError, QccHostBridge } from './qcc.js';
|
|
13
|
+
import { fingerprintRequest, G5RunStore } from './qcc-runs.js';
|
|
14
|
+
import { PHASE3_BATCH_LIMITS, Phase3BatchService, Phase3RunStore } from './qcc-phase3-batch.js';
|
|
12
15
|
|
|
13
16
|
const MAX_BODY = 16 * 1024 * 1024; // 16 MiB 上传上限(MVP)
|
|
14
17
|
|
|
@@ -52,6 +55,49 @@ async function readBody(req, max = MAX_BODY) {
|
|
|
52
55
|
return Buffer.concat(chunks);
|
|
53
56
|
}
|
|
54
57
|
|
|
58
|
+
function requirePaidConfirmation(payload) {
|
|
59
|
+
if (payload?.confirmPaidCalls !== true) {
|
|
60
|
+
throw new QccBridgeError(
|
|
61
|
+
'QCC_CONFIRM_REQUIRED',
|
|
62
|
+
'Set confirmPaidCalls=true after the current user confirms use of their own QCC account quota or billing.',
|
|
63
|
+
);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
function qccHttpStatus(code) {
|
|
68
|
+
if (code === 'QCC_RUN_NOT_FOUND') return 404;
|
|
69
|
+
if (code === 'QCC_AUTH_REQUIRED') return 401;
|
|
70
|
+
if (code === 'QCC_PERMISSION_DENIED') return 403;
|
|
71
|
+
if (code === 'QCC_QUOTA_EXHAUSTED') return 402;
|
|
72
|
+
if (code === 'QCC_RATE_LIMITED') return 429;
|
|
73
|
+
if (code === 'QCC_TIMEOUT') return 504;
|
|
74
|
+
if (
|
|
75
|
+
code === 'QCC_NOT_CONNECTED'
|
|
76
|
+
|| code === 'QCC_TOOL_UNAVAILABLE'
|
|
77
|
+
|| code === 'QCC_UPSTREAM_UNAVAILABLE'
|
|
78
|
+
|| code === 'QCC_IDEMPOTENCY_CAPACITY'
|
|
79
|
+
) return 503;
|
|
80
|
+
if (
|
|
81
|
+
code === 'QCC_CONFIRM_REQUIRED'
|
|
82
|
+
|| code === 'QCC_ABORTED'
|
|
83
|
+
|| code === 'QCC_IDEMPOTENCY_CONFLICT'
|
|
84
|
+
|| code === 'QCC_REVIEW_NOT_PENDING'
|
|
85
|
+
|| code === 'QCC_CANDIDATE_INVALID'
|
|
86
|
+
|| code === 'QCC_OPERATION_IN_PROGRESS'
|
|
87
|
+
|| code === 'QCC_RETRY_NOT_FAILED'
|
|
88
|
+
|| code === 'QCC_RETRY_NOT_ALLOWED'
|
|
89
|
+
) return 409;
|
|
90
|
+
return 400;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function writeQccError(res, error) {
|
|
94
|
+
const code = error?.code ?? 'QCC_BRIDGE';
|
|
95
|
+
const payload = error instanceof QccBridgeError
|
|
96
|
+
? error.toJSON()
|
|
97
|
+
: { code, message: 'G5 Host Bridge request failed' };
|
|
98
|
+
writeJson(res, qccHttpStatus(code), { ok: false, ...payload });
|
|
99
|
+
}
|
|
100
|
+
|
|
55
101
|
/** 从 JSON 协议解析上传:{ filename, content }。content 为字符串;xlsx 时为 base64。 */
|
|
56
102
|
async function parseUpload(body) {
|
|
57
103
|
let payload;
|
|
@@ -147,7 +193,12 @@ async function withRows() {
|
|
|
147
193
|
if (!p.ok) throw new Error(JSON.stringify(p));
|
|
148
194
|
return { rows: p.rows, headers: p.headers };
|
|
149
195
|
}
|
|
150
|
-
|
|
196
|
+
const content = $('src').value.trim();
|
|
197
|
+
if (!content) return { rows: [], headers: null };
|
|
198
|
+
if (content.startsWith('[')) return { rows: JSON.parse(content), headers: null };
|
|
199
|
+
const p = await call('/data-cleaning/api/mvp/parse', { filename: 'data.csv', content });
|
|
200
|
+
if (!p.ok) throw new Error(JSON.stringify(p));
|
|
201
|
+
return { rows: p.rows, headers: p.headers };
|
|
151
202
|
}
|
|
152
203
|
|
|
153
204
|
$('parse').onclick = async () => {
|
|
@@ -212,6 +263,11 @@ export function mountWebRoutes(wctx, { logger, report, TOOL_NAME, SKILL_NAME })
|
|
|
212
263
|
const tools = wctx.tools;
|
|
213
264
|
const skills = wctx.skills;
|
|
214
265
|
const disposers = [];
|
|
266
|
+
const qccBridge = new QccHostBridge({ tools, logger });
|
|
267
|
+
const g5Runs = new G5RunStore();
|
|
268
|
+
const phase3Service = new Phase3BatchService(qccBridge);
|
|
269
|
+
const phase3Runs = new Phase3RunStore();
|
|
270
|
+
report.qccBridgeMounted = true;
|
|
215
271
|
let state = null; // DataCleaningJobs,惰性初始化
|
|
216
272
|
let stateReady = null;
|
|
217
273
|
|
|
@@ -248,10 +304,288 @@ export function mountWebRoutes(wctx, { logger, report, TOOL_NAME, SKILL_NAME })
|
|
|
248
304
|
skillListed: null,
|
|
249
305
|
jobs: Boolean(wctx.jobs),
|
|
250
306
|
storageDomain: Boolean(wctx.storageDomain),
|
|
307
|
+
qccBridge: qccBridge.capabilities(),
|
|
251
308
|
},
|
|
252
309
|
});
|
|
253
310
|
});
|
|
254
311
|
|
|
312
|
+
register('/data-cleaning/api/g5/capabilities', (req, res) => {
|
|
313
|
+
if (!isTrusted(req)) return writeJson(res, 403, { ok: false, error: 'untrusted origin' });
|
|
314
|
+
writeJson(res, 200, {
|
|
315
|
+
ok: true,
|
|
316
|
+
marker: 'g5-host-bridge',
|
|
317
|
+
capabilities: qccBridge.capabilities(),
|
|
318
|
+
limits: { maxRows: 100, maxConcurrency: 4 },
|
|
319
|
+
paidCallConfirmationRequired: true,
|
|
320
|
+
idempotencyRequired: true,
|
|
321
|
+
candidateResume: true,
|
|
322
|
+
manualRetry: true,
|
|
323
|
+
runPersistence: 'host-memory',
|
|
324
|
+
});
|
|
325
|
+
});
|
|
326
|
+
|
|
327
|
+
register('/data-cleaning/api/phase2/capabilities', (req, res) => {
|
|
328
|
+
if (!isTrusted(req)) return writeJson(res, 403, { ok: false, error: 'untrusted origin' });
|
|
329
|
+
if (req.method !== 'GET') {
|
|
330
|
+
return writeJson(res, 405, { ok: false, code: 'DC_METHOD', message: 'GET required' });
|
|
331
|
+
}
|
|
332
|
+
writeJson(res, 200, {
|
|
333
|
+
ok: true,
|
|
334
|
+
marker: 'qcc-phase2-capabilities',
|
|
335
|
+
capabilities: qccBridge.phase2Capabilities(),
|
|
336
|
+
executesTools: false,
|
|
337
|
+
paidCalls: false,
|
|
338
|
+
historyAuthorizationRequiresRealCall: true,
|
|
339
|
+
});
|
|
340
|
+
});
|
|
341
|
+
|
|
342
|
+
register('/data-cleaning/api/phase3/capabilities', (req, res) => {
|
|
343
|
+
if (!isTrusted(req)) return writeJson(res, 403, { ok: false, error: 'untrusted origin' });
|
|
344
|
+
if (req.method !== 'GET') {
|
|
345
|
+
return writeJson(res, 405, { ok: false, code: 'DC_METHOD', message: 'GET required' });
|
|
346
|
+
}
|
|
347
|
+
writeJson(res, 200, {
|
|
348
|
+
ok: true,
|
|
349
|
+
marker: 'qcc-phase3-capabilities',
|
|
350
|
+
capabilities: qccBridge.phase3Capabilities(),
|
|
351
|
+
limits: PHASE3_BATCH_LIMITS,
|
|
352
|
+
executesTools: false,
|
|
353
|
+
paidCalls: false,
|
|
354
|
+
estimateEndpoint: '/data-cleaning/api/phase3/estimate',
|
|
355
|
+
paidCallConfirmationRequired: true,
|
|
356
|
+
idempotencyRequired: true,
|
|
357
|
+
candidateResume: true,
|
|
358
|
+
manualRetry: true,
|
|
359
|
+
runPersistence: 'host-memory',
|
|
360
|
+
});
|
|
361
|
+
});
|
|
362
|
+
|
|
363
|
+
const phase3Payload = (run, replayed = false) => {
|
|
364
|
+
const headers = [...new Set([...run.headers, ...run.rows.flatMap((row) => Object.keys(row))])];
|
|
365
|
+
const reviewRows = run.reviewQueue.flatMap((item) => item.candidates.map((candidate) => ({
|
|
366
|
+
input_company: item.companyName,
|
|
367
|
+
candidate_company: candidate.companyName,
|
|
368
|
+
credit_no: candidate.creditNo,
|
|
369
|
+
legal_rep: Array.isArray(candidate.legalRep) ? candidate.legalRep.join(';') : '',
|
|
370
|
+
status: candidate.status,
|
|
371
|
+
})));
|
|
372
|
+
return {
|
|
373
|
+
ok: true,
|
|
374
|
+
marker: 'qcc-phase3-batch',
|
|
375
|
+
...run,
|
|
376
|
+
idempotencyReplayed: replayed,
|
|
377
|
+
rowCount: run.rows.length,
|
|
378
|
+
csv: toCsv(headers, run.rows),
|
|
379
|
+
reviewCsv: toCsv(['input_company', 'candidate_company', 'credit_no', 'legal_rep', 'status'], reviewRows),
|
|
380
|
+
downloadName: 'qcc-phase3-enriched.csv',
|
|
381
|
+
reviewDownloadName: 'qcc-phase3-review.csv',
|
|
382
|
+
};
|
|
383
|
+
};
|
|
384
|
+
|
|
385
|
+
const phase3Options = (payload) => ({
|
|
386
|
+
nameField: String(payload?.nameField ?? 'name'),
|
|
387
|
+
domains: Array.isArray(payload?.domains) ? payload.domains.map(String) : [],
|
|
388
|
+
tools: Array.isArray(payload?.tools) ? payload.tools.map(String) : [],
|
|
389
|
+
toolArguments: payload?.toolArguments && typeof payload.toolArguments === 'object' ? payload.toolArguments : {},
|
|
390
|
+
concurrency: Number(payload?.concurrency ?? 2),
|
|
391
|
+
maxRows: Number(payload?.maxRows ?? PHASE3_BATCH_LIMITS.maxRows),
|
|
392
|
+
maxCalls: Number(payload?.maxCalls ?? PHASE3_BATCH_LIMITS.defaultMaxCalls),
|
|
393
|
+
});
|
|
394
|
+
|
|
395
|
+
register('/data-cleaning/api/phase3/estimate', async (req, res) => {
|
|
396
|
+
if (!isTrusted(req)) return writeJson(res, 403, { ok: false, error: 'untrusted origin' });
|
|
397
|
+
if (req.method !== 'POST') return writeJson(res, 405, { ok: false, code: 'DC_METHOD', message: 'POST required' });
|
|
398
|
+
try {
|
|
399
|
+
const payload = JSON.parse((await readBody(req)).toString('utf8'));
|
|
400
|
+
const rows = Array.isArray(payload?.rows) ? payload.rows : [];
|
|
401
|
+
writeJson(res, 200, { ok: true, marker: 'qcc-phase3-estimate', estimate: phase3Service.estimate(rows, phase3Options(payload)) });
|
|
402
|
+
} catch (error) {
|
|
403
|
+
writeQccError(res, error);
|
|
404
|
+
}
|
|
405
|
+
});
|
|
406
|
+
|
|
407
|
+
register('/data-cleaning/api/phase3/enrich', async (req, res) => {
|
|
408
|
+
if (!isTrusted(req)) return writeJson(res, 403, { ok: false, error: 'untrusted origin' });
|
|
409
|
+
if (req.method !== 'POST') return writeJson(res, 405, { ok: false, code: 'DC_METHOD', message: 'POST required' });
|
|
410
|
+
try {
|
|
411
|
+
const payload = JSON.parse((await readBody(req)).toString('utf8'));
|
|
412
|
+
requirePaidConfirmation(payload);
|
|
413
|
+
const rows = Array.isArray(payload?.rows) ? payload.rows : [];
|
|
414
|
+
const headers = Array.isArray(payload?.headers) ? payload.headers.map(String) : [];
|
|
415
|
+
const options = phase3Options(payload);
|
|
416
|
+
const fingerprintInput = { rows, headers, ...options };
|
|
417
|
+
const executed = await g5Runs.executeOnce({
|
|
418
|
+
key: payload?.idempotencyKey,
|
|
419
|
+
fingerprint: fingerprintRequest('phase3-enrich', fingerprintInput),
|
|
420
|
+
operation: async () => {
|
|
421
|
+
const result = await phase3Service.run(rows, options);
|
|
422
|
+
return phase3Runs.create({ headers, nameField: options.nameField, input: options, result });
|
|
423
|
+
},
|
|
424
|
+
});
|
|
425
|
+
writeJson(res, 200, phase3Payload(executed.value, executed.replayed));
|
|
426
|
+
} catch (error) {
|
|
427
|
+
writeQccError(res, error);
|
|
428
|
+
}
|
|
429
|
+
});
|
|
430
|
+
|
|
431
|
+
register('/data-cleaning/api/phase3/resolve', async (req, res) => {
|
|
432
|
+
if (!isTrusted(req)) return writeJson(res, 403, { ok: false, error: 'untrusted origin' });
|
|
433
|
+
if (req.method !== 'POST') return writeJson(res, 405, { ok: false, code: 'DC_METHOD', message: 'POST required' });
|
|
434
|
+
try {
|
|
435
|
+
const payload = JSON.parse((await readBody(req)).toString('utf8'));
|
|
436
|
+
requirePaidConfirmation(payload);
|
|
437
|
+
const input = {
|
|
438
|
+
runId: String(payload?.runId ?? ''),
|
|
439
|
+
companyName: String(payload?.companyName ?? ''),
|
|
440
|
+
selectedCreditNo: String(payload?.selectedCreditNo ?? ''),
|
|
441
|
+
};
|
|
442
|
+
const executed = await g5Runs.executeOnce({
|
|
443
|
+
key: payload?.idempotencyKey,
|
|
444
|
+
fingerprint: fingerprintRequest('phase3-resolve', input),
|
|
445
|
+
operation: () => phase3Runs.resolve(input.runId, input, phase3Service),
|
|
446
|
+
});
|
|
447
|
+
writeJson(res, 200, phase3Payload(executed.value, executed.replayed));
|
|
448
|
+
} catch (error) {
|
|
449
|
+
writeQccError(res, error);
|
|
450
|
+
}
|
|
451
|
+
});
|
|
452
|
+
|
|
453
|
+
register('/data-cleaning/api/phase3/retry', async (req, res) => {
|
|
454
|
+
if (!isTrusted(req)) return writeJson(res, 403, { ok: false, error: 'untrusted origin' });
|
|
455
|
+
if (req.method !== 'POST') return writeJson(res, 405, { ok: false, code: 'DC_METHOD', message: 'POST required' });
|
|
456
|
+
try {
|
|
457
|
+
const payload = JSON.parse((await readBody(req)).toString('utf8'));
|
|
458
|
+
requirePaidConfirmation(payload);
|
|
459
|
+
const input = {
|
|
460
|
+
runId: String(payload?.runId ?? ''),
|
|
461
|
+
companyNames: Array.isArray(payload?.companyNames) ? payload.companyNames.map(String) : [],
|
|
462
|
+
};
|
|
463
|
+
const executed = await g5Runs.executeOnce({
|
|
464
|
+
key: payload?.idempotencyKey,
|
|
465
|
+
fingerprint: fingerprintRequest('phase3-retry', input),
|
|
466
|
+
operation: () => phase3Runs.retry(input.runId, input.companyNames, phase3Service),
|
|
467
|
+
});
|
|
468
|
+
writeJson(res, 200, phase3Payload(executed.value, executed.replayed));
|
|
469
|
+
} catch (error) {
|
|
470
|
+
writeQccError(res, error);
|
|
471
|
+
}
|
|
472
|
+
});
|
|
473
|
+
|
|
474
|
+
register('/data-cleaning/api/phase3/run', (req, res) => {
|
|
475
|
+
if (!isTrusted(req)) return writeJson(res, 403, { ok: false, error: 'untrusted origin' });
|
|
476
|
+
if (req.method !== 'GET') return writeJson(res, 405, { ok: false, code: 'DC_METHOD', message: 'GET required' });
|
|
477
|
+
try {
|
|
478
|
+
const runId = String((req.url ?? '').split('/').filter(Boolean).pop() ?? '');
|
|
479
|
+
writeJson(res, 200, phase3Payload(phase3Runs.get(runId)));
|
|
480
|
+
} catch (error) {
|
|
481
|
+
writeQccError(res, error);
|
|
482
|
+
}
|
|
483
|
+
});
|
|
484
|
+
|
|
485
|
+
const runPayload = (run, replayed = false) => {
|
|
486
|
+
const headers = [...new Set([
|
|
487
|
+
...run.headers,
|
|
488
|
+
...run.rows.flatMap((row) => Object.keys(row)),
|
|
489
|
+
])];
|
|
490
|
+
return {
|
|
491
|
+
ok: true,
|
|
492
|
+
marker: 'g5-host-bridge',
|
|
493
|
+
...run,
|
|
494
|
+
idempotencyReplayed: replayed,
|
|
495
|
+
rowCount: run.rows.length,
|
|
496
|
+
csv: toCsv(headers, run.rows),
|
|
497
|
+
downloadName: 'qcc-enriched.csv',
|
|
498
|
+
};
|
|
499
|
+
};
|
|
500
|
+
|
|
501
|
+
register('/data-cleaning/api/g5/enrich', async (req, res) => {
|
|
502
|
+
if (!isTrusted(req)) return writeJson(res, 403, { ok: false, error: 'untrusted origin' });
|
|
503
|
+
if (req.method !== 'POST') return writeJson(res, 405, { ok: false, code: 'DC_METHOD', message: 'POST required' });
|
|
504
|
+
try {
|
|
505
|
+
const body = await readBody(req);
|
|
506
|
+
const payload = JSON.parse(body.toString('utf8'));
|
|
507
|
+
requirePaidConfirmation(payload);
|
|
508
|
+
const rows = Array.isArray(payload?.rows) ? payload.rows : [];
|
|
509
|
+
const headers = Array.isArray(payload?.headers) ? payload.headers.map(String) : [];
|
|
510
|
+
const nameField = String(payload?.nameField ?? 'name');
|
|
511
|
+
const includeRisk = payload?.includeRisk === true;
|
|
512
|
+
const concurrency = Number(payload?.concurrency ?? 2);
|
|
513
|
+
const input = { rows, headers, nameField, includeRisk, concurrency };
|
|
514
|
+
const executed = await g5Runs.executeOnce({
|
|
515
|
+
key: payload?.idempotencyKey,
|
|
516
|
+
fingerprint: fingerprintRequest('enrich', input),
|
|
517
|
+
operation: async () => {
|
|
518
|
+
const audit = [];
|
|
519
|
+
const result = await qccBridge.enrichRows(rows, {
|
|
520
|
+
nameField,
|
|
521
|
+
includeRisk,
|
|
522
|
+
concurrency,
|
|
523
|
+
maxRows: 100,
|
|
524
|
+
onAudit: (event) => audit.push(event),
|
|
525
|
+
});
|
|
526
|
+
return g5Runs.createRun({ headers, nameField, includeRisk, concurrency, result, audit });
|
|
527
|
+
},
|
|
528
|
+
});
|
|
529
|
+
writeJson(res, 200, runPayload(executed.value, executed.replayed));
|
|
530
|
+
} catch (error) {
|
|
531
|
+
writeQccError(res, error);
|
|
532
|
+
}
|
|
533
|
+
});
|
|
534
|
+
|
|
535
|
+
register('/data-cleaning/api/g5/resolve', async (req, res) => {
|
|
536
|
+
if (!isTrusted(req)) return writeJson(res, 403, { ok: false, error: 'untrusted origin' });
|
|
537
|
+
if (req.method !== 'POST') return writeJson(res, 405, { ok: false, code: 'DC_METHOD', message: 'POST required' });
|
|
538
|
+
try {
|
|
539
|
+
const payload = JSON.parse((await readBody(req)).toString('utf8'));
|
|
540
|
+
requirePaidConfirmation(payload);
|
|
541
|
+
const input = {
|
|
542
|
+
runId: String(payload?.runId ?? ''),
|
|
543
|
+
companyName: String(payload?.companyName ?? ''),
|
|
544
|
+
selectedCreditNo: String(payload?.selectedCreditNo ?? ''),
|
|
545
|
+
};
|
|
546
|
+
const executed = await g5Runs.executeOnce({
|
|
547
|
+
key: payload?.idempotencyKey,
|
|
548
|
+
fingerprint: fingerprintRequest('resolve', input),
|
|
549
|
+
operation: () => g5Runs.resolveCandidate(input.runId, input, qccBridge),
|
|
550
|
+
});
|
|
551
|
+
writeJson(res, 200, runPayload(executed.value, executed.replayed));
|
|
552
|
+
} catch (error) {
|
|
553
|
+
writeQccError(res, error);
|
|
554
|
+
}
|
|
555
|
+
});
|
|
556
|
+
|
|
557
|
+
register('/data-cleaning/api/g5/retry', async (req, res) => {
|
|
558
|
+
if (!isTrusted(req)) return writeJson(res, 403, { ok: false, error: 'untrusted origin' });
|
|
559
|
+
if (req.method !== 'POST') return writeJson(res, 405, { ok: false, code: 'DC_METHOD', message: 'POST required' });
|
|
560
|
+
try {
|
|
561
|
+
const payload = JSON.parse((await readBody(req)).toString('utf8'));
|
|
562
|
+
requirePaidConfirmation(payload);
|
|
563
|
+
const input = {
|
|
564
|
+
runId: String(payload?.runId ?? ''),
|
|
565
|
+
companyNames: Array.isArray(payload?.companyNames) ? payload.companyNames.map(String) : [],
|
|
566
|
+
};
|
|
567
|
+
const executed = await g5Runs.executeOnce({
|
|
568
|
+
key: payload?.idempotencyKey,
|
|
569
|
+
fingerprint: fingerprintRequest('retry', input),
|
|
570
|
+
operation: () => g5Runs.retryCompanies(input.runId, input.companyNames, qccBridge),
|
|
571
|
+
});
|
|
572
|
+
writeJson(res, 200, runPayload(executed.value, executed.replayed));
|
|
573
|
+
} catch (error) {
|
|
574
|
+
writeQccError(res, error);
|
|
575
|
+
}
|
|
576
|
+
});
|
|
577
|
+
|
|
578
|
+
register('/data-cleaning/api/g5/run', (req, res) => {
|
|
579
|
+
if (!isTrusted(req)) return writeJson(res, 403, { ok: false, error: 'untrusted origin' });
|
|
580
|
+
if (req.method !== 'GET') return writeJson(res, 405, { ok: false, code: 'DC_METHOD', message: 'GET required' });
|
|
581
|
+
try {
|
|
582
|
+
const runId = String((req.url ?? '').split('/').filter(Boolean).pop() ?? '');
|
|
583
|
+
writeJson(res, 200, runPayload(g5Runs.get(runId)));
|
|
584
|
+
} catch (error) {
|
|
585
|
+
writeQccError(res, error);
|
|
586
|
+
}
|
|
587
|
+
});
|
|
588
|
+
|
|
255
589
|
register('/data-cleaning/api/mvp/parse', async (req, res) => {
|
|
256
590
|
if (!isTrusted(req)) return writeJson(res, 403, { ok: false, error: 'untrusted origin' });
|
|
257
591
|
try {
|
|
@@ -277,7 +611,22 @@ export function mountWebRoutes(wctx, { logger, report, TOOL_NAME, SKILL_NAME })
|
|
|
277
611
|
const payload = JSON.parse(body.toString('utf8'));
|
|
278
612
|
const rows = Array.isArray(payload?.rows) ? payload.rows : [];
|
|
279
613
|
const headers = Array.isArray(payload?.headers) ? payload.headers : [];
|
|
280
|
-
const
|
|
614
|
+
const requested = payload?.options && typeof payload.options === 'object'
|
|
615
|
+
? payload.options
|
|
616
|
+
: {};
|
|
617
|
+
const options = { headers };
|
|
618
|
+
if (Array.isArray(requested.required)) {
|
|
619
|
+
options.required = requested.required.map(String).filter(Boolean).slice(0, 32);
|
|
620
|
+
}
|
|
621
|
+
for (const key of ['amountField', 'dedupeOn', 'phoneField']) {
|
|
622
|
+
if (Object.hasOwn(requested, key)) {
|
|
623
|
+
options[key] = requested[key] === null ? null : String(requested[key]);
|
|
624
|
+
}
|
|
625
|
+
}
|
|
626
|
+
if (Object.hasOwn(requested, 'fillableName')) {
|
|
627
|
+
options.fillableName = requested.fillableName === true;
|
|
628
|
+
}
|
|
629
|
+
const result = runSync(kind, rows, options);
|
|
281
630
|
const csv = result.rows.length ? toCsv(headers.length ? headers : result.rows[0] ? Object.keys(result.rows[0]) : [], result.rows) : '';
|
|
282
631
|
writeJson(res, 200, {
|
|
283
632
|
ok: true,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dsh-data-cleaning-agent",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.5.0",
|
|
4
4
|
"description": "Clean, complete, and profile enterprise name lists in DeepSeek Harness — a data cleaning & completion agent plugin with local CSV/XLSX/JSON engine and optional Qichacha (QCC) MCP enrichment. Maintained by Qichacha/QCC.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "lib/index.js",
|
|
@@ -23,14 +23,24 @@
|
|
|
23
23
|
"docs/FIRST-CONTRIBUTION.md",
|
|
24
24
|
"docs/COMPATIBILITY.md",
|
|
25
25
|
"docs/QCC-ENRICHMENT-DESIGN.md",
|
|
26
|
-
"docs/QCC-PHASES-ROADMAP.md"
|
|
26
|
+
"docs/QCC-PHASES-ROADMAP.md",
|
|
27
|
+
"docs/PHASE2-ACCEPTANCE.md",
|
|
28
|
+
"docs/PHASE3-ACCEPTANCE.md",
|
|
29
|
+
"docs/RELEASE-0.4.0.md",
|
|
30
|
+
"docs/RELEASE-0.5.0.md",
|
|
31
|
+
"docs/G5-HOST-BRIDGE.md",
|
|
32
|
+
"docs/G5-E2E-RUNBOOK.md"
|
|
27
33
|
],
|
|
28
34
|
"scripts": {
|
|
29
35
|
"test": "node --test",
|
|
30
|
-
"lint": "node --check lib/index.js && node --check lib/engine.js && node --check lib/tools.js && node --check lib/skill.js && node --check lib/skill-enrich.js && node --check lib/jobs.js && node --check lib/web.js && node --check lib/client.js",
|
|
36
|
+
"lint": "node --check lib/index.js && node --check lib/engine.js && node --check lib/tools.js && node --check lib/skill.js && node --check lib/qcc-phase2.js && node --check lib/qcc-phase3.js && node --check lib/qcc-phase3-batch.js && node --check lib/qcc-phase2-acceptance.js && node --check lib/skill-enrich.js && node --check lib/jobs.js && node --check lib/qcc-safety.js && node --check lib/qcc.js && node --check lib/qcc-runs.js && node --check lib/web.js && node --check lib/client.js && node --check scripts/check-market-registration.mjs && node --check scripts/g5-e2e.mjs && node --check scripts/phase3-e2e.mjs && node --check scripts/phase2-acceptance.mjs",
|
|
31
37
|
"docs:check": "node scripts/check-readme-version.mjs",
|
|
32
38
|
"marketing:check": "node scripts/check-marketing.mjs",
|
|
33
39
|
"verify-pack": "node scripts/verify-pack.mjs",
|
|
40
|
+
"market:check": "node scripts/check-market-registration.mjs",
|
|
41
|
+
"e2e:g5": "node scripts/g5-e2e.mjs",
|
|
42
|
+
"e2e:phase3": "node scripts/phase3-e2e.mjs",
|
|
43
|
+
"e2e:phase2": "node scripts/phase2-acceptance.mjs",
|
|
34
44
|
"check": "npm run lint && npm run docs:check && npm run marketing:check && npm run verify-pack && npm test",
|
|
35
45
|
"prepublishOnly": "npm run check"
|
|
36
46
|
},
|
|
@@ -77,12 +87,21 @@
|
|
|
77
87
|
"dependencies": {
|
|
78
88
|
"xlsx": "^0.18.5"
|
|
79
89
|
},
|
|
90
|
+
"peerDependencies": {
|
|
91
|
+
"@deepseek-ai/dsh-client-ui-conversation": "*",
|
|
92
|
+
"@deepseek-ai/dsh-client-ui-layout": "*",
|
|
93
|
+
"react": "^18.2.0",
|
|
94
|
+
"react-dom": "^18.2.0"
|
|
95
|
+
},
|
|
80
96
|
"dsh": {
|
|
81
97
|
"bundle": {
|
|
82
98
|
"patch": "./cordis.patch.yml"
|
|
83
99
|
},
|
|
84
100
|
"client": {
|
|
85
|
-
"inject": [
|
|
101
|
+
"inject": [
|
|
102
|
+
"@deepseek-ai/dsh-client-ui-layout",
|
|
103
|
+
"@deepseek-ai/dsh-client-ui-conversation"
|
|
104
|
+
],
|
|
86
105
|
"platform": "web"
|
|
87
106
|
}
|
|
88
107
|
}
|