@chenchaolong/plugin-trade-compliance-workbench 1.0.64 → 1.0.66

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@chenchaolong/plugin-trade-compliance-workbench",
-  "version": "1.0.64",
+  "version": "1.0.66",
   "description": "外贸合规工作台新版组织级插件",
   "license": "AGPL-3.0",
   "type": "module",
@@ -17,6 +17,9 @@
   "files": [
     "dist",
     "scripts/seed-demo-data.mjs",
+    "scripts/verify-catalog-e2e.mjs",
+    "scripts/verify-catalog-samples.mjs",
+    "scripts/verify-qwen-translation.mjs",
     "!**/*.tsbuildinfo"
   ],
   "scripts": {
@@ -33,6 +36,8 @@
     "test": "vitest run",
     "test:watch": "vitest",
     "verify:catalog-samples": "pnpm build && node scripts/verify-catalog-samples.mjs",
+    "verify:catalog-e2e": "pnpm build && node scripts/verify-catalog-e2e.mjs",
+    "verify:qwen-translation": "pnpm build && node scripts/verify-qwen-translation.mjs",
     "verify:flow-local": "node scripts/verify-flow-local.mjs",
     "verify:home-layout": "node scripts/visual-check.mjs --home-layout-only",
     "verify:runtime-local": "node scripts/verify-runtime-local.mjs",

package/scripts/verify-catalog-e2e.mjs

@@ -0,0 +1,250 @@
+import 'reflect-metadata'
+import { randomUUID } from 'node:crypto'
+import { basename } from 'node:path'
+import { readFile } from 'node:fs/promises'
+import { DataSource, IsNull } from 'typeorm'
+import {
+  TRADE_COMPLIANCE_ENTITIES,
+  CatalogReviewBlock,
+  ControlledCatalogBatch,
+  ControlledGoodsRecord,
+  ImportTask,
+  SanctionCatalogBatch,
+  SanctionedCompanyAlias,
+  SanctionedCompanyRecord
+} from '../dist/lib/entities/index.js'
+import { TranslationProvider } from '../dist/lib/adapters/translation.provider.js'
+import { CatalogImportPipeline } from '../dist/lib/catalog-import/catalog-import.pipeline.js'
+import { parseCatalogDocument } from '../dist/lib/catalog-import/catalog-document.parser.js'
+import { TradeComplianceWorkbenchService } from '../dist/lib/workbench.service.js'
+
+const postgresUrl = process.env.TRADE_COMPLIANCE_TEST_POSTGRES_URL?.trim()
+const apiKey = process.env.TRADE_COMPLIANCE_TRANSLATION_API_KEY?.trim()
+const paths = process.argv.slice(2).filter(value => value !== '--')
+if (!postgresUrl || !apiKey || paths.length !== 2) {
+  console.error('Usage: TRADE_COMPLIANCE_TEST_POSTGRES_URL=... TRADE_COMPLIANCE_TRANSLATION_API_KEY=... pnpm verify:catalog-e2e -- <controlled.pdf> <sanctions.docx>')
+  process.exit(2)
+}
+
+const [controlledPath, sanctionPath] = paths
+const schema = `tcw_e2e_${process.pid}_${randomUUID().replaceAll('-', '')}`
+const scope = {
+  tenantId: '10000000-0000-4000-8000-000000000001',
+  organizationId: '20000000-0000-4000-8000-000000000001',
+  userId: '30000000-0000-4000-8000-000000000001'
+}
+const dbScope = { tenantId: scope.tenantId, organizationId: scope.organizationId }
+const targetAssistantId = '40000000-0000-4000-8000-000000000001'
+const admin = new DataSource({ type: 'postgres', url: postgresUrl })
+let dataSource
+const startedAt = Date.now()
+
+try {
+  await admin.initialize()
+  await admin.query(`CREATE SCHEMA "${schema}"`)
+  dataSource = new DataSource({
+    type: 'postgres', url: postgresUrl, schema, entities: [...TRADE_COMPLIANCE_ENTITIES],
+    synchronize: true, extra: { application_name: `tcw-catalog-e2e-${process.pid}` }
+  })
+  await dataSource.initialize()
+
+  const config = { translation: {
+    endpoint: process.env.TRADE_COMPLIANCE_TRANSLATION_ENDPOINT || 'https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions',
+    apiKey,
+    model: process.env.TRADE_COMPLIANCE_TRANSLATION_MODEL || 'qwen3.6-flash',
+    targetLanguage: 'zh-CN', batchSize: 10, concurrency: 5, timeoutMs: 120_000
+  } }
+  const qwen = new TranslationProvider({ resolve: () => config })
+  const observedTranslationStages = []
+  const translator = {
+    toChinese: (...args) => qwen.toChinese(...args),
+    toEnglish: (...args) => qwen.toEnglish(...args),
+    translateBatch: async (...args) => {
+      const tasks = await dataSource.getRepository(ImportTask).find({ where: { ...dbScope, deletedAt: IsNull() } })
+      observedTranslationStages.push(...tasks.filter(task => task.progress?.stage === 'TRANSLATING').map(task => task.id))
+      const translationStartedAt = Date.now()
+      const result = await qwen.translateBatch(...args)
+      console.error(JSON.stringify({ phase: 'translation', requested: args[0].length, returned: result.length, elapsedMs: Date.now() - translationStartedAt }))
+      return result
+    }
+  }
+  const service = new TradeComplianceWorkbenchService(dataSource, undefined, translator)
+  const submitAutomaticReview = service.submitCatalogReviewBatchInternal.bind(service)
+  service.submitCatalogReviewBatchInternal = async (...args) => {
+    const result = await submitAutomaticReview(...args)
+    if (args[4] === true) {
+      const reasons = result.errors.reduce((counts, error) => {
+        for (const reason of error.reasons) counts[reason] = (counts[reason] ?? 0) + 1
+        return counts
+      }, {})
+      console.error(JSON.stringify({ phase: 'automatic-review', submitted: args[3].length, resolved: result.resolved, remaining: result.remaining, reasons }))
+    }
+    return result
+  }
+
+  const controlled = await runDocument({
+    path: controlledPath, format: 'PDF', kind: 'CONTROLLED', documentType: 'CONTROLLED_CATALOG',
+    executionInput: { actionKey: 'upload_controlled_catalog', catalogType: 'DUAL_USE' }, service, dataSource
+  })
+  const sanction = await runDocument({
+    path: sanctionPath, format: 'DOCX', kind: 'SANCTION', documentType: 'SANCTION_CATALOG',
+    executionInput: { actionKey: 'upload_sanction_catalog' }, service, dataSource
+  })
+
+  assert(controlled.task.status === 'AWAITING_REVIEW', `controlled status before review: ${controlled.task.status}`)
+  assert(controlled.task.progress?.reviewPendingCount === controlled.parsed.reviewBlocks.length, 'controlled pending review count mismatch')
+  assert(sanction.task.status === 'AWAITING_REVIEW', `sanction status before review: ${sanction.task.status}`)
+  assert(sanction.task.progress?.reviewPendingCount === 1, `sanction pending review count: ${sanction.task.progress?.reviewPendingCount}`)
+  assert(observedTranslationStages.includes(sanction.task.id), 'sanction translation stage was not persisted before the Qwen request')
+  assert(sanction.observerSummaries.at(-1)?.reviewCount === 1, 'sanction observer did not receive the remaining review count')
+
+  const controlledBefore = await dataSource.getRepository(ControlledCatalogBatch).findOneByOrFail({ id: controlled.task.resultEntityId })
+  const sanctionBefore = await dataSource.getRepository(SanctionCatalogBatch).findOneByOrFail({ id: sanction.task.resultEntityId })
+  assert(controlledBefore.successCount > 0, 'controlled catalog persisted no valid rows')
+  assert(sanctionBefore.successCount === 167, `sanction translated success count: ${sanctionBefore.successCount}`)
+
+  const sanctionRows = await dataSource.getRepository(SanctionedCompanyRecord).find({ where: { batchId: sanctionBefore.id, deletedAt: IsNull() } })
+  const sanctionByName = new Map(sanctionRows.map(row => [row.companyName, row]))
+  const translatedBlocks = sanction.parsed.reviewBlocks.filter(block => block.reasons.length === 1 && block.reasons[0] === 'TRANSLATION_REQUIRED')
+  assert(translatedBlocks.length === 167, `sanction translation block count: ${translatedBlocks.length}`)
+  for (const block of translatedBlocks) {
+    const companyName = String(block.payload.companyName ?? '')
+    const row = sanctionByName.get(companyName)
+    assert(row, `translated sanction company missing: ${companyName}`)
+    assert(row.sourceLocation && row.sourceContent, `sanction source evidence missing: ${companyName}`)
+    for (const field of ['sanctionReason', 'remarks']) {
+      const source = block.payload[field]
+      const translated = row[field]
+      if (typeof source === 'string' && /[A-Za-zА-Яа-яЁё]/u.test(source)) {
+        assert(typeof translated === 'string' && /\p{Script=Han}/u.test(translated), `${field} was not translated for ${companyName}`)
+      }
+    }
+  }
+
+  const expectedAliases = translatedBlocks.reduce((total, block) => total + (Array.isArray(block.payload.aliases) ? block.payload.aliases.length : 0), 0)
+  const aliasCount = await dataSource.getRepository(SanctionedCompanyAlias).count({ where: { deletedAt: IsNull() } })
+  assert(aliasCount === expectedAliases, `sanction alias count: expected ${expectedAliases}, received ${aliasCount}`)
+
+  const controlledRows = await dataSource.getRepository(ControlledGoodsRecord).find({ where: { batchId: controlledBefore.id, deletedAt: IsNull() } })
+  assert(controlledRows.length === controlledBefore.successCount, 'controlled batch and record counts differ')
+  assert(controlledRows.every(row => row.sourceLocation && row.sourceContent), 'controlled source evidence is incomplete')
+  assert(controlledRows.every(row => row.translationSource === 'SOURCE_CHINESE'), 'controlled Chinese source rows have an incorrect translation source')
+
+  const controlledRejected = await rejectRemainingReviews(service, controlled.task.id)
+  const sanctionRejected = await rejectRemainingReviews(service, sanction.task.id)
+  assert(controlledRejected === controlled.parsed.reviewBlocks.length, 'controlled review rejection count mismatch')
+  assert(sanctionRejected === 1, 'sanction review rejection count mismatch')
+
+  const controlledFinalTask = await dataSource.getRepository(ImportTask).findOneByOrFail({ id: controlled.task.id })
+  const sanctionFinalTask = await dataSource.getRepository(ImportTask).findOneByOrFail({ id: sanction.task.id })
+  const controlledFinalBatch = await dataSource.getRepository(ControlledCatalogBatch).findOneByOrFail({ id: controlledBefore.id })
+  const sanctionFinalBatch = await dataSource.getRepository(SanctionCatalogBatch).findOneByOrFail({ id: sanctionBefore.id })
+  assert(controlledFinalTask.status === 'SUCCEEDED' && controlledFinalBatch.lifecycleStatus === 'ACTIVE', 'controlled catalog was not published')
+  assert(sanctionFinalTask.status === 'SUCCEEDED' && sanctionFinalBatch.lifecycleStatus === 'ACTIVE', 'sanction catalog was not published')
+  const unresolved = await dataSource.getRepository(CatalogReviewBlock).count({ where: { ...dbScope, status: 'PENDING' } }) +
+    await dataSource.getRepository(CatalogReviewBlock).count({ where: { ...dbScope, status: 'LEASED' } })
+  assert(unresolved === 0, `unresolved review blocks remain: ${unresolved}`)
+
+  console.log(JSON.stringify({
+    success: true,
+    elapsedMs: Date.now() - startedAt,
+    model: config.translation.model,
+    controlled: {
+      parsedCandidates: controlled.parsed.accepted.length,
+      reviewBlocks: controlled.parsed.reviewBlocks.length,
+      successCount: controlledFinalBatch.successCount,
+      duplicateCount: controlledFinalBatch.duplicateCount,
+      invalidCount: controlledFinalBatch.invalidCount,
+      lifecycleStatus: controlledFinalBatch.lifecycleStatus
+    },
+    sanction: {
+      translatedRecords: translatedBlocks.length,
+      successCount: sanctionFinalBatch.successCount,
+      aliases: aliasCount,
+      rejectedReviews: sanctionRejected,
+      lifecycleStatus: sanctionFinalBatch.lifecycleStatus
+    }
+  }, null, 2))
+} finally {
+  if (dataSource?.isInitialized) await dataSource.destroy()
+  if (admin.isInitialized) {
+    await admin.query(`DROP SCHEMA IF EXISTS "${schema}" CASCADE`)
+    await admin.destroy()
+  }
+}
+
+async function runDocument(input) {
+  const buffer = await readFile(input.path)
+  const parsed = await parseCatalogDocument({ format: input.format, kind: input.kind, buffer })
+  assert(parsed.mode === 'FAST_PIPELINE', `${input.kind} parser mode: ${parsed.mode}`)
+  const taskRepo = input.dataSource.getRepository(ImportTask)
+  const task = await taskRepo.save(taskRepo.create({
+    ...scope,
+    documentType: input.documentType,
+    sourceFileId: randomUUID(),
+    sourceFileName: basename(input.path),
+    originalSourceFileName: null,
+    fileNameEncodingStatus: 'UNCHANGED',
+    sourceFileFormat: input.format,
+    sourcePlatformFileId: null,
+    sourceWorkspacePath: null,
+    status: 'AI_PARSING',
+    targetAssistantId,
+    queueJobId: null,
+    executionInput: input.executionInput,
+    executionCommand: null,
+    commandDispatchedAt: null,
+    progress: null,
+    observerConversationId: null,
+    handoffCallbackSequence: 0,
+    claimAttemptToken: randomUUID(),
+    resultPayloadVersion: null,
+    resultPayload: null,
+    resultEntityId: null,
+    resultProcessingClaimId: null,
+    resultProcessingClaimExpiresAt: null,
+    currentReanalysisRunId: null,
+    reviewSavedAt: null,
+    retryOfTaskId: null,
+    errorCode: null,
+    errorMessage: null,
+    queuedAt: new Date(),
+    startedAt: new Date(),
+    completedAt: null,
+    canceledAt: null,
+    canceledById: null,
+    deletedAt: null,
+    deletedById: null,
+    deleteOperation: null,
+    machineElapsedMs: null,
+    createdById: scope.userId
+  }))
+  const observerSummaries = []
+  const pipeline = new CatalogImportPipeline(
+    input.service,
+    { read: async fileId => {
+      assert(fileId === task.sourceFileId, `unexpected source file id: ${fileId}`)
+      return buffer
+    } },
+    async () => parsed,
+    async summary => { observerSummaries.push(summary) }
+  )
+  const outcome = await pipeline.process(scope, task)
+  assert(outcome.kind === 'COMPLETED', `${input.kind} pipeline outcome: ${outcome.kind}`)
+  return { parsed, task: await taskRepo.findOneByOrFail({ id: task.id }), observerSummaries }
+}
+
+async function rejectRemainingReviews(service, taskId) {
+  let rejected = 0
+  while (true) {
+    const leased = await service.nextCatalogReviewBatch(scope, taskId, 100)
+    if (!leased.leaseToken || leased.items.length === 0) return rejected
+    const result = await service.submitCatalogReviewBatch(scope, taskId, leased.leaseToken,
+      leased.items.map(item => ({ blockId: item.blockId, decision: 'REJECT' })))
+    rejected += result.rejected
+  }
+}
+
+function assert(condition, message) {
+  if (!condition) throw new Error(message)
+}

package/scripts/verify-catalog-samples.mjs

@@ -0,0 +1,61 @@
+import { readFile } from 'node:fs/promises'
+import { parseDocxCatalog } from '../dist/lib/catalog-import/docx.parser.js'
+import { parsePdfCatalog } from '../dist/lib/catalog-import/pdf-layout.parser.js'
+
+const [pdfPath, docxPath] = process.argv.slice(2).filter(argument => argument !== '--')
+if (!pdfPath || !docxPath) {
+  console.error('Usage: pnpm verify:catalog-samples -- <controlled-catalog.pdf> <sanction-catalog.docx>')
+  process.exit(2)
+}
+
+const startedAt = Date.now()
+const [pdf, docx] = await Promise.all([
+  parsePdfCatalog(await readFile(pdfPath)),
+  parseDocxCatalog({ buffer: await readFile(docxPath) })
+])
+
+const docxCandidates = [
+  ...docx.accepted.map(candidate => candidate.values),
+  ...docx.reviewBlocks.map(block => block.payload)
+]
+const reviewReasons = (blocks) => blocks.reduce((counts, block) => {
+  for (const reason of block.reasons) counts[reason] = (counts[reason] ?? 0) + 1
+  return counts
+}, {})
+const aliasStats = {
+  localLanguageRecords: docxCandidates.filter(values => values.aliases?.some(alias => alias.aliasType === 'LOCAL_LANGUAGE')).length,
+  akaRecords: docxCandidates.filter(values => values.aliases?.some(alias => alias.aliasType === 'AKA')).length
+}
+
+const report = {
+  elapsedMs: Date.now() - startedAt,
+  pdf: {
+    schemas: pdf.schemas,
+    accepted: pdf.accepted.length,
+    reviewBlocks: pdf.reviewBlocks.length,
+    reviewReasons: reviewReasons(pdf.reviewBlocks)
+  },
+  docx: {
+    entityCandidates: docxCandidates.length,
+    accepted: docx.accepted.length,
+    reviewBlocks: docx.reviewBlocks.length,
+    skippedPersons: docx.skipped.filter(item => item.reason === 'SANCTION_PERSON_OUT_OF_SCOPE').length,
+    reviewReasons: reviewReasons(docx.reviewBlocks),
+    aliases: aliasStats
+  }
+}
+console.log(JSON.stringify(report, null, 2))
+
+const failures = []
+if (!pdf.schemas.includes('IMPORT_FIVE_COLUMN')) failures.push('PDF import five-column schema was not detected')
+if (!pdf.schemas.includes('EXPORT_SIX_COLUMN')) failures.push('PDF export six-column schema was not detected')
+if (docxCandidates.length !== 168) failures.push(`DOCX entity candidate count is ${docxCandidates.length}, expected 168`)
+if (report.docx.skippedPersons !== 48) failures.push(`DOCX skipped person count is ${report.docx.skippedPersons}, expected 48`)
+if (report.docx.reviewReasons.EFFECTIVE_DATE_INVALID !== 1) failures.push('DOCX incomplete listing date was not isolated exactly once')
+if (aliasStats.localLanguageRecords !== 157) failures.push(`DOCX local-language alias record count is ${aliasStats.localLanguageRecords}, expected 157`)
+if (aliasStats.akaRecords !== 78) failures.push(`DOCX AKA record count is ${aliasStats.akaRecords}, expected 78`)
+
+if (failures.length) {
+  for (const failure of failures) console.error(`Verification failed: ${failure}`)
+  process.exit(1)
+}

package/scripts/verify-qwen-translation.mjs

@@ -0,0 +1,46 @@
+import { TranslationProvider } from '../dist/lib/adapters/translation.provider.js'
+
+const apiKey = process.env.TRADE_COMPLIANCE_TRANSLATION_API_KEY?.trim()
+if (!apiKey) {
+  console.error('TRADE_COMPLIANCE_TRANSLATION_API_KEY is required')
+  process.exit(2)
+}
+
+const config = {
+  translation: {
+    endpoint: process.env.TRADE_COMPLIANCE_TRANSLATION_ENDPOINT || 'https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions',
+    apiKey,
+    model: process.env.TRADE_COMPLIANCE_TRANSLATION_MODEL || 'qwen3.6-flash',
+    targetLanguage: process.env.TRADE_COMPLIANCE_TRANSLATION_TARGET_LANGUAGE || 'zh-CN',
+    batchSize: 10,
+    concurrency: 5,
+    timeoutMs: 120_000
+  }
+}
+const provider = new TranslationProvider({ resolve: () => config })
+const startedAt = Date.now()
+const records = await provider.translateBatch([
+  { recordId: 'controlled-1', fields: { productName: 'Electric motor', controlDescription: 'Subject to export controls' } },
+  { recordId: 'sanction-1', fields: { sanctionReason: 'Supporting restricted exports and concealing end-user information' } }
+])
+const singleTranslation = await provider.toChinese('Commercial aircraft engine')
+
+const byId = new Map(records.map(record => [record.recordId, record.fields]))
+const failures = []
+for (const [recordId, fields] of [
+  ['controlled-1', ['productName', 'controlDescription']],
+  ['sanction-1', ['sanctionReason']]
+]) {
+  const translated = byId.get(recordId)
+  if (!translated) failures.push(`${recordId} is missing`)
+  for (const field of fields) {
+    const value = translated?.[field]
+    if (typeof value !== 'string' || !/\p{Script=Han}/u.test(value)) failures.push(`${recordId}.${field} is not Chinese`)
+  }
+}
+if (typeof singleTranslation !== 'string' || !/\p{Script=Han}/u.test(singleTranslation)) {
+  failures.push('single text translation is not Chinese')
+}
+
+console.log(JSON.stringify({ model: config.translation.model, elapsedMs: Date.now() - startedAt, batchRecordCount: records.length, singleTranslation: Boolean(singleTranslation), success: failures.length === 0, failures }, null, 2))
+if (failures.length) process.exit(1)