@customize-agent/server 4.60.6 → 4.60.11

package/scripts/upload-matrix.cjs

@@ -12,7 +12,7 @@ const serverRoot = packageJson?.name === '@customize-agent/server'
   : path.dirname(require.resolve('@customize-agent/server/package.json', { paths: [packageRoot, installRoot] }));
 const nextBin = require.resolve('next/dist/bin/next', { paths: [serverRoot, packageRoot, installRoot] });
 const tempRoot = process.env.RUNNER_TEMP || fs.mkdtempSync(path.join(require('os').tmpdir(), 'customize-upload-'));
-const home = path.join(tempRoot, 'upload-home');
+const uploadDataHome = path.join(tempRoot, 'upload-home');
 const projectRoot = path.join(tempRoot, 'upload-project');
 const dashboardLog = path.join(tempRoot, 'upload-dashboard.log');
 const basePort = 18000 + Math.floor(Math.random() * 1000);
@@ -20,7 +20,7 @@ const port = Number(process.env.CUSTOMIZE_UPLOAD_PORT || basePort);
 const base = `http://127.0.0.1:${port}`;
 const marker = `uploadmatrix${Date.now()}`;
 
-fs.rmSync(home, { recursive: true, force: true });
+fs.rmSync(uploadDataHome, { recursive: true, force: true });
 fs.rmSync(projectRoot, { recursive: true, force: true });
 fs.rmSync(dashboardLog, { force: true });
 fs.mkdirSync(projectRoot, { recursive: true });
@@ -29,6 +29,19 @@ function minimalPng() {
   return Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==', 'base64');
 }
 
+function ocrPng() {
+  const canvasModule = require(require.resolve('@napi-rs/canvas', { paths: [serverRoot] }));
+  const font = path.join(serverRoot, 'public', 'fonts', 'NotoSansCJKsc-Regular.otf');
+  assert(fs.existsSync(font), 'Packaged review font is missing');
+  canvasModule.GlobalFonts.registerFromPath(font, 'UploadFixtureFont');
+  const canvas = canvasModule.createCanvas(1200, 240), context = canvas.getContext('2d');
+  context.fillStyle = '#fff'; context.fillRect(0, 0, 1200, 240);
+  context.fillStyle = '#000'; context.font = '44px UploadFixtureFont';
+  context.fillText('混凝土强度等级 C35', 30, 75);
+  context.fillText('合同工期 180 日历天', 30, 165);
+  return canvas.toBuffer('image/png');
+}
+
 function minimalPdf(text) {
   const stream = `BT /F1 12 Tf 72 720 Td (${text.replace(/[()\\]/g, ' ')}) Tj ET`;
   const objs = [
@@ -57,6 +70,7 @@ const samples = [
   { name: 'sheet-tsv.tsv', kind: 'spreadsheet/tsv', content: `name\tvalue\n${marker}\ttsv\n` },
   { name: 'image-svg.svg', kind: 'image/vector', content: `<svg xmlns="http://www.w3.org/2000/svg"><text>${text('svg')}</text></svg>` },
   { name: 'image-png.png', kind: 'image/raster', content: minimalPng(), allowZeroChunks: true },
+  { name: 'image-ocr.png', kind: 'image/raster-ocr', content: ocrPng(), expectedText: 'C35', additionalText: '180', query: '混凝土 C35 工期 180' },
   { name: 'cad-dxf.dxf', kind: 'cad/autocad', content: `0\nSECTION\n2\nENTITIES\n0\nTEXT\n1\n${text('dxf')}\n0\nENDSEC\n0\nEOF\n` },
   { name: 'cad-step.step', kind: 'cad/step', content: `ISO-10303-21; HEADER; FILE_DESCRIPTION(('${text('step')}'),'2;1'); ENDSEC; DATA; ENDSEC; END-ISO-10303-21;` },
   { name: 'code-js.js', kind: 'code/javascript', content: `export const marker = '${text('javascript')}';\n` },
@@ -65,7 +79,7 @@ const samples = [
   { name: 'data-yaml.yaml', kind: 'data/yaml', content: `marker: ${marker}\ntext: ${text('yaml')}\n` },
   { name: 'web-html.html', kind: 'web/html', content: `<html><body>${text('html')}</body></html>` },
   { name: 'web-css.css', kind: 'web/stylesheet', content: `/* ${text('css')} */\nbody{color:#111}` },
-  { name: 'diagram-drawio.drawio', kind: 'diagram/drawio', content: `<mxfile><diagram>${text('drawio')}</diagram></mxfile>`, searchable: false },
+  { name: 'diagram-drawio.drawio', kind: 'diagram/drawio', content: `<mxfile><diagram><mxGraphModel><root><mxCell id="0"/><mxCell id="1" parent="0"/><mxCell id="2" parent="1" vertex="1" value="${text('drawio')}"><mxGeometry x="10" y="10" width="100" height="40" as="geometry"/></mxCell></root></mxGraphModel></diagram></mxfile>` },
   { name: 'diagram-mermaid.mmd', kind: 'diagram/mermaid', content: `graph TD\nA[${text('mermaid')}] --> B` },
 ];
 
@@ -86,13 +100,13 @@ async function uploadSample(baseUrl, root, sample, uploadId) {
   const form = new FormData();
   form.append('uploadId', uploadId);
   form.append('projectRoot', root);
-  form.append('relativePaths', sample.name);
+  form.append('relativePaths[0]', sample.name);
   form.append('batchIndex', '0');
   form.append('totalBatches', '1');
   form.append('startIndex', '1');
   form.append('uploadComplete', '1');
   form.append('fileOffset', '0');
-  form.append('files', new Blob([sample.content]), sample.name);
+  form.append('files[0]', new Blob([sample.content]), sample.name);
   const res = await fetch(`${baseUrl}/api/kb/upload`, { method: 'POST', body: form, signal: AbortSignal.timeout(30000) });
   const raw = await res.text();
   let json;
@@ -124,11 +138,9 @@ function indexedDetail(detail, allowZeroChunks = false) {
   return file && file.status !== 'pending' && Number(file.indexedAt) > 0 && (allowZeroChunks || Number(file.chunkCount) > 0);
 }
 
-function searchHasHit(search, relativePath, expectedText) {
-  return Array.isArray(search.json?.results) && search.json.results.some(result => {
-    const filePath = result.filePath || result.relativePath;
-    return filePath === relativePath && (!expectedText || String(result.content || '').includes(expectedText));
-  });
+function searchHasHit(search, relativePath) {
+  return search.json?.contentSearchStatus === 'success' && !search.json.failures?.length
+    && search.json.files?.some(result => result.relativePath === relativePath && result.matchedBy === 'content');
 }
 
 async function startServer(serverPort, root, logPath) {
@@ -137,12 +149,12 @@ async function startServer(serverPort, root, logPath) {
   const child = spawn(node, [nextBin, 'start', '-p', String(serverPort), '-H', '127.0.0.1'], {
     cwd: serverRoot,
     stdio: ['ignore', out, out],
-    env: { ...process.env, HOME: home, CUSTOMIZE_AGENT_HOME: home, NODE_ENV: 'production', CUSTOMIZE_PROJECT_ROOT: root, CUSTOMIZE_AGENT_DISABLE_OCR: '1', LOG_LEVEL: 'debug' },
+    env: { ...process.env, CUSTOMIZE_AGENT_HOME: uploadDataHome, CUSTOMIZE_EMBEDDING_PROVIDER: 'transformers-local', NODE_ENV: 'production', CUSTOMIZE_PROJECT_ROOT: root, LOG_LEVEL: 'debug' },
   });
   fs.closeSync(out);
   await waitFor(`dashboard ${serverPort}`, async () => {
     const health = await request('GET', `http://127.0.0.1:${serverPort}/api/health`);
-    return health.status < 500 ? true : undefined;
+    return health.status === 200 && health.json?.status === 'ok' ? true : undefined;
   }, 120000);
   return child;
 }
@@ -156,7 +168,7 @@ function dumpEnv() {
   });
 }
 
-const pages = ['/', '/overview', '/knowledge', '/knowledge/files', '/knowledge/manage', '/knowledge/search', '/models', '/prompt', '/settings'];
+const pages = ['/', '/overview', '/knowledge', '/knowledge/files', '/knowledge/manage', '/models', '/prompt', '/settings'];
 const apiChecks = [
   { method: 'GET', path: '/api/health' },
   { method: 'GET', path: '/api/config/models' },
@@ -166,7 +178,7 @@ const apiChecks = [
   { method: 'GET', path: '/api/kb/tags' },
   { method: 'GET', path: '/api/kb/files', query: root => `projectRoot=${encodeURIComponent(root)}&limit=10` },
   { method: 'GET', path: '/api/kb/stats', query: root => `projectRoot=${encodeURIComponent(root)}` },
-  { method: 'GET', path: '/api/kb/search', query: root => `projectRoot=${encodeURIComponent(root)}&q=${encodeURIComponent(marker)}&limit=5` },
+  { method: 'GET', path: '/api/kb/files/search', query: root => `projectRoot=${encodeURIComponent(root)}&q=${encodeURIComponent(marker)}&limit=50` },
   { method: 'GET', path: '/api/kb/duplicates', query: root => `projectRoot=${encodeURIComponent(root)}` },
   { method: 'GET', path: '/api/kb/operations', query: root => `projectRoot=${encodeURIComponent(root)}` },
   { method: 'GET', path: '/api/prompt' },
@@ -201,13 +213,13 @@ async function verifyWebSurface(baseUrl, root, label) {
       assert(res.status === 202 && res.json.success, `${sample.name} upload failed: ${res.status} ${res.raw}`);
       const relativePath = res.json.relativePath;
       assert(relativePath, `${sample.name} missing relativePath`);
-      const progress = await request('GET', `${base}/api/kb/upload/progress?id=${encodeURIComponent(uploadId)}`);
-      assert(progress.status === 200, `${sample.name} progress failed: ${progress.status} ${progress.raw}`);
-      const files = await request('GET', `${base}/api/kb/files?projectRoot=${encodeURIComponent(projectRoot)}&limit=500`);
-      assert(files.status === 200, `${sample.name} files failed: ${files.status} ${files.raw}`);
-      assert(files.json.files?.some(f => f.relativePath === relativePath), `${sample.name} not listed`);
-      const reindex = await request('POST', `${base}/api/kb/files/reindex`, { projectRoot, relativePath });
-      assert(reindex.status === 200 || reindex.status === 202, `${sample.name} reindex failed: ${reindex.status} ${reindex.raw}`);
+      await waitFor(`${sample.name} background upload operation`, async () => {
+        const progress = await request('GET', `${base}/api/kb/operations?projectRoot=${encodeURIComponent(projectRoot)}`);
+        assert(progress.status === 200, `${sample.name} operations failed: ${progress.status}`);
+        const operation = progress.json.operations?.find(item => item.id === uploadId);
+        if (operation?.status === 'error') throw new Error(operation.error || operation.message);
+        return operation?.status === 'success' ? operation : undefined;
+      }, 120000);
       const detail = await waitFor(`${sample.name} indexed detail`, async () => {
         const current = await request('GET', `${base}/api/kb/files/detail?projectRoot=${encodeURIComponent(projectRoot)}&relativePath=${encodeURIComponent(relativePath)}`);
         if (current.status !== 200) throw new Error(`${sample.name} detail failed: ${current.status} ${current.raw}`);
@@ -216,16 +228,23 @@ async function verifyWebSurface(baseUrl, root, label) {
       }, 120000);
       if (!sample.allowZeroChunks && sample.searchable !== false) {
         const search = await waitFor(`${sample.name} searchable marker`, async () => {
-          const current = await request('GET', `${base}/api/kb/search?projectRoot=${encodeURIComponent(projectRoot)}&q=${encodeURIComponent(marker)}&limit=50&vectorWeight=1&rewriteWeight=0&keywordWeight=1`);
+          const current = await request('GET', `${base}/api/kb/files/search?projectRoot=${encodeURIComponent(projectRoot)}&q=${encodeURIComponent(sample.query ?? marker)}&limit=500`);
           if (current.status !== 200) throw new Error(`${sample.name} search failed: ${current.status} ${current.raw}`);
-          return searchHasHit(current, relativePath, marker) ? current : undefined;
+          return searchHasHit(current, relativePath) ? current : undefined;
         }, 120000);
-        assert(searchHasHit(search, relativePath, marker), `${sample.name} search did not include marker hit`);
+        assert(searchHasHit(search, relativePath), `${sample.name} search did not include marker hit`);
       }
+      const preview = await request('GET', `${base}/api/kb/files/source-preview?projectRoot=${encodeURIComponent(projectRoot)}&relativePath=${encodeURIComponent(relativePath)}`);
+      assert(preview.status === 200 && preview.json.status === 'recorded' && preview.json.conservation?.status === 'verified'
+        && preview.json.conservation.chunkCountMatches && !preview.json.conservation.staleChunkIdentities.length, `${sample.name} parsed source/chunk conservation failed`);
+      if (!sample.allowZeroChunks) assert(preview.json.text?.includes(sample.expectedText ?? marker)
+        && (!sample.additionalText || preview.json.text.includes(sample.additionalText)), `${sample.name} indexed text lost submitted content`);
+      else assert(detail.json.file?.chunkCount === 0 && preview.json.text === '', `${sample.name} empty image produced fictional source text`);
       console.log('UPLOAD', sample.kind, sample.name, JSON.stringify({ relativePath, chunks: detail.json.file?.chunkCount, vector: res.json.vectorStatus?.status }));
     }
 
-    const incrementalPath = path.join(projectRoot, 'knowledgeBase', '文档资料', 'incremental-local.txt');
+    const sdk = await import(require.resolve('@customize-agent/knowledge', { paths: [serverRoot] }));
+    const incrementalPath = path.join(sdk.getProjectKbPath(projectRoot, path.join(uploadDataHome, '.customize-agent')), '文档资料', 'incremental-local.txt');
     fs.mkdirSync(path.dirname(incrementalPath), { recursive: true });
     fs.writeFileSync(incrementalPath, text('incremental-local'));
     const reindexAll = await request('POST', `${base}/api/kb/reindex`, { projectRoot });