@customize-agent/knowledge 2.0.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunking/text-chunker.d.ts +12 -1
- package/dist/chunking/text-chunker.js +219 -41
- package/dist/core/change-tracker.d.ts +1 -0
- package/dist/core/change-tracker.js +21 -0
- package/dist/core/index-state-store.d.ts +35 -1
- package/dist/core/index-state-store.js +249 -16
- package/dist/core/knowledge-base-manager.d.ts +69 -3
- package/dist/core/knowledge-base-manager.js +535 -132
- package/dist/core/multi-project-manager.d.ts +7 -4
- package/dist/core/multi-project-manager.js +35 -23
- package/dist/embedding/embedding-provider.d.ts +1 -0
- package/dist/embedding/embedding-provider.js +16 -1
- package/dist/extraction/content-extractor.d.ts +2 -0
- package/dist/extraction/content-extractor.js +209 -42
- package/dist/extraction/module-resolver.d.ts +17 -0
- package/dist/extraction/module-resolver.js +113 -0
- package/dist/index.d.ts +2 -3
- package/dist/index.js +2 -2
- package/dist/llm/llm-search-provider.d.ts +23 -0
- package/dist/llm/llm-search-provider.js +1 -0
- package/dist/search/federation-search.d.ts +29 -0
- package/dist/search/federation-search.js +8 -1
- package/dist/vector/chroma-store.d.ts +2 -0
- package/dist/vector/chroma-store.js +53 -22
- package/dist/vector/vector-indexer.d.ts +3 -0
- package/dist/vector/vector-indexer.js +23 -0
- package/package.json +11 -3
- package/dist/server/dashboard-client.d.ts +0 -2
- package/dist/server/dashboard-client.js +0 -396
- package/dist/server/dashboard-i18n.d.ts +0 -112
- package/dist/server/dashboard-i18n.js +0 -220
- package/dist/server/dashboard-page.d.ts +0 -6
- package/dist/server/dashboard-page.js +0 -138
- package/dist/server/dashboard-server.d.ts +0 -13
- package/dist/server/dashboard-server.js +0 -225
- package/dist/server/dashboard-styles.d.ts +0 -1
- package/dist/server/dashboard-styles.js +0 -152
|
@@ -1,21 +1,23 @@
|
|
|
1
|
-
import type { FederatedResult, SearchFilters, SearchScope } from '../search/federation-search.js';
|
|
2
|
-
import { FederationSearch } from '../search/federation-search.js';
|
|
1
|
+
import type { FederatedResult, RetrievalWeights, SearchFilters, SearchScope } from '../search/federation-search.js';
|
|
3
2
|
import type { CrossProjectDuplicate, ProjectInfo } from '../types.js';
|
|
4
3
|
import { KnowledgeBaseManager } from './knowledge-base-manager.js';
|
|
4
|
+
import type { LLMSearchProvider } from '../llm/llm-search-provider.js';
|
|
5
5
|
export declare class MultiProjectManager {
|
|
6
6
|
private readonly storageRoot;
|
|
7
|
+
private readonly llmProvider?;
|
|
7
8
|
private readonly registry;
|
|
8
9
|
private readonly configManager;
|
|
9
10
|
private readonly projects;
|
|
10
11
|
private globalKB?;
|
|
11
|
-
constructor(storageRoot?: string);
|
|
12
|
+
constructor(storageRoot?: string, llmProvider?: LLMSearchProvider);
|
|
12
13
|
getProject(projectRoot: string): Promise<KnowledgeBaseManager>;
|
|
13
14
|
getGlobalKB(): Promise<KnowledgeBaseManager>;
|
|
14
15
|
listProjects(): Promise<ProjectInfo[]>;
|
|
15
16
|
search(projectRoot: string, query: string, options?: {
|
|
16
17
|
limit?: number;
|
|
17
18
|
scope?: SearchScope;
|
|
18
|
-
|
|
19
|
+
weights?: RetrievalWeights;
|
|
20
|
+
}): Promise<FederatedResult>;
|
|
19
21
|
semanticSearch(projectRoot: string, query: string, options?: {
|
|
20
22
|
limit?: number;
|
|
21
23
|
scope?: SearchScope;
|
|
@@ -26,5 +28,6 @@ export declare class MultiProjectManager {
|
|
|
26
28
|
forgetProject(projectId: string): Promise<void>;
|
|
27
29
|
closeProject(projectId: string): Promise<void>;
|
|
28
30
|
shutdown(): Promise<void>;
|
|
31
|
+
private mergeDebug;
|
|
29
32
|
private updateRegistry;
|
|
30
33
|
}
|
|
@@ -8,12 +8,14 @@ import { getProjectKbPath, ProjectConfigManager } from './project-config.js';
|
|
|
8
8
|
import { ProjectRegistry } from './project-registry.js';
|
|
9
9
|
export class MultiProjectManager {
|
|
10
10
|
storageRoot;
|
|
11
|
+
llmProvider;
|
|
11
12
|
registry;
|
|
12
13
|
configManager = new ProjectConfigManager();
|
|
13
14
|
projects = new Map();
|
|
14
15
|
globalKB;
|
|
15
|
-
constructor(storageRoot = path.join(os.homedir(), USER_DATA_DIR)) {
|
|
16
|
+
constructor(storageRoot = path.join(os.homedir(), USER_DATA_DIR), llmProvider) {
|
|
16
17
|
this.storageRoot = storageRoot;
|
|
18
|
+
this.llmProvider = llmProvider;
|
|
17
19
|
this.registry = new ProjectRegistry(path.join(storageRoot, 'projects', 'registry.db'));
|
|
18
20
|
}
|
|
19
21
|
async getProject(projectRoot) {
|
|
@@ -29,6 +31,7 @@ export class MultiProjectManager {
|
|
|
29
31
|
projectId,
|
|
30
32
|
kbPath: getProjectKbPath(resolvedRoot),
|
|
31
33
|
storageRoot: this.storageRoot,
|
|
34
|
+
llmProvider: this.llmProvider,
|
|
32
35
|
});
|
|
33
36
|
manager.initialize();
|
|
34
37
|
this.projects.set(projectId, manager);
|
|
@@ -38,7 +41,7 @@ export class MultiProjectManager {
|
|
|
38
41
|
async getGlobalKB() {
|
|
39
42
|
if (this.globalKB)
|
|
40
43
|
return this.globalKB;
|
|
41
|
-
const manager = new KnowledgeBaseManager({ scope: 'global', storageRoot: this.storageRoot });
|
|
44
|
+
const manager = new KnowledgeBaseManager({ scope: 'global', storageRoot: this.storageRoot, llmProvider: this.llmProvider });
|
|
42
45
|
manager.initialize();
|
|
43
46
|
await manager.incrementalIndex();
|
|
44
47
|
this.globalKB = manager;
|
|
@@ -52,29 +55,22 @@ export class MultiProjectManager {
|
|
|
52
55
|
const scope = options.scope ?? 'all';
|
|
53
56
|
const project = await this.getProject(projectRoot);
|
|
54
57
|
await project.incrementalIndex();
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
scope: 'project',
|
|
62
|
-
collection: result.collectionName,
|
|
63
|
-
score: result.score,
|
|
64
|
-
})));
|
|
65
|
-
}
|
|
66
|
-
if (scope === 'global' || scope === 'all') {
|
|
58
|
+
if (scope === 'project')
|
|
59
|
+
return project.hybridSearch(query, { limit, weights: options.weights });
|
|
60
|
+
const projectResults = scope === 'all'
|
|
61
|
+
? await project.hybridSearch(query, { limit, weights: options.weights })
|
|
62
|
+
: { results: [], scopesSearched: [], queryTimeMs: 0 };
|
|
63
|
+
if (scope === 'global') {
|
|
67
64
|
const global = await this.getGlobalKB();
|
|
68
|
-
|
|
69
|
-
id: result.id,
|
|
70
|
-
content: result.content,
|
|
71
|
-
filePath: result.relativePath,
|
|
72
|
-
scope: 'global',
|
|
73
|
-
collection: result.collectionName,
|
|
74
|
-
score: result.score,
|
|
75
|
-
})));
|
|
65
|
+
return global.hybridSearch(query, { limit, weights: options.weights });
|
|
76
66
|
}
|
|
77
|
-
|
|
67
|
+
const global = await this.getGlobalKB();
|
|
68
|
+
const globalResults = await global.hybridSearch(query, { limit, weights: options.weights });
|
|
69
|
+
const merged = new FederationSearch().merge([...projectResults.results, ...globalResults.results], limit, 'all');
|
|
70
|
+
return {
|
|
71
|
+
...merged,
|
|
72
|
+
debug: this.mergeDebug(projectResults.debug, globalResults.debug),
|
|
73
|
+
};
|
|
78
74
|
}
|
|
79
75
|
async semanticSearch(projectRoot, query, options = {}) {
|
|
80
76
|
const scope = options.scope ?? 'all';
|
|
@@ -104,6 +100,7 @@ export class MultiProjectManager {
|
|
|
104
100
|
projectId: project.projectId,
|
|
105
101
|
kbPath: project.kbPath,
|
|
106
102
|
storageRoot: this.storageRoot,
|
|
103
|
+
llmProvider: this.llmProvider,
|
|
107
104
|
});
|
|
108
105
|
for (const item of manager.store.listContentHashes()) {
|
|
109
106
|
const duplicate = byHash.get(item.contentHash) ?? { contentHash: item.contentHash, files: [] };
|
|
@@ -143,6 +140,21 @@ export class MultiProjectManager {
|
|
|
143
140
|
this.globalKB = undefined;
|
|
144
141
|
this.registry.close();
|
|
145
142
|
}
|
|
143
|
+
mergeDebug(projectDebug, globalDebug) {
|
|
144
|
+
if (!projectDebug && !globalDebug)
|
|
145
|
+
return undefined;
|
|
146
|
+
return {
|
|
147
|
+
originalQuery: projectDebug?.originalQuery ?? globalDebug?.originalQuery,
|
|
148
|
+
rewrittenQueries: [...new Set([...(projectDebug?.rewrittenQueries ?? []), ...(globalDebug?.rewrittenQueries ?? [])])],
|
|
149
|
+
weights: projectDebug?.weights ?? globalDebug?.weights,
|
|
150
|
+
recallCounts: {
|
|
151
|
+
keyword: (projectDebug?.recallCounts?.keyword ?? 0) + (globalDebug?.recallCounts?.keyword ?? 0),
|
|
152
|
+
vector: (projectDebug?.recallCounts?.vector ?? 0) + (globalDebug?.recallCounts?.vector ?? 0),
|
|
153
|
+
merged: (projectDebug?.recallCounts?.merged ?? 0) + (globalDebug?.recallCounts?.merged ?? 0),
|
|
154
|
+
},
|
|
155
|
+
reranker: projectDebug?.reranker ?? globalDebug?.reranker,
|
|
156
|
+
};
|
|
157
|
+
}
|
|
146
158
|
updateRegistry(manager, projectRoot, projectName, lastOpenedAt) {
|
|
147
159
|
const stats = manager.getStats();
|
|
148
160
|
if (!manager.projectId)
|
|
@@ -13,7 +13,7 @@ export class HashEmbeddingProvider {
|
|
|
13
13
|
}
|
|
14
14
|
embed(text) {
|
|
15
15
|
const vector = Array.from({ length: this.dimensions }, () => 0);
|
|
16
|
-
const tokens =
|
|
16
|
+
const tokens = this.tokenize(text);
|
|
17
17
|
for (const token of tokens) {
|
|
18
18
|
const hash = crypto.createHash('sha256').update(token).digest();
|
|
19
19
|
const index = hash.readUInt32BE(0) % this.dimensions;
|
|
@@ -22,6 +22,21 @@ export class HashEmbeddingProvider {
|
|
|
22
22
|
}
|
|
23
23
|
return this.normalize(vector);
|
|
24
24
|
}
|
|
25
|
+
tokenize(text) {
|
|
26
|
+
const normalized = text.toLowerCase().normalize('NFKC');
|
|
27
|
+
const tokens = normalized.match(/[\p{Script=Han}]{1,4}|[\p{Letter}\p{Number}_-]+/gu) ?? [];
|
|
28
|
+
const grams = [];
|
|
29
|
+
for (const token of tokens) {
|
|
30
|
+
grams.push(token);
|
|
31
|
+
if (/^[\p{Script=Han}]+$/u.test(token) && token.length > 1) {
|
|
32
|
+
for (let i = 0; i < token.length - 1; i++)
|
|
33
|
+
grams.push(token.slice(i, i + 2));
|
|
34
|
+
for (let i = 0; i < token.length - 2; i++)
|
|
35
|
+
grams.push(token.slice(i, i + 3));
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
return grams.filter(Boolean);
|
|
39
|
+
}
|
|
25
40
|
normalize(vector) {
|
|
26
41
|
const norm = Math.sqrt(vector.reduce((sum, value) => sum + value * value, 0));
|
|
27
42
|
if (norm === 0)
|
|
@@ -25,6 +25,8 @@ export declare class ContentExtractor {
|
|
|
25
25
|
private extractRasterImage;
|
|
26
26
|
private validateRasterImage;
|
|
27
27
|
private extractPdf;
|
|
28
|
+
private pdfOcrPageLimit;
|
|
29
|
+
private extractScannedPdfOcr;
|
|
28
30
|
private extractPdfText;
|
|
29
31
|
private extractSvg;
|
|
30
32
|
private isTextReadable;
|
|
@@ -2,6 +2,7 @@ import { spawnSync } from 'node:child_process';
|
|
|
2
2
|
import * as fs from 'node:fs';
|
|
3
3
|
import * as path from 'node:path';
|
|
4
4
|
import { ExternalExtractorRegistry } from './external-extractor.js';
|
|
5
|
+
import { resolveAndImport, resolvePackage, getNodeModulesRoot } from './module-resolver.js';
|
|
5
6
|
export class ContentExtractor {
|
|
6
7
|
externalExtractors;
|
|
7
8
|
constructor(externalExtractors = ExternalExtractorRegistry.fromEnvironment()) {
|
|
@@ -142,17 +143,23 @@ export class ContentExtractor {
|
|
|
142
143
|
const textEntities = this.matchAll(raw, /\n\s*(?:1|3)\s*\n([^\n]+)/gu).slice(0, 500);
|
|
143
144
|
const blocks = this.matchAll(raw, /\n\s*2\s*\n([^\n]+)/gu).slice(0, 300);
|
|
144
145
|
const entityTypes = this.matchAll(raw, /\n\s*0\s*\n([A-Z][A-Z0-9_]+)/gu).slice(0, 1000);
|
|
145
|
-
|
|
146
|
+
const uniqueLayers = Array.from(new Set(layers));
|
|
147
|
+
const uniqueBlocks = Array.from(new Set(blocks));
|
|
148
|
+
const uniqueEntityTypes = Array.from(new Set(entityTypes));
|
|
149
|
+
metadata.layerCount = uniqueLayers.length;
|
|
150
|
+
metadata.layerNames = uniqueLayers.slice(0, 80);
|
|
146
151
|
metadata.textEntityCount = textEntities.length;
|
|
147
|
-
metadata.blockCount =
|
|
148
|
-
metadata.
|
|
152
|
+
metadata.blockCount = uniqueBlocks.length;
|
|
153
|
+
metadata.blockNames = uniqueBlocks.slice(0, 80);
|
|
154
|
+
metadata.entityTypeCount = uniqueEntityTypes.length;
|
|
155
|
+
metadata.entityTypes = uniqueEntityTypes.slice(0, 80);
|
|
149
156
|
metadata.contentCoverage = 'dxf_layers_blocks_entities_text';
|
|
150
157
|
return {
|
|
151
158
|
text: [
|
|
152
159
|
this.metadataOnlyText(file),
|
|
153
|
-
`CAD DXF 图层: ${
|
|
154
|
-
`CAD DXF 块/符号: ${
|
|
155
|
-
`CAD DXF 实体类型: ${
|
|
160
|
+
`CAD DXF 图层: ${uniqueLayers.join(', ')}`,
|
|
161
|
+
`CAD DXF 块/符号: ${uniqueBlocks.join(', ')}`,
|
|
162
|
+
`CAD DXF 实体类型: ${uniqueEntityTypes.join(', ')}`,
|
|
156
163
|
`CAD DXF 标注/文本:\n${textEntities.join('\n')}`,
|
|
157
164
|
].join('\n'),
|
|
158
165
|
metadata,
|
|
@@ -165,16 +172,20 @@ export class ContentExtractor {
|
|
|
165
172
|
const materials = this.matchAll(raw, /MATERIAL[^']*'([^']+)'/giu).slice(0, 120);
|
|
166
173
|
const entities = this.matchAll(raw, /#\d+\s*=\s*([A-Z0-9_]+)/gu).slice(0, 1000);
|
|
167
174
|
const names = this.matchAll(raw, /'([^']{2,120})'/gu).slice(0, 500);
|
|
175
|
+
const uniqueStepEntities = Array.from(new Set(entities));
|
|
168
176
|
metadata.productCount = products.length;
|
|
177
|
+
metadata.productNames = products.slice(0, 80);
|
|
169
178
|
metadata.materialCount = materials.length;
|
|
170
|
-
metadata.
|
|
179
|
+
metadata.materialNames = materials.slice(0, 80);
|
|
180
|
+
metadata.entityTypeCount = uniqueStepEntities.length;
|
|
181
|
+
metadata.entityTypes = uniqueStepEntities.slice(0, 80);
|
|
171
182
|
metadata.contentCoverage = 'step_products_materials_entities_names';
|
|
172
183
|
return {
|
|
173
184
|
text: [
|
|
174
185
|
this.metadataOnlyText(file),
|
|
175
186
|
`STEP 产品/零件:\n${products.join('\n')}`,
|
|
176
187
|
`STEP 材料: ${materials.join(', ')}`,
|
|
177
|
-
`STEP 实体类型: ${
|
|
188
|
+
`STEP 实体类型: ${uniqueStepEntities.join(', ')}`,
|
|
178
189
|
`STEP 名称/属性:\n${names.join('\n')}`,
|
|
179
190
|
].join('\n'),
|
|
180
191
|
metadata,
|
|
@@ -185,11 +196,14 @@ export class ContentExtractor {
|
|
|
185
196
|
const raw = fs.readFileSync(file.absolutePath, 'utf8');
|
|
186
197
|
const names = this.matchAll(raw, /'([^']{2,120})'/gu).slice(0, 500);
|
|
187
198
|
const entityTypes = this.matchAll(raw, /^\s*(\d{3,4})\s*,/gmu).slice(0, 1000);
|
|
199
|
+
const uniqueIgesTypes = Array.from(new Set(entityTypes));
|
|
188
200
|
metadata.entityNameCount = names.length;
|
|
189
|
-
metadata.
|
|
201
|
+
metadata.entityNames = names.slice(0, 80);
|
|
202
|
+
metadata.entityTypeCount = uniqueIgesTypes.length;
|
|
203
|
+
metadata.entityTypes = uniqueIgesTypes.slice(0, 80);
|
|
190
204
|
metadata.contentCoverage = 'iges_entity_names_types';
|
|
191
205
|
return {
|
|
192
|
-
text: [this.metadataOnlyText(file), `IGES 实体类型: ${
|
|
206
|
+
text: [this.metadataOnlyText(file), `IGES 实体类型: ${uniqueIgesTypes.join(', ')}`, `IGES 实体/名称:\n${names.join('\n')}`].join('\n'),
|
|
193
207
|
metadata,
|
|
194
208
|
warnings,
|
|
195
209
|
};
|
|
@@ -256,20 +270,31 @@ export class ContentExtractor {
|
|
|
256
270
|
const metadata = { extractionMode: 'structured_data', vectorizable: true };
|
|
257
271
|
try {
|
|
258
272
|
if (file.format === 'json') {
|
|
259
|
-
const
|
|
260
|
-
|
|
273
|
+
const isJsonl = path.extname(file.absolutePath).toLowerCase() === '.jsonl';
|
|
274
|
+
const lines = isJsonl
|
|
275
|
+
? raw.split(/\r?\n/u).filter(Boolean).slice(0, 500).flatMap((line, index) => this.flattenJson(JSON.parse(line), `line${index + 1}`))
|
|
261
276
|
: this.flattenJson(JSON.parse(raw));
|
|
262
277
|
metadata.fieldCount = lines.length;
|
|
263
|
-
metadata.
|
|
264
|
-
|
|
278
|
+
metadata.recordCount = isJsonl ? raw.split(/\r?\n/u).filter(Boolean).length : 1;
|
|
279
|
+
metadata.dataPaths = lines.map(line => line.split(':')[0]).slice(0, 200);
|
|
280
|
+
metadata.contentCoverage = isJsonl ? 'jsonl_records_paths_values' : 'json_paths_values';
|
|
281
|
+
return { text: [this.metadataOnlyText(file), ...lines.slice(0, 1500)].join('\n'), metadata, warnings: [] };
|
|
265
282
|
}
|
|
266
283
|
}
|
|
267
284
|
catch {
|
|
268
285
|
metadata.parseError = true;
|
|
269
286
|
}
|
|
287
|
+
if (file.format === 'yaml') {
|
|
288
|
+
const pairs = this.matchAll(raw, /^\s*([\w.-]+)\s*:\s*(.{1,300})$/gmu).slice(0, 1500);
|
|
289
|
+
metadata.fieldCount = pairs.length;
|
|
290
|
+
metadata.dataPaths = pairs.map(line => (line.split(':')[0] ?? '').trim()).slice(0, 200);
|
|
291
|
+
metadata.contentCoverage = 'yaml_key_values';
|
|
292
|
+
return { text: [this.metadataOnlyText(file), ...pairs].join('\n'), metadata, warnings: [] };
|
|
293
|
+
}
|
|
270
294
|
if (file.format === 'xml') {
|
|
271
295
|
const elements = this.matchAll(raw, /<([A-Za-z_][\w:.-]*)\b[^>]*>([^<]{1,200})<\/\1>/gu).slice(0, 1000);
|
|
272
296
|
metadata.elementTextCount = elements.length;
|
|
297
|
+
metadata.dataPaths = elements.map(line => line.match(/^<([A-Za-z_][\w:.-]*)/u)?.[1]).filter(Boolean).slice(0, 200);
|
|
273
298
|
metadata.contentCoverage = 'xml_element_text';
|
|
274
299
|
return { text: [this.metadataOnlyText(file), ...elements].join('\n'), metadata, warnings: [] };
|
|
275
300
|
}
|
|
@@ -346,6 +371,7 @@ export class ContentExtractor {
|
|
|
346
371
|
delimiter: file.format === 'tsv' ? 'tab' : 'comma',
|
|
347
372
|
rowCount: rows.length,
|
|
348
373
|
columnCount: header.length,
|
|
374
|
+
columnNames: header.slice(0, 120),
|
|
349
375
|
contentCoverage: 'table_headers_cells_text',
|
|
350
376
|
},
|
|
351
377
|
warnings: [],
|
|
@@ -354,14 +380,14 @@ export class ContentExtractor {
|
|
|
354
380
|
async extractOfficeDocument(file) {
|
|
355
381
|
if (path.extname(file.absolutePath).toLowerCase() === '.docx') {
|
|
356
382
|
try {
|
|
357
|
-
const mammoth = await
|
|
383
|
+
const mammoth = await resolveAndImport('mammoth');
|
|
358
384
|
const result = await mammoth.extractRawText({ path: file.absolutePath });
|
|
359
385
|
const text = result.value.trim();
|
|
360
386
|
if (text) {
|
|
361
387
|
return {
|
|
362
388
|
text,
|
|
363
389
|
metadata: { extractionMode: 'builtin_mammoth', vectorizable: true, contentCoverage: 'office_full_text' },
|
|
364
|
-
warnings: result.messages.map(
|
|
390
|
+
warnings: result.messages.map(m => m.message),
|
|
365
391
|
};
|
|
366
392
|
}
|
|
367
393
|
}
|
|
@@ -373,7 +399,7 @@ export class ContentExtractor {
|
|
|
373
399
|
}
|
|
374
400
|
async extractSpreadsheet(file) {
|
|
375
401
|
try {
|
|
376
|
-
const XLSX = await
|
|
402
|
+
const XLSX = await resolveAndImport('xlsx');
|
|
377
403
|
const workbook = XLSX.readFile(file.absolutePath, { cellDates: true, cellFormula: true, cellNF: true, cellStyles: true });
|
|
378
404
|
const sheetTexts = [];
|
|
379
405
|
let cellCount = 0;
|
|
@@ -410,7 +436,7 @@ export class ContentExtractor {
|
|
|
410
436
|
if (sheetTexts.length > 0) {
|
|
411
437
|
return {
|
|
412
438
|
text: sheetTexts.join('\n\n'),
|
|
413
|
-
metadata: { extractionMode: 'builtin_xlsx_structured_cells', vectorizable: true, sheetCount: sheetTexts.length, cellCount, formulaCount, mergeCount, contentCoverage: 'spreadsheet_cells_formulas_merges' },
|
|
439
|
+
metadata: { extractionMode: 'builtin_xlsx_structured_cells', vectorizable: true, sheetCount: sheetTexts.length, sheetNames: workbook.SheetNames.slice(0, 120), cellCount, formulaCount, mergeCount, contentCoverage: 'spreadsheet_cells_formulas_merges' },
|
|
414
440
|
warnings: [],
|
|
415
441
|
};
|
|
416
442
|
}
|
|
@@ -423,7 +449,8 @@ export class ContentExtractor {
|
|
|
423
449
|
async extractOfficeZip(file) {
|
|
424
450
|
const metadata = { extractionMode: 'office_zip_text', vectorizable: true };
|
|
425
451
|
try {
|
|
426
|
-
const
|
|
452
|
+
const jszipMod = await resolveAndImport('jszip');
|
|
453
|
+
const JSZip = jszipMod.default ?? jszipMod;
|
|
427
454
|
const zip = await JSZip.loadAsync(fs.readFileSync(file.absolutePath));
|
|
428
455
|
const texts = [];
|
|
429
456
|
const xmlEntries = Object.values(zip.files).filter(entry => !entry.dir && /\.(xml|rels)$/iu.test(entry.name)).slice(0, 80);
|
|
@@ -450,7 +477,8 @@ export class ContentExtractor {
|
|
|
450
477
|
return { text: this.metadataOnlyText(file), metadata, warnings: ['压缩包未提取到正文,未入库;仅 zip 可提取文件清单'] };
|
|
451
478
|
}
|
|
452
479
|
try {
|
|
453
|
-
const
|
|
480
|
+
const jszipMod = await resolveAndImport('jszip');
|
|
481
|
+
const JSZip = jszipMod.default ?? jszipMod;
|
|
454
482
|
const zip = await JSZip.loadAsync(fs.readFileSync(file.absolutePath));
|
|
455
483
|
const entries = Object.values(zip.files).map(entry => `${entry.dir ? '目录' : '文件'}: ${entry.name}`).slice(0, 1_000);
|
|
456
484
|
metadata.entryCount = Object.keys(zip.files).length;
|
|
@@ -470,10 +498,27 @@ export class ContentExtractor {
|
|
|
470
498
|
metadata.parseError = validationError;
|
|
471
499
|
return { text: '', metadata, warnings: [`图片文件无效或不完整:${validationError},未入库`] };
|
|
472
500
|
}
|
|
501
|
+
let tesseractPath;
|
|
502
|
+
try {
|
|
503
|
+
tesseractPath = resolvePackage('tesseract.js');
|
|
504
|
+
}
|
|
505
|
+
catch (e) {
|
|
506
|
+
metadata.contentCoverage = 'ocr_unavailable';
|
|
507
|
+
metadata.parseError = e.message;
|
|
508
|
+
return { text: '', metadata, warnings: [`内置 OCR 不可用:${e.message}`] };
|
|
509
|
+
}
|
|
510
|
+
// 子进程用 createRequire 加载 tesseract.js(兼容打包 Server 的 vendor 目录)
|
|
473
511
|
const result = spawnSync(process.execPath, [
|
|
474
512
|
'--input-type=module',
|
|
475
513
|
'-e',
|
|
476
|
-
`
|
|
514
|
+
`import { createRequire } from 'node:module';
|
|
515
|
+
import { fileURLToPath } from 'node:url';
|
|
516
|
+
import path from 'node:path';
|
|
517
|
+
|
|
518
|
+
const localRequire = createRequire(import.meta.url);
|
|
519
|
+
const tesseractMod = localRequire(${JSON.stringify(tesseractPath)});
|
|
520
|
+
const { createWorker } = tesseractMod;
|
|
521
|
+
|
|
477
522
|
const worker = await createWorker('chi_sim+eng');
|
|
478
523
|
try {
|
|
479
524
|
const result = await worker.recognize(process.argv[1]);
|
|
@@ -482,7 +527,7 @@ try {
|
|
|
482
527
|
await worker.terminate();
|
|
483
528
|
}`,
|
|
484
529
|
file.absolutePath,
|
|
485
|
-
], { encoding: 'utf8', timeout:
|
|
530
|
+
], { encoding: 'utf8', timeout: 120_000, maxBuffer: 20 * 1024 * 1024 });
|
|
486
531
|
if (result.status !== 0 || result.error) {
|
|
487
532
|
const message = result.error?.message || result.stderr.trim() || `OCR 子进程退出码 ${result.status ?? 'unknown'}`;
|
|
488
533
|
metadata.contentCoverage = 'ocr_failed';
|
|
@@ -491,8 +536,11 @@ try {
|
|
|
491
536
|
}
|
|
492
537
|
const text = result.stdout.trim();
|
|
493
538
|
metadata.contentCoverage = text ? 'ocr_text' : 'metadata_filename';
|
|
539
|
+
metadata.ocrProvider = 'tesseract.js';
|
|
540
|
+
metadata.ocrLanguages = 'chi_sim+eng';
|
|
541
|
+
metadata.ocrTextLength = text.length;
|
|
494
542
|
return {
|
|
495
|
-
text,
|
|
543
|
+
text: text ? [this.metadataOnlyText(file), `OCR 识别文本:\n${text}`].join('\n') : '',
|
|
496
544
|
metadata,
|
|
497
545
|
warnings: text ? [] : ['内置 OCR 未识别到文字,未入库'],
|
|
498
546
|
};
|
|
@@ -523,46 +571,164 @@ try {
|
|
|
523
571
|
}
|
|
524
572
|
async extractPdf(file) {
|
|
525
573
|
const metadata = { extractionMode: 'pdf_text', vectorizable: true };
|
|
574
|
+
const warnings = [];
|
|
575
|
+
// Tier 1: pdfjs-dist 文本提取(处理常规 PDF、压缩内容流、CJK 字体等)
|
|
526
576
|
try {
|
|
527
577
|
const raw = fs.readFileSync(file.absolutePath);
|
|
528
578
|
const text = await this.extractPdfText(raw);
|
|
529
579
|
if (text.trim()) {
|
|
530
580
|
metadata.contentCoverage = 'pdf_text_streams';
|
|
531
|
-
|
|
581
|
+
metadata.pdfExtractor = 'pdfjs-dist';
|
|
582
|
+
return { text: [this.metadataOnlyText(file), text].join('\n'), metadata, warnings };
|
|
532
583
|
}
|
|
533
584
|
}
|
|
534
585
|
catch (error) {
|
|
586
|
+
warnings.push(`PDF 文本提取失败: ${error instanceof Error ? error.message : String(error)}`);
|
|
535
587
|
metadata.parseError = error instanceof Error ? error.message : String(error);
|
|
536
588
|
}
|
|
589
|
+
// Tier 2: OCR(扫描件/图片型 PDF)—— 必须保留并确保可用
|
|
590
|
+
const ocr = await this.extractScannedPdfOcr(file);
|
|
591
|
+
if (ocr.text.trim())
|
|
592
|
+
return ocr;
|
|
593
|
+
// Tier 3: 仅索引元数据(兜底)
|
|
537
594
|
metadata.extractionMode = 'pdf_metadata_only';
|
|
538
595
|
metadata.contentCoverage = 'metadata_filename';
|
|
596
|
+
metadata.ocrRecommended = true;
|
|
597
|
+
metadata.ocrReason = ocr.metadata.ocrReason ?? 'pdf_text_stream_empty_or_unavailable';
|
|
598
|
+
metadata.pdfPageOcrSupported = true;
|
|
539
599
|
return {
|
|
540
|
-
text:
|
|
600
|
+
text: this.metadataOnlyText(file),
|
|
541
601
|
metadata,
|
|
542
|
-
warnings: ['PDF
|
|
602
|
+
warnings: [...warnings, 'PDF 正文暂未提取到文本,已索引文件名、路径和类型元数据', ...ocr.warnings],
|
|
543
603
|
};
|
|
544
604
|
}
|
|
605
|
+
pdfOcrPageLimit() {
|
|
606
|
+
const value = Number(process.env.KB_PDF_OCR_PAGE_LIMIT ?? 20);
|
|
607
|
+
return Number.isFinite(value) ? Math.min(500, Math.max(1, Math.floor(value))) : 20;
|
|
608
|
+
}
|
|
609
|
+
async extractScannedPdfOcr(file) {
|
|
610
|
+
const metadata = {
|
|
611
|
+
extractionMode: 'pdf_page_ocr_embedded',
|
|
612
|
+
vectorizable: true,
|
|
613
|
+
pdfPageOcrSupported: true,
|
|
614
|
+
ocrProvider: 'tesseract.js',
|
|
615
|
+
ocrLanguages: 'chi_sim+eng',
|
|
616
|
+
pdfRenderer: 'pdfjs-dist + @napi-rs/canvas',
|
|
617
|
+
pdfOcrPageLimit: this.pdfOcrPageLimit(),
|
|
618
|
+
};
|
|
619
|
+
const pageLimitConfig = this.pdfOcrPageLimit();
|
|
620
|
+
// 解析模块路径 —— 确保在打包 Server 等上下文中子进程也能正确加载
|
|
621
|
+
let canvasPath;
|
|
622
|
+
let pdfjsPath;
|
|
623
|
+
let tesseractPath;
|
|
624
|
+
try {
|
|
625
|
+
canvasPath = resolvePackage('@napi-rs/canvas');
|
|
626
|
+
pdfjsPath = resolvePackage('pdfjs-dist/legacy/build/pdf.mjs');
|
|
627
|
+
tesseractPath = resolvePackage('tesseract.js');
|
|
628
|
+
}
|
|
629
|
+
catch (e) {
|
|
630
|
+
metadata.ocrRecommended = true;
|
|
631
|
+
metadata.ocrReason = `OCR 依赖解析失败: ${e.message}`;
|
|
632
|
+
return { text: '', metadata, warnings: [`内置扫描 PDF OCR 不可用:${metadata.ocrReason}`] };
|
|
633
|
+
}
|
|
634
|
+
// NODE_PATH 确保子进程能解析 tesseract.js 的依赖
|
|
635
|
+
const childEnv = { ...process.env };
|
|
636
|
+
const nmRoot = getNodeModulesRoot();
|
|
637
|
+
if (nmRoot)
|
|
638
|
+
childEnv.NODE_PATH = nmRoot;
|
|
639
|
+
const result = spawnSync(process.execPath, [
|
|
640
|
+
'--input-type=module',
|
|
641
|
+
'-e',
|
|
642
|
+
`import fs from 'node:fs';
|
|
643
|
+
import os from 'node:os';
|
|
644
|
+
import path from 'node:path';
|
|
645
|
+
import { createCanvas } from ${JSON.stringify(canvasPath)};
|
|
646
|
+
import * as pdfjs from ${JSON.stringify(pdfjsPath)};
|
|
647
|
+
import { createWorker } from ${JSON.stringify(tesseractPath)};
|
|
648
|
+
const filePath = process.argv[1];
|
|
649
|
+
const configuredLimit = Math.max(1, Number(process.argv[2] || 5));
|
|
650
|
+
const bytes = new Uint8Array(fs.readFileSync(filePath));
|
|
651
|
+
const doc = await pdfjs.getDocument({ data: bytes, verbosity: 0 }).promise;
|
|
652
|
+
const pageLimit = Math.min(doc.numPages, configuredLimit);
|
|
653
|
+
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'kb-pdf-ocr-'));
|
|
654
|
+
const worker = await createWorker('chi_sim+eng');
|
|
655
|
+
const pages = [];
|
|
656
|
+
try {
|
|
657
|
+
for (let i = 1; i <= pageLimit; i += 1) {
|
|
658
|
+
const page = await doc.getPage(i);
|
|
659
|
+
const viewport = page.getViewport({ scale: 2 });
|
|
660
|
+
const canvas = createCanvas(Math.ceil(viewport.width), Math.ceil(viewport.height));
|
|
661
|
+
const context = canvas.getContext('2d');
|
|
662
|
+
await page.render({ canvasContext: context, viewport }).promise;
|
|
663
|
+
const imagePath = path.join(tmpDir, 'page-' + i + '.png');
|
|
664
|
+
fs.writeFileSync(imagePath, canvas.toBuffer('image/png'));
|
|
665
|
+
const recognized = await worker.recognize(imagePath);
|
|
666
|
+
const text = (recognized.data.text || '').trim();
|
|
667
|
+
if (text) pages.push('PDF OCR 第 ' + i + ' 页:\\n' + text);
|
|
668
|
+
}
|
|
669
|
+
} finally {
|
|
670
|
+
await worker.terminate();
|
|
671
|
+
fs.rmSync(tmpDir, { recursive: true, force: true });
|
|
672
|
+
}
|
|
673
|
+
process.stdout.write(JSON.stringify({ pageCount: doc.numPages, pageLimit, text: pages.join('\\n\\n') }));`,
|
|
674
|
+
file.absolutePath,
|
|
675
|
+
String(pageLimitConfig),
|
|
676
|
+
], { encoding: 'utf8', timeout: Math.max(120_000, pageLimitConfig * 60_000), maxBuffer: 50 * 1024 * 1024, env: childEnv });
|
|
677
|
+
if (result.status !== 0 || result.error) {
|
|
678
|
+
metadata.ocrRecommended = true;
|
|
679
|
+
metadata.ocrReason = result.error?.message ?? result.stderr.trim() ?? `pdf_page_ocr_exit_${result.status ?? 'unknown'}`;
|
|
680
|
+
return { text: '', metadata, warnings: [`内置扫描 PDF OCR 失败:${metadata.ocrReason}`] };
|
|
681
|
+
}
|
|
682
|
+
try {
|
|
683
|
+
const parsed = JSON.parse(result.stdout);
|
|
684
|
+
const text = parsed.text?.trim() ?? '';
|
|
685
|
+
metadata.ocrPageCount = parsed.pageCount ?? 0;
|
|
686
|
+
metadata.pdfOcrPageLimit = parsed.pageLimit ?? 5;
|
|
687
|
+
metadata.ocrTextLength = text.length;
|
|
688
|
+
metadata.contentCoverage = text ? 'pdf_page_ocr_text' : 'metadata_filename';
|
|
689
|
+
return { text: text ? [this.metadataOnlyText(file), text].join('\n') : '', metadata, warnings: text ? [] : ['内置扫描 PDF OCR 未识别到文字'] };
|
|
690
|
+
}
|
|
691
|
+
catch {
|
|
692
|
+
metadata.ocrRecommended = true;
|
|
693
|
+
metadata.ocrReason = 'pdf_page_ocr_output_parse_failed';
|
|
694
|
+
return { text: '', metadata, warnings: ['内置扫描 PDF OCR 输出解析失败'] };
|
|
695
|
+
}
|
|
696
|
+
}
|
|
545
697
|
async extractPdfText(buffer) {
|
|
698
|
+
// Tier 1: pdfjs-dist 文本提取(处理压缩内容流、CJK 字体、现代 PDF)
|
|
546
699
|
try {
|
|
547
|
-
const mod = await
|
|
548
|
-
const
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
700
|
+
const mod = await resolveAndImport('pdfjs-dist/legacy/build/pdf.mjs');
|
|
701
|
+
const loadingTask = mod.getDocument({ data: new Uint8Array(buffer), verbosity: 0 });
|
|
702
|
+
const doc = await loadingTask.promise;
|
|
703
|
+
const pages = [];
|
|
704
|
+
const pageLimit = Math.min(doc.numPages, 200);
|
|
705
|
+
for (let i = 1; i <= pageLimit; i++) {
|
|
706
|
+
const page = await doc.getPage(i);
|
|
707
|
+
const content = await page.getTextContent();
|
|
708
|
+
const pageText = content.items
|
|
709
|
+
.map((item) => {
|
|
710
|
+
const hasStr = item != null && typeof item === 'object' && 'str' in item;
|
|
711
|
+
return hasStr ? String(item.str) : '';
|
|
712
|
+
})
|
|
713
|
+
.filter((s) => s.trim().length > 0)
|
|
714
|
+
.join(' ');
|
|
715
|
+
if (pageText.trim())
|
|
716
|
+
pages.push(pageText.trim());
|
|
717
|
+
}
|
|
718
|
+
await doc.destroy();
|
|
719
|
+
if (pages.length > 0) {
|
|
720
|
+
const combined = pages.join('\n\n');
|
|
721
|
+
if (combined.trim())
|
|
722
|
+
return combined.slice(0, 250_000);
|
|
559
723
|
}
|
|
560
724
|
}
|
|
561
|
-
catch {
|
|
562
|
-
|
|
725
|
+
catch (e) {
|
|
726
|
+
if (process.env.KB_DEBUG === '1')
|
|
727
|
+
console.warn('[kb] pdfjs-dist extraction failed:', e.message);
|
|
563
728
|
}
|
|
729
|
+
// Tier 2: pdf-parse(兼容旧版 PDF)
|
|
564
730
|
try {
|
|
565
|
-
const mod = await
|
|
731
|
+
const mod = await resolveAndImport('pdf-parse');
|
|
566
732
|
const pdfParse = mod.default;
|
|
567
733
|
if (pdfParse) {
|
|
568
734
|
const result = await pdfParse(buffer);
|
|
@@ -571,8 +737,9 @@ try {
|
|
|
571
737
|
}
|
|
572
738
|
}
|
|
573
739
|
catch {
|
|
574
|
-
// fallback below
|
|
740
|
+
// fallback to raw regex below
|
|
575
741
|
}
|
|
742
|
+
// Tier 3: raw regex 回退(未压缩的古老 PDF)
|
|
576
743
|
const raw = buffer.toString('latin1');
|
|
577
744
|
const matches = Array.from(raw.matchAll(/\(([^()]{2,500})\)\s*T[jJ]/gu), match => match[1] ?? '')
|
|
578
745
|
.concat(Array.from(raw.matchAll(/\[([^\]]{2,2000})\]\s*TJ/gu), match => match[1] ?? ''));
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 使用 CJS require 解析 npm 包为绝对路径。
|
|
3
|
+
* 多层回退:CJS require.resolve → .pnpm 遍历。
|
|
4
|
+
*/
|
|
5
|
+
export declare function resolvePackage(specifier: string): string;
|
|
6
|
+
/**
|
|
7
|
+
* 解析并加载一个 npm 包。
|
|
8
|
+
* 使用 CJS require() 而非 ESM import(),确保在 Next.js standalone /
|
|
9
|
+
* 打包 Server 上下文中也能正确加载(CJS 走 Module._resolveFilename
|
|
10
|
+
* monkey-patch,不受 ESM 自定义 loader 影响)。
|
|
11
|
+
*/
|
|
12
|
+
export declare function resolveAndImport<T = unknown>(specifier: string): Promise<T>;
|
|
13
|
+
/**
|
|
14
|
+
* 获取 node_modules 根目录路径(包含 .pnpm 的那个)。
|
|
15
|
+
* 用于设置子进程 NODE_PATH,确保 OCR 等子进程能解析依赖。
|
|
16
|
+
*/
|
|
17
|
+
export declare function getNodeModulesRoot(): string | null;
|