@customize-agent/knowledge 1.0.1 → 2.1.0

package/dist/extraction/module-resolver.d.ts

@@ -0,0 +1,17 @@
+/**
+ * 使用 CJS require 解析 npm 包为绝对路径。
+ * 多层回退：CJS require.resolve → .pnpm 遍历。
+ */
+export declare function resolvePackage(specifier: string): string;
+/**
+ * 解析并加载一个 npm 包。
+ * 使用 CJS require() 而非 ESM import()，确保在 Next.js standalone /
+ * 打包 Server 上下文中也能正确加载（CJS 走 Module._resolveFilename
+ * monkey-patch，不受 ESM 自定义 loader 影响）。
+ */
+export declare function resolveAndImport<T = unknown>(specifier: string): Promise<T>;
+/**
+ * 获取 node_modules 根目录路径（包含 .pnpm 的那个）。
+ * 用于设置子进程 NODE_PATH，确保 OCR 等子进程能解析依赖。
+ */
+export declare function getNodeModulesRoot(): string | null;

package/dist/extraction/module-resolver.js

@@ -0,0 +1,113 @@
+/**
+ * 模块解析工具。
+ *
+ * Next.js standalone / 打包 Server 中，ESM 动态 import(绝对路径) 被自定义
+ * loader 拦截后可能失败。但 CJS require() 走 Module._resolveFilename
+ * monkey-patch，能正确找到 pnpm store 和 vendor 目录中的包。
+ *
+ * 此模块统一使用 CJS require 加载依赖，确保在所有上下文中可靠工作。
+ */
+import { createRequire } from 'node:module';
+import { fileURLToPath } from 'node:url';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+/** 从 knowledge 包自身目录解析的 CJS require 函数 */
+const localRequire = createRequire(import.meta.url);
+/** knowledge 包所在的实际目录 */
+const knowledgeDir = path.dirname(fileURLToPath(import.meta.url));
+/**
+ * 在 .pnpm 目录中查找包的实际路径（处理 standalone 缺少顶层 symlink 的情况）。
+ */
+function findInPnpm(packageName) {
+    let dir = knowledgeDir;
+    for (let i = 0; i < 10; i++) {
+        const nodeModules = path.join(dir, 'node_modules');
+        const pnpmDir = path.join(nodeModules, '.pnpm');
+        if (fs.existsSync(pnpmDir)) {
+            const pkgParts = packageName.split('/');
+            const flatName = pkgParts.length > 1
+                ? `${pkgParts[0]}+${pkgParts.slice(1).join('/')}`
+                : packageName;
+            try {
+                for (const entry of fs.readdirSync(pnpmDir)) {
+                    if (entry.startsWith(flatName + '@')) {
+                        const pkgDir = path.join(pnpmDir, entry, 'node_modules', packageName);
+                        if (fs.existsSync(pkgDir))
+                            return pkgDir;
+                    }
+                }
+            }
+            catch { /* continue upward */ }
+        }
+        const parent = path.dirname(dir);
+        if (parent === dir)
+            break;
+        dir = parent;
+    }
+    return null;
+}
+/**
+ * 使用 CJS require 解析 npm 包为绝对路径。
+ * 多层回退：CJS require.resolve → .pnpm 遍历。
+ */
+export function resolvePackage(specifier) {
+    try {
+        return localRequire.resolve(specifier);
+    }
+    catch { /* fall through */ }
+    const parts = specifier.split('/');
+    let packageName;
+    if (parts[0]?.startsWith('@')) {
+        packageName = `${parts[0]}/${parts[1]}`;
+    }
+    else {
+        packageName = parts[0] ?? specifier;
+    }
+    const pnpmRoot = findInPnpm(packageName);
+    if (pnpmRoot) {
+        const subPath = packageName === specifier ? '' : specifier.slice(packageName.length + 1);
+        const fullPath = subPath ? path.join(pnpmRoot, subPath) : pnpmRoot;
+        if (fs.existsSync(fullPath))
+            return fullPath;
+    }
+    throw new Error(`Cannot resolve package: ${specifier}`);
+}
+/**
+ * 解析并加载一个 npm 包。
+ * 使用 CJS require() 而非 ESM import()，确保在 Next.js standalone /
+ * 打包 Server 上下文中也能正确加载（CJS 走 Module._resolveFilename
+ * monkey-patch，不受 ESM 自定义 loader 影响）。
+ */
+export async function resolveAndImport(specifier) {
+    const resolvedPath = resolvePackage(specifier);
+    // CJS require 能穿透 pnpm store 和 Next.js bundle，兼容性最好
+    try {
+        return localRequire(resolvedPath);
+    }
+    catch {
+        // 某些包可能是纯 ESM（如 pdfjs-dist v5），回退到 import()
+        return import(resolvedPath);
+    }
+}
+/**
+ * 获取 node_modules 根目录路径（包含 .pnpm 的那个）。
+ * 用于设置子进程 NODE_PATH，确保 OCR 等子进程能解析依赖。
+ */
+export function getNodeModulesRoot() {
+    let dir = path.dirname(knowledgeDir);
+    for (let i = 0; i < 10; i++) {
+        // 开发环境: node_modules/.pnpm
+        const nm = path.join(dir, 'node_modules');
+        if (fs.existsSync(path.join(nm, '.pnpm')))
+            return nm;
+        // 打包环境: vendor/.pnpm（bundle-server 把 node_modules 改名 vendor）
+        const vendor = path.join(dir, 'vendor');
+        if (fs.existsSync(path.join(vendor, '.pnpm')))
+            return vendor;
+        const parent = path.dirname(dir);
+        if (parent === dir)
+            break;
+        dir = parent;
+    }
+    return null;
+}

package/dist/index.d.ts

@@ -1,5 +1,3 @@
-export * from './types.js';
-export * from './constants.js';
 export { TextChunker, type ChunkConfig, type TextChunk } from './chunking/text-chunker.js';
 export { FileClassifier } from './classification/classifier.js';
 export { DedupEngine, type MinHashSignature, type SimilarityMatch } from './dedup/dedup-engine.js';
@@ -10,7 +8,7 @@ export { CommandExternalExtractor, ExternalExtractorRegistry, type CommandExtern
 export { ChangeTracker } from './core/change-tracker.js';
 export { KnowledgeFileScanner, type DiskFileStat } from './core/file-scanner.js';
 export { IndexStateStore, type ChunkSearchResult, type FileHashRecord, type FileRelationship, type StoredChunk } from './core/index-state-store.js';
-export { KnowledgeBaseManager, type KnowledgeBaseManagerOptions } from './core/knowledge-base-manager.js';
+export { KnowledgeBaseManager, type KnowledgeBaseManagerOptions, type KnowledgeIndexProgress } from './core/knowledge-base-manager.js';
 export { MultiProjectManager } from './core/multi-project-manager.js';
 export { computeProjectId } from './core/project-id.js';
 export { ensureProjectCustomizeFile, getProjectConfigPath, getProjectKbPath, ProjectConfigManager } from './core/project-config.js';
@@ -20,4 +18,4 @@ export { CollectionManager, globalCollectionName, projectCollectionName } from '
 export type { CollectionClient, VectorCollectionInfo, VectorDocument, VectorSearchQuery, VectorSearchResult, VectorStoreInterface } from './vector/types.js';
 export { VectorIndexer, type VectorIndexResult } from './vector/vector-indexer.js';
 export { FederationSearch, type FederatedQuery, type FederatedResult, type FederatedSearchItem, type SearchFilters, type SearchScope } from './search/federation-search.js';
-export { startKnowledgeDashboard, type DashboardServerHandle, type DashboardServerOptions } from './server/dashboard-server.js';
+export type { LLMChatMessage, LLMChatOptions, LLMChatResponse, LLMSearchProvider } from './llm/llm-search-provider.js';

package/dist/index.js

@@ -1,5 +1,5 @@
-export * from './types.js';
-export * from './constants.js';
+// Types and constants from types.ts and constants.ts are not exported
+// as they are only used internally within the knowledge package.
 export { TextChunker } from './chunking/text-chunker.js';
 export { FileClassifier } from './classification/classifier.js';
 export { DedupEngine } from './dedup/dedup-engine.js';
@@ -19,4 +19,3 @@ export { ChromaHttpClient, ChromaVectorStore } from './vector/chroma-store.js';
 export { CollectionManager, globalCollectionName, projectCollectionName } from './vector/collection-manager.js';
 export { VectorIndexer } from './vector/vector-indexer.js';
 export { FederationSearch } from './search/federation-search.js';
-export { startKnowledgeDashboard } from './server/dashboard-server.js';

package/dist/llm/llm-search-provider.d.ts

@@ -0,0 +1,23 @@
+/**
+ * 轻量级 LLM 搜索 Provider 接口。
+ *
+ * 定义在 knowledge 包内部，避免直接依赖 @customize-agent/llm。
+ * CLI 层的 ILLMProvider 在结构上兼容此接口，可直接传入。
+ */
+export interface LLMChatMessage {
+    role: 'system' | 'user' | 'assistant';
+    content: string;
+}
+export interface LLMChatOptions {
+    temperature?: number;
+    maxTokens?: number;
+}
+export interface LLMChatResponse {
+    content: string;
+}
+/**
+ * LLM 搜索 Provider —— 用于查询扩展和语义重排序。
+ */
+export interface LLMSearchProvider {
+    chat(messages: LLMChatMessage[], options?: LLMChatOptions): Promise<LLMChatResponse>;
+}

package/dist/llm/llm-search-provider.js

@@ -0,0 +1 @@
+export {};

package/dist/search/federation-search.d.ts

@@ -8,16 +8,45 @@ export interface FederatedSearchItem {
     collection: string;
     score: number;
     contentHash?: string;
+    chunkIndex?: number;
+    parentId?: string;
+    source?: 'keyword' | 'vector' | 'hybrid';
+    sectionTitle?: string;
+    rowRange?: string;
+    chunkKind?: string;
+    scoreDetails?: {
+        keywordScore?: number;
+        bm25Score?: number;
+        vectorScore?: number;
+        hybridScore?: number;
+        exactPhraseBoost?: number;
+        rerankBoost?: number;
+        llmRelevanceScore?: number;
+    };
+    facets?: Record<string, string | number | string[]>;
 }
 export interface FederatedResult {
     results: FederatedSearchItem[];
     scopesSearched: Array<'project' | 'global'>;
     queryTimeMs: number;
+    debug?: {
+        originalQuery?: string;
+        rewrittenQueries?: string[];
+        weights?: Record<string, number>;
+        recallCounts?: Record<string, number>;
+        reranker?: string;
+    };
 }
 export interface SearchFilters {
     category?: string;
     filePath?: string;
 }
+export interface RetrievalWeights {
+    keyword?: number;
+    vector?: number;
+    rewrite?: number;
+    hybridBonus?: number;
+}
 export interface FederatedQuery {
     query: string;
     queryEmbedding: number[];

package/dist/search/federation-search.js

@@ -79,6 +79,13 @@ export class FederationSearch {
             collection: result.collection,
             score: result.score,
             contentHash: typeof result.document.metadata.content_hash === 'string' ? result.document.metadata.content_hash : undefined,
+            chunkIndex: typeof result.document.metadata.chunk_index === 'number' ? result.document.metadata.chunk_index : undefined,
+            parentId: typeof result.document.metadata.parent_id === 'string' ? result.document.metadata.parent_id : undefined,
+            source: 'vector',
+            sectionTitle: typeof result.document.metadata.section_title === 'string' ? result.document.metadata.section_title : undefined,
+            rowRange: typeof result.document.metadata.row_range === 'string' ? result.document.metadata.row_range : undefined,
+            chunkKind: typeof result.document.metadata.chunk_kind === 'string' ? result.document.metadata.chunk_kind : undefined,
+            scoreDetails: { vectorScore: result.score },
         };
     }
     resolveScopes(scope) {
@@ -91,7 +98,7 @@ export class FederationSearch {
     crossScopeDedup(results) {
         const byKey = new Map();
         for (const result of results) {
-            const key = result.contentHash ?? result.filePath;
+            const key = result.contentHash ?? `${result.filePath}#${result.parentId ?? result.chunkIndex ?? result.id}`;
             const existing = byKey.get(key);
             if (!existing || (existing.scope === 'global' && result.scope === 'project')) {
                 byKey.set(key, result);

package/dist/vector/chroma-store.d.ts

@@ -14,6 +14,7 @@ export declare class ChromaHttpClient implements CollectionClient {
     readonly baseUrl: string;
     readonly tenant: string;
     readonly database: string;
+    private readonly collectionIds;
     constructor(options?: ChromaClientOptions);
     heartbeat(): Promise<boolean>;
     getOrCreateCollection(name: string, metadata?: Record<string, unknown>): Promise<VectorCollectionInfo>;
@@ -22,6 +23,7 @@ export declare class ChromaHttpClient implements CollectionClient {
     upsert(collectionName: string, documents: VectorDocument[]): Promise<void>;
     deleteWhere(collectionName: string, where: Record<string, string | number | boolean>): Promise<void>;
     query(collectionName: string, query: VectorSearchQuery): Promise<ChromaQueryResponse>;
+    private getCollectionId;
     private collectionsPath;
     private request;
     private toCollectionInfo;

package/dist/vector/chroma-store.js

@@ -2,14 +2,15 @@ export class ChromaHttpClient {
     baseUrl;
     tenant;
     database;
+    collectionIds = new Map();
     constructor(options = {}) {
-        this.baseUrl = options.baseUrl ?? 'http://localhost:8000';
+        this.baseUrl = options.baseUrl ?? process.env.CHROMA_URL ?? process.env.CHROMA_BASE_URL ?? 'http://localhost:17322';
         this.tenant = options.tenant ?? 'default_tenant';
         this.database = options.database ?? 'default_database';
     }
     async heartbeat() {
         try {
-            await this.request('/api/v1/heartbeat');
+            await this.request('/api/v2/heartbeat');
             return true;
         }
         catch {
@@ -17,24 +18,33 @@ export class ChromaHttpClient {
         }
     }
     async getOrCreateCollection(name, metadata = {}) {
+        const body = { name, get_or_create: true };
+        if (Object.keys(metadata).length > 0)
+            body.metadata = metadata;
         const response = await this.request(this.collectionsPath(), {
             method: 'POST',
-            body: JSON.stringify({ name, metadata, get_or_create: true }),
-        });
+            body: JSON.stringify(body),
+        }, 10000);
+        if (response.id)
+            this.collectionIds.set(name, response.id);
         return this.toCollectionInfo(response);
     }
     async listCollections() {
-        const response = await this.request(this.collectionsPath());
+        const response = await this.request(this.collectionsPath(), {}, 10000);
+        for (const collection of response)
+            if (collection.id)
+                this.collectionIds.set(collection.name, collection.id);
         return response.map(collection => this.toCollectionInfo(collection));
     }
     async deleteCollection(name) {
         await this.request(`${this.collectionsPath()}/${encodeURIComponent(name)}`, { method: 'DELETE' });
+        this.collectionIds.delete(name);
     }
     async upsert(collectionName, documents) {
         if (documents.length === 0)
             return;
-        await this.getOrCreateCollection(collectionName);
-        await this.request(`${this.collectionsPath()}/${encodeURIComponent(collectionName)}/upsert`, {
+        const collectionId = await this.getCollectionId(collectionName);
+        await this.request(`${this.collectionsPath()}/${encodeURIComponent(collectionId)}/upsert`, {
             method: 'POST',
             body: JSON.stringify({
                 ids: documents.map(document => document.id),
@@ -42,16 +52,18 @@ export class ChromaHttpClient {
                 documents: documents.map(document => document.content),
                 metadatas: documents.map(document => document.metadata),
             }),
-        });
+        }, 30000);
     }
     async deleteWhere(collectionName, where) {
-        await this.request(`${this.collectionsPath()}/${encodeURIComponent(collectionName)}/delete`, {
+        const collectionId = await this.getCollectionId(collectionName);
+        await this.request(`${this.collectionsPath()}/${encodeURIComponent(collectionId)}/delete`, {
             method: 'POST',
             body: JSON.stringify({ where }),
-        });
+        }, 10000);
     }
     async query(collectionName, query) {
-        return this.request(`${this.collectionsPath()}/${encodeURIComponent(collectionName)}/query`, {
+        const collectionId = await this.getCollectionId(collectionName);
+        return this.request(`${this.collectionsPath()}/${encodeURIComponent(collectionId)}/query`, {
             method: 'POST',
             body: JSON.stringify({
                 query_embeddings: [query.queryEmbedding],
@@ -59,19 +71,38 @@ export class ChromaHttpClient {
                 where: query.where,
                 include: ['documents', 'metadatas', 'distances'],
             }),
-        });
+        }, 10000);
+    }
+    async getCollectionId(name) {
+        const cached = this.collectionIds.get(name);
+        if (cached)
+            return cached;
+        const collection = await this.getOrCreateCollection(name);
+        if (!collection.id)
+            throw new Error(`ChromaDB collection has no id: ${name}`);
+        this.collectionIds.set(name, collection.id);
+        return collection.id;
     }
     collectionsPath() {
-        return `/api/v1/tenants/${encodeURIComponent(this.tenant)}/databases/${encodeURIComponent(this.database)}/collections`;
-    }
-    async request(path, init = {}) {
-        const response = await fetch(`${this.baseUrl}${path}`, {
-            ...init,
-            headers: {
-                'content-type': 'application/json',
-                ...init.headers,
-            },
-        });
+        return `/api/v2/tenants/${encodeURIComponent(this.tenant)}/databases/${encodeURIComponent(this.database)}/collections`;
+    }
+    async request(path, init = {}, timeoutMs = 3000) {
+        const controller = new AbortController();
+        const timeout = setTimeout(() => controller.abort(), timeoutMs);
+        let response;
+        try {
+            response = await fetch(`${this.baseUrl}${path}`, {
+                ...init,
+                signal: controller.signal,
+                headers: {
+                    'content-type': 'application/json',
+                    ...(init.headers ?? {}),
+                },
+            });
+        }
+        finally {
+            clearTimeout(timeout);
+        }
         if (!response.ok) {
             throw new Error(`ChromaDB request failed: ${response.status} ${response.statusText}`);
         }

package/dist/vector/vector-indexer.d.ts

@@ -15,4 +15,7 @@ export declare class VectorIndexer {
     deleteFile(collectionName: string, filePath: string): Promise<void>;
     private groupByCollection;
     private toVectorDocument;
+    private parseMetadata;
+    private metadataString;
+    private metadataNumber;
 }

package/dist/vector/vector-indexer.js

@@ -44,6 +44,7 @@ export class VectorIndexer {
         return grouped;
     }
     toVectorDocument(chunk, embedding) {
+        const chunkMetadata = this.parseMetadata(chunk.metadataJson);
         return {
             id: chunk.id,
             content: chunk.content,
@@ -55,7 +56,29 @@ export class VectorIndexer {
                 format: chunk.format,
                 token_count: chunk.tokenCount,
                 section_title: chunk.sectionTitle ?? null,
+                parent_id: this.metadataString(chunkMetadata.parentId),
+                parent_index: this.metadataNumber(chunkMetadata.parentIndex),
+                child_index: this.metadataNumber(chunkMetadata.childIndex),
+                chunk_kind: this.metadataString(chunkMetadata.chunkKind),
+                row_range: this.metadataString(chunkMetadata.rowRange),
+                split_strategy: this.metadataString(chunkMetadata.splitStrategy),
             },
         };
     }
+    parseMetadata(metadataJson) {
+        if (!metadataJson)
+            return {};
+        try {
+            return JSON.parse(metadataJson);
+        }
+        catch {
+            return {};
+        }
+    }
+    metadataString(value) {
+        return typeof value === 'string' ? value : null;
+    }
+    metadataNumber(value) {
+        return typeof value === 'number' ? value : null;
+    }
 }

package/package.json

@@ -1,10 +1,16 @@
 {
   "name": "@customize-agent/knowledge",
-  "version": "1.0.1",
+  "version": "2.1.0",
   "description": "Local knowledge base infrastructure for customize-agent",
   "type": "module",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
   "license": "MIT",
   "author": "Pan-jijian",
   "repository": {
@@ -28,9 +34,11 @@
     "better-sqlite3": "^12.10.0",
     "fast-glob": "^3.3.3",
     "jszip": "^3.10.1",
-    "mammoth": "^1.11.0",
+    "@napi-rs/canvas": "^0.1.82",
+    "mammoth": "^1.12.0",
     "pdf-parse": "^2.4.5",
-    "tesseract.js": "^6.0.1",
+    "pdfjs-dist": "^5.4.394",
+    "tesseract.js": "^7.0.0",
     "xlsx": "^0.18.5"
   },
   "devDependencies": {

package/dist/server/dashboard-client.d.ts

@@ -1,2 +0,0 @@
-import type { DashboardMessages } from './dashboard-i18n.js';
-export declare function renderDashboardClient(messages: DashboardMessages): string;