@customize-agent/knowledge 1.0.1

package/dist/extraction/external-extractor.d.ts

@@ -0,0 +1,63 @@
+import type { ClassifiedFile, FileCategory } from '../types.js';
+export interface ExternalExtractionResult {
+    text: string;
+    metadata?: Record<string, unknown>;
+    warnings?: string[];
+}
+export interface ExternalExtractor {
+    readonly id: string;
+    readonly name: string;
+    readonly category?: FileCategory;
+    readonly formats?: string[];
+    readonly extensions?: string[];
+    readonly available: boolean;
+    supports(file: ClassifiedFile): boolean;
+    extract(file: ClassifiedFile): ExternalExtractionResult;
+    describe(): ExternalExtractorCapability;
+}
+export interface ExternalExtractorCapability {
+    id: string;
+    name: string;
+    category?: FileCategory;
+    formats?: string[];
+    extensions?: string[];
+    available: boolean;
+    kind: 'command';
+}
+export interface CommandExternalExtractorOptions {
+    id: string;
+    name: string;
+    command: string;
+    args?: string[];
+    category?: FileCategory;
+    formats?: string[];
+    extensions?: string[];
+    timeoutMs?: number;
+}
+export declare class CommandExternalExtractor implements ExternalExtractor {
+    readonly id: string;
+    readonly name: string;
+    readonly category?: FileCategory;
+    readonly formats?: string[];
+    readonly extensions?: string[];
+    readonly available: boolean;
+    private readonly command;
+    private readonly args;
+    private readonly timeoutMs;
+    constructor(options: CommandExternalExtractorOptions);
+    supports(file: ClassifiedFile): boolean;
+    extract(file: ClassifiedFile): ExternalExtractionResult;
+    describe(): ExternalExtractorCapability;
+    private interpolate;
+    private checkAvailable;
+}
+export declare class ExternalExtractorRegistry {
+    private readonly extractors;
+    static fromEnvironment(env?: NodeJS.ProcessEnv): ExternalExtractorRegistry;
+    register(extractor: ExternalExtractor): void;
+    findAll(file: ClassifiedFile): ExternalExtractor[];
+    find(file: ClassifiedFile): ExternalExtractor | undefined;
+    listCapabilities(): ExternalExtractorCapability[];
+    private registerConfiguredOrAuto;
+    private commandExists;
+}

package/dist/extraction/external-extractor.js

@@ -0,0 +1,139 @@
+import { spawnSync } from 'node:child_process';
+import * as path from 'node:path';
+export class CommandExternalExtractor {
+    id;
+    name;
+    category;
+    formats;
+    extensions;
+    available;
+    command;
+    args;
+    timeoutMs;
+    constructor(options) {
+        this.id = options.id;
+        this.name = options.name;
+        this.command = options.command;
+        this.args = options.args ?? ['{file}'];
+        this.category = options.category;
+        this.formats = options.formats;
+        this.extensions = options.extensions?.map(ext => ext.toLowerCase());
+        this.timeoutMs = options.timeoutMs ?? 30_000;
+        this.available = this.checkAvailable();
+    }
+    supports(file) {
+        if (this.category && file.category !== this.category)
+            return false;
+        if (this.formats && !this.formats.includes(file.format))
+            return false;
+        if (this.extensions && !this.extensions.includes(path.extname(file.absolutePath).toLowerCase()))
+            return false;
+        return true;
+    }
+    extract(file) {
+        const args = this.args.map(arg => this.interpolate(arg, file));
+        const result = spawnSync(this.command, args, {
+            encoding: 'utf8',
+            timeout: this.timeoutMs,
+            maxBuffer: 20 * 1024 * 1024,
+        });
+        if (result.error)
+            throw result.error;
+        if (result.status !== 0) {
+            throw new Error(`${this.name} exited with ${result.status}: ${result.stderr}`);
+        }
+        const stdout = result.stdout.trim();
+        if (!stdout)
+            return { text: '', metadata: { externalExtractor: this.id } };
+        try {
+            const parsed = JSON.parse(stdout);
+            return {
+                text: parsed.text ?? stdout,
+                metadata: { externalExtractor: this.id, ...(parsed.metadata ?? {}) },
+                warnings: parsed.warnings,
+            };
+        }
+        catch {
+            return {
+                text: stdout,
+                metadata: { externalExtractor: this.id, externalOutput: 'text' },
+            };
+        }
+    }
+    describe() {
+        return {
+            id: this.id,
+            name: this.name,
+            category: this.category,
+            formats: this.formats,
+            extensions: this.extensions,
+            available: this.available,
+            kind: 'command',
+        };
+    }
+    interpolate(value, file) {
+        return value
+            .replaceAll('{file}', file.absolutePath)
+            .replaceAll('{relativePath}', file.relativePath)
+            .replaceAll('{category}', file.category)
+            .replaceAll('{format}', file.format);
+    }
+    checkAvailable() {
+        const result = spawnSync(this.command, ['--version'], {
+            encoding: 'utf8',
+            timeout: 3_000,
+            stdio: 'ignore',
+        });
+        return !result.error;
+    }
+}
+export class ExternalExtractorRegistry {
+    extractors = [];
+    static fromEnvironment(env = process.env) {
+        const registry = new ExternalExtractorRegistry();
+        registry.registerConfiguredOrAuto('cad-dwg', 'DWG Advanced Parser', env.CUSTOMIZE_AGENT_DWG_PARSER, [{ command: 'dwg-parser' }, { command: 'oda-dwg-parser' }], { category: 'cad', formats: ['autocad'], extensions: ['.dwg', '.dwt'] });
+        registry.registerConfiguredOrAuto('cad-solidworks', 'SolidWorks Advanced Parser', env.CUSTOMIZE_AGENT_SOLIDWORKS_PARSER, [{ command: 'solidworks-parser' }, { command: 'sldworks-parser' }], { category: 'cad', formats: ['solidworks'] });
+        registry.registerConfiguredOrAuto('diagram-visio', 'Visio Advanced Parser', env.CUSTOMIZE_AGENT_VISIO_PARSER, [{ command: 'visio-parser' }, { command: 'vsdx-parser' }], { category: 'diagram', formats: ['visio'] });
+        registry.registerConfiguredOrAuto('ocr-image', 'OCR Image Parser', env.CUSTOMIZE_AGENT_OCR_PARSER, [{ command: 'tesseract', args: ['{file}', 'stdout', '-l', 'chi_sim+eng'] }], { category: 'image', formats: ['raster', 'raw'] });
+        registry.registerConfiguredOrAuto('vision-image', 'Vision Image Parser', env.CUSTOMIZE_AGENT_VISION_PARSER, [{ command: 'vision-parser' }, { command: 'llava-parser' }], { category: 'image', formats: ['raster', 'raw'] });
+        registry.registerConfiguredOrAuto('pdf-advanced', 'PDF Advanced Parser', env.CUSTOMIZE_AGENT_PDF_PARSER, [{ command: 'pdftotext', args: ['{file}', '-'] }, { command: 'pdf-parser' }], { category: 'document', formats: ['pdf'] });
+        registry.registerConfiguredOrAuto('ocr-pdf', 'OCR PDF Parser', env.CUSTOMIZE_AGENT_OCR_PARSER, [{ command: 'ocrmypdf-text' }, { command: 'tesseract', args: ['{file}', 'stdout', '-l', 'chi_sim+eng'] }], { category: 'document', formats: ['pdf'] });
+        registry.registerConfiguredOrAuto('vision-pdf', 'Vision PDF Parser', env.CUSTOMIZE_AGENT_VISION_PARSER, [{ command: 'vision-parser' }, { command: 'llava-parser' }], { category: 'document', formats: ['pdf'] });
+        registry.registerConfiguredOrAuto('office-advanced', 'Office Advanced Parser', env.CUSTOMIZE_AGENT_OFFICE_PARSER, [{ command: 'pandoc', args: ['-t', 'plain', '{file}'] }, { command: 'textutil', args: ['-convert', 'txt', '-stdout', '{file}'] }, { command: 'mammoth', args: ['{file}'] }], { category: 'document', formats: ['office', 'presentation'] });
+        registry.registerConfiguredOrAuto('spreadsheet-advanced', 'Spreadsheet Advanced Parser', env.CUSTOMIZE_AGENT_SPREADSHEET_PARSER, [{ command: 'xlsx2csv' }, { command: 'in2csv' }], { category: 'spreadsheet', formats: ['excel', 'opendoc'] });
+        return registry;
+    }
+    register(extractor) {
+        this.extractors.push(extractor);
+    }
+    findAll(file) {
+        return this.extractors.filter(extractor => extractor.available && extractor.supports(file));
+    }
+    find(file) {
+        return this.findAll(file)[0];
+    }
+    listCapabilities() {
+        return this.extractors.map(extractor => extractor.describe());
+    }
+    registerConfiguredOrAuto(id, name, configuredCommand, candidates, supports) {
+        const configured = configuredCommand ? { command: configuredCommand } : undefined;
+        const candidate = configured
+            ?? candidates.find(item => this.commandExists(typeof item === 'string' ? item : item.command))
+            ?? candidates[0];
+        if (!candidate)
+            return;
+        const command = typeof candidate === 'string' ? candidate : candidate.command;
+        const args = typeof candidate === 'string' ? undefined : candidate.args;
+        this.register(new CommandExternalExtractor({
+            id,
+            name,
+            command,
+            args,
+            ...supports,
+        }));
+    }
+    commandExists(command) {
+        const result = spawnSync(command, ['--version'], { stdio: 'ignore', timeout: 2_000 });
+        return !result.error;
+    }
+}

package/dist/index.d.ts

@@ -0,0 +1,23 @@
+export * from './types.js';
+export * from './constants.js';
+export { TextChunker, type ChunkConfig, type TextChunk } from './chunking/text-chunker.js';
+export { FileClassifier } from './classification/classifier.js';
+export { DedupEngine, type MinHashSignature, type SimilarityMatch } from './dedup/dedup-engine.js';
+export { RelationshipDetector } from './dedup/relationship-detector.js';
+export { HashEmbeddingProvider, type EmbeddingProvider } from './embedding/embedding-provider.js';
+export { ContentExtractor, type ExtractionResult } from './extraction/content-extractor.js';
+export { CommandExternalExtractor, ExternalExtractorRegistry, type CommandExternalExtractorOptions, type ExternalExtractionResult, type ExternalExtractor, type ExternalExtractorCapability } from './extraction/external-extractor.js';
+export { ChangeTracker } from './core/change-tracker.js';
+export { KnowledgeFileScanner, type DiskFileStat } from './core/file-scanner.js';
+export { IndexStateStore, type ChunkSearchResult, type FileHashRecord, type FileRelationship, type StoredChunk } from './core/index-state-store.js';
+export { KnowledgeBaseManager, type KnowledgeBaseManagerOptions } from './core/knowledge-base-manager.js';
+export { MultiProjectManager } from './core/multi-project-manager.js';
+export { computeProjectId } from './core/project-id.js';
+export { ensureProjectCustomizeFile, getProjectConfigPath, getProjectKbPath, ProjectConfigManager } from './core/project-config.js';
+export { ProjectRegistry } from './core/project-registry.js';
+export { ChromaHttpClient, ChromaVectorStore, type ChromaClientOptions } from './vector/chroma-store.js';
+export { CollectionManager, globalCollectionName, projectCollectionName } from './vector/collection-manager.js';
+export type { CollectionClient, VectorCollectionInfo, VectorDocument, VectorSearchQuery, VectorSearchResult, VectorStoreInterface } from './vector/types.js';
+export { VectorIndexer, type VectorIndexResult } from './vector/vector-indexer.js';
+export { FederationSearch, type FederatedQuery, type FederatedResult, type FederatedSearchItem, type SearchFilters, type SearchScope } from './search/federation-search.js';
+export { startKnowledgeDashboard, type DashboardServerHandle, type DashboardServerOptions } from './server/dashboard-server.js';

package/dist/index.js

@@ -0,0 +1,22 @@
+export * from './types.js';
+export * from './constants.js';
+export { TextChunker } from './chunking/text-chunker.js';
+export { FileClassifier } from './classification/classifier.js';
+export { DedupEngine } from './dedup/dedup-engine.js';
+export { RelationshipDetector } from './dedup/relationship-detector.js';
+export { HashEmbeddingProvider } from './embedding/embedding-provider.js';
+export { ContentExtractor } from './extraction/content-extractor.js';
+export { CommandExternalExtractor, ExternalExtractorRegistry } from './extraction/external-extractor.js';
+export { ChangeTracker } from './core/change-tracker.js';
+export { KnowledgeFileScanner } from './core/file-scanner.js';
+export { IndexStateStore } from './core/index-state-store.js';
+export { KnowledgeBaseManager } from './core/knowledge-base-manager.js';
+export { MultiProjectManager } from './core/multi-project-manager.js';
+export { computeProjectId } from './core/project-id.js';
+export { ensureProjectCustomizeFile, getProjectConfigPath, getProjectKbPath, ProjectConfigManager } from './core/project-config.js';
+export { ProjectRegistry } from './core/project-registry.js';
+export { ChromaHttpClient, ChromaVectorStore } from './vector/chroma-store.js';
+export { CollectionManager, globalCollectionName, projectCollectionName } from './vector/collection-manager.js';
+export { VectorIndexer } from './vector/vector-indexer.js';
+export { FederationSearch } from './search/federation-search.js';
+export { startKnowledgeDashboard } from './server/dashboard-server.js';

package/dist/search/federation-search.d.ts

@@ -0,0 +1,41 @@
+import type { VectorStoreInterface } from '../vector/types.js';
+export type SearchScope = 'project' | 'global' | 'all';
+export interface FederatedSearchItem {
+    id: string;
+    content: string;
+    filePath: string;
+    scope: 'project' | 'global';
+    collection: string;
+    score: number;
+    contentHash?: string;
+}
+export interface FederatedResult {
+    results: FederatedSearchItem[];
+    scopesSearched: Array<'project' | 'global'>;
+    queryTimeMs: number;
+}
+export interface SearchFilters {
+    category?: string;
+    filePath?: string;
+}
+export interface FederatedQuery {
+    query: string;
+    queryEmbedding: number[];
+    topK: number;
+    scope: SearchScope;
+    projectId?: string;
+    collections?: string[];
+    filters?: SearchFilters;
+}
+export declare class FederationSearch {
+    private readonly vectorStores;
+    static readonly SCOPE_WEIGHTS: Record<'project' | 'global', number>;
+    constructor(vectorStores?: Map<string, VectorStoreInterface>);
+    search(query: FederatedQuery): Promise<FederatedResult>;
+    merge(results: FederatedSearchItem[], topK: number, scope?: SearchScope): FederatedResult;
+    private resolveCollectionsForQuery;
+    private buildFilter;
+    private toFederatedItem;
+    private resolveScopes;
+    private crossScopeDedup;
+}

package/dist/search/federation-search.js

@@ -0,0 +1,102 @@
+import { ALL_CATEGORIES } from '../constants.js';
+import { globalCollectionName, projectCollectionName } from '../vector/collection-manager.js';
+export class FederationSearch {
+    vectorStores;
+    static SCOPE_WEIGHTS = {
+        project: 1.0,
+        global: 0.7,
+    };
+    constructor(vectorStores = new Map()) {
+        this.vectorStores = vectorStores;
+    }
+    async search(query) {
+        const start = Date.now();
+        const collectionNames = this.resolveCollectionsForQuery(query);
+        const where = this.buildFilter(query.filters);
+        const perCollection = await Promise.all(collectionNames.map(async (collectionName) => {
+            const store = this.vectorStores.get(collectionName);
+            if (!store)
+                return [];
+            const results = await store.search({
+                queryEmbedding: query.queryEmbedding,
+                topK: query.topK * 3,
+                where,
+            });
+            const scope = collectionName.startsWith('proj_') ? 'project' : 'global';
+            return results.map(result => this.toFederatedItem(result, scope));
+        }));
+        const merged = this.merge(perCollection.flat(), query.topK, query.scope);
+        return { ...merged, queryTimeMs: Date.now() - start };
+    }
+    merge(results, topK, scope = 'all') {
+        const start = Date.now();
+        const allowedScopes = this.resolveScopes(scope);
+        const weighted = results
+            .filter(result => allowedScopes.includes(result.scope))
+            .map(result => ({
+            ...result,
+            score: result.score * FederationSearch.SCOPE_WEIGHTS[result.scope],
+        }));
+        const deduped = this.crossScopeDedup(weighted);
+        return {
+            results: deduped.sort((a, b) => b.score - a.score).slice(0, topK),
+            scopesSearched: allowedScopes,
+            queryTimeMs: Date.now() - start,
+        };
+    }
+    resolveCollectionsForQuery(query) {
+        const names = [];
+        if (query.scope === 'project' || query.scope === 'all') {
+            if (!query.projectId)
+                throw new Error('project scope requires projectId');
+            for (const category of ALL_CATEGORIES) {
+                names.push(projectCollectionName(query.projectId, category));
+            }
+        }
+        if (query.scope === 'global' || query.scope === 'all') {
+            for (const category of ALL_CATEGORIES) {
+                names.push(globalCollectionName(category));
+            }
+        }
+        return query.collections ? names.filter(name => query.collections?.includes(name)) : names;
+    }
+    buildFilter(filters) {
+        if (!filters)
+            return undefined;
+        const where = {};
+        if (filters.category)
+            where.category = filters.category;
+        if (filters.filePath)
+            where.file_path = filters.filePath;
+        return Object.keys(where).length > 0 ? where : undefined;
+    }
+    toFederatedItem(result, scope) {
+        return {
+            id: result.document.id,
+            content: result.document.content,
+            filePath: String(result.document.metadata.file_path ?? ''),
+            scope,
+            collection: result.collection,
+            score: result.score,
+            contentHash: typeof result.document.metadata.content_hash === 'string' ? result.document.metadata.content_hash : undefined,
+        };
+    }
+    resolveScopes(scope) {
+        if (scope === 'project')
+            return ['project'];
+        if (scope === 'global')
+            return ['global'];
+        return ['project', 'global'];
+    }
+    crossScopeDedup(results) {
+        const byKey = new Map();
+        for (const result of results) {
+            const key = result.contentHash ?? result.filePath;
+            const existing = byKey.get(key);
+            if (!existing || (existing.scope === 'global' && result.scope === 'project')) {
+                byKey.set(key, result);
+            }
+        }
+        return [...byKey.values()];
+    }
+}

package/dist/server/dashboard-client.d.ts

@@ -0,0 +1,2 @@
+import type { DashboardMessages } from './dashboard-i18n.js';
+export declare function renderDashboardClient(messages: DashboardMessages): string;