@matperez/coderag 0.1.24

package/dist/storage-persistent-pg.d.ts

@@ -0,0 +1,75 @@
+/**
+ * Persistent storage implementation using PostgreSQL + Drizzle ORM (pg Pool)
+ */
+import type { CodebaseFile } from './storage.js';
+import type { ChunkData, PersistentChunkStorage, PostgresDbConfig, SearchByTermsResultItem, StoredChunk } from './storage-persistent-types.js';
+export declare class PostgresPersistentStorage implements PersistentChunkStorage {
+    private dbInstance;
+    private initPromise;
+    constructor(config: PostgresDbConfig);
+    getBackend(): 'postgres';
+    private initialize;
+    private ensureInit;
+    private getOne;
+    storeFile(file: CodebaseFile): Promise<void>;
+    storeFiles(files: CodebaseFile[]): Promise<void>;
+    getFile(path: string): Promise<CodebaseFile | null>;
+    getAllFiles(): Promise<CodebaseFile[]>;
+    deleteFile(path: string): Promise<void>;
+    clear(): Promise<void>;
+    storeChunks(filePath: string, chunks: ChunkData[]): Promise<number[]>;
+    storeManyChunks(fileChunks: Array<{
+        filePath: string;
+        chunks: ChunkData[];
+    }>): Promise<Map<string, number[]>>;
+    getChunksForFile(filePath: string): Promise<StoredChunk[]>;
+    getChunkCount(): Promise<number>;
+    count(): Promise<number>;
+    exists(path: string): Promise<boolean>;
+    storeChunkVectors(chunkId: number, terms: Map<string, {
+        tf: number;
+        tfidf: number;
+        rawFreq: number;
+    }>, tokenCount?: number): Promise<void>;
+    storeManyChunkVectors(chunkVectors: Array<{
+        chunkId: number;
+        terms: Map<string, {
+            tf: number;
+            tfidf: number;
+            rawFreq: number;
+        }>;
+        tokenCount?: number;
+    }>): Promise<void>;
+    storeIdfScores(idf: Map<string, number>, docFreq: Map<string, number>): Promise<void>;
+    getIdfScores(): Promise<Map<string, number>>;
+    getChunkVectors(chunkId: number): Promise<Map<string, {
+        tf: number;
+        tfidf: number;
+        rawFreq: number;
+    }> | null>;
+    getIdfScoresForTerms(terms: string[]): Promise<Map<string, number>>;
+    getTotalDocuments(): Promise<number>;
+    getAllFileMetadata(): Promise<Map<string, {
+        mtime: number;
+        hash: string;
+    }>>;
+    deleteFiles(paths: string[]): Promise<void>;
+    setMetadata(key: string, value: string): Promise<void>;
+    getMetadata(key: string): Promise<string | null>;
+    getAverageDocLength(): Promise<number>;
+    updateAverageDocLength(): Promise<number>;
+    rebuildIdfScoresFromVectors(): Promise<void>;
+    recalculateTfidfScores(): Promise<void>;
+    updateChunkMagnitudes(): Promise<void>;
+    getTermsForFiles(paths: string[]): Promise<Set<string>>;
+    searchByTerms(queryTerms: string[], options?: {
+        limit?: number;
+    }): Promise<SearchByTermsResultItem[]>;
+    getAllChunks(): Promise<StoredChunk[]>;
+    close(): void;
+}
+/**
+ * Create Postgres-backed persistent storage (used by createPersistentStorage when backend === 'postgres').
+ */
+export declare function createPostgresPersistentStorage(config: PostgresDbConfig): PersistentChunkStorage;
+//# sourceMappingURL=storage-persistent-pg.d.ts.map

package/dist/storage-persistent-pg.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"storage-persistent-pg.d.ts","sourceRoot":"","sources":["../src/storage-persistent-pg.ts"],"names":[],"mappings":"AAAA;;GAEG;AAKH,OAAO,KAAK,EAAE,YAAY,EAAW,MAAM,cAAc,CAAA;AACzD,OAAO,KAAK,EACX,SAAS,EACT,sBAAsB,EACtB,gBAAgB,EAChB,uBAAuB,EACvB,WAAW,EACX,MAAM,+BAA+B,CAAA;AAEtC,qBAAa,yBAA0B,YAAW,sBAAsB;IACvE,OAAO,CAAC,UAAU,CAAe;IACjC,OAAO,CAAC,WAAW,CAAe;gBAEtB,MAAM,EAAE,gBAAgB;IAIpC,UAAU,IAAI,UAAU;YAIV,UAAU;YAIV,UAAU;YAIV,MAAM;IAKd,SAAS,CAAC,IAAI,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IA6B5C,UAAU,CAAC,KAAK,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IA+BhD,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;IAiBnD,WAAW,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;IActC,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAMvC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAUtB,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IA2BrE,eAAe,CACpB,UAAU,EAAE,KAAK,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,SAAS,EAAE,CAAA;KAAE,CAAC,GAC1D,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IAwC3B,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IA6B1D,aAAa,IAAI,OAAO,CAAC,MAAM,CAAC;IAOhC,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC;IAOxB,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAStC,iBAAiB,CACtB,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,EAClE,UAAU,CAAC,EAAE,MAAM,GACjB,OAAO,CAAC,IAAI,CAAC;IAwBV,qBAAqB,CAC1B,YAAY,EAAE,KAAK,CAAC;QACnB,OAAO,EAAE,MAAM,CAAA;QACf,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE;YAAE,EAAE,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAC;YAAC,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;QAClE,UAAU,CAAC,EAAE,MAAM,CAAA;KACnB,CAAC,GACA,OAAO,CAAC,IAAI,CAAC;IAmCV,cAAc,CAAC,GAAG,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAgBrF,YAAY,IAAI,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAS5C,eAAe,CACpB,OAAO,EAAE,MAAM,GACb,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,GAAG,IAAI,CAAC;IAexE,oBAAoB,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAUnE,iBAAiB,IAAI,OAAO,CAAC,MAAM,CAAC;IAIpC,kBAAkB,IAAI,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAa3E,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAW3C,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAYtD,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAShD,mBAAmB,IAAI,OAAO,CAAC,MAAM,CAAC;IAetC,sBAAsB,IAAI,OAAO,CAAC,MAAM,CAAC;IAazC,2BAA2B,IAAI,OAAO,CAAC,IAAI,CAAC;IA4B5C,sBAAsB,IAAI,OAAO,CAAC,IAAI,CAAC;IAWvC,qBAAqB,IAAI,OAAO,CAAC,IAAI,CAAC;IAWtC,gBAAgB,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAgBvD,aAAa,CAClB,UAAU,EAAE,MAAM,EAAE,EACpB,OAAO,GAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAO,GAC9B,OAAO,CAAC,uBAAuB,EAAE,CAAC;IAgF/B,YAAY,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;IA4B5C,KAAK,IAAI,IAAI;CAGb;AAED;;GAEG;AACH,wBAAgB,+BAA+B,CAAC,MAAM,EAAE,gBAAgB,GAAG,sBAAsB,CAEhG"}

package/dist/storage-persistent-pg.js

@@ -0,0 +1,579 @@
+/**
+ * Persistent storage implementation using PostgreSQL + Drizzle ORM (pg Pool)
+ */
+import { desc, eq, inArray, sql } from 'drizzle-orm';
+import { createPostgresDb } from './db/client-pg.js';
+import * as schema from './db/schema-pg.js';
+export class PostgresPersistentStorage {
+    dbInstance;
+    initPromise;
+    constructor(config) {
+        this.initPromise = this.initialize(config);
+    }
+    getBackend() {
+        return 'postgres';
+    }
+    async initialize(config) {
+        this.dbInstance = await createPostgresDb(config);
+    }
+    async ensureInit() {
+        await this.initPromise;
+    }
+    async getOne(promise) {
+        const rows = await promise;
+        return rows[0];
+    }
+    async storeFile(file) {
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        const mtime = typeof file.mtime === 'number' ? file.mtime : file.mtime.getTime();
+        const values = {
+            path: file.path,
+            content: file.content,
+            hash: file.hash,
+            size: file.size,
+            mtime,
+            ...(file.language ? { language: file.language } : {}),
+            indexedAt: Date.now(),
+        };
+        await db
+            .insert(schema.files)
+            .values(values)
+            .onConflictDoUpdate({
+            target: schema.files.path,
+            set: {
+                content: values.content,
+                hash: values.hash,
+                size: values.size,
+                mtime: values.mtime,
+                indexedAt: values.indexedAt,
+                ...(values.language != null ? { language: values.language } : {}),
+            },
+        });
+    }
+    async storeFiles(files) {
+        if (files.length === 0)
+            return;
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        for (const file of files) {
+            const mtime = typeof file.mtime === 'number' ? file.mtime : file.mtime.getTime();
+            await db
+                .insert(schema.files)
+                .values({
+                path: file.path,
+                content: file.content,
+                hash: file.hash,
+                size: file.size,
+                mtime,
+                indexedAt: Date.now(),
+                ...(file.language != null ? { language: file.language } : {}),
+            })
+                .onConflictDoUpdate({
+                target: schema.files.path,
+                set: {
+                    content: file.content,
+                    hash: file.hash,
+                    size: file.size,
+                    mtime,
+                    indexedAt: Date.now(),
+                    ...(file.language != null ? { language: file.language } : {}),
+                },
+            });
+        }
+    }
+    async getFile(path) {
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        const result = await this.getOne(db.select().from(schema.files).where(eq(schema.files.path, path)).limit(1));
+        if (!result)
+            return null;
+        return {
+            path: result.path,
+            content: result.content,
+            hash: result.hash,
+            size: result.size,
+            mtime: result.mtime,
+            language: result.language ?? undefined,
+        };
+    }
+    async getAllFiles() {
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        const results = await db.select().from(schema.files);
+        return results.map((file) => ({
+            path: file.path,
+            content: file.content,
+            hash: file.hash,
+            size: file.size,
+            mtime: file.mtime,
+            language: file.language ?? undefined,
+        }));
+    }
+    async deleteFile(path) {
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        await db.delete(schema.files).where(eq(schema.files.path, path));
+    }
+    async clear() {
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        await db.delete(schema.chunks);
+        await db.delete(schema.documentVectors);
+        await db.delete(schema.files);
+        await db.delete(schema.idfScores);
+        await db.delete(schema.indexMetadata);
+    }
+    async storeChunks(filePath, chunks) {
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        const file = await this.getOne(db.select().from(schema.files).where(eq(schema.files.path, filePath)).limit(1));
+        if (!file)
+            throw new Error(`File not found: ${filePath}`);
+        await db.delete(schema.chunks).where(eq(schema.chunks.fileId, file.id));
+        const chunkIds = [];
+        for (const chunk of chunks) {
+            const insertValues = {
+                fileId: file.id,
+                content: chunk.content,
+                type: chunk.type,
+                startLine: chunk.startLine,
+                endLine: chunk.endLine,
+            };
+            if (chunk.metadata)
+                insertValues.metadata = JSON.stringify(chunk.metadata);
+            const insertResult = await db
+                .insert(schema.chunks)
+                .values(insertValues)
+                .returning({ id: schema.chunks.id });
+            if (insertResult[0])
+                chunkIds.push(insertResult[0].id);
+        }
+        return chunkIds;
+    }
+    async storeManyChunks(fileChunks) {
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        const result = new Map();
+        if (fileChunks.length === 0)
+            return result;
+        const filePaths = fileChunks.map((fc) => fc.filePath);
+        const files = await db
+            .select({ id: schema.files.id, path: schema.files.path })
+            .from(schema.files)
+            .where(inArray(schema.files.path, filePaths));
+        const fileIdMap = new Map();
+        for (const file of files)
+            fileIdMap.set(file.path, file.id);
+        const fileIds = Array.from(fileIdMap.values());
+        if (fileIds.length > 0) {
+            await db.delete(schema.chunks).where(inArray(schema.chunks.fileId, fileIds));
+        }
+        for (const fc of fileChunks) {
+            const fileId = fileIdMap.get(fc.filePath);
+            if (!fileId)
+                continue;
+            const chunkIds = [];
+            for (const chunk of fc.chunks) {
+                const insertValues = {
+                    fileId,
+                    content: chunk.content,
+                    type: chunk.type,
+                    startLine: chunk.startLine,
+                    endLine: chunk.endLine,
+                };
+                if (chunk.metadata)
+                    insertValues.metadata = JSON.stringify(chunk.metadata);
+                const insertResult = await db
+                    .insert(schema.chunks)
+                    .values(insertValues)
+                    .returning({ id: schema.chunks.id });
+                if (insertResult[0])
+                    chunkIds.push(insertResult[0].id);
+            }
+            result.set(fc.filePath, chunkIds);
+        }
+        return result;
+    }
+    async getChunksForFile(filePath) {
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        const results = await db
+            .select({
+            id: schema.chunks.id,
+            fileId: schema.chunks.fileId,
+            content: schema.chunks.content,
+            type: schema.chunks.type,
+            startLine: schema.chunks.startLine,
+            endLine: schema.chunks.endLine,
+            metadata: schema.chunks.metadata,
+            filePath: schema.files.path,
+        })
+            .from(schema.chunks)
+            .innerJoin(schema.files, eq(schema.chunks.fileId, schema.files.id))
+            .where(eq(schema.files.path, filePath));
+        return results.map((r) => ({
+            id: r.id,
+            fileId: r.fileId,
+            filePath: r.filePath,
+            content: r.content,
+            type: r.type,
+            startLine: r.startLine,
+            endLine: r.endLine,
+            metadata: r.metadata ? JSON.parse(r.metadata) : undefined,
+        }));
+    }
+    async getChunkCount() {
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        const rows = await db.select({ count: sql `count(*)::int` }).from(schema.chunks);
+        return Number(rows[0]?.count ?? 0);
+    }
+    async count() {
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        const rows = await db.select({ count: sql `count(*)::int` }).from(schema.files);
+        return Number(rows[0]?.count ?? 0);
+    }
+    async exists(path) {
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        const result = await this.getOne(db.select().from(schema.files).where(eq(schema.files.path, path)).limit(1));
+        return result !== undefined;
+    }
+    async storeChunkVectors(chunkId, terms, tokenCount) {
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        await db.delete(schema.documentVectors).where(eq(schema.documentVectors.chunkId, chunkId));
+        if (tokenCount !== undefined) {
+            await this.dbInstance.pool.query('UPDATE chunks SET token_count = $1 WHERE id = $2', [
+                tokenCount,
+                chunkId,
+            ]);
+        }
+        const BATCH_SIZE = 500;
+        const vectors = Array.from(terms.entries()).map(([term, scores]) => ({
+            chunkId,
+            term,
+            tf: scores.tf,
+            tfidf: scores.tfidf,
+            rawFreq: scores.rawFreq,
+        }));
+        for (let i = 0; i < vectors.length; i += BATCH_SIZE) {
+            const batch = vectors.slice(i, i + BATCH_SIZE);
+            await db.insert(schema.documentVectors).values(batch);
+        }
+    }
+    async storeManyChunkVectors(chunkVectors) {
+        if (chunkVectors.length === 0)
+            return;
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        const chunkIds = chunkVectors.map((cv) => cv.chunkId);
+        if (chunkIds.length > 0) {
+            await db.delete(schema.documentVectors).where(inArray(schema.documentVectors.chunkId, chunkIds));
+        }
+        for (const cv of chunkVectors) {
+            if (cv.tokenCount !== undefined) {
+                await this.dbInstance.pool.query('UPDATE chunks SET token_count = $1 WHERE id = $2', [cv.tokenCount, cv.chunkId]);
+            }
+        }
+        const allVectors = [];
+        for (const cv of chunkVectors) {
+            for (const [term, scores] of cv.terms.entries()) {
+                allVectors.push({
+                    chunkId: cv.chunkId,
+                    term,
+                    tf: scores.tf,
+                    tfidf: scores.tfidf,
+                    rawFreq: scores.rawFreq,
+                });
+            }
+        }
+        const batchSize = 500;
+        for (let i = 0; i < allVectors.length; i += batchSize) {
+            const batch = allVectors.slice(i, i + batchSize);
+            if (batch.length > 0)
+                await db.insert(schema.documentVectors).values(batch);
+        }
+    }
+    async storeIdfScores(idf, docFreq) {
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        await db.delete(schema.idfScores);
+        const BATCH_SIZE = 300;
+        const scores = Array.from(idf.entries()).map(([term, idfScore]) => ({
+            term,
+            idf: idfScore,
+            documentFrequency: docFreq.get(term) ?? 0,
+        }));
+        for (let i = 0; i < scores.length; i += BATCH_SIZE) {
+            const batch = scores.slice(i, i + BATCH_SIZE);
+            await db.insert(schema.idfScores).values(batch);
+        }
+    }
+    async getIdfScores() {
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        const scores = await db.select().from(schema.idfScores);
+        const idf = new Map();
+        for (const score of scores)
+            idf.set(score.term, score.idf);
+        return idf;
+    }
+    async getChunkVectors(chunkId) {
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        const vectors = await db
+            .select()
+            .from(schema.documentVectors)
+            .where(eq(schema.documentVectors.chunkId, chunkId));
+        if (vectors.length === 0)
+            return null;
+        const map = new Map();
+        for (const v of vectors) {
+            map.set(v.term, { tf: v.tf, tfidf: v.tfidf, rawFreq: v.rawFreq });
+        }
+        return map;
+    }
+    async getIdfScoresForTerms(terms) {
+        if (terms.length === 0)
+            return new Map();
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        const scores = await db.select().from(schema.idfScores).where(inArray(schema.idfScores.term, terms));
+        const idf = new Map();
+        for (const score of scores)
+            idf.set(score.term, score.idf);
+        return idf;
+    }
+    async getTotalDocuments() {
+        return this.getChunkCount();
+    }
+    async getAllFileMetadata() {
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        const results = await db.select({
+            path: schema.files.path,
+            mtime: schema.files.mtime,
+            hash: schema.files.hash,
+        }).from(schema.files);
+        const metadata = new Map();
+        for (const row of results)
+            metadata.set(row.path, { mtime: row.mtime, hash: row.hash });
+        return metadata;
+    }
+    async deleteFiles(paths) {
+        if (paths.length === 0)
+            return;
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        const chunkSize = 500;
+        for (let i = 0; i < paths.length; i += chunkSize) {
+            const chunk = paths.slice(i, i + chunkSize);
+            await db.delete(schema.files).where(inArray(schema.files.path, chunk));
+        }
+    }
+    async setMetadata(key, value) {
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        await db
+            .insert(schema.indexMetadata)
+            .values({ key, value, updatedAt: Date.now() })
+            .onConflictDoUpdate({
+            target: schema.indexMetadata.key,
+            set: { value, updatedAt: Date.now() },
+        });
+    }
+    async getMetadata(key) {
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        const result = await this.getOne(db.select().from(schema.indexMetadata).where(eq(schema.indexMetadata.key, key)).limit(1));
+        return result?.value ?? null;
+    }
+    async getAverageDocLength() {
+        await this.ensureInit();
+        const cached = await this.getMetadata('avgDocLength');
+        if (cached)
+            return parseFloat(cached);
+        const { db } = this.dbInstance;
+        const result = await this.getOne(db.select({
+            avgLen: sql `AVG(COALESCE(${schema.chunks.tokenCount}, 0))::double precision`,
+        }).from(schema.chunks));
+        const avgLen = result?.avgLen ?? 0;
+        await this.setMetadata('avgDocLength', String(avgLen));
+        return avgLen;
+    }
+    async updateAverageDocLength() {
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        const result = await this.getOne(db.select({
+            avgLen: sql `AVG(COALESCE(${schema.chunks.tokenCount}, 0))::double precision`,
+        }).from(schema.chunks));
+        const avgLen = result?.avgLen ?? 0;
+        await this.setMetadata('avgDocLength', String(avgLen));
+        return avgLen;
+    }
+    async rebuildIdfScoresFromVectors() {
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        const totalChunks = await this.getChunkCount();
+        if (totalChunks === 0) {
+            await db.delete(schema.idfScores);
+            return;
+        }
+        const dfResults = await db
+            .select({
+            term: schema.documentVectors.term,
+            df: sql `COUNT(DISTINCT ${schema.documentVectors.chunkId})::int`,
+        })
+            .from(schema.documentVectors)
+            .groupBy(schema.documentVectors.term);
+        await db.delete(schema.idfScores);
+        const BATCH_SIZE = 300;
+        const scores = dfResults.map((row) => ({
+            term: row.term,
+            idf: Math.log((totalChunks + 1) / (row.df + 1)) + 1,
+            documentFrequency: row.df,
+        }));
+        for (let i = 0; i < scores.length; i += BATCH_SIZE) {
+            const batch = scores.slice(i, i + BATCH_SIZE);
+            if (batch.length > 0)
+                await db.insert(schema.idfScores).values(batch);
+        }
+    }
+    async recalculateTfidfScores() {
+        await this.ensureInit();
+        await this.dbInstance.pool.query(`
+			UPDATE document_vectors
+			SET tfidf = tf * COALESCE(
+				(SELECT idf FROM idf_scores WHERE idf_scores.term = document_vectors.term),
+				0
+			)
+		`);
+    }
+    async updateChunkMagnitudes() {
+        await this.ensureInit();
+        await this.dbInstance.pool.query(`
+			UPDATE chunks
+			SET magnitude = COALESCE(
+				(SELECT SQRT(SUM(tfidf * tfidf))::real FROM document_vectors WHERE document_vectors.chunk_id = chunks.id),
+				0
+			)
+		`);
+    }
+    async getTermsForFiles(paths) {
+        if (paths.length === 0)
+            return new Set();
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        const terms = new Set();
+        const files = await db.select({ id: schema.files.id }).from(schema.files).where(inArray(schema.files.path, paths));
+        if (files.length === 0)
+            return terms;
+        const fileIds = files.map((f) => f.id);
+        const chunks = await db.select({ id: schema.chunks.id }).from(schema.chunks).where(inArray(schema.chunks.fileId, fileIds));
+        if (chunks.length === 0)
+            return terms;
+        const chunkIds = chunks.map((c) => c.id);
+        const results = await db.select({ term: schema.documentVectors.term }).from(schema.documentVectors).where(inArray(schema.documentVectors.chunkId, chunkIds));
+        for (const row of results)
+            terms.add(row.term);
+        return terms;
+    }
+    async searchByTerms(queryTerms, options = {}) {
+        if (queryTerms.length === 0)
+            return [];
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        const limit = options.limit ?? 100;
+        const matchingChunks = await db
+            .select({
+            chunkId: schema.documentVectors.chunkId,
+            filePath: schema.files.path,
+            content: schema.chunks.content,
+            type: schema.chunks.type,
+            startLine: schema.chunks.startLine,
+            endLine: schema.chunks.endLine,
+            magnitude: schema.chunks.magnitude,
+            tokenCount: schema.chunks.tokenCount,
+            matchCount: sql `COUNT(DISTINCT ${schema.documentVectors.term})::int`,
+        })
+            .from(schema.documentVectors)
+            .innerJoin(schema.chunks, eq(schema.documentVectors.chunkId, schema.chunks.id))
+            .innerJoin(schema.files, eq(schema.chunks.fileId, schema.files.id))
+            .where(inArray(schema.documentVectors.term, queryTerms))
+            .groupBy(schema.documentVectors.chunkId, schema.files.path, schema.chunks.content, schema.chunks.type, schema.chunks.startLine, schema.chunks.endLine, schema.chunks.magnitude, schema.chunks.tokenCount)
+            .orderBy(desc(sql `COUNT(DISTINCT ${schema.documentVectors.term})`))
+            .limit(limit * 2);
+        if (matchingChunks.length === 0)
+            return [];
+        const chunkIds = matchingChunks.map((c) => c.chunkId);
+        const matchedVectors = await db
+            .select({
+            chunkId: schema.documentVectors.chunkId,
+            term: schema.documentVectors.term,
+            tfidf: schema.documentVectors.tfidf,
+            rawFreq: schema.documentVectors.rawFreq,
+        })
+            .from(schema.documentVectors)
+            .where(sql `${schema.documentVectors.chunkId} IN (${sql.join(chunkIds.map((id) => sql `${id}`), sql `, `)}) AND ${schema.documentVectors.term} IN (${sql.join(queryTerms.map((t) => sql `${t}`), sql `, `)})`);
+        const resultMap = new Map();
+        for (const c of matchingChunks) {
+            resultMap.set(c.chunkId, {
+                chunkId: c.chunkId,
+                filePath: c.filePath,
+                content: c.content,
+                type: c.type,
+                startLine: c.startLine,
+                endLine: c.endLine,
+                matchedTerms: new Map(),
+                magnitude: c.magnitude ?? 0,
+                tokenCount: c.tokenCount ?? 0,
+            });
+        }
+        for (const v of matchedVectors) {
+            const entry = resultMap.get(v.chunkId);
+            if (entry)
+                entry.matchedTerms.set(v.term, { tfidf: v.tfidf, rawFreq: v.rawFreq });
+        }
+        return Array.from(resultMap.values());
+    }
+    async getAllChunks() {
+        await this.ensureInit();
+        const { db } = this.dbInstance;
+        const results = await db
+            .select({
+            id: schema.chunks.id,
+            fileId: schema.chunks.fileId,
+            content: schema.chunks.content,
+            type: schema.chunks.type,
+            startLine: schema.chunks.startLine,
+            endLine: schema.chunks.endLine,
+            metadata: schema.chunks.metadata,
+            filePath: schema.files.path,
+        })
+            .from(schema.chunks)
+            .innerJoin(schema.files, eq(schema.chunks.fileId, schema.files.id));
+        return results.map((r) => ({
+            id: r.id,
+            fileId: r.fileId,
+            filePath: r.filePath,
+            content: r.content,
+            type: r.type,
+            startLine: r.startLine,
+            endLine: r.endLine,
+            metadata: r.metadata ? JSON.parse(r.metadata) : undefined,
+        }));
+    }
+    close() {
+        this.dbInstance.pool.end();
+    }
+}
+/**
+ * Create Postgres-backed persistent storage (used by createPersistentStorage when backend === 'postgres').
+ */
+export function createPostgresPersistentStorage(config) {
+    return new PostgresPersistentStorage(config);
+}
+//# sourceMappingURL=storage-persistent-pg.js.map