@vinaes/succ 1.4.0 → 1.5.42

package/dist/lib/storage/backends/postgresql.js

@@ -9,7 +9,10 @@
  * - vector(N) type from pgvector extension
  * - ON CONFLICT instead of INSERT OR REPLACE
  */
+import { SOURCE_TYPES } from '../types.js';
 import { StorageError, ConfigError } from '../../errors.js';
+import { tokenizeCode, tokenizeCodeWithAST, tokenizeDocs, reciprocalRankFusion, } from '../../bm25.js';
+import { logWarn } from '../../fault-logger.js';
 // Lazy-load pg to make it optional
 let pg = null;
 async function loadPg() {
@@ -19,7 +22,10 @@ async function loadPg() {
         pg = await import('pg');
         return pg;
     }
-    catch {
+    catch (error) {
+        logWarn('postgresql', 'Failed to import pg package for PostgreSQL backend', {
+            error: error instanceof Error ? error.message : String(error),
+        });
         throw new ConfigError('PostgreSQL support requires the "pg" package. ' + 'Install it with: npm install pg');
     }
 }
@@ -105,7 +111,10 @@ export class PostgresBackend {
             const { getEmbeddingInfo } = await import('../../embeddings.js');
             return getEmbeddingInfo().dimensions ?? 384;
         }
-        catch {
+        catch (error) {
+            logWarn('postgresql', 'Failed to read embedding dimensions, using default 384', {
+                error: error instanceof Error ? error.message : String(error),
+            });
             return 384;
         }
     }
@@ -193,6 +202,9 @@ export class PostgresBackend {
         await pool.query('CREATE INDEX IF NOT EXISTS idx_documents_project_id ON documents(project_id)');
         await pool.query('CREATE INDEX IF NOT EXISTS idx_documents_file_path ON documents(file_path)');
         await pool.query('CREATE INDEX IF NOT EXISTS idx_documents_embedding ON documents USING ivfflat (embedding vector_cosine_ops) WITH (lists = 100)');
+        // Migration: add search_vector column for full-text search (tsvector)
+        await pool.query('ALTER TABLE documents ADD COLUMN IF NOT EXISTS search_vector tsvector');
+        await pool.query('CREATE INDEX IF NOT EXISTS idx_documents_search_vector ON documents USING GIN(search_vector)');
         // Metadata table
         await pool.query(`
       CREATE TABLE IF NOT EXISTS metadata (
@@ -264,6 +276,9 @@ export class PostgresBackend {
         await pool.query('CREATE INDEX IF NOT EXISTS idx_memories_type ON memories(type)');
         await pool.query('CREATE INDEX IF NOT EXISTS idx_memories_project_id ON memories(project_id)');
         await pool.query('CREATE INDEX IF NOT EXISTS idx_memories_embedding ON memories USING ivfflat (embedding vector_cosine_ops) WITH (lists = 100)');
+        // Migration: add search_vector column for full-text search (tsvector)
+        await pool.query('ALTER TABLE memories ADD COLUMN IF NOT EXISTS search_vector tsvector');
+        await pool.query('CREATE INDEX IF NOT EXISTS idx_memories_search_vector ON memories USING GIN(search_vector)');
         // Memory links table
         await pool.query(`
       CREATE TABLE IF NOT EXISTS memory_links (
@@ -282,6 +297,8 @@ export class PostgresBackend {
         await pool.query('CREATE INDEX IF NOT EXISTS idx_memory_links_target ON memory_links(target_id)');
         // Migration: add llm_enriched column to memory_links
         await pool.query('ALTER TABLE memory_links ADD COLUMN IF NOT EXISTS llm_enriched INTEGER DEFAULT 0');
+        // Migration: add metadata JSON column to memory_links (for bridge edges)
+        await pool.query('ALTER TABLE memory_links ADD COLUMN IF NOT EXISTS metadata JSONB');
         // Migration: add project_id column to memory_links (for multi-project PG)
         await pool.query(`
       DO $$
@@ -295,6 +312,72 @@ export class PostgresBackend {
       END $$;
     `);
         await pool.query('CREATE INDEX IF NOT EXISTS idx_memory_links_project_id ON memory_links(project_id)');
+        // Migration: add confidence and source_type columns for memory provenance
+        await pool.query('ALTER TABLE memories ADD COLUMN IF NOT EXISTS confidence REAL DEFAULT 0.5');
+        await pool.query("ALTER TABLE memories ADD COLUMN IF NOT EXISTS source_type TEXT DEFAULT 'human'");
+        // CHECK constraints for provenance columns (defense-in-depth)
+        await pool.query(`
+      DO $$ BEGIN
+        IF NOT EXISTS (
+          SELECT 1 FROM pg_constraint
+          WHERE conname = 'chk_memories_confidence'
+            AND conrelid = 'memories'::regclass
+        ) THEN
+          UPDATE memories
+            SET confidence = 0.5
+            WHERE confidence IS NULL
+               OR confidence < 0
+               OR confidence > 1
+               OR confidence <> confidence;
+          ALTER TABLE memories
+            ADD CONSTRAINT chk_memories_confidence
+            CHECK (confidence >= 0 AND confidence <= 1 AND confidence = confidence);
+        END IF;
+
+        IF NOT EXISTS (
+          SELECT 1 FROM pg_constraint
+          WHERE conname = 'chk_memories_source_type'
+            AND conrelid = 'memories'::regclass
+        ) THEN
+          UPDATE memories
+            SET source_type = 'human'
+            WHERE source_type IS NULL
+               OR source_type NOT IN ('human','agent','canonical_doc','imported','auto_extracted');
+          ALTER TABLE memories
+            ADD CONSTRAINT chk_memories_source_type
+            CHECK (source_type IN ('human','agent','canonical_doc','imported','auto_extracted'));
+        END IF;
+      END $$;
+    `);
+        // CHECK constraint for non-negative link weights (required for Dijkstra correctness)
+        await pool.query(`
+      DO $$ BEGIN
+        IF NOT EXISTS (
+          SELECT 1 FROM pg_constraint
+          WHERE conname = 'chk_memory_links_weight'
+            AND conrelid = 'memory_links'::regclass
+        ) THEN
+          UPDATE memory_links SET weight = GREATEST(weight, 0)
+            WHERE weight < 0;
+          ALTER TABLE memory_links
+            ADD CONSTRAINT chk_memory_links_weight CHECK (weight >= 0);
+        END IF;
+      END $$;
+    `);
+        // Retrieval feedback table
+        await pool.query(`
+      CREATE TABLE IF NOT EXISTS recall_events (
+        id SERIAL PRIMARY KEY,
+        memory_id INTEGER NOT NULL REFERENCES memories(id) ON DELETE CASCADE,
+        query TEXT NOT NULL,
+        was_used INTEGER NOT NULL DEFAULT 0,
+        rank_position INTEGER,
+        similarity_score REAL,
+        created_at TIMESTAMPTZ DEFAULT NOW()
+      )
+    `);
+        await pool.query('CREATE INDEX IF NOT EXISTS idx_recall_events_memory ON recall_events(memory_id)');
+        await pool.query('CREATE INDEX IF NOT EXISTS idx_recall_events_created ON recall_events(created_at)');
         // Memory centrality cache table
         await pool.query(`
       CREATE TABLE IF NOT EXISTS memory_centrality (
@@ -471,6 +554,20 @@ export class PostgresBackend {
         await pool.query('CREATE INDEX IF NOT EXISTS idx_wsh_project_id ON web_search_history(project_id)');
         await pool.query('CREATE INDEX IF NOT EXISTS idx_wsh_created ON web_search_history(created_at)');
         await pool.query('CREATE INDEX IF NOT EXISTS idx_wsh_tool ON web_search_history(tool_name)');
+        // One-time backfill: populate search_vector for rows that existed before tsvector migration
+        const migrationKey = 'search_vector_backfill_done';
+        const migCheck = await pool.query('SELECT value FROM metadata WHERE key = $1', [migrationKey]);
+        if (migCheck.rows.length === 0) {
+            try {
+                await this.rebuildAllSearchVectors();
+                await pool.query(`INSERT INTO metadata (key, value) VALUES ($1, '1') ON CONFLICT(key) DO NOTHING`, [migrationKey]);
+            }
+            catch (error) {
+                logWarn('postgresql', 'search_vector backfill failed, will retry on next init', {
+                    error,
+                });
+            }
+        }
     }
     /**
      * Close all connections.
@@ -513,7 +610,14 @@ export class PostgresBackend {
             symbolType ?? null,
             signature ?? null,
         ]);
-        return result.rows[0].id;
+        const docId = result.rows[0].id;
+        // Compute and store search_vector for full-text search
+        const tokens = filePath.startsWith('code:')
+            ? tokenizeCodeWithAST(content, signature ? tokenizeCode(signature) : [], symbolName)
+            : tokenizeDocs(content);
+        const tokenStr = tokens.join(' ');
+        await pool.query(`UPDATE documents SET search_vector = CASE WHEN $1 = '' THEN NULL ELSE to_tsvector('simple', $1) END WHERE id = $2`, [tokenStr, docId]);
+        return docId;
     }
     async upsertDocumentsBatch(documents) {
         if (documents.length === 0)
@@ -550,7 +654,14 @@ export class PostgresBackend {
                     doc.symbolType ?? null,
                     doc.signature ?? null,
                 ]);
-                ids.push(result.rows[0].id);
+                const docId = result.rows[0].id;
+                ids.push(docId);
+                // Compute and store search_vector for full-text search
+                const tokens = doc.filePath.startsWith('code:')
+                    ? tokenizeCodeWithAST(doc.content, doc.signature ? tokenizeCode(doc.signature) : [], doc.symbolName)
+                    : tokenizeDocs(doc.content);
+                const tokenStr = tokens.join(' ');
+                await client.query(`UPDATE documents SET search_vector = CASE WHEN $1 = '' THEN NULL ELSE to_tsvector('simple', $1) END WHERE id = $2`, [tokenStr, docId]);
             }
             await client.query('COMMIT');
         }
@@ -599,7 +710,14 @@ export class PostgresBackend {
                     doc.symbolType ?? null,
                     doc.signature ?? null,
                 ]);
-                ids.push(result.rows[0].id);
+                const docId = result.rows[0].id;
+                ids.push(docId);
+                // Compute and store search_vector for full-text search
+                const tokens = doc.filePath.startsWith('code:')
+                    ? tokenizeCodeWithAST(doc.content, doc.signature ? tokenizeCode(doc.signature) : [], doc.symbolName)
+                    : tokenizeDocs(doc.content);
+                const tokenStr = tokens.join(' ');
+                await client.query(`UPDATE documents SET search_vector = CASE WHEN $1 = '' THEN NULL ELSE to_tsvector('simple', $1) END WHERE id = $2`, [tokenStr, docId]);
                 if (!processedFiles.has(doc.filePath)) {
                     await client.query(`INSERT INTO file_hashes (project_id, file_path, content_hash, indexed_at)
              VALUES ($1, $2, $3, NOW())
@@ -683,7 +801,9 @@ export class PostgresBackend {
         const pool = await this.getPool();
         let query = `
       SELECT id, content, tags, source, type, quality_score, quality_factors,
-             access_count, last_accessed, valid_from, valid_until, created_at
+             access_count, last_accessed, correction_count, is_invariant,
+             priority_score, valid_from, valid_until, confidence, source_type,
+             created_at
       FROM memories WHERE id = ANY($1)`;
         const params = [ids];
         let idx = 2;
@@ -711,7 +831,6 @@ export class PostgresBackend {
         if (filters?.createdBefore) {
             query += ` AND created_at <= $${idx}`;
             params.push(filters.createdBefore.toISOString());
-            idx++;
         }
         const result = await pool.query(query, params);
         return result.rows.map((row) => ({
@@ -733,6 +852,8 @@ export class PostgresBackend {
             priority_score: row.priority_score ?? null,
             valid_from: row.valid_from,
             valid_until: row.valid_until,
+            confidence: row.confidence ?? null,
+            source_type: (row.source_type ?? null),
             created_at: row.created_at,
         }));
     }
@@ -774,6 +895,432 @@ export class PostgresBackend {
         await pool.query("DELETE FROM metadata WHERE key = 'embedding_model'");
     }
     // ============================================================================
+    // Full-Text Search Vector Updates
+    // ============================================================================
+    /**
+     * Update the search_vector for a single document (called from BM25 dispatcher).
+     * tokens: pre-tokenized string joined with spaces, e.g. "auth user login".
+     */
+    async updateDocumentSearchVector(docId, tokens) {
+        const pool = await this.getPool();
+        const trimmed = (tokens ?? '').trim();
+        await pool.query(`UPDATE documents SET search_vector = CASE WHEN $1 = '' THEN NULL ELSE to_tsvector('simple', $1) END WHERE id = $2`, [trimmed, docId]);
+    }
+    /**
+     * Update the search_vector for a single memory (called from BM25 dispatcher).
+     */
+    async updateMemorySearchVector(memoryId, tokens) {
+        const pool = await this.getPool();
+        const trimmed = (tokens ?? '').trim();
+        await pool.query(`UPDATE memories SET search_vector = CASE WHEN $1 = '' THEN NULL ELSE to_tsvector('simple', $1) END WHERE id = $2`, [trimmed, memoryId]);
+    }
+    /**
+     * Rebuild all search_vector values from content.
+     * Use after tokenizer logic changes or manual DB edits.
+     * Processes in batches to avoid memory issues on large datasets.
+     */
+    async rebuildAllSearchVectors(batchSize = 500) {
+        const pool = await this.getPool();
+        let docCount = 0;
+        let memCount = 0;
+        // Rebuild documents in batches with batch UPDATE
+        const docBaseParams = [];
+        let docWhere;
+        if (this.projectId) {
+            docBaseParams.push(this.projectId);
+            docWhere = `LOWER(project_id) = $1`;
+        }
+        else {
+            docWhere = `TRUE`;
+        }
+        const docLimitIdx = docBaseParams.length + 1;
+        const docOffsetIdx = docBaseParams.length + 2;
+        let offset = 0;
+        while (true) {
+            const batch = await pool.query(`SELECT id, file_path, content, symbol_name, signature FROM documents
+         WHERE ${docWhere} ORDER BY id LIMIT $${docLimitIdx} OFFSET $${docOffsetIdx}`, [...docBaseParams, batchSize, offset]);
+            if (batch.rows.length === 0)
+                break;
+            const ids = [];
+            const tokenStrs = [];
+            for (const row of batch.rows) {
+                const tokens = row.file_path.startsWith('code:')
+                    ? tokenizeCodeWithAST(row.content, row.signature ? tokenizeCode(row.signature) : [], row.symbol_name ?? undefined)
+                    : tokenizeDocs(row.content);
+                ids.push(row.id);
+                tokenStrs.push(tokens.join(' '));
+                docCount++;
+            }
+            if (ids.length > 0) {
+                await pool.query(`UPDATE documents SET search_vector = CASE WHEN v.tokens = '' THEN NULL ELSE to_tsvector('simple', v.tokens) END
+           FROM unnest($1::int[], $2::text[]) AS v(id, tokens)
+           WHERE documents.id = v.id`, [ids, tokenStrs]);
+            }
+            offset += batch.rows.length;
+        }
+        // Rebuild memories in batches with batch UPDATE
+        const memBaseParams = [];
+        let memWhere;
+        if (this.projectId) {
+            memBaseParams.push(this.projectId);
+            memWhere = `(LOWER(project_id) = $1 OR project_id IS NULL)`;
+        }
+        else {
+            memWhere = `project_id IS NULL`;
+        }
+        const limitIdx = memBaseParams.length + 1;
+        const offsetIdx = memBaseParams.length + 2;
+        offset = 0;
+        while (true) {
+            const batch = await pool.query(`SELECT id, content FROM memories WHERE ${memWhere} ORDER BY id LIMIT $${limitIdx} OFFSET $${offsetIdx}`, [...memBaseParams, batchSize, offset]);
+            if (batch.rows.length === 0)
+                break;
+            const ids = [];
+            const tokenStrs = [];
+            for (const row of batch.rows) {
+                const tokens = tokenizeDocs(row.content);
+                ids.push(row.id);
+                tokenStrs.push(tokens.join(' '));
+                memCount++;
+            }
+            if (ids.length > 0) {
+                await pool.query(`UPDATE memories SET search_vector = CASE WHEN v.tokens = '' THEN NULL ELSE to_tsvector('simple', v.tokens) END
+           FROM unnest($1::int[], $2::text[]) AS v(id, tokens)
+           WHERE memories.id = v.id`, [ids, tokenStrs]);
+            }
+            offset += batch.rows.length;
+        }
+        return { documents: docCount, memories: memCount };
+    }
+    // ============================================================================
+    // Hybrid Search (tsvector + pgvector + RRF)
+    // ============================================================================
+    /**
+     * Build a tsquery string from tokenized query terms using OR semantics.
+     * Returns null if no valid tokens (caller should skip text search).
+     */
+    buildTsquery(query, isCode) {
+        const tokens = isCode ? tokenizeCode(query) : tokenizeDocs(query);
+        const valid = tokens.filter((t) => t.length > 0);
+        if (valid.length === 0)
+            return null;
+        // De-duplicate and join with OR operator
+        return [...new Set(valid)].join(' | ');
+    }
+    /**
+     * Hybrid search over documents (code + brain docs).
+     * Combines tsvector full-text search with pgvector cosine similarity via RRF.
+     */
+    async hybridSearchDocuments(query, queryEmbedding, limit, threshold, options) {
+        if (!this.projectId) {
+            throw new StorageError('Project ID must be set before searching documents');
+        }
+        const pool = await this.getPool();
+        const isCode = !!options?.codeOnly;
+        // Build tsquery for text search
+        const tsq = this.buildTsquery(query, isCode);
+        // --- Text search ---
+        const textResults = [];
+        if (tsq) {
+            let textWhere = `WHERE LOWER(project_id) = $2 AND search_vector @@ to_tsquery('simple', $1)`;
+            const textParams = [tsq, this.projectId];
+            if (options?.codeOnly)
+                textWhere += ` AND file_path LIKE 'code:%'`;
+            else if (options?.docsOnly)
+                textWhere += ` AND file_path NOT LIKE 'code:%'`;
+            if (options?.symbolType) {
+                textParams.push(options.symbolType);
+                textWhere += ` AND symbol_type = $${textParams.length}`;
+            }
+            textParams.push(limit * 3); // overfetch for RRF
+            const textQ = `
+        SELECT id, ts_rank_cd(search_vector, to_tsquery('simple', $1)) as score
+        FROM documents
+        ${textWhere}
+        ORDER BY score DESC
+        LIMIT $${textParams.length}
+      `;
+            try {
+                const r = await pool.query(textQ, textParams);
+                for (const row of r.rows) {
+                    textResults.push({ docId: row.id, score: parseFloat(String(row.score)) });
+                }
+            }
+            catch (error) {
+                logWarn('postgresql', 'hybridSearchDocuments text search failed, using vector-only', {
+                    error: error instanceof Error ? error.message : String(error),
+                });
+            }
+        }
+        // --- Vector search ---
+        const vectorResults = [];
+        {
+            let vecWhere = `WHERE LOWER(project_id) = $2 AND 1 - (embedding <=> $1) >= $3`;
+            const vecParams = [toPgVector(queryEmbedding), this.projectId, threshold];
+            if (options?.codeOnly)
+                vecWhere += ` AND file_path LIKE 'code:%'`;
+            else if (options?.docsOnly)
+                vecWhere += ` AND file_path NOT LIKE 'code:%'`;
+            if (options?.symbolType) {
+                vecParams.push(options.symbolType);
+                vecWhere += ` AND symbol_type = $${vecParams.length}`;
+            }
+            vecParams.push(limit * 3); // overfetch for RRF
+            const vecQ = `
+        SELECT id, 1 - (embedding <=> $1) as score
+        FROM documents
+        ${vecWhere}
+        ORDER BY embedding <=> $1
+        LIMIT $${vecParams.length}
+      `;
+            const r = await pool.query(vecQ, vecParams);
+            for (const row of r.rows) {
+                vectorResults.push({ docId: row.id, score: parseFloat(String(row.score)) });
+            }
+        }
+        // --- RRF fusion ---
+        const fused = reciprocalRankFusion(textResults, vectorResults, 0.5, limit);
+        if (fused.length === 0)
+            return [];
+        const fusedIds = fused.map((r) => r.docId);
+        const fusedScoreMap = new Map(fused.map((r) => [r.docId, r.score]));
+        const textScoreMap = new Map(textResults.map((r) => [r.docId, r.score]));
+        const vecScoreMap = new Map(vectorResults.map((r) => [r.docId, r.score]));
+        // Fetch full rows for fused IDs
+        const rows = await pool.query(`SELECT id, file_path, content, start_line, end_line, symbol_name, symbol_type, signature
+       FROM documents WHERE id = ANY($1)`, [fusedIds]);
+        // Map rows by id for ordering
+        const rowMap = new Map(rows.rows.map((r) => [r.id, r]));
+        const results = [];
+        for (const { docId } of fused) {
+            const row = rowMap.get(docId);
+            if (!row)
+                continue;
+            results.push({
+                file_path: row.file_path,
+                content: row.content,
+                start_line: row.start_line,
+                end_line: row.end_line,
+                similarity: fusedScoreMap.get(docId) ?? 0,
+                bm25Score: textScoreMap.get(docId) ?? 0,
+                vectorScore: vecScoreMap.get(docId) ?? 0,
+                symbol_name: row.symbol_name,
+                symbol_type: row.symbol_type,
+                signature: row.signature,
+            });
+        }
+        return results;
+    }
+    /**
+     * Hybrid search over project memories.
+     */
+    async hybridSearchMemories(query, queryEmbedding, limit, threshold) {
+        const pool = await this.getPool();
+        const tsq = this.buildTsquery(query, false);
+        const now = new Date().toISOString();
+        // --- Text search ---
+        const textResults = [];
+        if (tsq) {
+            const textParams = [tsq];
+            let paramIdx = 2;
+            let whereCond = `WHERE search_vector @@ to_tsquery('simple', $1) AND invalidated_by IS NULL`;
+            whereCond += ` AND (valid_from IS NULL OR valid_from <= $${paramIdx})`;
+            textParams.push(now);
+            paramIdx++;
+            whereCond += ` AND (valid_until IS NULL OR valid_until > $${paramIdx})`;
+            textParams.push(now);
+            paramIdx++;
+            if (this.projectId) {
+                whereCond += ` AND (LOWER(project_id) = $${paramIdx} OR project_id IS NULL)`;
+                textParams.push(this.projectId);
+            }
+            else {
+                whereCond += ` AND project_id IS NULL`;
+            }
+            textParams.push(limit * 3);
+            const textQ = `
+        SELECT id, ts_rank_cd(search_vector, to_tsquery('simple', $1)) as score
+        FROM memories
+        ${whereCond}
+        ORDER BY score DESC
+        LIMIT $${textParams.length}
+      `;
+            try {
+                const r = await pool.query(textQ, textParams);
+                for (const row of r.rows) {
+                    textResults.push({ docId: row.id, score: parseFloat(String(row.score)) });
+                }
+            }
+            catch (error) {
+                logWarn('postgresql', 'hybridSearchMemories text search failed, using vector-only', {
+                    error: error instanceof Error ? error.message : String(error),
+                });
+            }
+        }
+        // --- Vector search ---
+        const vectorResults = [];
+        {
+            const vecParams = [toPgVector(queryEmbedding), threshold];
+            let paramIdx = 3;
+            let whereVec = `WHERE 1 - (embedding <=> $1) >= $2 AND invalidated_by IS NULL`;
+            whereVec += ` AND (valid_from IS NULL OR valid_from <= $${paramIdx})`;
+            vecParams.push(now);
+            paramIdx++;
+            whereVec += ` AND (valid_until IS NULL OR valid_until > $${paramIdx})`;
+            vecParams.push(now);
+            paramIdx++;
+            if (this.projectId) {
+                whereVec += ` AND (LOWER(project_id) = $${paramIdx} OR project_id IS NULL)`;
+                vecParams.push(this.projectId);
+            }
+            else {
+                whereVec += ` AND project_id IS NULL`;
+            }
+            vecParams.push(limit * 3);
+            const vecQ = `
+        SELECT id, 1 - (embedding <=> $1) as score
+        FROM memories
+        ${whereVec}
+        ORDER BY embedding <=> $1
+        LIMIT $${vecParams.length}
+      `;
+            const r = await pool.query(vecQ, vecParams);
+            for (const row of r.rows) {
+                vectorResults.push({ docId: row.id, score: parseFloat(String(row.score)) });
+            }
+        }
+        // --- RRF fusion ---
+        const fused = reciprocalRankFusion(textResults, vectorResults, 0.5, limit);
+        if (fused.length === 0)
+            return [];
+        const fusedIds = fused.map((r) => r.docId);
+        const fusedScoreMap = new Map(fused.map((r) => [r.docId, r.score]));
+        const textScoreMap = new Map(textResults.map((r) => [r.docId, r.score]));
+        const vecScoreMap = new Map(vectorResults.map((r) => [r.docId, r.score]));
+        // Fetch full rows
+        const rows = await pool.query(`SELECT id, content, tags, source, type, created_at,
+              access_count, last_accessed, valid_from, valid_until
+       FROM memories WHERE id = ANY($1)`, [fusedIds]);
+        const rowMap = new Map(rows.rows.map((r) => [r.id, r]));
+        const results = [];
+        for (const { docId } of fused) {
+            const row = rowMap.get(docId);
+            if (!row)
+                continue;
+            results.push({
+                id: row.id,
+                content: row.content,
+                tags: row.tags ? (typeof row.tags === 'string' ? JSON.parse(row.tags) : row.tags) : [],
+                source: row.source,
+                type: (row.type ?? null),
+                created_at: row.created_at,
+                similarity: fusedScoreMap.get(docId) ?? 0,
+                bm25Score: textScoreMap.get(docId) ?? 0,
+                vectorScore: vecScoreMap.get(docId) ?? 0,
+                access_count: row.access_count ?? 0,
+                last_accessed: row.last_accessed,
+                valid_from: row.valid_from,
+                valid_until: row.valid_until,
+            });
+        }
+        return results;
+    }
+    /**
+     * Hybrid search over global memories (project_id IS NULL).
+     */
+    async hybridSearchGlobalMemories(query, queryEmbedding, limit, threshold, tags, since) {
+        const pool = await this.getPool();
+        const tsq = this.buildTsquery(query, false);
+        // --- Text search ---
+        const textResults = [];
+        if (tsq) {
+            const textParams = [tsq];
+            const paramIdx = 2;
+            let whereText = `WHERE search_vector @@ to_tsquery('simple', $1) AND project_id IS NULL AND invalidated_by IS NULL`;
+            if (since) {
+                whereText += ` AND created_at >= $${paramIdx}`;
+                textParams.push(since.toISOString());
+            }
+            textParams.push(limit * 3);
+            const textQ = `
+        SELECT id, ts_rank_cd(search_vector, to_tsquery('simple', $1)) as score
+        FROM memories
+        ${whereText}
+        ORDER BY score DESC
+        LIMIT $${textParams.length}
+      `;
+            try {
+                const r = await pool.query(textQ, textParams);
+                for (const row of r.rows) {
+                    textResults.push({ docId: row.id, score: parseFloat(String(row.score)) });
+                }
+            }
+            catch (error) {
+                logWarn('postgresql', 'hybridSearchGlobalMemories text search failed, using vector-only', {
+                    error: error instanceof Error ? error.message : String(error),
+                });
+            }
+        }
+        // --- Vector search ---
+        const vectorResults = [];
+        {
+            const vecParams = [toPgVector(queryEmbedding), threshold];
+            const paramIdx = 3;
+            let whereVec = `WHERE 1 - (embedding <=> $1) >= $2 AND project_id IS NULL AND invalidated_by IS NULL`;
+            if (since) {
+                whereVec += ` AND created_at >= $${paramIdx}`;
+                vecParams.push(since.toISOString());
+            }
+            vecParams.push(limit * 3);
+            const vecQ = `
+        SELECT id, 1 - (embedding <=> $1) as score
+        FROM memories
+        ${whereVec}
+        ORDER BY embedding <=> $1
+        LIMIT $${vecParams.length}
+      `;
+            const r = await pool.query(vecQ, vecParams);
+            for (const row of r.rows) {
+                vectorResults.push({ docId: row.id, score: parseFloat(String(row.score)) });
+            }
+        }
+        // --- RRF fusion ---
+        const fused = reciprocalRankFusion(textResults, vectorResults, 0.5, limit);
+        if (fused.length === 0)
+            return [];
+        const fusedIds = fused.map((r) => r.docId);
+        const fusedScoreMap = new Map(fused.map((r) => [r.docId, r.score]));
+        const textScoreMap = new Map(textResults.map((r) => [r.docId, r.score]));
+        const vecScoreMap = new Map(vectorResults.map((r) => [r.docId, r.score]));
+        // Fetch full rows
+        const rows = await pool.query(`SELECT id, project_id, content, tags, source, type, created_at
+       FROM memories WHERE id = ANY($1)`, [fusedIds]);
+        const rowMap = new Map(rows.rows.map((r) => [r.id, r]));
+        let results = [];
+        for (const { docId } of fused) {
+            const row = rowMap.get(docId);
+            if (!row)
+                continue;
+            results.push({
+                id: row.id,
+                content: row.content,
+                tags: row.tags ? (typeof row.tags === 'string' ? JSON.parse(row.tags) : row.tags) : [],
+                source: row.source,
+                project: row.project_id,
+                type: (row.type ?? null),
+                created_at: row.created_at,
+                similarity: fusedScoreMap.get(docId) ?? 0,
+                bm25Score: textScoreMap.get(docId) ?? 0,
+                vectorScore: vecScoreMap.get(docId) ?? 0,
+            });
+        }
+        // Tag filter (post-filter in JS, consistent with searchGlobalMemories)
+        if (tags && tags.length > 0) {
+            results = results.filter((m) => tags.some((t) => m.tags.some((rt) => rt.toLowerCase().includes(t.toLowerCase()))));
+        }
+        return results;
+    }
+    // ============================================================================
     // File Hashes
     // ============================================================================
     async getFileHash(filePath) {
@@ -824,11 +1371,18 @@ export class PostgresBackend {
     // ============================================================================
     // Memory Operations
     // ============================================================================
-    async saveMemory(content, embedding, tags = [], source, type = 'observation', qualityScore, qualityFactors, validFrom, validUntil, isGlobal = false) {
+    async saveMemory(content, embedding, tags = [], source, type = 'observation', qualityScore, qualityFactors, validFrom, validUntil, isGlobal = false, confidence = 0.5, sourceType = 'human') {
+        // Validate provenance fields
+        if (!Number.isFinite(confidence) || confidence < 0 || confidence > 1) {
+            confidence = 0.5;
+        }
+        if (!SOURCE_TYPES.includes(sourceType)) {
+            sourceType = 'human';
+        }
         const pool = await this.getPool();
         const projectId = isGlobal ? null : this.projectId;
-        const result = await pool.query(`INSERT INTO memories (project_id, content, tags, source, type, quality_score, quality_factors, embedding, valid_from, valid_until)
-       VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
+        const result = await pool.query(`INSERT INTO memories (project_id, content, tags, source, type, quality_score, quality_factors, embedding, valid_from, valid_until, confidence, source_type)
+       VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
        RETURNING id`, [
             projectId,
             content,
@@ -840,8 +1394,15 @@ export class PostgresBackend {
             toPgVector(embedding),
             validFrom ?? null,
             validUntil ?? null,
+            confidence,
+            sourceType,
         ]);
-        return result.rows[0].id;
+        const memoryId = result.rows[0].id;
+        // Compute and store search_vector for full-text search
+        const tokens = tokenizeDocs(content);
+        const tokenStr = tokens.join(' ');
+        await pool.query(`UPDATE memories SET search_vector = CASE WHEN $1 = '' THEN NULL ELSE to_tsvector('simple', $1) END WHERE id = $2`, [tokenStr, memoryId]);
+        return memoryId;
     }
     async searchMemories(queryEmbedding, limit = 5, threshold = 0.3, tags, since, options) {
         const pool = await this.getPool();
@@ -850,8 +1411,9 @@ export class PostgresBackend {
         const includeGlobal = options?.includeGlobal ?? true;
         let query = `
       SELECT id, project_id, content, tags, source, type, quality_score, quality_factors,
-             access_count, last_accessed, valid_from, valid_until, created_at,
-             1 - (embedding <=> $1) as similarity
+             access_count, last_accessed, correction_count, is_invariant,
+             priority_score, valid_from, valid_until, confidence, source_type,
+             created_at, 1 - (embedding <=> $1) as similarity
       FROM memories
       WHERE 1 - (embedding <=> $1) >= $2
         AND invalidated_by IS NULL
@@ -908,6 +1470,8 @@ export class PostgresBackend {
             priority_score: row.priority_score ?? null,
             valid_from: row.valid_from,
             valid_until: row.valid_until,
+            confidence: row.confidence ?? null,
+            source_type: (row.source_type ?? null),
             created_at: row.created_at,
             similarity: parseFloat(row.similarity),
         }));
@@ -921,7 +1485,7 @@ export class PostgresBackend {
         const pool = await this.getPool();
         const result = await pool.query(`SELECT id, content, tags, source, type, quality_score, quality_factors,
               access_count, last_accessed, valid_from, valid_until,
-              correction_count, is_invariant, priority_score, created_at
+              correction_count, is_invariant, priority_score, confidence, source_type, created_at
        FROM memories WHERE id = $1`, [id]);
         if (result.rows.length === 0)
             return null;
@@ -945,6 +1509,8 @@ export class PostgresBackend {
             priority_score: row.priority_score ?? null,
             valid_from: row.valid_from,
             valid_until: row.valid_until,
+            confidence: row.confidence ?? null,
+            source_type: (row.source_type ?? null),
             created_at: row.created_at,
         };
     }
@@ -952,7 +1518,7 @@ export class PostgresBackend {
         const pool = await this.getPool();
         const result = await pool.query(`SELECT id, content, tags, source, type, quality_score, quality_factors,
               access_count, last_accessed, valid_from, valid_until,
-              correction_count, is_invariant, priority_score, created_at
+              correction_count, is_invariant, priority_score, confidence, source_type, created_at
        FROM memories
        WHERE tags::jsonb ? $1
          AND invalidated_by IS NULL
@@ -978,6 +1544,8 @@ export class PostgresBackend {
             priority_score: row.priority_score ?? null,
             valid_from: row.valid_from,
             valid_until: row.valid_until,
+            confidence: row.confidence ?? null,
+            source_type: (row.source_type ?? null),
             created_at: row.created_at,
         }));
     }
@@ -1012,7 +1580,7 @@ export class PostgresBackend {
         let query = `
       SELECT id, project_id, content, tags, source, type, quality_score, quality_factors,
              access_count, last_accessed, valid_from, valid_until,
-             correction_count, is_invariant, priority_score, created_at
+             correction_count, is_invariant, priority_score, confidence, source_type, created_at
       FROM memories
     `;
         const params = [];
@@ -1053,6 +1621,8 @@ export class PostgresBackend {
             correction_count: row.correction_count ?? 0,
             is_invariant: !!row.is_invariant,
             priority_score: row.priority_score ?? null,
+            confidence: row.confidence ?? null,
+            source_type: (row.source_type ?? null),
             created_at: row.created_at,
         }));
     }
@@ -1078,7 +1648,7 @@ export class PostgresBackend {
         let query = `
       SELECT id, content, tags, source, type, quality_score, quality_factors,
              access_count, last_accessed, valid_from, valid_until,
-             correction_count, is_invariant, priority_score, created_at
+             correction_count, is_invariant, priority_score, confidence, source_type, created_at
       FROM memories
     `;
         const params = [];
@@ -1093,7 +1663,6 @@ export class PostgresBackend {
         }
         query += ` AND (correction_count >= $${paramIndex} OR is_invariant = 1)`;
         params.push(threshold);
-        paramIndex++;
         query += ` ORDER BY is_invariant DESC, correction_count DESC, quality_score DESC`;
         const result = await pool.query(query, params);
         return result.rows.map((row) => ({
@@ -1115,6 +1684,8 @@ export class PostgresBackend {
             correction_count: row.correction_count ?? 0,
             is_invariant: !!row.is_invariant,
             priority_score: row.priority_score ?? null,
+            confidence: row.confidence ?? null,
+            source_type: (row.source_type ?? null),
             created_at: row.created_at,
         }));
     }
@@ -1128,7 +1699,7 @@ export class PostgresBackend {
     // ============================================================================
     // Memory Links
     // ============================================================================
-    async createMemoryLink(sourceId, targetId, relation = 'related', weight = 1.0, validFrom, validUntil) {
+    async createMemoryLink(sourceId, targetId, relation = 'related', weight = 1.0, validFrom, validUntil, metadata) {
         const pool = await this.getPool();
         // Validate that both memories belong to current project (case-insensitive for Windows paths)
         const validation = await pool.query(`SELECT COUNT(*) as count FROM memories
@@ -1136,16 +1707,35 @@ export class PostgresBackend {
         if (parseInt(validation.rows[0].count) !== 2) {
             throw new StorageError('Cannot link memories from different projects');
         }
-        const result = await pool.query(`INSERT INTO memory_links (source_id, target_id, relation, weight, valid_from, valid_until)
-       VALUES ($1, $2, $3, $4, $5, $6)
-       ON CONFLICT (source_id, target_id, relation) DO NOTHING
-       RETURNING id`, [sourceId, targetId, relation, weight, validFrom ?? null, validUntil ?? null]);
-        if (result.rows.length > 0) {
-            return { id: result.rows[0].id, created: true };
-        }
-        // Link already existed — fetch its id
-        const existing = await pool.query('SELECT id FROM memory_links WHERE source_id = $1 AND target_id = $2 AND relation = $3', [sourceId, targetId, relation]);
-        return { id: existing.rows[0].id, created: false };
+        const metadataJson = metadata ? JSON.stringify(metadata) : null;
+        const result = await pool.query(`INSERT INTO memory_links (source_id, target_id, relation, weight, valid_from, valid_until, metadata)
+       VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb)
+       ON CONFLICT (source_id, target_id, relation) DO UPDATE
+         SET metadata = CASE
+               WHEN memory_links.metadata IS NULL THEN EXCLUDED.metadata
+               WHEN EXCLUDED.metadata IS NULL THEN memory_links.metadata
+               ELSE memory_links.metadata || EXCLUDED.metadata || jsonb_build_object(
+                 'code_paths',
+                 COALESCE((SELECT jsonb_agg(DISTINCT p)
+                  FROM jsonb_array_elements_text(
+                    COALESCE(memory_links.metadata->'code_paths', '[]'::jsonb) ||
+                    COALESCE(EXCLUDED.metadata->'code_paths', '[]'::jsonb) ||
+                    CASE WHEN memory_links.metadata ? 'code_path'
+                         THEN jsonb_build_array(memory_links.metadata->>'code_path')
+                         ELSE '[]'::jsonb END ||
+                    CASE WHEN EXCLUDED.metadata ? 'code_path'
+                         THEN jsonb_build_array(EXCLUDED.metadata->>'code_path')
+                         ELSE '[]'::jsonb END
+                  ) p), '[]'::jsonb)
+               )
+             END,
+             weight = EXCLUDED.weight,
+             valid_from = COALESCE(EXCLUDED.valid_from, memory_links.valid_from),
+             valid_until = COALESCE(EXCLUDED.valid_until, memory_links.valid_until)
+       RETURNING id, xmax::text`, [sourceId, targetId, relation, weight, validFrom ?? null, validUntil ?? null, metadataJson]);
+        const row = result.rows[0];
+        // xmax = '0' means a fresh insert; non-zero means an update (conflict resolved)
+        return { id: row.id, created: row.xmax === '0' };
     }
     async deleteMemoryLink(sourceId, targetId, relation) {
         const pool = await this.getPool();
@@ -1296,7 +1886,8 @@ export class PostgresBackend {
         const pool = await this.getPool();
         const query = `
       SELECT id, project_id, content, tags, source, type, quality_score, quality_factors,
-             access_count, last_accessed, valid_from, valid_until, created_at,
+             access_count, last_accessed, valid_from, valid_until,
+             correction_count, is_invariant, priority_score, confidence, source_type, created_at,
              1 - (embedding <=> $1) as similarity
       FROM memories
       WHERE 1 - (embedding <=> $1) >= $2
@@ -1326,6 +1917,8 @@ export class PostgresBackend {
             priority_score: row.priority_score ?? null,
             valid_from: row.valid_from,
             valid_until: row.valid_until,
+            confidence: row.confidence ?? null,
+            source_type: (row.source_type ?? null),
             created_at: row.created_at,
             similarity: parseFloat(row.similarity),
         }));
@@ -1341,7 +1934,8 @@ export class PostgresBackend {
     async getRecentGlobalMemories(limit = 10) {
         const pool = await this.getPool();
         const result = await pool.query(`SELECT id, project_id, content, tags, source, type, quality_score, quality_factors,
-              access_count, last_accessed, valid_from, valid_until, created_at
+              access_count, last_accessed, valid_from, valid_until,
+              correction_count, is_invariant, priority_score, confidence, source_type, created_at
        FROM memories
        WHERE project_id IS NULL
          AND invalidated_by IS NULL
@@ -1366,6 +1960,8 @@ export class PostgresBackend {
             priority_score: row.priority_score ?? null,
             valid_from: row.valid_from,
             valid_until: row.valid_until,
+            confidence: row.confidence ?? null,
+            source_type: (row.source_type ?? null),
             created_at: row.created_at,
         }));
     }
@@ -1651,7 +2247,11 @@ export class PostgresBackend {
             catch (err) {
                 // Duplicate key: a link with the target (source_id, target_id, relation) already exists.
                 // Delete this link and mark the existing one as enriched instead.
-                if (err.code === '23505' && updates.relation) {
+                if (typeof err === 'object' &&
+                    err !== null &&
+                    'code' in err &&
+                    err.code === '23505' &&
+                    updates.relation) {
                     const row = await pool.query('SELECT source_id, target_id FROM memory_links WHERE id = $1', [linkId]);
                     if (row.rows.length > 0) {
                         const { source_id, target_id } = row.rows[0];
@@ -1793,11 +2393,11 @@ export class PostgresBackend {
     `, params);
         return result.rows.map((row) => ({
             event_type: row.event_type,
-            query_count: parseInt(row.query_count),
-            total_returned_tokens: parseInt(row.total_returned_tokens),
-            total_full_source_tokens: parseInt(row.total_full_source_tokens),
-            total_savings_tokens: parseInt(row.total_savings_tokens),
-            total_estimated_cost: parseFloat(row.total_estimated_cost),
+            query_count: parseInt(String(row.query_count), 10),
+            total_returned_tokens: parseInt(String(row.total_returned_tokens), 10),
+            total_full_source_tokens: parseInt(String(row.total_full_source_tokens), 10),
+            total_savings_tokens: parseInt(String(row.total_savings_tokens), 10),
+            total_estimated_cost: parseFloat(String(row.total_estimated_cost)),
         }));
     }
     async clearTokenStats() {
@@ -1864,16 +2464,16 @@ export class PostgresBackend {
         const result = await pool.query(`SELECT id, tool_name, model, query, prompt_tokens, completion_tokens, estimated_cost_usd, citations_count, has_reasoning, response_length_chars, created_at
        FROM web_search_history ${where} ORDER BY created_at DESC LIMIT $${paramIdx}`, params);
         return result.rows.map((row) => ({
-            id: parseInt(row.id),
+            id: parseInt(String(row.id), 10),
             tool_name: row.tool_name,
             model: row.model,
             query: row.query,
-            prompt_tokens: parseInt(row.prompt_tokens),
-            completion_tokens: parseInt(row.completion_tokens),
-            estimated_cost_usd: parseFloat(row.estimated_cost_usd),
-            citations_count: parseInt(row.citations_count),
+            prompt_tokens: parseInt(String(row.prompt_tokens), 10),
+            completion_tokens: parseInt(String(row.completion_tokens), 10),
+            estimated_cost_usd: parseFloat(String(row.estimated_cost_usd)),
+            citations_count: parseInt(String(row.citations_count), 10),
             has_reasoning: !!row.has_reasoning,
-            response_length_chars: parseInt(row.response_length_chars),
+            response_length_chars: parseInt(String(row.response_length_chars), 10),
             created_at: row.created_at instanceof Date ? row.created_at.toISOString() : row.created_at,
         }));
     }
@@ -2066,8 +2666,16 @@ export class PostgresBackend {
                         ? memory.validUntil.toISOString()
                         : memory.validUntil
                     : null;
-                const insertResult = await client.query(`INSERT INTO memories (project_id, content, tags, source, type, quality_score, quality_factors, embedding, valid_from, valid_until)
-           VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
+                // Validate provenance fields — mirror saveMemory() behaviour
+                const rawConfidence = memory.confidence ?? 0.5;
+                const confidence = !Number.isFinite(rawConfidence) || rawConfidence < 0 || rawConfidence > 1
+                    ? 0.5
+                    : rawConfidence;
+                const sourceType = memory.sourceType && SOURCE_TYPES.includes(memory.sourceType)
+                    ? memory.sourceType
+                    : 'human';
+                const insertResult = await client.query(`INSERT INTO memories (project_id, content, tags, source, type, quality_score, quality_factors, embedding, valid_from, valid_until, confidence, source_type)
+           VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
            RETURNING id`, [
                     this.projectId,
                     memory.content,
@@ -2079,11 +2687,18 @@ export class PostgresBackend {
                     toPgVector(memory.embedding),
                     validFromStr,
                     validUntilStr,
+                    confidence,
+                    sourceType,
                 ]);
+                const memId = insertResult.rows[0].id;
+                // Compute and store search_vector for full-text search
+                const memTokens = tokenizeDocs(memory.content);
+                const memTokenStr = memTokens.join(' ');
+                await client.query(`UPDATE memories SET search_vector = CASE WHEN $1 = '' THEN NULL ELSE to_tsvector('simple', $1) END WHERE id = $2`, [memTokenStr, memId]);
                 results.push({
                     index: i,
                     isDuplicate: false,
-                    id: insertResult.rows[0].id,
+                    id: memId,
                     reason: 'saved',
                 });
                 saved++;
@@ -2134,6 +2749,29 @@ export class PostgresBackend {
             stale_count: parseInt(stale.rows[0].count),
         };
     }
+    async getMemoryHealth() {
+        const pool = await this.getPool();
+        const scopeCond = this.projectId
+            ? '(LOWER(project_id) = $1 OR project_id IS NULL)'
+            : 'project_id IS NULL';
+        const scopeParams = this.projectId ? [this.projectId] : [];
+        const result = await pool.query(`SELECT
+         COUNT(*) as total,
+         SUM(CASE WHEN access_count = 0 THEN 1 ELSE 0 END) as never_accessed,
+         SUM(CASE WHEN EXTRACT(EPOCH FROM (now() - COALESCE(last_accessed::timestamp, created_at))) / 86400 > 90
+             THEN 1 ELSE 0 END) as stale_unused_90d,
+         AVG(EXTRACT(EPOCH FROM (now() - created_at)) / 86400) as avg_age_days,
+         AVG(access_count) as avg_access
+       FROM memories WHERE ${scopeCond} AND invalidated_by IS NULL`, scopeParams);
+        const row = result.rows[0];
+        return {
+            total: parseInt(row.total) || 0,
+            never_accessed: parseInt(row.never_accessed) || 0,
+            stale_unused_90d: parseInt(row.stale_unused_90d) || 0,
+            avg_age_days: parseFloat(row.avg_age_days ?? '0') || 0,
+            avg_access: parseFloat(row.avg_access ?? '0') || 0,
+        };
+    }
     async deleteMemoriesOlderThan(date) {
         const pool = await this.getPool();
         const result = await pool.query('DELETE FROM memories WHERE created_at < $1', [
@@ -2169,7 +2807,8 @@ export class PostgresBackend {
             : 'project_id IS NULL';
         const scopeParams = this.projectId ? [this.projectId] : [];
         const result = await pool.query(`SELECT id, project_id, content, tags, source, type, quality_score, quality_factors,
-              access_count, last_accessed, valid_from, valid_until, created_at,
+              access_count, last_accessed, valid_from, valid_until,
+              correction_count, is_invariant, priority_score, confidence, source_type, created_at,
               1 - (embedding <=> $1) as similarity
        FROM memories
        WHERE created_at <= $2
@@ -2198,8 +2837,10 @@ export class PostgresBackend {
             priority_score: row.priority_score ?? null,
             valid_from: row.valid_from,
             valid_until: row.valid_until,
+            confidence: row.confidence ?? null,
+            source_type: (row.source_type ?? null),
             created_at: row.created_at,
-            similarity: parseFloat(row.similarity),
+            similarity: parseFloat(String(row.similarity)),
         }));
     }
     async getConsolidationHistory(limit = 20) {
@@ -2288,7 +2929,11 @@ export class PostgresBackend {
         return result.rows.map((row) => ({
             id: row.id,
             content: row.content,
-            tags: row.tags ?? [],
+            tags: Array.isArray(row.tags)
+                ? row.tags
+                : typeof row.tags === 'string'
+                    ? JSON.parse(row.tags)
+                    : [],
             source: row.source,
             type: row.type,
             projectId: row.project_id,