@chatcops/store-postgres 0.0.0-stage → 0.2.0

package/dist/vectors.js

@@ -0,0 +1,147 @@
+import { segmentWords } from '@chatcops/core';
+/** Postgres pgvector-backed storage for embedded page chunks. */
+export class PostgresVectorStore {
+    dimensions;
+    db;
+    table;
+    constructor(dimensions, db, table) {
+        this.dimensions = dimensions;
+        this.db = db;
+        this.table = table;
+        if (!Number.isInteger(dimensions) || dimensions < 1 || dimensions > 16_000) {
+            throw new Error(`dimensions must be a whole number from 1 to 16000; got ${dimensions}.`);
+        }
+    }
+    async upsert(records) {
+        for (const record of records)
+            this.checkVector(record.vector, `record ${record.id}`);
+        for (const record of records) {
+            const words = segmentWords(record.text).join(' ');
+            await this.db.query(`INSERT INTO ${this.table} (id, source_id, source, text, metadata, embedding, keyword_vector)
+         VALUES ($1, $2, $3::text::jsonb, $4, $5::text::jsonb, $6::text::vector, to_tsvector('simple', $7))
+         ON CONFLICT (id) DO UPDATE SET source_id = EXCLUDED.source_id, source = EXCLUDED.source,
+           text = EXCLUDED.text, metadata = EXCLUDED.metadata, embedding = EXCLUDED.embedding,
+           keyword_vector = EXCLUDED.keyword_vector`, [record.id, record.source.id, JSON.stringify(record.source), record.text, JSON.stringify(record.metadata ?? {}), toVector(record.vector), words]);
+        }
+    }
+    async query({ vector, topK, filter }) {
+        this.checkVector(vector, 'the query');
+        const limit = limitFor(topK);
+        if (!limit)
+            return [];
+        const params = [toVector(vector)];
+        const where = metadataWhere(filter, params);
+        params.push(limit);
+        const sql = `WITH nearest AS MATERIALIZED (
+      SELECT id, text, source, metadata, embedding <=> $1::text::vector AS distance
+      FROM ${this.table} ${where}
+      ORDER BY embedding <=> $1::text::vector
+      LIMIT $${params.length}
+    )
+    SELECT id, text, source, metadata, coalesce(1 - distance, 0)::float8 AS score
+    FROM nearest ORDER BY distance + 0, id`;
+        const hasFilter = Object.keys(filter ?? {}).length > 0;
+        const result = hasFilter
+            ? await inTransaction(this.db, async (tx) => {
+                await tx.query("SET LOCAL hnsw.iterative_scan = 'relaxed_order'");
+                return tx.query(sql, params);
+            })
+            : await this.db.query(sql, params);
+        return result.rows.map(asScoredRecord);
+    }
+    async keywordQuery({ text, topK, filter }) {
+        const limit = limitFor(topK);
+        const words = segmentWords(text).join(' ');
+        if (!limit || !words)
+            return [];
+        const params = [words];
+        const where = metadataWhere(filter, params);
+        params.push(limit);
+        const { rows } = await this.db.query(
+        // Any of the words, not all of them (plainto_tsquery would AND them):
+        // a question has more words than any one page. Each lexeme is quoted,
+        // so nothing in the text is read as tsquery syntax.
+        `WITH search AS (
+         SELECT to_tsquery('simple', string_agg(quote_literal(lexeme), ' | ')) AS query
+         FROM unnest(tsvector_to_array(to_tsvector('simple', $1::text))) AS lexeme
+       )
+       SELECT id, text, source, metadata, ts_rank(keyword_vector, search.query)::float8 AS score
+       FROM ${this.table}, search
+       WHERE keyword_vector @@ search.query${where ? ` AND ${where.slice('WHERE '.length)}` : ''}
+       ORDER BY score DESC, id LIMIT $${params.length}`, params);
+        return rows.map(asScoredRecord);
+    }
+    async deleteBySource(sourceId) {
+        await this.db.query(`DELETE FROM ${this.table} WHERE source_id = $1`, [sourceId]);
+    }
+    checkVector(vector, what) {
+        if (vector.length !== this.dimensions) {
+            throw new Error(`The vector for ${what} has ${vector.length} dimensions, but this store holds ${this.dimensions}.`);
+        }
+        if (!vector.every(Number.isFinite))
+            throw new Error(`The vector for ${what} has a value that isn't a finite number.`);
+    }
+}
+function toVector(vector) {
+    return `[${Array.from(vector).join(',')}]`;
+}
+function limitFor(topK) {
+    return Number.isFinite(topK) && topK >= 1 ? Math.floor(topK) : 0;
+}
+function metadataWhere(filter, params) {
+    if (!filter)
+        return '';
+    const clauses = [];
+    for (const [key, value] of Object.entries(filter)) {
+        params.push(key);
+        const keyParam = `$${params.length}`;
+        if (Array.isArray(value)) {
+            params.push(JSON.stringify(value));
+            clauses.push(`EXISTS (SELECT 1 FROM jsonb_array_elements($${params.length}::text::jsonb) AS wanted(value) WHERE metadata -> ${keyParam}::text = wanted.value)`);
+        }
+        else {
+            params.push(JSON.stringify(value));
+            clauses.push(`metadata -> ${keyParam}::text = $${params.length}::text::jsonb`);
+        }
+    }
+    return clauses.length ? `WHERE ${clauses.join(' AND ')}` : '';
+}
+function asScoredRecord(row) {
+    return {
+        id: row.id,
+        text: row.text,
+        source: row.source,
+        ...(row.metadata && Object.keys(row.metadata).length ? { metadata: row.metadata } : {}),
+        score: Number(row.score),
+    };
+}
+async function inTransaction(db, run) {
+    if (db.transaction)
+        return db.transaction(run);
+    const pool = db;
+    const fromPool = typeof pool.connect === 'function';
+    const client = fromPool ? await pool.connect() : db;
+    if (!fromPool && !client.connection) {
+        throw new Error('Filtered pgvector queries need a transaction-capable database connection for HNSW iterative scans.');
+    }
+    try {
+        await client.query('BEGIN');
+        const result = await run(client);
+        await client.query('COMMIT');
+        return result;
+    }
+    catch (error) {
+        try {
+            await client.query('ROLLBACK');
+        }
+        catch {
+            // Keep the query error if the connection has already been lost.
+        }
+        throw error;
+    }
+    finally {
+        if (fromPool)
+            client.release?.();
+    }
+}
+//# sourceMappingURL=vectors.js.map

package/dist/vectors.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"vectors.js","sourceRoot":"","sources":["../src/vectors.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAoB9C,iEAAiE;AACjE,MAAM,OAAO,mBAAmB;IAEnB;IACQ;IACA;IAHnB,YACW,UAAkB,EACV,EAAa,EACb,KAAa;QAFrB,eAAU,GAAV,UAAU,CAAQ;QACV,OAAE,GAAF,EAAE,CAAW;QACb,UAAK,GAAL,KAAK,CAAQ;QAE9B,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,UAAU,GAAG,CAAC,IAAI,UAAU,GAAG,MAAM,EAAE,CAAC;YAC3E,MAAM,IAAI,KAAK,CAAC,0DAA0D,UAAU,GAAG,CAAC,CAAC;QAC3F,CAAC;IACH,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,OAAuB;QAClC,KAAK,MAAM,MAAM,IAAI,OAAO;YAAE,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,EAAE,UAAU,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC;QACrF,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,KAAK,GAAG,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAClD,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CACjB,eAAe,IAAI,CAAC,KAAK;;;;oDAImB,EAC5C,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,QAAQ,IAAI,EAAE,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,CAAC,CACjJ,CAAC;QACJ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAe;QAC/C,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;QACtC,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC7B,IAAI,CAAC,KAAK;YAAE,OAAO,EAAE,CAAC;QACtB,MAAM,MAAM,GAAc,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;QAC7C,MAAM,KAAK,GAAG,aAAa,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC5C,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnB,MAAM,GAAG,GAAG;;aAEH,IAAI,CAAC,KAAK,IAAI,KAAK;;eAEjB,MAAM,CAAC,MAAM;;;2CAGe,CAAC;QACxC,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;QACvD,MAAM,MAAM,GAAG,SAAS;YACtB,CAAC,CAAC,MAAM,aAAa,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE;gBACxC,MAAM,EAAE,CAAC,KAAK,CAAC,iDAAiD,CAAC,CAAC;gBAClE,OAAO,EAAE,CAAC,KAAK,CAAM,GAAG,EAAE,MAAM,CAAC,CAAC;YACpC,CAAC,CAAC;YACJ,CAAC,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAAM,GAAG,EAAE,MAAM,CAAC,CAAC;QAC1C,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;IACzC,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAgB;QACrD,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC7B,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC3C,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK;YAAE,OAAO,EAAE,CAAC;QAChC,MAAM,MAAM,GAAc,CAAC,KAAK,CAAC,CAAC;QAClC,MAAM,KAAK,GAAG,aAAa,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC5C,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnB,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK;QAClC,sEAAsE;QACtE,sEAAsE;QACtE,oDAAoD;QACpD;;;;;cAKQ,IAAI,CAAC,KAAK;6CACqB,KAAK,CAAC,CAAC,CAAC,QAAQ,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE;wCACxD,MAAM,CAAC,MAAM,EAAE,EACjD,MAAM,CACP,CAAC;QACF,OAAO,IAAI,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;IAClC,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,QAAgB;QACnC,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,eAAe,IAAI,CAAC,KAAK,uBAAuB,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC;IACpF,CAAC;IAEO,WAAW,CAAC,MAAoB,EAAE,IAAY;QACpD,IAAI,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC,UAAU,EAAE,CAAC;YACtC,MAAM,IAAI,KAAK,CAAC,kBAAkB,IAAI,QAAQ,MAAM,CAAC,MAAM,qCAAqC,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC;QACtH,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,kBAAkB,IAAI,0CAA0C,CAAC,CAAC;IACxH,CAAC;CACF;AAED,SAAS,QAAQ,CAAC,MAAoB;IACpC,OAAO,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;AAC7C,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY;IAC5B,OAAO,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACnE,CAAC;AAED,SAAS,aAAa,CAAC,MAAkC,EAAE,MAAiB;IAC1E,IAAI,CAAC,MAAM;QAAE,OAAO,EAAE,CAAC;IACvB,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAClD,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACjB,MAAM,QAAQ,GAAG,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QACrC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YACzB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;YACnC,OAAO,CAAC,IAAI,CAAC,+CAA+C,MAAM,CAAC,MAAM,qDAAqD,QAAQ,wBAAwB,CAAC,CAAC;QAClK,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;YACnC,OAAO,CAAC,IAAI,CAAC,eAAe,QAAQ,aAAa,MAAM,CAAC,MAAM,eAAe,CAAC,CAAC;QACjF,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;AAChE,CAAC;AAED,SAAS,cAAc,CAAC,GAAQ;IAC9B,OAAO;QACL,EAAE,EAAE,GAAG,CAAC,EAAE;QACV,IAAI,EAAE,GAAG,CAAC,IAAI;QACd,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,GAAG,CAAC,GAAG,CAAC,QAAQ,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvF,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC;KACzB,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,aAAa,CAAI,EAAa,EAAE,GAAkC;IAC/E,IAAI,EAAE,CAAC,WAAW;QAAE,OAAO,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAC/C,MAAM,IAAI,GAAG,EAAY,CAAC;IAC1B,MAAM,QAAQ,GAAG,OAAO,IAAI,CAAC,OAAO,KAAK,UAAU,CAAC;IACpD,MAAM,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAE,EAAmB,CAAC;IACtE,IAAI,CAAC,QAAQ,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;QACpC,MAAM,IAAI,KAAK,CAAC,oGAAoG,CAAC,CAAC;IACxH,CAAC;IACD,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC5B,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC7B,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QACjC,CAAC;QAAC,MAAM,CAAC;YACP,gEAAgE;QAClE,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;YAAS,CAAC;QACT,IAAI,QAAQ;YAAE,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;IACnC,CAAC;AACH,CAAC"}

package/package.json

@@ -1,6 +1,67 @@
 {
   "name": "@chatcops/store-postgres",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
+  "version": "0.2.0",
+  "description": "Postgres storage for ChatCops: conversations, counters, keys and analytics events",
+  "keywords": [
+    "chatcops",
+    "chatbot",
+    "postgres",
+    "neon",
+    "storage"
+  ],
+  "homepage": "https://chat.codercops.com",
+  "bugs": "https://github.com/codercops/chatcops/issues",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "bin": {
+    "chatcops-postgres": "./dist/bin.js"
+  },
+  "files": [
+    "dist"
+  ],
+  "peerDependencies": {
+    "pg": "^8.13.0",
+    "@chatcops/core": "^0.6.0"
+  },
+  "peerDependenciesMeta": {
+    "pg": {
+      "optional": true
+    }
+  },
+  "devDependencies": {
+    "@types/node": "^22",
+    "@types/pg": "^8.11.10",
+    "pg": "^8.13.1",
+    "postgres": "^3.4.5",
+    "typescript": "^5.7.0",
+    "vitest": "^4.1.11",
+    "@chatcops/core": "0.6.0"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/codercops/chatcops.git",
+    "directory": "packages/store-postgres"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "license": "MIT",
+  "engines": {
+    "node": ">=22"
+  },
+  "sideEffects": false,
+  "scripts": {
+    "build": "tsc",
+    "typecheck": "tsc --noEmit",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "clean": "rm -rf dist *.tsbuildinfo"
+  }
 }