ahok-skill 1.3.1

package/src/core/vector/postgres.ts

@@ -0,0 +1,61 @@
+import { VectorStore } from "../vector_store";
+import { cosineSimilarity, bufferToVector, vectorToBuffer } from "../../memory/embed";
+
+export interface DbOps {
+    run_async: (sql: string, params?: any[]) => Promise<void>;
+    get_async: (sql: string, params?: any[]) => Promise<any>;
+    all_async: (sql: string, params?: any[]) => Promise<any[]>;
+}
+
+export class PostgresVectorStore implements VectorStore {
+    private table: string;
+
+    constructor(private db: DbOps, tableName: string = "vectors") {
+        this.table = tableName;
+    }
+
+    async storeVector(id: string, sector: string, vector: number[], dim: number, user_id?: string): Promise<void> {
+        console.error(`[Vector] Storing ID: ${id}, Sector: ${sector}, Dim: ${dim}`);
+        const v = vectorToBuffer(vector);
+        const sql = `insert into ${this.table}(id,sector,user_id,v,dim) values($1,$2,$3,$4,$5) on conflict(id,sector) do update set user_id=excluded.user_id,v=excluded.v,dim=excluded.dim`;
+        await this.db.run_async(sql, [id, sector, user_id || "anonymous", v, dim]);
+    }
+
+    async deleteVector(id: string, sector: string): Promise<void> {
+        await this.db.run_async(`delete from ${this.table} where id=$1 and sector=$2`, [id, sector]);
+    }
+
+    async deleteVectors(id: string): Promise<void> {
+        await this.db.run_async(`delete from ${this.table} where id=$1`, [id]);
+    }
+
+    async searchSimilar(sector: string, queryVec: number[], topK: number): Promise<Array<{ id: string; score: number }>> {
+        // Postgres implementation (in-memory cosine sim for now, as per original)
+        const rows = await this.db.all_async(`select id,v,dim from ${this.table} where sector=$1`, [sector]);
+        console.error(`[Vector] Search Sector: ${sector}, Found ${rows.length} rows.`);
+        const sims: Array<{ id: string; score: number }> = [];
+        for (const row of rows) {
+            const vec = bufferToVector(row.v);
+            const sim = cosineSimilarity(queryVec, vec);
+            sims.push({ id: row.id, score: sim });
+        }
+        sims.sort((a, b) => b.score - a.score);
+        return sims.slice(0, topK);
+    }
+
+    async getVector(id: string, sector: string): Promise<{ vector: number[]; dim: number } | null> {
+        const row = await this.db.get_async(`select v,dim from ${this.table} where id=$1 and sector=$2`, [id, sector]);
+        if (!row) return null;
+        return { vector: bufferToVector(row.v), dim: row.dim };
+    }
+
+    async getVectorsById(id: string): Promise<Array<{ sector: string; vector: number[]; dim: number }>> {
+        const rows = await this.db.all_async(`select sector,v,dim from ${this.table} where id=$1`, [id]);
+        return rows.map(row => ({ sector: row.sector, vector: bufferToVector(row.v), dim: row.dim }));
+    }
+
+    async getVectorsBySector(sector: string): Promise<Array<{ id: string; vector: number[]; dim: number }>> {
+        const rows = await this.db.all_async(`select id,v,dim from ${this.table} where sector=$1`, [sector]);
+        return rows.map(row => ({ id: row.id, vector: bufferToVector(row.v), dim: row.dim }));
+    }
+}

package/src/core/vector/valkey.ts

@@ -0,0 +1,261 @@
+import { VectorStore } from "../vector_store";
+import Redis from "ioredis";
+import { env } from "../cfg";
+import { vectorToBuffer, bufferToVector } from "../../memory/embed";
+
+export class ValkeyVectorStore implements VectorStore {
+    private client: Redis;
+
+    constructor() {
+        this.client = new Redis({
+            host: env.valkey_host || "localhost",
+            port: env.valkey_port || 6379,
+            password: env.valkey_password,
+        });
+    }
+
+    private getKey(id: string, sector: string): string {
+        return `vec:${sector}:${id}`;
+    }
+
+    async storeVector(id: string, sector: string, vector: number[], dim: number, user_id?: string): Promise<void> {
+        const key = this.getKey(id, sector);
+        const buf = vectorToBuffer(vector);
+        // Store as Hash: v (blob), dim (int), user_id (string)
+        await this.client.hset(key, {
+            v: buf,
+            dim: dim,
+            user_id: user_id || "anonymous",
+            id: id,
+            sector: sector
+        });
+    }
+
+    async deleteVector(id: string, sector: string): Promise<void> {
+        const key = this.getKey(id, sector);
+        await this.client.del(key);
+    }
+
+    async deleteVectors(id: string): Promise<void> {
+        // This is inefficient in Redis without an index on ID across sectors.
+        // We might need to track which sectors an ID has.
+        // For now, we can scan or just assume we know the sectors?
+        // The interface implies we delete ALL vectors for this ID.
+        // In Postgres we did `delete from vectors where id=$1`.
+        // In Redis, we might need to know the sectors.
+        // Or we can use `keys vec:*:${id}` which is slow.
+        // Better approach: maintain a set of sectors for each ID?
+        // Or just iterate over known sectors (from hsg.ts sectors list).
+        // I'll import `sectors` from `../../memory/hsg`? No, circular dependency risk.
+        // I'll use a scan for now as it's safer, or just accept it might be slow.
+        // Actually, `keys` is blocking. `scan` is better.
+        // But `id` is at the end of the key `vec:{sector}:{id}`.
+        // Pattern: `vec:*:${id}`.
+
+        let cursor = "0";
+        do {
+            const res = await this.client.scan(cursor, "MATCH", `vec:*:${id}`, "COUNT", 100);
+            cursor = res[0];
+            const keys = res[1];
+            if (keys.length) await this.client.del(...keys);
+        } while (cursor !== "0");
+    }
+
+    async searchSimilar(sector: string, queryVec: number[], topK: number): Promise<Array<{ id: string; score: number }>> {
+        // Try to use FT.SEARCH if index exists.
+        // Index name assumption: `idx:{sector}`
+        // Query: `*=>[KNN {k} @v $blob AS score]`
+        const indexName = `idx:${sector}`;
+        const blob = vectorToBuffer(queryVec);
+
+        try {
+            // Check if index exists (optional, or just try search)
+            // We'll try the search.
+            // FT.SEARCH idx:sector "*=>[KNN 10 @v $blob AS score]" PARAMS 2 blob "\x..." DIALECT 2
+            const res = await this.client.call(
+                "FT.SEARCH",
+                indexName,
+                `*=>[KNN ${topK} @v $blob AS score]`,
+                "PARAMS",
+                "2",
+                "blob",
+                blob,
+                "DIALECT",
+                "2"
+            ) as any[];
+
+            // Parse result
+            // [total_results, key1, [field1, val1, ...], key2, ...]
+            // We need to parse the array.
+            const count = res[0];
+            const results: Array<{ id: string; score: number }> = [];
+            for (let i = 1; i < res.length; i += 2) {
+                const key = res[i] as string; // e.g. vec:semantic:123
+                const fields = res[i + 1] as any[];
+                let id = "";
+                let score = 0;
+                // fields is array [k, v, k, v...]
+                for (let j = 0; j < fields.length; j += 2) {
+                    const f = fields[j];
+                    const v = fields[j + 1];
+                    if (f === "id") id = v;
+                    if (f === "score") score = 1 - (parseFloat(v) / 2); // Cosine distance to similarity?
+                    // Valkey HNSW usually returns distance.
+                    // If metric is COSINE, distance is 1 - cosine_sim.
+                    // So sim = 1 - dist.
+                    // Wait, standard RediSearch KNN distance depends on metric.
+                    // Assuming COSINE metric.
+                }
+                // If id not in fields, extract from key
+                if (!id) {
+                    const parts = key.split(":");
+                    id = parts[parts.length - 1];
+                }
+                // If score not found (should be there as 'score'), default 0
+                // Actually, 'score' is the alias we gave.
+                // Wait, in FT.SEARCH response, the score is returned if we ask for it?
+                // Yes, "AS score" puts it in the fields.
+
+                // Correction: RediSearch returns distance by default for KNN?
+                // Yes, but we aliased it "AS score".
+                // So it should be in the fields as "score".
+
+                // Distance conversion:
+                // If distance is d, and we want cosine similarity.
+                // Cosine distance = 1 - cosine similarity.
+                // So similarity = 1 - distance.
+                results.push({ id, score: 1 - parseFloat(fields.find((f: any, idx: number) => f === "score" && idx % 2 === 0 ? false : fields[idx - 1] === "score") || "0") });
+                // The find logic above is tricky.
+                // Let's just loop.
+            }
+
+            // Fix score parsing loop
+            results.length = 0; // clear
+            for (let i = 1; i < res.length; i += 2) {
+                const key = res[i] as string;
+                const fields = res[i + 1] as any[];
+                let id = "";
+                let dist = 0;
+                for (let j = 0; j < fields.length; j += 2) {
+                    if (fields[j] === "id") id = fields[j + 1];
+                    if (fields[j] === "score") dist = parseFloat(fields[j + 1]);
+                }
+                if (!id) id = key.split(":").pop()!;
+                results.push({ id, score: 1 - dist });
+            }
+
+            return results;
+
+        } catch (e) {
+            console.warn(`[Valkey] FT.SEARCH failed for ${sector}, falling back to scan (slow):`, e);
+            // Fallback: Scan all vectors in sector and compute cosine sim
+            // This is very slow but ensures correctness if index is missing.
+            let cursor = "0";
+            const allVecs: Array<{ id: string; vector: number[] }> = [];
+            do {
+                const res = await this.client.scan(cursor, "MATCH", `vec:${sector}:*`, "COUNT", 100);
+                cursor = res[0];
+                const keys = res[1];
+                if (keys.length) {
+                    // Pipeline get all
+                    const pipe = this.client.pipeline();
+                    keys.forEach(k => pipe.hget(k, "v"));
+                    const buffers = await pipe.exec();
+                    buffers?.forEach((b, idx) => {
+                        if (b && b[1]) {
+                            const buf = b[1] as Buffer;
+                            const id = keys[idx].split(":").pop()!;
+                            allVecs.push({ id, vector: bufferToVector(buf) });
+                        }
+                    });
+                }
+            } while (cursor !== "0");
+
+            const sims = allVecs.map(v => ({
+                id: v.id,
+                score: this.cosineSimilarity(queryVec, v.vector)
+            }));
+            sims.sort((a, b) => b.score - a.score);
+            return sims.slice(0, topK);
+        }
+    }
+
+    private cosineSimilarity(a: number[], b: number[]) {
+        if (a.length !== b.length) return 0;
+        let dot = 0, na = 0, nb = 0;
+        for (let i = 0; i < a.length; i++) {
+            dot += a[i] * b[i];
+            na += a[i] * a[i];
+            nb += b[i] * b[i];
+        }
+        return na && nb ? dot / (Math.sqrt(na) * Math.sqrt(nb)) : 0;
+    }
+
+    async getVector(id: string, sector: string): Promise<{ vector: number[]; dim: number } | null> {
+        const key = this.getKey(id, sector);
+        const res = await this.client.hmget(key, "v", "dim");
+        if (!res[0]) return null;
+        return {
+            vector: bufferToVector(res[0] as unknown as Buffer),
+            dim: parseInt(res[1] as string)
+        };
+    }
+
+    async getVectorsById(id: string): Promise<Array<{ sector: string; vector: number[]; dim: number }>> {
+        // Scan for vec:*:{id}
+        const results: Array<{ sector: string; vector: number[]; dim: number }> = [];
+        let cursor = "0";
+        do {
+            const res = await this.client.scan(cursor, "MATCH", `vec:*:${id}`, "COUNT", 100);
+            cursor = res[0];
+            const keys = res[1];
+            if (keys.length) {
+                const pipe = this.client.pipeline();
+                keys.forEach(k => pipe.hmget(k, "v", "dim"));
+                const res = await pipe.exec();
+                res?.forEach((r, idx) => {
+                    if (r && r[1]) {
+                        const [v, dim] = r[1] as [Buffer, string];
+                        const key = keys[idx];
+                        const parts = key.split(":");
+                        const sector = parts[1];
+                        results.push({
+                            sector,
+                            vector: bufferToVector(v),
+                            dim: parseInt(dim)
+                        });
+                    }
+                });
+            }
+        } while (cursor !== "0");
+        return results;
+    }
+
+    async getVectorsBySector(sector: string): Promise<Array<{ id: string; vector: number[]; dim: number }>> {
+        const results: Array<{ id: string; vector: number[]; dim: number }> = [];
+        let cursor = "0";
+        do {
+            const res = await this.client.scan(cursor, "MATCH", `vec:${sector}:*`, "COUNT", 100);
+            cursor = res[0];
+            const keys = res[1];
+            if (keys.length) {
+                const pipe = this.client.pipeline();
+                keys.forEach(k => pipe.hmget(k, "v", "dim"));
+                const res = await pipe.exec();
+                res?.forEach((r, idx) => {
+                    if (r && r[1]) {
+                        const [v, dim] = r[1] as [Buffer, string];
+                        const key = keys[idx];
+                        const id = key.split(":").pop()!;
+                        results.push({
+                            id,
+                            vector: bufferToVector(v),
+                            dim: parseInt(dim)
+                        });
+                    }
+                });
+            }
+        } while (cursor !== "0");
+        return results;
+    }
+}

package/src/core/vector_store.ts

@@ -0,0 +1,9 @@
+export interface VectorStore {
+    storeVector(id: string, sector: string, vector: number[], dim: number, user_id?: string): Promise<void>;
+    deleteVector(id: string, sector: string): Promise<void>;
+    deleteVectors(id: string): Promise<void>;
+    searchSimilar(sector: string, queryVec: number[], topK: number): Promise<Array<{ id: string; score: number }>>;
+    getVector(id: string, sector: string): Promise<{ vector: number[]; dim: number } | null>;
+    getVectorsById(id: string): Promise<Array<{ sector: string; vector: number[]; dim: number }>>;
+    getVectorsBySector(sector: string): Promise<Array<{ id: string; vector: number[]; dim: number }>>;
+}

package/src/index.ts

@@ -0,0 +1,5 @@
+// Export core functionality for use as a package
+export * from "./core/memory";
+export * from "./server/index";
+export * from "./ops/ingest";
+export * as sources from "./sources";