halbot 1995.1.58 → 1995.1.61

package/index.mjs

@@ -11,11 +11,27 @@ const init = async (options = {}) => {
     const info = bot.lines([
         `[${bot.BOT} ${pkg.title}](${pkg.homepage})`, pkg.description
     ]);
-    // use google's search if google is enabled
-    options.googleApiKey && options.googleCx && await web.initSearch({
-        provider: 'GOOGLE', apiKey: options.googleApiKey, cx: options.googleCx,
-    });
-    // use openrouter's AI models, embedding if OpenRouter is enabled
+    // use google rerank
+    if (options?.googleCredentials && options.googleProjectId) {
+        opts = {
+            provider: 'GOOGLE', credentials: options.googleCredentials,
+            projectId: options.googleProjectId,
+        };
+        if (!_rerank) {
+            await rag.initReranker(opts);
+            _rerank = rag.rerank;
+        }
+    }
+    // use jina search and reranker
+    if (options.jinaApiKey) {
+        opts = { provider: 'JINA', apiKey: options.jinaApiKey };
+        await web.initSearch(opts);
+        if (!_rerank) {
+            await rag.initReranker(opts);
+            _rerank = rag.rerank;
+        }
+    }
+    // use openrouter's AI models
     if (options.openrouterApiKey) {
         opts = { provider: 'OPENROUTER', apiKey: options.openrouterApiKey };
         await alan.init({
@@ -27,17 +43,8 @@ const init = async (options = {}) => {
             await rag.initEmbedding(opts);
             _embed = rag.embed;
         }
-    }
-    // use google's imagen, veo, tts if google is enabled
-    if (options.googleApiKey) {
-        opts = { provider: 'GOOGLE', apiKey: options.googleApiKey };
-        await alan.init({
-            ...opts, model: options.googleModel || '*',
-            priority: options.googlePriority, ...options,
-        });
-    }
-    // use openai's embedding, tts if openai is enabled, and google is not
-    if (options.openaiApiKey) {
+        // use openai models, embedding if openrouter is disabled
+    } else if (options.openaiApiKey && options.openaiModels) {
         opts = { provider: 'OPENAI', apiKey: options.openaiApiKey };
         await alan.init({
             ...opts, model: options.openaiModel || '*',
@@ -48,31 +55,15 @@ const init = async (options = {}) => {
             _embed = rag.embed;
         }
     }
-    // use google rerank if google is enabled
-    if (options?.googleCredentials && options.googleProjectId) {
-        opts = {
-            provider: 'GOOGLE', credentials: options.googleCredentials,
-            projectId: options.googleProjectId,
-        };
-        if (!_rerank) {
-            await rag.initReranker(opts);
-            _rerank = rag.rerank;
-        }
-    }
-    // init other ai providers
-    options.siliconflowApiKey && await alan.init({
-        provider: 'SILICONFLOW', apiKey: options.siliconflowApiKey,
-        model: options.siliconflowModel || '*',
-        priority: options.siliconflowPriority, ...options,
-    });
-    if (options.jinaApiKey) {
-        opts = { provider: 'JINA', apiKey: options.jinaApiKey };
-        await web.initSearch(opts);
-        if (!_rerank) {
-            await rag.initReranker(opts);
-            _rerank = rag.rerank;
-        }
+    // use google's veo
+    if (options.googleApiKey) {
+        opts = { provider: 'GOOGLE', apiKey: options.googleApiKey };
+        await alan.init({
+            ...opts, model: options.googleModel || '*',
+            priority: options.googlePriority, ...options,
+        });
     }
+    // use ollama
     if (options?.ollamaEnabled || options?.ollamaEndpoint) {
         await alan.init({
             provider: 'OLLAMA', model: options?.ollamaModel || '*',

package/lib/hal.mjs

@@ -20,6 +20,9 @@ const { __filename } = utilitas.__(import.meta.url);
 const workdir = path.dirname(__filename);
 const getPath = (subPath) => { return path.join(workdir, subPath || ''); };
 const table = 'utilitas_hal_events';
+const vectorIndex = `${table}_distilled_vector_index`;
+const vectorIndexLists = 1000;
+const vectorIndexProbes = 20;
 const log = (c, o) => utilitas.log(c, import.meta.url, { time: 1, ...o || {} });
 const [end] = [bot.end];
 const uList = arr => bot.lines(arr.map(x => `- ${x}`));
@@ -65,6 +68,8 @@ const initSql = {
         ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4`), [table],
     ]],
     [dbio.POSTGRESQL]: [[
+        'CREATE EXTENSION IF NOT EXISTS vchord CASCADE',
+    ], [
         dbio.cleanSql(`CREATE TABLE IF NOT EXISTS ${table} (
             id               SERIAL       PRIMARY KEY,
             bot_id           BIGINT       NOT NULL,
@@ -77,7 +82,7 @@ const initSql = {
             response_text    TEXT         NOT NULL,
             collected        TEXT         NOT NULL,
             distilled        TEXT         NOT NULL,
-            distilled_vector VECTOR(768)  NOT NULL,
+            distilled_vector RABITQ8(768) NOT NULL,
             token            VARCHAR(255) NOT NULL DEFAULT '',
             created_at       TIMESTAMP    NOT NULL DEFAULT CURRENT_TIMESTAMP,
             updated_at       TIMESTAMP    NOT NULL DEFAULT CURRENT_TIMESTAMP
@@ -102,8 +107,6 @@ const initSql = {
         `CREATE INDEX IF NOT EXISTS ${table}_collected_index ON ${table} USING GIN(to_tsvector('english', collected))`,
     ], [
         `CREATE INDEX IF NOT EXISTS ${table}_distilled_index ON ${table} USING GIN(to_tsvector('english', distilled))`,
-    ], [
-        `CREATE INDEX IF NOT EXISTS ${table}_distilled_vector_index ON ${table} USING hnsw(distilled_vector vector_cosine_ops)`,
     ], [
         `CREATE INDEX IF NOT EXISTS ${table}_token_index ON ${table} (token)`,
     ], [
@@ -115,6 +118,24 @@ const initSql = {
 
 let _;
 
+const ensurePostgresqlRagIndex = async (client) => {
+    const index = await client.query(
+        `SELECT indexdef FROM pg_indexes
+        WHERE tablename = $1 AND indexname = $2`,
+        [table, vectorIndex]
+    );
+    if (index?.length) { return; }
+    await client.query(`CREATE INDEX IF NOT EXISTS ${vectorIndex}
+        ON ${table} USING vchordrq (distilled_vector rabitq8_cosine_ops)
+        WITH (options = $$
+        [build.internal]
+        lists = [${vectorIndexLists}]
+        spherical_centroids = true
+        build_threads = 8
+        $$, probes = '${vectorIndexProbes}')`
+    );
+};
+
 const newCommand = (command, description) => ({
     command: utilitas.ensureString(command, { case: 'SNAKE' }).slice(0, COMMAND_LENGTH),
     description: utilitas.trim(description).slice(0, COMMAND_DESCRIPTION_LENGTH),
@@ -219,6 +240,9 @@ const init = async (options) => {
                         for (const act of initSql[_.storage?.provider]) {
                             dbResult.push(await _.storage.client.query(...act));
                         }
+                        if (_.storage?.provider === dbio.POSTGRESQL) {
+                            await ensurePostgresqlRagIndex(_.storage.client);
+                        }
                     } catch (e) { console.error(e); }
                 }
             }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "halbot",
   "description": "Just another AI powered Telegram bot, which is simple design, easy to use, extendable and fun.",
-  "version": "1995.1.58",
+  "version": "1995.1.61",
   "private": false,
   "homepage": "https://github.com/Leask/halbot",
   "type": "module",
@@ -31,27 +31,27 @@
   "dependencies": {
     "@ffmpeg-installer/ffmpeg": "^1.1.0",
     "@ffprobe-installer/ffprobe": "^2.1.2",
-    "@google-cloud/discoveryengine": "^2.5.2",
-    "@google/genai": "^1.43.0",
+    "@google-cloud/discoveryengine": "^2.6.0",
+    "@google/genai": "^1.51.0",
     "@mozilla/readability": "^0.6.0",
     "@resvg/resvg-js": "^2.6.2",
     "fluent-ffmpeg": "^2.1.3",
     "google-gax": "^5.0.6",
-    "ioredis": "^5.9.3",
-    "jsdom": "^28.1.0",
+    "ioredis": "^5.10.1",
+    "jsdom": "^29.1.1",
     "lorem-ipsum": "^2.0.8",
     "mime": "^4.1.0",
-    "mysql2": "^3.18.1",
+    "mysql2": "^3.22.3",
     "office-text-extractor": "^4.0.0",
-    "openai": "^6.25.0",
+    "openai": "^6.35.0",
     "parse-numeric-range": "^1.3.0",
-    "pg": "^8.19.0",
+    "pg": "^8.20.0",
     "pgvector": "^0.2.1",
-    "satori": "^0.19.2",
+    "satori": "^0.26.0",
     "telegraf": "^4.16.3",
     "tellegram": "^1.1.18",
     "tesseract.js": "^7.0.0",
-    "webjam": "^1995.3.10",
-    "youtube-transcript": "^1.2.1"
+    "webjam": "^1995.3.21",
+    "youtube-transcript": "^1.3.1"
   }
 }

package/pipeline/010_broca.mjs

@@ -1,5 +1,6 @@
-import { bot, hal, uoid, utilitas } from '../index.mjs';
+import { bot, hal, storage, uoid, utilitas } from '../index.mjs';
 import { convert, paginate } from 'tellegram';
+import { v4 as uuidv4 } from 'uuid';
 
 const _name = 'Broca';
 const [PRIVATE_LIMIT, GROUP_LIMIT] = [60 / 60, 60 / 20].map(x => x * 1000);
@@ -44,6 +45,24 @@ const getExtra = (ctx, options) => {
     return resp;
 };
 
+const getMediaName = func => func.replace(/^replyWith/, '') || 'File';
+
+const packInputFile = async (func, media) => {
+    if (media.filename || media.url || !Buffer.isBuffer(media.source)) {
+        return media;
+    }
+    const { extension } = await storage.getMime(media.source);
+    return { ...media, filename: `${getMediaName(func)}_${uuidv4()}.${extension}` };
+};
+
+const getMedia = async (func, src) => {
+    if (src?.source || src?.url) { return await packInputFile(func, src); }
+    if (Buffer.isBuffer(src)) {
+        return await packInputFile(func, { source: src });
+    }
+    return { [getKey(src)]: src };
+};
+
 const resp = async (ctx, text, extra) => {
     // if (ctx._.type === 'inline_query') {
     //     return await ctx.answerInlineQuery([{}, {}]);
@@ -72,7 +91,7 @@ const resp = async (ctx, text, extra) => {
 const replyWith = async (ctx, func, src, options) => ctx._.done.push(
     await ctx[func](Array.isArray(src) ? src.map(x => ({
         type: x.type || 'photo', media: { [getKey(x.src)]: x.src },
-    })) : { [getKey(src)]: src }, getExtra(ctx, options))
+    })) : await getMedia(func, src), getExtra(ctx, options))
 );
 
 const edit = async (ctx, lastMsgId, text, extra) => {

package/pipeline/080_history.mjs

@@ -3,12 +3,31 @@ import { bot, dbio, hal, utilitas } from '../index.mjs';
 const [RELEVANCE, SEARCH_LIMIT, SUB_LIMIT] = [0.2, 10, 200]; // Google Rerank limit
 const compact = (str, op) => utilitas.ensureString(str, { ...op || {}, compact: true });
 const compactLimit = (str, op) => compact(str, { ...op || {}, limit: 140 });
+const quote = key => `"${key}"`;
 
 const packMessage = (messages) => messages.map(x => ({
     message_id: x.message_id, score: x.score, created_at: x.created_at,
     request: x.received_text, response: x.response_text,
 }));
 
+const upsertPostgresqlEvent = async event => {
+    event = {
+        ...event,
+        distilled_vector: await dbio.encodeVector(event.distilled_vector),
+    };
+    const fields = Object.keys(event);
+    const values = fields.map(key => event[key]);
+    const placeholders = fields.map((key, i) => key === 'distilled_vector'
+        ? `quantize_to_rabitq8($${i + 1}::vector)` : `$${i + 1}`);
+    return await hal._.storage?.client?.query?.(
+        `INSERT INTO ${hal.table} (${fields.map(quote).join(', ')}) `
+        + `VALUES (${placeholders.join(', ')}) `
+        + `ON CONFLICT ("id") DO UPDATE SET `
+        + fields.map(key => `${quote(key)} = EXCLUDED.${quote(key)}`).join(', '),
+        values
+    );
+};
+
 const recall = async (sessionId, keyword, offset = 0, limit = SEARCH_LIMIT, options = {}) => {
     assert(sessionId, 'Session ID is required.');
     let [result, _limit, exclude] = [
@@ -35,13 +54,14 @@ const recall = async (sessionId, keyword, offset = 0, limit = SEARCH_LIMIT, opti
             // globalThis.debug = 2;
             const vector = await dbio.encodeVector(await hal._.embed(keyword));
             result = await hal._.storage?.client?.query?.(
-                `SELECT *, (1 - (distilled_vector <=> $1)) as relevance `
+                `SELECT *, (1 - (distilled_vector <=> `
+                + `quantize_to_rabitq8($1::vector))) as relevance `
                 + `FROM ${hal.table} WHERE bot_id = $2 AND chat_id = $3 `
                 + `AND received_text != '' `
                 + `AND received_text NOT LIKE '/%' `
                 + `AND response_text != '' `
                 + (exclude.length ? `AND message_id NOT IN (${exclude.join(',')}) ` : '')
-                + `ORDER BY (distilled_vector <=> $1) ASC `
+                + `ORDER BY (distilled_vector <=> quantize_to_rabitq8($1::vector)) ASC `
                 + `LIMIT ${_limit} OFFSET $4`,
                 [vector, hal._.bot.botInfo.id, sessionId, offset]
             );
@@ -127,6 +147,8 @@ const memorize = async (ctx) => {
             case dbio.MYSQL:
                 event.distilled_vector = JSON.stringify(event.distilled_vector);
                 break;
+            case dbio.POSTGRESQL:
+                return await upsertPostgresqlEvent(event);
         }
         await hal._.storage?.client?.upsert?.(hal.table, event, { skipEcho: true });
     }, hal.logOptions);

package/pipeline/100_chat.mjs

@@ -1,4 +1,4 @@
-import { callosum, alan } from '../index.mjs';
+import { callosum, alan, utilitas } from '../index.mjs';
 import { token } from 'webjam';
 
 const _name = 'Chat';