@sjovanovic/recall.js 1.0.3 → 1.0.4

package/README.md

@@ -8,11 +8,11 @@ Recall.js is long term memory for AI apps!
 
 It is a tool for building RAG (Retrieval-augmented generation) in a form of JavaScript library and command line utility focused on speed, ease of use and embeddability.
 
-It is versatile and you don't have to use it exclusively for RAG, use it for generic Semantic Search, as expert memory for your AI app, as a  recommendation system, there are so many possibilities...
+It is versatile and you don't have to use it exclusively for RAG, it can also be used for generic Semantic Search, as expert memory for your AI app, as a  recommendation system, there are many possibilities...
 
 Recall.js supports multilingual embeddings out of the box so you can add data in one language and then query it in another.
 
-Under the hood, recall.js uses sentence vector embeddings and a vector database to index and query your data. It is a light wrapper around local language models such as [MiniLM-L12-v2](https://huggingface.co/sentence-transformers/all-MiniLM-L12-v2) and [CozoDB](https://www.cozodb.org/) vector database.
+Under the hood, recall.js uses [Transformers.js](https://huggingface.co/docs/transformers.js/index) feature extraction and a vector database to index and query your data. It is a light wrapper around local language models such as [Multilingual-MiniLM-L12-v2](https://huggingface.co/Xenova/paraphrase-multilingual-MiniLM-L12-v2) and [CozoDB](https://www.cozodb.org/) vector database.
 
 ## Install
 
@@ -26,7 +26,7 @@ Console:
 recall --add 'The quick brown fox jumps over the lazy dog|Fox|{"foo":"bar"}'
 recall --query "Un animal saute par-dessus un autre animal" --limit 1
 ```
-**Warning:** when this library is used for the first time, it will download a local language model MiniLM-L12-v2 which may take long time depending on your Internet connectivity. Please be patient.
+**Warning:** when this library is used for the first time, it will download a local language model Multilingual-MiniLM-L12-v2 which may take a while depending on your Internet connectivity. Please be patient.
 
 Below is the same example in JavaScript:
 
@@ -58,16 +58,18 @@ response:
     "dist",
     "result",
     "id",
-    "data"
+    "data",
+    "category"
   ],
   "rows": [
     [
-      0.5840495824813843, // vector similarity
+      0.6840495824813843, // vector similarity
       "Fox and dog",
       "08840189191373282",
       {
         "foo": "bar"
-      }
+      },
+      ""
     ]
   ]
 }
@@ -84,20 +86,21 @@ Easy way to view all the options is via command line:
 recall --help
 
 Usage:
-recall --query "Foo Bar"
+recall.js --query "Foo Bar"
 
 Options:
---query "SEARCH_STRING"                - search
---limit 2                              - limit number of results (used with --query)
---add 'input|result|{"foo":"bar"}'     - add data
---remove 'id'                          - remove data
---nuke                                 - destroy database
---mcp                                  - run as MCP server
---db "FILE_NAME"                       - database file (SQLite)
---import "file.csv | file.tsv"         - import from CSV or TSV w/ columns: 1. input 2. result 3. and remaining columns are additional data
---input-header "foo"                   - when used with --import designates specific header column as input
---result-header "bar"                  - when used with --import designates specific header column as result
---json "FILE_NAME"                     - import from file which has one json object per line: {input:"", result:"", data:{}}
+--query "SEARCH_STRING"                    - search
+--limit 2                                  - limit number of results (used with --query)
+--add 'input|result|{"foo":"bar"}|categ'   - add data
+--remove 'id'                              - remove data
+--nuke                                     - destroy database
+--mcp                                      - run as MCP server (experimental)
+--db "FILE_NAME"                           - database file (SQLite)
+--import "file.csv | file.tsv"             - import from CSV or TSV w/ columns: 1. input 2. result 3. and remaining columns are additional data
+--input-header "foo"                       - when used with --import designates specific header column as input
+--result-header "bar"                      - when used with --import designates specific header column as result
+--json "FILE_NAME"                         - import from file which has one json object per line: {input:"", result:"", data:{}}
+--category "CATEGORY"                      - specify category when adding data and to filter by when querying (defaults to empty string)
 ```
 
 **Note:** when adding data recall will generate unique id automatically. To set custom id add it as a string property named "id" in the data object (i.e. `{"id":"customID"}`).
@@ -111,11 +114,14 @@ Configuration object.
 
 ```javascript
 export const config = {
-    VECTOR_SIZE: 384, // number of dimensions
-    MODEL_NAME: 'Xenova/paraphrase-multilingual-MiniLM-L12-v2', // model to use 
+    VECTOR_SIZE: 384, // number of dimensions (must match the models output)
+    MODEL_NAME: 'Xenova/paraphrase-multilingual-MiniLM-L12-v2', // model to use (passed to Transformers.js)
     SHOW_ERRORS: true, // Show errors
     DB_FILE: join(PATH, 'vector.db'), // Path to the datbase file (SQLite file used by CozoDB)
-    PATH: PATH // directory of recall.js
+    PATH: PATH, // directory of recall.js
+    DEVICE: undefined, // Transformers.js device
+    DTYPE: undefined, // Transformers.js dtype
+    PROGRESS_CALLBACK: undefined // Transformers.js progress_callback
 }
 ```
 
@@ -127,7 +133,7 @@ Returns reference to the CozoDB instance.
 
 Given text calculates the embeddings vector
 
-### RECALL.add(input, result, data={}) -> Promise(Object)
+### RECALL.add(input, result, data={}, category="") -> Promise(Object)
 
 Add data. `input` is the sentence to get embeddings from. `result` is the string to show in the results. `data` is arbitrary object intended to hold related pieces of information and references. If `data` object contains `id` property it will be used as unique id of the record.
 
@@ -136,14 +142,14 @@ Add data. `input` is the sentence to get embeddings from. `result` is the string
 Add data in batches (faster than using add repeteadely). 
 `batch` is an Array that looks like this:
 ```
-let batch = [{input:"", result:"", data:{}}]
+let batch = [{input:"", result:"", data:{}, category:""}]
 ```
 
 ### RECALL.remove(id) -> Promise(Object)
 
 Remove data by id. id is a string.
 
-### RECALL.searchText(text, numResults = 5) ->  Promise(Object)
+### RECALL.searchText(text, category="", numResults = 5, includeInput=false) ->  Promise(Object)
 
 Query the vector database. Accepts query text and number of results to return.
 
@@ -155,8 +161,8 @@ Deletes the database.
 
 Imports from readable stream or file which consists of JSON objects, one per line. e.g.
 ```
-{input:"one", result:"one result", data:{"id":"123"}}
-{input:"", result:"", data:{}}
+{input:"one", result:"one result", data:{"id":"123"}, category:""}
+{input:"", result:"", data:{}, category:""}
 ...
 ```
 This is the most efficient way to import data.

package/package.json

@@ -1,24 +1,24 @@
 {
   "name": "@sjovanovic/recall.js",
-  "version": "1.0.3",
-  "description": "Semantic search as long term memory for LLMs",
+  "version": "1.0.4",
+  "description": "Easy RAG with semantic search and long term memory",
   "main": "recall.js",
   "bin": {
     "recall": "recall.js"
   },
   "type": "module",
   "scripts": {
+    "start": "node recall.js",
     "test": "echo \"Error: no test specified\" && exit 1",
     "query": "node recall.js --query "
   },
   "author": "Slobodan Jovanovic",
   "license": "ISC",
   "dependencies": {
-    "@modelcontextprotocol/sdk": "^1.8.0",
-    "@themaximalist/embeddings.js": "^0.1.3",
-    "@xenova/transformers": "^2.17.2",
+    "@huggingface/transformers": "^4.2.0",
+    "@modelcontextprotocol/sdk": "^1.29.0",
     "cozo-node": "^0.7.6",
     "csv-parser": "^3.2.0",
-    "zod": "^3.24.2"
+    "zod": "^4.3.6"
   }
 }

package/recall.js

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
 import {CozoDb} from 'cozo-node'
-import embeddings from "@themaximalist/embeddings.js";
+import { pipeline } from "@huggingface/transformers";
 import csv from 'csv-parser'
 import fs from 'fs'
 import { resolve, join, dirname, sep } from 'path'
@@ -10,17 +10,22 @@ import { McpServer, ResourceTemplate } from "@modelcontextprotocol/sdk/server/mc
 import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
 import { z } from "zod";
 
+// import {sanitizeValue} from './utils/sanitize.js'
+
 const pathToThisFile = resolve(fileURLToPath(import.meta.url))
 const pathPassedToNode = resolve(process.argv[1])
 const isThisFileBeingRunViaCLI = pathToThisFile.includes(pathPassedToNode) || pathPassedToNode.includes('.npm-global')
 const PATH = dirname(pathToThisFile)
 
 export const config = {
-    VECTOR_SIZE: 384, // number of dimensions
-    MODEL_NAME: 'Xenova/paraphrase-multilingual-MiniLM-L12-v2', // model to use 
+    VECTOR_SIZE: 384, // number of dimensions (must match the models output)
+    MODEL_NAME: 'Xenova/paraphrase-multilingual-MiniLM-L12-v2', // model to use (passed to Transformers.js)
     SHOW_ERRORS: true, // Show errors
     DB_FILE: join(PATH, 'vector.db'), // Path to the datbase file (SQLite file used by CozoDB)
-    PATH: PATH // directory of recall.js
+    PATH: PATH, // directory of recall.js
+    DEVICE: undefined, // Transformers.js device
+    DTYPE: undefined, // Transformers.js dtype
+    PROGRESS_CALLBACK: undefined // Transformers.js progress_callback
 }
 
 var db = null, initDone = false
@@ -33,7 +38,6 @@ export const getDb = () => {
 }
 
 async function printQuery(query, params = {}) {
-
     try{
         if(!initDone) {
             initDone = true
@@ -51,12 +55,22 @@ async function printQuery(query, params = {}) {
 }
 
 export const getEmbeddings = async (text) => {
-    const embedding = await embeddings(text,  {
-        service:'transformers',
-        model: config.MODEL_NAME,
-        cache_file: join(config.PATH, "cache", ".embeddings.cache.json")
-    });
-    return embedding
+    let pipe = config._pipe
+    if(!pipe) {
+        config._pipe = await pipeline("feature-extraction", config.MODEL_NAME, {
+            progress_callback:(progress) => {
+                if(config.PROGRESS_CALLBACK) return config.PROGRESS_CALLBACK();
+                if(progress.status === "progress_total"){
+                    process.stdout.write(`\r\x1b[K✅ Loaded ${ Math.round(progress.progress)}% ${progress.name || "model"}`)
+                }
+            },
+            device: config.DEVICE,
+            dtype: config.DTYPE
+        });
+        pipe = config._pipe
+    }
+    const embedding = await pipe(text, { pooling: "mean", normalize: true });
+    return Array.from(embedding.data)
 }
 
 export const createTable = async () => {
@@ -81,7 +95,6 @@ export const createTable = async () => {
 
 export const add = async (input, result, data={}, category="") => {
     if(!input || !result) return
-
     input = sanitizeString(input)
     result = sanitizeString(result)
     const embedding = await getEmbeddings(input)
@@ -99,7 +112,7 @@ export const add = async (input, result, data={}, category="") => {
  * @param {Array} batch 
  * @returns 
  */
-export const addBatch = async (batch) => {
+export const addBatch = async (batch, opts={onProgress:null}) => {
     if(!batch || !Array.isArray(batch)) return
     let vectorBatch = []
     for(let i=0;i<batch.length; i++){
@@ -125,27 +138,46 @@ export const addBatch = async (batch) => {
             :put embeddings {id, category => v, input, result, data}`
         }
         vectorBatch.push(item)
+
+        if(opts.onProgress && typeof opts.onProgress == 'function') {
+            await opts.onProgress({index: i+1, total:batch.length, item: batch[i], embedding, percent: Math.round((i+1) / batch.length * 100)})
+        }
     }
     return await printQuery(vectorBatch.join("\n"))
 }
 
 const sanitizeString = (str)=>{
-    return str.replace(/[\/#$%\^&\*{}=_`~()\"]/g," ").replace(/\s{2,}/g, " ")
+    return str.replace(/[\/#$%\^&\*{}=_`~()\"]/g," ").replace(/\s{2,}/g, " ").trim()
 }
 
 export const remove = async (id, category="") => {
     if(!id || typeof id != 'string') return 
-    id.replace(/[^a-zA-Z0-9]/g, '')
-    if(!id) return
+    id = id.replace(/[^a-zA-Z0-9]/g, '')
+    category = sanitizeString(category)
+    if(!id || !category) return
     let results = await printQuery(
         `?[id, category] <- [['${id}', '${category}']]
-        ::remove embeddings {id}`)
+        ::rm embeddings {id, category}`)
+    return results
+}
+
+export const removeAllByCategory = async (category="") => {
+    category = sanitizeString(category)
+    if(!category) return
+    let results
+    try {
+        results = await printQuery(
+            `?[id, category] := *embeddings{id, category}, category = "${category}"
+            :rm embeddings {id, category}`)
+    }catch(err){
+        console.error(err)
+    }
     return results
 }
 
-export const searchText = async (text, category="", numResults = 5) => {
+export const searchText = async (text, category="", numResults = 5, includeInput=false) => {
     const embedding = await getEmbeddings(text)
-    let results = await printQuery(`?[dist, result, id, data, category] := ~embeddings:index_name { id, v, input, result, data, category |
+    let results = await printQuery(`?[dist, result, id, data, category${includeInput? ', input' : ''}] := ~embeddings:index_name { id, v, input, result, data, category${includeInput? ', input' : ''} |
         query: q,
         k: ${numResults}, # number of results
         ef: 50, # number of neighbours to consider 

package/utils/sanitize.js

@@ -0,0 +1,34 @@
+export function sanitizeValue(stringValue, maxChars=1000) {
+    if (typeof stringValue !== 'string') {
+        throw new Error('stringValue must be a string');
+    }
+    
+    let sanitized = stringValue.normalize('NFC').trim();
+    
+    // Basic validation
+    if (sanitized.length === 0) {
+        throw new Error('stringValue name cannot be empty');
+    }
+    
+    if (sanitized.length > maxChars) {
+        throw new Error(`stringValue name too long (max ${maxChars} characters)`);
+    }
+    
+    // Block control characters (primary security concern)
+    // This allows all other Unicode characters including emojis, Chinese, Arabic, etc.
+    if (/[\x00-\x1F\x7F-\x9F\u200B\u200E\u200F\u202A-\u202E\u2060-\u2069\uFEFF]/.test(sanitized)) {
+        throw new Error('stringValue contains disallowed control characters');
+    }
+    
+    // Block private use areas
+    if (/[\uE000-\uF8FF\uFFF0-\uFFFF]/.test(sanitized)) {
+        throw new Error('stringValue contains disallowed Unicode characters');
+    }
+    
+    // Block surrogate pairs (invalid alone)
+    if (/[\uD800-\uDFFF]/.test(sanitized)) {
+        throw new Error('stringValue contains invalid Unicode characters');
+    }
+    
+    return sanitized;
+}