@storecraft/database-turso 1.0.8 → 1.0.10

package/README.md

@@ -12,7 +12,10 @@ npm i @storecraft/database-turso
 ```
 
 ## Setup
+You can run a local database,
+or,
 
+connect to a cloud `libsql` and `Turso` platform
 - First, login to your [turso](https://turso.tech) account.
 - Create a database.
 - Create an API Key.
@@ -41,10 +44,11 @@ const app = new App(
   new Turso(
     { 
       prefers_batch_over_transactions: true,
-      libsqlConfig: {
-        url: process.env.TURSO_URL,
-        authToken: process.env.TURSO_API_TOKEN,
-      }
+      // all of these configurations can be inferred by env variables at init
+      url: process.env.LIBSQL_URL,
+      authToken: process.env.LIBSQL_API_TOKEN,
+      // or local
+      url: 'file:local.db',
     }
   )
 )

package/index.js

@@ -1,23 +1,51 @@
+/**
+ * @import { Config } from './types.public.js';
+ * @import { ENV } from '@storecraft/core';
+ */
+
 import { SQL } from '@storecraft/database-sql-base';
 import { LibsqlDialect } from './kysely.turso.dialect.js';
 
+export { LibSQLVectorStore } from './vector-store/index.js'
+
 /**
  * @extends {SQL}
  */
 export class Turso extends SQL {
 
+  /** @satisfies {ENV<Config>} */
+  static EnvConfig = /** @type{const} */ ({
+    authToken: 'LIBSQL_AUTH_TOKEN',
+    url: 'LIBSQL_URL'
+  });
+  
   /**
    * 
-   * @param {import('./types.public.d.ts').Config} [config] config 
+   * @param {Config} [config] config 
    */
-  constructor(config) {
+  constructor(config={}) {
     super(
       {
         dialect_type: 'SQLITE',
-        dialect: new LibsqlDialect(config),
+        dialect: new LibsqlDialect(
+          {
+            ...config,
+            prefers_batch_over_transactions: config.prefers_batch_over_transactions ?? true
+          }
+        ),
       }
     );
-
   }
+  
 
+  /** @type {SQL["init"]} */
+  init = (app) => { 
+    const dialect = /** @type {LibsqlDialect}*/ (this.config.dialect);
+    const dconfig = dialect.config;
+    
+    dconfig.authToken ??= app.platform.env[Turso.EnvConfig.authToken];
+    dconfig.url ??= app.platform.env[Turso.EnvConfig.url];
+        
+    super.init(app);
+  }
 }

package/jsconfig.json

@@ -9,6 +9,6 @@
     "*", 
     "src/*", 
     "tests/*.js",
-    "d1/*"
+    "vector-store/*"
   ]
 }

package/kysely.turso.dialect.js

@@ -1,12 +1,10 @@
-import * as libsql from "@libsql/client";
-import * as kysely from "kysely";
-
 /**
- * @typedef {import('kysely').Driver} Driver
- * @typedef {import('kysely').Dialect} Dialect
- * @typedef {import('kysely').DatabaseConnection} DatabaseConnection
- * @typedef {import('./types.public.d.ts').Config} Config
+ * @import { Driver, Dialect, DatabaseConnection, QueryResult } from 'kysely';
+ * @import { Config } from './types.public.js';
+ * @import { Row, InArgs } from '@libsql/client';
  */
+import * as libsql from "@libsql/client";
+import * as kysely from "kysely";
 
 /**
  * 
@@ -24,18 +22,23 @@ export class LibsqlDialect {
     this.#config = config;
   }
 
+  get config() {
+    return this.#config;
+  }
+
   createAdapter() { return new kysely.SqliteAdapter(); }
   createQueryCompiler() { return new kysely.SqliteQueryCompiler(); }
   createDriver() {
 
-    if (this.#config?.libsqlConfig?.url===undefined) {
+    if (this.#config?.url===undefined) {
       throw new Error(
         "Please specify either `client` or `url` in the LibsqlDialect config"
       );
     }
 
     return new LibsqlDriver(
-      libsql.createClient(this.#config.libsqlConfig), 
+      // @ts-ignore
+      libsql.createClient(this.#config), 
       this.#config
     );
   }
@@ -134,16 +137,17 @@ export class LibsqlConnection {
   /**
    * @param {kysely.CompiledQuery[]} compiledQueries 
    * 
-   * @returns {Promise<import('kysely').QueryResult<import("@libsql/client").Row>>}
+   * @returns {Promise<QueryResult<Row>>}
    */
   async #internal_executeQuery(compiledQueries) {
+    // console.log(compiledQueries)
     const target = this.#transaction ?? this.client;
 
     const stmts = compiledQueries.map(
       cq => (
         {
           sql: cq.sql,
-          args: (/** @type {import("@libsql/client").InArgs} */ (cq.parameters)),
+          args: (/** @type {InArgs} */ (cq.parameters)),
         }
       )
     );
@@ -153,8 +157,8 @@ export class LibsqlConnection {
     );
 
     // console.log('q', JSON.stringify({sql, params}, null, 2))
-    console.log('stmts', JSON.stringify(stmts, null, 2))
-    console.log('result', JSON.stringify(results, null, 2))
+    // console.log('stmts', JSON.stringify(stmts, null, 2))
+    // console.log('result', JSON.stringify(results, null, 2))
 
     const last_result = results?.at(-1);
 
@@ -172,10 +176,10 @@ export class LibsqlConnection {
    * 
    * @param {kysely.CompiledQuery} compiledQuery 
    * 
-   * @returns {Promise<import('kysely').QueryResult>}
+   * @returns {Promise<QueryResult>}
    */
   async executeQuery(compiledQuery) {
-    console.log('this.isBatch', this.isBatch)
+    // console.log('this.isBatch', this.isBatch)
     if(this.isBatch) {
       this.batch.push(compiledQuery);
       return Promise.resolve(
@@ -202,7 +206,7 @@ export class LibsqlConnection {
   }
 
   async commitTransaction() {
-    console.log('commitTransaction')
+    // console.log('commitTransaction')
     if(this.isBatch) {
       // console.trace()
       await this.#internal_executeQuery(this.batch);
@@ -237,7 +241,7 @@ export class LibsqlConnection {
    * @param {kysely.CompiledQuery} compiledQuery 
    * @param {number} chunkSize 
    * 
-   * @returns {AsyncIterableIterator<import('kysely').QueryResult<R>>}
+   * @returns {AsyncIterableIterator<QueryResult<R>>}
    */
   async *streamQuery(compiledQuery, chunkSize) {
     throw new Error("Libsql Driver does not support streaming yet");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@storecraft/database-turso",
-  "version": "1.0.8",
+  "version": "1.0.10",
   "description": "`Storecraft` database driver for `Turso` (cloud sqlite)",
   "license": "MIT",
   "author": "Tomer Shalev (https://github.com/store-craft)",
@@ -24,6 +24,10 @@
       "import": "./index.js",
       "types": "./types.public.d.ts"
     },
+    "./vector-store": {
+      "import": "./vector-store/index.js",
+      "types": "./vector-store/types.d.ts"
+    },
     "./*": {
       "import": "./*"
     }

package/tests/runner.test.js

@@ -18,11 +18,8 @@ export const create_app = async () => {
   .withDatabase(
     new Turso(
       { 
+        url: ':memory:',
         prefers_batch_over_transactions: true,
-        libsqlConfig: {
-          url: process.env.TURSO_URL,
-          authToken: process.env.TURSO_API_TOKEN,
-        }
       }
     )
   );

package/tests/sandbox.js

@@ -17,10 +17,6 @@ export const test = async () => {
     new Turso(
       { 
         prefers_batch_over_transactions: true,
-        libsqlConfig: {
-          url: process.env.TURSO_URL,
-          authToken: process.env.TURSO_API_TOKEN,
-        }
       }
     )
   );

package/types.public.d.ts

@@ -1,16 +1,29 @@
 import type { Config as LibSqlConfig } from '@libsql/client'
-export { Turso } from './index.js';
+export * from './index.js';
 
-export type Config = {
+export type Config = Partial<Omit<LibSqlConfig, 'url' | 'authToken'>> & {
 
-  /**
-   * @description Official `libsql` config
+  /** The database URL.
+   *
+   * The client supports `libsql:`, `http:`/`https:`, `ws:`/`wss:` and `file:` URL. For more infomation,
+   * please refer to the project README:
+   *
+   * https://github.com/libsql/libsql-client-ts#supported-urls
+   * 
+   * If missing, it will be inferred by env variable `LIBSQL_URL`
+   */
+  url?: string;
+  /** 
+   * Authentication token for the database. Not applicable for `url`=`file:local.db`.
+   * 
+   * If missing, it will be inferred by env variable `LIBSQL_AUTH_TOKEN`
    */
-  libsqlConfig: LibSqlConfig
+  authToken?: string;
 
   /**
    * @description if `true`, transactions are converted into a non-interactive batch,
    * use with caution and prefer this when transactions are non-interactive
+   * @default true
    */
   prefers_batch_over_transactions?: boolean;
 }

package/vector-store/index.js

@@ -0,0 +1,302 @@
+/**
+ * @import { 
+ *  AIEmbedder, VectorStore 
+ * } from '@storecraft/core/ai/core/types.private.js'
+ * @import { ENV } from '@storecraft/core';
+ * @import {
+ *  Config
+ * } from './types.js'
+ * @import { 
+ *  VectorDocumentUpsert 
+ * } from './types.private.js'
+ * @import { InArgs } from '@libsql/client';
+ */
+
+import * as libsql from "@libsql/client";
+import { 
+  truncate_or_pad_vector 
+} from "@storecraft/core/ai/models/vector-stores/index.js";
+
+export const DEFAULT_INDEX_NAME = 'vector_store';
+
+/** @param {any} json */
+const parse_json_safely = json => {
+  try {
+    return JSON.parse(json);
+  } catch (e) {
+    return {};
+  } finally {
+  }
+}
+
+/**
+ * Implementation referenes:
+ * - https://docs.turso.tech/features/ai-and-embeddings#vectors-usage
+ * - https://github.com/langchain-ai/langchainjs/blob/9dfaae7e36a1ddce586b9c44fb96785fa38b36ec/libs/langchain-community/src/vectorstores/libsql.ts
+ */
+
+/**
+ * @typedef {VectorStore} Impl
+ */
+
+/**
+ * @description LibSQL / Turso Vector Store
+ * 
+ * @implements {VectorStore}
+ */
+export class LibSQLVectorStore {
+  
+  /** @satisfies {ENV<Config>} */
+  static EnvConfig = /** @type{const} */ ({
+    authToken: 'LIBSQL_VECTOR_AUTH_TOKEN',
+    url: 'LIBSQL_VECTOR_URL',
+  });
+
+  /** @type {Config} */
+  config;
+
+  /** @type {libsql.Client} */
+  #client
+
+  /**
+   * 
+   * @param {Config} config 
+   */
+  constructor(config) {
+    this.config = {
+      index_name: DEFAULT_INDEX_NAME,
+      similarity: 'cosine',
+      dimensions: 1536,
+      ...config,
+    };
+  }
+
+  get client() {
+    if(!this.config.url) {
+      throw new Error('LibSQLVectorStore::client() - missing url');
+    }
+    
+    // @ts-ignore
+    this.#client = this.#client ?? libsql.createClient(this.config);
+
+    return this.#client;
+  }
+
+  get index_name() {
+    return this.config.index_name;
+  }
+
+  get table_name() {
+    return `${this.index_name}_table`;
+  }
+
+  /** @type {VectorStore["onInit"]} */
+  onInit = (app) => {
+    this.config.authToken ??= app.platform.env[
+      LibSQLVectorStore.EnvConfig.authToken ?? 'LIBSQL_AUTH_TOKEN'
+    ];
+    this.config.url ??= app.platform.env[
+      LibSQLVectorStore.EnvConfig.url ?? 'LIBSQL_URL'
+    ];
+  }
+
+  /** @type {VectorStore["embedder"]} */
+  get embedder() {
+    return this.config.embedder
+  }
+
+  // (id TEXT, metadata TEXT, pageContent Text, updated_at TEXT, namespace TEXT, embedding F32_BLOB
+  /** @type {VectorStore["upsertVectors"]} */
+  upsertVectors = async (vectors, documents, options) => {
+
+    const updated_at = new Date().toISOString();
+    /** @type {VectorDocumentUpsert[]} */
+    const docs_upsert = documents.map(
+      (doc, ix) => (
+        {
+          embedding: `[${truncate_or_pad_vector(vectors[ix], this.config.dimensions).join(',')}]`,
+          id: doc.id,
+          metadata: JSON.stringify(doc.metadata ?? {}),
+          pageContent: doc.pageContent,
+          updated_at,
+          namespace: doc.namespace,
+        }
+      )
+    );
+
+    /** @type {import("@libsql/client").InStatement[]} */
+    const stmts_delete = docs_upsert.map(
+      (doc, ix) => (
+        {
+          sql: `DELETE FROM ${this.table_name} WHERE id=?`,
+          args: [doc.id]
+        }
+      )
+    );
+
+    /** @type {import("@libsql/client").InStatement[]} */
+    const stmts_insert = docs_upsert.map(
+      (doc, ix) => (
+        {
+          sql: `
+          INSERT INTO ${this.table_name} (id, metadata, pageContent, updated_at, namespace, embedding) 
+          VALUES (:id, :metadata, :pageContent, :updated_at, :namespace, vector(:embedding))
+          `,
+          args: doc
+        }
+      )
+    );
+
+    const result = await this.client.batch(
+      [
+        ...stmts_delete,
+        ...stmts_insert,
+      ]
+    );
+
+  }
+
+  /** @type {VectorStore["upsertDocuments"]} */
+  upsertDocuments = async (documents, options) => {
+    // first, generate embeddings for the documents
+    const result = await this.embedder.generateEmbeddings(
+      {
+        content: documents.map(
+          doc => (
+            {
+              content: doc.pageContent,
+              type: 'text'
+            }
+          )
+        )
+      }
+    );
+
+    const vectors = result.content;
+
+    // console.log(vectors)
+
+    return this.upsertVectors(
+      vectors, documents, options
+    )
+  }
+
+  /** @type {VectorStore["delete"]} */
+  delete = async (ids) => {
+    await this.client.execute(
+      {
+        sql: `DELETE FROM ${this.table_name} WHERE id IN (${ids.map(id => '?').join(',')})`,
+        args: ids
+      }
+    );
+  }
+
+  /** @type {VectorStore["similaritySearch"]} */
+  similaritySearch = async (query, k, namespaces) => {
+
+    const embedding_result = await this.embedder.generateEmbeddings(
+      {
+        content: [
+          {
+            content: query, 
+            type: 'text'
+          }
+        ]
+      }
+    );
+    const vector = truncate_or_pad_vector(
+      embedding_result.content[0], this.config.dimensions
+    );
+    const vector_sql_value = `[${vector.join(',')}]`
+    const distance_fn = this.config.similarity==='cosine' ? 'vector_distance_cos' : 'vector_distance_l2'
+    // SELECT title, year
+    // FROM vector_top_k('movies_idx', vector32('[0.064, 0.777, 0.661, 0.687]'), 3)
+    // JOIN movies ON movies.rowid = id
+    // WHERE year >= 2020;    
+    const table = this.table_name;
+    const index_name = this.index_name;
+    /** @type {InArgs} */
+    let args = [];
+    let sql = `
+    SELECT id, metadata, pageContent, updated_at, namespace, ${distance_fn}(embedding, vector(?)) AS score
+    FROM vector_top_k('${index_name}', vector(?), ?) as top_k_view
+    JOIN ${table} ON ${table}.rowid = top_k_view.id
+    `;
+    args.push(vector_sql_value, vector_sql_value, k);
+
+    if(Array.isArray(namespaces) && namespaces.length) {
+      sql += `\nWHERE namespace IN (${namespaces.map(n => '?').join(',')})`
+      args.push(...namespaces);
+    }
+
+    sql += `
+    ORDER BY
+       ${distance_fn}(embedding, vector(?))
+    ASC;
+    `
+    args.push(vector_sql_value);
+
+
+    const result = await this.client.execute({ sql, args });
+
+    return result.rows.map(
+      (row) => (
+        {
+          document: {
+            pageContent: String(row.pageContent),
+            id: String(row.id),
+            metadata: parse_json_safely(row.metadata),
+            namespace: String(row.namespace),
+          },
+          score: Number(row.score)
+        }
+      )
+    );
+
+  }
+
+
+  /**
+   * 
+   * @param {boolean} [delete_index_if_exists_before=false] 
+   * @returns {Promise<boolean>}
+   */
+  createVectorIndex = async (delete_index_if_exists_before=false) => {
+
+    /** @type {string[]} */
+    const batch = [];
+
+    if(delete_index_if_exists_before) {
+      await this.deleteVectorIndex();
+    }
+
+    batch.push(
+      `CREATE TABLE IF NOT EXISTS ${this.table_name} (id TEXT, metadata TEXT, pageContent Text, updated_at TEXT, namespace TEXT, embedding F32_BLOB(${this.config.dimensions}));`,
+      `CREATE INDEX IF NOT EXISTS ${this.index_name} ON ${this.table_name}(libsql_vector_idx(embedding));`
+    );
+
+    const result = await this.client.batch(batch);
+    return true;
+  }
+
+/**
+   * 
+   * @returns {Promise<boolean>}
+   */
+deleteVectorIndex = async () => {
+
+  /** @type {string[]} */
+  const batch = [];
+
+  batch.push(
+    `DROP INDEX IF EXISTS ${this.index_name}`,
+    `DROP TABLE IF EXISTS ${this.table_name}`,
+  );
+
+  const result = await this.client.batch(batch);
+  return true;
+}
+
+  
+}
+

package/vector-store/types.d.ts

@@ -0,0 +1,48 @@
+
+import type { AIEmbedder } from '@storecraft/core/ai';
+export * from './index.js';
+
+export type Config = {
+  /** 
+   * @description The database URL.
+   *
+   * The client supports `libsql:`, `http:`/`https:`, `ws:`/`wss:` and `file:` URL. For more infomation,
+   * please refer to the project README:
+   *
+   * https://github.com/libsql/libsql-client-ts#supported-urls
+   * 
+   * If missing, it will be inferred by env variable `LIBSQL_VECTOR_URL` or `LIBSQL_URL`
+   */
+  url?: string;
+  /** 
+   * @description Authentication token for the database. Not applicable for local `url`=`file:local.db`.
+   * 
+   * If missing, it will be inferred by env variable `LIBSQL_VECTOR_AUTH_TOKEN` or `LIBSQL_AUTH_TOKEN`
+   * 
+   * @default ENV variable `LIBSQL_AUTH_TOKEN`
+   */
+  authToken?: string;
+
+  /**
+   * @description The name of the index
+   * @default 'vector_store'
+   */
+  index_name?: string,
+
+  /** 
+   * @description The dimensions of the vectors to be inserted in the index. 
+   * @default 1536
+   */
+  dimensions?: number,
+
+  /**
+   * @description The similiarity metric
+   * @default 'cosine'
+   */
+  similarity?: 'euclidean' | 'cosine',
+
+   /**
+   * @description Embedding model provider
+   */
+   embedder: AIEmbedder
+}

package/vector-store/types.private.d.ts

@@ -0,0 +1,16 @@
+
+export type VectorDocument = {
+  id: string,
+  metadata: Record<string, any>,
+  embedding: number[],
+  pageContent: string,
+  updated_at: string,
+  score?: number
+  namespace?: string
+}
+
+export type VectorDocumentUpsert = Omit<VectorDocument, 'embedding' | 'metadata' | 'score'> & {
+  embedding: string,
+  metadata: string,
+}
+