@storecraft/database-turso 1.0.14 → 1.0.16

package/README.md

@@ -5,16 +5,85 @@
        width='90%' />
 </div><hr/><br/>
 
-Official `libSql` / `Turso` driver for `StoreCraft` on any platforms.
+Official `libSql` / `Turso` driver for `StoreCraft` on any platform.
 Includes a vector store.
 
 ```bash
 npm i @storecraft/database-turso
 ```
 
-## Setup
-You can run a local database,
-or,
+## Local usage
+
+```js
+import 'dotenv/config';
+import http from "node:http";
+import { App } from '@storecraft/core'
+import { NodePlatform } from '@storecraft/core/platform/node';
+import { NodeLocalStorage } from '@storecraft/core/storage/node'
+import { Turso, LibSQLVectorStore } from '@storecraft/database-turso'
+import { migrateToLatest } from '@storecraft/database-turso/migrate.js'
+
+const app = new App(
+  {
+    auth_admins_emails: ['admin@sc.com'],
+  }
+)
+.withPlatform(new NodePlatform())
+.withDatabase(
+  new Turso(
+    { 
+      url: 'file:local.db',
+    }
+  )
+)
+.withStorage(new NodeLocalStorage('storage'))
+.withVectorStore(
+  new LibSQLVectorStore(
+    {
+      embedder: new OpenAIEmbedder(),
+      url: 'file:local-vector.db'
+    }
+  )
+)
+
+await app.init();
+await migrateToLatest(app.db, false);
+await app.vectorstore.createVectorIndex();
+
+const server = http.createServer(app.handler).listen(
+  8000,
+  () => {
+    console.log(`Server is running on http://localhost:8000`);
+  }
+); 
+
+```
+
+Storecraft will search the following `env` variables
+
+```bash
+LIBSQL_URL=./data.db
+# optional, if absent will use the same as LIBSQL_URL
+LIBSQL_VECTOR_URL=./data-vector.db
+```
+
+So, you can instantiate with empty config
+
+```ts
+.withDatabase(
+  new Turso()
+)
+.withVectorStore(
+  new LibSQLVectorStore(
+    {
+      embedder: new OpenAIEmbedder(),
+    }
+  )
+)
+```
+
+
+## Cloud usage
 
 connect to a cloud `libsql` and `Turso` platform
 - First, login to your [turso](https://turso.tech) account.
@@ -36,8 +105,6 @@ import { migrateToLatest } from '@storecraft/database-turso/migrate.js'
 const app = new App(
   {
     auth_admins_emails: ['admin@sc.com'],
-    auth_secret_access_token: 'auth_secret_access_token',
-    auth_secret_refresh_token: 'auth_secret_refresh_token'
   }
 )
 .withPlatform(new NodePlatform())
@@ -48,8 +115,6 @@ const app = new App(
       // all of these configurations can be inferred by env variables at init
       url: process.env.LIBSQL_URL,
       authToken: process.env.LIBSQL_API_TOKEN,
-      // or local
-      url: 'file:local.db',
     }
   )
 )
@@ -57,14 +122,17 @@ const app = new App(
 .withVectorStore(
   new LibSQLVectorStore(
     {
-      embedder: new OpenAIEmbedder()
+      embedder: new OpenAIEmbedder(),
+      url: process.env.LIBSQL_URL,
+      authToken: process.env.LIBSQL_API_TOKEN,
     }
   )
 )
 
 await app.init();
 await migrateToLatest(app.db, false);
- 
+await app.vectorstore.createVectorIndex();
+
 const server = http.createServer(app.handler).listen(
   8000,
   () => {
@@ -74,6 +142,30 @@ const server = http.createServer(app.handler).listen(
 
 ```
 
+Storecraft will search the following `env` variables
+
+```bash
+LIBSQL_URL=libsql://<your_database_name>.something.com
+LIBSQL_AUTH_TOKEN=your_api_key
+```
+
+So, you can instantiate with empty config
+
+```ts
+.withDatabase(
+  new Turso()
+)
+.withVectorStore(
+  new LibSQLVectorStore(
+    {
+      embedder: new OpenAIEmbedder(),
+    }
+  )
+)
+```
+
+
+
 ```text
 Author: Tomer Shalev <tomer.shalev@gmail.com>
 ```

package/index.js

@@ -29,8 +29,8 @@ export class Turso extends SQL {
         dialect_type: 'SQLITE',
         dialect: new LibsqlDialect(
           {
+            prefers_batch_over_transactions: true,
             ...config,
-            prefers_batch_over_transactions: config.prefers_batch_over_transactions ?? true
           }
         ),
       }

package/kysely.turso.dialect.js

@@ -26,8 +26,12 @@ export class LibsqlDialect {
     return this.#config;
   }
 
-  createAdapter() { return new kysely.SqliteAdapter(); }
-  createQueryCompiler() { return new kysely.SqliteQueryCompiler(); }
+  createAdapter() { 
+    return new kysely.SqliteAdapter(); 
+  }
+  createQueryCompiler() { 
+    return new kysely.SqliteQueryCompiler(); 
+  }
   createDriver() {
 
     if (this.#config?.url===undefined) {
@@ -173,9 +177,7 @@ export class LibsqlConnection {
   } 
 
   /**
-   * 
    * @param {kysely.CompiledQuery} compiledQuery 
-   * 
    * @returns {Promise<QueryResult>}
    */
   async executeQuery(compiledQuery) {
@@ -193,6 +195,7 @@ export class LibsqlConnection {
   }
 
   async beginTransaction() {
+    // console.log('beginTransaction')
     if(this.config.prefers_batch_over_transactions) {
       this.isBatch = true;
       this.batch = [];

package/migrate.js

@@ -1,5 +1,123 @@
-export { migrateToLatest } from '@storecraft/database-sql-base/migrate.js';
+import { SQL } from "@storecraft/database-sql-base";
+import { AggregateDialect } from "@storecraft/database-sql-base/kysely.aggregate.dialect.js";
+import { get_migrations, prepare_and_bind } from "@storecraft/database-sql-base/migrate.js";
+import { Kysely, Migrator } from "kysely";
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from "node:url";
 
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
 
+/**
+ * 
+ * @param {SQL} db_driver 
+ * @param {boolean} [destroy_db_upon_completion=true] 
+ */
+export async function migrateToLatest(
+  db_driver, destroy_db_upon_completion=true
+) {
+  const db = db_driver?.client;
 
+  if(!db)
+    throw new Error('No Kysely client found !!!');
 
+  console.log(
+    'Resolving migrations. This may take 2 minutes ...'
+  );
+
+  const migrations = await get_migrations(db_driver.dialectType);
+
+  if(!migrations)
+    throw new Error('No migrations found !!!');
+
+  const sorted = Object
+  .entries(migrations)
+  .sort((a, b) => a[0].localeCompare(b[0]));
+
+  /** @type {Record<string, import("kysely").Migration>} */
+  const rewritten_migrations = {};
+
+  for (const [name, migration] of sorted) {
+    // console.log(`Executing virtual migration "${name}" ...`);
+    const up_agg_dialect = new AggregateDialect(
+      { dialect: db_driver.config.dialect, }
+    );
+    const up_agg_kysely = new Kysely(
+      { dialect: up_agg_dialect }
+    );
+    const down_agg_dialect = new AggregateDialect(
+      { dialect: db_driver.config.dialect, }
+    );
+    const down_agg_kysely = new Kysely(
+      { dialect: down_agg_dialect }
+    );
+    
+    await migration.up(up_agg_kysely);
+    await migration.down(down_agg_kysely);
+
+    // now get the queries
+    const queries_up = up_agg_dialect.queries;
+    const queries_down = down_agg_dialect.queries;
+    
+    // turso dialect supports transactions and also batch
+    // we take adavantage of this
+    rewritten_migrations[name] = {
+      up: async (db) => {
+        await db.transaction().execute(
+          async (trx) => {
+            for (const query of queries_up) {
+              await trx.executeQuery(query)
+            }
+          }
+        );
+      },
+      down: async (db) => {
+        await db.transaction().execute(
+          async (trx) => {
+            for (const query of queries_down) {
+              await trx.executeQuery(query)
+            }
+          }
+        );
+      }
+    }
+  }
+
+  const migrator = new Migrator(
+    {
+      db,
+      provider: {
+        getMigrations: async () => {
+          return rewritten_migrations;
+        }
+      }
+    }
+  );
+
+  const { error, results } = await migrator.migrateToLatest();
+
+  results?.forEach(
+    (it) => {
+      if (it.status === 'Success') {
+        console.log(
+          `migration "${it.migrationName}" was executed successfully`
+        );
+      } else if (it.status === 'Error') {
+        console.error(
+          `failed to execute migration "${it.migrationName}"`
+        );
+      }
+    }
+  );
+
+  if (error) {
+    console.error('failed to migrate')
+    console.error(JSON.stringify(error, null, 2))
+    process.exit(1)
+  }
+
+  if(destroy_db_upon_completion)
+    await db.destroy();
+}
+  

package/migrate_old.js

@@ -0,0 +1,5 @@
+export { migrateToLatest } from '@storecraft/database-sql-base/migrate.js';
+
+
+
+

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@storecraft/database-turso",
-  "version": "1.0.14",
+  "version": "1.0.16",
   "description": "`Storecraft` database driver for `Turso` (cloud sqlite)",
   "license": "MIT",
   "author": "Tomer Shalev (https://github.com/store-craft)",

package/tests/runner.test.js

@@ -11,7 +11,9 @@ export const create_app = async () => {
     {
       auth_admins_emails: ['admin@sc.com'],
       auth_secret_access_token: 'auth_secret_access_token',
-      auth_secret_refresh_token: 'auth_secret_refresh_token'
+      auth_secret_refresh_token: 'auth_secret_refresh_token',
+      auth_secret_confirm_email_token: 'auth_secret_confirm_email_token',
+      auth_secret_forgot_password_token: 'auth_secret_forgot_password_token',
     }
   )
   .withPlatform(new NodePlatform())

package/tests/sandbox.js

@@ -9,7 +9,9 @@ export const test = async () => {
     {
       auth_admins_emails: ['admin@sc.com'],
       auth_secret_access_token: 'auth_secret_access_token',
-      auth_secret_refresh_token: 'auth_secret_refresh_token'
+      auth_secret_refresh_token: 'auth_secret_refresh_token',
+      auth_secret_confirm_email_token: 'auth_secret_confirm_email_token',
+      auth_secret_forgot_password_token: 'auth_secret_forgot_password_token',
     }
   )
   .withPlatform(new NodePlatform())

package/vector-store/index.js

@@ -71,10 +71,12 @@ export class LibSQLVectorStore {
     };
   }
 
+  /** @type {VectorStore["metric"]} */
   get metric() {
     return this.config.similarity;
   };
 
+  /** @type {VectorStore["dimensions"]} */
   get dimensions() {
     return this.config.dimensions;
   };

package/vector-store/types.d.ts

@@ -1,6 +1,6 @@
 
 import type { AIEmbedder } from '@storecraft/core/ai';
-export * from './index.js';
+export { LibSQLVectorStore, DEFAULT_INDEX_NAME } from './index.js';
 
 export type Config = {
   /** 

package/kysely.turso.utils.js

@@ -1,33 +0,0 @@
-
-/**
- * 
- * @param {string} stmt 
- * @param {any[] | Record<string, any>} params 
- */
-export const prepare_and_bind = (stmt='', params=[]) => {
-  const params_object = Array.isArray(params) ? 
-      params.reduce((a, v, idx) => ({ ...a, [idx+1]: v}), {}) : 
-      params; 
-
-  let current = 0;
-  let result = ''
-  let index_run = 1;
-  for (let m of stmt.matchAll(/\?[0-9]*/g)) {
-    result += stmt.slice(current, m.index);
-  
-    const match_string = m[0];
-    let index_access = match_string.length > 1 ? 
-          Number(match_string.slice(1)) : 
-          index_run;
-  
-    result += "'" + params_object[index_access] + "'";
-  
-    current = m.index + m[0].length;
-    index_run+=1;
-  }
-  
-  result += stmt.slice(current);
-
-  return result;
-}
-