@workglow/storage 0.0.125 → 0.1.0

package/README.md

@@ -142,6 +142,10 @@ The package uses conditional exports, so importing from `@workglow/storage` auto
 import { InMemoryKvStorage, SqliteTabularStorage } from "@workglow/storage";
 ```
 
+### SQLite setup (`@workglow/storage/sqlite`)
+
+Before you open SQLite with a **file path** or construct **`new Sqlite.Database(...)`**, call **`await Sqlite.init()`** once per runtime (Node.js, Bun, or browser). Export is available from **`workglow`** and **`@workglow/storage/sqlite`**. The call is idempotent. On the browser it loads the SQLite WASM build; on Node.js it loads `better-sqlite3`; on Bun it resolves `bun:sqlite`.
+
 ## Storage Types
 
 ### Key-Value Storage
@@ -171,12 +175,15 @@ const browserStore = new IndexedDbKvRepository("my-app-storage");
 
 // Node.js/Bun (using SQLite)
 import { SqliteKvRepository } from "@workglow/storage";
-// Pass a file path or a Database instance (see @workglow/sqlite)
+import { Sqlite } from "@workglow/storage/sqlite";
+
+await Sqlite.init();
+// Pass a file path or a Sqlite.Database instance (see @workglow/storage/sqlite)
 const sqliteStore = new SqliteKvRepository("./data.db", "config_table");
 
 // PostgreSQL (Node.js/Bun)
 import { PostgresKvRepository } from "@workglow/storage";
-import { Pool } from "pg";
+import type { Pool } from "@workglow/storage/postgres";
 const pool = new Pool({ connectionString: "postgresql://..." });
 const pgStore = new PostgresKvRepository(pool, "settings");
 
@@ -390,7 +397,9 @@ await userRepo.deleteSearch({
 ```typescript
 // SQLite (Node.js/Bun)
 import { SqliteTabularStorage } from "@workglow/storage";
+import { Sqlite } from "@workglow/storage/sqlite";
 
+await Sqlite.init();
 const sqliteUsers = new SqliteTabularStorage<typeof UserSchema, ["id"]>(
   "./users.db",
   "users",
@@ -401,7 +410,7 @@ const sqliteUsers = new SqliteTabularStorage<typeof UserSchema, ["id"]>(
 
 // PostgreSQL (Node.js/Bun)
 import { PostgresTabularStorage } from "@workglow/storage";
-import { Pool } from "pg";
+import type { Pool } from "@workglow/storage/postgres";
 
 const pool = new Pool({ connectionString: "postgresql://..." });
 const pgUsers = new PostgresTabularStorage<typeof UserSchema, ["id"]>(
@@ -554,11 +563,11 @@ import {
   PostgresQueueStorage,
   SupabaseTabularStorage,
 } from "@workglow/storage";
-
-import { Database } from "bun:sqlite";
+import { Sqlite } from "@workglow/storage/sqlite";
 import { createClient } from "@supabase/supabase-js";
 
-const db = new Database("./app.db");
+await Sqlite.init();
+const db = new Sqlite.Database("./app.db");
 const data = new SqliteTabularStorage(db, "items", ItemSchema, ["id"]);
 
 // Or use Supabase for cloud storage

package/dist/browser.d.ts

@@ -0,0 +1,18 @@
+/**
+ * @license
+ * Copyright 2025 Steven Roussey <sroussey@gmail.com>
+ * SPDX-License-Identifier: Apache-2.0
+ */
+export * from "./common";
+export * from "./tabular/IndexedDbTabularStorage";
+export * from "./tabular/SharedInMemoryTabularStorage";
+export * from "./tabular/SupabaseTabularStorage";
+export * from "./kv/IndexedDbKvStorage";
+export * from "./kv/SupabaseKvStorage";
+export * from "./queue/IndexedDbQueueStorage";
+export * from "./queue/SupabaseQueueStorage";
+export * from "./queue-limiter/IndexedDbRateLimiterStorage";
+export * from "./queue-limiter/SupabaseRateLimiterStorage";
+export * from "./vector/IndexedDbVectorStorage";
+export * from "./util/IndexedDbTable";
+//# sourceMappingURL=browser.d.ts.map

package/dist/browser.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"browser.d.ts","sourceRoot":"","sources":["../src/browser.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,UAAU,CAAC;AAEzB,cAAc,mCAAmC,CAAC;AAClD,cAAc,wCAAwC,CAAC;AACvD,cAAc,kCAAkC,CAAC;AAEjD,cAAc,yBAAyB,CAAC;AACxC,cAAc,wBAAwB,CAAC;AAEvC,cAAc,+BAA+B,CAAC;AAC9C,cAAc,8BAA8B,CAAC;AAE7C,cAAc,6CAA6C,CAAC;AAC5D,cAAc,4CAA4C,CAAC;AAE3D,cAAc,iCAAiC,CAAC;AAEhD,cAAc,uBAAuB,CAAC"}

package/dist/browser.js

@@ -391,6 +391,7 @@ var MEMORY_TABULAR_REPOSITORY = createServiceToken2("storage.tabularRepository.i
 class InMemoryTabularStorage extends BaseTabularStorage {
   values = new Map;
   autoIncrementCounter = 0;
+  _lastPutWasInsert = false;
   constructor(schema, primaryKeyNames, indexes = [], clientProvidedKeys = "if-missing") {
     super(schema, primaryKeyNames, indexes, clientProvidedKeys);
   }
@@ -426,6 +427,7 @@ class InMemoryTabularStorage extends BaseTabularStorage {
     }
     const { key } = this.separateKeyValueFromCombined(entityToStore);
     const id = await makeFingerprint2(key);
+    this._lastPutWasInsert = !this.values.has(id);
     this.values.set(id, entityToStore);
     this.events.emit("put", entityToStore);
     return entityToStore;
@@ -622,7 +624,7 @@ class InMemoryTabularStorage extends BaseTabularStorage {
   }
   subscribeToChanges(callback, options) {
     const handlePut = (entity) => {
-      callback({ type: "UPDATE", new: entity });
+      callback({ type: this._lastPutWasInsert ? "INSERT" : "UPDATE", new: entity });
     };
     const handleDelete = (_key) => {
       callback({ type: "DELETE" });
@@ -651,6 +653,7 @@ class CachedTabularStorage extends BaseTabularStorage {
   cache;
   durable;
   cacheInitialized = false;
+  cacheInitPromise = null;
   constructor(durable, cache, schema, primaryKeyNames, indexes, clientProvidedKeys = "if-missing") {
     if (!schema || !primaryKeyNames) {
       throw new Error("Schema and primaryKeyNames must be provided when creating CachedTabularStorage");
@@ -684,16 +687,23 @@ class CachedTabularStorage extends BaseTabularStorage {
   async initializeCache() {
     if (this.cacheInitialized)
       return;
-    try {
-      const all = await this.durable.getAll();
-      if (all && all.length > 0) {
-        await this.cache.putBulk(all);
-      }
-      this.cacheInitialized = true;
-    } catch (error) {
-      getLogger().warn("Failed to initialize cache from durable repository:", { error });
-      this.cacheInitialized = true;
+    if (this.cacheInitPromise) {
+      return this.cacheInitPromise;
     }
+    this.cacheInitPromise = (async () => {
+      try {
+        const all = await this.durable.getAll();
+        if (all && all.length > 0) {
+          await this.cache.putBulk(all);
+        }
+        this.cacheInitialized = true;
+      } catch (error) {
+        getLogger().warn("Failed to initialize cache from durable repository:", { error });
+      } finally {
+        this.cacheInitPromise = null;
+      }
+    })();
+    return this.cacheInitPromise;
   }
   async put(value) {
     await this.initializeCache();
@@ -1529,13 +1539,17 @@ class InMemoryQueueStorage {
   }
   async complete(job) {
     await sleep(0);
-    const index = this.jobQueue.findIndex((j) => j.id === job.id);
+    const jobWithPrefixes = job;
+    const index = this.jobQueue.findIndex((j) => j.id === job.id && this.matchesPrefixes(j));
     if (index !== -1) {
       const existing = this.jobQueue[index];
       const currentAttempts = existing?.run_attempts ?? 0;
-      job.run_attempts = currentAttempts + 1;
-      this.jobQueue[index] = job;
-      this.events.emit("change", { type: "UPDATE", old: existing, new: job });
+      jobWithPrefixes.run_attempts = currentAttempts + 1;
+      for (const [key, value] of Object.entries(this.prefixValues)) {
+        jobWithPrefixes[key] = value;
+      }
+      this.jobQueue[index] = jobWithPrefixes;
+      this.events.emit("change", { type: "UPDATE", old: existing, new: jobWithPrefixes });
     }
   }
   async abort(id) {
@@ -1890,6 +1904,7 @@ class PollingSubscriptionManager {
   intervals = new Map;
   lastKnownState = new Map;
   initialized = false;
+  initializing = false;
   fetchState;
   compareItems;
   payloadFactory;
@@ -1914,7 +1929,8 @@ class PollingSubscriptionManager {
       this.intervals.set(interval, intervalGroup);
       if (!this.initialized) {
         this.initialized = true;
-        this.initAndPoll(subscribers, subscription);
+        this.initializing = true;
+        this.initAndPoll(subscription);
       } else {
         this.pollForNewSubscriber(subscription);
       }
@@ -1933,7 +1949,7 @@ class PollingSubscriptionManager {
       }
     };
   }
-  async initAndPoll(subscribers, newSubscription) {
+  async initAndPoll(newSubscription) {
     try {
       this.lastKnownState = await this.fetchState();
       for (const [, item] of this.lastKnownState) {
@@ -1942,7 +1958,9 @@ class PollingSubscriptionManager {
           newSubscription.callback(payload);
         } catch {}
       }
-    } catch {}
+    } catch {} finally {
+      this.initializing = false;
+    }
   }
   pollForNewSubscriber(subscription) {
     for (const [, item] of this.lastKnownState) {
@@ -1955,6 +1973,8 @@ class PollingSubscriptionManager {
   async poll(subscribers) {
     if (subscribers.size === 0)
       return;
+    if (this.initializing)
+      return;
     try {
       const currentState = await this.fetchState();
       const changes = [];
@@ -1998,6 +2018,7 @@ class PollingSubscriptionManager {
     this.intervals.clear();
     this.lastKnownState.clear();
     this.initialized = false;
+    this.initializing = false;
   }
 }
 // src/vector/InMemoryVectorStorage.ts
@@ -2102,7 +2123,7 @@ class InMemoryVectorStorage extends InMemoryTabularStorage {
         continue;
       }
       const vectorScore = cosineSimilarity(query, vector);
-      const metadataText = Object.values(metadata).join(" ").toLowerCase();
+      const metadataText = Object.values(metadata ?? {}).join(" ").toLowerCase();
       const textScore = textRelevance(metadataText, textQuery);
       const combinedScore = vectorWeight * vectorScore + (1 - vectorWeight) * textScore;
       if (combinedScore < scoreThreshold) {
@@ -4484,12 +4505,24 @@ class SupabaseQueueStorage {
     const conditions = this.prefixes.map((p) => {
       const value = this.prefixValues[p.name];
       if (p.type === "uuid") {
-        return `${p.name} = '${this.escapeSqlString(String(value))}'`;
+        const validated = this.validateSqlValue(String(value), `prefix "${p.name}"`);
+        return `${p.name} = '${this.escapeSqlString(validated)}'`;
+      }
+      const numValue = Number(value ?? 0);
+      if (!Number.isFinite(numValue)) {
+        throw new Error(`Invalid numeric prefix value for "${p.name}": ${value}`);
       }
-      return `${p.name} = ${Number(value ?? 0)}`;
+      return `${p.name} = ${numValue}`;
     }).join(" AND ");
     return " AND " + conditions;
   }
+  static SAFE_SQL_VALUE_RE = /^[a-zA-Z0-9_\-.:]+$/;
+  validateSqlValue(value, context) {
+    if (!SupabaseQueueStorage.SAFE_SQL_VALUE_RE.test(value)) {
+      throw new Error(`Unsafe value for ${context}: "${value}". Values must match /^[a-zA-Z0-9_\\-.:]+$/.`);
+    }
+    return value;
+  }
   escapeSqlString(value) {
     return value.replace(/'/g, "''");
   }
@@ -4596,8 +4629,10 @@ class SupabaseQueueStorage {
   }
   async next(workerId) {
     const prefixConditions = this.buildPrefixWhereSql();
-    const escapedQueueName = this.escapeSqlString(this.queueName);
-    const escapedWorkerId = this.escapeSqlString(workerId);
+    const validatedQueueName = this.validateSqlValue(this.queueName, "queueName");
+    const validatedWorkerId = this.validateSqlValue(workerId, "workerId");
+    const escapedQueueName = this.escapeSqlString(validatedQueueName);
+    const escapedWorkerId = this.escapeSqlString(validatedWorkerId);
     const sql = `
       UPDATE ${this.tableName}
       SET status = '${JobStatus.PROCESSING}', last_ran_at = NOW() AT TIME ZONE 'UTC', worker_id = '${escapedWorkerId}'
@@ -5469,4 +5504,4 @@ export {
   BaseTabularStorage
 };
 
-//# debugId=D02D6875010DA7A164756E2164756E21
+//# debugId=09B8497E3C06E04164756E2164756E21