@workglow/storage 0.0.126 → 0.1.1

package/dist/node.js

@@ -394,6 +394,7 @@ var MEMORY_TABULAR_REPOSITORY = createServiceToken2("storage.tabularRepository.i
 class InMemoryTabularStorage extends BaseTabularStorage {
   values = new Map;
   autoIncrementCounter = 0;
+  _lastPutWasInsert = false;
   constructor(schema, primaryKeyNames, indexes = [], clientProvidedKeys = "if-missing") {
     super(schema, primaryKeyNames, indexes, clientProvidedKeys);
   }
@@ -429,6 +430,7 @@ class InMemoryTabularStorage extends BaseTabularStorage {
     }
     const { key } = this.separateKeyValueFromCombined(entityToStore);
     const id = await makeFingerprint2(key);
+    this._lastPutWasInsert = !this.values.has(id);
     this.values.set(id, entityToStore);
     this.events.emit("put", entityToStore);
     return entityToStore;
@@ -625,7 +627,7 @@ class InMemoryTabularStorage extends BaseTabularStorage {
   }
   subscribeToChanges(callback, options) {
     const handlePut = (entity) => {
-      callback({ type: "UPDATE", new: entity });
+      callback({ type: this._lastPutWasInsert ? "INSERT" : "UPDATE", new: entity });
     };
     const handleDelete = (_key) => {
       callback({ type: "DELETE" });
@@ -654,6 +656,7 @@ class CachedTabularStorage extends BaseTabularStorage {
   cache;
   durable;
   cacheInitialized = false;
+  cacheInitPromise = null;
   constructor(durable, cache, schema, primaryKeyNames, indexes, clientProvidedKeys = "if-missing") {
     if (!schema || !primaryKeyNames) {
       throw new Error("Schema and primaryKeyNames must be provided when creating CachedTabularStorage");
@@ -687,16 +690,23 @@ class CachedTabularStorage extends BaseTabularStorage {
   async initializeCache() {
     if (this.cacheInitialized)
       return;
-    try {
-      const all = await this.durable.getAll();
-      if (all && all.length > 0) {
-        await this.cache.putBulk(all);
-      }
-      this.cacheInitialized = true;
-    } catch (error) {
-      getLogger().warn("Failed to initialize cache from durable repository:", { error });
-      this.cacheInitialized = true;
+    if (this.cacheInitPromise) {
+      return this.cacheInitPromise;
     }
+    this.cacheInitPromise = (async () => {
+      try {
+        const all = await this.durable.getAll();
+        if (all && all.length > 0) {
+          await this.cache.putBulk(all);
+        }
+        this.cacheInitialized = true;
+      } catch (error) {
+        getLogger().warn("Failed to initialize cache from durable repository:", { error });
+      } finally {
+        this.cacheInitPromise = null;
+      }
+    })();
+    return this.cacheInitPromise;
   }
   async put(value) {
     await this.initializeCache();
@@ -1532,13 +1542,17 @@ class InMemoryQueueStorage {
   }
   async complete(job) {
     await sleep(0);
-    const index = this.jobQueue.findIndex((j) => j.id === job.id);
+    const jobWithPrefixes = job;
+    const index = this.jobQueue.findIndex((j) => j.id === job.id && this.matchesPrefixes(j));
     if (index !== -1) {
       const existing = this.jobQueue[index];
       const currentAttempts = existing?.run_attempts ?? 0;
-      job.run_attempts = currentAttempts + 1;
-      this.jobQueue[index] = job;
-      this.events.emit("change", { type: "UPDATE", old: existing, new: job });
+      jobWithPrefixes.run_attempts = currentAttempts + 1;
+      for (const [key, value] of Object.entries(this.prefixValues)) {
+        jobWithPrefixes[key] = value;
+      }
+      this.jobQueue[index] = jobWithPrefixes;
+      this.events.emit("change", { type: "UPDATE", old: existing, new: jobWithPrefixes });
     }
   }
   async abort(id) {
@@ -1893,6 +1907,7 @@ class PollingSubscriptionManager {
   intervals = new Map;
   lastKnownState = new Map;
   initialized = false;
+  initializing = false;
   fetchState;
   compareItems;
   payloadFactory;
@@ -1917,7 +1932,8 @@ class PollingSubscriptionManager {
       this.intervals.set(interval, intervalGroup);
       if (!this.initialized) {
         this.initialized = true;
-        this.initAndPoll(subscribers, subscription);
+        this.initializing = true;
+        this.initAndPoll(subscription);
       } else {
         this.pollForNewSubscriber(subscription);
       }
@@ -1936,7 +1952,7 @@ class PollingSubscriptionManager {
       }
     };
   }
-  async initAndPoll(subscribers, newSubscription) {
+  async initAndPoll(newSubscription) {
     try {
       this.lastKnownState = await this.fetchState();
       for (const [, item] of this.lastKnownState) {
@@ -1945,7 +1961,9 @@ class PollingSubscriptionManager {
           newSubscription.callback(payload);
         } catch {}
       }
-    } catch {}
+    } catch {} finally {
+      this.initializing = false;
+    }
   }
   pollForNewSubscriber(subscription) {
     for (const [, item] of this.lastKnownState) {
@@ -1958,6 +1976,8 @@ class PollingSubscriptionManager {
   async poll(subscribers) {
     if (subscribers.size === 0)
       return;
+    if (this.initializing)
+      return;
     try {
       const currentState = await this.fetchState();
       const changes = [];
@@ -2001,6 +2021,7 @@ class PollingSubscriptionManager {
     this.intervals.clear();
     this.lastKnownState.clear();
     this.initialized = false;
+    this.initializing = false;
   }
 }
 // src/vector/InMemoryVectorStorage.ts
@@ -2105,7 +2126,7 @@ class InMemoryVectorStorage extends InMemoryTabularStorage {
         continue;
       }
       const vectorScore = cosineSimilarity(query, vector);
-      const metadataText = Object.values(metadata).join(" ").toLowerCase();
+      const metadataText = Object.values(metadata ?? {}).join(" ").toLowerCase();
       const textScore = textRelevance(metadataText, textQuery);
       const combinedScore = vectorWeight * vectorScore + (1 - vectorWeight) * textScore;
       if (combinedScore < scoreThreshold) {
@@ -2217,7 +2238,7 @@ class EncryptedKvCredentialStore {
   }
 }
 // src/tabular/FsFolderTabularStorage.ts
-import { createServiceToken as createServiceToken12, makeFingerprint as makeFingerprint5, sleep as sleep3, uuid4 as uuid43 } from "@workglow/util";
+import { createServiceToken as createServiceToken12, getLogger as getLogger3, makeFingerprint as makeFingerprint5, sleep as sleep3, uuid4 as uuid43 } from "@workglow/util";
 import { mkdir, readdir, readFile, rm, writeFile } from "node:fs/promises";
 import path from "node:path";
 var FS_FOLDER_TABULAR_REPOSITORY = createServiceToken12("storage.tabularRepository.fsFolder");
@@ -2274,11 +2295,11 @@ class FsFolderTabularStorage extends BaseTabularStorage {
     try {
       await writeFile(filePath, JSON.stringify(entityToStore));
     } catch (error) {
+      await sleep3(1);
       try {
-        await sleep3(1);
-        await writeFile(filePath, JSON.stringify(entity));
-      } catch (error2) {
-        console.error("Error writing file", filePath, error2);
+        await writeFile(filePath, JSON.stringify(entityToStore));
+      } catch (retryError) {
+        throw new Error(`Failed to write file "${filePath}" after retry: ${retryError instanceof Error ? retryError.message : String(retryError)}`, { cause: retryError });
       }
     }
     this.events.emit("put", entityToStore);
@@ -2357,11 +2378,24 @@ class FsFolderTabularStorage extends BaseTabularStorage {
     if (jsonFiles.length === 0) {
       return;
     }
-    const allEntities = await Promise.all(jsonFiles.map(async (file) => {
+    const results = await Promise.allSettled(jsonFiles.map(async (file) => {
       const filePath = path.join(this.folderPath, file);
-      const content = await readFile(filePath, "utf8");
-      return JSON.parse(content);
+      try {
+        const content = await readFile(filePath, "utf8");
+        return JSON.parse(content);
+      } catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        throw new Error(`Failed to read or parse "${filePath}": ${message}`);
+      }
     }));
+    const allEntities = [];
+    for (const result of results) {
+      if (result.status === "fulfilled") {
+        allEntities.push(result.value);
+      } else {
+        getLogger3().warn(`Skipping corrupted file in getBulk: ${result.reason?.message ?? result.reason}`);
+      }
+    }
     allEntities.sort((a, b) => {
       for (const key of this.primaryKeyNames) {
         const aVal = a[key];
@@ -4675,7 +4709,7 @@ class SqliteQueueStorage {
     return result;
   }
   async peek(status = JobStatus.PENDING, num = 100) {
-    num = Number(num) || 100;
+    num = Math.max(1, Math.min(1e4, Math.floor(Number(num) || 100)));
     const prefixConditions = this.buildPrefixWhereClause();
     const prefixParams = this.getPrefixParamValues();
     const FutureJobQuery = `
@@ -4930,12 +4964,24 @@ class SupabaseQueueStorage {
     const conditions = this.prefixes.map((p) => {
       const value = this.prefixValues[p.name];
       if (p.type === "uuid") {
-        return `${p.name} = '${this.escapeSqlString(String(value))}'`;
+        const validated = this.validateSqlValue(String(value), `prefix "${p.name}"`);
+        return `${p.name} = '${this.escapeSqlString(validated)}'`;
+      }
+      const numValue = Number(value ?? 0);
+      if (!Number.isFinite(numValue)) {
+        throw new Error(`Invalid numeric prefix value for "${p.name}": ${value}`);
       }
-      return `${p.name} = ${Number(value ?? 0)}`;
+      return `${p.name} = ${numValue}`;
     }).join(" AND ");
     return " AND " + conditions;
   }
+  static SAFE_SQL_VALUE_RE = /^[a-zA-Z0-9_\-.:]+$/;
+  validateSqlValue(value, context) {
+    if (!SupabaseQueueStorage.SAFE_SQL_VALUE_RE.test(value)) {
+      throw new Error(`Unsafe value for ${context}: "${value}". Values must match /^[a-zA-Z0-9_\\-.:]+$/.`);
+    }
+    return value;
+  }
   escapeSqlString(value) {
     return value.replace(/'/g, "''");
   }
@@ -5042,8 +5088,10 @@ class SupabaseQueueStorage {
   }
   async next(workerId) {
     const prefixConditions = this.buildPrefixWhereSql();
-    const escapedQueueName = this.escapeSqlString(this.queueName);
-    const escapedWorkerId = this.escapeSqlString(workerId);
+    const validatedQueueName = this.validateSqlValue(this.queueName, "queueName");
+    const validatedWorkerId = this.validateSqlValue(workerId, "workerId");
+    const escapedQueueName = this.escapeSqlString(validatedQueueName);
+    const escapedWorkerId = this.escapeSqlString(validatedWorkerId);
     const sql = `
       UPDATE ${this.tableName}
       SET status = '${JobStatus.PROCESSING}', last_ran_at = NOW() AT TIME ZONE 'UTC', worker_id = '${escapedWorkerId}'
@@ -5804,6 +5852,8 @@ class SupabaseRateLimiterStorage {
 }
 // src/vector/PostgresVectorStorage.ts
 import { cosineSimilarity as cosineSimilarity2 } from "@workglow/util/schema";
+var SAFE_IDENTIFIER_RE = /^[a-zA-Z_][a-zA-Z0-9_]*$/;
+
 class PostgresVectorStorage extends PostgresTabularStorage {
   vectorDimensions;
   VectorType;
@@ -5840,6 +5890,9 @@ class PostgresVectorStorage extends PostgresTabularStorage {
       if (filter && Object.keys(filter).length > 0 && metadataCol) {
         const conditions = [];
         for (const [key, value] of Object.entries(filter)) {
+          if (!SAFE_IDENTIFIER_RE.test(key)) {
+            throw new StorageValidationError(`Invalid metadata filter key: "${key}". Keys must match /^[a-zA-Z_][a-zA-Z0-9_]*$/.`);
+          }
           conditions.push(`${metadataCol}->>'${key}' = $${paramIndex}`);
           params.push(String(value));
           paramIndex++;
@@ -5868,7 +5921,10 @@ class PostgresVectorStorage extends PostgresTabularStorage {
       }
       return results;
     } catch (error) {
-      console.warn("pgvector query failed, falling back to in-memory search:", error);
+      if (error instanceof StorageValidationError) {
+        throw error;
+      }
+      console.error("pgvector query failed, falling back to in-memory search:", error);
       return this.searchFallback(query, options);
     }
   }
@@ -5879,7 +5935,7 @@ class PostgresVectorStorage extends PostgresTabularStorage {
     }
     try {
       const queryVector = `[${Array.from(query).join(",")}]`;
-      const tsQuery = textQuery.split(/\s+/).join(" & ");
+      const tsQueryText = textQuery;
       const vectorCol = String(this.vectorPropertyName);
       const metadataCol = this.metadataPropertyName ? String(this.metadataPropertyName) : null;
       let sql = `
@@ -5887,15 +5943,18 @@ class PostgresVectorStorage extends PostgresTabularStorage {
           *,
           (
             $2 * (1 - (${vectorCol} <=> $1::vector)) +
-            $3 * ts_rank(to_tsvector('english', ${metadataCol || "''"}::text), to_tsquery('english', $4))
+            $3 * ts_rank(to_tsvector('english', ${metadataCol || "''"}::text), plainto_tsquery('english', $4))
           ) as score
         FROM "${this.table}"
       `;
-      const params = [queryVector, vectorWeight, 1 - vectorWeight, tsQuery];
+      const params = [queryVector, vectorWeight, 1 - vectorWeight, tsQueryText];
       let paramIndex = 5;
       if (filter && Object.keys(filter).length > 0 && metadataCol) {
         const conditions = [];
         for (const [key, value] of Object.entries(filter)) {
+          if (!SAFE_IDENTIFIER_RE.test(key)) {
+            throw new StorageValidationError(`Invalid metadata filter key: "${key}". Keys must match /^[a-zA-Z_][a-zA-Z0-9_]*$/.`);
+          }
           conditions.push(`${metadataCol}->>'${key}' = $${paramIndex}`);
           params.push(String(value));
           paramIndex++;
@@ -5906,7 +5965,7 @@ class PostgresVectorStorage extends PostgresTabularStorage {
         sql += filter ? " AND" : " WHERE";
         sql += ` (
           $2 * (1 - (${vectorCol} <=> $1::vector)) +
-          $3 * ts_rank(to_tsvector('english', ${metadataCol || "''"}::text), to_tsquery('english', $4))
+          $3 * ts_rank(to_tsvector('english', ${metadataCol || "''"}::text), plainto_tsquery('english', $4))
         ) >= $${paramIndex}`;
         params.push(scoreThreshold);
         paramIndex++;
@@ -5927,7 +5986,10 @@ class PostgresVectorStorage extends PostgresTabularStorage {
       }
       return results;
     } catch (error) {
-      console.warn("pgvector hybrid query failed, falling back to in-memory search:", error);
+      if (error instanceof StorageValidationError) {
+        throw error;
+      }
+      console.error("pgvector hybrid query failed, falling back to in-memory search:", error);
       return this.hybridSearchFallback(query, options);
     }
   }
@@ -5963,7 +6025,7 @@ class PostgresVectorStorage extends PostgresTabularStorage {
         continue;
       }
       const vectorScore = cosineSimilarity2(query, vector);
-      const metadataText = JSON.stringify(metadata).toLowerCase();
+      const metadataText = Object.values(metadata ?? {}).join(" ").toLowerCase();
       let textScore = 0;
       if (queryWords.length > 0) {
         let matches = 0;
@@ -6074,7 +6136,7 @@ class SqliteVectorStorage extends SqliteTabularStorage {
         continue;
       }
       const vectorScore = cosineSimilarity3(query, vector);
-      const metadataText = JSON.stringify(metadata).toLowerCase();
+      const metadataText = Object.values(metadata ?? {}).join(" ").toLowerCase();
       let textScore = 0;
       if (queryWords.length > 0) {
         let matches = 0;
@@ -6408,7 +6470,7 @@ class SqliteAiVectorStorage extends SqliteTabularStorage {
         if (filter && !matchesFilter3(metadata, filter)) {
           continue;
         }
-        const metadataText = JSON.stringify(metadata).toLowerCase();
+        const metadataText = Object.values(metadata ?? {}).join(" ").toLowerCase();
         let textScore = 0;
         if (queryWords.length > 0) {
           let matches = 0;
@@ -6463,7 +6525,7 @@ class SqliteAiVectorStorage extends SqliteTabularStorage {
         continue;
       }
       const vectorScore = cosineSimilarity4(query, vector);
-      const metadataText = JSON.stringify(metadata).toLowerCase();
+      const metadataText = Object.values(metadata ?? {}).join(" ").toLowerCase();
       let textScore = 0;
       if (queryWords.length > 0) {
         let matches = 0;
@@ -8258,4 +8320,4 @@ export {
   BaseTabularStorage
 };
 
-//# debugId=CB06EB0A7CC1A40D64756E2164756E21
+//# debugId=120AE6AECCC1B17664756E2164756E21