@workglow/storage 0.0.97 → 0.0.99

package/dist/browser.js

@@ -123,6 +123,42 @@ class BaseTabularStorage {
   waitOn(name) {
     return this.events.waitOn(name);
   }
+  async* records(pageSize = 100) {
+    if (pageSize <= 0) {
+      throw new RangeError(`pageSize must be greater than 0, got ${pageSize}`);
+    }
+    let offset = 0;
+    while (true) {
+      const page = await this.getBulk(offset, pageSize);
+      if (!page || page.length === 0) {
+        break;
+      }
+      for (const entity of page) {
+        yield entity;
+      }
+      if (page.length < pageSize) {
+        break;
+      }
+      offset += pageSize;
+    }
+  }
+  async* pages(pageSize = 100) {
+    if (pageSize <= 0) {
+      throw new RangeError(`pageSize must be greater than 0, got ${pageSize}`);
+    }
+    let offset = 0;
+    while (true) {
+      const page = await this.getBulk(offset, pageSize);
+      if (!page || page.length === 0) {
+        break;
+      }
+      yield page;
+      if (page.length < pageSize) {
+        break;
+      }
+      offset += pageSize;
+    }
+  }
   subscribeToChanges(_callback, _options) {
     throw new Error(`subscribeToChanges is not implemented for ${this.constructor.name}. ` + `All concrete repository implementations should override this method.`);
   }
@@ -346,6 +382,22 @@ class InMemoryTabularStorage extends BaseTabularStorage {
   async size() {
     return this.values.size;
   }
+  async getBulk(offset, limit) {
+    const all = Array.from(this.values.values());
+    all.sort((a, b) => {
+      for (const key of this.primaryKeyNames) {
+        const aVal = a[key];
+        const bVal = b[key];
+        if (aVal < bVal)
+          return -1;
+        if (aVal > bVal)
+          return 1;
+      }
+      return 0;
+    });
+    const page = all.slice(offset, offset + limit);
+    return page.length > 0 ? page : undefined;
+  }
   async deleteSearch(criteria) {
     const criteriaKeys = Object.keys(criteria);
     if (criteriaKeys.length === 0) {
@@ -531,6 +583,10 @@ class CachedTabularStorage extends BaseTabularStorage {
     await this.initializeCache();
     return await this.durable.size();
   }
+  async getBulk(offset, limit) {
+    await this.initializeCache();
+    return await this.durable.getBulk(offset, limit);
+  }
   async deleteSearch(criteria) {
     await this.initializeCache();
     await this.durable.deleteSearch(criteria);
@@ -564,13 +620,262 @@ class CachedTabularStorage extends BaseTabularStorage {
     this.cache.destroy();
   }
 }
+// src/tabular/HuggingFaceTabularStorage.ts
+import {
+  createServiceToken as createServiceToken4
+} from "@workglow/util";
+var HF_TABULAR_REPOSITORY = createServiceToken4("storage.tabularRepository.huggingface");
+
+class HuggingFaceTabularStorage extends BaseTabularStorage {
+  dataset;
+  config;
+  split;
+  token;
+  baseUrl;
+  constructor(dataset, config, split, schema, primaryKeyNames, options) {
+    super(schema, primaryKeyNames, options?.indexes ?? [], "never");
+    this.dataset = dataset;
+    this.config = config;
+    this.split = split;
+    this.token = options?.token;
+    this.baseUrl = options?.baseUrl ?? "https://datasets-server.huggingface.co";
+  }
+  static async fromDataset(dataset, config, split, options) {
+    const baseUrl = options?.baseUrl ?? "https://datasets-server.huggingface.co";
+    const token = options?.token;
+    const url = new URL(`${baseUrl}/first-rows`);
+    url.searchParams.set("dataset", dataset);
+    url.searchParams.set("config", config);
+    url.searchParams.set("split", split);
+    const headers = {};
+    if (token) {
+      headers["Authorization"] = `Bearer ${token}`;
+    }
+    const response = await fetch(url.toString(), { headers });
+    if (!response.ok) {
+      throw new Error(`Failed to fetch dataset features: ${response.status} ${response.statusText}`);
+    }
+    const data = await response.json();
+    const properties = {};
+    const required = [];
+    properties["row_idx"] = { type: "integer", "x-auto-generated": true };
+    required.push("row_idx");
+    for (const feature of data.features) {
+      const jsonSchema = hfFeatureToJsonSchema(feature.type);
+      properties[feature.name] = jsonSchema;
+      required.push(feature.name);
+    }
+    const schema = {
+      type: "object",
+      properties,
+      required,
+      additionalProperties: false
+    };
+    return new HuggingFaceTabularStorage(dataset, config, split, schema, ["row_idx"], options);
+  }
+  async setupDatabase() {
+    const data = await this.fetchApi("/first-rows", {});
+    const schemaColumns = Object.keys(this.schema.properties);
+    const hfColumns = data.features.map((f) => f.name);
+    const hasRowIdx = schemaColumns.includes("row_idx");
+    if (!hasRowIdx) {
+      for (const column of schemaColumns) {
+        if (!hfColumns.includes(column) && !this.primaryKeyNames.includes(column)) {
+          throw new Error(`Schema column "${column}" not found in HuggingFace dataset features`);
+        }
+      }
+    }
+  }
+  async get(key) {
+    const keyObj = this.separateKeyValueFromCombined({ ...key }).key;
+    const whereConditions = [];
+    for (const [k, v] of Object.entries(keyObj)) {
+      if (typeof v === "string") {
+        const escaped = v.replace(/\\/g, "\\\\").replace(/'/g, "\\'");
+        whereConditions.push(`${k}='${escaped}'`);
+      } else {
+        whereConditions.push(`${k}=${v}`);
+      }
+    }
+    const where = whereConditions.join(" AND ");
+    const data = await this.fetchApi("/filter", { where, limit: "1" });
+    if (data.rows.length > 0) {
+      const entity = this.rowToEntity(data.rows[0]);
+      this.events.emit("get", key, entity);
+      return entity;
+    }
+    this.events.emit("get", key, undefined);
+    return;
+  }
+  async getAll() {
+    const allEntities = [];
+    let offset = 0;
+    const pageSize = 100;
+    while (true) {
+      const page = await this.getBulk(offset, pageSize);
+      if (!page || page.length === 0) {
+        break;
+      }
+      allEntities.push(...page);
+      offset += page.length;
+      if (page.length < pageSize) {
+        break;
+      }
+    }
+    return allEntities.length > 0 ? allEntities : undefined;
+  }
+  async getBulk(offset, limit) {
+    const data = await this.fetchApi("/rows", {
+      offset: offset.toString(),
+      length: Math.min(limit, 100).toString()
+    });
+    if (data.rows.length === 0) {
+      return;
+    }
+    const entities = [];
+    for (const row of data.rows) {
+      entities.push(this.rowToEntity(row));
+    }
+    return entities;
+  }
+  async search(key) {
+    const searchKeys = Object.keys(key);
+    if (searchKeys.length === 0) {
+      return;
+    }
+    const bestIndex = this.findBestMatchingIndex(searchKeys);
+    if (!bestIndex) {
+      throw new Error(`No suitable index found for the search criteria, searching for ['${searchKeys.join("', '")}'] with pk ['${this.primaryKeyNames.join("', '")}'] and indexes ['${this.indexes.map((idx) => idx.join(",")).join("', '")}'`);
+    }
+    const whereConditions = [];
+    for (const [k, v] of Object.entries(key)) {
+      if (v !== undefined && v !== null) {
+        if (typeof v === "string") {
+          const escaped = v.replace(/\\/g, "\\\\").replace(/'/g, "\\'");
+          whereConditions.push(`${k}='${escaped}'`);
+        } else {
+          whereConditions.push(`${k}=${v}`);
+        }
+      }
+    }
+    if (whereConditions.length === 0) {
+      throw new Error("Search criteria must include at least one non-null and non-undefined value to build a valid WHERE clause.");
+    }
+    const where = whereConditions.join(" AND ");
+    const allEntities = [];
+    let offset = 0;
+    const limit = 100;
+    while (true) {
+      const data = await this.fetchApi("/filter", {
+        where,
+        offset: offset.toString(),
+        limit: limit.toString()
+      });
+      for (const row of data.rows) {
+        allEntities.push(this.rowToEntity(row));
+      }
+      offset += data.rows.length;
+      if (offset >= data.num_rows_total || data.rows.length < limit) {
+        break;
+      }
+    }
+    if (allEntities.length > 0) {
+      this.events.emit("search", key, allEntities);
+      return allEntities;
+    } else {
+      this.events.emit("search", key, undefined);
+      return;
+    }
+  }
+  async size() {
+    const data = await this.fetchApi("/size", {});
+    return data.size.num_rows;
+  }
+  async put(_value) {
+    throw new Error("HuggingFaceTabularStorage is readonly");
+  }
+  async putBulk(_values) {
+    throw new Error("HuggingFaceTabularStorage is readonly");
+  }
+  async delete(_value) {
+    throw new Error("HuggingFaceTabularStorage is readonly");
+  }
+  async deleteAll() {
+    throw new Error("HuggingFaceTabularStorage is readonly");
+  }
+  async deleteSearch(_criteria) {
+    throw new Error("HuggingFaceTabularStorage is readonly");
+  }
+  subscribeToChanges(_callback, _options) {
+    throw new Error("HuggingFaceTabularStorage does not support subscriptions");
+  }
+  destroy() {}
+  async fetchApi(endpoint, params) {
+    const url = new URL(`${this.baseUrl}${endpoint}`);
+    url.searchParams.set("dataset", this.dataset);
+    url.searchParams.set("config", this.config);
+    url.searchParams.set("split", this.split);
+    for (const [key, value] of Object.entries(params)) {
+      if (value !== undefined) {
+        url.searchParams.set(key, value);
+      }
+    }
+    const headers = {};
+    if (this.token) {
+      headers["Authorization"] = `Bearer ${this.token}`;
+    }
+    const response = await fetch(url.toString(), { headers });
+    if (!response.ok) {
+      throw new Error(`HuggingFace API error: ${response.status} ${response.statusText}`);
+    }
+    return await response.json();
+  }
+  rowToEntity(row) {
+    return { row_idx: row.row_idx, ...row.row };
+  }
+}
+function hfFeatureToJsonSchema(feature) {
+  if (feature._type === "Value") {
+    switch (feature.dtype) {
+      case "string":
+        return { type: "string" };
+      case "int64":
+      case "int32":
+      case "int16":
+      case "int8":
+      case "uint64":
+      case "uint32":
+      case "uint16":
+      case "uint8":
+        return { type: "integer" };
+      case "float64":
+      case "float32":
+      case "float16":
+        return { type: "number" };
+      case "bool":
+        return { type: "boolean" };
+      default:
+        return {};
+    }
+  }
+  if (feature._type === "ClassLabel") {
+    return { type: "integer" };
+  }
+  if (feature._type === "Sequence") {
+    return {
+      type: "array",
+      items: hfFeatureToJsonSchema(feature.feature)
+    };
+  }
+  return {};
+}
 // src/tabular/TabularStorageRegistry.ts
 import {
-  createServiceToken as createServiceToken4,
+  createServiceToken as createServiceToken5,
   globalServiceRegistry,
   registerInputResolver
 } from "@workglow/util";
-var TABULAR_REPOSITORIES = createServiceToken4("storage.tabular.repositories");
+var TABULAR_REPOSITORIES = createServiceToken5("storage.tabular.repositories");
 if (!globalServiceRegistry.has(TABULAR_REPOSITORIES)) {
   globalServiceRegistry.register(TABULAR_REPOSITORIES, () => new Map, true);
 }
@@ -604,11 +909,11 @@ var DefaultKeyValueSchema = {
 };
 var DefaultKeyValueKey = ["key"];
 // src/kv/InMemoryKvStorage.ts
-import { createServiceToken as createServiceToken6 } from "@workglow/util";
+import { createServiceToken as createServiceToken7 } from "@workglow/util";
 
 // src/kv/KvStorage.ts
-import { createServiceToken as createServiceToken5, EventEmitter as EventEmitter2, makeFingerprint as makeFingerprint3 } from "@workglow/util";
-var KV_REPOSITORY = createServiceToken5("storage.kvRepository");
+import { createServiceToken as createServiceToken6, EventEmitter as EventEmitter2, makeFingerprint as makeFingerprint3 } from "@workglow/util";
+var KV_REPOSITORY = createServiceToken6("storage.kvRepository");
 
 class KvStorage {
   keySchema;
@@ -715,7 +1020,7 @@ class KvViaTabularStorage extends KvStorage {
 }
 
 // src/kv/InMemoryKvStorage.ts
-var MEMORY_KV_REPOSITORY = createServiceToken6("storage.kvRepository.inMemory");
+var MEMORY_KV_REPOSITORY = createServiceToken7("storage.kvRepository.inMemory");
 
 class InMemoryKvStorage extends KvViaTabularStorage {
   tabularRepository;
@@ -725,11 +1030,11 @@ class InMemoryKvStorage extends KvViaTabularStorage {
   }
 }
 // src/queue/InMemoryQueueStorage.ts
-import { createServiceToken as createServiceToken8, EventEmitter as EventEmitter3, makeFingerprint as makeFingerprint4, sleep, uuid4 as uuid42 } from "@workglow/util";
+import { createServiceToken as createServiceToken9, EventEmitter as EventEmitter3, makeFingerprint as makeFingerprint4, sleep, uuid4 as uuid42 } from "@workglow/util";
 
 // src/queue/IQueueStorage.ts
-import { createServiceToken as createServiceToken7 } from "@workglow/util";
-var QUEUE_STORAGE = createServiceToken7("jobqueue.storage");
+import { createServiceToken as createServiceToken8 } from "@workglow/util";
+var QUEUE_STORAGE = createServiceToken8("jobqueue.storage");
 var JobStatus = {
   PENDING: "PENDING",
   PROCESSING: "PROCESSING",
@@ -740,7 +1045,7 @@ var JobStatus = {
 };
 
 // src/queue/InMemoryQueueStorage.ts
-var IN_MEMORY_QUEUE_STORAGE = createServiceToken8("jobqueue.storage.inMemory");
+var IN_MEMORY_QUEUE_STORAGE = createServiceToken9("jobqueue.storage.inMemory");
 
 class InMemoryQueueStorage {
   queueName;
@@ -917,8 +1222,8 @@ class InMemoryQueueStorage {
   }
 }
 // src/queue-limiter/InMemoryRateLimiterStorage.ts
-import { createServiceToken as createServiceToken9, sleep as sleep2 } from "@workglow/util";
-var IN_MEMORY_RATE_LIMITER_STORAGE = createServiceToken9("ratelimiter.storage.inMemory");
+import { createServiceToken as createServiceToken10, sleep as sleep2 } from "@workglow/util";
+var IN_MEMORY_RATE_LIMITER_STORAGE = createServiceToken10("ratelimiter.storage.inMemory");
 
 class InMemoryRateLimiterStorage {
   prefixValues;
@@ -976,8 +1281,8 @@ class InMemoryRateLimiterStorage {
   }
 }
 // src/queue-limiter/IRateLimiterStorage.ts
-import { createServiceToken as createServiceToken10 } from "@workglow/util";
-var RATE_LIMITER_STORAGE = createServiceToken10("ratelimiter.storage");
+import { createServiceToken as createServiceToken11 } from "@workglow/util";
+var RATE_LIMITER_STORAGE = createServiceToken11("ratelimiter.storage");
 // src/util/HybridSubscriptionManager.ts
 class HybridSubscriptionManager {
   subscribers = new Set;
@@ -1371,7 +1676,7 @@ class InMemoryVectorStorage extends InMemoryTabularStorage {
 }
 // src/tabular/IndexedDbTabularStorage.ts
 import {
-  createServiceToken as createServiceToken11,
+  createServiceToken as createServiceToken12,
   makeFingerprint as makeFingerprint5,
   uuid4 as uuid43
 } from "@workglow/util";
@@ -1714,7 +2019,7 @@ async function dropIndexedDbTable(tableName) {
 }
 
 // src/tabular/IndexedDbTabularStorage.ts
-var IDB_TABULAR_REPOSITORY = createServiceToken11("storage.tabularRepository.indexedDb");
+var IDB_TABULAR_REPOSITORY = createServiceToken12("storage.tabularRepository.indexedDb");
 
 class IndexedDbTabularStorage extends BaseTabularStorage {
   table;
@@ -2005,6 +2310,41 @@ class IndexedDbTabularStorage extends BaseTabularStorage {
       request.onsuccess = () => resolve(request.result);
     });
   }
+  async getBulk(offset, limit) {
+    if (offset < 0) {
+      throw new RangeError(`offset must be non-negative, got ${offset}`);
+    }
+    if (limit <= 0) {
+      return;
+    }
+    const db = await this.getDb();
+    return new Promise((resolve, reject) => {
+      const transaction = db.transaction(this.table, "readonly");
+      const store = transaction.objectStore(this.table);
+      const request = store.openCursor();
+      const entities = [];
+      let skipped = false;
+      request.onerror = () => reject(request.error);
+      request.onsuccess = () => {
+        const cursor = request.result;
+        if (cursor) {
+          if (!skipped && offset > 0) {
+            skipped = true;
+            cursor.advance(offset);
+            return;
+          }
+          entities.push(cursor.value);
+          if (entities.length === limit) {
+            resolve(entities);
+            return;
+          }
+          cursor.continue();
+        } else {
+          resolve(entities.length > 0 ? entities : undefined);
+        }
+      };
+    });
+  }
   matchesCriteria(record, criteria) {
     for (const column of Object.keys(criteria)) {
       const criterion = criteria[column];
@@ -2130,9 +2470,9 @@ class IndexedDbTabularStorage extends BaseTabularStorage {
 }
 // src/tabular/SharedInMemoryTabularStorage.ts
 import {
-  createServiceToken as createServiceToken12
+  createServiceToken as createServiceToken13
 } from "@workglow/util";
-var SHARED_IN_MEMORY_TABULAR_REPOSITORY = createServiceToken12("storage.tabularRepository.sharedInMemory");
+var SHARED_IN_MEMORY_TABULAR_REPOSITORY = createServiceToken13("storage.tabularRepository.sharedInMemory");
 
 class SharedInMemoryTabularStorage extends BaseTabularStorage {
   channel = null;
@@ -2276,6 +2616,9 @@ class SharedInMemoryTabularStorage extends BaseTabularStorage {
   async size() {
     return await this.inMemoryRepo.size();
   }
+  async getBulk(offset, limit) {
+    return await this.inMemoryRepo.getBulk(offset, limit);
+  }
   async deleteSearch(criteria) {
     await this.inMemoryRepo.deleteSearch(criteria);
     this.broadcast({
@@ -2296,7 +2639,7 @@ class SharedInMemoryTabularStorage extends BaseTabularStorage {
 }
 // src/tabular/SupabaseTabularStorage.ts
 import {
-  createServiceToken as createServiceToken13
+  createServiceToken as createServiceToken14
 } from "@workglow/util";
 
 // src/tabular/BaseSqlTabularStorage.ts
@@ -2483,7 +2826,7 @@ class BaseSqlTabularStorage extends BaseTabularStorage {
 }
 
 // src/tabular/SupabaseTabularStorage.ts
-var SUPABASE_TABULAR_REPOSITORY = createServiceToken13("storage.tabularRepository.supabase");
+var SUPABASE_TABULAR_REPOSITORY = createServiceToken14("storage.tabularRepository.supabase");
 
 class SupabaseTabularStorage extends BaseSqlTabularStorage {
   client;
@@ -2824,6 +3167,24 @@ class SupabaseTabularStorage extends BaseSqlTabularStorage {
       throw error;
     return count ?? 0;
   }
+  async getBulk(offset, limit) {
+    let query = this.client.from(this.table).select("*");
+    for (const pkName of this.primaryKeyNames) {
+      query = query.order(String(pkName));
+    }
+    const { data, error } = await query.range(offset, offset + limit - 1);
+    if (error)
+      throw error;
+    if (!data || data.length === 0) {
+      return;
+    }
+    for (const row of data) {
+      for (const key in this.schema.properties) {
+        row[key] = this.sqlToJsValue(key, row[key]);
+      }
+    }
+    return data;
+  }
   async deleteSearch(criteria) {
     const criteriaKeys = Object.keys(criteria);
     if (criteriaKeys.length === 0) {
@@ -2902,8 +3263,8 @@ class SupabaseTabularStorage extends BaseSqlTabularStorage {
   }
 }
 // src/kv/IndexedDbKvStorage.ts
-import { createServiceToken as createServiceToken14 } from "@workglow/util";
-var IDB_KV_REPOSITORY = createServiceToken14("storage.kvRepository.indexedDb");
+import { createServiceToken as createServiceToken15 } from "@workglow/util";
+var IDB_KV_REPOSITORY = createServiceToken15("storage.kvRepository.indexedDb");
 
 class IndexedDbKvStorage extends KvViaTabularStorage {
   dbName;
@@ -2915,8 +3276,8 @@ class IndexedDbKvStorage extends KvViaTabularStorage {
   }
 }
 // src/kv/SupabaseKvStorage.ts
-import { createServiceToken as createServiceToken15 } from "@workglow/util";
-var SUPABASE_KV_REPOSITORY = createServiceToken15("storage.kvRepository.supabase");
+import { createServiceToken as createServiceToken16 } from "@workglow/util";
+var SUPABASE_KV_REPOSITORY = createServiceToken16("storage.kvRepository.supabase");
 
 class SupabaseKvStorage extends KvViaTabularStorage {
   client;
@@ -2930,8 +3291,8 @@ class SupabaseKvStorage extends KvViaTabularStorage {
   }
 }
 // src/queue/IndexedDbQueueStorage.ts
-import { createServiceToken as createServiceToken16, makeFingerprint as makeFingerprint6, uuid4 as uuid44 } from "@workglow/util";
-var INDEXED_DB_QUEUE_STORAGE = createServiceToken16("jobqueue.storage.indexedDb");
+import { createServiceToken as createServiceToken17, makeFingerprint as makeFingerprint6, uuid4 as uuid44 } from "@workglow/util";
+var INDEXED_DB_QUEUE_STORAGE = createServiceToken17("jobqueue.storage.indexedDb");
 
 class IndexedDbQueueStorage {
   queueName;
@@ -3499,8 +3860,8 @@ class IndexedDbQueueStorage {
   }
 }
 // src/queue/SupabaseQueueStorage.ts
-import { createServiceToken as createServiceToken17, makeFingerprint as makeFingerprint7, uuid4 as uuid45 } from "@workglow/util";
-var SUPABASE_QUEUE_STORAGE = createServiceToken17("jobqueue.storage.supabase");
+import { createServiceToken as createServiceToken18, makeFingerprint as makeFingerprint7, uuid4 as uuid45 } from "@workglow/util";
+var SUPABASE_QUEUE_STORAGE = createServiceToken18("jobqueue.storage.supabase");
 
 class SupabaseQueueStorage {
   client;
@@ -3999,8 +4360,8 @@ class SupabaseQueueStorage {
   }
 }
 // src/queue-limiter/IndexedDbRateLimiterStorage.ts
-import { createServiceToken as createServiceToken18 } from "@workglow/util";
-var INDEXED_DB_RATE_LIMITER_STORAGE = createServiceToken18("ratelimiter.storage.indexedDb");
+import { createServiceToken as createServiceToken19 } from "@workglow/util";
+var INDEXED_DB_RATE_LIMITER_STORAGE = createServiceToken19("ratelimiter.storage.indexedDb");
 
 class IndexedDbRateLimiterStorage {
   executionDb;
@@ -4220,8 +4581,8 @@ class IndexedDbRateLimiterStorage {
   }
 }
 // src/queue-limiter/SupabaseRateLimiterStorage.ts
-import { createServiceToken as createServiceToken19 } from "@workglow/util";
-var SUPABASE_RATE_LIMITER_STORAGE = createServiceToken19("ratelimiter.storage.supabase");
+import { createServiceToken as createServiceToken20 } from "@workglow/util";
+var SUPABASE_RATE_LIMITER_STORAGE = createServiceToken20("ratelimiter.storage.supabase");
 
 class SupabaseRateLimiterStorage {
   client;
@@ -4418,6 +4779,8 @@ export {
   IDB_TABULAR_REPOSITORY,
   IDB_KV_REPOSITORY,
   HybridSubscriptionManager,
+  HuggingFaceTabularStorage,
+  HF_TABULAR_REPOSITORY,
   DefaultKeyValueSchema,
   DefaultKeyValueKey,
   CachedTabularStorage,
@@ -4425,4 +4788,4 @@ export {
   BaseTabularStorage
 };
 
-//# debugId=DD95869160C198CF64756E2164756E21
+//# debugId=7DE4087156DCB1A264756E2164756E21