@workglow/storage 0.0.85 → 0.0.87

package/README.md

@@ -28,12 +28,13 @@ Modular storage solutions for Workglow.AI platform with multiple backend impleme
   - [Node.js Environment](#nodejs-environment)
   - [Bun Environment](#bun-environment)
 - [Advanced Features](#advanced-features)
+  - [Repository Registry](#repository-registry)
   - [Event-Driven Architecture](#event-driven-architecture)
   - [Compound Primary Keys](#compound-primary-keys)
   - [Custom File Layout (KV on filesystem)](#custom-file-layout-kv-on-filesystem)
 - [API Reference](#api-reference)
-  - [IKvRepository\<Key, Value\>](#ikvrepositorykey-value)
-  - [ITabularRepository\<Schema, PrimaryKeyNames\>](#itabularrepositoryschema-primarykeynames)
+  - [IKvStorage\<Key, Value\>](#ikvrepositorykey-value)
+  - [ITabularStorage\<Schema, PrimaryKeyNames\>](#itabularrepositoryschema-primarykeynames)
   - [IQueueStorage\<Input, Output\>](#iqueuestorageinput-output)
 - [Examples](#examples)
   - [User Management System](#user-management-system)
@@ -46,16 +47,16 @@ Modular storage solutions for Workglow.AI platform with multiple backend impleme
 
 ```typescript
 // Key-Value Storage (simple data)
-import { InMemoryKvRepository } from "@workglow/storage";
+import { InMemoryKvStorage } from "@workglow/storage";
 
-const kvStore = new InMemoryKvRepository<string, { name: string; age: number }>();
+const kvStore = new InMemoryKvStorage<string, { name: string; age: number }>();
 await kvStore.put("user:123", { name: "Alice", age: 30 });
 const kvUser = await kvStore.get("user:123"); // { name: "Alice", age: 30 }
 ```
 
 ```typescript
 // Tabular Storage (structured data with schemas)
-import { InMemoryTabularRepository } from "@workglow/storage";
+import { InMemoryTabularStorage } from "@workglow/storage";
 import { JsonSchema } from "@workglow/util";
 
 const userSchema = {
@@ -70,7 +71,7 @@ const userSchema = {
   additionalProperties: false,
 } as const satisfies JsonSchema;
 
-const userRepo = new InMemoryTabularRepository<typeof userSchema, ["id"]>(
+const userRepo = new InMemoryTabularStorage<typeof userSchema, ["id"]>(
   userSchema,
   ["id"], // primary key
   ["email"] // additional indexes
@@ -138,7 +139,7 @@ The package uses conditional exports, so importing from `@workglow/storage` auto
 
 ```typescript
 // Import from the top-level package; it resolves to the correct target per environment
-import { InMemoryKvRepository, SqliteTabularRepository } from "@workglow/storage";
+import { InMemoryKvStorage, SqliteTabularStorage } from "@workglow/storage";
 ```
 
 ## Storage Types
@@ -150,10 +151,10 @@ Simple key-value storage for unstructured or semi-structured data.
 #### Basic Usage
 
 ```typescript
-import { InMemoryKvRepository, FsFolderJsonKvRepository } from "@workglow/storage";
+import { InMemoryKvStorage, FsFolderJsonKvRepository } from "@workglow/storage";
 
 // In-memory (for testing/caching)
-const cache = new InMemoryKvRepository<string, any>();
+const cache = new InMemoryKvStorage<string, any>();
 await cache.put("config", { theme: "dark", language: "en" });
 
 // File-based JSON (persistent)
@@ -189,7 +190,7 @@ const supabaseStore = new SupabaseKvRepository(supabase, "settings");
 #### Bulk Operations
 
 ```typescript
-const store = new InMemoryKvRepository<string, { name: string; score: number }>();
+const store = new InMemoryKvStorage<string, { name: string; score: number }>();
 
 // Bulk insert
 await store.putBulk([
@@ -208,7 +209,7 @@ const count = await store.size(); // 2
 #### Event Handling
 
 ```typescript
-const store = new InMemoryKvRepository<string, any>();
+const store = new InMemoryKvStorage<string, any>();
 
 // Listen to storage events
 store.on("put", (key, value) => {
@@ -231,7 +232,7 @@ Structured storage with schemas, primary keys, and indexing for complex data rel
 
 ```typescript
 import { JsonSchema } from "@workglow/util";
-import { InMemoryTabularRepository } from "@workglow/storage";
+import { InMemoryTabularStorage } from "@workglow/storage";
 
 // Define your entity schema
 const UserSchema = {
@@ -249,13 +250,59 @@ const UserSchema = {
 } as const satisfies JsonSchema;
 
 // Create repository with primary key and indexes
-const userRepo = new InMemoryTabularRepository<typeof UserSchema, ["id"]>(
+const userRepo = new InMemoryTabularStorage<typeof UserSchema, ["id"]>(
   UserSchema,
   ["id"], // Primary key (can be compound: ["dept", "id"])
   ["email", "department", ["department", "age"]] // Indexes for fast lookups
 );
 ```
 
+#### Auto-Generated Primary Keys
+
+TabularStorage supports automatic ID generation by marking schema properties with `x-auto-generated: true`:
+
+```typescript
+const UserSchema = {
+  type: "object",
+  properties: {
+    id: { type: "integer", "x-auto-generated": true }, // Auto-increment
+    name: { type: "string" },
+    email: { type: "string" },
+  },
+  required: ["id", "name", "email"],
+} as const;
+
+const storage = new PostgresTabularStorage(db, "users", UserSchema, ["id"] as const);
+
+// Insert without providing ID - database generates it
+const user = await storage.put({ name: "Alice", email: "alice@example.com" });
+console.log(user.id); // 1 (auto-generated)
+```
+
+**Generation Strategy** (inferred from type):
+
+- `type: "integer"` → Auto-increment (SERIAL, INTEGER PRIMARY KEY, counter)
+- `type: "string"` → UUID via `uuid4()` from `@workglow/util`
+
+**Configuration Options:**
+
+```typescript
+new PostgresTabularStorage(
+  db,
+  "users",
+  UserSchema,
+  ["id"],
+  [],
+  { clientProvidedKeys: "if-missing" } // "never" | "if-missing" | "always"
+);
+```
+
+- `"if-missing"` (default): Use client value if provided, generate otherwise
+- `"never"`: Always generate (most secure)
+- `"always"`: Require client value (for testing)
+
+See [Tabular Storage README](./src/tabular/README.md) for detailed documentation.
+
 #### CRUD Operations
 
 ```typescript
@@ -342,9 +389,9 @@ await userRepo.deleteSearch({
 
 ```typescript
 // SQLite (Node.js/Bun)
-import { SqliteTabularRepository } from "@workglow/storage";
+import { SqliteTabularStorage } from "@workglow/storage";
 
-const sqliteUsers = new SqliteTabularRepository<typeof UserSchema, ["id"]>(
+const sqliteUsers = new SqliteTabularStorage<typeof UserSchema, ["id"]>(
   "./users.db",
   "users",
   UserSchema,
@@ -353,11 +400,11 @@ const sqliteUsers = new SqliteTabularRepository<typeof UserSchema, ["id"]>(
 );
 
 // PostgreSQL (Node.js/Bun)
-import { PostgresTabularRepository } from "@workglow/storage";
+import { PostgresTabularStorage } from "@workglow/storage";
 import { Pool } from "pg";
 
 const pool = new Pool({ connectionString: "postgresql://..." });
-const pgUsers = new PostgresTabularRepository<typeof UserSchema, ["id"]>(
+const pgUsers = new PostgresTabularStorage<typeof UserSchema, ["id"]>(
   pool,
   "users",
   UserSchema,
@@ -366,11 +413,11 @@ const pgUsers = new PostgresTabularRepository<typeof UserSchema, ["id"]>(
 );
 
 // Supabase (Node.js/Bun)
-import { SupabaseTabularRepository } from "@workglow/storage";
+import { SupabaseTabularStorage } from "@workglow/storage";
 import { createClient } from "@supabase/supabase-js";
 
 const supabase = createClient("https://your-project.supabase.co", "your-anon-key");
-const supabaseUsers = new SupabaseTabularRepository<typeof UserSchema, ["id"]>(
+const supabaseUsers = new SupabaseTabularStorage<typeof UserSchema, ["id"]>(
   supabase,
   "users",
   UserSchema,
@@ -379,8 +426,8 @@ const supabaseUsers = new SupabaseTabularRepository<typeof UserSchema, ["id"]>(
 );
 
 // IndexedDB (Browser)
-import { IndexedDbTabularRepository } from "@workglow/storage";
-const browserUsers = new IndexedDbTabularRepository<typeof UserSchema, ["id"]>(
+import { IndexedDbTabularStorage } from "@workglow/storage";
+const browserUsers = new IndexedDbTabularStorage<typeof UserSchema, ["id"]>(
   "users",
   UserSchema,
   ["id"],
@@ -388,8 +435,8 @@ const browserUsers = new IndexedDbTabularRepository<typeof UserSchema, ["id"]>(
 );
 
 // File-based (Node.js/Bun)
-import { FsFolderTabularRepository } from "@workglow/storage";
-const fileUsers = new FsFolderTabularRepository<typeof UserSchema, ["id"]>(
+import { FsFolderTabularStorage } from "@workglow/storage";
+const fileUsers = new FsFolderTabularStorage<typeof UserSchema, ["id"]>(
   "./data/users",
   UserSchema,
   ["id"],
@@ -463,23 +510,23 @@ await jobQueue.deleteJobsByStatusAndAge(JobStatus.COMPLETED, 24 * 60 * 60 * 1000
 ```typescript
 import {
   IndexedDbKvRepository,
-  IndexedDbTabularRepository,
+  IndexedDbTabularStorage,
   IndexedDbQueueStorage,
   SupabaseKvRepository,
-  SupabaseTabularRepository,
+  SupabaseTabularStorage,
   SupabaseQueueStorage,
 } from "@workglow/storage";
 import { createClient } from "@supabase/supabase-js";
 
 // Local browser storage with IndexedDB
 const settings = new IndexedDbKvRepository("app-settings");
-const userData = new IndexedDbTabularRepository("users", UserSchema, ["id"]);
+const userData = new IndexedDbTabularStorage("users", UserSchema, ["id"]);
 const jobQueue = new IndexedDbQueueStorage<any, any>("background-jobs");
 
 // Or use Supabase for cloud storage from the browser
 const supabase = createClient("https://your-project.supabase.co", "your-anon-key");
 const cloudSettings = new SupabaseKvRepository(supabase, "app-settings");
-const cloudUserData = new SupabaseTabularRepository(supabase, "users", UserSchema, ["id"]);
+const cloudUserData = new SupabaseTabularStorage(supabase, "users", UserSchema, ["id"]);
 const cloudJobQueue = new SupabaseQueueStorage(supabase, "background-jobs");
 ```
 
@@ -488,13 +535,13 @@ const cloudJobQueue = new SupabaseQueueStorage(supabase, "background-jobs");
 ```typescript
 import {
   SqliteKvRepository,
-  PostgresTabularRepository,
+  PostgresTabularStorage,
   FsFolderJsonKvRepository,
 } from "@workglow/storage";
 
 // Mix and match storage backends
 const cache = new FsFolderJsonKvRepository("./cache");
-const users = new PostgresTabularRepository(pool, "users", UserSchema, ["id"]);
+const users = new PostgresTabularStorage(pool, "users", UserSchema, ["id"]);
 ```
 
 ### Bun Environment
@@ -502,31 +549,120 @@ const users = new PostgresTabularRepository(pool, "users", UserSchema, ["id"]);
 ```typescript
 // Bun has access to all implementations
 import {
-  SqliteTabularRepository,
+  SqliteTabularStorage,
   FsFolderJsonKvRepository,
   PostgresQueueStorage,
-  SupabaseTabularRepository,
+  SupabaseTabularStorage,
 } from "@workglow/storage";
 
 import { Database } from "bun:sqlite";
 import { createClient } from "@supabase/supabase-js";
 
 const db = new Database("./app.db");
-const data = new SqliteTabularRepository(db, "items", ItemSchema, ["id"]);
+const data = new SqliteTabularStorage(db, "items", ItemSchema, ["id"]);
 
 // Or use Supabase for cloud storage
 const supabase = createClient("https://your-project.supabase.co", "your-anon-key");
-const cloudData = new SupabaseTabularRepository(supabase, "items", ItemSchema, ["id"]);
+const cloudData = new SupabaseTabularStorage(supabase, "items", ItemSchema, ["id"]);
 ```
 
 ## Advanced Features
 
+### Repository Registry
+
+Repositories can be registered globally by ID, allowing tasks to reference them by name rather than passing direct instances. This is useful for configuring repositories once at application startup and referencing them throughout your task graphs.
+
+#### Registering Repositories
+
+```typescript
+import {
+  registerTabularStorage,
+  getTabularStorage,
+  InMemoryTabularStorage,
+} from "@workglow/storage";
+
+// Define your schema
+const userSchema = {
+  type: "object",
+  properties: {
+    id: { type: "string" },
+    name: { type: "string" },
+    email: { type: "string" },
+  },
+  required: ["id", "name", "email"],
+  additionalProperties: false,
+} as const;
+
+// Create and register a repository
+const userRepo = new InMemoryTabularStorage(userSchema, ["id"] as const);
+registerTabularStorage("users", userRepo);
+
+// Later, retrieve the repository by ID
+const repo = getTabularStorage("users");
+```
+
+#### Using Repositories in Tasks
+
+When using repositories with tasks, you can pass either the repository ID or a direct instance. The TaskRunner automatically resolves string IDs using the registry.
+
+```typescript
+import { TypeTabularStorage } from "@workglow/storage";
+
+// In your task's input schema, use TypeTabularStorage
+static inputSchema() {
+  return {
+    type: "object",
+    properties: {
+      dataSource: TypeTabularStorage({
+        title: "User Repository",
+        description: "Repository containing user records",
+      }),
+    },
+    required: ["dataSource"],
+  };
+}
+
+// Both approaches work:
+await task.run({ dataSource: "users" });           // Resolved from registry
+await task.run({ dataSource: userRepoInstance }); // Direct instance
+```
+
+#### Schema Helper Functions
+
+The package provides schema helper functions for defining repository inputs with proper format annotations:
+
+```typescript
+import {
+  TypeTabularStorage,
+  TypeVectorRepository,
+  TypeDocumentRepository,
+} from "@workglow/storage";
+
+// Tabular repository (format: "storage:tabular")
+const tabularSchema = TypeTabularStorage({
+  title: "Data Source",
+  description: "Tabular data repository",
+});
+
+// Vector repository (format: "repository:document-node-vector")
+const vectorSchema = TypeVectorRepository({
+  title: "Embeddings Store",
+  description: "Vector embeddings repository",
+});
+
+// Document repository (format: "repository:document")
+const docSchema = TypeDocumentRepository({
+  title: "Document Store",
+  description: "Document storage repository",
+});
+```
+
 ### Event-Driven Architecture
 
 All storage implementations support event emission for monitoring and reactive programming:
 
 ```typescript
-const store = new InMemoryTabularRepository(UserSchema, ["id"]);
+const store = new InMemoryTabularStorage(UserSchema, ["id"]);
 
 // Monitor all operations
 store.on("put", (entity) => console.log("User created/updated:", entity));
@@ -555,7 +691,7 @@ const OrderLineSchema = {
   additionalProperties: false,
 } as const satisfies JsonSchema;
 
-const orderLines = new InMemoryTabularRepository<typeof OrderLineSchema, ["orderId", "lineNumber"]>(
+const orderLines = new InMemoryTabularStorage<typeof OrderLineSchema, ["orderId", "lineNumber"]>(
   OrderLineSchema,
   ["orderId", "lineNumber"], // Compound primary key
   ["productId"] // Additional index
@@ -594,12 +730,12 @@ await files.put("note-1", "Hello world");
 
 ## API Reference
 
-### IKvRepository<Key, Value>
+### IKvStorage<Key, Value>
 
 Core interface for key-value storage:
 
 ```typescript
-interface IKvRepository<Key, Value> {
+interface IKvStorage<Key, Value> {
   // Core operations
   put(key: Key, value: Value): Promise<void>;
   putBulk(items: Array<{ key: Key; value: Value }>): Promise<void>;
@@ -618,12 +754,12 @@ interface IKvRepository<Key, Value> {
 }
 ```
 
-### ITabularRepository<Schema, PrimaryKeyNames>
+### ITabularStorage<Schema, PrimaryKeyNames>
 
 Core interface for tabular storage:
 
 ```typescript
-interface ITabularRepository<Schema, PrimaryKeyNames, Entity, PrimaryKey, Value> {
+interface ITabularStorage<Schema, PrimaryKeyNames, Entity, PrimaryKey, Value> {
   // Core operations
   put(entity: Entity): Promise<void>;
   putBulk(entities: Entity[]): Promise<void>;
@@ -705,7 +841,7 @@ interface IQueueStorage<Input, Output> {
 
 ```typescript
 import { JsonSchema, FromSchema } from "@workglow/util";
-import { InMemoryTabularRepository, InMemoryKvRepository } from "@workglow/storage";
+import { InMemoryTabularStorage, InMemoryKvStorage } from "@workglow/storage";
 
 // User profile with tabular storage
 const UserSchema = {
@@ -727,14 +863,14 @@ const UserSchema = {
   additionalProperties: false,
 } as const satisfies JsonSchema;
 
-const userRepo = new InMemoryTabularRepository<typeof UserSchema, ["id"]>(
+const userRepo = new InMemoryTabularStorage<typeof UserSchema, ["id"]>(
   UserSchema,
   ["id"],
   ["email", "username"]
 );
 
 // User sessions with KV storage
-const sessionStore = new InMemoryKvRepository<string, { userId: string; expiresAt: string }>();
+const sessionStore = new InMemoryKvStorage<string, { userId: string; expiresAt: string }>();
 
 // User management class
 class UserManager {
@@ -841,7 +977,7 @@ class ConfigManager {
 import { createClient } from "@supabase/supabase-js";
 import { JsonSchema } from "@workglow/util";
 import {
-  SupabaseTabularRepository,
+  SupabaseTabularStorage,
   SupabaseKvRepository,
   SupabaseQueueStorage,
 } from "@workglow/storage";
@@ -882,7 +1018,7 @@ const OrderSchema = {
 } as const satisfies JsonSchema;
 
 // Create repositories
-const products = new SupabaseTabularRepository<typeof ProductSchema, ["id"]>(
+const products = new SupabaseTabularStorage<typeof ProductSchema, ["id"]>(
   supabase,
   "products",
   ProductSchema,
@@ -890,7 +1026,7 @@ const products = new SupabaseTabularRepository<typeof ProductSchema, ["id"]>(
   ["category", "name"] // Indexed columns for fast searching
 );
 
-const orders = new SupabaseTabularRepository<typeof OrderSchema, ["id"]>(
+const orders = new SupabaseTabularStorage<typeof OrderSchema, ["id"]>(
   supabase,
   "orders",
   OrderSchema,
@@ -972,7 +1108,7 @@ bun test
 
 # Run specific test suites
 bun test --grep "KvRepository"
-bun test --grep "TabularRepository"
+bun test --grep "TabularStorage"
 bun test --grep "QueueStorage"
 
 # Test specific environments
@@ -985,17 +1121,13 @@ bun test --grep "Sqlite"      # Native tests
 
 ```typescript
 import { describe, test, expect, beforeEach } from "vitest";
-import { InMemoryTabularRepository } from "@workglow/storage";
+import { InMemoryTabularStorage } from "@workglow/storage";
 
 describe("UserRepository", () => {
-  let userRepo: InMemoryTabularRepository<typeof UserSchema, ["id"]>;
+  let userRepo: InMemoryTabularStorage<typeof UserSchema, ["id"]>;
 
   beforeEach(() => {
-    userRepo = new InMemoryTabularRepository<typeof UserSchema, ["id"]>(
-      UserSchema,
-      ["id"],
-      ["email"]
-    );
+    userRepo = new InMemoryTabularStorage<typeof UserSchema, ["id"]>(UserSchema, ["id"], ["email"]);
   });
 
   test("should create and retrieve user", async () => {