@workglow/storage 0.0.84 → 0.0.86
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +185 -53
- package/dist/browser.js +451 -188
- package/dist/browser.js.map +21 -18
- package/dist/bun.js +1069 -356
- package/dist/bun.js.map +29 -24
- package/dist/common-server.d.ts +17 -15
- package/dist/common-server.d.ts.map +1 -1
- package/dist/common.d.ts +13 -10
- package/dist/common.d.ts.map +1 -1
- package/dist/kv/{FsFolderJsonKvRepository.d.ts → FsFolderJsonKvStorage.d.ts} +8 -8
- package/dist/kv/FsFolderJsonKvStorage.d.ts.map +1 -0
- package/dist/kv/{FsFolderKvRepository.d.ts → FsFolderKvStorage.d.ts} +7 -7
- package/dist/kv/FsFolderKvStorage.d.ts.map +1 -0
- package/dist/kv/{IKvRepository.d.ts → IKvStorage.d.ts} +3 -3
- package/dist/kv/IKvStorage.d.ts.map +1 -0
- package/dist/kv/{InMemoryKvRepository.d.ts → InMemoryKvStorage.d.ts} +8 -8
- package/dist/kv/InMemoryKvStorage.d.ts.map +1 -0
- package/dist/kv/{IndexedDbKvRepository.d.ts → IndexedDbKvStorage.d.ts} +8 -8
- package/dist/kv/IndexedDbKvStorage.d.ts.map +1 -0
- package/dist/kv/{KvRepository.d.ts → KvStorage.d.ts} +7 -7
- package/dist/kv/KvStorage.d.ts.map +1 -0
- package/dist/kv/{KvViaTabularRepository.d.ts → KvViaTabularStorage.d.ts} +7 -7
- package/dist/kv/KvViaTabularStorage.d.ts.map +1 -0
- package/dist/kv/{PostgresKvRepository.d.ts → PostgresKvStorage.d.ts} +8 -8
- package/dist/kv/PostgresKvStorage.d.ts.map +1 -0
- package/dist/kv/{SqliteKvRepository.d.ts → SqliteKvStorage.d.ts} +8 -8
- package/dist/kv/SqliteKvStorage.d.ts.map +1 -0
- package/dist/kv/{SupabaseKvRepository.d.ts → SupabaseKvStorage.d.ts} +9 -9
- package/dist/kv/SupabaseKvStorage.d.ts.map +1 -0
- package/dist/node.js +1069 -356
- package/dist/node.js.map +29 -24
- package/dist/queue-limiter/IRateLimiterStorage.d.ts.map +1 -0
- package/dist/queue-limiter/InMemoryRateLimiterStorage.d.ts.map +1 -0
- package/dist/queue-limiter/IndexedDbRateLimiterStorage.d.ts.map +1 -0
- package/dist/queue-limiter/PostgresRateLimiterStorage.d.ts.map +1 -0
- package/dist/queue-limiter/SqliteRateLimiterStorage.d.ts.map +1 -0
- package/dist/queue-limiter/SupabaseRateLimiterStorage.d.ts.map +1 -0
- package/dist/tabular/{BaseSqlTabularRepository.d.ts → BaseSqlTabularStorage.d.ts} +8 -7
- package/dist/tabular/BaseSqlTabularStorage.d.ts.map +1 -0
- package/dist/tabular/{TabularRepository.d.ts → BaseTabularStorage.d.ts} +52 -10
- package/dist/tabular/BaseTabularStorage.d.ts.map +1 -0
- package/dist/tabular/{CachedTabularRepository.d.ts → CachedTabularStorage.d.ts} +15 -14
- package/dist/tabular/CachedTabularStorage.d.ts.map +1 -0
- package/dist/tabular/{FsFolderTabularRepository.d.ts → FsFolderTabularStorage.d.ts} +22 -12
- package/dist/tabular/FsFolderTabularStorage.d.ts.map +1 -0
- package/dist/tabular/{ITabularRepository.d.ts → ITabularStorage.d.ts} +29 -6
- package/dist/tabular/ITabularStorage.d.ts.map +1 -0
- package/dist/tabular/{InMemoryTabularRepository.d.ts → InMemoryTabularStorage.d.ts} +24 -14
- package/dist/tabular/InMemoryTabularStorage.d.ts.map +1 -0
- package/dist/tabular/{IndexedDbTabularRepository.d.ts → IndexedDbTabularStorage.d.ts} +20 -11
- package/dist/tabular/IndexedDbTabularStorage.d.ts.map +1 -0
- package/dist/tabular/{PostgresTabularRepository.d.ts → PostgresTabularStorage.d.ts} +37 -15
- package/dist/tabular/PostgresTabularStorage.d.ts.map +1 -0
- package/dist/tabular/{SharedInMemoryTabularRepository.d.ts → SharedInMemoryTabularStorage.d.ts} +14 -13
- package/dist/tabular/SharedInMemoryTabularStorage.d.ts.map +1 -0
- package/dist/tabular/{SqliteTabularRepository.d.ts → SqliteTabularStorage.d.ts} +25 -11
- package/dist/tabular/SqliteTabularStorage.d.ts.map +1 -0
- package/dist/tabular/{SupabaseTabularRepository.d.ts → SupabaseTabularStorage.d.ts} +17 -15
- package/dist/tabular/SupabaseTabularStorage.d.ts.map +1 -0
- package/dist/tabular/TabularStorageRegistry.d.ts +29 -0
- package/dist/tabular/TabularStorageRegistry.d.ts.map +1 -0
- package/dist/util/IndexedDbTable.d.ts +1 -1
- package/dist/util/IndexedDbTable.d.ts.map +1 -1
- package/dist/vector/IVectorStorage.d.ts +83 -0
- package/dist/vector/IVectorStorage.d.ts.map +1 -0
- package/dist/vector/InMemoryVectorStorage.d.ts +41 -0
- package/dist/vector/InMemoryVectorStorage.d.ts.map +1 -0
- package/dist/vector/PostgresVectorStorage.d.ts +57 -0
- package/dist/vector/PostgresVectorStorage.d.ts.map +1 -0
- package/dist/vector/SqliteVectorStorage.d.ts +45 -0
- package/dist/vector/SqliteVectorStorage.d.ts.map +1 -0
- package/package.json +5 -5
- package/src/kv/README.md +3 -3
- package/src/tabular/README.md +186 -23
- package/src/vector/README.md +393 -0
- package/dist/kv/FsFolderJsonKvRepository.d.ts.map +0 -1
- package/dist/kv/FsFolderKvRepository.d.ts.map +0 -1
- package/dist/kv/IKvRepository.d.ts.map +0 -1
- package/dist/kv/InMemoryKvRepository.d.ts.map +0 -1
- package/dist/kv/IndexedDbKvRepository.d.ts.map +0 -1
- package/dist/kv/KvRepository.d.ts.map +0 -1
- package/dist/kv/KvViaTabularRepository.d.ts.map +0 -1
- package/dist/kv/PostgresKvRepository.d.ts.map +0 -1
- package/dist/kv/SqliteKvRepository.d.ts.map +0 -1
- package/dist/kv/SupabaseKvRepository.d.ts.map +0 -1
- package/dist/limiter/IRateLimiterStorage.d.ts.map +0 -1
- package/dist/limiter/InMemoryRateLimiterStorage.d.ts.map +0 -1
- package/dist/limiter/IndexedDbRateLimiterStorage.d.ts.map +0 -1
- package/dist/limiter/PostgresRateLimiterStorage.d.ts.map +0 -1
- package/dist/limiter/SqliteRateLimiterStorage.d.ts.map +0 -1
- package/dist/limiter/SupabaseRateLimiterStorage.d.ts.map +0 -1
- package/dist/tabular/BaseSqlTabularRepository.d.ts.map +0 -1
- package/dist/tabular/CachedTabularRepository.d.ts.map +0 -1
- package/dist/tabular/FsFolderTabularRepository.d.ts.map +0 -1
- package/dist/tabular/ITabularRepository.d.ts.map +0 -1
- package/dist/tabular/InMemoryTabularRepository.d.ts.map +0 -1
- package/dist/tabular/IndexedDbTabularRepository.d.ts.map +0 -1
- package/dist/tabular/PostgresTabularRepository.d.ts.map +0 -1
- package/dist/tabular/SharedInMemoryTabularRepository.d.ts.map +0 -1
- package/dist/tabular/SqliteTabularRepository.d.ts.map +0 -1
- package/dist/tabular/SupabaseTabularRepository.d.ts.map +0 -1
- package/dist/tabular/TabularRepository.d.ts.map +0 -1
- /package/dist/{limiter → queue-limiter}/IRateLimiterStorage.d.ts +0 -0
- /package/dist/{limiter → queue-limiter}/InMemoryRateLimiterStorage.d.ts +0 -0
- /package/dist/{limiter → queue-limiter}/IndexedDbRateLimiterStorage.d.ts +0 -0
- /package/dist/{limiter → queue-limiter}/PostgresRateLimiterStorage.d.ts +0 -0
- /package/dist/{limiter → queue-limiter}/SqliteRateLimiterStorage.d.ts +0 -0
- /package/dist/{limiter → queue-limiter}/SupabaseRateLimiterStorage.d.ts +0 -0
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @license
|
|
3
|
+
* Copyright 2025 Steven Roussey <sroussey@gmail.com>
|
|
4
|
+
* SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
*/
|
|
6
|
+
import type { DataPortSchemaObject, FromSchema, TypedArray, TypedArraySchemaOptions } from "@workglow/util";
|
|
7
|
+
import type { Pool } from "pg";
|
|
8
|
+
import { PostgresTabularStorage } from "../tabular/PostgresTabularStorage";
|
|
9
|
+
import { type HybridSearchOptions, type IVectorStorage, type VectorSearchOptions } from "./IVectorStorage";
|
|
10
|
+
/**
|
|
11
|
+
* PostgreSQL vector repository implementation using pgvector extension.
|
|
12
|
+
* Extends PostgresTabularStorage for storage.
|
|
13
|
+
* Provides efficient vector similarity search with native database support.
|
|
14
|
+
*
|
|
15
|
+
* Requirements:
|
|
16
|
+
* - PostgreSQL database with pgvector extension installed
|
|
17
|
+
* - CREATE EXTENSION vector;
|
|
18
|
+
*
|
|
19
|
+
* @template Metadata - The metadata type
|
|
20
|
+
* @template Vector - The vector type
|
|
21
|
+
*/
|
|
22
|
+
export declare class PostgresVectorStorage<Schema extends DataPortSchemaObject, PrimaryKeyNames extends ReadonlyArray<keyof Schema["properties"]>, Metadata extends Record<string, unknown> = Record<string, unknown>, Vector extends TypedArray = Float32Array, Entity = FromSchema<Schema, TypedArraySchemaOptions>> extends PostgresTabularStorage<Schema, PrimaryKeyNames, Entity> implements IVectorStorage<Metadata, Schema, Entity, PrimaryKeyNames> {
|
|
23
|
+
private vectorDimensions;
|
|
24
|
+
private VectorType;
|
|
25
|
+
private vectorPropertyName;
|
|
26
|
+
private metadataPropertyName;
|
|
27
|
+
/**
|
|
28
|
+
* Creates a new PostgreSQL vector repository
|
|
29
|
+
* @param db - PostgreSQL connection pool
|
|
30
|
+
* @param table - The name of the table to use for storage
|
|
31
|
+
* @param schema - The schema definition for the entity
|
|
32
|
+
* @param primaryKeyNames - Array of property names that form the primary key
|
|
33
|
+
* @param indexes - Array of columns or column arrays to make searchable
|
|
34
|
+
* @param dimensions - The number of dimensions of the vector
|
|
35
|
+
* @param VectorType - The type of vector to use (defaults to Float32Array)
|
|
36
|
+
*/
|
|
37
|
+
constructor(db: Pool, table: string, schema: Schema, primaryKeyNames: PrimaryKeyNames, indexes: readonly (keyof Entity | readonly (keyof Entity)[])[] | undefined, dimensions: number, VectorType?: new (array: number[]) => TypedArray);
|
|
38
|
+
getVectorDimensions(): number;
|
|
39
|
+
similaritySearch(query: TypedArray, options?: VectorSearchOptions<Metadata>): Promise<Array<Entity & {
|
|
40
|
+
score: number;
|
|
41
|
+
}>>;
|
|
42
|
+
hybridSearch(query: TypedArray, options: HybridSearchOptions<Metadata>): Promise<(Entity & {
|
|
43
|
+
score: number;
|
|
44
|
+
})[]>;
|
|
45
|
+
/**
|
|
46
|
+
* Fallback search using in-memory cosine similarity
|
|
47
|
+
*/
|
|
48
|
+
private searchFallback;
|
|
49
|
+
/**
|
|
50
|
+
* Fallback hybrid search
|
|
51
|
+
*/
|
|
52
|
+
private hybridSearchFallback;
|
|
53
|
+
private getPrimaryKeyWhereClause;
|
|
54
|
+
private getPrimaryKeyValues;
|
|
55
|
+
private matchesFilter;
|
|
56
|
+
}
|
|
57
|
+
//# sourceMappingURL=PostgresVectorStorage.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"PostgresVectorStorage.d.ts","sourceRoot":"","sources":["../../src/vector/PostgresVectorStorage.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EACV,oBAAoB,EACpB,UAAU,EACV,UAAU,EACV,uBAAuB,EACxB,MAAM,gBAAgB,CAAC;AAExB,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC/B,OAAO,EAAE,sBAAsB,EAAE,MAAM,mCAAmC,CAAC;AAC3E,OAAO,EAGL,KAAK,mBAAmB,EACxB,KAAK,cAAc,EACnB,KAAK,mBAAmB,EACzB,MAAM,kBAAkB,CAAC;AAE1B;;;;;;;;;;;GAWG;AACH,qBAAa,qBAAqB,CAChC,MAAM,SAAS,oBAAoB,EACnC,eAAe,SAAS,aAAa,CAAC,MAAM,MAAM,CAAC,YAAY,CAAC,CAAC,EACjE,QAAQ,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAClE,MAAM,SAAS,UAAU,GAAG,YAAY,EACxC,MAAM,GAAG,UAAU,CAAC,MAAM,EAAE,uBAAuB,CAAC,CAEpD,SAAQ,sBAAsB,CAAC,MAAM,EAAE,eAAe,EAAE,MAAM,CAC9D,YAAW,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,eAAe,CAAC;IAEpE,OAAO,CAAC,gBAAgB,CAAS;IACjC,OAAO,CAAC,UAAU,CAAsC;IACxD,OAAO,CAAC,kBAAkB,CAAe;IACzC,OAAO,CAAC,oBAAoB,CAA2B;IAEvD;;;;;;;;;OASG;gBAED,EAAE,EAAE,IAAI,EACR,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EACd,eAAe,EAAE,eAAe,EAChC,OAAO,EAAE,SAAS,CAAC,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,MAAM,CAAC,EAAE,CAAC,EAAE,YAAK,EACnE,UAAU,EAAE,MAAM,EAClB,UAAU,GAAE,KAAK,KAAK,EAAE,MAAM,EAAE,KAAK,UAAyB;IAgBhE,mBAAmB,IAAI,MAAM;IAIvB,gBAAgB,CACpB,KAAK,EAAE,UAAU,EACjB,OAAO,GAAE,mBAAmB,CAAC,QAAQ,CAAM,GAC1C,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG;QAAE,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAkEvC,YAAY,CAAC,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,mBAAmB,CAAC,QAAQ,CAAC;eAlEzC,MAAM;;IA+IzC;;OAEG;YACW,cAAc;IA4B5B;;OAEG;YACW,oBAAoB;IA4ClC,OAAO,CAAC,wBAAwB;IAOhC,OAAO,CAAC,mBAAmB;IAI3B,OAAO,CAAC,aAAa;CAQtB"}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @license
|
|
3
|
+
* Copyright 2025 Steven Roussey <sroussey@gmail.com>
|
|
4
|
+
* SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
*/
|
|
6
|
+
import { Sqlite } from "@workglow/sqlite";
|
|
7
|
+
import type { DataPortSchemaObject, FromSchema, TypedArray, TypedArraySchemaOptions } from "@workglow/util";
|
|
8
|
+
import { SqliteTabularStorage } from "../tabular/SqliteTabularStorage";
|
|
9
|
+
import { type HybridSearchOptions, type IVectorStorage, type VectorSearchOptions } from "./IVectorStorage";
|
|
10
|
+
/**
|
|
11
|
+
* SQLite vector repository implementation using tabular storage underneath.
|
|
12
|
+
* Stores vectors as JSON-encoded arrays with metadata.
|
|
13
|
+
*
|
|
14
|
+
* @template Vector - The vector type for the vector
|
|
15
|
+
* @template Metadata - The metadata type for the vector
|
|
16
|
+
* @template Schema - The schema for the vector
|
|
17
|
+
* @template PrimaryKeyNames - The primary key names for the vector
|
|
18
|
+
*/
|
|
19
|
+
export declare class SqliteVectorStorage<Schema extends DataPortSchemaObject, PrimaryKeyNames extends ReadonlyArray<keyof Schema["properties"]>, Vector extends TypedArray = Float32Array, Metadata extends Record<string, unknown> | undefined = Record<string, unknown>, Entity = FromSchema<Schema, TypedArraySchemaOptions>> extends SqliteTabularStorage<Schema, PrimaryKeyNames, Entity> implements IVectorStorage<Metadata, Schema, Entity, PrimaryKeyNames> {
|
|
20
|
+
private vectorDimensions;
|
|
21
|
+
private VectorType;
|
|
22
|
+
private vectorPropertyName;
|
|
23
|
+
private metadataPropertyName;
|
|
24
|
+
/**
|
|
25
|
+
* Creates a new SQLite vector repository
|
|
26
|
+
* @param dbOrPath - Either a Database instance or a path to the SQLite database file
|
|
27
|
+
* @param table - The name of the table to use for storage (defaults to 'vectors')
|
|
28
|
+
* @param dimensions - The number of dimensions of the vector
|
|
29
|
+
* @param VectorType - The type of vector to use (defaults to Float32Array)
|
|
30
|
+
*/
|
|
31
|
+
constructor(dbOrPath: string | Sqlite.Database, table: string | undefined, schema: Schema, primaryKeyNames: PrimaryKeyNames, indexes: readonly (keyof Entity | readonly (keyof Entity)[])[] | undefined, dimensions: number, VectorType?: new (array: number[]) => TypedArray);
|
|
32
|
+
getVectorDimensions(): number;
|
|
33
|
+
/**
|
|
34
|
+
* Deserialize vector from JSON string
|
|
35
|
+
* Defaults to Float32Array for compatibility with typical embedding vectors
|
|
36
|
+
*/
|
|
37
|
+
private deserializeVector;
|
|
38
|
+
similaritySearch(query: TypedArray, options?: VectorSearchOptions<Metadata>): Promise<(Entity & {
|
|
39
|
+
score: number;
|
|
40
|
+
})[]>;
|
|
41
|
+
hybridSearch(query: TypedArray, options: HybridSearchOptions<Metadata>): Promise<(Entity & {
|
|
42
|
+
score: number;
|
|
43
|
+
})[]>;
|
|
44
|
+
}
|
|
45
|
+
//# sourceMappingURL=SqliteVectorStorage.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"SqliteVectorStorage.d.ts","sourceRoot":"","sources":["../../src/vector/SqliteVectorStorage.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,OAAO,KAAK,EACV,oBAAoB,EACpB,UAAU,EACV,UAAU,EACV,uBAAuB,EACxB,MAAM,gBAAgB,CAAC;AAExB,OAAO,EAAE,oBAAoB,EAAE,MAAM,iCAAiC,CAAC;AACvE,OAAO,EAGL,KAAK,mBAAmB,EACxB,KAAK,cAAc,EACnB,KAAK,mBAAmB,EACzB,MAAM,kBAAkB,CAAC;AAc1B;;;;;;;;GAQG;AACH,qBAAa,mBAAmB,CAC9B,MAAM,SAAS,oBAAoB,EACnC,eAAe,SAAS,aAAa,CAAC,MAAM,MAAM,CAAC,YAAY,CAAC,CAAC,EACjE,MAAM,SAAS,UAAU,GAAG,YAAY,EACxC,QAAQ,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC9E,MAAM,GAAG,UAAU,CAAC,MAAM,EAAE,uBAAuB,CAAC,CAEpD,SAAQ,oBAAoB,CAAC,MAAM,EAAE,eAAe,EAAE,MAAM,CAC5D,YAAW,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,eAAe,CAAC;IAEpE,OAAO,CAAC,gBAAgB,CAAS;IACjC,OAAO,CAAC,UAAU,CAAsC;IACxD,OAAO,CAAC,kBAAkB,CAAe;IACzC,OAAO,CAAC,oBAAoB,CAA2B;IAEvD;;;;;;OAMG;gBAED,QAAQ,EAAE,MAAM,GAAG,MAAM,CAAC,QAAQ,EAClC,KAAK,EAAE,MAAM,YAAY,EACzB,MAAM,EAAE,MAAM,EACd,eAAe,EAAE,eAAe,EAChC,OAAO,EAAE,SAAS,CAAC,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,MAAM,CAAC,EAAE,CAAC,EAAE,YAAK,EACnE,UAAU,EAAE,MAAM,EAClB,UAAU,GAAE,KAAK,KAAK,EAAE,MAAM,EAAE,KAAK,UAAyB;IAgBhE,mBAAmB,IAAI,MAAM;IAI7B;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAMnB,gBAAgB,CAAC,KAAK,EAAE,UAAU,EAAE,OAAO,GAAE,mBAAmB,CAAC,QAAQ,CAAM;eAE5C,MAAM;;IAsCzC,YAAY,CAAC,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,mBAAmB,CAAC,QAAQ,CAAC;eAtCnC,MAAM;;CAoGhD"}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@workglow/storage",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "0.0.
|
|
4
|
+
"version": "0.0.86",
|
|
5
5
|
"description": "Storage abstraction layer for Workglow, supporting IndexedDB, PostgreSQL, and Supabase with unified interfaces.",
|
|
6
6
|
"scripts": {
|
|
7
7
|
"watch": "concurrently -c 'auto' 'bun:watch-*'",
|
|
@@ -20,8 +20,8 @@
|
|
|
20
20
|
"prepare": "node -e \"const pkg=require('./package.json');pkg.exports['.'].bun='./dist/bun.js';pkg.exports['.'].types='./dist/types.d.ts';require('fs').writeFileSync('package.json',JSON.stringify(pkg,null,2))\""
|
|
21
21
|
},
|
|
22
22
|
"peerDependencies": {
|
|
23
|
-
"@workglow/sqlite": "0.0.
|
|
24
|
-
"@workglow/util": "0.0.
|
|
23
|
+
"@workglow/sqlite": "0.0.86",
|
|
24
|
+
"@workglow/util": "0.0.86",
|
|
25
25
|
"pg": "^8.16.3",
|
|
26
26
|
"@supabase/supabase-js": "^2.89.0"
|
|
27
27
|
},
|
|
@@ -34,8 +34,8 @@
|
|
|
34
34
|
}
|
|
35
35
|
},
|
|
36
36
|
"devDependencies": {
|
|
37
|
-
"@workglow/sqlite": "0.0.
|
|
38
|
-
"@workglow/util": "0.0.
|
|
37
|
+
"@workglow/sqlite": "0.0.86",
|
|
38
|
+
"@workglow/util": "0.0.86",
|
|
39
39
|
"pg": "^8.16.3",
|
|
40
40
|
"@types/pg": "^8.15.5",
|
|
41
41
|
"@supabase/supabase-js": "^2.89.0",
|
package/src/kv/README.md
CHANGED
|
@@ -23,7 +23,7 @@ A flexible key-value storage solution with multiple backend implementations. Pro
|
|
|
23
23
|
- 💾 `IndexedDbKvRepository` - Browser IndexedDB storage
|
|
24
24
|
- 🐘 `PostgresKvRepository` - PostgreSQL database storage
|
|
25
25
|
- 📁 `SqliteKvRepository` - SQLite database storage
|
|
26
|
-
- 🧠 `
|
|
26
|
+
- 🧠 `InMemoryKvStorage` - Volatile memory storage
|
|
27
27
|
- Type-safe key/value definitions
|
|
28
28
|
- JSON value serialization support
|
|
29
29
|
- Event emitter for storage operations (put/get/delete)
|
|
@@ -104,9 +104,9 @@ await sqliteRepo.put("temp:789", "cached_value");
|
|
|
104
104
|
### In-Memory Storage
|
|
105
105
|
|
|
106
106
|
```typescript
|
|
107
|
-
import {
|
|
107
|
+
import { InMemoryKvStorage } from "@workglow/storage/kv";
|
|
108
108
|
|
|
109
|
-
const memRepo = new
|
|
109
|
+
const memRepo = new InMemoryKvStorage(
|
|
110
110
|
"string", // Key type
|
|
111
111
|
"json" // Value type
|
|
112
112
|
);
|
package/src/tabular/README.md
CHANGED
|
@@ -9,11 +9,11 @@ A collection of storage implementations for tabular data with multiple backend s
|
|
|
9
9
|
- [Using TypeBox](#using-typebox)
|
|
10
10
|
- [Using Zod 4](#using-zod-4)
|
|
11
11
|
- [Implementations](#implementations)
|
|
12
|
-
- [
|
|
13
|
-
- [
|
|
14
|
-
- [
|
|
15
|
-
- [
|
|
16
|
-
- [
|
|
12
|
+
- [InMemoryTabularStorage](#inmemorytabularrepository)
|
|
13
|
+
- [SqliteTabularStorage](#sqlitetabularrepository)
|
|
14
|
+
- [PostgresTabularStorage](#postgrestabularrepository)
|
|
15
|
+
- [IndexedDbTabularStorage](#indexeddbtabularrepository)
|
|
16
|
+
- [FsFolderTabularStorage](#fsfoldertabularrepository)
|
|
17
17
|
- [Events](#events)
|
|
18
18
|
- [Testing](#testing)
|
|
19
19
|
- [License](#license)
|
|
@@ -43,7 +43,7 @@ npm install @workglow/storage
|
|
|
43
43
|
## Basic Usage
|
|
44
44
|
|
|
45
45
|
```typescript
|
|
46
|
-
import {
|
|
46
|
+
import { InMemoryTabularStorage } from "@workglow/storage/tabular";
|
|
47
47
|
|
|
48
48
|
// Define schema and primary keys
|
|
49
49
|
const schema = {
|
|
@@ -55,8 +55,8 @@ const schema = {
|
|
|
55
55
|
|
|
56
56
|
const primaryKeys = ["id"] as const;
|
|
57
57
|
// Create repository instance (when using const schemas, the next three generics
|
|
58
|
-
// on
|
|
59
|
-
const repo = new
|
|
58
|
+
// on InMemoryTabularStorage are automatically created for you)
|
|
59
|
+
const repo = new InMemoryTabularStorage<typeof schema, typeof primaryKeys>(schema, primaryKeys);
|
|
60
60
|
|
|
61
61
|
// Basic operations
|
|
62
62
|
await repo.put({ id: "1", name: "Alice", age: 30, active: true });
|
|
@@ -75,9 +75,9 @@ You can define schemas using plain JSON Schema objects, or use schema libraries
|
|
|
75
75
|
TypeBox schemas are JSON Schema compatible and can be used directly:
|
|
76
76
|
|
|
77
77
|
```typescript
|
|
78
|
-
import {
|
|
78
|
+
import { InMemoryTabularStorage } from "@workglow/storage/tabular";
|
|
79
79
|
import { Type, Static } from "@sinclair/typebox";
|
|
80
|
-
import { DataPortSchemaObject, FromSchema
|
|
80
|
+
import { DataPortSchemaObject, FromSchema } from "@workglow/util";
|
|
81
81
|
|
|
82
82
|
// Define schema using TypeBox
|
|
83
83
|
const userSchema = Type.Object({
|
|
@@ -98,7 +98,7 @@ type UserEntity = FromSchema<typeof userSchema>;
|
|
|
98
98
|
|
|
99
99
|
// IMPORTANT: You must explicitly provide generic type parameters for t
|
|
100
100
|
// TypeScript cannot infer them from TypeBox schemas
|
|
101
|
-
const repo = new
|
|
101
|
+
const repo = new InMemoryTabularStorage<typeof userSchema, typeof primaryKeys, UserEntity>(
|
|
102
102
|
userSchema,
|
|
103
103
|
primaryKeys,
|
|
104
104
|
["email", "active"] as const // Indexes
|
|
@@ -119,7 +119,7 @@ await repo.put({
|
|
|
119
119
|
Zod 4 has built-in JSON Schema support using the `.toJSONSchema()` method:
|
|
120
120
|
|
|
121
121
|
```typescript
|
|
122
|
-
import {
|
|
122
|
+
import { InMemoryTabularStorage } from "@workglow/storage/tabular";
|
|
123
123
|
import { z } from "zod";
|
|
124
124
|
import { DataPortSchemaObject } from "@workglow/util";
|
|
125
125
|
|
|
@@ -141,7 +141,7 @@ type UserEntity = z.infer<typeof userSchemaZod>;
|
|
|
141
141
|
|
|
142
142
|
// IMPORTANT: You must explicitly provide generic type parameters
|
|
143
143
|
// TypeScript cannot infer them from Zod schemas (even after conversion)
|
|
144
|
-
const repo = new
|
|
144
|
+
const repo = new InMemoryTabularStorage<typeof userSchema, typeof primaryKeys, UserEntity>(
|
|
145
145
|
userSchema,
|
|
146
146
|
primaryKeys,
|
|
147
147
|
["email", "active"] as const // Indexes
|
|
@@ -157,16 +157,179 @@ await repo.put({
|
|
|
157
157
|
});
|
|
158
158
|
```
|
|
159
159
|
|
|
160
|
+
## Auto-Generated Primary Keys
|
|
161
|
+
|
|
162
|
+
TabularStorage supports automatic generation of primary keys, allowing the storage backend to generate IDs when entities are inserted without them. This is useful for:
|
|
163
|
+
|
|
164
|
+
- Security: Preventing clients from choosing arbitrary IDs
|
|
165
|
+
- Simplicity: No need to generate IDs client-side
|
|
166
|
+
- Database features: Leveraging native auto-increment and UUID generation
|
|
167
|
+
|
|
168
|
+
### Schema Configuration
|
|
169
|
+
|
|
170
|
+
Mark a primary key column as auto-generated using the `x-auto-generated: true` annotation:
|
|
171
|
+
|
|
172
|
+
```typescript
|
|
173
|
+
const UserSchema = {
|
|
174
|
+
type: "object",
|
|
175
|
+
properties: {
|
|
176
|
+
id: { type: "integer", "x-auto-generated": true }, // Auto-increment
|
|
177
|
+
name: { type: "string" },
|
|
178
|
+
email: { type: "string" },
|
|
179
|
+
},
|
|
180
|
+
required: ["id", "name", "email"],
|
|
181
|
+
additionalProperties: false,
|
|
182
|
+
} as const satisfies DataPortSchemaObject;
|
|
183
|
+
|
|
184
|
+
const DocumentSchema = {
|
|
185
|
+
type: "object",
|
|
186
|
+
properties: {
|
|
187
|
+
id: { type: "string", "x-auto-generated": true }, // UUID
|
|
188
|
+
title: { type: "string" },
|
|
189
|
+
content: { type: "string" },
|
|
190
|
+
},
|
|
191
|
+
required: ["id", "title", "content"],
|
|
192
|
+
additionalProperties: false,
|
|
193
|
+
} as const satisfies DataPortSchemaObject;
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
**Generation Strategy (inferred from column type):**
|
|
197
|
+
|
|
198
|
+
- `type: "integer"` → Auto-increment (SERIAL, INTEGER PRIMARY KEY, counter)
|
|
199
|
+
- `type: "string"` → UUID via `uuid4()` from `@workglow/util`
|
|
200
|
+
|
|
201
|
+
**Constraints:**
|
|
202
|
+
|
|
203
|
+
- Only the **first column** in a compound primary key can be auto-generated
|
|
204
|
+
- Only **one column** can be auto-generated per table
|
|
205
|
+
|
|
206
|
+
### Basic Usage
|
|
207
|
+
|
|
208
|
+
```typescript
|
|
209
|
+
import { InMemoryTabularStorage } from "@workglow/storage/tabular";
|
|
210
|
+
|
|
211
|
+
const userStorage = new InMemoryTabularStorage(UserSchema, ["id"] as const);
|
|
212
|
+
await userStorage.setupDatabase();
|
|
213
|
+
|
|
214
|
+
// Insert without providing ID - it will be auto-generated
|
|
215
|
+
const user = await userStorage.put({
|
|
216
|
+
name: "Alice",
|
|
217
|
+
email: "alice@example.com",
|
|
218
|
+
});
|
|
219
|
+
console.log(user.id); // 1 (auto-generated)
|
|
220
|
+
|
|
221
|
+
// TypeScript enforces: id is optional on insert, required on returned entity
|
|
222
|
+
```
|
|
223
|
+
|
|
224
|
+
### Client-Provided Keys Configuration
|
|
225
|
+
|
|
226
|
+
Control whether clients can provide values for auto-generated keys:
|
|
227
|
+
|
|
228
|
+
```typescript
|
|
229
|
+
const storage = new PostgresTabularStorage(
|
|
230
|
+
db,
|
|
231
|
+
"users",
|
|
232
|
+
UserSchema,
|
|
233
|
+
["id"] as const,
|
|
234
|
+
[], // indexes
|
|
235
|
+
{ clientProvidedKeys: "if-missing" } // configuration
|
|
236
|
+
);
|
|
237
|
+
```
|
|
238
|
+
|
|
239
|
+
**Options:**
|
|
240
|
+
|
|
241
|
+
| Setting | Behavior | Use Case |
|
|
242
|
+
| ------------------------ | ------------------------------------------------ | ----------------------------------------------------------------- |
|
|
243
|
+
| `"if-missing"` (default) | Use client value if provided, generate otherwise | Flexible - supports both auto-generation and client-specified IDs |
|
|
244
|
+
| `"never"` | Always generate, ignore client values | Maximum security - never trust client IDs |
|
|
245
|
+
| `"always"` | Require client to provide value | Testing/migration - enforce client-side ID generation |
|
|
246
|
+
|
|
247
|
+
**Examples:**
|
|
248
|
+
|
|
249
|
+
```typescript
|
|
250
|
+
// Default: "if-missing" - flexible
|
|
251
|
+
const flexibleStorage = new InMemoryTabularStorage(UserSchema, ["id"] as const);
|
|
252
|
+
|
|
253
|
+
// Without ID - auto-generated
|
|
254
|
+
await flexibleStorage.put({ name: "Bob", email: "bob@example.com" });
|
|
255
|
+
|
|
256
|
+
// With ID - uses client value
|
|
257
|
+
await flexibleStorage.put({ id: 999, name: "Charlie", email: "charlie@example.com" });
|
|
258
|
+
|
|
259
|
+
// Secure mode: "never" - always generate
|
|
260
|
+
const secureStorage = new PostgresTabularStorage(db, "users", UserSchema, ["id"] as const, [], {
|
|
261
|
+
clientProvidedKeys: "never",
|
|
262
|
+
});
|
|
263
|
+
|
|
264
|
+
// Even if client provides id, it will be ignored and regenerated
|
|
265
|
+
const result = await secureStorage.put({
|
|
266
|
+
id: 999, // Ignored!
|
|
267
|
+
name: "Diana",
|
|
268
|
+
email: "diana@example.com",
|
|
269
|
+
});
|
|
270
|
+
// result.id will be database-generated, NOT 999
|
|
271
|
+
|
|
272
|
+
// Testing mode: "always" - require client ID
|
|
273
|
+
const testStorage = new InMemoryTabularStorage(UserSchema, ["id"] as const, [], {
|
|
274
|
+
clientProvidedKeys: "always",
|
|
275
|
+
});
|
|
276
|
+
|
|
277
|
+
// Must provide ID or throws error
|
|
278
|
+
await testStorage.put({
|
|
279
|
+
id: 1,
|
|
280
|
+
name: "Eve",
|
|
281
|
+
email: "eve@example.com",
|
|
282
|
+
}); // OK
|
|
283
|
+
|
|
284
|
+
await testStorage.put({
|
|
285
|
+
name: "Frank",
|
|
286
|
+
email: "frank@example.com",
|
|
287
|
+
}); // Throws Error!
|
|
288
|
+
```
|
|
289
|
+
|
|
290
|
+
### Backend-Specific Behavior
|
|
291
|
+
|
|
292
|
+
Each storage backend implements auto-generation differently:
|
|
293
|
+
|
|
294
|
+
| Backend | Integer (autoincrement) | String (UUID) |
|
|
295
|
+
| --------------------------- | ----------------------------------- | --------------------------------- |
|
|
296
|
+
| **InMemoryTabularStorage** | Internal counter (1, 2, 3...) | `uuid4()` from `@workglow/util` |
|
|
297
|
+
| **SqliteTabularStorage** | `INTEGER PRIMARY KEY AUTOINCREMENT` | `uuid4()` client-side |
|
|
298
|
+
| **PostgresTabularStorage** | `SERIAL`/`BIGSERIAL` | `gen_random_uuid()` database-side |
|
|
299
|
+
| **SupabaseTabularStorage** | `SERIAL` | `gen_random_uuid()` database-side |
|
|
300
|
+
| **IndexedDbTabularStorage** | `autoIncrement: true` | `uuid4()` client-side |
|
|
301
|
+
| **FsFolderTabularStorage** | Internal counter | `uuid4()` from `@workglow/util` |
|
|
302
|
+
|
|
303
|
+
### Constraints
|
|
304
|
+
|
|
305
|
+
1. **Only first column**: Only the first primary key column can be auto-generated
|
|
306
|
+
2. **Single auto-gen key**: Only one column per table can be auto-generated
|
|
307
|
+
3. **Type inference**: Generation strategy is inferred from column type (integer → autoincrement, string → UUID)
|
|
308
|
+
|
|
309
|
+
### Type Safety
|
|
310
|
+
|
|
311
|
+
TypeScript enforces correct usage through the type system:
|
|
312
|
+
|
|
313
|
+
```typescript
|
|
314
|
+
// Auto-generated key is OPTIONAL on insert
|
|
315
|
+
const entity = { name: "Alice", email: "alice@example.com" };
|
|
316
|
+
await storage.put(entity); // ✅ OK - id can be omitted
|
|
317
|
+
|
|
318
|
+
// Returned entity has ALL fields REQUIRED
|
|
319
|
+
const result = await storage.put(entity);
|
|
320
|
+
const id: number = result.id; // ✅ OK - id is guaranteed to exist
|
|
321
|
+
```
|
|
322
|
+
|
|
160
323
|
## Implementations
|
|
161
324
|
|
|
162
|
-
###
|
|
325
|
+
### InMemoryTabularStorage
|
|
163
326
|
|
|
164
327
|
- Ideal for testing/development
|
|
165
328
|
- No persistence
|
|
166
329
|
- Fast search capabilities
|
|
167
330
|
|
|
168
331
|
```typescript
|
|
169
|
-
const repo = new
|
|
332
|
+
const repo = new InMemoryTabularStorage<
|
|
170
333
|
typeof schema,
|
|
171
334
|
typeof primaryKeys,
|
|
172
335
|
Entity, // required if using TypeBox, Zod, etc, otherwise automatically created
|
|
@@ -175,13 +338,13 @@ const repo = new InMemoryTabularRepository<
|
|
|
175
338
|
>(schema, primaryKeys, ["name", "active"]);
|
|
176
339
|
```
|
|
177
340
|
|
|
178
|
-
###
|
|
341
|
+
### SqliteTabularStorage
|
|
179
342
|
|
|
180
343
|
- Embedded SQLite database
|
|
181
344
|
- File-based or in-memory
|
|
182
345
|
|
|
183
346
|
```typescript
|
|
184
|
-
const repo = new
|
|
347
|
+
const repo = new SqliteTabularStorage<
|
|
185
348
|
typeof schema,
|
|
186
349
|
typeof primaryKeys,
|
|
187
350
|
Entity, // required if using TypeBox, Zod, etc, otherwise automatically created
|
|
@@ -196,7 +359,7 @@ const repo = new SqliteTabularRepository<
|
|
|
196
359
|
);
|
|
197
360
|
```
|
|
198
361
|
|
|
199
|
-
###
|
|
362
|
+
### PostgresTabularStorage
|
|
200
363
|
|
|
201
364
|
- PostgreSQL backend
|
|
202
365
|
- Connection pooling support
|
|
@@ -207,7 +370,7 @@ import { Pool } from "pg";
|
|
|
207
370
|
const pool = new Pool({
|
|
208
371
|
/* config */
|
|
209
372
|
});
|
|
210
|
-
const repo = new
|
|
373
|
+
const repo = new PostgresTabularStorage<
|
|
211
374
|
typeof schema,
|
|
212
375
|
typeof primaryKeys,
|
|
213
376
|
Entity, // required if using TypeBox, Zod, etc, otherwise automatically created
|
|
@@ -222,13 +385,13 @@ const repo = new PostgresTabularRepository<
|
|
|
222
385
|
);
|
|
223
386
|
```
|
|
224
387
|
|
|
225
|
-
###
|
|
388
|
+
### IndexedDbTabularStorage
|
|
226
389
|
|
|
227
390
|
- Browser-based storage
|
|
228
391
|
- Automatic schema migration
|
|
229
392
|
|
|
230
393
|
```typescript
|
|
231
|
-
const repo = new
|
|
394
|
+
const repo = new IndexedDbTabularStorage<
|
|
232
395
|
typeof schema,
|
|
233
396
|
typeof primaryKeys,
|
|
234
397
|
Entity, // required if using TypeBox, Zod, etc, otherwise automatically created
|
|
@@ -242,13 +405,13 @@ const repo = new IndexedDbTabularRepository<
|
|
|
242
405
|
);
|
|
243
406
|
```
|
|
244
407
|
|
|
245
|
-
###
|
|
408
|
+
### FsFolderTabularStorage
|
|
246
409
|
|
|
247
410
|
- Filesystem storage (one JSON file per record)
|
|
248
411
|
- Simple persistence format
|
|
249
412
|
|
|
250
413
|
```typescript
|
|
251
|
-
const repo = new
|
|
414
|
+
const repo = new FsFolderTabularStorage<
|
|
252
415
|
typeof schema,
|
|
253
416
|
typeof primaryKeys,
|
|
254
417
|
Entity, // required if using TypeBox, Zod, etc, otherwise automatically created
|