@workglow/storage 0.0.52
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +1015 -0
- package/dist/browser.js +2635 -0
- package/dist/browser.js.map +27 -0
- package/dist/bun.js +3880 -0
- package/dist/bun.js.map +35 -0
- package/dist/common-server.d.ts +23 -0
- package/dist/common-server.d.ts.map +1 -0
- package/dist/common.d.ts +16 -0
- package/dist/common.d.ts.map +1 -0
- package/dist/kv/FsFolderJsonKvRepository.d.ts +27 -0
- package/dist/kv/FsFolderJsonKvRepository.d.ts.map +1 -0
- package/dist/kv/FsFolderKvRepository.d.ts +74 -0
- package/dist/kv/FsFolderKvRepository.d.ts.map +1 -0
- package/dist/kv/IKvRepository.d.ts +65 -0
- package/dist/kv/IKvRepository.d.ts.map +1 -0
- package/dist/kv/InMemoryKvRepository.d.ts +26 -0
- package/dist/kv/InMemoryKvRepository.d.ts.map +1 -0
- package/dist/kv/IndexedDbKvRepository.d.ts +27 -0
- package/dist/kv/IndexedDbKvRepository.d.ts.map +1 -0
- package/dist/kv/KvRepository.d.ts +109 -0
- package/dist/kv/KvRepository.d.ts.map +1 -0
- package/dist/kv/KvViaTabularRepository.d.ts +64 -0
- package/dist/kv/KvViaTabularRepository.d.ts.map +1 -0
- package/dist/kv/PostgresKvRepository.d.ts +28 -0
- package/dist/kv/PostgresKvRepository.d.ts.map +1 -0
- package/dist/kv/SqliteKvRepository.d.ts +28 -0
- package/dist/kv/SqliteKvRepository.d.ts.map +1 -0
- package/dist/kv/SupabaseKvRepository.d.ts +34 -0
- package/dist/kv/SupabaseKvRepository.d.ts.map +1 -0
- package/dist/node.js +3879 -0
- package/dist/node.js.map +35 -0
- package/dist/queue/IQueueStorage.d.ts +125 -0
- package/dist/queue/IQueueStorage.d.ts.map +1 -0
- package/dist/queue/InMemoryQueueStorage.d.ts +109 -0
- package/dist/queue/InMemoryQueueStorage.d.ts.map +1 -0
- package/dist/queue/IndexedDbQueueStorage.d.ts +89 -0
- package/dist/queue/IndexedDbQueueStorage.d.ts.map +1 -0
- package/dist/queue/PostgresQueueStorage.d.ts +92 -0
- package/dist/queue/PostgresQueueStorage.d.ts.map +1 -0
- package/dist/queue/SqliteQueueStorage.d.ts +116 -0
- package/dist/queue/SqliteQueueStorage.d.ts.map +1 -0
- package/dist/queue/SupabaseQueueStorage.d.ts +93 -0
- package/dist/queue/SupabaseQueueStorage.d.ts.map +1 -0
- package/dist/tabular/BaseSqlTabularRepository.d.ts +94 -0
- package/dist/tabular/BaseSqlTabularRepository.d.ts.map +1 -0
- package/dist/tabular/CachedTabularRepository.d.ts +110 -0
- package/dist/tabular/CachedTabularRepository.d.ts.map +1 -0
- package/dist/tabular/FsFolderTabularRepository.d.ts +92 -0
- package/dist/tabular/FsFolderTabularRepository.d.ts.map +1 -0
- package/dist/tabular/ITabularRepository.d.ts +52 -0
- package/dist/tabular/ITabularRepository.d.ts.map +1 -0
- package/dist/tabular/InMemoryTabularRepository.d.ts +93 -0
- package/dist/tabular/InMemoryTabularRepository.d.ts.map +1 -0
- package/dist/tabular/IndexedDbTabularRepository.d.ts +100 -0
- package/dist/tabular/IndexedDbTabularRepository.d.ts.map +1 -0
- package/dist/tabular/PostgresTabularRepository.d.ts +133 -0
- package/dist/tabular/PostgresTabularRepository.d.ts.map +1 -0
- package/dist/tabular/SharedInMemoryTabularRepository.d.ts +126 -0
- package/dist/tabular/SharedInMemoryTabularRepository.d.ts.map +1 -0
- package/dist/tabular/SqliteTabularRepository.d.ts +110 -0
- package/dist/tabular/SqliteTabularRepository.d.ts.map +1 -0
- package/dist/tabular/SupabaseTabularRepository.d.ts +132 -0
- package/dist/tabular/SupabaseTabularRepository.d.ts.map +1 -0
- package/dist/tabular/TabularRepository.d.ts +123 -0
- package/dist/tabular/TabularRepository.d.ts.map +1 -0
- package/dist/types.d.ts +7 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/util/IndexedDbTable.d.ts +40 -0
- package/dist/util/IndexedDbTable.d.ts.map +1 -0
- package/package.json +60 -0
- package/src/kv/README.md +159 -0
- package/src/queue/README.md +41 -0
- package/src/tabular/README.md +298 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"TabularRepository.d.ts","sourceRoot":"","sources":["../../src/tabular/TabularRepository.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAEH,oBAAoB,EACpB,YAAY,EACZ,UAAU,EAEb,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACH,kBAAkB,EAClB,oBAAoB,EACpB,qBAAqB,EACrB,gBAAgB,EAChB,sBAAsB,EACtB,eAAe,EAClB,MAAM,sBAAsB,CAAC;AAE9B,eAAO,MAAM,kBAAkB,oFAE9B,CAAC;AAEF;;;;;;;;GAQG;AACH,8BAAsB,iBAAiB,CACrC,MAAM,SAAS,oBAAoB,EACnC,eAAe,SAAS,aAAa,CAAC,MAAM,MAAM,CAAC,YAAY,CAAC,CAAC,EAEjE,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,EAC3B,UAAU,GAAG,IAAI,CAAC,MAAM,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,MAAM,MAAM,CAAC,EACjE,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,MAAM,MAAM,CAAC,CAC5D,YAAW,kBAAkB,CAAC,MAAM,EAAE,eAAe,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,CAAC;IAiB/E,SAAS,CAAC,MAAM,EAAE,MAAM;IACxB,SAAS,CAAC,eAAe,EAAE,eAAe;IAhB5C,0CAA0C;IAC1C,SAAS,CAAC,MAAM,0DAAiE;IAEjF,SAAS,CAAC,OAAO,EAAE,KAAK,CAAC,MAAM,MAAM,CAAC,EAAE,CAAC;IACzC,SAAS,CAAC,gBAAgB,EAAE,oBAAoB,CAAC;IACjD,SAAS,CAAC,WAAW,EAAE,oBAAoB,CAAC;IAE5C;;;;;;OAMG;gBAES,MAAM,EAAE,MAAM,EACd,eAAe,EAAE,eAAe,EAC1C,OAAO,GAAE,KAAK,CAAC,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,MAAM,CAAC,CAAM;IA6EzD,SAAS,CAAC,kBAAkB,CAC1B,UAAU,EAAE,KAAK,CAAC,MAAM,MAAM,CAAC,EAC/B,aAAa,EAAE,KAAK,CAAC,MAAM,MAAM,CAAC,EAAE,GACnC,KAAK,CAAC,MAAM,MAAM,CAAC,EAAE;IAkCxB;;;;OAIG;IACH,EAAE,CAAC,KAAK,SAAS,gBAAgB,EAC/B,IAAI,EAAE,KAAK,EACX,EAAE,EAAE,oBAAoB,CAAC,KAAK,EAAE,UAAU,EAAE,MAAM,CAAC;IAKrD;;;;OAIG;IACH,GAAG,CAAC,KAAK,SAAS,gBAAgB,EAChC,IAAI,EAAE,KAAK,EACX,EAAE,EAAE,oBAAoB,CAAC,KAAK,EAAE,UAAU,EAAE,MAAM,CAAC;IAKrD;;;;OAIG;IACH,IAAI,CAAC,KAAK,SAAS,gBAAgB,EACjC,IAAI,EAAE,KAAK,EACX,EAAE,EAAE,oBAAoB,CAAC,KAAK,EAAE,UAAU,EAAE,MAAM,CAAC;IAKrD;;;;OAIG;IACH,IAAI,CAAC,KAAK,SAAS,gBAAgB,EACjC,IAAI,EAAE,KAAK,EACX,GAAG,IAAI,EAAE,sBAAsB,CAAC,KAAK,EAAE,UAAU,EAAE,MAAM,CAAC;IAK5D;;;;OAIG;IACH,MAAM,CAAC,KAAK,SAAS,gBAAgB,EACnC,IAAI,EAAE,KAAK,GACV,OAAO,CAAC,sBAAsB,CAAC,KAAK,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;IAI7D;;OAEG;IACH,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAC5C,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IACrD,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IAC1D,QAAQ,CAAC,MAAM,CAAC,GAAG,EAAE,UAAU,GAAG,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IACxD,QAAQ,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,EAAE,GAAG,SAAS,CAAC;IAChD,QAAQ,CAAC,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IACnC,QAAQ,CAAC,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC;IAChC,QAAQ,CAAC,YAAY,CACnB,MAAM,EAAE,MAAM,MAAM,EACpB,KAAK,EAAE,MAAM,CAAC,MAAM,MAAM,CAAC,EAC3B,QAAQ,EAAE,GAAG,GAAG,GAAG,GAAG,IAAI,GAAG,GAAG,GAAG,IAAI,GACtC,OAAO,CAAC,IAAI,CAAC;IAEhB;;;;;;OAMG;aACa,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,GAAG,SAAS,CAAC;IAE3E,SAAS,CAAC,iBAAiB,IAAI,KAAK,CAAC,MAAM,UAAU,CAAC;IAQtD,SAAS,CAAC,YAAY,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;IAQ5C;;;;;OAKG;IACH,SAAS,CAAC,4BAA4B,CAAC,GAAG,EAAE,MAAM,GAAG;QAAE,KAAK,EAAE,KAAK,CAAC;QAAC,GAAG,EAAE,UAAU,CAAA;KAAE;IAuBtF;;;;;OAKG;cACa,gBAAgB,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC;IAIlE;;;;;OAKG;IACH,SAAS,CAAC,2BAA2B,CAAC,GAAG,EAAE,UAAU,GAAG,eAAe,EAAE;IAazE;;;;OAIG;IACI,qBAAqB,CAC1B,kBAAkB,EAAE,KAAK,CAAC,MAAM,MAAM,CAAC,GACtC,KAAK,CAAC,MAAM,MAAM,CAAC,GAAG,SAAS;IAqClC;;OAEG;IACH,OAAO,IAAI,IAAI;IAIT,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5C,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI;CAGzB"}
|
package/dist/types.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,iBAAiB,CAAC"}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @license
|
|
3
|
+
* Copyright 2025 Steven Roussey <sroussey@gmail.com>
|
|
4
|
+
* SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
*/
|
|
6
|
+
export interface ExpectedIndexDefinition {
|
|
7
|
+
name: string;
|
|
8
|
+
keyPath: string | string[];
|
|
9
|
+
options?: IDBIndexParameters;
|
|
10
|
+
}
|
|
11
|
+
export interface MigrationContext {
|
|
12
|
+
db: IDBDatabase;
|
|
13
|
+
transaction: IDBTransaction;
|
|
14
|
+
oldVersion: number;
|
|
15
|
+
newVersion: number;
|
|
16
|
+
tableName: string;
|
|
17
|
+
}
|
|
18
|
+
export interface DataTransformer {
|
|
19
|
+
(oldData: any): any | Promise<any>;
|
|
20
|
+
}
|
|
21
|
+
export interface MigrationOptions {
|
|
22
|
+
/** Custom data transformer to apply during migration */
|
|
23
|
+
dataTransformer?: DataTransformer;
|
|
24
|
+
/** Whether to allow destructive operations (delete and recreate). Default: false */
|
|
25
|
+
allowDestructiveMigration?: boolean;
|
|
26
|
+
/** Callback for migration progress/logging */
|
|
27
|
+
onMigrationProgress?: (message: string, progress?: number) => void;
|
|
28
|
+
/** Callback for migration errors (non-fatal warnings) */
|
|
29
|
+
onMigrationWarning?: (message: string, error?: Error) => void;
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Ensures that an IndexedDB table exists with the specified schema.
|
|
33
|
+
* Performs migrations as needed without data loss when possible.
|
|
34
|
+
*/
|
|
35
|
+
export declare function ensureIndexedDbTable(tableName: string, primaryKey: string | string[], expectedIndexes?: ExpectedIndexDefinition[], options?: MigrationOptions): Promise<IDBDatabase>;
|
|
36
|
+
/**
|
|
37
|
+
* Utility function to delete a database (for testing or cleanup)
|
|
38
|
+
*/
|
|
39
|
+
export declare function dropIndexedDbTable(tableName: string): Promise<void>;
|
|
40
|
+
//# sourceMappingURL=IndexedDbTable.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"IndexedDbTable.d.ts","sourceRoot":"","sources":["../../src/util/IndexedDbTable.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAMH,MAAM,WAAW,uBAAuB;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC3B,OAAO,CAAC,EAAE,kBAAkB,CAAC;CAC9B;AAED,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,WAAW,CAAC;IAChB,WAAW,EAAE,cAAc,CAAC;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,eAAe;IAC9B,CAAC,OAAO,EAAE,GAAG,GAAG,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,gBAAgB;IAC/B,wDAAwD;IACxD,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,oFAAoF;IACpF,yBAAyB,CAAC,EAAE,OAAO,CAAC;IACpC,8CAA8C;IAC9C,mBAAmB,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,KAAK,IAAI,CAAC;IACnE,yDAAyD;IACzD,kBAAkB,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,KAAK,KAAK,IAAI,CAAC;CAC/D;AAycD;;;GAGG;AACH,wBAAsB,oBAAoB,CACxC,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,EAC7B,eAAe,GAAE,uBAAuB,EAAO,EAC/C,OAAO,GAAE,gBAAqB,GAC7B,OAAO,CAAC,WAAW,CAAC,CA6JtB;AAED;;GAEG;AACH,wBAAsB,kBAAkB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAEzE"}
|
package/package.json
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@workglow/storage",
|
|
3
|
+
"type": "module",
|
|
4
|
+
"version": "0.0.52",
|
|
5
|
+
"description": "Storage abstraction layer for Workglow, supporting IndexedDB, PostgreSQL, and Supabase with unified interfaces.",
|
|
6
|
+
"scripts": {
|
|
7
|
+
"watch": "concurrently -c 'auto' 'bun:watch-*'",
|
|
8
|
+
"watch-browser": "bun build --watch --no-clear-screen --target=browser --sourcemap=external --packages=external --outdir ./dist ./src/browser.ts",
|
|
9
|
+
"watch-node": "bun build --watch --no-clear-screen --target=node --sourcemap=external --packages=external --outdir ./dist ./src/node.ts",
|
|
10
|
+
"watch-bun": "bun build --watch --no-clear-screen --target=bun --sourcemap=external --packages=external --outdir ./dist ./src/bun.ts",
|
|
11
|
+
"watch-types": "tsc --watch --preserveWatchOutput",
|
|
12
|
+
"build-package": "bun run build-clean && concurrently -c 'auto' -n 'browser,node,bun,types' 'bun run build-browser' 'bun run build-node' 'bun run build-bun' 'bun run build-types'",
|
|
13
|
+
"build-clean": "rm -fr dist/*",
|
|
14
|
+
"build-browser": "bun build --target=browser --sourcemap=external --packages=external --outdir ./dist ./src/browser.ts",
|
|
15
|
+
"build-node": "bun build --target=node --sourcemap=external --packages=external --outdir ./dist ./src/node.ts",
|
|
16
|
+
"build-bun": "bun build --target=bun --sourcemap=external --packages=external --outdir ./dist ./src/bun.ts",
|
|
17
|
+
"build-types": "rm -f tsconfig.tsbuildinfo && tsc",
|
|
18
|
+
"lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0",
|
|
19
|
+
"test": "bun test",
|
|
20
|
+
"prepare": "node -e \"const pkg=require('./package.json');pkg.exports['.'].bun='./dist/bun.js';pkg.exports['.'].types='./dist/types.d.ts';require('fs').writeFileSync('package.json',JSON.stringify(pkg,null,2))\""
|
|
21
|
+
},
|
|
22
|
+
"peerDependencies": {
|
|
23
|
+
"@workglow/sqlite": "0.0.52",
|
|
24
|
+
"@workglow/util": "0.0.52",
|
|
25
|
+
"pg": "^8.16.3",
|
|
26
|
+
"@supabase/supabase-js": "^2.84.0"
|
|
27
|
+
},
|
|
28
|
+
"peerDependenciesMeta": {
|
|
29
|
+
"@workglow/sqlite": {
|
|
30
|
+
"optional": false
|
|
31
|
+
},
|
|
32
|
+
"@workglow/util": {
|
|
33
|
+
"optional": false
|
|
34
|
+
}
|
|
35
|
+
},
|
|
36
|
+
"devDependencies": {
|
|
37
|
+
"@workglow/sqlite": "0.0.52",
|
|
38
|
+
"@workglow/util": "0.0.52",
|
|
39
|
+
"pg": "^8.16.3",
|
|
40
|
+
"@types/pg": "^8.15.5",
|
|
41
|
+
"@supabase/supabase-js": "^2.84.0",
|
|
42
|
+
"fake-indexeddb": "^6.2.4"
|
|
43
|
+
},
|
|
44
|
+
"exports": {
|
|
45
|
+
".": {
|
|
46
|
+
"react-native": "./dist/browser.js",
|
|
47
|
+
"browser": "./dist/browser.js",
|
|
48
|
+
"bun": "./dist/bun.js",
|
|
49
|
+
"types": "./dist/types.d.ts",
|
|
50
|
+
"node": "./dist/node.js"
|
|
51
|
+
}
|
|
52
|
+
},
|
|
53
|
+
"files": [
|
|
54
|
+
"dist",
|
|
55
|
+
"src/**/*.md"
|
|
56
|
+
],
|
|
57
|
+
"publishConfig": {
|
|
58
|
+
"access": "public"
|
|
59
|
+
}
|
|
60
|
+
}
|
package/src/kv/README.md
ADDED
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
# Key-Value Storage Module
|
|
2
|
+
|
|
3
|
+
A flexible key-value storage solution with multiple backend implementations. Provides a consistent interface for CRUD operations with event monitoring capabilities.
|
|
4
|
+
|
|
5
|
+
- [Features](#features)
|
|
6
|
+
- [Installation](#installation)
|
|
7
|
+
- [Usage](#usage)
|
|
8
|
+
- [File System Storage](#file-system-storage)
|
|
9
|
+
- [Browser IndexedDB Storage](#browser-indexeddb-storage)
|
|
10
|
+
- [PostgreSQL Storage](#postgresql-storage)
|
|
11
|
+
- [SQLite Storage](#sqlite-storage)
|
|
12
|
+
- [In-Memory Storage](#in-memory-storage)
|
|
13
|
+
- [API Documentation](#api-documentation)
|
|
14
|
+
- [Core Methods](#core-methods)
|
|
15
|
+
- [Events](#events)
|
|
16
|
+
- [Testing](#testing)
|
|
17
|
+
- [License](#license)
|
|
18
|
+
|
|
19
|
+
## Features
|
|
20
|
+
|
|
21
|
+
- Multiple storage backends:
|
|
22
|
+
- 🗂️ `FsFolderKvRepository` - File system storage
|
|
23
|
+
- 💾 `IndexedDbKvRepository` - Browser IndexedDB storage
|
|
24
|
+
- 🐘 `PostgresKvRepository` - PostgreSQL database storage
|
|
25
|
+
- 📁 `SqliteKvRepository` - SQLite database storage
|
|
26
|
+
- 🧠 `InMemoryKvRepository` - Volatile memory storage
|
|
27
|
+
- Type-safe key/value definitions
|
|
28
|
+
- JSON value serialization support
|
|
29
|
+
- Event emitter for storage operations (put/get/delete)
|
|
30
|
+
- Cross-platform compatibility (Node.js and browser)
|
|
31
|
+
|
|
32
|
+
## Installation
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
bun install @workglow/storage
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
## Usage
|
|
39
|
+
|
|
40
|
+
### File System Storage
|
|
41
|
+
|
|
42
|
+
```typescript
|
|
43
|
+
import { FsFolderKvRepository } from "@workglow/storage/kv";
|
|
44
|
+
|
|
45
|
+
const fsRepo = new FsFolderKvRepository(
|
|
46
|
+
"./data-storage", // Storage directory
|
|
47
|
+
"string", // Key type (string)
|
|
48
|
+
"json" // Value type (JSON serialized)
|
|
49
|
+
);
|
|
50
|
+
|
|
51
|
+
await fsRepo.put("config", { theme: "dark", notifications: true });
|
|
52
|
+
const config = await fsRepo.get("config");
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
### Browser IndexedDB Storage
|
|
56
|
+
|
|
57
|
+
```typescript
|
|
58
|
+
import { IndexedDbKvRepository } from "@workglow/storage/kv";
|
|
59
|
+
|
|
60
|
+
const idbRepo = new IndexedDbKvRepository(
|
|
61
|
+
"my-app-db", // Database name
|
|
62
|
+
"string", // Key type
|
|
63
|
+
"json" // Value type
|
|
64
|
+
);
|
|
65
|
+
|
|
66
|
+
// Works in browser environments
|
|
67
|
+
await idbRepo.put("session", { userId: 123, token: "abc" });
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
### PostgreSQL Storage
|
|
71
|
+
|
|
72
|
+
```typescript
|
|
73
|
+
import { PostgresKvRepository } from "@workglow/storage/kv";
|
|
74
|
+
import { PGlite } from "@electric-sql/pglite";
|
|
75
|
+
|
|
76
|
+
const pg = new PGlite(); // Requires @electric-sql/pglite
|
|
77
|
+
const pgRepo = new PostgresKvRepository(
|
|
78
|
+
pg, // PostgreSQL connection
|
|
79
|
+
"user_data", // Table name
|
|
80
|
+
"string", // Key type
|
|
81
|
+
"json" // Value type
|
|
82
|
+
);
|
|
83
|
+
|
|
84
|
+
await pgRepo.put("preferences:456", { lang: "en", fontSize: 16 });
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
### SQLite Storage
|
|
88
|
+
|
|
89
|
+
```typescript
|
|
90
|
+
import { SqliteKvRepository } from "@workglow/storage/kv";
|
|
91
|
+
import { Sqlite } from "@workglow/sqlite";
|
|
92
|
+
|
|
93
|
+
const db = new Sqlite(":memory:"); // In-memory database
|
|
94
|
+
const sqliteRepo = new SqliteKvRepository(
|
|
95
|
+
db, // SQLite connection
|
|
96
|
+
"cache_data", // Table name
|
|
97
|
+
"string", // Key type
|
|
98
|
+
"string" // Value type (raw strings)
|
|
99
|
+
);
|
|
100
|
+
|
|
101
|
+
await sqliteRepo.put("temp:789", "cached_value");
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
### In-Memory Storage
|
|
105
|
+
|
|
106
|
+
```typescript
|
|
107
|
+
import { InMemoryKvRepository } from "@workglow/storage/kv";
|
|
108
|
+
|
|
109
|
+
const memRepo = new InMemoryKvRepository(
|
|
110
|
+
"string", // Key type
|
|
111
|
+
"json" // Value type
|
|
112
|
+
);
|
|
113
|
+
|
|
114
|
+
// Volatile storage - data lost on process exit
|
|
115
|
+
await memRepo.put("counter", { value: 42 });
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
## API Documentation
|
|
119
|
+
|
|
120
|
+
### Core Methods
|
|
121
|
+
|
|
122
|
+
- `put(key: Key, value: Value): Promise<void>`
|
|
123
|
+
- `get(key: Key): Promise<Value | undefined>`
|
|
124
|
+
- `delete(key: Key): Promise<void>`
|
|
125
|
+
- `getAll(): Promise<Combined[] | undefined>`
|
|
126
|
+
- `deleteAll(): Promise<void>`
|
|
127
|
+
- `size(): Promise<number>`
|
|
128
|
+
|
|
129
|
+
### Events
|
|
130
|
+
|
|
131
|
+
```typescript
|
|
132
|
+
repo.on("put", (key, value) => {
|
|
133
|
+
console.log(`Stored ${key}:`, value);
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
repo.on("delete", (key) => {
|
|
137
|
+
console.log(`Deleted ${key}`);
|
|
138
|
+
});
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
## Testing
|
|
142
|
+
|
|
143
|
+
Run all tests:
|
|
144
|
+
|
|
145
|
+
```bash
|
|
146
|
+
bun test
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
Run specific implementation tests:
|
|
150
|
+
|
|
151
|
+
```bash
|
|
152
|
+
bun test FsFolderKvRepository
|
|
153
|
+
bun test IndexedDbKvRepository
|
|
154
|
+
bun test PostgresKvRepository
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
## License
|
|
158
|
+
|
|
159
|
+
Apache 2.0 - See LICENSE for details
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# ELLMERS Queue Storage Package
|
|
2
|
+
|
|
3
|
+
This is not a job queue implementation. It is a storage implementation for job queues. See the [@workglow/job-queue](../../../job-queue/README.md) package for a job queue implementation that uses these storage implementations.
|
|
4
|
+
|
|
5
|
+
- [Features](#features)
|
|
6
|
+
- [Performance Considerations](#performance-considerations)
|
|
7
|
+
- [Job Lifecycle](#job-lifecycle)
|
|
8
|
+
|
|
9
|
+
## Features
|
|
10
|
+
|
|
11
|
+
- Multiple storage implementations:
|
|
12
|
+
- `InMemoryQueueStorage` - Volatile memory (dev/testing)
|
|
13
|
+
- `IndexedDbQueueStorage` - Browser-based storage
|
|
14
|
+
- `SqliteQueueStorage` - Embedded SQLite
|
|
15
|
+
- `PostgresQueueStorage` - Production-grade PostgreSQL
|
|
16
|
+
- Job lifecycle management:
|
|
17
|
+
- PENDING → PROCESSING → COMPLETED/FAILED/ABORTED
|
|
18
|
+
- PENDING → DISABLED
|
|
19
|
+
- Automatic retry mechanisms
|
|
20
|
+
- Progress tracking with message/details
|
|
21
|
+
- Fingerprint-based input deduplication
|
|
22
|
+
- Transactional operations with SKIP LOCKED
|
|
23
|
+
- Job expiration policies
|
|
24
|
+
|
|
25
|
+
## Performance Considerations
|
|
26
|
+
|
|
27
|
+
1. **IndexedDB**: Best for client-side applications with <10k jobs
|
|
28
|
+
2. **SQLite**: Ideal for single-process applications
|
|
29
|
+
3. **PostgreSQL**: Recommended for distributed systems with high throughput
|
|
30
|
+
4. **In-Memory**: Suitable for testing/development only
|
|
31
|
+
|
|
32
|
+
## Job Lifecycle
|
|
33
|
+
|
|
34
|
+
1. Jobs start as `PENDING`
|
|
35
|
+
2. Acquired via `next()` → `PROCESSING`
|
|
36
|
+
3. Final states:
|
|
37
|
+
- `COMPLETED`: Successful execution
|
|
38
|
+
- `FAILED`: Unrecoverable error
|
|
39
|
+
- `ABORTED`: Manual cancellation
|
|
40
|
+
- `DISABLED`: Disabled due to conditions not met
|
|
41
|
+
- Auto-retried while `PENDING` if within retry limits
|
|
@@ -0,0 +1,298 @@
|
|
|
1
|
+
# Tabular Repository Implementations
|
|
2
|
+
|
|
3
|
+
A collection of storage implementations for tabular data with multiple backend support. Provides consistent CRUD operations, search capabilities, and event monitoring across different storage technologies.
|
|
4
|
+
|
|
5
|
+
- [Features](#features)
|
|
6
|
+
- [Installation](#installation)
|
|
7
|
+
- [Basic Usage](#basic-usage)
|
|
8
|
+
- [Schema Definitions](#schema-definitions)
|
|
9
|
+
- [Using TypeBox](#using-typebox)
|
|
10
|
+
- [Using Zod 4](#using-zod-4)
|
|
11
|
+
- [Implementations](#implementations)
|
|
12
|
+
- [InMemoryTabularRepository](#inmemorytabularrepository)
|
|
13
|
+
- [SqliteTabularRepository](#sqlitetabularrepository)
|
|
14
|
+
- [PostgresTabularRepository](#postgrestabularrepository)
|
|
15
|
+
- [IndexedDbTabularRepository](#indexeddbtabularrepository)
|
|
16
|
+
- [FsFolderTabularRepository](#fsfoldertabularrepository)
|
|
17
|
+
- [Events](#events)
|
|
18
|
+
- [Testing](#testing)
|
|
19
|
+
- [License](#license)
|
|
20
|
+
|
|
21
|
+
## Features
|
|
22
|
+
|
|
23
|
+
- Multiple storage backends:
|
|
24
|
+
- In-memory (for testing/caching)
|
|
25
|
+
- SQLite (embedded database)
|
|
26
|
+
- PostgreSQL (relational database)
|
|
27
|
+
- IndexedDB (browser storage)
|
|
28
|
+
- Filesystem (JSON file per record)
|
|
29
|
+
- Type-safe schema definitions
|
|
30
|
+
- Compound primary keys support
|
|
31
|
+
- Indexing for efficient search
|
|
32
|
+
- Event-driven architecture
|
|
33
|
+
- Cross-implementation test suite
|
|
34
|
+
|
|
35
|
+
## Installation
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
bun add @workglow/storage
|
|
39
|
+
# or
|
|
40
|
+
npm install @workglow/storage
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
## Basic Usage
|
|
44
|
+
|
|
45
|
+
```typescript
|
|
46
|
+
import { InMemoryTabularRepository } from "@workglow/storage/tabular";
|
|
47
|
+
|
|
48
|
+
// Define schema and primary keys
|
|
49
|
+
const schema = {
|
|
50
|
+
id: "string",
|
|
51
|
+
name: "string",
|
|
52
|
+
age: "number",
|
|
53
|
+
active: "boolean",
|
|
54
|
+
} as const;
|
|
55
|
+
|
|
56
|
+
const primaryKeys = ["id"] as const;
|
|
57
|
+
// Create repository instance (when using const schemas, the next three generics
|
|
58
|
+
// on InMemoryTabularRepository are automatically created for you)
|
|
59
|
+
const repo = new InMemoryTabularRepository<typeof schema, typeof primaryKeys>(schema, primaryKeys);
|
|
60
|
+
|
|
61
|
+
// Basic operations
|
|
62
|
+
await repo.put({ id: "1", name: "Alice", age: 30, active: true });
|
|
63
|
+
const result = await repo.get({ id: "1" });
|
|
64
|
+
await repo.delete({ id: "1" });
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
## Schema Definitions
|
|
68
|
+
|
|
69
|
+
You can define schemas using plain JSON Schema objects, or use schema libraries like TypeBox or Zod 4 to create them. All schemas must be compatible with `DataPortSchemaObject` from `@workglow/util`.
|
|
70
|
+
|
|
71
|
+
**Note:** When using TypeBox or Zod schemas, you **must** explicitly provide the generic type parameters to the repository constructor, as TypeScript cannot infer them from non-const schema definitions.
|
|
72
|
+
|
|
73
|
+
### Using TypeBox
|
|
74
|
+
|
|
75
|
+
TypeBox schemas are JSON Schema compatible and can be used directly:
|
|
76
|
+
|
|
77
|
+
```typescript
|
|
78
|
+
import { InMemoryTabularRepository } from "@workglow/storage/tabular";
|
|
79
|
+
import { Type, Static } from "@sinclair/typebox";
|
|
80
|
+
import { DataPortSchemaObject, FromSchema, IncludeProps, ExcludeProps } from "@workglow/util";
|
|
81
|
+
|
|
82
|
+
// Define schema using TypeBox
|
|
83
|
+
const userSchema = Type.Object({
|
|
84
|
+
id: Type.String({ format: "uuid" }),
|
|
85
|
+
name: Type.String({ minLength: 1, maxLength: 100 }),
|
|
86
|
+
email: Type.String({ format: "email" }),
|
|
87
|
+
age: Type.Optional(Type.Number({ minimum: 0, maximum: 150 })),
|
|
88
|
+
active: Type.Boolean({ default: true }),
|
|
89
|
+
}) satisfies DataPortSchemaObject;
|
|
90
|
+
|
|
91
|
+
// Infer TypeScript types from schema
|
|
92
|
+
type User = FromSchema<typeof userSchema>;
|
|
93
|
+
// => { id: string; name: string; email: string; age?: number; active: boolean }
|
|
94
|
+
|
|
95
|
+
const primaryKeys = ["id"] as const;
|
|
96
|
+
|
|
97
|
+
type UserEntity = FromSchema<typeof userSchema>;
|
|
98
|
+
|
|
99
|
+
// IMPORTANT: You must explicitly provide generic type parameters for t
|
|
100
|
+
// TypeScript cannot infer them from TypeBox schemas
|
|
101
|
+
const repo = new InMemoryTabularRepository<typeof userSchema, typeof primaryKeys, UserEntity>(
|
|
102
|
+
userSchema,
|
|
103
|
+
primaryKeys,
|
|
104
|
+
["email", "active"] // Indexes
|
|
105
|
+
);
|
|
106
|
+
|
|
107
|
+
// Use with type safety
|
|
108
|
+
await repo.put({
|
|
109
|
+
id: "550e8400-e29b-41d4-a716-446655440000",
|
|
110
|
+
name: "Alice",
|
|
111
|
+
email: "alice@example.com",
|
|
112
|
+
age: 30,
|
|
113
|
+
active: true,
|
|
114
|
+
});
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
### Using Zod 4
|
|
118
|
+
|
|
119
|
+
Zod 4 has built-in JSON Schema support using the `.toJSONSchema()` method:
|
|
120
|
+
|
|
121
|
+
```typescript
|
|
122
|
+
import { InMemoryTabularRepository } from "@workglow/storage/tabular";
|
|
123
|
+
import { z } from "zod";
|
|
124
|
+
import { DataPortSchemaObject } from "@workglow/util";
|
|
125
|
+
|
|
126
|
+
// Define schema using Zod
|
|
127
|
+
const userSchemaZod = z.object({
|
|
128
|
+
id: z.string().uuid(),
|
|
129
|
+
name: z.string().min(1).max(100),
|
|
130
|
+
email: z.string().email(),
|
|
131
|
+
age: z.number().min(0).max(150).optional(),
|
|
132
|
+
active: z.boolean().default(true),
|
|
133
|
+
});
|
|
134
|
+
|
|
135
|
+
// Convert Zod schema to JSON Schema using built-in method
|
|
136
|
+
const userSchema = userSchemaZod.toJSONSchema() as DataPortSchemaObject;
|
|
137
|
+
const primaryKeys = ["id"] as const;
|
|
138
|
+
|
|
139
|
+
// Define computed types for the repository generics
|
|
140
|
+
type UserEntity = z.infer<typeof userSchemaZod>;
|
|
141
|
+
|
|
142
|
+
// IMPORTANT: You must explicitly provide generic type parameters
|
|
143
|
+
// TypeScript cannot infer them from Zod schemas (even after conversion)
|
|
144
|
+
const repo = new InMemoryTabularRepository<typeof userSchema, typeof primaryKeys, UserEntity>(
|
|
145
|
+
userSchema,
|
|
146
|
+
primaryKeys,
|
|
147
|
+
["email", "active"] // Indexes
|
|
148
|
+
);
|
|
149
|
+
|
|
150
|
+
// Use with type safety
|
|
151
|
+
await repo.put({
|
|
152
|
+
id: "550e8400-e29b-41d4-a716-446655440000",
|
|
153
|
+
name: "Alice",
|
|
154
|
+
email: "alice@example.com",
|
|
155
|
+
age: 30,
|
|
156
|
+
active: true,
|
|
157
|
+
});
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
## Implementations
|
|
161
|
+
|
|
162
|
+
### InMemoryTabularRepository
|
|
163
|
+
|
|
164
|
+
- Ideal for testing/development
|
|
165
|
+
- No persistence
|
|
166
|
+
- Fast search capabilities
|
|
167
|
+
|
|
168
|
+
```typescript
|
|
169
|
+
const repo = new InMemoryTabularRepository<
|
|
170
|
+
typeof schema,
|
|
171
|
+
typeof primaryKeys,
|
|
172
|
+
Entity, // required if using TypeBox, Zod, etc, otherwise automatically created
|
|
173
|
+
PrimaryKeyEntity, // should be automatically created
|
|
174
|
+
ValueEntity // should be automatically created
|
|
175
|
+
>(schema, primaryKeys, ["name", "active"]);
|
|
176
|
+
```
|
|
177
|
+
|
|
178
|
+
### SqliteTabularRepository
|
|
179
|
+
|
|
180
|
+
- Embedded SQLite database
|
|
181
|
+
- File-based or in-memory
|
|
182
|
+
|
|
183
|
+
```typescript
|
|
184
|
+
const repo = new SqliteTabularRepository<
|
|
185
|
+
typeof schema,
|
|
186
|
+
typeof primaryKeys,
|
|
187
|
+
Entity, // required if using TypeBox, Zod, etc, otherwise automatically created
|
|
188
|
+
PrimaryKeyEntity, // should be automatically created
|
|
189
|
+
ValueEntity // should be automatically created
|
|
190
|
+
>(
|
|
191
|
+
":memory:", // Database path
|
|
192
|
+
"users", // Table name
|
|
193
|
+
schema,
|
|
194
|
+
primaryKeys,
|
|
195
|
+
[["name", "active"], "age"] // Indexes
|
|
196
|
+
);
|
|
197
|
+
```
|
|
198
|
+
|
|
199
|
+
### PostgresTabularRepository
|
|
200
|
+
|
|
201
|
+
- PostgreSQL backend
|
|
202
|
+
- Connection pooling support
|
|
203
|
+
|
|
204
|
+
```typescript
|
|
205
|
+
import { Pool } from "pg";
|
|
206
|
+
|
|
207
|
+
const pool = new Pool({
|
|
208
|
+
/* config */
|
|
209
|
+
});
|
|
210
|
+
const repo = new PostgresTabularRepository<
|
|
211
|
+
typeof schema,
|
|
212
|
+
typeof primaryKeys,
|
|
213
|
+
Entity, // required if using TypeBox, Zod, etc, otherwise automatically created
|
|
214
|
+
PrimaryKeyEntity, // should be automatically created
|
|
215
|
+
ValueEntity // should be automatically created
|
|
216
|
+
>(
|
|
217
|
+
pool, // postgres connection pool
|
|
218
|
+
"users",
|
|
219
|
+
schema,
|
|
220
|
+
primaryKeys,
|
|
221
|
+
[["name", "active"], "age"]
|
|
222
|
+
);
|
|
223
|
+
```
|
|
224
|
+
|
|
225
|
+
### IndexedDbTabularRepository
|
|
226
|
+
|
|
227
|
+
- Browser-based storage
|
|
228
|
+
- Automatic schema migration
|
|
229
|
+
|
|
230
|
+
```typescript
|
|
231
|
+
const repo = new IndexedDbTabularRepository<
|
|
232
|
+
typeof schema,
|
|
233
|
+
typeof primaryKeys,
|
|
234
|
+
Entity, // required if using TypeBox, Zod, etc, otherwise automatically created
|
|
235
|
+
PrimaryKeyEntity, // should be automatically created
|
|
236
|
+
ValueEntity // should be automatically created
|
|
237
|
+
>(
|
|
238
|
+
"user_db", // Database name
|
|
239
|
+
schema,
|
|
240
|
+
primaryKeys,
|
|
241
|
+
[["name", "active"], "age"]
|
|
242
|
+
);
|
|
243
|
+
```
|
|
244
|
+
|
|
245
|
+
### FsFolderTabularRepository
|
|
246
|
+
|
|
247
|
+
- Filesystem storage (one JSON file per record)
|
|
248
|
+
- Simple persistence format
|
|
249
|
+
|
|
250
|
+
```typescript
|
|
251
|
+
const repo = new FsFolderTabularRepository<
|
|
252
|
+
typeof schema,
|
|
253
|
+
typeof primaryKeys,
|
|
254
|
+
Entity, // required if using TypeBox, Zod, etc, otherwise automatically created
|
|
255
|
+
PrimaryKeyEntity, // should be automatically created
|
|
256
|
+
ValueEntity // should be automatically created
|
|
257
|
+
>("./data/users", schema, primaryKeys);
|
|
258
|
+
```
|
|
259
|
+
|
|
260
|
+
## Events
|
|
261
|
+
|
|
262
|
+
All implementations emit events:
|
|
263
|
+
|
|
264
|
+
- `put`: When a record is created/updated
|
|
265
|
+
- `get`: When a record is retrieved
|
|
266
|
+
- `delete`: When a record is deleted
|
|
267
|
+
- `clearall`: When all records are deleted
|
|
268
|
+
- `search`: When a search is performed
|
|
269
|
+
|
|
270
|
+
```typescript
|
|
271
|
+
repo.on("put", (entity) => {
|
|
272
|
+
console.log("Record stored:", entity);
|
|
273
|
+
});
|
|
274
|
+
|
|
275
|
+
repo.on("delete", (key) => {
|
|
276
|
+
console.log("Record deleted:", key);
|
|
277
|
+
});
|
|
278
|
+
```
|
|
279
|
+
|
|
280
|
+
## Testing
|
|
281
|
+
|
|
282
|
+
The implementations share a common test suite. To run tests:
|
|
283
|
+
|
|
284
|
+
```bash
|
|
285
|
+
bun test
|
|
286
|
+
```
|
|
287
|
+
|
|
288
|
+
Test includes:
|
|
289
|
+
|
|
290
|
+
- Basic CRUD operations
|
|
291
|
+
- Compound key handling
|
|
292
|
+
- Index-based search
|
|
293
|
+
- Event emission
|
|
294
|
+
- Concurrency tests
|
|
295
|
+
|
|
296
|
+
## License
|
|
297
|
+
|
|
298
|
+
Apache 2.0
|