@workglow/storage 0.2.27 → 0.2.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser.d.ts +0 -10
- package/dist/browser.d.ts.map +1 -1
- package/dist/browser.js +406 -4207
- package/dist/browser.js.map +8 -24
- package/dist/bun.js +341 -7235
- package/dist/bun.js.map +8 -35
- package/dist/common-server.d.ts +0 -21
- package/dist/common-server.d.ts.map +1 -1
- package/dist/common.d.ts +1 -6
- package/dist/common.d.ts.map +1 -1
- package/dist/node.js +341 -7236
- package/dist/node.js.map +8 -35
- package/package.json +5 -71
- package/dist/kv/IndexedDbKvStorage.d.ts +0 -27
- package/dist/kv/IndexedDbKvStorage.d.ts.map +0 -1
- package/dist/kv/PostgresKvStorage.d.ts +0 -28
- package/dist/kv/PostgresKvStorage.d.ts.map +0 -1
- package/dist/kv/SqliteKvStorage.d.ts +0 -28
- package/dist/kv/SqliteKvStorage.d.ts.map +0 -1
- package/dist/kv/SupabaseKvStorage.d.ts +0 -33
- package/dist/kv/SupabaseKvStorage.d.ts.map +0 -1
- package/dist/postgres/browser.d.ts +0 -32
- package/dist/postgres/browser.d.ts.map +0 -1
- package/dist/postgres/browser.js +0 -150
- package/dist/postgres/browser.js.map +0 -11
- package/dist/postgres/node-bun.d.ts +0 -26
- package/dist/postgres/node-bun.d.ts.map +0 -1
- package/dist/postgres/node-bun.js +0 -41
- package/dist/postgres/node-bun.js.map +0 -10
- package/dist/postgres/pglite-pool.d.ts +0 -21
- package/dist/postgres/pglite-pool.d.ts.map +0 -1
- package/dist/queue/IQueueStorage.d.ts +0 -229
- package/dist/queue/IQueueStorage.d.ts.map +0 -1
- package/dist/queue/InMemoryQueueStorage.d.ts +0 -149
- package/dist/queue/InMemoryQueueStorage.d.ts.map +0 -1
- package/dist/queue/IndexedDbQueueStorage.d.ts +0 -166
- package/dist/queue/IndexedDbQueueStorage.d.ts.map +0 -1
- package/dist/queue/PostgresQueueStorage.d.ts +0 -154
- package/dist/queue/PostgresQueueStorage.d.ts.map +0 -1
- package/dist/queue/SqliteQueueStorage.d.ts +0 -149
- package/dist/queue/SqliteQueueStorage.d.ts.map +0 -1
- package/dist/queue/SupabaseQueueStorage.d.ts +0 -195
- package/dist/queue/SupabaseQueueStorage.d.ts.map +0 -1
- package/dist/queue/TelemetryQueueStorage.d.ts +0 -33
- package/dist/queue/TelemetryQueueStorage.d.ts.map +0 -1
- package/dist/queue-limiter/IRateLimiterStorage.d.ts +0 -127
- package/dist/queue-limiter/IRateLimiterStorage.d.ts.map +0 -1
- package/dist/queue-limiter/InMemoryRateLimiterStorage.d.ts +0 -43
- package/dist/queue-limiter/InMemoryRateLimiterStorage.d.ts.map +0 -1
- package/dist/queue-limiter/IndexedDbRateLimiterStorage.d.ts +0 -79
- package/dist/queue-limiter/IndexedDbRateLimiterStorage.d.ts.map +0 -1
- package/dist/queue-limiter/PostgresRateLimiterStorage.d.ts +0 -57
- package/dist/queue-limiter/PostgresRateLimiterStorage.d.ts.map +0 -1
- package/dist/queue-limiter/SqliteRateLimiterStorage.d.ts +0 -62
- package/dist/queue-limiter/SqliteRateLimiterStorage.d.ts.map +0 -1
- package/dist/queue-limiter/SupabaseRateLimiterStorage.d.ts +0 -54
- package/dist/queue-limiter/SupabaseRateLimiterStorage.d.ts.map +0 -1
- package/dist/sqlite/browser.d.ts +0 -37
- package/dist/sqlite/browser.d.ts.map +0 -1
- package/dist/sqlite/browser.js +0 -125
- package/dist/sqlite/browser.js.map +0 -10
- package/dist/sqlite/bun.d.ts +0 -32
- package/dist/sqlite/bun.d.ts.map +0 -1
- package/dist/sqlite/bun.js +0 -84
- package/dist/sqlite/bun.js.map +0 -10
- package/dist/sqlite/canonical-api.d.ts +0 -34
- package/dist/sqlite/canonical-api.d.ts.map +0 -1
- package/dist/sqlite/node.d.ts +0 -34
- package/dist/sqlite/node.d.ts.map +0 -1
- package/dist/sqlite/node.js +0 -65
- package/dist/sqlite/node.js.map +0 -10
- package/dist/tabular/IndexedDbTabularStorage.d.ts +0 -199
- package/dist/tabular/IndexedDbTabularStorage.d.ts.map +0 -1
- package/dist/tabular/PostgresTabularStorage.d.ts +0 -196
- package/dist/tabular/PostgresTabularStorage.d.ts.map +0 -1
- package/dist/tabular/SqliteTabularStorage.d.ts +0 -167
- package/dist/tabular/SqliteTabularStorage.d.ts.map +0 -1
- package/dist/tabular/SupabaseTabularStorage.d.ts +0 -174
- package/dist/tabular/SupabaseTabularStorage.d.ts.map +0 -1
- package/dist/util/IndexedDbTable.d.ts +0 -40
- package/dist/util/IndexedDbTable.d.ts.map +0 -1
- package/dist/util/traced.d.ts +0 -10
- package/dist/util/traced.d.ts.map +0 -1
- package/dist/vector/IndexedDbVectorStorage.d.ts +0 -53
- package/dist/vector/IndexedDbVectorStorage.d.ts.map +0 -1
- package/dist/vector/PostgresVectorStorage.d.ts +0 -39
- package/dist/vector/PostgresVectorStorage.d.ts.map +0 -1
- package/dist/vector/SqliteAiVectorStorage.d.ts +0 -100
- package/dist/vector/SqliteAiVectorStorage.d.ts.map +0 -1
- package/dist/vector/SqliteVectorStorage.d.ts +0 -49
- package/dist/vector/SqliteVectorStorage.d.ts.map +0 -1
- package/src/queue/README.md +0 -41
|
@@ -1,53 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @license
|
|
3
|
-
* Copyright 2025 Steven Roussey <sroussey@gmail.com>
|
|
4
|
-
* SPDX-License-Identifier: Apache-2.0
|
|
5
|
-
*/
|
|
6
|
-
import type { DataPortSchemaObject, FromSchema, TypedArray, TypedArrayConstructor, TypedArraySchemaOptions } from "@workglow/util/schema";
|
|
7
|
-
import type { ClientProvidedKeysOption } from "../tabular/BaseTabularStorage";
|
|
8
|
-
import { IndexedDbTabularStorage } from "../tabular/IndexedDbTabularStorage";
|
|
9
|
-
import type { MigrationOptions } from "../util/IndexedDbTable";
|
|
10
|
-
import type { AnyVectorStorage, HybridSearchOptions, IVectorStorage, VectorSearchOptions } from "./IVectorStorage";
|
|
11
|
-
export declare const IDB_VECTOR_REPOSITORY: import("@workglow/util").ServiceToken<AnyVectorStorage>;
|
|
12
|
-
/**
|
|
13
|
-
* IndexedDB vector storage implementation.
|
|
14
|
-
* Extends IndexedDbTabularStorage for storage.
|
|
15
|
-
* Suitable for browser applications that need persistent vector storage.
|
|
16
|
-
* No vector serialization needed since IndexedDB supports TypedArrays
|
|
17
|
-
* natively via structured clone.
|
|
18
|
-
*
|
|
19
|
-
* @template Schema - The schema definition for the entity
|
|
20
|
-
* @template PrimaryKeyNames - The primary key names
|
|
21
|
-
* @template Metadata - The metadata type for the vector
|
|
22
|
-
* @template VectorCtor - Constructor for stored vectors (default {@link typeof Float32Array})
|
|
23
|
-
* @template Entity - The entity type
|
|
24
|
-
*/
|
|
25
|
-
export declare class IndexedDbVectorStorage<Schema extends DataPortSchemaObject, PrimaryKeyNames extends ReadonlyArray<keyof Schema["properties"]>, Metadata extends Record<string, unknown> = Record<string, unknown>, Entity = FromSchema<Schema, TypedArraySchemaOptions>> extends IndexedDbTabularStorage<Schema, PrimaryKeyNames, Entity> implements IVectorStorage<Metadata, Schema, Entity, PrimaryKeyNames> {
|
|
26
|
-
private vectorDimensions;
|
|
27
|
-
private vectorPropertyName;
|
|
28
|
-
private metadataPropertyName;
|
|
29
|
-
/**
|
|
30
|
-
* Creates a new IndexedDB vector storage
|
|
31
|
-
* @param table - The name of the IndexedDB store (defaults to 'vectors')
|
|
32
|
-
* @param schema - The schema definition for the entity
|
|
33
|
-
* @param primaryKeyNames - Array of property names that form the primary key
|
|
34
|
-
* @param indexes - Array of columns or column arrays to make searchable
|
|
35
|
-
* @param dimensions - The number of dimensions of the vector
|
|
36
|
-
* @param _vectorCtor - TypedArray constructor (unused, IndexedDB stores typed arrays natively)
|
|
37
|
-
* @param migrationOptions - Options for handling database schema migrations
|
|
38
|
-
* @param clientProvidedKeys - How to handle client-provided values for auto-generated keys
|
|
39
|
-
*/
|
|
40
|
-
constructor(table: string | undefined, schema: Schema, primaryKeyNames: PrimaryKeyNames, indexes: readonly (keyof NoInfer<Entity> | readonly (keyof NoInfer<Entity>)[])[] | undefined, dimensions: number, _vectorCtor?: TypedArrayConstructor, migrationOptions?: MigrationOptions, clientProvidedKeys?: ClientProvidedKeysOption);
|
|
41
|
-
/**
|
|
42
|
-
* Get the vector dimensions
|
|
43
|
-
* @returns The vector dimensions
|
|
44
|
-
*/
|
|
45
|
-
getVectorDimensions(): number;
|
|
46
|
-
similaritySearch(query: TypedArray, options?: VectorSearchOptions<Record<string, unknown>>): Promise<(Entity & {
|
|
47
|
-
score: number;
|
|
48
|
-
})[]>;
|
|
49
|
-
hybridSearch(query: TypedArray, options: HybridSearchOptions<Record<string, unknown>>): Promise<(Entity & {
|
|
50
|
-
score: number;
|
|
51
|
-
})[]>;
|
|
52
|
-
}
|
|
53
|
-
//# sourceMappingURL=IndexedDbVectorStorage.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"IndexedDbVectorStorage.d.ts","sourceRoot":"","sources":["../../src/vector/IndexedDbVectorStorage.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,KAAK,EACV,oBAAoB,EACpB,UAAU,EACV,UAAU,EACV,qBAAqB,EACrB,uBAAuB,EACxB,MAAM,uBAAuB,CAAC;AAE/B,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAC9E,OAAO,EAAE,uBAAuB,EAAE,MAAM,oCAAoC,CAAC;AAC7E,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC/D,OAAO,KAAK,EACV,gBAAgB,EAChB,mBAAmB,EACnB,cAAc,EACd,mBAAmB,EACpB,MAAM,kBAAkB,CAAC;AAG1B,eAAO,MAAM,qBAAqB,yDAEjC,CAAC;AAiCF;;;;;;;;;;;;GAYG;AACH,qBAAa,sBAAsB,CACjC,MAAM,SAAS,oBAAoB,EACnC,eAAe,SAAS,aAAa,CAAC,MAAM,MAAM,CAAC,YAAY,CAAC,CAAC,EACjE,QAAQ,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAClE,MAAM,GAAG,UAAU,CAAC,MAAM,EAAE,uBAAuB,CAAC,CAEpD,SAAQ,uBAAuB,CAAC,MAAM,EAAE,eAAe,EAAE,MAAM,CAC/D,YAAW,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,eAAe,CAAC;IAEpE,OAAO,CAAC,gBAAgB,CAAS;IACjC,OAAO,CAAC,kBAAkB,CAAe;IACzC,OAAO,CAAC,oBAAoB,CAA2B;IAEvD;;;;;;;;;;OAUG;IACH,YACE,KAAK,EAAE,MAAM,YAAY,EACzB,MAAM,EAAE,MAAM,EACd,eAAe,EAAE,eAAe,EAChC,OAAO,EAAE,SAAS,CAAC,MAAM,OAAO,CAAC,MAAM,CAAC,GAAG,SAAS,CAAC,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,YAAK,EACrF,UAAU,EAAE,MAAM,EAClB,WAAW,GAAE,qBAAoC,EACjD,gBAAgB,GAAE,gBAAqB,EACvC,kBAAkB,GAAE,wBAAuC,EAa5D;IAED;;;OAGG;IACH,mBAAmB,IAAI,MAAM,CAE5B;IAEK,gBAAgB,CACpB,KAAK,EAAE,UAAU,EACjB,OAAO,GAAE,mBAAmB,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAM;eAGnB,MAAM;UAmC9C;IAEK,YAAY,CAAC,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,mBAAmB,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;eArClD,MAAM;UAsF9C;CACF"}
|
|
@@ -1,39 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @license
|
|
3
|
-
* Copyright 2025 Steven Roussey <sroussey@gmail.com>
|
|
4
|
-
* SPDX-License-Identifier: Apache-2.0
|
|
5
|
-
*/
|
|
6
|
-
import type { Pool } from "@workglow/storage/postgres";
|
|
7
|
-
import type { DataPortSchemaObject, FromSchema, TypedArray, TypedArrayConstructor, TypedArraySchemaOptions } from "@workglow/util/schema";
|
|
8
|
-
import { PostgresTabularStorage } from "../tabular/PostgresTabularStorage";
|
|
9
|
-
import type { HybridSearchOptions, IVectorStorage, VectorSearchOptions } from "./IVectorStorage";
|
|
10
|
-
export declare class PostgresVectorStorage<Schema extends DataPortSchemaObject, PrimaryKeyNames extends ReadonlyArray<keyof Schema["properties"]>, Metadata extends Record<string, unknown> = Record<string, unknown>, Entity = FromSchema<Schema, TypedArraySchemaOptions>> extends PostgresTabularStorage<Schema, PrimaryKeyNames, Entity> implements IVectorStorage<Metadata, Schema, Entity, PrimaryKeyNames> {
|
|
11
|
-
private vectorDimensions;
|
|
12
|
-
private readonly vectorCtor;
|
|
13
|
-
private vectorPropertyName;
|
|
14
|
-
private metadataPropertyName;
|
|
15
|
-
/**
|
|
16
|
-
* Creates a new PostgreSQL vector repository
|
|
17
|
-
* @param db - PostgreSQL connection pool
|
|
18
|
-
* @param table - The name of the table to use for storage
|
|
19
|
-
* @param schema - The schema definition for the entity
|
|
20
|
-
* @param primaryKeyNames - Array of property names that form the primary key
|
|
21
|
-
* @param indexes - Array of columns or column arrays to make searchable
|
|
22
|
-
* @param dimensions - The number of dimensions of the vector
|
|
23
|
-
* @param vectorCtor - TypedArray constructor used when hydrating vectors from SQL text (e.g. {@link Float32Array})
|
|
24
|
-
*/
|
|
25
|
-
constructor(db: Pool, table: string, schema: Schema, primaryKeyNames: PrimaryKeyNames, indexes: readonly (keyof NoInfer<Entity> | readonly (keyof NoInfer<Entity>)[])[] | undefined, dimensions: number, vectorCtor?: TypedArrayConstructor);
|
|
26
|
-
getVectorDimensions(): number;
|
|
27
|
-
similaritySearch(query: TypedArray, options?: VectorSearchOptions<Metadata>): Promise<Array<Entity & {
|
|
28
|
-
score: number;
|
|
29
|
-
}>>;
|
|
30
|
-
hybridSearch(query: TypedArray, options: HybridSearchOptions<Metadata>): Promise<(Entity & {
|
|
31
|
-
score: number;
|
|
32
|
-
})[]>;
|
|
33
|
-
private searchFallback;
|
|
34
|
-
private hybridSearchFallback;
|
|
35
|
-
private getPrimaryKeyWhereClause;
|
|
36
|
-
private getPrimaryKeyValues;
|
|
37
|
-
private matchesFilter;
|
|
38
|
-
}
|
|
39
|
-
//# sourceMappingURL=PostgresVectorStorage.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"PostgresVectorStorage.d.ts","sourceRoot":"","sources":["../../src/vector/PostgresVectorStorage.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,4BAA4B,CAAC;AACvD,OAAO,KAAK,EACV,oBAAoB,EACpB,UAAU,EACV,UAAU,EACV,qBAAqB,EACrB,uBAAuB,EACxB,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAAE,sBAAsB,EAAE,MAAM,mCAAmC,CAAC;AAE3E,OAAO,KAAK,EAAE,mBAAmB,EAAE,cAAc,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AAqBjG,qBAAa,qBAAqB,CAChC,MAAM,SAAS,oBAAoB,EACnC,eAAe,SAAS,aAAa,CAAC,MAAM,MAAM,CAAC,YAAY,CAAC,CAAC,EACjE,QAAQ,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAClE,MAAM,GAAG,UAAU,CAAC,MAAM,EAAE,uBAAuB,CAAC,CAEpD,SAAQ,sBAAsB,CAAC,MAAM,EAAE,eAAe,EAAE,MAAM,CAC9D,YAAW,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,eAAe,CAAC;IAEpE,OAAO,CAAC,gBAAgB,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAwB;IACnD,OAAO,CAAC,kBAAkB,CAAe;IACzC,OAAO,CAAC,oBAAoB,CAA2B;IAEvD;;;;;;;;;OASG;IACH,YACE,EAAE,EAAE,IAAI,EACR,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EACd,eAAe,EAAE,eAAe,EAChC,OAAO,EAAE,SAAS,CAAC,MAAM,OAAO,CAAC,MAAM,CAAC,GAAG,SAAS,CAAC,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,YAAK,EACrF,UAAU,EAAE,MAAM,EAClB,UAAU,GAAE,qBAAoC,EAcjD;IAEe,mBAAmB,IAAI,MAAM,CAE5C;IAEY,gBAAgB,CAC3B,KAAK,EAAE,UAAU,EACjB,OAAO,GAAE,mBAAmB,CAAC,QAAQ,CAAM,GAC1C,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG;QAAE,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC,CAwE5C;IAEK,YAAY,CAAC,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,mBAAmB,CAAC,QAAQ,CAAC;eA1EzC,MAAM;UA8JxC;YAKa,cAAc;YA+Bd,oBAAoB;IA8ClC,OAAO,CAAC,wBAAwB;IAKhC,OAAO,CAAC,mBAAmB;IAI3B,OAAO,CAAC,aAAa;CAQtB"}
|
|
@@ -1,100 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @license
|
|
3
|
-
* Copyright 2025 Steven Roussey <sroussey@gmail.com>
|
|
4
|
-
* SPDX-License-Identifier: Apache-2.0
|
|
5
|
-
*/
|
|
6
|
-
import { Sqlite } from "@workglow/storage/sqlite";
|
|
7
|
-
import type { DataPortSchemaObject, FromSchema, TypedArray, TypedArrayConstructor, TypedArraySchemaOptions } from "@workglow/util/schema";
|
|
8
|
-
import { SqliteTabularStorage } from "../tabular/SqliteTabularStorage";
|
|
9
|
-
import type { HybridSearchOptions, IVectorStorage, VectorSearchOptions } from "./IVectorStorage";
|
|
10
|
-
/**
|
|
11
|
-
* SQLite vector storage implementation using the @sqliteai/sqlite-vector extension.
|
|
12
|
-
* Provides native vector similarity search via SQLite virtual table functions
|
|
13
|
-
* instead of in-memory brute-force search.
|
|
14
|
-
*
|
|
15
|
-
* Requirements:
|
|
16
|
-
* - @sqliteai/sqlite-vector package installed
|
|
17
|
-
* - Extension loaded via db.loadExtension(getExtensionPath())
|
|
18
|
-
*
|
|
19
|
-
* Vectors are stored as BLOBs using sqlite-vector encoding functions (vector_as_f32, etc.)
|
|
20
|
-
* and searched using vector_full_scan for efficient KNN queries.
|
|
21
|
-
*
|
|
22
|
-
* @template Schema - The schema for the vector storage
|
|
23
|
-
* @template PrimaryKeyNames - The primary key names
|
|
24
|
-
* @template VectorCtor - Constructor for stored vectors (default {@link typeof Float32Array})
|
|
25
|
-
* @template Metadata - The metadata type
|
|
26
|
-
*/
|
|
27
|
-
export declare class SqliteAiVectorStorage<Schema extends DataPortSchemaObject, PrimaryKeyNames extends ReadonlyArray<keyof Schema["properties"]>, Metadata extends Record<string, unknown> | undefined = Record<string, unknown>, Entity = FromSchema<Schema, TypedArraySchemaOptions>> extends SqliteTabularStorage<Schema, PrimaryKeyNames, Entity> implements IVectorStorage<Metadata, Schema, Entity, PrimaryKeyNames> {
|
|
28
|
-
private vectorDimensions;
|
|
29
|
-
private readonly vectorCtor;
|
|
30
|
-
private vectorPropertyName;
|
|
31
|
-
private metadataPropertyName;
|
|
32
|
-
private vectorTypeSuffix;
|
|
33
|
-
private extensionLoaded;
|
|
34
|
-
/**
|
|
35
|
-
* Creates a new SQLite AI vector storage
|
|
36
|
-
* @param dbOrPath - Either a Database instance or a path to the SQLite database file
|
|
37
|
-
* @param table - The name of the table to use for storage
|
|
38
|
-
* @param schema - The schema for the entity
|
|
39
|
-
* @param primaryKeyNames - Array of property names forming the primary key
|
|
40
|
-
* @param indexes - Array of columns to index
|
|
41
|
-
* @param dimensions - The number of dimensions of the vector
|
|
42
|
-
* @param vectorCtor - TypedArray constructor for stored vectors (e.g. {@link Float32Array})
|
|
43
|
-
*/
|
|
44
|
-
constructor(dbOrPath: string | Sqlite.Database, table: string | undefined, schema: Schema, primaryKeyNames: PrimaryKeyNames, indexes: readonly (keyof NoInfer<Entity> | readonly (keyof NoInfer<Entity>)[])[] | undefined, dimensions: number, vectorCtor?: TypedArrayConstructor);
|
|
45
|
-
getVectorDimensions(): number;
|
|
46
|
-
/**
|
|
47
|
-
* Load the sqlite-vector extension and initialize vector indexing on the vector column.
|
|
48
|
-
* Extension loading is best-effort: if unavailable, operations fall back to in-memory search.
|
|
49
|
-
*/
|
|
50
|
-
setupDatabase(): Promise<void>;
|
|
51
|
-
/**
|
|
52
|
-
* Encode a vector as a BLOB using sqlite-vector functions.
|
|
53
|
-
* Returns a JSON string representation suitable for vector_as_f32() etc.
|
|
54
|
-
*/
|
|
55
|
-
private encodeVectorJson;
|
|
56
|
-
/**
|
|
57
|
-
* Decode a vector BLOB from SQLite back to a TypedArray.
|
|
58
|
-
* sqlite-vector stores vectors as BLOBs, but when we SELECT them
|
|
59
|
-
* they come back as Buffer/Uint8Array. We also handle JSON string fallback.
|
|
60
|
-
*/
|
|
61
|
-
private decodeVector;
|
|
62
|
-
/**
|
|
63
|
-
* Override jsToSqlValue to encode vectors as BLOBs via sqlite-vector functions
|
|
64
|
-
*/
|
|
65
|
-
protected jsToSqlValue(column: string, value: Entity[keyof Entity]): ReturnType<SqliteTabularStorage<Schema, PrimaryKeyNames, Entity>["jsToSqlValue"]>;
|
|
66
|
-
/**
|
|
67
|
-
* Override sqlToJsValue to decode vector BLOBs back to TypedArrays
|
|
68
|
-
*/
|
|
69
|
-
protected sqlToJsValue(column: string, value: any): Entity[keyof Entity];
|
|
70
|
-
/**
|
|
71
|
-
* Override mapTypeToSQL to use BLOB for vector columns instead of TEXT
|
|
72
|
-
*/
|
|
73
|
-
protected mapTypeToSQL(typeDef: any): string;
|
|
74
|
-
/**
|
|
75
|
-
* Override put to use sqlite-vector encoding for vector data.
|
|
76
|
-
* Builds a custom INSERT OR REPLACE that wraps the vector column
|
|
77
|
-
* with vector_as_fXX() to encode as a native vector BLOB.
|
|
78
|
-
* Falls back to base class put() if the extension is not available.
|
|
79
|
-
*/
|
|
80
|
-
put(entity: any): Promise<Entity>;
|
|
81
|
-
/**
|
|
82
|
-
* Perform similarity search using sqlite-vector's vector_full_scan.
|
|
83
|
-
* Uses native COSINE distance computation in SQLite rather than in-memory JS.
|
|
84
|
-
* Falls back to in-memory search if the extension is unavailable.
|
|
85
|
-
*/
|
|
86
|
-
similaritySearch(query: TypedArray, options?: VectorSearchOptions<Metadata>): Promise<(Entity & {
|
|
87
|
-
score: number;
|
|
88
|
-
})[]>;
|
|
89
|
-
/**
|
|
90
|
-
* Hybrid search combining vector similarity with text relevance.
|
|
91
|
-
* Uses sqlite-vector for the vector component and keyword matching for text.
|
|
92
|
-
* Falls back to in-memory search if the extension is unavailable.
|
|
93
|
-
*/
|
|
94
|
-
hybridSearch(query: TypedArray, options: HybridSearchOptions<Metadata>): Promise<(Entity & {
|
|
95
|
-
score: number;
|
|
96
|
-
})[]>;
|
|
97
|
-
private searchFallback;
|
|
98
|
-
private hybridSearchFallback;
|
|
99
|
-
}
|
|
100
|
-
//# sourceMappingURL=SqliteAiVectorStorage.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"SqliteAiVectorStorage.d.ts","sourceRoot":"","sources":["../../src/vector/SqliteAiVectorStorage.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAClD,OAAO,KAAK,EACV,oBAAoB,EACpB,UAAU,EACV,UAAU,EACV,qBAAqB,EACrB,uBAAuB,EACxB,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAAE,oBAAoB,EAAE,MAAM,iCAAiC,CAAC;AACvE,OAAO,KAAK,EAAE,mBAAmB,EAAE,cAAc,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AAwDjG;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,qBAAqB,CAChC,MAAM,SAAS,oBAAoB,EACnC,eAAe,SAAS,aAAa,CAAC,MAAM,MAAM,CAAC,YAAY,CAAC,CAAC,EACjE,QAAQ,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC9E,MAAM,GAAG,UAAU,CAAC,MAAM,EAAE,uBAAuB,CAAC,CAEpD,SAAQ,oBAAoB,CAAC,MAAM,EAAE,eAAe,EAAE,MAAM,CAC5D,YAAW,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,eAAe,CAAC;IAEpE,OAAO,CAAC,gBAAgB,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAwB;IACnD,OAAO,CAAC,kBAAkB,CAAe;IACzC,OAAO,CAAC,oBAAoB,CAA2B;IACvD,OAAO,CAAC,gBAAgB,CAAS;IACjC,OAAO,CAAC,eAAe,CAAkB;IAEzC;;;;;;;;;OASG;IACH,YACE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAAC,QAAQ,EAClC,KAAK,EAAE,MAAM,YAAY,EACzB,MAAM,EAAE,MAAM,EACd,eAAe,EAAE,eAAe,EAChC,OAAO,EAAE,SAAS,CAAC,MAAM,OAAO,CAAC,MAAM,CAAC,GAAG,SAAS,CAAC,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,YAAK,EACrF,UAAU,EAAE,MAAM,EAClB,UAAU,GAAE,qBAAoC,EAejD;IAED,mBAAmB,IAAI,MAAM,CAE5B;IAED;;;OAGG;IACmB,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC,CAsCnD;IAED;;;OAGG;IACH,OAAO,CAAC,gBAAgB;IAIxB;;;;OAIG;IACH,OAAO,CAAC,YAAY;IA8BpB;;OAEG;IACH,UAAmB,YAAY,CAC7B,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,CAAC,MAAM,MAAM,CAAC,GAC1B,UAAU,CAAC,oBAAoB,CAAC,MAAM,EAAE,eAAe,EAAE,MAAM,CAAC,CAAC,cAAc,CAAC,CAAC,CAOnF;IAED;;OAEG;IACH,UAAmB,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,GAAG,MAAM,CAAC,MAAM,MAAM,CAAC,CAKhF;IAED;;OAEG;IACH,UAAmB,YAAY,CAAC,OAAO,EAAE,GAAG,GAAG,MAAM,CAQpD;IAED;;;;;OAKG;IACmB,GAAG,CAAC,MAAM,EAAE,GAAG,GAAG,OAAO,CAAC,MAAM,CAAC,CAwGtD;IAED;;;;OAIG;IACG,gBAAgB,CAAC,KAAK,EAAE,UAAU,EAAE,OAAO,GAAE,mBAAmB,CAAC,QAAQ,CAAM;eA+BxC,MAAM;UAqElD;IAED;;;;OAIG;IACG,YAAY,CAAC,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,mBAAmB,CAAC,QAAQ,CAAC;eA5E/B,MAAM;UAiKlD;YAKa,cAAc;YA6Bd,oBAAoB;CA2CnC"}
|
|
@@ -1,49 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @license
|
|
3
|
-
* Copyright 2025 Steven Roussey <sroussey@gmail.com>
|
|
4
|
-
* SPDX-License-Identifier: Apache-2.0
|
|
5
|
-
*/
|
|
6
|
-
import { Sqlite } from "@workglow/storage/sqlite";
|
|
7
|
-
import type { DataPortSchemaObject, FromSchema, TypedArray, TypedArrayConstructor, TypedArraySchemaOptions } from "@workglow/util/schema";
|
|
8
|
-
import { SqliteTabularStorage } from "../tabular/SqliteTabularStorage";
|
|
9
|
-
import type { HybridSearchOptions, IVectorStorage, VectorSearchOptions } from "./IVectorStorage";
|
|
10
|
-
/**
|
|
11
|
-
* SQLite vector repository implementation using tabular storage underneath.
|
|
12
|
-
* Stores vectors as JSON-encoded arrays with metadata.
|
|
13
|
-
*
|
|
14
|
-
* @template Schema - The schema for the vector
|
|
15
|
-
* @template PrimaryKeyNames - The primary key names for the vector
|
|
16
|
-
* @template VectorCtor - Constructor for stored vectors (default {@link typeof Float32Array})
|
|
17
|
-
* @template Metadata - The metadata type for the vector
|
|
18
|
-
* @template Entity - The entity type for the vector
|
|
19
|
-
*/
|
|
20
|
-
export declare class SqliteVectorStorage<Schema extends DataPortSchemaObject, PrimaryKeyNames extends ReadonlyArray<keyof Schema["properties"]>, Metadata extends Record<string, unknown> | undefined = Record<string, unknown>, Entity = FromSchema<Schema, TypedArraySchemaOptions>> extends SqliteTabularStorage<Schema, PrimaryKeyNames, Entity> implements IVectorStorage<Metadata, Schema, Entity, PrimaryKeyNames> {
|
|
21
|
-
private vectorDimensions;
|
|
22
|
-
private readonly vectorCtor;
|
|
23
|
-
private vectorPropertyName;
|
|
24
|
-
private metadataPropertyName;
|
|
25
|
-
/**
|
|
26
|
-
* Creates a new SQLite vector repository
|
|
27
|
-
* @param dbOrPath - Either a Database instance or a path to the SQLite database file
|
|
28
|
-
* @param table - The name of the table to use for storage (defaults to 'vectors')
|
|
29
|
-
* @param schema - The schema for the entity
|
|
30
|
-
* @param primaryKeyNames - Array of property names forming the primary key
|
|
31
|
-
* @param indexes - Array of columns to index
|
|
32
|
-
* @param dimensions - The number of dimensions of the vector
|
|
33
|
-
* @param vectorCtor - TypedArray constructor used when deserializing JSON vectors (e.g. {@link Float32Array})
|
|
34
|
-
*/
|
|
35
|
-
constructor(dbOrPath: string | Sqlite.Database, table: string | undefined, schema: Schema, primaryKeyNames: PrimaryKeyNames, indexes: readonly (keyof NoInfer<Entity> | readonly (keyof NoInfer<Entity>)[])[] | undefined, dimensions: number, vectorCtor?: TypedArrayConstructor);
|
|
36
|
-
getVectorDimensions(): number;
|
|
37
|
-
/**
|
|
38
|
-
* Deserialize vector from JSON string
|
|
39
|
-
* Defaults to Float32Array for compatibility with typical embedding vectors
|
|
40
|
-
*/
|
|
41
|
-
private deserializeVector;
|
|
42
|
-
similaritySearch(query: TypedArray, options?: VectorSearchOptions<Metadata>): Promise<(Entity & {
|
|
43
|
-
score: number;
|
|
44
|
-
})[]>;
|
|
45
|
-
hybridSearch(query: TypedArray, options: HybridSearchOptions<Metadata>): Promise<(Entity & {
|
|
46
|
-
score: number;
|
|
47
|
-
})[]>;
|
|
48
|
-
}
|
|
49
|
-
//# sourceMappingURL=SqliteVectorStorage.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"SqliteVectorStorage.d.ts","sourceRoot":"","sources":["../../src/vector/SqliteVectorStorage.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAClD,OAAO,KAAK,EACV,oBAAoB,EACpB,UAAU,EACV,UAAU,EACV,qBAAqB,EACrB,uBAAuB,EACxB,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAAE,oBAAoB,EAAE,MAAM,iCAAiC,CAAC;AACvE,OAAO,KAAK,EAAE,mBAAmB,EAAE,cAAc,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AAejG;;;;;;;;;GASG;AACH,qBAAa,mBAAmB,CAC9B,MAAM,SAAS,oBAAoB,EACnC,eAAe,SAAS,aAAa,CAAC,MAAM,MAAM,CAAC,YAAY,CAAC,CAAC,EACjE,QAAQ,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC9E,MAAM,GAAG,UAAU,CAAC,MAAM,EAAE,uBAAuB,CAAC,CAEpD,SAAQ,oBAAoB,CAAC,MAAM,EAAE,eAAe,EAAE,MAAM,CAC5D,YAAW,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,eAAe,CAAC;IAEpE,OAAO,CAAC,gBAAgB,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAwB;IACnD,OAAO,CAAC,kBAAkB,CAAe;IACzC,OAAO,CAAC,oBAAoB,CAA2B;IAEvD;;;;;;;;;OASG;IACH,YACE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAAC,QAAQ,EAClC,KAAK,EAAE,MAAM,YAAY,EACzB,MAAM,EAAE,MAAM,EACd,eAAe,EAAE,eAAe,EAChC,OAAO,EAAE,SAAS,CAAC,MAAM,OAAO,CAAC,MAAM,CAAC,GAAG,SAAS,CAAC,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,YAAK,EACrF,UAAU,EAAE,MAAM,EAClB,UAAU,GAAE,qBAAoC,EAcjD;IAED,mBAAmB,IAAI,MAAM,CAE5B;IAED;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAMnB,gBAAgB,CAAC,KAAK,EAAE,UAAU,EAAE,OAAO,GAAE,mBAAmB,CAAC,QAAQ,CAAM;eAE5C,MAAM;UAoC9C;IAEK,YAAY,CAAC,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,mBAAmB,CAAC,QAAQ,CAAC;eAtCnC,MAAM;UAqG9C;CACF"}
|
package/src/queue/README.md
DELETED
|
@@ -1,41 +0,0 @@
|
|
|
1
|
-
# ELLMERS Queue Storage Package
|
|
2
|
-
|
|
3
|
-
This is not a job queue implementation. It is a storage implementation for job queues. See the [@workglow/job-queue](../../../job-queue/README.md) package for a job queue implementation that uses these storage implementations.
|
|
4
|
-
|
|
5
|
-
- [Features](#features)
|
|
6
|
-
- [Performance Considerations](#performance-considerations)
|
|
7
|
-
- [Job Lifecycle](#job-lifecycle)
|
|
8
|
-
|
|
9
|
-
## Features
|
|
10
|
-
|
|
11
|
-
- Multiple storage implementations:
|
|
12
|
-
- `InMemoryQueueStorage` - Volatile memory (dev/testing)
|
|
13
|
-
- `IndexedDbQueueStorage` - Browser-based storage
|
|
14
|
-
- `SqliteQueueStorage` - Embedded SQLite (pass a `Sqlite.Database` from `@workglow/storage/sqlite`; call `await Sqlite.init()` before opening the DB)
|
|
15
|
-
- `PostgresQueueStorage` - Production-grade PostgreSQL
|
|
16
|
-
- Job lifecycle management:
|
|
17
|
-
- PENDING → PROCESSING → COMPLETED/FAILED/ABORTED
|
|
18
|
-
- PENDING → DISABLED
|
|
19
|
-
- Automatic retry mechanisms
|
|
20
|
-
- Progress tracking with message/details
|
|
21
|
-
- Fingerprint-based input deduplication
|
|
22
|
-
- Transactional operations with SKIP LOCKED
|
|
23
|
-
- Job expiration policies
|
|
24
|
-
|
|
25
|
-
## Performance Considerations
|
|
26
|
-
|
|
27
|
-
1. **IndexedDB**: Best for client-side applications with <10k jobs
|
|
28
|
-
2. **SQLite**: Ideal for single-process applications
|
|
29
|
-
3. **PostgreSQL**: Recommended for distributed systems with high throughput
|
|
30
|
-
4. **In-Memory**: Suitable for testing/development only
|
|
31
|
-
|
|
32
|
-
## Job Lifecycle
|
|
33
|
-
|
|
34
|
-
1. Jobs start as `PENDING`
|
|
35
|
-
2. Acquired via `next()` → `PROCESSING`
|
|
36
|
-
3. Final states:
|
|
37
|
-
- `COMPLETED`: Successful execution
|
|
38
|
-
- `FAILED`: Unrecoverable error
|
|
39
|
-
- `ABORTED`: Manual cancellation
|
|
40
|
-
- `DISABLED`: Disabled due to conditions not met
|
|
41
|
-
- Auto-retried while `PENDING` if within retry limits
|