@alineo-labs/postgres-memory 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +61 -0
- package/dist/index.d.mts +92 -0
- package/dist/index.mjs +371 -0
- package/package.json +38 -0
package/README.md
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
# @alineo-labs/postgres-memory
|
|
2
|
+
|
|
3
|
+
Shared, multi-process `@alineo-labs/memory` backend for [alineo](https://alineo.tech), via Postgres.
|
|
4
|
+
|
|
5
|
+
```bash
|
|
6
|
+
bun add @alineo-labs/postgres-memory
|
|
7
|
+
```
|
|
8
|
+
|
|
9
|
+
For local dev with zero infrastructure, use [`@alineo-labs/sqlite-memory`](https://github.com/DrejT/alineo/tree/main/packages/adapters/sqlite-memory) instead.
|
|
10
|
+
|
|
11
|
+
---
|
|
12
|
+
|
|
13
|
+
## Usage
|
|
14
|
+
|
|
15
|
+
```ts
|
|
16
|
+
import { Memory } from "@alineo-labs/memory";
|
|
17
|
+
import {
|
|
18
|
+
PostgresWorkingMemoryProvider,
|
|
19
|
+
PostgresSemanticMemoryProvider,
|
|
20
|
+
} from "@alineo-labs/postgres-memory";
|
|
21
|
+
|
|
22
|
+
const memory = new Memory({
|
|
23
|
+
workingMemory: new PostgresWorkingMemoryProvider(connectionString),
|
|
24
|
+
semantic: new PostgresSemanticMemoryProvider(connectionString, myEmbeddingProvider),
|
|
25
|
+
});
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
Migrations run lazily on first query — no separate `connect()` step.
|
|
29
|
+
|
|
30
|
+
## Team isolation
|
|
31
|
+
|
|
32
|
+
Rows carrying a `ResourceRef.teamId` are isolated via row-level security: every query runs
|
|
33
|
+
inside a transaction that first sets `app.team_id` for that transaction, and each table's RLS
|
|
34
|
+
policy only allows access to rows where `team_id` is `NULL` or matches. `FORCE ROW LEVEL
|
|
35
|
+
SECURITY` is set so this applies even to the role that owns the tables. See `src/migrations.ts`
|
|
36
|
+
for the exact policy.
|
|
37
|
+
|
|
38
|
+
## Vector search
|
|
39
|
+
|
|
40
|
+
`recall()` uses a real `pgvector` HNSW index whenever the connected Postgres instance allows
|
|
41
|
+
`CREATE EXTENSION IF NOT EXISTS vector` (on by default on Supabase/Neon, allowlisted on RDS,
|
|
42
|
+
unavailable on a bare Postgres with no superuser access) — falling back to an in-JS
|
|
43
|
+
cosine-similarity scan over every row for the resource otherwise. Check
|
|
44
|
+
`PostgresSemanticMemoryProvider.hasVectorIndex` to see which path is active; the plain `vector`
|
|
45
|
+
(`double precision[]`) column on `alineo_semantic_memory` is always populated regardless, so the
|
|
46
|
+
fallback is never a degraded schema, only a slower code path over the same data. The indexed
|
|
47
|
+
path's own table (`alineo_semantic_vec`, created lazily once the embedding dimension is known)
|
|
48
|
+
gets identical row-level-security policies to the primary table, so scoping doesn't depend on
|
|
49
|
+
which path a given call takes.
|
|
50
|
+
|
|
51
|
+
This package has not been run against a live Postgres instance in this repo's own test suite
|
|
52
|
+
(none is available here) — it ships type-checked, matching `@alineo-labs/postgres`'s own
|
|
53
|
+
testing posture, not integration-tested against a real server. That applies to this HNSW path
|
|
54
|
+
too: it's implemented against `pgvector`'s documented SQL surface, not verified against a live
|
|
55
|
+
instance with the extension installed.
|
|
56
|
+
|
|
57
|
+
---
|
|
58
|
+
|
|
59
|
+
## License
|
|
60
|
+
|
|
61
|
+
Apache 2.0
|
package/dist/index.d.mts
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import { EmbeddingProvider, IBulkSemanticMemoryProvider, IPrunableSemanticMemoryProvider, IWorkingMemoryProvider, MemoryFact, RememberedFact, ResourceRef } from "@alineo-labs/memory";
|
|
2
|
+
|
|
3
|
+
//#region src/working.d.ts
|
|
4
|
+
/**
|
|
5
|
+
* Persisted, multi-process-safe `IWorkingMemoryProvider` backed by Postgres — the shared,
|
|
6
|
+
* production-shaped counterpart to `@alineo-labs/sqlite-memory`'s file-based provider. Rows
|
|
7
|
+
* are additionally isolated by `ResourceRef.teamId` via row-level security (see
|
|
8
|
+
* `migrations.ts`) whenever a team is set.
|
|
9
|
+
*
|
|
10
|
+
* Written against the `postgres` package the same way `@alineo-labs/postgres`'s ledger
|
|
11
|
+
* adapter is, but — like that adapter — has no live database to run against in this repo's
|
|
12
|
+
* test environment, so this ships type-checked, not integration-tested against a real
|
|
13
|
+
* Postgres instance.
|
|
14
|
+
*/
|
|
15
|
+
declare class PostgresWorkingMemoryProvider implements IWorkingMemoryProvider {
|
|
16
|
+
private readonly conn;
|
|
17
|
+
constructor(connectionString: string);
|
|
18
|
+
get(ref: ResourceRef, key: string): Promise<unknown | undefined>;
|
|
19
|
+
set(ref: ResourceRef, key: string, value: unknown): Promise<void>;
|
|
20
|
+
list(ref: ResourceRef): Promise<Record<string, unknown>>;
|
|
21
|
+
delete(ref: ResourceRef, key: string): Promise<void>;
|
|
22
|
+
/** Release the underlying connection pool. */
|
|
23
|
+
close(): Promise<void>;
|
|
24
|
+
}
|
|
25
|
+
//#endregion
|
|
26
|
+
//#region src/semantic.d.ts
|
|
27
|
+
/**
|
|
28
|
+
* Persisted, multi-process-safe `ISemanticMemoryProvider` (+ pruning) backed by Postgres.
|
|
29
|
+
*
|
|
30
|
+
* Ranks `recall()` with a real `pgvector` HNSW index whenever the `vector` extension can be
|
|
31
|
+
* installed on the connected Postgres instance (`CREATE EXTENSION IF NOT EXISTS vector` —
|
|
32
|
+
* enabled by default on Supabase/Neon/RDS-with-the-extension-allowlisted; unavailable on a
|
|
33
|
+
* bare Postgres with no superuser access), falling back to the same in-JS cosine-similarity
|
|
34
|
+
* scan `InMemorySemanticMemoryProvider`/`SQLiteSemanticMemoryProvider`'s no-index path uses
|
|
35
|
+
* otherwise. `hasVectorIndex` reports which path is active — mirrors
|
|
36
|
+
* `SQLiteSemanticMemoryProvider`'s own `hasVectorIndex`/`vecAvailable` split exactly.
|
|
37
|
+
*
|
|
38
|
+
* The plain `vector` (`double precision[]`) column on `alineo_semantic_memory` is always
|
|
39
|
+
* populated regardless of which path is active, so the fallback is never a degraded schema —
|
|
40
|
+
* it's a genuinely lower-performance code path over the same data, same as the SQLite adapter.
|
|
41
|
+
* The indexed path additionally maintains a separate `alineo_semantic_vec` table (id, scope,
|
|
42
|
+
* team_id, a real `vector(N)` column + HNSW index), created lazily once the embedding
|
|
43
|
+
* dimension is known from the first fact remembered — an `ON DELETE CASCADE` foreign key back
|
|
44
|
+
* to `alineo_semantic_memory.id` keeps it in sync with `forget()` for free, no extra delete
|
|
45
|
+
* needed there. Both tables get identical row-level-security policies, so scoping doesn't
|
|
46
|
+
* depend on which path a given `recall()` call takes.
|
|
47
|
+
*
|
|
48
|
+
* Unlike `SQLiteSemanticMemoryProvider`'s `vec0` virtual table (which needs `scope` declared
|
|
49
|
+
* as a native *partition key* so vec0's own KNN traversal applies it before counting `k`
|
|
50
|
+
* neighbors — see that file's `ensureVecTable` comment), a plain `WHERE v.scope = ...` alongside
|
|
51
|
+
* `ORDER BY ... LIMIT k` is sufficient and correct here: Postgres's planner evaluates the filter
|
|
52
|
+
* as part of the same query regardless of whether it uses the HNSW index or falls back to a
|
|
53
|
+
* sequential scan, so a resource never receives fewer than `k` matching rows the way an
|
|
54
|
+
* outer-join-after-KNN approach could.
|
|
55
|
+
*
|
|
56
|
+
* Mixing embedding models with different output dimensions on one instance throws a real
|
|
57
|
+
* Postgres dimension-mismatch error at insert time once `alineo_semantic_vec` is sized to the
|
|
58
|
+
* first dimension seen — same documented behavior as the SQLite adapter, not silently wrong.
|
|
59
|
+
*
|
|
60
|
+
* Like `PostgresWorkingMemoryProvider`, this ships type-checked against no live database.
|
|
61
|
+
*/
|
|
62
|
+
declare class PostgresSemanticMemoryProvider implements IPrunableSemanticMemoryProvider, IBulkSemanticMemoryProvider {
|
|
63
|
+
private readonly embeddings;
|
|
64
|
+
private readonly conn;
|
|
65
|
+
private vecDimensions;
|
|
66
|
+
private ensureVecTablePromise;
|
|
67
|
+
constructor(connectionString: string, embeddings: EmbeddingProvider);
|
|
68
|
+
/** Whether `recall()` is using the native `pgvector` HNSW index (true) or the in-JS cosine
|
|
69
|
+
* fallback scan (false, either because the extension isn't available on this Postgres
|
|
70
|
+
* instance or no fact has been remembered yet to size the index from). */
|
|
71
|
+
get hasVectorIndex(): boolean;
|
|
72
|
+
/** Lazily creates `alineo_semantic_vec`, sized to `dimensions` — memoized, and (like
|
|
73
|
+
* `PostgresMemoryConnection.ensureMigrated()`) cleared on rejection so a transient failure
|
|
74
|
+
* here doesn't permanently pin this instance to the fallback path the way a permanent one
|
|
75
|
+
* reasonably should. */
|
|
76
|
+
private ensureVecTable;
|
|
77
|
+
remember(ref: ResourceRef, fact: MemoryFact): Promise<void>;
|
|
78
|
+
/** Batches the embedding call for all `facts` into one `embed()` invocation, and runs every
|
|
79
|
+
* insert (both tables, when the vec index is active) inside the single transaction
|
|
80
|
+
* `withTeamContext` already opens, instead of one transaction per fact — see
|
|
81
|
+
* `IBulkSemanticMemoryProvider`. */
|
|
82
|
+
rememberMany(ref: ResourceRef, facts: MemoryFact[]): Promise<void>;
|
|
83
|
+
recall(ref: ResourceRef, query: string, opts?: {
|
|
84
|
+
topK?: number;
|
|
85
|
+
}): Promise<MemoryFact[]>;
|
|
86
|
+
listAll(ref: ResourceRef): Promise<RememberedFact[]>;
|
|
87
|
+
forget(ref: ResourceRef, ids: string[]): Promise<number>;
|
|
88
|
+
/** Release the underlying connection pool. */
|
|
89
|
+
close(): Promise<void>;
|
|
90
|
+
}
|
|
91
|
+
//#endregion
|
|
92
|
+
export { PostgresSemanticMemoryProvider, PostgresWorkingMemoryProvider };
|
package/dist/index.mjs
ADDED
|
@@ -0,0 +1,371 @@
|
|
|
1
|
+
import { cosineSimilarity, factFromRow, scopeKey } from "@alineo-labs/memory";
|
|
2
|
+
import postgres from "postgres";
|
|
3
|
+
//#region src/migrations.ts
|
|
4
|
+
/**
|
|
5
|
+
* `team_id` is a plain nullable column, not the primary scoping key (`scope`, same
|
|
6
|
+
* resourceId[:teamId] composite every provider in this package family uses) — it exists
|
|
7
|
+
* purely so the row-level security policies below have something to check. A row with
|
|
8
|
+
* `team_id IS NULL` (an untenanted resource) is visible to every session; a row with a
|
|
9
|
+
* `team_id` is visible only when the connection has `SET LOCAL app.team_id` to a matching
|
|
10
|
+
* value first — which `PostgresWorkingMemoryProvider`/`PostgresSemanticMemoryProvider` do
|
|
11
|
+
* automatically, per call, via a wrapping transaction (see `withTeamContext` in `shared.ts`).
|
|
12
|
+
*
|
|
13
|
+
* `FORCE ROW LEVEL SECURITY` matters here specifically because the same role that runs these
|
|
14
|
+
* migrations (typically the table owner) is also the role the app connects as — Postgres
|
|
15
|
+
* exempts the table owner from RLS by default, which would make the policy a no-op for the
|
|
16
|
+
* exact role most likely to be running application queries. `FORCE` closes that gap.
|
|
17
|
+
*
|
|
18
|
+
* `CREATE POLICY` has no `IF NOT EXISTS` in any Postgres version — the `DO $$ ... EXCEPTION
|
|
19
|
+
* WHEN duplicate_object` wrapper is the standard idiom for making it re-runnable, matching
|
|
20
|
+
* this migration file's `CREATE TABLE IF NOT EXISTS` idempotency everywhere else.
|
|
21
|
+
*/
|
|
22
|
+
const MIGRATION_SQL = `
|
|
23
|
+
CREATE TABLE IF NOT EXISTS alineo_working_memory (
|
|
24
|
+
scope TEXT NOT NULL,
|
|
25
|
+
team_id TEXT,
|
|
26
|
+
key TEXT NOT NULL,
|
|
27
|
+
value JSONB NOT NULL,
|
|
28
|
+
PRIMARY KEY (scope, key)
|
|
29
|
+
);
|
|
30
|
+
|
|
31
|
+
ALTER TABLE alineo_working_memory ENABLE ROW LEVEL SECURITY;
|
|
32
|
+
ALTER TABLE alineo_working_memory FORCE ROW LEVEL SECURITY;
|
|
33
|
+
|
|
34
|
+
DO $$ BEGIN
|
|
35
|
+
CREATE POLICY alineo_working_memory_team_isolation ON alineo_working_memory
|
|
36
|
+
FOR ALL
|
|
37
|
+
USING (team_id IS NULL OR team_id = current_setting('app.team_id', true))
|
|
38
|
+
WITH CHECK (team_id IS NULL OR team_id = current_setting('app.team_id', true));
|
|
39
|
+
EXCEPTION WHEN duplicate_object THEN NULL;
|
|
40
|
+
END $$;
|
|
41
|
+
|
|
42
|
+
CREATE TABLE IF NOT EXISTS alineo_semantic_memory (
|
|
43
|
+
id TEXT PRIMARY KEY,
|
|
44
|
+
scope TEXT NOT NULL,
|
|
45
|
+
team_id TEXT,
|
|
46
|
+
content TEXT NOT NULL,
|
|
47
|
+
vector DOUBLE PRECISION[] NOT NULL,
|
|
48
|
+
source_sandbox_id TEXT,
|
|
49
|
+
source_entry_index INTEGER,
|
|
50
|
+
remembered_at BIGINT NOT NULL
|
|
51
|
+
);
|
|
52
|
+
|
|
53
|
+
CREATE INDEX IF NOT EXISTS alineo_semantic_memory_scope ON alineo_semantic_memory(scope);
|
|
54
|
+
|
|
55
|
+
ALTER TABLE alineo_semantic_memory ENABLE ROW LEVEL SECURITY;
|
|
56
|
+
ALTER TABLE alineo_semantic_memory FORCE ROW LEVEL SECURITY;
|
|
57
|
+
|
|
58
|
+
DO $$ BEGIN
|
|
59
|
+
CREATE POLICY alineo_semantic_memory_team_isolation ON alineo_semantic_memory
|
|
60
|
+
FOR ALL
|
|
61
|
+
USING (team_id IS NULL OR team_id = current_setting('app.team_id', true))
|
|
62
|
+
WITH CHECK (team_id IS NULL OR team_id = current_setting('app.team_id', true));
|
|
63
|
+
EXCEPTION WHEN duplicate_object THEN NULL;
|
|
64
|
+
END $$;
|
|
65
|
+
`;
|
|
66
|
+
//#endregion
|
|
67
|
+
//#region src/shared.ts
|
|
68
|
+
/**
|
|
69
|
+
* Lazily runs migrations on first use, memoized — same pattern `Sandbox._ensureConnected()`
|
|
70
|
+
* uses in `@alineo-labs/sandbox` — so callers don't need to remember an explicit `connect()`
|
|
71
|
+
* step before their first query.
|
|
72
|
+
*/
|
|
73
|
+
var PostgresMemoryConnection = class {
|
|
74
|
+
sql;
|
|
75
|
+
migratePromise = null;
|
|
76
|
+
pgvectorPromise = null;
|
|
77
|
+
constructor(connectionString) {
|
|
78
|
+
this.sql = postgres(connectionString);
|
|
79
|
+
}
|
|
80
|
+
ensureMigrated() {
|
|
81
|
+
this.migratePromise ??= this.sql.unsafe(MIGRATION_SQL).then(() => void 0, (err) => {
|
|
82
|
+
this.migratePromise = null;
|
|
83
|
+
throw err;
|
|
84
|
+
});
|
|
85
|
+
return this.migratePromise;
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Best-effort `CREATE EXTENSION IF NOT EXISTS vector`, memoized. Deliberately kept separate
|
|
89
|
+
* from `ensureMigrated()` above: the base schema (working memory, semantic memory, RLS) must
|
|
90
|
+
* succeed on any Postgres instance, including managed ones where the connecting role isn't
|
|
91
|
+
* allowed to create extensions at all or where `pgvector` isn't installed — this call is
|
|
92
|
+
* allowed to fail without taking the rest of the package down with it.
|
|
93
|
+
*
|
|
94
|
+
* Unlike `ensureMigrated()`, this never rejects and is memoized permanently once resolved —
|
|
95
|
+
* a `false` result (extension missing, or no privilege) means "use the in-JS cosine-scan
|
|
96
|
+
* fallback for the rest of this process," which is a correctness-preserving, merely-slower
|
|
97
|
+
* degradation, not a broken instance. That also means a *transient* failure here (e.g. a
|
|
98
|
+
* connection blip on the very first call) is indistinguishable from a permanent one and
|
|
99
|
+
* pins this connection to the fallback path for its whole lifetime — an accepted trade-off,
|
|
100
|
+
* not an oversight: retrying this on every call would mean every `remember()`/`recall()`
|
|
101
|
+
* pays for a fresh `CREATE EXTENSION` attempt against a Postgres instance that may never
|
|
102
|
+
* grant it, which is worse than staying on the always-correct fallback.
|
|
103
|
+
*/
|
|
104
|
+
async ensurePgvectorExtension() {
|
|
105
|
+
if (!this.pgvectorPromise) this.pgvectorPromise = this.ensureMigrated().then(() => this.sql.unsafe("CREATE EXTENSION IF NOT EXISTS vector")).then(() => true, () => false);
|
|
106
|
+
return this.pgvectorPromise;
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* Runs `fn` inside a transaction that first sets `app.team_id` for the duration of that
|
|
110
|
+
* transaction (`SET LOCAL`, via `set_config(..., true)`) whenever `ref.teamId` is set — the
|
|
111
|
+
* mechanism the RLS policies in `migrations.ts` check against. A `ref` with no `teamId`
|
|
112
|
+
* skips the `set_config` call entirely; rows with `team_id IS NULL` are visible regardless.
|
|
113
|
+
*/
|
|
114
|
+
async withTeamContext(ref, fn) {
|
|
115
|
+
await this.ensureMigrated();
|
|
116
|
+
return await this.sql.begin(async (tx) => {
|
|
117
|
+
if (ref.teamId) await tx`SELECT set_config('app.team_id', ${ref.teamId}, true)`;
|
|
118
|
+
return fn(tx);
|
|
119
|
+
});
|
|
120
|
+
}
|
|
121
|
+
/** Release the underlying connection pool. */
|
|
122
|
+
async close() {
|
|
123
|
+
await this.sql.end();
|
|
124
|
+
}
|
|
125
|
+
};
|
|
126
|
+
//#endregion
|
|
127
|
+
//#region src/working.ts
|
|
128
|
+
/**
|
|
129
|
+
* Persisted, multi-process-safe `IWorkingMemoryProvider` backed by Postgres — the shared,
|
|
130
|
+
* production-shaped counterpart to `@alineo-labs/sqlite-memory`'s file-based provider. Rows
|
|
131
|
+
* are additionally isolated by `ResourceRef.teamId` via row-level security (see
|
|
132
|
+
* `migrations.ts`) whenever a team is set.
|
|
133
|
+
*
|
|
134
|
+
* Written against the `postgres` package the same way `@alineo-labs/postgres`'s ledger
|
|
135
|
+
* adapter is, but — like that adapter — has no live database to run against in this repo's
|
|
136
|
+
* test environment, so this ships type-checked, not integration-tested against a real
|
|
137
|
+
* Postgres instance.
|
|
138
|
+
*/
|
|
139
|
+
var PostgresWorkingMemoryProvider = class {
|
|
140
|
+
conn;
|
|
141
|
+
constructor(connectionString) {
|
|
142
|
+
this.conn = new PostgresMemoryConnection(connectionString);
|
|
143
|
+
}
|
|
144
|
+
async get(ref, key) {
|
|
145
|
+
const rows = await this.conn.withTeamContext(ref, (tx) => tx`
|
|
146
|
+
SELECT value FROM alineo_working_memory WHERE scope = ${scopeKey(ref)} AND key = ${key}
|
|
147
|
+
`);
|
|
148
|
+
return rows.length ? rows[0].value : void 0;
|
|
149
|
+
}
|
|
150
|
+
async set(ref, key, value) {
|
|
151
|
+
await this.conn.withTeamContext(ref, (tx) => tx`
|
|
152
|
+
INSERT INTO alineo_working_memory (scope, team_id, key, value)
|
|
153
|
+
VALUES (${scopeKey(ref)}, ${ref.teamId ?? null}, ${key}, ${JSON.stringify(value)}::jsonb)
|
|
154
|
+
ON CONFLICT (scope, key) DO UPDATE SET value = excluded.value
|
|
155
|
+
`);
|
|
156
|
+
}
|
|
157
|
+
async list(ref) {
|
|
158
|
+
const rows = await this.conn.withTeamContext(ref, (tx) => tx`
|
|
159
|
+
SELECT key, value FROM alineo_working_memory WHERE scope = ${scopeKey(ref)}
|
|
160
|
+
`);
|
|
161
|
+
return Object.fromEntries(rows.map((r) => [r.key, r.value]));
|
|
162
|
+
}
|
|
163
|
+
async delete(ref, key) {
|
|
164
|
+
await this.conn.withTeamContext(ref, (tx) => tx`
|
|
165
|
+
DELETE FROM alineo_working_memory WHERE scope = ${scopeKey(ref)} AND key = ${key}
|
|
166
|
+
`);
|
|
167
|
+
}
|
|
168
|
+
/** Release the underlying connection pool. */
|
|
169
|
+
async close() {
|
|
170
|
+
await this.conn.close();
|
|
171
|
+
}
|
|
172
|
+
};
|
|
173
|
+
//#endregion
|
|
174
|
+
//#region src/semantic.ts
|
|
175
|
+
/** `remembered_at` comes back from Postgres's `BIGINT` column as a numeric string, unlike
|
|
176
|
+
* `@alineo-labs/sqlite-memory`'s `number` — coerce before handing off to the shared mapping. */
|
|
177
|
+
function toSharedRow(row) {
|
|
178
|
+
return {
|
|
179
|
+
...row,
|
|
180
|
+
remembered_at: Number(row.remembered_at)
|
|
181
|
+
};
|
|
182
|
+
}
|
|
183
|
+
/** Encodes a raw embedding as the text literal `pgvector` accepts via an explicit `::vector`
|
|
184
|
+
* cast — the `postgres` driver has no native binary type for it, so a plain interpolated
|
|
185
|
+
* string plus a SQL-side cast is the standard way to pass one through this driver. */
|
|
186
|
+
function toVectorLiteral(vector) {
|
|
187
|
+
return `[${vector.join(",")}]`;
|
|
188
|
+
}
|
|
189
|
+
const VEC_TABLE = "alineo_semantic_vec";
|
|
190
|
+
/**
|
|
191
|
+
* Persisted, multi-process-safe `ISemanticMemoryProvider` (+ pruning) backed by Postgres.
|
|
192
|
+
*
|
|
193
|
+
* Ranks `recall()` with a real `pgvector` HNSW index whenever the `vector` extension can be
|
|
194
|
+
* installed on the connected Postgres instance (`CREATE EXTENSION IF NOT EXISTS vector` —
|
|
195
|
+
* enabled by default on Supabase/Neon/RDS-with-the-extension-allowlisted; unavailable on a
|
|
196
|
+
* bare Postgres with no superuser access), falling back to the same in-JS cosine-similarity
|
|
197
|
+
* scan `InMemorySemanticMemoryProvider`/`SQLiteSemanticMemoryProvider`'s no-index path uses
|
|
198
|
+
* otherwise. `hasVectorIndex` reports which path is active — mirrors
|
|
199
|
+
* `SQLiteSemanticMemoryProvider`'s own `hasVectorIndex`/`vecAvailable` split exactly.
|
|
200
|
+
*
|
|
201
|
+
* The plain `vector` (`double precision[]`) column on `alineo_semantic_memory` is always
|
|
202
|
+
* populated regardless of which path is active, so the fallback is never a degraded schema —
|
|
203
|
+
* it's a genuinely lower-performance code path over the same data, same as the SQLite adapter.
|
|
204
|
+
* The indexed path additionally maintains a separate `alineo_semantic_vec` table (id, scope,
|
|
205
|
+
* team_id, a real `vector(N)` column + HNSW index), created lazily once the embedding
|
|
206
|
+
* dimension is known from the first fact remembered — an `ON DELETE CASCADE` foreign key back
|
|
207
|
+
* to `alineo_semantic_memory.id` keeps it in sync with `forget()` for free, no extra delete
|
|
208
|
+
* needed there. Both tables get identical row-level-security policies, so scoping doesn't
|
|
209
|
+
* depend on which path a given `recall()` call takes.
|
|
210
|
+
*
|
|
211
|
+
* Unlike `SQLiteSemanticMemoryProvider`'s `vec0` virtual table (which needs `scope` declared
|
|
212
|
+
* as a native *partition key* so vec0's own KNN traversal applies it before counting `k`
|
|
213
|
+
* neighbors — see that file's `ensureVecTable` comment), a plain `WHERE v.scope = ...` alongside
|
|
214
|
+
* `ORDER BY ... LIMIT k` is sufficient and correct here: Postgres's planner evaluates the filter
|
|
215
|
+
* as part of the same query regardless of whether it uses the HNSW index or falls back to a
|
|
216
|
+
* sequential scan, so a resource never receives fewer than `k` matching rows the way an
|
|
217
|
+
* outer-join-after-KNN approach could.
|
|
218
|
+
*
|
|
219
|
+
* Mixing embedding models with different output dimensions on one instance throws a real
|
|
220
|
+
* Postgres dimension-mismatch error at insert time once `alineo_semantic_vec` is sized to the
|
|
221
|
+
* first dimension seen — same documented behavior as the SQLite adapter, not silently wrong.
|
|
222
|
+
*
|
|
223
|
+
* Like `PostgresWorkingMemoryProvider`, this ships type-checked against no live database.
|
|
224
|
+
*/
|
|
225
|
+
var PostgresSemanticMemoryProvider = class {
|
|
226
|
+
embeddings;
|
|
227
|
+
conn;
|
|
228
|
+
vecDimensions = null;
|
|
229
|
+
ensureVecTablePromise = null;
|
|
230
|
+
constructor(connectionString, embeddings) {
|
|
231
|
+
this.embeddings = embeddings;
|
|
232
|
+
this.conn = new PostgresMemoryConnection(connectionString);
|
|
233
|
+
}
|
|
234
|
+
/** Whether `recall()` is using the native `pgvector` HNSW index (true) or the in-JS cosine
|
|
235
|
+
* fallback scan (false, either because the extension isn't available on this Postgres
|
|
236
|
+
* instance or no fact has been remembered yet to size the index from). */
|
|
237
|
+
get hasVectorIndex() {
|
|
238
|
+
return this.vecDimensions != null;
|
|
239
|
+
}
|
|
240
|
+
/** Lazily creates `alineo_semantic_vec`, sized to `dimensions` — memoized, and (like
|
|
241
|
+
* `PostgresMemoryConnection.ensureMigrated()`) cleared on rejection so a transient failure
|
|
242
|
+
* here doesn't permanently pin this instance to the fallback path the way a permanent one
|
|
243
|
+
* reasonably should. */
|
|
244
|
+
ensureVecTable(dimensions) {
|
|
245
|
+
if (this.vecDimensions === dimensions) return Promise.resolve();
|
|
246
|
+
this.ensureVecTablePromise ??= this.conn.sql.unsafe(`
|
|
247
|
+
CREATE TABLE IF NOT EXISTS ${VEC_TABLE} (
|
|
248
|
+
id TEXT PRIMARY KEY REFERENCES alineo_semantic_memory(id) ON DELETE CASCADE,
|
|
249
|
+
scope TEXT NOT NULL,
|
|
250
|
+
team_id TEXT,
|
|
251
|
+
embedding VECTOR(${dimensions}) NOT NULL
|
|
252
|
+
);
|
|
253
|
+
CREATE INDEX IF NOT EXISTS ${VEC_TABLE}_hnsw ON ${VEC_TABLE}
|
|
254
|
+
USING hnsw (embedding vector_cosine_ops);
|
|
255
|
+
ALTER TABLE ${VEC_TABLE} ENABLE ROW LEVEL SECURITY;
|
|
256
|
+
ALTER TABLE ${VEC_TABLE} FORCE ROW LEVEL SECURITY;
|
|
257
|
+
DO $$ BEGIN
|
|
258
|
+
CREATE POLICY ${VEC_TABLE}_team_isolation ON ${VEC_TABLE}
|
|
259
|
+
FOR ALL
|
|
260
|
+
USING (team_id IS NULL OR team_id = current_setting('app.team_id', true))
|
|
261
|
+
WITH CHECK (team_id IS NULL OR team_id = current_setting('app.team_id', true));
|
|
262
|
+
EXCEPTION WHEN duplicate_object THEN NULL; END $$;
|
|
263
|
+
`).then(() => {
|
|
264
|
+
this.vecDimensions = dimensions;
|
|
265
|
+
}, (err) => {
|
|
266
|
+
this.ensureVecTablePromise = null;
|
|
267
|
+
throw err;
|
|
268
|
+
});
|
|
269
|
+
return this.ensureVecTablePromise;
|
|
270
|
+
}
|
|
271
|
+
async remember(ref, fact) {
|
|
272
|
+
const [vector] = await this.embeddings.embed([fact.content], { type: "passage" });
|
|
273
|
+
if (!vector) return;
|
|
274
|
+
const usePgvector = await this.conn.ensurePgvectorExtension();
|
|
275
|
+
if (usePgvector) await this.ensureVecTable(vector.length);
|
|
276
|
+
const id = crypto.randomUUID();
|
|
277
|
+
await this.conn.withTeamContext(ref, async (tx) => {
|
|
278
|
+
await tx`
|
|
279
|
+
INSERT INTO alineo_semantic_memory
|
|
280
|
+
(id, scope, team_id, content, vector, source_sandbox_id, source_entry_index, remembered_at)
|
|
281
|
+
VALUES (
|
|
282
|
+
${id},
|
|
283
|
+
${scopeKey(ref)},
|
|
284
|
+
${ref.teamId ?? null},
|
|
285
|
+
${fact.content},
|
|
286
|
+
${tx.array(vector)},
|
|
287
|
+
${fact.sourceRef?.sandboxId ?? null},
|
|
288
|
+
${fact.sourceRef?.entryIndex ?? null},
|
|
289
|
+
${Date.now()}
|
|
290
|
+
)
|
|
291
|
+
`;
|
|
292
|
+
if (usePgvector && this.hasVectorIndex) await tx`
|
|
293
|
+
INSERT INTO alineo_semantic_vec (id, scope, team_id, embedding)
|
|
294
|
+
VALUES (${id}, ${scopeKey(ref)}, ${ref.teamId ?? null}, ${toVectorLiteral(vector)}::vector)
|
|
295
|
+
`;
|
|
296
|
+
});
|
|
297
|
+
}
|
|
298
|
+
/** Batches the embedding call for all `facts` into one `embed()` invocation, and runs every
|
|
299
|
+
* insert (both tables, when the vec index is active) inside the single transaction
|
|
300
|
+
* `withTeamContext` already opens, instead of one transaction per fact — see
|
|
301
|
+
* `IBulkSemanticMemoryProvider`. */
|
|
302
|
+
async rememberMany(ref, facts) {
|
|
303
|
+
if (facts.length === 0) return;
|
|
304
|
+
const vectors = await this.embeddings.embed(facts.map((f) => f.content), { type: "passage" });
|
|
305
|
+
const usePgvector = await this.conn.ensurePgvectorExtension();
|
|
306
|
+
if (usePgvector) {
|
|
307
|
+
const firstVector = vectors.find((v) => v != null);
|
|
308
|
+
if (firstVector) await this.ensureVecTable(firstVector.length);
|
|
309
|
+
}
|
|
310
|
+
await this.conn.withTeamContext(ref, async (tx) => {
|
|
311
|
+
for (let i = 0; i < facts.length; i++) {
|
|
312
|
+
const vector = vectors[i];
|
|
313
|
+
const fact = facts[i];
|
|
314
|
+
if (!vector) continue;
|
|
315
|
+
const id = crypto.randomUUID();
|
|
316
|
+
await tx`
|
|
317
|
+
INSERT INTO alineo_semantic_memory
|
|
318
|
+
(id, scope, team_id, content, vector, source_sandbox_id, source_entry_index, remembered_at)
|
|
319
|
+
VALUES (
|
|
320
|
+
${id},
|
|
321
|
+
${scopeKey(ref)},
|
|
322
|
+
${ref.teamId ?? null},
|
|
323
|
+
${fact.content},
|
|
324
|
+
${tx.array(vector)},
|
|
325
|
+
${fact.sourceRef?.sandboxId ?? null},
|
|
326
|
+
${fact.sourceRef?.entryIndex ?? null},
|
|
327
|
+
${Date.now()}
|
|
328
|
+
)
|
|
329
|
+
`;
|
|
330
|
+
if (usePgvector && this.hasVectorIndex) await tx`
|
|
331
|
+
INSERT INTO alineo_semantic_vec (id, scope, team_id, embedding)
|
|
332
|
+
VALUES (${id}, ${scopeKey(ref)}, ${ref.teamId ?? null}, ${toVectorLiteral(vector)}::vector)
|
|
333
|
+
`;
|
|
334
|
+
}
|
|
335
|
+
});
|
|
336
|
+
}
|
|
337
|
+
async recall(ref, query, opts = {}) {
|
|
338
|
+
const topK = opts.topK ?? 5;
|
|
339
|
+
const [queryVector] = await this.embeddings.embed([query], { type: "query" });
|
|
340
|
+
if (!queryVector) return [];
|
|
341
|
+
if (this.hasVectorIndex) return (await this.conn.withTeamContext(ref, (tx) => tx`
|
|
342
|
+
SELECT m.id, m.content, m.source_sandbox_id, m.source_entry_index, m.remembered_at
|
|
343
|
+
FROM ${tx(VEC_TABLE)} v
|
|
344
|
+
JOIN alineo_semantic_memory m ON m.id = v.id
|
|
345
|
+
WHERE v.scope = ${scopeKey(ref)}
|
|
346
|
+
ORDER BY v.embedding <=> ${toVectorLiteral(queryVector)}::vector
|
|
347
|
+
LIMIT ${topK}
|
|
348
|
+
`)).map((row) => factFromRow(toSharedRow(row)));
|
|
349
|
+
const rows = await this.conn.withTeamContext(ref, (tx) => tx`SELECT * FROM alineo_semantic_memory WHERE scope = ${scopeKey(ref)}`);
|
|
350
|
+
if (rows.length === 0) return [];
|
|
351
|
+
return rows.map((row) => ({
|
|
352
|
+
row,
|
|
353
|
+
score: cosineSimilarity(queryVector, row.vector)
|
|
354
|
+
})).sort((a, b) => b.score - a.score).slice(0, topK).map(({ row }) => factFromRow(toSharedRow(row)));
|
|
355
|
+
}
|
|
356
|
+
async listAll(ref) {
|
|
357
|
+
return (await this.conn.withTeamContext(ref, (tx) => tx`SELECT * FROM alineo_semantic_memory WHERE scope = ${scopeKey(ref)}`)).map((row) => factFromRow(toSharedRow(row)));
|
|
358
|
+
}
|
|
359
|
+
async forget(ref, ids) {
|
|
360
|
+
if (ids.length === 0) return 0;
|
|
361
|
+
return (await this.conn.withTeamContext(ref, (tx) => tx`
|
|
362
|
+
DELETE FROM alineo_semantic_memory WHERE scope = ${scopeKey(ref)} AND id IN ${tx(ids)}
|
|
363
|
+
`)).count;
|
|
364
|
+
}
|
|
365
|
+
/** Release the underlying connection pool. */
|
|
366
|
+
async close() {
|
|
367
|
+
await this.conn.close();
|
|
368
|
+
}
|
|
369
|
+
};
|
|
370
|
+
//#endregion
|
|
371
|
+
export { PostgresSemanticMemoryProvider, PostgresWorkingMemoryProvider };
|
package/package.json
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@alineo-labs/postgres-memory",
|
|
3
|
+
"version": "0.2.0",
|
|
4
|
+
"license": "Apache-2.0",
|
|
5
|
+
"repository": {
|
|
6
|
+
"type": "git",
|
|
7
|
+
"url": "git+https://github.com/DrejT/alineo.git",
|
|
8
|
+
"directory": "packages/adapters/postgres-memory"
|
|
9
|
+
},
|
|
10
|
+
"files": [
|
|
11
|
+
"dist"
|
|
12
|
+
],
|
|
13
|
+
"type": "module",
|
|
14
|
+
"main": "./dist/index.mjs",
|
|
15
|
+
"types": "./dist/index.d.mts",
|
|
16
|
+
"exports": {
|
|
17
|
+
".": {
|
|
18
|
+
"import": "./dist/index.mjs",
|
|
19
|
+
"types": "./dist/index.d.mts"
|
|
20
|
+
}
|
|
21
|
+
},
|
|
22
|
+
"publishConfig": {
|
|
23
|
+
"access": "public",
|
|
24
|
+
"provenance": true
|
|
25
|
+
},
|
|
26
|
+
"scripts": {
|
|
27
|
+
"build": "tsdown"
|
|
28
|
+
},
|
|
29
|
+
"dependencies": {
|
|
30
|
+
"@alineo-labs/memory": "0.2.0",
|
|
31
|
+
"postgres": "3.4.9"
|
|
32
|
+
},
|
|
33
|
+
"devDependencies": {
|
|
34
|
+
"bun-types": "1.3.14",
|
|
35
|
+
"tsdown": "0.22.3",
|
|
36
|
+
"typescript": "6.0.3"
|
|
37
|
+
}
|
|
38
|
+
}
|