okengine 0.3.5 → 0.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +4 -0
- package/manifest.v1.schema.json +43 -1
- package/package.json +16 -1
- package/site/content/docs/ai/skills.mdx +9 -7
- package/site/content/docs/console/vault.mdx +4 -0
- package/site/content/docs/elements/channel.mdx +5 -4
- package/site/content/docs/elements/clock.mdx +1 -0
- package/site/content/docs/elements/flow.mdx +15 -0
- package/site/content/docs/elements/gate.mdx +16 -11
- package/site/content/docs/elements/signal.mdx +8 -7
- package/site/content/docs/elements/store.mdx +48 -8
- package/site/content/docs/elements/vault.mdx +1 -1
- package/site/content/docs/plugins/ip-allowlist.mdx +2 -2
- package/site/content/docs/reference/configuration.mdx +2 -2
- package/site/content/docs/reference/environment-variables.mdx +8 -0
- package/site/content/docs/reference/fx.mdx +40 -0
- package/site/content/docs/reference/plugins.mdx +18 -18
- package/src/compiler/extract.test.ts +63 -0
- package/src/compiler/extract.ts +36 -15
- package/src/console/server/channels.ts +2 -0
- package/src/console/server/clock.ts +3 -0
- package/src/console/server/flows.ts +13 -0
- package/src/console/server/gates.ts +2 -0
- package/src/console/server/plugins.ts +20 -1
- package/src/console/server/signals.ts +5 -0
- package/src/console/server/store.test.ts +17 -0
- package/src/console/server/store.ts +24 -0
- package/src/console/ui/channels/types.ts +1 -0
- package/src/console/ui/clock/types.ts +1 -0
- package/src/console/ui/display.test.ts +14 -0
- package/src/console/ui/display.ts +9 -0
- package/src/console/ui/dist/assets/{index-BWo8R7NR.js → index-CrKMmO__.js} +2 -2
- package/src/console/ui/dist/assets/panel-channels-DCDd4WAC.js +1 -0
- package/src/console/ui/dist/assets/panel-clock-DjGGFPzr.js +1 -0
- package/src/console/ui/dist/assets/panel-gates-B5eTE8XH.js +1 -0
- package/src/console/ui/dist/assets/{panel-overview-BznEOTnb.js → panel-overview-BsFvDdts.js} +1 -1
- package/src/console/ui/dist/assets/panel-plugins-Cj7DK1er.js +1 -0
- package/src/console/ui/dist/assets/{panel-runs-CGWNHLR4.js → panel-runs-C0gmnoYL.js} +1 -1
- package/src/console/ui/dist/assets/panel-signals-whmDXIg3.js +1 -0
- package/src/console/ui/dist/assets/panel-store-CEMHLvaw.js +1 -0
- package/src/console/ui/dist/assets/{panel-traces-DBLx2ilD.js → panel-traces-BDiAuVSK.js} +1 -1
- package/src/console/ui/dist/assets/panel-vault-C9wjbki8.js +1 -0
- package/src/console/ui/dist/index.html +1 -1
- package/src/console/ui/gates/types.ts +1 -0
- package/src/console/ui/plugins/fixture.ts +7 -0
- package/src/console/ui/plugins/types.ts +3 -0
- package/src/console/ui/shell/client.ts +3 -0
- package/src/console/ui/shell/panels/channels/ChannelsPanel.tsx +7 -2
- package/src/console/ui/shell/panels/clock/ClockPanel.tsx +9 -2
- package/src/console/ui/shell/panels/gates/GatesPanel.tsx +12 -5
- package/src/console/ui/shell/panels/plugins/PluginsPanel.tsx +18 -4
- package/src/console/ui/shell/panels/signals/SignalsPanel.tsx +13 -2
- package/src/console/ui/shell/panels/store/StorePanel.tsx +11 -4
- package/src/console/ui/shell/panels/vault/VaultPanel.tsx +9 -3
- package/src/console/ui/signals/types.ts +1 -0
- package/src/console/ui/store/fixture.ts +5 -0
- package/src/console/ui/store/types.ts +2 -0
- package/src/drivers/conformance.test.ts +11 -0
- package/src/drivers/drizzle-dialect.test.ts +4 -0
- package/src/drivers/drizzle-dialect.ts +8 -4
- package/src/drivers/index.ts +4 -0
- package/src/drivers/libsql.ts +179 -0
- package/src/drivers/pglite.ts +79 -0
- package/src/drivers/pgvector.ts +54 -19
- package/src/drivers/types.ts +16 -7
- package/src/elements/channel/declare.ts +5 -0
- package/src/elements/clock/declare.ts +5 -0
- package/src/elements/clock/durable.ts +7 -1
- package/src/elements/gate/declare.ts +28 -5
- package/src/elements/gate.ts +1 -0
- package/src/elements/signal/declare.ts +5 -0
- package/src/elements/store/declare.ts +10 -2
- package/src/elements/store/index-boot.test.ts +257 -0
- package/src/elements/store/runtime.ts +67 -9
- package/src/elements/store/schema-decl.ts +7 -0
- package/src/kernel/abort-scope.ts +116 -0
- package/src/kernel/app.ts +12 -2
- package/src/kernel/boot-bind/store.test.ts +59 -1
- package/src/kernel/boot-bind/store.ts +64 -2
- package/src/kernel/concurrency.test.ts +236 -0
- package/src/kernel/concurrency.ts +172 -0
- package/src/kernel/flow.ts +9 -0
- package/src/kernel/fx.test.ts +12 -0
- package/src/kernel/fx.ts +44 -0
- package/src/kernel/index.ts +14 -0
- package/src/kernel/journal.ts +9 -0
- package/src/kernel/plugin/capabilities.test.ts +18 -0
- package/src/kernel/plugin.ts +11 -3
- package/src/kernel/registry.ts +29 -6
- package/src/manifest/types.ts +21 -0
- package/src/release/measure.ts +4 -0
- package/src/console/ui/dist/assets/panel-channels-BOmQ-onL.js +0 -1
- package/src/console/ui/dist/assets/panel-clock-giAq0Ccv.js +0 -1
- package/src/console/ui/dist/assets/panel-gates-XclZxWD5.js +0 -1
- package/src/console/ui/dist/assets/panel-plugins-CcGM1g64.js +0 -1
- package/src/console/ui/dist/assets/panel-signals-CNywkdak.js +0 -1
- package/src/console/ui/dist/assets/panel-store-KmTbFHMH.js +0 -1
- package/src/console/ui/dist/assets/panel-vault-CEnFc0dk.js +0 -1
|
@@ -40,6 +40,8 @@ export interface StoreDeclBase {
|
|
|
40
40
|
|
|
41
41
|
/** Options for {@link store.sql}. */
|
|
42
42
|
export interface SqlStoreOptions {
|
|
43
|
+
/** Optional human description for Console / docs (falls back to the store name). */
|
|
44
|
+
readonly description?: string;
|
|
43
45
|
/**
|
|
44
46
|
* Schema tables — okengine {@link TableHandle}s and/or Drizzle tables.
|
|
45
47
|
* Classification is read from TableHandle columns when present.
|
|
@@ -56,6 +58,7 @@ export interface SqlStoreOptions {
|
|
|
56
58
|
export interface SqlStoreDecl extends StoreDeclBase {
|
|
57
59
|
readonly facet: "sql";
|
|
58
60
|
readonly ref: `sql:${string}`;
|
|
61
|
+
readonly description?: string;
|
|
59
62
|
readonly schema?: SqlStoreOptions["schema"];
|
|
60
63
|
readonly classify?: SqlStoreOptions["classify"];
|
|
61
64
|
/**
|
|
@@ -94,6 +97,8 @@ export interface FilesStoreDecl extends StoreDeclBase {
|
|
|
94
97
|
|
|
95
98
|
/** Options for {@link store.index}. */
|
|
96
99
|
export interface IndexStoreOptions {
|
|
100
|
+
/** Optional human description for Console / docs (falls back to the index name). */
|
|
101
|
+
readonly description?: string;
|
|
97
102
|
/** Vector dimensions (default 3 for tests; set explicitly in apps). */
|
|
98
103
|
readonly dims?: number;
|
|
99
104
|
}
|
|
@@ -102,6 +107,7 @@ export interface IndexStoreOptions {
|
|
|
102
107
|
export interface IndexStoreDecl extends StoreDeclBase {
|
|
103
108
|
readonly facet: "index";
|
|
104
109
|
readonly ref: `index:${string}`;
|
|
110
|
+
readonly description?: string;
|
|
105
111
|
readonly dims?: number;
|
|
106
112
|
}
|
|
107
113
|
|
|
@@ -119,6 +125,7 @@ export function sql(name: string, options: SqlStoreOptions = {}): SqlStoreDecl {
|
|
|
119
125
|
facet: "sql",
|
|
120
126
|
name,
|
|
121
127
|
ref: `sql:${name}`,
|
|
128
|
+
...(options.description !== undefined ? { description: options.description } : {}),
|
|
122
129
|
schema: options.schema,
|
|
123
130
|
classify: options.classify,
|
|
124
131
|
table(table) {
|
|
@@ -147,7 +154,7 @@ export function kv(name: string, options: KvStoreOptions = {}): KvStoreDecl {
|
|
|
147
154
|
facet: "kv",
|
|
148
155
|
name,
|
|
149
156
|
ref: `kv:${name}`,
|
|
150
|
-
description: options.description,
|
|
157
|
+
...(options.description !== undefined ? { description: options.description } : {}),
|
|
151
158
|
};
|
|
152
159
|
}
|
|
153
160
|
|
|
@@ -162,7 +169,7 @@ export function files(name: string, options: FilesStoreOptions = {}): FilesStore
|
|
|
162
169
|
facet: "files",
|
|
163
170
|
name,
|
|
164
171
|
ref: `files:${name}`,
|
|
165
|
-
description: options.description,
|
|
172
|
+
...(options.description !== undefined ? { description: options.description } : {}),
|
|
166
173
|
};
|
|
167
174
|
}
|
|
168
175
|
|
|
@@ -177,6 +184,7 @@ export function index(name: string, options: IndexStoreOptions = {}): IndexStore
|
|
|
177
184
|
facet: "index",
|
|
178
185
|
name,
|
|
179
186
|
ref: `index:${name}`,
|
|
187
|
+
...(options.description !== undefined ? { description: options.description } : {}),
|
|
180
188
|
dims: options.dims,
|
|
181
189
|
};
|
|
182
190
|
}
|
|
@@ -0,0 +1,257 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* End-to-end `store.index()` boot wiring — the CONFIGURED driver resolves at
|
|
3
|
+
* real boot for every backend (memory / libsql / pgvector over pglite /
|
|
4
|
+
* pgvector over real Postgres), SQL-backed indexes share store.sql's
|
|
5
|
+
* already-open connection (never a second, redundant one), and search runs
|
|
6
|
+
* through each engine's native ANN index — no JS-side full scan survives.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { afterEach, describe, expect, test } from "bun:test";
|
|
10
|
+
import {
|
|
11
|
+
connectLibsql,
|
|
12
|
+
libsqlDriver,
|
|
13
|
+
libsqlIndexDriver,
|
|
14
|
+
openLibsqlIndex,
|
|
15
|
+
} from "../../drivers/libsql.ts";
|
|
16
|
+
import { connectPglite } from "../../drivers/pglite.ts";
|
|
17
|
+
import { openPgvectorIndex } from "../../drivers/pgvector.ts";
|
|
18
|
+
import { connectPostgres } from "../../drivers/postgres.ts";
|
|
19
|
+
import type { SqlConnection, SqlDriver } from "../../drivers/types.ts";
|
|
20
|
+
import { bindStore } from "../../kernel/boot-bind/store.ts";
|
|
21
|
+
import {
|
|
22
|
+
createStoreRuntime,
|
|
23
|
+
index as declareIndex,
|
|
24
|
+
sql as declareSql,
|
|
25
|
+
type IndexStoreFxHandle,
|
|
26
|
+
type SqlStoreHandle,
|
|
27
|
+
} from "../store.ts";
|
|
28
|
+
|
|
29
|
+
const WRITE_CTX = (name: string) => ({
|
|
30
|
+
effects: { reads: [`index:${name}` as const], writes: [`index:${name}` as const] },
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
const prev = {
|
|
34
|
+
libsql: process.env.OKE_LIBSQL_URL,
|
|
35
|
+
pglite: process.env.OKE_PGLITE_URL,
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
afterEach(() => {
|
|
39
|
+
if (prev.libsql === undefined) delete process.env.OKE_LIBSQL_URL;
|
|
40
|
+
else process.env.OKE_LIBSQL_URL = prev.libsql;
|
|
41
|
+
if (prev.pglite === undefined) delete process.env.OKE_PGLITE_URL;
|
|
42
|
+
else process.env.OKE_PGLITE_URL = prev.pglite;
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
/** Wrap a real connection so the SQL it executes can be asserted on. */
|
|
46
|
+
function recordingConn(conn: SqlConnection): { conn: SqlConnection; statements: string[] } {
|
|
47
|
+
const statements: string[] = [];
|
|
48
|
+
return {
|
|
49
|
+
statements,
|
|
50
|
+
conn: {
|
|
51
|
+
driverId: conn.driverId,
|
|
52
|
+
role: conn.role,
|
|
53
|
+
query: (sql, params) => {
|
|
54
|
+
statements.push(sql);
|
|
55
|
+
return conn.query(sql, params);
|
|
56
|
+
},
|
|
57
|
+
exec: (sql, params) => {
|
|
58
|
+
statements.push(sql);
|
|
59
|
+
return conn.exec(sql, params);
|
|
60
|
+
},
|
|
61
|
+
close: () => conn.close(),
|
|
62
|
+
},
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
describe("store.index boot wiring — memory", () => {
|
|
67
|
+
test("boot with no configured index driver resolves memory end to end", async () => {
|
|
68
|
+
const kb = declareIndex("kb", { dims: 3 });
|
|
69
|
+
const runtime = bindStore({ stores: [kb] }, "local", () => Date.now());
|
|
70
|
+
const handle = (await runtime.open(kb, WRITE_CTX("kb"))) as IndexStoreFxHandle;
|
|
71
|
+
expect(handle.driverId).toBe("memory");
|
|
72
|
+
await handle.upsert("near", [1, 0, 0]);
|
|
73
|
+
await handle.upsert("far", [0, 1, 0]);
|
|
74
|
+
const hits = await handle.search([1, 0, 0], 2);
|
|
75
|
+
expect(hits[0]?.id).toBe("near");
|
|
76
|
+
await runtime.close();
|
|
77
|
+
});
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
describe("store.index boot wiring — libsql", () => {
|
|
81
|
+
test("boot: sql=libsql + index=libsql resolves native ANN end to end", async () => {
|
|
82
|
+
process.env.OKE_LIBSQL_URL = ":memory:";
|
|
83
|
+
const kb = declareIndex("kb", { dims: 3 });
|
|
84
|
+
const runtime = bindStore(
|
|
85
|
+
{
|
|
86
|
+
config: { drivers: { store: { sql: { local: "libsql" }, index: { local: "libsql" } } } },
|
|
87
|
+
stores: [kb],
|
|
88
|
+
},
|
|
89
|
+
"local",
|
|
90
|
+
() => Date.now(),
|
|
91
|
+
);
|
|
92
|
+
const handle = (await runtime.open(kb, WRITE_CTX("kb"))) as IndexStoreFxHandle;
|
|
93
|
+
expect(handle.driverId).toBe("libsql");
|
|
94
|
+
|
|
95
|
+
await handle.upsert("near", [1, 0.1, 0], { label: "near" });
|
|
96
|
+
await handle.upsert("far", [0, 1, 0]);
|
|
97
|
+
await handle.upsert("farther", [-1, 0, 0]);
|
|
98
|
+
const hits = await handle.search([1, 0, 0], 3);
|
|
99
|
+
expect(hits.map((h) => h.id)).toEqual(["near", "far", "farther"]);
|
|
100
|
+
expect(hits[0]!.score).toBeGreaterThan(hits[1]!.score);
|
|
101
|
+
expect(hits[0]!.meta).toEqual({ label: "near" });
|
|
102
|
+
expect(await handle.delete("farther")).toBe(true);
|
|
103
|
+
await runtime.close();
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
test("sql and index facets share one already-open libsql connection", async () => {
|
|
107
|
+
let connects = 0;
|
|
108
|
+
const countingLibsql: SqlDriver = {
|
|
109
|
+
...libsqlDriver,
|
|
110
|
+
connect: async (opts) => {
|
|
111
|
+
connects++;
|
|
112
|
+
return libsqlDriver.connect(opts);
|
|
113
|
+
},
|
|
114
|
+
};
|
|
115
|
+
const docs = declareSql("docs");
|
|
116
|
+
const kb = declareIndex("kb", { dims: 3 });
|
|
117
|
+
const runtime = createStoreRuntime({
|
|
118
|
+
drivers: { sql: countingLibsql, index: libsqlIndexDriver },
|
|
119
|
+
sql: { docs: { name: "docs", primary: { url: ":memory:" } } },
|
|
120
|
+
index: { kb: { url: ":memory:" } },
|
|
121
|
+
});
|
|
122
|
+
// Index first: it must open the connection the sql facet then reuses.
|
|
123
|
+
const idxHandle = (await runtime.open(kb, WRITE_CTX("kb"))) as IndexStoreFxHandle;
|
|
124
|
+
await idxHandle.upsert("a", [1, 0, 0]);
|
|
125
|
+
const sqlHandle = (await runtime.open(docs, {
|
|
126
|
+
effects: { reads: ["sql:docs"], writes: ["sql:docs"] },
|
|
127
|
+
})) as SqlStoreHandle;
|
|
128
|
+
await sqlHandle.raw("CREATE TABLE IF NOT EXISTS t_share (id TEXT)", []);
|
|
129
|
+
expect(connects).toBe(1);
|
|
130
|
+
await runtime.close();
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
test("libsql index searches through vector_top_k — real ANN, not a scan", async () => {
|
|
134
|
+
const real = await connectLibsql({ url: ":memory:" });
|
|
135
|
+
const { conn, statements } = recordingConn(real);
|
|
136
|
+
const idx = await openLibsqlIndex({ name: "ann", dims: 3, sql: conn });
|
|
137
|
+
await idx.upsert("x", [1, 0, 0]);
|
|
138
|
+
await idx.upsert("y", [0, 1, 0]);
|
|
139
|
+
const hits = await idx.search([1, 0, 0], 1);
|
|
140
|
+
expect(hits[0]?.id).toBe("x");
|
|
141
|
+
expect(statements.some((s) => s.includes("libsql_vector_idx"))).toBe(true);
|
|
142
|
+
expect(statements.some((s) => s.includes("vector_top_k"))).toBe(true);
|
|
143
|
+
const plan = await real.query(
|
|
144
|
+
`EXPLAIN QUERY PLAN SELECT * FROM vector_top_k('oke_idx_ann_vec', vector32('[1,0,0]'), 1)`,
|
|
145
|
+
);
|
|
146
|
+
expect(JSON.stringify(plan)).toContain("vector_top_k");
|
|
147
|
+
await idx.close();
|
|
148
|
+
await real.close();
|
|
149
|
+
});
|
|
150
|
+
});
|
|
151
|
+
|
|
152
|
+
describe("store.index boot wiring — pglite + pgvector", () => {
|
|
153
|
+
test("boot: sql=pglite + index=pgvector resolves end to end", async () => {
|
|
154
|
+
process.env.OKE_PGLITE_URL = "memory://";
|
|
155
|
+
const kb = declareIndex("kb", { dims: 3 });
|
|
156
|
+
const runtime = bindStore(
|
|
157
|
+
{
|
|
158
|
+
config: { drivers: { store: { sql: { local: "pglite" }, index: { local: "pgvector" } } } },
|
|
159
|
+
stores: [kb],
|
|
160
|
+
},
|
|
161
|
+
"local",
|
|
162
|
+
() => Date.now(),
|
|
163
|
+
);
|
|
164
|
+
const handle = (await runtime.open(kb, WRITE_CTX("kb"))) as IndexStoreFxHandle;
|
|
165
|
+
expect(handle.driverId).toBe("pgvector");
|
|
166
|
+
|
|
167
|
+
await handle.upsert("near", [1, 0.1, 0], { label: "near" });
|
|
168
|
+
await handle.upsert("far", [0, 1, 0]);
|
|
169
|
+
await handle.upsert("farther", [-1, 0, 0]);
|
|
170
|
+
const hits = await handle.search([1, 0, 0], 3);
|
|
171
|
+
expect(hits.map((h) => h.id)).toEqual(["near", "far", "farther"]);
|
|
172
|
+
expect(hits[0]!.score).toBeGreaterThan(hits[1]!.score);
|
|
173
|
+
expect(hits[0]!.meta).toEqual({ label: "near" });
|
|
174
|
+
expect(await handle.delete("farther")).toBe(true);
|
|
175
|
+
await runtime.close();
|
|
176
|
+
});
|
|
177
|
+
|
|
178
|
+
test("pglite runs the shared pgvector HNSW path — real ANN, not a scan", async () => {
|
|
179
|
+
const real = await connectPglite({ url: "memory://" });
|
|
180
|
+
const { conn, statements } = recordingConn(real);
|
|
181
|
+
const idx = await openPgvectorIndex({ name: "ann", dims: 3, sql: conn });
|
|
182
|
+
await idx.upsert("a", [1, 0, 0]);
|
|
183
|
+
await idx.upsert("b", [0, 1, 0]);
|
|
184
|
+
const hits = await idx.search([1, 0, 0], 2);
|
|
185
|
+
expect(hits[0]?.id).toBe("a");
|
|
186
|
+
expect(statements.some((s) => s.includes("CREATE EXTENSION IF NOT EXISTS vector"))).toBe(true);
|
|
187
|
+
expect(
|
|
188
|
+
statements.some((s) => s.includes("USING hnsw") && s.includes("vector_cosine_ops")),
|
|
189
|
+
).toBe(true);
|
|
190
|
+
expect(statements.some((s) => s.includes("<=>"))).toBe(true);
|
|
191
|
+
const indexes = await real.query(
|
|
192
|
+
`SELECT indexdef FROM pg_indexes WHERE tablename = 'oke_idx_ann'`,
|
|
193
|
+
);
|
|
194
|
+
expect(JSON.stringify(indexes)).toContain("USING hnsw");
|
|
195
|
+
await idx.close();
|
|
196
|
+
await real.close();
|
|
197
|
+
});
|
|
198
|
+
});
|
|
199
|
+
|
|
200
|
+
describe("store.index boot wiring — fail loud", () => {
|
|
201
|
+
test("pgvector index with memory sql driver throws — never silently memory", async () => {
|
|
202
|
+
const kb = declareIndex("kb", { dims: 3 });
|
|
203
|
+
const runtime = bindStore(
|
|
204
|
+
{
|
|
205
|
+
config: { drivers: { store: { sql: { local: "memory" }, index: { local: "pgvector" } } } },
|
|
206
|
+
stores: [kb],
|
|
207
|
+
},
|
|
208
|
+
"local",
|
|
209
|
+
() => Date.now(),
|
|
210
|
+
);
|
|
211
|
+
await expect(runtime.open(kb, WRITE_CTX("kb"))).rejects.toThrow(/cannot share sql driver/);
|
|
212
|
+
await runtime.close();
|
|
213
|
+
});
|
|
214
|
+
|
|
215
|
+
test("libsql index with sqlite sql driver throws", async () => {
|
|
216
|
+
const kb = declareIndex("kb", { dims: 3 });
|
|
217
|
+
const runtime = bindStore(
|
|
218
|
+
{
|
|
219
|
+
config: { drivers: { store: { sql: { local: "sqlite" }, index: { local: "libsql" } } } },
|
|
220
|
+
stores: [kb],
|
|
221
|
+
},
|
|
222
|
+
"local",
|
|
223
|
+
() => Date.now(),
|
|
224
|
+
);
|
|
225
|
+
await expect(runtime.open(kb, WRITE_CTX("kb"))).rejects.toThrow(/cannot share sql driver/);
|
|
226
|
+
await runtime.close();
|
|
227
|
+
});
|
|
228
|
+
});
|
|
229
|
+
|
|
230
|
+
const PGVECTOR_URL = process.env.OKE_TEST_PGVECTOR_URL;
|
|
231
|
+
if (!PGVECTOR_URL) {
|
|
232
|
+
console.log("skip: live pgvector e2e (OKE_TEST_PGVECTOR_URL not set)");
|
|
233
|
+
}
|
|
234
|
+
const liveTest = PGVECTOR_URL ? test : test.skip;
|
|
235
|
+
|
|
236
|
+
describe("store.index boot wiring — real Postgres pgvector", () => {
|
|
237
|
+
liveTest("live Postgres runs real HNSW ANN, not a scan", async () => {
|
|
238
|
+
const real = await connectPostgres({ url: PGVECTOR_URL });
|
|
239
|
+
const { conn, statements } = recordingConn(real);
|
|
240
|
+
const idx = await openPgvectorIndex({ name: "live", dims: 3, sql: conn });
|
|
241
|
+
await idx.upsert("near", [1, 0.1, 0]);
|
|
242
|
+
await idx.upsert("far", [0, 1, 0]);
|
|
243
|
+
await idx.upsert("farther", [-1, 0, 0]);
|
|
244
|
+
const hits = await idx.search([1, 0, 0], 3);
|
|
245
|
+
expect(hits.map((h) => h.id)).toEqual(["near", "far", "farther"]);
|
|
246
|
+
expect(statements.some((s) => s.includes("CREATE EXTENSION IF NOT EXISTS vector"))).toBe(true);
|
|
247
|
+
expect(statements.some((s) => s.includes("USING hnsw"))).toBe(true);
|
|
248
|
+
expect(statements.some((s) => s.includes("<=>"))).toBe(true);
|
|
249
|
+
const indexes = await real.query(
|
|
250
|
+
`SELECT indexdef FROM pg_indexes WHERE tablename = 'oke_idx_live'`,
|
|
251
|
+
);
|
|
252
|
+
expect(JSON.stringify(indexes)).toContain("USING hnsw");
|
|
253
|
+
await idx.delete("farther");
|
|
254
|
+
await idx.close();
|
|
255
|
+
await real.close();
|
|
256
|
+
});
|
|
257
|
+
});
|
|
@@ -14,7 +14,9 @@ import type {
|
|
|
14
14
|
KvDriver,
|
|
15
15
|
KvNamespace,
|
|
16
16
|
SqlConnection,
|
|
17
|
+
SqlConnectOptions,
|
|
17
18
|
SqlDriver,
|
|
19
|
+
SqlRole,
|
|
18
20
|
SqlRow,
|
|
19
21
|
} from "../../drivers/types.ts";
|
|
20
22
|
import { buildClassificationMap, type MaskRowsOptions } from "./classify.ts";
|
|
@@ -116,7 +118,7 @@ export interface FilesStoreFxHandle {
|
|
|
116
118
|
/** Index handle on `fx.store`. */
|
|
117
119
|
export interface IndexStoreFxHandle {
|
|
118
120
|
readonly ref: `index:${string}`;
|
|
119
|
-
readonly driverId: "memory" | "pgvector";
|
|
121
|
+
readonly driverId: "memory" | "pgvector" | "libsql";
|
|
120
122
|
upsert(id: string, vector: readonly number[], meta?: Record<string, unknown>): Promise<void>;
|
|
121
123
|
search(
|
|
122
124
|
vector: readonly number[],
|
|
@@ -186,6 +188,19 @@ export function createStoreRuntime(options: CreateStoreRuntimeOptions): StoreRun
|
|
|
186
188
|
const kvNs = new Map<string, KvNamespace>();
|
|
187
189
|
const fileBuckets = new Map<string, FilesBucket>();
|
|
188
190
|
const indexes = new Map<string, IndexStore>();
|
|
191
|
+
const clientIds = new WeakMap<object, number>();
|
|
192
|
+
let nextClientId = 0;
|
|
193
|
+
|
|
194
|
+
/** Injected test clients with the same URL still get distinct cache entries. */
|
|
195
|
+
function clientTag(client: unknown): string {
|
|
196
|
+
if ((typeof client !== "object" && typeof client !== "function") || client === null) return "";
|
|
197
|
+
let id = clientIds.get(client as object);
|
|
198
|
+
if (id === undefined) {
|
|
199
|
+
id = ++nextClientId;
|
|
200
|
+
clientIds.set(client as object, id);
|
|
201
|
+
}
|
|
202
|
+
return `#c${id}`;
|
|
203
|
+
}
|
|
189
204
|
|
|
190
205
|
function classificationsFor(decl: SqlStoreDecl): ClassificationMap {
|
|
191
206
|
const nested: Record<
|
|
@@ -215,13 +230,7 @@ export function createStoreRuntime(options: CreateStoreRuntimeOptions): StoreRun
|
|
|
215
230
|
const driver = options.drivers.sql;
|
|
216
231
|
if (!driver) throw new Error("No sql driver configured");
|
|
217
232
|
|
|
218
|
-
const
|
|
219
|
-
let conn = sqlConns.get(cacheKey);
|
|
220
|
-
if (!conn) {
|
|
221
|
-
conn = await driver.connect(target.options);
|
|
222
|
-
sqlConns.set(cacheKey, conn);
|
|
223
|
-
}
|
|
224
|
-
|
|
233
|
+
const conn = await sharedSqlConn(driver, target.role, target.options);
|
|
225
234
|
return createSqlStoreHandle(`sql:${decl.name}`, {
|
|
226
235
|
connection: conn,
|
|
227
236
|
classifications: classificationsFor(decl),
|
|
@@ -231,6 +240,51 @@ export function createStoreRuntime(options: CreateStoreRuntimeOptions): StoreRun
|
|
|
231
240
|
});
|
|
232
241
|
}
|
|
233
242
|
|
|
243
|
+
/**
|
|
244
|
+
* One connection per driver+role+URL (+ injected client) — `store.sql` and a
|
|
245
|
+
* SQL-backed `store.index` share the same already-open connection, never a
|
|
246
|
+
* second one.
|
|
247
|
+
*/
|
|
248
|
+
async function sharedSqlConn(
|
|
249
|
+
driver: SqlDriver,
|
|
250
|
+
role: SqlRole,
|
|
251
|
+
connectOptions: SqlConnectOptions,
|
|
252
|
+
): Promise<SqlConnection> {
|
|
253
|
+
const cacheKey = `${driver.id}:${role}:${connectOptions.url ?? ""}${clientTag(connectOptions.client)}`;
|
|
254
|
+
let conn = sqlConns.get(cacheKey);
|
|
255
|
+
if (!conn) {
|
|
256
|
+
conn = await driver.connect({ ...connectOptions, role });
|
|
257
|
+
sqlConns.set(cacheKey, conn);
|
|
258
|
+
}
|
|
259
|
+
return conn;
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
/** Index drivers backed by a SQL engine borrow the sql facet's connection. */
|
|
263
|
+
async function sqlConnForIndex(
|
|
264
|
+
driver: IndexDriver,
|
|
265
|
+
url: string | undefined,
|
|
266
|
+
): Promise<SqlConnection> {
|
|
267
|
+
const sqlDriver = options.drivers.sql;
|
|
268
|
+
if (!sqlDriver) {
|
|
269
|
+
throw new Error(
|
|
270
|
+
`oke store: index driver "${driver.id}" needs a configured sql driver to share its connection`,
|
|
271
|
+
);
|
|
272
|
+
}
|
|
273
|
+
const compatible =
|
|
274
|
+
driver.id === "pgvector"
|
|
275
|
+
? sqlDriver.id === "postgres" || sqlDriver.id === "pglite"
|
|
276
|
+
: sqlDriver.id === "libsql";
|
|
277
|
+
if (!compatible) {
|
|
278
|
+
throw new Error(
|
|
279
|
+
`oke store: index driver "${driver.id}" cannot share sql driver "${sqlDriver.id}" — ` +
|
|
280
|
+
(driver.id === "pgvector"
|
|
281
|
+
? `pgvector needs store.sql on "postgres" or "pglite"`
|
|
282
|
+
: `libsql index needs store.sql on "libsql"`),
|
|
283
|
+
);
|
|
284
|
+
}
|
|
285
|
+
return sharedSqlConn(sqlDriver, "primary", { url });
|
|
286
|
+
}
|
|
287
|
+
|
|
234
288
|
async function openKv(decl: KvStoreDecl): Promise<KvStoreFxHandle> {
|
|
235
289
|
const driver = options.drivers.kv;
|
|
236
290
|
if (!driver) throw new Error("No kv driver configured");
|
|
@@ -284,11 +338,15 @@ export function createStoreRuntime(options: CreateStoreRuntimeOptions): StoreRun
|
|
|
284
338
|
let idx = indexes.get(decl.name);
|
|
285
339
|
if (!idx) {
|
|
286
340
|
const binding = options.index?.[decl.name] ?? {};
|
|
341
|
+
let sql = binding.sql;
|
|
342
|
+
if (!sql && driver.id !== "memory") {
|
|
343
|
+
sql = await sqlConnForIndex(driver, binding.url);
|
|
344
|
+
}
|
|
287
345
|
idx = await driver.open({
|
|
288
346
|
name: decl.name,
|
|
289
347
|
dims: binding.dims ?? decl.dims ?? 3,
|
|
290
348
|
url: binding.url,
|
|
291
|
-
sql
|
|
349
|
+
sql,
|
|
292
350
|
});
|
|
293
351
|
indexes.set(decl.name, idx);
|
|
294
352
|
}
|
|
@@ -55,6 +55,8 @@ export interface SchemaColumnDecl extends ColumnDef {
|
|
|
55
55
|
readonly tableName?: string;
|
|
56
56
|
/** Foreign key when `.references()` was used. */
|
|
57
57
|
readonly references?: ColumnReference;
|
|
58
|
+
/** Optional human description for Console / docs (falls back to the JS key). */
|
|
59
|
+
readonly description?: string;
|
|
58
60
|
}
|
|
59
61
|
|
|
60
62
|
/** Table from {@link store.schema.table} — extends {@link TableHandle}. */
|
|
@@ -75,6 +77,8 @@ export interface FieldBuilder {
|
|
|
75
77
|
retain(duration: string): FieldBuilder;
|
|
76
78
|
/** Override snake_case SQL name. */
|
|
77
79
|
as(sqlName: string): FieldBuilder;
|
|
80
|
+
/** Optional human description for Console / docs (falls back to the JS key). */
|
|
81
|
+
describe(description: string): FieldBuilder;
|
|
78
82
|
/**
|
|
79
83
|
* Declare a foreign key to another column (dialect-agnostic).
|
|
80
84
|
*
|
|
@@ -100,6 +104,7 @@ interface FieldState {
|
|
|
100
104
|
readonly defaultFnKind?: DefaultFnKind;
|
|
101
105
|
readonly classification?: ColumnClassification;
|
|
102
106
|
readonly sqlName?: string;
|
|
107
|
+
readonly description?: string;
|
|
103
108
|
readonly references?: ColumnReference;
|
|
104
109
|
}
|
|
105
110
|
|
|
@@ -145,6 +150,7 @@ function createBuilder(state: FieldState): FieldBuilder {
|
|
|
145
150
|
retain: (duration) =>
|
|
146
151
|
next({ classification: mergeClassification(state.classification, { retain: duration }) }),
|
|
147
152
|
as: (sqlName) => next({ sqlName }),
|
|
153
|
+
describe: (description) => next({ description }),
|
|
148
154
|
references: (ref, actions) =>
|
|
149
155
|
next({
|
|
150
156
|
references: actions ? { ref, actions } : { ref },
|
|
@@ -163,6 +169,7 @@ function createBuilder(state: FieldState): FieldBuilder {
|
|
|
163
169
|
...(state.defaultFn ? { defaultFn: state.defaultFn } : {}),
|
|
164
170
|
...(state.defaultFnKind ? { defaultFnKind: state.defaultFnKind } : {}),
|
|
165
171
|
...(state.classification ? { classification: state.classification } : {}),
|
|
172
|
+
...(state.description !== undefined ? { description: state.description } : {}),
|
|
166
173
|
...(state.references ? { references: state.references } : {}),
|
|
167
174
|
};
|
|
168
175
|
},
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Ambient AbortSignal scope for structured concurrency.
|
|
3
|
+
*
|
|
4
|
+
* Same ALS pattern as dry-run: `fx.all` / `fx.race` enter a child signal so
|
|
5
|
+
* cooperative branches (and future driver plumbing) can observe cancellation
|
|
6
|
+
* without every call site passing a signal argument.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { AsyncLocalStorage } from "node:async_hooks";
|
|
10
|
+
|
|
11
|
+
/** Never-aborted signal used outside an `all` / `race` branch. */
|
|
12
|
+
const NEVER_ABORTED = new AbortController().signal;
|
|
13
|
+
|
|
14
|
+
const storage = new AsyncLocalStorage<AbortSignal>();
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* AbortSignal for the current async chain, or a never-aborted signal when
|
|
18
|
+
* no structured-concurrency scope is active.
|
|
19
|
+
*/
|
|
20
|
+
export function currentAbortSignal(): AbortSignal {
|
|
21
|
+
return storage.getStore() ?? NEVER_ABORTED;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Run `fn` with `signal` as the ambient abort signal.
|
|
26
|
+
*
|
|
27
|
+
* @param signal - Branch / scope signal
|
|
28
|
+
* @param fn - Work that may read {@link currentAbortSignal}
|
|
29
|
+
*/
|
|
30
|
+
export async function withAbortSignal<T>(
|
|
31
|
+
signal: AbortSignal,
|
|
32
|
+
fn: () => T | Promise<T>,
|
|
33
|
+
): Promise<T> {
|
|
34
|
+
return storage.run(signal, fn);
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* When `parent` aborts, abort `child` with the same reason.
|
|
39
|
+
*
|
|
40
|
+
* @param parent - Outer signal
|
|
41
|
+
* @param child - Controller to abort
|
|
42
|
+
* @returns Unsubscribe
|
|
43
|
+
*/
|
|
44
|
+
export function linkAbort(parent: AbortSignal, child: AbortController): () => void {
|
|
45
|
+
if (parent.aborted) {
|
|
46
|
+
child.abort(parent.reason);
|
|
47
|
+
return () => undefined;
|
|
48
|
+
}
|
|
49
|
+
const onAbort = (): void => {
|
|
50
|
+
child.abort(parent.reason);
|
|
51
|
+
};
|
|
52
|
+
parent.addEventListener("abort", onAbort);
|
|
53
|
+
return () => parent.removeEventListener("abort", onAbort);
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* True when `err` is an abort rejection (`AbortError` name).
|
|
58
|
+
*
|
|
59
|
+
* @param err - Unknown
|
|
60
|
+
*/
|
|
61
|
+
export function isAbortError(err: unknown): boolean {
|
|
62
|
+
return err instanceof Error && err.name === "AbortError";
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Create an `AbortError` matching Web / fetch abort rejections.
|
|
67
|
+
*
|
|
68
|
+
* @param reason - Optional abort reason
|
|
69
|
+
*/
|
|
70
|
+
export function abortError(reason?: unknown): Error {
|
|
71
|
+
if (typeof DOMException === "function") {
|
|
72
|
+
const message =
|
|
73
|
+
reason instanceof Error
|
|
74
|
+
? reason.message
|
|
75
|
+
: reason !== undefined
|
|
76
|
+
? String(reason)
|
|
77
|
+
: "This operation was aborted";
|
|
78
|
+
return new DOMException(message, "AbortError");
|
|
79
|
+
}
|
|
80
|
+
const err = new Error(
|
|
81
|
+
reason instanceof Error
|
|
82
|
+
? reason.message
|
|
83
|
+
: reason !== undefined
|
|
84
|
+
? String(reason)
|
|
85
|
+
: "This operation was aborted",
|
|
86
|
+
);
|
|
87
|
+
err.name = "AbortError";
|
|
88
|
+
return err;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Sleep that rejects promptly when `signal` aborts.
|
|
93
|
+
*
|
|
94
|
+
* Not durable — retry backoff must not journal via `fx.clock.sleep`.
|
|
95
|
+
*
|
|
96
|
+
* @param ms - Milliseconds
|
|
97
|
+
* @param signal - Optional abort signal
|
|
98
|
+
*/
|
|
99
|
+
export function abortableSleep(ms: number, signal?: AbortSignal): Promise<void> {
|
|
100
|
+
if (ms <= 0) return Promise.resolve();
|
|
101
|
+
return new Promise((resolve, reject) => {
|
|
102
|
+
if (signal?.aborted) {
|
|
103
|
+
reject(abortError(signal.reason));
|
|
104
|
+
return;
|
|
105
|
+
}
|
|
106
|
+
const timer = setTimeout(() => {
|
|
107
|
+
signal?.removeEventListener("abort", onAbort);
|
|
108
|
+
resolve();
|
|
109
|
+
}, ms);
|
|
110
|
+
const onAbort = (): void => {
|
|
111
|
+
clearTimeout(timer);
|
|
112
|
+
reject(abortError(signal?.reason));
|
|
113
|
+
};
|
|
114
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
115
|
+
});
|
|
116
|
+
}
|
package/src/kernel/app.ts
CHANGED
|
@@ -33,6 +33,7 @@ import {
|
|
|
33
33
|
} from "./auth-resolve.ts";
|
|
34
34
|
import { runDurable } from "../elements/clock/durable.ts";
|
|
35
35
|
import { isFlow, type AnyFlowDef } from "./flow.ts";
|
|
36
|
+
import { fxRetry } from "./concurrency.ts";
|
|
36
37
|
import {
|
|
37
38
|
createFxContext,
|
|
38
39
|
resolveName,
|
|
@@ -680,13 +681,22 @@ export function oke(options: OkeOptions): OkeApp {
|
|
|
680
681
|
hooks,
|
|
681
682
|
async () => {
|
|
682
683
|
try {
|
|
684
|
+
const invoke = async (input: unknown) => {
|
|
685
|
+
const run = () => {
|
|
686
|
+
// Flow-level retry re-enters `do` on the same journal session —
|
|
687
|
+
// rewind so completed steps/effects replay instead of re-executing.
|
|
688
|
+
journalSession?.rewind();
|
|
689
|
+
return flowDef.do(input as never, fx);
|
|
690
|
+
};
|
|
691
|
+
return flowDef.retry ? fxRetry(run, flowDef.retry) : run();
|
|
692
|
+
};
|
|
683
693
|
if (!alreadyValidated) {
|
|
684
694
|
const parsed = await validate(flowDef.in, ctx.input);
|
|
685
695
|
if (!parsed.ok) return parsed.failure;
|
|
686
696
|
ctx.input = parsed.value;
|
|
687
|
-
return await
|
|
697
|
+
return await invoke(parsed.value);
|
|
688
698
|
}
|
|
689
|
-
return await
|
|
699
|
+
return await invoke(ctx.input);
|
|
690
700
|
} catch (err) {
|
|
691
701
|
// A durable sleep that has not yet elapsed suspends the run — this
|
|
692
702
|
// is a park, not a pipeline failure, so it must not throw upward.
|