okengine 0.3.6 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +2 -0
- package/package.json +12 -12
- package/site/content/docs/ai/skills.mdx +5 -3
- package/site/content/docs/elements/ai.mdx +2 -0
- package/site/content/docs/elements/channel.mdx +2 -0
- package/site/content/docs/elements/clock.mdx +1 -4
- package/site/content/docs/elements/flow.mdx +4 -10
- package/site/content/docs/elements/gate.mdx +2 -0
- package/site/content/docs/elements/signal.mdx +1 -5
- package/site/content/docs/elements/store.mdx +27 -6
- package/site/content/docs/elements/vault.mdx +10 -11
- package/site/content/docs/get-started/basic-usage.mdx +76 -41
- package/site/content/docs/get-started/installation.mdx +95 -43
- package/site/content/docs/get-started/introduction.mdx +128 -75
- package/site/content/docs/get-started/meta.json +1 -1
- package/site/content/docs/get-started/why.mdx +141 -0
- package/site/content/docs/plugins/ip-allowlist.mdx +1 -2
- package/site/content/docs/plugins/security-headers.mdx +1 -1
- package/site/content/docs/reference/configuration.mdx +1 -1
- package/site/content/docs/reference/environment-variables.mdx +5 -3
- package/site/content/docs/reference/fx.mdx +16 -3
- package/src/cli/competitor-mention-removal.test.ts +117 -0
- package/src/cli/dev.ts +20 -0
- package/src/cli/meilisearch-local.test.ts +69 -0
- package/src/cli/meilisearch-local.ts +188 -0
- package/src/compiler/fixtures/skyport/src/flows/payments/index.ts +5 -1
- package/src/console/server/store.test.ts +1 -1
- package/src/console/server/store.ts +11 -1
- package/src/console/ui/dist/assets/index-CjxwRGVv.js +10 -0
- package/src/console/ui/dist/assets/panel-access-BGv45snf.js +64 -0
- package/src/console/ui/dist/assets/{panel-ai-D_m6WQI8.js → panel-ai-B2S7LEii.js} +1 -1
- package/src/console/ui/dist/assets/{panel-architecture-CKnXFyUx.js → panel-architecture-D7UJh91v.js} +1 -1
- package/src/console/ui/dist/assets/{panel-channels-DCDd4WAC.js → panel-channels-9T3ybqRu.js} +1 -1
- package/src/console/ui/dist/assets/panel-clock-Cb1UXGRQ.js +1 -0
- package/src/console/ui/dist/assets/{panel-diff-cdonmH8c.js → panel-diff-DmYbKWmN.js} +1 -1
- package/src/console/ui/dist/assets/panel-flows-PiHwT55z.js +48 -0
- package/src/console/ui/dist/assets/{panel-gates-B5eTE8XH.js → panel-gates-BQGYXvjT.js} +1 -1
- package/src/console/ui/dist/assets/panel-overview-BBnRO18l.js +1 -0
- package/src/console/ui/dist/assets/{panel-plugins-Cj7DK1er.js → panel-plugins-D0PsmVw2.js} +1 -1
- package/src/console/ui/dist/assets/panel-runs-CWuRDe0r.js +1 -0
- package/src/console/ui/dist/assets/{panel-signals-whmDXIg3.js → panel-signals-Bbg4ewpP.js} +1 -1
- package/src/console/ui/dist/assets/{panel-store-CEMHLvaw.js → panel-store-CPCbsDRa.js} +1 -1
- package/src/console/ui/dist/assets/panel-traces-DVAzuA_S.js +1 -0
- package/src/console/ui/dist/assets/{panel-vault-C9wjbki8.js → panel-vault-D1_MvOmo.js} +1 -1
- package/src/console/ui/dist/assets/{rolldown-runtime-CNC7AqOf.js → rolldown-runtime-B0Z9INg1.js} +1 -1
- package/src/console/ui/dist/index.html +2 -2
- package/src/docker/compose.ts +5 -0
- package/src/docker/docker.test.ts +41 -0
- package/src/docker/recipes/index.ts +10 -2
- package/src/docker/recipes/meilisearch.ts +31 -0
- package/src/drivers/conformance.test.ts +16 -1
- package/src/drivers/conformance.ts +40 -3
- package/src/drivers/index.ts +14 -2
- package/src/drivers/libsql.ts +4 -4
- package/src/drivers/meilisearch.integration.test.ts +77 -0
- package/src/drivers/meilisearch.test.ts +181 -0
- package/src/drivers/meilisearch.ts +208 -0
- package/src/drivers/memory.ts +4 -4
- package/src/drivers/pgvector.ts +6 -6
- package/src/drivers/types.ts +93 -12
- package/src/drivers/vault-driver-removal.test.ts +6 -0
- package/src/drivers/vault-types.ts +4 -4
- package/src/elements/ai/runtime.ts +6 -0
- package/src/elements/ai.test.ts +22 -0
- package/src/elements/store/index-boot.test.ts +49 -7
- package/src/elements/store/runtime.ts +50 -15
- package/src/elements/store.ts +2 -0
- package/src/elements/vault.test.ts +27 -4
- package/src/elements/vault.ts +1 -1
- package/src/index.ts +4 -0
- package/src/kernel/boot-bind/store.test.ts +9 -0
- package/src/kernel/boot-bind/store.ts +30 -2
- package/src/kernel/concurrency.test.ts +58 -0
- package/src/kernel/concurrency.ts +48 -0
- package/src/kernel/fx.test.ts +11 -2
- package/src/kernel/fx.ts +37 -5
- package/src/kernel/index.ts +10 -1
- package/src/kernel/redacted.ts +74 -0
- package/src/kernel/router.ts +3 -3
- package/src/test/provisions.integration.test.ts +1 -1
- package/site/content/docs/get-started/comparison.mdx +0 -65
- package/src/console/ui/dist/assets/index-CrKMmO__.js +0 -10
- package/src/console/ui/dist/assets/panel-access-C0J2D-a2.js +0 -64
- package/src/console/ui/dist/assets/panel-clock-DjGGFPzr.js +0 -1
- package/src/console/ui/dist/assets/panel-flows-DlCU5zjA.js +0 -45
- package/src/console/ui/dist/assets/panel-overview-BsFvDdts.js +0 -1
- package/src/console/ui/dist/assets/panel-runs-C0gmnoYL.js +0 -1
- package/src/console/ui/dist/assets/panel-traces-BDiAuVSK.js +0 -1
- package/src/drivers/vault-infisical.ts +0 -57
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `meilisearch` index driver — full-text search over the Meilisearch HTTP API.
|
|
3
|
+
*
|
|
4
|
+
* Product-as-protocol id (same pattern as `openbao` / `pgvector`); the image
|
|
5
|
+
* pin lives in `images["store.index"]` (e.g. `getmeili/meilisearch`). This is
|
|
6
|
+
* a genuinely different capability from the vector ANN drivers — typo-tolerant
|
|
7
|
+
* / faceted full-text relevance, not cosine similarity.
|
|
8
|
+
*
|
|
9
|
+
* Fail-loud: a configured Meilisearch that is unreachable, unhealthy, or
|
|
10
|
+
* rejects a call throws {@link MeilisearchUnavailableError} — never silently
|
|
11
|
+
* falls back to memory. Writes wait for their task to succeed (Meilisearch is
|
|
12
|
+
* asynchronous), so an `upsert`/`delete` only resolves once the server has
|
|
13
|
+
* actually indexed it.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import type {
|
|
17
|
+
IndexHit,
|
|
18
|
+
IndexOpenOptions,
|
|
19
|
+
TextIndexDriver,
|
|
20
|
+
TextIndexSearchOptions,
|
|
21
|
+
TextIndexSearchResult,
|
|
22
|
+
TextIndexStore,
|
|
23
|
+
} from "./types.ts";
|
|
24
|
+
|
|
25
|
+
/** Error thrown when the remote Meilisearch is unreachable / unhealthy / errors. */
|
|
26
|
+
export class MeilisearchUnavailableError extends Error {
|
|
27
|
+
constructor(message: string) {
|
|
28
|
+
super(message);
|
|
29
|
+
this.name = "MeilisearchUnavailableError";
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** Minimal response shape of a Meilisearch summarized / finished task. */
|
|
34
|
+
interface MeiliTask {
|
|
35
|
+
readonly taskUid: number;
|
|
36
|
+
readonly status: "enqueued" | "processing" | "succeeded" | "failed" | "canceled";
|
|
37
|
+
readonly error?: { readonly message?: string } | null;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** Search response hit shape we consume. */
|
|
41
|
+
interface MeiliSearchResponse {
|
|
42
|
+
readonly hits?: ReadonlyArray<Record<string, unknown> & { readonly _rankingScore?: number }>;
|
|
43
|
+
readonly facetDistribution?: Record<string, Record<string, number>>;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** Options for opening a Meilisearch-backed text index. */
|
|
47
|
+
export interface MeilisearchIndexOptions extends IndexOpenOptions {
|
|
48
|
+
/** Max ms to wait for an indexing task to finish (default 5000). */
|
|
49
|
+
readonly taskTimeoutMs?: number;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Open a full-text index.
|
|
54
|
+
*
|
|
55
|
+
* Ensures the index exists, health-checks the server, and returns a handle
|
|
56
|
+
* whose writes are confirmed (task-waited) and whose search maps
|
|
57
|
+
* `_rankingScore` into `IndexHit.score` as relevance.
|
|
58
|
+
*
|
|
59
|
+
* @param options - name / url / apiKey / injected fetch
|
|
60
|
+
*/
|
|
61
|
+
export async function openMeilisearchIndex(
|
|
62
|
+
options: MeilisearchIndexOptions,
|
|
63
|
+
): Promise<TextIndexStore> {
|
|
64
|
+
const url = options.url?.replace(/\/$/, "");
|
|
65
|
+
if (!url) {
|
|
66
|
+
throw new MeilisearchUnavailableError(
|
|
67
|
+
"meilisearch index: needs an HTTP URL (OKE_STORE_INDEX_URL) — no silent memory fallback",
|
|
68
|
+
);
|
|
69
|
+
}
|
|
70
|
+
const apiKey = options.apiKey;
|
|
71
|
+
const fetchFn = options.fetch ?? globalThis.fetch;
|
|
72
|
+
const taskTimeoutMs = options.taskTimeoutMs ?? 5_000;
|
|
73
|
+
const uid = encodeURIComponent(options.name);
|
|
74
|
+
|
|
75
|
+
const headers: Record<string, string> = { "content-type": "application/json" };
|
|
76
|
+
if (apiKey) headers.Authorization = `Bearer ${apiKey}`;
|
|
77
|
+
|
|
78
|
+
async function request(path: string, init?: RequestInit): Promise<Response> {
|
|
79
|
+
let res: Response;
|
|
80
|
+
try {
|
|
81
|
+
res = await fetchFn(`${url}${path}`, {
|
|
82
|
+
...init,
|
|
83
|
+
headers: { ...headers, ...(init?.headers ?? {}) },
|
|
84
|
+
});
|
|
85
|
+
} catch (err) {
|
|
86
|
+
throw new MeilisearchUnavailableError(
|
|
87
|
+
`meilisearch index: unreachable at ${url} — ${err instanceof Error ? err.message : String(err)}`,
|
|
88
|
+
);
|
|
89
|
+
}
|
|
90
|
+
return res;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
async function requestOk(path: string, init?: RequestInit): Promise<Response> {
|
|
94
|
+
const res = await request(path, init);
|
|
95
|
+
if (!res.ok) {
|
|
96
|
+
const detail = await res.text().catch(() => "");
|
|
97
|
+
throw new MeilisearchUnavailableError(
|
|
98
|
+
`meilisearch index: ${init?.method ?? "GET"} ${path} failed (${res.status})${detail ? ` — ${detail.slice(0, 200)}` : ""}`,
|
|
99
|
+
);
|
|
100
|
+
}
|
|
101
|
+
return res;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
async function enqueue(path: string, init: RequestInit): Promise<MeiliTask> {
|
|
105
|
+
const res = await requestOk(path, init);
|
|
106
|
+
return (await res.json()) as MeiliTask;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
async function waitForTask(taskUid: number): Promise<void> {
|
|
110
|
+
const deadline = Date.now() + taskTimeoutMs;
|
|
111
|
+
// Poll the task until it leaves the queue; treat a failed/canceled task as
|
|
112
|
+
// a loud error rather than an optimistic write.
|
|
113
|
+
for (;;) {
|
|
114
|
+
const res = await requestOk(`/tasks/${taskUid}`);
|
|
115
|
+
const task = (await res.json()) as MeiliTask;
|
|
116
|
+
if (task.status === "succeeded") return;
|
|
117
|
+
if (task.status === "failed" || task.status === "canceled") {
|
|
118
|
+
throw new MeilisearchUnavailableError(
|
|
119
|
+
`meilisearch index: task ${taskUid} ${task.status}${task.error?.message ? ` — ${task.error.message}` : ""}`,
|
|
120
|
+
);
|
|
121
|
+
}
|
|
122
|
+
if (Date.now() > deadline) {
|
|
123
|
+
throw new MeilisearchUnavailableError(
|
|
124
|
+
`meilisearch index: task ${taskUid} did not finish within ${taskTimeoutMs}ms`,
|
|
125
|
+
);
|
|
126
|
+
}
|
|
127
|
+
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
// Fail loud up front: server must be healthy and the index must exist
|
|
132
|
+
// (created if absent) before we hand back a usable handle.
|
|
133
|
+
await requestOk("/health");
|
|
134
|
+
const created = await enqueue("/indexes", {
|
|
135
|
+
method: "POST",
|
|
136
|
+
body: JSON.stringify({ uid: options.name, primaryKey: "id" }),
|
|
137
|
+
});
|
|
138
|
+
await waitForTask(created.taskUid);
|
|
139
|
+
|
|
140
|
+
const scoreOf = (hit: { readonly _rankingScore?: unknown }): number =>
|
|
141
|
+
typeof hit._rankingScore === "number" ? hit._rankingScore : 0;
|
|
142
|
+
|
|
143
|
+
return {
|
|
144
|
+
driverId: "meilisearch",
|
|
145
|
+
async upsert(id, document) {
|
|
146
|
+
const body = [{ ...document, id }];
|
|
147
|
+
const task = await enqueue(`/indexes/${uid}/documents`, {
|
|
148
|
+
method: "POST",
|
|
149
|
+
body: JSON.stringify(body),
|
|
150
|
+
});
|
|
151
|
+
await waitForTask(task.taskUid);
|
|
152
|
+
},
|
|
153
|
+
async search(q, opts?: TextIndexSearchOptions): Promise<TextIndexSearchResult> {
|
|
154
|
+
const payload: Record<string, unknown> = {
|
|
155
|
+
q,
|
|
156
|
+
limit: opts?.topK ?? 10,
|
|
157
|
+
showRankingScore: true,
|
|
158
|
+
};
|
|
159
|
+
if (opts?.filter !== undefined) payload.filter = opts.filter;
|
|
160
|
+
if (opts?.facets !== undefined) payload.facets = opts.facets;
|
|
161
|
+
const res = await requestOk(`/indexes/${uid}/search`, {
|
|
162
|
+
method: "POST",
|
|
163
|
+
body: JSON.stringify(payload),
|
|
164
|
+
});
|
|
165
|
+
const body = (await res.json()) as MeiliSearchResponse;
|
|
166
|
+
const hits: IndexHit[] = (body.hits ?? []).map((hit) => {
|
|
167
|
+
const { _rankingScore, ...rest } = hit;
|
|
168
|
+
void _rankingScore;
|
|
169
|
+
const meta = Object.fromEntries(Object.entries(rest).filter(([key]) => key !== "id"));
|
|
170
|
+
return {
|
|
171
|
+
id: String(rest.id ?? ""),
|
|
172
|
+
score: scoreOf(hit),
|
|
173
|
+
...(Object.keys(meta).length > 0 ? { meta } : {}),
|
|
174
|
+
};
|
|
175
|
+
});
|
|
176
|
+
return {
|
|
177
|
+
hits,
|
|
178
|
+
...(body.facetDistribution !== undefined
|
|
179
|
+
? { facetDistribution: body.facetDistribution }
|
|
180
|
+
: {}),
|
|
181
|
+
};
|
|
182
|
+
},
|
|
183
|
+
async delete(id) {
|
|
184
|
+
const res = await request(`/indexes/${uid}/documents/${encodeURIComponent(id)}`, {
|
|
185
|
+
method: "DELETE",
|
|
186
|
+
});
|
|
187
|
+
if (res.status === 404) return false;
|
|
188
|
+
if (!res.ok) {
|
|
189
|
+
throw new MeilisearchUnavailableError(
|
|
190
|
+
`meilisearch index: delete ${id} failed (${res.status})`,
|
|
191
|
+
);
|
|
192
|
+
}
|
|
193
|
+
const task = (await res.json()) as MeiliTask;
|
|
194
|
+
await waitForTask(task.taskUid);
|
|
195
|
+
return true;
|
|
196
|
+
},
|
|
197
|
+
async close() {
|
|
198
|
+
// Stateless HTTP client — nothing to release.
|
|
199
|
+
},
|
|
200
|
+
};
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/** Protocol-named Meilisearch driver. */
|
|
204
|
+
export const meilisearchDriver: TextIndexDriver = {
|
|
205
|
+
id: "meilisearch",
|
|
206
|
+
facet: "index",
|
|
207
|
+
open: openMeilisearchIndex,
|
|
208
|
+
};
|
package/src/drivers/memory.ts
CHANGED
|
@@ -7,10 +7,10 @@ import type {
|
|
|
7
7
|
FilesBucket,
|
|
8
8
|
FilesDriver,
|
|
9
9
|
FilesOpenOptions,
|
|
10
|
-
IndexDriver,
|
|
11
10
|
IndexHit,
|
|
12
11
|
IndexOpenOptions,
|
|
13
|
-
|
|
12
|
+
VectorIndexDriver,
|
|
13
|
+
VectorIndexStore,
|
|
14
14
|
KvDriver,
|
|
15
15
|
KvNamespace,
|
|
16
16
|
KvOpenOptions,
|
|
@@ -638,10 +638,10 @@ function cosine(a: readonly number[], b: readonly number[]): number {
|
|
|
638
638
|
}
|
|
639
639
|
|
|
640
640
|
/** Memory index driver. */
|
|
641
|
-
export const memoryIndexDriver:
|
|
641
|
+
export const memoryIndexDriver: VectorIndexDriver = {
|
|
642
642
|
id: "memory",
|
|
643
643
|
facet: "index",
|
|
644
|
-
async open(options: IndexOpenOptions): Promise<
|
|
644
|
+
async open(options: IndexOpenOptions): Promise<VectorIndexStore> {
|
|
645
645
|
const docs = new Map<string, { vector: number[]; meta?: Record<string, unknown> }>();
|
|
646
646
|
return {
|
|
647
647
|
driverId: "memory",
|
package/src/drivers/pgvector.ts
CHANGED
|
@@ -14,11 +14,11 @@
|
|
|
14
14
|
import { cosineDistance } from "drizzle-orm";
|
|
15
15
|
import { PgDialect, index as pgIndex, pgTable, text, vector } from "drizzle-orm/pg-core";
|
|
16
16
|
import type {
|
|
17
|
-
IndexDriver,
|
|
18
17
|
IndexHit,
|
|
19
18
|
IndexOpenOptions,
|
|
20
|
-
IndexStore,
|
|
21
19
|
SqlConnection,
|
|
20
|
+
VectorIndexDriver,
|
|
21
|
+
VectorIndexStore,
|
|
22
22
|
} from "./types.ts";
|
|
23
23
|
|
|
24
24
|
/**
|
|
@@ -26,7 +26,7 @@ import type {
|
|
|
26
26
|
*
|
|
27
27
|
* @param options - Name / dims / SQL connection
|
|
28
28
|
*/
|
|
29
|
-
export async function openPgvectorIndex(options: IndexOpenOptions): Promise<
|
|
29
|
+
export async function openPgvectorIndex(options: IndexOpenOptions): Promise<VectorIndexStore> {
|
|
30
30
|
if (options.sql) {
|
|
31
31
|
return openPgvectorSql(options.name, options.dims, options.sql);
|
|
32
32
|
}
|
|
@@ -40,7 +40,7 @@ async function openPgvectorSql(
|
|
|
40
40
|
name: string,
|
|
41
41
|
dims: number,
|
|
42
42
|
sql: SqlConnection,
|
|
43
|
-
): Promise<
|
|
43
|
+
): Promise<VectorIndexStore> {
|
|
44
44
|
if (sql.driverId !== "postgres" && sql.driverId !== "pglite") {
|
|
45
45
|
throw new Error(
|
|
46
46
|
`pgvector index: needs a postgres/pglite SQL connection, got "${sql.driverId}" — ` +
|
|
@@ -111,7 +111,7 @@ async function openPgvectorSql(
|
|
|
111
111
|
};
|
|
112
112
|
}
|
|
113
113
|
|
|
114
|
-
function openPgvectorMemory(dims: number):
|
|
114
|
+
function openPgvectorMemory(dims: number): VectorIndexStore {
|
|
115
115
|
const docs = new Map<string, { vector: number[]; meta?: Record<string, unknown> }>();
|
|
116
116
|
return {
|
|
117
117
|
driverId: "pgvector",
|
|
@@ -154,7 +154,7 @@ function cosine(a: readonly number[], b: readonly number[]): number {
|
|
|
154
154
|
}
|
|
155
155
|
|
|
156
156
|
/** Protocol-named pgvector driver. */
|
|
157
|
-
export const pgvectorDriver:
|
|
157
|
+
export const pgvectorDriver: VectorIndexDriver = {
|
|
158
158
|
id: "pgvector",
|
|
159
159
|
facet: "index",
|
|
160
160
|
open: openPgvectorIndex,
|
package/src/drivers/types.ts
CHANGED
|
@@ -17,7 +17,8 @@ export type StoreDriverId =
|
|
|
17
17
|
| "redis"
|
|
18
18
|
| "fs"
|
|
19
19
|
| "s3"
|
|
20
|
-
| "pgvector"
|
|
20
|
+
| "pgvector"
|
|
21
|
+
| "meilisearch";
|
|
21
22
|
|
|
22
23
|
/** Facets a driver may serve. */
|
|
23
24
|
export type DriverFacet = StoreFacet;
|
|
@@ -248,15 +249,19 @@ export interface FilesDriver {
|
|
|
248
249
|
open(options: FilesOpenOptions): Promise<FilesBucket>;
|
|
249
250
|
}
|
|
250
251
|
|
|
251
|
-
/**
|
|
252
|
+
/** Index hit — score meaning depends on the driver. */
|
|
252
253
|
export interface IndexHit {
|
|
253
254
|
readonly id: string;
|
|
255
|
+
/**
|
|
256
|
+
* Vector drivers: cosine similarity (higher = more similar).
|
|
257
|
+
* Full-text (`meilisearch`): relevance score (`_rankingScore`, higher = more relevant).
|
|
258
|
+
*/
|
|
254
259
|
readonly score: number;
|
|
255
260
|
readonly meta?: Record<string, unknown>;
|
|
256
261
|
}
|
|
257
262
|
|
|
258
|
-
/** Vector index handle. */
|
|
259
|
-
export interface
|
|
263
|
+
/** Vector (ANN) index handle. */
|
|
264
|
+
export interface VectorIndexStore {
|
|
260
265
|
readonly driverId: "memory" | "pgvector" | "libsql";
|
|
261
266
|
/**
|
|
262
267
|
* Upsert a vector.
|
|
@@ -272,7 +277,7 @@ export interface IndexStore {
|
|
|
272
277
|
* @param vector - Query embedding
|
|
273
278
|
* @param topK - Max hits
|
|
274
279
|
*/
|
|
275
|
-
search(vector: readonly number[], topK?: number): Promise<IndexHit[]>;
|
|
280
|
+
search(vector: readonly number[], topK?: number): Promise<readonly IndexHit[]>;
|
|
276
281
|
/**
|
|
277
282
|
* Delete a vector.
|
|
278
283
|
*
|
|
@@ -283,28 +288,104 @@ export interface IndexStore {
|
|
|
283
288
|
close(): Promise<void>;
|
|
284
289
|
}
|
|
285
290
|
|
|
291
|
+
/** Full-text search options (meilisearch). */
|
|
292
|
+
export interface TextIndexSearchOptions {
|
|
293
|
+
/** Max hits (Meilisearch `limit`). */
|
|
294
|
+
readonly topK?: number;
|
|
295
|
+
/** Filter expression (attribute must be filterable). */
|
|
296
|
+
readonly filter?: string | string[];
|
|
297
|
+
/** Facet attributes to aggregate. */
|
|
298
|
+
readonly facets?: string[];
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
/** Full-text search result (meilisearch). */
|
|
302
|
+
export interface TextIndexSearchResult {
|
|
303
|
+
readonly hits: readonly IndexHit[];
|
|
304
|
+
/** Facet counts per attribute, when `facets` was requested. */
|
|
305
|
+
readonly facetDistribution?: Record<string, Record<string, number>>;
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
/** Full-text index handle. */
|
|
309
|
+
export interface TextIndexStore {
|
|
310
|
+
readonly driverId: "meilisearch";
|
|
311
|
+
/**
|
|
312
|
+
* Upsert a document.
|
|
313
|
+
*
|
|
314
|
+
* @param id - Document id
|
|
315
|
+
* @param document - Field map (must include the id field)
|
|
316
|
+
*/
|
|
317
|
+
upsert(id: string, document: Record<string, unknown>): Promise<void>;
|
|
318
|
+
/**
|
|
319
|
+
* Full-text search.
|
|
320
|
+
*
|
|
321
|
+
* @param q - Query text (typo tolerant)
|
|
322
|
+
* @param opts - topK / filter / facets
|
|
323
|
+
*/
|
|
324
|
+
search(q: string, opts?: TextIndexSearchOptions): Promise<TextIndexSearchResult>;
|
|
325
|
+
/**
|
|
326
|
+
* Delete a document.
|
|
327
|
+
*
|
|
328
|
+
* @param id - Document id
|
|
329
|
+
*/
|
|
330
|
+
delete(id: string): Promise<boolean>;
|
|
331
|
+
/** Close / release. */
|
|
332
|
+
close(): Promise<void>;
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
/** Index handle — discriminated by `driverId`. */
|
|
336
|
+
export type IndexStore = VectorIndexStore | TextIndexStore;
|
|
337
|
+
|
|
286
338
|
/** Options when opening an index. */
|
|
287
339
|
export interface IndexOpenOptions {
|
|
288
340
|
/** Index name. */
|
|
289
341
|
readonly name: string;
|
|
290
|
-
/** Vector dimensions. */
|
|
342
|
+
/** Vector dimensions (vector drivers only). */
|
|
291
343
|
readonly dims: number;
|
|
292
|
-
/**
|
|
344
|
+
/**
|
|
345
|
+
* Vector: SQL URL when the driver shares the sql facet's engine.
|
|
346
|
+
* Meilisearch: base HTTP URL.
|
|
347
|
+
*/
|
|
293
348
|
readonly url?: string;
|
|
294
|
-
/** Injected SQL connection — shared with `store.sql` at boot. */
|
|
349
|
+
/** Injected SQL connection — shared with `store.sql` at boot (vector only). */
|
|
295
350
|
readonly sql?: SqlConnection;
|
|
351
|
+
/** Meilisearch API / master key. */
|
|
352
|
+
readonly apiKey?: string;
|
|
353
|
+
/** Injected fetch (tests). */
|
|
354
|
+
readonly fetch?: typeof fetch;
|
|
296
355
|
}
|
|
297
356
|
|
|
298
|
-
/**
|
|
299
|
-
export interface
|
|
357
|
+
/** Vector index driver factory. */
|
|
358
|
+
export interface VectorIndexDriver {
|
|
300
359
|
readonly id: "memory" | "pgvector" | "libsql";
|
|
301
360
|
readonly facet: "index";
|
|
302
361
|
/**
|
|
303
|
-
* Open
|
|
362
|
+
* Open a vector index.
|
|
304
363
|
*
|
|
305
364
|
* @param options - Name / dims / SQL
|
|
306
365
|
*/
|
|
307
|
-
open(options: IndexOpenOptions): Promise<
|
|
366
|
+
open(options: IndexOpenOptions): Promise<VectorIndexStore>;
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
/** Full-text index driver factory. */
|
|
370
|
+
export interface TextIndexDriver {
|
|
371
|
+
readonly id: "meilisearch";
|
|
372
|
+
readonly facet: "index";
|
|
373
|
+
/**
|
|
374
|
+
* Open a full-text index.
|
|
375
|
+
*
|
|
376
|
+
* @param options - Name / HTTP URL / apiKey
|
|
377
|
+
*/
|
|
378
|
+
open(options: IndexOpenOptions): Promise<TextIndexStore>;
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
/** Index driver factory — discriminated by `id`. */
|
|
382
|
+
export type IndexDriver = VectorIndexDriver | TextIndexDriver;
|
|
383
|
+
|
|
384
|
+
/** True when the index driver shares the sql facet's engine (never memory). */
|
|
385
|
+
export function indexDriverNeedsSql(
|
|
386
|
+
driver: IndexDriver,
|
|
387
|
+
): driver is VectorIndexDriver & { id: "pgvector" | "libsql" } {
|
|
388
|
+
return driver.id === "pgvector" || driver.id === "libsql";
|
|
308
389
|
}
|
|
309
390
|
|
|
310
391
|
/** Union of all store drivers. */
|
|
@@ -12,6 +12,8 @@ const ROOT = join(import.meta.dir, "../..");
|
|
|
12
12
|
|
|
13
13
|
/** Forbidden driver id, split so this file does not match itself. */
|
|
14
14
|
const FORBIDDEN_DRIVER = ["so", "ps"].join("");
|
|
15
|
+
/** Forbidden vault driver id, split so this file does not match itself. */
|
|
16
|
+
const FORBIDDEN_SECRETS_SAAS = ["infi", "sical"].join("");
|
|
15
17
|
/** Forbidden optional peer, split so this file does not match itself. */
|
|
16
18
|
const FORBIDDEN_AGE = ["age", "-", "encryption"].join("");
|
|
17
19
|
|
|
@@ -49,6 +51,10 @@ describe("removed vault driver gate", () => {
|
|
|
49
51
|
assertZeroGitGrep(FORBIDDEN_DRIVER);
|
|
50
52
|
});
|
|
51
53
|
|
|
54
|
+
test("tracked tree has zero matches for the removed secrets-saas driver id", () => {
|
|
55
|
+
assertZeroGitGrep(FORBIDDEN_SECRETS_SAAS, ["changelog.md:"]);
|
|
56
|
+
});
|
|
57
|
+
|
|
52
58
|
test("tracked tree has zero matches for the removed age peer outside lockfiles", () => {
|
|
53
59
|
assertZeroGitGrep(FORBIDDEN_AGE, LOCKFILE_PREFIXES);
|
|
54
60
|
});
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Protocol-named vault driver contracts.
|
|
3
3
|
*
|
|
4
|
-
* Driver ids: `env` · `openbao` · `
|
|
5
|
-
* Never vendor names (`neon`,
|
|
4
|
+
* Driver ids: `env` · `openbao` · `managed` · `memory`.
|
|
5
|
+
* Never vendor names (`neon`, …) — vendor choice lives in `images`.
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
8
|
/** Protocol ids for vault drivers. */
|
|
9
|
-
export type VaultDriverId = "env" | "openbao" | "
|
|
9
|
+
export type VaultDriverId = "env" | "openbao" | "managed" | "memory";
|
|
10
10
|
|
|
11
11
|
/** Options when opening a vault backend. */
|
|
12
12
|
export interface VaultOpenOptions {
|
|
@@ -16,7 +16,7 @@ export interface VaultOpenOptions {
|
|
|
16
16
|
readonly envPrefix?: string;
|
|
17
17
|
/** Injected env map for tests (defaults to `process.env`). */
|
|
18
18
|
readonly env?: Readonly<Record<string, string | undefined>>;
|
|
19
|
-
/** OpenBao /
|
|
19
|
+
/** OpenBao / managed base URL. */
|
|
20
20
|
readonly url?: string;
|
|
21
21
|
/** Token / API key for remote vaults. */
|
|
22
22
|
readonly token?: string;
|
|
@@ -485,6 +485,12 @@ export function createAiRuntime(options: CreateAiRuntimeOptions = {}): AiRuntime
|
|
|
485
485
|
if (!index) {
|
|
486
486
|
throw new Error(`ai: index "${decl.into}" not registered`);
|
|
487
487
|
}
|
|
488
|
+
if (index.driverId === "meilisearch") {
|
|
489
|
+
throw new Error(
|
|
490
|
+
`ai: embed into "${decl.into}" needs a vector index (memory/pgvector/libsql) — ` +
|
|
491
|
+
`"${index.driverId}" is full-text; embeddings don't apply`,
|
|
492
|
+
);
|
|
493
|
+
}
|
|
488
494
|
const modelName = decl.model ?? [...models.keys()][0] ?? "mock";
|
|
489
495
|
const client = await clientFor(modelName);
|
|
490
496
|
if (!client.embed) {
|
package/src/elements/ai.test.ts
CHANGED
|
@@ -263,6 +263,28 @@ describe("embeddings into store.index", () => {
|
|
|
263
263
|
const hits = await index.search((await client.embed!({ input: "hello world" })).vectors[0]!, 1);
|
|
264
264
|
expect(hits[0]?.id).toBe("doc-1");
|
|
265
265
|
});
|
|
266
|
+
|
|
267
|
+
test("embed into a meilisearch (full-text) index fails loud", async () => {
|
|
268
|
+
const smart = ai.model("smart", { provider: "mock" });
|
|
269
|
+
const embed = ai.embed("docs", { model: smart, into: "kb" });
|
|
270
|
+
const client = await mockAiDriver.open({ model: "mock" });
|
|
271
|
+
const textIndex = {
|
|
272
|
+
driverId: "meilisearch" as const,
|
|
273
|
+
upsert: async () => {},
|
|
274
|
+
search: async () => ({ hits: [] }),
|
|
275
|
+
delete: async () => true,
|
|
276
|
+
close: async () => {},
|
|
277
|
+
};
|
|
278
|
+
const runtime = createAiRuntime({
|
|
279
|
+
models: [smart],
|
|
280
|
+
embeds: [embed],
|
|
281
|
+
clients: { smart: client },
|
|
282
|
+
indexes: { kb: textIndex as never },
|
|
283
|
+
});
|
|
284
|
+
await expect(runtime.embed("docs", "doc-1", "hello world")).rejects.toThrow(
|
|
285
|
+
/needs a vector index/,
|
|
286
|
+
);
|
|
287
|
+
});
|
|
266
288
|
});
|
|
267
289
|
|
|
268
290
|
describe("model fallback chain", () => {
|
|
@@ -22,8 +22,8 @@ import {
|
|
|
22
22
|
createStoreRuntime,
|
|
23
23
|
index as declareIndex,
|
|
24
24
|
sql as declareSql,
|
|
25
|
-
type IndexStoreFxHandle,
|
|
26
25
|
type SqlStoreHandle,
|
|
26
|
+
type VectorIndexStoreFxHandle,
|
|
27
27
|
} from "../store.ts";
|
|
28
28
|
|
|
29
29
|
const WRITE_CTX = (name: string) => ({
|
|
@@ -67,7 +67,7 @@ describe("store.index boot wiring — memory", () => {
|
|
|
67
67
|
test("boot with no configured index driver resolves memory end to end", async () => {
|
|
68
68
|
const kb = declareIndex("kb", { dims: 3 });
|
|
69
69
|
const runtime = bindStore({ stores: [kb] }, "local", () => Date.now());
|
|
70
|
-
const handle = (await runtime.open(kb, WRITE_CTX("kb"))) as
|
|
70
|
+
const handle = (await runtime.open(kb, WRITE_CTX("kb"))) as VectorIndexStoreFxHandle;
|
|
71
71
|
expect(handle.driverId).toBe("memory");
|
|
72
72
|
await handle.upsert("near", [1, 0, 0]);
|
|
73
73
|
await handle.upsert("far", [0, 1, 0]);
|
|
@@ -89,14 +89,14 @@ describe("store.index boot wiring — libsql", () => {
|
|
|
89
89
|
"local",
|
|
90
90
|
() => Date.now(),
|
|
91
91
|
);
|
|
92
|
-
const handle = (await runtime.open(kb, WRITE_CTX("kb"))) as
|
|
92
|
+
const handle = (await runtime.open(kb, WRITE_CTX("kb"))) as VectorIndexStoreFxHandle;
|
|
93
93
|
expect(handle.driverId).toBe("libsql");
|
|
94
94
|
|
|
95
95
|
await handle.upsert("near", [1, 0.1, 0], { label: "near" });
|
|
96
96
|
await handle.upsert("far", [0, 1, 0]);
|
|
97
97
|
await handle.upsert("farther", [-1, 0, 0]);
|
|
98
98
|
const hits = await handle.search([1, 0, 0], 3);
|
|
99
|
-
expect(hits.map((h) => h.id)).toEqual(["near", "far", "farther"]);
|
|
99
|
+
expect(hits.map((h: { id: string }) => h.id)).toEqual(["near", "far", "farther"]);
|
|
100
100
|
expect(hits[0]!.score).toBeGreaterThan(hits[1]!.score);
|
|
101
101
|
expect(hits[0]!.meta).toEqual({ label: "near" });
|
|
102
102
|
expect(await handle.delete("farther")).toBe(true);
|
|
@@ -120,7 +120,7 @@ describe("store.index boot wiring — libsql", () => {
|
|
|
120
120
|
index: { kb: { url: ":memory:" } },
|
|
121
121
|
});
|
|
122
122
|
// Index first: it must open the connection the sql facet then reuses.
|
|
123
|
-
const idxHandle = (await runtime.open(kb, WRITE_CTX("kb"))) as
|
|
123
|
+
const idxHandle = (await runtime.open(kb, WRITE_CTX("kb"))) as VectorIndexStoreFxHandle;
|
|
124
124
|
await idxHandle.upsert("a", [1, 0, 0]);
|
|
125
125
|
const sqlHandle = (await runtime.open(docs, {
|
|
126
126
|
effects: { reads: ["sql:docs"], writes: ["sql:docs"] },
|
|
@@ -161,14 +161,14 @@ describe("store.index boot wiring — pglite + pgvector", () => {
|
|
|
161
161
|
"local",
|
|
162
162
|
() => Date.now(),
|
|
163
163
|
);
|
|
164
|
-
const handle = (await runtime.open(kb, WRITE_CTX("kb"))) as
|
|
164
|
+
const handle = (await runtime.open(kb, WRITE_CTX("kb"))) as VectorIndexStoreFxHandle;
|
|
165
165
|
expect(handle.driverId).toBe("pgvector");
|
|
166
166
|
|
|
167
167
|
await handle.upsert("near", [1, 0.1, 0], { label: "near" });
|
|
168
168
|
await handle.upsert("far", [0, 1, 0]);
|
|
169
169
|
await handle.upsert("farther", [-1, 0, 0]);
|
|
170
170
|
const hits = await handle.search([1, 0, 0], 3);
|
|
171
|
-
expect(hits.map((h) => h.id)).toEqual(["near", "far", "farther"]);
|
|
171
|
+
expect(hits.map((h: { id: string }) => h.id)).toEqual(["near", "far", "farther"]);
|
|
172
172
|
expect(hits[0]!.score).toBeGreaterThan(hits[1]!.score);
|
|
173
173
|
expect(hits[0]!.meta).toEqual({ label: "near" });
|
|
174
174
|
expect(await handle.delete("farther")).toBe(true);
|
|
@@ -225,6 +225,48 @@ describe("store.index boot wiring — fail loud", () => {
|
|
|
225
225
|
await expect(runtime.open(kb, WRITE_CTX("kb"))).rejects.toThrow(/cannot share sql driver/);
|
|
226
226
|
await runtime.close();
|
|
227
227
|
});
|
|
228
|
+
|
|
229
|
+
test("meilisearch without OKE_STORE_INDEX_URL throws at bind — never sqlUrl, never memory", () => {
|
|
230
|
+
const prev = process.env.OKE_STORE_INDEX_URL;
|
|
231
|
+
delete process.env.OKE_STORE_INDEX_URL;
|
|
232
|
+
try {
|
|
233
|
+
const kb = declareIndex("kb");
|
|
234
|
+
expect(() =>
|
|
235
|
+
bindStore(
|
|
236
|
+
{ config: { drivers: { store: { index: { local: "meilisearch" } } } }, stores: [kb] },
|
|
237
|
+
"local",
|
|
238
|
+
() => Date.now(),
|
|
239
|
+
),
|
|
240
|
+
).toThrow(/OKE_STORE_INDEX_URL/);
|
|
241
|
+
} finally {
|
|
242
|
+
if (prev === undefined) delete process.env.OKE_STORE_INDEX_URL;
|
|
243
|
+
else process.env.OKE_STORE_INDEX_URL = prev;
|
|
244
|
+
}
|
|
245
|
+
});
|
|
246
|
+
|
|
247
|
+
test("meilisearch binds its own URL + key (never the sql connection)", async () => {
|
|
248
|
+
const prevUrl = process.env.OKE_STORE_INDEX_URL;
|
|
249
|
+
const prevKey = process.env.OKE_STORE_INDEX_KEY;
|
|
250
|
+
process.env.OKE_STORE_INDEX_URL = "http://127.0.0.1:7700";
|
|
251
|
+
process.env.OKE_STORE_INDEX_KEY = "master-key";
|
|
252
|
+
try {
|
|
253
|
+
const kb = declareIndex("kb");
|
|
254
|
+
const runtime = bindStore(
|
|
255
|
+
{ config: { drivers: { store: { index: { local: "meilisearch" } } } }, stores: [kb] },
|
|
256
|
+
"local",
|
|
257
|
+
() => Date.now(),
|
|
258
|
+
);
|
|
259
|
+
// No live server in unit tests: opening must fail loud (unreachable),
|
|
260
|
+
// proving the binding pointed at HTTP, not silently degraded to memory.
|
|
261
|
+
await expect(runtime.open(kb, WRITE_CTX("kb"))).rejects.toThrow(/meilisearch index/);
|
|
262
|
+
await runtime.close();
|
|
263
|
+
} finally {
|
|
264
|
+
if (prevUrl === undefined) delete process.env.OKE_STORE_INDEX_URL;
|
|
265
|
+
else process.env.OKE_STORE_INDEX_URL = prevUrl;
|
|
266
|
+
if (prevKey === undefined) delete process.env.OKE_STORE_INDEX_KEY;
|
|
267
|
+
else process.env.OKE_STORE_INDEX_KEY = prevKey;
|
|
268
|
+
}
|
|
269
|
+
});
|
|
228
270
|
});
|
|
229
271
|
|
|
230
272
|
const PGVECTOR_URL = process.env.OKE_TEST_PGVECTOR_URL;
|