okengine 0.3.5 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +6 -0
- package/manifest.v1.schema.json +43 -1
- package/package.json +27 -12
- package/site/content/docs/ai/skills.mdx +11 -7
- package/site/content/docs/console/vault.mdx +4 -0
- package/site/content/docs/elements/ai.mdx +2 -0
- package/site/content/docs/elements/channel.mdx +7 -4
- package/site/content/docs/elements/clock.mdx +2 -4
- package/site/content/docs/elements/flow.mdx +19 -10
- package/site/content/docs/elements/gate.mdx +18 -11
- package/site/content/docs/elements/signal.mdx +9 -12
- package/site/content/docs/elements/store.mdx +69 -8
- package/site/content/docs/elements/vault.mdx +11 -12
- package/site/content/docs/get-started/basic-usage.mdx +76 -41
- package/site/content/docs/get-started/installation.mdx +95 -43
- package/site/content/docs/get-started/introduction.mdx +128 -75
- package/site/content/docs/get-started/meta.json +1 -1
- package/site/content/docs/get-started/why.mdx +141 -0
- package/site/content/docs/plugins/ip-allowlist.mdx +1 -2
- package/site/content/docs/plugins/security-headers.mdx +1 -1
- package/site/content/docs/reference/configuration.mdx +2 -2
- package/site/content/docs/reference/environment-variables.mdx +10 -0
- package/site/content/docs/reference/fx.mdx +56 -3
- package/site/content/docs/reference/plugins.mdx +18 -18
- package/src/cli/competitor-mention-removal.test.ts +117 -0
- package/src/cli/dev.ts +20 -0
- package/src/cli/meilisearch-local.test.ts +69 -0
- package/src/cli/meilisearch-local.ts +188 -0
- package/src/compiler/extract.test.ts +63 -0
- package/src/compiler/extract.ts +36 -15
- package/src/compiler/fixtures/skyport/src/flows/payments/index.ts +5 -1
- package/src/console/server/channels.ts +2 -0
- package/src/console/server/clock.ts +3 -0
- package/src/console/server/flows.ts +13 -0
- package/src/console/server/gates.ts +2 -0
- package/src/console/server/plugins.ts +20 -1
- package/src/console/server/signals.ts +5 -0
- package/src/console/server/store.test.ts +17 -0
- package/src/console/server/store.ts +35 -1
- package/src/console/ui/channels/types.ts +1 -0
- package/src/console/ui/clock/types.ts +1 -0
- package/src/console/ui/display.test.ts +14 -0
- package/src/console/ui/display.ts +9 -0
- package/src/console/ui/dist/assets/index-CjxwRGVv.js +10 -0
- package/src/console/ui/dist/assets/panel-access-BGv45snf.js +64 -0
- package/src/console/ui/dist/assets/{panel-ai-D_m6WQI8.js → panel-ai-B2S7LEii.js} +1 -1
- package/src/console/ui/dist/assets/{panel-architecture-CKnXFyUx.js → panel-architecture-D7UJh91v.js} +1 -1
- package/src/console/ui/dist/assets/panel-channels-9T3ybqRu.js +1 -0
- package/src/console/ui/dist/assets/panel-clock-Cb1UXGRQ.js +1 -0
- package/src/console/ui/dist/assets/{panel-diff-cdonmH8c.js → panel-diff-DmYbKWmN.js} +1 -1
- package/src/console/ui/dist/assets/panel-flows-PiHwT55z.js +48 -0
- package/src/console/ui/dist/assets/panel-gates-BQGYXvjT.js +1 -0
- package/src/console/ui/dist/assets/panel-overview-BBnRO18l.js +1 -0
- package/src/console/ui/dist/assets/panel-plugins-D0PsmVw2.js +1 -0
- package/src/console/ui/dist/assets/panel-runs-CWuRDe0r.js +1 -0
- package/src/console/ui/dist/assets/panel-signals-Bbg4ewpP.js +1 -0
- package/src/console/ui/dist/assets/panel-store-CPCbsDRa.js +1 -0
- package/src/console/ui/dist/assets/panel-traces-DVAzuA_S.js +1 -0
- package/src/console/ui/dist/assets/panel-vault-D1_MvOmo.js +1 -0
- package/src/console/ui/dist/assets/{rolldown-runtime-CNC7AqOf.js → rolldown-runtime-B0Z9INg1.js} +1 -1
- package/src/console/ui/dist/index.html +2 -2
- package/src/console/ui/gates/types.ts +1 -0
- package/src/console/ui/plugins/fixture.ts +7 -0
- package/src/console/ui/plugins/types.ts +3 -0
- package/src/console/ui/shell/client.ts +3 -0
- package/src/console/ui/shell/panels/channels/ChannelsPanel.tsx +7 -2
- package/src/console/ui/shell/panels/clock/ClockPanel.tsx +9 -2
- package/src/console/ui/shell/panels/gates/GatesPanel.tsx +12 -5
- package/src/console/ui/shell/panels/plugins/PluginsPanel.tsx +18 -4
- package/src/console/ui/shell/panels/signals/SignalsPanel.tsx +13 -2
- package/src/console/ui/shell/panels/store/StorePanel.tsx +11 -4
- package/src/console/ui/shell/panels/vault/VaultPanel.tsx +9 -3
- package/src/console/ui/signals/types.ts +1 -0
- package/src/console/ui/store/fixture.ts +5 -0
- package/src/console/ui/store/types.ts +2 -0
- package/src/docker/compose.ts +5 -0
- package/src/docker/docker.test.ts +41 -0
- package/src/docker/recipes/index.ts +10 -2
- package/src/docker/recipes/meilisearch.ts +31 -0
- package/src/drivers/conformance.test.ts +26 -0
- package/src/drivers/conformance.ts +40 -3
- package/src/drivers/drizzle-dialect.test.ts +4 -0
- package/src/drivers/drizzle-dialect.ts +8 -4
- package/src/drivers/index.ts +18 -2
- package/src/drivers/libsql.ts +179 -0
- package/src/drivers/meilisearch.integration.test.ts +77 -0
- package/src/drivers/meilisearch.test.ts +181 -0
- package/src/drivers/meilisearch.ts +208 -0
- package/src/drivers/memory.ts +4 -4
- package/src/drivers/pglite.ts +79 -0
- package/src/drivers/pgvector.ts +60 -25
- package/src/drivers/types.ts +106 -16
- package/src/drivers/vault-driver-removal.test.ts +6 -0
- package/src/drivers/vault-types.ts +4 -4
- package/src/elements/ai/runtime.ts +6 -0
- package/src/elements/ai.test.ts +22 -0
- package/src/elements/channel/declare.ts +5 -0
- package/src/elements/clock/declare.ts +5 -0
- package/src/elements/clock/durable.ts +7 -1
- package/src/elements/gate/declare.ts +28 -5
- package/src/elements/gate.ts +1 -0
- package/src/elements/signal/declare.ts +5 -0
- package/src/elements/store/declare.ts +10 -2
- package/src/elements/store/index-boot.test.ts +299 -0
- package/src/elements/store/runtime.ts +110 -17
- package/src/elements/store/schema-decl.ts +7 -0
- package/src/elements/store.ts +2 -0
- package/src/elements/vault.test.ts +27 -4
- package/src/elements/vault.ts +1 -1
- package/src/index.ts +4 -0
- package/src/kernel/abort-scope.ts +116 -0
- package/src/kernel/app.ts +12 -2
- package/src/kernel/boot-bind/store.test.ts +68 -1
- package/src/kernel/boot-bind/store.ts +92 -2
- package/src/kernel/concurrency.test.ts +294 -0
- package/src/kernel/concurrency.ts +220 -0
- package/src/kernel/flow.ts +9 -0
- package/src/kernel/fx.test.ts +23 -2
- package/src/kernel/fx.ts +80 -4
- package/src/kernel/index.ts +23 -0
- package/src/kernel/journal.ts +9 -0
- package/src/kernel/plugin/capabilities.test.ts +18 -0
- package/src/kernel/plugin.ts +11 -3
- package/src/kernel/redacted.ts +74 -0
- package/src/kernel/registry.ts +29 -6
- package/src/kernel/router.ts +3 -3
- package/src/manifest/types.ts +21 -0
- package/src/release/measure.ts +4 -0
- package/src/test/provisions.integration.test.ts +1 -1
- package/site/content/docs/get-started/comparison.mdx +0 -65
- package/src/console/ui/dist/assets/index-BWo8R7NR.js +0 -10
- package/src/console/ui/dist/assets/panel-access-C0J2D-a2.js +0 -64
- package/src/console/ui/dist/assets/panel-channels-BOmQ-onL.js +0 -1
- package/src/console/ui/dist/assets/panel-clock-giAq0Ccv.js +0 -1
- package/src/console/ui/dist/assets/panel-flows-DlCU5zjA.js +0 -45
- package/src/console/ui/dist/assets/panel-gates-XclZxWD5.js +0 -1
- package/src/console/ui/dist/assets/panel-overview-BznEOTnb.js +0 -1
- package/src/console/ui/dist/assets/panel-plugins-CcGM1g64.js +0 -1
- package/src/console/ui/dist/assets/panel-runs-CGWNHLR4.js +0 -1
- package/src/console/ui/dist/assets/panel-signals-CNywkdak.js +0 -1
- package/src/console/ui/dist/assets/panel-store-KmTbFHMH.js +0 -1
- package/src/console/ui/dist/assets/panel-traces-DBLx2ilD.js +0 -1
- package/src/console/ui/dist/assets/panel-vault-CEnFc0dk.js +0 -1
- package/src/drivers/vault-infisical.ts +0 -57
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `libsql` driver — binds `@libsql/client` (optional peer, dynamic import).
|
|
3
|
+
*
|
|
4
|
+
* Protocol-named: libSQL speaks the sqlite wire dialect plus native ANN
|
|
5
|
+
* vectors (`F32_BLOB` + `libsql_vector_idx` + `vector_top_k`). Classic libSQL
|
|
6
|
+
* only — never the Turso Rust rewrite (`@tursodatabase/database`).
|
|
7
|
+
*
|
|
8
|
+
* Serves both the `sql` facet (same `?` placeholder convention as `sqlite`)
|
|
9
|
+
* and the `index` facet (native ANN, no JS-side distance math).
|
|
10
|
+
*
|
|
11
|
+
* Latency note: local file / `:memory:` opens are in-process like sqlite.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { customType, sqliteTable, text } from "drizzle-orm/sqlite-core";
|
|
15
|
+
import type {
|
|
16
|
+
IndexHit,
|
|
17
|
+
IndexOpenOptions,
|
|
18
|
+
SqlConnectOptions,
|
|
19
|
+
SqlConnection,
|
|
20
|
+
SqlDriver,
|
|
21
|
+
SqlRow,
|
|
22
|
+
VectorIndexDriver,
|
|
23
|
+
VectorIndexStore,
|
|
24
|
+
} from "./types.ts";
|
|
25
|
+
|
|
26
|
+
/** `@libsql/client` is an optional peer — loaded only when this driver runs. */
|
|
27
|
+
type LibsqlModule = typeof import("@libsql/client");
|
|
28
|
+
|
|
29
|
+
async function loadLibsql(): Promise<LibsqlModule> {
|
|
30
|
+
try {
|
|
31
|
+
return await import("@libsql/client");
|
|
32
|
+
} catch {
|
|
33
|
+
throw new Error(
|
|
34
|
+
"libsql driver: install optional peer `@libsql/client` (bun add @libsql/client)",
|
|
35
|
+
);
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** Bare file paths become `file:` URLs; `:memory:` / scheme URLs pass through. */
|
|
40
|
+
function normalizeLibsqlUrl(url: string): string {
|
|
41
|
+
if (url === ":memory:" || url.includes("://") || url.startsWith("file:")) return url;
|
|
42
|
+
return `file:${url}`;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Open a libSQL connection implementing {@link SqlConnection}.
|
|
47
|
+
*
|
|
48
|
+
* @param options - URL (`:memory:` / `file:…` / `libsql://…`) / role
|
|
49
|
+
*/
|
|
50
|
+
export async function connectLibsql(options: SqlConnectOptions = {}): Promise<SqlConnection> {
|
|
51
|
+
const role = options.role ?? "primary";
|
|
52
|
+
const { createClient } = await loadLibsql();
|
|
53
|
+
const client = createClient({ url: normalizeLibsqlUrl(options.url ?? ":memory:") });
|
|
54
|
+
return {
|
|
55
|
+
driverId: "libsql",
|
|
56
|
+
role,
|
|
57
|
+
async query(sql, params = []) {
|
|
58
|
+
const rs = await client.execute({ sql, args: params as never });
|
|
59
|
+
return rs.rows as unknown as SqlRow[];
|
|
60
|
+
},
|
|
61
|
+
async exec(sql, params = []) {
|
|
62
|
+
const rs = await client.execute({ sql, args: params as never });
|
|
63
|
+
return { changes: rs.rowsAffected };
|
|
64
|
+
},
|
|
65
|
+
async close() {
|
|
66
|
+
client.close();
|
|
67
|
+
},
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** Protocol-named libsql sql driver. */
|
|
72
|
+
export const libsqlDriver: SqlDriver = {
|
|
73
|
+
id: "libsql",
|
|
74
|
+
facet: "sql",
|
|
75
|
+
connect: connectLibsql,
|
|
76
|
+
};
|
|
77
|
+
|
|
78
|
+
/** libSQL native vector column: `F32_BLOB(dims)` (float32, validated by the engine). */
|
|
79
|
+
const f32Blob = customType<{
|
|
80
|
+
data: readonly number[];
|
|
81
|
+
driverData: string;
|
|
82
|
+
config: { dimensions: number };
|
|
83
|
+
}>({
|
|
84
|
+
dataType(config) {
|
|
85
|
+
return `F32_BLOB(${config?.dimensions ?? 0})`;
|
|
86
|
+
},
|
|
87
|
+
toDriver(value) {
|
|
88
|
+
return JSON.stringify(value);
|
|
89
|
+
},
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* Open a libSQL-backed index with native ANN.
|
|
94
|
+
*
|
|
95
|
+
* DDL: `F32_BLOB(dims)` column + `libsql_vector_idx` ANN index (validated
|
|
96
|
+
* against `@libsql/client@0.17`). Queries run `vector_top_k` joined back to
|
|
97
|
+
* the base table — approximate nearest neighbours, never a full scan. Score
|
|
98
|
+
* is cosine similarity (`1 - vector_distance_cos`), matching the memory
|
|
99
|
+
* driver's higher-is-better convention.
|
|
100
|
+
*
|
|
101
|
+
* @param options - Name / dims / shared SQL connection
|
|
102
|
+
*/
|
|
103
|
+
export async function openLibsqlIndex(options: IndexOpenOptions): Promise<VectorIndexStore> {
|
|
104
|
+
const ownsSql = !options.sql;
|
|
105
|
+
const sql = options.sql ?? (await connectLibsql({ url: options.url }));
|
|
106
|
+
if (sql.driverId !== "libsql") {
|
|
107
|
+
if (ownsSql) await sql.close();
|
|
108
|
+
throw new Error(
|
|
109
|
+
`libsql index: needs a libsql SQL connection, got "${sql.driverId}" — ` +
|
|
110
|
+
`set store.sql to "libsql" so the index shares its connection`,
|
|
111
|
+
);
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
const tableName = `oke_idx_${options.name.replace(/[^a-zA-Z0-9_]/g, "_")}`;
|
|
115
|
+
const indexName = `${tableName}_vec`;
|
|
116
|
+
const table = sqliteTable(tableName, {
|
|
117
|
+
id: text("id").primaryKey(),
|
|
118
|
+
embedding: f32Blob("embedding", { dimensions: options.dims }).notNull(),
|
|
119
|
+
meta: text("meta"),
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
await sql.exec(
|
|
123
|
+
`CREATE TABLE IF NOT EXISTS "${tableName}" (id TEXT PRIMARY KEY, embedding ${table.embedding.getSQLType()} NOT NULL, meta TEXT)`,
|
|
124
|
+
);
|
|
125
|
+
// ANN index — fails loud on engines without vector support (never falls back).
|
|
126
|
+
await sql.exec(
|
|
127
|
+
`CREATE INDEX IF NOT EXISTS "${indexName}" ON "${tableName}" (libsql_vector_idx(embedding))`,
|
|
128
|
+
);
|
|
129
|
+
|
|
130
|
+
return {
|
|
131
|
+
driverId: "libsql",
|
|
132
|
+
async upsert(id, vector, meta) {
|
|
133
|
+
if (vector.length !== options.dims) {
|
|
134
|
+
throw new Error(`vector dims ${vector.length} !== index dims ${options.dims}`);
|
|
135
|
+
}
|
|
136
|
+
await sql.exec(
|
|
137
|
+
`INSERT INTO "${tableName}" (id, embedding, meta) VALUES (?, vector32(?), ?) ` +
|
|
138
|
+
`ON CONFLICT (id) DO UPDATE SET embedding = excluded.embedding, meta = excluded.meta`,
|
|
139
|
+
[id, JSON.stringify(vector), meta ? JSON.stringify(meta) : null],
|
|
140
|
+
);
|
|
141
|
+
},
|
|
142
|
+
async search(vector, topK = 10): Promise<IndexHit[]> {
|
|
143
|
+
const k = Math.max(1, Math.floor(topK));
|
|
144
|
+
const rows = await sql.query(
|
|
145
|
+
`SELECT t.id AS id, t.meta AS meta, ` +
|
|
146
|
+
`vector_distance_cos(t.embedding, vector32(?)) AS distance ` +
|
|
147
|
+
`FROM vector_top_k('${indexName}', vector32(?), ${k}) AS i ` +
|
|
148
|
+
`JOIN "${tableName}" AS t ON t.rowid = i.id ` +
|
|
149
|
+
`ORDER BY distance ASC`,
|
|
150
|
+
[JSON.stringify(vector), JSON.stringify(vector)],
|
|
151
|
+
);
|
|
152
|
+
return rows.map((row) => {
|
|
153
|
+
const metaRaw = row.meta;
|
|
154
|
+
return {
|
|
155
|
+
id: String(row.id),
|
|
156
|
+
score: 1 - Number(row.distance),
|
|
157
|
+
meta:
|
|
158
|
+
typeof metaRaw === "string"
|
|
159
|
+
? (JSON.parse(metaRaw) as Record<string, unknown>)
|
|
160
|
+
: undefined,
|
|
161
|
+
};
|
|
162
|
+
});
|
|
163
|
+
},
|
|
164
|
+
async delete(id) {
|
|
165
|
+
const result = await sql.exec(`DELETE FROM "${tableName}" WHERE id = ?`, [id]);
|
|
166
|
+
return result.changes > 0;
|
|
167
|
+
},
|
|
168
|
+
async close() {
|
|
169
|
+
if (ownsSql) await sql.close();
|
|
170
|
+
},
|
|
171
|
+
};
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/** Protocol-named libsql index driver. */
|
|
175
|
+
export const libsqlIndexDriver: VectorIndexDriver = {
|
|
176
|
+
id: "libsql",
|
|
177
|
+
facet: "index",
|
|
178
|
+
open: openLibsqlIndex,
|
|
179
|
+
};
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Live Meilisearch round-trip — Arabic + typo tolerance.
|
|
3
|
+
*
|
|
4
|
+
* The only acceptable evidence that Meilisearch actually handles Arabic (not
|
|
5
|
+
* just "supported" in marketing copy) is a real round-trip against a live
|
|
6
|
+
* server: index Arabic content, then search it — with and without the
|
|
7
|
+
* definite article (ال), with diacritics stripped, and with a Latin typo that
|
|
8
|
+
* typo-tolerance must forgive.
|
|
9
|
+
*
|
|
10
|
+
* Gated on a live server: set `OKE_TEST_MEILI_URL` (and optionally
|
|
11
|
+
* `OKE_TEST_MEILI_KEY`). Without it the whole suite skips, so unit runs never
|
|
12
|
+
* depend on Docker or a spawned binary.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import { afterEach, describe, expect, test } from "bun:test";
|
|
16
|
+
import { openMeilisearchIndex, type MeilisearchIndexOptions } from "./meilisearch.ts";
|
|
17
|
+
|
|
18
|
+
const URL = process.env.OKE_TEST_MEILI_URL;
|
|
19
|
+
const KEY = process.env.OKE_TEST_MEILI_KEY;
|
|
20
|
+
if (!URL) {
|
|
21
|
+
console.log("skip: live meilisearch e2e (OKE_TEST_MEILI_URL not set)");
|
|
22
|
+
}
|
|
23
|
+
const live = URL ? test : test.skip;
|
|
24
|
+
|
|
25
|
+
const opened: Array<{ close(): Promise<void> }> = [];
|
|
26
|
+
afterEach(async () => {
|
|
27
|
+
while (opened.length > 0) {
|
|
28
|
+
await opened.pop()?.close();
|
|
29
|
+
}
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
async function liveIndex(name: string): ReturnType<typeof openMeilisearchIndex> {
|
|
33
|
+
const opts: MeilisearchIndexOptions = { name, dims: 0, url: URL, apiKey: KEY };
|
|
34
|
+
const index = await openMeilisearchIndex(opts);
|
|
35
|
+
opened.push(index);
|
|
36
|
+
return index;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
describe("meilisearch live — arabic + typo tolerance", () => {
|
|
40
|
+
live("indexes and searches Arabic content (round-trip)", async () => {
|
|
41
|
+
const index = await liveIndex(`ar-${crypto.randomUUID().slice(0, 8)}`);
|
|
42
|
+
await index.upsert("d1", { title: "كتاب عن محركات البحث" });
|
|
43
|
+
await index.upsert("d2", { title: "مقال عن قواعد البيانات" });
|
|
44
|
+
|
|
45
|
+
const found = await index.search("البحث", { topK: 5 });
|
|
46
|
+
expect(found.hits.length).toBeGreaterThan(0);
|
|
47
|
+
expect(found.hits[0]?.id).toBe("d1");
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
live("matches a query without the definite article ال", async () => {
|
|
51
|
+
const index = await liveIndex(`ar-${crypto.randomUUID().slice(0, 8)}`);
|
|
52
|
+
await index.upsert("d1", { title: "الكتاب" });
|
|
53
|
+
// Charabia segments/normalises ال, so the bare stem should still match.
|
|
54
|
+
const found = await index.search("كتاب", { topK: 5 });
|
|
55
|
+
expect(found.hits.map((h) => h.id)).toContain("d1");
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
live("typo tolerance forgives a latin misspelling", async () => {
|
|
59
|
+
const index = await liveIndex(`en-${crypto.randomUUID().slice(0, 8)}`);
|
|
60
|
+
await index.upsert("d1", { title: "search engine" });
|
|
61
|
+
// One-transposition typo ("serach") must still hit the document.
|
|
62
|
+
const found = await index.search("serach", { topK: 5 });
|
|
63
|
+
expect(found.hits.map((h) => h.id)).toContain("d1");
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
live("filter + facets aggregate real results", async () => {
|
|
67
|
+
const index = await liveIndex(`fac-${crypto.randomUUID().slice(0, 8)}`);
|
|
68
|
+
await index.upsert("d1", { title: "red apple", color: "red" });
|
|
69
|
+
await index.upsert("d2", { title: "green apple", color: "green" });
|
|
70
|
+
const found = await index.search("apple", {
|
|
71
|
+
topK: 5,
|
|
72
|
+
filter: "color = red",
|
|
73
|
+
facets: ["color"],
|
|
74
|
+
});
|
|
75
|
+
expect(found.hits.map((h) => h.id)).toEqual(["d1"]);
|
|
76
|
+
});
|
|
77
|
+
});
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `meilisearch` index driver — task-waited writes, relevance search mapping,
|
|
3
|
+
* and fail-loud semantics against an injected fetch (never a live server).
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import { describe, expect, test } from "bun:test";
|
|
7
|
+
import {
|
|
8
|
+
MeilisearchUnavailableError,
|
|
9
|
+
meilisearchDriver,
|
|
10
|
+
openMeilisearchIndex,
|
|
11
|
+
} from "./meilisearch.ts";
|
|
12
|
+
|
|
13
|
+
/** A scripted Meilisearch over the wire — in-memory task + document store. */
|
|
14
|
+
export function createMeilisearchFakeFetch(opts?: {
|
|
15
|
+
readonly failHealth?: boolean;
|
|
16
|
+
readonly failTask?: boolean;
|
|
17
|
+
}): typeof fetch {
|
|
18
|
+
const documents = new Map<string, Record<string, unknown>>();
|
|
19
|
+
const taskResults = new Map<number, { fail: boolean }>();
|
|
20
|
+
let taskSeq = 0;
|
|
21
|
+
|
|
22
|
+
const json = (body: unknown, status = 200): Response =>
|
|
23
|
+
new Response(JSON.stringify(body), {
|
|
24
|
+
status,
|
|
25
|
+
headers: { "content-type": "application/json" },
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
const task = (type: string, fail: boolean): object => {
|
|
29
|
+
taskSeq += 1;
|
|
30
|
+
taskResults.set(taskSeq, { fail });
|
|
31
|
+
return { taskUid: taskSeq, indexUid: "kb", status: "enqueued", type };
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
return (async (input: string | URL | Request, init?: RequestInit): Promise<Response> => {
|
|
35
|
+
const url = typeof input === "string" ? input : input instanceof URL ? input.href : input.url;
|
|
36
|
+
const method = init?.method ?? "GET";
|
|
37
|
+
|
|
38
|
+
if (url.endsWith("/health")) {
|
|
39
|
+
if (opts?.failHealth) return json({ message: "down" }, 503);
|
|
40
|
+
return json({ status: "available" });
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
if (url.endsWith("/indexes") && method === "POST") {
|
|
44
|
+
return json(task("indexCreation", false), 202);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
if (url.includes("/tasks/")) {
|
|
48
|
+
const taskUid = Number(url.split("/tasks/")[1]);
|
|
49
|
+
const fail = taskResults.get(taskUid)?.fail ?? false;
|
|
50
|
+
return json({
|
|
51
|
+
taskUid,
|
|
52
|
+
status: fail ? "failed" : "succeeded",
|
|
53
|
+
error: fail ? { message: "boom" } : null,
|
|
54
|
+
});
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
if (url.includes("/documents/") && method === "DELETE") {
|
|
58
|
+
const id = decodeURIComponent(url.split("/documents/")[1] ?? "");
|
|
59
|
+
if (!documents.delete(id)) return json({ message: "not found" }, 404);
|
|
60
|
+
return json(task("documentDeletion", opts?.failTask ?? false), 202);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
if (url.endsWith("/documents") && method === "POST") {
|
|
64
|
+
const docs = JSON.parse(String(init?.body ?? "[]")) as Record<string, unknown>[];
|
|
65
|
+
if (!(opts?.failTask ?? false)) {
|
|
66
|
+
for (const doc of docs) documents.set(String(doc.id), doc);
|
|
67
|
+
}
|
|
68
|
+
return json(task("documentAdditionOrUpdate", opts?.failTask ?? false), 202);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
if (url.endsWith("/search") && method === "POST") {
|
|
72
|
+
const {
|
|
73
|
+
q,
|
|
74
|
+
limit = 10,
|
|
75
|
+
facets,
|
|
76
|
+
} = JSON.parse(String(init?.body ?? "{}")) as {
|
|
77
|
+
q?: string;
|
|
78
|
+
limit?: number;
|
|
79
|
+
facets?: string[];
|
|
80
|
+
};
|
|
81
|
+
const query = (q ?? "").toLowerCase();
|
|
82
|
+
const hits = [...documents.values()]
|
|
83
|
+
.filter((doc) => query === "" || JSON.stringify(doc).toLowerCase().includes(query))
|
|
84
|
+
.slice(0, limit)
|
|
85
|
+
.map((doc) => ({ ...doc, _rankingScore: 0.95 }));
|
|
86
|
+
const facetDistribution: Record<string, Record<string, number>> = {};
|
|
87
|
+
for (const facet of facets ?? []) {
|
|
88
|
+
facetDistribution[facet] = { tagged: documents.size };
|
|
89
|
+
}
|
|
90
|
+
return json({
|
|
91
|
+
hits,
|
|
92
|
+
...(facets && facets.length > 0 ? { facetDistribution } : {}),
|
|
93
|
+
estimatedTotalHits: hits.length,
|
|
94
|
+
});
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
return json({ message: `no route for ${method} ${url}` }, 404);
|
|
98
|
+
}) as typeof fetch;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
describe("meilisearch driver", () => {
|
|
102
|
+
test("upsert → search maps _rankingScore into relevance score", async () => {
|
|
103
|
+
const index = await openMeilisearchIndex({
|
|
104
|
+
name: "kb",
|
|
105
|
+
dims: 0,
|
|
106
|
+
url: "http://127.0.0.1:7700",
|
|
107
|
+
apiKey: "master-key",
|
|
108
|
+
fetch: createMeilisearchFakeFetch(),
|
|
109
|
+
});
|
|
110
|
+
await index.upsert("d1", { title: "hello world", tag: "a" });
|
|
111
|
+
await index.upsert("d2", { title: "goodbye world", tag: "b" });
|
|
112
|
+
|
|
113
|
+
const found = await index.search("hello", { topK: 1 });
|
|
114
|
+
expect(found.hits).toHaveLength(1);
|
|
115
|
+
expect(found.hits[0]?.id).toBe("d1");
|
|
116
|
+
expect(found.hits[0]?.score).toBeGreaterThan(0);
|
|
117
|
+
expect(found.hits[0]?.meta).toMatchObject({ title: "hello world", tag: "a" });
|
|
118
|
+
await index.close();
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
test("facets return a facetDistribution", async () => {
|
|
122
|
+
const index = await openMeilisearchIndex({
|
|
123
|
+
name: "kb",
|
|
124
|
+
dims: 0,
|
|
125
|
+
url: "http://127.0.0.1:7700",
|
|
126
|
+
fetch: createMeilisearchFakeFetch(),
|
|
127
|
+
});
|
|
128
|
+
await index.upsert("d1", { title: "x", tag: "a" });
|
|
129
|
+
const found = await index.search("x", { facets: ["tag"] });
|
|
130
|
+
expect(found.facetDistribution).toEqual({ tag: { tagged: 1 } });
|
|
131
|
+
await index.close();
|
|
132
|
+
});
|
|
133
|
+
|
|
134
|
+
test("delete removes the document; missing id is false", async () => {
|
|
135
|
+
const index = await openMeilisearchIndex({
|
|
136
|
+
name: "kb",
|
|
137
|
+
dims: 0,
|
|
138
|
+
url: "http://127.0.0.1:7700",
|
|
139
|
+
fetch: createMeilisearchFakeFetch(),
|
|
140
|
+
});
|
|
141
|
+
await index.upsert("d1", { title: "x" });
|
|
142
|
+
expect(await index.delete("d1")).toBe(true);
|
|
143
|
+
expect(await index.delete("d1")).toBe(false);
|
|
144
|
+
await index.close();
|
|
145
|
+
});
|
|
146
|
+
|
|
147
|
+
test("missing URL fails loud (no memory fallback)", async () => {
|
|
148
|
+
await expect(openMeilisearchIndex({ name: "kb", dims: 0 })).rejects.toBeInstanceOf(
|
|
149
|
+
MeilisearchUnavailableError,
|
|
150
|
+
);
|
|
151
|
+
});
|
|
152
|
+
|
|
153
|
+
test("unhealthy server fails loud at open", async () => {
|
|
154
|
+
await expect(
|
|
155
|
+
openMeilisearchIndex({
|
|
156
|
+
name: "kb",
|
|
157
|
+
dims: 0,
|
|
158
|
+
url: "http://127.0.0.1:7700",
|
|
159
|
+
fetch: createMeilisearchFakeFetch({ failHealth: true }),
|
|
160
|
+
}),
|
|
161
|
+
).rejects.toBeInstanceOf(MeilisearchUnavailableError);
|
|
162
|
+
});
|
|
163
|
+
|
|
164
|
+
test("a failed indexing task surfaces as a loud error, not a silent write", async () => {
|
|
165
|
+
const index = await openMeilisearchIndex({
|
|
166
|
+
name: "kb",
|
|
167
|
+
dims: 0,
|
|
168
|
+
url: "http://127.0.0.1:7700",
|
|
169
|
+
fetch: createMeilisearchFakeFetch({ failTask: true }),
|
|
170
|
+
});
|
|
171
|
+
await expect(index.upsert("d1", { title: "x" })).rejects.toBeInstanceOf(
|
|
172
|
+
MeilisearchUnavailableError,
|
|
173
|
+
);
|
|
174
|
+
await index.close();
|
|
175
|
+
});
|
|
176
|
+
|
|
177
|
+
test("driver id + facet", () => {
|
|
178
|
+
expect(meilisearchDriver.id).toBe("meilisearch");
|
|
179
|
+
expect(meilisearchDriver.facet).toBe("index");
|
|
180
|
+
});
|
|
181
|
+
});
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `meilisearch` index driver — full-text search over the Meilisearch HTTP API.
|
|
3
|
+
*
|
|
4
|
+
* Product-as-protocol id (same pattern as `openbao` / `pgvector`); the image
|
|
5
|
+
* pin lives in `images["store.index"]` (e.g. `getmeili/meilisearch`). This is
|
|
6
|
+
* a genuinely different capability from the vector ANN drivers — typo-tolerant
|
|
7
|
+
* / faceted full-text relevance, not cosine similarity.
|
|
8
|
+
*
|
|
9
|
+
* Fail-loud: a configured Meilisearch that is unreachable, unhealthy, or
|
|
10
|
+
* rejects a call throws {@link MeilisearchUnavailableError} — never silently
|
|
11
|
+
* falls back to memory. Writes wait for their task to succeed (Meilisearch is
|
|
12
|
+
* asynchronous), so an `upsert`/`delete` only resolves once the server has
|
|
13
|
+
* actually indexed it.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import type {
|
|
17
|
+
IndexHit,
|
|
18
|
+
IndexOpenOptions,
|
|
19
|
+
TextIndexDriver,
|
|
20
|
+
TextIndexSearchOptions,
|
|
21
|
+
TextIndexSearchResult,
|
|
22
|
+
TextIndexStore,
|
|
23
|
+
} from "./types.ts";
|
|
24
|
+
|
|
25
|
+
/** Error thrown when the remote Meilisearch is unreachable / unhealthy / errors. */
|
|
26
|
+
export class MeilisearchUnavailableError extends Error {
|
|
27
|
+
constructor(message: string) {
|
|
28
|
+
super(message);
|
|
29
|
+
this.name = "MeilisearchUnavailableError";
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** Minimal response shape of a Meilisearch summarized / finished task. */
|
|
34
|
+
interface MeiliTask {
|
|
35
|
+
readonly taskUid: number;
|
|
36
|
+
readonly status: "enqueued" | "processing" | "succeeded" | "failed" | "canceled";
|
|
37
|
+
readonly error?: { readonly message?: string } | null;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** Search response hit shape we consume. */
|
|
41
|
+
interface MeiliSearchResponse {
|
|
42
|
+
readonly hits?: ReadonlyArray<Record<string, unknown> & { readonly _rankingScore?: number }>;
|
|
43
|
+
readonly facetDistribution?: Record<string, Record<string, number>>;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** Options for opening a Meilisearch-backed text index. */
|
|
47
|
+
export interface MeilisearchIndexOptions extends IndexOpenOptions {
|
|
48
|
+
/** Max ms to wait for an indexing task to finish (default 5000). */
|
|
49
|
+
readonly taskTimeoutMs?: number;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Open a full-text index.
|
|
54
|
+
*
|
|
55
|
+
* Ensures the index exists, health-checks the server, and returns a handle
|
|
56
|
+
* whose writes are confirmed (task-waited) and whose search maps
|
|
57
|
+
* `_rankingScore` into `IndexHit.score` as relevance.
|
|
58
|
+
*
|
|
59
|
+
* @param options - name / url / apiKey / injected fetch
|
|
60
|
+
*/
|
|
61
|
+
export async function openMeilisearchIndex(
|
|
62
|
+
options: MeilisearchIndexOptions,
|
|
63
|
+
): Promise<TextIndexStore> {
|
|
64
|
+
const url = options.url?.replace(/\/$/, "");
|
|
65
|
+
if (!url) {
|
|
66
|
+
throw new MeilisearchUnavailableError(
|
|
67
|
+
"meilisearch index: needs an HTTP URL (OKE_STORE_INDEX_URL) — no silent memory fallback",
|
|
68
|
+
);
|
|
69
|
+
}
|
|
70
|
+
const apiKey = options.apiKey;
|
|
71
|
+
const fetchFn = options.fetch ?? globalThis.fetch;
|
|
72
|
+
const taskTimeoutMs = options.taskTimeoutMs ?? 5_000;
|
|
73
|
+
const uid = encodeURIComponent(options.name);
|
|
74
|
+
|
|
75
|
+
const headers: Record<string, string> = { "content-type": "application/json" };
|
|
76
|
+
if (apiKey) headers.Authorization = `Bearer ${apiKey}`;
|
|
77
|
+
|
|
78
|
+
async function request(path: string, init?: RequestInit): Promise<Response> {
|
|
79
|
+
let res: Response;
|
|
80
|
+
try {
|
|
81
|
+
res = await fetchFn(`${url}${path}`, {
|
|
82
|
+
...init,
|
|
83
|
+
headers: { ...headers, ...(init?.headers ?? {}) },
|
|
84
|
+
});
|
|
85
|
+
} catch (err) {
|
|
86
|
+
throw new MeilisearchUnavailableError(
|
|
87
|
+
`meilisearch index: unreachable at ${url} — ${err instanceof Error ? err.message : String(err)}`,
|
|
88
|
+
);
|
|
89
|
+
}
|
|
90
|
+
return res;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
async function requestOk(path: string, init?: RequestInit): Promise<Response> {
|
|
94
|
+
const res = await request(path, init);
|
|
95
|
+
if (!res.ok) {
|
|
96
|
+
const detail = await res.text().catch(() => "");
|
|
97
|
+
throw new MeilisearchUnavailableError(
|
|
98
|
+
`meilisearch index: ${init?.method ?? "GET"} ${path} failed (${res.status})${detail ? ` — ${detail.slice(0, 200)}` : ""}`,
|
|
99
|
+
);
|
|
100
|
+
}
|
|
101
|
+
return res;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
async function enqueue(path: string, init: RequestInit): Promise<MeiliTask> {
|
|
105
|
+
const res = await requestOk(path, init);
|
|
106
|
+
return (await res.json()) as MeiliTask;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
async function waitForTask(taskUid: number): Promise<void> {
|
|
110
|
+
const deadline = Date.now() + taskTimeoutMs;
|
|
111
|
+
// Poll the task until it leaves the queue; treat a failed/canceled task as
|
|
112
|
+
// a loud error rather than an optimistic write.
|
|
113
|
+
for (;;) {
|
|
114
|
+
const res = await requestOk(`/tasks/${taskUid}`);
|
|
115
|
+
const task = (await res.json()) as MeiliTask;
|
|
116
|
+
if (task.status === "succeeded") return;
|
|
117
|
+
if (task.status === "failed" || task.status === "canceled") {
|
|
118
|
+
throw new MeilisearchUnavailableError(
|
|
119
|
+
`meilisearch index: task ${taskUid} ${task.status}${task.error?.message ? ` — ${task.error.message}` : ""}`,
|
|
120
|
+
);
|
|
121
|
+
}
|
|
122
|
+
if (Date.now() > deadline) {
|
|
123
|
+
throw new MeilisearchUnavailableError(
|
|
124
|
+
`meilisearch index: task ${taskUid} did not finish within ${taskTimeoutMs}ms`,
|
|
125
|
+
);
|
|
126
|
+
}
|
|
127
|
+
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
// Fail loud up front: server must be healthy and the index must exist
|
|
132
|
+
// (created if absent) before we hand back a usable handle.
|
|
133
|
+
await requestOk("/health");
|
|
134
|
+
const created = await enqueue("/indexes", {
|
|
135
|
+
method: "POST",
|
|
136
|
+
body: JSON.stringify({ uid: options.name, primaryKey: "id" }),
|
|
137
|
+
});
|
|
138
|
+
await waitForTask(created.taskUid);
|
|
139
|
+
|
|
140
|
+
const scoreOf = (hit: { readonly _rankingScore?: unknown }): number =>
|
|
141
|
+
typeof hit._rankingScore === "number" ? hit._rankingScore : 0;
|
|
142
|
+
|
|
143
|
+
return {
|
|
144
|
+
driverId: "meilisearch",
|
|
145
|
+
async upsert(id, document) {
|
|
146
|
+
const body = [{ ...document, id }];
|
|
147
|
+
const task = await enqueue(`/indexes/${uid}/documents`, {
|
|
148
|
+
method: "POST",
|
|
149
|
+
body: JSON.stringify(body),
|
|
150
|
+
});
|
|
151
|
+
await waitForTask(task.taskUid);
|
|
152
|
+
},
|
|
153
|
+
async search(q, opts?: TextIndexSearchOptions): Promise<TextIndexSearchResult> {
|
|
154
|
+
const payload: Record<string, unknown> = {
|
|
155
|
+
q,
|
|
156
|
+
limit: opts?.topK ?? 10,
|
|
157
|
+
showRankingScore: true,
|
|
158
|
+
};
|
|
159
|
+
if (opts?.filter !== undefined) payload.filter = opts.filter;
|
|
160
|
+
if (opts?.facets !== undefined) payload.facets = opts.facets;
|
|
161
|
+
const res = await requestOk(`/indexes/${uid}/search`, {
|
|
162
|
+
method: "POST",
|
|
163
|
+
body: JSON.stringify(payload),
|
|
164
|
+
});
|
|
165
|
+
const body = (await res.json()) as MeiliSearchResponse;
|
|
166
|
+
const hits: IndexHit[] = (body.hits ?? []).map((hit) => {
|
|
167
|
+
const { _rankingScore, ...rest } = hit;
|
|
168
|
+
void _rankingScore;
|
|
169
|
+
const meta = Object.fromEntries(Object.entries(rest).filter(([key]) => key !== "id"));
|
|
170
|
+
return {
|
|
171
|
+
id: String(rest.id ?? ""),
|
|
172
|
+
score: scoreOf(hit),
|
|
173
|
+
...(Object.keys(meta).length > 0 ? { meta } : {}),
|
|
174
|
+
};
|
|
175
|
+
});
|
|
176
|
+
return {
|
|
177
|
+
hits,
|
|
178
|
+
...(body.facetDistribution !== undefined
|
|
179
|
+
? { facetDistribution: body.facetDistribution }
|
|
180
|
+
: {}),
|
|
181
|
+
};
|
|
182
|
+
},
|
|
183
|
+
async delete(id) {
|
|
184
|
+
const res = await request(`/indexes/${uid}/documents/${encodeURIComponent(id)}`, {
|
|
185
|
+
method: "DELETE",
|
|
186
|
+
});
|
|
187
|
+
if (res.status === 404) return false;
|
|
188
|
+
if (!res.ok) {
|
|
189
|
+
throw new MeilisearchUnavailableError(
|
|
190
|
+
`meilisearch index: delete ${id} failed (${res.status})`,
|
|
191
|
+
);
|
|
192
|
+
}
|
|
193
|
+
const task = (await res.json()) as MeiliTask;
|
|
194
|
+
await waitForTask(task.taskUid);
|
|
195
|
+
return true;
|
|
196
|
+
},
|
|
197
|
+
async close() {
|
|
198
|
+
// Stateless HTTP client — nothing to release.
|
|
199
|
+
},
|
|
200
|
+
};
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/** Protocol-named Meilisearch driver. */
|
|
204
|
+
export const meilisearchDriver: TextIndexDriver = {
|
|
205
|
+
id: "meilisearch",
|
|
206
|
+
facet: "index",
|
|
207
|
+
open: openMeilisearchIndex,
|
|
208
|
+
};
|
package/src/drivers/memory.ts
CHANGED
|
@@ -7,10 +7,10 @@ import type {
|
|
|
7
7
|
FilesBucket,
|
|
8
8
|
FilesDriver,
|
|
9
9
|
FilesOpenOptions,
|
|
10
|
-
IndexDriver,
|
|
11
10
|
IndexHit,
|
|
12
11
|
IndexOpenOptions,
|
|
13
|
-
|
|
12
|
+
VectorIndexDriver,
|
|
13
|
+
VectorIndexStore,
|
|
14
14
|
KvDriver,
|
|
15
15
|
KvNamespace,
|
|
16
16
|
KvOpenOptions,
|
|
@@ -638,10 +638,10 @@ function cosine(a: readonly number[], b: readonly number[]): number {
|
|
|
638
638
|
}
|
|
639
639
|
|
|
640
640
|
/** Memory index driver. */
|
|
641
|
-
export const memoryIndexDriver:
|
|
641
|
+
export const memoryIndexDriver: VectorIndexDriver = {
|
|
642
642
|
id: "memory",
|
|
643
643
|
facet: "index",
|
|
644
|
-
async open(options: IndexOpenOptions): Promise<
|
|
644
|
+
async open(options: IndexOpenOptions): Promise<VectorIndexStore> {
|
|
645
645
|
const docs = new Map<string, { vector: number[]; meta?: Record<string, unknown> }>();
|
|
646
646
|
return {
|
|
647
647
|
driverId: "memory",
|