@hraness/oh 0.2.7 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. package/README.md +116 -12
  2. package/dist/canonical.d.ts.map +1 -1
  3. package/dist/cli.d.ts +1 -1
  4. package/dist/cli.js +91 -19
  5. package/dist/cloudflare-embedding.d.ts +104 -0
  6. package/dist/cloudflare-embedding.d.ts.map +1 -0
  7. package/dist/graph.d.ts.map +1 -1
  8. package/dist/index.js +90 -18
  9. package/dist/libsql-semantic.d.ts +111 -0
  10. package/dist/libsql-semantic.d.ts.map +1 -0
  11. package/dist/libsql.js +105 -18
  12. package/dist/memory-page.d.ts +2 -0
  13. package/dist/memory-page.d.ts.map +1 -0
  14. package/dist/memory-page.js +725 -0
  15. package/dist/memory-pages.d.ts +76 -0
  16. package/dist/memory-pages.d.ts.map +1 -0
  17. package/dist/memory.d.ts +1 -0
  18. package/dist/memory.d.ts.map +1 -1
  19. package/dist/memory.js +478 -18
  20. package/dist/projection-public.js +105 -18
  21. package/dist/projection-suss.js +105 -18
  22. package/dist/sdk.js +90 -18
  23. package/dist/semantic-cloud.d.ts +3 -0
  24. package/dist/semantic-cloud.d.ts.map +1 -0
  25. package/dist/semantic-cloud.js +1843 -0
  26. package/dist/semantic.d.ts.map +1 -1
  27. package/dist/semantic.js +104 -21
  28. package/dist/sqlite/index.js +90 -18
  29. package/dist/store.js +105 -18
  30. package/dist/sync.js +90 -18
  31. package/package.json +10 -2
  32. package/skills/oh/SKILL.md +28 -2
  33. package/spec/README.md +11 -3
  34. package/spec/manifest.json +9 -1
  35. package/spec/v1/cloudflare-embedding-profile.json +13 -0
  36. package/spec/v1/cloudflare-embedding-renderer.json +8 -0
  37. package/spec/v1/memory-page.md +153 -0
  38. package/spec/v1/memory-page.schema.json +154 -0
  39. package/spec/v1/memory.md +18 -0
  40. package/spec/v1/migration.md +13 -0
  41. package/spec/v1/semantic-cloud.md +87 -0
  42. package/src/canonical.ts +28 -13
  43. package/src/cli.ts +1 -1
  44. package/src/cloudflare-embedding.test.ts +306 -0
  45. package/src/cloudflare-embedding.ts +385 -0
  46. package/src/contracts.test.ts +20 -0
  47. package/src/graph.ts +63 -6
  48. package/src/libsql-semantic.test.ts +478 -0
  49. package/src/libsql-semantic.ts +1117 -0
  50. package/src/memory-page.ts +1 -0
  51. package/src/memory-pages.test.ts +277 -0
  52. package/src/memory-pages.ts +440 -0
  53. package/src/memory.ts +2 -0
  54. package/src/semantic-cloud.ts +2 -0
  55. package/src/semantic.ts +14 -3
@@ -0,0 +1,1117 @@
1
+ import {
2
+ canonicalJson,
3
+ canonicalNow,
4
+ canonicalSha256,
5
+ parseCanonicalInstantV1,
6
+ parseSha256Hex,
7
+ safeCode,
8
+ sha256Hex,
9
+ type Sha256Hex,
10
+ } from "./canonical";
11
+ import {
12
+ OH_CLOUDFLARE_EMBEDDING_PROFILE_V1,
13
+ OH_SEMANTIC_RENDERER_V1,
14
+ renderOhCloudflareEmbeddingDocumentV1,
15
+ renderOhCloudflareEmbeddingQueryV1,
16
+ type OhCloudflareEmbeddingClientV1,
17
+ type OhRenderedEmbeddingInputV1,
18
+ } from "./cloudflare-embedding";
19
+ import type {
20
+ OhLibSqlClientV1,
21
+ OhLibSqlResultV1,
22
+ OhLibSqlStatementV1,
23
+ } from "./libsql";
24
+ import { normalizeOhEmbeddingV1 } from "./semantic";
25
+
26
+ export const OH_LIBSQL_SEMANTIC_LIMITS_V1 = Object.freeze({
27
+ chunksPerDocument: 64,
28
+ chunksPerGeneration: 4_096,
29
+ documentsPerGeneration: 512,
30
+ embeddingBatch: 16,
31
+ searchLimit: 100,
32
+ searchPage: 128,
33
+ });
34
+
35
+ export class OhLibSqlSemanticError extends Error {
36
+ readonly code: "conflict" | "integrity" | "invalid-input" | "purged" | "schema-unavailable";
37
+
38
+ constructor(code: OhLibSqlSemanticError["code"], message: string) {
39
+ super(message);
40
+ this.name = "OhLibSqlSemanticError";
41
+ this.code = code;
42
+ }
43
+ }
44
+
45
+ export type OhSemanticAuthorityRefV1 = Readonly<{
46
+ authorityId: string;
47
+ authoritySha256: Sha256Hex;
48
+ generation: number;
49
+ records: readonly Readonly<{ key: string; recordSha256: Sha256Hex }>[];
50
+ v: 1;
51
+ }>;
52
+
53
+ export type OhSemanticDocumentV1 = Readonly<{
54
+ content: string;
55
+ key: string;
56
+ recordSha256: Sha256Hex;
57
+ title: string;
58
+ v: 1;
59
+ }>;
60
+
61
+ export type OhSemanticStageResultV1 = Readonly<{
62
+ authorityId: string;
63
+ chunks: number;
64
+ documents: number;
65
+ embedded: number;
66
+ generation: number;
67
+ generationSha256: Sha256Hex;
68
+ membershipSha256: Sha256Hex;
69
+ reused: number;
70
+ status: "staged";
71
+ v: 1;
72
+ }>;
73
+
74
+ export type OhSemanticPublishResultV1 = Readonly<{
75
+ authorityId: string;
76
+ generation: number;
77
+ generationSha256: Sha256Hex;
78
+ published: boolean;
79
+ v: 1;
80
+ }>;
81
+
82
+ export type OhSemanticSearchResultV1 = Readonly<{
83
+ chunkOrdinal: number;
84
+ key: string;
85
+ recordSha256: Sha256Hex;
86
+ score: number;
87
+ v: 1;
88
+ }>;
89
+
90
+ export type OhSemanticPurgeResultV1 = Readonly<{
91
+ authorityId: string;
92
+ generations: number;
93
+ memberships: number;
94
+ orphanVectors: number;
95
+ purgedAt: string;
96
+ v: 1;
97
+ }>;
98
+
99
+ const SCHEMA_NAME = "oh.libsql-semantic-cache.v1";
100
+ const SCHEMA_VERSION = 1;
101
+ const VECTOR_BYTES = OH_CLOUDFLARE_EMBEDDING_PROFILE_V1.dimensions * 4;
102
+
103
+ const SCHEMA_TABLE = `CREATE TABLE IF NOT EXISTS oh_semantic_schemas (
104
+ version INTEGER PRIMARY KEY,
105
+ name TEXT NOT NULL UNIQUE,
106
+ schema_sha256 TEXT NOT NULL,
107
+ applied_at TEXT NOT NULL
108
+ ) STRICT`;
109
+
110
+ const SCHEMA_STATEMENTS = Object.freeze([
111
+ `CREATE TABLE IF NOT EXISTS oh_semantic_vectors (
112
+ profile_sha256 TEXT NOT NULL,
113
+ renderer_sha256 TEXT NOT NULL,
114
+ input_sha256 TEXT NOT NULL,
115
+ vector_sha256 TEXT NOT NULL,
116
+ vector BLOB NOT NULL,
117
+ created_at TEXT NOT NULL,
118
+ PRIMARY KEY(profile_sha256, renderer_sha256, input_sha256)
119
+ ) STRICT`,
120
+ `CREATE TABLE IF NOT EXISTS oh_semantic_generations (
121
+ authority_id TEXT NOT NULL,
122
+ generation INTEGER NOT NULL CHECK(generation >= 0),
123
+ authority_sha256 TEXT NOT NULL,
124
+ profile_sha256 TEXT NOT NULL,
125
+ renderer_sha256 TEXT NOT NULL,
126
+ membership_sha256 TEXT NOT NULL,
127
+ generation_sha256 TEXT NOT NULL UNIQUE,
128
+ document_count INTEGER NOT NULL CHECK(document_count >= 0),
129
+ chunk_count INTEGER NOT NULL CHECK(chunk_count >= 0),
130
+ created_at TEXT NOT NULL,
131
+ PRIMARY KEY(authority_id, generation)
132
+ ) STRICT`,
133
+ `CREATE TABLE IF NOT EXISTS oh_semantic_memberships (
134
+ authority_id TEXT NOT NULL,
135
+ generation INTEGER NOT NULL CHECK(generation >= 0),
136
+ generation_sha256 TEXT NOT NULL,
137
+ record_key TEXT NOT NULL,
138
+ record_sha256 TEXT NOT NULL,
139
+ ordinal INTEGER NOT NULL CHECK(ordinal >= 0),
140
+ input_sha256 TEXT NOT NULL,
141
+ PRIMARY KEY(authority_id, generation, record_key, ordinal)
142
+ ) STRICT`,
143
+ `CREATE TABLE IF NOT EXISTS oh_semantic_heads (
144
+ authority_id TEXT PRIMARY KEY,
145
+ generation INTEGER NOT NULL CHECK(generation >= 0),
146
+ authority_sha256 TEXT NOT NULL,
147
+ profile_sha256 TEXT NOT NULL,
148
+ renderer_sha256 TEXT NOT NULL,
149
+ membership_sha256 TEXT NOT NULL,
150
+ generation_sha256 TEXT NOT NULL,
151
+ published_at TEXT NOT NULL
152
+ ) STRICT`,
153
+ `CREATE TABLE IF NOT EXISTS oh_semantic_purges (
154
+ authority_id TEXT PRIMARY KEY,
155
+ purged_at TEXT NOT NULL
156
+ ) STRICT`,
157
+ `CREATE INDEX IF NOT EXISTS oh_semantic_memberships_generation
158
+ ON oh_semantic_memberships(authority_id, generation, record_key, ordinal)`,
159
+ `CREATE INDEX IF NOT EXISTS oh_semantic_memberships_input
160
+ ON oh_semantic_memberships(input_sha256)`,
161
+ `CREATE TRIGGER IF NOT EXISTS oh_semantic_vectors_no_update
162
+ BEFORE UPDATE ON oh_semantic_vectors
163
+ BEGIN SELECT RAISE(ABORT, 'Oh semantic vectors are immutable'); END`,
164
+ `CREATE TRIGGER IF NOT EXISTS oh_semantic_generations_no_update
165
+ BEFORE UPDATE ON oh_semantic_generations
166
+ BEGIN SELECT RAISE(ABORT, 'Oh semantic generations are immutable'); END`,
167
+ `CREATE TRIGGER IF NOT EXISTS oh_semantic_memberships_no_update
168
+ BEFORE UPDATE ON oh_semantic_memberships
169
+ BEGIN SELECT RAISE(ABORT, 'Oh semantic memberships are immutable'); END`,
170
+ `CREATE TRIGGER IF NOT EXISTS oh_semantic_generations_purge_guard
171
+ BEFORE INSERT ON oh_semantic_generations
172
+ WHEN EXISTS (SELECT 1 FROM oh_semantic_purges WHERE authority_id = NEW.authority_id)
173
+ BEGIN SELECT RAISE(ABORT, 'Oh semantic authority was purged'); END`,
174
+ `CREATE TRIGGER IF NOT EXISTS oh_semantic_memberships_purge_guard
175
+ BEFORE INSERT ON oh_semantic_memberships
176
+ WHEN EXISTS (SELECT 1 FROM oh_semantic_purges WHERE authority_id = NEW.authority_id)
177
+ BEGIN SELECT RAISE(ABORT, 'Oh semantic authority was purged'); END`,
178
+ `CREATE TRIGGER IF NOT EXISTS oh_semantic_memberships_published_guard
179
+ BEFORE INSERT ON oh_semantic_memberships
180
+ WHEN EXISTS (SELECT 1 FROM oh_semantic_heads
181
+ WHERE authority_id = NEW.authority_id AND generation = NEW.generation)
182
+ AND NOT EXISTS (SELECT 1 FROM oh_semantic_memberships
183
+ WHERE authority_id = NEW.authority_id AND generation = NEW.generation
184
+ AND generation_sha256 = NEW.generation_sha256
185
+ AND record_key = NEW.record_key AND record_sha256 = NEW.record_sha256
186
+ AND ordinal = NEW.ordinal AND input_sha256 = NEW.input_sha256)
187
+ BEGIN SELECT RAISE(ABORT, 'Oh semantic generation is published'); END`,
188
+ `CREATE TRIGGER IF NOT EXISTS oh_semantic_heads_insert_purge_guard
189
+ BEFORE INSERT ON oh_semantic_heads
190
+ WHEN EXISTS (SELECT 1 FROM oh_semantic_purges WHERE authority_id = NEW.authority_id)
191
+ BEGIN SELECT RAISE(ABORT, 'Oh semantic authority was purged'); END`,
192
+ `CREATE TRIGGER IF NOT EXISTS oh_semantic_heads_update_purge_guard
193
+ BEFORE UPDATE ON oh_semantic_heads
194
+ WHEN EXISTS (SELECT 1 FROM oh_semantic_purges WHERE authority_id = NEW.authority_id)
195
+ BEGIN SELECT RAISE(ABORT, 'Oh semantic authority was purged'); END`,
196
+ `CREATE TRIGGER IF NOT EXISTS oh_semantic_purges_no_update
197
+ BEFORE UPDATE ON oh_semantic_purges
198
+ BEGIN SELECT RAISE(ABORT, 'Oh semantic purge markers are immutable'); END`,
199
+ `CREATE TRIGGER IF NOT EXISTS oh_semantic_purges_no_delete
200
+ BEFORE DELETE ON oh_semantic_purges
201
+ BEGIN SELECT RAISE(ABORT, 'Oh semantic purge markers are immutable'); END`,
202
+ ]);
203
+
204
+ function normalizedSchemaSql(sql: string): string {
205
+ return sql.replace(/\bIF\s+NOT\s+EXISTS\b/giu, "").replace(/\s+/gu, " ").trim();
206
+ }
207
+
208
+ type SchemaObject = Readonly<{
209
+ name: string;
210
+ sql: string;
211
+ tableName: string;
212
+ type: "index" | "table" | "trigger";
213
+ }>;
214
+
215
+ function expectedSchemaObject(statement: string): SchemaObject {
216
+ const match = /^CREATE\s+(TABLE|INDEX|TRIGGER)(?:\s+IF\s+NOT\s+EXISTS)?\s+([a-z0-9_]+)/iu
217
+ .exec(statement.trim());
218
+ if (match === null) throw new Error("Invalid compiled semantic schema statement.");
219
+ const declared = match[1]?.toLowerCase();
220
+ const type = declared === "index" ? "index" as const
221
+ : declared === "trigger" ? "trigger" as const : "table" as const;
222
+ const name = match[2] as string;
223
+ const owner = type === "table" ? name : /\bON\s+([a-z0-9_]+)/iu.exec(statement)?.[1];
224
+ if (owner === undefined) throw new Error("Invalid compiled semantic schema owner.");
225
+ return { name, sql: normalizedSchemaSql(statement), tableName: owner, type };
226
+ }
227
+
228
+ const EXPECTED_SCHEMA_OBJECTS = Object.freeze(
229
+ [SCHEMA_TABLE, ...SCHEMA_STATEMENTS]
230
+ .map(expectedSchemaObject)
231
+ .sort((left, right) => canonicalJson([left.type, left.name])
232
+ .localeCompare(canonicalJson([right.type, right.name]))),
233
+ );
234
+ const SCHEMA_SHA256 = canonicalSha256(EXPECTED_SCHEMA_OBJECTS);
235
+
236
+ function rowValue(
237
+ row: Readonly<Record<string, unknown>> | readonly unknown[],
238
+ key: string,
239
+ index: number,
240
+ ): unknown {
241
+ return Array.isArray(row) ? row[index] : (row as Readonly<Record<string, unknown>>)[key];
242
+ }
243
+
244
+ function integer(value: unknown): number | null {
245
+ if (typeof value === "number") return Number.isSafeInteger(value) ? value : null;
246
+ if (typeof value === "bigint") {
247
+ const converted = Number(value);
248
+ return Number.isSafeInteger(converted) ? converted : null;
249
+ }
250
+ return null;
251
+ }
252
+
253
+ function rowsAffected(result: OhLibSqlResultV1): number {
254
+ return typeof result.rowsAffected === "number" && Number.isSafeInteger(result.rowsAffected)
255
+ && result.rowsAffected >= 0 ? result.rowsAffected : 0;
256
+ }
257
+
258
+ function parseAuthorityId(value: unknown): string {
259
+ const parsed = safeCode(value, 256);
260
+ if (parsed === null) throw new OhLibSqlSemanticError("invalid-input", "Invalid semantic authority ID.");
261
+ return parsed;
262
+ }
263
+
264
+ function parseRecordKey(value: unknown): string {
265
+ const parsed = safeCode(value, 512);
266
+ if (parsed === null) throw new OhLibSqlSemanticError("invalid-input", "Invalid semantic record key.");
267
+ return parsed;
268
+ }
269
+
270
+ function parseGeneration(value: unknown): number {
271
+ if (!Number.isSafeInteger(value) || (value as number) < 0) {
272
+ throw new OhLibSqlSemanticError("invalid-input", "Invalid semantic authority generation.");
273
+ }
274
+ return value as number;
275
+ }
276
+
277
+ function parseDigest(value: unknown, label: string): Sha256Hex {
278
+ const digest = parseSha256Hex(value);
279
+ if (digest === null) throw new OhLibSqlSemanticError("invalid-input", `Invalid ${label} digest.`);
280
+ return digest;
281
+ }
282
+
283
+ function parseInstant(value: unknown): string {
284
+ const instant = parseCanonicalInstantV1(value);
285
+ if (instant === null) throw new OhLibSqlSemanticError("invalid-input", "Invalid semantic instant.");
286
+ return instant;
287
+ }
288
+
289
+ async function schemaObjects(client: OhLibSqlClientV1): Promise<readonly SchemaObject[]> {
290
+ const result = await client.execute(`SELECT type, name, tbl_name, sql FROM sqlite_schema
291
+ WHERE sql IS NOT NULL AND (name GLOB 'oh_semantic_*' OR tbl_name GLOB 'oh_semantic_*')
292
+ ORDER BY type, name`);
293
+ return result.rows.map((row) => {
294
+ const type = rowValue(row, "type", 0);
295
+ const name = rowValue(row, "name", 1);
296
+ const tableName = rowValue(row, "tbl_name", 2);
297
+ const sql = rowValue(row, "sql", 3);
298
+ if ((type !== "index" && type !== "table" && type !== "trigger")
299
+ || typeof name !== "string" || typeof tableName !== "string" || typeof sql !== "string") {
300
+ throw new OhLibSqlSemanticError("integrity", "The semantic schema inventory is malformed.");
301
+ }
302
+ const schemaType: SchemaObject["type"] = type;
303
+ return { name, sql: normalizedSchemaSql(sql), tableName, type: schemaType };
304
+ }).sort((left, right) => canonicalJson([left.type, left.name])
305
+ .localeCompare(canonicalJson([right.type, right.name])));
306
+ }
307
+
308
+ async function verifySchema(client: OhLibSqlClientV1): Promise<void> {
309
+ let marker: OhLibSqlResultV1;
310
+ try {
311
+ marker = await client.execute({
312
+ args: [SCHEMA_VERSION],
313
+ sql: "SELECT name, schema_sha256 FROM oh_semantic_schemas WHERE version = ?",
314
+ });
315
+ } catch {
316
+ throw new OhLibSqlSemanticError("schema-unavailable", "The semantic cache schema is unavailable.");
317
+ }
318
+ const row = marker.rows[0];
319
+ if (marker.rows.length !== 1 || row === undefined
320
+ || rowValue(row, "name", 0) !== SCHEMA_NAME
321
+ || rowValue(row, "schema_sha256", 1) !== SCHEMA_SHA256) {
322
+ throw new OhLibSqlSemanticError("schema-unavailable", "The semantic cache schema marker is invalid.");
323
+ }
324
+ if (canonicalJson(await schemaObjects(client)) !== canonicalJson(EXPECTED_SCHEMA_OBJECTS)) {
325
+ throw new OhLibSqlSemanticError("integrity", "The semantic cache schema has drifted.");
326
+ }
327
+ }
328
+
329
+ export async function bootstrapOhLibSqlSemanticCacheV1(
330
+ client: OhLibSqlClientV1,
331
+ options: Readonly<{ appliedAt?: string }> = {},
332
+ ): Promise<Readonly<{ schemaSha256: Sha256Hex; schemaVersion: 1; v: 1 }>> {
333
+ const appliedAt = parseInstant(options.appliedAt ?? canonicalNow());
334
+ const existing = await schemaObjects(client);
335
+ if (existing.length === 0) {
336
+ await client.batch([
337
+ { sql: SCHEMA_TABLE },
338
+ ...SCHEMA_STATEMENTS.map((sql) => ({ sql })),
339
+ {
340
+ args: [SCHEMA_VERSION, SCHEMA_NAME, SCHEMA_SHA256, appliedAt],
341
+ sql: `INSERT INTO oh_semantic_schemas(version, name, schema_sha256, applied_at)
342
+ VALUES (?, ?, ?, ?) ON CONFLICT(version) DO NOTHING`,
343
+ },
344
+ ], "write");
345
+ } else if (canonicalJson(existing) !== canonicalJson(EXPECTED_SCHEMA_OBJECTS)) {
346
+ throw new OhLibSqlSemanticError("integrity", "Refusing to bless a partial or drifted semantic schema.");
347
+ }
348
+ await verifySchema(client);
349
+ return Object.freeze({ schemaSha256: SCHEMA_SHA256, schemaVersion: 1, v: 1 });
350
+ }
351
+
352
+ function vectorBytes(vector: readonly number[]): Uint8Array {
353
+ const normalized = normalizeOhEmbeddingV1(vector);
354
+ const bytes = new Uint8Array(VECTOR_BYTES);
355
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
356
+ for (const [index, component] of normalized.entries()) view.setFloat32(index * 4, component, true);
357
+ return bytes;
358
+ }
359
+
360
+ function storedBytes(value: unknown): Uint8Array | null {
361
+ if (value instanceof Uint8Array) return new Uint8Array(value);
362
+ if (value instanceof ArrayBuffer) return new Uint8Array(value.slice(0));
363
+ return null;
364
+ }
365
+
366
+ function decodeVector(value: unknown, expectedSha256: unknown): readonly number[] {
367
+ const bytes = storedBytes(value);
368
+ const digest = parseSha256Hex(expectedSha256);
369
+ if (bytes === null || bytes.byteLength !== VECTOR_BYTES || digest === null
370
+ || sha256Hex(bytes) !== digest) {
371
+ throw new OhLibSqlSemanticError("integrity", "A cached semantic vector is corrupt.");
372
+ }
373
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
374
+ const vector = Array.from({ length: OH_CLOUDFLARE_EMBEDDING_PROFILE_V1.dimensions },
375
+ (_, index) => view.getFloat32(index * 4, true));
376
+ try { return Object.freeze([...normalizeOhEmbeddingV1(vector)]); }
377
+ catch { throw new OhLibSqlSemanticError("integrity", "A cached semantic vector is invalid."); }
378
+ }
379
+
380
+ type Membership = Readonly<{
381
+ input: OhRenderedEmbeddingInputV1;
382
+ inputSha256: Sha256Hex;
383
+ ordinal: number;
384
+ recordKey: string;
385
+ recordSha256: Sha256Hex;
386
+ }>;
387
+
388
+ type Generation = Readonly<{
389
+ authorityId: string;
390
+ authoritySha256: Sha256Hex;
391
+ chunkCount: number;
392
+ createdAt: string;
393
+ documentCount: number;
394
+ generation: number;
395
+ generationSha256: Sha256Hex;
396
+ membershipSha256: Sha256Hex;
397
+ memberships: readonly Membership[];
398
+ }>;
399
+
400
+ function prepareGeneration(input: Readonly<{
401
+ authorityId: string;
402
+ authoritySha256: Sha256Hex;
403
+ createdAt?: string;
404
+ documents: readonly OhSemanticDocumentV1[];
405
+ generation: number;
406
+ maximumChunksPerDocument?: number;
407
+ }>): Generation {
408
+ const authorityId = parseAuthorityId(input.authorityId);
409
+ const authoritySha256 = parseDigest(input.authoritySha256, "authority");
410
+ const generation = parseGeneration(input.generation);
411
+ const createdAt = parseInstant(input.createdAt ?? canonicalNow());
412
+ const maximumChunks = input.maximumChunksPerDocument
413
+ ?? OH_LIBSQL_SEMANTIC_LIMITS_V1.chunksPerDocument;
414
+ if (!Number.isSafeInteger(maximumChunks) || maximumChunks < 1
415
+ || maximumChunks > OH_LIBSQL_SEMANTIC_LIMITS_V1.chunksPerDocument
416
+ || !Array.isArray(input.documents) || input.documents.length < 1
417
+ || input.documents.length > OH_LIBSQL_SEMANTIC_LIMITS_V1.documentsPerGeneration) {
418
+ throw new OhLibSqlSemanticError("invalid-input", "Invalid semantic generation bounds.");
419
+ }
420
+ const documents = input.documents.map((document) => {
421
+ if (document.v !== 1) throw new OhLibSqlSemanticError("invalid-input", "Invalid semantic document version.");
422
+ return {
423
+ ...document,
424
+ key: parseRecordKey(document.key),
425
+ recordSha256: parseDigest(document.recordSha256, "record"),
426
+ };
427
+ }).sort((left, right) => left.key < right.key ? -1 : left.key > right.key ? 1 : 0);
428
+ if (new Set(documents.map(({ key }) => key)).size !== documents.length) {
429
+ throw new OhLibSqlSemanticError("invalid-input", "Semantic document keys must be unique.");
430
+ }
431
+ const memberships: Membership[] = [];
432
+ for (const document of documents) {
433
+ const rendered = renderOhCloudflareEmbeddingDocumentV1({
434
+ content: document.content,
435
+ maximumChunks,
436
+ title: document.title,
437
+ });
438
+ if (rendered.status !== "complete") {
439
+ throw new OhLibSqlSemanticError("invalid-input", "A semantic document exceeds the complete renderer bound.");
440
+ }
441
+ for (const chunk of rendered.chunks) {
442
+ memberships.push(Object.freeze({
443
+ input: chunk.input,
444
+ inputSha256: chunk.input.inputSha256,
445
+ ordinal: chunk.ordinal,
446
+ recordKey: document.key,
447
+ recordSha256: document.recordSha256,
448
+ }));
449
+ }
450
+ }
451
+ if (memberships.length < 1
452
+ || memberships.length > OH_LIBSQL_SEMANTIC_LIMITS_V1.chunksPerGeneration) {
453
+ throw new OhLibSqlSemanticError("invalid-input", "The semantic generation exceeds its chunk bound.");
454
+ }
455
+ const membershipSha256 = canonicalSha256(memberships.map((membership) => ({
456
+ inputSha256: membership.inputSha256,
457
+ ordinal: membership.ordinal,
458
+ recordKey: membership.recordKey,
459
+ recordSha256: membership.recordSha256,
460
+ })));
461
+ const generationSha256 = canonicalSha256({
462
+ authorityId,
463
+ authoritySha256,
464
+ chunkCount: memberships.length,
465
+ documentCount: documents.length,
466
+ generation,
467
+ membershipSha256,
468
+ profileSha256: OH_CLOUDFLARE_EMBEDDING_PROFILE_V1.profileSha256,
469
+ rendererSha256: OH_SEMANTIC_RENDERER_V1.rendererSha256,
470
+ v: 1,
471
+ });
472
+ return Object.freeze({
473
+ authorityId,
474
+ authoritySha256,
475
+ chunkCount: memberships.length,
476
+ createdAt,
477
+ documentCount: documents.length,
478
+ generation,
479
+ generationSha256,
480
+ membershipSha256,
481
+ memberships: Object.freeze(memberships),
482
+ });
483
+ }
484
+
485
+ type StoredGeneration = Readonly<{
486
+ authorityId: string;
487
+ authoritySha256: Sha256Hex;
488
+ chunkCount: number;
489
+ createdAt: string;
490
+ documentCount: number;
491
+ generation: number;
492
+ generationSha256: Sha256Hex;
493
+ membershipSha256: Sha256Hex;
494
+ profileSha256: Sha256Hex;
495
+ rendererSha256: Sha256Hex;
496
+ }>;
497
+
498
+ type StoredHead = Readonly<{
499
+ authorityId: string;
500
+ authoritySha256: Sha256Hex;
501
+ generation: number;
502
+ generationSha256: Sha256Hex;
503
+ membershipSha256: Sha256Hex;
504
+ profileSha256: Sha256Hex;
505
+ publishedAt: string;
506
+ rendererSha256: Sha256Hex;
507
+ }>;
508
+
509
+ function parseStoredGeneration(
510
+ row: Readonly<Record<string, unknown>> | readonly unknown[],
511
+ ): StoredGeneration {
512
+ const authorityId = safeCode(rowValue(row, "authority_id", 0), 256);
513
+ const generation = integer(rowValue(row, "generation", 1));
514
+ const authoritySha256 = parseSha256Hex(rowValue(row, "authority_sha256", 2));
515
+ const profileSha256 = parseSha256Hex(rowValue(row, "profile_sha256", 3));
516
+ const rendererSha256 = parseSha256Hex(rowValue(row, "renderer_sha256", 4));
517
+ const membershipSha256 = parseSha256Hex(rowValue(row, "membership_sha256", 5));
518
+ const generationSha256 = parseSha256Hex(rowValue(row, "generation_sha256", 6));
519
+ const documentCount = integer(rowValue(row, "document_count", 7));
520
+ const chunkCount = integer(rowValue(row, "chunk_count", 8));
521
+ const createdAtValue = rowValue(row, "created_at", 9);
522
+ const createdAt = parseCanonicalInstantV1(createdAtValue);
523
+ if (authorityId === null || generation === null || generation < 0
524
+ || authoritySha256 === null || profileSha256 === null || rendererSha256 === null
525
+ || membershipSha256 === null || generationSha256 === null
526
+ || documentCount === null || documentCount < 1
527
+ || documentCount > OH_LIBSQL_SEMANTIC_LIMITS_V1.documentsPerGeneration
528
+ || chunkCount === null || chunkCount < 1
529
+ || chunkCount > OH_LIBSQL_SEMANTIC_LIMITS_V1.chunksPerGeneration
530
+ || createdAt === null) {
531
+ throw new OhLibSqlSemanticError("integrity", "A stored semantic generation is invalid.");
532
+ }
533
+ return Object.freeze({
534
+ authorityId,
535
+ authoritySha256,
536
+ chunkCount,
537
+ createdAt,
538
+ documentCount,
539
+ generation,
540
+ generationSha256,
541
+ membershipSha256,
542
+ profileSha256,
543
+ rendererSha256,
544
+ });
545
+ }
546
+
547
+ function generationMatches(left: StoredGeneration, right: Generation): boolean {
548
+ return left.authorityId === right.authorityId
549
+ && left.authoritySha256 === right.authoritySha256
550
+ && left.chunkCount === right.chunkCount
551
+ && left.documentCount === right.documentCount
552
+ && left.generation === right.generation
553
+ && left.generationSha256 === right.generationSha256
554
+ && left.membershipSha256 === right.membershipSha256
555
+ && left.profileSha256 === OH_CLOUDFLARE_EMBEDDING_PROFILE_V1.profileSha256
556
+ && left.rendererSha256 === OH_SEMANTIC_RENDERER_V1.rendererSha256;
557
+ }
558
+
559
+ function parseStoredHead(row: Readonly<Record<string, unknown>> | readonly unknown[]): StoredHead {
560
+ const authorityId = safeCode(rowValue(row, "authority_id", 0), 256);
561
+ const generation = integer(rowValue(row, "generation", 1));
562
+ const authoritySha256 = parseSha256Hex(rowValue(row, "authority_sha256", 2));
563
+ const profileSha256 = parseSha256Hex(rowValue(row, "profile_sha256", 3));
564
+ const rendererSha256 = parseSha256Hex(rowValue(row, "renderer_sha256", 4));
565
+ const membershipSha256 = parseSha256Hex(rowValue(row, "membership_sha256", 5));
566
+ const generationSha256 = parseSha256Hex(rowValue(row, "generation_sha256", 6));
567
+ const publishedAtValue = rowValue(row, "published_at", 7);
568
+ const publishedAt = parseCanonicalInstantV1(publishedAtValue);
569
+ if (authorityId === null || generation === null || generation < 0
570
+ || authoritySha256 === null || profileSha256 === null || rendererSha256 === null
571
+ || membershipSha256 === null || generationSha256 === null || publishedAt === null) {
572
+ throw new OhLibSqlSemanticError("integrity", "A stored semantic head is invalid.");
573
+ }
574
+ return Object.freeze({
575
+ authorityId,
576
+ authoritySha256,
577
+ generation,
578
+ generationSha256,
579
+ membershipSha256,
580
+ profileSha256,
581
+ publishedAt,
582
+ rendererSha256,
583
+ });
584
+ }
585
+
586
+ function headMatchesGeneration(head: StoredHead, generation: StoredGeneration): boolean {
587
+ return head.authorityId === generation.authorityId
588
+ && head.authoritySha256 === generation.authoritySha256
589
+ && head.generation === generation.generation
590
+ && head.generationSha256 === generation.generationSha256
591
+ && head.membershipSha256 === generation.membershipSha256
592
+ && head.profileSha256 === generation.profileSha256
593
+ && head.rendererSha256 === generation.rendererSha256;
594
+ }
595
+
596
+ const GENERATION_SELECT = `SELECT authority_id, generation, authority_sha256,
597
+ profile_sha256, renderer_sha256, membership_sha256, generation_sha256,
598
+ document_count, chunk_count, created_at
599
+ FROM oh_semantic_generations WHERE authority_id = ? AND generation = ?`;
600
+ const HEAD_SELECT = `SELECT authority_id, generation, authority_sha256,
601
+ profile_sha256, renderer_sha256, membership_sha256, generation_sha256, published_at
602
+ FROM oh_semantic_heads WHERE authority_id = ?`;
603
+
604
+ async function readGeneration(
605
+ client: OhLibSqlClientV1,
606
+ authorityId: string,
607
+ generation: number,
608
+ ): Promise<StoredGeneration | null> {
609
+ const result = await client.execute({ args: [authorityId, generation], sql: GENERATION_SELECT });
610
+ if (result.rows.length > 1) throw new OhLibSqlSemanticError("integrity", "Duplicate semantic generations.");
611
+ const row = result.rows[0];
612
+ return row === undefined ? null : parseStoredGeneration(row);
613
+ }
614
+
615
+ async function readHead(client: OhLibSqlClientV1, authorityId: string): Promise<StoredHead | null> {
616
+ const result = await client.execute({ args: [authorityId], sql: HEAD_SELECT });
617
+ if (result.rows.length > 1) throw new OhLibSqlSemanticError("integrity", "Duplicate semantic heads.");
618
+ const row = result.rows[0];
619
+ return row === undefined ? null : parseStoredHead(row);
620
+ }
621
+
622
+ async function readPurge(client: OhLibSqlClientV1, authorityId: string): Promise<string | null> {
623
+ const result = await client.execute({
624
+ args: [authorityId],
625
+ sql: "SELECT purged_at FROM oh_semantic_purges WHERE authority_id = ?",
626
+ });
627
+ if (result.rows.length > 1) throw new OhLibSqlSemanticError("integrity", "Duplicate semantic purge markers.");
628
+ const row = result.rows[0];
629
+ if (row === undefined) return null;
630
+ const purgedAt = parseCanonicalInstantV1(rowValue(row, "purged_at", 0));
631
+ if (purgedAt === null) throw new OhLibSqlSemanticError("integrity", "The semantic purge marker is invalid.");
632
+ return purgedAt;
633
+ }
634
+
635
+ async function readMemberships(
636
+ client: OhLibSqlClientV1,
637
+ generation: StoredGeneration,
638
+ ): Promise<readonly Omit<Membership, "input">[]> {
639
+ const memberships: Array<Omit<Membership, "input">> = [];
640
+ for (let offset = 0; offset < generation.chunkCount; offset += OH_LIBSQL_SEMANTIC_LIMITS_V1.searchPage) {
641
+ const result = await client.execute({
642
+ args: [generation.authorityId, generation.generation,
643
+ OH_LIBSQL_SEMANTIC_LIMITS_V1.searchPage, offset],
644
+ sql: `SELECT generation_sha256, record_key, record_sha256, ordinal, input_sha256
645
+ FROM oh_semantic_memberships
646
+ WHERE authority_id = ? AND generation = ?
647
+ ORDER BY record_key, ordinal LIMIT ? OFFSET ?`,
648
+ });
649
+ for (const row of result.rows) {
650
+ const generationSha256 = parseSha256Hex(rowValue(row, "generation_sha256", 0));
651
+ const recordKey = safeCode(rowValue(row, "record_key", 1), 512);
652
+ const recordSha256 = parseSha256Hex(rowValue(row, "record_sha256", 2));
653
+ const ordinal = integer(rowValue(row, "ordinal", 3));
654
+ const inputSha256 = parseSha256Hex(rowValue(row, "input_sha256", 4));
655
+ if (generationSha256 !== generation.generationSha256 || recordKey === null
656
+ || recordSha256 === null || ordinal === null || ordinal < 0
657
+ || ordinal >= OH_LIBSQL_SEMANTIC_LIMITS_V1.chunksPerDocument
658
+ || inputSha256 === null) {
659
+ throw new OhLibSqlSemanticError("integrity", "A semantic generation membership is invalid.");
660
+ }
661
+ memberships.push(Object.freeze({ inputSha256, ordinal, recordKey, recordSha256 }));
662
+ }
663
+ }
664
+ if (memberships.length !== generation.chunkCount
665
+ || canonicalSha256(memberships.map((membership) => ({
666
+ inputSha256: membership.inputSha256,
667
+ ordinal: membership.ordinal,
668
+ recordKey: membership.recordKey,
669
+ recordSha256: membership.recordSha256,
670
+ }))) !== generation.membershipSha256) {
671
+ throw new OhLibSqlSemanticError("integrity", "A semantic generation membership digest is invalid.");
672
+ }
673
+ return Object.freeze(memberships);
674
+ }
675
+
676
+ type StoredVector = Readonly<{
677
+ bytes: Uint8Array;
678
+ inputSha256: Sha256Hex;
679
+ vectorSha256: Sha256Hex;
680
+ }>;
681
+
682
+ async function readVectors(
683
+ client: OhLibSqlClientV1,
684
+ inputSha256s: readonly Sha256Hex[],
685
+ ): Promise<ReadonlyMap<Sha256Hex, StoredVector>> {
686
+ const vectors = new Map<Sha256Hex, StoredVector>();
687
+ for (let offset = 0; offset < inputSha256s.length; offset += 64) {
688
+ const page = inputSha256s.slice(offset, offset + 64);
689
+ if (page.length === 0) continue;
690
+ const placeholders = page.map(() => "?").join(", ");
691
+ const result = await client.execute({
692
+ args: [OH_CLOUDFLARE_EMBEDDING_PROFILE_V1.profileSha256,
693
+ OH_SEMANTIC_RENDERER_V1.rendererSha256, ...page],
694
+ sql: `SELECT input_sha256, vector_sha256, vector FROM oh_semantic_vectors
695
+ WHERE profile_sha256 = ? AND renderer_sha256 = ?
696
+ AND input_sha256 IN (${placeholders}) ORDER BY input_sha256`,
697
+ });
698
+ for (const row of result.rows) {
699
+ const inputSha256 = parseSha256Hex(rowValue(row, "input_sha256", 0));
700
+ const vectorSha256 = parseSha256Hex(rowValue(row, "vector_sha256", 1));
701
+ if (inputSha256 === null || vectorSha256 === null || !page.includes(inputSha256)
702
+ || vectors.has(inputSha256)) {
703
+ throw new OhLibSqlSemanticError("integrity", "A cached semantic vector identity is invalid.");
704
+ }
705
+ const bytes = storedBytes(rowValue(row, "vector", 2));
706
+ if (bytes === null) {
707
+ throw new OhLibSqlSemanticError("integrity", "A cached semantic vector is corrupt.");
708
+ }
709
+ decodeVector(bytes, vectorSha256);
710
+ vectors.set(inputSha256, Object.freeze({
711
+ bytes,
712
+ inputSha256,
713
+ vectorSha256,
714
+ }));
715
+ }
716
+ }
717
+ return vectors;
718
+ }
719
+
720
+ function validateEmbeddingClient(client: OhCloudflareEmbeddingClientV1): void {
721
+ if (client.profile.profileSha256 !== OH_CLOUDFLARE_EMBEDDING_PROFILE_V1.profileSha256) {
722
+ throw new OhLibSqlSemanticError("invalid-input", "The embedding client profile is incompatible.");
723
+ }
724
+ }
725
+
726
+ export class OhLibSqlSemanticCacheV1 {
727
+ readonly #client: OhLibSqlClientV1;
728
+ readonly #closeClient: boolean;
729
+ #closed = false;
730
+
731
+ private constructor(client: OhLibSqlClientV1, closeClient: boolean) {
732
+ this.#client = client;
733
+ this.#closeClient = closeClient;
734
+ }
735
+
736
+ /** @internal Public callers should use `openOhLibSqlSemanticCacheV1`. */
737
+ static async open(client: OhLibSqlClientV1, closeClient: boolean): Promise<OhLibSqlSemanticCacheV1> {
738
+ await verifySchema(client);
739
+ return new OhLibSqlSemanticCacheV1(client, closeClient);
740
+ }
741
+
742
+ #open(): void {
743
+ if (this.#closed) throw new OhLibSqlSemanticError("schema-unavailable", "The semantic cache is closed.");
744
+ }
745
+
746
+ async close(): Promise<void> {
747
+ if (this.#closed) return;
748
+ this.#closed = true;
749
+ if (this.#closeClient) this.#client.close?.();
750
+ }
751
+
752
+ async stage(input: Readonly<{
753
+ authorityId: string;
754
+ authoritySha256: Sha256Hex;
755
+ createdAt?: string;
756
+ documents: readonly OhSemanticDocumentV1[];
757
+ embeddingClient: OhCloudflareEmbeddingClientV1;
758
+ generation: number;
759
+ maximumChunksPerDocument?: number;
760
+ signal?: AbortSignal;
761
+ }>): Promise<OhSemanticStageResultV1> {
762
+ this.#open();
763
+ validateEmbeddingClient(input.embeddingClient);
764
+ const prepared = prepareGeneration(input);
765
+ if (await readPurge(this.#client, prepared.authorityId) !== null) {
766
+ throw new OhLibSqlSemanticError("purged", "The semantic authority was purged.");
767
+ }
768
+ const uniqueInputs = new Map<Sha256Hex, OhRenderedEmbeddingInputV1>();
769
+ for (const membership of prepared.memberships) uniqueInputs.set(membership.inputSha256, membership.input);
770
+ const orderedInputs = [...uniqueInputs.entries()].sort(([left], [right]) => left < right ? -1 : 1);
771
+ const existingVectors = await readVectors(this.#client, orderedInputs.map(([digest]) => digest));
772
+ const missing = orderedInputs.filter(([digest]) => !existingVectors.has(digest));
773
+ const candidateVectors = new Map(existingVectors);
774
+ for (let offset = 0; offset < missing.length; offset += OH_LIBSQL_SEMANTIC_LIMITS_V1.embeddingBatch) {
775
+ const page = missing.slice(offset, offset + OH_LIBSQL_SEMANTIC_LIMITS_V1.embeddingBatch);
776
+ const vectors = await input.embeddingClient.embed(
777
+ page.map(([, rendered]) => rendered),
778
+ input.signal === undefined ? {} : { signal: input.signal },
779
+ );
780
+ if (vectors.length !== page.length) {
781
+ throw new OhLibSqlSemanticError("integrity", "The embedding client returned a mismatched vector batch.");
782
+ }
783
+ for (const [index, [inputSha256]] of page.entries()) {
784
+ const vector = vectors[index];
785
+ if (vector === undefined) throw new OhLibSqlSemanticError("integrity", "A semantic vector is missing.");
786
+ const bytes = vectorBytes(vector);
787
+ const vectorSha256 = sha256Hex(bytes);
788
+ decodeVector(bytes, vectorSha256);
789
+ candidateVectors.set(inputSha256, Object.freeze({ bytes, inputSha256, vectorSha256 }));
790
+ }
791
+ }
792
+ const statements: OhLibSqlStatementV1[] = [{
793
+ args: [prepared.authorityId, prepared.generation, prepared.authoritySha256,
794
+ OH_CLOUDFLARE_EMBEDDING_PROFILE_V1.profileSha256,
795
+ OH_SEMANTIC_RENDERER_V1.rendererSha256, prepared.membershipSha256,
796
+ prepared.generationSha256, prepared.documentCount, prepared.chunkCount,
797
+ prepared.createdAt, prepared.authorityId],
798
+ sql: `INSERT INTO oh_semantic_generations(authority_id, generation,
799
+ authority_sha256, profile_sha256, renderer_sha256, membership_sha256,
800
+ generation_sha256, document_count, chunk_count, created_at)
801
+ SELECT ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
802
+ WHERE NOT EXISTS (SELECT 1 FROM oh_semantic_purges WHERE authority_id = ?)
803
+ ON CONFLICT DO NOTHING`,
804
+ }];
805
+ for (const [inputSha256] of orderedInputs) {
806
+ const candidate = candidateVectors.get(inputSha256);
807
+ if (candidate === undefined) {
808
+ throw new OhLibSqlSemanticError("integrity", "A semantic vector candidate is missing.");
809
+ }
810
+ statements.push({
811
+ args: [OH_CLOUDFLARE_EMBEDDING_PROFILE_V1.profileSha256,
812
+ OH_SEMANTIC_RENDERER_V1.rendererSha256, inputSha256, candidate.vectorSha256,
813
+ candidate.bytes, prepared.createdAt, prepared.authorityId, prepared.generation,
814
+ prepared.generationSha256, prepared.authorityId],
815
+ sql: `INSERT INTO oh_semantic_vectors(profile_sha256, renderer_sha256,
816
+ input_sha256, vector_sha256, vector, created_at)
817
+ SELECT ?, ?, ?, ?, ?, ?
818
+ WHERE EXISTS (SELECT 1 FROM oh_semantic_generations
819
+ WHERE authority_id = ? AND generation = ? AND generation_sha256 = ?)
820
+ AND NOT EXISTS (SELECT 1 FROM oh_semantic_purges WHERE authority_id = ?)
821
+ ON CONFLICT DO NOTHING`,
822
+ });
823
+ }
824
+ for (const membership of prepared.memberships) {
825
+ statements.push({
826
+ args: [prepared.authorityId, prepared.generation, prepared.generationSha256,
827
+ membership.recordKey, membership.recordSha256, membership.ordinal,
828
+ membership.inputSha256, prepared.authorityId, prepared.generation,
829
+ prepared.generationSha256, prepared.authorityId],
830
+ sql: `INSERT INTO oh_semantic_memberships(authority_id, generation,
831
+ generation_sha256, record_key, record_sha256, ordinal, input_sha256)
832
+ SELECT ?, ?, ?, ?, ?, ?, ?
833
+ WHERE EXISTS (SELECT 1 FROM oh_semantic_generations
834
+ WHERE authority_id = ? AND generation = ? AND generation_sha256 = ?)
835
+ AND NOT EXISTS (SELECT 1 FROM oh_semantic_purges WHERE authority_id = ?)
836
+ ON CONFLICT DO NOTHING`,
837
+ });
838
+ }
839
+ await this.#client.batch(statements, "write");
840
+ const completeVectors = await readVectors(this.#client, orderedInputs.map(([digest]) => digest));
841
+ if (completeVectors.size !== orderedInputs.length) {
842
+ if (await readPurge(this.#client, prepared.authorityId) !== null) {
843
+ throw new OhLibSqlSemanticError("purged", "The semantic authority was purged.");
844
+ }
845
+ throw new OhLibSqlSemanticError("integrity", "The semantic vector cache did not converge.");
846
+ }
847
+ const stored = await readGeneration(this.#client, prepared.authorityId, prepared.generation);
848
+ if (await readPurge(this.#client, prepared.authorityId) !== null) {
849
+ throw new OhLibSqlSemanticError("purged", "The semantic authority was purged.");
850
+ }
851
+ if (stored === null || !generationMatches(stored, prepared)) {
852
+ throw new OhLibSqlSemanticError("conflict", "The semantic generation identity conflicts.");
853
+ }
854
+ const memberships = await readMemberships(this.#client, stored);
855
+ if (canonicalJson(memberships) !== canonicalJson(prepared.memberships.map((membership) => ({
856
+ inputSha256: membership.inputSha256,
857
+ ordinal: membership.ordinal,
858
+ recordKey: membership.recordKey,
859
+ recordSha256: membership.recordSha256,
860
+ })))) {
861
+ throw new OhLibSqlSemanticError("conflict", "The semantic generation membership conflicts.");
862
+ }
863
+ return Object.freeze({
864
+ authorityId: prepared.authorityId,
865
+ chunks: prepared.chunkCount,
866
+ documents: prepared.documentCount,
867
+ embedded: missing.length,
868
+ generation: prepared.generation,
869
+ generationSha256: prepared.generationSha256,
870
+ membershipSha256: prepared.membershipSha256,
871
+ reused: orderedInputs.length - missing.length,
872
+ status: "staged",
873
+ v: 1,
874
+ });
875
+ }
876
+
877
+ async publish(input: Readonly<{
878
+ authorityId: string;
879
+ expectedPublishedGeneration: number | null;
880
+ generation: number;
881
+ publishedAt?: string;
882
+ }>): Promise<OhSemanticPublishResultV1> {
883
+ this.#open();
884
+ const authorityId = parseAuthorityId(input.authorityId);
885
+ const generationNumber = parseGeneration(input.generation);
886
+ const expected = input.expectedPublishedGeneration === null
887
+ ? null : parseGeneration(input.expectedPublishedGeneration);
888
+ const publishedAt = parseInstant(input.publishedAt ?? canonicalNow());
889
+ if (await readPurge(this.#client, authorityId) !== null) {
890
+ throw new OhLibSqlSemanticError("purged", "The semantic authority was purged.");
891
+ }
892
+ const generation = await readGeneration(this.#client, authorityId, generationNumber);
893
+ if (generation === null) {
894
+ throw new OhLibSqlSemanticError("conflict", "The semantic generation is not staged.");
895
+ }
896
+ await readMemberships(this.#client, generation);
897
+ const before = await readHead(this.#client, authorityId);
898
+ if (before !== null && headMatchesGeneration(before, generation)) {
899
+ return Object.freeze({
900
+ authorityId,
901
+ generation: generationNumber,
902
+ generationSha256: generation.generationSha256,
903
+ published: false,
904
+ v: 1,
905
+ });
906
+ }
907
+ if ((before === null) !== (expected === null)
908
+ || (before !== null && before.generation !== expected)
909
+ || (before !== null && generationNumber < before.generation)) {
910
+ throw new OhLibSqlSemanticError("conflict", "The semantic published-head precondition failed.");
911
+ }
912
+ let result: OhLibSqlResultV1;
913
+ const values = [generation.authorityId, generation.generation,
914
+ generation.authoritySha256, generation.profileSha256, generation.rendererSha256,
915
+ generation.membershipSha256, generation.generationSha256, publishedAt];
916
+ if (expected === null) {
917
+ result = await this.#client.execute({
918
+ args: [...values, generation.authorityId, generation.generation,
919
+ generation.generationSha256, authorityId, authorityId],
920
+ sql: `INSERT INTO oh_semantic_heads(authority_id, generation,
921
+ authority_sha256, profile_sha256, renderer_sha256, membership_sha256,
922
+ generation_sha256, published_at)
923
+ SELECT ?, ?, ?, ?, ?, ?, ?, ?
924
+ WHERE EXISTS (SELECT 1 FROM oh_semantic_generations
925
+ WHERE authority_id = ? AND generation = ? AND generation_sha256 = ?)
926
+ AND NOT EXISTS (SELECT 1 FROM oh_semantic_purges WHERE authority_id = ?)
927
+ AND NOT EXISTS (SELECT 1 FROM oh_semantic_heads WHERE authority_id = ?)
928
+ ON CONFLICT DO NOTHING`,
929
+ });
930
+ } else {
931
+ result = await this.#client.execute({
932
+ args: [generation.generation, generation.authoritySha256, generation.profileSha256,
933
+ generation.rendererSha256, generation.membershipSha256,
934
+ generation.generationSha256, publishedAt, authorityId, expected,
935
+ generation.authorityId, generation.generation, generation.generationSha256,
936
+ authorityId],
937
+ sql: `UPDATE oh_semantic_heads SET generation = ?, authority_sha256 = ?,
938
+ profile_sha256 = ?, renderer_sha256 = ?, membership_sha256 = ?,
939
+ generation_sha256 = ?, published_at = ?
940
+ WHERE authority_id = ? AND generation = ?
941
+ AND EXISTS (SELECT 1 FROM oh_semantic_generations
942
+ WHERE authority_id = ? AND generation = ? AND generation_sha256 = ?)
943
+ AND NOT EXISTS (SELECT 1 FROM oh_semantic_purges WHERE authority_id = ?)`,
944
+ });
945
+ }
946
+ if (await readPurge(this.#client, authorityId) !== null) {
947
+ throw new OhLibSqlSemanticError("purged", "The semantic authority was purged.");
948
+ }
949
+ const after = await readHead(this.#client, authorityId);
950
+ if (after === null || !headMatchesGeneration(after, generation)) {
951
+ throw new OhLibSqlSemanticError("conflict", "The semantic published head did not converge.");
952
+ }
953
+ return Object.freeze({
954
+ authorityId,
955
+ generation: generationNumber,
956
+ generationSha256: generation.generationSha256,
957
+ published: rowsAffected(result) > 0,
958
+ v: 1,
959
+ });
960
+ }
961
+
962
+ async search(input: Readonly<{
963
+ authority: OhSemanticAuthorityRefV1;
964
+ embeddingClient: OhCloudflareEmbeddingClientV1;
965
+ limit?: number;
966
+ query: string;
967
+ signal?: AbortSignal;
968
+ }>): Promise<readonly OhSemanticSearchResultV1[]> {
969
+ this.#open();
970
+ validateEmbeddingClient(input.embeddingClient);
971
+ const authorityId = parseAuthorityId(input.authority.authorityId);
972
+ const authoritySha256 = parseDigest(input.authority.authoritySha256, "authority");
973
+ const authorityGeneration = parseGeneration(input.authority.generation);
974
+ const limit = input.limit ?? 10;
975
+ if (input.authority.v !== 1 || !Number.isSafeInteger(limit) || limit < 1
976
+ || limit > OH_LIBSQL_SEMANTIC_LIMITS_V1.searchLimit
977
+ || !Array.isArray(input.authority.records)
978
+ || input.authority.records.length > OH_LIBSQL_SEMANTIC_LIMITS_V1.documentsPerGeneration) {
979
+ throw new OhLibSqlSemanticError("invalid-input", "Invalid semantic search authority or limit.");
980
+ }
981
+ const records = new Map<string, Sha256Hex>();
982
+ for (const record of input.authority.records) {
983
+ const key = parseRecordKey(record.key);
984
+ const recordSha256 = parseDigest(record.recordSha256, "record");
985
+ if (records.has(key)) throw new OhLibSqlSemanticError("invalid-input", "Duplicate authority record key.");
986
+ records.set(key, recordSha256);
987
+ }
988
+ if (await readPurge(this.#client, authorityId) !== null) return Object.freeze([]);
989
+ const head = await readHead(this.#client, authorityId);
990
+ if (head === null || head.authoritySha256 !== authoritySha256
991
+ || head.generation !== authorityGeneration
992
+ || head.profileSha256 !== OH_CLOUDFLARE_EMBEDDING_PROFILE_V1.profileSha256
993
+ || head.rendererSha256 !== OH_SEMANTIC_RENDERER_V1.rendererSha256) {
994
+ return Object.freeze([]);
995
+ }
996
+ const generation = await readGeneration(this.#client, authorityId, authorityGeneration);
997
+ if (generation === null || !headMatchesGeneration(head, generation)) return Object.freeze([]);
998
+ const renderedQuery = renderOhCloudflareEmbeddingQueryV1(input.query);
999
+ const queryVectors = await input.embeddingClient.embed(
1000
+ [renderedQuery],
1001
+ input.signal === undefined ? {} : { signal: input.signal },
1002
+ );
1003
+ const queryVector = queryVectors[0];
1004
+ if (queryVectors.length !== 1 || queryVector === undefined) {
1005
+ throw new OhLibSqlSemanticError("integrity", "The query embedding response is invalid.");
1006
+ }
1007
+ const normalizedQuery = normalizeOhEmbeddingV1(queryVector);
1008
+ const best = new Map<string, OhSemanticSearchResultV1>();
1009
+ let scanned = 0;
1010
+ for (let offset = 0; offset < generation.chunkCount;
1011
+ offset += OH_LIBSQL_SEMANTIC_LIMITS_V1.searchPage) {
1012
+ const result = await this.#client.execute({
1013
+ args: [OH_CLOUDFLARE_EMBEDDING_PROFILE_V1.profileSha256,
1014
+ OH_SEMANTIC_RENDERER_V1.rendererSha256, authorityId, authorityGeneration,
1015
+ OH_LIBSQL_SEMANTIC_LIMITS_V1.searchPage, offset],
1016
+ sql: `SELECT membership.generation_sha256, membership.record_key,
1017
+ membership.record_sha256, membership.ordinal, membership.input_sha256,
1018
+ vector.vector_sha256, vector.vector
1019
+ FROM oh_semantic_memberships AS membership
1020
+ JOIN oh_semantic_vectors AS vector
1021
+ ON vector.input_sha256 = membership.input_sha256
1022
+ AND vector.profile_sha256 = ? AND vector.renderer_sha256 = ?
1023
+ WHERE membership.authority_id = ? AND membership.generation = ?
1024
+ ORDER BY membership.record_key, membership.ordinal LIMIT ? OFFSET ?`,
1025
+ });
1026
+ for (const row of result.rows) {
1027
+ scanned += 1;
1028
+ const generationSha256 = parseSha256Hex(rowValue(row, "generation_sha256", 0));
1029
+ const key = safeCode(rowValue(row, "record_key", 1), 512);
1030
+ const recordSha256 = parseSha256Hex(rowValue(row, "record_sha256", 2));
1031
+ const ordinal = integer(rowValue(row, "ordinal", 3));
1032
+ const inputSha256 = parseSha256Hex(rowValue(row, "input_sha256", 4));
1033
+ const vectorSha256 = parseSha256Hex(rowValue(row, "vector_sha256", 5));
1034
+ if (generationSha256 !== generation.generationSha256 || key === null
1035
+ || recordSha256 === null || ordinal === null || ordinal < 0
1036
+ || ordinal >= OH_LIBSQL_SEMANTIC_LIMITS_V1.chunksPerDocument
1037
+ || inputSha256 === null || vectorSha256 === null) {
1038
+ throw new OhLibSqlSemanticError("integrity", "A semantic search row is invalid.");
1039
+ }
1040
+ if (records.get(key) !== recordSha256) continue;
1041
+ const vector = decodeVector(rowValue(row, "vector", 6), vectorSha256);
1042
+ let score = 0;
1043
+ for (let index = 0; index < normalizedQuery.length; index += 1) {
1044
+ score += (normalizedQuery[index] as number) * (vector[index] as number);
1045
+ }
1046
+ score = Math.max(-1, Math.min(1, score));
1047
+ const previous = best.get(key);
1048
+ if (previous === undefined || score > previous.score
1049
+ || (score === previous.score && ordinal < previous.chunkOrdinal)) {
1050
+ best.set(key, Object.freeze({ chunkOrdinal: ordinal, key, recordSha256, score, v: 1 }));
1051
+ }
1052
+ }
1053
+ }
1054
+ if (scanned !== generation.chunkCount) {
1055
+ throw new OhLibSqlSemanticError("integrity", "The semantic search scan is incomplete.");
1056
+ }
1057
+ const finalHead = await readHead(this.#client, authorityId);
1058
+ if (finalHead === null || canonicalJson(finalHead) !== canonicalJson(head)
1059
+ || await readPurge(this.#client, authorityId) !== null) return Object.freeze([]);
1060
+ return Object.freeze([...best.values()]
1061
+ .sort((left, right) => right.score - left.score
1062
+ || (left.key < right.key ? -1 : left.key > right.key ? 1 : 0))
1063
+ .slice(0, limit));
1064
+ }
1065
+
1066
+ async purgeAuthority(input: Readonly<{
1067
+ authorityId: string;
1068
+ purgedAt?: string;
1069
+ }>): Promise<OhSemanticPurgeResultV1> {
1070
+ this.#open();
1071
+ const authorityId = parseAuthorityId(input.authorityId);
1072
+ const requestedAt = parseInstant(input.purgedAt ?? canonicalNow());
1073
+ const previous = await readPurge(this.#client, authorityId);
1074
+ const results = await this.#client.batch([
1075
+ {
1076
+ args: [authorityId, requestedAt],
1077
+ sql: `INSERT INTO oh_semantic_purges(authority_id, purged_at)
1078
+ VALUES (?, ?) ON CONFLICT DO NOTHING`,
1079
+ },
1080
+ { args: [authorityId], sql: "DELETE FROM oh_semantic_heads WHERE authority_id = ?" },
1081
+ { args: [authorityId], sql: "DELETE FROM oh_semantic_memberships WHERE authority_id = ?" },
1082
+ { args: [authorityId], sql: "DELETE FROM oh_semantic_generations WHERE authority_id = ?" },
1083
+ {
1084
+ sql: `DELETE FROM oh_semantic_vectors AS vector
1085
+ WHERE NOT EXISTS (SELECT 1 FROM oh_semantic_memberships AS membership
1086
+ WHERE membership.input_sha256 = vector.input_sha256)`,
1087
+ },
1088
+ ], "write");
1089
+ const purgedAt = await readPurge(this.#client, authorityId);
1090
+ if (purgedAt === null || (previous !== null && purgedAt !== previous)) {
1091
+ throw new OhLibSqlSemanticError("integrity", "The semantic purge did not converge.");
1092
+ }
1093
+ if (await readHead(this.#client, authorityId) !== null
1094
+ || (await this.#client.execute({
1095
+ args: [authorityId, authorityId],
1096
+ sql: `SELECT authority_id FROM oh_semantic_generations WHERE authority_id = ?
1097
+ UNION ALL SELECT authority_id FROM oh_semantic_memberships WHERE authority_id = ? LIMIT 1`,
1098
+ })).rows.length !== 0) {
1099
+ throw new OhLibSqlSemanticError("integrity", "The semantic authority purge is incomplete.");
1100
+ }
1101
+ return Object.freeze({
1102
+ authorityId,
1103
+ generations: rowsAffected(results[3] ?? { rows: [] }),
1104
+ memberships: rowsAffected(results[2] ?? { rows: [] }),
1105
+ orphanVectors: rowsAffected(results[4] ?? { rows: [] }),
1106
+ purgedAt,
1107
+ v: 1,
1108
+ });
1109
+ }
1110
+ }
1111
+
1112
+ export async function openOhLibSqlSemanticCacheV1(
1113
+ client: OhLibSqlClientV1,
1114
+ options: Readonly<{ closeClient?: boolean }> = {},
1115
+ ): Promise<OhLibSqlSemanticCacheV1> {
1116
+ return await OhLibSqlSemanticCacheV1.open(client, options.closeClient ?? false);
1117
+ }