@hraness/oh 0.3.1 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -21
- package/dist/cli.d.ts +1 -1
- package/dist/cli.js +1 -1
- package/dist/libsql-semantic-v2.d.ts +160 -0
- package/dist/libsql-semantic-v2.d.ts.map +1 -0
- package/dist/semantic-cloud.d.ts +1 -0
- package/dist/semantic-cloud.d.ts.map +1 -1
- package/dist/semantic-cloud.js +1543 -0
- package/package.json +1 -1
- package/skills/oh/SKILL.md +2 -2
- package/spec/README.md +8 -1
- package/spec/manifest.json +5 -1
- package/spec/v1/libsql-semantic-cache-schema-v1.sql +114 -0
- package/spec/v1/libsql-semantic-digest-fixture-v1.json +13 -0
- package/spec/v2/libsql-semantic-cache-schema-v2.sql +175 -0
- package/spec/v2/libsql-semantic-digest-fixture-v2.json +14 -0
- package/spec/v2/manifest.json +11 -0
- package/spec/v2/semantic-cloud.md +106 -0
- package/src/cli.ts +1 -1
- package/src/libsql-semantic-v2.test.ts +1114 -0
- package/src/libsql-semantic-v2.ts +1891 -0
- package/src/libsql-semantic.test.ts +50 -1
- package/src/semantic-cloud.ts +1 -0
|
@@ -0,0 +1,1891 @@
|
|
|
1
|
+
import {
|
|
2
|
+
canonicalJson,
|
|
3
|
+
canonicalNow,
|
|
4
|
+
canonicalSha256,
|
|
5
|
+
parseCanonicalInstantV1,
|
|
6
|
+
parseSha256Hex,
|
|
7
|
+
safeCode,
|
|
8
|
+
sha256Hex,
|
|
9
|
+
type Sha256Hex,
|
|
10
|
+
} from "./canonical";
|
|
11
|
+
import {
|
|
12
|
+
OH_CLOUDFLARE_EMBEDDING_PROFILE_V1,
|
|
13
|
+
OH_SEMANTIC_RENDERER_V1,
|
|
14
|
+
renderOhCloudflareEmbeddingDocumentV1,
|
|
15
|
+
renderOhCloudflareEmbeddingQueryV1,
|
|
16
|
+
type OhCloudflareEmbeddingClientV1,
|
|
17
|
+
type OhRenderedEmbeddingInputV1,
|
|
18
|
+
} from "./cloudflare-embedding";
|
|
19
|
+
import type {
|
|
20
|
+
OhLibSqlClientV1,
|
|
21
|
+
OhLibSqlResultV1,
|
|
22
|
+
OhLibSqlStatementV1,
|
|
23
|
+
} from "./libsql";
|
|
24
|
+
import { normalizeOhEmbeddingV1 } from "./semantic";
|
|
25
|
+
|
|
26
|
+
export const OH_LIBSQL_SEMANTIC_LIMITS_V2 = Object.freeze({
|
|
27
|
+
chunksPerDocument: 64,
|
|
28
|
+
chunksPerGeneration: 4_096,
|
|
29
|
+
documentsPerGeneration: 512,
|
|
30
|
+
embeddingBatch: 16,
|
|
31
|
+
searchLimit: 100,
|
|
32
|
+
searchPage: 128,
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
export class OhLibSqlSemanticV2Error extends Error {
|
|
36
|
+
readonly code: "conflict" | "integrity" | "invalid-input" | "purged" | "schema-unavailable";
|
|
37
|
+
|
|
38
|
+
constructor(code: OhLibSqlSemanticV2Error["code"], message: string) {
|
|
39
|
+
super(message);
|
|
40
|
+
this.name = "OhLibSqlSemanticV2Error";
|
|
41
|
+
this.code = code;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export type OhSemanticAuthorityRefV2 = Readonly<{
|
|
46
|
+
authorityId: string;
|
|
47
|
+
authoritySha256: Sha256Hex;
|
|
48
|
+
generation: number;
|
|
49
|
+
/** Defaults to an authority-specific isolation when omitted. */
|
|
50
|
+
isolationSha256?: Sha256Hex;
|
|
51
|
+
records: readonly Readonly<{ key: string; recordSha256: Sha256Hex }>[];
|
|
52
|
+
v: 2;
|
|
53
|
+
}>;
|
|
54
|
+
|
|
55
|
+
export type OhSemanticDocumentV2 = Readonly<{
|
|
56
|
+
content: string;
|
|
57
|
+
key: string;
|
|
58
|
+
recordSha256: Sha256Hex;
|
|
59
|
+
title: string;
|
|
60
|
+
v: 2;
|
|
61
|
+
}>;
|
|
62
|
+
|
|
63
|
+
export type OhSemanticStageResultV2 = Readonly<{
|
|
64
|
+
authorityId: string;
|
|
65
|
+
chunks: number;
|
|
66
|
+
documents: number;
|
|
67
|
+
embedded: number;
|
|
68
|
+
generation: number;
|
|
69
|
+
generationSha256: Sha256Hex;
|
|
70
|
+
isolationSha256: Sha256Hex;
|
|
71
|
+
membershipSha256: Sha256Hex;
|
|
72
|
+
reused: number;
|
|
73
|
+
status: "staged";
|
|
74
|
+
v: 2;
|
|
75
|
+
}>;
|
|
76
|
+
|
|
77
|
+
export type OhSemanticPublishResultV2 = Readonly<{
|
|
78
|
+
authorityId: string;
|
|
79
|
+
generation: number;
|
|
80
|
+
generationSha256: Sha256Hex;
|
|
81
|
+
isolationSha256: Sha256Hex;
|
|
82
|
+
published: boolean;
|
|
83
|
+
v: 2;
|
|
84
|
+
}>;
|
|
85
|
+
|
|
86
|
+
/** The exact, currently published cache pointer for one semantic authority. */
|
|
87
|
+
export type OhSemanticPublishedHeadV2 = Readonly<{
|
|
88
|
+
authorityId: string;
|
|
89
|
+
authoritySha256: Sha256Hex;
|
|
90
|
+
generation: number;
|
|
91
|
+
generationSha256: Sha256Hex;
|
|
92
|
+
isolationSha256: Sha256Hex;
|
|
93
|
+
membershipSha256: Sha256Hex;
|
|
94
|
+
profileSha256: Sha256Hex;
|
|
95
|
+
publishedAt: string;
|
|
96
|
+
rendererSha256: Sha256Hex;
|
|
97
|
+
v: 2;
|
|
98
|
+
}>;
|
|
99
|
+
|
|
100
|
+
export type OhSemanticSearchResultV2 = Readonly<{
|
|
101
|
+
chunkOrdinal: number;
|
|
102
|
+
key: string;
|
|
103
|
+
recordSha256: Sha256Hex;
|
|
104
|
+
score: number;
|
|
105
|
+
v: 2;
|
|
106
|
+
}>;
|
|
107
|
+
|
|
108
|
+
export type OhSemanticPurgeResultV2 = Readonly<{
|
|
109
|
+
authorityId: string;
|
|
110
|
+
countsRecorded: boolean;
|
|
111
|
+
generations: number;
|
|
112
|
+
isolationScopes: number;
|
|
113
|
+
isolationSha256: Sha256Hex;
|
|
114
|
+
memberships: number;
|
|
115
|
+
orphanVectors: number;
|
|
116
|
+
profileSha256: Sha256Hex;
|
|
117
|
+
publishedGeneration: number | null;
|
|
118
|
+
publishedGenerationSha256: Sha256Hex | null;
|
|
119
|
+
purgeMarkerSha256: Sha256Hex;
|
|
120
|
+
purgeReceiptSha256: Sha256Hex;
|
|
121
|
+
purgedAt: string;
|
|
122
|
+
residualGenerations: 0;
|
|
123
|
+
residualMemberships: 0;
|
|
124
|
+
residualScopedVectors: 0;
|
|
125
|
+
v: 2;
|
|
126
|
+
}>;
|
|
127
|
+
|
|
128
|
+
const SCHEMA_NAME_V1 = "oh.libsql-semantic-cache.v1";
|
|
129
|
+
const SCHEMA_VERSION_V1 = 1;
|
|
130
|
+
const SCHEMA_NAME = "oh.libsql-semantic-cache.v2";
|
|
131
|
+
const SCHEMA_VERSION = 2;
|
|
132
|
+
const VECTOR_BYTES = OH_CLOUDFLARE_EMBEDDING_PROFILE_V1.dimensions * 4;
|
|
133
|
+
const DEFAULT_ISOLATION_KIND = "oh.semantic-authority-isolation.v2";
|
|
134
|
+
const GENERATION_KIND = "oh.semantic-generation.v2";
|
|
135
|
+
const MEMBERSHIP_KIND = "oh.semantic-membership.v2";
|
|
136
|
+
const PURGE_MARKER_KIND = "oh.semantic-purge-marker.v2";
|
|
137
|
+
const PURGE_RECEIPT_KIND = "oh.semantic-purge-receipt.v2";
|
|
138
|
+
const TRANSITION_PAGE_SIZE = 32;
|
|
139
|
+
const TRANSITION_TABLE_NAME = "oh_semantic_v1_purge_transition";
|
|
140
|
+
|
|
141
|
+
const SCHEMA_TABLE = `CREATE TABLE IF NOT EXISTS oh_semantic_schemas (
|
|
142
|
+
version INTEGER PRIMARY KEY,
|
|
143
|
+
name TEXT NOT NULL UNIQUE,
|
|
144
|
+
schema_sha256 TEXT NOT NULL,
|
|
145
|
+
applied_at TEXT NOT NULL
|
|
146
|
+
) STRICT`;
|
|
147
|
+
|
|
148
|
+
const TRANSITION_TABLE = `CREATE TABLE oh_semantic_v1_purge_transition (
|
|
149
|
+
authority_id TEXT PRIMARY KEY,
|
|
150
|
+
purged_at TEXT NOT NULL
|
|
151
|
+
) STRICT`;
|
|
152
|
+
|
|
153
|
+
const SCHEMA_STATEMENTS_V1 = Object.freeze([
|
|
154
|
+
`CREATE TABLE IF NOT EXISTS oh_semantic_vectors (
|
|
155
|
+
profile_sha256 TEXT NOT NULL,
|
|
156
|
+
renderer_sha256 TEXT NOT NULL,
|
|
157
|
+
input_sha256 TEXT NOT NULL,
|
|
158
|
+
vector_sha256 TEXT NOT NULL,
|
|
159
|
+
vector BLOB NOT NULL,
|
|
160
|
+
created_at TEXT NOT NULL,
|
|
161
|
+
PRIMARY KEY(profile_sha256, renderer_sha256, input_sha256)
|
|
162
|
+
) STRICT`,
|
|
163
|
+
`CREATE TABLE IF NOT EXISTS oh_semantic_generations (
|
|
164
|
+
authority_id TEXT NOT NULL,
|
|
165
|
+
generation INTEGER NOT NULL CHECK(generation >= 0),
|
|
166
|
+
authority_sha256 TEXT NOT NULL,
|
|
167
|
+
profile_sha256 TEXT NOT NULL,
|
|
168
|
+
renderer_sha256 TEXT NOT NULL,
|
|
169
|
+
membership_sha256 TEXT NOT NULL,
|
|
170
|
+
generation_sha256 TEXT NOT NULL UNIQUE,
|
|
171
|
+
document_count INTEGER NOT NULL CHECK(document_count >= 0),
|
|
172
|
+
chunk_count INTEGER NOT NULL CHECK(chunk_count >= 0),
|
|
173
|
+
created_at TEXT NOT NULL,
|
|
174
|
+
PRIMARY KEY(authority_id, generation)
|
|
175
|
+
) STRICT`,
|
|
176
|
+
`CREATE TABLE IF NOT EXISTS oh_semantic_memberships (
|
|
177
|
+
authority_id TEXT NOT NULL,
|
|
178
|
+
generation INTEGER NOT NULL CHECK(generation >= 0),
|
|
179
|
+
generation_sha256 TEXT NOT NULL,
|
|
180
|
+
record_key TEXT NOT NULL,
|
|
181
|
+
record_sha256 TEXT NOT NULL,
|
|
182
|
+
ordinal INTEGER NOT NULL CHECK(ordinal >= 0),
|
|
183
|
+
input_sha256 TEXT NOT NULL,
|
|
184
|
+
PRIMARY KEY(authority_id, generation, record_key, ordinal)
|
|
185
|
+
) STRICT`,
|
|
186
|
+
`CREATE TABLE IF NOT EXISTS oh_semantic_heads (
|
|
187
|
+
authority_id TEXT PRIMARY KEY,
|
|
188
|
+
generation INTEGER NOT NULL CHECK(generation >= 0),
|
|
189
|
+
authority_sha256 TEXT NOT NULL,
|
|
190
|
+
profile_sha256 TEXT NOT NULL,
|
|
191
|
+
renderer_sha256 TEXT NOT NULL,
|
|
192
|
+
membership_sha256 TEXT NOT NULL,
|
|
193
|
+
generation_sha256 TEXT NOT NULL,
|
|
194
|
+
published_at TEXT NOT NULL
|
|
195
|
+
) STRICT`,
|
|
196
|
+
`CREATE TABLE IF NOT EXISTS oh_semantic_purges (
|
|
197
|
+
authority_id TEXT PRIMARY KEY,
|
|
198
|
+
purged_at TEXT NOT NULL
|
|
199
|
+
) STRICT`,
|
|
200
|
+
`CREATE INDEX IF NOT EXISTS oh_semantic_memberships_generation
|
|
201
|
+
ON oh_semantic_memberships(authority_id, generation, record_key, ordinal)`,
|
|
202
|
+
`CREATE INDEX IF NOT EXISTS oh_semantic_memberships_input
|
|
203
|
+
ON oh_semantic_memberships(input_sha256)`,
|
|
204
|
+
`CREATE TRIGGER IF NOT EXISTS oh_semantic_vectors_no_update
|
|
205
|
+
BEFORE UPDATE ON oh_semantic_vectors
|
|
206
|
+
BEGIN SELECT RAISE(ABORT, 'Oh semantic vectors are immutable'); END`,
|
|
207
|
+
`CREATE TRIGGER IF NOT EXISTS oh_semantic_generations_no_update
|
|
208
|
+
BEFORE UPDATE ON oh_semantic_generations
|
|
209
|
+
BEGIN SELECT RAISE(ABORT, 'Oh semantic generations are immutable'); END`,
|
|
210
|
+
`CREATE TRIGGER IF NOT EXISTS oh_semantic_memberships_no_update
|
|
211
|
+
BEFORE UPDATE ON oh_semantic_memberships
|
|
212
|
+
BEGIN SELECT RAISE(ABORT, 'Oh semantic memberships are immutable'); END`,
|
|
213
|
+
`CREATE TRIGGER IF NOT EXISTS oh_semantic_generations_purge_guard
|
|
214
|
+
BEFORE INSERT ON oh_semantic_generations
|
|
215
|
+
WHEN EXISTS (SELECT 1 FROM oh_semantic_purges WHERE authority_id = NEW.authority_id)
|
|
216
|
+
BEGIN SELECT RAISE(ABORT, 'Oh semantic authority was purged'); END`,
|
|
217
|
+
`CREATE TRIGGER IF NOT EXISTS oh_semantic_memberships_purge_guard
|
|
218
|
+
BEFORE INSERT ON oh_semantic_memberships
|
|
219
|
+
WHEN EXISTS (SELECT 1 FROM oh_semantic_purges WHERE authority_id = NEW.authority_id)
|
|
220
|
+
BEGIN SELECT RAISE(ABORT, 'Oh semantic authority was purged'); END`,
|
|
221
|
+
`CREATE TRIGGER IF NOT EXISTS oh_semantic_memberships_published_guard
|
|
222
|
+
BEFORE INSERT ON oh_semantic_memberships
|
|
223
|
+
WHEN EXISTS (SELECT 1 FROM oh_semantic_heads
|
|
224
|
+
WHERE authority_id = NEW.authority_id AND generation = NEW.generation)
|
|
225
|
+
AND NOT EXISTS (SELECT 1 FROM oh_semantic_memberships
|
|
226
|
+
WHERE authority_id = NEW.authority_id AND generation = NEW.generation
|
|
227
|
+
AND generation_sha256 = NEW.generation_sha256
|
|
228
|
+
AND record_key = NEW.record_key AND record_sha256 = NEW.record_sha256
|
|
229
|
+
AND ordinal = NEW.ordinal AND input_sha256 = NEW.input_sha256)
|
|
230
|
+
BEGIN SELECT RAISE(ABORT, 'Oh semantic generation is published'); END`,
|
|
231
|
+
`CREATE TRIGGER IF NOT EXISTS oh_semantic_heads_insert_purge_guard
|
|
232
|
+
BEFORE INSERT ON oh_semantic_heads
|
|
233
|
+
WHEN EXISTS (SELECT 1 FROM oh_semantic_purges WHERE authority_id = NEW.authority_id)
|
|
234
|
+
BEGIN SELECT RAISE(ABORT, 'Oh semantic authority was purged'); END`,
|
|
235
|
+
`CREATE TRIGGER IF NOT EXISTS oh_semantic_heads_update_purge_guard
|
|
236
|
+
BEFORE UPDATE ON oh_semantic_heads
|
|
237
|
+
WHEN EXISTS (SELECT 1 FROM oh_semantic_purges WHERE authority_id = NEW.authority_id)
|
|
238
|
+
BEGIN SELECT RAISE(ABORT, 'Oh semantic authority was purged'); END`,
|
|
239
|
+
`CREATE TRIGGER IF NOT EXISTS oh_semantic_purges_no_update
|
|
240
|
+
BEFORE UPDATE ON oh_semantic_purges
|
|
241
|
+
BEGIN SELECT RAISE(ABORT, 'Oh semantic purge markers are immutable'); END`,
|
|
242
|
+
`CREATE TRIGGER IF NOT EXISTS oh_semantic_purges_no_delete
|
|
243
|
+
BEFORE DELETE ON oh_semantic_purges
|
|
244
|
+
BEGIN SELECT RAISE(ABORT, 'Oh semantic purge markers are immutable'); END`,
|
|
245
|
+
]);
|
|
246
|
+
|
|
247
|
+
const SCHEMA_STATEMENTS = Object.freeze([
|
|
248
|
+
`CREATE TABLE IF NOT EXISTS oh_semantic_isolations (
|
|
249
|
+
isolation_sha256 TEXT PRIMARY KEY,
|
|
250
|
+
authority_id TEXT NOT NULL,
|
|
251
|
+
created_at TEXT NOT NULL
|
|
252
|
+
) STRICT`,
|
|
253
|
+
`CREATE TABLE IF NOT EXISTS oh_semantic_vectors (
|
|
254
|
+
isolation_sha256 TEXT NOT NULL,
|
|
255
|
+
profile_sha256 TEXT NOT NULL,
|
|
256
|
+
renderer_sha256 TEXT NOT NULL,
|
|
257
|
+
input_sha256 TEXT NOT NULL,
|
|
258
|
+
vector_sha256 TEXT NOT NULL,
|
|
259
|
+
vector BLOB NOT NULL,
|
|
260
|
+
created_at TEXT NOT NULL,
|
|
261
|
+
PRIMARY KEY(isolation_sha256, profile_sha256, renderer_sha256, input_sha256)
|
|
262
|
+
) STRICT`,
|
|
263
|
+
`CREATE TABLE IF NOT EXISTS oh_semantic_generations (
|
|
264
|
+
authority_id TEXT NOT NULL,
|
|
265
|
+
generation INTEGER NOT NULL CHECK(generation >= 0),
|
|
266
|
+
authority_sha256 TEXT NOT NULL,
|
|
267
|
+
isolation_sha256 TEXT NOT NULL,
|
|
268
|
+
profile_sha256 TEXT NOT NULL,
|
|
269
|
+
renderer_sha256 TEXT NOT NULL,
|
|
270
|
+
membership_sha256 TEXT NOT NULL,
|
|
271
|
+
generation_sha256 TEXT NOT NULL UNIQUE,
|
|
272
|
+
document_count INTEGER NOT NULL CHECK(document_count >= 0),
|
|
273
|
+
chunk_count INTEGER NOT NULL CHECK(chunk_count >= 0),
|
|
274
|
+
created_at TEXT NOT NULL,
|
|
275
|
+
PRIMARY KEY(authority_id, generation)
|
|
276
|
+
) STRICT`,
|
|
277
|
+
`CREATE TABLE IF NOT EXISTS oh_semantic_memberships (
|
|
278
|
+
authority_id TEXT NOT NULL,
|
|
279
|
+
generation INTEGER NOT NULL CHECK(generation >= 0),
|
|
280
|
+
generation_sha256 TEXT NOT NULL,
|
|
281
|
+
isolation_sha256 TEXT NOT NULL,
|
|
282
|
+
record_key TEXT NOT NULL,
|
|
283
|
+
record_sha256 TEXT NOT NULL,
|
|
284
|
+
ordinal INTEGER NOT NULL CHECK(ordinal >= 0),
|
|
285
|
+
input_sha256 TEXT NOT NULL,
|
|
286
|
+
PRIMARY KEY(authority_id, generation, record_key, ordinal)
|
|
287
|
+
) STRICT`,
|
|
288
|
+
`CREATE TABLE IF NOT EXISTS oh_semantic_heads (
|
|
289
|
+
authority_id TEXT PRIMARY KEY,
|
|
290
|
+
generation INTEGER NOT NULL CHECK(generation >= 0),
|
|
291
|
+
authority_sha256 TEXT NOT NULL,
|
|
292
|
+
isolation_sha256 TEXT NOT NULL,
|
|
293
|
+
profile_sha256 TEXT NOT NULL,
|
|
294
|
+
renderer_sha256 TEXT NOT NULL,
|
|
295
|
+
membership_sha256 TEXT NOT NULL,
|
|
296
|
+
generation_sha256 TEXT NOT NULL,
|
|
297
|
+
published_at TEXT NOT NULL
|
|
298
|
+
) STRICT`,
|
|
299
|
+
`CREATE TABLE IF NOT EXISTS oh_semantic_purges (
|
|
300
|
+
authority_id TEXT PRIMARY KEY,
|
|
301
|
+
isolation_sha256 TEXT NOT NULL,
|
|
302
|
+
profile_sha256 TEXT NOT NULL,
|
|
303
|
+
published_generation INTEGER CHECK(published_generation IS NULL OR published_generation >= 0),
|
|
304
|
+
published_generation_sha256 TEXT,
|
|
305
|
+
purged_at TEXT NOT NULL,
|
|
306
|
+
purge_marker_sha256 TEXT NOT NULL,
|
|
307
|
+
generation_count INTEGER NOT NULL CHECK(generation_count >= 0),
|
|
308
|
+
membership_count INTEGER NOT NULL CHECK(membership_count >= 0),
|
|
309
|
+
orphan_vector_count INTEGER NOT NULL CHECK(orphan_vector_count >= 0),
|
|
310
|
+
isolation_scope_count INTEGER NOT NULL CHECK(isolation_scope_count >= 0),
|
|
311
|
+
counts_recorded INTEGER NOT NULL CHECK(counts_recorded IN (0, 1))
|
|
312
|
+
) STRICT`,
|
|
313
|
+
`CREATE INDEX IF NOT EXISTS oh_semantic_isolations_authority
|
|
314
|
+
ON oh_semantic_isolations(authority_id, isolation_sha256)`,
|
|
315
|
+
`CREATE INDEX IF NOT EXISTS oh_semantic_memberships_generation
|
|
316
|
+
ON oh_semantic_memberships(authority_id, generation, record_key, ordinal)`,
|
|
317
|
+
`CREATE INDEX IF NOT EXISTS oh_semantic_memberships_input
|
|
318
|
+
ON oh_semantic_memberships(isolation_sha256, input_sha256)`,
|
|
319
|
+
`CREATE TRIGGER IF NOT EXISTS oh_semantic_isolations_no_update
|
|
320
|
+
BEFORE UPDATE ON oh_semantic_isolations
|
|
321
|
+
BEGIN SELECT RAISE(ABORT, 'Oh semantic isolations are immutable'); END`,
|
|
322
|
+
`CREATE TRIGGER IF NOT EXISTS oh_semantic_isolations_no_delete
|
|
323
|
+
BEFORE DELETE ON oh_semantic_isolations
|
|
324
|
+
BEGIN SELECT RAISE(ABORT, 'Oh semantic isolations are immutable'); END`,
|
|
325
|
+
`CREATE TRIGGER IF NOT EXISTS oh_semantic_isolations_purge_guard
|
|
326
|
+
BEFORE INSERT ON oh_semantic_isolations
|
|
327
|
+
WHEN EXISTS (SELECT 1 FROM oh_semantic_purges WHERE authority_id = NEW.authority_id)
|
|
328
|
+
BEGIN SELECT RAISE(ABORT, 'Oh semantic authority was purged'); END`,
|
|
329
|
+
`CREATE TRIGGER IF NOT EXISTS oh_semantic_vectors_no_update
|
|
330
|
+
BEFORE UPDATE ON oh_semantic_vectors
|
|
331
|
+
BEGIN SELECT RAISE(ABORT, 'Oh semantic vectors are immutable'); END`,
|
|
332
|
+
`CREATE TRIGGER IF NOT EXISTS oh_semantic_vectors_isolation_guard
|
|
333
|
+
BEFORE INSERT ON oh_semantic_vectors
|
|
334
|
+
WHEN NOT EXISTS (SELECT 1 FROM oh_semantic_isolations
|
|
335
|
+
WHERE isolation_sha256 = NEW.isolation_sha256)
|
|
336
|
+
OR EXISTS (SELECT 1 FROM oh_semantic_purges AS purge
|
|
337
|
+
JOIN oh_semantic_isolations AS isolation
|
|
338
|
+
ON isolation.authority_id = purge.authority_id
|
|
339
|
+
WHERE isolation.isolation_sha256 = NEW.isolation_sha256)
|
|
340
|
+
BEGIN SELECT RAISE(ABORT, 'Oh semantic vector isolation is unavailable'); END`,
|
|
341
|
+
`CREATE TRIGGER IF NOT EXISTS oh_semantic_generations_no_update
|
|
342
|
+
BEFORE UPDATE ON oh_semantic_generations
|
|
343
|
+
BEGIN SELECT RAISE(ABORT, 'Oh semantic generations are immutable'); END`,
|
|
344
|
+
`CREATE TRIGGER IF NOT EXISTS oh_semantic_memberships_no_update
|
|
345
|
+
BEFORE UPDATE ON oh_semantic_memberships
|
|
346
|
+
BEGIN SELECT RAISE(ABORT, 'Oh semantic memberships are immutable'); END`,
|
|
347
|
+
`CREATE TRIGGER IF NOT EXISTS oh_semantic_generations_purge_guard
|
|
348
|
+
BEFORE INSERT ON oh_semantic_generations
|
|
349
|
+
WHEN EXISTS (SELECT 1 FROM oh_semantic_purges WHERE authority_id = NEW.authority_id)
|
|
350
|
+
OR NOT EXISTS (SELECT 1 FROM oh_semantic_isolations
|
|
351
|
+
WHERE isolation_sha256 = NEW.isolation_sha256 AND authority_id = NEW.authority_id)
|
|
352
|
+
BEGIN SELECT RAISE(ABORT, 'Oh semantic authority or isolation is unavailable'); END`,
|
|
353
|
+
`CREATE TRIGGER IF NOT EXISTS oh_semantic_memberships_purge_guard
|
|
354
|
+
BEFORE INSERT ON oh_semantic_memberships
|
|
355
|
+
WHEN EXISTS (SELECT 1 FROM oh_semantic_purges WHERE authority_id = NEW.authority_id)
|
|
356
|
+
OR NOT EXISTS (SELECT 1 FROM oh_semantic_generations
|
|
357
|
+
WHERE authority_id = NEW.authority_id AND generation = NEW.generation
|
|
358
|
+
AND generation_sha256 = NEW.generation_sha256
|
|
359
|
+
AND isolation_sha256 = NEW.isolation_sha256)
|
|
360
|
+
BEGIN SELECT RAISE(ABORT, 'Oh semantic authority or isolation is unavailable'); END`,
|
|
361
|
+
`CREATE TRIGGER IF NOT EXISTS oh_semantic_memberships_published_guard
|
|
362
|
+
BEFORE INSERT ON oh_semantic_memberships
|
|
363
|
+
WHEN EXISTS (SELECT 1 FROM oh_semantic_heads
|
|
364
|
+
WHERE authority_id = NEW.authority_id AND generation = NEW.generation)
|
|
365
|
+
AND NOT EXISTS (SELECT 1 FROM oh_semantic_memberships
|
|
366
|
+
WHERE authority_id = NEW.authority_id AND generation = NEW.generation
|
|
367
|
+
AND generation_sha256 = NEW.generation_sha256
|
|
368
|
+
AND isolation_sha256 = NEW.isolation_sha256
|
|
369
|
+
AND record_key = NEW.record_key AND record_sha256 = NEW.record_sha256
|
|
370
|
+
AND ordinal = NEW.ordinal AND input_sha256 = NEW.input_sha256)
|
|
371
|
+
BEGIN SELECT RAISE(ABORT, 'Oh semantic generation is published'); END`,
|
|
372
|
+
`CREATE TRIGGER IF NOT EXISTS oh_semantic_heads_insert_purge_guard
|
|
373
|
+
BEFORE INSERT ON oh_semantic_heads
|
|
374
|
+
WHEN EXISTS (SELECT 1 FROM oh_semantic_purges WHERE authority_id = NEW.authority_id)
|
|
375
|
+
OR NOT EXISTS (SELECT 1 FROM oh_semantic_generations
|
|
376
|
+
WHERE authority_id = NEW.authority_id AND generation = NEW.generation
|
|
377
|
+
AND generation_sha256 = NEW.generation_sha256
|
|
378
|
+
AND isolation_sha256 = NEW.isolation_sha256)
|
|
379
|
+
BEGIN SELECT RAISE(ABORT, 'Oh semantic authority or isolation is unavailable'); END`,
|
|
380
|
+
`CREATE TRIGGER IF NOT EXISTS oh_semantic_heads_update_purge_guard
|
|
381
|
+
BEFORE UPDATE ON oh_semantic_heads
|
|
382
|
+
WHEN EXISTS (SELECT 1 FROM oh_semantic_purges WHERE authority_id = NEW.authority_id)
|
|
383
|
+
OR NOT EXISTS (SELECT 1 FROM oh_semantic_generations
|
|
384
|
+
WHERE authority_id = NEW.authority_id AND generation = NEW.generation
|
|
385
|
+
AND generation_sha256 = NEW.generation_sha256
|
|
386
|
+
AND isolation_sha256 = NEW.isolation_sha256)
|
|
387
|
+
BEGIN SELECT RAISE(ABORT, 'Oh semantic authority or isolation is unavailable'); END`,
|
|
388
|
+
`CREATE TRIGGER IF NOT EXISTS oh_semantic_purges_no_update
|
|
389
|
+
BEFORE UPDATE ON oh_semantic_purges
|
|
390
|
+
BEGIN SELECT RAISE(ABORT, 'Oh semantic purge markers are immutable'); END`,
|
|
391
|
+
`CREATE TRIGGER IF NOT EXISTS oh_semantic_purges_no_delete
|
|
392
|
+
BEFORE DELETE ON oh_semantic_purges
|
|
393
|
+
BEGIN SELECT RAISE(ABORT, 'Oh semantic purge markers are immutable'); END`,
|
|
394
|
+
]);
|
|
395
|
+
|
|
396
|
+
function normalizedSchemaSql(sql: string): string {
|
|
397
|
+
return sql.replace(/\bIF\s+NOT\s+EXISTS\b/giu, "").replace(/\s+/gu, " ").trim();
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
type SchemaObject = Readonly<{
|
|
401
|
+
name: string;
|
|
402
|
+
sql: string;
|
|
403
|
+
tableName: string;
|
|
404
|
+
type: "index" | "table" | "trigger";
|
|
405
|
+
}>;
|
|
406
|
+
|
|
407
|
+
function expectedSchemaObject(statement: string): SchemaObject {
|
|
408
|
+
const match = /^CREATE\s+(TABLE|INDEX|TRIGGER)(?:\s+IF\s+NOT\s+EXISTS)?\s+([a-z0-9_]+)/iu
|
|
409
|
+
.exec(statement.trim());
|
|
410
|
+
if (match === null) throw new Error("Invalid compiled semantic schema statement.");
|
|
411
|
+
const declared = match[1]?.toLowerCase();
|
|
412
|
+
const type = declared === "index" ? "index" as const
|
|
413
|
+
: declared === "trigger" ? "trigger" as const : "table" as const;
|
|
414
|
+
const name = match[2] as string;
|
|
415
|
+
const owner = type === "table" ? name : /\bON\s+([a-z0-9_]+)/iu.exec(statement)?.[1];
|
|
416
|
+
if (owner === undefined) throw new Error("Invalid compiled semantic schema owner.");
|
|
417
|
+
return { name, sql: normalizedSchemaSql(statement), tableName: owner, type };
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
const EXPECTED_SCHEMA_OBJECTS = Object.freeze(
|
|
421
|
+
[SCHEMA_TABLE, ...SCHEMA_STATEMENTS]
|
|
422
|
+
.map(expectedSchemaObject)
|
|
423
|
+
.sort((left, right) => canonicalJson([left.type, left.name])
|
|
424
|
+
.localeCompare(canonicalJson([right.type, right.name]))),
|
|
425
|
+
);
|
|
426
|
+
const SCHEMA_SHA256 = canonicalSha256(EXPECTED_SCHEMA_OBJECTS);
|
|
427
|
+
const EXPECTED_SCHEMA_OBJECTS_V1 = Object.freeze(
|
|
428
|
+
[SCHEMA_TABLE, ...SCHEMA_STATEMENTS_V1]
|
|
429
|
+
.map(expectedSchemaObject)
|
|
430
|
+
.sort((left, right) => canonicalJson([left.type, left.name])
|
|
431
|
+
.localeCompare(canonicalJson([right.type, right.name]))),
|
|
432
|
+
);
|
|
433
|
+
const SCHEMA_SHA256_V1 = canonicalSha256(EXPECTED_SCHEMA_OBJECTS_V1);
|
|
434
|
+
const EXPECTED_TRANSITION_SCHEMA_OBJECTS = Object.freeze(
|
|
435
|
+
[...EXPECTED_SCHEMA_OBJECTS, expectedSchemaObject(TRANSITION_TABLE)]
|
|
436
|
+
.sort((left, right) => canonicalJson([left.type, left.name])
|
|
437
|
+
.localeCompare(canonicalJson([right.type, right.name]))),
|
|
438
|
+
);
|
|
439
|
+
|
|
440
|
+
function rowValue(
|
|
441
|
+
row: Readonly<Record<string, unknown>> | readonly unknown[],
|
|
442
|
+
key: string,
|
|
443
|
+
index: number,
|
|
444
|
+
): unknown {
|
|
445
|
+
return Array.isArray(row) ? row[index] : (row as Readonly<Record<string, unknown>>)[key];
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
function integer(value: unknown): number | null {
|
|
449
|
+
if (typeof value === "number") return Number.isSafeInteger(value) ? value : null;
|
|
450
|
+
if (typeof value === "bigint") {
|
|
451
|
+
const converted = Number(value);
|
|
452
|
+
return Number.isSafeInteger(converted) ? converted : null;
|
|
453
|
+
}
|
|
454
|
+
return null;
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
function rowsAffected(result: OhLibSqlResultV1): number {
|
|
458
|
+
return typeof result.rowsAffected === "number" && Number.isSafeInteger(result.rowsAffected)
|
|
459
|
+
&& result.rowsAffected >= 0 ? result.rowsAffected : 0;
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
function parseAuthorityId(value: unknown): string {
|
|
463
|
+
const parsed = safeCode(value, 256);
|
|
464
|
+
if (parsed === null) throw new OhLibSqlSemanticV2Error("invalid-input", "Invalid semantic authority ID.");
|
|
465
|
+
return parsed;
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
/**
|
|
469
|
+
* Derives the private-by-default cache scope used when a host does not supply
|
|
470
|
+
* its own epoch/profile isolation digest.
|
|
471
|
+
*/
|
|
472
|
+
export function deriveOhSemanticIsolationSha256V2(authorityId: string): Sha256Hex {
|
|
473
|
+
return canonicalSha256({
|
|
474
|
+
authorityId: parseAuthorityId(authorityId),
|
|
475
|
+
kind: DEFAULT_ISOLATION_KIND,
|
|
476
|
+
v: 2,
|
|
477
|
+
});
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
function isolationSha256(authorityId: string, value: unknown): Sha256Hex {
|
|
481
|
+
return value === undefined
|
|
482
|
+
? deriveOhSemanticIsolationSha256V2(authorityId)
|
|
483
|
+
: parseDigest(value, "semantic isolation");
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
function purgeMarkerSha256(
|
|
487
|
+
authorityId: string,
|
|
488
|
+
isolation: Sha256Hex,
|
|
489
|
+
purgedAt: string,
|
|
490
|
+
): Sha256Hex {
|
|
491
|
+
return canonicalSha256({
|
|
492
|
+
authorityId,
|
|
493
|
+
isolationSha256: isolation,
|
|
494
|
+
kind: PURGE_MARKER_KIND,
|
|
495
|
+
purgedAt,
|
|
496
|
+
v: 2,
|
|
497
|
+
});
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
function parseRecordKey(value: unknown): string {
|
|
501
|
+
const parsed = safeCode(value, 512);
|
|
502
|
+
if (parsed === null) throw new OhLibSqlSemanticV2Error("invalid-input", "Invalid semantic record key.");
|
|
503
|
+
return parsed;
|
|
504
|
+
}
|
|
505
|
+
|
|
506
|
+
function parseGeneration(value: unknown): number {
|
|
507
|
+
if (!Number.isSafeInteger(value) || (value as number) < 0) {
|
|
508
|
+
throw new OhLibSqlSemanticV2Error("invalid-input", "Invalid semantic authority generation.");
|
|
509
|
+
}
|
|
510
|
+
return value as number;
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
function parseDigest(value: unknown, label: string): Sha256Hex {
|
|
514
|
+
const digest = parseSha256Hex(value);
|
|
515
|
+
if (digest === null) throw new OhLibSqlSemanticV2Error("invalid-input", `Invalid ${label} digest.`);
|
|
516
|
+
return digest;
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
function parseInstant(value: unknown): string {
|
|
520
|
+
const instant = parseCanonicalInstantV1(value);
|
|
521
|
+
if (instant === null) throw new OhLibSqlSemanticV2Error("invalid-input", "Invalid semantic instant.");
|
|
522
|
+
return instant;
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
async function schemaObjects(client: OhLibSqlClientV1): Promise<readonly SchemaObject[]> {
|
|
526
|
+
const result = await client.execute(`SELECT type, name, tbl_name, sql FROM sqlite_schema
|
|
527
|
+
WHERE sql IS NOT NULL AND (name GLOB 'oh_semantic_*' OR tbl_name GLOB 'oh_semantic_*')
|
|
528
|
+
ORDER BY type, name`);
|
|
529
|
+
return result.rows.map((row) => {
|
|
530
|
+
const type = rowValue(row, "type", 0);
|
|
531
|
+
const name = rowValue(row, "name", 1);
|
|
532
|
+
const tableName = rowValue(row, "tbl_name", 2);
|
|
533
|
+
const sql = rowValue(row, "sql", 3);
|
|
534
|
+
if ((type !== "index" && type !== "table" && type !== "trigger")
|
|
535
|
+
|| typeof name !== "string" || typeof tableName !== "string" || typeof sql !== "string") {
|
|
536
|
+
throw new OhLibSqlSemanticV2Error("integrity", "The semantic schema inventory is malformed.");
|
|
537
|
+
}
|
|
538
|
+
const schemaType: SchemaObject["type"] = type;
|
|
539
|
+
return { name, sql: normalizedSchemaSql(sql), tableName, type: schemaType };
|
|
540
|
+
}).sort((left, right) => canonicalJson([left.type, left.name])
|
|
541
|
+
.localeCompare(canonicalJson([right.type, right.name])));
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
async function verifySchemaRevision(
|
|
545
|
+
client: OhLibSqlClientV1,
|
|
546
|
+
revision: Readonly<{
|
|
547
|
+
expected: readonly SchemaObject[];
|
|
548
|
+
name: string;
|
|
549
|
+
schemaSha256: Sha256Hex;
|
|
550
|
+
version: number;
|
|
551
|
+
}>,
|
|
552
|
+
): Promise<void> {
|
|
553
|
+
let marker: OhLibSqlResultV1;
|
|
554
|
+
try {
|
|
555
|
+
marker = await client.execute({
|
|
556
|
+
args: [revision.version],
|
|
557
|
+
sql: "SELECT name, schema_sha256 FROM oh_semantic_schemas WHERE version = ?",
|
|
558
|
+
});
|
|
559
|
+
} catch {
|
|
560
|
+
throw new OhLibSqlSemanticV2Error("schema-unavailable", "The semantic cache schema is unavailable.");
|
|
561
|
+
}
|
|
562
|
+
const row = marker.rows[0];
|
|
563
|
+
if (marker.rows.length !== 1 || row === undefined
|
|
564
|
+
|| rowValue(row, "name", 0) !== revision.name
|
|
565
|
+
|| rowValue(row, "schema_sha256", 1) !== revision.schemaSha256) {
|
|
566
|
+
throw new OhLibSqlSemanticV2Error("schema-unavailable", "The semantic cache schema marker is invalid.");
|
|
567
|
+
}
|
|
568
|
+
if (canonicalJson(await schemaObjects(client)) !== canonicalJson(revision.expected)) {
|
|
569
|
+
throw new OhLibSqlSemanticV2Error("integrity", "The semantic cache schema has drifted.");
|
|
570
|
+
}
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
async function verifySchema(client: OhLibSqlClientV1): Promise<void> {
|
|
574
|
+
const transition = await client.execute({
|
|
575
|
+
args: [TRANSITION_TABLE_NAME],
|
|
576
|
+
sql: "SELECT name FROM sqlite_schema WHERE type = 'table' AND name = ?",
|
|
577
|
+
});
|
|
578
|
+
if (transition.rows.length !== 0) {
|
|
579
|
+
throw new OhLibSqlSemanticV2Error(
|
|
580
|
+
"schema-unavailable",
|
|
581
|
+
"The semantic V2 cache upgrade is still materializing purge custody.",
|
|
582
|
+
);
|
|
583
|
+
}
|
|
584
|
+
await verifySchemaRevision(client, {
|
|
585
|
+
expected: EXPECTED_SCHEMA_OBJECTS,
|
|
586
|
+
name: SCHEMA_NAME,
|
|
587
|
+
schemaSha256: SCHEMA_SHA256,
|
|
588
|
+
version: SCHEMA_VERSION,
|
|
589
|
+
});
|
|
590
|
+
}
|
|
591
|
+
|
|
592
|
+
type LegacyPurge = Readonly<{ authorityId: string; purgedAt: string }>;
|
|
593
|
+
|
|
594
|
+
async function transitionPage(client: OhLibSqlClientV1): Promise<readonly LegacyPurge[]> {
|
|
595
|
+
const result = await client.execute({
|
|
596
|
+
args: [TRANSITION_PAGE_SIZE],
|
|
597
|
+
sql: `SELECT authority_id, purged_at FROM oh_semantic_v1_purge_transition
|
|
598
|
+
ORDER BY authority_id LIMIT ?`,
|
|
599
|
+
});
|
|
600
|
+
return Object.freeze(result.rows.map((row) => {
|
|
601
|
+
const authorityId = safeCode(rowValue(row, "authority_id", 0), 256);
|
|
602
|
+
const purgedAt = parseCanonicalInstantV1(rowValue(row, "purged_at", 1));
|
|
603
|
+
if (authorityId === null || purgedAt === null) {
|
|
604
|
+
throw new OhLibSqlSemanticV2Error("integrity", "A semantic v1 purge marker is invalid.");
|
|
605
|
+
}
|
|
606
|
+
return Object.freeze({ authorityId, purgedAt });
|
|
607
|
+
}));
|
|
608
|
+
}
|
|
609
|
+
|
|
610
|
+
async function verifyTransitionSchema(client: OhLibSqlClientV1): Promise<void> {
|
|
611
|
+
if (canonicalJson(await schemaObjects(client))
|
|
612
|
+
!== canonicalJson(EXPECTED_TRANSITION_SCHEMA_OBJECTS)) {
|
|
613
|
+
throw new OhLibSqlSemanticV2Error("integrity", "The semantic V2 transition schema has drifted.");
|
|
614
|
+
}
|
|
615
|
+
const markers = await client.execute(
|
|
616
|
+
"SELECT version FROM oh_semantic_schemas ORDER BY version LIMIT 1",
|
|
617
|
+
);
|
|
618
|
+
if (markers.rows.length !== 0) {
|
|
619
|
+
throw new OhLibSqlSemanticV2Error(
|
|
620
|
+
"integrity",
|
|
621
|
+
"The semantic V2 marker cannot exist during purge transition.",
|
|
622
|
+
);
|
|
623
|
+
}
|
|
624
|
+
}
|
|
625
|
+
|
|
626
|
+
async function startSemanticCacheV1ToV2Transition(
|
|
627
|
+
client: OhLibSqlClientV1,
|
|
628
|
+
): Promise<void> {
|
|
629
|
+
try {
|
|
630
|
+
await verifySchemaRevision(client, {
|
|
631
|
+
expected: EXPECTED_SCHEMA_OBJECTS_V1,
|
|
632
|
+
name: SCHEMA_NAME_V1,
|
|
633
|
+
schemaSha256: SCHEMA_SHA256_V1,
|
|
634
|
+
version: SCHEMA_VERSION_V1,
|
|
635
|
+
});
|
|
636
|
+
await client.batch([
|
|
637
|
+
{ sql: TRANSITION_TABLE },
|
|
638
|
+
{
|
|
639
|
+
sql: `INSERT INTO oh_semantic_v1_purge_transition(authority_id, purged_at)
|
|
640
|
+
SELECT authority_id, purged_at FROM oh_semantic_purges`,
|
|
641
|
+
},
|
|
642
|
+
{ sql: "DROP TABLE oh_semantic_heads" },
|
|
643
|
+
{ sql: "DROP TABLE oh_semantic_memberships" },
|
|
644
|
+
{ sql: "DROP TABLE oh_semantic_generations" },
|
|
645
|
+
{ sql: "DROP TABLE oh_semantic_vectors" },
|
|
646
|
+
{ sql: "DROP TABLE oh_semantic_purges" },
|
|
647
|
+
{ sql: "DROP TABLE oh_semantic_schemas" },
|
|
648
|
+
{ sql: SCHEMA_TABLE },
|
|
649
|
+
...SCHEMA_STATEMENTS.map((sql) => ({ sql })),
|
|
650
|
+
], "write");
|
|
651
|
+
} catch {
|
|
652
|
+
const inventory = await schemaObjects(client);
|
|
653
|
+
if (canonicalJson(inventory) === canonicalJson(EXPECTED_SCHEMA_OBJECTS)) {
|
|
654
|
+
await verifySchema(client);
|
|
655
|
+
return;
|
|
656
|
+
}
|
|
657
|
+
if (canonicalJson(inventory) !== canonicalJson(EXPECTED_TRANSITION_SCHEMA_OBJECTS)) {
|
|
658
|
+
throw new OhLibSqlSemanticV2Error(
|
|
659
|
+
"integrity",
|
|
660
|
+
"The semantic V1-to-V2 transition did not begin atomically.",
|
|
661
|
+
);
|
|
662
|
+
}
|
|
663
|
+
}
|
|
664
|
+
await verifyTransitionSchema(client);
|
|
665
|
+
}
|
|
666
|
+
|
|
667
|
+
async function materializeTransitionPage(
|
|
668
|
+
client: OhLibSqlClientV1,
|
|
669
|
+
purges: readonly LegacyPurge[],
|
|
670
|
+
): Promise<void> {
|
|
671
|
+
const statements: OhLibSqlStatementV1[] = [];
|
|
672
|
+
for (const purge of purges) {
|
|
673
|
+
const isolation = deriveOhSemanticIsolationSha256V2(purge.authorityId);
|
|
674
|
+
const marker = purgeMarkerSha256(purge.authorityId, isolation, purge.purgedAt);
|
|
675
|
+
statements.push({
|
|
676
|
+
args: [isolation, purge.authorityId, purge.purgedAt,
|
|
677
|
+
isolation, purge.authorityId, purge.purgedAt],
|
|
678
|
+
sql: `INSERT INTO oh_semantic_isolations(isolation_sha256, authority_id, created_at)
|
|
679
|
+
SELECT ?, ?, ? WHERE NOT EXISTS (SELECT 1 FROM oh_semantic_isolations
|
|
680
|
+
WHERE isolation_sha256 = ? AND authority_id = ? AND created_at = ?)
|
|
681
|
+
ON CONFLICT DO NOTHING`,
|
|
682
|
+
});
|
|
683
|
+
statements.push({
|
|
684
|
+
args: [purge.authorityId, isolation,
|
|
685
|
+
OH_CLOUDFLARE_EMBEDDING_PROFILE_V1.profileSha256, purge.purgedAt, marker],
|
|
686
|
+
sql: `INSERT INTO oh_semantic_purges(authority_id, isolation_sha256,
|
|
687
|
+
profile_sha256, published_generation, published_generation_sha256,
|
|
688
|
+
purged_at, purge_marker_sha256, generation_count, membership_count,
|
|
689
|
+
orphan_vector_count, isolation_scope_count, counts_recorded)
|
|
690
|
+
VALUES (?, ?, ?, NULL, NULL, ?, ?, 0, 0, 0, 1, 0)
|
|
691
|
+
ON CONFLICT DO NOTHING`,
|
|
692
|
+
});
|
|
693
|
+
statements.push({
|
|
694
|
+
args: [purge.authorityId, purge.purgedAt,
|
|
695
|
+
isolation, purge.authorityId, purge.purgedAt,
|
|
696
|
+
purge.authorityId, isolation, OH_CLOUDFLARE_EMBEDDING_PROFILE_V1.profileSha256,
|
|
697
|
+
purge.purgedAt, marker],
|
|
698
|
+
sql: `DELETE FROM oh_semantic_v1_purge_transition
|
|
699
|
+
WHERE authority_id = ? AND purged_at = ?
|
|
700
|
+
AND EXISTS (SELECT 1 FROM oh_semantic_isolations
|
|
701
|
+
WHERE isolation_sha256 = ? AND authority_id = ? AND created_at = ?)
|
|
702
|
+
AND EXISTS (SELECT 1 FROM oh_semantic_purges
|
|
703
|
+
WHERE authority_id = ? AND isolation_sha256 = ? AND profile_sha256 = ?
|
|
704
|
+
AND published_generation IS NULL AND published_generation_sha256 IS NULL
|
|
705
|
+
AND purged_at = ? AND purge_marker_sha256 = ?
|
|
706
|
+
AND generation_count = 0 AND membership_count = 0
|
|
707
|
+
AND orphan_vector_count = 0 AND isolation_scope_count = 1
|
|
708
|
+
AND counts_recorded = 0)`,
|
|
709
|
+
});
|
|
710
|
+
}
|
|
711
|
+
await client.batch(statements, "write");
|
|
712
|
+
const placeholders = purges.map(() => "?").join(", ");
|
|
713
|
+
const remaining = await client.execute({
|
|
714
|
+
args: purges.map(({ authorityId }) => authorityId),
|
|
715
|
+
sql: `SELECT authority_id FROM oh_semantic_v1_purge_transition
|
|
716
|
+
WHERE authority_id IN (${placeholders}) LIMIT 1`,
|
|
717
|
+
});
|
|
718
|
+
if (remaining.rows.length !== 0) {
|
|
719
|
+
throw new OhLibSqlSemanticV2Error(
|
|
720
|
+
"integrity",
|
|
721
|
+
"A semantic V1 purge tombstone did not materialize exactly in V2.",
|
|
722
|
+
);
|
|
723
|
+
}
|
|
724
|
+
}
|
|
725
|
+
|
|
726
|
+
async function finishSemanticCacheV2Transition(
|
|
727
|
+
client: OhLibSqlClientV1,
|
|
728
|
+
appliedAt: string,
|
|
729
|
+
): Promise<void> {
|
|
730
|
+
try {
|
|
731
|
+
await client.batch([
|
|
732
|
+
{
|
|
733
|
+
args: [SCHEMA_VERSION, SCHEMA_NAME, SCHEMA_SHA256, appliedAt],
|
|
734
|
+
sql: `INSERT INTO oh_semantic_schemas(version, name, schema_sha256, applied_at)
|
|
735
|
+
SELECT ?, CASE WHEN NOT EXISTS
|
|
736
|
+
(SELECT 1 FROM oh_semantic_v1_purge_transition) THEN ? END, ?, ?`,
|
|
737
|
+
},
|
|
738
|
+
{ sql: "DROP TABLE oh_semantic_v1_purge_transition" },
|
|
739
|
+
], "write");
|
|
740
|
+
} catch {
|
|
741
|
+
await verifySchema(client);
|
|
742
|
+
}
|
|
743
|
+
}
|
|
744
|
+
|
|
745
|
+
async function hasConvergedSemanticCacheV2(client: OhLibSqlClientV1): Promise<boolean> {
|
|
746
|
+
try {
|
|
747
|
+
await verifySchema(client);
|
|
748
|
+
return true;
|
|
749
|
+
} catch {
|
|
750
|
+
return false;
|
|
751
|
+
}
|
|
752
|
+
}
|
|
753
|
+
|
|
754
|
+
async function resumeSemanticCacheV2Transition(
|
|
755
|
+
client: OhLibSqlClientV1,
|
|
756
|
+
appliedAt: string,
|
|
757
|
+
): Promise<void> {
|
|
758
|
+
try {
|
|
759
|
+
await verifyTransitionSchema(client);
|
|
760
|
+
for (;;) {
|
|
761
|
+
const page = await transitionPage(client);
|
|
762
|
+
if (page.length === 0) break;
|
|
763
|
+
await materializeTransitionPage(client, page);
|
|
764
|
+
}
|
|
765
|
+
await finishSemanticCacheV2Transition(client, appliedAt);
|
|
766
|
+
} catch (error) {
|
|
767
|
+
if (await hasConvergedSemanticCacheV2(client)) return;
|
|
768
|
+
throw error;
|
|
769
|
+
}
|
|
770
|
+
}
|
|
771
|
+
|
|
772
|
+
export async function bootstrapOhLibSqlSemanticCacheV2(
|
|
773
|
+
client: OhLibSqlClientV1,
|
|
774
|
+
options: Readonly<{ appliedAt?: string }> = {},
|
|
775
|
+
): Promise<Readonly<{ schemaSha256: Sha256Hex; schemaVersion: 2; v: 2 }>> {
|
|
776
|
+
const appliedAt = parseInstant(options.appliedAt ?? canonicalNow());
|
|
777
|
+
let existing = await schemaObjects(client);
|
|
778
|
+
if (existing.length === 0) {
|
|
779
|
+
await client.batch([
|
|
780
|
+
{ sql: SCHEMA_TABLE },
|
|
781
|
+
...SCHEMA_STATEMENTS.map((sql) => ({ sql })),
|
|
782
|
+
{
|
|
783
|
+
args: [SCHEMA_VERSION, SCHEMA_NAME, SCHEMA_SHA256, appliedAt],
|
|
784
|
+
sql: `INSERT INTO oh_semantic_schemas(version, name, schema_sha256, applied_at)
|
|
785
|
+
VALUES (?, ?, ?, ?) ON CONFLICT(version) DO NOTHING`,
|
|
786
|
+
},
|
|
787
|
+
], "write");
|
|
788
|
+
} else if (canonicalJson(existing) === canonicalJson(EXPECTED_SCHEMA_OBJECTS_V1)) {
|
|
789
|
+
await startSemanticCacheV1ToV2Transition(client);
|
|
790
|
+
}
|
|
791
|
+
existing = await schemaObjects(client);
|
|
792
|
+
if (canonicalJson(existing) === canonicalJson(EXPECTED_TRANSITION_SCHEMA_OBJECTS)) {
|
|
793
|
+
await resumeSemanticCacheV2Transition(client, appliedAt);
|
|
794
|
+
} else if (canonicalJson(existing) !== canonicalJson(EXPECTED_SCHEMA_OBJECTS)) {
|
|
795
|
+
throw new OhLibSqlSemanticV2Error("integrity", "Refusing to bless a partial or drifted semantic V2 schema.");
|
|
796
|
+
}
|
|
797
|
+
await verifySchema(client);
|
|
798
|
+
return Object.freeze({ schemaSha256: SCHEMA_SHA256, schemaVersion: 2, v: 2 });
|
|
799
|
+
}
|
|
800
|
+
|
|
801
|
+
function vectorBytes(vector: readonly number[]): Uint8Array {
|
|
802
|
+
const normalized = normalizeOhEmbeddingV1(vector);
|
|
803
|
+
const bytes = new Uint8Array(VECTOR_BYTES);
|
|
804
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
805
|
+
for (const [index, component] of normalized.entries()) view.setFloat32(index * 4, component, true);
|
|
806
|
+
return bytes;
|
|
807
|
+
}
|
|
808
|
+
|
|
809
|
+
function storedBytes(value: unknown): Uint8Array | null {
|
|
810
|
+
if (value instanceof Uint8Array) return new Uint8Array(value);
|
|
811
|
+
if (value instanceof ArrayBuffer) return new Uint8Array(value.slice(0));
|
|
812
|
+
return null;
|
|
813
|
+
}
|
|
814
|
+
|
|
815
|
+
function decodeVector(value: unknown, expectedSha256: unknown): readonly number[] {
|
|
816
|
+
const bytes = storedBytes(value);
|
|
817
|
+
const digest = parseSha256Hex(expectedSha256);
|
|
818
|
+
if (bytes === null || bytes.byteLength !== VECTOR_BYTES || digest === null
|
|
819
|
+
|| sha256Hex(bytes) !== digest) {
|
|
820
|
+
throw new OhLibSqlSemanticV2Error("integrity", "A cached semantic vector is corrupt.");
|
|
821
|
+
}
|
|
822
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
823
|
+
const vector = Array.from({ length: OH_CLOUDFLARE_EMBEDDING_PROFILE_V1.dimensions },
|
|
824
|
+
(_, index) => view.getFloat32(index * 4, true));
|
|
825
|
+
try { return Object.freeze([...normalizeOhEmbeddingV1(vector)]); }
|
|
826
|
+
catch { throw new OhLibSqlSemanticV2Error("integrity", "A cached semantic vector is invalid."); }
|
|
827
|
+
}
|
|
828
|
+
|
|
829
|
+
type Membership = Readonly<{
|
|
830
|
+
input: OhRenderedEmbeddingInputV1;
|
|
831
|
+
inputSha256: Sha256Hex;
|
|
832
|
+
ordinal: number;
|
|
833
|
+
recordKey: string;
|
|
834
|
+
recordSha256: Sha256Hex;
|
|
835
|
+
}>;
|
|
836
|
+
|
|
837
|
+
type Generation = Readonly<{
|
|
838
|
+
authorityId: string;
|
|
839
|
+
authoritySha256: Sha256Hex;
|
|
840
|
+
chunkCount: number;
|
|
841
|
+
createdAt: string;
|
|
842
|
+
documentCount: number;
|
|
843
|
+
generation: number;
|
|
844
|
+
generationSha256: Sha256Hex;
|
|
845
|
+
isolationSha256: Sha256Hex;
|
|
846
|
+
membershipSha256: Sha256Hex;
|
|
847
|
+
memberships: readonly Membership[];
|
|
848
|
+
}>;
|
|
849
|
+
|
|
850
|
+
function prepareGeneration(input: Readonly<{
|
|
851
|
+
authorityId: string;
|
|
852
|
+
authoritySha256: Sha256Hex;
|
|
853
|
+
createdAt?: string;
|
|
854
|
+
documents: readonly OhSemanticDocumentV2[];
|
|
855
|
+
generation: number;
|
|
856
|
+
isolationSha256?: Sha256Hex;
|
|
857
|
+
maximumChunksPerDocument?: number;
|
|
858
|
+
}>): Generation {
|
|
859
|
+
const authorityId = parseAuthorityId(input.authorityId);
|
|
860
|
+
const authoritySha256 = parseDigest(input.authoritySha256, "authority");
|
|
861
|
+
const isolation = isolationSha256(authorityId, input.isolationSha256);
|
|
862
|
+
const generation = parseGeneration(input.generation);
|
|
863
|
+
const createdAt = parseInstant(input.createdAt ?? canonicalNow());
|
|
864
|
+
const maximumChunks = input.maximumChunksPerDocument
|
|
865
|
+
?? OH_LIBSQL_SEMANTIC_LIMITS_V2.chunksPerDocument;
|
|
866
|
+
if (!Number.isSafeInteger(maximumChunks) || maximumChunks < 1
|
|
867
|
+
|| maximumChunks > OH_LIBSQL_SEMANTIC_LIMITS_V2.chunksPerDocument
|
|
868
|
+
|| !Array.isArray(input.documents) || input.documents.length < 1
|
|
869
|
+
|| input.documents.length > OH_LIBSQL_SEMANTIC_LIMITS_V2.documentsPerGeneration) {
|
|
870
|
+
throw new OhLibSqlSemanticV2Error("invalid-input", "Invalid semantic generation bounds.");
|
|
871
|
+
}
|
|
872
|
+
const documents = input.documents.map((document) => {
|
|
873
|
+
if (document.v !== 2) throw new OhLibSqlSemanticV2Error("invalid-input", "Invalid semantic document version.");
|
|
874
|
+
return {
|
|
875
|
+
...document,
|
|
876
|
+
key: parseRecordKey(document.key),
|
|
877
|
+
recordSha256: parseDigest(document.recordSha256, "record"),
|
|
878
|
+
};
|
|
879
|
+
}).sort((left, right) => left.key < right.key ? -1 : left.key > right.key ? 1 : 0);
|
|
880
|
+
if (new Set(documents.map(({ key }) => key)).size !== documents.length) {
|
|
881
|
+
throw new OhLibSqlSemanticV2Error("invalid-input", "Semantic document keys must be unique.");
|
|
882
|
+
}
|
|
883
|
+
const memberships: Membership[] = [];
|
|
884
|
+
for (const document of documents) {
|
|
885
|
+
const rendered = renderOhCloudflareEmbeddingDocumentV1({
|
|
886
|
+
content: document.content,
|
|
887
|
+
maximumChunks,
|
|
888
|
+
title: document.title,
|
|
889
|
+
});
|
|
890
|
+
if (rendered.status !== "complete") {
|
|
891
|
+
throw new OhLibSqlSemanticV2Error("invalid-input", "A semantic document exceeds the complete renderer bound.");
|
|
892
|
+
}
|
|
893
|
+
for (const chunk of rendered.chunks) {
|
|
894
|
+
memberships.push(Object.freeze({
|
|
895
|
+
input: chunk.input,
|
|
896
|
+
inputSha256: chunk.input.inputSha256,
|
|
897
|
+
ordinal: chunk.ordinal,
|
|
898
|
+
recordKey: document.key,
|
|
899
|
+
recordSha256: document.recordSha256,
|
|
900
|
+
}));
|
|
901
|
+
}
|
|
902
|
+
}
|
|
903
|
+
if (memberships.length < 1
|
|
904
|
+
|| memberships.length > OH_LIBSQL_SEMANTIC_LIMITS_V2.chunksPerGeneration) {
|
|
905
|
+
throw new OhLibSqlSemanticV2Error("invalid-input", "The semantic generation exceeds its chunk bound.");
|
|
906
|
+
}
|
|
907
|
+
const membershipSha256 = canonicalSha256({
|
|
908
|
+
kind: MEMBERSHIP_KIND,
|
|
909
|
+
memberships: memberships.map((membership) => ({
|
|
910
|
+
inputSha256: membership.inputSha256,
|
|
911
|
+
isolationSha256: isolation,
|
|
912
|
+
ordinal: membership.ordinal,
|
|
913
|
+
recordKey: membership.recordKey,
|
|
914
|
+
recordSha256: membership.recordSha256,
|
|
915
|
+
})),
|
|
916
|
+
v: 2,
|
|
917
|
+
});
|
|
918
|
+
const generationSha256 = canonicalSha256({
|
|
919
|
+
authorityId,
|
|
920
|
+
authoritySha256,
|
|
921
|
+
chunkCount: memberships.length,
|
|
922
|
+
documentCount: documents.length,
|
|
923
|
+
generation,
|
|
924
|
+
isolationSha256: isolation,
|
|
925
|
+
kind: GENERATION_KIND,
|
|
926
|
+
membershipSha256,
|
|
927
|
+
profileSha256: OH_CLOUDFLARE_EMBEDDING_PROFILE_V1.profileSha256,
|
|
928
|
+
rendererSha256: OH_SEMANTIC_RENDERER_V1.rendererSha256,
|
|
929
|
+
v: 2,
|
|
930
|
+
});
|
|
931
|
+
return Object.freeze({
|
|
932
|
+
authorityId,
|
|
933
|
+
authoritySha256,
|
|
934
|
+
chunkCount: memberships.length,
|
|
935
|
+
createdAt,
|
|
936
|
+
documentCount: documents.length,
|
|
937
|
+
generation,
|
|
938
|
+
generationSha256,
|
|
939
|
+
isolationSha256: isolation,
|
|
940
|
+
membershipSha256,
|
|
941
|
+
memberships: Object.freeze(memberships),
|
|
942
|
+
});
|
|
943
|
+
}
|
|
944
|
+
|
|
945
|
+
type StoredGeneration = Readonly<{
|
|
946
|
+
authorityId: string;
|
|
947
|
+
authoritySha256: Sha256Hex;
|
|
948
|
+
chunkCount: number;
|
|
949
|
+
createdAt: string;
|
|
950
|
+
documentCount: number;
|
|
951
|
+
generation: number;
|
|
952
|
+
generationSha256: Sha256Hex;
|
|
953
|
+
isolationSha256: Sha256Hex;
|
|
954
|
+
membershipSha256: Sha256Hex;
|
|
955
|
+
profileSha256: Sha256Hex;
|
|
956
|
+
rendererSha256: Sha256Hex;
|
|
957
|
+
}>;
|
|
958
|
+
|
|
959
|
+
type StoredHead = Readonly<{
|
|
960
|
+
authorityId: string;
|
|
961
|
+
authoritySha256: Sha256Hex;
|
|
962
|
+
generation: number;
|
|
963
|
+
generationSha256: Sha256Hex;
|
|
964
|
+
isolationSha256: Sha256Hex;
|
|
965
|
+
membershipSha256: Sha256Hex;
|
|
966
|
+
profileSha256: Sha256Hex;
|
|
967
|
+
publishedAt: string;
|
|
968
|
+
rendererSha256: Sha256Hex;
|
|
969
|
+
}>;
|
|
970
|
+
|
|
971
|
+
function parseStoredGeneration(
|
|
972
|
+
row: Readonly<Record<string, unknown>> | readonly unknown[],
|
|
973
|
+
): StoredGeneration {
|
|
974
|
+
const authorityId = safeCode(rowValue(row, "authority_id", 0), 256);
|
|
975
|
+
const generation = integer(rowValue(row, "generation", 1));
|
|
976
|
+
const authoritySha256 = parseSha256Hex(rowValue(row, "authority_sha256", 2));
|
|
977
|
+
const isolation = parseSha256Hex(rowValue(row, "isolation_sha256", 3));
|
|
978
|
+
const profileSha256 = parseSha256Hex(rowValue(row, "profile_sha256", 4));
|
|
979
|
+
const rendererSha256 = parseSha256Hex(rowValue(row, "renderer_sha256", 5));
|
|
980
|
+
const membershipSha256 = parseSha256Hex(rowValue(row, "membership_sha256", 6));
|
|
981
|
+
const generationSha256 = parseSha256Hex(rowValue(row, "generation_sha256", 7));
|
|
982
|
+
const documentCount = integer(rowValue(row, "document_count", 8));
|
|
983
|
+
const chunkCount = integer(rowValue(row, "chunk_count", 9));
|
|
984
|
+
const createdAtValue = rowValue(row, "created_at", 10);
|
|
985
|
+
const createdAt = parseCanonicalInstantV1(createdAtValue);
|
|
986
|
+
if (authorityId === null || generation === null || generation < 0
|
|
987
|
+
|| authoritySha256 === null || isolation === null
|
|
988
|
+
|| profileSha256 === null || rendererSha256 === null
|
|
989
|
+
|| membershipSha256 === null || generationSha256 === null
|
|
990
|
+
|| documentCount === null || documentCount < 1
|
|
991
|
+
|| documentCount > OH_LIBSQL_SEMANTIC_LIMITS_V2.documentsPerGeneration
|
|
992
|
+
|| chunkCount === null || chunkCount < 1
|
|
993
|
+
|| chunkCount > OH_LIBSQL_SEMANTIC_LIMITS_V2.chunksPerGeneration
|
|
994
|
+
|| createdAt === null) {
|
|
995
|
+
throw new OhLibSqlSemanticV2Error("integrity", "A stored semantic generation is invalid.");
|
|
996
|
+
}
|
|
997
|
+
return Object.freeze({
|
|
998
|
+
authorityId,
|
|
999
|
+
authoritySha256,
|
|
1000
|
+
chunkCount,
|
|
1001
|
+
createdAt,
|
|
1002
|
+
documentCount,
|
|
1003
|
+
generation,
|
|
1004
|
+
generationSha256,
|
|
1005
|
+
isolationSha256: isolation,
|
|
1006
|
+
membershipSha256,
|
|
1007
|
+
profileSha256,
|
|
1008
|
+
rendererSha256,
|
|
1009
|
+
});
|
|
1010
|
+
}
|
|
1011
|
+
|
|
1012
|
+
function generationMatches(left: StoredGeneration, right: Generation): boolean {
|
|
1013
|
+
return left.authorityId === right.authorityId
|
|
1014
|
+
&& left.authoritySha256 === right.authoritySha256
|
|
1015
|
+
&& left.chunkCount === right.chunkCount
|
|
1016
|
+
&& left.documentCount === right.documentCount
|
|
1017
|
+
&& left.generation === right.generation
|
|
1018
|
+
&& left.generationSha256 === right.generationSha256
|
|
1019
|
+
&& left.isolationSha256 === right.isolationSha256
|
|
1020
|
+
&& left.membershipSha256 === right.membershipSha256
|
|
1021
|
+
&& left.profileSha256 === OH_CLOUDFLARE_EMBEDDING_PROFILE_V1.profileSha256
|
|
1022
|
+
&& left.rendererSha256 === OH_SEMANTIC_RENDERER_V1.rendererSha256;
|
|
1023
|
+
}
|
|
1024
|
+
|
|
1025
|
+
function parseStoredHead(row: Readonly<Record<string, unknown>> | readonly unknown[]): StoredHead {
|
|
1026
|
+
const authorityId = safeCode(rowValue(row, "authority_id", 0), 256);
|
|
1027
|
+
const generation = integer(rowValue(row, "generation", 1));
|
|
1028
|
+
const authoritySha256 = parseSha256Hex(rowValue(row, "authority_sha256", 2));
|
|
1029
|
+
const isolation = parseSha256Hex(rowValue(row, "isolation_sha256", 3));
|
|
1030
|
+
const profileSha256 = parseSha256Hex(rowValue(row, "profile_sha256", 4));
|
|
1031
|
+
const rendererSha256 = parseSha256Hex(rowValue(row, "renderer_sha256", 5));
|
|
1032
|
+
const membershipSha256 = parseSha256Hex(rowValue(row, "membership_sha256", 6));
|
|
1033
|
+
const generationSha256 = parseSha256Hex(rowValue(row, "generation_sha256", 7));
|
|
1034
|
+
const publishedAtValue = rowValue(row, "published_at", 8);
|
|
1035
|
+
const publishedAt = parseCanonicalInstantV1(publishedAtValue);
|
|
1036
|
+
if (authorityId === null || generation === null || generation < 0
|
|
1037
|
+
|| authoritySha256 === null || isolation === null
|
|
1038
|
+
|| profileSha256 === null || rendererSha256 === null
|
|
1039
|
+
|| membershipSha256 === null || generationSha256 === null || publishedAt === null) {
|
|
1040
|
+
throw new OhLibSqlSemanticV2Error("integrity", "A stored semantic head is invalid.");
|
|
1041
|
+
}
|
|
1042
|
+
return Object.freeze({
|
|
1043
|
+
authorityId,
|
|
1044
|
+
authoritySha256,
|
|
1045
|
+
generation,
|
|
1046
|
+
generationSha256,
|
|
1047
|
+
isolationSha256: isolation,
|
|
1048
|
+
membershipSha256,
|
|
1049
|
+
profileSha256,
|
|
1050
|
+
publishedAt,
|
|
1051
|
+
rendererSha256,
|
|
1052
|
+
});
|
|
1053
|
+
}
|
|
1054
|
+
|
|
1055
|
+
function headMatchesGeneration(head: StoredHead, generation: StoredGeneration): boolean {
|
|
1056
|
+
return head.authorityId === generation.authorityId
|
|
1057
|
+
&& head.authoritySha256 === generation.authoritySha256
|
|
1058
|
+
&& head.generation === generation.generation
|
|
1059
|
+
&& head.generationSha256 === generation.generationSha256
|
|
1060
|
+
&& head.isolationSha256 === generation.isolationSha256
|
|
1061
|
+
&& head.membershipSha256 === generation.membershipSha256
|
|
1062
|
+
&& head.profileSha256 === generation.profileSha256
|
|
1063
|
+
&& head.rendererSha256 === generation.rendererSha256;
|
|
1064
|
+
}
|
|
1065
|
+
|
|
1066
|
+
const GENERATION_SELECT = `SELECT authority_id, generation, authority_sha256,
|
|
1067
|
+
isolation_sha256, profile_sha256, renderer_sha256, membership_sha256, generation_sha256,
|
|
1068
|
+
document_count, chunk_count, created_at
|
|
1069
|
+
FROM oh_semantic_generations WHERE authority_id = ? AND generation = ?`;
|
|
1070
|
+
const HEAD_SELECT = `SELECT authority_id, generation, authority_sha256,
|
|
1071
|
+
isolation_sha256, profile_sha256, renderer_sha256, membership_sha256,
|
|
1072
|
+
generation_sha256, published_at
|
|
1073
|
+
FROM oh_semantic_heads WHERE authority_id = ?`;
|
|
1074
|
+
|
|
1075
|
+
async function readGeneration(
|
|
1076
|
+
client: OhLibSqlClientV1,
|
|
1077
|
+
authorityId: string,
|
|
1078
|
+
generation: number,
|
|
1079
|
+
): Promise<StoredGeneration | null> {
|
|
1080
|
+
const result = await client.execute({ args: [authorityId, generation], sql: GENERATION_SELECT });
|
|
1081
|
+
if (result.rows.length > 1) throw new OhLibSqlSemanticV2Error("integrity", "Duplicate semantic generations.");
|
|
1082
|
+
const row = result.rows[0];
|
|
1083
|
+
return row === undefined ? null : parseStoredGeneration(row);
|
|
1084
|
+
}
|
|
1085
|
+
|
|
1086
|
+
async function readHead(client: OhLibSqlClientV1, authorityId: string): Promise<StoredHead | null> {
|
|
1087
|
+
const result = await client.execute({ args: [authorityId], sql: HEAD_SELECT });
|
|
1088
|
+
if (result.rows.length > 1) throw new OhLibSqlSemanticV2Error("integrity", "Duplicate semantic heads.");
|
|
1089
|
+
const row = result.rows[0];
|
|
1090
|
+
return row === undefined ? null : parseStoredHead(row);
|
|
1091
|
+
}
|
|
1092
|
+
|
|
1093
|
+
type StoredPurge = Readonly<Omit<OhSemanticPurgeResultV2, "purgeReceiptSha256">>;
|
|
1094
|
+
|
|
1095
|
+
function purgeResult(stored: StoredPurge): OhSemanticPurgeResultV2 {
|
|
1096
|
+
return Object.freeze({
|
|
1097
|
+
...stored,
|
|
1098
|
+
purgeReceiptSha256: canonicalSha256({
|
|
1099
|
+
kind: PURGE_RECEIPT_KIND,
|
|
1100
|
+
receipt: stored,
|
|
1101
|
+
v: 2,
|
|
1102
|
+
}),
|
|
1103
|
+
});
|
|
1104
|
+
}
|
|
1105
|
+
|
|
1106
|
+
async function readPurge(
|
|
1107
|
+
client: OhLibSqlClientV1,
|
|
1108
|
+
authorityId: string,
|
|
1109
|
+
): Promise<OhSemanticPurgeResultV2 | null> {
|
|
1110
|
+
const result = await client.execute({
|
|
1111
|
+
args: [authorityId],
|
|
1112
|
+
sql: `SELECT isolation_sha256, profile_sha256, published_generation,
|
|
1113
|
+
published_generation_sha256, purged_at, purge_marker_sha256,
|
|
1114
|
+
generation_count, membership_count, orphan_vector_count,
|
|
1115
|
+
isolation_scope_count, counts_recorded
|
|
1116
|
+
FROM oh_semantic_purges WHERE authority_id = ?`,
|
|
1117
|
+
});
|
|
1118
|
+
if (result.rows.length > 1) throw new OhLibSqlSemanticV2Error("integrity", "Duplicate semantic purge markers.");
|
|
1119
|
+
const row = result.rows[0];
|
|
1120
|
+
if (row === undefined) return null;
|
|
1121
|
+
const isolation = parseSha256Hex(rowValue(row, "isolation_sha256", 0));
|
|
1122
|
+
const profileSha256 = parseSha256Hex(rowValue(row, "profile_sha256", 1));
|
|
1123
|
+
const publishedGenerationValue = rowValue(row, "published_generation", 2);
|
|
1124
|
+
const publishedGeneration = publishedGenerationValue === null
|
|
1125
|
+
? null : integer(publishedGenerationValue);
|
|
1126
|
+
const publishedGenerationSha256Value = rowValue(row, "published_generation_sha256", 3);
|
|
1127
|
+
const publishedGenerationSha256 = publishedGenerationSha256Value === null
|
|
1128
|
+
? null : parseSha256Hex(publishedGenerationSha256Value);
|
|
1129
|
+
const purgedAt = parseCanonicalInstantV1(rowValue(row, "purged_at", 4));
|
|
1130
|
+
const storedMarker = parseSha256Hex(rowValue(row, "purge_marker_sha256", 5));
|
|
1131
|
+
const generations = integer(rowValue(row, "generation_count", 6));
|
|
1132
|
+
const memberships = integer(rowValue(row, "membership_count", 7));
|
|
1133
|
+
const orphanVectors = integer(rowValue(row, "orphan_vector_count", 8));
|
|
1134
|
+
const isolationScopes = integer(rowValue(row, "isolation_scope_count", 9));
|
|
1135
|
+
const countsRecordedValue = integer(rowValue(row, "counts_recorded", 10));
|
|
1136
|
+
if (isolation === null
|
|
1137
|
+
|| profileSha256 !== OH_CLOUDFLARE_EMBEDDING_PROFILE_V1.profileSha256
|
|
1138
|
+
|| (publishedGeneration !== null && publishedGeneration < 0)
|
|
1139
|
+
|| (publishedGeneration === null) !== (publishedGenerationSha256 === null)
|
|
1140
|
+
|| purgedAt === null || storedMarker === null
|
|
1141
|
+
|| storedMarker !== purgeMarkerSha256(authorityId, isolation, purgedAt)
|
|
1142
|
+
|| generations === null || generations < 0
|
|
1143
|
+
|| memberships === null || memberships < 0
|
|
1144
|
+
|| orphanVectors === null || orphanVectors < 0
|
|
1145
|
+
|| isolationScopes === null || isolationScopes < 1
|
|
1146
|
+
|| (countsRecordedValue !== 0 && countsRecordedValue !== 1)) {
|
|
1147
|
+
throw new OhLibSqlSemanticV2Error("integrity", "The semantic purge marker is invalid.");
|
|
1148
|
+
}
|
|
1149
|
+
return purgeResult(Object.freeze({
|
|
1150
|
+
authorityId,
|
|
1151
|
+
countsRecorded: countsRecordedValue === 1,
|
|
1152
|
+
generations,
|
|
1153
|
+
isolationScopes,
|
|
1154
|
+
isolationSha256: isolation,
|
|
1155
|
+
memberships,
|
|
1156
|
+
orphanVectors,
|
|
1157
|
+
profileSha256,
|
|
1158
|
+
publishedGeneration,
|
|
1159
|
+
publishedGenerationSha256,
|
|
1160
|
+
purgeMarkerSha256: storedMarker,
|
|
1161
|
+
purgedAt,
|
|
1162
|
+
residualGenerations: 0,
|
|
1163
|
+
residualMemberships: 0,
|
|
1164
|
+
residualScopedVectors: 0,
|
|
1165
|
+
v: 2,
|
|
1166
|
+
}));
|
|
1167
|
+
}
|
|
1168
|
+
|
|
1169
|
+
async function readIsolationOwner(
|
|
1170
|
+
client: OhLibSqlClientV1,
|
|
1171
|
+
isolation: Sha256Hex,
|
|
1172
|
+
): Promise<string | null> {
|
|
1173
|
+
const result = await client.execute({
|
|
1174
|
+
args: [isolation],
|
|
1175
|
+
sql: "SELECT authority_id FROM oh_semantic_isolations WHERE isolation_sha256 = ?",
|
|
1176
|
+
});
|
|
1177
|
+
if (result.rows.length > 1) throw new OhLibSqlSemanticV2Error("integrity", "Duplicate semantic isolations.");
|
|
1178
|
+
const row = result.rows[0];
|
|
1179
|
+
if (row === undefined) return null;
|
|
1180
|
+
const authorityId = safeCode(rowValue(row, "authority_id", 0), 256);
|
|
1181
|
+
if (authorityId === null) throw new OhLibSqlSemanticV2Error("integrity", "A semantic isolation is invalid.");
|
|
1182
|
+
return authorityId;
|
|
1183
|
+
}
|
|
1184
|
+
|
|
1185
|
+
async function reserveIsolation(
|
|
1186
|
+
client: OhLibSqlClientV1,
|
|
1187
|
+
authorityId: string,
|
|
1188
|
+
isolation: Sha256Hex,
|
|
1189
|
+
createdAt: string,
|
|
1190
|
+
): Promise<void> {
|
|
1191
|
+
if (await readPurge(client, authorityId) !== null) {
|
|
1192
|
+
throw new OhLibSqlSemanticV2Error("purged", "The semantic authority was purged.");
|
|
1193
|
+
}
|
|
1194
|
+
await client.execute({
|
|
1195
|
+
args: [isolation, authorityId, createdAt, authorityId],
|
|
1196
|
+
sql: `INSERT INTO oh_semantic_isolations(isolation_sha256, authority_id, created_at)
|
|
1197
|
+
SELECT ?, ?, ?
|
|
1198
|
+
WHERE NOT EXISTS (SELECT 1 FROM oh_semantic_purges WHERE authority_id = ?)
|
|
1199
|
+
ON CONFLICT DO NOTHING`,
|
|
1200
|
+
});
|
|
1201
|
+
const owner = await readIsolationOwner(client, isolation);
|
|
1202
|
+
if (await readPurge(client, authorityId) !== null) {
|
|
1203
|
+
throw new OhLibSqlSemanticV2Error("purged", "The semantic authority was purged.");
|
|
1204
|
+
}
|
|
1205
|
+
if (owner !== authorityId) {
|
|
1206
|
+
throw new OhLibSqlSemanticV2Error("conflict", "The semantic isolation belongs to another authority.");
|
|
1207
|
+
}
|
|
1208
|
+
}
|
|
1209
|
+
|
|
1210
|
+
async function reservePurgeIsolation(
|
|
1211
|
+
client: OhLibSqlClientV1,
|
|
1212
|
+
authorityId: string,
|
|
1213
|
+
isolation: Sha256Hex,
|
|
1214
|
+
createdAt: string,
|
|
1215
|
+
): Promise<void> {
|
|
1216
|
+
const head = await readHead(client, authorityId);
|
|
1217
|
+
if (head !== null && head.isolationSha256 !== isolation) {
|
|
1218
|
+
throw new OhLibSqlSemanticV2Error("conflict", "The semantic purge isolation conflicts.");
|
|
1219
|
+
}
|
|
1220
|
+
const owner = await readIsolationOwner(client, isolation);
|
|
1221
|
+
if (owner !== null && owner !== authorityId) {
|
|
1222
|
+
throw new OhLibSqlSemanticV2Error("conflict", "The semantic isolation belongs to another authority.");
|
|
1223
|
+
}
|
|
1224
|
+
if (owner === null) {
|
|
1225
|
+
const existing = await client.execute({
|
|
1226
|
+
args: [authorityId],
|
|
1227
|
+
sql: "SELECT isolation_sha256 FROM oh_semantic_isolations WHERE authority_id = ? LIMIT 1",
|
|
1228
|
+
});
|
|
1229
|
+
if (existing.rows.length !== 0) {
|
|
1230
|
+
throw new OhLibSqlSemanticV2Error("conflict", "The semantic purge isolation conflicts.");
|
|
1231
|
+
}
|
|
1232
|
+
await reserveIsolation(client, authorityId, isolation, createdAt);
|
|
1233
|
+
}
|
|
1234
|
+
}
|
|
1235
|
+
|
|
1236
|
+
async function verifyPurgeResidual(
|
|
1237
|
+
client: OhLibSqlClientV1,
|
|
1238
|
+
authorityId: string,
|
|
1239
|
+
): Promise<void> {
|
|
1240
|
+
const result = await client.execute({
|
|
1241
|
+
args: [authorityId, authorityId, authorityId, authorityId],
|
|
1242
|
+
sql: `SELECT
|
|
1243
|
+
(SELECT count(*) FROM oh_semantic_heads WHERE authority_id = ?) AS heads,
|
|
1244
|
+
(SELECT count(*) FROM oh_semantic_generations WHERE authority_id = ?) AS generations,
|
|
1245
|
+
(SELECT count(*) FROM oh_semantic_memberships WHERE authority_id = ?) AS memberships,
|
|
1246
|
+
(SELECT count(*) FROM oh_semantic_vectors AS vector
|
|
1247
|
+
JOIN oh_semantic_isolations AS isolation
|
|
1248
|
+
ON isolation.isolation_sha256 = vector.isolation_sha256
|
|
1249
|
+
WHERE isolation.authority_id = ?) AS vectors`,
|
|
1250
|
+
});
|
|
1251
|
+
const row = result.rows[0];
|
|
1252
|
+
if (result.rows.length !== 1 || row === undefined
|
|
1253
|
+
|| integer(rowValue(row, "heads", 0)) !== 0
|
|
1254
|
+
|| integer(rowValue(row, "generations", 1)) !== 0
|
|
1255
|
+
|| integer(rowValue(row, "memberships", 2)) !== 0
|
|
1256
|
+
|| integer(rowValue(row, "vectors", 3)) !== 0) {
|
|
1257
|
+
throw new OhLibSqlSemanticV2Error("integrity", "The semantic authority purge is incomplete.");
|
|
1258
|
+
}
|
|
1259
|
+
}
|
|
1260
|
+
|
|
1261
|
+
async function readMemberships(
|
|
1262
|
+
client: OhLibSqlClientV1,
|
|
1263
|
+
generation: StoredGeneration,
|
|
1264
|
+
): Promise<readonly Omit<Membership, "input">[]> {
|
|
1265
|
+
const memberships: Array<Omit<Membership, "input">> = [];
|
|
1266
|
+
for (let offset = 0; offset < generation.chunkCount; offset += OH_LIBSQL_SEMANTIC_LIMITS_V2.searchPage) {
|
|
1267
|
+
const result = await client.execute({
|
|
1268
|
+
args: [generation.authorityId, generation.generation,
|
|
1269
|
+
OH_LIBSQL_SEMANTIC_LIMITS_V2.searchPage, offset],
|
|
1270
|
+
sql: `SELECT generation_sha256, isolation_sha256, record_key,
|
|
1271
|
+
record_sha256, ordinal, input_sha256
|
|
1272
|
+
FROM oh_semantic_memberships
|
|
1273
|
+
WHERE authority_id = ? AND generation = ?
|
|
1274
|
+
ORDER BY record_key, ordinal LIMIT ? OFFSET ?`,
|
|
1275
|
+
});
|
|
1276
|
+
for (const row of result.rows) {
|
|
1277
|
+
const generationSha256 = parseSha256Hex(rowValue(row, "generation_sha256", 0));
|
|
1278
|
+
const isolation = parseSha256Hex(rowValue(row, "isolation_sha256", 1));
|
|
1279
|
+
const recordKey = safeCode(rowValue(row, "record_key", 2), 512);
|
|
1280
|
+
const recordSha256 = parseSha256Hex(rowValue(row, "record_sha256", 3));
|
|
1281
|
+
const ordinal = integer(rowValue(row, "ordinal", 4));
|
|
1282
|
+
const inputSha256 = parseSha256Hex(rowValue(row, "input_sha256", 5));
|
|
1283
|
+
if (generationSha256 !== generation.generationSha256
|
|
1284
|
+
|| isolation !== generation.isolationSha256 || recordKey === null
|
|
1285
|
+
|| recordSha256 === null || ordinal === null || ordinal < 0
|
|
1286
|
+
|| ordinal >= OH_LIBSQL_SEMANTIC_LIMITS_V2.chunksPerDocument
|
|
1287
|
+
|| inputSha256 === null) {
|
|
1288
|
+
throw new OhLibSqlSemanticV2Error("integrity", "A semantic generation membership is invalid.");
|
|
1289
|
+
}
|
|
1290
|
+
memberships.push(Object.freeze({ inputSha256, ordinal, recordKey, recordSha256 }));
|
|
1291
|
+
}
|
|
1292
|
+
}
|
|
1293
|
+
if (memberships.length !== generation.chunkCount
|
|
1294
|
+
|| canonicalSha256({
|
|
1295
|
+
kind: MEMBERSHIP_KIND,
|
|
1296
|
+
memberships: memberships.map((membership) => ({
|
|
1297
|
+
inputSha256: membership.inputSha256,
|
|
1298
|
+
isolationSha256: generation.isolationSha256,
|
|
1299
|
+
ordinal: membership.ordinal,
|
|
1300
|
+
recordKey: membership.recordKey,
|
|
1301
|
+
recordSha256: membership.recordSha256,
|
|
1302
|
+
})),
|
|
1303
|
+
v: 2,
|
|
1304
|
+
}) !== generation.membershipSha256) {
|
|
1305
|
+
throw new OhLibSqlSemanticV2Error("integrity", "A semantic generation membership digest is invalid.");
|
|
1306
|
+
}
|
|
1307
|
+
return Object.freeze(memberships);
|
|
1308
|
+
}
|
|
1309
|
+
|
|
1310
|
+
type StoredVector = Readonly<{
|
|
1311
|
+
bytes: Uint8Array;
|
|
1312
|
+
inputSha256: Sha256Hex;
|
|
1313
|
+
vectorSha256: Sha256Hex;
|
|
1314
|
+
}>;
|
|
1315
|
+
|
|
1316
|
+
async function readVectors(
|
|
1317
|
+
client: OhLibSqlClientV1,
|
|
1318
|
+
isolationSha256: Sha256Hex,
|
|
1319
|
+
inputSha256s: readonly Sha256Hex[],
|
|
1320
|
+
): Promise<ReadonlyMap<Sha256Hex, StoredVector>> {
|
|
1321
|
+
const vectors = new Map<Sha256Hex, StoredVector>();
|
|
1322
|
+
for (let offset = 0; offset < inputSha256s.length; offset += 64) {
|
|
1323
|
+
const page = inputSha256s.slice(offset, offset + 64);
|
|
1324
|
+
if (page.length === 0) continue;
|
|
1325
|
+
const placeholders = page.map(() => "?").join(", ");
|
|
1326
|
+
const result = await client.execute({
|
|
1327
|
+
args: [isolationSha256, OH_CLOUDFLARE_EMBEDDING_PROFILE_V1.profileSha256,
|
|
1328
|
+
OH_SEMANTIC_RENDERER_V1.rendererSha256, ...page],
|
|
1329
|
+
sql: `SELECT input_sha256, vector_sha256, vector FROM oh_semantic_vectors
|
|
1330
|
+
WHERE isolation_sha256 = ? AND profile_sha256 = ? AND renderer_sha256 = ?
|
|
1331
|
+
AND input_sha256 IN (${placeholders}) ORDER BY input_sha256`,
|
|
1332
|
+
});
|
|
1333
|
+
for (const row of result.rows) {
|
|
1334
|
+
const inputSha256 = parseSha256Hex(rowValue(row, "input_sha256", 0));
|
|
1335
|
+
const vectorSha256 = parseSha256Hex(rowValue(row, "vector_sha256", 1));
|
|
1336
|
+
if (inputSha256 === null || vectorSha256 === null || !page.includes(inputSha256)
|
|
1337
|
+
|| vectors.has(inputSha256)) {
|
|
1338
|
+
throw new OhLibSqlSemanticV2Error("integrity", "A cached semantic vector identity is invalid.");
|
|
1339
|
+
}
|
|
1340
|
+
const bytes = storedBytes(rowValue(row, "vector", 2));
|
|
1341
|
+
if (bytes === null) {
|
|
1342
|
+
throw new OhLibSqlSemanticV2Error("integrity", "A cached semantic vector is corrupt.");
|
|
1343
|
+
}
|
|
1344
|
+
decodeVector(bytes, vectorSha256);
|
|
1345
|
+
vectors.set(inputSha256, Object.freeze({
|
|
1346
|
+
bytes,
|
|
1347
|
+
inputSha256,
|
|
1348
|
+
vectorSha256,
|
|
1349
|
+
}));
|
|
1350
|
+
}
|
|
1351
|
+
}
|
|
1352
|
+
return vectors;
|
|
1353
|
+
}
|
|
1354
|
+
|
|
1355
|
+
function validateEmbeddingClient(client: OhCloudflareEmbeddingClientV1): void {
|
|
1356
|
+
if (client.profile.profileSha256 !== OH_CLOUDFLARE_EMBEDDING_PROFILE_V1.profileSha256) {
|
|
1357
|
+
throw new OhLibSqlSemanticV2Error("invalid-input", "The embedding client profile is incompatible.");
|
|
1358
|
+
}
|
|
1359
|
+
}
|
|
1360
|
+
|
|
1361
|
+
export class OhLibSqlSemanticCacheV2 {
|
|
1362
|
+
readonly #client: OhLibSqlClientV1;
|
|
1363
|
+
readonly #closeClient: boolean;
|
|
1364
|
+
#closed = false;
|
|
1365
|
+
|
|
1366
|
+
private constructor(client: OhLibSqlClientV1, closeClient: boolean) {
|
|
1367
|
+
this.#client = client;
|
|
1368
|
+
this.#closeClient = closeClient;
|
|
1369
|
+
}
|
|
1370
|
+
|
|
1371
|
+
/** @internal Public callers should use `openOhLibSqlSemanticCacheV2`. */
|
|
1372
|
+
static async open(client: OhLibSqlClientV1, closeClient: boolean): Promise<OhLibSqlSemanticCacheV2> {
|
|
1373
|
+
await verifySchema(client);
|
|
1374
|
+
return new OhLibSqlSemanticCacheV2(client, closeClient);
|
|
1375
|
+
}
|
|
1376
|
+
|
|
1377
|
+
#open(): void {
|
|
1378
|
+
if (this.#closed) throw new OhLibSqlSemanticV2Error("schema-unavailable", "The semantic cache is closed.");
|
|
1379
|
+
}
|
|
1380
|
+
|
|
1381
|
+
async close(): Promise<void> {
|
|
1382
|
+
if (this.#closed) return;
|
|
1383
|
+
this.#closed = true;
|
|
1384
|
+
if (this.#closeClient) this.#client.close?.();
|
|
1385
|
+
}
|
|
1386
|
+
|
|
1387
|
+
/**
|
|
1388
|
+
* Reads the current compare-and-swap base without exposing cache rows or
|
|
1389
|
+
* private source text. An absent or purged authority has no published head.
|
|
1390
|
+
*/
|
|
1391
|
+
async publishedHead(input: Readonly<{
|
|
1392
|
+
authorityId: string;
|
|
1393
|
+
isolationSha256?: Sha256Hex;
|
|
1394
|
+
}>): Promise<OhSemanticPublishedHeadV2 | null> {
|
|
1395
|
+
this.#open();
|
|
1396
|
+
const authorityId = parseAuthorityId(input.authorityId);
|
|
1397
|
+
const isolation = isolationSha256(authorityId, input.isolationSha256);
|
|
1398
|
+
if (await readPurge(this.#client, authorityId) !== null) return null;
|
|
1399
|
+
const head = await readHead(this.#client, authorityId);
|
|
1400
|
+
if (head === null || head.isolationSha256 !== isolation) return null;
|
|
1401
|
+
const generation = await readGeneration(this.#client, authorityId, head.generation);
|
|
1402
|
+
if (generation === null || !headMatchesGeneration(head, generation)) {
|
|
1403
|
+
if (await readPurge(this.#client, authorityId) !== null) return null;
|
|
1404
|
+
throw new OhLibSqlSemanticV2Error(
|
|
1405
|
+
"integrity",
|
|
1406
|
+
"The semantic published head does not match its immutable generation.",
|
|
1407
|
+
);
|
|
1408
|
+
}
|
|
1409
|
+
if (await readPurge(this.#client, authorityId) !== null) return null;
|
|
1410
|
+
const finalHead = await readHead(this.#client, authorityId);
|
|
1411
|
+
if (finalHead === null) {
|
|
1412
|
+
if (await readPurge(this.#client, authorityId) !== null) return null;
|
|
1413
|
+
throw new OhLibSqlSemanticV2Error(
|
|
1414
|
+
"integrity",
|
|
1415
|
+
"The semantic published head disappeared during its read.",
|
|
1416
|
+
);
|
|
1417
|
+
}
|
|
1418
|
+
if (canonicalJson(finalHead) !== canonicalJson(head)) {
|
|
1419
|
+
throw new OhLibSqlSemanticV2Error(
|
|
1420
|
+
"conflict",
|
|
1421
|
+
"The semantic published head changed during its read.",
|
|
1422
|
+
);
|
|
1423
|
+
}
|
|
1424
|
+
return Object.freeze({ ...head, v: 2 });
|
|
1425
|
+
}
|
|
1426
|
+
|
|
1427
|
+
async stage(input: Readonly<{
|
|
1428
|
+
authorityId: string;
|
|
1429
|
+
authoritySha256: Sha256Hex;
|
|
1430
|
+
createdAt?: string;
|
|
1431
|
+
documents: readonly OhSemanticDocumentV2[];
|
|
1432
|
+
embeddingClient: OhCloudflareEmbeddingClientV1;
|
|
1433
|
+
generation: number;
|
|
1434
|
+
isolationSha256?: Sha256Hex;
|
|
1435
|
+
maximumChunksPerDocument?: number;
|
|
1436
|
+
signal?: AbortSignal;
|
|
1437
|
+
}>): Promise<OhSemanticStageResultV2> {
|
|
1438
|
+
this.#open();
|
|
1439
|
+
validateEmbeddingClient(input.embeddingClient);
|
|
1440
|
+
const prepared = prepareGeneration(input);
|
|
1441
|
+
await reserveIsolation(
|
|
1442
|
+
this.#client,
|
|
1443
|
+
prepared.authorityId,
|
|
1444
|
+
prepared.isolationSha256,
|
|
1445
|
+
prepared.createdAt,
|
|
1446
|
+
);
|
|
1447
|
+
if (await readPurge(this.#client, prepared.authorityId) !== null) {
|
|
1448
|
+
throw new OhLibSqlSemanticV2Error("purged", "The semantic authority was purged.");
|
|
1449
|
+
}
|
|
1450
|
+
const existingGeneration = await readGeneration(
|
|
1451
|
+
this.#client,
|
|
1452
|
+
prepared.authorityId,
|
|
1453
|
+
prepared.generation,
|
|
1454
|
+
);
|
|
1455
|
+
if (existingGeneration !== null && !generationMatches(existingGeneration, prepared)) {
|
|
1456
|
+
throw new OhLibSqlSemanticV2Error("conflict", "The semantic generation identity conflicts.");
|
|
1457
|
+
}
|
|
1458
|
+
const uniqueInputs = new Map<Sha256Hex, OhRenderedEmbeddingInputV1>();
|
|
1459
|
+
for (const membership of prepared.memberships) uniqueInputs.set(membership.inputSha256, membership.input);
|
|
1460
|
+
const orderedInputs = [...uniqueInputs.entries()].sort(([left], [right]) => left < right ? -1 : 1);
|
|
1461
|
+
const existingVectors = await readVectors(
|
|
1462
|
+
this.#client,
|
|
1463
|
+
prepared.isolationSha256,
|
|
1464
|
+
orderedInputs.map(([digest]) => digest),
|
|
1465
|
+
);
|
|
1466
|
+
const missing = orderedInputs.filter(([digest]) => !existingVectors.has(digest));
|
|
1467
|
+
const candidateVectors = new Map(existingVectors);
|
|
1468
|
+
for (let offset = 0; offset < missing.length; offset += OH_LIBSQL_SEMANTIC_LIMITS_V2.embeddingBatch) {
|
|
1469
|
+
const page = missing.slice(offset, offset + OH_LIBSQL_SEMANTIC_LIMITS_V2.embeddingBatch);
|
|
1470
|
+
const vectors = await input.embeddingClient.embed(
|
|
1471
|
+
page.map(([, rendered]) => rendered),
|
|
1472
|
+
input.signal === undefined ? {} : { signal: input.signal },
|
|
1473
|
+
);
|
|
1474
|
+
if (vectors.length !== page.length) {
|
|
1475
|
+
throw new OhLibSqlSemanticV2Error("integrity", "The embedding client returned a mismatched vector batch.");
|
|
1476
|
+
}
|
|
1477
|
+
for (const [index, [inputSha256]] of page.entries()) {
|
|
1478
|
+
const vector = vectors[index];
|
|
1479
|
+
if (vector === undefined) throw new OhLibSqlSemanticV2Error("integrity", "A semantic vector is missing.");
|
|
1480
|
+
const bytes = vectorBytes(vector);
|
|
1481
|
+
const vectorSha256 = sha256Hex(bytes);
|
|
1482
|
+
decodeVector(bytes, vectorSha256);
|
|
1483
|
+
candidateVectors.set(inputSha256, Object.freeze({ bytes, inputSha256, vectorSha256 }));
|
|
1484
|
+
}
|
|
1485
|
+
}
|
|
1486
|
+
const statements: OhLibSqlStatementV1[] = [{
|
|
1487
|
+
args: [prepared.authorityId, prepared.generation, prepared.authoritySha256,
|
|
1488
|
+
prepared.isolationSha256,
|
|
1489
|
+
OH_CLOUDFLARE_EMBEDDING_PROFILE_V1.profileSha256,
|
|
1490
|
+
OH_SEMANTIC_RENDERER_V1.rendererSha256, prepared.membershipSha256,
|
|
1491
|
+
prepared.generationSha256, prepared.documentCount, prepared.chunkCount,
|
|
1492
|
+
prepared.createdAt, prepared.authorityId],
|
|
1493
|
+
sql: `INSERT INTO oh_semantic_generations(authority_id, generation,
|
|
1494
|
+
authority_sha256, isolation_sha256, profile_sha256, renderer_sha256, membership_sha256,
|
|
1495
|
+
generation_sha256, document_count, chunk_count, created_at)
|
|
1496
|
+
SELECT ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
|
|
1497
|
+
WHERE NOT EXISTS (SELECT 1 FROM oh_semantic_purges WHERE authority_id = ?)
|
|
1498
|
+
ON CONFLICT DO NOTHING`,
|
|
1499
|
+
}];
|
|
1500
|
+
for (const [inputSha256] of orderedInputs) {
|
|
1501
|
+
const candidate = candidateVectors.get(inputSha256);
|
|
1502
|
+
if (candidate === undefined) {
|
|
1503
|
+
throw new OhLibSqlSemanticV2Error("integrity", "A semantic vector candidate is missing.");
|
|
1504
|
+
}
|
|
1505
|
+
statements.push({
|
|
1506
|
+
args: [prepared.isolationSha256,
|
|
1507
|
+
OH_CLOUDFLARE_EMBEDDING_PROFILE_V1.profileSha256,
|
|
1508
|
+
OH_SEMANTIC_RENDERER_V1.rendererSha256, inputSha256, candidate.vectorSha256,
|
|
1509
|
+
candidate.bytes, prepared.createdAt, prepared.authorityId, prepared.generation,
|
|
1510
|
+
prepared.generationSha256, prepared.isolationSha256, prepared.authorityId],
|
|
1511
|
+
sql: `INSERT INTO oh_semantic_vectors(isolation_sha256, profile_sha256,
|
|
1512
|
+
renderer_sha256, input_sha256, vector_sha256, vector, created_at)
|
|
1513
|
+
SELECT ?, ?, ?, ?, ?, ?, ?
|
|
1514
|
+
WHERE EXISTS (SELECT 1 FROM oh_semantic_generations
|
|
1515
|
+
WHERE authority_id = ? AND generation = ? AND generation_sha256 = ?
|
|
1516
|
+
AND isolation_sha256 = ?)
|
|
1517
|
+
AND NOT EXISTS (SELECT 1 FROM oh_semantic_purges WHERE authority_id = ?)
|
|
1518
|
+
ON CONFLICT DO NOTHING`,
|
|
1519
|
+
});
|
|
1520
|
+
}
|
|
1521
|
+
for (const membership of prepared.memberships) {
|
|
1522
|
+
statements.push({
|
|
1523
|
+
args: [prepared.authorityId, prepared.generation, prepared.generationSha256,
|
|
1524
|
+
prepared.isolationSha256, membership.recordKey, membership.recordSha256, membership.ordinal,
|
|
1525
|
+
membership.inputSha256, prepared.authorityId, prepared.generation,
|
|
1526
|
+
prepared.generationSha256, prepared.isolationSha256, prepared.authorityId],
|
|
1527
|
+
sql: `INSERT INTO oh_semantic_memberships(authority_id, generation,
|
|
1528
|
+
generation_sha256, isolation_sha256, record_key, record_sha256, ordinal, input_sha256)
|
|
1529
|
+
SELECT ?, ?, ?, ?, ?, ?, ?, ?
|
|
1530
|
+
WHERE EXISTS (SELECT 1 FROM oh_semantic_generations
|
|
1531
|
+
WHERE authority_id = ? AND generation = ? AND generation_sha256 = ?
|
|
1532
|
+
AND isolation_sha256 = ?)
|
|
1533
|
+
AND NOT EXISTS (SELECT 1 FROM oh_semantic_purges WHERE authority_id = ?)
|
|
1534
|
+
ON CONFLICT DO NOTHING`,
|
|
1535
|
+
});
|
|
1536
|
+
}
|
|
1537
|
+
await this.#client.batch(statements, "write");
|
|
1538
|
+
const completeVectors = await readVectors(
|
|
1539
|
+
this.#client,
|
|
1540
|
+
prepared.isolationSha256,
|
|
1541
|
+
orderedInputs.map(([digest]) => digest),
|
|
1542
|
+
);
|
|
1543
|
+
if (completeVectors.size !== orderedInputs.length) {
|
|
1544
|
+
if (await readPurge(this.#client, prepared.authorityId) !== null) {
|
|
1545
|
+
throw new OhLibSqlSemanticV2Error("purged", "The semantic authority was purged.");
|
|
1546
|
+
}
|
|
1547
|
+
throw new OhLibSqlSemanticV2Error("integrity", "The semantic vector cache did not converge.");
|
|
1548
|
+
}
|
|
1549
|
+
const stored = await readGeneration(this.#client, prepared.authorityId, prepared.generation);
|
|
1550
|
+
if (await readPurge(this.#client, prepared.authorityId) !== null) {
|
|
1551
|
+
throw new OhLibSqlSemanticV2Error("purged", "The semantic authority was purged.");
|
|
1552
|
+
}
|
|
1553
|
+
if (stored === null || !generationMatches(stored, prepared)) {
|
|
1554
|
+
throw new OhLibSqlSemanticV2Error("conflict", "The semantic generation identity conflicts.");
|
|
1555
|
+
}
|
|
1556
|
+
const memberships = await readMemberships(this.#client, stored);
|
|
1557
|
+
if (canonicalJson(memberships) !== canonicalJson(prepared.memberships.map((membership) => ({
|
|
1558
|
+
inputSha256: membership.inputSha256,
|
|
1559
|
+
ordinal: membership.ordinal,
|
|
1560
|
+
recordKey: membership.recordKey,
|
|
1561
|
+
recordSha256: membership.recordSha256,
|
|
1562
|
+
})))) {
|
|
1563
|
+
throw new OhLibSqlSemanticV2Error("conflict", "The semantic generation membership conflicts.");
|
|
1564
|
+
}
|
|
1565
|
+
return Object.freeze({
|
|
1566
|
+
authorityId: prepared.authorityId,
|
|
1567
|
+
chunks: prepared.chunkCount,
|
|
1568
|
+
documents: prepared.documentCount,
|
|
1569
|
+
embedded: missing.length,
|
|
1570
|
+
generation: prepared.generation,
|
|
1571
|
+
generationSha256: prepared.generationSha256,
|
|
1572
|
+
isolationSha256: prepared.isolationSha256,
|
|
1573
|
+
membershipSha256: prepared.membershipSha256,
|
|
1574
|
+
reused: orderedInputs.length - missing.length,
|
|
1575
|
+
status: "staged",
|
|
1576
|
+
v: 2,
|
|
1577
|
+
});
|
|
1578
|
+
}
|
|
1579
|
+
|
|
1580
|
+
async publish(input: Readonly<{
|
|
1581
|
+
authorityId: string;
|
|
1582
|
+
expectedPublishedGeneration: number | null;
|
|
1583
|
+
generation: number;
|
|
1584
|
+
isolationSha256?: Sha256Hex;
|
|
1585
|
+
publishedAt?: string;
|
|
1586
|
+
}>): Promise<OhSemanticPublishResultV2> {
|
|
1587
|
+
this.#open();
|
|
1588
|
+
const authorityId = parseAuthorityId(input.authorityId);
|
|
1589
|
+
const isolation = isolationSha256(authorityId, input.isolationSha256);
|
|
1590
|
+
const generationNumber = parseGeneration(input.generation);
|
|
1591
|
+
const expected = input.expectedPublishedGeneration === null
|
|
1592
|
+
? null : parseGeneration(input.expectedPublishedGeneration);
|
|
1593
|
+
const publishedAt = parseInstant(input.publishedAt ?? canonicalNow());
|
|
1594
|
+
if (await readPurge(this.#client, authorityId) !== null) {
|
|
1595
|
+
throw new OhLibSqlSemanticV2Error("purged", "The semantic authority was purged.");
|
|
1596
|
+
}
|
|
1597
|
+
const generation = await readGeneration(this.#client, authorityId, generationNumber);
|
|
1598
|
+
if (generation === null || generation.isolationSha256 !== isolation) {
|
|
1599
|
+
throw new OhLibSqlSemanticV2Error("conflict", "The semantic generation is not staged.");
|
|
1600
|
+
}
|
|
1601
|
+
await readMemberships(this.#client, generation);
|
|
1602
|
+
const before = await readHead(this.#client, authorityId);
|
|
1603
|
+
if (before !== null && headMatchesGeneration(before, generation)) {
|
|
1604
|
+
return Object.freeze({
|
|
1605
|
+
authorityId,
|
|
1606
|
+
generation: generationNumber,
|
|
1607
|
+
generationSha256: generation.generationSha256,
|
|
1608
|
+
isolationSha256: isolation,
|
|
1609
|
+
published: false,
|
|
1610
|
+
v: 2,
|
|
1611
|
+
});
|
|
1612
|
+
}
|
|
1613
|
+
if ((before === null) !== (expected === null)
|
|
1614
|
+
|| (before !== null && before.generation !== expected)
|
|
1615
|
+
|| (before !== null && generationNumber < before.generation)) {
|
|
1616
|
+
throw new OhLibSqlSemanticV2Error("conflict", "The semantic published-head precondition failed.");
|
|
1617
|
+
}
|
|
1618
|
+
let result: OhLibSqlResultV1;
|
|
1619
|
+
const values = [generation.authorityId, generation.generation,
|
|
1620
|
+
generation.authoritySha256, generation.isolationSha256,
|
|
1621
|
+
generation.profileSha256, generation.rendererSha256,
|
|
1622
|
+
generation.membershipSha256, generation.generationSha256, publishedAt];
|
|
1623
|
+
if (expected === null) {
|
|
1624
|
+
result = await this.#client.execute({
|
|
1625
|
+
args: [...values, generation.authorityId, generation.generation,
|
|
1626
|
+
generation.generationSha256, generation.isolationSha256, authorityId, authorityId],
|
|
1627
|
+
sql: `INSERT INTO oh_semantic_heads(authority_id, generation,
|
|
1628
|
+
authority_sha256, isolation_sha256, profile_sha256, renderer_sha256, membership_sha256,
|
|
1629
|
+
generation_sha256, published_at)
|
|
1630
|
+
SELECT ?, ?, ?, ?, ?, ?, ?, ?, ?
|
|
1631
|
+
WHERE EXISTS (SELECT 1 FROM oh_semantic_generations
|
|
1632
|
+
WHERE authority_id = ? AND generation = ? AND generation_sha256 = ?
|
|
1633
|
+
AND isolation_sha256 = ?)
|
|
1634
|
+
AND NOT EXISTS (SELECT 1 FROM oh_semantic_purges WHERE authority_id = ?)
|
|
1635
|
+
AND NOT EXISTS (SELECT 1 FROM oh_semantic_heads WHERE authority_id = ?)
|
|
1636
|
+
ON CONFLICT DO NOTHING`,
|
|
1637
|
+
});
|
|
1638
|
+
} else {
|
|
1639
|
+
result = await this.#client.execute({
|
|
1640
|
+
args: [generation.generation, generation.authoritySha256, generation.isolationSha256,
|
|
1641
|
+
generation.profileSha256, generation.rendererSha256, generation.membershipSha256,
|
|
1642
|
+
generation.generationSha256, publishedAt, authorityId, expected,
|
|
1643
|
+
generation.authorityId, generation.generation, generation.generationSha256,
|
|
1644
|
+
generation.isolationSha256, authorityId],
|
|
1645
|
+
sql: `UPDATE oh_semantic_heads SET generation = ?, authority_sha256 = ?,
|
|
1646
|
+
isolation_sha256 = ?, profile_sha256 = ?, renderer_sha256 = ?, membership_sha256 = ?,
|
|
1647
|
+
generation_sha256 = ?, published_at = ?
|
|
1648
|
+
WHERE authority_id = ? AND generation = ?
|
|
1649
|
+
AND EXISTS (SELECT 1 FROM oh_semantic_generations
|
|
1650
|
+
WHERE authority_id = ? AND generation = ? AND generation_sha256 = ?
|
|
1651
|
+
AND isolation_sha256 = ?)
|
|
1652
|
+
AND NOT EXISTS (SELECT 1 FROM oh_semantic_purges WHERE authority_id = ?)`,
|
|
1653
|
+
});
|
|
1654
|
+
}
|
|
1655
|
+
if (await readPurge(this.#client, authorityId) !== null) {
|
|
1656
|
+
throw new OhLibSqlSemanticV2Error("purged", "The semantic authority was purged.");
|
|
1657
|
+
}
|
|
1658
|
+
const after = await readHead(this.#client, authorityId);
|
|
1659
|
+
if (after === null || !headMatchesGeneration(after, generation)) {
|
|
1660
|
+
throw new OhLibSqlSemanticV2Error("conflict", "The semantic published head did not converge.");
|
|
1661
|
+
}
|
|
1662
|
+
return Object.freeze({
|
|
1663
|
+
authorityId,
|
|
1664
|
+
generation: generationNumber,
|
|
1665
|
+
generationSha256: generation.generationSha256,
|
|
1666
|
+
isolationSha256: isolation,
|
|
1667
|
+
published: rowsAffected(result) > 0,
|
|
1668
|
+
v: 2,
|
|
1669
|
+
});
|
|
1670
|
+
}
|
|
1671
|
+
|
|
1672
|
+
async search(input: Readonly<{
|
|
1673
|
+
authority: OhSemanticAuthorityRefV2;
|
|
1674
|
+
embeddingClient: OhCloudflareEmbeddingClientV1;
|
|
1675
|
+
limit?: number;
|
|
1676
|
+
query: string;
|
|
1677
|
+
signal?: AbortSignal;
|
|
1678
|
+
}>): Promise<readonly OhSemanticSearchResultV2[]> {
|
|
1679
|
+
this.#open();
|
|
1680
|
+
validateEmbeddingClient(input.embeddingClient);
|
|
1681
|
+
const authorityId = parseAuthorityId(input.authority.authorityId);
|
|
1682
|
+
const authoritySha256 = parseDigest(input.authority.authoritySha256, "authority");
|
|
1683
|
+
const isolation = isolationSha256(authorityId, input.authority.isolationSha256);
|
|
1684
|
+
const authorityGeneration = parseGeneration(input.authority.generation);
|
|
1685
|
+
const limit = input.limit ?? 10;
|
|
1686
|
+
if (input.authority.v !== 2 || !Number.isSafeInteger(limit) || limit < 1
|
|
1687
|
+
|| limit > OH_LIBSQL_SEMANTIC_LIMITS_V2.searchLimit
|
|
1688
|
+
|| !Array.isArray(input.authority.records)
|
|
1689
|
+
|| input.authority.records.length > OH_LIBSQL_SEMANTIC_LIMITS_V2.documentsPerGeneration) {
|
|
1690
|
+
throw new OhLibSqlSemanticV2Error("invalid-input", "Invalid semantic search authority or limit.");
|
|
1691
|
+
}
|
|
1692
|
+
const records = new Map<string, Sha256Hex>();
|
|
1693
|
+
for (const record of input.authority.records) {
|
|
1694
|
+
const key = parseRecordKey(record.key);
|
|
1695
|
+
const recordSha256 = parseDigest(record.recordSha256, "record");
|
|
1696
|
+
if (records.has(key)) throw new OhLibSqlSemanticV2Error("invalid-input", "Duplicate authority record key.");
|
|
1697
|
+
records.set(key, recordSha256);
|
|
1698
|
+
}
|
|
1699
|
+
if (await readPurge(this.#client, authorityId) !== null) return Object.freeze([]);
|
|
1700
|
+
const head = await readHead(this.#client, authorityId);
|
|
1701
|
+
if (head === null || head.authoritySha256 !== authoritySha256
|
|
1702
|
+
|| head.generation !== authorityGeneration
|
|
1703
|
+
|| head.isolationSha256 !== isolation
|
|
1704
|
+
|| head.profileSha256 !== OH_CLOUDFLARE_EMBEDDING_PROFILE_V1.profileSha256
|
|
1705
|
+
|| head.rendererSha256 !== OH_SEMANTIC_RENDERER_V1.rendererSha256) {
|
|
1706
|
+
return Object.freeze([]);
|
|
1707
|
+
}
|
|
1708
|
+
const generation = await readGeneration(this.#client, authorityId, authorityGeneration);
|
|
1709
|
+
if (generation === null || !headMatchesGeneration(head, generation)) return Object.freeze([]);
|
|
1710
|
+
const renderedQuery = renderOhCloudflareEmbeddingQueryV1(input.query);
|
|
1711
|
+
const queryVectors = await input.embeddingClient.embed(
|
|
1712
|
+
[renderedQuery],
|
|
1713
|
+
input.signal === undefined ? {} : { signal: input.signal },
|
|
1714
|
+
);
|
|
1715
|
+
const queryVector = queryVectors[0];
|
|
1716
|
+
if (queryVectors.length !== 1 || queryVector === undefined) {
|
|
1717
|
+
throw new OhLibSqlSemanticV2Error("integrity", "The query embedding response is invalid.");
|
|
1718
|
+
}
|
|
1719
|
+
const normalizedQuery = normalizeOhEmbeddingV1(queryVector);
|
|
1720
|
+
const best = new Map<string, OhSemanticSearchResultV2>();
|
|
1721
|
+
let scanned = 0;
|
|
1722
|
+
for (let offset = 0; offset < generation.chunkCount;
|
|
1723
|
+
offset += OH_LIBSQL_SEMANTIC_LIMITS_V2.searchPage) {
|
|
1724
|
+
const result = await this.#client.execute({
|
|
1725
|
+
args: [OH_CLOUDFLARE_EMBEDDING_PROFILE_V1.profileSha256,
|
|
1726
|
+
OH_SEMANTIC_RENDERER_V1.rendererSha256, authorityId, authorityGeneration,
|
|
1727
|
+
isolation, OH_LIBSQL_SEMANTIC_LIMITS_V2.searchPage, offset],
|
|
1728
|
+
sql: `SELECT membership.generation_sha256, membership.isolation_sha256,
|
|
1729
|
+
membership.record_key,
|
|
1730
|
+
membership.record_sha256, membership.ordinal, membership.input_sha256,
|
|
1731
|
+
vector.vector_sha256, vector.vector
|
|
1732
|
+
FROM oh_semantic_memberships AS membership
|
|
1733
|
+
JOIN oh_semantic_vectors AS vector
|
|
1734
|
+
ON vector.input_sha256 = membership.input_sha256
|
|
1735
|
+
AND vector.isolation_sha256 = membership.isolation_sha256
|
|
1736
|
+
AND vector.profile_sha256 = ? AND vector.renderer_sha256 = ?
|
|
1737
|
+
WHERE membership.authority_id = ? AND membership.generation = ?
|
|
1738
|
+
AND membership.isolation_sha256 = ?
|
|
1739
|
+
ORDER BY membership.record_key, membership.ordinal LIMIT ? OFFSET ?`,
|
|
1740
|
+
});
|
|
1741
|
+
for (const row of result.rows) {
|
|
1742
|
+
scanned += 1;
|
|
1743
|
+
const generationSha256 = parseSha256Hex(rowValue(row, "generation_sha256", 0));
|
|
1744
|
+
const storedIsolation = parseSha256Hex(rowValue(row, "isolation_sha256", 1));
|
|
1745
|
+
const key = safeCode(rowValue(row, "record_key", 2), 512);
|
|
1746
|
+
const recordSha256 = parseSha256Hex(rowValue(row, "record_sha256", 3));
|
|
1747
|
+
const ordinal = integer(rowValue(row, "ordinal", 4));
|
|
1748
|
+
const inputSha256 = parseSha256Hex(rowValue(row, "input_sha256", 5));
|
|
1749
|
+
const vectorSha256 = parseSha256Hex(rowValue(row, "vector_sha256", 6));
|
|
1750
|
+
if (generationSha256 !== generation.generationSha256
|
|
1751
|
+
|| storedIsolation !== isolation || key === null
|
|
1752
|
+
|| recordSha256 === null || ordinal === null || ordinal < 0
|
|
1753
|
+
|| ordinal >= OH_LIBSQL_SEMANTIC_LIMITS_V2.chunksPerDocument
|
|
1754
|
+
|| inputSha256 === null || vectorSha256 === null) {
|
|
1755
|
+
throw new OhLibSqlSemanticV2Error("integrity", "A semantic search row is invalid.");
|
|
1756
|
+
}
|
|
1757
|
+
if (records.get(key) !== recordSha256) continue;
|
|
1758
|
+
const vector = decodeVector(rowValue(row, "vector", 7), vectorSha256);
|
|
1759
|
+
let score = 0;
|
|
1760
|
+
for (let index = 0; index < normalizedQuery.length; index += 1) {
|
|
1761
|
+
score += (normalizedQuery[index] as number) * (vector[index] as number);
|
|
1762
|
+
}
|
|
1763
|
+
score = Math.max(-1, Math.min(1, score));
|
|
1764
|
+
const previous = best.get(key);
|
|
1765
|
+
if (previous === undefined || score > previous.score
|
|
1766
|
+
|| (score === previous.score && ordinal < previous.chunkOrdinal)) {
|
|
1767
|
+
best.set(key, Object.freeze({ chunkOrdinal: ordinal, key, recordSha256, score, v: 2 }));
|
|
1768
|
+
}
|
|
1769
|
+
}
|
|
1770
|
+
}
|
|
1771
|
+
if (scanned !== generation.chunkCount) {
|
|
1772
|
+
throw new OhLibSqlSemanticV2Error("integrity", "The semantic search scan is incomplete.");
|
|
1773
|
+
}
|
|
1774
|
+
const finalHead = await readHead(this.#client, authorityId);
|
|
1775
|
+
if (finalHead === null || canonicalJson(finalHead) !== canonicalJson(head)
|
|
1776
|
+
|| await readPurge(this.#client, authorityId) !== null) return Object.freeze([]);
|
|
1777
|
+
return Object.freeze([...best.values()]
|
|
1778
|
+
.sort((left, right) => right.score - left.score
|
|
1779
|
+
|| (left.key < right.key ? -1 : left.key > right.key ? 1 : 0))
|
|
1780
|
+
.slice(0, limit));
|
|
1781
|
+
}
|
|
1782
|
+
|
|
1783
|
+
/** Reads the immutable, content-free receipt for a completed purge. */
|
|
1784
|
+
async purgeReceipt(input: Readonly<{
|
|
1785
|
+
authorityId: string;
|
|
1786
|
+
isolationSha256?: Sha256Hex;
|
|
1787
|
+
}>): Promise<OhSemanticPurgeResultV2 | null> {
|
|
1788
|
+
this.#open();
|
|
1789
|
+
const authorityId = parseAuthorityId(input.authorityId);
|
|
1790
|
+
const isolation = isolationSha256(authorityId, input.isolationSha256);
|
|
1791
|
+
const receipt = await readPurge(this.#client, authorityId);
|
|
1792
|
+
if (receipt === null) return null;
|
|
1793
|
+
if (receipt.isolationSha256 !== isolation) {
|
|
1794
|
+
throw new OhLibSqlSemanticV2Error("conflict", "The semantic purge isolation conflicts.");
|
|
1795
|
+
}
|
|
1796
|
+
await verifyPurgeResidual(this.#client, authorityId);
|
|
1797
|
+
return receipt;
|
|
1798
|
+
}
|
|
1799
|
+
|
|
1800
|
+
async purgeAuthority(input: Readonly<{
|
|
1801
|
+
authorityId: string;
|
|
1802
|
+
isolationSha256?: Sha256Hex;
|
|
1803
|
+
purgedAt?: string;
|
|
1804
|
+
}>): Promise<OhSemanticPurgeResultV2> {
|
|
1805
|
+
this.#open();
|
|
1806
|
+
const authorityId = parseAuthorityId(input.authorityId);
|
|
1807
|
+
const isolation = isolationSha256(authorityId, input.isolationSha256);
|
|
1808
|
+
const previous = await readPurge(this.#client, authorityId);
|
|
1809
|
+
if (previous !== null) {
|
|
1810
|
+
if (previous.isolationSha256 !== isolation) {
|
|
1811
|
+
throw new OhLibSqlSemanticV2Error("conflict", "The semantic purge isolation conflicts.");
|
|
1812
|
+
}
|
|
1813
|
+
await verifyPurgeResidual(this.#client, authorityId);
|
|
1814
|
+
return previous;
|
|
1815
|
+
}
|
|
1816
|
+
const requestedAt = parseInstant(input.purgedAt ?? canonicalNow());
|
|
1817
|
+
await reservePurgeIsolation(this.#client, authorityId, isolation, requestedAt);
|
|
1818
|
+
const markerSha256 = purgeMarkerSha256(authorityId, isolation, requestedAt);
|
|
1819
|
+
await this.#client.batch([
|
|
1820
|
+
{
|
|
1821
|
+
args: [authorityId, isolation, OH_CLOUDFLARE_EMBEDDING_PROFILE_V1.profileSha256,
|
|
1822
|
+
authorityId, authorityId, requestedAt, markerSha256,
|
|
1823
|
+
authorityId, authorityId, authorityId, authorityId,
|
|
1824
|
+
isolation, authorityId, authorityId, isolation],
|
|
1825
|
+
sql: `INSERT INTO oh_semantic_purges(authority_id, isolation_sha256,
|
|
1826
|
+
profile_sha256, published_generation, published_generation_sha256,
|
|
1827
|
+
purged_at, purge_marker_sha256, generation_count, membership_count,
|
|
1828
|
+
orphan_vector_count, isolation_scope_count, counts_recorded)
|
|
1829
|
+
SELECT ?, ?, ?,
|
|
1830
|
+
(SELECT generation FROM oh_semantic_heads WHERE authority_id = ?),
|
|
1831
|
+
(SELECT generation_sha256 FROM oh_semantic_heads WHERE authority_id = ?),
|
|
1832
|
+
?, ?,
|
|
1833
|
+
(SELECT count(*) FROM oh_semantic_generations WHERE authority_id = ?),
|
|
1834
|
+
(SELECT count(*) FROM oh_semantic_memberships WHERE authority_id = ?),
|
|
1835
|
+
(SELECT count(*) FROM oh_semantic_vectors AS vector
|
|
1836
|
+
JOIN oh_semantic_isolations AS isolation
|
|
1837
|
+
ON isolation.isolation_sha256 = vector.isolation_sha256
|
|
1838
|
+
WHERE isolation.authority_id = ?),
|
|
1839
|
+
(SELECT count(*) FROM oh_semantic_isolations WHERE authority_id = ?),
|
|
1840
|
+
1
|
|
1841
|
+
WHERE EXISTS (SELECT 1 FROM oh_semantic_isolations
|
|
1842
|
+
WHERE isolation_sha256 = ? AND authority_id = ?)
|
|
1843
|
+
AND NOT EXISTS (SELECT 1 FROM oh_semantic_heads
|
|
1844
|
+
WHERE authority_id = ? AND isolation_sha256 <> ?)
|
|
1845
|
+
ON CONFLICT DO NOTHING`,
|
|
1846
|
+
},
|
|
1847
|
+
{
|
|
1848
|
+
args: [authorityId, authorityId, isolation],
|
|
1849
|
+
sql: `DELETE FROM oh_semantic_heads WHERE authority_id = ?
|
|
1850
|
+
AND EXISTS (SELECT 1 FROM oh_semantic_purges
|
|
1851
|
+
WHERE authority_id = ? AND isolation_sha256 = ?)`,
|
|
1852
|
+
},
|
|
1853
|
+
{
|
|
1854
|
+
args: [authorityId, authorityId, isolation],
|
|
1855
|
+
sql: `DELETE FROM oh_semantic_memberships WHERE authority_id = ?
|
|
1856
|
+
AND EXISTS (SELECT 1 FROM oh_semantic_purges
|
|
1857
|
+
WHERE authority_id = ? AND isolation_sha256 = ?)`,
|
|
1858
|
+
},
|
|
1859
|
+
{
|
|
1860
|
+
args: [authorityId, authorityId, isolation],
|
|
1861
|
+
sql: `DELETE FROM oh_semantic_vectors
|
|
1862
|
+
WHERE isolation_sha256 IN (SELECT isolation_sha256
|
|
1863
|
+
FROM oh_semantic_isolations WHERE authority_id = ?)
|
|
1864
|
+
AND EXISTS (SELECT 1 FROM oh_semantic_purges
|
|
1865
|
+
WHERE authority_id = ? AND isolation_sha256 = ?)`,
|
|
1866
|
+
},
|
|
1867
|
+
{
|
|
1868
|
+
args: [authorityId, authorityId, isolation],
|
|
1869
|
+
sql: `DELETE FROM oh_semantic_generations WHERE authority_id = ?
|
|
1870
|
+
AND EXISTS (SELECT 1 FROM oh_semantic_purges
|
|
1871
|
+
WHERE authority_id = ? AND isolation_sha256 = ?)`,
|
|
1872
|
+
},
|
|
1873
|
+
], "write");
|
|
1874
|
+
const receipt = await readPurge(this.#client, authorityId);
|
|
1875
|
+
if (receipt === null) {
|
|
1876
|
+
throw new OhLibSqlSemanticV2Error("conflict", "The semantic purge identity changed before tombstoning.");
|
|
1877
|
+
}
|
|
1878
|
+
if (receipt.isolationSha256 !== isolation) {
|
|
1879
|
+
throw new OhLibSqlSemanticV2Error("conflict", "The semantic purge isolation conflicts.");
|
|
1880
|
+
}
|
|
1881
|
+
await verifyPurgeResidual(this.#client, authorityId);
|
|
1882
|
+
return receipt;
|
|
1883
|
+
}
|
|
1884
|
+
}
|
|
1885
|
+
|
|
1886
|
+
export async function openOhLibSqlSemanticCacheV2(
|
|
1887
|
+
client: OhLibSqlClientV1,
|
|
1888
|
+
options: Readonly<{ closeClient?: boolean }> = {},
|
|
1889
|
+
): Promise<OhLibSqlSemanticCacheV2> {
|
|
1890
|
+
return await OhLibSqlSemanticCacheV2.open(client, options.closeClient ?? false);
|
|
1891
|
+
}
|