@hraness/oh 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +598 -0
- package/dist/canonical.d.ts +32 -0
- package/dist/canonical.d.ts.map +1 -0
- package/dist/cli.d.ts +4 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +3419 -0
- package/dist/contract.d.ts +33 -0
- package/dist/contract.d.ts.map +1 -0
- package/dist/graph.d.ts +67 -0
- package/dist/graph.d.ts.map +1 -0
- package/dist/index.d.ts +9 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +1988 -0
- package/dist/libsql.d.ts +57 -0
- package/dist/libsql.d.ts.map +1 -0
- package/dist/libsql.js +2662 -0
- package/dist/memory.d.ts +366 -0
- package/dist/memory.d.ts.map +1 -0
- package/dist/memory.js +3650 -0
- package/dist/ontology.d.ts +242 -0
- package/dist/ontology.d.ts.map +1 -0
- package/dist/operation.d.ts +24 -0
- package/dist/operation.d.ts.map +1 -0
- package/dist/projection-public.d.ts +59 -0
- package/dist/projection-public.d.ts.map +1 -0
- package/dist/projection-public.js +1682 -0
- package/dist/projection-suss.d.ts +17 -0
- package/dist/projection-suss.d.ts.map +1 -0
- package/dist/projection-suss.js +1721 -0
- package/dist/projection.d.ts +315 -0
- package/dist/projection.d.ts.map +1 -0
- package/dist/schema.d.ts +45 -0
- package/dist/schema.d.ts.map +1 -0
- package/dist/sdk.d.ts +51 -0
- package/dist/sdk.d.ts.map +1 -0
- package/dist/sdk.js +3072 -0
- package/dist/search.d.ts +34 -0
- package/dist/search.d.ts.map +1 -0
- package/dist/semantic.d.ts +83 -0
- package/dist/semantic.d.ts.map +1 -0
- package/dist/semantic.js +706 -0
- package/dist/sqlite/driver.d.ts +6 -0
- package/dist/sqlite/driver.d.ts.map +1 -0
- package/dist/sqlite/index.d.ts +5 -0
- package/dist/sqlite/index.d.ts.map +1 -0
- package/dist/sqlite/index.js +2840 -0
- package/dist/sqlite/migrations.d.ts +10 -0
- package/dist/sqlite/migrations.d.ts.map +1 -0
- package/dist/sqlite/port.d.ts +40 -0
- package/dist/sqlite/port.d.ts.map +1 -0
- package/dist/sqlite/runtime.d.ts +23 -0
- package/dist/sqlite/runtime.d.ts.map +1 -0
- package/dist/sqlite/store.d.ts +85 -0
- package/dist/sqlite/store.d.ts.map +1 -0
- package/dist/store.d.ts +236 -0
- package/dist/store.d.ts.map +1 -0
- package/dist/store.js +996 -0
- package/dist/sync.d.ts +57 -0
- package/dist/sync.d.ts.map +1 -0
- package/dist/sync.js +1328 -0
- package/package.json +121 -0
- package/skills/oh/SKILL.md +206 -0
- package/skills/oh/agents/openai.yaml +4 -0
- package/spec/README.md +74 -0
- package/spec/manifest.json +41 -0
- package/spec/v1/canonical-json.md +59 -0
- package/spec/v1/contract.json +28 -0
- package/spec/v1/contract.schema.json +58 -0
- package/spec/v1/embedding-profile.json +11 -0
- package/spec/v1/embedding.md +56 -0
- package/spec/v1/graph.md +87 -0
- package/spec/v1/memory.md +193 -0
- package/spec/v1/migration.md +92 -0
- package/spec/v1/ontology.json +55 -0
- package/spec/v1/ontology.md +80 -0
- package/spec/v1/operation.schema.json +138 -0
- package/spec/v1/projection-identity.schema.json +58 -0
- package/spec/v1/projection-query.schema.json +60 -0
- package/spec/v1/projection-result.schema.json +452 -0
- package/spec/v1/projection-rule-pack.schema.json +182 -0
- package/spec/v1/projection.md +165 -0
- package/spec/v1/record.schema.json +95 -0
- package/spec/v1/schema-evolution.md +51 -0
- package/spec/v1/schema-revision.schema.json +178 -0
- package/spec/v1/storage.md +88 -0
- package/spec/v1/store.md +131 -0
- package/spec/v1/sync-bundle.schema.json +51 -0
- package/spec/v1/sync.md +67 -0
- package/src/canonical.test.ts +46 -0
- package/src/canonical.ts +203 -0
- package/src/cli.test.ts +103 -0
- package/src/cli.ts +308 -0
- package/src/contract.ts +87 -0
- package/src/contracts.test.ts +147 -0
- package/src/graph.ts +248 -0
- package/src/index.ts +8 -0
- package/src/libsql.test.ts +657 -0
- package/src/libsql.ts +1687 -0
- package/src/memory.test.ts +783 -0
- package/src/memory.ts +1684 -0
- package/src/ontology.ts +573 -0
- package/src/operation.ts +80 -0
- package/src/projection-public.ts +53 -0
- package/src/projection-suss.ts +129 -0
- package/src/projection.test.ts +418 -0
- package/src/projection.ts +1457 -0
- package/src/schema.ts +156 -0
- package/src/sdk.ts +96 -0
- package/src/search.ts +66 -0
- package/src/semantic.test.ts +480 -0
- package/src/semantic.ts +333 -0
- package/src/sqlite/driver.ts +47 -0
- package/src/sqlite/index.ts +4 -0
- package/src/sqlite/migrations.test.ts +44 -0
- package/src/sqlite/migrations.ts +178 -0
- package/src/sqlite/port.test.ts +127 -0
- package/src/sqlite/port.ts +120 -0
- package/src/sqlite/runtime.test.ts +68 -0
- package/src/sqlite/runtime.ts +53 -0
- package/src/sqlite/store.test.ts +295 -0
- package/src/sqlite/store.ts +988 -0
- package/src/store.test.ts +121 -0
- package/src/store.ts +701 -0
- package/src/sync.test.ts +117 -0
- package/src/sync.ts +227 -0
package/src/semantic.ts
ADDED
|
@@ -0,0 +1,333 @@
|
|
|
1
|
+
import { mkdir, readFile, readdir, rename, unlink, writeFile } from "node:fs/promises";
|
|
2
|
+
import { join, resolve } from "node:path";
|
|
3
|
+
|
|
4
|
+
import { canonicalJson, canonicalSha256, hasExactKeys, isPlainRecord, parseSha256Hex,
|
|
5
|
+
safeCode, sha256Hex, type Sha256Hex } from "./canonical";
|
|
6
|
+
import { OH_GRAPH_LIMITS_V1, type KnowledgeGraphRecordV1 } from "./graph";
|
|
7
|
+
import type { OhSqliteStore } from "./sqlite/store";
|
|
8
|
+
|
|
9
|
+
export const OH_EMBEDDING_PROFILE_V1 = Object.freeze({
|
|
10
|
+
dimensions: 768,
|
|
11
|
+
distance: "cosine",
|
|
12
|
+
documentation: "https://ai.google.dev/gemma/docs/embeddinggemma",
|
|
13
|
+
documentFormat: "title: {title} | text: {content}",
|
|
14
|
+
engine: "@tobilu/qmd@2.5.3",
|
|
15
|
+
model: "hf:ggml-org/embeddinggemma-300M-GGUF/embeddinggemma-300M-Q8_0.gguf",
|
|
16
|
+
normalization: "l2",
|
|
17
|
+
queryFormat: "task: search result | query: {query}",
|
|
18
|
+
v: 1,
|
|
19
|
+
} as const);
|
|
20
|
+
|
|
21
|
+
export type OhEmbeddingProfileV1 = typeof OH_EMBEDDING_PROFILE_V1;
|
|
22
|
+
|
|
23
|
+
export function formatOhEmbeddingQueryV1(query: string): string {
|
|
24
|
+
return `task: search result | query: ${query}`;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export function formatOhEmbeddingDocumentV1(title: string, content: string): string {
|
|
28
|
+
return `title: ${title} | text: ${content}`;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export function normalizeOhEmbeddingV1(vector: readonly number[]): readonly number[] {
|
|
32
|
+
if (vector.length !== OH_EMBEDDING_PROFILE_V1.dimensions
|
|
33
|
+
|| vector.some((component) => !Number.isFinite(component))) {
|
|
34
|
+
throw new TypeError(`Embedding vectors must contain ${OH_EMBEDDING_PROFILE_V1.dimensions} finite values.`);
|
|
35
|
+
}
|
|
36
|
+
const magnitude = Math.sqrt(vector.reduce((sum, component) => sum + component * component, 0));
|
|
37
|
+
if (magnitude === 0) throw new TypeError("Embedding vectors must have nonzero magnitude.");
|
|
38
|
+
return vector.map((component) => component / magnitude);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export function cosineSimilarityV1(left: readonly number[], right: readonly number[]): number {
|
|
42
|
+
const normalizedLeft = normalizeOhEmbeddingV1(left);
|
|
43
|
+
const normalizedRight = normalizeOhEmbeddingV1(right);
|
|
44
|
+
return normalizedLeft.reduce((sum, component, index) => sum + component * (normalizedRight[index] as number), 0);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export type OhSemanticSearchResultV1 = Readonly<{
|
|
48
|
+
key: string;
|
|
49
|
+
recordSha256: Sha256Hex;
|
|
50
|
+
score: number;
|
|
51
|
+
v: 1;
|
|
52
|
+
}>;
|
|
53
|
+
|
|
54
|
+
export interface OhSemanticSearchBackendV1 {
|
|
55
|
+
readonly profile: OhEmbeddingProfileV1;
|
|
56
|
+
close(): Promise<void>;
|
|
57
|
+
index(records: readonly KnowledgeGraphRecordV1[]): Promise<Readonly<{ indexed: number; v: 1 }>>;
|
|
58
|
+
search(query: string, limit: number, authority: OhSqliteStore): Promise<readonly OhSemanticSearchResultV1[]>;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
type QmdStore = {
|
|
62
|
+
close(): Promise<void>;
|
|
63
|
+
embed(options: Readonly<{ collection: string; model: string }>): Promise<unknown>;
|
|
64
|
+
searchVector(query: string, options: Readonly<{ collection: string; limit: number }>): Promise<unknown>;
|
|
65
|
+
update(options: Readonly<{ collections: readonly string[] }>): Promise<unknown>;
|
|
66
|
+
};
|
|
67
|
+
export type QmdStoreFactoryV1 = (options: Readonly<{
|
|
68
|
+
config: Readonly<Record<string, unknown>>;
|
|
69
|
+
dbPath: string;
|
|
70
|
+
}>) => Promise<QmdStore>;
|
|
71
|
+
|
|
72
|
+
const qmdModuleSpecifier: string = "@tobilu/qmd";
|
|
73
|
+
async function defaultQmdStoreFactory(options: Parameters<QmdStoreFactoryV1>[0]): Promise<QmdStore> {
|
|
74
|
+
let module: unknown;
|
|
75
|
+
try { module = await import(qmdModuleSpecifier); } catch {
|
|
76
|
+
throw new Error("Semantic search needs the optional @tobilu/qmd@2.5.3 package.");
|
|
77
|
+
}
|
|
78
|
+
const createStore = (module as { createStore?: unknown }).createStore;
|
|
79
|
+
if (typeof createStore !== "function") throw new Error("The installed QMD package has no compatible createStore export.");
|
|
80
|
+
return await (createStore as QmdStoreFactoryV1)(options);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
type SemanticManifestV1 = Readonly<{
|
|
84
|
+
entries: Readonly<Record<string, Readonly<{ key: string; recordSha256: Sha256Hex }>>>;
|
|
85
|
+
profileSha256: Sha256Hex;
|
|
86
|
+
v: 1;
|
|
87
|
+
}>;
|
|
88
|
+
|
|
89
|
+
function recordDocument(record: KnowledgeGraphRecordV1): string {
|
|
90
|
+
return `# ${record.key}\n\nkind: ${record.kind}\n\n${canonicalJson(record.value)}\n`;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
const QMD_VECTOR_RESULT_KEYS = [
|
|
94
|
+
"body", "bodyLength", "chunkPos", "collectionName", "context", "displayPath", "docid",
|
|
95
|
+
"filepath", "hash", "modifiedAt", "score", "source", "title",
|
|
96
|
+
] as const;
|
|
97
|
+
type ParsedQmdVectorResult = Readonly<{
|
|
98
|
+
body: string;
|
|
99
|
+
filename: string;
|
|
100
|
+
hash: Sha256Hex;
|
|
101
|
+
score: number;
|
|
102
|
+
title: string;
|
|
103
|
+
}>;
|
|
104
|
+
|
|
105
|
+
function semanticManifest(entries: Record<string, { key: string; recordSha256: Sha256Hex }>): SemanticManifestV1 {
|
|
106
|
+
const immutableEntries: Record<string, Readonly<{ key: string; recordSha256: Sha256Hex }>> = {};
|
|
107
|
+
for (const [filename, entry] of Object.entries(entries)) {
|
|
108
|
+
immutableEntries[filename] = Object.freeze({ ...entry });
|
|
109
|
+
}
|
|
110
|
+
return Object.freeze({
|
|
111
|
+
entries: Object.freeze(immutableEntries),
|
|
112
|
+
profileSha256: canonicalSha256(OH_EMBEDDING_PROFILE_V1),
|
|
113
|
+
v: 1,
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
function parseQmdVectorResult(value: unknown): ParsedQmdVectorResult | null {
|
|
118
|
+
if (!isPlainRecord(value) || !hasExactKeys(value, QMD_VECTOR_RESULT_KEYS)) return null;
|
|
119
|
+
const pathMatch = typeof value.filepath === "string"
|
|
120
|
+
? /^qmd:\/\/oh\/([a-f0-9]{64}\.md)$/u.exec(value.filepath)
|
|
121
|
+
: null;
|
|
122
|
+
const filename = pathMatch?.[1];
|
|
123
|
+
const hash = parseSha256Hex(value.hash);
|
|
124
|
+
const title = safeCode(value.title, 512);
|
|
125
|
+
if (filename === undefined || value.displayPath !== `oh/${filename}` || value.collectionName !== "oh"
|
|
126
|
+
|| value.source !== "vec" || value.context !== null || value.modifiedAt !== ""
|
|
127
|
+
|| hash === null || value.docid !== hash.slice(0, 6) || title === null
|
|
128
|
+
|| typeof value.body !== "string" || typeof value.bodyLength !== "number" || !Number.isSafeInteger(value.bodyLength)
|
|
129
|
+
|| value.bodyLength < 0 || value.bodyLength > OH_GRAPH_LIMITS_V1.recordBytes + 4096
|
|
130
|
+
|| value.body.length !== value.bodyLength || hash !== sha256Hex(value.body) || typeof value.chunkPos !== "number"
|
|
131
|
+
|| !Number.isSafeInteger(value.chunkPos) || value.chunkPos < 0
|
|
132
|
+
|| typeof value.score !== "number" || !Number.isFinite(value.score) || value.score < 0 || value.score > 1) {
|
|
133
|
+
return null;
|
|
134
|
+
}
|
|
135
|
+
return { body: value.body, filename, hash, score: value.score, title };
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
function parseQmdVectorResults(value: unknown): readonly ParsedQmdVectorResult[] {
|
|
139
|
+
if (!Array.isArray(value) || value.length > 100) {
|
|
140
|
+
throw new Error("QMD returned an invalid vector result batch.");
|
|
141
|
+
}
|
|
142
|
+
return value.map(parseQmdVectorResult)
|
|
143
|
+
.filter((result): result is ParsedQmdVectorResult => result !== null);
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/**
|
|
147
|
+
* Optional, rebuildable local QMD index. SQLite records remain authoritative;
|
|
148
|
+
* every returned hit is rejoined to its exact current record digest.
|
|
149
|
+
*/
|
|
150
|
+
export class OhQmdSemanticBackendV1 implements OhSemanticSearchBackendV1 {
|
|
151
|
+
readonly profile = OH_EMBEDDING_PROFILE_V1;
|
|
152
|
+
readonly #cacheDirectory: string;
|
|
153
|
+
readonly #databasePath: string;
|
|
154
|
+
readonly #factory: QmdStoreFactoryV1;
|
|
155
|
+
#manifest: SemanticManifestV1 = semanticManifest({});
|
|
156
|
+
#manifestLoad: Promise<void> | null = null;
|
|
157
|
+
#store: Promise<QmdStore> | null = null;
|
|
158
|
+
#storeClose: Promise<void> | null = null;
|
|
159
|
+
#closure: Promise<void> | null = null;
|
|
160
|
+
#indexQueue: Promise<void> = Promise.resolve();
|
|
161
|
+
readonly #activeSearches = new Set<Promise<unknown>>();
|
|
162
|
+
#closed = false;
|
|
163
|
+
|
|
164
|
+
constructor(options: Readonly<{ cacheDirectory: string; databasePath?: string; storeFactory?: QmdStoreFactoryV1 }>) {
|
|
165
|
+
this.#cacheDirectory = resolve(options.cacheDirectory);
|
|
166
|
+
this.#databasePath = resolve(options.databasePath ?? join(this.#cacheDirectory, "qmd.sqlite"));
|
|
167
|
+
this.#factory = options.storeFactory ?? defaultQmdStoreFactory;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
async #open(): Promise<QmdStore> {
|
|
171
|
+
if (this.#closed) throw new Error("The semantic backend is closed.");
|
|
172
|
+
this.#store ??= this.#initializeStore();
|
|
173
|
+
const store = await this.#store;
|
|
174
|
+
if (this.#closed) {
|
|
175
|
+
await this.#closeStore(store);
|
|
176
|
+
throw new Error("The semantic backend is closed.");
|
|
177
|
+
}
|
|
178
|
+
return store;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
async #initializeStore(): Promise<QmdStore> {
|
|
182
|
+
const documents = join(this.#cacheDirectory, "documents");
|
|
183
|
+
await mkdir(documents, { recursive: true });
|
|
184
|
+
await this.#loadManifest();
|
|
185
|
+
if (this.#closed) throw new Error("The semantic backend is closed.");
|
|
186
|
+
const store = await this.#factory({
|
|
187
|
+
dbPath: this.#databasePath,
|
|
188
|
+
config: {
|
|
189
|
+
collections: { oh: { path: documents, pattern: "*.md" } },
|
|
190
|
+
models: { embed: OH_EMBEDDING_PROFILE_V1.model },
|
|
191
|
+
},
|
|
192
|
+
});
|
|
193
|
+
if (this.#closed) {
|
|
194
|
+
await this.#closeStore(store);
|
|
195
|
+
throw new Error("The semantic backend is closed.");
|
|
196
|
+
}
|
|
197
|
+
return store;
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
#closeStore(store: QmdStore): Promise<void> {
|
|
201
|
+
this.#storeClose ??= Promise.resolve().then(() => store.close());
|
|
202
|
+
return this.#storeClose;
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
#loadManifest(): Promise<void> {
|
|
206
|
+
this.#manifestLoad ??= this.#readManifest();
|
|
207
|
+
return this.#manifestLoad;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
async #readManifest(): Promise<void> {
|
|
211
|
+
let text: string;
|
|
212
|
+
try { text = await readFile(join(this.#cacheDirectory, "manifest.json"), "utf8"); }
|
|
213
|
+
catch (error) {
|
|
214
|
+
if ((error as NodeJS.ErrnoException).code === "ENOENT") return;
|
|
215
|
+
throw error;
|
|
216
|
+
}
|
|
217
|
+
let value: unknown;
|
|
218
|
+
try { value = JSON.parse(text); } catch { throw new Error("The semantic manifest is not JSON."); }
|
|
219
|
+
if (canonicalJson(value) !== text || !isPlainRecord(value)
|
|
220
|
+
|| !hasExactKeys(value, ["entries", "profileSha256", "v"]) || value.v !== 1
|
|
221
|
+
|| value.profileSha256 !== canonicalSha256(OH_EMBEDDING_PROFILE_V1) || !isPlainRecord(value.entries)
|
|
222
|
+
|| Object.keys(value.entries).length > 65_536) throw new Error("The semantic manifest is incompatible or invalid.");
|
|
223
|
+
const entries: Record<string, { key: string; recordSha256: Sha256Hex }> = {};
|
|
224
|
+
for (const [filename, candidate] of Object.entries(value.entries)) {
|
|
225
|
+
if (!/^[a-f0-9]{64}\.md$/u.test(filename) || !isPlainRecord(candidate)
|
|
226
|
+
|| !hasExactKeys(candidate, ["key", "recordSha256"])) throw new Error("The semantic manifest has an invalid entry.");
|
|
227
|
+
const key = safeCode(candidate.key, 512);
|
|
228
|
+
const recordSha256 = parseSha256Hex(candidate.recordSha256);
|
|
229
|
+
if (key === null || recordSha256 === null || filename !== `${sha256Hex(key)}.md`) {
|
|
230
|
+
throw new Error("The semantic manifest entry identity is invalid.");
|
|
231
|
+
}
|
|
232
|
+
entries[filename] = { key, recordSha256 };
|
|
233
|
+
}
|
|
234
|
+
this.#manifest = semanticManifest(entries);
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
index(records: readonly KnowledgeGraphRecordV1[]): Promise<Readonly<{ indexed: number; v: 1 }>> {
|
|
238
|
+
if (records.length > 65_536) {
|
|
239
|
+
return Promise.reject(new RangeError("A semantic snapshot may contain at most 65,536 records."));
|
|
240
|
+
}
|
|
241
|
+
if (this.#closed) return Promise.reject(new Error("The semantic backend is closed."));
|
|
242
|
+
const snapshot = [...records];
|
|
243
|
+
const operation = this.#indexQueue.then(() => this.#indexSnapshot(snapshot));
|
|
244
|
+
this.#indexQueue = operation.then(() => undefined, () => undefined);
|
|
245
|
+
return operation;
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
async #indexSnapshot(records: readonly KnowledgeGraphRecordV1[]): Promise<Readonly<{ indexed: number; v: 1 }>> {
|
|
249
|
+
if (this.#closed) throw new Error("The semantic backend is closed.");
|
|
250
|
+
const documents = join(this.#cacheDirectory, "documents");
|
|
251
|
+
await mkdir(documents, { recursive: true });
|
|
252
|
+
await this.#loadManifest();
|
|
253
|
+
const entries: Record<string, { key: string; recordSha256: Sha256Hex }> = {};
|
|
254
|
+
for (const record of records) {
|
|
255
|
+
const filename = `${sha256Hex(record.key)}.md`;
|
|
256
|
+
entries[filename] = { key: record.key, recordSha256: record.recordSha256 };
|
|
257
|
+
const path = join(documents, filename);
|
|
258
|
+
const temporary = `${path}.${process.pid}.tmp`;
|
|
259
|
+
await writeFile(temporary, recordDocument(record), { encoding: "utf8", mode: 0o600 });
|
|
260
|
+
await rename(temporary, path);
|
|
261
|
+
}
|
|
262
|
+
const retained = new Set(Object.keys(entries));
|
|
263
|
+
for (const filename of await readdir(documents)) {
|
|
264
|
+
if (/^[a-f0-9]{64}\.md$/u.test(filename) && !retained.has(filename)) await unlink(join(documents, filename));
|
|
265
|
+
}
|
|
266
|
+
const nextManifest = semanticManifest(entries);
|
|
267
|
+
const store = await this.#open();
|
|
268
|
+
await store.update({ collections: ["oh"] });
|
|
269
|
+
await store.embed({ collection: "oh", model: OH_EMBEDDING_PROFILE_V1.model });
|
|
270
|
+
const manifestPath = join(this.#cacheDirectory, "manifest.json");
|
|
271
|
+
const temporaryManifest = `${manifestPath}.${process.pid}.tmp`;
|
|
272
|
+
await writeFile(temporaryManifest, canonicalJson(nextManifest), { encoding: "utf8", mode: 0o600 });
|
|
273
|
+
await rename(temporaryManifest, manifestPath);
|
|
274
|
+
this.#manifest = nextManifest;
|
|
275
|
+
return { indexed: records.length, v: 1 };
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
search(query: string, limit: number, authority: OhSqliteStore): Promise<readonly OhSemanticSearchResultV1[]> {
|
|
279
|
+
if (!Number.isSafeInteger(limit) || limit < 1 || limit > 100) {
|
|
280
|
+
return Promise.reject(new RangeError("Semantic limit must be 1 through 100."));
|
|
281
|
+
}
|
|
282
|
+
if (this.#closed) return Promise.reject(new Error("The semantic backend is closed."));
|
|
283
|
+
const operation = this.#searchSnapshot(query, limit, authority);
|
|
284
|
+
this.#activeSearches.add(operation);
|
|
285
|
+
void operation.then(
|
|
286
|
+
() => { this.#activeSearches.delete(operation); },
|
|
287
|
+
() => { this.#activeSearches.delete(operation); },
|
|
288
|
+
);
|
|
289
|
+
return operation;
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
async #searchSnapshot(query: string, limit: number,
|
|
293
|
+
authority: OhSqliteStore): Promise<readonly OhSemanticSearchResultV1[]> {
|
|
294
|
+
const store = await this.#open();
|
|
295
|
+
const manifest = this.#manifest;
|
|
296
|
+
const results = parseQmdVectorResults(await store.searchVector(query,
|
|
297
|
+
{ collection: "oh", limit: Math.min(100, limit * 3) }));
|
|
298
|
+
const output: OhSemanticSearchResultV1[] = [];
|
|
299
|
+
const seen = new Set<string>();
|
|
300
|
+
for (const result of results) {
|
|
301
|
+
const entry = manifest.entries[result.filename];
|
|
302
|
+
if (entry === undefined || result.title !== entry.key || seen.has(entry.key)) continue;
|
|
303
|
+
const current = authority.get(entry.key);
|
|
304
|
+
if (current === null || current.recordSha256 !== entry.recordSha256) continue;
|
|
305
|
+
const expectedDocument = recordDocument(current);
|
|
306
|
+
if (result.body !== expectedDocument || result.hash !== sha256Hex(expectedDocument)) continue;
|
|
307
|
+
seen.add(entry.key);
|
|
308
|
+
output.push({ key: entry.key, recordSha256: entry.recordSha256, score: result.score, v: 1 });
|
|
309
|
+
if (output.length === limit) break;
|
|
310
|
+
}
|
|
311
|
+
return output;
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
async #finishClose(): Promise<void> {
|
|
315
|
+
await this.#indexQueue;
|
|
316
|
+
await Promise.allSettled([...this.#activeSearches]);
|
|
317
|
+
if (this.#store === null) return;
|
|
318
|
+
try {
|
|
319
|
+
const store = await this.#store;
|
|
320
|
+
await this.#closeStore(store);
|
|
321
|
+
} catch {
|
|
322
|
+
// Initialization failures are not close failures. A close initiated by
|
|
323
|
+
// late initialization is shared separately and must still be observed.
|
|
324
|
+
if (this.#storeClose !== null) await this.#storeClose;
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
close(): Promise<void> {
|
|
329
|
+
this.#closed = true;
|
|
330
|
+
this.#closure ??= this.#finishClose();
|
|
331
|
+
return this.#closure;
|
|
332
|
+
}
|
|
333
|
+
}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { existsSync } from "node:fs";
|
|
2
|
+
import { Database } from "bun:sqlite";
|
|
3
|
+
|
|
4
|
+
import { createOhSqliteRuntime } from "./runtime";
|
|
5
|
+
|
|
6
|
+
export type OhSqliteDatabase = Database;
|
|
7
|
+
|
|
8
|
+
const SQLITE_RUNTIME = createOhSqliteRuntime({
|
|
9
|
+
exists: existsSync,
|
|
10
|
+
open: (path, options) => new Database(path, options),
|
|
11
|
+
platform: process.platform,
|
|
12
|
+
setCustomSQLite: (path) => Database.setCustomSQLite(path),
|
|
13
|
+
});
|
|
14
|
+
|
|
15
|
+
export function openOhSqliteDatabase(path: string): OhSqliteDatabase {
|
|
16
|
+
const database = SQLITE_RUNTIME.open(path);
|
|
17
|
+
database.exec("PRAGMA foreign_keys = ON");
|
|
18
|
+
database.exec("PRAGMA journal_mode = WAL");
|
|
19
|
+
database.exec("PRAGMA synchronous = NORMAL");
|
|
20
|
+
database.exec("PRAGMA busy_timeout = 5000");
|
|
21
|
+
database.exec("PRAGMA trusted_schema = OFF");
|
|
22
|
+
return database;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export function withImmediateTransaction<T>(database: OhSqliteDatabase, work: () => T): T {
|
|
26
|
+
database.exec("BEGIN IMMEDIATE");
|
|
27
|
+
try {
|
|
28
|
+
const result = work();
|
|
29
|
+
database.exec("COMMIT");
|
|
30
|
+
return result;
|
|
31
|
+
} catch (error) {
|
|
32
|
+
try { database.exec("ROLLBACK"); } catch { /* preserve the original failure */ }
|
|
33
|
+
throw error;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export function withReadTransaction<T>(database: OhSqliteDatabase, work: () => T): T {
|
|
38
|
+
database.exec("BEGIN");
|
|
39
|
+
try {
|
|
40
|
+
const result = work();
|
|
41
|
+
database.exec("COMMIT");
|
|
42
|
+
return result;
|
|
43
|
+
} catch (error) {
|
|
44
|
+
try { database.exec("ROLLBACK"); } catch { /* preserve the original failure */ }
|
|
45
|
+
throw error;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { describe, expect, test } from "bun:test";
|
|
2
|
+
|
|
3
|
+
import { canonicalNow, sha256Hex } from "../canonical";
|
|
4
|
+
import { openOhSqliteDatabase } from "./driver";
|
|
5
|
+
import { OH_SQLITE_MIGRATIONS, OH_SQLITE_SCHEMA_VERSION } from "./migrations";
|
|
6
|
+
import { OhSqliteStore } from "./store";
|
|
7
|
+
|
|
8
|
+
describe("SQLite schema evolution", () => {
|
|
9
|
+
test("preserves the released 0001 bytes and appends store realms as version 2", () => {
|
|
10
|
+
expect(OH_SQLITE_SCHEMA_VERSION).toBe(2);
|
|
11
|
+
expect(OH_SQLITE_MIGRATIONS.map(({ name, version }) => ({ name, version }))).toEqual([
|
|
12
|
+
{ name: "0001_oh_core", version: 1 },
|
|
13
|
+
{ name: "0002_store_realms", version: 2 },
|
|
14
|
+
]);
|
|
15
|
+
expect(String(sha256Hex(OH_SQLITE_MIGRATIONS[0]?.sql ?? ""))).toBe(
|
|
16
|
+
"f525fc4f521b544d8e00526f2993a3b6bf81ae936950765c51406566ea4b1b7c",
|
|
17
|
+
);
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
test("upgrades an applied version-1 authority without rewriting its migration evidence", () => {
|
|
21
|
+
const database = openOhSqliteDatabase(":memory:");
|
|
22
|
+
const first = OH_SQLITE_MIGRATIONS[0];
|
|
23
|
+
if (first === undefined) throw new Error("Missing first migration.");
|
|
24
|
+
database.exec(`CREATE TABLE oh_migrations (
|
|
25
|
+
version INTEGER PRIMARY KEY,
|
|
26
|
+
name TEXT NOT NULL UNIQUE,
|
|
27
|
+
migration_sha256 TEXT NOT NULL CHECK(length(migration_sha256) = 64),
|
|
28
|
+
applied_at TEXT NOT NULL
|
|
29
|
+
) STRICT`);
|
|
30
|
+
database.exec(first.sql);
|
|
31
|
+
database.query(`INSERT INTO oh_migrations(version, name, migration_sha256, applied_at)
|
|
32
|
+
VALUES (?, ?, ?, ?)`).run(first.version, first.name, sha256Hex(first.sql), canonicalNow());
|
|
33
|
+
|
|
34
|
+
const store = new OhSqliteStore({ database, spaceId: "upgraded" });
|
|
35
|
+
expect(store.contract().sqliteSchemaVersion).toBe(2);
|
|
36
|
+
expect(database.query<{ count: number }, []>(
|
|
37
|
+
"SELECT count(*) AS count FROM oh_migrations",
|
|
38
|
+
).get()?.count).toBe(2);
|
|
39
|
+
expect(database.query<{ count: number }, []>(
|
|
40
|
+
"SELECT count(*) AS count FROM sqlite_schema WHERE name IN ('oh_space_bindings', 'oh_space_purges')",
|
|
41
|
+
).get()?.count).toBe(2);
|
|
42
|
+
store.close();
|
|
43
|
+
});
|
|
44
|
+
});
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
import { canonicalNow, sha256Hex } from "../canonical";
|
|
2
|
+
import type { OhSqliteDatabase } from "./driver";
|
|
3
|
+
|
|
4
|
+
export const OH_SQLITE_SCHEMA_VERSION = 2 as const;
|
|
5
|
+
|
|
6
|
+
export type OhSqliteMigration = Readonly<{ name: string; sql: string; version: number }>;
|
|
7
|
+
|
|
8
|
+
export const OH_SQLITE_MIGRATIONS: readonly OhSqliteMigration[] = Object.freeze([
|
|
9
|
+
Object.freeze({
|
|
10
|
+
name: "0001_oh_core",
|
|
11
|
+
version: 1,
|
|
12
|
+
sql: `
|
|
13
|
+
CREATE TABLE oh_contracts (
|
|
14
|
+
contract_id TEXT PRIMARY KEY,
|
|
15
|
+
contract_sha256 TEXT NOT NULL CHECK(length(contract_sha256) = 64),
|
|
16
|
+
manifest_json TEXT NOT NULL CHECK(json_valid(manifest_json)),
|
|
17
|
+
created_at TEXT NOT NULL
|
|
18
|
+
) STRICT;
|
|
19
|
+
|
|
20
|
+
CREATE TABLE oh_spaces (
|
|
21
|
+
space_id TEXT PRIMARY KEY,
|
|
22
|
+
contract_id TEXT NOT NULL REFERENCES oh_contracts(contract_id),
|
|
23
|
+
generation INTEGER NOT NULL CHECK(generation >= 0),
|
|
24
|
+
head_operation_sha256 TEXT CHECK(head_operation_sha256 IS NULL OR length(head_operation_sha256) = 64),
|
|
25
|
+
graph_revision_sha256 TEXT CHECK(graph_revision_sha256 IS NULL OR length(graph_revision_sha256) = 64),
|
|
26
|
+
records_sha256 TEXT NOT NULL CHECK(length(records_sha256) = 64),
|
|
27
|
+
sequence INTEGER NOT NULL CHECK(sequence >= 0),
|
|
28
|
+
created_at TEXT NOT NULL,
|
|
29
|
+
updated_at TEXT NOT NULL,
|
|
30
|
+
CHECK(generation = sequence),
|
|
31
|
+
CHECK((sequence = 0) = (head_operation_sha256 IS NULL)),
|
|
32
|
+
CHECK((sequence = 0) = (graph_revision_sha256 IS NULL))
|
|
33
|
+
) STRICT;
|
|
34
|
+
|
|
35
|
+
CREATE TABLE oh_operations (
|
|
36
|
+
operation_sha256 TEXT PRIMARY KEY CHECK(length(operation_sha256) = 64),
|
|
37
|
+
space_id TEXT NOT NULL REFERENCES oh_spaces(space_id),
|
|
38
|
+
sequence INTEGER NOT NULL CHECK(sequence > 0),
|
|
39
|
+
operation_id TEXT NOT NULL,
|
|
40
|
+
parent_operation_sha256 TEXT CHECK(parent_operation_sha256 IS NULL OR length(parent_operation_sha256) = 64),
|
|
41
|
+
graph_revision_sha256 TEXT NOT NULL CHECK(length(graph_revision_sha256) = 64),
|
|
42
|
+
records_sha256 TEXT NOT NULL CHECK(length(records_sha256) = 64),
|
|
43
|
+
operation_json TEXT NOT NULL CHECK(json_valid(operation_json)),
|
|
44
|
+
instant TEXT NOT NULL,
|
|
45
|
+
UNIQUE(space_id, sequence),
|
|
46
|
+
UNIQUE(space_id, operation_id)
|
|
47
|
+
) STRICT;
|
|
48
|
+
|
|
49
|
+
CREATE TABLE oh_operation_records (
|
|
50
|
+
operation_sha256 TEXT NOT NULL REFERENCES oh_operations(operation_sha256),
|
|
51
|
+
ordinal INTEGER NOT NULL CHECK(ordinal >= 0),
|
|
52
|
+
record_key TEXT NOT NULL,
|
|
53
|
+
change_kind TEXT NOT NULL CHECK(change_kind IN ('put', 'tombstone')),
|
|
54
|
+
record_sha256 TEXT CHECK(record_sha256 IS NULL OR length(record_sha256) = 64),
|
|
55
|
+
PRIMARY KEY(operation_sha256, ordinal),
|
|
56
|
+
UNIQUE(operation_sha256, record_key)
|
|
57
|
+
) STRICT;
|
|
58
|
+
|
|
59
|
+
CREATE TABLE oh_records (
|
|
60
|
+
space_id TEXT NOT NULL REFERENCES oh_spaces(space_id),
|
|
61
|
+
record_key TEXT NOT NULL,
|
|
62
|
+
kind TEXT NOT NULL,
|
|
63
|
+
record_sha256 TEXT NOT NULL CHECK(length(record_sha256) = 64),
|
|
64
|
+
record_json TEXT NOT NULL CHECK(json_valid(record_json)),
|
|
65
|
+
operation_sha256 TEXT NOT NULL REFERENCES oh_operations(operation_sha256),
|
|
66
|
+
sequence INTEGER NOT NULL CHECK(sequence > 0),
|
|
67
|
+
PRIMARY KEY(space_id, record_key)
|
|
68
|
+
) STRICT;
|
|
69
|
+
|
|
70
|
+
CREATE TABLE oh_dependencies (
|
|
71
|
+
space_id TEXT NOT NULL,
|
|
72
|
+
record_key TEXT NOT NULL,
|
|
73
|
+
dependency_key TEXT NOT NULL,
|
|
74
|
+
PRIMARY KEY(space_id, record_key, dependency_key),
|
|
75
|
+
FOREIGN KEY(space_id, record_key) REFERENCES oh_records(space_id, record_key) ON DELETE CASCADE,
|
|
76
|
+
FOREIGN KEY(space_id, dependency_key) REFERENCES oh_records(space_id, record_key)
|
|
77
|
+
) STRICT;
|
|
78
|
+
|
|
79
|
+
CREATE TABLE oh_sync_outbox (
|
|
80
|
+
space_id TEXT NOT NULL REFERENCES oh_spaces(space_id),
|
|
81
|
+
sequence INTEGER NOT NULL,
|
|
82
|
+
operation_sha256 TEXT NOT NULL REFERENCES oh_operations(operation_sha256),
|
|
83
|
+
PRIMARY KEY(space_id, sequence),
|
|
84
|
+
UNIQUE(operation_sha256)
|
|
85
|
+
) STRICT;
|
|
86
|
+
|
|
87
|
+
CREATE TABLE oh_sync_state (
|
|
88
|
+
remote_id TEXT NOT NULL,
|
|
89
|
+
space_id TEXT NOT NULL REFERENCES oh_spaces(space_id),
|
|
90
|
+
pulled_sequence INTEGER NOT NULL CHECK(pulled_sequence >= 0),
|
|
91
|
+
pushed_sequence INTEGER NOT NULL CHECK(pushed_sequence >= 0),
|
|
92
|
+
remote_head_sha256 TEXT CHECK(remote_head_sha256 IS NULL OR length(remote_head_sha256) = 64),
|
|
93
|
+
updated_at TEXT NOT NULL,
|
|
94
|
+
PRIMARY KEY(remote_id, space_id)
|
|
95
|
+
) STRICT;
|
|
96
|
+
|
|
97
|
+
CREATE TABLE oh_search_documents (
|
|
98
|
+
space_id TEXT NOT NULL,
|
|
99
|
+
record_key TEXT NOT NULL,
|
|
100
|
+
record_sha256 TEXT NOT NULL CHECK(length(record_sha256) = 64),
|
|
101
|
+
text TEXT NOT NULL,
|
|
102
|
+
PRIMARY KEY(space_id, record_key),
|
|
103
|
+
FOREIGN KEY(space_id, record_key) REFERENCES oh_records(space_id, record_key) ON DELETE CASCADE
|
|
104
|
+
) STRICT;
|
|
105
|
+
|
|
106
|
+
CREATE VIRTUAL TABLE oh_search_fts USING fts5(
|
|
107
|
+
space_id UNINDEXED,
|
|
108
|
+
record_key UNINDEXED,
|
|
109
|
+
text,
|
|
110
|
+
tokenize='unicode61 remove_diacritics 2'
|
|
111
|
+
);
|
|
112
|
+
|
|
113
|
+
CREATE INDEX oh_operations_space_sequence ON oh_operations(space_id, sequence);
|
|
114
|
+
CREATE INDEX oh_records_space_kind ON oh_records(space_id, kind, record_key);
|
|
115
|
+
CREATE INDEX oh_dependencies_dependency ON oh_dependencies(space_id, dependency_key);
|
|
116
|
+
`,
|
|
117
|
+
}),
|
|
118
|
+
Object.freeze({
|
|
119
|
+
name: "0002_store_realms",
|
|
120
|
+
version: 2,
|
|
121
|
+
sql: `
|
|
122
|
+
CREATE TABLE oh_space_bindings (
|
|
123
|
+
space_id TEXT PRIMARY KEY REFERENCES oh_spaces(space_id),
|
|
124
|
+
realm_id TEXT NOT NULL,
|
|
125
|
+
profile_id TEXT NOT NULL,
|
|
126
|
+
profile_kind TEXT NOT NULL CHECK(profile_kind IN ('canonical', 'working')),
|
|
127
|
+
profile_sha256 TEXT NOT NULL CHECK(length(profile_sha256) = 64),
|
|
128
|
+
binding_sha256 TEXT NOT NULL UNIQUE CHECK(length(binding_sha256) = 64),
|
|
129
|
+
binding_json TEXT NOT NULL CHECK(json_valid(binding_json)),
|
|
130
|
+
created_at TEXT NOT NULL
|
|
131
|
+
) STRICT;
|
|
132
|
+
|
|
133
|
+
CREATE TABLE oh_space_purges (
|
|
134
|
+
space_id TEXT PRIMARY KEY,
|
|
135
|
+
binding_sha256 TEXT NOT NULL CHECK(length(binding_sha256) = 64),
|
|
136
|
+
prior_operation_sha256 TEXT CHECK(prior_operation_sha256 IS NULL OR length(prior_operation_sha256) = 64),
|
|
137
|
+
prior_sequence INTEGER NOT NULL CHECK(prior_sequence >= 0),
|
|
138
|
+
purged_at TEXT NOT NULL,
|
|
139
|
+
receipt_sha256 TEXT NOT NULL UNIQUE CHECK(length(receipt_sha256) = 64),
|
|
140
|
+
receipt_json TEXT NOT NULL CHECK(json_valid(receipt_json))
|
|
141
|
+
) STRICT;
|
|
142
|
+
`,
|
|
143
|
+
}),
|
|
144
|
+
]);
|
|
145
|
+
|
|
146
|
+
export function applyOhSqliteMigrations(database: OhSqliteDatabase): void {
|
|
147
|
+
database.exec(`CREATE TABLE IF NOT EXISTS oh_migrations (
|
|
148
|
+
version INTEGER PRIMARY KEY,
|
|
149
|
+
name TEXT NOT NULL UNIQUE,
|
|
150
|
+
migration_sha256 TEXT NOT NULL CHECK(length(migration_sha256) = 64),
|
|
151
|
+
applied_at TEXT NOT NULL
|
|
152
|
+
) STRICT`);
|
|
153
|
+
const select = database.query<{ migration_sha256: string; name: string }, [number]>(
|
|
154
|
+
"SELECT name, migration_sha256 FROM oh_migrations WHERE version = ?",
|
|
155
|
+
);
|
|
156
|
+
const insert = database.query(
|
|
157
|
+
"INSERT INTO oh_migrations(version, name, migration_sha256, applied_at) VALUES (?, ?, ?, ?)",
|
|
158
|
+
);
|
|
159
|
+
for (const migration of OH_SQLITE_MIGRATIONS) {
|
|
160
|
+
const digest = sha256Hex(migration.sql);
|
|
161
|
+
database.exec("BEGIN IMMEDIATE");
|
|
162
|
+
try {
|
|
163
|
+
const existing = select.get(migration.version);
|
|
164
|
+
if (existing !== null) {
|
|
165
|
+
if (existing.name !== migration.name || existing.migration_sha256 !== digest) {
|
|
166
|
+
throw new Error(`SQLite migration ${migration.version} does not match the applied migration.`);
|
|
167
|
+
}
|
|
168
|
+
} else {
|
|
169
|
+
database.exec(migration.sql);
|
|
170
|
+
insert.run(migration.version, migration.name, digest, canonicalNow());
|
|
171
|
+
}
|
|
172
|
+
database.exec("COMMIT");
|
|
173
|
+
} catch (error) {
|
|
174
|
+
try { database.exec("ROLLBACK"); } catch { /* preserve the migration error */ }
|
|
175
|
+
throw error;
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
}
|