@hraness/oh 0.4.1 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +3 -0
- package/README.md +4 -1
- package/dist/cli.d.ts +1 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +15590 -1705
- package/dist/index.js +15209 -261
- package/dist/libsql-model.d.ts +75 -0
- package/dist/libsql-model.d.ts.map +1 -0
- package/dist/libsql-platform.d.ts +24 -0
- package/dist/libsql-platform.d.ts.map +1 -0
- package/dist/libsql-program.d.ts +64 -0
- package/dist/libsql-program.d.ts.map +1 -0
- package/dist/libsql-runtime.d.ts +13 -0
- package/dist/libsql-runtime.d.ts.map +1 -0
- package/dist/libsql.d.ts +4 -36
- package/dist/libsql.d.ts.map +1 -1
- package/dist/libsql.js +16583 -1426
- package/dist/memory-authority-platform.d.ts +210 -0
- package/dist/memory-authority-platform.d.ts.map +1 -0
- package/dist/memory-authority-program.d.ts +15 -0
- package/dist/memory-authority-program.d.ts.map +1 -0
- package/dist/memory-authority-runtime.d.ts +4 -0
- package/dist/memory-authority-runtime.d.ts.map +1 -0
- package/dist/memory-core.d.ts +522 -0
- package/dist/memory-core.d.ts.map +1 -0
- package/dist/memory.d.ts +5 -445
- package/dist/memory.d.ts.map +1 -1
- package/dist/memory.js +18232 -4143
- package/dist/sdk.js +15266 -318
- package/dist/semantic-cloud.js +1 -260
- package/dist/semantic-model.d.ts +75 -0
- package/dist/semantic-model.d.ts.map +1 -0
- package/dist/semantic-platform.d.ts +32 -0
- package/dist/semantic-platform.d.ts.map +1 -0
- package/dist/semantic-program.d.ts +18 -0
- package/dist/semantic-program.d.ts.map +1 -0
- package/dist/semantic-runtime.d.ts +6 -0
- package/dist/semantic-runtime.d.ts.map +1 -0
- package/dist/semantic.d.ts +6 -57
- package/dist/semantic.d.ts.map +1 -1
- package/dist/semantic.js +15541 -514
- package/dist/sqlite/index.js +1148 -1378
- package/dist/sqlite/port.d.ts +1 -1
- package/dist/sqlite/port.d.ts.map +1 -1
- package/dist/sync-libsql-program.d.ts +5 -0
- package/dist/sync-libsql-program.d.ts.map +1 -0
- package/dist/sync-model.d.ts +60 -0
- package/dist/sync-model.d.ts.map +1 -0
- package/dist/sync-platform.d.ts +41 -0
- package/dist/sync-platform.d.ts.map +1 -0
- package/dist/sync-program.d.ts +11 -0
- package/dist/sync-program.d.ts.map +1 -0
- package/dist/sync-runtime.d.ts +6 -0
- package/dist/sync-runtime.d.ts.map +1 -0
- package/dist/sync.d.ts +5 -55
- package/dist/sync.d.ts.map +1 -1
- package/dist/sync.js +15174 -226
- package/package.json +7 -5
- package/skills/oh/SKILL.md +2 -1
- package/spec/v1/store.md +14 -0
- package/src/cli.test.ts +35 -0
- package/src/cli.ts +4 -3
- package/src/cloudflare-embedding.ts +1 -1
- package/src/libsql-lifecycle.test.ts +76 -0
- package/src/libsql-model.ts +350 -0
- package/src/libsql-platform.ts +51 -0
- package/src/libsql-program.ts +1583 -0
- package/src/libsql-runtime.ts +89 -0
- package/src/libsql-semantic-v2.ts +1 -1
- package/src/libsql-semantic.ts +1 -1
- package/src/libsql.test.ts +245 -0
- package/src/libsql.ts +26 -1698
- package/src/memory-authority-platform.ts +133 -0
- package/src/memory-authority-program.ts +309 -0
- package/src/memory-authority-runtime.ts +40 -0
- package/src/memory-core.ts +2401 -0
- package/src/memory.test.ts +55 -0
- package/src/memory.ts +68 -2759
- package/src/semantic-model.ts +142 -0
- package/src/semantic-platform.ts +130 -0
- package/src/semantic-program.ts +113 -0
- package/src/semantic-runtime.ts +53 -0
- package/src/semantic.test.ts +115 -1
- package/src/semantic.ts +14 -324
- package/src/sqlite/port.ts +1 -1
- package/src/sync-libsql-program.ts +170 -0
- package/src/sync-lifecycle.test.ts +110 -0
- package/src/sync-model.ts +530 -0
- package/src/sync-platform.ts +87 -0
- package/src/sync-program.ts +117 -0
- package/src/sync-runtime.ts +33 -0
- package/src/sync.ts +10 -765
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
import { canonicalJson, canonicalSha256, hasExactKeys, isPlainRecord, parseSha256Hex,
|
|
2
|
+
safeCode, sha256Hex, type Sha256Hex } from "./canonical";
|
|
3
|
+
import { OH_GRAPH_LIMITS_V1, type KnowledgeGraphRecordV1 } from "./graph";
|
|
4
|
+
import type { OhSqliteStore } from "./sqlite/store";
|
|
5
|
+
|
|
6
|
+
export const OH_EMBEDDING_PROFILE_V1 = Object.freeze({
|
|
7
|
+
dimensions: 768,
|
|
8
|
+
distance: "cosine",
|
|
9
|
+
documentation: "https://ai.google.dev/gemma/docs/embeddinggemma",
|
|
10
|
+
documentFormat: "title: {title} | text: {content}",
|
|
11
|
+
engine: "@tobilu/qmd@2.5.3",
|
|
12
|
+
model: "hf:ggml-org/embeddinggemma-300M-GGUF/embeddinggemma-300M-Q8_0.gguf",
|
|
13
|
+
normalization: "l2",
|
|
14
|
+
queryFormat: "task: search result | query: {query}",
|
|
15
|
+
v: 1,
|
|
16
|
+
} as const);
|
|
17
|
+
|
|
18
|
+
export type OhEmbeddingProfileV1 = typeof OH_EMBEDDING_PROFILE_V1;
|
|
19
|
+
|
|
20
|
+
export function formatOhEmbeddingQueryV1(query: string): string {
|
|
21
|
+
return `task: search result | query: ${query}`;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export function formatOhEmbeddingDocumentV1(title: string, content: string): string {
|
|
25
|
+
return `title: ${title} | text: ${content}`;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export function normalizeOhEmbeddingV1(vector: readonly number[]): readonly number[] {
|
|
29
|
+
if (vector.length !== OH_EMBEDDING_PROFILE_V1.dimensions
|
|
30
|
+
|| vector.some((component) => !Number.isFinite(component))) {
|
|
31
|
+
throw new TypeError(`Embedding vectors must contain ${OH_EMBEDDING_PROFILE_V1.dimensions} finite values.`);
|
|
32
|
+
}
|
|
33
|
+
const scale = vector.reduce((maximum, component) => Math.max(maximum, Math.abs(component)), 0);
|
|
34
|
+
if (scale === 0) throw new TypeError("Embedding vectors must have nonzero magnitude.");
|
|
35
|
+
const scaledMagnitude = Math.sqrt(vector.reduce((sum, component) => {
|
|
36
|
+
const scaled = component / scale;
|
|
37
|
+
return sum + scaled * scaled;
|
|
38
|
+
}, 0));
|
|
39
|
+
if (!Number.isFinite(scaledMagnitude) || scaledMagnitude === 0) {
|
|
40
|
+
throw new TypeError("Embedding vectors must have finite nonzero magnitude.");
|
|
41
|
+
}
|
|
42
|
+
const normalized = vector.map((component) => (component / scale) / scaledMagnitude);
|
|
43
|
+
if (normalized.some((component) => !Number.isFinite(component))) {
|
|
44
|
+
throw new TypeError("Embedding vectors must normalize to finite values.");
|
|
45
|
+
}
|
|
46
|
+
return normalized;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export function cosineSimilarityV1(left: readonly number[], right: readonly number[]): number {
|
|
50
|
+
const normalizedLeft = normalizeOhEmbeddingV1(left);
|
|
51
|
+
const normalizedRight = normalizeOhEmbeddingV1(right);
|
|
52
|
+
return normalizedLeft.reduce((sum, component, index) => sum + component * (normalizedRight[index] as number), 0);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export type OhSemanticSearchResultV1 = Readonly<{
|
|
56
|
+
key: string;
|
|
57
|
+
recordSha256: Sha256Hex;
|
|
58
|
+
score: number;
|
|
59
|
+
v: 1;
|
|
60
|
+
}>;
|
|
61
|
+
|
|
62
|
+
export interface OhSemanticSearchBackendV1 {
|
|
63
|
+
readonly profile: OhEmbeddingProfileV1;
|
|
64
|
+
close(): Promise<void>;
|
|
65
|
+
index(records: readonly KnowledgeGraphRecordV1[]): Promise<Readonly<{ indexed: number; v: 1 }>>;
|
|
66
|
+
search(query: string, limit: number, authority: OhSqliteStore): Promise<readonly OhSemanticSearchResultV1[]>;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export type QmdStore = {
|
|
70
|
+
close(): Promise<void>;
|
|
71
|
+
embed(options: Readonly<{ collection: string; model: string }>): Promise<unknown>;
|
|
72
|
+
searchVector(query: string, options: Readonly<{ collection: string; limit: number }>): Promise<unknown>;
|
|
73
|
+
update(options: Readonly<{ collections: readonly string[] }>): Promise<unknown>;
|
|
74
|
+
};
|
|
75
|
+
export type QmdStoreFactoryV1 = (options: Readonly<{
|
|
76
|
+
config: Readonly<Record<string, unknown>>;
|
|
77
|
+
dbPath: string;
|
|
78
|
+
}>) => Promise<QmdStore>;
|
|
79
|
+
|
|
80
|
+
export type SemanticManifestV1 = Readonly<{
|
|
81
|
+
entries: Readonly<Record<string, Readonly<{ key: string; recordSha256: Sha256Hex }>>>;
|
|
82
|
+
profileSha256: Sha256Hex;
|
|
83
|
+
v: 1;
|
|
84
|
+
}>;
|
|
85
|
+
|
|
86
|
+
export function recordDocument(record: KnowledgeGraphRecordV1): string {
|
|
87
|
+
return `# ${record.key}\n\nkind: ${record.kind}\n\n${canonicalJson(record.value)}\n`;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
const QMD_VECTOR_RESULT_KEYS = [
|
|
91
|
+
"body", "bodyLength", "chunkPos", "collectionName", "context", "displayPath", "docid",
|
|
92
|
+
"filepath", "hash", "modifiedAt", "score", "source", "title",
|
|
93
|
+
] as const;
|
|
94
|
+
type ParsedQmdVectorResult = Readonly<{
|
|
95
|
+
body: string;
|
|
96
|
+
filename: string;
|
|
97
|
+
hash: Sha256Hex;
|
|
98
|
+
score: number;
|
|
99
|
+
title: string;
|
|
100
|
+
}>;
|
|
101
|
+
|
|
102
|
+
export function semanticManifest(entries: Record<string, { key: string; recordSha256: Sha256Hex }>): SemanticManifestV1 {
|
|
103
|
+
const immutableEntries: Record<string, Readonly<{ key: string; recordSha256: Sha256Hex }>> = {};
|
|
104
|
+
for (const [filename, entry] of Object.entries(entries)) {
|
|
105
|
+
immutableEntries[filename] = Object.freeze({ ...entry });
|
|
106
|
+
}
|
|
107
|
+
return Object.freeze({
|
|
108
|
+
entries: Object.freeze(immutableEntries),
|
|
109
|
+
profileSha256: canonicalSha256(OH_EMBEDDING_PROFILE_V1),
|
|
110
|
+
v: 1,
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
function parseQmdVectorResult(value: unknown): ParsedQmdVectorResult | null {
|
|
115
|
+
if (!isPlainRecord(value) || !hasExactKeys(value, QMD_VECTOR_RESULT_KEYS)) return null;
|
|
116
|
+
const pathMatch = typeof value.filepath === "string"
|
|
117
|
+
? /^qmd:\/\/oh\/([a-f0-9]{64}\.md)$/u.exec(value.filepath)
|
|
118
|
+
: null;
|
|
119
|
+
const filename = pathMatch?.[1];
|
|
120
|
+
const hash = parseSha256Hex(value.hash);
|
|
121
|
+
const title = safeCode(value.title, 512);
|
|
122
|
+
if (filename === undefined || value.displayPath !== `oh/${filename}` || value.collectionName !== "oh"
|
|
123
|
+
|| value.source !== "vec" || value.context !== null || value.modifiedAt !== ""
|
|
124
|
+
|| hash === null || value.docid !== hash.slice(0, 6) || title === null
|
|
125
|
+
|| typeof value.body !== "string" || typeof value.bodyLength !== "number" || !Number.isSafeInteger(value.bodyLength)
|
|
126
|
+
|| value.bodyLength < 0 || value.bodyLength > OH_GRAPH_LIMITS_V1.recordBytes + 4096
|
|
127
|
+
|| value.body.length !== value.bodyLength || hash !== sha256Hex(value.body) || typeof value.chunkPos !== "number"
|
|
128
|
+
|| !Number.isSafeInteger(value.chunkPos) || value.chunkPos < 0
|
|
129
|
+
|| typeof value.score !== "number" || !Number.isFinite(value.score) || value.score < 0 || value.score > 1) {
|
|
130
|
+
return null;
|
|
131
|
+
}
|
|
132
|
+
return { body: value.body, filename, hash, score: value.score, title };
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
export function parseQmdVectorResults(value: unknown): readonly ParsedQmdVectorResult[] {
|
|
136
|
+
if (!Array.isArray(value) || value.length > 100) {
|
|
137
|
+
throw new Error("QMD returned an invalid vector result batch.");
|
|
138
|
+
}
|
|
139
|
+
return value.map(parseQmdVectorResult)
|
|
140
|
+
.filter((result): result is ParsedQmdVectorResult => result !== null);
|
|
141
|
+
}
|
|
142
|
+
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
import { mkdir, readFile, readdir, rename, unlink, writeFile } from "node:fs/promises";
|
|
2
|
+
import { join, resolve } from "node:path";
|
|
3
|
+
import { Context, Effect, Layer } from "effect";
|
|
4
|
+
|
|
5
|
+
import { canonicalJson, canonicalSha256, hasExactKeys, isPlainRecord, parseSha256Hex,
|
|
6
|
+
safeCode, sha256Hex, type Sha256Hex } from "./canonical";
|
|
7
|
+
import type { KnowledgeGraphRecordV1 } from "./graph";
|
|
8
|
+
import type { OhSqliteStore } from "./sqlite/store";
|
|
9
|
+
import { OH_EMBEDDING_PROFILE_V1, recordDocument, semanticManifest,
|
|
10
|
+
type QmdStore, type QmdStoreFactoryV1, type SemanticManifestV1 } from "./semantic-model";
|
|
11
|
+
|
|
12
|
+
export type SemanticFailure = Readonly<{
|
|
13
|
+
_tag: "FilesystemFailure" | "BackendFailure" | "ManifestFailure" | "ResultFailure" | "AuthorityFailure" | "Closed";
|
|
14
|
+
// Retained only for the existing Promise boundary; never log or serialize foreign causes.
|
|
15
|
+
cause: unknown;
|
|
16
|
+
}>;
|
|
17
|
+
|
|
18
|
+
export type SemanticOptions = Readonly<{
|
|
19
|
+
cacheDirectory: string;
|
|
20
|
+
databasePath?: string;
|
|
21
|
+
storeFactory?: QmdStoreFactoryV1;
|
|
22
|
+
}>;
|
|
23
|
+
|
|
24
|
+
export interface SemanticPlatformService {
|
|
25
|
+
readonly prepare: Effect.Effect<void, SemanticFailure>;
|
|
26
|
+
readonly loadManifest: Effect.Effect<SemanticManifestV1, SemanticFailure>;
|
|
27
|
+
readonly open: Effect.Effect<QmdStore, SemanticFailure>;
|
|
28
|
+
close(store: QmdStore): Effect.Effect<void, SemanticFailure>;
|
|
29
|
+
update(store: QmdStore): Effect.Effect<unknown, SemanticFailure>;
|
|
30
|
+
embed(store: QmdStore): Effect.Effect<unknown, SemanticFailure>;
|
|
31
|
+
search(store: QmdStore, query: string, limit: number): Effect.Effect<unknown, SemanticFailure>;
|
|
32
|
+
writeDocuments(records: readonly KnowledgeGraphRecordV1[]): Effect.Effect<SemanticManifestV1, SemanticFailure>;
|
|
33
|
+
publishManifest(manifest: SemanticManifestV1): Effect.Effect<void, SemanticFailure>;
|
|
34
|
+
readAuthority(authority: OhSqliteStore, key: string): Effect.Effect<KnowledgeGraphRecordV1 | null, SemanticFailure>;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export class SemanticPlatform extends Context.Tag("@hraness/oh/SemanticPlatform")<
|
|
38
|
+
SemanticPlatform, SemanticPlatformService
|
|
39
|
+
>() {}
|
|
40
|
+
|
|
41
|
+
function foreign<A>(tag: SemanticFailure["_tag"], operation: () => Promise<A>): Effect.Effect<A, SemanticFailure> {
|
|
42
|
+
return Effect.tryPromise({ try: operation, catch: (cause) => ({ _tag: tag, cause }) });
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
const qmdModuleSpecifier: string = "@tobilu/qmd";
|
|
46
|
+
async function defaultQmdStoreFactory(options: Parameters<QmdStoreFactoryV1>[0]): Promise<QmdStore> {
|
|
47
|
+
let module: unknown;
|
|
48
|
+
try { module = await import(qmdModuleSpecifier); } catch {
|
|
49
|
+
throw new Error("Semantic search needs the optional @tobilu/qmd@2.5.3 package.");
|
|
50
|
+
}
|
|
51
|
+
const createStore = (module as { createStore?: unknown }).createStore;
|
|
52
|
+
if (typeof createStore !== "function") throw new Error("The installed QMD package has no compatible createStore export.");
|
|
53
|
+
return await (createStore as QmdStoreFactoryV1)(options);
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function parseManifest(text: string): SemanticManifestV1 {
|
|
57
|
+
let value: unknown;
|
|
58
|
+
try { value = JSON.parse(text); } catch { throw new Error("The semantic manifest is not JSON."); }
|
|
59
|
+
if (canonicalJson(value) !== text || !isPlainRecord(value)
|
|
60
|
+
|| !hasExactKeys(value, ["entries", "profileSha256", "v"]) || value.v !== 1
|
|
61
|
+
|| value.profileSha256 !== canonicalSha256(OH_EMBEDDING_PROFILE_V1) || !isPlainRecord(value.entries)
|
|
62
|
+
|| Object.keys(value.entries).length > 65_536) throw new Error("The semantic manifest is incompatible or invalid.");
|
|
63
|
+
const entries: Record<string, { key: string; recordSha256: Sha256Hex }> = {};
|
|
64
|
+
for (const [filename, candidate] of Object.entries(value.entries)) {
|
|
65
|
+
if (!/^[a-f0-9]{64}\.md$/u.test(filename) || !isPlainRecord(candidate)
|
|
66
|
+
|| !hasExactKeys(candidate, ["key", "recordSha256"])) throw new Error("The semantic manifest has an invalid entry.");
|
|
67
|
+
const key = safeCode(candidate.key, 512);
|
|
68
|
+
const recordSha256 = parseSha256Hex(candidate.recordSha256);
|
|
69
|
+
if (key === null || recordSha256 === null || filename !== `${sha256Hex(key)}.md`) {
|
|
70
|
+
throw new Error("The semantic manifest entry identity is invalid.");
|
|
71
|
+
}
|
|
72
|
+
entries[filename] = { key, recordSha256 };
|
|
73
|
+
}
|
|
74
|
+
return semanticManifest(entries);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** Foreign filesystem and optional QMD calls are confined to this adapter. */
|
|
78
|
+
export function semanticPlatformLive(options: SemanticOptions): Layer.Layer<SemanticPlatform> {
|
|
79
|
+
const cacheDirectory = resolve(options.cacheDirectory);
|
|
80
|
+
const databasePath = resolve(options.databasePath ?? join(cacheDirectory, "qmd.sqlite"));
|
|
81
|
+
const documents = join(cacheDirectory, "documents");
|
|
82
|
+
const factory = options.storeFactory ?? defaultQmdStoreFactory;
|
|
83
|
+
return Layer.succeed(SemanticPlatform, {
|
|
84
|
+
prepare: foreign("FilesystemFailure", () => mkdir(documents, { recursive: true })).pipe(Effect.asVoid),
|
|
85
|
+
loadManifest: foreign("FilesystemFailure", async () => {
|
|
86
|
+
try { return await readFile(join(cacheDirectory, "manifest.json"), "utf8"); }
|
|
87
|
+
catch (error) {
|
|
88
|
+
if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
|
|
89
|
+
throw error;
|
|
90
|
+
}
|
|
91
|
+
}).pipe(Effect.flatMap((text) => Effect.try({
|
|
92
|
+
try: () => text === null ? semanticManifest({}) : parseManifest(text),
|
|
93
|
+
catch: (cause): SemanticFailure => ({ _tag: "ManifestFailure", cause }),
|
|
94
|
+
}))),
|
|
95
|
+
open: foreign("BackendFailure", () => factory({ dbPath: databasePath, config: {
|
|
96
|
+
collections: { oh: { path: documents, pattern: "*.md" } },
|
|
97
|
+
models: { embed: OH_EMBEDDING_PROFILE_V1.model },
|
|
98
|
+
} })),
|
|
99
|
+
close: (store) => foreign("BackendFailure", () => store.close()),
|
|
100
|
+
update: (store) => foreign("BackendFailure", () => store.update({ collections: ["oh"] })),
|
|
101
|
+
embed: (store) => foreign("BackendFailure", () => store.embed({ collection: "oh", model: OH_EMBEDDING_PROFILE_V1.model })),
|
|
102
|
+
search: (store, query, limit) => foreign("BackendFailure", () => store.searchVector(query, { collection: "oh", limit })),
|
|
103
|
+
writeDocuments: (records) => foreign("FilesystemFailure", async () => {
|
|
104
|
+
const entries: Record<string, { key: string; recordSha256: Sha256Hex }> = {};
|
|
105
|
+
for (const record of records) {
|
|
106
|
+
const filename = `${sha256Hex(record.key)}.md`;
|
|
107
|
+
entries[filename] = { key: record.key, recordSha256: record.recordSha256 };
|
|
108
|
+
const path = join(documents, filename);
|
|
109
|
+
const temporary = `${path}.${process.pid}.tmp`;
|
|
110
|
+
await writeFile(temporary, recordDocument(record), { encoding: "utf8", mode: 0o600 });
|
|
111
|
+
await rename(temporary, path);
|
|
112
|
+
}
|
|
113
|
+
const retained = new Set(Object.keys(entries));
|
|
114
|
+
for (const filename of await readdir(documents)) {
|
|
115
|
+
if (/^[a-f0-9]{64}\.md$/u.test(filename) && !retained.has(filename)) await unlink(join(documents, filename));
|
|
116
|
+
}
|
|
117
|
+
return semanticManifest(entries);
|
|
118
|
+
}),
|
|
119
|
+
publishManifest: (manifest) => foreign("FilesystemFailure", async () => {
|
|
120
|
+
const path = join(cacheDirectory, "manifest.json");
|
|
121
|
+
const temporary = `${path}.${process.pid}.tmp`;
|
|
122
|
+
await writeFile(temporary, canonicalJson(manifest), { encoding: "utf8", mode: 0o600 });
|
|
123
|
+
await rename(temporary, path);
|
|
124
|
+
}),
|
|
125
|
+
readAuthority: (authority, key) => Effect.try({
|
|
126
|
+
try: () => authority.get(key),
|
|
127
|
+
catch: (cause): SemanticFailure => ({ _tag: "AuthorityFailure", cause }),
|
|
128
|
+
}),
|
|
129
|
+
});
|
|
130
|
+
}
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
import { Effect, Exit, FiberSet, Ref, Scope } from "effect";
|
|
2
|
+
|
|
3
|
+
import { sha256Hex } from "./canonical";
|
|
4
|
+
import type { KnowledgeGraphRecordV1 } from "./graph";
|
|
5
|
+
import { parseQmdVectorResults, recordDocument, semanticManifest,
|
|
6
|
+
type OhSemanticSearchResultV1, type SemanticManifestV1 } from "./semantic-model";
|
|
7
|
+
import { SemanticPlatform, type SemanticFailure } from "./semantic-platform";
|
|
8
|
+
import type { OhSqliteStore } from "./sqlite/store";
|
|
9
|
+
|
|
10
|
+
export interface SemanticLifecycle {
|
|
11
|
+
readonly closed: Ref.Ref<boolean>;
|
|
12
|
+
readonly operations: FiberSet.FiberSet<Exit.Exit<unknown, SemanticFailure>, never>;
|
|
13
|
+
readonly close: Effect.Effect<void, SemanticFailure>;
|
|
14
|
+
index(records: readonly KnowledgeGraphRecordV1[]): Effect.Effect<Readonly<{ indexed: number; v: 1 }>, SemanticFailure>;
|
|
15
|
+
search(query: string, limit: number, authority: OhSqliteStore): Effect.Effect<readonly OhSemanticSearchResultV1[], SemanticFailure>;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/** One owner scope holds the optional store until all admitted operations settle. */
|
|
19
|
+
export const makeSemanticLifecycle: Effect.Effect<SemanticLifecycle, never, SemanticPlatform> = Effect.gen(function* () {
|
|
20
|
+
const platform = yield* SemanticPlatform;
|
|
21
|
+
const owner = yield* Scope.make();
|
|
22
|
+
const operations = yield* Scope.extend(FiberSet.make<Exit.Exit<unknown, SemanticFailure>, never>(), owner);
|
|
23
|
+
const closed = yield* Ref.make(false);
|
|
24
|
+
const manifest = yield* Ref.make<SemanticManifestV1>(semanticManifest({}));
|
|
25
|
+
const releaseResult = yield* Ref.make<Exit.Exit<void, SemanticFailure>>(Exit.void);
|
|
26
|
+
const indexing = yield* Effect.makeSemaphore(1);
|
|
27
|
+
|
|
28
|
+
const ensureOpen: Effect.Effect<void, SemanticFailure> = Effect.gen(function* () {
|
|
29
|
+
if (yield* Ref.get(closed)) {
|
|
30
|
+
return yield* Effect.fail({ _tag: "Closed", cause: new Error("The semantic backend is closed.") } satisfies SemanticFailure);
|
|
31
|
+
}
|
|
32
|
+
});
|
|
33
|
+
const loadManifest = yield* Effect.cached(platform.loadManifest.pipe(
|
|
34
|
+
Effect.flatMap((value) => Ref.set(manifest, value)),
|
|
35
|
+
));
|
|
36
|
+
const initialize = yield* Effect.cached(Effect.gen(function* () {
|
|
37
|
+
yield* platform.prepare;
|
|
38
|
+
yield* loadManifest;
|
|
39
|
+
yield* ensureOpen;
|
|
40
|
+
// Acquisition and finalizer registration are uninterruptible as one unit.
|
|
41
|
+
return yield* Scope.extend(Effect.acquireRelease(
|
|
42
|
+
Effect.gen(function* () {
|
|
43
|
+
const store = yield* platform.open;
|
|
44
|
+
const release = yield* Effect.cached(platform.close(store));
|
|
45
|
+
return { store, release };
|
|
46
|
+
}),
|
|
47
|
+
({ release }) => Effect.exit(release).pipe(Effect.flatMap((result) => Ref.set(releaseResult, result))),
|
|
48
|
+
), owner);
|
|
49
|
+
}));
|
|
50
|
+
|
|
51
|
+
const open = Effect.gen(function* () {
|
|
52
|
+
yield* ensureOpen;
|
|
53
|
+
const acquired = yield* initialize;
|
|
54
|
+
if (yield* Ref.get(closed)) {
|
|
55
|
+
// A late store must close even though the operation that acquired it fails.
|
|
56
|
+
// The scope finalizer shares this cached release, including its exact error.
|
|
57
|
+
yield* acquired.release;
|
|
58
|
+
yield* ensureOpen;
|
|
59
|
+
}
|
|
60
|
+
return acquired.store;
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
function index(records: readonly KnowledgeGraphRecordV1[]): Effect.Effect<Readonly<{ indexed: number; v: 1 }>, SemanticFailure> {
|
|
64
|
+
return indexing.withPermits(1)(Effect.gen(function* () {
|
|
65
|
+
yield* ensureOpen;
|
|
66
|
+
yield* platform.prepare;
|
|
67
|
+
yield* loadManifest;
|
|
68
|
+
const next = yield* platform.writeDocuments(records);
|
|
69
|
+
const store = yield* open;
|
|
70
|
+
yield* platform.update(store);
|
|
71
|
+
yield* platform.embed(store);
|
|
72
|
+
yield* platform.publishManifest(next);
|
|
73
|
+
yield* Ref.set(manifest, next);
|
|
74
|
+
return { indexed: records.length, v: 1 };
|
|
75
|
+
}));
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function search(query: string, limit: number,
|
|
79
|
+
authority: OhSqliteStore): Effect.Effect<readonly OhSemanticSearchResultV1[], SemanticFailure> {
|
|
80
|
+
return Effect.gen(function* () {
|
|
81
|
+
const store = yield* open;
|
|
82
|
+
const snapshot = yield* Ref.get(manifest);
|
|
83
|
+
const rawResults = yield* platform.search(store, query, Math.min(100, limit * 3));
|
|
84
|
+
const results = yield* Effect.try({
|
|
85
|
+
try: () => parseQmdVectorResults(rawResults),
|
|
86
|
+
catch: (cause): SemanticFailure => ({ _tag: "ResultFailure", cause }),
|
|
87
|
+
});
|
|
88
|
+
const output: OhSemanticSearchResultV1[] = [];
|
|
89
|
+
const seen = new Set<string>();
|
|
90
|
+
for (const result of results) {
|
|
91
|
+
const entry = snapshot.entries[result.filename];
|
|
92
|
+
if (entry === undefined || result.title !== entry.key || seen.has(entry.key)) continue;
|
|
93
|
+
const current = yield* platform.readAuthority(authority, entry.key);
|
|
94
|
+
if (current === null || current.recordSha256 !== entry.recordSha256) continue;
|
|
95
|
+
const expectedDocument = recordDocument(current);
|
|
96
|
+
if (result.body !== expectedDocument || result.hash !== sha256Hex(expectedDocument)) continue;
|
|
97
|
+
seen.add(entry.key);
|
|
98
|
+
output.push({ key: entry.key, recordSha256: entry.recordSha256, score: result.score, v: 1 });
|
|
99
|
+
if (output.length === limit) break;
|
|
100
|
+
}
|
|
101
|
+
return output;
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
const close = yield* Effect.cached(Effect.gen(function* () {
|
|
106
|
+
// A failed operation does not abort draining its siblings or poison close.
|
|
107
|
+
yield* FiberSet.awaitEmpty(operations);
|
|
108
|
+
yield* Scope.close(owner, Exit.void);
|
|
109
|
+
const result = yield* Ref.get(releaseResult);
|
|
110
|
+
return yield* result;
|
|
111
|
+
}));
|
|
112
|
+
return { closed, operations, close, index, search };
|
|
113
|
+
});
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { Cause, Effect, Exit, Fiber, FiberSet, Option, Ref } from "effect";
|
|
2
|
+
|
|
3
|
+
import type { OhSemanticSearchBackendV1 } from "./semantic-model";
|
|
4
|
+
import { semanticPlatformLive, type SemanticFailure, type SemanticOptions } from "./semantic-platform";
|
|
5
|
+
import { makeSemanticLifecycle } from "./semantic-program";
|
|
6
|
+
|
|
7
|
+
type SemanticBoundary = Omit<OhSemanticSearchBackendV1, "profile">;
|
|
8
|
+
|
|
9
|
+
/** The only asynchronous runner accepts an already closed, total boundary. */
|
|
10
|
+
async function runClosed<A>(effect: Effect.Effect<Exit.Exit<A, SemanticFailure>, never, never>): Promise<A> {
|
|
11
|
+
const result = await Effect.runPromise(effect);
|
|
12
|
+
if (Exit.isSuccess(result)) return result.value;
|
|
13
|
+
const expected = Cause.failureOption(result.cause);
|
|
14
|
+
if (Option.isSome(expected)) throw expected.value.cause;
|
|
15
|
+
throw Cause.squash(result.cause);
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export function makeSemanticBoundary(options: SemanticOptions): SemanticBoundary {
|
|
19
|
+
const lifecycle = Effect.runSync(makeSemanticLifecycle.pipe(Effect.provide(semanticPlatformLive(options))));
|
|
20
|
+
const start = Effect.runSync(FiberSet.runtime(lifecycle.operations)<never>());
|
|
21
|
+
|
|
22
|
+
function admitted<A>(operation: Effect.Effect<A, SemanticFailure, never>, immediate = true): Promise<A> {
|
|
23
|
+
// Register synchronously before returning to the caller, so immediate close
|
|
24
|
+
// cannot overtake an admitted operation before its fiber has started.
|
|
25
|
+
const fiber = start(Effect.exit(operation), { immediate });
|
|
26
|
+
return runClosed(Fiber.join(fiber));
|
|
27
|
+
}
|
|
28
|
+
function isClosed(): boolean {
|
|
29
|
+
return Effect.runSync(Ref.get(lifecycle.closed));
|
|
30
|
+
}
|
|
31
|
+
return {
|
|
32
|
+
index(records) {
|
|
33
|
+
if (records.length > 65_536) {
|
|
34
|
+
return Promise.reject(new RangeError("A semantic snapshot may contain at most 65,536 records."));
|
|
35
|
+
}
|
|
36
|
+
if (isClosed()) return Promise.reject(new Error("The semantic backend is closed."));
|
|
37
|
+
// Index snapshots have always entered an asynchronous FIFO queue. Keep
|
|
38
|
+
// same-turn close ahead of their first filesystem operation.
|
|
39
|
+
return admitted(lifecycle.index([...records]), false);
|
|
40
|
+
},
|
|
41
|
+
search(query, limit, authority) {
|
|
42
|
+
if (!Number.isSafeInteger(limit) || limit < 1 || limit > 100) {
|
|
43
|
+
return Promise.reject(new RangeError("Semantic limit must be 1 through 100."));
|
|
44
|
+
}
|
|
45
|
+
if (isClosed()) return Promise.reject(new Error("The semantic backend is closed."));
|
|
46
|
+
return admitted(lifecycle.search(query, limit, authority));
|
|
47
|
+
},
|
|
48
|
+
close() {
|
|
49
|
+
Effect.runSync(Ref.set(lifecycle.closed, true));
|
|
50
|
+
return runClosed(Effect.exit(lifecycle.close));
|
|
51
|
+
},
|
|
52
|
+
};
|
|
53
|
+
}
|
package/src/semantic.test.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { afterEach, describe, expect, test } from "bun:test";
|
|
2
|
-
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
|
2
|
+
import { mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises";
|
|
3
3
|
import { isAbsolute, join, relative, resolve } from "node:path";
|
|
4
4
|
import { tmpdir } from "node:os";
|
|
5
5
|
|
|
@@ -87,6 +87,120 @@ describe("local EmbeddingGemma profile", () => {
|
|
|
87
87
|
});
|
|
88
88
|
|
|
89
89
|
describe("QMD derived index confinement", () => {
|
|
90
|
+
test("same-turn close fences the first queued index before filesystem work", async () => {
|
|
91
|
+
const root = await mkdtemp(join(tmpdir(), "oh-semantic-immediate-close-test-"));
|
|
92
|
+
roots.push(root);
|
|
93
|
+
let factoryCalls = 0;
|
|
94
|
+
const backend = new OhQmdSemanticBackendV1({ cacheDirectory: root, storeFactory: async () => {
|
|
95
|
+
factoryCalls += 1;
|
|
96
|
+
throw new Error("must not acquire");
|
|
97
|
+
} });
|
|
98
|
+
const indexing = backend.index([record("Ada")]);
|
|
99
|
+
const rejection = indexing.then(() => null, (error: unknown) => error);
|
|
100
|
+
await backend.close();
|
|
101
|
+
expect(await rejection).toMatchObject({ message: "The semantic backend is closed." });
|
|
102
|
+
expect(factoryCalls).toBe(0);
|
|
103
|
+
expect(await readdir(root)).toEqual([]);
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
test("closing before use never acquires a store and preserves validation precedence", async () => {
|
|
107
|
+
const root = await mkdtemp(join(tmpdir(), "oh-semantic-unused-close-test-"));
|
|
108
|
+
roots.push(root);
|
|
109
|
+
let factoryCalls = 0;
|
|
110
|
+
const backend = new OhQmdSemanticBackendV1({ cacheDirectory: root, storeFactory: async () => {
|
|
111
|
+
factoryCalls += 1;
|
|
112
|
+
throw new Error("must not acquire");
|
|
113
|
+
} });
|
|
114
|
+
const authority = new OhSqliteStore({ path: ":memory:" });
|
|
115
|
+
await Promise.all([backend.close(), backend.close()]);
|
|
116
|
+
await expect(backend.search("late", 0, authority)).rejects.toBeInstanceOf(RangeError);
|
|
117
|
+
await expect(backend.search("late", 1, authority)).rejects.toThrow("closed");
|
|
118
|
+
await expect(backend.index([])).rejects.toThrow("closed");
|
|
119
|
+
expect(factoryCalls).toBe(0);
|
|
120
|
+
authority.close();
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
test("preserves the same late-acquisition close error for the caller and every close", async () => {
|
|
124
|
+
const root = await mkdtemp(join(tmpdir(), "oh-semantic-late-close-failure-test-"));
|
|
125
|
+
roots.push(root);
|
|
126
|
+
const entered = deferred<void>();
|
|
127
|
+
const release = deferred<void>();
|
|
128
|
+
const failure = Object.freeze({ code: "CLOSE_FAILED", message: "foreign close failed" });
|
|
129
|
+
let closeCalls = 0;
|
|
130
|
+
const backend = new OhQmdSemanticBackendV1({ cacheDirectory: root, storeFactory: async () => {
|
|
131
|
+
entered.resolve();
|
|
132
|
+
await release.promise;
|
|
133
|
+
return {
|
|
134
|
+
close: async () => { closeCalls += 1; throw failure; },
|
|
135
|
+
embed: async () => ({}), searchVector: async () => [], update: async () => ({}),
|
|
136
|
+
};
|
|
137
|
+
} });
|
|
138
|
+
const authority = new OhSqliteStore({ path: ":memory:" });
|
|
139
|
+
const searching = backend.search("first", 1, authority);
|
|
140
|
+
await entered.promise;
|
|
141
|
+
const closing = backend.close();
|
|
142
|
+
const searchFailure = searching.then(() => null, (error: unknown) => error);
|
|
143
|
+
const closeFailure = closing.then(() => null, (error: unknown) => error);
|
|
144
|
+
release.resolve();
|
|
145
|
+
expect(await searchFailure).toBe(failure);
|
|
146
|
+
expect(await closeFailure).toBe(failure);
|
|
147
|
+
await expect(backend.close()).rejects.toBe(failure);
|
|
148
|
+
expect(closeCalls).toBe(1);
|
|
149
|
+
authority.close();
|
|
150
|
+
});
|
|
151
|
+
|
|
152
|
+
test("drains an active index, rejects the queued snapshot, and closes once after publication", async () => {
|
|
153
|
+
const root = await mkdtemp(join(tmpdir(), "oh-semantic-queued-close-test-"));
|
|
154
|
+
roots.push(root);
|
|
155
|
+
const first = record("First edition");
|
|
156
|
+
const second = record("Second edition");
|
|
157
|
+
const entered = deferred<void>();
|
|
158
|
+
const release = deferred<void>();
|
|
159
|
+
let updateCalls = 0;
|
|
160
|
+
let closeCalls = 0;
|
|
161
|
+
const backend = new OhQmdSemanticBackendV1({ cacheDirectory: root, storeFactory: async () => ({
|
|
162
|
+
close: async () => { closeCalls += 1; },
|
|
163
|
+
update: async () => { updateCalls += 1; entered.resolve(); await release.promise; },
|
|
164
|
+
embed: async () => ({}), searchVector: async () => [],
|
|
165
|
+
}) });
|
|
166
|
+
const active = backend.index([first]);
|
|
167
|
+
await entered.promise;
|
|
168
|
+
const queued = backend.index([second]);
|
|
169
|
+
const queuedRejection = queued.then(() => null, (error: unknown) => error);
|
|
170
|
+
const closed = backend.close();
|
|
171
|
+
expect(closeCalls).toBe(0);
|
|
172
|
+
release.resolve();
|
|
173
|
+
await expect(active).resolves.toEqual({ indexed: 1, v: 1 });
|
|
174
|
+
expect(await queuedRejection).toMatchObject({ message: "The semantic backend is closed." });
|
|
175
|
+
await closed;
|
|
176
|
+
expect(updateCalls).toBe(1);
|
|
177
|
+
expect(closeCalls).toBe(1);
|
|
178
|
+
expect(await readFile(join(root, "manifest.json"), "utf8")).toBe(canonicalJson({
|
|
179
|
+
entries: { [`${sha256Hex(first.key)}.md`]: { key: first.key, recordSha256: first.recordSha256 } },
|
|
180
|
+
profileSha256: canonicalSha256(OH_EMBEDDING_PROFILE_V1), v: 1,
|
|
181
|
+
}));
|
|
182
|
+
});
|
|
183
|
+
|
|
184
|
+
test("a failed index releases the queue and preserves the original rejection object", async () => {
|
|
185
|
+
const root = await mkdtemp(join(tmpdir(), "oh-semantic-failed-queue-test-"));
|
|
186
|
+
roots.push(root);
|
|
187
|
+
const failure = new TypeError("synthetic embedding failure");
|
|
188
|
+
const first = record("First edition");
|
|
189
|
+
const second = record("Second edition");
|
|
190
|
+
let embeds = 0;
|
|
191
|
+
const backend = new OhQmdSemanticBackendV1({ cacheDirectory: root, storeFactory: async () => ({
|
|
192
|
+
close: async () => {}, update: async () => ({}), searchVector: async () => [],
|
|
193
|
+
embed: async () => { embeds += 1; if (embeds === 1) throw failure; },
|
|
194
|
+
}) });
|
|
195
|
+
const failed = backend.index([first]);
|
|
196
|
+
const succeeded = backend.index([second]);
|
|
197
|
+
await expect(failed).rejects.toBe(failure);
|
|
198
|
+
await expect(succeeded).resolves.toEqual({ indexed: 1, v: 1 });
|
|
199
|
+
expect(embeds).toBe(2);
|
|
200
|
+
expect(await readFile(join(root, "manifest.json"), "utf8")).toContain(second.recordSha256);
|
|
201
|
+
await backend.close();
|
|
202
|
+
});
|
|
203
|
+
|
|
90
204
|
test("closes idempotently after optional backend initialization fails", async () => {
|
|
91
205
|
const root = await mkdtemp(join(tmpdir(), "oh-semantic-failed-open-test-"));
|
|
92
206
|
roots.push(root);
|