gnosys 5.12.2 → 5.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +36 -5
- package/dist/lib/archive.js +10 -4
- package/dist/lib/db.d.ts +15 -0
- package/dist/lib/db.js +94 -34
- package/dist/lib/dbWrite.js +12 -0
- package/dist/lib/dream.d.ts +9 -0
- package/dist/lib/dream.js +98 -4
- package/dist/lib/dreamRunLog.d.ts +4 -4
- package/dist/lib/dreamRunLog.js +9 -6
- package/dist/lib/embedDb.d.ts +53 -0
- package/dist/lib/embedDb.js +84 -0
- package/dist/lib/embedQueue.d.ts +28 -0
- package/dist/lib/embedQueue.js +80 -0
- package/dist/lib/ftsQuery.d.ts +25 -0
- package/dist/lib/ftsQuery.js +42 -0
- package/dist/lib/hybridSearch.d.ts +21 -0
- package/dist/lib/hybridSearch.js +37 -1
- package/dist/lib/hybridSearchCommand.js +5 -0
- package/dist/lib/reindexCommand.js +26 -0
- package/dist/lib/resolver.d.ts +17 -0
- package/dist/lib/resolver.js +43 -17
- package/dist/lib/search.js +33 -20
- package/package.json +1 -1
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Central-DB embedding utilities (v5.13.0).
|
|
3
|
+
*
|
|
4
|
+
* GnosysDbSearch reads vectors from the memories.embedding column of
|
|
5
|
+
* gnosys.db, but before v5.13.0 nothing ever wrote that column: the
|
|
6
|
+
* reindex flow embedded file-store memories into the store-local
|
|
7
|
+
* embeddings.db, so DB-mode hybrid search could never run its semantic
|
|
8
|
+
* leg. This module owns the column — full backfill for reindex,
|
|
9
|
+
* missing-only backfill for Dream/maintenance, and single-memory
|
|
10
|
+
* embedding for the write-time queue.
|
|
11
|
+
*
|
|
12
|
+
* Embeddings are derived, rebuildable data: regenerating them is always
|
|
13
|
+
* safe, and a sync snapshot refresh replacing them is not a loss.
|
|
14
|
+
*/
|
|
15
|
+
/**
|
|
16
|
+
* The text recipe — identical to the file-store reindex in hybridSearch.ts
|
|
17
|
+
* (title, relevance keywords, tags, content) so query-vs-document
|
|
18
|
+
* similarity behaves the same in both modes.
|
|
19
|
+
*/
|
|
20
|
+
export function embeddingText(m) {
|
|
21
|
+
let tags = m.tags || "";
|
|
22
|
+
try {
|
|
23
|
+
const parsed = JSON.parse(tags);
|
|
24
|
+
if (Array.isArray(parsed)) {
|
|
25
|
+
tags = parsed.join(" ");
|
|
26
|
+
}
|
|
27
|
+
else if (parsed && typeof parsed === "object") {
|
|
28
|
+
tags = Object.values(parsed).flat().join(" ");
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
catch {
|
|
32
|
+
// already a plain string
|
|
33
|
+
}
|
|
34
|
+
return `${m.title}\n${m.relevance || ""}\n${tags}\n${m.content}`;
|
|
35
|
+
}
|
|
36
|
+
/** View a Float32Array as the Buffer shape memories.embedding stores. */
|
|
37
|
+
export function float32ToBuffer(vec) {
|
|
38
|
+
return Buffer.from(vec.buffer, vec.byteOffset, vec.byteLength);
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* (Re)build the memories.embedding column.
|
|
42
|
+
*
|
|
43
|
+
* mode "all" regenerates every memory (reindex semantics); "missing" only
|
|
44
|
+
* fills NULL rows (write-gap backfill for Dream/maintenance, newest first).
|
|
45
|
+
* Batched through embedBatch for model efficiency.
|
|
46
|
+
*/
|
|
47
|
+
export async function backfillCentralDbEmbeddings(db, embeddings, opts = {}) {
|
|
48
|
+
if (!db.isAvailable() || !db.isMigrated())
|
|
49
|
+
return { embedded: 0, total: 0 };
|
|
50
|
+
const rows = db.getMemoriesForEmbedding(opts.mode || "missing", opts.limit);
|
|
51
|
+
const total = rows.length;
|
|
52
|
+
if (total === 0)
|
|
53
|
+
return { embedded: 0, total: 0 };
|
|
54
|
+
const batchSize = opts.batchSize || 32;
|
|
55
|
+
let embedded = 0;
|
|
56
|
+
for (let i = 0; i < rows.length; i += batchSize) {
|
|
57
|
+
const batch = rows.slice(i, i + batchSize);
|
|
58
|
+
const vectors = await embeddings.embedBatch(batch.map(embeddingText));
|
|
59
|
+
for (let j = 0; j < batch.length; j++) {
|
|
60
|
+
db.updateEmbedding(batch[j].id, float32ToBuffer(vectors[j]));
|
|
61
|
+
embedded++;
|
|
62
|
+
opts.onProgress?.(embedded, total, batch[j].id);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
return { embedded, total };
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Embed one memory and store its vector. Returns false if the memory
|
|
69
|
+
* doesn't exist; throws on embedder failure — callers decide how loud.
|
|
70
|
+
*/
|
|
71
|
+
export async function embedMemoryIntoDb(db, embeddings, id) {
|
|
72
|
+
const mem = db.getMemory(id);
|
|
73
|
+
if (!mem)
|
|
74
|
+
return false;
|
|
75
|
+
const vec = await embeddings.embed(embeddingText({
|
|
76
|
+
id: mem.id,
|
|
77
|
+
title: mem.title,
|
|
78
|
+
relevance: mem.relevance,
|
|
79
|
+
tags: mem.tags,
|
|
80
|
+
content: mem.content,
|
|
81
|
+
}));
|
|
82
|
+
db.updateEmbedding(id, float32ToBuffer(vec));
|
|
83
|
+
return true;
|
|
84
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Write-time embedding queue (v5.13.0).
|
|
3
|
+
*
|
|
4
|
+
* Keeps memories.embedding fresh as memories are written, so semantic
|
|
5
|
+
* search doesn't drift stale between reindexes. Disabled by default —
|
|
6
|
+
* the MCP server enables it once its long-lived central-DB handle and
|
|
7
|
+
* embeddings instance exist. CLI one-shot processes stay disabled (they
|
|
8
|
+
* exit before the model could load) and rely on reindex / Dream backfill.
|
|
9
|
+
*
|
|
10
|
+
* Contract: a memory write must NEVER block on or fail because of
|
|
11
|
+
* embedding. queueMemoryEmbedding returns immediately; the drain runs on
|
|
12
|
+
* an unref'd timer, lazily imports the embedder path, logs at most one
|
|
13
|
+
* stderr warning per process on failure, and drops rather than retries.
|
|
14
|
+
*/
|
|
15
|
+
import type { GnosysDB } from "./db.js";
|
|
16
|
+
import type { GnosysEmbeddings } from "./embeddings.js";
|
|
17
|
+
/** Turn on write-time embedding (serve mode). Getter is called at drain time. */
|
|
18
|
+
export declare function enableWriteTimeEmbedding(dbGetter: () => GnosysDB | null, embeddings: GnosysEmbeddings): void;
|
|
19
|
+
/** Turn off and clear the queue (tests, shutdown). */
|
|
20
|
+
export declare function disableWriteTimeEmbedding(): void;
|
|
21
|
+
export declare function isWriteTimeEmbeddingEnabled(): boolean;
|
|
22
|
+
/**
|
|
23
|
+
* Queue a memory for embedding. No-op unless enabled. Never throws,
|
|
24
|
+
* never blocks — safe to call from any write path.
|
|
25
|
+
*/
|
|
26
|
+
export declare function queueMemoryEmbedding(id: string): void;
|
|
27
|
+
/** Await in-flight embedding work. Used by tests and graceful shutdown. */
|
|
28
|
+
export declare function flushWriteTimeEmbeddings(): Promise<void>;
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Write-time embedding queue (v5.13.0).
|
|
3
|
+
*
|
|
4
|
+
* Keeps memories.embedding fresh as memories are written, so semantic
|
|
5
|
+
* search doesn't drift stale between reindexes. Disabled by default —
|
|
6
|
+
* the MCP server enables it once its long-lived central-DB handle and
|
|
7
|
+
* embeddings instance exist. CLI one-shot processes stay disabled (they
|
|
8
|
+
* exit before the model could load) and rely on reindex / Dream backfill.
|
|
9
|
+
*
|
|
10
|
+
* Contract: a memory write must NEVER block on or fail because of
|
|
11
|
+
* embedding. queueMemoryEmbedding returns immediately; the drain runs on
|
|
12
|
+
* an unref'd timer, lazily imports the embedder path, logs at most one
|
|
13
|
+
* stderr warning per process on failure, and drops rather than retries.
|
|
14
|
+
*/
|
|
15
|
+
let getDb = null;
|
|
16
|
+
let embedder = null;
|
|
17
|
+
const pending = new Set();
|
|
18
|
+
let drainPromise = null;
|
|
19
|
+
let warnedOnce = false;
|
|
20
|
+
/** Turn on write-time embedding (serve mode). Getter is called at drain time. */
|
|
21
|
+
export function enableWriteTimeEmbedding(dbGetter, embeddings) {
|
|
22
|
+
getDb = dbGetter;
|
|
23
|
+
embedder = embeddings;
|
|
24
|
+
}
|
|
25
|
+
/** Turn off and clear the queue (tests, shutdown). */
|
|
26
|
+
export function disableWriteTimeEmbedding() {
|
|
27
|
+
getDb = null;
|
|
28
|
+
embedder = null;
|
|
29
|
+
pending.clear();
|
|
30
|
+
}
|
|
31
|
+
export function isWriteTimeEmbeddingEnabled() {
|
|
32
|
+
return getDb !== null && embedder !== null;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Queue a memory for embedding. No-op unless enabled. Never throws,
|
|
36
|
+
* never blocks — safe to call from any write path.
|
|
37
|
+
*/
|
|
38
|
+
export function queueMemoryEmbedding(id) {
|
|
39
|
+
if (!getDb || !embedder || !id)
|
|
40
|
+
return;
|
|
41
|
+
pending.add(id);
|
|
42
|
+
if (!drainPromise) {
|
|
43
|
+
drainPromise = new Promise((resolve) => {
|
|
44
|
+
const timer = setTimeout(() => {
|
|
45
|
+
drain().finally(() => {
|
|
46
|
+
drainPromise = null;
|
|
47
|
+
resolve();
|
|
48
|
+
});
|
|
49
|
+
}, 25);
|
|
50
|
+
timer.unref?.();
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
/** Await in-flight embedding work. Used by tests and graceful shutdown. */
|
|
55
|
+
export async function flushWriteTimeEmbeddings() {
|
|
56
|
+
while (drainPromise) {
|
|
57
|
+
await drainPromise;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
async function drain() {
|
|
61
|
+
while (pending.size > 0) {
|
|
62
|
+
const id = pending.values().next().value;
|
|
63
|
+
pending.delete(id);
|
|
64
|
+
const db = getDb?.();
|
|
65
|
+
if (!db || !embedder)
|
|
66
|
+
return;
|
|
67
|
+
try {
|
|
68
|
+
const { embedMemoryIntoDb } = await import("./embedDb.js");
|
|
69
|
+
await embedMemoryIntoDb(db, embedder, id);
|
|
70
|
+
}
|
|
71
|
+
catch (err) {
|
|
72
|
+
if (!warnedOnce) {
|
|
73
|
+
warnedOnce = true;
|
|
74
|
+
console.error(`Gnosys: write-time embedding failed (${err instanceof Error ? err.message : err}). ` +
|
|
75
|
+
`New memories will be embedded by the next gnosys_reindex or Dream run instead.`);
|
|
76
|
+
}
|
|
77
|
+
return; // drop this round; anything still pending waits for the next queue call
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* FTS5 MATCH query construction — shared by the central-DB search (db.ts),
|
|
3
|
+
* the per-store search index (search.ts), and archive search (archive.ts).
|
|
4
|
+
*
|
|
5
|
+
* FTS5 treats space-separated bare terms as implicit AND, so the long
|
|
6
|
+
* descriptive queries the tool docs encourage ("auth JWT session tokens
|
|
7
|
+
* refresh") return zero results unless EVERY term matches. Callers use
|
|
8
|
+
* these helpers to try AND first (precision), then retry with OR
|
|
9
|
+
* (recall) when AND finds nothing — BM25 still ranks the best-covered
|
|
10
|
+
* memories first in the OR pass.
|
|
11
|
+
*
|
|
12
|
+
* Every term is emitted as a quoted phrase, which also makes previously
|
|
13
|
+
* syntax-error-prone input (hyphens, colons, FTS5 keywords like NOT)
|
|
14
|
+
* safe. A trailing `*` is preserved as an FTS5 prefix query (`"term"*`).
|
|
15
|
+
*/
|
|
16
|
+
/**
|
|
17
|
+
* Split a raw query into sanitized terms. Drops quote characters and any
|
|
18
|
+
* token with no letters or digits (pure punctuation can't match anything
|
|
19
|
+
* under the unicode61 tokenizer).
|
|
20
|
+
*/
|
|
21
|
+
export declare function ftsTerms(query: string): string[];
|
|
22
|
+
/** Implicit-AND MATCH expression: all terms must match. */
|
|
23
|
+
export declare function ftsAndQuery(terms: string[]): string;
|
|
24
|
+
/** OR MATCH expression: any term may match; BM25 ranks fuller matches higher. */
|
|
25
|
+
export declare function ftsOrQuery(terms: string[]): string;
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* FTS5 MATCH query construction — shared by the central-DB search (db.ts),
|
|
3
|
+
* the per-store search index (search.ts), and archive search (archive.ts).
|
|
4
|
+
*
|
|
5
|
+
* FTS5 treats space-separated bare terms as implicit AND, so the long
|
|
6
|
+
* descriptive queries the tool docs encourage ("auth JWT session tokens
|
|
7
|
+
* refresh") return zero results unless EVERY term matches. Callers use
|
|
8
|
+
* these helpers to try AND first (precision), then retry with OR
|
|
9
|
+
* (recall) when AND finds nothing — BM25 still ranks the best-covered
|
|
10
|
+
* memories first in the OR pass.
|
|
11
|
+
*
|
|
12
|
+
* Every term is emitted as a quoted phrase, which also makes previously
|
|
13
|
+
* syntax-error-prone input (hyphens, colons, FTS5 keywords like NOT)
|
|
14
|
+
* safe. A trailing `*` is preserved as an FTS5 prefix query (`"term"*`).
|
|
15
|
+
*/
|
|
16
|
+
/**
|
|
17
|
+
* Split a raw query into sanitized terms. Drops quote characters and any
|
|
18
|
+
* token with no letters or digits (pure punctuation can't match anything
|
|
19
|
+
* under the unicode61 tokenizer).
|
|
20
|
+
*/
|
|
21
|
+
export function ftsTerms(query) {
|
|
22
|
+
return query
|
|
23
|
+
.replace(/['"]/g, "")
|
|
24
|
+
.split(/\s+/)
|
|
25
|
+
.filter((t) => /[\p{L}\p{N}]/u.test(t));
|
|
26
|
+
}
|
|
27
|
+
/** Render one term as a safe FTS5 phrase, preserving trailing-`*` prefix queries. */
|
|
28
|
+
function ftsPhrase(term) {
|
|
29
|
+
const prefixMatch = term.match(/^(.*?)\*+$/);
|
|
30
|
+
if (prefixMatch && /[\p{L}\p{N}]/u.test(prefixMatch[1])) {
|
|
31
|
+
return `"${prefixMatch[1]}"*`;
|
|
32
|
+
}
|
|
33
|
+
return `"${term}"`;
|
|
34
|
+
}
|
|
35
|
+
/** Implicit-AND MATCH expression: all terms must match. */
|
|
36
|
+
export function ftsAndQuery(terms) {
|
|
37
|
+
return terms.map(ftsPhrase).join(" ");
|
|
38
|
+
}
|
|
39
|
+
/** OR MATCH expression: any term may match; BM25 ranks fuller matches higher. */
|
|
40
|
+
export function ftsOrQuery(terms) {
|
|
41
|
+
return terms.map(ftsPhrase).join(" OR ");
|
|
42
|
+
}
|
|
@@ -20,6 +20,8 @@ export declare class GnosysHybridSearch {
|
|
|
20
20
|
private storePath;
|
|
21
21
|
/** v2.0: When set, hybrid search uses SQLite directly */
|
|
22
22
|
private dbSearch;
|
|
23
|
+
/** v5.13.0: kept for central-DB embedding backfill (reindexCentralDb) */
|
|
24
|
+
private gnosysDb;
|
|
23
25
|
constructor(search: GnosysSearch, embeddings: GnosysEmbeddings, resolver: GnosysResolver, storePath: string, gnosysDb?: GnosysDB);
|
|
24
26
|
/**
|
|
25
27
|
* Main hybrid search entry point.
|
|
@@ -48,6 +50,17 @@ export declare class GnosysHybridSearch {
|
|
|
48
50
|
* Returns count of files indexed.
|
|
49
51
|
*/
|
|
50
52
|
reindex(onProgress?: (current: number, total: number, filePath: string) => void): Promise<number>;
|
|
53
|
+
/**
|
|
54
|
+
* v5.13.0: (re)build the central-DB memories.embedding column — the
|
|
55
|
+
* vectors GnosysDbSearch actually reads. Before this, reindex only
|
|
56
|
+
* embedded file-store memories into the store-local embeddings.db, so
|
|
57
|
+
* DB-mode semantic search could never activate. Returns counts; no-op
|
|
58
|
+
* ({0,0}) when no migrated central DB is attached.
|
|
59
|
+
*/
|
|
60
|
+
reindexCentralDb(onProgress?: (current: number, total: number, id: string) => void): Promise<{
|
|
61
|
+
embedded: number;
|
|
62
|
+
total: number;
|
|
63
|
+
}>;
|
|
51
64
|
/**
|
|
52
65
|
* Load full content for search results (used by Ask engine).
|
|
53
66
|
* Handles both active memories and archived memories.
|
|
@@ -57,6 +70,14 @@ export declare class GnosysHybridSearch {
|
|
|
57
70
|
* Check if embeddings are available.
|
|
58
71
|
*/
|
|
59
72
|
hasEmbeddings(): boolean;
|
|
73
|
+
/**
|
|
74
|
+
* True when hybrid/semantic search can actually run its semantic leg.
|
|
75
|
+
* Mirrors the embedQuery gate in hybridSearch(): in DB mode, stored
|
|
76
|
+
* central-DB vectors are what matter (the query embedder loads the
|
|
77
|
+
* local model on demand); in legacy file mode, the store-local
|
|
78
|
+
* embeddings.db must be populated.
|
|
79
|
+
*/
|
|
80
|
+
canRunSemantic(): boolean;
|
|
60
81
|
/**
|
|
61
82
|
* Get embedding count.
|
|
62
83
|
*/
|
package/dist/lib/hybridSearch.js
CHANGED
|
@@ -19,6 +19,8 @@ export class GnosysHybridSearch {
|
|
|
19
19
|
storePath;
|
|
20
20
|
/** v2.0: When set, hybrid search uses SQLite directly */
|
|
21
21
|
dbSearch = null;
|
|
22
|
+
/** v5.13.0: kept for central-DB embedding backfill (reindexCentralDb) */
|
|
23
|
+
gnosysDb = null;
|
|
22
24
|
constructor(search, embeddings, resolver, storePath, gnosysDb) {
|
|
23
25
|
this.search = search;
|
|
24
26
|
this.embeddings = embeddings;
|
|
@@ -27,6 +29,7 @@ export class GnosysHybridSearch {
|
|
|
27
29
|
// v2.0: If GnosysDB is migrated, create a DB search adapter
|
|
28
30
|
if (gnosysDb?.isAvailable() && gnosysDb?.isMigrated()) {
|
|
29
31
|
this.dbSearch = new GnosysDbSearch(gnosysDb);
|
|
32
|
+
this.gnosysDb = gnosysDb;
|
|
30
33
|
}
|
|
31
34
|
}
|
|
32
35
|
/**
|
|
@@ -36,7 +39,11 @@ export class GnosysHybridSearch {
|
|
|
36
39
|
async hybridSearch(query, limit = 15, mode = "hybrid") {
|
|
37
40
|
// v2.0 DB-backed fast path: run entirely from gnosys.db
|
|
38
41
|
if (this.dbSearch) {
|
|
39
|
-
|
|
42
|
+
// v5.13.0: gate the semantic leg on stored central-DB vectors — the
|
|
43
|
+
// query embedder loads the local model on demand, so the store-local
|
|
44
|
+
// embeddings.db (the old gate) is irrelevant in DB mode. A machine
|
|
45
|
+
// that only syncs gnosys.db gets semantic search once vectors exist.
|
|
46
|
+
const embedQuery = this.dbSearch.hasEmbeddings()
|
|
40
47
|
? (text) => this.embeddings.embed(text)
|
|
41
48
|
: undefined;
|
|
42
49
|
return this.dbSearch.hybridSearch(query, limit, mode, embedQuery);
|
|
@@ -230,6 +237,22 @@ export class GnosysHybridSearch {
|
|
|
230
237
|
}
|
|
231
238
|
return indexed;
|
|
232
239
|
}
|
|
240
|
+
/**
|
|
241
|
+
* v5.13.0: (re)build the central-DB memories.embedding column — the
|
|
242
|
+
* vectors GnosysDbSearch actually reads. Before this, reindex only
|
|
243
|
+
* embedded file-store memories into the store-local embeddings.db, so
|
|
244
|
+
* DB-mode semantic search could never activate. Returns counts; no-op
|
|
245
|
+
* ({0,0}) when no migrated central DB is attached.
|
|
246
|
+
*/
|
|
247
|
+
async reindexCentralDb(onProgress) {
|
|
248
|
+
if (!this.gnosysDb)
|
|
249
|
+
return { embedded: 0, total: 0 };
|
|
250
|
+
const { backfillCentralDbEmbeddings } = await import("./embedDb.js");
|
|
251
|
+
return backfillCentralDbEmbeddings(this.gnosysDb, this.embeddings, {
|
|
252
|
+
mode: "all",
|
|
253
|
+
onProgress,
|
|
254
|
+
});
|
|
255
|
+
}
|
|
233
256
|
/**
|
|
234
257
|
* Load full content for search results (used by Ask engine).
|
|
235
258
|
* Handles both active memories and archived memories.
|
|
@@ -284,6 +307,19 @@ export class GnosysHybridSearch {
|
|
|
284
307
|
return this.dbSearch.hasEmbeddings();
|
|
285
308
|
return this.embeddings.hasEmbeddings();
|
|
286
309
|
}
|
|
310
|
+
/**
|
|
311
|
+
* True when hybrid/semantic search can actually run its semantic leg.
|
|
312
|
+
* Mirrors the embedQuery gate in hybridSearch(): in DB mode, stored
|
|
313
|
+
* central-DB vectors are what matter (the query embedder loads the
|
|
314
|
+
* local model on demand); in legacy file mode, the store-local
|
|
315
|
+
* embeddings.db must be populated.
|
|
316
|
+
*/
|
|
317
|
+
canRunSemantic() {
|
|
318
|
+
if (this.dbSearch) {
|
|
319
|
+
return this.dbSearch.hasEmbeddings();
|
|
320
|
+
}
|
|
321
|
+
return this.embeddings.hasEmbeddings();
|
|
322
|
+
}
|
|
287
323
|
/**
|
|
288
324
|
* Get embedding count.
|
|
289
325
|
*/
|
|
@@ -67,6 +67,11 @@ export async function runHybridSearchCommand(getResolver, query, opts) {
|
|
|
67
67
|
const embeddings = new GnosysEmbeddings(storePath);
|
|
68
68
|
const hybridSearch = new GnosysHybridSearch(search, embeddings, resolver, storePath);
|
|
69
69
|
const mode = opts.mode;
|
|
70
|
+
// v5.12.3: hybrid used to degrade to keyword-only silently when the
|
|
71
|
+
// semantic leg can't run. Warn on stderr so --json stdout stays clean.
|
|
72
|
+
if (mode !== "keyword" && !hybridSearch.canRunSemantic()) {
|
|
73
|
+
console.error(`⚠ Semantic embeddings unavailable — ${mode} search will run keyword-only. Run 'gnosys reindex' to build embeddings.`);
|
|
74
|
+
}
|
|
70
75
|
const results = await hybridSearch.hybridSearch(query, parseInt(opts.limit, 10), mode);
|
|
71
76
|
if (results.length === 0) {
|
|
72
77
|
outputResult(!!opts.json, { query, mode, results: [] }, () => {
|
|
@@ -24,7 +24,33 @@ export async function runReindexCommand(getResolver) {
|
|
|
24
24
|
const count = await hybridSearch.reindex((current, total, filePath) => {
|
|
25
25
|
process.stdout.write(`\r Indexing: ${current}/${total} — ${filePath.substring(0, 60)}`);
|
|
26
26
|
});
|
|
27
|
+
// v5.13.0: also (re)build the central-DB embedding column — the vectors
|
|
28
|
+
// DB-mode hybrid/semantic search actually reads. Own handle: the
|
|
29
|
+
// resolver-based hybridSearch above only covers file stores.
|
|
30
|
+
let dbEmbedded = 0;
|
|
31
|
+
let dbTotal = 0;
|
|
32
|
+
const { GnosysDB } = await import("./db.js");
|
|
33
|
+
const centralDb = GnosysDB.openCentral();
|
|
34
|
+
try {
|
|
35
|
+
if (centralDb.isAvailable() && centralDb.isMigrated()) {
|
|
36
|
+
const { backfillCentralDbEmbeddings } = await import("./embedDb.js");
|
|
37
|
+
const result = await backfillCentralDbEmbeddings(centralDb, embeddings, {
|
|
38
|
+
mode: "all",
|
|
39
|
+
onProgress: (current, total, id) => {
|
|
40
|
+
process.stdout.write(`\r Central DB: ${current}/${total} — ${id.substring(0, 40)} `);
|
|
41
|
+
},
|
|
42
|
+
});
|
|
43
|
+
dbEmbedded = result.embedded;
|
|
44
|
+
dbTotal = result.total;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
finally {
|
|
48
|
+
centralDb.close();
|
|
49
|
+
}
|
|
27
50
|
console.log(`\n\nReindex complete: ${count} memories embedded.`);
|
|
51
|
+
if (dbTotal > 0) {
|
|
52
|
+
console.log(`Central DB: ${dbEmbedded}/${dbTotal} memories embedded.`);
|
|
53
|
+
}
|
|
28
54
|
console.log("Hybrid and semantic search are now available.");
|
|
29
55
|
}
|
|
30
56
|
finally {
|
package/dist/lib/resolver.d.ts
CHANGED
|
@@ -46,6 +46,23 @@ export declare class GnosysResolver {
|
|
|
46
46
|
* Discover and initialize all store layers.
|
|
47
47
|
*/
|
|
48
48
|
resolve(): Promise<ResolvedStore[]>;
|
|
49
|
+
/**
|
|
50
|
+
* Load the env-configured store tiers (optional `GNOSYS_STORES`, personal
|
|
51
|
+
* `GNOSYS_PERSONAL`, global `GNOSYS_GLOBAL`) onto `this.stores`.
|
|
52
|
+
*
|
|
53
|
+
* Shared by {@link resolve} and {@link resolveForProject}: a per-tool
|
|
54
|
+
* `projectRoot` call must STILL see personal/global, otherwise cross-project
|
|
55
|
+
* and global memories are unreachable — you couldn't write `store: "global"`
|
|
56
|
+
* / `"personal"` nor read them back from another project. Since every tool
|
|
57
|
+
* is told to always pass `projectRoot`, omitting these here silently
|
|
58
|
+
* disabled the entire cross-project tier system.
|
|
59
|
+
*
|
|
60
|
+
* Global is loaded whenever `GNOSYS_GLOBAL` is set, creating its directory on
|
|
61
|
+
* demand (parity with personal) so a configured-but-empty global store still
|
|
62
|
+
* works. It remains write-only-when-explicitly-targeted: {@link getWriteTarget}
|
|
63
|
+
* never auto-selects it.
|
|
64
|
+
*/
|
|
65
|
+
private loadEnvTiers;
|
|
49
66
|
/**
|
|
50
67
|
* Get all stores in precedence order.
|
|
51
68
|
*/
|
package/dist/lib/resolver.js
CHANGED
|
@@ -79,8 +79,13 @@ export class GnosysResolver {
|
|
|
79
79
|
}
|
|
80
80
|
}
|
|
81
81
|
catch {
|
|
82
|
-
//
|
|
82
|
+
// No project store at projectRoot — still load the env tiers below.
|
|
83
83
|
}
|
|
84
|
+
// Even when scoped to a projectRoot, load the personal/global/optional
|
|
85
|
+
// tiers so cross-project + global memories stay reachable (write store:
|
|
86
|
+
// "global"/"personal", and read them back from any project). Without this,
|
|
87
|
+
// the universal "always pass projectRoot" guidance silently disabled them.
|
|
88
|
+
await resolver.loadEnvTiers();
|
|
84
89
|
return resolver;
|
|
85
90
|
}
|
|
86
91
|
/**
|
|
@@ -104,7 +109,30 @@ export class GnosysResolver {
|
|
|
104
109
|
path: projectPath,
|
|
105
110
|
});
|
|
106
111
|
}
|
|
107
|
-
// 2.
|
|
112
|
+
// 2-4. Env-configured tiers (optional, personal, global). Shared with
|
|
113
|
+
// resolveForProject so passing a projectRoot does NOT strip the
|
|
114
|
+
// cross-project tiers.
|
|
115
|
+
await this.loadEnvTiers();
|
|
116
|
+
return this.stores;
|
|
117
|
+
}
|
|
118
|
+
/**
|
|
119
|
+
* Load the env-configured store tiers (optional `GNOSYS_STORES`, personal
|
|
120
|
+
* `GNOSYS_PERSONAL`, global `GNOSYS_GLOBAL`) onto `this.stores`.
|
|
121
|
+
*
|
|
122
|
+
* Shared by {@link resolve} and {@link resolveForProject}: a per-tool
|
|
123
|
+
* `projectRoot` call must STILL see personal/global, otherwise cross-project
|
|
124
|
+
* and global memories are unreachable — you couldn't write `store: "global"`
|
|
125
|
+
* / `"personal"` nor read them back from another project. Since every tool
|
|
126
|
+
* is told to always pass `projectRoot`, omitting these here silently
|
|
127
|
+
* disabled the entire cross-project tier system.
|
|
128
|
+
*
|
|
129
|
+
* Global is loaded whenever `GNOSYS_GLOBAL` is set, creating its directory on
|
|
130
|
+
* demand (parity with personal) so a configured-but-empty global store still
|
|
131
|
+
* works. It remains write-only-when-explicitly-targeted: {@link getWriteTarget}
|
|
132
|
+
* never auto-selects it.
|
|
133
|
+
*/
|
|
134
|
+
async loadEnvTiers() {
|
|
135
|
+
// Optional stores (GNOSYS_STORES — colon-separated, read-only)
|
|
108
136
|
const optionalPaths = process.env.GNOSYS_STORES;
|
|
109
137
|
if (optionalPaths) {
|
|
110
138
|
const paths = optionalPaths.split(":").filter(Boolean);
|
|
@@ -124,7 +152,7 @@ export class GnosysResolver {
|
|
|
124
152
|
}
|
|
125
153
|
}
|
|
126
154
|
}
|
|
127
|
-
//
|
|
155
|
+
// Personal store (GNOSYS_PERSONAL — writable fallback target)
|
|
128
156
|
const personalPath = process.env.GNOSYS_PERSONAL;
|
|
129
157
|
if (personalPath) {
|
|
130
158
|
const p = path.resolve(personalPath);
|
|
@@ -138,24 +166,22 @@ export class GnosysResolver {
|
|
|
138
166
|
path: p,
|
|
139
167
|
});
|
|
140
168
|
}
|
|
141
|
-
//
|
|
142
|
-
//
|
|
169
|
+
// Global store (GNOSYS_GLOBAL — writable, but never auto-selected). Loaded
|
|
170
|
+
// whenever set; store.init() creates the dir on demand (parity with
|
|
171
|
+
// personal), so a configured-but-empty GNOSYS_GLOBAL still works.
|
|
143
172
|
const globalPath = process.env.GNOSYS_GLOBAL;
|
|
144
173
|
if (globalPath) {
|
|
145
174
|
const p = path.resolve(globalPath);
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
});
|
|
156
|
-
}
|
|
175
|
+
const store = new GnosysStore(p);
|
|
176
|
+
await store.init();
|
|
177
|
+
this.stores.push({
|
|
178
|
+
layer: "global",
|
|
179
|
+
label: "global",
|
|
180
|
+
store,
|
|
181
|
+
writable: true,
|
|
182
|
+
path: p,
|
|
183
|
+
});
|
|
157
184
|
}
|
|
158
|
-
return this.stores;
|
|
159
185
|
}
|
|
160
186
|
/**
|
|
161
187
|
* Get all stores in precedence order.
|
package/dist/lib/search.js
CHANGED
|
@@ -11,6 +11,7 @@ catch {
|
|
|
11
11
|
// better-sqlite3 native module not available — search degrades gracefully
|
|
12
12
|
}
|
|
13
13
|
import path from "path";
|
|
14
|
+
import { ftsTerms, ftsAndQuery, ftsOrQuery } from "./ftsQuery.js";
|
|
14
15
|
export class GnosysSearch {
|
|
15
16
|
db = null;
|
|
16
17
|
constructor(storePath) {
|
|
@@ -133,9 +134,8 @@ export class GnosysSearch {
|
|
|
133
134
|
search(query, limit = 20) {
|
|
134
135
|
if (!this.db)
|
|
135
136
|
return [];
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
if (!safeQuery)
|
|
137
|
+
const terms = ftsTerms(query);
|
|
138
|
+
if (terms.length === 0)
|
|
139
139
|
return [];
|
|
140
140
|
const stmt = this.db.prepare(`
|
|
141
141
|
SELECT
|
|
@@ -149,7 +149,12 @@ export class GnosysSearch {
|
|
|
149
149
|
LIMIT ?
|
|
150
150
|
`);
|
|
151
151
|
try {
|
|
152
|
-
|
|
152
|
+
// v5.12.3: AND first (precision), OR retry when AND finds nothing —
|
|
153
|
+
// multi-word queries previously required every term to match.
|
|
154
|
+
const results = stmt.all(ftsAndQuery(terms), limit);
|
|
155
|
+
if (results.length > 0 || terms.length === 1)
|
|
156
|
+
return results;
|
|
157
|
+
return stmt.all(ftsOrQuery(terms), limit);
|
|
153
158
|
}
|
|
154
159
|
catch {
|
|
155
160
|
// If FTS5 query fails, fall back to simple LIKE search
|
|
@@ -163,7 +168,7 @@ export class GnosysSearch {
|
|
|
163
168
|
WHERE content LIKE ? OR title LIKE ? OR tags LIKE ?
|
|
164
169
|
LIMIT ?
|
|
165
170
|
`);
|
|
166
|
-
const pattern = `%${
|
|
171
|
+
const pattern = `%${terms.join(" ")}%`;
|
|
167
172
|
return likeStmt.all(pattern, pattern, pattern, limit);
|
|
168
173
|
}
|
|
169
174
|
}
|
|
@@ -175,8 +180,8 @@ export class GnosysSearch {
|
|
|
175
180
|
discover(query, limit = 20) {
|
|
176
181
|
if (!this.db)
|
|
177
182
|
return [];
|
|
178
|
-
const
|
|
179
|
-
if (
|
|
183
|
+
const terms = ftsTerms(query);
|
|
184
|
+
if (terms.length === 0)
|
|
180
185
|
return [];
|
|
181
186
|
// Search primarily on relevance + title + tags (not content body)
|
|
182
187
|
// FTS5 column filter: {relevance title tags}
|
|
@@ -191,24 +196,32 @@ export class GnosysSearch {
|
|
|
191
196
|
ORDER BY rank
|
|
192
197
|
LIMIT ?
|
|
193
198
|
`);
|
|
194
|
-
|
|
195
|
-
// Try column-filtered search on relevance/title/tags first
|
|
196
|
-
const colQuery = `{relevance title tags} : ${safeQuery}`;
|
|
197
|
-
const results = stmt.all(colQuery, limit);
|
|
198
|
-
if (results.length > 0)
|
|
199
|
-
return results;
|
|
200
|
-
// Fall back to full-text search if column filter finds nothing
|
|
201
|
-
return stmt.all(safeQuery, limit);
|
|
202
|
-
}
|
|
203
|
-
catch {
|
|
204
|
-
// If FTS5 column filter syntax fails, fall back to full search
|
|
199
|
+
const tryRun = (match) => {
|
|
205
200
|
try {
|
|
206
|
-
return stmt.all(
|
|
201
|
+
return stmt.all(match, limit);
|
|
207
202
|
}
|
|
208
203
|
catch {
|
|
209
204
|
return [];
|
|
210
205
|
}
|
|
211
|
-
}
|
|
206
|
+
};
|
|
207
|
+
// v5.12.3: precision-to-recall ladder. AND on the metadata columns,
|
|
208
|
+
// AND anywhere, then OR retries — multi-word queries previously
|
|
209
|
+
// required every term to match, so long queries returned nothing.
|
|
210
|
+
// Parens scope the column filter to the whole expression (a bare
|
|
211
|
+
// `{cols} : a b` only filtered the first term).
|
|
212
|
+
const colScoped = (expr) => `{relevance title tags} : (${expr})`;
|
|
213
|
+
const andExpr = ftsAndQuery(terms);
|
|
214
|
+
let results = tryRun(colScoped(andExpr));
|
|
215
|
+
if (results.length > 0)
|
|
216
|
+
return results;
|
|
217
|
+
results = tryRun(andExpr);
|
|
218
|
+
if (results.length > 0 || terms.length === 1)
|
|
219
|
+
return results;
|
|
220
|
+
const orExpr = ftsOrQuery(terms);
|
|
221
|
+
results = tryRun(colScoped(orExpr));
|
|
222
|
+
if (results.length > 0)
|
|
223
|
+
return results;
|
|
224
|
+
return tryRun(orExpr);
|
|
212
225
|
}
|
|
213
226
|
close() {
|
|
214
227
|
this.db?.close();
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "gnosys",
|
|
3
|
-
"version": "5.
|
|
3
|
+
"version": "5.13.0",
|
|
4
4
|
"description": "Gnosys — Persistent Memory for AI Agents. Sandbox-first runtime, central SQLite brain, federated search, Dream Mode, Web Knowledge Base, Obsidian export.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|