@gmickel/gno 1.2.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/assets/skill/SKILL.md +3 -0
- package/assets/skill/cli-reference.md +5 -0
- package/assets/skill/examples.md +2 -0
- package/package.json +1 -1
- package/src/app/constants.ts +64 -8
- package/src/cli/commands/embed.ts +6 -2
- package/src/cli/commands/get.ts +15 -5
- package/src/cli/commands/index-cmd.ts +4 -0
- package/src/cli/commands/multi-get.ts +62 -1
- package/src/cli/commands/query.ts +8 -2
- package/src/cli/commands/search.ts +8 -2
- package/src/cli/commands/shared.ts +18 -1
- package/src/cli/commands/status.ts +4 -2
- package/src/cli/commands/update.ts +6 -1
- package/src/cli/commands/vsearch.ts +8 -2
- package/src/cli/format/search-results.ts +1 -1
- package/src/cli/program.ts +22 -1
- package/src/ingestion/chunker.ts +6 -0
- package/src/llm/cache.ts +162 -28
- package/src/llm/errors.ts +32 -0
- package/src/llm/lockfile.ts +49 -4
- package/src/llm/nodeLlamaCpp/embedding.ts +69 -3
- package/src/llm/nodeLlamaCpp/lifecycle.ts +60 -4
- package/src/mcp/resources/index.ts +13 -4
- package/src/mcp/server.ts +2 -0
- package/src/mcp/tools/get.ts +7 -2
- package/src/mcp/tools/multi-get.ts +2 -2
- package/src/mcp/tools/query.ts +2 -1
- package/src/mcp/tools/search.ts +2 -1
- package/src/mcp/tools/vsearch.ts +2 -1
- package/src/pipeline/explain.ts +12 -2
- package/src/pipeline/hybrid.ts +9 -1
- package/src/pipeline/search.ts +16 -7
- package/src/pipeline/types.ts +2 -0
- package/src/pipeline/vsearch.ts +29 -15
- package/src/publish/export-service.ts +27 -2
- package/src/sdk/client.ts +83 -28
- package/src/store/content-batch.ts +38 -0
- package/src/store/sqlite/adapter.ts +38 -2
- package/src/store/types.ts +8 -0
- package/src/store/vector/sqlite-vec.ts +10 -4
- package/src/store/vector/types.ts +2 -0
|
@@ -16,6 +16,7 @@ import { loadFailedError, outOfMemoryError, timeoutError } from "../errors";
|
|
|
16
16
|
|
|
17
17
|
type Llama = Awaited<ReturnType<typeof import("node-llama-cpp").getLlama>>;
|
|
18
18
|
type LlamaModel = Awaited<ReturnType<Llama["loadModel"]>>;
|
|
19
|
+
export type LlamaGpuMode = "auto" | "metal" | "vulkan" | "cuda" | false;
|
|
19
20
|
|
|
20
21
|
interface CachedModel {
|
|
21
22
|
uri: string;
|
|
@@ -24,6 +25,40 @@ interface CachedModel {
|
|
|
24
25
|
loadedAt: number;
|
|
25
26
|
}
|
|
26
27
|
|
|
28
|
+
let invalidGpuModeWarned = false;
|
|
29
|
+
let gpuFallbackWarned = false;
|
|
30
|
+
|
|
31
|
+
export function resolveLlamaGpuMode(
|
|
32
|
+
env: NodeJS.ProcessEnv = process.env
|
|
33
|
+
): LlamaGpuMode {
|
|
34
|
+
const raw = (env.GNO_LLAMA_GPU ?? env.NODE_LLAMA_CPP_GPU ?? "auto")
|
|
35
|
+
.trim()
|
|
36
|
+
.toLowerCase();
|
|
37
|
+
if (!raw || raw === "auto") {
|
|
38
|
+
return "auto";
|
|
39
|
+
}
|
|
40
|
+
if (raw === "metal" || raw === "vulkan" || raw === "cuda") {
|
|
41
|
+
return raw;
|
|
42
|
+
}
|
|
43
|
+
if (
|
|
44
|
+
raw === "false" ||
|
|
45
|
+
raw === "off" ||
|
|
46
|
+
raw === "none" ||
|
|
47
|
+
raw === "disable" ||
|
|
48
|
+
raw === "disabled" ||
|
|
49
|
+
raw === "0"
|
|
50
|
+
) {
|
|
51
|
+
return false;
|
|
52
|
+
}
|
|
53
|
+
if (!invalidGpuModeWarned) {
|
|
54
|
+
invalidGpuModeWarned = true;
|
|
55
|
+
console.warn(
|
|
56
|
+
`[llama] Invalid GNO_LLAMA_GPU/NODE_LLAMA_CPP_GPU value "${raw}", using auto`
|
|
57
|
+
);
|
|
58
|
+
}
|
|
59
|
+
return "auto";
|
|
60
|
+
}
|
|
61
|
+
|
|
27
62
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
28
63
|
// ModelManager
|
|
29
64
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
@@ -48,11 +83,32 @@ export class ModelManager {
|
|
|
48
83
|
async getLlama(): Promise<Llama> {
|
|
49
84
|
if (!this.llama) {
|
|
50
85
|
const { getLlama, LlamaLogLevel } = await import("node-llama-cpp");
|
|
86
|
+
const gpu = resolveLlamaGpuMode();
|
|
51
87
|
// Suppress model loading warnings (vocab tokens, pooling type)
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
88
|
+
try {
|
|
89
|
+
this.llama = await getLlama({
|
|
90
|
+
build: "autoAttempt",
|
|
91
|
+
gpu,
|
|
92
|
+
logLevel: LlamaLogLevel.error,
|
|
93
|
+
});
|
|
94
|
+
} catch (error) {
|
|
95
|
+
if (gpu === "auto" || gpu === false) {
|
|
96
|
+
throw error;
|
|
97
|
+
}
|
|
98
|
+
if (!gpuFallbackWarned) {
|
|
99
|
+
gpuFallbackWarned = true;
|
|
100
|
+
console.warn(
|
|
101
|
+
`[llama] GPU backend "${gpu}" failed, retrying with CPU: ${
|
|
102
|
+
error instanceof Error ? error.message : String(error)
|
|
103
|
+
}`
|
|
104
|
+
);
|
|
105
|
+
}
|
|
106
|
+
this.llama = await getLlama({
|
|
107
|
+
build: "autoAttempt",
|
|
108
|
+
gpu: false,
|
|
109
|
+
logLevel: LlamaLogLevel.error,
|
|
110
|
+
});
|
|
111
|
+
}
|
|
56
112
|
}
|
|
57
113
|
return this.llama;
|
|
58
114
|
}
|
|
@@ -13,7 +13,12 @@ import { join as pathJoin } from "node:path";
|
|
|
13
13
|
import type { DocumentRow, TagCount } from "../../store/types";
|
|
14
14
|
import type { ToolContext } from "../server";
|
|
15
15
|
|
|
16
|
-
import {
|
|
16
|
+
import {
|
|
17
|
+
buildUri,
|
|
18
|
+
decorateUriForIndex,
|
|
19
|
+
parseUri,
|
|
20
|
+
URI_PREFIX,
|
|
21
|
+
} from "../../app/constants";
|
|
17
22
|
import { MCP_ERRORS } from "../../core/errors";
|
|
18
23
|
import { normalizeTag, validateTag } from "../../core/tags";
|
|
19
24
|
import { normalizeCollectionName } from "../../core/validation";
|
|
@@ -64,7 +69,8 @@ function formatResourceContent(
|
|
|
64
69
|
const langLine = doc.languageHint
|
|
65
70
|
? `\n language: ${doc.languageHint}`
|
|
66
71
|
: "";
|
|
67
|
-
const
|
|
72
|
+
const displayUri = decorateUriForIndex(doc.uri, ctx.indexName);
|
|
73
|
+
const header = `<!-- ${displayUri}
|
|
68
74
|
docid: ${doc.docid}
|
|
69
75
|
source: ${absPath}
|
|
70
76
|
mime: ${doc.sourceMime}${langLine}
|
|
@@ -94,7 +100,7 @@ export function registerResources(server: McpServer, ctx: ToolContext): void {
|
|
|
94
100
|
|
|
95
101
|
return {
|
|
96
102
|
resources: listResult.value.map((doc) => ({
|
|
97
|
-
uri: doc.uri,
|
|
103
|
+
uri: decorateUriForIndex(doc.uri, ctx.indexName),
|
|
98
104
|
name: doc.relPath,
|
|
99
105
|
mimeType: doc.sourceMime || "text/markdown",
|
|
100
106
|
description: doc.title ?? undefined,
|
|
@@ -160,7 +166,10 @@ export function registerResources(server: McpServer, ctx: ToolContext): void {
|
|
|
160
166
|
const formattedContent = formatResourceContent(doc, content, ctx);
|
|
161
167
|
|
|
162
168
|
// Build canonical URI
|
|
163
|
-
const canonicalUri =
|
|
169
|
+
const canonicalUri = decorateUriForIndex(
|
|
170
|
+
buildUri(collection, path),
|
|
171
|
+
parsed.indexName ?? ctx.indexName
|
|
172
|
+
);
|
|
164
173
|
|
|
165
174
|
return {
|
|
166
175
|
contents: [
|
package/src/mcp/server.ts
CHANGED
|
@@ -57,6 +57,7 @@ export interface ToolContext {
|
|
|
57
57
|
config: Config;
|
|
58
58
|
collections: Collection[];
|
|
59
59
|
actualConfigPath: string;
|
|
60
|
+
indexName?: string;
|
|
60
61
|
toolMutex: Mutex;
|
|
61
62
|
jobManager: JobManager;
|
|
62
63
|
serverInstanceId: string;
|
|
@@ -164,6 +165,7 @@ export async function startMcpServer(options: McpServerOptions): Promise<void> {
|
|
|
164
165
|
config,
|
|
165
166
|
collections,
|
|
166
167
|
actualConfigPath,
|
|
168
|
+
indexName: options.indexName,
|
|
167
169
|
toolMutex,
|
|
168
170
|
jobManager,
|
|
169
171
|
serverInstanceId,
|
package/src/mcp/tools/get.ts
CHANGED
|
@@ -9,7 +9,7 @@ import { join as pathJoin } from "node:path";
|
|
|
9
9
|
import type { DocumentRow, StorePort } from "../../store/types";
|
|
10
10
|
import type { ToolContext } from "../server";
|
|
11
11
|
|
|
12
|
-
import { parseUri } from "../../app/constants";
|
|
12
|
+
import { decorateUriForIndex, parseUri } from "../../app/constants";
|
|
13
13
|
import { parseRef } from "../../cli/commands/ref-parser";
|
|
14
14
|
import {
|
|
15
15
|
getDocumentCapabilities,
|
|
@@ -196,7 +196,12 @@ export function handleGet(
|
|
|
196
196
|
|
|
197
197
|
const response: GetResponse = {
|
|
198
198
|
docid: doc.docid,
|
|
199
|
-
uri:
|
|
199
|
+
uri: decorateUriForIndex(
|
|
200
|
+
doc.uri,
|
|
201
|
+
parsed.type === "uri"
|
|
202
|
+
? (parseUri(parsed.value)?.indexName ?? ctx.indexName)
|
|
203
|
+
: ctx.indexName
|
|
204
|
+
),
|
|
200
205
|
title: doc.title ?? undefined,
|
|
201
206
|
content,
|
|
202
207
|
totalLines,
|
|
@@ -9,7 +9,7 @@ import { join as pathJoin } from "node:path";
|
|
|
9
9
|
import type { DocumentRow, StorePort } from "../../store/types";
|
|
10
10
|
import type { ToolContext } from "../server";
|
|
11
11
|
|
|
12
|
-
import { parseUri } from "../../app/constants";
|
|
12
|
+
import { decorateUriForIndex, parseUri } from "../../app/constants";
|
|
13
13
|
import { parseRef } from "../../cli/commands/ref-parser";
|
|
14
14
|
import { runTool, type ToolResult } from "./index";
|
|
15
15
|
|
|
@@ -232,7 +232,7 @@ export function handleMultiGet(
|
|
|
232
232
|
|
|
233
233
|
documents.push({
|
|
234
234
|
docid: doc.docid,
|
|
235
|
-
uri: doc.uri,
|
|
235
|
+
uri: decorateUriForIndex(doc.uri, ctx.indexName),
|
|
236
236
|
title: doc.title ?? undefined,
|
|
237
237
|
content,
|
|
238
238
|
totalLines: (contentResult.value ?? "").split("\n").length,
|
package/src/mcp/tools/query.ts
CHANGED
|
@@ -18,7 +18,7 @@ import type {
|
|
|
18
18
|
} from "../../pipeline/types";
|
|
19
19
|
import type { ToolContext } from "../server";
|
|
20
20
|
|
|
21
|
-
import { parseUri } from "../../app/constants";
|
|
21
|
+
import { decorateUriForIndex, parseUri } from "../../app/constants";
|
|
22
22
|
import { createNonTtyProgressRenderer } from "../../cli/progress";
|
|
23
23
|
import { resolveDepthPolicy } from "../../core/depth-policy";
|
|
24
24
|
import { normalizeStructuredQueryInput } from "../../core/structured-query";
|
|
@@ -76,6 +76,7 @@ function enrichWithAbsPath(
|
|
|
76
76
|
|
|
77
77
|
return {
|
|
78
78
|
...r,
|
|
79
|
+
uri: decorateUriForIndex(r.uri, ctx.indexName),
|
|
79
80
|
source: {
|
|
80
81
|
...r.source,
|
|
81
82
|
absPath: pathJoin(collection.path, r.source.relPath),
|
package/src/mcp/tools/search.ts
CHANGED
|
@@ -9,7 +9,7 @@ import { join as pathJoin } from "node:path";
|
|
|
9
9
|
import type { SearchResult, SearchResults } from "../../pipeline/types";
|
|
10
10
|
import type { ToolContext } from "../server";
|
|
11
11
|
|
|
12
|
-
import { parseUri } from "../../app/constants";
|
|
12
|
+
import { decorateUriForIndex, parseUri } from "../../app/constants";
|
|
13
13
|
import { searchBm25 } from "../../pipeline/search";
|
|
14
14
|
import { normalizeTagFilters, runTool, type ToolResult } from "./index";
|
|
15
15
|
|
|
@@ -51,6 +51,7 @@ function enrichWithAbsPath(
|
|
|
51
51
|
|
|
52
52
|
return {
|
|
53
53
|
...r,
|
|
54
|
+
uri: decorateUriForIndex(r.uri, ctx.indexName),
|
|
54
55
|
source: {
|
|
55
56
|
...r.source,
|
|
56
57
|
absPath: pathJoin(collection.path, r.source.relPath),
|
package/src/mcp/tools/vsearch.ts
CHANGED
|
@@ -9,7 +9,7 @@ import { join as pathJoin } from "node:path";
|
|
|
9
9
|
import type { SearchResult, SearchResults } from "../../pipeline/types";
|
|
10
10
|
import type { ToolContext } from "../server";
|
|
11
11
|
|
|
12
|
-
import { parseUri } from "../../app/constants";
|
|
12
|
+
import { decorateUriForIndex, parseUri } from "../../app/constants";
|
|
13
13
|
import { createNonTtyProgressRenderer } from "../../cli/progress";
|
|
14
14
|
import { LlmAdapter } from "../../llm/nodeLlamaCpp/adapter";
|
|
15
15
|
import { resolveDownloadPolicy } from "../../llm/policy";
|
|
@@ -60,6 +60,7 @@ function enrichWithAbsPath(
|
|
|
60
60
|
|
|
61
61
|
return {
|
|
62
62
|
...r,
|
|
63
|
+
uri: decorateUriForIndex(r.uri, ctx.indexName),
|
|
63
64
|
source: {
|
|
64
65
|
...r.source,
|
|
65
66
|
absPath: pathJoin(collection.path, r.source.relPath),
|
package/src/pipeline/explain.ts
CHANGED
|
@@ -122,9 +122,19 @@ export function explainBm25(count: number): ExplainLine {
|
|
|
122
122
|
return { stage: "bm25", message: `${count} candidates` };
|
|
123
123
|
}
|
|
124
124
|
|
|
125
|
-
export function explainVector(
|
|
125
|
+
export function explainVector(
|
|
126
|
+
count: number,
|
|
127
|
+
available: boolean,
|
|
128
|
+
unavailableReason?: string,
|
|
129
|
+
guidance?: string
|
|
130
|
+
): ExplainLine {
|
|
126
131
|
if (!available) {
|
|
127
|
-
|
|
132
|
+
const detail = unavailableReason ? `: ${unavailableReason}` : "";
|
|
133
|
+
const hint = guidance ? `; ${guidance}` : "";
|
|
134
|
+
return {
|
|
135
|
+
stage: "vector",
|
|
136
|
+
message: `unavailable (sqlite-vec not loaded${detail})${hint}`,
|
|
137
|
+
};
|
|
128
138
|
}
|
|
129
139
|
return { stage: "vector", message: `${count} candidates` };
|
|
130
140
|
}
|
package/src/pipeline/hybrid.ts
CHANGED
|
@@ -528,7 +528,14 @@ export async function searchHybrid(
|
|
|
528
528
|
}
|
|
529
529
|
timings.vectorMs = performance.now() - vectorStartedAt;
|
|
530
530
|
|
|
531
|
-
explainLines.push(
|
|
531
|
+
explainLines.push(
|
|
532
|
+
explainVector(
|
|
533
|
+
vecCount,
|
|
534
|
+
vectorAvailable,
|
|
535
|
+
vectorIndex?.loadError,
|
|
536
|
+
vectorIndex?.guidance
|
|
537
|
+
)
|
|
538
|
+
);
|
|
532
539
|
|
|
533
540
|
// ─────────────────────────────────────────────────────────────────────────
|
|
534
541
|
// 3. RRF Fusion
|
|
@@ -800,6 +807,7 @@ export async function searchHybrid(
|
|
|
800
807
|
score: candidate.blendedScore,
|
|
801
808
|
uri: doc.uri,
|
|
802
809
|
title: doc.title ?? undefined,
|
|
810
|
+
line: snippetChunk.startLine,
|
|
803
811
|
snippet,
|
|
804
812
|
snippetLanguage: chunk.language ?? undefined,
|
|
805
813
|
snippetRange,
|
package/src/pipeline/search.ts
CHANGED
|
@@ -15,6 +15,7 @@ import type {
|
|
|
15
15
|
SearchResults,
|
|
16
16
|
} from "./types";
|
|
17
17
|
|
|
18
|
+
import { getContentBatch } from "../store/content-batch";
|
|
18
19
|
import { err, ok } from "../store/types";
|
|
19
20
|
import { createChunkLookup } from "./chunk-lookup";
|
|
20
21
|
import { matchesExcludedChunks, matchesExcludedText } from "./exclude";
|
|
@@ -116,6 +117,7 @@ function buildSearchResult(ctx: BuildResultContext): SearchResult {
|
|
|
116
117
|
score: fts.score, // Raw score, normalized later as batch
|
|
117
118
|
uri: fts.uri ?? "",
|
|
118
119
|
title: fts.title,
|
|
120
|
+
line: chunk?.startLine,
|
|
119
121
|
snippet,
|
|
120
122
|
snippetLanguage: chunk?.language ?? undefined,
|
|
121
123
|
snippetRange,
|
|
@@ -277,14 +279,21 @@ export async function searchBm25(
|
|
|
277
279
|
const sortedEntries = [...bestByDocid.values()].sort(
|
|
278
280
|
(a, b) => a.score - b.score
|
|
279
281
|
);
|
|
282
|
+
const fullContentResult = await getContentBatch(
|
|
283
|
+
store,
|
|
284
|
+
sortedEntries
|
|
285
|
+
.map(({ fts }) => fts.mirrorHash)
|
|
286
|
+
.filter((mirrorHash): mirrorHash is string => Boolean(mirrorHash))
|
|
287
|
+
);
|
|
288
|
+
if (!fullContentResult.ok) {
|
|
289
|
+
return err("QUERY_FAILED", fullContentResult.error.message);
|
|
290
|
+
}
|
|
291
|
+
const fullContentByHash = fullContentResult.value;
|
|
292
|
+
|
|
280
293
|
for (const { fts, chunk } of sortedEntries) {
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
if (contentResult.ok && contentResult.value) {
|
|
285
|
-
fullContent = contentResult.value;
|
|
286
|
-
}
|
|
287
|
-
}
|
|
294
|
+
const fullContent = fts.mirrorHash
|
|
295
|
+
? fullContentByHash.get(fts.mirrorHash)
|
|
296
|
+
: undefined;
|
|
288
297
|
const collectionPath = fts.collection
|
|
289
298
|
? collectionPaths.get(fts.collection)
|
|
290
299
|
: undefined;
|
package/src/pipeline/types.ts
CHANGED
package/src/pipeline/vsearch.ts
CHANGED
|
@@ -11,6 +11,7 @@ import type { StorePort } from "../store/types";
|
|
|
11
11
|
import type { VectorIndexPort } from "../store/vector/types";
|
|
12
12
|
import type { SearchOptions, SearchResult, SearchResults } from "./types";
|
|
13
13
|
|
|
14
|
+
import { getContentBatch } from "../store/content-batch";
|
|
14
15
|
import { err, ok } from "../store/types";
|
|
15
16
|
import { createChunkLookup } from "./chunk-lookup";
|
|
16
17
|
import { formatQueryForEmbedding } from "./contextual";
|
|
@@ -49,6 +50,16 @@ export interface VectorSearchDeps {
|
|
|
49
50
|
config: Config;
|
|
50
51
|
}
|
|
51
52
|
|
|
53
|
+
function vectorUnavailableMessage(vectorIndex: VectorIndexPort): string {
|
|
54
|
+
const reason = vectorIndex.loadError
|
|
55
|
+
? ` Reason: ${vectorIndex.loadError}`
|
|
56
|
+
: "";
|
|
57
|
+
const guidance = vectorIndex.guidance
|
|
58
|
+
? ` ${vectorIndex.guidance}`
|
|
59
|
+
: " Run: gno doctor";
|
|
60
|
+
return `Vector search unavailable.${reason}${guidance}`;
|
|
61
|
+
}
|
|
62
|
+
|
|
52
63
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
53
64
|
// Search Function (with pre-computed embedding)
|
|
54
65
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
@@ -81,10 +92,7 @@ export async function searchVectorWithEmbedding(
|
|
|
81
92
|
|
|
82
93
|
// Check if vector search is available
|
|
83
94
|
if (!vectorIndex.searchAvailable) {
|
|
84
|
-
return err(
|
|
85
|
-
"VEC_SEARCH_UNAVAILABLE",
|
|
86
|
-
"Vector search requires sqlite-vec. Run: gno embed"
|
|
87
|
-
);
|
|
95
|
+
return err("VEC_SEARCH_UNAVAILABLE", vectorUnavailableMessage(vectorIndex));
|
|
88
96
|
}
|
|
89
97
|
|
|
90
98
|
// Search nearest neighbors
|
|
@@ -219,6 +227,7 @@ export async function searchVectorWithEmbedding(
|
|
|
219
227
|
score,
|
|
220
228
|
uri: doc.uri,
|
|
221
229
|
title: doc.title ?? undefined,
|
|
230
|
+
line: chunk.startLine,
|
|
222
231
|
snippet: chunk.text,
|
|
223
232
|
snippetLanguage: chunk.language ?? undefined,
|
|
224
233
|
snippetRange: {
|
|
@@ -249,14 +258,21 @@ export async function searchVectorWithEmbedding(
|
|
|
249
258
|
|
|
250
259
|
// For --full, fetch full content and build results
|
|
251
260
|
if (options.full) {
|
|
261
|
+
const fullContentResult = await getContentBatch(
|
|
262
|
+
store,
|
|
263
|
+
[...bestByDocid.values()]
|
|
264
|
+
.map(({ doc }) => doc.mirrorHash)
|
|
265
|
+
.filter((mirrorHash): mirrorHash is string => Boolean(mirrorHash))
|
|
266
|
+
);
|
|
267
|
+
if (!fullContentResult.ok) {
|
|
268
|
+
return err("QUERY_FAILED", fullContentResult.error.message);
|
|
269
|
+
}
|
|
270
|
+
const fullContentByHash = fullContentResult.value;
|
|
271
|
+
|
|
252
272
|
for (const { doc, chunk, score } of bestByDocid.values()) {
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
if (contentResult.ok && contentResult.value) {
|
|
257
|
-
fullContent = contentResult.value;
|
|
258
|
-
}
|
|
259
|
-
}
|
|
273
|
+
const fullContent = doc.mirrorHash
|
|
274
|
+
? fullContentByHash.get(doc.mirrorHash)
|
|
275
|
+
: undefined;
|
|
260
276
|
|
|
261
277
|
const collectionPath = collectionPaths.get(doc.collection);
|
|
262
278
|
|
|
@@ -265,6 +281,7 @@ export async function searchVectorWithEmbedding(
|
|
|
265
281
|
score,
|
|
266
282
|
uri: doc.uri,
|
|
267
283
|
title: doc.title ?? undefined,
|
|
284
|
+
line: chunk.startLine,
|
|
268
285
|
snippet: fullContent ?? chunk.text,
|
|
269
286
|
snippetLanguage: chunk.language ?? undefined,
|
|
270
287
|
// --full: no snippetRange (full doc content)
|
|
@@ -346,10 +363,7 @@ export async function searchVector(
|
|
|
346
363
|
|
|
347
364
|
// Check if vector search is available
|
|
348
365
|
if (!vectorIndex.searchAvailable) {
|
|
349
|
-
return err(
|
|
350
|
-
"VEC_SEARCH_UNAVAILABLE",
|
|
351
|
-
"Vector search requires sqlite-vec. Run: gno embed"
|
|
352
|
-
);
|
|
366
|
+
return err("VEC_SEARCH_UNAVAILABLE", vectorUnavailableMessage(vectorIndex));
|
|
353
367
|
}
|
|
354
368
|
|
|
355
369
|
// Embed query with contextual formatting
|
|
@@ -9,6 +9,7 @@ import type { DocumentRow, StorePort, TagRow } from "../store/types";
|
|
|
9
9
|
|
|
10
10
|
import { parseRef } from "../cli/commands/ref-parser";
|
|
11
11
|
import { parseFrontmatter } from "../ingestion/frontmatter";
|
|
12
|
+
import { getContentBatch } from "../store/content-batch";
|
|
12
13
|
import {
|
|
13
14
|
buildEncryptedPublishArtifact,
|
|
14
15
|
buildPublishArtifact,
|
|
@@ -155,16 +156,40 @@ async function exportCollectionArtifact(
|
|
|
155
156
|
throw new Error(`Collection "${collection.name}" has no active documents`);
|
|
156
157
|
}
|
|
157
158
|
|
|
159
|
+
const contentResult = await getContentBatch(
|
|
160
|
+
store,
|
|
161
|
+
activeDocs
|
|
162
|
+
.map((doc) => doc.mirrorHash)
|
|
163
|
+
.filter((mirrorHash): mirrorHash is string => Boolean(mirrorHash))
|
|
164
|
+
);
|
|
165
|
+
if (!contentResult.ok) {
|
|
166
|
+
throw new Error(contentResult.error.message);
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
const tagsResult = await store.getTagsBatch(activeDocs.map((doc) => doc.id));
|
|
170
|
+
if (!tagsResult.ok) {
|
|
171
|
+
throw new Error(tagsResult.error.message);
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
const contentByHash = contentResult.value;
|
|
175
|
+
const tagsByDocId = tagsResult.value;
|
|
176
|
+
|
|
158
177
|
const notes: PublishArtifactNote[] = [];
|
|
159
178
|
for (const doc of activeDocs) {
|
|
160
|
-
|
|
179
|
+
if (!doc.mirrorHash) {
|
|
180
|
+
throw new Error(`Document has no converted content: ${doc.uri}`);
|
|
181
|
+
}
|
|
182
|
+
const rawMarkdown = contentByHash.get(doc.mirrorHash);
|
|
183
|
+
if (rawMarkdown === undefined) {
|
|
184
|
+
throw new Error(`Unable to load content for ${doc.uri}`);
|
|
185
|
+
}
|
|
161
186
|
if (isPublishDisabledByFrontmatter(rawMarkdown)) {
|
|
162
187
|
continue;
|
|
163
188
|
}
|
|
164
189
|
const sanitized = sanitizeObsidianMarkdown(rawMarkdown);
|
|
165
190
|
warnings.push(...sanitized.warnings);
|
|
166
191
|
const markdown = sanitized.markdown;
|
|
167
|
-
const tags =
|
|
192
|
+
const tags = tagsByDocId.get(doc.id) ?? [];
|
|
168
193
|
const frontmatter = parseFrontmatter(markdown).metadata;
|
|
169
194
|
const title = deriveExportedTitle(doc);
|
|
170
195
|
notes.push({
|
package/src/sdk/client.ts
CHANGED
|
@@ -37,7 +37,11 @@ import type {
|
|
|
37
37
|
GnoVectorSearchOptions,
|
|
38
38
|
} from "./types";
|
|
39
39
|
|
|
40
|
-
import {
|
|
40
|
+
import {
|
|
41
|
+
decorateUriForIndex,
|
|
42
|
+
getIndexDbPath,
|
|
43
|
+
parseUri,
|
|
44
|
+
} from "../app/constants";
|
|
41
45
|
import { ConfigSchema, loadConfig } from "../config";
|
|
42
46
|
import {
|
|
43
47
|
atomicWrite,
|
|
@@ -88,6 +92,7 @@ interface OpenedClientState {
|
|
|
88
92
|
store: SqliteAdapter;
|
|
89
93
|
llm: LlmAdapter;
|
|
90
94
|
downloadPolicy: DownloadPolicy;
|
|
95
|
+
indexName?: string;
|
|
91
96
|
}
|
|
92
97
|
|
|
93
98
|
interface RuntimePorts {
|
|
@@ -158,6 +163,7 @@ async function resolveClientState(
|
|
|
158
163
|
llm: new LlmAdapter(config, options.cacheDir),
|
|
159
164
|
downloadPolicy:
|
|
160
165
|
options.downloadPolicy ?? resolveDownloadPolicy(process.env, {}),
|
|
166
|
+
indexName: options.indexName,
|
|
161
167
|
};
|
|
162
168
|
}
|
|
163
169
|
|
|
@@ -170,6 +176,7 @@ class GnoClientImpl implements GnoClient {
|
|
|
170
176
|
private readonly store: SqliteAdapter;
|
|
171
177
|
private readonly llm: LlmAdapter;
|
|
172
178
|
private readonly downloadPolicy: DownloadPolicy;
|
|
179
|
+
private readonly indexName?: string;
|
|
173
180
|
private closed = false;
|
|
174
181
|
|
|
175
182
|
constructor(state: OpenedClientState) {
|
|
@@ -180,6 +187,7 @@ class GnoClientImpl implements GnoClient {
|
|
|
180
187
|
this.store = state.store;
|
|
181
188
|
this.llm = state.llm;
|
|
182
189
|
this.downloadPolicy = state.downloadPolicy;
|
|
190
|
+
this.indexName = state.indexName;
|
|
183
191
|
}
|
|
184
192
|
|
|
185
193
|
isOpen(): boolean {
|
|
@@ -371,12 +379,24 @@ class GnoClientImpl implements GnoClient {
|
|
|
371
379
|
}
|
|
372
380
|
}
|
|
373
381
|
|
|
382
|
+
private decorateSearchResults(results: SearchResults): SearchResults {
|
|
383
|
+
return {
|
|
384
|
+
...results,
|
|
385
|
+
results: results.results.map((result) => ({
|
|
386
|
+
...result,
|
|
387
|
+
uri: decorateUriForIndex(result.uri, this.indexName),
|
|
388
|
+
})),
|
|
389
|
+
};
|
|
390
|
+
}
|
|
391
|
+
|
|
374
392
|
async search(
|
|
375
393
|
query: string,
|
|
376
394
|
options: import("../pipeline/types").SearchOptions = {}
|
|
377
395
|
): Promise<SearchResults> {
|
|
378
396
|
this.assertOpen();
|
|
379
|
-
return
|
|
397
|
+
return this.decorateSearchResults(
|
|
398
|
+
unwrapStore(await searchBm25(this.store, query, options))
|
|
399
|
+
);
|
|
380
400
|
}
|
|
381
401
|
|
|
382
402
|
async vsearch(
|
|
@@ -409,17 +429,19 @@ class GnoClientImpl implements GnoClient {
|
|
|
409
429
|
});
|
|
410
430
|
}
|
|
411
431
|
|
|
412
|
-
return
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
432
|
+
return this.decorateSearchResults(
|
|
433
|
+
unwrapStore(
|
|
434
|
+
await searchVectorWithEmbedding(
|
|
435
|
+
{
|
|
436
|
+
store: this.store,
|
|
437
|
+
vectorIndex: ports.vectorIndex,
|
|
438
|
+
embedPort: ports.embedPort,
|
|
439
|
+
config: this.config,
|
|
440
|
+
},
|
|
441
|
+
query,
|
|
442
|
+
new Float32Array(queryEmbedResult.value),
|
|
443
|
+
options
|
|
444
|
+
)
|
|
423
445
|
)
|
|
424
446
|
);
|
|
425
447
|
} finally {
|
|
@@ -461,18 +483,20 @@ class GnoClientImpl implements GnoClient {
|
|
|
461
483
|
});
|
|
462
484
|
|
|
463
485
|
try {
|
|
464
|
-
return
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
486
|
+
return this.decorateSearchResults(
|
|
487
|
+
unwrapStore(
|
|
488
|
+
await searchHybrid(
|
|
489
|
+
{
|
|
490
|
+
store: this.store,
|
|
491
|
+
config: this.config,
|
|
492
|
+
vectorIndex: ports.vectorIndex,
|
|
493
|
+
embedPort: ports.embedPort,
|
|
494
|
+
expandPort: ports.expandPort,
|
|
495
|
+
rerankPort: ports.rerankPort,
|
|
496
|
+
},
|
|
497
|
+
query,
|
|
498
|
+
options
|
|
499
|
+
)
|
|
476
500
|
)
|
|
477
501
|
);
|
|
478
502
|
} finally {
|
|
@@ -606,17 +630,48 @@ class GnoClientImpl implements GnoClient {
|
|
|
606
630
|
|
|
607
631
|
async get(ref: string, options: GnoGetOptions = {}) {
|
|
608
632
|
this.assertOpen();
|
|
609
|
-
|
|
633
|
+
const explicitIndex = ref.startsWith("gno://")
|
|
634
|
+
? parseUri(ref)?.indexName
|
|
635
|
+
: undefined;
|
|
636
|
+
const result = await getDocumentByRef(
|
|
637
|
+
this.store,
|
|
638
|
+
this.config,
|
|
639
|
+
ref,
|
|
640
|
+
options
|
|
641
|
+
);
|
|
642
|
+
return {
|
|
643
|
+
...result,
|
|
644
|
+
uri: decorateUriForIndex(result.uri, explicitIndex ?? this.indexName),
|
|
645
|
+
};
|
|
610
646
|
}
|
|
611
647
|
|
|
612
648
|
async multiGet(refs: string[], options: GnoMultiGetOptions = {}) {
|
|
613
649
|
this.assertOpen();
|
|
614
|
-
|
|
650
|
+
const result = await multiGetDocuments(
|
|
651
|
+
this.store,
|
|
652
|
+
this.config,
|
|
653
|
+
refs,
|
|
654
|
+
options
|
|
655
|
+
);
|
|
656
|
+
return {
|
|
657
|
+
...result,
|
|
658
|
+
documents: result.documents.map((doc) => ({
|
|
659
|
+
...doc,
|
|
660
|
+
uri: decorateUriForIndex(doc.uri, this.indexName),
|
|
661
|
+
})),
|
|
662
|
+
};
|
|
615
663
|
}
|
|
616
664
|
|
|
617
665
|
async list(options: GnoListOptions = {}) {
|
|
618
666
|
this.assertOpen();
|
|
619
|
-
|
|
667
|
+
const result = await listDocuments(this.store, options);
|
|
668
|
+
return {
|
|
669
|
+
...result,
|
|
670
|
+
documents: result.documents.map((doc) => ({
|
|
671
|
+
...doc,
|
|
672
|
+
uri: decorateUriForIndex(doc.uri, this.indexName),
|
|
673
|
+
})),
|
|
674
|
+
};
|
|
620
675
|
}
|
|
621
676
|
|
|
622
677
|
async status(): Promise<IndexStatus> {
|