@gmickel/gno 2.3.3 → 2.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -3
- package/assets/skill/SKILL.md +53 -0
- package/assets/skill/cli-reference.md +29 -0
- package/assets/skill/examples.md +30 -0
- package/assets/skill/mcp-reference.md +26 -0
- package/assets/spa-production.json.gz +0 -0
- package/browser-extension/artifacts/{gno-browser-clipper-v2.3.3.zip → gno-browser-clipper-v2.5.1.zip} +0 -0
- package/browser-extension/artifacts/gno-browser-clipper-v2.5.1.zip.sha256 +1 -0
- package/browser-extension/dist/manifest.json +1 -1
- package/package.json +1 -1
- package/spec/cli.md +41 -0
- package/spec/compiled-context.md +68 -0
- package/spec/db/schema.sql +2 -0
- package/spec/evals.md +48 -0
- package/spec/mcp.md +37 -0
- package/spec/output-schemas/ask.schema.json +12 -0
- package/spec/output-schemas/compiled-context-check.schema.json +44 -0
- package/spec/output-schemas/compiled-context-file.schema.json +165 -0
- package/spec/output-schemas/compiled-context-preview.schema.json +142 -0
- package/spec/output-schemas/context-capsule-v1.schema.json +205 -32
- package/spec/output-schemas/context-capsule-verification.schema.json +32 -9
- package/spec/output-schemas/evidence-fixtures.schema.json +176 -0
- package/spec/output-schemas/evidence-report.schema.json +322 -0
- package/spec/output-schemas/evidence-run.schema.json +277 -0
- package/spec/output-schemas/metadata-predicate.schema.json +460 -0
- package/spec/output-schemas/retrieval-trace-filters.schema.json +13 -4
- package/spec/output-schemas/search-results.schema.json +12 -0
- package/spec/output-schemas/status.schema.json +37 -6
- package/src/app/compiled-context-files.ts +361 -0
- package/src/app/compiled-context.ts +240 -0
- package/src/app/context-runtime-contract.ts +33 -2
- package/src/app/context-runtime-input.ts +17 -0
- package/src/app/context-runtime-types.ts +2 -0
- package/src/app/context-runtime.ts +27 -3
- package/src/app/context-surface.ts +7 -0
- package/src/app/verified-ask.ts +1 -0
- package/src/cli/commands/ask.ts +1 -0
- package/src/cli/commands/audit.ts +4 -0
- package/src/cli/commands/context-build.ts +1 -0
- package/src/cli/commands/context-compiled.ts +136 -0
- package/src/cli/commands/search.ts +1 -0
- package/src/cli/commands/status.ts +10 -0
- package/src/cli/errors.ts +10 -3
- package/src/cli/options.ts +19 -0
- package/src/cli/program.ts +82 -0
- package/src/converters/types.ts +2 -0
- package/src/core/compiled-context.ts +254 -0
- package/src/core/context-budget.ts +2 -10
- package/src/core/context-capsule-retrieval-schema.ts +8 -0
- package/src/core/context-capsule-schema.ts +47 -3
- package/src/core/context-capsule-validation.ts +14 -2
- package/src/core/context-capsule-verification.ts +1 -0
- package/src/core/context-capsule.ts +16 -2
- package/src/core/context-compiler.ts +3 -0
- package/src/core/file-lock.ts +22 -5
- package/src/core/folder-setup-planning.ts +2 -1
- package/src/core/network-boundary-inventory.ts +8 -0
- package/src/core/record-metadata.ts +1 -1
- package/src/core/retrieval-replay-candidate.ts +5 -0
- package/src/core/retrieval-trace-filter-normalization.ts +4 -0
- package/src/core/retrieval-trace-request.ts +1 -0
- package/src/core/retrieval-trace-session.ts +5 -1
- package/src/core/retrieval-trace.ts +2 -0
- package/src/core/setup-receipt.ts +27 -20
- package/src/core/typed-metadata.ts +228 -0
- package/src/core/validation.ts +9 -2
- package/src/core/windows-private-path.ts +96 -0
- package/src/index.ts +2 -2
- package/src/ingestion/compiled-context.ts +15 -0
- package/src/ingestion/record-adapter-canonical.ts +13 -2
- package/src/ingestion/record-container.ts +9 -0
- package/src/ingestion/sync.ts +51 -9
- package/src/ingestion/typed-metadata.ts +55 -0
- package/src/ingestion/walker.ts +5 -4
- package/src/llm/nodeLlamaCpp/simulator-install.ts +6 -2
- package/src/mcp/http-egress.ts +11 -3
- package/src/mcp/retrieval-warnings.ts +24 -0
- package/src/mcp/tools/ask.ts +23 -2
- package/src/mcp/tools/context.ts +47 -2
- package/src/mcp/tools/index.ts +58 -1
- package/src/mcp/tools/query.ts +16 -1
- package/src/mcp/tools/search.ts +12 -1
- package/src/mcp/tools/status.ts +4 -0
- package/src/mcp/tools/vsearch.ts +12 -1
- package/src/pipeline/diagnose.ts +6 -0
- package/src/pipeline/filters.ts +65 -0
- package/src/pipeline/graph-retrieval.ts +9 -1
- package/src/pipeline/hybrid.ts +22 -2
- package/src/pipeline/search.ts +11 -0
- package/src/pipeline/types.ts +7 -1
- package/src/pipeline/vsearch.ts +21 -1
- package/src/sdk/client.ts +98 -3
- package/src/sdk/index.ts +11 -0
- package/src/sdk/types.ts +35 -1
- package/src/serve/compiled-context.ts +84 -0
- package/src/serve/context-capsule.ts +1 -0
- package/src/serve/public/app.tsx +12 -1
- package/src/serve/public/components/MetadataFilter.tsx +186 -0
- package/src/serve/public/globals.built.css +1 -1
- package/src/serve/public/lib/metadata-filter.ts +28 -0
- package/src/serve/public/lib/retrieval-filters.ts +3 -0
- package/src/serve/public/lib/workspace-tabs.ts +2 -0
- package/src/serve/public/pages/Ask.tsx +48 -3
- package/src/serve/public/pages/CompiledContext.tsx +364 -0
- package/src/serve/public/pages/Dashboard.tsx +7 -0
- package/src/serve/public/pages/DocView.tsx +33 -0
- package/src/serve/public/pages/Search.tsx +44 -1
- package/src/serve/routes/api.ts +50 -0
- package/src/serve/server.ts +43 -2
- package/src/serve/spa-production-build.ts +6 -5
- package/src/store/migrations/030-typed-metadata.ts +13 -0
- package/src/store/migrations/index.ts +2 -0
- package/src/store/sqlite/adapter.ts +69 -5
- package/src/store/sqlite/eligibility.ts +12 -0
- package/src/store/sqlite/metadata-predicate.ts +71 -0
- package/src/store/types.ts +11 -0
- package/browser-extension/artifacts/gno-browser-clipper-v2.3.3.zip.sha256 +0 -1
package/src/mcp/tools/vsearch.ts
CHANGED
|
@@ -22,6 +22,10 @@ import {
|
|
|
22
22
|
attachRetrievalTraceMetadata,
|
|
23
23
|
type RetrievalTraceSession,
|
|
24
24
|
} from "../../core/retrieval-trace-session";
|
|
25
|
+
import {
|
|
26
|
+
normalizeMetadataPredicate,
|
|
27
|
+
type MetadataPredicate,
|
|
28
|
+
} from "../../core/typed-metadata";
|
|
25
29
|
import { LlmAdapter } from "../../llm/nodeLlamaCpp/adapter";
|
|
26
30
|
import { resolveDownloadPolicy } from "../../llm/policy";
|
|
27
31
|
import { resolveModelUri } from "../../llm/registry";
|
|
@@ -31,6 +35,7 @@ import {
|
|
|
31
35
|
type VectorSearchDeps,
|
|
32
36
|
} from "../../pipeline/vsearch";
|
|
33
37
|
import { createVectorIndexPort } from "../../store/vector";
|
|
38
|
+
import { appendRetrievalWarnings } from "../retrieval-warnings";
|
|
34
39
|
import { normalizeTagFilters, runTool, type ToolResult } from "./index";
|
|
35
40
|
|
|
36
41
|
interface VsearchInput {
|
|
@@ -46,6 +51,7 @@ interface VsearchInput {
|
|
|
46
51
|
until?: string;
|
|
47
52
|
categories?: string[];
|
|
48
53
|
author?: string;
|
|
54
|
+
filter?: MetadataPredicate;
|
|
49
55
|
tagsAll?: string[];
|
|
50
56
|
tagsAny?: string[];
|
|
51
57
|
}
|
|
@@ -151,6 +157,10 @@ export function handleVsearch(
|
|
|
151
157
|
until: args.until,
|
|
152
158
|
categories: args.categories,
|
|
153
159
|
author: args.author,
|
|
160
|
+
filter:
|
|
161
|
+
args.filter === undefined
|
|
162
|
+
? undefined
|
|
163
|
+
: normalizeMetadataPredicate(args.filter),
|
|
154
164
|
tagsAll: normalizeTagFilters(args.tagsAll),
|
|
155
165
|
tagsAny: normalizeTagFilters(args.tagsAny),
|
|
156
166
|
projectAffinity,
|
|
@@ -271,6 +281,7 @@ export function handleVsearch(
|
|
|
271
281
|
await embedPort?.dispose();
|
|
272
282
|
}
|
|
273
283
|
},
|
|
274
|
-
|
|
284
|
+
(data) =>
|
|
285
|
+
appendRetrievalWarnings(formatSearchResults(data), data.meta.warnings)
|
|
275
286
|
);
|
|
276
287
|
}
|
package/src/pipeline/diagnose.ts
CHANGED
|
@@ -19,6 +19,7 @@ import {
|
|
|
19
19
|
normalizeContentTypes,
|
|
20
20
|
} from "../config";
|
|
21
21
|
import { resolveDocRef } from "../core/ref-parser";
|
|
22
|
+
import { normalizeMetadataPredicate } from "../core/typed-metadata";
|
|
22
23
|
import { err, ok } from "../store/types";
|
|
23
24
|
import {
|
|
24
25
|
getContentTypeBoostMetadata,
|
|
@@ -184,6 +185,11 @@ export async function diagnoseQueryTarget(
|
|
|
184
185
|
query: string,
|
|
185
186
|
options: QueryDiagnoseOptions
|
|
186
187
|
): Promise<StoreResult<QueryDiagnoseResult>> {
|
|
188
|
+
if (options.filter !== undefined)
|
|
189
|
+
options = {
|
|
190
|
+
...options,
|
|
191
|
+
filter: normalizeMetadataPredicate(options.filter),
|
|
192
|
+
};
|
|
187
193
|
const resolved = await resolveDocRef(deps.store, options.target);
|
|
188
194
|
if ("error" in resolved) {
|
|
189
195
|
return ok(buildBaseResult(query, options.target, "not_found", null));
|
package/src/pipeline/filters.ts
CHANGED
|
@@ -7,6 +7,11 @@
|
|
|
7
7
|
import type { ChunkRow, DocumentRow, StorePort } from "../store/types";
|
|
8
8
|
import type { HybridSearchOptions } from "./types";
|
|
9
9
|
|
|
10
|
+
import {
|
|
11
|
+
matchesMetadataPredicate,
|
|
12
|
+
TYPED_METADATA_INGEST_VERSION,
|
|
13
|
+
type MetadataPredicate,
|
|
14
|
+
} from "../core/typed-metadata";
|
|
10
15
|
import { matchesExcludedChunks, matchesExcludedText } from "./exclude";
|
|
11
16
|
import { isWithinTemporalRange, resolveTemporalRange } from "./temporal";
|
|
12
17
|
|
|
@@ -85,6 +90,8 @@ export function evaluateDocumentChunkFilters(
|
|
|
85
90
|
options: HybridSearchOptions
|
|
86
91
|
): QueryFilterEvaluation {
|
|
87
92
|
const reasons: string[] = [];
|
|
93
|
+
const metadataReason = typedMetadataFilterReason(doc, options.filter);
|
|
94
|
+
if (metadataReason) reasons.push(metadataReason);
|
|
88
95
|
const temporalRange = resolveTemporalRange(
|
|
89
96
|
query,
|
|
90
97
|
options.since,
|
|
@@ -180,3 +187,61 @@ export async function evaluateQueryTargetFilters(
|
|
|
180
187
|
reasons,
|
|
181
188
|
};
|
|
182
189
|
}
|
|
190
|
+
|
|
191
|
+
/** Invalid or unextracted documents are ineligible even for negated predicates. */
|
|
192
|
+
export function typedMetadataFilterReason(
|
|
193
|
+
doc: DocumentRow,
|
|
194
|
+
filter?: MetadataPredicate
|
|
195
|
+
): string | undefined {
|
|
196
|
+
if (!filter) return undefined;
|
|
197
|
+
if ((doc.ingestVersion ?? 0) < TYPED_METADATA_INGEST_VERSION)
|
|
198
|
+
return "metadata_backfill";
|
|
199
|
+
if (doc.metadataError || !doc.typedMetadata) return "metadata_invalid";
|
|
200
|
+
return matchesMetadataPredicate(doc.typedMetadata, filter)
|
|
201
|
+
? undefined
|
|
202
|
+
: "metadata_filter";
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
export async function typedMetadataWarnings(
|
|
206
|
+
store: StorePort,
|
|
207
|
+
query: string,
|
|
208
|
+
options: HybridSearchOptions
|
|
209
|
+
): Promise<{ code: string; message: string }[] | undefined> {
|
|
210
|
+
if (!options.filter) return undefined;
|
|
211
|
+
const unknown = [
|
|
212
|
+
{
|
|
213
|
+
code: "METADATA_COVERAGE_UNKNOWN",
|
|
214
|
+
message:
|
|
215
|
+
"Typed metadata coverage could not be checked; filtered results may be incomplete.",
|
|
216
|
+
},
|
|
217
|
+
];
|
|
218
|
+
if (!store.getTypedMetadataCoverage) return unknown;
|
|
219
|
+
const range = resolveTemporalRange(query, options.since, options.until);
|
|
220
|
+
const result = await store.getTypedMetadataCoverage({
|
|
221
|
+
collection: options.collection,
|
|
222
|
+
relPathPrefix: options.retrievalScope?.relPathPrefix,
|
|
223
|
+
allowedMirrorHashes: options.retrievalScope?.allowedMirrorHashes,
|
|
224
|
+
chunkLanguage: options.lang,
|
|
225
|
+
tagsAll: options.tagsAll,
|
|
226
|
+
tagsAny: options.tagsAny,
|
|
227
|
+
since: range.since,
|
|
228
|
+
until: range.until,
|
|
229
|
+
categories: options.categories,
|
|
230
|
+
author: options.author,
|
|
231
|
+
exclude: options.exclude,
|
|
232
|
+
excludeMetadata: true,
|
|
233
|
+
semanticMetadata: true,
|
|
234
|
+
memoryScopesAny: options.memoryFilter?.scopes,
|
|
235
|
+
excludeSuperseded: options.memoryFilter?.excludeSuperseded,
|
|
236
|
+
});
|
|
237
|
+
if (!result.ok) return unknown;
|
|
238
|
+
const { pending, invalid } = result.value;
|
|
239
|
+
return pending || invalid
|
|
240
|
+
? [
|
|
241
|
+
{
|
|
242
|
+
code: "METADATA_COVERAGE_INCOMPLETE",
|
|
243
|
+
message: `Typed metadata coverage is incomplete: ${pending} documents need re-ingestion and ${invalid} have invalid metadata within the query scope.`,
|
|
244
|
+
},
|
|
245
|
+
]
|
|
246
|
+
: undefined;
|
|
247
|
+
}
|
|
@@ -4,15 +4,18 @@
|
|
|
4
4
|
* @module src/pipeline/graph-retrieval
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
|
+
import type { MetadataPredicate } from "../core/typed-metadata";
|
|
7
8
|
import type {
|
|
8
9
|
ChunkRow,
|
|
9
10
|
DocumentRow,
|
|
11
|
+
DocumentEligibilityOptions,
|
|
10
12
|
GraphEdgeConfidence,
|
|
11
13
|
GraphLink,
|
|
12
14
|
StorePort,
|
|
13
15
|
} from "../store/types";
|
|
14
16
|
import type { FusionCandidate } from "./types";
|
|
15
17
|
|
|
18
|
+
import { typedMetadataFilterReason } from "./filters";
|
|
16
19
|
import { isWithinTemporalRange } from "./temporal";
|
|
17
20
|
|
|
18
21
|
export interface GraphRetrievalMeta {
|
|
@@ -65,9 +68,11 @@ const matchesDocumentFilters = (
|
|
|
65
68
|
until?: string;
|
|
66
69
|
categories?: string[];
|
|
67
70
|
author?: string;
|
|
71
|
+
filter?: MetadataPredicate;
|
|
68
72
|
relPathPrefix?: string;
|
|
69
73
|
}
|
|
70
74
|
): boolean => {
|
|
75
|
+
if (typedMetadataFilterReason(doc, options.filter)) return false;
|
|
71
76
|
if (
|
|
72
77
|
!isWithinTemporalRange(doc.sourceMtime, {
|
|
73
78
|
since: options.since,
|
|
@@ -235,6 +240,7 @@ export async function expandGraphCandidates(
|
|
|
235
240
|
options: {
|
|
236
241
|
collection?: string;
|
|
237
242
|
includeSimilar?: boolean;
|
|
243
|
+
eligibility?: DocumentEligibilityOptions;
|
|
238
244
|
limit?: number;
|
|
239
245
|
candidateLimit?: number;
|
|
240
246
|
disabled?: boolean;
|
|
@@ -245,6 +251,7 @@ export async function expandGraphCandidates(
|
|
|
245
251
|
until?: string;
|
|
246
252
|
categories?: string[];
|
|
247
253
|
author?: string;
|
|
254
|
+
filter?: MetadataPredicate;
|
|
248
255
|
relPathPrefix?: string;
|
|
249
256
|
} = {},
|
|
250
257
|
hydration: Pick<
|
|
@@ -300,7 +307,7 @@ export async function expandGraphCandidates(
|
|
|
300
307
|
|
|
301
308
|
const rankedSeedDocs: Array<{ doc: DocumentRow; rank: number }> = [];
|
|
302
309
|
for (const doc of seedDocsResult.value) {
|
|
303
|
-
if (!doc.mirrorHash) {
|
|
310
|
+
if (!doc.mirrorHash || typedMetadataFilterReason(doc, options.filter)) {
|
|
304
311
|
continue;
|
|
305
312
|
}
|
|
306
313
|
if (
|
|
@@ -394,6 +401,7 @@ export async function expandGraphCandidates(
|
|
|
394
401
|
}
|
|
395
402
|
|
|
396
403
|
const docsResult = await store.getDocumentsByDocids(rankedNeighborDocids, {
|
|
404
|
+
eligibility: options.eligibility,
|
|
397
405
|
collection: options.collection,
|
|
398
406
|
activeOnly: true,
|
|
399
407
|
});
|
package/src/pipeline/hybrid.ts
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
import type { Config } from "../config/types";
|
|
2
1
|
/**
|
|
3
2
|
* Hybrid search orchestrator.
|
|
4
3
|
* Combines BM25, vector search, expansion, fusion, and reranking.
|
|
5
4
|
*
|
|
6
5
|
* @module src/pipeline/hybrid
|
|
7
6
|
*/
|
|
7
|
+
|
|
8
|
+
import type { Config } from "../config/types";
|
|
8
9
|
import type { EmbeddingPort, GenerationPort, RerankPort } from "../llm/types";
|
|
9
10
|
import type { DocumentRow, StorePort } from "../store/types";
|
|
10
11
|
import type {
|
|
@@ -24,6 +25,7 @@ import type {
|
|
|
24
25
|
|
|
25
26
|
import { normalizeContentTypes } from "../config/content-types";
|
|
26
27
|
import { projectRecordEvidenceMetadata } from "../core/record-metadata";
|
|
28
|
+
import { normalizeMetadataPredicate } from "../core/typed-metadata";
|
|
27
29
|
import { embedTextsWithRecovery } from "../embed/batch";
|
|
28
30
|
import {
|
|
29
31
|
assertInferenceActive,
|
|
@@ -54,7 +56,7 @@ import {
|
|
|
54
56
|
explainTimings,
|
|
55
57
|
explainVector,
|
|
56
58
|
} from "./explain";
|
|
57
|
-
import { evaluateDocumentChunkFilters } from "./filters";
|
|
59
|
+
import { evaluateDocumentChunkFilters, typedMetadataWarnings } from "./filters";
|
|
58
60
|
import { type RankedInput, rrfFuse, toRankedInput } from "./fusion";
|
|
59
61
|
import { expandGraphCandidates } from "./graph-retrieval";
|
|
60
62
|
import { RequestHydration } from "./hydration";
|
|
@@ -148,6 +150,7 @@ async function checkBm25Strength(
|
|
|
148
150
|
until?: string;
|
|
149
151
|
categories?: string[];
|
|
150
152
|
author?: string;
|
|
153
|
+
filter?: HybridSearchOptions["filter"];
|
|
151
154
|
relPathPrefix?: string;
|
|
152
155
|
allowedMirrorHashes?: string[];
|
|
153
156
|
exclude?: string[];
|
|
@@ -173,6 +176,7 @@ async function checkBm25Strength(
|
|
|
173
176
|
until: options?.until,
|
|
174
177
|
categories: options?.categories,
|
|
175
178
|
author: options?.author,
|
|
179
|
+
filter: options?.filter,
|
|
176
180
|
});
|
|
177
181
|
|
|
178
182
|
if (!result.ok || result.value.length === 0) {
|
|
@@ -221,6 +225,7 @@ async function searchFtsChunks(
|
|
|
221
225
|
until?: string;
|
|
222
226
|
categories?: string[];
|
|
223
227
|
author?: string;
|
|
228
|
+
filter?: HybridSearchOptions["filter"];
|
|
224
229
|
relPathPrefix?: string;
|
|
225
230
|
allowedMirrorHashes?: string[];
|
|
226
231
|
exclude?: string[];
|
|
@@ -246,6 +251,7 @@ async function searchFtsChunks(
|
|
|
246
251
|
until: options.until,
|
|
247
252
|
categories: options.categories,
|
|
248
253
|
author: options.author,
|
|
254
|
+
filter: options.filter,
|
|
249
255
|
});
|
|
250
256
|
if (!result.ok) {
|
|
251
257
|
// Propagate INVALID_INPUT for FTS syntax errors
|
|
@@ -356,6 +362,11 @@ export async function searchHybrid(
|
|
|
356
362
|
query: string,
|
|
357
363
|
options: HybridSearchOptions = {}
|
|
358
364
|
): Promise<ReturnType<typeof ok<SearchResults>>> {
|
|
365
|
+
if (options.filter !== undefined)
|
|
366
|
+
options = {
|
|
367
|
+
...options,
|
|
368
|
+
filter: normalizeMetadataPredicate(options.filter),
|
|
369
|
+
};
|
|
359
370
|
const hydration = deps.hydration ?? new RequestHydration(deps.store);
|
|
360
371
|
try {
|
|
361
372
|
return await withInferenceScope(options, () =>
|
|
@@ -483,6 +494,7 @@ async function searchHybridWithHydration(
|
|
|
483
494
|
until: temporalRange.until,
|
|
484
495
|
categories: options.categories,
|
|
485
496
|
author: options.author,
|
|
497
|
+
filter: options.filter,
|
|
486
498
|
relPathPrefix: options.retrievalScope?.relPathPrefix,
|
|
487
499
|
});
|
|
488
500
|
|
|
@@ -531,6 +543,7 @@ async function searchHybridWithHydration(
|
|
|
531
543
|
until: temporalRange.until,
|
|
532
544
|
categories: options.categories,
|
|
533
545
|
author: options.author,
|
|
546
|
+
filter: options.filter,
|
|
534
547
|
exclude: options.exclude,
|
|
535
548
|
excludeMetadata: true,
|
|
536
549
|
semanticMetadata: true,
|
|
@@ -553,6 +566,7 @@ async function searchHybridWithHydration(
|
|
|
553
566
|
until: temporalRange.until,
|
|
554
567
|
categories: options.categories,
|
|
555
568
|
author: options.author,
|
|
569
|
+
filter: options.filter,
|
|
556
570
|
relPathPrefix: options.retrievalScope?.relPathPrefix,
|
|
557
571
|
});
|
|
558
572
|
|
|
@@ -592,6 +606,7 @@ async function searchHybridWithHydration(
|
|
|
592
606
|
until: temporalRange.until,
|
|
593
607
|
categories: options.categories,
|
|
594
608
|
author: options.author,
|
|
609
|
+
filter: options.filter,
|
|
595
610
|
relPathPrefix: options.retrievalScope?.relPathPrefix,
|
|
596
611
|
})
|
|
597
612
|
)
|
|
@@ -767,6 +782,7 @@ async function searchHybridWithHydration(
|
|
|
767
782
|
{
|
|
768
783
|
collection: options.collection,
|
|
769
784
|
includeSimilar: vectorAvailable,
|
|
785
|
+
eligibility: vectorEligibility,
|
|
770
786
|
limit,
|
|
771
787
|
candidateLimit,
|
|
772
788
|
disabled: options.graph === false || options.noGraph === true,
|
|
@@ -778,6 +794,7 @@ async function searchHybridWithHydration(
|
|
|
778
794
|
until: temporalRange.until,
|
|
779
795
|
categories: options.categories,
|
|
780
796
|
author: options.author,
|
|
797
|
+
filter: options.filter,
|
|
781
798
|
},
|
|
782
799
|
hydration
|
|
783
800
|
);
|
|
@@ -1355,6 +1372,9 @@ async function searchHybridWithHydration(
|
|
|
1355
1372
|
until: temporalRange.until,
|
|
1356
1373
|
categories: options.categories,
|
|
1357
1374
|
author: options.author,
|
|
1375
|
+
...(options.filter
|
|
1376
|
+
? { warnings: await typedMetadataWarnings(store, query, options) }
|
|
1377
|
+
: {}),
|
|
1358
1378
|
candidateLimit,
|
|
1359
1379
|
graphExpansion: {
|
|
1360
1380
|
enabled: graphExpansion.meta.enabled,
|
package/src/pipeline/search.ts
CHANGED
|
@@ -16,6 +16,7 @@ import type {
|
|
|
16
16
|
} from "./types";
|
|
17
17
|
|
|
18
18
|
import { projectRecordEvidenceMetadata } from "../core/record-metadata";
|
|
19
|
+
import { normalizeMetadataPredicate } from "../core/typed-metadata";
|
|
19
20
|
import { getContentBatch } from "../store/content-batch";
|
|
20
21
|
import { err, ok } from "../store/types";
|
|
21
22
|
import { createChunkLookup } from "./chunk-lookup";
|
|
@@ -26,6 +27,7 @@ import {
|
|
|
26
27
|
} from "./content-type-boost";
|
|
27
28
|
import { attachSearchResultEgressLineage } from "./egress-lineage";
|
|
28
29
|
import { matchesExcludedChunks, matchesExcludedText } from "./exclude";
|
|
30
|
+
import { typedMetadataWarnings } from "./filters";
|
|
29
31
|
import { selectBestChunkForSteering } from "./intent";
|
|
30
32
|
import { hasProjectAffinity } from "./project-affinity";
|
|
31
33
|
import { detectQueryLanguage } from "./query-language";
|
|
@@ -176,6 +178,11 @@ export async function searchBm25(
|
|
|
176
178
|
query: string,
|
|
177
179
|
options: SearchOptions = {}
|
|
178
180
|
): Promise<ReturnType<typeof ok<SearchResults>>> {
|
|
181
|
+
if (options.filter !== undefined)
|
|
182
|
+
options = {
|
|
183
|
+
...options,
|
|
184
|
+
filter: normalizeMetadataPredicate(options.filter),
|
|
185
|
+
};
|
|
179
186
|
const traceStartedAt = options.traceSession ? performance.now() : 0;
|
|
180
187
|
const limit = options.limit ?? 20;
|
|
181
188
|
const minScore = options.minScore ?? 0;
|
|
@@ -213,6 +220,7 @@ export async function searchBm25(
|
|
|
213
220
|
until: temporalRange.until,
|
|
214
221
|
categories: options.categories,
|
|
215
222
|
author: options.author,
|
|
223
|
+
filter: options.filter,
|
|
216
224
|
memoryScopesAny: options.memoryFilter?.scopes,
|
|
217
225
|
excludeSuperseded: options.memoryFilter?.excludeSuperseded,
|
|
218
226
|
});
|
|
@@ -482,6 +490,9 @@ export async function searchBm25(
|
|
|
482
490
|
until: temporalRange.until,
|
|
483
491
|
categories: options.categories,
|
|
484
492
|
author: options.author,
|
|
493
|
+
...(options.filter
|
|
494
|
+
? { warnings: await typedMetadataWarnings(store, query, options) }
|
|
495
|
+
: {}),
|
|
485
496
|
queryLanguage,
|
|
486
497
|
},
|
|
487
498
|
};
|
package/src/pipeline/types.ts
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
import type { NormalizedContentTypeRule } from "../config/content-types";
|
|
2
1
|
/**
|
|
3
2
|
* Search pipeline types.
|
|
4
3
|
* Defines SearchPipelinePort and related types for search operations.
|
|
5
4
|
*
|
|
6
5
|
* @module src/pipeline/types
|
|
7
6
|
*/
|
|
7
|
+
|
|
8
|
+
import type { NormalizedContentTypeRule } from "../config/content-types";
|
|
8
9
|
import type {
|
|
9
10
|
ContextCapsuleV1,
|
|
10
11
|
ContextCapsuleVerification,
|
|
@@ -12,6 +13,7 @@ import type {
|
|
|
12
13
|
import type { EgressLineage } from "../core/egress-provenance";
|
|
13
14
|
import type { RecordEvidenceMetadata } from "../core/record-metadata";
|
|
14
15
|
import type { RetrievalTraceSession } from "../core/retrieval-trace-session";
|
|
16
|
+
import type { MetadataPredicate } from "../core/typed-metadata";
|
|
15
17
|
import type { InferenceOptions } from "../llm/types";
|
|
16
18
|
import type { StoreResult } from "../store/types";
|
|
17
19
|
import type { ClaimVerificationResult } from "./claim-verification";
|
|
@@ -100,6 +102,8 @@ export type SearchMode = "bm25" | "vector" | "hybrid" | "bm25_only";
|
|
|
100
102
|
|
|
101
103
|
/** Search metadata */
|
|
102
104
|
export interface SearchMeta {
|
|
105
|
+
/** Typed-filter coverage limitations; absence does not certify model quality. */
|
|
106
|
+
warnings?: { code: string; message: string }[];
|
|
103
107
|
query: string;
|
|
104
108
|
mode: SearchMode;
|
|
105
109
|
expanded?: boolean;
|
|
@@ -173,6 +177,8 @@ export interface SearchResults {
|
|
|
173
177
|
|
|
174
178
|
/** Common options for all search commands */
|
|
175
179
|
export interface SearchOptions extends InferenceOptions {
|
|
180
|
+
/** Type-strict custom metadata predicate, applied before candidate limits. */
|
|
181
|
+
filter?: MetadataPredicate;
|
|
176
182
|
/** Internal receipt seam; never serialized or included in public schemas. */
|
|
177
183
|
traceSession?: RetrievalTraceSession;
|
|
178
184
|
/** Trusted, already-resolved project affinity; never accepts raw roots. */
|
package/src/pipeline/vsearch.ts
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
import type { Config } from "../config/types";
|
|
2
1
|
/**
|
|
3
2
|
* Vector search pipeline.
|
|
4
3
|
* Wraps VectorIndexPort.searchNearest() to produce SearchResults.
|
|
5
4
|
*
|
|
6
5
|
* @module src/pipeline/vsearch
|
|
7
6
|
*/
|
|
7
|
+
|
|
8
|
+
import type { Config } from "../config/types";
|
|
8
9
|
import type { EmbeddingPort } from "../llm/types";
|
|
9
10
|
import type { DocumentRow, StorePort } from "../store/types";
|
|
10
11
|
import type { VectorIndexPort } from "../store/vector/types";
|
|
@@ -12,6 +13,7 @@ import type { SearchOptions, SearchResult, SearchResults } from "./types";
|
|
|
12
13
|
|
|
13
14
|
import { normalizeContentTypes } from "../config/content-types";
|
|
14
15
|
import { projectRecordEvidenceMetadata } from "../core/record-metadata";
|
|
16
|
+
import { normalizeMetadataPredicate } from "../core/typed-metadata";
|
|
15
17
|
import {
|
|
16
18
|
assertInferenceActive,
|
|
17
19
|
assertInferenceResult,
|
|
@@ -29,6 +31,7 @@ import {
|
|
|
29
31
|
import { formatQueryForEmbedding } from "./contextual";
|
|
30
32
|
import { attachSearchResultEgressLineage } from "./egress-lineage";
|
|
31
33
|
import { matchesExcludedChunks, matchesExcludedText } from "./exclude";
|
|
34
|
+
import { typedMetadataWarnings, typedMetadataFilterReason } from "./filters";
|
|
32
35
|
import { selectBestChunkForSteering } from "./intent";
|
|
33
36
|
import { hasProjectAffinity } from "./project-affinity";
|
|
34
37
|
import { detectQueryLanguage } from "./query-language";
|
|
@@ -93,6 +96,11 @@ export async function searchVectorWithEmbedding(
|
|
|
93
96
|
queryEmbedding: Float32Array,
|
|
94
97
|
options: SearchOptions = {}
|
|
95
98
|
): Promise<ReturnType<typeof ok<SearchResults>>> {
|
|
99
|
+
if (options.filter !== undefined)
|
|
100
|
+
options = {
|
|
101
|
+
...options,
|
|
102
|
+
filter: normalizeMetadataPredicate(options.filter),
|
|
103
|
+
};
|
|
96
104
|
return withInferenceScope(options, () =>
|
|
97
105
|
searchVectorWithEmbeddingOwned(deps, query, queryEmbedding, options)
|
|
98
106
|
);
|
|
@@ -162,6 +170,7 @@ async function searchVectorWithEmbeddingOwned(
|
|
|
162
170
|
until: temporalRange.until,
|
|
163
171
|
categories: options.categories,
|
|
164
172
|
author: options.author,
|
|
173
|
+
filter: options.filter,
|
|
165
174
|
exclude: options.exclude,
|
|
166
175
|
language: options.lang,
|
|
167
176
|
},
|
|
@@ -198,6 +207,7 @@ async function searchVectorWithEmbeddingOwned(
|
|
|
198
207
|
until: temporalRange.until,
|
|
199
208
|
categories: options.categories,
|
|
200
209
|
author: options.author,
|
|
210
|
+
filter: options.filter,
|
|
201
211
|
mirrorHashes: uniqueHashes,
|
|
202
212
|
});
|
|
203
213
|
|
|
@@ -526,6 +536,9 @@ async function searchVectorWithEmbeddingOwned(
|
|
|
526
536
|
until: temporalRange.until,
|
|
527
537
|
categories: options.categories,
|
|
528
538
|
author: options.author,
|
|
539
|
+
...(options.filter
|
|
540
|
+
? { warnings: await typedMetadataWarnings(store, query, options) }
|
|
541
|
+
: {}),
|
|
529
542
|
queryLanguage,
|
|
530
543
|
},
|
|
531
544
|
};
|
|
@@ -552,6 +565,11 @@ export async function searchVector(
|
|
|
552
565
|
query: string,
|
|
553
566
|
options: SearchOptions = {}
|
|
554
567
|
): Promise<ReturnType<typeof ok<SearchResults>>> {
|
|
568
|
+
if (options.filter !== undefined)
|
|
569
|
+
options = {
|
|
570
|
+
...options,
|
|
571
|
+
filter: normalizeMetadataPredicate(options.filter),
|
|
572
|
+
};
|
|
555
573
|
return withInferenceScope(options, () =>
|
|
556
574
|
searchVectorOwned(deps, query, options)
|
|
557
575
|
);
|
|
@@ -639,6 +657,7 @@ interface DocumentMapOptions {
|
|
|
639
657
|
until?: string;
|
|
640
658
|
categories?: string[];
|
|
641
659
|
author?: string;
|
|
660
|
+
filter?: SearchOptions["filter"];
|
|
642
661
|
mirrorHashes?: string[];
|
|
643
662
|
}
|
|
644
663
|
|
|
@@ -731,6 +750,7 @@ async function buildDocumentMap(
|
|
|
731
750
|
|
|
732
751
|
for (const doc of activeDocs) {
|
|
733
752
|
assertInferenceActive();
|
|
753
|
+
if (typedMetadataFilterReason(doc, options.filter)) continue;
|
|
734
754
|
const sourceRelPath = doc.recordSourcePath ?? doc.relPath;
|
|
735
755
|
if (
|
|
736
756
|
options.relPathPrefix !== undefined &&
|
package/src/sdk/client.ts
CHANGED
|
@@ -1,12 +1,16 @@
|
|
|
1
1
|
import { mkdir } from "node:fs/promises";
|
|
2
|
+
import { dirname } from "node:path";
|
|
3
|
+
|
|
4
|
+
import type { Config } from "../config/types";
|
|
2
5
|
/**
|
|
3
6
|
* GNO SDK client.
|
|
4
7
|
*
|
|
5
8
|
* @module src/sdk/client
|
|
6
9
|
*/
|
|
7
|
-
import {
|
|
8
|
-
|
|
9
|
-
|
|
10
|
+
import type {
|
|
11
|
+
CompiledContextPreviewInput,
|
|
12
|
+
CompiledContextCheckInput,
|
|
13
|
+
} from "../core/compiled-context";
|
|
10
14
|
import type { DownloadPolicy } from "../llm/policy";
|
|
11
15
|
import type { EmbeddingPort, GenerationPort, RerankPort } from "../llm/types";
|
|
12
16
|
import type { AskResult, SearchResults } from "../pipeline/types";
|
|
@@ -58,6 +62,15 @@ import type {
|
|
|
58
62
|
SectionTargetV1,
|
|
59
63
|
} from "./types";
|
|
60
64
|
|
|
65
|
+
import {
|
|
66
|
+
previewCompiledContext,
|
|
67
|
+
checkCompiledContext,
|
|
68
|
+
} from "../app/compiled-context";
|
|
69
|
+
import {
|
|
70
|
+
compileContextFile,
|
|
71
|
+
checkContextFile,
|
|
72
|
+
refreshContextFile,
|
|
73
|
+
} from "../app/compiled-context-files";
|
|
61
74
|
import {
|
|
62
75
|
decorateUriForIndex,
|
|
63
76
|
DEFAULT_INDEX_NAME,
|
|
@@ -152,6 +165,10 @@ import {
|
|
|
152
165
|
} from "../core/sections";
|
|
153
166
|
import { normalizeStructuredQueryInput } from "../core/structured-query";
|
|
154
167
|
import { parseAndValidateTagFilter } from "../core/tags";
|
|
168
|
+
import {
|
|
169
|
+
normalizeMetadataPredicate,
|
|
170
|
+
type MetadataPredicate,
|
|
171
|
+
} from "../core/typed-metadata";
|
|
155
172
|
import { writeLeasePath } from "../core/write-lease";
|
|
156
173
|
import {
|
|
157
174
|
defaultSyncService,
|
|
@@ -206,6 +223,17 @@ interface RuntimePorts {
|
|
|
206
223
|
vectorIndex: VectorIndexPort | null;
|
|
207
224
|
}
|
|
208
225
|
|
|
226
|
+
const normalizeSdkMetadataFilter = (value: unknown): MetadataPredicate => {
|
|
227
|
+
try {
|
|
228
|
+
return normalizeMetadataPredicate(value);
|
|
229
|
+
} catch (error) {
|
|
230
|
+
throw sdkError(
|
|
231
|
+
"VALIDATION",
|
|
232
|
+
`filter: ${error instanceof Error ? error.message : "Invalid predicate"}`
|
|
233
|
+
);
|
|
234
|
+
}
|
|
235
|
+
};
|
|
236
|
+
|
|
209
237
|
const resolveSdkProjectAffinity = async (
|
|
210
238
|
config: Config,
|
|
211
239
|
projectHints: readonly string[] | undefined
|
|
@@ -640,6 +668,8 @@ class GnoClientImpl implements GnoClient {
|
|
|
640
668
|
let traceSession: RetrievalTraceSession | null = null;
|
|
641
669
|
try {
|
|
642
670
|
const { projectHints, ...searchOptions } = options;
|
|
671
|
+
if (searchOptions.filter !== undefined)
|
|
672
|
+
searchOptions.filter = normalizeSdkMetadataFilter(searchOptions.filter);
|
|
643
673
|
const projectAffinity = await resolveSdkProjectAffinity(
|
|
644
674
|
this.config,
|
|
645
675
|
projectHints
|
|
@@ -695,6 +725,8 @@ class GnoClientImpl implements GnoClient {
|
|
|
695
725
|
|
|
696
726
|
try {
|
|
697
727
|
const { projectHints, ...searchOptions } = options;
|
|
728
|
+
if (searchOptions.filter !== undefined)
|
|
729
|
+
searchOptions.filter = normalizeSdkMetadataFilter(searchOptions.filter);
|
|
698
730
|
const projectAffinity = await resolveSdkProjectAffinity(
|
|
699
731
|
this.config,
|
|
700
732
|
projectHints
|
|
@@ -826,6 +858,8 @@ class GnoClientImpl implements GnoClient {
|
|
|
826
858
|
|
|
827
859
|
try {
|
|
828
860
|
const { projectHints, ...queryOptions } = options;
|
|
861
|
+
if (queryOptions.filter !== undefined)
|
|
862
|
+
queryOptions.filter = normalizeSdkMetadataFilter(queryOptions.filter);
|
|
829
863
|
const projectAffinity = await resolveSdkProjectAffinity(
|
|
830
864
|
this.config,
|
|
831
865
|
projectHints
|
|
@@ -954,6 +988,8 @@ class GnoClientImpl implements GnoClient {
|
|
|
954
988
|
const hydration = new RequestHydration(this.store);
|
|
955
989
|
try {
|
|
956
990
|
const { projectHints, ...askOptions } = options;
|
|
991
|
+
if (askOptions.filter !== undefined)
|
|
992
|
+
askOptions.filter = normalizeSdkMetadataFilter(askOptions.filter);
|
|
957
993
|
const projectAffinity = await resolveSdkProjectAffinity(
|
|
958
994
|
this.config,
|
|
959
995
|
projectHints
|
|
@@ -1046,6 +1082,7 @@ class GnoClientImpl implements GnoClient {
|
|
|
1046
1082
|
until: options.until,
|
|
1047
1083
|
categories: options.categories,
|
|
1048
1084
|
author: options.author,
|
|
1085
|
+
filter: askOptions.filter,
|
|
1049
1086
|
tagsAll: options.tagsAll,
|
|
1050
1087
|
tagsAny: options.tagsAny,
|
|
1051
1088
|
exclude: options.exclude,
|
|
@@ -1151,6 +1188,8 @@ class GnoClientImpl implements GnoClient {
|
|
|
1151
1188
|
async context(input: GnoContextInput): Promise<GnoContextResult> {
|
|
1152
1189
|
this.assertOpen();
|
|
1153
1190
|
const { projectHints, ...contextInput } = input;
|
|
1191
|
+
if (contextInput.filter !== undefined)
|
|
1192
|
+
contextInput.filter = normalizeSdkMetadataFilter(contextInput.filter);
|
|
1154
1193
|
validateContextCapsuleBuildInput(
|
|
1155
1194
|
{ ...contextInput, indexName: this.indexName },
|
|
1156
1195
|
this.indexName,
|
|
@@ -1189,6 +1228,7 @@ class GnoClientImpl implements GnoClient {
|
|
|
1189
1228
|
until: input.until,
|
|
1190
1229
|
categories: input.categories,
|
|
1191
1230
|
author: input.author,
|
|
1231
|
+
filter: contextInput.filter,
|
|
1192
1232
|
graph: input.graph,
|
|
1193
1233
|
candidateLimit: input.candidateLimit,
|
|
1194
1234
|
queryModes: input.queryModes,
|
|
@@ -1227,6 +1267,61 @@ class GnoClientImpl implements GnoClient {
|
|
|
1227
1267
|
}
|
|
1228
1268
|
}
|
|
1229
1269
|
|
|
1270
|
+
compiledContextPreview(
|
|
1271
|
+
input: CompiledContextPreviewInput
|
|
1272
|
+
): ReturnType<typeof previewCompiledContext> {
|
|
1273
|
+
this.assertOpen();
|
|
1274
|
+
return previewCompiledContext(input, {
|
|
1275
|
+
store: this.store,
|
|
1276
|
+
config: this.config,
|
|
1277
|
+
indexName: this.indexName,
|
|
1278
|
+
});
|
|
1279
|
+
}
|
|
1280
|
+
|
|
1281
|
+
compiledContextCheck(
|
|
1282
|
+
input: CompiledContextCheckInput
|
|
1283
|
+
): ReturnType<typeof checkCompiledContext> {
|
|
1284
|
+
this.assertOpen();
|
|
1285
|
+
return checkCompiledContext(input, {
|
|
1286
|
+
store: this.store,
|
|
1287
|
+
config: this.config,
|
|
1288
|
+
indexName: this.indexName,
|
|
1289
|
+
});
|
|
1290
|
+
}
|
|
1291
|
+
|
|
1292
|
+
compileContextFile(
|
|
1293
|
+
input: Parameters<typeof compileContextFile>[0]
|
|
1294
|
+
): ReturnType<typeof compileContextFile> {
|
|
1295
|
+
this.assertOpen();
|
|
1296
|
+
return compileContextFile(input, {
|
|
1297
|
+
store: this.store,
|
|
1298
|
+
config: this.config,
|
|
1299
|
+
indexName: this.indexName,
|
|
1300
|
+
});
|
|
1301
|
+
}
|
|
1302
|
+
|
|
1303
|
+
checkContextFile(
|
|
1304
|
+
input: Parameters<typeof checkContextFile>[0]
|
|
1305
|
+
): ReturnType<typeof checkContextFile> {
|
|
1306
|
+
this.assertOpen();
|
|
1307
|
+
return checkContextFile(input, {
|
|
1308
|
+
store: this.store,
|
|
1309
|
+
config: this.config,
|
|
1310
|
+
indexName: this.indexName,
|
|
1311
|
+
});
|
|
1312
|
+
}
|
|
1313
|
+
|
|
1314
|
+
refreshContextFile(
|
|
1315
|
+
input: Parameters<typeof refreshContextFile>[0]
|
|
1316
|
+
): ReturnType<typeof refreshContextFile> {
|
|
1317
|
+
this.assertOpen();
|
|
1318
|
+
return refreshContextFile(
|
|
1319
|
+
input,
|
|
1320
|
+
{ store: this.store, config: this.config, indexName: this.indexName },
|
|
1321
|
+
(request) => this.context(request)
|
|
1322
|
+
);
|
|
1323
|
+
}
|
|
1324
|
+
|
|
1230
1325
|
async verifyContext(
|
|
1231
1326
|
capsule: GnoContextResult
|
|
1232
1327
|
): Promise<GnoContextVerificationResult> {
|