@gmickel/gno 2.7.1 → 2.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -2
- package/assets/skill/SKILL.md +11 -1
- package/assets/skill/cli-reference.md +8 -1
- package/assets/skill/examples.md +2 -1
- package/assets/skill/mcp-reference.md +3 -1
- package/assets/skill/recipes/memory-scoped-recall.md +10 -5
- package/assets/spa-production.json.gz +0 -0
- package/browser-extension/artifacts/{gno-browser-clipper-v2.7.1.zip → gno-browser-clipper-v2.8.1.zip} +0 -0
- package/browser-extension/artifacts/gno-browser-clipper-v2.8.1.zip.sha256 +1 -0
- package/browser-extension/dist/manifest.json +1 -1
- package/package.json +1 -1
- package/spec/cli.md +86 -9
- package/spec/db/schema.sql +0 -1
- package/spec/mcp.md +26 -7
- package/spec/output-schemas/audit-report.schema.json +18 -4
- package/spec/output-schemas/backlinks.schema.json +4 -0
- package/spec/output-schemas/collection-list.schema.json +13 -0
- package/spec/output-schemas/graph.schema.json +2 -0
- package/spec/output-schemas/links-list.schema.json +4 -0
- package/spec/output-schemas/memory-recall.schema.json +1 -1
- package/spec/output-schemas/status.schema.json +18 -3
- package/src/cli/commands/audit.ts +23 -4
- package/src/cli/commands/collection/list.ts +39 -5
- package/src/cli/commands/embed.ts +3 -3
- package/src/cli/commands/graph.ts +3 -1
- package/src/cli/commands/links.ts +61 -180
- package/src/cli/commands/shared.ts +7 -0
- package/src/cli/commands/status.ts +6 -0
- package/src/cli/program.ts +12 -2
- package/src/config/loader.ts +43 -0
- package/src/config/types.ts +8 -0
- package/src/core/audit-contract.ts +16 -4
- package/src/core/audit-freshness.ts +11 -1
- package/src/core/audit-links.ts +197 -25
- package/src/core/audit-outside-index.ts +215 -0
- package/src/core/audit-provenance.ts +11 -4
- package/src/core/audit-workspace.ts +30 -9
- package/src/core/audit.ts +76 -16
- package/src/core/context-compiler.ts +3 -0
- package/src/core/context-evidence.ts +11 -0
- package/src/core/graph-edge-confidence.ts +23 -1
- package/src/core/host-paths.ts +1 -0
- package/src/core/knowledge-impact.ts +28 -0
- package/src/core/link-inventory-markdown.ts +2 -3
- package/src/core/link-workspace.ts +324 -0
- package/src/core/links.ts +40 -17
- package/src/core/memory-recall.ts +254 -15
- package/src/core/memory-types.ts +12 -0
- package/src/core/memory.ts +2 -0
- package/src/core/retrieval-replay-candidate.ts +6 -0
- package/src/core/retrieval-trace-request.ts +3 -0
- package/src/index.ts +14 -1
- package/src/ingestion/graph-reconciliation.ts +77 -15
- package/src/ingestion/source-availability/darwin-path.ts +9 -3
- package/src/ingestion/sync.ts +27 -4
- package/src/ingestion/types.ts +14 -0
- package/src/llm/inference-scope.ts +4 -3
- package/src/mcp/http-egress.ts +42 -3
- package/src/mcp/tools/audit.ts +11 -2
- package/src/mcp/tools/changes.ts +1 -0
- package/src/mcp/tools/links.ts +74 -93
- package/src/mcp/tools/sessions.ts +33 -4
- package/src/mcp/tools/status.ts +4 -0
- package/src/pipeline/expansion.ts +19 -31
- package/src/pipeline/graph-retrieval.ts +22 -2
- package/src/pipeline/hybrid.ts +1 -1
- package/src/pipeline/search.ts +2 -0
- package/src/pipeline/types.ts +10 -3
- package/src/sdk/client.ts +1 -0
- package/src/serve/findings-pass.ts +1 -1
- package/src/serve/public/components/editor/MarkdownPreview.tsx +5 -3
- package/src/serve/public/pages/GraphView.tsx +2 -0
- package/src/serve/routes/changes.ts +6 -1
- package/src/serve/routes/graph.ts +3 -1
- package/src/serve/routes/links.ts +45 -50
- package/src/serve/routes/sessions.ts +41 -53
- package/src/serve/server.ts +2 -1
- package/src/serve/status.ts +1 -0
- package/src/sessions/config-refresh.ts +111 -0
- package/src/store/migrations/033-drop-documents-active-index.ts +30 -0
- package/src/store/migrations/034-collection-link-workspace.ts +47 -0
- package/src/store/migrations/index.ts +4 -0
- package/src/store/sqlite/adapter.ts +477 -337
- package/src/store/sqlite/eligibility.ts +8 -2
- package/src/store/sqlite/graph-link-resolver.ts +259 -5
- package/src/store/sqlite/graph-neighbors.ts +147 -40
- package/src/store/sqlite/graph-reference-state.ts +13 -2
- package/src/store/sqlite/graph-similarity.ts +96 -0
- package/src/store/sqlite/workspace-link-resolver.ts +742 -0
- package/src/store/types.ts +64 -5
- package/src/store/vector/stats.ts +1 -1
- package/src/store/vector/status.ts +27 -0
- package/src/store/vector/stored-vectors.ts +158 -0
- package/src/store/vector/types.ts +6 -0
- package/src/store/vector/variant-search.ts +30 -14
- package/browser-extension/artifacts/gno-browser-clipper-v2.7.1.zip.sha256 +0 -1
|
@@ -30,6 +30,9 @@ import {
|
|
|
30
30
|
} from "./memory-fence";
|
|
31
31
|
import {
|
|
32
32
|
MEMORY_EMPTY_RECALL_HINT,
|
|
33
|
+
MEMORY_NO_MATCH_RECALL_HINT,
|
|
34
|
+
MEMORY_OVER_BUDGET_RECALL_HINT,
|
|
35
|
+
MEMORY_RECALL_ANY_TERM_MIN_RELATIVE_SCORE,
|
|
33
36
|
MEMORY_RECALL_MAX_FACTS,
|
|
34
37
|
MEMORY_RECALL_MAX_TOKENS,
|
|
35
38
|
MEMORY_RECALL_RETRIEVAL_LIMIT,
|
|
@@ -40,6 +43,218 @@ import {
|
|
|
40
43
|
|
|
41
44
|
type RetrievalLeg = { source: "bm25" | "vector"; results: SearchResult[] };
|
|
42
45
|
|
|
46
|
+
/**
|
|
47
|
+
* Function words dropped from the lexical leg so a question-shaped turn
|
|
48
|
+
* retrieves on its content terms. English plus the common German, French,
|
|
49
|
+
* and Italian question and function words.
|
|
50
|
+
*/
|
|
51
|
+
const RECALL_STOPWORDS = new Set([
|
|
52
|
+
// English
|
|
53
|
+
"a",
|
|
54
|
+
"about",
|
|
55
|
+
"am",
|
|
56
|
+
"an",
|
|
57
|
+
"and",
|
|
58
|
+
"any",
|
|
59
|
+
"anything",
|
|
60
|
+
"are",
|
|
61
|
+
"as",
|
|
62
|
+
"at",
|
|
63
|
+
"be",
|
|
64
|
+
"been",
|
|
65
|
+
"but",
|
|
66
|
+
"by",
|
|
67
|
+
"can",
|
|
68
|
+
"could",
|
|
69
|
+
"did",
|
|
70
|
+
"do",
|
|
71
|
+
"does",
|
|
72
|
+
"for",
|
|
73
|
+
"from",
|
|
74
|
+
"had",
|
|
75
|
+
"has",
|
|
76
|
+
"have",
|
|
77
|
+
"how",
|
|
78
|
+
"how's",
|
|
79
|
+
"i",
|
|
80
|
+
"if",
|
|
81
|
+
"in",
|
|
82
|
+
"into",
|
|
83
|
+
"is",
|
|
84
|
+
"it",
|
|
85
|
+
"it's",
|
|
86
|
+
"its",
|
|
87
|
+
"know",
|
|
88
|
+
"me",
|
|
89
|
+
"my",
|
|
90
|
+
"of",
|
|
91
|
+
"on",
|
|
92
|
+
"or",
|
|
93
|
+
"our",
|
|
94
|
+
"please",
|
|
95
|
+
"should",
|
|
96
|
+
"so",
|
|
97
|
+
"tell",
|
|
98
|
+
"that",
|
|
99
|
+
"the",
|
|
100
|
+
"their",
|
|
101
|
+
"them",
|
|
102
|
+
"there",
|
|
103
|
+
"these",
|
|
104
|
+
"they",
|
|
105
|
+
"this",
|
|
106
|
+
"those",
|
|
107
|
+
"to",
|
|
108
|
+
"us",
|
|
109
|
+
"was",
|
|
110
|
+
"we",
|
|
111
|
+
"were",
|
|
112
|
+
"what",
|
|
113
|
+
"what's",
|
|
114
|
+
"when",
|
|
115
|
+
"where",
|
|
116
|
+
"where's",
|
|
117
|
+
"which",
|
|
118
|
+
"who",
|
|
119
|
+
"who's",
|
|
120
|
+
"whom",
|
|
121
|
+
"whose",
|
|
122
|
+
"why",
|
|
123
|
+
"will",
|
|
124
|
+
"with",
|
|
125
|
+
"would",
|
|
126
|
+
"you",
|
|
127
|
+
"your",
|
|
128
|
+
// German
|
|
129
|
+
"das",
|
|
130
|
+
"dem",
|
|
131
|
+
"den",
|
|
132
|
+
"der",
|
|
133
|
+
"des",
|
|
134
|
+
"ein",
|
|
135
|
+
"eine",
|
|
136
|
+
"ist",
|
|
137
|
+
"mit",
|
|
138
|
+
"oder",
|
|
139
|
+
"sind",
|
|
140
|
+
"und",
|
|
141
|
+
"uns",
|
|
142
|
+
"von",
|
|
143
|
+
"wann",
|
|
144
|
+
"warum",
|
|
145
|
+
"welche",
|
|
146
|
+
"welcher",
|
|
147
|
+
"welches",
|
|
148
|
+
"wer",
|
|
149
|
+
"wie",
|
|
150
|
+
"wir",
|
|
151
|
+
"wo",
|
|
152
|
+
"zu",
|
|
153
|
+
// French
|
|
154
|
+
"avec",
|
|
155
|
+
"comment",
|
|
156
|
+
"dans",
|
|
157
|
+
"de",
|
|
158
|
+
"du",
|
|
159
|
+
"est",
|
|
160
|
+
"et",
|
|
161
|
+
"la",
|
|
162
|
+
"le",
|
|
163
|
+
"les",
|
|
164
|
+
"nous",
|
|
165
|
+
"ou",
|
|
166
|
+
"où",
|
|
167
|
+
"pour",
|
|
168
|
+
"pourquoi",
|
|
169
|
+
"quand",
|
|
170
|
+
"que",
|
|
171
|
+
"quel",
|
|
172
|
+
"quelle",
|
|
173
|
+
"qui",
|
|
174
|
+
"quoi",
|
|
175
|
+
"sont",
|
|
176
|
+
"sur",
|
|
177
|
+
"un",
|
|
178
|
+
"une",
|
|
179
|
+
"vous",
|
|
180
|
+
// Italian
|
|
181
|
+
"che",
|
|
182
|
+
"chi",
|
|
183
|
+
"come",
|
|
184
|
+
"con",
|
|
185
|
+
"cosa",
|
|
186
|
+
"da",
|
|
187
|
+
"del",
|
|
188
|
+
"della",
|
|
189
|
+
"di",
|
|
190
|
+
"dove",
|
|
191
|
+
"il",
|
|
192
|
+
"per",
|
|
193
|
+
"perché",
|
|
194
|
+
"quale",
|
|
195
|
+
"quali",
|
|
196
|
+
"sono",
|
|
197
|
+
]);
|
|
198
|
+
|
|
199
|
+
/** Whitespace tokens, keeping a quoted phrase (optionally negated) whole. */
|
|
200
|
+
const QUERY_TOKEN_PATTERN = /-?"[^"]*"|\S+/g;
|
|
201
|
+
/** Same character class the FTS term sanitizer keeps. */
|
|
202
|
+
const NON_TERM_CHARS = /[^\p{L}\p{N}'_]/gu;
|
|
203
|
+
|
|
204
|
+
/**
|
|
205
|
+
* Content terms of a recall query: bare stopword tokens are dropped; quoted
|
|
206
|
+
* phrases, negations, and compounds pass through. A query with no positive
|
|
207
|
+
* content term left is returned unchanged.
|
|
208
|
+
*/
|
|
209
|
+
function recallContentQuery(query: string): string {
|
|
210
|
+
const tokens = query.match(QUERY_TOKEN_PATTERN) ?? [];
|
|
211
|
+
const kept = tokens.filter((token) => {
|
|
212
|
+
if (token.startsWith("-") || token.includes('"')) return true;
|
|
213
|
+
return !RECALL_STOPWORDS.has(
|
|
214
|
+
token.replace(NON_TERM_CHARS, "").toLowerCase()
|
|
215
|
+
);
|
|
216
|
+
});
|
|
217
|
+
const hasPositiveTerm = kept.some(
|
|
218
|
+
(token) =>
|
|
219
|
+
!token.startsWith("-") && token.replace(NON_TERM_CHARS, "").length > 0
|
|
220
|
+
);
|
|
221
|
+
return hasPositiveTerm ? kept.join(" ") : query;
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
/**
|
|
225
|
+
* Lexical leg: BM25 over the content terms, every term required first. When
|
|
226
|
+
* no fact carries all of them (a question-shaped turn), fall back to
|
|
227
|
+
* any-term matching so facts sharing a content term still rank, best BM25
|
|
228
|
+
* first, above a relative score floor. `null` means the query has no
|
|
229
|
+
* searchable terms.
|
|
230
|
+
*/
|
|
231
|
+
async function searchLexical(
|
|
232
|
+
deps: MemoryServiceDeps,
|
|
233
|
+
input: { query: string; collection: string; scopes: string[] }
|
|
234
|
+
): Promise<SearchResult[] | null> {
|
|
235
|
+
const contentQuery = recallContentQuery(input.query);
|
|
236
|
+
const run = async (anyTerm: boolean) => {
|
|
237
|
+
const bm25 = await searchBm25(deps.store, contentQuery, {
|
|
238
|
+
collection: input.collection,
|
|
239
|
+
limit: MEMORY_RECALL_RETRIEVAL_LIMIT,
|
|
240
|
+
memoryFilter: { scopes: input.scopes, excludeSuperseded: true },
|
|
241
|
+
...(anyTerm
|
|
242
|
+
? {
|
|
243
|
+
anyTerm,
|
|
244
|
+
minRelativeScore: MEMORY_RECALL_ANY_TERM_MIN_RELATIVE_SCORE,
|
|
245
|
+
}
|
|
246
|
+
: {}),
|
|
247
|
+
});
|
|
248
|
+
if (bm25.ok) return bm25.value.results;
|
|
249
|
+
if (bm25.error.code !== "INVALID_INPUT") {
|
|
250
|
+
throw new MemoryError("MEMORY_QUERY_FAILED", bm25.error.message);
|
|
251
|
+
}
|
|
252
|
+
return null;
|
|
253
|
+
};
|
|
254
|
+
const allTerms = await run(false);
|
|
255
|
+
return allTerms?.length === 0 ? run(true) : allTerms;
|
|
256
|
+
}
|
|
257
|
+
|
|
43
258
|
/**
|
|
44
259
|
* Retrieval legs: BM25 always; vectors when an embedding port and a searchable
|
|
45
260
|
* vector index are present. The eligible set is one unbounded in-query
|
|
@@ -52,17 +267,9 @@ async function retrieveLegs(
|
|
|
52
267
|
const { store, config } = deps;
|
|
53
268
|
const { query, collection, scopes } = input;
|
|
54
269
|
const legs: RetrievalLeg[] = [];
|
|
55
|
-
const
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
memoryFilter: { scopes, excludeSuperseded: true },
|
|
59
|
-
});
|
|
60
|
-
if (!bm25.ok) {
|
|
61
|
-
if (bm25.error.code !== "INVALID_INPUT") {
|
|
62
|
-
throw new MemoryError("MEMORY_QUERY_FAILED", bm25.error.message);
|
|
63
|
-
}
|
|
64
|
-
} else {
|
|
65
|
-
legs.push({ source: "bm25", results: bm25.value.results });
|
|
270
|
+
const lexical = await searchLexical(deps, input);
|
|
271
|
+
if (lexical) {
|
|
272
|
+
legs.push({ source: "bm25", results: lexical });
|
|
66
273
|
}
|
|
67
274
|
|
|
68
275
|
const retrieval: RecallResult["retrieval"] = { mode: "lexical" };
|
|
@@ -159,7 +366,11 @@ async function materializeFacts(
|
|
|
159
366
|
return materialized;
|
|
160
367
|
}
|
|
161
368
|
|
|
162
|
-
/**
|
|
369
|
+
/**
|
|
370
|
+
* Token budget via the shared context-evidence selector, then the fact cap.
|
|
371
|
+
* Facts carry no facets, so the selector fills the budget in retrieval-rank
|
|
372
|
+
* order (per-fact facets would make it prefer the shortest facts).
|
|
373
|
+
*/
|
|
163
374
|
function selectWithinBudget(
|
|
164
375
|
materialized: Array<{ fact: RecalledFact; rank: number }>,
|
|
165
376
|
maxFacts: number,
|
|
@@ -176,11 +387,11 @@ function selectWithinBudget(
|
|
|
176
387
|
sourceHash: fact.contentHash,
|
|
177
388
|
mirrorHash: fact.contentHash,
|
|
178
389
|
text: fact.text,
|
|
179
|
-
facets: [
|
|
390
|
+
facets: [],
|
|
180
391
|
retrievalRank: rank,
|
|
181
392
|
value: fact,
|
|
182
393
|
})),
|
|
183
|
-
requestedFacets:
|
|
394
|
+
requestedFacets: [],
|
|
184
395
|
limits: {
|
|
185
396
|
requestedBytes: maxTokens * MEMORY_TOKEN_BYTES_ESTIMATE,
|
|
186
397
|
requestedTokens: maxTokens,
|
|
@@ -201,6 +412,28 @@ function selectWithinBudget(
|
|
|
201
412
|
return selection.selected.slice(0, maxFacts).map((item) => item.value);
|
|
202
413
|
}
|
|
203
414
|
|
|
415
|
+
/**
|
|
416
|
+
* Why nothing came back: facts matched but none fit the budget, the scope
|
|
417
|
+
* holds facts but none matched, or the scope holds no current fact at all.
|
|
418
|
+
*/
|
|
419
|
+
async function emptyRecallHint(
|
|
420
|
+
deps: MemoryServiceDeps,
|
|
421
|
+
input: { collection: string; scopes: string[]; matched: number }
|
|
422
|
+
): Promise<string> {
|
|
423
|
+
if (input.matched > 0) return MEMORY_OVER_BUDGET_RECALL_HINT;
|
|
424
|
+
const eligible = await deps.store.listMemoryEligibleDocuments({
|
|
425
|
+
collection: input.collection,
|
|
426
|
+
scopes: input.scopes,
|
|
427
|
+
excludeSuperseded: true,
|
|
428
|
+
});
|
|
429
|
+
if (!eligible.ok) {
|
|
430
|
+
throw new MemoryError("MEMORY_QUERY_FAILED", eligible.error.message);
|
|
431
|
+
}
|
|
432
|
+
return eligible.value.length > 0
|
|
433
|
+
? MEMORY_NO_MATCH_RECALL_HINT
|
|
434
|
+
: MEMORY_EMPTY_RECALL_HINT;
|
|
435
|
+
}
|
|
436
|
+
|
|
204
437
|
export async function recallFacts(
|
|
205
438
|
deps: MemoryServiceDeps,
|
|
206
439
|
rawInput: RecallInput
|
|
@@ -264,6 +497,12 @@ export async function recallFacts(
|
|
|
264
497
|
facts.map((fact) => fact.egressLineage)
|
|
265
498
|
),
|
|
266
499
|
}
|
|
267
|
-
: {
|
|
500
|
+
: {
|
|
501
|
+
hint: await emptyRecallHint(deps, {
|
|
502
|
+
collection: collection.name,
|
|
503
|
+
scopes,
|
|
504
|
+
matched: materialized.length,
|
|
505
|
+
}),
|
|
506
|
+
}),
|
|
268
507
|
};
|
|
269
508
|
}
|
package/src/core/memory-types.ts
CHANGED
|
@@ -33,11 +33,23 @@ export const MEMORY_RECALL_MAX_TOKENS = 512;
|
|
|
33
33
|
/** Retrieval depth per leg before fusion and budgeting. */
|
|
34
34
|
export const MEMORY_RECALL_RETRIEVAL_LIMIT = 32;
|
|
35
35
|
export const MEMORY_RRF_K = 60;
|
|
36
|
+
/**
|
|
37
|
+
* Any-term fallback floor: a fact must score at least this fraction of the
|
|
38
|
+
* best fact's raw BM25, so a term shared by most facts cannot pull them in.
|
|
39
|
+
*/
|
|
40
|
+
export const MEMORY_RECALL_ANY_TERM_MIN_RELATIVE_SCORE = 0.1;
|
|
36
41
|
export const MEMORY_DEFAULT_LOCK_WAIT_MS = 120_000;
|
|
37
42
|
export const MEMORY_TOKEN_BYTES_ESTIMATE = 4;
|
|
38
43
|
|
|
44
|
+
/** Recall hint when the scope holds no current fact at all. */
|
|
39
45
|
export const MEMORY_EMPTY_RECALL_HINT =
|
|
40
46
|
'No memories in scope yet. Store one with: gno remember "<fact>" --scope <scope> --decision add';
|
|
47
|
+
/** Recall hint when the scope holds facts but none matched the query. */
|
|
48
|
+
export const MEMORY_NO_MATCH_RECALL_HINT =
|
|
49
|
+
'No memories in scope matched this query. Rephrase with words the fact uses, or store one with: gno remember "<fact>" --scope <scope> --decision add';
|
|
50
|
+
/** Recall hint when facts matched but none fit the token budget. */
|
|
51
|
+
export const MEMORY_OVER_BUDGET_RECALL_HINT =
|
|
52
|
+
"Matching memories did not fit the token budget. Raise --max-tokens (maxTokens) to return them.";
|
|
41
53
|
|
|
42
54
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
43
55
|
// Errors
|
package/src/core/memory.ts
CHANGED
|
@@ -34,6 +34,8 @@ export {
|
|
|
34
34
|
MEMORY_CANDIDATE_POOL,
|
|
35
35
|
MEMORY_EMPTY_RECALL_HINT,
|
|
36
36
|
MEMORY_LEXICAL_LIKELY_THRESHOLD,
|
|
37
|
+
MEMORY_NO_MATCH_RECALL_HINT,
|
|
38
|
+
MEMORY_OVER_BUDGET_RECALL_HINT,
|
|
37
39
|
MEMORY_RECALL_MAX_FACTS,
|
|
38
40
|
MEMORY_RECALL_MAX_TOKENS,
|
|
39
41
|
MEMORY_SEMANTIC_LIKELY_THRESHOLD,
|
|
@@ -303,6 +303,12 @@ export const runRetrievalReplayCandidate = async (
|
|
|
303
303
|
retrievalScope
|
|
304
304
|
);
|
|
305
305
|
options.limit = scope.value.fetchLimit;
|
|
306
|
+
const scopedCollections = scope.value.collections.filter(
|
|
307
|
+
(name): name is string => name !== undefined
|
|
308
|
+
);
|
|
309
|
+
if (scopedCollections.length > 0) {
|
|
310
|
+
options.graphCollections = scopedCollections;
|
|
311
|
+
}
|
|
306
312
|
const result = await runCandidateOnce(deps, source, candidate, options);
|
|
307
313
|
if (!result.ok) return result;
|
|
308
314
|
outputs.push(result.value);
|
|
@@ -8,6 +8,7 @@ import type { RetrievalTraceTerminalStatus } from "../store/types";
|
|
|
8
8
|
|
|
9
9
|
import { canonicalTraceJson } from "../store/retrieval-trace-codec";
|
|
10
10
|
import { err, ok } from "../store/types";
|
|
11
|
+
import { linkResolutionFingerprintInput } from "./link-workspace";
|
|
11
12
|
import { RetrievalTraceSession } from "./retrieval-trace-session";
|
|
12
13
|
|
|
13
14
|
export const retrievalTraceFailureStatus = (
|
|
@@ -45,6 +46,7 @@ export const buildRetrievalTraceFingerprints = async (input: {
|
|
|
45
46
|
}): Promise<RetrievalTraceFingerprints> => {
|
|
46
47
|
const collections = await input.store.getCollections();
|
|
47
48
|
if (!collections.ok) throw new Error(collections.error.message);
|
|
49
|
+
const linkResolution = linkResolutionFingerprintInput(collections.value);
|
|
48
50
|
const snapshots = [];
|
|
49
51
|
for (const collection of [...collections.value].sort((left, right) =>
|
|
50
52
|
left.name.localeCompare(right.name)
|
|
@@ -70,6 +72,7 @@ export const buildRetrievalTraceFingerprints = async (input: {
|
|
|
70
72
|
index: fingerprint({
|
|
71
73
|
indexName: input.indexName ?? "default",
|
|
72
74
|
snapshots,
|
|
75
|
+
...(linkResolution ? { linkResolution } : {}),
|
|
73
76
|
}),
|
|
74
77
|
};
|
|
75
78
|
};
|
package/src/index.ts
CHANGED
|
@@ -18,14 +18,27 @@ import { IMPORT_CHILD_ENV } from "./sessions/import-child-env";
|
|
|
18
18
|
* is the flush that actually waits. A closed consumer (EPIPE) settles
|
|
19
19
|
* through the callback or the 'error' event.
|
|
20
20
|
*/
|
|
21
|
+
const pendingFlushes = new WeakMap<NodeJS.WriteStream, Promise<void>>();
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Concurrent exit paths (normal completion racing a SIGINT) share one flush:
|
|
25
|
+
* a second caller must wait for the drain the first end() started, not treat
|
|
26
|
+
* the already-ended stream as flushed and exit mid-write.
|
|
27
|
+
*/
|
|
21
28
|
function flushStream(stream: NodeJS.WriteStream): Promise<void> {
|
|
29
|
+
const pending = pendingFlushes.get(stream);
|
|
30
|
+
if (pending) {
|
|
31
|
+
return pending;
|
|
32
|
+
}
|
|
22
33
|
if (stream.destroyed || stream.writableEnded) {
|
|
23
34
|
return Promise.resolve();
|
|
24
35
|
}
|
|
25
|
-
|
|
36
|
+
const flush = new Promise<void>((resolve) => {
|
|
26
37
|
stream.once("error", () => resolve());
|
|
27
38
|
stream.end(() => resolve());
|
|
28
39
|
});
|
|
40
|
+
pendingFlushes.set(stream, flush);
|
|
41
|
+
return flush;
|
|
29
42
|
}
|
|
30
43
|
|
|
31
44
|
/**
|
|
@@ -13,15 +13,25 @@ import {
|
|
|
13
13
|
normalizeRelationEdgeType,
|
|
14
14
|
normalizeRelationTarget,
|
|
15
15
|
} from "../core/change-diff";
|
|
16
|
+
import { LINK_RESOLVER_VERSION } from "../core/link-workspace";
|
|
16
17
|
import {
|
|
17
18
|
normalizeMarkdownPath,
|
|
18
19
|
normalizeWikiName,
|
|
19
20
|
parseTargetParts,
|
|
20
21
|
} from "../core/links";
|
|
22
|
+
import {
|
|
23
|
+
createInMemoryWorkspaceResolver,
|
|
24
|
+
membershipsFromCollectionRows,
|
|
25
|
+
} from "../store/sqlite/workspace-link-resolver";
|
|
21
26
|
import { parseFrontmatter } from "./frontmatter";
|
|
22
27
|
|
|
23
|
-
|
|
28
|
+
/**
|
|
29
|
+
* Projection version. 2: workspace-wide wiki resolution; a stored version
|
|
30
|
+
* mismatch forces one full projection on the first sync after upgrade.
|
|
31
|
+
*/
|
|
32
|
+
const VERSION = 2;
|
|
24
33
|
const EDGE_TYPE = /^[a-z][a-z0-9_]*$/;
|
|
34
|
+
const RELATIVE_REF = /^\.\.?\//;
|
|
25
35
|
type ProjectionError = { relPath: string; code: string; message: string };
|
|
26
36
|
|
|
27
37
|
function identity(doc: DocumentRow): GraphReferenceDocument {
|
|
@@ -38,8 +48,27 @@ function identity(doc: DocumentRow): GraphReferenceDocument {
|
|
|
38
48
|
};
|
|
39
49
|
}
|
|
40
50
|
|
|
41
|
-
/**
|
|
42
|
-
|
|
51
|
+
/**
|
|
52
|
+
* Insert once in catalog order: preserves legacy Array.find ambiguity precedence.
|
|
53
|
+
* Plain wiki-style targets from a document inside a link workspace use the
|
|
54
|
+
* shared workspace ranking (a tie resolves to nothing); URIs, docids,
|
|
55
|
+
* collection-qualified paths and relative markdown keep their contracts.
|
|
56
|
+
*/
|
|
57
|
+
function relationResolver(
|
|
58
|
+
docs: GraphReferenceDocument[],
|
|
59
|
+
memberships: ReturnType<typeof membershipsFromCollectionRows>
|
|
60
|
+
) {
|
|
61
|
+
const workspace = createInMemoryWorkspaceResolver(
|
|
62
|
+
memberships,
|
|
63
|
+
docs.map((doc) => ({
|
|
64
|
+
id: doc.documentId,
|
|
65
|
+
docid: doc.docid,
|
|
66
|
+
collection: doc.collection,
|
|
67
|
+
relPath: doc.relPath,
|
|
68
|
+
title: doc.title,
|
|
69
|
+
}))
|
|
70
|
+
);
|
|
71
|
+
const byId = new Map(docs.map((doc) => [doc.documentId, doc]));
|
|
43
72
|
const docids = new Map<string, GraphReferenceDocument>();
|
|
44
73
|
const uris = new Map<string, GraphReferenceDocument>();
|
|
45
74
|
const paths = new Map<string, GraphReferenceDocument>();
|
|
@@ -84,14 +113,27 @@ function relationResolver(docs: GraphReferenceDocument[]) {
|
|
|
84
113
|
localWiki.get(`${parts.collection}\0${key}`)
|
|
85
114
|
);
|
|
86
115
|
const relativePath = normalizeMarkdownPath(parts.ref, source.relPath);
|
|
87
|
-
|
|
88
|
-
(relativePath
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
116
|
+
const relativeHit = relativePath
|
|
117
|
+
? paths.get(`${source.collection}/${relativePath}`)
|
|
118
|
+
: undefined;
|
|
119
|
+
const ranked = workspace(source, key);
|
|
120
|
+
if (ranked === undefined) {
|
|
121
|
+
// Source outside any link workspace: the collection-scoped contract.
|
|
122
|
+
return (
|
|
123
|
+
relativeHit ??
|
|
124
|
+
paths.get(parts.ref) ??
|
|
125
|
+
localWiki.get(`${source.collection}\0${key}`) ??
|
|
126
|
+
wiki.get(key)
|
|
127
|
+
);
|
|
128
|
+
}
|
|
129
|
+
// Inside a workspace only the preserved contracts bypass the shared
|
|
130
|
+
// ranking: a collection-qualified path (`collection/relPath`) and an
|
|
131
|
+
// explicitly relative Markdown path (`./x.md`, `../x.md`). Every other
|
|
132
|
+
// name or path resolves exactly as the shared link resolver does.
|
|
133
|
+
const qualified = paths.get(parts.ref);
|
|
134
|
+
if (qualified) return qualified;
|
|
135
|
+
if (RELATIVE_REF.test(parts.ref) && relativeHit) return relativeHit;
|
|
136
|
+
return ranked?.traversable ? byId.get(ranked.target.id) : undefined;
|
|
95
137
|
};
|
|
96
138
|
}
|
|
97
139
|
|
|
@@ -105,12 +147,23 @@ export async function projectGraph(
|
|
|
105
147
|
try {
|
|
106
148
|
const graph = store.graphReferenceStore?.();
|
|
107
149
|
let fingerprint = "";
|
|
150
|
+
// Partial ports (tests, adapters without collections) resolve links
|
|
151
|
+
// collection-scoped; the SQLite store always reports memberships.
|
|
152
|
+
const collections =
|
|
153
|
+
typeof store.getCollections === "function"
|
|
154
|
+
? await store.getCollections()
|
|
155
|
+
: undefined;
|
|
156
|
+
if (collections && !collections.ok)
|
|
157
|
+
throw new Error(collections.error.message);
|
|
158
|
+
const memberships = membershipsFromCollectionRows(collections?.value ?? []);
|
|
108
159
|
if (graph) {
|
|
109
|
-
|
|
110
|
-
|
|
160
|
+
if (!collections) throw new Error("Graph projection needs collections");
|
|
161
|
+
// Effective workspace membership (roots, nested vaults, real paths) is
|
|
162
|
+
// part of each collection row, so a membership change re-projects.
|
|
111
163
|
fingerprint = new Bun.CryptoHasher("sha256")
|
|
112
164
|
.update(
|
|
113
165
|
JSON.stringify({
|
|
166
|
+
resolver: LINK_RESOLVER_VERSION,
|
|
114
167
|
rules: options.contentTypeRules ?? [],
|
|
115
168
|
collections: collections.value
|
|
116
169
|
.map(({ syncedAt: _syncedAt, ...config }) => config)
|
|
@@ -138,7 +191,15 @@ export async function projectGraph(
|
|
|
138
191
|
state.inProgress ||
|
|
139
192
|
state.version !== VERSION ||
|
|
140
193
|
state.configFingerprint !== fingerprint;
|
|
141
|
-
|
|
194
|
+
// An existing graph whose resolver or collection membership changed is
|
|
195
|
+
// rebuilt in full; report it so the fallback is never silent.
|
|
196
|
+
if (!forceFull && state && state.version !== null) {
|
|
197
|
+
if (state.version !== VERSION)
|
|
198
|
+
options.onGraphRebuild?.("resolver-upgrade");
|
|
199
|
+
else if (state.configFingerprint !== fingerprint)
|
|
200
|
+
options.onGraphRebuild?.("collections-changed");
|
|
201
|
+
}
|
|
202
|
+
const resolve = relationResolver(current, memberships);
|
|
142
203
|
let selected: Set<number> | undefined;
|
|
143
204
|
if (!full && graph) {
|
|
144
205
|
selected = new Set(requestedSources);
|
|
@@ -165,7 +226,8 @@ export async function projectGraph(
|
|
|
165
226
|
))
|
|
166
227
|
selected.add(id);
|
|
167
228
|
const resolveOld = relationResolver(
|
|
168
|
-
previous.map((row) => row.document)
|
|
229
|
+
previous.map((row) => row.document),
|
|
230
|
+
memberships
|
|
169
231
|
);
|
|
170
232
|
for (const row of previous) {
|
|
171
233
|
const source = currentById.get(row.document.documentId);
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Metadata-free recognition of the macOS File Provider layouts covered by the
|
|
3
|
-
* physical
|
|
3
|
+
* physical evidence in research/file-provider/ (fn-118; Google Shared drives in
|
|
4
|
+
* fn-179). This is intentionally narrower than all paths on
|
|
4
5
|
* Darwin: unknown storage must not inherit a no-materialization guarantee.
|
|
5
6
|
*/
|
|
6
7
|
|
|
@@ -43,8 +44,13 @@ export function classifyDarwinFileProviderPath(
|
|
|
43
44
|
return "unsupported";
|
|
44
45
|
}
|
|
45
46
|
const domain = parts[2];
|
|
46
|
-
if (domain?.startsWith("GoogleDrive-")
|
|
47
|
-
|
|
47
|
+
if (domain?.startsWith("GoogleDrive-")) {
|
|
48
|
+
const isMyDrive = parts[3] === "My Drive";
|
|
49
|
+
const isSharedDrive =
|
|
50
|
+
parts[3] === "Shared drives" &&
|
|
51
|
+
typeof parts[4] === "string" &&
|
|
52
|
+
parts[4].length > 0;
|
|
53
|
+
return isMyDrive || isSharedDrive ? "google-drive" : "unsupported";
|
|
48
54
|
}
|
|
49
55
|
if (
|
|
50
56
|
domain?.startsWith("OneDrive-") &&
|
package/src/ingestion/sync.ts
CHANGED
|
@@ -57,7 +57,6 @@ import {
|
|
|
57
57
|
} from "../core/links";
|
|
58
58
|
import { extractMemoryScopes } from "../core/memory-record";
|
|
59
59
|
import { normalizeTag, validateTag } from "../core/tags";
|
|
60
|
-
import { TYPED_METADATA_INGEST_VERSION } from "../core/typed-metadata";
|
|
61
60
|
import { defaultChunker } from "./chunker";
|
|
62
61
|
import { persistChunkLayout, prepareChunking } from "./chunking";
|
|
63
62
|
import {
|
|
@@ -102,9 +101,12 @@ const MAX_CONCURRENCY = 16;
|
|
|
102
101
|
/**
|
|
103
102
|
* Current ingest schema version.
|
|
104
103
|
* Increment when ingestion adds new derived data (tags, metadata, etc.)
|
|
105
|
-
* Documents with ingestVersion < INGEST_VERSION
|
|
104
|
+
* or changes how it is parsed. Documents with ingestVersion < INGEST_VERSION
|
|
105
|
+
* will be re-processed. Must stay >= TYPED_METADATA_INGEST_VERSION (tested).
|
|
106
|
+
* 8: wiki links with a table-escaped alias (`[[Note\|Alias]]`) and Markdown
|
|
107
|
+
* link text with square brackets parse the way Obsidian renders them.
|
|
106
108
|
*/
|
|
107
|
-
export const INGEST_VERSION =
|
|
109
|
+
export const INGEST_VERSION = 8;
|
|
108
110
|
const EMPTY_CONTENT_TYPE_RULES_FINGERPRINT =
|
|
109
111
|
fingerprintContentTypeMetadataRules([]);
|
|
110
112
|
const NON_RETRYABLE_CONVERSION_ERROR_CODES = new Set([
|
|
@@ -2069,6 +2071,19 @@ export class SyncService {
|
|
|
2069
2071
|
}
|
|
2070
2072
|
}
|
|
2071
2073
|
|
|
2074
|
+
// Nested vaults (their own `.obsidian/`) are discovered from the synced
|
|
2075
|
+
// document directories; a change re-fingerprints the graph projection.
|
|
2076
|
+
const nestedRefresh = await store.refreshCollectionNestedWorkspaces?.(
|
|
2077
|
+
collection.name
|
|
2078
|
+
);
|
|
2079
|
+
if (nestedRefresh && !nestedRefresh.ok) {
|
|
2080
|
+
errors.push({
|
|
2081
|
+
relPath: "(link workspace)",
|
|
2082
|
+
code: nestedRefresh.error.code,
|
|
2083
|
+
message: nestedRefresh.error.message,
|
|
2084
|
+
});
|
|
2085
|
+
}
|
|
2086
|
+
|
|
2072
2087
|
if (syncOptions.projectTypedEdges !== false) {
|
|
2073
2088
|
errors.push(...(await this.projectTypedEdges(store, syncOptions)));
|
|
2074
2089
|
}
|
|
@@ -2123,8 +2138,15 @@ export class SyncService {
|
|
|
2123
2138
|
results.push(result);
|
|
2124
2139
|
}
|
|
2125
2140
|
|
|
2141
|
+
let graphRebuild: SyncResult["graphRebuild"];
|
|
2126
2142
|
if (results.length > 0) {
|
|
2127
|
-
const projectionErrors = await this.projectTypedEdges(store,
|
|
2143
|
+
const projectionErrors = await this.projectTypedEdges(store, {
|
|
2144
|
+
...options,
|
|
2145
|
+
onGraphRebuild: (reason) => {
|
|
2146
|
+
graphRebuild = reason;
|
|
2147
|
+
options.onGraphRebuild?.(reason);
|
|
2148
|
+
},
|
|
2149
|
+
});
|
|
2128
2150
|
results.at(-1)?.errors.push(...projectionErrors);
|
|
2129
2151
|
}
|
|
2130
2152
|
|
|
@@ -2145,6 +2167,7 @@ export class SyncService {
|
|
|
2145
2167
|
...(prepared.rechunkedMirrors
|
|
2146
2168
|
? { rechunkedMirrors: prepared.rechunkedMirrors }
|
|
2147
2169
|
: {}),
|
|
2170
|
+
...(graphRebuild ? { graphRebuild } : {}),
|
|
2148
2171
|
totalDurationMs: Date.now() - startTime,
|
|
2149
2172
|
totalFilesProcessed: totals.processed,
|
|
2150
2173
|
totalFilesAdded: totals.added,
|
package/src/ingestion/types.ts
CHANGED
|
@@ -161,7 +161,19 @@ export interface ChunkerPort {
|
|
|
161
161
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
162
162
|
|
|
163
163
|
/** Sync options */
|
|
164
|
+
/** Why a sync rebuilt the whole link graph instead of reconciling it. */
|
|
165
|
+
export type GraphRebuildReason =
|
|
166
|
+
/** Link resolution semantics changed (first sync after an upgrade). */
|
|
167
|
+
| "resolver-upgrade"
|
|
168
|
+
/** Collection settings changed, including link workspace membership. */
|
|
169
|
+
| "collections-changed";
|
|
170
|
+
|
|
164
171
|
export interface SyncOptions {
|
|
172
|
+
/**
|
|
173
|
+
* Internal: called when graph reconciliation cannot proceed incrementally
|
|
174
|
+
* and falls back to a full projection of an existing graph.
|
|
175
|
+
*/
|
|
176
|
+
onGraphRebuild?: (reason: GraphRebuildReason) => void;
|
|
165
177
|
/** Index-wide configured policy; omitted means the existing defaults. */
|
|
166
178
|
chunking?: Partial<ChunkingParams>;
|
|
167
179
|
/** Internal token passed from an outer sync; never a public CLI override. */
|
|
@@ -298,6 +310,8 @@ export interface CollectionSyncResult {
|
|
|
298
310
|
export interface SyncResult {
|
|
299
311
|
/** Cached layouts updated independently of source-refresh file counters. */
|
|
300
312
|
rechunkedMirrors?: number;
|
|
313
|
+
/** Present when the link graph was rebuilt in full rather than reconciled. */
|
|
314
|
+
graphRebuild?: GraphRebuildReason;
|
|
301
315
|
collections: CollectionSyncResult[];
|
|
302
316
|
totalDurationMs: number;
|
|
303
317
|
totalFilesProcessed: number;
|