archgraph-argo 0.20.2 → 0.20.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/argo/.env.example +11 -1
- package/argo/scripts/argo-mcp-server.js +11 -0
- package/argo/scripts/graph-rag/ARCHITECTURE.md +1 -1
- package/argo/scripts/graph-rag/defaultSemanticRetrieval.js +30 -30
- package/argo/scripts/graph-rag/liveEmbeddingProviderConfig.js +2 -0
- package/argo/scripts/graph-rag/rerankRetrieval.js +63 -12
- package/argo/scripts/graph-rag/semantic-persistence/productionSemanticBackfill.js +56 -1
- package/argo/scripts/graph-rag/semanticAlignmentRunner.js +95 -0
- package/argo/scripts/systemarchitecture-mcp-server.js +1 -1
- package/package.json +1 -1
package/argo/.env.example
CHANGED
|
@@ -89,7 +89,7 @@ ARGO_SEMANTIC_RERANK_MODEL=
|
|
|
89
89
|
ARGO_SEMANTIC_RERANK_POOL=
|
|
90
90
|
# Max ids the reranker may return. Default 8.
|
|
91
91
|
ARGO_SEMANTIC_RERANK_RETURN=
|
|
92
|
-
# Per-request rerank timeout in ms (AbortController). Default
|
|
92
|
+
# Per-request rerank timeout in ms (AbortController). Default 3500.
|
|
93
93
|
ARGO_SEMANTIC_RERANK_TIMEOUT_MS=
|
|
94
94
|
# Dedicated rerank provider (optional). When unset, rerank falls back to the
|
|
95
95
|
# embedding provider above. Use these to point rerank at another provider/model
|
|
@@ -101,6 +101,16 @@ ARGO_RERANK_API_KEY=
|
|
|
101
101
|
ARGO_RERANK_PROVIDER=
|
|
102
102
|
# Dedicated rerank model id (overrides ARGO_SEMANTIC_RERANK_MODEL).
|
|
103
103
|
ARGO_RERANK_MODEL=
|
|
104
|
+
# Disable the rerank model's hidden "thinking"/reasoning (reasoning models spend
|
|
105
|
+
# 6-12s per call for a listwise ranking with no accuracy gain). Default 1 =
|
|
106
|
+
# disabled; set 0 to send nothing. Applies to every provider so a model swap
|
|
107
|
+
# keeps the fast path. If a provider rejects the fragment it is retried without.
|
|
108
|
+
ARGO_RERANK_DISABLE_THINKING=
|
|
109
|
+
# Exact JSON body fragment merged into the rerank request to disable thinking
|
|
110
|
+
# (default {"thinking":{"type":"disabled"}}). Override for another model/provider
|
|
111
|
+
# that uses a different field, e.g. {"reasoning_effort":"none"} or
|
|
112
|
+
# {"enable_thinking":false}.
|
|
113
|
+
ARGO_RERANK_THINKING_PARAM=
|
|
104
114
|
|
|
105
115
|
# --- Live end-to-end opt-ins (optional; normally unset) ---------------------
|
|
106
116
|
# "1" allows the live embedding-provider E2E to hit the real network.
|
|
@@ -773,6 +773,17 @@ async function handleRequest(request, dependencies = undefined) {
|
|
|
773
773
|
}
|
|
774
774
|
|
|
775
775
|
async function main() {
|
|
776
|
+
// Recall-safe warm-up: if the semantic readiness record is not Aligned when the
|
|
777
|
+
// server starts, rebuild the index in the BACKGROUND (async, logged) so the
|
|
778
|
+
// first query rarely pays the multi-second reconstruction. No-op when aligned.
|
|
779
|
+
if (process.env.ARGO_REPO_ROOT && process.env.ARGO_REPO_ROOT.trim() !== '') {
|
|
780
|
+
try {
|
|
781
|
+
require('./graph-rag/semanticAlignmentRunner.js')
|
|
782
|
+
.preheatSemanticAlignment(process.env.ARGO_REPO_ROOT);
|
|
783
|
+
} catch {
|
|
784
|
+
// preheat is best-effort; never block server startup
|
|
785
|
+
}
|
|
786
|
+
}
|
|
776
787
|
const rl = readline.createInterface({
|
|
777
788
|
input: process.stdin,
|
|
778
789
|
crlfDelay: Infinity,
|
|
@@ -27,7 +27,7 @@ This local contract refines `OVERALL_ARCHITECTURE.md`.
|
|
|
27
27
|
- DT-19 capacity evidence begins after real DT-18 quality evidence. It must preserve explicit `observedResultIds` or equivalent result ids from actual phase-1 query observations, never use closure fallback as result evidence, record measured precision for every declared purpose, validate any provided result cardinality against the result ID count so cardinality mismatch blocks, block explicitly when quality evidence or result evidence is missing, and must not return or infer cap, budget, pagination, truncation, continuation, top-k, token-budget, result-limit, or capacity-policy decisions. This is evidence for later human capacity governance, not a release threshold.
|
|
28
28
|
- Seven-wave delivery gating must block W7 when W2-W6 prerequisite evidence is incomplete, block whole delivery when the W7 DT-18 business benchmark is missing, failing, incomplete, or carries precision outside `[0, 1]`, and permit overall delivery only after prerequisites and the W7 benchmark pass. DT-19 capacity controls remain evidence-only and cannot become a W7 release rule.
|
|
29
29
|
- WP-P1 delegates durable production persistence and full-backfill orchestration inward to `semantic-persistence/`. `productionGraphRagRuntime.js` composes concrete production Neo4j projection/checkpoint adapters from `semanticPersistence` dependencies and exposes `runSemanticBackfill(request)`; it must not absorb checkpoint, durable-store, tombstone, or production-cleanup policy.
|
|
30
|
-
- WP-P1 semantic backfill is invoked only by private canonical argo init composition after internal exact consent and verified same-version structural projection, never by a public standalone MCP route or inferred mutation signal. Missing consent, version match, external credentials, or provider qualification blocks before provider/store/checkpoint/index side effects. It independently completes Element, ArchitectureRelationship, and View through bounded durable checkpoints before Aligned.
|
|
30
|
+
- WP-P1 semantic backfill is invoked only by private canonical argo init composition after internal exact consent and verified same-version structural projection, never by a public standalone MCP route or inferred mutation signal. Missing consent, version match, external credentials, or provider qualification blocks before provider/store/checkpoint/index side effects. It independently completes Element, ArchitectureRelationship, and View through bounded durable checkpoints before Aligned. Its resume checkpoint is contentHash-incremental: when the canonical version changes, records whose curated semantic text (contentHash) and embedding provider/model/dimensions already match are skipped (not re-embedded); only changed/new records are embedded, so recall is preserved and a version bump stays cheap.
|
|
31
31
|
- The separate production store upserts changed stable canonical identities, deletes tombstones, preserves complete canonical/content/index/provider/model/version/dimensions/vector metadata across restart, rejects `runId` records, and exposes exactly four methods with no cleanup API. Existing `liveEmbeddingNeo4jBoundary.js` remains test-evidence infrastructure and unchanged.
|
|
32
32
|
- `defaultSemanticRetrieval.js` is the WP-P2 deep composition module for the shipped uninjected MCP semantic path. `createDefaultSemanticRetrieval(dependencies)` accepts the unified production `readinessBoundary` and returns `retrieve(request)` plus the narrow read-only `readReadiness()`. Both methods resolve through one private `readAndEvaluatePersistentReadiness` path that retains the existing approved composition, normalization, threshold filtering, expanding-window exhaustion, purpose closure, endpoint/View closure, provenance, and canonical completion. The safe failure envelope preserves stored `category`, `message`, and `action` plus state/version/channel evidence and `fullSnapshotFallback: false`; unknown diagnostic fields and secrets never cross the boundary.
|
|
33
33
|
- Approved external-source resolution must directly read and attribute all required embedding-profile, Neo4j, and provider-secret keys before readiness; success with zero or partial approved-source reads is forbidden. Production-semantic-query directly inspects all three legacy Neo4j alias keys and rejects legacy-only or mixed valid-canonical-plus-any-legacy-alias sources with `SECRET_SOURCE_PROVENANCE_PROHIBITED` before attribution, selection, readiness, provider, or driver effects. Inspection cannot attribute, normalize, return, or use a legacy value. It also rejects missing secrets, unsafe `.argo/.env` path/git/file/reparse/ACL state, conflicting process/file values, and test/default, literal, fallback, alias, or indirect credential operations. The test composition accepts raw `sourceBehavior` plus filesystem/git/ACL adapters; ready-made environment or normalized configuration objects are prohibited.
|
|
@@ -29,6 +29,9 @@ const {
|
|
|
29
29
|
rerankCandidates,
|
|
30
30
|
applyRerankOrder,
|
|
31
31
|
} = require('./rerankRetrieval.js');
|
|
32
|
+
const {
|
|
33
|
+
runSemanticAlignment,
|
|
34
|
+
} = require('./semanticAlignmentRunner.js');
|
|
32
35
|
|
|
33
36
|
const APPROVED_SOURCE_KEYS = Object.freeze([
|
|
34
37
|
'ARGO_EMBEDDING_BASE_URL',
|
|
@@ -240,12 +243,12 @@ async function executeWpP2Retrieval({
|
|
|
240
243
|
const lexicalTopK = hybridTopK();
|
|
241
244
|
const fusionK = rrfK();
|
|
242
245
|
const fusionWeights = hybridWeights();
|
|
243
|
-
const rerank = isRerankEnabled();
|
|
246
|
+
const rerank = isRerankEnabled() && request.rerank !== false;
|
|
244
247
|
const rerankOptions = rerankConfig();
|
|
245
248
|
const rerankProvider = rerank ? resolveRerankProvider(configurationEvidence.configuration) : null;
|
|
246
249
|
// Rerank needs a larger candidate pool than the final top-K.
|
|
247
250
|
const pool = rerank ? Math.max(topK, rerankOptions.poolSize) : topK;
|
|
248
|
-
const
|
|
251
|
+
const channelSeeds = [];
|
|
249
252
|
for (const channel of CHANNELS) {
|
|
250
253
|
const vectorSeeds = await exhaustChannel({
|
|
251
254
|
channel,
|
|
@@ -266,17 +269,27 @@ async function executeWpP2Retrieval({
|
|
|
266
269
|
});
|
|
267
270
|
seeds = fuseChannelSeeds({ vectorSeeds, lexicalSeeds, k: fusionK, limit: Math.max(pool, lexicalTopK), weights: fusionWeights });
|
|
268
271
|
}
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
272
|
+
channelSeeds.push({ channel, seeds });
|
|
273
|
+
}
|
|
274
|
+
if (rerank) {
|
|
275
|
+
// Rerank every channel CONCURRENTLY: the LLM calls dominate latency and are
|
|
276
|
+
// independent, so parallelizing turns the cost from sum(channels) into
|
|
277
|
+
// ~one call. fail-open: a null/empty order keeps the original ordering.
|
|
278
|
+
const rerankedSeeds = await Promise.all(channelSeeds.map(({ seeds }) => (
|
|
279
|
+
seeds.length > 1
|
|
280
|
+
? rerankCandidates({
|
|
281
|
+
query: request.intent,
|
|
282
|
+
candidates: seeds,
|
|
283
|
+
provider: rerankProvider,
|
|
284
|
+
transport: composition.transport,
|
|
285
|
+
maxReturn: rerankOptions.maxReturn,
|
|
286
|
+
}).then(ordered => applyRerankOrder(seeds, ordered, topK))
|
|
287
|
+
: seeds
|
|
288
|
+
)));
|
|
289
|
+
channelSeeds.forEach((entry, index) => { entry.seeds = rerankedSeeds[index]; });
|
|
290
|
+
}
|
|
291
|
+
const seedsByType = {};
|
|
292
|
+
for (const { channel, seeds } of channelSeeds) {
|
|
280
293
|
seedsByType[channel.key] = seeds;
|
|
281
294
|
}
|
|
282
295
|
return completeSemanticResult({
|
|
@@ -413,25 +426,12 @@ async function attemptAutomaticAlignment({ composition, request, alignment }) {
|
|
|
413
426
|
}
|
|
414
427
|
|
|
415
428
|
function runScriptOwnedSemanticAlignment(operation) {
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
const scriptPath = resolveArgoPath('scripts', 'ensureArgoHarnessEnvironment.js');
|
|
419
|
-
const result = childProcess.spawnSync(process.execPath, [scriptPath], {
|
|
420
|
-
cwd: repositoryRoot,
|
|
421
|
-
encoding: 'utf8',
|
|
422
|
-
env: process.env,
|
|
423
|
-
});
|
|
424
|
-
if (result.status === 0) {
|
|
425
|
-
return Object.freeze({
|
|
429
|
+
return runSemanticAlignment(getWorkspaceRoot()).then(
|
|
430
|
+
() => Object.freeze({
|
|
426
431
|
status: 'aligned',
|
|
427
432
|
originalQuery: operation && operation.originalQuery,
|
|
428
|
-
})
|
|
429
|
-
|
|
430
|
-
const error = safeError('SEMANTIC_AUTO_ALIGNMENT_FAILED');
|
|
431
|
-
error.message = 'Semantic automatic alignment failed before retry.';
|
|
432
|
-
error.action = 'Repair semantic lifecycle alignment, then retry the original query.';
|
|
433
|
-
error.fullSnapshotFallback = false;
|
|
434
|
-
return Promise.reject(error);
|
|
433
|
+
}),
|
|
434
|
+
);
|
|
435
435
|
}
|
|
436
436
|
|
|
437
437
|
async function resolveRawTestConfiguration(sourceBehavior, sourceAdapters) {
|
|
@@ -52,6 +52,8 @@ const RETRIEVAL_TUNING_KEYS = Object.freeze([
|
|
|
52
52
|
'ARGO_RERANK_API_KEY',
|
|
53
53
|
'ARGO_RERANK_PROVIDER',
|
|
54
54
|
'ARGO_RERANK_MODEL',
|
|
55
|
+
'ARGO_RERANK_DISABLE_THINKING',
|
|
56
|
+
'ARGO_RERANK_THINKING_PARAM',
|
|
55
57
|
]);
|
|
56
58
|
const OPT_IN_KEYS = Object.freeze({
|
|
57
59
|
ARGO_LIVE_PROVIDER_E2E: 'LIVE_PROVIDER_E2E_OPT_IN_REQUIRED',
|
|
@@ -6,9 +6,18 @@
|
|
|
6
6
|
// the caller keeps the original ordering. Pure helpers are exported for tests.
|
|
7
7
|
|
|
8
8
|
const DEFAULT_RERANK_MODEL = 'qwen-turbo';
|
|
9
|
+
// Candidate pool size. The pool is the recall CEILING: a target the seed stage
|
|
10
|
+
// ranks outside the pool can never be recovered by rerank, so it must NOT be
|
|
11
|
+
// shrunk for speed. Keep the original 20. With DeepSeek thinking disabled a
|
|
12
|
+
// single rerank call is ~flat across candidate counts (8-40 all ~1s), so a
|
|
13
|
+
// larger pool costs ~nothing -- never trade recall for latency here.
|
|
9
14
|
const DEFAULT_RERANK_POOL = 20;
|
|
10
15
|
const DEFAULT_RERANK_RETURN = 8;
|
|
11
|
-
|
|
16
|
+
// Per-call timeout. Channel reranks run concurrently, so the end-to-end rerank
|
|
17
|
+
// cost is ~one timeout, not N. 3.5s keeps the whole semantic query under ~5s
|
|
18
|
+
// while still letting typical calls (1-4s) complete; slower calls fail open to
|
|
19
|
+
// the pre-rerank (fused) order. Overridable via ARGO_SEMANTIC_RERANK_TIMEOUT_MS.
|
|
20
|
+
const DEFAULT_RERANK_TIMEOUT_MS = 3500;
|
|
12
21
|
|
|
13
22
|
function rerankTimeoutMs(env = process.env) {
|
|
14
23
|
const value = Number(env && env.ARGO_SEMANTIC_RERANK_TIMEOUT_MS);
|
|
@@ -51,6 +60,32 @@ function rerankConfig(env = process.env) {
|
|
|
51
60
|
};
|
|
52
61
|
}
|
|
53
62
|
|
|
63
|
+
// Model-agnostic "disable thinking/reasoning" body fragment. Reasoning models
|
|
64
|
+
// spend seconds on hidden reasoning tokens for a simple listwise ranking with no
|
|
65
|
+
// accuracy gain, so disable it by default for EVERY rerank provider (not just
|
|
66
|
+
// deepseek), so switching models keeps the same behaviour. Providers use
|
|
67
|
+
// different field names, so it is fully overridable:
|
|
68
|
+
// ARGO_RERANK_DISABLE_THINKING=0 -> send nothing
|
|
69
|
+
// ARGO_RERANK_THINKING_PARAM='{...}' -> send this exact JSON fragment
|
|
70
|
+
// rerankCandidates retries WITHOUT the fragment if the provider rejects it, so an
|
|
71
|
+
// unsupported field can never break rerank (recall-first: never lose rerank to a
|
|
72
|
+
// 400).
|
|
73
|
+
const DEFAULT_RERANK_THINKING_OFF = Object.freeze({ thinking: { type: 'disabled' } });
|
|
74
|
+
|
|
75
|
+
function resolveRerankThinkingOff(env = process.env) {
|
|
76
|
+
if (String((env && env.ARGO_RERANK_DISABLE_THINKING) || '') === '0') return null;
|
|
77
|
+
const raw = env && env.ARGO_RERANK_THINKING_PARAM;
|
|
78
|
+
if (typeof raw === 'string' && raw.trim() !== '') {
|
|
79
|
+
try {
|
|
80
|
+
const parsed = JSON.parse(raw);
|
|
81
|
+
return parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? parsed : null;
|
|
82
|
+
} catch {
|
|
83
|
+
return null;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
return DEFAULT_RERANK_THINKING_OFF;
|
|
87
|
+
}
|
|
88
|
+
|
|
54
89
|
function candidateText(record) {
|
|
55
90
|
const raw = record && (record.searchText || record.description || record.name || '');
|
|
56
91
|
return String(raw).replace(/\s+/g, ' ').slice(0, 160);
|
|
@@ -106,7 +141,8 @@ async function rerankCandidates({ query, candidates, provider, transport, maxRet
|
|
|
106
141
|
const apiKey = typeof provider.apiKey === 'string' ? provider.apiKey : '';
|
|
107
142
|
if (baseUrl === '' || apiKey === '') return null;
|
|
108
143
|
const model = provider.model || DEFAULT_RERANK_MODEL;
|
|
109
|
-
const
|
|
144
|
+
const thinkingOff = resolveRerankThinkingOff();
|
|
145
|
+
const baseBody = {
|
|
110
146
|
model,
|
|
111
147
|
temperature: 0,
|
|
112
148
|
response_format: { type: 'json_object' },
|
|
@@ -115,23 +151,36 @@ async function rerankCandidates({ query, candidates, provider, transport, maxRet
|
|
|
115
151
|
{ role: 'user', content: `Query: ${query}\n\nCandidates (id\\ttext):\n${list.map(candidate => `${candidate.id}\t${candidateText(candidate)}`).join('\n')}\n\nReturn up to ${Math.min(maxReturn || DEFAULT_RERANK_RETURN, list.length)} ids best-first.` },
|
|
116
152
|
],
|
|
117
153
|
};
|
|
118
|
-
const
|
|
119
|
-
const
|
|
154
|
+
const timeout = timeoutMs || rerankTimeoutMs();
|
|
155
|
+
const send = async (extra) => {
|
|
156
|
+
const controller = typeof AbortController === 'function' ? new AbortController() : null;
|
|
157
|
+
const timer = controller ? setTimeout(() => controller.abort(), timeout) : null;
|
|
158
|
+
try {
|
|
159
|
+
return await transport.request(`${baseUrl}/chat/completions`, {
|
|
160
|
+
method: 'POST',
|
|
161
|
+
headers: { Authorization: `Bearer ${apiKey}`, 'Content-Type': 'application/json' },
|
|
162
|
+
body: JSON.stringify({ ...baseBody, ...extra }),
|
|
163
|
+
...(controller ? { signal: controller.signal } : {}),
|
|
164
|
+
});
|
|
165
|
+
} catch {
|
|
166
|
+
return null;
|
|
167
|
+
} finally {
|
|
168
|
+
if (timer) clearTimeout(timer);
|
|
169
|
+
}
|
|
170
|
+
};
|
|
120
171
|
try {
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
}
|
|
172
|
+
let response = await send(thinkingOff || {});
|
|
173
|
+
// A provider may reject the thinking-off fragment (unknown field -> 400).
|
|
174
|
+
// Retry WITHOUT it so rerank still works: never lose rerank/recall to a 400.
|
|
175
|
+
if ((!response || response.ok !== true) && thinkingOff) {
|
|
176
|
+
response = await send({});
|
|
177
|
+
}
|
|
127
178
|
if (!response || response.ok !== true || typeof response.json !== 'function') return null;
|
|
128
179
|
const payload = await response.json();
|
|
129
180
|
const content = payload && payload.choices && payload.choices[0] && payload.choices[0].message && payload.choices[0].message.content;
|
|
130
181
|
return parseRerankOrder(content, list.map(candidate => candidate.id));
|
|
131
182
|
} catch {
|
|
132
183
|
return null;
|
|
133
|
-
} finally {
|
|
134
|
-
if (timer) clearTimeout(timer);
|
|
135
184
|
}
|
|
136
185
|
}
|
|
137
186
|
|
|
@@ -144,6 +193,8 @@ module.exports = {
|
|
|
144
193
|
rerankConfig,
|
|
145
194
|
rerankTimeoutMs,
|
|
146
195
|
resolveRerankProvider,
|
|
196
|
+
resolveRerankThinkingOff,
|
|
197
|
+
DEFAULT_RERANK_THINKING_OFF,
|
|
147
198
|
parseRerankOrder,
|
|
148
199
|
applyRerankOrder,
|
|
149
200
|
rerankCandidates,
|
|
@@ -90,7 +90,7 @@ async function processChannel(options) {
|
|
|
90
90
|
const stored = await options.checkpointStore.readCheckpoint(options.channel);
|
|
91
91
|
const checkpoint = stored && stored.canonicalVersion === options.canonicalVersion
|
|
92
92
|
? mutableCheckpoint(stored, canonicalRecords.length)
|
|
93
|
-
:
|
|
93
|
+
: await resumeCheckpointByContent(options, canonicalRecords);
|
|
94
94
|
const attempts = new Map();
|
|
95
95
|
|
|
96
96
|
while (checkpoint.completedCanonicalIdentities.length < canonicalRecords.length) {
|
|
@@ -153,6 +153,61 @@ async function processChannel(options) {
|
|
|
153
153
|
});
|
|
154
154
|
}
|
|
155
155
|
|
|
156
|
+
// Cross-version incremental resume. The resume checkpoint is keyed by
|
|
157
|
+
// canonicalVersion, but ANY add/remove of an element/relationship/view changes
|
|
158
|
+
// that version. Re-embedding every record on every such change is wasteful and
|
|
159
|
+
// slow (~hundreds of provider calls: measured ~174s). Instead, when the stored
|
|
160
|
+
// checkpoint is for a different version, seed the completed set from the
|
|
161
|
+
// persisted records whose curated semantic text (contentHash) AND embedding
|
|
162
|
+
// provider/model/dimensions already match the current graph; only changed/new
|
|
163
|
+
// records get embedded. Recall is unchanged: changed content is ALWAYS
|
|
164
|
+
// re-embedded, only byte-identical text is skipped.
|
|
165
|
+
async function resumeCheckpointByContent(options, canonicalRecords) {
|
|
166
|
+
const checkpoint = emptyCheckpoint(options.channel, options.canonicalVersion, canonicalRecords.length);
|
|
167
|
+
let persisted = [];
|
|
168
|
+
try {
|
|
169
|
+
const stored = await options.projectionStore.readRecords();
|
|
170
|
+
persisted = Array.isArray(stored) ? stored : [];
|
|
171
|
+
} catch {
|
|
172
|
+
persisted = [];
|
|
173
|
+
}
|
|
174
|
+
const byIdentity = new Map();
|
|
175
|
+
for (const record of persisted) {
|
|
176
|
+
if (record && record.channel === options.channel && isNonBlankString(record.canonicalIdentity)) {
|
|
177
|
+
byIdentity.set(record.canonicalIdentity, record);
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
const completed = [];
|
|
181
|
+
for (const record of canonicalRecords) {
|
|
182
|
+
const existing = byIdentity.get(record.canonicalIdentity);
|
|
183
|
+
if (existing && persistedRecordMatches(existing, record, options.qualification)) {
|
|
184
|
+
completed.push(record.canonicalIdentity);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
checkpoint.completedCanonicalIdentities = completed;
|
|
188
|
+
checkpoint.completedCount = completed.length;
|
|
189
|
+
checkpoint.cursor = completed.length;
|
|
190
|
+
checkpoint.status = completed.length === canonicalRecords.length ? 'complete' : 'pending';
|
|
191
|
+
return checkpoint;
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
function persistedRecordMatches(existing, canonicalRecord, qualification) {
|
|
195
|
+
if (!Array.isArray(existing.vector)) {
|
|
196
|
+
return false;
|
|
197
|
+
}
|
|
198
|
+
const searchText = buildSemanticRecordText(canonicalRecord.channel, canonicalRecord.canonicalObject);
|
|
199
|
+
const contentHash = crypto.createHash('sha256').update(searchText).digest('hex');
|
|
200
|
+
return existing.contentVersion === `content:${contentHash}`
|
|
201
|
+
&& existing.provider === qualification.provider
|
|
202
|
+
&& existing.model === qualification.model
|
|
203
|
+
&& existing.modelVersion === qualification.version
|
|
204
|
+
&& toNumber(existing.dimensions) === toNumber(qualification.dimensions);
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
function toNumber(value) {
|
|
208
|
+
return value && typeof value.toNumber === 'function' ? value.toNumber() : Number(value);
|
|
209
|
+
}
|
|
210
|
+
|
|
156
211
|
function buildSemanticRecord(record, vector, options) {
|
|
157
212
|
const searchText = buildSemanticRecordText(record.channel, record.canonicalObject);
|
|
158
213
|
const contentHash = crypto.createHash('sha256')
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Owns the (expensive) semantic-index alignment child process. When readiness is
|
|
4
|
+
// NOT aligned, the index must be rebuilt to serve correct (full-recall) results;
|
|
5
|
+
// that rebuild can take seconds to ~90s. This module:
|
|
6
|
+
// - runs it ASYNCHRONOUSLY (child_process.spawn, not spawnSync) so the MCP
|
|
7
|
+
// event loop is never frozen while it runs;
|
|
8
|
+
// - de-duplicates concurrent requests (one rebuild shared by all waiters);
|
|
9
|
+
// - logs start/finish/failure for observability;
|
|
10
|
+
// - exposes preheatSemanticAlignment() for a background warm-up at startup.
|
|
11
|
+
// It never changes WHAT is retrieved (recall/precision), only how the rebuild is
|
|
12
|
+
// scheduled/observed.
|
|
13
|
+
|
|
14
|
+
const fs = require('node:fs');
|
|
15
|
+
const path = require('node:path');
|
|
16
|
+
const { spawn } = require('node:child_process');
|
|
17
|
+
const { getWorkspaceRoot, resolveArgoPath } = require('../argo-paths.js');
|
|
18
|
+
|
|
19
|
+
const READINESS_RELATIVE_PATH = path.join('.argo', 'temp', 'system-architecture-semantic-readiness.json');
|
|
20
|
+
|
|
21
|
+
let inFlight = null;
|
|
22
|
+
let preheated = false;
|
|
23
|
+
|
|
24
|
+
function readinessRecordPath(repositoryRoot) {
|
|
25
|
+
return path.join(repositoryRoot, ...READINESS_RELATIVE_PATH.split(path.sep));
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function isSemanticReady(repositoryRoot) {
|
|
29
|
+
try {
|
|
30
|
+
const parsed = JSON.parse(fs.readFileSync(readinessRecordPath(repositoryRoot), 'utf8'));
|
|
31
|
+
return !!parsed && parsed.state === 'Aligned' && parsed.verified === true;
|
|
32
|
+
} catch {
|
|
33
|
+
return false;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function alignmentError() {
|
|
38
|
+
const error = new Error('SEMANTIC_AUTO_ALIGNMENT_FAILED');
|
|
39
|
+
error.category = 'SEMANTIC_AUTO_ALIGNMENT_FAILED';
|
|
40
|
+
error.message = 'Semantic automatic alignment failed before retry.';
|
|
41
|
+
error.action = 'Repair semantic lifecycle alignment, then retry the original query.';
|
|
42
|
+
error.fullSnapshotFallback = false;
|
|
43
|
+
return error;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
// Run the alignment (once). Returns a promise resolving to { status: 'aligned' }
|
|
47
|
+
// or rejecting with the SEMANTIC_AUTO_ALIGNMENT_FAILED envelope.
|
|
48
|
+
function runSemanticAlignment(repositoryRoot = getWorkspaceRoot()) {
|
|
49
|
+
if (inFlight) {
|
|
50
|
+
return inFlight;
|
|
51
|
+
}
|
|
52
|
+
const scriptPath = resolveArgoPath('scripts', 'ensureArgoHarnessEnvironment.js');
|
|
53
|
+
const startedAt = Date.now();
|
|
54
|
+
console.error('[argo] semantic auto-alignment: index not aligned, rebuilding (recall-safe)…');
|
|
55
|
+
inFlight = new Promise((resolve, reject) => {
|
|
56
|
+
const child = spawn(process.execPath, [scriptPath], {
|
|
57
|
+
cwd: repositoryRoot,
|
|
58
|
+
env: process.env,
|
|
59
|
+
stdio: ['ignore', 'ignore', 'inherit'],
|
|
60
|
+
});
|
|
61
|
+
const finish = (failed, cause) => {
|
|
62
|
+
inFlight = null;
|
|
63
|
+
const ms = Date.now() - startedAt;
|
|
64
|
+
if (failed) {
|
|
65
|
+
console.error(`[argo] semantic auto-alignment FAILED in ${ms}ms${cause ? ': ' + cause : ''}`);
|
|
66
|
+
reject(alignmentError());
|
|
67
|
+
} else {
|
|
68
|
+
console.error(`[argo] semantic auto-alignment done in ${ms}ms`);
|
|
69
|
+
resolve(Object.freeze({ status: 'aligned' }));
|
|
70
|
+
}
|
|
71
|
+
};
|
|
72
|
+
child.on('error', (error) => finish(true, error && error.message));
|
|
73
|
+
child.on('exit', (code, signal) => finish(code !== 0, `code=${code} signal=${signal || '-'}`));
|
|
74
|
+
});
|
|
75
|
+
return inFlight;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
// Fire-and-forget warm-up: if readiness is already Aligned it does nothing;
|
|
79
|
+
// otherwise it starts the rebuild in the background so the first query rarely
|
|
80
|
+
// pays for it. Safe to call repeatedly (guarded + de-duplicated).
|
|
81
|
+
function preheatSemanticAlignment(repositoryRoot = getWorkspaceRoot()) {
|
|
82
|
+
if (preheated || !repositoryRoot || isSemanticReady(repositoryRoot)) {
|
|
83
|
+
return;
|
|
84
|
+
}
|
|
85
|
+
preheated = true;
|
|
86
|
+
console.error('[argo] readiness not aligned at startup; preheating semantic alignment in background…');
|
|
87
|
+
runSemanticAlignment(repositoryRoot).catch(() => {});
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
module.exports = {
|
|
91
|
+
runSemanticAlignment,
|
|
92
|
+
preheatSemanticAlignment,
|
|
93
|
+
isSemanticReady,
|
|
94
|
+
readinessRecordPath,
|
|
95
|
+
};
|
|
@@ -2801,7 +2801,7 @@ async function buildSemanticDedupAdvisory(context, mutations, dependencies) {
|
|
|
2801
2801
|
const intent = [element.type, element.name, element.description]
|
|
2802
2802
|
.filter(part => typeof part === 'string' && part.trim() !== '')
|
|
2803
2803
|
.join(' ');
|
|
2804
|
-
const retrieved = await journey.query({ purpose: 'general', intent, topK: SEMANTIC_DEDUP_TOP_K });
|
|
2804
|
+
const retrieved = await journey.query({ purpose: 'general', intent, topK: SEMANTIC_DEDUP_TOP_K, rerank: false });
|
|
2805
2805
|
const source = retrieved && (retrieved.result || retrieved.document) || retrieved;
|
|
2806
2806
|
const subset = buildCanonicalSemanticDocumentSubset(source, context.document);
|
|
2807
2807
|
const elements = subset && subset.status === 'passed' && subset.document
|
package/package.json
CHANGED