archgraph-argo 0.20.3 → 0.20.4

package/argo/.env.example

@@ -89,7 +89,7 @@ ARGO_SEMANTIC_RERANK_MODEL=
 ARGO_SEMANTIC_RERANK_POOL=
 # Max ids the reranker may return. Default 8.
 ARGO_SEMANTIC_RERANK_RETURN=
-# Per-request rerank timeout in ms (AbortController). Default 8000.
+# Per-request rerank timeout in ms (AbortController). Default 3500.
 ARGO_SEMANTIC_RERANK_TIMEOUT_MS=
 # Dedicated rerank provider (optional). When unset, rerank falls back to the
 # embedding provider above. Use these to point rerank at another provider/model
@@ -101,6 +101,16 @@ ARGO_RERANK_API_KEY=
 ARGO_RERANK_PROVIDER=
 # Dedicated rerank model id (overrides ARGO_SEMANTIC_RERANK_MODEL).
 ARGO_RERANK_MODEL=
+# Disable the rerank model's hidden "thinking"/reasoning (reasoning models spend
+# 6-12s per call for a listwise ranking with no accuracy gain). Default 1 =
+# disabled; set 0 to send nothing. Applies to every provider so a model swap
+# keeps the fast path. If a provider rejects the fragment it is retried without.
+ARGO_RERANK_DISABLE_THINKING=
+# Exact JSON body fragment merged into the rerank request to disable thinking
+# (default {"thinking":{"type":"disabled"}}). Override for another model/provider
+# that uses a different field, e.g. {"reasoning_effort":"none"} or
+# {"enable_thinking":false}.
+ARGO_RERANK_THINKING_PARAM=
 
 # --- Live end-to-end opt-ins (optional; normally unset) ---------------------
 # "1" allows the live embedding-provider E2E to hit the real network.

package/argo/scripts/argo-mcp-server.js

@@ -773,6 +773,17 @@ async function handleRequest(request, dependencies = undefined) {
 }
 
 async function main() {
+  // Recall-safe warm-up: if the semantic readiness record is not Aligned when the
+  // server starts, rebuild the index in the BACKGROUND (async, logged) so the
+  // first query rarely pays the multi-second reconstruction. No-op when aligned.
+  if (process.env.ARGO_REPO_ROOT && process.env.ARGO_REPO_ROOT.trim() !== '') {
+    try {
+      require('./graph-rag/semanticAlignmentRunner.js')
+        .preheatSemanticAlignment(process.env.ARGO_REPO_ROOT);
+    } catch {
+      // preheat is best-effort; never block server startup
+    }
+  }
   const rl = readline.createInterface({
     input: process.stdin,
     crlfDelay: Infinity,

package/argo/scripts/graph-rag/ARCHITECTURE.md

@@ -27,7 +27,7 @@ This local contract refines `OVERALL_ARCHITECTURE.md`.
 - DT-19 capacity evidence begins after real DT-18 quality evidence. It must preserve explicit `observedResultIds` or equivalent result ids from actual phase-1 query observations, never use closure fallback as result evidence, record measured precision for every declared purpose, validate any provided result cardinality against the result ID count so cardinality mismatch blocks, block explicitly when quality evidence or result evidence is missing, and must not return or infer cap, budget, pagination, truncation, continuation, top-k, token-budget, result-limit, or capacity-policy decisions. This is evidence for later human capacity governance, not a release threshold.
 - Seven-wave delivery gating must block W7 when W2-W6 prerequisite evidence is incomplete, block whole delivery when the W7 DT-18 business benchmark is missing, failing, incomplete, or carries precision outside `[0, 1]`, and permit overall delivery only after prerequisites and the W7 benchmark pass. DT-19 capacity controls remain evidence-only and cannot become a W7 release rule.
 - WP-P1 delegates durable production persistence and full-backfill orchestration inward to `semantic-persistence/`. `productionGraphRagRuntime.js` composes concrete production Neo4j projection/checkpoint adapters from `semanticPersistence` dependencies and exposes `runSemanticBackfill(request)`; it must not absorb checkpoint, durable-store, tombstone, or production-cleanup policy.
-- WP-P1 semantic backfill is invoked only by private canonical argo init composition after internal exact consent and verified same-version structural projection, never by a public standalone MCP route or inferred mutation signal. Missing consent, version match, external credentials, or provider qualification blocks before provider/store/checkpoint/index side effects. It independently completes Element, ArchitectureRelationship, and View through bounded durable checkpoints before Aligned.
+- WP-P1 semantic backfill is invoked only by private canonical argo init composition after internal exact consent and verified same-version structural projection, never by a public standalone MCP route or inferred mutation signal. Missing consent, version match, external credentials, or provider qualification blocks before provider/store/checkpoint/index side effects. It independently completes Element, ArchitectureRelationship, and View through bounded durable checkpoints before Aligned. Its resume checkpoint is contentHash-incremental: when the canonical version changes, records whose curated semantic text (contentHash) and embedding provider/model/dimensions already match are skipped (not re-embedded); only changed/new records are embedded, so recall is preserved and a version bump stays cheap.
 - The separate production store upserts changed stable canonical identities, deletes tombstones, preserves complete canonical/content/index/provider/model/version/dimensions/vector metadata across restart, rejects `runId` records, and exposes exactly four methods with no cleanup API. Existing `liveEmbeddingNeo4jBoundary.js` remains test-evidence infrastructure and unchanged.
 - `defaultSemanticRetrieval.js` is the WP-P2 deep composition module for the shipped uninjected MCP semantic path. `createDefaultSemanticRetrieval(dependencies)` accepts the unified production `readinessBoundary` and returns `retrieve(request)` plus the narrow read-only `readReadiness()`. Both methods resolve through one private `readAndEvaluatePersistentReadiness` path that retains the existing approved composition, normalization, threshold filtering, expanding-window exhaustion, purpose closure, endpoint/View closure, provenance, and canonical completion. The safe failure envelope preserves stored `category`, `message`, and `action` plus state/version/channel evidence and `fullSnapshotFallback: false`; unknown diagnostic fields and secrets never cross the boundary.
 - Approved external-source resolution must directly read and attribute all required embedding-profile, Neo4j, and provider-secret keys before readiness; success with zero or partial approved-source reads is forbidden. Production-semantic-query directly inspects all three legacy Neo4j alias keys and rejects legacy-only or mixed valid-canonical-plus-any-legacy-alias sources with `SECRET_SOURCE_PROVENANCE_PROHIBITED` before attribution, selection, readiness, provider, or driver effects. Inspection cannot attribute, normalize, return, or use a legacy value. It also rejects missing secrets, unsafe `.argo/.env` path/git/file/reparse/ACL state, conflicting process/file values, and test/default, literal, fallback, alias, or indirect credential operations. The test composition accepts raw `sourceBehavior` plus filesystem/git/ACL adapters; ready-made environment or normalized configuration objects are prohibited.

package/argo/scripts/graph-rag/defaultSemanticRetrieval.js

@@ -29,6 +29,9 @@ const {
   rerankCandidates,
   applyRerankOrder,
 } = require('./rerankRetrieval.js');
+const {
+  runSemanticAlignment,
+} = require('./semanticAlignmentRunner.js');
 
 const APPROVED_SOURCE_KEYS = Object.freeze([
   'ARGO_EMBEDDING_BASE_URL',
@@ -423,25 +426,12 @@ async function attemptAutomaticAlignment({ composition, request, alignment }) {
 }
 
 function runScriptOwnedSemanticAlignment(operation) {
-  const childProcess = require('node:child_process');
-  const repositoryRoot = getWorkspaceRoot();
-  const scriptPath = resolveArgoPath('scripts', 'ensureArgoHarnessEnvironment.js');
-  const result = childProcess.spawnSync(process.execPath, [scriptPath], {
-    cwd: repositoryRoot,
-    encoding: 'utf8',
-    env: process.env,
-  });
-  if (result.status === 0) {
-    return Object.freeze({
+  return runSemanticAlignment(getWorkspaceRoot()).then(
+    () => Object.freeze({
       status: 'aligned',
       originalQuery: operation && operation.originalQuery,
-    });
-  }
-  const error = safeError('SEMANTIC_AUTO_ALIGNMENT_FAILED');
-  error.message = 'Semantic automatic alignment failed before retry.';
-  error.action = 'Repair semantic lifecycle alignment, then retry the original query.';
-  error.fullSnapshotFallback = false;
-  return Promise.reject(error);
+    }),
+  );
 }
 
 async function resolveRawTestConfiguration(sourceBehavior, sourceAdapters) {

package/argo/scripts/graph-rag/liveEmbeddingProviderConfig.js

@@ -52,6 +52,8 @@ const RETRIEVAL_TUNING_KEYS = Object.freeze([
   'ARGO_RERANK_API_KEY',
   'ARGO_RERANK_PROVIDER',
   'ARGO_RERANK_MODEL',
+  'ARGO_RERANK_DISABLE_THINKING',
+  'ARGO_RERANK_THINKING_PARAM',
 ]);
 const OPT_IN_KEYS = Object.freeze({
   ARGO_LIVE_PROVIDER_E2E: 'LIVE_PROVIDER_E2E_OPT_IN_REQUIRED',

package/argo/scripts/graph-rag/rerankRetrieval.js

@@ -60,6 +60,32 @@ function rerankConfig(env = process.env) {
   };
 }
 
+// Model-agnostic "disable thinking/reasoning" body fragment. Reasoning models
+// spend seconds on hidden reasoning tokens for a simple listwise ranking with no
+// accuracy gain, so disable it by default for EVERY rerank provider (not just
+// deepseek), so switching models keeps the same behaviour. Providers use
+// different field names, so it is fully overridable:
+//   ARGO_RERANK_DISABLE_THINKING=0      -> send nothing
+//   ARGO_RERANK_THINKING_PARAM='{...}'  -> send this exact JSON fragment
+// rerankCandidates retries WITHOUT the fragment if the provider rejects it, so an
+// unsupported field can never break rerank (recall-first: never lose rerank to a
+// 400).
+const DEFAULT_RERANK_THINKING_OFF = Object.freeze({ thinking: { type: 'disabled' } });
+
+function resolveRerankThinkingOff(env = process.env) {
+  if (String((env && env.ARGO_RERANK_DISABLE_THINKING) || '') === '0') return null;
+  const raw = env && env.ARGO_RERANK_THINKING_PARAM;
+  if (typeof raw === 'string' && raw.trim() !== '') {
+    try {
+      const parsed = JSON.parse(raw);
+      return parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? parsed : null;
+    } catch {
+      return null;
+    }
+  }
+  return DEFAULT_RERANK_THINKING_OFF;
+}
+
 function candidateText(record) {
   const raw = record && (record.searchText || record.description || record.name || '');
   return String(raw).replace(/\s+/g, ' ').slice(0, 160);
@@ -115,37 +141,46 @@ async function rerankCandidates({ query, candidates, provider, transport, maxRet
   const apiKey = typeof provider.apiKey === 'string' ? provider.apiKey : '';
   if (baseUrl === '' || apiKey === '') return null;
   const model = provider.model || DEFAULT_RERANK_MODEL;
-  const body = {
+  const thinkingOff = resolveRerankThinkingOff();
+  const baseBody = {
     model,
     temperature: 0,
     response_format: { type: 'json_object' },
-    // DeepSeek's default is a "thinking" model: for a listwise ranking it spends
-    // 5k-10k reasoning tokens per call (measured 6-12s, highly variable) with no
-    // accuracy gain. Disable thinking for a fast, deterministic rerank
-    // (deepseek-flash: ~6s -> ~0.7s, reasoning tokens -> 0).
-    ...(provider.provider === 'deepseek' ? { thinking: { type: 'disabled' } } : {}),
     messages: [
       { role: 'system', content: 'You rank architecture elements by relevance to a query. Return ONLY JSON {"order":[ids best-first]} using only the candidate ids.' },
       { role: 'user', content: `Query: ${query}\n\nCandidates (id\\ttext):\n${list.map(candidate => `${candidate.id}\t${candidateText(candidate)}`).join('\n')}\n\nReturn up to ${Math.min(maxReturn || DEFAULT_RERANK_RETURN, list.length)} ids best-first.` },
     ],
   };
-  const controller = typeof AbortController === 'function' ? new AbortController() : null;
-  const timer = controller ? setTimeout(() => controller.abort(), timeoutMs || rerankTimeoutMs()) : null;
+  const timeout = timeoutMs || rerankTimeoutMs();
+  const send = async (extra) => {
+    const controller = typeof AbortController === 'function' ? new AbortController() : null;
+    const timer = controller ? setTimeout(() => controller.abort(), timeout) : null;
+    try {
+      return await transport.request(`${baseUrl}/chat/completions`, {
+        method: 'POST',
+        headers: { Authorization: `Bearer ${apiKey}`, 'Content-Type': 'application/json' },
+        body: JSON.stringify({ ...baseBody, ...extra }),
+        ...(controller ? { signal: controller.signal } : {}),
+      });
+    } catch {
+      return null;
+    } finally {
+      if (timer) clearTimeout(timer);
+    }
+  };
   try {
-    const response = await transport.request(`${baseUrl}/chat/completions`, {
-      method: 'POST',
-      headers: { Authorization: `Bearer ${apiKey}`, 'Content-Type': 'application/json' },
-      body: JSON.stringify(body),
-      ...(controller ? { signal: controller.signal } : {}),
-    });
+    let response = await send(thinkingOff || {});
+    // A provider may reject the thinking-off fragment (unknown field -> 400).
+    // Retry WITHOUT it so rerank still works: never lose rerank/recall to a 400.
+    if ((!response || response.ok !== true) && thinkingOff) {
+      response = await send({});
+    }
     if (!response || response.ok !== true || typeof response.json !== 'function') return null;
     const payload = await response.json();
     const content = payload && payload.choices && payload.choices[0] && payload.choices[0].message && payload.choices[0].message.content;
     return parseRerankOrder(content, list.map(candidate => candidate.id));
   } catch {
     return null;
-  } finally {
-    if (timer) clearTimeout(timer);
   }
 }
 
@@ -158,6 +193,8 @@ module.exports = {
   rerankConfig,
   rerankTimeoutMs,
   resolveRerankProvider,
+  resolveRerankThinkingOff,
+  DEFAULT_RERANK_THINKING_OFF,
   parseRerankOrder,
   applyRerankOrder,
   rerankCandidates,

package/argo/scripts/graph-rag/semantic-persistence/productionSemanticBackfill.js

@@ -90,7 +90,7 @@ async function processChannel(options) {
   const stored = await options.checkpointStore.readCheckpoint(options.channel);
   const checkpoint = stored && stored.canonicalVersion === options.canonicalVersion
     ? mutableCheckpoint(stored, canonicalRecords.length)
-    : emptyCheckpoint(options.channel, options.canonicalVersion, canonicalRecords.length);
+    : await resumeCheckpointByContent(options, canonicalRecords);
   const attempts = new Map();
 
   while (checkpoint.completedCanonicalIdentities.length < canonicalRecords.length) {
@@ -153,6 +153,61 @@ async function processChannel(options) {
   });
 }
 
+// Cross-version incremental resume. The resume checkpoint is keyed by
+// canonicalVersion, but ANY add/remove of an element/relationship/view changes
+// that version. Re-embedding every record on every such change is wasteful and
+// slow (~hundreds of provider calls: measured ~174s). Instead, when the stored
+// checkpoint is for a different version, seed the completed set from the
+// persisted records whose curated semantic text (contentHash) AND embedding
+// provider/model/dimensions already match the current graph; only changed/new
+// records get embedded. Recall is unchanged: changed content is ALWAYS
+// re-embedded, only byte-identical text is skipped.
+async function resumeCheckpointByContent(options, canonicalRecords) {
+  const checkpoint = emptyCheckpoint(options.channel, options.canonicalVersion, canonicalRecords.length);
+  let persisted = [];
+  try {
+    const stored = await options.projectionStore.readRecords();
+    persisted = Array.isArray(stored) ? stored : [];
+  } catch {
+    persisted = [];
+  }
+  const byIdentity = new Map();
+  for (const record of persisted) {
+    if (record && record.channel === options.channel && isNonBlankString(record.canonicalIdentity)) {
+      byIdentity.set(record.canonicalIdentity, record);
+    }
+  }
+  const completed = [];
+  for (const record of canonicalRecords) {
+    const existing = byIdentity.get(record.canonicalIdentity);
+    if (existing && persistedRecordMatches(existing, record, options.qualification)) {
+      completed.push(record.canonicalIdentity);
+    }
+  }
+  checkpoint.completedCanonicalIdentities = completed;
+  checkpoint.completedCount = completed.length;
+  checkpoint.cursor = completed.length;
+  checkpoint.status = completed.length === canonicalRecords.length ? 'complete' : 'pending';
+  return checkpoint;
+}
+
+function persistedRecordMatches(existing, canonicalRecord, qualification) {
+  if (!Array.isArray(existing.vector)) {
+    return false;
+  }
+  const searchText = buildSemanticRecordText(canonicalRecord.channel, canonicalRecord.canonicalObject);
+  const contentHash = crypto.createHash('sha256').update(searchText).digest('hex');
+  return existing.contentVersion === `content:${contentHash}`
+    && existing.provider === qualification.provider
+    && existing.model === qualification.model
+    && existing.modelVersion === qualification.version
+    && toNumber(existing.dimensions) === toNumber(qualification.dimensions);
+}
+
+function toNumber(value) {
+  return value && typeof value.toNumber === 'function' ? value.toNumber() : Number(value);
+}
+
 function buildSemanticRecord(record, vector, options) {
   const searchText = buildSemanticRecordText(record.channel, record.canonicalObject);
   const contentHash = crypto.createHash('sha256')

package/argo/scripts/graph-rag/semanticAlignmentRunner.js

@@ -0,0 +1,95 @@
+'use strict';
+
+// Owns the (expensive) semantic-index alignment child process. When readiness is
+// NOT aligned, the index must be rebuilt to serve correct (full-recall) results;
+// that rebuild can take seconds to ~90s. This module:
+//   - runs it ASYNCHRONOUSLY (child_process.spawn, not spawnSync) so the MCP
+//     event loop is never frozen while it runs;
+//   - de-duplicates concurrent requests (one rebuild shared by all waiters);
+//   - logs start/finish/failure for observability;
+//   - exposes preheatSemanticAlignment() for a background warm-up at startup.
+// It never changes WHAT is retrieved (recall/precision), only how the rebuild is
+// scheduled/observed.
+
+const fs = require('node:fs');
+const path = require('node:path');
+const { spawn } = require('node:child_process');
+const { getWorkspaceRoot, resolveArgoPath } = require('../argo-paths.js');
+
+const READINESS_RELATIVE_PATH = path.join('.argo', 'temp', 'system-architecture-semantic-readiness.json');
+
+let inFlight = null;
+let preheated = false;
+
+function readinessRecordPath(repositoryRoot) {
+  return path.join(repositoryRoot, ...READINESS_RELATIVE_PATH.split(path.sep));
+}
+
+function isSemanticReady(repositoryRoot) {
+  try {
+    const parsed = JSON.parse(fs.readFileSync(readinessRecordPath(repositoryRoot), 'utf8'));
+    return !!parsed && parsed.state === 'Aligned' && parsed.verified === true;
+  } catch {
+    return false;
+  }
+}
+
+function alignmentError() {
+  const error = new Error('SEMANTIC_AUTO_ALIGNMENT_FAILED');
+  error.category = 'SEMANTIC_AUTO_ALIGNMENT_FAILED';
+  error.message = 'Semantic automatic alignment failed before retry.';
+  error.action = 'Repair semantic lifecycle alignment, then retry the original query.';
+  error.fullSnapshotFallback = false;
+  return error;
+}
+
+// Run the alignment (once). Returns a promise resolving to { status: 'aligned' }
+// or rejecting with the SEMANTIC_AUTO_ALIGNMENT_FAILED envelope.
+function runSemanticAlignment(repositoryRoot = getWorkspaceRoot()) {
+  if (inFlight) {
+    return inFlight;
+  }
+  const scriptPath = resolveArgoPath('scripts', 'ensureArgoHarnessEnvironment.js');
+  const startedAt = Date.now();
+  console.error('[argo] semantic auto-alignment: index not aligned, rebuilding (recall-safe)…');
+  inFlight = new Promise((resolve, reject) => {
+    const child = spawn(process.execPath, [scriptPath], {
+      cwd: repositoryRoot,
+      env: process.env,
+      stdio: ['ignore', 'ignore', 'inherit'],
+    });
+    const finish = (failed, cause) => {
+      inFlight = null;
+      const ms = Date.now() - startedAt;
+      if (failed) {
+        console.error(`[argo] semantic auto-alignment FAILED in ${ms}ms${cause ? ': ' + cause : ''}`);
+        reject(alignmentError());
+      } else {
+        console.error(`[argo] semantic auto-alignment done in ${ms}ms`);
+        resolve(Object.freeze({ status: 'aligned' }));
+      }
+    };
+    child.on('error', (error) => finish(true, error && error.message));
+    child.on('exit', (code, signal) => finish(code !== 0, `code=${code} signal=${signal || '-'}`));
+  });
+  return inFlight;
+}
+
+// Fire-and-forget warm-up: if readiness is already Aligned it does nothing;
+// otherwise it starts the rebuild in the background so the first query rarely
+// pays for it. Safe to call repeatedly (guarded + de-duplicated).
+function preheatSemanticAlignment(repositoryRoot = getWorkspaceRoot()) {
+  if (preheated || !repositoryRoot || isSemanticReady(repositoryRoot)) {
+    return;
+  }
+  preheated = true;
+  console.error('[argo] readiness not aligned at startup; preheating semantic alignment in background…');
+  runSemanticAlignment(repositoryRoot).catch(() => {});
+}
+
+module.exports = {
+  runSemanticAlignment,
+  preheatSemanticAlignment,
+  isSemanticReady,
+  readinessRecordPath,
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "archgraph-argo",
-  "version": "0.20.3",
+  "version": "0.20.4",
   "description": "Deploy the ArchGraph ARGO toolchain, skills, and rules (schema, scripts, argo-init skill, global rule) with one command.",
   "license": "MIT",
   "bin": {