archgraph-argo 0.24.0 → 0.24.2

package/README.md

@@ -57,12 +57,15 @@ Everything works out of the box except **semantic (Graph RAG) queries**, which n
 - **Embedding / vector engine** — powers semantic Graph RAG retrieval. Configure
   `ARGO_EMBEDDING_BASE_URL`, `ARGO_EMBEDDING_MODEL`, `ARGO_EMBEDDING_PROVIDER`,
   `ARGO_EMBEDDING_MODEL_VERSION`, `ARGO_EMBEDDING_DIMENSIONS`, plus the API key `QWEN_KEY`.
+  It points at **any OpenAI-compatible embedding endpoint** — a cloud provider, or a self-hosted
+  server for offline / intranet / private deployments via `ARGO_EMBEDDING_PROFILE=openai-compatible`
+  (see the [self-hosted embedding guide](docs/self-hosted-embedding-deployment.md)).
 
 Where do the values come from? The Neo4j credentials come from the Neo4j instance you own or
 provision (URI, username, password). The embedding configuration and `QWEN_KEY` come from your
-embedding provider's dashboard — for example Alibaba DashScope. `argo-deploy` walks you through the
-prompt (existing non-empty values in `~/.argo/.env` are kept); you can also edit the file afterwards
-and re-run.
+embedding provider's dashboard — for example Alibaba DashScope — or from a self-hosted
+OpenAI-compatible server. `argo-deploy` walks you through the prompt (existing non-empty values in
+`~/.argo/.env` are kept); you can also edit the file afterwards and re-run.
 
 ## How to use
 

package/argo/scripts/graph-rag/defaultSemanticRetrieval.js

@@ -85,32 +85,35 @@ const CHANNELS = Object.freeze([
     indexName: 'argo_production_semantic_view_vector',
   }),
 ]);
+// Project an explicit metadata allowlist instead of properties(node): Neo4j map
+// projection `.*` lets an existing property win, so properties(node) / node{.*, vector:null}
+// would still ship the 1536-dim vector; the read path only consumes canonicalIdentity, channel, searchText and score.
 const VECTOR_QUERY_CYPHER = [
   'CALL db.index.vector.queryNodes($indexName, $topK, $vector)',
   'YIELD node, score',
   'WHERE node.channel = $channel',
-  'RETURN properties(node) AS record, score',
+  'RETURN {canonicalIdentity: node.canonicalIdentity, channel: node.channel, searchText: node.searchText, dimensions: node.dimensions, provider: node.provider, model: node.model, modelVersion: node.modelVersion, canonicalVersion: node.canonicalVersion, contentVersion: node.contentVersion, indexVersion: node.indexVersion} AS record, score',
   'ORDER BY score DESC',
 ].join('\n');
 const VECTOR_QUERY_CYPHER_SCOPED = [
   'CALL db.index.vector.queryNodes($indexName, $topK, $vector)',
   'YIELD node, score',
   'WHERE node.channel = $channel AND node.canonicalIdentity IN $canonicalIdentities',
-  'RETURN properties(node) AS record, score',
+  'RETURN {canonicalIdentity: node.canonicalIdentity, channel: node.channel, searchText: node.searchText, dimensions: node.dimensions, provider: node.provider, model: node.model, modelVersion: node.modelVersion, canonicalVersion: node.canonicalVersion, contentVersion: node.contentVersion, indexVersion: node.indexVersion} AS record, score',
   'ORDER BY score DESC',
 ].join('\n');
 const LEXICAL_QUERY_CYPHER = [
   'CALL db.index.fulltext.queryNodes($indexName, $queryText, { limit: $topK })',
   'YIELD node, score',
   'WHERE node.channel = $channel',
-  'RETURN properties(node) AS record, score',
+  'RETURN {canonicalIdentity: node.canonicalIdentity, channel: node.channel, searchText: node.searchText, dimensions: node.dimensions, provider: node.provider, model: node.model, modelVersion: node.modelVersion, canonicalVersion: node.canonicalVersion, contentVersion: node.contentVersion, indexVersion: node.indexVersion} AS record, score',
   'ORDER BY score DESC',
 ].join('\n');
 const LEXICAL_QUERY_CYPHER_SCOPED = [
   'CALL db.index.fulltext.queryNodes($indexName, $queryText, { limit: $topK })',
   'YIELD node, score',
   'WHERE node.channel = $channel AND node.canonicalIdentity IN $canonicalIdentities',
-  'RETURN properties(node) AS record, score',
+  'RETURN {canonicalIdentity: node.canonicalIdentity, channel: node.channel, searchText: node.searchText, dimensions: node.dimensions, provider: node.provider, model: node.model, modelVersion: node.modelVersion, canonicalVersion: node.canonicalVersion, contentVersion: node.contentVersion, indexVersion: node.indexVersion} AS record, score',
   'ORDER BY score DESC',
 ].join('\n');
 const READINESS_QUERY_CYPHER = [
@@ -149,6 +152,12 @@ function resolveTopK() {
   const k = envNumber('ARGO_SEMANTIC_TOP_K', DEFAULT_TOP_K);
   return Number.isInteger(k) && k > 0 ? k : DEFAULT_TOP_K;
 }
+// The single production seed gate: a retrieved record becomes a seed only when
+// its TRUE vector similarity reaches the purpose-aware threshold. Exposed so
+// acceptance tests exercise the exact predicate the retrieval runtime uses.
+function isSeedAboveThreshold(score, threshold) {
+  return Number.isFinite(score) && Number.isFinite(threshold) && score >= threshold;
+}
 const testCompositionStorage = new AsyncLocalStorage();
 
 function createDefaultSemanticRetrieval(dependencies = {}) {
@@ -828,7 +837,7 @@ async function exhaustChannel({
     for (const raw of newlyVisible) {
       if (accepted.length >= effectiveMax) break;
       const record = normalizeVectorRecord(raw, channel);
-      if (record && record.score >= effectiveThreshold && !seen.has(record.id)) {
+      if (record && isSeedAboveThreshold(record.score, effectiveThreshold) && !seen.has(record.id)) {
         seen.add(record.id);
         accepted.push(Object.freeze(record));
       }
@@ -1238,8 +1247,11 @@ module.exports = {
   memoryThresholdFor,
   auditThresholdFor,
   resolveTopK,
+  isSeedAboveThreshold,
   scopeCanonicalIdentitiesForChannel,
   bareCanonicalId,
   normalizeVectorRecord,
+  exhaustChannel,
+  exhaustLexicalChannel,
   AUDIT_PURPOSES,
 };

package/argo/scripts/graph-rag/embeddingProviderProfile.js

@@ -1,89 +1,89 @@
-'use strict';
-
-// Embedding provider profile (ArchGraph ARGO).
-//
-// The semantic-retrieval embedding backend is either the human-approved cloud
-// profile (`approved`, the default, byte-for-byte unchanged) or a self-hosted
-// OpenAI-compatible endpoint (`openai-compatible`) for intranet / offline
-// deployments. This module owns the profile vocabulary, the dimension contract,
-// and the query-side instruction composition so the live configuration resolver
-// and the retrieval runtime share one implementation.
-
-const PROFILE_KEY = 'ARGO_EMBEDDING_PROFILE';
-const QUERY_INSTRUCTION_KEY = 'ARGO_EMBEDDING_QUERY_INSTRUCTION';
-const API_KEY = 'ARGO_EMBEDDING_API_KEY';
-const PROFILE_APPROVED = 'approved';
-const PROFILE_OPENAI_COMPATIBLE = 'openai-compatible';
-const SUPPORTED_EMBEDDING_PROFILES = Object.freeze([PROFILE_APPROVED, PROFILE_OPENAI_COMPATIBLE]);
-const DEFAULT_EMBEDDING_DIMENSIONS = 1536;
-
-function resolveEmbeddingProfile(value) {
-  const normalized = typeof value === 'string' ? value.trim().toLowerCase() : '';
-  return normalized === '' ? PROFILE_APPROVED : normalized;
-}
-
-function isApprovedEmbeddingProfile(value) {
-  return resolveEmbeddingProfile(value) === PROFILE_APPROVED;
-}
-
-function requireSupportedEmbeddingProfile(value) {
-  const profile = resolveEmbeddingProfile(value);
-  if (!SUPPORTED_EMBEDDING_PROFILES.includes(profile)) {
-    throw profileError('EMBEDDING_PROFILE_UNSUPPORTED');
-  }
-  return profile;
-}
-
-function resolveEmbeddingDimensions(value, fallback = DEFAULT_EMBEDDING_DIMENSIONS) {
-  const parsed = typeof value === 'string' && value.trim() !== '' ? Number(value) : NaN;
-  return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
-}
-
-function requireEmbeddingDimensions(value) {
-  const parsed = typeof value === 'string' && value.trim() !== '' ? Number(value) : NaN;
-  if (!Number.isInteger(parsed) || parsed <= 0) {
-    throw profileError('EMBEDDING_DIMENSIONS_INVALID');
-  }
-  return parsed;
-}
-
-// The query-side input may carry an instruction prefix (e.g. gte-Qwen2's
-// "Instruct: <task>\nQuery: <query>"); document-side input stays raw so stored
-// vectors are never polluted by a query-only prefix. The prefix commonly comes
-// from a single-line `.env` value, so `\n`/`\r`/`\t` escapes are decoded here.
-function normalizeInstruction(instruction) {
-  if (typeof instruction !== 'string') return '';
-  return instruction
-    .replace(/\\r\\n/g, '\n')
-    .replace(/\\n/g, '\n')
-    .replace(/\\r/g, '\r')
-    .replace(/\\t/g, '\t');
-}
-
-function composeQueryEmbeddingInput(text, instruction) {
-  const query = text == null ? '' : String(text);
-  const prefix = normalizeInstruction(instruction);
-  return prefix === '' ? query : `${prefix}${query}`;
-}
-
-function profileError(category) {
-  const error = new Error(category);
-  error.category = category;
-  return error;
-}
-
-module.exports = {
-  PROFILE_KEY,
-  QUERY_INSTRUCTION_KEY,
-  API_KEY,
-  PROFILE_APPROVED,
-  PROFILE_OPENAI_COMPATIBLE,
-  SUPPORTED_EMBEDDING_PROFILES,
-  DEFAULT_EMBEDDING_DIMENSIONS,
-  resolveEmbeddingProfile,
-  isApprovedEmbeddingProfile,
-  requireSupportedEmbeddingProfile,
-  resolveEmbeddingDimensions,
-  requireEmbeddingDimensions,
-  composeQueryEmbeddingInput,
-};
+'use strict';
+
+// Embedding provider profile (ArchGraph ARGO).
+//
+// The semantic-retrieval embedding backend is either the human-approved cloud
+// profile (`approved`, the default, byte-for-byte unchanged) or a self-hosted
+// OpenAI-compatible endpoint (`openai-compatible`) for intranet / offline
+// deployments. This module owns the profile vocabulary, the dimension contract,
+// and the query-side instruction composition so the live configuration resolver
+// and the retrieval runtime share one implementation.
+
+const PROFILE_KEY = 'ARGO_EMBEDDING_PROFILE';
+const QUERY_INSTRUCTION_KEY = 'ARGO_EMBEDDING_QUERY_INSTRUCTION';
+const API_KEY = 'ARGO_EMBEDDING_API_KEY';
+const PROFILE_APPROVED = 'approved';
+const PROFILE_OPENAI_COMPATIBLE = 'openai-compatible';
+const SUPPORTED_EMBEDDING_PROFILES = Object.freeze([PROFILE_APPROVED, PROFILE_OPENAI_COMPATIBLE]);
+const DEFAULT_EMBEDDING_DIMENSIONS = 1536;
+
+function resolveEmbeddingProfile(value) {
+  const normalized = typeof value === 'string' ? value.trim().toLowerCase() : '';
+  return normalized === '' ? PROFILE_APPROVED : normalized;
+}
+
+function isApprovedEmbeddingProfile(value) {
+  return resolveEmbeddingProfile(value) === PROFILE_APPROVED;
+}
+
+function requireSupportedEmbeddingProfile(value) {
+  const profile = resolveEmbeddingProfile(value);
+  if (!SUPPORTED_EMBEDDING_PROFILES.includes(profile)) {
+    throw profileError('EMBEDDING_PROFILE_UNSUPPORTED');
+  }
+  return profile;
+}
+
+function resolveEmbeddingDimensions(value, fallback = DEFAULT_EMBEDDING_DIMENSIONS) {
+  const parsed = typeof value === 'string' && value.trim() !== '' ? Number(value) : NaN;
+  return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
+}
+
+function requireEmbeddingDimensions(value) {
+  const parsed = typeof value === 'string' && value.trim() !== '' ? Number(value) : NaN;
+  if (!Number.isInteger(parsed) || parsed <= 0) {
+    throw profileError('EMBEDDING_DIMENSIONS_INVALID');
+  }
+  return parsed;
+}
+
+// The query-side input may carry an instruction prefix (e.g. gte-Qwen2's
+// "Instruct: <task>\nQuery: <query>"); document-side input stays raw so stored
+// vectors are never polluted by a query-only prefix. The prefix commonly comes
+// from a single-line `.env` value, so `\n`/`\r`/`\t` escapes are decoded here.
+function normalizeInstruction(instruction) {
+  if (typeof instruction !== 'string') return '';
+  return instruction
+    .replace(/\\r\\n/g, '\n')
+    .replace(/\\n/g, '\n')
+    .replace(/\\r/g, '\r')
+    .replace(/\\t/g, '\t');
+}
+
+function composeQueryEmbeddingInput(text, instruction) {
+  const query = text == null ? '' : String(text);
+  const prefix = normalizeInstruction(instruction);
+  return prefix === '' ? query : `${prefix}${query}`;
+}
+
+function profileError(category) {
+  const error = new Error(category);
+  error.category = category;
+  return error;
+}
+
+module.exports = {
+  PROFILE_KEY,
+  QUERY_INSTRUCTION_KEY,
+  API_KEY,
+  PROFILE_APPROVED,
+  PROFILE_OPENAI_COMPATIBLE,
+  SUPPORTED_EMBEDDING_PROFILES,
+  DEFAULT_EMBEDDING_DIMENSIONS,
+  resolveEmbeddingProfile,
+  isApprovedEmbeddingProfile,
+  requireSupportedEmbeddingProfile,
+  resolveEmbeddingDimensions,
+  requireEmbeddingDimensions,
+  composeQueryEmbeddingInput,
+};

package/argo/scripts/graph-rag/liveEmbeddingProviderConfig.js

@@ -77,6 +77,27 @@ const READABLE_KEYS = Object.freeze([
   ...Object.keys(OPT_IN_KEYS),
 ]);
 const LEGACY_KEYS = Object.freeze(['ARGO_NEO4J_URI', 'ARGO_NEO4J_USERNAME', 'ARGO_NEO4J_PASSWORD']);
+// Host / process-level keys: read from the environment at launch, but NOT
+// accepted inside the .env file (see argo/.env.example Part 2). Kept explicit so
+// the coverage test (tests/env-key-coverage.test.js) can prove every env key
+// referenced by argo/scripts is classified — either a .env key, a host-only key,
+// or a legacy alias. This is what prevents "a key the code already uses is still
+// rejected as unknown".
+const HOST_ONLY_ENV_KEYS = Object.freeze([
+  'ARGO_ENV_FILE',
+  'ARGO_REPO_ROOT',
+  'ARGO_EA_QEA',
+  'ARGO_WORKSPACE_ROOTS',
+  'ARGO_SERVER_PATH',
+  'GRAPH_MCP_URL',
+  'EA_QEA_DEBUG',
+  'WORKSPACE_FOLDER',
+  'ARGO_TEST_TIMEOUT_MS',
+  'ARGO_MCP_MUTATION_RESPONSE_DEBUG',
+  'ARGO_MCP_SEMANTIC_DEDUP',
+  'ARGO_MCP_SEMANTIC_DEDUP_THRESHOLD',
+  'ARGO_SEMANTIC_DEDUP_THRESHOLD',
+]);
 const PROHIBITED_RUNTIME_FIELD_KEYS = Object.freeze(['neo4jUri', 'embeddingCredential']);
 const SECRET_KEYS = new Set(['ARGO_NEO4J_DATABASE_PASSWORD', 'QWEN_KEY', 'ARGO_RERANK_API_KEY', EMBEDDING_API_KEY_KEY]);
 const APPROVED = Object.freeze({
@@ -600,4 +621,6 @@ module.exports = {
   // preflighted as secrets.
   ENV_FILE_KEYS: READABLE_KEYS,
   ENV_FILE_SECRET_KEYS: Object.freeze(Array.from(SECRET_KEYS)),
+  HOST_ONLY_ENV_KEYS,
+  ENV_FILE_LEGACY_KEYS: LEGACY_KEYS,
 };

package/argo/scripts/graph-rag/semanticInitConfiguration.js

@@ -0,0 +1,24 @@
+'use strict';
+
+// Maps profile-aware configuration evidence (from resolveApprovedLiveConfiguration)
+// into the shape the canonical semantic-init / backfill runtime consumes.
+//
+// This is the single place that turns "which embedding profile" into the init
+// backfill's configuration, so argo init embeds through the SAME provider the
+// retrieval path uses (approved cloud by default, or a self-hosted endpoint via
+// ARGO_EMBEDDING_PROFILE=openai-compatible) — instead of a hardcoded profile.
+
+function buildDefaultSemanticConfiguration(evidence) {
+  const configuration = evidence && typeof evidence === 'object' && evidence.configuration
+    ? evidence.configuration
+    : evidence;
+  if (!configuration || typeof configuration !== 'object') {
+    throw new TypeError('resolved configuration evidence is required');
+  }
+  return Object.freeze({
+    ...configuration,
+    embeddingCredential: configuration.embeddingApiKey || configuration.qwenKey,
+  });
+}
+
+module.exports = { buildDefaultSemanticConfiguration };

package/argo/scripts/systemarchitecture-mcp-server.js

@@ -144,12 +144,12 @@ const {
 const {
   semanticOperatorErrorResult,
 } = require('./graph-rag/semanticOperatorError.js');
-const {
-  resolveExternalProductionConfig,
-} = require('./graph-rag/externalProductionConfig.js');
 const {
   resolveApprovedLiveConfiguration,
 } = require('./graph-rag/liveEmbeddingProviderConfig.js');
+const {
+  buildDefaultSemanticConfiguration,
+} = require('./graph-rag/semanticInitConfiguration.js');
 const {
   createLiveEmbeddingProviderClient,
 } = require('./graph-rag/liveEmbeddingProviderClient.js');
@@ -4056,59 +4056,17 @@ async function createDefaultProductionSemanticRuntime(options = {}) {
   });
 }
 
+// Resolve the canonical semantic-init/backfill configuration through the SAME
+// profile-aware resolver the retrieval path uses, so argo init embeds via the
+// configured provider (ARGO_EMBEDDING_PROFILE) instead of a hardcoded cloud
+// profile. `resolveApprovedLiveConfiguration` still fails closed when a required
+// key is missing or an unknown key is present in the .env file.
 async function resolveDefaultSemanticConfiguration(repositoryRoot) {
-  let external;
-  try {
-    external = resolveExternalProductionConfig({
-      neo4jUri: process.env.ARGO_NEO4J_DATABASE_URL,
-      neo4jUsername: process.env.ARGO_NEO4J_DATABASE_USERNAME,
-      neo4jPassword: process.env.ARGO_NEO4J_DATABASE_PASSWORD,
-      embeddingCredential: process.env.QWEN_KEY,
-      neo4jDatabase: process.env.ARGO_NEO4J_DATABASE || getDefaultSemanticNeo4jDatabaseName(repositoryRoot),
-    }, {
-      operation: 'semantic-backfill',
-      sourceKeys: new Map([
-        ['neo4jUri', 'ARGO_NEO4J_DATABASE_URL'],
-        ['neo4jUsername', 'ARGO_NEO4J_DATABASE_USERNAME'],
-        ['neo4jPassword', 'ARGO_NEO4J_DATABASE_PASSWORD'],
-        ['embeddingCredential', 'QWEN_KEY'],
-      ]),
-    });
-  } catch (error) {
-    if (error && error.category === 'EXTERNAL_CREDENTIALS_REQUIRED') {
-      const missing = new Error('EXTERNAL_CREDENTIALS_REQUIRED');
-      missing.category = 'EXTERNAL_CREDENTIALS_REQUIRED';
-      missing.field = error.field;
-      throw missing;
-    }
-    throw error;
-  }
-  return Object.freeze({
-    embeddingBaseUrl: W31_APPROVED_PROFILE.baseUrl,
-    embeddingModel: W31_APPROVED_PROFILE.model,
-    embeddingProvider: W31_APPROVED_PROFILE.provider,
-    embeddingModelVersion: W31_APPROVED_PROFILE.version,
-    embeddingDimensions: W31_APPROVED_PROFILE.dimensions,
-    neo4jDatabaseUrl: external.neo4jUri,
-    neo4jDatabaseUsername: external.neo4jUsername,
-    neo4jDatabasePassword: external.neo4jPassword,
-    qwenKey: external.embeddingCredential,
-    embeddingCredential: external.embeddingCredential,
-    ...(external.neo4jDatabase === undefined ? {} : { neo4jDatabase: external.neo4jDatabase }),
+  const evidence = await resolveApprovedLiveConfiguration({
+    repositoryRoot,
+    requiredOptIns: [LIVE_PROVIDER_OPT_IN, W31_LIVE_OPT_IN],
   });
-}
-
-function getDefaultSemanticNeo4jDatabaseName(repositoryRoot) {
-  const repoName = path.basename(repositoryRoot || resolveWorkspaceRoot());
-  const normalized = String(repoName)
-    .toLowerCase()
-    .replace(/[^a-z0-9.-]+/g, '-')
-    .replace(/^-+|-+$/g, '')
-    .replace(/\.{2,}/g, '.')
-    .replace(/-{2,}/g, '-');
-  const safe = normalized || 'workspace';
-  const prefixed = /^[a-z]/.test(safe) ? safe : `db-${safe}`;
-  return prefixed.slice(0, 63);
+  return buildDefaultSemanticConfiguration(evidence);
 }
 
 function deriveSemanticCanonicalVersion(document) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "archgraph-argo",
-  "version": "0.24.0",
+  "version": "0.24.2",
   "description": "Deploy the ArchGraph ARGO toolchain, skills, and rules (schema, scripts, argo-init skill, global rule) with one command.",
   "license": "MIT",
   "bin": {