archgraph-argo 0.23.0 → 0.24.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -3
- package/argo/.env.example +14 -1
- package/argo/scripts/graph-rag/ARCHITECTURE.md +1 -1
- package/argo/scripts/graph-rag/defaultSemanticRetrieval.js +38 -19
- package/argo/scripts/graph-rag/embeddingProviderProfile.js +89 -0
- package/argo/scripts/graph-rag/liveEmbeddingIndexGate.js +1 -2
- package/argo/scripts/graph-rag/liveEmbeddingProviderClient.js +1 -1
- package/argo/scripts/graph-rag/liveEmbeddingProviderConfig.js +92 -9
- package/argo/scripts/graph-rag/semantic-persistence/productionSemanticNeo4jAdapter.js +12 -6
- package/argo/scripts/graph-rag/semanticInitConfiguration.js +24 -0
- package/argo/scripts/systemarchitecture-mcp-server.js +12 -54
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -57,12 +57,15 @@ Everything works out of the box except **semantic (Graph RAG) queries**, which n
|
|
|
57
57
|
- **Embedding / vector engine** — powers semantic Graph RAG retrieval. Configure
|
|
58
58
|
`ARGO_EMBEDDING_BASE_URL`, `ARGO_EMBEDDING_MODEL`, `ARGO_EMBEDDING_PROVIDER`,
|
|
59
59
|
`ARGO_EMBEDDING_MODEL_VERSION`, `ARGO_EMBEDDING_DIMENSIONS`, plus the API key `QWEN_KEY`.
|
|
60
|
+
It points at **any OpenAI-compatible embedding endpoint** — a cloud provider, or a self-hosted
|
|
61
|
+
server for offline / intranet / private deployments via `ARGO_EMBEDDING_PROFILE=openai-compatible`
|
|
62
|
+
(see the [self-hosted embedding guide](docs/self-hosted-embedding-deployment.md)).
|
|
60
63
|
|
|
61
64
|
Where do the values come from? The Neo4j credentials come from the Neo4j instance you own or
|
|
62
65
|
provision (URI, username, password). The embedding configuration and `QWEN_KEY` come from your
|
|
63
|
-
embedding provider's dashboard — for example Alibaba DashScope
|
|
64
|
-
|
|
65
|
-
and re-run.
|
|
66
|
+
embedding provider's dashboard — for example Alibaba DashScope — or from a self-hosted
|
|
67
|
+
OpenAI-compatible server. `argo-deploy` walks you through the prompt (existing non-empty values in
|
|
68
|
+
`~/.argo/.env` are kept); you can also edit the file afterwards and re-run.
|
|
66
69
|
|
|
67
70
|
## How to use
|
|
68
71
|
|
package/argo/.env.example
CHANGED
|
@@ -19,6 +19,10 @@
|
|
|
19
19
|
# -----------------------------------------------------------------------------
|
|
20
20
|
|
|
21
21
|
# --- Embedding provider (required) — powers vector semantic retrieval -------
|
|
22
|
+
# Provider profile: "approved" (default) = the human-approved cloud profile
|
|
23
|
+
# below; "openai-compatible" = a self-hosted OpenAI-compatible endpoint
|
|
24
|
+
# (intranet/offline) whose URL/model/label/dimension are read verbatim.
|
|
25
|
+
ARGO_EMBEDDING_PROFILE=
|
|
22
26
|
# OpenAI-compatible embedding endpoint base URL (no trailing slash).
|
|
23
27
|
ARGO_EMBEDDING_BASE_URL=
|
|
24
28
|
# Embedding model id (e.g. qwen3.7-text-embedding).
|
|
@@ -27,8 +31,17 @@ ARGO_EMBEDDING_MODEL=
|
|
|
27
31
|
ARGO_EMBEDDING_PROVIDER=
|
|
28
32
|
# Model version / qualification label (recorded evidence only).
|
|
29
33
|
ARGO_EMBEDDING_MODEL_VERSION=
|
|
30
|
-
# Embedding vector dimension; must match the model
|
|
34
|
+
# Embedding vector dimension; must match the model and the vector index
|
|
35
|
+
# (current profiles: 1536).
|
|
31
36
|
ARGO_EMBEDDING_DIMENSIONS=
|
|
37
|
+
# Query-side instruction prefix for instruction-tuned embedding models (e.g.
|
|
38
|
+
# gte-Qwen2: "Instruct: <task>\nQuery: "). Empty = no prefix. Documents are
|
|
39
|
+
# never prefixed; only the query side is.
|
|
40
|
+
ARGO_EMBEDDING_QUERY_INSTRUCTION=
|
|
41
|
+
# Optional embedding API key. SECRET: overrides QWEN_KEY as the Bearer token for
|
|
42
|
+
# the embeddings call when set (useful for a self-hosted endpoint). Leave empty
|
|
43
|
+
# to use QWEN_KEY.
|
|
44
|
+
ARGO_EMBEDDING_API_KEY=
|
|
32
45
|
|
|
33
46
|
# --- Neo4j (required) — structural projection + vector/full-text store ------
|
|
34
47
|
# Neo4j connection URI (e.g. neo4j://127.0.0.1:7687).
|
|
@@ -137,7 +137,7 @@ Successful query evidence identifies `nodejs` as runtime, `neo4j-native` as retr
|
|
|
137
137
|
- Frozen accepted source fixtures supply all five `ARGO_EMBEDDING_*` keys, `ARGO_NEO4J_DATABASE_URL`, `ARGO_NEO4J_DATABASE_USERNAME`, and both secret keys from process-only, file-only, or matching dual sources. Every rejected fixture starts from that complete profile and mutates only its named rejection dimension; no implicit default may satisfy normalization.
|
|
138
138
|
- Loader provenance rejects root/alternate/tracked files, CLI, literal/default/fallback, alias, destructured, generated, or indirect secret sources.
|
|
139
139
|
- The resolver validates every trace before consuming its value using its private composition capability: trace/alias-chain immutability, exact field set with no extras, field types, issued-object identity, requested-key equality, exact requested file path, source-kind/path/operation correlation, and a one-element alias chain equal to the requested key for approved direct reads. Trusted test composition can deliberately issue key/path mismatch, missing/extra-field, or wrong-type traces; each must fail exactly `SOURCE_TRACE_INVALID`. The same value with CLI/literal operation methods, fallback/indirect operation methods, or alias methods remains prohibited.
|
|
140
|
-
- The approved live profile is provider `alibaba-cloud-model-studio-openai-compatible-cn-beijing`, endpoint `https://llm-clids9mqc5o1mbvb.cn-beijing.maas.aliyuncs.com/compatible-mode/v1`, model `qwen3.7-text-embedding`, qualification `qualification-2026-07-25`, dimensions `
|
|
140
|
+
- The approved live profile is provider `alibaba-cloud-model-studio-openai-compatible-cn-beijing`, endpoint `https://llm-clids9mqc5o1mbvb.cn-beijing.maas.aliyuncs.com/compatible-mode/v1`, model `qwen3.7-text-embedding`, qualification `qualification-2026-07-25`, dimensions `1536`. The embedding backend is profile-selectable via `ARGO_EMBEDDING_PROFILE`: `approved` (default) enforces those values byte-for-byte; `openai-compatible` reads a self-hosted OpenAI-compatible endpoint verbatim but still fails closed on a missing key or an invalid dimension, with the vector-index dimension driven by `ARGO_EMBEDDING_DIMENSIONS`. `ARGO_EMBEDDING_QUERY_INSTRUCTION` prefixes the query side only (instruction-tuned models), never stored documents; `ARGO_EMBEDDING_API_KEY` (optional secret) overrides `QWEN_KEY` as the embeddings Bearer token.
|
|
141
141
|
- Live network access requires explicit opt-in through the approved configuration source, normally the ignored/untracked `.argo/.env` containing `ARGO_LIVE_PROVIDER_E2E=1` and, for W3.1, `ARGO_W31_LIVE_MUTATION_VECTOR_E2E=1`; controlled process injection is allowed only through the same provenance-checked resolver. Default/offline CI remains deterministic but never substitutes fake evidence for a live pass.
|
|
142
142
|
- Redaction verification includes a synthetic-success recording boundary that captures full Cypher text/parameter and graph-evidence values, detects canaries in neutral fields, and clears all in-memory persistence before inspecting generated artifacts.
|
|
143
143
|
- The controlled Neo4j test boundary uses `ARGO_NEO4J_DATABASE_URL`, `ARGO_NEO4J_DATABASE_USERNAME`, and `ARGO_NEO4J_DATABASE_PASSWORD`; the password flows only to `neo4j.auth.basic`, never to Cypher or evidence.
|
|
@@ -35,6 +35,9 @@ const {
|
|
|
35
35
|
const {
|
|
36
36
|
markPhase,
|
|
37
37
|
} = require('./mcpCrashDiagnostics.js');
|
|
38
|
+
const {
|
|
39
|
+
composeQueryEmbeddingInput,
|
|
40
|
+
} = require('./embeddingProviderProfile.js');
|
|
38
41
|
|
|
39
42
|
const APPROVED_SOURCE_KEYS = Object.freeze([
|
|
40
43
|
'ARGO_EMBEDDING_BASE_URL',
|
|
@@ -165,6 +168,7 @@ function createDefaultSemanticRetrieval(dependencies = {}) {
|
|
|
165
168
|
composition,
|
|
166
169
|
canonicalGraph,
|
|
167
170
|
activeReadinessBoundary,
|
|
171
|
+
expectedEmbeddingQualification(configurationEvidence.configuration),
|
|
168
172
|
);
|
|
169
173
|
if (!evidence.alignment.aligned) {
|
|
170
174
|
await attemptAutomaticAlignment({
|
|
@@ -176,6 +180,7 @@ function createDefaultSemanticRetrieval(dependencies = {}) {
|
|
|
176
180
|
composition,
|
|
177
181
|
canonicalGraph,
|
|
178
182
|
activeReadinessBoundary,
|
|
183
|
+
expectedEmbeddingQualification(configurationEvidence.configuration),
|
|
179
184
|
);
|
|
180
185
|
if (!evidence.alignment.aligned) {
|
|
181
186
|
throw semanticAutomaticAlignmentFailed(evidence.alignment);
|
|
@@ -219,11 +224,12 @@ function createDefaultSemanticRetrieval(dependencies = {}) {
|
|
|
219
224
|
&& activeTestComposition.useReadinessBoundary !== true
|
|
220
225
|
? undefined
|
|
221
226
|
: readinessBoundary;
|
|
222
|
-
await composition.resolveConfiguration();
|
|
227
|
+
const configurationEvidence = await composition.resolveConfiguration();
|
|
223
228
|
const evidence = await readAndEvaluatePersistentReadiness(
|
|
224
229
|
composition,
|
|
225
230
|
canonicalGraph,
|
|
226
231
|
activeReadinessBoundary,
|
|
232
|
+
expectedEmbeddingQualification(configurationEvidence.configuration),
|
|
227
233
|
);
|
|
228
234
|
return publicReadinessOutcome(evidence.alignment);
|
|
229
235
|
} finally {
|
|
@@ -255,9 +261,12 @@ async function executeWpP2Retrieval({
|
|
|
255
261
|
transport: composition.transport,
|
|
256
262
|
});
|
|
257
263
|
markPhase('retrieval:embed:start');
|
|
258
|
-
const vector = await provider.embed(
|
|
264
|
+
const vector = await provider.embed(composeQueryEmbeddingInput(
|
|
265
|
+
request.intent,
|
|
266
|
+
configurationEvidence.configuration.embeddingQueryInstruction,
|
|
267
|
+
));
|
|
259
268
|
markPhase('retrieval:embed:done');
|
|
260
|
-
requireQualifiedVector(vector);
|
|
269
|
+
requireQualifiedVector(vector, configurationEvidence.configuration.embeddingDimensions);
|
|
261
270
|
const purpose = request && typeof request.purpose === 'string' ? request.purpose : '';
|
|
262
271
|
const strict = AUDIT_PURPOSES.has(purpose);
|
|
263
272
|
const topK = Number.isInteger(request.topK) && request.topK > 0 ? request.topK : resolveTopK();
|
|
@@ -652,7 +661,7 @@ async function readPersistentReadiness(neo4jDriver, readinessBoundary) {
|
|
|
652
661
|
return { readiness, requireQualification: false };
|
|
653
662
|
}
|
|
654
663
|
|
|
655
|
-
function evaluatePersistentReadiness(readiness, canonicalGraph, requireQualification = false) {
|
|
664
|
+
function evaluatePersistentReadiness(readiness, canonicalGraph, requireQualification = false, expectedQualification = APPROVED_PROFILE) {
|
|
656
665
|
const expectedCanonicalVersion = deriveCanonicalVersion(canonicalGraph);
|
|
657
666
|
const records = new Map((Array.isArray(readiness.channels) ? readiness.channels : [])
|
|
658
667
|
.map(record => [record.channel, record]));
|
|
@@ -669,10 +678,10 @@ function evaluatePersistentReadiness(readiness, canonicalGraph, requireQualifica
|
|
|
669
678
|
|| record.contentVersion !== readiness.contentVersion
|
|
670
679
|
|| record.indexVersion !== readiness.indexVersion
|
|
671
680
|
|| (requireQualification && (
|
|
672
|
-
record.provider !==
|
|
673
|
-
|| record.model !==
|
|
674
|
-
|| record.modelVersion !==
|
|
675
|
-
|| record.dimensions !==
|
|
681
|
+
record.provider !== expectedQualification.provider
|
|
682
|
+
|| record.model !== expectedQualification.model
|
|
683
|
+
|| record.modelVersion !== expectedQualification.version
|
|
684
|
+
|| record.dimensions !== expectedQualification.dimensions
|
|
676
685
|
|| record.queryable !== true
|
|
677
686
|
|| record.coherent !== true
|
|
678
687
|
))
|
|
@@ -720,17 +729,27 @@ function arrayEvidence(value, fallback = []) {
|
|
|
720
729
|
: fallback;
|
|
721
730
|
}
|
|
722
731
|
|
|
723
|
-
async function readAndEvaluatePersistentReadiness(composition, canonicalGraph, readinessBoundary) {
|
|
732
|
+
async function readAndEvaluatePersistentReadiness(composition, canonicalGraph, readinessBoundary, expectedQualification) {
|
|
724
733
|
const persistent = await readPersistentReadiness(composition.neo4jDriver, readinessBoundary);
|
|
725
734
|
const readiness = persistent.readiness;
|
|
726
735
|
const alignment = evaluatePersistentReadiness(
|
|
727
736
|
readiness,
|
|
728
737
|
canonicalGraph,
|
|
729
738
|
persistent.requireQualification,
|
|
739
|
+
expectedQualification,
|
|
730
740
|
);
|
|
731
741
|
return { composition, readiness, alignment };
|
|
732
742
|
}
|
|
733
743
|
|
|
744
|
+
function expectedEmbeddingQualification(configuration) {
|
|
745
|
+
return {
|
|
746
|
+
provider: configuration.embeddingProvider,
|
|
747
|
+
model: configuration.embeddingModel,
|
|
748
|
+
version: configuration.embeddingModelVersion,
|
|
749
|
+
dimensions: configuration.embeddingDimensions,
|
|
750
|
+
};
|
|
751
|
+
}
|
|
752
|
+
|
|
734
753
|
function publicFailureEvidence(readiness) {
|
|
735
754
|
const categories = new Set([
|
|
736
755
|
'APPROVED_SECRET_REQUIRED',
|
|
@@ -929,10 +948,10 @@ async function completeSemanticResult({
|
|
|
929
948
|
canonicalGraph,
|
|
930
949
|
embeddingQualification: {
|
|
931
950
|
approvedByHuman: true,
|
|
932
|
-
provider:
|
|
933
|
-
model:
|
|
934
|
-
version:
|
|
935
|
-
dimensions:
|
|
951
|
+
provider: configurationEvidence.configuration.embeddingProvider,
|
|
952
|
+
model: configurationEvidence.configuration.embeddingModel,
|
|
953
|
+
version: configurationEvidence.configuration.embeddingModelVersion,
|
|
954
|
+
dimensions: configurationEvidence.configuration.embeddingDimensions,
|
|
936
955
|
},
|
|
937
956
|
neo4jRetrievalBoundary: {
|
|
938
957
|
async retrieve() {
|
|
@@ -967,10 +986,10 @@ async function completeSemanticResult({
|
|
|
967
986
|
seedsByType: Object.freeze(seedsByType),
|
|
968
987
|
configurationEvidence: Object.freeze({
|
|
969
988
|
attribution: configurationEvidence.attribution,
|
|
970
|
-
provider:
|
|
971
|
-
model:
|
|
972
|
-
modelVersion:
|
|
973
|
-
dimensions:
|
|
989
|
+
provider: configurationEvidence.configuration.embeddingProvider,
|
|
990
|
+
model: configurationEvidence.configuration.embeddingModel,
|
|
991
|
+
modelVersion: configurationEvidence.configuration.embeddingModelVersion,
|
|
992
|
+
dimensions: configurationEvidence.configuration.embeddingDimensions,
|
|
974
993
|
}),
|
|
975
994
|
canonicalVersion: versions.canonicalVersion,
|
|
976
995
|
contentVersion: versions.contentVersion,
|
|
@@ -1159,10 +1178,10 @@ function semanticAutomaticAlignmentFailed(alignment, sourceError) {
|
|
|
1159
1178
|
return error;
|
|
1160
1179
|
}
|
|
1161
1180
|
|
|
1162
|
-
function requireQualifiedVector(vector) {
|
|
1181
|
+
function requireQualifiedVector(vector, dimensions = APPROVED_PROFILE.dimensions) {
|
|
1163
1182
|
if (
|
|
1164
1183
|
!Array.isArray(vector)
|
|
1165
|
-
|| vector.length !==
|
|
1184
|
+
|| vector.length !== dimensions
|
|
1166
1185
|
|| vector.some(value => typeof value !== 'number' || !Number.isFinite(value))
|
|
1167
1186
|
) {
|
|
1168
1187
|
throw safeError('LIVE_PROVIDER_RESPONSE_INVALID');
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Embedding provider profile (ArchGraph ARGO).
|
|
4
|
+
//
|
|
5
|
+
// The semantic-retrieval embedding backend is either the human-approved cloud
|
|
6
|
+
// profile (`approved`, the default, byte-for-byte unchanged) or a self-hosted
|
|
7
|
+
// OpenAI-compatible endpoint (`openai-compatible`) for intranet / offline
|
|
8
|
+
// deployments. This module owns the profile vocabulary, the dimension contract,
|
|
9
|
+
// and the query-side instruction composition so the live configuration resolver
|
|
10
|
+
// and the retrieval runtime share one implementation.
|
|
11
|
+
|
|
12
|
+
const PROFILE_KEY = 'ARGO_EMBEDDING_PROFILE';
|
|
13
|
+
const QUERY_INSTRUCTION_KEY = 'ARGO_EMBEDDING_QUERY_INSTRUCTION';
|
|
14
|
+
const API_KEY = 'ARGO_EMBEDDING_API_KEY';
|
|
15
|
+
const PROFILE_APPROVED = 'approved';
|
|
16
|
+
const PROFILE_OPENAI_COMPATIBLE = 'openai-compatible';
|
|
17
|
+
const SUPPORTED_EMBEDDING_PROFILES = Object.freeze([PROFILE_APPROVED, PROFILE_OPENAI_COMPATIBLE]);
|
|
18
|
+
const DEFAULT_EMBEDDING_DIMENSIONS = 1536;
|
|
19
|
+
|
|
20
|
+
function resolveEmbeddingProfile(value) {
|
|
21
|
+
const normalized = typeof value === 'string' ? value.trim().toLowerCase() : '';
|
|
22
|
+
return normalized === '' ? PROFILE_APPROVED : normalized;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function isApprovedEmbeddingProfile(value) {
|
|
26
|
+
return resolveEmbeddingProfile(value) === PROFILE_APPROVED;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function requireSupportedEmbeddingProfile(value) {
|
|
30
|
+
const profile = resolveEmbeddingProfile(value);
|
|
31
|
+
if (!SUPPORTED_EMBEDDING_PROFILES.includes(profile)) {
|
|
32
|
+
throw profileError('EMBEDDING_PROFILE_UNSUPPORTED');
|
|
33
|
+
}
|
|
34
|
+
return profile;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function resolveEmbeddingDimensions(value, fallback = DEFAULT_EMBEDDING_DIMENSIONS) {
|
|
38
|
+
const parsed = typeof value === 'string' && value.trim() !== '' ? Number(value) : NaN;
|
|
39
|
+
return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function requireEmbeddingDimensions(value) {
|
|
43
|
+
const parsed = typeof value === 'string' && value.trim() !== '' ? Number(value) : NaN;
|
|
44
|
+
if (!Number.isInteger(parsed) || parsed <= 0) {
|
|
45
|
+
throw profileError('EMBEDDING_DIMENSIONS_INVALID');
|
|
46
|
+
}
|
|
47
|
+
return parsed;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
// The query-side input may carry an instruction prefix (e.g. gte-Qwen2's
|
|
51
|
+
// "Instruct: <task>\nQuery: <query>"); document-side input stays raw so stored
|
|
52
|
+
// vectors are never polluted by a query-only prefix. The prefix commonly comes
|
|
53
|
+
// from a single-line `.env` value, so `\n`/`\r`/`\t` escapes are decoded here.
|
|
54
|
+
function normalizeInstruction(instruction) {
|
|
55
|
+
if (typeof instruction !== 'string') return '';
|
|
56
|
+
return instruction
|
|
57
|
+
.replace(/\\r\\n/g, '\n')
|
|
58
|
+
.replace(/\\n/g, '\n')
|
|
59
|
+
.replace(/\\r/g, '\r')
|
|
60
|
+
.replace(/\\t/g, '\t');
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function composeQueryEmbeddingInput(text, instruction) {
|
|
64
|
+
const query = text == null ? '' : String(text);
|
|
65
|
+
const prefix = normalizeInstruction(instruction);
|
|
66
|
+
return prefix === '' ? query : `${prefix}${query}`;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function profileError(category) {
|
|
70
|
+
const error = new Error(category);
|
|
71
|
+
error.category = category;
|
|
72
|
+
return error;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
module.exports = {
|
|
76
|
+
PROFILE_KEY,
|
|
77
|
+
QUERY_INSTRUCTION_KEY,
|
|
78
|
+
API_KEY,
|
|
79
|
+
PROFILE_APPROVED,
|
|
80
|
+
PROFILE_OPENAI_COMPATIBLE,
|
|
81
|
+
SUPPORTED_EMBEDDING_PROFILES,
|
|
82
|
+
DEFAULT_EMBEDDING_DIMENSIONS,
|
|
83
|
+
resolveEmbeddingProfile,
|
|
84
|
+
isApprovedEmbeddingProfile,
|
|
85
|
+
requireSupportedEmbeddingProfile,
|
|
86
|
+
resolveEmbeddingDimensions,
|
|
87
|
+
requireEmbeddingDimensions,
|
|
88
|
+
composeQueryEmbeddingInput,
|
|
89
|
+
};
|
|
@@ -46,7 +46,7 @@ function createLiveEmbeddingIndexGate(dependencies = {}) {
|
|
|
46
46
|
requireInput(input);
|
|
47
47
|
const vector = await client.embed(input.input);
|
|
48
48
|
if (
|
|
49
|
-
vector.length !==
|
|
49
|
+
vector.length !== configuration.embeddingDimensions
|
|
50
50
|
|| vector.some(value => typeof value !== 'number' || !Number.isFinite(value))
|
|
51
51
|
) {
|
|
52
52
|
throw safeError('LIVE_PROVIDER_RESPONSE_INVALID');
|
|
@@ -79,7 +79,6 @@ function requireApprovedConfiguration(configuration, qualification) {
|
|
|
79
79
|
|| qualification.model !== configuration.embeddingModel
|
|
80
80
|
|| qualification.version !== configuration.embeddingModelVersion
|
|
81
81
|
|| qualification.dimensions !== configuration.embeddingDimensions
|
|
82
|
-
|| qualification.dimensions !== 1536
|
|
83
82
|
) {
|
|
84
83
|
throw safeError('LIVE_PROVIDER_INDEX_WRITE_PROHIBITED');
|
|
85
84
|
}
|
|
@@ -11,7 +11,7 @@ function createLiveEmbeddingProviderClient({ configuration, transport }) {
|
|
|
11
11
|
{
|
|
12
12
|
method: 'POST',
|
|
13
13
|
headers: {
|
|
14
|
-
Authorization: `Bearer ${configuration.qwenKey}`,
|
|
14
|
+
Authorization: `Bearer ${configuration.embeddingApiKey || configuration.qwenKey}`,
|
|
15
15
|
'Content-Type': 'application/json',
|
|
16
16
|
},
|
|
17
17
|
body: JSON.stringify({
|
|
@@ -6,6 +6,14 @@ const {
|
|
|
6
6
|
const {
|
|
7
7
|
getArgoEnvPath,
|
|
8
8
|
} = require('../argo-paths.js');
|
|
9
|
+
const {
|
|
10
|
+
PROFILE_KEY,
|
|
11
|
+
QUERY_INSTRUCTION_KEY,
|
|
12
|
+
API_KEY: EMBEDDING_API_KEY_KEY,
|
|
13
|
+
PROFILE_APPROVED,
|
|
14
|
+
requireSupportedEmbeddingProfile,
|
|
15
|
+
requireEmbeddingDimensions,
|
|
16
|
+
} = require('./embeddingProviderProfile.js');
|
|
9
17
|
|
|
10
18
|
const CONFIG_KEYS = Object.freeze([
|
|
11
19
|
'ARGO_EMBEDDING_BASE_URL',
|
|
@@ -63,11 +71,35 @@ const READABLE_KEYS = Object.freeze([
|
|
|
63
71
|
...CONFIG_KEYS,
|
|
64
72
|
...OPTIONAL_CONFIG_KEYS,
|
|
65
73
|
...RETRIEVAL_TUNING_KEYS,
|
|
74
|
+
PROFILE_KEY,
|
|
75
|
+
QUERY_INSTRUCTION_KEY,
|
|
76
|
+
EMBEDDING_API_KEY_KEY,
|
|
66
77
|
...Object.keys(OPT_IN_KEYS),
|
|
67
78
|
]);
|
|
68
79
|
const LEGACY_KEYS = Object.freeze(['ARGO_NEO4J_URI', 'ARGO_NEO4J_USERNAME', 'ARGO_NEO4J_PASSWORD']);
|
|
80
|
+
// Host / process-level keys: read from the environment at launch, but NOT
|
|
81
|
+
// accepted inside the .env file (see argo/.env.example Part 2). Kept explicit so
|
|
82
|
+
// the coverage test (tests/env-key-coverage.test.js) can prove every env key
|
|
83
|
+
// referenced by argo/scripts is classified — either a .env key, a host-only key,
|
|
84
|
+
// or a legacy alias. This is what prevents "a key the code already uses is still
|
|
85
|
+
// rejected as unknown".
|
|
86
|
+
const HOST_ONLY_ENV_KEYS = Object.freeze([
|
|
87
|
+
'ARGO_ENV_FILE',
|
|
88
|
+
'ARGO_REPO_ROOT',
|
|
89
|
+
'ARGO_EA_QEA',
|
|
90
|
+
'ARGO_WORKSPACE_ROOTS',
|
|
91
|
+
'ARGO_SERVER_PATH',
|
|
92
|
+
'GRAPH_MCP_URL',
|
|
93
|
+
'EA_QEA_DEBUG',
|
|
94
|
+
'WORKSPACE_FOLDER',
|
|
95
|
+
'ARGO_TEST_TIMEOUT_MS',
|
|
96
|
+
'ARGO_MCP_MUTATION_RESPONSE_DEBUG',
|
|
97
|
+
'ARGO_MCP_SEMANTIC_DEDUP',
|
|
98
|
+
'ARGO_MCP_SEMANTIC_DEDUP_THRESHOLD',
|
|
99
|
+
'ARGO_SEMANTIC_DEDUP_THRESHOLD',
|
|
100
|
+
]);
|
|
69
101
|
const PROHIBITED_RUNTIME_FIELD_KEYS = Object.freeze(['neo4jUri', 'embeddingCredential']);
|
|
70
|
-
const SECRET_KEYS = new Set(['ARGO_NEO4J_DATABASE_PASSWORD', 'QWEN_KEY', 'ARGO_RERANK_API_KEY']);
|
|
102
|
+
const SECRET_KEYS = new Set(['ARGO_NEO4J_DATABASE_PASSWORD', 'QWEN_KEY', 'ARGO_RERANK_API_KEY', EMBEDDING_API_KEY_KEY]);
|
|
71
103
|
const APPROVED = Object.freeze({
|
|
72
104
|
ARGO_EMBEDDING_BASE_URL: 'https://llm-clids9mqc5o1mbvb.cn-beijing.maas.aliyuncs.com/compatible-mode/v1',
|
|
73
105
|
ARGO_EMBEDDING_MODEL: 'qwen3.7-text-embedding',
|
|
@@ -194,15 +226,16 @@ async function resolveTrusted({
|
|
|
194
226
|
}
|
|
195
227
|
attribution[key] = present(processValue) ? 'process' : 'file';
|
|
196
228
|
}
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
229
|
+
const profile = requireSupportedEmbeddingProfile(
|
|
230
|
+
selectOptional(processValues, fileValues, PROFILE_KEY),
|
|
231
|
+
);
|
|
232
|
+
const profileConfiguration = buildProfileConfiguration(normalized, profile);
|
|
233
|
+
const embeddingQueryInstruction = selectOptional(processValues, fileValues, QUERY_INSTRUCTION_KEY);
|
|
234
|
+
const explicitEmbeddingApiKey = selectOptional(processValues, fileValues, EMBEDDING_API_KEY_KEY);
|
|
200
235
|
const configuration = Object.freeze({
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
embeddingModelVersion: normalized.ARGO_EMBEDDING_MODEL_VERSION,
|
|
205
|
-
embeddingDimensions: 1536,
|
|
236
|
+
...profileConfiguration,
|
|
237
|
+
embeddingQueryInstruction,
|
|
238
|
+
embeddingApiKey: explicitEmbeddingApiKey || normalized.QWEN_KEY,
|
|
206
239
|
neo4jDatabaseUrl: normalized.ARGO_NEO4J_DATABASE_URL,
|
|
207
240
|
neo4jDatabaseUsername: normalized.ARGO_NEO4J_DATABASE_USERNAME,
|
|
208
241
|
neo4jDatabasePassword: normalized.ARGO_NEO4J_DATABASE_PASSWORD,
|
|
@@ -216,6 +249,52 @@ async function resolveTrusted({
|
|
|
216
249
|
});
|
|
217
250
|
}
|
|
218
251
|
|
|
252
|
+
// Profile-aware embedding identity. `approved` (the default) enforces the
|
|
253
|
+
// human-approved cloud values byte-for-byte; `openai-compatible` accepts the
|
|
254
|
+
// configured self-hosted endpoint verbatim but still fails closed on any missing
|
|
255
|
+
// key or invalid dimension (no implicit default).
|
|
256
|
+
function buildProfileConfiguration(normalized, profileValue) {
|
|
257
|
+
const profile = requireSupportedEmbeddingProfile(profileValue);
|
|
258
|
+
for (const key of [
|
|
259
|
+
'ARGO_EMBEDDING_BASE_URL',
|
|
260
|
+
'ARGO_EMBEDDING_MODEL',
|
|
261
|
+
'ARGO_EMBEDDING_PROVIDER',
|
|
262
|
+
'ARGO_EMBEDDING_MODEL_VERSION',
|
|
263
|
+
]) {
|
|
264
|
+
if (!present(normalized[key])) {
|
|
265
|
+
const error = safeError('LIVE_PROVIDER_CONFIGURATION_REQUIRED');
|
|
266
|
+
error.field = key;
|
|
267
|
+
throw error;
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
if (profile === PROFILE_APPROVED) {
|
|
271
|
+
for (const [key, expected] of Object.entries(APPROVED)) {
|
|
272
|
+
if (normalized[key] !== expected) throw safeError('LIVE_PROVIDER_CONFIGURATION_REQUIRED');
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
return Object.freeze({
|
|
276
|
+
embeddingProfile: profile,
|
|
277
|
+
embeddingBaseUrl: normalized.ARGO_EMBEDDING_BASE_URL,
|
|
278
|
+
embeddingModel: normalized.ARGO_EMBEDDING_MODEL,
|
|
279
|
+
embeddingProvider: normalized.ARGO_EMBEDDING_PROVIDER,
|
|
280
|
+
embeddingModelVersion: normalized.ARGO_EMBEDDING_MODEL_VERSION,
|
|
281
|
+
embeddingDimensions: requireEmbeddingDimensions(normalized.ARGO_EMBEDDING_DIMENSIONS),
|
|
282
|
+
});
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
function selectOptional(processValues, fileValues, key) {
|
|
286
|
+
const processValue = processValues.get(key);
|
|
287
|
+
const fileValue = fileValues.get(key);
|
|
288
|
+
if (present(processValue) && present(fileValue) && processValue !== fileValue) {
|
|
289
|
+
throw safeError(SECRET_KEYS.has(key)
|
|
290
|
+
? 'SECRET_SOURCE_CONFLICT'
|
|
291
|
+
: 'LIVE_PROVIDER_CONFIGURATION_CONFLICT');
|
|
292
|
+
}
|
|
293
|
+
if (present(processValue)) return processValue;
|
|
294
|
+
if (present(fileValue)) return fileValue;
|
|
295
|
+
return '';
|
|
296
|
+
}
|
|
297
|
+
|
|
219
298
|
function optionalDatabaseName(value, repositoryRoot) {
|
|
220
299
|
if (present(value)) return String(value).trim();
|
|
221
300
|
const repoName = path.basename(repositoryRoot);
|
|
@@ -534,10 +613,14 @@ module.exports = {
|
|
|
534
613
|
resolveApprovedLiveConfiguration,
|
|
535
614
|
withApprovedLiveConfigurationTestComposition,
|
|
536
615
|
posixModeIsSecretSafe,
|
|
616
|
+
buildProfileConfiguration,
|
|
617
|
+
APPROVED_EMBEDDING_VALUES: APPROVED,
|
|
537
618
|
// The authoritative set of keys an approved `.env` file may carry. The
|
|
538
619
|
// committed `.env.example` must document exactly this set (see
|
|
539
620
|
// tests/env-example.test.js); SECRET_KEYS marks the subset that must be
|
|
540
621
|
// preflighted as secrets.
|
|
541
622
|
ENV_FILE_KEYS: READABLE_KEYS,
|
|
542
623
|
ENV_FILE_SECRET_KEYS: Object.freeze(Array.from(SECRET_KEYS)),
|
|
624
|
+
HOST_ONLY_ENV_KEYS,
|
|
625
|
+
ENV_FILE_LEGACY_KEYS: LEGACY_KEYS,
|
|
543
626
|
};
|
|
@@ -29,7 +29,7 @@ function createProductionSemanticNeo4jAdapter(dependencies = {}) {
|
|
|
29
29
|
}));
|
|
30
30
|
}
|
|
31
31
|
return withSession(driver, dependencies.configuration, async session => {
|
|
32
|
-
await ensureVectorIndexes(session);
|
|
32
|
+
await ensureVectorIndexes(session, resolveEmbeddingDimensions(dependencies.configuration));
|
|
33
33
|
await ensureFulltextIndexes(session);
|
|
34
34
|
const results = [];
|
|
35
35
|
for (const [channel, definition] of Object.entries(CHANNEL_INDEXES)) {
|
|
@@ -93,11 +93,16 @@ function createProductionSemanticNeo4jAdapter(dependencies = {}) {
|
|
|
93
93
|
|
|
94
94
|
const EMBEDDING_DIMENSIONS = 1536;
|
|
95
95
|
|
|
96
|
-
|
|
96
|
+
function resolveEmbeddingDimensions(configuration) {
|
|
97
|
+
const value = configuration && configuration.embeddingDimensions;
|
|
98
|
+
return Number.isInteger(value) && value > 0 ? value : EMBEDDING_DIMENSIONS;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
async function ensureVectorIndexes(session, dimensions = EMBEDDING_DIMENSIONS) {
|
|
97
102
|
for (const definition of Object.values(CHANNEL_INDEXES)) {
|
|
98
103
|
// CREATE ... IF NOT EXISTS cannot change an existing index's dimensions, so
|
|
99
|
-
// when the
|
|
100
|
-
// stale index first and recreate it at the
|
|
104
|
+
// when the configured embedding dimension changes (e.g. 1024 -> 1536) drop
|
|
105
|
+
// the stale index first and recreate it at the configured dimension.
|
|
101
106
|
const existing = await executeRead(
|
|
102
107
|
session,
|
|
103
108
|
'SHOW INDEXES YIELD name, type, options WHERE type = \'VECTOR\' AND name = $name RETURN name, options',
|
|
@@ -106,7 +111,7 @@ async function ensureVectorIndexes(session) {
|
|
|
106
111
|
if (existing.records.length > 0) {
|
|
107
112
|
const options = existing.records[0].get('options');
|
|
108
113
|
const dim = options && options.indexConfig && options.indexConfig['vector.dimensions'];
|
|
109
|
-
if (dim !==
|
|
114
|
+
if (dim !== dimensions) {
|
|
110
115
|
await executeWrite(session, `DROP INDEX ${definition.indexName} IF EXISTS`, {});
|
|
111
116
|
}
|
|
112
117
|
}
|
|
@@ -115,7 +120,7 @@ async function ensureVectorIndexes(session) {
|
|
|
115
120
|
[
|
|
116
121
|
`CREATE VECTOR INDEX ${definition.indexName} IF NOT EXISTS`,
|
|
117
122
|
`FOR (semantic:${definition.label}) ON (semantic.vector)`,
|
|
118
|
-
`OPTIONS { indexConfig: { \`vector.dimensions\`: ${
|
|
123
|
+
`OPTIONS { indexConfig: { \`vector.dimensions\`: ${dimensions}, \`vector.similarity_function\`: "cosine" } }`,
|
|
119
124
|
].join('\n'),
|
|
120
125
|
{},
|
|
121
126
|
);
|
|
@@ -177,4 +182,5 @@ function cloneRecord(record) {
|
|
|
177
182
|
|
|
178
183
|
module.exports = {
|
|
179
184
|
createProductionSemanticNeo4jAdapter,
|
|
185
|
+
resolveEmbeddingDimensions,
|
|
180
186
|
};
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Maps profile-aware configuration evidence (from resolveApprovedLiveConfiguration)
|
|
4
|
+
// into the shape the canonical semantic-init / backfill runtime consumes.
|
|
5
|
+
//
|
|
6
|
+
// This is the single place that turns "which embedding profile" into the init
|
|
7
|
+
// backfill's configuration, so argo init embeds through the SAME provider the
|
|
8
|
+
// retrieval path uses (approved cloud by default, or a self-hosted endpoint via
|
|
9
|
+
// ARGO_EMBEDDING_PROFILE=openai-compatible) — instead of a hardcoded profile.
|
|
10
|
+
|
|
11
|
+
function buildDefaultSemanticConfiguration(evidence) {
|
|
12
|
+
const configuration = evidence && typeof evidence === 'object' && evidence.configuration
|
|
13
|
+
? evidence.configuration
|
|
14
|
+
: evidence;
|
|
15
|
+
if (!configuration || typeof configuration !== 'object') {
|
|
16
|
+
throw new TypeError('resolved configuration evidence is required');
|
|
17
|
+
}
|
|
18
|
+
return Object.freeze({
|
|
19
|
+
...configuration,
|
|
20
|
+
embeddingCredential: configuration.embeddingApiKey || configuration.qwenKey,
|
|
21
|
+
});
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
module.exports = { buildDefaultSemanticConfiguration };
|
|
@@ -144,12 +144,12 @@ const {
|
|
|
144
144
|
const {
|
|
145
145
|
semanticOperatorErrorResult,
|
|
146
146
|
} = require('./graph-rag/semanticOperatorError.js');
|
|
147
|
-
const {
|
|
148
|
-
resolveExternalProductionConfig,
|
|
149
|
-
} = require('./graph-rag/externalProductionConfig.js');
|
|
150
147
|
const {
|
|
151
148
|
resolveApprovedLiveConfiguration,
|
|
152
149
|
} = require('./graph-rag/liveEmbeddingProviderConfig.js');
|
|
150
|
+
const {
|
|
151
|
+
buildDefaultSemanticConfiguration,
|
|
152
|
+
} = require('./graph-rag/semanticInitConfiguration.js');
|
|
153
153
|
const {
|
|
154
154
|
createLiveEmbeddingProviderClient,
|
|
155
155
|
} = require('./graph-rag/liveEmbeddingProviderClient.js');
|
|
@@ -4056,59 +4056,17 @@ async function createDefaultProductionSemanticRuntime(options = {}) {
|
|
|
4056
4056
|
});
|
|
4057
4057
|
}
|
|
4058
4058
|
|
|
4059
|
+
// Resolve the canonical semantic-init/backfill configuration through the SAME
|
|
4060
|
+
// profile-aware resolver the retrieval path uses, so argo init embeds via the
|
|
4061
|
+
// configured provider (ARGO_EMBEDDING_PROFILE) instead of a hardcoded cloud
|
|
4062
|
+
// profile. `resolveApprovedLiveConfiguration` still fails closed when a required
|
|
4063
|
+
// key is missing or an unknown key is present in the .env file.
|
|
4059
4064
|
async function resolveDefaultSemanticConfiguration(repositoryRoot) {
|
|
4060
|
-
|
|
4061
|
-
|
|
4062
|
-
|
|
4063
|
-
neo4jUri: process.env.ARGO_NEO4J_DATABASE_URL,
|
|
4064
|
-
neo4jUsername: process.env.ARGO_NEO4J_DATABASE_USERNAME,
|
|
4065
|
-
neo4jPassword: process.env.ARGO_NEO4J_DATABASE_PASSWORD,
|
|
4066
|
-
embeddingCredential: process.env.QWEN_KEY,
|
|
4067
|
-
neo4jDatabase: process.env.ARGO_NEO4J_DATABASE || getDefaultSemanticNeo4jDatabaseName(repositoryRoot),
|
|
4068
|
-
}, {
|
|
4069
|
-
operation: 'semantic-backfill',
|
|
4070
|
-
sourceKeys: new Map([
|
|
4071
|
-
['neo4jUri', 'ARGO_NEO4J_DATABASE_URL'],
|
|
4072
|
-
['neo4jUsername', 'ARGO_NEO4J_DATABASE_USERNAME'],
|
|
4073
|
-
['neo4jPassword', 'ARGO_NEO4J_DATABASE_PASSWORD'],
|
|
4074
|
-
['embeddingCredential', 'QWEN_KEY'],
|
|
4075
|
-
]),
|
|
4076
|
-
});
|
|
4077
|
-
} catch (error) {
|
|
4078
|
-
if (error && error.category === 'EXTERNAL_CREDENTIALS_REQUIRED') {
|
|
4079
|
-
const missing = new Error('EXTERNAL_CREDENTIALS_REQUIRED');
|
|
4080
|
-
missing.category = 'EXTERNAL_CREDENTIALS_REQUIRED';
|
|
4081
|
-
missing.field = error.field;
|
|
4082
|
-
throw missing;
|
|
4083
|
-
}
|
|
4084
|
-
throw error;
|
|
4085
|
-
}
|
|
4086
|
-
return Object.freeze({
|
|
4087
|
-
embeddingBaseUrl: W31_APPROVED_PROFILE.baseUrl,
|
|
4088
|
-
embeddingModel: W31_APPROVED_PROFILE.model,
|
|
4089
|
-
embeddingProvider: W31_APPROVED_PROFILE.provider,
|
|
4090
|
-
embeddingModelVersion: W31_APPROVED_PROFILE.version,
|
|
4091
|
-
embeddingDimensions: W31_APPROVED_PROFILE.dimensions,
|
|
4092
|
-
neo4jDatabaseUrl: external.neo4jUri,
|
|
4093
|
-
neo4jDatabaseUsername: external.neo4jUsername,
|
|
4094
|
-
neo4jDatabasePassword: external.neo4jPassword,
|
|
4095
|
-
qwenKey: external.embeddingCredential,
|
|
4096
|
-
embeddingCredential: external.embeddingCredential,
|
|
4097
|
-
...(external.neo4jDatabase === undefined ? {} : { neo4jDatabase: external.neo4jDatabase }),
|
|
4065
|
+
const evidence = await resolveApprovedLiveConfiguration({
|
|
4066
|
+
repositoryRoot,
|
|
4067
|
+
requiredOptIns: [LIVE_PROVIDER_OPT_IN, W31_LIVE_OPT_IN],
|
|
4098
4068
|
});
|
|
4099
|
-
|
|
4100
|
-
|
|
4101
|
-
function getDefaultSemanticNeo4jDatabaseName(repositoryRoot) {
|
|
4102
|
-
const repoName = path.basename(repositoryRoot || resolveWorkspaceRoot());
|
|
4103
|
-
const normalized = String(repoName)
|
|
4104
|
-
.toLowerCase()
|
|
4105
|
-
.replace(/[^a-z0-9.-]+/g, '-')
|
|
4106
|
-
.replace(/^-+|-+$/g, '')
|
|
4107
|
-
.replace(/\.{2,}/g, '.')
|
|
4108
|
-
.replace(/-{2,}/g, '-');
|
|
4109
|
-
const safe = normalized || 'workspace';
|
|
4110
|
-
const prefixed = /^[a-z]/.test(safe) ? safe : `db-${safe}`;
|
|
4111
|
-
return prefixed.slice(0, 63);
|
|
4069
|
+
return buildDefaultSemanticConfiguration(evidence);
|
|
4112
4070
|
}
|
|
4113
4071
|
|
|
4114
4072
|
function deriveSemanticCanonicalVersion(document) {
|
package/package.json
CHANGED