archgraph-argo 0.19.1 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -1
- package/argo/scripts/graph-rag/defaultSemanticRetrieval.js +115 -5
- package/argo/scripts/graph-rag/hybridRetrieval.js +114 -0
- package/argo/scripts/graph-rag/liveEmbeddingProviderConfig.js +12 -1
- package/argo/scripts/graph-rag/mutationEmbeddingVectorLifecycle.js +1 -0
- package/argo/scripts/graph-rag/rerankRetrieval.js +150 -0
- package/argo/scripts/graph-rag/semantic-persistence/productionSemanticBackfill.js +3 -1
- package/argo/scripts/graph-rag/semantic-persistence/productionSemanticNeo4jAdapter.js +14 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -7,6 +7,10 @@ An architecture-graph driven framework for Agentic Engineering.
|
|
|
7
7
|
ArchGraph builds a **unified language** that puts harness design and target product design into
|
|
8
8
|
**one model** — so you get a single view to work and observe, and real control over your agents.
|
|
9
9
|
|
|
10
|
+
It doubles as a **long-term memory for coding agents**: an ArchiMate 3.2 intent graph exposed through
|
|
11
|
+
a single read/write MCP interface. Writes are deduplicated, so the graph stays clean and semantic
|
|
12
|
+
recall stays precise. See the [home page](https://archgraph.org/) for the full capability set.
|
|
13
|
+
|
|
10
14
|

|
|
11
15
|
|
|
12
16
|
## Architecture
|
|
@@ -71,7 +75,9 @@ After installing, open your project and start a coding agent. It will:
|
|
|
71
75
|
|
|
72
76
|
1. locate the architecture element behind the task before changing anything,
|
|
73
77
|
2. arm itself with that element's Skills and Rules,
|
|
74
|
-
3. work test-first (GIVEN-WHEN-THEN), and trace every commit back to the graph
|
|
78
|
+
3. work test-first (GIVEN-WHEN-THEN), and trace every commit back to the graph,
|
|
79
|
+
4. reuse an existing element, relationship, or view instead of creating a duplicate — the write path
|
|
80
|
+
deduplicates by identity and flags a semantically near element of the same type.
|
|
75
81
|
|
|
76
82
|
The intent architecture graph — modelled in **ArchiMate 3.2** — is the single source of truth.
|
|
77
83
|
|
|
@@ -14,6 +14,21 @@ const {
|
|
|
14
14
|
getWorkspaceRoot,
|
|
15
15
|
resolveArgoPath,
|
|
16
16
|
} = require('../argo-paths.js');
|
|
17
|
+
const {
|
|
18
|
+
isHybridEnabled,
|
|
19
|
+
hybridTopK,
|
|
20
|
+
hybridWeights,
|
|
21
|
+
rrfK,
|
|
22
|
+
fuseChannelSeeds,
|
|
23
|
+
sanitizeFulltextQuery,
|
|
24
|
+
} = require('./hybridRetrieval.js');
|
|
25
|
+
const {
|
|
26
|
+
isRerankEnabled,
|
|
27
|
+
rerankConfig,
|
|
28
|
+
resolveRerankProvider,
|
|
29
|
+
rerankCandidates,
|
|
30
|
+
applyRerankOrder,
|
|
31
|
+
} = require('./rerankRetrieval.js');
|
|
17
32
|
|
|
18
33
|
const APPROVED_SOURCE_KEYS = Object.freeze([
|
|
19
34
|
'ARGO_EMBEDDING_BASE_URL',
|
|
@@ -75,6 +90,20 @@ const VECTOR_QUERY_CYPHER_SCOPED = [
|
|
|
75
90
|
'RETURN properties(node) AS record, score',
|
|
76
91
|
'ORDER BY score DESC',
|
|
77
92
|
].join('\n');
|
|
93
|
+
const LEXICAL_QUERY_CYPHER = [
|
|
94
|
+
'CALL db.index.fulltext.queryNodes($indexName, $queryText, { limit: $topK })',
|
|
95
|
+
'YIELD node, score',
|
|
96
|
+
'WHERE node.channel = $channel',
|
|
97
|
+
'RETURN properties(node) AS record, score',
|
|
98
|
+
'ORDER BY score DESC',
|
|
99
|
+
].join('\n');
|
|
100
|
+
const LEXICAL_QUERY_CYPHER_SCOPED = [
|
|
101
|
+
'CALL db.index.fulltext.queryNodes($indexName, $queryText, { limit: $topK })',
|
|
102
|
+
'YIELD node, score',
|
|
103
|
+
'WHERE node.channel = $channel AND node.canonicalIdentity IN $canonicalIdentities',
|
|
104
|
+
'RETURN properties(node) AS record, score',
|
|
105
|
+
'ORDER BY score DESC',
|
|
106
|
+
].join('\n');
|
|
78
107
|
const READINESS_QUERY_CYPHER = [
|
|
79
108
|
'MATCH (readiness:ArgoProductionSemanticReadiness {identity: $identity})',
|
|
80
109
|
'RETURN properties(readiness) AS readiness',
|
|
@@ -206,18 +235,49 @@ async function executeWpP2Retrieval({
|
|
|
206
235
|
const purpose = request && typeof request.purpose === 'string' ? request.purpose : '';
|
|
207
236
|
const strict = AUDIT_PURPOSES.has(purpose);
|
|
208
237
|
const topK = Number.isInteger(request.topK) && request.topK > 0 ? request.topK : resolveTopK();
|
|
238
|
+
const scoped = Array.isArray(canonicalIdentities) && canonicalIdentities.length > 0;
|
|
239
|
+
const hybrid = isHybridEnabled();
|
|
240
|
+
const lexicalTopK = hybridTopK();
|
|
241
|
+
const fusionK = rrfK();
|
|
242
|
+
const fusionWeights = hybridWeights();
|
|
243
|
+
const rerank = isRerankEnabled();
|
|
244
|
+
const rerankOptions = rerankConfig();
|
|
245
|
+
const rerankProvider = rerank ? resolveRerankProvider(configurationEvidence.configuration) : null;
|
|
246
|
+
// Rerank needs a larger candidate pool than the final top-K.
|
|
247
|
+
const pool = rerank ? Math.max(topK, rerankOptions.poolSize) : topK;
|
|
209
248
|
const seedsByType = {};
|
|
210
249
|
for (const channel of CHANNELS) {
|
|
211
|
-
|
|
250
|
+
const vectorSeeds = await exhaustChannel({
|
|
212
251
|
channel,
|
|
213
252
|
neo4jDriver: composition.neo4jDriver,
|
|
214
253
|
vector,
|
|
215
254
|
threshold: strict ? auditThresholdFor(channel) : memoryThresholdFor(channel),
|
|
216
|
-
maxSeeds:
|
|
217
|
-
...(
|
|
218
|
-
? { canonicalIdentities }
|
|
219
|
-
: {}),
|
|
255
|
+
maxSeeds: pool,
|
|
256
|
+
...(scoped ? { canonicalIdentities } : {}),
|
|
220
257
|
});
|
|
258
|
+
let seeds = vectorSeeds;
|
|
259
|
+
if (hybrid) {
|
|
260
|
+
const lexicalSeeds = await exhaustLexicalChannel({
|
|
261
|
+
channel,
|
|
262
|
+
neo4jDriver: composition.neo4jDriver,
|
|
263
|
+
queryText: request.intent,
|
|
264
|
+
maxSeeds: lexicalTopK,
|
|
265
|
+
...(scoped ? { canonicalIdentities } : {}),
|
|
266
|
+
});
|
|
267
|
+
seeds = fuseChannelSeeds({ vectorSeeds, lexicalSeeds, k: fusionK, limit: Math.max(pool, lexicalTopK), weights: fusionWeights });
|
|
268
|
+
}
|
|
269
|
+
if (rerank && seeds.length > 1) {
|
|
270
|
+
const ordered = await rerankCandidates({
|
|
271
|
+
query: request.intent,
|
|
272
|
+
candidates: seeds,
|
|
273
|
+
provider: rerankProvider,
|
|
274
|
+
transport: composition.transport,
|
|
275
|
+
maxReturn: rerankOptions.maxReturn,
|
|
276
|
+
});
|
|
277
|
+
// fail-open: a null/empty order keeps the original ordering
|
|
278
|
+
seeds = applyRerankOrder(seeds, ordered, topK);
|
|
279
|
+
}
|
|
280
|
+
seedsByType[channel.key] = seeds;
|
|
221
281
|
}
|
|
222
282
|
return completeSemanticResult({
|
|
223
283
|
request: completeRequest,
|
|
@@ -303,6 +363,9 @@ async function executeProductionNeo4jOperation(configuration, operation) {
|
|
|
303
363
|
...record.get('record'),
|
|
304
364
|
score: numberValue(record.get('score')),
|
|
305
365
|
}));
|
|
366
|
+
if (operation.kind === 'semantic-lexical-query') {
|
|
367
|
+
return { records };
|
|
368
|
+
}
|
|
306
369
|
const offset = operation.parameters.offset;
|
|
307
370
|
const windowSize = operation.parameters.windowSize;
|
|
308
371
|
const returnedCount = Math.max(0, records.length - offset);
|
|
@@ -709,6 +772,53 @@ async function exhaustChannel({
|
|
|
709
772
|
return Object.freeze(accepted);
|
|
710
773
|
}
|
|
711
774
|
|
|
775
|
+
// Lexical (full-text/BM25) sibling of exhaustChannel. Additive and fail-open:
|
|
776
|
+
// it returns [] when the query text is blank, when the full-text index is
|
|
777
|
+
// missing, or on any error, so hybrid retrieval can never break the baseline.
|
|
778
|
+
async function exhaustLexicalChannel({
|
|
779
|
+
channel,
|
|
780
|
+
neo4jDriver,
|
|
781
|
+
queryText,
|
|
782
|
+
maxSeeds,
|
|
783
|
+
canonicalIdentities,
|
|
784
|
+
}) {
|
|
785
|
+
if (typeof queryText !== 'string' || queryText.trim() === '') {
|
|
786
|
+
return Object.freeze([]);
|
|
787
|
+
}
|
|
788
|
+
const safeQuery = sanitizeFulltextQuery(queryText);
|
|
789
|
+
if (safeQuery === '') {
|
|
790
|
+
return Object.freeze([]);
|
|
791
|
+
}
|
|
792
|
+
const scoped = Array.isArray(canonicalIdentities) && canonicalIdentities.length > 0;
|
|
793
|
+
const scopedCanonicalIdentities = scoped
|
|
794
|
+
? scopeCanonicalIdentitiesForChannel(canonicalIdentities, channel)
|
|
795
|
+
: canonicalIdentities;
|
|
796
|
+
const effectiveMax = Number.isInteger(maxSeeds) && maxSeeds > 0 ? maxSeeds : INITIAL_WINDOW_SIZE;
|
|
797
|
+
const indexName = `${channel.indexName}_fulltext`;
|
|
798
|
+
try {
|
|
799
|
+
const result = await neo4jDriver.execute(Object.freeze({
|
|
800
|
+
kind: 'semantic-lexical-query',
|
|
801
|
+
channel: channel.channel,
|
|
802
|
+
indexName,
|
|
803
|
+
cypher: scoped ? LEXICAL_QUERY_CYPHER_SCOPED : LEXICAL_QUERY_CYPHER,
|
|
804
|
+
parameters: Object.freeze({
|
|
805
|
+
indexName,
|
|
806
|
+
channel: channel.channel,
|
|
807
|
+
queryText: safeQuery,
|
|
808
|
+
topK: effectiveMax,
|
|
809
|
+
...(scoped ? { canonicalIdentities: scopedCanonicalIdentities } : {}),
|
|
810
|
+
}),
|
|
811
|
+
}));
|
|
812
|
+
const records = Array.isArray(result && result.records) ? result.records : [];
|
|
813
|
+
return Object.freeze(records
|
|
814
|
+
.map(raw => normalizeVectorRecord(raw, channel))
|
|
815
|
+
.filter(Boolean)
|
|
816
|
+
.slice(0, effectiveMax));
|
|
817
|
+
} catch {
|
|
818
|
+
return Object.freeze([]);
|
|
819
|
+
}
|
|
820
|
+
}
|
|
821
|
+
|
|
712
822
|
// Scope identities come from the canonical graph as BARE ids (e.g.
|
|
713
823
|
// "memory-eval-bench-wp-001"), but the semantic vector records store their
|
|
714
824
|
// canonicalIdentity with a channel prefix (e.g. "Element:memory-eval-bench-wp-001").
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Hybrid retrieval fusion (P2). Pure, dependency-free helpers so the fusion
|
|
4
|
+
// logic is fully unit-testable and the production retrieval path can stay
|
|
5
|
+
// additive: when hybrid is OFF nothing here is invoked.
|
|
6
|
+
//
|
|
7
|
+
// Design:
|
|
8
|
+
// - vector (dense) and lexical (full-text/BM25) ranked lists are fused with
|
|
9
|
+
// Reciprocal Rank Fusion: score(id) = sum over lists of 1/(k + rank).
|
|
10
|
+
// - RRF needs no score normalization between the two very different score
|
|
11
|
+
// scales, and guarantees the union is retained (recall can only increase).
|
|
12
|
+
|
|
13
|
+
const DEFAULT_RRF_K = 60;
|
|
14
|
+
const DEFAULT_HYBRID_TOP_K = 16;
|
|
15
|
+
// Vector-first weighting: the dense channel is the stronger signal on this
|
|
16
|
+
// graph, so lexical only nudges the ranking and rescues misses.
|
|
17
|
+
const DEFAULT_VECTOR_WEIGHT = 3;
|
|
18
|
+
const DEFAULT_LEXICAL_WEIGHT = 1;
|
|
19
|
+
|
|
20
|
+
function isHybridEnabled(env = process.env) {
|
|
21
|
+
return String((env && env.ARGO_SEMANTIC_HYBRID) || '') === '1';
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function hybridWeights(env = process.env) {
|
|
25
|
+
const vector = Number(env && env.ARGO_SEMANTIC_HYBRID_VECTOR_WEIGHT);
|
|
26
|
+
const lexical = Number(env && env.ARGO_SEMANTIC_HYBRID_LEXICAL_WEIGHT);
|
|
27
|
+
return [
|
|
28
|
+
Number.isFinite(vector) && vector >= 0 ? vector : DEFAULT_VECTOR_WEIGHT,
|
|
29
|
+
Number.isFinite(lexical) && lexical >= 0 ? lexical : DEFAULT_LEXICAL_WEIGHT,
|
|
30
|
+
];
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function hybridTopK(env = process.env) {
|
|
34
|
+
const value = Number(env && env.ARGO_SEMANTIC_HYBRID_TOP_K);
|
|
35
|
+
return Number.isInteger(value) && value > 0 ? value : DEFAULT_HYBRID_TOP_K;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function rrfK(env = process.env) {
|
|
39
|
+
const value = Number(env && env.ARGO_SEMANTIC_HYBRID_RRF_K);
|
|
40
|
+
return Number.isFinite(value) && value > 0 ? value : DEFAULT_RRF_K;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
// Lucene full-text queries reject raw punctuation (e.g. "A --(Flow)--> B") and
|
|
44
|
+
// blow the maxClauseCount on very long text (long Chinese descriptions become
|
|
45
|
+
// hundreds of single-character clauses). Truncate first, then escape the
|
|
46
|
+
// reserved query-syntax characters so any intent/name/statement is literal.
|
|
47
|
+
// Returns '' for blank input (caller skips).
|
|
48
|
+
const MAX_FULLTEXT_QUERY_LENGTH = 256;
|
|
49
|
+
|
|
50
|
+
function sanitizeFulltextQuery(text) {
|
|
51
|
+
const value = String(text === undefined || text === null ? '' : text)
|
|
52
|
+
.trim()
|
|
53
|
+
.slice(0, MAX_FULLTEXT_QUERY_LENGTH);
|
|
54
|
+
if (!value) return '';
|
|
55
|
+
return value
|
|
56
|
+
.replace(/&&/g, '\\&\\&')
|
|
57
|
+
.replace(/\|\|/g, '\\|\\|')
|
|
58
|
+
.replace(/([+\-!(){}[\]^"~*?:\\/])/g, '\\$1');
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// Fuse ranked record lists by id (canonicalIdentity). Each list is expected to be
|
|
62
|
+
// already sorted best-first. Returns records carrying the fused `score`, with
|
|
63
|
+
// the first-seen record object preserved (so downstream keeps its fields).
|
|
64
|
+
function rrfFuse(lists, options = {}) {
|
|
65
|
+
const k = Number.isFinite(options.k) && options.k > 0 ? options.k : DEFAULT_RRF_K;
|
|
66
|
+
const weights = Array.isArray(options.weights) ? options.weights : [];
|
|
67
|
+
const byId = new Map();
|
|
68
|
+
(Array.isArray(lists) ? lists : []).forEach((list, listIndex) => {
|
|
69
|
+
const weight = Number.isFinite(weights[listIndex]) && weights[listIndex] >= 0 ? weights[listIndex] : 1;
|
|
70
|
+
(Array.isArray(list) ? list : []).forEach((record, rank) => {
|
|
71
|
+
if (!record || !record.id) return;
|
|
72
|
+
const entry = byId.get(record.id) || { id: record.id, record, score: 0, matchedLists: 0 };
|
|
73
|
+
entry.score += weight / (k + rank + 1);
|
|
74
|
+
entry.matchedLists += 1;
|
|
75
|
+
if (!entry.record) entry.record = record;
|
|
76
|
+
byId.set(record.id, entry);
|
|
77
|
+
});
|
|
78
|
+
});
|
|
79
|
+
return [...byId.values()]
|
|
80
|
+
.sort((left, right) => (right.score - left.score) || String(left.id).localeCompare(String(right.id)))
|
|
81
|
+
.map(entry => Object.freeze({
|
|
82
|
+
...entry.record,
|
|
83
|
+
id: entry.id,
|
|
84
|
+
canonicalIdentity: entry.id,
|
|
85
|
+
score: entry.score,
|
|
86
|
+
rrfScore: entry.score,
|
|
87
|
+
matchedLists: entry.matchedLists,
|
|
88
|
+
}));
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
// Fuse the two per-channel seed lists. With an empty lexical list this preserves
|
|
92
|
+
// the vector ordering (RRF on a single list is order-preserving) — which is what
|
|
93
|
+
// makes "hybrid ON but no lexical hit" behave like the vector-only baseline.
|
|
94
|
+
function fuseChannelSeeds({ vectorSeeds = [], lexicalSeeds = [], k = DEFAULT_RRF_K, limit, weights } = {}) {
|
|
95
|
+
const fused = rrfFuse([
|
|
96
|
+
Array.isArray(vectorSeeds) ? vectorSeeds.filter(Boolean) : [],
|
|
97
|
+
Array.isArray(lexicalSeeds) ? lexicalSeeds.filter(Boolean) : [],
|
|
98
|
+
], { k, ...(Array.isArray(weights) ? { weights } : {}) });
|
|
99
|
+
return Number.isInteger(limit) && limit > 0 ? fused.slice(0, limit) : fused;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
module.exports = {
|
|
103
|
+
DEFAULT_RRF_K,
|
|
104
|
+
DEFAULT_HYBRID_TOP_K,
|
|
105
|
+
DEFAULT_VECTOR_WEIGHT,
|
|
106
|
+
DEFAULT_LEXICAL_WEIGHT,
|
|
107
|
+
isHybridEnabled,
|
|
108
|
+
hybridTopK,
|
|
109
|
+
hybridWeights,
|
|
110
|
+
rrfK,
|
|
111
|
+
rrfFuse,
|
|
112
|
+
fuseChannelSeeds,
|
|
113
|
+
sanitizeFulltextQuery,
|
|
114
|
+
};
|
|
@@ -34,6 +34,17 @@ const RETRIEVAL_TUNING_KEYS = Object.freeze([
|
|
|
34
34
|
'ARGO_SEMANTIC_AUDIT_THRESHOLD_RELATIONSHIP',
|
|
35
35
|
'ARGO_SEMANTIC_AUDIT_THRESHOLD_VIEW',
|
|
36
36
|
'ARGO_SEMANTIC_TOP_K',
|
|
37
|
+
// Rerank (P3) tuning + optional dedicated rerank provider. When the provider
|
|
38
|
+
// keys are unset the reranker falls back to the embedding provider (qwen).
|
|
39
|
+
'ARGO_SEMANTIC_RERANK',
|
|
40
|
+
'ARGO_SEMANTIC_RERANK_MODEL',
|
|
41
|
+
'ARGO_SEMANTIC_RERANK_POOL',
|
|
42
|
+
'ARGO_SEMANTIC_RERANK_RETURN',
|
|
43
|
+
'ARGO_SEMANTIC_RERANK_TIMEOUT_MS',
|
|
44
|
+
'ARGO_RERANK_BASE_URL',
|
|
45
|
+
'ARGO_RERANK_API_KEY',
|
|
46
|
+
'ARGO_RERANK_PROVIDER',
|
|
47
|
+
'ARGO_RERANK_MODEL',
|
|
37
48
|
]);
|
|
38
49
|
const OPT_IN_KEYS = Object.freeze({
|
|
39
50
|
ARGO_LIVE_PROVIDER_E2E: 'LIVE_PROVIDER_E2E_OPT_IN_REQUIRED',
|
|
@@ -47,7 +58,7 @@ const READABLE_KEYS = Object.freeze([
|
|
|
47
58
|
]);
|
|
48
59
|
const LEGACY_KEYS = Object.freeze(['ARGO_NEO4J_URI', 'ARGO_NEO4J_USERNAME', 'ARGO_NEO4J_PASSWORD']);
|
|
49
60
|
const PROHIBITED_RUNTIME_FIELD_KEYS = Object.freeze(['neo4jUri', 'embeddingCredential']);
|
|
50
|
-
const SECRET_KEYS = new Set(['ARGO_NEO4J_DATABASE_PASSWORD', 'QWEN_KEY']);
|
|
61
|
+
const SECRET_KEYS = new Set(['ARGO_NEO4J_DATABASE_PASSWORD', 'QWEN_KEY', 'ARGO_RERANK_API_KEY']);
|
|
51
62
|
const APPROVED = Object.freeze({
|
|
52
63
|
ARGO_EMBEDDING_BASE_URL: 'https://llm-clids9mqc5o1mbvb.cn-beijing.maas.aliyuncs.com/compatible-mode/v1',
|
|
53
64
|
ARGO_EMBEDDING_MODEL: 'qwen3.7-text-embedding',
|
|
@@ -531,6 +531,7 @@ function buildPersistentWork(canonicalWrite, configuration, versions) {
|
|
|
531
531
|
contentVersion: `content:${fingerprint(content)}`,
|
|
532
532
|
indexVersion: `index:${fingerprint({ objectId, content, canonicalVersion: versions.canonicalVersion })}`,
|
|
533
533
|
content,
|
|
534
|
+
searchText: content,
|
|
534
535
|
}));
|
|
535
536
|
} else {
|
|
536
537
|
tombstones.push(Object.freeze(base));
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Two-stage rerank (P3): after the (vector, or hybrid) candidate pool is
|
|
4
|
+
// retrieved, an LLM listwise reranker reorders it by relevance to the query.
|
|
5
|
+
// Additive and fail-open: when disabled nothing runs; any error returns null so
|
|
6
|
+
// the caller keeps the original ordering. Pure helpers are exported for tests.
|
|
7
|
+
|
|
8
|
+
const DEFAULT_RERANK_MODEL = 'qwen-turbo';
|
|
9
|
+
const DEFAULT_RERANK_POOL = 20;
|
|
10
|
+
const DEFAULT_RERANK_RETURN = 8;
|
|
11
|
+
const DEFAULT_RERANK_TIMEOUT_MS = 8000;
|
|
12
|
+
|
|
13
|
+
function rerankTimeoutMs(env = process.env) {
|
|
14
|
+
const value = Number(env && env.ARGO_SEMANTIC_RERANK_TIMEOUT_MS);
|
|
15
|
+
return Number.isFinite(value) && value > 0 ? value : DEFAULT_RERANK_TIMEOUT_MS;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
function isRerankEnabled(env = process.env) {
|
|
19
|
+
return String((env && env.ARGO_SEMANTIC_RERANK) || '') === '1';
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function firstNonEmpty(...values) {
|
|
23
|
+
for (const value of values) {
|
|
24
|
+
if (typeof value === 'string' && value.trim() !== '') return value.trim();
|
|
25
|
+
}
|
|
26
|
+
return '';
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
// Rerank provider resolution, DECOUPLED from the embedding provider:
|
|
30
|
+
// - dedicated keys (ARGO_RERANK_BASE_URL / ARGO_RERANK_API_KEY / ARGO_RERANK_PROVIDER
|
|
31
|
+
// / ARGO_RERANK_MODEL) win when set, so a different provider/model can be used;
|
|
32
|
+
// - otherwise it falls back to the embedding configuration (qwen by default).
|
|
33
|
+
function resolveRerankProvider(configuration = {}, env = process.env) {
|
|
34
|
+
const baseUrl = firstNonEmpty(env.ARGO_RERANK_BASE_URL, configuration.embeddingBaseUrl);
|
|
35
|
+
const apiKey = firstNonEmpty(env.ARGO_RERANK_API_KEY, configuration.qwenKey);
|
|
36
|
+
const model = firstNonEmpty(env.ARGO_RERANK_MODEL, env.ARGO_SEMANTIC_RERANK_MODEL, DEFAULT_RERANK_MODEL);
|
|
37
|
+
const provider = firstNonEmpty(env.ARGO_RERANK_PROVIDER, configuration.embeddingProvider, 'qwen');
|
|
38
|
+
return { baseUrl, apiKey, model, provider };
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function rerankConfig(env = process.env) {
|
|
42
|
+
const model = (env && typeof env.ARGO_SEMANTIC_RERANK_MODEL === 'string' && env.ARGO_SEMANTIC_RERANK_MODEL.trim())
|
|
43
|
+
? env.ARGO_SEMANTIC_RERANK_MODEL.trim()
|
|
44
|
+
: DEFAULT_RERANK_MODEL;
|
|
45
|
+
const pool = Number(env && env.ARGO_SEMANTIC_RERANK_POOL);
|
|
46
|
+
const maxReturn = Number(env && env.ARGO_SEMANTIC_RERANK_RETURN);
|
|
47
|
+
return {
|
|
48
|
+
model,
|
|
49
|
+
poolSize: Number.isInteger(pool) && pool > 0 ? pool : DEFAULT_RERANK_POOL,
|
|
50
|
+
maxReturn: Number.isInteger(maxReturn) && maxReturn > 0 ? maxReturn : DEFAULT_RERANK_RETURN,
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function candidateText(record) {
|
|
55
|
+
const raw = record && (record.searchText || record.description || record.name || '');
|
|
56
|
+
return String(raw).replace(/\s+/g, ' ').slice(0, 160);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
// Accept the canonical id or its bare/suffix form (models often drop the
|
|
60
|
+
// channel prefix), dedupe, ignore anything not in the candidate set.
|
|
61
|
+
function parseRerankOrder(content, candidateIds) {
|
|
62
|
+
let parsed;
|
|
63
|
+
try { parsed = JSON.parse(content); } catch { return []; }
|
|
64
|
+
const raw = Array.isArray(parsed && parsed.order) ? parsed.order : [];
|
|
65
|
+
const byKey = new Map();
|
|
66
|
+
for (const id of candidateIds) {
|
|
67
|
+
const text = String(id);
|
|
68
|
+
byKey.set(text, id);
|
|
69
|
+
byKey.set(text.split(':').slice(1).join(':'), id);
|
|
70
|
+
byKey.set(text.split(':').pop(), id);
|
|
71
|
+
}
|
|
72
|
+
const seen = new Set();
|
|
73
|
+
const order = [];
|
|
74
|
+
for (const value of raw) {
|
|
75
|
+
const canonical = byKey.get(String(value));
|
|
76
|
+
if (canonical && !seen.has(canonical)) {
|
|
77
|
+
seen.add(canonical);
|
|
78
|
+
order.push(canonical);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
return order;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
// Reorder seeds by the model's order, then append any seeds the model omitted
|
|
85
|
+
// (nothing is lost), bounded by `limit`.
|
|
86
|
+
function applyRerankOrder(seeds, orderedIds, limit) {
|
|
87
|
+
const list = Array.isArray(seeds) ? seeds.filter(Boolean) : [];
|
|
88
|
+
const byId = new Map(list.map(seed => [seed.id, seed]));
|
|
89
|
+
const out = [];
|
|
90
|
+
const seen = new Set();
|
|
91
|
+
for (const id of (Array.isArray(orderedIds) ? orderedIds : [])) {
|
|
92
|
+
const seed = byId.get(id);
|
|
93
|
+
if (seed && !seen.has(id)) { seen.add(id); out.push(seed); }
|
|
94
|
+
}
|
|
95
|
+
for (const seed of list) {
|
|
96
|
+
if (!seen.has(seed.id)) { seen.add(seed.id); out.push(seed); }
|
|
97
|
+
}
|
|
98
|
+
return Number.isInteger(limit) && limit > 0 ? out.slice(0, limit) : out;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
async function rerankCandidates({ query, candidates, provider, transport, maxReturn, timeoutMs } = {}) {
|
|
102
|
+
const list = Array.isArray(candidates) ? candidates.filter(candidate => candidate && candidate.id) : [];
|
|
103
|
+
if (list.length < 2 || typeof query !== 'string' || query.trim() === '') return null;
|
|
104
|
+
if (!provider || !transport || typeof transport.request !== 'function') return null;
|
|
105
|
+
const baseUrl = typeof provider.baseUrl === 'string' ? provider.baseUrl : '';
|
|
106
|
+
const apiKey = typeof provider.apiKey === 'string' ? provider.apiKey : '';
|
|
107
|
+
if (baseUrl === '' || apiKey === '') return null;
|
|
108
|
+
const model = provider.model || DEFAULT_RERANK_MODEL;
|
|
109
|
+
const body = {
|
|
110
|
+
model,
|
|
111
|
+
temperature: 0,
|
|
112
|
+
response_format: { type: 'json_object' },
|
|
113
|
+
messages: [
|
|
114
|
+
{ role: 'system', content: 'You rank architecture elements by relevance to a query. Return ONLY JSON {"order":[ids best-first]} using only the candidate ids.' },
|
|
115
|
+
{ role: 'user', content: `Query: ${query}\n\nCandidates (id\\ttext):\n${list.map(candidate => `${candidate.id}\t${candidateText(candidate)}`).join('\n')}\n\nReturn up to ${Math.min(maxReturn || DEFAULT_RERANK_RETURN, list.length)} ids best-first.` },
|
|
116
|
+
],
|
|
117
|
+
};
|
|
118
|
+
const controller = typeof AbortController === 'function' ? new AbortController() : null;
|
|
119
|
+
const timer = controller ? setTimeout(() => controller.abort(), timeoutMs || rerankTimeoutMs()) : null;
|
|
120
|
+
try {
|
|
121
|
+
const response = await transport.request(`${baseUrl}/chat/completions`, {
|
|
122
|
+
method: 'POST',
|
|
123
|
+
headers: { Authorization: `Bearer ${apiKey}`, 'Content-Type': 'application/json' },
|
|
124
|
+
body: JSON.stringify(body),
|
|
125
|
+
...(controller ? { signal: controller.signal } : {}),
|
|
126
|
+
});
|
|
127
|
+
if (!response || response.ok !== true || typeof response.json !== 'function') return null;
|
|
128
|
+
const payload = await response.json();
|
|
129
|
+
const content = payload && payload.choices && payload.choices[0] && payload.choices[0].message && payload.choices[0].message.content;
|
|
130
|
+
return parseRerankOrder(content, list.map(candidate => candidate.id));
|
|
131
|
+
} catch {
|
|
132
|
+
return null;
|
|
133
|
+
} finally {
|
|
134
|
+
if (timer) clearTimeout(timer);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
module.exports = {
|
|
139
|
+
DEFAULT_RERANK_MODEL,
|
|
140
|
+
DEFAULT_RERANK_POOL,
|
|
141
|
+
DEFAULT_RERANK_RETURN,
|
|
142
|
+
DEFAULT_RERANK_TIMEOUT_MS,
|
|
143
|
+
isRerankEnabled,
|
|
144
|
+
rerankConfig,
|
|
145
|
+
rerankTimeoutMs,
|
|
146
|
+
resolveRerankProvider,
|
|
147
|
+
parseRerankOrder,
|
|
148
|
+
applyRerankOrder,
|
|
149
|
+
rerankCandidates,
|
|
150
|
+
};
|
|
@@ -154,8 +154,9 @@ async function processChannel(options) {
|
|
|
154
154
|
}
|
|
155
155
|
|
|
156
156
|
function buildSemanticRecord(record, vector, options) {
|
|
157
|
+
const searchText = buildSemanticRecordText(record.channel, record.canonicalObject);
|
|
157
158
|
const contentHash = crypto.createHash('sha256')
|
|
158
|
-
.update(
|
|
159
|
+
.update(searchText)
|
|
159
160
|
.digest('hex');
|
|
160
161
|
return Object.freeze({
|
|
161
162
|
canonicalIdentity: record.canonicalIdentity,
|
|
@@ -167,6 +168,7 @@ function buildSemanticRecord(record, vector, options) {
|
|
|
167
168
|
model: options.qualification.model,
|
|
168
169
|
modelVersion: options.qualification.version,
|
|
169
170
|
dimensions: options.qualification.dimensions,
|
|
171
|
+
searchText,
|
|
170
172
|
vector: Object.freeze([...vector]),
|
|
171
173
|
});
|
|
172
174
|
}
|
|
@@ -30,6 +30,7 @@ function createProductionSemanticNeo4jAdapter(dependencies = {}) {
|
|
|
30
30
|
}
|
|
31
31
|
return withSession(driver, dependencies.configuration, async session => {
|
|
32
32
|
await ensureVectorIndexes(session);
|
|
33
|
+
await ensureFulltextIndexes(session);
|
|
33
34
|
const results = [];
|
|
34
35
|
for (const [channel, definition] of Object.entries(CHANNEL_INDEXES)) {
|
|
35
36
|
const channelRecords = records.filter(record => record.channel === channel).map(cloneRecord);
|
|
@@ -121,6 +122,19 @@ async function ensureVectorIndexes(session) {
|
|
|
121
122
|
}
|
|
122
123
|
}
|
|
123
124
|
|
|
125
|
+
async function ensureFulltextIndexes(session) {
|
|
126
|
+
for (const definition of Object.values(CHANNEL_INDEXES)) {
|
|
127
|
+
await executeWrite(
|
|
128
|
+
session,
|
|
129
|
+
[
|
|
130
|
+
`CREATE FULLTEXT INDEX ${definition.indexName}_fulltext IF NOT EXISTS`,
|
|
131
|
+
`FOR (semantic:${definition.label}) ON EACH [semantic.searchText]`,
|
|
132
|
+
].join('\n'),
|
|
133
|
+
{},
|
|
134
|
+
);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
|
|
124
138
|
async function withSession(driver, configuration, action) {
|
|
125
139
|
const database = configuration && configuration.neo4jDatabase;
|
|
126
140
|
const session = driver.session(database === undefined ? undefined : { database });
|
package/package.json
CHANGED