@shadowforge0/aquifer-memory 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +62 -9
- package/consumers/cli.js +11 -78
- package/consumers/mcp.js +68 -4
- package/consumers/openclaw-plugin.js +18 -5
- package/consumers/shared/config.js +21 -1
- package/consumers/shared/factory.js +26 -5
- package/core/aquifer.js +237 -24
- package/core/storage.js +60 -16
- package/index.js +3 -1
- package/package.json +3 -3
- package/pipeline/_http.js +67 -0
- package/pipeline/embed.js +1 -63
- package/pipeline/normalize/adapters/claude-code.js +90 -0
- package/pipeline/normalize/adapters/gateway.js +67 -0
- package/pipeline/normalize/constants.js +12 -0
- package/pipeline/normalize/detect.js +52 -0
- package/pipeline/normalize/extract.js +49 -0
- package/pipeline/normalize/index.js +129 -0
- package/pipeline/normalize/timestamp.js +33 -0
- package/pipeline/rerank.js +161 -0
package/core/aquifer.js
CHANGED
|
@@ -36,6 +36,24 @@ function loadSql(filename, schema) {
|
|
|
36
36
|
return raw.replace(/\$\{schema\}/g, qi(schema));
|
|
37
37
|
}
|
|
38
38
|
|
|
39
|
+
// ---------------------------------------------------------------------------
|
|
40
|
+
// buildRerankDocument — assemble text for cross-encoder reranking
|
|
41
|
+
// ---------------------------------------------------------------------------
|
|
42
|
+
|
|
43
|
+
function buildRerankDocument(row, maxChars) {
|
|
44
|
+
let text = (row.summary_text || row.summary_snippet || '').replace(/\s+/g, ' ').trim();
|
|
45
|
+
const turn = (row.matched_turn_text || '').replace(/\s+/g, ' ').trim();
|
|
46
|
+
|
|
47
|
+
if (!text) {
|
|
48
|
+
text = turn;
|
|
49
|
+
} else if (turn && !text.includes(turn)) {
|
|
50
|
+
text = `${text}\n\nMatched turn:\n${turn}`;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
if (text.length > maxChars) text = text.slice(0, maxChars);
|
|
54
|
+
return text;
|
|
55
|
+
}
|
|
56
|
+
|
|
39
57
|
// ---------------------------------------------------------------------------
|
|
40
58
|
// createAquifer
|
|
41
59
|
// ---------------------------------------------------------------------------
|
|
@@ -59,6 +77,7 @@ function createAquifer(config) {
|
|
|
59
77
|
ownsPool = true;
|
|
60
78
|
} else {
|
|
61
79
|
pool = config.db;
|
|
80
|
+
ownsPool = !!config.ownsPool; // allow factory to claim ownership
|
|
62
81
|
}
|
|
63
82
|
|
|
64
83
|
// Embed config (lazy — only required for recall/enrich)
|
|
@@ -81,6 +100,19 @@ function createAquifer(config) {
|
|
|
81
100
|
const entityPromptFn = config.entities && config.entities.prompt ? config.entities.prompt : null;
|
|
82
101
|
const entityScope = (config.entities && config.entities.scope) || 'default';
|
|
83
102
|
|
|
103
|
+
// FTS config — locked to 'simple'.
|
|
104
|
+
// The search_tsv trigger always uses to_tsvector('simple', ...), so query-time
|
|
105
|
+
// config must match. Warn and override if someone passes anything else.
|
|
106
|
+
const _rawFtsConfig = config.ftsConfig || 'simple';
|
|
107
|
+
if (_rawFtsConfig !== 'simple') {
|
|
108
|
+
console.warn(
|
|
109
|
+
`[aquifer] ftsConfig '${_rawFtsConfig}' is not currently supported. ` +
|
|
110
|
+
`The search_tsv index is built with 'simple'; only 'simple' is valid at query time. ` +
|
|
111
|
+
`Overriding to 'simple'.`
|
|
112
|
+
);
|
|
113
|
+
}
|
|
114
|
+
const ftsConfig = 'simple';
|
|
115
|
+
|
|
84
116
|
// Rank weights
|
|
85
117
|
const rankWeights = {
|
|
86
118
|
rrf: 0.65,
|
|
@@ -90,6 +122,16 @@ function createAquifer(config) {
|
|
|
90
122
|
...(config.rank || {}),
|
|
91
123
|
};
|
|
92
124
|
|
|
125
|
+
// Reranker config (optional)
|
|
126
|
+
const rerankConfig = config.rerank || null;
|
|
127
|
+
let reranker = null;
|
|
128
|
+
if (rerankConfig) {
|
|
129
|
+
const { createReranker } = require('../pipeline/rerank');
|
|
130
|
+
reranker = createReranker(rerankConfig);
|
|
131
|
+
}
|
|
132
|
+
const defaultRerankTopK = rerankConfig ? Math.max(1, rerankConfig.topK || 20) : 0;
|
|
133
|
+
const rerankMaxChars = rerankConfig ? Math.max(200, rerankConfig.maxChars || 1600) : 0;
|
|
134
|
+
|
|
93
135
|
// Source registry (in-memory)
|
|
94
136
|
const sources = new Map();
|
|
95
137
|
|
|
@@ -106,7 +148,7 @@ function createAquifer(config) {
|
|
|
106
148
|
|
|
107
149
|
// --- Helper: embed search on summaries ---
|
|
108
150
|
async function embeddingSearchSummaries(queryVec, opts) {
|
|
109
|
-
const {
|
|
151
|
+
const { agentIds, source, dateFrom, dateTo, limit = 20 } = opts;
|
|
110
152
|
const where = [`s.tenant_id = $1`];
|
|
111
153
|
const params = [tenantId];
|
|
112
154
|
|
|
@@ -121,9 +163,9 @@ function createAquifer(config) {
|
|
|
121
163
|
params.push(dateTo);
|
|
122
164
|
where.push(`($${params.length}::date IS NULL OR s.started_at::date <= $${params.length}::date)`);
|
|
123
165
|
}
|
|
124
|
-
if (
|
|
125
|
-
params.push(
|
|
126
|
-
where.push(`s.agent_id = $${params.length}`);
|
|
166
|
+
if (agentIds && agentIds.length > 0) {
|
|
167
|
+
params.push(agentIds);
|
|
168
|
+
where.push(`s.agent_id = ANY($${params.length})`);
|
|
127
169
|
}
|
|
128
170
|
if (source) {
|
|
129
171
|
params.push(source);
|
|
@@ -520,10 +562,20 @@ function createAquifer(config) {
|
|
|
520
562
|
|
|
521
563
|
async recall(query, opts = {}) {
|
|
522
564
|
if (!query) return [];
|
|
523
|
-
|
|
565
|
+
|
|
566
|
+
const VALID_MODES = ['fts', 'hybrid', 'vector'];
|
|
567
|
+
const mode = opts.mode !== undefined ? opts.mode : 'hybrid';
|
|
568
|
+
if (!VALID_MODES.includes(mode)) {
|
|
569
|
+
throw new Error(`Invalid recall mode: "${mode}". Must be one of: ${VALID_MODES.join(', ')}`);
|
|
570
|
+
}
|
|
571
|
+
|
|
572
|
+
if (mode === 'hybrid' || mode === 'vector') {
|
|
573
|
+
requireEmbed('recall');
|
|
574
|
+
}
|
|
524
575
|
|
|
525
576
|
const {
|
|
526
577
|
agentId,
|
|
578
|
+
agentIds: rawAgentIds,
|
|
527
579
|
source,
|
|
528
580
|
dateFrom,
|
|
529
581
|
dateTo,
|
|
@@ -533,6 +585,12 @@ function createAquifer(config) {
|
|
|
533
585
|
entityMode = 'any',
|
|
534
586
|
} = opts;
|
|
535
587
|
|
|
588
|
+
// Normalize agentId/agentIds into a single resolved value
|
|
589
|
+
// agentIds takes precedence; agentId is sugar for agentIds: [agentId]
|
|
590
|
+
const resolvedAgentIds = rawAgentIds && rawAgentIds.length > 0
|
|
591
|
+
? rawAgentIds
|
|
592
|
+
: (agentId ? [agentId] : null);
|
|
593
|
+
|
|
536
594
|
// Validate before touching DB
|
|
537
595
|
if (explicitEntities && explicitEntities.length > 0 && !entitiesEnabled) {
|
|
538
596
|
throw new Error('Entities are not enabled');
|
|
@@ -540,12 +598,17 @@ function createAquifer(config) {
|
|
|
540
598
|
|
|
541
599
|
await ensureMigrated();
|
|
542
600
|
|
|
543
|
-
const
|
|
601
|
+
const rerankEnabled = !!reranker && opts.rerank !== false;
|
|
602
|
+
const rerankTopK = rerankEnabled ? Math.max(limit, opts.rerankTopK || defaultRerankTopK) : limit;
|
|
603
|
+
const fetchLimit = rerankTopK * 4;
|
|
544
604
|
|
|
545
|
-
// 1. Embed query
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
605
|
+
// 1. Embed query (only needed for hybrid/vector modes)
|
|
606
|
+
let queryVec = null;
|
|
607
|
+
if (mode === 'hybrid' || mode === 'vector') {
|
|
608
|
+
const queryVecResult = await embedFn([query]);
|
|
609
|
+
queryVec = queryVecResult[0];
|
|
610
|
+
if (!queryVec || !queryVec.length) return []; // m3: guard empty array too
|
|
611
|
+
}
|
|
549
612
|
|
|
550
613
|
// 2. Entity intersection pre-filter (when entityMode === 'all')
|
|
551
614
|
let candidateSessionIds = null; // null = no filter
|
|
@@ -621,17 +684,26 @@ function createAquifer(config) {
|
|
|
621
684
|
} catch (_) { /* entity search failure non-fatal */ }
|
|
622
685
|
}
|
|
623
686
|
|
|
624
|
-
// 3. Run
|
|
687
|
+
// 3. Run search paths in parallel (conditioned on mode)
|
|
688
|
+
const runFts = mode === 'fts' || mode === 'hybrid';
|
|
689
|
+
const runVector = mode === 'vector' || mode === 'hybrid';
|
|
690
|
+
|
|
625
691
|
const [ftsRows, embRows, turnResult] = await Promise.all([
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
692
|
+
runFts
|
|
693
|
+
? storage.searchSessions(pool, query, {
|
|
694
|
+
schema, tenantId, agentIds: resolvedAgentIds, source, dateFrom, dateTo, limit: fetchLimit, ftsConfig,
|
|
695
|
+
}).catch(() => [])
|
|
696
|
+
: Promise.resolve([]),
|
|
697
|
+
runVector
|
|
698
|
+
? embeddingSearchSummaries(queryVec, {
|
|
699
|
+
agentIds: resolvedAgentIds, source, dateFrom, dateTo, limit: fetchLimit,
|
|
700
|
+
}).catch(() => [])
|
|
701
|
+
: Promise.resolve([]),
|
|
702
|
+
runVector
|
|
703
|
+
? storage.searchTurnEmbeddings(pool, {
|
|
704
|
+
schema, tenantId, queryVec, dateFrom, dateTo, agentIds: resolvedAgentIds, source, limit: fetchLimit,
|
|
705
|
+
}).catch(() => ({ rows: [] }))
|
|
706
|
+
: Promise.resolve({ rows: [] }),
|
|
635
707
|
]);
|
|
636
708
|
|
|
637
709
|
const turnRows = turnResult.rows || [];
|
|
@@ -691,15 +763,45 @@ function createAquifer(config) {
|
|
|
691
763
|
[...filteredEmb, ...filterFn(externalRows)],
|
|
692
764
|
filteredTurn,
|
|
693
765
|
{
|
|
694
|
-
limit,
|
|
766
|
+
limit: rerankTopK,
|
|
695
767
|
weights: mergedWeights,
|
|
696
768
|
entityScoreBySession,
|
|
697
769
|
openLoopSet,
|
|
698
770
|
},
|
|
699
771
|
);
|
|
700
772
|
|
|
773
|
+
// 6b. Rerank (optional)
|
|
774
|
+
let finalRanked = ranked;
|
|
775
|
+
if (rerankEnabled && ranked.length > 1) {
|
|
776
|
+
try {
|
|
777
|
+
const docs = ranked.map(r => buildRerankDocument(r, rerankMaxChars));
|
|
778
|
+
const rerankResult = await reranker.rerank(query, docs, { topN: ranked.length });
|
|
779
|
+
const scoreMap = new Map(rerankResult.map(r => [r.index, r.score]));
|
|
780
|
+
|
|
781
|
+
finalRanked = ranked.map((r, i) => ({
|
|
782
|
+
...r,
|
|
783
|
+
_hybridScore: r._score,
|
|
784
|
+
_rerankScore: scoreMap.has(i) ? scoreMap.get(i) : null,
|
|
785
|
+
}));
|
|
786
|
+
|
|
787
|
+
finalRanked.sort((a, b) => {
|
|
788
|
+
const aR = a._rerankScore ?? -Infinity;
|
|
789
|
+
const bR = b._rerankScore ?? -Infinity;
|
|
790
|
+
if (aR !== bR) return bR - aR;
|
|
791
|
+
return (b._hybridScore || 0) - (a._hybridScore || 0);
|
|
792
|
+
});
|
|
793
|
+
finalRanked = finalRanked.slice(0, limit);
|
|
794
|
+
} catch (rerankErr) {
|
|
795
|
+
// Fallback: use original hybrid-rank order, flag in debug
|
|
796
|
+
if (process.env.AQUIFER_DEBUG) console.error('[aquifer] rerank error:', rerankErr.message);
|
|
797
|
+
finalRanked = ranked.slice(0, limit).map(r => ({ ...r, _rerankFallback: true }));
|
|
798
|
+
}
|
|
799
|
+
} else {
|
|
800
|
+
finalRanked = ranked.slice(0, limit);
|
|
801
|
+
}
|
|
802
|
+
|
|
701
803
|
// 7. Record access
|
|
702
|
-
const sessionRowIds =
|
|
804
|
+
const sessionRowIds = finalRanked
|
|
703
805
|
.map(r => r.id || r.session_row_id)
|
|
704
806
|
.filter(Boolean);
|
|
705
807
|
|
|
@@ -710,7 +812,7 @@ function createAquifer(config) {
|
|
|
710
812
|
}
|
|
711
813
|
|
|
712
814
|
// 8. Format results
|
|
713
|
-
return
|
|
815
|
+
return finalRanked.map(r => ({
|
|
714
816
|
sessionId: r.session_id,
|
|
715
817
|
agentId: r.agent_id,
|
|
716
818
|
source: r.source,
|
|
@@ -720,7 +822,7 @@ function createAquifer(config) {
|
|
|
720
822
|
summarySnippet: r.summary_snippet || null,
|
|
721
823
|
matchedTurnText: r.matched_turn_text || null,
|
|
722
824
|
matchedTurnIndex: r.matched_turn_index || null,
|
|
723
|
-
score: r._score,
|
|
825
|
+
score: r._rerankScore ?? r._score,
|
|
724
826
|
trustScore: r._trustScore ?? 0.5,
|
|
725
827
|
_debug: {
|
|
726
828
|
rrf: r._rrf,
|
|
@@ -730,6 +832,9 @@ function createAquifer(config) {
|
|
|
730
832
|
trustScore: r._trustScore,
|
|
731
833
|
trustMultiplier: r._trustMultiplier,
|
|
732
834
|
openLoopBoost: r._openLoopBoost,
|
|
835
|
+
hybridScore: r._hybridScore ?? r._score,
|
|
836
|
+
rerankScore: r._rerankScore ?? null,
|
|
837
|
+
rerankFallback: r._rerankFallback || false,
|
|
733
838
|
},
|
|
734
839
|
}));
|
|
735
840
|
},
|
|
@@ -763,6 +868,27 @@ function createAquifer(config) {
|
|
|
763
868
|
return storage.getSession(pool, sessionId, agentId, opts, { schema, tenantId });
|
|
764
869
|
},
|
|
765
870
|
|
|
871
|
+
async skip(sessionId, opts = {}) {
|
|
872
|
+
const agentId = opts.agentId || 'agent';
|
|
873
|
+
const reason = opts.reason || null;
|
|
874
|
+
// Atomic CAS: only skip if still pending (avoids race with concurrent enrich)
|
|
875
|
+
const result = await pool.query(
|
|
876
|
+
`UPDATE ${qi(schema)}.sessions
|
|
877
|
+
SET processing_status = 'skipped', processing_error = $1
|
|
878
|
+
WHERE session_id = $2 AND agent_id = $3 AND tenant_id = $4
|
|
879
|
+
AND processing_status = 'pending'
|
|
880
|
+
RETURNING id`,
|
|
881
|
+
[reason, sessionId, agentId, tenantId]
|
|
882
|
+
);
|
|
883
|
+
if (result.rows.length === 0) {
|
|
884
|
+
// Check if session exists at all
|
|
885
|
+
const existing = await storage.getSession(pool, sessionId, agentId, {}, { schema, tenantId });
|
|
886
|
+
if (!existing) throw new Error(`Session not found: ${sessionId} (agentId=${agentId})`);
|
|
887
|
+
return null; // exists but not pending — no-op
|
|
888
|
+
}
|
|
889
|
+
return { id: result.rows[0].id, sessionId, agentId, status: 'skipped' };
|
|
890
|
+
},
|
|
891
|
+
|
|
766
892
|
async getSessionFull(sessionId) {
|
|
767
893
|
// Try to find the session across agents by querying directly
|
|
768
894
|
const result = await pool.query(
|
|
@@ -795,6 +921,93 @@ function createAquifer(config) {
|
|
|
795
921
|
summary: sumResult.rows[0] || null,
|
|
796
922
|
};
|
|
797
923
|
},
|
|
924
|
+
|
|
925
|
+
// --- public config accessor ---
|
|
926
|
+
|
|
927
|
+
getConfig() {
|
|
928
|
+
return { schema, tenantId };
|
|
929
|
+
},
|
|
930
|
+
|
|
931
|
+
// --- admin query helpers ---
|
|
932
|
+
|
|
933
|
+
async getStats() {
|
|
934
|
+
const [sessions, summaries, turns, timeRange] = await Promise.all([
|
|
935
|
+
pool.query(
|
|
936
|
+
`SELECT processing_status, COUNT(*)::int as count
|
|
937
|
+
FROM ${qi(schema)}.sessions WHERE tenant_id = $1
|
|
938
|
+
GROUP BY processing_status`,
|
|
939
|
+
[tenantId]
|
|
940
|
+
),
|
|
941
|
+
pool.query(
|
|
942
|
+
`SELECT COUNT(*)::int as count FROM ${qi(schema)}.session_summaries WHERE tenant_id = $1`,
|
|
943
|
+
[tenantId]
|
|
944
|
+
),
|
|
945
|
+
pool.query(
|
|
946
|
+
`SELECT COUNT(*)::int as count FROM ${qi(schema)}.turn_embeddings WHERE tenant_id = $1`,
|
|
947
|
+
[tenantId]
|
|
948
|
+
),
|
|
949
|
+
pool.query(
|
|
950
|
+
`SELECT MIN(started_at) as earliest, MAX(started_at) as latest
|
|
951
|
+
FROM ${qi(schema)}.sessions WHERE tenant_id = $1`,
|
|
952
|
+
[tenantId]
|
|
953
|
+
),
|
|
954
|
+
]);
|
|
955
|
+
|
|
956
|
+
let entityCount = 0;
|
|
957
|
+
try {
|
|
958
|
+
const entResult = await pool.query(
|
|
959
|
+
`SELECT COUNT(*)::int as count FROM ${qi(schema)}.entities WHERE tenant_id = $1`,
|
|
960
|
+
[tenantId]
|
|
961
|
+
);
|
|
962
|
+
entityCount = entResult.rows[0]?.count || 0;
|
|
963
|
+
} catch (_) { /* entities table may not exist */ }
|
|
964
|
+
|
|
965
|
+
return {
|
|
966
|
+
sessions: Object.fromEntries(sessions.rows.map(r => [r.processing_status, r.count])),
|
|
967
|
+
sessionTotal: sessions.rows.reduce((s, r) => s + r.count, 0),
|
|
968
|
+
summaries: summaries.rows[0]?.count || 0,
|
|
969
|
+
turnEmbeddings: turns.rows[0]?.count || 0,
|
|
970
|
+
entities: entityCount,
|
|
971
|
+
earliest: timeRange.rows[0]?.earliest || null,
|
|
972
|
+
latest: timeRange.rows[0]?.latest || null,
|
|
973
|
+
};
|
|
974
|
+
},
|
|
975
|
+
|
|
976
|
+
async getPendingSessions(opts = {}) {
|
|
977
|
+
const limit = opts.limit !== undefined ? opts.limit : 100;
|
|
978
|
+
const result = await pool.query(
|
|
979
|
+
`SELECT session_id, agent_id, processing_status
|
|
980
|
+
FROM ${qi(schema)}.sessions
|
|
981
|
+
WHERE tenant_id = $1
|
|
982
|
+
AND processing_status IN ('pending', 'failed')
|
|
983
|
+
ORDER BY started_at DESC
|
|
984
|
+
LIMIT $2`,
|
|
985
|
+
[tenantId, limit]
|
|
986
|
+
);
|
|
987
|
+
return result.rows;
|
|
988
|
+
},
|
|
989
|
+
|
|
990
|
+
async exportSessions(opts = {}) {
|
|
991
|
+
const { agentId, source, limit = 1000 } = opts;
|
|
992
|
+
const where = [`s.tenant_id = $1`];
|
|
993
|
+
const params = [tenantId];
|
|
994
|
+
|
|
995
|
+
if (agentId) { params.push(agentId); where.push(`s.agent_id = $${params.length}`); }
|
|
996
|
+
if (source) { params.push(source); where.push(`s.source = $${params.length}`); }
|
|
997
|
+
params.push(limit);
|
|
998
|
+
|
|
999
|
+
const result = await pool.query(
|
|
1000
|
+
`SELECT s.session_id, s.agent_id, s.source, s.started_at, s.msg_count,
|
|
1001
|
+
s.processing_status, ss.summary_text, ss.structured_summary
|
|
1002
|
+
FROM ${qi(schema)}.sessions s
|
|
1003
|
+
LEFT JOIN ${qi(schema)}.session_summaries ss ON ss.session_row_id = s.id
|
|
1004
|
+
WHERE ${where.join(' AND ')}
|
|
1005
|
+
ORDER BY s.started_at DESC
|
|
1006
|
+
LIMIT $${params.length}`,
|
|
1007
|
+
params
|
|
1008
|
+
);
|
|
1009
|
+
return result.rows;
|
|
1010
|
+
},
|
|
798
1011
|
};
|
|
799
1012
|
|
|
800
1013
|
return aquifer;
|
package/core/storage.js
CHANGED
|
@@ -31,7 +31,7 @@ const TURN_NOISE_RE = [
|
|
|
31
31
|
/^A new session was started via \/new/,
|
|
32
32
|
];
|
|
33
33
|
|
|
34
|
-
const VALID_STATUSES = new Set(['pending', 'processing', 'succeeded', 'partial', 'failed']);
|
|
34
|
+
const VALID_STATUSES = new Set(['pending', 'processing', 'succeeded', 'partial', 'failed', 'skipped']);
|
|
35
35
|
|
|
36
36
|
// ---------------------------------------------------------------------------
|
|
37
37
|
// upsertSession
|
|
@@ -331,12 +331,55 @@ async function searchSessions(pool, query, {
|
|
|
331
331
|
schema,
|
|
332
332
|
tenantId,
|
|
333
333
|
agentId,
|
|
334
|
+
agentIds: rawAgentIds,
|
|
334
335
|
source,
|
|
335
336
|
dateFrom, // m1: add date filtering
|
|
336
337
|
dateTo,
|
|
337
338
|
limit = 20,
|
|
339
|
+
ftsConfig = 'simple',
|
|
338
340
|
} = {}) {
|
|
339
341
|
const clampedLimit = Math.max(1, Math.min(100, limit));
|
|
342
|
+
// FTS config is locked to 'simple' — the search_tsv trigger always uses
|
|
343
|
+
// to_tsvector('simple', ...) so query semantics must match. Warn callers
|
|
344
|
+
// that pass a different value rather than silently honouring it.
|
|
345
|
+
if (ftsConfig !== 'simple') {
|
|
346
|
+
console.warn(
|
|
347
|
+
`[aquifer/storage] searchSessions: ftsConfig '${ftsConfig}' ignored. ` +
|
|
348
|
+
`Only 'simple' is supported (index is built with simple tokenizer). ` +
|
|
349
|
+
`Using 'simple'.`
|
|
350
|
+
);
|
|
351
|
+
}
|
|
352
|
+
const safeFts = 'simple';
|
|
353
|
+
|
|
354
|
+
// Normalize agentId/agentIds
|
|
355
|
+
const agentIds = rawAgentIds && rawAgentIds.length > 0
|
|
356
|
+
? rawAgentIds
|
|
357
|
+
: (agentId ? [agentId] : null);
|
|
358
|
+
|
|
359
|
+
const where = [
|
|
360
|
+
`ss.search_tsv @@ plainto_tsquery('${safeFts}', $1)`,
|
|
361
|
+
`s.tenant_id = $2`,
|
|
362
|
+
];
|
|
363
|
+
const params = [query, tenantId];
|
|
364
|
+
|
|
365
|
+
if (agentIds) {
|
|
366
|
+
params.push(agentIds);
|
|
367
|
+
where.push(`s.agent_id = ANY($${params.length})`);
|
|
368
|
+
}
|
|
369
|
+
if (source) {
|
|
370
|
+
params.push(source);
|
|
371
|
+
where.push(`s.source = $${params.length}`);
|
|
372
|
+
}
|
|
373
|
+
if (dateFrom) {
|
|
374
|
+
params.push(dateFrom);
|
|
375
|
+
where.push(`s.started_at::date >= $${params.length}::date`);
|
|
376
|
+
}
|
|
377
|
+
if (dateTo) {
|
|
378
|
+
params.push(dateTo);
|
|
379
|
+
where.push(`s.started_at::date <= $${params.length}::date`);
|
|
380
|
+
}
|
|
381
|
+
params.push(clampedLimit);
|
|
382
|
+
|
|
340
383
|
const result = await pool.query(
|
|
341
384
|
`SELECT
|
|
342
385
|
s.id,
|
|
@@ -351,19 +394,14 @@ async function searchSessions(pool, query, {
|
|
|
351
394
|
ss.access_count,
|
|
352
395
|
ss.last_accessed_at,
|
|
353
396
|
ss.trust_score,
|
|
354
|
-
ts_headline('
|
|
355
|
-
ts_rank(ss.search_tsv, plainto_tsquery('
|
|
397
|
+
ts_headline('${safeFts}', COALESCE(ss.summary_text, ''), plainto_tsquery('${safeFts}', $1)) AS summary_snippet,
|
|
398
|
+
ts_rank(ss.search_tsv, plainto_tsquery('${safeFts}', $1)) AS fts_rank
|
|
356
399
|
FROM ${qi(schema)}.sessions s
|
|
357
400
|
LEFT JOIN ${qi(schema)}.session_summaries ss ON ss.session_row_id = s.id
|
|
358
|
-
WHERE
|
|
359
|
-
AND s.tenant_id = $2
|
|
360
|
-
AND ($3::text IS NULL OR s.agent_id = $3)
|
|
361
|
-
AND ($4::text IS NULL OR s.source = $4)
|
|
362
|
-
AND ($5::date IS NULL OR s.started_at::date >= $5::date)
|
|
363
|
-
AND ($6::date IS NULL OR s.started_at::date <= $6::date)
|
|
401
|
+
WHERE ${where.join(' AND ')}
|
|
364
402
|
ORDER BY fts_rank DESC, s.last_message_at DESC NULLS LAST
|
|
365
|
-
LIMIT
|
|
366
|
-
|
|
403
|
+
LIMIT $${params.length}`,
|
|
404
|
+
params
|
|
367
405
|
);
|
|
368
406
|
return result.rows;
|
|
369
407
|
}
|
|
@@ -479,23 +517,29 @@ async function searchTurnEmbeddings(pool, {
|
|
|
479
517
|
dateFrom,
|
|
480
518
|
dateTo,
|
|
481
519
|
agentId,
|
|
520
|
+
agentIds: rawAgentIds,
|
|
482
521
|
source,
|
|
483
522
|
limit = 15,
|
|
484
523
|
}) {
|
|
485
524
|
const where = ['s.tenant_id = $1'];
|
|
486
525
|
const params = [tenantId];
|
|
487
526
|
|
|
527
|
+
// Normalize agentId/agentIds
|
|
528
|
+
const agentIds = rawAgentIds && rawAgentIds.length > 0
|
|
529
|
+
? rawAgentIds
|
|
530
|
+
: (agentId ? [agentId] : null);
|
|
531
|
+
|
|
488
532
|
if (dateFrom) {
|
|
489
533
|
params.push(dateFrom);
|
|
490
|
-
where.push(`
|
|
534
|
+
where.push(`s.started_at::date >= $${params.length}::date`);
|
|
491
535
|
}
|
|
492
536
|
if (dateTo) {
|
|
493
537
|
params.push(dateTo);
|
|
494
|
-
where.push(`
|
|
538
|
+
where.push(`s.started_at::date <= $${params.length}::date`);
|
|
495
539
|
}
|
|
496
|
-
if (
|
|
497
|
-
params.push(
|
|
498
|
-
where.push(`t.agent_id = $${params.length}`);
|
|
540
|
+
if (agentIds) {
|
|
541
|
+
params.push(agentIds);
|
|
542
|
+
where.push(`t.agent_id = ANY($${params.length})`);
|
|
499
543
|
}
|
|
500
544
|
if (source) {
|
|
501
545
|
params.push(source);
|
package/index.js
CHANGED
|
@@ -2,5 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
const { createAquifer } = require('./core/aquifer');
|
|
4
4
|
const { createEmbedder } = require('./pipeline/embed');
|
|
5
|
+
const { createReranker } = require('./pipeline/rerank');
|
|
6
|
+
const { normalizeSession, detectClient } = require('./pipeline/normalize');
|
|
5
7
|
|
|
6
|
-
module.exports = { createAquifer, createEmbedder };
|
|
8
|
+
module.exports = { createAquifer, createEmbedder, createReranker, normalizeSession, detectClient };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@shadowforge0/aquifer-memory",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.8.0",
|
|
4
4
|
"description": "PG-native long-term memory for AI agents. Turn-level embedding, hybrid RRF ranking, optional knowledge graph. Includes CLI, MCP server, and OpenClaw plugin.",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"files": [
|
|
@@ -35,8 +35,8 @@
|
|
|
35
35
|
"pg": "^8.13.0"
|
|
36
36
|
},
|
|
37
37
|
"optionalDependencies": {
|
|
38
|
-
"@modelcontextprotocol/sdk": "^1.
|
|
39
|
-
"zod": "^3.
|
|
38
|
+
"@modelcontextprotocol/sdk": "^1.29.0",
|
|
39
|
+
"zod": "^3.25.76"
|
|
40
40
|
},
|
|
41
41
|
"engines": {
|
|
42
42
|
"node": ">=18.0.0"
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const http = require('http');
|
|
4
|
+
const https = require('https');
|
|
5
|
+
|
|
6
|
+
// ---------------------------------------------------------------------------
|
|
7
|
+
// HTTP helpers (shared by embed.js and rerank.js)
|
|
8
|
+
// ---------------------------------------------------------------------------
|
|
9
|
+
|
|
10
|
+
function httpRequest(url, options, body) {
|
|
11
|
+
return new Promise((resolve, reject) => {
|
|
12
|
+
const parsedUrl = new URL(url);
|
|
13
|
+
const transport = parsedUrl.protocol === 'https:' ? https : http;
|
|
14
|
+
|
|
15
|
+
// M8 fix: settled flag to prevent double-settle on timeout race
|
|
16
|
+
let settled = false;
|
|
17
|
+
const finish = (fn, val) => { if (!settled) { settled = true; fn(val); } };
|
|
18
|
+
|
|
19
|
+
const req = transport.request(parsedUrl, options, (res) => {
|
|
20
|
+
const chunks = [];
|
|
21
|
+
res.on('data', (chunk) => chunks.push(chunk));
|
|
22
|
+
res.on('end', () => {
|
|
23
|
+
if (timer) clearTimeout(timer);
|
|
24
|
+
const raw = Buffer.concat(chunks).toString();
|
|
25
|
+
if (res.statusCode < 200 || res.statusCode >= 300) {
|
|
26
|
+
finish(reject, new Error(`HTTP ${res.statusCode}: ${raw.slice(0, 500)}`));
|
|
27
|
+
return;
|
|
28
|
+
}
|
|
29
|
+
try {
|
|
30
|
+
finish(resolve, JSON.parse(raw));
|
|
31
|
+
} catch (e) {
|
|
32
|
+
finish(reject, new Error(`Invalid JSON response: ${raw.slice(0, 200)}`));
|
|
33
|
+
}
|
|
34
|
+
});
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
const timer = options.timeout
|
|
38
|
+
? setTimeout(() => { req.destroy(); finish(reject, new Error('Request timeout')); }, options.timeout)
|
|
39
|
+
: null;
|
|
40
|
+
|
|
41
|
+
req.on('error', (err) => { if (timer) clearTimeout(timer); finish(reject, err); });
|
|
42
|
+
if (body) req.write(JSON.stringify(body));
|
|
43
|
+
req.end();
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
// ---------------------------------------------------------------------------
|
|
48
|
+
// Retry wrapper
|
|
49
|
+
// ---------------------------------------------------------------------------
|
|
50
|
+
|
|
51
|
+
async function withRetry(fn, { maxRetries = 3, initialBackoffMs = 2000 } = {}) {
|
|
52
|
+
let lastErr;
|
|
53
|
+
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
54
|
+
try {
|
|
55
|
+
return await fn();
|
|
56
|
+
} catch (err) {
|
|
57
|
+
lastErr = err;
|
|
58
|
+
if (attempt < maxRetries - 1) {
|
|
59
|
+
const delay = initialBackoffMs * Math.pow(2, attempt);
|
|
60
|
+
await new Promise(r => setTimeout(r, delay));
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
throw lastErr;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
module.exports = { httpRequest, withRetry };
|
package/pipeline/embed.js
CHANGED
|
@@ -1,68 +1,6 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
|
-
const
|
|
4
|
-
const https = require('https');
|
|
5
|
-
|
|
6
|
-
// ---------------------------------------------------------------------------
|
|
7
|
-
// HTTP helpers
|
|
8
|
-
// ---------------------------------------------------------------------------
|
|
9
|
-
|
|
10
|
-
function httpRequest(url, options, body) {
|
|
11
|
-
return new Promise((resolve, reject) => {
|
|
12
|
-
const parsedUrl = new URL(url);
|
|
13
|
-
const transport = parsedUrl.protocol === 'https:' ? https : http;
|
|
14
|
-
|
|
15
|
-
// M8 fix: settled flag to prevent double-settle on timeout race
|
|
16
|
-
let settled = false;
|
|
17
|
-
const finish = (fn, val) => { if (!settled) { settled = true; fn(val); } };
|
|
18
|
-
|
|
19
|
-
const req = transport.request(parsedUrl, options, (res) => {
|
|
20
|
-
const chunks = [];
|
|
21
|
-
res.on('data', (chunk) => chunks.push(chunk));
|
|
22
|
-
res.on('end', () => {
|
|
23
|
-
if (timer) clearTimeout(timer);
|
|
24
|
-
const raw = Buffer.concat(chunks).toString();
|
|
25
|
-
if (res.statusCode < 200 || res.statusCode >= 300) {
|
|
26
|
-
finish(reject, new Error(`HTTP ${res.statusCode}: ${raw.slice(0, 500)}`));
|
|
27
|
-
return;
|
|
28
|
-
}
|
|
29
|
-
try {
|
|
30
|
-
finish(resolve, JSON.parse(raw));
|
|
31
|
-
} catch (e) {
|
|
32
|
-
finish(reject, new Error(`Invalid JSON response: ${raw.slice(0, 200)}`));
|
|
33
|
-
}
|
|
34
|
-
});
|
|
35
|
-
});
|
|
36
|
-
|
|
37
|
-
const timer = options.timeout
|
|
38
|
-
? setTimeout(() => { req.destroy(); finish(reject, new Error('Request timeout')); }, options.timeout)
|
|
39
|
-
: null;
|
|
40
|
-
|
|
41
|
-
req.on('error', (err) => { if (timer) clearTimeout(timer); finish(reject, err); });
|
|
42
|
-
if (body) req.write(JSON.stringify(body));
|
|
43
|
-
req.end();
|
|
44
|
-
});
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
// ---------------------------------------------------------------------------
|
|
48
|
-
// Retry wrapper
|
|
49
|
-
// ---------------------------------------------------------------------------
|
|
50
|
-
|
|
51
|
-
async function withRetry(fn, { maxRetries = 3, initialBackoffMs = 2000 } = {}) {
|
|
52
|
-
let lastErr;
|
|
53
|
-
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
54
|
-
try {
|
|
55
|
-
return await fn();
|
|
56
|
-
} catch (err) {
|
|
57
|
-
lastErr = err;
|
|
58
|
-
if (attempt < maxRetries - 1) {
|
|
59
|
-
const delay = initialBackoffMs * Math.pow(2, attempt);
|
|
60
|
-
await new Promise(r => setTimeout(r, delay));
|
|
61
|
-
}
|
|
62
|
-
}
|
|
63
|
-
}
|
|
64
|
-
throw lastErr;
|
|
65
|
-
}
|
|
3
|
+
const { httpRequest, withRetry } = require('./_http');
|
|
66
4
|
|
|
67
5
|
// ---------------------------------------------------------------------------
|
|
68
6
|
// Ollama adapter
|