@hanhnd/agent-kit 1.0.38 → 1.0.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +286 -140
- package/dist/entrypoints/server.js +3 -3
- package/dist/mcp/memory.d.ts +4 -9
- package/dist/mcp/memory.js +43 -19
- package/dist/server.js +445 -212
- package/dist/services/digest/digest-processor.test.js +1 -3
- package/dist/services/memory/index.d.ts +3 -1
- package/dist/services/memory/index.js +1 -0
- package/dist/services/memory/indexer.d.ts +5 -6
- package/dist/services/memory/indexer.js +71 -101
- package/dist/services/memory/indexer.test.js +347 -572
- package/dist/services/memory/memory.test.js +58 -0
- package/dist/services/memory/store.d.ts +4 -11
- package/dist/services/memory/store.js +200 -66
- package/dist/services/memory/store.test.js +359 -108
- package/dist/services/memory/subsystem.d.ts +19 -0
- package/dist/services/memory/subsystem.js +67 -0
- package/dist/services/memory/types.d.ts +48 -0
- package/dist/utils/async.d.ts +1 -0
- package/dist/utils/async.js +17 -0
- package/dist/utils/async.test.d.ts +1 -0
- package/dist/utils/async.test.js +48 -0
- package/package.json +1 -1
package/dist/cli.js
CHANGED
|
@@ -15195,6 +15195,22 @@ async function withTimeout(promise, timeoutMs, createError) {
|
|
|
15195
15195
|
if (timer) clearTimeout(timer);
|
|
15196
15196
|
}
|
|
15197
15197
|
}
|
|
15198
|
+
async function mapLimit(items, limit, worker) {
|
|
15199
|
+
if (items.length === 0) return [];
|
|
15200
|
+
const concurrency = Math.max(1, Math.floor(limit));
|
|
15201
|
+
const results = new Array(items.length);
|
|
15202
|
+
let nextIndex = 0;
|
|
15203
|
+
async function runWorker() {
|
|
15204
|
+
while (nextIndex < items.length) {
|
|
15205
|
+
const currentIndex = nextIndex;
|
|
15206
|
+
nextIndex++;
|
|
15207
|
+
results[currentIndex] = await worker(items[currentIndex], currentIndex);
|
|
15208
|
+
}
|
|
15209
|
+
}
|
|
15210
|
+
const runners = Array.from({ length: Math.min(concurrency, items.length) }, () => runWorker());
|
|
15211
|
+
await Promise.all(runners);
|
|
15212
|
+
return results;
|
|
15213
|
+
}
|
|
15198
15214
|
|
|
15199
15215
|
// src/utils/paths.ts
|
|
15200
15216
|
import * as os2 from "os";
|
|
@@ -15504,10 +15520,6 @@ var MemoryStore = class {
|
|
|
15504
15520
|
}
|
|
15505
15521
|
return Buffer.from(embedding.buffer, embedding.byteOffset, embedding.byteLength);
|
|
15506
15522
|
}
|
|
15507
|
-
normalizeVectorDistance(distance) {
|
|
15508
|
-
if (distance <= 0) return 1;
|
|
15509
|
-
return Math.max(0, Math.min(1, 1 - distance));
|
|
15510
|
-
}
|
|
15511
15523
|
hashesBySource(source) {
|
|
15512
15524
|
const rows = this.db.prepare(`SELECT id FROM memory_chunks WHERE source = ?`).all(source);
|
|
15513
15525
|
return new Set(rows.map((r) => r.id));
|
|
@@ -15544,61 +15556,12 @@ var MemoryStore = class {
|
|
|
15544
15556
|
});
|
|
15545
15557
|
deleteAll();
|
|
15546
15558
|
}
|
|
15547
|
-
|
|
15548
|
-
try {
|
|
15549
|
-
const queryVector = this.toVectorBinding(embedding);
|
|
15550
|
-
const rows = this.db.prepare(
|
|
15551
|
-
`SELECT mc.id, vector_distance_cos(mc.embedding, vector32(?)) AS distance
|
|
15552
|
-
FROM vector_top_k('idx_memory_chunks_embedding', vector32(?), ?) AS vector_matches
|
|
15553
|
-
JOIN memory_chunks mc ON mc.rowid = vector_matches.id
|
|
15554
|
-
ORDER BY distance
|
|
15555
|
-
LIMIT ?`
|
|
15556
|
-
).all(queryVector, queryVector, limit, limit);
|
|
15557
|
-
return rows.map((r) => ({
|
|
15558
|
-
id: r.id,
|
|
15559
|
-
score: this.normalizeVectorDistance(r.distance)
|
|
15560
|
-
}));
|
|
15561
|
-
} catch (err) {
|
|
15562
|
-
console.warn("[memory-store] Dense search failed:", err);
|
|
15563
|
-
return [];
|
|
15564
|
-
}
|
|
15565
|
-
}
|
|
15566
|
-
searchBm25(query, limit) {
|
|
15567
|
-
if (!query.trim()) return [];
|
|
15559
|
+
buildFtsQuery(query) {
|
|
15568
15560
|
const tokens = query.trim().split(/\s+/).map((t) => t.replace(/[^a-zA-Z0-9_]/g, "").toLowerCase()).filter((t) => t.length > 1);
|
|
15569
|
-
|
|
15570
|
-
if (!ftsQuery) return [];
|
|
15571
|
-
try {
|
|
15572
|
-
const rows = this.db.prepare(
|
|
15573
|
-
`SELECT mc.id, bm25(memory_fts, 0.2, 2.0, 5.0) AS rank
|
|
15574
|
-
FROM memory_fts
|
|
15575
|
-
JOIN memory_chunks mc ON mc.rowid = memory_fts.rowid
|
|
15576
|
-
WHERE memory_fts MATCH ?
|
|
15577
|
-
ORDER BY rank
|
|
15578
|
-
LIMIT ?`
|
|
15579
|
-
).all(ftsQuery, limit);
|
|
15580
|
-
if (rows.length === 0) return [];
|
|
15581
|
-
const ranks = rows.map((r) => r.rank);
|
|
15582
|
-
const minRank = Math.min(...ranks);
|
|
15583
|
-
const maxRank = Math.max(...ranks);
|
|
15584
|
-
const range = maxRank - minRank;
|
|
15585
|
-
return rows.map((r) => ({
|
|
15586
|
-
id: r.id,
|
|
15587
|
-
score: range > 0 ? (maxRank - r.rank) / range : 1
|
|
15588
|
-
}));
|
|
15589
|
-
} catch (err) {
|
|
15590
|
-
console.warn("[memory-store] BM25 search failed:", err);
|
|
15591
|
-
return [];
|
|
15592
|
-
}
|
|
15561
|
+
return (0, import_stopword.removeStopwords)(tokens, import_stopword.eng).map((t) => `${t.replace(/s$/, "")}*`).join(" OR ");
|
|
15593
15562
|
}
|
|
15594
|
-
|
|
15595
|
-
|
|
15596
|
-
const placeholders = ids.map(() => "?").join(", ");
|
|
15597
|
-
const rows = this.db.prepare(
|
|
15598
|
-
`SELECT id, source, source_type, heading, heading_level, content, line_start, line_end, file_mtime_at
|
|
15599
|
-
FROM memory_chunks WHERE id IN (${placeholders})`
|
|
15600
|
-
).all(...ids);
|
|
15601
|
-
return rows.map((r) => ({
|
|
15563
|
+
mapMemoryChunkRow(r) {
|
|
15564
|
+
return {
|
|
15602
15565
|
id: r.id,
|
|
15603
15566
|
source: r.source,
|
|
15604
15567
|
sourceType: r.source_type,
|
|
@@ -15608,7 +15571,201 @@ var MemoryStore = class {
|
|
|
15608
15571
|
lineStart: r.line_start,
|
|
15609
15572
|
lineEnd: r.line_end,
|
|
15610
15573
|
fileMtimeAt: r.file_mtime_at
|
|
15611
|
-
}
|
|
15574
|
+
};
|
|
15575
|
+
}
|
|
15576
|
+
searchHybrid(options) {
|
|
15577
|
+
const {
|
|
15578
|
+
query,
|
|
15579
|
+
embedding,
|
|
15580
|
+
topK,
|
|
15581
|
+
fetchLimit,
|
|
15582
|
+
denseScoreFloor,
|
|
15583
|
+
recencyWeight,
|
|
15584
|
+
rrfK,
|
|
15585
|
+
sourceType,
|
|
15586
|
+
sinceMtimeAt,
|
|
15587
|
+
includeDebug
|
|
15588
|
+
} = options;
|
|
15589
|
+
const ftsQuery = this.buildFtsQuery(query);
|
|
15590
|
+
const hasDense = embedding !== void 0;
|
|
15591
|
+
const hasBm25 = ftsQuery.length > 0;
|
|
15592
|
+
if (!hasDense && !hasBm25) return [];
|
|
15593
|
+
const hasFilters = sourceType !== void 0 || sinceMtimeAt !== void 0;
|
|
15594
|
+
try {
|
|
15595
|
+
const params = [];
|
|
15596
|
+
let denseCte = "";
|
|
15597
|
+
let denseRankedCte = "";
|
|
15598
|
+
if (hasDense) {
|
|
15599
|
+
const queryBuf = this.toVectorBinding(embedding);
|
|
15600
|
+
if (hasFilters) {
|
|
15601
|
+
const filterClauses = [];
|
|
15602
|
+
if (sourceType) {
|
|
15603
|
+
filterClauses.push("mc.source_type = ?");
|
|
15604
|
+
params.push(sourceType);
|
|
15605
|
+
}
|
|
15606
|
+
if (sinceMtimeAt !== void 0) {
|
|
15607
|
+
filterClauses.push("mc.file_mtime_at >= ?");
|
|
15608
|
+
params.push(sinceMtimeAt);
|
|
15609
|
+
}
|
|
15610
|
+
const whereClause = filterClauses.length > 0 ? `WHERE ${filterClauses.join(" AND ")}` : "";
|
|
15611
|
+
params.push(queryBuf);
|
|
15612
|
+
params.push(fetchLimit);
|
|
15613
|
+
denseCte = `dense_data AS (
|
|
15614
|
+
SELECT mc.id, mc.source, mc.source_type, mc.file_mtime_at,
|
|
15615
|
+
mc.heading, mc.heading_level, mc.content, mc.line_start, mc.line_end,
|
|
15616
|
+
vector_distance_cos(mc.embedding, vector32(?)) AS dense_dist
|
|
15617
|
+
FROM memory_chunks mc
|
|
15618
|
+
${whereClause}
|
|
15619
|
+
ORDER BY dense_dist
|
|
15620
|
+
LIMIT ?
|
|
15621
|
+
)`;
|
|
15622
|
+
} else {
|
|
15623
|
+
params.push(queryBuf);
|
|
15624
|
+
params.push(queryBuf);
|
|
15625
|
+
params.push(fetchLimit);
|
|
15626
|
+
denseCte = `dense_data AS (
|
|
15627
|
+
SELECT mc.id, mc.source, mc.source_type, mc.file_mtime_at,
|
|
15628
|
+
mc.heading, mc.heading_level, mc.content, mc.line_start, mc.line_end,
|
|
15629
|
+
vector_distance_cos(mc.embedding, vector32(?)) AS dense_dist
|
|
15630
|
+
FROM vector_top_k('idx_memory_chunks_embedding', vector32(?), ?) AS vtk
|
|
15631
|
+
JOIN memory_chunks mc ON mc.rowid = vtk.id
|
|
15632
|
+
)`;
|
|
15633
|
+
}
|
|
15634
|
+
denseRankedCte = `dense_ranked AS (
|
|
15635
|
+
SELECT *, ROW_NUMBER() OVER (ORDER BY dense_dist, id) AS dense_rank
|
|
15636
|
+
FROM dense_data
|
|
15637
|
+
)`;
|
|
15638
|
+
}
|
|
15639
|
+
let bm25Cte = "";
|
|
15640
|
+
let bm25RankedCte = "";
|
|
15641
|
+
if (hasBm25) {
|
|
15642
|
+
const bm25FilterClauses = [];
|
|
15643
|
+
if (sourceType) bm25FilterClauses.push("mc.source_type = ?");
|
|
15644
|
+
if (sinceMtimeAt !== void 0) bm25FilterClauses.push("mc.file_mtime_at >= ?");
|
|
15645
|
+
const bm25WhereExtra = bm25FilterClauses.length > 0 ? ` AND ${bm25FilterClauses.join(" AND ")}` : "";
|
|
15646
|
+
params.push(ftsQuery);
|
|
15647
|
+
if (sourceType) params.push(sourceType);
|
|
15648
|
+
if (sinceMtimeAt !== void 0) params.push(sinceMtimeAt);
|
|
15649
|
+
params.push(fetchLimit);
|
|
15650
|
+
bm25Cte = `bm25_data AS (
|
|
15651
|
+
SELECT mc.id, mc.source, mc.source_type, mc.file_mtime_at,
|
|
15652
|
+
mc.heading, mc.heading_level, mc.content, mc.line_start, mc.line_end,
|
|
15653
|
+
bm25(memory_fts, 0.2, 2.0, 5.0) AS bm25_neg
|
|
15654
|
+
FROM memory_fts
|
|
15655
|
+
JOIN memory_chunks mc ON mc.rowid = memory_fts.rowid
|
|
15656
|
+
WHERE memory_fts MATCH ?${bm25WhereExtra}
|
|
15657
|
+
LIMIT ?
|
|
15658
|
+
)`;
|
|
15659
|
+
bm25RankedCte = `bm25_ranked AS (
|
|
15660
|
+
SELECT *, ROW_NUMBER() OVER (ORDER BY bm25_neg, id) AS bm25_rank
|
|
15661
|
+
FROM bm25_data
|
|
15662
|
+
)`;
|
|
15663
|
+
}
|
|
15664
|
+
let candidatesCte = "";
|
|
15665
|
+
if (hasDense && hasBm25) {
|
|
15666
|
+
candidatesCte = `all_candidates AS (
|
|
15667
|
+
SELECT d.id, d.source, d.source_type, d.file_mtime_at,
|
|
15668
|
+
d.heading, d.heading_level, d.content, d.line_start, d.line_end,
|
|
15669
|
+
CAST(d.dense_rank AS INTEGER) AS dense_rank, d.dense_dist,
|
|
15670
|
+
b.bm25_rank,
|
|
15671
|
+
CASE WHEN b.id IS NOT NULL THEN 'both' ELSE 'dense' END AS retriever
|
|
15672
|
+
FROM dense_ranked d
|
|
15673
|
+
LEFT JOIN bm25_ranked b ON b.id = d.id
|
|
15674
|
+
UNION ALL
|
|
15675
|
+
SELECT b.id, b.source, b.source_type, b.file_mtime_at,
|
|
15676
|
+
b.heading, b.heading_level, b.content, b.line_start, b.line_end,
|
|
15677
|
+
NULL AS dense_rank, NULL AS dense_dist,
|
|
15678
|
+
CAST(b.bm25_rank AS INTEGER) AS bm25_rank,
|
|
15679
|
+
'bm25' AS retriever
|
|
15680
|
+
FROM bm25_ranked b
|
|
15681
|
+
LEFT JOIN dense_ranked d ON d.id = b.id
|
|
15682
|
+
WHERE d.id IS NULL
|
|
15683
|
+
)`;
|
|
15684
|
+
} else if (hasDense) {
|
|
15685
|
+
candidatesCte = `all_candidates AS (
|
|
15686
|
+
SELECT id, source, source_type, file_mtime_at,
|
|
15687
|
+
heading, heading_level, content, line_start, line_end,
|
|
15688
|
+
CAST(dense_rank AS INTEGER) AS dense_rank, dense_dist,
|
|
15689
|
+
NULL AS bm25_rank, 'dense' AS retriever
|
|
15690
|
+
FROM dense_ranked
|
|
15691
|
+
)`;
|
|
15692
|
+
} else {
|
|
15693
|
+
candidatesCte = `all_candidates AS (
|
|
15694
|
+
SELECT id, source, source_type, file_mtime_at,
|
|
15695
|
+
heading, heading_level, content, line_start, line_end,
|
|
15696
|
+
NULL AS dense_rank, NULL AS dense_dist,
|
|
15697
|
+
CAST(bm25_rank AS INTEGER) AS bm25_rank, 'bm25' AS retriever
|
|
15698
|
+
FROM bm25_ranked
|
|
15699
|
+
)`;
|
|
15700
|
+
}
|
|
15701
|
+
params.push(rrfK, rrfK, recencyWeight, rrfK, denseScoreFloor, topK);
|
|
15702
|
+
const sql = `
|
|
15703
|
+
WITH
|
|
15704
|
+
${[denseCte, denseRankedCte, bm25Cte, bm25RankedCte, candidatesCte].filter(Boolean).join(",\n")}
|
|
15705
|
+
,
|
|
15706
|
+
recency_ranked AS (
|
|
15707
|
+
SELECT id, ROW_NUMBER() OVER (ORDER BY file_mtime_at DESC, source, id) AS recency_rank
|
|
15708
|
+
FROM all_candidates
|
|
15709
|
+
),
|
|
15710
|
+
scored AS (
|
|
15711
|
+
SELECT c.id, c.source, c.source_type, c.file_mtime_at,
|
|
15712
|
+
c.heading, c.heading_level, c.content, c.line_start, c.line_end,
|
|
15713
|
+
c.dense_rank, c.dense_dist, c.bm25_rank, r.recency_rank,
|
|
15714
|
+
c.retriever,
|
|
15715
|
+
COALESCE(1.0 / (? + c.dense_rank), 0.0) +
|
|
15716
|
+
COALESCE(1.0 / (? + c.bm25_rank), 0.0) +
|
|
15717
|
+
? * (1.0 / (? + r.recency_rank)) AS total_score
|
|
15718
|
+
FROM all_candidates c
|
|
15719
|
+
JOIN recency_ranked r ON r.id = c.id
|
|
15720
|
+
WHERE c.retriever != 'dense' OR (1.0 - c.dense_dist) >= ?
|
|
15721
|
+
),
|
|
15722
|
+
best_per_source AS (
|
|
15723
|
+
SELECT *,
|
|
15724
|
+
ROW_NUMBER() OVER (PARTITION BY source ORDER BY total_score DESC, source, id) AS source_rank
|
|
15725
|
+
FROM scored
|
|
15726
|
+
)
|
|
15727
|
+
SELECT id, source, source_type, file_mtime_at, heading, heading_level, content, line_start, line_end,
|
|
15728
|
+
dense_rank, dense_dist, bm25_rank, recency_rank, total_score, retriever
|
|
15729
|
+
FROM best_per_source
|
|
15730
|
+
WHERE source_rank = 1
|
|
15731
|
+
ORDER BY total_score DESC, source, id
|
|
15732
|
+
LIMIT ?
|
|
15733
|
+
`;
|
|
15734
|
+
const rows = this.db.prepare(sql).all(...params);
|
|
15735
|
+
if (rows.length === 0) return [];
|
|
15736
|
+
const denseActive = hasDense;
|
|
15737
|
+
const bm25Active = hasBm25;
|
|
15738
|
+
const recencyActive = rows.length > 0;
|
|
15739
|
+
const maxScore = ((denseActive ? 1 : 0) + (bm25Active ? 1 : 0) + (recencyActive ? recencyWeight : 0)) / (rrfK + 1);
|
|
15740
|
+
return rows.map((r) => {
|
|
15741
|
+
const chunk = this.mapMemoryChunkRow(r);
|
|
15742
|
+
const normalizedScore = maxScore > 0 ? r.total_score / maxScore : 0;
|
|
15743
|
+
const row = {
|
|
15744
|
+
chunk,
|
|
15745
|
+
score: normalizedScore,
|
|
15746
|
+
retriever: r.retriever
|
|
15747
|
+
};
|
|
15748
|
+
if (includeDebug) {
|
|
15749
|
+
const denseScore = r.dense_dist !== null ? Math.max(0, Math.min(1, 1 - r.dense_dist)) : void 0;
|
|
15750
|
+
const bm25Score = r.bm25_rank !== null ? 1 / (rrfK + r.bm25_rank) : void 0;
|
|
15751
|
+
const recencyScore = recencyWeight * (1 / (rrfK + r.recency_rank));
|
|
15752
|
+
const debug = {
|
|
15753
|
+
denseRank: r.dense_rank ?? void 0,
|
|
15754
|
+
denseScore,
|
|
15755
|
+
bm25Rank: r.bm25_rank ?? void 0,
|
|
15756
|
+
bm25Score,
|
|
15757
|
+
recencyRank: r.recency_rank,
|
|
15758
|
+
recencyScore,
|
|
15759
|
+
totalScore: r.total_score
|
|
15760
|
+
};
|
|
15761
|
+
row.debug = debug;
|
|
15762
|
+
}
|
|
15763
|
+
return row;
|
|
15764
|
+
});
|
|
15765
|
+
} catch (err) {
|
|
15766
|
+
console.warn("[memory-store] Hybrid search failed:", err);
|
|
15767
|
+
return [];
|
|
15768
|
+
}
|
|
15612
15769
|
}
|
|
15613
15770
|
get vecAvailable() {
|
|
15614
15771
|
return true;
|
|
@@ -15771,21 +15928,31 @@ var MemoryIndexer = class {
|
|
|
15771
15928
|
}
|
|
15772
15929
|
_ready = false;
|
|
15773
15930
|
async indexFile(absolutePath) {
|
|
15774
|
-
|
|
15931
|
+
const mutation = await this.prepareIndexFileRelativeTo(absolutePath, this.config.wikiDir);
|
|
15932
|
+
return this.commitPreparedMutation(mutation);
|
|
15933
|
+
}
|
|
15934
|
+
makeSkippedMutation(source, skipped) {
|
|
15935
|
+
return {
|
|
15936
|
+
source,
|
|
15937
|
+
chunksToUpsert: [],
|
|
15938
|
+
embeddings: [],
|
|
15939
|
+
idsToDelete: [],
|
|
15940
|
+
stats: { indexed: 0, deleted: 0, skipped }
|
|
15941
|
+
};
|
|
15775
15942
|
}
|
|
15776
|
-
async
|
|
15943
|
+
async prepareIndexFileRelativeTo(absolutePath, sourceRoot) {
|
|
15777
15944
|
const source = path8.relative(sourceRoot, absolutePath);
|
|
15778
15945
|
const existingChunkIds = this.store.hashesBySource(source);
|
|
15779
15946
|
let text;
|
|
15780
15947
|
try {
|
|
15781
|
-
text = fs6.
|
|
15948
|
+
text = await fs6.promises.readFile(absolutePath, "utf8");
|
|
15782
15949
|
} catch (err) {
|
|
15783
15950
|
console.warn(`[memory-indexer] Cannot read file ${absolutePath}:`, err);
|
|
15784
|
-
return
|
|
15951
|
+
return this.makeSkippedMutation(source, existingChunkIds.size);
|
|
15785
15952
|
}
|
|
15786
15953
|
let fileMtimeAt;
|
|
15787
15954
|
try {
|
|
15788
|
-
fileMtimeAt = fs6.
|
|
15955
|
+
fileMtimeAt = (await fs6.promises.stat(absolutePath)).mtimeMs;
|
|
15789
15956
|
} catch (err) {
|
|
15790
15957
|
console.warn(`[memory-indexer] Cannot stat file ${absolutePath}:`, err);
|
|
15791
15958
|
fileMtimeAt = Date.now();
|
|
@@ -15801,39 +15968,56 @@ var MemoryIndexer = class {
|
|
|
15801
15968
|
embeddings = await this.embedder.embed(toIndex.map((c) => c.content));
|
|
15802
15969
|
} catch (err) {
|
|
15803
15970
|
console.warn(`[memory-indexer] Embedding failed for ${absolutePath}:`, err);
|
|
15804
|
-
return
|
|
15971
|
+
return this.makeSkippedMutation(source, existingChunkIds.size);
|
|
15805
15972
|
}
|
|
15806
15973
|
}
|
|
15807
|
-
if (toIndex.length > 0) {
|
|
15808
|
-
this.store.upsert(toIndex, embeddings);
|
|
15809
|
-
}
|
|
15810
|
-
if (toDelete.length > 0) {
|
|
15811
|
-
this.store.deleteByIds(toDelete);
|
|
15812
|
-
}
|
|
15813
15974
|
return {
|
|
15814
|
-
|
|
15815
|
-
|
|
15816
|
-
|
|
15975
|
+
source,
|
|
15976
|
+
chunksToUpsert: toIndex,
|
|
15977
|
+
embeddings,
|
|
15978
|
+
idsToDelete: toDelete,
|
|
15979
|
+
stats: {
|
|
15980
|
+
indexed: toIndex.length,
|
|
15981
|
+
deleted: toDelete.length,
|
|
15982
|
+
skipped: allChunks.length - toIndex.length
|
|
15983
|
+
}
|
|
15817
15984
|
};
|
|
15818
15985
|
}
|
|
15986
|
+
commitPreparedMutation(mutation) {
|
|
15987
|
+
if (mutation.chunksToUpsert.length > 0) {
|
|
15988
|
+
this.store.upsert(mutation.chunksToUpsert, mutation.embeddings);
|
|
15989
|
+
}
|
|
15990
|
+
if (mutation.idsToDelete.length > 0) {
|
|
15991
|
+
this.store.deleteByIds(mutation.idsToDelete);
|
|
15992
|
+
}
|
|
15993
|
+
return mutation.stats;
|
|
15994
|
+
}
|
|
15819
15995
|
async indexDirectory(rootDir, opts = {}) {
|
|
15820
15996
|
const totals = { indexed: 0, deleted: 0, skipped: 0 };
|
|
15821
|
-
|
|
15997
|
+
let rootExists;
|
|
15998
|
+
try {
|
|
15999
|
+
await fs6.promises.stat(rootDir);
|
|
16000
|
+
rootExists = true;
|
|
16001
|
+
} catch {
|
|
16002
|
+
rootExists = false;
|
|
16003
|
+
}
|
|
16004
|
+
if (!rootExists) return totals;
|
|
15822
16005
|
const relativeBase = opts.relativeBase ?? this.config.wikiDir;
|
|
15823
16006
|
const excludeFiles = new Set(opts.excludeFiles ?? []);
|
|
15824
16007
|
const files = [];
|
|
15825
|
-
const walk = (dirPath) => {
|
|
16008
|
+
const walk = async (dirPath) => {
|
|
15826
16009
|
let entries;
|
|
15827
16010
|
try {
|
|
15828
|
-
entries = fs6.
|
|
16011
|
+
entries = await fs6.promises.readdir(dirPath, { withFileTypes: true });
|
|
15829
16012
|
} catch (err) {
|
|
15830
16013
|
console.warn(`[memory-indexer] Cannot scan directory ${dirPath}:`, err);
|
|
15831
16014
|
return;
|
|
15832
16015
|
}
|
|
16016
|
+
entries.sort((a, b) => a.name.localeCompare(b.name));
|
|
15833
16017
|
for (const entry of entries) {
|
|
15834
16018
|
const entryPath = path8.join(dirPath, entry.name);
|
|
15835
16019
|
if (entry.isDirectory()) {
|
|
15836
|
-
walk(entryPath);
|
|
16020
|
+
await walk(entryPath);
|
|
15837
16021
|
continue;
|
|
15838
16022
|
}
|
|
15839
16023
|
if (entry.isFile() && /\.md$/i.test(entry.name) && !excludeFiles.has(entry.name)) {
|
|
@@ -15841,7 +16025,7 @@ var MemoryIndexer = class {
|
|
|
15841
16025
|
}
|
|
15842
16026
|
}
|
|
15843
16027
|
};
|
|
15844
|
-
walk(rootDir);
|
|
16028
|
+
await walk(rootDir);
|
|
15845
16029
|
const currentSources = new Set(files.map((file) => path8.relative(relativeBase, file)));
|
|
15846
16030
|
for (const stale of this.store.indexedSources()) {
|
|
15847
16031
|
if (!currentSources.has(stale)) {
|
|
@@ -15849,8 +16033,13 @@ var MemoryIndexer = class {
|
|
|
15849
16033
|
totals.deleted += 1;
|
|
15850
16034
|
}
|
|
15851
16035
|
}
|
|
15852
|
-
|
|
15853
|
-
|
|
16036
|
+
const preparedMutations = await mapLimit(
|
|
16037
|
+
files,
|
|
16038
|
+
opts.fileConcurrency ?? 2,
|
|
16039
|
+
(file) => this.prepareIndexFileRelativeTo(file, relativeBase)
|
|
16040
|
+
);
|
|
16041
|
+
for (const mutation of preparedMutations) {
|
|
16042
|
+
const stats = this.commitPreparedMutation(mutation);
|
|
15854
16043
|
totals.indexed += stats.indexed;
|
|
15855
16044
|
totals.deleted += stats.deleted;
|
|
15856
16045
|
totals.skipped += stats.skipped;
|
|
@@ -15859,77 +16048,34 @@ var MemoryIndexer = class {
|
|
|
15859
16048
|
}
|
|
15860
16049
|
async search(query, topK) {
|
|
15861
16050
|
const fetchLimit = topK * FETCH_MULTIPLIER;
|
|
15862
|
-
let
|
|
16051
|
+
let embedding;
|
|
15863
16052
|
if (this.store.vecAvailable) {
|
|
15864
16053
|
try {
|
|
15865
|
-
const
|
|
15866
|
-
|
|
16054
|
+
const embeddings = await this.embedder.embed([query]);
|
|
16055
|
+
embedding = embeddings[0];
|
|
15867
16056
|
} catch (err) {
|
|
15868
16057
|
console.warn("[memory-indexer] Dense search embedding failed:", err);
|
|
15869
16058
|
}
|
|
15870
16059
|
}
|
|
15871
|
-
const
|
|
15872
|
-
|
|
15873
|
-
|
|
15874
|
-
|
|
15875
|
-
|
|
15876
|
-
|
|
15877
|
-
|
|
15878
|
-
|
|
15879
|
-
const unionIndexOf = new Map(unionArray.map((id, idx) => [id, idx]));
|
|
15880
|
-
const recencyOrdered = [...unionArray].sort((a, b) => {
|
|
15881
|
-
const mtimeA = chunkMap.get(a)?.fileMtimeAt ?? 0;
|
|
15882
|
-
const mtimeB = chunkMap.get(b)?.fileMtimeAt ?? 0;
|
|
15883
|
-
if (mtimeB !== mtimeA) return mtimeB - mtimeA;
|
|
15884
|
-
return (unionIndexOf.get(a) ?? 0) - (unionIndexOf.get(b) ?? 0);
|
|
15885
|
-
});
|
|
15886
|
-
const scoreMap = /* @__PURE__ */ new Map();
|
|
15887
|
-
denseResults.forEach((r, rank) => {
|
|
15888
|
-
scoreMap.set(r.id, { dense: 1 / (RRF_K + rank + 1), bm25: 0, recency: 0 });
|
|
15889
|
-
});
|
|
15890
|
-
bm25Results.forEach((r, rank) => {
|
|
15891
|
-
const entry = scoreMap.get(r.id) ?? { dense: 0, bm25: 0, recency: 0 };
|
|
15892
|
-
entry.bm25 = 1 / (RRF_K + rank + 1);
|
|
15893
|
-
scoreMap.set(r.id, entry);
|
|
15894
|
-
});
|
|
15895
|
-
recencyOrdered.forEach((id, rank) => {
|
|
15896
|
-
const entry = scoreMap.get(id);
|
|
15897
|
-
if (!entry) return;
|
|
15898
|
-
entry.recency = RECENCY_WEIGHT * (1 / (RRF_K + rank + 1));
|
|
16060
|
+
const rows = this.store.searchHybrid({
|
|
16061
|
+
query,
|
|
16062
|
+
embedding,
|
|
16063
|
+
topK,
|
|
16064
|
+
fetchLimit,
|
|
16065
|
+
denseScoreFloor: DENSE_SCORE_FLOOR,
|
|
16066
|
+
recencyWeight: RECENCY_WEIGHT,
|
|
16067
|
+
rrfK: RRF_K
|
|
15899
16068
|
});
|
|
15900
|
-
|
|
15901
|
-
const bm25Active = bm25Results.length > 0;
|
|
15902
|
-
const recencyActive = unionIds.size > 0;
|
|
15903
|
-
const maxScore = ((denseActive ? 1 : 0) + (bm25Active ? 1 : 0) + (recencyActive ? RECENCY_WEIGHT : 0)) / (RRF_K + 1);
|
|
15904
|
-
const ranked = [...scoreMap.entries()].map(([id, scores]) => ({
|
|
15905
|
-
id,
|
|
15906
|
-
totalScore: scores.dense + scores.bm25 + scores.recency,
|
|
15907
|
-
hasDense: scores.dense > 0,
|
|
15908
|
-
hasBm25: scores.bm25 > 0,
|
|
15909
|
-
isDenseOnly: denseIds.has(id) && !bm25Ids.has(id)
|
|
15910
|
-
})).filter((candidate) => {
|
|
15911
|
-
if (!candidate.isDenseOnly) return true;
|
|
15912
|
-
const retrieval = denseRetrievalScore.get(candidate.id);
|
|
15913
|
-
if (retrieval === void 0) return true;
|
|
15914
|
-
return retrieval >= DENSE_SCORE_FLOOR;
|
|
15915
|
-
}).sort((a, b) => b.totalScore - a.totalScore);
|
|
16069
|
+
if (rows.length === 0) return [];
|
|
15916
16070
|
const results = [];
|
|
15917
|
-
const
|
|
15918
|
-
for (const r of ranked) {
|
|
15919
|
-
if (results.length >= topK) break;
|
|
15920
|
-
const chunk = chunkMap.get(r.id);
|
|
15921
|
-
if (!chunk) continue;
|
|
15922
|
-
if (seenSources.has(chunk.source)) continue;
|
|
15923
|
-
seenSources.add(chunk.source);
|
|
15924
|
-
const normalizedScore = maxScore > 0 ? r.totalScore / maxScore : 0;
|
|
15925
|
-
const retriever = r.hasDense && r.hasBm25 ? "both" : r.hasDense ? "dense" : "bm25";
|
|
16071
|
+
for (const row of rows) {
|
|
15926
16072
|
let contentSource = "file";
|
|
15927
16073
|
try {
|
|
15928
|
-
chunk.content = fs6.
|
|
16074
|
+
row.chunk.content = await fs6.promises.readFile(path8.join(this.config.wikiDir, row.chunk.source), "utf8");
|
|
15929
16075
|
} catch {
|
|
15930
16076
|
contentSource = "fallback";
|
|
15931
16077
|
}
|
|
15932
|
-
results.push({ chunk, score:
|
|
16078
|
+
results.push({ chunk: row.chunk, score: row.score, retriever: row.retriever, contentSource });
|
|
15933
16079
|
}
|
|
15934
16080
|
return results;
|
|
15935
16081
|
}
|
|
@@ -15938,13 +16084,13 @@ var MemoryIndexer = class {
|
|
|
15938
16084
|
const rawDir = path8.join(this.config.wikiDir, "raw");
|
|
15939
16085
|
const savePath = path8.join(rawDir, `conv_save_${datePart}.md`);
|
|
15940
16086
|
const lockPath = `${savePath}.lock`;
|
|
15941
|
-
fs6.
|
|
16087
|
+
await fs6.promises.mkdir(rawDir, { recursive: true });
|
|
15942
16088
|
const acquired = await acquireLock(lockPath, LOCK_TIMEOUT_MS, LOCK_RETRY_MS);
|
|
15943
16089
|
if (!acquired) {
|
|
15944
16090
|
console.warn("[memory-indexer] Could not acquire write lock, writing anyway (fail-open)");
|
|
15945
16091
|
}
|
|
15946
16092
|
try {
|
|
15947
|
-
fs6.
|
|
16093
|
+
await fs6.promises.appendFile(savePath, `
|
|
15948
16094
|
${content}
|
|
15949
16095
|
`, "utf8");
|
|
15950
16096
|
} finally {
|
|
@@ -23,13 +23,13 @@ registerIntegrationTools(server);
|
|
|
23
23
|
registerCoreTools(server);
|
|
24
24
|
registerAgentTools(server);
|
|
25
25
|
const workspaceRoot = getWorkspaceRoot();
|
|
26
|
-
const
|
|
26
|
+
const memorySubsystem = registerMemoryTools(server, workspaceRoot);
|
|
27
27
|
// ═══════════════════════════════════════════════════════════════
|
|
28
28
|
// START SERVER
|
|
29
29
|
// ═══════════════════════════════════════════════════════════════
|
|
30
30
|
const transport = new StdioServerTransport();
|
|
31
31
|
await server.connect(transport);
|
|
32
32
|
// Fire-and-forget startup indexing — must run after server.connect
|
|
33
|
-
if (
|
|
34
|
-
void
|
|
33
|
+
if (memorySubsystem) {
|
|
34
|
+
void memorySubsystem.startWarmup();
|
|
35
35
|
}
|
package/dist/mcp/memory.d.ts
CHANGED
|
@@ -1,16 +1,11 @@
|
|
|
1
1
|
import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
|
2
|
-
import {
|
|
3
|
-
import { MemoryStore } from '../services/memory/store.js';
|
|
4
|
-
import type { MemoryConfig } from '../services/memory/types.js';
|
|
2
|
+
import { type MemorySubsystem, type MemorySubsystemOverrides } from '../services/memory/subsystem.js';
|
|
5
3
|
import { type ProjectSettings } from '../core/config/index.js';
|
|
6
4
|
/**
|
|
7
5
|
* Initializes the memory subsystem and registers all memory tools.
|
|
8
|
-
* Returns the
|
|
6
|
+
* Returns the MemorySubsystem so the caller can fire startWarmup() after server.connect().
|
|
9
7
|
* Returns null when memory is disabled (settings.memory.enabled !== true).
|
|
10
8
|
*/
|
|
11
|
-
export declare function registerMemoryTools(server: McpServer, workspaceRoot: string, overrides?: {
|
|
12
|
-
indexer?: MemoryIndexer;
|
|
13
|
-
store?: MemoryStore;
|
|
14
|
-
config?: MemoryConfig;
|
|
9
|
+
export declare function registerMemoryTools(server: McpServer, workspaceRoot: string, overrides?: MemorySubsystemOverrides & {
|
|
15
10
|
settings?: ProjectSettings;
|
|
16
|
-
}):
|
|
11
|
+
}): MemorySubsystem | null;
|