@hanhnd/agent-kit 1.0.31 → 1.0.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands/cli-digest-pending.test.js +111 -0
- package/dist/cli/commands/cli-memory.test.js +34 -0
- package/dist/cli/commands/memory.d.ts +1 -0
- package/dist/cli/commands/memory.js +90 -0
- package/dist/cli/index.d.ts +1 -0
- package/dist/cli/index.js +9 -0
- package/dist/cli/utils/args.d.ts +8 -0
- package/dist/cli/utils/args.js +31 -0
- package/dist/cli.js +16489 -0
- package/dist/{tools/config.d.ts → core/config/index.d.ts} +4 -9
- package/dist/{tools/config.js → core/config/index.js} +20 -23
- package/dist/core/handoffs/index.d.ts +16 -0
- package/dist/{tools/handoffs.js → core/handoffs/index.js} +4 -5
- package/dist/{tools/security.d.ts → core/security/index.d.ts} +0 -19
- package/dist/{tools/security.js → core/security/index.js} +3 -80
- package/dist/entrypoints/cli.d.ts +2 -0
- package/dist/entrypoints/cli.js +11 -0
- package/dist/entrypoints/server.js +37 -0
- package/dist/{tools → mcp}/agent.js +3 -14
- package/dist/{tools → mcp}/core.d.ts +1 -1
- package/dist/{tools → mcp}/core.js +3 -5
- package/dist/{tools → mcp}/integration.js +5 -11
- package/dist/mcp/memory.d.ts +15 -0
- package/dist/{tools → mcp}/memory.js +40 -12
- package/dist/{kit-server.js → server.js} +720 -224
- package/dist/services/digest/background.d.ts +11 -0
- package/dist/services/digest/background.js +258 -0
- package/dist/services/digest/constants.d.ts +17 -0
- package/dist/services/digest/constants.js +19 -0
- package/dist/services/digest/digest-background.test.js +206 -0
- package/dist/services/digest/digest-init.test.js +33 -0
- package/dist/services/digest/digest-llama-local.test.js +21 -0
- package/dist/services/digest/digest-pending.test.d.ts +1 -0
- package/dist/services/digest/digest-pending.test.js +192 -0
- package/dist/services/digest/digest-processor.test.d.ts +1 -0
- package/dist/services/digest/digest-processor.test.js +32 -0
- package/dist/services/digest/files.d.ts +6 -0
- package/dist/services/digest/files.js +45 -0
- package/dist/services/digest/model-registry.d.ts +2 -0
- package/dist/services/digest/model-registry.js +49 -0
- package/dist/services/digest/processor.d.ts +13 -0
- package/dist/services/digest/processor.js +188 -0
- package/dist/services/digest/providers/llama-local.d.ts +2 -0
- package/dist/services/digest/providers/llama-local.js +135 -0
- package/dist/services/digest/types.d.ts +117 -0
- package/dist/services/digest/types.js +6 -0
- package/dist/services/digest/worker.d.ts +2 -0
- package/dist/services/digest/worker.js +51 -0
- package/dist/{utils → services/integration}/credentials.js +1 -1
- package/dist/{utils/parser.js → services/integration/parser/adf.js} +3 -6
- package/dist/{memory → services/memory}/chunker.d.ts +0 -2
- package/dist/{memory → services/memory}/chunker.js +5 -7
- package/dist/services/memory/chunker.test.d.ts +1 -0
- package/dist/{memory/__tests__ → services/memory}/chunker.test.js +1 -34
- package/dist/services/memory/constants.d.ts +5 -0
- package/dist/services/memory/constants.js +13 -0
- package/dist/{memory → services/memory}/embedder.d.ts +3 -2
- package/dist/{memory → services/memory}/embedder.js +15 -6
- package/dist/services/memory/embedder.test.d.ts +1 -0
- package/dist/services/memory/embedder.test.js +13 -0
- package/dist/{memory → services/memory}/index.d.ts +1 -1
- package/dist/{memory → services/memory}/index.js +1 -1
- package/dist/{memory → services/memory}/indexer.d.ts +6 -2
- package/dist/{memory → services/memory}/indexer.js +15 -37
- package/dist/services/memory/indexer.test.d.ts +1 -0
- package/dist/{memory/__tests__ → services/memory}/indexer.test.js +11 -15
- package/dist/services/memory/memory.test.d.ts +1 -0
- package/dist/{tools/__tests__ → services/memory}/memory.test.js +82 -19
- package/dist/{memory → services/memory}/store.js +6 -18
- package/dist/services/memory/store.test.d.ts +1 -0
- package/dist/{memory/__tests__ → services/memory}/store.test.js +18 -5
- package/dist/{memory → services/memory}/types.d.ts +8 -2
- package/dist/services/memory/types.js +6 -0
- package/dist/utils/async.d.ts +1 -0
- package/dist/utils/async.js +15 -0
- package/dist/utils/files.d.ts +7 -0
- package/dist/utils/files.js +77 -0
- package/dist/utils/files.test.d.ts +1 -0
- package/dist/utils/files.test.js +81 -0
- package/dist/utils/hash.d.ts +1 -0
- package/dist/utils/hash.js +4 -0
- package/dist/utils/index.d.ts +6 -0
- package/dist/utils/index.js +6 -0
- package/dist/utils/json.d.ts +1 -0
- package/dist/utils/json.js +3 -0
- package/dist/utils/paths.d.ts +0 -1
- package/dist/utils/paths.js +1 -1
- package/dist/utils/temp-dir.test.d.ts +4 -0
- package/dist/utils/temp-dir.test.js +20 -0
- package/package.json +8 -6
- package/dist/memory/__tests__/embedder.test.js +0 -14
- package/dist/memory/types.js +0 -9
- package/dist/tools/handoffs.d.ts +0 -22
- package/dist/tools/memory.d.ts +0 -16
- /package/dist/{memory/__tests__/chunker.test.d.ts → cli/commands/cli-digest-pending.test.d.ts} +0 -0
- /package/dist/{memory/__tests__/embedder.test.d.ts → cli/commands/cli-memory.test.d.ts} +0 -0
- /package/dist/{kit-server.d.ts → entrypoints/server.d.ts} +0 -0
- /package/dist/{tools → mcp}/agent.d.ts +0 -0
- /package/dist/{tools → mcp}/integration.d.ts +0 -0
- /package/dist/{memory/__tests__/indexer.test.d.ts → services/digest/digest-background.test.d.ts} +0 -0
- /package/dist/{memory/__tests__/store.test.d.ts → services/digest/digest-init.test.d.ts} +0 -0
- /package/dist/{tools/__tests__/memory.test.d.ts → services/digest/digest-llama-local.test.d.ts} +0 -0
- /package/dist/{utils → services/integration}/credentials.d.ts +0 -0
- /package/dist/{utils/parser.d.ts → services/integration/parser/adf.d.ts} +0 -0
- /package/dist/{memory → services/memory}/store.d.ts +0 -0
|
@@ -1,29 +1,8 @@
|
|
|
1
1
|
import * as fs from 'fs';
|
|
2
2
|
import * as path from 'path';
|
|
3
3
|
import { chunkMarkdown } from './chunker.js';
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
async function acquireLock(lockPath) {
|
|
7
|
-
const deadline = Date.now() + LOCK_TIMEOUT_MS;
|
|
8
|
-
while (Date.now() < deadline) {
|
|
9
|
-
try {
|
|
10
|
-
fs.writeFileSync(lockPath, String(process.pid), { flag: 'wx' });
|
|
11
|
-
return true;
|
|
12
|
-
}
|
|
13
|
-
catch {
|
|
14
|
-
await new Promise((r) => setTimeout(r, LOCK_RETRY_MS));
|
|
15
|
-
}
|
|
16
|
-
}
|
|
17
|
-
return false;
|
|
18
|
-
}
|
|
19
|
-
function releaseLock(lockPath) {
|
|
20
|
-
try {
|
|
21
|
-
fs.unlinkSync(lockPath);
|
|
22
|
-
}
|
|
23
|
-
catch {
|
|
24
|
-
// ignore
|
|
25
|
-
}
|
|
26
|
-
}
|
|
4
|
+
import { LOCK_RETRY_MS, LOCK_TIMEOUT_MS, RRF_K } from './constants.js';
|
|
5
|
+
import { acquireLock, releaseLock } from '../../utils/files.js';
|
|
27
6
|
export class MemoryIndexer {
|
|
28
7
|
store;
|
|
29
8
|
embedder;
|
|
@@ -39,19 +18,19 @@ export class MemoryIndexer {
|
|
|
39
18
|
}
|
|
40
19
|
async indexFileRelativeTo(absolutePath, sourceRoot) {
|
|
41
20
|
const source = path.relative(sourceRoot, absolutePath);
|
|
42
|
-
const
|
|
21
|
+
const existingChunkIds = this.store.hashesBySource(source);
|
|
43
22
|
let text;
|
|
44
23
|
try {
|
|
45
24
|
text = fs.readFileSync(absolutePath, 'utf8');
|
|
46
25
|
}
|
|
47
26
|
catch (err) {
|
|
48
27
|
console.warn(`[memory-indexer] Cannot read file ${absolutePath}:`, err);
|
|
49
|
-
return { indexed: 0, deleted: 0, skipped:
|
|
28
|
+
return { indexed: 0, deleted: 0, skipped: existingChunkIds.size };
|
|
50
29
|
}
|
|
51
30
|
const allChunks = chunkMarkdown(text, source, this.config);
|
|
52
|
-
const
|
|
53
|
-
const toIndex = allChunks.filter((
|
|
54
|
-
const toDelete = [...
|
|
31
|
+
const currentChunkIds = new Set(allChunks.map((chunk) => chunk.id));
|
|
32
|
+
const toIndex = allChunks.filter((chunk) => !existingChunkIds.has(chunk.id));
|
|
33
|
+
const toDelete = [...existingChunkIds].filter((id) => !currentChunkIds.has(id));
|
|
55
34
|
let embeddings = [];
|
|
56
35
|
if (toIndex.length > 0) {
|
|
57
36
|
try {
|
|
@@ -59,7 +38,7 @@ export class MemoryIndexer {
|
|
|
59
38
|
}
|
|
60
39
|
catch (err) {
|
|
61
40
|
console.warn(`[memory-indexer] Embedding failed for ${absolutePath}:`, err);
|
|
62
|
-
return { indexed: 0, deleted: 0, skipped:
|
|
41
|
+
return { indexed: 0, deleted: 0, skipped: existingChunkIds.size };
|
|
63
42
|
}
|
|
64
43
|
}
|
|
65
44
|
if (toIndex.length > 0) {
|
|
@@ -135,24 +114,22 @@ export class MemoryIndexer {
|
|
|
135
114
|
if (bm25Ids.size > 0) {
|
|
136
115
|
denseResults = denseResults.filter((result) => bm25Ids.has(result.id));
|
|
137
116
|
}
|
|
138
|
-
// RRF fusion (k=60)
|
|
139
|
-
const k = 60;
|
|
140
117
|
const scoreMap = new Map();
|
|
141
118
|
denseResults.forEach((r, rank) => {
|
|
142
|
-
scoreMap.set(r.id, { dense: 1 / (
|
|
119
|
+
scoreMap.set(r.id, { dense: 1 / (RRF_K + rank + 1), bm25: 0 });
|
|
143
120
|
});
|
|
144
121
|
bm25Results.forEach((r, rank) => {
|
|
145
122
|
const existing = scoreMap.get(r.id);
|
|
146
123
|
if (existing) {
|
|
147
|
-
existing.bm25 = 1 / (
|
|
124
|
+
existing.bm25 = 1 / (RRF_K + rank + 1);
|
|
148
125
|
}
|
|
149
126
|
else {
|
|
150
|
-
scoreMap.set(r.id, { dense: 0, bm25: 1 / (
|
|
127
|
+
scoreMap.set(r.id, { dense: 0, bm25: 1 / (RRF_K + rank + 1) });
|
|
151
128
|
}
|
|
152
129
|
});
|
|
153
130
|
const hasDense = this.store.vecAvailable && denseResults.length > 0;
|
|
154
131
|
const numRetrievers = hasDense ? 2 : 1;
|
|
155
|
-
const maxScore = numRetrievers /
|
|
132
|
+
const maxScore = numRetrievers / (RRF_K + 1);
|
|
156
133
|
const ranked = [...scoreMap.entries()]
|
|
157
134
|
.map(([id, scores]) => ({
|
|
158
135
|
id,
|
|
@@ -193,7 +170,7 @@ export class MemoryIndexer {
|
|
|
193
170
|
const savePath = path.join(rawDir, `conv_save_${datePart}.md`);
|
|
194
171
|
const lockPath = `${savePath}.lock`;
|
|
195
172
|
fs.mkdirSync(rawDir, { recursive: true });
|
|
196
|
-
const acquired = await acquireLock(lockPath);
|
|
173
|
+
const acquired = await acquireLock(lockPath, LOCK_TIMEOUT_MS, LOCK_RETRY_MS);
|
|
197
174
|
if (!acquired) {
|
|
198
175
|
console.warn('[memory-indexer] Could not acquire write lock, writing anyway (fail-open)');
|
|
199
176
|
}
|
|
@@ -202,12 +179,13 @@ export class MemoryIndexer {
|
|
|
202
179
|
}
|
|
203
180
|
finally {
|
|
204
181
|
if (acquired)
|
|
205
|
-
releaseLock(lockPath);
|
|
182
|
+
releaseLock(lockPath, { ignoreErrors: true });
|
|
206
183
|
}
|
|
207
184
|
return { indexed: 0, deleted: 0, skipped: 0 };
|
|
208
185
|
}
|
|
209
186
|
async startupIndex() {
|
|
210
187
|
try {
|
|
188
|
+
await this.embedder.initialize();
|
|
211
189
|
await this.indexDirectory(path.join(this.config.wikiDir, 'compiled'), {
|
|
212
190
|
relativeBase: this.config.wikiDir,
|
|
213
191
|
excludeFiles: ['index.md', 'log.md'],
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -5,18 +5,15 @@ import { syncBuiltinESMExports } from 'node:module';
|
|
|
5
5
|
import * as os from 'node:os';
|
|
6
6
|
import * as path from 'node:path';
|
|
7
7
|
import { after, before, describe, test } from 'node:test';
|
|
8
|
-
import { MemoryIndexer } from '
|
|
9
|
-
import { MemoryStore } from '
|
|
10
|
-
|
|
8
|
+
import { MemoryIndexer } from './indexer.js';
|
|
9
|
+
import { MemoryStore } from './store.js';
|
|
10
|
+
import { EmbeddingModelName } from './types.js';
|
|
11
11
|
class StubEmbedder {
|
|
12
12
|
async embed(texts) {
|
|
13
13
|
return texts.map(() => new Float32Array(384).fill(0.05));
|
|
14
14
|
}
|
|
15
|
-
|
|
16
|
-
return
|
|
17
|
-
}
|
|
18
|
-
isReady() {
|
|
19
|
-
return true;
|
|
15
|
+
initialize() {
|
|
16
|
+
return Promise.resolve();
|
|
20
17
|
}
|
|
21
18
|
}
|
|
22
19
|
function makeConfig(wikiDir) {
|
|
@@ -26,7 +23,7 @@ function makeConfig(wikiDir) {
|
|
|
26
23
|
topK: 5,
|
|
27
24
|
chunkSize: 1500,
|
|
28
25
|
overlapLines: 2,
|
|
29
|
-
embeddingModel:
|
|
26
|
+
embeddingModel: EmbeddingModelName.SMALL,
|
|
30
27
|
vectorDimension: 384,
|
|
31
28
|
};
|
|
32
29
|
}
|
|
@@ -140,7 +137,8 @@ describe('MemoryIndexer', () => {
|
|
|
140
137
|
const compiledDir = path.join(testCfg.wikiDir, 'compiled');
|
|
141
138
|
try {
|
|
142
139
|
fs.mkdirSync(compiledDir, { recursive: true });
|
|
143
|
-
testStore.upsert([
|
|
140
|
+
testStore.upsert([
|
|
141
|
+
{
|
|
144
142
|
id: 'stale-daily-file-0001',
|
|
145
143
|
source: '2026-05-18.md',
|
|
146
144
|
heading: 'Stale',
|
|
@@ -148,7 +146,8 @@ describe('MemoryIndexer', () => {
|
|
|
148
146
|
content: 'pre migration content',
|
|
149
147
|
lineStart: 1,
|
|
150
148
|
lineEnd: 2,
|
|
151
|
-
}
|
|
149
|
+
},
|
|
150
|
+
], [new Float32Array(384)]);
|
|
152
151
|
assert.ok(testStore.hashesBySource('2026-05-18.md').size > 0);
|
|
153
152
|
const stats = await testIndexer.indexDirectory(compiledDir, {
|
|
154
153
|
relativeBase: testCfg.wikiDir,
|
|
@@ -257,10 +256,7 @@ describe('MemoryIndexer', () => {
|
|
|
257
256
|
fs.writeFileSync(secondPath, secondContent, 'utf8');
|
|
258
257
|
const results = await testIndexer.search('duplicate source query', 2);
|
|
259
258
|
assert.equal(results.length, 2);
|
|
260
|
-
assert.deepEqual(results.map((result) => result.chunk.source), [
|
|
261
|
-
'compiled/entities/first.md',
|
|
262
|
-
'compiled/entities/second.md',
|
|
263
|
-
]);
|
|
259
|
+
assert.deepEqual(results.map((result) => result.chunk.source), ['compiled/entities/first.md', 'compiled/entities/second.md']);
|
|
264
260
|
assert.equal(results[0].chunk.content, firstContent);
|
|
265
261
|
assert.equal(results[1].chunk.content, secondContent);
|
|
266
262
|
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import * as assert from 'node:assert/strict';
|
|
2
2
|
import { describe, test } from 'node:test';
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
3
|
+
import { EmbeddingModelName } from './types.js';
|
|
4
|
+
import { resolveMemoryConfig } from '../../core/config/index.js';
|
|
5
|
+
import { registerMemoryTools } from '../../mcp/memory.js';
|
|
5
6
|
// Minimal McpServer stub that captures tool registrations
|
|
6
7
|
function makeMockServer() {
|
|
7
8
|
const tools = new Map();
|
|
@@ -30,7 +31,7 @@ const BASE_CONFIG = {
|
|
|
30
31
|
topK: 5,
|
|
31
32
|
chunkSize: 1500,
|
|
32
33
|
overlapLines: 2,
|
|
33
|
-
embeddingModel:
|
|
34
|
+
embeddingModel: EmbeddingModelName.SMALL,
|
|
34
35
|
vectorDimension: 384,
|
|
35
36
|
};
|
|
36
37
|
function makeIndexerStub(overrides = {}) {
|
|
@@ -48,12 +49,24 @@ function extractText(result) {
|
|
|
48
49
|
}
|
|
49
50
|
describe('kit_memory_search', () => {
|
|
50
51
|
test('returns formatted full content and display source when results exist', async () => {
|
|
51
|
-
const chunk = makeChunk({
|
|
52
|
-
|
|
52
|
+
const chunk = makeChunk({
|
|
53
|
+
content: 'Important memory content',
|
|
54
|
+
source: 'compiled/entities/foo.md',
|
|
55
|
+
});
|
|
56
|
+
const searchResult = {
|
|
57
|
+
chunk,
|
|
58
|
+
score: 0.85,
|
|
59
|
+
retriever: 'bm25',
|
|
60
|
+
contentSource: 'file',
|
|
61
|
+
};
|
|
53
62
|
const indexer = makeIndexerStub({ search: async () => [searchResult] });
|
|
54
63
|
const store = makeStoreStub(true);
|
|
55
64
|
const { server, tools } = makeMockServer();
|
|
56
|
-
|
|
65
|
+
registerMemoryTools(server, { memory: { enabled: true } }, '/tmp', {
|
|
66
|
+
indexer,
|
|
67
|
+
store,
|
|
68
|
+
config: BASE_CONFIG,
|
|
69
|
+
});
|
|
57
70
|
const result = await tools.get('kit_memory_search')({ query: 'test query' });
|
|
58
71
|
const text = extractText(result);
|
|
59
72
|
assert.ok(text.includes('Important memory content'), 'Result must include chunk content');
|
|
@@ -65,7 +78,11 @@ describe('kit_memory_search', () => {
|
|
|
65
78
|
const indexer = makeIndexerStub({ search: async () => [] });
|
|
66
79
|
const store = makeStoreStub(true);
|
|
67
80
|
const { server, tools } = makeMockServer();
|
|
68
|
-
|
|
81
|
+
registerMemoryTools(server, { memory: { enabled: true } }, '/tmp', {
|
|
82
|
+
indexer,
|
|
83
|
+
store,
|
|
84
|
+
config: BASE_CONFIG,
|
|
85
|
+
});
|
|
69
86
|
const result = await tools.get('kit_memory_search')({ query: 'nothing here' });
|
|
70
87
|
const text = extractText(result);
|
|
71
88
|
assert.ok(text.toLowerCase().includes('no memories') || text.includes('No memories'), `Expected "no memories" message, got: ${text.slice(0, 100)}`);
|
|
@@ -74,18 +91,31 @@ describe('kit_memory_search', () => {
|
|
|
74
91
|
const indexer = makeIndexerStub({ search: async () => [] });
|
|
75
92
|
const store = makeStoreStub(false);
|
|
76
93
|
const { server, tools } = makeMockServer();
|
|
77
|
-
|
|
94
|
+
registerMemoryTools(server, { memory: { enabled: true } }, '/tmp', {
|
|
95
|
+
indexer,
|
|
96
|
+
store,
|
|
97
|
+
config: BASE_CONFIG,
|
|
98
|
+
});
|
|
78
99
|
const result = await tools.get('kit_memory_search')({ query: 'test' });
|
|
79
100
|
const text = extractText(result);
|
|
80
101
|
assert.ok(text.includes('Vector search unavailable') || text.includes('keyword-only'), `Expected degraded warning in: ${text.slice(0, 200)}`);
|
|
81
102
|
});
|
|
82
103
|
test('prepends degraded warning when vecAvailable is false with results', async () => {
|
|
83
104
|
const chunk = makeChunk({ content: 'Found content', source: 'test.md' });
|
|
84
|
-
const searchResult = {
|
|
105
|
+
const searchResult = {
|
|
106
|
+
chunk,
|
|
107
|
+
score: 0.5,
|
|
108
|
+
retriever: 'bm25',
|
|
109
|
+
contentSource: 'file',
|
|
110
|
+
};
|
|
85
111
|
const indexer = makeIndexerStub({ search: async () => [searchResult] });
|
|
86
112
|
const store = makeStoreStub(false);
|
|
87
113
|
const { server, tools } = makeMockServer();
|
|
88
|
-
|
|
114
|
+
registerMemoryTools(server, { memory: { enabled: true } }, '/tmp', {
|
|
115
|
+
indexer,
|
|
116
|
+
store,
|
|
117
|
+
config: BASE_CONFIG,
|
|
118
|
+
});
|
|
89
119
|
const result = await tools.get('kit_memory_search')({ query: 'test' });
|
|
90
120
|
const text = extractText(result);
|
|
91
121
|
assert.ok(text.includes('Vector search unavailable') || text.includes('keyword-only'), `Expected degraded warning with results in: ${text.slice(0, 200)}`);
|
|
@@ -93,11 +123,20 @@ describe('kit_memory_search', () => {
|
|
|
93
123
|
});
|
|
94
124
|
test('prepends source-unavailable warning for fallback results', async () => {
|
|
95
125
|
const chunk = makeChunk({ content: 'Stored chunk only', source: 'compiled/entities/foo.md' });
|
|
96
|
-
const searchResult = {
|
|
126
|
+
const searchResult = {
|
|
127
|
+
chunk,
|
|
128
|
+
score: 0.5,
|
|
129
|
+
retriever: 'bm25',
|
|
130
|
+
contentSource: 'fallback',
|
|
131
|
+
};
|
|
97
132
|
const indexer = makeIndexerStub({ search: async () => [searchResult] });
|
|
98
133
|
const store = makeStoreStub(true);
|
|
99
134
|
const { server, tools } = makeMockServer();
|
|
100
|
-
|
|
135
|
+
registerMemoryTools(server, { memory: { enabled: true } }, '/tmp', {
|
|
136
|
+
indexer,
|
|
137
|
+
store,
|
|
138
|
+
config: BASE_CONFIG,
|
|
139
|
+
});
|
|
101
140
|
const result = await tools.get('kit_memory_search')({ query: 'test' });
|
|
102
141
|
const text = extractText(result);
|
|
103
142
|
assert.ok(text.includes('Source file unavailable'), `Expected fallback warning in: ${text}`);
|
|
@@ -113,7 +152,11 @@ describe('kit_memory_search', () => {
|
|
|
113
152
|
});
|
|
114
153
|
const store = makeStoreStub(true);
|
|
115
154
|
const { server, tools } = makeMockServer();
|
|
116
|
-
|
|
155
|
+
registerMemoryTools(server, { memory: { enabled: true } }, '/tmp', {
|
|
156
|
+
indexer,
|
|
157
|
+
store,
|
|
158
|
+
config: BASE_CONFIG,
|
|
159
|
+
});
|
|
117
160
|
await tools.get('kit_memory_search')({ query: 'test', top_k: 3 });
|
|
118
161
|
assert.equal(capturedTopK, 3);
|
|
119
162
|
});
|
|
@@ -127,17 +170,27 @@ describe('kit_memory_search', () => {
|
|
|
127
170
|
});
|
|
128
171
|
const store = makeStoreStub(true);
|
|
129
172
|
const { server, tools } = makeMockServer();
|
|
130
|
-
|
|
173
|
+
registerMemoryTools(server, { memory: { enabled: true } }, '/tmp', {
|
|
174
|
+
indexer,
|
|
175
|
+
store,
|
|
176
|
+
config: BASE_CONFIG,
|
|
177
|
+
});
|
|
131
178
|
await tools.get('kit_memory_search')({ query: 'test' });
|
|
132
179
|
assert.equal(capturedTopK, BASE_CONFIG.topK);
|
|
133
180
|
});
|
|
134
181
|
test('returns error message when indexer.search throws', async () => {
|
|
135
182
|
const indexer = makeIndexerStub({
|
|
136
|
-
search: async () => {
|
|
183
|
+
search: async () => {
|
|
184
|
+
throw new Error('search exploded');
|
|
185
|
+
},
|
|
137
186
|
});
|
|
138
187
|
const store = makeStoreStub(true);
|
|
139
188
|
const { server, tools } = makeMockServer();
|
|
140
|
-
|
|
189
|
+
registerMemoryTools(server, { memory: { enabled: true } }, '/tmp', {
|
|
190
|
+
indexer,
|
|
191
|
+
store,
|
|
192
|
+
config: BASE_CONFIG,
|
|
193
|
+
});
|
|
141
194
|
const result = await tools.get('kit_memory_search')({ query: 'test' });
|
|
142
195
|
const text = extractText(result);
|
|
143
196
|
assert.ok(text.includes('kit_memory_search failed'), `Expected error prefix, got: ${text}`);
|
|
@@ -151,7 +204,11 @@ describe('kit_memory_save', () => {
|
|
|
151
204
|
});
|
|
152
205
|
const store = makeStoreStub(true);
|
|
153
206
|
const { server, tools } = makeMockServer();
|
|
154
|
-
|
|
207
|
+
registerMemoryTools(server, { memory: { enabled: true } }, '/tmp', {
|
|
208
|
+
indexer,
|
|
209
|
+
store,
|
|
210
|
+
config: BASE_CONFIG,
|
|
211
|
+
});
|
|
155
212
|
const result = await tools.get('kit_memory_save')({ content: 'Some important note' });
|
|
156
213
|
const text = extractText(result);
|
|
157
214
|
const parsed = JSON.parse(text);
|
|
@@ -161,11 +218,17 @@ describe('kit_memory_save', () => {
|
|
|
161
218
|
});
|
|
162
219
|
test('returns saved:false and error message when indexer.save throws', async () => {
|
|
163
220
|
const indexer = makeIndexerStub({
|
|
164
|
-
save: async () => {
|
|
221
|
+
save: async () => {
|
|
222
|
+
throw new Error('disk full');
|
|
223
|
+
},
|
|
165
224
|
});
|
|
166
225
|
const store = makeStoreStub(true);
|
|
167
226
|
const { server, tools } = makeMockServer();
|
|
168
|
-
|
|
227
|
+
registerMemoryTools(server, { memory: { enabled: true } }, '/tmp', {
|
|
228
|
+
indexer,
|
|
229
|
+
store,
|
|
230
|
+
config: BASE_CONFIG,
|
|
231
|
+
});
|
|
169
232
|
const result = await tools.get('kit_memory_save')({ content: 'note' });
|
|
170
233
|
const text = extractText(result);
|
|
171
234
|
const parsed = JSON.parse(text);
|
|
@@ -141,27 +141,19 @@ export class MemoryStore {
|
|
|
141
141
|
}
|
|
142
142
|
}
|
|
143
143
|
hashesBySource(source) {
|
|
144
|
-
const rows = this.db
|
|
145
|
-
.prepare(`SELECT id FROM memory_chunks WHERE source = ?`)
|
|
146
|
-
.all(source);
|
|
144
|
+
const rows = this.db.prepare(`SELECT id FROM memory_chunks WHERE source = ?`).all(source);
|
|
147
145
|
return new Set(rows.map((r) => r.id));
|
|
148
146
|
}
|
|
149
147
|
indexedSources() {
|
|
150
|
-
const rows = this.db
|
|
151
|
-
.prepare(`SELECT DISTINCT source FROM memory_chunks`)
|
|
152
|
-
.all();
|
|
148
|
+
const rows = this.db.prepare(`SELECT DISTINCT source FROM memory_chunks`).all();
|
|
153
149
|
return rows.map((r) => r.source);
|
|
154
150
|
}
|
|
155
151
|
deleteBySource(source) {
|
|
156
|
-
const rows = this.db
|
|
157
|
-
.prepare(`SELECT rowid FROM memory_chunks WHERE source = ?`)
|
|
158
|
-
.all(source);
|
|
152
|
+
const rows = this.db.prepare(`SELECT rowid FROM memory_chunks WHERE source = ?`).all(source);
|
|
159
153
|
const deleteAll = this.db.transaction(() => {
|
|
160
154
|
if (this._vecAvailable && rows.length > 0) {
|
|
161
155
|
const placeholders = rows.map(() => '?').join(', ');
|
|
162
|
-
this.db
|
|
163
|
-
.prepare(`DELETE FROM memory_vec WHERE rowid IN (${placeholders})`)
|
|
164
|
-
.run(...rows.map((r) => r.rowid));
|
|
156
|
+
this.db.prepare(`DELETE FROM memory_vec WHERE rowid IN (${placeholders})`).run(...rows.map((r) => r.rowid));
|
|
165
157
|
}
|
|
166
158
|
this.db.prepare(`DELETE FROM memory_chunks WHERE source = ?`).run(source);
|
|
167
159
|
});
|
|
@@ -171,15 +163,11 @@ export class MemoryStore {
|
|
|
171
163
|
if (ids.length === 0)
|
|
172
164
|
return;
|
|
173
165
|
const placeholders = ids.map(() => '?').join(', ');
|
|
174
|
-
const rows = this.db
|
|
175
|
-
.prepare(`SELECT rowid FROM memory_chunks WHERE id IN (${placeholders})`)
|
|
176
|
-
.all(...ids);
|
|
166
|
+
const rows = this.db.prepare(`SELECT rowid FROM memory_chunks WHERE id IN (${placeholders})`).all(...ids);
|
|
177
167
|
const deleteAll = this.db.transaction(() => {
|
|
178
168
|
if (this._vecAvailable && rows.length > 0) {
|
|
179
169
|
const rPlaceholders = rows.map(() => '?').join(', ');
|
|
180
|
-
this.db
|
|
181
|
-
.prepare(`DELETE FROM memory_vec WHERE rowid IN (${rPlaceholders})`)
|
|
182
|
-
.run(...rows.map((r) => r.rowid));
|
|
170
|
+
this.db.prepare(`DELETE FROM memory_vec WHERE rowid IN (${rPlaceholders})`).run(...rows.map((r) => r.rowid));
|
|
183
171
|
}
|
|
184
172
|
this.db.prepare(`DELETE FROM memory_chunks WHERE id IN (${placeholders})`).run(...ids);
|
|
185
173
|
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -3,14 +3,15 @@ import * as fs from 'node:fs';
|
|
|
3
3
|
import * as os from 'node:os';
|
|
4
4
|
import * as path from 'node:path';
|
|
5
5
|
import { after, before, describe, test } from 'node:test';
|
|
6
|
-
import { MemoryStore } from '
|
|
6
|
+
import { MemoryStore } from './store.js';
|
|
7
|
+
import { EmbeddingModelName } from './types.js';
|
|
7
8
|
const TEST_CONFIG = {
|
|
8
9
|
enabled: true,
|
|
9
10
|
wikiDir: '',
|
|
10
11
|
topK: 5,
|
|
11
12
|
chunkSize: 1500,
|
|
12
13
|
overlapLines: 2,
|
|
13
|
-
embeddingModel:
|
|
14
|
+
embeddingModel: EmbeddingModelName.SMALL,
|
|
14
15
|
vectorDimension: 384,
|
|
15
16
|
};
|
|
16
17
|
function makeChunk(overrides = {}) {
|
|
@@ -54,8 +55,16 @@ describe('MemoryStore', () => {
|
|
|
54
55
|
assert.ok(hashes.has('upsert-test-0001'));
|
|
55
56
|
});
|
|
56
57
|
test('deleteBySource removes all chunks for that source', () => {
|
|
57
|
-
const c1 = makeChunk({
|
|
58
|
-
|
|
58
|
+
const c1 = makeChunk({
|
|
59
|
+
id: 'del-src-0001',
|
|
60
|
+
source: 'deleteme.md',
|
|
61
|
+
content: 'delete source chunk 1',
|
|
62
|
+
});
|
|
63
|
+
const c2 = makeChunk({
|
|
64
|
+
id: 'del-src-0002',
|
|
65
|
+
source: 'deleteme.md',
|
|
66
|
+
content: 'delete source chunk 2',
|
|
67
|
+
});
|
|
59
68
|
store.upsert([c1, c2], [new Float32Array(384), new Float32Array(384)]);
|
|
60
69
|
store.deleteBySource('deleteme.md');
|
|
61
70
|
const hashes = store.hashesBySource('deleteme.md');
|
|
@@ -124,7 +133,11 @@ describe('MemoryStore', () => {
|
|
|
124
133
|
assert.equal(results[0].id, 'partial-found-0001');
|
|
125
134
|
});
|
|
126
135
|
test('indexedSources includes source after upsert', () => {
|
|
127
|
-
const chunk = makeChunk({
|
|
136
|
+
const chunk = makeChunk({
|
|
137
|
+
id: 'indexed-src-0001',
|
|
138
|
+
source: 'indexed-source.md',
|
|
139
|
+
content: 'indexed',
|
|
140
|
+
});
|
|
128
141
|
store.upsert([chunk], [new Float32Array(384)]);
|
|
129
142
|
const sources = store.indexedSources();
|
|
130
143
|
assert.ok(sources.includes('indexed-source.md'), `Expected 'indexed-source.md' in ${sources.join(', ')}`);
|
|
@@ -1,3 +1,9 @@
|
|
|
1
|
+
import type { ConversationDigestSettings } from '../digest/types.js';
|
|
2
|
+
export declare enum EmbeddingModelName {
|
|
3
|
+
TINY = "tiny",
|
|
4
|
+
SMALL = "small",
|
|
5
|
+
LARGE = "large"
|
|
6
|
+
}
|
|
1
7
|
export interface MemoryChunk {
|
|
2
8
|
id: string;
|
|
3
9
|
source: string;
|
|
@@ -19,12 +25,12 @@ export interface MemoryConfig {
|
|
|
19
25
|
topK: number;
|
|
20
26
|
chunkSize: number;
|
|
21
27
|
overlapLines: number;
|
|
22
|
-
embeddingModel:
|
|
28
|
+
embeddingModel: EmbeddingModelName;
|
|
23
29
|
vectorDimension: number;
|
|
30
|
+
conversationDigest?: ConversationDigestSettings;
|
|
24
31
|
}
|
|
25
32
|
export interface IndexStats {
|
|
26
33
|
indexed: number;
|
|
27
34
|
deleted: number;
|
|
28
35
|
skipped: number;
|
|
29
36
|
}
|
|
30
|
-
export declare const DEFAULT_MEMORY_CONFIG: MemoryConfig;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function withTimeout<T>(promise: Promise<T>, timeoutMs: number, createError: () => Error): Promise<T>;
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
export async function withTimeout(promise, timeoutMs, createError) {
|
|
2
|
+
let timer;
|
|
3
|
+
const timeout = new Promise((_resolve, reject) => {
|
|
4
|
+
timer = setTimeout(() => {
|
|
5
|
+
reject(createError());
|
|
6
|
+
}, timeoutMs);
|
|
7
|
+
});
|
|
8
|
+
try {
|
|
9
|
+
return await Promise.race([promise, timeout]);
|
|
10
|
+
}
|
|
11
|
+
finally {
|
|
12
|
+
if (timer)
|
|
13
|
+
clearTimeout(timer);
|
|
14
|
+
}
|
|
15
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
export declare function atomicWriteTextFile(filePath: string, content: string): void;
|
|
2
|
+
export declare function atomicWriteJsonFile(filePath: string, value: unknown): void;
|
|
3
|
+
export declare function acquireLock(lockPath: string, timeoutMs: number, retryMs: number): Promise<boolean>;
|
|
4
|
+
export declare function releaseLock(lockPath: string, opts?: {
|
|
5
|
+
ignoreErrors?: boolean;
|
|
6
|
+
}): void;
|
|
7
|
+
export declare function tryAcquireProcessLock(lockPath: string): boolean;
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import * as fs from 'node:fs';
|
|
2
|
+
import * as path from 'node:path';
|
|
3
|
+
export function atomicWriteTextFile(filePath, content) {
|
|
4
|
+
fs.mkdirSync(path.dirname(filePath), { recursive: true });
|
|
5
|
+
const tempPath = `${filePath}.${process.pid}.${Date.now()}.tmp`;
|
|
6
|
+
fs.writeFileSync(tempPath, content, 'utf8');
|
|
7
|
+
fs.renameSync(tempPath, filePath);
|
|
8
|
+
}
|
|
9
|
+
export function atomicWriteJsonFile(filePath, value) {
|
|
10
|
+
atomicWriteTextFile(filePath, `${JSON.stringify(value, null, 2)}\n`);
|
|
11
|
+
}
|
|
12
|
+
function tryWriteFileExclusive(filePath, content) {
|
|
13
|
+
fs.mkdirSync(path.dirname(filePath), { recursive: true });
|
|
14
|
+
try {
|
|
15
|
+
fs.writeFileSync(filePath, content, { flag: 'wx' });
|
|
16
|
+
return true;
|
|
17
|
+
}
|
|
18
|
+
catch (err) {
|
|
19
|
+
if (err.code === 'EEXIST')
|
|
20
|
+
return false;
|
|
21
|
+
throw err;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
export async function acquireLock(lockPath, timeoutMs, retryMs) {
|
|
25
|
+
const deadline = Date.now() + timeoutMs;
|
|
26
|
+
while (Date.now() < deadline) {
|
|
27
|
+
try {
|
|
28
|
+
if (tryWriteFileExclusive(lockPath, String(process.pid)))
|
|
29
|
+
return true;
|
|
30
|
+
}
|
|
31
|
+
catch {
|
|
32
|
+
// Preserve fail-open retry behavior for transient lock-file write errors.
|
|
33
|
+
}
|
|
34
|
+
await new Promise((r) => setTimeout(r, retryMs));
|
|
35
|
+
}
|
|
36
|
+
return false;
|
|
37
|
+
}
|
|
38
|
+
export function releaseLock(lockPath, opts = {}) {
|
|
39
|
+
try {
|
|
40
|
+
fs.unlinkSync(lockPath);
|
|
41
|
+
}
|
|
42
|
+
catch (err) {
|
|
43
|
+
if (opts.ignoreErrors || err.code === 'ENOENT')
|
|
44
|
+
return;
|
|
45
|
+
throw err;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
export function tryAcquireProcessLock(lockPath) {
|
|
49
|
+
const lockContent = JSON.stringify({ pid: process.pid });
|
|
50
|
+
if (tryWriteFileExclusive(lockPath, lockContent))
|
|
51
|
+
return true;
|
|
52
|
+
let pid;
|
|
53
|
+
try {
|
|
54
|
+
const raw = fs.readFileSync(lockPath, 'utf8');
|
|
55
|
+
pid = JSON.parse(raw).pid;
|
|
56
|
+
}
|
|
57
|
+
catch {
|
|
58
|
+
// Unparseable lockfiles are treated as stale.
|
|
59
|
+
}
|
|
60
|
+
if (pid !== undefined) {
|
|
61
|
+
try {
|
|
62
|
+
process.kill(pid, 0);
|
|
63
|
+
return false;
|
|
64
|
+
}
|
|
65
|
+
catch (err) {
|
|
66
|
+
if (err.code !== 'ESRCH')
|
|
67
|
+
return false;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
try {
|
|
71
|
+
fs.unlinkSync(lockPath);
|
|
72
|
+
}
|
|
73
|
+
catch {
|
|
74
|
+
// Ignore if another process already removed the stale lock.
|
|
75
|
+
}
|
|
76
|
+
return tryWriteFileExclusive(lockPath, lockContent);
|
|
77
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|