@hanhnd/agent-kit 1.0.34 → 1.0.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands/cli-digest-pending.test.js +19 -10
- package/dist/cli/commands/memory.js +44 -4
- package/dist/cli.js +348 -179
- package/dist/core/config/index.d.ts +3 -0
- package/dist/core/config/index.js +20 -0
- package/dist/entrypoints/server.js +1 -3
- package/dist/mcp/memory.d.ts +2 -1
- package/dist/mcp/memory.js +45 -18
- package/dist/server.js +294 -200
- package/dist/services/digest/constants.d.ts +3 -3
- package/dist/services/digest/constants.js +3 -3
- package/dist/services/digest/digest-background.test.js +19 -8
- package/dist/services/digest/digest-init.test.js +13 -2
- package/dist/services/digest/digest-pending.test.js +13 -2
- package/dist/services/digest/digest-processor.test.js +52 -0
- package/dist/services/digest/files.js +3 -3
- package/dist/services/digest/model-registry.js +2 -2
- package/dist/services/digest/processor.js +43 -13
- package/dist/services/digest/providers/llama-local.js +17 -28
- package/dist/services/digest/types.d.ts +3 -1
- package/dist/services/digest/types.js +1 -1
- package/dist/services/memory/chunker.d.ts +5 -2
- package/dist/services/memory/chunker.js +3 -1
- package/dist/services/memory/chunker.test.js +21 -11
- package/dist/services/memory/constants.js +1 -1
- package/dist/services/memory/embedder.js +1 -1
- package/dist/services/memory/index.d.ts +1 -1
- package/dist/services/memory/indexer.d.ts +2 -1
- package/dist/services/memory/indexer.js +24 -8
- package/dist/services/memory/indexer.test.js +38 -2
- package/dist/services/memory/memory.test.js +172 -15
- package/dist/services/memory/store.d.ts +9 -3
- package/dist/services/memory/store.js +132 -98
- package/dist/services/memory/store.test.js +239 -14
- package/dist/services/memory/types.d.ts +9 -1
- package/dist/services/memory/types.js +2 -1
- package/dist/utils/paths.d.ts +2 -0
- package/dist/utils/paths.js +2 -1
- package/dist/utils/utils.d.ts +1 -0
- package/dist/utils/utils.js +3 -3
- package/package.json +3 -5
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { DigestModelId } from './types.js';
|
|
2
|
-
export declare const DEFAULT_DIGEST_MODEL_ID = DigestModelId.
|
|
3
|
-
export declare const DEFAULT_DIGEST_MAX_INPUT_CHARS =
|
|
2
|
+
export declare const DEFAULT_DIGEST_MODEL_ID = DigestModelId.BASE;
|
|
3
|
+
export declare const DEFAULT_DIGEST_MAX_INPUT_CHARS = 10000;
|
|
4
4
|
export declare const DEFAULT_DIGEST_TIMEOUT_MS = 120000;
|
|
5
5
|
export declare const DIGEST_TIMEOUT_GRACE_MS = 5000;
|
|
6
6
|
export declare const DIGEST_WORKER_FLAG = "__agent-kit-digest-worker";
|
|
@@ -12,6 +12,6 @@ export declare const PROVISIONAL_DIGEST_DIR: string;
|
|
|
12
12
|
export declare const DIGEST_LOCKFILE_REL_PATH: string;
|
|
13
13
|
export declare const DIGEST_WORKER_STATUS_REL_PATH: string;
|
|
14
14
|
export declare const DIGEST_WORKER_LOG_REL_PATH: string;
|
|
15
|
-
export declare const LLAMA_CONTEXT_SIZE =
|
|
15
|
+
export declare const LLAMA_CONTEXT_SIZE = 8192;
|
|
16
16
|
export declare const LLAMA_MAX_GENERATED_TOKENS = 512;
|
|
17
17
|
export declare const LLAMA_TEMPERATURE = 0.1;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import * as path from 'node:path';
|
|
2
2
|
import { DigestModelId } from './types.js';
|
|
3
|
-
export const DEFAULT_DIGEST_MODEL_ID = DigestModelId.
|
|
4
|
-
export const DEFAULT_DIGEST_MAX_INPUT_CHARS =
|
|
3
|
+
export const DEFAULT_DIGEST_MODEL_ID = DigestModelId.BASE;
|
|
4
|
+
export const DEFAULT_DIGEST_MAX_INPUT_CHARS = 10000;
|
|
5
5
|
export const DEFAULT_DIGEST_TIMEOUT_MS = 120_000;
|
|
6
6
|
export const DIGEST_TIMEOUT_GRACE_MS = 5_000;
|
|
7
7
|
export const DIGEST_WORKER_FLAG = '__agent-kit-digest-worker';
|
|
@@ -14,6 +14,6 @@ const DIGEST_WORKER_DIR = path.join(WIKI_DIR, 'digest');
|
|
|
14
14
|
export const DIGEST_LOCKFILE_REL_PATH = path.join(DIGEST_WORKER_DIR, 'digest-worker.lock');
|
|
15
15
|
export const DIGEST_WORKER_STATUS_REL_PATH = path.join(DIGEST_WORKER_DIR, 'digest-worker.status.json');
|
|
16
16
|
export const DIGEST_WORKER_LOG_REL_PATH = path.join(DIGEST_WORKER_DIR, 'digest-worker.log');
|
|
17
|
-
export const LLAMA_CONTEXT_SIZE =
|
|
17
|
+
export const LLAMA_CONTEXT_SIZE = 8192;
|
|
18
18
|
export const LLAMA_MAX_GENERATED_TOKENS = 512;
|
|
19
19
|
export const LLAMA_TEMPERATURE = 0.1;
|
|
@@ -1,15 +1,26 @@
|
|
|
1
1
|
import * as assert from 'node:assert/strict';
|
|
2
2
|
import * as fs from 'node:fs';
|
|
3
3
|
import * as path from 'node:path';
|
|
4
|
-
import { afterEach, describe, test } from 'node:test';
|
|
4
|
+
import { afterEach, describe, test, beforeEach } from 'node:test';
|
|
5
5
|
import { createTempDirTracker } from '../../utils/temp-dir.test.js';
|
|
6
6
|
import { writeConversationDigestSettings } from './files.js';
|
|
7
7
|
import { launchDigestPendingWorker, runDigestPendingWorker } from './background.js';
|
|
8
8
|
import { DIGEST_WORKER_LOG_REL_PATH, DIGEST_WORKER_STATUS_REL_PATH } from './constants.js';
|
|
9
9
|
import { DigestModelId } from './types.js';
|
|
10
10
|
const tempDirs = createTempDirTracker();
|
|
11
|
+
let originalHome;
|
|
12
|
+
let originalUserProfile;
|
|
13
|
+
beforeEach(() => {
|
|
14
|
+
originalHome = process.env.HOME;
|
|
15
|
+
originalUserProfile = process.env.USERPROFILE;
|
|
16
|
+
const mockHome = tempDirs.makeTempDir('mock-home-');
|
|
17
|
+
process.env.HOME = mockHome;
|
|
18
|
+
process.env.USERPROFILE = mockHome;
|
|
19
|
+
});
|
|
11
20
|
afterEach(() => {
|
|
12
21
|
tempDirs.cleanup();
|
|
22
|
+
process.env.HOME = originalHome;
|
|
23
|
+
process.env.USERPROFILE = originalUserProfile;
|
|
13
24
|
});
|
|
14
25
|
function digestWorkerLogPath(workspace) {
|
|
15
26
|
return path.join(workspace, DIGEST_WORKER_LOG_REL_PATH);
|
|
@@ -35,7 +46,7 @@ function writeInitState(workspace) {
|
|
|
35
46
|
writeConversationDigestSettings(workspace, {
|
|
36
47
|
enabled: true,
|
|
37
48
|
initialized: true,
|
|
38
|
-
modelId: DigestModelId.
|
|
49
|
+
modelId: DigestModelId.BASE,
|
|
39
50
|
initializedAt: new Date().toISOString(),
|
|
40
51
|
});
|
|
41
52
|
}
|
|
@@ -74,7 +85,7 @@ describe('digest background helpers', () => {
|
|
|
74
85
|
});
|
|
75
86
|
});
|
|
76
87
|
describe('launchDigestPendingWorker', () => {
|
|
77
|
-
test('writes not-initialized and does not spawn before digest init', () => {
|
|
88
|
+
test('writes not-initialized and does not spawn before digest init', async () => {
|
|
78
89
|
const workspace = tempDirs.makeTempDir('digest-bg-not-init-');
|
|
79
90
|
const result = launchDigestPendingWorker({
|
|
80
91
|
workspaceRoot: workspace,
|
|
@@ -86,7 +97,7 @@ describe('launchDigestPendingWorker', () => {
|
|
|
86
97
|
assert.equal(result.status.state, 'not-initialized');
|
|
87
98
|
assert.equal(readDigestWorkerStatus(workspace)?.state, 'not-initialized');
|
|
88
99
|
});
|
|
89
|
-
test('writes no-pending and does not spawn when initialized with no candidates', () => {
|
|
100
|
+
test('writes no-pending and does not spawn when initialized with no candidates', async () => {
|
|
90
101
|
const workspace = tempDirs.makeTempDir('digest-bg-no-pending-');
|
|
91
102
|
writeInitState(workspace);
|
|
92
103
|
const result = launchDigestPendingWorker({
|
|
@@ -98,7 +109,7 @@ describe('launchDigestPendingWorker', () => {
|
|
|
98
109
|
assert.equal(result.spawned, false);
|
|
99
110
|
assert.equal(result.status.state, 'no-pending');
|
|
100
111
|
});
|
|
101
|
-
test('writes locked when an existing running worker pid is live', () => {
|
|
112
|
+
test('writes locked when an existing running worker pid is live', async () => {
|
|
102
113
|
const workspace = tempDirs.makeTempDir('digest-bg-locked-status-');
|
|
103
114
|
writeDigestWorkerStatus(workspace, status());
|
|
104
115
|
const result = launchDigestPendingWorker({
|
|
@@ -109,7 +120,7 @@ describe('launchDigestPendingWorker', () => {
|
|
|
109
120
|
assert.equal(result.spawned, false);
|
|
110
121
|
assert.equal(result.status.state, 'locked');
|
|
111
122
|
});
|
|
112
|
-
test('marks stale running status before continuing to no-pending preflight', () => {
|
|
123
|
+
test('marks stale running status before continuing to no-pending preflight', async () => {
|
|
113
124
|
const workspace = tempDirs.makeTempDir('digest-bg-stale-status-');
|
|
114
125
|
writeInitState(workspace);
|
|
115
126
|
writeDigestWorkerStatus(workspace, status({ pid: 999999999 }));
|
|
@@ -121,7 +132,7 @@ describe('launchDigestPendingWorker', () => {
|
|
|
121
132
|
assert.equal(result.spawned, false);
|
|
122
133
|
assert.equal(result.status.state, 'no-pending');
|
|
123
134
|
});
|
|
124
|
-
test('tolerates corrupt stale lock files and continues preflight', () => {
|
|
135
|
+
test('tolerates corrupt stale lock files and continues preflight', async () => {
|
|
125
136
|
const workspace = tempDirs.makeTempDir('digest-bg-stale-lock-');
|
|
126
137
|
writeInitState(workspace);
|
|
127
138
|
const lockDir = path.join(workspace, '.agent-kit', 'wiki', 'digest');
|
|
@@ -135,7 +146,7 @@ describe('launchDigestPendingWorker', () => {
|
|
|
135
146
|
assert.equal(result.spawned, false);
|
|
136
147
|
assert.equal(result.status.state, 'no-pending');
|
|
137
148
|
});
|
|
138
|
-
test('writes failed status when pending work exists but entrypoint is unavailable', () => {
|
|
149
|
+
test('writes failed status when pending work exists but entrypoint is unavailable', async () => {
|
|
139
150
|
const workspace = tempDirs.makeTempDir('digest-bg-spawn-failed-');
|
|
140
151
|
writeInitState(workspace);
|
|
141
152
|
writeConvFile(workspace);
|
|
@@ -1,13 +1,24 @@
|
|
|
1
1
|
import * as assert from 'node:assert/strict';
|
|
2
2
|
import * as fs from 'node:fs';
|
|
3
3
|
import * as path from 'node:path';
|
|
4
|
-
import { afterEach, describe, test } from 'node:test';
|
|
4
|
+
import { afterEach, describe, test, beforeEach } from 'node:test';
|
|
5
5
|
import { createTempDirTracker } from '../../utils/temp-dir.test.js';
|
|
6
6
|
import { initializeConversationDigestModel } from './processor.js';
|
|
7
7
|
import { DigestModelId } from './types.js';
|
|
8
8
|
const tempDirs = createTempDirTracker();
|
|
9
|
+
let originalHome;
|
|
10
|
+
let originalUserProfile;
|
|
11
|
+
beforeEach(() => {
|
|
12
|
+
originalHome = process.env.HOME;
|
|
13
|
+
originalUserProfile = process.env.USERPROFILE;
|
|
14
|
+
const mockHome = tempDirs.makeTempDir('mock-home-');
|
|
15
|
+
process.env.HOME = mockHome;
|
|
16
|
+
process.env.USERPROFILE = mockHome;
|
|
17
|
+
});
|
|
9
18
|
afterEach(() => {
|
|
10
19
|
tempDirs.cleanup();
|
|
20
|
+
process.env.HOME = originalHome;
|
|
21
|
+
process.env.USERPROFILE = originalUserProfile;
|
|
11
22
|
});
|
|
12
23
|
describe('initializeConversationDigestModel', () => {
|
|
13
24
|
test('rejects unknown model and leaves settings absent', async () => {
|
|
@@ -24,7 +35,7 @@ describe('initializeConversationDigestModel', () => {
|
|
|
24
35
|
const workspace = tempDirs.makeTempDir('digest-init-');
|
|
25
36
|
const result = await initializeConversationDigestModel({
|
|
26
37
|
workspaceRoot: workspace,
|
|
27
|
-
modelId: DigestModelId.
|
|
38
|
+
modelId: DigestModelId.BASE,
|
|
28
39
|
allowDownload: false,
|
|
29
40
|
});
|
|
30
41
|
assert.equal(result.initialized, false);
|
|
@@ -1,16 +1,27 @@
|
|
|
1
1
|
import * as assert from 'node:assert/strict';
|
|
2
2
|
import * as fs from 'node:fs';
|
|
3
3
|
import * as path from 'node:path';
|
|
4
|
-
import { afterEach, describe, test } from 'node:test';
|
|
4
|
+
import { afterEach, describe, test, beforeEach } from 'node:test';
|
|
5
5
|
import { createTempDirTracker } from '../../utils/temp-dir.test.js';
|
|
6
6
|
import { defaultProvisionalDigestDir, readConversationDigestInput, writeConversationDigestSettings, writeProvisionalDigestFile, } from './files.js';
|
|
7
7
|
import { DigestModelId } from './types.js';
|
|
8
8
|
import { digestPendingConversations, summarizePendingConversations } from './processor.js';
|
|
9
9
|
const tempDirs = createTempDirTracker();
|
|
10
|
+
let originalHome;
|
|
11
|
+
let originalUserProfile;
|
|
12
|
+
beforeEach(() => {
|
|
13
|
+
originalHome = process.env.HOME;
|
|
14
|
+
originalUserProfile = process.env.USERPROFILE;
|
|
15
|
+
const mockHome = tempDirs.makeTempDir('mock-home-');
|
|
16
|
+
process.env.HOME = mockHome;
|
|
17
|
+
process.env.USERPROFILE = mockHome;
|
|
18
|
+
});
|
|
10
19
|
afterEach(() => {
|
|
11
20
|
tempDirs.cleanup();
|
|
21
|
+
process.env.HOME = originalHome;
|
|
22
|
+
process.env.USERPROFILE = originalUserProfile;
|
|
12
23
|
});
|
|
13
|
-
function writeInitState(workspace, modelId = DigestModelId.
|
|
24
|
+
function writeInitState(workspace, modelId = DigestModelId.BASE) {
|
|
14
25
|
writeConversationDigestSettings(workspace, {
|
|
15
26
|
enabled: true,
|
|
16
27
|
initialized: true,
|
|
@@ -6,10 +6,32 @@ import { createTempDirTracker } from '../../utils/temp-dir.test.js';
|
|
|
6
6
|
import { DEFAULT_DIGEST_MODEL_ID } from './constants.js';
|
|
7
7
|
import { defaultProvisionalDigestDir, writeProvisionalDigestFile, readConversationDigestInput } from './files.js';
|
|
8
8
|
import { digestConversationFile } from './processor.js';
|
|
9
|
+
import { MemoryIndexer } from '../memory/indexer.js';
|
|
10
|
+
import { MemoryStore } from '../memory/store.js';
|
|
11
|
+
import { EmbeddingModelName } from '../memory/types.js';
|
|
9
12
|
const tempDirs = createTempDirTracker();
|
|
10
13
|
afterEach(() => {
|
|
11
14
|
tempDirs.cleanup();
|
|
12
15
|
});
|
|
16
|
+
class StubEmbedder {
|
|
17
|
+
async embed(texts) {
|
|
18
|
+
return texts.map(() => new Float32Array(384).fill(0.05));
|
|
19
|
+
}
|
|
20
|
+
initialize() {
|
|
21
|
+
return Promise.resolve();
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
function makeConfig(workspace) {
|
|
25
|
+
return {
|
|
26
|
+
enabled: true,
|
|
27
|
+
wikiDir: path.join(workspace, '.agent-kit', 'wiki'),
|
|
28
|
+
topK: 5,
|
|
29
|
+
chunkSize: 1500,
|
|
30
|
+
overlapLines: 2,
|
|
31
|
+
embeddingModel: EmbeddingModelName.BASE,
|
|
32
|
+
vectorDimension: 384,
|
|
33
|
+
};
|
|
34
|
+
}
|
|
13
35
|
describe('digestConversationFile', () => {
|
|
14
36
|
test('returns existing content-hash named provisional digest without loading model', async () => {
|
|
15
37
|
const workspace = tempDirs.makeTempDir('digest-processor-');
|
|
@@ -29,4 +51,34 @@ describe('digestConversationFile', () => {
|
|
|
29
51
|
assert.equal(result.contentHash, input.contentHash);
|
|
30
52
|
assert.match(path.basename(result.markdown), /^[a-f0-9]{16}-conv\.md$/);
|
|
31
53
|
});
|
|
54
|
+
test('indexes existing provisional digest with searchable store vector data', async () => {
|
|
55
|
+
const workspace = tempDirs.makeTempDir('digest-processor-index-');
|
|
56
|
+
const inputPath = path.join(workspace, '.agent-kit', 'wiki', 'archive', 'conversations', 'conv.md');
|
|
57
|
+
fs.mkdirSync(path.dirname(inputPath), { recursive: true });
|
|
58
|
+
fs.writeFileSync(inputPath, '**User:** remember vector indexing\n\n**Assistant:** done', 'utf8');
|
|
59
|
+
const input = readConversationDigestInput(workspace, inputPath);
|
|
60
|
+
const outDir = defaultProvisionalDigestDir(workspace);
|
|
61
|
+
const markdownPath = writeProvisionalDigestFile(outDir, input, '# Conversation Digest: conv\nindexed digest content\n');
|
|
62
|
+
const config = makeConfig(workspace);
|
|
63
|
+
const dbPath = path.join(config.wikiDir, 'index.db');
|
|
64
|
+
const store = new MemoryStore(dbPath, config);
|
|
65
|
+
const indexer = new MemoryIndexer(store, new StubEmbedder(), config);
|
|
66
|
+
try {
|
|
67
|
+
const result = await digestConversationFile({
|
|
68
|
+
workspaceRoot: workspace,
|
|
69
|
+
inputPath,
|
|
70
|
+
modelId: DEFAULT_DIGEST_MODEL_ID,
|
|
71
|
+
indexer,
|
|
72
|
+
});
|
|
73
|
+
assert.equal(result.indexed, true);
|
|
74
|
+
assert.equal(result.markdown, markdownPath);
|
|
75
|
+
const denseResults = store.searchDense(new Float32Array(384).fill(0.05), 5);
|
|
76
|
+
const indexedSource = path.relative(config.wikiDir, markdownPath);
|
|
77
|
+
const indexedChunks = store.getChunksByIds(denseResults.map((denseResult) => denseResult.id));
|
|
78
|
+
assert.ok(indexedChunks.some((chunk) => chunk.source === indexedSource), `Expected dense search to return a chunk for ${indexedSource}`);
|
|
79
|
+
}
|
|
80
|
+
finally {
|
|
81
|
+
store.close();
|
|
82
|
+
}
|
|
83
|
+
});
|
|
32
84
|
});
|
|
@@ -3,7 +3,7 @@ import * as path from 'node:path';
|
|
|
3
3
|
import { PROVISIONAL_DIGEST_DIR } from './constants.js';
|
|
4
4
|
import { atomicWriteTextFile } from '../../utils/files.js';
|
|
5
5
|
import { sha256Hex } from '../../utils/hash.js';
|
|
6
|
-
import {
|
|
6
|
+
import { loadGlobalSettings, writeGlobalSettings } from '../../core/config/index.js';
|
|
7
7
|
export function defaultProvisionalDigestDir(workspaceRoot) {
|
|
8
8
|
return path.join(workspaceRoot, PROVISIONAL_DIGEST_DIR);
|
|
9
9
|
}
|
|
@@ -33,9 +33,9 @@ export function writeProvisionalDigestFile(outDir, input, markdown) {
|
|
|
33
33
|
return markdownPath;
|
|
34
34
|
}
|
|
35
35
|
export function writeConversationDigestSettings(workspaceRoot, digest) {
|
|
36
|
-
const current =
|
|
36
|
+
const current = loadGlobalSettings();
|
|
37
37
|
const memory = current.memory ?? {};
|
|
38
|
-
|
|
38
|
+
writeGlobalSettings({
|
|
39
39
|
memory: {
|
|
40
40
|
...memory,
|
|
41
41
|
enabled: memory.enabled ?? true,
|
|
@@ -14,8 +14,8 @@ const DIGEST_MODEL_REGISTRY = {
|
|
|
14
14
|
sourceUrl: 'https://huggingface.co/bartowski/Qwen2.5-0.5B-Instruct-GGUF',
|
|
15
15
|
enabled: true,
|
|
16
16
|
},
|
|
17
|
-
[DigestModelId.
|
|
18
|
-
id: DigestModelId.
|
|
17
|
+
[DigestModelId.BASE]: {
|
|
18
|
+
id: DigestModelId.BASE,
|
|
19
19
|
ggufUri: 'hf:bartowski/Qwen2.5-1.5B-Instruct-GGUF/Qwen2.5-1.5B-Instruct-Q4_K_M.gguf',
|
|
20
20
|
approxSizeBytes: 1_000_000_000,
|
|
21
21
|
license: 'Apache-2.0',
|
|
@@ -4,9 +4,10 @@ import { DEFAULT_DIGEST_TIMEOUT_MS, DEFAULT_DIGEST_MAX_INPUT_CHARS, DIGEST_LOCKF
|
|
|
4
4
|
import { defaultProvisionalDigestDir, readConversationDigestInput, findExistingProvisionalDigest, writeProvisionalDigestFile, writeConversationDigestSettings, } from './files.js';
|
|
5
5
|
import { getDigestModelSpec } from './model-registry.js';
|
|
6
6
|
import { createLlamaLocalDigestProvider } from './providers/llama-local.js';
|
|
7
|
-
import {
|
|
7
|
+
import { loadGlobalSettings, resolveConversationDigestConfig, resolveMemoryConfig } from '../../core/config/index.js';
|
|
8
8
|
import { MemoryStore } from '../memory/store.js';
|
|
9
9
|
import { MemoryIndexer } from '../memory/indexer.js';
|
|
10
|
+
import { Embedder } from '../memory/embedder.js';
|
|
10
11
|
import { releaseLock, tryAcquireProcessLock } from '../../utils/files.js';
|
|
11
12
|
function digestLockPath(workspaceRoot) {
|
|
12
13
|
return path.join(workspaceRoot, DIGEST_LOCKFILE_REL_PATH);
|
|
@@ -40,19 +41,28 @@ function discoverPendingConversationFiles(workspaceRoot) {
|
|
|
40
41
|
}
|
|
41
42
|
});
|
|
42
43
|
}
|
|
43
|
-
async function indexProvisionalDigestFile(workspaceRoot, markdownPath) {
|
|
44
|
+
async function indexProvisionalDigestFile(workspaceRoot, markdownPath, indexer) {
|
|
45
|
+
let store;
|
|
44
46
|
try {
|
|
45
|
-
|
|
47
|
+
if (indexer) {
|
|
48
|
+
await indexer.indexFile(markdownPath);
|
|
49
|
+
return { indexed: true };
|
|
50
|
+
}
|
|
51
|
+
const settings = loadGlobalSettings();
|
|
46
52
|
const config = resolveMemoryConfig(settings, workspaceRoot);
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
53
|
+
if (config.enabled !== true)
|
|
54
|
+
return { indexed: false };
|
|
55
|
+
store = new MemoryStore(path.join(config.wikiDir, 'index.db'), config);
|
|
56
|
+
const embedder = new Embedder(config.embeddingModel);
|
|
57
|
+
await embedder.initialize();
|
|
58
|
+
await new MemoryIndexer(store, embedder, config).indexFile(markdownPath);
|
|
51
59
|
return { indexed: true };
|
|
52
60
|
}
|
|
53
61
|
catch (err) {
|
|
54
|
-
|
|
55
|
-
|
|
62
|
+
return { indexed: false, error: err instanceof Error ? err.message : String(err) };
|
|
63
|
+
}
|
|
64
|
+
finally {
|
|
65
|
+
store?.close();
|
|
56
66
|
}
|
|
57
67
|
}
|
|
58
68
|
export async function digestConversationFile(options) {
|
|
@@ -62,7 +72,7 @@ export async function digestConversationFile(options) {
|
|
|
62
72
|
const input = readConversationDigestInput(options.workspaceRoot, options.inputPath);
|
|
63
73
|
const existingPath = findExistingProvisionalDigest(outDir, input);
|
|
64
74
|
if (existingPath) {
|
|
65
|
-
const indexResult = await indexProvisionalDigestFile(options.workspaceRoot, existingPath);
|
|
75
|
+
const indexResult = await indexProvisionalDigestFile(options.workspaceRoot, existingPath, options.indexer);
|
|
66
76
|
return {
|
|
67
77
|
markdown: existingPath,
|
|
68
78
|
status: 'provisional',
|
|
@@ -79,7 +89,7 @@ export async function digestConversationFile(options) {
|
|
|
79
89
|
timeoutMs: options.timeoutMs ?? DEFAULT_DIGEST_TIMEOUT_MS,
|
|
80
90
|
});
|
|
81
91
|
const markdownPath = writeProvisionalDigestFile(outDir, input, markdown);
|
|
82
|
-
const indexResult = await indexProvisionalDigestFile(options.workspaceRoot, markdownPath);
|
|
92
|
+
const indexResult = await indexProvisionalDigestFile(options.workspaceRoot, markdownPath, options.indexer);
|
|
83
93
|
return {
|
|
84
94
|
markdown: markdownPath,
|
|
85
95
|
status: 'provisional',
|
|
@@ -93,7 +103,7 @@ export async function digestConversationFile(options) {
|
|
|
93
103
|
}
|
|
94
104
|
}
|
|
95
105
|
export function summarizePendingConversations(workspaceRoot) {
|
|
96
|
-
const digestConfig = resolveConversationDigestConfig(
|
|
106
|
+
const digestConfig = resolveConversationDigestConfig(loadGlobalSettings());
|
|
97
107
|
if (!digestConfig || digestConfig.enabled === false) {
|
|
98
108
|
return { initialized: false, pending: 0, reason: 'not-initialized' };
|
|
99
109
|
}
|
|
@@ -104,7 +114,7 @@ export function summarizePendingConversations(workspaceRoot) {
|
|
|
104
114
|
return { initialized: true, pending: candidates.length };
|
|
105
115
|
}
|
|
106
116
|
export async function digestPendingConversations({ workspaceRoot, digestFn = digestConversationFile, onProgress, }) {
|
|
107
|
-
const digestConfig = resolveConversationDigestConfig(
|
|
117
|
+
const digestConfig = resolveConversationDigestConfig(loadGlobalSettings());
|
|
108
118
|
if (!digestConfig || digestConfig.enabled === false) {
|
|
109
119
|
return { ok: true, initialized: false, action: 'noop', reason: 'not-initialized' };
|
|
110
120
|
}
|
|
@@ -112,11 +122,21 @@ export async function digestPendingConversations({ workspaceRoot, digestFn = dig
|
|
|
112
122
|
if (!tryAcquireProcessLock(lockPath)) {
|
|
113
123
|
return { ok: true, initialized: true, action: 'noop', reason: 'locked' };
|
|
114
124
|
}
|
|
125
|
+
let closeIndexer;
|
|
115
126
|
try {
|
|
116
127
|
const candidates = discoverPendingConversationFiles(workspaceRoot);
|
|
117
128
|
if (candidates.length === 0) {
|
|
118
129
|
return { ok: true, initialized: true, action: 'noop', reason: 'no-pending' };
|
|
119
130
|
}
|
|
131
|
+
let indexer;
|
|
132
|
+
const memoryConfig = resolveMemoryConfig(loadGlobalSettings(), workspaceRoot);
|
|
133
|
+
if (memoryConfig.enabled === true) {
|
|
134
|
+
const store = new MemoryStore(path.join(memoryConfig.wikiDir, 'index.db'), memoryConfig);
|
|
135
|
+
const embedder = new Embedder(memoryConfig.embeddingModel);
|
|
136
|
+
indexer = new MemoryIndexer(store, embedder, memoryConfig);
|
|
137
|
+
closeIndexer = () => store.close();
|
|
138
|
+
await embedder.initialize();
|
|
139
|
+
}
|
|
120
140
|
let count = 0;
|
|
121
141
|
let skipped = 0;
|
|
122
142
|
let errors = 0;
|
|
@@ -126,6 +146,7 @@ export async function digestPendingConversations({ workspaceRoot, digestFn = dig
|
|
|
126
146
|
workspaceRoot,
|
|
127
147
|
inputPath: filePath,
|
|
128
148
|
modelId: digestConfig.modelId,
|
|
149
|
+
indexer,
|
|
129
150
|
});
|
|
130
151
|
if (result.skipped) {
|
|
131
152
|
skipped++;
|
|
@@ -145,10 +166,19 @@ export async function digestPendingConversations({ workspaceRoot, digestFn = dig
|
|
|
145
166
|
}
|
|
146
167
|
catch (err) {
|
|
147
168
|
const message = err instanceof Error ? err.message : String(err);
|
|
169
|
+
console.warn('[digest] Pending conversation digest failed:', err);
|
|
148
170
|
return { ok: false, initialized: true, action: 'error', error: message };
|
|
149
171
|
}
|
|
150
172
|
finally {
|
|
151
173
|
releaseLock(lockPath);
|
|
174
|
+
if (closeIndexer) {
|
|
175
|
+
try {
|
|
176
|
+
closeIndexer();
|
|
177
|
+
}
|
|
178
|
+
catch (err) {
|
|
179
|
+
console.warn('[digest] Failed to close memory indexer store:', err);
|
|
180
|
+
}
|
|
181
|
+
}
|
|
152
182
|
}
|
|
153
183
|
}
|
|
154
184
|
export async function initializeConversationDigestModel(input) {
|
|
@@ -32,39 +32,19 @@ function buildPrompt(input, maxInputChars) {
|
|
|
32
32
|
{
|
|
33
33
|
role: 'system',
|
|
34
34
|
content: [
|
|
35
|
-
'
|
|
36
|
-
'
|
|
37
|
-
'
|
|
38
|
-
'',
|
|
39
|
-
'# RULES',
|
|
40
|
-
'1. Read the end of the export FIRST to discover the ultimate resolution.',
|
|
41
|
-
'2. What the user says LAST ALWAYS overrides earlier agreements.',
|
|
42
|
-
'3. Capture absolute engineering conclusions, never conversational pleasantries.',
|
|
43
|
-
'4. Return Markdown only. Do not wrap output in code blocks (```markdown ... ```).',
|
|
35
|
+
'You are a technical summarizer.',
|
|
36
|
+
'Focus ONLY on the final decisions made at the end of the transcript.',
|
|
37
|
+
'Never invent or guess details.',
|
|
38
|
+
'Keep the output concise and strictly factual.',
|
|
44
39
|
].join('\n'),
|
|
45
40
|
},
|
|
46
41
|
{
|
|
47
42
|
role: 'user',
|
|
48
43
|
content: [
|
|
49
|
-
'
|
|
50
|
-
'',
|
|
51
|
-
'<layout>',
|
|
52
|
-
'# Digest: ' + titleFromSource(input.sourcePath),
|
|
53
|
-
'- **Ultimate Core Resolution**: [Describe the final working state of the feature and what problem it solves in 1-2 sentences. No conversational filler.]',
|
|
54
|
-
'- **Architectural Changes**: [List 3-4 high-level engineering decisions made (e.g., shifts in hook timing, data isolation, atomic locks). Focus on systemic changes, NOT individual files.]',
|
|
55
|
-
'- **Component State**: [Briefly state how the Plugin side and the MCP/CLI side now interact based on the final decision.]',
|
|
56
|
-
'- **Considered & Rejected**: [List concepts proposed but discarded (e.g., initial hooks location, mocking strategies). Omit if empty.]',
|
|
57
|
-
'</layout>',
|
|
58
|
-
'',
|
|
59
|
-
'# CRITICAL RULES:',
|
|
60
|
-
'- DO NOT list individual file paths, line numbers, or specific test cases (EXCLUDE lists of files).',
|
|
61
|
-
'- Focus entirely on the SYSTEM DESIGN and ARCHITECTURE that the next engineer needs to know.',
|
|
62
|
-
'- Keep output under 300 tokens. Stop immediately after the last valid section.',
|
|
63
|
-
'',
|
|
64
|
-
'Source path: ' + input.sourcePath,
|
|
65
|
-
'<conversation_export format="memory-kit">',
|
|
44
|
+
'Transcript:',
|
|
66
45
|
conversationExport,
|
|
67
|
-
'
|
|
46
|
+
'',
|
|
47
|
+
'Task: Write a brief summary paragraph of the final agreed-upon decisions in the transcript above. Then, provide a simple bulleted list of the specific technical changes or outcomes.',
|
|
68
48
|
].join('\n'),
|
|
69
49
|
},
|
|
70
50
|
];
|
|
@@ -125,7 +105,16 @@ export async function createLlamaLocalDigestProvider(modelId) {
|
|
|
125
105
|
maxTokens: LLAMA_MAX_GENERATED_TOKENS,
|
|
126
106
|
temperature: LLAMA_TEMPERATURE,
|
|
127
107
|
}), options.timeoutMs, () => new LlamaLocalDigestProviderError(`Llama provider timed out after ${options.timeoutMs}ms`));
|
|
128
|
-
|
|
108
|
+
const metadata = `## Digest: ${titleFromSource(input.sourcePath)}
|
|
109
|
+
|
|
110
|
+
| Key | Value |
|
|
111
|
+
| ------ | ----- |
|
|
112
|
+
| **Source** | ${input.sourcePath} |
|
|
113
|
+
| **Generated** | ${new Date().toISOString().split('T')[0]} |
|
|
114
|
+
| **Model** | ${modelId} |
|
|
115
|
+
`;
|
|
116
|
+
const sanitized = sanitizeConversationDigestMarkdown(response);
|
|
117
|
+
return `${metadata}\n\n${sanitized}`;
|
|
129
118
|
}
|
|
130
119
|
finally {
|
|
131
120
|
await context.dispose();
|
|
@@ -1,6 +1,7 @@
|
|
|
1
|
+
import type { MemoryIndexer } from '../memory/indexer.js';
|
|
1
2
|
export declare enum DigestModelId {
|
|
2
3
|
TINY = "tiny",
|
|
3
|
-
|
|
4
|
+
BASE = "base",
|
|
4
5
|
LARGE = "large"
|
|
5
6
|
}
|
|
6
7
|
export interface ConversationDigestSettings {
|
|
@@ -45,6 +46,7 @@ export interface DigestFileOptions {
|
|
|
45
46
|
outDir?: string;
|
|
46
47
|
maxInputChars?: number;
|
|
47
48
|
timeoutMs?: number;
|
|
49
|
+
indexer?: MemoryIndexer;
|
|
48
50
|
}
|
|
49
51
|
export interface ProvisionalDigestResult {
|
|
50
52
|
markdown: string;
|
|
@@ -1,2 +1,5 @@
|
|
|
1
|
-
import type { MemoryChunk, MemoryConfig } from './types.js';
|
|
2
|
-
export declare function chunkMarkdown(text: string, source: string, config: Pick<MemoryConfig, 'chunkSize' | 'overlapLines'
|
|
1
|
+
import type { MemoryChunk, MemoryConfig, SourceType } from './types.js';
|
|
2
|
+
export declare function chunkMarkdown(text: string, source: string, config: Pick<MemoryConfig, 'chunkSize' | 'overlapLines'>, meta: {
|
|
3
|
+
sourceType: SourceType;
|
|
4
|
+
fileMtimeAt: number;
|
|
5
|
+
}): MemoryChunk[];
|
|
@@ -85,7 +85,7 @@ function splitSection(text, maxLen) {
|
|
|
85
85
|
return byPara;
|
|
86
86
|
return byPara.flatMap((p) => (p.length <= maxLen ? [p] : splitAtLine(p, maxLen)));
|
|
87
87
|
}
|
|
88
|
-
export function chunkMarkdown(text, source, config) {
|
|
88
|
+
export function chunkMarkdown(text, source, config, meta) {
|
|
89
89
|
if (!text || !text.trim())
|
|
90
90
|
return [];
|
|
91
91
|
const { chunkSize, overlapLines } = config;
|
|
@@ -129,11 +129,13 @@ export function chunkMarkdown(text, source, config) {
|
|
|
129
129
|
chunks.push({
|
|
130
130
|
id: computeChunkId(content),
|
|
131
131
|
source,
|
|
132
|
+
sourceType: meta.sourceType,
|
|
132
133
|
heading: section.heading,
|
|
133
134
|
headingLevel: section.headingLevel,
|
|
134
135
|
content,
|
|
135
136
|
lineStart: firstLine,
|
|
136
137
|
lineEnd: lastLine,
|
|
138
|
+
fileMtimeAt: meta.fileMtimeAt,
|
|
137
139
|
});
|
|
138
140
|
// Update overlap for next iteration (last N lines of this chunk)
|
|
139
141
|
if (pi === parts.length - 1) {
|
|
@@ -3,16 +3,17 @@ import { describe, test } from 'node:test';
|
|
|
3
3
|
import { chunkMarkdown } from './chunker.js';
|
|
4
4
|
const CFG = { chunkSize: 200, overlapLines: 2 };
|
|
5
5
|
const SRC = 'test.md';
|
|
6
|
+
const META = { sourceType: 'wiki', fileMtimeAt: 123 };
|
|
6
7
|
describe('chunkMarkdown', () => {
|
|
7
8
|
test('returns [] for empty string', () => {
|
|
8
|
-
assert.deepEqual(chunkMarkdown('', SRC, CFG), []);
|
|
9
|
+
assert.deepEqual(chunkMarkdown('', SRC, CFG, META), []);
|
|
9
10
|
});
|
|
10
11
|
test('returns [] for whitespace-only string', () => {
|
|
11
|
-
assert.deepEqual(chunkMarkdown(' \n\n ', SRC, CFG), []);
|
|
12
|
+
assert.deepEqual(chunkMarkdown(' \n\n ', SRC, CFG, META), []);
|
|
12
13
|
});
|
|
13
14
|
test('single heading + short body produces one chunk with correct metadata', () => {
|
|
14
15
|
const text = '# My Heading\nThis is the body text.';
|
|
15
|
-
const chunks = chunkMarkdown(text, SRC, CFG);
|
|
16
|
+
const chunks = chunkMarkdown(text, SRC, CFG, META);
|
|
16
17
|
assert.equal(chunks.length, 1);
|
|
17
18
|
const [c] = chunks;
|
|
18
19
|
assert.equal(c.heading, 'My Heading');
|
|
@@ -23,20 +24,20 @@ describe('chunkMarkdown', () => {
|
|
|
23
24
|
});
|
|
24
25
|
test('chunk id is deterministic — same content produces same id', () => {
|
|
25
26
|
const text = '# Section\nHello world.';
|
|
26
|
-
const [a] = chunkMarkdown(text, SRC, CFG);
|
|
27
|
-
const [b] = chunkMarkdown(text, 'other-source.md', CFG);
|
|
27
|
+
const [a] = chunkMarkdown(text, SRC, CFG, META);
|
|
28
|
+
const [b] = chunkMarkdown(text, 'other-source.md', CFG, META);
|
|
28
29
|
assert.equal(a.id, b.id, 'id must depend on content only, not source');
|
|
29
30
|
});
|
|
30
31
|
test('chunk id changes when content changes', () => {
|
|
31
|
-
const [a] = chunkMarkdown('# H\nVersion A', SRC, CFG);
|
|
32
|
-
const [b] = chunkMarkdown('# H\nVersion B', SRC, CFG);
|
|
32
|
+
const [a] = chunkMarkdown('# H\nVersion A', SRC, CFG, META);
|
|
33
|
+
const [b] = chunkMarkdown('# H\nVersion B', SRC, CFG, META);
|
|
33
34
|
assert.notEqual(a.id, b.id);
|
|
34
35
|
});
|
|
35
36
|
test('body exceeding chunkSize produces multiple chunks', () => {
|
|
36
37
|
const word = 'a'.repeat(50);
|
|
37
38
|
const body = Array.from({ length: 10 }, (_, i) => `Paragraph ${i}: ${word}`).join('\n\n');
|
|
38
39
|
const text = `# Big Section\n${body}`;
|
|
39
|
-
const chunks = chunkMarkdown(text, SRC, { chunkSize: 100, overlapLines: 0 });
|
|
40
|
+
const chunks = chunkMarkdown(text, SRC, { chunkSize: 100, overlapLines: 0 }, META);
|
|
40
41
|
assert.ok(chunks.length > 1, `Expected >1 chunks, got ${chunks.length}`);
|
|
41
42
|
for (const c of chunks) {
|
|
42
43
|
assert.equal(c.heading, 'Big Section');
|
|
@@ -51,7 +52,7 @@ describe('chunkMarkdown', () => {
|
|
|
51
52
|
'### Deep Level',
|
|
52
53
|
'Deep content.',
|
|
53
54
|
].join('\n');
|
|
54
|
-
const chunks = chunkMarkdown(text, SRC, CFG);
|
|
55
|
+
const chunks = chunkMarkdown(text, SRC, CFG, META);
|
|
55
56
|
const levels = chunks.map((c) => c.headingLevel);
|
|
56
57
|
assert.ok(levels.includes(1), 'Expected headingLevel 1');
|
|
57
58
|
assert.ok(levels.includes(2), 'Expected headingLevel 2');
|
|
@@ -59,14 +60,23 @@ describe('chunkMarkdown', () => {
|
|
|
59
60
|
});
|
|
60
61
|
test('HTML comment is stripped from chunk content', () => {
|
|
61
62
|
const text = '# Section\nVisible text <!-- hidden comment --> more visible text.';
|
|
62
|
-
const [chunk] = chunkMarkdown(text, SRC, CFG);
|
|
63
|
+
const [chunk] = chunkMarkdown(text, SRC, CFG, META);
|
|
63
64
|
assert.ok(!chunk.content.includes('hidden comment'), 'HTML comment must be stripped');
|
|
64
65
|
assert.ok(chunk.content.includes('Visible text'), 'Visible text must remain');
|
|
65
66
|
});
|
|
66
67
|
test('multiline HTML comment is stripped', () => {
|
|
67
68
|
const text = '# Section\nBefore.\n<!-- multi\nline\ncomment -->\nAfter.';
|
|
68
|
-
const [chunk] = chunkMarkdown(text, SRC, CFG);
|
|
69
|
+
const [chunk] = chunkMarkdown(text, SRC, CFG, META);
|
|
69
70
|
assert.ok(!chunk.content.includes('multi'), 'Multiline comment must be stripped');
|
|
70
71
|
assert.ok(chunk.content.includes('Before.'), 'Text before comment must remain');
|
|
71
72
|
});
|
|
73
|
+
test('propagates source type and file mtime metadata to every chunk', () => {
|
|
74
|
+
const text = '# One\nBody one.\n\n# Two\nBody two.';
|
|
75
|
+
const chunks = chunkMarkdown(text, SRC, CFG, { sourceType: 'digest', fileMtimeAt: 456 });
|
|
76
|
+
assert.ok(chunks.length > 0);
|
|
77
|
+
for (const chunk of chunks) {
|
|
78
|
+
assert.equal(chunk.sourceType, 'digest');
|
|
79
|
+
assert.equal(chunk.fileMtimeAt, 456);
|
|
80
|
+
}
|
|
81
|
+
});
|
|
72
82
|
});
|