@hanhnd/agent-kit 1.0.31 → 1.0.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands/cli-digest-pending.test.js +111 -0
- package/dist/cli/commands/cli-memory.test.js +34 -0
- package/dist/cli/commands/memory.d.ts +1 -0
- package/dist/cli/commands/memory.js +90 -0
- package/dist/cli/index.d.ts +1 -0
- package/dist/cli/index.js +9 -0
- package/dist/cli/utils/args.d.ts +8 -0
- package/dist/cli/utils/args.js +31 -0
- package/dist/cli.js +16489 -0
- package/dist/{tools/config.d.ts → core/config/index.d.ts} +4 -9
- package/dist/{tools/config.js → core/config/index.js} +20 -23
- package/dist/core/handoffs/index.d.ts +16 -0
- package/dist/{tools/handoffs.js → core/handoffs/index.js} +4 -5
- package/dist/{tools/security.d.ts → core/security/index.d.ts} +0 -19
- package/dist/{tools/security.js → core/security/index.js} +3 -80
- package/dist/entrypoints/cli.d.ts +2 -0
- package/dist/entrypoints/cli.js +11 -0
- package/dist/entrypoints/server.js +37 -0
- package/dist/{tools → mcp}/agent.js +3 -14
- package/dist/{tools → mcp}/core.d.ts +1 -1
- package/dist/{tools → mcp}/core.js +3 -5
- package/dist/{tools → mcp}/integration.js +5 -11
- package/dist/mcp/memory.d.ts +15 -0
- package/dist/{tools → mcp}/memory.js +40 -12
- package/dist/{kit-server.js → server.js} +720 -224
- package/dist/services/digest/background.d.ts +11 -0
- package/dist/services/digest/background.js +258 -0
- package/dist/services/digest/constants.d.ts +17 -0
- package/dist/services/digest/constants.js +19 -0
- package/dist/services/digest/digest-background.test.js +206 -0
- package/dist/services/digest/digest-init.test.js +33 -0
- package/dist/services/digest/digest-llama-local.test.js +21 -0
- package/dist/services/digest/digest-pending.test.d.ts +1 -0
- package/dist/services/digest/digest-pending.test.js +192 -0
- package/dist/services/digest/digest-processor.test.d.ts +1 -0
- package/dist/services/digest/digest-processor.test.js +32 -0
- package/dist/services/digest/files.d.ts +6 -0
- package/dist/services/digest/files.js +45 -0
- package/dist/services/digest/model-registry.d.ts +2 -0
- package/dist/services/digest/model-registry.js +49 -0
- package/dist/services/digest/processor.d.ts +13 -0
- package/dist/services/digest/processor.js +188 -0
- package/dist/services/digest/providers/llama-local.d.ts +2 -0
- package/dist/services/digest/providers/llama-local.js +135 -0
- package/dist/services/digest/types.d.ts +117 -0
- package/dist/services/digest/types.js +6 -0
- package/dist/services/digest/worker.d.ts +2 -0
- package/dist/services/digest/worker.js +51 -0
- package/dist/{utils → services/integration}/credentials.js +1 -1
- package/dist/{utils/parser.js → services/integration/parser/adf.js} +3 -6
- package/dist/{memory → services/memory}/chunker.d.ts +0 -2
- package/dist/{memory → services/memory}/chunker.js +5 -7
- package/dist/services/memory/chunker.test.d.ts +1 -0
- package/dist/{memory/__tests__ → services/memory}/chunker.test.js +1 -34
- package/dist/services/memory/constants.d.ts +5 -0
- package/dist/services/memory/constants.js +13 -0
- package/dist/{memory → services/memory}/embedder.d.ts +3 -2
- package/dist/{memory → services/memory}/embedder.js +15 -6
- package/dist/services/memory/embedder.test.d.ts +1 -0
- package/dist/services/memory/embedder.test.js +13 -0
- package/dist/{memory → services/memory}/index.d.ts +1 -1
- package/dist/{memory → services/memory}/index.js +1 -1
- package/dist/{memory → services/memory}/indexer.d.ts +6 -2
- package/dist/{memory → services/memory}/indexer.js +15 -37
- package/dist/services/memory/indexer.test.d.ts +1 -0
- package/dist/{memory/__tests__ → services/memory}/indexer.test.js +11 -15
- package/dist/services/memory/memory.test.d.ts +1 -0
- package/dist/{tools/__tests__ → services/memory}/memory.test.js +82 -19
- package/dist/{memory → services/memory}/store.js +6 -18
- package/dist/services/memory/store.test.d.ts +1 -0
- package/dist/{memory/__tests__ → services/memory}/store.test.js +18 -5
- package/dist/{memory → services/memory}/types.d.ts +8 -2
- package/dist/services/memory/types.js +6 -0
- package/dist/utils/async.d.ts +1 -0
- package/dist/utils/async.js +15 -0
- package/dist/utils/files.d.ts +7 -0
- package/dist/utils/files.js +77 -0
- package/dist/utils/files.test.d.ts +1 -0
- package/dist/utils/files.test.js +81 -0
- package/dist/utils/hash.d.ts +1 -0
- package/dist/utils/hash.js +4 -0
- package/dist/utils/index.d.ts +6 -0
- package/dist/utils/index.js +6 -0
- package/dist/utils/json.d.ts +1 -0
- package/dist/utils/json.js +3 -0
- package/dist/utils/paths.d.ts +0 -1
- package/dist/utils/paths.js +1 -1
- package/dist/utils/temp-dir.test.d.ts +4 -0
- package/dist/utils/temp-dir.test.js +20 -0
- package/package.json +8 -6
- package/dist/memory/__tests__/embedder.test.js +0 -14
- package/dist/memory/types.js +0 -9
- package/dist/tools/handoffs.d.ts +0 -22
- package/dist/tools/memory.d.ts +0 -16
- /package/dist/{memory/__tests__/chunker.test.d.ts → cli/commands/cli-digest-pending.test.d.ts} +0 -0
- /package/dist/{memory/__tests__/embedder.test.d.ts → cli/commands/cli-memory.test.d.ts} +0 -0
- /package/dist/{kit-server.d.ts → entrypoints/server.d.ts} +0 -0
- /package/dist/{tools → mcp}/agent.d.ts +0 -0
- /package/dist/{tools → mcp}/integration.d.ts +0 -0
- /package/dist/{memory/__tests__/indexer.test.d.ts → services/digest/digest-background.test.d.ts} +0 -0
- /package/dist/{memory/__tests__/store.test.d.ts → services/digest/digest-init.test.d.ts} +0 -0
- /package/dist/{tools/__tests__/memory.test.d.ts → services/digest/digest-llama-local.test.d.ts} +0 -0
- /package/dist/{utils → services/integration}/credentials.d.ts +0 -0
- /package/dist/{utils/parser.d.ts → services/integration/parser/adf.d.ts} +0 -0
- /package/dist/{memory → services/memory}/store.d.ts +0 -0
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
import * as assert from 'node:assert/strict';
|
|
2
|
+
import * as fs from 'node:fs';
|
|
3
|
+
import * as path from 'node:path';
|
|
4
|
+
import { afterEach, describe, test } from 'node:test';
|
|
5
|
+
import { createTempDirTracker } from '../../utils/temp-dir.test.js';
|
|
6
|
+
import { defaultProvisionalDigestDir, readConversationDigestInput, writeConversationDigestSettings, writeProvisionalDigestFile, } from './files.js';
|
|
7
|
+
import { DigestModelId } from './types.js';
|
|
8
|
+
import { digestPendingConversations, summarizePendingConversations } from './processor.js';
|
|
9
|
+
const tempDirs = createTempDirTracker();
|
|
10
|
+
afterEach(() => {
|
|
11
|
+
tempDirs.cleanup();
|
|
12
|
+
});
|
|
13
|
+
function writeInitState(workspace, modelId = DigestModelId.DEFAULT) {
|
|
14
|
+
writeConversationDigestSettings(workspace, {
|
|
15
|
+
enabled: true,
|
|
16
|
+
initialized: true,
|
|
17
|
+
modelId,
|
|
18
|
+
initializedAt: new Date().toISOString(),
|
|
19
|
+
});
|
|
20
|
+
}
|
|
21
|
+
function writeConvFile(workspace, name, content) {
|
|
22
|
+
const rawDir = path.join(workspace, '.agent-kit', 'wiki', 'raw');
|
|
23
|
+
fs.mkdirSync(rawDir, { recursive: true });
|
|
24
|
+
const filePath = path.join(rawDir, name);
|
|
25
|
+
fs.writeFileSync(filePath, content, 'utf8');
|
|
26
|
+
return filePath;
|
|
27
|
+
}
|
|
28
|
+
function stubDigest(result = {}) {
|
|
29
|
+
return async () => ({
|
|
30
|
+
markdown: 'out.md',
|
|
31
|
+
status: 'provisional',
|
|
32
|
+
contentHash: 'abc123',
|
|
33
|
+
skipped: false,
|
|
34
|
+
indexed: false,
|
|
35
|
+
...result,
|
|
36
|
+
});
|
|
37
|
+
}
|
|
38
|
+
describe('digestPendingConversations', () => {
|
|
39
|
+
test('C13: no init state → noop not-initialized', async () => {
|
|
40
|
+
const workspace = tempDirs.makeTempDir('pending-c13-');
|
|
41
|
+
const result = await digestPendingConversations({ workspaceRoot: workspace });
|
|
42
|
+
assert.equal(result.ok, true);
|
|
43
|
+
assert.equal(result.initialized, false);
|
|
44
|
+
assert.equal(result.action, 'noop');
|
|
45
|
+
if (result.action === 'noop')
|
|
46
|
+
assert.equal(result.reason, 'not-initialized');
|
|
47
|
+
});
|
|
48
|
+
test('C14: init state, live lockfile → noop locked', async () => {
|
|
49
|
+
const workspace = tempDirs.makeTempDir('pending-c14-');
|
|
50
|
+
writeInitState(workspace);
|
|
51
|
+
const lockDir = path.join(workspace, '.agent-kit', 'wiki', 'digest');
|
|
52
|
+
fs.mkdirSync(lockDir, { recursive: true });
|
|
53
|
+
fs.writeFileSync(path.join(lockDir, 'digest-worker.lock'), JSON.stringify({ pid: process.pid }));
|
|
54
|
+
const result = await digestPendingConversations({ workspaceRoot: workspace });
|
|
55
|
+
assert.equal(result.ok, true);
|
|
56
|
+
assert.equal(result.initialized, true);
|
|
57
|
+
assert.equal(result.action, 'noop');
|
|
58
|
+
if (result.action === 'noop')
|
|
59
|
+
assert.equal(result.reason, 'locked');
|
|
60
|
+
});
|
|
61
|
+
test('C15: init state, no lockfile, no conv files → noop no-pending, lock released', async () => {
|
|
62
|
+
const workspace = tempDirs.makeTempDir('pending-c15-');
|
|
63
|
+
writeInitState(workspace);
|
|
64
|
+
const result = await digestPendingConversations({ workspaceRoot: workspace });
|
|
65
|
+
assert.equal(result.ok, true);
|
|
66
|
+
assert.equal(result.action, 'noop');
|
|
67
|
+
if (result.action === 'noop')
|
|
68
|
+
assert.equal(result.reason, 'no-pending');
|
|
69
|
+
assert.equal(fs.existsSync(path.join(workspace, '.agent-kit', 'wiki', 'digest', 'digest-worker.lock')), false);
|
|
70
|
+
});
|
|
71
|
+
test('C19: wiki/raw dir does not exist → noop no-pending', async () => {
|
|
72
|
+
const workspace = tempDirs.makeTempDir('pending-c19-');
|
|
73
|
+
writeInitState(workspace);
|
|
74
|
+
const result = await digestPendingConversations({ workspaceRoot: workspace });
|
|
75
|
+
assert.equal(result.action, 'noop');
|
|
76
|
+
if (result.action === 'noop')
|
|
77
|
+
assert.equal(result.reason, 'no-pending');
|
|
78
|
+
});
|
|
79
|
+
test('C16: one undigested conv file → digested count:1, lock released', async () => {
|
|
80
|
+
const workspace = tempDirs.makeTempDir('pending-c16-');
|
|
81
|
+
writeInitState(workspace);
|
|
82
|
+
writeConvFile(workspace, 'conv_2026-01-01T00-00-00-001Z.md', '**User:** hello\n\n**Assistant:** hi\n');
|
|
83
|
+
const result = await digestPendingConversations({
|
|
84
|
+
workspaceRoot: workspace,
|
|
85
|
+
digestFn: stubDigest(),
|
|
86
|
+
});
|
|
87
|
+
assert.equal(result.ok, true);
|
|
88
|
+
assert.equal(result.action, 'digested');
|
|
89
|
+
if (result.action === 'digested') {
|
|
90
|
+
assert.equal(result.count, 1);
|
|
91
|
+
assert.equal(result.errors, 0);
|
|
92
|
+
}
|
|
93
|
+
assert.equal(fs.existsSync(path.join(workspace, '.agent-kit', 'wiki', 'digest', 'digest-worker.lock')), false);
|
|
94
|
+
});
|
|
95
|
+
test('C17: digestFn throws for one file → errors:1, lock released', async () => {
|
|
96
|
+
const workspace = tempDirs.makeTempDir('pending-c17-');
|
|
97
|
+
writeInitState(workspace);
|
|
98
|
+
writeConvFile(workspace, 'conv_2026-01-01T00-00-00-001Z.md', '**User:** hello\n\n**Assistant:** hi\n');
|
|
99
|
+
const throwingFn = async () => {
|
|
100
|
+
throw new Error('model error');
|
|
101
|
+
};
|
|
102
|
+
const result = await digestPendingConversations({
|
|
103
|
+
workspaceRoot: workspace,
|
|
104
|
+
digestFn: throwingFn,
|
|
105
|
+
});
|
|
106
|
+
assert.equal(result.ok, true);
|
|
107
|
+
assert.equal(result.action, 'digested');
|
|
108
|
+
if (result.action === 'digested') {
|
|
109
|
+
assert.equal(result.errors, 1);
|
|
110
|
+
assert.equal(result.count, 0);
|
|
111
|
+
}
|
|
112
|
+
assert.equal(fs.existsSync(path.join(workspace, '.agent-kit', 'wiki', 'digest', 'digest-worker.lock')), false);
|
|
113
|
+
});
|
|
114
|
+
test('C18: stale lockfile (dead PID) → reclaims lock and returns (no-pending since no conv files)', async () => {
|
|
115
|
+
const workspace = tempDirs.makeTempDir('pending-c18-');
|
|
116
|
+
writeInitState(workspace);
|
|
117
|
+
const lockDir = path.join(workspace, '.agent-kit', 'wiki', 'digest');
|
|
118
|
+
fs.mkdirSync(lockDir, { recursive: true });
|
|
119
|
+
fs.writeFileSync(path.join(lockDir, 'digest-worker.lock'), JSON.stringify({ pid: 999999999 }));
|
|
120
|
+
const result = await digestPendingConversations({ workspaceRoot: workspace });
|
|
121
|
+
assert.ok(result.ok);
|
|
122
|
+
if (result.action === 'noop')
|
|
123
|
+
assert.notEqual(result.reason, 'locked');
|
|
124
|
+
assert.equal(fs.existsSync(path.join(workspace, '.agent-kit', 'wiki', 'digest', 'digest-worker.lock')), false);
|
|
125
|
+
});
|
|
126
|
+
test('C20: 3 undigested conv files → digestFn called 3 times, count+skipped+errors=3', async () => {
|
|
127
|
+
const workspace = tempDirs.makeTempDir('pending-c20-');
|
|
128
|
+
writeInitState(workspace);
|
|
129
|
+
writeConvFile(workspace, 'conv_2026-01-01T00-00-00-001Z.md', '**User:** a\n\n**Assistant:** b\n');
|
|
130
|
+
writeConvFile(workspace, 'conv_2026-01-01T00-00-00-002Z.md', '**User:** c\n\n**Assistant:** d\n');
|
|
131
|
+
writeConvFile(workspace, 'conv_2026-01-01T00-00-00-003Z.md', '**User:** e\n\n**Assistant:** f\n');
|
|
132
|
+
let callCount = 0;
|
|
133
|
+
const countingFn = async () => {
|
|
134
|
+
callCount++;
|
|
135
|
+
return {
|
|
136
|
+
markdown: 'out.md',
|
|
137
|
+
status: 'provisional',
|
|
138
|
+
contentHash: 'abc',
|
|
139
|
+
skipped: false,
|
|
140
|
+
indexed: false,
|
|
141
|
+
};
|
|
142
|
+
};
|
|
143
|
+
const result = await digestPendingConversations({
|
|
144
|
+
workspaceRoot: workspace,
|
|
145
|
+
digestFn: countingFn,
|
|
146
|
+
});
|
|
147
|
+
assert.equal(callCount, 3);
|
|
148
|
+
assert.equal(result.action, 'digested');
|
|
149
|
+
if (result.action === 'digested') {
|
|
150
|
+
assert.equal(result.count + result.skipped + result.errors, 3);
|
|
151
|
+
}
|
|
152
|
+
});
|
|
153
|
+
test('C21: outer error (fs throws unexpected) → returns ok:false, releases lock', async () => {
|
|
154
|
+
const workspace = tempDirs.makeTempDir('pending-c21-');
|
|
155
|
+
writeInitState(workspace);
|
|
156
|
+
// Create wiki/raw as a file (not a dir) so readdirSync throws ENOTDIR
|
|
157
|
+
const wikiDir = path.join(workspace, '.agent-kit', 'wiki');
|
|
158
|
+
fs.mkdirSync(wikiDir, { recursive: true });
|
|
159
|
+
fs.writeFileSync(path.join(wikiDir, 'raw'), 'not-a-directory', 'utf8');
|
|
160
|
+
const result = await digestPendingConversations({ workspaceRoot: workspace });
|
|
161
|
+
assert.equal(result.ok, false);
|
|
162
|
+
assert.equal(result.action, 'error');
|
|
163
|
+
assert.equal(fs.existsSync(path.join(workspace, '.agent-kit', 'wiki', 'digest', 'digest-worker.lock')), false);
|
|
164
|
+
});
|
|
165
|
+
});
|
|
166
|
+
describe('summarizePendingConversations', () => {
|
|
167
|
+
test('returns not-initialized without raw discovery side effects', () => {
|
|
168
|
+
const workspace = tempDirs.makeTempDir('pending-summary-init-');
|
|
169
|
+
const result = summarizePendingConversations(workspace);
|
|
170
|
+
assert.equal(result.initialized, false);
|
|
171
|
+
assert.equal(result.pending, 0);
|
|
172
|
+
assert.equal(result.reason, 'not-initialized');
|
|
173
|
+
});
|
|
174
|
+
test('returns no-pending when initialized workspace has no raw conversations', () => {
|
|
175
|
+
const workspace = tempDirs.makeTempDir('pending-summary-empty-');
|
|
176
|
+
writeInitState(workspace);
|
|
177
|
+
const result = summarizePendingConversations(workspace);
|
|
178
|
+
assert.equal(result.initialized, true);
|
|
179
|
+
assert.equal(result.pending, 0);
|
|
180
|
+
assert.equal(result.reason, 'no-pending');
|
|
181
|
+
});
|
|
182
|
+
test('counts only conversations without existing provisional digests', () => {
|
|
183
|
+
const workspace = tempDirs.makeTempDir('pending-summary-count-');
|
|
184
|
+
writeInitState(workspace);
|
|
185
|
+
const first = writeConvFile(workspace, 'conv_2026-01-01T00-00-00-001Z.md', '**User:** a\n');
|
|
186
|
+
writeConvFile(workspace, 'conv_2026-01-01T00-00-00-002Z.md', '**User:** b\n');
|
|
187
|
+
writeProvisionalDigestFile(defaultProvisionalDigestDir(workspace), readConversationDigestInput(workspace, first), '# Existing\n');
|
|
188
|
+
const result = summarizePendingConversations(workspace);
|
|
189
|
+
assert.equal(result.initialized, true);
|
|
190
|
+
assert.equal(result.pending, 1);
|
|
191
|
+
});
|
|
192
|
+
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import * as assert from 'node:assert/strict';
|
|
2
|
+
import * as fs from 'node:fs';
|
|
3
|
+
import * as path from 'node:path';
|
|
4
|
+
import { afterEach, describe, test } from 'node:test';
|
|
5
|
+
import { createTempDirTracker } from '../../utils/temp-dir.test.js';
|
|
6
|
+
import { DEFAULT_DIGEST_MODEL_ID } from './constants.js';
|
|
7
|
+
import { defaultProvisionalDigestDir, writeProvisionalDigestFile, readConversationDigestInput } from './files.js';
|
|
8
|
+
import { digestConversationFile } from './processor.js';
|
|
9
|
+
const tempDirs = createTempDirTracker();
|
|
10
|
+
afterEach(() => {
|
|
11
|
+
tempDirs.cleanup();
|
|
12
|
+
});
|
|
13
|
+
describe('digestConversationFile', () => {
|
|
14
|
+
test('returns existing content-hash named provisional digest without loading model', async () => {
|
|
15
|
+
const workspace = tempDirs.makeTempDir('digest-processor-');
|
|
16
|
+
const inputPath = path.join(workspace, '.agent-kit', 'wiki', 'archive', 'conversations', 'conv.md');
|
|
17
|
+
fs.mkdirSync(path.dirname(inputPath), { recursive: true });
|
|
18
|
+
fs.writeFileSync(inputPath, '**User:** remember this decision', 'utf8');
|
|
19
|
+
const input = readConversationDigestInput(workspace, inputPath);
|
|
20
|
+
const outDir = defaultProvisionalDigestDir(workspace);
|
|
21
|
+
const markdownPath = writeProvisionalDigestFile(outDir, input, '# Conversation Digest: conv\n');
|
|
22
|
+
const result = await digestConversationFile({
|
|
23
|
+
workspaceRoot: workspace,
|
|
24
|
+
inputPath,
|
|
25
|
+
modelId: DEFAULT_DIGEST_MODEL_ID,
|
|
26
|
+
});
|
|
27
|
+
assert.equal(result.skipped, true);
|
|
28
|
+
assert.equal(result.markdown, markdownPath);
|
|
29
|
+
assert.equal(result.contentHash, input.contentHash);
|
|
30
|
+
assert.match(path.basename(result.markdown), /^[a-f0-9]{16}-conv\.md$/);
|
|
31
|
+
});
|
|
32
|
+
});
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { ConversationDigestInput, ConversationDigestSettings } from './types.js';
|
|
2
|
+
export declare function defaultProvisionalDigestDir(workspaceRoot: string): string;
|
|
3
|
+
export declare function readConversationDigestInput(workspaceRoot: string, inputPath: string): ConversationDigestInput;
|
|
4
|
+
export declare function findExistingProvisionalDigest(outDir: string, input: ConversationDigestInput): string | undefined;
|
|
5
|
+
export declare function writeProvisionalDigestFile(outDir: string, input: ConversationDigestInput, markdown: string): string;
|
|
6
|
+
export declare function writeConversationDigestSettings(workspaceRoot: string, digest: ConversationDigestSettings): void;
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import * as fs from 'node:fs';
|
|
2
|
+
import * as path from 'node:path';
|
|
3
|
+
import { PROVISIONAL_DIGEST_DIR } from './constants.js';
|
|
4
|
+
import { atomicWriteTextFile } from '../../utils/files.js';
|
|
5
|
+
import { sha256Hex } from '../../utils/hash.js';
|
|
6
|
+
import { loadProjectSettings, writeProjectSettings } from '../../core/config/index.js';
|
|
7
|
+
export function defaultProvisionalDigestDir(workspaceRoot) {
|
|
8
|
+
return path.join(workspaceRoot, PROVISIONAL_DIGEST_DIR);
|
|
9
|
+
}
|
|
10
|
+
export function readConversationDigestInput(workspaceRoot, inputPath) {
|
|
11
|
+
const absoluteInput = path.isAbsolute(inputPath) ? inputPath : path.resolve(workspaceRoot, inputPath);
|
|
12
|
+
const content = fs.readFileSync(absoluteInput, 'utf8');
|
|
13
|
+
return {
|
|
14
|
+
sourcePath: path.relative(workspaceRoot, absoluteInput),
|
|
15
|
+
content,
|
|
16
|
+
contentHash: sha256Hex(content),
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
function provisionalDigestPath(outDir, input) {
|
|
20
|
+
const safeName = path
|
|
21
|
+
.basename(input.sourcePath)
|
|
22
|
+
.replace(/\.[^.]+$/, '')
|
|
23
|
+
.replace(/[^a-z0-9._-]+/gi, '-');
|
|
24
|
+
return path.join(outDir, `${input.contentHash.slice(0, 16)}-${safeName}.md`);
|
|
25
|
+
}
|
|
26
|
+
export function findExistingProvisionalDigest(outDir, input) {
|
|
27
|
+
const markdownPath = provisionalDigestPath(outDir, input);
|
|
28
|
+
return fs.existsSync(markdownPath) ? markdownPath : undefined;
|
|
29
|
+
}
|
|
30
|
+
export function writeProvisionalDigestFile(outDir, input, markdown) {
|
|
31
|
+
const markdownPath = provisionalDigestPath(outDir, input);
|
|
32
|
+
atomicWriteTextFile(markdownPath, markdown);
|
|
33
|
+
return markdownPath;
|
|
34
|
+
}
|
|
35
|
+
export function writeConversationDigestSettings(workspaceRoot, digest) {
|
|
36
|
+
const current = loadProjectSettings(workspaceRoot);
|
|
37
|
+
const memory = current.memory ?? {};
|
|
38
|
+
writeProjectSettings(workspaceRoot, {
|
|
39
|
+
memory: {
|
|
40
|
+
...memory,
|
|
41
|
+
enabled: memory.enabled ?? true,
|
|
42
|
+
conversationDigest: digest,
|
|
43
|
+
},
|
|
44
|
+
});
|
|
45
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { DigestModelId } from './types.js';
|
|
2
|
+
class DigestModelRegistryError extends Error {
|
|
3
|
+
constructor(message) {
|
|
4
|
+
super(message);
|
|
5
|
+
this.name = 'DigestModelRegistryError';
|
|
6
|
+
}
|
|
7
|
+
}
|
|
8
|
+
const DIGEST_MODEL_REGISTRY = {
|
|
9
|
+
[DigestModelId.TINY]: {
|
|
10
|
+
id: DigestModelId.TINY,
|
|
11
|
+
ggufUri: 'hf:bartowski/Qwen2.5-0.5B-Instruct-GGUF/Qwen2.5-0.5B-Instruct-Q4_K_M.gguf',
|
|
12
|
+
approxSizeBytes: 400_000_000,
|
|
13
|
+
license: 'Apache-2.0',
|
|
14
|
+
sourceUrl: 'https://huggingface.co/bartowski/Qwen2.5-0.5B-Instruct-GGUF',
|
|
15
|
+
enabled: true,
|
|
16
|
+
},
|
|
17
|
+
[DigestModelId.DEFAULT]: {
|
|
18
|
+
id: DigestModelId.DEFAULT,
|
|
19
|
+
ggufUri: 'hf:bartowski/Qwen2.5-1.5B-Instruct-GGUF/Qwen2.5-1.5B-Instruct-Q4_K_M.gguf',
|
|
20
|
+
approxSizeBytes: 1_000_000_000,
|
|
21
|
+
license: 'Apache-2.0',
|
|
22
|
+
sourceUrl: 'https://huggingface.co/bartowski/Qwen2.5-1.5B-Instruct-GGUF',
|
|
23
|
+
enabled: true,
|
|
24
|
+
},
|
|
25
|
+
[DigestModelId.LARGE]: {
|
|
26
|
+
id: DigestModelId.LARGE,
|
|
27
|
+
ggufUri: 'hf:bartowski/Qwen2.5-3B-Instruct-GGUF/Qwen2.5-3B-Instruct-Q4_K_M.gguf',
|
|
28
|
+
approxSizeBytes: 1_900_000_000,
|
|
29
|
+
license: 'Apache-2.0',
|
|
30
|
+
sourceUrl: 'https://huggingface.co/bartowski/Qwen2.5-3B-Instruct-GGUF',
|
|
31
|
+
enabled: true,
|
|
32
|
+
},
|
|
33
|
+
};
|
|
34
|
+
export function getDigestModelSpec(modelId) {
|
|
35
|
+
if (!modelId.trim()) {
|
|
36
|
+
throw new DigestModelRegistryError('Digest model id is required');
|
|
37
|
+
}
|
|
38
|
+
const spec = DIGEST_MODEL_REGISTRY[modelId];
|
|
39
|
+
if (!spec) {
|
|
40
|
+
throw new DigestModelRegistryError(`Unknown digest model: ${modelId}`);
|
|
41
|
+
}
|
|
42
|
+
if (spec.enabled !== true) {
|
|
43
|
+
throw new DigestModelRegistryError(`Digest model is disabled: ${modelId}`);
|
|
44
|
+
}
|
|
45
|
+
if (!spec.ggufUri || !spec.sourceUrl) {
|
|
46
|
+
throw new DigestModelRegistryError(`Digest model spec is incomplete: ${modelId}`);
|
|
47
|
+
}
|
|
48
|
+
return spec;
|
|
49
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { DigestFileOptions, DigestPendingCandidateSummary, DigestPendingResult, ProvisionalDigestResult, ConversationDigestInitResult, InitializeConversationDigestInput } from './types.js';
|
|
2
|
+
export declare function digestConversationFile(options: DigestFileOptions): Promise<ProvisionalDigestResult>;
|
|
3
|
+
export declare function summarizePendingConversations(workspaceRoot: string): DigestPendingCandidateSummary;
|
|
4
|
+
export declare function digestPendingConversations({ workspaceRoot, digestFn, onProgress, }: {
|
|
5
|
+
workspaceRoot: string;
|
|
6
|
+
digestFn?: (opts: DigestFileOptions) => Promise<ProvisionalDigestResult>;
|
|
7
|
+
onProgress?: (progress: {
|
|
8
|
+
processed: number;
|
|
9
|
+
skipped: number;
|
|
10
|
+
errors: number;
|
|
11
|
+
}) => void | Promise<void>;
|
|
12
|
+
}): Promise<DigestPendingResult>;
|
|
13
|
+
export declare function initializeConversationDigestModel(input: InitializeConversationDigestInput): Promise<ConversationDigestInitResult>;
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
import * as fs from 'node:fs';
|
|
2
|
+
import * as path from 'node:path';
|
|
3
|
+
import { DEFAULT_DIGEST_TIMEOUT_MS, DEFAULT_DIGEST_MAX_INPUT_CHARS, DIGEST_LOCKFILE_REL_PATH, WIKI_RAW_DIR, } from './constants.js';
|
|
4
|
+
import { defaultProvisionalDigestDir, readConversationDigestInput, findExistingProvisionalDigest, writeProvisionalDigestFile, writeConversationDigestSettings, } from './files.js';
|
|
5
|
+
import { getDigestModelSpec } from './model-registry.js';
|
|
6
|
+
import { createLlamaLocalDigestProvider } from './providers/llama-local.js';
|
|
7
|
+
import { loadProjectSettings, resolveConversationDigestConfig, resolveMemoryConfig } from '../../core/config/index.js';
|
|
8
|
+
import { MemoryStore } from '../memory/store.js';
|
|
9
|
+
import { MemoryIndexer } from '../memory/indexer.js';
|
|
10
|
+
import { releaseLock, tryAcquireProcessLock } from '../../utils/files.js';
|
|
11
|
+
function digestLockPath(workspaceRoot) {
|
|
12
|
+
return path.join(workspaceRoot, DIGEST_LOCKFILE_REL_PATH);
|
|
13
|
+
}
|
|
14
|
+
function discoverPendingConversationFiles(workspaceRoot) {
|
|
15
|
+
const rawDir = path.join(workspaceRoot, WIKI_RAW_DIR);
|
|
16
|
+
let files;
|
|
17
|
+
try {
|
|
18
|
+
files = fs
|
|
19
|
+
.readdirSync(rawDir)
|
|
20
|
+
.filter((f) => f.startsWith('conv_') && f.endsWith('.md'))
|
|
21
|
+
.map((f) => path.join(rawDir, f))
|
|
22
|
+
.sort((a, b) => fs.statSync(a).mtimeMs - fs.statSync(b).mtimeMs);
|
|
23
|
+
}
|
|
24
|
+
catch (err) {
|
|
25
|
+
if (err.code === 'ENOENT') {
|
|
26
|
+
files = [];
|
|
27
|
+
}
|
|
28
|
+
else {
|
|
29
|
+
throw err;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
const outDir = defaultProvisionalDigestDir(workspaceRoot);
|
|
33
|
+
return files.filter((f) => {
|
|
34
|
+
try {
|
|
35
|
+
const input = readConversationDigestInput(workspaceRoot, f);
|
|
36
|
+
return !findExistingProvisionalDigest(outDir, input);
|
|
37
|
+
}
|
|
38
|
+
catch {
|
|
39
|
+
return false;
|
|
40
|
+
}
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
async function indexProvisionalDigestFile(workspaceRoot, markdownPath) {
|
|
44
|
+
try {
|
|
45
|
+
const settings = loadProjectSettings(workspaceRoot);
|
|
46
|
+
const config = resolveMemoryConfig(settings, workspaceRoot);
|
|
47
|
+
const store = new MemoryStore(path.join(config.wikiDir, 'index.db'), config);
|
|
48
|
+
const embedder = { embed: async (_texts) => [], initialize: async () => { } };
|
|
49
|
+
const indexer = new MemoryIndexer(store, embedder, config);
|
|
50
|
+
await indexer.indexFile(markdownPath);
|
|
51
|
+
return { indexed: true };
|
|
52
|
+
}
|
|
53
|
+
catch (err) {
|
|
54
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
55
|
+
return { indexed: false, error: message };
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
export async function digestConversationFile(options) {
|
|
59
|
+
const outDir = options.outDir
|
|
60
|
+
? path.resolve(options.workspaceRoot, options.outDir)
|
|
61
|
+
: defaultProvisionalDigestDir(options.workspaceRoot);
|
|
62
|
+
const input = readConversationDigestInput(options.workspaceRoot, options.inputPath);
|
|
63
|
+
const existingPath = findExistingProvisionalDigest(outDir, input);
|
|
64
|
+
if (existingPath) {
|
|
65
|
+
const indexResult = await indexProvisionalDigestFile(options.workspaceRoot, existingPath);
|
|
66
|
+
return {
|
|
67
|
+
markdown: existingPath,
|
|
68
|
+
status: 'provisional',
|
|
69
|
+
contentHash: input.contentHash,
|
|
70
|
+
skipped: true,
|
|
71
|
+
...indexResult,
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
const provider = await createLlamaLocalDigestProvider(options.modelId);
|
|
75
|
+
try {
|
|
76
|
+
const markdown = await provider.generateDigestMarkdown(input, {
|
|
77
|
+
modelId: options.modelId,
|
|
78
|
+
maxInputChars: options.maxInputChars ?? DEFAULT_DIGEST_MAX_INPUT_CHARS,
|
|
79
|
+
timeoutMs: options.timeoutMs ?? DEFAULT_DIGEST_TIMEOUT_MS,
|
|
80
|
+
});
|
|
81
|
+
const markdownPath = writeProvisionalDigestFile(outDir, input, markdown);
|
|
82
|
+
const indexResult = await indexProvisionalDigestFile(options.workspaceRoot, markdownPath);
|
|
83
|
+
return {
|
|
84
|
+
markdown: markdownPath,
|
|
85
|
+
status: 'provisional',
|
|
86
|
+
contentHash: input.contentHash,
|
|
87
|
+
skipped: false,
|
|
88
|
+
...indexResult,
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
finally {
|
|
92
|
+
await provider.dispose?.();
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
export function summarizePendingConversations(workspaceRoot) {
|
|
96
|
+
const digestConfig = resolveConversationDigestConfig(loadProjectSettings(workspaceRoot));
|
|
97
|
+
if (!digestConfig || digestConfig.enabled === false) {
|
|
98
|
+
return { initialized: false, pending: 0, reason: 'not-initialized' };
|
|
99
|
+
}
|
|
100
|
+
const candidates = discoverPendingConversationFiles(workspaceRoot);
|
|
101
|
+
if (candidates.length === 0) {
|
|
102
|
+
return { initialized: true, pending: 0, reason: 'no-pending' };
|
|
103
|
+
}
|
|
104
|
+
return { initialized: true, pending: candidates.length };
|
|
105
|
+
}
|
|
106
|
+
export async function digestPendingConversations({ workspaceRoot, digestFn = digestConversationFile, onProgress, }) {
|
|
107
|
+
const digestConfig = resolveConversationDigestConfig(loadProjectSettings(workspaceRoot));
|
|
108
|
+
if (!digestConfig || digestConfig.enabled === false) {
|
|
109
|
+
return { ok: true, initialized: false, action: 'noop', reason: 'not-initialized' };
|
|
110
|
+
}
|
|
111
|
+
const lockPath = digestLockPath(workspaceRoot);
|
|
112
|
+
if (!tryAcquireProcessLock(lockPath)) {
|
|
113
|
+
return { ok: true, initialized: true, action: 'noop', reason: 'locked' };
|
|
114
|
+
}
|
|
115
|
+
try {
|
|
116
|
+
const candidates = discoverPendingConversationFiles(workspaceRoot);
|
|
117
|
+
if (candidates.length === 0) {
|
|
118
|
+
return { ok: true, initialized: true, action: 'noop', reason: 'no-pending' };
|
|
119
|
+
}
|
|
120
|
+
let count = 0;
|
|
121
|
+
let skipped = 0;
|
|
122
|
+
let errors = 0;
|
|
123
|
+
for (const filePath of candidates) {
|
|
124
|
+
try {
|
|
125
|
+
const result = await digestFn({
|
|
126
|
+
workspaceRoot,
|
|
127
|
+
inputPath: filePath,
|
|
128
|
+
modelId: digestConfig.modelId,
|
|
129
|
+
});
|
|
130
|
+
if (result.skipped) {
|
|
131
|
+
skipped++;
|
|
132
|
+
}
|
|
133
|
+
else {
|
|
134
|
+
count++;
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
catch {
|
|
138
|
+
errors++;
|
|
139
|
+
}
|
|
140
|
+
if (onProgress) {
|
|
141
|
+
await onProgress({ processed: count, skipped, errors });
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
return { ok: true, initialized: true, action: 'digested', count, skipped, errors };
|
|
145
|
+
}
|
|
146
|
+
catch (err) {
|
|
147
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
148
|
+
return { ok: false, initialized: true, action: 'error', error: message };
|
|
149
|
+
}
|
|
150
|
+
finally {
|
|
151
|
+
releaseLock(lockPath);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
export async function initializeConversationDigestModel(input) {
|
|
155
|
+
try {
|
|
156
|
+
getDigestModelSpec(input.modelId);
|
|
157
|
+
if (input.allowDownload !== true) {
|
|
158
|
+
return {
|
|
159
|
+
initialized: false,
|
|
160
|
+
modelId: input.modelId,
|
|
161
|
+
error: 'Digest model initialization requires explicit download permission',
|
|
162
|
+
};
|
|
163
|
+
}
|
|
164
|
+
const loadedProvider = await createLlamaLocalDigestProvider(input.modelId);
|
|
165
|
+
await loadedProvider?.dispose?.();
|
|
166
|
+
const initializedAt = new Date().toISOString();
|
|
167
|
+
writeConversationDigestSettings(input.workspaceRoot, {
|
|
168
|
+
enabled: input.enabled ?? true,
|
|
169
|
+
initialized: true,
|
|
170
|
+
modelId: input.modelId,
|
|
171
|
+
initializedAt,
|
|
172
|
+
});
|
|
173
|
+
await digestPendingConversations({ workspaceRoot: input.workspaceRoot });
|
|
174
|
+
return {
|
|
175
|
+
initialized: true,
|
|
176
|
+
modelId: input.modelId,
|
|
177
|
+
initializedAt,
|
|
178
|
+
};
|
|
179
|
+
}
|
|
180
|
+
catch (err) {
|
|
181
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
182
|
+
return {
|
|
183
|
+
initialized: false,
|
|
184
|
+
modelId: input.modelId,
|
|
185
|
+
error: message,
|
|
186
|
+
};
|
|
187
|
+
}
|
|
188
|
+
}
|