@hanhnd/agent-kit 1.0.31 → 1.0.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/__tests__/temp-dir.d.ts +4 -0
- package/dist/__tests__/temp-dir.js +20 -0
- package/dist/cli/index.d.ts +1 -0
- package/dist/cli/index.js +9 -0
- package/dist/cli/memory.d.ts +1 -0
- package/dist/cli/memory.js +77 -0
- package/dist/kit-cli.d.ts +2 -0
- package/dist/kit-cli.js +16205 -0
- package/dist/kit-server.js +679 -193
- package/dist/memory/__tests__/digest-llama-local.test.d.ts +1 -0
- package/dist/memory/__tests__/digest-llama-local.test.js +33 -0
- package/dist/memory/__tests__/digest-pending.test.d.ts +1 -0
- package/dist/memory/__tests__/digest-pending.test.js +164 -0
- package/dist/memory/__tests__/digest-processor.test.d.ts +1 -0
- package/dist/memory/__tests__/digest-processor.test.js +34 -0
- package/dist/memory/__tests__/digest-worker.test.d.ts +1 -0
- package/dist/memory/__tests__/digest-worker.test.js +46 -0
- package/dist/memory/__tests__/indexer.test.js +7 -12
- package/dist/memory/__tests__/lockfile.test.d.ts +1 -0
- package/dist/memory/__tests__/lockfile.test.js +81 -0
- package/dist/memory/__tests__/store.test.js +15 -3
- package/dist/memory/chunker.js +3 -5
- package/dist/memory/constants.d.ts +5 -0
- package/dist/memory/constants.js +12 -0
- package/dist/memory/digest/constants.d.ts +14 -0
- package/dist/memory/digest/constants.js +15 -0
- package/dist/memory/digest/files.d.ts +7 -0
- package/dist/memory/digest/files.js +45 -0
- package/dist/memory/digest/model-registry.d.ts +6 -0
- package/dist/memory/digest/model-registry.js +48 -0
- package/dist/memory/digest/processor.d.ts +7 -0
- package/dist/memory/digest/processor.js +171 -0
- package/dist/memory/digest/providers/llama-local.d.ts +7 -0
- package/dist/memory/digest/providers/llama-local.js +136 -0
- package/dist/memory/digest/types.d.ts +85 -0
- package/dist/memory/digest/types.js +1 -0
- package/dist/memory/digest/worker.d.ts +3 -0
- package/dist/memory/digest/worker.js +51 -0
- package/dist/memory/embedder.d.ts +1 -0
- package/dist/memory/embedder.js +10 -1
- package/dist/memory/index.d.ts +1 -1
- package/dist/memory/index.js +1 -1
- package/dist/memory/indexer.d.ts +5 -2
- package/dist/memory/indexer.js +15 -37
- package/dist/memory/store.js +6 -18
- package/dist/memory/types.d.ts +2 -1
- package/dist/memory/types.js +1 -9
- package/dist/tools/__tests__/cli-digest-pending.test.d.ts +1 -0
- package/dist/tools/__tests__/cli-digest-pending.test.js +51 -0
- package/dist/tools/__tests__/cli-memory.test.d.ts +1 -0
- package/dist/tools/__tests__/cli-memory.test.js +22 -0
- package/dist/tools/__tests__/digest-init.test.d.ts +1 -0
- package/dist/tools/__tests__/digest-init.test.js +32 -0
- package/dist/tools/__tests__/memory.test.js +28 -6
- package/dist/tools/agent.js +2 -13
- package/dist/tools/config.d.ts +4 -4
- package/dist/tools/config.js +20 -12
- package/dist/tools/core.js +1 -3
- package/dist/tools/handoffs.js +1 -2
- package/dist/tools/integration.js +2 -8
- package/dist/tools/memory.d.ts +1 -1
- package/dist/tools/memory.js +29 -3
- package/dist/tools/security.js +2 -6
- package/dist/utils/args.d.ts +7 -0
- package/dist/utils/args.js +31 -0
- package/dist/utils/async.d.ts +1 -0
- package/dist/utils/async.js +15 -0
- package/dist/utils/files.d.ts +8 -0
- package/dist/utils/files.js +77 -0
- package/dist/utils/hash.d.ts +1 -0
- package/dist/utils/hash.js +4 -0
- package/dist/utils/json.d.ts +2 -0
- package/dist/utils/json.js +13 -0
- package/dist/utils/parser.js +2 -6
- package/package.json +7 -5
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { DigestFileOptions, DigestPendingResult, ProvisionalDigestResult, ConversationDigestInitResult, InitializeConversationDigestInput } from './types.js';
|
|
2
|
+
export declare function digestConversationFile(options: DigestFileOptions): Promise<ProvisionalDigestResult>;
|
|
3
|
+
export declare function digestPendingConversations({ workspaceRoot, digestFn, }: {
|
|
4
|
+
workspaceRoot: string;
|
|
5
|
+
digestFn?: (opts: DigestFileOptions) => Promise<ProvisionalDigestResult>;
|
|
6
|
+
}): Promise<DigestPendingResult>;
|
|
7
|
+
export declare function initializeConversationDigestModel(input: InitializeConversationDigestInput): Promise<ConversationDigestInitResult>;
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
import * as fs from 'node:fs';
|
|
2
|
+
import * as path from 'node:path';
|
|
3
|
+
import { DEFAULT_DIGEST_TIMEOUT_MS, DEFAULT_DIGEST_MAX_INPUT_CHARS, DIGEST_LOCKFILE_REL_PATH, WIKI_RAW_DIR, } from './constants.js';
|
|
4
|
+
import { defaultProvisionalDigestDir, readConversationDigestInput, findExistingProvisionalDigest, writeProvisionalDigestFile, writeConversationDigestSettings, } from './files.js';
|
|
5
|
+
import { getDigestModelSpec } from './model-registry.js';
|
|
6
|
+
import { createLlamaLocalDigestProvider } from './providers/llama-local.js';
|
|
7
|
+
import { loadProjectSettings, resolveConversationDigestConfig, resolveMemoryConfig } from '../../tools/config.js';
|
|
8
|
+
import { MemoryStore } from '../store.js';
|
|
9
|
+
import { MemoryIndexer } from '../indexer.js';
|
|
10
|
+
import { releaseLock, tryAcquireProcessLock } from '../../utils/files.js';
|
|
11
|
+
function digestLockPath(workspaceRoot) {
|
|
12
|
+
return path.join(workspaceRoot, DIGEST_LOCKFILE_REL_PATH);
|
|
13
|
+
}
|
|
14
|
+
async function indexProvisionalDigestFile(workspaceRoot, markdownPath) {
|
|
15
|
+
try {
|
|
16
|
+
const settings = loadProjectSettings(workspaceRoot);
|
|
17
|
+
const config = resolveMemoryConfig(settings, workspaceRoot);
|
|
18
|
+
const store = new MemoryStore(path.join(config.wikiDir, 'index.db'), config);
|
|
19
|
+
const embedder = { embed: async (_texts) => [], initialize: async () => { } };
|
|
20
|
+
const indexer = new MemoryIndexer(store, embedder, config);
|
|
21
|
+
await indexer.indexFile(markdownPath);
|
|
22
|
+
return { indexed: true };
|
|
23
|
+
}
|
|
24
|
+
catch (err) {
|
|
25
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
26
|
+
return { indexed: false, error: message };
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
export async function digestConversationFile(options) {
|
|
30
|
+
const outDir = options.outDir
|
|
31
|
+
? path.resolve(options.workspaceRoot, options.outDir)
|
|
32
|
+
: defaultProvisionalDigestDir(options.workspaceRoot);
|
|
33
|
+
const input = readConversationDigestInput(options.workspaceRoot, options.inputPath);
|
|
34
|
+
const existingPath = findExistingProvisionalDigest(outDir, input);
|
|
35
|
+
if (existingPath) {
|
|
36
|
+
const indexResult = await indexProvisionalDigestFile(options.workspaceRoot, existingPath);
|
|
37
|
+
return {
|
|
38
|
+
markdown: existingPath,
|
|
39
|
+
status: 'provisional',
|
|
40
|
+
contentHash: input.contentHash,
|
|
41
|
+
skipped: true,
|
|
42
|
+
...indexResult,
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
const provider = await createLlamaLocalDigestProvider(options.modelId);
|
|
46
|
+
try {
|
|
47
|
+
const markdown = await provider.generateDigestMarkdown(input, {
|
|
48
|
+
modelId: options.modelId,
|
|
49
|
+
maxInputChars: options.maxInputChars ?? DEFAULT_DIGEST_MAX_INPUT_CHARS,
|
|
50
|
+
timeoutMs: options.timeoutMs ?? DEFAULT_DIGEST_TIMEOUT_MS,
|
|
51
|
+
});
|
|
52
|
+
const markdownPath = writeProvisionalDigestFile(outDir, input, markdown);
|
|
53
|
+
const indexResult = await indexProvisionalDigestFile(options.workspaceRoot, markdownPath);
|
|
54
|
+
return {
|
|
55
|
+
markdown: markdownPath,
|
|
56
|
+
status: 'provisional',
|
|
57
|
+
contentHash: input.contentHash,
|
|
58
|
+
skipped: false,
|
|
59
|
+
...indexResult,
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
finally {
|
|
63
|
+
await provider.dispose?.();
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
export async function digestPendingConversations({ workspaceRoot, digestFn = digestConversationFile, }) {
|
|
67
|
+
const digestConfig = resolveConversationDigestConfig(loadProjectSettings(workspaceRoot));
|
|
68
|
+
if (!digestConfig || digestConfig.enabled === false) {
|
|
69
|
+
return { ok: true, initialized: false, action: 'noop', reason: 'not-initialized' };
|
|
70
|
+
}
|
|
71
|
+
const lockPath = digestLockPath(workspaceRoot);
|
|
72
|
+
if (!tryAcquireProcessLock(lockPath)) {
|
|
73
|
+
return { ok: true, initialized: true, action: 'noop', reason: 'locked' };
|
|
74
|
+
}
|
|
75
|
+
try {
|
|
76
|
+
const rawDir = path.join(workspaceRoot, WIKI_RAW_DIR);
|
|
77
|
+
let files;
|
|
78
|
+
try {
|
|
79
|
+
files = fs
|
|
80
|
+
.readdirSync(rawDir)
|
|
81
|
+
.filter((f) => f.startsWith('conv_') && f.endsWith('.md'))
|
|
82
|
+
.map((f) => path.join(rawDir, f))
|
|
83
|
+
.sort((a, b) => fs.statSync(a).mtimeMs - fs.statSync(b).mtimeMs);
|
|
84
|
+
}
|
|
85
|
+
catch (err) {
|
|
86
|
+
if (err.code === 'ENOENT') {
|
|
87
|
+
files = [];
|
|
88
|
+
}
|
|
89
|
+
else {
|
|
90
|
+
throw err;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
const outDir = defaultProvisionalDigestDir(workspaceRoot);
|
|
94
|
+
const candidates = files.filter((f) => {
|
|
95
|
+
try {
|
|
96
|
+
const input = readConversationDigestInput(workspaceRoot, f);
|
|
97
|
+
return !findExistingProvisionalDigest(outDir, input);
|
|
98
|
+
}
|
|
99
|
+
catch {
|
|
100
|
+
return false;
|
|
101
|
+
}
|
|
102
|
+
});
|
|
103
|
+
if (candidates.length === 0) {
|
|
104
|
+
return { ok: true, initialized: true, action: 'noop', reason: 'no-pending' };
|
|
105
|
+
}
|
|
106
|
+
let count = 0;
|
|
107
|
+
let skipped = 0;
|
|
108
|
+
let errors = 0;
|
|
109
|
+
for (const filePath of candidates) {
|
|
110
|
+
try {
|
|
111
|
+
const result = await digestFn({
|
|
112
|
+
workspaceRoot,
|
|
113
|
+
inputPath: filePath,
|
|
114
|
+
modelId: digestConfig.modelId,
|
|
115
|
+
});
|
|
116
|
+
if (result.skipped) {
|
|
117
|
+
skipped++;
|
|
118
|
+
}
|
|
119
|
+
else {
|
|
120
|
+
count++;
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
catch {
|
|
124
|
+
errors++;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
return { ok: true, initialized: true, action: 'digested', count, skipped, errors };
|
|
128
|
+
}
|
|
129
|
+
catch (err) {
|
|
130
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
131
|
+
return { ok: false, initialized: true, action: 'error', error: message };
|
|
132
|
+
}
|
|
133
|
+
finally {
|
|
134
|
+
releaseLock(lockPath);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
export async function initializeConversationDigestModel(input) {
|
|
138
|
+
try {
|
|
139
|
+
getDigestModelSpec(input.modelId);
|
|
140
|
+
if (input.allowDownload !== true) {
|
|
141
|
+
return {
|
|
142
|
+
initialized: false,
|
|
143
|
+
modelId: input.modelId,
|
|
144
|
+
error: 'Digest model initialization requires explicit download permission',
|
|
145
|
+
};
|
|
146
|
+
}
|
|
147
|
+
const loadedProvider = await createLlamaLocalDigestProvider(input.modelId);
|
|
148
|
+
await loadedProvider?.dispose?.();
|
|
149
|
+
const initializedAt = new Date().toISOString();
|
|
150
|
+
writeConversationDigestSettings(input.workspaceRoot, {
|
|
151
|
+
enabled: input.enabled ?? true,
|
|
152
|
+
initialized: true,
|
|
153
|
+
modelId: input.modelId,
|
|
154
|
+
initializedAt,
|
|
155
|
+
});
|
|
156
|
+
await digestPendingConversations({ workspaceRoot: input.workspaceRoot });
|
|
157
|
+
return {
|
|
158
|
+
initialized: true,
|
|
159
|
+
modelId: input.modelId,
|
|
160
|
+
initializedAt,
|
|
161
|
+
};
|
|
162
|
+
}
|
|
163
|
+
catch (err) {
|
|
164
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
165
|
+
return {
|
|
166
|
+
initialized: false,
|
|
167
|
+
modelId: input.modelId,
|
|
168
|
+
error: message,
|
|
169
|
+
};
|
|
170
|
+
}
|
|
171
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { ConversationDigestProvider } from '../types.js';
|
|
2
|
+
export declare class LlamaLocalDigestProviderError extends Error {
|
|
3
|
+
constructor(message: string);
|
|
4
|
+
}
|
|
5
|
+
export declare function trimConversationExport(content: string, maxInputChars: number): string;
|
|
6
|
+
export declare function sanitizeConversationDigestMarkdown(generatedText: string): string;
|
|
7
|
+
export declare function createLlamaLocalDigestProvider(modelId: string): Promise<ConversationDigestProvider>;
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
import * as fs from 'node:fs';
|
|
2
|
+
import * as path from 'node:path';
|
|
3
|
+
import { getLlama, LlamaChatSession, createModelDownloader } from 'node-llama-cpp';
|
|
4
|
+
import { LLAMA_CONTEXT_SIZE, LLAMA_MAX_GENERATED_TOKENS, LLAMA_TEMPERATURE } from '../constants.js';
|
|
5
|
+
import { getDigestModelSpec } from '../model-registry.js';
|
|
6
|
+
import { withTimeout } from '../../../utils/async.js';
|
|
7
|
+
import { MODEL_CACHE_DIR } from '../../../utils/paths.js';
|
|
8
|
+
export class LlamaLocalDigestProviderError extends Error {
|
|
9
|
+
constructor(message) {
|
|
10
|
+
super(message);
|
|
11
|
+
this.name = 'LlamaLocalDigestProviderError';
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
function titleFromSource(sourcePath) {
|
|
15
|
+
return path
|
|
16
|
+
.basename(sourcePath)
|
|
17
|
+
.replace(/\.[^.]+$/, '')
|
|
18
|
+
.replace(/[_-]+/g, ' ');
|
|
19
|
+
}
|
|
20
|
+
export function trimConversationExport(content, maxInputChars) {
|
|
21
|
+
const trimmed = content.trim();
|
|
22
|
+
if (trimmed.length <= maxInputChars)
|
|
23
|
+
return trimmed;
|
|
24
|
+
return [
|
|
25
|
+
'[Earlier conversation content omitted to fit the local digest context window.]',
|
|
26
|
+
trimmed.slice(-maxInputChars),
|
|
27
|
+
].join('\n\n');
|
|
28
|
+
}
|
|
29
|
+
function buildPrompt(input, maxInputChars) {
|
|
30
|
+
const conversationExport = trimConversationExport(input.content, maxInputChars);
|
|
31
|
+
const sourceName = path.basename(input.sourcePath);
|
|
32
|
+
return [
|
|
33
|
+
{
|
|
34
|
+
role: 'system',
|
|
35
|
+
content: [
|
|
36
|
+
'# ROLE',
|
|
37
|
+
'You write short project-memory wiki pages from developer transcripts.',
|
|
38
|
+
'Your output is temporary recall to prevent blind steps.',
|
|
39
|
+
'',
|
|
40
|
+
'# RULES',
|
|
41
|
+
'1. Read the end of the export FIRST to discover the ultimate resolution.',
|
|
42
|
+
'2. What the user says LAST ALWAYS overrides earlier agreements.',
|
|
43
|
+
'3. Capture absolute engineering conclusions, never conversational pleasantries.',
|
|
44
|
+
'4. Return Markdown only. Do not wrap output in code blocks (```markdown ... ```).',
|
|
45
|
+
].join('\n'),
|
|
46
|
+
},
|
|
47
|
+
{
|
|
48
|
+
role: 'user',
|
|
49
|
+
content: [
|
|
50
|
+
'Generate a high-level project memory page. STRICTLY adhere to this architectural layout:',
|
|
51
|
+
'',
|
|
52
|
+
'<layout>',
|
|
53
|
+
'# Digest: ' + titleFromSource(input.sourcePath),
|
|
54
|
+
'- **Ultimate Core Resolution**: [Describe the final working state of the feature and what problem it solves in 1-2 sentences. No conversational filler.]',
|
|
55
|
+
'- **Architectural Changes**: [List 3-4 high-level engineering decisions made (e.g., shifts in hook timing, data isolation, atomic locks). Focus on systemic changes, NOT individual files.]',
|
|
56
|
+
'- **Component State**: [Briefly state how the Plugin side and the MCP/CLI side now interact based on the final decision.]',
|
|
57
|
+
'- **Considered & Rejected**: [List concepts proposed but discarded (e.g., initial hooks location, mocking strategies). Omit if empty.]',
|
|
58
|
+
'</layout>',
|
|
59
|
+
'',
|
|
60
|
+
'# CRITICAL RULES:',
|
|
61
|
+
'- DO NOT list individual file paths, line numbers, or specific test cases (EXCLUDE lists of files).',
|
|
62
|
+
'- Focus entirely on the SYSTEM DESIGN and ARCHITECTURE that the next engineer needs to know.',
|
|
63
|
+
'- Keep output under 300 tokens. Stop immediately after the last valid section.',
|
|
64
|
+
'',
|
|
65
|
+
'Source path: ' + input.sourcePath,
|
|
66
|
+
'<conversation_export format="memory-kit">',
|
|
67
|
+
conversationExport,
|
|
68
|
+
'</conversation_export>',
|
|
69
|
+
].join('\n'),
|
|
70
|
+
},
|
|
71
|
+
];
|
|
72
|
+
}
|
|
73
|
+
function stripCodeFence(text) {
|
|
74
|
+
const trimmed = text.trim();
|
|
75
|
+
const match = trimmed.match(/^```(?:markdown|md)?\s*\n([\s\S]*?)\n```$/i);
|
|
76
|
+
return match ? match[1].trim() : trimmed;
|
|
77
|
+
}
|
|
78
|
+
export function sanitizeConversationDigestMarkdown(generatedText) {
|
|
79
|
+
return stripCodeFence(generatedText).trim() + '\n';
|
|
80
|
+
}
|
|
81
|
+
export async function createLlamaLocalDigestProvider(modelId) {
|
|
82
|
+
const spec = getDigestModelSpec(modelId);
|
|
83
|
+
const modelCacheDir = path.join(MODEL_CACHE_DIR, 'llama');
|
|
84
|
+
fs.mkdirSync(modelCacheDir, { recursive: true });
|
|
85
|
+
let modelFilePath;
|
|
86
|
+
try {
|
|
87
|
+
const downloader = await createModelDownloader({
|
|
88
|
+
modelUri: spec.ggufUri,
|
|
89
|
+
dirPath: modelCacheDir,
|
|
90
|
+
});
|
|
91
|
+
modelFilePath = downloader.entrypointFilePath;
|
|
92
|
+
if (!fs.existsSync(modelFilePath)) {
|
|
93
|
+
await downloader.download();
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
catch (err) {
|
|
97
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
98
|
+
throw new LlamaLocalDigestProviderError(`Failed to download model: ${message}`);
|
|
99
|
+
}
|
|
100
|
+
let llama;
|
|
101
|
+
let model;
|
|
102
|
+
try {
|
|
103
|
+
llama = await getLlama();
|
|
104
|
+
model = await llama.loadModel({ modelPath: modelFilePath });
|
|
105
|
+
}
|
|
106
|
+
catch (err) {
|
|
107
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
108
|
+
throw new LlamaLocalDigestProviderError(`Failed to load model: ${message}`);
|
|
109
|
+
}
|
|
110
|
+
return {
|
|
111
|
+
id: 'llama-local',
|
|
112
|
+
async dispose() {
|
|
113
|
+
await model.dispose();
|
|
114
|
+
},
|
|
115
|
+
async generateDigestMarkdown(input, options) {
|
|
116
|
+
const messages = buildPrompt(input, options.maxInputChars);
|
|
117
|
+
const systemContent = messages[0].content;
|
|
118
|
+
const userContent = messages[1].content;
|
|
119
|
+
const context = await model.createContext({ contextSize: LLAMA_CONTEXT_SIZE });
|
|
120
|
+
try {
|
|
121
|
+
const session = new LlamaChatSession({
|
|
122
|
+
contextSequence: context.getSequence(),
|
|
123
|
+
systemPrompt: systemContent,
|
|
124
|
+
});
|
|
125
|
+
const response = await withTimeout(session.prompt(userContent, {
|
|
126
|
+
maxTokens: LLAMA_MAX_GENERATED_TOKENS,
|
|
127
|
+
temperature: LLAMA_TEMPERATURE,
|
|
128
|
+
}), options.timeoutMs, () => new LlamaLocalDigestProviderError(`Llama provider timed out after ${options.timeoutMs}ms`));
|
|
129
|
+
return sanitizeConversationDigestMarkdown(response);
|
|
130
|
+
}
|
|
131
|
+
finally {
|
|
132
|
+
await context.dispose();
|
|
133
|
+
}
|
|
134
|
+
},
|
|
135
|
+
};
|
|
136
|
+
}
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
export interface ConversationDigestSettings {
|
|
2
|
+
enabled?: boolean;
|
|
3
|
+
initialized: boolean;
|
|
4
|
+
modelId: string;
|
|
5
|
+
initializedAt: string;
|
|
6
|
+
}
|
|
7
|
+
export interface DigestModelSpec {
|
|
8
|
+
id: string;
|
|
9
|
+
ggufUri: string;
|
|
10
|
+
approxSizeBytes: number;
|
|
11
|
+
license: string;
|
|
12
|
+
sourceUrl: string;
|
|
13
|
+
enabled: boolean;
|
|
14
|
+
}
|
|
15
|
+
export interface ConversationDigestInput {
|
|
16
|
+
sourcePath: string;
|
|
17
|
+
content: string;
|
|
18
|
+
contentHash: string;
|
|
19
|
+
}
|
|
20
|
+
export interface ConversationDigestOptions {
|
|
21
|
+
modelId: string;
|
|
22
|
+
maxInputChars: number;
|
|
23
|
+
timeoutMs: number;
|
|
24
|
+
}
|
|
25
|
+
export interface ConversationDigestInitResult {
|
|
26
|
+
initialized: boolean;
|
|
27
|
+
modelId: string;
|
|
28
|
+
initializedAt?: string;
|
|
29
|
+
error?: string;
|
|
30
|
+
}
|
|
31
|
+
export interface ConversationDigestProvider {
|
|
32
|
+
readonly id: string;
|
|
33
|
+
dispose?(): Promise<void>;
|
|
34
|
+
generateDigestMarkdown(input: ConversationDigestInput, options: ConversationDigestOptions): Promise<string>;
|
|
35
|
+
}
|
|
36
|
+
export interface DigestFileOptions {
|
|
37
|
+
workspaceRoot: string;
|
|
38
|
+
inputPath: string;
|
|
39
|
+
modelId: string;
|
|
40
|
+
outDir?: string;
|
|
41
|
+
maxInputChars?: number;
|
|
42
|
+
timeoutMs?: number;
|
|
43
|
+
}
|
|
44
|
+
export interface ProvisionalDigestResult {
|
|
45
|
+
markdown: string;
|
|
46
|
+
status: 'provisional';
|
|
47
|
+
contentHash: string;
|
|
48
|
+
indexed: boolean;
|
|
49
|
+
skipped: boolean;
|
|
50
|
+
error?: string;
|
|
51
|
+
}
|
|
52
|
+
export interface InitializeConversationDigestInput {
|
|
53
|
+
workspaceRoot: string;
|
|
54
|
+
modelId: string;
|
|
55
|
+
allowDownload: boolean;
|
|
56
|
+
enabled?: boolean;
|
|
57
|
+
}
|
|
58
|
+
export type DigestPendingResult = {
|
|
59
|
+
ok: true;
|
|
60
|
+
initialized: false;
|
|
61
|
+
action: 'noop';
|
|
62
|
+
reason: 'not-initialized';
|
|
63
|
+
} | {
|
|
64
|
+
ok: true;
|
|
65
|
+
initialized: true;
|
|
66
|
+
action: 'noop';
|
|
67
|
+
reason: 'locked';
|
|
68
|
+
} | {
|
|
69
|
+
ok: true;
|
|
70
|
+
initialized: true;
|
|
71
|
+
action: 'noop';
|
|
72
|
+
reason: 'no-pending';
|
|
73
|
+
} | {
|
|
74
|
+
ok: true;
|
|
75
|
+
initialized: true;
|
|
76
|
+
action: 'digested';
|
|
77
|
+
count: number;
|
|
78
|
+
skipped: number;
|
|
79
|
+
errors: number;
|
|
80
|
+
} | {
|
|
81
|
+
ok: false;
|
|
82
|
+
initialized: true;
|
|
83
|
+
action: 'error';
|
|
84
|
+
error: string;
|
|
85
|
+
};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { spawnSync } from 'node:child_process';
|
|
2
|
+
import { parseArgs, numberFlag } from '../../utils/args.js';
|
|
3
|
+
import { DEFAULT_DIGEST_TIMEOUT_MS, DIGEST_TIMEOUT_GRACE_MS, DIGEST_WORKER_FLAG, DIGEST_WORKER_RESULT_PREFIX, } from './constants.js';
|
|
4
|
+
function isDigestWorkerResult(value) {
|
|
5
|
+
if (typeof value !== 'object' || value === null || Array.isArray(value))
|
|
6
|
+
return false;
|
|
7
|
+
const record = value;
|
|
8
|
+
return typeof record.markdown === 'string' && record.status === 'provisional';
|
|
9
|
+
}
|
|
10
|
+
export function parseDigestWorkerSuccess(stdout) {
|
|
11
|
+
const line = stdout
|
|
12
|
+
.split(/\r?\n/)
|
|
13
|
+
.reverse()
|
|
14
|
+
.find((candidate) => candidate.startsWith(DIGEST_WORKER_RESULT_PREFIX));
|
|
15
|
+
if (!line)
|
|
16
|
+
return false;
|
|
17
|
+
try {
|
|
18
|
+
return isDigestWorkerResult(JSON.parse(line.slice(DIGEST_WORKER_RESULT_PREFIX.length)));
|
|
19
|
+
}
|
|
20
|
+
catch {
|
|
21
|
+
return false;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
export function isDigestWorkerInvocation(args) {
|
|
25
|
+
return parseArgs(args).flags.get(DIGEST_WORKER_FLAG) === true;
|
|
26
|
+
}
|
|
27
|
+
export function runDigestFileInWorker(args) {
|
|
28
|
+
const parsed = parseArgs(args);
|
|
29
|
+
if (parsed.flags.get(DIGEST_WORKER_FLAG) === true)
|
|
30
|
+
return undefined;
|
|
31
|
+
const entrypoint = process.argv[1];
|
|
32
|
+
if (!entrypoint)
|
|
33
|
+
return undefined;
|
|
34
|
+
const timeoutMs = numberFlag(parsed.flags, 'timeout-ms', DEFAULT_DIGEST_TIMEOUT_MS);
|
|
35
|
+
const result = spawnSync(process.execPath, [entrypoint, 'memory', 'digest-file', ...args, `--${DIGEST_WORKER_FLAG}`], {
|
|
36
|
+
cwd: process.cwd(),
|
|
37
|
+
encoding: 'utf8',
|
|
38
|
+
maxBuffer: 1024 * 1024 * 20,
|
|
39
|
+
timeout: timeoutMs + DIGEST_TIMEOUT_GRACE_MS,
|
|
40
|
+
});
|
|
41
|
+
if (result.stdout)
|
|
42
|
+
process.stdout.write(result.stdout);
|
|
43
|
+
const workerSucceeded = parseDigestWorkerSuccess(result.stdout);
|
|
44
|
+
if (result.stderr && !workerSucceeded)
|
|
45
|
+
process.stderr.write(result.stderr);
|
|
46
|
+
if (result.status === 0 || workerSucceeded)
|
|
47
|
+
return 0;
|
|
48
|
+
if (result.status !== null)
|
|
49
|
+
return result.status;
|
|
50
|
+
return 1;
|
|
51
|
+
}
|
package/dist/memory/embedder.js
CHANGED
|
@@ -40,6 +40,11 @@ export class Embedder {
|
|
|
40
40
|
isReady() {
|
|
41
41
|
return this.pipeline !== null;
|
|
42
42
|
}
|
|
43
|
+
async initialize() {
|
|
44
|
+
if (!this.pipeline) {
|
|
45
|
+
await this.loadModel();
|
|
46
|
+
}
|
|
47
|
+
}
|
|
43
48
|
async loadModel() {
|
|
44
49
|
try {
|
|
45
50
|
const { EmbeddingModel, FlagEmbedding } = await import('fastembed');
|
|
@@ -49,7 +54,11 @@ export class Embedder {
|
|
|
49
54
|
'Xenova/all-MiniLM-L6-v2': EmbeddingModel.AllMiniLML6V2,
|
|
50
55
|
};
|
|
51
56
|
const model = modelMap[this.modelName] ?? EmbeddingModel.BGESmallENV15;
|
|
52
|
-
const embedder = await FlagEmbedding.init({
|
|
57
|
+
const embedder = await FlagEmbedding.init({
|
|
58
|
+
model: model,
|
|
59
|
+
cacheDir: FASTEMBED_CACHE_DIR,
|
|
60
|
+
showDownloadProgress: false,
|
|
61
|
+
});
|
|
53
62
|
this.pipeline = async (texts) => {
|
|
54
63
|
const result = [];
|
|
55
64
|
for await (const batch of embedder.embed(texts, texts.length)) {
|
package/dist/memory/index.d.ts
CHANGED
|
@@ -2,4 +2,4 @@ export { MemoryStore, StoreError } from './store.js';
|
|
|
2
2
|
export { Embedder, EmbedderError } from './embedder.js';
|
|
3
3
|
export { MemoryIndexer } from './indexer.js';
|
|
4
4
|
export type { MemoryChunk, SearchResult, MemoryConfig, IndexStats } from './types.js';
|
|
5
|
-
export { DEFAULT_MEMORY_CONFIG } from './
|
|
5
|
+
export { DEFAULT_MEMORY_CONFIG } from './constants.js';
|
package/dist/memory/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
export { MemoryStore, StoreError } from './store.js';
|
|
2
2
|
export { Embedder, EmbedderError } from './embedder.js';
|
|
3
3
|
export { MemoryIndexer } from './indexer.js';
|
|
4
|
-
export { DEFAULT_MEMORY_CONFIG } from './
|
|
4
|
+
export { DEFAULT_MEMORY_CONFIG } from './constants.js';
|
package/dist/memory/indexer.d.ts
CHANGED
|
@@ -1,12 +1,15 @@
|
|
|
1
|
-
import type { Embedder } from './embedder.js';
|
|
2
1
|
import type { MemoryStore } from './store.js';
|
|
3
2
|
import type { IndexStats, MemoryConfig, SearchResult } from './types.js';
|
|
3
|
+
export interface TextEmbedder {
|
|
4
|
+
embed(texts: string[]): Promise<Float32Array[]>;
|
|
5
|
+
initialize(): Promise<void>;
|
|
6
|
+
}
|
|
4
7
|
export declare class MemoryIndexer {
|
|
5
8
|
private readonly store;
|
|
6
9
|
private readonly embedder;
|
|
7
10
|
private readonly config;
|
|
8
11
|
private _ready;
|
|
9
|
-
constructor(store: MemoryStore, embedder:
|
|
12
|
+
constructor(store: MemoryStore, embedder: TextEmbedder, config: MemoryConfig);
|
|
10
13
|
indexFile(absolutePath: string): Promise<IndexStats>;
|
|
11
14
|
private indexFileRelativeTo;
|
|
12
15
|
indexDirectory(rootDir: string, opts?: {
|
package/dist/memory/indexer.js
CHANGED
|
@@ -1,29 +1,8 @@
|
|
|
1
1
|
import * as fs from 'fs';
|
|
2
2
|
import * as path from 'path';
|
|
3
3
|
import { chunkMarkdown } from './chunker.js';
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
async function acquireLock(lockPath) {
|
|
7
|
-
const deadline = Date.now() + LOCK_TIMEOUT_MS;
|
|
8
|
-
while (Date.now() < deadline) {
|
|
9
|
-
try {
|
|
10
|
-
fs.writeFileSync(lockPath, String(process.pid), { flag: 'wx' });
|
|
11
|
-
return true;
|
|
12
|
-
}
|
|
13
|
-
catch {
|
|
14
|
-
await new Promise((r) => setTimeout(r, LOCK_RETRY_MS));
|
|
15
|
-
}
|
|
16
|
-
}
|
|
17
|
-
return false;
|
|
18
|
-
}
|
|
19
|
-
function releaseLock(lockPath) {
|
|
20
|
-
try {
|
|
21
|
-
fs.unlinkSync(lockPath);
|
|
22
|
-
}
|
|
23
|
-
catch {
|
|
24
|
-
// ignore
|
|
25
|
-
}
|
|
26
|
-
}
|
|
4
|
+
import { LOCK_RETRY_MS, LOCK_TIMEOUT_MS, RRF_K } from './constants.js';
|
|
5
|
+
import { acquireLock, releaseLock } from '../utils/files.js';
|
|
27
6
|
export class MemoryIndexer {
|
|
28
7
|
store;
|
|
29
8
|
embedder;
|
|
@@ -39,19 +18,19 @@ export class MemoryIndexer {
|
|
|
39
18
|
}
|
|
40
19
|
async indexFileRelativeTo(absolutePath, sourceRoot) {
|
|
41
20
|
const source = path.relative(sourceRoot, absolutePath);
|
|
42
|
-
const
|
|
21
|
+
const existingChunkIds = this.store.hashesBySource(source);
|
|
43
22
|
let text;
|
|
44
23
|
try {
|
|
45
24
|
text = fs.readFileSync(absolutePath, 'utf8');
|
|
46
25
|
}
|
|
47
26
|
catch (err) {
|
|
48
27
|
console.warn(`[memory-indexer] Cannot read file ${absolutePath}:`, err);
|
|
49
|
-
return { indexed: 0, deleted: 0, skipped:
|
|
28
|
+
return { indexed: 0, deleted: 0, skipped: existingChunkIds.size };
|
|
50
29
|
}
|
|
51
30
|
const allChunks = chunkMarkdown(text, source, this.config);
|
|
52
|
-
const
|
|
53
|
-
const toIndex = allChunks.filter((
|
|
54
|
-
const toDelete = [...
|
|
31
|
+
const currentChunkIds = new Set(allChunks.map((chunk) => chunk.id));
|
|
32
|
+
const toIndex = allChunks.filter((chunk) => !existingChunkIds.has(chunk.id));
|
|
33
|
+
const toDelete = [...existingChunkIds].filter((id) => !currentChunkIds.has(id));
|
|
55
34
|
let embeddings = [];
|
|
56
35
|
if (toIndex.length > 0) {
|
|
57
36
|
try {
|
|
@@ -59,7 +38,7 @@ export class MemoryIndexer {
|
|
|
59
38
|
}
|
|
60
39
|
catch (err) {
|
|
61
40
|
console.warn(`[memory-indexer] Embedding failed for ${absolutePath}:`, err);
|
|
62
|
-
return { indexed: 0, deleted: 0, skipped:
|
|
41
|
+
return { indexed: 0, deleted: 0, skipped: existingChunkIds.size };
|
|
63
42
|
}
|
|
64
43
|
}
|
|
65
44
|
if (toIndex.length > 0) {
|
|
@@ -135,24 +114,22 @@ export class MemoryIndexer {
|
|
|
135
114
|
if (bm25Ids.size > 0) {
|
|
136
115
|
denseResults = denseResults.filter((result) => bm25Ids.has(result.id));
|
|
137
116
|
}
|
|
138
|
-
// RRF fusion (k=60)
|
|
139
|
-
const k = 60;
|
|
140
117
|
const scoreMap = new Map();
|
|
141
118
|
denseResults.forEach((r, rank) => {
|
|
142
|
-
scoreMap.set(r.id, { dense: 1 / (
|
|
119
|
+
scoreMap.set(r.id, { dense: 1 / (RRF_K + rank + 1), bm25: 0 });
|
|
143
120
|
});
|
|
144
121
|
bm25Results.forEach((r, rank) => {
|
|
145
122
|
const existing = scoreMap.get(r.id);
|
|
146
123
|
if (existing) {
|
|
147
|
-
existing.bm25 = 1 / (
|
|
124
|
+
existing.bm25 = 1 / (RRF_K + rank + 1);
|
|
148
125
|
}
|
|
149
126
|
else {
|
|
150
|
-
scoreMap.set(r.id, { dense: 0, bm25: 1 / (
|
|
127
|
+
scoreMap.set(r.id, { dense: 0, bm25: 1 / (RRF_K + rank + 1) });
|
|
151
128
|
}
|
|
152
129
|
});
|
|
153
130
|
const hasDense = this.store.vecAvailable && denseResults.length > 0;
|
|
154
131
|
const numRetrievers = hasDense ? 2 : 1;
|
|
155
|
-
const maxScore = numRetrievers /
|
|
132
|
+
const maxScore = numRetrievers / (RRF_K + 1);
|
|
156
133
|
const ranked = [...scoreMap.entries()]
|
|
157
134
|
.map(([id, scores]) => ({
|
|
158
135
|
id,
|
|
@@ -193,7 +170,7 @@ export class MemoryIndexer {
|
|
|
193
170
|
const savePath = path.join(rawDir, `conv_save_${datePart}.md`);
|
|
194
171
|
const lockPath = `${savePath}.lock`;
|
|
195
172
|
fs.mkdirSync(rawDir, { recursive: true });
|
|
196
|
-
const acquired = await acquireLock(lockPath);
|
|
173
|
+
const acquired = await acquireLock(lockPath, LOCK_TIMEOUT_MS, LOCK_RETRY_MS);
|
|
197
174
|
if (!acquired) {
|
|
198
175
|
console.warn('[memory-indexer] Could not acquire write lock, writing anyway (fail-open)');
|
|
199
176
|
}
|
|
@@ -202,12 +179,13 @@ export class MemoryIndexer {
|
|
|
202
179
|
}
|
|
203
180
|
finally {
|
|
204
181
|
if (acquired)
|
|
205
|
-
releaseLock(lockPath);
|
|
182
|
+
releaseLock(lockPath, { ignoreErrors: true });
|
|
206
183
|
}
|
|
207
184
|
return { indexed: 0, deleted: 0, skipped: 0 };
|
|
208
185
|
}
|
|
209
186
|
async startupIndex() {
|
|
210
187
|
try {
|
|
188
|
+
await this.embedder.initialize();
|
|
211
189
|
await this.indexDirectory(path.join(this.config.wikiDir, 'compiled'), {
|
|
212
190
|
relativeBase: this.config.wikiDir,
|
|
213
191
|
excludeFiles: ['index.md', 'log.md'],
|