@hanhnd/agent-kit 1.0.31 → 1.0.32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. package/dist/__tests__/temp-dir.d.ts +4 -0
  2. package/dist/__tests__/temp-dir.js +20 -0
  3. package/dist/cli/index.d.ts +1 -0
  4. package/dist/cli/index.js +9 -0
  5. package/dist/cli/memory.d.ts +1 -0
  6. package/dist/cli/memory.js +77 -0
  7. package/dist/kit-cli.d.ts +2 -0
  8. package/dist/kit-cli.js +16205 -0
  9. package/dist/kit-server.js +679 -193
  10. package/dist/memory/__tests__/digest-llama-local.test.d.ts +1 -0
  11. package/dist/memory/__tests__/digest-llama-local.test.js +33 -0
  12. package/dist/memory/__tests__/digest-pending.test.d.ts +1 -0
  13. package/dist/memory/__tests__/digest-pending.test.js +164 -0
  14. package/dist/memory/__tests__/digest-processor.test.d.ts +1 -0
  15. package/dist/memory/__tests__/digest-processor.test.js +34 -0
  16. package/dist/memory/__tests__/digest-worker.test.d.ts +1 -0
  17. package/dist/memory/__tests__/digest-worker.test.js +46 -0
  18. package/dist/memory/__tests__/indexer.test.js +7 -12
  19. package/dist/memory/__tests__/lockfile.test.d.ts +1 -0
  20. package/dist/memory/__tests__/lockfile.test.js +81 -0
  21. package/dist/memory/__tests__/store.test.js +15 -3
  22. package/dist/memory/chunker.js +3 -5
  23. package/dist/memory/constants.d.ts +5 -0
  24. package/dist/memory/constants.js +12 -0
  25. package/dist/memory/digest/constants.d.ts +14 -0
  26. package/dist/memory/digest/constants.js +15 -0
  27. package/dist/memory/digest/files.d.ts +7 -0
  28. package/dist/memory/digest/files.js +45 -0
  29. package/dist/memory/digest/model-registry.d.ts +6 -0
  30. package/dist/memory/digest/model-registry.js +48 -0
  31. package/dist/memory/digest/processor.d.ts +7 -0
  32. package/dist/memory/digest/processor.js +171 -0
  33. package/dist/memory/digest/providers/llama-local.d.ts +7 -0
  34. package/dist/memory/digest/providers/llama-local.js +136 -0
  35. package/dist/memory/digest/types.d.ts +85 -0
  36. package/dist/memory/digest/types.js +1 -0
  37. package/dist/memory/digest/worker.d.ts +3 -0
  38. package/dist/memory/digest/worker.js +51 -0
  39. package/dist/memory/embedder.d.ts +1 -0
  40. package/dist/memory/embedder.js +10 -1
  41. package/dist/memory/index.d.ts +1 -1
  42. package/dist/memory/index.js +1 -1
  43. package/dist/memory/indexer.d.ts +5 -2
  44. package/dist/memory/indexer.js +15 -37
  45. package/dist/memory/store.js +6 -18
  46. package/dist/memory/types.d.ts +2 -1
  47. package/dist/memory/types.js +1 -9
  48. package/dist/tools/__tests__/cli-digest-pending.test.d.ts +1 -0
  49. package/dist/tools/__tests__/cli-digest-pending.test.js +51 -0
  50. package/dist/tools/__tests__/cli-memory.test.d.ts +1 -0
  51. package/dist/tools/__tests__/cli-memory.test.js +22 -0
  52. package/dist/tools/__tests__/digest-init.test.d.ts +1 -0
  53. package/dist/tools/__tests__/digest-init.test.js +32 -0
  54. package/dist/tools/__tests__/memory.test.js +28 -6
  55. package/dist/tools/agent.js +2 -13
  56. package/dist/tools/config.d.ts +4 -4
  57. package/dist/tools/config.js +20 -12
  58. package/dist/tools/core.js +1 -3
  59. package/dist/tools/handoffs.js +1 -2
  60. package/dist/tools/integration.js +2 -8
  61. package/dist/tools/memory.d.ts +1 -1
  62. package/dist/tools/memory.js +29 -3
  63. package/dist/tools/security.js +2 -6
  64. package/dist/utils/args.d.ts +7 -0
  65. package/dist/utils/args.js +31 -0
  66. package/dist/utils/async.d.ts +1 -0
  67. package/dist/utils/async.js +15 -0
  68. package/dist/utils/files.d.ts +8 -0
  69. package/dist/utils/files.js +77 -0
  70. package/dist/utils/hash.d.ts +1 -0
  71. package/dist/utils/hash.js +4 -0
  72. package/dist/utils/json.d.ts +2 -0
  73. package/dist/utils/json.js +13 -0
  74. package/dist/utils/parser.js +2 -6
  75. package/package.json +7 -5
@@ -0,0 +1,7 @@
1
+ import type { DigestFileOptions, DigestPendingResult, ProvisionalDigestResult, ConversationDigestInitResult, InitializeConversationDigestInput } from './types.js';
2
+ export declare function digestConversationFile(options: DigestFileOptions): Promise<ProvisionalDigestResult>;
3
+ export declare function digestPendingConversations({ workspaceRoot, digestFn, }: {
4
+ workspaceRoot: string;
5
+ digestFn?: (opts: DigestFileOptions) => Promise<ProvisionalDigestResult>;
6
+ }): Promise<DigestPendingResult>;
7
+ export declare function initializeConversationDigestModel(input: InitializeConversationDigestInput): Promise<ConversationDigestInitResult>;
@@ -0,0 +1,171 @@
1
+ import * as fs from 'node:fs';
2
+ import * as path from 'node:path';
3
+ import { DEFAULT_DIGEST_TIMEOUT_MS, DEFAULT_DIGEST_MAX_INPUT_CHARS, DIGEST_LOCKFILE_REL_PATH, WIKI_RAW_DIR, } from './constants.js';
4
+ import { defaultProvisionalDigestDir, readConversationDigestInput, findExistingProvisionalDigest, writeProvisionalDigestFile, writeConversationDigestSettings, } from './files.js';
5
+ import { getDigestModelSpec } from './model-registry.js';
6
+ import { createLlamaLocalDigestProvider } from './providers/llama-local.js';
7
+ import { loadProjectSettings, resolveConversationDigestConfig, resolveMemoryConfig } from '../../tools/config.js';
8
+ import { MemoryStore } from '../store.js';
9
+ import { MemoryIndexer } from '../indexer.js';
10
+ import { releaseLock, tryAcquireProcessLock } from '../../utils/files.js';
11
+ function digestLockPath(workspaceRoot) {
12
+ return path.join(workspaceRoot, DIGEST_LOCKFILE_REL_PATH);
13
+ }
14
+ async function indexProvisionalDigestFile(workspaceRoot, markdownPath) {
15
+ try {
16
+ const settings = loadProjectSettings(workspaceRoot);
17
+ const config = resolveMemoryConfig(settings, workspaceRoot);
18
+ const store = new MemoryStore(path.join(config.wikiDir, 'index.db'), config);
19
+ const embedder = { embed: async (_texts) => [], initialize: async () => { } };
20
+ const indexer = new MemoryIndexer(store, embedder, config);
21
+ await indexer.indexFile(markdownPath);
22
+ return { indexed: true };
23
+ }
24
+ catch (err) {
25
+ const message = err instanceof Error ? err.message : String(err);
26
+ return { indexed: false, error: message };
27
+ }
28
+ }
29
+ export async function digestConversationFile(options) {
30
+ const outDir = options.outDir
31
+ ? path.resolve(options.workspaceRoot, options.outDir)
32
+ : defaultProvisionalDigestDir(options.workspaceRoot);
33
+ const input = readConversationDigestInput(options.workspaceRoot, options.inputPath);
34
+ const existingPath = findExistingProvisionalDigest(outDir, input);
35
+ if (existingPath) {
36
+ const indexResult = await indexProvisionalDigestFile(options.workspaceRoot, existingPath);
37
+ return {
38
+ markdown: existingPath,
39
+ status: 'provisional',
40
+ contentHash: input.contentHash,
41
+ skipped: true,
42
+ ...indexResult,
43
+ };
44
+ }
45
+ const provider = await createLlamaLocalDigestProvider(options.modelId);
46
+ try {
47
+ const markdown = await provider.generateDigestMarkdown(input, {
48
+ modelId: options.modelId,
49
+ maxInputChars: options.maxInputChars ?? DEFAULT_DIGEST_MAX_INPUT_CHARS,
50
+ timeoutMs: options.timeoutMs ?? DEFAULT_DIGEST_TIMEOUT_MS,
51
+ });
52
+ const markdownPath = writeProvisionalDigestFile(outDir, input, markdown);
53
+ const indexResult = await indexProvisionalDigestFile(options.workspaceRoot, markdownPath);
54
+ return {
55
+ markdown: markdownPath,
56
+ status: 'provisional',
57
+ contentHash: input.contentHash,
58
+ skipped: false,
59
+ ...indexResult,
60
+ };
61
+ }
62
+ finally {
63
+ await provider.dispose?.();
64
+ }
65
+ }
66
+ export async function digestPendingConversations({ workspaceRoot, digestFn = digestConversationFile, }) {
67
+ const digestConfig = resolveConversationDigestConfig(loadProjectSettings(workspaceRoot));
68
+ if (!digestConfig || digestConfig.enabled === false) {
69
+ return { ok: true, initialized: false, action: 'noop', reason: 'not-initialized' };
70
+ }
71
+ const lockPath = digestLockPath(workspaceRoot);
72
+ if (!tryAcquireProcessLock(lockPath)) {
73
+ return { ok: true, initialized: true, action: 'noop', reason: 'locked' };
74
+ }
75
+ try {
76
+ const rawDir = path.join(workspaceRoot, WIKI_RAW_DIR);
77
+ let files;
78
+ try {
79
+ files = fs
80
+ .readdirSync(rawDir)
81
+ .filter((f) => f.startsWith('conv_') && f.endsWith('.md'))
82
+ .map((f) => path.join(rawDir, f))
83
+ .sort((a, b) => fs.statSync(a).mtimeMs - fs.statSync(b).mtimeMs);
84
+ }
85
+ catch (err) {
86
+ if (err.code === 'ENOENT') {
87
+ files = [];
88
+ }
89
+ else {
90
+ throw err;
91
+ }
92
+ }
93
+ const outDir = defaultProvisionalDigestDir(workspaceRoot);
94
+ const candidates = files.filter((f) => {
95
+ try {
96
+ const input = readConversationDigestInput(workspaceRoot, f);
97
+ return !findExistingProvisionalDigest(outDir, input);
98
+ }
99
+ catch {
100
+ return false;
101
+ }
102
+ });
103
+ if (candidates.length === 0) {
104
+ return { ok: true, initialized: true, action: 'noop', reason: 'no-pending' };
105
+ }
106
+ let count = 0;
107
+ let skipped = 0;
108
+ let errors = 0;
109
+ for (const filePath of candidates) {
110
+ try {
111
+ const result = await digestFn({
112
+ workspaceRoot,
113
+ inputPath: filePath,
114
+ modelId: digestConfig.modelId,
115
+ });
116
+ if (result.skipped) {
117
+ skipped++;
118
+ }
119
+ else {
120
+ count++;
121
+ }
122
+ }
123
+ catch {
124
+ errors++;
125
+ }
126
+ }
127
+ return { ok: true, initialized: true, action: 'digested', count, skipped, errors };
128
+ }
129
+ catch (err) {
130
+ const message = err instanceof Error ? err.message : String(err);
131
+ return { ok: false, initialized: true, action: 'error', error: message };
132
+ }
133
+ finally {
134
+ releaseLock(lockPath);
135
+ }
136
+ }
137
+ export async function initializeConversationDigestModel(input) {
138
+ try {
139
+ getDigestModelSpec(input.modelId);
140
+ if (input.allowDownload !== true) {
141
+ return {
142
+ initialized: false,
143
+ modelId: input.modelId,
144
+ error: 'Digest model initialization requires explicit download permission',
145
+ };
146
+ }
147
+ const loadedProvider = await createLlamaLocalDigestProvider(input.modelId);
148
+ await loadedProvider?.dispose?.();
149
+ const initializedAt = new Date().toISOString();
150
+ writeConversationDigestSettings(input.workspaceRoot, {
151
+ enabled: input.enabled ?? true,
152
+ initialized: true,
153
+ modelId: input.modelId,
154
+ initializedAt,
155
+ });
156
+ await digestPendingConversations({ workspaceRoot: input.workspaceRoot });
157
+ return {
158
+ initialized: true,
159
+ modelId: input.modelId,
160
+ initializedAt,
161
+ };
162
+ }
163
+ catch (err) {
164
+ const message = err instanceof Error ? err.message : String(err);
165
+ return {
166
+ initialized: false,
167
+ modelId: input.modelId,
168
+ error: message,
169
+ };
170
+ }
171
+ }
@@ -0,0 +1,7 @@
1
+ import type { ConversationDigestProvider } from '../types.js';
2
+ export declare class LlamaLocalDigestProviderError extends Error {
3
+ constructor(message: string);
4
+ }
5
+ export declare function trimConversationExport(content: string, maxInputChars: number): string;
6
+ export declare function sanitizeConversationDigestMarkdown(generatedText: string): string;
7
+ export declare function createLlamaLocalDigestProvider(modelId: string): Promise<ConversationDigestProvider>;
@@ -0,0 +1,136 @@
1
+ import * as fs from 'node:fs';
2
+ import * as path from 'node:path';
3
+ import { getLlama, LlamaChatSession, createModelDownloader } from 'node-llama-cpp';
4
+ import { LLAMA_CONTEXT_SIZE, LLAMA_MAX_GENERATED_TOKENS, LLAMA_TEMPERATURE } from '../constants.js';
5
+ import { getDigestModelSpec } from '../model-registry.js';
6
+ import { withTimeout } from '../../../utils/async.js';
7
+ import { MODEL_CACHE_DIR } from '../../../utils/paths.js';
8
+ export class LlamaLocalDigestProviderError extends Error {
9
+ constructor(message) {
10
+ super(message);
11
+ this.name = 'LlamaLocalDigestProviderError';
12
+ }
13
+ }
14
+ function titleFromSource(sourcePath) {
15
+ return path
16
+ .basename(sourcePath)
17
+ .replace(/\.[^.]+$/, '')
18
+ .replace(/[_-]+/g, ' ');
19
+ }
20
+ export function trimConversationExport(content, maxInputChars) {
21
+ const trimmed = content.trim();
22
+ if (trimmed.length <= maxInputChars)
23
+ return trimmed;
24
+ return [
25
+ '[Earlier conversation content omitted to fit the local digest context window.]',
26
+ trimmed.slice(-maxInputChars),
27
+ ].join('\n\n');
28
+ }
29
+ function buildPrompt(input, maxInputChars) {
30
+ const conversationExport = trimConversationExport(input.content, maxInputChars);
31
+ const sourceName = path.basename(input.sourcePath);
32
+ return [
33
+ {
34
+ role: 'system',
35
+ content: [
36
+ '# ROLE',
37
+ 'You write short project-memory wiki pages from developer transcripts.',
38
+ 'Your output is temporary recall to prevent blind steps.',
39
+ '',
40
+ '# RULES',
41
+ '1. Read the end of the export FIRST to discover the ultimate resolution.',
42
+ '2. What the user says LAST ALWAYS overrides earlier agreements.',
43
+ '3. Capture absolute engineering conclusions, never conversational pleasantries.',
44
+ '4. Return Markdown only. Do not wrap output in code blocks (```markdown ... ```).',
45
+ ].join('\n'),
46
+ },
47
+ {
48
+ role: 'user',
49
+ content: [
50
+ 'Generate a high-level project memory page. STRICTLY adhere to this architectural layout:',
51
+ '',
52
+ '<layout>',
53
+ '# Digest: ' + titleFromSource(input.sourcePath),
54
+ '- **Ultimate Core Resolution**: [Describe the final working state of the feature and what problem it solves in 1-2 sentences. No conversational filler.]',
55
+ '- **Architectural Changes**: [List 3-4 high-level engineering decisions made (e.g., shifts in hook timing, data isolation, atomic locks). Focus on systemic changes, NOT individual files.]',
56
+ '- **Component State**: [Briefly state how the Plugin side and the MCP/CLI side now interact based on the final decision.]',
57
+ '- **Considered & Rejected**: [List concepts proposed but discarded (e.g., initial hooks location, mocking strategies). Omit if empty.]',
58
+ '</layout>',
59
+ '',
60
+ '# CRITICAL RULES:',
61
+ '- DO NOT list individual file paths, line numbers, or specific test cases (EXCLUDE lists of files).',
62
+ '- Focus entirely on the SYSTEM DESIGN and ARCHITECTURE that the next engineer needs to know.',
63
+ '- Keep output under 300 tokens. Stop immediately after the last valid section.',
64
+ '',
65
+ 'Source path: ' + input.sourcePath,
66
+ '<conversation_export format="memory-kit">',
67
+ conversationExport,
68
+ '</conversation_export>',
69
+ ].join('\n'),
70
+ },
71
+ ];
72
+ }
73
+ function stripCodeFence(text) {
74
+ const trimmed = text.trim();
75
+ const match = trimmed.match(/^```(?:markdown|md)?\s*\n([\s\S]*?)\n```$/i);
76
+ return match ? match[1].trim() : trimmed;
77
+ }
78
+ export function sanitizeConversationDigestMarkdown(generatedText) {
79
+ return stripCodeFence(generatedText).trim() + '\n';
80
+ }
81
+ export async function createLlamaLocalDigestProvider(modelId) {
82
+ const spec = getDigestModelSpec(modelId);
83
+ const modelCacheDir = path.join(MODEL_CACHE_DIR, 'llama');
84
+ fs.mkdirSync(modelCacheDir, { recursive: true });
85
+ let modelFilePath;
86
+ try {
87
+ const downloader = await createModelDownloader({
88
+ modelUri: spec.ggufUri,
89
+ dirPath: modelCacheDir,
90
+ });
91
+ modelFilePath = downloader.entrypointFilePath;
92
+ if (!fs.existsSync(modelFilePath)) {
93
+ await downloader.download();
94
+ }
95
+ }
96
+ catch (err) {
97
+ const message = err instanceof Error ? err.message : String(err);
98
+ throw new LlamaLocalDigestProviderError(`Failed to download model: ${message}`);
99
+ }
100
+ let llama;
101
+ let model;
102
+ try {
103
+ llama = await getLlama();
104
+ model = await llama.loadModel({ modelPath: modelFilePath });
105
+ }
106
+ catch (err) {
107
+ const message = err instanceof Error ? err.message : String(err);
108
+ throw new LlamaLocalDigestProviderError(`Failed to load model: ${message}`);
109
+ }
110
+ return {
111
+ id: 'llama-local',
112
+ async dispose() {
113
+ await model.dispose();
114
+ },
115
+ async generateDigestMarkdown(input, options) {
116
+ const messages = buildPrompt(input, options.maxInputChars);
117
+ const systemContent = messages[0].content;
118
+ const userContent = messages[1].content;
119
+ const context = await model.createContext({ contextSize: LLAMA_CONTEXT_SIZE });
120
+ try {
121
+ const session = new LlamaChatSession({
122
+ contextSequence: context.getSequence(),
123
+ systemPrompt: systemContent,
124
+ });
125
+ const response = await withTimeout(session.prompt(userContent, {
126
+ maxTokens: LLAMA_MAX_GENERATED_TOKENS,
127
+ temperature: LLAMA_TEMPERATURE,
128
+ }), options.timeoutMs, () => new LlamaLocalDigestProviderError(`Llama provider timed out after ${options.timeoutMs}ms`));
129
+ return sanitizeConversationDigestMarkdown(response);
130
+ }
131
+ finally {
132
+ await context.dispose();
133
+ }
134
+ },
135
+ };
136
+ }
@@ -0,0 +1,85 @@
1
+ export interface ConversationDigestSettings {
2
+ enabled?: boolean;
3
+ initialized: boolean;
4
+ modelId: string;
5
+ initializedAt: string;
6
+ }
7
+ export interface DigestModelSpec {
8
+ id: string;
9
+ ggufUri: string;
10
+ approxSizeBytes: number;
11
+ license: string;
12
+ sourceUrl: string;
13
+ enabled: boolean;
14
+ }
15
+ export interface ConversationDigestInput {
16
+ sourcePath: string;
17
+ content: string;
18
+ contentHash: string;
19
+ }
20
+ export interface ConversationDigestOptions {
21
+ modelId: string;
22
+ maxInputChars: number;
23
+ timeoutMs: number;
24
+ }
25
+ export interface ConversationDigestInitResult {
26
+ initialized: boolean;
27
+ modelId: string;
28
+ initializedAt?: string;
29
+ error?: string;
30
+ }
31
+ export interface ConversationDigestProvider {
32
+ readonly id: string;
33
+ dispose?(): Promise<void>;
34
+ generateDigestMarkdown(input: ConversationDigestInput, options: ConversationDigestOptions): Promise<string>;
35
+ }
36
+ export interface DigestFileOptions {
37
+ workspaceRoot: string;
38
+ inputPath: string;
39
+ modelId: string;
40
+ outDir?: string;
41
+ maxInputChars?: number;
42
+ timeoutMs?: number;
43
+ }
44
+ export interface ProvisionalDigestResult {
45
+ markdown: string;
46
+ status: 'provisional';
47
+ contentHash: string;
48
+ indexed: boolean;
49
+ skipped: boolean;
50
+ error?: string;
51
+ }
52
+ export interface InitializeConversationDigestInput {
53
+ workspaceRoot: string;
54
+ modelId: string;
55
+ allowDownload: boolean;
56
+ enabled?: boolean;
57
+ }
58
+ export type DigestPendingResult = {
59
+ ok: true;
60
+ initialized: false;
61
+ action: 'noop';
62
+ reason: 'not-initialized';
63
+ } | {
64
+ ok: true;
65
+ initialized: true;
66
+ action: 'noop';
67
+ reason: 'locked';
68
+ } | {
69
+ ok: true;
70
+ initialized: true;
71
+ action: 'noop';
72
+ reason: 'no-pending';
73
+ } | {
74
+ ok: true;
75
+ initialized: true;
76
+ action: 'digested';
77
+ count: number;
78
+ skipped: number;
79
+ errors: number;
80
+ } | {
81
+ ok: false;
82
+ initialized: true;
83
+ action: 'error';
84
+ error: string;
85
+ };
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,3 @@
1
+ export declare function parseDigestWorkerSuccess(stdout: string): boolean;
2
+ export declare function isDigestWorkerInvocation(args: string[]): boolean;
3
+ export declare function runDigestFileInWorker(args: string[]): number | undefined;
@@ -0,0 +1,51 @@
1
+ import { spawnSync } from 'node:child_process';
2
+ import { parseArgs, numberFlag } from '../../utils/args.js';
3
+ import { DEFAULT_DIGEST_TIMEOUT_MS, DIGEST_TIMEOUT_GRACE_MS, DIGEST_WORKER_FLAG, DIGEST_WORKER_RESULT_PREFIX, } from './constants.js';
4
+ function isDigestWorkerResult(value) {
5
+ if (typeof value !== 'object' || value === null || Array.isArray(value))
6
+ return false;
7
+ const record = value;
8
+ return typeof record.markdown === 'string' && record.status === 'provisional';
9
+ }
10
+ export function parseDigestWorkerSuccess(stdout) {
11
+ const line = stdout
12
+ .split(/\r?\n/)
13
+ .reverse()
14
+ .find((candidate) => candidate.startsWith(DIGEST_WORKER_RESULT_PREFIX));
15
+ if (!line)
16
+ return false;
17
+ try {
18
+ return isDigestWorkerResult(JSON.parse(line.slice(DIGEST_WORKER_RESULT_PREFIX.length)));
19
+ }
20
+ catch {
21
+ return false;
22
+ }
23
+ }
24
+ export function isDigestWorkerInvocation(args) {
25
+ return parseArgs(args).flags.get(DIGEST_WORKER_FLAG) === true;
26
+ }
27
+ export function runDigestFileInWorker(args) {
28
+ const parsed = parseArgs(args);
29
+ if (parsed.flags.get(DIGEST_WORKER_FLAG) === true)
30
+ return undefined;
31
+ const entrypoint = process.argv[1];
32
+ if (!entrypoint)
33
+ return undefined;
34
+ const timeoutMs = numberFlag(parsed.flags, 'timeout-ms', DEFAULT_DIGEST_TIMEOUT_MS);
35
+ const result = spawnSync(process.execPath, [entrypoint, 'memory', 'digest-file', ...args, `--${DIGEST_WORKER_FLAG}`], {
36
+ cwd: process.cwd(),
37
+ encoding: 'utf8',
38
+ maxBuffer: 1024 * 1024 * 20,
39
+ timeout: timeoutMs + DIGEST_TIMEOUT_GRACE_MS,
40
+ });
41
+ if (result.stdout)
42
+ process.stdout.write(result.stdout);
43
+ const workerSucceeded = parseDigestWorkerSuccess(result.stdout);
44
+ if (result.stderr && !workerSucceeded)
45
+ process.stderr.write(result.stderr);
46
+ if (result.status === 0 || workerSucceeded)
47
+ return 0;
48
+ if (result.status !== null)
49
+ return result.status;
50
+ return 1;
51
+ }
@@ -10,5 +10,6 @@ export declare class Embedder {
10
10
  embed(texts: string[]): Promise<Float32Array[]>;
11
11
  get dimension(): number | undefined;
12
12
  isReady(): boolean;
13
+ initialize(): Promise<void>;
13
14
  private loadModel;
14
15
  }
@@ -40,6 +40,11 @@ export class Embedder {
40
40
  isReady() {
41
41
  return this.pipeline !== null;
42
42
  }
43
+ async initialize() {
44
+ if (!this.pipeline) {
45
+ await this.loadModel();
46
+ }
47
+ }
43
48
  async loadModel() {
44
49
  try {
45
50
  const { EmbeddingModel, FlagEmbedding } = await import('fastembed');
@@ -49,7 +54,11 @@ export class Embedder {
49
54
  'Xenova/all-MiniLM-L6-v2': EmbeddingModel.AllMiniLML6V2,
50
55
  };
51
56
  const model = modelMap[this.modelName] ?? EmbeddingModel.BGESmallENV15;
52
- const embedder = await FlagEmbedding.init({ model: model, cacheDir: FASTEMBED_CACHE_DIR, showDownloadProgress: false });
57
+ const embedder = await FlagEmbedding.init({
58
+ model: model,
59
+ cacheDir: FASTEMBED_CACHE_DIR,
60
+ showDownloadProgress: false,
61
+ });
53
62
  this.pipeline = async (texts) => {
54
63
  const result = [];
55
64
  for await (const batch of embedder.embed(texts, texts.length)) {
@@ -2,4 +2,4 @@ export { MemoryStore, StoreError } from './store.js';
2
2
  export { Embedder, EmbedderError } from './embedder.js';
3
3
  export { MemoryIndexer } from './indexer.js';
4
4
  export type { MemoryChunk, SearchResult, MemoryConfig, IndexStats } from './types.js';
5
- export { DEFAULT_MEMORY_CONFIG } from './types.js';
5
+ export { DEFAULT_MEMORY_CONFIG } from './constants.js';
@@ -1,4 +1,4 @@
1
1
  export { MemoryStore, StoreError } from './store.js';
2
2
  export { Embedder, EmbedderError } from './embedder.js';
3
3
  export { MemoryIndexer } from './indexer.js';
4
- export { DEFAULT_MEMORY_CONFIG } from './types.js';
4
+ export { DEFAULT_MEMORY_CONFIG } from './constants.js';
@@ -1,12 +1,15 @@
1
- import type { Embedder } from './embedder.js';
2
1
  import type { MemoryStore } from './store.js';
3
2
  import type { IndexStats, MemoryConfig, SearchResult } from './types.js';
3
+ export interface TextEmbedder {
4
+ embed(texts: string[]): Promise<Float32Array[]>;
5
+ initialize(): Promise<void>;
6
+ }
4
7
  export declare class MemoryIndexer {
5
8
  private readonly store;
6
9
  private readonly embedder;
7
10
  private readonly config;
8
11
  private _ready;
9
- constructor(store: MemoryStore, embedder: Embedder, config: MemoryConfig);
12
+ constructor(store: MemoryStore, embedder: TextEmbedder, config: MemoryConfig);
10
13
  indexFile(absolutePath: string): Promise<IndexStats>;
11
14
  private indexFileRelativeTo;
12
15
  indexDirectory(rootDir: string, opts?: {
@@ -1,29 +1,8 @@
1
1
  import * as fs from 'fs';
2
2
  import * as path from 'path';
3
3
  import { chunkMarkdown } from './chunker.js';
4
- const LOCK_RETRY_MS = 50;
5
- const LOCK_TIMEOUT_MS = 500;
6
- async function acquireLock(lockPath) {
7
- const deadline = Date.now() + LOCK_TIMEOUT_MS;
8
- while (Date.now() < deadline) {
9
- try {
10
- fs.writeFileSync(lockPath, String(process.pid), { flag: 'wx' });
11
- return true;
12
- }
13
- catch {
14
- await new Promise((r) => setTimeout(r, LOCK_RETRY_MS));
15
- }
16
- }
17
- return false;
18
- }
19
- function releaseLock(lockPath) {
20
- try {
21
- fs.unlinkSync(lockPath);
22
- }
23
- catch {
24
- // ignore
25
- }
26
- }
4
+ import { LOCK_RETRY_MS, LOCK_TIMEOUT_MS, RRF_K } from './constants.js';
5
+ import { acquireLock, releaseLock } from '../utils/files.js';
27
6
  export class MemoryIndexer {
28
7
  store;
29
8
  embedder;
@@ -39,19 +18,19 @@ export class MemoryIndexer {
39
18
  }
40
19
  async indexFileRelativeTo(absolutePath, sourceRoot) {
41
20
  const source = path.relative(sourceRoot, absolutePath);
42
- const existingHashes = this.store.hashesBySource(source);
21
+ const existingChunkIds = this.store.hashesBySource(source);
43
22
  let text;
44
23
  try {
45
24
  text = fs.readFileSync(absolutePath, 'utf8');
46
25
  }
47
26
  catch (err) {
48
27
  console.warn(`[memory-indexer] Cannot read file ${absolutePath}:`, err);
49
- return { indexed: 0, deleted: 0, skipped: existingHashes.size };
28
+ return { indexed: 0, deleted: 0, skipped: existingChunkIds.size };
50
29
  }
51
30
  const allChunks = chunkMarkdown(text, source, this.config);
52
- const newHashes = new Set(allChunks.map((c) => c.id));
53
- const toIndex = allChunks.filter((c) => !existingHashes.has(c.id));
54
- const toDelete = [...existingHashes].filter((h) => !newHashes.has(h));
31
+ const currentChunkIds = new Set(allChunks.map((chunk) => chunk.id));
32
+ const toIndex = allChunks.filter((chunk) => !existingChunkIds.has(chunk.id));
33
+ const toDelete = [...existingChunkIds].filter((id) => !currentChunkIds.has(id));
55
34
  let embeddings = [];
56
35
  if (toIndex.length > 0) {
57
36
  try {
@@ -59,7 +38,7 @@ export class MemoryIndexer {
59
38
  }
60
39
  catch (err) {
61
40
  console.warn(`[memory-indexer] Embedding failed for ${absolutePath}:`, err);
62
- return { indexed: 0, deleted: 0, skipped: existingHashes.size };
41
+ return { indexed: 0, deleted: 0, skipped: existingChunkIds.size };
63
42
  }
64
43
  }
65
44
  if (toIndex.length > 0) {
@@ -135,24 +114,22 @@ export class MemoryIndexer {
135
114
  if (bm25Ids.size > 0) {
136
115
  denseResults = denseResults.filter((result) => bm25Ids.has(result.id));
137
116
  }
138
- // RRF fusion (k=60)
139
- const k = 60;
140
117
  const scoreMap = new Map();
141
118
  denseResults.forEach((r, rank) => {
142
- scoreMap.set(r.id, { dense: 1 / (k + rank + 1), bm25: 0 });
119
+ scoreMap.set(r.id, { dense: 1 / (RRF_K + rank + 1), bm25: 0 });
143
120
  });
144
121
  bm25Results.forEach((r, rank) => {
145
122
  const existing = scoreMap.get(r.id);
146
123
  if (existing) {
147
- existing.bm25 = 1 / (k + rank + 1);
124
+ existing.bm25 = 1 / (RRF_K + rank + 1);
148
125
  }
149
126
  else {
150
- scoreMap.set(r.id, { dense: 0, bm25: 1 / (k + rank + 1) });
127
+ scoreMap.set(r.id, { dense: 0, bm25: 1 / (RRF_K + rank + 1) });
151
128
  }
152
129
  });
153
130
  const hasDense = this.store.vecAvailable && denseResults.length > 0;
154
131
  const numRetrievers = hasDense ? 2 : 1;
155
- const maxScore = numRetrievers / 61;
132
+ const maxScore = numRetrievers / (RRF_K + 1);
156
133
  const ranked = [...scoreMap.entries()]
157
134
  .map(([id, scores]) => ({
158
135
  id,
@@ -193,7 +170,7 @@ export class MemoryIndexer {
193
170
  const savePath = path.join(rawDir, `conv_save_${datePart}.md`);
194
171
  const lockPath = `${savePath}.lock`;
195
172
  fs.mkdirSync(rawDir, { recursive: true });
196
- const acquired = await acquireLock(lockPath);
173
+ const acquired = await acquireLock(lockPath, LOCK_TIMEOUT_MS, LOCK_RETRY_MS);
197
174
  if (!acquired) {
198
175
  console.warn('[memory-indexer] Could not acquire write lock, writing anyway (fail-open)');
199
176
  }
@@ -202,12 +179,13 @@ export class MemoryIndexer {
202
179
  }
203
180
  finally {
204
181
  if (acquired)
205
- releaseLock(lockPath);
182
+ releaseLock(lockPath, { ignoreErrors: true });
206
183
  }
207
184
  return { indexed: 0, deleted: 0, skipped: 0 };
208
185
  }
209
186
  async startupIndex() {
210
187
  try {
188
+ await this.embedder.initialize();
211
189
  await this.indexDirectory(path.join(this.config.wikiDir, 'compiled'), {
212
190
  relativeBase: this.config.wikiDir,
213
191
  excludeFiles: ['index.md', 'log.md'],