@hanhnd/agent-kit 1.0.32 → 1.0.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands/cli-digest-pending.test.js +111 -0
- package/dist/cli/commands/cli-memory.test.js +34 -0
- package/dist/cli/{memory.js → commands/memory.js} +24 -11
- package/dist/cli/index.js +2 -2
- package/dist/{utils → cli/utils}/args.d.ts +2 -1
- package/dist/{kit-cli.js → cli.js} +342 -58
- package/dist/{tools/config.d.ts → core/config/index.d.ts} +2 -7
- package/dist/{tools/config.js → core/config/index.js} +2 -13
- package/dist/core/handoffs/index.d.ts +17 -0
- package/dist/{tools/handoffs.js → core/handoffs/index.js} +15 -7
- package/dist/core/handoffs/index.test.js +88 -0
- package/dist/{tools/security.d.ts → core/security/index.d.ts} +0 -19
- package/dist/{tools/security.js → core/security/index.js} +3 -76
- package/dist/entrypoints/cli.js +11 -0
- package/dist/entrypoints/server.js +37 -0
- package/dist/{tools → mcp}/agent.d.ts +1 -1
- package/dist/{tools → mcp}/agent.js +4 -4
- package/dist/mcp/core.d.ts +17 -0
- package/dist/mcp/core.js +60 -0
- package/dist/mcp/core.test.js +98 -0
- package/dist/{tools → mcp}/integration.js +3 -3
- package/dist/mcp/memory.d.ts +15 -0
- package/dist/{tools → mcp}/memory.js +15 -13
- package/dist/{kit-server.js → server.js} +129 -98
- package/dist/services/digest/background.d.ts +11 -0
- package/dist/services/digest/background.js +258 -0
- package/dist/{memory → services}/digest/constants.d.ts +4 -1
- package/dist/{memory → services}/digest/constants.js +6 -2
- package/dist/services/digest/digest-background.test.js +206 -0
- package/dist/{tools/__tests__ → services/digest}/digest-init.test.js +4 -3
- package/dist/{memory/__tests__ → services/digest}/digest-llama-local.test.js +1 -13
- package/dist/{memory/__tests__ → services/digest}/digest-pending.test.js +39 -11
- package/dist/{memory/__tests__ → services/digest}/digest-processor.test.js +5 -7
- package/dist/{memory → services}/digest/files.d.ts +0 -1
- package/dist/{memory → services}/digest/files.js +2 -2
- package/dist/services/digest/model-registry.d.ts +2 -0
- package/dist/{memory → services}/digest/model-registry.js +9 -8
- package/dist/services/digest/processor.d.ts +13 -0
- package/dist/{memory → services}/digest/processor.js +48 -31
- package/dist/services/digest/providers/llama-local.d.ts +2 -0
- package/dist/{memory → services}/digest/providers/llama-local.js +3 -4
- package/dist/{memory → services}/digest/types.d.ts +34 -2
- package/dist/services/digest/types.js +6 -0
- package/dist/{memory → services}/digest/worker.d.ts +0 -1
- package/dist/{memory → services}/digest/worker.js +2 -2
- package/dist/{utils → services/integration}/credentials.js +1 -1
- package/dist/{utils/parser.js → services/integration/parser/adf.js} +1 -0
- package/dist/{memory → services/memory}/chunker.d.ts +0 -2
- package/dist/{memory → services/memory}/chunker.js +3 -3
- package/dist/{memory/__tests__ → services/memory}/chunker.test.js +1 -34
- package/dist/{memory → services/memory}/constants.d.ts +1 -1
- package/dist/{memory → services/memory}/constants.js +2 -1
- package/dist/{memory → services/memory}/embedder.d.ts +2 -2
- package/dist/{memory → services/memory}/embedder.js +5 -5
- package/dist/services/memory/embedder.test.js +13 -0
- package/dist/{memory → services/memory}/indexer.d.ts +2 -1
- package/dist/{memory → services/memory}/indexer.js +1 -1
- package/dist/{memory/__tests__ → services/memory}/indexer.test.js +4 -3
- package/dist/{tools/__tests__ → services/memory}/memory.test.js +54 -13
- package/dist/{memory/__tests__ → services/memory}/store.test.js +3 -2
- package/dist/{memory → services/memory}/types.d.ts +7 -2
- package/dist/services/memory/types.js +6 -0
- package/dist/utils/files.d.ts +0 -1
- package/dist/utils/files.js +1 -1
- package/dist/{memory/__tests__/lockfile.test.js → utils/files.test.js} +14 -14
- package/dist/utils/index.d.ts +6 -0
- package/dist/utils/index.js +6 -0
- package/dist/utils/json.d.ts +0 -1
- package/dist/utils/json.js +0 -10
- package/dist/utils/paths.d.ts +0 -1
- package/dist/utils/paths.js +1 -1
- package/package.json +6 -6
- package/dist/memory/__tests__/digest-worker.test.js +0 -46
- package/dist/memory/__tests__/embedder.test.js +0 -14
- package/dist/memory/digest/model-registry.d.ts +0 -6
- package/dist/memory/digest/processor.d.ts +0 -7
- package/dist/memory/digest/providers/llama-local.d.ts +0 -7
- package/dist/tools/__tests__/cli-digest-pending.test.js +0 -51
- package/dist/tools/__tests__/cli-memory.test.js +0 -22
- package/dist/tools/core.d.ts +0 -10
- package/dist/tools/core.js +0 -46
- package/dist/tools/handoffs.d.ts +0 -22
- package/dist/tools/memory.d.ts +0 -16
- /package/dist/{tools/__tests__ → cli/commands}/cli-digest-pending.test.d.ts +0 -0
- /package/dist/{tools/__tests__ → cli/commands}/cli-memory.test.d.ts +0 -0
- /package/dist/cli/{memory.d.ts → commands/memory.d.ts} +0 -0
- /package/dist/{utils → cli/utils}/args.js +0 -0
- /package/dist/{memory/__tests__/chunker.test.d.ts → core/handoffs/index.test.d.ts} +0 -0
- /package/dist/{kit-cli.d.ts → entrypoints/cli.d.ts} +0 -0
- /package/dist/{kit-server.d.ts → entrypoints/server.d.ts} +0 -0
- /package/dist/{memory/__tests__/digest-llama-local.test.d.ts → mcp/core.test.d.ts} +0 -0
- /package/dist/{tools → mcp}/integration.d.ts +0 -0
- /package/dist/{memory/__tests__/digest-pending.test.d.ts → services/digest/digest-background.test.d.ts} +0 -0
- /package/dist/{tools/__tests__ → services/digest}/digest-init.test.d.ts +0 -0
- /package/dist/{memory/__tests__/digest-processor.test.d.ts → services/digest/digest-llama-local.test.d.ts} +0 -0
- /package/dist/{memory/__tests__/digest-worker.test.d.ts → services/digest/digest-pending.test.d.ts} +0 -0
- /package/dist/{memory/__tests__/embedder.test.d.ts → services/digest/digest-processor.test.d.ts} +0 -0
- /package/dist/{utils → services/integration}/credentials.d.ts +0 -0
- /package/dist/{utils/parser.d.ts → services/integration/parser/adf.d.ts} +0 -0
- /package/dist/{memory/__tests__/indexer.test.d.ts → services/memory/chunker.test.d.ts} +0 -0
- /package/dist/{memory/__tests__/lockfile.test.d.ts → services/memory/embedder.test.d.ts} +0 -0
- /package/dist/{memory → services/memory}/index.d.ts +0 -0
- /package/dist/{memory → services/memory}/index.js +0 -0
- /package/dist/{memory/__tests__/store.test.d.ts → services/memory/indexer.test.d.ts} +0 -0
- /package/dist/{tools/__tests__ → services/memory}/memory.test.d.ts +0 -0
- /package/dist/{memory → services/memory}/store.d.ts +0 -0
- /package/dist/{memory → services/memory}/store.js +0 -0
- /package/dist/{memory/digest/types.js → services/memory/store.test.d.ts} +0 -0
- /package/dist/{memory/types.js → utils/files.test.d.ts} +0 -0
- /package/dist/{__tests__/temp-dir.d.ts → utils/temp-dir.test.d.ts} +0 -0
- /package/dist/{__tests__/temp-dir.js → utils/temp-dir.test.js} +0 -0
|
@@ -2,10 +2,10 @@ import * as assert from 'node:assert/strict';
|
|
|
2
2
|
import * as fs from 'node:fs';
|
|
3
3
|
import * as path from 'node:path';
|
|
4
4
|
import { afterEach, describe, test } from 'node:test';
|
|
5
|
-
import { createTempDirTracker } from '../../
|
|
6
|
-
import { DEFAULT_DIGEST_MODEL_ID } from '
|
|
7
|
-
import { defaultProvisionalDigestDir,
|
|
8
|
-
import { digestConversationFile } from '
|
|
5
|
+
import { createTempDirTracker } from '../../utils/temp-dir.test.js';
|
|
6
|
+
import { DEFAULT_DIGEST_MODEL_ID } from './constants.js';
|
|
7
|
+
import { defaultProvisionalDigestDir, writeProvisionalDigestFile, readConversationDigestInput } from './files.js';
|
|
8
|
+
import { digestConversationFile } from './processor.js';
|
|
9
9
|
const tempDirs = createTempDirTracker();
|
|
10
10
|
afterEach(() => {
|
|
11
11
|
tempDirs.cleanup();
|
|
@@ -18,9 +18,7 @@ describe('digestConversationFile', () => {
|
|
|
18
18
|
fs.writeFileSync(inputPath, '**User:** remember this decision', 'utf8');
|
|
19
19
|
const input = readConversationDigestInput(workspace, inputPath);
|
|
20
20
|
const outDir = defaultProvisionalDigestDir(workspace);
|
|
21
|
-
const markdownPath =
|
|
22
|
-
fs.mkdirSync(path.dirname(markdownPath), { recursive: true });
|
|
23
|
-
fs.writeFileSync(markdownPath, '# Conversation Digest: conv\n', 'utf8');
|
|
21
|
+
const markdownPath = writeProvisionalDigestFile(outDir, input, '# Conversation Digest: conv\n');
|
|
24
22
|
const result = await digestConversationFile({
|
|
25
23
|
workspaceRoot: workspace,
|
|
26
24
|
inputPath,
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import type { ConversationDigestInput, ConversationDigestSettings } from './types.js';
|
|
2
2
|
export declare function defaultProvisionalDigestDir(workspaceRoot: string): string;
|
|
3
3
|
export declare function readConversationDigestInput(workspaceRoot: string, inputPath: string): ConversationDigestInput;
|
|
4
|
-
export declare function provisionalDigestPath(outDir: string, input: Pick<ConversationDigestInput, 'sourcePath' | 'contentHash'>): string;
|
|
5
4
|
export declare function findExistingProvisionalDigest(outDir: string, input: ConversationDigestInput): string | undefined;
|
|
6
5
|
export declare function writeProvisionalDigestFile(outDir: string, input: ConversationDigestInput, markdown: string): string;
|
|
7
6
|
export declare function writeConversationDigestSettings(workspaceRoot: string, digest: ConversationDigestSettings): void;
|
|
@@ -3,7 +3,7 @@ import * as path from 'node:path';
|
|
|
3
3
|
import { PROVISIONAL_DIGEST_DIR } from './constants.js';
|
|
4
4
|
import { atomicWriteTextFile } from '../../utils/files.js';
|
|
5
5
|
import { sha256Hex } from '../../utils/hash.js';
|
|
6
|
-
import { loadProjectSettings, writeProjectSettings } from '../../
|
|
6
|
+
import { loadProjectSettings, writeProjectSettings } from '../../core/config/index.js';
|
|
7
7
|
export function defaultProvisionalDigestDir(workspaceRoot) {
|
|
8
8
|
return path.join(workspaceRoot, PROVISIONAL_DIGEST_DIR);
|
|
9
9
|
}
|
|
@@ -16,7 +16,7 @@ export function readConversationDigestInput(workspaceRoot, inputPath) {
|
|
|
16
16
|
contentHash: sha256Hex(content),
|
|
17
17
|
};
|
|
18
18
|
}
|
|
19
|
-
|
|
19
|
+
function provisionalDigestPath(outDir, input) {
|
|
20
20
|
const safeName = path
|
|
21
21
|
.basename(input.sourcePath)
|
|
22
22
|
.replace(/\.[^.]+$/, '')
|
|
@@ -1,28 +1,29 @@
|
|
|
1
|
-
|
|
1
|
+
import { DigestModelId } from './types.js';
|
|
2
|
+
class DigestModelRegistryError extends Error {
|
|
2
3
|
constructor(message) {
|
|
3
4
|
super(message);
|
|
4
5
|
this.name = 'DigestModelRegistryError';
|
|
5
6
|
}
|
|
6
7
|
}
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
id:
|
|
8
|
+
const DIGEST_MODEL_REGISTRY = {
|
|
9
|
+
[DigestModelId.TINY]: {
|
|
10
|
+
id: DigestModelId.TINY,
|
|
10
11
|
ggufUri: 'hf:bartowski/Qwen2.5-0.5B-Instruct-GGUF/Qwen2.5-0.5B-Instruct-Q4_K_M.gguf',
|
|
11
12
|
approxSizeBytes: 400_000_000,
|
|
12
13
|
license: 'Apache-2.0',
|
|
13
14
|
sourceUrl: 'https://huggingface.co/bartowski/Qwen2.5-0.5B-Instruct-GGUF',
|
|
14
15
|
enabled: true,
|
|
15
16
|
},
|
|
16
|
-
|
|
17
|
-
id:
|
|
17
|
+
[DigestModelId.DEFAULT]: {
|
|
18
|
+
id: DigestModelId.DEFAULT,
|
|
18
19
|
ggufUri: 'hf:bartowski/Qwen2.5-1.5B-Instruct-GGUF/Qwen2.5-1.5B-Instruct-Q4_K_M.gguf',
|
|
19
20
|
approxSizeBytes: 1_000_000_000,
|
|
20
21
|
license: 'Apache-2.0',
|
|
21
22
|
sourceUrl: 'https://huggingface.co/bartowski/Qwen2.5-1.5B-Instruct-GGUF',
|
|
22
23
|
enabled: true,
|
|
23
24
|
},
|
|
24
|
-
|
|
25
|
-
id:
|
|
25
|
+
[DigestModelId.LARGE]: {
|
|
26
|
+
id: DigestModelId.LARGE,
|
|
26
27
|
ggufUri: 'hf:bartowski/Qwen2.5-3B-Instruct-GGUF/Qwen2.5-3B-Instruct-Q4_K_M.gguf',
|
|
27
28
|
approxSizeBytes: 1_900_000_000,
|
|
28
29
|
license: 'Apache-2.0',
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { DigestFileOptions, DigestPendingCandidateSummary, DigestPendingResult, ProvisionalDigestResult, ConversationDigestInitResult, InitializeConversationDigestInput } from './types.js';
|
|
2
|
+
export declare function digestConversationFile(options: DigestFileOptions): Promise<ProvisionalDigestResult>;
|
|
3
|
+
export declare function summarizePendingConversations(workspaceRoot: string): DigestPendingCandidateSummary;
|
|
4
|
+
export declare function digestPendingConversations({ workspaceRoot, digestFn, onProgress, }: {
|
|
5
|
+
workspaceRoot: string;
|
|
6
|
+
digestFn?: (opts: DigestFileOptions) => Promise<ProvisionalDigestResult>;
|
|
7
|
+
onProgress?: (progress: {
|
|
8
|
+
processed: number;
|
|
9
|
+
skipped: number;
|
|
10
|
+
errors: number;
|
|
11
|
+
}) => void | Promise<void>;
|
|
12
|
+
}): Promise<DigestPendingResult>;
|
|
13
|
+
export declare function initializeConversationDigestModel(input: InitializeConversationDigestInput): Promise<ConversationDigestInitResult>;
|
|
@@ -4,13 +4,42 @@ import { DEFAULT_DIGEST_TIMEOUT_MS, DEFAULT_DIGEST_MAX_INPUT_CHARS, DIGEST_LOCKF
|
|
|
4
4
|
import { defaultProvisionalDigestDir, readConversationDigestInput, findExistingProvisionalDigest, writeProvisionalDigestFile, writeConversationDigestSettings, } from './files.js';
|
|
5
5
|
import { getDigestModelSpec } from './model-registry.js';
|
|
6
6
|
import { createLlamaLocalDigestProvider } from './providers/llama-local.js';
|
|
7
|
-
import { loadProjectSettings, resolveConversationDigestConfig, resolveMemoryConfig } from '../../
|
|
8
|
-
import { MemoryStore } from '../store.js';
|
|
9
|
-
import { MemoryIndexer } from '../indexer.js';
|
|
7
|
+
import { loadProjectSettings, resolveConversationDigestConfig, resolveMemoryConfig } from '../../core/config/index.js';
|
|
8
|
+
import { MemoryStore } from '../memory/store.js';
|
|
9
|
+
import { MemoryIndexer } from '../memory/indexer.js';
|
|
10
10
|
import { releaseLock, tryAcquireProcessLock } from '../../utils/files.js';
|
|
11
11
|
function digestLockPath(workspaceRoot) {
|
|
12
12
|
return path.join(workspaceRoot, DIGEST_LOCKFILE_REL_PATH);
|
|
13
13
|
}
|
|
14
|
+
function discoverPendingConversationFiles(workspaceRoot) {
|
|
15
|
+
const rawDir = path.join(workspaceRoot, WIKI_RAW_DIR);
|
|
16
|
+
let files;
|
|
17
|
+
try {
|
|
18
|
+
files = fs
|
|
19
|
+
.readdirSync(rawDir)
|
|
20
|
+
.filter((f) => f.startsWith('conv_') && f.endsWith('.md'))
|
|
21
|
+
.map((f) => path.join(rawDir, f))
|
|
22
|
+
.sort((a, b) => fs.statSync(a).mtimeMs - fs.statSync(b).mtimeMs);
|
|
23
|
+
}
|
|
24
|
+
catch (err) {
|
|
25
|
+
if (err.code === 'ENOENT') {
|
|
26
|
+
files = [];
|
|
27
|
+
}
|
|
28
|
+
else {
|
|
29
|
+
throw err;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
const outDir = defaultProvisionalDigestDir(workspaceRoot);
|
|
33
|
+
return files.filter((f) => {
|
|
34
|
+
try {
|
|
35
|
+
const input = readConversationDigestInput(workspaceRoot, f);
|
|
36
|
+
return !findExistingProvisionalDigest(outDir, input);
|
|
37
|
+
}
|
|
38
|
+
catch {
|
|
39
|
+
return false;
|
|
40
|
+
}
|
|
41
|
+
});
|
|
42
|
+
}
|
|
14
43
|
async function indexProvisionalDigestFile(workspaceRoot, markdownPath) {
|
|
15
44
|
try {
|
|
16
45
|
const settings = loadProjectSettings(workspaceRoot);
|
|
@@ -63,7 +92,18 @@ export async function digestConversationFile(options) {
|
|
|
63
92
|
await provider.dispose?.();
|
|
64
93
|
}
|
|
65
94
|
}
|
|
66
|
-
export
|
|
95
|
+
export function summarizePendingConversations(workspaceRoot) {
|
|
96
|
+
const digestConfig = resolveConversationDigestConfig(loadProjectSettings(workspaceRoot));
|
|
97
|
+
if (!digestConfig || digestConfig.enabled === false) {
|
|
98
|
+
return { initialized: false, pending: 0, reason: 'not-initialized' };
|
|
99
|
+
}
|
|
100
|
+
const candidates = discoverPendingConversationFiles(workspaceRoot);
|
|
101
|
+
if (candidates.length === 0) {
|
|
102
|
+
return { initialized: true, pending: 0, reason: 'no-pending' };
|
|
103
|
+
}
|
|
104
|
+
return { initialized: true, pending: candidates.length };
|
|
105
|
+
}
|
|
106
|
+
export async function digestPendingConversations({ workspaceRoot, digestFn = digestConversationFile, onProgress, }) {
|
|
67
107
|
const digestConfig = resolveConversationDigestConfig(loadProjectSettings(workspaceRoot));
|
|
68
108
|
if (!digestConfig || digestConfig.enabled === false) {
|
|
69
109
|
return { ok: true, initialized: false, action: 'noop', reason: 'not-initialized' };
|
|
@@ -73,33 +113,7 @@ export async function digestPendingConversations({ workspaceRoot, digestFn = dig
|
|
|
73
113
|
return { ok: true, initialized: true, action: 'noop', reason: 'locked' };
|
|
74
114
|
}
|
|
75
115
|
try {
|
|
76
|
-
const
|
|
77
|
-
let files;
|
|
78
|
-
try {
|
|
79
|
-
files = fs
|
|
80
|
-
.readdirSync(rawDir)
|
|
81
|
-
.filter((f) => f.startsWith('conv_') && f.endsWith('.md'))
|
|
82
|
-
.map((f) => path.join(rawDir, f))
|
|
83
|
-
.sort((a, b) => fs.statSync(a).mtimeMs - fs.statSync(b).mtimeMs);
|
|
84
|
-
}
|
|
85
|
-
catch (err) {
|
|
86
|
-
if (err.code === 'ENOENT') {
|
|
87
|
-
files = [];
|
|
88
|
-
}
|
|
89
|
-
else {
|
|
90
|
-
throw err;
|
|
91
|
-
}
|
|
92
|
-
}
|
|
93
|
-
const outDir = defaultProvisionalDigestDir(workspaceRoot);
|
|
94
|
-
const candidates = files.filter((f) => {
|
|
95
|
-
try {
|
|
96
|
-
const input = readConversationDigestInput(workspaceRoot, f);
|
|
97
|
-
return !findExistingProvisionalDigest(outDir, input);
|
|
98
|
-
}
|
|
99
|
-
catch {
|
|
100
|
-
return false;
|
|
101
|
-
}
|
|
102
|
-
});
|
|
116
|
+
const candidates = discoverPendingConversationFiles(workspaceRoot);
|
|
103
117
|
if (candidates.length === 0) {
|
|
104
118
|
return { ok: true, initialized: true, action: 'noop', reason: 'no-pending' };
|
|
105
119
|
}
|
|
@@ -123,6 +137,9 @@ export async function digestPendingConversations({ workspaceRoot, digestFn = dig
|
|
|
123
137
|
catch {
|
|
124
138
|
errors++;
|
|
125
139
|
}
|
|
140
|
+
if (onProgress) {
|
|
141
|
+
await onProgress({ processed: count, skipped, errors });
|
|
142
|
+
}
|
|
126
143
|
}
|
|
127
144
|
return { ok: true, initialized: true, action: 'digested', count, skipped, errors };
|
|
128
145
|
}
|
|
@@ -5,7 +5,7 @@ import { LLAMA_CONTEXT_SIZE, LLAMA_MAX_GENERATED_TOKENS, LLAMA_TEMPERATURE } fro
|
|
|
5
5
|
import { getDigestModelSpec } from '../model-registry.js';
|
|
6
6
|
import { withTimeout } from '../../../utils/async.js';
|
|
7
7
|
import { MODEL_CACHE_DIR } from '../../../utils/paths.js';
|
|
8
|
-
|
|
8
|
+
class LlamaLocalDigestProviderError extends Error {
|
|
9
9
|
constructor(message) {
|
|
10
10
|
super(message);
|
|
11
11
|
this.name = 'LlamaLocalDigestProviderError';
|
|
@@ -17,7 +17,7 @@ function titleFromSource(sourcePath) {
|
|
|
17
17
|
.replace(/\.[^.]+$/, '')
|
|
18
18
|
.replace(/[_-]+/g, ' ');
|
|
19
19
|
}
|
|
20
|
-
|
|
20
|
+
function trimConversationExport(content, maxInputChars) {
|
|
21
21
|
const trimmed = content.trim();
|
|
22
22
|
if (trimmed.length <= maxInputChars)
|
|
23
23
|
return trimmed;
|
|
@@ -28,7 +28,6 @@ export function trimConversationExport(content, maxInputChars) {
|
|
|
28
28
|
}
|
|
29
29
|
function buildPrompt(input, maxInputChars) {
|
|
30
30
|
const conversationExport = trimConversationExport(input.content, maxInputChars);
|
|
31
|
-
const sourceName = path.basename(input.sourcePath);
|
|
32
31
|
return [
|
|
33
32
|
{
|
|
34
33
|
role: 'system',
|
|
@@ -75,7 +74,7 @@ function stripCodeFence(text) {
|
|
|
75
74
|
const match = trimmed.match(/^```(?:markdown|md)?\s*\n([\s\S]*?)\n```$/i);
|
|
76
75
|
return match ? match[1].trim() : trimmed;
|
|
77
76
|
}
|
|
78
|
-
|
|
77
|
+
function sanitizeConversationDigestMarkdown(generatedText) {
|
|
79
78
|
return stripCodeFence(generatedText).trim() + '\n';
|
|
80
79
|
}
|
|
81
80
|
export async function createLlamaLocalDigestProvider(modelId) {
|
|
@@ -1,3 +1,8 @@
|
|
|
1
|
+
export declare enum DigestModelId {
|
|
2
|
+
TINY = "tiny",
|
|
3
|
+
DEFAULT = "default",
|
|
4
|
+
LARGE = "large"
|
|
5
|
+
}
|
|
1
6
|
export interface ConversationDigestSettings {
|
|
2
7
|
enabled?: boolean;
|
|
3
8
|
initialized: boolean;
|
|
@@ -5,7 +10,7 @@ export interface ConversationDigestSettings {
|
|
|
5
10
|
initializedAt: string;
|
|
6
11
|
}
|
|
7
12
|
export interface DigestModelSpec {
|
|
8
|
-
id:
|
|
13
|
+
id: DigestModelId;
|
|
9
14
|
ggufUri: string;
|
|
10
15
|
approxSizeBytes: number;
|
|
11
16
|
license: string;
|
|
@@ -17,7 +22,7 @@ export interface ConversationDigestInput {
|
|
|
17
22
|
content: string;
|
|
18
23
|
contentHash: string;
|
|
19
24
|
}
|
|
20
|
-
|
|
25
|
+
interface ConversationDigestOptions {
|
|
21
26
|
modelId: string;
|
|
22
27
|
maxInputChars: number;
|
|
23
28
|
timeoutMs: number;
|
|
@@ -83,3 +88,30 @@ export type DigestPendingResult = {
|
|
|
83
88
|
action: 'error';
|
|
84
89
|
error: string;
|
|
85
90
|
};
|
|
91
|
+
export type DigestWorkerState = 'running' | 'complete' | 'failed' | 'not-initialized' | 'no-pending' | 'locked' | 'stale';
|
|
92
|
+
export interface DigestWorkerStatus {
|
|
93
|
+
state: DigestWorkerState;
|
|
94
|
+
pid: number | null;
|
|
95
|
+
startedAt: string;
|
|
96
|
+
updatedAt: string;
|
|
97
|
+
pendingAtStart: number;
|
|
98
|
+
processed: number;
|
|
99
|
+
skipped: number;
|
|
100
|
+
errors: number;
|
|
101
|
+
lastError?: string;
|
|
102
|
+
}
|
|
103
|
+
export interface DigestPendingCandidateSummary {
|
|
104
|
+
initialized: boolean;
|
|
105
|
+
pending: number;
|
|
106
|
+
reason?: 'not-initialized' | 'no-pending';
|
|
107
|
+
}
|
|
108
|
+
export interface DigestPendingLauncherResult {
|
|
109
|
+
ok: boolean;
|
|
110
|
+
mode: 'background-launcher';
|
|
111
|
+
status: DigestWorkerStatus;
|
|
112
|
+
spawned: boolean;
|
|
113
|
+
pid?: number;
|
|
114
|
+
reason?: string;
|
|
115
|
+
error?: string;
|
|
116
|
+
}
|
|
117
|
+
export {};
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { spawnSync } from 'node:child_process';
|
|
2
|
-
import { parseArgs, numberFlag } from '../../utils/args.js';
|
|
2
|
+
import { parseArgs, numberFlag } from '../../cli/utils/args.js';
|
|
3
3
|
import { DEFAULT_DIGEST_TIMEOUT_MS, DIGEST_TIMEOUT_GRACE_MS, DIGEST_WORKER_FLAG, DIGEST_WORKER_RESULT_PREFIX, } from './constants.js';
|
|
4
4
|
function isDigestWorkerResult(value) {
|
|
5
5
|
if (typeof value !== 'object' || value === null || Array.isArray(value))
|
|
@@ -7,7 +7,7 @@ function isDigestWorkerResult(value) {
|
|
|
7
7
|
const record = value;
|
|
8
8
|
return typeof record.markdown === 'string' && record.status === 'provisional';
|
|
9
9
|
}
|
|
10
|
-
|
|
10
|
+
function parseDigestWorkerSuccess(stdout) {
|
|
11
11
|
const line = stdout
|
|
12
12
|
.split(/\r?\n/)
|
|
13
13
|
.reverse()
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
* Active profile: KIT_PROFILE env var (defaults to "default")
|
|
15
15
|
*/
|
|
16
16
|
import * as fs from 'fs';
|
|
17
|
-
import { CREDENTIALS_FILE } from '
|
|
17
|
+
import { CREDENTIALS_FILE } from '../../utils/paths.js';
|
|
18
18
|
let cachedCredentials = null;
|
|
19
19
|
/**
|
|
20
20
|
* Parse a simple INI file into a map of profiles.
|
|
@@ -1,4 +1,2 @@
|
|
|
1
1
|
import type { MemoryChunk, MemoryConfig } from './types.js';
|
|
2
|
-
export declare function computeChunkId(content: string): string;
|
|
3
|
-
export declare function cleanContentForEmbedding(text: string): string;
|
|
4
2
|
export declare function chunkMarkdown(text: string, source: string, config: Pick<MemoryConfig, 'chunkSize' | 'overlapLines'>): MemoryChunk[];
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { sha256Hex } from '
|
|
2
|
-
|
|
1
|
+
import { sha256Hex } from '../../utils/hash.js';
|
|
2
|
+
function computeChunkId(content) {
|
|
3
3
|
return sha256Hex(content).slice(0, 16);
|
|
4
4
|
}
|
|
5
|
-
|
|
5
|
+
function cleanContentForEmbedding(text) {
|
|
6
6
|
return text.replace(/<!--[\s\S]*?-->/g, '');
|
|
7
7
|
}
|
|
8
8
|
const HEADING_RE = /^(#{1,6}) (.+)$/;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import * as assert from 'node:assert/strict';
|
|
2
2
|
import { describe, test } from 'node:test';
|
|
3
|
-
import { chunkMarkdown
|
|
3
|
+
import { chunkMarkdown } from './chunker.js';
|
|
4
4
|
const CFG = { chunkSize: 200, overlapLines: 2 };
|
|
5
5
|
const SRC = 'test.md';
|
|
6
6
|
describe('chunkMarkdown', () => {
|
|
@@ -70,36 +70,3 @@ describe('chunkMarkdown', () => {
|
|
|
70
70
|
assert.ok(chunk.content.includes('Before.'), 'Text before comment must remain');
|
|
71
71
|
});
|
|
72
72
|
});
|
|
73
|
-
describe('computeChunkId', () => {
|
|
74
|
-
test('returns 16-character hex string', () => {
|
|
75
|
-
const id = computeChunkId('hello world');
|
|
76
|
-
assert.equal(id.length, 16);
|
|
77
|
-
assert.match(id, /^[0-9a-f]{16}$/);
|
|
78
|
-
});
|
|
79
|
-
test('same content always produces same id', () => {
|
|
80
|
-
assert.equal(computeChunkId('abc'), computeChunkId('abc'));
|
|
81
|
-
});
|
|
82
|
-
test('different content produces different id', () => {
|
|
83
|
-
assert.notEqual(computeChunkId('abc'), computeChunkId('xyz'));
|
|
84
|
-
});
|
|
85
|
-
});
|
|
86
|
-
describe('cleanContentForEmbedding', () => {
|
|
87
|
-
test('strips HTML comment', () => {
|
|
88
|
-
assert.equal(cleanContentForEmbedding('hello <!-- comment --> world'), 'hello world');
|
|
89
|
-
});
|
|
90
|
-
test('strips multiline HTML comment', () => {
|
|
91
|
-
const input = 'before <!-- line1\nline2 --> after';
|
|
92
|
-
const result = cleanContentForEmbedding(input);
|
|
93
|
-
assert.ok(!result.includes('line1'));
|
|
94
|
-
assert.ok(result.includes('before'));
|
|
95
|
-
assert.ok(result.includes('after'));
|
|
96
|
-
});
|
|
97
|
-
test('returns original string when no comments present', () => {
|
|
98
|
-
const text = 'plain text without comments';
|
|
99
|
-
assert.equal(cleanContentForEmbedding(text), text);
|
|
100
|
-
});
|
|
101
|
-
test('preserves fenced code block content', () => {
|
|
102
|
-
const text = '```js\nconst x = 1;\n```';
|
|
103
|
-
assert.equal(cleanContentForEmbedding(text), text);
|
|
104
|
-
});
|
|
105
|
-
});
|
|
@@ -1,10 +1,11 @@
|
|
|
1
|
+
import { EmbeddingModelName } from './types.js';
|
|
1
2
|
export const DEFAULT_MEMORY_CONFIG = {
|
|
2
3
|
enabled: false,
|
|
3
4
|
wikiDir: '',
|
|
4
5
|
topK: 5,
|
|
5
6
|
chunkSize: 1500,
|
|
6
7
|
overlapLines: 2,
|
|
7
|
-
embeddingModel:
|
|
8
|
+
embeddingModel: EmbeddingModelName.SMALL,
|
|
8
9
|
vectorDimension: 384,
|
|
9
10
|
};
|
|
10
11
|
export const LOCK_RETRY_MS = 50;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
|
|
1
|
+
import { EmbeddingModelName } from './types.js';
|
|
2
2
|
export declare class EmbedderError extends Error {
|
|
3
3
|
constructor(message: string, cause?: unknown);
|
|
4
4
|
}
|
|
@@ -6,7 +6,7 @@ export declare class Embedder {
|
|
|
6
6
|
private readonly modelName;
|
|
7
7
|
private pipeline;
|
|
8
8
|
private _dimension;
|
|
9
|
-
constructor(modelName:
|
|
9
|
+
constructor(modelName: EmbeddingModelName);
|
|
10
10
|
embed(texts: string[]): Promise<Float32Array[]>;
|
|
11
11
|
get dimension(): number | undefined;
|
|
12
12
|
isReady(): boolean;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { FASTEMBED_CACHE_DIR } from '
|
|
2
|
-
|
|
1
|
+
import { FASTEMBED_CACHE_DIR } from '../../utils/paths.js';
|
|
2
|
+
import { EmbeddingModelName } from './types.js';
|
|
3
3
|
export class EmbedderError extends Error {
|
|
4
4
|
constructor(message, cause) {
|
|
5
5
|
super(message);
|
|
@@ -49,9 +49,9 @@ export class Embedder {
|
|
|
49
49
|
try {
|
|
50
50
|
const { EmbeddingModel, FlagEmbedding } = await import('fastembed');
|
|
51
51
|
const modelMap = {
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
52
|
+
[EmbeddingModelName.TINY]: EmbeddingModel.AllMiniLML6V2,
|
|
53
|
+
[EmbeddingModelName.SMALL]: EmbeddingModel.BGESmallENV15,
|
|
54
|
+
[EmbeddingModelName.LARGE]: EmbeddingModel.BGEBaseENV15,
|
|
55
55
|
};
|
|
56
56
|
const model = modelMap[this.modelName] ?? EmbeddingModel.BGESmallENV15;
|
|
57
57
|
const embedder = await FlagEmbedding.init({
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import * as assert from 'node:assert/strict';
|
|
2
|
+
import * as os from 'node:os';
|
|
3
|
+
import * as path from 'node:path';
|
|
4
|
+
import { describe, test } from 'node:test';
|
|
5
|
+
import { CREDENTIALS_FILE, MODEL_CACHE_DIR, FASTEMBED_CACHE_DIR } from '../../utils/paths.js';
|
|
6
|
+
describe('Embedder cache paths', () => {
|
|
7
|
+
test('uses global Agent Kit home for user-level files', () => {
|
|
8
|
+
const home = path.join(os.homedir(), '.agent-kit');
|
|
9
|
+
assert.equal(CREDENTIALS_FILE, path.join(home, 'credentials'));
|
|
10
|
+
assert.equal(MODEL_CACHE_DIR, path.join(home, 'cache', 'models'));
|
|
11
|
+
assert.equal(FASTEMBED_CACHE_DIR, path.join(MODEL_CACHE_DIR, 'fastembed'));
|
|
12
|
+
});
|
|
13
|
+
});
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { MemoryStore } from './store.js';
|
|
2
2
|
import type { IndexStats, MemoryConfig, SearchResult } from './types.js';
|
|
3
|
-
|
|
3
|
+
interface TextEmbedder {
|
|
4
4
|
embed(texts: string[]): Promise<Float32Array[]>;
|
|
5
5
|
initialize(): Promise<void>;
|
|
6
6
|
}
|
|
@@ -21,3 +21,4 @@ export declare class MemoryIndexer {
|
|
|
21
21
|
startupIndex(): Promise<void>;
|
|
22
22
|
get ready(): boolean;
|
|
23
23
|
}
|
|
24
|
+
export {};
|
|
@@ -2,7 +2,7 @@ import * as fs from 'fs';
|
|
|
2
2
|
import * as path from 'path';
|
|
3
3
|
import { chunkMarkdown } from './chunker.js';
|
|
4
4
|
import { LOCK_RETRY_MS, LOCK_TIMEOUT_MS, RRF_K } from './constants.js';
|
|
5
|
-
import { acquireLock, releaseLock } from '
|
|
5
|
+
import { acquireLock, releaseLock } from '../../utils/files.js';
|
|
6
6
|
export class MemoryIndexer {
|
|
7
7
|
store;
|
|
8
8
|
embedder;
|
|
@@ -5,8 +5,9 @@ import { syncBuiltinESMExports } from 'node:module';
|
|
|
5
5
|
import * as os from 'node:os';
|
|
6
6
|
import * as path from 'node:path';
|
|
7
7
|
import { after, before, describe, test } from 'node:test';
|
|
8
|
-
import { MemoryIndexer } from '
|
|
9
|
-
import { MemoryStore } from '
|
|
8
|
+
import { MemoryIndexer } from './indexer.js';
|
|
9
|
+
import { MemoryStore } from './store.js';
|
|
10
|
+
import { EmbeddingModelName } from './types.js';
|
|
10
11
|
class StubEmbedder {
|
|
11
12
|
async embed(texts) {
|
|
12
13
|
return texts.map(() => new Float32Array(384).fill(0.05));
|
|
@@ -22,7 +23,7 @@ function makeConfig(wikiDir) {
|
|
|
22
23
|
topK: 5,
|
|
23
24
|
chunkSize: 1500,
|
|
24
25
|
overlapLines: 2,
|
|
25
|
-
embeddingModel:
|
|
26
|
+
embeddingModel: EmbeddingModelName.SMALL,
|
|
26
27
|
vectorDimension: 384,
|
|
27
28
|
};
|
|
28
29
|
}
|