byterover-cli 3.0.0 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/core/domain/tools/constants.d.ts +1 -0
- package/dist/agent/core/domain/tools/constants.js +1 -0
- package/dist/agent/core/interfaces/cipher-services.d.ts +8 -0
- package/dist/agent/core/interfaces/i-cipher-agent.d.ts +1 -0
- package/dist/agent/infra/agent/agent-error-codes.d.ts +0 -1
- package/dist/agent/infra/agent/agent-error-codes.js +0 -1
- package/dist/agent/infra/agent/agent-error.d.ts +0 -1
- package/dist/agent/infra/agent/agent-error.js +0 -1
- package/dist/agent/infra/agent/agent-state-manager.d.ts +1 -3
- package/dist/agent/infra/agent/agent-state-manager.js +1 -3
- package/dist/agent/infra/agent/base-agent.d.ts +1 -1
- package/dist/agent/infra/agent/base-agent.js +1 -1
- package/dist/agent/infra/agent/cipher-agent.d.ts +15 -1
- package/dist/agent/infra/agent/cipher-agent.js +188 -3
- package/dist/agent/infra/agent/index.d.ts +1 -1
- package/dist/agent/infra/agent/index.js +1 -1
- package/dist/agent/infra/agent/service-initializer.d.ts +3 -3
- package/dist/agent/infra/agent/service-initializer.js +14 -8
- package/dist/agent/infra/agent/types.d.ts +0 -1
- package/dist/agent/infra/file-system/file-system-service.js +6 -5
- package/dist/agent/infra/folder-pack/folder-pack-service.d.ts +1 -0
- package/dist/agent/infra/folder-pack/folder-pack-service.js +29 -15
- package/dist/agent/infra/llm/providers/openai.js +12 -0
- package/dist/agent/infra/llm/stream-to-text.d.ts +7 -0
- package/dist/agent/infra/llm/stream-to-text.js +14 -0
- package/dist/agent/infra/map/abstract-generator.d.ts +22 -0
- package/dist/agent/infra/map/abstract-generator.js +67 -0
- package/dist/agent/infra/map/abstract-queue.d.ts +67 -0
- package/dist/agent/infra/map/abstract-queue.js +218 -0
- package/dist/agent/infra/memory/memory-deduplicator.d.ts +44 -0
- package/dist/agent/infra/memory/memory-deduplicator.js +88 -0
- package/dist/agent/infra/memory/memory-manager.d.ts +1 -0
- package/dist/agent/infra/memory/memory-manager.js +6 -5
- package/dist/agent/infra/sandbox/curate-service.d.ts +4 -2
- package/dist/agent/infra/sandbox/curate-service.js +6 -7
- package/dist/agent/infra/sandbox/local-sandbox.d.ts +5 -0
- package/dist/agent/infra/sandbox/local-sandbox.js +57 -1
- package/dist/agent/infra/sandbox/tools-sdk.d.ts +3 -1
- package/dist/agent/infra/session/session-compressor.d.ts +43 -0
- package/dist/agent/infra/session/session-compressor.js +296 -0
- package/dist/agent/infra/session/session-manager.d.ts +7 -0
- package/dist/agent/infra/session/session-manager.js +9 -0
- package/dist/agent/infra/tools/implementations/curate-tool.d.ts +3 -2
- package/dist/agent/infra/tools/implementations/curate-tool.js +54 -27
- package/dist/agent/infra/tools/implementations/expand-knowledge-tool.d.ts +3 -3
- package/dist/agent/infra/tools/implementations/expand-knowledge-tool.js +34 -7
- package/dist/agent/infra/tools/implementations/ingest-resource-tool.d.ts +17 -0
- package/dist/agent/infra/tools/implementations/ingest-resource-tool.js +224 -0
- package/dist/agent/infra/tools/implementations/memory-symbol-tree.d.ts +8 -0
- package/dist/agent/infra/tools/implementations/search-knowledge-service.d.ts +1 -1
- package/dist/agent/infra/tools/implementations/search-knowledge-service.js +207 -34
- package/dist/agent/infra/tools/implementations/search-knowledge-tool.js +2 -2
- package/dist/agent/infra/tools/tool-provider.js +1 -0
- package/dist/agent/infra/tools/tool-registry.d.ts +3 -0
- package/dist/agent/infra/tools/tool-registry.js +15 -4
- package/dist/server/constants.d.ts +2 -0
- package/dist/server/constants.js +2 -0
- package/dist/server/core/domain/knowledge/memory-scoring.d.ts +3 -3
- package/dist/server/core/domain/knowledge/memory-scoring.js +5 -5
- package/dist/server/core/domain/knowledge/summary-types.d.ts +4 -0
- package/dist/server/core/domain/transport/schemas.d.ts +10 -10
- package/dist/server/infra/context-tree/derived-artifact.js +5 -1
- package/dist/server/infra/context-tree/file-context-tree-manifest-service.d.ts +2 -1
- package/dist/server/infra/context-tree/file-context-tree-manifest-service.js +43 -7
- package/dist/server/infra/context-tree/file-context-tree-summary-service.js +20 -2
- package/dist/server/infra/executor/curate-executor.js +2 -1
- package/dist/server/infra/executor/folder-pack-executor.js +72 -2
- package/dist/server/infra/executor/query-executor.js +11 -3
- package/dist/server/infra/transport/handlers/status-handler.js +10 -0
- package/dist/server/utils/curate-result-parser.d.ts +4 -4
- package/dist/shared/transport/types/dto.d.ts +7 -0
- package/oclif.manifest.json +1 -1
- package/package.json +10 -4
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import { streamToText } from '../llm/stream-to-text.js';
|
|
3
|
+
const SYSTEM_PROMPT = `You are a memory deduplication assistant. Given a new draft memory and a list of existing memories, decide one of:
|
|
4
|
+
- CREATE: the draft is new and should be stored as-is
|
|
5
|
+
- MERGE: the draft overlaps with an existing memory; provide merged content
|
|
6
|
+
- SKIP: the draft is already covered by an existing memory
|
|
7
|
+
|
|
8
|
+
Respond with ONLY a JSON object:
|
|
9
|
+
{"action": "CREATE"}
|
|
10
|
+
{"action": "MERGE", "targetId": "<id>", "mergedContent": "<combined content>"}
|
|
11
|
+
{"action": "SKIP"}`;
|
|
12
|
+
const DEDUPLICATION_CONCURRENCY = 4;
|
|
13
|
+
/**
|
|
14
|
+
* LLM-based deduplicator for agent-extracted memories.
|
|
15
|
+
*
|
|
16
|
+
* For each draft, checks against existing memories via an LLM call.
|
|
17
|
+
* DECISIONS category drafts always result in CREATE (immutable log).
|
|
18
|
+
*/
|
|
19
|
+
export class MemoryDeduplicator {
|
|
20
|
+
generator;
|
|
21
|
+
constructor(generator) {
|
|
22
|
+
this.generator = generator;
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Deduplicate a list of draft memories against existing stored memories.
|
|
26
|
+
*
|
|
27
|
+
* @param drafts - Draft memories to check
|
|
28
|
+
* @param existing - Existing memories to compare against
|
|
29
|
+
* @returns Deduplication action for each draft
|
|
30
|
+
*/
|
|
31
|
+
async deduplicate(drafts, existing) {
|
|
32
|
+
if (existing.length === 0) {
|
|
33
|
+
return drafts.map((memory) => ({ action: 'CREATE', memory }));
|
|
34
|
+
}
|
|
35
|
+
const actions = Array.from({ length: drafts.length });
|
|
36
|
+
const concurrency = Math.min(DEDUPLICATION_CONCURRENCY, drafts.length);
|
|
37
|
+
const worker = async (workerIndex) => {
|
|
38
|
+
for (let draftIndex = workerIndex; draftIndex < drafts.length; draftIndex += concurrency) {
|
|
39
|
+
const draft = drafts[draftIndex];
|
|
40
|
+
if (draft.category === 'DECISIONS') {
|
|
41
|
+
actions[draftIndex] = { action: 'CREATE', memory: draft };
|
|
42
|
+
continue;
|
|
43
|
+
}
|
|
44
|
+
// eslint-disable-next-line no-await-in-loop
|
|
45
|
+
actions[draftIndex] = await this.deduplicateSingle(draft, existing);
|
|
46
|
+
}
|
|
47
|
+
};
|
|
48
|
+
await Promise.all(Array.from({ length: concurrency }, async (_, workerIndex) => worker(workerIndex)));
|
|
49
|
+
return actions;
|
|
50
|
+
}
|
|
51
|
+
async deduplicateSingle(draft, existing) {
|
|
52
|
+
const existingSummary = JSON.stringify(existing.map((m) => ({ content: m.content.slice(0, 300), id: m.id })));
|
|
53
|
+
const prompt = `## Draft Memory (category: ${draft.category})
|
|
54
|
+
${draft.content}
|
|
55
|
+
|
|
56
|
+
## Existing Memories (JSON)
|
|
57
|
+
${existingSummary}
|
|
58
|
+
|
|
59
|
+
Decide: CREATE, MERGE (with targetId and mergedContent), or SKIP.`;
|
|
60
|
+
try {
|
|
61
|
+
// Use streaming — ChatGPT OAuth Codex endpoint requires stream: true
|
|
62
|
+
const responseText = await streamToText(this.generator, {
|
|
63
|
+
config: { maxTokens: 300, temperature: 0 },
|
|
64
|
+
contents: [{ content: prompt, role: 'user' }],
|
|
65
|
+
model: 'default',
|
|
66
|
+
systemPrompt: SYSTEM_PROMPT,
|
|
67
|
+
taskId: randomUUID(),
|
|
68
|
+
});
|
|
69
|
+
// Strip markdown code fences — some providers wrap JSON in ```json ... ```
|
|
70
|
+
const jsonText = responseText.replace(/^```(?:json)?\s*\n?/i, '').replace(/\n?```\s*$/i, '').trim();
|
|
71
|
+
const parsed = JSON.parse(jsonText);
|
|
72
|
+
const targetExists = parsed.targetId ? existing.some((memory) => memory.id === parsed.targetId) : false;
|
|
73
|
+
if (parsed.action === 'MERGE' && targetExists && parsed.mergedContent && parsed.targetId) {
|
|
74
|
+
return { action: 'MERGE', memory: draft, mergedContent: parsed.mergedContent, targetId: parsed.targetId };
|
|
75
|
+
}
|
|
76
|
+
if (parsed.action === 'SKIP') {
|
|
77
|
+
return { action: 'SKIP', memory: draft };
|
|
78
|
+
}
|
|
79
|
+
return { action: 'CREATE', memory: draft };
|
|
80
|
+
}
|
|
81
|
+
catch (error) {
|
|
82
|
+
// On any error, default to CREATE (fail-open)
|
|
83
|
+
const msg = error instanceof Error ? error.message : String(error);
|
|
84
|
+
console.debug(`[MemoryDeduplicator] Failed for ${draft.category}: ${msg}`);
|
|
85
|
+
return { action: 'CREATE', memory: draft };
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
}
|
|
@@ -20,6 +20,7 @@ import type { ILogger } from '../../core/interfaces/i-logger.js';
|
|
|
20
20
|
*/
|
|
21
21
|
export declare class MemoryManager {
|
|
22
22
|
private blobStorage;
|
|
23
|
+
private static readonly MEMORY_ID_LENGTH;
|
|
23
24
|
private static readonly MEMORY_KEY_PREFIX;
|
|
24
25
|
private readonly logger;
|
|
25
26
|
constructor(blobStorage: IBlobStorage, logger?: ILogger);
|
|
@@ -105,6 +105,7 @@ const ListMemoriesOptionsSchema = z
|
|
|
105
105
|
*/
|
|
106
106
|
export class MemoryManager {
|
|
107
107
|
blobStorage;
|
|
108
|
+
static MEMORY_ID_LENGTH = 12;
|
|
108
109
|
static MEMORY_KEY_PREFIX = 'memory-';
|
|
109
110
|
logger;
|
|
110
111
|
constructor(blobStorage, logger) {
|
|
@@ -468,11 +469,11 @@ export class MemoryManager {
|
|
|
468
469
|
if (!key.startsWith(MemoryManager.MEMORY_KEY_PREFIX)) {
|
|
469
470
|
return false;
|
|
470
471
|
}
|
|
471
|
-
// Memory keys have format: memory-{id}
|
|
472
|
-
// Attachment keys
|
|
473
|
-
//
|
|
474
|
-
const
|
|
475
|
-
return
|
|
472
|
+
// Memory keys have format: memory-{id} where id is a fixed-length nanoid(12).
|
|
473
|
+
// Attachment keys append an extra suffix: memory-{id}-{suffix}.
|
|
474
|
+
// A valid memory id may itself contain '-', so counting dashes is incorrect.
|
|
475
|
+
const suffix = key.slice(MemoryManager.MEMORY_KEY_PREFIX.length);
|
|
476
|
+
return suffix.length === MemoryManager.MEMORY_ID_LENGTH;
|
|
476
477
|
}
|
|
477
478
|
/**
|
|
478
479
|
* Load all memories from blob storage
|
|
@@ -3,13 +3,15 @@
|
|
|
3
3
|
* Wraps the curate-tool logic for use in the sandbox's tools.* SDK.
|
|
4
4
|
*/
|
|
5
5
|
import type { CurateOperation, CurateOptions, CurateResult, DetectDomainsInput, DetectDomainsResult, ICurateService } from '../../core/interfaces/i-curate-service.js';
|
|
6
|
+
import type { AbstractGenerationQueue } from '../map/abstract-queue.js';
|
|
6
7
|
/**
|
|
7
8
|
* Curate service implementation.
|
|
8
9
|
* Provides curate and domain detection operations for the sandbox.
|
|
9
10
|
*/
|
|
10
11
|
export declare class CurateService implements ICurateService {
|
|
12
|
+
private readonly abstractQueue?;
|
|
11
13
|
private readonly workingDirectory;
|
|
12
|
-
constructor(workingDirectory?: string);
|
|
14
|
+
constructor(workingDirectory?: string, abstractQueue?: AbstractGenerationQueue | undefined);
|
|
13
15
|
/**
|
|
14
16
|
* Execute curate operations on knowledge topics.
|
|
15
17
|
*
|
|
@@ -33,4 +35,4 @@ export declare class CurateService implements ICurateService {
|
|
|
33
35
|
* @param workingDirectory - Working directory for resolving relative paths
|
|
34
36
|
* @returns CurateService instance
|
|
35
37
|
*/
|
|
36
|
-
export declare function createCurateService(workingDirectory?: string): ICurateService;
|
|
38
|
+
export declare function createCurateService(workingDirectory?: string, abstractQueue?: AbstractGenerationQueue): ICurateService;
|
|
@@ -72,8 +72,10 @@ function validateOperations(operations) {
|
|
|
72
72
|
* Provides curate and domain detection operations for the sandbox.
|
|
73
73
|
*/
|
|
74
74
|
export class CurateService {
|
|
75
|
+
abstractQueue;
|
|
75
76
|
workingDirectory;
|
|
76
|
-
constructor(workingDirectory) {
|
|
77
|
+
constructor(workingDirectory, abstractQueue) {
|
|
78
|
+
this.abstractQueue = abstractQueue;
|
|
77
79
|
this.workingDirectory = workingDirectory ?? process.cwd();
|
|
78
80
|
}
|
|
79
81
|
/**
|
|
@@ -104,10 +106,7 @@ export class CurateService {
|
|
|
104
106
|
};
|
|
105
107
|
}
|
|
106
108
|
// Call the underlying executeCurate function from curate-tool
|
|
107
|
-
const result = await executeCurate({
|
|
108
|
-
basePath,
|
|
109
|
-
operations,
|
|
110
|
-
});
|
|
109
|
+
const result = await executeCurate({ basePath, operations }, undefined, this.abstractQueue);
|
|
111
110
|
return result;
|
|
112
111
|
}
|
|
113
112
|
/**
|
|
@@ -146,6 +145,6 @@ export class CurateService {
|
|
|
146
145
|
* @param workingDirectory - Working directory for resolving relative paths
|
|
147
146
|
* @returns CurateService instance
|
|
148
147
|
*/
|
|
149
|
-
export function createCurateService(workingDirectory) {
|
|
150
|
-
return new CurateService(workingDirectory);
|
|
148
|
+
export function createCurateService(workingDirectory, abstractQueue) {
|
|
149
|
+
return new CurateService(workingDirectory, abstractQueue);
|
|
151
150
|
}
|
|
@@ -22,6 +22,8 @@ export declare class LocalSandbox {
|
|
|
22
22
|
/** Value set by setFinalResult() — signals early exit */
|
|
23
23
|
private finalResult?;
|
|
24
24
|
private outputBuffer;
|
|
25
|
+
/** Async tool calls started inside the sandbox, including unawaited ones. */
|
|
26
|
+
private pendingToolPromises;
|
|
25
27
|
/** Current stdout cap in chars (undefined = unlimited) */
|
|
26
28
|
private stdoutCap?;
|
|
27
29
|
/** Running count of chars written to outputBuffer in current execution */
|
|
@@ -50,9 +52,12 @@ export declare class LocalSandbox {
|
|
|
50
52
|
* @param updates - Key-value pairs to add to context
|
|
51
53
|
*/
|
|
52
54
|
updateContext(updates: Record<string, unknown>): void;
|
|
55
|
+
private awaitPendingToolPromises;
|
|
53
56
|
/**
|
|
54
57
|
* Push a line to stdout, respecting the optional character cap.
|
|
55
58
|
* When the cap is reached, appends a truncation notice and drops further output.
|
|
56
59
|
*/
|
|
57
60
|
private pushStdout;
|
|
61
|
+
private trackPromise;
|
|
62
|
+
private wrapToolObject;
|
|
58
63
|
}
|
|
@@ -87,6 +87,8 @@ export class LocalSandbox {
|
|
|
87
87
|
/** Value set by setFinalResult() — signals early exit */
|
|
88
88
|
finalResult;
|
|
89
89
|
outputBuffer = [];
|
|
90
|
+
/** Async tool calls started inside the sandbox, including unawaited ones. */
|
|
91
|
+
pendingToolPromises = new Set();
|
|
90
92
|
/** Current stdout cap in chars (undefined = unlimited) */
|
|
91
93
|
stdoutCap;
|
|
92
94
|
/** Running count of chars written to outputBuffer in current execution */
|
|
@@ -118,7 +120,7 @@ export class LocalSandbox {
|
|
|
118
120
|
};
|
|
119
121
|
// Inject Tools SDK if provided (for file system operations)
|
|
120
122
|
if (toolsSDK) {
|
|
121
|
-
sandbox.tools = toolsSDK;
|
|
123
|
+
sandbox.tools = this.wrapToolObject(toolsSDK);
|
|
122
124
|
}
|
|
123
125
|
// Inject environment context as `env` object if provided
|
|
124
126
|
if (environmentContext) {
|
|
@@ -152,6 +154,7 @@ export class LocalSandbox {
|
|
|
152
154
|
this.outputBuffer = [];
|
|
153
155
|
this.errorBuffer = [];
|
|
154
156
|
this.finalResult = undefined;
|
|
157
|
+
this.pendingToolPromises.clear();
|
|
155
158
|
this.stdoutCap = config?.maxStdoutChars;
|
|
156
159
|
this.stdoutCharsWritten = 0;
|
|
157
160
|
this.stdoutTruncated = false;
|
|
@@ -192,6 +195,8 @@ export class LocalSandbox {
|
|
|
192
195
|
}
|
|
193
196
|
}
|
|
194
197
|
}
|
|
198
|
+
const elapsedMs = performance.now() - startTime;
|
|
199
|
+
await this.awaitPendingToolPromises(Math.max(0, timeout - elapsedMs));
|
|
195
200
|
}
|
|
196
201
|
catch (error) {
|
|
197
202
|
const errorMessage = error instanceof Error ? `${error.name}: ${error.message}` : String(error);
|
|
@@ -243,6 +248,32 @@ export class LocalSandbox {
|
|
|
243
248
|
this.context[key] = value;
|
|
244
249
|
}
|
|
245
250
|
}
|
|
251
|
+
async awaitPendingToolPromises(timeoutMs) {
|
|
252
|
+
const deadline = Date.now() + timeoutMs;
|
|
253
|
+
/* eslint-disable no-await-in-loop */
|
|
254
|
+
while (this.pendingToolPromises.size > 0) {
|
|
255
|
+
const pending = [...this.pendingToolPromises];
|
|
256
|
+
const remaining = deadline - Date.now();
|
|
257
|
+
if (remaining <= 0) {
|
|
258
|
+
throw new Error('Async execution timeout');
|
|
259
|
+
}
|
|
260
|
+
let timeoutId;
|
|
261
|
+
try {
|
|
262
|
+
await Promise.race([
|
|
263
|
+
Promise.allSettled(pending),
|
|
264
|
+
new Promise((_, reject) => {
|
|
265
|
+
timeoutId = setTimeout(() => reject(new Error('Async execution timeout')), remaining);
|
|
266
|
+
}),
|
|
267
|
+
]);
|
|
268
|
+
}
|
|
269
|
+
finally {
|
|
270
|
+
if (timeoutId !== undefined) {
|
|
271
|
+
clearTimeout(timeoutId);
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
/* eslint-enable no-await-in-loop */
|
|
276
|
+
}
|
|
246
277
|
/**
|
|
247
278
|
* Push a line to stdout, respecting the optional character cap.
|
|
248
279
|
* When the cap is reached, appends a truncation notice and drops further output.
|
|
@@ -263,4 +294,29 @@ export class LocalSandbox {
|
|
|
263
294
|
this.stdoutCharsWritten += line.length + 1; // +1 for join newline
|
|
264
295
|
this.outputBuffer.push(line);
|
|
265
296
|
}
|
|
297
|
+
trackPromise(promise) {
|
|
298
|
+
const trackedPromise = promise.finally(() => {
|
|
299
|
+
this.pendingToolPromises.delete(trackedPromise);
|
|
300
|
+
});
|
|
301
|
+
this.pendingToolPromises.add(trackedPromise);
|
|
302
|
+
return trackedPromise;
|
|
303
|
+
}
|
|
304
|
+
wrapToolObject(value) {
|
|
305
|
+
if (typeof value === 'function') {
|
|
306
|
+
const original = value;
|
|
307
|
+
const wrapped = (...args) => {
|
|
308
|
+
const result = original(...args);
|
|
309
|
+
const isPromiseLike = result !== null &&
|
|
310
|
+
typeof result === 'object' &&
|
|
311
|
+
typeof result.then === 'function';
|
|
312
|
+
return isPromiseLike ? this.trackPromise(Promise.resolve(result)) : result;
|
|
313
|
+
};
|
|
314
|
+
return wrapped;
|
|
315
|
+
}
|
|
316
|
+
if (value && typeof value === 'object' && !Array.isArray(value)) {
|
|
317
|
+
const wrappedEntries = Object.entries(value).map(([key, child]) => ([key, this.wrapToolObject(child)]));
|
|
318
|
+
return Object.fromEntries(wrappedEntries);
|
|
319
|
+
}
|
|
320
|
+
return value;
|
|
321
|
+
}
|
|
266
322
|
}
|
|
@@ -74,11 +74,13 @@ export interface SearchKnowledgeResult {
|
|
|
74
74
|
/** Number of other memories that reference this one */
|
|
75
75
|
backlinkCount?: number;
|
|
76
76
|
excerpt: string;
|
|
77
|
+
/** Path to .overview.md for this entry; present when L1 overview exists */
|
|
78
|
+
overviewPath?: string;
|
|
77
79
|
path: string;
|
|
78
80
|
/** Top backlink source paths (max 3) */
|
|
79
81
|
relatedPaths?: string[];
|
|
80
82
|
score: number;
|
|
81
|
-
/** Symbol kind: 'domain' | 'topic' | 'subtopic' | 'context' | 'archive_stub' */
|
|
83
|
+
/** Symbol kind: 'domain' | 'topic' | 'subtopic' | 'context' | 'archive_stub' | 'summary' */
|
|
82
84
|
symbolKind?: string;
|
|
83
85
|
/** Resolved hierarchical path in the symbol tree */
|
|
84
86
|
symbolPath?: string;
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import type { IContentGenerator } from '../../core/interfaces/i-content-generator.js';
|
|
2
|
+
import type { InternalMessage } from '../../core/interfaces/message-types.js';
|
|
3
|
+
import type { MemoryDeduplicator } from '../memory/memory-deduplicator.js';
|
|
4
|
+
import type { MemoryManager } from '../memory/memory-manager.js';
|
|
5
|
+
/**
|
|
6
|
+
* Result of a session compression pass.
|
|
7
|
+
*/
|
|
8
|
+
export interface CompressionResult {
|
|
9
|
+
created: number;
|
|
10
|
+
merged: number;
|
|
11
|
+
skipped: number;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Extracts and persists memories from a completed task session.
|
|
15
|
+
*
|
|
16
|
+
* Flow:
|
|
17
|
+
* 1. Serialize session messages into a text digest
|
|
18
|
+
* 2. LLM call: extract 5-category draft memories
|
|
19
|
+
* 3. Load existing agent memories for deduplication
|
|
20
|
+
* 4. Apply deduplication decisions (CREATE/MERGE/SKIP)
|
|
21
|
+
*/
|
|
22
|
+
export declare class SessionCompressor {
|
|
23
|
+
private readonly deduplicator;
|
|
24
|
+
private readonly generator;
|
|
25
|
+
private readonly memoryManager;
|
|
26
|
+
constructor(deduplicator: MemoryDeduplicator, generator: IContentGenerator, memoryManager: MemoryManager);
|
|
27
|
+
/**
|
|
28
|
+
* Compress a session into persistent memories.
|
|
29
|
+
*
|
|
30
|
+
* @param messages - Session message history
|
|
31
|
+
* @param commandType - Session command type (e.g. 'curate', 'query')
|
|
32
|
+
* @param options - Compression options
|
|
33
|
+
* @param options.minMessages - Minimum message count required before compression runs
|
|
34
|
+
* @returns Summary of actions taken
|
|
35
|
+
*/
|
|
36
|
+
compress(messages: InternalMessage[], commandType: string, options?: {
|
|
37
|
+
minMessages?: number;
|
|
38
|
+
}): Promise<CompressionResult>;
|
|
39
|
+
private buildFallbackDrafts;
|
|
40
|
+
private deduplicateFallbackDrafts;
|
|
41
|
+
private extractDrafts;
|
|
42
|
+
private serializeMessages;
|
|
43
|
+
}
|
|
@@ -0,0 +1,296 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import { streamToText } from '../llm/stream-to-text.js';
|
|
3
|
+
/**
|
|
4
|
+
* Five extraction categories for ByteRover session memories.
|
|
5
|
+
*/
|
|
6
|
+
const CATEGORIES = ['DECISIONS', 'ENTITIES', 'PATTERNS', 'PREFERENCES', 'SKILLS'];
|
|
7
|
+
const SYSTEM_PROMPT = `You are a session memory extractor for ByteRover, a code intelligence tool.
|
|
8
|
+
Extract reusable memories from the conversation in exactly these 5 categories:
|
|
9
|
+
- PATTERNS: reusable code or workflow patterns discovered
|
|
10
|
+
- PREFERENCES: user style/naming/structure decisions
|
|
11
|
+
- ENTITIES: key files, modules, APIs, dependencies discovered
|
|
12
|
+
- DECISIONS: architectural choices (always extract, even if already known — immutable log)
|
|
13
|
+
- SKILLS: tool invocation recipes that worked
|
|
14
|
+
|
|
15
|
+
Return ONLY a JSON array of memory objects:
|
|
16
|
+
[{"category": "PATTERNS", "content": "...", "tags": ["optional"]}, ...]
|
|
17
|
+
|
|
18
|
+
Extract 0-3 memories per category. Skip categories with nothing new. Be concise (max 200 chars per memory).`;
|
|
19
|
+
const MAX_DIGEST_CHARS = 12_000;
|
|
20
|
+
const FALLBACK_DIGEST_PREVIEW_CHARS = 4000;
|
|
21
|
+
const MIN_BOUNDARY_RATIO = 0.6;
|
|
22
|
+
const SOURCE_PATH_PATTERN = /\b(?:src|app|lib|packages|docs|test|tests)\/[A-Za-z0-9_./-]+\b/g;
|
|
23
|
+
function truncateDigestAtBoundary(digest, maxChars = MAX_DIGEST_CHARS) {
|
|
24
|
+
if (digest.length <= maxChars) {
|
|
25
|
+
return digest;
|
|
26
|
+
}
|
|
27
|
+
const clipped = digest.slice(0, maxChars);
|
|
28
|
+
const boundary = clipped.lastIndexOf('\n\n');
|
|
29
|
+
// Prefer a natural message boundary when it falls reasonably close to the cap.
|
|
30
|
+
return boundary >= Math.floor(maxChars * MIN_BOUNDARY_RATIO) ? clipped.slice(0, boundary) : clipped;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Extracts and persists memories from a completed task session.
|
|
34
|
+
*
|
|
35
|
+
* Flow:
|
|
36
|
+
* 1. Serialize session messages into a text digest
|
|
37
|
+
* 2. LLM call: extract 5-category draft memories
|
|
38
|
+
* 3. Load existing agent memories for deduplication
|
|
39
|
+
* 4. Apply deduplication decisions (CREATE/MERGE/SKIP)
|
|
40
|
+
*/
|
|
41
|
+
export class SessionCompressor {
|
|
42
|
+
deduplicator;
|
|
43
|
+
generator;
|
|
44
|
+
memoryManager;
|
|
45
|
+
constructor(deduplicator, generator, memoryManager) {
|
|
46
|
+
this.deduplicator = deduplicator;
|
|
47
|
+
this.generator = generator;
|
|
48
|
+
this.memoryManager = memoryManager;
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Compress a session into persistent memories.
|
|
52
|
+
*
|
|
53
|
+
* @param messages - Session message history
|
|
54
|
+
* @param commandType - Session command type (e.g. 'curate', 'query')
|
|
55
|
+
* @param options - Compression options
|
|
56
|
+
* @param options.minMessages - Minimum message count required before compression runs
|
|
57
|
+
* @returns Summary of actions taken
|
|
58
|
+
*/
|
|
59
|
+
async compress(messages, commandType, options) {
|
|
60
|
+
const minMessages = options?.minMessages ?? 4;
|
|
61
|
+
const hasAssistantContent = messages.some((message) => (message.role === 'assistant' &&
|
|
62
|
+
getMessageText(message).trim().length > 0));
|
|
63
|
+
const effectiveMinMessages = commandType.startsWith('curate') && hasAssistantContent
|
|
64
|
+
? Math.min(minMessages, 1)
|
|
65
|
+
: minMessages;
|
|
66
|
+
if (messages.length < effectiveMinMessages) {
|
|
67
|
+
return { created: 0, merged: 0, skipped: 0 };
|
|
68
|
+
}
|
|
69
|
+
const digest = this.serializeMessages(messages);
|
|
70
|
+
if (!digest.trim()) {
|
|
71
|
+
return { created: 0, merged: 0, skipped: 0 };
|
|
72
|
+
}
|
|
73
|
+
// Step 1: Extract draft memories via LLM
|
|
74
|
+
const useFallbackDraftsFirst = shouldPreferFallbackDrafts(commandType);
|
|
75
|
+
let drafts = useFallbackDraftsFirst ? this.buildFallbackDrafts(digest, commandType) : await this.extractDrafts(digest, commandType);
|
|
76
|
+
let usedFallbackDrafts = useFallbackDraftsFirst && drafts.length > 0;
|
|
77
|
+
if (drafts.length === 0) {
|
|
78
|
+
drafts = this.buildFallbackDrafts(digest, commandType);
|
|
79
|
+
usedFallbackDrafts = drafts.length > 0;
|
|
80
|
+
}
|
|
81
|
+
if (drafts.length === 0) {
|
|
82
|
+
return { created: 0, merged: 0, skipped: 0 };
|
|
83
|
+
}
|
|
84
|
+
// Step 2: Load the most recently updated agent memories for deduplication.
|
|
85
|
+
// MemoryManager.list() sorts by updatedAt DESC before applying the limit.
|
|
86
|
+
const existing = await this.memoryManager.list({ limit: 60, source: 'agent' });
|
|
87
|
+
// Step 3: Deduplicate
|
|
88
|
+
const actions = usedFallbackDrafts
|
|
89
|
+
? this.deduplicateFallbackDrafts(drafts, existing)
|
|
90
|
+
: await this.deduplicator.deduplicate(drafts, existing);
|
|
91
|
+
// Step 4: Apply decisions
|
|
92
|
+
let created = 0;
|
|
93
|
+
let merged = 0;
|
|
94
|
+
let skipped = 0;
|
|
95
|
+
/* eslint-disable no-await-in-loop */
|
|
96
|
+
for (const action of actions) {
|
|
97
|
+
try {
|
|
98
|
+
if (action.action === 'CREATE') {
|
|
99
|
+
await this.memoryManager.create({
|
|
100
|
+
content: action.memory.content,
|
|
101
|
+
metadata: { category: action.memory.category, source: 'agent' },
|
|
102
|
+
tags: action.memory.tags,
|
|
103
|
+
});
|
|
104
|
+
created++;
|
|
105
|
+
}
|
|
106
|
+
else if (action.action === 'MERGE') {
|
|
107
|
+
await this.memoryManager.update(action.targetId, { content: action.mergedContent });
|
|
108
|
+
merged++;
|
|
109
|
+
}
|
|
110
|
+
else {
|
|
111
|
+
skipped++;
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
catch (error) {
|
|
115
|
+
// Fail-open: skip individual memory errors
|
|
116
|
+
const msg = error instanceof Error ? error.message : String(error);
|
|
117
|
+
console.debug(`[SessionCompressor] Failed to apply ${action.action} action: ${msg}`);
|
|
118
|
+
skipped++;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
/* eslint-enable no-await-in-loop */
|
|
122
|
+
return { created, merged, skipped };
|
|
123
|
+
}
|
|
124
|
+
buildFallbackDrafts(digest, commandType) {
|
|
125
|
+
if (!commandType.startsWith('curate')) {
|
|
126
|
+
return [];
|
|
127
|
+
}
|
|
128
|
+
const preview = digest.slice(0, FALLBACK_DIGEST_PREVIEW_CHARS);
|
|
129
|
+
const fingerprint = computeFingerprint(preview);
|
|
130
|
+
const sourcePaths = [...new Set((preview.match(SOURCE_PATH_PATTERN) ?? []).filter((path) => !path.startsWith('.brv/')))];
|
|
131
|
+
const moduleLabel = deriveModuleLabel(sourcePaths);
|
|
132
|
+
const tags = moduleLabel === 'the working module' ? undefined : [moduleLabel];
|
|
133
|
+
return [
|
|
134
|
+
{
|
|
135
|
+
category: 'DECISIONS',
|
|
136
|
+
content: `Session ${fingerprint}: curated ${moduleLabel} knowledge into the context tree.`,
|
|
137
|
+
tags,
|
|
138
|
+
},
|
|
139
|
+
{
|
|
140
|
+
category: 'DECISIONS',
|
|
141
|
+
content: `Session ${fingerprint}: preserved ${moduleLabel} findings as durable knowledge instead of chat-only context.`,
|
|
142
|
+
tags,
|
|
143
|
+
},
|
|
144
|
+
{
|
|
145
|
+
category: 'PATTERNS',
|
|
146
|
+
content: `Session ${fingerprint}: used recon -> extraction -> curate apply workflow for ${moduleLabel}.`,
|
|
147
|
+
tags,
|
|
148
|
+
},
|
|
149
|
+
{
|
|
150
|
+
category: 'PATTERNS',
|
|
151
|
+
content: `Session ${fingerprint}: separated durable notes from raw source snippets while curating ${moduleLabel}.`,
|
|
152
|
+
tags,
|
|
153
|
+
},
|
|
154
|
+
{
|
|
155
|
+
category: 'SKILLS',
|
|
156
|
+
content: `Session ${fingerprint}: start ${commandType} with tools.curation.recon, then mapExtract, then verify applied file paths.`,
|
|
157
|
+
tags,
|
|
158
|
+
},
|
|
159
|
+
{
|
|
160
|
+
category: 'ENTITIES',
|
|
161
|
+
content: `${moduleLabel} is an actively curated module surfaced during ${commandType}.`,
|
|
162
|
+
tags,
|
|
163
|
+
},
|
|
164
|
+
];
|
|
165
|
+
}
|
|
166
|
+
deduplicateFallbackDrafts(drafts, existing) {
|
|
167
|
+
const dedupCategories = ['ENTITIES', 'PATTERNS', 'SKILLS'];
|
|
168
|
+
const existingKeys = new Map();
|
|
169
|
+
for (const category of dedupCategories) {
|
|
170
|
+
existingKeys.set(category, new Set(existing
|
|
171
|
+
.filter((memory) => getMemoryCategory(memory) === category)
|
|
172
|
+
.map((memory) => normalizeForFallbackDedup(memory.content, category))));
|
|
173
|
+
}
|
|
174
|
+
return drafts.map((memory) => {
|
|
175
|
+
// DECISIONS always CREATE — temporal audit records that should accumulate
|
|
176
|
+
if (memory.category === 'DECISIONS') {
|
|
177
|
+
return { action: 'CREATE', memory };
|
|
178
|
+
}
|
|
179
|
+
const categorySet = existingKeys.get(memory.category);
|
|
180
|
+
if (!categorySet) {
|
|
181
|
+
return { action: 'CREATE', memory };
|
|
182
|
+
}
|
|
183
|
+
const key = normalizeForFallbackDedup(memory.content, memory.category);
|
|
184
|
+
if (categorySet.has(key)) {
|
|
185
|
+
return { action: 'SKIP', memory };
|
|
186
|
+
}
|
|
187
|
+
categorySet.add(key);
|
|
188
|
+
return { action: 'CREATE', memory };
|
|
189
|
+
});
|
|
190
|
+
}
|
|
191
|
+
async extractDrafts(digest, commandType) {
|
|
192
|
+
try {
|
|
193
|
+
const truncatedDigest = truncateDigestAtBoundary(digest);
|
|
194
|
+
const prompt = `## Session Type: ${commandType}
|
|
195
|
+
|
|
196
|
+
## Conversation
|
|
197
|
+
${truncatedDigest}
|
|
198
|
+
|
|
199
|
+
Extract reusable memories from this session.`;
|
|
200
|
+
// Use streaming — ChatGPT OAuth Codex endpoint requires stream: true
|
|
201
|
+
const responseText = await streamToText(this.generator, {
|
|
202
|
+
config: { maxTokens: 1000, temperature: 0 },
|
|
203
|
+
contents: [{ content: prompt, role: 'user' }],
|
|
204
|
+
model: 'default',
|
|
205
|
+
systemPrompt: SYSTEM_PROMPT,
|
|
206
|
+
taskId: randomUUID(),
|
|
207
|
+
});
|
|
208
|
+
// Strip markdown code fences — some providers wrap JSON in ```json ... ```
|
|
209
|
+
const jsonText = responseText.replace(/^```(?:json)?\s*\n?/i, '').replace(/\n?```\s*$/i, '').trim();
|
|
210
|
+
const parsed = JSON.parse(jsonText);
|
|
211
|
+
if (!Array.isArray(parsed))
|
|
212
|
+
return [];
|
|
213
|
+
return parsed
|
|
214
|
+
.filter((item) => CATEGORIES.includes(item.category) && item.content?.trim())
|
|
215
|
+
.map((item) => ({
|
|
216
|
+
category: item.category,
|
|
217
|
+
content: item.content.trim(),
|
|
218
|
+
tags: item.tags,
|
|
219
|
+
}));
|
|
220
|
+
}
|
|
221
|
+
catch (error) {
|
|
222
|
+
const msg = error instanceof Error ? error.message : String(error);
|
|
223
|
+
console.debug(`[SessionCompressor] Failed to extract drafts (${commandType}): ${msg}`);
|
|
224
|
+
return [];
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
serializeMessages(messages) {
|
|
228
|
+
const lines = [];
|
|
229
|
+
for (const msg of messages) {
|
|
230
|
+
const role = msg.role.toUpperCase();
|
|
231
|
+
const text = getMessageText(msg);
|
|
232
|
+
if (text.trim()) {
|
|
233
|
+
lines.push(`[${role}]: ${text.slice(0, 2000)}`);
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
return lines.join('\n\n');
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
function getMessageText(message) {
|
|
240
|
+
if (typeof message.content === 'string') {
|
|
241
|
+
return message.content;
|
|
242
|
+
}
|
|
243
|
+
if (!Array.isArray(message.content)) {
|
|
244
|
+
return '';
|
|
245
|
+
}
|
|
246
|
+
return message.content
|
|
247
|
+
.filter((part) => 'text' in part && typeof part.text === 'string')
|
|
248
|
+
.map((part) => part.text)
|
|
249
|
+
.join(' ');
|
|
250
|
+
}
|
|
251
|
+
function computeFingerprint(text) {
|
|
252
|
+
/* eslint-disable no-bitwise, unicorn/prefer-code-point */
|
|
253
|
+
let hash = 0;
|
|
254
|
+
for (const char of text) {
|
|
255
|
+
hash = (hash * 31 + char.charCodeAt(0)) >>> 0;
|
|
256
|
+
}
|
|
257
|
+
/* eslint-enable no-bitwise, unicorn/prefer-code-point */
|
|
258
|
+
return hash.toString(16).padStart(8, '0').slice(0, 8);
|
|
259
|
+
}
|
|
260
|
+
function deriveModuleLabel(sourcePaths) {
|
|
261
|
+
if (sourcePaths.length === 0) {
|
|
262
|
+
return 'the working module';
|
|
263
|
+
}
|
|
264
|
+
const firstPath = sourcePaths[0];
|
|
265
|
+
const segments = firstPath.split('/').filter(Boolean);
|
|
266
|
+
if (segments.length >= 2) {
|
|
267
|
+
return `${segments[0]}/${segments[1]}`;
|
|
268
|
+
}
|
|
269
|
+
return firstPath;
|
|
270
|
+
}
|
|
271
|
+
function normalizeMemoryContent(content) {
|
|
272
|
+
return content.trim().toLowerCase().replaceAll(/\s+/g, ' ');
|
|
273
|
+
}
|
|
274
|
+
function normalizeForFallbackDedup(content, category) {
|
|
275
|
+
let normalized = normalizeMemoryContent(content);
|
|
276
|
+
// PATTERNS and SKILLS are session-fingerprinted ("Session abc123: ...").
|
|
277
|
+
// Strip the prefix so repeated curate sessions on the same module are detected as duplicates.
|
|
278
|
+
if (category === 'PATTERNS' || category === 'SKILLS') {
|
|
279
|
+
normalized = normalized.replace(/^session\s+\S+:\s*/, '');
|
|
280
|
+
}
|
|
281
|
+
return normalized;
|
|
282
|
+
}
|
|
283
|
+
function getMemoryCategory(memory) {
|
|
284
|
+
if (!memory.metadata || typeof memory.metadata !== 'object') {
|
|
285
|
+
return undefined;
|
|
286
|
+
}
|
|
287
|
+
const { category } = memory.metadata;
|
|
288
|
+
return typeof category === 'string' ? category : undefined;
|
|
289
|
+
}
|
|
290
|
+
// Curate sessions always use deterministic fallback drafts instead of LLM extraction.
|
|
291
|
+
// Fallback drafts are faster (no LLM call), cheaper, and produce consistent categorized
|
|
292
|
+
// memories that can be deduped via string matching. LLM extraction is reserved for
|
|
293
|
+
// non-curate sessions (e.g., query) where conversation content is unpredictable.
|
|
294
|
+
function shouldPreferFallbackDrafts(commandType) {
|
|
295
|
+
return commandType.startsWith('curate');
|
|
296
|
+
}
|
|
@@ -160,6 +160,13 @@ export declare class SessionManager {
|
|
|
160
160
|
* @returns Session instance or undefined if not found
|
|
161
161
|
*/
|
|
162
162
|
getSession(id: string): IChatSession | undefined;
|
|
163
|
+
/**
|
|
164
|
+
* Get the command type (agent name) registered for a session.
|
|
165
|
+
*
|
|
166
|
+
* @param id - Session ID
|
|
167
|
+
* @returns Command type string (e.g. 'curate', 'query') or undefined if not found
|
|
168
|
+
*/
|
|
169
|
+
getSessionCommandType(id: string): string | undefined;
|
|
163
170
|
/**
|
|
164
171
|
* Get the number of active sessions.
|
|
165
172
|
*
|