byterover-cli 3.0.0 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/core/domain/tools/constants.d.ts +1 -0
- package/dist/agent/core/domain/tools/constants.js +1 -0
- package/dist/agent/core/interfaces/cipher-services.d.ts +8 -0
- package/dist/agent/core/interfaces/i-cipher-agent.d.ts +1 -0
- package/dist/agent/infra/agent/agent-error-codes.d.ts +0 -1
- package/dist/agent/infra/agent/agent-error-codes.js +0 -1
- package/dist/agent/infra/agent/agent-error.d.ts +0 -1
- package/dist/agent/infra/agent/agent-error.js +0 -1
- package/dist/agent/infra/agent/agent-state-manager.d.ts +1 -3
- package/dist/agent/infra/agent/agent-state-manager.js +1 -3
- package/dist/agent/infra/agent/base-agent.d.ts +1 -1
- package/dist/agent/infra/agent/base-agent.js +1 -1
- package/dist/agent/infra/agent/cipher-agent.d.ts +15 -1
- package/dist/agent/infra/agent/cipher-agent.js +188 -3
- package/dist/agent/infra/agent/index.d.ts +1 -1
- package/dist/agent/infra/agent/index.js +1 -1
- package/dist/agent/infra/agent/service-initializer.d.ts +3 -3
- package/dist/agent/infra/agent/service-initializer.js +14 -8
- package/dist/agent/infra/agent/types.d.ts +0 -1
- package/dist/agent/infra/file-system/file-system-service.js +6 -5
- package/dist/agent/infra/folder-pack/folder-pack-service.d.ts +1 -0
- package/dist/agent/infra/folder-pack/folder-pack-service.js +29 -15
- package/dist/agent/infra/llm/providers/openai.js +12 -0
- package/dist/agent/infra/llm/stream-to-text.d.ts +7 -0
- package/dist/agent/infra/llm/stream-to-text.js +14 -0
- package/dist/agent/infra/map/abstract-generator.d.ts +22 -0
- package/dist/agent/infra/map/abstract-generator.js +67 -0
- package/dist/agent/infra/map/abstract-queue.d.ts +67 -0
- package/dist/agent/infra/map/abstract-queue.js +218 -0
- package/dist/agent/infra/memory/memory-deduplicator.d.ts +44 -0
- package/dist/agent/infra/memory/memory-deduplicator.js +88 -0
- package/dist/agent/infra/memory/memory-manager.d.ts +1 -0
- package/dist/agent/infra/memory/memory-manager.js +6 -5
- package/dist/agent/infra/sandbox/curate-service.d.ts +4 -2
- package/dist/agent/infra/sandbox/curate-service.js +6 -7
- package/dist/agent/infra/sandbox/local-sandbox.d.ts +5 -0
- package/dist/agent/infra/sandbox/local-sandbox.js +57 -1
- package/dist/agent/infra/sandbox/tools-sdk.d.ts +3 -1
- package/dist/agent/infra/session/session-compressor.d.ts +43 -0
- package/dist/agent/infra/session/session-compressor.js +296 -0
- package/dist/agent/infra/session/session-manager.d.ts +7 -0
- package/dist/agent/infra/session/session-manager.js +9 -0
- package/dist/agent/infra/tools/implementations/curate-tool.d.ts +3 -2
- package/dist/agent/infra/tools/implementations/curate-tool.js +54 -27
- package/dist/agent/infra/tools/implementations/expand-knowledge-tool.d.ts +3 -3
- package/dist/agent/infra/tools/implementations/expand-knowledge-tool.js +34 -7
- package/dist/agent/infra/tools/implementations/ingest-resource-tool.d.ts +17 -0
- package/dist/agent/infra/tools/implementations/ingest-resource-tool.js +224 -0
- package/dist/agent/infra/tools/implementations/memory-symbol-tree.d.ts +8 -0
- package/dist/agent/infra/tools/implementations/search-knowledge-service.d.ts +1 -1
- package/dist/agent/infra/tools/implementations/search-knowledge-service.js +207 -34
- package/dist/agent/infra/tools/implementations/search-knowledge-tool.js +2 -2
- package/dist/agent/infra/tools/tool-provider.js +1 -0
- package/dist/agent/infra/tools/tool-registry.d.ts +3 -0
- package/dist/agent/infra/tools/tool-registry.js +15 -4
- package/dist/server/constants.d.ts +2 -0
- package/dist/server/constants.js +2 -0
- package/dist/server/core/domain/knowledge/memory-scoring.d.ts +3 -3
- package/dist/server/core/domain/knowledge/memory-scoring.js +5 -5
- package/dist/server/core/domain/knowledge/summary-types.d.ts +4 -0
- package/dist/server/core/domain/transport/schemas.d.ts +10 -10
- package/dist/server/infra/context-tree/derived-artifact.js +5 -1
- package/dist/server/infra/context-tree/file-context-tree-manifest-service.d.ts +2 -1
- package/dist/server/infra/context-tree/file-context-tree-manifest-service.js +43 -7
- package/dist/server/infra/context-tree/file-context-tree-summary-service.js +20 -2
- package/dist/server/infra/executor/curate-executor.js +2 -1
- package/dist/server/infra/executor/folder-pack-executor.js +72 -2
- package/dist/server/infra/executor/query-executor.js +11 -3
- package/dist/server/infra/transport/handlers/status-handler.js +10 -0
- package/dist/server/utils/curate-result-parser.d.ts +4 -4
- package/dist/shared/transport/types/dto.d.ts +7 -0
- package/oclif.manifest.json +1 -1
- package/package.json +10 -4
|
@@ -208,17 +208,18 @@ export class FileSystemService {
|
|
|
208
208
|
this.ensureInitialized();
|
|
209
209
|
const rawCwd = options.cwd ?? this.config.workingDirectory;
|
|
210
210
|
const cwd = path.isAbsolute(rawCwd) ? rawCwd : path.resolve(this.config.workingDirectory, rawCwd);
|
|
211
|
+
const normalizedCwd = await fs.realpath(cwd).catch(() => cwd);
|
|
211
212
|
const maxResults = options.maxResults ?? 1000;
|
|
212
213
|
const includeMetadata = options.includeMetadata ?? true;
|
|
213
214
|
const caseSensitive = options.caseSensitive ?? true;
|
|
214
215
|
const respectGitignore = options.respectGitignore ?? true;
|
|
215
216
|
try {
|
|
216
217
|
// Handle special characters - escape pattern if it matches an existing file
|
|
217
|
-
const escapedPattern = await escapeIfExactMatch(pattern,
|
|
218
|
+
const escapedPattern = await escapeIfExactMatch(pattern, normalizedCwd);
|
|
218
219
|
// Execute glob with case sensitivity option
|
|
219
220
|
const files = await glob(escapedPattern, {
|
|
220
221
|
absolute: true,
|
|
221
|
-
cwd,
|
|
222
|
+
cwd: normalizedCwd,
|
|
222
223
|
follow: false, // Don't follow symlinks
|
|
223
224
|
nocase: !caseSensitive, // Case insensitive if caseSensitive is false
|
|
224
225
|
nodir: true, // Only files
|
|
@@ -226,7 +227,7 @@ export class FileSystemService {
|
|
|
226
227
|
// Initialize gitignore filter if requested
|
|
227
228
|
let gitignoreFilter = null;
|
|
228
229
|
if (respectGitignore) {
|
|
229
|
-
gitignoreFilter = await createGitignoreFilter(
|
|
230
|
+
gitignoreFilter = await createGitignoreFilter(normalizedCwd);
|
|
230
231
|
}
|
|
231
232
|
// Validate paths and apply gitignore filtering
|
|
232
233
|
const validPaths = [];
|
|
@@ -240,7 +241,7 @@ export class FileSystemService {
|
|
|
240
241
|
}
|
|
241
242
|
// Apply gitignore filter if enabled
|
|
242
243
|
if (gitignoreFilter) {
|
|
243
|
-
const relativePath = path.relative(
|
|
244
|
+
const relativePath = path.relative(normalizedCwd, validation.normalizedPath);
|
|
244
245
|
if (gitignoreFilter.isIgnored(relativePath)) {
|
|
245
246
|
ignoredCount++;
|
|
246
247
|
continue;
|
|
@@ -250,7 +251,7 @@ export class FileSystemService {
|
|
|
250
251
|
}
|
|
251
252
|
const totalFound = validPaths.length;
|
|
252
253
|
// Collect metadata for all valid paths
|
|
253
|
-
const filesWithMetadata = await collectFileMetadata(validPaths,
|
|
254
|
+
const filesWithMetadata = await collectFileMetadata(validPaths, normalizedCwd);
|
|
254
255
|
// Sort files: recent files first (within 24h), then alphabetical
|
|
255
256
|
const sortedFiles = sortFilesByRecency(filesWithMetadata);
|
|
256
257
|
// Apply maxResults limit after sorting
|
|
@@ -109,9 +109,10 @@ export class FolderPackService {
|
|
|
109
109
|
const startTime = Date.now();
|
|
110
110
|
const mergedConfig = this.mergeConfig(config);
|
|
111
111
|
// Resolve to absolute path
|
|
112
|
-
const
|
|
112
|
+
const rawAbsolutePath = path.isAbsolute(folderPath)
|
|
113
113
|
? folderPath
|
|
114
114
|
: path.resolve(process.cwd(), folderPath);
|
|
115
|
+
const absolutePath = await fs.realpath(rawAbsolutePath).catch(() => rawAbsolutePath);
|
|
115
116
|
// Phase 1: Search for files
|
|
116
117
|
onProgress?.({ current: 0, message: 'Searching for files...', phase: 'searching' });
|
|
117
118
|
const ignorePatterns = [...getDefaultIgnorePatterns(), ...mergedConfig.ignore];
|
|
@@ -130,8 +131,18 @@ export class FolderPackService {
|
|
|
130
131
|
}
|
|
131
132
|
throw error;
|
|
132
133
|
}
|
|
133
|
-
|
|
134
|
-
|
|
134
|
+
const discoveredFiles = globResult.files
|
|
135
|
+
.map((file) => {
|
|
136
|
+
const absoluteFilePath = path.isAbsolute(file.path) ? file.path : path.resolve(absolutePath, file.path);
|
|
137
|
+
return {
|
|
138
|
+
absolutePath: absoluteFilePath,
|
|
139
|
+
relativePath: this.toRelativePackPath(absolutePath, absoluteFilePath),
|
|
140
|
+
size: file.size,
|
|
141
|
+
};
|
|
142
|
+
})
|
|
143
|
+
.filter((file) => file.relativePath.length > 0 && !file.relativePath.startsWith('../'));
|
|
144
|
+
// Filter files based on ignore patterns using paths relative to the packed root.
|
|
145
|
+
const filteredFiles = discoveredFiles.filter((file) => !this.matchesIgnorePattern(file.relativePath, ignorePatterns));
|
|
135
146
|
onProgress?.({
|
|
136
147
|
current: filteredFiles.length,
|
|
137
148
|
message: `Found ${filteredFiles.length} files`,
|
|
@@ -152,7 +163,7 @@ export class FolderPackService {
|
|
|
152
163
|
// Report progress (note: may be out of order due to parallelism)
|
|
153
164
|
onProgress?.({
|
|
154
165
|
current: index + 1,
|
|
155
|
-
message: `Reading ${fileInfo.
|
|
166
|
+
message: `Reading ${fileInfo.relativePath}`,
|
|
156
167
|
phase: 'collecting',
|
|
157
168
|
total: totalFiles,
|
|
158
169
|
});
|
|
@@ -162,19 +173,19 @@ export class FolderPackService {
|
|
|
162
173
|
return {
|
|
163
174
|
skipped: {
|
|
164
175
|
message: `File size ${fileInfo.size} exceeds limit ${mergedConfig.maxFileSize}`,
|
|
165
|
-
path: fileInfo.
|
|
176
|
+
path: fileInfo.relativePath,
|
|
166
177
|
reason: 'size-limit',
|
|
167
178
|
},
|
|
168
179
|
type: 'skipped',
|
|
169
180
|
};
|
|
170
181
|
}
|
|
171
182
|
// Check if this is an Office document that should be parsed
|
|
172
|
-
if (mergedConfig.extractDocuments && this.documentParser && isOfficeFile(fileInfo.
|
|
173
|
-
return this.parseOfficeDocument(fileInfo.
|
|
183
|
+
if (mergedConfig.extractDocuments && this.documentParser && isOfficeFile(fileInfo.absolutePath)) {
|
|
184
|
+
return this.parseOfficeDocument(fileInfo.absolutePath, fileInfo.relativePath, fileInfo.size);
|
|
174
185
|
}
|
|
175
186
|
// Read file content using FileSystemService
|
|
176
187
|
// This handles binary detection, PDF extraction, encoding, etc.
|
|
177
|
-
const fileContent = await this.fileSystemService.readFile(fileInfo.
|
|
188
|
+
const fileContent = await this.fileSystemService.readFile(fileInfo.absolutePath, {
|
|
178
189
|
limit: mergedConfig.maxLinesPerFile,
|
|
179
190
|
});
|
|
180
191
|
// Check if it's a PDF with extracted text
|
|
@@ -182,9 +193,9 @@ export class FolderPackService {
|
|
|
182
193
|
return {
|
|
183
194
|
file: {
|
|
184
195
|
content: fileContent.content,
|
|
185
|
-
fileType: isPdf ? 'pdf' : this.detectFileType(fileInfo.
|
|
196
|
+
fileType: isPdf ? 'pdf' : this.detectFileType(fileInfo.relativePath),
|
|
186
197
|
lineCount: fileContent.lines,
|
|
187
|
-
path: fileInfo.
|
|
198
|
+
path: fileInfo.relativePath,
|
|
188
199
|
size: fileInfo.size,
|
|
189
200
|
truncated: fileContent.truncated,
|
|
190
201
|
},
|
|
@@ -197,7 +208,7 @@ export class FolderPackService {
|
|
|
197
208
|
return {
|
|
198
209
|
skipped: {
|
|
199
210
|
message: error instanceof Error ? error.message : String(error),
|
|
200
|
-
path: fileInfo.
|
|
211
|
+
path: fileInfo.relativePath,
|
|
201
212
|
reason: skipReason,
|
|
202
213
|
},
|
|
203
214
|
type: 'skipped',
|
|
@@ -329,12 +340,12 @@ export class FolderPackService {
|
|
|
329
340
|
/**
|
|
330
341
|
* Parse an Office document using the document parser.
|
|
331
342
|
*/
|
|
332
|
-
async parseOfficeDocument(filePath, size) {
|
|
343
|
+
async parseOfficeDocument(filePath, outputPath, size) {
|
|
333
344
|
if (!this.documentParser) {
|
|
334
345
|
return {
|
|
335
346
|
skipped: {
|
|
336
347
|
message: 'Document parser not available',
|
|
337
|
-
path:
|
|
348
|
+
path: outputPath,
|
|
338
349
|
reason: 'read-error',
|
|
339
350
|
},
|
|
340
351
|
type: 'skipped',
|
|
@@ -351,7 +362,7 @@ export class FolderPackService {
|
|
|
351
362
|
content: result.content,
|
|
352
363
|
fileType: 'document',
|
|
353
364
|
lineCount: lines.length,
|
|
354
|
-
path:
|
|
365
|
+
path: outputPath,
|
|
355
366
|
size,
|
|
356
367
|
truncated: false,
|
|
357
368
|
},
|
|
@@ -362,11 +373,14 @@ export class FolderPackService {
|
|
|
362
373
|
return {
|
|
363
374
|
skipped: {
|
|
364
375
|
message: error instanceof Error ? error.message : String(error),
|
|
365
|
-
path:
|
|
376
|
+
path: outputPath,
|
|
366
377
|
reason: 'read-error',
|
|
367
378
|
},
|
|
368
379
|
type: 'skipped',
|
|
369
380
|
};
|
|
370
381
|
}
|
|
371
382
|
}
|
|
383
|
+
toRelativePackPath(rootPath, filePath) {
|
|
384
|
+
return path.relative(rootPath, filePath).replaceAll('\\', '/');
|
|
385
|
+
}
|
|
372
386
|
}
|
|
@@ -31,6 +31,18 @@ export function createChatGptOAuthFetch() {
|
|
|
31
31
|
catch {
|
|
32
32
|
return globalThis.fetch(input, init);
|
|
33
33
|
}
|
|
34
|
+
// The AI SDK sends systemPrompt as input[0] with role "system" or "developer".
|
|
35
|
+
// The ChatGPT OAuth Responses endpoint expects it in top-level `instructions`.
|
|
36
|
+
if (!body.instructions && Array.isArray(body.input) && body.input.length > 0) {
|
|
37
|
+
const first = body.input[0];
|
|
38
|
+
if (typeof first === 'object' && first !== null) {
|
|
39
|
+
const record = first;
|
|
40
|
+
if ((record.role === 'system' || record.role === 'developer') && typeof record.content === 'string') {
|
|
41
|
+
body.instructions = record.content;
|
|
42
|
+
body.input.splice(0, 1);
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
}
|
|
34
46
|
if (!body.instructions) {
|
|
35
47
|
body.instructions = '';
|
|
36
48
|
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { GenerateContentRequest, IContentGenerator } from '../../core/interfaces/i-content-generator.js';
|
|
2
|
+
/**
|
|
3
|
+
* Consume generateContentStream() and return accumulated text.
|
|
4
|
+
* Used instead of generateContent() because the ChatGPT OAuth Codex
|
|
5
|
+
* endpoint requires stream: true in all requests.
|
|
6
|
+
*/
|
|
7
|
+
export declare function streamToText(generator: IContentGenerator, request: GenerateContentRequest): Promise<string>;
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Consume generateContentStream() and return accumulated text.
|
|
3
|
+
* Used instead of generateContent() because the ChatGPT OAuth Codex
|
|
4
|
+
* endpoint requires stream: true in all requests.
|
|
5
|
+
*/
|
|
6
|
+
export async function streamToText(generator, request) {
|
|
7
|
+
const chunks = [];
|
|
8
|
+
for await (const chunk of generator.generateContentStream(request)) {
|
|
9
|
+
if (chunk.content) {
|
|
10
|
+
chunks.push(chunk.content);
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
return chunks.join('');
|
|
14
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { IContentGenerator } from '../../core/interfaces/i-content-generator.js';
|
|
2
|
+
/**
|
|
3
|
+
* Result from abstract generation.
|
|
4
|
+
*/
|
|
5
|
+
export interface AbstractGenerateResult {
|
|
6
|
+
/** L0: one-line summary (~80 tokens) */
|
|
7
|
+
abstractContent: string;
|
|
8
|
+
/** L1: key points + structure (~1500 tokens) */
|
|
9
|
+
overviewContent: string;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Generate L0 abstract and L1 overview for a knowledge file.
|
|
13
|
+
*
|
|
14
|
+
* Makes two parallel LLM calls at temperature=0:
|
|
15
|
+
* 1. L0 .abstract.md — one-line summary (~80 tokens)
|
|
16
|
+
* 2. L1 .overview.md — key points + structure (~1500 tokens)
|
|
17
|
+
*
|
|
18
|
+
* @param fullContent - Full markdown content of the knowledge file
|
|
19
|
+
* @param generator - LLM content generator
|
|
20
|
+
* @returns Abstract and overview content strings
|
|
21
|
+
*/
|
|
22
|
+
export declare function generateFileAbstracts(fullContent: string, generator: IContentGenerator): Promise<AbstractGenerateResult>;
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import { streamToText } from '../llm/stream-to-text.js';
|
|
3
|
+
const ABSTRACT_SYSTEM_PROMPT = `You are a technical documentation assistant.
|
|
4
|
+
Your job is to produce precise, factual summaries of knowledge documents.
|
|
5
|
+
Output only the requested content — no preamble, no commentary.`;
|
|
6
|
+
const OVERVIEW_SYSTEM_PROMPT = `You are a technical documentation assistant.
|
|
7
|
+
Your job is to produce structured overviews of knowledge documents.
|
|
8
|
+
Preserve factual accuracy, surface important entities and decisions, and format the result in concise markdown.`;
|
|
9
|
+
function buildAbstractPrompt(content) {
|
|
10
|
+
return `Produce a ONE-LINE summary (max 80 tokens) of the following knowledge document.
|
|
11
|
+
The line must be a complete sentence that captures the core topic and key insight.
|
|
12
|
+
Output only the single line — nothing else.
|
|
13
|
+
|
|
14
|
+
<document>
|
|
15
|
+
${content}
|
|
16
|
+
</document>`;
|
|
17
|
+
}
|
|
18
|
+
function buildOverviewPrompt(content) {
|
|
19
|
+
return `Produce a structured overview of the following knowledge document.
|
|
20
|
+
Include:
|
|
21
|
+
- Key points (3-7 bullet points)
|
|
22
|
+
- Structure / sections summary
|
|
23
|
+
- Any notable entities, patterns, or decisions mentioned
|
|
24
|
+
|
|
25
|
+
Keep it under 1500 tokens. Use markdown formatting.
|
|
26
|
+
Output only the overview — no preamble.
|
|
27
|
+
|
|
28
|
+
<document>
|
|
29
|
+
${content}
|
|
30
|
+
</document>`;
|
|
31
|
+
}
|
|
32
|
+
/** Truncate content before embedding in LLM prompts to avoid exceeding model context windows during bulk ingest. */
|
|
33
|
+
const MAX_ABSTRACT_CONTENT_CHARS = 20_000;
|
|
34
|
+
/**
|
|
35
|
+
* Generate L0 abstract and L1 overview for a knowledge file.
|
|
36
|
+
*
|
|
37
|
+
* Makes two parallel LLM calls at temperature=0:
|
|
38
|
+
* 1. L0 .abstract.md — one-line summary (~80 tokens)
|
|
39
|
+
* 2. L1 .overview.md — key points + structure (~1500 tokens)
|
|
40
|
+
*
|
|
41
|
+
* @param fullContent - Full markdown content of the knowledge file
|
|
42
|
+
* @param generator - LLM content generator
|
|
43
|
+
* @returns Abstract and overview content strings
|
|
44
|
+
*/
|
|
45
|
+
export async function generateFileAbstracts(fullContent, generator) {
|
|
46
|
+
const truncated = fullContent.slice(0, MAX_ABSTRACT_CONTENT_CHARS);
|
|
47
|
+
const [abstractText, overviewText] = await Promise.all([
|
|
48
|
+
streamToText(generator, {
|
|
49
|
+
config: { maxTokens: 150, temperature: 0 },
|
|
50
|
+
contents: [{ content: buildAbstractPrompt(truncated), role: 'user' }],
|
|
51
|
+
model: 'default',
|
|
52
|
+
systemPrompt: ABSTRACT_SYSTEM_PROMPT,
|
|
53
|
+
taskId: randomUUID(),
|
|
54
|
+
}),
|
|
55
|
+
streamToText(generator, {
|
|
56
|
+
config: { maxTokens: 2000, temperature: 0 },
|
|
57
|
+
contents: [{ content: buildOverviewPrompt(truncated), role: 'user' }],
|
|
58
|
+
model: 'default',
|
|
59
|
+
systemPrompt: OVERVIEW_SYSTEM_PROMPT,
|
|
60
|
+
taskId: randomUUID(),
|
|
61
|
+
}),
|
|
62
|
+
]);
|
|
63
|
+
return {
|
|
64
|
+
abstractContent: abstractText.trim(),
|
|
65
|
+
overviewContent: overviewText.trim(),
|
|
66
|
+
};
|
|
67
|
+
}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import type { IContentGenerator } from '../../core/interfaces/i-content-generator.js';
|
|
2
|
+
/**
|
|
3
|
+
* Observable status of the abstract generation queue.
|
|
4
|
+
*/
|
|
5
|
+
export interface AbstractQueueStatus {
|
|
6
|
+
failed: number;
|
|
7
|
+
pending: number;
|
|
8
|
+
processed: number;
|
|
9
|
+
processing: boolean;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Background queue for generating L0/L1 abstract files (.abstract.md, .overview.md).
|
|
13
|
+
*
|
|
14
|
+
* - Generator is injected lazily via setGenerator() (mirrors rebindMapTools pattern)
|
|
15
|
+
* - Items arriving before setGenerator() are buffered and processed once generator is set
|
|
16
|
+
* - Writes status to <projectRoot>/.brv/_queue_status.json after each state transition
|
|
17
|
+
* - Retries up to maxAttempts with exponential backoff (500ms base)
|
|
18
|
+
* - drain() waits for all pending/processing items to complete (for graceful shutdown)
|
|
19
|
+
*/
|
|
20
|
+
export declare class AbstractGenerationQueue {
|
|
21
|
+
private readonly projectRoot;
|
|
22
|
+
private readonly maxAttempts;
|
|
23
|
+
private drainResolvers;
|
|
24
|
+
private failed;
|
|
25
|
+
private generator;
|
|
26
|
+
private onBeforeProcess?;
|
|
27
|
+
private pending;
|
|
28
|
+
private processed;
|
|
29
|
+
private processing;
|
|
30
|
+
/** Number of items currently in retry backoff (removed from pending but not yet re-enqueued). */
|
|
31
|
+
private retrying;
|
|
32
|
+
private statusDirCreated;
|
|
33
|
+
private statusWriteFailed;
|
|
34
|
+
private statusWritePromise;
|
|
35
|
+
constructor(projectRoot: string, maxAttempts?: number);
|
|
36
|
+
/**
|
|
37
|
+
* Wait for all pending items to finish processing (graceful shutdown).
|
|
38
|
+
* Includes items currently in retry backoff so drain() does not resolve prematurely.
|
|
39
|
+
*/
|
|
40
|
+
drain(): Promise<void>;
|
|
41
|
+
/**
|
|
42
|
+
* Add a file to the abstract generation queue.
|
|
43
|
+
*/
|
|
44
|
+
enqueue(item: {
|
|
45
|
+
contextPath: string;
|
|
46
|
+
fullContent: string;
|
|
47
|
+
}): void;
|
|
48
|
+
/**
|
|
49
|
+
* Return current queue status snapshot.
|
|
50
|
+
*/
|
|
51
|
+
getStatus(): AbstractQueueStatus;
|
|
52
|
+
/**
|
|
53
|
+
* Set a callback that runs before each item is processed.
|
|
54
|
+
* Used to refresh OAuth tokens before LLM calls.
|
|
55
|
+
*/
|
|
56
|
+
setBeforeProcess(fn: () => Promise<void>): void;
|
|
57
|
+
/**
|
|
58
|
+
* Inject the LLM generator. Triggers processing of any buffered items.
|
|
59
|
+
*/
|
|
60
|
+
setGenerator(generator: IContentGenerator): void;
|
|
61
|
+
private isIdle;
|
|
62
|
+
private processNext;
|
|
63
|
+
private queueStatusWrite;
|
|
64
|
+
private resolveDrainersIfIdle;
|
|
65
|
+
private scheduleNext;
|
|
66
|
+
private writeStatusFile;
|
|
67
|
+
}
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
import { appendFileSync } from 'node:fs';
|
|
2
|
+
import { mkdir, writeFile } from 'node:fs/promises';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
import { generateFileAbstracts } from './abstract-generator.js';
|
|
5
|
+
const QUEUE_TRACE_ENABLED = process.env.BRV_QUEUE_TRACE === '1';
|
|
6
|
+
const LOG_PATH = process.env.BRV_SESSION_LOG;
|
|
7
|
+
function queueLog(message) {
|
|
8
|
+
if (!QUEUE_TRACE_ENABLED || !LOG_PATH)
|
|
9
|
+
return;
|
|
10
|
+
try {
|
|
11
|
+
appendFileSync(LOG_PATH, `${new Date().toISOString()} [abstract-queue] ${message}\n`);
|
|
12
|
+
}
|
|
13
|
+
catch {
|
|
14
|
+
// ignore — tracing must never block queue progress
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Background queue for generating L0/L1 abstract files (.abstract.md, .overview.md).
|
|
19
|
+
*
|
|
20
|
+
* - Generator is injected lazily via setGenerator() (mirrors rebindMapTools pattern)
|
|
21
|
+
* - Items arriving before setGenerator() are buffered and processed once generator is set
|
|
22
|
+
* - Writes status to <projectRoot>/.brv/_queue_status.json after each state transition
|
|
23
|
+
* - Retries up to maxAttempts with exponential backoff (500ms base)
|
|
24
|
+
* - drain() waits for all pending/processing items to complete (for graceful shutdown)
|
|
25
|
+
*/
|
|
26
|
+
export class AbstractGenerationQueue {
|
|
27
|
+
projectRoot;
|
|
28
|
+
maxAttempts;
|
|
29
|
+
drainResolvers = [];
|
|
30
|
+
failed = 0;
|
|
31
|
+
generator;
|
|
32
|
+
onBeforeProcess;
|
|
33
|
+
pending = [];
|
|
34
|
+
processed = 0;
|
|
35
|
+
processing = false;
|
|
36
|
+
/** Number of items currently in retry backoff (removed from pending but not yet re-enqueued). */
|
|
37
|
+
retrying = 0;
|
|
38
|
+
statusDirCreated = false;
|
|
39
|
+
statusWriteFailed = false;
|
|
40
|
+
statusWritePromise = Promise.resolve();
|
|
41
|
+
constructor(projectRoot, maxAttempts = 3) {
|
|
42
|
+
this.projectRoot = projectRoot;
|
|
43
|
+
this.maxAttempts = maxAttempts;
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Wait for all pending items to finish processing (graceful shutdown).
|
|
47
|
+
* Includes items currently in retry backoff so drain() does not resolve prematurely.
|
|
48
|
+
*/
|
|
49
|
+
async drain() {
|
|
50
|
+
queueLog(`drain:start idle=${this.isIdle()} pending=${this.pending.length} retrying=${this.retrying} processing=${this.processing}`);
|
|
51
|
+
if (this.isIdle()) {
|
|
52
|
+
await this.statusWritePromise.catch(() => { });
|
|
53
|
+
queueLog('drain:resolved-immediate');
|
|
54
|
+
return;
|
|
55
|
+
}
|
|
56
|
+
await new Promise((resolve) => {
|
|
57
|
+
this.drainResolvers.push(resolve);
|
|
58
|
+
this.resolveDrainersIfIdle();
|
|
59
|
+
});
|
|
60
|
+
queueLog('drain:resolved-deferred');
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Add a file to the abstract generation queue.
|
|
64
|
+
*/
|
|
65
|
+
enqueue(item) {
|
|
66
|
+
// Guard against paths that must never trigger abstract generation:
|
|
67
|
+
// - derived artifacts (.abstract.md, .overview.md) — would produce .abstract.abstract.md
|
|
68
|
+
// - summary index files (_index.md) — domain/topic summaries, not knowledge nodes
|
|
69
|
+
// - hierarchy scaffolding (context.md) — helper files, not leaf knowledge entries
|
|
70
|
+
const fileName = item.contextPath.split('/').at(-1) ?? item.contextPath;
|
|
71
|
+
if (fileName === 'context.md' ||
|
|
72
|
+
fileName === '_index.md' ||
|
|
73
|
+
item.contextPath.endsWith('.abstract.md') ||
|
|
74
|
+
item.contextPath.endsWith('.overview.md')) {
|
|
75
|
+
return;
|
|
76
|
+
}
|
|
77
|
+
this.pending.push({ attempts: 0, contextPath: item.contextPath, fullContent: item.fullContent });
|
|
78
|
+
queueLog(`enqueue path=${item.contextPath} pending=${this.pending.length} retrying=${this.retrying} processing=${this.processing}`);
|
|
79
|
+
this.queueStatusWrite();
|
|
80
|
+
this.scheduleNext();
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Return current queue status snapshot.
|
|
84
|
+
*/
|
|
85
|
+
getStatus() {
|
|
86
|
+
return {
|
|
87
|
+
failed: this.failed,
|
|
88
|
+
// Items in retry backoff are still pending work — include them so the status
|
|
89
|
+
// does not falsely report the queue as idle during backoff windows.
|
|
90
|
+
pending: this.pending.length + this.retrying,
|
|
91
|
+
processed: this.processed,
|
|
92
|
+
processing: this.processing,
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* Set a callback that runs before each item is processed.
|
|
97
|
+
* Used to refresh OAuth tokens before LLM calls.
|
|
98
|
+
*/
|
|
99
|
+
setBeforeProcess(fn) {
|
|
100
|
+
this.onBeforeProcess = fn;
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* Inject the LLM generator. Triggers processing of any buffered items.
|
|
104
|
+
*/
|
|
105
|
+
setGenerator(generator) {
|
|
106
|
+
this.generator = generator;
|
|
107
|
+
this.scheduleNext();
|
|
108
|
+
}
|
|
109
|
+
isIdle() {
|
|
110
|
+
return this.pending.length === 0 && !this.processing && this.retrying === 0;
|
|
111
|
+
}
|
|
112
|
+
async processNext() {
|
|
113
|
+
if (!this.generator || this.processing || this.pending.length === 0) {
|
|
114
|
+
this.resolveDrainersIfIdle();
|
|
115
|
+
return;
|
|
116
|
+
}
|
|
117
|
+
this.processing = true;
|
|
118
|
+
this.queueStatusWrite();
|
|
119
|
+
const item = this.pending.shift();
|
|
120
|
+
queueLog(`process:start path=${item.contextPath} remaining=${this.pending.length} retrying=${this.retrying}`);
|
|
121
|
+
try {
|
|
122
|
+
// Refresh credentials before each generation (OAuth tokens may expire)
|
|
123
|
+
try {
|
|
124
|
+
await this.onBeforeProcess?.();
|
|
125
|
+
}
|
|
126
|
+
catch (error) {
|
|
127
|
+
const msg = error instanceof Error ? error.message : String(error);
|
|
128
|
+
console.debug(`[AbstractQueue] token refresh failed, proceeding with existing generator: ${msg}`);
|
|
129
|
+
}
|
|
130
|
+
const { abstractContent, overviewContent } = await generateFileAbstracts(item.fullContent, this.generator);
|
|
131
|
+
// Derive sibling paths: replace .md with .abstract.md and .overview.md
|
|
132
|
+
const abstractPath = item.contextPath.replace(/\.md$/, '.abstract.md');
|
|
133
|
+
const overviewPath = item.contextPath.replace(/\.md$/, '.overview.md');
|
|
134
|
+
await Promise.all([
|
|
135
|
+
writeFile(abstractPath, abstractContent, 'utf8'),
|
|
136
|
+
writeFile(overviewPath, overviewContent, 'utf8'),
|
|
137
|
+
]);
|
|
138
|
+
this.processed++;
|
|
139
|
+
queueLog(`process:success path=${item.contextPath} processed=${this.processed}`);
|
|
140
|
+
}
|
|
141
|
+
catch (error) {
|
|
142
|
+
const msg = error instanceof Error ? error.message : String(error);
|
|
143
|
+
console.debug(`[AbstractQueue] ${item.contextPath} attempt ${item.attempts + 1}/${this.maxAttempts}: ${msg}`);
|
|
144
|
+
item.attempts++;
|
|
145
|
+
if (item.attempts < this.maxAttempts) {
|
|
146
|
+
// Exponential backoff: 500ms, 1000ms, 2000ms, ...
|
|
147
|
+
const delay = 500 * 2 ** (item.attempts - 1);
|
|
148
|
+
this.retrying++;
|
|
149
|
+
this.queueStatusWrite();
|
|
150
|
+
setTimeout(() => {
|
|
151
|
+
this.retrying--;
|
|
152
|
+
this.pending.unshift(item);
|
|
153
|
+
queueLog(`process:retry-requeue path=${item.contextPath} pending=${this.pending.length} retrying=${this.retrying}`);
|
|
154
|
+
this.queueStatusWrite();
|
|
155
|
+
this.scheduleNext();
|
|
156
|
+
}, delay);
|
|
157
|
+
}
|
|
158
|
+
else {
|
|
159
|
+
this.failed++;
|
|
160
|
+
queueLog(`process:failed path=${item.contextPath} failed=${this.failed}`);
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
finally {
|
|
164
|
+
this.processing = false;
|
|
165
|
+
queueLog(`process:finally path=${item.contextPath} pending=${this.pending.length} retrying=${this.retrying} processed=${this.processed} failed=${this.failed}`);
|
|
166
|
+
this.queueStatusWrite();
|
|
167
|
+
}
|
|
168
|
+
this.scheduleNext();
|
|
169
|
+
this.resolveDrainersIfIdle();
|
|
170
|
+
}
|
|
171
|
+
queueStatusWrite() {
|
|
172
|
+
this.statusWritePromise = this.statusWritePromise
|
|
173
|
+
.catch(() => { })
|
|
174
|
+
.then(async () => this.writeStatusFile());
|
|
175
|
+
}
|
|
176
|
+
resolveDrainersIfIdle() {
|
|
177
|
+
if (!this.isIdle() || this.drainResolvers.length === 0) {
|
|
178
|
+
return;
|
|
179
|
+
}
|
|
180
|
+
queueLog(`drain:idle pending=${this.pending.length} retrying=${this.retrying} processed=${this.processed} failed=${this.failed}`);
|
|
181
|
+
const resolvers = this.drainResolvers.splice(0);
|
|
182
|
+
const settledStatusWrite = this.statusWritePromise.catch(() => { });
|
|
183
|
+
for (const resolve of resolvers) {
|
|
184
|
+
settledStatusWrite.then(() => resolve()).catch(() => { });
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
scheduleNext() {
|
|
188
|
+
if (!this.generator || this.processing || this.pending.length === 0) {
|
|
189
|
+
this.resolveDrainersIfIdle();
|
|
190
|
+
return;
|
|
191
|
+
}
|
|
192
|
+
// eslint-disable-next-line no-void
|
|
193
|
+
setImmediate(() => { void this.processNext(); });
|
|
194
|
+
}
|
|
195
|
+
async writeStatusFile() {
|
|
196
|
+
const statusPath = join(this.projectRoot, '.brv', '_queue_status.json');
|
|
197
|
+
try {
|
|
198
|
+
if (!this.statusDirCreated) {
|
|
199
|
+
await mkdir(join(this.projectRoot, '.brv'), { recursive: true });
|
|
200
|
+
this.statusDirCreated = true;
|
|
201
|
+
}
|
|
202
|
+
await writeFile(statusPath, JSON.stringify(this.getStatus()), 'utf8');
|
|
203
|
+
this.statusWriteFailed = false;
|
|
204
|
+
}
|
|
205
|
+
catch (error) {
|
|
206
|
+
const errorCode = typeof error === 'object' && error !== null && 'code' in error
|
|
207
|
+
? error.code
|
|
208
|
+
: undefined;
|
|
209
|
+
if (errorCode === 'ENOENT') {
|
|
210
|
+
return;
|
|
211
|
+
}
|
|
212
|
+
if (!this.statusWriteFailed) {
|
|
213
|
+
this.statusWriteFailed = true;
|
|
214
|
+
console.debug(`[AbstractGenerationQueue] Failed to write queue status: ${error instanceof Error ? error.message : String(error)}`);
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import type { Memory } from '../../core/domain/memory/types.js';
|
|
2
|
+
import type { IContentGenerator } from '../../core/interfaces/i-content-generator.js';
|
|
3
|
+
/**
|
|
4
|
+
* A draft memory extracted from a session, before deduplication.
|
|
5
|
+
*/
|
|
6
|
+
export interface DraftMemory {
|
|
7
|
+
category: string;
|
|
8
|
+
content: string;
|
|
9
|
+
tags?: string[];
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Deduplication decision for a single draft memory.
|
|
13
|
+
*/
|
|
14
|
+
export type DeduplicationAction = {
|
|
15
|
+
action: 'CREATE';
|
|
16
|
+
memory: DraftMemory;
|
|
17
|
+
} | {
|
|
18
|
+
action: 'MERGE';
|
|
19
|
+
memory: DraftMemory;
|
|
20
|
+
mergedContent: string;
|
|
21
|
+
targetId: string;
|
|
22
|
+
} | {
|
|
23
|
+
action: 'SKIP';
|
|
24
|
+
memory: DraftMemory;
|
|
25
|
+
};
|
|
26
|
+
/**
|
|
27
|
+
* LLM-based deduplicator for agent-extracted memories.
|
|
28
|
+
*
|
|
29
|
+
* For each draft, checks against existing memories via an LLM call.
|
|
30
|
+
* DECISIONS category drafts always result in CREATE (immutable log).
|
|
31
|
+
*/
|
|
32
|
+
export declare class MemoryDeduplicator {
|
|
33
|
+
private readonly generator;
|
|
34
|
+
constructor(generator: IContentGenerator);
|
|
35
|
+
/**
|
|
36
|
+
* Deduplicate a list of draft memories against existing stored memories.
|
|
37
|
+
*
|
|
38
|
+
* @param drafts - Draft memories to check
|
|
39
|
+
* @param existing - Existing memories to compare against
|
|
40
|
+
* @returns Deduplication action for each draft
|
|
41
|
+
*/
|
|
42
|
+
deduplicate(drafts: DraftMemory[], existing: Memory[]): Promise<DeduplicationAction[]>;
|
|
43
|
+
private deduplicateSingle;
|
|
44
|
+
}
|