byterover-cli 3.0.0 → 3.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/dist/agent/core/domain/tools/constants.d.ts +1 -0
  2. package/dist/agent/core/domain/tools/constants.js +1 -0
  3. package/dist/agent/core/interfaces/cipher-services.d.ts +8 -0
  4. package/dist/agent/core/interfaces/i-cipher-agent.d.ts +1 -0
  5. package/dist/agent/infra/agent/agent-error-codes.d.ts +0 -1
  6. package/dist/agent/infra/agent/agent-error-codes.js +0 -1
  7. package/dist/agent/infra/agent/agent-error.d.ts +0 -1
  8. package/dist/agent/infra/agent/agent-error.js +0 -1
  9. package/dist/agent/infra/agent/agent-state-manager.d.ts +1 -3
  10. package/dist/agent/infra/agent/agent-state-manager.js +1 -3
  11. package/dist/agent/infra/agent/base-agent.d.ts +1 -1
  12. package/dist/agent/infra/agent/base-agent.js +1 -1
  13. package/dist/agent/infra/agent/cipher-agent.d.ts +15 -1
  14. package/dist/agent/infra/agent/cipher-agent.js +188 -3
  15. package/dist/agent/infra/agent/index.d.ts +1 -1
  16. package/dist/agent/infra/agent/index.js +1 -1
  17. package/dist/agent/infra/agent/service-initializer.d.ts +3 -3
  18. package/dist/agent/infra/agent/service-initializer.js +14 -8
  19. package/dist/agent/infra/agent/types.d.ts +0 -1
  20. package/dist/agent/infra/file-system/file-system-service.js +6 -5
  21. package/dist/agent/infra/folder-pack/folder-pack-service.d.ts +1 -0
  22. package/dist/agent/infra/folder-pack/folder-pack-service.js +29 -15
  23. package/dist/agent/infra/llm/providers/openai.js +12 -0
  24. package/dist/agent/infra/llm/stream-to-text.d.ts +7 -0
  25. package/dist/agent/infra/llm/stream-to-text.js +14 -0
  26. package/dist/agent/infra/map/abstract-generator.d.ts +22 -0
  27. package/dist/agent/infra/map/abstract-generator.js +67 -0
  28. package/dist/agent/infra/map/abstract-queue.d.ts +67 -0
  29. package/dist/agent/infra/map/abstract-queue.js +218 -0
  30. package/dist/agent/infra/memory/memory-deduplicator.d.ts +44 -0
  31. package/dist/agent/infra/memory/memory-deduplicator.js +88 -0
  32. package/dist/agent/infra/memory/memory-manager.d.ts +1 -0
  33. package/dist/agent/infra/memory/memory-manager.js +6 -5
  34. package/dist/agent/infra/sandbox/curate-service.d.ts +4 -2
  35. package/dist/agent/infra/sandbox/curate-service.js +6 -7
  36. package/dist/agent/infra/sandbox/local-sandbox.d.ts +5 -0
  37. package/dist/agent/infra/sandbox/local-sandbox.js +57 -1
  38. package/dist/agent/infra/sandbox/tools-sdk.d.ts +3 -1
  39. package/dist/agent/infra/session/session-compressor.d.ts +43 -0
  40. package/dist/agent/infra/session/session-compressor.js +296 -0
  41. package/dist/agent/infra/session/session-manager.d.ts +7 -0
  42. package/dist/agent/infra/session/session-manager.js +9 -0
  43. package/dist/agent/infra/tools/implementations/curate-tool.d.ts +3 -2
  44. package/dist/agent/infra/tools/implementations/curate-tool.js +54 -27
  45. package/dist/agent/infra/tools/implementations/expand-knowledge-tool.d.ts +3 -3
  46. package/dist/agent/infra/tools/implementations/expand-knowledge-tool.js +34 -7
  47. package/dist/agent/infra/tools/implementations/ingest-resource-tool.d.ts +17 -0
  48. package/dist/agent/infra/tools/implementations/ingest-resource-tool.js +224 -0
  49. package/dist/agent/infra/tools/implementations/memory-symbol-tree.d.ts +8 -0
  50. package/dist/agent/infra/tools/implementations/search-knowledge-service.d.ts +1 -1
  51. package/dist/agent/infra/tools/implementations/search-knowledge-service.js +207 -34
  52. package/dist/agent/infra/tools/implementations/search-knowledge-tool.js +2 -2
  53. package/dist/agent/infra/tools/tool-provider.js +1 -0
  54. package/dist/agent/infra/tools/tool-registry.d.ts +3 -0
  55. package/dist/agent/infra/tools/tool-registry.js +15 -4
  56. package/dist/server/constants.d.ts +2 -0
  57. package/dist/server/constants.js +2 -0
  58. package/dist/server/core/domain/knowledge/memory-scoring.d.ts +3 -3
  59. package/dist/server/core/domain/knowledge/memory-scoring.js +5 -5
  60. package/dist/server/core/domain/knowledge/summary-types.d.ts +4 -0
  61. package/dist/server/core/domain/transport/schemas.d.ts +10 -10
  62. package/dist/server/infra/context-tree/derived-artifact.js +5 -1
  63. package/dist/server/infra/context-tree/file-context-tree-manifest-service.d.ts +2 -1
  64. package/dist/server/infra/context-tree/file-context-tree-manifest-service.js +43 -7
  65. package/dist/server/infra/context-tree/file-context-tree-summary-service.js +20 -2
  66. package/dist/server/infra/executor/curate-executor.js +2 -1
  67. package/dist/server/infra/executor/folder-pack-executor.js +72 -2
  68. package/dist/server/infra/executor/query-executor.js +11 -3
  69. package/dist/server/infra/transport/handlers/status-handler.js +10 -0
  70. package/dist/server/utils/curate-result-parser.d.ts +4 -4
  71. package/dist/shared/transport/types/dto.d.ts +7 -0
  72. package/oclif.manifest.json +1 -1
  73. package/package.json +10 -4
@@ -208,17 +208,18 @@ export class FileSystemService {
208
208
  this.ensureInitialized();
209
209
  const rawCwd = options.cwd ?? this.config.workingDirectory;
210
210
  const cwd = path.isAbsolute(rawCwd) ? rawCwd : path.resolve(this.config.workingDirectory, rawCwd);
211
+ const normalizedCwd = await fs.realpath(cwd).catch(() => cwd);
211
212
  const maxResults = options.maxResults ?? 1000;
212
213
  const includeMetadata = options.includeMetadata ?? true;
213
214
  const caseSensitive = options.caseSensitive ?? true;
214
215
  const respectGitignore = options.respectGitignore ?? true;
215
216
  try {
216
217
  // Handle special characters - escape pattern if it matches an existing file
217
- const escapedPattern = await escapeIfExactMatch(pattern, cwd);
218
+ const escapedPattern = await escapeIfExactMatch(pattern, normalizedCwd);
218
219
  // Execute glob with case sensitivity option
219
220
  const files = await glob(escapedPattern, {
220
221
  absolute: true,
221
- cwd,
222
+ cwd: normalizedCwd,
222
223
  follow: false, // Don't follow symlinks
223
224
  nocase: !caseSensitive, // Case insensitive if caseSensitive is false
224
225
  nodir: true, // Only files
@@ -226,7 +227,7 @@ export class FileSystemService {
226
227
  // Initialize gitignore filter if requested
227
228
  let gitignoreFilter = null;
228
229
  if (respectGitignore) {
229
- gitignoreFilter = await createGitignoreFilter(cwd);
230
+ gitignoreFilter = await createGitignoreFilter(normalizedCwd);
230
231
  }
231
232
  // Validate paths and apply gitignore filtering
232
233
  const validPaths = [];
@@ -240,7 +241,7 @@ export class FileSystemService {
240
241
  }
241
242
  // Apply gitignore filter if enabled
242
243
  if (gitignoreFilter) {
243
- const relativePath = path.relative(cwd, validation.normalizedPath);
244
+ const relativePath = path.relative(normalizedCwd, validation.normalizedPath);
244
245
  if (gitignoreFilter.isIgnored(relativePath)) {
245
246
  ignoredCount++;
246
247
  continue;
@@ -250,7 +251,7 @@ export class FileSystemService {
250
251
  }
251
252
  const totalFound = validPaths.length;
252
253
  // Collect metadata for all valid paths
253
- const filesWithMetadata = await collectFileMetadata(validPaths, cwd);
254
+ const filesWithMetadata = await collectFileMetadata(validPaths, normalizedCwd);
254
255
  // Sort files: recent files first (within 24h), then alphabetical
255
256
  const sortedFiles = sortFilesByRecency(filesWithMetadata);
256
257
  // Apply maxResults limit after sorting
@@ -54,4 +54,5 @@ export declare class FolderPackService implements IFolderPackService {
54
54
  * Parse an Office document using the document parser.
55
55
  */
56
56
  private parseOfficeDocument;
57
+ private toRelativePackPath;
57
58
  }
@@ -109,9 +109,10 @@ export class FolderPackService {
109
109
  const startTime = Date.now();
110
110
  const mergedConfig = this.mergeConfig(config);
111
111
  // Resolve to absolute path
112
- const absolutePath = path.isAbsolute(folderPath)
112
+ const rawAbsolutePath = path.isAbsolute(folderPath)
113
113
  ? folderPath
114
114
  : path.resolve(process.cwd(), folderPath);
115
+ const absolutePath = await fs.realpath(rawAbsolutePath).catch(() => rawAbsolutePath);
115
116
  // Phase 1: Search for files
116
117
  onProgress?.({ current: 0, message: 'Searching for files...', phase: 'searching' });
117
118
  const ignorePatterns = [...getDefaultIgnorePatterns(), ...mergedConfig.ignore];
@@ -130,8 +131,18 @@ export class FolderPackService {
130
131
  }
131
132
  throw error;
132
133
  }
133
- // Filter files based on ignore patterns
134
- const filteredFiles = globResult.files.filter((file) => !this.matchesIgnorePattern(file.path, ignorePatterns));
134
+ const discoveredFiles = globResult.files
135
+ .map((file) => {
136
+ const absoluteFilePath = path.isAbsolute(file.path) ? file.path : path.resolve(absolutePath, file.path);
137
+ return {
138
+ absolutePath: absoluteFilePath,
139
+ relativePath: this.toRelativePackPath(absolutePath, absoluteFilePath),
140
+ size: file.size,
141
+ };
142
+ })
143
+ .filter((file) => file.relativePath.length > 0 && !file.relativePath.startsWith('../'));
144
+ // Filter files based on ignore patterns using paths relative to the packed root.
145
+ const filteredFiles = discoveredFiles.filter((file) => !this.matchesIgnorePattern(file.relativePath, ignorePatterns));
135
146
  onProgress?.({
136
147
  current: filteredFiles.length,
137
148
  message: `Found ${filteredFiles.length} files`,
@@ -152,7 +163,7 @@ export class FolderPackService {
152
163
  // Report progress (note: may be out of order due to parallelism)
153
164
  onProgress?.({
154
165
  current: index + 1,
155
- message: `Reading ${fileInfo.path}`,
166
+ message: `Reading ${fileInfo.relativePath}`,
156
167
  phase: 'collecting',
157
168
  total: totalFiles,
158
169
  });
@@ -162,19 +173,19 @@ export class FolderPackService {
162
173
  return {
163
174
  skipped: {
164
175
  message: `File size ${fileInfo.size} exceeds limit ${mergedConfig.maxFileSize}`,
165
- path: fileInfo.path,
176
+ path: fileInfo.relativePath,
166
177
  reason: 'size-limit',
167
178
  },
168
179
  type: 'skipped',
169
180
  };
170
181
  }
171
182
  // Check if this is an Office document that should be parsed
172
- if (mergedConfig.extractDocuments && this.documentParser && isOfficeFile(fileInfo.path)) {
173
- return this.parseOfficeDocument(fileInfo.path, fileInfo.size);
183
+ if (mergedConfig.extractDocuments && this.documentParser && isOfficeFile(fileInfo.absolutePath)) {
184
+ return this.parseOfficeDocument(fileInfo.absolutePath, fileInfo.relativePath, fileInfo.size);
174
185
  }
175
186
  // Read file content using FileSystemService
176
187
  // This handles binary detection, PDF extraction, encoding, etc.
177
- const fileContent = await this.fileSystemService.readFile(fileInfo.path, {
188
+ const fileContent = await this.fileSystemService.readFile(fileInfo.absolutePath, {
178
189
  limit: mergedConfig.maxLinesPerFile,
179
190
  });
180
191
  // Check if it's a PDF with extracted text
@@ -182,9 +193,9 @@ export class FolderPackService {
182
193
  return {
183
194
  file: {
184
195
  content: fileContent.content,
185
- fileType: isPdf ? 'pdf' : this.detectFileType(fileInfo.path),
196
+ fileType: isPdf ? 'pdf' : this.detectFileType(fileInfo.relativePath),
186
197
  lineCount: fileContent.lines,
187
- path: fileInfo.path,
198
+ path: fileInfo.relativePath,
188
199
  size: fileInfo.size,
189
200
  truncated: fileContent.truncated,
190
201
  },
@@ -197,7 +208,7 @@ export class FolderPackService {
197
208
  return {
198
209
  skipped: {
199
210
  message: error instanceof Error ? error.message : String(error),
200
- path: fileInfo.path,
211
+ path: fileInfo.relativePath,
201
212
  reason: skipReason,
202
213
  },
203
214
  type: 'skipped',
@@ -329,12 +340,12 @@ export class FolderPackService {
329
340
  /**
330
341
  * Parse an Office document using the document parser.
331
342
  */
332
- async parseOfficeDocument(filePath, size) {
343
+ async parseOfficeDocument(filePath, outputPath, size) {
333
344
  if (!this.documentParser) {
334
345
  return {
335
346
  skipped: {
336
347
  message: 'Document parser not available',
337
- path: filePath,
348
+ path: outputPath,
338
349
  reason: 'read-error',
339
350
  },
340
351
  type: 'skipped',
@@ -351,7 +362,7 @@ export class FolderPackService {
351
362
  content: result.content,
352
363
  fileType: 'document',
353
364
  lineCount: lines.length,
354
- path: filePath,
365
+ path: outputPath,
355
366
  size,
356
367
  truncated: false,
357
368
  },
@@ -362,11 +373,14 @@ export class FolderPackService {
362
373
  return {
363
374
  skipped: {
364
375
  message: error instanceof Error ? error.message : String(error),
365
- path: filePath,
376
+ path: outputPath,
366
377
  reason: 'read-error',
367
378
  },
368
379
  type: 'skipped',
369
380
  };
370
381
  }
371
382
  }
383
+ toRelativePackPath(rootPath, filePath) {
384
+ return path.relative(rootPath, filePath).replaceAll('\\', '/');
385
+ }
372
386
  }
@@ -31,6 +31,18 @@ export function createChatGptOAuthFetch() {
31
31
  catch {
32
32
  return globalThis.fetch(input, init);
33
33
  }
34
+ // The AI SDK sends systemPrompt as input[0] with role "system" or "developer".
35
+ // The ChatGPT OAuth Responses endpoint expects it in top-level `instructions`.
36
+ if (!body.instructions && Array.isArray(body.input) && body.input.length > 0) {
37
+ const first = body.input[0];
38
+ if (typeof first === 'object' && first !== null) {
39
+ const record = first;
40
+ if ((record.role === 'system' || record.role === 'developer') && typeof record.content === 'string') {
41
+ body.instructions = record.content;
42
+ body.input.splice(0, 1);
43
+ }
44
+ }
45
+ }
34
46
  if (!body.instructions) {
35
47
  body.instructions = '';
36
48
  }
@@ -0,0 +1,7 @@
1
+ import type { GenerateContentRequest, IContentGenerator } from '../../core/interfaces/i-content-generator.js';
2
+ /**
3
+ * Consume generateContentStream() and return accumulated text.
4
+ * Used instead of generateContent() because the ChatGPT OAuth Codex
5
+ * endpoint requires stream: true in all requests.
6
+ */
7
+ export declare function streamToText(generator: IContentGenerator, request: GenerateContentRequest): Promise<string>;
@@ -0,0 +1,14 @@
1
+ /**
2
+ * Consume generateContentStream() and return accumulated text.
3
+ * Used instead of generateContent() because the ChatGPT OAuth Codex
4
+ * endpoint requires stream: true in all requests.
5
+ */
6
+ export async function streamToText(generator, request) {
7
+ const chunks = [];
8
+ for await (const chunk of generator.generateContentStream(request)) {
9
+ if (chunk.content) {
10
+ chunks.push(chunk.content);
11
+ }
12
+ }
13
+ return chunks.join('');
14
+ }
@@ -0,0 +1,22 @@
1
+ import type { IContentGenerator } from '../../core/interfaces/i-content-generator.js';
2
+ /**
3
+ * Result from abstract generation.
4
+ */
5
+ export interface AbstractGenerateResult {
6
+ /** L0: one-line summary (~80 tokens) */
7
+ abstractContent: string;
8
+ /** L1: key points + structure (~1500 tokens) */
9
+ overviewContent: string;
10
+ }
11
+ /**
12
+ * Generate L0 abstract and L1 overview for a knowledge file.
13
+ *
14
+ * Makes two parallel LLM calls at temperature=0:
15
+ * 1. L0 .abstract.md — one-line summary (~80 tokens)
16
+ * 2. L1 .overview.md — key points + structure (~1500 tokens)
17
+ *
18
+ * @param fullContent - Full markdown content of the knowledge file
19
+ * @param generator - LLM content generator
20
+ * @returns Abstract and overview content strings
21
+ */
22
+ export declare function generateFileAbstracts(fullContent: string, generator: IContentGenerator): Promise<AbstractGenerateResult>;
@@ -0,0 +1,67 @@
1
+ import { randomUUID } from 'node:crypto';
2
+ import { streamToText } from '../llm/stream-to-text.js';
3
+ const ABSTRACT_SYSTEM_PROMPT = `You are a technical documentation assistant.
4
+ Your job is to produce precise, factual summaries of knowledge documents.
5
+ Output only the requested content — no preamble, no commentary.`;
6
+ const OVERVIEW_SYSTEM_PROMPT = `You are a technical documentation assistant.
7
+ Your job is to produce structured overviews of knowledge documents.
8
+ Preserve factual accuracy, surface important entities and decisions, and format the result in concise markdown.`;
9
+ function buildAbstractPrompt(content) {
10
+ return `Produce a ONE-LINE summary (max 80 tokens) of the following knowledge document.
11
+ The line must be a complete sentence that captures the core topic and key insight.
12
+ Output only the single line — nothing else.
13
+
14
+ <document>
15
+ ${content}
16
+ </document>`;
17
+ }
18
+ function buildOverviewPrompt(content) {
19
+ return `Produce a structured overview of the following knowledge document.
20
+ Include:
21
+ - Key points (3-7 bullet points)
22
+ - Structure / sections summary
23
+ - Any notable entities, patterns, or decisions mentioned
24
+
25
+ Keep it under 1500 tokens. Use markdown formatting.
26
+ Output only the overview — no preamble.
27
+
28
+ <document>
29
+ ${content}
30
+ </document>`;
31
+ }
32
+ /** Truncate content before embedding in LLM prompts to avoid exceeding model context windows during bulk ingest. */
33
+ const MAX_ABSTRACT_CONTENT_CHARS = 20_000;
34
+ /**
35
+ * Generate L0 abstract and L1 overview for a knowledge file.
36
+ *
37
+ * Makes two parallel LLM calls at temperature=0:
38
+ * 1. L0 .abstract.md — one-line summary (~80 tokens)
39
+ * 2. L1 .overview.md — key points + structure (~1500 tokens)
40
+ *
41
+ * @param fullContent - Full markdown content of the knowledge file
42
+ * @param generator - LLM content generator
43
+ * @returns Abstract and overview content strings
44
+ */
45
+ export async function generateFileAbstracts(fullContent, generator) {
46
+ const truncated = fullContent.slice(0, MAX_ABSTRACT_CONTENT_CHARS);
47
+ const [abstractText, overviewText] = await Promise.all([
48
+ streamToText(generator, {
49
+ config: { maxTokens: 150, temperature: 0 },
50
+ contents: [{ content: buildAbstractPrompt(truncated), role: 'user' }],
51
+ model: 'default',
52
+ systemPrompt: ABSTRACT_SYSTEM_PROMPT,
53
+ taskId: randomUUID(),
54
+ }),
55
+ streamToText(generator, {
56
+ config: { maxTokens: 2000, temperature: 0 },
57
+ contents: [{ content: buildOverviewPrompt(truncated), role: 'user' }],
58
+ model: 'default',
59
+ systemPrompt: OVERVIEW_SYSTEM_PROMPT,
60
+ taskId: randomUUID(),
61
+ }),
62
+ ]);
63
+ return {
64
+ abstractContent: abstractText.trim(),
65
+ overviewContent: overviewText.trim(),
66
+ };
67
+ }
@@ -0,0 +1,67 @@
1
+ import type { IContentGenerator } from '../../core/interfaces/i-content-generator.js';
2
+ /**
3
+ * Observable status of the abstract generation queue.
4
+ */
5
+ export interface AbstractQueueStatus {
6
+ failed: number;
7
+ pending: number;
8
+ processed: number;
9
+ processing: boolean;
10
+ }
11
+ /**
12
+ * Background queue for generating L0/L1 abstract files (.abstract.md, .overview.md).
13
+ *
14
+ * - Generator is injected lazily via setGenerator() (mirrors rebindMapTools pattern)
15
+ * - Items arriving before setGenerator() are buffered and processed once generator is set
16
+ * - Writes status to <projectRoot>/.brv/_queue_status.json after each state transition
17
+ * - Retries up to maxAttempts with exponential backoff (500ms base)
18
+ * - drain() waits for all pending/processing items to complete (for graceful shutdown)
19
+ */
20
+ export declare class AbstractGenerationQueue {
21
+ private readonly projectRoot;
22
+ private readonly maxAttempts;
23
+ private drainResolvers;
24
+ private failed;
25
+ private generator;
26
+ private onBeforeProcess?;
27
+ private pending;
28
+ private processed;
29
+ private processing;
30
+ /** Number of items currently in retry backoff (removed from pending but not yet re-enqueued). */
31
+ private retrying;
32
+ private statusDirCreated;
33
+ private statusWriteFailed;
34
+ private statusWritePromise;
35
+ constructor(projectRoot: string, maxAttempts?: number);
36
+ /**
37
+ * Wait for all pending items to finish processing (graceful shutdown).
38
+ * Includes items currently in retry backoff so drain() does not resolve prematurely.
39
+ */
40
+ drain(): Promise<void>;
41
+ /**
42
+ * Add a file to the abstract generation queue.
43
+ */
44
+ enqueue(item: {
45
+ contextPath: string;
46
+ fullContent: string;
47
+ }): void;
48
+ /**
49
+ * Return current queue status snapshot.
50
+ */
51
+ getStatus(): AbstractQueueStatus;
52
+ /**
53
+ * Set a callback that runs before each item is processed.
54
+ * Used to refresh OAuth tokens before LLM calls.
55
+ */
56
+ setBeforeProcess(fn: () => Promise<void>): void;
57
+ /**
58
+ * Inject the LLM generator. Triggers processing of any buffered items.
59
+ */
60
+ setGenerator(generator: IContentGenerator): void;
61
+ private isIdle;
62
+ private processNext;
63
+ private queueStatusWrite;
64
+ private resolveDrainersIfIdle;
65
+ private scheduleNext;
66
+ private writeStatusFile;
67
+ }
@@ -0,0 +1,218 @@
1
+ import { appendFileSync } from 'node:fs';
2
+ import { mkdir, writeFile } from 'node:fs/promises';
3
+ import { join } from 'node:path';
4
+ import { generateFileAbstracts } from './abstract-generator.js';
5
+ const QUEUE_TRACE_ENABLED = process.env.BRV_QUEUE_TRACE === '1';
6
+ const LOG_PATH = process.env.BRV_SESSION_LOG;
7
+ function queueLog(message) {
8
+ if (!QUEUE_TRACE_ENABLED || !LOG_PATH)
9
+ return;
10
+ try {
11
+ appendFileSync(LOG_PATH, `${new Date().toISOString()} [abstract-queue] ${message}\n`);
12
+ }
13
+ catch {
14
+ // ignore — tracing must never block queue progress
15
+ }
16
+ }
17
+ /**
18
+ * Background queue for generating L0/L1 abstract files (.abstract.md, .overview.md).
19
+ *
20
+ * - Generator is injected lazily via setGenerator() (mirrors rebindMapTools pattern)
21
+ * - Items arriving before setGenerator() are buffered and processed once generator is set
22
+ * - Writes status to <projectRoot>/.brv/_queue_status.json after each state transition
23
+ * - Retries up to maxAttempts with exponential backoff (500ms base)
24
+ * - drain() waits for all pending/processing items to complete (for graceful shutdown)
25
+ */
26
+ export class AbstractGenerationQueue {
27
+ projectRoot;
28
+ maxAttempts;
29
+ drainResolvers = [];
30
+ failed = 0;
31
+ generator;
32
+ onBeforeProcess;
33
+ pending = [];
34
+ processed = 0;
35
+ processing = false;
36
+ /** Number of items currently in retry backoff (removed from pending but not yet re-enqueued). */
37
+ retrying = 0;
38
+ statusDirCreated = false;
39
+ statusWriteFailed = false;
40
+ statusWritePromise = Promise.resolve();
41
+ constructor(projectRoot, maxAttempts = 3) {
42
+ this.projectRoot = projectRoot;
43
+ this.maxAttempts = maxAttempts;
44
+ }
45
+ /**
46
+ * Wait for all pending items to finish processing (graceful shutdown).
47
+ * Includes items currently in retry backoff so drain() does not resolve prematurely.
48
+ */
49
+ async drain() {
50
+ queueLog(`drain:start idle=${this.isIdle()} pending=${this.pending.length} retrying=${this.retrying} processing=${this.processing}`);
51
+ if (this.isIdle()) {
52
+ await this.statusWritePromise.catch(() => { });
53
+ queueLog('drain:resolved-immediate');
54
+ return;
55
+ }
56
+ await new Promise((resolve) => {
57
+ this.drainResolvers.push(resolve);
58
+ this.resolveDrainersIfIdle();
59
+ });
60
+ queueLog('drain:resolved-deferred');
61
+ }
62
+ /**
63
+ * Add a file to the abstract generation queue.
64
+ */
65
+ enqueue(item) {
66
+ // Guard against paths that must never trigger abstract generation:
67
+ // - derived artifacts (.abstract.md, .overview.md) — would produce .abstract.abstract.md
68
+ // - summary index files (_index.md) — domain/topic summaries, not knowledge nodes
69
+ // - hierarchy scaffolding (context.md) — helper files, not leaf knowledge entries
70
+ const fileName = item.contextPath.split('/').at(-1) ?? item.contextPath;
71
+ if (fileName === 'context.md' ||
72
+ fileName === '_index.md' ||
73
+ item.contextPath.endsWith('.abstract.md') ||
74
+ item.contextPath.endsWith('.overview.md')) {
75
+ return;
76
+ }
77
+ this.pending.push({ attempts: 0, contextPath: item.contextPath, fullContent: item.fullContent });
78
+ queueLog(`enqueue path=${item.contextPath} pending=${this.pending.length} retrying=${this.retrying} processing=${this.processing}`);
79
+ this.queueStatusWrite();
80
+ this.scheduleNext();
81
+ }
82
+ /**
83
+ * Return current queue status snapshot.
84
+ */
85
+ getStatus() {
86
+ return {
87
+ failed: this.failed,
88
+ // Items in retry backoff are still pending work — include them so the status
89
+ // does not falsely report the queue as idle during backoff windows.
90
+ pending: this.pending.length + this.retrying,
91
+ processed: this.processed,
92
+ processing: this.processing,
93
+ };
94
+ }
95
+ /**
96
+ * Set a callback that runs before each item is processed.
97
+ * Used to refresh OAuth tokens before LLM calls.
98
+ */
99
+ setBeforeProcess(fn) {
100
+ this.onBeforeProcess = fn;
101
+ }
102
+ /**
103
+ * Inject the LLM generator. Triggers processing of any buffered items.
104
+ */
105
+ setGenerator(generator) {
106
+ this.generator = generator;
107
+ this.scheduleNext();
108
+ }
109
+ isIdle() {
110
+ return this.pending.length === 0 && !this.processing && this.retrying === 0;
111
+ }
112
+ async processNext() {
113
+ if (!this.generator || this.processing || this.pending.length === 0) {
114
+ this.resolveDrainersIfIdle();
115
+ return;
116
+ }
117
+ this.processing = true;
118
+ this.queueStatusWrite();
119
+ const item = this.pending.shift();
120
+ queueLog(`process:start path=${item.contextPath} remaining=${this.pending.length} retrying=${this.retrying}`);
121
+ try {
122
+ // Refresh credentials before each generation (OAuth tokens may expire)
123
+ try {
124
+ await this.onBeforeProcess?.();
125
+ }
126
+ catch (error) {
127
+ const msg = error instanceof Error ? error.message : String(error);
128
+ console.debug(`[AbstractQueue] token refresh failed, proceeding with existing generator: ${msg}`);
129
+ }
130
+ const { abstractContent, overviewContent } = await generateFileAbstracts(item.fullContent, this.generator);
131
+ // Derive sibling paths: replace .md with .abstract.md and .overview.md
132
+ const abstractPath = item.contextPath.replace(/\.md$/, '.abstract.md');
133
+ const overviewPath = item.contextPath.replace(/\.md$/, '.overview.md');
134
+ await Promise.all([
135
+ writeFile(abstractPath, abstractContent, 'utf8'),
136
+ writeFile(overviewPath, overviewContent, 'utf8'),
137
+ ]);
138
+ this.processed++;
139
+ queueLog(`process:success path=${item.contextPath} processed=${this.processed}`);
140
+ }
141
+ catch (error) {
142
+ const msg = error instanceof Error ? error.message : String(error);
143
+ console.debug(`[AbstractQueue] ${item.contextPath} attempt ${item.attempts + 1}/${this.maxAttempts}: ${msg}`);
144
+ item.attempts++;
145
+ if (item.attempts < this.maxAttempts) {
146
+ // Exponential backoff: 500ms, 1000ms, 2000ms, ...
147
+ const delay = 500 * 2 ** (item.attempts - 1);
148
+ this.retrying++;
149
+ this.queueStatusWrite();
150
+ setTimeout(() => {
151
+ this.retrying--;
152
+ this.pending.unshift(item);
153
+ queueLog(`process:retry-requeue path=${item.contextPath} pending=${this.pending.length} retrying=${this.retrying}`);
154
+ this.queueStatusWrite();
155
+ this.scheduleNext();
156
+ }, delay);
157
+ }
158
+ else {
159
+ this.failed++;
160
+ queueLog(`process:failed path=${item.contextPath} failed=${this.failed}`);
161
+ }
162
+ }
163
+ finally {
164
+ this.processing = false;
165
+ queueLog(`process:finally path=${item.contextPath} pending=${this.pending.length} retrying=${this.retrying} processed=${this.processed} failed=${this.failed}`);
166
+ this.queueStatusWrite();
167
+ }
168
+ this.scheduleNext();
169
+ this.resolveDrainersIfIdle();
170
+ }
171
+ queueStatusWrite() {
172
+ this.statusWritePromise = this.statusWritePromise
173
+ .catch(() => { })
174
+ .then(async () => this.writeStatusFile());
175
+ }
176
+ resolveDrainersIfIdle() {
177
+ if (!this.isIdle() || this.drainResolvers.length === 0) {
178
+ return;
179
+ }
180
+ queueLog(`drain:idle pending=${this.pending.length} retrying=${this.retrying} processed=${this.processed} failed=${this.failed}`);
181
+ const resolvers = this.drainResolvers.splice(0);
182
+ const settledStatusWrite = this.statusWritePromise.catch(() => { });
183
+ for (const resolve of resolvers) {
184
+ settledStatusWrite.then(() => resolve()).catch(() => { });
185
+ }
186
+ }
187
+ scheduleNext() {
188
+ if (!this.generator || this.processing || this.pending.length === 0) {
189
+ this.resolveDrainersIfIdle();
190
+ return;
191
+ }
192
+ // eslint-disable-next-line no-void
193
+ setImmediate(() => { void this.processNext(); });
194
+ }
195
+ async writeStatusFile() {
196
+ const statusPath = join(this.projectRoot, '.brv', '_queue_status.json');
197
+ try {
198
+ if (!this.statusDirCreated) {
199
+ await mkdir(join(this.projectRoot, '.brv'), { recursive: true });
200
+ this.statusDirCreated = true;
201
+ }
202
+ await writeFile(statusPath, JSON.stringify(this.getStatus()), 'utf8');
203
+ this.statusWriteFailed = false;
204
+ }
205
+ catch (error) {
206
+ const errorCode = typeof error === 'object' && error !== null && 'code' in error
207
+ ? error.code
208
+ : undefined;
209
+ if (errorCode === 'ENOENT') {
210
+ return;
211
+ }
212
+ if (!this.statusWriteFailed) {
213
+ this.statusWriteFailed = true;
214
+ console.debug(`[AbstractGenerationQueue] Failed to write queue status: ${error instanceof Error ? error.message : String(error)}`);
215
+ }
216
+ }
217
+ }
218
+ }
@@ -0,0 +1,44 @@
1
+ import type { Memory } from '../../core/domain/memory/types.js';
2
+ import type { IContentGenerator } from '../../core/interfaces/i-content-generator.js';
3
+ /**
4
+ * A draft memory extracted from a session, before deduplication.
5
+ */
6
+ export interface DraftMemory {
7
+ category: string;
8
+ content: string;
9
+ tags?: string[];
10
+ }
11
+ /**
12
+ * Deduplication decision for a single draft memory.
13
+ */
14
+ export type DeduplicationAction = {
15
+ action: 'CREATE';
16
+ memory: DraftMemory;
17
+ } | {
18
+ action: 'MERGE';
19
+ memory: DraftMemory;
20
+ mergedContent: string;
21
+ targetId: string;
22
+ } | {
23
+ action: 'SKIP';
24
+ memory: DraftMemory;
25
+ };
26
+ /**
27
+ * LLM-based deduplicator for agent-extracted memories.
28
+ *
29
+ * For each draft, checks against existing memories via an LLM call.
30
+ * DECISIONS category drafts always result in CREATE (immutable log).
31
+ */
32
+ export declare class MemoryDeduplicator {
33
+ private readonly generator;
34
+ constructor(generator: IContentGenerator);
35
+ /**
36
+ * Deduplicate a list of draft memories against existing stored memories.
37
+ *
38
+ * @param drafts - Draft memories to check
39
+ * @param existing - Existing memories to compare against
40
+ * @returns Deduplication action for each draft
41
+ */
42
+ deduplicate(drafts: DraftMemory[], existing: Memory[]): Promise<DeduplicationAction[]>;
43
+ private deduplicateSingle;
44
+ }