byterover-cli 2.0.0 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (181) hide show
  1. package/README.md +6 -81
  2. package/dist/agent/core/domain/llm/index.d.ts +1 -1
  3. package/dist/agent/core/domain/llm/index.js +1 -1
  4. package/dist/agent/core/domain/llm/registry.d.ts +8 -0
  5. package/dist/agent/core/domain/llm/registry.js +34 -0
  6. package/dist/agent/core/domain/sandbox/types.d.ts +2 -0
  7. package/dist/agent/core/domain/tools/constants.d.ts +3 -0
  8. package/dist/agent/core/domain/tools/constants.js +3 -0
  9. package/dist/agent/core/interfaces/cipher-services.d.ts +2 -4
  10. package/dist/agent/core/interfaces/i-cipher-agent.d.ts +9 -1
  11. package/dist/agent/core/interfaces/i-sandbox-service.d.ts +8 -0
  12. package/dist/agent/core/interfaces/i-tool-provider.d.ts +10 -0
  13. package/dist/agent/core/interfaces/i-tool-scheduler.d.ts +9 -0
  14. package/dist/agent/infra/agent/agent-schemas.d.ts +0 -9
  15. package/dist/agent/infra/agent/agent-schemas.js +0 -3
  16. package/dist/agent/infra/agent/cipher-agent.d.ts +25 -1
  17. package/dist/agent/infra/agent/cipher-agent.js +138 -11
  18. package/dist/agent/infra/agent/provider-update-config.d.ts +0 -2
  19. package/dist/agent/infra/agent/service-initializer.d.ts +2 -6
  20. package/dist/agent/infra/agent/service-initializer.js +45 -38
  21. package/dist/agent/infra/blob/blob-storage-factory.d.ts +2 -2
  22. package/dist/agent/infra/blob/blob-storage-factory.js +4 -4
  23. package/dist/agent/infra/blob/file-blob-storage.d.ts +96 -0
  24. package/dist/agent/infra/blob/file-blob-storage.js +454 -0
  25. package/dist/agent/infra/blob/index.d.ts +2 -3
  26. package/dist/agent/infra/blob/index.js +4 -6
  27. package/dist/agent/infra/llm/agent-llm-service.d.ts +3 -0
  28. package/dist/agent/infra/llm/agent-llm-service.js +34 -52
  29. package/dist/agent/infra/llm/context/compression/compression-helpers.d.ts +35 -0
  30. package/dist/agent/infra/llm/context/compression/compression-helpers.js +124 -0
  31. package/dist/agent/infra/llm/context/compression/escalated-compression.d.ts +62 -0
  32. package/dist/agent/infra/llm/context/compression/escalated-compression.js +144 -0
  33. package/dist/agent/infra/llm/context/compression/index.d.ts +3 -0
  34. package/dist/agent/infra/llm/context/compression/index.js +3 -0
  35. package/dist/agent/infra/llm/context/compression/reactive-overflow.d.ts +0 -27
  36. package/dist/agent/infra/llm/context/compression/reactive-overflow.js +5 -122
  37. package/dist/agent/infra/llm/context/context-manager.d.ts +20 -1
  38. package/dist/agent/infra/llm/context/context-manager.js +37 -7
  39. package/dist/agent/infra/llm/providers/index.js +0 -2
  40. package/dist/agent/infra/llm/providers/types.d.ts +1 -5
  41. package/dist/agent/infra/map/agentic-map-service.d.ts +97 -0
  42. package/dist/agent/infra/map/agentic-map-service.js +309 -0
  43. package/dist/agent/infra/map/context-tree-store.d.ts +94 -0
  44. package/dist/agent/infra/map/context-tree-store.js +278 -0
  45. package/dist/agent/infra/map/index.d.ts +4 -0
  46. package/dist/agent/infra/map/index.js +4 -0
  47. package/dist/agent/infra/map/llm-map-memory.d.ts +59 -0
  48. package/dist/agent/infra/map/llm-map-memory.js +187 -0
  49. package/dist/agent/infra/map/llm-map-service.d.ts +36 -0
  50. package/dist/agent/infra/map/llm-map-service.js +118 -0
  51. package/dist/agent/infra/map/map-shared.d.ts +140 -0
  52. package/dist/agent/infra/map/map-shared.js +325 -0
  53. package/dist/agent/infra/map/worker-pool.d.ts +45 -0
  54. package/dist/agent/infra/map/worker-pool.js +73 -0
  55. package/dist/agent/infra/sandbox/curation-helpers.d.ts +62 -0
  56. package/dist/agent/infra/sandbox/curation-helpers.js +219 -0
  57. package/dist/agent/infra/sandbox/sandbox-service.d.ts +12 -0
  58. package/dist/agent/infra/sandbox/sandbox-service.js +39 -7
  59. package/dist/agent/infra/sandbox/tools-sdk.d.ts +48 -1
  60. package/dist/agent/infra/sandbox/tools-sdk.js +52 -1
  61. package/dist/agent/infra/session/session-manager.d.ts +8 -1
  62. package/dist/agent/infra/session/session-manager.js +24 -4
  63. package/dist/agent/infra/storage/file-key-storage.d.ts +142 -0
  64. package/dist/agent/infra/storage/file-key-storage.js +572 -0
  65. package/dist/agent/infra/storage/granular-history-storage.d.ts +1 -1
  66. package/dist/agent/infra/storage/granular-history-storage.js +1 -1
  67. package/dist/agent/infra/system-prompt/contributors/context-tree-structure-contributor.d.ts +4 -0
  68. package/dist/agent/infra/system-prompt/contributors/context-tree-structure-contributor.js +42 -14
  69. package/dist/agent/infra/system-prompt/contributors/map-selection-contributor.d.ts +16 -0
  70. package/dist/agent/infra/system-prompt/contributors/map-selection-contributor.js +47 -0
  71. package/dist/agent/infra/tools/core-tool-scheduler.js +3 -1
  72. package/dist/agent/infra/tools/implementations/agentic-map-tool.d.ts +35 -0
  73. package/dist/agent/infra/tools/implementations/agentic-map-tool.js +156 -0
  74. package/dist/agent/infra/tools/implementations/code-exec-tool.js +1 -0
  75. package/dist/agent/infra/tools/implementations/curate-tool.d.ts +9 -9
  76. package/dist/agent/infra/tools/implementations/expand-knowledge-tool.d.ts +18 -0
  77. package/dist/agent/infra/tools/implementations/expand-knowledge-tool.js +43 -0
  78. package/dist/agent/infra/tools/implementations/llm-map-tool.d.ts +24 -0
  79. package/dist/agent/infra/tools/implementations/llm-map-tool.js +87 -0
  80. package/dist/agent/infra/tools/implementations/memory-symbol-tree.d.ts +28 -1
  81. package/dist/agent/infra/tools/implementations/memory-symbol-tree.js +27 -3
  82. package/dist/agent/infra/tools/implementations/search-knowledge-service.d.ts +1 -0
  83. package/dist/agent/infra/tools/implementations/search-knowledge-service.js +83 -12
  84. package/dist/agent/infra/tools/implementations/search-knowledge-tool.js +2 -2
  85. package/dist/agent/infra/tools/tool-manager.js +6 -0
  86. package/dist/agent/infra/tools/tool-provider.d.ts +12 -0
  87. package/dist/agent/infra/tools/tool-provider.js +78 -0
  88. package/dist/agent/infra/tools/tool-registry.d.ts +14 -0
  89. package/dist/agent/infra/tools/tool-registry.js +32 -0
  90. package/dist/agent/resources/prompts/system-prompt.yml +48 -74
  91. package/dist/agent/resources/tools/expand_knowledge.txt +20 -0
  92. package/dist/oclif/commands/curate/index.js +1 -2
  93. package/dist/oclif/commands/main.js +1 -0
  94. package/dist/oclif/commands/providers/connect.d.ts +1 -3
  95. package/dist/oclif/commands/providers/connect.js +7 -29
  96. package/dist/oclif/commands/query.js +1 -2
  97. package/dist/server/constants.d.ts +7 -0
  98. package/dist/server/constants.js +8 -0
  99. package/dist/server/core/domain/entities/provider-registry.js +1 -15
  100. package/dist/server/core/domain/knowledge/memory-scoring.js +1 -1
  101. package/dist/server/core/domain/knowledge/summary-types.d.ts +126 -0
  102. package/dist/server/core/domain/knowledge/summary-types.js +7 -0
  103. package/dist/server/core/domain/transport/schemas.d.ts +0 -4
  104. package/dist/server/core/interfaces/context-tree/i-context-tree-archive-service.d.ts +30 -0
  105. package/dist/server/core/interfaces/context-tree/i-context-tree-archive-service.js +1 -0
  106. package/dist/server/core/interfaces/context-tree/i-context-tree-manifest-service.d.ts +30 -0
  107. package/dist/server/core/interfaces/context-tree/i-context-tree-manifest-service.js +1 -0
  108. package/dist/server/core/interfaces/context-tree/i-context-tree-summary-service.d.ts +29 -0
  109. package/dist/server/core/interfaces/context-tree/i-context-tree-summary-service.js +1 -0
  110. package/dist/server/infra/cogit/context-tree-to-push-context-mapper.js +10 -3
  111. package/dist/server/infra/connectors/skill/skill-connector.d.ts +4 -0
  112. package/dist/server/infra/connectors/skill/skill-connector.js +4 -0
  113. package/dist/server/infra/context-tree/children-hash.d.ts +20 -0
  114. package/dist/server/infra/context-tree/children-hash.js +22 -0
  115. package/dist/server/infra/context-tree/derived-artifact.d.ts +28 -0
  116. package/dist/server/infra/context-tree/derived-artifact.js +48 -0
  117. package/dist/server/infra/context-tree/file-context-tree-archive-service.d.ts +37 -0
  118. package/dist/server/infra/context-tree/file-context-tree-archive-service.js +219 -0
  119. package/dist/server/infra/context-tree/file-context-tree-manifest-service.d.ts +50 -0
  120. package/dist/server/infra/context-tree/file-context-tree-manifest-service.js +278 -0
  121. package/dist/server/infra/context-tree/file-context-tree-merger.js +4 -0
  122. package/dist/server/infra/context-tree/file-context-tree-snapshot-service.js +12 -4
  123. package/dist/server/infra/context-tree/file-context-tree-summary-service.d.ts +44 -0
  124. package/dist/server/infra/context-tree/file-context-tree-summary-service.js +313 -0
  125. package/dist/server/infra/context-tree/file-context-tree-writer-service.js +5 -0
  126. package/dist/server/infra/context-tree/prompts/summary-generation.d.ts +22 -0
  127. package/dist/server/infra/context-tree/prompts/summary-generation.js +45 -0
  128. package/dist/server/infra/context-tree/snapshot-diff.d.ts +19 -0
  129. package/dist/server/infra/context-tree/snapshot-diff.js +39 -0
  130. package/dist/server/infra/context-tree/summary-frontmatter.d.ts +24 -0
  131. package/dist/server/infra/context-tree/summary-frontmatter.js +111 -0
  132. package/dist/server/infra/daemon/agent-process.js +2 -14
  133. package/dist/server/infra/executor/curate-executor.d.ts +1 -0
  134. package/dist/server/infra/executor/curate-executor.js +82 -34
  135. package/dist/server/infra/executor/folder-pack-executor.js +1 -1
  136. package/dist/server/infra/executor/pre-compaction/compaction-escalation.d.ts +6 -0
  137. package/dist/server/infra/executor/pre-compaction/compaction-escalation.js +6 -0
  138. package/dist/server/infra/executor/pre-compaction/index.d.ts +3 -0
  139. package/dist/server/infra/executor/pre-compaction/index.js +1 -0
  140. package/dist/server/infra/executor/pre-compaction/pre-compaction-service.d.ts +59 -0
  141. package/dist/server/infra/executor/pre-compaction/pre-compaction-service.js +124 -0
  142. package/dist/server/infra/executor/pre-compaction/prompts.d.ts +24 -0
  143. package/dist/server/infra/executor/pre-compaction/prompts.js +47 -0
  144. package/dist/server/infra/executor/query-executor.d.ts +3 -0
  145. package/dist/server/infra/executor/query-executor.js +39 -4
  146. package/dist/server/infra/http/authenticated-http-client.js +4 -0
  147. package/dist/server/infra/http/provider-model-fetcher-registry.js +1 -5
  148. package/dist/server/infra/http/provider-model-fetchers.d.ts +0 -14
  149. package/dist/server/infra/http/provider-model-fetchers.js +0 -132
  150. package/dist/server/infra/provider/provider-config-resolver.js +0 -55
  151. package/dist/server/utils/curate-result-parser.d.ts +4 -4
  152. package/dist/shared/constants/curation.d.ts +6 -0
  153. package/dist/shared/constants/curation.js +6 -0
  154. package/dist/shared/utils/escalation-utils.d.ts +59 -0
  155. package/dist/shared/utils/escalation-utils.js +141 -0
  156. package/dist/tui/components/command-input.js +1 -1
  157. package/dist/tui/components/inline-prompts/inline-confirm.js +6 -1
  158. package/dist/tui/features/commands/definitions/exit.d.ts +2 -0
  159. package/dist/tui/features/commands/definitions/exit.js +9 -0
  160. package/dist/tui/features/commands/definitions/index.js +3 -0
  161. package/dist/tui/features/exit/components/exit-flow.d.ts +10 -0
  162. package/dist/tui/features/exit/components/exit-flow.js +19 -0
  163. package/dist/tui/features/provider/components/provider-flow.js +1 -21
  164. package/oclif.manifest.json +100 -109
  165. package/package.json +11 -4
  166. package/dist/agent/infra/blob/migrations.d.ts +0 -63
  167. package/dist/agent/infra/blob/migrations.js +0 -148
  168. package/dist/agent/infra/blob/sqlite-blob-storage.d.ts +0 -82
  169. package/dist/agent/infra/blob/sqlite-blob-storage.js +0 -307
  170. package/dist/agent/infra/llm/providers/google-vertex.d.ts +0 -15
  171. package/dist/agent/infra/llm/providers/google-vertex.js +0 -36
  172. package/dist/agent/infra/storage/blob-history-storage.d.ts +0 -81
  173. package/dist/agent/infra/storage/blob-history-storage.js +0 -193
  174. package/dist/agent/infra/storage/dual-format-history-storage.d.ts +0 -83
  175. package/dist/agent/infra/storage/dual-format-history-storage.js +0 -165
  176. package/dist/agent/infra/storage/sqlite-key-storage.d.ts +0 -113
  177. package/dist/agent/infra/storage/sqlite-key-storage.js +0 -438
  178. package/dist/server/infra/provider/vertex-ai-utils.d.ts +0 -10
  179. package/dist/server/infra/provider/vertex-ai-utils.js +0 -28
  180. package/dist/tui/features/provider/components/credential-path-dialog.d.ts +0 -30
  181. package/dist/tui/features/provider/components/credential-path-dialog.js +0 -85
@@ -0,0 +1,118 @@
1
+ import { writeFile } from 'node:fs/promises';
2
+ import { buildRetryMessage, buildUserMessage, callLlm, parseJsonlFile, resolveAndValidatePath, validateAgainstSchema, withTimeout, } from './map-shared.js';
3
+ import { runMapWorkerPool } from './worker-pool.js';
4
+ // ── LLM-Map Service ──────────────────────────────────────────────────────────
5
+ /**
6
+ * Execute an LLM-Map: parallel, stateless LLM calls over a JSONL file.
7
+ *
8
+ * For each item (line), makes a single LLM API call (no tools, no file I/O)
9
+ * that must return one JSON value conforming to the provided output schema.
10
+ * If validation fails, the system retries with the error and prior response.
11
+ *
12
+ * Ported from VoltCode's llm-map.ts, adapted for byterover-cli:
13
+ * - Uses IContentGenerator instead of AI SDK's generateText()
14
+ * - Uses in-memory worker pool (no FileMapStore / PostgreSQL)
15
+ * - Runs in-process (no SQS)
16
+ */
17
+ export async function executeLlmMap(options) {
18
+ const { abortSignal, generator, onProgress, params, taskId, workingDirectory, } = options;
19
+ const { concurrency = 8, input_path: inputPath, max_attempts: maxAttempts = 3, output_path: outputPath, output_schema: outputSchema, prompt, timeout_seconds: timeoutSeconds = 120, } = params;
20
+ // Resolve paths relative to working directory and validate they don't escape it
21
+ const resolvedInputPath = resolveAndValidatePath(workingDirectory, inputPath);
22
+ const resolvedOutputPath = resolveAndValidatePath(workingDirectory, outputPath);
23
+ // 1. Parse input JSONL
24
+ const items = await parseJsonlFile(resolvedInputPath);
25
+ if (items.length === 0) {
26
+ // Write empty output file
27
+ await writeFile(resolvedOutputPath, '', 'utf8');
28
+ return { failed: 0, mapId: 'empty', succeeded: 0, total: 0 };
29
+ }
30
+ // 2. Prepare run metadata
31
+ const runStartedAt = new Date().toISOString();
32
+ // 3. Define per-item processing function
33
+ async function processItem(itemIndex, item) {
34
+ const userMessage = buildUserMessage(prompt, 'pending', runStartedAt, itemIndex, item, outputSchema);
35
+ // Per-item timeout
36
+ const timeoutController = new AbortController();
37
+ const timeoutHandle = setTimeout(() => {
38
+ timeoutController.abort();
39
+ }, timeoutSeconds * 1000);
40
+ try {
41
+ let attemptsUsed = 1;
42
+ let lastResponse = '';
43
+ let lastError = '';
44
+ // Initial LLM call (stateless — no tool access)
45
+ // Wrap with timeout so a hung generateContent() doesn't block forever
46
+ const response = await withTimeout(callLlm(generator, userMessage, taskId, abortSignal), timeoutController.signal);
47
+ lastResponse = response.content;
48
+ // Validation loop with retry
49
+ while (true) {
50
+ // Try to parse JSON
51
+ let parsed;
52
+ try {
53
+ parsed = JSON.parse(lastResponse);
54
+ }
55
+ catch (error) {
56
+ lastError = `JSON parse error: ${error instanceof Error ? error.message : String(error)}`;
57
+ }
58
+ // Validate against schema
59
+ if (parsed !== undefined) {
60
+ const validation = validateAgainstSchema(parsed, outputSchema);
61
+ if (validation.valid) {
62
+ // Fail-open: store result in context tree if available
63
+ if (options.contextTreeStore) {
64
+ try {
65
+ options.contextTreeStore.store(itemIndex, JSON.stringify(parsed));
66
+ }
67
+ catch (storeError) {
68
+ options.logger?.warn('Context tree store failed', { error: String(storeError), itemIndex });
69
+ }
70
+ }
71
+ return parsed;
72
+ }
73
+ lastError = `Schema validation failed: ${validation.error}`;
74
+ }
75
+ // Check retry budget
76
+ if (attemptsUsed >= maxAttempts) {
77
+ throw new Error(`Failed after ${attemptsUsed} attempts. Last error: ${lastError}`);
78
+ }
79
+ // Check abort or timeout
80
+ if (abortSignal?.aborted || timeoutController.signal.aborted) {
81
+ throw new Error('Aborted or timed out');
82
+ }
83
+ // Retry with error context + prior response
84
+ attemptsUsed++;
85
+ const retryMessage = buildRetryMessage(userMessage, lastError, lastResponse);
86
+ // eslint-disable-next-line no-await-in-loop
87
+ const retryResponse = await withTimeout(callLlm(generator, retryMessage, taskId, abortSignal), timeoutController.signal);
88
+ lastResponse = retryResponse.content;
89
+ }
90
+ }
91
+ finally {
92
+ clearTimeout(timeoutHandle);
93
+ }
94
+ }
95
+ // 4. Run in-memory worker pool
96
+ const result = await runMapWorkerPool({
97
+ abortSignal,
98
+ concurrency,
99
+ items,
100
+ onProgress,
101
+ processItem,
102
+ });
103
+ // 5. Compact context tree and attach summaryHandle (fail-open)
104
+ if (options.contextTreeStore) {
105
+ try {
106
+ await options.contextTreeStore.compact();
107
+ result.summaryHandle = options.contextTreeStore.getSummaryHandle();
108
+ }
109
+ catch (compactError) {
110
+ options.logger?.warn('Context tree compaction failed', { error: String(compactError) });
111
+ }
112
+ }
113
+ // 6. Write output JSONL from in-memory results (sorted by index)
114
+ const sorted = [...result.results.entries()].sort(([a], [b]) => a - b);
115
+ const outputContent = sorted.map(([, r]) => JSON.stringify(r)).join('\n');
116
+ await writeFile(resolvedOutputPath, outputContent, 'utf8');
117
+ return result;
118
+ }
@@ -0,0 +1,140 @@
1
+ import { z } from 'zod';
2
+ import type { GenerateContentResponse, IContentGenerator } from '../../core/interfaces/i-content-generator.js';
3
+ /**
4
+ * Canonicalize a path by resolving symlinks.
5
+ * For existing paths, uses realpathSync.native().
6
+ * For non-existent paths (e.g., output files), walks up to the nearest
7
+ * existing ancestor directory, canonicalizes it, then reconstructs the
8
+ * remaining non-existent segments on top. This catches symlink escapes
9
+ * at any depth (e.g., workspace/symlink-to-outside/new-dir/file.jsonl).
10
+ */
11
+ /**
12
+ * Export the canonical path resolver for use in anti-cycle checks.
13
+ * Returns the real path via realpathSync.native(); falls back to walking
14
+ * up to the nearest existing ancestor for non-existent paths.
15
+ */
16
+ export declare function canonicalizePath(absolutePath: string): string;
17
+ /**
18
+ * Resolve a path relative to the working directory and validate it stays within bounds.
19
+ * Prevents both path traversal (e.g., `../../etc/passwd`) and symlink escape
20
+ * (e.g., a symlink under the repo pointing to an external location).
21
+ *
22
+ * @throws Error if the resolved path escapes the working directory
23
+ */
24
+ export declare function resolveAndValidatePath(workingDirectory: string, filePath: string): string;
25
+ /**
26
+ * Deterministic JSON serialization with recursively sorted keys.
27
+ * Ensures identical output regardless of key insertion order.
28
+ *
29
+ * Ported from VoltCode's map-shared.ts.
30
+ */
31
+ export declare function stableStringify(value: unknown): string;
32
+ /**
33
+ * Build the user message with prompt + `<map-details-json>` block.
34
+ * Each item gets a deterministic metadata block for reproducibility.
35
+ */
36
+ export declare function buildUserMessage(promptText: string, mapId: string, runStartedAt: string, itemIndex: number, item: unknown, outputSchema: Record<string, unknown>): string;
37
+ /**
38
+ * Build a retry message that includes the original prompt, the error, and the prior response.
39
+ */
40
+ export declare function buildRetryMessage(originalUserMessage: string, error: string, priorResponse: string): string;
41
+ /**
42
+ * Parse a JSONL file into an array of JSON values.
43
+ * Each line must be valid JSON. Empty trailing lines are tolerated.
44
+ */
45
+ export declare function parseJsonlFile(filePath: string): Promise<unknown[]>;
46
+ /**
47
+ * Write items as a JSONL file. Each item is serialized to one line.
48
+ */
49
+ export declare function itemsToJsonl(items: unknown[]): string;
50
+ /**
51
+ * Validate a parsed value against a JSON Schema using Zod passthrough.
52
+ * Returns {valid: true, value} on success, {valid: false, error} on failure.
53
+ *
54
+ * For simplicity, we use a basic JSON Schema validation approach rather than
55
+ * requiring ajv as a dependency. The schema is used mainly for type checking
56
+ * (object with expected properties).
57
+ */
58
+ export declare function validateAgainstSchema(value: unknown, schema: Record<string, unknown>): {
59
+ error?: string;
60
+ valid: boolean;
61
+ };
62
+ export declare const LlmMapParametersSchema: z.ZodObject<{
63
+ concurrency: z.ZodOptional<z.ZodNumber>;
64
+ input_path: z.ZodString;
65
+ max_attempts: z.ZodOptional<z.ZodNumber>;
66
+ output_path: z.ZodString;
67
+ output_schema: z.ZodRecord<z.ZodString, z.ZodAny>;
68
+ prompt: z.ZodString;
69
+ timeout_seconds: z.ZodOptional<z.ZodNumber>;
70
+ }, "strip", z.ZodTypeAny, {
71
+ prompt: string;
72
+ output_schema: Record<string, any>;
73
+ input_path: string;
74
+ output_path: string;
75
+ concurrency?: number | undefined;
76
+ max_attempts?: number | undefined;
77
+ timeout_seconds?: number | undefined;
78
+ }, {
79
+ prompt: string;
80
+ output_schema: Record<string, any>;
81
+ input_path: string;
82
+ output_path: string;
83
+ concurrency?: number | undefined;
84
+ max_attempts?: number | undefined;
85
+ timeout_seconds?: number | undefined;
86
+ }>;
87
+ export type LlmMapParameters = z.infer<typeof LlmMapParametersSchema>;
88
+ export declare const AgenticMapParametersSchema: z.ZodObject<{
89
+ input_path: z.ZodString;
90
+ max_attempts: z.ZodOptional<z.ZodNumber>;
91
+ max_depth: z.ZodOptional<z.ZodNumber>;
92
+ output_path: z.ZodString;
93
+ output_schema: z.ZodRecord<z.ZodString, z.ZodAny>;
94
+ prompt: z.ZodString;
95
+ read_only: z.ZodOptional<z.ZodBoolean>;
96
+ timeout_seconds: z.ZodOptional<z.ZodNumber>;
97
+ }, "strip", z.ZodTypeAny, {
98
+ prompt: string;
99
+ output_schema: Record<string, any>;
100
+ input_path: string;
101
+ output_path: string;
102
+ max_attempts?: number | undefined;
103
+ timeout_seconds?: number | undefined;
104
+ max_depth?: number | undefined;
105
+ read_only?: boolean | undefined;
106
+ }, {
107
+ prompt: string;
108
+ output_schema: Record<string, any>;
109
+ input_path: string;
110
+ output_path: string;
111
+ max_attempts?: number | undefined;
112
+ timeout_seconds?: number | undefined;
113
+ max_depth?: number | undefined;
114
+ read_only?: boolean | undefined;
115
+ }>;
116
+ export type AgenticMapParameters = z.infer<typeof AgenticMapParametersSchema>;
117
+ /**
118
+ * System message for LLM-Map: enforces stateless JSON-only output.
119
+ */
120
+ export declare const LLM_MAP_SYSTEM_MESSAGE: string;
121
+ /**
122
+ * System message for Agentic-Map: allows tool access with JSON output.
123
+ */
124
+ export declare function buildAgenticMapSystemMessage(readOnly: boolean): string;
125
+ /**
126
+ * Build the recursive composition guidance block appended to sub-agent user messages
127
+ * when this depth level allows further nesting.
128
+ */
129
+ export declare function buildRecursiveCompositionGuidance(currentDepth: number, maxDepth: number): string;
130
+ /**
131
+ * Race a promise against an abort signal.
132
+ * Used across map services to enforce per-item timeouts on calls that
133
+ * don't natively accept AbortSignal (e.g., generateContent, executeOnSession).
134
+ */
135
+ export declare function withTimeout<T>(promise: Promise<T>, signal: AbortSignal): Promise<T>;
136
+ /**
137
+ * Stateless LLM call for map item processing.
138
+ * Shared by llm-map-service and llm-map-memory.
139
+ */
140
+ export declare function callLlm(generator: IContentGenerator, userMessage: string, taskId?: string, abortSignal?: AbortSignal): Promise<GenerateContentResponse>;
@@ -0,0 +1,325 @@
1
+ /* eslint-disable camelcase */
2
+ import { randomUUID } from 'node:crypto';
3
+ import { realpathSync } from 'node:fs';
4
+ import { readFile } from 'node:fs/promises';
5
+ import { basename, dirname, join, normalize, resolve, sep } from 'node:path';
6
+ import { z } from 'zod';
7
+ // ── Path Safety ─────────────────────────────────────────────────────────────
8
+ /**
9
+ * Canonicalize a path by resolving symlinks.
10
+ * For existing paths, uses realpathSync.native().
11
+ * For non-existent paths (e.g., output files), walks up to the nearest
12
+ * existing ancestor directory, canonicalizes it, then reconstructs the
13
+ * remaining non-existent segments on top. This catches symlink escapes
14
+ * at any depth (e.g., workspace/symlink-to-outside/new-dir/file.jsonl).
15
+ */
16
+ /**
17
+ * Export the canonical path resolver for use in anti-cycle checks.
18
+ * Returns the real path via realpathSync.native(); falls back to walking
19
+ * up to the nearest existing ancestor for non-existent paths.
20
+ */
21
+ export function canonicalizePath(absolutePath) {
22
+ return canonicalize(absolutePath);
23
+ }
24
+ function canonicalize(absolutePath) {
25
+ try {
26
+ return realpathSync.native(absolutePath);
27
+ }
28
+ catch {
29
+ // Path doesn't exist — recurse up to the nearest existing ancestor
30
+ const dir = dirname(absolutePath);
31
+ const base = basename(absolutePath);
32
+ // Guard: reached filesystem root without finding an existing path
33
+ if (dir === absolutePath) {
34
+ return normalize(absolutePath);
35
+ }
36
+ return join(canonicalize(dir), base);
37
+ }
38
+ }
39
+ /**
40
+ * Resolve a path relative to the working directory and validate it stays within bounds.
41
+ * Prevents both path traversal (e.g., `../../etc/passwd`) and symlink escape
42
+ * (e.g., a symlink under the repo pointing to an external location).
43
+ *
44
+ * @throws Error if the resolved path escapes the working directory
45
+ */
46
+ export function resolveAndValidatePath(workingDirectory, filePath) {
47
+ const resolved = resolve(workingDirectory, filePath);
48
+ // Canonicalize both paths to resolve symlinks before containment check
49
+ const canonicalRoot = canonicalize(resolve(workingDirectory));
50
+ const canonicalResolved = canonicalize(resolved);
51
+ if (!canonicalResolved.startsWith(canonicalRoot + sep) && canonicalResolved !== canonicalRoot) {
52
+ throw new Error(`Path "${filePath}" resolves outside the working directory`);
53
+ }
54
+ return resolved;
55
+ }
56
+ // ── Deterministic JSON Serialization ─────────────────────────────────────────
57
+ /**
58
+ * Deterministic JSON serialization with recursively sorted keys.
59
+ * Ensures identical output regardless of key insertion order.
60
+ *
61
+ * Ported from VoltCode's map-shared.ts.
62
+ */
63
+ export function stableStringify(value) {
64
+ return JSON.stringify(value, (_key, val) => {
65
+ if (val !== null && typeof val === 'object' && !Array.isArray(val)) {
66
+ const sorted = {};
67
+ for (const k of Object.keys(val).sort()) {
68
+ sorted[k] = val[k];
69
+ }
70
+ return sorted;
71
+ }
72
+ return val;
73
+ });
74
+ }
75
+ // ── User Message Builder ─────────────────────────────────────────────────────
76
+ /**
77
+ * Build the user message with prompt + `<map-details-json>` block.
78
+ * Each item gets a deterministic metadata block for reproducibility.
79
+ */
80
+ export function buildUserMessage(promptText, mapId, runStartedAt, itemIndex, item, outputSchema) {
81
+ const details = {
82
+ item,
83
+ item_index: itemIndex,
84
+ map_id: mapId,
85
+ output_schema: outputSchema,
86
+ run_started_at: runStartedAt,
87
+ };
88
+ return `${promptText}\n\n<map-details-json>\n${stableStringify(details)}\n</map-details-json>`;
89
+ }
90
+ /**
91
+ * Build a retry message that includes the original prompt, the error, and the prior response.
92
+ */
93
+ export function buildRetryMessage(originalUserMessage, error, priorResponse) {
94
+ return [
95
+ originalUserMessage,
96
+ '',
97
+ '<map-retry>',
98
+ `Your previous response could not be used: ${error}`,
99
+ '',
100
+ 'Your prior response was:',
101
+ priorResponse,
102
+ '',
103
+ 'Please output corrected JSON only. No explanations, no markdown fences.',
104
+ '</map-retry>',
105
+ ].join('\n');
106
+ }
107
+ // ── JSONL Parsing ────────────────────────────────────────────────────────────
108
+ /**
109
+ * Parse a JSONL file into an array of JSON values.
110
+ * Each line must be valid JSON. Empty trailing lines are tolerated.
111
+ */
112
+ export async function parseJsonlFile(filePath) {
113
+ const rawText = await readFile(filePath, 'utf8');
114
+ const rawLines = rawText.split('\n');
115
+ const lines = rawLines.at(-1) === '' ? rawLines.slice(0, -1) : rawLines;
116
+ const items = [];
117
+ for (const [i, line] of lines.entries()) {
118
+ if (line.trim() === '') {
119
+ throw new Error(`Line ${i} is empty. Every line must be valid JSON.`);
120
+ }
121
+ try {
122
+ items.push(JSON.parse(line));
123
+ }
124
+ catch (error) {
125
+ throw new Error(`Line ${i} is not valid JSON: ${error}`);
126
+ }
127
+ }
128
+ return items;
129
+ }
130
+ /**
131
+ * Write items as a JSONL file. Each item is serialized to one line.
132
+ */
133
+ export function itemsToJsonl(items) {
134
+ return items.map((item) => JSON.stringify(item)).join('\n');
135
+ }
136
+ // ── Schema Validation ────────────────────────────────────────────────────────
137
+ /**
138
+ * Validate a parsed value against a JSON Schema using Zod passthrough.
139
+ * Returns {valid: true, value} on success, {valid: false, error} on failure.
140
+ *
141
+ * For simplicity, we use a basic JSON Schema validation approach rather than
142
+ * requiring ajv as a dependency. The schema is used mainly for type checking
143
+ * (object with expected properties).
144
+ */
145
+ export function validateAgainstSchema(value, schema) {
146
+ // Basic type check
147
+ if (schema.type === 'object' && (typeof value !== 'object' || value === null || Array.isArray(value))) {
148
+ return { error: `Expected object, got ${typeof value}`, valid: false };
149
+ }
150
+ if (schema.type === 'array' && !Array.isArray(value)) {
151
+ return { error: `Expected array, got ${typeof value}`, valid: false };
152
+ }
153
+ if (schema.type === 'string' && typeof value !== 'string') {
154
+ return { error: `Expected string, got ${typeof value}`, valid: false };
155
+ }
156
+ if (schema.type === 'number' && typeof value !== 'number') {
157
+ return { error: `Expected number, got ${typeof value}`, valid: false };
158
+ }
159
+ // Check required properties for objects
160
+ if (schema.type === 'object' && schema.required && Array.isArray(schema.required)) {
161
+ const obj = value;
162
+ for (const key of schema.required) {
163
+ if (!(key in obj)) {
164
+ return { error: `Missing required property: ${key}`, valid: false };
165
+ }
166
+ }
167
+ }
168
+ return { valid: true };
169
+ }
170
+ // ── Zod Schemas for Map Tool Parameters ──────────────────────────────────────
171
+ export const LlmMapParametersSchema = z.object({
172
+ concurrency: z
173
+ .number()
174
+ .int()
175
+ .positive()
176
+ .optional()
177
+ .describe('Max parallel LLM requests (default: 8)'),
178
+ input_path: z.string().describe('File path to JSONL input'),
179
+ max_attempts: z
180
+ .number()
181
+ .int()
182
+ .positive()
183
+ .optional()
184
+ .describe('Max LLM calls per item (default: 3)'),
185
+ output_path: z.string().describe('File path where JSONL output will be written'),
186
+ output_schema: z
187
+ .record(z.string(), z.any())
188
+ .describe('JSON Schema for LLM output validation'),
189
+ prompt: z.string().describe('Base instruction text sent to the LLM for each item'),
190
+ timeout_seconds: z
191
+ .number()
192
+ .int()
193
+ .positive()
194
+ .optional()
195
+ .describe('Max seconds per item (default: 120)'),
196
+ });
197
+ export const AgenticMapParametersSchema = z.object({
198
+ input_path: z.string().describe('File path to JSONL input'),
199
+ max_attempts: z
200
+ .number()
201
+ .int()
202
+ .positive()
203
+ .optional()
204
+ .describe('Max attempts per item (default: 3)'),
205
+ max_depth: z
206
+ .number()
207
+ .int()
208
+ .min(1)
209
+ .optional()
210
+ .describe('Maximum nesting depth for recursive agentic_map calls (default: 1). ' +
211
+ 'Requires read_only=false. Capped at a hard server-side ceiling of 3. ' +
212
+ 'Nested calls cannot increase the depth limit set by the root call.'),
213
+ output_path: z.string().describe('File path where JSONL output will be written'),
214
+ output_schema: z
215
+ .record(z.string(), z.any())
216
+ .describe('JSON Schema for sub-agent output validation'),
217
+ prompt: z.string().describe('Base instruction text for sub-agents'),
218
+ read_only: z
219
+ .boolean()
220
+ .optional()
221
+ .describe('If true, sub-agent write operations are disabled (default: true)'),
222
+ timeout_seconds: z
223
+ .number()
224
+ .int()
225
+ .positive()
226
+ .optional()
227
+ .describe('Max seconds per item (default: 300)'),
228
+ });
229
+ // ── System Messages ──────────────────────────────────────────────────────────
230
+ /**
231
+ * System message for LLM-Map: enforces stateless JSON-only output.
232
+ */
233
+ export const LLM_MAP_SYSTEM_MESSAGE = [
234
+ 'You are processing one item from a parallel LLM map.',
235
+ '',
236
+ 'You must return exactly one JSON value as the entire response:',
237
+ '- No surrounding prose, explanations, or commentary.',
238
+ '- No markdown fences (no ```json blocks).',
239
+ '- No trailing text after the JSON.',
240
+ '',
241
+ 'Your JSON output must validate against the schema provided in <map-details-json>.',
242
+ 'If asked to retry due to an error, output corrected JSON only.',
243
+ '',
244
+ 'No external tools exist and no actions can be taken beyond returning JSON text.',
245
+ ].join('\n');
246
+ /**
247
+ * System message for Agentic-Map: allows tool access with JSON output.
248
+ */
249
+ export function buildAgenticMapSystemMessage(readOnly) {
250
+ const lines = [
251
+ 'You are operating on one item from a parallel agentic map.',
252
+ '',
253
+ 'You must output exactly one JSON value as your final answer:',
254
+ '- No surrounding prose, explanations, or commentary.',
255
+ '- No markdown fences (no ```json blocks).',
256
+ '- No trailing text after the JSON.',
257
+ '',
258
+ 'Your JSON output must validate against the schema provided in <map-details-json>.',
259
+ 'If the system reports a JSON parsing or schema validation error, respond with corrected JSON only.',
260
+ ];
261
+ if (readOnly) {
262
+ lines.push('', 'Write operations (edit, write, bash) are disabled for this task.');
263
+ }
264
+ return lines.join('\n');
265
+ }
266
+ /**
267
+ * Build the recursive composition guidance block appended to sub-agent user messages
268
+ * when this depth level allows further nesting.
269
+ */
270
+ export function buildRecursiveCompositionGuidance(currentDepth, maxDepth) {
271
+ const remaining = maxDepth - currentDepth;
272
+ return [
273
+ '<recursive-map-guidance>',
274
+ `You are at nesting depth ${currentDepth} (hard limit: ${maxDepth}).`,
275
+ `You may call agentic_map up to ${remaining} more level(s).`,
276
+ 'Each nested call MUST use a distinct JSONL input_path.',
277
+ "Reusing an ancestor's input_path is a cycle and will fail immediately.",
278
+ '</recursive-map-guidance>',
279
+ ].join('\n');
280
+ }
281
+ // ── Shared Helpers ──────────────────────────────────────────────────────────
282
+ /**
283
+ * Race a promise against an abort signal.
284
+ * Used across map services to enforce per-item timeouts on calls that
285
+ * don't natively accept AbortSignal (e.g., generateContent, executeOnSession).
286
+ */
287
+ export function withTimeout(promise, signal) {
288
+ if (signal.aborted) {
289
+ return Promise.reject(new Error('Timed out'));
290
+ }
291
+ return new Promise((resolve, reject) => {
292
+ const onAbort = () => {
293
+ reject(new Error('Timed out'));
294
+ };
295
+ signal.addEventListener('abort', onAbort, { once: true });
296
+ promise.then((value) => {
297
+ signal.removeEventListener('abort', onAbort);
298
+ resolve(value);
299
+ }, (error) => {
300
+ signal.removeEventListener('abort', onAbort);
301
+ reject(error instanceof Error ? error : new Error(String(error)));
302
+ });
303
+ });
304
+ }
305
+ /**
306
+ * Stateless LLM call for map item processing.
307
+ * Shared by llm-map-service and llm-map-memory.
308
+ */
309
+ export async function callLlm(generator, userMessage, taskId, abortSignal) {
310
+ if (abortSignal?.aborted) {
311
+ throw new Error('Aborted');
312
+ }
313
+ return generator.generateContent({
314
+ config: {
315
+ maxTokens: 4096,
316
+ temperature: 0,
317
+ },
318
+ contents: [
319
+ { content: userMessage, role: 'user' },
320
+ ],
321
+ model: 'default',
322
+ systemPrompt: LLM_MAP_SYSTEM_MESSAGE,
323
+ taskId: taskId ?? randomUUID(),
324
+ });
325
+ }
@@ -0,0 +1,45 @@
1
+ export interface MapProgress {
2
+ failed: number;
3
+ mapId: string;
4
+ running: number;
5
+ succeeded: number;
6
+ total: number;
7
+ }
8
+ export interface MapRunResult {
9
+ failed: number;
10
+ mapId: string;
11
+ succeeded: number;
12
+ summaryHandle?: string;
13
+ total: number;
14
+ }
15
+ export interface InMemoryMapRunResult extends MapRunResult {
16
+ /** Per-item results keyed by input index */
17
+ results: Map<number, unknown>;
18
+ }
19
+ export interface WorkerPoolOptions {
20
+ /** Abort signal to cancel all workers */
21
+ abortSignal?: AbortSignal;
22
+ /** Number of parallel workers */
23
+ concurrency: number;
24
+ /** Pre-loaded items from JSONL input */
25
+ items: unknown[];
26
+ /** Progress callback for streaming updates */
27
+ onProgress?: (progress: MapProgress) => void;
28
+ /** Function to process a single item. Must return the result or throw. */
29
+ processItem: (index: number, item: unknown) => Promise<unknown>;
30
+ }
31
+ /**
32
+ * In-memory parallel worker pool for map operations.
33
+ *
34
+ * N workers run in parallel via Promise.all(). Each worker claims items by
35
+ * incrementing a shared index counter (safe because JS is single-threaded
36
+ * for synchronous code — no cross-process races).
37
+ *
38
+ * Replaces the previous FileMapStore-backed implementation which used atomic
39
+ * file renames for item claiming. That pattern was ported from VoltCode's
40
+ * multi-process PostgreSQL architecture but is unnecessary for byterover-cli's
41
+ * single-process execution model.
42
+ *
43
+ * @returns Summary of the map run including a results Map (index → result)
44
+ */
45
+ export declare function runMapWorkerPool(options: WorkerPoolOptions): Promise<InMemoryMapRunResult>;