march-cli 0.1.42 → 0.1.46
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/agent/code-search/cache.mjs +16 -7
- package/src/agent/code-search/engine.mjs +9 -2
- package/src/agent/code-search/retrieval/resilient-vectorizer.mjs +59 -0
- package/src/agent/code-search/retrieval/safetensors.mjs +16 -10
- package/src/agent/code-search/scanner.mjs +11 -5
- package/src/agent/code-search/tool.mjs +11 -5
- package/src/agent/model-payload-dumper.mjs +1 -1
- package/src/agent/runner/payload/provider-payload-transform.mjs +59 -0
- package/src/agent/runner/recall/mid-turn-recall-bridge.mjs +23 -0
- package/src/agent/runner.mjs +28 -27
- package/src/agent/runtime/remote-ui-client.mjs +1 -1
- package/src/agent/runtime/resource/context-resource-loader.mjs +17 -0
- package/src/agent/runtime/runtime-factory.mjs +5 -1
- package/src/agent/runtime/state/runner-state.mjs +10 -3
- package/src/agent/runtime/ui-event-bridge.mjs +2 -2
- package/src/agent/turn/turn-runner.mjs +35 -24
- package/src/cli/fallback-ui.mjs +2 -2
- package/src/cli/repl-loop.mjs +9 -8
- package/src/cli/startup/app-runtime.mjs +4 -2
- package/src/cli/tui/input/mouse-selection-controller.mjs +19 -0
- package/src/cli/tui/output/selectable-copy.mjs +3 -3
- package/src/cli/tui/output/timeline-block-restore.mjs +1 -1
- package/src/cli/tui/output-buffer.mjs +18 -0
- package/src/cli/tui/recall-rendering.mjs +38 -9
- package/src/cli/tui/selection/ansi-range.mjs +88 -0
- package/src/cli/tui/selection-screen.mjs +31 -99
- package/src/cli/turn/turn-input-preparer.mjs +15 -6
- package/src/cli/ui.mjs +2 -2
- package/src/cli/workspace/tui-timeline-projection.mjs +1 -1
- package/src/context/engine.mjs +15 -5
- package/src/context/system-core/base.md +1 -1
- package/src/memory/markdown/markdown-format.mjs +0 -17
- package/src/memory/markdown/markdown-recall.mjs +11 -19
- package/src/memory/markdown/semantic-preload.mjs +26 -0
- package/src/memory/markdown/semantic-recall.mjs +169 -0
- package/src/memory/markdown/sqlite-index.mjs +1 -13
- package/src/memory/markdown-store.mjs +34 -54
- package/src/web-ui/dist/assets/{index-BQtl1uQs.css → index-BG1Pxf1k.css} +1 -1
- package/src/web-ui/dist/assets/{index-DrlJis_D.js → index-C0xOHlDz.js} +1 -1
- package/src/web-ui/dist/index.html +2 -2
- package/src/web-ui/runtime-host.mjs +18 -3
- package/src/web-ui/src/components/timeline/TimelineBlocks.tsx +27 -0
- package/src/web-ui/src/model.ts +18 -0
- package/src/web-ui/src/runtime/client.ts +2 -1
- package/src/web-ui/src/runtime/runtimeTimeline.ts +5 -0
- package/src/web-ui/src/styles/shell.css +7 -0
- package/src/web-ui/src/timelineAdapter.ts +2 -0
package/package.json
CHANGED
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
|
|
2
2
|
import { dirname } from "node:path";
|
|
3
3
|
import { chunkFile } from "./chunker.mjs";
|
|
4
|
+
import { readCodeFileContent } from "./scanner.mjs";
|
|
4
5
|
import { Bm25Index } from "./retrieval/bm25.mjs";
|
|
6
|
+
import { describeVectorizer } from "./retrieval/resilient-vectorizer.mjs";
|
|
5
7
|
import { LocalVectorIndex, defaultVectorizer } from "./retrieval/vector.mjs";
|
|
6
8
|
|
|
7
9
|
const DEFAULT_MAX_FILE_ENTRIES = 8_000;
|
|
8
|
-
const DEFAULT_MAX_INDEX_ENTRIES =
|
|
10
|
+
const DEFAULT_MAX_INDEX_ENTRIES = 6;
|
|
9
11
|
|
|
10
12
|
export class CodeSearchIndexCache {
|
|
11
13
|
constructor({
|
|
@@ -24,7 +26,7 @@ export class CodeSearchIndexCache {
|
|
|
24
26
|
this.dirty = false;
|
|
25
27
|
}
|
|
26
28
|
|
|
27
|
-
async build(files) {
|
|
29
|
+
async build(files, { includeVector = true } = {}) {
|
|
28
30
|
await this.load();
|
|
29
31
|
const chunks = [];
|
|
30
32
|
let reusedFiles = 0;
|
|
@@ -38,7 +40,9 @@ export class CodeSearchIndexCache {
|
|
|
38
40
|
reusedFiles += 1;
|
|
39
41
|
continue;
|
|
40
42
|
}
|
|
41
|
-
const
|
|
43
|
+
const content = await readCodeFileContent(file.absPath);
|
|
44
|
+
if (content === null) continue;
|
|
45
|
+
const fileChunks = await chunkFile({ ...file, content });
|
|
42
46
|
this.fileChunks.set(key, { signature, chunks: fileChunks });
|
|
43
47
|
this.dirty = true;
|
|
44
48
|
chunks.push(...fileChunks);
|
|
@@ -48,21 +52,26 @@ export class CodeSearchIndexCache {
|
|
|
48
52
|
this.pruneFileCache();
|
|
49
53
|
await this.persist();
|
|
50
54
|
|
|
51
|
-
const indexSignature =
|
|
55
|
+
const indexSignature = this.indexSignature(files, { includeVector });
|
|
52
56
|
const cachedIndex = this.indices.get(indexSignature);
|
|
53
57
|
if (cachedIndex) {
|
|
54
58
|
this.indices.delete(indexSignature);
|
|
55
59
|
this.indices.set(indexSignature, cachedIndex);
|
|
56
|
-
return { chunks, index: cachedIndex, reusedFiles, indexedFiles, reusedIndex: true,
|
|
60
|
+
return { chunks, index: cachedIndex, reusedFiles, indexedFiles, reusedIndex: true, ...describeVectorizer(this.vectorizer) };
|
|
57
61
|
}
|
|
58
62
|
|
|
59
63
|
const index = {
|
|
60
64
|
lexical: new Bm25Index(chunks),
|
|
61
|
-
vector: await LocalVectorIndex.create(chunks, { vectorizer: this.vectorizer }),
|
|
65
|
+
vector: includeVector ? await LocalVectorIndex.create(chunks, { vectorizer: this.vectorizer }) : null,
|
|
62
66
|
};
|
|
63
67
|
this.indices.set(indexSignature, index);
|
|
64
68
|
this.pruneIndexCache();
|
|
65
|
-
return { chunks, index, reusedFiles, indexedFiles, reusedIndex: false,
|
|
69
|
+
return { chunks, index, reusedFiles, indexedFiles, reusedIndex: false, ...describeVectorizer(this.vectorizer) };
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
indexSignature(files, { includeVector }) {
|
|
73
|
+
const vectorKey = includeVector ? this.vectorizer.id : "lexical";
|
|
74
|
+
return [vectorKey, ...files.map(fileSignature)].join("\n");
|
|
66
75
|
}
|
|
67
76
|
|
|
68
77
|
clear() {
|
|
@@ -22,7 +22,7 @@ export async function searchCode(options = {}) {
|
|
|
22
22
|
if (!normalizedQuery && !related_to) return { results: [], stats: { files: 0, chunks: 0 } };
|
|
23
23
|
|
|
24
24
|
const files = await scanCodeFiles({ root, path });
|
|
25
|
-
const built = await activeCache.build(files);
|
|
25
|
+
const built = await activeCache.build(files, { includeVector: needsVectorIndex({ mode, related_to }) });
|
|
26
26
|
const related = related_to ? relatedQuery(built.chunks, related_to, normalizedQuery) : null;
|
|
27
27
|
const queryText = related?.query ?? normalizedQuery;
|
|
28
28
|
const retrieved = await retrieveChunks(built.index, queryText, mode);
|
|
@@ -38,7 +38,7 @@ export async function searchCode(options = {}) {
|
|
|
38
38
|
async function retrieveChunks(index, queryText, mode) {
|
|
39
39
|
const lexical = index.lexical.search(queryText, { limit: RETRIEVAL_LIMIT });
|
|
40
40
|
if (mode === "lexical" || mode === "symbol") return lexical;
|
|
41
|
-
const semantic = await index.vector.search(queryText, { limit: RETRIEVAL_LIMIT });
|
|
41
|
+
const semantic = index.vector ? await index.vector.search(queryText, { limit: RETRIEVAL_LIMIT }) : [];
|
|
42
42
|
if (mode === "semantic") return semantic;
|
|
43
43
|
return rrfFuse([
|
|
44
44
|
{ results: lexical, weight: 1.2 },
|
|
@@ -46,6 +46,11 @@ async function retrieveChunks(index, queryText, mode) {
|
|
|
46
46
|
], { limit: RETRIEVAL_LIMIT });
|
|
47
47
|
}
|
|
48
48
|
|
|
49
|
+
function needsVectorIndex({ mode, related_to }) {
|
|
50
|
+
if (related_to) return true;
|
|
51
|
+
return mode !== "lexical" && mode !== "symbol";
|
|
52
|
+
}
|
|
53
|
+
|
|
49
54
|
function resultMode({ related_to, mode }) {
|
|
50
55
|
if (related_to) return "related";
|
|
51
56
|
if (mode === "symbol") return "symbol";
|
|
@@ -79,6 +84,8 @@ function formatStats(files, built, mode) {
|
|
|
79
84
|
indexed_files: built.indexedFiles,
|
|
80
85
|
reused_index: built.reusedIndex,
|
|
81
86
|
vectorizer: built.vectorizer,
|
|
87
|
+
vectorizer_status: built.vectorizer_status,
|
|
88
|
+
vectorizer_warning: built.vectorizer_warning,
|
|
82
89
|
};
|
|
83
90
|
}
|
|
84
91
|
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { HashingVectorizer } from "./vector.mjs";
|
|
2
|
+
|
|
3
|
+
export class ResilientVectorizer {
|
|
4
|
+
constructor({ primary, fallback = new HashingVectorizer(), label = "embedding" } = {}) {
|
|
5
|
+
if (!primary) throw new Error("ResilientVectorizer requires primary");
|
|
6
|
+
this.primary = primary;
|
|
7
|
+
this.fallback = fallback;
|
|
8
|
+
this.label = label;
|
|
9
|
+
this.id = `resilient:${primary.id}->${fallback.id}`;
|
|
10
|
+
this.active = primary;
|
|
11
|
+
this.status = "primary";
|
|
12
|
+
this.warning = null;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
get dimensions() {
|
|
16
|
+
return this.active.dimensions;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
get activeId() {
|
|
20
|
+
return this.active.id;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
async load() {
|
|
24
|
+
if (this.status === "fallback") return false;
|
|
25
|
+
try {
|
|
26
|
+
if (typeof this.primary.load === "function") await this.primary.load();
|
|
27
|
+
else await this.primary.encode([`${this.label} warmup`]);
|
|
28
|
+
return true;
|
|
29
|
+
} catch (err) {
|
|
30
|
+
this.#activateFallback(err);
|
|
31
|
+
return false;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
async encode(texts) {
|
|
36
|
+
if (this.status === "fallback") return this.fallback.encode(texts);
|
|
37
|
+
try {
|
|
38
|
+
return await this.primary.encode(texts);
|
|
39
|
+
} catch (err) {
|
|
40
|
+
this.#activateFallback(err);
|
|
41
|
+
return this.fallback.encode(texts);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
#activateFallback(err) {
|
|
46
|
+
this.active = this.fallback;
|
|
47
|
+
this.status = "fallback";
|
|
48
|
+
const message = err?.message ?? String(err);
|
|
49
|
+
this.warning = `${this.label} Model2Vec unavailable; using local hashing fallback: ${message}`;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export function describeVectorizer(vectorizer) {
|
|
54
|
+
return {
|
|
55
|
+
vectorizer: vectorizer?.activeId ?? vectorizer?.id ?? "unknown",
|
|
56
|
+
vectorizer_status: vectorizer?.status ?? "primary",
|
|
57
|
+
vectorizer_warning: vectorizer?.warning ?? null,
|
|
58
|
+
};
|
|
59
|
+
}
|
|
@@ -16,34 +16,40 @@ export async function readSafetensors(filePath) {
|
|
|
16
16
|
}
|
|
17
17
|
|
|
18
18
|
export function parseSafetensors(buffer) {
|
|
19
|
-
const
|
|
20
|
-
const headerLength = Number(new DataView(
|
|
19
|
+
const source = toByteSource(buffer);
|
|
20
|
+
const headerLength = Number(new DataView(source.buffer, source.byteOffset, HEADER_BYTES).getBigUint64(0, true));
|
|
21
21
|
const headerStart = HEADER_BYTES;
|
|
22
22
|
const headerEnd = headerStart + headerLength;
|
|
23
|
-
const headerJson = new TextDecoder().decode(new Uint8Array(
|
|
23
|
+
const headerJson = new TextDecoder().decode(new Uint8Array(source.buffer, source.byteOffset + headerStart, headerLength));
|
|
24
24
|
const header = JSON.parse(headerJson);
|
|
25
25
|
return {
|
|
26
26
|
names: Object.keys(header).filter((name) => name !== "__metadata__"),
|
|
27
27
|
getTensor(name) {
|
|
28
28
|
const descriptor = header[name];
|
|
29
29
|
if (!descriptor) throw new Error(`Missing safetensors tensor: ${name}`);
|
|
30
|
-
return readTensor(
|
|
30
|
+
return readTensor(source, headerEnd, descriptor);
|
|
31
31
|
},
|
|
32
32
|
};
|
|
33
33
|
}
|
|
34
34
|
|
|
35
|
-
function readTensor(
|
|
35
|
+
function readTensor(source, dataStart, descriptor) {
|
|
36
36
|
const reader = DTYPE_READERS[descriptor.dtype];
|
|
37
37
|
if (!reader) throw new Error(`Unsupported safetensors dtype: ${descriptor.dtype}`);
|
|
38
38
|
const [start, end] = descriptor.data_offsets;
|
|
39
|
-
const byteOffset = dataStart + start;
|
|
40
|
-
const
|
|
41
|
-
const
|
|
39
|
+
const byteOffset = source.byteOffset + dataStart + start;
|
|
40
|
+
const byteLength = end - start;
|
|
41
|
+
const length = byteLength / reader.size;
|
|
42
|
+
if (descriptor.dtype === "F32" && byteOffset % Float32Array.BYTES_PER_ELEMENT === 0) {
|
|
43
|
+
return { values: new Float32Array(source.buffer, byteOffset, length), shape: descriptor.shape, dtype: descriptor.dtype };
|
|
44
|
+
}
|
|
45
|
+
const view = new DataView(source.buffer, byteOffset, byteLength);
|
|
42
46
|
const values = new Float32Array(length);
|
|
43
47
|
for (let index = 0; index < length; index += 1) values[index] = reader.read(view, index * reader.size);
|
|
44
48
|
return { values, shape: descriptor.shape, dtype: descriptor.dtype };
|
|
45
49
|
}
|
|
46
50
|
|
|
47
|
-
function
|
|
48
|
-
return buffer
|
|
51
|
+
function toByteSource(buffer) {
|
|
52
|
+
if (buffer instanceof ArrayBuffer) return { buffer, byteOffset: 0, byteLength: buffer.byteLength };
|
|
53
|
+
if (ArrayBuffer.isView(buffer)) return { buffer: buffer.buffer, byteOffset: buffer.byteOffset, byteLength: buffer.byteLength };
|
|
54
|
+
throw new TypeError("safetensors input must be an ArrayBuffer or typed array view");
|
|
49
55
|
}
|
|
@@ -26,13 +26,11 @@ export async function scanCodeFiles({ root, path = ".", maxFiles = DEFAULT_MAX_F
|
|
|
26
26
|
let info;
|
|
27
27
|
try { info = await stat(absPath); } catch { continue; }
|
|
28
28
|
if (!info.isFile() || info.size > maxFileBytes) continue;
|
|
29
|
-
|
|
30
|
-
if (content === null) continue;
|
|
29
|
+
if (await isSymlink(absPath)) continue;
|
|
31
30
|
files.push({
|
|
32
31
|
absPath,
|
|
33
32
|
relPath: relPath.replace(/\\/g, "/"),
|
|
34
33
|
language: languageForPath(relPath),
|
|
35
|
-
content,
|
|
36
34
|
size: info.size,
|
|
37
35
|
mtimeMs: info.mtimeMs,
|
|
38
36
|
});
|
|
@@ -40,6 +38,7 @@ export async function scanCodeFiles({ root, path = ".", maxFiles = DEFAULT_MAX_F
|
|
|
40
38
|
return files;
|
|
41
39
|
}
|
|
42
40
|
|
|
41
|
+
|
|
43
42
|
async function listCandidateFiles(root, base) {
|
|
44
43
|
const fromRipgrep = await listRipgrepFiles(root, base);
|
|
45
44
|
if (fromRipgrep) return fromRipgrep;
|
|
@@ -72,9 +71,8 @@ async function listFilesRecursively(root, dir) {
|
|
|
72
71
|
return files;
|
|
73
72
|
}
|
|
74
73
|
|
|
75
|
-
async function
|
|
74
|
+
export async function readCodeFileContent(path) {
|
|
76
75
|
try {
|
|
77
|
-
if ((await lstat(path)).isSymbolicLink()) return null;
|
|
78
76
|
const buffer = await readFile(path);
|
|
79
77
|
if (buffer.includes(0)) return null;
|
|
80
78
|
return buffer.toString("utf8");
|
|
@@ -82,3 +80,11 @@ async function readUtf8Text(path) {
|
|
|
82
80
|
return null;
|
|
83
81
|
}
|
|
84
82
|
}
|
|
83
|
+
|
|
84
|
+
async function isSymlink(path) {
|
|
85
|
+
try {
|
|
86
|
+
return (await lstat(path)).isSymbolicLink();
|
|
87
|
+
} catch {
|
|
88
|
+
return true;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
@@ -5,6 +5,7 @@ import { toolText } from "../tool-result.mjs";
|
|
|
5
5
|
import { CodeSearchIndexCache } from "./cache.mjs";
|
|
6
6
|
import { searchCode } from "./engine.mjs";
|
|
7
7
|
import { Model2VecVectorizer, POTION_CODE_MODEL_ID } from "./retrieval/model2vec.mjs";
|
|
8
|
+
import { ResilientVectorizer } from "./retrieval/resilient-vectorizer.mjs";
|
|
8
9
|
|
|
9
10
|
const persistentCaches = new Map();
|
|
10
11
|
|
|
@@ -23,7 +24,7 @@ export function createCodeSearchTool({ engine, stateRoot = null }) {
|
|
|
23
24
|
Type.Literal("symbol"),
|
|
24
25
|
Type.Literal("lexical"),
|
|
25
26
|
Type.Literal("semantic"),
|
|
26
|
-
], { description: "Search mode. auto uses BM25 + Model2Vec retrieval with RRF fusion." })),
|
|
27
|
+
], { description: "Search mode. auto uses BM25 + Model2Vec retrieval with RRF fusion; falls back to local hashing if the model is unavailable." })),
|
|
27
28
|
include_tests: Type.Optional(Type.Boolean({ description: "Include test/spec paths without penalty; default false" })),
|
|
28
29
|
related_to: Type.Optional(Type.Object({
|
|
29
30
|
file_path: Type.String({ description: "Workspace-relative file path containing the known code" }),
|
|
@@ -53,7 +54,10 @@ function persistentCacheFor(stateRoot) {
|
|
|
53
54
|
if (!cache) {
|
|
54
55
|
cache = new CodeSearchIndexCache({
|
|
55
56
|
storagePath,
|
|
56
|
-
vectorizer: new
|
|
57
|
+
vectorizer: new ResilientVectorizer({
|
|
58
|
+
primary: new Model2VecVectorizer({ modelDir }),
|
|
59
|
+
label: "code_search",
|
|
60
|
+
}),
|
|
57
61
|
});
|
|
58
62
|
persistentCaches.set(cacheKey, cache);
|
|
59
63
|
}
|
|
@@ -61,13 +65,15 @@ function persistentCacheFor(stateRoot) {
|
|
|
61
65
|
}
|
|
62
66
|
|
|
63
67
|
function formatSearchOutput({ results, stats }) {
|
|
64
|
-
const
|
|
65
|
-
|
|
68
|
+
const mode = stats.vectorizer_status === "fallback" ? `${stats.mode}-fallback` : stats.mode;
|
|
69
|
+
const header = `--- code_search (${results.length} results, ${stats.files} files, ${stats.chunks} chunks, ${mode}) ---`;
|
|
70
|
+
const warning = stats.vectorizer_warning ? `\nwarning: ${stats.vectorizer_warning}` : "";
|
|
71
|
+
if (results.length === 0) return `${header}${warning}\nNo matching code snippets found.`;
|
|
66
72
|
const body = results.map((result, index) => [
|
|
67
73
|
`${index + 1}. ${result.file_path}:${result.start_line}-${result.end_line} score=${result.score} kind=${result.kind}${result.symbols.length ? ` symbols=${result.symbols.join(",")}` : ""}`,
|
|
68
74
|
fenceSnippet(result.snippet),
|
|
69
75
|
].join("\n")).join("\n\n");
|
|
70
|
-
return `${header}\n${body}`;
|
|
76
|
+
return `${header}${warning}\n${body}`;
|
|
71
77
|
}
|
|
72
78
|
|
|
73
79
|
function fenceSnippet(snippet) {
|
|
@@ -14,7 +14,7 @@ export function installModelPayloadDumper(session, modelContextDumper, getKind =
|
|
|
14
14
|
const originalEffectivePayload = replacement === undefined ? payload : replacement;
|
|
15
15
|
const kind = getKind();
|
|
16
16
|
const effectivePayload = typeof transformPayload === "function"
|
|
17
|
-
? transformPayload(originalEffectivePayload, { kind, model })
|
|
17
|
+
? await transformPayload(originalEffectivePayload, { kind, model })
|
|
18
18
|
: originalEffectivePayload;
|
|
19
19
|
onModelPayload?.({
|
|
20
20
|
payload: effectivePayload,
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { injectHostedTools } from "../../../provider/hosted-tools.mjs";
|
|
2
|
+
import { replaceProviderContextMessages } from "../../model-payload-dumper.mjs";
|
|
3
|
+
import { applyCodexLargeContextGuardToPayload } from "../codex-large-context-guard.mjs";
|
|
4
|
+
|
|
5
|
+
export function createRunnerProviderPayloadTransform({
|
|
6
|
+
engine,
|
|
7
|
+
sessionBinding,
|
|
8
|
+
hostedTools,
|
|
9
|
+
getCurrentPrompt,
|
|
10
|
+
getContextMode,
|
|
11
|
+
getFastEntry,
|
|
12
|
+
waitForMidTurnRecall = null,
|
|
13
|
+
getMidTurnRecallMessages = null,
|
|
14
|
+
}) {
|
|
15
|
+
let didReplaceProviderContext = false;
|
|
16
|
+
|
|
17
|
+
return {
|
|
18
|
+
resetTurn() {
|
|
19
|
+
didReplaceProviderContext = false;
|
|
20
|
+
},
|
|
21
|
+
async transform(payload, { kind, model } = {}) {
|
|
22
|
+
if (kind !== "user") return payload;
|
|
23
|
+
const shouldReplaceProviderContext = getContextMode() !== "continueExistingPiTranscript"
|
|
24
|
+
&& !didReplaceProviderContext;
|
|
25
|
+
let nextPayload = payload;
|
|
26
|
+
if (shouldReplaceProviderContext) {
|
|
27
|
+
nextPayload = replaceProviderContextMessages(payload, engine.buildProviderContext(getCurrentPrompt()));
|
|
28
|
+
didReplaceProviderContext = true;
|
|
29
|
+
} else {
|
|
30
|
+
await waitForMidTurnRecall?.();
|
|
31
|
+
nextPayload = appendMissingMidTurnRecallMessages(nextPayload, getMidTurnRecallMessages?.() ?? []);
|
|
32
|
+
}
|
|
33
|
+
nextPayload = injectHostedTools(nextPayload, model, hostedTools);
|
|
34
|
+
nextPayload = applyCodexLargeContextGuardToPayload(nextPayload, { model, session: sessionBinding.get() });
|
|
35
|
+
if (getFastEntry()) nextPayload = { ...nextPayload, service_tier: "priority" };
|
|
36
|
+
return nextPayload;
|
|
37
|
+
},
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function appendMissingMidTurnRecallMessages(payload, recallMessages) {
|
|
42
|
+
if (!Array.isArray(payload?.messages) || recallMessages.length === 0) return payload;
|
|
43
|
+
const existingText = payload.messages.map((message) => providerMessageText(message)).join("\n");
|
|
44
|
+
const missing = recallMessages.filter((content) => content && !existingText.includes(content));
|
|
45
|
+
if (missing.length === 0) return payload;
|
|
46
|
+
return {
|
|
47
|
+
...payload,
|
|
48
|
+
messages: [
|
|
49
|
+
...payload.messages,
|
|
50
|
+
...missing.map((content) => ({ role: "user", content })),
|
|
51
|
+
],
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function providerMessageText(message) {
|
|
56
|
+
if (typeof message?.content === "string") return message.content;
|
|
57
|
+
if (Array.isArray(message?.content)) return message.content.map((part) => part?.text ?? "").join("");
|
|
58
|
+
return "";
|
|
59
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
export function createMidTurnRecallBridge() {
|
|
2
|
+
const messages = [];
|
|
3
|
+
const tasks = new Set();
|
|
4
|
+
return {
|
|
5
|
+
reset() {
|
|
6
|
+
messages.length = 0;
|
|
7
|
+
tasks.clear();
|
|
8
|
+
},
|
|
9
|
+
track({ content, task } = {}) {
|
|
10
|
+
if (content && !messages.includes(content)) messages.push(content);
|
|
11
|
+
if (!task?.finally) return;
|
|
12
|
+
tasks.add(task);
|
|
13
|
+
task.finally(() => tasks.delete(task));
|
|
14
|
+
},
|
|
15
|
+
async wait() {
|
|
16
|
+
if (tasks.size === 0) return;
|
|
17
|
+
await Promise.allSettled([...tasks]);
|
|
18
|
+
},
|
|
19
|
+
messages() {
|
|
20
|
+
return [...messages];
|
|
21
|
+
},
|
|
22
|
+
};
|
|
23
|
+
}
|
package/src/agent/runner.mjs
CHANGED
|
@@ -5,15 +5,17 @@ import { createMarchLifecycleAdapter } from "../extensions/lifecycle-adapter.mjs
|
|
|
5
5
|
import { syncMarchSessionState } from "../session/state/march-session-sync.mjs";
|
|
6
6
|
import { LspService } from "../lsp/service.mjs";
|
|
7
7
|
import { formatLspServiceEvent } from "../lsp/status-message.mjs";
|
|
8
|
-
import { estimateProviderPayloadTokens, installModelPayloadDumper
|
|
8
|
+
import { estimateProviderPayloadTokens, installModelPayloadDumper } from "./model-payload-dumper.mjs";
|
|
9
9
|
import { resolveInitialModel, resolveRunnerSessionManager } from "./runner/runner-init.mjs";
|
|
10
10
|
import { runRunnerCleanup } from "./runner/runner-cleanup.mjs";
|
|
11
11
|
import { createRunnerRuntimeHost } from "./runtime/runner-runtime-host.mjs";
|
|
12
|
+
import { createMarchPiResourceLoader } from "./runtime/resource/context-resource-loader.mjs";
|
|
12
13
|
import { createRuntimeUiBridge } from "./runtime/ui-event-bridge.mjs";
|
|
13
14
|
import { getRunnerSessionStats, syncEngineSessionState } from "./runner/runner-session-state.mjs";
|
|
14
15
|
import { buildNotificationActivation, installRunnerProcessGuards, notifyTurnEndBestEffort, notifyTurnEndDetached, providerContextToPayload } from "./runner/runner-utils.mjs";
|
|
15
16
|
import { dumpCodexTransportDebug, getCodexTransportDebugSnapshot } from "./runner/codex-transport-debug.mjs";
|
|
16
|
-
import {
|
|
17
|
+
import { createRunnerProviderPayloadTransform } from "./runner/payload/provider-payload-transform.mjs";
|
|
18
|
+
import { createMidTurnRecallBridge } from "./runner/recall/mid-turn-recall-bridge.mjs";
|
|
17
19
|
import { resolveRunnerSessionOptions } from "./session/session-options.mjs";
|
|
18
20
|
import { createSessionBinding } from "./session/session-binding.mjs";
|
|
19
21
|
import { maybeAutoNameSession } from "./session/session-auto-name.mjs";
|
|
@@ -23,13 +25,11 @@ import { beginLoggedTurn } from "./turn/turn-logging.mjs";
|
|
|
23
25
|
import { appendFastVariants, createFastModelEntry, fromFastEntryModel, isFastProvider } from "./runner/fast-model.mjs";
|
|
24
26
|
import { registerSuperGrokProvider } from "../supergrok/provider.mjs";
|
|
25
27
|
import { registerCustomProviders } from "../provider/custom-provider.mjs";
|
|
26
|
-
import { injectHostedTools } from "../provider/hosted-tools.mjs";
|
|
27
28
|
import { createRunnerLifecycle } from "./lifecycle/runner-lifecycle.mjs";
|
|
28
29
|
import { createRunnerProviderQuotaRuntime } from "./runner/provider-quota-runtime.mjs";
|
|
29
30
|
import { appendRunnerTurnHistory, createRunnerHistoryStore } from "../history/runner.mjs";
|
|
30
31
|
export { MARCH_BASE_TOOL_NAMES, installModelPayloadDumper };
|
|
31
|
-
export { createDefaultSessionManager, resolveRunnerSessionManager } from "./runner/runner-init.mjs";
|
|
32
|
-
export { getRunnerSessionStats, syncEngineSessionState } from "./runner/runner-session-state.mjs";
|
|
32
|
+
export { createDefaultSessionManager, resolveRunnerSessionManager } from "./runner/runner-init.mjs"; export { getRunnerSessionStats, syncEngineSessionState } from "./runner/runner-session-state.mjs";
|
|
33
33
|
export async function createRunner({ cwd, modelId = null, provider = null, providers = {}, stateRoot, ui, memoryRoot = null, profilePaths = null, memoryStore = null, memoryTools = [], remoteMemorySources = [], shellRuntime = null, mcpTools = [], mcpInjections = [], mcpClientManager = null, webTools = [], namespace = "", sessionManager = null, useRuntimeHost = false, projectMarchDir = null, syncMarchSessionState: syncMarchSessionStateEnabled = false, syncPiSidecar = syncMarchSessionStateEnabled, extensionPaths = [], lifecycleHooks = [], lifecycleDiagnostics = [], authStorage = null, modelContextDumper = null, turnNotifier = null, logger = null, onModelPayload = null, onLspStatusChange = null, createAgentSessionImpl = createAgentSession, createAgentSessionRuntimeImpl, createRuntimeServices, createRuntimeSessionFromServices, maxTurns, trimBatch, serviceTier = null, hostedTools = {}, notificationContext = null }) {
|
|
34
34
|
installRunnerProcessGuards();
|
|
35
35
|
if (!useRuntimeHost && extensionPaths.length > 0) throw new Error("--extension requires the default pi runtime host path");
|
|
@@ -43,10 +43,7 @@ export async function createRunner({ cwd, modelId = null, provider = null, provi
|
|
|
43
43
|
if (!selectedModel) throw new Error("No authenticated models available. Run: march provider --config");
|
|
44
44
|
provider = selectedModel.provider;
|
|
45
45
|
modelId = selectedModel.id;
|
|
46
|
-
const settingsManager = SettingsManager.inMemory({
|
|
47
|
-
compaction: { enabled: false },
|
|
48
|
-
retry: { enabled: true, maxRetries: 3, baseDelayMs: 2000 },
|
|
49
|
-
});
|
|
46
|
+
const settingsManager = SettingsManager.inMemory({ compaction: { enabled: false }, retry: { enabled: true, maxRetries: 3, baseDelayMs: 2000, provider: { timeoutMs: 20000, maxRetries: 3, maxRetryDelayMs: 60000 } } });
|
|
50
47
|
const { ui: runtimeUi, eventBus: runtimeUiEvents, detach: detachRuntimeUi } = createRuntimeUiBridge(ui);
|
|
51
48
|
const lspService = new LspService({ cwd, onEvent: (event) => runtimeUi.status?.(formatLspServiceEvent(event)), onStatusChange: (event) => onLspStatusChange?.(event) });
|
|
52
49
|
const engine = new ContextEngine({ cwd, modelId, provider, namespace, memoryRoot, profilePaths, remoteMemorySources, shellRuntime, lspService, injections: mcpInjections, maxTurns, trimBatch });
|
|
@@ -54,11 +51,18 @@ export async function createRunner({ cwd, modelId = null, provider = null, provi
|
|
|
54
51
|
const resolvedSessionManager = resolveRunnerSessionManager(cwd, sessionManager);
|
|
55
52
|
const sessionBinding = createSessionBinding(null);
|
|
56
53
|
let currentModelCallKind = "model", currentTurnId = null, currentPromptForContext = "";
|
|
54
|
+
const midTurnRecallBridge = createMidTurnRecallBridge();
|
|
57
55
|
const lifecycle = createRunnerLifecycle();
|
|
58
|
-
let currentTurnContextMode = "rebuild";
|
|
59
|
-
let
|
|
60
|
-
|
|
61
|
-
|
|
56
|
+
let currentTurnContextMode = "rebuild", nextTurnContextMode = "rebuild";
|
|
57
|
+
let lastNotificationResult = null, runtimeHost = null, lifecycleAdapter = null, _currentFastEntry = null;
|
|
58
|
+
const providerPayloadTransform = createRunnerProviderPayloadTransform({
|
|
59
|
+
engine, sessionBinding, hostedTools,
|
|
60
|
+
getCurrentPrompt: () => currentPromptForContext,
|
|
61
|
+
getContextMode: () => currentTurnContextMode,
|
|
62
|
+
getFastEntry: () => _currentFastEntry,
|
|
63
|
+
waitForMidTurnRecall: () => midTurnRecallBridge.wait(),
|
|
64
|
+
getMidTurnRecallMessages: () => midTurnRecallBridge.messages(),
|
|
65
|
+
});
|
|
62
66
|
if (useRuntimeHost) {
|
|
63
67
|
runtimeHost = await createRunnerRuntimeHost({
|
|
64
68
|
cwd, stateRoot, provider, modelId,
|
|
@@ -69,7 +73,7 @@ export async function createRunner({ cwd, modelId = null, provider = null, provi
|
|
|
69
73
|
memoryTools, memoryStore, historyStore, shellRuntime, lspService, mcpTools, webTools,
|
|
70
74
|
lifecycle, extensionPaths, hostedTools,
|
|
71
75
|
onRebind: (session) => {
|
|
72
|
-
installModelPayloadDumper(session, modelContextDumper, () => currentModelCallKind, onLoggedModelPayload,
|
|
76
|
+
installModelPayloadDumper(session, modelContextDumper, () => currentModelCallKind, onLoggedModelPayload, providerPayloadTransform.transform);
|
|
73
77
|
syncEngineSessionState(engine, session);
|
|
74
78
|
},
|
|
75
79
|
createAgentSessionRuntimeImpl,
|
|
@@ -83,13 +87,18 @@ export async function createRunner({ cwd, modelId = null, provider = null, provi
|
|
|
83
87
|
authStorage: resolvedAuth, projectMarchDir,
|
|
84
88
|
getCurrentModel: () => sessionBinding.get()?.model ?? selectedModel,
|
|
85
89
|
});
|
|
90
|
+
const resourceLoader = await createMarchPiResourceLoader({
|
|
91
|
+
cwd,
|
|
92
|
+
agentDir: stateRoot,
|
|
93
|
+
settingsManager,
|
|
94
|
+
});
|
|
86
95
|
const { session } = await createAgentSessionImpl({
|
|
87
96
|
cwd, agentDir: stateRoot, ...sessionOptions,
|
|
88
97
|
authStorage: resolvedAuth, modelRegistry,
|
|
89
|
-
sessionManager: resolvedSessionManager, settingsManager,
|
|
98
|
+
sessionManager: resolvedSessionManager, settingsManager, resourceLoader,
|
|
90
99
|
});
|
|
91
100
|
sessionBinding.set(session);
|
|
92
|
-
installModelPayloadDumper(session, modelContextDumper, () => currentModelCallKind, onLoggedModelPayload,
|
|
101
|
+
installModelPayloadDumper(session, modelContextDumper, () => currentModelCallKind, onLoggedModelPayload, providerPayloadTransform.transform);
|
|
93
102
|
}
|
|
94
103
|
syncEngineSessionState(engine, sessionBinding.get());
|
|
95
104
|
lifecycleAdapter = createMarchLifecycleAdapter({
|
|
@@ -113,6 +122,8 @@ export async function createRunner({ cwd, modelId = null, provider = null, provi
|
|
|
113
122
|
currentPromptForContext = prompt;
|
|
114
123
|
const contextMode = nextTurnContextMode;
|
|
115
124
|
currentTurnContextMode = contextMode;
|
|
125
|
+
providerPayloadTransform.resetTurn();
|
|
126
|
+
midTurnRecallBridge.reset();
|
|
116
127
|
nextTurnContextMode = "rebuild";
|
|
117
128
|
lifecycle.clearPendingAction();
|
|
118
129
|
const turnStartedAt = Date.now();
|
|
@@ -129,6 +140,7 @@ export async function createRunner({ cwd, modelId = null, provider = null, provi
|
|
|
129
140
|
autoNameSession,
|
|
130
141
|
contextMode,
|
|
131
142
|
recordHistory: (turn) => appendRunnerTurnHistory({ store: historyStore, turn, sessionStats: getRunnerSessionStats(sessionBinding.get(), runtimeHost), modelId: engine.modelId, provider: engine.provider }),
|
|
143
|
+
trackMidTurnRecallInjection: (injection) => midTurnRecallBridge.track(injection),
|
|
132
144
|
});
|
|
133
145
|
notifyTurnEndDetached(turnNotifier, {
|
|
134
146
|
status: "success",
|
|
@@ -256,7 +268,6 @@ export async function createRunner({ cwd, modelId = null, provider = null, provi
|
|
|
256
268
|
]);
|
|
257
269
|
},
|
|
258
270
|
};
|
|
259
|
-
return runner;
|
|
260
271
|
function syncCurrentMarchSessionState() {
|
|
261
272
|
return syncMarchSessionState({
|
|
262
273
|
enabled: syncPiSidecar || syncMarchSessionStateEnabled, projectMarchDir, engine,
|
|
@@ -285,14 +296,4 @@ export async function createRunner({ cwd, modelId = null, provider = null, provi
|
|
|
285
296
|
});
|
|
286
297
|
onModelPayload?.(event);
|
|
287
298
|
}
|
|
288
|
-
function injectMarchSystemContext(payload, { kind, model } = {}) {
|
|
289
|
-
if (kind !== "user") return payload;
|
|
290
|
-
let nextPayload = currentTurnContextMode === "continueExistingPiTranscript"
|
|
291
|
-
? payload
|
|
292
|
-
: replaceProviderContextMessages(payload, engine.buildProviderContext(currentPromptForContext));
|
|
293
|
-
nextPayload = injectHostedTools(nextPayload, model, hostedTools);
|
|
294
|
-
nextPayload = applyCodexLargeContextGuardToPayload(nextPayload, { model, session: sessionBinding.get() });
|
|
295
|
-
if (_currentFastEntry) nextPayload = { ...nextPayload, service_tier: "priority" };
|
|
296
|
-
return nextPayload;
|
|
297
|
-
}
|
|
298
299
|
}
|
|
@@ -13,7 +13,7 @@ export function createRemoteRuntimeUiClient(peer) {
|
|
|
13
13
|
retryEnd: (event) => peer.notify("uiEvent", { type: "retry_end", ...event }),
|
|
14
14
|
status: (text) => peer.notify("uiEvent", { type: "status", text }),
|
|
15
15
|
debugLines: (lines) => peer.notify("uiEvent", { type: "debug_lines", lines }),
|
|
16
|
-
recall: ({
|
|
16
|
+
recall: ({ hints, report, variant }) => peer.notify("uiEvent", { type: "recall", hints, report, variant }),
|
|
17
17
|
editDiff: (path, diffLines) => peer.notify("uiEvent", { type: "edit_diff", path, diffLines }),
|
|
18
18
|
};
|
|
19
19
|
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { DefaultResourceLoader } from "@earendil-works/pi-coding-agent";
|
|
2
|
+
|
|
3
|
+
export const MARCH_PI_RESOURCE_LOADER_OPTIONS = Object.freeze({
|
|
4
|
+
noContextFiles: true,
|
|
5
|
+
});
|
|
6
|
+
|
|
7
|
+
export async function createMarchPiResourceLoader({ cwd, agentDir, settingsManager, extraOptions = {} }) {
|
|
8
|
+
const resourceLoader = new DefaultResourceLoader({
|
|
9
|
+
...MARCH_PI_RESOURCE_LOADER_OPTIONS,
|
|
10
|
+
...extraOptions,
|
|
11
|
+
cwd,
|
|
12
|
+
agentDir,
|
|
13
|
+
settingsManager,
|
|
14
|
+
});
|
|
15
|
+
await resourceLoader.reload();
|
|
16
|
+
return resourceLoader;
|
|
17
|
+
}
|
|
@@ -2,6 +2,7 @@ import {
|
|
|
2
2
|
createAgentSessionFromServices,
|
|
3
3
|
createAgentSessionServices,
|
|
4
4
|
} from "@earendil-works/pi-coding-agent";
|
|
5
|
+
import { MARCH_PI_RESOURCE_LOADER_OPTIONS } from "./resource/context-resource-loader.mjs";
|
|
5
6
|
|
|
6
7
|
export function createMarchRuntimeFactory({
|
|
7
8
|
agentDir,
|
|
@@ -24,7 +25,10 @@ export function createMarchRuntimeFactory({
|
|
|
24
25
|
authStorage,
|
|
25
26
|
settingsManager,
|
|
26
27
|
modelRegistry,
|
|
27
|
-
resourceLoaderOptions
|
|
28
|
+
resourceLoaderOptions: {
|
|
29
|
+
...MARCH_PI_RESOURCE_LOADER_OPTIONS,
|
|
30
|
+
...resourceLoaderOptions,
|
|
31
|
+
},
|
|
28
32
|
});
|
|
29
33
|
const sessionOptions = await resolveSessionOptions({ cwd, services });
|
|
30
34
|
const result = await createFromServices({
|
|
@@ -13,6 +13,7 @@ export function createRunnerStateSnapshot(runner) {
|
|
|
13
13
|
remoteMemorySources: engine.remoteMemorySources ?? [],
|
|
14
14
|
turns: engine.turns ?? [],
|
|
15
15
|
pendingAssistantRecallHints: engine.peekPendingAssistantRecallHints?.() ?? engine.pendingAssistantRecallHints ?? [],
|
|
16
|
+
pendingAssistantRecallReport: engine.peekPendingAssistantRecallReport?.() ?? engine.pendingAssistantRecallReport ?? null,
|
|
16
17
|
pendingAssistantRecallHintsRendered: engine.hasRenderedPendingAssistantRecallHints?.() ?? engine.pendingAssistantRecallHintsRendered ?? false,
|
|
17
18
|
recentRecallMemoryIds: [...(engine.getRecentRecallMemoryIds?.() ?? [])],
|
|
18
19
|
},
|
|
@@ -41,23 +42,29 @@ export function createRunnerEngineStateFacade({ getState, setState }) {
|
|
|
41
42
|
peekPendingAssistantRecallHints() {
|
|
42
43
|
return engineState(getState()).pendingAssistantRecallHints ?? [];
|
|
43
44
|
},
|
|
45
|
+
peekPendingAssistantRecallReport() {
|
|
46
|
+
return engineState(getState()).pendingAssistantRecallReport ?? null;
|
|
47
|
+
},
|
|
44
48
|
hasRenderedPendingAssistantRecallHints() {
|
|
45
49
|
return Boolean(engineState(getState()).pendingAssistantRecallHintsRendered);
|
|
46
50
|
},
|
|
47
51
|
markPendingAssistantRecallHintsRendered() {
|
|
48
52
|
updateEngineState(getState, setState, (engine) => {
|
|
49
|
-
if ((engine.pendingAssistantRecallHints ?? []).length > 0) {
|
|
53
|
+
if ((engine.pendingAssistantRecallHints ?? []).length > 0 || engine.pendingAssistantRecallReport) {
|
|
50
54
|
engine.pendingAssistantRecallHintsRendered = true;
|
|
51
55
|
}
|
|
52
56
|
});
|
|
53
57
|
},
|
|
54
58
|
takePendingAssistantRecallHints() {
|
|
55
|
-
const
|
|
59
|
+
const state = engineState(getState());
|
|
60
|
+
const hints = state.pendingAssistantRecallHints ?? [];
|
|
61
|
+
const report = state.pendingAssistantRecallReport ?? null;
|
|
56
62
|
updateEngineState(getState, setState, (engine) => {
|
|
57
63
|
engine.pendingAssistantRecallHints = [];
|
|
64
|
+
engine.pendingAssistantRecallReport = null;
|
|
58
65
|
engine.pendingAssistantRecallHintsRendered = false;
|
|
59
66
|
});
|
|
60
|
-
return hints;
|
|
67
|
+
return { hints, report };
|
|
61
68
|
},
|
|
62
69
|
getRecentRecallMemoryIds() {
|
|
63
70
|
return engineState(getState()).recentRecallMemoryIds ?? [];
|