@phamvuhoang/otto-core 0.33.1 → 0.35.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cbm-index.d.ts +57 -0
- package/dist/cbm-index.d.ts.map +1 -0
- package/dist/cbm-index.js +62 -0
- package/dist/cbm-index.js.map +1 -0
- package/dist/cbm-inject.d.ts +50 -0
- package/dist/cbm-inject.d.ts.map +1 -0
- package/dist/cbm-inject.js +104 -0
- package/dist/cbm-inject.js.map +1 -0
- package/dist/codebase-memory-adapter.d.ts +15 -7
- package/dist/codebase-memory-adapter.d.ts.map +1 -1
- package/dist/codebase-memory-adapter.js +23 -7
- package/dist/codebase-memory-adapter.js.map +1 -1
- package/dist/context-report.d.ts.map +1 -1
- package/dist/context-report.js +3 -0
- package/dist/context-report.js.map +1 -1
- package/dist/eval-run.d.ts +6 -0
- package/dist/eval-run.d.ts.map +1 -1
- package/dist/eval-run.js +27 -5
- package/dist/eval-run.js.map +1 -1
- package/dist/eval.d.ts +5 -0
- package/dist/eval.d.ts.map +1 -1
- package/dist/eval.js +1 -1
- package/dist/eval.js.map +1 -1
- package/dist/extension-profiles.js +1 -1
- package/dist/extension-profiles.js.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/loop.d.ts +7 -0
- package/dist/loop.d.ts.map +1 -1
- package/dist/loop.js +220 -6
- package/dist/loop.js.map +1 -1
- package/package.json +1 -1
- package/templates/ghprompt-workflow.md +2 -0
- package/templates/prompt.md +2 -0
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import { type CbmIndexIdentity, type CbmRunner, type IndexFreshness, type WriteInventory } from "./codebase-memory-adapter.js";
|
|
2
|
+
export type IndexResult = {
|
|
3
|
+
ok: boolean;
|
|
4
|
+
identity?: CbmIndexIdentity;
|
|
5
|
+
fallbackReason?: string;
|
|
6
|
+
writeInventory: WriteInventory;
|
|
7
|
+
};
|
|
8
|
+
export type IndexInputs = {
|
|
9
|
+
runner: CbmRunner;
|
|
10
|
+
scratchDir: string;
|
|
11
|
+
/**
|
|
12
|
+
* Directory the before/after snapshot walks to detect confinement escapes.
|
|
13
|
+
* Must be broader than (or equal to) `scratchDir` — an escaped write that
|
|
14
|
+
* lands outside `scratchDir` is only visible if the snapshot actually walks
|
|
15
|
+
* past it. Defaults to `scratchDir` (preserves prior, scratch-only
|
|
16
|
+
* behavior) when omitted.
|
|
17
|
+
*/
|
|
18
|
+
watchDir?: string;
|
|
19
|
+
declaredRoots: string[];
|
|
20
|
+
snapshot: (dir: string) => string[];
|
|
21
|
+
identity: Omit<CbmIndexIdentity, "indexStatus">;
|
|
22
|
+
};
|
|
23
|
+
/**
|
|
24
|
+
* Run `index_repository` confined to `scratchDir`, then verify no write escaped
|
|
25
|
+
* the declared roots. The before/after snapshot walks `watchDir` (default
|
|
26
|
+
* `scratchDir`) rather than `scratchDir` itself, so a write that lands outside
|
|
27
|
+
* the declared roots — e.g. at the workspace root, if the underlying tool
|
|
28
|
+
* ignores `cacheDir` — is actually visible to the diff instead of silently
|
|
29
|
+
* escaping detection. Any escape (or a failed call) aborts: the index is not
|
|
30
|
+
* trusted and the caller falls back to normal search. Pure w.r.t. its injected
|
|
31
|
+
* `snapshot`/`runner` — the loop supplies fs-backed impls.
|
|
32
|
+
*/
|
|
33
|
+
export declare function runIndexRepository(inputs: IndexInputs): IndexResult;
|
|
34
|
+
export type IndexAction = {
|
|
35
|
+
action: "reuse" | "reindex";
|
|
36
|
+
freshness: IndexFreshness;
|
|
37
|
+
reason: string;
|
|
38
|
+
};
|
|
39
|
+
/**
|
|
40
|
+
* Decides whether a persisted index can be reused as-is or must be rebuilt,
|
|
41
|
+
* based on {@link classifyIndexFreshness}'s verdict against the current
|
|
42
|
+
* workspace state. Pure — no filesystem access.
|
|
43
|
+
*/
|
|
44
|
+
export declare function decideIndexAction(current: {
|
|
45
|
+
workspace: string;
|
|
46
|
+
sourceRevision: string;
|
|
47
|
+
worktreeDirty: boolean;
|
|
48
|
+
} | null, persisted: CbmIndexIdentity | null): IndexAction;
|
|
49
|
+
/**
|
|
50
|
+
* Gates whether index-derived context is safe to inject into a prompt: only
|
|
51
|
+
* a `fresh` index is trusted; anything else falls back with a reason.
|
|
52
|
+
*/
|
|
53
|
+
export declare function canInject(freshness: IndexFreshness): {
|
|
54
|
+
inject: boolean;
|
|
55
|
+
fallbackReason?: string;
|
|
56
|
+
};
|
|
57
|
+
//# sourceMappingURL=cbm-index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cbm-index.d.ts","sourceRoot":"","sources":["../src/cbm-index.ts"],"names":[],"mappings":"AAAA,OAAO,EAGL,KAAK,gBAAgB,EACrB,KAAK,SAAS,EACd,KAAK,cAAc,EACnB,KAAK,cAAc,EACpB,MAAM,8BAA8B,CAAC;AAEtC,MAAM,MAAM,WAAW,GAAG;IACxB,EAAE,EAAE,OAAO,CAAC;IACZ,QAAQ,CAAC,EAAE,gBAAgB,CAAC;IAC5B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,cAAc,CAAC;CAChC,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,MAAM,EAAE,SAAS,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,QAAQ,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,MAAM,EAAE,CAAC;IACpC,QAAQ,EAAE,IAAI,CAAC,gBAAgB,EAAE,aAAa,CAAC,CAAC;CACjD,CAAC;AAIF;;;;;;;;;GASG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,WAAW,GAAG,WAAW,CA4BnE;AAED,MAAM,MAAM,WAAW,GAAG;IACxB,MAAM,EAAE,OAAO,GAAG,SAAS,CAAC;IAC5B,SAAS,EAAE,cAAc,CAAC;IAC1B,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF;;;;GAIG;AACH,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE;IACP,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,OAAO,CAAC;CACxB,GAAG,IAAI,EACR,SAAS,EAAE,gBAAgB,GAAG,IAAI,GACjC,WAAW,CAKb;AAED;;;GAGG;AACH,wBAAgB,SAAS,CAAC,SAAS,EAAE,cAAc,GAAG;IACpD,MAAM,EAAE,OAAO,CAAC;IAChB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB,CAIA"}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { diffWriteInventory, classifyIndexFreshness, } from "./codebase-memory-adapter.js";
|
|
2
|
+
const EMPTY_INVENTORY = { files: [], escaped: [] };
|
|
3
|
+
/**
|
|
4
|
+
* Run `index_repository` confined to `scratchDir`, then verify no write escaped
|
|
5
|
+
* the declared roots. The before/after snapshot walks `watchDir` (default
|
|
6
|
+
* `scratchDir`) rather than `scratchDir` itself, so a write that lands outside
|
|
7
|
+
* the declared roots — e.g. at the workspace root, if the underlying tool
|
|
8
|
+
* ignores `cacheDir` — is actually visible to the diff instead of silently
|
|
9
|
+
* escaping detection. Any escape (or a failed call) aborts: the index is not
|
|
10
|
+
* trusted and the caller falls back to normal search. Pure w.r.t. its injected
|
|
11
|
+
* `snapshot`/`runner` — the loop supplies fs-backed impls.
|
|
12
|
+
*/
|
|
13
|
+
export function runIndexRepository(inputs) {
|
|
14
|
+
const watchDir = inputs.watchDir ?? inputs.scratchDir;
|
|
15
|
+
const before = inputs.snapshot(watchDir);
|
|
16
|
+
const res = inputs.runner.call({
|
|
17
|
+
operation: "index_repository",
|
|
18
|
+
params: { cacheDir: inputs.scratchDir },
|
|
19
|
+
});
|
|
20
|
+
if (!res.ok) {
|
|
21
|
+
return {
|
|
22
|
+
ok: false,
|
|
23
|
+
fallbackReason: `index_repository failed: ${res.error ?? "unknown"}`,
|
|
24
|
+
writeInventory: EMPTY_INVENTORY,
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
const after = inputs.snapshot(watchDir);
|
|
28
|
+
const inventory = diffWriteInventory(before, after, inputs.declaredRoots);
|
|
29
|
+
if (inventory.escaped.length > 0) {
|
|
30
|
+
return {
|
|
31
|
+
ok: false,
|
|
32
|
+
fallbackReason: `index writes escaped confinement: ${inventory.escaped.join(", ")}`,
|
|
33
|
+
writeInventory: inventory,
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
return {
|
|
37
|
+
ok: true,
|
|
38
|
+
identity: { ...inputs.identity, indexStatus: "ready" },
|
|
39
|
+
writeInventory: inventory,
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Decides whether a persisted index can be reused as-is or must be rebuilt,
|
|
44
|
+
* based on {@link classifyIndexFreshness}'s verdict against the current
|
|
45
|
+
* workspace state. Pure — no filesystem access.
|
|
46
|
+
*/
|
|
47
|
+
export function decideIndexAction(current, persisted) {
|
|
48
|
+
const freshness = classifyIndexFreshness(current, persisted);
|
|
49
|
+
return freshness === "fresh"
|
|
50
|
+
? { action: "reuse", freshness, reason: "index fresh" }
|
|
51
|
+
: { action: "reindex", freshness, reason: `index ${freshness}` };
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Gates whether index-derived context is safe to inject into a prompt: only
|
|
55
|
+
* a `fresh` index is trusted; anything else falls back with a reason.
|
|
56
|
+
*/
|
|
57
|
+
export function canInject(freshness) {
|
|
58
|
+
return freshness === "fresh"
|
|
59
|
+
? { inject: true }
|
|
60
|
+
: { inject: false, fallbackReason: `index ${freshness}` };
|
|
61
|
+
}
|
|
62
|
+
//# sourceMappingURL=cbm-index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cbm-index.js","sourceRoot":"","sources":["../src/cbm-index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,kBAAkB,EAClB,sBAAsB,GAKvB,MAAM,8BAA8B,CAAC;AAyBtC,MAAM,eAAe,GAAmB,EAAE,KAAK,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;AAEnE;;;;;;;;;GASG;AACH,MAAM,UAAU,kBAAkB,CAAC,MAAmB;IACpD,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,UAAU,CAAC;IACtD,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACzC,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;QAC7B,SAAS,EAAE,kBAAkB;QAC7B,MAAM,EAAE,EAAE,QAAQ,EAAE,MAAM,CAAC,UAAU,EAAE;KACxC,CAAC,CAAC;IACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;QACZ,OAAO;YACL,EAAE,EAAE,KAAK;YACT,cAAc,EAAE,4BAA4B,GAAG,CAAC,KAAK,IAAI,SAAS,EAAE;YACpE,cAAc,EAAE,eAAe;SAChC,CAAC;IACJ,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACxC,MAAM,SAAS,GAAG,kBAAkB,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,aAAa,CAAC,CAAC;IAC1E,IAAI,SAAS,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjC,OAAO;YACL,EAAE,EAAE,KAAK;YACT,cAAc,EAAE,qCAAqC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;YACnF,cAAc,EAAE,SAAS;SAC1B,CAAC;IACJ,CAAC;IACD,OAAO;QACL,EAAE,EAAE,IAAI;QACR,QAAQ,EAAE,EAAE,GAAG,MAAM,CAAC,QAAQ,EAAE,WAAW,EAAE,OAAO,EAAE;QACtD,cAAc,EAAE,SAAS;KAC1B,CAAC;AACJ,CAAC;AAQD;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAC/B,OAIQ,EACR,SAAkC;IAElC,MAAM,SAAS,GAAG,sBAAsB,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;IAC7D,OAAO,SAAS,KAAK,OAAO;QAC1B,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,aAAa,EAAE;QACvD,CAAC,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,SAAS,EAAE,EAAE,CAAC;AACrE,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,SAAS,CAAC,SAAyB;IAIjD,OAAO,SAAS,KAAK,OAAO;QAC1B,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE;QAClB,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,cAAc,EAAE,SAAS,SAAS,EAAE,EAAE,CAAC;AAC9D,CAAC"}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import type { CbmRequest, CbmRunner, IndexFreshness } from "./codebase-memory-adapter.js";
|
|
2
|
+
import type { RetrievalStore } from "./context-compressor.js";
|
|
3
|
+
import type { ToolUsage } from "./run-report.js";
|
|
4
|
+
/**
|
|
5
|
+
* Injection builder for the codebase-memory graph map (P26 slice2, Task 3).
|
|
6
|
+
* Picks the {@link CbmRequest}s a stage needs ({@link stageQueries}), then
|
|
7
|
+
* drives them through a {@link CbmRunner} and wraps the result in a bounded,
|
|
8
|
+
* navigation-only `<graph-map>` block ({@link buildCbmInjection}). Injection
|
|
9
|
+
* only happens when the index is `fresh` ({@link canInject}, Task 2) and every
|
|
10
|
+
* query succeeds — anything else degrades to an empty block plus a
|
|
11
|
+
* `fallbackReason` so the caller can record why nothing was injected. Pure
|
|
12
|
+
* w.r.t. the injected `runner`/`store`; no filesystem or process access here.
|
|
13
|
+
*/
|
|
14
|
+
export declare const GRAPH_BLOCK_TAG = "graph-map";
|
|
15
|
+
/**
|
|
16
|
+
* The {@link CbmRequest}s a given stage should ask the codebase-memory index
|
|
17
|
+
* for. `plan` wants the architecture overview; `implementer` searches the
|
|
18
|
+
* graph for the task at hand; `reviewer`/`verifier` want change-impact
|
|
19
|
+
* (detected changes + traced call paths) over the files that changed. Any
|
|
20
|
+
* other stage gets no queries — injection is opt-in per stage, not global.
|
|
21
|
+
*/
|
|
22
|
+
export declare function stageQueries(stage: string, ctx: {
|
|
23
|
+
changedFiles?: string[];
|
|
24
|
+
taskHint?: string;
|
|
25
|
+
}): CbmRequest[];
|
|
26
|
+
/** The rendered navigation-only block plus its evidence record. */
|
|
27
|
+
export type CbmInjection = {
|
|
28
|
+
block: string;
|
|
29
|
+
toolUsage: ToolUsage;
|
|
30
|
+
};
|
|
31
|
+
/**
|
|
32
|
+
* Runs `opts.requests` against `opts.runner` and, if the index is fresh and
|
|
33
|
+
* every query succeeds, wraps the concatenated results in a bounded
|
|
34
|
+
* `<graph-map>` block carrying the "read the actual source" navigation
|
|
35
|
+
* header. The full (unbounded) result is handed to `opts.store` — when
|
|
36
|
+
* provided — so a run report can retrieve it later via the returned
|
|
37
|
+
* `retrievalHandle`; only the `maxChars`-bounded copy is actually injected
|
|
38
|
+
* into the prompt. A non-fresh index, an empty request list, or any failed
|
|
39
|
+
* call short-circuits to an empty block plus a `fallbackReason`. Always
|
|
40
|
+
* returns a `ToolUsage` so the caller has evidence of the attempt either way.
|
|
41
|
+
*/
|
|
42
|
+
export declare function buildCbmInjection(opts: {
|
|
43
|
+
stage: string;
|
|
44
|
+
requests: CbmRequest[];
|
|
45
|
+
runner: CbmRunner;
|
|
46
|
+
freshness: IndexFreshness;
|
|
47
|
+
maxChars: number;
|
|
48
|
+
store?: RetrievalStore | null;
|
|
49
|
+
}): CbmInjection;
|
|
50
|
+
//# sourceMappingURL=cbm-inject.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cbm-inject.d.ts","sourceRoot":"","sources":["../src/cbm-inject.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,UAAU,EACV,SAAS,EACT,cAAc,EACf,MAAM,8BAA8B,CAAC;AAEtC,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AAC9D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAEjD;;;;;;;;;GASG;AACH,eAAO,MAAM,eAAe,cAAc,CAAC;AAM3C;;;;;;GAMG;AACH,wBAAgB,YAAY,CAC1B,KAAK,EAAE,MAAM,EACb,GAAG,EAAE;IAAE,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,GAClD,UAAU,EAAE,CAoBd;AAED,mEAAmE;AACnE,MAAM,MAAM,YAAY,GAAG;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,SAAS,CAAA;CAAE,CAAC;AAgBnE;;;;;;;;;;GAUG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE;IACtC,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,UAAU,EAAE,CAAC;IACvB,MAAM,EAAE,SAAS,CAAC;IAClB,SAAS,EAAE,cAAc,CAAC;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,cAAc,GAAG,IAAI,CAAC;CAC/B,GAAG,YAAY,CA2Cf"}
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import { canInject } from "./cbm-index.js";
|
|
2
|
+
/**
|
|
3
|
+
* Injection builder for the codebase-memory graph map (P26 slice2, Task 3).
|
|
4
|
+
* Picks the {@link CbmRequest}s a stage needs ({@link stageQueries}), then
|
|
5
|
+
* drives them through a {@link CbmRunner} and wraps the result in a bounded,
|
|
6
|
+
* navigation-only `<graph-map>` block ({@link buildCbmInjection}). Injection
|
|
7
|
+
* only happens when the index is `fresh` ({@link canInject}, Task 2) and every
|
|
8
|
+
* query succeeds — anything else degrades to an empty block plus a
|
|
9
|
+
* `fallbackReason` so the caller can record why nothing was injected. Pure
|
|
10
|
+
* w.r.t. the injected `runner`/`store`; no filesystem or process access here.
|
|
11
|
+
*/
|
|
12
|
+
export const GRAPH_BLOCK_TAG = "graph-map";
|
|
13
|
+
const NAV_HEADER = "Code-navigation map (local graph index, may be stale). Read the actual " +
|
|
14
|
+
"source before changing it; tests remain the gate.";
|
|
15
|
+
/**
|
|
16
|
+
* The {@link CbmRequest}s a given stage should ask the codebase-memory index
|
|
17
|
+
* for. `plan` wants the architecture overview; `implementer` searches the
|
|
18
|
+
* graph for the task at hand; `reviewer`/`verifier` want change-impact
|
|
19
|
+
* (detected changes + traced call paths) over the files that changed. Any
|
|
20
|
+
* other stage gets no queries — injection is opt-in per stage, not global.
|
|
21
|
+
*/
|
|
22
|
+
export function stageQueries(stage, ctx) {
|
|
23
|
+
switch (stage) {
|
|
24
|
+
case "plan":
|
|
25
|
+
return [{ operation: "get_architecture", params: {} }];
|
|
26
|
+
case "implementer":
|
|
27
|
+
return [
|
|
28
|
+
{ operation: "search_graph", params: { query: ctx.taskHint ?? "" } },
|
|
29
|
+
];
|
|
30
|
+
case "reviewer":
|
|
31
|
+
case "verifier":
|
|
32
|
+
return [
|
|
33
|
+
{
|
|
34
|
+
operation: "detect_changes",
|
|
35
|
+
params: { files: ctx.changedFiles ?? [] },
|
|
36
|
+
},
|
|
37
|
+
{ operation: "trace_path", params: { files: ctx.changedFiles ?? [] } },
|
|
38
|
+
];
|
|
39
|
+
default:
|
|
40
|
+
return [];
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
function usage(stage, freshness, over) {
|
|
44
|
+
return {
|
|
45
|
+
name: "codebase-memory",
|
|
46
|
+
kind: "mcp",
|
|
47
|
+
stage,
|
|
48
|
+
indexFreshness: freshness,
|
|
49
|
+
...over,
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Runs `opts.requests` against `opts.runner` and, if the index is fresh and
|
|
54
|
+
* every query succeeds, wraps the concatenated results in a bounded
|
|
55
|
+
* `<graph-map>` block carrying the "read the actual source" navigation
|
|
56
|
+
* header. The full (unbounded) result is handed to `opts.store` — when
|
|
57
|
+
* provided — so a run report can retrieve it later via the returned
|
|
58
|
+
* `retrievalHandle`; only the `maxChars`-bounded copy is actually injected
|
|
59
|
+
* into the prompt. A non-fresh index, an empty request list, or any failed
|
|
60
|
+
* call short-circuits to an empty block plus a `fallbackReason`. Always
|
|
61
|
+
* returns a `ToolUsage` so the caller has evidence of the attempt either way.
|
|
62
|
+
*/
|
|
63
|
+
export function buildCbmInjection(opts) {
|
|
64
|
+
const gate = canInject(opts.freshness);
|
|
65
|
+
const query = opts.requests.map((r) => r.operation).join(",");
|
|
66
|
+
if (!gate.inject || opts.requests.length === 0) {
|
|
67
|
+
return {
|
|
68
|
+
block: "",
|
|
69
|
+
toolUsage: usage(opts.stage, opts.freshness, {
|
|
70
|
+
query,
|
|
71
|
+
fallbackReason: gate.fallbackReason ?? "no query for stage",
|
|
72
|
+
}),
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
const parts = [];
|
|
76
|
+
for (const req of opts.requests) {
|
|
77
|
+
const res = opts.runner.call(req);
|
|
78
|
+
if (!res.ok) {
|
|
79
|
+
return {
|
|
80
|
+
block: "",
|
|
81
|
+
toolUsage: usage(opts.stage, opts.freshness, {
|
|
82
|
+
query,
|
|
83
|
+
fallbackReason: `query failed: ${res.error ?? "unknown"}`,
|
|
84
|
+
}),
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
parts.push(typeof res.result === "string" ? res.result : JSON.stringify(res.result));
|
|
88
|
+
}
|
|
89
|
+
const full = parts.join("\n");
|
|
90
|
+
const handle = opts.store
|
|
91
|
+
? opts.store(`graph-map-${opts.stage}`, full)
|
|
92
|
+
: undefined;
|
|
93
|
+
const bounded = full.length > opts.maxChars ? `${full.slice(0, opts.maxChars)}…` : full;
|
|
94
|
+
const block = `<${GRAPH_BLOCK_TAG}>\n${NAV_HEADER}\n\n${bounded}\n</${GRAPH_BLOCK_TAG}>`;
|
|
95
|
+
return {
|
|
96
|
+
block,
|
|
97
|
+
toolUsage: usage(opts.stage, opts.freshness, {
|
|
98
|
+
query,
|
|
99
|
+
resultSize: full.length,
|
|
100
|
+
retrievalHandle: handle,
|
|
101
|
+
}),
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
//# sourceMappingURL=cbm-inject.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cbm-inject.js","sourceRoot":"","sources":["../src/cbm-inject.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAI3C;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,WAAW,CAAC;AAE3C,MAAM,UAAU,GACd,yEAAyE;IACzE,mDAAmD,CAAC;AAEtD;;;;;;GAMG;AACH,MAAM,UAAU,YAAY,CAC1B,KAAa,EACb,GAAmD;IAEnD,QAAQ,KAAK,EAAE,CAAC;QACd,KAAK,MAAM;YACT,OAAO,CAAC,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC;QACzD,KAAK,aAAa;YAChB,OAAO;gBACL,EAAE,SAAS,EAAE,cAAc,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,QAAQ,IAAI,EAAE,EAAE,EAAE;aACrE,CAAC;QACJ,KAAK,UAAU,CAAC;QAChB,KAAK,UAAU;YACb,OAAO;gBACL;oBACE,SAAS,EAAE,gBAAgB;oBAC3B,MAAM,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,YAAY,IAAI,EAAE,EAAE;iBAC1C;gBACD,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,YAAY,IAAI,EAAE,EAAE,EAAE;aACvE,CAAC;QACJ;YACE,OAAO,EAAE,CAAC;IACd,CAAC;AACH,CAAC;AAKD,SAAS,KAAK,CACZ,KAAa,EACb,SAAyB,EACzB,IAAwB;IAExB,OAAO;QACL,IAAI,EAAE,iBAAiB;QACvB,IAAI,EAAE,KAAK;QACX,KAAK;QACL,cAAc,EAAE,SAAS;QACzB,GAAG,IAAI;KACR,CAAC;AACJ,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAOjC;IACC,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACvC,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC9D,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/C,OAAO;YACL,KAAK,EAAE,EAAE;YACT,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,SAAS,EAAE;gBAC3C,KAAK;gBACL,cAAc,EAAE,IAAI,CAAC,cAAc,IAAI,oBAAoB;aAC5D,CAAC;SACH,CAAC;IACJ,CAAC;IACD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;QAChC,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAClC,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,OAAO;gBACL,KAAK,EAAE,EAAE;gBACT,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,SAAS,EAAE;oBAC3C,KAAK;oBACL,cAAc,EAAE,iBAAiB,GAAG,CAAC,KAAK,IAAI,SAAS,EAAE;iBAC1D,CAAC;aACH,CAAC;QACJ,CAAC;QACD,KAAK,CAAC,IAAI,CACR,OAAO,GAAG,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CACzE,CAAC;IACJ,CAAC;IACD,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9B,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK;QACvB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,IAAI,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC;QAC7C,CAAC,CAAC,SAAS,CAAC;IACd,MAAM,OAAO,GACX,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;IAC1E,MAAM,KAAK,GAAG,IAAI,eAAe,MAAM,UAAU,OAAO,OAAO,OAAO,eAAe,GAAG,CAAC;IACzF,OAAO;QACL,KAAK;QACL,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,SAAS,EAAE;YAC3C,KAAK;YACL,UAAU,EAAE,IAAI,CAAC,MAAM;YACvB,eAAe,EAAE,MAAM;SACxB,CAAC;KACH,CAAC;AACJ,CAAC"}
|
|
@@ -13,7 +13,7 @@ import type { ToolDefinition } from "./tools.js";
|
|
|
13
13
|
* `stages: []` (no default injection into any stage template), `enabled:
|
|
14
14
|
* false` (registry-level off switch), `networkDomains: []` (no runtime
|
|
15
15
|
* network — the graph lives entirely in a local cache), and `writeRoots:
|
|
16
|
-
* [".
|
|
16
|
+
* [".otto/cbm-scratch"]` (writes confined to Otto's own scratch directory).
|
|
17
17
|
*/
|
|
18
18
|
/** One JSON-RPC request to the codebase-memory MCP child. */
|
|
19
19
|
export type CbmRequest = {
|
|
@@ -38,9 +38,10 @@ export type CbmRunner = {
|
|
|
38
38
|
/**
|
|
39
39
|
* The {@link ToolDefinition} a repo drops into `.otto/tools/codebase-memory.json`
|
|
40
40
|
* (mirrors {@link ./headroom-adapter.js}'s `headroomToolDefinition`). Declares
|
|
41
|
-
* no runtime network,
|
|
42
|
-
* (`operations`) that a
|
|
43
|
-
*
|
|
41
|
+
* no runtime network, writes confined to the Otto-owned scratch directory
|
|
42
|
+
* `.otto/cbm-scratch`, and the operation allowlist (`operations`) that a
|
|
43
|
+
* caller can gate calls against. `stages: []` and `enabled: false` keep it
|
|
44
|
+
* fully opt-in — a bare repo behaves as before.
|
|
44
45
|
*/
|
|
45
46
|
export declare function codebaseMemoryToolDefinition(command?: string): ToolDefinition;
|
|
46
47
|
/**
|
|
@@ -51,9 +52,16 @@ export declare function codebaseMemoryToolDefinition(command?: string): ToolDefi
|
|
|
51
52
|
* constraints as the Headroom runners. Only exercised under the gated e2e —
|
|
52
53
|
* unit tests inject a {@link CbmRunner} stub instead.
|
|
53
54
|
*/
|
|
55
|
+
/**
|
|
56
|
+
* Split a command string into argv, respecting single/double quotes so a
|
|
57
|
+
* binary path containing spaces (e.g. `"/opt/My Tools/codebase-memory" serve`)
|
|
58
|
+
* stays one token instead of being shredded by a naive `split(" ")`. Minimal:
|
|
59
|
+
* no escape sequences or environment expansion. Pure.
|
|
60
|
+
*/
|
|
61
|
+
export declare function tokenizeCommand(command: string): string[];
|
|
54
62
|
export declare function createStdioCbmRunner(command: string, cwd: string, timeoutMs: number): CbmRunner;
|
|
55
63
|
/**
|
|
56
|
-
* Identity stamp a persisted `.
|
|
64
|
+
* Identity stamp a persisted `.otto/cbm-scratch` index carries, so a later
|
|
57
65
|
* run can tell whether it's safe to trust without re-indexing: which
|
|
58
66
|
* workspace it was built for, the source revision it was built at, whether
|
|
59
67
|
* the worktree was dirty at build time, and tool/index bookkeeping.
|
|
@@ -86,9 +94,9 @@ export declare function classifyIndexFreshness(current: {
|
|
|
86
94
|
/**
|
|
87
95
|
* New files written during a run (`after \ before`) plus which of those
|
|
88
96
|
* escaped the tool's declared write roots — i.e. wrote outside
|
|
89
|
-
* `.
|
|
97
|
+
* `.otto/cbm-scratch` (see `writeRoots` on {@link codebaseMemoryToolDefinition}).
|
|
90
98
|
* A declared root matches itself exactly or any path under `root + "/"`, so
|
|
91
|
-
* a root doesn't accidentally prefix-match a sibling like `.
|
|
99
|
+
* a root doesn't accidentally prefix-match a sibling like `.otto/cbm-scratch-foo`.
|
|
92
100
|
*/
|
|
93
101
|
export type WriteInventory = {
|
|
94
102
|
files: string[];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"codebase-memory-adapter.d.ts","sourceRoot":"","sources":["../src/codebase-memory-adapter.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAiB,MAAM,YAAY,CAAC;AAEhE;;;;;;;;;;;;;;;GAeG;AAEH,6DAA6D;AAC7D,MAAM,MAAM,UAAU,GAAG;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACjC,CAAC;AAEF,6DAA6D;AAC7D,MAAM,MAAM,WAAW,GAAG;IAAE,EAAE,EAAE,OAAO,CAAC;IAAC,MAAM,CAAC,EAAE,OAAO,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAE5E;;;;GAIG;AACH,MAAM,MAAM,SAAS,GAAG;IACtB,SAAS,IAAI,OAAO,CAAC;IACrB,IAAI,CAAC,GAAG,EAAE,UAAU,GAAG,WAAW,CAAC;CACpC,CAAC;AAoBF
|
|
1
|
+
{"version":3,"file":"codebase-memory-adapter.d.ts","sourceRoot":"","sources":["../src/codebase-memory-adapter.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAiB,MAAM,YAAY,CAAC;AAEhE;;;;;;;;;;;;;;;GAeG;AAEH,6DAA6D;AAC7D,MAAM,MAAM,UAAU,GAAG;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACjC,CAAC;AAEF,6DAA6D;AAC7D,MAAM,MAAM,WAAW,GAAG;IAAE,EAAE,EAAE,OAAO,CAAC;IAAC,MAAM,CAAC,EAAE,OAAO,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAE5E;;;;GAIG;AACH,MAAM,MAAM,SAAS,GAAG;IACtB,SAAS,IAAI,OAAO,CAAC;IACrB,IAAI,CAAC,GAAG,EAAE,UAAU,GAAG,WAAW,CAAC;CACpC,CAAC;AAoBF;;;;;;;GAOG;AACH,wBAAgB,4BAA4B,CAC1C,OAAO,SAAoB,GAC1B,cAAc,CAwBhB;AAED;;;;;;;GAOG;AACH;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAQzD;AAED,wBAAgB,oBAAoB,CAClC,OAAO,EAAE,MAAM,EACf,GAAG,EAAE,MAAM,EACX,SAAS,EAAE,MAAM,GAChB,SAAS,CA0CX;AAED;;;;;GAKG;AACH,MAAM,MAAM,gBAAgB,GAAG;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,OAAO,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF;;;;;GAKG;AACH,MAAM,MAAM,cAAc,GAAG,OAAO,GAAG,OAAO,GAAG,QAAQ,GAAG,eAAe,CAAC;AAE5E;;;;GAIG;AACH,wBAAgB,sBAAsB,CACpC,OAAO,EAAE;IACP,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,OAAO,CAAC;CACxB,GAAG,IAAI,EACR,KAAK,EAAE,gBAAgB,GAAG,IAAI,GAC7B,cAAc,CAMhB;AAED;;;;;;GAMG;AACH,MAAM,MAAM,cAAc,GAAG;IAAE,KAAK,EAAE,MAAM,EAAE,CAAC;IAAC,OAAO,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC;AAEpE,kFAAkF;AAClF,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,MAAM,EAAE,EAChB,KAAK,EAAE,MAAM,EAAE,EACf,aAAa,EAAE,MAAM,EAAE,GACtB,cAAc,CAWhB"}
|
|
@@ -18,9 +18,10 @@ const OPERATIONS = [
|
|
|
18
18
|
/**
|
|
19
19
|
* The {@link ToolDefinition} a repo drops into `.otto/tools/codebase-memory.json`
|
|
20
20
|
* (mirrors {@link ./headroom-adapter.js}'s `headroomToolDefinition`). Declares
|
|
21
|
-
* no runtime network,
|
|
22
|
-
* (`operations`) that a
|
|
23
|
-
*
|
|
21
|
+
* no runtime network, writes confined to the Otto-owned scratch directory
|
|
22
|
+
* `.otto/cbm-scratch`, and the operation allowlist (`operations`) that a
|
|
23
|
+
* caller can gate calls against. `stages: []` and `enabled: false` keep it
|
|
24
|
+
* fully opt-in — a bare repo behaves as before.
|
|
24
25
|
*/
|
|
25
26
|
export function codebaseMemoryToolDefinition(command = "codebase-memory") {
|
|
26
27
|
return {
|
|
@@ -37,7 +38,7 @@ export function codebaseMemoryToolDefinition(command = "codebase-memory") {
|
|
|
37
38
|
command,
|
|
38
39
|
env: [],
|
|
39
40
|
networkDomains: [], // no runtime network
|
|
40
|
-
writeRoots: [".
|
|
41
|
+
writeRoots: [".otto/cbm-scratch"], // Otto-owned confined scratch
|
|
41
42
|
secretRefs: [],
|
|
42
43
|
timeoutMs: 120_000,
|
|
43
44
|
healthCheck: `${command} --version`,
|
|
@@ -54,8 +55,23 @@ export function codebaseMemoryToolDefinition(command = "codebase-memory") {
|
|
|
54
55
|
* constraints as the Headroom runners. Only exercised under the gated e2e —
|
|
55
56
|
* unit tests inject a {@link CbmRunner} stub instead.
|
|
56
57
|
*/
|
|
58
|
+
/**
|
|
59
|
+
* Split a command string into argv, respecting single/double quotes so a
|
|
60
|
+
* binary path containing spaces (e.g. `"/opt/My Tools/codebase-memory" serve`)
|
|
61
|
+
* stays one token instead of being shredded by a naive `split(" ")`. Minimal:
|
|
62
|
+
* no escape sequences or environment expansion. Pure.
|
|
63
|
+
*/
|
|
64
|
+
export function tokenizeCommand(command) {
|
|
65
|
+
const tokens = [];
|
|
66
|
+
const re = /"([^"]*)"|'([^']*)'|(\S+)/g;
|
|
67
|
+
let m;
|
|
68
|
+
while ((m = re.exec(command)) !== null) {
|
|
69
|
+
tokens.push(m[1] ?? m[2] ?? m[3]);
|
|
70
|
+
}
|
|
71
|
+
return tokens;
|
|
72
|
+
}
|
|
57
73
|
export function createStdioCbmRunner(command, cwd, timeoutMs) {
|
|
58
|
-
const [bin, ...args] = command
|
|
74
|
+
const [bin, ...args] = tokenizeCommand(command);
|
|
59
75
|
const available = () => {
|
|
60
76
|
const probe = spawnSync(bin, ["--version"], { cwd, timeout: 5000 });
|
|
61
77
|
return probe.status === 0;
|
|
@@ -117,8 +133,8 @@ export function diffWriteInventory(before, after, declaredRoots) {
|
|
|
117
133
|
const beforeSet = new Set(before);
|
|
118
134
|
const files = after.filter((f) => !beforeSet.has(f));
|
|
119
135
|
const under = (f) => declaredRoots.some((raw) => {
|
|
120
|
-
// Normalize a trailing slash so a root declared as `.
|
|
121
|
-
// classifies the bare directory the same as `.
|
|
136
|
+
// Normalize a trailing slash so a root declared as `.otto/cbm-scratch/`
|
|
137
|
+
// classifies the bare directory the same as `.otto/cbm-scratch` does.
|
|
122
138
|
const r = raw.replace(/\/+$/, "");
|
|
123
139
|
return f === r || f.startsWith(`${r}/`);
|
|
124
140
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"codebase-memory-adapter.js","sourceRoot":"","sources":["../src/codebase-memory-adapter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAuC/C,+DAA+D;AAC/D,MAAM,QAAQ,GAAG;IACf,cAAc;IACd,kBAAkB;IAClB,kBAAkB;IAClB,cAAc;IACd,YAAY;IACZ,gBAAgB;IAChB,aAAa;IACb,kBAAkB;CACnB,CAAC;AAEF,0EAA0E;AAC1E,MAAM,UAAU,GAAoB;IAClC,EAAE,IAAI,EAAE,kBAAkB,EAAE,KAAK,EAAE,IAAI,EAAE;IACzC,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;CACpD,CAAC;AAEF
|
|
1
|
+
{"version":3,"file":"codebase-memory-adapter.js","sourceRoot":"","sources":["../src/codebase-memory-adapter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAuC/C,+DAA+D;AAC/D,MAAM,QAAQ,GAAG;IACf,cAAc;IACd,kBAAkB;IAClB,kBAAkB;IAClB,cAAc;IACd,YAAY;IACZ,gBAAgB;IAChB,aAAa;IACb,kBAAkB;CACnB,CAAC;AAEF,0EAA0E;AAC1E,MAAM,UAAU,GAAoB;IAClC,EAAE,IAAI,EAAE,kBAAkB,EAAE,KAAK,EAAE,IAAI,EAAE;IACzC,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;CACpD,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,UAAU,4BAA4B,CAC1C,OAAO,GAAG,iBAAiB;IAE3B,OAAO;QACL,IAAI,EAAE,iBAAiB;QACvB,IAAI,EAAE,KAAK;QACX,WAAW,EACT,+DAA+D;QACjE,YAAY,EAAE;YACZ,cAAc;YACd,WAAW;YACX,eAAe;YACf,eAAe;SAChB;QACD,MAAM,EAAE,EAAE,EAAE,0CAA0C;QACtD,OAAO;QACP,GAAG,EAAE,EAAE;QACP,cAAc,EAAE,EAAE,EAAE,qBAAqB;QACzC,UAAU,EAAE,CAAC,mBAAmB,CAAC,EAAE,8BAA8B;QACjE,UAAU,EAAE,EAAE;QACd,SAAS,EAAE,OAAO;QAClB,WAAW,EAAE,GAAG,OAAO,YAAY;QACnC,eAAe,EAAE,EAAE;QACnB,OAAO,EAAE,KAAK;QACd,UAAU,EAAE,UAAU;KACvB,CAAC;AACJ,CAAC;AAED;;;;;;;GAOG;AACH;;;;;GAKG;AACH,MAAM,UAAU,eAAe,CAAC,OAAe;IAC7C,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,MAAM,EAAE,GAAG,4BAA4B,CAAC;IACxC,IAAI,CAAyB,CAAC;IAC9B,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACvC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACpC,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,OAAe,EACf,GAAW,EACX,SAAiB;IAEjB,MAAM,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;IAChD,MAAM,SAAS,GAAG,GAAG,EAAE;QACrB,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,EAAE,CAAC,WAAW,CAAC,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QACpE,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC;IAC5B,CAAC,CAAC;IACF,gFAAgF;IAChF,MAAM,IAAI,GAAG,CAAC,GAAe,EAAe,EAAE;QAC5C,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC;YAC1B,OAAO,EAAE,KAAK;YACd,EAAE,EAAE,CAAC;YACL,MAAM,EAAE,YAAY;YACpB,MAAM,EAAE,EAAE;SACX,CAAC,CAAC;QACH,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC;YAC7B,OAAO,EAAE,KAAK;YACd,EAAE,EAAE,CAAC;YACL,MAAM,EAAE,YAAY;YACpB,MAAM,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,SAAS,EAAE,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE;SACvD,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE;YAChC,GAAG;YACH,OAAO,EAAE,SAAS;YAClB,KAAK,EAAE,GAAG,IAAI,KAAK,OAAO,IAAI;SAC/B,CAAC,CAAC;QACH,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YACnB,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,CAAC,MAAM,EAAE,QAAQ,EAAE,IAAI,cAAc,EAAE,CAAC;QACzE,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACjE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAC7B,IAAI,GAAG,CAAC,EAAE,KAAK,CAAC,IAAI,QAAQ,IAAI,GAAG;oBACjC,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC;gBAC1C,IAAI,GAAG,CAAC,EAAE,KAAK,CAAC,IAAI,OAAO,IAAI,GAAG;oBAChC,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,IAAI,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YACxE,CAAC;YAAC,MAAM,CAAC;gBACP,kCAAkC;YACpC,CAAC;QACH,CAAC;QACD,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,4BAA4B,EAAE,CAAC;IAC5D,CAAC,CAAC;IACF,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;AAC7B,CAAC;AAyBD;;;;GAIG;AACH,MAAM,UAAU,sBAAsB,CACpC,OAIQ,EACR,KAA8B;IAE9B,IAAI,CAAC,KAAK,IAAI,CAAC,OAAO;QAAE,OAAO,QAAQ,CAAC;IACxC,IAAI,KAAK,CAAC,SAAS,KAAK,OAAO,CAAC,SAAS;QAAE,OAAO,eAAe,CAAC;IAClE,IAAI,KAAK,CAAC,cAAc,KAAK,OAAO,CAAC,cAAc,IAAI,OAAO,CAAC,aAAa;QAC1E,OAAO,OAAO,CAAC;IACjB,OAAO,OAAO,CAAC;AACjB,CAAC;AAWD,kFAAkF;AAClF,MAAM,UAAU,kBAAkB,CAChC,MAAgB,EAChB,KAAe,EACf,aAAuB;IAEvB,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;IAClC,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACrD,MAAM,KAAK,GAAG,CAAC,CAAS,EAAE,EAAE,CAC1B,aAAa,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE;QACzB,wEAAwE;QACxE,sEAAsE;QACtE,MAAM,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAClC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC1C,CAAC,CAAC,CAAC;IACL,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;AAC5D,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"context-report.d.ts","sourceRoot":"","sources":["../src/context-report.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAEL,KAAK,gBAAgB,EACtB,MAAM,wBAAwB,CAAC;AAEhC,MAAM,MAAM,eAAe,GACvB,SAAS,GACT,WAAW,GACX,QAAQ,GACR,UAAU,GACV,UAAU,CAAC;AAEf,MAAM,MAAM,cAAc,GAAG;IAC3B,QAAQ,EAAE,eAAe,CAAC;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,eAAe,EAAE,MAAM,CAAC;IACxB;;;;OAIG;IACH,SAAS,CAAC,EAAE,gBAAgB,CAAC;CAC9B,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;IACxB,oFAAoF;IACpF,QAAQ,EAAE,cAAc,EAAE,CAAC;CAC5B,CAAC;AAEF,qEAAqE;AACrE,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAEpD;
|
|
1
|
+
{"version":3,"file":"context-report.d.ts","sourceRoot":"","sources":["../src/context-report.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAEL,KAAK,gBAAgB,EACtB,MAAM,wBAAwB,CAAC;AAEhC,MAAM,MAAM,eAAe,GACvB,SAAS,GACT,WAAW,GACX,QAAQ,GACR,UAAU,GACV,UAAU,CAAC;AAEf,MAAM,MAAM,cAAc,GAAG;IAC3B,QAAQ,EAAE,eAAe,CAAC;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,eAAe,EAAE,MAAM,CAAC;IACxB;;;;OAIG;IACH,SAAS,CAAC,EAAE,gBAAgB,CAAC;CAC9B,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;IACxB,oFAAoF;IACpF,QAAQ,EAAE,cAAc,EAAE,CAAC;CAC5B,CAAC;AAEF,qEAAqE;AACrE,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAEpD;AAwCD;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,gBAAgB,CAyB/D;AAKD;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,CAAC,EAAE,gBAAgB,GAAG,MAAM,CAa/D"}
|
package/dist/context-report.js
CHANGED
|
@@ -34,6 +34,9 @@ const BLOCK_CATEGORY = {
|
|
|
34
34
|
issue: "evidence",
|
|
35
35
|
"issues-summary": "evidence",
|
|
36
36
|
"issues-full-file": "evidence",
|
|
37
|
+
// Injected code-graph navigation results (P26 slice2): retrievable evidence —
|
|
38
|
+
// the full result is stored via runRetrievalStore, so later prompts retire it.
|
|
39
|
+
"graph-map": "evidence",
|
|
37
40
|
};
|
|
38
41
|
/** Sum the char footprint of every recognized `<tag>…</tag>` span, by category. */
|
|
39
42
|
function recognizedSpans(prompt) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"context-report.js","sourceRoot":"","sources":["../src/context-report.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EACL,iBAAiB,GAElB,MAAM,wBAAwB,CAAC;AA4BhC,qEAAqE;AACrE,MAAM,UAAU,cAAc,CAAC,KAAa;IAC1C,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;AAC3C,CAAC;AAED;;;;;GAKG;AACH,MAAM,cAAc,GAAoC;IACtD,OAAO,EAAE,SAAS;IAClB,SAAS,EAAE,WAAW;IACtB,MAAM,EAAE,QAAQ;IAChB,sEAAsE;IACtE,6EAA6E;IAC7E,wEAAwE;IACxE,+EAA+E;IAC/E,KAAK,EAAE,UAAU;IACjB,gBAAgB,EAAE,UAAU;IAC5B,kBAAkB,EAAE,UAAU;
|
|
1
|
+
{"version":3,"file":"context-report.js","sourceRoot":"","sources":["../src/context-report.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EACL,iBAAiB,GAElB,MAAM,wBAAwB,CAAC;AA4BhC,qEAAqE;AACrE,MAAM,UAAU,cAAc,CAAC,KAAa;IAC1C,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;AAC3C,CAAC;AAED;;;;;GAKG;AACH,MAAM,cAAc,GAAoC;IACtD,OAAO,EAAE,SAAS;IAClB,SAAS,EAAE,WAAW;IACtB,MAAM,EAAE,QAAQ;IAChB,sEAAsE;IACtE,6EAA6E;IAC7E,wEAAwE;IACxE,+EAA+E;IAC/E,KAAK,EAAE,UAAU;IACjB,gBAAgB,EAAE,UAAU;IAC5B,kBAAkB,EAAE,UAAU;IAC9B,8EAA8E;IAC9E,+EAA+E;IAC/E,WAAW,EAAE,UAAU;CACxB,CAAC;AAEF,mFAAmF;AACnF,SAAS,eAAe,CAAC,MAAc;IACrC,MAAM,UAAU,GAAG,IAAI,GAAG,EAA2B,CAAC;IACtD,KAAK,MAAM,CAAC,GAAG,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,CAAC;QAC7D,6EAA6E;QAC7E,MAAM,EAAE,GAAG,IAAI,MAAM,CAAC,IAAI,GAAG,gBAAgB,GAAG,GAAG,EAAE,GAAG,CAAC,CAAC;QAC1D,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC;YACxC,UAAU,CAAC,GAAG,CACZ,QAAQ,EACR,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAClD,CAAC;QACJ,CAAC;IACH,CAAC;IACD,OAAO,UAAU,CAAC;AACpB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAAC,MAAc;IAC3C,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC;IACjC,MAAM,UAAU,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC;IAE3C,MAAM,eAAe,GAAG,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5E,MAAM,QAAQ,GAAG,UAAU,GAAG,eAAe,CAAC;IAC9C,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;QACjB,UAAU,CAAC,GAAG,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;IACvC,CAAC;IAED,MAAM,QAAQ,GAAqB,CAAC,GAAG,UAAU,CAAC,OAAO,EAAE,CAAC;SACzD,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,GAAG,CAAC,CAAC;SAChC,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC;QAC3B,QAAQ;QACR,KAAK;QACL,eAAe,EAAE,cAAc,CAAC,KAAK,CAAC;QACtC,SAAS,EAAE,iBAAiB,CAAC,QAAQ,CAAC;KACvC,CAAC,CAAC;SACF,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IAErC,OAAO;QACL,UAAU;QACV,eAAe,EAAE,cAAc,CAAC,UAAU,CAAC;QAC3C,QAAQ;KACT,CAAC;AACJ,CAAC;AAED,MAAM,GAAG,GAAG,IAAI,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,EAAE,qBAAqB,EAAE,CAAC,EAAE,CAAC,CAAC;AACzE,MAAM,GAAG,GAAG,IAAI,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;AAE3C;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CAAC,CAAmB;IACrD,MAAM,MAAM,GAAG,YAAY,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,YAAY,GAAG,CAAC,MAAM,CACvE,CAAC,CAAC,eAAe,CAClB,UAAU,CAAC;IACZ,IAAI,CAAC,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC;IAE3C,MAAM,KAAK,GAAG,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACjC,MAAM,KAAK,GAAG,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACpE,OAAO,KAAK,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,YAAY,GAAG,CAAC,MAAM,CAC5E,CAAC,CAAC,eAAe,CAClB,aAAa,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC;IACrC,CAAC,CAAC,CAAC;IACH,OAAO,CAAC,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACvC,CAAC"}
|
package/dist/eval-run.d.ts
CHANGED
|
@@ -43,6 +43,12 @@ export type EvalDeps = {
|
|
|
43
43
|
readManifest: (workspaceDir: string, runId: string) => RunManifest | null;
|
|
44
44
|
readStageRecords: (workspaceDir: string, runId: string) => StageRecord[];
|
|
45
45
|
runChecks: (checks: BenchmarkCheck[], cwd: string) => CheckResult[];
|
|
46
|
+
/** Known impacted-file set a fixture declares (its `impact.json`); `[]` when
|
|
47
|
+
* the fixture declares none, which leaves impact recall at 0 (P26). */
|
|
48
|
+
readImpactSet: (fixtureDir: string) => string[];
|
|
49
|
+
/** The run's answer text (its rendered report) impact recall is scored
|
|
50
|
+
* against; `""` when no report was written (P26). */
|
|
51
|
+
readReportText: (fixtureDir: string, runId: string) => string;
|
|
46
52
|
};
|
|
47
53
|
/** Injectable host surface for {@link runEvalCompare} (mirrors `InspectDeps`). */
|
|
48
54
|
export type CompareDeps = {
|
package/dist/eval-run.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eval-run.d.ts","sourceRoot":"","sources":["../src/eval-run.ts"],"names":[],"mappings":"AAIA,OAAO,EAIL,KAAK,cAAc,EACnB,KAAK,aAAa,EAClB,KAAK,WAAW,EACjB,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"eval-run.d.ts","sourceRoot":"","sources":["../src/eval-run.ts"],"names":[],"mappings":"AAIA,OAAO,EAIL,KAAK,cAAc,EACnB,KAAK,aAAa,EAClB,KAAK,WAAW,EACjB,MAAM,YAAY,CAAC;AAMpB,OAAO,EAKL,KAAK,WAAW,EAChB,KAAK,WAAW,EACjB,MAAM,iBAAiB,CAAC;AAEzB,2EAA2E;AAC3E,MAAM,MAAM,UAAU,GAAG;IACvB,+DAA+D;IAC/D,KAAK,EAAE,MAAM,CAAC;IACd,6DAA6D;IAC7D,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,0DAA0D;IAC1D,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC7B,CAAC;AAEF,gFAAgF;AAChF,MAAM,MAAM,cAAc,GAAG;IAC3B,IAAI,EAAE,aAAa,CAAC;IACpB,MAAM,EAAE,UAAU,CAAC;IACnB,0DAA0D;IAC1D,UAAU,EAAE,MAAM,CAAC;IACnB,qDAAqD;IACrD,GAAG,EAAE,MAAM,CAAC;IACZ,+BAA+B;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,8DAA8D;IAC9D,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,gEAAgE;IAChE,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC7B,CAAC;AAEF;;;;;GAKG;AACH,MAAM,MAAM,WAAW,GAAG,CAAC,GAAG,EAAE,cAAc,KAAK,OAAO,CAAC;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC;AAE9E,mDAAmD;AACnD,MAAM,MAAM,QAAQ,GAAG;IACrB,GAAG,EAAE,MAAM,CAAC,UAAU,CAAC;IACvB,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;IAC3B,GAAG,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;IAC3B,MAAM,EAAE,WAAW,CAAC;IACpB,YAAY,EAAE,CAAC,YAAY,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,WAAW,GAAG,IAAI,CAAC;IAC1E,gBAAgB,EAAE,CAAC,YAAY,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,WAAW,EAAE,CAAC;IACzE,SAAS,EAAE,CAAC,MAAM,EAAE,cAAc,EAAE,EAAE,GAAG,EAAE,MAAM,KAAK,WAAW,EAAE,CAAC;IACpE;4EACwE;IACxE,aAAa,EAAE,CAAC,UAAU,EAAE,MAAM,KAAK,MAAM,EAAE,CAAC;IAChD;0DACsD;IACtD,cAAc,EAAE,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,MAAM,CAAC;CAC/D,CAAC;AAkDF,kFAAkF;AAClF,MAAM,MAAM,WAAW,GAAG;IACxB,GAAG,EAAE,MAAM,CAAC,UAAU,CAAC;IACvB,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;IAC3B,GAAG,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;CAC5B,CAAC;AAoBF;;;;;;;GAOG;AACH,wBAAsB,cAAc,CAClC,IAAI,EAAE,MAAM,EAAE,EACd,IAAI,GAAE,WAAgC,GACrC,OAAO,CAAC,MAAM,CAAC,CAqCjB;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,OAAO,GAAG,UAAU,EAAE,CAkC3D;AAuBD;;;;;;;GAOG;AACH,wBAAsB,OAAO,CAC3B,IAAI,EAAE,MAAM,EAAE,EACd,IAAI,GAAE,QAAsB,GAC3B,OAAO,CAAC,MAAM,CAAC,CA8GjB"}
|
package/dist/eval-run.js
CHANGED
|
@@ -2,8 +2,8 @@ import { spawn } from "node:child_process";
|
|
|
2
2
|
import { readFileSync } from "node:fs";
|
|
3
3
|
import { dirname, resolve } from "node:path";
|
|
4
4
|
import { evaluateExpectation, readBenchmarkSuite, runFixtureChecks, } from "./bench.js";
|
|
5
|
-
import { compareTrajectories, scoreTrajectory } from "./eval.js";
|
|
6
|
-
import { listRunIds, readManifest as readManifestFs, readStageRecords as readStageRecordsFs, } from "./run-report.js";
|
|
5
|
+
import { compareTrajectories, scoreImpactRecall, scoreTrajectory, } from "./eval.js";
|
|
6
|
+
import { listRunIds, readManifest as readManifestFs, readRunReport, readStageRecords as readStageRecordsFs, } from "./run-report.js";
|
|
7
7
|
const defaultInvoke = async (inv) => {
|
|
8
8
|
const argv = inv.bin === "otto-afk"
|
|
9
9
|
? [inv.task.inputs, String(inv.iterations), ...inv.args]
|
|
@@ -29,6 +29,18 @@ const defaultDeps = {
|
|
|
29
29
|
readManifest: readManifestFs,
|
|
30
30
|
readStageRecords: readStageRecordsFs,
|
|
31
31
|
runChecks: runFixtureChecks,
|
|
32
|
+
readImpactSet: (fixtureDir) => {
|
|
33
|
+
try {
|
|
34
|
+
const parsed = JSON.parse(readFileSync(resolve(fixtureDir, "impact.json"), "utf8"));
|
|
35
|
+
return Array.isArray(parsed)
|
|
36
|
+
? parsed.filter((f) => typeof f === "string")
|
|
37
|
+
: [];
|
|
38
|
+
}
|
|
39
|
+
catch {
|
|
40
|
+
return [];
|
|
41
|
+
}
|
|
42
|
+
},
|
|
43
|
+
readReportText: (fixtureDir, runId) => readRunReport(fixtureDir, runId) ?? "",
|
|
32
44
|
};
|
|
33
45
|
const USAGE = "Usage: otto-eval <suite.json> [<configs.json>] [--iterations <n>]\n" +
|
|
34
46
|
" otto-eval compare <run-a> <run-b>";
|
|
@@ -103,11 +115,13 @@ export function parseEvalConfigs(raw) {
|
|
|
103
115
|
throw new Error(`eval config [${i}]: 'label' must be a non-empty string`);
|
|
104
116
|
}
|
|
105
117
|
const args = rec.args;
|
|
106
|
-
if (args !== undefined &&
|
|
118
|
+
if (args !== undefined &&
|
|
119
|
+
(!Array.isArray(args) || args.some((a) => typeof a !== "string"))) {
|
|
107
120
|
throw new Error(`eval config '${rec.label}': 'args' must be an array of strings`);
|
|
108
121
|
}
|
|
109
122
|
const env = rec.env;
|
|
110
|
-
if (env !== undefined &&
|
|
123
|
+
if (env !== undefined &&
|
|
124
|
+
(env == null || typeof env !== "object" || Array.isArray(env))) {
|
|
111
125
|
throw new Error(`eval config '${rec.label}': 'env' must be an object`);
|
|
112
126
|
}
|
|
113
127
|
return {
|
|
@@ -203,7 +217,15 @@ export async function runEval(argv, deps = defaultDeps) {
|
|
|
203
217
|
verdictLines.push(` - ${config.label}: FAIL (no evidence bundle for run '${runId}')`);
|
|
204
218
|
continue;
|
|
205
219
|
}
|
|
206
|
-
|
|
220
|
+
// Impact recall only applies to fixtures that declare a known
|
|
221
|
+
// impacted-file set; otherwise it stays 0 (scoreImpactRecall is
|
|
222
|
+
// vacuously 1 for an empty set, which would wrongly inflate it).
|
|
223
|
+
const impactSet = deps.readImpactSet(fixtureDir);
|
|
224
|
+
const signals = scoreTrajectory(manifest, deps.readStageRecords(fixtureDir, runId), impactSet.length > 0
|
|
225
|
+
? {
|
|
226
|
+
impactRecall: scoreImpactRecall(impactSet, deps.readReportText(fixtureDir, runId)),
|
|
227
|
+
}
|
|
228
|
+
: {});
|
|
207
229
|
const checks = deps.runChecks(task.expect.checks ?? [], fixtureDir);
|
|
208
230
|
const verdict = evaluateExpectation(task.expect, signals, checks);
|
|
209
231
|
if (!verdict.passed)
|
package/dist/eval-run.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eval-run.js","sourceRoot":"","sources":["../src/eval-run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAE7C,OAAO,EACL,mBAAmB,EACnB,kBAAkB,EAClB,gBAAgB,GAIjB,MAAM,YAAY,CAAC;AACpB,OAAO,
|
|
1
|
+
{"version":3,"file":"eval-run.js","sourceRoot":"","sources":["../src/eval-run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAE7C,OAAO,EACL,mBAAmB,EACnB,kBAAkB,EAClB,gBAAgB,GAIjB,MAAM,YAAY,CAAC;AACpB,OAAO,EACL,mBAAmB,EACnB,iBAAiB,EACjB,eAAe,GAChB,MAAM,WAAW,CAAC;AACnB,OAAO,EACL,UAAU,EACV,YAAY,IAAI,cAAc,EAC9B,aAAa,EACb,gBAAgB,IAAI,kBAAkB,GAGvC,MAAM,iBAAiB,CAAC;AAsDzB,MAAM,aAAa,GAAgB,KAAK,EAAE,GAAG,EAAE,EAAE;IAC/C,MAAM,IAAI,GACR,GAAG,CAAC,GAAG,KAAK,UAAU;QACpB,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC;QACxD,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC;IAC5C,MAAM,IAAI,OAAO,CAAO,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;QACnC,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,EAAE;YACjC,GAAG,EAAE,GAAG,CAAC,UAAU;YACnB,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,GAAG,GAAG,CAAC,GAAG,EAAE;YACnC,KAAK,EAAE,SAAS;SACjB,CAAC,CAAC;QACH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;QACvB,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC;IACjC,CAAC,CAAC,CAAC;IACH,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IACvC,OAAO,EAAE,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;AAC9C,CAAC,CAAC;AAEF,MAAM,WAAW,GAAa;IAC5B,GAAG,EAAE,OAAO,CAAC,GAAG;IAChB,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE;IAClB,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC;IAC1C,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC;IAC1C,MAAM,EAAE,aAAa;IACrB,YAAY,EAAE,cAAc;IAC5B,gBAAgB,EAAE,kBAAkB;IACpC,SAAS,EAAE,gBAAgB;IAC3B,aAAa,EAAE,CAAC,UAAU,EAAE,EAAE;QAC5B,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CACvB,YAAY,CAAC,OAAO,CAAC,UAAU,EAAE,aAAa,CAAC,EAAE,MAAM,CAAC,CACzD,CAAC;YACF,OAAO,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;gBAC1B,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC;gBAC1D,CAAC,CAAC,EAAE,CAAC;QACT,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IACD,cAAc,EAAE,CAAC,UAAU,EAAE,KAAK,EAAE,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,KAAK,CAAC,IAAI,EAAE;CAC9E,CAAC;AAEF,MAAM,KAAK,GACT,qEAAqE;IACrE,0CAA0C,CAAC;AAE7C,MAAM,aAAa,GAAG,wDAAwD,CAAC;AAU/E,MAAM,kBAAkB,GAAgB;IACtC,GAAG,EAAE,OAAO,CAAC,GAAG;IAChB,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE;IAClB,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC;IAC1C,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC;CAC3C,CAAC;AAEF;;;;GAIG;AACH,SAAS,YAAY,CAAC,YAAoB,EAAE,GAAW;IACrD,IAAI,GAAG,KAAK,QAAQ;QAAE,OAAO,GAAG,CAAC;IACjC,MAAM,GAAG,GAAG,UAAU,CAAC,YAAY,CAAC,CAAC;IACrC,OAAO,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACjD,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,IAAc,EACd,OAAoB,kBAAkB;IAEtC,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,QAAQ,EAAE,CAAC;QAC7C,IAAI,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;QACxB,OAAO,CAAC,CAAC;IACX,CAAC;IACD,MAAM,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,KAAK,CAAC,GAAG,IAAI,CAAC;IACpC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvC,IAAI,CAAC,GAAG,CAAC,uCAAuC,aAAa,EAAE,CAAC,CAAC;QACjE,OAAO,CAAC,CAAC;IACX,CAAC;IAED,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,cAAc,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC;IAClE,MAAM,QAAQ,GAGR,EAAE,CAAC;IACT,KAAK,MAAM,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,YAAY,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC;QAC9C,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,cAAc,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACpE,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,IAAI,CAAC,GAAG,CACN,+BAA+B,GAAG,WAAW,YAAY,gBAAgB;gBACvE,sCAAsC,CACzC,CAAC;YACF,OAAO,CAAC,CAAC;QACX,CAAC;QACD,QAAQ,CAAC,IAAI,CAAC;YACZ,KAAK,EAAE,KAAM;YACb,OAAO,EAAE,eAAe,CACtB,QAAQ,EACR,kBAAkB,CAAC,YAAY,EAAE,KAAM,CAAC,CACzC;SACF,CAAC,CAAC;IACL,CAAC;IAED,IAAI,CAAC,GAAG,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC,CAAC;IACxC,OAAO,CAAC,CAAC;AACX,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,GAAY;IAC3C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;IAC7E,CAAC;IACD,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACtB,IAAI,CAAC,IAAI,IAAI,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3D,MAAM,IAAI,KAAK,CAAC,gBAAgB,CAAC,uBAAuB,CAAC,CAAC;QAC5D,CAAC;QACD,MAAM,GAAG,GAAG,CAA4B,CAAC;QACzC,IAAI,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5D,MAAM,IAAI,KAAK,CAAC,gBAAgB,CAAC,uCAAuC,CAAC,CAAC;QAC5E,CAAC;QACD,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;QACtB,IACE,IAAI,KAAK,SAAS;YAClB,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,EACjE,CAAC;YACD,MAAM,IAAI,KAAK,CACb,gBAAgB,GAAG,CAAC,KAAK,uCAAuC,CACjE,CAAC;QACJ,CAAC;QACD,MAAM,GAAG,GAAG,GAAG,CAAC,GAAG,CAAC;QACpB,IACE,GAAG,KAAK,SAAS;YACjB,CAAC,GAAG,IAAI,IAAI,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,EAC9D,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,gBAAgB,GAAG,CAAC,KAAK,4BAA4B,CAAC,CAAC;QACzE,CAAC;QACD,OAAO;YACL,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,IAAI,EAAG,IAAiB,IAAI,EAAE;YAC9B,GAAG,EAAG,GAA8B,IAAI,EAAE;SAC3C,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AASD,SAAS,SAAS,CAAC,IAAc;IAC/B,MAAM,MAAM,GAAe,EAAE,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;IAC1D,MAAM,WAAW,GAAa,EAAE,CAAC;IACjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,QAAQ;YAAE,MAAM,CAAC,IAAI,GAAG,IAAI,CAAC;aAChD,IAAI,CAAC,KAAK,cAAc;YAAE,MAAM,CAAC,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;;YAChE,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC3B,CAAC;IACD,MAAM,CAAC,SAAS,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;IAClC,MAAM,CAAC,WAAW,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;IACpC,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,IAAc,EACd,OAAiB,WAAW;IAE5B,4EAA4E;IAC5E,wEAAwE;IACxE,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,SAAS,EAAE,CAAC;QAC1B,OAAO,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;YACnC,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,GAAG,EAAE,IAAI,CAAC,GAAG;SACd,CAAC,CAAC;IACL,CAAC;IAED,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAC7B,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;QACd,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAChB,OAAO,CAAC,CAAC;IACX,CAAC;IACD,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;QACpB,IAAI,CAAC,GAAG,CAAC,8BAA8B,KAAK,EAAE,CAAC,CAAC;QAChD,OAAO,CAAC,CAAC;IACX,CAAC;IAED,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACpD,IAAI,KAAsB,CAAC;IAC3B,IAAI,CAAC;QACH,KAAK,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAC;IACxC,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAI,CAAC,GAAG,CAAE,CAAW,CAAC,OAAO,CAAC,CAAC;QAC/B,OAAO,CAAC,CAAC;IACX,CAAC;IAED,IAAI,OAAO,GAAiB,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC;IACtE,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;QACrB,IAAI,CAAC;YACH,OAAO,GAAG,gBAAgB,CACxB,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC,CAAC,CACtE,CAAC;QACJ,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,CAAC,GAAG,CAAC,gBAAgB,IAAI,CAAC,WAAW,KAAM,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;YACtE,OAAO,CAAC,CAAC;QACX,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;IACpC,IAAI,SAAS,GAAG,IAAI,CAAC;IAErB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,UAAU,GAAG,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QACnD,MAAM,QAAQ,GAGR,EAAE,CAAC;QACT,MAAM,YAAY,GAAa,EAAE,CAAC;QAElC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,GAAG,GAAmB;gBAC1B,IAAI;gBACJ,MAAM;gBACN,UAAU;gBACV,GAAG,EAAE,IAAI,CAAC,GAAG;gBACb,UAAU,EAAE,IAAI,CAAC,UAAU;gBAC3B,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC;gBACpC,GAAG,EAAE,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,GAAG,EAAE;aACpC,CAAC;YACF,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;YACtD,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,SAAS,GAAG,KAAK,CAAC;gBAClB,YAAY,CAAC,IAAI,CACf,OAAO,MAAM,CAAC,KAAK,uCAAuC,KAAK,IAAI,CACpE,CAAC;gBACF,SAAS;YACX,CAAC;YACD,8DAA8D;YAC9D,gEAAgE;YAChE,iEAAiE;YACjE,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;YACjD,MAAM,OAAO,GAAG,eAAe,CAC7B,QAAQ,EACR,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,KAAK,CAAC,EACxC,SAAS,CAAC,MAAM,GAAG,CAAC;gBAClB,CAAC,CAAC;oBACE,YAAY,EAAE,iBAAiB,CAC7B,SAAS,EACT,IAAI,CAAC,cAAc,CAAC,UAAU,EAAE,KAAK,CAAC,CACvC;iBACF;gBACH,CAAC,CAAC,EAAE,CACP,CAAC;YACF,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,EAAE,UAAU,CAAC,CAAC;YACpE,MAAM,OAAO,GAAG,mBAAmB,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;YAClE,IAAI,CAAC,OAAO,CAAC,MAAM;gBAAE,SAAS,GAAG,KAAK,CAAC;YACvC,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;YAChD,YAAY,CAAC,IAAI,CACf,OAAO,CAAC,MAAM;gBACZ,CAAC,CAAC,OAAO,MAAM,CAAC,KAAK,QAAQ;gBAC7B,CAAC,CAAC,OAAO,MAAM,CAAC,KAAK,WAAW,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CACjE,CAAC;QACJ,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,MAAM,IAAI,CAAC,EAAE,KAAK,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC;QACzC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACb,IAAI,CAAC,GAAG,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC,CAAC;QACxC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACb,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QACtB,KAAK,MAAM,IAAI,IAAI,YAAY;YAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAChD,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACf,CAAC;IAED,OAAO,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC3B,CAAC"}
|
package/dist/eval.d.ts
CHANGED
|
@@ -80,6 +80,11 @@ export type EvalSignals = {
|
|
|
80
80
|
export declare function scoreTrajectory(manifest: RunManifest, stages: StageRecord[], opts?: {
|
|
81
81
|
planScore?: PlanRubricScore;
|
|
82
82
|
reportScore?: ReportRubricScore;
|
|
83
|
+
/** Precomputed impact recall for fixtures that declare a known impacted-file
|
|
84
|
+
* set (P26). The trajectory alone can't derive it — the eval harness scores
|
|
85
|
+
* it via {@link scoreImpactRecall} against the fixture's impact list and the
|
|
86
|
+
* run's answer text, then threads it in here. Absent ⇒ 0 (not applicable). */
|
|
87
|
+
impactRecall?: number;
|
|
83
88
|
}): EvalSignals;
|
|
84
89
|
/**
|
|
85
90
|
* Score how many `knownImpactedFiles` a run's answer text surfaced — a thin
|