@tangle-network/traces 0.11.4 → 0.11.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +61 -2
- package/dist/{chunk-AGPOIWPQ.js → chunk-MEAEXKUV.js} +2475 -316
- package/dist/cli.js +226 -65
- package/dist/index.d.ts +738 -4
- package/dist/index.js +68 -74
- package/docs/replay-verify.md +17 -0
- package/docs/trace-analysts.md +67 -1
- package/examples/external-engines.ts +14 -1
- package/package.json +6 -5
- package/skills/build-trace-analyst/SKILL.md +13 -0
- package/skills/inspect-agent-traces/SKILL.md +30 -4
|
@@ -3,6 +3,145 @@ function appendAll(target, values) {
|
|
|
3
3
|
for (const value of values) target.push(value);
|
|
4
4
|
}
|
|
5
5
|
|
|
6
|
+
// src/attributes.ts
|
|
7
|
+
var ATTR = {
|
|
8
|
+
/** = the trace/session id. Wire contract (dedup linkage). */
|
|
9
|
+
SESSION_ID: "tangle.sessionId",
|
|
10
|
+
/** "cli" marks a CLI upload so the platform can drop a re-upload of a session
|
|
11
|
+
* that also streamed live. Wire contract. */
|
|
12
|
+
INGEST_SOURCE: "tangle.ingest_source",
|
|
13
|
+
HARNESS: "tangle.harness",
|
|
14
|
+
CWD: "tangle.cwd",
|
|
15
|
+
/** THE per-session grouping key the spine derives subjects from
|
|
16
|
+
* (`deriveSubjectKey` reads `tangle.subject.key` first). Resolves to the git
|
|
17
|
+
* remote (host/owner/repo) when readable, else the cwd path basename. */
|
|
18
|
+
SUBJECT_KEY: "tangle.subject.key",
|
|
19
|
+
/** Normalized git remote (e.g. `github.com/tangle-network/agent-dev-container`). */
|
|
20
|
+
GIT_REPOSITORY: "git.repository",
|
|
21
|
+
/** Current branch at conversion time. */
|
|
22
|
+
GIT_BRANCH_NAME: "git.branch",
|
|
23
|
+
/** HEAD short sha at conversion time. */
|
|
24
|
+
GIT_COMMIT: "git.commit",
|
|
25
|
+
/** How traces resolved the session cwd: recorded cwd, repaired cwd, or span path. */
|
|
26
|
+
REPO_RESOLUTION_SOURCE: "traces.repo_resolution_source",
|
|
27
|
+
GIT_BRANCH: "tangle.git_branch",
|
|
28
|
+
HOST: "tangle.host",
|
|
29
|
+
/** Basename of the session file. (Renamed from the ambiguous `tangle.source`,
|
|
30
|
+
* which collided with `tangle.ingest_source`.) */
|
|
31
|
+
SESSION_FILE: "tangle.session_file",
|
|
32
|
+
UPLOADED_AT: "tangle.uploaded_at",
|
|
33
|
+
UPLOADER: "tangle.uploader",
|
|
34
|
+
REDACTION_VERSION: "redaction.version",
|
|
35
|
+
REDACTION_COUNT: "redaction.count",
|
|
36
|
+
/** Upload contract: bounded root summary plus location-only receipt child spans.
|
|
37
|
+
* Receipt hashes are deliberate integrity evidence; raw malformed bytes never upload. */
|
|
38
|
+
SESSION_INTEGRITY: "traces.session.integrity",
|
|
39
|
+
CORRUPTION_COUNT: "traces.session.corruption_count",
|
|
40
|
+
CORRUPTION_DIGEST: "traces.session.corruption_digest",
|
|
41
|
+
CORRUPTION_RECEIPT_VERSION: "traces.session.corruption.receipt_version",
|
|
42
|
+
CORRUPTION_RECEIPT_KIND: "traces.session.corruption.kind",
|
|
43
|
+
CORRUPTION_SOURCE_PATH: "traces.session.corruption.source_path",
|
|
44
|
+
CORRUPTION_LINE_NUMBER: "traces.session.corruption.line_number",
|
|
45
|
+
CORRUPTION_BYTE_OFFSET: "traces.session.corruption.byte_offset",
|
|
46
|
+
CORRUPTION_BYTE_LENGTH: "traces.session.corruption.byte_length",
|
|
47
|
+
CORRUPTION_SHA256: "traces.session.corruption.sha256",
|
|
48
|
+
RAW_SOURCE_RETENTION: "traces.session.raw_source_retention",
|
|
49
|
+
/** Container image reference the session executed in (OTel semconv key). */
|
|
50
|
+
CONTAINER_IMAGE: "container.image.name",
|
|
51
|
+
/** Immutable image digest (`sha256:…`) — the replay-grade environment pin. */
|
|
52
|
+
CONTAINER_IMAGE_DIGEST: "container.image.digest",
|
|
53
|
+
/** Sandbox/container instance id that executed the session. */
|
|
54
|
+
SANDBOX_ID: "tangle.sandbox.id",
|
|
55
|
+
/** Recorded execution cwd, verbatim. Unlike `tangle.cwd` it is never
|
|
56
|
+
* repaired against the host filesystem — replay verification needs the
|
|
57
|
+
* in-container path even when no such path exists on this host. */
|
|
58
|
+
ENVIRONMENT_CWD: "tangle.environment.cwd"
|
|
59
|
+
};
|
|
60
|
+
var INGEST_SOURCE_CLI = "cli";
|
|
61
|
+
function environmentAttributes(environment) {
|
|
62
|
+
if (!environment) return {};
|
|
63
|
+
const attrs = {};
|
|
64
|
+
if (environment.image) attrs[ATTR.CONTAINER_IMAGE] = environment.image;
|
|
65
|
+
if (environment.imageDigest) attrs[ATTR.CONTAINER_IMAGE_DIGEST] = environment.imageDigest;
|
|
66
|
+
if (environment.sandboxId) attrs[ATTR.SANDBOX_ID] = environment.sandboxId;
|
|
67
|
+
if (environment.cwd) attrs[ATTR.ENVIRONMENT_CWD] = environment.cwd;
|
|
68
|
+
if (environment.gitCommit) attrs[ATTR.GIT_COMMIT] = environment.gitCommit;
|
|
69
|
+
return attrs;
|
|
70
|
+
}
|
|
71
|
+
function stampEnvironmentAttrs(spans, environment) {
|
|
72
|
+
const attrs = environmentAttributes(environment);
|
|
73
|
+
const keys = Object.keys(attrs);
|
|
74
|
+
if (keys.length === 0) return;
|
|
75
|
+
for (const span2 of spans) {
|
|
76
|
+
for (const key of keys) {
|
|
77
|
+
if (span2.attributes[key] === void 0) span2.attributes[key] = attrs[key];
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
function environmentFromSpanAttributes(spans) {
|
|
82
|
+
const first = (key) => {
|
|
83
|
+
for (const span2 of spans) {
|
|
84
|
+
const value = span2.attributes[key];
|
|
85
|
+
if (typeof value === "string" && value.length > 0) return value;
|
|
86
|
+
}
|
|
87
|
+
return void 0;
|
|
88
|
+
};
|
|
89
|
+
const image = first(ATTR.CONTAINER_IMAGE);
|
|
90
|
+
const imageDigest = first(ATTR.CONTAINER_IMAGE_DIGEST);
|
|
91
|
+
const sandboxId = first(ATTR.SANDBOX_ID);
|
|
92
|
+
const cwd = first(ATTR.ENVIRONMENT_CWD);
|
|
93
|
+
if (!image && !imageDigest && !sandboxId && !cwd) return null;
|
|
94
|
+
return {
|
|
95
|
+
...image ? { image } : {},
|
|
96
|
+
...imageDigest ? { imageDigest } : {},
|
|
97
|
+
...sandboxId ? { sandboxId } : {},
|
|
98
|
+
cwd: cwd ?? null
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
var DEFAULT_HARNESS = "claude-code";
|
|
102
|
+
var SESSION_ID_ATTRIBUTE_KEYS = [ATTR.SESSION_ID, "session.id", "traces.session.id"];
|
|
103
|
+
function sessionIdFromAttributes(attributes2) {
|
|
104
|
+
return SESSION_ID_ATTRIBUTE_KEYS.map((key) => attributes2[key]).find(
|
|
105
|
+
(value) => typeof value === "string" && value.length > 0
|
|
106
|
+
);
|
|
107
|
+
}
|
|
108
|
+
function indexSessionIdsByTrace(spans) {
|
|
109
|
+
const candidatesByTrace = /* @__PURE__ */ new Map();
|
|
110
|
+
for (const span2 of spans) {
|
|
111
|
+
const candidates = candidatesByTrace.get(span2.trace_id) ?? /* @__PURE__ */ new Set();
|
|
112
|
+
for (const key of SESSION_ID_ATTRIBUTE_KEYS) {
|
|
113
|
+
const value = span2.attributes[key];
|
|
114
|
+
if (typeof value === "string" && value.length > 0) candidates.add(value);
|
|
115
|
+
}
|
|
116
|
+
if (candidates.size > 0) candidatesByTrace.set(span2.trace_id, candidates);
|
|
117
|
+
}
|
|
118
|
+
const sessionByTrace = /* @__PURE__ */ new Map();
|
|
119
|
+
const conflicts = [];
|
|
120
|
+
for (const [traceId, candidates] of candidatesByTrace) {
|
|
121
|
+
const sessionIds = [...candidates].sort();
|
|
122
|
+
if (sessionIds.length === 1) {
|
|
123
|
+
sessionByTrace.set(traceId, sessionIds[0]);
|
|
124
|
+
} else {
|
|
125
|
+
conflicts.push({ traceId, sessionIds });
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
conflicts.sort((a, b) => a.traceId.localeCompare(b.traceId));
|
|
129
|
+
return { sessionByTrace, conflicts };
|
|
130
|
+
}
|
|
131
|
+
function stampSessionIdentity(spans, fallbackSessionId) {
|
|
132
|
+
const { sessionByTrace, conflicts } = indexSessionIdsByTrace(spans);
|
|
133
|
+
const conflictingTraceIds = new Set(conflicts.map((conflict) => conflict.traceId));
|
|
134
|
+
const traceIds = new Set(spans.map((span2) => span2.trace_id));
|
|
135
|
+
const singleTraceId = traceIds.size === 1 ? traceIds.values().next().value : void 0;
|
|
136
|
+
for (const span2 of spans) {
|
|
137
|
+
if (conflictingTraceIds.has(span2.trace_id)) continue;
|
|
138
|
+
const sessionId = sessionByTrace.get(span2.trace_id) ?? singleTraceId ?? fallbackSessionId;
|
|
139
|
+
if (!sessionIdFromAttributes(span2.attributes)) {
|
|
140
|
+
span2.attributes[ATTR.SESSION_ID] = sessionId;
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
|
|
6
145
|
// src/time.ts
|
|
7
146
|
function parseIsoToEpochMs(ts) {
|
|
8
147
|
if (typeof ts !== "string" || ts.trim().length === 0) {
|
|
@@ -31,7 +170,7 @@ function parseSince(value) {
|
|
|
31
170
|
import { readdir, readFile, stat } from "fs/promises";
|
|
32
171
|
import { basename, join } from "path";
|
|
33
172
|
import {
|
|
34
|
-
ATTR,
|
|
173
|
+
ATTR as ATTR2,
|
|
35
174
|
declaredSpanKind,
|
|
36
175
|
resolveSpanKind,
|
|
37
176
|
SPAN_KIND_ATTR_KEYS,
|
|
@@ -325,7 +464,7 @@ function readLinks(value, traceId) {
|
|
|
325
464
|
return links;
|
|
326
465
|
}
|
|
327
466
|
function bridgeContractAttributes(attributes2) {
|
|
328
|
-
const toolName4 = nonEmptyString(attributes2[
|
|
467
|
+
const toolName4 = nonEmptyString(attributes2[ATTR2.toolName]);
|
|
329
468
|
if (toolName4 !== null && nonEmptyString(attributes2[TOOL_NAME]) === null) {
|
|
330
469
|
attributes2[TOOL_NAME] = toolName4;
|
|
331
470
|
}
|
|
@@ -385,7 +524,7 @@ function otlpRowToSpan(row, fallbackTraceId) {
|
|
|
385
524
|
bridgeContractAttributes(attributes2);
|
|
386
525
|
const { kind, declaredRaw, substituted: kindSubstituted } = resolveRowKind(row, attributes2, name);
|
|
387
526
|
if (kindSubstituted) substitute("kind", declaredRaw ?? void 0);
|
|
388
|
-
attributes2[
|
|
527
|
+
attributes2[ATTR2.spanKind] = kind;
|
|
389
528
|
if (kind === "TOOL") Object.assign(attributes2, capturedToolIo(attributes2));
|
|
390
529
|
const status = isObject(row.status) ? row.status : void 0;
|
|
391
530
|
const statusMessage = nonEmptyString(status?.message);
|
|
@@ -725,6 +864,9 @@ function toOpenInferenceSpan(s) {
|
|
|
725
864
|
for (const k of ["tangle.subject.key", "git.repository", "git.branch", "git.commit", "tangle.cwd", "traces.repo_resolution_source"]) {
|
|
726
865
|
if (a[k] != null) resourceAttrs[k] = a[k];
|
|
727
866
|
}
|
|
867
|
+
for (const k of ["container.image.name", "container.image.digest", "tangle.sandbox.id", "tangle.environment.cwd"]) {
|
|
868
|
+
if (a[k] != null) resourceAttrs[k] = a[k];
|
|
869
|
+
}
|
|
728
870
|
for (const k of ["traces.source.unreadable_rows", "traces.source.unreadable_row_kinds"]) {
|
|
729
871
|
if (a[k] != null) resourceAttrs[k] = a[k];
|
|
730
872
|
}
|
|
@@ -764,11 +906,11 @@ function serializeSpans(spans) {
|
|
|
764
906
|
`;
|
|
765
907
|
}
|
|
766
908
|
async function writeOtlpFile(spans, outPath) {
|
|
767
|
-
const { mkdtemp: mkdtemp8, writeFile:
|
|
909
|
+
const { mkdtemp: mkdtemp8, writeFile: writeFile12 } = await import("fs/promises");
|
|
768
910
|
const { tmpdir: tmpdir9 } = await import("os");
|
|
769
|
-
const { join:
|
|
770
|
-
const path = outPath ??
|
|
771
|
-
await
|
|
911
|
+
const { join: join36 } = await import("path");
|
|
912
|
+
const path = outPath ?? join36(await mkdtemp8(join36(tmpdir9(), "traces-")), "spans.otlp.jsonl");
|
|
913
|
+
await writeFile12(path, serializeSpans(spans), "utf8");
|
|
772
914
|
return path;
|
|
773
915
|
}
|
|
774
916
|
|
|
@@ -796,95 +938,6 @@ function userPromptSpan(o) {
|
|
|
796
938
|
});
|
|
797
939
|
}
|
|
798
940
|
|
|
799
|
-
// src/attributes.ts
|
|
800
|
-
var ATTR2 = {
|
|
801
|
-
/** = the trace/session id. Wire contract (dedup linkage). */
|
|
802
|
-
SESSION_ID: "tangle.sessionId",
|
|
803
|
-
/** "cli" marks a CLI upload so the platform can drop a re-upload of a session
|
|
804
|
-
* that also streamed live. Wire contract. */
|
|
805
|
-
INGEST_SOURCE: "tangle.ingest_source",
|
|
806
|
-
HARNESS: "tangle.harness",
|
|
807
|
-
CWD: "tangle.cwd",
|
|
808
|
-
/** THE per-session grouping key the spine derives subjects from
|
|
809
|
-
* (`deriveSubjectKey` reads `tangle.subject.key` first). Resolves to the git
|
|
810
|
-
* remote (host/owner/repo) when readable, else the cwd path basename. */
|
|
811
|
-
SUBJECT_KEY: "tangle.subject.key",
|
|
812
|
-
/** Normalized git remote (e.g. `github.com/tangle-network/agent-dev-container`). */
|
|
813
|
-
GIT_REPOSITORY: "git.repository",
|
|
814
|
-
/** Current branch at conversion time. */
|
|
815
|
-
GIT_BRANCH_NAME: "git.branch",
|
|
816
|
-
/** HEAD short sha at conversion time. */
|
|
817
|
-
GIT_COMMIT: "git.commit",
|
|
818
|
-
/** How traces resolved the session cwd: recorded cwd, repaired cwd, or span path. */
|
|
819
|
-
REPO_RESOLUTION_SOURCE: "traces.repo_resolution_source",
|
|
820
|
-
GIT_BRANCH: "tangle.git_branch",
|
|
821
|
-
HOST: "tangle.host",
|
|
822
|
-
/** Basename of the session file. (Renamed from the ambiguous `tangle.source`,
|
|
823
|
-
* which collided with `tangle.ingest_source`.) */
|
|
824
|
-
SESSION_FILE: "tangle.session_file",
|
|
825
|
-
UPLOADED_AT: "tangle.uploaded_at",
|
|
826
|
-
UPLOADER: "tangle.uploader",
|
|
827
|
-
REDACTION_VERSION: "redaction.version",
|
|
828
|
-
REDACTION_COUNT: "redaction.count",
|
|
829
|
-
/** Upload contract: bounded root summary plus location-only receipt child spans.
|
|
830
|
-
* Receipt hashes are deliberate integrity evidence; raw malformed bytes never upload. */
|
|
831
|
-
SESSION_INTEGRITY: "traces.session.integrity",
|
|
832
|
-
CORRUPTION_COUNT: "traces.session.corruption_count",
|
|
833
|
-
CORRUPTION_DIGEST: "traces.session.corruption_digest",
|
|
834
|
-
CORRUPTION_RECEIPT_VERSION: "traces.session.corruption.receipt_version",
|
|
835
|
-
CORRUPTION_RECEIPT_KIND: "traces.session.corruption.kind",
|
|
836
|
-
CORRUPTION_SOURCE_PATH: "traces.session.corruption.source_path",
|
|
837
|
-
CORRUPTION_LINE_NUMBER: "traces.session.corruption.line_number",
|
|
838
|
-
CORRUPTION_BYTE_OFFSET: "traces.session.corruption.byte_offset",
|
|
839
|
-
CORRUPTION_BYTE_LENGTH: "traces.session.corruption.byte_length",
|
|
840
|
-
CORRUPTION_SHA256: "traces.session.corruption.sha256",
|
|
841
|
-
RAW_SOURCE_RETENTION: "traces.session.raw_source_retention"
|
|
842
|
-
};
|
|
843
|
-
var INGEST_SOURCE_CLI = "cli";
|
|
844
|
-
var DEFAULT_HARNESS = "claude-code";
|
|
845
|
-
var SESSION_ID_ATTRIBUTE_KEYS = [ATTR2.SESSION_ID, "session.id", "traces.session.id"];
|
|
846
|
-
function sessionIdFromAttributes(attributes2) {
|
|
847
|
-
return SESSION_ID_ATTRIBUTE_KEYS.map((key) => attributes2[key]).find(
|
|
848
|
-
(value) => typeof value === "string" && value.length > 0
|
|
849
|
-
);
|
|
850
|
-
}
|
|
851
|
-
function indexSessionIdsByTrace(spans) {
|
|
852
|
-
const candidatesByTrace = /* @__PURE__ */ new Map();
|
|
853
|
-
for (const span2 of spans) {
|
|
854
|
-
const candidates = candidatesByTrace.get(span2.trace_id) ?? /* @__PURE__ */ new Set();
|
|
855
|
-
for (const key of SESSION_ID_ATTRIBUTE_KEYS) {
|
|
856
|
-
const value = span2.attributes[key];
|
|
857
|
-
if (typeof value === "string" && value.length > 0) candidates.add(value);
|
|
858
|
-
}
|
|
859
|
-
if (candidates.size > 0) candidatesByTrace.set(span2.trace_id, candidates);
|
|
860
|
-
}
|
|
861
|
-
const sessionByTrace = /* @__PURE__ */ new Map();
|
|
862
|
-
const conflicts = [];
|
|
863
|
-
for (const [traceId, candidates] of candidatesByTrace) {
|
|
864
|
-
const sessionIds = [...candidates].sort();
|
|
865
|
-
if (sessionIds.length === 1) {
|
|
866
|
-
sessionByTrace.set(traceId, sessionIds[0]);
|
|
867
|
-
} else {
|
|
868
|
-
conflicts.push({ traceId, sessionIds });
|
|
869
|
-
}
|
|
870
|
-
}
|
|
871
|
-
conflicts.sort((a, b) => a.traceId.localeCompare(b.traceId));
|
|
872
|
-
return { sessionByTrace, conflicts };
|
|
873
|
-
}
|
|
874
|
-
function stampSessionIdentity(spans, fallbackSessionId) {
|
|
875
|
-
const { sessionByTrace, conflicts } = indexSessionIdsByTrace(spans);
|
|
876
|
-
const conflictingTraceIds = new Set(conflicts.map((conflict) => conflict.traceId));
|
|
877
|
-
const traceIds = new Set(spans.map((span2) => span2.trace_id));
|
|
878
|
-
const singleTraceId = traceIds.size === 1 ? traceIds.values().next().value : void 0;
|
|
879
|
-
for (const span2 of spans) {
|
|
880
|
-
if (conflictingTraceIds.has(span2.trace_id)) continue;
|
|
881
|
-
const sessionId = sessionByTrace.get(span2.trace_id) ?? singleTraceId ?? fallbackSessionId;
|
|
882
|
-
if (!sessionIdFromAttributes(span2.attributes)) {
|
|
883
|
-
span2.attributes[ATTR2.SESSION_ID] = sessionId;
|
|
884
|
-
}
|
|
885
|
-
}
|
|
886
|
-
}
|
|
887
|
-
|
|
888
941
|
// src/chat-trajectory.ts
|
|
889
942
|
import { createHash as createHash3 } from "crypto";
|
|
890
943
|
function isChatTrajectoryMessage(value) {
|
|
@@ -1907,10 +1960,10 @@ function sessionJsonlOptions(ref, options = {}) {
|
|
|
1907
1960
|
function sessionIntegrityAttributes(ref) {
|
|
1908
1961
|
if (!ref.integrity) return {};
|
|
1909
1962
|
return {
|
|
1910
|
-
[
|
|
1911
|
-
[
|
|
1912
|
-
[
|
|
1913
|
-
[
|
|
1963
|
+
[ATTR.SESSION_INTEGRITY]: ref.integrity.status,
|
|
1964
|
+
[ATTR.CORRUPTION_COUNT]: ref.integrity.corruptions.length,
|
|
1965
|
+
[ATTR.CORRUPTION_DIGEST]: corruptionDigest(ref.integrity.corruptions),
|
|
1966
|
+
[ATTR.RAW_SOURCE_RETENTION]: "local_source_only"
|
|
1914
1967
|
};
|
|
1915
1968
|
}
|
|
1916
1969
|
function corruptionDigest(receipts) {
|
|
@@ -1951,15 +2004,15 @@ function stampSessionIntegrity(ref, spans) {
|
|
|
1951
2004
|
status: "ERROR",
|
|
1952
2005
|
service: typeof parent.attributes["service.name"] === "string" ? parent.attributes["service.name"] : ref.harness,
|
|
1953
2006
|
extra: {
|
|
1954
|
-
[
|
|
1955
|
-
[
|
|
1956
|
-
[
|
|
1957
|
-
[
|
|
1958
|
-
[
|
|
1959
|
-
[
|
|
1960
|
-
[
|
|
1961
|
-
[
|
|
1962
|
-
[
|
|
2007
|
+
[ATTR.SESSION_INTEGRITY]: ref.integrity.status,
|
|
2008
|
+
[ATTR.CORRUPTION_RECEIPT_VERSION]: receipt.receiptVersion,
|
|
2009
|
+
[ATTR.CORRUPTION_RECEIPT_KIND]: receipt.kind,
|
|
2010
|
+
[ATTR.CORRUPTION_SOURCE_PATH]: receipt.sourcePath,
|
|
2011
|
+
[ATTR.CORRUPTION_LINE_NUMBER]: receipt.lineNumber,
|
|
2012
|
+
[ATTR.CORRUPTION_BYTE_OFFSET]: receipt.byteOffset,
|
|
2013
|
+
[ATTR.CORRUPTION_BYTE_LENGTH]: receipt.byteLength,
|
|
2014
|
+
[ATTR.CORRUPTION_SHA256]: receipt.sha256,
|
|
2015
|
+
[ATTR.RAW_SOURCE_RETENTION]: "local_source_only"
|
|
1963
2016
|
}
|
|
1964
2017
|
}));
|
|
1965
2018
|
existingSpanIds.add(spanId);
|
|
@@ -6236,7 +6289,7 @@ function extractAbsolutePaths(spans) {
|
|
|
6236
6289
|
}
|
|
6237
6290
|
async function resolveRepoAttrs(cwd) {
|
|
6238
6291
|
if (!cwd) return {};
|
|
6239
|
-
const attrs = { [
|
|
6292
|
+
const attrs = { [ATTR.CWD]: cwd };
|
|
6240
6293
|
const basename16 = (() => {
|
|
6241
6294
|
const trimmed = cwd.replace(/\/+$/g, "");
|
|
6242
6295
|
const idx = trimmed.lastIndexOf("/");
|
|
@@ -6249,17 +6302,17 @@ async function resolveRepoAttrs(cwd) {
|
|
|
6249
6302
|
const { remote, branch, commit } = await readGit(gitRoot);
|
|
6250
6303
|
const normalized = remote ? normalizeRemote(remote) : null;
|
|
6251
6304
|
if (normalized) {
|
|
6252
|
-
attrs[
|
|
6253
|
-
attrs[
|
|
6254
|
-
if (branch) attrs[
|
|
6255
|
-
if (commit) attrs[
|
|
6305
|
+
attrs[ATTR.SUBJECT_KEY] = normalized;
|
|
6306
|
+
attrs[ATTR.GIT_REPOSITORY] = normalized;
|
|
6307
|
+
if (branch) attrs[ATTR.GIT_BRANCH_NAME] = branch;
|
|
6308
|
+
if (commit) attrs[ATTR.GIT_COMMIT] = commit;
|
|
6256
6309
|
return attrs;
|
|
6257
6310
|
}
|
|
6258
|
-
if (branch) attrs[
|
|
6259
|
-
if (commit) attrs[
|
|
6311
|
+
if (branch) attrs[ATTR.GIT_BRANCH_NAME] = branch;
|
|
6312
|
+
if (commit) attrs[ATTR.GIT_COMMIT] = commit;
|
|
6260
6313
|
} catch {
|
|
6261
6314
|
}
|
|
6262
|
-
attrs[
|
|
6315
|
+
attrs[ATTR.SUBJECT_KEY] = basename16;
|
|
6263
6316
|
return attrs;
|
|
6264
6317
|
}
|
|
6265
6318
|
async function repoCandidateForPath(path) {
|
|
@@ -6290,12 +6343,12 @@ async function resolveSessionRepoAttrs(cwd, spans) {
|
|
|
6290
6343
|
resolved.push({ ...candidate, attrs });
|
|
6291
6344
|
}
|
|
6292
6345
|
const byScore = (a, b) => b.score - a.score || a.order - b.order;
|
|
6293
|
-
const withGit = resolved.filter((candidate) => candidate.attrs[
|
|
6294
|
-
const selected = withGit[0] ?? resolved.filter((candidate) => candidate.attrs[
|
|
6346
|
+
const withGit = resolved.filter((candidate) => candidate.attrs[ATTR.GIT_REPOSITORY]).sort(byScore);
|
|
6347
|
+
const selected = withGit[0] ?? resolved.filter((candidate) => candidate.attrs[ATTR.SUBJECT_KEY]).sort(byScore)[0];
|
|
6295
6348
|
if (!selected) return { attrs: {}, cwd: null, source: "none" };
|
|
6296
6349
|
return {
|
|
6297
|
-
attrs: { ...selected.attrs, [
|
|
6298
|
-
cwd: selected.attrs[
|
|
6350
|
+
attrs: { ...selected.attrs, [ATTR.REPO_RESOLUTION_SOURCE]: selected.source },
|
|
6351
|
+
cwd: selected.attrs[ATTR.CWD] ?? selected.cwd,
|
|
6299
6352
|
source: selected.source
|
|
6300
6353
|
};
|
|
6301
6354
|
}
|
|
@@ -6323,9 +6376,9 @@ async function resolutionForWorkdirEvidence(evidence3, cache) {
|
|
|
6323
6376
|
return null;
|
|
6324
6377
|
}
|
|
6325
6378
|
const attrs = await resolveRepoAttrs(cwd);
|
|
6326
|
-
const resolved = attrs[
|
|
6327
|
-
attrs: { ...attrs, [
|
|
6328
|
-
cwd: attrs[
|
|
6379
|
+
const resolved = attrs[ATTR.SUBJECT_KEY] ? {
|
|
6380
|
+
attrs: { ...attrs, [ATTR.REPO_RESOLUTION_SOURCE]: evidence3.source },
|
|
6381
|
+
cwd: attrs[ATTR.CWD] ?? cwd,
|
|
6329
6382
|
source: evidence3.source
|
|
6330
6383
|
} : null;
|
|
6331
6384
|
cache.set(evidence3.path, resolved);
|
|
@@ -6348,7 +6401,7 @@ async function resolveSpanWorkdirRepoAttrs(span2, cache) {
|
|
|
6348
6401
|
}
|
|
6349
6402
|
const byScore = (a, b) => b.score - a.score || a.order - b.order;
|
|
6350
6403
|
const resolved = [...candidates.values()];
|
|
6351
|
-
const selected = resolved.filter((candidate) => candidate.repo.attrs[
|
|
6404
|
+
const selected = resolved.filter((candidate) => candidate.repo.attrs[ATTR.GIT_REPOSITORY]).sort(byScore)[0] ?? resolved.filter((candidate) => candidate.repo.attrs[ATTR.SUBJECT_KEY]).sort(byScore)[0];
|
|
6352
6405
|
if (!selected) return null;
|
|
6353
6406
|
return selected.repo;
|
|
6354
6407
|
}
|
|
@@ -6364,7 +6417,7 @@ async function stampSpanWorkdirRepoAttrs(spans) {
|
|
|
6364
6417
|
async function awaitWithAbort(operation, signal) {
|
|
6365
6418
|
if (!signal) return operation;
|
|
6366
6419
|
signal.throwIfAborted();
|
|
6367
|
-
return new Promise((
|
|
6420
|
+
return new Promise((resolve10, reject) => {
|
|
6368
6421
|
const stop = () => {
|
|
6369
6422
|
signal.removeEventListener("abort", stop);
|
|
6370
6423
|
reject(signal.reason);
|
|
@@ -6373,7 +6426,7 @@ async function awaitWithAbort(operation, signal) {
|
|
|
6373
6426
|
operation.then(
|
|
6374
6427
|
(value) => {
|
|
6375
6428
|
signal.removeEventListener("abort", stop);
|
|
6376
|
-
|
|
6429
|
+
resolve10(value);
|
|
6377
6430
|
},
|
|
6378
6431
|
(error) => {
|
|
6379
6432
|
signal.removeEventListener("abort", stop);
|
|
@@ -6392,6 +6445,8 @@ async function parseSession(adapter, ref, options = {}) {
|
|
|
6392
6445
|
if (spans.length === 0) throw new EmptySessionError(ref.path);
|
|
6393
6446
|
stampSessionIntegrity(ref, spans);
|
|
6394
6447
|
stampSessionIdentity(spans, ref.sessionId);
|
|
6448
|
+
stampEnvironmentAttrs(spans, ref.environment);
|
|
6449
|
+
if (ref.cwd === null && ref.environment?.cwd) ref.cwd = ref.environment.cwd;
|
|
6395
6450
|
const repo = await resolveSessionRepoAttrs(ref.cwd, spans);
|
|
6396
6451
|
if (repo.cwd) ref.cwd = repo.cwd;
|
|
6397
6452
|
stampRepoAttrs(spans, repo.attrs);
|
|
@@ -6553,6 +6608,7 @@ import {
|
|
|
6553
6608
|
OPENINFERENCE_SPAN_KIND as OPENINFERENCE_SPAN_KIND2,
|
|
6554
6609
|
TOOL_NAME as TOOL_NAME3
|
|
6555
6610
|
} from "@tangle-network/agent-eval/trace-attributes";
|
|
6611
|
+
var POLICY_EVIDENCE_NOTE = "This is normalized coding-agent session evidence for downstream policy mining; it is not an eval campaign cell.";
|
|
6556
6612
|
function stringAttr(span2, key) {
|
|
6557
6613
|
const value = span2.attributes[key];
|
|
6558
6614
|
return typeof value === "string" && value.length > 0 ? value : void 0;
|
|
@@ -6563,12 +6619,12 @@ function spanKind(span2) {
|
|
|
6563
6619
|
function repoFromSpans(spans) {
|
|
6564
6620
|
const attrs = {};
|
|
6565
6621
|
for (const span2 of spans) {
|
|
6566
|
-
attrs.subjectKey ??= stringAttr(span2,
|
|
6567
|
-
attrs.repository ??= stringAttr(span2,
|
|
6568
|
-
attrs.branch ??= stringAttr(span2,
|
|
6569
|
-
attrs.commit ??= stringAttr(span2,
|
|
6570
|
-
attrs.cwd ??= stringAttr(span2,
|
|
6571
|
-
attrs.resolutionSource ??= stringAttr(span2,
|
|
6622
|
+
attrs.subjectKey ??= stringAttr(span2, ATTR.SUBJECT_KEY);
|
|
6623
|
+
attrs.repository ??= stringAttr(span2, ATTR.GIT_REPOSITORY);
|
|
6624
|
+
attrs.branch ??= stringAttr(span2, ATTR.GIT_BRANCH_NAME);
|
|
6625
|
+
attrs.commit ??= stringAttr(span2, ATTR.GIT_COMMIT);
|
|
6626
|
+
attrs.cwd ??= stringAttr(span2, ATTR.CWD);
|
|
6627
|
+
attrs.resolutionSource ??= stringAttr(span2, ATTR.REPO_RESOLUTION_SOURCE);
|
|
6572
6628
|
if (attrs.subjectKey && attrs.repository && attrs.branch && attrs.commit && attrs.cwd && attrs.resolutionSource) break;
|
|
6573
6629
|
}
|
|
6574
6630
|
return attrs;
|
|
@@ -6657,7 +6713,7 @@ async function buildPolicyEvidenceRecord(ref, spans, opts = {}) {
|
|
|
6657
6713
|
...opts.otlpPath ? { otlpPath: opts.otlpPath } : {},
|
|
6658
6714
|
...opts.sourceSha256 ? { sourceSha256: opts.sourceSha256 } : {},
|
|
6659
6715
|
notCampaignCell: true,
|
|
6660
|
-
note:
|
|
6716
|
+
note: POLICY_EVIDENCE_NOTE
|
|
6661
6717
|
}
|
|
6662
6718
|
};
|
|
6663
6719
|
}
|
|
@@ -7181,11 +7237,11 @@ function parseCorpusFlag(value) {
|
|
|
7181
7237
|
if (eq <= 0) throw new Error(`--corpus must be name=<labels>::<prepared>, got: ${value}`);
|
|
7182
7238
|
const name = value.slice(0, eq);
|
|
7183
7239
|
const rest = value.slice(eq + 1);
|
|
7184
|
-
const
|
|
7185
|
-
if (
|
|
7240
|
+
const sep3 = rest.indexOf("::");
|
|
7241
|
+
if (sep3 <= 0 || sep3 === rest.length - 2) {
|
|
7186
7242
|
throw new Error(`--corpus must be name=<labels>::<prepared>, got: ${value}`);
|
|
7187
7243
|
}
|
|
7188
|
-
return { name, labelsPath: rest.slice(0,
|
|
7244
|
+
return { name, labelsPath: rest.slice(0, sep3), preparedDir: rest.slice(sep3 + 2) };
|
|
7189
7245
|
}
|
|
7190
7246
|
var SUBMIT_ACTION_SIGNATURE = "COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT";
|
|
7191
7247
|
function isSubmitAction(action) {
|
|
@@ -7211,7 +7267,7 @@ function goldIncorrectSteps(entry) {
|
|
|
7211
7267
|
function findRawTrajFiles(sweRawDir) {
|
|
7212
7268
|
if (!existsSync2(sweRawDir)) return [];
|
|
7213
7269
|
const entries = readdirSync2(sweRawDir, { recursive: true, encoding: "utf8" });
|
|
7214
|
-
return entries.filter((
|
|
7270
|
+
return entries.filter((relative6) => relative6.endsWith(".traj.json")).map((relative6) => join20(sweRawDir, relative6)).sort();
|
|
7215
7271
|
}
|
|
7216
7272
|
function cwdFromPwdObservation(steps) {
|
|
7217
7273
|
for (const step of steps) {
|
|
@@ -7419,7 +7475,7 @@ function zaiChatCaller(options) {
|
|
|
7419
7475
|
}
|
|
7420
7476
|
};
|
|
7421
7477
|
}
|
|
7422
|
-
function
|
|
7478
|
+
function clipText(text2, limit) {
|
|
7423
7479
|
if (text2.length <= limit) return text2;
|
|
7424
7480
|
const half = Math.floor(limit / 2);
|
|
7425
7481
|
return `${text2.slice(0, half)}
|
|
@@ -7428,11 +7484,11 @@ ${text2.slice(-half)}`;
|
|
|
7428
7484
|
}
|
|
7429
7485
|
function renderStep(step, marker) {
|
|
7430
7486
|
const rc = parseRecordedReturncode(step.observation);
|
|
7431
|
-
const output =
|
|
7487
|
+
const output = clipText(parseObservationOutput(step.observation).trim(), 1600);
|
|
7432
7488
|
return [
|
|
7433
7489
|
`### step ${step.step_id}${marker}`,
|
|
7434
7490
|
"```sh",
|
|
7435
|
-
|
|
7491
|
+
clipText(step.action, 2e3),
|
|
7436
7492
|
"```",
|
|
7437
7493
|
`returncode: ${rc ?? "none recorded"}`,
|
|
7438
7494
|
output.length > 0 ? `output:
|
|
@@ -7441,13 +7497,25 @@ ${output}
|
|
|
7441
7497
|
\`\`\`` : "output: (empty)"
|
|
7442
7498
|
].join("\n");
|
|
7443
7499
|
}
|
|
7444
|
-
function
|
|
7500
|
+
function promptBody(input) {
|
|
7445
7501
|
const radius = input.contextRadius ?? 3;
|
|
7446
7502
|
const target = input.steps.find((s) => s.step_id === input.k);
|
|
7447
7503
|
if (!target) throw new Error(`replay-fix: no step with step_id ${input.k}`);
|
|
7448
7504
|
const context = input.steps.filter(
|
|
7449
7505
|
(s) => s.step_id !== input.k && Math.abs(s.step_id - input.k) <= radius
|
|
7450
7506
|
);
|
|
7507
|
+
return [
|
|
7508
|
+
"## Task the agent was solving",
|
|
7509
|
+
input.taskStatement ? clipText(input.taskStatement, 4e3) : "(no task statement recorded)",
|
|
7510
|
+
"",
|
|
7511
|
+
"## Surrounding steps",
|
|
7512
|
+
...context.map((s) => renderStep(s, "")),
|
|
7513
|
+
"",
|
|
7514
|
+
"## Failing step to correct",
|
|
7515
|
+
renderStep(target, " (INCORRECT \u2014 correct this one)")
|
|
7516
|
+
];
|
|
7517
|
+
}
|
|
7518
|
+
function buildFixPrompt(input) {
|
|
7451
7519
|
const system = [
|
|
7452
7520
|
"You repair one failed shell command from a recorded coding-agent trajectory.",
|
|
7453
7521
|
"The trajectory replays inside the original docker image; every command runs as a fresh /bin/sh subshell from a fixed working directory.",
|
|
@@ -7456,19 +7524,62 @@ function buildFixPrompt(input) {
|
|
|
7456
7524
|
"The corrected command must accomplish the failing step's intent and exit 0. No prose outside the fenced block."
|
|
7457
7525
|
].join("\n");
|
|
7458
7526
|
const user = [
|
|
7459
|
-
|
|
7460
|
-
input.taskStatement ? clip(input.taskStatement, 4e3) : "(no task statement recorded)",
|
|
7527
|
+
...promptBody(input),
|
|
7461
7528
|
"",
|
|
7462
|
-
"
|
|
7463
|
-
|
|
7529
|
+
"Output the single corrected replacement for the failing step now."
|
|
7530
|
+
].join("\n");
|
|
7531
|
+
return { system, user };
|
|
7532
|
+
}
|
|
7533
|
+
function renderFailedAttempt(prior) {
|
|
7534
|
+
if (prior.command === null) {
|
|
7535
|
+
return [
|
|
7536
|
+
`### attempt ${prior.attempt}`,
|
|
7537
|
+
`model call failed before producing a command: ${prior.llmError ?? "unknown error"}`
|
|
7538
|
+
].join("\n");
|
|
7539
|
+
}
|
|
7540
|
+
const stdout = (prior.stdoutTail ?? "").trim();
|
|
7541
|
+
const stderr = (prior.stderrTail ?? "").trim();
|
|
7542
|
+
return [
|
|
7543
|
+
`### attempt ${prior.attempt}`,
|
|
7544
|
+
"```sh",
|
|
7545
|
+
clipText(prior.command, 2e3),
|
|
7546
|
+
"```",
|
|
7547
|
+
`exit code: ${prior.exitCode ?? "not executed"}`,
|
|
7548
|
+
stdout.length > 0 ? `stdout:
|
|
7549
|
+
\`\`\`
|
|
7550
|
+
${stdout}
|
|
7551
|
+
\`\`\`` : "stdout: (empty)",
|
|
7552
|
+
stderr.length > 0 ? `stderr:
|
|
7553
|
+
\`\`\`
|
|
7554
|
+
${stderr}
|
|
7555
|
+
\`\`\`` : "stderr: (empty)"
|
|
7556
|
+
].join("\n");
|
|
7557
|
+
}
|
|
7558
|
+
function buildRetryFixPrompt(input, priorAttempts, maxScriptCommands = 5) {
|
|
7559
|
+
if (priorAttempts.length === 0) {
|
|
7560
|
+
throw new Error("replay-fix: buildRetryFixPrompt requires at least one prior attempt");
|
|
7561
|
+
}
|
|
7562
|
+
const system = [
|
|
7563
|
+
"You repair one failed shell command from a recorded coding-agent trajectory.",
|
|
7564
|
+
"The trajectory replays inside the original docker image; every command runs as a fresh /bin/sh subshell from a fixed working directory.",
|
|
7565
|
+
"Earlier corrected commands were executed for real and failed; their actual output is included below.",
|
|
7566
|
+
`Reply with a corrected fix inside a single \`\`\`sh fenced block: either one command, or a short script of at most ${maxScriptCommands} commands (one per line).`,
|
|
7567
|
+
"The whole block executes as ONE /bin/sh unit from the fixed working directory and must exit 0.",
|
|
7568
|
+
"Do not repeat a command that already failed. Keep reasoning brief. No prose outside the fenced block."
|
|
7569
|
+
].join("\n");
|
|
7570
|
+
const user = [
|
|
7571
|
+
...promptBody(input),
|
|
7464
7572
|
"",
|
|
7465
|
-
"##
|
|
7466
|
-
|
|
7573
|
+
"## Previous fix attempts (executed for real \u2014 all failed)",
|
|
7574
|
+
...priorAttempts.map((prior) => renderFailedAttempt(prior)),
|
|
7467
7575
|
"",
|
|
7468
|
-
|
|
7576
|
+
`Output a corrected fix now \u2014 one \`\`\`sh block, at most ${maxScriptCommands} commands.`
|
|
7469
7577
|
].join("\n");
|
|
7470
7578
|
return { system, user };
|
|
7471
7579
|
}
|
|
7580
|
+
function countScriptCommands(script) {
|
|
7581
|
+
return script.split("\n").filter((line) => line.trim().length > 0 && !line.trim().startsWith("#")).length;
|
|
7582
|
+
}
|
|
7472
7583
|
function extractFixCommand(content) {
|
|
7473
7584
|
const blocks = [...content.matchAll(/```(?:sh|bash|shell)?\n([\s\S]*?)```/g)];
|
|
7474
7585
|
const candidate = blocks.length > 0 ? blocks[blocks.length - 1][1] : content;
|
|
@@ -7484,43 +7595,198 @@ async function generateFixCommand(caller, input) {
|
|
|
7484
7595
|
if (command === null) {
|
|
7485
7596
|
return {
|
|
7486
7597
|
succeeded: false,
|
|
7487
|
-
error: `completion carried no usable command: ${
|
|
7598
|
+
error: `completion carried no usable command: ${clipText(outcome.value.content, 300)}`
|
|
7488
7599
|
};
|
|
7489
7600
|
}
|
|
7490
7601
|
return { succeeded: true, value: { command, usage: outcome.value.usage } };
|
|
7491
7602
|
}
|
|
7492
7603
|
|
|
7493
|
-
// src/replay-
|
|
7494
|
-
|
|
7495
|
-
|
|
7496
|
-
|
|
7497
|
-
|
|
7498
|
-
|
|
7499
|
-
|
|
7500
|
-
|
|
7501
|
-
|
|
7502
|
-
runCounterfactual as runCounterfactual2
|
|
7503
|
-
} from "@tangle-network/agent-eval";
|
|
7504
|
-
var execFileAsync = promisify(execFile);
|
|
7505
|
-
function derivedImageTag(image, cwd) {
|
|
7506
|
-
const digest = createHash8("sha256").update(`${image}
|
|
7507
|
-
${cwd}`).digest("hex").slice(0, 12);
|
|
7508
|
-
return `ctb-replay:${digest}-uid1000`;
|
|
7509
|
-
}
|
|
7510
|
-
function dockerImagePreparer(options = {}) {
|
|
7511
|
-
const pullTimeoutMs = options.pullTimeoutMs ?? 12e5;
|
|
7512
|
-
const buildTimeoutMs = options.buildTimeoutMs ?? 9e5;
|
|
7513
|
-
const imageExists = async (tag) => {
|
|
7514
|
-
try {
|
|
7515
|
-
await execFileAsync("docker", ["image", "inspect", tag], { maxBuffer: 8 * 1024 * 1024 });
|
|
7516
|
-
return true;
|
|
7517
|
-
} catch {
|
|
7518
|
-
return false;
|
|
7519
|
-
}
|
|
7604
|
+
// src/replay-fix-loop.ts
|
|
7605
|
+
function toFailedAttempt(record2) {
|
|
7606
|
+
return {
|
|
7607
|
+
attempt: record2.attempt,
|
|
7608
|
+
command: record2.command,
|
|
7609
|
+
exitCode: record2.exitCode,
|
|
7610
|
+
stdoutTail: record2.stdoutTail,
|
|
7611
|
+
stderrTail: record2.stderrTail,
|
|
7612
|
+
llmError: record2.llmError ?? record2.armBError
|
|
7520
7613
|
};
|
|
7521
|
-
|
|
7522
|
-
|
|
7523
|
-
|
|
7614
|
+
}
|
|
7615
|
+
async function runFixLoop(caller, input, executor, options) {
|
|
7616
|
+
if (!Number.isInteger(options.maxAttempts) || options.maxAttempts < 1) {
|
|
7617
|
+
throw new Error(
|
|
7618
|
+
`replay-fix-loop: maxAttempts must be a positive integer, got ${options.maxAttempts}`
|
|
7619
|
+
);
|
|
7620
|
+
}
|
|
7621
|
+
const scriptCap = options.maxScriptCommands ?? 5;
|
|
7622
|
+
const tailChars = options.outputTailChars ?? 1600;
|
|
7623
|
+
const attempts = [];
|
|
7624
|
+
let llmCalls = 0;
|
|
7625
|
+
let llmFailures = 0;
|
|
7626
|
+
let promptTokens = 0;
|
|
7627
|
+
let completionTokens = 0;
|
|
7628
|
+
const unexecuted = (attempt, llmError, usage, command, started) => ({
|
|
7629
|
+
attempt,
|
|
7630
|
+
command,
|
|
7631
|
+
llmError,
|
|
7632
|
+
usage,
|
|
7633
|
+
executed: false,
|
|
7634
|
+
exitCode: null,
|
|
7635
|
+
prefixExecuted: null,
|
|
7636
|
+
prefixDivergences: null,
|
|
7637
|
+
failureVanished: null,
|
|
7638
|
+
stdoutTail: null,
|
|
7639
|
+
stderrTail: null,
|
|
7640
|
+
armBError: null,
|
|
7641
|
+
wallMs: Date.now() - started
|
|
7642
|
+
});
|
|
7643
|
+
for (let attempt = 1; attempt <= options.maxAttempts; attempt++) {
|
|
7644
|
+
const started = Date.now();
|
|
7645
|
+
const prompt = attempt === 1 ? buildFixPrompt(input) : buildRetryFixPrompt(input, attempts.map(toFailedAttempt), scriptCap);
|
|
7646
|
+
llmCalls += 1;
|
|
7647
|
+
const outcome = await caller.complete(prompt.system, prompt.user);
|
|
7648
|
+
if (!outcome.succeeded) {
|
|
7649
|
+
llmFailures += 1;
|
|
7650
|
+
attempts.push(unexecuted(attempt, outcome.error, null, null, started));
|
|
7651
|
+
options.onProgress?.(
|
|
7652
|
+
`fix-loop attempt ${attempt}: LLM failed \u2014 ${outcome.error.slice(0, 160)}`
|
|
7653
|
+
);
|
|
7654
|
+
continue;
|
|
7655
|
+
}
|
|
7656
|
+
const usage = outcome.value.usage;
|
|
7657
|
+
promptTokens += usage?.promptTokens ?? 0;
|
|
7658
|
+
completionTokens += usage?.completionTokens ?? 0;
|
|
7659
|
+
const command = extractFixCommand(outcome.value.content);
|
|
7660
|
+
if (command === null) {
|
|
7661
|
+
llmFailures += 1;
|
|
7662
|
+
attempts.push(
|
|
7663
|
+
unexecuted(
|
|
7664
|
+
attempt,
|
|
7665
|
+
`completion carried no usable command: ${clipText(outcome.value.content, 300)}`,
|
|
7666
|
+
usage,
|
|
7667
|
+
null,
|
|
7668
|
+
started
|
|
7669
|
+
)
|
|
7670
|
+
);
|
|
7671
|
+
continue;
|
|
7672
|
+
}
|
|
7673
|
+
if (attempt > 1) {
|
|
7674
|
+
const commandLines = countScriptCommands(command);
|
|
7675
|
+
if (commandLines > scriptCap) {
|
|
7676
|
+
llmFailures += 1;
|
|
7677
|
+
attempts.push(
|
|
7678
|
+
unexecuted(
|
|
7679
|
+
attempt,
|
|
7680
|
+
`script exceeds ${scriptCap} command lines (${commandLines})`,
|
|
7681
|
+
usage,
|
|
7682
|
+
null,
|
|
7683
|
+
started
|
|
7684
|
+
)
|
|
7685
|
+
);
|
|
7686
|
+
options.onProgress?.(
|
|
7687
|
+
`fix-loop attempt ${attempt}: rejected script with ${commandLines} command lines`
|
|
7688
|
+
);
|
|
7689
|
+
continue;
|
|
7690
|
+
}
|
|
7691
|
+
}
|
|
7692
|
+
let execution;
|
|
7693
|
+
try {
|
|
7694
|
+
execution = await executor(command, attempt);
|
|
7695
|
+
} catch (err) {
|
|
7696
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
7697
|
+
attempts.push({
|
|
7698
|
+
...unexecuted(attempt, "", usage, command, started),
|
|
7699
|
+
llmError: null,
|
|
7700
|
+
armBError: message.slice(0, 500)
|
|
7701
|
+
});
|
|
7702
|
+
options.onProgress?.(
|
|
7703
|
+
`fix-loop attempt ${attempt}: sandbox error \u2014 ${message.slice(0, 160)}`
|
|
7704
|
+
);
|
|
7705
|
+
return {
|
|
7706
|
+
flipped: false,
|
|
7707
|
+
flippedAtAttempt: null,
|
|
7708
|
+
attempts,
|
|
7709
|
+
aborted: true,
|
|
7710
|
+
llmCalls,
|
|
7711
|
+
llmFailures,
|
|
7712
|
+
promptTokens,
|
|
7713
|
+
completionTokens
|
|
7714
|
+
};
|
|
7715
|
+
}
|
|
7716
|
+
attempts.push({
|
|
7717
|
+
attempt,
|
|
7718
|
+
command,
|
|
7719
|
+
llmError: null,
|
|
7720
|
+
usage,
|
|
7721
|
+
executed: true,
|
|
7722
|
+
exitCode: execution.exitCode,
|
|
7723
|
+
prefixExecuted: execution.prefixExecuted,
|
|
7724
|
+
prefixDivergences: execution.prefixDivergences,
|
|
7725
|
+
failureVanished: execution.failureVanished,
|
|
7726
|
+
stdoutTail: clipText(execution.stdout, tailChars),
|
|
7727
|
+
stderrTail: clipText(execution.stderr, tailChars),
|
|
7728
|
+
armBError: null,
|
|
7729
|
+
wallMs: Date.now() - started
|
|
7730
|
+
});
|
|
7731
|
+
options.onProgress?.(
|
|
7732
|
+
`fix-loop attempt ${attempt}: exit=${execution.exitCode} failureVanished=${execution.failureVanished}`
|
|
7733
|
+
);
|
|
7734
|
+
if (execution.failureVanished) {
|
|
7735
|
+
return {
|
|
7736
|
+
flipped: true,
|
|
7737
|
+
flippedAtAttempt: attempt,
|
|
7738
|
+
attempts,
|
|
7739
|
+
aborted: false,
|
|
7740
|
+
llmCalls,
|
|
7741
|
+
llmFailures,
|
|
7742
|
+
promptTokens,
|
|
7743
|
+
completionTokens
|
|
7744
|
+
};
|
|
7745
|
+
}
|
|
7746
|
+
}
|
|
7747
|
+
return {
|
|
7748
|
+
flipped: false,
|
|
7749
|
+
flippedAtAttempt: null,
|
|
7750
|
+
attempts,
|
|
7751
|
+
aborted: false,
|
|
7752
|
+
llmCalls,
|
|
7753
|
+
llmFailures,
|
|
7754
|
+
promptTokens,
|
|
7755
|
+
completionTokens
|
|
7756
|
+
};
|
|
7757
|
+
}
|
|
7758
|
+
|
|
7759
|
+
// src/replay-batch.ts
|
|
7760
|
+
import { execFile } from "child_process";
|
|
7761
|
+
import { createHash as createHash8 } from "crypto";
|
|
7762
|
+
import { appendFileSync, mkdirSync as mkdirSync2, mkdtempSync, writeFileSync as writeFileSync2 } from "fs";
|
|
7763
|
+
import { tmpdir as tmpdir3 } from "os";
|
|
7764
|
+
import { join as join21 } from "path";
|
|
7765
|
+
import { promisify } from "util";
|
|
7766
|
+
import {
|
|
7767
|
+
InMemoryTraceStore as InMemoryTraceStore3,
|
|
7768
|
+
runCounterfactual as runCounterfactual2
|
|
7769
|
+
} from "@tangle-network/agent-eval";
|
|
7770
|
+
var execFileAsync = promisify(execFile);
|
|
7771
|
+
function derivedImageTag(image, cwd) {
|
|
7772
|
+
const digest = createHash8("sha256").update(`${image}
|
|
7773
|
+
${cwd}`).digest("hex").slice(0, 12);
|
|
7774
|
+
return `ctb-replay:${digest}-uid1000`;
|
|
7775
|
+
}
|
|
7776
|
+
function dockerImagePreparer(options = {}) {
|
|
7777
|
+
const pullTimeoutMs = options.pullTimeoutMs ?? 12e5;
|
|
7778
|
+
const buildTimeoutMs = options.buildTimeoutMs ?? 9e5;
|
|
7779
|
+
const imageExists = async (tag) => {
|
|
7780
|
+
try {
|
|
7781
|
+
await execFileAsync("docker", ["image", "inspect", tag], { maxBuffer: 8 * 1024 * 1024 });
|
|
7782
|
+
return true;
|
|
7783
|
+
} catch {
|
|
7784
|
+
return false;
|
|
7785
|
+
}
|
|
7786
|
+
};
|
|
7787
|
+
const errorTail = (err) => {
|
|
7788
|
+
const raw = err && typeof err === "object" && "stderr" in err && typeof err.stderr === "string" && err.stderr.length > 0 ? err.stderr : err instanceof Error ? err.message : String(err);
|
|
7789
|
+
return raw.trim().split("\n").slice(-3).join(" | ").slice(0, 400);
|
|
7524
7790
|
};
|
|
7525
7791
|
return {
|
|
7526
7792
|
async ensure(image, cwd) {
|
|
@@ -7778,9 +8044,10 @@ async function runReplayBatch(options) {
|
|
|
7778
8044
|
}
|
|
7779
8045
|
}
|
|
7780
8046
|
let llm = null;
|
|
7781
|
-
if (options.fix
|
|
8047
|
+
if (options.fix !== "none") {
|
|
7782
8048
|
const caller = options.fixCaller;
|
|
7783
|
-
if (!caller) throw new Error(
|
|
8049
|
+
if (!caller) throw new Error(`replay-verify-batch: fix=${options.fix} requires a fixCaller`);
|
|
8050
|
+
const fixAttempts = options.fixAttempts ?? 3;
|
|
7784
8051
|
const maxFixCases = options.maxFixCases ?? 30;
|
|
7785
8052
|
const seed = options.seed ?? 17;
|
|
7786
8053
|
const eligible = rows.filter((r) => r.status === "ok" && r.replayed);
|
|
@@ -7809,20 +8076,115 @@ async function runReplayBatch(options) {
|
|
|
7809
8076
|
armBPrefixExecuted: null,
|
|
7810
8077
|
armBPrefixDivergences: null,
|
|
7811
8078
|
failureVanished: null,
|
|
7812
|
-
armBError: null
|
|
8079
|
+
armBError: null,
|
|
8080
|
+
attempts: null,
|
|
8081
|
+
flippedAtAttempt: null
|
|
7813
8082
|
}
|
|
7814
8083
|
};
|
|
7815
8084
|
continue;
|
|
7816
8085
|
}
|
|
7817
8086
|
const replayCase = selected.find((c) => c.trajId === row.trajId && c.corpus === row.corpus);
|
|
7818
8087
|
const label = `fix ${row.corpus}/${row.trajId}`;
|
|
7819
|
-
|
|
7820
|
-
|
|
7821
|
-
const
|
|
8088
|
+
const signature = verdictByTraj.get(row.trajId)?.signature ?? null;
|
|
8089
|
+
const stepTimeoutMs = options.stepTimeoutMs ?? replayCase.recordedStepTimeoutMs ?? 12e4;
|
|
8090
|
+
const promptInput = {
|
|
7822
8091
|
taskStatement: replayCase.taskStatement,
|
|
7823
8092
|
steps: replayCase.steps,
|
|
7824
8093
|
k: replayCase.k
|
|
7825
|
-
}
|
|
8094
|
+
};
|
|
8095
|
+
const caseOut = join21(options.out, caseOutDirName(row));
|
|
8096
|
+
if (options.fix === "loop") {
|
|
8097
|
+
onProgress(`${label}: fix loop (budget ${fixAttempts} attempts)`);
|
|
8098
|
+
const result = await runFixLoop(
|
|
8099
|
+
caller,
|
|
8100
|
+
promptInput,
|
|
8101
|
+
async (command, attempt) => {
|
|
8102
|
+
onProgress(
|
|
8103
|
+
`${label}: attempt ${attempt} arm B \u2014 ${command.split("\n")[0].slice(0, 120)}`
|
|
8104
|
+
);
|
|
8105
|
+
const armB = await executeArmB(
|
|
8106
|
+
replayCase,
|
|
8107
|
+
command,
|
|
8108
|
+
backendFactory(row.derivedImage),
|
|
8109
|
+
signature,
|
|
8110
|
+
stepTimeoutMs,
|
|
8111
|
+
options.prefixLimit,
|
|
8112
|
+
(message) => onProgress(`${label}: attempt ${attempt}: ${message}`)
|
|
8113
|
+
);
|
|
8114
|
+
mkdirSync2(caseOut, { recursive: true });
|
|
8115
|
+
writeFileSync2(
|
|
8116
|
+
join21(caseOut, `armB-attempt${attempt}-result.json`),
|
|
8117
|
+
`${JSON.stringify({ command, ...armB }, null, 2)}
|
|
8118
|
+
`
|
|
8119
|
+
);
|
|
8120
|
+
writeFileSync2(
|
|
8121
|
+
join21(caseOut, "armB-result.json"),
|
|
8122
|
+
`${JSON.stringify({ command, attempt, ...armB }, null, 2)}
|
|
8123
|
+
`
|
|
8124
|
+
);
|
|
8125
|
+
return armB;
|
|
8126
|
+
},
|
|
8127
|
+
{ maxAttempts: fixAttempts, onProgress: (message) => onProgress(`${label}: ${message}`) }
|
|
8128
|
+
);
|
|
8129
|
+
calls += result.llmCalls;
|
|
8130
|
+
failures += result.llmFailures;
|
|
8131
|
+
promptTokens += result.promptTokens;
|
|
8132
|
+
completionTokens += result.completionTokens;
|
|
8133
|
+
const usage = result.promptTokens + result.completionTokens > 0 ? { promptTokens: result.promptTokens, completionTokens: result.completionTokens } : null;
|
|
8134
|
+
const flippedRecord = result.flippedAtAttempt !== null ? result.attempts.find((a) => a.attempt === result.flippedAtAttempt) : null;
|
|
8135
|
+
const summary = flippedRecord ?? [...result.attempts].reverse().find((a) => a.executed) ?? null;
|
|
8136
|
+
const lastRecord = result.attempts.at(-1) ?? null;
|
|
8137
|
+
rows[index] = {
|
|
8138
|
+
...row,
|
|
8139
|
+
fix: summary ? {
|
|
8140
|
+
attempted: true,
|
|
8141
|
+
sampledOut: false,
|
|
8142
|
+
command: summary.command,
|
|
8143
|
+
llmError: null,
|
|
8144
|
+
usage,
|
|
8145
|
+
armBExit: summary.exitCode,
|
|
8146
|
+
armBPrefixExecuted: summary.prefixExecuted,
|
|
8147
|
+
armBPrefixDivergences: summary.prefixDivergences,
|
|
8148
|
+
failureVanished: summary.failureVanished,
|
|
8149
|
+
armBError: null,
|
|
8150
|
+
attempts: result.attempts,
|
|
8151
|
+
flippedAtAttempt: result.flippedAtAttempt
|
|
8152
|
+
} : result.aborted ? {
|
|
8153
|
+
attempted: true,
|
|
8154
|
+
sampledOut: false,
|
|
8155
|
+
command: lastRecord?.command ?? null,
|
|
8156
|
+
llmError: null,
|
|
8157
|
+
usage,
|
|
8158
|
+
armBExit: null,
|
|
8159
|
+
armBPrefixExecuted: null,
|
|
8160
|
+
armBPrefixDivergences: null,
|
|
8161
|
+
failureVanished: null,
|
|
8162
|
+
armBError: lastRecord?.armBError ?? "sandbox error",
|
|
8163
|
+
attempts: result.attempts,
|
|
8164
|
+
flippedAtAttempt: null
|
|
8165
|
+
} : {
|
|
8166
|
+
attempted: true,
|
|
8167
|
+
sampledOut: false,
|
|
8168
|
+
command: null,
|
|
8169
|
+
llmError: lastRecord?.llmError ?? "no attempt produced a runnable fix",
|
|
8170
|
+
usage,
|
|
8171
|
+
armBExit: null,
|
|
8172
|
+
armBPrefixExecuted: null,
|
|
8173
|
+
armBPrefixDivergences: null,
|
|
8174
|
+
failureVanished: null,
|
|
8175
|
+
armBError: null,
|
|
8176
|
+
attempts: result.attempts,
|
|
8177
|
+
flippedAtAttempt: null
|
|
8178
|
+
}
|
|
8179
|
+
};
|
|
8180
|
+
onProgress(
|
|
8181
|
+
`${label}: loop done \u2014 flipped=${result.flipped}` + (result.flippedAtAttempt !== null ? ` at attempt ${result.flippedAtAttempt}` : "") + ` (${result.llmCalls} calls, ${result.attempts.filter((a) => a.executed).length} arms)`
|
|
8182
|
+
);
|
|
8183
|
+
continue;
|
|
8184
|
+
}
|
|
8185
|
+
onProgress(`${label}: generating corrected command`);
|
|
8186
|
+
calls += 1;
|
|
8187
|
+
const generated = await generateFixCommand(caller, promptInput);
|
|
7826
8188
|
if (!generated.succeeded) {
|
|
7827
8189
|
failures += 1;
|
|
7828
8190
|
rows[index] = {
|
|
@@ -7837,7 +8199,9 @@ async function runReplayBatch(options) {
|
|
|
7837
8199
|
armBPrefixExecuted: null,
|
|
7838
8200
|
armBPrefixDivergences: null,
|
|
7839
8201
|
failureVanished: null,
|
|
7840
|
-
armBError: null
|
|
8202
|
+
armBError: null,
|
|
8203
|
+
attempts: null,
|
|
8204
|
+
flippedAtAttempt: null
|
|
7841
8205
|
}
|
|
7842
8206
|
};
|
|
7843
8207
|
onProgress(`${label}: LLM failed \u2014 ${generated.error.slice(0, 200)}`);
|
|
@@ -7845,8 +8209,6 @@ async function runReplayBatch(options) {
|
|
|
7845
8209
|
}
|
|
7846
8210
|
promptTokens += generated.value.usage?.promptTokens ?? 0;
|
|
7847
8211
|
completionTokens += generated.value.usage?.completionTokens ?? 0;
|
|
7848
|
-
const signature = verdictByTraj.get(row.trajId)?.signature ?? null;
|
|
7849
|
-
const stepTimeoutMs = options.stepTimeoutMs ?? replayCase.recordedStepTimeoutMs ?? 12e4;
|
|
7850
8212
|
onProgress(`${label}: arm B \u2014 ${generated.value.command.split("\n")[0].slice(0, 120)}`);
|
|
7851
8213
|
try {
|
|
7852
8214
|
const armB = await executeArmB(
|
|
@@ -7870,10 +8232,11 @@ async function runReplayBatch(options) {
|
|
|
7870
8232
|
armBPrefixExecuted: armB.prefixExecuted,
|
|
7871
8233
|
armBPrefixDivergences: armB.prefixDivergences,
|
|
7872
8234
|
failureVanished: armB.failureVanished,
|
|
7873
|
-
armBError: null
|
|
8235
|
+
armBError: null,
|
|
8236
|
+
attempts: null,
|
|
8237
|
+
flippedAtAttempt: armB.failureVanished ? 1 : null
|
|
7874
8238
|
}
|
|
7875
8239
|
};
|
|
7876
|
-
const caseOut = join21(options.out, caseOutDirName(row));
|
|
7877
8240
|
mkdirSync2(caseOut, { recursive: true });
|
|
7878
8241
|
writeFileSync2(
|
|
7879
8242
|
join21(caseOut, "armB-result.json"),
|
|
@@ -7895,7 +8258,9 @@ async function runReplayBatch(options) {
|
|
|
7895
8258
|
armBPrefixExecuted: null,
|
|
7896
8259
|
armBPrefixDivergences: null,
|
|
7897
8260
|
failureVanished: null,
|
|
7898
|
-
armBError: message.slice(0, 500)
|
|
8261
|
+
armBError: message.slice(0, 500),
|
|
8262
|
+
attempts: null,
|
|
8263
|
+
flippedAtAttempt: null
|
|
7899
8264
|
}
|
|
7900
8265
|
};
|
|
7901
8266
|
onProgress(`${label}: arm B error \u2014 ${message.slice(0, 200)}`);
|
|
@@ -7920,6 +8285,15 @@ async function runReplayBatch(options) {
|
|
|
7920
8285
|
(r) => r.recordedReturncodeAtK !== null && r.recordedReturncodeAtK !== 0
|
|
7921
8286
|
);
|
|
7922
8287
|
const flippedNonzeroRc = armBNonzeroRc.filter((r) => r.fix.failureVanished === true);
|
|
8288
|
+
const attempt1Executed = rows.filter(
|
|
8289
|
+
(r) => r.fix?.attempts?.find((a) => a.attempt === 1)?.executed === true
|
|
8290
|
+
);
|
|
8291
|
+
const flippedAt1 = rows.filter((r) => r.fix?.flippedAtAttempt === 1);
|
|
8292
|
+
const flipsByAttempt = {};
|
|
8293
|
+
for (const row of rows) {
|
|
8294
|
+
const at = row.fix?.flippedAtAttempt;
|
|
8295
|
+
if (typeof at === "number") flipsByAttempt[String(at)] = (flipsByAttempt[String(at)] ?? 0) + 1;
|
|
8296
|
+
}
|
|
7923
8297
|
const excludedByReason = {};
|
|
7924
8298
|
for (const excluded of enumeration.excluded) {
|
|
7925
8299
|
excludedByReason[excluded.reason] = (excludedByReason[excluded.reason] ?? 0) + 1;
|
|
@@ -7959,16 +8333,22 @@ async function runReplayBatch(options) {
|
|
|
7959
8333
|
denominator: selected.length,
|
|
7960
8334
|
value: rate(signatureStrict.length, selected.length)
|
|
7961
8335
|
},
|
|
7962
|
-
fixFlipRate: options.fix
|
|
8336
|
+
fixFlipRate: options.fix !== "none" ? {
|
|
7963
8337
|
numerator: flipped.length,
|
|
7964
8338
|
denominator: armBExecuted.length,
|
|
7965
8339
|
value: rate(flipped.length, armBExecuted.length)
|
|
7966
8340
|
} : null,
|
|
7967
|
-
fixFlipRateNonzeroRc: options.fix
|
|
8341
|
+
fixFlipRateNonzeroRc: options.fix !== "none" ? {
|
|
7968
8342
|
numerator: flippedNonzeroRc.length,
|
|
7969
8343
|
denominator: armBNonzeroRc.length,
|
|
7970
8344
|
value: rate(flippedNonzeroRc.length, armBNonzeroRc.length)
|
|
7971
|
-
} : null
|
|
8345
|
+
} : null,
|
|
8346
|
+
fixFlipAttempt1: options.fix === "loop" ? {
|
|
8347
|
+
numerator: flippedAt1.length,
|
|
8348
|
+
denominator: attempt1Executed.length,
|
|
8349
|
+
value: rate(flippedAt1.length, attempt1Executed.length)
|
|
8350
|
+
} : null,
|
|
8351
|
+
flipsByAttempt: options.fix === "loop" ? flipsByAttempt : null
|
|
7972
8352
|
},
|
|
7973
8353
|
llm,
|
|
7974
8354
|
excluded: enumeration.excluded,
|
|
@@ -8012,6 +8392,15 @@ function renderBatchReport(report) {
|
|
|
8012
8392
|
`- Fix-flip rate on recorded-rc\u22600 cases: ${pct(headline.fixFlipRateNonzeroRc.value)} (${headline.fixFlipRateNonzeroRc.numerator}/${headline.fixFlipRateNonzeroRc.denominator}; real recorded failures \u2014 a gold step recorded with rc 0 flips vacuously).`
|
|
8013
8393
|
);
|
|
8014
8394
|
}
|
|
8395
|
+
if (headline.fixFlipAttempt1) {
|
|
8396
|
+
lines.push(
|
|
8397
|
+
`- Fix-flip@1: ${pct(headline.fixFlipAttempt1.value)} (${headline.fixFlipAttempt1.numerator}/${headline.fixFlipAttempt1.denominator} cases whose attempt 1 executed \u2014 the one-shot-comparable number).`
|
|
8398
|
+
);
|
|
8399
|
+
}
|
|
8400
|
+
if (headline.flipsByAttempt && Object.keys(headline.flipsByAttempt).length > 0) {
|
|
8401
|
+
const parts = Object.entries(headline.flipsByAttempt).sort((a, b) => Number(a[0]) - Number(b[0])).map(([attempt, count2]) => `attempt ${attempt}: ${count2}`);
|
|
8402
|
+
lines.push(`- Flips by attempt: ${parts.join(", ")}.`);
|
|
8403
|
+
}
|
|
8015
8404
|
lines.push("");
|
|
8016
8405
|
lines.push("## Enumeration");
|
|
8017
8406
|
lines.push("");
|
|
@@ -8043,9 +8432,9 @@ function renderBatchReport(report) {
|
|
|
8043
8432
|
lines.push("");
|
|
8044
8433
|
lines.push("| corpus | trajectory | image | error |");
|
|
8045
8434
|
lines.push("| --- | --- | --- | --- |");
|
|
8046
|
-
for (const
|
|
8435
|
+
for (const failure2 of report.pullFailures) {
|
|
8047
8436
|
lines.push(
|
|
8048
|
-
`| ${
|
|
8437
|
+
`| ${failure2.corpus} | ${failure2.trajId} | \`${failure2.image}\` | ${failure2.error.replaceAll("|", "\\|")} |`
|
|
8049
8438
|
);
|
|
8050
8439
|
}
|
|
8051
8440
|
}
|
|
@@ -8060,7 +8449,7 @@ function renderBatchReport(report) {
|
|
|
8060
8449
|
);
|
|
8061
8450
|
for (const row of report.cases) {
|
|
8062
8451
|
const fix = row.fix;
|
|
8063
|
-
const fixCell = !fix ? "\u2014" : fix.sampledOut ? "sampled-out" : fix.llmError ? "llm-failed" : fix.armBError ? "armB-error" : "generated";
|
|
8452
|
+
const fixCell = !fix ? "\u2014" : fix.sampledOut ? "sampled-out" : fix.llmError ? "llm-failed" : fix.armBError ? "armB-error" : fix.attempts ? fix.flippedAtAttempt !== null ? `flip@${fix.flippedAtAttempt}` : `exhausted(${fix.attempts.length})` : "generated";
|
|
8064
8453
|
lines.push(
|
|
8065
8454
|
[
|
|
8066
8455
|
row.corpus,
|
|
@@ -8122,13 +8511,20 @@ function parseReplayBatchArgs(argv) {
|
|
|
8122
8511
|
return n;
|
|
8123
8512
|
};
|
|
8124
8513
|
const fix = values.get("--fix") ?? "none";
|
|
8125
|
-
if (fix !== "none" && fix !== "generate") {
|
|
8126
|
-
throw new Error(`replay-verify-batch: --fix must be none or
|
|
8514
|
+
if (fix !== "none" && fix !== "generate" && fix !== "loop") {
|
|
8515
|
+
throw new Error(`replay-verify-batch: --fix must be none, generate, or loop, got ${fix}`);
|
|
8516
|
+
}
|
|
8517
|
+
const fixAttempts = optionalNumber("--fix-attempts") ?? 3;
|
|
8518
|
+
if (!Number.isInteger(fixAttempts) || fixAttempts < 1) {
|
|
8519
|
+
throw new Error(
|
|
8520
|
+
`replay-verify-batch: --fix-attempts must be a positive integer, got ${values.get("--fix-attempts")}`
|
|
8521
|
+
);
|
|
8127
8522
|
}
|
|
8128
8523
|
return {
|
|
8129
8524
|
corpora,
|
|
8130
8525
|
out: out ?? ".",
|
|
8131
8526
|
fix,
|
|
8527
|
+
fixAttempts,
|
|
8132
8528
|
fixModel: values.get("--fix-model") ?? "glm-5.2",
|
|
8133
8529
|
fixBaseUrl: values.get("--fix-base-url") ?? "https://api.z.ai/api/coding/paas/v4",
|
|
8134
8530
|
fixApiKeyEnv: values.get("--fix-api-key-env") ?? "ZAI_GLM_API_KEY",
|
|
@@ -8150,7 +8546,7 @@ function replayBatchUsage() {
|
|
|
8150
8546
|
Usage:
|
|
8151
8547
|
traces replay-verify-batch \\
|
|
8152
8548
|
--corpus NAME=<labels.json>::<prepared-dir> [--corpus ...] \\
|
|
8153
|
-
--out DIR [--fix none|generate] [--enumerate-only] \\
|
|
8549
|
+
--out DIR [--fix none|generate|loop] [--fix-attempts 3] [--enumerate-only] \\
|
|
8154
8550
|
[--fix-model glm-5.2] [--fix-base-url URL] [--fix-api-key-env ZAI_GLM_API_KEY] \\
|
|
8155
8551
|
[--max-fix-cases 30] [--seed 17] [--step-timeout MS] [--prefix-limit N] \\
|
|
8156
8552
|
[--case-filter SUBSTRING] [--case-limit N] \\
|
|
@@ -8169,6 +8565,13 @@ Usage:
|
|
|
8169
8565
|
--max-fix-cases, seeded sample beyond it) and executes the corrected command
|
|
8170
8566
|
as arm B in a fresh sandbox.
|
|
8171
8567
|
|
|
8568
|
+
--fix loop iterates: attempt 1 is the one-shot prompt; when an arm fails
|
|
8569
|
+
(nonzero exit or the failure signature persists) or the model call fails,
|
|
8570
|
+
the next prompt carries every prior command with its REAL executed
|
|
8571
|
+
stdout/stderr, up to --fix-attempts attempts (default 3). Retries may
|
|
8572
|
+
answer with a short script (<=5 commands) executed as one /bin/sh unit.
|
|
8573
|
+
Every attempt runs in its own fresh sandbox with the same replayed prefix.
|
|
8574
|
+
|
|
8172
8575
|
Outputs batch-report.json, batch-report.md, cases.jsonl (incremental), and one
|
|
8173
8576
|
directory per case with the full replay-verify artifacts.
|
|
8174
8577
|
See docs/replay-verify.md for orchestrator setup and honest limits.
|
|
@@ -8213,7 +8616,7 @@ async function cmdReplayVerifyBatch(argv) {
|
|
|
8213
8616
|
);
|
|
8214
8617
|
}
|
|
8215
8618
|
let fixCaller;
|
|
8216
|
-
if (parsed.fix
|
|
8619
|
+
if (parsed.fix !== "none") {
|
|
8217
8620
|
const fixApiKey = process.env[parsed.fixApiKeyEnv];
|
|
8218
8621
|
if (!fixApiKey) {
|
|
8219
8622
|
throw new Error(
|
|
@@ -8232,6 +8635,7 @@ async function cmdReplayVerifyBatch(argv) {
|
|
|
8232
8635
|
baseUrl: parsed.baseUrl,
|
|
8233
8636
|
apiKey,
|
|
8234
8637
|
fix: parsed.fix,
|
|
8638
|
+
fixAttempts: parsed.fixAttempts,
|
|
8235
8639
|
fixCaller,
|
|
8236
8640
|
fixModelLabel: parsed.fixModel,
|
|
8237
8641
|
maxFixCases: parsed.maxFixCases,
|
|
@@ -8251,6 +8655,573 @@ async function cmdReplayVerifyBatch(argv) {
|
|
|
8251
8655
|
);
|
|
8252
8656
|
}
|
|
8253
8657
|
|
|
8658
|
+
// src/replay-wire.ts
|
|
8659
|
+
var SUBJECT_PATTERN = /^incorrect-steps-(\d+)-(\d+)-(escaped|unescaped)-consequence-(\d+)$/;
|
|
8660
|
+
function parseIncorrectStepsSubject(subject) {
|
|
8661
|
+
const match = SUBJECT_PATTERN.exec(subject);
|
|
8662
|
+
if (!match) return null;
|
|
8663
|
+
return {
|
|
8664
|
+
firstStep: Number(match[1]),
|
|
8665
|
+
lastStep: Number(match[2]),
|
|
8666
|
+
escapeStatus: match[3],
|
|
8667
|
+
consequenceStep: Number(match[4])
|
|
8668
|
+
};
|
|
8669
|
+
}
|
|
8670
|
+
function resolveFindingInvocation(finding2, corpora) {
|
|
8671
|
+
const subject = parseIncorrectStepsSubject(finding2.subject);
|
|
8672
|
+
if (!subject) {
|
|
8673
|
+
throw new Error(
|
|
8674
|
+
`replay-wire: subject '${finding2.subject}' is not incorrect-steps-<first>-<last>-<escaped|unescaped>-consequence-<step>`
|
|
8675
|
+
);
|
|
8676
|
+
}
|
|
8677
|
+
const failures = [];
|
|
8678
|
+
for (const corpus of corpora) {
|
|
8679
|
+
const resolution = resolveCaseResources(corpus, finding2.trajId);
|
|
8680
|
+
if (resolution.resolved) {
|
|
8681
|
+
const at = subject.firstStep;
|
|
8682
|
+
const step = resolution.resources.steps.find((s) => s.step_id === at);
|
|
8683
|
+
if (!step) {
|
|
8684
|
+
throw new Error(
|
|
8685
|
+
`replay-wire: finding step ${at} is outside ${finding2.trajId} (${resolution.resources.steps.length} steps)`
|
|
8686
|
+
);
|
|
8687
|
+
}
|
|
8688
|
+
return { resources: resolution.resources, subject, at };
|
|
8689
|
+
}
|
|
8690
|
+
failures.push(`${corpus.name}: ${resolution.reason}`);
|
|
8691
|
+
}
|
|
8692
|
+
throw new Error(
|
|
8693
|
+
`replay-wire: trajectory ${finding2.trajId} is not replayable in any corpus \u2014 ${failures.join("; ")}`
|
|
8694
|
+
);
|
|
8695
|
+
}
|
|
8696
|
+
async function replayVerifyFinding(finding2, options) {
|
|
8697
|
+
if (options.fixCaller && options.fixCommand !== void 0) {
|
|
8698
|
+
throw new Error("replay-wire: pass fixCaller or fixCommand, not both");
|
|
8699
|
+
}
|
|
8700
|
+
const invocation = resolveFindingInvocation(finding2, options.corpora);
|
|
8701
|
+
let fixCommand = options.fixCommand ?? null;
|
|
8702
|
+
if (options.fixCaller) {
|
|
8703
|
+
const generated = await generateFixCommand(options.fixCaller, {
|
|
8704
|
+
taskStatement: invocation.resources.taskStatement,
|
|
8705
|
+
steps: invocation.resources.steps,
|
|
8706
|
+
k: invocation.at
|
|
8707
|
+
});
|
|
8708
|
+
if (!generated.succeeded) {
|
|
8709
|
+
throw new Error(`replay-wire: fix generation failed \u2014 ${generated.error}`);
|
|
8710
|
+
}
|
|
8711
|
+
fixCommand = generated.value.command;
|
|
8712
|
+
}
|
|
8713
|
+
const verdict = await replayVerify({
|
|
8714
|
+
stepsPath: invocation.resources.stepsPath,
|
|
8715
|
+
image: invocation.resources.image,
|
|
8716
|
+
at: invocation.at,
|
|
8717
|
+
fixCommand: fixCommand ?? void 0,
|
|
8718
|
+
cwd: invocation.resources.cwd,
|
|
8719
|
+
out: options.out,
|
|
8720
|
+
caseId: invocation.resources.trajId,
|
|
8721
|
+
stepTimeoutMs: options.stepTimeoutMs ?? invocation.resources.recordedStepTimeoutMs ?? void 0,
|
|
8722
|
+
prefixLimit: options.prefixLimit,
|
|
8723
|
+
backend: options.backend,
|
|
8724
|
+
apiKey: options.apiKey,
|
|
8725
|
+
baseUrl: options.baseUrl,
|
|
8726
|
+
onProgress: options.onProgress
|
|
8727
|
+
});
|
|
8728
|
+
return { invocation, fixCommand, verdict };
|
|
8729
|
+
}
|
|
8730
|
+
|
|
8731
|
+
// src/analyze-verify.ts
|
|
8732
|
+
import { mkdirSync as mkdirSync3, readFileSync as readFileSync4, writeFileSync as writeFileSync3 } from "fs";
|
|
8733
|
+
import { join as join22 } from "path";
|
|
8734
|
+
var SINGLE_STEP_SUBJECT = /^incorrect-step-(\d+)$/;
|
|
8735
|
+
function findingReplayStep(finding2) {
|
|
8736
|
+
const fromMetadata = finding2.metadata?.block_first_step;
|
|
8737
|
+
if (typeof fromMetadata === "number" && Number.isInteger(fromMetadata) && fromMetadata >= 1) {
|
|
8738
|
+
return fromMetadata;
|
|
8739
|
+
}
|
|
8740
|
+
const subject = finding2.subject ?? "";
|
|
8741
|
+
const wire = parseIncorrectStepsSubject(subject);
|
|
8742
|
+
if (wire) return wire.firstStep;
|
|
8743
|
+
const single = SINGLE_STEP_SUBJECT.exec(subject);
|
|
8744
|
+
if (single) return Number(single[1]);
|
|
8745
|
+
return null;
|
|
8746
|
+
}
|
|
8747
|
+
var TRACE_EVIDENCE_URI = /^trace:\/\/([^/]+)\//;
|
|
8748
|
+
function findingTrajectoryId(finding2) {
|
|
8749
|
+
for (const ref of finding2.evidence_refs ?? []) {
|
|
8750
|
+
if (typeof ref.uri !== "string") continue;
|
|
8751
|
+
const match = TRACE_EVIDENCE_URI.exec(ref.uri);
|
|
8752
|
+
if (match) return match[1];
|
|
8753
|
+
}
|
|
8754
|
+
return null;
|
|
8755
|
+
}
|
|
8756
|
+
function checkStep(steps, at) {
|
|
8757
|
+
const step = steps.find((s) => s.step_id === at);
|
|
8758
|
+
if (!step) {
|
|
8759
|
+
return { ok: false, reason: `step ${at} is outside the trajectory (${steps.length} steps)` };
|
|
8760
|
+
}
|
|
8761
|
+
if (isSubmitAction(step.action)) {
|
|
8762
|
+
return {
|
|
8763
|
+
ok: false,
|
|
8764
|
+
reason: `step ${at} is the submit action \u2014 a submit decision has no executable failure to replay`
|
|
8765
|
+
};
|
|
8766
|
+
}
|
|
8767
|
+
const recordedReturncode = parseRecordedReturncode(step.observation);
|
|
8768
|
+
if (recordedReturncode === null) {
|
|
8769
|
+
return {
|
|
8770
|
+
ok: false,
|
|
8771
|
+
reason: `step ${at} recorded no returncode \u2014 there is no executable failure signature to reproduce`
|
|
8772
|
+
};
|
|
8773
|
+
}
|
|
8774
|
+
return { ok: true, recordedReturncode };
|
|
8775
|
+
}
|
|
8776
|
+
function resolveFindingReplayability(finding2, source) {
|
|
8777
|
+
const at = findingReplayStep(finding2);
|
|
8778
|
+
if (at === null) {
|
|
8779
|
+
return {
|
|
8780
|
+
replayable: false,
|
|
8781
|
+
reason: `subject '${finding2.subject ?? "(none)"}' names no trajectory step (expected incorrect-step-<n>, incorrect-steps-<f>-<l>-\u2026, or metadata.block_first_step)`
|
|
8782
|
+
};
|
|
8783
|
+
}
|
|
8784
|
+
if (source.kind === "direct") {
|
|
8785
|
+
const trajId2 = findingTrajectoryId(finding2);
|
|
8786
|
+
if (trajId2 && source.caseId && trajId2 !== source.caseId) {
|
|
8787
|
+
return {
|
|
8788
|
+
replayable: false,
|
|
8789
|
+
reason: `finding cites trajectory '${trajId2}' but the supplied steps are case '${source.caseId}'`
|
|
8790
|
+
};
|
|
8791
|
+
}
|
|
8792
|
+
let steps;
|
|
8793
|
+
try {
|
|
8794
|
+
steps = JSON.parse(readFileSync4(source.stepsPath, "utf8"));
|
|
8795
|
+
} catch (err) {
|
|
8796
|
+
throw new Error(
|
|
8797
|
+
`verify-findings: cannot read steps file ${source.stepsPath} \u2014 ${err instanceof Error ? err.message : String(err)}`
|
|
8798
|
+
);
|
|
8799
|
+
}
|
|
8800
|
+
if (!Array.isArray(steps) || steps.length === 0) {
|
|
8801
|
+
throw new Error(`verify-findings: ${source.stepsPath} is not a non-empty steps array`);
|
|
8802
|
+
}
|
|
8803
|
+
const step = checkStep(steps, at);
|
|
8804
|
+
if (!step.ok) return { replayable: false, reason: step.reason };
|
|
8805
|
+
return {
|
|
8806
|
+
replayable: true,
|
|
8807
|
+
resolved: {
|
|
8808
|
+
caseId: source.caseId ?? trajId2 ?? source.stepsPath,
|
|
8809
|
+
stepsPath: source.stepsPath,
|
|
8810
|
+
image: source.image,
|
|
8811
|
+
cwd: source.cwd,
|
|
8812
|
+
at,
|
|
8813
|
+
recordedReturncode: step.recordedReturncode,
|
|
8814
|
+
recordedStepTimeoutMs: null
|
|
8815
|
+
}
|
|
8816
|
+
};
|
|
8817
|
+
}
|
|
8818
|
+
const trajId = findingTrajectoryId(finding2);
|
|
8819
|
+
if (!trajId) {
|
|
8820
|
+
return {
|
|
8821
|
+
replayable: false,
|
|
8822
|
+
reason: "finding carries no trace://<trajectory>/ evidence ref naming its trajectory"
|
|
8823
|
+
};
|
|
8824
|
+
}
|
|
8825
|
+
const failures = [];
|
|
8826
|
+
for (const corpus of source.corpora) {
|
|
8827
|
+
const resolution = resolveCaseResources(corpus, trajId);
|
|
8828
|
+
if (!resolution.resolved) {
|
|
8829
|
+
failures.push(`${corpus.name}: ${resolution.reason}${resolution.detail ? ` (${resolution.detail})` : ""}`);
|
|
8830
|
+
continue;
|
|
8831
|
+
}
|
|
8832
|
+
const step = checkStep(resolution.resources.steps, at);
|
|
8833
|
+
if (!step.ok) return { replayable: false, reason: step.reason };
|
|
8834
|
+
return {
|
|
8835
|
+
replayable: true,
|
|
8836
|
+
resolved: {
|
|
8837
|
+
caseId: trajId,
|
|
8838
|
+
stepsPath: resolution.resources.stepsPath,
|
|
8839
|
+
image: resolution.resources.image,
|
|
8840
|
+
cwd: resolution.resources.cwd,
|
|
8841
|
+
at,
|
|
8842
|
+
recordedReturncode: step.recordedReturncode,
|
|
8843
|
+
recordedStepTimeoutMs: resolution.resources.recordedStepTimeoutMs
|
|
8844
|
+
}
|
|
8845
|
+
};
|
|
8846
|
+
}
|
|
8847
|
+
return {
|
|
8848
|
+
replayable: false,
|
|
8849
|
+
reason: `trajectory ${trajId} is not replayable in any corpus \u2014 ${failures.join("; ")}`
|
|
8850
|
+
};
|
|
8851
|
+
}
|
|
8852
|
+
function classifyVerdict(verdict) {
|
|
8853
|
+
if (!verdict.armA.failureSignatureMatch) return "divergent";
|
|
8854
|
+
if (verdict.armB?.failureVanished) return "fix-flipped";
|
|
8855
|
+
return "reproduced";
|
|
8856
|
+
}
|
|
8857
|
+
var DEFAULT_SANDBOX_BASE_URL = "http://127.0.0.1:4097";
|
|
8858
|
+
async function assertSandboxReachable(baseUrl) {
|
|
8859
|
+
let response;
|
|
8860
|
+
try {
|
|
8861
|
+
response = await fetch(new URL("/health", baseUrl), { signal: AbortSignal.timeout(5e3) });
|
|
8862
|
+
} catch (err) {
|
|
8863
|
+
throw new Error(
|
|
8864
|
+
`verify-findings: sandbox API unreachable at ${baseUrl} \u2014 ${err instanceof Error ? err.message : String(err)}. Executing findings requires a running sandbox orchestrator + SDK adapter (see docs/replay-verify.md); start them or pass --base-url.`
|
|
8865
|
+
);
|
|
8866
|
+
}
|
|
8867
|
+
if (!response.ok) {
|
|
8868
|
+
throw new Error(
|
|
8869
|
+
`verify-findings: sandbox API at ${baseUrl} answered /health with HTTP ${response.status} \u2014 refusing to run proofs against degraded infrastructure`
|
|
8870
|
+
);
|
|
8871
|
+
}
|
|
8872
|
+
}
|
|
8873
|
+
function receiptExecution(verdict) {
|
|
8874
|
+
return {
|
|
8875
|
+
image: verdict.image,
|
|
8876
|
+
cwd: verdict.cwd,
|
|
8877
|
+
recordedReturncode: verdict.recordedReturncode,
|
|
8878
|
+
signature: verdict.signature,
|
|
8879
|
+
signatureBasis: verdict.signatureBasis,
|
|
8880
|
+
armA: {
|
|
8881
|
+
command: verdict.armA.command,
|
|
8882
|
+
exitCode: verdict.armA.exitCode,
|
|
8883
|
+
wallMs: verdict.armA.wallMs,
|
|
8884
|
+
failureSignatureMatch: verdict.armA.failureSignatureMatch
|
|
8885
|
+
},
|
|
8886
|
+
armB: verdict.armB ? {
|
|
8887
|
+
command: verdict.armB.command,
|
|
8888
|
+
exitCode: verdict.armB.exitCode,
|
|
8889
|
+
wallMs: verdict.armB.wallMs,
|
|
8890
|
+
failureVanished: verdict.armB.failureVanished
|
|
8891
|
+
} : null,
|
|
8892
|
+
prefixExecuted: verdict.prefixExecuted,
|
|
8893
|
+
prefixDivergences: verdict.prefixDivergences.length,
|
|
8894
|
+
totalMs: verdict.timings.totalMs
|
|
8895
|
+
};
|
|
8896
|
+
}
|
|
8897
|
+
function writeReceipt(receiptDir, finding2, verification, execution) {
|
|
8898
|
+
const receipt = {
|
|
8899
|
+
schema_version: "1.0.0",
|
|
8900
|
+
produced_at: (/* @__PURE__ */ new Date()).toISOString(),
|
|
8901
|
+
finding_id: verification.finding_id,
|
|
8902
|
+
analyst_id: finding2.analyst_id ?? null,
|
|
8903
|
+
subject: verification.subject,
|
|
8904
|
+
claim: typeof finding2.claim === "string" ? finding2.claim.slice(0, 600) : null,
|
|
8905
|
+
trajectory_id: verification.trajectory_id,
|
|
8906
|
+
step: verification.step,
|
|
8907
|
+
verified: verification.verified,
|
|
8908
|
+
reason: verification.reason,
|
|
8909
|
+
execution,
|
|
8910
|
+
verdict_path: verification.verdict_path,
|
|
8911
|
+
deduplicated_with: verification.deduplicated_with
|
|
8912
|
+
};
|
|
8913
|
+
writeFileSync3(join22(receiptDir, "receipt.json"), `${JSON.stringify(receipt, null, 2)}
|
|
8914
|
+
`);
|
|
8915
|
+
}
|
|
8916
|
+
function receiptDirName(index, finding2) {
|
|
8917
|
+
const id = typeof finding2.finding_id === "string" && finding2.finding_id.length > 0 ? finding2.finding_id.replace(/[^A-Za-z0-9_-]/g, "_") : "finding";
|
|
8918
|
+
return `${String(index + 1).padStart(3, "0")}-${id}`;
|
|
8919
|
+
}
|
|
8920
|
+
async function verifyFindings(findings, options) {
|
|
8921
|
+
if (findings.length === 0) throw new Error("verify-findings: no findings to verify");
|
|
8922
|
+
mkdirSync3(options.out, { recursive: true });
|
|
8923
|
+
const resolutions = findings.map((finding2) => resolveFindingReplayability(finding2, options.source));
|
|
8924
|
+
const baseUrl = options.baseUrl ?? DEFAULT_SANDBOX_BASE_URL;
|
|
8925
|
+
if (resolutions.some((resolution) => resolution.replayable) && !options.backend) {
|
|
8926
|
+
if (!options.apiKey) {
|
|
8927
|
+
throw new Error(
|
|
8928
|
+
"verify-findings: replayable findings need a sandbox API key \u2014 export SANDBOX_API_KEY or pass --api-key-env"
|
|
8929
|
+
);
|
|
8930
|
+
}
|
|
8931
|
+
await assertSandboxReachable(baseUrl);
|
|
8932
|
+
}
|
|
8933
|
+
const preparer = options.source.kind === "corpus" && !options.backend ? options.source.preparer ?? dockerImagePreparer() : null;
|
|
8934
|
+
const preparedImages = /* @__PURE__ */ new Map();
|
|
8935
|
+
const executedByKey = /* @__PURE__ */ new Map();
|
|
8936
|
+
const verifications = [];
|
|
8937
|
+
let executions = 0;
|
|
8938
|
+
for (let index = 0; index < findings.length; index++) {
|
|
8939
|
+
const finding2 = findings[index];
|
|
8940
|
+
const resolution = resolutions[index];
|
|
8941
|
+
const receiptDir = join22(options.out, receiptDirName(index, finding2));
|
|
8942
|
+
mkdirSync3(receiptDir, { recursive: true });
|
|
8943
|
+
const identity = {
|
|
8944
|
+
finding_id: finding2.finding_id ?? null,
|
|
8945
|
+
subject: finding2.subject ?? null,
|
|
8946
|
+
trajectory_id: findingTrajectoryId(finding2)
|
|
8947
|
+
};
|
|
8948
|
+
if (!resolution.replayable) {
|
|
8949
|
+
const verification2 = {
|
|
8950
|
+
...identity,
|
|
8951
|
+
step: findingReplayStep(finding2),
|
|
8952
|
+
verified: "not-replayable",
|
|
8953
|
+
reason: resolution.reason,
|
|
8954
|
+
receipt: receiptDir,
|
|
8955
|
+
verdict_path: null,
|
|
8956
|
+
deduplicated_with: null
|
|
8957
|
+
};
|
|
8958
|
+
writeReceipt(receiptDir, finding2, verification2, null);
|
|
8959
|
+
verifications.push(verification2);
|
|
8960
|
+
options.onProgress?.(`${identity.finding_id ?? `finding ${index + 1}`}: not-replayable \u2014 ${resolution.reason}`);
|
|
8961
|
+
continue;
|
|
8962
|
+
}
|
|
8963
|
+
const resolved = resolution.resolved;
|
|
8964
|
+
const dedupeKey = `${resolved.caseId}::${resolved.at}::${options.fixCommand ?? ""}`;
|
|
8965
|
+
const prior = executedByKey.get(dedupeKey);
|
|
8966
|
+
if (prior) {
|
|
8967
|
+
const verification2 = {
|
|
8968
|
+
...identity,
|
|
8969
|
+
step: resolved.at,
|
|
8970
|
+
verified: prior.verified,
|
|
8971
|
+
reason: prior.reason,
|
|
8972
|
+
receipt: receiptDir,
|
|
8973
|
+
verdict_path: prior.verdict_path,
|
|
8974
|
+
deduplicated_with: prior.receipt
|
|
8975
|
+
};
|
|
8976
|
+
const priorVerdict = prior.verdict_path ? JSON.parse(readFileSync4(prior.verdict_path, "utf8")) : null;
|
|
8977
|
+
writeReceipt(receiptDir, finding2, verification2, priorVerdict ? receiptExecution(priorVerdict) : null);
|
|
8978
|
+
verifications.push(verification2);
|
|
8979
|
+
options.onProgress?.(
|
|
8980
|
+
`${identity.finding_id ?? `finding ${index + 1}`}: ${prior.verified} (shares proof with ${prior.finding_id ?? prior.receipt})`
|
|
8981
|
+
);
|
|
8982
|
+
continue;
|
|
8983
|
+
}
|
|
8984
|
+
let image = resolved.image;
|
|
8985
|
+
if (preparer) {
|
|
8986
|
+
const preparationKey = `${resolved.image}::${resolved.cwd}`;
|
|
8987
|
+
let preparation = preparedImages.get(preparationKey);
|
|
8988
|
+
if (!preparation) {
|
|
8989
|
+
const ensured = await preparer.ensure(resolved.image, resolved.cwd);
|
|
8990
|
+
preparation = ensured.succeeded ? { succeeded: true, image: ensured.value.derivedImage } : { succeeded: false, error: ensured.error };
|
|
8991
|
+
preparedImages.set(preparationKey, preparation);
|
|
8992
|
+
}
|
|
8993
|
+
if (!preparation.succeeded) {
|
|
8994
|
+
const verification2 = {
|
|
8995
|
+
...identity,
|
|
8996
|
+
step: resolved.at,
|
|
8997
|
+
verified: "not-replayable",
|
|
8998
|
+
reason: `replay image could not be prepared \u2014 ${preparation.error}`,
|
|
8999
|
+
receipt: receiptDir,
|
|
9000
|
+
verdict_path: null,
|
|
9001
|
+
deduplicated_with: null
|
|
9002
|
+
};
|
|
9003
|
+
writeReceipt(receiptDir, finding2, verification2, null);
|
|
9004
|
+
verifications.push(verification2);
|
|
9005
|
+
options.onProgress?.(`${identity.finding_id ?? `finding ${index + 1}`}: not-replayable \u2014 image preparation failed`);
|
|
9006
|
+
continue;
|
|
9007
|
+
}
|
|
9008
|
+
image = preparation.image;
|
|
9009
|
+
}
|
|
9010
|
+
options.onProgress?.(
|
|
9011
|
+
`${identity.finding_id ?? `finding ${index + 1}`}: executing arm A at step ${resolved.at} of ${resolved.caseId} on ${image}`
|
|
9012
|
+
);
|
|
9013
|
+
const verdict = await replayVerify({
|
|
9014
|
+
stepsPath: resolved.stepsPath,
|
|
9015
|
+
image,
|
|
9016
|
+
at: resolved.at,
|
|
9017
|
+
fixCommand: options.fixCommand,
|
|
9018
|
+
cwd: resolved.cwd,
|
|
9019
|
+
out: receiptDir,
|
|
9020
|
+
caseId: resolved.caseId,
|
|
9021
|
+
stepTimeoutMs: options.stepTimeoutMs ?? resolved.recordedStepTimeoutMs ?? void 0,
|
|
9022
|
+
prefixLimit: options.prefixLimit,
|
|
9023
|
+
backend: options.backend,
|
|
9024
|
+
apiKey: options.apiKey,
|
|
9025
|
+
baseUrl,
|
|
9026
|
+
maxLifetimeSeconds: options.maxLifetimeSeconds,
|
|
9027
|
+
onProgress: options.onProgress
|
|
9028
|
+
});
|
|
9029
|
+
executions += 1;
|
|
9030
|
+
const verification = {
|
|
9031
|
+
...identity,
|
|
9032
|
+
step: resolved.at,
|
|
9033
|
+
verified: classifyVerdict(verdict),
|
|
9034
|
+
reason: null,
|
|
9035
|
+
receipt: receiptDir,
|
|
9036
|
+
verdict_path: join22(receiptDir, "replay-verdict.json"),
|
|
9037
|
+
deduplicated_with: null
|
|
9038
|
+
};
|
|
9039
|
+
executedByKey.set(dedupeKey, verification);
|
|
9040
|
+
writeReceipt(receiptDir, finding2, verification, receiptExecution(verdict));
|
|
9041
|
+
verifications.push(verification);
|
|
9042
|
+
options.onProgress?.(`${identity.finding_id ?? `finding ${index + 1}`}: ${verification.verified}`);
|
|
9043
|
+
}
|
|
9044
|
+
const counts = {
|
|
9045
|
+
reproduced: 0,
|
|
9046
|
+
"fix-flipped": 0,
|
|
9047
|
+
divergent: 0,
|
|
9048
|
+
"not-replayable": 0
|
|
9049
|
+
};
|
|
9050
|
+
for (const verification of verifications) counts[verification.verified] += 1;
|
|
9051
|
+
const run = { out: options.out, verifications, counts, executions };
|
|
9052
|
+
writeFileSync3(
|
|
9053
|
+
join22(options.out, "verifications.json"),
|
|
9054
|
+
`${JSON.stringify({ schema_version: "1.0.0", ...run }, null, 2)}
|
|
9055
|
+
`
|
|
9056
|
+
);
|
|
9057
|
+
return run;
|
|
9058
|
+
}
|
|
9059
|
+
function verdictCell(verification) {
|
|
9060
|
+
switch (verification.verified) {
|
|
9061
|
+
case "reproduced":
|
|
9062
|
+
return "**VERIFIED** \u2014 reproduced";
|
|
9063
|
+
case "fix-flipped":
|
|
9064
|
+
return "**VERIFIED** \u2014 fix-flipped";
|
|
9065
|
+
case "divergent":
|
|
9066
|
+
return "DIVERGENT \u2014 recorded failure did not reproduce";
|
|
9067
|
+
case "not-replayable":
|
|
9068
|
+
return `UNVERIFIABLE \u2014 ${verification.reason ?? "no reason recorded"}`;
|
|
9069
|
+
}
|
|
9070
|
+
}
|
|
9071
|
+
function renderVerifiedFindingsSection(run) {
|
|
9072
|
+
const lines = ["## Verified findings (executed replay)", ""];
|
|
9073
|
+
const total = run.verifications.length;
|
|
9074
|
+
lines.push(
|
|
9075
|
+
`${total} finding(s) \u2192 ${run.counts.reproduced} reproduced, ${run.counts["fix-flipped"]} fix-flipped, ${run.counts.divergent} divergent, ${run.counts["not-replayable"]} not replayable (${run.executions} sandbox execution(s); findings accusing the same step share one proof).`
|
|
9076
|
+
);
|
|
9077
|
+
lines.push("");
|
|
9078
|
+
lines.push("| Finding | Subject | Step | Verdict | Receipt |");
|
|
9079
|
+
lines.push("|---|---|---:|---|---|");
|
|
9080
|
+
for (const verification of run.verifications) {
|
|
9081
|
+
const shared = verification.deduplicated_with ? " (shared proof)" : "";
|
|
9082
|
+
lines.push(
|
|
9083
|
+
`| \`${verification.finding_id ?? "\u2014"}\` | \`${verification.subject ?? "\u2014"}\` | ${verification.step ?? "\u2014"} | ${verdictCell(verification)} | \`${verification.receipt}\`${shared} |`
|
|
9084
|
+
);
|
|
9085
|
+
}
|
|
9086
|
+
lines.push("");
|
|
9087
|
+
lines.push(
|
|
9088
|
+
"VERIFIED = the accused step was re-executed in a sandbox after replaying the trajectory prefix, and the recorded failure signature reproduced (fix-flipped: a corrected command additionally made it vanish). Each receipt directory carries receipt.json and, when executed, replay-verdict.json + report.md with real stdout/stderr."
|
|
9089
|
+
);
|
|
9090
|
+
lines.push("");
|
|
9091
|
+
return lines.join("\n");
|
|
9092
|
+
}
|
|
9093
|
+
function readFindingsFile(path) {
|
|
9094
|
+
const parsed = JSON.parse(readFileSync4(path, "utf8"));
|
|
9095
|
+
const array = Array.isArray(parsed) ? parsed : parsed && typeof parsed === "object" && Array.isArray(parsed.findings) ? parsed.findings : null;
|
|
9096
|
+
if (!array) {
|
|
9097
|
+
throw new Error(`${path} is neither a findings array nor an object with a findings array`);
|
|
9098
|
+
}
|
|
9099
|
+
for (const entry of array) {
|
|
9100
|
+
if (!entry || typeof entry !== "object") {
|
|
9101
|
+
throw new Error(`${path}: every finding must be an object, got ${JSON.stringify(entry)}`);
|
|
9102
|
+
}
|
|
9103
|
+
}
|
|
9104
|
+
return array;
|
|
9105
|
+
}
|
|
9106
|
+
function verifyFindingsUsage() {
|
|
9107
|
+
return `traces verify-findings \u2014 execute analyst findings as sandbox replay proofs
|
|
9108
|
+
|
|
9109
|
+
Usage:
|
|
9110
|
+
traces verify-findings --findings FINDINGS.json --out DIR \\
|
|
9111
|
+
( --steps STEPS.json --image IMG --cwd DIR [--case ID]
|
|
9112
|
+
| --corpus name=<labels.json>::<preparedDir> [--corpus ...] ) \\
|
|
9113
|
+
[--fix-command CMD] [--step-timeout MS] [--prefix-limit N] \\
|
|
9114
|
+
[--base-url URL] [--api-key-env VAR] [--max-lifetime SECONDS]
|
|
9115
|
+
|
|
9116
|
+
--findings JSON array of analyst findings (or an object with a findings
|
|
9117
|
+
array) \u2014 the shape agent-eval analysts emit: subject
|
|
9118
|
+
incorrect-step-<n>, metadata.block_first_step, trace:// evidence
|
|
9119
|
+
--out receipt root; one directory per finding + verifications.json
|
|
9120
|
+
--steps/--image/--cwd
|
|
9121
|
+
verify against one trajectory; the image must be replay-ready
|
|
9122
|
+
(uid-1000 derived), exactly like traces replay-verify
|
|
9123
|
+
--case trajectory id of --steps; findings citing other trajectories
|
|
9124
|
+
are honestly marked not-replayable
|
|
9125
|
+
--corpus resolve each finding's trajectory in CodeTraceBench corpora;
|
|
9126
|
+
images are derived via the batch uid-1000 preparer (docker)
|
|
9127
|
+
--fix-command corrected step for arm B; a reproduced finding whose fix
|
|
9128
|
+
flips becomes fix-flipped
|
|
9129
|
+
--base-url sandbox API url (default $SANDBOX_API_URL or ${DEFAULT_SANDBOX_BASE_URL})
|
|
9130
|
+
--api-key-env env var holding the sandbox API key (default SANDBOX_API_KEY)
|
|
9131
|
+
|
|
9132
|
+
Every finding gets a verdict: reproduced | fix-flipped | divergent | not-replayable,
|
|
9133
|
+
with a receipt directory carrying the executed evidence or the precise reason.
|
|
9134
|
+
Requires a running sandbox orchestrator when any finding is replayable \u2014 infra
|
|
9135
|
+
absence is an error, never a silent skip. See docs/replay-verify.md for setup.
|
|
9136
|
+
`;
|
|
9137
|
+
}
|
|
9138
|
+
function parseVerifyFindingsArgs(argv) {
|
|
9139
|
+
const values = /* @__PURE__ */ new Map();
|
|
9140
|
+
const corpora = [];
|
|
9141
|
+
for (let i = 0; i < argv.length; i++) {
|
|
9142
|
+
const arg = argv[i];
|
|
9143
|
+
if (arg === "--help" || arg === "-h") return "help";
|
|
9144
|
+
if (!arg.startsWith("--")) throw new Error(`unexpected positional argument: ${arg}`);
|
|
9145
|
+
const value = argv[++i];
|
|
9146
|
+
if (value === void 0) throw new Error(`missing value for ${arg}`);
|
|
9147
|
+
if (arg === "--corpus") {
|
|
9148
|
+
corpora.push(parseCorpusFlag(value));
|
|
9149
|
+
continue;
|
|
9150
|
+
}
|
|
9151
|
+
values.set(arg, value);
|
|
9152
|
+
}
|
|
9153
|
+
const required = (flag) => {
|
|
9154
|
+
const v = values.get(flag);
|
|
9155
|
+
if (v === void 0) throw new Error(`verify-findings: ${flag} is required`);
|
|
9156
|
+
return v;
|
|
9157
|
+
};
|
|
9158
|
+
const optionalNumber = (flag) => {
|
|
9159
|
+
const v = values.get(flag);
|
|
9160
|
+
if (v === void 0) return void 0;
|
|
9161
|
+
const n = Number(v);
|
|
9162
|
+
if (!Number.isFinite(n)) throw new Error(`verify-findings: ${flag} must be a number, got ${v}`);
|
|
9163
|
+
return n;
|
|
9164
|
+
};
|
|
9165
|
+
const findingsPath = required("--findings");
|
|
9166
|
+
const out = required("--out");
|
|
9167
|
+
const direct = values.has("--steps") || values.has("--image") || values.has("--cwd");
|
|
9168
|
+
if (direct && corpora.length > 0) {
|
|
9169
|
+
throw new Error("verify-findings: pass --steps/--image/--cwd or --corpus, not both");
|
|
9170
|
+
}
|
|
9171
|
+
let source;
|
|
9172
|
+
if (direct) {
|
|
9173
|
+
source = {
|
|
9174
|
+
kind: "direct",
|
|
9175
|
+
stepsPath: required("--steps"),
|
|
9176
|
+
image: required("--image"),
|
|
9177
|
+
cwd: required("--cwd"),
|
|
9178
|
+
...values.has("--case") ? { caseId: values.get("--case") } : {}
|
|
9179
|
+
};
|
|
9180
|
+
} else if (corpora.length > 0) {
|
|
9181
|
+
source = { kind: "corpus", corpora };
|
|
9182
|
+
} else {
|
|
9183
|
+
throw new Error("verify-findings: a replay source is required \u2014 --steps/--image/--cwd or --corpus");
|
|
9184
|
+
}
|
|
9185
|
+
return {
|
|
9186
|
+
findingsPath,
|
|
9187
|
+
source,
|
|
9188
|
+
out,
|
|
9189
|
+
fixCommand: values.get("--fix-command"),
|
|
9190
|
+
stepTimeoutMs: optionalNumber("--step-timeout"),
|
|
9191
|
+
prefixLimit: optionalNumber("--prefix-limit"),
|
|
9192
|
+
baseUrl: values.get("--base-url") ?? process.env.SANDBOX_API_URL ?? DEFAULT_SANDBOX_BASE_URL,
|
|
9193
|
+
apiKeyEnv: values.get("--api-key-env") ?? "SANDBOX_API_KEY",
|
|
9194
|
+
maxLifetimeSeconds: optionalNumber("--max-lifetime")
|
|
9195
|
+
};
|
|
9196
|
+
}
|
|
9197
|
+
async function cmdVerifyFindings(argv) {
|
|
9198
|
+
const parsed = parseVerifyFindingsArgs(argv);
|
|
9199
|
+
if (parsed === "help") {
|
|
9200
|
+
process.stdout.write(verifyFindingsUsage());
|
|
9201
|
+
return;
|
|
9202
|
+
}
|
|
9203
|
+
const findings = readFindingsFile(parsed.findingsPath);
|
|
9204
|
+
const apiKey = process.env[parsed.apiKeyEnv];
|
|
9205
|
+
const run = await verifyFindings(findings, {
|
|
9206
|
+
source: parsed.source,
|
|
9207
|
+
out: parsed.out,
|
|
9208
|
+
fixCommand: parsed.fixCommand,
|
|
9209
|
+
stepTimeoutMs: parsed.stepTimeoutMs,
|
|
9210
|
+
prefixLimit: parsed.prefixLimit,
|
|
9211
|
+
apiKey,
|
|
9212
|
+
baseUrl: parsed.baseUrl,
|
|
9213
|
+
maxLifetimeSeconds: parsed.maxLifetimeSeconds,
|
|
9214
|
+
onProgress: (message) => process.stderr.write(`${message}
|
|
9215
|
+
`)
|
|
9216
|
+
});
|
|
9217
|
+
process.stdout.write(`${JSON.stringify(run, null, 2)}
|
|
9218
|
+
`);
|
|
9219
|
+
process.stderr.write(
|
|
9220
|
+
`verify-findings: ${run.verifications.length} finding(s) \u2192 ${run.counts.reproduced} reproduced, ${run.counts["fix-flipped"]} fix-flipped, ${run.counts.divergent} divergent, ${run.counts["not-replayable"]} not-replayable (${run.executions} execution(s)) \u2192 ${run.out}
|
|
9221
|
+
`
|
|
9222
|
+
);
|
|
9223
|
+
}
|
|
9224
|
+
|
|
8254
9225
|
// src/external.ts
|
|
8255
9226
|
import { spawn } from "child_process";
|
|
8256
9227
|
import { readdir as readdir14, readFile as readFile7 } from "fs/promises";
|
|
@@ -8529,13 +9500,13 @@ function terminateWindowsProcessTree(child) {
|
|
|
8529
9500
|
forceKill(child);
|
|
8530
9501
|
return Promise.resolve();
|
|
8531
9502
|
}
|
|
8532
|
-
return new Promise((
|
|
9503
|
+
return new Promise((resolve10) => {
|
|
8533
9504
|
let finished = false;
|
|
8534
9505
|
const finish = () => {
|
|
8535
9506
|
if (finished) return;
|
|
8536
9507
|
finished = true;
|
|
8537
9508
|
forceKill(child);
|
|
8538
|
-
|
|
9509
|
+
resolve10();
|
|
8539
9510
|
};
|
|
8540
9511
|
let killer;
|
|
8541
9512
|
try {
|
|
@@ -8562,7 +9533,7 @@ function signalProcess(pid, signal) {
|
|
|
8562
9533
|
}
|
|
8563
9534
|
}
|
|
8564
9535
|
function readProcessTableWithPs() {
|
|
8565
|
-
return new Promise((
|
|
9536
|
+
return new Promise((resolve10, reject) => {
|
|
8566
9537
|
let child;
|
|
8567
9538
|
try {
|
|
8568
9539
|
child = spawn("ps", ["-A", "-o", "pid=", "-o", "ppid="], {
|
|
@@ -8616,7 +9587,7 @@ function readProcessTableWithPs() {
|
|
|
8616
9587
|
reject(new Error("ps: returned no process records"));
|
|
8617
9588
|
return;
|
|
8618
9589
|
}
|
|
8619
|
-
|
|
9590
|
+
resolve10(rows);
|
|
8620
9591
|
});
|
|
8621
9592
|
});
|
|
8622
9593
|
});
|
|
@@ -8626,10 +9597,10 @@ async function readLinuxProcProcessTable() {
|
|
|
8626
9597
|
const rows = await Promise.all(
|
|
8627
9598
|
entries.filter((entry) => entry.isDirectory() && /^\d+$/.test(entry.name)).map(async (entry) => {
|
|
8628
9599
|
try {
|
|
8629
|
-
const
|
|
8630
|
-
const commandEnd =
|
|
9600
|
+
const stat18 = await readFile7(`/proc/${entry.name}/stat`, "utf8");
|
|
9601
|
+
const commandEnd = stat18.lastIndexOf(")");
|
|
8631
9602
|
if (commandEnd < 0) return void 0;
|
|
8632
|
-
const fields =
|
|
9603
|
+
const fields = stat18.slice(commandEnd + 1).trim().split(/\s+/);
|
|
8633
9604
|
const pid = Number(entry.name);
|
|
8634
9605
|
const parentPid = Number(fields[1]);
|
|
8635
9606
|
if (!Number.isSafeInteger(pid) || pid < 1 || !Number.isSafeInteger(parentPid) || parentPid < 0) {
|
|
@@ -8691,7 +9662,7 @@ async function terminatePosixProcessTree(child) {
|
|
|
8691
9662
|
}
|
|
8692
9663
|
const rootPid = child.pid;
|
|
8693
9664
|
const descendants = /* @__PURE__ */ new Map();
|
|
8694
|
-
let
|
|
9665
|
+
let failure2;
|
|
8695
9666
|
try {
|
|
8696
9667
|
signalProcess(-rootPid, "SIGSTOP");
|
|
8697
9668
|
signalProcess(rootPid, "SIGSTOP");
|
|
@@ -8708,7 +9679,7 @@ async function terminatePosixProcessTree(child) {
|
|
|
8708
9679
|
previous = signature;
|
|
8709
9680
|
}
|
|
8710
9681
|
} catch (error) {
|
|
8711
|
-
|
|
9682
|
+
failure2 = error;
|
|
8712
9683
|
} finally {
|
|
8713
9684
|
for (const processEntry of [...descendants.values()].sort(
|
|
8714
9685
|
(left, right) => right.depth - left.depth
|
|
@@ -8716,18 +9687,18 @@ async function terminatePosixProcessTree(child) {
|
|
|
8716
9687
|
try {
|
|
8717
9688
|
signalProcess(processEntry.pid, "SIGKILL");
|
|
8718
9689
|
} catch (error) {
|
|
8719
|
-
|
|
9690
|
+
failure2 ??= error;
|
|
8720
9691
|
}
|
|
8721
9692
|
}
|
|
8722
9693
|
try {
|
|
8723
9694
|
signalProcess(-rootPid, "SIGKILL");
|
|
8724
9695
|
} catch (error) {
|
|
8725
|
-
|
|
9696
|
+
failure2 ??= error;
|
|
8726
9697
|
}
|
|
8727
9698
|
forceKill(child);
|
|
8728
9699
|
}
|
|
8729
|
-
if (
|
|
8730
|
-
throw
|
|
9700
|
+
if (failure2) {
|
|
9701
|
+
throw failure2 instanceof Error ? failure2 : new Error(String(failure2));
|
|
8731
9702
|
}
|
|
8732
9703
|
}
|
|
8733
9704
|
function terminateProcessTree(child) {
|
|
@@ -8751,7 +9722,7 @@ function runCommand(command, args = [], opts = {}) {
|
|
|
8751
9722
|
throw new RangeError("maxBuffer must be a positive safe integer");
|
|
8752
9723
|
}
|
|
8753
9724
|
if (opts.signal?.aborted) return Promise.reject(abortError(opts.signal));
|
|
8754
|
-
return new Promise((
|
|
9725
|
+
return new Promise((resolve10, reject) => {
|
|
8755
9726
|
let child;
|
|
8756
9727
|
try {
|
|
8757
9728
|
child = spawn(command, [...args], { detached: process.platform !== "win32" });
|
|
@@ -8783,7 +9754,7 @@ function runCommand(command, args = [], opts = {}) {
|
|
|
8783
9754
|
} else if (spawnError) {
|
|
8784
9755
|
reject(spawnError);
|
|
8785
9756
|
} else {
|
|
8786
|
-
|
|
9757
|
+
resolve10({ stdout: out, stderr: err, code: closeCode });
|
|
8787
9758
|
}
|
|
8788
9759
|
};
|
|
8789
9760
|
function requestTermination(error) {
|
|
@@ -9015,18 +9986,18 @@ function commandRedactor(spec) {
|
|
|
9015
9986
|
import { createHash as createHash9 } from "crypto";
|
|
9016
9987
|
import { mkdir as mkdir3, mkdtemp as mkdtemp3, readFile as readFile8, readdir as readdir15, writeFile as writeFile4 } from "fs/promises";
|
|
9017
9988
|
import { endianness, tmpdir as tmpdir4 } from "os";
|
|
9018
|
-
import { join as
|
|
9989
|
+
import { join as join23, resolve as resolve6 } from "path";
|
|
9019
9990
|
var HODOSCOPE_VERSION = "0.2.4";
|
|
9020
9991
|
async function writeHodoscopeInput(otlpPath, options = {}) {
|
|
9021
9992
|
const explicitDirectory = options.directory ? resolve6(options.directory) : void 0;
|
|
9022
|
-
const directory = explicitDirectory ?? await mkdtemp3(
|
|
9993
|
+
const directory = explicitDirectory ?? await mkdtemp3(join23(tmpdir4(), "traces-hodoscope-input-"));
|
|
9023
9994
|
if (explicitDirectory) {
|
|
9024
9995
|
await mkdir3(directory, { recursive: true });
|
|
9025
9996
|
if ((await readdir15(directory)).length > 0) {
|
|
9026
9997
|
throw new Error(`Hodoscope explicit directory must be empty: ${directory}`);
|
|
9027
9998
|
}
|
|
9028
9999
|
}
|
|
9029
|
-
const samples =
|
|
10000
|
+
const samples = join23(directory, "samples");
|
|
9030
10001
|
await mkdir3(samples);
|
|
9031
10002
|
const rows = [];
|
|
9032
10003
|
for await (const value of readJsonl(otlpPath)) rows.push(parseOpenInferenceRow(value));
|
|
@@ -9052,7 +10023,7 @@ async function writeHodoscopeInput(otlpPath, options = {}) {
|
|
|
9052
10023
|
}
|
|
9053
10024
|
actionCount += actions;
|
|
9054
10025
|
const file = `${createHash9("sha256").update(traceId).digest("hex").slice(0, 24)}.json`;
|
|
9055
|
-
await writeFile4(
|
|
10026
|
+
await writeFile4(join23(samples, file), `${JSON.stringify(trajectory, null, 2)}
|
|
9056
10027
|
`, "utf8");
|
|
9057
10028
|
}
|
|
9058
10029
|
return {
|
|
@@ -9076,10 +10047,10 @@ function hodoscopeAnalyzer(options = {}) {
|
|
|
9076
10047
|
async analyze(otlpPath, runOptions2 = {}) {
|
|
9077
10048
|
const outputRoot = options.outputDir ? resolve6(options.outputDir) : tmpdir4();
|
|
9078
10049
|
await mkdir3(outputRoot, { recursive: true });
|
|
9079
|
-
const directory = await mkdtemp3(
|
|
9080
|
-
const input =
|
|
9081
|
-
const analysisFile =
|
|
9082
|
-
const samplesFile =
|
|
10050
|
+
const directory = await mkdtemp3(join23(outputRoot, "traces-hodoscope-"));
|
|
10051
|
+
const input = join23(directory, "input");
|
|
10052
|
+
const analysisFile = join23(directory, "analysis.hodoscope.json");
|
|
10053
|
+
const samplesFile = join23(directory, "samples.json");
|
|
9083
10054
|
try {
|
|
9084
10055
|
const prepared = await writeHodoscopeInput(otlpPath, {
|
|
9085
10056
|
directory: input,
|
|
@@ -9698,6 +10669,425 @@ function assertPositiveInteger(name, value) {
|
|
|
9698
10669
|
}
|
|
9699
10670
|
}
|
|
9700
10671
|
|
|
10672
|
+
// src/analyst-engine-prime.ts
|
|
10673
|
+
import { createHash as createHash10 } from "crypto";
|
|
10674
|
+
import { request as httpRequest } from "http";
|
|
10675
|
+
import { request as httpsRequest } from "https";
|
|
10676
|
+
var DEFAULT_BRIDGE_URL = "http://localhost:4181";
|
|
10677
|
+
var DEFAULT_MODEL = "prime/zai/glm-5.2";
|
|
10678
|
+
var DEFAULT_TIMEOUT_MS = 12e5;
|
|
10679
|
+
var DEFAULT_MAX_INLINE_CHARS = 36e4;
|
|
10680
|
+
var DEFAULT_PER_ATTRIBUTE_CHAR_CAP = 1200;
|
|
10681
|
+
var DEFAULT_QUESTION = "Diagnose this trajectory: identify concrete agent failures, wasted work, and unsupported claims.";
|
|
10682
|
+
var MAX_FINDINGS = 10;
|
|
10683
|
+
var MAX_SPAN_IDS_PER_FINDING = 8;
|
|
10684
|
+
var SEVERITIES = /* @__PURE__ */ new Set(["critical", "high", "medium", "low", "info"]);
|
|
10685
|
+
var ANALYST_ID = "prime";
|
|
10686
|
+
function positiveIntegerEnv(name) {
|
|
10687
|
+
const raw = process.env[name];
|
|
10688
|
+
if (raw === void 0 || raw === "") return void 0;
|
|
10689
|
+
const value = Number(raw);
|
|
10690
|
+
if (!Number.isSafeInteger(value) || value < 1) {
|
|
10691
|
+
throw new TypeError(`${name} must be a positive integer, got '${raw}'`);
|
|
10692
|
+
}
|
|
10693
|
+
return value;
|
|
10694
|
+
}
|
|
10695
|
+
function finiteOrNull(value) {
|
|
10696
|
+
return typeof value === "number" && Number.isFinite(value) ? value : null;
|
|
10697
|
+
}
|
|
10698
|
+
function normalizeUsage(raw) {
|
|
10699
|
+
if (!raw || typeof raw !== "object") return { calls: null, tokens: null, estimated: false };
|
|
10700
|
+
const usage = raw;
|
|
10701
|
+
const input = finiteOrNull(usage.prompt_tokens);
|
|
10702
|
+
const output = finiteOrNull(usage.completion_tokens);
|
|
10703
|
+
return {
|
|
10704
|
+
calls: finiteOrNull(usage.model_requests),
|
|
10705
|
+
tokens: input === null && output === null ? null : { input, output },
|
|
10706
|
+
estimated: usage.estimated === true
|
|
10707
|
+
};
|
|
10708
|
+
}
|
|
10709
|
+
function mergeUsage(a, b) {
|
|
10710
|
+
const calls = a.calls !== null && b.calls !== null ? a.calls + b.calls : null;
|
|
10711
|
+
const tokens3 = a.tokens !== null && b.tokens !== null && a.tokens.input !== null && b.tokens.input !== null && a.tokens.output !== null && b.tokens.output !== null ? { input: a.tokens.input + b.tokens.input, output: a.tokens.output + b.tokens.output } : null;
|
|
10712
|
+
return { calls, tokens: tokens3, estimated: a.estimated || b.estimated };
|
|
10713
|
+
}
|
|
10714
|
+
function renderUsage(usage) {
|
|
10715
|
+
const calls = usage.calls === null ? "uncaptured" : String(usage.calls);
|
|
10716
|
+
const input = usage.tokens?.input ?? "uncaptured";
|
|
10717
|
+
const output = usage.tokens?.output ?? "uncaptured";
|
|
10718
|
+
const estimated = usage.estimated ? " (bridge-estimated)" : "";
|
|
10719
|
+
return `usage: calls=${calls} input_tokens=${input} output_tokens=${output}${estimated}; cost=uncaptured (no pricing table)`;
|
|
10720
|
+
}
|
|
10721
|
+
function capAttributeValue(value, cap) {
|
|
10722
|
+
const rendered = JSON.stringify(value);
|
|
10723
|
+
if (rendered === void 0 || rendered.length <= cap) return value;
|
|
10724
|
+
if (typeof value === "string") return `${value.slice(0, cap)}\u2026[truncated ${value.length - cap} chars]`;
|
|
10725
|
+
return `[omitted non-string attribute: ${rendered.length} JSON chars]`;
|
|
10726
|
+
}
|
|
10727
|
+
function capRowAttributes(row, cap) {
|
|
10728
|
+
const attributes2 = row.attributes;
|
|
10729
|
+
if (!attributes2 || typeof attributes2 !== "object" || Array.isArray(attributes2)) return row;
|
|
10730
|
+
const capped = {};
|
|
10731
|
+
for (const [key, value] of Object.entries(attributes2)) {
|
|
10732
|
+
capped[key] = capAttributeValue(value, cap);
|
|
10733
|
+
}
|
|
10734
|
+
return { ...row, attributes: capped };
|
|
10735
|
+
}
|
|
10736
|
+
async function projectArtifact(otlpPath, maxInlineChars, perAttributeCharCap, signal) {
|
|
10737
|
+
const rows = [];
|
|
10738
|
+
const spanTraces = /* @__PURE__ */ new Map();
|
|
10739
|
+
let index = 0;
|
|
10740
|
+
for await (const value of readJsonl(otlpPath, { signal })) {
|
|
10741
|
+
const label = `${otlpPath}:${index + 1}`;
|
|
10742
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) {
|
|
10743
|
+
throw new TypeError(`${label} must contain an object`);
|
|
10744
|
+
}
|
|
10745
|
+
const row = value;
|
|
10746
|
+
if (typeof row.trace_id !== "string" || row.trace_id.length === 0) {
|
|
10747
|
+
throw new TypeError(`${label}.trace_id must be a non-empty string`);
|
|
10748
|
+
}
|
|
10749
|
+
if (typeof row.span_id !== "string" || row.span_id.length === 0) {
|
|
10750
|
+
throw new TypeError(`${label}.span_id must be a non-empty string`);
|
|
10751
|
+
}
|
|
10752
|
+
rows.push(row);
|
|
10753
|
+
const traces = spanTraces.get(row.span_id) ?? /* @__PURE__ */ new Set();
|
|
10754
|
+
traces.add(row.trace_id);
|
|
10755
|
+
spanTraces.set(row.span_id, traces);
|
|
10756
|
+
index += 1;
|
|
10757
|
+
}
|
|
10758
|
+
if (rows.length === 0) throw new Error(`${otlpPath} contains no spans`);
|
|
10759
|
+
const traceCount = new Set(rows.map((row) => row.trace_id)).size;
|
|
10760
|
+
let projected = rows;
|
|
10761
|
+
let rendered = JSON.stringify(projected);
|
|
10762
|
+
const delivery = {
|
|
10763
|
+
mode: "inline-json",
|
|
10764
|
+
perAttributeCharCap: null,
|
|
10765
|
+
renderedChars: rendered.length
|
|
10766
|
+
};
|
|
10767
|
+
if (rendered.length > maxInlineChars) {
|
|
10768
|
+
projected = rows.map((row) => capRowAttributes(row, perAttributeCharCap));
|
|
10769
|
+
rendered = JSON.stringify(projected);
|
|
10770
|
+
delivery.perAttributeCharCap = perAttributeCharCap;
|
|
10771
|
+
delivery.renderedChars = rendered.length;
|
|
10772
|
+
}
|
|
10773
|
+
if (rendered.length > maxInlineChars) {
|
|
10774
|
+
throw new Error(
|
|
10775
|
+
`trajectory renders to ${rendered.length} chars even at per-attribute cap ${perAttributeCharCap} (budget ${maxInlineChars}); inline delivery impossible`
|
|
10776
|
+
);
|
|
10777
|
+
}
|
|
10778
|
+
return { rows: projected, rendered, delivery, traceCount, spanTraces };
|
|
10779
|
+
}
|
|
10780
|
+
var OUTPUT_CONTRACT_LINES = [
|
|
10781
|
+
"OUTPUT CONTRACT (you have no trace tools and no REPL):",
|
|
10782
|
+
"You are a one-shot analyst. Every fact you need is in the TRAJECTORY JSON below.",
|
|
10783
|
+
"Do not run shell commands, do not read or write files, do not use any tools.",
|
|
10784
|
+
"Reply with EXACTLY one fenced ```json code block and no other fenced block. The JSON object has exactly two fields:",
|
|
10785
|
+
' "answer": string \u2014 ONE short sentence (max 300 chars) summarizing your verdict.',
|
|
10786
|
+
' "findings": array (possibly empty) of finding rows, each exactly:',
|
|
10787
|
+
' {"span_ids": [string, ...],',
|
|
10788
|
+
' "severity": "critical"|"high"|"medium"|"low"|"info",',
|
|
10789
|
+
' "area": string (short kebab-case topic, max 40 chars),',
|
|
10790
|
+
' "claim": string (ONE short sentence, max 200 chars),',
|
|
10791
|
+
' "action": string (optional, ONE short imperative sentence, max 200 chars),',
|
|
10792
|
+
' "confidence": number 0..1}',
|
|
10793
|
+
"Do NOT include a rationale field. Keep every string SHORT \u2014 long strings get corrupted in transport and void your work.",
|
|
10794
|
+
`Report at most ${MAX_FINDINGS} findings; a finding cites 1..${MAX_SPAN_IDS_PER_FINDING} span_ids, each copied VERBATIM from a span in the trajectory below.`,
|
|
10795
|
+
'"findings" is [] only for a clean trajectory.'
|
|
10796
|
+
];
|
|
10797
|
+
function buildPrompt(question, projection) {
|
|
10798
|
+
return [
|
|
10799
|
+
`QUESTION: ${question}`,
|
|
10800
|
+
"",
|
|
10801
|
+
...OUTPUT_CONTRACT_LINES,
|
|
10802
|
+
"",
|
|
10803
|
+
`TRAJECTORY (${projection.traceCount} trace(s); ${projection.rows.length} spans; full OpenInference span projection as JSON):`,
|
|
10804
|
+
projection.rendered
|
|
10805
|
+
].join("\n");
|
|
10806
|
+
}
|
|
10807
|
+
function buildRepairPrompt(parseDefect, previousReply) {
|
|
10808
|
+
return [
|
|
10809
|
+
"Your previous reply to a trace-analysis task was structurally malformed and could not be parsed",
|
|
10810
|
+
`(${parseDefect}). Below is your previous reply verbatim. Re-emit ONLY the corrected JSON \u2014 one`,
|
|
10811
|
+
"fenced ```json block, no other text, no tools. The JSON object has exactly two fields:",
|
|
10812
|
+
' "answer": string (ONE short sentence, max 300 chars)',
|
|
10813
|
+
' "findings": array (possibly empty) of {"span_ids": [string, ...],',
|
|
10814
|
+
' "severity": "critical"|"high"|"medium"|"low"|"info", "area": string (max 40 chars),',
|
|
10815
|
+
' "claim": string (max 200 chars), "action": string (optional, max 200 chars), "confidence": number 0..1}',
|
|
10816
|
+
"No rationale field. Keep every string SHORT. Preserve the span ids and verdicts of your previous reply exactly; shorten prose freely.",
|
|
10817
|
+
"",
|
|
10818
|
+
"PREVIOUS REPLY:",
|
|
10819
|
+
previousReply
|
|
10820
|
+
].join("\n");
|
|
10821
|
+
}
|
|
10822
|
+
function tryParseObject(text2) {
|
|
10823
|
+
try {
|
|
10824
|
+
const value = JSON.parse(text2.trim());
|
|
10825
|
+
return value && typeof value === "object" && !Array.isArray(value) ? value : null;
|
|
10826
|
+
} catch {
|
|
10827
|
+
return null;
|
|
10828
|
+
}
|
|
10829
|
+
}
|
|
10830
|
+
function extractJsonObject(text2) {
|
|
10831
|
+
const direct = tryParseObject(text2);
|
|
10832
|
+
if (direct) return direct;
|
|
10833
|
+
const fenced = [...text2.matchAll(/```(?:json)?\s*\n?([\s\S]*?)```/g)];
|
|
10834
|
+
for (let i = fenced.length - 1; i >= 0; i -= 1) {
|
|
10835
|
+
const candidate = tryParseObject(fenced[i][1]);
|
|
10836
|
+
if (candidate) return candidate;
|
|
10837
|
+
}
|
|
10838
|
+
const start = text2.indexOf("{");
|
|
10839
|
+
const end = text2.lastIndexOf("}");
|
|
10840
|
+
if (start >= 0 && end > start) {
|
|
10841
|
+
const candidate = tryParseObject(text2.slice(start, end + 1));
|
|
10842
|
+
if (candidate) return candidate;
|
|
10843
|
+
}
|
|
10844
|
+
return null;
|
|
10845
|
+
}
|
|
10846
|
+
function parseDefectOf(parsed) {
|
|
10847
|
+
if (parsed === null) return "no parseable JSON object";
|
|
10848
|
+
if (!Array.isArray(parsed.findings)) return 'JSON has no "findings" array';
|
|
10849
|
+
return null;
|
|
10850
|
+
}
|
|
10851
|
+
function findingRowDefect(row, spanTraces) {
|
|
10852
|
+
if (!row || typeof row !== "object" || Array.isArray(row)) return "row is not an object";
|
|
10853
|
+
const record2 = row;
|
|
10854
|
+
if (!Array.isArray(record2.span_ids) || record2.span_ids.length === 0) {
|
|
10855
|
+
return "span_ids must be a non-empty array";
|
|
10856
|
+
}
|
|
10857
|
+
const spanIds = [];
|
|
10858
|
+
for (const id of record2.span_ids) {
|
|
10859
|
+
if (typeof id !== "string" || id.length === 0) return "span_ids must contain non-empty strings";
|
|
10860
|
+
const traces = spanTraces.get(id);
|
|
10861
|
+
if (!traces) return `span_id '${id}' is not in the trajectory`;
|
|
10862
|
+
if (traces.size > 1) return `span_id '${id}' is ambiguous across ${traces.size} traces`;
|
|
10863
|
+
if (!spanIds.includes(id)) spanIds.push(id);
|
|
10864
|
+
}
|
|
10865
|
+
if (spanIds.length > MAX_SPAN_IDS_PER_FINDING) {
|
|
10866
|
+
return `span_ids cites ${spanIds.length} distinct spans (cap ${MAX_SPAN_IDS_PER_FINDING})`;
|
|
10867
|
+
}
|
|
10868
|
+
if (typeof record2.severity !== "string" || !SEVERITIES.has(record2.severity)) {
|
|
10869
|
+
return "severity outside the analyst severity enum";
|
|
10870
|
+
}
|
|
10871
|
+
if (typeof record2.area !== "string" || record2.area.trim().length === 0 || record2.area.length > 200) {
|
|
10872
|
+
return "area must be a 1-200 char string";
|
|
10873
|
+
}
|
|
10874
|
+
if (typeof record2.claim !== "string" || record2.claim.trim().length === 0 || record2.claim.length > 2e3) {
|
|
10875
|
+
return "claim must be a 1-2000 char string";
|
|
10876
|
+
}
|
|
10877
|
+
if (record2.action !== void 0 && (typeof record2.action !== "string" || record2.action.trim().length === 0 || record2.action.length > 2e3)) {
|
|
10878
|
+
return "action must be a 1-2000 char string when present";
|
|
10879
|
+
}
|
|
10880
|
+
if (typeof record2.confidence !== "number" || !Number.isFinite(record2.confidence) || record2.confidence < 0 || record2.confidence > 1) {
|
|
10881
|
+
return "confidence must be 0..1";
|
|
10882
|
+
}
|
|
10883
|
+
return {
|
|
10884
|
+
spanIds,
|
|
10885
|
+
severity: record2.severity,
|
|
10886
|
+
area: record2.area.trim(),
|
|
10887
|
+
claim: record2.claim.trim(),
|
|
10888
|
+
...typeof record2.action === "string" ? { action: record2.action.trim() } : {},
|
|
10889
|
+
confidence: record2.confidence
|
|
10890
|
+
};
|
|
10891
|
+
}
|
|
10892
|
+
function toAnalystFinding(row, index, spanTraces, model, producedAt) {
|
|
10893
|
+
const uris = row.spanIds.map((spanId) => {
|
|
10894
|
+
const traces = spanTraces.get(spanId);
|
|
10895
|
+
return spanEvidenceUri([...traces][0], spanId);
|
|
10896
|
+
});
|
|
10897
|
+
const findingId = `prime-${createHash10("sha256").update(`${index}
|
|
10898
|
+
${row.claim}
|
|
10899
|
+
${uris.join("\n")}`).digest("hex").slice(0, 12)}`;
|
|
10900
|
+
return {
|
|
10901
|
+
schema_version: "1.0.0",
|
|
10902
|
+
finding_id: findingId,
|
|
10903
|
+
analyst_id: ANALYST_ID,
|
|
10904
|
+
produced_at: producedAt,
|
|
10905
|
+
severity: row.severity,
|
|
10906
|
+
area: row.area,
|
|
10907
|
+
claim: row.claim,
|
|
10908
|
+
evidence_refs: uris.map((uri) => ({ kind: "span", uri })),
|
|
10909
|
+
confidence: row.confidence,
|
|
10910
|
+
...row.action ? { recommended_action: row.action } : {},
|
|
10911
|
+
metadata: { engine: ANALYST_ID, model }
|
|
10912
|
+
};
|
|
10913
|
+
}
|
|
10914
|
+
var httpJsonTransport = ({ url, body, signal }) => {
|
|
10915
|
+
const target = new URL(url);
|
|
10916
|
+
if (target.protocol !== "http:" && target.protocol !== "https:") {
|
|
10917
|
+
throw new TypeError(`bridge URL must be http: or https:, got ${target.protocol}`);
|
|
10918
|
+
}
|
|
10919
|
+
const requestFn = target.protocol === "https:" ? httpsRequest : httpRequest;
|
|
10920
|
+
const payload = JSON.stringify(body);
|
|
10921
|
+
return new Promise((resolve10, reject) => {
|
|
10922
|
+
const req = requestFn(
|
|
10923
|
+
{
|
|
10924
|
+
hostname: target.hostname,
|
|
10925
|
+
port: target.port,
|
|
10926
|
+
path: `${target.pathname}${target.search}`,
|
|
10927
|
+
method: "POST",
|
|
10928
|
+
headers: { "content-type": "application/json", "content-length": Buffer.byteLength(payload) },
|
|
10929
|
+
signal
|
|
10930
|
+
},
|
|
10931
|
+
(res) => {
|
|
10932
|
+
const chunks = [];
|
|
10933
|
+
res.on("data", (chunk) => chunks.push(chunk));
|
|
10934
|
+
res.on("end", () => resolve10({ status: res.statusCode ?? 0, text: Buffer.concat(chunks).toString("utf8") }));
|
|
10935
|
+
res.on("error", reject);
|
|
10936
|
+
}
|
|
10937
|
+
);
|
|
10938
|
+
req.on("error", reject);
|
|
10939
|
+
req.end(payload);
|
|
10940
|
+
});
|
|
10941
|
+
};
|
|
10942
|
+
var PrimeBridgeError = class extends Error {
|
|
10943
|
+
};
|
|
10944
|
+
async function callBridge(transport, url, model, content, timeoutMs, callerSignal, turn) {
|
|
10945
|
+
const controller = new AbortController();
|
|
10946
|
+
const timer = setTimeout(
|
|
10947
|
+
() => controller.abort(new Error(`${turn}bridge call exceeded ${timeoutMs}ms`)),
|
|
10948
|
+
timeoutMs
|
|
10949
|
+
);
|
|
10950
|
+
const onCallerAbort = () => controller.abort(callerSignal.reason);
|
|
10951
|
+
callerSignal?.addEventListener("abort", onCallerAbort, { once: true });
|
|
10952
|
+
if (callerSignal?.aborted) onCallerAbort();
|
|
10953
|
+
let response;
|
|
10954
|
+
try {
|
|
10955
|
+
response = await transport({
|
|
10956
|
+
url,
|
|
10957
|
+
body: { model, messages: [{ role: "user", content }] },
|
|
10958
|
+
signal: controller.signal
|
|
10959
|
+
});
|
|
10960
|
+
} catch (error) {
|
|
10961
|
+
const reason = controller.signal.aborted ? controller.signal.reason : error;
|
|
10962
|
+
throw new PrimeBridgeError(
|
|
10963
|
+
`${turn}bridge transport failure: ${reason instanceof Error ? reason.message : String(reason)}`
|
|
10964
|
+
);
|
|
10965
|
+
} finally {
|
|
10966
|
+
clearTimeout(timer);
|
|
10967
|
+
callerSignal?.removeEventListener("abort", onCallerAbort);
|
|
10968
|
+
}
|
|
10969
|
+
if (response.status !== 200) {
|
|
10970
|
+
throw new PrimeBridgeError(`${turn}bridge HTTP ${response.status}: ${response.text.slice(0, 500)}`);
|
|
10971
|
+
}
|
|
10972
|
+
let parsed;
|
|
10973
|
+
try {
|
|
10974
|
+
parsed = JSON.parse(response.text);
|
|
10975
|
+
} catch {
|
|
10976
|
+
throw new PrimeBridgeError(`${turn}bridge returned unparseable JSON (${response.text.length} bytes)`);
|
|
10977
|
+
}
|
|
10978
|
+
const record2 = parsed;
|
|
10979
|
+
const replyContent = record2.choices?.[0]?.message?.content;
|
|
10980
|
+
if (typeof replyContent !== "string" || replyContent.length === 0) {
|
|
10981
|
+
throw new PrimeBridgeError(`${turn}bridge reply carries no message content`);
|
|
10982
|
+
}
|
|
10983
|
+
return { content: replyContent, usage: normalizeUsage(record2.usage) };
|
|
10984
|
+
}
|
|
10985
|
+
function failure(output, error) {
|
|
10986
|
+
return { analyzer: ANALYST_ID, kind: "report", ok: false, output, error };
|
|
10987
|
+
}
|
|
10988
|
+
function primeAnalyzer(opts = {}) {
|
|
10989
|
+
const bridgeUrl = (opts.bridgeUrl ?? process.env.TRACES_PRIME_BRIDGE_URL ?? DEFAULT_BRIDGE_URL).replace(/\/+$/, "");
|
|
10990
|
+
const model = opts.model ?? process.env.TRACES_PRIME_MODEL ?? DEFAULT_MODEL;
|
|
10991
|
+
const timeoutMs = opts.timeoutMs ?? positiveIntegerEnv("TRACES_PRIME_TIMEOUT_MS") ?? DEFAULT_TIMEOUT_MS;
|
|
10992
|
+
if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1) {
|
|
10993
|
+
throw new RangeError("timeoutMs must be a positive safe integer");
|
|
10994
|
+
}
|
|
10995
|
+
const maxInlineChars = opts.maxInlineChars ?? DEFAULT_MAX_INLINE_CHARS;
|
|
10996
|
+
const perAttributeCharCap = opts.perAttributeCharCap ?? DEFAULT_PER_ATTRIBUTE_CHAR_CAP;
|
|
10997
|
+
if (!Number.isSafeInteger(maxInlineChars) || maxInlineChars < 1) {
|
|
10998
|
+
throw new RangeError("maxInlineChars must be a positive safe integer");
|
|
10999
|
+
}
|
|
11000
|
+
if (!Number.isSafeInteger(perAttributeCharCap) || perAttributeCharCap < 1) {
|
|
11001
|
+
throw new RangeError("perAttributeCharCap must be a positive safe integer");
|
|
11002
|
+
}
|
|
11003
|
+
const repairEnabled = opts.repair ?? true;
|
|
11004
|
+
const transport = opts.transport ?? httpJsonTransport;
|
|
11005
|
+
const url = `${bridgeUrl}/v1/chat/completions`;
|
|
11006
|
+
return {
|
|
11007
|
+
name: ANALYST_ID,
|
|
11008
|
+
async analyze(otlpPath, analyzeOpts = {}) {
|
|
11009
|
+
let projection;
|
|
11010
|
+
try {
|
|
11011
|
+
projection = await projectArtifact(otlpPath, maxInlineChars, perAttributeCharCap, analyzeOpts.signal);
|
|
11012
|
+
} catch (error) {
|
|
11013
|
+
return failure("", error instanceof Error ? error.message : String(error));
|
|
11014
|
+
}
|
|
11015
|
+
const question = analyzeOpts.prompt ?? opts.defaultPrompt ?? DEFAULT_QUESTION;
|
|
11016
|
+
const deliveryLine = `delivery: ${projection.delivery.mode} (${projection.delivery.renderedChars} chars, per-attribute cap ${projection.delivery.perAttributeCharCap ?? "none"})`;
|
|
11017
|
+
let reply;
|
|
11018
|
+
try {
|
|
11019
|
+
reply = await callBridge(
|
|
11020
|
+
transport,
|
|
11021
|
+
url,
|
|
11022
|
+
model,
|
|
11023
|
+
buildPrompt(question, projection),
|
|
11024
|
+
timeoutMs,
|
|
11025
|
+
analyzeOpts.signal,
|
|
11026
|
+
""
|
|
11027
|
+
);
|
|
11028
|
+
} catch (error) {
|
|
11029
|
+
if (!(error instanceof PrimeBridgeError)) throw error;
|
|
11030
|
+
return failure("", `${error.message}; ${deliveryLine}`);
|
|
11031
|
+
}
|
|
11032
|
+
let usage = reply.usage;
|
|
11033
|
+
let parsed = extractJsonObject(reply.content);
|
|
11034
|
+
let parseDefect = parseDefectOf(parsed);
|
|
11035
|
+
let repairAttempted = false;
|
|
11036
|
+
if (parseDefect !== null && repairEnabled) {
|
|
11037
|
+
repairAttempted = true;
|
|
11038
|
+
let repairReply;
|
|
11039
|
+
try {
|
|
11040
|
+
repairReply = await callBridge(
|
|
11041
|
+
transport,
|
|
11042
|
+
url,
|
|
11043
|
+
model,
|
|
11044
|
+
buildRepairPrompt(parseDefect, reply.content),
|
|
11045
|
+
timeoutMs,
|
|
11046
|
+
analyzeOpts.signal,
|
|
11047
|
+
"repair-turn "
|
|
11048
|
+
);
|
|
11049
|
+
} catch (error) {
|
|
11050
|
+
if (!(error instanceof PrimeBridgeError)) throw error;
|
|
11051
|
+
return failure(reply.content, `${error.message}; ${renderUsage(usage)}`);
|
|
11052
|
+
}
|
|
11053
|
+
usage = mergeUsage(usage, repairReply.usage);
|
|
11054
|
+
parsed = extractJsonObject(repairReply.content);
|
|
11055
|
+
parseDefect = parseDefectOf(parsed);
|
|
11056
|
+
}
|
|
11057
|
+
if (parseDefect !== null) {
|
|
11058
|
+
return failure(
|
|
11059
|
+
reply.content,
|
|
11060
|
+
`${parseDefect} in prime reply${repairAttempted ? " even after the bounded repair turn" : ""}; ` + renderUsage(usage)
|
|
11061
|
+
);
|
|
11062
|
+
}
|
|
11063
|
+
const rows = parsed.findings.slice(0, MAX_FINDINGS);
|
|
11064
|
+
const overflow = parsed.findings.length - rows.length;
|
|
11065
|
+
const producedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
11066
|
+
const rejected = [];
|
|
11067
|
+
const findings = [];
|
|
11068
|
+
rows.forEach((row, index) => {
|
|
11069
|
+
const decoded = findingRowDefect(row, projection.spanTraces);
|
|
11070
|
+
if (typeof decoded === "string") {
|
|
11071
|
+
rejected.push(`rejected[${index}]: ${decoded}`);
|
|
11072
|
+
return;
|
|
11073
|
+
}
|
|
11074
|
+
findings.push(toAnalystFinding(decoded, index, projection.spanTraces, model, producedAt));
|
|
11075
|
+
});
|
|
11076
|
+
const answer = typeof parsed.answer === "string" ? parsed.answer : null;
|
|
11077
|
+
const output = [
|
|
11078
|
+
...answer ? [`answer: ${answer}`] : [],
|
|
11079
|
+
`findings: ${findings.length} mapped, ${rejected.length} rejected` + (overflow > 0 ? `, ${overflow} over the ${MAX_FINDINGS}-finding cap dropped` : ""),
|
|
11080
|
+
...rejected,
|
|
11081
|
+
deliveryLine,
|
|
11082
|
+
`repair: ${repairAttempted ? "attempted (succeeded)" : "not needed"}`,
|
|
11083
|
+
renderUsage(usage),
|
|
11084
|
+
...findings.length === 0 ? ["zero findings \u2014 an honest null, not a failure"] : []
|
|
11085
|
+
].join("\n");
|
|
11086
|
+
return { analyzer: ANALYST_ID, kind: "findings", ok: true, output, findings };
|
|
11087
|
+
}
|
|
11088
|
+
};
|
|
11089
|
+
}
|
|
11090
|
+
|
|
9701
11091
|
// src/redact.ts
|
|
9702
11092
|
import { DEFAULT_REDACTION_RULES, redactValue } from "@tangle-network/agent-eval/traces";
|
|
9703
11093
|
var CODING_REDACTION_RULES = [
|
|
@@ -9788,10 +11178,10 @@ async function applyRedactor(spans, redactor) {
|
|
|
9788
11178
|
}
|
|
9789
11179
|
|
|
9790
11180
|
// src/file-export.ts
|
|
9791
|
-
import { createHash as
|
|
11181
|
+
import { createHash as createHash11 } from "crypto";
|
|
9792
11182
|
import { mkdtemp as mkdtemp4, readFile as readFile9, writeFile as writeFile5 } from "fs/promises";
|
|
9793
11183
|
import { tmpdir as tmpdir5 } from "os";
|
|
9794
|
-
import { join as
|
|
11184
|
+
import { join as join24 } from "path";
|
|
9795
11185
|
import {
|
|
9796
11186
|
firstNumberAttr,
|
|
9797
11187
|
LLM_CACHED_TOKEN_ATTR_KEYS,
|
|
@@ -9826,7 +11216,7 @@ function stableJson(value) {
|
|
|
9826
11216
|
});
|
|
9827
11217
|
}
|
|
9828
11218
|
function hashId(value, chars = 16) {
|
|
9829
|
-
return
|
|
11219
|
+
return createHash11("sha256").update(stableJson(value)).digest("hex").slice(0, chars);
|
|
9830
11220
|
}
|
|
9831
11221
|
function isoTime2(value) {
|
|
9832
11222
|
if (typeof value === "string") {
|
|
@@ -9987,12 +11377,12 @@ function policyEvidenceToSpans(records) {
|
|
|
9987
11377
|
"traces.provenance.evidence_kind": record2.provenance.evidenceKind,
|
|
9988
11378
|
"traces.provenance.otlp_path": record2.provenance.otlpPath,
|
|
9989
11379
|
"traces.provenance.source_sha256": record2.provenance.sourceSha256,
|
|
9990
|
-
[
|
|
9991
|
-
[
|
|
9992
|
-
[
|
|
9993
|
-
[
|
|
9994
|
-
[
|
|
9995
|
-
[
|
|
11380
|
+
[ATTR.SUBJECT_KEY]: record2.repo.subjectKey,
|
|
11381
|
+
[ATTR.GIT_REPOSITORY]: record2.repo.repository,
|
|
11382
|
+
[ATTR.GIT_BRANCH_NAME]: record2.repo.branch,
|
|
11383
|
+
[ATTR.GIT_COMMIT]: record2.repo.commit,
|
|
11384
|
+
[ATTR.CWD]: record2.repo.cwd ?? record2.session.cwd,
|
|
11385
|
+
[ATTR.REPO_RESOLUTION_SOURCE]: record2.repo.resolutionSource
|
|
9996
11386
|
};
|
|
9997
11387
|
return span({
|
|
9998
11388
|
traceId,
|
|
@@ -10089,6 +11479,54 @@ function eventKind(row, type) {
|
|
|
10089
11479
|
if (lowered.includes("llm") || lowered === "token") return { kind: "LLM" };
|
|
10090
11480
|
return { kind: "CHAIN" };
|
|
10091
11481
|
}
|
|
11482
|
+
function sandboxEventsEnvironment(context, rows) {
|
|
11483
|
+
const ready = rows.find((row) => eventType(row) === "runtime.ready");
|
|
11484
|
+
const readyData = ready && isObject5(ready.data) ? ready.data : void 0;
|
|
11485
|
+
const pick = (keys) => findStringKey(context, keys, 2) ?? (readyData ? findStringKey(readyData, keys, 2) : void 0);
|
|
11486
|
+
const image = pick(["image"]);
|
|
11487
|
+
const imageDigest = pick(["imageDigest", "image_digest"]);
|
|
11488
|
+
const sandboxId = pick(["sandboxId", "sandbox_id"]);
|
|
11489
|
+
const cwd = pick(["cwd", "workspaceRoot", "workspace_root"]);
|
|
11490
|
+
if (!image && !imageDigest && !sandboxId && !cwd) return null;
|
|
11491
|
+
return {
|
|
11492
|
+
...image ? { image } : {},
|
|
11493
|
+
...imageDigest ? { imageDigest } : {},
|
|
11494
|
+
...sandboxId ? { sandboxId } : {},
|
|
11495
|
+
cwd: cwd ?? null
|
|
11496
|
+
};
|
|
11497
|
+
}
|
|
11498
|
+
function llmEventUsage(row) {
|
|
11499
|
+
const num = (keys) => {
|
|
11500
|
+
for (const key of keys) {
|
|
11501
|
+
const direct = numberValue(row[key]);
|
|
11502
|
+
if (direct !== void 0) return direct;
|
|
11503
|
+
for (const nest of ["data", "usage", "tokenUsage"]) {
|
|
11504
|
+
const container = row[nest];
|
|
11505
|
+
if (!isObject5(container)) continue;
|
|
11506
|
+
const value = numberValue(container[key]);
|
|
11507
|
+
if (value !== void 0) return value;
|
|
11508
|
+
for (const inner of ["usage", "tokenUsage"]) {
|
|
11509
|
+
const deep = container[inner];
|
|
11510
|
+
if (isObject5(deep)) {
|
|
11511
|
+
const deepValue = numberValue(deep[key]);
|
|
11512
|
+
if (deepValue !== void 0) return deepValue;
|
|
11513
|
+
}
|
|
11514
|
+
}
|
|
11515
|
+
}
|
|
11516
|
+
}
|
|
11517
|
+
return void 0;
|
|
11518
|
+
};
|
|
11519
|
+
const model = findStringKey(row, ["model"], 3);
|
|
11520
|
+
const inputTokens = num(["prompt_tokens", "input_tokens", "inputTokens"]);
|
|
11521
|
+
const outputTokens = num(["completion_tokens", "output_tokens", "outputTokens"]);
|
|
11522
|
+
const costUsd = num(["totalCostUsd", "total_cost_usd", "costUsd", "cost_usd"]);
|
|
11523
|
+
return {
|
|
11524
|
+
...model ? { model } : {},
|
|
11525
|
+
...inputTokens !== void 0 ? { inputTokens } : {},
|
|
11526
|
+
...outputTokens !== void 0 ? { outputTokens } : {},
|
|
11527
|
+
...costUsd !== void 0 ? { costUsd } : {}
|
|
11528
|
+
};
|
|
11529
|
+
}
|
|
10092
11530
|
function sandboxEventsToSpans(rows, wrapper) {
|
|
10093
11531
|
const context = wrapper ?? {};
|
|
10094
11532
|
const sessionId = findStringKey(context, ["session_id", "sessionId"], 3) ?? findStringKey(rows, ["session_id", "sessionId"], 4);
|
|
@@ -10159,13 +11597,15 @@ function sandboxEventsToSpans(rows, wrapper) {
|
|
|
10159
11597
|
tool: kind === "TOOL" ? tool ?? type : null,
|
|
10160
11598
|
step: index,
|
|
10161
11599
|
content: capText(stableJson(row)),
|
|
10162
|
-
extra
|
|
11600
|
+
extra,
|
|
11601
|
+
...kind === "LLM" ? llmEventUsage(row) : {}
|
|
10163
11602
|
}));
|
|
10164
11603
|
});
|
|
11604
|
+
stampEnvironmentAttrs(spans, sandboxEventsEnvironment(context, rows) ?? void 0);
|
|
10165
11605
|
if (!sessionId) return spans;
|
|
10166
11606
|
return spans.map((item) => ({
|
|
10167
11607
|
...item,
|
|
10168
|
-
attributes: { ...item.attributes, [
|
|
11608
|
+
attributes: { ...item.attributes, [ATTR.SESSION_ID]: sessionId }
|
|
10169
11609
|
}));
|
|
10170
11610
|
}
|
|
10171
11611
|
function statusCode2(value) {
|
|
@@ -10244,7 +11684,7 @@ function intelligenceSpansToSpans(rows) {
|
|
|
10244
11684
|
};
|
|
10245
11685
|
copyDefined(extra, {
|
|
10246
11686
|
"traces.row.id": stringValue3(row.id),
|
|
10247
|
-
[
|
|
11687
|
+
[ATTR.SESSION_ID]: sessionId,
|
|
10248
11688
|
"session.id": sessionId,
|
|
10249
11689
|
"run.id": stringValue3(row.run_id),
|
|
10250
11690
|
"thread.id": stringValue3(row.thread_id),
|
|
@@ -10387,7 +11827,7 @@ async function exportTraceEvidenceFile(inputPath, opts = {}) {
|
|
|
10387
11827
|
}
|
|
10388
11828
|
async function writeTraceEvidenceExportFile(inputPath, outPath, opts = {}) {
|
|
10389
11829
|
const result = await exportTraceEvidenceFile(inputPath, opts);
|
|
10390
|
-
const path = outPath ??
|
|
11830
|
+
const path = outPath ?? join24(await mkdtemp4(join24(tmpdir5(), "traces-export-")), "spans.openinference.jsonl");
|
|
10391
11831
|
await writeFile5(path, serializeSpans(result.spans), "utf8");
|
|
10392
11832
|
return { ...result, path };
|
|
10393
11833
|
}
|
|
@@ -10395,7 +11835,7 @@ async function writeTraceEvidenceExportFile(inputPath, outPath, opts = {}) {
|
|
|
10395
11835
|
// src/inspect.ts
|
|
10396
11836
|
import { mkdtemp as mkdtemp5, readFile as readFile10, writeFile as writeFile6 } from "fs/promises";
|
|
10397
11837
|
import { tmpdir as tmpdir6 } from "os";
|
|
10398
|
-
import { basename as basename13, join as
|
|
11838
|
+
import { basename as basename13, join as join25 } from "path";
|
|
10399
11839
|
var severityRank = {
|
|
10400
11840
|
high: 0,
|
|
10401
11841
|
medium: 1,
|
|
@@ -10684,14 +12124,14 @@ async function readSessionIndexFile(path) {
|
|
|
10684
12124
|
return parsed;
|
|
10685
12125
|
}
|
|
10686
12126
|
async function writeInspectionReportFile(report, outPath) {
|
|
10687
|
-
const path = outPath ??
|
|
12127
|
+
const path = outPath ?? join25(await mkdtemp5(join25(tmpdir6(), "traces-inspect-")), "inspection-report.md");
|
|
10688
12128
|
await writeFile6(path, renderInspectionReport(report), "utf8");
|
|
10689
12129
|
return path;
|
|
10690
12130
|
}
|
|
10691
12131
|
|
|
10692
12132
|
// src/adoption.ts
|
|
10693
12133
|
import { readFile as readFile11 } from "fs/promises";
|
|
10694
|
-
import { join as
|
|
12134
|
+
import { join as join26 } from "path";
|
|
10695
12135
|
function toolName2(s) {
|
|
10696
12136
|
const n = s.attributes["tool.name"];
|
|
10697
12137
|
if (typeof n === "string") return n;
|
|
@@ -10811,7 +12251,7 @@ async function readLoopRuns(cwds, sessionIds) {
|
|
|
10811
12251
|
seenCwds.add(cwd);
|
|
10812
12252
|
const firstSourceByRun = /* @__PURE__ */ new Map();
|
|
10813
12253
|
for (const artifactDir of [".agent", ".evolve"]) {
|
|
10814
|
-
const path =
|
|
12254
|
+
const path = join26(cwd, artifactDir, "skill-runs.jsonl");
|
|
10815
12255
|
let raw;
|
|
10816
12256
|
try {
|
|
10817
12257
|
raw = await readFile11(path, "utf8");
|
|
@@ -11619,6 +13059,47 @@ var SEVERITY_BADGE2 = {
|
|
|
11619
13059
|
low: "\u{1F535} LOW",
|
|
11620
13060
|
info: "\u2139\uFE0F INFO"
|
|
11621
13061
|
};
|
|
13062
|
+
function sessionReportSource(ref, spans, sessionIdOverride) {
|
|
13063
|
+
const root = spans.find((item) => item.parent_span_id === null) ?? spans[0];
|
|
13064
|
+
const prompt = spans.find(
|
|
13065
|
+
(item) => item.name === "user.prompt" && item.attributes[ACTOR_ATTR] === "human"
|
|
13066
|
+
) ?? spans.find((item) => item.name === "user.prompt") ?? spans.find(
|
|
13067
|
+
(item) => item.attributes["span.type"] === "interaction" && typeof item.attributes.content === "string"
|
|
13068
|
+
);
|
|
13069
|
+
const content = typeof prompt?.attributes.content === "string" ? prompt.attributes.content : "";
|
|
13070
|
+
const firstLine = content.split(/\r?\n/, 1)[0].trim();
|
|
13071
|
+
const subject = firstLine.length > 240 ? `${firstLine.slice(0, 240)}\u2026 [+${firstLine.length - 240} chars]` : firstLine;
|
|
13072
|
+
const role = root?.attributes["traces.session.role"];
|
|
13073
|
+
const parentSessionId = root?.attributes["traces.parent_session_id"];
|
|
13074
|
+
const relationship = describeSessionRelationship(ref, spans);
|
|
13075
|
+
const corruptionDigest2 = root?.attributes[ATTR.CORRUPTION_DIGEST];
|
|
13076
|
+
const sessionId = [
|
|
13077
|
+
sessionIdOverride,
|
|
13078
|
+
root ? sessionIdFromAttributes(root.attributes) : void 0,
|
|
13079
|
+
root?.trace_id,
|
|
13080
|
+
ref.sessionId
|
|
13081
|
+
].find((value) => typeof value === "string" && value.length > 0);
|
|
13082
|
+
return {
|
|
13083
|
+
sessionId,
|
|
13084
|
+
path: ref.path,
|
|
13085
|
+
subject,
|
|
13086
|
+
role: role === "operator" || role === "child" ? role : "unknown",
|
|
13087
|
+
...typeof parentSessionId === "string" ? { parentSessionId } : {},
|
|
13088
|
+
childSessionIds: relationship.childSessionIds,
|
|
13089
|
+
...relationship.depth !== void 0 ? { depth: relationship.depth } : {},
|
|
13090
|
+
...relationship.agentNickname ? { agentNickname: relationship.agentNickname } : {},
|
|
13091
|
+
...relationship.agentRole ? { agentRole: relationship.agentRole } : {},
|
|
13092
|
+
...relationship.agentPath ? { agentPath: relationship.agentPath } : {},
|
|
13093
|
+
...relationship.taskScope ? { taskScope: relationship.taskScope } : {},
|
|
13094
|
+
...relationship.turnId ? { turnId: relationship.turnId } : {},
|
|
13095
|
+
integrity: ref.integrity?.status ?? "complete",
|
|
13096
|
+
...ref.integrity ? {
|
|
13097
|
+
corruptionCount: ref.integrity.corruptions.length,
|
|
13098
|
+
...typeof corruptionDigest2 === "string" ? { corruptionDigest: corruptionDigest2 } : {},
|
|
13099
|
+
corruptions: ref.integrity.corruptions.slice(0, CORRUPTION_RECEIPT_DISPLAY_LIMIT)
|
|
13100
|
+
} : {}
|
|
13101
|
+
};
|
|
13102
|
+
}
|
|
11622
13103
|
function deterministicSummaryText(summary) {
|
|
11623
13104
|
const parts = [];
|
|
11624
13105
|
if (summary.stuckLoops > 0) parts.push(`${summary.stuckLoops} stuck loop(s)`);
|
|
@@ -12223,7 +13704,7 @@ function renderAdoption(ar) {
|
|
|
12223
13704
|
// src/improvement.ts
|
|
12224
13705
|
import { access, mkdir as mkdir4, mkdtemp as mkdtemp6, writeFile as writeFile7 } from "fs/promises";
|
|
12225
13706
|
import { tmpdir as tmpdir7 } from "os";
|
|
12226
|
-
import { dirname as dirname7, join as
|
|
13707
|
+
import { dirname as dirname7, join as join27, resolve as resolve7 } from "path";
|
|
12227
13708
|
import { pathToFileURL } from "url";
|
|
12228
13709
|
import {
|
|
12229
13710
|
AnalystRegistry,
|
|
@@ -12730,13 +14211,13 @@ async function runTraceStoreInvestigation(opts) {
|
|
|
12730
14211
|
};
|
|
12731
14212
|
}
|
|
12732
14213
|
async function writeTraceImprovementArtifacts(result, outDir) {
|
|
12733
|
-
const directory = outDir ? resolve7(outDir) : await mkdtemp6(
|
|
14214
|
+
const directory = outDir ? resolve7(outDir) : await mkdtemp6(join27(tmpdir7(), "traces-improvement-"));
|
|
12734
14215
|
await mkdir4(directory, { recursive: true });
|
|
12735
14216
|
const paths = {
|
|
12736
14217
|
directory,
|
|
12737
|
-
result:
|
|
12738
|
-
evidence:
|
|
12739
|
-
report:
|
|
14218
|
+
result: join27(directory, "result.json"),
|
|
14219
|
+
evidence: join27(directory, "evidence.jsonl"),
|
|
14220
|
+
report: join27(directory, "report.md"),
|
|
12740
14221
|
traces: result.otlpPath
|
|
12741
14222
|
};
|
|
12742
14223
|
const { report, ...machineResult } = result;
|
|
@@ -12748,11 +14229,11 @@ async function writeTraceImprovementArtifacts(result, outDir) {
|
|
|
12748
14229
|
return paths;
|
|
12749
14230
|
}
|
|
12750
14231
|
async function runTraceImprovement(opts) {
|
|
12751
|
-
const directory = opts.outDir ? resolve7(opts.outDir) : await mkdtemp6(
|
|
14232
|
+
const directory = opts.outDir ? resolve7(opts.outDir) : await mkdtemp6(join27(tmpdir7(), "traces-improvement-"));
|
|
12752
14233
|
await mkdir4(directory, { recursive: true });
|
|
12753
14234
|
const investigation = await runTraceInvestigation({
|
|
12754
14235
|
...opts,
|
|
12755
|
-
otlpOutPath: opts.otlpOutPath ??
|
|
14236
|
+
otlpOutPath: opts.otlpOutPath ?? join27(directory, "traces.otlp.jsonl")
|
|
12756
14237
|
});
|
|
12757
14238
|
const result = {
|
|
12758
14239
|
...investigation,
|
|
@@ -12769,27 +14250,27 @@ async function saveReport(path, report) {
|
|
|
12769
14250
|
}
|
|
12770
14251
|
|
|
12771
14252
|
// src/live.ts
|
|
12772
|
-
import { createHash as
|
|
14253
|
+
import { createHash as createHash12 } from "crypto";
|
|
12773
14254
|
var VERIFY_RE = /\b(test|tests|vitest|jest|pytest|go test|cargo test|tsc|typecheck|lint|biome|eslint|build|check:invariants|preflight)\b/i;
|
|
12774
14255
|
var CHANGE_RE = /\b(apply_patch|edit|write|create|delete|rename|replace|update|git apply|sed -i|perl -pi|tee\s|cat\s+>)/i;
|
|
12775
14256
|
var CLAIM_DONE_RE = /\b(done|fixed|complete|completed|implemented|merged|verified|all green|tests pass|ready|works now)\b/i;
|
|
12776
|
-
var sleep = (ms2, signal) => new Promise((
|
|
14257
|
+
var sleep = (ms2, signal) => new Promise((resolve10) => {
|
|
12777
14258
|
if (signal?.aborted) {
|
|
12778
|
-
|
|
14259
|
+
resolve10();
|
|
12779
14260
|
return;
|
|
12780
14261
|
}
|
|
12781
14262
|
const onAbort = () => {
|
|
12782
14263
|
clearTimeout(t);
|
|
12783
|
-
|
|
14264
|
+
resolve10();
|
|
12784
14265
|
};
|
|
12785
14266
|
const t = setTimeout(() => {
|
|
12786
14267
|
signal?.removeEventListener("abort", onAbort);
|
|
12787
|
-
|
|
14268
|
+
resolve10();
|
|
12788
14269
|
}, ms2);
|
|
12789
14270
|
signal?.addEventListener("abort", onAbort, { once: true });
|
|
12790
14271
|
});
|
|
12791
14272
|
function hash(parts) {
|
|
12792
|
-
return
|
|
14273
|
+
return createHash12("sha256").update(parts.join("\0")).digest("hex").slice(0, 16);
|
|
12793
14274
|
}
|
|
12794
14275
|
function asString(value) {
|
|
12795
14276
|
if (typeof value === "string" && value.trim()) return value;
|
|
@@ -13195,7 +14676,7 @@ async function streamSessions(opts) {
|
|
|
13195
14676
|
|
|
13196
14677
|
// src/supervisor-run-context.ts
|
|
13197
14678
|
import { readFile as readFile12, stat as stat14 } from "fs/promises";
|
|
13198
|
-
import { basename as basename14, join as
|
|
14679
|
+
import { basename as basename14, join as join28 } from "path";
|
|
13199
14680
|
import { materializeTreeView } from "@tangle-network/agent-runtime/kernel";
|
|
13200
14681
|
import {
|
|
13201
14682
|
NO_SOURCE_LIMITS
|
|
@@ -13356,10 +14837,10 @@ function parseInstant(v) {
|
|
|
13356
14837
|
return Number.isFinite(t) ? t : null;
|
|
13357
14838
|
}
|
|
13358
14839
|
async function readFileRunContext(runDir, opts = {}) {
|
|
13359
|
-
const journalText = await readMaybe(
|
|
14840
|
+
const journalText = await readMaybe(join28(runDir, SPAWN_JOURNAL_FILE));
|
|
13360
14841
|
const journal = readRunContextJournal(journalText);
|
|
13361
|
-
const coordination = readCoordinationLog(await readMaybe(
|
|
13362
|
-
const result = await readMaybe(
|
|
14842
|
+
const coordination = readCoordinationLog(await readMaybe(join28(runDir, COORDINATION_LOG_FILE)));
|
|
14843
|
+
const result = await readMaybe(join28(runDir, RESULT_FILE));
|
|
13363
14844
|
return {
|
|
13364
14845
|
runRef: runDir,
|
|
13365
14846
|
instanceId: opts.instanceId !== void 0 ? opts.instanceId : journal.root,
|
|
@@ -13382,7 +14863,7 @@ async function readFileRunContext(runDir, opts = {}) {
|
|
|
13382
14863
|
harnessWorkerTokens: null,
|
|
13383
14864
|
harnessMissingReason: "the file run context journals each child\u2019s own settled spend, so no external harness join is used",
|
|
13384
14865
|
limits: sourceLimits(journal, coordination),
|
|
13385
|
-
rootTranscriptRef: journal.present ?
|
|
14866
|
+
rootTranscriptRef: journal.present ? join28(runDir, SPAWN_JOURNAL_FILE) : null,
|
|
13386
14867
|
traceCommand: `traces watch ${runDir}`
|
|
13387
14868
|
};
|
|
13388
14869
|
}
|
|
@@ -13390,7 +14871,7 @@ function fileRunContextSupervisorRunReader(runDir, opts = {}) {
|
|
|
13390
14871
|
return { runRef: runDir, read: () => readFileRunContext(runDir, opts) };
|
|
13391
14872
|
}
|
|
13392
14873
|
async function isFileRunContextDir(runDir) {
|
|
13393
|
-
return stat14(
|
|
14874
|
+
return stat14(join28(runDir, SPAWN_JOURNAL_FILE)).then((s) => s.isFile()).catch(() => false);
|
|
13394
14875
|
}
|
|
13395
14876
|
function flattenJournal(journal) {
|
|
13396
14877
|
return journal.events.map((event) => {
|
|
@@ -13536,9 +15017,9 @@ function mergeRolls(a, b) {
|
|
|
13536
15017
|
};
|
|
13537
15018
|
}
|
|
13538
15019
|
async function readRunContextSnapshot(runDir) {
|
|
13539
|
-
const journal = readRunContextJournal(await readMaybe(
|
|
13540
|
-
const coordination = readCoordinationLog(await readMaybe(
|
|
13541
|
-
const resultText = await readMaybe(
|
|
15020
|
+
const journal = readRunContextJournal(await readMaybe(join28(runDir, SPAWN_JOURNAL_FILE)));
|
|
15021
|
+
const coordination = readCoordinationLog(await readMaybe(join28(runDir, COORDINATION_LOG_FILE)));
|
|
15022
|
+
const resultText = await readMaybe(join28(runDir, RESULT_FILE));
|
|
13542
15023
|
const result = parseResult(resultText);
|
|
13543
15024
|
const view = safeTreeView(journal.events);
|
|
13544
15025
|
const nodes = view === null ? [] : orderNodes(view, journal);
|
|
@@ -13721,7 +15202,7 @@ function wrap(text2, indent) {
|
|
|
13721
15202
|
|
|
13722
15203
|
// src/run-span-tree.ts
|
|
13723
15204
|
import { readdir as readdir16 } from "fs/promises";
|
|
13724
|
-
import { join as
|
|
15205
|
+
import { join as join29 } from "path";
|
|
13725
15206
|
import {
|
|
13726
15207
|
firstNumberAttr as firstNumberAttr3,
|
|
13727
15208
|
LLM_COST_ATTR_KEYS as LLM_COST_ATTR_KEYS2,
|
|
@@ -13994,7 +15475,7 @@ async function findSpanSourceFiles(dir) {
|
|
|
13994
15475
|
const found = [];
|
|
13995
15476
|
const entries = await readdir16(dir, { withFileTypes: true }).catch(() => []);
|
|
13996
15477
|
for (const entry of entries) {
|
|
13997
|
-
const child =
|
|
15478
|
+
const child = join29(dir, entry.name);
|
|
13998
15479
|
if (entry.isFile()) {
|
|
13999
15480
|
if (SPAN_FILE_PATTERN.test(entry.name)) found.push(child);
|
|
14000
15481
|
continue;
|
|
@@ -14002,7 +15483,7 @@ async function findSpanSourceFiles(dir) {
|
|
|
14002
15483
|
if (!entry.isDirectory()) continue;
|
|
14003
15484
|
const nested = await readdir16(child, { withFileTypes: true }).catch(() => []);
|
|
14004
15485
|
for (const inner of nested) {
|
|
14005
|
-
if (inner.isFile() && SPAN_FILE_PATTERN.test(inner.name)) found.push(
|
|
15486
|
+
if (inner.isFile() && SPAN_FILE_PATTERN.test(inner.name)) found.push(join29(child, inner.name));
|
|
14006
15487
|
}
|
|
14007
15488
|
}
|
|
14008
15489
|
return found.sort();
|
|
@@ -14010,7 +15491,7 @@ async function findSpanSourceFiles(dir) {
|
|
|
14010
15491
|
|
|
14011
15492
|
// src/supervisor-run-watch.ts
|
|
14012
15493
|
import { readdir as readdir17 } from "fs/promises";
|
|
14013
|
-
import { join as
|
|
15494
|
+
import { join as join30 } from "path";
|
|
14014
15495
|
function budget(node) {
|
|
14015
15496
|
const parts = [`${int(node.budget.maxIterations)}it`, `${int(node.budget.maxTokens)}tok`];
|
|
14016
15497
|
if (node.budget.maxUsd !== void 0) parts.push(`$${node.budget.maxUsd.toFixed(2)}`);
|
|
@@ -14154,7 +15635,7 @@ async function resolveRunContextDir(dir) {
|
|
|
14154
15635
|
const candidates = [];
|
|
14155
15636
|
for (const entry of entries) {
|
|
14156
15637
|
if (!entry.isDirectory()) continue;
|
|
14157
|
-
const child =
|
|
15638
|
+
const child = join30(dir, entry.name);
|
|
14158
15639
|
if (await isFileRunContextDir(child)) candidates.push(child);
|
|
14159
15640
|
}
|
|
14160
15641
|
if (candidates.length === 1) return candidates[0];
|
|
@@ -14259,13 +15740,13 @@ function aborted(signal) {
|
|
|
14259
15740
|
return signal !== void 0 && signal.aborted;
|
|
14260
15741
|
}
|
|
14261
15742
|
function sleep2(msDelay, signal) {
|
|
14262
|
-
return new Promise((
|
|
14263
|
-
const timer = setTimeout(
|
|
15743
|
+
return new Promise((resolve10) => {
|
|
15744
|
+
const timer = setTimeout(resolve10, msDelay);
|
|
14264
15745
|
signal?.addEventListener(
|
|
14265
15746
|
"abort",
|
|
14266
15747
|
() => {
|
|
14267
15748
|
clearTimeout(timer);
|
|
14268
|
-
|
|
15749
|
+
resolve10();
|
|
14269
15750
|
},
|
|
14270
15751
|
{ once: true }
|
|
14271
15752
|
);
|
|
@@ -14605,11 +16086,11 @@ async function collectSessionWorkflow(adapter, seeds, options = {}) {
|
|
|
14605
16086
|
}
|
|
14606
16087
|
|
|
14607
16088
|
// src/session-selection.ts
|
|
14608
|
-
import { createHash as
|
|
16089
|
+
import { createHash as createHash13 } from "crypto";
|
|
14609
16090
|
import { createReadStream as createReadStream2 } from "fs";
|
|
14610
16091
|
async function fileSha256(path, signal) {
|
|
14611
16092
|
signal?.throwIfAborted();
|
|
14612
|
-
const hash2 =
|
|
16093
|
+
const hash2 = createHash13("sha256");
|
|
14613
16094
|
try {
|
|
14614
16095
|
for await (const chunk of createReadStream2(path, { signal })) {
|
|
14615
16096
|
signal?.throwIfAborted();
|
|
@@ -14637,7 +16118,7 @@ async function sourceSnapshot(adapter, ref, signal) {
|
|
|
14637
16118
|
signal?.throwIfAborted();
|
|
14638
16119
|
files.push({ path, sha256: await fileSha256(path, signal) });
|
|
14639
16120
|
}
|
|
14640
|
-
const sha2562 = files.length === 1 ? files[0].sha256 :
|
|
16121
|
+
const sha2562 = files.length === 1 ? files[0].sha256 : createHash13("sha256").update(JSON.stringify(files)).digest("hex");
|
|
14641
16122
|
return { sha256: sha2562, files };
|
|
14642
16123
|
}
|
|
14643
16124
|
function summarizeWorkflows(workflows) {
|
|
@@ -14734,7 +16215,7 @@ async function collectSessionSelection(groups, options = {}) {
|
|
|
14734
16215
|
// src/session-index.ts
|
|
14735
16216
|
import { mkdtemp as mkdtemp7, readdir as readdir18, readFile as readFile13, stat as stat16, writeFile as writeFile8 } from "fs/promises";
|
|
14736
16217
|
import { tmpdir as tmpdir8 } from "os";
|
|
14737
|
-
import { dirname as dirname8, join as
|
|
16218
|
+
import { dirname as dirname8, join as join31 } from "path";
|
|
14738
16219
|
function indexRow(record2) {
|
|
14739
16220
|
return {
|
|
14740
16221
|
session: record2.session,
|
|
@@ -14807,7 +16288,7 @@ async function findContextRoot(cwd) {
|
|
|
14807
16288
|
if (!s) return null;
|
|
14808
16289
|
if (!s.isDirectory()) current = dirname8(current);
|
|
14809
16290
|
for (; ; ) {
|
|
14810
|
-
if (await pathStat2(
|
|
16291
|
+
if (await pathStat2(join31(current, ".git")) || await dirExists(join31(current, ".evolve")) || await fileExists(join31(current, "AGENTS.md")) || await fileExists(join31(current, "CLAUDE.md"))) {
|
|
14811
16292
|
return current;
|
|
14812
16293
|
}
|
|
14813
16294
|
const parent = dirname8(current);
|
|
@@ -14875,7 +16356,7 @@ async function walkMarkdown(root, kind, maxFiles = 100) {
|
|
|
14875
16356
|
}
|
|
14876
16357
|
for (const entry of entries) {
|
|
14877
16358
|
if (out.length >= maxFiles) return;
|
|
14878
|
-
const path =
|
|
16359
|
+
const path = join31(dir, entry.name);
|
|
14879
16360
|
if (entry.isDirectory()) {
|
|
14880
16361
|
await walk(path);
|
|
14881
16362
|
} else if (entry.isFile() && entry.name.endsWith(".md")) {
|
|
@@ -14890,22 +16371,38 @@ async function walkMarkdown(root, kind, maxFiles = 100) {
|
|
|
14890
16371
|
async function collectContextRoot(root) {
|
|
14891
16372
|
const files = [];
|
|
14892
16373
|
for (const name of ["AGENTS.md", "CLAUDE.md", "GEMINI.md"]) {
|
|
14893
|
-
const summary = await summarizeFile(
|
|
16374
|
+
const summary = await summarizeFile(join31(root, name), "instruction-doc");
|
|
14894
16375
|
if (summary) files.push(summary);
|
|
14895
16376
|
}
|
|
14896
|
-
const evolve =
|
|
16377
|
+
const evolve = join31(root, ".evolve");
|
|
14897
16378
|
for (const name of ["skill-runs.jsonl", "governor.jsonl", "experiments.jsonl"]) {
|
|
14898
|
-
const summary = await summarizeFile(
|
|
16379
|
+
const summary = await summarizeFile(join31(evolve, name), "evolve-jsonl");
|
|
14899
16380
|
if (summary) files.push(summary);
|
|
14900
16381
|
}
|
|
14901
|
-
for (const name of ["scorecard.json"]) {
|
|
14902
|
-
const summary = await summarizeFile(
|
|
16382
|
+
for (const name of ["scorecard.json", "current.json"]) {
|
|
16383
|
+
const summary = await summarizeFile(join31(evolve, name), "evolve-json");
|
|
14903
16384
|
if (summary) files.push(summary);
|
|
14904
16385
|
}
|
|
14905
|
-
files.push(...await walkMarkdown(
|
|
14906
|
-
files.push(...await walkMarkdown(
|
|
16386
|
+
files.push(...await walkMarkdown(join31(evolve, "reflections"), "reflection"));
|
|
16387
|
+
files.push(...await walkMarkdown(join31(evolve, "handoffs"), "handoff"));
|
|
16388
|
+
for (const name of await listEvolveRootFiles(evolve)) {
|
|
16389
|
+
if (/^handoff-.*\.md$/.test(name)) {
|
|
16390
|
+
const summary = await summarizeFile(join31(evolve, name), "handoff");
|
|
16391
|
+
if (summary) files.push(summary);
|
|
16392
|
+
}
|
|
16393
|
+
}
|
|
16394
|
+
const progress = await summarizeFile(join31(evolve, "progress.md"), "other");
|
|
16395
|
+
if (progress) files.push(progress);
|
|
14907
16396
|
return { root, files: files.sort((a, b) => a.path.localeCompare(b.path)) };
|
|
14908
16397
|
}
|
|
16398
|
+
async function listEvolveRootFiles(evolve) {
|
|
16399
|
+
try {
|
|
16400
|
+
const entries = await readdir18(evolve, { withFileTypes: true });
|
|
16401
|
+
return entries.filter((entry) => entry.isFile()).map((entry) => entry.name).sort();
|
|
16402
|
+
} catch {
|
|
16403
|
+
return [];
|
|
16404
|
+
}
|
|
16405
|
+
}
|
|
14909
16406
|
async function collectContextIndex(records) {
|
|
14910
16407
|
const rootSet = /* @__PURE__ */ new Set();
|
|
14911
16408
|
for (const record2 of records) {
|
|
@@ -14949,19 +16446,646 @@ function serializeSessionIndex(index) {
|
|
|
14949
16446
|
`;
|
|
14950
16447
|
}
|
|
14951
16448
|
async function writeSessionIndexFile(index, outPath) {
|
|
14952
|
-
const path = outPath ??
|
|
16449
|
+
const path = outPath ?? join31(await mkdtemp7(join31(tmpdir8(), "traces-index-")), "session-index.json");
|
|
14953
16450
|
await writeFile8(path, serializeSessionIndex(index), "utf8");
|
|
14954
16451
|
return path;
|
|
14955
16452
|
}
|
|
14956
16453
|
|
|
16454
|
+
// src/bundle.ts
|
|
16455
|
+
import { createHash as createHash14 } from "crypto";
|
|
16456
|
+
import { readFileSync as readFileSync5 } from "fs";
|
|
16457
|
+
import { cp, mkdir as mkdir5, readdir as readdir19, readFile as readFile14, stat as stat17, writeFile as writeFile9 } from "fs/promises";
|
|
16458
|
+
import { join as join32, relative as relative5, resolve as resolve8, sep as sep2 } from "path";
|
|
16459
|
+
var SESSION_WINDOW_PAD_MS = 15 * 60 * 1e3;
|
|
16460
|
+
var KNOWN_LIMITS = [
|
|
16461
|
+
"'what was decided' is prose-only: handoff, progress, and reflection files are copied as markdown with no structured decision extraction",
|
|
16462
|
+
"'what was measured' is split between experiments.jsonl free-text fields and progress/handoff markdown tables",
|
|
16463
|
+
"session-to-ledger join is by timestamp/cwd inference: ledger rows are window-sliced, not linked by transcript path",
|
|
16464
|
+
"progress.md is copied whole; no session-dated section extraction is performed",
|
|
16465
|
+
"open pull-request state is not captured: the bundler makes no network calls"
|
|
16466
|
+
];
|
|
16467
|
+
function sha256Hex(bytes) {
|
|
16468
|
+
return createHash14("sha256").update(bytes).digest("hex");
|
|
16469
|
+
}
|
|
16470
|
+
function bundlePath(...parts) {
|
|
16471
|
+
return parts.join("/");
|
|
16472
|
+
}
|
|
16473
|
+
async function statOrNull(path) {
|
|
16474
|
+
try {
|
|
16475
|
+
return await stat17(path);
|
|
16476
|
+
} catch (error) {
|
|
16477
|
+
if (isMissingPathError(error)) return null;
|
|
16478
|
+
if (error.code === "ENOTDIR") return null;
|
|
16479
|
+
throw error;
|
|
16480
|
+
}
|
|
16481
|
+
}
|
|
16482
|
+
function sliceJsonlByWindow(text2, windowStartMs, windowEndMs) {
|
|
16483
|
+
const kept = [];
|
|
16484
|
+
let totalRows = 0;
|
|
16485
|
+
let unparseableTsRows = 0;
|
|
16486
|
+
for (const line of text2.split("\n")) {
|
|
16487
|
+
if (!line.trim()) continue;
|
|
16488
|
+
totalRows += 1;
|
|
16489
|
+
let tsMs = Number.NaN;
|
|
16490
|
+
try {
|
|
16491
|
+
const row = JSON.parse(line);
|
|
16492
|
+
const ts = row !== null && typeof row === "object" && !Array.isArray(row) ? row.ts : void 0;
|
|
16493
|
+
if (typeof ts === "string") tsMs = Date.parse(ts);
|
|
16494
|
+
} catch {
|
|
16495
|
+
}
|
|
16496
|
+
if (Number.isNaN(tsMs)) {
|
|
16497
|
+
unparseableTsRows += 1;
|
|
16498
|
+
continue;
|
|
16499
|
+
}
|
|
16500
|
+
if (tsMs >= windowStartMs && tsMs <= windowEndMs) kept.push(line);
|
|
16501
|
+
}
|
|
16502
|
+
return { kept, totalRows, unparseableTsRows };
|
|
16503
|
+
}
|
|
16504
|
+
async function runGitLog(cwd, sinceIso, untilIso) {
|
|
16505
|
+
const { execFile: execFile2 } = await import("child_process");
|
|
16506
|
+
const { promisify: promisify2 } = await import("util");
|
|
16507
|
+
const run = promisify2(execFile2);
|
|
16508
|
+
try {
|
|
16509
|
+
const { stdout } = await run(
|
|
16510
|
+
"git",
|
|
16511
|
+
["-C", cwd, "log", `--since=${sinceIso}`, `--until=${untilIso}`, "--date=iso-strict", "--pretty=format:%H %ad %an %s"],
|
|
16512
|
+
{ timeout: 5e3 }
|
|
16513
|
+
);
|
|
16514
|
+
return { succeeded: true, stdout };
|
|
16515
|
+
} catch (error) {
|
|
16516
|
+
return { succeeded: false, error: error instanceof Error ? error.message.split("\n", 1)[0] : String(error) };
|
|
16517
|
+
}
|
|
16518
|
+
}
|
|
16519
|
+
async function listSessionBundleFiles(root) {
|
|
16520
|
+
const out = [];
|
|
16521
|
+
const pending = [resolve8(root)];
|
|
16522
|
+
while (pending.length > 0) {
|
|
16523
|
+
const dir = pending.pop();
|
|
16524
|
+
for (const entry of await readdir19(dir, { withFileTypes: true })) {
|
|
16525
|
+
const path = join32(dir, entry.name);
|
|
16526
|
+
if (entry.isDirectory()) pending.push(path);
|
|
16527
|
+
else if (entry.isFile()) out.push(relative5(resolve8(root), path).split(sep2).join("/"));
|
|
16528
|
+
}
|
|
16529
|
+
}
|
|
16530
|
+
return out.sort();
|
|
16531
|
+
}
|
|
16532
|
+
async function sliceLedgerJsonl(ctx, name) {
|
|
16533
|
+
const source = join32(ctx.evolveDir, name);
|
|
16534
|
+
const s = await statOrNull(source);
|
|
16535
|
+
if (!s?.isFile()) {
|
|
16536
|
+
ctx.absent.push({ path: bundlePath("ledger", name), reason: `no ${name} at ${source}` });
|
|
16537
|
+
return;
|
|
16538
|
+
}
|
|
16539
|
+
const text2 = await readFile14(source, "utf8");
|
|
16540
|
+
const target = join32(ctx.ledgerDir, name);
|
|
16541
|
+
if (ctx.windowStartMs === null || ctx.windowEndMs === null) {
|
|
16542
|
+
await writeFile9(target, text2, "utf8");
|
|
16543
|
+
ctx.slices.push({ path: bundlePath("ledger", name), source, rule: "copied-whole" });
|
|
16544
|
+
return;
|
|
16545
|
+
}
|
|
16546
|
+
const slice = sliceJsonlByWindow(text2, ctx.windowStartMs, ctx.windowEndMs);
|
|
16547
|
+
await writeFile9(target, slice.kept.length > 0 ? `${slice.kept.join("\n")}
|
|
16548
|
+
` : "", "utf8");
|
|
16549
|
+
ctx.slices.push({
|
|
16550
|
+
path: bundlePath("ledger", name),
|
|
16551
|
+
source,
|
|
16552
|
+
rule: "session-window-rows",
|
|
16553
|
+
totalRows: slice.totalRows,
|
|
16554
|
+
keptRows: slice.kept.length,
|
|
16555
|
+
unparseableTsRows: slice.unparseableTsRows
|
|
16556
|
+
});
|
|
16557
|
+
}
|
|
16558
|
+
async function copyLedgerWhole(ctx, name) {
|
|
16559
|
+
const source = join32(ctx.evolveDir, name);
|
|
16560
|
+
const s = await statOrNull(source);
|
|
16561
|
+
if (!s?.isFile()) {
|
|
16562
|
+
ctx.absent.push({ path: bundlePath("ledger", name), reason: `no ${name} at ${source}` });
|
|
16563
|
+
return;
|
|
16564
|
+
}
|
|
16565
|
+
await cp(source, join32(ctx.ledgerDir, name));
|
|
16566
|
+
ctx.slices.push({ path: bundlePath("ledger", name), source, rule: "copied-whole" });
|
|
16567
|
+
}
|
|
16568
|
+
async function copyLatestHandoff(ctx) {
|
|
16569
|
+
let names;
|
|
16570
|
+
try {
|
|
16571
|
+
names = (await readdir19(ctx.evolveDir, { withFileTypes: true })).filter((entry) => entry.isFile() && /^handoff-.*\.md$/.test(entry.name)).map((entry) => entry.name).sort();
|
|
16572
|
+
} catch {
|
|
16573
|
+
names = [];
|
|
16574
|
+
}
|
|
16575
|
+
const latest = names[names.length - 1];
|
|
16576
|
+
if (!latest) {
|
|
16577
|
+
ctx.absent.push({
|
|
16578
|
+
path: bundlePath("ledger", "handoff-*.md"),
|
|
16579
|
+
reason: `no flat handoff-*.md files in ${ctx.evolveDir}`
|
|
16580
|
+
});
|
|
16581
|
+
return;
|
|
16582
|
+
}
|
|
16583
|
+
const source = join32(ctx.evolveDir, latest);
|
|
16584
|
+
await cp(source, join32(ctx.ledgerDir, latest));
|
|
16585
|
+
ctx.slices.push({ path: bundlePath("ledger", latest), source, rule: "latest-by-name" });
|
|
16586
|
+
}
|
|
16587
|
+
async function copyWindowReflections(ctx) {
|
|
16588
|
+
const reflectionsDir = join32(ctx.evolveDir, "reflections");
|
|
16589
|
+
const s = await statOrNull(reflectionsDir);
|
|
16590
|
+
if (!s?.isDirectory()) {
|
|
16591
|
+
ctx.absent.push({
|
|
16592
|
+
path: bundlePath("ledger", "reflections"),
|
|
16593
|
+
reason: `no reflections directory at ${reflectionsDir}`
|
|
16594
|
+
});
|
|
16595
|
+
return;
|
|
16596
|
+
}
|
|
16597
|
+
const names = (await readdir19(reflectionsDir, { withFileTypes: true })).filter((entry) => entry.isFile() && entry.name.endsWith(".md")).map((entry) => entry.name).sort();
|
|
16598
|
+
const windowed = ctx.windowStartMs === null || ctx.windowEndMs === null ? names : names.filter((name) => {
|
|
16599
|
+
const date = name.match(/(\d{4}-\d{2}-\d{2})/)?.[1];
|
|
16600
|
+
if (!date) return false;
|
|
16601
|
+
const startDate = new Date(ctx.windowStartMs).toISOString().slice(0, 10);
|
|
16602
|
+
const endDate = new Date(ctx.windowEndMs).toISOString().slice(0, 10);
|
|
16603
|
+
return date >= startDate && date <= endDate;
|
|
16604
|
+
});
|
|
16605
|
+
if (windowed.length === 0) {
|
|
16606
|
+
ctx.absent.push({
|
|
16607
|
+
path: bundlePath("ledger", "reflections"),
|
|
16608
|
+
reason: `no reflections dated inside the session window in ${reflectionsDir}`
|
|
16609
|
+
});
|
|
16610
|
+
return;
|
|
16611
|
+
}
|
|
16612
|
+
await mkdir5(join32(ctx.ledgerDir, "reflections"), { recursive: true });
|
|
16613
|
+
const rule = ctx.windowStartMs === null ? "copied-whole" : "session-window-dated-files";
|
|
16614
|
+
for (const name of windowed) {
|
|
16615
|
+
const source = join32(reflectionsDir, name);
|
|
16616
|
+
await cp(source, join32(ctx.ledgerDir, "reflections", name));
|
|
16617
|
+
ctx.slices.push({ path: bundlePath("ledger", "reflections", name), source, rule });
|
|
16618
|
+
}
|
|
16619
|
+
}
|
|
16620
|
+
async function assembleSessionBundle(opts) {
|
|
16621
|
+
const { adapter, ref } = opts;
|
|
16622
|
+
const generatedAt = opts.generatedAt ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
16623
|
+
let transcriptBytes;
|
|
16624
|
+
try {
|
|
16625
|
+
transcriptBytes = await readFile14(ref.path);
|
|
16626
|
+
} catch (error) {
|
|
16627
|
+
if (isMissingPathError(error)) {
|
|
16628
|
+
throw new Error(
|
|
16629
|
+
`session transcript not found at ${ref.path} \u2014 a bundle cannot be assembled without its transcript`
|
|
16630
|
+
);
|
|
16631
|
+
}
|
|
16632
|
+
throw error;
|
|
16633
|
+
}
|
|
16634
|
+
const transcriptSha256 = sha256Hex(transcriptBytes);
|
|
16635
|
+
const spans = await parseSession(adapter, ref, { signal: opts.signal });
|
|
16636
|
+
const outDir = resolve8(opts.outDir);
|
|
16637
|
+
await mkdir5(outDir, { recursive: true });
|
|
16638
|
+
if ((await readdir19(outDir)).length > 0) {
|
|
16639
|
+
throw new Error(
|
|
16640
|
+
`bundle output directory ${outDir} is not empty \u2014 pass a new directory so one bundle holds exactly one session`
|
|
16641
|
+
);
|
|
16642
|
+
}
|
|
16643
|
+
const absent = [];
|
|
16644
|
+
const slices = [];
|
|
16645
|
+
await mkdir5(join32(outDir, "session"), { recursive: true });
|
|
16646
|
+
await writeFile9(join32(outDir, "session", "transcript.jsonl"), transcriptBytes);
|
|
16647
|
+
const subagentsSource = join32(ref.path.replace(/\.jsonl$/, ""), "subagents");
|
|
16648
|
+
if ((await statOrNull(subagentsSource))?.isDirectory()) {
|
|
16649
|
+
await cp(subagentsSource, join32(outDir, "session", "subagents"), { recursive: true });
|
|
16650
|
+
} else {
|
|
16651
|
+
absent.push({ path: bundlePath("session", "subagents"), reason: `no subagents directory at ${subagentsSource}` });
|
|
16652
|
+
}
|
|
16653
|
+
await mkdir5(join32(outDir, "derived"), { recursive: true });
|
|
16654
|
+
const otlpPath = join32(outDir, "derived", "trace.otlp.jsonl");
|
|
16655
|
+
const investigation = await runTraceInvestigation({
|
|
16656
|
+
spans,
|
|
16657
|
+
harness: ref.harness,
|
|
16658
|
+
sources: [sessionReportSource(ref, spans)],
|
|
16659
|
+
cwds: ref.cwd ? [ref.cwd] : [],
|
|
16660
|
+
minLoopOccurrences: opts.minLoopOccurrences,
|
|
16661
|
+
otlpOutPath: otlpPath,
|
|
16662
|
+
generatedAt,
|
|
16663
|
+
signal: opts.signal,
|
|
16664
|
+
log: opts.log
|
|
16665
|
+
});
|
|
16666
|
+
await writeFile9(join32(outDir, "derived", "report.md"), investigation.report, "utf8");
|
|
16667
|
+
const evidence3 = await buildPolicyEvidenceRecord(ref, spans, {
|
|
16668
|
+
generatedAt,
|
|
16669
|
+
minLoopOccurrences: opts.minLoopOccurrences,
|
|
16670
|
+
// Bundle-relative pointer: the bundle must stay internally valid when moved.
|
|
16671
|
+
otlpPath: bundlePath("derived", "trace.otlp.jsonl"),
|
|
16672
|
+
sourceSha256: transcriptSha256
|
|
16673
|
+
});
|
|
16674
|
+
await writeFile9(join32(outDir, "derived", "evidence.jsonl"), serializePolicyEvidence([evidence3]), "utf8");
|
|
16675
|
+
const index = await buildSessionIndexFromRows([{ ref, spans }], {
|
|
16676
|
+
generatedAt,
|
|
16677
|
+
minLoopOccurrences: opts.minLoopOccurrences,
|
|
16678
|
+
selection: { command: "bundle", harness: ref.harness, session: ref.sessionId }
|
|
16679
|
+
});
|
|
16680
|
+
await writeFile9(join32(outDir, "derived", "session-index.json"), serializeSessionIndex(index), "utf8");
|
|
16681
|
+
const { firstSpanAt, lastSpanAt } = evidence3.metrics;
|
|
16682
|
+
const windowStartMs = firstSpanAt !== null ? Date.parse(firstSpanAt) - SESSION_WINDOW_PAD_MS : null;
|
|
16683
|
+
const windowEndMs = lastSpanAt !== null ? Date.parse(lastSpanAt) + SESSION_WINDOW_PAD_MS : null;
|
|
16684
|
+
const contextRoot = await findContextRoot(ref.cwd);
|
|
16685
|
+
if (!contextRoot) {
|
|
16686
|
+
absent.push({ path: "ledger", reason: `no context root found from session cwd ${ref.cwd ?? "(unknown)"}` });
|
|
16687
|
+
} else {
|
|
16688
|
+
const evolveDir = join32(contextRoot, ".evolve");
|
|
16689
|
+
if (!(await statOrNull(evolveDir))?.isDirectory()) {
|
|
16690
|
+
absent.push({ path: "ledger", reason: `no .evolve ledger at ${evolveDir}` });
|
|
16691
|
+
} else {
|
|
16692
|
+
const ledgerDir = join32(outDir, "ledger");
|
|
16693
|
+
await mkdir5(ledgerDir, { recursive: true });
|
|
16694
|
+
const ctx = { evolveDir, ledgerDir, windowStartMs, windowEndMs, slices, absent };
|
|
16695
|
+
await sliceLedgerJsonl(ctx, "experiments.jsonl");
|
|
16696
|
+
await sliceLedgerJsonl(ctx, "skill-runs.jsonl");
|
|
16697
|
+
await copyLedgerWhole(ctx, "current.json");
|
|
16698
|
+
await copyLedgerWhole(ctx, "scorecard.json");
|
|
16699
|
+
await copyLedgerWhole(ctx, "progress.md");
|
|
16700
|
+
await copyLatestHandoff(ctx);
|
|
16701
|
+
await copyWindowReflections(ctx);
|
|
16702
|
+
}
|
|
16703
|
+
}
|
|
16704
|
+
const gitLogPath = bundlePath("repo", "git-log.txt");
|
|
16705
|
+
if (!ref.cwd) {
|
|
16706
|
+
absent.push({ path: gitLogPath, reason: "session has no recorded cwd to read git history from" });
|
|
16707
|
+
} else if (windowStartMs === null || windowEndMs === null) {
|
|
16708
|
+
absent.push({ path: gitLogPath, reason: "session window unavailable (spans carry no timestamps)" });
|
|
16709
|
+
} else {
|
|
16710
|
+
const log = await runGitLog(ref.cwd, new Date(windowStartMs).toISOString(), new Date(windowEndMs).toISOString());
|
|
16711
|
+
if (log.succeeded) {
|
|
16712
|
+
await mkdir5(join32(outDir, "repo"), { recursive: true });
|
|
16713
|
+
await writeFile9(join32(outDir, "repo", "git-log.txt"), log.stdout ? `${log.stdout}
|
|
16714
|
+
` : "", "utf8");
|
|
16715
|
+
} else {
|
|
16716
|
+
absent.push({ path: gitLogPath, reason: `git log failed in ${ref.cwd}: ${log.error}` });
|
|
16717
|
+
}
|
|
16718
|
+
}
|
|
16719
|
+
const files = await hashSessionBundleFiles(outDir);
|
|
16720
|
+
const manifest = {
|
|
16721
|
+
schemaVersion: 2,
|
|
16722
|
+
kind: "traces.session_bundle",
|
|
16723
|
+
view: "full",
|
|
16724
|
+
createdAt: generatedAt,
|
|
16725
|
+
provenance: {
|
|
16726
|
+
sessionId: ref.sessionId,
|
|
16727
|
+
harness: ref.harness,
|
|
16728
|
+
cwd: ref.cwd,
|
|
16729
|
+
transcriptPath: resolve8(ref.path),
|
|
16730
|
+
transcriptSha256,
|
|
16731
|
+
contextRoot,
|
|
16732
|
+
tracesVersion: tracesPackageVersion(),
|
|
16733
|
+
sessionWindow: { firstSpanAt, lastSpanAt, padMs: SESSION_WINDOW_PAD_MS }
|
|
16734
|
+
},
|
|
16735
|
+
files,
|
|
16736
|
+
absent,
|
|
16737
|
+
// The full view is the whole record: it drops nothing, and says so rather
|
|
16738
|
+
// than leaving a reader to infer completeness from a missing field.
|
|
16739
|
+
excluded: [],
|
|
16740
|
+
ledgerSlices: slices,
|
|
16741
|
+
knownLimits: KNOWN_LIMITS
|
|
16742
|
+
};
|
|
16743
|
+
const manifestPath = await writeSessionBundleManifest(outDir, manifest);
|
|
16744
|
+
return { directory: outDir, manifestPath, manifest };
|
|
16745
|
+
}
|
|
16746
|
+
async function hashSessionBundleFiles(root) {
|
|
16747
|
+
const files = [];
|
|
16748
|
+
for (const path of await listSessionBundleFiles(root)) {
|
|
16749
|
+
const bytes = await readFile14(join32(root, path));
|
|
16750
|
+
files.push({ path, bytes: bytes.length, sha256: sha256Hex(bytes) });
|
|
16751
|
+
}
|
|
16752
|
+
return files;
|
|
16753
|
+
}
|
|
16754
|
+
async function writeSessionBundleManifest(outDir, manifest) {
|
|
16755
|
+
const manifestPath = join32(outDir, "manifest.json");
|
|
16756
|
+
await writeFile9(manifestPath, `${JSON.stringify(manifest, null, 2)}
|
|
16757
|
+
`, "utf8");
|
|
16758
|
+
return manifestPath;
|
|
16759
|
+
}
|
|
16760
|
+
async function readSessionBundleManifest(bundleDir) {
|
|
16761
|
+
const path = join32(resolve8(bundleDir), "manifest.json");
|
|
16762
|
+
let text2;
|
|
16763
|
+
try {
|
|
16764
|
+
text2 = await readFile14(path, "utf8");
|
|
16765
|
+
} catch (error) {
|
|
16766
|
+
if (isMissingPathError(error)) {
|
|
16767
|
+
throw new Error(`no manifest.json at ${path} \u2014 ${bundleDir} is not a traces session bundle`);
|
|
16768
|
+
}
|
|
16769
|
+
throw error;
|
|
16770
|
+
}
|
|
16771
|
+
let parsed;
|
|
16772
|
+
try {
|
|
16773
|
+
parsed = JSON.parse(text2);
|
|
16774
|
+
} catch (error) {
|
|
16775
|
+
throw new Error(`${path} is not valid JSON: ${error instanceof Error ? error.message : String(error)}`);
|
|
16776
|
+
}
|
|
16777
|
+
const manifest = parsed;
|
|
16778
|
+
if (manifest?.kind !== "traces.session_bundle") {
|
|
16779
|
+
throw new Error(`${path} has kind ${JSON.stringify(manifest?.kind)}, expected "traces.session_bundle"`);
|
|
16780
|
+
}
|
|
16781
|
+
if (manifest.schemaVersion !== 2) {
|
|
16782
|
+
throw new Error(
|
|
16783
|
+
`${path} is bundle schemaVersion ${JSON.stringify(manifest.schemaVersion)}; this traces reads 2. Re-run \`traces bundle\` to assemble it with the view-aware manifest.`
|
|
16784
|
+
);
|
|
16785
|
+
}
|
|
16786
|
+
if (manifest.view !== "full" && manifest.view !== "evidence-only") {
|
|
16787
|
+
throw new Error(`${path} has view ${JSON.stringify(manifest.view)}, expected "full" or "evidence-only"`);
|
|
16788
|
+
}
|
|
16789
|
+
return manifest;
|
|
16790
|
+
}
|
|
16791
|
+
var cachedVersion;
|
|
16792
|
+
function tracesPackageVersion() {
|
|
16793
|
+
if (cachedVersion) return cachedVersion;
|
|
16794
|
+
const pkg = JSON.parse(
|
|
16795
|
+
readFileSync5(new URL("../package.json", import.meta.url), "utf8")
|
|
16796
|
+
);
|
|
16797
|
+
if (typeof pkg.version !== "string" || !pkg.version) throw new Error("package.json is missing version");
|
|
16798
|
+
cachedVersion = pkg.version;
|
|
16799
|
+
return cachedVersion;
|
|
16800
|
+
}
|
|
16801
|
+
|
|
16802
|
+
// src/bundle-view.ts
|
|
16803
|
+
import { createHash as createHash15 } from "crypto";
|
|
16804
|
+
import { mkdir as mkdir6, readdir as readdir20, readFile as readFile15, writeFile as writeFile10 } from "fs/promises";
|
|
16805
|
+
import { dirname as dirname9, join as join33, resolve as resolve9 } from "path";
|
|
16806
|
+
var EVIDENCE_ONLY_ALLOW_LIST = [
|
|
16807
|
+
{
|
|
16808
|
+
path: "derived/session-index.json",
|
|
16809
|
+
owned: true,
|
|
16810
|
+
carries: "per-session counts, tool names, models, and nearby-context file metadata"
|
|
16811
|
+
},
|
|
16812
|
+
{
|
|
16813
|
+
path: "derived/evidence.jsonl",
|
|
16814
|
+
owned: true,
|
|
16815
|
+
carries: "execution statistics, token and tool totals, loop and error counts"
|
|
16816
|
+
},
|
|
16817
|
+
{
|
|
16818
|
+
path: "ledger/current.json",
|
|
16819
|
+
owned: false,
|
|
16820
|
+
carries: "the repo ledger current-state record"
|
|
16821
|
+
},
|
|
16822
|
+
{
|
|
16823
|
+
path: "ledger/scorecard.json",
|
|
16824
|
+
owned: false,
|
|
16825
|
+
carries: "the repo ledger scorecard rows"
|
|
16826
|
+
},
|
|
16827
|
+
{
|
|
16828
|
+
path: "ledger/skill-runs.jsonl",
|
|
16829
|
+
owned: false,
|
|
16830
|
+
carries: "skill-run rows: skill name, timestamp, and outcome fields"
|
|
16831
|
+
}
|
|
16832
|
+
];
|
|
16833
|
+
var EXCLUSION_RULES = [
|
|
16834
|
+
{
|
|
16835
|
+
matches: (path) => path === "session/transcript.jsonl" || path.startsWith("session/"),
|
|
16836
|
+
rule: "session-source",
|
|
16837
|
+
reason: "raw session text: the transcript and any subagent transcript hold every word the session wrote, including the shell call that authored an earlier report"
|
|
16838
|
+
},
|
|
16839
|
+
{
|
|
16840
|
+
matches: (path) => path === "derived/report.md",
|
|
16841
|
+
rule: "session-text-derived",
|
|
16842
|
+
reason: "the deterministic report quotes the session verbatim: the first prompt line and evidence excerpts per finding"
|
|
16843
|
+
},
|
|
16844
|
+
{
|
|
16845
|
+
matches: (path) => path === "derived/trace.otlp.jsonl",
|
|
16846
|
+
rule: "session-text-derived",
|
|
16847
|
+
reason: "spans carry input.value, output.value and content: the full prompt and response text of every turn"
|
|
16848
|
+
},
|
|
16849
|
+
{
|
|
16850
|
+
matches: (path) => path === "ledger/progress.md",
|
|
16851
|
+
rule: "authored-prose",
|
|
16852
|
+
reason: "progress notes are written at session close and restate what the session concluded"
|
|
16853
|
+
},
|
|
16854
|
+
{
|
|
16855
|
+
matches: (path) => /^ledger\/handoff-.*\.md$/.test(path),
|
|
16856
|
+
rule: "authored-prose",
|
|
16857
|
+
reason: "a handoff is the session conclusion in prose"
|
|
16858
|
+
},
|
|
16859
|
+
{
|
|
16860
|
+
matches: (path) => path.startsWith("ledger/reflections/"),
|
|
16861
|
+
rule: "authored-prose",
|
|
16862
|
+
reason: "a reflection is the session conclusion in prose"
|
|
16863
|
+
},
|
|
16864
|
+
{
|
|
16865
|
+
matches: (path) => path === "ledger/experiments.jsonl",
|
|
16866
|
+
rule: "authored-prose",
|
|
16867
|
+
reason: "experiment rows carry free-text verdict and note fields that restate what the session concluded"
|
|
16868
|
+
},
|
|
16869
|
+
{
|
|
16870
|
+
matches: (path) => path === "repo/git-log.txt",
|
|
16871
|
+
rule: "authored-prose",
|
|
16872
|
+
reason: "commit subjects are prose the session authored"
|
|
16873
|
+
},
|
|
16874
|
+
{
|
|
16875
|
+
matches: () => true,
|
|
16876
|
+
rule: "not-allow-listed",
|
|
16877
|
+
reason: "the evidence-only view carries an explicit allow-list of counted artifacts, and this path is not on it"
|
|
16878
|
+
}
|
|
16879
|
+
];
|
|
16880
|
+
var SIGNATURE_WORDS = 8;
|
|
16881
|
+
var MIN_WORD_TOKENS = 7;
|
|
16882
|
+
function scrubIdentifiers(line) {
|
|
16883
|
+
return line.replace(/[a-z][a-z0-9+.-]*:\/\/\S*/gi, " ").replace(/\/?(?:[\w.@+~-]+\/)+[\w.@+~-]*/g, " ").replace(/\b[0-9a-f-]{8,}\b/gi, " ");
|
|
16884
|
+
}
|
|
16885
|
+
var PACKAGE_CONSTANT_PROSE = [POLICY_EVIDENCE_NOTE];
|
|
16886
|
+
function stripPackageConstants(text2) {
|
|
16887
|
+
let out = text2;
|
|
16888
|
+
for (const constant of PACKAGE_CONSTANT_PROSE) out = out.split(constant).join(" ");
|
|
16889
|
+
return out;
|
|
16890
|
+
}
|
|
16891
|
+
function collectJsonStrings(value, into) {
|
|
16892
|
+
if (typeof value === "string") into.push(value);
|
|
16893
|
+
else if (Array.isArray(value)) for (const item of value) collectJsonStrings(item, into);
|
|
16894
|
+
else if (value !== null && typeof value === "object") {
|
|
16895
|
+
for (const item of Object.values(value)) collectJsonStrings(item, into);
|
|
16896
|
+
}
|
|
16897
|
+
}
|
|
16898
|
+
function proseFragments(line) {
|
|
16899
|
+
const trimmed = line.trim();
|
|
16900
|
+
if (trimmed.startsWith("{") || trimmed.startsWith("[")) {
|
|
16901
|
+
try {
|
|
16902
|
+
const parsed = JSON.parse(trimmed);
|
|
16903
|
+
const strings = [];
|
|
16904
|
+
collectJsonStrings(parsed, strings);
|
|
16905
|
+
return strings;
|
|
16906
|
+
} catch {
|
|
16907
|
+
}
|
|
16908
|
+
}
|
|
16909
|
+
return [line];
|
|
16910
|
+
}
|
|
16911
|
+
function sessionTextSignatures(text2) {
|
|
16912
|
+
const out = /* @__PURE__ */ new Set();
|
|
16913
|
+
for (const line of stripPackageConstants(text2).split("\n")) {
|
|
16914
|
+
for (const fragment of proseFragments(line)) {
|
|
16915
|
+
const words = scrubIdentifiers(fragment).toLowerCase().replace(/[^a-z0-9]+/g, " ").trim().split(" ").filter(Boolean);
|
|
16916
|
+
for (let i = 0; i + SIGNATURE_WORDS <= words.length; i++) {
|
|
16917
|
+
const window = words.slice(i, i + SIGNATURE_WORDS);
|
|
16918
|
+
if (window.filter((word) => /^[a-z]+$/.test(word)).length < MIN_WORD_TOKENS) continue;
|
|
16919
|
+
out.add(window.join(" "));
|
|
16920
|
+
}
|
|
16921
|
+
}
|
|
16922
|
+
}
|
|
16923
|
+
return out;
|
|
16924
|
+
}
|
|
16925
|
+
async function findSessionTextLeaks(opts) {
|
|
16926
|
+
const index = /* @__PURE__ */ new Map();
|
|
16927
|
+
const carried = [];
|
|
16928
|
+
for (const path of opts.carriedPaths) {
|
|
16929
|
+
const signatures = sessionTextSignatures(await readFile15(join33(opts.viewDir, path), "utf8"));
|
|
16930
|
+
carried.push({ path, signatures: signatures.size });
|
|
16931
|
+
for (const signature of signatures) if (!index.has(signature)) index.set(signature, path);
|
|
16932
|
+
}
|
|
16933
|
+
const hits = /* @__PURE__ */ new Map();
|
|
16934
|
+
const comparedSources = [];
|
|
16935
|
+
let sourceSignatures = 0;
|
|
16936
|
+
for (const sourcePath of opts.excludedPaths) {
|
|
16937
|
+
const signatures = sessionTextSignatures(await readFile15(join33(opts.sourceDir, sourcePath), "utf8"));
|
|
16938
|
+
comparedSources.push(sourcePath);
|
|
16939
|
+
sourceSignatures += signatures.size;
|
|
16940
|
+
for (const signature of signatures) {
|
|
16941
|
+
const viewPath = index.get(signature);
|
|
16942
|
+
if (viewPath === void 0) continue;
|
|
16943
|
+
const hit = hits.get(viewPath) ?? { matched: /* @__PURE__ */ new Set(), sources: /* @__PURE__ */ new Set(), example: signature };
|
|
16944
|
+
hit.matched.add(signature);
|
|
16945
|
+
hit.sources.add(sourcePath);
|
|
16946
|
+
hits.set(viewPath, hit);
|
|
16947
|
+
}
|
|
16948
|
+
}
|
|
16949
|
+
const leaks = [...hits].map(([viewPath, hit]) => ({
|
|
16950
|
+
viewPath,
|
|
16951
|
+
sources: [...hit.sources].sort(),
|
|
16952
|
+
signatures: hit.matched.size,
|
|
16953
|
+
example: hit.example
|
|
16954
|
+
}));
|
|
16955
|
+
return { leaks, carried, comparedSources, sourceSignatures };
|
|
16956
|
+
}
|
|
16957
|
+
function classifyEvidenceOnlyPath(path) {
|
|
16958
|
+
const allowed = EVIDENCE_ONLY_ALLOW_LIST.find((entry) => entry.path === path);
|
|
16959
|
+
if (allowed) return { carried: true, owned: allowed.owned, carries: allowed.carries };
|
|
16960
|
+
const rule = EXCLUSION_RULES.find((entry) => entry.matches(path));
|
|
16961
|
+
return { carried: false, rule: rule.rule, reason: rule.reason };
|
|
16962
|
+
}
|
|
16963
|
+
var EVIDENCE_ONLY_LIMITS = [
|
|
16964
|
+
"the exclusion is verbatim-tight, not paraphrase-tight: a conclusion restated in different words inside a carried artifact is not detectable by content signatures",
|
|
16965
|
+
"ledger prose (progress, handoffs, reflections) and experiment free-text fields are excluded whole; a measured experiment result is readable only from the full bundle, under audit rules",
|
|
16966
|
+
"the view carries counts, names, identifiers, timestamps and hashes; a question that needs the words of a turn cannot be answered from it"
|
|
16967
|
+
];
|
|
16968
|
+
async function projectSessionBundle(opts) {
|
|
16969
|
+
if (opts.view !== "evidence-only") {
|
|
16970
|
+
throw new Error(
|
|
16971
|
+
`unsupported bundle view ${JSON.stringify(opts.view)}. The full view IS the bundle \`traces bundle\` assembles, so there is nothing to project into it; the projectable view is "evidence-only".`
|
|
16972
|
+
);
|
|
16973
|
+
}
|
|
16974
|
+
const generatedAt = opts.generatedAt ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
16975
|
+
const sourceDir = resolve9(opts.bundleDir);
|
|
16976
|
+
const source = await readSessionBundleManifest(sourceDir);
|
|
16977
|
+
if (source.view !== "full") {
|
|
16978
|
+
throw new Error(
|
|
16979
|
+
`${sourceDir} is already the ${source.view} view \u2014 project from the full bundle, not from a view of it`
|
|
16980
|
+
);
|
|
16981
|
+
}
|
|
16982
|
+
const outDir = resolve9(opts.outDir);
|
|
16983
|
+
await mkdir6(outDir, { recursive: true });
|
|
16984
|
+
if ((await readdir20(outDir)).length > 0) {
|
|
16985
|
+
throw new Error(
|
|
16986
|
+
`view output directory ${outDir} is not empty \u2014 pass a new directory so one directory holds exactly one view`
|
|
16987
|
+
);
|
|
16988
|
+
}
|
|
16989
|
+
const sourceHashes = new Map(source.files.map((file) => [file.path, file]));
|
|
16990
|
+
const carriedPaths = [];
|
|
16991
|
+
const carries = /* @__PURE__ */ new Map();
|
|
16992
|
+
const excluded = [];
|
|
16993
|
+
const candidates = [];
|
|
16994
|
+
for (const path of await listSessionBundleFiles(sourceDir)) {
|
|
16995
|
+
if (path === "manifest.json") continue;
|
|
16996
|
+
const bytes = await readFile15(join33(sourceDir, path));
|
|
16997
|
+
const sha2562 = createHash15("sha256").update(bytes).digest("hex");
|
|
16998
|
+
const recorded = sourceHashes.get(path);
|
|
16999
|
+
if (!recorded) {
|
|
17000
|
+
throw new Error(
|
|
17001
|
+
`${join33(sourceDir, path)} is not listed in the bundle manifest \u2014 the bundle changed after assembly and cannot be projected; re-run \`traces bundle\``
|
|
17002
|
+
);
|
|
17003
|
+
}
|
|
17004
|
+
if (recorded.sha256 !== sha2562) {
|
|
17005
|
+
throw new Error(
|
|
17006
|
+
`${join33(sourceDir, path)} does not match its manifest hash (${recorded.sha256} recorded, ${sha2562} on disk) \u2014 the bundle changed after assembly and cannot be projected; re-run \`traces bundle\``
|
|
17007
|
+
);
|
|
17008
|
+
}
|
|
17009
|
+
const verdict = classifyEvidenceOnlyPath(path);
|
|
17010
|
+
if (!verdict.carried) {
|
|
17011
|
+
excluded.push({ path, bytes: bytes.length, sha256: sha2562, rule: verdict.rule, reason: verdict.reason });
|
|
17012
|
+
continue;
|
|
17013
|
+
}
|
|
17014
|
+
candidates.push({ path, bytes, sha256: sha2562, carries: verdict.carries, owned: verdict.owned });
|
|
17015
|
+
}
|
|
17016
|
+
const report = await findSessionTextLeaks({
|
|
17017
|
+
viewDir: sourceDir,
|
|
17018
|
+
sourceDir,
|
|
17019
|
+
carriedPaths: candidates.map((candidate) => candidate.path),
|
|
17020
|
+
excludedPaths: excluded.map((entry) => entry.path)
|
|
17021
|
+
});
|
|
17022
|
+
const leakByPath = new Map(report.leaks.map((leak) => [leak.viewPath, leak]));
|
|
17023
|
+
const ownedLeak = report.leaks.find((leak) => candidates.some((c) => c.path === leak.viewPath && c.owned));
|
|
17024
|
+
if (ownedLeak) {
|
|
17025
|
+
throw new Error(
|
|
17026
|
+
`${ownedLeak.viewPath} repeats ${ownedLeak.signatures} ${SIGNATURE_WORDS}-word run(s) found in ${ownedLeak.sources.join(", ")}, for example "${ownedLeak.example}". That artifact is derived by traces and must carry counts only, so this is a defect in the derivation, not a property of the session. No view was written.`
|
|
17027
|
+
);
|
|
17028
|
+
}
|
|
17029
|
+
for (const candidate of candidates) {
|
|
17030
|
+
const leak = leakByPath.get(candidate.path);
|
|
17031
|
+
if (leak) {
|
|
17032
|
+
excluded.push({
|
|
17033
|
+
path: candidate.path,
|
|
17034
|
+
bytes: candidate.bytes.length,
|
|
17035
|
+
sha256: candidate.sha256,
|
|
17036
|
+
rule: "content-signature",
|
|
17037
|
+
reason: `the content check found ${leak.signatures} ${SIGNATURE_WORDS}-word run(s) this file shares with ${leak.sources.join(", ")}; carrying it would put back text the view removes`,
|
|
17038
|
+
contentMatch: { signatures: leak.signatures, sources: leak.sources }
|
|
17039
|
+
});
|
|
17040
|
+
continue;
|
|
17041
|
+
}
|
|
17042
|
+
const target = join33(outDir, candidate.path);
|
|
17043
|
+
await mkdir6(dirname9(target), { recursive: true });
|
|
17044
|
+
await writeFile10(target, candidate.bytes);
|
|
17045
|
+
carriedPaths.push(candidate.path);
|
|
17046
|
+
carries.set(candidate.path, candidate.carries);
|
|
17047
|
+
}
|
|
17048
|
+
const leakCheck = {
|
|
17049
|
+
signatureWords: SIGNATURE_WORDS,
|
|
17050
|
+
comparedSources: report.comparedSources,
|
|
17051
|
+
sourceSignatures: report.sourceSignatures,
|
|
17052
|
+
carried: report.carried.filter((entry) => carries.has(entry.path)).map((entry) => ({
|
|
17053
|
+
path: entry.path,
|
|
17054
|
+
carries: carries.get(entry.path),
|
|
17055
|
+
signatures: entry.signatures
|
|
17056
|
+
})),
|
|
17057
|
+
matches: 0
|
|
17058
|
+
};
|
|
17059
|
+
const sourceManifestBytes = await readFile15(join33(sourceDir, "manifest.json"));
|
|
17060
|
+
const manifest = {
|
|
17061
|
+
schemaVersion: 2,
|
|
17062
|
+
kind: "traces.session_bundle",
|
|
17063
|
+
view: "evidence-only",
|
|
17064
|
+
createdAt: generatedAt,
|
|
17065
|
+
provenance: source.provenance,
|
|
17066
|
+
files: await hashSessionBundleFiles(outDir),
|
|
17067
|
+
absent: source.absent,
|
|
17068
|
+
excluded,
|
|
17069
|
+
ledgerSlices: source.ledgerSlices.filter((slice) => carriedPaths.includes(slice.path)),
|
|
17070
|
+
knownLimits: [...source.knownLimits, ...EVIDENCE_ONLY_LIMITS],
|
|
17071
|
+
projection: {
|
|
17072
|
+
sourceDirectory: sourceDir,
|
|
17073
|
+
sourceManifestSha256: createHash15("sha256").update(sourceManifestBytes).digest("hex"),
|
|
17074
|
+
leakCheck
|
|
17075
|
+
}
|
|
17076
|
+
};
|
|
17077
|
+
const manifestPath = await writeSessionBundleManifest(outDir, manifest);
|
|
17078
|
+
return { directory: outDir, manifestPath, manifest, leakCheck };
|
|
17079
|
+
}
|
|
17080
|
+
|
|
14957
17081
|
// src/upload-state.ts
|
|
14958
|
-
import { mkdir as
|
|
17082
|
+
import { mkdir as mkdir7, readFile as readFile16, writeFile as writeFile11 } from "fs/promises";
|
|
14959
17083
|
import { homedir as homedir12 } from "os";
|
|
14960
|
-
import { dirname as
|
|
17084
|
+
import { dirname as dirname10, join as join34 } from "path";
|
|
14961
17085
|
import { contentHash } from "@tangle-network/agent-eval";
|
|
14962
17086
|
function stateFile() {
|
|
14963
|
-
const base = process.env.XDG_STATE_HOME ??
|
|
14964
|
-
return
|
|
17087
|
+
const base = process.env.XDG_STATE_HOME ?? join34(homedir12(), ".local", "state");
|
|
17088
|
+
return join34(base, "tangle-traces", "uploaded.json");
|
|
14965
17089
|
}
|
|
14966
17090
|
function uploadKey(harness, sessionId) {
|
|
14967
17091
|
return `${harness}:${sessionId}`;
|
|
@@ -14974,15 +17098,15 @@ function outboundHash(events, identity) {
|
|
|
14974
17098
|
}
|
|
14975
17099
|
async function loadState() {
|
|
14976
17100
|
try {
|
|
14977
|
-
return JSON.parse(await
|
|
17101
|
+
return JSON.parse(await readFile16(stateFile(), "utf8"));
|
|
14978
17102
|
} catch {
|
|
14979
17103
|
return {};
|
|
14980
17104
|
}
|
|
14981
17105
|
}
|
|
14982
17106
|
async function saveState(state) {
|
|
14983
17107
|
const f = stateFile();
|
|
14984
|
-
await
|
|
14985
|
-
await
|
|
17108
|
+
await mkdir7(dirname10(f), { recursive: true });
|
|
17109
|
+
await writeFile11(f, `${JSON.stringify(state, null, 2)}
|
|
14986
17110
|
`, "utf8");
|
|
14987
17111
|
}
|
|
14988
17112
|
function alreadyUploaded(state, harness, sessionId, hash2) {
|
|
@@ -14990,10 +17114,10 @@ function alreadyUploaded(state, harness, sessionId, hash2) {
|
|
|
14990
17114
|
}
|
|
14991
17115
|
|
|
14992
17116
|
// src/upload.ts
|
|
14993
|
-
import { readFile as
|
|
17117
|
+
import { readFile as readFile17 } from "fs/promises";
|
|
14994
17118
|
import { createRequire as createRequire2 } from "module";
|
|
14995
17119
|
import { hostname as hostname2 } from "os";
|
|
14996
|
-
import { basename as basename15, join as
|
|
17120
|
+
import { basename as basename15, join as join35 } from "path";
|
|
14997
17121
|
import { hostedClientFromEnv } from "@tangle-network/agent-eval/hosted";
|
|
14998
17122
|
import { REDACTION_VERSION } from "@tangle-network/agent-eval/traces";
|
|
14999
17123
|
var require2 = createRequire2(import.meta.url);
|
|
@@ -15017,7 +17141,7 @@ async function planUpload(opts = {}) {
|
|
|
15017
17141
|
async function gitBranch(cwd) {
|
|
15018
17142
|
if (!cwd) return void 0;
|
|
15019
17143
|
try {
|
|
15020
|
-
const head = await
|
|
17144
|
+
const head = await readFile17(join35(cwd, ".git", "HEAD"), "utf8");
|
|
15021
17145
|
const m = head.match(/^ref:\s*refs\/heads\/(.+)$/m);
|
|
15022
17146
|
return m?.[1]?.trim();
|
|
15023
17147
|
} catch {
|
|
@@ -15026,16 +17150,16 @@ async function gitBranch(cwd) {
|
|
|
15026
17150
|
}
|
|
15027
17151
|
async function sessionMeta(item) {
|
|
15028
17152
|
const meta = {
|
|
15029
|
-
[
|
|
15030
|
-
[
|
|
15031
|
-
[
|
|
15032
|
-
[
|
|
15033
|
-
[
|
|
15034
|
-
[
|
|
15035
|
-
};
|
|
15036
|
-
if (item.ref.cwd) meta[
|
|
17153
|
+
[ATTR.HARNESS]: item.ref.harness,
|
|
17154
|
+
[ATTR.SESSION_FILE]: basename15(item.ref.path),
|
|
17155
|
+
[ATTR.HOST]: hostname2(),
|
|
17156
|
+
[ATTR.UPLOADER]: `tangle-traces@${TRACES_VERSION}`,
|
|
17157
|
+
[ATTR.REDACTION_VERSION]: REDACTION_VERSION,
|
|
17158
|
+
[ATTR.REDACTION_COUNT]: item.redaction.redactionCount
|
|
17159
|
+
};
|
|
17160
|
+
if (item.ref.cwd) meta[ATTR.CWD] = item.ref.cwd;
|
|
15037
17161
|
const branch = await gitBranch(item.ref.cwd);
|
|
15038
|
-
if (branch) meta[
|
|
17162
|
+
if (branch) meta[ATTR.GIT_BRANCH] = branch;
|
|
15039
17163
|
return meta;
|
|
15040
17164
|
}
|
|
15041
17165
|
var msToNano = (epochMs) => (BigInt(epochMs) * 1000000n).toString();
|
|
@@ -15046,8 +17170,8 @@ function toTraceSpanEvents(spans, rootMeta) {
|
|
|
15046
17170
|
if (typeof v === "string" || typeof v === "number" || typeof v === "boolean") attributes2[k] = v;
|
|
15047
17171
|
else if (v != null) attributes2[k] = JSON.stringify(v);
|
|
15048
17172
|
}
|
|
15049
|
-
attributes2[
|
|
15050
|
-
attributes2[
|
|
17173
|
+
attributes2[ATTR.SESSION_ID] = s.trace_id;
|
|
17174
|
+
attributes2[ATTR.INGEST_SOURCE] = INGEST_SOURCE_CLI;
|
|
15051
17175
|
if (s.parent_span_id === null) Object.assign(attributes2, rootMeta);
|
|
15052
17176
|
const startMs = parseIsoToEpochMs(s.start_time);
|
|
15053
17177
|
const endMs = Math.max(startMs, parseIsoToEpochMs(s.end_time));
|
|
@@ -15106,7 +17230,7 @@ async function prepareOutbound(plan, opts, uploadedAt) {
|
|
|
15106
17230
|
const key = uploadKey(item.ref.harness, item.ref.sessionId);
|
|
15107
17231
|
const previous = plan.state[key];
|
|
15108
17232
|
const meta = await sessionMeta(item);
|
|
15109
|
-
const eventsAt = (timestamp) => toTraceSpanEvents(item.spans, { ...meta, [
|
|
17233
|
+
const eventsAt = (timestamp) => toTraceSpanEvents(item.spans, { ...meta, [ATTR.UPLOADED_AT]: timestamp });
|
|
15110
17234
|
let eventTimestamp2 = previous?.uploadedAt ?? uploadedAt;
|
|
15111
17235
|
let events = eventsAt(eventTimestamp2);
|
|
15112
17236
|
let hash2 = outboundHash(events, identity);
|
|
@@ -15184,6 +17308,15 @@ async function executeUpload(plan, opts = {}) {
|
|
|
15184
17308
|
|
|
15185
17309
|
export {
|
|
15186
17310
|
appendAll,
|
|
17311
|
+
ATTR,
|
|
17312
|
+
INGEST_SOURCE_CLI,
|
|
17313
|
+
environmentAttributes,
|
|
17314
|
+
stampEnvironmentAttrs,
|
|
17315
|
+
environmentFromSpanAttributes,
|
|
17316
|
+
DEFAULT_HARNESS,
|
|
17317
|
+
sessionIdFromAttributes,
|
|
17318
|
+
indexSessionIdsByTrace,
|
|
17319
|
+
stampSessionIdentity,
|
|
15187
17320
|
parseIsoToEpochMs,
|
|
15188
17321
|
parseSince,
|
|
15189
17322
|
resolveOtlpInputFiles,
|
|
@@ -15200,12 +17333,6 @@ export {
|
|
|
15200
17333
|
capText,
|
|
15201
17334
|
ACTOR_ATTR,
|
|
15202
17335
|
userPromptSpan,
|
|
15203
|
-
ATTR2 as ATTR,
|
|
15204
|
-
INGEST_SOURCE_CLI,
|
|
15205
|
-
DEFAULT_HARNESS,
|
|
15206
|
-
sessionIdFromAttributes,
|
|
15207
|
-
indexSessionIdsByTrace,
|
|
15208
|
-
stampSessionIdentity,
|
|
15209
17336
|
isChatTrajectoryMessage,
|
|
15210
17337
|
isChatTrajectory,
|
|
15211
17338
|
chatTrajectoryToSpans,
|
|
@@ -15258,6 +17385,7 @@ export {
|
|
|
15258
17385
|
scanSessions,
|
|
15259
17386
|
SessionWorkflowError,
|
|
15260
17387
|
describeSessionRelationship,
|
|
17388
|
+
POLICY_EVIDENCE_NOTE,
|
|
15261
17389
|
buildPolicyEvidenceRecord,
|
|
15262
17390
|
collectPolicyEvidence,
|
|
15263
17391
|
serializePolicyEvidence,
|
|
@@ -15281,9 +17409,13 @@ export {
|
|
|
15281
17409
|
resolveCaseResources,
|
|
15282
17410
|
enumerateReplayableCases,
|
|
15283
17411
|
zaiChatCaller,
|
|
17412
|
+
clipText,
|
|
15284
17413
|
buildFixPrompt,
|
|
17414
|
+
buildRetryFixPrompt,
|
|
17415
|
+
countScriptCommands,
|
|
15285
17416
|
extractFixCommand,
|
|
15286
17417
|
generateFixCommand,
|
|
17418
|
+
runFixLoop,
|
|
15287
17419
|
derivedImageTag,
|
|
15288
17420
|
dockerImagePreparer,
|
|
15289
17421
|
seededSample,
|
|
@@ -15292,6 +17424,21 @@ export {
|
|
|
15292
17424
|
parseReplayBatchArgs,
|
|
15293
17425
|
replayBatchUsage,
|
|
15294
17426
|
cmdReplayVerifyBatch,
|
|
17427
|
+
parseIncorrectStepsSubject,
|
|
17428
|
+
resolveFindingInvocation,
|
|
17429
|
+
replayVerifyFinding,
|
|
17430
|
+
findingReplayStep,
|
|
17431
|
+
findingTrajectoryId,
|
|
17432
|
+
resolveFindingReplayability,
|
|
17433
|
+
classifyVerdict,
|
|
17434
|
+
DEFAULT_SANDBOX_BASE_URL,
|
|
17435
|
+
assertSandboxReachable,
|
|
17436
|
+
verifyFindings,
|
|
17437
|
+
renderVerifiedFindingsSection,
|
|
17438
|
+
readFindingsFile,
|
|
17439
|
+
verifyFindingsUsage,
|
|
17440
|
+
parseVerifyFindingsArgs,
|
|
17441
|
+
cmdVerifyFindings,
|
|
15295
17442
|
runCommand,
|
|
15296
17443
|
commandAnalyzer,
|
|
15297
17444
|
haloAnalyzer,
|
|
@@ -15299,6 +17446,8 @@ export {
|
|
|
15299
17446
|
commandRedactor,
|
|
15300
17447
|
writeHodoscopeInput,
|
|
15301
17448
|
hodoscopeAnalyzer,
|
|
17449
|
+
httpJsonTransport,
|
|
17450
|
+
primeAnalyzer,
|
|
15302
17451
|
CODING_REDACTION_RULES,
|
|
15303
17452
|
TRACES_REDACTION_RULES,
|
|
15304
17453
|
redactSpans,
|
|
@@ -15329,7 +17478,7 @@ export {
|
|
|
15329
17478
|
analyzeSteeringChain,
|
|
15330
17479
|
REACTIONS,
|
|
15331
17480
|
analyzeReactions,
|
|
15332
|
-
|
|
17481
|
+
sessionReportSource,
|
|
15333
17482
|
buildTraceFindingPacket,
|
|
15334
17483
|
loadTracesConfig,
|
|
15335
17484
|
mergeTracesConfig,
|
|
@@ -15386,10 +17535,20 @@ export {
|
|
|
15386
17535
|
collectSessionWorkflow,
|
|
15387
17536
|
collectSessionSelection,
|
|
15388
17537
|
buildSessionIndex,
|
|
17538
|
+
findContextRoot,
|
|
15389
17539
|
buildSessionIndexFromRows,
|
|
15390
17540
|
collectSessionIndex,
|
|
15391
17541
|
serializeSessionIndex,
|
|
15392
17542
|
writeSessionIndexFile,
|
|
17543
|
+
listSessionBundleFiles,
|
|
17544
|
+
assembleSessionBundle,
|
|
17545
|
+
hashSessionBundleFiles,
|
|
17546
|
+
writeSessionBundleManifest,
|
|
17547
|
+
readSessionBundleManifest,
|
|
17548
|
+
sessionTextSignatures,
|
|
17549
|
+
findSessionTextLeaks,
|
|
17550
|
+
classifyEvidenceOnlyPath,
|
|
17551
|
+
projectSessionBundle,
|
|
15393
17552
|
uploadKey,
|
|
15394
17553
|
sessionHash,
|
|
15395
17554
|
outboundHash,
|