@tangle-network/agent-eval 0.221.8 → 0.222.1

package/dist/{hf-dataset-CKxIJQba.js → hf-dataset-DPVQXW5n.js}

@@ -1,9 +1,10 @@
-import { a as assertMinted, c as assertRolloutLine, d as isTrainableSplit, h as GATE_POLICIES, m as GATE_CHECK_IDS, n as ROLLOUT_ROLES, r as ROLLOUT_SCHEMA, v as undeclaredStepPayload } from "./schema-CvhfLjtZ.js";
-import { a as toRftItems, o as toSftRows, r as toJsonl, s as toVerifiersRolloutOutputs } from "./exporters-DWvM16LE.js";
+import { a as assertMinted, c as assertRolloutLine, d as isTrainableSplit, h as GATE_POLICIES, m as GATE_CHECK_IDS, n as ROLLOUT_ROLES, r as ROLLOUT_SCHEMA, v as undeclaredStepPayload, x as licenseClassOf } from "./schema-YUIXT8iW.js";
+import { a as toRftItems, o as toSftRows, r as toJsonl, s as toVerifiersRolloutOutputs } from "./exporters-ByLcILYU.js";
 import { basename, dirname, join } from "node:path";
 import { spawnSync } from "node:child_process";
-import { appendFile, mkdir, readFile, writeFile } from "node:fs/promises";
-//#region src/rollout/ledger.ts
+import { appendFile, mkdir, readFile, stat, writeFile } from "node:fs/promises";
+import { claudeCodeRefForFile, codexRefForFile, kimiRefForFile, listReaders, opencodeSessionsInStore, piRefForFile, readerFor, toChatMessages } from "@tangle-network/harness-sessions";
+// src/rollout/ledger.ts
 /**
 * Rollout-ledger file API — append-only JSONL of validated `tangle.rollout.v1`
 * lines. Writes validate BEFORE touching disk (a bad line never lands);
@@ -49,6 +50,17 @@ async function appendRolloutLines(path, lines) {
 async function readRolloutLedger(path) {
 	return readLines(path, (parsed, context) => assertMinted(parsed, context));
 }
+/**
+* Read under the write policy: imported and supervision rows whose positive
+* rewards no realness screen has seen stay readable, as they were writable.
+* Not training input until a screen mints them.
+*/
+async function readRolloutJournal(path) {
+	return readLines(path, (parsed, context) => {
+		assertRolloutLine(parsed, context);
+		return parsed;
+	});
+}
 async function readLines(path, admit) {
 	const raw = await readFile(path, "utf8");
 	const lines = [];
@@ -66,8 +78,243 @@ async function readLines(path, admit) {
 	}
 	return lines;
 }
-//#endregion
-//#region src/rollout/release/gate-report.ts
+// src/rollout/readers/gate-receipts.ts
+/**
+* `beelink-gate` receipts (`~/.local/state/beelink-gate/receipts.jsonl`) as
+* verifiable labels: one `judge` row per gated (repo, head SHA, command), with
+* reward 1 for exit 0 and 0 for any other recorded exit. A gate that did not
+* run its command to a verdict — exit 75 (TEMPFAIL), a timeout, the watchdog —
+* labels nothing and is skipped. The row carries no transcript; it labels the
+* commit, and joins to the sessions that produced it by `task.instance_id`.
+*/
+const TEMPFAIL = 75;
+async function gateReceiptRolloutLines(path, capturedAt = (/* @__PURE__ */ new Date()).toISOString()) {
+	const lines = [];
+	let skipped = 0;
+	for (const raw of (await readFile(path, "utf8")).split("\n")) {
+		if (!raw.trim()) continue;
+		let r;
+		try {
+			r = JSON.parse(raw);
+		} catch {
+			skipped += 1;
+			continue;
+		}
+		const { repo, sha, exit, gate_id: gateId } = r;
+		if (typeof repo !== "string" || typeof sha !== "string" || !/^[0-9a-f]{40}$/u.test(sha) || typeof exit !== "number" || exit === TEMPFAIL || r.timed_out === true || r.watchdog === true) {
+			skipped += 1;
+			continue;
+		}
+		const id = typeof gateId === "string" ? `beelink-gate:${gateId}` : `beelink-gate:${sha}:${r.at}`;
+		lines.push({
+			schema: ROLLOUT_SCHEMA,
+			rollout_id: id,
+			parent_rollout_id: null,
+			run_id: id,
+			experiment_id: null,
+			candidate_id: null,
+			generation: null,
+			candidate_index: null,
+			role: "judge",
+			task: {
+				suite: `beelink-gate/${repo}`,
+				instance_id: sha,
+				split: "search",
+				seed: null,
+				rep: 0
+			},
+			policy: {
+				harness: "beelink-gate",
+				harness_version: null,
+				model: null,
+				provider: null,
+				profile_commit: sha,
+				sampling: null
+			},
+			messages: [],
+			tool_defs: [],
+			outcome: {
+				reward: exit === 0 ? 1 : 0,
+				reward_source: "beelink-gate/exit",
+				verdict: {
+					exit,
+					command: r.command ?? null,
+					mode: r.mode ?? null,
+					class: r.class ?? null,
+					pr_number: r.pr_number ?? null,
+					base_sha: r.base_sha ?? null,
+					lane: r.lane ?? null,
+					host: r.host ?? null,
+					at: r.at ?? null
+				},
+				metrics: {},
+				is_completed: true,
+				is_truncated: false,
+				error: null,
+				realness_gated: false,
+				realness_screened: false
+			},
+			cost: {
+				usd: null,
+				tokens_in: null,
+				tokens_out: null,
+				tokens_reasoning: null,
+				cache_read: null,
+				cache_write: null,
+				wall_s: typeof r.ran_s === "number" ? r.ran_s : null
+			},
+			artifacts: {
+				patch_path: null,
+				run_dir: null,
+				transcript_ref: null
+			},
+			provenance: {
+				captured_at: capturedAt,
+				capture: "backfill",
+				gap: `gate receipt: verifiable label for ${repo}@${sha}; no transcript`
+			}
+		});
+	}
+	return {
+		lines,
+		skipped
+	};
+}
+// src/rollout/readers/harness-session-lines.ts
+/**
+* Native harness sessions (OpenCode, Kimi, Claude Code, Codex, Pi, Factory) as
+* `tangle.rollout.v1` rows, through @tangle-network/harness-sessions — the one
+* reader of those stores. A session carries no task verdict, so its reward is
+* null and `provenance.gap` says so; a later labeler supplies the reward.
+*
+* The producer comes from the models that answered (`servedModels`), never from
+* the request, and the license class from that model and the harness.
+*/
+const REF_FOR_FILE = {
+	"claude-code": (path) => claudeCodeRefForFile(path, null),
+	codex: codexRefForFile,
+	"kimi-code": kimiRefForFile,
+	pi: piRefForFile
+};
+/** Session refs under a HOME or store directory, or in one session file / OpenCode store. */
+async function harnessSessionRefs(target, options = {}) {
+	const entry = await stat(target);
+	let refs;
+	if (entry.isDirectory()) {
+		const readers = options.harness ? [readerFor(options.harness)] : listReaders();
+		refs = [];
+		const locate = options.sinceMs === void 0 ? {} : { sinceMs: options.sinceMs };
+		for (const reader of readers) refs.push(...await reader.locate(target, locate));
+	} else if (/\.db$/u.test(target)) refs = await opencodeSessionsInStore(target, null, {});
+	else {
+		const refFor = options.harness ? REF_FOR_FILE[options.harness] : void 0;
+		if (refFor === void 0) throw new Error(`${target}: pass --harness (${Object.keys(REF_FOR_FILE).join("|")}) for a file`);
+		refs = [refFor(target)];
+	}
+	return options.sinceMs === void 0 ? refs : refs.filter((ref) => ref.mtimeMs >= options.sinceMs);
+}
+/** The model that answered most calls, split into provider and model. */
+function producer(session) {
+	const [top] = Object.entries(session.servedModels).sort((a, b) => b[1] - a[1]);
+	if (top === void 0) return {
+		provider: null,
+		model: null
+	};
+	const call = session.modelCalls.find((c) => c.servedModel !== null && top[0].endsWith(c.servedModel));
+	return {
+		provider: call?.provider ?? null,
+		model: call?.servedModel ?? top[0]
+	};
+}
+const sum = (values) => values.some((v) => v !== null) ? values.reduce((a, v) => a + (v ?? 0), 0) : null;
+/** One normalized session as a rollout row. */
+function harnessSessionRolloutLine(session, ref, capturedAt = (/* @__PURE__ */ new Date()).toISOString()) {
+	const { provider, model } = producer(session);
+	const id = `${session.harness}:${session.nativeSessionId}`;
+	const startedMs = session.startedAt === null ? null : Date.parse(session.startedAt);
+	const endedMs = session.endedAt === null ? null : Date.parse(session.endedAt);
+	const line = {
+		schema: ROLLOUT_SCHEMA,
+		rollout_id: id,
+		parent_rollout_id: session.parentNativeSessionId === null ? null : `${session.harness}:${session.parentNativeSessionId}`,
+		run_id: id,
+		experiment_id: null,
+		candidate_id: null,
+		generation: null,
+		candidate_index: null,
+		role: "agent",
+		task: {
+			suite: `harness/${session.harness}`,
+			instance_id: session.nativeSessionId,
+			split: "search",
+			seed: null,
+			rep: 0
+		},
+		policy: {
+			harness: session.harness,
+			harness_version: null,
+			model,
+			provider,
+			profile_commit: null,
+			sampling: null
+		},
+		messages: toChatMessages(session),
+		tool_defs: [],
+		outcome: {
+			reward: null,
+			reward_source: null,
+			verdict: null,
+			metrics: {
+				ending: session.ending.status,
+				model_calls: session.modelCalls.length,
+				tool_calls: session.toolCalls.length,
+				served_models: session.servedModels
+			},
+			is_completed: session.ending.status === "completed",
+			is_truncated: ref.truncated === true,
+			error: session.ending.error?.message ?? null,
+			realness_gated: false
+		},
+		cost: {
+			usd: sum(session.modelCalls.map((c) => c.costUsd)),
+			tokens_in: session.usage?.input ?? null,
+			tokens_out: session.usage?.output ?? null,
+			tokens_reasoning: session.usage?.reasoning ?? null,
+			cache_read: session.usage?.cacheRead ?? null,
+			cache_write: session.usage?.cacheWrite ?? null,
+			wall_s: startedMs !== null && endedMs !== null && endedMs >= startedMs ? (endedMs - startedMs) / 1e3 : null,
+			llm_call_count: session.modelCalls.length
+		},
+		artifacts: {
+			patch_path: null,
+			run_dir: session.cwd,
+			transcript_ref: ref.path
+		},
+		provenance: {
+			captured_at: capturedAt,
+			capture: "backfill",
+			gap: "imported harness session: no task verdict"
+		}
+	};
+	const license = licenseClassOf(line);
+	if (license !== null) line.policy.license_class = license;
+	return line;
+}
+/** Read each ref in turn; a session its reader refuses is reported, never guessed. */
+async function* harnessSessionRolloutLines(refs, capturedAt) {
+	for (const ref of refs) try {
+		yield {
+			ref,
+			line: harnessSessionRolloutLine(await readerFor(ref.harness).read(ref), ref, capturedAt)
+		};
+	} catch (error) {
+		yield {
+			ref,
+			error: error instanceof Error ? error.message : String(error)
+		};
+	}
+}
+// src/rollout/release/gate-report.ts
 const FORMAT_GATE_DISPOSITION = {
 	sft: "exclude",
 	verifiers: "zero-and-flag",
@@ -192,8 +439,7 @@ function assertGateReport(report) {
 		if (FORMAT_GATE_DISPOSITION[format] === "exclude" && counts.emitted > 0) throw new Error(`release format "${format}" declares gated lines EXCLUDED but wrote ${counts.emitted} of them.`);
 	}
 }
-//#endregion
-//#region src/rollout/release/card.ts
+// src/rollout/release/card.ts
 /**
 * HuggingFace dataset-card (README.md) generation for a rollout-ledger release.
 *
@@ -360,7 +606,7 @@ function buildDatasetCard(inputs) {
 
 One line per agent invocation (supervisor episode, worker session, proposer shot, judge call, analyst pass) captured by the \`${ROLLOUT_SCHEMA}\` rollout ledger, labeled with improvement-loop coordinates and the official-judge reward, with the full message transcript inline.
 
-This release contains the **trainable split only** (\`search\`). Holdout, dev, and canary splits are structurally excluded at build time, and the build additionally drops any non-trainable line as a fail-closed filter (${excluded.nonTrain} dropped here).
+This release contains the **trainable split only** (\`search\`). Holdout, dev, and canary splits are structurally excluded at build time, and the build additionally drops any non-trainable line as a fail-closed filter (${excluded.nonTrain} dropped here). Lines with any secret or private detail (${excluded.privacy}) are dropped whole, never redacted into the release.
 
 ## Formats
 
@@ -433,8 +679,7 @@ ${scrubTable}
 \`\`\`
 `;
 }
-//#endregion
-//#region src/rollout/release/scrub.ts
+// src/rollout/release/scrub.ts
 /**
 * Deterministic scrubbing pass over rollout-ledger lines before public release.
 *
@@ -546,8 +791,93 @@ function scrubLines(lines) {
 function defaultRolloutScrubber(text) {
 	return scrubText(text, emptyScrubCounts());
 }
-//#endregion
-//#region src/rollout/release/hf-dataset.ts
+const scrubPattern = (name) => {
+	const rule = SCRUB_RULES.find((r) => r.name === name);
+	if (!rule) throw new Error(`no scrub rule "${name}"`);
+	return rule.pattern;
+};
+/** Bits per character over the token's own symbols. */
+function entropy(token) {
+	const counts = /* @__PURE__ */ new Map();
+	for (const char of token) counts.set(char, (counts.get(char) ?? 0) + 1);
+	let bits = 0;
+	for (const count of counts.values()) {
+		const p = count / token.length;
+		bits -= p * Math.log2(p);
+	}
+	return bits;
+}
+const ENTROPY_TOKEN = /[A-Za-z0-9+=_-]{32,}/g;
+const NAMED = /(?:^|[-_])[a-z]{4,}(?:$|[-_])/;
+const highEntropy = (text) => {
+	for (const [token] of text.matchAll(ENTROPY_TOKEN)) {
+		if (NAMED.test(token)) continue;
+		if (/[A-Z]/.test(token) && /[a-z]/.test(token) && /\d/.test(token) && entropy(token) > 4.3) return true;
+	}
+	return false;
+};
+const EMAIL = /[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)*\.[A-Za-z]{2,}/g;
+const NOT_PERSONAL_EMAIL = /^git@|@(?:[\w-]+\.)*example\.(?:com|org|net)$/i;
+const email = (text) => {
+	for (const [match] of text.matchAll(EMAIL)) if (!NOT_PERSONAL_EMAIL.test(match)) return true;
+	return false;
+};
+const PRIVACY_RULES = [
+	...[
+		"api-key",
+		"env-secret",
+		"bearer-token"
+	].map((name) => {
+		const pattern = scrubPattern(name);
+		return {
+			name,
+			hit: (text) => text.search(pattern) >= 0
+		};
+	}),
+	{
+		name: "private-key",
+		hit: (text) => /-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----/.test(text)
+	},
+	{
+		name: "high-entropy",
+		hit: highEntropy
+	},
+	{
+		name: "email",
+		hit: email
+	},
+	{
+		name: "phone",
+		hit: (text) => /(?<![\d.])(?:\+?1[ .-]?)?\(?\d{3}\)?[ .-]\d{3}[ .-]\d{4}(?![\d.])/.test(text)
+	},
+	{
+		name: "postal-address",
+		hit: (text) => /\b\d{1,6}\s+(?:[NSEW]\.?\s+)?(?:[A-Z][a-z]+\s+){1,3}(?:Street|St|Avenue|Ave|Road|Rd|Boulevard|Blvd|Lane|Ln|Drive|Dr|Court|Ct|Place|Pl|Way|Terrace|Parkway|Pkwy)\b/.test(text)
+	}
+];
+const PRIVACY_RULE_NAMES = [...PRIVACY_RULES.map((rule) => rule.name), "denylist"];
+function* stringsOf(value) {
+	if (typeof value === "string") yield value;
+	else if (Array.isArray(value)) for (const item of value) yield* stringsOf(item);
+	else if (value !== null && typeof value === "object") for (const item of Object.values(value)) yield* stringsOf(item);
+}
+/**
+* The first privacy rule any string in `value` hits, or `null` when clean.
+* `denylist` holds literal terms (names, addresses, numbers) matched without
+* case; it is supplied at run time and never committed.
+*/
+function privacyHit(value, denylist = []) {
+	const terms = denylist.map((term) => term.toLowerCase()).filter(Boolean);
+	for (const text of stringsOf(value)) {
+		for (const rule of PRIVACY_RULES) if (rule.hit(text)) return rule.name;
+		if (terms.length > 0) {
+			const lower = text.toLowerCase();
+			if (terms.some((term) => lower.includes(term))) return "denylist";
+		}
+	}
+	return null;
+}
+// src/rollout/release/hf-dataset.ts
 /**
 * One-command HuggingFace dataset release from rollout ledgers:
 *
@@ -556,7 +886,8 @@ function defaultRolloutScrubber(text) {
 *
 * Pipeline per input ledger: read + validate → fail-closed filters
 * (trainable split only; proposer sessions dropped unless
-* --include-proposers, they contain improvement-loop harness source) →
+* --include-proposers, they contain improvement-loop harness source; any line
+* with a secret or private detail, or a `--denylist` term, dropped whole) →
 * deterministic scrub → export the requested formats + scrub-report.json +
 * auto-generated README.md card. Deterministic: same inputs and flags →
 * byte-identical output dir.
@@ -573,8 +904,10 @@ async function buildHfDataset(inputs, options) {
 		totals: emptyScrubCounts(),
 		excluded: {
 			proposers: 0,
-			nonTrain: 0
-		}
+			nonTrain: 0,
+			privacy: 0
+		},
+		dropped: Object.fromEntries(PRIVACY_RULE_NAMES.map((name) => [name, 0]))
 	};
 	const kept = [];
 	let read = 0;
@@ -590,6 +923,12 @@ async function buildHfDataset(inputs, options) {
 				report.excluded.proposers += 1;
 				return false;
 			}
+			const hit = privacyHit(line, options.denylist);
+			if (hit !== null) {
+				report.excluded.privacy += 1;
+				report.dropped[hit] = (report.dropped[hit] ?? 0) + 1;
+				return false;
+			}
 			return true;
 		}));
 		report.files[input] = scrubbed.counts;
@@ -688,14 +1027,15 @@ function pushDataset(repo, outDir) {
 	const run = spawnSync(command, args, { stdio: "inherit" });
 	if (run.status !== 0) throw new Error(`huggingface-cli upload exited ${String(run.status)}`);
 }
-const ROLLOUT_RELEASE_USAGE = "usage: agent-eval rollout-release <ledger.jsonl...> --out <dir> [--formats sft,verifiers,rft,raw] [--include-proposers] [--push <org/name>]";
+const ROLLOUT_RELEASE_USAGE = "usage: agent-eval rollout-release <ledger.jsonl...> --out <dir> [--formats sft,verifiers,rft,raw] [--include-proposers] [--denylist <terms.txt>] [--push <org/name>]";
 function parseRolloutReleaseArgs(argv) {
 	const args = {
 		inputs: [],
 		out: "",
 		formats: [...RELEASE_FORMATS],
 		includeProposers: false,
-		push: null
+		push: null,
+		denylistPath: null
 	};
 	for (let i = 0; i < argv.length; i++) {
 		const arg = argv[i];
@@ -705,6 +1045,7 @@ function parseRolloutReleaseArgs(argv) {
 			for (const format of raw) if (!RELEASE_FORMATS.includes(format)) throw new Error(`unknown format "${format}" — expected one of ${RELEASE_FORMATS.join(",")}`);
 			args.formats = raw;
 		} else if (arg === "--include-proposers") args.includeProposers = true;
+		else if (arg === "--denylist") args.denylistPath = argv[++i] ?? null;
 		else if (arg === "--push") args.push = argv[++i] ?? null;
 		else if (arg.startsWith("--")) throw new Error(`unknown flag "${arg}"`);
 		else args.inputs.push(arg);
@@ -722,6 +1063,7 @@ async function runRolloutReleaseCli(argv) {
 		process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
 		return 2;
 	}
+	if (args.denylistPath !== null) args.denylist = (await readFile(args.denylistPath, "utf8")).split("\n").map((line) => line.replace(/#.*/, "").trim()).filter(Boolean);
 	const summary = await buildHfDataset(args.inputs, args);
 	process.stdout.write(`${JSON.stringify({
 		read: summary.read,
@@ -737,7 +1079,6 @@ async function runRolloutReleaseCli(argv) {
 	}
 	return 0;
 }
-//#endregion
-export { scrubText as a, appendRolloutLines as c, scrubLines as i, readRolloutLedger as l, runRolloutReleaseCli as n, buildDatasetCard as o, defaultRolloutScrubber as r, assertGateReport as s, buildHfDataset as t, writeRolloutLedger as u };
+export { writeRolloutLedger as _, privacyHit as a, buildDatasetCard as c, harnessSessionRolloutLine as d, harnessSessionRolloutLines as f, readRolloutLedger as g, readRolloutJournal as h, defaultRolloutScrubber as i, assertGateReport as l, appendRolloutLines as m, runRolloutReleaseCli as n, scrubLines as o, gateReceiptRolloutLines as p, PRIVACY_RULE_NAMES as r, scrubText as s, buildHfDataset as t, harnessSessionRefs as u };
 
-//# sourceMappingURL=hf-dataset-CKxIJQba.js.map
+//# sourceMappingURL=hf-dataset-DPVQXW5n.js.map