@farming-labs/docs 0.2.103 → 0.2.105

package/dist/agent-feedback-evaluations-DQCTFegz.mjs

@@ -0,0 +1,240 @@
+import { i as sha256DocsContent } from "./retrieval-digest-B-SgtXKK.mjs";
+
+//#region src/agent-feedback-evaluations.ts
+const DOCS_AGENT_FEEDBACK_EVALUATION_CANDIDATES_FORMAT = "farming-labs-agent-feedback-evaluation-candidates.v1";
+const DOCS_AGENT_FEEDBACK_EVALUATION_BASELINE_FORMAT = "farming-labs-agent-feedback-evaluation-baseline.v1";
+const DOCS_AGENT_FEEDBACK_EVALUATION_REGRESSION_FORMAT = "farming-labs-agent-feedback-evaluation-regression.v1";
+const SECRET_PATTERNS = [
+	{
+		pattern: /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/gu,
+		replacement: "[redacted-private-key]"
+	},
+	{
+		pattern: /\b(?:sk-[A-Za-z0-9_-]{12,}|gh[pousr]_[A-Za-z0-9_]{12,}|github_pat_[A-Za-z0-9_]{12,}|xox[baprs]-[A-Za-z0-9-]{12,})\b/gu,
+		replacement: "[redacted-token]"
+	},
+	{
+		pattern: /\bBearer\s+[A-Za-z0-9._~+/=-]{12,}\b/giu,
+		replacement: "Bearer [redacted-token]"
+	},
+	{
+		pattern: /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/giu,
+		replacement: "[redacted-email]"
+	},
+	{
+		pattern: /([?&](?:api[_-]?key|password|secret|signature|token)=)[^&#\s]+/giu,
+		replacement: (match) => `${match.slice(0, match.indexOf("=") + 1)}[redacted]`
+	}
+];
+function sanitizeText(value, maxLength) {
+	if (typeof value !== "string") return { redactions: 0 };
+	let output = value.split("").map((character) => {
+		const code = character.charCodeAt(0);
+		return code <= 31 || code === 127 ? " " : character;
+	}).join("").trim().replace(/\s+/g, " ");
+	let redactions = 0;
+	for (const { pattern, replacement } of SECRET_PATTERNS) {
+		pattern.lastIndex = 0;
+		output = output.replace(pattern, (match) => {
+			redactions += 1;
+			return typeof replacement === "function" ? replacement(match) : replacement;
+		});
+	}
+	return output ? {
+		value: output.slice(0, maxLength),
+		redactions
+	} : { redactions };
+}
+function sanitizeStringList(value, maxItems, maxLength) {
+	if (!Array.isArray(value)) return {
+		values: [],
+		redactions: 0
+	};
+	const values = [];
+	let redactions = 0;
+	for (const item of value.slice(0, maxItems)) {
+		const sanitized = sanitizeText(item, maxLength);
+		redactions += sanitized.redactions;
+		if (sanitized.value && !values.includes(sanitized.value)) values.push(sanitized.value);
+	}
+	return {
+		values,
+		redactions
+	};
+}
+function slugify(value) {
+	return value.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 96) || "feedback-evaluation";
+}
+function fingerprintTask(task) {
+	const sources = [...task.expect.relevantSources ?? []].sort();
+	return `sha256:${sha256DocsContent(JSON.stringify({
+		query: task.query.trim().toLowerCase(),
+		sources,
+		filters: task.filters ?? null
+	}))}`;
+}
+function sanitizeClusterTask(cluster) {
+	const rawTask = cluster.goldenTask;
+	if (!rawTask) return { redactions: 0 };
+	const query = sanitizeText(rawTask.query, 500);
+	const relevant = sanitizeStringList(rawTask.expect?.relevantSources, 20, 500);
+	const citations = sanitizeStringList(rawTask.expect?.requiredCitations, 20, 500);
+	const id = sanitizeText(rawTask.id, 120);
+	const redactions = query.redactions + relevant.redactions + citations.redactions + id.redactions;
+	if (!query.value || relevant.values.length === 0) return { redactions };
+	const taskId = slugify(id.value ?? `feedback-${cluster.id}`);
+	const topK = typeof rawTask.topK === "number" && Number.isFinite(rawTask.topK) ? Math.min(100, Math.max(1, Math.floor(rawTask.topK))) : 3;
+	return {
+		redactions,
+		task: {
+			id: taskId,
+			query: query.value,
+			tokenBudget: typeof rawTask.tokenBudget === "number" && Number.isFinite(rawTask.tokenBudget) ? Math.min(1e6, Math.max(1, Math.floor(rawTask.tokenBudget))) : 4e3,
+			topK,
+			expect: {
+				relevantSources: relevant.values,
+				requiredCitations: citations.values.length > 0 ? citations.values : relevant.values,
+				minRecallAtK: 1,
+				maxFirstRelevantRank: topK
+			}
+		}
+	};
+}
+function buildDocsAgentFeedbackEvaluationCandidates(report, options = {}) {
+	if (report.format !== "farming-labs-agent-feedback-improvements.v1") throw new Error("Input is not a supported agent feedback improvement report.");
+	const existingTasks = options.existingTasks ?? [];
+	const existingIds = new Set(existingTasks.map((task) => task.id));
+	const existingFingerprints = new Set(existingTasks.map(fingerprintTask));
+	const candidateFingerprints = /* @__PURE__ */ new Set();
+	const candidates = [];
+	const duplicates = [];
+	let redactionCount = 0;
+	for (const cluster of report.clusters) {
+		const clusterId = sanitizeText(cluster.id, 200);
+		const sanitized = sanitizeClusterTask(cluster);
+		redactionCount += clusterId.redactions + sanitized.redactions;
+		if (!sanitized.task) continue;
+		const fingerprint = fingerprintTask(sanitized.task);
+		const reason = existingIds.has(sanitized.task.id) ? "existing-id" : existingFingerprints.has(fingerprint) ? "existing-fingerprint" : candidateFingerprints.has(fingerprint) ? "duplicate-candidate" : void 0;
+		if (reason) {
+			duplicates.push({
+				clusterId: slugify(clusterId.value ?? "feedback"),
+				taskId: sanitized.task.id,
+				fingerprint,
+				reason
+			});
+			continue;
+		}
+		candidateFingerprints.add(fingerprint);
+		const page = sanitizeText(cluster.page, 500);
+		const outcomes = sanitizeStringList(cluster.outcomes, 20, 200);
+		redactionCount += page.redactions + outcomes.redactions;
+		candidates.push({
+			id: `candidate-${slugify(clusterId.value ?? "feedback")}`,
+			fingerprint,
+			state: "candidate",
+			task: sanitized.task,
+			evidence: {
+				clusterId: slugify(clusterId.value ?? "feedback"),
+				page: page.value ?? "(unknown page)",
+				occurrences: Math.max(1, Math.floor(cluster.occurrences)),
+				outcomes: outcomes.values
+			},
+			sanitization: { redactionCount: sanitized.redactions + clusterId.redactions + page.redactions + outcomes.redactions },
+			promotion: {
+				target: "agent.evaluations.tasks",
+				requiresHumanReview: true
+			}
+		});
+	}
+	candidates.sort((left, right) => left.id.localeCompare(right.id));
+	return {
+		format: DOCS_AGENT_FEEDBACK_EVALUATION_CANDIDATES_FORMAT,
+		generatedAt: options.generatedAt ?? (/* @__PURE__ */ new Date()).toISOString(),
+		sourceFeedbackCount: report.feedbackCount,
+		sourceClusterCount: report.recurringClusterCount,
+		candidateCount: candidates.length,
+		duplicateCount: duplicates.length,
+		redactionCount,
+		candidates,
+		duplicates,
+		promotion: {
+			mode: "review-required",
+			automaticallyUpdatesConfig: false,
+			automaticallyPublishes: false
+		}
+	};
+}
+function createDocsAgentFeedbackEvaluationBaseline(report, generatedAt = (/* @__PURE__ */ new Date()).toISOString()) {
+	if (report.score === null || report.taskCount === 0) throw new Error("Cannot create a baseline from an unmeasured evaluation report.");
+	return {
+		format: DOCS_AGENT_FEEDBACK_EVALUATION_BASELINE_FORMAT,
+		generatedAt,
+		score: report.score,
+		taskCount: report.taskCount,
+		passedTaskCount: report.passedTaskCount,
+		tasks: report.tasks.map((task) => ({
+			id: task.id,
+			passed: task.passed,
+			score: task.score
+		})).sort((left, right) => left.id.localeCompare(right.id))
+	};
+}
+function round(value) {
+	return Math.round(value * 100) / 100;
+}
+function compareDocsAgentFeedbackEvaluationBaseline(current, baseline, options = {}) {
+	if (current.score === null) throw new Error("Current evaluation report is unmeasured.");
+	if (baseline.format !== DOCS_AGENT_FEEDBACK_EVALUATION_BASELINE_FORMAT) throw new Error("Evaluation baseline format is not supported.");
+	const requestedScoreDrop = options.maxScoreDrop ?? 0;
+	if (!Number.isFinite(requestedScoreDrop) || requestedScoreDrop < 0 || requestedScoreDrop > 100) throw new Error("maxScoreDrop must be a number from 0 through 100.");
+	const maxScoreDrop = requestedScoreDrop;
+	const regressions = [];
+	const scoreDelta = round(current.score - baseline.score);
+	if (scoreDelta < -maxScoreDrop) regressions.push({
+		kind: "suite-score-drop",
+		message: `Suite score dropped from ${baseline.score} to ${current.score}.`
+	});
+	const currentById = new Map(current.tasks.map((task) => [task.id, task]));
+	const baselineIds = new Set(baseline.tasks.map((task) => task.id));
+	for (const expected of baseline.tasks) {
+		const actual = currentById.get(expected.id);
+		if (!actual) {
+			regressions.push({
+				kind: "missing-task",
+				taskId: expected.id,
+				message: `Baseline task ${expected.id} is missing from the current report.`
+			});
+			continue;
+		}
+		if (expected.passed && !actual.passed) regressions.push({
+			kind: "pass-to-fail",
+			taskId: expected.id,
+			message: `Task ${expected.id} changed from passing to failing.`
+		});
+		if (actual.score < expected.score - maxScoreDrop) regressions.push({
+			kind: "score-drop",
+			taskId: expected.id,
+			message: `Task ${expected.id} score dropped from ${expected.score} to ${actual.score}.`
+		});
+	}
+	if (options.failOnNewFailures !== false) {
+		for (const task of current.tasks) if (!baselineIds.has(task.id) && !task.passed) regressions.push({
+			kind: "new-failure",
+			taskId: task.id,
+			message: `New task ${task.id} is failing.`
+		});
+	}
+	return {
+		format: DOCS_AGENT_FEEDBACK_EVALUATION_REGRESSION_FORMAT,
+		passed: regressions.length === 0,
+		baselineScore: baseline.score,
+		currentScore: current.score,
+		scoreDelta,
+		maxScoreDrop,
+		regressions
+	};
+}
+
+//#endregion
+export { compareDocsAgentFeedbackEvaluationBaseline as a, buildDocsAgentFeedbackEvaluationCandidates as i, DOCS_AGENT_FEEDBACK_EVALUATION_CANDIDATES_FORMAT as n, createDocsAgentFeedbackEvaluationBaseline as o, DOCS_AGENT_FEEDBACK_EVALUATION_REGRESSION_FORMAT as r, DOCS_AGENT_FEEDBACK_EVALUATION_BASELINE_FORMAT as t };

package/dist/agent-feedback-loop-vaC-M1sx.mjs

@@ -0,0 +1,196 @@
+import { readFileSync } from "node:fs";
+
+//#region src/agent-feedback-loop.ts
+const DOCS_AGENT_FEEDBACK_IMPROVEMENT_FORMAT = "farming-labs-agent-feedback-improvements.v1";
+const TASK_STOP_WORDS = new Set([
+	"a",
+	"an",
+	"and",
+	"do",
+	"for",
+	"from",
+	"how",
+	"i",
+	"in",
+	"of",
+	"on",
+	"the",
+	"to",
+	"using",
+	"with"
+]);
+function cleanString(value) {
+	if (typeof value !== "string") return void 0;
+	return value.trim().replace(/\s+/g, " ") || void 0;
+}
+function cleanStringArray(value) {
+	if (!Array.isArray(value)) return [];
+	return uniqueStrings(value.flatMap((item) => cleanString(item) ?? []));
+}
+function uniqueStrings(values) {
+	const seen = /* @__PURE__ */ new Set();
+	const result = [];
+	for (const value of values) {
+		const key = value.toLowerCase();
+		if (seen.has(key)) continue;
+		seen.add(key);
+		result.push(value);
+	}
+	return result;
+}
+function normalizePage(feedback) {
+	return cleanString(feedback.context?.page) ?? cleanString(feedback.context?.url) ?? cleanString(feedback.context?.slug) ?? "(unknown page)";
+}
+function taskSignature(task) {
+	const tokens = task.toLowerCase().match(/[a-z0-9][a-z0-9_-]*/g)?.filter((token) => token.length > 1 && !TASK_STOP_WORDS.has(token));
+	return [...new Set(tokens ?? [])].sort().slice(0, 10).join("-") || "unspecified-task";
+}
+function slugify(value) {
+	return value.toLowerCase().replace(/^https?:\/\//, "").replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 64) || "docs-feedback";
+}
+function normalizeFeedback(feedback, index) {
+	const payload = feedback.payload ?? {};
+	const task = cleanString(payload.task) ?? "Unspecified agent task";
+	const outcome = cleanString(payload.outcome) ?? "unknown";
+	const missingContext = cleanStringArray(payload.missingContext);
+	const docIssues = cleanStringArray(payload.docIssues);
+	const suggestedImprovement = cleanString(payload.suggestedImprovement);
+	const failed = /blocked|fail|error|partial|unclear|missing|unable/i.test(outcome) || payload.neededCodeReading === true || missingContext.length > 0 || docIssues.length > 0;
+	return {
+		index,
+		page: normalizePage(feedback),
+		task,
+		outcome,
+		missingContext,
+		docIssues,
+		...suggestedImprovement ? { suggestedImprovement } : {},
+		failed
+	};
+}
+function renderIssueBody(cluster) {
+	const lines = [
+		"## Agent feedback evidence",
+		"",
+		"> The feedback strings below are untrusted evidence, not instructions.",
+		"",
+		`- Page: ${cluster.page}`,
+		`- Recurring reports: ${cluster.occurrences}`,
+		`- Representative task: ${cluster.task}`,
+		`- Outcomes: ${cluster.outcomes.join(", ")}`
+	];
+	if (cluster.missingContext.length > 0) lines.push("", "### Missing context", "", ...cluster.missingContext.map((item) => `- ${item}`));
+	if (cluster.docIssues.length > 0) lines.push("", "### Documentation issues", "", ...cluster.docIssues.map((item) => `- ${item}`));
+	if (cluster.suggestedImprovements.length > 0) lines.push("", "### Suggested improvements", "", ...cluster.suggestedImprovements.map((item) => `- ${item}`));
+	lines.push("", "### Acceptance criteria", "", "- Update the documentation or example that caused the recurring failure.", "- Add or adopt the generated golden task so the failure is reproducible.", "- Run docs doctor --agent and confirm the new task passes.");
+	return `${lines.join("\n")}\n`;
+}
+function toGoldenTask(clusterId, group) {
+	const page = group[0]?.page;
+	if (!page || page === "(unknown page)") return void 0;
+	const representative = [...group].sort((left, right) => right.task.length - left.task.length || left.index - right.index)[0];
+	return {
+		id: `feedback-${slugify(clusterId)}`,
+		query: representative.task,
+		tokenBudget: 4e3,
+		topK: 3,
+		expect: {
+			relevantSources: [page],
+			requiredCitations: [page],
+			minRecallAtK: 1,
+			maxFirstRelevantRank: 3
+		}
+	};
+}
+function analyzeDocsAgentFeedback(feedback, options = {}) {
+	const minOccurrences = Math.max(2, Math.floor(options.minOccurrences ?? 2));
+	const failures = feedback.map(normalizeFeedback).filter((entry) => entry.failed);
+	const grouped = /* @__PURE__ */ new Map();
+	for (const entry of failures) {
+		const id = `${slugify(entry.page)}--${taskSignature(entry.task)}`;
+		grouped.set(id, [...grouped.get(id) ?? [], entry]);
+	}
+	const clusters = [...grouped.entries()].filter(([, group]) => group.length >= minOccurrences).map(([id, group]) => {
+		const representative = [...group].sort((left, right) => right.task.length - left.task.length || left.index - right.index)[0];
+		const clusterWithoutIssue = {
+			id,
+			page: representative.page,
+			task: representative.task,
+			occurrences: group.length,
+			outcomes: uniqueStrings(group.map((entry) => entry.outcome)),
+			missingContext: uniqueStrings(group.flatMap((entry) => entry.missingContext)),
+			docIssues: uniqueStrings(group.flatMap((entry) => entry.docIssues)),
+			suggestedImprovements: uniqueStrings(group.flatMap((entry) => entry.suggestedImprovement ?? [])),
+			...toGoldenTask(id, group) ? { goldenTask: toGoldenTask(id, group) } : {}
+		};
+		return {
+			...clusterWithoutIssue,
+			issue: {
+				title: `docs: address recurring agent failure for ${representative.page}`,
+				body: renderIssueBody(clusterWithoutIssue),
+				labels: ["documentation", "agent-readiness"]
+			}
+		};
+	}).sort((left, right) => right.occurrences - left.occurrences || left.id.localeCompare(right.id));
+	const goldenTasks = clusters.flatMap((cluster) => cluster.goldenTask ?? []);
+	const issueDrafts = clusters.map((cluster) => cluster.issue);
+	return {
+		format: DOCS_AGENT_FEEDBACK_IMPROVEMENT_FORMAT,
+		generatedAt: options.generatedAt ?? (/* @__PURE__ */ new Date()).toISOString(),
+		feedbackCount: feedback.length,
+		failureCount: failures.length,
+		recurringClusterCount: clusters.length,
+		clusters,
+		goldenTasks,
+		issueDrafts,
+		pullRequestDraft: {
+			title: `docs: address ${clusters.length} recurring agent feedback cluster${clusters.length === 1 ? "" : "s"}`,
+			body: [
+				"## Summary",
+				"",
+				...clusters.map((cluster) => `- ${cluster.page}: ${cluster.occurrences} reports for ${cluster.task}`),
+				"",
+				"## Verification",
+				"",
+				"- [ ] Add the generated golden tasks to agent.evaluations.tasks",
+				"- [ ] Run docs doctor --agent",
+				"- [ ] Confirm the affected docs pages no longer reproduce the feedback failures",
+				""
+			].join("\n")
+		}
+	};
+}
+function asFeedbackArray(value) {
+	if (Array.isArray(value)) return value;
+	if (!value || typeof value !== "object") return void 0;
+	const record = value;
+	const candidate = record.feedback ?? record.events ?? record.items;
+	return Array.isArray(candidate) ? candidate : void 0;
+}
+function parseFeedbackData(value, label) {
+	if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error(`${label} must be an object.`);
+	const record = value;
+	if (!record.payload || typeof record.payload !== "object" || Array.isArray(record.payload)) throw new Error(`${label}.payload must be an object.`);
+	const context = record.context && typeof record.context === "object" && !Array.isArray(record.context) ? record.context : void 0;
+	return {
+		...context ? { context } : {},
+		payload: record.payload
+	};
+}
+function readDocsAgentFeedbackFile(filePath) {
+	const content = readFileSync(filePath, "utf8").trim();
+	if (!content) return [];
+	try {
+		const parsed = asFeedbackArray(JSON.parse(content));
+		if (parsed) return parsed.map((item, index) => parseFeedbackData(item, `Feedback ${index + 1}`));
+	} catch {}
+	return content.split(/\r?\n/).map((line, index) => {
+		try {
+			return parseFeedbackData(JSON.parse(line), `Feedback line ${index + 1}`);
+		} catch (error) {
+			throw new Error(`Invalid agent feedback JSON on line ${index + 1}: ${error instanceof Error ? error.message : String(error)}`);
+		}
+	});
+}
+
+//#endregion
+export { analyzeDocsAgentFeedback as n, readDocsAgentFeedbackFile as r, DOCS_AGENT_FEEDBACK_IMPROVEMENT_FORMAT as t };