@farming-labs/docs 0.2.120 → 0.2.122

package/dist/agent-detection.mjs

@@ -0,0 +1,83 @@
+//#region src/agent-detection.ts
+const DOCS_AI_AGENT_USER_AGENT_PATTERNS = [
+	"claudebot",
+	"claude-searchbot",
+	"claude-user",
+	"anthropic-ai",
+	"claude-web",
+	"chatgpt",
+	"gptbot",
+	"oai-searchbot",
+	"openai",
+	"gemini",
+	"bard",
+	"google-cloudvertexbot",
+	"google-extended",
+	"meta-externalagent",
+	"meta-externalfetcher",
+	"meta-webindexer",
+	"perplexity",
+	"youbot",
+	"you.com",
+	"deepseekbot",
+	"cursor",
+	"github-copilot",
+	"codeium",
+	"tabnine",
+	"sourcegraph",
+	"cohere-ai",
+	"bytespider",
+	"amazonbot",
+	"ai2bot",
+	"diffbot",
+	"omgili",
+	"omgilibot"
+];
+const DOCS_TRADITIONAL_BOT_USER_AGENT_PATTERNS = [
+	"googlebot",
+	"bingbot",
+	"yandexbot",
+	"baiduspider",
+	"duckduckbot",
+	"slurp",
+	"msnbot",
+	"facebot",
+	"twitterbot",
+	"linkedinbot",
+	"whatsapp",
+	"telegrambot",
+	"pingdom",
+	"uptimerobot",
+	"newrelic",
+	"datadog",
+	"statuspage",
+	"site24x7",
+	"applebot"
+];
+const DOCS_BOT_LIKE_USER_AGENT_PATTERN = /bot|agent|fetch|crawl|spider|search/i;
+const DOCS_BOT_LIKE_USER_AGENT_TERMS = [
+	"bot",
+	"agent",
+	"fetch",
+	"crawl",
+	"spider",
+	"search"
+];
+const DOCS_AI_AGENT_USER_AGENT_HEADER_PATTERN = buildDocsUserAgentHeaderPattern(DOCS_AI_AGENT_USER_AGENT_PATTERNS);
+const DOCS_TRADITIONAL_BOT_USER_AGENT_HEADER_PATTERN = buildDocsUserAgentHeaderPattern(DOCS_TRADITIONAL_BOT_USER_AGENT_PATTERNS);
+const DOCS_BOT_LIKE_USER_AGENT_HEADER_PATTERN = buildDocsUserAgentHeaderPattern(DOCS_BOT_LIKE_USER_AGENT_TERMS);
+function buildDocsUserAgentHeaderPattern(patterns) {
+	return `.*(?:${patterns.map(toCaseInsensitiveHeaderPattern).join("|")}).*`;
+}
+function toCaseInsensitiveHeaderPattern(value) {
+	return value.split("").map((char) => {
+		if (/^[a-z]$/i.test(char)) return `[${char.toLowerCase()}${char.toUpperCase()}]`;
+		return escapeRegex(char);
+	}).join("");
+}
+function escapeRegex(value) {
+	return value.replace(/[\\^$.*+?()[\]{}|]/g, "\\$&");
+}
+
+//#endregion
+export { DOCS_AI_AGENT_USER_AGENT_HEADER_PATTERN, DOCS_AI_AGENT_USER_AGENT_PATTERNS, DOCS_BOT_LIKE_USER_AGENT_HEADER_PATTERN, DOCS_BOT_LIKE_USER_AGENT_PATTERN, DOCS_TRADITIONAL_BOT_USER_AGENT_HEADER_PATTERN, DOCS_TRADITIONAL_BOT_USER_AGENT_PATTERNS };

package/dist/agent-evals.d.mts

@@ -0,0 +1,224 @@
+import { DocsAgentEvaluationAnswerProvider, DocsAgentEvaluationSurface, DocsAgentGoldenCoverageApplicability, DocsAgentGoldenExpectedExample, DocsAgentGoldenQueryVariant, DocsAgentGoldenTask, DocsAgentGoldenTaskExpectation, DocsAgentGoldenTaskFilters, DocsCodeBlocksValidateConfig, DocsSearchConfig } from "./types.mjs";
+import { DocsCodeBlockValidationResult } from "./code-blocks.mjs";
+import { DocsMcpPage } from "./mcp.mjs";
+
+//#region src/agent-evals.d.ts
+type DocsGoldenEvaluationStatus = "unmeasured" | "passed" | "failed";
+type DocsGoldenEvaluationCoverageStatus = "measured" | "partially-measured" | "unmeasured" | "not-applicable";
+type DocsGoldenTaskFilters = DocsAgentGoldenTaskFilters;
+type DocsGoldenExpectedExample = DocsAgentGoldenExpectedExample;
+type DocsGoldenTaskExpectation = DocsAgentGoldenTaskExpectation;
+type DocsGoldenTask = DocsAgentGoldenTask;
+interface DocsGoldenRetrievedSource {
+  rank: number;
+  url: string;
+  title: string;
+  framework?: string;
+  version?: string;
+  package?: string[];
+  tags?: string[];
+  utf8Bytes: number;
+  relevant: boolean;
+  truncated: boolean;
+}
+interface DocsGoldenRetrievalMetrics {
+  expectedRelevant: number;
+  retrievedRelevant: number;
+  recallAtK: number;
+  firstRelevantRank: number | null;
+  reciprocalRank: number;
+  forbiddenSources: string[];
+  passed: boolean;
+}
+interface DocsGoldenCitationMetrics {
+  /** Where the citation evidence came from. Context/results are not model-answer citations. */
+  evidence: "context" | "results";
+  expected: string[];
+  actual: string[];
+  missing: string[];
+  unexpected: string[];
+  precision: number;
+  recall: number;
+  /** True when every rendered Source line maps one-to-one to a returned source record. */
+  integrity: boolean;
+  passed: boolean;
+}
+interface DocsGoldenSelectionMetrics {
+  requestedFramework?: string;
+  requestedVersion?: string;
+  requestedPackage?: string | readonly string[];
+  requestedTags?: string | readonly string[];
+  requestedLocale?: string;
+  expectedFramework?: string;
+  expectedVersion?: string;
+  expectedPackage?: string | readonly string[];
+  expectedTags?: string | readonly string[];
+  expectedLocale?: string;
+  firstFrameworkMatchRank: number | null;
+  firstVersionMatchRank: number | null;
+  firstPackageMatchRank?: number | null;
+  firstTagsMatchRank?: number | null;
+  firstLocaleMatchRank: number | null;
+  conflictingSources: string[];
+  ambiguousSources: string[];
+  passed: boolean;
+}
+interface DocsGoldenExampleResult {
+  expected: DocsGoldenExpectedExample;
+  matchedId?: string;
+  source?: string;
+  verification: "present" | "syntax" | "execute";
+  status: "passed" | "failed" | "skipped";
+  reason?: string;
+  syntaxValid: boolean;
+  executed: boolean;
+  executionStatus?: DocsCodeBlockValidationResult["status"];
+  /** @deprecated Static/planner executability signal retained for report compatibility. */
+  executable: boolean;
+  passed: boolean;
+}
+interface DocsGoldenExampleMetrics {
+  expected: number;
+  matched: number;
+  syntaxValid: number;
+  executed: number;
+  /** @deprecated Count of statically/planner-executable examples retained for compatibility. */
+  executable: number;
+  results: DocsGoldenExampleResult[];
+  passed: boolean;
+}
+interface DocsGoldenUsageMetrics {
+  budgetUnit: "utf8-bytes";
+  tokenBudget: number;
+  usedUtf8Bytes: number;
+  remainingUtf8Bytes: number;
+  estimatedTokens: number;
+  /** Exact conservative upper bound: one token per UTF-8 byte. */
+  conservativeTokenUpperBound: number;
+  usefulUtf8Bytes: number;
+  usefulByteRatio: number;
+  truncated: boolean;
+  withinBudget: boolean;
+  passed: boolean;
+}
+interface DocsGoldenAnswerMetrics {
+  evidence: "answer";
+  expected: boolean;
+  provided: boolean;
+  textIncludesMissing: string[];
+  textExcludesPresent: string[];
+  citations: string[];
+  requiredCitations: string[];
+  missingCitations: string[];
+  unexpectedCitations: string[];
+  forbiddenCitations: string[];
+  passed: boolean;
+}
+type DocsGoldenSafetyCaseKind = "prompt-injection" | "poisoned-citations" | "authenticated-content" | "freshness" | "framework-version-conflict" | "deleted-section-tombstones";
+interface DocsGoldenSafetyCaseResult {
+  kind: DocsGoldenSafetyCaseKind;
+  passed: boolean;
+  issues: string[];
+}
+interface DocsGoldenQueryVariantResult {
+  kind: DocsAgentGoldenQueryVariant["kind"];
+  query: string;
+  passed: boolean;
+  score: number;
+  firstRelevantRank: number | null;
+  citationIntegrity: boolean;
+  issues: string[];
+}
+interface DocsGoldenSafetyMetrics {
+  expected: boolean;
+  passed: boolean;
+  cases: DocsGoldenSafetyCaseResult[];
+  queryVariants: DocsGoldenQueryVariantResult[];
+}
+interface DocsGoldenTaskReport {
+  id: string;
+  query: string;
+  surface: DocsAgentEvaluationSurface;
+  provider: string;
+  status: "passed" | "failed";
+  passed: boolean;
+  score: number;
+  context: string;
+  sources: DocsGoldenRetrievedSource[];
+  retrieval: DocsGoldenRetrievalMetrics;
+  citations: DocsGoldenCitationMetrics;
+  answer: DocsGoldenAnswerMetrics;
+  safety: DocsGoldenSafetyMetrics;
+  selection: DocsGoldenSelectionMetrics;
+  examples: DocsGoldenExampleMetrics;
+  usage: DocsGoldenUsageMetrics;
+  coverageApplicability: {
+    executableExamples: DocsAgentGoldenCoverageApplicability;
+  };
+  issues: string[];
+}
+interface DocsGoldenTasksReport {
+  status: DocsGoldenEvaluationStatus;
+  passed: boolean | null;
+  /** Compatibility alias for quality.score. */
+  score: number | null;
+  taskCount: number;
+  passedTaskCount: number;
+  failedTaskCount: number;
+  /** Quality across only the expectations configured by each task. */
+  quality: DocsGoldenEvaluationQuality;
+  /** Coverage is intentionally separate so absent evaluation dimensions cannot look perfect. */
+  coverage: DocsGoldenEvaluationCoverage;
+  tasks: DocsGoldenTaskReport[];
+}
+interface DocsGoldenEvaluationQuality {
+  status: DocsGoldenEvaluationStatus;
+  passed: boolean | null;
+  score: number | null;
+  taskCount: number;
+  passedTaskCount: number;
+  failedTaskCount: number;
+}
+interface DocsGoldenEvaluationDimensionCoverage {
+  status: DocsGoldenEvaluationCoverageStatus;
+  measuredTaskCount: number;
+  applicableTaskCount: number;
+  notApplicableTaskCount: number;
+  totalTaskCount: number;
+  coveragePercent: number;
+}
+interface DocsGoldenEvaluationCoverage {
+  status: DocsGoldenEvaluationCoverageStatus;
+  measuredTaskDimensions: number;
+  applicableTaskDimensions: number;
+  notApplicableTaskDimensions: number;
+  totalTaskDimensions: number;
+  coveragePercent: number;
+  dimensions: {
+    safety: DocsGoldenEvaluationDimensionCoverage;
+    answerQuality: DocsGoldenEvaluationDimensionCoverage;
+    executableExamples: DocsGoldenEvaluationDimensionCoverage;
+  };
+}
+interface RunDocsGoldenTasksOptions {
+  /** Default evaluation surface. Tasks may override this with `task.surface`. */
+  surface?: DocsAgentEvaluationSurface;
+  /** Search configuration used by configured-search and ask-ai-context. */
+  search?: boolean | DocsSearchConfig;
+  /** Production-resolved Ask AI retrieval config, including `ai.useMcp`. */
+  askAISearch?: boolean | DocsSearchConfig;
+  siteTitle?: string;
+  baseUrl?: string;
+  rootDir?: string;
+  /** Permit external configured search or HTTP answer calls. @default false */
+  allowNetwork?: boolean;
+  /** Per-attempt configured-search/Ask-AI retrieval timeout in milliseconds. @default 30000 */
+  searchTimeoutMs?: number;
+  /** Project code-block validation config used only by explicit execute expectations. */
+  codeBlocksValidate?: boolean | DocsCodeBlocksValidateConfig;
+  /** Optional actual-answer runner. No model or HTTP request is made when omitted. */
+  answer?: DocsAgentEvaluationAnswerProvider;
+}
+declare function runDocsGoldenTasks(pages: readonly DocsMcpPage[], tasks: readonly DocsGoldenTask[] | undefined, options?: RunDocsGoldenTasksOptions): Promise<DocsGoldenTasksReport>;
+//#endregion
+export { DocsGoldenAnswerMetrics, DocsGoldenCitationMetrics, DocsGoldenEvaluationCoverage, DocsGoldenEvaluationCoverageStatus, DocsGoldenEvaluationDimensionCoverage, DocsGoldenEvaluationQuality, DocsGoldenEvaluationStatus, DocsGoldenExampleMetrics, DocsGoldenExampleResult, DocsGoldenExpectedExample, DocsGoldenQueryVariantResult, DocsGoldenRetrievalMetrics, DocsGoldenRetrievedSource, DocsGoldenSafetyCaseKind, DocsGoldenSafetyCaseResult, DocsGoldenSafetyMetrics, DocsGoldenSelectionMetrics, DocsGoldenTask, DocsGoldenTaskExpectation, DocsGoldenTaskFilters, DocsGoldenTaskReport, DocsGoldenTasksReport, DocsGoldenUsageMetrics, RunDocsGoldenTasksOptions, runDocsGoldenTasks };