@x12i/ai-profiles 4.3.0 → 4.4.1

package/dist/evidence/patch.d.ts

@@ -0,0 +1,8 @@
+import type { FinalProfileModelMappingPatch } from "../finalProfileModelMappingPatch.js";
+import type { NormalizedModel, PatchValidationError, ProfileChoiceKey, ProfileModelRecommendation } from "./types.js";
+export declare function buildPatchFromApprovedRecommendations(recommendations: ProfileModelRecommendation[], approvedKeys: Set<ProfileChoiceKey | string>): FinalProfileModelMappingPatch;
+export declare function validatePatchAgainstCatalog(args: {
+    patch: FinalProfileModelMappingPatch;
+    models: NormalizedModel[];
+}): PatchValidationError[];
+//# sourceMappingURL=patch.d.ts.map

package/dist/evidence/patch.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"patch.d.ts","sourceRoot":"","sources":["../../src/evidence/patch.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,qCAAqC,CAAC;AACzF,OAAO,KAAK,EACV,eAAe,EACf,oBAAoB,EACpB,gBAAgB,EAChB,0BAA0B,EAC3B,MAAM,YAAY,CAAC;AAEpB,wBAAgB,qCAAqC,CACnD,eAAe,EAAE,0BAA0B,EAAE,EAC7C,YAAY,EAAE,GAAG,CAAC,gBAAgB,GAAG,MAAM,CAAC,GAC3C,6BAA6B,CAc/B;AAED,wBAAgB,2BAA2B,CAAC,IAAI,EAAE;IAChD,KAAK,EAAE,6BAA6B,CAAC;IACrC,MAAM,EAAE,eAAe,EAAE,CAAC;CAC3B,GAAG,oBAAoB,EAAE,CAuCzB"}

package/dist/evidence/patch.js

@@ -0,0 +1,50 @@
+export function buildPatchFromApprovedRecommendations(recommendations, approvedKeys) {
+    const patch = {};
+    for (const rec of recommendations) {
+        if (!approvedKeys.has(rec.profileChoiceKey))
+            continue;
+        patch[rec.profileChoiceKey] = {
+            provider: rec.recommended.provider,
+            modelId: rec.recommended.modelId,
+            modelStatus: "verified",
+            reason: rec.reason,
+        };
+    }
+    return patch;
+}
+export function validatePatchAgainstCatalog(args) {
+    const { patch, models } = args;
+    const modelSet = new Set(models.map((m) => `${m.identity.provider}/${m.identity.modelId}`));
+    const errors = [];
+    const seen = new Set();
+    for (const [key, value] of Object.entries(patch)) {
+        if (seen.has(key)) {
+            errors.push({
+                profileChoiceKey: key,
+                message: "Duplicate conflicting patch key",
+            });
+        }
+        seen.add(key);
+        const lookup = `${value.provider}/${value.modelId}`;
+        if (!modelSet.has(lookup) && value.provider !== "local") {
+            errors.push({
+                profileChoiceKey: key,
+                message: `Model not found in active catalog: ${lookup}`,
+            });
+        }
+        if (value.modelStatus !== "verified") {
+            errors.push({
+                profileChoiceKey: key,
+                message: "Functional patch must use modelStatus=verified",
+            });
+        }
+        if (value.modelId.includes("latest")) {
+            errors.push({
+                profileChoiceKey: key,
+                message: "Do not use latest alias in final profile patch",
+            });
+        }
+    }
+    return errors;
+}
+//# sourceMappingURL=patch.js.map

package/dist/evidence/patch.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"patch.js","sourceRoot":"","sources":["../../src/evidence/patch.ts"],"names":[],"mappings":"AAQA,MAAM,UAAU,qCAAqC,CACnD,eAA6C,EAC7C,YAA4C;IAE5C,MAAM,KAAK,GAAkC,EAAE,CAAC;IAEhD,KAAK,MAAM,GAAG,IAAI,eAAe,EAAE,CAAC;QAClC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,gBAAgB,CAAC;YAAE,SAAS;QACtD,KAAK,CAAC,GAAG,CAAC,gBAAgB,CAAC,GAAG;YAC5B,QAAQ,EAAE,GAAG,CAAC,WAAW,CAAC,QAAQ;YAClC,OAAO,EAAE,GAAG,CAAC,WAAW,CAAC,OAAO;YAChC,WAAW,EAAE,UAAU;YACvB,MAAM,EAAE,GAAG,CAAC,MAAM;SACnB,CAAC;IACJ,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,2BAA2B,CAAC,IAG3C;IACC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;IAC/B,MAAM,QAAQ,GAAG,IAAI,GAAG,CACtB,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,QAAQ,IAAI,CAAC,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,CAClE,CAAC;IACF,MAAM,MAAM,GAA2B,EAAE,CAAC;IAC1C,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAE/B,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACjD,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAClB,MAAM,CAAC,IAAI,CAAC;gBACV,gBAAgB,EAAE,GAAG;gBACrB,OAAO,EAAE,iCAAiC;aAC3C,CAAC,CAAC;QACL,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAEd,MAAM,MAAM,GAAG,GAAG,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;QACpD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;YACxD,MAAM,CAAC,IAAI,CAAC;gBACV,gBAAgB,EAAE,GAAG;gBACrB,OAAO,EAAE,sCAAsC,MAAM,EAAE;aACxD,CAAC,CAAC;QACL,CAAC;QACD,IAAI,KAAK,CAAC,WAAW,KAAK,UAAU,EAAE,CAAC;YACrC,MAAM,CAAC,IAAI,CAAC;gBACV,gBAAgB,EAAE,GAAG;gBACrB,OAAO,EAAE,gDAAgD;aAC1D,CAAC,CAAC;QACL,CAAC;QACD,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;YACrC,MAAM,CAAC,IAAI,CAAC;gBACV,gBAAgB,EAAE,GAAG;gBACrB,OAAO,EAAE,gDAAgD;aAC1D,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/evidence/recommend.d.ts

@@ -0,0 +1,20 @@
+import type { ProviderId } from "../types.js";
+import type { BenchmarkSignal, InternalEvalResult, ModelIssueSignal, ModelTaskScore, NormalizedModel, ProfileModelRecommendation, TaskProfile } from "./types.js";
+export declare const profileChoiceToTaskProfile: Record<string, TaskProfile>;
+export declare const candidateProviderFamiliesByTask: Record<TaskProfile, ProviderId[]>;
+export declare function buildRecommendationReason(score: ModelTaskScore, current?: {
+    provider: ProviderId;
+    modelId: string;
+}): string;
+export declare function summarizeEvidence(evidence: Array<BenchmarkSignal | InternalEvalResult>): string[];
+export declare function recommendModelsForProfiles(args: {
+    models: NormalizedModel[];
+    benchmarkSignals: BenchmarkSignal[];
+    internalResults: InternalEvalResult[];
+    issues: ModelIssueSignal[];
+    currentMappings: Record<string, {
+        provider: ProviderId;
+        modelId: string;
+    }>;
+}): ProfileModelRecommendation[];
+//# sourceMappingURL=recommend.d.ts.map

package/dist/evidence/recommend.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"recommend.d.ts","sourceRoot":"","sources":["../../src/evidence/recommend.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAE9C,OAAO,KAAK,EACV,eAAe,EACf,kBAAkB,EAClB,gBAAgB,EAChB,cAAc,EACd,eAAe,EAEf,0BAA0B,EAC1B,WAAW,EACZ,MAAM,YAAY,CAAC;AAGpB,eAAO,MAAM,0BAA0B,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAqClE,CAAC;AAEF,eAAO,MAAM,+BAA+B,EAAE,MAAM,CAAC,WAAW,EAAE,UAAU,EAAE,CAoB3E,CAAC;AAIJ,wBAAgB,yBAAyB,CACvC,KAAK,EAAE,cAAc,EACrB,OAAO,CAAC,EAAE;IAAE,QAAQ,EAAE,UAAU,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,GAClD,MAAM,CAMR;AAED,wBAAgB,iBAAiB,CAC/B,QAAQ,EAAE,KAAK,CAAC,eAAe,GAAG,kBAAkB,CAAC,GACpD,MAAM,EAAE,CAOV;AAaD,wBAAgB,0BAA0B,CAAC,IAAI,EAAE;IAC/C,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,gBAAgB,EAAE,eAAe,EAAE,CAAC;IACpC,eAAe,EAAE,kBAAkB,EAAE,CAAC;IACtC,MAAM,EAAE,gBAAgB,EAAE,CAAC;IAC3B,eAAe,EAAE,MAAM,CACrB,MAAM,EACN;QAAE,QAAQ,EAAE,UAAU,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAC1C,CAAC;CACH,GAAG,0BAA0B,EAAE,CAsF/B"}

package/dist/evidence/recommend.js

@@ -0,0 +1,142 @@
+import { resolveFinalProfileModelMappingPatch } from "../finalProfileModelMappingPatch.js";
+import { scoreModelForTask } from "./scoreModelForTask.js";
+export const profileChoiceToTaskProfile = {
+    "cheap/default": "cheap",
+    "cheap/google_floor": "cheap",
+    "routing/default": "routing",
+    "extraction/default": "extraction",
+    "json/default": "json",
+    "json/google_json": "json",
+    "json/deepseek_json": "json",
+    "fast/default": "fast",
+    "balanced/default": "balanced",
+    "value/default": "value",
+    "deep/default": "deep",
+    "deep/openai_deep": "deep",
+    "deep/google_deep": "deep",
+    "deep/deepseek_pro": "deep",
+    "pro/default": "pro",
+    "pro/claude_pro": "pro",
+    "agentic/default": "agentic",
+    "agentic/openai_agentic": "agentic",
+    "agentic/deepseek_agentic": "agentic",
+    "research/default": "research",
+    "research/openai_research": "research",
+    "research/google_research": "research",
+    "cyber/default": "cyber",
+    "cyber/deep_forensics": "cyber",
+    "cyber/gpt_forensics": "cyber",
+    "cyber/log_analysis": "cyber",
+    "coding/default": "coding",
+    "coding/deepseek_pro": "coding",
+    "coding/claude_coding": "coding",
+    "coding/openai_coding": "coding",
+    "content/default": "content",
+    "content/pseo_bulk": "content",
+    "content/brand_voice": "content",
+    "sum/default": "summarization",
+    "sum/mega_context": "long_context",
+    "vol/default": "vol",
+};
+export const candidateProviderFamiliesByTask = {
+    cheap: ["deepseek", "google", "openai", "minimax"],
+    routing: ["google", "openai", "anthropic", "deepseek"],
+    extraction: ["google", "openai", "anthropic", "deepseek"],
+    json: ["openai", "google", "deepseek", "anthropic"],
+    fast: ["openai", "google", "xai", "anthropic", "minimax"],
+    balanced: ["openai", "anthropic", "deepseek", "google", "minimax"],
+    value: ["deepseek", "minimax", "google", "qwen"],
+    deep: ["anthropic", "openai", "deepseek", "google", "xai"],
+    pro: ["openai", "anthropic"],
+    agentic: ["anthropic", "openai", "deepseek", "minimax", "google"],
+    research: ["anthropic", "openai", "google", "perplexity"],
+    cyber: ["deepseek", "openai", "xai", "google", "anthropic"],
+    coding: ["deepseek", "anthropic", "openai", "qwen", "moonshotai", "minimax"],
+    content: ["anthropic", "openai", "google"],
+    summarization: ["anthropic", "google", "deepseek"],
+    long_context: ["google", "deepseek", "xai"],
+    local: ["local"],
+    vol: ["deepseek", "google", "openai", "minimax"],
+};
+const POLICY_PRIOR_BOOST = 0.35;
+export function buildRecommendationReason(score, current) {
+    const target = `${score.model.provider}/${score.model.modelId}`;
+    if (!current) {
+        return `Recommended ${target} for ${score.taskProfile} based on score ${score.finalScore.toFixed(3)}.`;
+    }
+    return `Recommended ${target} over ${current.provider}/${current.modelId} for ${score.taskProfile}; score=${score.finalScore.toFixed(3)}, estimatedCostUsd=${score.estimatedCostUsd.toFixed(6)}.`;
+}
+export function summarizeEvidence(evidence) {
+    return evidence.slice(0, 6).map((item) => {
+        if ("metric" in item) {
+            return `${item.source}:${item.metric}=${item.value}`;
+        }
+        return `internal:${item.taskId}:quality=${item.qualityScore ?? item.passRate ?? "n/a"}`;
+    });
+}
+function hasInternalEvidence(internalResults, model) {
+    return internalResults.some((r) => r.provider === model.identity.provider &&
+        r.modelId === model.identity.modelId);
+}
+export function recommendModelsForProfiles(args) {
+    const { models, benchmarkSignals, internalResults, issues, currentMappings, } = args;
+    const policyPatch = resolveFinalProfileModelMappingPatch();
+    const recommendations = [];
+    for (const [profileChoiceKey, taskProfile] of Object.entries(profileChoiceToTaskProfile)) {
+        const allowedProviders = new Set(candidateProviderFamiliesByTask[taskProfile]);
+        const candidates = models.filter((m) => allowedProviders.has(m.identity.provider));
+        const policy = policyPatch[profileChoiceKey];
+        const scored = candidates
+            .map((model) => {
+            const base = scoreModelForTask({
+                model,
+                taskProfile,
+                benchmarkSignals,
+                internalResults,
+                issues,
+            });
+            if (base.disqualifiedReasons.length > 0)
+                return base;
+            const isPolicyPrior = policy &&
+                model.identity.provider === policy.provider &&
+                model.identity.modelId === policy.modelId;
+            const lacksInternal = !hasInternalEvidence(internalResults, model);
+            const boosted = isPolicyPrior && lacksInternal
+                ? Math.min(1, base.finalScore + POLICY_PRIOR_BOOST)
+                : base.finalScore;
+            return { ...base, finalScore: boosted };
+        })
+            .filter((s) => s.disqualifiedReasons.length === 0)
+            .sort((a, b) => b.finalScore - a.finalScore);
+        const policyHit = policy
+            ? scored.find((s) => s.model.provider === policy.provider &&
+                s.model.modelId === policy.modelId)
+            : undefined;
+        const best = policyHit ?? scored[0];
+        if (!best)
+            continue;
+        const current = currentMappings[profileChoiceKey];
+        const changed = !current ||
+            current.provider !== best.model.provider ||
+            current.modelId !== best.model.modelId;
+        recommendations.push({
+            profileChoiceKey: profileChoiceKey,
+            current,
+            recommended: {
+                provider: best.model.provider,
+                modelId: best.model.modelId,
+                modelStatus: "verified",
+            },
+            taskProfile,
+            score: best.finalScore,
+            estimatedCostUsd: best.estimatedCostUsd,
+            reason: buildRecommendationReason(best, current),
+            confidence: best.benchmarkConfidenceScore,
+            evidenceSummary: summarizeEvidence(best.evidence),
+            issuesSummary: best.issues.map((i) => `${i.issue}: ${i.severity}`),
+            requiresHumanApproval: changed,
+        });
+    }
+    return recommendations;
+}
+//# sourceMappingURL=recommend.js.map

package/dist/evidence/recommend.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"recommend.js","sourceRoot":"","sources":["../../src/evidence/recommend.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,oCAAoC,EAAE,MAAM,qCAAqC,CAAC;AAW3F,OAAO,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAE3D,MAAM,CAAC,MAAM,0BAA0B,GAAgC;IACrE,eAAe,EAAE,OAAO;IACxB,oBAAoB,EAAE,OAAO;IAC7B,iBAAiB,EAAE,SAAS;IAC5B,oBAAoB,EAAE,YAAY;IAClC,cAAc,EAAE,MAAM;IACtB,kBAAkB,EAAE,MAAM;IAC1B,oBAAoB,EAAE,MAAM;IAC5B,cAAc,EAAE,MAAM;IACtB,kBAAkB,EAAE,UAAU;IAC9B,eAAe,EAAE,OAAO;IACxB,cAAc,EAAE,MAAM;IACtB,kBAAkB,EAAE,MAAM;IAC1B,kBAAkB,EAAE,MAAM;IAC1B,mBAAmB,EAAE,MAAM;IAC3B,aAAa,EAAE,KAAK;IACpB,gBAAgB,EAAE,KAAK;IACvB,iBAAiB,EAAE,SAAS;IAC5B,wBAAwB,EAAE,SAAS;IACnC,0BAA0B,EAAE,SAAS;IACrC,kBAAkB,EAAE,UAAU;IAC9B,0BAA0B,EAAE,UAAU;IACtC,0BAA0B,EAAE,UAAU;IACtC,eAAe,EAAE,OAAO;IACxB,sBAAsB,EAAE,OAAO;IAC/B,qBAAqB,EAAE,OAAO;IAC9B,oBAAoB,EAAE,OAAO;IAC7B,gBAAgB,EAAE,QAAQ;IAC1B,qBAAqB,EAAE,QAAQ;IAC/B,sBAAsB,EAAE,QAAQ;IAChC,sBAAsB,EAAE,QAAQ;IAChC,iBAAiB,EAAE,SAAS;IAC5B,mBAAmB,EAAE,SAAS;IAC9B,qBAAqB,EAAE,SAAS;IAChC,aAAa,EAAE,eAAe;IAC9B,kBAAkB,EAAE,cAAc;IAClC,aAAa,EAAE,KAAK;CACrB,CAAC;AAEF,MAAM,CAAC,MAAM,+BAA+B,GAC1C;IACE,KAAK,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,QAAQ,EAAE,SAAS,CAAC;IAClD,OAAO,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,UAAU,CAAC;IACtD,UAAU,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,UAAU,CAAC;IACzD,IAAI,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAE,WAAW,CAAC;IACnD,IAAI,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,WAAW,EAAE,SAAS,CAAC;IACzD,QAAQ,EAAE,CAAC,QAAQ,EAAE,WAAW,EAAE,UAAU,EAAE,QAAQ,EAAE,SAAS,CAAC;IAClE,KAAK,EAAE,CAAC,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,CAAC;IAChD,IAAI,EAAE,CAAC,WAAW,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,CAAC;IAC1D,GAAG,EAAE,CAAC,QAAQ,EAAE,WAAW,CAAC;IAC5B,OAAO,EAAE,CAAC,WAAW,EAAE,QAAQ,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,CAAC;IACjE,QAAQ,EAAE,CAAC,WAAW,EAAE,QAAQ,EAAE,QAAQ,EAAE,YAAY,CAAC;IACzD,KAAK,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,WAAW,CAAC;IAC3D,MAAM,EAAE,CAAC,UAAU,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,EAAE,YAAY,EAAE,SAAS,CAAC;IAC5E,OAAO,EAAE,CAAC,WAAW,EAAE,QAAQ,EAAE,QAAQ,CAAC;IAC1C,aAAa,EAAE,CAAC,WAAW,EAAE,QAAQ,EAAE,UAAU,CAAC;IAClD,YAAY,EAAE,CAAC,QAAQ,EAAE,UAAU,EAAE,KAAK,CAAC;IAC3C,KAAK,EAAE,CAAC,OAAO,CAAC;IAChB,GAAG,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,QAAQ,EAAE,SAAS,CAAC;CACjD,CAAC;AAEJ,MAAM,kBAAkB,GAAG,IAAI,CAAC;AAEhC,MAAM,UAAU,yBAAyB,CACvC,KAAqB,EACrB,OAAmD;IAEnD,MAAM,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;IAChE,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO,eAAe,MAAM,QAAQ,KAAK,CAAC,WAAW,mBAAmB,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;IACzG,CAAC;IACD,OAAO,eAAe,MAAM,SAAS,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,OAAO,QAAQ,KAAK,CAAC,WAAW,WAAW,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,sBAAsB,KAAK,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AACpM,CAAC;AAED,MAAM,UAAU,iBAAiB,CAC/B,QAAqD;IAErD,OAAO,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACvC,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;YACrB,OAAO,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;QACvD,CAAC;QACD,OAAO,YAAY,IAAI,CAAC,MAAM,YAAY,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,QAAQ,IAAI,KAAK,EAAE,CAAC;IAC1F,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,mBAAmB,CAC1B,eAAqC,EACrC,KAAsB;IAEtB,OAAO,eAAe,CAAC,IAAI,CACzB,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,CAAC,QAAQ,KAAK,KAAK,CAAC,QAAQ,CAAC,QAAQ;QACtC,CAAC,CAAC,OAAO,KAAK,KAAK,CAAC,QAAQ,CAAC,OAAO,CACvC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,0BAA0B,CAAC,IAS1C;IACC,MAAM,EACJ,MAAM,EACN,gBAAgB,EAChB,eAAe,EACf,MAAM,EACN,eAAe,GAChB,GAAG,IAAI,CAAC;IAET,MAAM,WAAW,GAAG,oCAAoC,EAAE,CAAC;IAC3D,MAAM,eAAe,GAAiC,EAAE,CAAC;IAEzD,KAAK,MAAM,CAAC,gBAAgB,EAAE,WAAW,CAAC,IAAI,MAAM,CAAC,OAAO,CAC1D,0BAA0B,CAC3B,EAAE,CAAC;QACF,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAC9B,+BAA+B,CAAC,WAAW,CAAC,CAC7C,CAAC;QACF,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CACrC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAC1C,CAAC;QAEF,MAAM,MAAM,GAAG,WAAW,CAAC,gBAAoC,CAAC,CAAC;QAEjE,MAAM,MAAM,GAAG,UAAU;aACtB,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;YACb,MAAM,IAAI,GAAG,iBAAiB,CAAC;gBAC7B,KAAK;gBACL,WAAW;gBACX,gBAAgB;gBAChB,eAAe;gBACf,MAAM;aACP,CAAC,CAAC;YACH,IAAI,IAAI,CAAC,mBAAmB,CAAC,MAAM,GAAG,CAAC;gBAAE,OAAO,IAAI,CAAC;YAErD,MAAM,aAAa,GACjB,MAAM;gBACN,KAAK,CAAC,QAAQ,CAAC,QAAQ,KAAK,MAAM,CAAC,QAAQ;gBAC3C,KAAK,CAAC,QAAQ,CAAC,OAAO,KAAK,MAAM,CAAC,OAAO,CAAC;YAC5C,MAAM,aAAa,GAAG,CAAC,mBAAmB,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;YACnE,MAAM,OAAO,GACX,aAAa,IAAI,aAAa;gBAC5B,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,UAAU,GAAG,kBAAkB,CAAC;gBACnD,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC;YAEtB,OAAO,EAAE,GAAG,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,CAAC;QAC1C,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,mBAAmB,CAAC,MAAM,KAAK,CAAC,CAAC;aACjD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;QAE/C,MAAM,SAAS,GAAG,MAAM;YACtB,CAAC,CAAC,MAAM,CAAC,IAAI,CACT,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,CAAC,KAAK,CAAC,QAAQ,KAAK,MAAM,CAAC,QAAQ;gBACpC,CAAC,CAAC,KAAK,CAAC,OAAO,KAAK,MAAM,CAAC,OAAO,CACrC;YACH,CAAC,CAAC,SAAS,CAAC;QACd,MAAM,IAAI,GAAG,SAAS,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC;QACpC,IAAI,CAAC,IAAI;YAAE,SAAS;QAEpB,MAAM,OAAO,GAAG,eAAe,CAAC,gBAAgB,CAAC,CAAC;QAClD,MAAM,OAAO,GACX,CAAC,OAAO;YACR,OAAO,CAAC,QAAQ,KAAK,IAAI,CAAC,KAAK,CAAC,QAAQ;YACxC,OAAO,CAAC,OAAO,KAAK,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC;QAEzC,eAAe,CAAC,IAAI,CAAC;YACnB,gBAAgB,EAAE,gBAAoC;YACtD,OAAO;YACP,WAAW,EAAE;gBACX,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ;gBAC7B,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO;gBAC3B,WAAW,EAAE,UAAU;aACxB;YACD,WAAW;YACX,KAAK,EAAE,IAAI,CAAC,UAAU;YACtB,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;YACvC,MAAM,EAAE,yBAAyB,CAAC,IAAI,EAAE,OAAO,CAAC;YAChD,UAAU,EAAE,IAAI,CAAC,wBAAwB;YACzC,eAAe,EAAE,iBAAiB,CAAC,IAAI,CAAC,QAAQ,CAAC;YACjD,aAAa,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,QAAQ,EAAE,CAAC;YAClE,qBAAqB,EAAE,OAAO;SAC/B,CAAC,CAAC;IACL,CAAC;IAED,OAAO,eAAe,CAAC;AACzB,CAAC"}

package/dist/evidence/refreshModelEvidence.d.ts

@@ -0,0 +1,25 @@
+import type { ProviderId } from "../types.js";
+import type { BenchmarkSignal, InternalEvalResult, ModelEvidenceFile, ModelIssueSignal, NormalizedModel, ProfileModelRecommendation } from "./types.js";
+export type RefreshModelEvidenceOptions = {
+    includeArtificialAnalysis?: boolean;
+    artificialAnalysisPath?: string;
+    includeHuggingFace?: boolean;
+    includeInternalEvals?: boolean;
+    internalEvalsPath?: string;
+    includeLocalProbes?: boolean;
+    /** When set, use these models instead of live OpenRouter fetch (tests / offline). */
+    models?: NormalizedModel[];
+};
+export type RefreshModelEvidenceResult = {
+    models: NormalizedModel[];
+    benchmarkSignals: BenchmarkSignal[];
+    internalResults: InternalEvalResult[];
+    issues: ModelIssueSignal[];
+    recommendations: ProfileModelRecommendation[];
+    evidenceFile: ModelEvidenceFile;
+};
+export declare function refreshModelEvidence(options: RefreshModelEvidenceOptions, currentMappings: Record<string, {
+    provider: ProviderId;
+    modelId: string;
+}>): Promise<RefreshModelEvidenceResult>;
+//# sourceMappingURL=refreshModelEvidence.d.ts.map

package/dist/evidence/refreshModelEvidence.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"refreshModelEvidence.d.ts","sourceRoot":"","sources":["../../src/evidence/refreshModelEvidence.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAW9C,OAAO,KAAK,EACV,eAAe,EACf,kBAAkB,EAClB,iBAAiB,EACjB,gBAAgB,EAChB,eAAe,EACf,0BAA0B,EAC3B,MAAM,YAAY,CAAC;AAEpB,MAAM,MAAM,2BAA2B,GAAG;IACxC,yBAAyB,CAAC,EAAE,OAAO,CAAC;IACpC,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,qFAAqF;IACrF,MAAM,CAAC,EAAE,eAAe,EAAE,CAAC;CAC5B,CAAC;AAEF,MAAM,MAAM,0BAA0B,GAAG;IACvC,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,gBAAgB,EAAE,eAAe,EAAE,CAAC;IACpC,eAAe,EAAE,kBAAkB,EAAE,CAAC;IACtC,MAAM,EAAE,gBAAgB,EAAE,CAAC;IAC3B,eAAe,EAAE,0BAA0B,EAAE,CAAC;IAC9C,YAAY,EAAE,iBAAiB,CAAC;CACjC,CAAC;AAEF,wBAAsB,oBAAoB,CACxC,OAAO,EAAE,2BAA2B,EACpC,eAAe,EAAE,MAAM,CACrB,MAAM,EACN;IAAE,QAAQ,EAAE,UAAU,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAC1C,GACA,OAAO,CAAC,0BAA0B,CAAC,CA8GrC"}

package/dist/evidence/refreshModelEvidence.js

@@ -0,0 +1,104 @@
+import { loadArtificialAnalysisFromJson, loadInternalEvalResultsFromJson, } from "./adapters.js";
+import { probeAllLocalRuntimes } from "./local/probes.js";
+import { fetchOpenRouterModels, normalizeOpenRouterModel, } from "./openRouter.js";
+import { recommendModelsForProfiles } from "./recommend.js";
+export async function refreshModelEvidence(options, currentMappings) {
+    const openrouterFetchedAt = new Date().toISOString();
+    let models;
+    if (options.models) {
+        models = options.models;
+    }
+    else {
+        const openRouterRaw = await fetchOpenRouterModels({
+            outputModalities: ["text"],
+        });
+        models = openRouterRaw.map(normalizeOpenRouterModel);
+    }
+    const benchmarkSignals = [];
+    const internalResults = [];
+    const issues = [];
+    let artificialAnalysisFetchedAt;
+    let internalEvalFetchedAt;
+    if (options.includeArtificialAnalysis && options.artificialAnalysisPath) {
+        const aa = await loadArtificialAnalysisFromJson(options.artificialAnalysisPath);
+        benchmarkSignals.push(...aa);
+        artificialAnalysisFetchedAt = new Date().toISOString();
+    }
+    if (options.includeHuggingFace) {
+        // Optional: plug dataset adapters when configured. No aggressive scraping.
+    }
+    if (options.includeInternalEvals && options.internalEvalsPath) {
+        internalResults.push(...loadInternalEvalResultsFromJson(options.internalEvalsPath));
+        internalEvalFetchedAt = new Date().toISOString();
+    }
+    if (options.includeLocalProbes) {
+        const probes = await probeAllLocalRuntimes();
+        for (const probe of probes) {
+            if (!probe.available) {
+                issues.push({
+                    source: "internal_eval",
+                    issue: "localInstallRisk",
+                    provider: "local",
+                    modelId: probe.runtime,
+                    severity: "low",
+                    score: 0.2,
+                    evidence: probe.error ?? "runtime unavailable",
+                    fetchedAt: new Date().toISOString(),
+                });
+                continue;
+            }
+            for (const modelId of probe.installedModels) {
+                models.push({
+                    identity: {
+                        provider: "local",
+                        modelId: `${probe.runtime}:${modelId}`,
+                    },
+                    source: "local",
+                    capabilities: {
+                        inputModalities: ["text"],
+                        outputModalities: ["text"],
+                        supportsTools: false,
+                        supportsToolChoice: false,
+                        supportsStructuredOutputs: false,
+                        supportsResponseFormat: false,
+                        supportsReasoning: false,
+                        supportsIncludeReasoning: false,
+                        supportsTemperature: true,
+                        supportsSeed: false,
+                        isDeprecated: false,
+                    },
+                    pricing: {},
+                    fetchedAt: new Date().toISOString(),
+                });
+            }
+        }
+    }
+    const recommendations = recommendModelsForProfiles({
+        models,
+        benchmarkSignals,
+        internalResults,
+        issues,
+        currentMappings,
+    });
+    const evidenceFile = {
+        generatedAt: new Date().toISOString(),
+        sourceVersions: {
+            openrouterFetchedAt,
+            artificialAnalysisFetchedAt,
+            internalEvalFetchedAt,
+        },
+        models,
+        benchmarkSignals,
+        issues,
+        recommendations,
+    };
+    return {
+        models,
+        benchmarkSignals,
+        internalResults,
+        issues,
+        recommendations,
+        evidenceFile,
+    };
+}
+//# sourceMappingURL=refreshModelEvidence.js.map

package/dist/evidence/refreshModelEvidence.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"refreshModelEvidence.js","sourceRoot":"","sources":["../../src/evidence/refreshModelEvidence.ts"],"names":[],"mappings":"AACA,OAAO,EACL,8BAA8B,EAC9B,+BAA+B,GAChC,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AAC1D,OAAO,EACL,qBAAqB,EACrB,wBAAwB,GACzB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,0BAA0B,EAAE,MAAM,gBAAgB,CAAC;AA8B5D,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,OAAoC,EACpC,eAGC;IAED,MAAM,mBAAmB,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAErD,IAAI,MAAyB,CAAC;IAC9B,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAC1B,CAAC;SAAM,CAAC;QACN,MAAM,aAAa,GAAG,MAAM,qBAAqB,CAAC;YAChD,gBAAgB,EAAE,CAAC,MAAM,CAAC;SAC3B,CAAC,CAAC;QACH,MAAM,GAAG,aAAa,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;IACvD,CAAC;IAED,MAAM,gBAAgB,GAAsB,EAAE,CAAC;IAC/C,MAAM,eAAe,GAAyB,EAAE,CAAC;IACjD,MAAM,MAAM,GAAuB,EAAE,CAAC;IACtC,IAAI,2BAA+C,CAAC;IACpD,IAAI,qBAAyC,CAAC;IAE9C,IAAI,OAAO,CAAC,yBAAyB,IAAI,OAAO,CAAC,sBAAsB,EAAE,CAAC;QACxE,MAAM,EAAE,GAAG,MAAM,8BAA8B,CAC7C,OAAO,CAAC,sBAAsB,CAC/B,CAAC;QACF,gBAAgB,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QAC7B,2BAA2B,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACzD,CAAC;IAED,IAAI,OAAO,CAAC,kBAAkB,EAAE,CAAC;QAC/B,2EAA2E;IAC7E,CAAC;IAED,IAAI,OAAO,CAAC,oBAAoB,IAAI,OAAO,CAAC,iBAAiB,EAAE,CAAC;QAC9D,eAAe,CAAC,IAAI,CAClB,GAAG,+BAA+B,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAC9D,CAAC;QACF,qBAAqB,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACnD,CAAC;IAED,IAAI,OAAO,CAAC,kBAAkB,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAG,MAAM,qBAAqB,EAAE,CAAC;QAC7C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;gBACrB,MAAM,CAAC,IAAI,CAAC;oBACV,MAAM,EAAE,eAAe;oBACvB,KAAK,EAAE,kBAAkB;oBACzB,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,KAAK,CAAC,OAAO;oBACtB,QAAQ,EAAE,KAAK;oBACf,KAAK,EAAE,GAAG;oBACV,QAAQ,EAAE,KAAK,CAAC,KAAK,IAAI,qBAAqB;oBAC9C,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;iBACpC,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;YACD,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,eAAe,EAAE,CAAC;gBAC5C,MAAM,CAAC,IAAI,CAAC;oBACV,QAAQ,EAAE;wBACR,QAAQ,EAAE,OAAO;wBACjB,OAAO,EAAE,GAAG,KAAK,CAAC,OAAO,IAAI,OAAO,EAAE;qBACvC;oBACD,MAAM,EAAE,OAAO;oBACf,YAAY,EAAE;wBACZ,eAAe,EAAE,CAAC,MAAM,CAAC;wBACzB,gBAAgB,EAAE,CAAC,MAAM,CAAC;wBAC1B,aAAa,EAAE,KAAK;wBACpB,kBAAkB,EAAE,KAAK;wBACzB,yBAAyB,EAAE,KAAK;wBAChC,sBAAsB,EAAE,KAAK;wBAC7B,iBAAiB,EAAE,KAAK;wBACxB,wBAAwB,EAAE,KAAK;wBAC/B,mBAAmB,EAAE,IAAI;wBACzB,YAAY,EAAE,KAAK;wBACnB,YAAY,EAAE,KAAK;qBACpB;oBACD,OAAO,EAAE,EAAE;oBACX,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;iBACpC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,eAAe,GAAG,0BAA0B,CAAC;QACjD,MAAM;QACN,gBAAgB;QAChB,eAAe;QACf,MAAM;QACN,eAAe;KAChB,CAAC,CAAC;IAEH,MAAM,YAAY,GAAsB;QACtC,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACrC,cAAc,EAAE;YACd,mBAAmB;YACnB,2BAA2B;YAC3B,qBAAqB;SACtB;QACD,MAAM;QACN,gBAAgB;QAChB,MAAM;QACN,eAAe;KAChB,CAAC;IAEF,OAAO;QACL,MAAM;QACN,gBAAgB;QAChB,eAAe;QACf,MAAM;QACN,eAAe;QACf,YAAY;KACb,CAAC;AACJ,CAAC"}

package/dist/evidence/scoreModelForTask.d.ts

@@ -0,0 +1,13 @@
+import type { BenchmarkMetricName, BenchmarkSignal, InternalEvalResult, ModelIssueSignal, ModelTaskScore, NormalizedModel, TaskProfile } from "./types.js";
+import { taskRequirements } from "./scoringConfig.js";
+export declare function modelSatisfiesTask(model: NormalizedModel, requirements: (typeof taskRequirements)[TaskProfile]): boolean;
+export declare function normalizeRawBenchmarkValue(value: number, metric: BenchmarkMetricName, higherIsBetter: boolean): number;
+export declare function aggregateBenchmarkQuality(signals: BenchmarkSignal[], taskProfile: TaskProfile): number;
+export declare function scoreModelForTask(args: {
+    model: NormalizedModel;
+    taskProfile: TaskProfile;
+    benchmarkSignals: BenchmarkSignal[];
+    internalResults: InternalEvalResult[];
+    issues: ModelIssueSignal[];
+}): ModelTaskScore;
+//# sourceMappingURL=scoreModelForTask.d.ts.map

package/dist/evidence/scoreModelForTask.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scoreModelForTask.d.ts","sourceRoot":"","sources":["../../src/evidence/scoreModelForTask.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,mBAAmB,EACnB,eAAe,EACf,kBAAkB,EAClB,gBAAgB,EAChB,cAAc,EACd,eAAe,EACf,WAAW,EACZ,MAAM,YAAY,CAAC;AACpB,OAAO,EASL,gBAAgB,EACjB,MAAM,oBAAoB,CAAC;AAE5B,wBAAgB,kBAAkB,CAChC,KAAK,EAAE,eAAe,EACtB,YAAY,EAAE,CAAC,OAAO,gBAAgB,CAAC,CAAC,WAAW,CAAC,GACnD,OAAO,CAiCT;AAED,wBAAgB,0BAA0B,CACxC,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,mBAAmB,EAC3B,cAAc,EAAE,OAAO,GACtB,MAAM,CAqBR;AAED,wBAAgB,yBAAyB,CACvC,OAAO,EAAE,eAAe,EAAE,EAC1B,WAAW,EAAE,WAAW,GACvB,MAAM,CAkBR;AAED,wBAAgB,iBAAiB,CAAC,IAAI,EAAE;IACtC,KAAK,EAAE,eAAe,CAAC;IACvB,WAAW,EAAE,WAAW,CAAC;IACzB,gBAAgB,EAAE,eAAe,EAAE,CAAC;IACpC,eAAe,EAAE,kBAAkB,EAAE,CAAC;IACtC,MAAM,EAAE,gBAAgB,EAAE,CAAC;CAC5B,GAAG,cAAc,CA4GjB"}

package/dist/evidence/scoreModelForTask.js

@@ -0,0 +1,137 @@
+import { average, clamp01, contextScore, defaultProfileTaskShapes, estimateTaskCostUsd, inverseCostScore, issuePenalty, scoreWeightsByTask, taskRequirements, } from "./scoringConfig.js";
+export function modelSatisfiesTask(model, requirements) {
+    const c = model.capabilities;
+    if (!requirements.allowDeprecated && c.isDeprecated) {
+        return false;
+    }
+    if (requirements.minContextTokens != null &&
+        (c.contextLength ?? 0) < requirements.minContextTokens) {
+        return false;
+    }
+    if (requirements.requiresTools && !c.supportsTools) {
+        return false;
+    }
+    if (requirements.requiresStructuredOutputs &&
+        !c.supportsStructuredOutputs &&
+        !c.supportsResponseFormat) {
+        return false;
+    }
+    if (requirements.requiresReasoning && !c.supportsReasoning) {
+        return false;
+    }
+    if (requirements.requiresTextOutput &&
+        c.outputModalities.length > 0 &&
+        !c.outputModalities.includes("text")) {
+        return false;
+    }
+    return true;
+}
+export function normalizeRawBenchmarkValue(value, metric, higherIsBetter) {
+    const assumedMaxByMetric = {
+        intelligence: 100,
+        coding: 100,
+        agentic: 100,
+        functionCalling: 100,
+        factuality: 100,
+        nonHallucination: 100,
+        longContext: 100,
+        mathReasoning: 100,
+        instructionFollowing: 100,
+        writingPreference: 2000,
+        costPerTask: 1,
+        outputTokensPerTask: 10_000,
+        tokensPerSecond: 500,
+        latencyP50Ms: 10_000,
+        latencyP95Ms: 30_000,
+    };
+    const max = assumedMaxByMetric[metric] ?? 100;
+    const raw = clamp01(value / max);
+    return higherIsBetter ? raw : 1 - raw;
+}
+export function aggregateBenchmarkQuality(signals, taskProfile) {
+    const relevant = signals.filter((s) => s.taskProfiles.includes(taskProfile));
+    if (relevant.length === 0)
+        return 0.5;
+    let weightedTotal = 0;
+    let weightSum = 0;
+    for (const signal of relevant) {
+        const normalized = signal.normalizedValue ??
+            normalizeRawBenchmarkValue(signal.value, signal.metric, signal.higherIsBetter);
+        weightedTotal += normalized * signal.confidence;
+        weightSum += signal.confidence;
+    }
+    return weightSum > 0 ? weightedTotal / weightSum : 0.5;
+}
+export function scoreModelForTask(args) {
+    const { model, taskProfile, benchmarkSignals, internalResults, issues } = args;
+    const requirements = taskRequirements[taskProfile];
+    const shape = defaultProfileTaskShapes[taskProfile];
+    const weights = scoreWeightsByTask[taskProfile];
+    const disqualifiedReasons = [];
+    if (!modelSatisfiesTask(model, requirements)) {
+        disqualifiedReasons.push("Model does not satisfy task requirements");
+    }
+    const estimatedCostUsd = estimateTaskCostUsd(model.pricing, shape);
+    if (requirements.maxCostUsd != null &&
+        estimatedCostUsd > requirements.maxCostUsd) {
+        disqualifiedReasons.push(`Estimated cost ${estimatedCostUsd} exceeds max ${requirements.maxCostUsd}`);
+    }
+    const relevantInternal = internalResults.filter((r) => r.provider === model.identity.provider &&
+        r.modelId === model.identity.modelId);
+    const internalQuality = relevantInternal.length > 0
+        ? average(relevantInternal
+            .map((r) => r.qualityScore ?? r.passRate)
+            .filter((v) => v != null))
+        : undefined;
+    const modelSignals = benchmarkSignals.filter((s) => {
+        if (s.openRouterId && s.openRouterId === model.identity.openRouterId) {
+            return true;
+        }
+        return (s.provider === model.identity.provider &&
+            s.modelId === model.identity.modelId);
+    });
+    const benchmarkQuality = aggregateBenchmarkQuality(modelSignals, taskProfile);
+    const qualityScore = internalQuality ?? benchmarkQuality;
+    const maxCost = requirements.maxCostUsd ?? Math.max(estimatedCostUsd * 3, 0.01);
+    const costScore = inverseCostScore(estimatedCostUsd, maxCost);
+    const latencyFromInternal = average(relevantInternal
+        .map((r) => r.avgLatencyMs)
+        .filter((v) => v != null));
+    const latencyScore = latencyFromInternal != null && latencyFromInternal > 0
+        ? clamp01(1 - latencyFromInternal / (requirements.maxLatencyMs ?? 10_000))
+        : 0.5;
+    const modelIssues = issues.filter((i) => {
+        if (i.openRouterId && i.openRouterId === model.identity.openRouterId) {
+            return true;
+        }
+        return (i.provider === model.identity.provider &&
+            i.modelId === model.identity.modelId);
+    });
+    const reliabilityScore = 1 - issuePenalty(modelIssues);
+    const cScore = contextScore(model.capabilities.contextLength, requirements.minContextTokens);
+    const benchmarkConfidenceScore = modelSignals.length > 0
+        ? average(modelSignals.map((s) => s.confidence))
+        : 0.2;
+    const finalScore = qualityScore * weights.quality +
+        costScore * weights.cost +
+        latencyScore * weights.latency +
+        reliabilityScore * weights.reliability +
+        cScore * weights.context +
+        benchmarkConfidenceScore * weights.benchmarkConfidence;
+    return {
+        model: model.identity,
+        taskProfile,
+        qualityScore,
+        costScore,
+        latencyScore,
+        reliabilityScore,
+        contextScore: cScore,
+        benchmarkConfidenceScore,
+        finalScore: disqualifiedReasons.length ? 0 : clamp01(finalScore),
+        estimatedCostUsd,
+        evidence: [...modelSignals, ...relevantInternal],
+        issues: modelIssues,
+        disqualifiedReasons,
+    };
+}
+//# sourceMappingURL=scoreModelForTask.js.map

package/dist/evidence/scoreModelForTask.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scoreModelForTask.js","sourceRoot":"","sources":["../../src/evidence/scoreModelForTask.ts"],"names":[],"mappings":"AASA,OAAO,EACL,OAAO,EACP,OAAO,EACP,YAAY,EACZ,wBAAwB,EACxB,mBAAmB,EACnB,gBAAgB,EAChB,YAAY,EACZ,kBAAkB,EAClB,gBAAgB,GACjB,MAAM,oBAAoB,CAAC;AAE5B,MAAM,UAAU,kBAAkB,CAChC,KAAsB,EACtB,YAAoD;IAEpD,MAAM,CAAC,GAAG,KAAK,CAAC,YAAY,CAAC;IAE7B,IAAI,CAAC,YAAY,CAAC,eAAe,IAAI,CAAC,CAAC,YAAY,EAAE,CAAC;QACpD,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IACE,YAAY,CAAC,gBAAgB,IAAI,IAAI;QACrC,CAAC,CAAC,CAAC,aAAa,IAAI,CAAC,CAAC,GAAG,YAAY,CAAC,gBAAgB,EACtD,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IAAI,YAAY,CAAC,aAAa,IAAI,CAAC,CAAC,CAAC,aAAa,EAAE,CAAC;QACnD,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IACE,YAAY,CAAC,yBAAyB;QACtC,CAAC,CAAC,CAAC,yBAAyB;QAC5B,CAAC,CAAC,CAAC,sBAAsB,EACzB,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IAAI,YAAY,CAAC,iBAAiB,IAAI,CAAC,CAAC,CAAC,iBAAiB,EAAE,CAAC;QAC3D,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IACE,YAAY,CAAC,kBAAkB;QAC/B,CAAC,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC;QAC7B,CAAC,CAAC,CAAC,gBAAgB,CAAC,QAAQ,CAAC,MAAM,CAAC,EACpC,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,0BAA0B,CACxC,KAAa,EACb,MAA2B,EAC3B,cAAuB;IAEvB,MAAM,kBAAkB,GAAiD;QACvE,YAAY,EAAE,GAAG;QACjB,MAAM,EAAE,GAAG;QACX,OAAO,EAAE,GAAG;QACZ,eAAe,EAAE,GAAG;QACpB,UAAU,EAAE,GAAG;QACf,gBAAgB,EAAE,GAAG;QACrB,WAAW,EAAE,GAAG;QAChB,aAAa,EAAE,GAAG;QAClB,oBAAoB,EAAE,GAAG;QACzB,iBAAiB,EAAE,IAAI;QACvB,WAAW,EAAE,CAAC;QACd,mBAAmB,EAAE,MAAM;QAC3B,eAAe,EAAE,GAAG;QACpB,YAAY,EAAE,MAAM;QACpB,YAAY,EAAE,MAAM;KACrB,CAAC;IACF,MAAM,GAAG,GAAG,kBAAkB,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC;IAC9C,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC;IACjC,OAAO,cAAc,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC;AACxC,CAAC;AAED,MAAM,UAAU,yBAAyB,CACvC,OAA0B,EAC1B,WAAwB;IAExB,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC;IAC7E,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,GAAG,CAAC;IAEtC,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;QAC9B,MAAM,UAAU,GACd,MAAM,CAAC,eAAe;YACtB,0BAA0B,CACxB,MAAM,CAAC,KAAK,EACZ,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,cAAc,CACtB,CAAC;QACJ,aAAa,IAAI,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;QAChD,SAAS,IAAI,MAAM,CAAC,UAAU,CAAC;IACjC,CAAC;IACD,OAAO,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,GAAG,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC;AACzD,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,IAMjC;IACC,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,EAAE,GACrE,IAAI,CAAC;IAEP,MAAM,YAAY,GAAG,gBAAgB,CAAC,WAAW,CAAC,CAAC;IACnD,MAAM,KAAK,GAAG,wBAAwB,CAAC,WAAW,CAAC,CAAC;IACpD,MAAM,OAAO,GAAG,kBAAkB,CAAC,WAAW,CAAC,CAAC;IAChD,MAAM,mBAAmB,GAAa,EAAE,CAAC;IAEzC,IAAI,CAAC,kBAAkB,CAAC,KAAK,EAAE,YAAY,CAAC,EAAE,CAAC;QAC7C,mBAAmB,CAAC,IAAI,CAAC,0CAA0C,CAAC,CAAC;IACvE,CAAC;IAED,MAAM,gBAAgB,GAAG,mBAAmB,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;IACnE,IACE,YAAY,CAAC,UAAU,IAAI,IAAI;QAC/B,gBAAgB,GAAG,YAAY,CAAC,UAAU,EAC1C,CAAC;QACD,mBAAmB,CAAC,IAAI,CACtB,kBAAkB,gBAAgB,gBAAgB,YAAY,CAAC,UAAU,EAAE,CAC5E,CAAC;IACJ,CAAC;IAED,MAAM,gBAAgB,GAAG,eAAe,CAAC,MAAM,CAC7C,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,CAAC,QAAQ,KAAK,KAAK,CAAC,QAAQ,CAAC,QAAQ;QACtC,CAAC,CAAC,OAAO,KAAK,KAAK,CAAC,QAAQ,CAAC,OAAO,CACvC,CAAC;IAEF,MAAM,eAAe,GACnB,gBAAgB,CAAC,MAAM,GAAG,CAAC;QACzB,CAAC,CAAC,OAAO,CACL,gBAAgB;aACb,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC,QAAQ,CAAC;aACxC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,CAAC,IAAI,IAAI,CAAC,CACzC;QACH,CAAC,CAAC,SAAS,CAAC;IAEhB,MAAM,YAAY,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;QACjD,IAAI,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC,YAAY,KAAK,KAAK,CAAC,QAAQ,CAAC,YAAY,EAAE,CAAC;YACrE,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO,CACL,CAAC,CAAC,QAAQ,KAAK,KAAK,CAAC,QAAQ,CAAC,QAAQ;YACtC,CAAC,CAAC,OAAO,KAAK,KAAK,CAAC,QAAQ,CAAC,OAAO,CACrC,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,gBAAgB,GAAG,yBAAyB,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC;IAC9E,MAAM,YAAY,GAAG,eAAe,IAAI,gBAAgB,CAAC;IAEzD,MAAM,OAAO,GACX,YAAY,CAAC,UAAU,IAAI,IAAI,CAAC,GAAG,CAAC,gBAAgB,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;IAClE,MAAM,SAAS,GAAG,gBAAgB,CAAC,gBAAgB,EAAE,OAAO,CAAC,CAAC;IAE9D,MAAM,mBAAmB,GAAG,OAAO,CACjC,gBAAgB;SACb,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC;SAC1B,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,CAAC,IAAI,IAAI,CAAC,CACzC,CAAC;IACF,MAAM,YAAY,GAChB,mBAAmB,IAAI,IAAI,IAAI,mBAAmB,GAAG,CAAC;QACpD,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,mBAAmB,GAAG,CAAC,YAAY,CAAC,YAAY,IAAI,MAAM,CAAC,CAAC;QAC1E,CAAC,CAAC,GAAG,CAAC;IAEV,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;QACtC,IAAI,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC,YAAY,KAAK,KAAK,CAAC,QAAQ,CAAC,YAAY,EAAE,CAAC;YACrE,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO,CACL,CAAC,CAAC,QAAQ,KAAK,KAAK,CAAC,QAAQ,CAAC,QAAQ;YACtC,CAAC,CAAC,OAAO,KAAK,KAAK,CAAC,QAAQ,CAAC,OAAO,CACrC,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,gBAAgB,GAAG,CAAC,GAAG,YAAY,CAAC,WAAW,CAAC,CAAC;IACvD,MAAM,MAAM,GAAG,YAAY,CACzB,KAAK,CAAC,YAAY,CAAC,aAAa,EAChC,YAAY,CAAC,gBAAgB,CAC9B,CAAC;IACF,MAAM,wBAAwB,GAC5B,YAAY,CAAC,MAAM,GAAG,CAAC;QACrB,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;QAChD,CAAC,CAAC,GAAG,CAAC;IAEV,MAAM,UAAU,GACd,YAAY,GAAG,OAAO,CAAC,OAAO;QAC9B,SAAS,GAAG,OAAO,CAAC,IAAI;QACxB,YAAY,GAAG,OAAO,CAAC,OAAO;QAC9B,gBAAgB,GAAG,OAAO,CAAC,WAAW;QACtC,MAAM,GAAG,OAAO,CAAC,OAAO;QACxB,wBAAwB,GAAG,OAAO,CAAC,mBAAmB,CAAC;IAEzD,OAAO;QACL,KAAK,EAAE,KAAK,CAAC,QAAQ;QACrB,WAAW;QACX,YAAY;QACZ,SAAS;QACT,YAAY;QACZ,gBAAgB;QAChB,YAAY,EAAE,MAAM;QACpB,wBAAwB;QACxB,UAAU,EAAE,mBAAmB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC;QAChE,gBAAgB;QAChB,QAAQ,EAAE,CAAC,GAAG,YAAY,EAAE,GAAG,gBAAgB,CAAC;QAChD,MAAM,EAAE,WAAW;QACnB,mBAAmB;KACpB,CAAC;AACJ,CAAC"}

package/dist/evidence/scoringConfig.d.ts

@@ -0,0 +1,11 @@
+import type { EstimatedTaskShape, ModelIssueSignal, ModelPricing, ScoreWeights, TaskProfile, TaskRequirements } from "./types.js";
+export declare const defaultProfileTaskShapes: Record<TaskProfile, EstimatedTaskShape>;
+export declare const taskRequirements: Record<TaskProfile, TaskRequirements>;
+export declare const scoreWeightsByTask: Record<TaskProfile, ScoreWeights>;
+export declare function estimateTaskCostUsd(pricing: ModelPricing, task: EstimatedTaskShape): number;
+export declare function clamp01(value: number): number;
+export declare function inverseCostScore(costUsd: number, maxAcceptableCostUsd: number): number;
+export declare function contextScore(contextLength: number | undefined, required: number | undefined): number;
+export declare function issuePenalty(issues: ModelIssueSignal[]): number;
+export declare function average(values: number[]): number;
+//# sourceMappingURL=scoringConfig.d.ts.map

package/dist/evidence/scoringConfig.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scoringConfig.d.ts","sourceRoot":"","sources":["../../src/evidence/scoringConfig.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,kBAAkB,EAClB,gBAAgB,EAChB,YAAY,EACZ,YAAY,EACZ,WAAW,EACX,gBAAgB,EACjB,MAAM,YAAY,CAAC;AAEpB,eAAO,MAAM,wBAAwB,EAAE,MAAM,CAAC,WAAW,EAAE,kBAAkB,CAwC5E,CAAC;AAEF,eAAO,MAAM,gBAAgB,EAAE,MAAM,CAAC,WAAW,EAAE,gBAAgB,CA2BlE,CAAC;AAEF,eAAO,MAAM,kBAAkB,EAAE,MAAM,CAAC,WAAW,EAAE,YAAY,CAiJhE,CAAC;AAEF,wBAAgB,mBAAmB,CACjC,OAAO,EAAE,YAAY,EACrB,IAAI,EAAE,kBAAkB,GACvB,MAAM,CAaR;AAED,wBAAgB,OAAO,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAE7C;AAED,wBAAgB,gBAAgB,CAC9B,OAAO,EAAE,MAAM,EACf,oBAAoB,EAAE,MAAM,GAC3B,MAAM,CAGR;AAED,wBAAgB,YAAY,CAC1B,aAAa,EAAE,MAAM,GAAG,SAAS,EACjC,QAAQ,EAAE,MAAM,GAAG,SAAS,GAC3B,MAAM,CAIR;AAED,wBAAgB,YAAY,CAAC,MAAM,EAAE,gBAAgB,EAAE,GAAG,MAAM,CAa/D;AAED,wBAAgB,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,CAGhD"}