@babav/knowledge-core-client 0.45.0 → 0.46.0

package/dist/index.d.ts

@@ -552,7 +552,7 @@ export type QueryProfileWrite = Partial<Omit<QueryProfile, "id" | "tenant_id">>;
 export interface ModelOptions {
     models: Record<string, {
         label: string;
-        provider: "anthropic" | "vertex";
+        provider: "anthropic" | "vertex" | "moonshot";
         kind: "text" | "vision";
         supports_reasoning: boolean;
     }>;
@@ -568,6 +568,43 @@ export interface ModelOptions {
         values: string[];
     }>;
 }
+export type BenchmarkTier = "client_quick" | "screening" | "full";
+export type BenchmarkCategory = "quality" | "quality_cost" | "quality_latency";
+/** A query profile's benchmark score. `cache_hit=false` + `scoring_status="queued"` means scoring was
+ *  enqueued (poll again / check the leaderboard). */
+export interface BenchmarkScore {
+    cache_hit: boolean;
+    config_hash: string;
+    eval_version: string | null;
+    scoring_status: "unscored" | "queued" | "running" | "scored" | "failed" | "stale";
+    composite_q: number | null;
+    sub_scores: Record<string, unknown>;
+    per_language: Record<string, number>;
+    gates: Record<string, unknown>;
+    prod_cost_per_query_usd: number | null;
+    p50_ms: number | null;
+    p95_ms: number | null;
+    percentile: number | null;
+    job_id: string | null;
+}
+export interface LeaderboardEntry {
+    rank: number;
+    config_hash: string;
+    generation_model: string | null;
+    composite_q: number | null;
+    prod_cost_per_query_usd: number | null;
+    p50_ms: number | null;
+    p95_ms: number | null;
+    sub_scores: Record<string, unknown>;
+    per_language: Record<string, number>;
+    config: Record<string, unknown> | null;
+}
+export interface Leaderboard {
+    eval_version: string;
+    tier: BenchmarkTier;
+    category: BenchmarkCategory;
+    entries: LeaderboardEntry[];
+}
 export interface Tenant {
     id: UUID;
     name: string;
@@ -1007,6 +1044,21 @@ export declare class KnowledgeCoreClient extends HttpBase {
         /** The model catalog for an query-profile-config UI: supported models + default per field, per-model
          *  capabilities (`supports_reasoning`), and mode↔model dependencies. */
         modelOptions: () => Promise<ModelOptions>;
+        /** Benchmark this query profile against the eval-set (cache-first on config hash; a miss queues
+         *  scoring). Returns the score (quality composite + sub-scores + gates + cost + latency) or a
+         *  queued status. See also `benchmark.leaderboard`. */
+        benchmark: (id: UUID) => Promise<BenchmarkScore>;
+    };
+    /** Query-profile benchmark leaderboards — top-N system-swept profiles per eval version, per tier
+     *  (client_quick | screening | full) and per category (quality | quality_cost | quality_latency).
+     *  Tenant-anonymous (keyed by config hash); never a customer config. */
+    benchmark: {
+        leaderboard: (q?: {
+            eval_version?: string;
+            tier?: BenchmarkTier;
+            category?: BenchmarkCategory;
+            limit?: number;
+        }) => Promise<Leaderboard>;
     };
     /** Ingestion profiles — the build-side config object (counterpart to query profiles): a tenant-owned,
      *  named bundle of pipeline config (chunking + embedding + sparse + quant). Unlike a query profile

package/dist/index.js

@@ -468,6 +468,16 @@ export class KnowledgeCoreClient extends HttpBase {
         /** The model catalog for an query-profile-config UI: supported models + default per field, per-model
          *  capabilities (`supports_reasoning`), and mode↔model dependencies. */
         modelOptions: () => this.request("GET", "/v1/query-profiles/model-options"),
+        /** Benchmark this query profile against the eval-set (cache-first on config hash; a miss queues
+         *  scoring). Returns the score (quality composite + sub-scores + gates + cost + latency) or a
+         *  queued status. See also `benchmark.leaderboard`. */
+        benchmark: (id) => this.request("POST", `/v1/query-profiles/${id}/benchmark`),
+    };
+    /** Query-profile benchmark leaderboards — top-N system-swept profiles per eval version, per tier
+     *  (client_quick | screening | full) and per category (quality | quality_cost | quality_latency).
+     *  Tenant-anonymous (keyed by config hash); never a customer config. */
+    benchmark = {
+        leaderboard: (q) => this.request("GET", "/v1/benchmark/leaderboard", { query: q }),
     };
     /** Ingestion profiles — the build-side config object (counterpart to query profiles): a tenant-owned,
      *  named bundle of pipeline config (chunking + embedding + sparse + quant). Unlike a query profile

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@babav/knowledge-core-client",
-  "version": "0.45.0",
+  "version": "0.46.0",
   "description": "TypeScript client for the Babav Knowledge Core API (Deno + Node 18+, zero deps). Includes the babav.visual grammar TYPES at the ./visual subpath (types only; all visual rendering is server-side).",
   "type": "module",
   "main": "./dist/index.js",

package/src/index.ts

@@ -519,10 +519,50 @@ export type QueryProfileWrite = Partial<Omit<QueryProfile, "id" | "tenant_id">>;
  *  `modes` says which model field governs each mode (reasoning is valid only if that model's
  *  supports_reasoning is true). KC provides the data; the UI decides presentation. */
 export interface ModelOptions {
-  models: Record<string, { label: string; provider: "anthropic" | "vertex"; kind: "text" | "vision"; supports_reasoning: boolean }>;
+  models: Record<string, { label: string; provider: "anthropic" | "vertex" | "moonshot"; kind: "text" | "vision"; supports_reasoning: boolean }>;
   fields: Record<string, { supported: string[]; default: string; applies_when?: string; applies_to?: string[]; note?: string }>;
   modes: Record<string, { depends_on: string; values: string[] }>;
 }
+
+// ---- benchmark (score-my-profile + leaderboards) --------------------------
+export type BenchmarkTier = "client_quick" | "screening" | "full";
+export type BenchmarkCategory = "quality" | "quality_cost" | "quality_latency";
+
+/** A query profile's benchmark score. `cache_hit=false` + `scoring_status="queued"` means scoring was
+ *  enqueued (poll again / check the leaderboard). */
+export interface BenchmarkScore {
+  cache_hit: boolean;
+  config_hash: string;
+  eval_version: string | null;
+  scoring_status: "unscored" | "queued" | "running" | "scored" | "failed" | "stale";
+  composite_q: number | null;
+  sub_scores: Record<string, unknown>;
+  per_language: Record<string, number>;
+  gates: Record<string, unknown>;
+  prod_cost_per_query_usd: number | null;
+  p50_ms: number | null;
+  p95_ms: number | null;
+  percentile: number | null;
+  job_id: string | null;
+}
+export interface LeaderboardEntry {
+  rank: number;
+  config_hash: string;
+  generation_model: string | null;
+  composite_q: number | null;
+  prod_cost_per_query_usd: number | null;
+  p50_ms: number | null;
+  p95_ms: number | null;
+  sub_scores: Record<string, unknown>;
+  per_language: Record<string, number>;
+  config: Record<string, unknown> | null;
+}
+export interface Leaderboard {
+  eval_version: string;
+  tier: BenchmarkTier;
+  category: BenchmarkCategory;
+  entries: LeaderboardEntry[];
+}
 export interface Tenant {
   id: UUID;
   name: string;
@@ -1142,6 +1182,18 @@ export class KnowledgeCoreClient extends HttpBase {
     /** The model catalog for an query-profile-config UI: supported models + default per field, per-model
      *  capabilities (`supports_reasoning`), and mode↔model dependencies. */
     modelOptions: () => this.request<ModelOptions>("GET", "/v1/query-profiles/model-options"),
+    /** Benchmark this query profile against the eval-set (cache-first on config hash; a miss queues
+     *  scoring). Returns the score (quality composite + sub-scores + gates + cost + latency) or a
+     *  queued status. See also `benchmark.leaderboard`. */
+    benchmark: (id: UUID) => this.request<BenchmarkScore>("POST", `/v1/query-profiles/${id}/benchmark`),
+  };
+
+  /** Query-profile benchmark leaderboards — top-N system-swept profiles per eval version, per tier
+   *  (client_quick | screening | full) and per category (quality | quality_cost | quality_latency).
+   *  Tenant-anonymous (keyed by config hash); never a customer config. */
+  benchmark = {
+    leaderboard: (q?: { eval_version?: string; tier?: BenchmarkTier; category?: BenchmarkCategory; limit?: number }) =>
+      this.request<Leaderboard>("GET", "/v1/benchmark/leaderboard", { query: q }),
   };
 
   /** Ingestion profiles — the build-side config object (counterpart to query profiles): a tenant-owned,