@fabricorg/databricks-testkit 0.7.0 → 0.7.1

package/dist/index.cjs

@@ -2444,6 +2444,12 @@ function parseJson(value, name) {
     throw new Error(`${name} must contain valid JSON`);
   }
 }
+function isObject(value) {
+  return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+function isNonNegativeInteger(value) {
+  return typeof value === "number" && Number.isInteger(value) && value >= 0;
+}
 function assertNotFailed(label, state) {
   if (typeof state === "string" && /FAIL|ERROR|DELET/i.test(state)) {
     throw new Error(`${label} is ${state}`);
@@ -2678,6 +2684,46 @@ function modelServingCheck() {
     }
   };
 }
+function evalJudgeModelServingCheck() {
+  const names = ["DBX_TEST_EVAL_SERVING_ENDPOINT"];
+  return {
+    id: "model-eval-judge",
+    description: "Databricks Model Serving satisfies the deterministic evaluator chat contract",
+    configured: (env) => required2(env, names).length === 0,
+    async run({ env, client }) {
+      requireEnv(env, names);
+      const api = adapter(client);
+      const name = env.DBX_TEST_EVAL_SERVING_ENDPOINT;
+      const endpoint = await api.servingEndpoint(name);
+      assertReady("Evaluator serving endpoint", endpoint.state?.ready, ["READY"]);
+      const response = await api.invokeServingEndpoint(name, {
+        messages: [
+          {
+            role: "system",
+            content: 'Answer on the first line with exactly "LABEL: relevant".'
+          },
+          { role: "user", content: "Is this response relevant to this question?" }
+        ],
+        temperature: 0,
+        max_tokens: 32
+      });
+      const choices = Array.isArray(response.choices) ? response.choices : [];
+      const first = choices[0];
+      const message = isObject(first) && isObject(first.message) ? first.message : void 0;
+      const content = typeof message?.content === "string" ? message.content : "";
+      const usage = isObject(response.usage) ? response.usage : void 0;
+      const inputTokens = usage?.prompt_tokens;
+      const outputTokens = usage?.completion_tokens;
+      if (!/^\s*LABEL:\s*relevant\s*$/im.test(content)) {
+        throw new Error("Evaluator serving endpoint did not return the requested label contract");
+      }
+      if (!isNonNegativeInteger(inputTokens) || !isNonNegativeInteger(outputTokens)) {
+        throw new Error("Evaluator serving endpoint did not return integer token usage");
+      }
+      return { endpoint: endpoint.name ?? name, inputTokens, outputTokens };
+    }
+  };
+}
 function vectorSearchCheck() {
   const names = ["DBX_TEST_VECTOR_ENDPOINT", "DBX_TEST_VECTOR_INDEX"];
   return {
@@ -3224,6 +3270,7 @@ exports.deleteVolumeFile = deleteVolumeFile;
 exports.deltaContractCheck = deltaContractCheck;
 exports.doctorTargetPack = doctorTargetPack;
 exports.ensureVolumeDirectory = ensureVolumeDirectory;
+exports.evalJudgeModelServingCheck = evalJudgeModelServingCheck;
 exports.expectQueryToMatchGolden = expectQueryToMatchGolden;
 exports.expectTable = expectTable;
 exports.failureRecoveryCheck = failureRecoveryCheck;