@crowandraven/evalkeeper 0.2.0 → 0.3.0

package/src/types.ts

@@ -114,6 +114,11 @@ export type Suite<Input, Output, Label> = {
        * temperature was versioned.
        */
       temperature?: number | null;
+      /**
+       * The output-token cap the test runs at, like temperature. Null: send
+       * none (the call's own default).
+       */
+      maxOutputTokens?: number | null;
       onCost: (usd: number) => void;
     },
   ): Promise<Output>;

package/src/weekly.ts

@@ -104,8 +104,8 @@ export function weeklySummary(args: {
       continue;
     }
     if (r.status !== "SUCCEEDED") continue;
-    // The last check of the same model, temperature, instructions, and
-    // grading, a day or more earlier: anything else changed for a known
+    // The last check of the same model, temperature, output cap,
+    // instructions, and grading, a day or more earlier: anything else changed for a known
     // reason.
     const prior = scheduled.find(
       (p) =>
@@ -113,6 +113,7 @@ export function weeklySummary(args: {
         p.status === "SUCCEEDED" &&
         p.model === r.model &&
         (p.temperature ?? null) === (r.temperature ?? null) &&
+        (p.maxOutputTokens ?? null) === (r.maxOutputTokens ?? null) &&
         p.promptVersion === r.promptVersion &&
         p.graderVersion === r.graderVersion &&
         +r.startedAt - +p.startedAt >= DAY,

package/templates/app/evals/[suite]/page.tsx

@@ -7,7 +7,6 @@ import {
   money,
   probabilityQuality,
   perAnswer,
-  type ReasoningEffort,
   retiresLabel,
   seconds,
   seriousMistakes,
@@ -75,7 +74,7 @@ export default async function SuitePage({
           [
             r.key,
             await runner.estimateRun(key, r.model, "ALL", 1, {
-              reasoningEffort: r.effort as ReasoningEffort | null,
+              reasoningEffort: r.effort,
             }),
           ] as const,
       ),

package/templates/app/evals/feedback/page.tsx

@@ -1,4 +1,3 @@
-import Link from "next/link";
 import {
   asText,
   thumbsSummary,
@@ -103,11 +102,17 @@ export default async function FeedbackPage() {
         </h2>
         {open.length === 0 && (
           <p className="text-sm text-muted">
-            Nothing waiting. Rate an answer on the{" "}
-            <Link href="/try" className="underline">
-              try it
-            </Link>{" "}
-            page to see one here.
+            Nothing waiting. Ratings show up here once an AI feature has a
+            thumbs up and down (step 8 of{" "}
+            <a
+              href="https://github.com/seanhudson/evalkeeper/blob/main/docs/adding-to-your-app.md#8-add-thumbs-to-your-ai-answers-optional"
+              target="_blank"
+              rel="noreferrer"
+              className="underline"
+            >
+              the guide
+            </a>
+            ).
           </p>
         )}
         <div className="space-y-3">

package/templates/app/evals/page.tsx

@@ -120,6 +120,7 @@ export default async function EvalsHub() {
           Suites
           <InfoTip term="suites" />
         </h2>
+        {boards.length === 0 && <NoSuites />}
         <div className="space-y-4">
           {boards.map((b, i) => {
             const h = suiteHealth(
@@ -147,6 +148,45 @@ export default async function EvalsHub() {
   );
 }
 
+/** Before the first suite: what a suite is, and how to add one. */
+function NoSuites() {
+  return (
+    <Card className="readable space-y-3 text-sm">
+      <p className="font-semibold">No suites yet</p>
+      <p className="text-muted">
+        A suite tests one AI feature: its cases, the answers a person confirms,
+        and the checks that grade any model against them. To add one:
+      </p>
+      <ol className="list-decimal space-y-2 pl-4">
+        <li>
+          Write it: the input and answer shapes with zod, two or three checks,{" "}
+          <code>run</code>, and <code>grade</code>.
+        </li>
+        <li>
+          Add it to <code>SUITES</code> in <code>lib/evalkeeper/index.ts</code>,
+          with its instructions hash.
+        </li>
+        <li>
+          Add cases (here, or with <code>importCases</code> from files you
+          have), confirm their answers, and test models.
+        </li>
+      </ol>
+      <p className="text-muted">
+        Step by step:{" "}
+        <a
+          href="https://github.com/seanhudson/evalkeeper/blob/main/docs/adding-to-your-app.md#5-write-your-first-suite"
+          target="_blank"
+          rel="noreferrer"
+          className="underline"
+        >
+          Write your first suite
+        </a>
+        .
+      </p>
+    </Card>
+  );
+}
+
 function Stat({
   title,
   term,