@aglyn/plugins-ai 1.0.0-beta.250 → 1.0.0-beta.253

package/src/lib/runtime/ai-eval-live.js

@@ -14,7 +14,8 @@ import { _ as _extends } from "@swc/helpers/_/_extends";
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
- */ import { assistCreditsFromUsd } from "../usage/assist-credits.js";
+ */ import { createHash } from "node:crypto";
+import { assistCreditsFromUsd } from "../usage/assist-credits.js";
 import { LAYOUT_SLOT_COMPONENT_ID } from "@aglyn/aglyn/app-utils/compose-layout-nodes";
 import { stampDocumentLandmark } from "@aglyn/aglyn/app-utils/document-landmark";
 import { decodeStoredNodes } from "@aglyn/aglyn/app-utils/stored-nodes";
@@ -41,6 +42,7 @@ import { aiSiteInventoryBlock, runValidatedGeneration } from "./ai-doctrine.js";
 import { AI_EVAL_AUTOMATION_CAPABILITIES, AI_EVAL_SITE_ID, AI_EVAL_TREE_OUTPUT } from "./ai-eval.js";
 import { aiEvalMemoryFirestore, aiEvalMemoryInventory } from "./ai-eval-memory-firestore.js";
 import { aiCategoriesPrompt, aiProductCopyPrompt } from "./ai-products-generation.js";
+import { aiDevEffortArm, aiLiveBatchWanted } from "./ai-dev-env.js";
 import { aiModelReadsImages } from "./ai-runtime.js";
 /**
  * THE EVAL HARNESS, LIVE (AGL-2937): recording new answers from the model.
@@ -1188,6 +1190,49 @@ const looseText = (value)=>typeof value === 'string' ? value.replace(/\s+/g, ' '
     }
     return result.value;
 }
+/**
+ * The arm `AGLYN_AI_EFFORT` names, or null when it names none. A recording's
+ * filename carries only the tag, and the recording carries the map: the
+ * filename once spelled the whole list out, and a list of five kinds was
+ * longer than a filename may be (ENAMETOOLONG lost the round's recordings).
+ * The hash is of the parsed, sorted map, so the same efforts written in
+ * another order or spacing are the same arm.
+ */ export function aiEvalEffortArm(env) {
+    const efforts = aiDevEffortArm(env);
+    if (!Object.keys(efforts).length) return null;
+    const hash = createHash('sha256').update(JSON.stringify(efforts)).digest('hex').slice(0, 8);
+    return {
+        tag: `effort-${hash}`,
+        efforts
+    };
+}
+/** A recording's file name: `<case>.<model>.json`, with the arm's tag before `.json` when one ran. */ export function aiEvalRecordingFileName(caseId, model, arm) {
+    return `${caseId}.${model != null ? model : 'unknown'}${arm ? `.${arm.tag}` : ''}.json`;
+}
+/**
+ * What a recording holds: the brief, the answer, and under `arm` the arm's
+ * tag and every effort it named, when one ran — the candidate's own `effort`
+ * is the one its step was asked at.
+ */ export function aiEvalRecordingBody(caseId, candidate, arm) {
+    return `${JSON.stringify(_extends({
+        caseId
+    }, arm ? {
+        arm
+    } : {}, {
+        candidate
+    }), null, 2)}\n`;
+}
+/**
+ * One brief's score line. Everything before the colon is the pairing key —
+ * `score <case> (<kind>) <model>`, then the arm's tag when one ran — so a
+ * default run's line and an arm's line for the same brief sort side by side.
+ */ export function aiEvalScoreLine(entry, pass, arm) {
+    var _candidate_model, _ref, _ref1, _ref2, _ref3;
+    var _candidate_usage;
+    const { caseId, kind, candidate } = entry;
+    const rubric = candidate.rubric;
+    return `score ${caseId} (${kind}) ${(_candidate_model = candidate.model) != null ? _candidate_model : 'unknown'}${arm ? ` ${arm.tag}` : ''}: ` + `${pass ? 'pass' : 'fail'}, structure ${(_ref = rubric == null ? void 0 : rubric.structure) != null ? _ref : '-'}, copy ${(_ref1 = rubric == null ? void 0 : rubric.copy) != null ? _ref1 : '-'}` + `, reuse ${(_ref2 = rubric == null ? void 0 : rubric.reuse) != null ? _ref2 : '-'}, output ${(_ref3 = (_candidate_usage = candidate.usage) == null ? void 0 : _candidate_usage.outputTokens) != null ? _ref3 : '-'} tokens`;
+}
 /** The vendor's request id off a thrown `UpstreamServiceError`, when it carried one. */ function requestIdOfThrown(error) {
     const id = error == null ? void 0 : error.requestId;
     return typeof id === 'string' && id ? id : null;
@@ -1211,18 +1256,26 @@ function messageOfThrown(error) {
  *
  * A brief whose GRADE threw is in both halves: recorded, with its grade
  * marked missing, and failed, naming what the grader threw.
+ *
+ * ONE BATCH A ROUND (AGL-3781). In batch mode (`AGLYN_LIVE_AI_BATCH`) the
+ * briefs run side by side: a batch gathers the requests that arrive within
+ * its window, so briefs asked one after another went as one batch each,
+ * minutes apart, while side by side every brief's answer goes in one batch,
+ * each re-ask in the next and every grade in the one after. Each brief still
+ * runs on its own and the report keeps the briefs' order. Outside batch mode
+ * they run one after another, as a live run's rate limits want.
  */ export async function recordAiEvalLive(cases, options) {
     if (!aiEvalLiveAllowed(options.env)) throw new AiEvalLiveRefusedError();
     const unreadable = cases.filter((evalCase)=>evalCase.media && aiEvalRecorderFor(evalCase.kind) && !options.readFixture);
     if (unreadable.length) {
         throw new Error(`${unreadable.map((evalCase)=>evalCase.id).join(', ')} name media, and this run was given no readFixture to read them with`);
     }
-    const report = {
-        recorded: [],
-        skipped: [],
-        failed: []
-    };
-    for (const evalCase of cases){
+    const runCase = async (evalCase)=>{
+        const report = {
+            recorded: [],
+            skipped: [],
+            failed: []
+        };
         const recorder = aiEvalRecorderFor(evalCase.kind);
         if (!recorder) {
             report.skipped.push({
@@ -1230,7 +1283,7 @@ function messageOfThrown(error) {
                 kind: evalCase.kind,
                 why: aiEvalSkipReason(evalCase.kind)
             });
-            continue;
+            return report;
         }
         const failure = (step, error)=>{
             report.failed.push({
@@ -1246,7 +1299,7 @@ function messageOfThrown(error) {
             answer = await recorder(evalCase, options);
         } catch (error) {
             failure('record', error);
-            continue;
+            return report;
         }
         // A GRADER THAT THROWS DOES NOT TAKE THE ANSWER WITH IT (AGL-3143). The
         // brief is already paid for by here, and grading is a second request that
@@ -1273,6 +1326,23 @@ function messageOfThrown(error) {
             });
             failure('grade', error);
         }
+        return report;
+    };
+    const reports = [];
+    if (aiLiveBatchWanted(options.env)) {
+        reports.push(...await Promise.all(cases.map(runCase)));
+    } else {
+        for (const evalCase of cases)reports.push(await runCase(evalCase));
+    }
+    const report = {
+        recorded: [],
+        skipped: [],
+        failed: []
+    };
+    for (const one of reports){
+        report.recorded.push(...one.recorded);
+        report.skipped.push(...one.skipped);
+        report.failed.push(...one.failed);
     }
     return report;
 }