@aglyn/plugins-ai 1.0.0-beta.252 → 1.0.0-beta.254

package/src/lib/runtime/ai-eval-live.d.ts

@@ -16,7 +16,7 @@
  */
 import type { AiJob } from '../model/ai-jobs.types';
 import { type AiProductsProposal } from '../model/ai-products';
-import type { AiImagePart, AiProvider, AiSystemBlock, AiTool } from '../providers/contract';
+import type { AiEffort, AiImagePart, AiProvider, AiSystemBlock, AiTool } from '../providers/contract';
 import { type AiCustomGenerationInput } from './ai-doctrine';
 import { type AiEvalCandidate, type AiEvalCase, type AiEvalKind, type AiEvalRubric } from './ai-eval';
 /**
@@ -301,6 +301,40 @@ export interface AiEvalLiveReport {
     /** The briefs whose run threw; empty when every brief a recorder covers came back. */
     failed: AiEvalLiveFailure[];
 }
+/** The effort A/B arm a live run records under (AGL-3781). */
+export interface AiEvalEffortArm {
+    /** `effort-` and 8 hex characters of a hash of `efforts`: the same arm always gets the same tag. */
+    tag: string;
+    /** Each step kind the arm names, in kind order, with its effort; `*` for every other kind. */
+    efforts: Record<string, AiEffort>;
+}
+/**
+ * The arm `AGLYN_AI_EFFORT` names, or null when it names none. A recording's
+ * filename carries only the tag, and the recording carries the map: the
+ * filename once spelled the whole list out, and a list of five kinds was
+ * longer than a filename may be (ENAMETOOLONG lost the round's recordings).
+ * The hash is of the parsed, sorted map, so the same efforts written in
+ * another order or spacing are the same arm.
+ */
+export declare function aiEvalEffortArm(env: Record<string, string | undefined>): AiEvalEffortArm | null;
+/** A recording's file name: `<case>.<model>.json`, with the arm's tag before `.json` when one ran. */
+export declare function aiEvalRecordingFileName(caseId: string, model: string | null | undefined, arm: AiEvalEffortArm | null): string;
+/**
+ * What a recording holds: the brief, the answer, and under `arm` the arm's
+ * tag and every effort it named, when one ran — the candidate's own `effort`
+ * is the one its step was asked at.
+ */
+export declare function aiEvalRecordingBody(caseId: string, candidate: AiEvalCandidate, arm: AiEvalEffortArm | null): string;
+/**
+ * One brief's score line. Everything before the colon is the pairing key —
+ * `score <case> (<kind>) <model>`, then the arm's tag when one ran — so a
+ * default run's line and an arm's line for the same brief sort side by side.
+ */
+export declare function aiEvalScoreLine(entry: {
+    caseId: string;
+    kind: AiEvalKind;
+    candidate: AiEvalCandidate;
+}, pass: boolean | null, arm: AiEvalEffortArm | null): string;
 /**
  * Record and grade an answer for every brief a recorder covers. Refused
  * before anything else unless `AI_EVAL_LIVE=1` names the run, and before any

package/src/lib/runtime/ai-eval-live.js

@@ -14,7 +14,8 @@ import { _ as _extends } from "@swc/helpers/_/_extends";
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
- */ import { assistCreditsFromUsd } from "../usage/assist-credits.js";
+ */ import { createHash } from "node:crypto";
+import { assistCreditsFromUsd } from "../usage/assist-credits.js";
 import { LAYOUT_SLOT_COMPONENT_ID } from "@aglyn/aglyn/app-utils/compose-layout-nodes";
 import { stampDocumentLandmark } from "@aglyn/aglyn/app-utils/document-landmark";
 import { decodeStoredNodes } from "@aglyn/aglyn/app-utils/stored-nodes";
@@ -41,7 +42,7 @@ import { aiSiteInventoryBlock, runValidatedGeneration } from "./ai-doctrine.js";
 import { AI_EVAL_AUTOMATION_CAPABILITIES, AI_EVAL_SITE_ID, AI_EVAL_TREE_OUTPUT } from "./ai-eval.js";
 import { aiEvalMemoryFirestore, aiEvalMemoryInventory } from "./ai-eval-memory-firestore.js";
 import { aiCategoriesPrompt, aiProductCopyPrompt } from "./ai-products-generation.js";
-import { aiLiveBatchWanted } from "./ai-dev-env.js";
+import { aiDevEffortArm, aiLiveBatchWanted } from "./ai-dev-env.js";
 import { aiModelReadsImages } from "./ai-runtime.js";
 /**
  * THE EVAL HARNESS, LIVE (AGL-2937): recording new answers from the model.
@@ -1189,6 +1190,49 @@ const looseText = (value)=>typeof value === 'string' ? value.replace(/\s+/g, ' '
     }
     return result.value;
 }
+/**
+ * The arm `AGLYN_AI_EFFORT` names, or null when it names none. A recording's
+ * filename carries only the tag, and the recording carries the map: the
+ * filename once spelled the whole list out, and a list of five kinds was
+ * longer than a filename may be (ENAMETOOLONG lost the round's recordings).
+ * The hash is of the parsed, sorted map, so the same efforts written in
+ * another order or spacing are the same arm.
+ */ export function aiEvalEffortArm(env) {
+    const efforts = aiDevEffortArm(env);
+    if (!Object.keys(efforts).length) return null;
+    const hash = createHash('sha256').update(JSON.stringify(efforts)).digest('hex').slice(0, 8);
+    return {
+        tag: `effort-${hash}`,
+        efforts
+    };
+}
+/** A recording's file name: `<case>.<model>.json`, with the arm's tag before `.json` when one ran. */ export function aiEvalRecordingFileName(caseId, model, arm) {
+    return `${caseId}.${model != null ? model : 'unknown'}${arm ? `.${arm.tag}` : ''}.json`;
+}
+/**
+ * What a recording holds: the brief, the answer, and under `arm` the arm's
+ * tag and every effort it named, when one ran — the candidate's own `effort`
+ * is the one its step was asked at.
+ */ export function aiEvalRecordingBody(caseId, candidate, arm) {
+    return `${JSON.stringify(_extends({
+        caseId
+    }, arm ? {
+        arm
+    } : {}, {
+        candidate
+    }), null, 2)}\n`;
+}
+/**
+ * One brief's score line. Everything before the colon is the pairing key —
+ * `score <case> (<kind>) <model>`, then the arm's tag when one ran — so a
+ * default run's line and an arm's line for the same brief sort side by side.
+ */ export function aiEvalScoreLine(entry, pass, arm) {
+    var _candidate_model, _ref, _ref1, _ref2, _ref3;
+    var _candidate_usage;
+    const { caseId, kind, candidate } = entry;
+    const rubric = candidate.rubric;
+    return `score ${caseId} (${kind}) ${(_candidate_model = candidate.model) != null ? _candidate_model : 'unknown'}${arm ? ` ${arm.tag}` : ''}: ` + `${pass ? 'pass' : 'fail'}, structure ${(_ref = rubric == null ? void 0 : rubric.structure) != null ? _ref : '-'}, copy ${(_ref1 = rubric == null ? void 0 : rubric.copy) != null ? _ref1 : '-'}` + `, reuse ${(_ref2 = rubric == null ? void 0 : rubric.reuse) != null ? _ref2 : '-'}, output ${(_ref3 = (_candidate_usage = candidate.usage) == null ? void 0 : _candidate_usage.outputTokens) != null ? _ref3 : '-'} tokens`;
+}
 /** The vendor's request id off a thrown `UpstreamServiceError`, when it carried one. */ function requestIdOfThrown(error) {
     const id = error == null ? void 0 : error.requestId;
     return typeof id === 'string' && id ? id : null;