@loreai/core 0.11.1 → 0.13.0

package/src/worker-model.ts

@@ -25,7 +25,11 @@ export type ModelInfo = {
   providerID: string;
   cost: { input: number }; // per-token cost
   status: string;
-  capabilities: { input: { text: boolean } };
+  capabilities: {
+    input: { text: boolean };
+    /** Whether this model supports extended thinking/reasoning. */
+    reasoning?: boolean;
+  };
 };
 
 /** Result of a worker model validation stored in kv_meta. */
@@ -64,8 +68,17 @@ export function selectWorkerCandidates(
 
   if (eligible.length === 0) return [];
 
-  // Sort by cost ascending (cheapest first)
-  const sorted = [...eligible].sort((a, b) => a.cost.input - b.cost.input);
+  // Sort by cost ascending, then prefer non-reasoning models at equal cost.
+  // Non-reasoning models don't produce thinking tokens, avoiding wasted spend
+  // on tokens that background workers discard.
+  const sorted = [...eligible].sort((a, b) => {
+    const costDiff = a.cost.input - b.cost.input;
+    if (costDiff !== 0) return costDiff;
+    // At equal cost, non-reasoning (0) sorts before reasoning (1)
+    const aReasoning = a.capabilities.reasoning ? 1 : 0;
+    const bReasoning = b.capabilities.reasoning ? 1 : 0;
+    return aReasoning - bReasoning;
+  });
 
   // Cheapest overall
   const cheapest = sorted[0];
@@ -139,6 +152,11 @@ export function storeValidatedWorkerModel(result: WorkerModelResult): void {
     .run(key, value, value);
 }
 
+/** Clear a stored worker model validation (e.g. when the model is deprecated). */
+export function clearValidatedWorkerModel(providerID: string): void {
+  db().query("DELETE FROM kv_meta WHERE key = ?").run(`${KV_PREFIX}${providerID}`);
+}
+
 /**
  * Check whether the stored validation is stale (fingerprint mismatch).
  */
@@ -237,6 +255,122 @@ export function parseJudgeScore(response: string): number | null {
   return parseInt(match[1], 10);
 }
 
+// ---------------------------------------------------------------------------
+// Validation orchestration
+// ---------------------------------------------------------------------------
+
+import { DISTILLATION_SYSTEM, distillationUser } from "./prompt";
+import type { LLMClient } from "./types";
+
+export type ValidationInput = {
+  llm: LLMClient;
+  providerID: string;
+  sessionModelID: string;
+  candidates: ModelInfo[];
+  /** Recent gen-0 distillation to use as reference (observations text). */
+  referenceObservations: string;
+  /** Source messages text for re-running distillation with candidates. */
+  sourceMessagesText: string;
+  /** Date string for the distillation prompt. */
+  date: string;
+};
+
+/**
+ * Run the two-phase quality validation for worker model candidates.
+ * Returns the cheapest passing candidate, or null if none pass.
+ */
+export async function runValidation(
+  input: ValidationInput,
+): Promise<WorkerModelResult | null> {
+  const { llm, candidates, referenceObservations, sourceMessagesText, date } = input;
+
+  const userPrompt = distillationUser({
+    messages: sourceMessagesText,
+    date,
+  });
+
+  for (const candidate of candidates) {
+    // Skip the session model — it produced the reference, no need to test
+    if (candidate.id === input.sessionModelID) continue;
+
+    // Phase 1: run distillation with candidate model
+    let candidateObservations: string | null = null;
+    try {
+      const raw = await llm.prompt(DISTILLATION_SYSTEM, userPrompt, {
+        model: { providerID: candidate.providerID, modelID: candidate.id },
+        workerID: "lore-distill",
+        thinking: false,
+      });
+      if (raw) {
+        // Parse <observations>...</observations> block
+        const match = raw.match(/<observations>([\s\S]*?)<\/observations>/);
+        candidateObservations = match ? match[1].trim() : raw.trim();
+      }
+    } catch (e) {
+      log.warn(`worker model validation: candidate ${candidate.id} failed:`, e);
+      continue;
+    }
+
+    const structural = structuralCheck(candidateObservations, referenceObservations);
+    if (!structural.passed) {
+      log.info(
+        `worker model validation: ${candidate.id} failed structural check: ${structural.reason}`,
+      );
+      continue;
+    }
+
+    // Phase 2: LLM judge (using session model)
+    let judgeScore: number | null = null;
+    try {
+      const judgeResponse = await llm.prompt(
+        WORKER_JUDGE_SYSTEM,
+        workerJudgeUser(referenceObservations, candidateObservations!),
+        { workerID: "lore-distill", thinking: false }, // use session model (no model override)
+      );
+      if (judgeResponse) {
+        judgeScore = parseJudgeScore(judgeResponse);
+      }
+    } catch (e) {
+      log.warn(`worker model validation: judge call failed for ${candidate.id}:`, e);
+    }
+
+    if (judgeScore !== null && judgeScore < 3) {
+      log.info(
+        `worker model validation: ${candidate.id} failed judge (score=${judgeScore})`,
+      );
+      continue;
+    }
+
+    // Candidate passed both phases
+    const fingerprint = computeModelFingerprint(
+      input.providerID,
+      input.sessionModelID,
+      candidates.map((c) => c.id),
+    );
+
+    const result: WorkerModelResult = {
+      modelID: candidate.id,
+      providerID: candidate.providerID,
+      fingerprint,
+      validatedAt: Date.now(),
+      judgeScore,
+    };
+    storeValidatedWorkerModel(result);
+    log.info(
+      `worker model validated: ${candidate.id} (judge=${judgeScore}) for provider ${input.providerID}`,
+    );
+    return result;
+  }
+
+  // No candidate passed — clear any stale stored result so we don't keep
+  // routing worker calls to a potentially-deprecated model.
+  clearValidatedWorkerModel(input.providerID);
+  log.info(
+    `worker model validation: no candidate passed for ${input.providerID} — cleared stale entry`,
+  );
+  return null;
+}
+
 // ---------------------------------------------------------------------------
 // Effective worker model resolution
 // ---------------------------------------------------------------------------
@@ -253,9 +387,12 @@ export function resolveWorkerModel(
   // Explicit override wins
   if (configWorkerModel) return configWorkerModel;
 
-  // Check for validated auto-selection
+  // Check for validated auto-selection.
+  // Don't trust entries older than 24h — model may have been deprecated.
+  // Validation will re-run on next idle cycle and either re-confirm or clear.
   const validated = getValidatedWorkerModel(providerID);
-  if (validated) {
+  const MAX_AGE_MS = 24 * 60 * 60 * 1000;
+  if (validated && Date.now() - validated.validatedAt <= MAX_AGE_MS) {
     return { providerID: validated.providerID, modelID: validated.modelID };
   }