clementine-agent 1.18.77 → 1.18.78

package/dist/agent/goal-evaluator.d.ts

@@ -0,0 +1,54 @@
+/**
+ * Goal evaluation — PRD Phase 1.
+ *
+ * Two evaluators run at the END of a successful cron run, when the Task
+ * defines `successSchema` (JSON Schema validated against the agent's output)
+ * and/or `successCriteriaText` (free-text criterion graded by an evaluator
+ * sub-agent). The verdicts merge into a single `goalCheck` object that
+ * gets stamped on the run's CronRunEntry.
+ *
+ * Design constraints:
+ * - Never block run completion. Any thrown error becomes status='error' on
+ *   goalCheck and the rest of the run logs unchanged.
+ * - Bounded budgets — schema validation is sub-millisecond; evaluator agent
+ *   gets max_turns=1, ~30s wall clock, Haiku-class model.
+ * - No new top-level deps — ajv is a transitive install; we import it lazily
+ *   inside the function so test fixtures that don't need it never load it.
+ */
+import type { CronJobDefinition, CronRunEntry } from '../types.js';
+type SchemaResult = {
+    pass: boolean;
+    errors: string[];
+    tried: boolean;
+};
+type EvaluatorResult = {
+    pass: boolean;
+    reason: string;
+};
+/**
+ * Validate the agent's response against a JSON Schema. Returns:
+ *  - tried=false if no JSON could be extracted from the response
+ *  - tried=true with pass + errors otherwise
+ * Schema-compile errors throw — caller catches.
+ */
+export declare function validateAgainstSchema(responseText: string, schema: Record<string, unknown>): Promise<SchemaResult>;
+/**
+ * Ask a small evaluator sub-agent whether the run accomplished the
+ * `successCriteriaText` criterion. Returns null if the evaluator failed
+ * to produce a parseable verdict (caller treats null as goalCheck.status='error').
+ *
+ * The evaluator is intentionally minimal — Haiku, max_turns=1, focused
+ * system prompt, ~30s budget. We're grading text, not running tools.
+ */
+export declare function evaluateAgainstCriterion(responseText: string, criterion: string, opts?: {
+    model?: string;
+    timeoutMs?: number;
+}): Promise<EvaluatorResult | null>;
+/**
+ * Orchestrator: runs whichever evaluators are configured on the Task and
+ * merges their verdicts into a single goalCheck record. Returns undefined
+ * when no goal is configured — the field then stays absent on the run entry.
+ */
+export declare function runGoalCheck(responseText: string, job: CronJobDefinition): Promise<CronRunEntry['goalCheck']>;
+export {};
+//# sourceMappingURL=goal-evaluator.d.ts.map

package/dist/agent/goal-evaluator.js

@@ -0,0 +1,235 @@
+/**
+ * Goal evaluation — PRD Phase 1.
+ *
+ * Two evaluators run at the END of a successful cron run, when the Task
+ * defines `successSchema` (JSON Schema validated against the agent's output)
+ * and/or `successCriteriaText` (free-text criterion graded by an evaluator
+ * sub-agent). The verdicts merge into a single `goalCheck` object that
+ * gets stamped on the run's CronRunEntry.
+ *
+ * Design constraints:
+ * - Never block run completion. Any thrown error becomes status='error' on
+ *   goalCheck and the rest of the run logs unchanged.
+ * - Bounded budgets — schema validation is sub-millisecond; evaluator agent
+ *   gets max_turns=1, ~30s wall clock, Haiku-class model.
+ * - No new top-level deps — ajv is a transitive install; we import it lazily
+ *   inside the function so test fixtures that don't need it never load it.
+ */
+/**
+ * Try to extract a JSON object from the agent's response. Looks first at the
+ * whole text, then at fenced ```json blocks (the common Claude output shape),
+ * then at any {...} substring as a last resort.
+ */
+function extractJson(responseText) {
+    if (!responseText || typeof responseText !== 'string')
+        return null;
+    // Whole-text parse first.
+    try {
+        return JSON.parse(responseText);
+    }
+    catch { /* fall through */ }
+    // Fenced ```json ... ``` block.
+    const fenced = responseText.match(/```(?:json|JSON)?\s*([\s\S]*?)```/);
+    if (fenced && fenced[1]) {
+        try {
+            return JSON.parse(fenced[1].trim());
+        }
+        catch { /* fall through */ }
+    }
+    // First {...} substring (greedy through last brace).
+    const first = responseText.indexOf('{');
+    const last = responseText.lastIndexOf('}');
+    if (first >= 0 && last > first) {
+        try {
+            return JSON.parse(responseText.slice(first, last + 1));
+        }
+        catch { /* fall through */ }
+    }
+    return null;
+}
+/**
+ * Validate the agent's response against a JSON Schema. Returns:
+ *  - tried=false if no JSON could be extracted from the response
+ *  - tried=true with pass + errors otherwise
+ * Schema-compile errors throw — caller catches.
+ */
+export async function validateAgainstSchema(responseText, schema) {
+    const candidate = extractJson(responseText);
+    if (candidate === null) {
+        return { tried: false, pass: false, errors: ['No JSON object found in agent response'] };
+    }
+    // Lazy import so this module costs nothing when no Task has a schema.
+    const ajvMod = await import('ajv').catch(() => null);
+    if (!ajvMod) {
+        throw new Error('ajv not available — cannot validate success_schema');
+    }
+    // Handle CJS default-export interop (ajv@8 ships as CJS; the ESM bridge
+    // sometimes lands the constructor on .default and sometimes at the top
+    // level).
+    const AjvCtor = ajvMod.default ?? ajvMod;
+    const ajv = new AjvCtor({ allErrors: true, strict: false });
+    const validator = ajv.compile(schema);
+    const ok = validator(candidate);
+    if (ok)
+        return { tried: true, pass: true, errors: [] };
+    // ajv stamps errors on the compiled validator; the instance fallback covers
+    // older versions that put them on the ajv instance instead.
+    const rawErrors = validator.errors ?? ajv.errors ?? [];
+    const errs = rawErrors.slice(0, 5).map((e) => {
+        const path = e.instancePath || '';
+        const msg = e.message || 'invalid';
+        return path ? `${path} ${msg}` : msg;
+    });
+    return { tried: true, pass: false, errors: errs.length ? errs : ['validation failed'] };
+}
+/**
+ * Ask a small evaluator sub-agent whether the run accomplished the
+ * `successCriteriaText` criterion. Returns null if the evaluator failed
+ * to produce a parseable verdict (caller treats null as goalCheck.status='error').
+ *
+ * The evaluator is intentionally minimal — Haiku, max_turns=1, focused
+ * system prompt, ~30s budget. We're grading text, not running tools.
+ */
+export async function evaluateAgainstCriterion(responseText, criterion, opts = {}) {
+    const trimmedResponse = (responseText || '').slice(0, 8000);
+    const trimmedCriterion = (criterion || '').slice(0, 2000);
+    if (!trimmedCriterion)
+        return null;
+    const sdk = await import('@anthropic-ai/claude-agent-sdk').catch(() => null);
+    if (!sdk || typeof sdk.query !== 'function') {
+        return null;
+    }
+    const systemPrompt = 'You are a strict evaluator. Grade whether a scheduled task accomplished its stated goal.\n' +
+        'Reply with EXACTLY one line in this format:\n' +
+        'PASS — <one-sentence reason> | FAIL — <one-sentence reason>\n' +
+        'Be honest. If the run did not achieve the goal, say FAIL even if the agent claimed success.';
+    const userPrompt = `GOAL:\n${trimmedCriterion}\n\nRUN OUTPUT:\n${trimmedResponse}\n\nVerdict:`;
+    const timeoutMs = opts.timeoutMs ?? 30_000;
+    const model = opts.model ?? 'claude-haiku-4-5-20251001';
+    // Race the SDK query against a hard timeout so a hung evaluator never
+    // blocks run logging.
+    const queryPromise = (async () => {
+        let collected = '';
+        try {
+            const queryFn = sdk.query;
+            const iter = queryFn({
+                prompt: userPrompt,
+                options: {
+                    systemPrompt,
+                    model,
+                    maxTurns: 1,
+                    permissionMode: 'default',
+                    allowedTools: [],
+                    settingSources: [],
+                    // No tools, no network beyond model — purely text-in / text-out.
+                },
+            });
+            for await (const message of iter) {
+                const m = message;
+                if (m.type === 'assistant' && Array.isArray(m.content)) {
+                    for (const block of m.content) {
+                        const b = block;
+                        if (b.type === 'text' && typeof b.text === 'string')
+                            collected += b.text;
+                    }
+                }
+                else if (m.type === 'result' && typeof m.result === 'string') {
+                    collected += m.result;
+                }
+            }
+        }
+        catch {
+            return null;
+        }
+        return collected;
+    })();
+    const timeoutPromise = new Promise((resolve) => setTimeout(() => resolve(null), timeoutMs));
+    const collected = await Promise.race([queryPromise, timeoutPromise]);
+    if (!collected || typeof collected !== 'string')
+        return null;
+    // Parse the strict verdict line. Accept variants: "PASS — reason", "FAIL: reason",
+    // "Verdict: PASS — reason", etc.
+    const match = collected.match(/\b(PASS|FAIL)\b\s*[—\-:]?\s*(.+)/i);
+    if (!match)
+        return null;
+    const verdict = match[1].toUpperCase() === 'PASS';
+    const reason = (match[2] || '').replace(/[\r\n].*$/s, '').trim().slice(0, 280);
+    return { pass: verdict, reason: reason || (verdict ? 'Pass' : 'Fail') };
+}
+/**
+ * Orchestrator: runs whichever evaluators are configured on the Task and
+ * merges their verdicts into a single goalCheck record. Returns undefined
+ * when no goal is configured — the field then stays absent on the run entry.
+ */
+export async function runGoalCheck(responseText, job) {
+    const hasSchema = !!(job.successSchema && Object.keys(job.successSchema).length > 0);
+    const hasCriterion = !!(job.successCriteriaText && job.successCriteriaText.trim());
+    if (!hasSchema && !hasCriterion)
+        return undefined;
+    let schemaResult = null;
+    let evaluatorResult = null;
+    let errored = false;
+    let errorMessage = '';
+    if (hasSchema) {
+        try {
+            schemaResult = await validateAgainstSchema(responseText, job.successSchema);
+        }
+        catch (err) {
+            errored = true;
+            errorMessage = `schema validator threw: ${String(err).slice(0, 200)}`;
+        }
+    }
+    if (hasCriterion) {
+        try {
+            evaluatorResult = await evaluateAgainstCriterion(responseText, job.successCriteriaText);
+            if (evaluatorResult === null && !errored) {
+                // Treat unparseable evaluator output as 'error' rather than 'fail' — we
+                // don't want a flaky evaluator to mark a healthy run as failed.
+                errored = true;
+                errorMessage = 'evaluator did not return a parseable PASS/FAIL verdict';
+            }
+        }
+        catch (err) {
+            errored = true;
+            errorMessage = `evaluator threw: ${String(err).slice(0, 200)}`;
+        }
+    }
+    // Decide overall status. Both passed = pass. Either failed = fail. Neither
+    // ran cleanly but both were configured = error.
+    const mode = hasSchema && hasCriterion ? 'both' : hasSchema ? 'schema' : 'evaluator';
+    let status;
+    if (errored && (!schemaResult || !evaluatorResult)) {
+        status = 'error';
+    }
+    else {
+        const schemaPassed = schemaResult?.pass !== false; // true if not run, or true if run + passed
+        const evaluatorPassed = evaluatorResult?.pass !== false; // same
+        const schemaFailed = schemaResult ? !schemaResult.pass || !schemaResult.tried : false;
+        const evaluatorFailed = evaluatorResult ? !evaluatorResult.pass : false;
+        if (schemaFailed || evaluatorFailed)
+            status = 'fail';
+        else if (schemaPassed && evaluatorPassed)
+            status = 'pass';
+        else
+            status = 'error';
+    }
+    const out = { status, mode };
+    if (schemaResult) {
+        out.schemaPass = schemaResult.pass && schemaResult.tried;
+        if (!schemaResult.pass || !schemaResult.tried) {
+            out.schemaErrors = schemaResult.errors.slice(0, 5);
+        }
+    }
+    if (evaluatorResult) {
+        out.evaluatorPass = evaluatorResult.pass;
+        out.evaluatorReason = evaluatorResult.reason;
+    }
+    if (errored && errorMessage) {
+        // Stash the error in evaluatorReason if we don't already have one — the
+        // dashboard surfaces this string in the tooltip.
+        if (!out.evaluatorReason)
+            out.evaluatorReason = errorMessage;
+    }
+    return out;
+}
+//# sourceMappingURL=goal-evaluator.js.map

package/dist/cli/cron.js

@@ -140,7 +140,7 @@ export async function cmdCronRun(jobName) {
     try {
         const response = await gateway.handleCronJob(job.name, job.prompt, job.tier, job.maxTurns, job.model, job.workDir, job.mode, job.maxHours);
         const finishedAt = new Date();
-        runLog.append({
+        const entry = {
             jobName: job.name,
             startedAt: startedAt.toISOString(),
             finishedAt: finishedAt.toISOString(),
@@ -148,7 +148,21 @@ export async function cmdCronRun(jobName) {
             durationMs: finishedAt.getTime() - startedAt.getTime(),
             attempt: 1,
             outputPreview: response ? response.slice(0, 200) : undefined,
-        });
+        };
+        // PRD Phase 1.1: goal-orientation evaluator (mirrors the daemon path).
+        if (job.successSchema || (job.successCriteriaText && job.successCriteriaText.trim())) {
+            try {
+                const { runGoalCheck } = await import('../agent/goal-evaluator.js');
+                const goalCheck = await runGoalCheck(response ?? '', job);
+                if (goalCheck)
+                    entry.goalCheck = goalCheck;
+            }
+            catch (err) {
+                // Never block logging on evaluator failure.
+                entry.goalCheck = { status: 'error', mode: 'evaluator', evaluatorReason: `evaluator orchestrator threw: ${String(err).slice(0, 200)}` };
+            }
+        }
+        runLog.append(entry);
         console.log(response || '(no output)');
         if (response && response !== '__NOTHING__') {
             console.log('\n(Note: Standalone runner — output not delivered to channels. Use the daemon for channel delivery.)');

package/dist/cli/dashboard.js

@@ -23274,6 +23274,22 @@ function renderScheduledTaskCard(task) {
     var ok = lr.status === 'ok';
     var statusIcon = ok ? '<span style="color:var(--green)">&#10003;</span>' : '<span style="color:var(--red)">&#10007;</span>';
     lastRunHtml = statusIcon + ' ' + esc(lr.status || 'unknown') + ' · ' + esc(timeAgo(lr.finishedAt || lr.startedAt || ''));
+    // PRD Phase 1.1: goal pill. Orthogonal to status — a run can be status='ok'
+    // but goalCheck.status='fail' (the agent finished cleanly without
+    // accomplishing the stated goal). That's exactly the failure mode the
+    // PRD's goal-orientation feature is designed to surface.
+    if (lr.goalCheck) {
+      var gc = lr.goalCheck;
+      var gIcon = gc.status === 'pass' ? '🎯' : gc.status === 'fail' ? '✗' : gc.status === 'error' ? '⚠' : '';
+      var gColor = gc.status === 'pass' ? 'var(--green)' : gc.status === 'fail' ? 'var(--red)' : 'var(--yellow)';
+      var gLabel = gc.status === 'pass' ? 'goal met' : gc.status === 'fail' ? 'goal not met' : gc.status === 'error' ? 'goal eval failed' : '';
+      var gTip = '';
+      if (gc.evaluatorReason) gTip = gc.evaluatorReason;
+      else if (Array.isArray(gc.schemaErrors) && gc.schemaErrors.length) gTip = 'Schema errors: ' + gc.schemaErrors.join('; ');
+      if (gIcon && gLabel) {
+        lastRunHtml += ' <span style="color:' + gColor + ';font-size:11px;font-weight:500" title="' + esc(gTip || gLabel) + '">· ' + gIcon + ' ' + esc(gLabel) + '</span>';
+      }
+    }
     // "ran with: …" — surface the skills + MCP that were live for this run.
     var ranWith = [];
     if (Array.isArray(lr.skillsApplied) && lr.skillsApplied.length > 0) {
@@ -23380,8 +23396,23 @@ function renderRecentHistoryList(runs) {
       var preview = String(entry.outputPreview).slice(0, 140);
       errorPreview = '<div style="font-size:11px;color:var(--text-muted);margin-top:2px;word-break:break-word">' + esc(preview) + '</div>';
     }
-    rowsHtml += '<div class="history-row" data-trace-job="' + esc(jobName) + '" style="display:grid;grid-template-columns:24px minmax(180px,1.2fr) minmax(180px,1fr) 90px auto;gap:10px;align-items:start;padding:8px 14px;border-bottom:1px solid var(--border);cursor:pointer">'
+    // PRD Phase 1.1: goal cell. Empty cell when no goal configured (status='skipped'
+    // returned by runGoalCheck means "no goal" — but we omit goalCheck entirely
+    // in that case, so missing field == no goal). The cell stays present in the
+    // grid for column alignment.
+    var goalCellHtml = '<div></div>';
+    if (entry.goalCheck) {
+      var gc2 = entry.goalCheck;
+      var gIcon2 = gc2.status === 'pass' ? '🎯' : gc2.status === 'fail' ? '✗' : gc2.status === 'error' ? '⚠' : '';
+      var gColor2 = gc2.status === 'pass' ? 'var(--green)' : gc2.status === 'fail' ? 'var(--red)' : 'var(--yellow)';
+      var gTip2 = gc2.evaluatorReason
+        ? gc2.evaluatorReason
+        : (Array.isArray(gc2.schemaErrors) && gc2.schemaErrors.length ? 'Schema errors: ' + gc2.schemaErrors.join('; ') : gc2.status);
+      goalCellHtml = '<div style="color:' + gColor2 + ';font-size:13px;line-height:18px;text-align:center" title="' + esc(gTip2) + '">' + gIcon2 + '</div>';
+    }
+    rowsHtml += '<div class="history-row" data-trace-job="' + esc(jobName) + '" style="display:grid;grid-template-columns:24px 24px minmax(180px,1.2fr) minmax(180px,1fr) 90px auto;gap:10px;align-items:start;padding:8px 14px;border-bottom:1px solid var(--border);cursor:pointer">'
       + '<div style="color:' + statusColor + ';font-size:14px;line-height:18px;text-align:center" title="' + esc(status) + '">' + statusIcon + '</div>'
+      + goalCellHtml
       + '<div style="min-width:0">'
         + '<div style="font-weight:500;color:var(--text-primary);font-size:13px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap" title="' + esc(jobName) + '">' + esc(jobName) + attemptLabel + '</div>'
         + errorPreview
@@ -23392,8 +23423,10 @@ function renderRecentHistoryList(runs) {
       + '</div>';
   }
   return '<div class="history-list" style="background:var(--bg-secondary);border:1px solid var(--border);border-radius:var(--radius)">'
-    + '<div style="display:grid;grid-template-columns:24px minmax(180px,1.2fr) minmax(180px,1fr) 90px auto;gap:10px;padding:8px 14px;border-bottom:1px solid var(--border);font-size:11px;color:var(--text-muted);text-transform:uppercase;letter-spacing:0.04em;font-weight:500">'
-      + '<div></div><div>Task</div><div>Started</div><div>Duration</div><div></div>'
+    + '<div style="display:grid;grid-template-columns:24px 24px minmax(180px,1.2fr) minmax(180px,1fr) 90px auto;gap:10px;padding:8px 14px;border-bottom:1px solid var(--border);font-size:11px;color:var(--text-muted);text-transform:uppercase;letter-spacing:0.04em;font-weight:500">'
+      + '<div title="Run status (ok / error / etc.)"></div>'
+      + '<div title="Goal check result — orthogonal to run status">Goal</div>'
+      + '<div>Task</div><div>Started</div><div>Duration</div><div></div>'
     + '</div>'
     + rowsHtml
     + '</div>';

package/dist/gateway/cron-scheduler.js

@@ -1236,6 +1236,23 @@ export class CronScheduler {
                             this.gateway.injectContext(`discord:user:${DISCORD_OWNER_ID}`, `[Scheduled cron: ${job.name}]`, response);
                         }
                     }
+                    // PRD Phase 1.1: goal-orientation. If the Task has successSchema or
+                    // successCriteriaText, run the evaluator now (before logging) so the
+                    // entry carries the goalCheck verdict. Errors here NEVER block
+                    // logging — runGoalCheck catches its own throws and emits
+                    // status='error' on the goalCheck instead.
+                    if (job.successSchema || (job.successCriteriaText && job.successCriteriaText.trim())) {
+                        try {
+                            const { runGoalCheck } = await import('../agent/goal-evaluator.js');
+                            const goalCheck = await runGoalCheck(response ?? '', job);
+                            if (goalCheck)
+                                entry.goalCheck = goalCheck;
+                        }
+                        catch (err) {
+                            logger.warn({ err, job: job.name }, 'Goal evaluator failed — proceeding without goalCheck');
+                            entry.goalCheck = { status: 'error', mode: 'evaluator', evaluatorReason: `evaluator orchestrator threw: ${String(err).slice(0, 200)}` };
+                        }
+                    }
                     this._logRun(entry);
                     this.logAutonomy('completed', job, { durationMs: entry.durationMs, deliveryFailed: entry.deliveryFailed, advisorApplied: !!advisorApplied });
                     // Fire-and-forget: extract procedural skill from successful long-running cron jobs

package/dist/types.d.ts

@@ -447,6 +447,24 @@ export interface CronRunEntry {
     allowedToolsApplied?: string[];
     /** MCP servers live for this run (post profile + trick allowlist intersection). */
     mcpServersApplied?: string[];
+    /** PRD Phase 1: did the run accomplish what it was supposed to?
+     *  Computed at run-end when the Task has successSchema or successCriteriaText.
+     *  - status='pass'      both configured checks passed (or the only one configured did)
+     *  - status='fail'      a configured check failed
+     *  - status='skipped'   no goal configured on the Task (don't show the pill)
+     *  - status='error'     evaluator/validator threw; does NOT mark the run failed
+     *  This is orthogonal to CronRunEntry.status — a run can be status='ok' with
+     *  goalCheck.status='fail' (the agent finished cleanly but didn't accomplish
+     *  the stated goal), and that's the failure mode the PRD is designed to surface. */
+    goalCheck?: {
+        status: 'pass' | 'fail' | 'skipped' | 'error';
+        /** Which evaluators ran. 'both' means schema + evaluator agreed. */
+        mode: 'schema' | 'evaluator' | 'both';
+        schemaPass?: boolean;
+        schemaErrors?: string[];
+        evaluatorPass?: boolean;
+        evaluatorReason?: string;
+    };
 }
 export interface Models {
     haiku: string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clementine-agent",
-  "version": "1.18.77",
+  "version": "1.18.78",
   "description": "Clementine — Personal AI Assistant (TypeScript)",
   "type": "module",
   "main": "dist/index.js",