crewly 1.20.194 → 1.20.196

package/dist/backend/backend/src/services/experiments/experiment.service.js

@@ -10,7 +10,8 @@
  * resolves the agent's prediction, appends the result to the wiki
  * experiment log and tells the owner.
  *
- * The experiment is a trace root: `traceId` (`exp:EXP-n`) names the run, and
+ * The experiment is a trace root: `traceId` (a run trace, `tr-…`; the ticket's
+ * trace when its ticket already has one) names the run, and
  * the timeline records every step (created, shipped, baseline, each fetch
  * failure, measured, logged, reported), so the whole run can be reviewed.
  *
@@ -27,6 +28,7 @@ import { EXPERIMENT_CONSTANTS } from '../../constants.js';
 import { LoggerService } from '../core/logger.service.js';
 import { isExperimentSource, isExperimentStatus, } from '../../types/experiment.types.js';
 import { decideVerdict, defaultDirection, experimentWindows, formatValue } from './experiment-verdict.js';
+import { startExperimentTrace, traceExperimentEvent } from '../trace/trace-recorder.js';
 /** An error with an HTTP status. */
 export class ExperimentError extends Error {
     status;
@@ -40,6 +42,22 @@ export class ExperimentError extends Error {
         this.name = 'ExperimentError';
     }
 }
+/**
+ * A ship time: a valid ISO instant that is not in the future.
+ *
+ * @param raw - Value
+ * @param now - Current time
+ * @returns ISO time
+ * @throws ExperimentError(400)
+ */
+function parseShipTime(raw, now) {
+    const t = Date.parse(raw);
+    if (Number.isNaN(t))
+        throw new ExperimentError(400, 'shippedAt must be an ISO time');
+    if (t > now.getTime())
+        throw new ExperimentError(400, 'shippedAt is in the future');
+    return new Date(t).toISOString();
+}
 /**
  * Error text.
  *
@@ -49,6 +67,34 @@ export class ExperimentError extends Error {
 function errText(err) {
     return err instanceof Error ? err.message : String(err);
 }
+/**
+ * A local calendar day (the autopilot's days; the budget resets at local midnight).
+ *
+ * @param d - Time
+ * @returns YYYY-MM-DD
+ */
+export function localDay(d) {
+    const pad = (n) => String(n).padStart(2, '0');
+    return `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())}`;
+}
+/**
+ * The process windows of an autopilot card, in local days: the observation
+ * window starts ON the start day (the autopilot works that day too), the
+ * baseline is the equal window right before it.
+ *
+ * @param startedAt - When the card started (ISO)
+ * @param windowDays - Days per window
+ * @returns Baseline, observation, and when the observation window is over
+ */
+export function processWindows(startedAt, windowDays) {
+    const t = new Date(startedAt);
+    const at = (delta) => new Date(t.getFullYear(), t.getMonth(), t.getDate() + delta, 12);
+    return {
+        baseline: { start: localDay(at(-windowDays)), end: localDay(at(-1)) },
+        observation: { start: localDay(at(0)), end: localDay(at(windowDays - 1)) },
+        dueAtMs: new Date(t.getFullYear(), t.getMonth(), t.getDate() + windowDays, 0, 0, 0, 0).getTime(),
+    };
+}
 /**
  * Trimmed, length-capped string, or undefined.
  *
@@ -61,6 +107,30 @@ function text(v) {
     const t = v.trim();
     return t ? t.slice(0, EXPERIMENT_CONSTANTS.MAX_TEXT_LENGTH) : undefined;
 }
+/**
+ * A GA4 landing-page filter as a path: GA4 reports landing pages as paths, so
+ * a full URL is reduced to its path and any query string / fragment dropped
+ * (`https://site/x/?utm=1` → `/x/`).
+ *
+ * @param page - Page as given
+ * @returns The path
+ * @throws ExperimentError(400) when it is a URL that cannot be read
+ */
+export function ga4PagePath(page) {
+    let p = page.trim();
+    if (/^[a-z][a-z0-9+.-]*:\/\//i.test(p)) {
+        try {
+            p = new URL(p).pathname;
+        }
+        catch {
+            throw new ExperimentError(400, `metric.page is not a valid URL: ${page} (for ga4 give the landing page path, e.g. /h1b-guide)`);
+        }
+    }
+    p = p.split('#')[0].split('?')[0];
+    if (!p.startsWith('/'))
+        p = `/${p}`;
+    return p;
+}
 /**
  * Validate a metric definition.
  *
@@ -84,7 +154,7 @@ export function validateMetric(raw) {
     const out = { source: m.source, measure: m.measure, config };
     const page = text(m.page);
     if (page)
-        out.page = page;
+        out.page = m.source === 'ga4' ? ga4PagePath(page) : page;
     if (m.pageMatch !== undefined) {
         if (m.pageMatch !== 'exact' && m.pageMatch !== 'contains')
             throw new ExperimentError(400, 'metric.pageMatch must be exact or contains');
@@ -103,7 +173,7 @@ export function validateMetric(raw) {
     }
     const event = text(m.event);
     if (out.measure === 'events' && !event)
-        throw new ExperimentError(400, 'metric.event is required for ga4 events (e.g. generate_lead)');
+        throw new ExperimentError(400, 'metric.event is required for ga4 events (e.g. generate_lead); for every key event use measure conversions');
     if (event)
         out.event = event;
     const channel = text(m.channel);
@@ -196,11 +266,62 @@ export function resultSummary(e) {
     }
     if (e.baseline && e.result)
         lines.push(`Baseline ${e.baseline.start}..${e.baseline.end} · Result ${e.result.start}..${e.result.end}`);
+    if (e.autopilot)
+        lines.push(...autopilotSummaryLines(e.autopilot));
     if (e.ticket)
         lines.push(`Ticket: ${ticketLabel(e.ticket)}`);
-    lines.push(`Trace: ${e.traceId}`);
     return lines.join('\n');
 }
+/**
+ * Duration in words.
+ *
+ * @param ms - Milliseconds
+ * @returns "3h 10m" / "45m"
+ */
+function durationText(ms) {
+    const m = Math.round(ms / 60_000);
+    if (m < 60)
+        return `${m}m`;
+    const h = Math.floor(m / 60);
+    return m % 60 ? `${h}h ${m % 60}m` : `${h}h`;
+}
+/**
+ * The process numbers as one line.
+ *
+ * @param p - Process summary
+ * @returns e.g. "9 tickets shipped · 1.2 owner touches per ticket · stalls 3h 10m · $2.10 per shipped ticket"
+ */
+export function processLine(p) {
+    return [
+        `${p.ticketsShipped} ticket${p.ticketsShipped === 1 ? '' : 's'} shipped (${p.ticketsStarted} started)`,
+        `${p.ownerTouchesPerTicket === null ? `${p.ownerTouches} owner touches` : `${p.ownerTouchesPerTicket} owner touches per ticket`}`,
+        `stalls ${p.stalls === 0 ? 'none' : `${p.stalls} (${durationText(p.stallMs)})`}`,
+        p.costPerShippedTicket === null ? `$${p.costUsd.toFixed(2)} spent` : `$${p.costPerShippedTicket.toFixed(2)} per shipped ticket`,
+    ].join(' · ');
+}
+/**
+ * The autopilot part of a result: the other outcome metrics and the process.
+ *
+ * @param a - Scope
+ * @returns Lines
+ */
+export function autopilotSummaryLines(a) {
+    const lines = [];
+    if (a.outcomes.length > 0) {
+        lines.push('Other metrics:');
+        for (const o of a.outcomes) {
+            const word = o.verdict === 'worked' ? 'worked' : o.verdict === 'didnt' ? "didn't work" : o.verdict ? 'inconclusive' : 'not measured';
+            lines.push(`- ${metricLabel(o.metric)}: ${o.verdictReason ?? (o.lastError ? `fetch failed (${o.lastError.slice(0, 120)})` : 'n/a')} — ${word}`);
+        }
+    }
+    const scope = `autopilot of ${a.projectName}${a.label ? `, label ${a.label}` : ''}`;
+    if (a.processResult)
+        lines.push(`Process (${scope}): ${a.processResult.noData ? 'no autopilot work recorded' : processLine(a.processResult)}`);
+    // A baseline window without autopilot traces is no data, not zeros: left out.
+    if (a.processBaseline && !a.processBaseline.noData)
+        lines.push(`Process before: ${processLine(a.processBaseline)}`);
+    return lines;
+}
 /**
  * Experiment cards: store, lifecycle and the measuring tick.
  */
@@ -213,6 +334,10 @@ export class ExperimentService {
     timer = null;
     /** A tick is running (fetches can be slow; ticks never overlap) */
     ticking = false;
+    /** Check-ins whose reads / send keep failing: next try and failures so far */
+    checkInRetry = new Map();
+    /** Experiments being measured right now (a measureNow racing a tick fetches once) */
+    measuring = new Set();
     /**
      * @param deps - Dependencies
      */
@@ -331,6 +456,7 @@ export class ExperimentService {
      */
     record(e, event, detail) {
         e.timeline.push({ at: this.now().toISOString(), event, ...(detail ? { detail: detail.slice(0, 500) } : {}) });
+        traceExperimentEvent(e.traceId, e.id, event, detail);
         if (e.timeline.length > EXPERIMENT_CONSTANTS.MAX_TIMELINE)
             e.timeline.splice(0, e.timeline.length - EXPERIMENT_CONSTANTS.MAX_TIMELINE);
     }
@@ -414,14 +540,41 @@ export class ExperimentService {
         }
         const ticket = validateTicketLink(input.ticket);
         const title = text(input.title) ?? hypothesis.slice(0, 80);
+        const scope = await this.parseAutopilotScope(input, metric);
+        const startedAt = input.startedAt === undefined || input.startedAt === null || input.startedAt === '' ? undefined : parseShipTime(String(input.startedAt), this.now());
+        // An autopilot card starts at its creation (or startedAt): the baseline is
+        // the equal window before it (specs/2026-10-03-autopilot-experiments.md §3).
+        const explicitShip = scope
+            ? (startedAt ?? (input.shippedAt ? parseShipTime(String(input.shippedAt), this.now()) : this.now().toISOString()))
+            : input.shippedAt === undefined || input.shippedAt === null || input.shippedAt === ''
+                ? undefined
+                : parseShipTime(String(input.shippedAt), this.now());
+        // A ticket that is already done ships the card right away. Its done
+        // transition dates the ship; with no recorded done time the caller must
+        // say when the change went live (guessing would skew the baseline).
+        let ticketDoneAt;
+        if (ticket && this.deps.ticketShipState && !explicitShip) {
+            const state = await this.deps.ticketShipState(ticket).catch((err) => {
+                this.logger.debug('Could not read the linked ticket (non-fatal)', { ticket: ticketLabel(ticket), error: errText(err) });
+                return { done: false };
+            });
+            if (state.done && !state.at) {
+                throw new ExperimentError(400, `Ticket ${ticketLabel(ticket)} is already done but has no recorded done time; pass shippedAt (the ISO time the change went live)`);
+            }
+            if (state.done && state.at)
+                ticketDoneAt = new Date(Math.min(Date.parse(state.at), this.now().getTime())).toISOString();
+        }
         const created = await this.serial(async () => {
             const data = await this.load();
             const id = `${C.ID_PREFIX}${data.nextNumber}`;
             data.nextNumber += 1;
             const at = this.now().toISOString();
+            // The card's run: its ticket's trace, else a new `experiment` root
+            // (specs/2026-10-03-run-traces.md). `exp:EXP-n` only when tracing failed.
+            const traceId = startExperimentTrace({ id, title, ...(ticket ? { ticket: { kind: ticket.kind, id: ticket.id } } : {}) }, caller) ?? `exp:${id}`;
             const e = {
                 id,
-                traceId: `exp:${id}`,
+                traceId,
                 title,
                 hypothesis,
                 direction,
@@ -434,9 +587,13 @@ export class ExperimentService {
                 status: 'planned',
                 createdAt: at,
                 updatedAt: at,
+                ...(scope ? { autopilot: scope } : {}),
                 timeline: [],
             };
             this.record(e, 'created', `by ${caller}${ticket ? ` on ${ticketLabel(ticket)}` : ''}`);
+            if (scope) {
+                this.record(e, 'autopilot_scope', `autopilot of ${scope.projectName}${scope.label ? `, label ${scope.label}` : ''}; outcome metrics: ${[metricLabel(metric), ...scope.outcomes.map((o) => metricLabel(o.metric))].join('; ')}`);
+            }
             data.experiments.push(e);
             await this.save(data);
             return e;
@@ -445,8 +602,231 @@ export class ExperimentService {
             await this.deps.noteOnTicket(ticket, `Experiment ${created.id}: ${hypothesis} (metric: ${metricLabel(metric)}, ${windowDays}-day window; measured automatically after ship)`).catch((err) => this.logger.debug('Could not note the experiment on its ticket (non-fatal)', { id: created.id, error: errText(err) }));
         }
         this.logger.info('Experiment created', { id: created.id, caller, ticket: ticket ? ticketLabel(ticket) : undefined });
+        if (explicitShip)
+            return this.ship(created.id, caller, explicitShip);
+        if (ticketDoneAt && ticket)
+            return this.ship(created.id, `ticket ${ticketLabel(ticket)} done`, ticketDoneAt);
         return created;
     }
+    /**
+     * The autopilot scope of a new card, validated.
+     *
+     * @param input - Create input
+     * @param primary - The primary metric (its config is the default of the extra metrics)
+     * @returns Scope, or undefined for an ordinary card
+     * @throws ExperimentError(400)
+     */
+    async parseAutopilotScope(input, primary) {
+        if (input.autopilot === undefined || input.autopilot === null || input.autopilot === '' || input.autopilot === false) {
+            if (input.metrics !== undefined && input.metrics !== null)
+                throw new ExperimentError(400, 'metrics (extra outcome metrics) are for autopilot cards: add autopilot: {project}');
+            return undefined;
+        }
+        if (!this.deps.autopilot)
+            throw new ExperimentError(400, 'Autopilot experiments are not available (the ticket autopilot is not running)');
+        const raw = (typeof input.autopilot === 'object' ? input.autopilot : {});
+        const ref = text(raw.project) ?? text(raw.projectId);
+        if (!ref)
+            throw new ExperimentError(400, 'autopilot.project is required (project id, name or path)');
+        let project;
+        try {
+            project = await this.deps.autopilot.resolveProject(ref);
+        }
+        catch (err) {
+            throw new ExperimentError(400, `autopilot.project: ${errText(err)}`);
+        }
+        const label = text(raw.label);
+        const list = input.metrics === undefined || input.metrics === null ? [] : input.metrics;
+        if (!Array.isArray(list))
+            throw new ExperimentError(400, 'metrics must be a list of metric definitions');
+        if (list.length > EXPERIMENT_CONSTANTS.MAX_EXTRA_METRICS)
+            throw new ExperimentError(400, `at most ${EXPERIMENT_CONSTANTS.MAX_EXTRA_METRICS} extra metrics`);
+        const outcomes = [];
+        for (const [i, m] of list.entries()) {
+            const withConfig = m && typeof m === 'object' ? { config: primary.config, ...m } : m;
+            try {
+                const metric = validateMetric(withConfig);
+                if (metric.config !== primary.config && this.deps.fileExists && !(await this.deps.fileExists(metric.config))) {
+                    throw new ExperimentError(400, `seo-ops config not found: ${metric.config}`);
+                }
+                outcomes.push({ metric });
+            }
+            catch (err) {
+                throw new ExperimentError(400, `metrics[${i}]: ${errText(err)}`);
+            }
+        }
+        return { projectId: project.id, projectName: project.name, ...(label ? { label } : {}), outcomes, checkIns: 0 };
+    }
+    /**
+     * Autopilot cards: fetch the extra outcome metrics' baselines and the
+     * process baseline that are still missing (retried by the tick).
+     *
+     * @param id - EXP-n
+     */
+    async captureAutopilotBaseline(id) {
+        const e = await this.get(id);
+        if (!e || e.status !== 'running' || !e.autopilot || !e.shippedAt)
+            return;
+        const fetched = [];
+        for (const [index, o] of e.autopilot.outcomes.entries()) {
+            if (o.baseline)
+                continue;
+            const ow = experimentWindows(e.shippedAt, e.windowDays, o.metric.source);
+            try {
+                fetched.push({ index, m: await this.deps.fetchMetric(o.metric, ow.baseline) });
+            }
+            catch (err) {
+                fetched.push({ index, error: errText(err).slice(0, 500) });
+            }
+        }
+        if (fetched.length === 0)
+            return;
+        await this.mutate(id, (x) => {
+            if (!x.autopilot)
+                return;
+            for (const f of fetched) {
+                const o = x.autopilot.outcomes[f.index];
+                if (!o || o.baseline)
+                    continue;
+                if (f.m) {
+                    o.baseline = f.m;
+                    delete o.lastError;
+                    this.record(x, 'outcome_baseline', `${metricLabel(o.metric)}: ${formatValue(f.m.total, o.metric.measure)} over ${f.m.start}..${f.m.end}`);
+                }
+                else if (o.lastError !== f.error) {
+                    o.lastError = f.error;
+                    this.record(x, 'fetch_failed', `baseline of ${metricLabel(o.metric)}: ${f.error}`);
+                }
+            }
+        });
+    }
+    /**
+     * Autopilot cards: the process baseline (as soon as the card starts) and
+     * the process result (once the observation window's last local day is
+     * over), each recorded once, independent of the outcome metrics. A failed
+     * read is retried by the next tick; a window with no autopilot traces is
+     * stored as no-data, never as zeros.
+     *
+     * @param id - EXP-n
+     */
+    async captureProcess(id) {
+        const e = await this.get(id);
+        if (!e || e.status !== 'running' || !e.autopilot || !e.shippedAt || !this.deps.autopilot)
+            return;
+        const w = processWindows(e.shippedAt, e.windowDays);
+        const label = e.autopilot.label ?? null;
+        const read = async (range, what) => this.deps.autopilot?.process(e.autopilot.projectId, label, range).catch((err) => {
+            this.logger.debug(`Could not read the autopilot process ${what}`, { id, error: errText(err) });
+            return undefined;
+        });
+        const base = e.autopilot.processBaseline ? undefined : await read(w.baseline, 'baseline');
+        const result = !e.autopilot.processResult && this.now().getTime() >= w.dueAtMs ? await read(w.observation, 'result') : undefined;
+        if (!base && !result)
+            return;
+        await this.mutate(id, (x) => {
+            if (!x.autopilot)
+                return;
+            if (base && !x.autopilot.processBaseline) {
+                x.autopilot.processBaseline = base;
+                this.record(x, 'process_baseline', base.noData ? `no autopilot work in ${w.baseline.start}..${w.baseline.end}` : processLine(base));
+            }
+            if (result && !x.autopilot.processResult) {
+                x.autopilot.processResult = result;
+                this.record(x, 'process_result', result.noData ? `no autopilot work in ${w.observation.start}..${w.observation.end}` : processLine(result));
+            }
+        });
+    }
+    /**
+     * Autopilot cards whose outcome fetch keeps failing: remind the owner at
+     * most once a week (the first notice is {@link fetchFailed}'s).
+     *
+     * @param id - EXP-n
+     */
+    async remindStuck(id) {
+        const e = await this.get(id);
+        if (!e || !e.autopilot || !e.stuckReported || !this.deps.notifyOwner)
+            return;
+        const last = Date.parse(e.stuckNoticeAt ?? '') || 0;
+        if (this.now().getTime() - last < EXPERIMENT_CONSTANTS.CHECK_IN_INTERVAL_MS)
+            return;
+        const process = e.autopilot.processResult && !e.autopilot.processResult.noData ? `\nProcess so far: ${processLine(e.autopilot.processResult)}` : '';
+        const sent = await this.deps
+            .notifyOwner({
+            title: `Experiment ${e.id} still can't fetch its metric`,
+            message: `${e.title}\nMetric: ${metricLabel(e.metric)}\nStill failing: ${e.lastError ?? 'unknown error'}\nFix the seo-ops config or credentials (${e.metric.config}).${process}`,
+            urgent: false,
+        })
+            .catch(() => false);
+        if (sent)
+            await this.mutate(id, (x) => {
+                x.stuckNoticeAt = this.now().toISOString();
+                this.record(x, 'stuck_reminder', x.lastError ?? '');
+            });
+    }
+    /**
+     * Autopilot cards: one short owner note per week while running.
+     *
+     * @param id - EXP-n
+     * @returns True when a check-in went out
+     */
+    async checkIn(id) {
+        const e = await this.get(id);
+        if (!e || e.status !== 'running' || !e.autopilot || !e.shippedAt || !this.deps.autopilot)
+            return false;
+        const nowMs = this.now().getTime();
+        if (e.dueAt && Date.parse(e.dueAt) <= nowMs)
+            return false;
+        const week = Math.floor((nowMs - Date.parse(e.shippedAt)) / EXPERIMENT_CONSTANTS.CHECK_IN_INTERVAL_MS);
+        if (week < 1 || week <= e.autopilot.checkIns)
+            return false;
+        const retry = this.checkInRetry.get(id);
+        if (retry && nowMs < retry.at)
+            return false;
+        const backOff = () => {
+            const failures = (retry?.failures ?? 0) + 1;
+            const C = EXPERIMENT_CONSTANTS;
+            this.checkInRetry.set(id, { failures, at: nowMs + Math.min(C.CHECK_IN_RETRY_MAX_MS, C.CHECK_IN_RETRY_MIN_MS * 2 ** (failures - 1)) });
+            return false;
+        };
+        const w = experimentWindows(e.shippedAt, e.windowDays, e.metric.source);
+        // Process numbers in local days, start day included (the autopilot's days).
+        const pw = processWindows(e.shippedAt, e.windowDays);
+        const today = localDay(this.now());
+        const soFar = { start: pw.observation.start, end: today < pw.observation.end ? today : pw.observation.end };
+        const process = await this.deps.autopilot.process(e.autopilot.projectId, e.autopilot.label ?? null, soFar).catch(() => null);
+        // No process numbers: no note (it would say nothing); retry with a backoff.
+        if (!process)
+            return backOff();
+        // The primary metric so far: only days that have settled.
+        const lag = EXPERIMENT_CONSTANTS.SOURCE_LAG_DAYS[e.metric.source] ?? 0;
+        const settled = new Date(nowMs - (lag + 1) * 24 * 60 * 60 * 1000).toISOString().slice(0, 10);
+        const metricEnd = settled < w.observation.end ? settled : w.observation.end;
+        let metricSoFar = null;
+        if (metricEnd >= w.observation.start) {
+            metricSoFar = await this.deps.fetchMetric(e.metric, { start: w.observation.start, end: metricEnd }).catch(() => null);
+        }
+        const lines = [`${e.id} week ${week}: ${e.title}`];
+        lines.push(process.noData ? 'No autopilot work recorded yet.' : processLine(process));
+        if (metricSoFar) {
+            const days = Math.round((Date.parse(`${metricEnd}T00:00:00Z`) - Date.parse(`${w.observation.start}T00:00:00Z`)) / (24 * 60 * 60 * 1000)) + 1;
+            const base = e.baseline ? ` (baseline ${formatValue(e.baseline.total, e.metric.measure)} over ${e.windowDays} days)` : '';
+            lines.push(`${metricLabel(e.metric)}: ${formatValue(metricSoFar.total, e.metric.measure)} over the first ${days} day${days === 1 ? '' : 's'}${base}`);
+        }
+        lines.push(`Result due ${e.dueAt?.slice(0, 10) ?? 'later'}.`);
+        const message = lines.join('\n');
+        const sent = this.deps.notifyOwner ? await this.deps.notifyOwner({ title: `Experiment ${e.id}: week ${week}`, message, urgent: false }).catch(() => false) : true;
+        if (!sent)
+            return backOff();
+        this.checkInRetry.delete(id);
+        await this.mutate(id, (x) => {
+            if (!x.autopilot)
+                return;
+            x.autopilot.checkIns = week;
+            x.autopilot.lastCheckInAt = this.now().toISOString();
+            this.record(x, 'check_in', `week ${week}: ${process.noData ? 'no autopilot work yet' : processLine(process)}${metricSoFar ? `; ${metricLabel(e.metric)} so far ${formatValue(metricSoFar.total, e.metric.measure)}` : ''}`);
+        });
+        return true;
+    }
     /**
      * Mark an experiment shipped: fix its windows, record the prediction and
      * capture the baseline.
@@ -458,12 +838,7 @@ export class ExperimentService {
      * @throws ExperimentError(404 / 409 / 400)
      */
     async ship(id, caller, shippedAt) {
-        const at = shippedAt ?? this.now().toISOString();
-        const t = Date.parse(at);
-        if (Number.isNaN(t))
-            throw new ExperimentError(400, 'shippedAt must be an ISO time');
-        if (t > this.now().getTime())
-            throw new ExperimentError(400, 'shippedAt is in the future');
+        const t = Date.parse(parseShipTime(shippedAt ?? this.now().toISOString(), this.now()));
         await this.mutate(id, (e) => {
             if (e.status !== 'planned')
                 throw new ExperimentError(409, `${e.id} is ${e.status}; only a planned experiment can ship`);
@@ -473,10 +848,13 @@ export class ExperimentService {
             e.dueAt = w.dueAt;
             e.fetchAttempts = 0;
             e.stuckReported = false;
+            delete e.lastFetchAt;
             this.record(e, 'shipped', `by ${caller}; baseline ${w.baseline.start}..${w.baseline.end}, result ${w.observation.start}..${w.observation.end}, due ${w.dueAt.slice(0, 10)}`);
         });
         await this.recordPrediction(id);
         await this.captureBaseline(id);
+        // Autopilot cards: the process baseline does not wait for the outcome fetch.
+        await this.captureProcess(id);
         return (await this.get(id));
     }
     /**
@@ -550,18 +928,33 @@ export class ExperimentService {
         const nowMs = this.now().getTime();
         for (const e of all.experiments) {
             try {
-                if (e.status === 'planned' && e.ticket && this.deps.ticketShippedAt) {
-                    const at = await this.deps.ticketShippedAt(e.ticket);
-                    if (at) {
-                        const t = Math.min(Date.parse(at) || nowMs, nowMs);
+                if (e.status === 'planned' && e.ticket && this.deps.ticketShipState) {
+                    const state = await this.deps.ticketShipState(e.ticket);
+                    if (state.done && state.at) {
+                        const t = Math.min(Date.parse(state.at) || nowMs, nowMs);
                         await this.ship(e.id, `ticket ${ticketLabel(e.ticket)} done`, new Date(t).toISOString());
                         shipped += 1;
                     }
+                    else if (state.done) {
+                        await this.shipTimeUnknown(e.id);
+                    }
                     continue;
                 }
                 if (e.status === 'running') {
+                    // Autopilot process numbers and check-ins run on their own schedule:
+                    // a failing outcome fetch (e.g. missing credentials) never holds them.
+                    if (e.autopilot) {
+                        await this.captureProcess(e.id);
+                        await this.checkIn(e.id);
+                        await this.remindStuck(e.id);
+                    }
+                    // After MAX_FETCH_ATTEMPTS failures in a row, retry once a day.
+                    if (this.backingOff(e, nowMs))
+                        continue;
                     if (!e.baseline)
                         await this.captureBaseline(e.id);
+                    if (e.autopilot)
+                        await this.captureAutopilotBaseline(e.id);
                     if (e.dueAt && Date.parse(e.dueAt) <= nowMs) {
                         const after = await this.get(e.id);
                         if (after?.baseline) {
@@ -583,6 +976,49 @@ export class ExperimentService {
     // ---------------------------------------------------------------------------
     // Steps
     // ---------------------------------------------------------------------------
+    /**
+     * Whether a running experiment's fetches are backing off: after
+     * MAX_FETCH_ATTEMPTS failures in a row the tick retries once every
+     * FETCH_BACKOFF_MS instead of every tick.
+     *
+     * @param e - Experiment
+     * @param nowMs - Now
+     * @returns True to skip this tick
+     */
+    backingOff(e, nowMs) {
+        if ((e.fetchAttempts ?? 0) < EXPERIMENT_CONSTANTS.MAX_FETCH_ATTEMPTS || !e.lastFetchAt)
+            return false;
+        return nowMs - Date.parse(e.lastFetchAt) < EXPERIMENT_CONSTANTS.FETCH_BACKOFF_MS;
+    }
+    /**
+     * The linked ticket is done but its done time is not recorded: the card
+     * cannot be dated, so it waits for an explicit ship. Recorded (and the
+     * owner told) once.
+     *
+     * @param id - EXP-n
+     */
+    async shipTimeUnknown(id) {
+        let first = null;
+        await this.mutate(id, (e) => {
+            if (e.status !== 'planned' || e.timeline.some((t) => t.event === 'ship_time_unknown'))
+                return;
+            this.record(e, 'ship_time_unknown', 'the linked ticket is done but has no recorded done time; ship it with shippedAt');
+            first = { ...e };
+        });
+        if (!first)
+            return;
+        const e = first;
+        this.logger.warn('Experiment ticket is done without a done time; waiting for an explicit ship', { id });
+        if (this.deps.notifyOwner && e.ticket) {
+            await this.deps
+                .notifyOwner({
+                title: `Experiment ${e.id} needs its ship time`,
+                message: `${e.title}\nTicket ${ticketLabel(e.ticket)} is done, but its log has no done time, so the baseline can't be dated. Ship it with the time the change went live: experiment-card ship --id ${e.id} --shipped-at <ISO time>.`,
+                urgent: false,
+            })
+                .catch(() => false);
+        }
+    }
     /**
      * Record the creating agent's prediction that the hypothesis holds.
      *
@@ -614,22 +1050,27 @@ export class ExperimentService {
         let tell = null;
         await this.mutate(id, (e) => {
             e.fetchAttempts = (e.fetchAttempts ?? 0) + 1;
+            e.lastFetchAt = this.now().toISOString();
             e.lastError = errText(err).slice(0, 500);
             this.record(e, 'fetch_failed', `${step}: ${e.lastError}`);
             if (e.fetchAttempts >= EXPERIMENT_CONSTANTS.MAX_FETCH_ATTEMPTS && !e.stuckReported) {
                 e.stuckReported = true;
+                e.stuckNoticeAt = this.now().toISOString();
                 tell = { ...e };
             }
         });
         if (tell && this.deps.notifyOwner) {
             const e = tell;
-            await this.deps
+            const sent = await this.deps
                 .notifyOwner({
                 title: `Experiment ${e.id} can't fetch its ${step}`,
-                message: `${e.title}\nMetric: ${metricLabel(e.metric)}\nThe last ${EXPERIMENT_CONSTANTS.MAX_FETCH_ATTEMPTS} tries failed: ${e.lastError}\nIt keeps retrying; fix the seo-ops config or credentials (${e.metric.config}).`,
+                message: `${e.title}\nMetric: ${metricLabel(e.metric)}\nThe last ${EXPERIMENT_CONSTANTS.MAX_FETCH_ATTEMPTS} tries failed: ${e.lastError}\nIt now retries once a day; fix the seo-ops config or credentials (${e.metric.config}).`,
                 urgent: false,
             })
                 .catch(() => false);
+            // Not delivered: try again on the next failure.
+            if (!sent)
+                await this.mutate(id, (x) => { x.stuckReported = false; }).catch(() => undefined);
         }
     }
     /**
@@ -651,12 +1092,20 @@ export class ExperimentService {
             return;
         }
         await this.mutate(id, (x) => {
+            // A concurrent measureNow / tick may have captured it meanwhile.
+            if (x.status !== 'running' || x.baseline)
+                return;
             x.baseline = m;
             x.fetchAttempts = 0;
             x.stuckReported = false;
             delete x.lastError;
+            delete x.lastFetchAt;
             this.record(x, 'baseline_captured', `${formatValue(m.total, x.metric.measure)} over ${m.start}..${m.end} (volume ${m.volume})`);
         });
+        if (e.autopilot) {
+            await this.captureAutopilotBaseline(id);
+            await this.captureProcess(id);
+        }
     }
     /**
      * Fetch the observation window and label the result.
@@ -664,6 +1113,22 @@ export class ExperimentService {
      * @param id - EXP-n
      */
     async measure(id) {
+        if (this.measuring.has(id))
+            return;
+        this.measuring.add(id);
+        try {
+            await this.measureOnce(id);
+        }
+        finally {
+            this.measuring.delete(id);
+        }
+    }
+    /**
+     * One measurement (see {@link measure}).
+     *
+     * @param id - EXP-n
+     */
+    async measureOnce(id) {
         const e = await this.get(id);
         if (!e || e.status !== 'running' || !e.baseline || !e.shippedAt)
             return;
@@ -676,8 +1141,27 @@ export class ExperimentService {
             await this.fetchFailed(id, 'result', err);
             return;
         }
+        // Autopilot cards: the extra outcome metrics and the process numbers of
+        // the same window (a failed extra fetch never blocks the result).
+        const extras = [];
+        let process = null;
+        if (e.autopilot) {
+            for (const o of e.autopilot.outcomes) {
+                try {
+                    extras.push({ m: await this.deps.fetchMetric(o.metric, experimentWindows(e.shippedAt, e.windowDays, o.metric.source).observation) });
+                }
+                catch (err) {
+                    extras.push({ error: errText(err).slice(0, 500) });
+                }
+            }
+            if (this.deps.autopilot && !e.autopilot.processResult) {
+                process = await this.deps.autopilot.process(e.autopilot.projectId, e.autopilot.label ?? null, processWindows(e.shippedAt, e.windowDays).observation).catch(() => null);
+            }
+        }
         await this.mutate(id, (x) => {
-            if (!x.baseline)
+            // Re-checked under the store lock: a racing measureNow / tick may
+            // have measured it already.
+            if (x.status !== 'running' || !x.baseline)
                 return;
             const v = decideVerdict(x.metric.measure, x.direction, x.baseline, m);
             x.result = m;
@@ -686,7 +1170,33 @@ export class ExperimentService {
             x.status = 'done';
             x.fetchAttempts = 0;
             delete x.lastError;
+            delete x.lastFetchAt;
             this.record(x, 'measured', `${v.verdict}: ${v.reason}`);
+            if (x.autopilot) {
+                for (const [i, o] of x.autopilot.outcomes.entries()) {
+                    const got = extras[i];
+                    if (!got)
+                        continue;
+                    if (got.m) {
+                        o.result = got.m;
+                        delete o.lastError;
+                        if (o.baseline) {
+                            const ov = decideVerdict(o.metric.measure, defaultDirection(o.metric.measure), o.baseline, got.m);
+                            o.verdict = ov.verdict;
+                            o.verdictReason = ov.reason;
+                        }
+                        this.record(x, 'outcome_result', `${metricLabel(o.metric)}: ${o.verdictReason ?? `${formatValue(got.m.total, o.metric.measure)} (no baseline)`}`);
+                    }
+                    else {
+                        o.lastError = got.error;
+                        this.record(x, 'fetch_failed', `result of ${metricLabel(o.metric)}: ${got.error}`);
+                    }
+                }
+                if (process && !x.autopilot.processResult) {
+                    x.autopilot.processResult = process;
+                    this.record(x, 'process_result', process.noData ? 'no autopilot work in the window' : processLine(process));
+                }
+            }
         });
         this.logger.info('Experiment measured', { id });
     }
@@ -722,14 +1232,29 @@ export class ExperimentService {
                 });
         }
         if (!e.reportedAt && this.deps.notifyOwner) {
-            const sent = await this.deps
-                .notifyOwner({ title: `Experiment ${e.id} ${verdictWord(e)}`, message: summary, urgent: false })
-                .catch(() => false);
-            if (sent)
+            // Claim the report under the store lock before sending, so a racing
+            // followUp cannot send it twice; release the claim when not delivered.
+            const claimAt = this.now().toISOString();
+            let claimed = false;
+            await this.mutate(id, (x) => {
+                if (x.reportedAt)
+                    return;
+                x.reportedAt = claimAt;
+                claimed = true;
+            });
+            if (claimed) {
+                const sent = await this.deps
+                    .notifyOwner({ title: `Experiment ${e.id} ${verdictWord(e)}`, message: summary, urgent: false })
+                    .catch(() => false);
                 await this.mutate(id, (x) => {
-                    x.reportedAt = this.now().toISOString();
-                    this.record(x, 'reported', 'owner');
+                    if (x.reportedAt !== claimAt)
+                        return;
+                    if (sent)
+                        this.record(x, 'reported', 'owner');
+                    else
+                        delete x.reportedAt;
                 });
+            }
         }
         if (e.ticket && this.deps.noteOnTicket && !e.timeline.some((t) => t.event === 'ticket_noted')) {
             try {