@rudderhq/server 0.2.0-canary.9 → 0.2.0

package/dist/services/runtime-kernel/heartbeat.js

@@ -20,7 +20,7 @@ import { summarizeRuntimeSkillsForTrace } from "../runtime-trace-metadata.js";
 import { buildWorkspaceReadyComment, cleanupExecutionWorkspaceArtifacts, ensureRuntimeServicesForRun, persistAdapterManagedRuntimeServices, realizeExecutionWorkspace, releaseRuntimeServicesForRun, } from "../workspace-runtime.js";
 import { issueService } from "../issues.js";
 import { documentService } from "../documents.js";
-import { buildIssueConvergenceReviewWakeupOptions } from "../issue-review-wakeup.js";
+import { buildIssueConvergenceReviewWakeupOptions, buildIssueReviewCloseoutWakeupOptions, } from "../issue-review-wakeup.js";
 import { executionWorkspaceService } from "../execution-workspaces.js";
 import { buildObservedRunLangfuseScores } from "../run-intelligence.js";
 import { workspaceOperationService } from "../workspace-operations.js";
@@ -48,6 +48,9 @@ const ISSUE_PASSIVE_FOLLOWUP_REASON = "issue_passive_followup";
 const ISSUE_PASSIVE_FOLLOWUP_WAKE_SOURCE = "passive_issue_followup";
 const ISSUE_PASSIVE_FOLLOWUP_FAILURE_REASON = "missing_closure";
 const ISSUE_PASSIVE_FOLLOWUP_MAX_ATTEMPTS = 2;
+const ISSUE_REVIEW_CLOSEOUT_REASON = "issue_review_closeout_missing";
+const ISSUE_REVIEW_CLOSEOUT_FAILURE_REASON = "missing_review_decision";
+const ISSUE_REVIEW_CLOSEOUT_MAX_ATTEMPTS = 2;
 const ISSUE_PASSIVE_FOLLOWUP_COOLDOWN_MS_BY_ATTEMPT = new Map([
     [1, 2 * 60 * 1000],
     [2, 5 * 60 * 1000],
@@ -184,15 +187,42 @@ export function buildHeartbeatAdapterInvokePayload(input) {
             .map((entry) => normalizeLoadedSkill(entry))
             .filter((entry) => Boolean(entry))
         : [];
-    const usedSkills = explicitUsedSkills.length > 0
-        ? explicitUsedSkills
-        : inferUsedSkillsFromPrompt(input.meta.prompt, input.runtimeSkills);
+    const promptRequestedSkills = inferUsedSkillsFromPrompt(input.meta.prompt, input.runtimeSkills);
+    const loadedSkills = Array.isArray(input.meta.loadedSkills) && input.meta.loadedSkills.length > 0
+        ? input.meta.loadedSkills
+            .map((entry) => normalizeLoadedSkillForPayload(entry))
+            .filter((entry) => Boolean(entry))
+        : input.runtimeSkills
+            .map((entry) => ({
+            key: entry.key,
+            runtimeName: entry.runtimeName ?? null,
+            name: entry.name ?? null,
+            description: entry.description ?? null,
+        }));
+    const loadedSkillEvidence = loadedSkills
+        .map((entry) => normalizeLoadedSkill(entry))
+        .filter((entry) => Boolean(entry));
+    const skillEvidence = resolveSkillEvidence({
+        usedSkills: explicitUsedSkills,
+        requestedSkills: promptRequestedSkills,
+        loadedSkills: loadedSkillEvidence,
+    });
     return {
         ...input.meta,
         ...summarizeRuntimeSkillsForTrace(input.runtimeSkills),
-        usedSkillCount: usedSkills.length,
-        usedSkillKeys: usedSkills.map((entry) => entry.key),
-        usedSkills,
+        loadedSkillCount: loadedSkills.length,
+        loadedSkillKeys: loadedSkills.map((entry) => entry.key),
+        loadedSkills,
+        usedSkillCount: explicitUsedSkills.length,
+        usedSkillKeys: explicitUsedSkills.map((entry) => entry.key),
+        usedSkills: explicitUsedSkills,
+        promptRequestedSkillCount: promptRequestedSkills.length,
+        promptRequestedSkillKeys: promptRequestedSkills.map((entry) => entry.key),
+        promptRequestedSkills,
+        skillEvidenceType: skillEvidence.evidence,
+        skillEvidenceCount: skillEvidence.skills.length,
+        skillEvidenceKeys: skillEvidence.skills.map((entry) => entry.key),
+        skillEvidenceSkills: skillEvidence.skills,
     };
 }
 function buildRecentDateKeys(windowDays, now) {
@@ -238,6 +268,151 @@ function normalizeLoadedSkill(value) {
     const label = rawRuntimeName ?? rawName ?? fallbackSkillLabel(key);
     return { key, label };
 }
+function normalizeLoadedSkillForPayload(value) {
+    const skill = parseObject(value);
+    const rawKey = readNonEmptyString(skill.key);
+    const rawRuntimeName = readNonEmptyString(skill.runtimeName);
+    const rawName = readNonEmptyString(skill.name);
+    const key = rawKey ?? rawRuntimeName ?? rawName;
+    if (!key)
+        return null;
+    return {
+        key,
+        runtimeName: rawRuntimeName ?? null,
+        name: rawName ?? null,
+        description: readNonEmptyString(skill.description) ?? null,
+    };
+}
+function emptySkillEvidenceCounts() {
+    return { used: 0, requested: 0, loaded: 0 };
+}
+function incrementSkillEvidenceCount(counts, evidence) {
+    counts[evidence] += 1;
+}
+function strongestSkillEvidence(left, right) {
+    const rank = {
+        used: 3,
+        requested: 2,
+        loaded: 1,
+    };
+    return rank[right] > rank[left] ? right : left;
+}
+function resolveSkillEvidence(input) {
+    if (input.usedSkills.length > 0)
+        return { evidence: "used", skills: input.usedSkills };
+    if (input.requestedSkills.length > 0)
+        return { evidence: "requested", skills: input.requestedSkills };
+    return { evidence: "loaded", skills: [] };
+}
+function readSkillEvidenceFromPayload(payload) {
+    const loadedSkills = Array.isArray(payload.loadedSkills)
+        ? payload.loadedSkills
+            .map((entry) => normalizeLoadedSkill(entry))
+            .filter((entry) => Boolean(entry))
+        : [];
+    const usedSkills = Array.isArray(payload.usedSkills)
+        ? payload.usedSkills
+            .map((entry) => normalizeLoadedSkill(entry))
+            .filter((entry) => Boolean(entry))
+        : [];
+    const requestedSkills = Array.isArray(payload.promptRequestedSkills)
+        ? payload.promptRequestedSkills
+            .map((entry) => normalizeLoadedSkill(entry))
+            .filter((entry) => Boolean(entry))
+        : inferUsedSkillsFromPrompt(payload.prompt, loadedSkills);
+    return resolveSkillEvidence({ usedSkills, requestedSkills, loadedSkills });
+}
+function extractSkillSlugFromPath(value) {
+    const normalized = value.trim().replace(/\\/g, "/").replace(/[?#].*$/u, "").replace(/\/+$/u, "");
+    if (!normalized.endsWith("/SKILL.md"))
+        return null;
+    const parts = normalized.split("/").filter(Boolean);
+    if (parts.length < 2)
+        return null;
+    const slug = parts.at(-2);
+    if (!slug || slug === "." || slug === "..")
+        return null;
+    return slug;
+}
+function collectSkillPathsFromText(value) {
+    const paths = [];
+    const pattern = /(?:^|[\s"'`(=])((?:\.{1,2}\/|~\/|\/)?(?:[^\s"'`()<>|;&/]+\/)+SKILL\.md(?:[?#][^\s"'`()<>|;&]*)?)(?=$|[\s"'`()<>|;&])/giu;
+    for (const match of value.matchAll(pattern)) {
+        const pathValue = match[1]?.trim();
+        if (pathValue)
+            paths.push(pathValue);
+    }
+    return paths;
+}
+function collectStringValues(value, depth = 0) {
+    if (depth > 4)
+        return [];
+    if (typeof value === "string")
+        return [value];
+    if (Array.isArray(value))
+        return value.flatMap((entry) => collectStringValues(entry, depth + 1));
+    const record = parseObject(value);
+    return Object.values(record).flatMap((entry) => collectStringValues(entry, depth + 1));
+}
+function normalizeSkillUseFromPath(value) {
+    const slug = extractSkillSlugFromPath(value);
+    if (!slug)
+        return null;
+    return { key: slug, label: slug };
+}
+function dedupeSkillUses(skills) {
+    const seen = new Set();
+    const result = [];
+    for (const skill of skills) {
+        const normalized = normalizeLoadedSkill(skill);
+        if (!normalized || seen.has(normalized.key))
+            continue;
+        seen.add(normalized.key);
+        result.push(normalized);
+    }
+    return result;
+}
+function collectSkillUsesFromText(value) {
+    return collectSkillPathsFromText(value)
+        .map((entry) => normalizeSkillUseFromPath(entry))
+        .filter((entry) => Boolean(entry));
+}
+function readToolCommandInput(input) {
+    if (typeof input === "string")
+        return input;
+    const record = parseObject(input);
+    return readNonEmptyString(record.command) ?? readNonEmptyString(record.cmd);
+}
+function isCommandTranscriptTool(name) {
+    const normalized = name.trim().toLowerCase();
+    return ["command_execution", "shell", "shelltoolcall", "bash"].includes(normalized);
+}
+function isReadTranscriptTool(name) {
+    const normalized = name.trim().toLowerCase();
+    if (isCommandTranscriptTool(normalized))
+        return true;
+    if (/(?:^|[_-])(read|fetch|open|cat)(?:$|[_-])/.test(normalized))
+        return true;
+    return false;
+}
+function inferUsedSkillsFromTranscript(transcript) {
+    const skills = [];
+    for (const entry of transcript) {
+        if (entry.kind !== "tool_call")
+            continue;
+        if (!isReadTranscriptTool(entry.name))
+            continue;
+        const command = readToolCommandInput(entry.input);
+        if (command) {
+            skills.push(...collectSkillUsesFromText(command));
+            continue;
+        }
+        for (const value of collectStringValues(entry.input)) {
+            skills.push(...collectSkillUsesFromText(value));
+        }
+    }
+    return dedupeSkillUses(skills);
+}
 function normalizeSkillCandidate(value) {
     return value
         ?.trim()
@@ -787,6 +962,20 @@ function normalizePassiveFollowupContext(raw) {
         queuedAt: readNonEmptyString(parsed.queuedAt),
     };
 }
+function normalizeReviewCloseoutContext(raw) {
+    const parsed = parseObject(raw);
+    const originRunId = readNonEmptyString(parsed.originRunId);
+    if (!originRunId)
+        return null;
+    const attempt = Math.max(0, Math.floor(asNumber(parsed.attempt, 0)));
+    return {
+        originRunId,
+        previousRunId: readNonEmptyString(parsed.previousRunId),
+        attempt,
+        maxAttempts: Math.max(1, Math.floor(asNumber(parsed.maxAttempts, ISSUE_REVIEW_CLOSEOUT_MAX_ATTEMPTS))),
+        reason: ISSUE_REVIEW_CLOSEOUT_FAILURE_REASON,
+    };
+}
 function passiveFollowupCooldownMs(attempt) {
     return ISSUE_PASSIVE_FOLLOWUP_COOLDOWN_MS_BY_ATTEMPT.get(attempt) ?? 5 * 60 * 1000;
 }
@@ -1783,6 +1972,15 @@ export function heartbeatService(db) {
             .then((rows) => rows[0] ?? null);
         return Boolean(commentActivity);
     }
+    async function runHasIssueReviewDecision(tx, run, issueId) {
+        const decisionActivity = await tx
+            .select({ id: activityLog.id })
+            .from(activityLog)
+            .where(and(eq(activityLog.orgId, run.orgId), eq(activityLog.action, "issue.review_decision_recorded"), eq(activityLog.entityType, "issue"), eq(activityLog.entityId, issueId), eq(activityLog.runId, run.id)))
+            .limit(1)
+            .then((rows) => rows[0] ?? null);
+        return Boolean(decisionActivity);
+    }
     async function issueHasDeferredWake(tx, orgId, issueId) {
         const deferred = await tx
             .select({ id: agentWakeupRequests.id })
@@ -1813,6 +2011,67 @@ export function heartbeatService(db) {
             .then((rows) => rows[0] ?? null);
         return Boolean(existingReview);
     }
+    async function reviewerCloseoutAlreadyRecorded(tx, runId) {
+        const existingWake = await tx
+            .select({ id: agentWakeupRequests.id })
+            .from(agentWakeupRequests)
+            .where(eq(agentWakeupRequests.idempotencyKey, `${ISSUE_REVIEW_CLOSEOUT_REASON}:${runId}`))
+            .limit(1)
+            .then((rows) => rows[0] ?? null);
+        if (existingWake)
+            return true;
+        const existingReview = await tx
+            .select({ id: activityLog.id })
+            .from(activityLog)
+            .where(and(eq(activityLog.runId, runId), eq(activityLog.action, "issue.review_closure_needs_operator_review")))
+            .limit(1)
+            .then((rows) => rows[0] ?? null);
+        return Boolean(existingReview);
+    }
+    async function issueHasConfirmedBlockedReviewerHandoff(tx, issue, reviewerAgentId) {
+        if (issue.status !== "blocked")
+            return false;
+        const existingHandoff = await tx
+            .select({ id: activityLog.id })
+            .from(activityLog)
+            .where(and(eq(activityLog.orgId, issue.orgId), eq(activityLog.action, "issue.review_decision_recorded"), eq(activityLog.entityType, "issue"), sql `${activityLog.entityId} = ${issue.id}::text`, eq(activityLog.actorType, "agent"), sql `${activityLog.actorId} = ${reviewerAgentId}::text`, sql `${activityLog.details} ->> 'decision' = 'blocked'`, sql `${activityLog.createdAt} >= COALESCE((
+            SELECT MAX(material_activity.created_at)
+            FROM activity_log material_activity
+            WHERE material_activity.org_id = ${issue.orgId}
+              AND material_activity.entity_type = 'issue'
+              AND material_activity.entity_id = ${issue.id}::text
+              AND (
+                (
+                  material_activity.action = 'issue.updated'
+                  AND jsonb_typeof(material_activity.details) = 'object'
+                  AND EXISTS (
+                    SELECT 1
+                    FROM jsonb_object_keys(material_activity.details) AS detail_key(key)
+                    WHERE detail_key.key NOT IN (
+                      'identifier',
+                      'issueIdentifier',
+                      '_previous',
+                      'source',
+                      'reopened',
+                      'reopenedFrom',
+                      'normalizedFromStatus',
+                      'normalizedReason'
+                    )
+                  )
+                )
+                OR (
+                  material_activity.action = 'issue.comment_added'
+                  AND NOT (
+                    material_activity.actor_type = 'agent'
+                    AND material_activity.actor_id = ${reviewerAgentId}::text
+                  )
+                )
+              )
+          ), to_timestamp(0))`))
+            .limit(1)
+            .then((rows) => rows[0] ?? null);
+        return Boolean(existingHandoff);
+    }
     async function evaluatePassiveIssueClosureForLockedIssue(input) {
         const { tx, run, issue, now } = input;
         const context = parseObject(run.contextSnapshot);
@@ -1821,6 +2080,49 @@ export function heartbeatService(db) {
             return { kind: "none", reason: "run_not_issue_backed" };
         if (run.status !== "succeeded")
             return { kind: "none", reason: "run_not_successful" };
+        const reviewerRun = (issue.status === "in_review" || issue.status === "blocked") &&
+            issue.reviewerAgentId === run.agentId &&
+            (run.invocationSource === "review" ||
+                readNonEmptyString(context.role) === "reviewer" ||
+                readNonEmptyString(context.wakeSource) === "review");
+        if (reviewerRun) {
+            if (await runHasIssueReviewDecision(tx, run, issue.id)) {
+                return { kind: "none", reason: "review_decision_recorded" };
+            }
+            if (await issueHasConfirmedBlockedReviewerHandoff(tx, issue, run.agentId)) {
+                return { kind: "none", reason: "blocked_reviewer_handoff_confirmed" };
+            }
+            if (await issueHasDeferredWake(tx, issue.orgId, issue.id)) {
+                return { kind: "none", reason: "deferred_issue_wake_exists" };
+            }
+            if (await reviewerCloseoutAlreadyRecorded(tx, run.id)) {
+                return { kind: "none", reason: "reviewer_closeout_already_recorded" };
+            }
+            const reviewCloseout = normalizeReviewCloseoutContext(context.reviewCloseout);
+            const currentAttempt = reviewCloseout?.attempt ?? 0;
+            const maxAttempts = reviewCloseout?.maxAttempts ?? ISSUE_REVIEW_CLOSEOUT_MAX_ATTEMPTS;
+            const originRunId = reviewCloseout?.originRunId ?? run.id;
+            if (currentAttempt >= maxAttempts) {
+                return {
+                    kind: "reviewer_closeout_operator_review",
+                    issue,
+                    originRunId,
+                    previousRunId: run.id,
+                    attempts: currentAttempt,
+                    maxAttempts,
+                    reason: ISSUE_REVIEW_CLOSEOUT_FAILURE_REASON,
+                };
+            }
+            return {
+                kind: "reviewer_closeout",
+                issue,
+                originRunId,
+                previousRunId: run.id,
+                attempts: currentAttempt + 1,
+                maxAttempts,
+                reason: ISSUE_REVIEW_CLOSEOUT_FAILURE_REASON,
+            };
+        }
         if (issue.status !== "todo" && issue.status !== "in_progress") {
             return { kind: "none", reason: "issue_has_closure_status" };
         }
@@ -2329,6 +2631,7 @@ export function heartbeatService(db) {
                 let finalObservationSessionId = heartbeatObservationContext.sessionKey ?? null;
                 const runtime = await ensureRuntimeState(agent);
                 const context = parseObject(run.contextSnapshot);
+                delete context.rudderGitIdentity;
                 const taskKey = deriveTaskKey(context, null);
                 const sessionCodec = getAgentRuntimeSessionCodec(agent.agentRuntimeType);
                 const issueId = readNonEmptyString(context.issueId);
@@ -3062,6 +3365,25 @@ export function heartbeatService(db) {
                     });
                     const finalizedRun = await getRun(run.id);
                     if (finalizedRun) {
+                        const transcriptUsedSkills = inferUsedSkillsFromTranscript(executionTranscript);
+                        if (transcriptUsedSkills.length > 0) {
+                            await appendRunEvent(finalizedRun, seq++, {
+                                eventType: "adapter.skill_usage",
+                                stream: "system",
+                                level: "info",
+                                message: "skill usage inferred from transcript",
+                                payload: {
+                                    source: "transcript.skill_file_read",
+                                    usedSkillCount: transcriptUsedSkills.length,
+                                    usedSkillKeys: transcriptUsedSkills.map((entry) => entry.key),
+                                    usedSkills: transcriptUsedSkills,
+                                    skillEvidenceType: "used",
+                                    skillEvidenceCount: transcriptUsedSkills.length,
+                                    skillEvidenceKeys: transcriptUsedSkills.map((entry) => entry.key),
+                                    skillEvidenceSkills: transcriptUsedSkills,
+                                },
+                            });
+                        }
                         await appendRunEvent(finalizedRun, seq++, {
                             eventType: "lifecycle",
                             stream: "system",
@@ -3528,6 +3850,96 @@ export function heartbeatService(db) {
                 });
             }
         }
+        else if (passiveClosure?.kind === "reviewer_closeout") {
+            await appendRunEvent(run, await nextRunEventSeq(run.id), {
+                eventType: "issue.review_closeout_missing",
+                stream: "system",
+                level: "warn",
+                message: "Reviewer run finished without a structured review decision",
+                payload: {
+                    issueId: passiveClosure.issue.id,
+                    originRunId: passiveClosure.originRunId,
+                    previousRunId: passiveClosure.previousRunId,
+                    attempts: passiveClosure.attempts,
+                    maxAttempts: passiveClosure.maxAttempts,
+                    reason: passiveClosure.reason,
+                },
+            });
+            await logActivity(db, {
+                orgId: passiveClosure.issue.orgId,
+                actorType: "system",
+                actorId: "issue_review_closeout_governance",
+                action: "issue.review_closeout_missing",
+                entityType: "issue",
+                entityId: passiveClosure.issue.id,
+                agentId: run.agentId,
+                runId: run.id,
+                details: {
+                    issueId: passiveClosure.issue.id,
+                    issueTitle: passiveClosure.issue.title,
+                    reviewerAgentId: passiveClosure.issue.reviewerAgentId,
+                    originRunId: passiveClosure.originRunId,
+                    previousRunId: passiveClosure.previousRunId,
+                    attempts: passiveClosure.attempts,
+                    maxAttempts: passiveClosure.maxAttempts,
+                    reason: passiveClosure.reason,
+                },
+            });
+            if (passiveClosure.issue.reviewerAgentId) {
+                await enqueueWakeup(passiveClosure.issue.reviewerAgentId, {
+                    ...buildIssueReviewCloseoutWakeupOptions({
+                        issue: passiveClosure.issue,
+                        contextSource: "issue.review_closeout_missing",
+                        originRunId: passiveClosure.originRunId,
+                        previousRunId: passiveClosure.previousRunId,
+                        attempts: passiveClosure.attempts,
+                        maxAttempts: passiveClosure.maxAttempts,
+                        requestedByActorType: "system",
+                        requestedByActorId: "issue_review_closeout_governance",
+                    }),
+                    idempotencyKey: `${ISSUE_REVIEW_CLOSEOUT_REASON}:${run.id}`,
+                }).catch((err) => {
+                    logger.warn({ err, issueId: passiveClosure.issue.id }, "failed to wake reviewer after missing review close-out");
+                    return null;
+                });
+            }
+        }
+        else if (passiveClosure?.kind === "reviewer_closeout_operator_review") {
+            await appendRunEvent(run, await nextRunEventSeq(run.id), {
+                eventType: "issue.review_closure_needs_operator_review",
+                stream: "system",
+                level: "warn",
+                message: "Reviewer close-out attempts stopped and need operator review",
+                payload: {
+                    issueId: passiveClosure.issue.id,
+                    originRunId: passiveClosure.originRunId,
+                    previousRunId: passiveClosure.previousRunId,
+                    attempts: passiveClosure.attempts,
+                    maxAttempts: passiveClosure.maxAttempts,
+                    reason: passiveClosure.reason,
+                },
+            });
+            await logActivity(db, {
+                orgId: passiveClosure.issue.orgId,
+                actorType: "system",
+                actorId: "issue_review_closeout_governance",
+                action: "issue.review_closure_needs_operator_review",
+                entityType: "issue",
+                entityId: passiveClosure.issue.id,
+                agentId: run.agentId,
+                runId: run.id,
+                details: {
+                    issueId: passiveClosure.issue.id,
+                    issueTitle: passiveClosure.issue.title,
+                    reviewerAgentId: passiveClosure.issue.reviewerAgentId,
+                    originRunId: passiveClosure.originRunId,
+                    previousRunId: passiveClosure.previousRunId,
+                    attempts: passiveClosure.attempts,
+                    maxAttempts: passiveClosure.maxAttempts,
+                    reason: passiveClosure.reason,
+                },
+            });
+        }
         const promotedRun = outcome.promotedRun;
         if (!promotedRun)
             return;
@@ -4333,60 +4745,184 @@ export function heartbeatService(db) {
         const windowEnd = new Date(`${endDate}T23:59:59.999Z`);
         const rows = await db
             .select({
+            runId: heartbeatRunEvents.runId,
             createdAt: heartbeatRunEvents.createdAt,
+            eventType: heartbeatRunEvents.eventType,
             payload: heartbeatRunEvents.payload,
         })
             .from(heartbeatRunEvents)
-            .where(and(eq(heartbeatRunEvents.orgId, scope.orgId), ...(scope.agentId ? [eq(heartbeatRunEvents.agentId, scope.agentId)] : []), eq(heartbeatRunEvents.eventType, "adapter.invoke"), gte(heartbeatRunEvents.createdAt, windowStart), lte(heartbeatRunEvents.createdAt, windowEnd)))
+            .where(and(eq(heartbeatRunEvents.orgId, scope.orgId), ...(scope.agentId ? [eq(heartbeatRunEvents.agentId, scope.agentId)] : []), inArray(heartbeatRunEvents.eventType, ["adapter.invoke", "adapter.skill_usage"]), gte(heartbeatRunEvents.createdAt, windowStart), lte(heartbeatRunEvents.createdAt, windowEnd)))
             .orderBy(asc(heartbeatRunEvents.createdAt), asc(heartbeatRunEvents.id));
         const days = new Map();
         for (const date of dateKeys) {
-            days.set(date, { totalCount: 0, runCount: 0, skills: new Map() });
+            days.set(date, { totalCount: 0, runCount: 0, evidenceCounts: emptySkillEvidenceCounts(), skills: new Map() });
         }
         const overallSkills = new Map();
+        const runEvidence = new Map();
         let totalCount = 0;
         let totalRunsWithSkills = 0;
-        for (const row of rows) {
-            const date = new Date(row.createdAt).toISOString().slice(0, 10);
-            const bucket = days.get(date);
-            if (!bucket)
-                continue;
-            const payload = parseObject(row.payload);
-            const loadedSkills = Array.isArray(payload.loadedSkills) ? payload.loadedSkills : [];
-            const usedSkills = Array.isArray(payload.usedSkills)
-                ? payload.usedSkills
-                : inferUsedSkillsFromPrompt(payload.prompt, loadedSkills);
-            if (usedSkills.length === 0)
-                continue;
-            const eventSkills = new Map();
-            for (const entry of usedSkills) {
+        const evidenceCounts = emptySkillEvidenceCounts();
+        function addRunSkillEvidence(runId, date, evidence) {
+            if (!days.has(date))
+                return;
+            if (evidence.skills.length === 0)
+                return;
+            const runBucket = runEvidence.get(runId) ?? { date, skills: new Map() };
+            for (const entry of evidence.skills) {
                 const normalized = normalizeLoadedSkill(entry);
                 if (!normalized)
                     continue;
-                if (!eventSkills.has(normalized.key)) {
-                    eventSkills.set(normalized.key, normalized.label);
+                const existing = runBucket.skills.get(normalized.key);
+                if (existing) {
+                    existing.evidence = strongestSkillEvidence(existing.evidence, evidence.evidence);
+                    if (existing.label === fallbackSkillLabel(existing.key) && normalized.label !== fallbackSkillLabel(normalized.key)) {
+                        existing.label = normalized.label;
+                    }
+                }
+                else {
+                    runBucket.skills.set(normalized.key, {
+                        key: normalized.key,
+                        label: normalized.label,
+                        evidence: evidence.evidence,
+                    });
+                }
+            }
+            if (runBucket.skills.size > 0)
+                runEvidence.set(runId, runBucket);
+        }
+        async function inferUsedSkillsFromStoredRunLog(row) {
+            if (row.logStore !== "local_file" || !row.logRef)
+                return [];
+            const adapter = (() => {
+                try {
+                    return getServerAdapter(row.agentRuntimeType);
+                }
+                catch {
+                    return null;
+                }
+            })();
+            if (!adapter)
+                return [];
+            const parser = adapter.parseStdoutLine ?? null;
+            if (!parser)
+                return [];
+            const limitBytes = Math.min(Math.max(row.logBytes ?? 0, 256_000), 2_000_000);
+            const read = await runLogStore
+                .read({ store: "local_file", logRef: row.logRef }, { limitBytes })
+                .catch(() => null);
+            if (!read?.content)
+                return [];
+            const transcript = [];
+            let stdoutBuffer = "";
+            let stderrBuffer = "";
+            for (const line of read.content.split("\n")) {
+                if (!line.trim())
+                    continue;
+                let raw;
+                try {
+                    raw = JSON.parse(line);
+                }
+                catch {
+                    continue;
+                }
+                const parsed = parseObject(raw);
+                const stream = parsed.stream === "stderr" ? "stderr" : parsed.stream === "stdout" ? "stdout" : null;
+                const chunk = typeof parsed.chunk === "string" ? parsed.chunk : "";
+                if (!stream || !chunk)
+                    continue;
+                if (stream === "stdout") {
+                    stdoutBuffer = appendTranscriptEntriesFromChunk({
+                        buffer: stdoutBuffer,
+                        chunk,
+                        transcript,
+                        parser,
+                        kind: "stdout",
+                    });
+                }
+                else {
+                    stderrBuffer = appendTranscriptEntriesFromChunk({
+                        buffer: stderrBuffer,
+                        chunk,
+                        transcript,
+                        kind: "stderr",
+                    });
                 }
             }
-            if (eventSkills.size === 0)
+            appendTranscriptEntriesFromChunk({
+                buffer: stdoutBuffer,
+                chunk: "",
+                transcript,
+                parser,
+                kind: "stdout",
+                finalize: true,
+            });
+            appendTranscriptEntriesFromChunk({
+                buffer: stderrBuffer,
+                chunk: "",
+                transcript,
+                kind: "stderr",
+                finalize: true,
+            });
+            return inferUsedSkillsFromTranscript(transcript);
+        }
+        for (const row of rows) {
+            const date = new Date(row.createdAt).toISOString().slice(0, 10);
+            const payload = parseObject(row.payload);
+            addRunSkillEvidence(row.runId, date, readSkillEvidenceFromPayload(payload));
+        }
+        const runRows = await db
+            .select({
+            id: heartbeatRuns.id,
+            agentRuntimeType: agents.agentRuntimeType,
+            createdAt: heartbeatRuns.createdAt,
+            logStore: heartbeatRuns.logStore,
+            logRef: heartbeatRuns.logRef,
+            logBytes: heartbeatRuns.logBytes,
+        })
+            .from(heartbeatRuns)
+            .innerJoin(agents, eq(agents.id, heartbeatRuns.agentId))
+            .where(and(eq(heartbeatRuns.orgId, scope.orgId), ...(scope.agentId ? [eq(heartbeatRuns.agentId, scope.agentId)] : []), gte(heartbeatRuns.createdAt, windowStart), lte(heartbeatRuns.createdAt, windowEnd)));
+        for (const row of runRows) {
+            const usedSkills = await inferUsedSkillsFromStoredRunLog(row);
+            if (usedSkills.length === 0)
+                continue;
+            addRunSkillEvidence(row.id, new Date(row.createdAt).toISOString().slice(0, 10), {
+                evidence: "used",
+                skills: usedSkills,
+            });
+        }
+        for (const runBucket of runEvidence.values()) {
+            const bucket = days.get(runBucket.date);
+            if (!bucket || runBucket.skills.size === 0)
                 continue;
             bucket.runCount += 1;
             totalRunsWithSkills += 1;
-            for (const [key, label] of eventSkills) {
+            for (const { key, label, evidence } of runBucket.skills.values()) {
                 bucket.totalCount += 1;
                 totalCount += 1;
+                incrementSkillEvidenceCount(bucket.evidenceCounts, evidence);
+                incrementSkillEvidenceCount(evidenceCounts, evidence);
                 const existingDaySkill = bucket.skills.get(key);
                 if (existingDaySkill) {
                     existingDaySkill.count += 1;
+                    existingDaySkill.evidence = strongestSkillEvidence(existingDaySkill.evidence, evidence);
+                    incrementSkillEvidenceCount(existingDaySkill.evidenceCounts, evidence);
                 }
                 else {
-                    bucket.skills.set(key, { key, label, count: 1 });
+                    const skillEvidenceCounts = emptySkillEvidenceCounts();
+                    incrementSkillEvidenceCount(skillEvidenceCounts, evidence);
+                    bucket.skills.set(key, { key, label, count: 1, evidence, evidenceCounts: skillEvidenceCounts });
                 }
                 const existingOverallSkill = overallSkills.get(key);
                 if (existingOverallSkill) {
                     existingOverallSkill.count += 1;
+                    existingOverallSkill.evidence = strongestSkillEvidence(existingOverallSkill.evidence, evidence);
+                    incrementSkillEvidenceCount(existingOverallSkill.evidenceCounts, evidence);
                 }
                 else {
-                    overallSkills.set(key, { key, label, count: 1 });
+                    const skillEvidenceCounts = emptySkillEvidenceCounts();
+                    incrementSkillEvidenceCount(skillEvidenceCounts, evidence);
+                    overallSkills.set(key, { key, label, count: 1, evidence, evidenceCounts: skillEvidenceCounts });
                 }
             }
         }
@@ -4398,6 +4934,7 @@ export function heartbeatService(db) {
             endDate,
             totalCount,
             totalRunsWithSkills,
+            evidenceCounts,
             skills: Array.from(overallSkills.values()).sort((left, right) => (right.count - left.count
                 || left.label.localeCompare(right.label)
                 || left.key.localeCompare(right.key))),
@@ -4407,6 +4944,7 @@ export function heartbeatService(db) {
                     date,
                     totalCount: bucket.totalCount,
                     runCount: bucket.runCount,
+                    evidenceCounts: bucket.evidenceCounts,
                     skills: Array.from(bucket.skills.values()).sort((left, right) => (right.count - left.count
                         || left.label.localeCompare(right.label)
                         || left.key.localeCompare(right.key))),