@agent-native/core 0.183.0 → 0.184.0-nightly-20260921210811

package/dist/client/observability/ObservabilityDashboard.js

@@ -1,9 +1,10 @@
 import { jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
 import { IconActivity, IconMessages, IconThumbUp, IconThumbDown, IconClock, IconCoin, IconTool, IconMoodSmile, IconChartBar, IconAB2, IconMessageReport, IconChevronRight, IconArrowLeft, IconLoader2, IconAlertTriangle, } from "@tabler/icons-react";
-import { useState } from "react";
+import { useQueryClient } from "@tanstack/react-query";
+import { Fragment, useEffect, useState } from "react";
 import { useT } from "../i18n.js";
 import { cn } from "../utils.js";
-import { useObservabilityOverview, useTraces, useTraceDetail, useFeedbackList, useFeedbackStats, useSatisfaction, useEvalStats, useExperiments, useExperimentDetail, useExperimentResults, } from "./useObservability.js";
+import { useObservabilityOverview, useTraces, useTraceDetail, useFeedbackList, useFeedbackStats, useSatisfaction, useEvalStats, useExperiments, useExperimentDetail, useExperimentResults, useOutputReviews, useSaveInstructionUpdate, useSubmitFeedback, } from "./useObservability.js";
 // ─── Helpers ────────────────────────────────────────────────────────────
 function formatCost(centsX100) {
     const cents = centsX100 / 100;
@@ -138,6 +139,110 @@ function ExperimentDetailView({ id, onBack, }) {
     const { data: results } = useExperimentResults(id);
     return (_jsxs("div", { children: [_jsxs("button", { onClick: onBack, className: "flex items-center gap-1 text-xs text-muted-foreground hover:text-foreground mb-3", children: [_jsx(IconArrowLeft, { size: 14 }), t("observability.backToExperiments")] }), isLoading && _jsx(LoadingState, {}), exp && (_jsxs("div", { className: "space-y-4", children: [_jsxs("div", { className: "flex items-center justify-between gap-2 min-w-0", children: [_jsx("h3", { className: "text-sm font-medium text-foreground truncate min-w-0", children: exp.name }), _jsx(StatusBadge, { status: exp.status })] }), _jsxs("div", { className: "grid grid-cols-3 gap-3", children: [_jsxs("div", { className: "rounded-lg border border-border p-3", children: [_jsx("div", { className: "text-[10px] text-muted-foreground mb-1", children: t("observability.variants") }), _jsx("div", { className: "text-sm font-medium tabular-nums text-foreground", children: exp.variants.length })] }), _jsxs("div", { className: "rounded-lg border border-border p-3", children: [_jsx("div", { className: "text-[10px] text-muted-foreground mb-1", children: t("observability.metrics") }), _jsx("div", { className: "text-sm font-medium tabular-nums text-foreground", children: exp.metrics.length })] }), _jsxs("div", { className: "rounded-lg border border-border p-3", children: [_jsx("div", { className: "text-[10px] text-muted-foreground mb-1", children: t("observability.level") }), _jsx("div", { className: "text-sm font-medium text-foreground capitalize", children: exp.assignmentLevel })] })] }), exp.variants.length > 0 && (_jsxs("div", { children: [_jsx("h4", { className: "text-xs font-medium text-foreground mb-2", children: t("observability.variants") }), _jsx("div", { className: "space-y-1", children: exp.variants.map((v) => (_jsxs("div", { className: "flex items-center justify-between rounded border border-border px-3 py-2 text-xs", children: [_jsx("span", { className: "font-mono text-foreground", children: truncateId(v.id) }), _jsxs("span", { className: "text-muted-foreground tabular-nums", children: ["Weight: ", v.weight] })] }, v.id))) })] })), results && results.length > 0 && (_jsxs("div", { children: [_jsx("h4", { className: "text-xs font-medium text-foreground mb-2", children: t("observability.results") }), _jsx("div", { className: "rounded-lg border border-border overflow-hidden", children: _jsxs("table", { className: "w-full table-fixed text-left text-xs", children: [_jsx("thead", { children: _jsxs("tr", { className: "border-b border-border bg-muted/30", children: [_jsx("th", { className: "px-3 py-2 font-medium text-muted-foreground w-[20%]", children: t("observability.variant") }), _jsx("th", { className: "px-3 py-2 font-medium text-muted-foreground w-[25%]", children: t("observability.metric") }), _jsx("th", { className: "px-3 py-2 font-medium text-muted-foreground", children: t("observability.value") }), _jsx("th", { className: "px-3 py-2 font-medium text-muted-foreground", children: "CI" }), _jsx("th", { className: "px-3 py-2 font-medium text-muted-foreground", children: "N" })] }) }), _jsx("tbody", { children: results.map((r) => (_jsxs("tr", { className: "border-b border-border last:border-b-0", children: [_jsx("td", { className: "px-3 py-2 font-mono text-foreground truncate", children: truncateId(r.variantId) }), _jsx("td", { className: "px-3 py-2 text-foreground truncate", children: r.metric }), _jsx("td", { className: "px-3 py-2 tabular-nums text-foreground", children: r.value.toFixed(3) }), _jsxs("td", { className: "px-3 py-2 tabular-nums text-muted-foreground", children: ["[", r.confidenceLow.toFixed(3), ",", " ", r.confidenceHigh.toFixed(3), "]"] }), _jsx("td", { className: "px-3 py-2 tabular-nums text-muted-foreground", children: r.sampleSize })] }, r.id))) })] }) })] }))] }))] }));
 }
+// ─── Tab: Human review ─────────────────────────────────────────────────
+function ReviewTab({ days }) {
+    const t = useT();
+    const { data: reviews, isLoading } = useOutputReviews(days);
+    const feedbackMutation = useSubmitFeedback();
+    const instructionMutation = useSaveInstructionUpdate();
+    const queryClient = useQueryClient();
+    const [selectedRunId, setSelectedRunId] = useState(null);
+    const [feedbackNote, setFeedbackNote] = useState("");
+    const [instruction, setInstruction] = useState("");
+    const [instructionSaved, setInstructionSaved] = useState(false);
+    const [target, setTarget] = useState("agent");
+    const selected = reviews?.find((review) => review.runId === selectedRunId);
+    useEffect(() => {
+        if (selectedRunId &&
+            reviews &&
+            !reviews.some((review) => review.runId === selectedRunId)) {
+            setSelectedRunId(null);
+        }
+    }, [reviews, selectedRunId]);
+    if (isLoading)
+        return _jsx(LoadingState, {});
+    if (!reviews || reviews.length === 0) {
+        return _jsx(EmptyState, { message: t("observability.noReviews") });
+    }
+    const openReview = (runId) => {
+        setSelectedRunId((current) => (current === runId ? null : runId));
+        setFeedbackNote("");
+        setInstruction("");
+        setInstructionSaved(false);
+        setTarget("agent");
+    };
+    const saveFeedback = (feedbackType) => {
+        if (!selected)
+            return;
+        feedbackMutation.mutate({
+            runId: selected.runId,
+            threadId: selected.threadId ?? undefined,
+            feedbackType,
+        }, {
+            onSuccess: () => void queryClient.invalidateQueries({
+                queryKey: ["action", "list-observability-reviews"],
+            }),
+        });
+    };
+    const saveNote = () => {
+        if (!selected || !feedbackNote.trim())
+            return;
+        feedbackMutation.mutate({
+            runId: selected.runId,
+            threadId: selected.threadId ?? undefined,
+            feedbackType: "text",
+            value: feedbackNote.trim(),
+        }, {
+            onSuccess: () => {
+                setFeedbackNote("");
+                void queryClient.invalidateQueries({
+                    queryKey: ["action", "list-observability-reviews"],
+                });
+            },
+        });
+    };
+    const saveInstruction = () => {
+        if (!selected || !instruction.trim())
+            return;
+        instructionMutation.mutate({
+            runId: selected.runId,
+            threadId: selected.threadId,
+            target,
+            instruction: instruction.trim(),
+            feedback: feedbackNote.trim() || undefined,
+        }, {
+            onSuccess: () => {
+                setFeedbackNote("");
+                setInstruction("");
+                setInstructionSaved(true);
+                setTarget("agent");
+                void queryClient.invalidateQueries({
+                    queryKey: ["action", "list-observability-reviews"],
+                });
+            },
+        });
+    };
+    return (_jsxs("div", { className: "space-y-3", children: [_jsx("div", { className: "rounded-lg border border-border bg-muted/20 px-3 py-2 text-xs text-muted-foreground", children: t("observability.reviewDescription") }), _jsx("div", { className: "overflow-x-auto rounded-lg border border-border", children: _jsxs("table", { className: "w-full min-w-[760px] table-fixed text-left text-xs", children: [_jsx("thead", { className: "bg-muted/30 text-muted-foreground", children: _jsxs("tr", { children: [_jsx("th", { className: "w-[28%] px-3 py-2 font-medium", children: t("observability.ask") }), _jsx("th", { className: "w-[40%] px-3 py-2 font-medium", children: t("observability.answer") }), _jsx("th", { className: "w-[16%] px-3 py-2 font-medium", children: t("observability.reviewFeedback") }), _jsx("th", { className: "w-[16%] px-3 py-2 font-medium" })] }) }), _jsx("tbody", { children: reviews.map((review) => {
+                                const isSelected = review.runId === selectedRunId;
+                                const latestFeedback = review.feedback[0];
+                                return (_jsxs(Fragment, { children: [_jsxs("tr", { className: "border-t border-border align-top", children: [_jsx("td", { className: "px-3 py-3 text-foreground", children: _jsx("div", { className: "line-clamp-3 break-words", children: review.ask || "-" }) }), _jsx("td", { className: "px-3 py-3 text-foreground", children: _jsx("div", { className: "line-clamp-3 break-words", children: review.answer || "-" }) }), _jsx("td", { className: "px-3 py-3 text-muted-foreground", children: latestFeedback
+                                                        ? latestFeedback.feedbackType === "text"
+                                                            ? t("observability.noteSaved")
+                                                            : latestFeedback.feedbackType === "thumbs_up"
+                                                                ? t("observability.looksGood")
+                                                                : t("observability.needsChange")
+                                                        : t("observability.notReviewed") }), _jsx("td", { className: "px-3 py-3 text-right", children: _jsx("button", { type: "button", onClick: () => openReview(review.runId), className: "rounded-md border border-border px-2.5 py-1.5 font-medium text-foreground hover:bg-muted", children: isSelected
+                                                            ? t("observability.closeReview")
+                                                            : t("observability.reviewOutput") }) })] }), isSelected && selected && (_jsx("tr", { className: "border-t border-border", children: _jsx("td", { colSpan: 4, className: "bg-muted/10 p-4", children: _jsxs("div", { className: "grid gap-4 lg:grid-cols-2", children: [_jsxs("div", { className: "space-y-3", children: [_jsxs("div", { children: [_jsx("div", { className: "mb-1 text-[10px] font-medium uppercase tracking-wide text-muted-foreground", children: t("observability.ask") }), _jsx("p", { className: "whitespace-pre-wrap break-words text-sm text-foreground", children: selected.ask || "-" })] }), _jsxs("div", { children: [_jsx("div", { className: "mb-1 text-[10px] font-medium uppercase tracking-wide text-muted-foreground", children: t("observability.answer") }), _jsx("p", { className: "max-h-48 overflow-y-auto whitespace-pre-wrap break-words text-sm text-foreground", children: selected.answer || "-" })] }), _jsxs("div", { className: "flex flex-wrap gap-2", children: [_jsx("button", { type: "button", onClick: () => saveFeedback("thumbs_up"), className: "rounded-md border border-border px-2.5 py-1.5 text-xs text-foreground hover:bg-muted", children: t("observability.looksGood") }), _jsx("button", { type: "button", onClick: () => saveFeedback("thumbs_down"), className: "rounded-md border border-border px-2.5 py-1.5 text-xs text-foreground hover:bg-muted", children: t("observability.needsChange") })] })] }), _jsxs("div", { className: "space-y-3", children: [_jsxs("label", { className: "block", children: [_jsx("span", { className: "mb-1 block text-xs font-medium text-foreground", children: t("observability.feedbackNote") }), _jsx("textarea", { value: feedbackNote, onChange: (event) => setFeedbackNote(event.target.value), rows: 3, className: "w-full rounded-md border border-border bg-background px-3 py-2 text-sm text-foreground outline-none focus:ring-2 focus:ring-ring", placeholder: t("observability.feedbackPlaceholder") })] }), _jsx("button", { type: "button", onClick: saveNote, disabled: !feedbackNote.trim() ||
+                                                                        feedbackMutation.isPending, className: "rounded-md bg-primary px-2.5 py-1.5 text-xs font-medium text-primary-foreground disabled:opacity-50", children: t("observability.saveFeedback") }), _jsxs("div", { className: "border-t border-border pt-3", children: [_jsx("div", { className: "mb-1 text-xs font-medium text-foreground", children: t("observability.updateInstructions") }), _jsx("p", { className: "mb-2 text-[11px] text-muted-foreground", children: t("observability.draftNotice") }), _jsx("div", { className: "flex gap-2", children: _jsxs("select", { value: target, onChange: (event) => setTarget(event.target.value), className: "rounded-md border border-border bg-background px-2 py-1.5 text-xs text-foreground", "aria-label": t("observability.instructionTarget"), children: [_jsx("option", { value: "agent", children: t("observability.agentTarget") }), _jsx("option", { value: "developer", children: t("observability.developerTarget") }), _jsx("option", { value: "skill", children: t("observability.skillTarget") })] }) }), _jsx("textarea", { value: instruction, onChange: (event) => {
+                                                                                setInstruction(event.target.value);
+                                                                                setInstructionSaved(false);
+                                                                            }, rows: 4, className: "mt-2 w-full rounded-md border border-border bg-background px-3 py-2 text-sm text-foreground outline-none focus:ring-2 focus:ring-ring", placeholder: t("observability.instructionPlaceholder") }), _jsx("button", { type: "button", onClick: saveInstruction, disabled: !instruction.trim() ||
+                                                                                instructionMutation.isPending, className: "mt-2 rounded-md border border-border px-2.5 py-1.5 text-xs font-medium text-foreground hover:bg-muted disabled:opacity-50", children: instructionSaved
+                                                                                ? t("observability.draftSaved")
+                                                                                : t("observability.saveUpdate") })] })] })] }) }) }))] }, review.runId));
+                            }) })] }) })] }));
+}
 // ─── Tab: Feedback ──────────────────────────────────────────────────────
 function FeedbackTab({ days }) {
     const t = useT();
@@ -177,6 +282,11 @@ const TABS = [
         labelKey: "observability.feedback",
         icon: IconMessageReport,
     },
+    {
+        id: "review",
+        labelKey: "observability.review",
+        icon: IconMessageReport,
+    },
 ];
 export function ObservabilityDashboard({ className, }) {
     const t = useT();
@@ -187,5 +297,5 @@ export function ObservabilityDashboard({ className, }) {
                             return (_jsxs("button", { onClick: () => setActiveTab(tab.id), className: cn("flex items-center gap-1.5 rounded-md px-3 py-1.5 text-xs font-medium", activeTab === tab.id
                                     ? "bg-background text-foreground shadow-sm"
                                     : "text-muted-foreground hover:text-foreground"), children: [_jsx(Icon, { size: 14 }), t(tab.labelKey)] }, tab.id));
-                        }) }), activeTab !== "experiments" && (_jsx(RangeSelector, { value: days, onChange: setDays }))] }), activeTab === "overview" && _jsx(OverviewTab, { days: days }), activeTab === "conversations" && _jsx(ConversationsTab, { days: days }), activeTab === "evals" && _jsx(EvalsTab, { days: days }), activeTab === "experiments" && _jsx(ExperimentsTab, {}), activeTab === "feedback" && _jsx(FeedbackTab, { days: days })] }));
+                        }) }), activeTab !== "experiments" && (_jsx(RangeSelector, { value: days, onChange: setDays }))] }), activeTab === "overview" && _jsx(OverviewTab, { days: days }), activeTab === "conversations" && _jsx(ConversationsTab, { days: days }), activeTab === "evals" && _jsx(EvalsTab, { days: days }), activeTab === "experiments" && _jsx(ExperimentsTab, {}), activeTab === "feedback" && _jsx(FeedbackTab, { days: days }), activeTab === "review" && _jsx(ReviewTab, { days: days })] }));
 }

package/dist/client/observability/useObservability.d.ts

@@ -1,3 +1,4 @@
+import type { InstructionUpdate, OutputReviewRow } from "../../observability/types.js";
 export interface ObservabilityOverview {
     totalRuns: number;
     totalCostCents: number;
@@ -24,6 +25,14 @@ export interface TraceSummary {
     createdAt: number;
 }
 export declare function useTraces(sinceDays?: number, limit?: number): import("@tanstack/react-query").UseQueryResult<NoInfer<TraceSummary[]>, Error>;
+export declare function useOutputReviews(sinceDays?: number, limit?: number): import("@tanstack/react-query").UseQueryResult<NoInfer<OutputReviewRow[]>, Error>;
+export declare function useSaveInstructionUpdate(): import("@tanstack/react-query").UseMutationResult<InstructionUpdate, Error, {
+    runId: string;
+    threadId?: string | null;
+    target: "agent" | "developer" | "skill";
+    instruction: string;
+    feedback?: string;
+}, unknown>;
 export interface TraceSpan {
     id: string;
     runId: string;

package/dist/client/observability/useObservability.js

@@ -1,5 +1,7 @@
 import { useQuery, useMutation, useQueryClient } from "@tanstack/react-query";
+import { useMemo } from "react";
 import { agentNativePath } from "../api-path.js";
+import { useActionMutation, useActionQuery } from "../use-action.js";
 const BASE = agentNativePath("/_agent-native/observability");
 function fetchJson(url) {
     return fetch(url).then((r) => {
@@ -24,6 +26,16 @@ export function useTraces(sinceDays = 7, limit = 100) {
         refetchInterval: 30_000,
     });
 }
+export function useOutputReviews(sinceDays = 7, limit = 100) {
+    const params = useMemo(() => ({
+        sinceMs: Date.now() - sinceDays * 86_400_000,
+        limit,
+    }), [sinceDays, limit]);
+    return useActionQuery("list-observability-reviews", params, { refetchInterval: 30_000 });
+}
+export function useSaveInstructionUpdate() {
+    return useActionMutation("save-observability-instruction-update");
+}
 export function useTraceDetail(runId) {
     return useQuery({
         queryKey: ["observability", "trace", runId],

package/dist/collab/awareness.d.ts

@@ -62,11 +62,11 @@ export declare const postAwareness: import("h3").EventHandlerWithFetch<import("h
     error: string;
     states?: undefined;
 } | {
+    error?: undefined;
     states: {
         clientId: number;
         state: string;
     }[];
-    error?: undefined;
 }>>;
 /**
  * GET /_agent-native/collab/:docId/users
@@ -77,9 +77,9 @@ export declare const getActiveUsers: import("h3").EventHandlerWithFetch<import("
     error: string;
     users?: undefined;
 } | {
+    error?: undefined;
     users: {
         clientId: number;
         lastSeen: number;
     }[];
-    error?: undefined;
 }>>;

package/dist/collab/struct-routes.d.ts

@@ -13,8 +13,8 @@
  * Body: { json: any, fieldName?: string, type?: "map"|"array", requestSource?: string }
  */
 export declare const postCollabJson: import("h3").EventHandlerWithFetch<import("h3").EventHandlerRequest, Promise<{
-    error: string;
     ok?: undefined;
+    error: string;
 } | {
     error?: undefined;
     ok: boolean;

package/dist/file-upload/actions/upload-image.d.ts

@@ -34,12 +34,12 @@ declare const _default: import("../../action.js").ActionDefinition<{
     idempotencyKey: string;
     deleted: boolean;
 } | {
+    id?: undefined;
     error: string;
     url?: undefined;
     provider?: undefined;
     configured?: undefined;
     connectPath?: undefined;
-    id?: undefined;
 } | {
     url: string;
     id: string;
@@ -48,11 +48,11 @@ declare const _default: import("../../action.js").ActionDefinition<{
     connectPath?: undefined;
     error?: undefined;
 } | {
+    id?: undefined;
     url?: undefined;
     provider?: undefined;
     error: string;
     configured: boolean;
     connectPath: string;
-    id?: undefined;
 }>;
 export default _default;

package/dist/identity/rekey.js

@@ -28,6 +28,11 @@ export const IDENTITY_REKEY_COLUMNS = [
         column: "user_id",
         mode: "email-user-id",
     },
+    {
+        table: "agent_instruction_updates",
+        column: "user_id",
+        mode: "email-user-id",
+    },
     { table: "agent_evals", column: "user_id", mode: "email-user-id" },
     {
         table: "agent_experiment_assignments",

package/dist/localization/default-messages.d.ts

@@ -1547,6 +1547,30 @@ declare const messages: {
         frustration: string;
         thumbsUpRate: string;
         categories: string;
+        review: string;
+        reviewDescription: string;
+        ask: string;
+        answer: string;
+        reviewFeedback: string;
+        reviewOutput: string;
+        closeReview: string;
+        notReviewed: string;
+        noteSaved: string;
+        looksGood: string;
+        needsChange: string;
+        feedbackNote: string;
+        feedbackPlaceholder: string;
+        saveFeedback: string;
+        updateInstructions: string;
+        draftNotice: string;
+        instructionTarget: string;
+        agentTarget: string;
+        developerTarget: string;
+        skillTarget: string;
+        instructionPlaceholder: string;
+        saveUpdate: string;
+        draftSaved: string;
+        noReviews: string;
     };
     error: {
         genericTitle: string;

package/dist/localization/default-messages.js

@@ -1554,6 +1554,30 @@ const messages = {
         frustration: "Frustration",
         thumbsUpRate: "Thumbs up rate",
         categories: "Categories",
+        review: "Human review",
+        reviewDescription: "Review the ask and answer, record feedback, and draft an instruction update.",
+        ask: "What was asked",
+        answer: "What the agent answered",
+        reviewFeedback: "Feedback",
+        reviewOutput: "Review output",
+        closeReview: "Close",
+        notReviewed: "Not reviewed",
+        noteSaved: "Note saved",
+        looksGood: "Looks good",
+        needsChange: "Needs change",
+        feedbackNote: "Feedback note",
+        feedbackPlaceholder: "What should change or stay the same?",
+        saveFeedback: "Save feedback",
+        updateInstructions: "Update instructions",
+        draftNotice: "Draft only - nothing changes automatically.",
+        instructionTarget: "Instruction target",
+        agentTarget: "Agent",
+        developerTarget: "Developer",
+        skillTarget: "Skill",
+        instructionPlaceholder: "Write the instruction change for a human to review.",
+        saveUpdate: "Save draft update",
+        draftSaved: "Draft saved",
+        noReviews: "No agent outputs to review yet",
     },
     error: {
         genericTitle: "Something went wrong",

package/dist/notifications/routes.d.ts

@@ -28,6 +28,6 @@ export declare function createNotificationsHandler(): import("h3").EventHandlerW
 } | {
     count?: undefined;
     updated?: undefined;
-    ok: boolean;
     error?: undefined;
+    ok: boolean;
 }>>;

package/dist/observability/actions/list-observability-reviews.d.ts

@@ -0,0 +1,5 @@
+declare const _default: import("../../action.js").ActionDefinition<{
+    sinceMs?: number | undefined;
+    limit?: number | undefined;
+}, import("../types.js").OutputReviewRow[]>;
+export default _default;

package/dist/observability/actions/list-observability-reviews.js

@@ -0,0 +1,23 @@
+import { z } from "zod";
+import { fail, defineAction } from "../../action.js";
+import { listOutputReviews } from "../reviews.js";
+export default defineAction({
+    description: "List recent agent outputs as a human-review table with the original ask, answer, feedback, and pending instruction updates.",
+    schema: z.object({
+        sinceMs: z.number().int().nonnegative().optional(),
+        limit: z.number().int().positive().max(100).optional(),
+    }),
+    http: { method: "GET" },
+    readOnly: true,
+    parallelSafe: true,
+    run: async (args, ctx) => {
+        const userId = ctx?.userEmail;
+        if (!userId)
+            fail("Sign in to review agent outputs.", { statusCode: 401 });
+        return listOutputReviews({
+            sinceMs: args.sinceMs ?? Date.now() - 7 * 86_400_000,
+            limit: args.limit ?? 100,
+            userId,
+        });
+    },
+});

package/dist/observability/actions/save-observability-instruction-update.d.ts

@@ -0,0 +1,9 @@
+import type { InstructionUpdate } from "../types.js";
+declare const _default: import("../../action.js").ActionDefinition<{
+    runId: string;
+    threadId?: string | null | undefined;
+    target: "agent" | "developer" | "skill";
+    instruction: string;
+    feedback?: string | undefined;
+}, InstructionUpdate>;
+export default _default;

package/dist/observability/actions/save-observability-instruction-update.js

@@ -0,0 +1,38 @@
+import { randomUUID } from "node:crypto";
+import { z } from "zod";
+import { fail, defineAction } from "../../action.js";
+import { getTraceSummary, insertInstructionUpdate } from "../store.js";
+const schema = z.object({
+    runId: z.string().trim().min(1).max(200),
+    threadId: z.string().trim().max(200).nullable().optional(),
+    target: z.enum(["agent", "developer", "skill"]),
+    instruction: z.string().trim().min(1).max(10_000),
+    feedback: z.string().trim().max(4_000).optional(),
+});
+export default defineAction({
+    description: "Save a human-proposed instruction update for an agent output. The draft is explicit and never applied automatically.",
+    schema,
+    run: async (args, ctx) => {
+        const userId = ctx?.userEmail;
+        if (!userId)
+            fail("Sign in to save instruction updates.", { statusCode: 401 });
+        const summary = await getTraceSummary(args.runId, { userId });
+        if (!summary)
+            fail("That agent output is no longer available.", { statusCode: 404 });
+        const now = Date.now();
+        const update = {
+            id: `instruction-${randomUUID()}`,
+            runId: args.runId,
+            threadId: args.threadId ?? summary.threadId,
+            target: args.target,
+            instruction: args.instruction,
+            feedback: args.feedback ?? "",
+            status: "draft",
+            userId,
+            createdAt: now,
+            updatedAt: now,
+        };
+        await insertInstructionUpdate(update);
+        return update;
+    },
+});

package/dist/observability/index.d.ts

@@ -1,5 +1,5 @@
-export type { SpanType, SpanStatus, TraceSpan, TraceSummary, FeedbackType, FeedbackEntry, SatisfactionScore, EvalType, EvalResult, EvalDataset, EvalTestCase, EvalCriteria, ExperimentStatus, ExperimentVariant, Experiment, ExperimentAssignment, ExperimentMetricResult, ObservabilityConfig, } from "./types.js";
-export { ensureObservabilityTables, insertTraceSpan, upsertTraceSummary, getTraceSpansForRun, getTraceSummaries, getTraceSummary, deleteOldTraceData, insertFeedback, getFeedback, getFeedbackStats, upsertSatisfactionScore, getSatisfactionScores, insertEvalResult, getEvalsForRun, getEvalStats, insertEvalDataset, listEvalDatasets, getEvalDataset, updateEvalDataset, insertExperiment, updateExperiment, listExperiments, getExperiment, upsertAssignment, getAssignment, insertExperimentResult, getExperimentResults, getObservabilityOverview, } from "./store.js";
+export type { SpanType, SpanStatus, TraceSpan, TraceSummary, FeedbackType, FeedbackEntry, InstructionUpdateStatus, InstructionUpdate, OutputReviewRow, SatisfactionScore, EvalType, EvalResult, EvalDataset, EvalTestCase, EvalCriteria, ExperimentStatus, ExperimentVariant, Experiment, ExperimentAssignment, ExperimentMetricResult, ObservabilityConfig, } from "./types.js";
+export { ensureObservabilityTables, insertTraceSpan, upsertTraceSummary, getTraceSpansForRun, getTraceSummaries, getTraceSummary, deleteOldTraceData, insertFeedback, getFeedback, getFeedbackStats, insertInstructionUpdate, getInstructionUpdates, upsertSatisfactionScore, getSatisfactionScores, insertEvalResult, getEvalsForRun, getEvalStats, insertEvalDataset, listEvalDatasets, getEvalDataset, updateEvalDataset, insertExperiment, updateExperiment, listExperiments, getExperiment, upsertAssignment, getAssignment, insertExperimentResult, getExperimentResults, getObservabilityOverview, } from "./store.js";
 export { createObservabilityPlugin } from "./plugin.js";
 export { createObservabilityHandler } from "./routes.js";
 export { runTraceCleanupOnce, startTraceCleanupJob, stopTraceCleanupJob, } from "./cleanup-job.js";

package/dist/observability/index.js

@@ -1,4 +1,4 @@
-export { ensureObservabilityTables, insertTraceSpan, upsertTraceSummary, getTraceSpansForRun, getTraceSummaries, getTraceSummary, deleteOldTraceData, insertFeedback, getFeedback, getFeedbackStats, upsertSatisfactionScore, getSatisfactionScores, insertEvalResult, getEvalsForRun, getEvalStats, insertEvalDataset, listEvalDatasets, getEvalDataset, updateEvalDataset, insertExperiment, updateExperiment, listExperiments, getExperiment, upsertAssignment, getAssignment, insertExperimentResult, getExperimentResults, getObservabilityOverview, } from "./store.js";
+export { ensureObservabilityTables, insertTraceSpan, upsertTraceSummary, getTraceSpansForRun, getTraceSummaries, getTraceSummary, deleteOldTraceData, insertFeedback, getFeedback, getFeedbackStats, insertInstructionUpdate, getInstructionUpdates, upsertSatisfactionScore, getSatisfactionScores, insertEvalResult, getEvalsForRun, getEvalStats, insertEvalDataset, listEvalDatasets, getEvalDataset, updateEvalDataset, insertExperiment, updateExperiment, listExperiments, getExperiment, upsertAssignment, getAssignment, insertExperimentResult, getExperimentResults, getObservabilityOverview, } from "./store.js";
 export { createObservabilityPlugin } from "./plugin.js";
 export { createObservabilityHandler } from "./routes.js";
 export { runTraceCleanupOnce, startTraceCleanupJob, stopTraceCleanupJob, } from "./cleanup-job.js";

package/dist/observability/reviews.d.ts

@@ -0,0 +1,6 @@
+import type { OutputReviewRow } from "./types.js";
+export declare function listOutputReviews(opts: {
+    sinceMs: number;
+    limit: number;
+    userId: string;
+}): Promise<OutputReviewRow[]>;

package/dist/observability/reviews.js

@@ -0,0 +1,155 @@
+import { resolveThreadsAccess } from "../chat-threads/store.js";
+import { getFeedback, getInstructionUpdates, getTraceSummaries, } from "./store.js";
+function unwrapMessage(value) {
+    if (!value || typeof value !== "object" || Array.isArray(value))
+        return null;
+    const record = value;
+    const nested = record.message;
+    return nested && typeof nested === "object" && !Array.isArray(nested)
+        ? nested
+        : record;
+}
+function messageText(value) {
+    if (typeof value === "string")
+        return value;
+    if (!Array.isArray(value))
+        return "";
+    return value
+        .filter((part) => Boolean(part) &&
+        typeof part === "object" &&
+        !Array.isArray(part) &&
+        part.type === "text")
+        .map((part) => (typeof part.text === "string" ? part.text : ""))
+        .join("");
+}
+function messageRunId(message) {
+    const metadata = message.metadata;
+    if (!metadata || typeof metadata !== "object" || Array.isArray(metadata)) {
+        return undefined;
+    }
+    const meta = metadata;
+    const custom = meta.custom;
+    const customRecord = custom && typeof custom === "object" && !Array.isArray(custom)
+        ? custom
+        : null;
+    const candidates = [
+        meta.runId,
+        customRecord?.runId,
+        customRecord?.submittedRunId,
+    ];
+    return candidates.find((candidate) => typeof candidate === "string" && candidate.length > 0);
+}
+function readThreadMessages(threadData) {
+    try {
+        const repository = JSON.parse(threadData);
+        const values = Array.isArray(repository?.messages)
+            ? repository.messages
+            : [];
+        return values.flatMap((value) => {
+            const message = unwrapMessage(value);
+            if (!message ||
+                (message.role !== "user" && message.role !== "assistant")) {
+                return [];
+            }
+            if (!Object.prototype.hasOwnProperty.call(message, "content")) {
+                return [];
+            }
+            const content = message.content;
+            if (typeof content !== "string" && !Array.isArray(content)) {
+                return [];
+            }
+            const text = messageText(content).trim();
+            return text
+                ? [{ role: message.role, text, runId: messageRunId(message) }]
+                : [];
+        });
+    }
+    catch (error) {
+        throw new Error("Unable to parse observability thread data", {
+            cause: error,
+        });
+    }
+}
+function askAndAnswer(summary, threadData) {
+    if (!threadData)
+        return { ask: "", answer: "" };
+    const messages = readThreadMessages(threadData);
+    const askIndex = messages.findIndex((message) => message.role === "user" && message.runId === summary.runId);
+    const resolvedAskIndex = askIndex >= 0
+        ? askIndex
+        : messages.filter((message) => message.role === "user").length === 1
+            ? messages.findIndex((message) => message.role === "user")
+            : -1;
+    const answerIndex = resolvedAskIndex < 0
+        ? -1
+        : messages.findIndex((message, index) => {
+            if (index <= resolvedAskIndex || message.role === "user") {
+                return false;
+            }
+            if (message.role !== "assistant")
+                return false;
+            return message.runId === summary.runId || !message.runId;
+        });
+    return {
+        ask: resolvedAskIndex >= 0 ? messages[resolvedAskIndex].text : "",
+        answer: answerIndex >= 0 ? messages[answerIndex].text : "",
+    };
+}
+export async function listOutputReviews(opts) {
+    const summaries = await getTraceSummaries({
+        sinceMs: opts.sinceMs,
+        limit: opts.limit,
+        userId: opts.userId,
+    });
+    const [feedback, updates] = await Promise.all([
+        getFeedback({
+            sinceMs: opts.sinceMs,
+            limit: opts.limit * 4,
+            userId: opts.userId,
+        }),
+        getInstructionUpdates({
+            sinceMs: opts.sinceMs,
+            limit: opts.limit * 2,
+            userId: opts.userId,
+        }),
+    ]);
+    const feedbackByRun = groupByRun(feedback);
+    const updateByRun = new Map();
+    for (const update of updates) {
+        if (!updateByRun.has(update.runId))
+            updateByRun.set(update.runId, update);
+    }
+    const threadIds = summaries.flatMap((summary) => summary.threadId ? [summary.threadId] : []);
+    const threads = await resolveThreadsAccess(opts.userId, threadIds);
+    return summaries
+        .map((summary) => {
+        const thread = summary.threadId
+            ? (threads.get(summary.threadId) ?? null)
+            : null;
+        if (summary.threadId && !thread)
+            return null;
+        const { ask, answer } = askAndAnswer(summary, thread?.threadData ?? null);
+        return {
+            runId: summary.runId,
+            threadId: summary.threadId,
+            ask,
+            answer,
+            model: summary.model,
+            createdAt: summary.createdAt,
+            feedback: feedbackByRun.get(summary.runId) ?? [],
+            instructionUpdate: updateByRun.get(summary.runId) ?? null,
+        };
+    })
+        .filter((row) => row !== null);
+}
+function groupByRun(entries) {
+    const grouped = new Map();
+    for (const entry of entries) {
+        if (!entry.runId)
+            continue;
+        const current = grouped.get(entry.runId) ?? [];
+        current.push(entry);
+        grouped.set(entry.runId, current);
+    }
+    return grouped;
+}

package/dist/observability/routes.d.ts

@@ -41,27 +41,27 @@ export declare function createObservabilityHandler(): import("h3").EventHandlerW
     thumbsUpRate: number;
     avgEvalScore: number;
 } | {
-    summary: import("./types.js").TraceSummary;
-    spans: import("./types.js").TraceSpan[];
     id?: undefined;
     error?: undefined;
     ok?: undefined;
+    summary: import("./types.js").TraceSummary;
+    spans: import("./types.js").TraceSpan[];
 } | {
-    summary?: undefined;
-    spans?: undefined;
-    id: string;
     error?: undefined;
     ok?: undefined;
-} | {
     summary?: undefined;
     spans?: undefined;
+    id: string;
+} | {
     id?: undefined;
-    error: any;
     ok?: undefined;
-} | {
     summary?: undefined;
     spans?: undefined;
+    error: any;
+} | {
     id?: undefined;
     error?: undefined;
+    summary?: undefined;
+    spans?: undefined;
     ok: boolean;
 }>>;