@agent-native/core 0.7.12 → 0.7.14

package/dist/notifications/actions.js

@@ -2,112 +2,120 @@
  * Framework-level agent actions for the notifications primitive.
  *
  * Registered as native tools (not template actions) so they're available in
- * every template. The agent uses `notify` to surface progress, completions,
- * and alerts to the user through the in-app inbox and any registered channels.
+ * every template. Consolidated into a single `manage-notifications` tool with
+ * an `action` parameter that dispatches to the correct implementation.
  */
 import { notify, listNotifications, countUnread } from "./registry.js";
 export function createNotificationToolEntries(getCurrentUser) {
     return {
-        notify: {
+        "manage-notifications": {
             tool: {
-                description: "Send a notification to the user. Always persisted to the in-app inbox so the bell + toast surface shows it. Registered channels (webhook, Slack, etc.) also run. Use `info` for FYI, `warning` for things the user should look at, `critical` for things that need immediate attention.",
+                description: [
+                    "Manage user notifications. Available actions:",
+                    "",
+                    '• action="send" — Send a notification to the user. Persisted to the in-app inbox so the bell + toast surface shows it. Registered channels (webhook, Slack, etc.) also run.',
+                    "  Required: severity, title. Optional: body, metadataJson, channels.",
+                    "",
+                    '• action="list" — List recent notifications for the current user. Useful when the user asks about prior alerts.',
+                    "  Optional: unreadOnly (boolean), limit (number, default 20, max 200).",
+                ].join("\n"),
                 parameters: {
                     type: "object",
                     properties: {
+                        action: {
+                            type: "string",
+                            enum: ["send", "list"],
+                            description: "The notification action to perform.",
+                        },
                         severity: {
                             type: "string",
                             enum: ["info", "warning", "critical"],
-                            description: "Severity level — drives styling and per-severity channel routing.",
+                            description: '(send) Severity level — drives styling and per-severity channel routing. Use "info" for FYI, "warning" for things the user should look at, "critical" for things that need immediate attention.',
                         },
                         title: {
                             type: "string",
-                            description: "Short, human-readable headline (≤100 chars).",
+                            description: "(send) Short, human-readable headline (≤100 chars).",
                         },
                         body: {
                             type: "string",
-                            description: "Optional longer description.",
+                            description: "(send) Optional longer description.",
                         },
                         metadataJson: {
                             type: "string",
-                            description: 'Optional JSON metadata (URLs, entity ids, etc.). Example: \'{"threadId":"abc","link":"/inbox/abc"}\'.',
+                            description: '(send) Optional JSON metadata (URLs, entity ids, etc.). Example: \'{"threadId":"abc","link":"/inbox/abc"}\'.',
                         },
                         channels: {
                             type: "string",
-                            description: 'Optional comma-separated channel allowlist (e.g. "inbox,webhook"). Omit to run all registered channels.',
+                            description: '(send) Optional comma-separated channel allowlist (e.g. "inbox,webhook"). Omit to run all registered channels.',
                         },
-                    },
-                    required: ["severity", "title"],
-                },
-            },
-            run: async (args) => {
-                const owner = getCurrentUser();
-                if (!args.severity || !args.title) {
-                    return "Error: --severity and --title are required.";
-                }
-                const severity = args.severity;
-                if (!["info", "warning", "critical"].includes(severity)) {
-                    return `Error: severity must be info, warning, or critical (got "${severity}").`;
-                }
-                let metadata;
-                if (args.metadataJson) {
-                    try {
-                        metadata = JSON.parse(args.metadataJson);
-                    }
-                    catch {
-                        return "Error: metadataJson must be valid JSON.";
-                    }
-                }
-                const channels = args.channels
-                    ? args.channels
-                        .split(",")
-                        .map((s) => s.trim())
-                        .filter(Boolean)
-                    : undefined;
-                const stored = await notify({
-                    severity,
-                    title: args.title,
-                    body: args.body || undefined,
-                    metadata,
-                    channels,
-                }, { owner });
-                return stored
-                    ? `Notification sent (id: ${stored.id})`
-                    : "Notification dispatched to channels (not persisted).";
-            },
-        },
-        "list-notifications": {
-            tool: {
-                description: "List recent notifications for the current user. Useful when the user asks about prior alerts.",
-                parameters: {
-                    type: "object",
-                    properties: {
                         unreadOnly: {
                             type: "boolean",
-                            description: "When true, only include unread notifications.",
+                            description: "(list) When true, only include unread notifications.",
                         },
                         limit: {
                             type: "number",
-                            description: "Max rows to return (default 20, max 200).",
+                            description: "(list) Max rows to return (default 20, max 200).",
                         },
                     },
+                    required: ["action"],
                 },
             },
             run: async (args) => {
                 const owner = getCurrentUser();
-                const rows = await listNotifications(owner, {
-                    unreadOnly: args.unreadOnly === true || args.unreadOnly === "true",
-                    limit: Math.min(Number(args.limit ?? 20), 200),
-                });
-                if (rows.length === 0) {
-                    return args.unreadOnly
-                        ? "No unread notifications."
-                        : "No notifications.";
+                const action = args.action;
+                switch (action) {
+                    case "send": {
+                        if (!args.severity || !args.title) {
+                            return "Error: severity and title are required for action=send.";
+                        }
+                        const severity = args.severity;
+                        if (!["info", "warning", "critical"].includes(severity)) {
+                            return `Error: severity must be info, warning, or critical (got "${severity}").`;
+                        }
+                        let metadata;
+                        if (args.metadataJson) {
+                            try {
+                                metadata = JSON.parse(args.metadataJson);
+                            }
+                            catch {
+                                return "Error: metadataJson must be valid JSON.";
+                            }
+                        }
+                        const channels = typeof args.channels === "string"
+                            ? args.channels
+                                .split(",")
+                                .map((s) => s.trim())
+                                .filter(Boolean)
+                            : undefined;
+                        const stored = await notify({
+                            severity,
+                            title: args.title,
+                            body: args.body || undefined,
+                            metadata,
+                            channels,
+                        }, { owner });
+                        return stored
+                            ? `Notification sent (id: ${stored.id})`
+                            : "Notification dispatched to channels (not persisted).";
+                    }
+                    case "list": {
+                        const rows = await listNotifications(owner, {
+                            unreadOnly: args.unreadOnly === true || args.unreadOnly === "true",
+                            limit: Math.min(Number(args.limit ?? 20), 200),
+                        });
+                        if (rows.length === 0) {
+                            return args.unreadOnly
+                                ? "No unread notifications."
+                                : "No notifications.";
+                        }
+                        const unreadCount = await countUnread(owner);
+                        const lines = rows.map((n) => `[${n.readAt ? " " : "•"}] (${n.severity}) ${n.title}${n.body ? ` — ${n.body}` : ""} · ${n.createdAt}`);
+                        return `${unreadCount} unread\n\n${lines.join("\n")}`;
+                    }
+                    default:
+                        return `Error: unknown action "${action}". Must be one of: send, list.`;
                 }
-                const unreadCount = await countUnread(owner);
-                const lines = rows.map((n) => `[${n.readAt ? " " : "•"}] (${n.severity}) ${n.title}${n.body ? ` — ${n.body}` : ""} · ${n.createdAt}`);
-                return `${unreadCount} unread\n\n${lines.join("\n")}`;
             },
-            readOnly: true,
         },
     };
 }

package/dist/notifications/actions.js.map

@@ -1 +1 @@
-{"version":3,"file":"actions.js","sourceRoot":"","sources":["../../src/notifications/actions.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAGH,OAAO,EAAE,MAAM,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAGvE,MAAM,UAAU,6BAA6B,CAC3C,cAA4B;IAE5B,OAAO;QACL,MAAM,EAAE;YACN,IAAI,EAAE;gBACJ,WAAW,EACT,yRAAyR;gBAC3R,UAAU,EAAE;oBACV,IAAI,EAAE,QAAiB;oBACvB,UAAU,EAAE;wBACV,QAAQ,EAAE;4BACR,IAAI,EAAE,QAAQ;4BACd,IAAI,EAAE,CAAC,MAAM,EAAE,SAAS,EAAE,UAAU,CAAC;4BACrC,WAAW,EACT,mEAAmE;yBACtE;wBACD,KAAK,EAAE;4BACL,IAAI,EAAE,QAAQ;4BACd,WAAW,EAAE,8CAA8C;yBAC5D;wBACD,IAAI,EAAE;4BACJ,IAAI,EAAE,QAAQ;4BACd,WAAW,EAAE,8BAA8B;yBAC5C;wBACD,YAAY,EAAE;4BACZ,IAAI,EAAE,QAAQ;4BACd,WAAW,EACT,uGAAuG;yBAC1G;wBACD,QAAQ,EAAE;4BACR,IAAI,EAAE,QAAQ;4BACd,WAAW,EACT,yGAAyG;yBAC5G;qBACF;oBACD,QAAQ,EAAE,CAAC,UAAU,EAAE,OAAO,CAAC;iBAChC;aACF;YACD,GAAG,EAAE,KAAK,EAAE,IAA4B,EAAE,EAAE;gBAC1C,MAAM,KAAK,GAAG,cAAc,EAAE,CAAC;gBAC/B,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;oBAClC,OAAO,6CAA6C,CAAC;gBACvD,CAAC;gBACD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAgC,CAAC;gBACvD,IAAI,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;oBACxD,OAAO,4DAA4D,QAAQ,KAAK,CAAC;gBACnF,CAAC;gBAED,IAAI,QAA6C,CAAC;gBAClD,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;oBACtB,IAAI,CAAC;wBACH,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;oBAC3C,CAAC;oBAAC,MAAM,CAAC;wBACP,OAAO,yCAAyC,CAAC;oBACnD,CAAC;gBACH,CAAC;gBAED,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ;oBAC5B,CAAC,CAAC,IAAI,CAAC,QAAQ;yBACV,KAAK,CAAC,GAAG,CAAC;yBACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;yBACpB,MAAM,CAAC,OAAO,CAAC;oBACpB,CAAC,CAAC,SAAS,CAAC;gBAEd,MAAM,MAAM,GAAG,MAAM,MAAM,CACzB;oBACE,QAAQ;oBACR,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,IAAI,EAAE,IAAI,CAAC,IAAI,IAAI,SAAS;oBAC5B,QAAQ;oBACR,QAAQ;iBACT,EACD,EAAE,KAAK,EAAE,CACV,CAAC;gBACF,OAAO,MAAM;oBACX,CAAC,CAAC,0BAA0B,MAAM,CAAC,EAAE,GAAG;oBACxC,CAAC,CAAC,sDAAsD,CAAC;YAC7D,CAAC;SACF;QAED,oBAAoB,EAAE;YACpB,IAAI,EAAE;gBACJ,WAAW,EACT,+FAA+F;gBACjG,UAAU,EAAE;oBACV,IAAI,EAAE,QAAiB;oBACvB,UAAU,EAAE;wBACV,UAAU,EAAE;4BACV,IAAI,EAAE,SAAS;4BACf,WAAW,EAAE,+CAA+C;yBAC7D;wBACD,KAAK,EAAE;4BACL,IAAI,EAAE,QAAQ;4BACd,WAAW,EAAE,2CAA2C;yBACzD;qBACF;iBACF;aACF;YACD,GAAG,EAAE,KAAK,EAAE,IAA6B,EAAE,EAAE;gBAC3C,MAAM,KAAK,GAAG,cAAc,EAAE,CAAC;gBAC/B,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAC,KAAK,EAAE;oBAC1C,UAAU,EAAE,IAAI,CAAC,UAAU,KAAK,IAAI,IAAI,IAAI,CAAC,UAAU,KAAK,MAAM;oBAClE,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC;iBAC/C,CAAC,CAAC;gBACH,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBACtB,OAAO,IAAI,CAAC,UAAU;wBACpB,CAAC,CAAC,0BAA0B;wBAC5B,CAAC,CAAC,mBAAmB,CAAC;gBAC1B,CAAC;gBACD,MAAM,WAAW,GAAG,MAAM,WAAW,CAAC,KAAK,CAAC,CAAC;gBAC7C,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CACpB,CAAC,CAAC,EAAE,EAAE,CACJ,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,SAAS,EAAE,CACzG,CAAC;gBACF,OAAO,GAAG,WAAW,cAAc,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YACxD,CAAC;YACD,QAAQ,EAAE,IAAI;SACf;KACF,CAAC;AACJ,CAAC"}
+{"version":3,"file":"actions.js","sourceRoot":"","sources":["../../src/notifications/actions.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAGH,OAAO,EAAE,MAAM,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAGvE,MAAM,UAAU,6BAA6B,CAC3C,cAA4B;IAE5B,OAAO;QACL,sBAAsB,EAAE;YACtB,IAAI,EAAE;gBACJ,WAAW,EAAE;oBACX,+CAA+C;oBAC/C,EAAE;oBACF,6KAA6K;oBAC7K,sEAAsE;oBACtE,EAAE;oBACF,iHAAiH;oBACjH,wEAAwE;iBACzE,CAAC,IAAI,CAAC,IAAI,CAAC;gBACZ,UAAU,EAAE;oBACV,IAAI,EAAE,QAAiB;oBACvB,UAAU,EAAE;wBACV,MAAM,EAAE;4BACN,IAAI,EAAE,QAAQ;4BACd,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC;4BACtB,WAAW,EAAE,qCAAqC;yBACnD;wBACD,QAAQ,EAAE;4BACR,IAAI,EAAE,QAAQ;4BACd,IAAI,EAAE,CAAC,MAAM,EAAE,SAAS,EAAE,UAAU,CAAC;4BACrC,WAAW,EACT,iMAAiM;yBACpM;wBACD,KAAK,EAAE;4BACL,IAAI,EAAE,QAAQ;4BACd,WAAW,EACT,qDAAqD;yBACxD;wBACD,IAAI,EAAE;4BACJ,IAAI,EAAE,QAAQ;4BACd,WAAW,EAAE,qCAAqC;yBACnD;wBACD,YAAY,EAAE;4BACZ,IAAI,EAAE,QAAQ;4BACd,WAAW,EACT,8GAA8G;yBACjH;wBACD,QAAQ,EAAE;4BACR,IAAI,EAAE,QAAQ;4BACd,WAAW,EACT,gHAAgH;yBACnH;wBACD,UAAU,EAAE;4BACV,IAAI,EAAE,SAAS;4BACf,WAAW,EACT,sDAAsD;yBACzD;wBACD,KAAK,EAAE;4BACL,IAAI,EAAE,QAAQ;4BACd,WAAW,EAAE,kDAAkD;yBAChE;qBACF;oBACD,QAAQ,EAAE,CAAC,QAAQ,CAAC;iBACrB;aACF;YACD,GAAG,EAAE,KAAK,EAAE,IAA6B,EAAE,EAAE;gBAC3C,MAAM,KAAK,GAAG,cAAc,EAAE,CAAC;gBAC/B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAgB,CAAC;gBAErC,QAAQ,MAAM,EAAE,CAAC;oBACf,KAAK,MAAM,CAAC,CAAC,CAAC;wBACZ,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;4BAClC,OAAO,yDAAyD,CAAC;wBACnE,CAAC;wBACD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAgC,CAAC;wBACvD,IAAI,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;4BACxD,OAAO,4DAA4D,QAAQ,KAAK,CAAC;wBACnF,CAAC;wBAED,IAAI,QAA6C,CAAC;wBAClD,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;4BACtB,IAAI,CAAC;gCACH,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,YAAsB,CAAC,CAAC;4BACrD,CAAC;4BAAC,MAAM,CAAC;gCACP,OAAO,yCAAyC,CAAC;4BACnD,CAAC;wBACH,CAAC;wBAED,MAAM,QAAQ,GACZ,OAAO,IAAI,CAAC,QAAQ,KAAK,QAAQ;4BAC/B,CAAC,CAAC,IAAI,CAAC,QAAQ;iCACV,KAAK,CAAC,GAAG,CAAC;iCACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;iCACpB,MAAM,CAAC,OAAO,CAAC;4BACpB,CAAC,CAAC,SAAS,CAAC;wBAEhB,MAAM,MAAM,GAAG,MAAM,MAAM,CACzB;4BACE,QAAQ;4BACR,KAAK,EAAE,IAAI,CAAC,KAAe;4BAC3B,IAAI,EAAG,IAAI,CAAC,IAAe,IAAI,SAAS;4BACxC,QAAQ;4BACR,QAAQ;yBACT,EACD,EAAE,KAAK,EAAE,CACV,CAAC;wBACF,OAAO,MAAM;4BACX,CAAC,CAAC,0BAA0B,MAAM,CAAC,EAAE,GAAG;4BACxC,CAAC,CAAC,sDAAsD,CAAC;oBAC7D,CAAC;oBAED,KAAK,MAAM,CAAC,CAAC,CAAC;wBACZ,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAC,KAAK,EAAE;4BAC1C,UAAU,EACR,IAAI,CAAC,UAAU,KAAK,IAAI,IAAI,IAAI,CAAC,UAAU,KAAK,MAAM;4BACxD,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC;yBAC/C,CAAC,CAAC;wBACH,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;4BACtB,OAAO,IAAI,CAAC,UAAU;gCACpB,CAAC,CAAC,0BAA0B;gCAC5B,CAAC,CAAC,mBAAmB,CAAC;wBAC1B,CAAC;wBACD,MAAM,WAAW,GAAG,MAAM,WAAW,CAAC,KAAK,CAAC,CAAC;wBAC7C,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CACpB,CAAC,CAAC,EAAE,EAAE,CACJ,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,SAAS,EAAE,CACzG,CAAC;wBACF,OAAO,GAAG,WAAW,cAAc,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;oBACxD,CAAC;oBAED;wBACE,OAAO,0BAA0B,MAAM,gCAAgC,CAAC;gBAC5E,CAAC;YACH,CAAC;SACF;KACF,CAAC;AACJ,CAAC"}

package/dist/oauth-tokens/google-refresh.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"google-refresh.d.ts","sourceRoot":"","sources":["../../src/oauth-tokens/google-refresh.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AA+CH;;;;GAIG;AACH,wBAAsB,2BAA2B,CAC/C,IAAI,GAAE;IACJ,QAAQ,CAAC,EAAE,MAAM,CAAC;CACd,GACL,OAAO,CAAC,IAAI,CAAC,CAsCf;AAKD;;;GAGG;AACH,wBAAgB,2BAA2B,CACzC,IAAI,GAAE;IACJ,8CAA8C;IAC9C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,uEAAuE;IACvE,QAAQ,CAAC,EAAE,MAAM,CAAC;CACd,GACL,IAAI,CA+BN"}
+{"version":3,"file":"google-refresh.d.ts","sourceRoot":"","sources":["../../src/oauth-tokens/google-refresh.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AA+CH;;;;GAIG;AACH,wBAAsB,2BAA2B,CAC/C,IAAI,GAAE;IACJ,QAAQ,CAAC,EAAE,MAAM,CAAC;CACd,GACL,OAAO,CAAC,IAAI,CAAC,CAsCf;AAKD;;;GAGG;AACH,wBAAgB,2BAA2B,CACzC,IAAI,GAAE;IACJ,8CAA8C;IAC9C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,uEAAuE;IACvE,QAAQ,CAAC,EAAE,MAAM,CAAC;CACd,GACL,IAAI,CAsCN"}

package/dist/oauth-tokens/google-refresh.js

@@ -88,6 +88,12 @@ export function startGoogleTokenRefreshLoop(opts = {}) {
     if (_started)
         return;
     _started = true;
+    // In dev mode the local DB is usually the shared prod DB. Proactively
+    // refreshing every user's token with local OAuth credentials always fails
+    // (wrong client_id/secret). Tokens still refresh reactively on-demand.
+    if (process.env.NODE_ENV !== "production") {
+        return;
+    }
     const intervalMs = opts.intervalMs ?? 20 * 60 * 1000;
     const bufferMs = opts.bufferMs ?? 15 * 60 * 1000;
     // Kick off an initial pass shortly after startup (not immediately — the DB

package/dist/oauth-tokens/google-refresh.js.map

@@ -1 +1 @@
-{"version":3,"file":"google-refresh.js","sourceRoot":"","sources":["../../src/oauth-tokens/google-refresh.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAEhE,MAAM,gBAAgB,GAAG,qCAAqC,CAAC;AAiB/D,KAAK,UAAU,UAAU,CAAC,YAAoB;IAC5C,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC;IAC9C,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC;IACtD,IAAI,CAAC,QAAQ,IAAI,CAAC,YAAY,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;IACrD,CAAC;IACD,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,gBAAgB,EAAE;QACxC,MAAM,EAAE,MAAM;QACd,OAAO,EAAE,EAAE,cAAc,EAAE,mCAAmC,EAAE;QAChE,IAAI,EAAE,IAAI,eAAe,CAAC;YACxB,aAAa,EAAE,YAAY;YAC3B,SAAS,EAAE,QAAQ;YACnB,aAAa,EAAE,YAAY;YAC3B,UAAU,EAAE,eAAe;SAC5B,CAAC;KACH,CAAC,CAAC;IACH,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAA4B,CAAC;IAC3D,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;QACZ,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,iBAAiB;YACjC,IAAI,CAAC,KAAK;YACV,GAAG,CAAC,UAAU,CAAW,CAAC;QAC5B,MAAM,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC;IACvB,CAAC;IACD,OAAO,IAAkC,CAAC;AAC5C,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAC/C,OAEI,EAAE;IAEN,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;IACjD,IAAI,QAAuD,CAAC;IAC5D,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,iBAAiB,CAAC,QAAQ,CAAC,CAAC;IAC/C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,KAAK,CAAC,2CAA2C,EAAE,GAAG,CAAC,CAAC;QAChE,OAAO;IACT,CAAC;IACD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAsB,CAAC;QAC3C,IAAI,CAAC,MAAM,EAAE,aAAa;YAAE,SAAS;QACrC,IAAI,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,WAAW,GAAG,GAAG,GAAG,QAAQ;YAAE,SAAS;QACxE,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,MAAM,UAAU,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;YACzD,MAAM,MAAM,GAAiB;gBAC3B,GAAG,MAAM;gBACT,YAAY,EAAE,SAAS,CAAC,YAAY;gBACpC,WAAW,EAAE,GAAG,GAAG,SAAS,CAAC,UAAU,GAAG,IAAI;gBAC9C,UAAU,EAAE,SAAS,CAAC,UAAU;gBAChC,KAAK,EAAE,SAAS,CAAC,KAAK,IAAI,MAAM,CAAC,KAAK;aACvC,CAAC;YACF,MAAM,eAAe,CACnB,QAAQ,EACR,IAAI,CAAC,SAAS,EACd,MAA4C,CAC7C,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,qEAAqE;YACrE,oEAAoE;YACpE,yDAAyD;YACzD,OAAO,CAAC,IAAI,CACV,uCAAuC,IAAI,CAAC,SAAS,GAAG,EACvD,GAAa,CAAC,OAAO,CACvB,CAAC;QACJ,CAAC;IACH,CAAC;AACH,CAAC;AAED,IAAI,QAAQ,GAAG,KAAK,CAAC;AACrB,IAAI,MAAkD,CAAC;AAEvD;;;GAGG;AACH,MAAM,UAAU,2BAA2B,CACzC,OAKI,EAAE;IAEN,IAAI,QAAQ;QAAE,OAAO;IACrB,QAAQ,GAAG,IAAI,CAAC;IAChB,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;IACrD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;IAEjD,2EAA2E;IAC3E,8BAA8B;IAC9B,MAAM,YAAY,GAAG,UAAU,CAAC,GAAG,EAAE;QACnC,2BAA2B,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACtD,OAAO,CAAC,KAAK,CAAC,uCAAuC,EAAE,GAAG,CAAC,CAAC;QAC9D,CAAC,CAAC,CAAC;IACL,CAAC,EAAE,MAAM,CAAC,CAAC;IAEX,MAAM,GAAG,WAAW,CAAC,GAAG,EAAE;QACxB,2BAA2B,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACtD,OAAO,CAAC,KAAK,CAAC,wCAAwC,EAAE,GAAG,CAAC,CAAC;QAC/D,CAAC,CAAC,CAAC;IACL,CAAC,EAAE,UAAU,CAAC,CAAC;IAEf,4DAA4D;IAC5D,IACE,OAAO,YAAY,KAAK,QAAQ;QAChC,YAAY;QACZ,OAAO,IAAI,YAAY,EACvB,CAAC;QACA,YAAsC,CAAC,KAAK,EAAE,CAAC;IAClD,CAAC;IACD,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,IAAI,OAAO,IAAI,MAAM,EAAE,CAAC;QAC7D,MAAgC,CAAC,KAAK,EAAE,CAAC;IAC5C,CAAC;AACH,CAAC"}
+{"version":3,"file":"google-refresh.js","sourceRoot":"","sources":["../../src/oauth-tokens/google-refresh.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAEhE,MAAM,gBAAgB,GAAG,qCAAqC,CAAC;AAiB/D,KAAK,UAAU,UAAU,CAAC,YAAoB;IAC5C,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC;IAC9C,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC;IACtD,IAAI,CAAC,QAAQ,IAAI,CAAC,YAAY,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;IACrD,CAAC;IACD,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,gBAAgB,EAAE;QACxC,MAAM,EAAE,MAAM;QACd,OAAO,EAAE,EAAE,cAAc,EAAE,mCAAmC,EAAE;QAChE,IAAI,EAAE,IAAI,eAAe,CAAC;YACxB,aAAa,EAAE,YAAY;YAC3B,SAAS,EAAE,QAAQ;YACnB,aAAa,EAAE,YAAY;YAC3B,UAAU,EAAE,eAAe;SAC5B,CAAC;KACH,CAAC,CAAC;IACH,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAA4B,CAAC;IAC3D,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;QACZ,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,iBAAiB;YACjC,IAAI,CAAC,KAAK;YACV,GAAG,CAAC,UAAU,CAAW,CAAC;QAC5B,MAAM,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC;IACvB,CAAC;IACD,OAAO,IAAkC,CAAC;AAC5C,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAC/C,OAEI,EAAE;IAEN,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;IACjD,IAAI,QAAuD,CAAC;IAC5D,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,iBAAiB,CAAC,QAAQ,CAAC,CAAC;IAC/C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,KAAK,CAAC,2CAA2C,EAAE,GAAG,CAAC,CAAC;QAChE,OAAO;IACT,CAAC;IACD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAsB,CAAC;QAC3C,IAAI,CAAC,MAAM,EAAE,aAAa;YAAE,SAAS;QACrC,IAAI,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,WAAW,GAAG,GAAG,GAAG,QAAQ;YAAE,SAAS;QACxE,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,MAAM,UAAU,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;YACzD,MAAM,MAAM,GAAiB;gBAC3B,GAAG,MAAM;gBACT,YAAY,EAAE,SAAS,CAAC,YAAY;gBACpC,WAAW,EAAE,GAAG,GAAG,SAAS,CAAC,UAAU,GAAG,IAAI;gBAC9C,UAAU,EAAE,SAAS,CAAC,UAAU;gBAChC,KAAK,EAAE,SAAS,CAAC,KAAK,IAAI,MAAM,CAAC,KAAK;aACvC,CAAC;YACF,MAAM,eAAe,CACnB,QAAQ,EACR,IAAI,CAAC,SAAS,EACd,MAA4C,CAC7C,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,qEAAqE;YACrE,oEAAoE;YACpE,yDAAyD;YACzD,OAAO,CAAC,IAAI,CACV,uCAAuC,IAAI,CAAC,SAAS,GAAG,EACvD,GAAa,CAAC,OAAO,CACvB,CAAC;QACJ,CAAC;IACH,CAAC;AACH,CAAC;AAED,IAAI,QAAQ,GAAG,KAAK,CAAC;AACrB,IAAI,MAAkD,CAAC;AAEvD;;;GAGG;AACH,MAAM,UAAU,2BAA2B,CACzC,OAKI,EAAE;IAEN,IAAI,QAAQ;QAAE,OAAO;IACrB,QAAQ,GAAG,IAAI,CAAC;IAEhB,sEAAsE;IACtE,0EAA0E;IAC1E,uEAAuE;IACvE,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,YAAY,EAAE,CAAC;QAC1C,OAAO;IACT,CAAC;IACD,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;IACrD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;IAEjD,2EAA2E;IAC3E,8BAA8B;IAC9B,MAAM,YAAY,GAAG,UAAU,CAAC,GAAG,EAAE;QACnC,2BAA2B,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACtD,OAAO,CAAC,KAAK,CAAC,uCAAuC,EAAE,GAAG,CAAC,CAAC;QAC9D,CAAC,CAAC,CAAC;IACL,CAAC,EAAE,MAAM,CAAC,CAAC;IAEX,MAAM,GAAG,WAAW,CAAC,GAAG,EAAE;QACxB,2BAA2B,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACtD,OAAO,CAAC,KAAK,CAAC,wCAAwC,EAAE,GAAG,CAAC,CAAC;QAC/D,CAAC,CAAC,CAAC;IACL,CAAC,EAAE,UAAU,CAAC,CAAC;IAEf,4DAA4D;IAC5D,IACE,OAAO,YAAY,KAAK,QAAQ;QAChC,YAAY;QACZ,OAAO,IAAI,YAAY,EACvB,CAAC;QACA,YAAsC,CAAC,KAAK,EAAE,CAAC;IAClD,CAAC;IACD,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,IAAI,OAAO,IAAI,MAAM,EAAE,CAAC;QAC7D,MAAgC,CAAC,KAAK,EAAE,CAAC;IAC5C,CAAC;AACH,CAAC"}

package/dist/observability/evals.d.ts

@@ -0,0 +1,22 @@
+import type { EvalResult, EvalCriteria } from "./types.js";
+import type { AgentEngine } from "../agent/engine/types.js";
+export declare function runAutomatedEvals(runId: string): Promise<EvalResult[]>;
+export declare function runLlmJudgeEval(runId: string, criteria: EvalCriteria, opts?: {
+    engine?: AgentEngine;
+    model?: string;
+    userId?: string | null;
+}): Promise<EvalResult | null>;
+export declare function runDatasetEval(datasetId: string, opts?: {
+    criteria?: EvalCriteria[];
+    engine?: AgentEngine;
+    model?: string;
+}): Promise<{
+    datasetId: string;
+    totalCases: number;
+    avgScore: number;
+    results: EvalResult[];
+}>;
+export declare function evaluateRun(runId: string, opts?: {
+    sampleRate?: number;
+}): Promise<EvalResult[]>;
+//# sourceMappingURL=evals.d.ts.map

package/dist/observability/evals.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"evals.d.ts","sourceRoot":"","sources":["../../src/observability/evals.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,UAAU,EACV,YAAY,EAGb,MAAM,YAAY,CAAC;AAGpB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AA0I5D,wBAAsB,iBAAiB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC,CAsB5E;AA2DD,wBAAsB,eAAe,CACnC,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,YAAY,EACtB,IAAI,CAAC,EAAE;IAAE,MAAM,CAAC,EAAE,WAAW,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,GACtE,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,CA6E5B;AAID,wBAAsB,cAAc,CAClC,SAAS,EAAE,MAAM,EACjB,IAAI,CAAC,EAAE;IAAE,QAAQ,CAAC,EAAE,YAAY,EAAE,CAAC;IAAC,MAAM,CAAC,EAAE,WAAW,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,GACzE,OAAO,CAAC;IACT,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,UAAU,EAAE,CAAC;CACvB,CAAC,CAkDD;AAuGD,wBAAsB,WAAW,CAC/B,KAAK,EAAE,MAAM,EACb,IAAI,CAAC,EAAE;IAAE,UAAU,CAAC,EAAE,MAAM,CAAA;CAAE,GAC7B,OAAO,CAAC,UAAU,EAAE,CAAC,CA6BvB"}

package/dist/observability/evals.js

@@ -0,0 +1,371 @@
+import { getTraceSummary, insertEvalResult, getEvalDataset } from "./store.js";
+import { getRunById, getRunEventsSince } from "../agent/run-store.js";
+import { resolveEngine, getStoredModelForEngine, } from "../agent/engine/index.js";
+const LATENCY_BASELINE_PER_TOOL_MS = 10_000;
+const COST_BASELINE_PER_TOOL_CX100 = 50;
+const LLM_JUDGE_TIMEOUT_MS = 30_000;
+function makeEvalResult(opts) {
+    return {
+        id: crypto.randomUUID(),
+        runId: opts.runId,
+        threadId: opts.threadId,
+        userId: opts.userId,
+        evalType: opts.evalType,
+        criteria: opts.criteria,
+        score: Math.max(0, Math.min(1, opts.score)),
+        reasoning: opts.reasoning ?? null,
+        metadata: opts.metadata ?? null,
+        createdAt: Date.now(),
+    };
+}
+/** Lift the (runId, threadId, userId) triple off a TraceSummary —
+ *  every automated scorer pulls these together. */
+function fromSummary(summary) {
+    return {
+        runId: summary.runId,
+        threadId: summary.threadId,
+        userId: summary.userId,
+    };
+}
+// ─── Layer 1: Automated deterministic scorers ────────────────────────
+function scoreToolSuccessRate(summary) {
+    const total = summary.toolCalls;
+    const score = total > 0 ? summary.successfulTools / total : 1.0;
+    return makeEvalResult({
+        ...fromSummary(summary),
+        evalType: "automated",
+        criteria: "tool_success_rate",
+        score,
+        metadata: {
+            totalTools: total,
+            successfulTools: summary.successfulTools,
+            failedTools: summary.failedTools,
+        },
+    });
+}
+function scoreStepEfficiency(summary) {
+    // No tool calls = simple Q&A, maximally efficient.
+    // With tools: penalize excessive LLM iterations relative to tool calls.
+    const score = summary.toolCalls === 0
+        ? 1.0
+        : summary.llmCalls > 0
+            ? Math.min(1, summary.toolCalls / summary.llmCalls)
+            : 1.0;
+    return makeEvalResult({
+        ...fromSummary(summary),
+        evalType: "automated",
+        criteria: "step_efficiency",
+        score,
+        metadata: { llmCalls: summary.llmCalls, toolCalls: summary.toolCalls },
+    });
+}
+function scoreLatency(summary) {
+    const expectedMs = Math.max(LATENCY_BASELINE_PER_TOOL_MS, summary.toolCalls * LATENCY_BASELINE_PER_TOOL_MS);
+    const score = Math.max(0, 1 - summary.totalDurationMs / expectedMs);
+    return makeEvalResult({
+        ...fromSummary(summary),
+        evalType: "automated",
+        criteria: "latency_score",
+        score,
+        metadata: { actualMs: summary.totalDurationMs, expectedMs },
+    });
+}
+function scoreCostEfficiency(summary) {
+    const expectedCx100 = Math.max(COST_BASELINE_PER_TOOL_CX100, summary.toolCalls * COST_BASELINE_PER_TOOL_CX100);
+    const score = Math.max(0, 1 - summary.totalCostCentsX100 / expectedCx100);
+    return makeEvalResult({
+        ...fromSummary(summary),
+        evalType: "automated",
+        criteria: "cost_efficiency",
+        score,
+        metadata: { actualCx100: summary.totalCostCentsX100, expectedCx100 },
+    });
+}
+function scoreErrorRecovery(summary, runStatus) {
+    const hadErrors = summary.failedTools > 0;
+    let score;
+    if (!hadErrors) {
+        score = 1.0;
+    }
+    else if (runStatus === "completed") {
+        score = 1.0;
+    }
+    else {
+        score = 0;
+    }
+    return makeEvalResult({
+        ...fromSummary(summary),
+        evalType: "automated",
+        criteria: "error_recovery",
+        score,
+        metadata: { hadErrors, runStatus },
+    });
+}
+export async function runAutomatedEvals(runId) {
+    const [summary, run] = await Promise.all([
+        getTraceSummary(runId),
+        getRunById(runId),
+    ]);
+    if (!summary)
+        return [];
+    const runStatus = run?.status ?? "unknown";
+    const results = [
+        scoreToolSuccessRate(summary),
+        scoreStepEfficiency(summary),
+        scoreLatency(summary),
+        scoreCostEfficiency(summary),
+        scoreErrorRecovery(summary, runStatus),
+    ];
+    for (const result of results) {
+        insertEvalResult(result).catch(() => { });
+    }
+    return results;
+}
+// ─── Layer 2: LLM-as-Judge ───────────────────────────────────────────
+function buildConversationTranscript(events) {
+    const lines = [];
+    for (const { eventData } of events) {
+        try {
+            const event = JSON.parse(eventData);
+            if (event.type === "user-message") {
+                lines.push(`[User]: ${event.text ?? JSON.stringify(event.content)}`);
+            }
+            else if (event.type === "text-delta" || event.type === "text") {
+                lines.push(`[Agent]: ${event.text}`);
+            }
+            else if (event.type === "tool-call") {
+                lines.push(`[Tool Call: ${event.name}] ${JSON.stringify(event.input)}`);
+            }
+            else if (event.type === "tool-result") {
+                const snippet = typeof event.content === "string"
+                    ? event.content.slice(0, 500)
+                    : JSON.stringify(event.content).slice(0, 500);
+                lines.push(`[Tool Result${event.isError ? " (ERROR)" : ""}]: ${snippet}`);
+            }
+        }
+        catch {
+            // Skip unparseable events
+        }
+    }
+    return lines.join("\n");
+}
+function buildJudgePrompt(transcript, criteria) {
+    let prompt = `You are an expert evaluator. Assess the following agent conversation against the given criteria.
+
+## Criteria
+Name: ${criteria.name}
+Description: ${criteria.description}`;
+    if (criteria.rubric) {
+        prompt += `\nRubric: ${criteria.rubric}`;
+    }
+    const min = criteria.scoreRange?.min ?? 0;
+    const max = criteria.scoreRange?.max ?? 1;
+    prompt += `
+
+## Conversation Transcript
+${transcript}
+
+## Instructions
+Evaluate the conversation and respond with ONLY a JSON object (no markdown, no explanation outside the JSON):
+{"score": <number between ${min} and ${max}>, "reasoning": "<brief explanation>"}`;
+    return prompt;
+}
+export async function runLlmJudgeEval(runId, criteria, opts) {
+    try {
+        const [events, run] = await Promise.all([
+            getRunEventsSince(runId, 0),
+            getRunById(runId),
+        ]);
+        if (events.length === 0)
+            return null;
+        const transcript = buildConversationTranscript(events);
+        if (!transcript.trim())
+            return null;
+        const engine = opts?.engine ?? (await resolveEngine({ engineOption: undefined }));
+        const model = opts?.model ??
+            (await getStoredModelForEngine(engine)) ??
+            engine.defaultModel;
+        const judgePrompt = buildJudgePrompt(transcript, criteria);
+        const controller = new AbortController();
+        const timeout = setTimeout(() => controller.abort(), LLM_JUDGE_TIMEOUT_MS);
+        let responseText = "";
+        try {
+            const stream = engine.stream({
+                model,
+                systemPrompt: "You are an evaluation judge. Respond only with valid JSON.",
+                messages: [
+                    { role: "user", content: [{ type: "text", text: judgePrompt }] },
+                ],
+                tools: [],
+                abortSignal: controller.signal,
+                maxOutputTokens: 512,
+                temperature: 0,
+            });
+            for await (const event of stream) {
+                if (event.type === "text-delta") {
+                    responseText += event.text;
+                }
+            }
+        }
+        finally {
+            clearTimeout(timeout);
+        }
+        const jsonMatch = responseText.match(/\{[\s\S]*\}/);
+        if (!jsonMatch)
+            return null;
+        const parsed = JSON.parse(jsonMatch[0]);
+        const min = criteria.scoreRange?.min ?? 0;
+        const max = criteria.scoreRange?.max ?? 1;
+        const normalizedScore = max > min ? (parsed.score - min) / (max - min) : parsed.score;
+        const result = makeEvalResult({
+            runId,
+            threadId: run?.threadId ?? null,
+            userId: opts?.userId ?? null,
+            evalType: "llm_judge",
+            criteria: criteria.name,
+            score: normalizedScore,
+            reasoning: parsed.reasoning,
+            metadata: { model, rawScore: parsed.score, scoreRange: { min, max } },
+        });
+        insertEvalResult(result).catch(() => { });
+        return result;
+    }
+    catch {
+        return null;
+    }
+}
+// ─── Layer 3: Dataset evaluation ─────────────────────────────────────
+export async function runDatasetEval(datasetId, opts) {
+    const dataset = await getEvalDataset(datasetId);
+    if (!dataset) {
+        return { datasetId, totalCases: 0, avgScore: 0, results: [] };
+    }
+    const engine = opts?.engine ?? (await resolveEngine({ engineOption: undefined }));
+    const model = opts?.model ??
+        (await getStoredModelForEngine(engine)) ??
+        engine.defaultModel;
+    const criteria = opts?.criteria ?? [
+        {
+            name: "response_quality",
+            description: "How well the agent's response addresses the user's input, considering accuracy, completeness, and helpfulness.",
+        },
+    ];
+    const allResults = [];
+    for (const testCase of dataset.entries) {
+        const transcript = buildTestCaseTranscript(testCase, engine, model);
+        for (const c of criteria) {
+            const result = await evaluateTestCase(datasetId, testCase, transcript, c, engine, model);
+            if (result)
+                allResults.push(result);
+        }
+    }
+    const avgScore = allResults.length > 0
+        ? allResults.reduce((sum, r) => sum + r.score, 0) / allResults.length
+        : 0;
+    return {
+        datasetId,
+        totalCases: dataset.entries.length,
+        avgScore,
+        results: allResults,
+    };
+}
+function buildTestCaseTranscript(testCase, _engine, _model) {
+    let transcript = `[User]: ${testCase.input}`;
+    if (testCase.expectedOutput) {
+        transcript += `\n[Expected Output]: ${testCase.expectedOutput}`;
+    }
+    if (testCase.context) {
+        transcript += `\n[Context]: ${JSON.stringify(testCase.context)}`;
+    }
+    return transcript;
+}
+async function evaluateTestCase(datasetId, testCase, transcript, criteria, engine, model) {
+    try {
+        const judgePrompt = buildJudgePrompt(transcript, criteria);
+        const controller = new AbortController();
+        const timeout = setTimeout(() => controller.abort(), LLM_JUDGE_TIMEOUT_MS);
+        let responseText = "";
+        try {
+            const stream = engine.stream({
+                model,
+                systemPrompt: "You are an evaluation judge. Respond only with valid JSON.",
+                messages: [
+                    { role: "user", content: [{ type: "text", text: judgePrompt }] },
+                ],
+                tools: [],
+                abortSignal: controller.signal,
+                maxOutputTokens: 512,
+                temperature: 0,
+            });
+            for await (const event of stream) {
+                if (event.type === "text-delta") {
+                    responseText += event.text;
+                }
+            }
+        }
+        finally {
+            clearTimeout(timeout);
+        }
+        const jsonMatch = responseText.match(/\{[\s\S]*\}/);
+        if (!jsonMatch)
+            return null;
+        const parsed = JSON.parse(jsonMatch[0]);
+        const min = criteria.scoreRange?.min ?? 0;
+        const max = criteria.scoreRange?.max ?? 1;
+        const normalizedScore = max > min ? (parsed.score - min) / (max - min) : parsed.score;
+        // Dataset evals use a synthetic runId since there's no real run
+        const syntheticRunId = `dataset:${datasetId}:${crypto.randomUUID()}`;
+        // Dataset evals are administrative — there's no per-user runId, so
+        // we leave userId null. Per-user reads filter null rows out, which
+        // is the right default; admins can fetch dataset evals via the
+        // unfiltered call path.
+        const result = makeEvalResult({
+            runId: syntheticRunId,
+            threadId: null,
+            userId: null,
+            evalType: "llm_judge",
+            criteria: criteria.name,
+            score: normalizedScore,
+            reasoning: parsed.reasoning,
+            metadata: {
+                datasetId,
+                model,
+                testCaseInput: testCase.input,
+                expectedOutput: testCase.expectedOutput ?? null,
+                tags: testCase.tags ?? [],
+                rawScore: parsed.score,
+                scoreRange: { min, max },
+            },
+        });
+        insertEvalResult(result).catch(() => { });
+        return result;
+    }
+    catch {
+        return null;
+    }
+}
+// ─── Orchestrator ────────────────────────────────────────────────────
+export async function evaluateRun(runId, opts) {
+    const results = await runAutomatedEvals(runId);
+    const userId = results[0]?.userId ?? null;
+    const sampleRate = opts?.sampleRate ?? 0;
+    if (sampleRate > 0 && Math.random() < sampleRate) {
+        const defaultCriteria = [
+            {
+                name: "overall_quality",
+                description: "Overall quality of the agent's response, considering helpfulness, accuracy, and appropriate tool usage.",
+            },
+            {
+                name: "task_completion",
+                description: "Whether the agent successfully completed the user's requested task.",
+            },
+        ];
+        const judgeResults = await Promise.all(defaultCriteria.map((c) => runLlmJudgeEval(runId, c, { userId })));
+        for (const r of judgeResults) {
+            if (r)
+                results.push(r);
+        }
+    }
+    return results;
+}
+//# sourceMappingURL=evals.js.map