@vellumai/assistant 0.10.2 → 0.10.3-dev.202606252046.9075fd5

package/src/telemetry/usage-telemetry-reporter.ts

@@ -28,6 +28,7 @@ import {
   assembleBoundedTurnTrace,
   isTurnSettled,
 } from "../memory/turn-trace-store.js";
+import { queryUnreportedWatchdogEvents } from "../memory/watchdog-events-store.js";
 import { VellumPlatformClient } from "../platform/client.js";
 import {
   getCachedShareAnalytics,
@@ -76,6 +77,9 @@ const CHECKPOINT_KEY_SKILL_LOADED_WATERMARK =
   "telemetry:skill_loaded:last_reported_at";
 const CHECKPOINT_KEY_SKILL_LOADED_WATERMARK_ID =
   "telemetry:skill_loaded:last_reported_id";
+const CHECKPOINT_KEY_WATCHDOG_WATERMARK = "telemetry:watchdog:last_reported_at";
+const CHECKPOINT_KEY_WATCHDOG_WATERMARK_ID =
+  "telemetry:watchdog:last_reported_id";
 // Written into the `*_id` watermark checkpoints by the opt-out flush branch.
 // Sorts lexicographically above every real row ID (all event stores generate
 // lowercase v4 UUIDs), so the compound cursor's same-millisecond arm
@@ -100,6 +104,7 @@ const WATERMARK_KEY_PAIRS: ReadonlyArray<readonly [string, string]> = [
     CHECKPOINT_KEY_SKILL_LOADED_WATERMARK,
     CHECKPOINT_KEY_SKILL_LOADED_WATERMARK_ID,
   ],
+  [CHECKPOINT_KEY_WATCHDOG_WATERMARK, CHECKPOINT_KEY_WATCHDOG_WATERMARK_ID],
 ];
 const REPORT_INTERVAL_MS = 5 * 60 * 1000;
 const INITIAL_FLUSH_DELAY_MS = 30_000; // Delay first flush to let CES handshake complete
@@ -123,6 +128,37 @@ export function setUsageTelemetryReporter(
   _instance = reporter;
 }
 
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+/**
+ * Parse a stored `watchdog_events.detail` JSON text column into the object the
+ * platform expects. Returns null for a null column or an unparseable/corrupted
+ * blob (mirroring the turn `client` metadata parse: a bad blob emits null
+ * rather than failing the batch). A non-object (e.g. a bare number or string)
+ * also resolves to null, since the platform serializer treats `detail` as a
+ * JSON object bag.
+ */
+function parseWatchdogDetail(
+  raw: string | null,
+): Record<string, unknown> | null {
+  if (!raw) return null;
+  try {
+    const parsed = JSON.parse(raw) as unknown;
+    if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
+      return parsed as Record<string, unknown>;
+    }
+    return null;
+  } catch {
+    log.warn(
+      { rawDetail: raw.slice(0, 200) },
+      "Telemetry watchdog: failed to parse detail; emitting null",
+    );
+    return null;
+  }
+}
+
 // ---------------------------------------------------------------------------
 // Reporter
 // ---------------------------------------------------------------------------
@@ -307,7 +343,8 @@ export class UsageTelemetryReporter {
         undefined;
 
       // Read skill-loaded watermark (compound cursor: createdAt + id).
-      // Brand-new table, so the standard 0 default is safe.
+      // Writes are gated on share_analytics consent, so opted-out rows
+      // cannot exist and the standard 0 default is safe.
       const skillLoadedWatermark = Number(
         getMemoryCheckpoint(CHECKPOINT_KEY_SKILL_LOADED_WATERMARK) ?? "0",
       );
@@ -315,6 +352,15 @@ export class UsageTelemetryReporter {
         getMemoryCheckpoint(CHECKPOINT_KEY_SKILL_LOADED_WATERMARK_ID) ??
         undefined;
 
+      // Read watchdog watermark (compound cursor: createdAt + id).
+      // Writes are gated on share_analytics consent, so opted-out rows
+      // cannot exist and the standard 0 default is safe.
+      const watchdogWatermark = Number(
+        getMemoryCheckpoint(CHECKPOINT_KEY_WATCHDOG_WATERMARK) ?? "0",
+      );
+      const watchdogWatermarkId =
+        getMemoryCheckpoint(CHECKPOINT_KEY_WATCHDOG_WATERMARK_ID) ?? undefined;
+
       // Query unreported events
       const events = queryUnreportedUsageEvents(
         watermark,
@@ -351,6 +397,11 @@ export class UsageTelemetryReporter {
         skillLoadedWatermarkId,
         BATCH_SIZE,
       );
+      const watchdogEvents = queryUnreportedWatchdogEvents(
+        watchdogWatermark,
+        watchdogWatermarkId,
+        BATCH_SIZE,
+      );
 
       // Trace completeness barrier (trace-eligible owners only).
       //
@@ -418,7 +469,8 @@ export class UsageTelemetryReporter {
         onboardingEvents.length === 0 &&
         authFallbackEvents.length === 0 &&
         toolExecutedEvents.length === 0 &&
-        skillLoadedEvents.length === 0
+        skillLoadedEvents.length === 0 &&
+        watchdogEvents.length === 0
       )
         return;
 
@@ -442,6 +494,7 @@ export class UsageTelemetryReporter {
           authFallbackCount: authFallbackEvents.length,
           toolExecutedCount: toolExecutedEvents.length,
           skillLoadedCount: skillLoadedEvents.length,
+          watchdogCount: watchdogEvents.length,
         },
         "Telemetry flush: resolved auth context",
       );
@@ -674,6 +727,23 @@ export class UsageTelemetryReporter {
             assistant_version: APP_VERSION,
           }),
         ),
+        ...watchdogEvents.map(
+          (e): TelemetryEvent => ({
+            type: "watchdog",
+            daemon_event_id: e.id,
+            recorded_at: e.createdAt,
+            check_name: e.checkName,
+            value: e.value,
+            // `detail` is stored as JSON text; parse defensively so a
+            // corrupted blob never fails the batch flush. A parse failure
+            // emits null rather than dropping the event.
+            detail: parseWatchdogDetail(e.detail),
+            // `watchdog_events` has no record-time version column — same
+            // upload-time APP_VERSION stamping as the other non-llm_usage
+            // event types.
+            assistant_version: APP_VERSION,
+          }),
+        ),
       ];
 
       const organizationId = getPlatformOrganizationId() || undefined;
@@ -796,6 +866,19 @@ export class UsageTelemetryReporter {
         );
       }
 
+      // Advance watchdog watermark (compound cursor)
+      if (watchdogEvents.length > 0) {
+        const lastWatchdog = watchdogEvents[watchdogEvents.length - 1];
+        setMemoryCheckpoint(
+          CHECKPOINT_KEY_WATCHDOG_WATERMARK,
+          String(lastWatchdog.createdAt),
+        );
+        setMemoryCheckpoint(
+          CHECKPOINT_KEY_WATCHDOG_WATERMARK_ID,
+          lastWatchdog.id,
+        );
+      }
+
       // If we got a full batch of any type, there may be more — recurse.
       // Turns use the REPORTED count: when the completeness barrier truncates
       // the batch, the deferred turns must wait for a later flush (by which
@@ -808,7 +891,8 @@ export class UsageTelemetryReporter {
         onboardingEvents.length === BATCH_SIZE ||
         authFallbackEvents.length === BATCH_SIZE ||
         toolExecutedEvents.length === BATCH_SIZE ||
-        skillLoadedEvents.length === BATCH_SIZE
+        skillLoadedEvents.length === BATCH_SIZE ||
+        watchdogEvents.length === BATCH_SIZE
       ) {
         await this._doFlush(batchCount + 1);
       }

package/src/tools/ask-question/ask-question-tool.test.ts

@@ -206,6 +206,35 @@ describe("AskQuestionTool.execute", () => {
     expect(result.content).toBe("Question aborted");
   });
 
+  test("short-circuits without prompting when no interactive user is present", async () => {
+    setNextResult(singleCompleted({ decision: "option", optionId: "a" }));
+
+    const result = await askQuestionTool.execute(
+      validInput,
+      makeContext({ isInteractive: false }),
+    );
+
+    // The prompter must never be invoked — there is no one to answer, so the
+    // turn proceeds with defaults instead of parking on the response backstop.
+    expect(calls).toHaveLength(0);
+    expect(result.isError).toBe(false);
+    expect(result.content.toLowerCase()).toContain("no interactive user");
+  });
+
+  test("still prompts when isInteractive is true or unset", async () => {
+    setNextResult(singleCompleted({ decision: "option", optionId: "a" }));
+
+    // The short-circuit keys off an explicit `false`, not a missing flag, so an
+    // interactive turn (or one that never set the flag) still prompts.
+    await askQuestionTool.execute(
+      validInput,
+      makeContext({ isInteractive: true }),
+    );
+    await askQuestionTool.execute(validInput, makeContext());
+
+    expect(calls).toHaveLength(2);
+  });
+
   test("rejects a question with fewer than 2 options", async () => {
     setNextResult(singleCompleted({ decision: "option", optionId: "a" }));
     const result = await askQuestionTool.execute(

package/src/tools/ask-question/ask-question-tool.ts

@@ -182,6 +182,19 @@ export const askQuestionTool = {
 
     const questions: SingleQuestion[] = parsed.data.questions;
 
+    // No interactive user is present to answer (scheduled/headless/background
+    // turn). Don't park the turn on a prompt no one can resolve — proceed with
+    // defaults immediately. Non-interactive turns are already instructed not to
+    // ask (NON_INTERACTIVE_CONTEXT_BLOCK); this is the backstop for when the
+    // model asks anyway, so it doesn't wait out the full response timeout.
+    if (context.isInteractive === false) {
+      return {
+        content:
+          "No interactive user is present to answer; proceeding with reasonable defaults.",
+        isError: false,
+      };
+    }
+
     const prompter = new QuestionPrompter();
     const result = await prompter.prompt({
       conversationId: context.conversationId,

package/src/tools/executor.ts

@@ -528,8 +528,8 @@ export { isSideEffectTool } from "./side-effects.js";
  * handles cleanup before the executor wrapper trips.
  *
  * `ask_question` blocks on user input inside `execute()` via `QuestionPrompter`,
- * which waits up to `permissionTimeoutSec`. We give the wrapper the same 5s
- * buffer over that deadline so the prompter's own timeout fires first and
+ * which waits up to `questionResponseTimeoutSec`. We give the wrapper the same
+ * 5s buffer over that deadline so the prompter's own timeout fires first and
  * returns its clean "User did not respond within timeout" result — otherwise
  * the shorter generic budget trips first, orphaning the still-pending prompt
  * behind the confusing "may still be running in the background" error.
@@ -556,8 +556,8 @@ export function computePerToolTimeoutMs(
     return (shellTimeoutSec + 5) * 1000;
   }
   if (name === "ask_question") {
-    const { permissionTimeoutSec } = getConfig().timeouts;
-    return (permissionTimeoutSec + 5) * 1000;
+    const { questionResponseTimeoutSec } = getConfig().timeouts;
+    return (questionResponseTimeoutSec + 5) * 1000;
   }
   const rawTimeoutSec = getConfig().timeouts.toolExecutionTimeoutSec;
   return safeTimeoutMs(rawTimeoutSec);