@vellumai/assistant 0.8.12 → 0.9.0-staging.2

package/src/util/platform.ts

@@ -72,7 +72,7 @@ export function normalizeAssistantId(assistantId: string): string {
 }
 
 /**
- * Returns the internal data directory (~/.vellum/workspace/data). Runtime
+ * Returns the internal data directory ($VELLUM_WORKSPACE_DIR/data). Runtime
  * databases, logs, memory indices, and other internal state live here.
  */
 export function getDataDir(): string {
@@ -96,7 +96,7 @@ export function getConfigQuarantineNoticePath(): string {
 }
 
 /**
- * Returns the embedding models directory (~/.vellum/workspace/embedding-models).
+ * Returns the embedding models directory ($VELLUM_WORKSPACE_DIR/embedding-models).
  * Downloaded embedding runtime (onnxruntime-node, transformers bundle, model weights)
  * is stored here, downloaded post-hatch rather than shipped with the app.
  */
@@ -113,7 +113,7 @@ export function getSandboxRootDir(): string {
 }
 
 /**
- * Returns the default sandbox working directory (~/.vellum/workspace).
+ * Returns the default sandbox working directory ($VELLUM_WORKSPACE_DIR).
  * This is the workspace root — tool working directories should use this
  * path unless explicitly overridden.
  */
@@ -122,7 +122,7 @@ export function getSandboxWorkingDir(): string {
 }
 
 /**
- * Returns the sounds directory (~/.vellum/workspace/data/sounds).
+ * Returns the sounds directory ($VELLUM_WORKSPACE_DIR/data/sounds).
  * Custom sound files and sound configuration live here.
  */
 export function getSoundsDir(): string {
@@ -137,7 +137,7 @@ export function getAvatarDir(): string {
 /** Canonical filename for the custom avatar PNG. */
 export const AVATAR_IMAGE_FILENAME = "avatar-image.png";
 
-/** Returns the canonical avatar image path (~/.vellum/workspace/data/avatar/avatar-image.png). */
+/** Returns the canonical avatar image path ($VELLUM_WORKSPACE_DIR/data/avatar/avatar-image.png). */
 export function getAvatarImagePath(): string {
   return join(getAvatarDir(), AVATAR_IMAGE_FILENAME);
 }
@@ -145,7 +145,7 @@ export function getAvatarImagePath(): string {
 /** Canonical filename for the avatar state manifest. */
 export const AVATAR_MANIFEST_FILENAME = "avatar.json";
 
-/** Returns the canonical avatar manifest path (~/.vellum/workspace/data/avatar/avatar.json). */
+/** Returns the canonical avatar manifest path ($VELLUM_WORKSPACE_DIR/data/avatar/avatar.json). */
 export function getAvatarManifestPath(): string {
   return join(getAvatarDir(), AVATAR_MANIFEST_FILENAME);
 }
@@ -203,7 +203,7 @@ export function getProtectedDir(): string {
   return join(vellumRoot(), "protected");
 }
 
-/** Returns ~/.vellum/workspace/signals — the directory for IPC signal files. */
+/** Returns $VELLUM_WORKSPACE_DIR/signals — the directory for IPC signal files. */
 export function getSignalsDir(): string {
   return join(getWorkspaceDir(), "signals");
 }
@@ -212,22 +212,22 @@ export function getSignalsDir(): string {
 // These expose specific root-level file paths so callers don't need to
 // import getRootDir() directly. getRootDir() is intentionally unexported.
 
-/** Returns the path to the daemon stderr log (~/.vellum/workspace/logs/daemon-stderr.log). */
+/** Returns the path to the daemon stderr log ($VELLUM_WORKSPACE_DIR/logs/daemon-stderr.log). */
 export function getDaemonStderrLogPath(): string {
   return join(getWorkspaceDir(), "logs", "daemon-stderr.log");
 }
 
-/** Returns the path to the daemon startup lock file (~/.vellum/workspace/daemon-startup.lock). */
+/** Returns the path to the daemon startup lock file ($VELLUM_WORKSPACE_DIR/daemon-startup.lock). */
 export function getDaemonStartupLockPath(): string {
   return join(getWorkspaceDir(), "daemon-startup.lock");
 }
 
-/** Returns the directory for externally-installed packages (~/.vellum/workspace/external). */
+/** Returns the directory for externally-installed packages ($VELLUM_WORKSPACE_DIR/external). */
 export function getExternalDir(): string {
   return join(getWorkspaceDir(), "external");
 }
 
-/** Returns the directory for installed binaries (~/.vellum/workspace/bin). */
+/** Returns the directory for installed binaries ($VELLUM_WORKSPACE_DIR/bin). */
 export function getBinDir(): string {
   return join(getWorkspaceDir(), "bin");
 }
@@ -237,7 +237,7 @@ export function getDotEnvPath(): string {
   return join(vellumRoot(), ".env");
 }
 
-/** Returns the path to the embed-worker PID file (~/.vellum/workspace/embed-worker.pid). */
+/** Returns the path to the embed-worker PID file ($VELLUM_WORKSPACE_DIR/embed-worker.pid). */
 export function getEmbedWorkerPidPath(): string {
   return join(getWorkspaceDir(), "embed-worker.pid");
 }
@@ -273,17 +273,17 @@ export function getWorkspaceDirDisplay(): string {
   return abs;
 }
 
-/** Returns ~/.vellum/workspace/config.json */
+/** Returns $VELLUM_WORKSPACE_DIR/config.json */
 export function getWorkspaceConfigPath(): string {
   return join(getWorkspaceDir(), "config.json");
 }
 
-/** Returns ~/.vellum/workspace/skills */
+/** Returns $VELLUM_WORKSPACE_DIR/skills */
 export function getWorkspaceSkillsDir(): string {
   return join(getWorkspaceDir(), "skills");
 }
 
-/** Returns ~/.vellum/workspace/hooks */
+/** Returns $VELLUM_WORKSPACE_DIR/hooks */
 export function getWorkspaceHooksDir(): string {
   return join(getWorkspaceDir(), "hooks");
 }
@@ -319,12 +319,12 @@ export function getWorkspaceRoutesDir(): string {
   return join(getWorkspaceDir(), "routes");
 }
 
-/** Returns ~/.vellum/workspace/deprecated — transitional files slated for removal. */
+/** Returns $VELLUM_WORKSPACE_DIR/deprecated — transitional files slated for removal. */
 export function getDeprecatedDir(): string {
   return join(getWorkspaceDir(), "deprecated");
 }
 
-/** Returns ~/.vellum/workspace/conversations */
+/** Returns $VELLUM_WORKSPACE_DIR/conversations */
 export function getConversationsDir(): string {
   return join(getWorkspaceDir(), "conversations");
 }

package/src/watcher/__tests__/engine.test.ts

@@ -105,6 +105,21 @@ mock.module("../../runtime/background-job-runner.js", () => ({
   },
 }));
 
+const inventoryCalls: number[] = [];
+const llmProcessedCalls: Array<{
+  providerId: string;
+  conversationId: string;
+}> = [];
+
+mock.module("../telemetry.js", () => ({
+  recordWatcherInventoryIfDue: (now: number) => {
+    inventoryCalls.push(now);
+  },
+  recordWatcherLlmProcessed: (providerId: string, conversationId: string) => {
+    llmProcessedCalls.push({ providerId, conversationId });
+  },
+}));
+
 // Import after mocks are in place.
 const { runWatchersOnce } = await import("../engine.js");
 
@@ -156,6 +171,8 @@ beforeEach(() => {
   setConvCalls.length = 0;
   dispositionCalls.length = 0;
   runJobCalls.length = 0;
+  inventoryCalls.length = 0;
+  llmProcessedCalls.length = 0;
   runJobImpl = async () => ({ conversationId: "conv-stub", ok: true });
 });
 
@@ -223,6 +240,9 @@ describe("runWatchersOnce — Phase 2 runBackgroundJob integration", () => {
     expect(setConvCalls).toEqual([
       { watcherId: "watcher-1", conversationId: "conv-success" },
     ]);
+    expect(llmProcessedCalls).toEqual([
+      { providerId: "linear", conversationId: "conv-success" },
+    ]);
     expect(dispositionCalls).toHaveLength(2);
     for (const call of dispositionCalls) {
       expect(call.disposition).toBe("silent");
@@ -267,6 +287,8 @@ describe("runWatchersOnce — Phase 2 runBackgroundJob integration", () => {
     // Critical: we must NOT have called setWatcherConversationId with "",
     // which would clobber a valid prior conversation id in the DB.
     expect(setConvCalls).toEqual([]);
+    // No conversation was bootstrapped, so no usage breadcrumb either.
+    expect(llmProcessedCalls).toEqual([]);
     // Failure path still updates event dispositions.
     expect(dispositionCalls).toHaveLength(1);
     expect(dispositionCalls[0].disposition).toBe("error");
@@ -280,6 +302,8 @@ describe("runWatchersOnce — Phase 2 runBackgroundJob integration", () => {
 
     expect(runJobCalls).toHaveLength(0);
     expect(setConvCalls).toHaveLength(0);
+    // Inventory telemetry runs on every tick regardless of pending work.
+    expect(inventoryCalls).toHaveLength(1);
   });
 
   test("malicious payload reaches the runner only inside assistant-role sandwich.content", async () => {

package/src/watcher/__tests__/telemetry.test.ts

@@ -0,0 +1,135 @@
+/**
+ * Tests for watcher usage telemetry.
+ *
+ * Strategy: stub the checkpoint store, lifecycle-event store, and
+ * watcher store via `mock.module()` so we can drive the inventory
+ * throttle and event naming without touching the DB.
+ */
+
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+const checkpoints = new Map<string, string>();
+const recordedEvents: string[] = [];
+let fakeEnabledWatchers: Array<{ providerId: string }> = [];
+let recordImpl: (name: string) => void = (name) => {
+  recordedEvents.push(name);
+};
+
+mock.module("../../memory/checkpoints.js", () => ({
+  getMemoryCheckpoint: (key: string) => checkpoints.get(key) ?? null,
+  setMemoryCheckpoint: (key: string, value: string) => {
+    checkpoints.set(key, value);
+  },
+}));
+
+mock.module("../../memory/lifecycle-events-store.js", () => ({
+  recordLifecycleEvent: (name: string) => {
+    recordImpl(name);
+    return null;
+  },
+}));
+
+mock.module("../watcher-store.js", () => ({
+  listWatchers: () => fakeEnabledWatchers,
+}));
+
+const {
+  recordWatcherInventoryIfDue,
+  recordWatcherLlmProcessed,
+  WATCHER_INVENTORY_INTERVAL_MS,
+} = await import("../telemetry.js");
+
+const INVENTORY_KEY = "telemetry:watchers:inventory_last_recorded";
+// Realistic epoch base: an absent checkpoint reads as 0, so `now` must be
+// at least one interval past the epoch for the inventory to be due.
+const BASE = 1_700_000_000_000;
+
+beforeEach(() => {
+  checkpoints.clear();
+  recordedEvents.length = 0;
+  fakeEnabledWatchers = [];
+  recordImpl = (name) => {
+    recordedEvents.push(name);
+  };
+});
+
+describe("recordWatcherInventoryIfDue", () => {
+  test("records one event per enabled watcher when due", () => {
+    fakeEnabledWatchers = [
+      { providerId: "gmail" },
+      { providerId: "linear" },
+      { providerId: "gmail" },
+    ];
+
+    recordWatcherInventoryIfDue(BASE);
+
+    expect(recordedEvents).toEqual([
+      "watcher_enabled:gmail",
+      "watcher_enabled:linear",
+      "watcher_enabled:gmail",
+    ]);
+    expect(checkpoints.get(INVENTORY_KEY)).toBe(String(BASE));
+  });
+
+  test("skips when the last inventory is within the interval", () => {
+    fakeEnabledWatchers = [{ providerId: "gmail" }];
+    checkpoints.set(INVENTORY_KEY, String(BASE));
+
+    recordWatcherInventoryIfDue(BASE + WATCHER_INVENTORY_INTERVAL_MS - 1);
+
+    expect(recordedEvents).toEqual([]);
+    expect(checkpoints.get(INVENTORY_KEY)).toBe(String(BASE));
+  });
+
+  test("records again once the interval has elapsed", () => {
+    fakeEnabledWatchers = [{ providerId: "github" }];
+    checkpoints.set(INVENTORY_KEY, String(BASE));
+
+    recordWatcherInventoryIfDue(BASE + WATCHER_INVENTORY_INTERVAL_MS);
+
+    expect(recordedEvents).toEqual(["watcher_enabled:github"]);
+  });
+
+  test("advances the checkpoint even when no watchers are enabled", () => {
+    recordWatcherInventoryIfDue(BASE + 1);
+
+    expect(recordedEvents).toEqual([]);
+    expect(checkpoints.get(INVENTORY_KEY)).toBe(String(BASE + 1));
+  });
+
+  test("swallows storage errors and leaves the checkpoint for retry", () => {
+    fakeEnabledWatchers = [{ providerId: "gmail" }];
+    recordImpl = () => {
+      throw new Error("db unavailable");
+    };
+
+    expect(() => recordWatcherInventoryIfDue(BASE + 2)).not.toThrow();
+    // Checkpoint must not advance on failure: the next tick retries
+    // instead of skipping a day of inventory.
+    expect(checkpoints.get(INVENTORY_KEY)).toBeUndefined();
+
+    recordImpl = (name) => {
+      recordedEvents.push(name);
+    };
+    recordWatcherInventoryIfDue(BASE + 3);
+
+    expect(recordedEvents).toEqual(["watcher_enabled:gmail"]);
+    expect(checkpoints.get(INVENTORY_KEY)).toBe(String(BASE + 3));
+  });
+});
+
+describe("recordWatcherLlmProcessed", () => {
+  test("embeds provider and conversation id in the event name", () => {
+    recordWatcherLlmProcessed("linear", "conv-123");
+
+    expect(recordedEvents).toEqual(["watcher_llm_processed:linear:conv-123"]);
+  });
+
+  test("swallows storage errors", () => {
+    recordImpl = () => {
+      throw new Error("db unavailable");
+    };
+
+    expect(() => recordWatcherLlmProcessed("gmail", "conv-456")).not.toThrow();
+  });
+});

package/src/watcher/engine.ts

@@ -12,6 +12,10 @@ import { checkForSequenceReplies } from "../sequence/reply-matcher.js";
 import { getLogger } from "../util/logger.js";
 import { MAX_CONSECUTIVE_ERRORS, WATCHER_JOB_TIMEOUT_MS } from "./constants.js";
 import { getWatcherProvider } from "./provider-registry.js";
+import {
+  recordWatcherInventoryIfDue,
+  recordWatcherLlmProcessed,
+} from "./telemetry.js";
 import {
   claimDueWatchers,
   completeWatcherPoll,
@@ -69,6 +73,8 @@ export async function runWatchersOnce(
   const now = Date.now();
   let processed = 0;
 
+  recordWatcherInventoryIfDue(now);
+
   // ── Phase 1: Poll providers for new events ──────────────────────
   const claimed = claimDueWatchers(now);
   for (const watcher of claimed) {
@@ -269,6 +275,7 @@ export async function runWatchersOnce(
     // is empty) — otherwise we'd overwrite a valid prior id with "".
     if (result.conversationId !== "") {
       setWatcherConversationId(watcher.id, result.conversationId);
+      recordWatcherLlmProcessed(watcher.providerId, result.conversationId);
     }
 
     if (result.ok) {

package/src/watcher/telemetry.ts

@@ -0,0 +1,74 @@
+/**
+ * Watcher usage telemetry — lifecycle-event breadcrumbs that let fleet
+ * analytics answer "is anyone using watchers, and what do they cost?"
+ * ahead of the planned watcher → skills-and-schedules migration.
+ *
+ * Both event shapes ship through the existing lifecycle-event telemetry
+ * pipeline, so they inherit its `collectUsageData` opt-out gate and need
+ * no wire-contract or platform-side changes:
+ *
+ * - `watcher_enabled:<providerId>` — one per enabled watcher, at most
+ *   once per 24h per daemon. Counts devices that have watchers
+ *   configured, including ones whose providers rarely produce events.
+ * - `watcher_llm_processed:<providerId>:<conversationId>` — one per
+ *   watcher tick that ran an LLM background job. The conversation id is
+ *   embedded so analytics can join against `llm_usage` events for exact
+ *   cost attribution.
+ *
+ * Telemetry must never break the polling loop, so every entry point
+ * swallows storage errors.
+ */
+
+import {
+  getMemoryCheckpoint,
+  setMemoryCheckpoint,
+} from "../memory/checkpoints.js";
+import { recordLifecycleEvent } from "../memory/lifecycle-events-store.js";
+import { getLogger } from "../util/logger.js";
+import { listWatchers } from "./watcher-store.js";
+
+const log = getLogger("watcher-telemetry");
+
+const INVENTORY_CHECKPOINT_KEY = "telemetry:watchers:inventory_last_recorded";
+export const WATCHER_INVENTORY_INTERVAL_MS = 24 * 60 * 60 * 1000;
+
+/**
+ * Record one `watcher_enabled:<providerId>` lifecycle event per enabled
+ * watcher, at most once per 24h. Called from every watcher tick; the
+ * checkpoint advances only after the events are recorded, so a transient
+ * storage failure retries on the next tick instead of skipping a day.
+ * Retries can duplicate events for watchers recorded before the failure
+ * point — an acceptable overcount for best-effort telemetry, where losing
+ * a full day's inventory would not be.
+ */
+export function recordWatcherInventoryIfDue(now: number): void {
+  try {
+    const last = Number(getMemoryCheckpoint(INVENTORY_CHECKPOINT_KEY) ?? "0");
+    if (now - last < WATCHER_INVENTORY_INTERVAL_MS) return;
+    for (const watcher of listWatchers({ enabledOnly: true })) {
+      recordLifecycleEvent(`watcher_enabled:${watcher.providerId}`);
+    }
+    setMemoryCheckpoint(INVENTORY_CHECKPOINT_KEY, String(now));
+  } catch (err) {
+    log.warn({ err }, "Failed to record watcher inventory telemetry");
+  }
+}
+
+/**
+ * Record a `watcher_llm_processed:<providerId>:<conversationId>`
+ * lifecycle event for a watcher tick that bootstrapped an LLM background
+ * job. Recorded whenever a conversation was created — even if the job
+ * later failed — because tokens may already have been spent.
+ */
+export function recordWatcherLlmProcessed(
+  providerId: string,
+  conversationId: string,
+): void {
+  try {
+    recordLifecycleEvent(
+      `watcher_llm_processed:${providerId}:${conversationId}`,
+    );
+  } catch (err) {
+    log.warn({ err }, "Failed to record watcher LLM-processed telemetry");
+  }
+}