@vellumai/assistant 0.10.9-staging.1 → 0.10.9

package/src/runtime/routes/user-routes-cli.ts

@@ -13,10 +13,17 @@ import { z } from "zod";
 
 import { getConfig } from "../../config/loader.js";
 import { getPublicBaseUrl } from "../../inbound/public-ingress-urls.js";
-import { getWorkspaceRoutesDir } from "../../util/platform.js";
+import { getWorkspaceDir, getWorkspaceRoutesDir } from "../../util/platform.js";
 import { LOCAL_PRINCIPALS } from "../auth/route-policy.js";
 import { NotFoundError } from "./errors.js";
 import type { RouteDefinition, RouteHandlerArgs } from "./types.js";
+import {
+  HANDLER_EXTENSIONS,
+  isReservedWorkspaceRoutePath,
+  listPluginRouteRoots,
+  resolveHandlerFile,
+  resolveRouteLocation,
+} from "./user-route-resolution.js";
 
 // ── Constants ───────────────────────────────────────────────────────
 
@@ -32,8 +39,6 @@ const HTTP_METHODS = [
 
 type HttpMethod = (typeof HTTP_METHODS)[number];
 
-const HANDLER_EXTENSIONS = [".ts", ".js"] as const;
-
 type HandlerExtension = (typeof HANDLER_EXTENSIONS)[number];
 
 // ── Schemas ─────────────────────────────────────────────────────────
@@ -146,51 +151,67 @@ function tryGetPublicBaseUrl(): string | null {
   }
 }
 
-function resolveHandlerFile(
-  routesDir: string,
-  routePath: string,
-): string | null {
-  const basePath = join(routesDir, routePath);
-
-  for (const ext of HANDLER_EXTENSIONS) {
-    const candidate = `${basePath}${ext}`;
-    if (existsSync(candidate)) {
-      return candidate;
-    }
-  }
-
-  for (const ext of HANDLER_EXTENSIONS) {
-    const candidate = join(basePath, `index${ext}`);
-    if (existsSync(candidate)) {
-      return candidate;
-    }
-  }
-
-  return null;
+/**
+ * Strip a leading `/x/` (or `x/`) and surrounding slashes from a route path so
+ * `routes inspect` accepts both the bare sub-path (`ping`) and the `/x/`-prefixed
+ * form that `routes list` prints (`/x/plugins/demo/status`).
+ */
+function normalizeInspectPath(input: string): string {
+  const trimmed = input.replace(/^\/+/, "");
+  return trimmed.startsWith("x/") ? trimmed.slice(2) : trimmed;
 }
 
 // ── Handlers ────────────────────────────────────────────────────────
 
 async function handleUserRoutesList() {
-  const routesDir = getWorkspaceRoutesDir();
-  const discovered = await discoverRoutes(routesDir);
   const publicBase = tryGetPublicBaseUrl();
+  const workspaceDir = getWorkspaceDir();
 
-  const routes = discovered.map((r) => ({
-    routePath: `/x/${r.routePath}`,
+  const toEntry = (xPath: string, r: DiscoveredRoute) => ({
+    routePath: `/x/${xPath}`,
     methods: r.methods,
     description: r.description ?? null,
-    filePath: relative(routesDir, r.filePath),
-    publicUrl: publicBase ? `${publicBase}/x/${r.routePath}` : null,
-  }));
+    filePath: relative(workspaceDir, r.filePath),
+    publicUrl: publicBase ? `${publicBase}/x/${xPath}` : null,
+  });
+
+  const routes: ReturnType<typeof toEntry>[] = [];
+
+  // Workspace routes at `/x/<path>`. Paths shadowed by the reserved plugin
+  // namespace are skipped: `resolveRouteLocation` routes those to a plugin
+  // directory, so a `<workspace>/routes/plugins/…` file is never served and
+  // must not be advertised here.
+  for (const r of await discoverRoutes(getWorkspaceRoutesDir())) {
+    if (isReservedWorkspaceRoutePath(r.routePath)) {
+      continue;
+    }
+    routes.push(toEntry(r.routePath, r));
+  }
 
+  // Plugin routes at `/x/plugins/<name>/<sub>`, from each enabled plugin's
+  // `routes/` directory (same enumeration the dispatcher resolves against).
+  for (const { pluginName, routesDir } of listPluginRouteRoots()) {
+    for (const r of await discoverRoutes(routesDir)) {
+      const xPath = r.routePath
+        ? `plugins/${pluginName}/${r.routePath}`
+        : `plugins/${pluginName}`;
+      routes.push(toEntry(xPath, r));
+    }
+  }
+
+  routes.sort((a, b) => a.routePath.localeCompare(b.routePath));
   return { ok: true, routes };
 }
 
 async function handleUserRoutesInspect({ body = {} }: RouteHandlerArgs) {
-  const { path: routePath } = InspectParams.parse(body);
-  const routesDir = getWorkspaceRoutesDir();
-  const filePath = resolveHandlerFile(routesDir, routePath);
+  const routePath = normalizeInspectPath(InspectParams.parse(body).path);
+
+  const location = routePath.includes("..")
+    ? null
+    : resolveRouteLocation(routePath);
+  const filePath = location
+    ? resolveHandlerFile(location.routesDir, location.subPath)
+    : null;
 
   if (!filePath) {
     throw new NotFoundError(
@@ -209,7 +230,7 @@ async function handleUserRoutesInspect({ body = {} }: RouteHandlerArgs) {
       routePath: `/x/${routePath}`,
       methods,
       description: description ?? null,
-      filePath,
+      filePath: relative(getWorkspaceDir(), filePath),
       publicUrl,
       fileSize: stat.size,
       modifiedAt: stat.mtime.toISOString(),

package/src/telemetry/AGENTS.md

@@ -0,0 +1,15 @@
+# Telemetry — Agent Instructions
+
+## Wire contract
+
+`telemetry-wire.generated.ts` (in this directory) is generated from the platform's telemetry ingest serializers and auto-synced here on platform merges (the platform's `sync-telemetry-wire.yaml` workflow). **Never edit it by hand** — contract changes belong in `vellum-assistant-platform` at `django/app/assistant/self_hosted_local/serializers.py`.
+
+`types.ts` is the override layer on top of it: simple events flow through `WireEventMap` without restating fields — they use the generated types directly, so their construction sites get excess-property/missing-field errors when the contract moves. Events where the daemon's type is intentionally richer live in `Overrides`, each pinned to the wire type by compile-time guards covering both drift directions: `_*Narrows` (daemon values stay wire-assignable — catches wire-side tightening) and `_*KeysExist` (the daemon emits no field the wire no longer declares — catches platform-side field removals/renames, which structural subtyping would otherwise let through silently). Daemon-only events live in `Extensions`. A red guard or a failing `types.test.ts` on a sync PR means the platform contract moved — reconcile the override to the new wire shape, don't loosen the guard.
+
+Pre-flush validation (`telemetry-wire-validation.ts`) checks outgoing events against the wire schemas and logs any the server would silently drop; it is observability only and never blocks or mutates the batch.
+
+## Adding a new event type starts platform-side, not here
+
+The ingest endpoint silently skips events whose type has no registered serializer (the batch still 2xxes and the daemon acks away its outbox rows), so an emitter shipped before its platform serializer loses every event it records — pre-flush validation logs the drop but does not prevent it.
+
+The full cross-repo checklist lives with the serializer registry it governs, in `vellum-assistant-platform` at `django/app/assistant/self_hosted_local/AGENTS.md` ("Adding a new telemetry ingest event type"). The daemon emitter is the **last** step — added only after the platform serializer has merged and the wire sync PR has landed here. A new emitter is an outbox store plus registration in `telemetry-event-sources.ts` (or a watermark source for high-volume tables); if the type previously existed daemon-only, also move it out of `Extensions` in `types.ts`.

package/src/telemetry/__tests__/telemetry-event-fixtures.ts

@@ -0,0 +1,258 @@
+/**
+ * Shared sample telemetry events for the wire-contract test suites
+ * (`types.test.ts` and `telemetry-wire-validation.test.ts`).
+ *
+ * `wireEventSamples` holds exactly one daemon-typed, wire-valid sample per
+ * generated wire event type — when the wire contract gains an event type,
+ * add its sample here and both suites pick it up. The daemon-type
+ * annotations are themselves part of the contract test: they compile-check
+ * that realistic daemon values inhabit the hand-written override types.
+ *
+ * Imported by `*.test.ts` files only — never by the test preload — so
+ * importing daemon types from `src/` is safe (AGENTS.md "Test machinery
+ * isolation" scopes its no-`src/` rule to preload-time machinery). The
+ * imports are type-only; this module has no runtime side effects.
+ */
+import type {
+  AuthFallbackTelemetryEvent,
+  ConfigSettingTelemetryEvent,
+  LifecycleTelemetryEvent,
+  LlmUsageTelemetryEvent,
+  OnboardingResearchTelemetryEvent,
+  OnboardingTelemetryEvent,
+  SkillLoadedTelemetryEvent,
+  ToolExecutedTelemetryEvent,
+  TurnTelemetryEvent,
+  TurnTrace,
+  WatchdogTelemetryEvent,
+} from "../types.js";
+
+const RECORDED_AT = 1_750_000_000_000;
+
+const llmUsage: LlmUsageTelemetryEvent = {
+  type: "llm_usage",
+  daemon_event_id: "evt-llm-usage-001",
+  recorded_at: RECORDED_AT,
+  assistant_version: "1.2.3",
+  conversation_id: "conv-xyz",
+  conversation_type: "standard",
+  // Zero is legitimate here (wire bound is min(0)) — the wire schema must
+  // not reject it.
+  turn_index: 0,
+  provider: "anthropic",
+  model: "claude-fable-5",
+  input_tokens: 1200,
+  output_tokens: 340,
+  cache_creation_input_tokens: 256,
+  cache_read_input_tokens: 1024,
+  llm_call_count: 3,
+  raw_usage: {
+    input_tokens: 1200,
+    cache_creation: { ephemeral_5m_input_tokens: 256 },
+  },
+  actor: "user",
+  llm_call_site: "mainAgent",
+  inference_profile: "balanced",
+  inference_profile_source: "active",
+  cost: 0.0123,
+};
+
+const trace: TurnTrace = {
+  schema_version: 3,
+  messages: [
+    {
+      id: "msg-1",
+      role: "user",
+      created_at: RECORDED_AT - 5_000,
+      content: [{ type: "text", text: "What's on my calendar today?" }],
+      model: null,
+    },
+    {
+      id: "msg-2",
+      role: "assistant",
+      created_at: RECORDED_AT - 1_000,
+      content: [{ type: "text", text: "You have two meetings." }],
+      model: "claude-fable-5",
+    },
+  ],
+  tool_calls: [
+    {
+      id: "inv-1",
+      tool_name: "bash",
+      input: { command: "ls" },
+      result: "ok",
+      decision: "allow",
+      duration_ms: 42,
+      created_at: RECORDED_AT - 3_000,
+    },
+  ],
+  system_prompt: "You are a helpful assistant.",
+  tool_definitions: [
+    {
+      name: "bash",
+      description: "Run a shell command",
+      input_schema: { type: "object", properties: {} },
+    },
+  ],
+};
+
+/** Exported separately: both suites build negative `client`-bag cases on it. */
+export const turnEventSample: TurnTelemetryEvent = {
+  type: "turn",
+  daemon_event_id: "evt-turn-001",
+  recorded_at: RECORDED_AT,
+  assistant_version: "1.2.3",
+  conversation_id: "conv-xyz",
+  conversation_type: "standard",
+  turn_index: 1,
+  interface_id: "web",
+  channel_id: "vellum",
+  client: {
+    browser_family: "chrome",
+    browser_version: "124",
+    os: "macos",
+    interface_version: "0.8.2",
+  },
+  outcome: "failed",
+  failure_code: "PROVIDER_RATE_LIMIT",
+  trace,
+};
+
+const lifecycle: LifecycleTelemetryEvent = {
+  type: "lifecycle",
+  daemon_event_id: "evt-lifecycle-001",
+  recorded_at: RECORDED_AT,
+  assistant_version: "1.2.3",
+  event_name: "app_open",
+};
+
+const onboarding: OnboardingTelemetryEvent = {
+  type: "onboarding",
+  daemon_event_id: "evt-onboarding-001",
+  recorded_at: RECORDED_AT,
+  assistant_version: "1.2.3",
+  screen: "tools",
+  tools: ["gmail", "calendar"],
+  tasks: ["email-triage"],
+  tone: "friendly",
+  google_connected: true,
+  google_scopes: ["https://www.googleapis.com/auth/gmail.readonly"],
+  ab_variant: "control",
+  session_id: "sess-123",
+  step_name: "select-tools",
+  step_index: 2,
+  completed_at: "2026-07-13T00:00:00Z",
+  funnel_version: "v2",
+  user_id: "user-123",
+};
+
+const authFallback: AuthFallbackTelemetryEvent = {
+  type: "auth_fallback",
+  daemon_event_id: "evt-auth-fallback-001",
+  recorded_at: RECORDED_AT,
+  assistant_version: "1.2.3",
+  guard: "edge",
+  path: "/v1/conversations",
+  failure_kind: "missing_authorization",
+  count: 7,
+  window_start: RECORDED_AT - 300_000,
+  window_end: RECORDED_AT,
+};
+
+const toolExecuted: ToolExecutedTelemetryEvent = {
+  type: "tool_executed",
+  daemon_event_id: "evt-tool-executed-001",
+  recorded_at: RECORDED_AT,
+  assistant_version: "1.2.3",
+  provider: "anthropic",
+  model: "claude-fable-5",
+  inference_profile: "balanced",
+  inference_profile_source: "call_site",
+  tool_name: "bash",
+  status: "fulfilled",
+  duration_ms: 42,
+  arg_bytes: 128,
+  result_bytes: 2048,
+  conversation_id: "conv-xyz",
+};
+
+const skillLoaded: SkillLoadedTelemetryEvent = {
+  type: "skill_loaded",
+  daemon_event_id: "evt-skill-loaded-001",
+  recorded_at: RECORDED_AT,
+  assistant_version: "1.2.3",
+  provider: "anthropic",
+  model: "claude-fable-5",
+  inference_profile: "balanced",
+  inference_profile_source: "conversation",
+  skill_name: "plugin-builder",
+  skill_updated_at: "2026-06-01T00:00:00Z",
+  conversation_id: "conv-xyz",
+};
+
+const watchdog: WatchdogTelemetryEvent = {
+  type: "watchdog",
+  daemon_event_id: "evt-watchdog-001",
+  recorded_at: RECORDED_AT,
+  assistant_version: "1.2.3",
+  check_name: "event_loop_blocked",
+  value: 1200,
+  detail: { reason: "gc", blocked_ms: 1200 },
+};
+
+const configSetting: ConfigSettingTelemetryEvent = {
+  type: "config_setting",
+  daemon_event_id: "evt-config-setting-001",
+  recorded_at: RECORDED_AT,
+  assistant_version: "1.2.3",
+  config_key: "memory.enabled",
+  config_value: "true",
+};
+
+/**
+ * `onboarding_research` sample. The daemon's type is richer than the wire
+ * (structured `claims`/`suggestions`, closed `status`), so it is an `Overrides`
+ * entry; the wire schema (opaque JSON arrays, size-bounded) still accepts it.
+ */
+export const onboardingResearch: OnboardingResearchTelemetryEvent = {
+  type: "onboarding_research",
+  daemon_event_id: "evt-onboarding-research-1",
+  recorded_at: RECORDED_AT,
+  assistant_version: "1.2.3",
+  conversation_id: "conv-xyz",
+  status: "done",
+  claims: [
+    {
+      claim: "Works on developer tooling",
+      confidence: "confident",
+      sources: ["https://example.com/profile"],
+    },
+  ],
+  claim_count: 1,
+  claims_confident: 1,
+  claims_maybe: 0,
+  claims_guessing: 0,
+  suggestions: [
+    {
+      suggestion: "Set up email triage",
+      prompt: "Help me triage my inbox",
+    },
+  ],
+  suggestion_count: 1,
+  plugins: ["gmail"],
+  installed_plugins: ["gmail", "calendar"],
+};
+
+/** One daemon-typed, wire-valid sample per generated wire event type. */
+export const wireEventSamples = [
+  llmUsage,
+  turnEventSample,
+  lifecycle,
+  onboarding,
+  authFallback,
+  toolExecuted,
+  skillLoaded,
+  watchdog,
+  configSetting,
+  onboardingResearch,
+] as const;

package/src/telemetry/telemetry-wire-source.json

@@ -1,4 +1,4 @@
 {
   "sourceRepo": "vellum-ai/vellum-assistant-platform",
-  "sourceSha": "8bfa34aa5ea0a34dc9339ca0baf8bad4b124ce39"
+  "sourceSha": "701c9110672aa6448798b972c93f1b7d34f56dd6"
 }

package/src/telemetry/telemetry-wire-validation.test.ts

@@ -0,0 +1,157 @@
+/**
+ * Tests for pre-flush telemetry wire validation.
+ *
+ * Validation is observability only — it must count and warn, never mutate,
+ * filter, or block — and its warn payloads must carry the event type and
+ * issue `{ path, code }` shapes only, never field values. `daemon_event_id`
+ * is a field value too: activation-funnel ids embed the onboarding session
+ * id (traces/claims can hold PII). Issue paths are sanitized as well: dynamic
+ * record keys (e.g. `client` bag keys) are redacted to `*` before logging.
+ */
+import { beforeEach, describe, expect, test } from "bun:test";
+
+import type { Logger } from "pino";
+
+import {
+  turnEventSample,
+  wireEventSamples,
+} from "./__tests__/telemetry-event-fixtures.js";
+import { telemetryEventSchema } from "./telemetry-wire.generated.js";
+import {
+  resetUnknownTypeWarningsForTests,
+  validateWireEvents,
+} from "./telemetry-wire-validation.js";
+
+let warnCalls: unknown[][] = [];
+const stubLog = {
+  warn: (...args: unknown[]) => {
+    warnCalls.push(args);
+  },
+} as unknown as Logger;
+
+describe("validateWireEvents", () => {
+  beforeEach(() => {
+    warnCalls = [];
+    resetUnknownTypeWarningsForTests();
+  });
+
+  test("a valid event of each wire type passes with no warnings", () => {
+    // Fixture coverage: one sample per generated union discriminator (with a
+    // floor of the 9 shipped types), so a new wire type without a fixture —
+    // or a shrunken union — turns this red.
+    const discriminators = telemetryEventSchema.options.map(
+      (option) => option.shape.type.value,
+    );
+    expect(discriminators.length).toBeGreaterThanOrEqual(9);
+    expect(new Set(wireEventSamples.map((event) => event.type))).toEqual(
+      new Set(discriminators),
+    );
+
+    const result = validateWireEvents(wireEventSamples, stubLog);
+    expect(result).toEqual({
+      checked: wireEventSamples.length,
+      invalid: 0,
+      unknownTypes: [],
+    });
+    expect(warnCalls).toHaveLength(0);
+  });
+
+  test("invalid events are counted and warned with {path, code} issues and no field values", () => {
+    const sentinel = "SENTINEL_PII_VALUE_do_not_log";
+    const invalidTurn = {
+      ...turnEventSample,
+      // Nested object values violate the wire contract's client bounds; the
+      // sentinel must never appear in any warn payload.
+      client: { nested: { secret: sentinel } },
+    };
+    const invalidLifecycle = {
+      type: "lifecycle",
+      // An id embedding the onboarding session id and padded past the wire
+      // schema's 128-char daemon_event_id bound — the id value itself must
+      // never be logged, only its (redacted) issue path.
+      daemon_event_id: `activation_v1_2026_06:${sentinel}:${"x".repeat(100)}`,
+      recorded_at: 1_700_000_000_011,
+      assistant_version: sentinel,
+      // event_name is missing.
+    };
+
+    const result = validateWireEvents([invalidTurn, invalidLifecycle], stubLog);
+    expect(result).toEqual({ checked: 2, invalid: 2, unknownTypes: [] });
+    expect(warnCalls).toHaveLength(2);
+
+    for (const call of warnCalls) {
+      const bag = call[0] as {
+        eventType: string;
+        issues: Array<Record<string, unknown>>;
+      };
+      // The bag carries the event type and issues only — no daemon_event_id
+      // (it is a payload value; activation ids embed the session id).
+      expect(Object.keys(bag).sort()).toEqual(["eventType", "issues"]);
+      expect(bag.issues.length).toBeGreaterThan(0);
+      for (const issue of bag.issues) {
+        // Issues carry path + code only — never messages or field values.
+        expect(Object.keys(issue).sort()).toEqual(["code", "path"]);
+        expect(issue.path).toBeString();
+        expect(issue.code).toBeString();
+      }
+    }
+    expect(
+      (warnCalls[0][0] as { issues: Array<{ path: string }> }).issues[0].path,
+    ).toStartWith("client");
+    const lifecyclePaths = (
+      warnCalls[1][0] as { issues: Array<{ path: string }> }
+    ).issues.map((issue) => issue.path);
+    expect(lifecyclePaths).toContain("daemon_event_id");
+    expect(lifecyclePaths).toContain("event_name");
+
+    // The planted sentinel (a field value) must be absent from all log args.
+    expect(JSON.stringify(warnCalls)).not.toInclude(sentinel);
+  });
+
+  test("dynamic record keys in issue paths are redacted to *", () => {
+    const sentinel = "sentinel-user@example.com";
+    const invalidTurn = {
+      ...turnEventSample,
+      // The client bag's keys come from an uncontrolled JSON metadata column;
+      // the superRefine emits issues at ['client', <key>], so a key carrying
+      // user text must be redacted from the logged path. The nested object
+      // value makes the bag invalid.
+      client: { [sentinel]: { nested: true } },
+    };
+
+    const result = validateWireEvents([invalidTurn], stubLog);
+    expect(result).toEqual({ checked: 1, invalid: 1, unknownTypes: [] });
+    expect(warnCalls).toHaveLength(1);
+
+    const bag = warnCalls[0][0] as { issues: Array<{ path: string }> };
+    // The dynamic key component is replaced with a literal `*`; the
+    // schema-defined depth-0 component (`client`) is kept.
+    expect(bag.issues.map((issue) => issue.path)).toContain("client.*");
+    expect(JSON.stringify(warnCalls)).not.toInclude(sentinel);
+  });
+
+  test("unknown event types are reported and warned once per process per type", () => {
+    // A type the platform ingest endpoint has no serializer for: the server
+    // silently skips it. Uses a synthetic never-shipped discriminant so the
+    // test stays valid as real event types graduate into the wire contract.
+    const unknownEvent = {
+      type: "speculative_future_event",
+      daemon_event_id: "evt-unknown-1",
+      recorded_at: 1_700_000_000_020,
+      assistant_version: "1.2.3",
+    };
+    const first = validateWireEvents([unknownEvent], stubLog);
+    const second = validateWireEvents([unknownEvent], stubLog);
+
+    expect(first).toEqual({
+      checked: 0,
+      invalid: 0,
+      unknownTypes: ["speculative_future_event"],
+    });
+    expect(second.unknownTypes).toEqual(["speculative_future_event"]);
+
+    // Rate-limited: the warn fires exactly once across both calls.
+    expect(warnCalls).toHaveLength(1);
+    expect(warnCalls[0][0]).toEqual({ eventType: "speculative_future_event" });
+  });
+});