@ai-sdk/devtools 1.0.0-beta.2 → 1.0.0-beta.31

package/src/db.ts

@@ -1,8 +1,13 @@
 import path from 'node:path';
 import fs from 'node:fs';
 
-const DB_DIR = path.join(process.cwd(), '.devtools');
-const DB_PATH = path.join(DB_DIR, 'generations.json');
+const DEVTOOLS_DB_DIR = '.devtools';
+const DEVTOOLS_DB_FILE = 'generations.json';
+// Cap how many bytes we read from a remote database file. Guards against a
+// synchronous hang / OOM if the file is enormous.
+const MAX_DB_BYTES = 100 * 1024 * 1024; // 100 MB
+const DB_DIR = path.join(process.cwd(), DEVTOOLS_DB_DIR);
+const DB_PATH = path.join(DB_DIR, DEVTOOLS_DB_FILE);
 const DEVTOOLS_PORT = process.env.AI_SDK_DEVTOOLS_PORT
   ? parseInt(process.env.AI_SDK_DEVTOOLS_PORT)
   : 4983;
@@ -26,7 +31,7 @@ export const notifyServerAsync = async (
     await fetch(`http://localhost:${DEVTOOLS_PORT}/api/notify`, {
       method: 'POST',
       headers: { 'Content-Type': 'application/json' },
-      body: JSON.stringify({ event, timestamp: Date.now() }),
+      body: JSON.stringify({ event, timestamp: Date.now(), dbPath: DB_PATH }),
     });
   } catch {
     // Ignore errors - server might not be running
@@ -36,6 +41,9 @@ export const notifyServerAsync = async (
 export interface Run {
   id: string;
   started_at: string;
+  parent_run_id: string | null;
+  parent_step_id: string | null;
+  function_id: string | null;
 }
 
 export interface Step {
@@ -137,15 +145,69 @@ const saveDb = (db: Database): void => {
   writeDb(db);
 };
 
+const normalizeDevtoolsDbPath = (dbPath: string): string | undefined => {
+  const resolvedPath = path.resolve(dbPath);
+  const dbDir = path.dirname(resolvedPath);
+
+  if (
+    path.basename(resolvedPath) !== DEVTOOLS_DB_FILE ||
+    path.basename(dbDir) !== DEVTOOLS_DB_DIR
+  ) {
+    return undefined;
+  }
+
+  return resolvedPath;
+};
+
+export const validateRemoteDbPath = (dbPath: unknown): string | undefined => {
+  if (typeof dbPath !== 'string') {
+    return undefined;
+  }
+
+  const normalizedPath = normalizeDevtoolsDbPath(dbPath);
+  if (!normalizedPath) {
+    return undefined;
+  }
+
+  try {
+    const stats = fs.statSync(normalizedPath);
+    if (!stats.isFile() || stats.size > MAX_DB_BYTES) {
+      return undefined;
+    }
+
+    const realPath = fs.realpathSync(normalizedPath);
+    return normalizeDevtoolsDbPath(realPath);
+  } catch {
+    return undefined;
+  }
+};
+
 /**
  * Reload the database from disk.
- * Used by the viewer server to pick up changes made by the middleware.
+ * Used by the viewer server to pick up changes made by the middleware/integration.
+ * When a valid remote dbPath is provided (from a notify POST), reads from that
+ * path instead of the local CWD-based path, so the viewer works regardless of
+ * where it was started.
  */
-export const reloadDb = async (): Promise<void> => {
+export const reloadDb = async (remoteDbPath?: string): Promise<void> => {
+  const validatedRemoteDbPath = validateRemoteDbPath(remoteDbPath);
+  if (validatedRemoteDbPath) {
+    try {
+      const content = fs.readFileSync(validatedRemoteDbPath, 'utf-8');
+      dbCache = JSON.parse(content);
+      return;
+    } catch {
+      // Fall through to default
+    }
+  }
   dbCache = readDb();
 };
 
-export const createRun = async (id: string): Promise<Run> => {
+export const createRun = async (
+  id: string,
+  parent?: { runId: string; stepId: string },
+  functionId?: string,
+): Promise<Run> => {
   const db = getDb();
   const started_at = new Date().toISOString();
 
@@ -155,7 +217,13 @@ export const createRun = async (id: string): Promise<Run> => {
     return existing;
   }
 
-  const run: Run = { id, started_at };
+  const run: Run = {
+    id,
+    started_at,
+    parent_run_id: parent?.runId ?? null,
+    parent_step_id: parent?.stepId ?? null,
+    function_id: functionId ?? null,
+  };
   db.runs.push(run);
   saveDb(db);
   notifyServer('run');

package/src/index.ts

@@ -1 +1,2 @@
 export { devToolsMiddleware } from './middleware.js';
+export { DevToolsTelemetry } from './integration.js';

package/src/integration.ts

@@ -0,0 +1,447 @@
+import type {
+  GenerateTextStartEvent,
+  GenerateTextStepStartEvent,
+  GenerateTextStepEndEvent,
+  GenerateObjectStartEvent,
+  GenerateObjectStepStartEvent,
+  GenerateObjectStepEndEvent,
+  Telemetry,
+  ToolSet,
+} from 'ai';
+import {
+  createRun,
+  createStep,
+  updateStepResult,
+  notifyServerAsync,
+} from './db.js';
+
+type OperationType = 'generate' | 'stream';
+
+interface StepState {
+  stepId: string;
+  startTime: number;
+}
+
+interface CallState {
+  runId: string;
+  operationType: OperationType;
+  functionId: string | undefined;
+  settings: Record<string, unknown>;
+  stepStates: Map<number, StepState>;
+}
+
+const activeSteps = new Map<string, StepState>();
+
+let signalHandlersRegistered = false;
+const registerSignalHandlers = () => {
+  if (signalHandlersRegistered) return;
+  signalHandlersRegistered = true;
+
+  const cleanup = async () => {
+    if (activeSteps.size === 0) return;
+
+    const promises = Array.from(activeSteps.entries()).map(
+      async ([stepId, data]) => {
+        const durationMs = Date.now() - data.startTime;
+        await updateStepResult(stepId, {
+          duration_ms: durationMs,
+          output: null,
+          usage: null,
+          error: 'Request aborted',
+          raw_request: null,
+          raw_response: null,
+          raw_chunks: null,
+        });
+      },
+    );
+    await Promise.all(promises);
+    await notifyServerAsync('step-update');
+  };
+
+  process.on('SIGINT', () => {
+    cleanup().then(() => process.exit(130));
+  });
+
+  process.on('SIGTERM', () => {
+    cleanup().then(() => process.exit(143));
+  });
+};
+
+const generateRunId = (): string => {
+  const now = new Date();
+  const timestamp = now
+    .toISOString()
+    .replace(/[-:T.Z]/g, '')
+    .slice(0, 17);
+  const uniqueId = crypto.randomUUID().slice(0, 8);
+  return `${timestamp}-${uniqueId}`;
+};
+
+function getOperationType(operationId: string): OperationType {
+  if (operationId === 'ai.streamText' || operationId === 'ai.streamObject') {
+    return 'stream';
+  }
+  return 'generate';
+}
+
+/**
+ * Creates a devtools telemetry integration that logs all AI SDK operations
+ * to the devtools viewer.
+ *
+ * Usage:
+ * ```ts
+ * import { registerTelemetry } from 'ai';
+ * import { DevToolsTelemetry } from '@ai-sdk/devtools';
+ *
+ * registerTelemetry(DevToolsTelemetry());
+ * ```
+ *
+ * Telemetry is enabled by default — no need to set `telemetry`
+ * unless you want to configure `functionId`, `recordInputs`, or `recordOutputs`.
+ */
+export function DevToolsTelemetry(): Telemetry {
+  if (process.env.NODE_ENV === 'production') {
+    throw new Error(
+      '@ai-sdk/devtools should not be used in production. ' +
+        'Remove DevToolsTelemetry from your telemetry configuration for production builds.',
+    );
+  }
+
+  registerSignalHandlers();
+
+  const callStates = new Map<string, CallState>();
+
+  // When executeTool runs a tool's execute function, any nested generateText/
+  // streamText call inside that tool gets its own callId. We track the nesting
+  // so that the inner call's run can be linked to the parent.
+  //
+  // We use a per-toolCallId Map instead of a single variable so that parallel
+  // tool calls
+  const toolContextMap = new Map<
+    string,
+    { parentCallId: string; parentToolCallId: string }
+  >();
+  let currentToolCallId: string | null = null;
+
+  function resolveParentInfo(): { runId: string; stepId: string } | undefined {
+    if (!currentToolCallId) return undefined;
+
+    const toolCallContext = toolContextMap.get(currentToolCallId);
+    if (!toolCallContext) return undefined;
+
+    const parentState = callStates.get(toolCallContext.parentCallId);
+    if (!parentState) return undefined;
+
+    // Find the step that is currently executing the tool call.
+    // This is the most recent (highest step number) step in the parent call.
+    let latestStepId: string | undefined;
+    let latestStepNumber = -1;
+    for (const [stepNumber, stepState] of parentState.stepStates) {
+      if (stepNumber > latestStepNumber) {
+        latestStepNumber = stepNumber;
+        latestStepId = stepState.stepId;
+      }
+    }
+
+    if (!latestStepId) return undefined;
+
+    return { runId: parentState.runId, stepId: latestStepId };
+  }
+
+  function getOrCreateCallState(
+    callId: string,
+    operationId: string,
+    event: {
+      functionId?: string | undefined;
+      maxOutputTokens?: number | undefined;
+      temperature?: number | undefined;
+      topP?: number | undefined;
+      topK?: number | undefined;
+      presencePenalty?: number | undefined;
+      frequencyPenalty?: number | undefined;
+      seed?: number | undefined;
+    },
+  ): CallState {
+    let state = callStates.get(callId);
+    if (state) return state;
+
+    state = {
+      runId: generateRunId(),
+      operationType: getOperationType(operationId),
+      functionId: event.functionId,
+      settings: {
+        maxOutputTokens: event.maxOutputTokens,
+        temperature: event.temperature,
+        topP: event.topP,
+        topK: event.topK,
+        presencePenalty: event.presencePenalty,
+        frequencyPenalty: event.frequencyPenalty,
+        seed: event.seed,
+      },
+      stepStates: new Map(),
+    };
+    callStates.set(callId, state);
+    return state;
+  }
+
+  const integration: Telemetry = {
+    onStart: async event => {
+      const operationId = (event as { operationId: string }).operationId;
+
+      if (
+        operationId === 'ai.embed' ||
+        operationId === 'ai.embedMany' ||
+        operationId === 'ai.rerank'
+      ) {
+        return;
+      }
+
+      const startEvent = event as (
+        | GenerateTextStartEvent<ToolSet>
+        | GenerateObjectStartEvent
+      ) & {
+        functionId?: string | undefined;
+      };
+
+      const parentInfo = resolveParentInfo();
+
+      const state = getOrCreateCallState(
+        startEvent.callId,
+        operationId,
+        startEvent,
+      );
+
+      await createRun(state.runId, parentInfo, state.functionId);
+    },
+
+    onStepStart: async event => {
+      const stepStartEvent = event as GenerateTextStepStartEvent<ToolSet> & {
+        promptMessages?: unknown[];
+      };
+
+      const state = callStates.get(stepStartEvent.callId);
+      if (!state) return;
+
+      const stepId = crypto.randomUUID();
+      const startTime = Date.now();
+
+      const stepState: StepState = {
+        stepId,
+        startTime,
+      };
+      const stepNumber = stepStartEvent.steps.length;
+      state.stepStates.set(stepNumber, stepState);
+      activeSteps.set(stepId, stepState);
+
+      const prompt = stepStartEvent.promptMessages ?? stepStartEvent.messages;
+
+      await createStep({
+        id: stepId,
+        run_id: state.runId,
+        step_number: stepNumber + 1,
+        type: state.operationType,
+        model_id: stepStartEvent.modelId,
+        provider: stepStartEvent.provider ?? null,
+        started_at: new Date().toISOString(),
+        input: JSON.stringify({
+          prompt,
+          tools: stepStartEvent.tools
+            ? Object.entries(stepStartEvent.tools).map(([name, tool]) => ({
+                name,
+                description: (tool as { description?: string }).description,
+                parameters: (tool as { parameters?: unknown }).parameters,
+              }))
+            : undefined,
+          toolChoice: stepStartEvent.toolChoice,
+          maxOutputTokens: state.settings.maxOutputTokens,
+          temperature: state.settings.temperature,
+          topP: state.settings.topP,
+          topK: state.settings.topK,
+          presencePenalty: state.settings.presencePenalty,
+          frequencyPenalty: state.settings.frequencyPenalty,
+          seed: state.settings.seed,
+        }),
+        provider_options: stepStartEvent.providerOptions
+          ? JSON.stringify(stepStartEvent.providerOptions)
+          : null,
+      });
+    },
+
+    onObjectStepStart: async event => {
+      const stepStartEvent = event as GenerateObjectStepStartEvent & {
+        promptMessages?: unknown[];
+      };
+
+      const state = callStates.get(stepStartEvent.callId);
+      if (!state) return;
+
+      const stepId = crypto.randomUUID();
+      const startTime = Date.now();
+
+      const stepState: StepState = {
+        stepId,
+        startTime,
+      };
+      state.stepStates.set(stepStartEvent.stepNumber, stepState);
+      activeSteps.set(stepId, stepState);
+
+      await createStep({
+        id: stepId,
+        run_id: state.runId,
+        step_number: stepStartEvent.stepNumber + 1,
+        type: state.operationType,
+        model_id: stepStartEvent.modelId,
+        provider: stepStartEvent.provider ?? null,
+        started_at: new Date().toISOString(),
+        input: JSON.stringify({
+          prompt: stepStartEvent.promptMessages,
+          maxOutputTokens: state.settings.maxOutputTokens,
+          temperature: state.settings.temperature,
+          topP: state.settings.topP,
+          topK: state.settings.topK,
+          presencePenalty: state.settings.presencePenalty,
+          frequencyPenalty: state.settings.frequencyPenalty,
+          seed: state.settings.seed,
+        }),
+        provider_options: stepStartEvent.providerOptions
+          ? JSON.stringify(stepStartEvent.providerOptions)
+          : null,
+      });
+    },
+
+    onStepEnd: async event => {
+      const stepResult = event as GenerateTextStepEndEvent<ToolSet>;
+
+      const state = callStates.get(stepResult.callId);
+      if (!state) return;
+
+      const stepState = state.stepStates.get(stepResult.stepNumber);
+      if (!stepState) return;
+
+      activeSteps.delete(stepState.stepId);
+
+      const durationMs = Date.now() - stepState.startTime;
+
+      const output = {
+        content: stepResult.content,
+        finishReason: stepResult.finishReason,
+        response: {
+          id: stepResult.response.id,
+          modelId: stepResult.response.modelId,
+          timestamp: stepResult.response.timestamp,
+          messages: stepResult.response.messages,
+        },
+      };
+
+      await updateStepResult(stepState.stepId, {
+        duration_ms: durationMs,
+        output: JSON.stringify(output),
+        usage: stepResult.usage ? JSON.stringify(stepResult.usage) : null,
+        error: null,
+        raw_request: stepResult.request?.body
+          ? JSON.stringify(stepResult.request.body)
+          : null,
+        raw_response: stepResult.response?.body
+          ? JSON.stringify(stepResult.response.body)
+          : null,
+        raw_chunks: null,
+      });
+
+      state.stepStates.delete(stepResult.stepNumber);
+    },
+
+    onObjectStepEnd: async event => {
+      const stepResult = event as GenerateObjectStepEndEvent;
+
+      const state = callStates.get(stepResult.callId);
+      if (!state) return;
+
+      const stepState = state.stepStates.get(stepResult.stepNumber);
+      if (!stepState) return;
+
+      activeSteps.delete(stepState.stepId);
+
+      const durationMs = Date.now() - stepState.startTime;
+
+      const output = {
+        finishReason: stepResult.finishReason,
+        objectText: stepResult.objectText,
+        response: {
+          id: stepResult.response.id,
+          modelId: stepResult.response.modelId,
+          timestamp: stepResult.response.timestamp,
+        },
+      };
+
+      await updateStepResult(stepState.stepId, {
+        duration_ms: durationMs,
+        output: JSON.stringify(output),
+        usage: stepResult.usage ? JSON.stringify(stepResult.usage) : null,
+        error: null,
+        raw_request: stepResult.request?.body
+          ? JSON.stringify(stepResult.request.body)
+          : null,
+        raw_response: stepResult.response?.body
+          ? JSON.stringify(stepResult.response.body)
+          : null,
+      });
+
+      state.stepStates.delete(stepResult.stepNumber);
+    },
+
+    onEnd: async event => {
+      const endEvent = event as { callId: string };
+      callStates.delete(endEvent.callId);
+    },
+
+    onError: async error => {
+      const errorObj = error as
+        | { callId?: string; error?: unknown }
+        | undefined;
+      const callId = errorObj?.callId;
+      if (!callId) return;
+
+      const state = callStates.get(callId);
+      if (!state) return;
+
+      const cause = errorObj?.error ?? error;
+      const errorMessage =
+        cause instanceof Error ? cause.message : String(cause);
+
+      for (const [, stepState] of state.stepStates) {
+        activeSteps.delete(stepState.stepId);
+        const durationMs = Date.now() - stepState.startTime;
+        await updateStepResult(stepState.stepId, {
+          duration_ms: durationMs,
+          output: null,
+          usage: null,
+          error: errorMessage,
+          raw_request: null,
+          raw_response: null,
+          raw_chunks: null,
+        });
+      }
+
+      callStates.delete(callId);
+    },
+
+    executeTool: async ({ callId, toolCallId, execute }) => {
+      toolContextMap.set(toolCallId, {
+        parentCallId: callId,
+        parentToolCallId: toolCallId,
+      });
+
+      const previousToolCallId = currentToolCallId;
+      currentToolCallId = toolCallId;
+
+      try {
+        return await execute();
+      } finally {
+        currentToolCallId = previousToolCallId;
+        toolContextMap.delete(toolCallId);
+      }
+    },
+  };
+
+  return integration;
+}

package/src/middleware.ts

@@ -1,8 +1,8 @@
-import {
-  type LanguageModelV3FinishReason,
-  type LanguageModelV3Usage,
-  type LanguageModelV3Middleware,
-  type LanguageModelV3StreamPart,
+import type {
+  LanguageModelV4FinishReason,
+  LanguageModelV4Usage,
+  LanguageModelV4Middleware,
+  LanguageModelV4StreamPart,
 } from '@ai-sdk/provider';
 import {
   createRun,
@@ -97,7 +97,7 @@ const generateRunId = (): string => {
  * });
  * ```
  */
-export const devToolsMiddleware = (): LanguageModelV3Middleware => {
+export const devToolsMiddleware = (): LanguageModelV4Middleware => {
   if (process.env.NODE_ENV === 'production') {
     throw new Error(
       '@ai-sdk/devtools should not be used in production. ' +
@@ -125,7 +125,7 @@ export const devToolsMiddleware = (): LanguageModelV3Middleware => {
   };
 
   return {
-    specificationVersion: 'v3',
+    specificationVersion: 'v4',
 
     wrapGenerate: async ({ doGenerate, params, model }) => {
       const startTime = Date.now();
@@ -242,9 +242,9 @@ export const devToolsMiddleware = (): LanguageModelV3Middleware => {
         const collectedOutput: {
           textParts: Array<{ id: string; text: string }>;
           reasoningParts: Array<{ id: string; text: string }>;
-          toolCalls: LanguageModelV3StreamPart[];
-          finishReason?: LanguageModelV3FinishReason;
-          usage?: LanguageModelV3Usage;
+          toolCalls: LanguageModelV4StreamPart[];
+          finishReason?: LanguageModelV4FinishReason;
+          usage?: LanguageModelV4Usage;
         } = {
           textParts: [],
           reasoningParts: [],
@@ -253,7 +253,7 @@ export const devToolsMiddleware = (): LanguageModelV3Middleware => {
 
         const currentText: Map<string, string> = new Map();
         const currentReasoning: Map<string, string> = new Map();
-        const fullStreamChunks: LanguageModelV3StreamPart[] = [];
+        const fullStreamChunks: LanguageModelV4StreamPart[] = [];
         const rawChunks: unknown[] = [];
 
         // Track this step for cleanup on process exit
@@ -266,8 +266,8 @@ export const devToolsMiddleware = (): LanguageModelV3Middleware => {
         });
 
         const transformStream = new TransformStream<
-          LanguageModelV3StreamPart,
-          LanguageModelV3StreamPart
+          LanguageModelV4StreamPart,
+          LanguageModelV4StreamPart
         >({
           transform(chunk, controller) {
             // Separate raw provider chunks from other stream chunks