@ai-sdk/devtools 0.0.6 → 0.0.8

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ai-sdk/devtools",
-  "version": "0.0.6",
+  "version": "0.0.8",
   "type": "module",
   "main": "./dist/index.js",
   "module": "./dist/index.js",
@@ -17,12 +17,17 @@
   },
   "files": [
     "dist",
+    "src",
+    "!src/**/*.test.ts",
+    "!src/**/*.test-d.ts",
+    "!src/**/__snapshots__",
+    "!src/**/__fixtures__",
     "bin"
   ],
   "dependencies": {
     "@hono/node-server": "^1.13.7",
     "hono": "^4.6.14",
-    "@ai-sdk/provider": "3.0.4"
+    "@ai-sdk/provider": "3.0.5"
   },
   "devDependencies": {
     "@radix-ui/react-collapsible": "^1.1.12",
@@ -50,7 +55,7 @@
     "vaul": "^1.1.2",
     "vite": "^6.0.3",
     "zod": "3.25.76",
-    "ai": "6.0.38"
+    "ai": "6.0.48"
   },
   "scripts": {
     "dev": "concurrently -k \"pnpm dev:api\" \"pnpm dev:client\"",

package/src/db.ts

@@ -0,0 +1,242 @@
+import path from 'node:path';
+import fs from 'node:fs';
+
+const DB_DIR = path.join(process.cwd(), '.devtools');
+const DB_PATH = path.join(DB_DIR, 'generations.json');
+const DEVTOOLS_PORT = process.env.AI_SDK_DEVTOOLS_PORT
+  ? parseInt(process.env.AI_SDK_DEVTOOLS_PORT)
+  : 4983;
+
+/**
+ * Notify the devtools server that data has changed.
+ * Fire-and-forget: doesn't block, ignores errors if server isn't running.
+ */
+const notifyServer = (event: 'run' | 'step' | 'step-update' | 'clear') => {
+  notifyServerAsync(event);
+};
+
+/**
+ * Notify the devtools server and wait for the request to complete.
+ * Used during process cleanup to ensure notifications are sent before exit.
+ */
+export const notifyServerAsync = async (
+  event: 'run' | 'step' | 'step-update' | 'clear',
+): Promise<void> => {
+  try {
+    await fetch(`http://localhost:${DEVTOOLS_PORT}/api/notify`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ event, timestamp: Date.now() }),
+    });
+  } catch {
+    // Ignore errors - server might not be running
+  }
+};
+
+export interface Run {
+  id: string;
+  started_at: string;
+}
+
+export interface Step {
+  id: string;
+  run_id: string;
+  step_number: number;
+  type: 'generate' | 'stream';
+  model_id: string;
+  provider: string | null;
+  started_at: string;
+  duration_ms: number | null;
+  input: string;
+  output: string | null;
+  usage: string | null;
+  error: string | null;
+  raw_request: string | null;
+  raw_response: string | null;
+  raw_chunks: string | null;
+  provider_options: string | null;
+}
+
+export interface StepResult {
+  duration_ms: number;
+  output: string | null;
+  usage: string | null;
+  error: string | null;
+  raw_request?: string | null;
+  raw_response?: string | null;
+  raw_chunks?: string | null;
+}
+
+interface Database {
+  runs: Run[];
+  steps: Step[];
+}
+
+/**
+ * Ensure .devtools is in .gitignore.
+ * Only writes if .gitignore exists and doesn't already contain .devtools.
+ */
+const ensureGitignore = (): void => {
+  const gitignorePath = path.join(process.cwd(), '.gitignore');
+
+  if (!fs.existsSync(gitignorePath)) {
+    return;
+  }
+
+  const content = fs.readFileSync(gitignorePath, 'utf-8');
+  const lines = content.split('\n');
+
+  // Check if .devtools is already ignored (exact match or with trailing slash)
+  const alreadyIgnored = lines.some(
+    line => line.trim() === '.devtools' || line.trim() === '.devtools/',
+  );
+
+  if (!alreadyIgnored) {
+    const newContent = content.endsWith('\n')
+      ? `${content}.devtools\n`
+      : `${content}\n.devtools\n`;
+    fs.writeFileSync(gitignorePath, newContent);
+  }
+};
+
+const readDb = (): Database => {
+  try {
+    if (fs.existsSync(DB_PATH)) {
+      const content = fs.readFileSync(DB_PATH, 'utf-8');
+      return JSON.parse(content);
+    }
+  } catch {
+    // If file is corrupted, start fresh
+  }
+  return { runs: [], steps: [] };
+};
+
+const writeDb = (db: Database): void => {
+  const isFirstRun = !fs.existsSync(DB_DIR);
+
+  if (isFirstRun) {
+    fs.mkdirSync(DB_DIR, { recursive: true });
+    ensureGitignore();
+  }
+
+  fs.writeFileSync(DB_PATH, JSON.stringify(db, null, 2));
+};
+
+// In-memory cache for performance
+let dbCache: Database | null = null;
+
+const getDb = (): Database => {
+  if (!dbCache) {
+    dbCache = readDb();
+  }
+  return dbCache;
+};
+
+const saveDb = (db: Database): void => {
+  dbCache = db;
+  writeDb(db);
+};
+
+/**
+ * Reload the database from disk.
+ * Used by the viewer server to pick up changes made by the middleware.
+ */
+export const reloadDb = async (): Promise<void> => {
+  dbCache = readDb();
+};
+
+export const createRun = async (id: string): Promise<Run> => {
+  const db = getDb();
+  const started_at = new Date().toISOString();
+
+  // Check if run already exists
+  const existing = db.runs.find(r => r.id === id);
+  if (existing) {
+    return existing;
+  }
+
+  const run: Run = { id, started_at };
+  db.runs.push(run);
+  saveDb(db);
+  notifyServer('run');
+  return run;
+};
+
+export const createStep = async (
+  step: Omit<
+    Step,
+    | 'duration_ms'
+    | 'output'
+    | 'usage'
+    | 'error'
+    | 'raw_request'
+    | 'raw_response'
+    | 'raw_chunks'
+  >,
+): Promise<void> => {
+  const db = getDb();
+  const newStep: Step = {
+    ...step,
+    duration_ms: null,
+    output: null,
+    usage: null,
+    error: null,
+    raw_request: null,
+    raw_response: null,
+    raw_chunks: null,
+  };
+  db.steps.push(newStep);
+  saveDb(db);
+  notifyServer('step');
+};
+
+export const updateStepResult = async (
+  stepId: string,
+  result: StepResult,
+): Promise<void> => {
+  const db = getDb();
+  const step = db.steps.find(s => s.id === stepId);
+  if (step) {
+    step.duration_ms = result.duration_ms;
+    step.output = result.output;
+    step.usage = result.usage;
+    step.error = result.error;
+    step.raw_request = result.raw_request ?? null;
+    step.raw_response = result.raw_response ?? null;
+    step.raw_chunks = result.raw_chunks ?? null;
+    saveDb(db);
+    notifyServer('step-update');
+  }
+};
+
+export const getRuns = async (): Promise<Run[]> => {
+  const db = getDb();
+  // Return runs sorted by started_at DESC
+  return [...db.runs].sort(
+    (a, b) =>
+      new Date(b.started_at).getTime() - new Date(a.started_at).getTime(),
+  );
+};
+
+export const getStepsForRun = async (runId: string): Promise<Step[]> => {
+  const db = getDb();
+  return db.steps
+    .filter(s => s.run_id === runId)
+    .sort((a, b) => a.step_number - b.step_number);
+};
+
+export const getRunWithSteps = async (
+  runId: string,
+): Promise<{ run: Run; steps: Step[] } | null> => {
+  const db = getDb();
+  const run = db.runs.find(r => r.id === runId);
+  if (!run) return null;
+  const steps = await getStepsForRun(runId);
+  return { run, steps };
+};
+
+export const clearDatabase = async (): Promise<void> => {
+  const db: Database = { runs: [], steps: [] };
+  saveDb(db);
+  notifyServer('clear');
+};

package/src/index.ts

@@ -0,0 +1 @@
+export { devToolsMiddleware } from './middleware.js';

package/src/middleware.ts

@@ -0,0 +1,392 @@
+import {
+  type LanguageModelV3FinishReason,
+  type LanguageModelV3Usage,
+  type LanguageModelV3Middleware,
+  type LanguageModelV3StreamPart,
+} from '@ai-sdk/provider';
+import {
+  createRun,
+  createStep,
+  updateStepResult,
+  notifyServerAsync,
+} from './db.js';
+
+const generateId = () => crypto.randomUUID();
+
+// Track active streaming steps for cleanup on process exit
+const activeSteps = new Map<
+  string,
+  {
+    startTime: number;
+    collectedOutput: unknown;
+    request: unknown;
+    fullStreamChunks: unknown[];
+    rawChunks: unknown[];
+  }
+>();
+
+// Handle process termination signals
+let signalHandlersRegistered = false;
+const registerSignalHandlers = () => {
+  if (signalHandlersRegistered) return;
+  signalHandlersRegistered = true;
+
+  const cleanup = async () => {
+    if (activeSteps.size === 0) return;
+
+    const promises = Array.from(activeSteps.entries()).map(
+      async ([stepId, data]) => {
+        const durationMs = Date.now() - data.startTime;
+        await updateStepResult(stepId, {
+          duration_ms: durationMs,
+          output: JSON.stringify(data.collectedOutput),
+          usage: null,
+          error: 'Request aborted',
+          raw_request:
+            data.request &&
+            typeof data.request === 'object' &&
+            'body' in data.request
+              ? JSON.stringify((data.request as { body: unknown }).body)
+              : null,
+          raw_response: JSON.stringify(data.fullStreamChunks),
+          raw_chunks: JSON.stringify(data.rawChunks),
+        });
+      },
+    );
+    await Promise.all(promises);
+
+    // Wait for the server notification to complete before process exits
+    await notifyServerAsync('step-update');
+  };
+
+  process.on('SIGINT', () => {
+    cleanup().then(() => process.exit(130));
+  });
+
+  process.on('SIGTERM', () => {
+    cleanup().then(() => process.exit(143));
+  });
+};
+
+/**
+ * Generate a unique run ID with timestamp prefix for sorting.
+ */
+const generateRunId = (): string => {
+  const now = new Date();
+  const timestamp = now
+    .toISOString()
+    .replace(/[-:T.Z]/g, '')
+    .slice(0, 17);
+  const uniqueId = crypto.randomUUID().slice(0, 8);
+  return `${timestamp}-${uniqueId}`;
+};
+
+/**
+ * Factory function that creates a devtools middleware instance.
+ * Each call generates a unique run ID, so all steps within a single
+ * streamText/generateText call share the same run.
+ *
+ * Usage:
+ * ```ts
+ * const result = streamText({
+ *   model: wrapLanguageModel({
+ *     middleware: devToolsMiddleware(),
+ *     model: yourModel,
+ *   }),
+ *   prompt: "...",
+ * });
+ * ```
+ */
+export const devToolsMiddleware = (): LanguageModelV3Middleware => {
+  if (process.env.NODE_ENV === 'production') {
+    throw new Error(
+      '@ai-sdk/devtools should not be used in production. ' +
+        'Remove devToolsMiddleware from your model configuration for production builds.',
+    );
+  }
+
+  // Register signal handlers once for cleanup on process exit
+  registerSignalHandlers();
+
+  const runId = generateRunId();
+  let runCreated = false;
+  let stepCounter = 0;
+
+  const ensureRunCreated = async () => {
+    if (!runCreated) {
+      await createRun(runId);
+      runCreated = true;
+    }
+  };
+
+  const getNextStepNumber = () => {
+    stepCounter++;
+    return stepCounter;
+  };
+
+  return {
+    specificationVersion: 'v3',
+
+    wrapGenerate: async ({ doGenerate, params, model }) => {
+      const startTime = Date.now();
+      const stepId = generateId();
+      const stepNumber = getNextStepNumber();
+      await ensureRunCreated();
+
+      // Log step start
+      await createStep({
+        id: stepId,
+        run_id: runId,
+        step_number: stepNumber,
+        type: 'generate',
+        model_id: model.modelId,
+        // @ts-expect-error broken type
+        provider: model.config?.provider,
+        started_at: new Date().toISOString(),
+        input: JSON.stringify({
+          prompt: params.prompt,
+          tools: params.tools,
+          toolChoice: params.toolChoice,
+          maxOutputTokens: params.maxOutputTokens,
+          temperature: params.temperature,
+          topP: params.topP,
+          topK: params.topK,
+          presencePenalty: params.presencePenalty,
+          frequencyPenalty: params.frequencyPenalty,
+          seed: params.seed,
+          responseFormat: params.responseFormat,
+        }),
+        provider_options: params.providerOptions
+          ? JSON.stringify(params.providerOptions)
+          : null,
+      });
+
+      try {
+        const result = await doGenerate();
+        const durationMs = Date.now() - startTime;
+
+        await updateStepResult(stepId, {
+          duration_ms: durationMs,
+          output: JSON.stringify({
+            content: result.content,
+            finishReason: result.finishReason,
+            response: result.response,
+          }),
+          usage: result.usage ? JSON.stringify(result.usage) : null,
+          error: null,
+          raw_request: result.request?.body
+            ? JSON.stringify(result.request.body)
+            : null,
+          raw_response: result.response?.body
+            ? JSON.stringify(result.response.body)
+            : null,
+        });
+
+        return result;
+      } catch (error) {
+        const durationMs = Date.now() - startTime;
+        await updateStepResult(stepId, {
+          duration_ms: durationMs,
+          output: null,
+          usage: null,
+          error: error instanceof Error ? error.message : String(error),
+          raw_request: null,
+          raw_response: null,
+        });
+        throw error;
+      }
+    },
+
+    wrapStream: async ({ doStream, params, model }) => {
+      const startTime = Date.now();
+      const stepId = generateId();
+      const stepNumber = getNextStepNumber();
+      await ensureRunCreated();
+
+      // Store original setting before overriding
+      const userRequestedRawChunks = params.includeRawChunks === true;
+      params.includeRawChunks = true;
+
+      // Log step start
+      await createStep({
+        id: stepId,
+        run_id: runId,
+        step_number: stepNumber,
+        type: 'stream',
+        model_id: model.modelId,
+        // @ts-expect-error broken type
+        provider: model.config?.provider,
+        started_at: new Date().toISOString(),
+        input: JSON.stringify({
+          prompt: params.prompt,
+          tools: params.tools,
+          toolChoice: params.toolChoice,
+          maxOutputTokens: params.maxOutputTokens,
+          temperature: params.temperature,
+          topP: params.topP,
+          topK: params.topK,
+          presencePenalty: params.presencePenalty,
+          frequencyPenalty: params.frequencyPenalty,
+          seed: params.seed,
+          responseFormat: params.responseFormat,
+        }),
+        provider_options: params.providerOptions
+          ? JSON.stringify(params.providerOptions)
+          : null,
+      });
+
+      try {
+        const { stream, request, response, ...rest } = await doStream();
+
+        // Collect stream output for logging
+        const collectedOutput: {
+          textParts: Array<{ id: string; text: string }>;
+          reasoningParts: Array<{ id: string; text: string }>;
+          toolCalls: LanguageModelV3StreamPart[];
+          finishReason?: LanguageModelV3FinishReason;
+          usage?: LanguageModelV3Usage;
+        } = {
+          textParts: [],
+          reasoningParts: [],
+          toolCalls: [],
+        };
+
+        const currentText: Map<string, string> = new Map();
+        const currentReasoning: Map<string, string> = new Map();
+        const fullStreamChunks: LanguageModelV3StreamPart[] = [];
+        const rawChunks: unknown[] = [];
+
+        // Track this step for cleanup on process exit
+        activeSteps.set(stepId, {
+          startTime,
+          collectedOutput,
+          request,
+          fullStreamChunks,
+          rawChunks,
+        });
+
+        const transformStream = new TransformStream<
+          LanguageModelV3StreamPart,
+          LanguageModelV3StreamPart
+        >({
+          transform(chunk, controller) {
+            // Separate raw provider chunks from other stream chunks
+            if (chunk.type === 'raw') {
+              // Store just the unwrapped rawValue for cleaner data
+              rawChunks.push(chunk.rawValue);
+              // Only pass raw chunks through if user originally requested them
+              if (userRequestedRawChunks) {
+                controller.enqueue(chunk);
+              }
+              return;
+            }
+
+            // Collect all non-raw chunks for full stream logging
+            fullStreamChunks.push(chunk);
+
+            // Collect relevant data from stream
+            switch (chunk.type) {
+              case 'text-start':
+                currentText.set(chunk.id, '');
+                break;
+              case 'text-delta':
+                currentText.set(
+                  chunk.id,
+                  (currentText.get(chunk.id) ?? '') + chunk.delta,
+                );
+                break;
+              case 'text-end':
+                collectedOutput.textParts.push({
+                  id: chunk.id,
+                  text: currentText.get(chunk.id) ?? '',
+                });
+                break;
+              case 'reasoning-start':
+                currentReasoning.set(chunk.id, '');
+                break;
+              case 'reasoning-delta':
+                currentReasoning.set(
+                  chunk.id,
+                  (currentReasoning.get(chunk.id) ?? '') + chunk.delta,
+                );
+                break;
+              case 'reasoning-end':
+                collectedOutput.reasoningParts.push({
+                  id: chunk.id,
+                  text: currentReasoning.get(chunk.id) ?? '',
+                });
+                break;
+              case 'tool-call':
+                collectedOutput.toolCalls.push(chunk);
+                break;
+              case 'finish':
+                collectedOutput.finishReason = chunk.finishReason;
+                collectedOutput.usage = chunk.usage;
+                break;
+            }
+
+            controller.enqueue(chunk);
+          },
+
+          async flush() {
+            // Remove from active tracking - stream completed normally
+            activeSteps.delete(stepId);
+
+            const durationMs = Date.now() - startTime;
+            await updateStepResult(stepId, {
+              duration_ms: durationMs,
+              output: JSON.stringify(collectedOutput),
+              usage: collectedOutput.usage
+                ? JSON.stringify(collectedOutput.usage)
+                : null,
+              error: null,
+              raw_request: request?.body ? JSON.stringify(request.body) : null,
+              raw_response: JSON.stringify(fullStreamChunks),
+              raw_chunks: JSON.stringify(rawChunks),
+            });
+          },
+
+          // @ts-expect-error - cancel is valid per WHATWG Streams spec but missing from TS types
+          async cancel() {
+            // Remove from active tracking - stream was cancelled
+            activeSteps.delete(stepId);
+
+            const durationMs = Date.now() - startTime;
+            await updateStepResult(stepId, {
+              duration_ms: durationMs,
+              output: JSON.stringify(collectedOutput),
+              usage: collectedOutput.usage
+                ? JSON.stringify(collectedOutput.usage)
+                : null,
+              error: 'Request aborted',
+              raw_request: request?.body ? JSON.stringify(request.body) : null,
+              raw_response: JSON.stringify(fullStreamChunks),
+              raw_chunks: JSON.stringify(rawChunks),
+            });
+          },
+        });
+
+        return {
+          stream: stream.pipeThrough(transformStream),
+          request,
+          response,
+          ...rest,
+        };
+      } catch (error) {
+        activeSteps.delete(stepId);
+
+        const durationMs = Date.now() - startTime;
+        await updateStepResult(stepId, {
+          duration_ms: durationMs,
+          output: null,
+          usage: null,
+          error: error instanceof Error ? error.message : String(error),
+          raw_request: null,
+          raw_response: null,
+          raw_chunks: null,
+        });
+        throw error;
+      }
+    },
+  };
+};