@turingpulse/sdk 1.0.1

package/tests/compat/google-genai.test.ts

@@ -0,0 +1,61 @@
+/**
+ * Compatibility tests for the Google Generative AI TypeScript SDK.
+ */
+import { describe, it, expect } from 'vitest';
+import { FingerprintBuilder, classifyConfigKey } from '../../src/fingerprint.js';
+
+// ── Import guard ──────────────────────────────────────────────────────────
+
+describe('Google GenAI import', () => {
+  it('@google/generative-ai is importable', async () => {
+    const mod = await import('@google/generative-ai');
+    expect(mod).toBeDefined();
+  });
+});
+
+// ── Key classification ────────────────────────────────────────────────────
+
+describe('Google GenAI key classification', () => {
+  const modelKeys = [
+    'model', 'temperature', 'top_p', 'top_k',
+    'max_output_tokens', 'candidate_count', 'stop_sequences',
+    'maxOutputTokens', 'topP', 'topK', 'candidateCount', 'stopSequences',
+  ];
+  const generalKeys = ['safetySettings', 'generationConfig'];
+
+  it.each(modelKeys)('classifies %s as model', (key) => {
+    expect(classifyConfigKey(key)).toBe('model');
+  });
+
+  it.each(generalKeys)('classifies %s as general', (key) => {
+    expect(classifyConfigKey(key)).toBe('general');
+  });
+});
+
+// ── Fingerprint hashing ──────────────────────────────────────────────────
+
+describe('Google GenAI fingerprint', () => {
+  it('produces model hash', () => {
+    const builder = new FingerprintBuilder({ captureConfigs: true, capturePrompts: true });
+    builder.recordNode('gemini', 'llm', {
+      model: 'gemini-1.5-pro',
+      temperature: 0.4,
+      maxOutputTokens: 2048,
+      topP: 0.8,
+      topK: 32,
+    });
+    const fp = builder.getFingerprint();
+    expect(fp.node_model_hashes['gemini']).toBeDefined();
+  });
+
+  it('maxOutputTokens change alters model hash', () => {
+    const before = new FingerprintBuilder({ captureConfigs: true, capturePrompts: true });
+    const after = new FingerprintBuilder({ captureConfigs: true, capturePrompts: true });
+
+    before.recordNode('llm', 'llm', { model: 'gemini-1.5-pro', maxOutputTokens: 2048 });
+    after.recordNode('llm', 'llm', { model: 'gemini-1.5-pro', maxOutputTokens: 4096 });
+
+    expect(before.getFingerprint().node_model_hashes['llm'])
+      .not.toBe(after.getFingerprint().node_model_hashes['llm']);
+  });
+});

package/tests/compat/langchain-openai.test.ts

@@ -0,0 +1,41 @@
+/**
+ * Compatibility tests for the @langchain/openai package.
+ */
+import { describe, it, expect } from 'vitest';
+import { FingerprintBuilder, classifyConfigKey } from '../../src/fingerprint.js';
+
+// ── Import guard ──────────────────────────────────────────────────────────
+
+describe('LangChain-OpenAI import', () => {
+  it('@langchain/openai is importable', async () => {
+    const mod = await import('@langchain/openai');
+    expect(mod).toBeDefined();
+  });
+});
+
+// ── Fingerprint hashing ──────────────────────────────────────────────────
+
+describe('LangChain-OpenAI fingerprint', () => {
+  it('produces model hash', () => {
+    const builder = new FingerprintBuilder({ captureConfigs: true, capturePrompts: true });
+    builder.recordNode('lc_openai', 'llm', {
+      model: 'gpt-4-turbo',
+      temperature: 0.3,
+      maxTokens: 4096,
+      streaming: true,
+    });
+    const fp = builder.getFingerprint();
+    expect(fp.node_model_hashes['lc_openai']).toBeDefined();
+  });
+
+  it('streaming change does not affect model hash', () => {
+    const before = new FingerprintBuilder({ captureConfigs: true, capturePrompts: true });
+    const after = new FingerprintBuilder({ captureConfigs: true, capturePrompts: true });
+
+    before.recordNode('llm', 'llm', { model: 'gpt-4-turbo', temperature: 0.3, streaming: false });
+    after.recordNode('llm', 'llm', { model: 'gpt-4-turbo', temperature: 0.3, streaming: true });
+
+    expect(before.getFingerprint().node_model_hashes['llm'])
+      .toBe(after.getFingerprint().node_model_hashes['llm']);
+  });
+});

package/tests/compat/langchain.test.ts

@@ -0,0 +1,64 @@
+/**
+ * Compatibility tests for the LangChain.js framework.
+ */
+import { describe, it, expect } from 'vitest';
+import { FingerprintBuilder, classifyConfigKey } from '../../src/fingerprint.js';
+
+// ── Import guard ──────────────────────────────────────────────────────────
+
+describe('LangChain import', () => {
+  it('@langchain/core is importable', async () => {
+    const mod = await import('@langchain/core');
+    expect(mod).toBeDefined();
+  });
+});
+
+// ── Key classification ────────────────────────────────────────────────────
+
+describe('LangChain key classification', () => {
+  const modelKeys = [
+    'model', 'model_name', 'modelName', 'temperature',
+    'max_tokens', 'maxTokens', 'top_p', 'topP',
+    'frequency_penalty', 'frequencyPenalty',
+    'presence_penalty', 'presencePenalty',
+    'model_kwargs', 'modelKwargs',
+  ];
+  const generalKeys = ['verbose', 'callbacks', 'cache', 'streaming', 'tags'];
+
+  it.each(modelKeys)('classifies %s as model', (key) => {
+    expect(classifyConfigKey(key)).toBe('model');
+  });
+
+  it.each(generalKeys)('classifies %s as general', (key) => {
+    expect(classifyConfigKey(key)).toBe('general');
+  });
+});
+
+// ── Fingerprint hashing ──────────────────────────────────────────────────
+
+describe('LangChain fingerprint', () => {
+  it('produces model hash', () => {
+    const builder = new FingerprintBuilder({ captureConfigs: true, capturePrompts: true });
+    builder.recordNode('lc_chat', 'llm', {
+      modelName: 'gpt-4',
+      temperature: 0.0,
+      maxTokens: 2048,
+      verbose: true,
+    });
+    const fp = builder.getFingerprint();
+    expect(fp.node_model_hashes['lc_chat']).toBeDefined();
+  });
+
+  it('verbose change does not alter model hash', () => {
+    const before = new FingerprintBuilder({ captureConfigs: true, capturePrompts: true });
+    const after = new FingerprintBuilder({ captureConfigs: true, capturePrompts: true });
+
+    before.recordNode('llm', 'llm', { modelName: 'gpt-4', temperature: 0.0, verbose: false });
+    after.recordNode('llm', 'llm', { modelName: 'gpt-4', temperature: 0.0, verbose: true });
+
+    expect(before.getFingerprint().node_model_hashes['llm'])
+      .toBe(after.getFingerprint().node_model_hashes['llm']);
+    expect(before.getFingerprint().node_config_hashes['llm'])
+      .not.toBe(after.getFingerprint().node_config_hashes['llm']);
+  });
+});

package/tests/compat/mistral.test.ts

@@ -0,0 +1,58 @@
+/**
+ * Compatibility tests for the Mistral AI TypeScript SDK.
+ */
+import { describe, it, expect } from 'vitest';
+import { FingerprintBuilder, classifyConfigKey } from '../../src/fingerprint.js';
+
+// ── Import guard ──────────────────────────────────────────────────────────
+
+describe('Mistral import', () => {
+  it('@mistralai/mistralai is importable', async () => {
+    const mod = await import('@mistralai/mistralai');
+    expect(mod).toBeDefined();
+  });
+});
+
+// ── Key classification ────────────────────────────────────────────────────
+
+describe('Mistral key classification', () => {
+  const modelKeys = ['model', 'temperature', 'top_p', 'topP', 'max_tokens', 'maxTokens', 'stop'];
+  const generalKeys = ['stream', 'safe_prompt'];
+
+  it.each(modelKeys)('classifies %s as model', (key) => {
+    expect(classifyConfigKey(key)).toBe('model');
+  });
+
+  it.each(generalKeys)('classifies %s as general', (key) => {
+    expect(classifyConfigKey(key)).toBe('general');
+  });
+});
+
+// ── Fingerprint hashing ──────────────────────────────────────────────────
+
+describe('Mistral fingerprint', () => {
+  it('produces model hash', () => {
+    const builder = new FingerprintBuilder({ captureConfigs: true, capturePrompts: true });
+    builder.recordNode('mistral_chat', 'llm', {
+      model: 'mistral-large-latest',
+      temperature: 0.7,
+      maxTokens: 4096,
+      stream: false,
+    });
+    const fp = builder.getFingerprint();
+    expect(fp.node_model_hashes['mistral_chat']).toBeDefined();
+  });
+
+  it('safe_prompt is general', () => {
+    const before = new FingerprintBuilder({ captureConfigs: true, capturePrompts: true });
+    const after = new FingerprintBuilder({ captureConfigs: true, capturePrompts: true });
+
+    before.recordNode('llm', 'llm', { model: 'mistral-large-latest', temperature: 0.7, safe_prompt: false });
+    after.recordNode('llm', 'llm', { model: 'mistral-large-latest', temperature: 0.7, safe_prompt: true });
+
+    expect(before.getFingerprint().node_model_hashes['llm'])
+      .toBe(after.getFingerprint().node_model_hashes['llm']);
+    expect(before.getFingerprint().node_config_hashes['llm'])
+      .not.toBe(after.getFingerprint().node_config_hashes['llm']);
+  });
+});

package/tests/compat/openai.test.ts

@@ -0,0 +1,71 @@
+/**
+ * Compatibility tests for the OpenAI Node.js SDK.
+ */
+import { describe, it, expect } from 'vitest';
+import { FingerprintBuilder, classifyConfigKey, splitConfig } from '../../src/fingerprint.js';
+
+// ── Import guard ──────────────────────────────────────────────────────────
+
+describe('OpenAI import', () => {
+  it('openai package is importable', async () => {
+    const mod = await import('openai');
+    expect(mod).toBeDefined();
+  });
+});
+
+// ── Key classification ────────────────────────────────────────────────────
+
+describe('OpenAI key classification', () => {
+  const modelKeys = ['model', 'temperature', 'top_p', 'max_tokens', 'frequency_penalty', 'presence_penalty', 'stop', 'response_format'];
+  const camelKeys = ['maxTokens', 'topP', 'frequencyPenalty', 'presencePenalty', 'responseFormat'];
+  const generalKeys = ['stream', 'n', 'logprobs', 'user', 'seed'];
+
+  it.each(modelKeys)('classifies %s as model', (key) => {
+    expect(classifyConfigKey(key)).toBe('model');
+  });
+
+  it.each(camelKeys)('classifies camelCase %s as model', (key) => {
+    expect(classifyConfigKey(key)).toBe('model');
+  });
+
+  it.each(generalKeys)('classifies %s as general', (key) => {
+    expect(classifyConfigKey(key)).toBe('general');
+  });
+});
+
+// ── Fingerprint hashing ──────────────────────────────────────────────────
+
+describe('OpenAI fingerprint', () => {
+  it('produces model hash for OpenAI config', () => {
+    const builder = new FingerprintBuilder({ captureConfigs: true, capturePrompts: true });
+    builder.recordNode('chat_llm', 'llm', { model: 'gpt-4o', temperature: 0.7, max_tokens: 4096, stream: true });
+    const fp = builder.getFingerprint();
+    expect(fp.node_model_hashes['chat_llm']).toBeDefined();
+    expect(fp.node_config_hashes['chat_llm']).toBeDefined();
+  });
+
+  it('model change alters model hash only', () => {
+    const before = new FingerprintBuilder({ captureConfigs: true, capturePrompts: true });
+    const after = new FingerprintBuilder({ captureConfigs: true, capturePrompts: true });
+
+    const base = { model: 'gpt-4o', temperature: 0.7, stream: true };
+    before.recordNode('llm', 'llm', { ...base, temperature: 0.7 });
+    after.recordNode('llm', 'llm', { ...base, temperature: 0.2 });
+
+    expect(before.getFingerprint().node_model_hashes['llm'])
+      .not.toBe(after.getFingerprint().node_model_hashes['llm']);
+  });
+
+  it('general-only change does not affect model hash', () => {
+    const before = new FingerprintBuilder({ captureConfigs: true, capturePrompts: true });
+    const after = new FingerprintBuilder({ captureConfigs: true, capturePrompts: true });
+
+    before.recordNode('llm', 'llm', { model: 'gpt-4o', temperature: 0.7, stream: true });
+    after.recordNode('llm', 'llm', { model: 'gpt-4o', temperature: 0.7, stream: false });
+
+    expect(before.getFingerprint().node_model_hashes['llm'])
+      .toBe(after.getFingerprint().node_model_hashes['llm']);
+    expect(before.getFingerprint().node_config_hashes['llm'])
+      .not.toBe(after.getFingerprint().node_config_hashes['llm']);
+  });
+});

package/tests/compat/vercel-ai.test.ts

@@ -0,0 +1,56 @@
+/**
+ * Compatibility tests for the Vercel AI SDK.
+ */
+import { describe, it, expect } from 'vitest';
+import { FingerprintBuilder, classifyConfigKey } from '../../src/fingerprint.js';
+
+// ── Import guard ──────────────────────────────────────────────────────────
+
+describe('Vercel AI import', () => {
+  it('ai package is importable', async () => {
+    const mod = await import('ai');
+    expect(mod).toBeDefined();
+  });
+});
+
+// ── Key classification ────────────────────────────────────────────────────
+
+describe('Vercel AI key classification', () => {
+  const modelKeys = ['model', 'temperature', 'topP', 'maxTokens', 'frequencyPenalty', 'presencePenalty', 'stopSequences'];
+  const generalKeys = ['onFinish', 'onStepFinish', 'experimental_telemetry'];
+
+  it.each(modelKeys)('classifies %s as model', (key) => {
+    expect(classifyConfigKey(key)).toBe('model');
+  });
+
+  it.each(generalKeys)('classifies %s as general', (key) => {
+    expect(classifyConfigKey(key)).toBe('general');
+  });
+});
+
+// ── Fingerprint hashing ──────────────────────────────────────────────────
+
+describe('Vercel AI fingerprint', () => {
+  it('produces model hash', () => {
+    const builder = new FingerprintBuilder({ captureConfigs: true, capturePrompts: true });
+    builder.recordNode('vercel_chat', 'llm', {
+      model: 'gpt-4o',
+      temperature: 0.5,
+      maxTokens: 2048,
+      topP: 0.9,
+    });
+    const fp = builder.getFingerprint();
+    expect(fp.node_model_hashes['vercel_chat']).toBeDefined();
+  });
+
+  it('temperature change alters model hash', () => {
+    const before = new FingerprintBuilder({ captureConfigs: true, capturePrompts: true });
+    const after = new FingerprintBuilder({ captureConfigs: true, capturePrompts: true });
+
+    before.recordNode('llm', 'llm', { model: 'gpt-4o', temperature: 0.5 });
+    after.recordNode('llm', 'llm', { model: 'gpt-4o', temperature: 0.1 });
+
+    expect(before.getFingerprint().node_model_hashes['llm'])
+      .not.toBe(after.getFingerprint().node_model_hashes['llm']);
+  });
+});

package/tests/plugins/anthropic-wrapper.test.ts

@@ -0,0 +1,120 @@
+/**
+ * Integration tests for @turingpulse/sdk-anthropic plugin.
+ *
+ * Verifies patchAnthropic monkey-patches the caller's Anthropic module
+ * and emits events with correct metadata fields.
+ */
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+import { init } from '@turingpulse/sdk';
+import { patchAnthropic, unpatchAnthropic } from '../../packages/anthropic/src/wrapper.js';
+
+const MOCK_RESPONSE = {
+  id: 'msg_test',
+  type: 'message',
+  role: 'assistant',
+  model: 'claude-3-sonnet-20240229',
+  content: [{ type: 'text', text: 'instrumented reply' }],
+  stop_reason: 'end_turn',
+  usage: { input_tokens: 8, output_tokens: 4 },
+};
+
+let capturedEvents: Record<string, unknown>[] = [];
+let originalFetch: typeof globalThis.fetch;
+
+beforeEach(() => {
+  capturedEvents = [];
+  originalFetch = globalThis.fetch;
+
+  globalThis.fetch = async (input: RequestInfo | URL, initOpts?: RequestInit): Promise<Response> => {
+    const url = typeof input === 'string' ? input : input instanceof URL ? input.toString() : (input as Request).url;
+
+    if (url.includes('api.anthropic.com')) {
+      return new Response(JSON.stringify(MOCK_RESPONSE), {
+        status: 200,
+        headers: { 'Content-Type': 'application/json' },
+      });
+    }
+
+    if (url.includes('/api/v1/sdk/events') && initOpts?.body) {
+      try {
+        const parsed = JSON.parse(initOpts.body as string);
+        capturedEvents.push(...(parsed.events ?? [parsed]));
+      } catch { /* ignore */ }
+      return new Response(JSON.stringify({ success: true }), {
+        status: 200,
+        headers: { 'Content-Type': 'application/json' },
+      });
+    }
+
+    return new Response('{}', { status: 200 });
+  };
+
+  init({
+    apiKey: 'test-key',
+    workflowName: 'test-anthropic-plugin',
+    endpoint: 'https://test-endpoint.turingpulse.ai',
+    fetchImpl: globalThis.fetch,
+  });
+});
+
+afterEach(async () => {
+  await unpatchAnthropic();
+  globalThis.fetch = originalFetch;
+});
+
+describe('patchAnthropic plugin wrapper', () => {
+  it('patches the caller-supplied Anthropic module prototype', async () => {
+    const anthropicMod = await import('@anthropic-ai/sdk');
+    const cls = anthropicMod.default ?? anthropicMod;
+    const proto = (cls as Record<string, unknown>).Messages
+      ? ((cls as Record<string, unknown>).Messages as Record<string, unknown>).prototype as Record<string, unknown>
+      : undefined;
+
+    if (!proto) {
+      console.warn('Could not resolve Anthropic.Messages.prototype — skipping');
+      return;
+    }
+
+    const originalName = (proto.create as Function).name;
+    await patchAnthropic({ name: 'test-workflow', anthropicModule: anthropicMod });
+
+    expect((proto.create as Function).name).toBe('instrumented');
+    expect((proto.create as Function).name).not.toBe(originalName);
+  });
+
+  it('emits event with correct node_type, model, provider, tokens', async () => {
+    const anthropicMod = await import('@anthropic-ai/sdk');
+    await patchAnthropic({ name: 'test-anthropic', anthropicModule: anthropicMod });
+
+    const Anthropic = anthropicMod.default;
+    const client = new Anthropic({ apiKey: 'mock-key' });
+    await client.messages.create({
+      model: 'claude-3-sonnet-20240229',
+      max_tokens: 100,
+      messages: [{ role: 'user', content: 'hello from test' }],
+    });
+
+    await new Promise((r) => setTimeout(r, 500));
+
+    expect(capturedEvents.length).toBeGreaterThanOrEqual(1);
+
+    const event = capturedEvents[0] as Record<string, unknown>;
+    const payload = event.payload as Record<string, unknown>;
+    const metadata = payload.metadata as Record<string, string>;
+
+    expect(metadata.node_type).toBe('llm');
+    expect(metadata.model).toBe('claude-3-sonnet-20240229');
+    expect(metadata.provider).toBe('anthropic');
+    expect(metadata.framework).toBe('anthropic');
+    expect(metadata.input).toBeDefined();
+    expect(metadata.output).toBeDefined();
+
+    const tokens = payload.tokens as { prompt: number; completion: number };
+    expect(tokens.prompt).toBe(8);
+    expect(tokens.completion).toBe(4);
+  });
+
+  it('throws when name is missing', async () => {
+    await expect(patchAnthropic({})).rejects.toThrow(/name or TP_WORKFLOW_NAME/);
+  });
+});

package/tests/plugins/langchain-wrapper.test.ts

@@ -0,0 +1,128 @@
+/**
+ * Integration tests for @turingpulse/sdk-langchain plugin.
+ *
+ * Verifies instrumentLangchain wraps a Runnable and emits events
+ * with correct metadata fields (node_type, model, provider, I/O).
+ */
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+import { init } from '@turingpulse/sdk';
+import { instrumentLangchain } from '../../packages/langchain/src/wrapper.js';
+
+let capturedEvents: Record<string, unknown>[] = [];
+let originalFetch: typeof globalThis.fetch;
+
+beforeEach(() => {
+  capturedEvents = [];
+  originalFetch = globalThis.fetch;
+
+  globalThis.fetch = async (input: RequestInfo | URL, initOpts?: RequestInit): Promise<Response> => {
+    const url = typeof input === 'string' ? input : input instanceof URL ? input.toString() : (input as Request).url;
+
+    if (url.includes('/api/v1/sdk/events') && initOpts?.body) {
+      try {
+        const parsed = JSON.parse(initOpts.body as string);
+        capturedEvents.push(...(parsed.events ?? [parsed]));
+      } catch { /* ignore */ }
+      return new Response(JSON.stringify({ success: true }), {
+        status: 200,
+        headers: { 'Content-Type': 'application/json' },
+      });
+    }
+
+    return new Response('{}', { status: 200 });
+  };
+
+  init({
+    apiKey: 'test-key',
+    workflowName: 'test-langchain-plugin',
+    endpoint: 'https://test-endpoint.turingpulse.ai',
+    fetchImpl: globalThis.fetch,
+  });
+});
+
+afterEach(() => {
+  globalThis.fetch = originalFetch;
+});
+
+describe('instrumentLangchain plugin wrapper', () => {
+  it('wraps a runnable and emits event with correct metadata', async () => {
+    const mockRunnable = {
+      invoke: async (input: unknown) => ({
+        content: `Response to: ${input}`,
+        response_metadata: {
+          model_name: 'gpt-4o',
+          tokenUsage: { promptTokens: 15, completionTokens: 10 },
+        },
+      }),
+    };
+
+    const runner = instrumentLangchain(mockRunnable, {
+      name: 'langchain-test-agent',
+      model: 'gpt-4o',
+      provider: 'openai',
+    });
+
+    await runner('what is AI?');
+    await new Promise((r) => setTimeout(r, 500));
+
+    expect(capturedEvents.length).toBeGreaterThanOrEqual(1);
+
+    const event = capturedEvents[0] as Record<string, unknown>;
+    const payload = event.payload as Record<string, unknown>;
+    const metadata = payload.metadata as Record<string, string>;
+
+    expect(metadata.node_type).toBe('llm');
+    expect(metadata.framework).toBe('langchain');
+    expect(metadata.model).toBe('gpt-4o');
+    expect(metadata.provider).toBe('openai');
+    expect(metadata.input).toBeDefined();
+    expect(metadata.output).toBeDefined();
+  });
+
+  it('extracts token counts from response_metadata', async () => {
+    const mockRunnable = {
+      invoke: async () => ({
+        content: 'reply',
+        response_metadata: {
+          tokenUsage: { promptTokens: 20, completionTokens: 12 },
+        },
+      }),
+    };
+
+    const runner = instrumentLangchain(mockRunnable, {
+      name: 'token-test',
+      model: 'gpt-4o-mini',
+    });
+
+    await runner('test prompt');
+    await new Promise((r) => setTimeout(r, 500));
+
+    const payload = (capturedEvents[0] as Record<string, unknown>).payload as Record<string, unknown>;
+    const tokens = payload.tokens as { prompt: number; completion: number };
+
+    expect(tokens.prompt).toBe(20);
+    expect(tokens.completion).toBe(12);
+  });
+
+  it('uses provided model and provider when response has no model_name', async () => {
+    const mockRunnable = {
+      invoke: async () => ({ content: 'plain response' }),
+    };
+
+    const runner = instrumentLangchain(mockRunnable, {
+      name: 'model-fallback',
+      model: 'claude-3-haiku',
+      provider: 'anthropic',
+    });
+
+    await runner('test');
+    await new Promise((r) => setTimeout(r, 500));
+
+    const metadata = (
+      (capturedEvents[0] as Record<string, unknown>).payload as Record<string, unknown>
+    ).metadata as Record<string, string>;
+
+    expect(metadata.model).toBe('claude-3-haiku');
+    expect(metadata.provider).toBe('anthropic');
+  });
+});