@artemiskit/adapter-langchain 0.2.0

package/package.json

@@ -0,0 +1,54 @@
+{
+  "name": "@artemiskit/adapter-langchain",
+  "version": "0.2.0",
+  "description": "LangChain.js adapter for ArtemisKit - Test LangChain chains and agents",
+  "type": "module",
+  "license": "Apache-2.0",
+  "author": "code-sensei",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/code-sensei/artemiskit.git",
+    "directory": "packages/adapters/langchain"
+  },
+  "bugs": {
+    "url": "https://github.com/code-sensei/artemiskit/issues"
+  },
+  "homepage": "https://artemiskit.vercel.app",
+  "keywords": ["llm", "langchain", "agents", "chains", "adapter", "artemiskit", "testing"],
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    }
+  },
+  "scripts": {
+    "build": "tsc && bun build ./src/index.ts --outdir ./dist --target bun",
+    "typecheck": "tsc --noEmit",
+    "clean": "rm -rf dist",
+    "test": "bun test"
+  },
+  "dependencies": {
+    "@artemiskit/core": "workspace:*",
+    "nanoid": "^5.0.0"
+  },
+  "peerDependencies": {
+    "@langchain/core": ">=0.1.0",
+    "langchain": ">=0.1.0"
+  },
+  "peerDependenciesMeta": {
+    "@langchain/core": {
+      "optional": true
+    },
+    "langchain": {
+      "optional": true
+    }
+  },
+  "devDependencies": {
+    "@langchain/core": "^0.3.0",
+    "@types/bun": "^1.1.0",
+    "langchain": "^0.3.0",
+    "typescript": "^5.3.0"
+  }
+}

package/src/client.test.ts

@@ -0,0 +1,309 @@
+/**
+ * Tests for LangChain adapter
+ */
+
+import { describe, expect, it, mock, spyOn } from 'bun:test';
+import { LangChainAdapter, createLangChainAdapter } from './client';
+import type { LangChainRunnable, LangChainRunnableOutput } from './types';
+
+// Mock runnable factory for testing
+function createMockRunnable(
+  outputOrFn:
+    | string
+    | LangChainRunnableOutput
+    | ((input: unknown) => Promise<LangChainRunnableOutput>),
+  options?: { supportsStreaming?: boolean }
+): LangChainRunnable {
+  const runnable: LangChainRunnable = {
+    invoke: mock(async (input) => {
+      if (typeof outputOrFn === 'function') {
+        return outputOrFn(input);
+      }
+      if (typeof outputOrFn === 'string') {
+        return outputOrFn as unknown as LangChainRunnableOutput;
+      }
+      return outputOrFn;
+    }),
+  };
+
+  if (options?.supportsStreaming) {
+    runnable.stream = mock(async function* (input) {
+      const result = typeof outputOrFn === 'string' ? outputOrFn : 'streamed content';
+      for (const char of result) {
+        yield { content: char };
+      }
+    });
+  }
+
+  return runnable;
+}
+
+describe('LangChainAdapter', () => {
+  describe('constructor', () => {
+    it('should create adapter with runnable', () => {
+      const runnable = createMockRunnable('test output');
+      const adapter = new LangChainAdapter({ provider: 'langchain' }, runnable);
+
+      expect(adapter.provider).toBe('langchain');
+    });
+
+    it('should detect agent type from properties', () => {
+      const agentRunnable = {
+        ...createMockRunnable('test'),
+        agent: {},
+      } as LangChainRunnable;
+
+      const adapter = new LangChainAdapter({ provider: 'langchain' }, agentRunnable);
+      expect(adapter.provider).toBe('langchain');
+    });
+
+    it('should use configured runnable type', () => {
+      const runnable = createMockRunnable('test');
+      const adapter = new LangChainAdapter(
+        { provider: 'langchain', runnableType: 'chain' },
+        runnable
+      );
+
+      expect(adapter.provider).toBe('langchain');
+    });
+  });
+
+  describe('generate', () => {
+    it('should call runnable.invoke with string prompt', async () => {
+      const runnable = createMockRunnable({ output: 'Hello, world!' });
+      const adapter = new LangChainAdapter({ provider: 'langchain' }, runnable);
+
+      const result = await adapter.generate({ prompt: 'Say hello' });
+
+      expect(runnable.invoke).toHaveBeenCalledWith({ input: 'Say hello' });
+      expect(result.text).toBe('Hello, world!');
+      expect(result.finishReason).toBe('stop');
+    });
+
+    it('should handle chat message array prompts', async () => {
+      const runnable = createMockRunnable({ output: 'Response' });
+      const adapter = new LangChainAdapter({ provider: 'langchain' }, runnable);
+
+      const result = await adapter.generate({
+        prompt: [
+          { role: 'system', content: 'You are helpful' },
+          { role: 'user', content: 'Hello' },
+        ],
+      });
+
+      expect(runnable.invoke).toHaveBeenCalledWith(
+        expect.objectContaining({
+          input: 'Hello',
+          system: 'You are helpful',
+        })
+      );
+      expect(result.text).toBe('Response');
+    });
+
+    it('should handle direct string responses (StringOutputParser)', async () => {
+      const runnable = createMockRunnable('Direct string response');
+      const adapter = new LangChainAdapter({ provider: 'langchain' }, runnable);
+
+      const result = await adapter.generate({ prompt: 'Test' });
+
+      expect(result.text).toBe('Direct string response');
+    });
+
+    it('should handle content property responses', async () => {
+      const runnable = createMockRunnable({ content: 'Content response' });
+      const adapter = new LangChainAdapter({ provider: 'langchain' }, runnable);
+
+      const result = await adapter.generate({ prompt: 'Test' });
+
+      expect(result.text).toBe('Content response');
+    });
+
+    it('should handle text property responses', async () => {
+      const runnable = createMockRunnable({ text: 'Text response' });
+      const adapter = new LangChainAdapter({ provider: 'langchain' }, runnable);
+
+      const result = await adapter.generate({ prompt: 'Test' });
+
+      expect(result.text).toBe('Text response');
+    });
+
+    it('should handle custom input/output keys', async () => {
+      const runnable = createMockRunnable({ answer: 'Custom output' });
+      const adapter = new LangChainAdapter(
+        { provider: 'langchain', inputKey: 'question', outputKey: 'answer' },
+        runnable
+      );
+
+      const result = await adapter.generate({ prompt: 'What is 2+2?' });
+
+      expect(runnable.invoke).toHaveBeenCalledWith({ question: 'What is 2+2?' });
+      expect(result.text).toBe('Custom output');
+    });
+
+    it('should capture intermediate steps from agent execution', async () => {
+      const agentOutput: LangChainRunnableOutput = {
+        output: 'Final answer',
+        intermediateSteps: [
+          {
+            action: { tool: 'calculator', toolInput: { query: '2+2' } },
+            observation: '4',
+          },
+          {
+            action: { tool: 'search', toolInput: { query: 'meaning of life' } },
+            observation: '42',
+          },
+        ],
+      };
+
+      const runnable = createMockRunnable(agentOutput);
+      const adapter = new LangChainAdapter(
+        { provider: 'langchain', runnableType: 'agent' },
+        runnable
+      );
+
+      const result = await adapter.generate({ prompt: 'Complex question' });
+
+      expect(result.text).toBe('Final answer');
+      expect(result.raw).toMatchObject({
+        metadata: {
+          runnableType: 'agent',
+          toolsUsed: ['calculator', 'search'],
+          totalToolCalls: 2,
+        },
+      });
+    });
+
+    it('should track latency', async () => {
+      const runnable = createMockRunnable(async () => {
+        await new Promise((resolve) => setTimeout(resolve, 10));
+        return { output: 'Delayed response' };
+      });
+
+      const adapter = new LangChainAdapter({ provider: 'langchain' }, runnable);
+      const result = await adapter.generate({ prompt: 'Test' });
+
+      expect(result.latencyMs).toBeGreaterThanOrEqual(10);
+    });
+
+    it('should include model name in result', async () => {
+      const runnable = createMockRunnable({ output: 'Test' });
+      const adapter = new LangChainAdapter(
+        { provider: 'langchain', name: 'my-rag-chain' },
+        runnable
+      );
+
+      const result = await adapter.generate({ prompt: 'Test' });
+
+      expect(result.model).toBe('my-rag-chain');
+    });
+
+    it('should fallback to JSON for complex outputs', async () => {
+      const complexOutput = { data: { nested: 'value' }, count: 42 };
+      const runnable = createMockRunnable(complexOutput as unknown as LangChainRunnableOutput);
+      const adapter = new LangChainAdapter({ provider: 'langchain' }, runnable);
+
+      const result = await adapter.generate({ prompt: 'Test' });
+
+      expect(result.text).toBe(JSON.stringify(complexOutput));
+    });
+  });
+
+  describe('stream', () => {
+    it('should stream chunks when supported', async () => {
+      const runnable = createMockRunnable('streaming', { supportsStreaming: true });
+      const adapter = new LangChainAdapter({ provider: 'langchain' }, runnable);
+
+      const chunks: string[] = [];
+      const onChunk = (chunk: string) => chunks.push(chunk);
+
+      for await (const chunk of adapter.stream({ prompt: 'Test' }, onChunk)) {
+        // Collect
+      }
+
+      expect(chunks.length).toBeGreaterThan(0);
+      expect(chunks.join('')).toBe('streaming');
+    });
+
+    it('should fallback to generate when streaming not supported', async () => {
+      const runnable = createMockRunnable({ output: 'non-streaming' });
+      const adapter = new LangChainAdapter({ provider: 'langchain' }, runnable);
+
+      const chunks: string[] = [];
+      const onChunk = (chunk: string) => chunks.push(chunk);
+
+      for await (const chunk of adapter.stream({ prompt: 'Test' }, onChunk)) {
+        // Collect
+      }
+
+      expect(chunks).toEqual(['non-streaming']);
+    });
+  });
+
+  describe('capabilities', () => {
+    it('should report streaming capability based on runnable', async () => {
+      const streamingRunnable = createMockRunnable('test', { supportsStreaming: true });
+      const adapter = new LangChainAdapter({ provider: 'langchain' }, streamingRunnable);
+
+      const caps = await adapter.capabilities();
+      expect(caps.streaming).toBe(true);
+    });
+
+    it('should report no streaming when not supported', async () => {
+      const runnable = createMockRunnable('test');
+      const adapter = new LangChainAdapter({ provider: 'langchain' }, runnable);
+
+      const caps = await adapter.capabilities();
+      expect(caps.streaming).toBe(false);
+    });
+
+    it('should report tool capabilities for agents', async () => {
+      const runnable = createMockRunnable('test');
+      const adapter = new LangChainAdapter(
+        { provider: 'langchain', runnableType: 'agent' },
+        runnable
+      );
+
+      const caps = await adapter.capabilities();
+      expect(caps.functionCalling).toBe(true);
+      expect(caps.toolUse).toBe(true);
+    });
+
+    it('should report no tool capabilities for chains', async () => {
+      const runnable = createMockRunnable('test');
+      const adapter = new LangChainAdapter(
+        { provider: 'langchain', runnableType: 'chain' },
+        runnable
+      );
+
+      const caps = await adapter.capabilities();
+      expect(caps.functionCalling).toBe(false);
+      expect(caps.toolUse).toBe(false);
+    });
+  });
+
+  describe('close', () => {
+    it('should complete without error', async () => {
+      const runnable = createMockRunnable('test');
+      const adapter = new LangChainAdapter({ provider: 'langchain' }, runnable);
+
+      await expect(adapter.close()).resolves.toBeUndefined();
+    });
+  });
+});
+
+describe('createLangChainAdapter', () => {
+  it('should create adapter with factory function', () => {
+    const runnable = createMockRunnable('test');
+    const adapter = createLangChainAdapter(runnable, { name: 'test-chain' });
+
+    expect(adapter).toBeInstanceOf(LangChainAdapter);
+    expect(adapter.provider).toBe('langchain');
+  });
+
+  it('should work with minimal options', () => {
+    const runnable = createMockRunnable('test');
+    const adapter = createLangChainAdapter(runnable);
+
+    expect(adapter).toBeInstanceOf(LangChainAdapter);
+  });
+});

package/src/client.ts

@@ -0,0 +1,264 @@
+/**
+ * LangChain Adapter
+ * Wraps LangChain chains and agents for ArtemisKit testing
+ */
+
+import type {
+  AdapterConfig,
+  GenerateOptions,
+  GenerateResult,
+  ModelCapabilities,
+  ModelClient,
+} from '@artemiskit/core';
+import { nanoid } from 'nanoid';
+import type {
+  LangChainAdapterConfig,
+  LangChainExecutionMetadata,
+  LangChainRunnable,
+  LangChainRunnableOutput,
+  LangChainRunnableType,
+} from './types';
+
+/**
+ * Adapter for testing LangChain chains and agents with ArtemisKit
+ *
+ * @example
+ * ```typescript
+ * import { LangChainAdapter } from '@artemiskit/adapter-langchain';
+ * import { ChatOpenAI } from '@langchain/openai';
+ * import { StringOutputParser } from '@langchain/core/output_parsers';
+ * import { ChatPromptTemplate } from '@langchain/core/prompts';
+ *
+ * // Create a LangChain chain
+ * const model = new ChatOpenAI({ model: 'gpt-4' });
+ * const prompt = ChatPromptTemplate.fromTemplate('Answer: {input}');
+ * const chain = prompt.pipe(model).pipe(new StringOutputParser());
+ *
+ * // Wrap with ArtemisKit adapter
+ * const adapter = new LangChainAdapter({
+ *   provider: 'langchain',
+ *   runnable: chain,
+ *   runnableType: 'chain',
+ * });
+ *
+ * // Use in ArtemisKit tests
+ * const result = await adapter.generate({ prompt: 'What is 2+2?' });
+ * ```
+ */
+export class LangChainAdapter implements ModelClient {
+  private runnable: LangChainRunnable;
+  private config: LangChainAdapterConfig;
+  private runnableType: LangChainRunnableType;
+  readonly provider = 'langchain';
+
+  constructor(config: AdapterConfig, runnable: LangChainRunnable) {
+    this.config = config as LangChainAdapterConfig;
+    this.runnable = runnable;
+    this.runnableType = this.config.runnableType ?? this.detectRunnableType(runnable);
+  }
+
+  /**
+   * Attempt to detect the type of runnable based on its properties
+   */
+  private detectRunnableType(runnable: LangChainRunnable): LangChainRunnableType {
+    // Check for agent-specific properties
+    // Cast through unknown first since LangChainRunnable is a specific interface
+    const runnableAny = runnable as unknown as Record<string, unknown>;
+    if (
+      runnableAny.agent ||
+      runnableAny.agentExecutor ||
+      typeof runnableAny.runAgent === 'function'
+    ) {
+      return 'agent';
+    }
+
+    // Check for LLM-specific properties
+    if (runnableAny.modelName || runnableAny.model || runnableAny._llmType) {
+      return 'llm';
+    }
+
+    // Default to 'runnable' for generic LCEL chains
+    return 'runnable';
+  }
+
+  async generate(options: GenerateOptions): Promise<GenerateResult> {
+    const startTime = Date.now();
+
+    // Prepare input based on options
+    const input = this.prepareInput(options);
+
+    // Execute the runnable
+    const response = await this.runnable.invoke(input);
+
+    const latencyMs = Date.now() - startTime;
+
+    // Extract text output from various possible response shapes
+    const text = this.extractOutput(response);
+
+    // Extract metadata from execution
+    const metadata = this.extractMetadata(response);
+
+    return {
+      id: nanoid(),
+      model: this.config.name || `langchain:${this.runnableType}`,
+      text,
+      tokens: {
+        prompt: 0, // LangChain doesn't expose token counts directly
+        completion: 0,
+        total: 0,
+      },
+      latencyMs,
+      finishReason: 'stop',
+      raw: {
+        response,
+        metadata,
+      },
+    };
+  }
+
+  async *stream(options: GenerateOptions, onChunk: (chunk: string) => void): AsyncIterable<string> {
+    if (!this.runnable.stream) {
+      // Fallback to non-streaming if not supported
+      const result = await this.generate(options);
+      onChunk(result.text);
+      yield result.text;
+      return;
+    }
+
+    const input = this.prepareInput(options);
+    const stream = this.runnable.stream(input);
+
+    for await (const chunk of stream) {
+      const text = chunk.content?.toString() || chunk.text || '';
+      if (text) {
+        onChunk(text);
+        yield text;
+      }
+    }
+  }
+
+  async capabilities(): Promise<ModelCapabilities> {
+    return {
+      streaming: typeof this.runnable.stream === 'function',
+      functionCalling: this.runnableType === 'agent',
+      toolUse: this.runnableType === 'agent',
+      maxContext: 128000, // Varies by underlying model
+      vision: false, // Depends on underlying model
+      jsonMode: false,
+    };
+  }
+
+  async close(): Promise<void> {
+    // No cleanup needed for most LangChain runnables
+  }
+
+  /**
+   * Prepare input for the LangChain runnable
+   */
+  private prepareInput(options: GenerateOptions): Record<string, unknown> | string {
+    const inputKey = this.config.inputKey ?? 'input';
+
+    // Handle string prompts
+    if (typeof options.prompt === 'string') {
+      // Some runnables accept just a string, others need an object
+      return { [inputKey]: options.prompt };
+    }
+
+    // Handle chat message array - convert to single prompt string
+    const messages = options.prompt;
+    const lastUserMessage = messages.findLast((m) => m.role === 'user');
+    const systemMessage = messages.find((m) => m.role === 'system');
+
+    // Build input with system context if available
+    if (systemMessage) {
+      return {
+        [inputKey]: lastUserMessage?.content || '',
+        system: systemMessage.content,
+        messages: messages.map((m) => ({
+          role: m.role,
+          content: m.content,
+        })),
+      };
+    }
+
+    return { [inputKey]: lastUserMessage?.content || '' };
+  }
+
+  /**
+   * Extract the text output from a LangChain response
+   * Handles various response shapes from different runnable types
+   */
+  private extractOutput(response: LangChainRunnableOutput): string {
+    const outputKey = this.config.outputKey ?? 'output';
+
+    // Direct string response (from StringOutputParser)
+    if (typeof response === 'string') {
+      return response;
+    }
+
+    // Check common output keys in order of preference
+    const possibleKeys = [outputKey, 'output', 'content', 'text', 'result', 'answer'];
+
+    for (const key of possibleKeys) {
+      const value = response[key];
+      if (typeof value === 'string') {
+        return value;
+      }
+      if (value && typeof value === 'object' && 'content' in value) {
+        return String((value as { content: unknown }).content);
+      }
+    }
+
+    // For agent responses, extract from intermediate steps
+    if (response.intermediateSteps?.length) {
+      const steps = response.intermediateSteps;
+      const lastStep = steps[steps.length - 1];
+      return lastStep?.observation || JSON.stringify(response);
+    }
+
+    // Fallback to JSON stringification
+    return JSON.stringify(response);
+  }
+
+  /**
+   * Extract execution metadata from the response
+   */
+  private extractMetadata(response: LangChainRunnableOutput): LangChainExecutionMetadata {
+    const metadata: LangChainExecutionMetadata = {
+      runnableType: this.runnableType,
+      name: this.config.name,
+    };
+
+    // Capture intermediate steps if available and enabled
+    if (this.config.captureIntermediateSteps !== false && response.intermediateSteps) {
+      metadata.intermediateSteps = response.intermediateSteps;
+      metadata.toolsUsed = [...new Set(response.intermediateSteps.map((s) => s.action.tool))];
+      metadata.totalToolCalls = response.intermediateSteps.length;
+    }
+
+    return metadata;
+  }
+}
+
+/**
+ * Factory function to create a LangChain adapter
+ *
+ * @example
+ * ```typescript
+ * const adapter = createLangChainAdapter(myChain, {
+ *   name: 'my-rag-chain',
+ *   runnableType: 'chain',
+ * });
+ * ```
+ */
+export function createLangChainAdapter(
+  runnable: LangChainRunnable,
+  options?: Partial<LangChainAdapterConfig>
+): LangChainAdapter {
+  const config: LangChainAdapterConfig = {
+    provider: 'langchain',
+    ...options,
+  };
+
+  return new LangChainAdapter(config, runnable);
+}

package/src/index.ts

@@ -0,0 +1,29 @@
+/**
+ * @artemiskit/adapter-langchain
+ *
+ * LangChain.js adapter for ArtemisKit LLM evaluation toolkit.
+ * Enables testing of LangChain chains, agents, and runnables.
+ *
+ * @example
+ * ```typescript
+ * import { createLangChainAdapter } from '@artemiskit/adapter-langchain';
+ * import { ChatOpenAI } from '@langchain/openai';
+ *
+ * const model = new ChatOpenAI({ model: 'gpt-4' });
+ * const adapter = createLangChainAdapter(model, { name: 'gpt4-direct' });
+ *
+ * // Use with ArtemisKit
+ * const result = await adapter.generate({ prompt: 'Hello!' });
+ * ```
+ */
+
+export { LangChainAdapter, createLangChainAdapter } from './client';
+export type {
+  LangChainAdapterConfig,
+  LangChainRunnable,
+  LangChainRunnableOutput,
+  LangChainRunnableType,
+  LangChainIntermediateStep,
+  LangChainStreamChunk,
+  LangChainExecutionMetadata,
+} from './types';