genai-lite 0.3.3 → 0.4.0

package/dist/llm/clients/LlamaCppClientAdapter.test.js

@@ -0,0 +1,447 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const LlamaCppClientAdapter_1 = require("./LlamaCppClientAdapter");
+// Mock OpenAI SDK
+jest.mock('openai', () => {
+    return {
+        __esModule: true,
+        default: jest.fn().mockImplementation(() => ({
+            chat: {
+                completions: {
+                    create: mockCreate,
+                },
+            },
+        })),
+    };
+});
+// Mock LlamaCppServerClient
+jest.mock('./LlamaCppServerClient', () => {
+    return {
+        LlamaCppServerClient: jest.fn().mockImplementation(() => ({
+            getHealth: mockGetHealth,
+        })),
+    };
+});
+const mockCreate = jest.fn();
+const mockGetHealth = jest.fn();
+describe('LlamaCppClientAdapter', () => {
+    let adapter;
+    let basicRequest;
+    beforeEach(() => {
+        jest.clearAllMocks();
+        adapter = new LlamaCppClientAdapter_1.LlamaCppClientAdapter();
+        basicRequest = {
+            providerId: 'llamacpp',
+            modelId: 'llama-3-8b-instruct',
+            messages: [
+                { role: 'user', content: 'Hello, how are you?' },
+            ],
+            settings: {
+                temperature: 0.7,
+                maxTokens: 1000,
+                topP: 0.95,
+                stopSequences: [],
+                frequencyPenalty: 0.0,
+                presencePenalty: 0.0,
+                supportsSystemMessage: true,
+                user: '',
+                geminiSafetySettings: [],
+                reasoning: {
+                    enabled: false,
+                    exclude: false,
+                },
+                thinkingExtraction: {
+                    enabled: false,
+                    tag: 'thinking',
+                    onMissing: 'auto',
+                },
+            },
+        };
+    });
+    describe('constructor', () => {
+        it('should use default baseURL when not provided', () => {
+            const adapterInfo = adapter.getAdapterInfo();
+            expect(adapterInfo.baseURL).toBe('http://localhost:8080');
+        });
+        it('should use custom baseURL when provided', () => {
+            const customAdapter = new LlamaCppClientAdapter_1.LlamaCppClientAdapter({
+                baseURL: 'http://localhost:9090',
+            });
+            const adapterInfo = customAdapter.getAdapterInfo();
+            expect(adapterInfo.baseURL).toBe('http://localhost:9090');
+        });
+        it('should set checkHealth to false by default', () => {
+            const adapter = new LlamaCppClientAdapter_1.LlamaCppClientAdapter();
+            expect(adapter.checkHealth).toBe(false);
+        });
+        it('should set checkHealth when provided', () => {
+            const adapter = new LlamaCppClientAdapter_1.LlamaCppClientAdapter({ checkHealth: true });
+            expect(adapter.checkHealth).toBe(true);
+        });
+    });
+    describe('sendMessage', () => {
+        it('should send message successfully', async () => {
+            mockCreate.mockResolvedValueOnce({
+                id: 'chatcmpl-123',
+                object: 'chat.completion',
+                created: 1677652288,
+                model: 'llama-3-8b-instruct',
+                choices: [
+                    {
+                        index: 0,
+                        message: {
+                            role: 'assistant',
+                            content: 'I am doing well, thank you!',
+                        },
+                        finish_reason: 'stop',
+                    },
+                ],
+                usage: {
+                    prompt_tokens: 10,
+                    completion_tokens: 8,
+                    total_tokens: 18,
+                },
+            });
+            const response = await adapter.sendMessage(basicRequest, 'not-needed');
+            expect(response.object).toBe('chat.completion');
+            if (response.object === 'chat.completion') {
+                expect(response.choices[0].message.content).toBe('I am doing well, thank you!');
+                expect(response.choices[0].finish_reason).toBe('stop');
+                expect(response.usage).toEqual({
+                    prompt_tokens: 10,
+                    completion_tokens: 8,
+                    total_tokens: 18,
+                });
+            }
+        });
+        it('should include system message when provided', async () => {
+            mockCreate.mockResolvedValueOnce({
+                id: 'chatcmpl-124',
+                choices: [
+                    {
+                        message: { role: 'assistant', content: 'Response' },
+                        finish_reason: 'stop',
+                    },
+                ],
+            });
+            const requestWithSystem = {
+                ...basicRequest,
+                systemMessage: 'You are a helpful assistant.',
+            };
+            await adapter.sendMessage(requestWithSystem, 'not-needed');
+            expect(mockCreate).toHaveBeenCalledWith(expect.objectContaining({
+                messages: expect.arrayContaining([
+                    { role: 'system', content: 'You are a helpful assistant.' },
+                ]),
+            }));
+        });
+        it('should pass stop sequences when provided', async () => {
+            mockCreate.mockResolvedValueOnce({
+                id: 'chatcmpl-125',
+                choices: [
+                    {
+                        message: { role: 'assistant', content: 'Response' },
+                        finish_reason: 'stop',
+                    },
+                ],
+            });
+            const requestWithStop = {
+                ...basicRequest,
+                settings: {
+                    ...basicRequest.settings,
+                    stopSequences: ['END', 'STOP'],
+                },
+            };
+            await adapter.sendMessage(requestWithStop, 'not-needed');
+            expect(mockCreate).toHaveBeenCalledWith(expect.objectContaining({
+                stop: ['END', 'STOP'],
+            }));
+        });
+        it('should handle length finish reason', async () => {
+            mockCreate.mockResolvedValueOnce({
+                id: 'chatcmpl-126',
+                choices: [
+                    {
+                        message: { role: 'assistant', content: 'Response...' },
+                        finish_reason: 'length',
+                    },
+                ],
+            });
+            const response = await adapter.sendMessage(basicRequest, 'not-needed');
+            expect(response.object).toBe('chat.completion');
+            if (response.object === 'chat.completion') {
+                expect(response.choices[0].finish_reason).toBe('length');
+            }
+        });
+        it('should handle completion without usage data', async () => {
+            mockCreate.mockResolvedValueOnce({
+                id: 'chatcmpl-127',
+                choices: [
+                    {
+                        message: { role: 'assistant', content: 'Response' },
+                        finish_reason: 'stop',
+                    },
+                ],
+                // No usage field
+            });
+            const response = await adapter.sendMessage(basicRequest, 'not-needed');
+            expect(response.object).toBe('chat.completion');
+            if (response.object === 'chat.completion') {
+                expect(response.usage).toBeUndefined();
+            }
+        });
+        it('should handle multiple choices', async () => {
+            mockCreate.mockResolvedValueOnce({
+                id: 'chatcmpl-128',
+                choices: [
+                    {
+                        message: { role: 'assistant', content: 'First response' },
+                        finish_reason: 'stop',
+                    },
+                    {
+                        message: { role: 'assistant', content: 'Second response' },
+                        finish_reason: 'stop',
+                    },
+                ],
+            });
+            const response = await adapter.sendMessage(basicRequest, 'not-needed');
+            expect(response.object).toBe('chat.completion');
+            if (response.object === 'chat.completion') {
+                expect(response.choices[0].message.content).toBe('First response');
+                expect(response.choices).toHaveLength(2);
+                expect(response.choices[1].message.content).toBe('Second response');
+            }
+        });
+        it('should check health before request when enabled', async () => {
+            const healthCheckAdapter = new LlamaCppClientAdapter_1.LlamaCppClientAdapter({ checkHealth: true });
+            mockGetHealth.mockResolvedValueOnce({ status: 'ok' });
+            mockCreate.mockResolvedValueOnce({
+                id: 'chatcmpl-129',
+                choices: [
+                    {
+                        message: { role: 'assistant', content: 'Response' },
+                        finish_reason: 'stop',
+                    },
+                ],
+            });
+            const response = await healthCheckAdapter.sendMessage(basicRequest, 'not-needed');
+            expect(mockGetHealth).toHaveBeenCalled();
+            expect(response.object).toBe('chat.completion');
+        });
+        it('should return error when health check fails with error status', async () => {
+            const healthCheckAdapter = new LlamaCppClientAdapter_1.LlamaCppClientAdapter({ checkHealth: true });
+            mockGetHealth.mockResolvedValueOnce({
+                status: 'error',
+                error: 'Model load failed'
+            });
+            const response = await healthCheckAdapter.sendMessage(basicRequest, 'not-needed');
+            expect(response.object).toBe('error');
+            if (response.object === 'error') {
+                expect(response.error.message).toContain('server not ready');
+                expect(response.error.message).toContain('Model load failed');
+                expect(response.error.code).toBe('PROVIDER_ERROR');
+            }
+        });
+        it('should return error when health check fails with loading status', async () => {
+            const healthCheckAdapter = new LlamaCppClientAdapter_1.LlamaCppClientAdapter({ checkHealth: true });
+            mockGetHealth.mockResolvedValueOnce({ status: 'loading' });
+            const response = await healthCheckAdapter.sendMessage(basicRequest, 'not-needed');
+            expect(response.object).toBe('error');
+            if (response.object === 'error') {
+                expect(response.error.message).toContain('loading');
+            }
+        });
+        it('should proceed with request if health check throws error', async () => {
+            const healthCheckAdapter = new LlamaCppClientAdapter_1.LlamaCppClientAdapter({ checkHealth: true });
+            mockGetHealth.mockRejectedValueOnce(new Error('Connection refused'));
+            mockCreate.mockResolvedValueOnce({
+                id: 'chatcmpl-130',
+                choices: [
+                    {
+                        message: { role: 'assistant', content: 'Response' },
+                        finish_reason: 'stop',
+                    },
+                ],
+            });
+            const response = await healthCheckAdapter.sendMessage(basicRequest, 'not-needed');
+            expect(response.object).toBe('chat.completion');
+        });
+        it('should handle connection error to server', async () => {
+            mockCreate.mockRejectedValueOnce(new Error('fetch failed: ECONNREFUSED'));
+            const response = await adapter.sendMessage(basicRequest, 'not-needed');
+            expect(response.object).toBe('error');
+            if (response.object === 'error') {
+                expect(response.error.message).toContain('Cannot connect to llama.cpp server');
+                expect(response.error.message).toContain('Is the server running?');
+                expect(response.error.code).toBe('NETWORK_ERROR');
+            }
+        });
+        it('should handle API errors', async () => {
+            mockCreate.mockRejectedValueOnce({
+                status: 400,
+                message: 'Invalid request',
+            });
+            const response = await adapter.sendMessage(basicRequest, 'not-needed');
+            expect(response.object).toBe('error');
+            if (response.object === 'error') {
+                expect(response.error.code).toBeDefined();
+            }
+        });
+        it('should handle error when no choices in response', async () => {
+            mockCreate.mockResolvedValueOnce({
+                id: 'chatcmpl-131',
+                choices: [],
+            });
+            const response = await adapter.sendMessage(basicRequest, 'not-needed');
+            expect(response.object).toBe('error');
+        });
+    });
+    describe('validateApiKey', () => {
+        it('should always return true (no API key required)', () => {
+            expect(adapter.validateApiKey('')).toBe(true);
+            expect(adapter.validateApiKey('any-string')).toBe(true);
+            expect(adapter.validateApiKey('not-needed')).toBe(true);
+        });
+    });
+    describe('getAdapterInfo', () => {
+        it('should return adapter information', () => {
+            const info = adapter.getAdapterInfo();
+            expect(info.providerId).toBe('llamacpp');
+            expect(info.name).toBe('llama.cpp Client Adapter');
+            expect(info.version).toBe('1.0.0');
+            expect(info.baseURL).toBe('http://localhost:8080');
+        });
+        it('should include custom baseURL in info', () => {
+            const customAdapter = new LlamaCppClientAdapter_1.LlamaCppClientAdapter({
+                baseURL: 'http://gpu-server:8080',
+            });
+            const info = customAdapter.getAdapterInfo();
+            expect(info.baseURL).toBe('http://gpu-server:8080');
+        });
+    });
+    describe('getServerClient', () => {
+        it('should return the underlying server client', () => {
+            const serverClient = adapter.getServerClient();
+            expect(serverClient).toBeDefined();
+        });
+    });
+    describe('message formatting', () => {
+        it('should format user messages correctly', async () => {
+            mockCreate.mockResolvedValueOnce({
+                id: 'chatcmpl-132',
+                choices: [
+                    {
+                        message: { role: 'assistant', content: 'Response' },
+                        finish_reason: 'stop',
+                    },
+                ],
+            });
+            await adapter.sendMessage(basicRequest, 'not-needed');
+            expect(mockCreate).toHaveBeenCalledWith(expect.objectContaining({
+                messages: expect.arrayContaining([
+                    { role: 'user', content: 'Hello, how are you?' },
+                ]),
+            }));
+        });
+        it('should format assistant messages correctly', async () => {
+            mockCreate.mockResolvedValueOnce({
+                id: 'chatcmpl-133',
+                choices: [
+                    {
+                        message: { role: 'assistant', content: 'Response' },
+                        finish_reason: 'stop',
+                    },
+                ],
+            });
+            const requestWithHistory = {
+                ...basicRequest,
+                messages: [
+                    { role: 'user', content: 'Hi' },
+                    { role: 'assistant', content: 'Hello!' },
+                    { role: 'user', content: 'How are you?' },
+                ],
+            };
+            await adapter.sendMessage(requestWithHistory, 'not-needed');
+            expect(mockCreate).toHaveBeenCalledWith(expect.objectContaining({
+                messages: [
+                    { role: 'user', content: 'Hi' },
+                    { role: 'assistant', content: 'Hello!' },
+                    { role: 'user', content: 'How are you?' },
+                ],
+            }));
+        });
+        it('should handle system messages in conversation', async () => {
+            mockCreate.mockResolvedValueOnce({
+                id: 'chatcmpl-134',
+                choices: [
+                    {
+                        message: { role: 'assistant', content: 'Response' },
+                        finish_reason: 'stop',
+                    },
+                ],
+            });
+            const requestWithSystemInMessages = {
+                ...basicRequest,
+                messages: [
+                    { role: 'system', content: 'Be concise' },
+                    { role: 'user', content: 'Explain AI' },
+                ],
+            };
+            await adapter.sendMessage(requestWithSystemInMessages, 'not-needed');
+            expect(mockCreate).toHaveBeenCalledWith(expect.objectContaining({
+                messages: expect.arrayContaining([
+                    { role: 'system', content: 'Be concise' },
+                ]),
+            }));
+        });
+    });
+    describe('API parameter mapping', () => {
+        it('should pass all standard parameters', async () => {
+            mockCreate.mockResolvedValueOnce({
+                id: 'chatcmpl-135',
+                choices: [
+                    {
+                        message: { role: 'assistant', content: 'Response' },
+                        finish_reason: 'stop',
+                    },
+                ],
+            });
+            const fullRequest = {
+                ...basicRequest,
+                settings: {
+                    ...basicRequest.settings,
+                    temperature: 0.9,
+                    maxTokens: 2000,
+                    topP: 0.8,
+                    frequencyPenalty: 0.5,
+                    presencePenalty: 0.3,
+                    stopSequences: ['END'],
+                },
+            };
+            await adapter.sendMessage(fullRequest, 'not-needed');
+            expect(mockCreate).toHaveBeenCalledWith(expect.objectContaining({
+                model: 'llama-3-8b-instruct',
+                temperature: 0.9,
+                max_tokens: 2000,
+                top_p: 0.8,
+                frequency_penalty: 0.5,
+                presence_penalty: 0.3,
+                stop: ['END'],
+            }));
+        });
+        it('should omit frequency penalty when zero', async () => {
+            mockCreate.mockResolvedValueOnce({
+                id: 'chatcmpl-136',
+                choices: [
+                    {
+                        message: { role: 'assistant', content: 'Response' },
+                        finish_reason: 'stop',
+                    },
+                ],
+            });
+            await adapter.sendMessage(basicRequest, 'not-needed');
+            const callArgs = mockCreate.mock.calls[0][0];
+            expect(callArgs.frequency_penalty).toBeUndefined();
+        });
+    });
+});

package/dist/llm/clients/LlamaCppServerClient.d.ts

@@ -0,0 +1,161 @@
+/**
+ * Response from the /health endpoint
+ */
+export interface LlamaCppHealthResponse {
+    status: 'loading' | 'error' | 'ok';
+    error?: string;
+}
+/**
+ * Response from the /tokenize endpoint
+ */
+export interface LlamaCppTokenizeResponse {
+    tokens: number[];
+}
+/**
+ * Response from the /detokenize endpoint
+ */
+export interface LlamaCppDetokenizeResponse {
+    content: string;
+}
+/**
+ * Response from the /embedding endpoint
+ */
+export interface LlamaCppEmbeddingResponse {
+    embedding: number[];
+}
+/**
+ * Response from the /infill endpoint
+ */
+export interface LlamaCppInfillResponse {
+    content: string;
+    tokens?: number[];
+    stop?: boolean;
+}
+/**
+ * Response from the /props endpoint
+ */
+export interface LlamaCppPropsResponse {
+    assistant_name?: string;
+    user_name?: string;
+    default_generation_settings?: Record<string, any>;
+    total_slots?: number;
+    [key: string]: any;
+}
+/**
+ * Response from the /metrics endpoint
+ */
+export interface LlamaCppMetricsResponse {
+    [key: string]: any;
+}
+/**
+ * Individual slot information from /slots endpoint
+ */
+export interface LlamaCppSlot {
+    id: number;
+    state: number;
+    prompt?: string;
+    [key: string]: any;
+}
+/**
+ * Response from the /slots endpoint
+ */
+export interface LlamaCppSlotsResponse {
+    slots: LlamaCppSlot[];
+}
+/**
+ * Client for interacting with llama.cpp server's management and utility endpoints
+ *
+ * This class provides access to non-LLM endpoints like tokenization, embeddings,
+ * health checks, and server properties. For chat completions, use LlamaCppClientAdapter.
+ *
+ * @example
+ * ```typescript
+ * const client = new LlamaCppServerClient('http://localhost:8080');
+ *
+ * // Check if server is ready
+ * const health = await client.getHealth();
+ * console.log(health.status); // 'ok', 'loading', or 'error'
+ *
+ * // Tokenize text
+ * const { tokens } = await client.tokenize('Hello world');
+ * console.log(tokens); // [123, 456, 789]
+ *
+ * // Generate embeddings
+ * const { embedding } = await client.createEmbedding('Some text');
+ * ```
+ */
+export declare class LlamaCppServerClient {
+    private baseURL;
+    /**
+     * Creates a new llama.cpp server client
+     *
+     * @param baseURL - The base URL of the llama.cpp server (e.g., 'http://localhost:8080')
+     */
+    constructor(baseURL: string);
+    /**
+     * Checks the health and readiness of the server
+     *
+     * @returns Promise resolving to health status
+     * @throws Error if the request fails
+     */
+    getHealth(): Promise<LlamaCppHealthResponse>;
+    /**
+     * Converts text to tokens using the loaded model's tokenizer
+     *
+     * @param content - The text to tokenize
+     * @returns Promise resolving to array of token IDs
+     * @throws Error if the request fails
+     */
+    tokenize(content: string): Promise<LlamaCppTokenizeResponse>;
+    /**
+     * Converts tokens back to text using the loaded model's tokenizer
+     *
+     * @param tokens - Array of token IDs to convert
+     * @returns Promise resolving to the decoded text
+     * @throws Error if the request fails
+     */
+    detokenize(tokens: number[]): Promise<LlamaCppDetokenizeResponse>;
+    /**
+     * Generates an embedding vector for the given text
+     *
+     * @param content - The text to embed
+     * @param imageData - Optional base64-encoded image data for multimodal models
+     * @returns Promise resolving to the embedding vector
+     * @throws Error if the request fails
+     */
+    createEmbedding(content: string, imageData?: string): Promise<LlamaCppEmbeddingResponse>;
+    /**
+     * Performs code infilling (completing code between prefix and suffix)
+     *
+     * @param inputPrefix - The code before the cursor/gap
+     * @param inputSuffix - The code after the cursor/gap
+     * @returns Promise resolving to the infilled completion
+     * @throws Error if the request fails
+     */
+    infill(inputPrefix: string, inputSuffix: string): Promise<LlamaCppInfillResponse>;
+    /**
+     * Retrieves server properties and configuration
+     *
+     * @returns Promise resolving to server properties
+     * @throws Error if the request fails
+     */
+    getProps(): Promise<LlamaCppPropsResponse>;
+    /**
+     * Retrieves performance metrics from the server
+     *
+     * @returns Promise resolving to metrics data
+     * @throws Error if the request fails
+     */
+    getMetrics(): Promise<LlamaCppMetricsResponse>;
+    /**
+     * Retrieves processing slot status (debugging endpoint)
+     *
+     * WARNING: This endpoint may expose sensitive information including prompt content.
+     * The llama.cpp documentation strongly advises against enabling this in production.
+     * Only use this endpoint in development/debugging environments.
+     *
+     * @returns Promise resolving to slot status information
+     * @throws Error if the request fails or endpoint is not enabled
+     */
+    getSlots(): Promise<LlamaCppSlotsResponse>;
+}