@theia/ai-llamafile 1.46.0-next.241

package/src/browser/llamafile-preferences.ts

@@ -0,0 +1,62 @@
+// *****************************************************************************
+// Copyright (C) 2024 EclipseSource GmbH.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+// *****************************************************************************
+
+import { PreferenceContribution, PreferenceSchema } from '@theia/core/lib/browser';
+import { interfaces } from '@theia/core/shared/inversify';
+
+export const AI_LLAMAFILE_PREFERENCES_TITLE = '✨ AI LlamaFile';
+export const PREFERENCE_LLAMAFILE = 'ai-features.llamafile.llamafiles';
+
+export const aiLlamafilePreferencesSchema: PreferenceSchema = {
+    type: 'object',
+    properties: {
+        [PREFERENCE_LLAMAFILE]: {
+            title: AI_LLAMAFILE_PREFERENCES_TITLE,
+            markdownDescription: 'This setting allows you to configure and manage LlamaFile models in Theia IDE.\
+            \n\
+            Each entry requires a user-friendly `name`, the file `uri` pointing to your LlamaFile, and the `port` on which it will run.\
+            \n\
+            To start a LlamaFile, use the "Start LlamaFile" command, which enables you to select the desired model.\
+            \n\
+            If you edit an entry (e.g., change the port), any running instance will stop, and you will need to manually start it again.\
+            \n\
+            [Learn more about configuring and managing LlamaFiles in the Theia IDE documentation](https://theia-ide.org/docs/user_ai/#llamafile-models).',
+            type: 'array',
+            default: [],
+            items: {
+                type: 'object',
+                properties: {
+                    name: {
+                        type: 'string',
+                        description: 'The model name to use for this Llamafile.'
+                    },
+                    uri: {
+                        type: 'string',
+                        description: 'The file uri to the Llamafile.'
+                    },
+                    port: {
+                        type: 'number',
+                        description: 'The port to use to start the server.'
+                    }
+                }
+            }
+        }
+    }
+};
+
+export function bindAILlamafilePreferences(bind: interfaces.Bind): void {
+    bind(PreferenceContribution).toConstantValue({ schema: aiLlamafilePreferencesSchema });
+}

package/src/common/llamafile-language-model.ts

@@ -0,0 +1,126 @@
+// *****************************************************************************
+// Copyright (C) 2024 EclipseSource GmbH.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+// *****************************************************************************
+
+import { LanguageModel, LanguageModelRequest, LanguageModelResponse, LanguageModelStreamResponsePart } from '@theia/ai-core';
+import { CancellationToken } from '@theia/core';
+export class LlamafileLanguageModel implements LanguageModel {
+
+    readonly providerId = 'llamafile';
+    readonly vendor: string = 'Mozilla';
+
+    /**
+     * @param name the unique name for this language model. It will be used to identify the model in the UI.
+     * @param uri the URI pointing to the Llamafile model location.
+     * @param port the port on which the Llamafile model server operates.
+     * @param defaultRequestSettings optional default settings for requests made using this model.
+     */
+    constructor(
+        public readonly name: string,
+        public readonly uri: string,
+        public readonly port: number,
+        public defaultRequestSettings?: { [key: string]: unknown }
+    ) { }
+
+    get id(): string {
+        return this.name;
+    }
+    protected getSettings(request: LanguageModelRequest): Record<string, unknown> {
+        const settings = request.settings ? request.settings : this.defaultRequestSettings;
+        if (!settings) {
+            return {
+                n_predict: 200,
+                stream: true,
+                stop: ['</s>', 'Llama:', 'User:', '<|eot_id|>'],
+                cache_prompt: true,
+            };
+        }
+        return settings;
+    }
+
+    async request(request: LanguageModelRequest, cancellationToken?: CancellationToken): Promise<LanguageModelResponse> {
+        const settings = this.getSettings(request);
+        try {
+            let prompt = request.messages.map(message => {
+                switch (message.actor) {
+                    case 'user':
+                        return `User: ${message.query}`;
+                    case 'ai':
+                        return `Llama: ${message.query}`;
+                    case 'system':
+                        return `${message.query.replace(/\n\n/g, '\n')}`;
+                }
+            }).join('\n');
+            prompt += '\nLlama:';
+            const response = await fetch(`http://localhost:${this.port}/completion`, {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json',
+                },
+                body: JSON.stringify({
+                    prompt: prompt,
+                    ...settings
+                }),
+            });
+
+            if (!response.ok) {
+                throw new Error(`HTTP error! status: ${response.status}`);
+            }
+
+            if (!response.body) {
+                throw new Error('Response body is undefined');
+            }
+
+            const reader = response.body.getReader();
+            const decoder = new TextDecoder();
+            return {
+                stream: {
+                    [Symbol.asyncIterator](): AsyncIterator<LanguageModelStreamResponsePart> {
+                        return {
+                            async next(): Promise<IteratorResult<LanguageModelStreamResponsePart>> {
+                                if (cancellationToken?.isCancellationRequested) {
+                                    reader.cancel();
+                                    return { value: undefined, done: true };
+                                }
+                                const { value, done } = await reader.read();
+                                if (done) {
+                                    return { value: undefined, done: true };
+                                }
+                                const read = decoder.decode(value, { stream: true });
+                                const chunk = read.split('\n').filter(l => l.length !== 0).reduce((acc, line) => {
+                                    try {
+                                        const parsed = JSON.parse(line.substring(6));
+                                        acc += parsed.content;
+                                        return acc;
+                                    } catch (error) {
+                                        console.error('Error parsing JSON:', error);
+                                        return acc;
+                                    }
+                                }, '');
+                                return { value: { content: chunk }, done: false };
+                            }
+                        };
+                    }
+                }
+            };
+        } catch (error) {
+            console.error('Error:', error);
+            return {
+                text: `Error: ${error}`
+            };
+        }
+    }
+
+}

package/src/common/llamafile-manager.ts

@@ -0,0 +1,42 @@
+// *****************************************************************************
+// Copyright (C) 2024 EclipseSource GmbH.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+// *****************************************************************************
+export const LlamafileManager = Symbol('LlamafileManager');
+
+export const LlamafileManagerPath = '/services/llamafilemanager';
+
+export interface LlamafileModelDescription {
+    name: string;
+    uri: string;
+    port: number;
+    /**
+     * Default request settings for the Llama model.
+     */
+    defaultRequestSettings?: { [key: string]: unknown };
+}
+
+export interface LlamafileManager {
+    startServer(name: string): Promise<void>;
+    stopServer(name: string): void;
+    getStartedLlamafiles(): Promise<string[]>;
+    setClient(client: LlamafileServerManagerClient): void;
+    addLanguageModels(llamaFiles: LlamafileModelDescription[]): Promise<void>;
+    removeLanguageModels(modelIds: string[]): void;
+    updateRequestSettings(modelId: string, requestSettings?: { [key: string]: unknown }): void;
+}
+export interface LlamafileServerManagerClient {
+    log(llamafileName: string, message: string): void;
+    error(llamafileName: string, message: string): void;
+}

package/src/node/llamafile-backend-module.ts

@@ -0,0 +1,32 @@
+// *****************************************************************************
+// Copyright (C) 2024 EclipseSource GmbH.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+// *****************************************************************************
+
+import { ContainerModule } from '@theia/core/shared/inversify';
+import { LlamafileManagerImpl } from './llamafile-manager-impl';
+import { LlamafileManager, LlamafileServerManagerClient, LlamafileManagerPath } from '../common/llamafile-manager';
+import { ConnectionHandler, RpcConnectionHandler } from '@theia/core';
+
+export default new ContainerModule(bind => {
+    bind(LlamafileManager).to(LlamafileManagerImpl).inSingletonScope();
+    bind(ConnectionHandler).toDynamicValue(ctx => new RpcConnectionHandler<LlamafileServerManagerClient>(
+        LlamafileManagerPath,
+        client => {
+            const service = ctx.container.get<LlamafileManager>(LlamafileManager);
+            service.setClient(client);
+            return service;
+        }
+    )).inSingletonScope();
+});

package/src/node/llamafile-manager-impl.ts

@@ -0,0 +1,129 @@
+// *****************************************************************************
+// Copyright (C) 2024 EclipseSource GmbH.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+// *****************************************************************************
+import { LanguageModelRegistry } from '@theia/ai-core';
+import { inject, injectable } from '@theia/core/shared/inversify';
+import { ChildProcessWithoutNullStreams, spawn } from 'child_process';
+import { basename, dirname } from 'path';
+import { fileURLToPath } from 'url';
+import { LlamafileLanguageModel } from '../common/llamafile-language-model';
+import { LlamafileManager, LlamafileModelDescription, LlamafileServerManagerClient } from '../common/llamafile-manager';
+
+@injectable()
+export class LlamafileManagerImpl implements LlamafileManager {
+
+    @inject(LanguageModelRegistry)
+    protected languageModelRegistry: LanguageModelRegistry;
+
+    private processMap: Map<string, ChildProcessWithoutNullStreams> = new Map();
+    private client: LlamafileServerManagerClient;
+
+    async addLanguageModels(LlamafileModelDescriptions: LlamafileModelDescription[]): Promise<void> {
+        for (const llamafile of LlamafileModelDescriptions) {
+            const model = await this.languageModelRegistry.getLanguageModel(llamafile.name);
+            if (model) {
+                if (!(model instanceof LlamafileLanguageModel)) {
+                    console.warn(`Llamafile: model ${model.id} is not a Llamafile model`);
+                    continue;
+                } else {
+                    // This can happen during the initializing of more than one frontends, changes are handled in the frontend
+                    console.info(`Llamafile: skip creating or updating model ${llamafile.name} because it already exists.`);
+                }
+            } else {
+                this.languageModelRegistry.addLanguageModels([
+                    new LlamafileLanguageModel(
+                        llamafile.name,
+                        llamafile.uri,
+                        llamafile.port,
+                        llamafile.defaultRequestSettings
+                    )
+                ]);
+            }
+        }
+    }
+
+    async updateRequestSettings(modelId: string, requestSettings?: { [key: string]: unknown; }): Promise<void> {
+        const model = await this.languageModelRegistry.getLanguageModel(modelId);
+        if (model) {
+            if (!(model instanceof LlamafileLanguageModel)) {
+                console.warn(`Llamafile: model ${model.id} is not a Llamafile model`);
+                return;
+            } else {
+                model.defaultRequestSettings = requestSettings;
+            }
+        }
+    }
+
+    removeLanguageModels(modelIds: string[]): void {
+        modelIds.filter(modelId => this.isStarted(modelId)).forEach(modelId => this.stopServer(modelId));
+        this.languageModelRegistry.removeLanguageModels(modelIds);
+    }
+
+    async getStartedLlamafiles(): Promise<string[]> {
+        const models = await this.languageModelRegistry.getLanguageModels();
+        return models.filter(model => model instanceof LlamafileLanguageModel && this.isStarted(model.name)).map(model => model.id);
+    }
+
+    async startServer(name: string): Promise<void> {
+        if (!this.processMap.has(name)) {
+            const models = await this.languageModelRegistry.getLanguageModels();
+            const llm = models.find(model => model.id === name && model instanceof LlamafileLanguageModel) as LlamafileLanguageModel | undefined;
+            if (llm === undefined) {
+                return Promise.reject(`Llamafile ${name} not found`);
+            }
+            const filePath = fileURLToPath(llm.uri);
+
+            // Extract the directory and file name
+            const dir = dirname(filePath);
+            const fileName = basename(filePath);
+            const currentProcess = spawn(`./${fileName}`, ['--port', '' + llm.port, '--server', '--nobrowser'], { cwd: dir });
+            this.processMap.set(name, currentProcess);
+
+            currentProcess.stdout.on('data', (data: Buffer) => {
+                const output = data.toString();
+                this.client.log(name, output);
+            });
+            currentProcess.stderr.on('data', (data: Buffer) => {
+                const output = data.toString();
+                this.client.error(name, output);
+            });
+            currentProcess.on('close', code => {
+                this.client.log(name, `LlamaFile process for file ${name} exited with code ${code}`);
+                this.processMap.delete(name);
+            });
+            currentProcess.on('error', error => {
+                this.client.error(name, `Error starting LlamaFile process for file ${name}: ${error.message}`);
+                this.processMap.delete(name);
+            });
+        }
+    }
+
+    stopServer(name: string): void {
+        if (this.processMap.has(name)) {
+            const currentProcess = this.processMap.get(name);
+            currentProcess!.kill();
+            this.processMap.delete(name);
+        }
+    }
+
+    isStarted(name: string): boolean {
+        return this.processMap.has(name);
+    }
+
+    setClient(client: LlamafileServerManagerClient): void {
+        this.client = client;
+    }
+
+}

package/src/package.spec.ts

@@ -0,0 +1,27 @@
+// *****************************************************************************
+// Copyright (C) 2024 TypeFox GmbH and others.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+// *****************************************************************************
+
+/* note: this bogus test file is required so that
+   we are able to run mocha unit tests on this
+   package, without having any actual unit tests in it.
+   This way a coverage report will be generated,
+   showing 0% coverage, instead of no report.
+   This file can be removed once we have real unit
+   tests in place. */
+
+describe('ai-llamafile package', () => {
+    it('support code coverage statistics', () => true);
+});