@theia/ai-chat 1.46.0-next.241

package/src/common/parse-contents.spec.ts

@@ -0,0 +1,144 @@
+// *****************************************************************************
+// Copyright (C) 2024 EclipseSource GmbH.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+// *****************************************************************************
+
+import { expect } from 'chai';
+import { ChatRequestModelImpl, ChatResponseContent, CodeChatResponseContentImpl, MarkdownChatResponseContentImpl } from './chat-model';
+import { parseContents } from './parse-contents';
+import { CodeContentMatcher, ResponseContentMatcher } from './response-content-matcher';
+
+export class CommandChatResponseContentImpl implements ChatResponseContent {
+    constructor(public readonly command: string) { }
+    kind = 'command';
+}
+
+export const CommandContentMatcher: ResponseContentMatcher = {
+    start: /^<command>$/m,
+    end: /^<\/command>$/m,
+    contentFactory: (content: string) => {
+        const code = content.replace(/^<command>\n|<\/command>$/g, '');
+        return new CommandChatResponseContentImpl(code.trim());
+    }
+};
+
+const fakeRequest = {} as ChatRequestModelImpl;
+
+describe('parseContents', () => {
+    it('should parse code content', () => {
+        const text = '```typescript\nconsole.log("Hello World");\n```';
+        const result = parseContents(text, fakeRequest);
+        expect(result).to.deep.equal([new CodeChatResponseContentImpl('console.log("Hello World");', 'typescript')]);
+    });
+
+    it('should parse markdown content', () => {
+        const text = 'Hello **World**';
+        const result = parseContents(text, fakeRequest);
+        expect(result).to.deep.equal([new MarkdownChatResponseContentImpl('Hello **World**')]);
+    });
+
+    it('should parse multiple content blocks', () => {
+        const text = '```typescript\nconsole.log("Hello World");\n```\nHello **World**';
+        const result = parseContents(text, fakeRequest);
+        expect(result).to.deep.equal([
+            new CodeChatResponseContentImpl('console.log("Hello World");', 'typescript'),
+            new MarkdownChatResponseContentImpl('\nHello **World**')
+        ]);
+    });
+
+    it('should parse multiple content blocks with different languages', () => {
+        const text = '```typescript\nconsole.log("Hello World");\n```\n```python\nprint("Hello World")\n```';
+        const result = parseContents(text, fakeRequest);
+        expect(result).to.deep.equal([
+            new CodeChatResponseContentImpl('console.log("Hello World");', 'typescript'),
+            new CodeChatResponseContentImpl('print("Hello World")', 'python')
+        ]);
+    });
+
+    it('should parse multiple content blocks with different languages and markdown', () => {
+        const text = '```typescript\nconsole.log("Hello World");\n```\nHello **World**\n```python\nprint("Hello World")\n```';
+        const result = parseContents(text, fakeRequest);
+        expect(result).to.deep.equal([
+            new CodeChatResponseContentImpl('console.log("Hello World");', 'typescript'),
+            new MarkdownChatResponseContentImpl('\nHello **World**\n'),
+            new CodeChatResponseContentImpl('print("Hello World")', 'python')
+        ]);
+    });
+
+    it('should parse content blocks with empty content', () => {
+        const text = '```typescript\n```\nHello **World**\n```python\nprint("Hello World")\n```';
+        const result = parseContents(text, fakeRequest);
+        expect(result).to.deep.equal([
+            new CodeChatResponseContentImpl('', 'typescript'),
+            new MarkdownChatResponseContentImpl('\nHello **World**\n'),
+            new CodeChatResponseContentImpl('print("Hello World")', 'python')
+        ]);
+    });
+
+    it('should parse content with markdown, code, and markdown', () => {
+        const text = 'Hello **World**\n```typescript\nconsole.log("Hello World");\n```\nGoodbye **World**';
+        const result = parseContents(text, fakeRequest);
+        expect(result).to.deep.equal([
+            new MarkdownChatResponseContentImpl('Hello **World**\n'),
+            new CodeChatResponseContentImpl('console.log("Hello World");', 'typescript'),
+            new MarkdownChatResponseContentImpl('\nGoodbye **World**')
+        ]);
+    });
+
+    it('should handle text with no special content', () => {
+        const text = 'Just some plain text.';
+        const result = parseContents(text, fakeRequest);
+        expect(result).to.deep.equal([new MarkdownChatResponseContentImpl('Just some plain text.')]);
+    });
+
+    it('should handle text with only start code block', () => {
+        const text = '```typescript\nconsole.log("Hello World");';
+        const result = parseContents(text, fakeRequest);
+        expect(result).to.deep.equal([new MarkdownChatResponseContentImpl('```typescript\nconsole.log("Hello World");')]);
+    });
+
+    it('should handle text with only end code block', () => {
+        const text = 'console.log("Hello World");\n```';
+        const result = parseContents(text, fakeRequest);
+        expect(result).to.deep.equal([new MarkdownChatResponseContentImpl('console.log("Hello World");\n```')]);
+    });
+
+    it('should handle text with unmatched code block', () => {
+        const text = '```typescript\nconsole.log("Hello World");\n```\n```python\nprint("Hello World")';
+        const result = parseContents(text, fakeRequest);
+        expect(result).to.deep.equal([
+            new CodeChatResponseContentImpl('console.log("Hello World");', 'typescript'),
+            new MarkdownChatResponseContentImpl('\n```python\nprint("Hello World")')
+        ]);
+    });
+
+    it('should parse code block without newline after language', () => {
+        const text = '```typescript console.log("Hello World");```';
+        const result = parseContents(text, fakeRequest);
+        expect(result).to.deep.equal([
+            new MarkdownChatResponseContentImpl('```typescript console.log("Hello World");```')
+        ]);
+    });
+
+    it('should parse with matches of multiple different matchers and default', () => {
+        const text = '<command>\nMY_SPECIAL_COMMAND\n</command>\nHello **World**\n```python\nprint("Hello World")\n```\n<command>\nMY_SPECIAL_COMMAND2\n</command>';
+        const result = parseContents(text, fakeRequest, [CodeContentMatcher, CommandContentMatcher]);
+        expect(result).to.deep.equal([
+            new CommandChatResponseContentImpl('MY_SPECIAL_COMMAND'),
+            new MarkdownChatResponseContentImpl('\nHello **World**\n'),
+            new CodeChatResponseContentImpl('print("Hello World")', 'python'),
+            new CommandChatResponseContentImpl('MY_SPECIAL_COMMAND2'),
+        ]);
+    });
+});

package/src/common/parse-contents.ts

@@ -0,0 +1,93 @@
+/*
+ * Copyright (C) 2024 EclipseSource GmbH.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v. 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ *
+ * This Source Code may also be made available under the following Secondary
+ * Licenses when the conditions for such availability set forth in the Eclipse
+ * Public License v. 2.0 are satisfied: GNU General Public License, version 2
+ * with the GNU Classpath Exception which is available at
+ * https://www.gnu.org/software/classpath/license.html.
+ *
+ * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+ */
+import { ChatRequestModelImpl, ChatResponseContent } from './chat-model';
+import { CodeContentMatcher, MarkdownContentFactory, ResponseContentFactory, ResponseContentMatcher } from './response-content-matcher';
+
+interface Match {
+    matcher: ResponseContentMatcher;
+    index: number;
+    content: string;
+}
+
+export function parseContents(
+    text: string,
+    request: ChatRequestModelImpl,
+    contentMatchers: ResponseContentMatcher[] = [CodeContentMatcher],
+    defaultContentFactory: ResponseContentFactory = MarkdownContentFactory
+): ChatResponseContent[] {
+    const result: ChatResponseContent[] = [];
+
+    let currentIndex = 0;
+    while (currentIndex < text.length) {
+        const remainingText = text.substring(currentIndex);
+        const match = findFirstMatch(contentMatchers, remainingText);
+        if (!match) {
+            // Add the remaining text as default content
+            if (remainingText.length > 0) {
+                result.push(defaultContentFactory(remainingText, request));
+            }
+            break;
+        }
+        // We have a match
+        // 1. Add preceding text as default content
+        if (match.index > 0) {
+            const precedingContent = remainingText.substring(0, match.index);
+            if (precedingContent.trim().length > 0) {
+                result.push(defaultContentFactory(precedingContent, request));
+            }
+        }
+        // 2. Add the matched content object
+        result.push(match.matcher.contentFactory(match.content, request));
+        // Update currentIndex to the end of the end of the match
+        // And continue with the search after the end of the match
+        currentIndex += match.index + match.content.length;
+    }
+
+    return result;
+}
+
+export function findFirstMatch(contentMatchers: ResponseContentMatcher[], text: string): Match | undefined {
+    let firstMatch: { matcher: ResponseContentMatcher, index: number, content: string } | undefined;
+    for (const matcher of contentMatchers) {
+        const startMatch = matcher.start.exec(text);
+        if (!startMatch) {
+            // No start match found, try next matcher.
+            continue;
+        }
+        const endOfStartMatch = startMatch.index + startMatch[0].length;
+        if (endOfStartMatch >= text.length) {
+            // There is no text after the start match.
+            // No need to search for the end match yet, try next matcher.
+            continue;
+        }
+        const remainingTextAfterStartMatch = text.substring(endOfStartMatch);
+        const endMatch = matcher.end.exec(remainingTextAfterStartMatch);
+        if (!endMatch) {
+            // No end match found, try next matcher.
+            continue;
+        }
+        // Found start and end match.
+        // Record the full match, if it is the earliest found so far.
+        const index = startMatch.index;
+        const contentEnd = index + startMatch[0].length + endMatch.index + endMatch[0].length;
+        const content = text.substring(index, contentEnd);
+        if (!firstMatch || index < firstMatch.index) {
+            firstMatch = { matcher, index, content };
+        }
+    }
+    return firstMatch;
+}
+

package/src/common/parsed-chat-request.ts

@@ -0,0 +1,112 @@
+// *****************************************************************************
+// Copyright (C) 2024 EclipseSource GmbH.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+// *****************************************************************************
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+// Partially copied from https://github.com/microsoft/vscode/blob/a2cab7255c0df424027be05d58e1b7b941f4ea60/src/vs/workbench/contrib/chat/common/chatParserTypes.ts
+// Partially copied from https://github.com/microsoft/vscode/blob/a2cab7255c0df424027be05d58e1b7b941f4ea60/src/vs/editor/common/core/offsetRange.ts
+
+import { AIVariable, ResolvedAIVariable, ToolRequest, toolRequestToPromptText } from '@theia/ai-core';
+import { ChatRequest } from './chat-model';
+
+export const chatVariableLeader = '#';
+export const chatAgentLeader = '@';
+export const chatFunctionLeader = '~';
+export const chatSubcommandLeader = '/';
+
+/**********************
+ * INTERFACES AND TYPE GUARDS
+ **********************/
+
+export interface OffsetRange {
+    readonly start: number;
+    readonly endExclusive: number;
+}
+
+export interface ParsedChatRequest {
+    readonly request: ChatRequest;
+    readonly parts: ParsedChatRequestPart[];
+    readonly toolRequests: Map<string, ToolRequest>;
+    readonly variables: Map<string, AIVariable>;
+}
+
+export interface ParsedChatRequestPart {
+    readonly kind: string;
+    /**
+     * The text as represented in the ChatRequest
+     */
+    readonly text: string;
+    /**
+     * The text as will be sent to the LLM
+     */
+    readonly promptText: string;
+
+    readonly range: OffsetRange;
+}
+
+export class ParsedChatRequestTextPart implements ParsedChatRequestPart {
+    readonly kind: 'text';
+
+    constructor(readonly range: OffsetRange, readonly text: string) { }
+
+    get promptText(): string {
+        return this.text;
+    }
+}
+
+export class ParsedChatRequestVariablePart implements ParsedChatRequestPart {
+    readonly kind: 'var';
+
+    public resolution: ResolvedAIVariable;
+
+    constructor(readonly range: OffsetRange, readonly variableName: string, readonly variableArg: string | undefined) { }
+
+    get text(): string {
+        const argPart = this.variableArg ? `:${this.variableArg}` : '';
+        return `${chatVariableLeader}${this.variableName}${argPart}`;
+    }
+
+    get promptText(): string {
+        return this.resolution?.value ?? this.text;
+    }
+}
+
+export class ParsedChatRequestFunctionPart implements ParsedChatRequestPart {
+    readonly kind: 'function';
+    constructor(readonly range: OffsetRange, readonly toolRequest: ToolRequest) { }
+
+    get text(): string {
+        return `${chatFunctionLeader}${this.toolRequest.id}`;
+    }
+
+    get promptText(): string {
+        return toolRequestToPromptText(this.toolRequest);
+    }
+}
+
+export class ParsedChatRequestAgentPart implements ParsedChatRequestPart {
+    readonly kind: 'agent';
+    constructor(readonly range: OffsetRange, readonly agentId: string, readonly agentName: string) { }
+
+    get text(): string {
+        return `${chatAgentLeader}${this.agentName}`;
+    }
+
+    get promptText(): string {
+        return '';
+    }
+}

package/src/common/response-content-matcher.ts

@@ -0,0 +1,103 @@
+/*
+ * Copyright (C) 2024 EclipseSource GmbH.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v. 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ *
+ * This Source Code may also be made available under the following Secondary
+ * Licenses when the conditions for such availability set forth in the Eclipse
+ * Public License v. 2.0 are satisfied: GNU General Public License, version 2
+ * with the GNU Classpath Exception which is available at
+ * https://www.gnu.org/software/classpath/license.html.
+ *
+ * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+ */
+import {
+    ChatRequestModelImpl,
+    ChatResponseContent,
+    CodeChatResponseContentImpl,
+    MarkdownChatResponseContentImpl
+} from './chat-model';
+import { injectable } from '@theia/core/shared/inversify';
+
+export type ResponseContentFactory = (content: string, request: ChatRequestModelImpl) => ChatResponseContent;
+
+export const MarkdownContentFactory: ResponseContentFactory = (content: string) =>
+    new MarkdownChatResponseContentImpl(content);
+
+/**
+ * Default response content factory used if no other `ResponseContentMatcher` applies.
+ * By default, this factory creates a markdown content object.
+ *
+ * @see MarkdownChatResponseContentImpl
+ */
+@injectable()
+export class DefaultResponseContentFactory {
+    create(content: string, request: ChatRequestModelImpl): ChatResponseContent {
+        return MarkdownContentFactory(content, request);
+    }
+}
+
+/**
+ * Clients can contribute response content matchers to parse a chat response into specific
+ * `ChatResponseContent` instances.
+ */
+export interface ResponseContentMatcher {
+    /** Regular expression for finding the start delimiter. */
+    start: RegExp;
+    /** Regular expression for finding the start delimiter. */
+    end: RegExp;
+    /**
+     * The factory creating a response content from the matching content,
+     * from start index to end index of the match (including delimiters).
+     */
+    contentFactory: ResponseContentFactory;
+}
+
+export const CodeContentMatcher: ResponseContentMatcher = {
+    start: /^```.*?$/m,
+    end: /^```$/m,
+    contentFactory: (content: string) => {
+        const language = content.match(/^```(\w+)/)?.[1] || '';
+        const code = content.replace(/^```(\w+)\n|```$/g, '');
+        return new CodeChatResponseContentImpl(code.trim(), language);
+    }
+};
+
+/**
+ * Clients can contribute response content matchers to parse the response content.
+ *
+ * The default chat user interface will collect all contributed matchers and use them
+ * to parse the response into structured content parts (e.g. code blocks, markdown blocks),
+ * which are then rendered with a `ChatResponsePartRenderer` registered for the respective
+ * content part type.
+ *
+ * ### Example
+ * ```ts
+ * bind(ResponseContentMatcherProvider).to(MyResponseContentMatcherProvider);
+ * ...
+ * @injectable()
+ * export class MyResponseContentMatcherProvider implements ResponseContentMatcherProvider {
+ *     readonly matchers: ResponseContentMatcher[] = [{
+ *       start: /^<command>$/m,
+ *       end: /^</command>$/m,
+ *       contentFactory: (content: string) => {
+ *         const command = content.replace(/^<command>\n|<\/command>$/g, '');
+ *         return new MyChatResponseContentImpl(command.trim());
+ *       }
+ *   }];
+ * }
+ * ```
+ *
+ * @see ResponseContentMatcher
+ */
+export const ResponseContentMatcherProvider = Symbol('ResponseContentMatcherProvider');
+export interface ResponseContentMatcherProvider {
+    readonly matchers: ResponseContentMatcher[];
+}
+
+@injectable()
+export class DefaultResponseContentMatcherProvider implements ResponseContentMatcherProvider {
+    readonly matchers: ResponseContentMatcher[] = [CodeContentMatcher];
+}

package/src/common/universal-chat-agent.ts

@@ -0,0 +1,117 @@
+// *****************************************************************************
+// Copyright (C) 2024 EclipseSource GmbH.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+// *****************************************************************************
+
+import { AgentSpecificVariables } from '@theia/ai-core';
+import {
+   PromptTemplate
+} from '@theia/ai-core/lib/common';
+import { injectable } from '@theia/core/shared/inversify';
+import { AbstractStreamParsingChatAgent, ChatAgent, SystemMessageDescription } from './chat-agents';
+
+export const universalTemplate: PromptTemplate = {
+   id: 'universal-system',
+   template: `{{!-- Made improvements or adaptations to this prompt template? We’d love for you to share it with the community! Contribute back here:
+https://github.com/eclipse-theia/theia/discussions/new?category=prompt-template-contribution --}}
+# Instructions
+
+You are an AI assistant integrated into the Theia IDE, specifically designed to help software developers by
+providing concise and accurate answers to programming-related questions. Your role is to enhance the
+developer's productivity by offering quick solutions, explanations, and best practices.
+Keep responses short and to the point, focusing on delivering valuable insights, best practices and
+simple solutions.
+
+### Guidelines
+
+1. **Understand Context:**
+   - Assess the context of the code or issue when available.
+   - Tailor responses to be relevant to the programming language, framework, or tools like Eclipse Theia.
+   - Ask clarifying questions if necessary to provide accurate assistance.
+
+2. **Provide Clear Solutions:**
+   - Offer direct answers or code snippets that solve the problem or clarify the concept.
+   - Avoid lengthy explanations unless necessary for understanding.
+
+3. **Promote Best Practices:**
+   - Suggest best practices and common patterns relevant to the question.
+   - Provide links to official documentation for further reading when applicable.
+
+4. **Support Multiple Languages and Tools:**
+   - Be familiar with popular programming languages, frameworks, IDEs like Eclipse Theia, and command-line tools.
+   - Adapt advice based on the language, environment, or tools specified by the developer.
+
+5. **Facilitate Learning:**
+   - Encourage learning by explaining why a solution works or why a particular approach is recommended.
+   - Keep explanations concise and educational.
+
+6. **Maintain Professional Tone:**
+   - Communicate in a friendly, professional manner.
+   - Use technical jargon appropriately, ensuring clarity for the target audience.
+
+7. **Stay on Topic:**
+   - Limit responses strictly to topics related to software development, frameworks, Eclipse Theia, terminal usage, and relevant technologies.
+   - Politely decline to answer questions unrelated to these areas by saying, "I'm here to assist with programming-related questions.
+     For other topics, please refer to a specialized source."
+
+### Example Interactions
+
+- **Question:** "What's the difference between \`let\` and \`var\` in JavaScript?"
+  **Answer:** "\`let\` is block-scoped, while \`var\` is function-scoped. Prefer \`let\` to avoid scope-related bugs."
+
+- **Question:** "How do I handle exceptions in Java?"
+  **Answer:** "Use try-catch blocks: \`\`\`java try { /* code */ } catch (ExceptionType e) { /* handle exception */ }\`\`\`."
+
+- **Question:** "What is the capital of France?"
+  **Answer:** "I'm here to assist with programming-related queries. For other topics, please refer to a specialized source."
+`
+};
+
+export const universalTemplateVariant: PromptTemplate = {
+   id: 'universal-system-empty',
+   template: '',
+   variantOf: universalTemplate.id,
+};
+
+@injectable()
+export class UniversalChatAgent extends AbstractStreamParsingChatAgent implements ChatAgent {
+   name: string;
+   description: string;
+   variables: string[];
+   promptTemplates: PromptTemplate[];
+   readonly functions: string[];
+   readonly agentSpecificVariables: AgentSpecificVariables[];
+
+   constructor() {
+      super('Universal', [{
+         purpose: 'chat',
+         identifier: 'openai/gpt-4o',
+      }], 'chat');
+      this.name = 'Universal';
+      this.description = 'This agent is designed to help software developers by providing concise and accurate '
+         + 'answers to general programming and software development questions. It is also the fall-back for any generic '
+         + 'questions the user might ask. The universal agent currently does not have any context by default, i.e. it cannot '
+         + 'access the current user context or the workspace.';
+      this.variables = [];
+      this.promptTemplates = [universalTemplate, universalTemplateVariant];
+      this.functions = [];
+      this.agentSpecificVariables = [];
+   }
+
+   protected override async getSystemMessageDescription(): Promise<SystemMessageDescription | undefined> {
+      const resolvedPrompt = await this.promptService.getPrompt(universalTemplate.id);
+      return resolvedPrompt ? SystemMessageDescription.fromResolvedPromptTemplate(resolvedPrompt) : undefined;
+   }
+
+}