@theia/ai-chat 1.54.0 → 1.55.0-next.14

package/src/common/parse-contents.spec.ts

@@ -0,0 +1,142 @@
+// *****************************************************************************
+// Copyright (C) 2024 EclipseSource GmbH.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+// *****************************************************************************
+
+import { expect } from 'chai';
+import { ChatResponseContent, CodeChatResponseContentImpl, MarkdownChatResponseContentImpl } from './chat-model';
+import { parseContents } from './parse-contents';
+import { CodeContentMatcher, ResponseContentMatcher } from './response-content-matcher';
+
+export class CommandChatResponseContentImpl implements ChatResponseContent {
+    constructor(public readonly command: string) { }
+    kind = 'command';
+}
+
+export const CommandContentMatcher: ResponseContentMatcher = {
+    start: /^<command>$/m,
+    end: /^<\/command>$/m,
+    contentFactory: (content: string) => {
+        const code = content.replace(/^<command>\n|<\/command>$/g, '');
+        return new CommandChatResponseContentImpl(code.trim());
+    }
+};
+
+describe('parseContents', () => {
+    it('should parse code content', () => {
+        const text = '```typescript\nconsole.log("Hello World");\n```';
+        const result = parseContents(text);
+        expect(result).to.deep.equal([new CodeChatResponseContentImpl('console.log("Hello World");', 'typescript')]);
+    });
+
+    it('should parse markdown content', () => {
+        const text = 'Hello **World**';
+        const result = parseContents(text);
+        expect(result).to.deep.equal([new MarkdownChatResponseContentImpl('Hello **World**')]);
+    });
+
+    it('should parse multiple content blocks', () => {
+        const text = '```typescript\nconsole.log("Hello World");\n```\nHello **World**';
+        const result = parseContents(text);
+        expect(result).to.deep.equal([
+            new CodeChatResponseContentImpl('console.log("Hello World");', 'typescript'),
+            new MarkdownChatResponseContentImpl('\nHello **World**')
+        ]);
+    });
+
+    it('should parse multiple content blocks with different languages', () => {
+        const text = '```typescript\nconsole.log("Hello World");\n```\n```python\nprint("Hello World")\n```';
+        const result = parseContents(text);
+        expect(result).to.deep.equal([
+            new CodeChatResponseContentImpl('console.log("Hello World");', 'typescript'),
+            new CodeChatResponseContentImpl('print("Hello World")', 'python')
+        ]);
+    });
+
+    it('should parse multiple content blocks with different languages and markdown', () => {
+        const text = '```typescript\nconsole.log("Hello World");\n```\nHello **World**\n```python\nprint("Hello World")\n```';
+        const result = parseContents(text);
+        expect(result).to.deep.equal([
+            new CodeChatResponseContentImpl('console.log("Hello World");', 'typescript'),
+            new MarkdownChatResponseContentImpl('\nHello **World**\n'),
+            new CodeChatResponseContentImpl('print("Hello World")', 'python')
+        ]);
+    });
+
+    it('should parse content blocks with empty content', () => {
+        const text = '```typescript\n```\nHello **World**\n```python\nprint("Hello World")\n```';
+        const result = parseContents(text);
+        expect(result).to.deep.equal([
+            new CodeChatResponseContentImpl('', 'typescript'),
+            new MarkdownChatResponseContentImpl('\nHello **World**\n'),
+            new CodeChatResponseContentImpl('print("Hello World")', 'python')
+        ]);
+    });
+
+    it('should parse content with markdown, code, and markdown', () => {
+        const text = 'Hello **World**\n```typescript\nconsole.log("Hello World");\n```\nGoodbye **World**';
+        const result = parseContents(text);
+        expect(result).to.deep.equal([
+            new MarkdownChatResponseContentImpl('Hello **World**\n'),
+            new CodeChatResponseContentImpl('console.log("Hello World");', 'typescript'),
+            new MarkdownChatResponseContentImpl('\nGoodbye **World**')
+        ]);
+    });
+
+    it('should handle text with no special content', () => {
+        const text = 'Just some plain text.';
+        const result = parseContents(text);
+        expect(result).to.deep.equal([new MarkdownChatResponseContentImpl('Just some plain text.')]);
+    });
+
+    it('should handle text with only start code block', () => {
+        const text = '```typescript\nconsole.log("Hello World");';
+        const result = parseContents(text);
+        expect(result).to.deep.equal([new MarkdownChatResponseContentImpl('```typescript\nconsole.log("Hello World");')]);
+    });
+
+    it('should handle text with only end code block', () => {
+        const text = 'console.log("Hello World");\n```';
+        const result = parseContents(text);
+        expect(result).to.deep.equal([new MarkdownChatResponseContentImpl('console.log("Hello World");\n```')]);
+    });
+
+    it('should handle text with unmatched code block', () => {
+        const text = '```typescript\nconsole.log("Hello World");\n```\n```python\nprint("Hello World")';
+        const result = parseContents(text);
+        expect(result).to.deep.equal([
+            new CodeChatResponseContentImpl('console.log("Hello World");', 'typescript'),
+            new MarkdownChatResponseContentImpl('\n```python\nprint("Hello World")')
+        ]);
+    });
+
+    it('should parse code block without newline after language', () => {
+        const text = '```typescript console.log("Hello World");```';
+        const result = parseContents(text);
+        expect(result).to.deep.equal([
+            new MarkdownChatResponseContentImpl('```typescript console.log("Hello World");```')
+        ]);
+    });
+
+    it('should parse with matches of multiple different matchers and default', () => {
+        const text = '<command>\nMY_SPECIAL_COMMAND\n</command>\nHello **World**\n```python\nprint("Hello World")\n```\n<command>\nMY_SPECIAL_COMMAND2\n</command>';
+        const result = parseContents(text, [CodeContentMatcher, CommandContentMatcher]);
+        expect(result).to.deep.equal([
+            new CommandChatResponseContentImpl('MY_SPECIAL_COMMAND'),
+            new MarkdownChatResponseContentImpl('\nHello **World**\n'),
+            new CodeChatResponseContentImpl('print("Hello World")', 'python'),
+            new CommandChatResponseContentImpl('MY_SPECIAL_COMMAND2'),
+        ]);
+    });
+});

package/src/common/parse-contents.ts

@@ -0,0 +1,92 @@
+/*
+ * Copyright (C) 2024 EclipseSource GmbH.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v. 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ *
+ * This Source Code may also be made available under the following Secondary
+ * Licenses when the conditions for such availability set forth in the Eclipse
+ * Public License v. 2.0 are satisfied: GNU General Public License, version 2
+ * with the GNU Classpath Exception which is available at
+ * https://www.gnu.org/software/classpath/license.html.
+ *
+ * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+ */
+import { ChatResponseContent } from './chat-model';
+import { CodeContentMatcher, MarkdownContentFactory, ResponseContentFactory, ResponseContentMatcher } from './response-content-matcher';
+
+interface Match {
+    matcher: ResponseContentMatcher;
+    index: number;
+    content: string;
+}
+
+export function parseContents(
+    text: string,
+    contentMatchers: ResponseContentMatcher[] = [CodeContentMatcher],
+    defaultContentFactory: ResponseContentFactory = MarkdownContentFactory
+): ChatResponseContent[] {
+    const result: ChatResponseContent[] = [];
+
+    let currentIndex = 0;
+    while (currentIndex < text.length) {
+        const remainingText = text.substring(currentIndex);
+        const match = findFirstMatch(contentMatchers, remainingText);
+        if (!match) {
+            // Add the remaining text as default content
+            if (remainingText.length > 0) {
+                result.push(defaultContentFactory(remainingText));
+            }
+            break;
+        }
+        // We have a match
+        // 1. Add preceding text as default content
+        if (match.index > 0) {
+            const precedingContent = remainingText.substring(0, match.index);
+            if (precedingContent.trim().length > 0) {
+                result.push(defaultContentFactory(precedingContent));
+            }
+        }
+        // 2. Add the matched content object
+        result.push(match.matcher.contentFactory(match.content));
+        // Update currentIndex to the end of the end of the match
+        // And continue with the search after the end of the match
+        currentIndex += match.index + match.content.length;
+    }
+
+    return result;
+}
+
+export function findFirstMatch(contentMatchers: ResponseContentMatcher[], text: string): Match | undefined {
+    let firstMatch: { matcher: ResponseContentMatcher, index: number, content: string } | undefined;
+    for (const matcher of contentMatchers) {
+        const startMatch = matcher.start.exec(text);
+        if (!startMatch) {
+            // No start match found, try next matcher.
+            continue;
+        }
+        const endOfStartMatch = startMatch.index + startMatch[0].length;
+        if (endOfStartMatch >= text.length) {
+            // There is no text after the start match.
+            // No need to search for the end match yet, try next matcher.
+            continue;
+        }
+        const remainingTextAfterStartMatch = text.substring(endOfStartMatch);
+        const endMatch = matcher.end.exec(remainingTextAfterStartMatch);
+        if (!endMatch) {
+            // No end match found, try next matcher.
+            continue;
+        }
+        // Found start and end match.
+        // Record the full match, if it is the earliest found so far.
+        const index = startMatch.index;
+        const contentEnd = index + startMatch[0].length + endMatch.index + endMatch[0].length;
+        const content = text.substring(index, contentEnd);
+        if (!firstMatch || index < firstMatch.index) {
+            firstMatch = { matcher, index, content };
+        }
+    }
+    return firstMatch;
+}
+

package/src/common/response-content-matcher.ts

@@ -0,0 +1,102 @@
+/*
+ * Copyright (C) 2024 EclipseSource GmbH.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v. 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ *
+ * This Source Code may also be made available under the following Secondary
+ * Licenses when the conditions for such availability set forth in the Eclipse
+ * Public License v. 2.0 are satisfied: GNU General Public License, version 2
+ * with the GNU Classpath Exception which is available at
+ * https://www.gnu.org/software/classpath/license.html.
+ *
+ * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+ */
+import {
+    ChatResponseContent,
+    CodeChatResponseContentImpl,
+    MarkdownChatResponseContentImpl
+} from './chat-model';
+import { injectable } from '@theia/core/shared/inversify';
+
+export type ResponseContentFactory = (content: string) => ChatResponseContent;
+
+export const MarkdownContentFactory: ResponseContentFactory = (content: string) =>
+    new MarkdownChatResponseContentImpl(content);
+
+/**
+ * Default response content factory used if no other `ResponseContentMatcher` applies.
+ * By default, this factory creates a markdown content object.
+ *
+ * @see MarkdownChatResponseContentImpl
+ */
+@injectable()
+export class DefaultResponseContentFactory {
+    create(content: string): ChatResponseContent {
+        return MarkdownContentFactory(content);
+    }
+}
+
+/**
+ * Clients can contribute response content matchers to parse a chat response into specific
+ * `ChatResponseContent` instances.
+ */
+export interface ResponseContentMatcher {
+    /** Regular expression for finding the start delimiter. */
+    start: RegExp;
+    /** Regular expression for finding the start delimiter. */
+    end: RegExp;
+    /**
+     * The factory creating a response content from the matching content,
+     * from start index to end index of the match (including delimiters).
+     */
+    contentFactory: ResponseContentFactory;
+}
+
+export const CodeContentMatcher: ResponseContentMatcher = {
+    start: /^```.*?$/m,
+    end: /^```$/m,
+    contentFactory: (content: string) => {
+        const language = content.match(/^```(\w+)/)?.[1] || '';
+        const code = content.replace(/^```(\w+)\n|```$/g, '');
+        return new CodeChatResponseContentImpl(code.trim(), language);
+    }
+};
+
+/**
+ * Clients can contribute response content matchers to parse the response content.
+ *
+ * The default chat user interface will collect all contributed matchers and use them
+ * to parse the response into structured content parts (e.g. code blocks, markdown blocks),
+ * which are then rendered with a `ChatResponsePartRenderer` registered for the respective
+ * content part type.
+ *
+ * ### Example
+ * ```ts
+ * bind(ResponseContentMatcherProvider).to(MyResponseContentMatcherProvider);
+ * ...
+ * @injectable()
+ * export class MyResponseContentMatcherProvider implements ResponseContentMatcherProvider {
+ *     readonly matchers: ResponseContentMatcher[] = [{
+ *       start: /^<command>$/m,
+ *       end: /^</command>$/m,
+ *       contentFactory: (content: string) => {
+ *         const command = content.replace(/^<command>\n|<\/command>$/g, '');
+ *         return new MyChatResponseContentImpl(command.trim());
+ *       }
+ *   }];
+ * }
+ * ```
+ *
+ * @see ResponseContentMatcher
+ */
+export const ResponseContentMatcherProvider = Symbol('ResponseContentMatcherProvider');
+export interface ResponseContentMatcherProvider {
+    readonly matchers: ResponseContentMatcher[];
+}
+
+@injectable()
+export class DefaultResponseContentMatcherProvider implements ResponseContentMatcherProvider {
+    readonly matchers: ResponseContentMatcher[] = [CodeContentMatcher];
+}