@theia/ai-chat 1.54.0 → 1.55.0-next.14

package/lib/common/parse-contents.spec.js

@@ -0,0 +1,133 @@
+"use strict";
+// *****************************************************************************
+// Copyright (C) 2024 EclipseSource GmbH.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+// *****************************************************************************
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CommandContentMatcher = exports.CommandChatResponseContentImpl = void 0;
+const chai_1 = require("chai");
+const chat_model_1 = require("./chat-model");
+const parse_contents_1 = require("./parse-contents");
+const response_content_matcher_1 = require("./response-content-matcher");
+class CommandChatResponseContentImpl {
+    constructor(command) {
+        this.command = command;
+        this.kind = 'command';
+    }
+}
+exports.CommandChatResponseContentImpl = CommandChatResponseContentImpl;
+exports.CommandContentMatcher = {
+    start: /^<command>$/m,
+    end: /^<\/command>$/m,
+    contentFactory: (content) => {
+        const code = content.replace(/^<command>\n|<\/command>$/g, '');
+        return new CommandChatResponseContentImpl(code.trim());
+    }
+};
+describe('parseContents', () => {
+    it('should parse code content', () => {
+        const text = '```typescript\nconsole.log("Hello World");\n```';
+        const result = (0, parse_contents_1.parseContents)(text);
+        (0, chai_1.expect)(result).to.deep.equal([new chat_model_1.CodeChatResponseContentImpl('console.log("Hello World");', 'typescript')]);
+    });
+    it('should parse markdown content', () => {
+        const text = 'Hello **World**';
+        const result = (0, parse_contents_1.parseContents)(text);
+        (0, chai_1.expect)(result).to.deep.equal([new chat_model_1.MarkdownChatResponseContentImpl('Hello **World**')]);
+    });
+    it('should parse multiple content blocks', () => {
+        const text = '```typescript\nconsole.log("Hello World");\n```\nHello **World**';
+        const result = (0, parse_contents_1.parseContents)(text);
+        (0, chai_1.expect)(result).to.deep.equal([
+            new chat_model_1.CodeChatResponseContentImpl('console.log("Hello World");', 'typescript'),
+            new chat_model_1.MarkdownChatResponseContentImpl('\nHello **World**')
+        ]);
+    });
+    it('should parse multiple content blocks with different languages', () => {
+        const text = '```typescript\nconsole.log("Hello World");\n```\n```python\nprint("Hello World")\n```';
+        const result = (0, parse_contents_1.parseContents)(text);
+        (0, chai_1.expect)(result).to.deep.equal([
+            new chat_model_1.CodeChatResponseContentImpl('console.log("Hello World");', 'typescript'),
+            new chat_model_1.CodeChatResponseContentImpl('print("Hello World")', 'python')
+        ]);
+    });
+    it('should parse multiple content blocks with different languages and markdown', () => {
+        const text = '```typescript\nconsole.log("Hello World");\n```\nHello **World**\n```python\nprint("Hello World")\n```';
+        const result = (0, parse_contents_1.parseContents)(text);
+        (0, chai_1.expect)(result).to.deep.equal([
+            new chat_model_1.CodeChatResponseContentImpl('console.log("Hello World");', 'typescript'),
+            new chat_model_1.MarkdownChatResponseContentImpl('\nHello **World**\n'),
+            new chat_model_1.CodeChatResponseContentImpl('print("Hello World")', 'python')
+        ]);
+    });
+    it('should parse content blocks with empty content', () => {
+        const text = '```typescript\n```\nHello **World**\n```python\nprint("Hello World")\n```';
+        const result = (0, parse_contents_1.parseContents)(text);
+        (0, chai_1.expect)(result).to.deep.equal([
+            new chat_model_1.CodeChatResponseContentImpl('', 'typescript'),
+            new chat_model_1.MarkdownChatResponseContentImpl('\nHello **World**\n'),
+            new chat_model_1.CodeChatResponseContentImpl('print("Hello World")', 'python')
+        ]);
+    });
+    it('should parse content with markdown, code, and markdown', () => {
+        const text = 'Hello **World**\n```typescript\nconsole.log("Hello World");\n```\nGoodbye **World**';
+        const result = (0, parse_contents_1.parseContents)(text);
+        (0, chai_1.expect)(result).to.deep.equal([
+            new chat_model_1.MarkdownChatResponseContentImpl('Hello **World**\n'),
+            new chat_model_1.CodeChatResponseContentImpl('console.log("Hello World");', 'typescript'),
+            new chat_model_1.MarkdownChatResponseContentImpl('\nGoodbye **World**')
+        ]);
+    });
+    it('should handle text with no special content', () => {
+        const text = 'Just some plain text.';
+        const result = (0, parse_contents_1.parseContents)(text);
+        (0, chai_1.expect)(result).to.deep.equal([new chat_model_1.MarkdownChatResponseContentImpl('Just some plain text.')]);
+    });
+    it('should handle text with only start code block', () => {
+        const text = '```typescript\nconsole.log("Hello World");';
+        const result = (0, parse_contents_1.parseContents)(text);
+        (0, chai_1.expect)(result).to.deep.equal([new chat_model_1.MarkdownChatResponseContentImpl('```typescript\nconsole.log("Hello World");')]);
+    });
+    it('should handle text with only end code block', () => {
+        const text = 'console.log("Hello World");\n```';
+        const result = (0, parse_contents_1.parseContents)(text);
+        (0, chai_1.expect)(result).to.deep.equal([new chat_model_1.MarkdownChatResponseContentImpl('console.log("Hello World");\n```')]);
+    });
+    it('should handle text with unmatched code block', () => {
+        const text = '```typescript\nconsole.log("Hello World");\n```\n```python\nprint("Hello World")';
+        const result = (0, parse_contents_1.parseContents)(text);
+        (0, chai_1.expect)(result).to.deep.equal([
+            new chat_model_1.CodeChatResponseContentImpl('console.log("Hello World");', 'typescript'),
+            new chat_model_1.MarkdownChatResponseContentImpl('\n```python\nprint("Hello World")')
+        ]);
+    });
+    it('should parse code block without newline after language', () => {
+        const text = '```typescript console.log("Hello World");```';
+        const result = (0, parse_contents_1.parseContents)(text);
+        (0, chai_1.expect)(result).to.deep.equal([
+            new chat_model_1.MarkdownChatResponseContentImpl('```typescript console.log("Hello World");```')
+        ]);
+    });
+    it('should parse with matches of multiple different matchers and default', () => {
+        const text = '<command>\nMY_SPECIAL_COMMAND\n</command>\nHello **World**\n```python\nprint("Hello World")\n```\n<command>\nMY_SPECIAL_COMMAND2\n</command>';
+        const result = (0, parse_contents_1.parseContents)(text, [response_content_matcher_1.CodeContentMatcher, exports.CommandContentMatcher]);
+        (0, chai_1.expect)(result).to.deep.equal([
+            new CommandChatResponseContentImpl('MY_SPECIAL_COMMAND'),
+            new chat_model_1.MarkdownChatResponseContentImpl('\nHello **World**\n'),
+            new chat_model_1.CodeChatResponseContentImpl('print("Hello World")', 'python'),
+            new CommandChatResponseContentImpl('MY_SPECIAL_COMMAND2'),
+        ]);
+    });
+});
+//# sourceMappingURL=parse-contents.spec.js.map

package/lib/common/parse-contents.spec.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parse-contents.spec.js","sourceRoot":"","sources":["../../src/common/parse-contents.spec.ts"],"names":[],"mappings":";AAAA,gFAAgF;AAChF,yCAAyC;AACzC,EAAE;AACF,2EAA2E;AAC3E,mEAAmE;AACnE,wCAAwC;AACxC,EAAE;AACF,4EAA4E;AAC5E,8EAA8E;AAC9E,6EAA6E;AAC7E,yDAAyD;AACzD,uDAAuD;AACvD,EAAE;AACF,gFAAgF;AAChF,gFAAgF;;;AAEhF,+BAA8B;AAC9B,6CAAiH;AACjH,qDAAiD;AACjD,yEAAwF;AAExF,MAAa,8BAA8B;IACvC,YAA4B,OAAe;QAAf,YAAO,GAAP,OAAO,CAAQ;QAC3C,SAAI,GAAG,SAAS,CAAC;IAD8B,CAAC;CAEnD;AAHD,wEAGC;AAEY,QAAA,qBAAqB,GAA2B;IACzD,KAAK,EAAE,cAAc;IACrB,GAAG,EAAE,gBAAgB;IACrB,cAAc,EAAE,CAAC,OAAe,EAAE,EAAE;QAChC,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,4BAA4B,EAAE,EAAE,CAAC,CAAC;QAC/D,OAAO,IAAI,8BAA8B,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IAC3D,CAAC;CACJ,CAAC;AAEF,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;IAC3B,EAAE,CAAC,2BAA2B,EAAE,GAAG,EAAE;QACjC,MAAM,IAAI,GAAG,iDAAiD,CAAC;QAC/D,MAAM,MAAM,GAAG,IAAA,8BAAa,EAAC,IAAI,CAAC,CAAC;QACnC,IAAA,aAAM,EAAC,MAAM,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,wCAA2B,CAAC,6BAA6B,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC;IACjH,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+BAA+B,EAAE,GAAG,EAAE;QACrC,MAAM,IAAI,GAAG,iBAAiB,CAAC;QAC/B,MAAM,MAAM,GAAG,IAAA,8BAAa,EAAC,IAAI,CAAC,CAAC;QACnC,IAAA,aAAM,EAAC,MAAM,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,4CAA+B,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC;IAC3F,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sCAAsC,EAAE,GAAG,EAAE;QAC5C,MAAM,IAAI,GAAG,kEAAkE,CAAC;QAChF,MAAM,MAAM,GAAG,IAAA,8BAAa,EAAC,IAAI,CAAC,CAAC;QACnC,IAAA,aAAM,EAAC,MAAM,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC;YACzB,IAAI,wCAA2B,CAAC,6BAA6B,EAAE,YAAY,CAAC;YAC5E,IAAI,4CAA+B,CAAC,mBAAmB,CAAC;SAC3D,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+DAA+D,EAAE,GAAG,EAAE;QACrE,MAAM,IAAI,GAAG,uFAAuF,CAAC;QACrG,MAAM,MAAM,GAAG,IAAA,8BAAa,EAAC,IAAI,CAAC,CAAC;QACnC,IAAA,aAAM,EAAC,MAAM,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC;YACzB,IAAI,wCAA2B,CAAC,6BAA6B,EAAE,YAAY,CAAC;YAC5E,IAAI,wCAA2B,CAAC,sBAAsB,EAAE,QAAQ,CAAC;SACpE,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4EAA4E,EAAE,GAAG,EAAE;QAClF,MAAM,IAAI,GAAG,wGAAwG,CAAC;QACtH,MAAM,MAAM,GAAG,IAAA,8BAAa,EAAC,IAAI,CAAC,CAAC;QACnC,IAAA,aAAM,EAAC,MAAM,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC;YACzB,IAAI,wCAA2B,CAAC,6BAA6B,EAAE,YAAY,CAAC;YAC5E,IAAI,4CAA+B,CAAC,qBAAqB,CAAC;YAC1D,IAAI,wCAA2B,CAAC,sBAAsB,EAAE,QAAQ,CAAC;SACpE,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;QACtD,MAAM,IAAI,GAAG,2EAA2E,CAAC;QACzF,MAAM,MAAM,GAAG,IAAA,8BAAa,EAAC,IAAI,CAAC,CAAC;QACnC,IAAA,aAAM,EAAC,MAAM,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC;YACzB,IAAI,wCAA2B,CAAC,EAAE,EAAE,YAAY,CAAC;YACjD,IAAI,4CAA+B,CAAC,qBAAqB,CAAC;YAC1D,IAAI,wCAA2B,CAAC,sBAAsB,EAAE,QAAQ,CAAC;SACpE,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wDAAwD,EAAE,GAAG,EAAE;QAC9D,MAAM,IAAI,GAAG,qFAAqF,CAAC;QACnG,MAAM,MAAM,GAAG,IAAA,8BAAa,EAAC,IAAI,CAAC,CAAC;QACnC,IAAA,aAAM,EAAC,MAAM,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC;YACzB,IAAI,4CAA+B,CAAC,mBAAmB,CAAC;YACxD,IAAI,wCAA2B,CAAC,6BAA6B,EAAE,YAAY,CAAC;YAC5E,IAAI,4CAA+B,CAAC,qBAAqB,CAAC;SAC7D,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4CAA4C,EAAE,GAAG,EAAE;QAClD,MAAM,IAAI,GAAG,uBAAuB,CAAC;QACrC,MAAM,MAAM,GAAG,IAAA,8BAAa,EAAC,IAAI,CAAC,CAAC;QACnC,IAAA,aAAM,EAAC,MAAM,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,4CAA+B,CAAC,uBAAuB,CAAC,CAAC,CAAC,CAAC;IACjG,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+CAA+C,EAAE,GAAG,EAAE;QACrD,MAAM,IAAI,GAAG,4CAA4C,CAAC;QAC1D,MAAM,MAAM,GAAG,IAAA,8BAAa,EAAC,IAAI,CAAC,CAAC;QACnC,IAAA,aAAM,EAAC,MAAM,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,4CAA+B,CAAC,4CAA4C,CAAC,CAAC,CAAC,CAAC;IACtH,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6CAA6C,EAAE,GAAG,EAAE;QACnD,MAAM,IAAI,GAAG,kCAAkC,CAAC;QAChD,MAAM,MAAM,GAAG,IAAA,8BAAa,EAAC,IAAI,CAAC,CAAC;QACnC,IAAA,aAAM,EAAC,MAAM,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,4CAA+B,CAAC,kCAAkC,CAAC,CAAC,CAAC,CAAC;IAC5G,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;QACpD,MAAM,IAAI,GAAG,kFAAkF,CAAC;QAChG,MAAM,MAAM,GAAG,IAAA,8BAAa,EAAC,IAAI,CAAC,CAAC;QACnC,IAAA,aAAM,EAAC,MAAM,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC;YACzB,IAAI,wCAA2B,CAAC,6BAA6B,EAAE,YAAY,CAAC;YAC5E,IAAI,4CAA+B,CAAC,mCAAmC,CAAC;SAC3E,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wDAAwD,EAAE,GAAG,EAAE;QAC9D,MAAM,IAAI,GAAG,8CAA8C,CAAC;QAC5D,MAAM,MAAM,GAAG,IAAA,8BAAa,EAAC,IAAI,CAAC,CAAC;QACnC,IAAA,aAAM,EAAC,MAAM,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC;YACzB,IAAI,4CAA+B,CAAC,8CAA8C,CAAC;SACtF,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sEAAsE,EAAE,GAAG,EAAE;QAC5E,MAAM,IAAI,GAAG,8IAA8I,CAAC;QAC5J,MAAM,MAAM,GAAG,IAAA,8BAAa,EAAC,IAAI,EAAE,CAAC,6CAAkB,EAAE,6BAAqB,CAAC,CAAC,CAAC;QAChF,IAAA,aAAM,EAAC,MAAM,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC;YACzB,IAAI,8BAA8B,CAAC,oBAAoB,CAAC;YACxD,IAAI,4CAA+B,CAAC,qBAAqB,CAAC;YAC1D,IAAI,wCAA2B,CAAC,sBAAsB,EAAE,QAAQ,CAAC;YACjE,IAAI,8BAA8B,CAAC,qBAAqB,CAAC;SAC5D,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;AACP,CAAC,CAAC,CAAC"}

package/lib/common/response-content-matcher.d.ts

@@ -0,0 +1,63 @@
+import { ChatResponseContent } from './chat-model';
+export type ResponseContentFactory = (content: string) => ChatResponseContent;
+export declare const MarkdownContentFactory: ResponseContentFactory;
+/**
+ * Default response content factory used if no other `ResponseContentMatcher` applies.
+ * By default, this factory creates a markdown content object.
+ *
+ * @see MarkdownChatResponseContentImpl
+ */
+export declare class DefaultResponseContentFactory {
+    create(content: string): ChatResponseContent;
+}
+/**
+ * Clients can contribute response content matchers to parse a chat response into specific
+ * `ChatResponseContent` instances.
+ */
+export interface ResponseContentMatcher {
+    /** Regular expression for finding the start delimiter. */
+    start: RegExp;
+    /** Regular expression for finding the start delimiter. */
+    end: RegExp;
+    /**
+     * The factory creating a response content from the matching content,
+     * from start index to end index of the match (including delimiters).
+     */
+    contentFactory: ResponseContentFactory;
+}
+export declare const CodeContentMatcher: ResponseContentMatcher;
+/**
+ * Clients can contribute response content matchers to parse the response content.
+ *
+ * The default chat user interface will collect all contributed matchers and use them
+ * to parse the response into structured content parts (e.g. code blocks, markdown blocks),
+ * which are then rendered with a `ChatResponsePartRenderer` registered for the respective
+ * content part type.
+ *
+ * ### Example
+ * ```ts
+ * bind(ResponseContentMatcherProvider).to(MyResponseContentMatcherProvider);
+ * ...
+ * @injectable()
+ * export class MyResponseContentMatcherProvider implements ResponseContentMatcherProvider {
+ *     readonly matchers: ResponseContentMatcher[] = [{
+ *       start: /^<command>$/m,
+ *       end: /^</command>$/m,
+ *       contentFactory: (content: string) => {
+ *         const command = content.replace(/^<command>\n|<\/command>$/g, '');
+ *         return new MyChatResponseContentImpl(command.trim());
+ *       }
+ *   }];
+ * }
+ * ```
+ *
+ * @see ResponseContentMatcher
+ */
+export declare const ResponseContentMatcherProvider: unique symbol;
+export interface ResponseContentMatcherProvider {
+    readonly matchers: ResponseContentMatcher[];
+}
+export declare class DefaultResponseContentMatcherProvider implements ResponseContentMatcherProvider {
+    readonly matchers: ResponseContentMatcher[];
+}
+//# sourceMappingURL=response-content-matcher.d.ts.map

package/lib/common/response-content-matcher.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"response-content-matcher.d.ts","sourceRoot":"","sources":["../../src/common/response-content-matcher.ts"],"names":[],"mappings":"AAeA,OAAO,EACH,mBAAmB,EAGtB,MAAM,cAAc,CAAC;AAGtB,MAAM,MAAM,sBAAsB,GAAG,CAAC,OAAO,EAAE,MAAM,KAAK,mBAAmB,CAAC;AAE9E,eAAO,MAAM,sBAAsB,EAAE,sBACW,CAAC;AAEjD;;;;;GAKG;AACH,qBACa,6BAA6B;IACtC,MAAM,CAAC,OAAO,EAAE,MAAM,GAAG,mBAAmB;CAG/C;AAED;;;GAGG;AACH,MAAM,WAAW,sBAAsB;IACnC,0DAA0D;IAC1D,KAAK,EAAE,MAAM,CAAC;IACd,0DAA0D;IAC1D,GAAG,EAAE,MAAM,CAAC;IACZ;;;OAGG;IACH,cAAc,EAAE,sBAAsB,CAAC;CAC1C;AAED,eAAO,MAAM,kBAAkB,EAAE,sBAQhC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,eAAO,MAAM,8BAA8B,eAA2C,CAAC;AACvF,MAAM,WAAW,8BAA8B;IAC3C,QAAQ,CAAC,QAAQ,EAAE,sBAAsB,EAAE,CAAC;CAC/C;AAED,qBACa,qCAAsC,YAAW,8BAA8B;IACxF,QAAQ,CAAC,QAAQ,EAAE,sBAAsB,EAAE,CAAwB;CACtE"}

package/lib/common/response-content-matcher.js

@@ -0,0 +1,86 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DefaultResponseContentMatcherProvider = exports.ResponseContentMatcherProvider = exports.CodeContentMatcher = exports.DefaultResponseContentFactory = exports.MarkdownContentFactory = void 0;
+const tslib_1 = require("tslib");
+/*
+ * Copyright (C) 2024 EclipseSource GmbH.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v. 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ *
+ * This Source Code may also be made available under the following Secondary
+ * Licenses when the conditions for such availability set forth in the Eclipse
+ * Public License v. 2.0 are satisfied: GNU General Public License, version 2
+ * with the GNU Classpath Exception which is available at
+ * https://www.gnu.org/software/classpath/license.html.
+ *
+ * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+ */
+const chat_model_1 = require("./chat-model");
+const inversify_1 = require("@theia/core/shared/inversify");
+const MarkdownContentFactory = (content) => new chat_model_1.MarkdownChatResponseContentImpl(content);
+exports.MarkdownContentFactory = MarkdownContentFactory;
+/**
+ * Default response content factory used if no other `ResponseContentMatcher` applies.
+ * By default, this factory creates a markdown content object.
+ *
+ * @see MarkdownChatResponseContentImpl
+ */
+let DefaultResponseContentFactory = class DefaultResponseContentFactory {
+    create(content) {
+        return (0, exports.MarkdownContentFactory)(content);
+    }
+};
+exports.DefaultResponseContentFactory = DefaultResponseContentFactory;
+exports.DefaultResponseContentFactory = DefaultResponseContentFactory = tslib_1.__decorate([
+    (0, inversify_1.injectable)()
+], DefaultResponseContentFactory);
+exports.CodeContentMatcher = {
+    start: /^```.*?$/m,
+    end: /^```$/m,
+    contentFactory: (content) => {
+        var _a;
+        const language = ((_a = content.match(/^```(\w+)/)) === null || _a === void 0 ? void 0 : _a[1]) || '';
+        const code = content.replace(/^```(\w+)\n|```$/g, '');
+        return new chat_model_1.CodeChatResponseContentImpl(code.trim(), language);
+    }
+};
+/**
+ * Clients can contribute response content matchers to parse the response content.
+ *
+ * The default chat user interface will collect all contributed matchers and use them
+ * to parse the response into structured content parts (e.g. code blocks, markdown blocks),
+ * which are then rendered with a `ChatResponsePartRenderer` registered for the respective
+ * content part type.
+ *
+ * ### Example
+ * ```ts
+ * bind(ResponseContentMatcherProvider).to(MyResponseContentMatcherProvider);
+ * ...
+ * @injectable()
+ * export class MyResponseContentMatcherProvider implements ResponseContentMatcherProvider {
+ *     readonly matchers: ResponseContentMatcher[] = [{
+ *       start: /^<command>$/m,
+ *       end: /^</command>$/m,
+ *       contentFactory: (content: string) => {
+ *         const command = content.replace(/^<command>\n|<\/command>$/g, '');
+ *         return new MyChatResponseContentImpl(command.trim());
+ *       }
+ *   }];
+ * }
+ * ```
+ *
+ * @see ResponseContentMatcher
+ */
+exports.ResponseContentMatcherProvider = Symbol('ResponseContentMatcherProvider');
+let DefaultResponseContentMatcherProvider = class DefaultResponseContentMatcherProvider {
+    constructor() {
+        this.matchers = [exports.CodeContentMatcher];
+    }
+};
+exports.DefaultResponseContentMatcherProvider = DefaultResponseContentMatcherProvider;
+exports.DefaultResponseContentMatcherProvider = DefaultResponseContentMatcherProvider = tslib_1.__decorate([
+    (0, inversify_1.injectable)()
+], DefaultResponseContentMatcherProvider);
+//# sourceMappingURL=response-content-matcher.js.map

package/lib/common/response-content-matcher.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"response-content-matcher.js","sourceRoot":"","sources":["../../src/common/response-content-matcher.ts"],"names":[],"mappings":";;;;AAAA;;;;;;;;;;;;;;GAcG;AACH,6CAIsB;AACtB,4DAA0D;AAInD,MAAM,sBAAsB,GAA2B,CAAC,OAAe,EAAE,EAAE,CAC9E,IAAI,4CAA+B,CAAC,OAAO,CAAC,CAAC;AADpC,QAAA,sBAAsB,0BACc;AAEjD;;;;;GAKG;AAEI,IAAM,6BAA6B,GAAnC,MAAM,6BAA6B;IACtC,MAAM,CAAC,OAAe;QAClB,OAAO,IAAA,8BAAsB,EAAC,OAAO,CAAC,CAAC;IAC3C,CAAC;CACJ,CAAA;AAJY,sEAA6B;wCAA7B,6BAA6B;IADzC,IAAA,sBAAU,GAAE;GACA,6BAA6B,CAIzC;AAkBY,QAAA,kBAAkB,GAA2B;IACtD,KAAK,EAAE,WAAW;IAClB,GAAG,EAAE,QAAQ;IACb,cAAc,EAAE,CAAC,OAAe,EAAE,EAAE;;QAChC,MAAM,QAAQ,GAAG,CAAA,MAAA,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,0CAAG,CAAC,CAAC,KAAI,EAAE,CAAC;QACvD,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,mBAAmB,EAAE,EAAE,CAAC,CAAC;QACtD,OAAO,IAAI,wCAA2B,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,QAAQ,CAAC,CAAC;IAClE,CAAC;CACJ,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACU,QAAA,8BAA8B,GAAG,MAAM,CAAC,gCAAgC,CAAC,CAAC;AAMhF,IAAM,qCAAqC,GAA3C,MAAM,qCAAqC;IAA3C;QACM,aAAQ,GAA6B,CAAC,0BAAkB,CAAC,CAAC;IACvE,CAAC;CAAA,CAAA;AAFY,sFAAqC;gDAArC,qCAAqC;IADjD,IAAA,sBAAU,GAAE;GACA,qCAAqC,CAEjD"}

package/package.json

@@ -1,13 +1,13 @@
 {
   "name": "@theia/ai-chat",
-  "version": "1.54.0",
+  "version": "1.55.0-next.14+d999fe031",
   "description": "Theia - AI Chat Extension",
   "dependencies": {
-    "@theia/ai-core": "1.54.0",
-    "@theia/ai-history": "1.54.0",
-    "@theia/core": "1.54.0",
-    "@theia/filesystem": "1.54.0",
-    "@theia/workspace": "1.54.0",
+    "@theia/ai-core": "1.55.0-next.14+d999fe031",
+    "@theia/ai-history": "1.55.0-next.14+d999fe031",
+    "@theia/core": "1.55.0-next.14+d999fe031",
+    "@theia/filesystem": "1.55.0-next.14+d999fe031",
+    "@theia/workspace": "1.55.0-next.14+d999fe031",
     "minimatch": "^5.1.0",
     "tslib": "^2.6.2"
   },
@@ -50,5 +50,5 @@
   "nyc": {
     "extends": "../../configs/nyc.json"
   },
-  "gitHead": "8fb36a237db744cff6e78eaff1481e1f36bb7a69"
+  "gitHead": "d999fe0315a29bc5b154067621dbc1eb180fa567"
 }

package/src/browser/ai-chat-frontend-module.ts

@@ -33,6 +33,7 @@ import { UniversalChatAgent } from '../common/universal-chat-agent';
 import { aiChatPreferences } from './ai-chat-preferences';
 import { ChatAgentsVariableContribution } from '../common/chat-agents-variable-contribution';
 import { FrontendChatServiceImpl } from './frontend-chat-service';
+import { DefaultResponseContentMatcherProvider, DefaultResponseContentFactory, ResponseContentMatcherProvider } from '../common/response-content-matcher';
 
 export default new ContainerModule(bind => {
     bindContributionProvider(bind, Agent);
@@ -42,6 +43,11 @@ export default new ContainerModule(bind => {
     bind(ChatAgentService).toService(ChatAgentServiceImpl);
     bind(DefaultChatAgentId).toConstantValue({ id: OrchestratorChatAgentId });
 
+    bindContributionProvider(bind, ResponseContentMatcherProvider);
+    bind(DefaultResponseContentMatcherProvider).toSelf().inSingletonScope();
+    bind(ResponseContentMatcherProvider).toService(DefaultResponseContentMatcherProvider);
+    bind(DefaultResponseContentFactory).toSelf().inSingletonScope();
+
     bind(AIVariableContribution).to(ChatAgentsVariableContribution).inSingletonScope();
 
     bind(ChatRequestParserImpl).toSelf().inSingletonScope();

package/src/common/chat-agents.ts

@@ -25,6 +25,7 @@ import {
     LanguageModel,
     LanguageModelRequirement,
     LanguageModelResponse,
+    LanguageModelStreamResponse,
     PromptService,
     ResolvedPromptTemplate,
     ToolRequest,
@@ -37,19 +38,20 @@ import {
     LanguageModelStreamResponsePart,
     MessageActor,
 } from '@theia/ai-core/lib/common';
-import { CancellationToken, CancellationTokenSource, ILogger, isArray } from '@theia/core';
-import { inject, injectable } from '@theia/core/shared/inversify';
+import { CancellationToken, CancellationTokenSource, ContributionProvider, ILogger, isArray } from '@theia/core';
+import { inject, injectable, named, postConstruct } from '@theia/core/shared/inversify';
 import { ChatAgentService } from './chat-agent-service';
 import {
     ChatModel,
     ChatRequestModel,
     ChatRequestModelImpl,
     ChatResponseContent,
-    CodeChatResponseContentImpl,
     ErrorChatResponseContentImpl,
     MarkdownChatResponseContentImpl,
     ToolCallChatResponseContentImpl
 } from './chat-model';
+import { findFirstMatch, parseContents } from './parse-contents';
+import { DefaultResponseContentFactory, ResponseContentMatcher, ResponseContentMatcherProvider } from './response-content-matcher';
 
 /**
  * A conversation consists of a sequence of ChatMessages.
@@ -121,6 +123,14 @@ export abstract class AbstractChatAgent {
     @inject(ILogger) protected logger: ILogger;
     @inject(CommunicationRecordingService) protected recordingService: CommunicationRecordingService;
     @inject(PromptService) protected promptService: PromptService;
+
+    @inject(ContributionProvider) @named(ResponseContentMatcherProvider)
+    protected contentMatcherProviders: ContributionProvider<ResponseContentMatcherProvider>;
+    protected contentMatchers: ResponseContentMatcher[] = [];
+
+    @inject(DefaultResponseContentFactory)
+    protected defaultContentFactory: DefaultResponseContentFactory;
+
     constructor(
         public id: string,
         public languageModelRequirements: LanguageModelRequirement[],
@@ -130,6 +140,11 @@ export abstract class AbstractChatAgent {
         public tags: String[] = ['Chat']) {
     }
 
+    @postConstruct()
+    init(): void {
+        this.contentMatchers = this.contentMatcherProviders.getContributions().flatMap(provider => provider.matchers);
+    }
+
     async invoke(request: ChatRequestModelImpl): Promise<void> {
         try {
             const languageModel = await this.getLanguageModel(this.defaultLanguageModelPurpose);
@@ -189,6 +204,14 @@ export abstract class AbstractChatAgent {
         }
     }
 
+    protected parseContents(text: string): ChatResponseContent[] {
+        return parseContents(
+            text,
+            this.contentMatchers,
+            this.defaultContentFactory?.create.bind(this.defaultContentFactory)
+        );
+    };
+
     protected handleError(request: ChatRequestModelImpl, error: Error): void {
         request.response.response.addContent(new ErrorChatResponseContentImpl(error));
         request.response.error(error);
@@ -281,9 +304,8 @@ export abstract class AbstractStreamParsingChatAgent extends AbstractChatAgent {
 
     protected override async addContentsToResponse(languageModelResponse: LanguageModelResponse, request: ChatRequestModelImpl): Promise<void> {
         if (isLanguageModelTextResponse(languageModelResponse)) {
-            request.response.response.addContent(
-                new MarkdownChatResponseContentImpl(languageModelResponse.text)
-            );
+            const contents = this.parseContents(languageModelResponse.text);
+            request.response.response.addContents(contents);
             request.response.complete();
             this.recordingService.recordResponse({
                 agentId: this.id,
@@ -295,57 +317,7 @@ export abstract class AbstractStreamParsingChatAgent extends AbstractChatAgent {
             return;
         }
         if (isLanguageModelStreamResponse(languageModelResponse)) {
-            for await (const token of languageModelResponse.stream) {
-                const newContents = this.parse(token, request.response.response.content);
-                if (isArray(newContents)) {
-                    newContents.forEach(newContent => request.response.response.addContent(newContent));
-                } else {
-                    request.response.response.addContent(newContents);
-                }
-
-                const lastContent = request.response.response.content.pop();
-                if (lastContent === undefined) {
-                    return;
-                }
-                const text = lastContent.asString?.();
-                if (text === undefined) {
-                    return;
-                }
-                let curSearchIndex = 0;
-                const result: ChatResponseContent[] = [];
-                while (curSearchIndex < text.length) {
-                    // find start of code block: ```[language]\n<code>[\n]```
-                    const codeStartIndex = text.indexOf('```', curSearchIndex);
-                    if (codeStartIndex === -1) {
-                        break;
-                    }
-
-                    // find language specifier if present
-                    const newLineIndex = text.indexOf('\n', codeStartIndex + 3);
-                    const language = codeStartIndex + 3 < newLineIndex ? text.substring(codeStartIndex + 3, newLineIndex) : undefined;
-
-                    // find end of code block
-                    const codeEndIndex = text.indexOf('```', codeStartIndex + 3);
-                    if (codeEndIndex === -1) {
-                        break;
-                    }
-
-                    // add text before code block as markdown content
-                    result.push(new MarkdownChatResponseContentImpl(text.substring(curSearchIndex, codeStartIndex)));
-                    // add code block as code content
-                    const codeText = text.substring(newLineIndex + 1, codeEndIndex).trimEnd();
-                    result.push(new CodeChatResponseContentImpl(codeText, language));
-                    curSearchIndex = codeEndIndex + 3;
-                }
-
-                if (result.length > 0) {
-                    result.forEach(r => {
-                        request.response.response.addContent(r);
-                    });
-                } else {
-                    request.response.response.addContent(lastContent);
-                }
-            }
+            await this.addStreamResponse(languageModelResponse, request);
             request.response.complete();
             this.recordingService.recordResponse({
                 agentId: this.id,
@@ -366,11 +338,38 @@ export abstract class AbstractStreamParsingChatAgent extends AbstractChatAgent {
         );
     }
 
-    private parse(token: LanguageModelStreamResponsePart, previousContent: ChatResponseContent[]): ChatResponseContent | ChatResponseContent[] {
+    protected async addStreamResponse(languageModelResponse: LanguageModelStreamResponse, request: ChatRequestModelImpl): Promise<void> {
+        for await (const token of languageModelResponse.stream) {
+            const newContents = this.parse(token, request.response.response.content);
+            if (isArray(newContents)) {
+                request.response.response.addContents(newContents);
+            } else {
+                request.response.response.addContent(newContents);
+            }
+
+            const lastContent = request.response.response.content.pop();
+            if (lastContent === undefined) {
+                return;
+            }
+            const text = lastContent.asString?.();
+            if (text === undefined) {
+                return;
+            }
+
+            const result: ChatResponseContent[] = findFirstMatch(this.contentMatchers, text) ? this.parseContents(text) : [];
+            if (result.length > 0) {
+                request.response.response.addContents(result);
+            } else {
+                request.response.response.addContent(lastContent);
+            }
+        }
+    }
+
+    protected parse(token: LanguageModelStreamResponsePart, previousContent: ChatResponseContent[]): ChatResponseContent | ChatResponseContent[] {
         const content = token.content;
         // eslint-disable-next-line no-null/no-null
         if (content !== undefined && content !== null) {
-            return new MarkdownChatResponseContentImpl(content);
+            return this.defaultContentFactory.create(content);
         }
         const toolCalls = token.tool_calls;
         if (toolCalls !== undefined) {
@@ -378,7 +377,7 @@ export abstract class AbstractStreamParsingChatAgent extends AbstractChatAgent {
                 new ToolCallChatResponseContentImpl(toolCall.id, toolCall.function?.name, toolCall.function?.arguments, toolCall.finished, toolCall.result));
             return toolCallContents;
         }
-        return new MarkdownChatResponseContentImpl('');
+        return this.defaultContentFactory.create('');
     }
 
 }

package/src/common/chat-model.ts

@@ -601,10 +601,20 @@ class ChatResponseImpl implements ChatResponse {
         return this._content;
     }
 
+    addContents(contents: ChatResponseContent[]): void {
+        contents.forEach(c => this.doAddContent(c));
+        this._onDidChangeEmitter.fire();
+    }
+
     addContent(nextContent: ChatResponseContent): void {
         // TODO: Support more complex merges affecting different content than the last, e.g. via some kind of ProcessorRegistry
         // TODO: Support more of the built-in VS Code behavior, see
         //   https://github.com/microsoft/vscode/blob/a2cab7255c0df424027be05d58e1b7b941f4ea60/src/vs/workbench/contrib/chat/common/chatModel.ts#L188-L244
+        this.doAddContent(nextContent);
+        this._onDidChangeEmitter.fire();
+    }
+
+    protected doAddContent(nextContent: ChatResponseContent): void {
         if (ToolCallChatResponseContent.is(nextContent) && nextContent.id !== undefined) {
             const fittingTool = this._content.find(c => ToolCallChatResponseContent.is(c) && c.id === nextContent.id);
             if (fittingTool !== undefined) {
@@ -613,10 +623,9 @@ class ChatResponseImpl implements ChatResponse {
                 this._content.push(nextContent);
             }
         } else {
-            const lastElement =
-                this._content.length > 0
-                    ? this._content[this._content.length - 1]
-                    : undefined;
+            const lastElement = this._content.length > 0
+                ? this._content[this._content.length - 1]
+                : undefined;
             if (lastElement?.kind === nextContent.kind && ChatResponseContent.hasMerge(lastElement)) {
                 const mergeSuccess = lastElement.merge(nextContent);
                 if (!mergeSuccess) {
@@ -627,7 +636,6 @@ class ChatResponseImpl implements ChatResponse {
             }
         }
         this._updateResponseRepresentation();
-        this._onDidChangeEmitter.fire();
     }
 
     protected _updateResponseRepresentation(): void {