@codingame/monaco-vscode-chat-service-override 18.4.0 → 19.0.0

package/vscode/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/tokens/markdownImage.js

@@ -0,0 +1,54 @@
+
+import { MarkdownToken } from './markdownToken.js';
+import { Range } from '@codingame/monaco-vscode-api/vscode/vs/editor/common/core/range';
+import { assert } from '@codingame/monaco-vscode-api/vscode/vs/base/common/assert';
+
+class MarkdownImage extends MarkdownToken {
+    constructor(
+    lineNumber,
+    columnNumber,
+    caption,
+    reference) {
+        assert(!isNaN(lineNumber), `The line number must not be a NaN.`);
+        assert(lineNumber > 0, `The line number must be >= 1, got "${lineNumber}".`);
+        assert(columnNumber > 0, `The column number must be >= 1, got "${columnNumber}".`);
+        assert(caption[0] === '!', `The caption must start with '!' character, got "${caption}".`);
+        assert(caption[1] === '[' && caption[caption.length - 1] === ']', `The caption must be enclosed in square brackets, got "${caption}".`);
+        assert(reference[0] === '(' && reference[reference.length - 1] === ')', `The reference must be enclosed in parentheses, got "${reference}".`);
+        super(( new Range(
+         lineNumber,
+         columnNumber,
+         lineNumber,
+         columnNumber + caption.length + reference.length
+        )));
+        this.caption = caption;
+        this.reference = reference;
+        try {
+            ( new URL(this.path));
+            this.isURL = true;
+        }
+        catch {
+            this.isURL = false;
+        }
+    }
+    get text() {
+        return `${this.caption}${this.reference}`;
+    }
+    get path() {
+        return this.reference.slice(1, this.reference.length - 1);
+    }
+    get linkRange() {
+        if (this.path.length === 0) {
+            return undefined;
+        }
+        const { range } = this;
+        const startColumn = range.startColumn + this.caption.length + 1;
+        const endColumn = startColumn + this.path.length;
+        return ( new Range(range.startLineNumber, startColumn, range.endLineNumber, endColumn));
+    }
+    toString() {
+        return `md-image("${this.shortText()}")${this.range}`;
+    }
+}
+
+export { MarkdownImage };

package/vscode/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/tokens/markdownLink.d.ts

@@ -0,0 +1,12 @@
+import { MarkdownToken } from "./markdownToken.js";
+import { IRange } from "@codingame/monaco-vscode-api/vscode/vs/editor/common/core/range";
+export declare class MarkdownLink extends MarkdownToken {
+    readonly caption: string;
+    readonly reference: string;
+    readonly isURL: boolean;
+    constructor(lineNumber: number, columnNumber: number, caption: string, reference: string);
+    get text(): string;
+    get path(): string;
+    get linkRange(): IRange | undefined;
+    toString(): string;
+}

package/vscode/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/tokens/markdownLink.js

@@ -0,0 +1,53 @@
+
+import { MarkdownToken } from './markdownToken.js';
+import { Range } from '@codingame/monaco-vscode-api/vscode/vs/editor/common/core/range';
+import { assert } from '@codingame/monaco-vscode-api/vscode/vs/base/common/assert';
+
+class MarkdownLink extends MarkdownToken {
+    constructor(
+    lineNumber,
+    columnNumber,
+    caption,
+    reference) {
+        assert(!isNaN(lineNumber), `The line number must not be a NaN.`);
+        assert(lineNumber > 0, `The line number must be >= 1, got "${lineNumber}".`);
+        assert(columnNumber > 0, `The column number must be >= 1, got "${columnNumber}".`);
+        assert(caption[0] === '[' && caption[caption.length - 1] === ']', `The caption must be enclosed in square brackets, got "${caption}".`);
+        assert(reference[0] === '(' && reference[reference.length - 1] === ')', `The reference must be enclosed in parentheses, got "${reference}".`);
+        super(( new Range(
+         lineNumber,
+         columnNumber,
+         lineNumber,
+         columnNumber + caption.length + reference.length
+        )));
+        this.caption = caption;
+        this.reference = reference;
+        try {
+            ( new URL(this.path));
+            this.isURL = true;
+        }
+        catch {
+            this.isURL = false;
+        }
+    }
+    get text() {
+        return `${this.caption}${this.reference}`;
+    }
+    get path() {
+        return this.reference.slice(1, this.reference.length - 1);
+    }
+    get linkRange() {
+        if (this.path.length === 0) {
+            return undefined;
+        }
+        const { range } = this;
+        const startColumn = range.startColumn + this.caption.length + 1;
+        const endColumn = startColumn + this.path.length;
+        return ( new Range(range.startLineNumber, startColumn, range.endLineNumber, endColumn));
+    }
+    toString() {
+        return `md-link("${this.shortText()}")${this.range}`;
+    }
+}
+
+export { MarkdownLink };

package/vscode/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/tokens/markdownToken.d.ts

@@ -0,0 +1,3 @@
+import { BaseToken } from "../../baseToken.js";
+export declare abstract class MarkdownToken extends BaseToken {
+}

package/vscode/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/tokens/markdownToken.js

@@ -0,0 +1,7 @@
+
+import { BaseToken } from '../../baseToken.js';
+
+class MarkdownToken extends BaseToken {
+}
+
+export { MarkdownToken };

package/vscode/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/markdownExtensionsDecoder.d.ts

@@ -0,0 +1,13 @@
+import { VSBuffer } from "@codingame/monaco-vscode-api/vscode/vs/base/common/buffer";
+import { ReadableStream } from "@codingame/monaco-vscode-api/vscode/vs/base/common/stream";
+import { BaseDecoder } from "../baseDecoder.js";
+import { MarkdownExtensionsToken } from "./tokens/markdownExtensionsToken.js";
+import { TSimpleDecoderToken } from "../simpleCodec/simpleDecoder.js";
+export type TMarkdownExtensionsToken = MarkdownExtensionsToken | TSimpleDecoderToken;
+export declare class MarkdownExtensionsDecoder extends BaseDecoder<TMarkdownExtensionsToken, TSimpleDecoderToken> {
+    private current?;
+    constructor(stream: ReadableStream<VSBuffer>);
+    protected onStreamData(token: TSimpleDecoderToken): void;
+    protected onStreamEnd(): void;
+    protected reEmitCurrentTokens(): void;
+}

package/vscode/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/markdownExtensionsDecoder.js

@@ -0,0 +1,68 @@
+
+import { BaseDecoder } from '../baseDecoder.js';
+import { MarkdownExtensionsToken } from './tokens/markdownExtensionsToken.js';
+import { SimpleDecoder } from '../simpleCodec/simpleDecoder.js';
+import { PartialFrontMatterStartMarker, PartialFrontMatterHeader } from './parsers/frontMatterHeader.js';
+
+class MarkdownExtensionsDecoder extends BaseDecoder {
+    constructor(stream) {
+        super(( new SimpleDecoder(stream)));
+    }
+    onStreamData(token) {
+        if ((this.current === undefined) && PartialFrontMatterStartMarker.mayStartHeader(token)) {
+            this.current = ( new PartialFrontMatterStartMarker(token));
+            return;
+        }
+        if (this.current === undefined) {
+            this._onData.fire(token);
+            return;
+        }
+        const parseResult = this.current.accept(token);
+        if (parseResult.result === 'success') {
+            const { nextParser } = parseResult;
+            if (nextParser instanceof MarkdownExtensionsToken) {
+                this._onData.fire(nextParser);
+                delete this.current;
+            }
+            else {
+                this.current = nextParser;
+            }
+        }
+        else {
+            this.reEmitCurrentTokens();
+        }
+        if (!parseResult.wasTokenConsumed) {
+            this.onStreamData(token);
+        }
+    }
+    onStreamEnd() {
+        try {
+            if (this.current === undefined) {
+                return;
+            }
+            if (this.current instanceof PartialFrontMatterHeader) {
+                this._onData.fire(this.current.asFrontMatterHeader());
+                delete this.current;
+                return;
+            }
+        }
+        catch {
+            this.reEmitCurrentTokens();
+        }
+        finally {
+            delete this.current;
+            super.onStreamEnd();
+        }
+    }
+    reEmitCurrentTokens() {
+        if (this.current === undefined) {
+            return;
+        }
+        for (const token of this.current.tokens) {
+            this._onData.fire(token);
+        }
+        delete this.current;
+    }
+}
+
+export { MarkdownExtensionsDecoder };

package/vscode/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/parsers/frontMatterHeader.d.ts

@@ -0,0 +1,20 @@
+import { Dash } from "../../simpleCodec/tokens/dash.js";
+import { FrontMatterHeader } from "../tokens/frontMatterHeader.js";
+import { TSimpleDecoderToken } from "../../simpleCodec/simpleDecoder.js";
+import { FrontMatterMarker, TMarkerToken } from "../tokens/frontMatterMarker.js";
+import { ParserBase, TAcceptTokenResult } from "../../simpleCodec/parserBase.js";
+export declare class PartialFrontMatterStartMarker extends ParserBase<TMarkerToken, PartialFrontMatterStartMarker | PartialFrontMatterHeader> {
+    constructor(token: Dash);
+    accept(token: TSimpleDecoderToken): TAcceptTokenResult<PartialFrontMatterStartMarker | PartialFrontMatterHeader>;
+    static mayStartHeader(token: TSimpleDecoderToken): token is Dash;
+}
+export declare class PartialFrontMatterHeader extends ParserBase<TSimpleDecoderToken, PartialFrontMatterHeader | FrontMatterHeader> {
+    readonly startMarker: FrontMatterMarker;
+    private maybeEndMarker?;
+    constructor(startMarker: FrontMatterMarker);
+    get tokens(): readonly TSimpleDecoderToken[];
+    asFrontMatterHeader(): FrontMatterHeader;
+    accept(token: TSimpleDecoderToken): TAcceptTokenResult<PartialFrontMatterHeader | FrontMatterHeader>;
+    private acceptEndMarkerToken;
+    private handleEndMarkerParsingFailure;
+}

package/vscode/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/parsers/frontMatterHeader.js

@@ -0,0 +1,199 @@
+
+import { __decorate } from '@codingame/monaco-vscode-api/external/tslib/tslib.es6';
+import { Dash } from '../../simpleCodec/tokens/dash.js';
+import { NewLine } from '../../linesCodec/tokens/newLine.js';
+import { FrontMatterHeader } from '../tokens/frontMatterHeader.js';
+import { assertDefined } from '@codingame/monaco-vscode-api/vscode/vs/base/common/types';
+import { assert, assertNever } from '@codingame/monaco-vscode-api/vscode/vs/base/common/assert';
+import { CarriageReturn } from '../../linesCodec/tokens/carriageReturn.js';
+import { FrontMatterMarker } from '../tokens/frontMatterMarker.js';
+import { ParserBase, assertNotConsumed } from '../../simpleCodec/parserBase.js';
+
+class PartialFrontMatterStartMarker extends ParserBase {
+    constructor(token) {
+        const { range } = token;
+        assert(range.startLineNumber === 1, `Front Matter header must start at the first line, but it starts at line #${range.startLineNumber}.`);
+        assert(range.startColumn === 1, `Front Matter header must start at the beginning of the line, but it starts at ${range.startColumn}.`);
+        super([token]);
+    }
+    accept(token) {
+        const previousToken = this.currentTokens[this.currentTokens.length - 1];
+        if ((token instanceof Dash) || (token instanceof CarriageReturn)) {
+            if ((previousToken instanceof Dash) === false) {
+                this.isConsumed = true;
+                return {
+                    result: 'failure',
+                    wasTokenConsumed: false,
+                };
+            }
+            this.currentTokens.push(token);
+            return {
+                result: 'success',
+                wasTokenConsumed: true,
+                nextParser: this,
+            };
+        }
+        if (token instanceof NewLine) {
+            this.isConsumed = true;
+            return {
+                result: 'success',
+                wasTokenConsumed: true,
+                nextParser: ( new PartialFrontMatterHeader(FrontMatterMarker.fromTokens([
+                    ...this.currentTokens,
+                    token,
+                ]))),
+            };
+        }
+        this.isConsumed = true;
+        return {
+            result: 'failure',
+            wasTokenConsumed: false,
+        };
+    }
+    static mayStartHeader(token) {
+        return (token instanceof Dash)
+            && (token.range.startLineNumber === 1)
+            && (token.range.startColumn === 1);
+    }
+}
+PartialFrontMatterStartMarker.__decorator = ( __decorate([
+    assertNotConsumed
+], PartialFrontMatterStartMarker.prototype, "accept", null));
+class PartialFrontMatterHeader extends ParserBase {
+    constructor(startMarker) {
+        super([]);
+        this.startMarker = startMarker;
+    }
+    get tokens() {
+        const endMarkerTokens = (this.maybeEndMarker !== undefined)
+            ? this.maybeEndMarker.tokens
+            : [];
+        return [
+            ...this.startMarker.tokens,
+            ...this.currentTokens,
+            ...endMarkerTokens,
+        ];
+    }
+    asFrontMatterHeader() {
+        assertDefined(this.maybeEndMarker, 'Cannot convert to Front Matter header token without an end marker.');
+        assert(this.maybeEndMarker.dashCount === this.startMarker.dashTokens.length, [
+            'Start and end markers must have the same number of dashes',
+            `, got ${this.startMarker.dashTokens.length} / ${this.maybeEndMarker.dashCount}.`,
+        ].join(''));
+        this.isConsumed = true;
+        return FrontMatterHeader.fromTokens(this.startMarker.tokens, this.currentTokens, this.maybeEndMarker.tokens);
+    }
+    accept(token) {
+        if (this.maybeEndMarker !== undefined) {
+            return this.acceptEndMarkerToken(token);
+        }
+        if (((token instanceof Dash) === false) || (token.range.startColumn !== 1)) {
+            this.currentTokens.push(token);
+            return {
+                result: 'success',
+                wasTokenConsumed: true,
+                nextParser: this,
+            };
+        }
+        assert(this.maybeEndMarker === undefined, `End marker parser must not be present.`);
+        this.maybeEndMarker = ( new PartialFrontMatterEndMarker(token));
+        return {
+            result: 'success',
+            wasTokenConsumed: true,
+            nextParser: this,
+        };
+    }
+    acceptEndMarkerToken(token) {
+        assertDefined(this.maybeEndMarker, `Partial end marker parser must be initialized.`);
+        const acceptResult = this.maybeEndMarker.accept(token);
+        const { result, wasTokenConsumed } = acceptResult;
+        if (result === 'success') {
+            const { nextParser } = acceptResult;
+            const endMarkerParsingComplete = (nextParser instanceof FrontMatterMarker);
+            if (endMarkerParsingComplete === false) {
+                return {
+                    result: 'success',
+                    wasTokenConsumed,
+                    nextParser: this,
+                };
+            }
+            const endMarker = nextParser;
+            if (endMarker.dashTokens.length !== this.startMarker.dashTokens.length) {
+                return this.handleEndMarkerParsingFailure(endMarker.tokens, wasTokenConsumed);
+            }
+            this.isConsumed = true;
+            return {
+                result: 'success',
+                wasTokenConsumed: true,
+                nextParser: FrontMatterHeader.fromTokens(this.startMarker.tokens, this.currentTokens, this.maybeEndMarker.tokens),
+            };
+        }
+        if (result === 'failure') {
+            return this.handleEndMarkerParsingFailure(this.maybeEndMarker.tokens, wasTokenConsumed);
+        }
+        assertNever(result, `Unexpected result '${result}' while parsing the end marker.`);
+    }
+    handleEndMarkerParsingFailure(tokens, wasTokenConsumed) {
+        this.currentTokens.push(...tokens);
+        delete this.maybeEndMarker;
+        return {
+            result: 'success',
+            wasTokenConsumed,
+            nextParser: this,
+        };
+    }
+}
+PartialFrontMatterHeader.__decorator = ( __decorate([
+    assertNotConsumed
+], PartialFrontMatterHeader.prototype, "accept", null));
+class PartialFrontMatterEndMarker extends ParserBase {
+    constructor(token) {
+        const { range } = token;
+        assert(range.startColumn === 1, `Front Matter header must start at the beginning of the line, but it starts at ${range.startColumn}.`);
+        super([token]);
+    }
+    get dashCount() {
+        return this.tokens
+            .filter((token) => { return token instanceof Dash; })
+            .length;
+    }
+    accept(token) {
+        const previousToken = this.currentTokens[this.currentTokens.length - 1];
+        if ((token instanceof Dash) || (token instanceof CarriageReturn)) {
+            if ((previousToken instanceof Dash) === false) {
+                this.isConsumed = true;
+                return {
+                    result: 'failure',
+                    wasTokenConsumed: false,
+                };
+            }
+            this.currentTokens.push(token);
+            return {
+                result: 'success',
+                wasTokenConsumed: true,
+                nextParser: this,
+            };
+        }
+        if (token instanceof NewLine) {
+            this.isConsumed = true;
+            this.currentTokens.push(token);
+            return {
+                result: 'success',
+                wasTokenConsumed: true,
+                nextParser: FrontMatterMarker.fromTokens([
+                    ...this.currentTokens,
+                ]),
+            };
+        }
+        this.isConsumed = true;
+        return {
+            result: 'failure',
+            wasTokenConsumed: false,
+        };
+    }
+}
+PartialFrontMatterEndMarker.__decorator = ( __decorate([
+    assertNotConsumed
+], PartialFrontMatterEndMarker.prototype, "accept", null));
+
+export { PartialFrontMatterHeader, PartialFrontMatterStartMarker };

package/vscode/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/tokens/frontMatterHeader.d.ts

@@ -0,0 +1,16 @@
+import { Text } from "../../textToken.js";
+import { Range } from "@codingame/monaco-vscode-api/vscode/vs/editor/common/core/range";
+import { MarkdownExtensionsToken } from "./markdownExtensionsToken.js";
+import { TSimpleDecoderToken } from "../../simpleCodec/simpleDecoder.js";
+import { FrontMatterMarker, TMarkerToken } from "./frontMatterMarker.js";
+export declare class FrontMatterHeader extends MarkdownExtensionsToken {
+    readonly startMarker: FrontMatterMarker;
+    readonly content: Text;
+    readonly endMarker: FrontMatterMarker;
+    constructor(range: Range, startMarker: FrontMatterMarker, content: Text, endMarker: FrontMatterMarker);
+    get text(): string;
+    get contentRange(): Range;
+    get contentToken(): Text;
+    static fromTokens(startMarkerTokens: readonly TMarkerToken[], contentTokens: readonly TSimpleDecoderToken[], endMarkerTokens: readonly TMarkerToken[]): FrontMatterHeader;
+    toString(): string;
+}

package/vscode/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/tokens/frontMatterHeader.js

@@ -0,0 +1,37 @@
+
+import { Text } from '../../textToken.js';
+import { BaseToken } from '../../baseToken.js';
+import { MarkdownExtensionsToken } from './markdownExtensionsToken.js';
+import { FrontMatterMarker } from './frontMatterMarker.js';
+
+class FrontMatterHeader extends MarkdownExtensionsToken {
+    constructor(range, startMarker, content, endMarker) {
+        super(range);
+        this.startMarker = startMarker;
+        this.content = content;
+        this.endMarker = endMarker;
+    }
+    get text() {
+        const text = [
+            this.startMarker.text,
+            this.content.text,
+            this.endMarker.text,
+        ];
+        return text.join('');
+    }
+    get contentRange() {
+        return this.content.range;
+    }
+    get contentToken() {
+        return this.content;
+    }
+    static fromTokens(startMarkerTokens, contentTokens, endMarkerTokens) {
+        const range = BaseToken.fullRange([...startMarkerTokens, ...endMarkerTokens]);
+        return ( new FrontMatterHeader(range, FrontMatterMarker.fromTokens(startMarkerTokens), ( new Text(contentTokens)), FrontMatterMarker.fromTokens(endMarkerTokens)));
+    }
+    toString() {
+        return `frontmatter("${this.shortText()}")${this.range}`;
+    }
+}
+
+export { FrontMatterHeader };

package/vscode/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/tokens/frontMatterMarker.d.ts

@@ -0,0 +1,14 @@
+import { Range } from "@codingame/monaco-vscode-api/vscode/vs/editor/common/core/range";
+import { Dash } from "../../simpleCodec/tokens/dash.js";
+import { NewLine } from "../../linesCodec/tokens/newLine.js";
+import { MarkdownExtensionsToken } from "./markdownExtensionsToken.js";
+import { CarriageReturn } from "../../linesCodec/tokens/carriageReturn.js";
+export type TMarkerToken = Dash | CarriageReturn | NewLine;
+export declare class FrontMatterMarker extends MarkdownExtensionsToken {
+    readonly tokens: readonly TMarkerToken[];
+    get text(): string;
+    get dashTokens(): readonly Dash[];
+    constructor(range: Range, tokens: readonly TMarkerToken[]);
+    static fromTokens(tokens: readonly TMarkerToken[]): FrontMatterMarker;
+    toString(): string;
+}

package/vscode/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/tokens/frontMatterMarker.js

@@ -0,0 +1,27 @@
+
+import { BaseToken } from '../../baseToken.js';
+import { Dash } from '../../simpleCodec/tokens/dash.js';
+import { MarkdownExtensionsToken } from './markdownExtensionsToken.js';
+
+class FrontMatterMarker extends MarkdownExtensionsToken {
+    get text() {
+        return BaseToken.render(this.tokens);
+    }
+    get dashTokens() {
+        return this.tokens
+            .filter((token) => { return token instanceof Dash; });
+    }
+    constructor(range, tokens) {
+        super(range);
+        this.tokens = tokens;
+    }
+    static fromTokens(tokens) {
+        const range = BaseToken.fullRange(tokens);
+        return ( new FrontMatterMarker(range, tokens));
+    }
+    toString() {
+        return `frontmatter-marker(${this.dashTokens.length}:${this.range})`;
+    }
+}
+
+export { FrontMatterMarker };

package/vscode/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/tokens/markdownExtensionsToken.d.ts

@@ -0,0 +1,3 @@
+import { MarkdownToken } from "../../markdownCodec/tokens/markdownToken.js";
+export declare abstract class MarkdownExtensionsToken extends MarkdownToken {
+}

package/vscode/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/tokens/markdownExtensionsToken.js

@@ -0,0 +1,7 @@
+
+import { MarkdownToken } from '../../markdownCodec/tokens/markdownToken.js';
+
+class MarkdownExtensionsToken extends MarkdownToken {
+}
+
+export { MarkdownExtensionsToken };

package/vscode/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/parserBase.d.ts

@@ -0,0 +1,24 @@
+import { BaseToken } from "../baseToken.js";
+export interface IAcceptTokenResult {
+    result: "success" | "failure";
+    wasTokenConsumed: boolean;
+}
+export interface IAcceptTokenSuccess<T> extends IAcceptTokenResult {
+    result: "success";
+    nextParser: T;
+}
+export interface IAcceptTokenFailure extends IAcceptTokenResult {
+    result: "failure";
+}
+export type TAcceptTokenResult<T> = IAcceptTokenSuccess<T> | IAcceptTokenFailure;
+export declare abstract class ParserBase<TToken extends BaseToken, TNextObject> {
+    protected readonly currentTokens: TToken[];
+    protected isConsumed: boolean;
+    get consumed(): boolean;
+    protected readonly startTokensCount: number;
+    constructor(currentTokens?: TToken[]);
+    get tokens(): readonly TToken[];
+    abstract accept(token: TToken): TAcceptTokenResult<TNextObject>;
+    protected assertNotConsumed(): void;
+}
+export declare function assertNotConsumed<T extends ParserBase<BaseToken, unknown>>(_target: T, propertyKey: "accept", descriptor: PropertyDescriptor): PropertyDescriptor;

package/vscode/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/parserBase.js

@@ -0,0 +1,30 @@
+
+import { assert } from '@codingame/monaco-vscode-api/vscode/vs/base/common/assert';
+
+class ParserBase {
+    get consumed() {
+        return this.isConsumed;
+    }
+    constructor(
+    currentTokens = []) {
+        this.currentTokens = currentTokens;
+        this.isConsumed = false;
+        this.startTokensCount = this.currentTokens.length;
+    }
+    get tokens() {
+        return this.currentTokens;
+    }
+    assertNotConsumed() {
+        assert(this.isConsumed === false, `The parser object is already consumed and should not be used anymore.`);
+    }
+}
+function assertNotConsumed(_target, propertyKey, descriptor) {
+    const originalMethod = descriptor.value;
+    descriptor.value = function (...args) {
+        assert(this.isConsumed === false, `The parser object is already consumed and should not be used anymore.`);
+        return originalMethod.apply(this, args);
+    };
+    return descriptor;
+}
+
+export { ParserBase, assertNotConsumed };

package/vscode/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/simpleDecoder.d.ts

@@ -0,0 +1,15 @@
+import { NewLine } from "../linesCodec/tokens/newLine.js";
+import { CarriageReturn } from "../linesCodec/tokens/carriageReturn.js";
+import { TLineBreakToken, TLineToken } from "../linesCodec/linesDecoder.js";
+import { At, Tab, Word, Hash, Dash, Colon, Slash, Space, Quote, Comma, FormFeed, DollarSign, DoubleQuote, VerticalTab, type TBracket, type TCurlyBrace, ExclamationMark, type TParenthesis, type TAngleBracket } from "@codingame/monaco-vscode-api/vscode/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/tokens";
+import { ISimpleTokenClass } from "./tokens/simpleToken.js";
+import { VSBuffer } from "@codingame/monaco-vscode-api/vscode/vs/base/common/buffer";
+import { BaseDecoder } from "../baseDecoder.js";
+import { ReadableStream } from "@codingame/monaco-vscode-api/vscode/vs/base/common/stream";
+export type TSimpleToken = Space | Tab | VerticalTab | At | Quote | DoubleQuote | CarriageReturn | NewLine | FormFeed | TBracket | TAngleBracket | TCurlyBrace | TParenthesis | Colon | Hash | Dash | ExclamationMark | Slash | DollarSign | Comma | TLineBreakToken;
+export type TSimpleDecoderToken = TSimpleToken | Word;
+export declare const WELL_KNOWN_TOKENS: readonly ISimpleTokenClass<TSimpleToken>[];
+export declare class SimpleDecoder extends BaseDecoder<TSimpleDecoderToken, TLineToken> {
+    constructor(stream: ReadableStream<VSBuffer>);
+    protected onStreamData(line: TLineToken): void;
+}