@vscode/chat-lib 0.0.1

package/dist/src/_internal/platform/tokenizer/node/parseTikTokens.d.ts

@@ -0,0 +1,3 @@
+/** See `script/build/compressTikToken.ts` */
+export declare const parseTikTokenBinary: (file: string) => Map<Uint8Array, number>;
+//# sourceMappingURL=parseTikTokens.d.ts.map

package/dist/src/_internal/platform/tokenizer/node/parseTikTokens.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parseTikTokens.d.ts","sourceRoot":"","sources":["../../../../../../src/_internal/platform/tokenizer/node/parseTikTokens.ts"],"names":[],"mappings":"AASA,6CAA6C;AAC7C,eAAO,MAAM,mBAAmB,GAAI,MAAM,MAAM,KAAG,GAAG,CAAC,UAAU,EAAE,MAAM,CAYxE,CAAC"}

package/dist/src/_internal/platform/tokenizer/node/parseTikTokens.js

@@ -0,0 +1,24 @@
+"use strict";
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseTikTokenBinary = void 0;
+const fs_1 = require("fs");
+const variableLengthQuantity_1 = require("../../../util/common/variableLengthQuantity");
+const buffer_1 = require("../../../util/vs/base/common/buffer");
+/** See `script/build/compressTikToken.ts` */
+const parseTikTokenBinary = (file) => {
+    const contents = (0, fs_1.readFileSync)(file);
+    const result = new Map();
+    for (let i = 0; i < contents.length;) {
+        const termLength = (0, variableLengthQuantity_1.readVariableLengthQuantity)(buffer_1.VSBuffer.wrap(contents), i);
+        i += termLength.consumed;
+        result.set(contents.subarray(i, i + termLength.value), result.size);
+        i += termLength.value;
+    }
+    return result;
+};
+exports.parseTikTokenBinary = parseTikTokenBinary;
+//# sourceMappingURL=parseTikTokens.js.map

package/dist/src/_internal/platform/tokenizer/node/parseTikTokens.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parseTikTokens.js","sourceRoot":"","sources":["../../../../../../src/_internal/platform/tokenizer/node/parseTikTokens.ts"],"names":[],"mappings":";AAAA;;;gGAGgG;;;AAEhG,2BAAkC;AAClC,wFAAyF;AACzF,gEAA+D;AAE/D,6CAA6C;AACtC,MAAM,mBAAmB,GAAG,CAAC,IAAY,EAA2B,EAAE;IAC5E,MAAM,QAAQ,GAAG,IAAA,iBAAY,EAAC,IAAI,CAAC,CAAC;IACpC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAsB,CAAC;IAE7C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC;QACtC,MAAM,UAAU,GAAG,IAAA,mDAA0B,EAAC,iBAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;QAC1E,CAAC,IAAI,UAAU,CAAC,QAAQ,CAAC;QACzB,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;QACpE,CAAC,IAAI,UAAU,CAAC,KAAK,CAAC;IACvB,CAAC;IAED,OAAO,MAAM,CAAC;AACf,CAAC,CAAC;AAZW,QAAA,mBAAmB,uBAY9B"}

package/dist/src/_internal/platform/tokenizer/node/tikTokenizerImpl.d.ts

@@ -0,0 +1,17 @@
+export type TokenDictionaryParser = (file: string) => string | Map<Uint8Array, number>;
+export declare class TikTokenImpl {
+    private static _instance;
+    private _values;
+    private _stats;
+    private constructor();
+    static get instance(): TikTokenImpl;
+    init(tokenFilePath: string, encoderName: string, useBinaryTokens: boolean): number;
+    encode(handle: number, text: string, allowedSpecial?: readonly string[]): number[];
+    destroy(handle: number): void;
+    resetStats(): {
+        callCount: number;
+        encodeDuration: number;
+        textLength: number;
+    };
+}
+//# sourceMappingURL=tikTokenizerImpl.d.ts.map

package/dist/src/_internal/platform/tokenizer/node/tikTokenizerImpl.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tikTokenizerImpl.d.ts","sourceRoot":"","sources":["../../../../../../src/_internal/platform/tokenizer/node/tikTokenizerImpl.ts"],"names":[],"mappings":"AAUA,MAAM,MAAM,qBAAqB,GAAG,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,GAAG,GAAG,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;AAEvF,qBAAa,YAAY;IAExB,OAAO,CAAC,MAAM,CAAC,SAAS,CAA2B;IAEnD,OAAO,CAAC,OAAO,CAAoC;IACnD,OAAO,CAAC,MAAM,CAIZ;IAEF,OAAO;IAEP,MAAM,KAAK,QAAQ,IAAI,YAAY,CAKlC;IAED,IAAI,CAAC,aAAa,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,eAAe,EAAE,OAAO,GAAG,MAAM;IAclF,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,cAAc,CAAC,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,EAAE;IAWlF,OAAO,CAAC,MAAM,EAAE,MAAM;IAItB,UAAU;;;;;CAYV"}

package/dist/src/_internal/platform/tokenizer/node/tikTokenizerImpl.js

@@ -0,0 +1,58 @@
+"use strict";
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TikTokenImpl = void 0;
+const tiktokenizer_1 = require("@microsoft/tiktokenizer");
+const numbers_1 = require("../../../util/vs/base/common/numbers");
+const stopwatch_1 = require("../../../util/vs/base/common/stopwatch");
+const parseTikTokens_1 = require("./parseTikTokens");
+class TikTokenImpl {
+    constructor() {
+        this._values = [];
+        this._stats = {
+            encodeDuration: new numbers_1.MovingAverage(),
+            textLength: new numbers_1.MovingAverage(),
+            callCount: 0,
+        };
+    }
+    static get instance() {
+        if (!this._instance) {
+            this._instance = new TikTokenImpl();
+        }
+        return this._instance;
+    }
+    init(tokenFilePath, encoderName, useBinaryTokens) {
+        const handle = this._values.length;
+        const parser = useBinaryTokens ? parseTikTokens_1.parseTikTokenBinary : f => f;
+        this._values.push((0, tiktokenizer_1.createTokenizer)(parser(tokenFilePath), (0, tiktokenizer_1.getSpecialTokensByEncoder)(encoderName), (0, tiktokenizer_1.getRegexByEncoder)(encoderName), 64000));
+        return handle;
+    }
+    encode(handle, text, allowedSpecial) {
+        const sw = stopwatch_1.StopWatch.create(true);
+        const result = this._values[handle].encode(text, allowedSpecial);
+        this._stats.callCount += 1;
+        this._stats.encodeDuration.update(sw.elapsed());
+        this._stats.textLength.update(text.length);
+        return result;
+    }
+    destroy(handle) {
+        this._values[handle] = undefined;
+    }
+    resetStats() {
+        const oldValue = this._stats;
+        const result = {
+            callCount: oldValue.callCount,
+            encodeDuration: oldValue.encodeDuration.value,
+            textLength: oldValue.textLength.value
+        };
+        this._stats.encodeDuration = new numbers_1.MovingAverage();
+        this._stats.textLength = new numbers_1.MovingAverage();
+        this._stats.callCount = 0;
+        return result;
+    }
+}
+exports.TikTokenImpl = TikTokenImpl;
+//# sourceMappingURL=tikTokenizerImpl.js.map

package/dist/src/_internal/platform/tokenizer/node/tikTokenizerImpl.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tikTokenizerImpl.js","sourceRoot":"","sources":["../../../../../../src/_internal/platform/tokenizer/node/tikTokenizerImpl.ts"],"names":[],"mappings":";AAAA;;;gGAGgG;;;AAEhG,0DAAsH;AACtH,kEAAqE;AACrE,sEAAmE;AACnE,qDAAuD;AAIvD,MAAa,YAAY;IAWxB;QAPQ,YAAO,GAAiC,EAAE,CAAC;QAC3C,WAAM,GAAG;YAChB,cAAc,EAAE,IAAI,uBAAa,EAAE;YACnC,UAAU,EAAE,IAAI,uBAAa,EAAE;YAC/B,SAAS,EAAE,CAAC;SACZ,CAAC;IAEsB,CAAC;IAEzB,MAAM,KAAK,QAAQ;QAClB,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YACrB,IAAI,CAAC,SAAS,GAAG,IAAI,YAAY,EAAE,CAAC;QACrC,CAAC;QACD,OAAO,IAAI,CAAC,SAAS,CAAC;IACvB,CAAC;IAED,IAAI,CAAC,aAAqB,EAAE,WAAmB,EAAE,eAAwB;QACxE,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;QACnC,MAAM,MAAM,GAA0B,eAAe,CAAC,CAAC,CAAC,oCAAmB,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAErF,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAA,8BAAe,EAChC,MAAM,CAAC,aAAa,CAAC,EACrB,IAAA,wCAAyB,EAAC,WAAW,CAAC,EACtC,IAAA,gCAAiB,EAAC,WAAW,CAAC,EAC9B,KAAK,CACL,CAAC,CAAC;QAEH,OAAO,MAAM,CAAC;IACf,CAAC;IAED,MAAM,CAAC,MAAc,EAAE,IAAY,EAAE,cAAkC;QACtE,MAAM,EAAE,GAAG,qBAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAClC,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAE,CAAC,MAAM,CAAC,IAAI,EAAE,cAAc,CAAC,CAAC;QAElE,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,CAAC,CAAC;QAC3B,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;QAChD,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAE3C,OAAO,MAAM,CAAC;IACf,CAAC;IAED,OAAO,CAAC,MAAc;QACrB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,SAAS,CAAC;IAClC,CAAC;IAED,UAAU;QACT,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC;QAC7B,MAAM,MAAM,GAAG;YACd,SAAS,EAAE,QAAQ,CAAC,SAAS;YAC7B,cAAc,EAAE,QAAQ,CAAC,cAAc,CAAC,KAAK;YAC7C,UAAU,EAAE,QAAQ,CAAC,UAAU,CAAC,KAAK;SACrC,CAAC;QACF,IAAI,CAAC,MAAM,CAAC,cAAc,GAAG,IAAI,uBAAa,EAAE,CAAC;QACjD,IAAI,CAAC,MAAM,CAAC,UAAU,GAAG,IAAI,uBAAa,EAAE,CAAC;QAC7C,IAAI,CAAC,MAAM,CAAC,SAAS,GAAG,CAAC,CAAC;QAC1B,OAAO,MAAM,CAAC;IACf,CAAC;CACD;AA7DD,oCA6DC"}

package/dist/src/_internal/platform/tokenizer/node/tikTokenizerWorker.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=tikTokenizerWorker.d.ts.map

package/dist/src/_internal/platform/tokenizer/node/tikTokenizerWorker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tikTokenizerWorker.d.ts","sourceRoot":"","sources":["../../../../../../src/_internal/platform/tokenizer/node/tikTokenizerWorker.ts"],"names":[],"mappings":""}

package/dist/src/_internal/platform/tokenizer/node/tikTokenizerWorker.js

@@ -0,0 +1,25 @@
+"use strict";
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+Object.defineProperty(exports, "__esModule", { value: true });
+const worker_threads_1 = require("worker_threads");
+const tikTokenizerImpl_1 = require("./tikTokenizerImpl");
+function main() {
+    const port = worker_threads_1.parentPort;
+    if (!port) {
+        throw new Error(`This module should only be used in a worker thread.`);
+    }
+    port.on('message', async (message) => {
+        try {
+            const res = await tikTokenizerImpl_1.TikTokenImpl.instance[message.fn](...message.args);
+            port.postMessage({ id: message.id, res });
+        }
+        catch (err) {
+            port.postMessage({ id: message.id, err });
+        }
+    });
+}
+main();
+//# sourceMappingURL=tikTokenizerWorker.js.map

package/dist/src/_internal/platform/tokenizer/node/tikTokenizerWorker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tikTokenizerWorker.js","sourceRoot":"","sources":["../../../../../../src/_internal/platform/tokenizer/node/tikTokenizerWorker.ts"],"names":[],"mappings":";AAAA;;;gGAGgG;;AAEhG,mDAA4C;AAC5C,yDAAkD;AAElD,SAAS,IAAI;IACZ,MAAM,IAAI,GAAG,2BAAU,CAAC;IACxB,IAAI,CAAC,IAAI,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;IACxE,CAAC;IACD,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,KAAK,EAAE,OAAgD,EAAE,EAAE;QAC7E,IAAI,CAAC;YACJ,MAAM,GAAG,GAAG,MAAY,+BAAY,CAAC,QAAS,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;YAC5E,IAAI,CAAC,WAAW,CAAC,EAAE,EAAE,EAAE,OAAO,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;QAC3C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,IAAI,CAAC,WAAW,CAAC,EAAE,EAAE,EAAE,OAAO,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;QAC3C,CAAC;IACF,CAAC,CAAC,CAAC;AACJ,CAAC;AAED,IAAI,EAAE,CAAC"}

package/dist/src/_internal/platform/tokenizer/node/tokenizer.d.ts

@@ -0,0 +1,32 @@
+import { ITokenizer, TokenizerType } from '../../../util/common/tokenizer';
+import { ITelemetryService } from '../../telemetry/common/telemetry';
+export declare const ITokenizerProvider: import("../../../util/common/services").ServiceIdentifier<ITokenizerProvider>;
+export interface TokenizationEndpoint {
+    readonly tokenizer: TokenizerType;
+}
+export interface ITokenizerProvider {
+    readonly _serviceBrand: undefined;
+    acquireTokenizer(endpoint: TokenizationEndpoint): ITokenizer;
+}
+/**
+ * BaseTokensPerCompletion is the minimum tokens for a completion request.
+ * Replies are primed with <|im_start|>assistant<|message|>, so these tokens represent the
+ * special token and the role name.
+ */
+export declare const BaseTokensPerCompletion = 3;
+export declare const BaseTokensPerMessage = 3;
+export declare const BaseTokensPerName = 1;
+export declare class TokenizerProvider implements ITokenizerProvider {
+    readonly _serviceBrand: undefined;
+    private readonly _cl100kTokenizer;
+    private readonly _o200kTokenizer;
+    constructor(useWorker: boolean, telmetryService: ITelemetryService);
+    dispose(): void;
+    /**
+     * Gets a tokenizer for a given model family
+     * @param endpoint The endpoint you want to acquire a tokenizer for
+     */
+    acquireTokenizer(endpoint: TokenizationEndpoint): ITokenizer;
+}
+export declare function calculateImageTokenCost(imageUrl: string, detail: 'low' | 'high' | undefined): number;
+//# sourceMappingURL=tokenizer.d.ts.map

package/dist/src/_internal/platform/tokenizer/node/tokenizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenizer.d.ts","sourceRoot":"","sources":["../../../../../../src/_internal/platform/tokenizer/node/tokenizer.ts"],"names":[],"mappings":"AAUA,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAK3E,OAAO,EAAE,iBAAiB,EAAE,MAAM,kCAAkC,CAAC;AAGrE,eAAO,MAAM,kBAAkB,+EAAoE,CAAC;AAEpG,MAAM,WAAW,oBAAoB;IACpC,QAAQ,CAAC,SAAS,EAAE,aAAa,CAAC;CAClC;AAED,MAAM,WAAW,kBAAkB;IAClC,QAAQ,CAAC,aAAa,EAAE,SAAS,CAAC;IAElC,gBAAgB,CAAC,QAAQ,EAAE,oBAAoB,GAAG,UAAU,CAAC;CAC7D;AAED;;;;GAIG;AACH,eAAO,MAAM,uBAAuB,IAAI,CAAC;AAIzC,eAAO,MAAM,oBAAoB,IAAI,CAAC;AAItC,eAAO,MAAM,iBAAiB,IAAI,CAAC;AAGnC,qBAAa,iBAAkB,YAAW,kBAAkB;IAC3D,SAAiB,aAAa,EAAE,SAAS,CAAC;IAG1C,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAe;IAChD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAe;gBAG9C,SAAS,EAAE,OAAO,EACC,eAAe,EAAE,iBAAiB;IAStD,OAAO;IAKP;;;OAGG;IACI,gBAAgB,CAAC,QAAQ,EAAE,oBAAoB,GAAG,UAAU;CAUnE;AA0PD,wBAAgB,uBAAuB,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,GAAG,MAAM,GAAG,SAAS,GAAG,MAAM,CAqBpG"}

package/dist/src/_internal/platform/tokenizer/node/tokenizer.js

@@ -0,0 +1,308 @@
+"use strict";
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __param = (this && this.__param) || function (paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TokenizerProvider = exports.BaseTokensPerName = exports.BaseTokensPerMessage = exports.BaseTokensPerCompletion = exports.ITokenizerProvider = void 0;
+exports.calculateImageTokenCost = calculateImageTokenCost;
+const prompt_tsx_1 = require("@vscode/prompt-tsx");
+const cache_1 = require("../../../util/common/cache");
+const imageUtils_1 = require("../../../util/common/imageUtils");
+const services_1 = require("../../../util/common/services");
+const tokenizer_1 = require("../../../util/common/tokenizer");
+const worker_1 = require("../../../util/node/worker");
+const assert_1 = require("../../../util/vs/base/common/assert");
+const lifecycle_1 = require("../../../util/vs/base/common/lifecycle");
+const path_1 = require("../../../util/vs/base/common/path");
+const telemetry_1 = require("../../telemetry/common/telemetry");
+const tikTokenizerImpl_1 = require("./tikTokenizerImpl");
+exports.ITokenizerProvider = (0, services_1.createServiceIdentifier)('ITokenizerProvider');
+/**
+ * BaseTokensPerCompletion is the minimum tokens for a completion request.
+ * Replies are primed with <|im_start|>assistant<|message|>, so these tokens represent the
+ * special token and the role name.
+ */
+exports.BaseTokensPerCompletion = 3;
+/*
+ * Each GPT 3.5 / GPT 4 message comes with 3 tokens per message due to special characters
+ */
+exports.BaseTokensPerMessage = 3;
+/*
+ * Since gpt-3.5-turbo-0613 each name costs 1 token
+ */
+exports.BaseTokensPerName = 1;
+let TokenizerProvider = class TokenizerProvider {
+    constructor(useWorker, telmetryService) {
+        // if we're running from dist, the dictionary is compressed, but if we're  running
+        // in e.g. a `spec` file we should load the dictionary using default behavior.
+        // todo: cleanup a bit, have an IS_BUILT constant?
+        this._cl100kTokenizer = new BPETokenizer(useWorker, (0, path_1.join)(__dirname, './cl100k_base.tiktoken'), 'cl100k_base', telmetryService);
+        this._o200kTokenizer = new BPETokenizer(useWorker, (0, path_1.join)(__dirname, './o200k_base.tiktoken'), 'o200k_base', telmetryService);
+    }
+    dispose() {
+        this._cl100kTokenizer.dispose();
+        this._o200kTokenizer.dispose();
+    }
+    /**
+     * Gets a tokenizer for a given model family
+     * @param endpoint The endpoint you want to acquire a tokenizer for
+     */
+    acquireTokenizer(endpoint) {
+        switch (endpoint.tokenizer) {
+            case tokenizer_1.TokenizerType.CL100K:
+                return this._cl100kTokenizer;
+            case tokenizer_1.TokenizerType.O200K:
+                return this._o200kTokenizer;
+            default:
+                throw new Error(`Unknown tokenizer: ${endpoint.tokenizer}`);
+        }
+    }
+};
+exports.TokenizerProvider = TokenizerProvider;
+exports.TokenizerProvider = TokenizerProvider = __decorate([
+    __param(1, telemetry_1.ITelemetryService)
+], TokenizerProvider);
+let BPETokenizer = class BPETokenizer extends lifecycle_1.Disposable {
+    constructor(_useWorker, _tokenFilePath, _encoderName, _telemetryService) {
+        super();
+        this._useWorker = _useWorker;
+        this._tokenFilePath = _tokenFilePath;
+        this._encoderName = _encoderName;
+        this._telemetryService = _telemetryService;
+        /**
+         * TikToken has its own cache, but it still does some processing
+         * until a cache hit. We can have a much more efficient cache that
+         * directly looks up string -> token length
+         */
+        this._cache = new cache_1.LRUCache(5000);
+        this.baseTokensPerMessage = exports.BaseTokensPerMessage;
+        this.baseTokensPerName = exports.BaseTokensPerName;
+        this.mode = prompt_tsx_1.OutputMode.Raw;
+    }
+    async countMessagesTokens(messages) {
+        let numTokens = exports.BaseTokensPerMessage;
+        for (const message of messages) {
+            numTokens += await this.countMessageTokens(message);
+        }
+        return numTokens;
+    }
+    /**
+     * Tokenizes the given text.
+     * @param text The text to tokenize.
+     * @returns The tokenized text.
+     */
+    async tokenize(text) {
+        return (await this.ensureTokenizer()).encode(text);
+    }
+    /**
+     * Calculates the token length of the given text.
+     * @param text The text to calculate the token length for.
+     * @returns The number of tokens in the text.
+     */
+    async tokenLength(text) {
+        if (typeof text === 'string') {
+            return this._textTokenLength(text);
+        }
+        switch (text.type) {
+            case prompt_tsx_1.Raw.ChatCompletionContentPartKind.Text:
+                return this._textTokenLength(text.text);
+            case prompt_tsx_1.Raw.ChatCompletionContentPartKind.Opaque:
+                return text.tokenUsage || 0;
+            case prompt_tsx_1.Raw.ChatCompletionContentPartKind.Image:
+                if (text.imageUrl.url.startsWith('data:image/')) {
+                    try {
+                        return calculateImageTokenCost(text.imageUrl.url, text.imageUrl.detail);
+                    }
+                    catch {
+                        return this._textTokenLength(text.imageUrl.url);
+                    }
+                }
+                return this._textTokenLength(text.imageUrl.url);
+            case prompt_tsx_1.Raw.ChatCompletionContentPartKind.CacheBreakpoint:
+                return 0;
+            default:
+                (0, assert_1.assertNever)(text, `unknown content part (${JSON.stringify(text)})`);
+        }
+    }
+    async _textTokenLength(text) {
+        if (!text) {
+            return 0;
+        }
+        let cacheValue = this._cache.get(text);
+        if (!cacheValue) {
+            cacheValue = (await this.tokenize(text)).length;
+            this._cache.put(text, cacheValue);
+        }
+        return cacheValue;
+    }
+    /**
+     * Counts tokens for a single chat message within a completion request.
+     *
+     * Follows https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb for GPT 3.5/4 models.
+     *
+     * **Note**: The result does not include base tokens for the completion itself.
+     */
+    async countMessageTokens(message) {
+        return this.baseTokensPerMessage + (await this.countMessageObjectTokens((0, prompt_tsx_1.toMode)(prompt_tsx_1.OutputMode.OpenAI, message)));
+    }
+    async countToolTokens(tools) {
+        const baseToolTokens = 16;
+        let numTokens = 0;
+        if (tools.length) {
+            numTokens += baseToolTokens;
+        }
+        const baseTokensPerTool = 8;
+        for (const tool of tools) {
+            numTokens += baseTokensPerTool;
+            numTokens += await this.countObjectTokens({ name: tool.name, description: tool.description, parameters: tool.inputSchema });
+        }
+        // This is an estimate, so give a little safety margin
+        return Math.floor(numTokens * 1.1);
+    }
+    async countMessageObjectTokens(obj) {
+        let numTokens = 0;
+        for (const [key, value] of Object.entries(obj)) {
+            if (!value) {
+                continue;
+            }
+            if (typeof value === 'string') {
+                numTokens += await this.tokenLength(value);
+            }
+            else if (value) {
+                const casted = value;
+                if (casted.type === 'text') {
+                    numTokens += await this.tokenLength(casted.text);
+                }
+                else if (casted.type === 'image_url' && casted.image_url) {
+                    if (casted.image_url.url.startsWith('data:image/')) {
+                        try {
+                            numTokens += calculateImageTokenCost(casted.image_url.url, casted.image_url.detail);
+                        }
+                        catch {
+                            numTokens += await this.tokenLength(casted.image_url.url);
+                        }
+                    }
+                    else {
+                        numTokens += await this.tokenLength(casted.image_url.url);
+                    }
+                }
+                else {
+                    let newTokens = await this.countMessageObjectTokens(value);
+                    if (key === 'tool_calls') {
+                        // This is an estimate, not including all of the overhead, so give a little safety margin
+                        newTokens = Math.floor(newTokens * 1.5);
+                    }
+                    numTokens += newTokens;
+                }
+            }
+            if (key === 'name' && value !== undefined) {
+                numTokens += this.baseTokensPerName;
+            }
+        }
+        return numTokens;
+    }
+    async countObjectTokens(obj) {
+        let numTokens = 0;
+        for (const [key, value] of Object.entries(obj)) {
+            if (!value) {
+                continue;
+            }
+            numTokens += await this.tokenLength(key);
+            if (typeof value === 'string') {
+                numTokens += await this.tokenLength(value);
+            }
+            else if (value) {
+                numTokens += await this.countMessageObjectTokens(value);
+            }
+        }
+        return numTokens;
+    }
+    ensureTokenizer() {
+        this._tokenizer ??= this.doInitTokenizer();
+        return this._tokenizer;
+    }
+    async doInitTokenizer() {
+        const useBinaryTokens = (0, path_1.basename)(__dirname) === 'dist';
+        if (!this._useWorker) {
+            const handle = tikTokenizerImpl_1.TikTokenImpl.instance.init(this._tokenFilePath, this._encoderName, useBinaryTokens);
+            const cleanup = (0, lifecycle_1.toDisposable)(() => {
+                tikTokenizerImpl_1.TikTokenImpl.instance.destroy(handle);
+                this._store.deleteAndLeak(cleanup);
+                this._tokenizer = undefined;
+            });
+            this._store.add(cleanup);
+            return {
+                encode: async (text, allowedSpecial) => {
+                    return tikTokenizerImpl_1.TikTokenImpl.instance.encode(handle, text, allowedSpecial);
+                }
+            };
+        }
+        else {
+            const workerPath = (0, path_1.join)(__dirname, 'tikTokenizerWorker.js');
+            const worker = new worker_1.WorkerWithRpcProxy(workerPath, { name: `TikToken worker (${this._encoderName})` });
+            const handle = await worker.proxy.init(this._tokenFilePath, this._encoderName, useBinaryTokens);
+            const cleanup = (0, lifecycle_1.toDisposable)(() => {
+                worker.terminate();
+                this._store.deleteAndLeak(cleanup);
+                this._tokenizer = undefined;
+            });
+            let timeout;
+            return {
+                encode: (text, allowedSpecial) => {
+                    const result = worker.proxy.encode(handle, text, allowedSpecial);
+                    clearTimeout(timeout);
+                    timeout = setTimeout(() => cleanup.dispose(), 15000);
+                    if (Math.random() < 1 / 1000) {
+                        worker.proxy.resetStats().then(stats => {
+                            /* __GDPR__
+                                "tokenizer.stats" : {
+                                    "owner": "jrieken",
+                                    "comment": "Perf stats about tokenizers",
+                                    "callCount": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "isMeasurement": true, "comment": "How often tokenize was called" },
+                                    "encodeDuration": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "isMeasurement": true, "comment": "Average time encode took" },
+                                    "textLength": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "isMeasurement": true, "comment": "Average length of text that got encoded" }
+                                }
+                            */
+                            this._telemetryService.sendMSFTTelemetryEvent('tokenizer.stats', undefined, stats);
+                        });
+                    }
+                    return result;
+                }
+            };
+        }
+    }
+};
+BPETokenizer = __decorate([
+    __param(3, telemetry_1.ITelemetryService)
+], BPETokenizer);
+//#region Image tokenizer helpers
+// https://platform.openai.com/docs/guides/vision#calculating-costs
+function calculateImageTokenCost(imageUrl, detail) {
+    let { width, height } = (0, imageUtils_1.getImageDimensions)(imageUrl);
+    if (detail === 'low') {
+        return 85;
+    }
+    // Scale image to fit within a 2048 x 2048 square if necessary.
+    if (width > 2048 || height > 2048) {
+        const scaleFactor = 2048 / Math.max(width, height);
+        width = Math.round(width * scaleFactor);
+        height = Math.round(height * scaleFactor);
+    }
+    const scaleFactor = 768 / Math.min(width, height);
+    width = Math.round(width * scaleFactor);
+    height = Math.round(height * scaleFactor);
+    const tiles = Math.ceil(width / 512) * Math.ceil(height / 512);
+    return tiles * 170 + 85;
+}
+//#endregion
+//# sourceMappingURL=tokenizer.js.map

package/dist/src/_internal/platform/tokenizer/node/tokenizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["../../../../../../src/_internal/platform/tokenizer/node/tokenizer.ts"],"names":[],"mappings":";AAAA;;;gGAGgG;;;;;;;;;;;;AA0UhG,0DAqBC;AA7VD,mDAA6D;AAE7D,sDAAsD;AACtD,gEAAqE;AACrE,4DAAwE;AACxE,8DAA2E;AAC3E,sDAA+D;AAC/D,gEAAkE;AAClE,sEAAkF;AAClF,4DAAmE;AACnE,gEAAqE;AACrE,yDAAkD;AAErC,QAAA,kBAAkB,GAAG,IAAA,kCAAuB,EAAqB,oBAAoB,CAAC,CAAC;AAYpG;;;;GAIG;AACU,QAAA,uBAAuB,GAAG,CAAC,CAAC;AACzC;;GAEG;AACU,QAAA,oBAAoB,GAAG,CAAC,CAAC;AACtC;;GAEG;AACU,QAAA,iBAAiB,GAAG,CAAC,CAAC;AAG5B,IAAM,iBAAiB,GAAvB,MAAM,iBAAiB;IAO7B,YACC,SAAkB,EACC,eAAkC;QAErD,kFAAkF;QAClF,8EAA8E;QAC9E,kDAAkD;QAClD,IAAI,CAAC,gBAAgB,GAAG,IAAI,YAAY,CAAC,SAAS,EAAE,IAAA,WAAI,EAAC,SAAS,EAAE,wBAAwB,CAAC,EAAE,aAAa,EAAE,eAAe,CAAC,CAAC;QAC/H,IAAI,CAAC,eAAe,GAAG,IAAI,YAAY,CAAC,SAAS,EAAE,IAAA,WAAI,EAAC,SAAS,EAAE,uBAAuB,CAAC,EAAE,YAAY,EAAE,eAAe,CAAC,CAAC;IAC7H,CAAC;IAED,OAAO;QACN,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,CAAC;QAChC,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC;IAChC,CAAC;IAED;;;OAGG;IACI,gBAAgB,CAAC,QAA8B;QACrD,QAAQ,QAAQ,CAAC,SAAS,EAAE,CAAC;YAC5B,KAAK,yBAAa,CAAC,MAAM;gBACxB,OAAO,IAAI,CAAC,gBAAgB,CAAC;YAC9B,KAAK,yBAAa,CAAC,KAAK;gBACvB,OAAO,IAAI,CAAC,eAAe,CAAC;YAC7B;gBACC,MAAM,IAAI,KAAK,CAAC,sBAAsB,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC;QAC9D,CAAC;IACF,CAAC;CACD,CAAA;AArCY,8CAAiB;4BAAjB,iBAAiB;IAS3B,WAAA,6BAAiB,CAAA;GATP,iBAAiB,CAqC7B;AAMD,IAAM,YAAY,GAAlB,MAAM,YAAa,SAAQ,sBAAU;IAgBpC,YACkB,UAAmB,EACnB,cAAsB,EACtB,YAAoB,EAClB,iBAAqD;QAExE,KAAK,EAAE,CAAC;QALS,eAAU,GAAV,UAAU,CAAS;QACnB,mBAAc,GAAd,cAAc,CAAQ;QACtB,iBAAY,GAAZ,YAAY,CAAQ;QACD,sBAAiB,GAAjB,iBAAiB,CAAmB;QAhBzE;;;;WAIG;QACc,WAAM,GAAG,IAAI,gBAAQ,CAAS,IAAI,CAAC,CAAC;QAElC,yBAAoB,GAAG,4BAAoB,CAAC;QAC5C,sBAAiB,GAAG,yBAAiB,CAAC;QAEzC,SAAI,GAAG,uBAAU,CAAC,GAAG,CAAC;IAStC,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,QAA2B;QACpD,IAAI,SAAS,GAAG,4BAAoB,CAAC;QACrC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAChC,SAAS,IAAI,MAAM,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;QACrD,CAAC;QACD,OAAO,SAAS,CAAC;IAClB,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,QAAQ,CAAC,IAAY;QAClC,OAAO,CAAC,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACpD,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,WAAW,CAAC,IAA4C;QAC7D,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;QACpC,CAAC;QAED,QAAQ,IAAI,CAAC,IAAI,EAAE,CAAC;YACnB,KAAK,gBAAG,CAAC,6BAA6B,CAAC,IAAI;gBAC1C,OAAO,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzC,KAAK,gBAAG,CAAC,6BAA6B,CAAC,MAAM;gBAC5C,OAAO,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;YAC7B,KAAK,gBAAG,CAAC,6BAA6B,CAAC,KAAK;gBAC3C,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;oBACjD,IAAI,CAAC;wBACJ,OAAO,uBAAuB,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;oBACzE,CAAC;oBAAC,MAAM,CAAC;wBACR,OAAO,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;oBACjD,CAAC;gBACF,CAAC;gBACD,OAAO,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;YACjD,KAAK,gBAAG,CAAC,6BAA6B,CAAC,eAAe;gBACrD,OAAO,CAAC,CAAC;YACV;gBACC,IAAA,oBAAW,EAAC,IAAI,EAAE,yBAAyB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACtE,CAAC;IACF,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAAC,IAAY;QAC1C,IAAI,CAAC,IAAI,EAAE,CAAC;YACX,OAAO,CAAC,CAAC;QACV,CAAC;QACD,IAAI,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACvC,IAAI,CAAC,UAAU,EAAE,CAAC;YACjB,UAAU,GAAG,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;YAChD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;QACnC,CAAC;QACD,OAAO,UAAU,CAAC;IACnB,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,kBAAkB,CAAC,OAAwB;QAChD,OAAO,IAAI,CAAC,oBAAoB,GAAG,CAAC,MAAM,IAAI,CAAC,wBAAwB,CAAC,IAAA,mBAAM,EAAC,uBAAU,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;IAC9G,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,KAA8B;QACnD,MAAM,cAAc,GAAG,EAAE,CAAC;QAC1B,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;YAClB,SAAS,IAAI,cAAc,CAAC;QAC7B,CAAC;QAED,MAAM,iBAAiB,GAAG,CAAC,CAAC;QAC5B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YAC1B,SAAS,IAAI,iBAAiB,CAAC;YAC/B,SAAS,IAAI,MAAM,IAAI,CAAC,iBAAiB,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,UAAU,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;QAC7H,CAAC;QAED,sDAAsD;QACtD,OAAO,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC;IACpC,CAAC;IAEO,KAAK,CAAC,wBAAwB,CAAC,GAAQ;QAC9C,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;YAChD,IAAI,CAAC,KAAK,EAAE,CAAC;gBACZ,SAAS;YACV,CAAC;YAED,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;gBAC/B,SAAS,IAAI,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;YAC5C,CAAC;iBAAM,IAAI,KAAK,EAAE,CAAC;gBAClB,MAAM,MAAM,GAAG,KAAY,CAAC;gBAC5B,IAAI,MAAM,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;oBAC5B,SAAS,IAAI,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBAClD,CAAC;qBAAM,IAAI,MAAM,CAAC,IAAI,KAAK,WAAW,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;oBAC5D,IAAI,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;wBACpD,IAAI,CAAC;4BACJ,SAAS,IAAI,uBAAuB,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;wBACrF,CAAC;wBAAC,MAAM,CAAC;4BACR,SAAS,IAAI,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;wBAC3D,CAAC;oBACF,CAAC;yBAAM,CAAC;wBACP,SAAS,IAAI,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;oBAC3D,CAAC;gBACF,CAAC;qBAAM,CAAC;oBACP,IAAI,SAAS,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAAC,KAAK,CAAC,CAAC;oBAC3D,IAAI,GAAG,KAAK,YAAY,EAAE,CAAC;wBAC1B,yFAAyF;wBACzF,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC;oBACzC,CAAC;oBAED,SAAS,IAAI,SAAS,CAAC;gBACxB,CAAC;YACF,CAAC;YAED,IAAI,GAAG,KAAK,MAAM,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC3C,SAAS,IAAI,IAAI,CAAC,iBAAiB,CAAC;YACrC,CAAC;QACF,CAAC;QAED,OAAO,SAAS,CAAC;IAClB,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAAC,GAAQ;QACvC,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;YAChD,IAAI,CAAC,KAAK,EAAE,CAAC;gBACZ,SAAS;YACV,CAAC;YAED,SAAS,IAAI,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;YACzC,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;gBAC/B,SAAS,IAAI,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;YAC5C,CAAC;iBAAM,IAAI,KAAK,EAAE,CAAC;gBAClB,SAAS,IAAI,MAAM,IAAI,CAAC,wBAAwB,CAAC,KAAK,CAAC,CAAC;YACzD,CAAC;QACF,CAAC;QAED,OAAO,SAAS,CAAC;IAClB,CAAC;IAEO,eAAe;QACtB,IAAI,CAAC,UAAU,KAAK,IAAI,CAAC,eAAe,EAAE,CAAC;QAC3C,OAAO,IAAI,CAAC,UAAU,CAAC;IACxB,CAAC;IAEO,KAAK,CAAC,eAAe;QAE5B,MAAM,eAAe,GAAG,IAAA,eAAQ,EAAC,SAAS,CAAC,KAAK,MAAM,CAAC;QAEvD,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;YACtB,MAAM,MAAM,GAAG,+BAAY,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC;YAEnG,MAAM,OAAO,GAAG,IAAA,wBAAY,EAAC,GAAG,EAAE;gBACjC,+BAAY,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;gBACtC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;gBACnC,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;YAC7B,CAAC,CAAC,CAAC;YACH,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAEzB,OAAO;gBACN,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,cAAc,EAAE,EAAE;oBACtC,OAAO,+BAAY,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,cAAc,CAAC,CAAC;gBACnE,CAAC;aACD,CAAC;QACH,CAAC;aAAM,CAAC;YAEP,MAAM,UAAU,GAAG,IAAA,WAAI,EAAC,SAAS,EAAE,uBAAuB,CAAC,CAAC;YAC5D,MAAM,MAAM,GAAG,IAAI,2BAAkB,CAAe,UAAU,EAAE,EAAE,IAAI,EAAE,oBAAoB,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC,CAAC;YACpH,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC;YAEhG,MAAM,OAAO,GAAG,IAAA,wBAAY,EAAC,GAAG,EAAE;gBACjC,MAAM,CAAC,SAAS,EAAE,CAAC;gBACnB,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;gBACnC,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;YAC7B,CAAC,CAAC,CAAC;YAEH,IAAI,OAAsB,CAAC;YAE3B,OAAO;gBACN,MAAM,EAAE,CAAC,IAAI,EAAE,cAAc,EAAE,EAAE;oBAChC,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,cAAc,CAAC,CAAC;oBAEjE,YAAY,CAAC,OAAO,CAAC,CAAC;oBACtB,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,KAAK,CAAC,CAAC;oBAErD,IAAI,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,IAAI,EAAE,CAAC;wBAC9B,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;4BACtC;;;;;;;;8BAQE;4BACF,IAAI,CAAC,iBAAiB,CAAC,sBAAsB,CAAC,iBAAiB,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;wBACpF,CAAC,CAAC,CAAC;oBACJ,CAAC;oBAED,OAAO,MAAM,CAAC;gBACf,CAAC;aACD,CAAC;QACH,CAAC;IACF,CAAC;CACD,CAAA;AA9OK,YAAY;IAoBf,WAAA,6BAAiB,CAAA;GApBd,YAAY,CA8OjB;AAGD,iCAAiC;AAEjC,mEAAmE;AACnE,SAAgB,uBAAuB,CAAC,QAAgB,EAAE,MAAkC;IAC3F,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,IAAA,+BAAkB,EAAC,QAAQ,CAAC,CAAC;IAErD,IAAI,MAAM,KAAK,KAAK,EAAE,CAAC;QACtB,OAAO,EAAE,CAAC;IACX,CAAC;IAED,+DAA+D;IAC/D,IAAI,KAAK,GAAG,IAAI,IAAI,MAAM,GAAG,IAAI,EAAE,CAAC;QACnC,MAAM,WAAW,GAAG,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QACnD,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,WAAW,CAAC,CAAC;QACxC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,WAAW,CAAC,CAAC;IAC3C,CAAC;IAED,MAAM,WAAW,GAAG,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAClD,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,WAAW,CAAC,CAAC;IACxC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,WAAW,CAAC,CAAC;IAE1C,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;IAE/D,OAAO,KAAK,GAAG,GAAG,GAAG,EAAE,CAAC;AACzB,CAAC;AAED,YAAY"}

package/dist/src/_internal/platform/workspace/common/workspaceService.d.ts

@@ -0,0 +1,104 @@
+import type { Event, FileSystem, NotebookData, NotebookDocument, NotebookDocumentChangeEvent, TextDocument, TextDocumentChangeEvent, TextEditorSelectionChangeEvent, Uri, WorkspaceEdit, WorkspaceFolder, WorkspaceFoldersChangeEvent } from 'vscode';
+import { URI } from '../../../util/vs/base/common/uri';
+import { NotebookDocumentSnapshot } from '../../editing/common/notebookDocumentSnapshot';
+import { TextDocumentSnapshot } from '../../editing/common/textDocumentSnapshot';
+import { IDisposable } from '../../../util/vs/base/common/lifecycle';
+import { Emitter } from '../../../util/vs/base/common/event';
+export declare const IWorkspaceService: import("../../../util/common/services").ServiceIdentifier<IWorkspaceService>;
+export interface IWorkspaceService {
+    readonly _serviceBrand: undefined;
+    textDocuments: readonly TextDocument[];
+    notebookDocuments: readonly NotebookDocument[];
+    readonly onDidOpenTextDocument: Event<TextDocument>;
+    readonly onDidCloseTextDocument: Event<TextDocument>;
+    readonly onDidOpenNotebookDocument: Event<NotebookDocument>;
+    readonly onDidCloseNotebookDocument: Event<NotebookDocument>;
+    readonly onDidChangeTextDocument: Event<TextDocumentChangeEvent>;
+    readonly onDidChangeNotebookDocument: Event<NotebookDocumentChangeEvent>;
+    readonly onDidChangeWorkspaceFolders: Event<WorkspaceFoldersChangeEvent>;
+    readonly onDidChangeTextEditorSelection: Event<TextEditorSelectionChangeEvent>;
+    openTextDocument(uri: Uri): Promise<TextDocument>;
+    fs: FileSystem;
+    showTextDocument(document: TextDocument): Promise<void>;
+    openTextDocumentAndSnapshot(uri: Uri): Promise<TextDocumentSnapshot>;
+    openNotebookDocumentAndSnapshot(uri: Uri, format: 'xml' | 'json' | 'text'): Promise<NotebookDocumentSnapshot>;
+    openNotebookDocument(uri: Uri): Promise<NotebookDocument>;
+    openNotebookDocument(notebookType: string, content?: NotebookData): Promise<NotebookDocument>;
+    getWorkspaceFolders(): URI[];
+    getWorkspaceFolder(resource: URI): URI | undefined;
+    getWorkspaceFolderName(workspaceFolderUri: URI): string;
+    showWorkspaceFolderPicker(): Promise<WorkspaceFolder | undefined>;
+    asRelativePath(pathOrUri: string | Uri, includeWorkspaceFolder?: boolean): string;
+    applyEdit(edit: WorkspaceEdit): Thenable<boolean>;
+    /**
+     * Ensures that the workspace has fully loaded before returning. This is useful for
+     * virtual workspaces where we need to ensure that the contents of the workspace
+     * has been downloaded before we can use them.
+     */
+    ensureWorkspaceIsFullyLoaded(): Promise<void>;
+}
+export declare abstract class AbstractWorkspaceService implements IWorkspaceService {
+    readonly _serviceBrand: undefined;
+    abstract textDocuments: readonly TextDocument[];
+    abstract notebookDocuments: readonly NotebookDocument[];
+    abstract readonly onDidOpenTextDocument: Event<TextDocument>;
+    abstract readonly onDidCloseTextDocument: Event<TextDocument>;
+    abstract readonly onDidOpenNotebookDocument: Event<NotebookDocument>;
+    abstract readonly onDidCloseNotebookDocument: Event<NotebookDocument>;
+    abstract readonly onDidChangeTextDocument: Event<TextDocumentChangeEvent>;
+    abstract readonly onDidChangeWorkspaceFolders: Event<WorkspaceFoldersChangeEvent>;
+    abstract readonly onDidChangeNotebookDocument: Event<NotebookDocumentChangeEvent>;
+    abstract readonly onDidChangeTextEditorSelection: Event<TextEditorSelectionChangeEvent>;
+    abstract openTextDocument(uri: Uri): Promise<TextDocument>;
+    abstract fs: FileSystem;
+    abstract showTextDocument(document: TextDocument): Promise<void>;
+    abstract openNotebookDocument(uri: Uri): Promise<NotebookDocument>;
+    abstract openNotebookDocument(notebookType: string, content?: NotebookData): Promise<NotebookDocument>;
+    abstract getWorkspaceFolders(): URI[];
+    abstract ensureWorkspaceIsFullyLoaded(): Promise<void>;
+    abstract showWorkspaceFolderPicker(): Promise<WorkspaceFolder | undefined>;
+    abstract getWorkspaceFolderName(workspaceFolderUri: URI): string;
+    abstract applyEdit(edit: WorkspaceEdit): Thenable<boolean>;
+    asRelativePath(pathOrUri: string | Uri, includeWorkspaceFolder?: boolean): string;
+    openTextDocumentAndSnapshot(uri: Uri): Promise<TextDocumentSnapshot>;
+    openNotebookDocumentAndSnapshot(uri: Uri, format: 'xml' | 'json' | 'text'): Promise<NotebookDocumentSnapshot>;
+    getWorkspaceFolder(resource: URI): URI | undefined;
+}
+export declare function getWorkspaceFileDisplayPath(workspaceService: IWorkspaceService, file: URI): string;
+export declare class NullWorkspaceService extends AbstractWorkspaceService implements IDisposable {
+    fs: FileSystem;
+    private readonly disposables;
+    readonly didOpenTextDocumentEmitter: Emitter<TextDocument>;
+    readonly didCloseTextDocumentEmitter: Emitter<TextDocument>;
+    readonly didOpenNotebookDocumentEmitter: Emitter<NotebookDocument>;
+    readonly didCloseNotebookDocumentEmitter: Emitter<NotebookDocument>;
+    readonly didChangeTextDocumentEmitter: Emitter<TextDocumentChangeEvent>;
+    readonly didChangeWorkspaceFoldersEmitter: Emitter<WorkspaceFoldersChangeEvent>;
+    readonly didChangeNotebookDocumentEmitter: Emitter<NotebookDocumentChangeEvent>;
+    readonly didChangeTextEditorSelectionEmitter: Emitter<TextEditorSelectionChangeEvent>;
+    readonly onDidChangeTextDocument: import("../../../util/vs/base/common/event").Event<TextDocumentChangeEvent>;
+    readonly onDidCloseTextDocument: import("../../../util/vs/base/common/event").Event<TextDocument>;
+    readonly onDidOpenNotebookDocument: import("../../../util/vs/base/common/event").Event<NotebookDocument>;
+    readonly onDidCloseNotebookDocument: import("../../../util/vs/base/common/event").Event<NotebookDocument>;
+    readonly onDidOpenTextDocument: import("../../../util/vs/base/common/event").Event<TextDocument>;
+    readonly onDidChangeWorkspaceFolders: import("../../../util/vs/base/common/event").Event<WorkspaceFoldersChangeEvent>;
+    readonly onDidChangeNotebookDocument: import("../../../util/vs/base/common/event").Event<NotebookDocumentChangeEvent>;
+    readonly onDidChangeTextEditorSelection: import("../../../util/vs/base/common/event").Event<TextEditorSelectionChangeEvent>;
+    private readonly workspaceFolder;
+    private readonly _textDocuments;
+    private readonly _notebookDocuments;
+    constructor(workspaceFolders?: URI[], textDocuments?: TextDocument[], notebookDocuments?: NotebookDocument[]);
+    get textDocuments(): TextDocument[];
+    showTextDocument(document: TextDocument): Promise<void>;
+    openTextDocument(uri: Uri): Promise<TextDocument>;
+    openNotebookDocument(uri: Uri): Promise<NotebookDocument>;
+    openNotebookDocument(notebookType: string, content?: NotebookData): Promise<NotebookDocument>;
+    get notebookDocuments(): readonly NotebookDocument[];
+    getWorkspaceFolders(): URI[];
+    getWorkspaceFolderName(workspaceFolderUri: URI): string;
+    ensureWorkspaceIsFullyLoaded(): Promise<void>;
+    showWorkspaceFolderPicker(): Promise<undefined>;
+    applyEdit(): Promise<boolean>;
+    dispose(): void;
+}
+//# sourceMappingURL=workspaceService.d.ts.map