node-llama-cpp 3.0.0-beta.4 → 3.0.0-beta.41

package/dist/utils/createModelDownloader.js

@@ -0,0 +1,273 @@
+import process from "process";
+import path from "path";
+import os from "os";
+import { downloadFile, downloadSequence } from "ipull";
+import fs from "fs-extra";
+import { normalizeGgufDownloadUrl } from "../gguf/utils/normalizeGgufDownloadUrl.js";
+import { createSplitPartFilename, resolveSplitGgufParts } from "../gguf/utils/resolveSplitGgufParts.js";
+import { getFilenameForBinarySplitGgufPartUrls, resolveBinarySplitGgufPartUrls } from "../gguf/utils/resolveBinarySplitGgufPartUrls.js";
+import { cliModelsDirectory } from "../config.js";
+import { safeEventCallback } from "./safeEventCallback.js";
+/**
+ * Create a model downloader to download a model from a URL.
+ * Uses [`ipull`](https://github.com/ido-pluto/ipull) to download a model file as fast as possible with parallel connections
+ * and other optimizations.
+ *
+ * If the url points to a `.gguf` file that is split into multiple parts (for example, `model-00001-of-00009.gguf`),
+ * all the parts will be downloaded to the specified directory.
+ *
+ * If the url points to a `.gguf` file that is binary spliced into multiple parts (for example, `model.gguf.part1of9`),
+ * all the parts will be spliced into a single file and be downloaded to the specified directory.
+ *
+ * If the url points to a `.gguf` file that is not split or binary spliced (for example, `model.gguf`),
+ * the file will be downloaded to the specified directory.
+ * @example
+ * ```typescript
+ * import {fileURLToPath} from "url";
+ * import path from "path";
+ * import {createModelDownloader, getLlama} from "node-llama-cpp";
+ *
+ * const __dirname = path.dirname(fileURLToPath(import.meta.url));
+ *
+ * const downloader = await createModelDownloader({
+ *     modelUrl: "https://example.com/model.gguf",
+ *     dirPath: path.join(__dirname, "models")
+ * });
+ * const modelPath = await downloader.download();
+ *
+ * const llama = await getLlama();
+ * const model = llama.loadModel({
+ *     modelPath
+ * });
+ * ```
+ */
+export async function createModelDownloader(options) {
+    const downloader = ModelDownloader._create(options);
+    await downloader._init();
+    return downloader;
+}
+export class ModelDownloader {
+    /** @internal */ _modelUrl;
+    /** @internal */ _dirPath;
+    /** @internal */ _fileName;
+    /** @internal */ _headers;
+    /** @internal */ _showCliProgress;
+    /** @internal */ _onProgress;
+    /** @internal */ _tokens;
+    /** @internal */ _deleteTempFileOnCancel;
+    /** @internal */ _skipExisting;
+    /** @internal */ _parallelDownloads;
+    /** @internal */ _downloader;
+    /** @internal */ _specificFileDownloaders = [];
+    /** @internal */ _entrypointFilename;
+    /** @internal */ _splitBinaryParts;
+    /** @internal */ _totalFiles;
+    /** @internal */ _tryHeaders = [];
+    constructor({ modelUrl, dirPath = cliModelsDirectory, fileName, headers, showCliProgress = false, onProgress, deleteTempFileOnCancel = true, skipExisting = true, parallelDownloads = 4, tokens }) {
+        if (modelUrl == null || dirPath == null)
+            throw new Error("modelUrl and dirPath cannot be null");
+        this._modelUrl = normalizeGgufDownloadUrl(modelUrl);
+        this._dirPath = path.resolve(process.cwd(), dirPath);
+        this._fileName = fileName;
+        this._headers = headers;
+        this._showCliProgress = showCliProgress;
+        this._onProgress = safeEventCallback(onProgress);
+        this._deleteTempFileOnCancel = deleteTempFileOnCancel;
+        this._skipExisting = skipExisting;
+        this._parallelDownloads = parallelDownloads;
+        this._tokens = tokens;
+        this._onDownloadProgress = this._onDownloadProgress.bind(this);
+    }
+    /**
+     * The filename of the entrypoint file that should be used to load the model.
+     */
+    get entrypointFilename() {
+        return this._entrypointFilename;
+    }
+    /**
+     * The full path to the entrypoint file that should be used to load the model.
+     */
+    get entrypointFilePath() {
+        return path.join(this._dirPath, this.entrypointFilename);
+    }
+    /**
+     * If the model is binary spliced from multiple parts, this will return the number of those binary parts.
+     */
+    get splitBinaryParts() {
+        return this._splitBinaryParts;
+    }
+    /**
+     * The total number of files that will be saved to the directory.
+     * For split files, this will be the number of split parts, as multiple files will be saved.
+     * For binary-split files, this will be 1, as the parts will be spliced into a single file.
+     */
+    get totalFiles() {
+        return this._totalFiles;
+    }
+    get totalSize() {
+        return this._downloader.downloadStatues
+            .map(status => status.totalBytes)
+            .reduce((acc, totalBytes) => acc + totalBytes, 0);
+    }
+    get downloadedSize() {
+        return this._downloader.downloadStatues
+            .map(status => status.transferredBytes)
+            .reduce((acc, transferredBytes) => acc + transferredBytes, 0);
+    }
+    /**
+     * @returns The path to the entrypoint file that should be used to load the model
+     */
+    async download({ signal } = {}) {
+        if (signal?.aborted)
+            throw signal.reason;
+        if (this._skipExisting) {
+            if (this._specificFileDownloaders.length === 1 && await fs.pathExists(this.entrypointFilePath)) {
+                const fileStat = await fs.stat(this.entrypointFilePath);
+                if (this._specificFileDownloaders[0].status.totalBytes === fileStat.size)
+                    return this.entrypointFilePath;
+            }
+            else {
+                // TODO: skip existing split files
+            }
+        }
+        const onAbort = () => {
+            signal?.removeEventListener("abort", onAbort);
+            this.cancel();
+        };
+        if (signal != null)
+            signal.addEventListener("abort", onAbort);
+        try {
+            if (this._onProgress)
+                this._downloader.on("progress", this._onDownloadProgress);
+            await this._downloader.download();
+        }
+        catch (err) {
+            if (signal?.aborted)
+                throw signal.reason;
+            throw err;
+        }
+        finally {
+            if (this._onProgress)
+                this._downloader.off("progress", this._onDownloadProgress);
+            if (signal != null)
+                signal.removeEventListener("abort", onAbort);
+        }
+        return this.entrypointFilePath;
+    }
+    async cancel({ deleteTempFile = this._deleteTempFileOnCancel } = {}) {
+        for (const downloader of this._specificFileDownloaders) {
+            if (deleteTempFile)
+                await downloader.closeAndDeleteFile();
+            else
+                await downloader.close();
+        }
+        if (this._downloader !== this._specificFileDownloaders[0])
+            await this._downloader?.close();
+    }
+    /** @internal */
+    _onDownloadProgress() {
+        this._onProgress?.({
+            totalSize: this.totalSize,
+            downloadedSize: this.downloadedSize
+        });
+    }
+    /** @internal */
+    async resolveTryHeaders() {
+        if (this._tokens == null)
+            return;
+        const { huggingFace } = this._tokens;
+        const [hfToken] = await Promise.all([
+            resolveHfToken(huggingFace)
+        ]);
+        if (hfToken != null && hfToken !== "")
+            this._tryHeaders?.push({
+                ...(this._headers ?? {}),
+                "Authorization": `Bearer ${hfToken}`
+            });
+    }
+    /** @internal */
+    async _init() {
+        await this.resolveTryHeaders();
+        const binarySplitPartUrls = resolveBinarySplitGgufPartUrls(this._modelUrl);
+        await fs.ensureDir(this._dirPath);
+        if (binarySplitPartUrls instanceof Array) {
+            this._downloader = await downloadFile({
+                partURLs: binarySplitPartUrls,
+                directory: this._dirPath,
+                fileName: this._fileName ?? getFilenameForBinarySplitGgufPartUrls(binarySplitPartUrls),
+                cliProgress: this._showCliProgress,
+                headers: this._headers ?? {},
+                tryHeaders: this._tryHeaders.slice()
+            });
+            this._specificFileDownloaders.push(this._downloader);
+            this._entrypointFilename = this._downloader.fileName;
+            this._splitBinaryParts = binarySplitPartUrls.length;
+            this._totalFiles = 1;
+            if (this._downloader.fileName == null || this._downloader.fileName === "")
+                throw new Error("Failed to get the file name from the given URL");
+            return;
+        }
+        const splitGgufPartUrls = resolveSplitGgufParts(this._modelUrl);
+        if (splitGgufPartUrls.length === 1) {
+            this._downloader = await downloadFile({
+                url: splitGgufPartUrls[0],
+                directory: this._dirPath,
+                fileName: this._fileName ?? undefined,
+                cliProgress: this._showCliProgress,
+                headers: this._headers ?? {},
+                tryHeaders: this._tryHeaders.slice()
+            });
+            this._specificFileDownloaders.push(this._downloader);
+            this._entrypointFilename = this._downloader.fileName;
+            this._totalFiles = 1;
+            if (this._downloader.fileName == null || this._downloader.fileName === "")
+                throw new Error("Failed to get the file name from the given URL");
+            return;
+        }
+        const partDownloads = splitGgufPartUrls.map((url, index) => downloadFile({
+            url,
+            directory: this._dirPath,
+            fileName: this._fileName != null
+                ? createSplitPartFilename(this._fileName, index + 1, splitGgufPartUrls.length)
+                : undefined,
+            headers: this._headers ?? {},
+            tryHeaders: this._tryHeaders.slice()
+        }));
+        this._downloader = await downloadSequence({
+            cliProgress: this._showCliProgress,
+            parallelDownloads: this._parallelDownloads
+        }, ...partDownloads);
+        const firstDownload = await partDownloads[0];
+        this._specificFileDownloaders = await Promise.all(partDownloads);
+        this._entrypointFilename = firstDownload.fileName;
+        this._totalFiles = partDownloads.length;
+        if (this._entrypointFilename == null || this._entrypointFilename === "")
+            throw new Error("Failed to get the file name from the given URL");
+        return;
+    }
+    /** @internal */
+    static _create(options) {
+        return new ModelDownloader(options);
+    }
+}
+async function resolveHfToken(providedToken) {
+    if (providedToken !== null)
+        return providedToken;
+    if (process.env.HF_TOKEN != null)
+        return process.env.HF_TOKEN;
+    const hfHomePath = process.env.HF_HOME ||
+        path.join(process.env.XDG_CACHE_HOME || path.join(os.homedir(), ".cache"), "huggingface");
+    const hfTokenPath = process.env.HF_TOKEN_PATH || path.join(hfHomePath, "token");
+    try {
+        if (await fs.pathExists(hfTokenPath)) {
+            const token = (await fs.readFile(hfTokenPath, "utf8")).trim();
+            if (token !== "")
+                return token;
+        }
+    }
+    catch (err) {
+        // do nothing
+    }
+    return undefined;
+}
+//# sourceMappingURL=createModelDownloader.js.map

package/dist/utils/createModelDownloader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"createModelDownloader.js","sourceRoot":"","sources":["../../src/utils/createModelDownloader.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAoD,YAAY,EAAE,gBAAgB,EAAC,MAAM,OAAO,CAAC;AACxG,OAAO,EAAE,MAAM,UAAU,CAAC;AAC1B,OAAO,EAAC,wBAAwB,EAAC,MAAM,2CAA2C,CAAC;AACnF,OAAO,EAAC,uBAAuB,EAAE,qBAAqB,EAAC,MAAM,wCAAwC,CAAC;AACtG,OAAO,EAAC,qCAAqC,EAAE,8BAA8B,EAAC,MAAM,iDAAiD,CAAC;AACtI,OAAO,EAAC,kBAAkB,EAAC,MAAM,cAAc,CAAC;AAChD,OAAO,EAAC,iBAAiB,EAAC,MAAM,wBAAwB,CAAC;AA+CzD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,OAA+B;IACvE,MAAM,UAAU,GAAG,eAAe,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACpD,MAAM,UAAU,CAAC,KAAK,EAAE,CAAC;IACzB,OAAO,UAAU,CAAC;AACtB,CAAC;AAED,MAAM,OAAO,eAAe;IACxB,gBAAgB,CAAkB,SAAS,CAAS;IACpD,gBAAgB,CAAkB,QAAQ,CAAS;IACnD,gBAAgB,CAAkB,SAAS,CAAU;IACrD,gBAAgB,CAAkB,QAAQ,CAA0B;IACpE,gBAAgB,CAAkB,gBAAgB,CAAU;IAC5D,gBAAgB,CAAkB,WAAW,CAAwC;IACrF,gBAAgB,CAAkB,OAAO,CAAoC;IAC7E,gBAAgB,CAAkB,uBAAuB,CAAU;IACnE,gBAAgB,CAAkB,aAAa,CAAU;IACzD,gBAAgB,CAAkB,kBAAkB,CAAS;IAE7D,gBAAgB,CAAS,WAAW,CAAsD;IAC1F,gBAAgB,CAAS,wBAAwB,GAA2B,EAAE,CAAC;IAC/E,gBAAgB,CAAS,mBAAmB,CAAU;IACtD,gBAAgB,CAAS,iBAAiB,CAAU;IACpD,gBAAgB,CAAS,WAAW,CAAU;IAC9C,gBAAgB,CAAS,WAAW,GAA6B,EAAE,CAAC;IAEpE,YAAoB,EAChB,QAAQ,EAAE,OAAO,GAAG,kBAAkB,EAAE,QAAQ,EAAE,OAAO,EAAE,eAAe,GAAG,KAAK,EAAE,UAAU,EAAE,sBAAsB,GAAG,IAAI,EAC7H,YAAY,GAAG,IAAI,EAAE,iBAAiB,GAAG,CAAC,EAAE,MAAM,EAC7B;QACrB,IAAI,QAAQ,IAAI,IAAI,IAAI,OAAO,IAAI,IAAI;YACnC,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QAE3D,IAAI,CAAC,SAAS,GAAG,wBAAwB,CAAC,QAAQ,CAAC,CAAC;QACpD,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,OAAO,CAAC,CAAC;QACrD,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;QAC1B,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;QACxB,IAAI,CAAC,gBAAgB,GAAG,eAAe,CAAC;QACxC,IAAI,CAAC,WAAW,GAAG,iBAAiB,CAAC,UAAU,CAAC,CAAC;QACjD,IAAI,CAAC,uBAAuB,GAAG,sBAAsB,CAAC;QACtD,IAAI,CAAC,aAAa,GAAG,YAAY,CAAC;QAClC,IAAI,CAAC,kBAAkB,GAAG,iBAAiB,CAAC;QAC5C,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QAEtB,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACnE,CAAC;IAED;;OAEG;IACH,IAAW,kBAAkB;QACzB,OAAO,IAAI,CAAC,mBAAoB,CAAC;IACrC,CAAC;IAED;;OAEG;IACH,IAAW,kBAAkB;QACzB,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,kBAAkB,CAAC,CAAC;IAC7D,CAAC;IAED;;OAEG;IACH,IAAW,gBAAgB;QACvB,OAAO,IAAI,CAAC,iBAAiB,CAAC;IAClC,CAAC;IAED;;;;OAIG;IACH,IAAW,UAAU;QACjB,OAAO,IAAI,CAAC,WAAY,CAAC;IAC7B,CAAC;IAED,IAAW,SAAS;QAChB,OAAO,IAAI,CAAC,WAAY,CAAC,eAAe;aACnC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC;aAChC,MAAM,CAAC,CAAC,GAAG,EAAE,UAAU,EAAE,EAAE,CAAC,GAAG,GAAG,UAAU,EAAE,CAAC,CAAC,CAAC;IAC1D,CAAC;IAED,IAAW,cAAc;QACrB,OAAO,IAAI,CAAC,WAAY,CAAC,eAAe;aACnC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,gBAAgB,CAAC;aACtC,MAAM,CAAC,CAAC,GAAG,EAAE,gBAAgB,EAAE,EAAE,CAAC,GAAG,GAAG,gBAAgB,EAAE,CAAC,CAAC,CAAC;IACtE,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,QAAQ,CAAC,EAClB,MAAM,KAGN,EAAE;QACF,IAAI,MAAM,EAAE,OAAO;YACf,MAAM,MAAM,CAAC,MAAM,CAAC;QAExB,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACrB,IAAI,IAAI,CAAC,wBAAwB,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,kBAAkB,CAAC,EAAE,CAAC;gBAC7F,MAAM,QAAQ,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;gBAExD,IAAI,IAAI,CAAC,wBAAwB,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,KAAK,QAAQ,CAAC,IAAI;oBACpE,OAAO,IAAI,CAAC,kBAAkB,CAAC;YACvC,CAAC;iBAAM,CAAC;gBACJ,kCAAkC;YACtC,CAAC;QACL,CAAC;QAED,MAAM,OAAO,GAAG,GAAG,EAAE;YACjB,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAC9C,IAAI,CAAC,MAAM,EAAE,CAAC;QAClB,CAAC,CAAC;QAEF,IAAI,MAAM,IAAI,IAAI;YACd,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAE9C,IAAI,CAAC;YACD,IAAI,IAAI,CAAC,WAAW;gBAChB,IAAI,CAAC,WAAY,CAAC,EAAE,CAAC,UAAU,EAAE,IAAI,CAAC,mBAAmB,CAAC,CAAC;YAE/D,MAAM,IAAI,CAAC,WAAY,CAAC,QAAQ,EAAE,CAAC;QACvC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,IAAI,MAAM,EAAE,OAAO;gBACf,MAAM,MAAM,CAAC,MAAM,CAAC;YAExB,MAAM,GAAG,CAAC;QACd,CAAC;gBAAS,CAAC;YACP,IAAI,IAAI,CAAC,WAAW;gBAChB,IAAI,CAAC,WAAY,CAAC,GAAG,CAAC,UAAU,EAAE,IAAI,CAAC,mBAAmB,CAAC,CAAC;YAEhE,IAAI,MAAM,IAAI,IAAI;gBACd,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QACrD,CAAC;QAED,OAAO,IAAI,CAAC,kBAAkB,CAAC;IACnC,CAAC;IAEM,KAAK,CAAC,MAAM,CAAC,EAChB,cAAc,GAAG,IAAI,CAAC,uBAAuB,KAQ7C,EAAE;QACF,KAAK,MAAM,UAAU,IAAI,IAAI,CAAC,wBAAwB,EAAE,CAAC;YACrD,IAAI,cAAc;gBACd,MAAM,UAAU,CAAC,kBAAkB,EAAE,CAAC;;gBAEtC,MAAM,UAAU,CAAC,KAAK,EAAE,CAAC;QACjC,CAAC;QAED,IAAI,IAAI,CAAC,WAAW,KAAK,IAAI,CAAC,wBAAwB,CAAC,CAAC,CAAC;YACrD,MAAM,IAAI,CAAC,WAAW,EAAE,KAAK,EAAE,CAAC;IACxC,CAAC;IAED,gBAAgB;IACR,mBAAmB;QACvB,IAAI,CAAC,WAAW,EAAE,CAAC;YACf,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,cAAc,EAAE,IAAI,CAAC,cAAc;SACtC,CAAC,CAAC;IACP,CAAC;IAED,gBAAgB;IACR,KAAK,CAAC,iBAAiB;QAC3B,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI;YACpB,OAAO;QAEX,MAAM,EAAC,WAAW,EAAC,GAAG,IAAI,CAAC,OAAO,CAAC;QAEnC,MAAM,CACF,OAAO,CACV,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAClB,cAAc,CAAC,WAAW,CAAC;SAC9B,CAAC,CAAC;QAEH,IAAI,OAAO,IAAI,IAAI,IAAI,OAAO,KAAK,EAAE;YACjC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC;gBACnB,GAAG,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC;gBACxB,eAAe,EAAE,UAAU,OAAO,EAAE;aACvC,CAAC,CAAC;IACX,CAAC;IAED,gBAAgB;IACT,KAAK,CAAC,KAAK;QACd,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC/B,MAAM,mBAAmB,GAAG,8BAA8B,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAE3E,MAAM,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAClC,IAAI,mBAAmB,YAAY,KAAK,EAAE,CAAC;YACvC,IAAI,CAAC,WAAW,GAAG,MAAM,YAAY,CAAC;gBAClC,QAAQ,EAAE,mBAAmB;gBAC7B,SAAS,EAAE,IAAI,CAAC,QAAQ;gBACxB,QAAQ,EAAE,IAAI,CAAC,SAAS,IAAI,qCAAqC,CAAC,mBAAmB,CAAC;gBACtF,WAAW,EAAE,IAAI,CAAC,gBAAgB;gBAClC,OAAO,EAAE,IAAI,CAAC,QAAQ,IAAI,EAAE;gBAC5B,UAAU,EAAE,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE;aACvC,CAAC,CAAC;YACH,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAErD,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC;YACrD,IAAI,CAAC,iBAAiB,GAAG,mBAAmB,CAAC,MAAM,CAAC;YACpD,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC;YAErB,IAAI,IAAI,CAAC,WAAW,CAAC,QAAQ,IAAI,IAAI,IAAI,IAAI,CAAC,WAAW,CAAC,QAAQ,KAAK,EAAE;gBACrE,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;YAEtE,OAAO;QACX,CAAC;QAED,MAAM,iBAAiB,GAAG,qBAAqB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAChE,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjC,IAAI,CAAC,WAAW,GAAG,MAAM,YAAY,CAAC;gBAClC,GAAG,EAAE,iBAAiB,CAAC,CAAC,CAAC;gBACzB,SAAS,EAAE,IAAI,CAAC,QAAQ;gBACxB,QAAQ,EAAE,IAAI,CAAC,SAAS,IAAI,SAAS;gBACrC,WAAW,EAAE,IAAI,CAAC,gBAAgB;gBAClC,OAAO,EAAE,IAAI,CAAC,QAAQ,IAAI,EAAE;gBAC5B,UAAU,EAAE,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE;aACvC,CAAC,CAAC;YACH,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAErD,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC;YACrD,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC;YAErB,IAAI,IAAI,CAAC,WAAW,CAAC,QAAQ,IAAI,IAAI,IAAI,IAAI,CAAC,WAAW,CAAC,QAAQ,KAAK,EAAE;gBACrE,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;YAEtE,OAAO;QACX,CAAC;QAED,MAAM,aAAa,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,YAAY,CAAC;YACrE,GAAG;YACH,SAAS,EAAE,IAAI,CAAC,QAAQ;YACxB,QAAQ,EAAE,IAAI,CAAC,SAAS,IAAI,IAAI;gBAC5B,CAAC,CAAC,uBAAuB,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,GAAG,CAAC,EAAE,iBAAiB,CAAC,MAAM,CAAC;gBAC9E,CAAC,CAAC,SAAS;YACf,OAAO,EAAE,IAAI,CAAC,QAAQ,IAAI,EAAE;YAC5B,UAAU,EAAE,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE;SACvC,CAAC,CAAC,CAAC;QAEJ,IAAI,CAAC,WAAW,GAAG,MAAM,gBAAgB,CACrC;YACI,WAAW,EAAE,IAAI,CAAC,gBAAgB;YAClC,iBAAiB,EAAE,IAAI,CAAC,kBAAkB;SAC7C,EACD,GAAG,aAAa,CACnB,CAAC;QACF,MAAM,aAAa,GAAG,MAAM,aAAa,CAAC,CAAC,CAAC,CAAC;QAC7C,IAAI,CAAC,wBAAwB,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;QAEjE,IAAI,CAAC,mBAAmB,GAAG,aAAa,CAAC,QAAQ,CAAC;QAClD,IAAI,CAAC,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC;QAExC,IAAI,IAAI,CAAC,mBAAmB,IAAI,IAAI,IAAI,IAAI,CAAC,mBAAmB,KAAK,EAAE;YACnE,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;QAEtE,OAAO;IACX,CAAC;IAED,gBAAgB;IACT,MAAM,CAAC,OAAO,CAAC,OAA+B;QACjD,OAAO,IAAI,eAAe,CAAC,OAAO,CAAC,CAAC;IACxC,CAAC;CACJ;AAED,KAAK,UAAU,cAAc,CAAC,aAAsB;IAChD,IAAI,aAAa,KAAK,IAAI;QACtB,OAAO,aAAa,CAAC;IAEzB,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,IAAI,IAAI;QAC5B,OAAO,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;IAEhC,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,OAAO;QAClC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,QAAQ,CAAC,EAAE,aAAa,CAAC,CAAC;IAE9F,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,aAAa,IAAI,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IAChF,IAAI,CAAC;QACD,IAAI,MAAM,EAAE,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;YACnC,MAAM,KAAK,GAAG,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC9D,IAAI,KAAK,KAAK,EAAE;gBACZ,OAAO,KAAK,CAAC;QACrB,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACX,aAAa;IACjB,CAAC;IAED,OAAO,SAAS,CAAC;AACrB,CAAC"}

package/dist/utils/findBestOption.d.ts

@@ -0,0 +1,4 @@
+export declare function findBestOption<const O>({ generator, score }: {
+    generator: () => Generator<O>;
+    score: (option: O) => number | null;
+}): O | null;

package/dist/utils/findBestOption.js

@@ -0,0 +1,15 @@
+export function findBestOption({ generator, score }) {
+    let bestOption = null;
+    let bestScore = null;
+    for (const option of generator()) {
+        const currentScore = score(option);
+        if (currentScore === Infinity)
+            return option;
+        if (currentScore != null && (bestScore == null || currentScore > bestScore)) {
+            bestOption = option;
+            bestScore = currentScore;
+        }
+    }
+    return bestOption;
+}
+//# sourceMappingURL=findBestOption.js.map

package/dist/utils/findBestOption.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"findBestOption.js","sourceRoot":"","sources":["../../src/utils/findBestOption.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,cAAc,CAAU,EAAC,SAAS,EAAE,KAAK,EAGxD;IACG,IAAI,UAAU,GAAa,IAAI,CAAC;IAChC,IAAI,SAAS,GAAkB,IAAI,CAAC;IAEpC,KAAK,MAAM,MAAM,IAAI,SAAS,EAAE,EAAE,CAAC;QAC/B,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;QAEnC,IAAI,YAAY,KAAK,QAAQ;YACzB,OAAO,MAAM,CAAC;QAElB,IAAI,YAAY,IAAI,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,IAAI,YAAY,GAAG,SAAS,CAAC,EAAE,CAAC;YAC1E,UAAU,GAAG,MAAM,CAAC;YACpB,SAAS,GAAG,YAAY,CAAC;QAC7B,CAAC;IACL,CAAC;IAED,OAAO,UAAU,CAAC;AACtB,CAAC"}

package/dist/utils/findCharacterRemovalCountToFitChatHistoryInContext.d.ts

@@ -4,6 +4,7 @@ export declare function findCharacterRemovalCountToFitChatHistoryInContext({ com
     compressChatHistory(options: {
         chatHistory: readonly ChatHistoryItem[];
         charactersToRemove: number;
+        estimatedCharactersPerToken: number;
     }): ChatHistoryItem[] | Promise<ChatHistoryItem[]>;
     chatHistory: ChatHistoryItem[];
     tokensCountToFit: number;

package/dist/utils/findCharacterRemovalCountToFitChatHistoryInContext.js

@@ -1,7 +1,8 @@
 export async function findCharacterRemovalCountToFitChatHistoryInContext({ compressChatHistory, chatHistory, tokensCountToFit, tokenizer, chatWrapper, initialCharactersRemovalCount = 0, estimatedCharactersPerToken = 5, maxDecompressionAttempts = 2 }) {
+    let currentEstimatedCharactersPerToken = estimatedCharactersPerToken;
     function getTokensCountForChatHistory(chatHistory) {
-        const { contextText } = chatWrapper.generateContextText(chatHistory);
-        return contextText.tokenize(tokenizer).length;
+        const { contextText } = chatWrapper.generateContextState({ chatHistory });
+        return contextText.tokenize(tokenizer, "trimLeadingSpace").length;
     }
     async function getResultForCharacterRemovalCount(characterRemovalCount) {
         if (characterRemovalCount === 0)
@@ -12,7 +13,8 @@ export async function findCharacterRemovalCountToFitChatHistoryInContext({ compr
             };
         const compressedHistory = await compressChatHistory({
             chatHistory,
-            charactersToRemove: characterRemovalCount
+            charactersToRemove: characterRemovalCount,
+            estimatedCharactersPerToken: currentEstimatedCharactersPerToken
         });
         return {
             compressedHistory,
@@ -22,22 +24,24 @@ export async function findCharacterRemovalCountToFitChatHistoryInContext({ compr
     }
     let latestCompressionAttempt = await getResultForCharacterRemovalCount(initialCharactersRemovalCount);
     const firstCompressionAttempt = latestCompressionAttempt;
-    let currentEstimatedCharactersPerToken = estimatedCharactersPerToken;
     if (latestCompressionAttempt.tokensCount === tokensCountToFit ||
         (latestCompressionAttempt.tokensCount < tokensCountToFit && latestCompressionAttempt.characterRemovalCount === 0))
         return {
             removedCharactersCount: initialCharactersRemovalCount,
             compressedChatHistory: latestCompressionAttempt.compressedHistory
         };
-    for (let compressionAttempts = 0, decompressionAttempts = 0; latestCompressionAttempt.tokensCount !== tokensCountToFit;) {
+    let bestCompressionAttempt = latestCompressionAttempt;
+    for (let compressionAttempts = 0, decompressionAttempts = 0; bestCompressionAttempt.tokensCount !== tokensCountToFit;) {
         if (compressionAttempts > 0) {
-            currentEstimatedCharactersPerToken =
-                Math.abs(latestCompressionAttempt.characterRemovalCount - firstCompressionAttempt.characterRemovalCount) /
-                    Math.abs(latestCompressionAttempt.tokensCount - firstCompressionAttempt.tokensCount);
-            if (!Number.isFinite(currentEstimatedCharactersPerToken))
+            if (latestCompressionAttempt.tokensCount != firstCompressionAttempt.tokensCount &&
+                latestCompressionAttempt.characterRemovalCount != firstCompressionAttempt.characterRemovalCount)
+                currentEstimatedCharactersPerToken =
+                    Math.abs(latestCompressionAttempt.characterRemovalCount - firstCompressionAttempt.characterRemovalCount) /
+                        Math.abs(latestCompressionAttempt.tokensCount - firstCompressionAttempt.tokensCount);
+            if (!Number.isFinite(currentEstimatedCharactersPerToken) || currentEstimatedCharactersPerToken === 0)
                 currentEstimatedCharactersPerToken = estimatedCharactersPerToken;
         }
-        const tokensLeftToRemove = tokensCountToFit - latestCompressionAttempt.tokensCount;
+        const tokensLeftToRemove = latestCompressionAttempt.tokensCount - tokensCountToFit;
         let additionalCharactersToRemove = Math.round(tokensLeftToRemove * currentEstimatedCharactersPerToken);
         if (additionalCharactersToRemove === 0) {
             if (tokensLeftToRemove > 0)
@@ -52,10 +56,17 @@ export async function findCharacterRemovalCountToFitChatHistoryInContext({ compr
         if (decompressionAttempts >= maxDecompressionAttempts)
             break;
         latestCompressionAttempt = await getResultForCharacterRemovalCount(latestCompressionAttempt.characterRemovalCount + additionalCharactersToRemove);
+        if ((bestCompressionAttempt.tokensCount > tokensCountToFit &&
+            latestCompressionAttempt.tokensCount <= bestCompressionAttempt.tokensCount) || (bestCompressionAttempt.tokensCount < tokensCountToFit &&
+            latestCompressionAttempt.tokensCount < tokensCountToFit &&
+            latestCompressionAttempt.tokensCount > bestCompressionAttempt.tokensCount) || (bestCompressionAttempt.tokensCount <= tokensCountToFit &&
+            latestCompressionAttempt.tokensCount <= tokensCountToFit &&
+            latestCompressionAttempt.characterRemovalCount < bestCompressionAttempt.characterRemovalCount))
+            bestCompressionAttempt = latestCompressionAttempt;
     }
     return {
-        removedCharactersCount: latestCompressionAttempt.characterRemovalCount,
-        compressedChatHistory: latestCompressionAttempt.compressedHistory
+        removedCharactersCount: bestCompressionAttempt.characterRemovalCount,
+        compressedChatHistory: bestCompressionAttempt.compressedHistory
     };
 }
 //# sourceMappingURL=findCharacterRemovalCountToFitChatHistoryInContext.js.map

package/dist/utils/findCharacterRemovalCountToFitChatHistoryInContext.js.map

@@ -1 +1 @@
-{"version":3,"file":"findCharacterRemovalCountToFitChatHistoryInContext.js","sourceRoot":"","sources":["../../src/utils/findCharacterRemovalCountToFitChatHistoryInContext.ts"],"names":[],"mappings":"AAGA,MAAM,CAAC,KAAK,UAAU,kDAAkD,CAAC,EACrE,mBAAmB,EACnB,WAAW,EACX,gBAAgB,EAChB,SAAS,EACT,WAAW,EACX,6BAA6B,GAAG,CAAC,EACjC,2BAA2B,GAAG,CAAC,EAC/B,wBAAwB,GAAG,CAAC,EAY/B;IAIG,SAAS,4BAA4B,CAAC,WAAuC;QACzE,MAAM,EAAC,WAAW,EAAC,GAAG,WAAW,CAAC,mBAAmB,CAAC,WAAW,CAAC,CAAC;QACnE,OAAO,WAAW,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC;IAClD,CAAC;IAED,KAAK,UAAU,iCAAiC,CAAC,qBAA6B;QAC1E,IAAI,qBAAqB,KAAK,CAAC;YAC3B,OAAO;gBACH,iBAAiB,EAAE,WAAW;gBAC9B,WAAW,EAAE,4BAA4B,CAAC,WAAW,CAAC;gBACtD,qBAAqB;aACxB,CAAC;QAEN,MAAM,iBAAiB,GAAG,MAAM,mBAAmB,CAAC;YAChD,WAAW;YACX,kBAAkB,EAAE,qBAAqB;SAC5C,CAAC,CAAC;QAEH,OAAO;YACH,iBAAiB;YACjB,WAAW,EAAE,4BAA4B,CAAC,iBAAiB,CAAC;YAC5D,qBAAqB;SACxB,CAAC;IACN,CAAC;IAED,IAAI,wBAAwB,GAAG,MAAM,iCAAiC,CAAC,6BAA6B,CAAC,CAAC;IACtG,MAAM,uBAAuB,GAAG,wBAAwB,CAAC;IACzD,IAAI,kCAAkC,GAAG,2BAA2B,CAAC;IAErE,IAAI,wBAAwB,CAAC,WAAW,KAAK,gBAAgB;QACzD,CAAC,wBAAwB,CAAC,WAAW,GAAG,gBAAgB,IAAI,wBAAwB,CAAC,qBAAqB,KAAK,CAAC,CAAC;QAEjH,OAAO;YACH,sBAAsB,EAAE,6BAA6B;YACrD,qBAAqB,EAAE,wBAAwB,CAAC,iBAAiB;SACpE,CAAC;IAEN,KACI,IAAI,mBAAmB,GAAG,CAAC,EAAE,qBAAqB,GAAG,CAAC,EACtD,wBAAwB,CAAC,WAAW,KAAK,gBAAgB,GAC3D;QACE,IAAI,mBAAmB,GAAG,CAAC,EAAE;YACzB,kCAAkC;gBAC9B,IAAI,CAAC,GAAG,CAAC,wBAAwB,CAAC,qBAAqB,GAAG,uBAAuB,CAAC,qBAAqB,CAAC;oBACxG,IAAI,CAAC,GAAG,CAAC,wBAAwB,CAAC,WAAW,GAAG,uBAAuB,CAAC,WAAW,CAAC,CAAC;YAEzF,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,kCAAkC,CAAC;gBACpD,kCAAkC,GAAG,2BAA2B,CAAC;SACxE;QAED,MAAM,kBAAkB,GAAG,gBAAgB,GAAG,wBAAwB,CAAC,WAAW,CAAC;QACnF,IAAI,4BAA4B,GAAG,IAAI,CAAC,KAAK,CAAC,kBAAkB,GAAG,kCAAkC,CAAC,CAAC;QAEvG,IAAI,4BAA4B,KAAK,CAAC,EAAE;YACpC,IAAI,kBAAkB,GAAG,CAAC;gBACtB,4BAA4B,GAAG,CAAC,CAAC;iBAChC,IAAI,kBAAkB,GAAG,CAAC;gBAC3B,4BAA4B,GAAG,CAAC,CAAC,CAAC;SACzC;QAED,IAAI,kBAAkB,GAAG,CAAC;YACtB,mBAAmB,EAAE,CAAC;aACrB,IAAI,kBAAkB,GAAG,CAAC;YAC3B,qBAAqB,EAAE,CAAC;QAE5B,IAAI,qBAAqB,IAAI,wBAAwB;YACjD,MAAM;QAEV,wBAAwB,GAAG,MAAM,iCAAiC,CAC9D,wBAAwB,CAAC,qBAAqB,GAAG,4BAA4B,CAChF,CAAC;KACL;IAED,OAAO;QACH,sBAAsB,EAAE,wBAAwB,CAAC,qBAAqB;QACtE,qBAAqB,EAAE,wBAAwB,CAAC,iBAAiB;KACpE,CAAC;AACN,CAAC"}
+{"version":3,"file":"findCharacterRemovalCountToFitChatHistoryInContext.js","sourceRoot":"","sources":["../../src/utils/findCharacterRemovalCountToFitChatHistoryInContext.ts"],"names":[],"mappings":"AAGA,MAAM,CAAC,KAAK,UAAU,kDAAkD,CAAC,EACrE,mBAAmB,EACnB,WAAW,EACX,gBAAgB,EAChB,SAAS,EACT,WAAW,EACX,6BAA6B,GAAG,CAAC,EACjC,2BAA2B,GAAG,CAAC,EAC/B,wBAAwB,GAAG,CAAC,EAY/B;IAIG,IAAI,kCAAkC,GAAG,2BAA2B,CAAC;IAErE,SAAS,4BAA4B,CAAC,WAAuC;QACzE,MAAM,EAAC,WAAW,EAAC,GAAG,WAAW,CAAC,oBAAoB,CAAC,EAAC,WAAW,EAAC,CAAC,CAAC;QACtE,OAAO,WAAW,CAAC,QAAQ,CAAC,SAAS,EAAE,kBAAkB,CAAC,CAAC,MAAM,CAAC;IACtE,CAAC;IAED,KAAK,UAAU,iCAAiC,CAAC,qBAA6B;QAC1E,IAAI,qBAAqB,KAAK,CAAC;YAC3B,OAAO;gBACH,iBAAiB,EAAE,WAAW;gBAC9B,WAAW,EAAE,4BAA4B,CAAC,WAAW,CAAC;gBACtD,qBAAqB;aACxB,CAAC;QAEN,MAAM,iBAAiB,GAAG,MAAM,mBAAmB,CAAC;YAChD,WAAW;YACX,kBAAkB,EAAE,qBAAqB;YACzC,2BAA2B,EAAE,kCAAkC;SAClE,CAAC,CAAC;QAEH,OAAO;YACH,iBAAiB;YACjB,WAAW,EAAE,4BAA4B,CAAC,iBAAiB,CAAC;YAC5D,qBAAqB;SACxB,CAAC;IACN,CAAC;IAED,IAAI,wBAAwB,GAAG,MAAM,iCAAiC,CAAC,6BAA6B,CAAC,CAAC;IACtG,MAAM,uBAAuB,GAAG,wBAAwB,CAAC;IAEzD,IAAI,wBAAwB,CAAC,WAAW,KAAK,gBAAgB;QACzD,CAAC,wBAAwB,CAAC,WAAW,GAAG,gBAAgB,IAAI,wBAAwB,CAAC,qBAAqB,KAAK,CAAC,CAAC;QAEjH,OAAO;YACH,sBAAsB,EAAE,6BAA6B;YACrD,qBAAqB,EAAE,wBAAwB,CAAC,iBAAiB;SACpE,CAAC;IAEN,IAAI,sBAAsB,GAAG,wBAAwB,CAAC;IACtD,KACI,IAAI,mBAAmB,GAAG,CAAC,EAAE,qBAAqB,GAAG,CAAC,EACtD,sBAAsB,CAAC,WAAW,KAAK,gBAAgB,GACzD,CAAC;QACC,IAAI,mBAAmB,GAAG,CAAC,EAAE,CAAC;YAC1B,IAAI,wBAAwB,CAAC,WAAW,IAAI,uBAAuB,CAAC,WAAW;gBAC3E,wBAAwB,CAAC,qBAAqB,IAAI,uBAAuB,CAAC,qBAAqB;gBAE/F,kCAAkC;oBAC9B,IAAI,CAAC,GAAG,CAAC,wBAAwB,CAAC,qBAAqB,GAAG,uBAAuB,CAAC,qBAAqB,CAAC;wBACxG,IAAI,CAAC,GAAG,CAAC,wBAAwB,CAAC,WAAW,GAAG,uBAAuB,CAAC,WAAW,CAAC,CAAC;YAE7F,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,kCAAkC,CAAC,IAAI,kCAAkC,KAAK,CAAC;gBAChG,kCAAkC,GAAG,2BAA2B,CAAC;QACzE,CAAC;QAED,MAAM,kBAAkB,GAAG,wBAAwB,CAAC,WAAW,GAAG,gBAAgB,CAAC;QACnF,IAAI,4BAA4B,GAAG,IAAI,CAAC,KAAK,CAAC,kBAAkB,GAAG,kCAAkC,CAAC,CAAC;QAEvG,IAAI,4BAA4B,KAAK,CAAC,EAAE,CAAC;YACrC,IAAI,kBAAkB,GAAG,CAAC;gBACtB,4BAA4B,GAAG,CAAC,CAAC;iBAChC,IAAI,kBAAkB,GAAG,CAAC;gBAC3B,4BAA4B,GAAG,CAAC,CAAC,CAAC;QAC1C,CAAC;QAED,IAAI,kBAAkB,GAAG,CAAC;YACtB,mBAAmB,EAAE,CAAC;aACrB,IAAI,kBAAkB,GAAG,CAAC;YAC3B,qBAAqB,EAAE,CAAC;QAE5B,IAAI,qBAAqB,IAAI,wBAAwB;YACjD,MAAM;QAEV,wBAAwB,GAAG,MAAM,iCAAiC,CAC9D,wBAAwB,CAAC,qBAAqB,GAAG,4BAA4B,CAChF,CAAC;QAEF,IAAI,CACA,sBAAsB,CAAC,WAAW,GAAG,gBAAgB;YACrD,wBAAwB,CAAC,WAAW,IAAI,sBAAsB,CAAC,WAAW,CAC7E,IAAI,CACD,sBAAsB,CAAC,WAAW,GAAG,gBAAgB;YACrD,wBAAwB,CAAC,WAAW,GAAG,gBAAgB;YACvD,wBAAwB,CAAC,WAAW,GAAG,sBAAsB,CAAC,WAAW,CAC5E,IAAI,CACD,sBAAsB,CAAC,WAAW,IAAI,gBAAgB;YACtD,wBAAwB,CAAC,WAAW,IAAI,gBAAgB;YACxD,wBAAwB,CAAC,qBAAqB,GAAG,sBAAsB,CAAC,qBAAqB,CAChG;YACG,sBAAsB,GAAG,wBAAwB,CAAC;IAC1D,CAAC;IAED,OAAO;QACH,sBAAsB,EAAE,sBAAsB,CAAC,qBAAqB;QACpE,qBAAqB,EAAE,sBAAsB,CAAC,iBAAiB;KAClE,CAAC;AACN,CAAC"}

package/dist/utils/gbnfJson/GbnfGrammarGenerator.js.map

@@ -1 +1 @@
-{"version":3,"file":"GbnfGrammarGenerator.js","sourceRoot":"","sources":["../../../src/utils/gbnfJson/GbnfGrammarGenerator.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,oBAAoB;IACtB,KAAK,GAAG,IAAI,GAAG,EAAyB,CAAC;IACxC,MAAM,GAAW,CAAC,CAAC;IAEpB,gBAAgB;QACnB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC3B,IAAI,CAAC,MAAM,EAAE,CAAC;QAEd,OAAO,OAAO,MAAM,EAAE,CAAC;IAC3B,CAAC;IAEM,gBAAgB,CAAC,WAAmB;QACvC,MAAM,KAAK,GAAsC,CAAC;gBAC9C,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE,WAAW;aACvB,CAAC,CAAC;QAEH,KAAK,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE;YACpD,IAAI,OAAO,IAAI,IAAI;gBACf,SAAS;YAEb,KAAK,CAAC,IAAI,CAAC;gBACP,IAAI,EAAE,QAAQ;gBACd,OAAO;aACV,CAAC,CAAC;SACN;QAED,MAAM,WAAW,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,GAAG,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC;QAC5E,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEpC,OAAO,IAAI,CAAC;IAChB,CAAC;CACJ"}
+{"version":3,"file":"GbnfGrammarGenerator.js","sourceRoot":"","sources":["../../../src/utils/gbnfJson/GbnfGrammarGenerator.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,oBAAoB;IACtB,KAAK,GAAG,IAAI,GAAG,EAAyB,CAAC;IACxC,MAAM,GAAW,CAAC,CAAC;IAEpB,gBAAgB;QACnB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC3B,IAAI,CAAC,MAAM,EAAE,CAAC;QAEd,OAAO,OAAO,MAAM,EAAE,CAAC;IAC3B,CAAC;IAEM,gBAAgB,CAAC,WAAmB;QACvC,MAAM,KAAK,GAAsC,CAAC;gBAC9C,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE,WAAW;aACvB,CAAC,CAAC;QAEH,KAAK,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;YACrD,IAAI,OAAO,IAAI,IAAI;gBACf,SAAS;YAEb,KAAK,CAAC,IAAI,CAAC;gBACP,IAAI,EAAE,QAAQ;gBACd,OAAO;aACV,CAAC,CAAC;QACP,CAAC;QAED,MAAM,WAAW,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,GAAG,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC;QAC5E,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEpC,OAAO,IAAI,CAAC;IAChB,CAAC;CACJ"}

package/dist/utils/gbnfJson/getGbnfGrammarForGbnfJsonSchema.d.ts

@@ -0,0 +1,5 @@
+import { GbnfJsonSchema } from "./types.js";
+export declare function getGbnfGrammarForGbnfJsonSchema(schema: GbnfJsonSchema, { allowNewLines, scopePadSpaces }?: {
+    allowNewLines?: boolean;
+    scopePadSpaces?: number;
+}): string;

package/dist/utils/gbnfJson/getGbnfGrammarForGbnfJsonSchema.js

@@ -0,0 +1,11 @@
+import { getGbnfJsonTerminalForGbnfJsonSchema } from "./utils/getGbnfJsonTerminalForGbnfJsonSchema.js";
+import { GbnfGrammarGenerator } from "./GbnfGrammarGenerator.js";
+import { GbnfJsonScopeState } from "./utils/GbnfJsonScopeState.js";
+export function getGbnfGrammarForGbnfJsonSchema(schema, { allowNewLines = true, scopePadSpaces = 4 } = {}) {
+    const grammarGenerator = new GbnfGrammarGenerator();
+    const scopeState = new GbnfJsonScopeState({ allowNewLines, scopePadSpaces });
+    const rootTerminal = getGbnfJsonTerminalForGbnfJsonSchema(schema, grammarGenerator, scopeState);
+    const rootGrammar = rootTerminal.getGrammar(grammarGenerator);
+    return grammarGenerator.generateGbnfFile(rootGrammar + ` "${"\\n".repeat(4)}"` + " [\\n]*");
+}
+//# sourceMappingURL=getGbnfGrammarForGbnfJsonSchema.js.map

package/dist/utils/gbnfJson/getGbnfGrammarForGbnfJsonSchema.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"getGbnfGrammarForGbnfJsonSchema.js","sourceRoot":"","sources":["../../../src/utils/gbnfJson/getGbnfGrammarForGbnfJsonSchema.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,oCAAoC,EAAC,MAAM,iDAAiD,CAAC;AACrG,OAAO,EAAC,oBAAoB,EAAC,MAAM,2BAA2B,CAAC;AAC/D,OAAO,EAAC,kBAAkB,EAAC,MAAM,+BAA+B,CAAC;AAGjE,MAAM,UAAU,+BAA+B,CAAC,MAAsB,EAAE,EACpE,aAAa,GAAG,IAAI,EACpB,cAAc,GAAG,CAAC,KAIlB,EAAE;IACF,MAAM,gBAAgB,GAAG,IAAI,oBAAoB,EAAE,CAAC;IACpD,MAAM,UAAU,GAAG,IAAI,kBAAkB,CAAC,EAAC,aAAa,EAAE,cAAc,EAAC,CAAC,CAAC;IAC3E,MAAM,YAAY,GAAG,oCAAoC,CAAC,MAAM,EAAE,gBAAgB,EAAE,UAAU,CAAC,CAAC;IAChG,MAAM,WAAW,GAAG,YAAY,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC;IAE9D,OAAO,gBAAgB,CAAC,gBAAgB,CAAC,WAAW,GAAG,KAAK,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,GAAG,SAAS,CAAC,CAAC;AAChG,CAAC"}

package/dist/utils/gbnfJson/terminals/GbnfArray.d.ts

@@ -1,7 +1,9 @@
 import { GbnfTerminal } from "../GbnfTerminal.js";
 import { GbnfGrammarGenerator } from "../GbnfGrammarGenerator.js";
+import { GbnfJsonScopeState } from "../utils/GbnfJsonScopeState.js";
 export declare class GbnfArray extends GbnfTerminal {
     readonly items: GbnfTerminal;
-    constructor(items: GbnfTerminal);
+    readonly scopeState: GbnfJsonScopeState;
+    constructor(items: GbnfTerminal, scopeState?: GbnfJsonScopeState);
     getGrammar(grammarGenerator: GbnfGrammarGenerator): string;
 }

package/dist/utils/gbnfJson/terminals/GbnfArray.js

@@ -1,28 +1,33 @@
 import { GbnfTerminal } from "../GbnfTerminal.js";
+import { GbnfJsonScopeState } from "../utils/GbnfJsonScopeState.js";
 import { GbnfWhitespace } from "./GbnfWhitespace.js";
 import { GbnfGrammar } from "./GbnfGrammar.js";
 import { GbnfOr } from "./GbnfOr.js";
 export class GbnfArray extends GbnfTerminal {
     items;
-    constructor(items) {
+    scopeState;
+    constructor(items, scopeState = new GbnfJsonScopeState()) {
         super();
         this.items = items;
+        this.scopeState = scopeState;
     }
     getGrammar(grammarGenerator) {
-        const whitespaceRuleName = new GbnfWhitespace().resolve(grammarGenerator);
+        const getWhitespaceRuleName = (newScope, newLine) => (newScope
+            ? new GbnfWhitespace(this.scopeState.getForNewScope(), { newLine }).resolve(grammarGenerator)
+            : new GbnfWhitespace(this.scopeState, { newLine }).resolve(grammarGenerator));
         const itemsGrammarRuleName = this.items.resolve(grammarGenerator);
         return new GbnfGrammar([
-            '"["', whitespaceRuleName,
+            '"["', getWhitespaceRuleName(true, "before"),
             new GbnfOr([
                 new GbnfGrammar([
                     "(", itemsGrammarRuleName, ")",
-                    "(", '","', whitespaceRuleName, itemsGrammarRuleName, ")*"
+                    "(", '","', getWhitespaceRuleName(true, "before"), itemsGrammarRuleName, ")*"
                 ]),
                 new GbnfGrammar([
                     "(", itemsGrammarRuleName, ")?"
                 ])
             ]).getGrammar(grammarGenerator),
-            whitespaceRuleName, '"]"'
+            getWhitespaceRuleName(false, "before"), '"]"'
         ]).getGrammar();
     }
 }

package/dist/utils/gbnfJson/terminals/GbnfArray.js.map

@@ -1 +1 @@
-{"version":3,"file":"GbnfArray.js","sourceRoot":"","sources":["../../../../src/utils/gbnfJson/terminals/GbnfArray.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAEhD,OAAO,EAAC,cAAc,EAAC,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAC,WAAW,EAAC,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EAAC,MAAM,EAAC,MAAM,aAAa,CAAC;AAGnC,MAAM,OAAO,SAAU,SAAQ,YAAY;IACvB,KAAK,CAAe;IAEpC,YAAmB,KAAmB;QAClC,KAAK,EAAE,CAAC;QAER,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACvB,CAAC;IAED,UAAU,CAAC,gBAAsC;QAC7C,MAAM,kBAAkB,GAAG,IAAI,cAAc,EAAE,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;QAC1E,MAAM,oBAAoB,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;QAElE,OAAO,IAAI,WAAW,CAAC;YACnB,KAAK,EAAE,kBAAkB;YACzB,IAAI,MAAM,CAAC;gBACP,IAAI,WAAW,CAAC;oBACZ,GAAG,EAAE,oBAAoB,EAAE,GAAG;oBAC9B,GAAG,EAAE,KAAK,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,IAAI;iBAC7D,CAAC;gBACF,IAAI,WAAW,CAAC;oBACZ,GAAG,EAAE,oBAAoB,EAAE,IAAI;iBAClC,CAAC;aACL,CAAC,CAAC,UAAU,CAAC,gBAAgB,CAAC;YAC/B,kBAAkB,EAAE,KAAK;SAC5B,CAAC,CAAC,UAAU,EAAE,CAAC;IACpB,CAAC;CACJ"}
+{"version":3,"file":"GbnfArray.js","sourceRoot":"","sources":["../../../../src/utils/gbnfJson/terminals/GbnfArray.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAEhD,OAAO,EAAC,kBAAkB,EAAC,MAAM,gCAAgC,CAAC;AAClE,OAAO,EAAC,cAAc,EAAC,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAC,WAAW,EAAC,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EAAC,MAAM,EAAC,MAAM,aAAa,CAAC;AAGnC,MAAM,OAAO,SAAU,SAAQ,YAAY;IACvB,KAAK,CAAe;IACpB,UAAU,CAAqB;IAE/C,YAAmB,KAAmB,EAAE,aAAiC,IAAI,kBAAkB,EAAE;QAC7F,KAAK,EAAE,CAAC;QAER,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IACjC,CAAC;IAEM,UAAU,CAAC,gBAAsC;QACpD,MAAM,qBAAqB,GAAG,CAAC,QAAiB,EAAE,OAAmC,EAAE,EAAE,CAAC,CACtF,QAAQ;YACJ,CAAC,CAAC,IAAI,cAAc,CAAC,IAAI,CAAC,UAAU,CAAC,cAAc,EAAE,EAAE,EAAC,OAAO,EAAC,CAAC,CAAC,OAAO,CAAC,gBAAgB,CAAC;YAC3F,CAAC,CAAC,IAAI,cAAc,CAAC,IAAI,CAAC,UAAU,EAAE,EAAC,OAAO,EAAC,CAAC,CAAC,OAAO,CAAC,gBAAgB,CAAC,CACjF,CAAC;QACF,MAAM,oBAAoB,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;QAElE,OAAO,IAAI,WAAW,CAAC;YACnB,KAAK,EAAE,qBAAqB,CAAC,IAAI,EAAE,QAAQ,CAAC;YAC5C,IAAI,MAAM,CAAC;gBACP,IAAI,WAAW,CAAC;oBACZ,GAAG,EAAE,oBAAoB,EAAE,GAAG;oBAC9B,GAAG,EAAE,KAAK,EAAE,qBAAqB,CAAC,IAAI,EAAE,QAAQ,CAAC,EAAE,oBAAoB,EAAE,IAAI;iBAChF,CAAC;gBACF,IAAI,WAAW,CAAC;oBACZ,GAAG,EAAE,oBAAoB,EAAE,IAAI;iBAClC,CAAC;aACL,CAAC,CAAC,UAAU,CAAC,gBAAgB,CAAC;YAC/B,qBAAqB,CAAC,KAAK,EAAE,QAAQ,CAAC,EAAE,KAAK;SAChD,CAAC,CAAC,UAAU,EAAE,CAAC;IACpB,CAAC;CACJ"}

package/dist/utils/gbnfJson/terminals/GbnfBoolean.d.ts

@@ -2,5 +2,5 @@ import { GbnfTerminal } from "../GbnfTerminal.js";
 import { GbnfGrammarGenerator } from "../GbnfGrammarGenerator.js";
 export declare class GbnfBoolean extends GbnfTerminal {
     getGrammar(grammarGenerator: GbnfGrammarGenerator): string;
-    getRuleName(): string;
+    protected getRuleName(): string;
 }

package/dist/utils/gbnfJson/terminals/GbnfBoolean.js.map

@@ -1 +1 @@
-{"version":3,"file":"GbnfBoolean.js","sourceRoot":"","sources":["../../../../src/utils/gbnfJson/terminals/GbnfBoolean.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAEhD,OAAO,EAAC,MAAM,EAAC,MAAM,aAAa,CAAC;AACnC,OAAO,EAAC,WAAW,EAAC,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EAAC,iBAAiB,EAAC,MAAM,iBAAiB,CAAC;AAGlD,MAAM,OAAO,WAAY,SAAQ,YAAY;IACzC,UAAU,CAAC,gBAAsC;QAC7C,OAAO,IAAI,MAAM,CAAC;YACd,IAAI,WAAW,CAAC,QAAQ,CAAC;YACzB,IAAI,WAAW,CAAC,SAAS,CAAC;SAC7B,CAAC,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC;IACpC,CAAC;IAEQ,WAAW;QAChB,OAAO,iBAAiB,CAAC,OAAO,CAAC;IACrC,CAAC;CACJ"}
+{"version":3,"file":"GbnfBoolean.js","sourceRoot":"","sources":["../../../../src/utils/gbnfJson/terminals/GbnfBoolean.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAEhD,OAAO,EAAC,MAAM,EAAC,MAAM,aAAa,CAAC;AACnC,OAAO,EAAC,WAAW,EAAC,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EAAC,iBAAiB,EAAC,MAAM,iBAAiB,CAAC;AAGlD,MAAM,OAAO,WAAY,SAAQ,YAAY;IAClC,UAAU,CAAC,gBAAsC;QACpD,OAAO,IAAI,MAAM,CAAC;YACd,IAAI,WAAW,CAAC,QAAQ,CAAC;YACzB,IAAI,WAAW,CAAC,SAAS,CAAC;SAC7B,CAAC,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC;IACpC,CAAC;IAEkB,WAAW;QAC1B,OAAO,iBAAiB,CAAC,OAAO,CAAC;IACrC,CAAC;CACJ"}

package/dist/utils/gbnfJson/terminals/GbnfBooleanValue.js.map

@@ -1 +1 @@
-{"version":3,"file":"GbnfBooleanValue.js","sourceRoot":"","sources":["../../../../src/utils/gbnfJson/terminals/GbnfBooleanValue.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAGhD,MAAM,OAAO,gBAAiB,SAAQ,YAAY;IAC9B,KAAK,CAAU;IAE/B,YAAmB,KAAc;QAC7B,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACvB,CAAC;IAED,UAAU;QACN,IAAI,IAAI,CAAC,KAAK;YACV,OAAO,QAAQ,CAAC;QAEpB,OAAO,SAAS,CAAC;IACrB,CAAC;IAEe,OAAO;QACnB,OAAO,IAAI,CAAC,UAAU,EAAE,CAAC;IAC7B,CAAC;CACJ"}
+{"version":3,"file":"GbnfBooleanValue.js","sourceRoot":"","sources":["../../../../src/utils/gbnfJson/terminals/GbnfBooleanValue.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAGhD,MAAM,OAAO,gBAAiB,SAAQ,YAAY;IAC9B,KAAK,CAAU;IAE/B,YAAmB,KAAc;QAC7B,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACvB,CAAC;IAEM,UAAU;QACb,IAAI,IAAI,CAAC,KAAK;YACV,OAAO,QAAQ,CAAC;QAEpB,OAAO,SAAS,CAAC;IACrB,CAAC;IAEe,OAAO;QACnB,OAAO,IAAI,CAAC,UAAU,EAAE,CAAC;IAC7B,CAAC;CACJ"}

package/dist/utils/gbnfJson/terminals/GbnfGrammar.js.map

@@ -1 +1 @@
-{"version":3,"file":"GbnfGrammar.js","sourceRoot":"","sources":["../../../../src/utils/gbnfJson/terminals/GbnfGrammar.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAGhD,MAAM,OAAO,WAAY,SAAQ,YAAY;IACzB,OAAO,CAAoB;IAE3C,YAAmB,OAA0B;QACzC,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IAC3B,CAAC;IAED,UAAU;QACN,IAAI,IAAI,CAAC,OAAO,YAAY,KAAK;YAC7B,OAAO,IAAI,CAAC,OAAO;iBACd,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC;iBAC7B,IAAI,CAAC,GAAG,CAAC,CAAC;QAEnB,OAAO,IAAI,CAAC,OAAO,CAAC;IACxB,CAAC;CACJ"}
+{"version":3,"file":"GbnfGrammar.js","sourceRoot":"","sources":["../../../../src/utils/gbnfJson/terminals/GbnfGrammar.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAGhD,MAAM,OAAO,WAAY,SAAQ,YAAY;IACzB,OAAO,CAAoB;IAE3C,YAAmB,OAA0B;QACzC,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IAC3B,CAAC;IAEM,UAAU;QACb,IAAI,IAAI,CAAC,OAAO,YAAY,KAAK;YAC7B,OAAO,IAAI,CAAC,OAAO;iBACd,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC;iBAC7B,IAAI,CAAC,GAAG,CAAC,CAAC;QAEnB,OAAO,IAAI,CAAC,OAAO,CAAC;IACxB,CAAC;CACJ"}

package/dist/utils/gbnfJson/terminals/GbnfNull.d.ts

@@ -1,5 +1,5 @@
 import { GbnfTerminal } from "../GbnfTerminal.js";
 export declare class GbnfNull extends GbnfTerminal {
     getGrammar(): string;
-    getRuleName(): string;
+    protected getRuleName(): string;
 }

package/dist/utils/gbnfJson/terminals/GbnfNull.js.map

@@ -1 +1 @@
-{"version":3,"file":"GbnfNull.js","sourceRoot":"","sources":["../../../../src/utils/gbnfJson/terminals/GbnfNull.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAC,iBAAiB,EAAC,MAAM,iBAAiB,CAAC;AAGlD,MAAM,OAAO,QAAS,SAAQ,YAAY;IACtC,UAAU;QACN,OAAO,QAAQ,CAAC;IACpB,CAAC;IAEQ,WAAW;QAChB,OAAO,iBAAiB,CAAC,IAAI,CAAC;IAClC,CAAC;CACJ"}
+{"version":3,"file":"GbnfNull.js","sourceRoot":"","sources":["../../../../src/utils/gbnfJson/terminals/GbnfNull.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAC,iBAAiB,EAAC,MAAM,iBAAiB,CAAC;AAGlD,MAAM,OAAO,QAAS,SAAQ,YAAY;IAC/B,UAAU;QACb,OAAO,QAAQ,CAAC;IACpB,CAAC;IAEkB,WAAW;QAC1B,OAAO,iBAAiB,CAAC,IAAI,CAAC;IAClC,CAAC;CACJ"}

package/dist/utils/gbnfJson/terminals/GbnfNumber.d.ts

@@ -5,5 +5,5 @@ export declare class GbnfNumber extends GbnfTerminal {
         allowFractional: boolean;
     });
     getGrammar(): string;
-    getRuleName(): string;
+    protected getRuleName(): string;
 }

package/dist/utils/gbnfJson/terminals/GbnfNumber.js.map

@@ -1 +1 @@
-{"version":3,"file":"GbnfNumber.js","sourceRoot":"","sources":["../../../../src/utils/gbnfJson/terminals/GbnfNumber.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAC,iBAAiB,EAAC,MAAM,iBAAiB,CAAC;AAGlD,MAAM,OAAO,UAAW,SAAQ,YAAY;IACxB,eAAe,CAAU;IAEzC,YAAmB,EAAC,eAAe,GAAG,IAAI,EAA+B;QACrE,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IAC3C,CAAC;IAED,UAAU;QACN,MAAM,aAAa,GAAG,+BAA+B,CAAC;QAEtD,IAAI,IAAI,CAAC,eAAe;YACpB,OAAO,aAAa,GAAG,qCAAqC,CAAC;QAEjE,OAAO,aAAa,CAAC;IACzB,CAAC;IAEQ,WAAW;QAChB,IAAI,IAAI,CAAC,eAAe;YACpB,OAAO,iBAAiB,CAAC,MAAM,CAAC,UAAU,CAAC;QAE/C,OAAO,iBAAiB,CAAC,MAAM,CAAC,OAAO,CAAC;IAC5C,CAAC;CACJ"}
+{"version":3,"file":"GbnfNumber.js","sourceRoot":"","sources":["../../../../src/utils/gbnfJson/terminals/GbnfNumber.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAC,iBAAiB,EAAC,MAAM,iBAAiB,CAAC;AAGlD,MAAM,OAAO,UAAW,SAAQ,YAAY;IACxB,eAAe,CAAU;IAEzC,YAAmB,EAAC,eAAe,GAAG,IAAI,EAA+B;QACrE,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,eAAe,GAAG,eAAe,CAAC;IAC3C,CAAC;IAEM,UAAU;QACb,MAAM,aAAa,GAAG,+BAA+B,CAAC;QAEtD,IAAI,IAAI,CAAC,eAAe;YACpB,OAAO,aAAa,GAAG,qCAAqC,CAAC;QAEjE,OAAO,aAAa,CAAC;IACzB,CAAC;IAEkB,WAAW;QAC1B,IAAI,IAAI,CAAC,eAAe;YACpB,OAAO,iBAAiB,CAAC,MAAM,CAAC,UAAU,CAAC;QAE/C,OAAO,iBAAiB,CAAC,MAAM,CAAC,OAAO,CAAC;IAC5C,CAAC;CACJ"}

package/dist/utils/gbnfJson/terminals/GbnfNumberValue.js.map

@@ -1 +1 @@
-{"version":3,"file":"GbnfNumberValue.js","sourceRoot":"","sources":["../../../../src/utils/gbnfJson/terminals/GbnfNumberValue.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAGhD,MAAM,OAAO,eAAgB,SAAQ,YAAY;IAC7B,KAAK,CAAS;IAE9B,YAAmB,KAAa;QAC5B,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACvB,CAAC;IAEQ,UAAU;QACf,OAAO,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC;IAClD,CAAC;IAEe,OAAO;QACnB,OAAO,IAAI,CAAC,UAAU,EAAE,CAAC;IAC7B,CAAC;CACJ"}
+{"version":3,"file":"GbnfNumberValue.js","sourceRoot":"","sources":["../../../../src/utils/gbnfJson/terminals/GbnfNumberValue.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAGhD,MAAM,OAAO,eAAgB,SAAQ,YAAY;IAC7B,KAAK,CAAS;IAE9B,YAAmB,KAAa;QAC5B,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACvB,CAAC;IAEe,UAAU;QACtB,OAAO,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC;IAClD,CAAC;IAEe,OAAO;QACnB,OAAO,IAAI,CAAC,UAAU,EAAE,CAAC;IAC7B,CAAC;CACJ"}

package/dist/utils/gbnfJson/terminals/GbnfObjectMap.d.ts

@@ -1,5 +1,6 @@
 import { GbnfTerminal } from "../GbnfTerminal.js";
 import { GbnfGrammarGenerator } from "../GbnfGrammarGenerator.js";
+import { GbnfJsonScopeState } from "../utils/GbnfJsonScopeState.js";
 import { GbnfString } from "./GbnfString.js";
 import { GbnfStringValue } from "./GbnfStringValue.js";
 export declare class GbnfObjectMap extends GbnfTerminal {
@@ -8,10 +9,11 @@ export declare class GbnfObjectMap extends GbnfTerminal {
         value: GbnfTerminal;
         required: true;
     }>>;
+    readonly scopeState: GbnfJsonScopeState;
     constructor(fields: Array<Readonly<{
         key: GbnfString | GbnfStringValue;
         value: GbnfTerminal;
         required: true;
-    }>>);
+    }>>, scopeState?: GbnfJsonScopeState);
     getGrammar(grammarGenerator: GbnfGrammarGenerator): string;
 }

package/dist/utils/gbnfJson/terminals/GbnfObjectMap.js

@@ -1,21 +1,26 @@
 import { GbnfTerminal } from "../GbnfTerminal.js";
+import { GbnfJsonScopeState } from "../utils/GbnfJsonScopeState.js";
 import { GbnfWhitespace } from "./GbnfWhitespace.js";
 import { GbnfGrammar } from "./GbnfGrammar.js";
 export class GbnfObjectMap extends GbnfTerminal {
     fields;
-    constructor(fields) {
+    scopeState;
+    constructor(fields, scopeState = new GbnfJsonScopeState()) {
         super();
         this.fields = fields;
+        this.scopeState = scopeState;
     }
     getGrammar(grammarGenerator) {
-        const whitespaceRuleName = new GbnfWhitespace().resolve(grammarGenerator);
+        const getWhitespaceRuleName = (newScope, newLine) => (newScope
+            ? new GbnfWhitespace(this.scopeState.getForNewScope(), { newLine }).resolve(grammarGenerator)
+            : new GbnfWhitespace(this.scopeState, { newLine }).resolve(grammarGenerator));
         return new GbnfGrammar([
-            '"{"', whitespaceRuleName,
+            '"{"', getWhitespaceRuleName(true, "before"),
             ...this.fields.map(({ key, value }, index) => {
                 return new GbnfGrammar([
                     key.getGrammar(), '":"', "[ ]?", value.resolve(grammarGenerator),
                     index < this.fields.length - 1 ? '","' : "",
-                    whitespaceRuleName
+                    getWhitespaceRuleName(index < this.fields.length - 1, "before")
                 ]).getGrammar();
             }),
             '"}"'