@cortexkit/pi-magic-context 0.42.5 → 0.43.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{index-a0960w4w.js → index-5tw61yhp.js} +11963 -13799
- package/dist/index-83wwtw1q.js +466 -0
- package/dist/index-vcjgpb6f.js +4653 -0
- package/dist/index-y33j7m6y.js +16 -0
- package/dist/index.js +3594 -2591
- package/dist/message-time-backfill-dah83bcz.js +108 -0
- package/dist/read-session-chunk-3zybnn0f.js +47 -0
- package/dist/subagent-entry.js +7 -3
- package/dist/transformers-node-wasm.js +1918 -826
- package/dist/transformers-web.js +1736 -636
- package/dist/{transformers.node-dpmf3wpx.js → transformers.node-3zqr6k1w.js} +4087 -2898
- package/package.json +6 -3
package/dist/transformers-web.js
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
// ../../node_modules/.bun/@huggingface+transformers@4.
|
|
1
|
+
// ../../node_modules/.bun/@huggingface+transformers@4.3.0+4d96f45ad8218cb4/node_modules/@huggingface/transformers/dist/transformers.web.js
|
|
2
2
|
import * as ONNX_WEB from "onnxruntime-web/webgpu";
|
|
3
3
|
|
|
4
|
-
// ../../node_modules/.bun/onnxruntime-common@1.
|
|
4
|
+
// ../../node_modules/.bun/onnxruntime-common@1.30.0/node_modules/onnxruntime-common/dist/esm/backend-impl.js
|
|
5
5
|
var backends = new Map;
|
|
6
|
-
// ../../node_modules/.bun/onnxruntime-common@1.
|
|
7
|
-
var version = "1.
|
|
6
|
+
// ../../node_modules/.bun/onnxruntime-common@1.30.0/node_modules/onnxruntime-common/dist/esm/version.js
|
|
7
|
+
var version = "1.30.0";
|
|
8
8
|
|
|
9
|
-
// ../../node_modules/.bun/onnxruntime-common@1.
|
|
9
|
+
// ../../node_modules/.bun/onnxruntime-common@1.30.0/node_modules/onnxruntime-common/dist/esm/env-impl.js
|
|
10
10
|
var logLevelValue = "warning";
|
|
11
11
|
var env = {
|
|
12
12
|
wasm: {},
|
|
@@ -27,7 +27,7 @@ var env = {
|
|
|
27
27
|
}
|
|
28
28
|
};
|
|
29
29
|
Object.defineProperty(env, "logLevel", { enumerable: true });
|
|
30
|
-
// ../../node_modules/.bun/onnxruntime-common@1.
|
|
30
|
+
// ../../node_modules/.bun/onnxruntime-common@1.30.0/node_modules/onnxruntime-common/dist/esm/tensor-conversion-impl.js
|
|
31
31
|
var tensorToDataURL = (tensor, options) => {
|
|
32
32
|
const canvas = typeof document !== "undefined" ? document.createElement("canvas") : new OffscreenCanvas(1, 1);
|
|
33
33
|
canvas.width = tensor.dims[3];
|
|
@@ -186,7 +186,7 @@ var tensorToImageData = (tensor, options) => {
|
|
|
186
186
|
return image;
|
|
187
187
|
};
|
|
188
188
|
|
|
189
|
-
// ../../node_modules/.bun/onnxruntime-common@1.
|
|
189
|
+
// ../../node_modules/.bun/onnxruntime-common@1.30.0/node_modules/onnxruntime-common/dist/esm/tensor-factory-impl.js
|
|
190
190
|
var bufferToTensor = (buffer, options) => {
|
|
191
191
|
if (buffer === undefined) {
|
|
192
192
|
throw new Error("Image buffer must be defined");
|
|
@@ -395,7 +395,7 @@ var tensorFromMLTensor = (mlTensor, options) => {
|
|
|
395
395
|
};
|
|
396
396
|
var tensorFromPinnedBuffer = (type, buffer, dims) => new Tensor({ location: "cpu-pinned", type, data: buffer, dims: dims ?? [buffer.length] });
|
|
397
397
|
|
|
398
|
-
// ../../node_modules/.bun/onnxruntime-common@1.
|
|
398
|
+
// ../../node_modules/.bun/onnxruntime-common@1.30.0/node_modules/onnxruntime-common/dist/esm/tensor-impl-type-mapping.js
|
|
399
399
|
var NUMERIC_TENSOR_TYPE_TO_TYPEDARRAY_MAP = new Map([
|
|
400
400
|
["float32", Float32Array],
|
|
401
401
|
["uint8", Uint8Array],
|
|
@@ -444,7 +444,7 @@ var checkTypedArray = () => {
|
|
|
444
444
|
}
|
|
445
445
|
};
|
|
446
446
|
|
|
447
|
-
// ../../node_modules/.bun/onnxruntime-common@1.
|
|
447
|
+
// ../../node_modules/.bun/onnxruntime-common@1.30.0/node_modules/onnxruntime-common/dist/esm/tensor-utils-impl.js
|
|
448
448
|
var calculateSize = (dims) => {
|
|
449
449
|
let size = 1;
|
|
450
450
|
for (let i = 0;i < dims.length; i++) {
|
|
@@ -496,7 +496,7 @@ var tensorReshape = (tensor, dims) => {
|
|
|
496
496
|
}
|
|
497
497
|
};
|
|
498
498
|
|
|
499
|
-
// ../../node_modules/.bun/onnxruntime-common@1.
|
|
499
|
+
// ../../node_modules/.bun/onnxruntime-common@1.30.0/node_modules/onnxruntime-common/dist/esm/tensor-impl.js
|
|
500
500
|
class Tensor {
|
|
501
501
|
constructor(arg0, arg1, arg2) {
|
|
502
502
|
checkTypedArray();
|
|
@@ -749,10 +749,10 @@ class Tensor {
|
|
|
749
749
|
}
|
|
750
750
|
}
|
|
751
751
|
|
|
752
|
-
// ../../node_modules/.bun/onnxruntime-common@1.
|
|
752
|
+
// ../../node_modules/.bun/onnxruntime-common@1.30.0/node_modules/onnxruntime-common/dist/esm/tensor.js
|
|
753
753
|
var Tensor2 = Tensor;
|
|
754
|
-
// ../../node_modules/.bun/@huggingface+transformers@4.
|
|
755
|
-
var __dirname = "/home/runner/work/magic-context/magic-context/node_modules/.bun/@huggingface+transformers@4.
|
|
754
|
+
// ../../node_modules/.bun/@huggingface+transformers@4.3.0+4d96f45ad8218cb4/node_modules/@huggingface/transformers/dist/transformers.web.js
|
|
755
|
+
var __dirname = "/home/runner/work/magic-context/magic-context/node_modules/.bun/@huggingface+transformers@4.3.0+4d96f45ad8218cb4/node_modules/@huggingface/transformers/dist";
|
|
756
756
|
var __defProp = Object.defineProperty;
|
|
757
757
|
var __export = (target, all) => {
|
|
758
758
|
for (var name in all)
|
|
@@ -761,7 +761,7 @@ var __export = (target, all) => {
|
|
|
761
761
|
var node_fs_default = {};
|
|
762
762
|
var node_path_default = {};
|
|
763
763
|
var node_url_default = {};
|
|
764
|
-
var VERSION = "4.
|
|
764
|
+
var VERSION = "4.3.0";
|
|
765
765
|
var HAS_SELF = typeof self !== "undefined";
|
|
766
766
|
var IS_FS_AVAILABLE = !isEmpty(node_fs_default);
|
|
767
767
|
var IS_PATH_AVAILABLE = !isEmpty(node_path_default);
|
|
@@ -779,17 +779,19 @@ var IS_WEBNN_AVAILABLE = typeof navigator !== "undefined" && "ml" in navigator;
|
|
|
779
779
|
var IS_CRYPTO_AVAILABLE = typeof crypto !== "undefined" && typeof crypto.getRandomValues === "function";
|
|
780
780
|
var IS_CHROME_AVAILABLE = typeof chrome !== "undefined" && typeof chrome.runtime !== "undefined" && typeof chrome.runtime.id === "string";
|
|
781
781
|
var IS_SERVICE_WORKER_ENV = typeof ServiceWorkerGlobalScope !== "undefined" && HAS_SELF && self instanceof ServiceWorkerGlobalScope;
|
|
782
|
-
var
|
|
782
|
+
var isSafariBelow26 = () => {
|
|
783
783
|
if (typeof navigator === "undefined") {
|
|
784
784
|
return false;
|
|
785
785
|
}
|
|
786
786
|
const userAgent = navigator.userAgent;
|
|
787
|
-
const
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
787
|
+
const isSafari = (navigator.vendor || "").indexOf("Apple") > -1 && !userAgent.match(/CriOS|FxiOS|EdgiOS|OPiOS|mercury|brave/i) && !userAgent.includes("Chrome") && !userAgent.includes("Android");
|
|
788
|
+
if (!isSafari) {
|
|
789
|
+
return false;
|
|
790
|
+
}
|
|
791
|
+
const match = userAgent.match(/Version\/(\d+)/);
|
|
792
|
+
return match ? parseInt(match[1], 10) < 26 : false;
|
|
791
793
|
};
|
|
792
|
-
var
|
|
794
|
+
var IS_SAFARI_BELOW_26 = isSafariBelow26();
|
|
793
795
|
var apis = Object.freeze({
|
|
794
796
|
IS_BROWSER_ENV,
|
|
795
797
|
IS_WEBWORKER_ENV,
|
|
@@ -799,7 +801,7 @@ var apis = Object.freeze({
|
|
|
799
801
|
IS_WEB_CACHE_AVAILABLE,
|
|
800
802
|
IS_WEBGPU_AVAILABLE,
|
|
801
803
|
IS_WEBNN_AVAILABLE,
|
|
802
|
-
|
|
804
|
+
IS_SAFARI_BELOW_26,
|
|
803
805
|
IS_PROCESS_AVAILABLE,
|
|
804
806
|
IS_NODE_ENV,
|
|
805
807
|
IS_FS_AVAILABLE,
|
|
@@ -810,7 +812,7 @@ var apis = Object.freeze({
|
|
|
810
812
|
var RUNNING_LOCALLY = IS_FS_AVAILABLE && IS_PATH_AVAILABLE;
|
|
811
813
|
var dirname__ = "./";
|
|
812
814
|
if (RUNNING_LOCALLY) {
|
|
813
|
-
const _import_meta_url =
|
|
815
|
+
const _import_meta_url = import.meta.url;
|
|
814
816
|
if (_import_meta_url) {
|
|
815
817
|
dirname__ = node_path_default.dirname(node_path_default.dirname(node_url_default.fileURLToPath(_import_meta_url)));
|
|
816
818
|
} else if (typeof __dirname !== "undefined") {
|
|
@@ -880,6 +882,7 @@ var DefaultProgressCallback = class extends Callable {
|
|
|
880
882
|
super();
|
|
881
883
|
this.callback = callback;
|
|
882
884
|
this.files_loading = files_loading;
|
|
885
|
+
this.loads = /* @__PURE__ */ new Map;
|
|
883
886
|
}
|
|
884
887
|
_call(info) {
|
|
885
888
|
if (info.status === "progress") {
|
|
@@ -1027,75 +1030,46 @@ var AddedToken = class {
|
|
|
1027
1030
|
}
|
|
1028
1031
|
};
|
|
1029
1032
|
var AddedToken_default = AddedToken;
|
|
1030
|
-
var
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
return Object.fromEntries(bs.map((b, i) => [b, ccs[i]]));
|
|
1047
|
-
})();
|
|
1048
|
-
var reverse_dictionary = (data) => Object.fromEntries(Object.entries(data).map(([key, value]) => [value, key]));
|
|
1049
|
-
var UNICODE_TO_BYTES = reverse_dictionary(BYTES_TO_UNICODE);
|
|
1050
|
-
var BLOOM_SPLIT_CHARS = ".,!?…。,、।۔،";
|
|
1051
|
-
var PROBLEMATIC_REGEX_MAP = /* @__PURE__ */ new Map([
|
|
1052
|
-
[
|
|
1053
|
-
"(?i:'s|'t|'re|'ve|'m|'ll|'d)",
|
|
1054
|
-
"(?:'([sS]|[tT]|[rR][eE]|[vV][eE]|[mM]|[lL][lL]|[dD]))"
|
|
1055
|
-
],
|
|
1056
|
-
[
|
|
1057
|
-
"(?i:[sdmt]|ll|ve|re)",
|
|
1058
|
-
"(?:[sS]|[dD]|[mM]|[tT]|[lL][lL]|[vV][eE]|[rR][eE])"
|
|
1059
|
-
],
|
|
1060
|
-
["[^\\r\\n\\p{L}\\p{N}]?+", "[^\\r\\n\\p{L}\\p{N}]?"],
|
|
1061
|
-
["[^\\s\\p{L}\\p{N}]++", "[^\\s\\p{L}\\p{N}]+"],
|
|
1062
|
-
["(?>\\p{Nd}{510})", "(?:\\p{Nd}{510})"],
|
|
1063
|
-
["\\p{Nd}{3}+", "(?:\\p{Nd}{3})+"],
|
|
1064
|
-
["\\G", ""],
|
|
1065
|
-
[` ?[^(\\s|[${BLOOM_SPLIT_CHARS}])]+`, ` ?[^\\s${BLOOM_SPLIT_CHARS}]+`]
|
|
1066
|
-
]);
|
|
1067
|
-
var PUNCTUATION_REGEX = "\\p{P}\\u0021-\\u002F\\u003A-\\u0040\\u005B-\\u0060\\u007B-\\u007E";
|
|
1068
|
-
var clean_up_tokenization = (text) => text.replace(/ \./g, ".").replace(/ \?/g, "?").replace(/ \!/g, "!").replace(/ ,/g, ",").replace(/ \' /g, "'").replace(/ n't/g, "n't").replace(/ 'm/g, "'m").replace(/ 's/g, "'s").replace(/ 've/g, "'ve").replace(/ 're/g, "'re");
|
|
1069
|
-
var create_pattern = (pattern, invert = true) => {
|
|
1070
|
-
if (pattern.Regex !== undefined) {
|
|
1071
|
-
let regex = pattern.Regex.replace(/\\([#&~])/g, "$1");
|
|
1072
|
-
regex = regex.replace(/\\A/g, "^").replace(/\\z/g, "$").replace(/\\Z/g, "(?=\\r?\\n?$)");
|
|
1073
|
-
for (const [key, value] of PROBLEMATIC_REGEX_MAP) {
|
|
1074
|
-
regex = regex.replaceAll(key, value);
|
|
1075
|
-
}
|
|
1076
|
-
try {
|
|
1077
|
-
return new RegExp(regex, "gu");
|
|
1078
|
-
} catch (error) {
|
|
1079
|
-
if (!(error instanceof SyntaxError) || !error.message.toLowerCase().includes("invalid property name"))
|
|
1080
|
-
throw error;
|
|
1081
|
-
let changed = false;
|
|
1082
|
-
const fixed = regex.replace(/(\\[pP])\{([^}=]+)\}/g, (_, p, n) => {
|
|
1033
|
+
var compile_unicode_regexp = (source, flags) => {
|
|
1034
|
+
try {
|
|
1035
|
+
return new RegExp(source, flags);
|
|
1036
|
+
} catch (error) {
|
|
1037
|
+
if (!(error instanceof SyntaxError))
|
|
1038
|
+
throw error;
|
|
1039
|
+
const property_names = /* @__PURE__ */ new Map;
|
|
1040
|
+
const rewritten = source.replace(/(\\[pP])\{([^}=]+)\}/g, (text, p, n, offset) => {
|
|
1041
|
+
let preceding_backslashes = 0;
|
|
1042
|
+
for (let i = offset - 1;i >= 0 && source[i] === "\\"; --i) {
|
|
1043
|
+
++preceding_backslashes;
|
|
1044
|
+
}
|
|
1045
|
+
if (preceding_backslashes % 2 === 1)
|
|
1046
|
+
return text;
|
|
1047
|
+
let property_name = property_names.get(n);
|
|
1048
|
+
if (property_name === undefined) {
|
|
1083
1049
|
try {
|
|
1084
1050
|
new RegExp(`\\p{${n}}`, "u");
|
|
1085
|
-
|
|
1051
|
+
property_name = n;
|
|
1086
1052
|
} catch {
|
|
1087
|
-
|
|
1088
|
-
return `${p}{Script=${n}}`;
|
|
1053
|
+
property_name = `Script=${n}`;
|
|
1089
1054
|
}
|
|
1090
|
-
|
|
1091
|
-
if (!changed)
|
|
1092
|
-
throw error;
|
|
1093
|
-
try {
|
|
1094
|
-
return new RegExp(fixed, "gu");
|
|
1095
|
-
} catch (e) {
|
|
1096
|
-
throw error;
|
|
1055
|
+
property_names.set(n, property_name);
|
|
1097
1056
|
}
|
|
1057
|
+
return `${p}{${property_name}}`;
|
|
1058
|
+
});
|
|
1059
|
+
if (rewritten === source)
|
|
1060
|
+
throw error;
|
|
1061
|
+
try {
|
|
1062
|
+
return new RegExp(rewritten, flags);
|
|
1063
|
+
} catch {
|
|
1064
|
+
throw error;
|
|
1098
1065
|
}
|
|
1066
|
+
}
|
|
1067
|
+
};
|
|
1068
|
+
var clean_up_tokenization = (text) => text.replace(/ \./g, ".").replace(/ \?/g, "?").replace(/ \!/g, "!").replace(/ ,/g, ",").replace(/ \' /g, "'").replace(/ n't/g, "n't").replace(/ 'm/g, "'m").replace(/ 's/g, "'s").replace(/ 've/g, "'ve").replace(/ 're/g, "'re");
|
|
1069
|
+
var create_pattern = (pattern, invert = true) => {
|
|
1070
|
+
if (pattern.Regex !== undefined) {
|
|
1071
|
+
const regex = rewrite_oniguruma_to_js(normalize_bloom_split_char_class(pattern.Regex));
|
|
1072
|
+
return compile_unicode_regexp(regex, "gu");
|
|
1099
1073
|
} else if (pattern.String !== undefined) {
|
|
1100
1074
|
const escaped = escape_reg_exp(pattern.String);
|
|
1101
1075
|
return new RegExp(invert ? escaped : `(${escaped})`, "gu");
|
|
@@ -1104,6 +1078,473 @@ var create_pattern = (pattern, invert = true) => {
|
|
|
1104
1078
|
return null;
|
|
1105
1079
|
}
|
|
1106
1080
|
};
|
|
1081
|
+
var UNICODE_WORD_CHARS_IN_CLASS = "\\p{Alphabetic}\\p{M}\\p{Nd}\\p{Pc}";
|
|
1082
|
+
var UNICODE_WORD_CHARS = `${UNICODE_WORD_CHARS_IN_CLASS}\\u00B2\\u00B3\\u00B9\\u00BC-\\u00BE`;
|
|
1083
|
+
var UNICODE_WORD_CLASS = `[${UNICODE_WORD_CHARS}]`;
|
|
1084
|
+
var UNICODE_NON_WORD_CLASS = `[^${UNICODE_WORD_CHARS}]`;
|
|
1085
|
+
var UNICODE_WORD_BOUNDARY = `(?:(?<!${UNICODE_WORD_CLASS})(?=${UNICODE_WORD_CLASS})|(?<=${UNICODE_WORD_CLASS})(?!${UNICODE_WORD_CLASS}))`;
|
|
1086
|
+
var UNICODE_NON_WORD_BOUNDARY = `(?:(?<!${UNICODE_WORD_CLASS})(?!${UNICODE_WORD_CLASS})|(?<=${UNICODE_WORD_CLASS})(?=${UNICODE_WORD_CLASS}))`;
|
|
1087
|
+
var LINE_START_ANCHOR = "(?:(?<![\\s\\S])|(?<=\\n))";
|
|
1088
|
+
var LINE_END_ANCHOR = "(?:(?=\\n)|(?![\\s\\S]))";
|
|
1089
|
+
var HEX_DIGIT_CHARS = "0-9A-Fa-f";
|
|
1090
|
+
var ESCAPE_REWRITES = /* @__PURE__ */ new Map([
|
|
1091
|
+
["A", "(?<![\\s\\S])"],
|
|
1092
|
+
["z", "(?![\\s\\S])"],
|
|
1093
|
+
["Z", "(?=\\n?(?![\\s\\S]))"],
|
|
1094
|
+
["h", `[${HEX_DIGIT_CHARS}]`],
|
|
1095
|
+
["H", `[^${HEX_DIGIT_CHARS}]`],
|
|
1096
|
+
["w", UNICODE_WORD_CLASS],
|
|
1097
|
+
["W", UNICODE_NON_WORD_CLASS],
|
|
1098
|
+
["d", "\\p{Nd}"],
|
|
1099
|
+
["D", "\\P{Nd}"],
|
|
1100
|
+
["s", "\\p{White_Space}"],
|
|
1101
|
+
["S", "\\P{White_Space}"],
|
|
1102
|
+
["b", UNICODE_WORD_BOUNDARY],
|
|
1103
|
+
["B", UNICODE_NON_WORD_BOUNDARY],
|
|
1104
|
+
["a", "\\x07"],
|
|
1105
|
+
["e", "\\x1B"]
|
|
1106
|
+
]);
|
|
1107
|
+
var CLASS_ESCAPE_REWRITES = /* @__PURE__ */ new Map([
|
|
1108
|
+
["h", HEX_DIGIT_CHARS],
|
|
1109
|
+
["w", UNICODE_WORD_CHARS_IN_CLASS],
|
|
1110
|
+
["d", "\\p{Nd}"],
|
|
1111
|
+
["D", "\\P{Nd}"],
|
|
1112
|
+
["s", "\\p{White_Space}"],
|
|
1113
|
+
["S", "\\P{White_Space}"],
|
|
1114
|
+
["a", "\\x07"],
|
|
1115
|
+
["e", "\\x1B"]
|
|
1116
|
+
]);
|
|
1117
|
+
var CLASS_COMPLEMENT_ALTERNATIVES = /* @__PURE__ */ new Map([
|
|
1118
|
+
["W", `[^${UNICODE_WORD_CHARS_IN_CLASS}]`],
|
|
1119
|
+
["H", `[^${HEX_DIGIT_CHARS}]`]
|
|
1120
|
+
]);
|
|
1121
|
+
var RAW_WHITESPACE_ESCAPES = /* @__PURE__ */ new Map([
|
|
1122
|
+
[`
|
|
1123
|
+
`, "\\n"],
|
|
1124
|
+
["\r", "\\r"],
|
|
1125
|
+
["\t", "\\t"],
|
|
1126
|
+
["\f", "\\f"],
|
|
1127
|
+
["\v", "\\v"]
|
|
1128
|
+
]);
|
|
1129
|
+
var POSIX_CLASS_FRAGMENTS = /* @__PURE__ */ new Map([
|
|
1130
|
+
["alpha", "\\p{Alphabetic}"],
|
|
1131
|
+
["alnum", "\\p{Alphabetic}\\p{Nd}"],
|
|
1132
|
+
["digit", "\\p{Nd}"],
|
|
1133
|
+
["lower", "\\p{Lowercase}"],
|
|
1134
|
+
["upper", "\\p{Uppercase}"],
|
|
1135
|
+
["space", "\\p{White_Space}"],
|
|
1136
|
+
["blank", "\\t\\p{Zs}"],
|
|
1137
|
+
["punct", "\\p{P}\\p{S}"],
|
|
1138
|
+
["cntrl", "\\p{Cc}"],
|
|
1139
|
+
["word", UNICODE_WORD_CHARS_IN_CLASS],
|
|
1140
|
+
["xdigit", HEX_DIGIT_CHARS]
|
|
1141
|
+
]);
|
|
1142
|
+
var JS_SYNTAX_CHARS = "^$\\.*+?()[]{}|/";
|
|
1143
|
+
var GROUP_PREFIX_RE = /^\(\?(?:<[=!]|<[A-Za-z_][A-Za-z0-9_]*>|[:=!>])/;
|
|
1144
|
+
var BRACED_ESCAPE_RE = /^\\([pPxu])\{([^}]*)\}/;
|
|
1145
|
+
var QUANTIFIER_BRACE_RE = /^\{(\d+(?:,\d*)?|,\d+)\}/;
|
|
1146
|
+
var POSIX_BRACKET_RE = /^\[:(\^?)(\p{Alphabetic}+):\]/u;
|
|
1147
|
+
var EMPTY_NEGATED_POSIX_BRACKET_RE = /^\[:\^:\]/;
|
|
1148
|
+
var UNSUPPORTED_POSIX_BRACKET_RE = /^\[(?:\.[^\]]*\.\]|=[^\]]*=\])/;
|
|
1149
|
+
var FIXED_WIDTH_ESCAPE_RE = /^(?:\\x[0-9A-Fa-f]{2}|\\u[0-9A-Fa-f]{4}|\\c[A-Za-z])/;
|
|
1150
|
+
var is_ascii_letter = (char) => char >= "A" && char <= "Z" || char >= "a" && char <= "z";
|
|
1151
|
+
var character_at = (text, index) => String.fromCodePoint(text.codePointAt(index));
|
|
1152
|
+
var get_ascii_folded_hex_atom = (hex) => {
|
|
1153
|
+
if (!/^[0-9A-Fa-f]{1,8}$/.test(hex))
|
|
1154
|
+
return null;
|
|
1155
|
+
const code_point = Number.parseInt(hex, 16);
|
|
1156
|
+
if (code_point > 127)
|
|
1157
|
+
return null;
|
|
1158
|
+
const letter = String.fromCharCode(code_point);
|
|
1159
|
+
return is_ascii_letter(letter) ? `[${letter.toLowerCase()}${letter.toUpperCase()}]` : null;
|
|
1160
|
+
};
|
|
1161
|
+
var normalize_bloom_split_char_class = (regex) => regex.replace(/\[\^\(\\s\|\[([^\]]+)\]\)\]/g, "[^()|\\s$1]");
|
|
1162
|
+
var ANY_CODE_POINT = "[\\s\\S]";
|
|
1163
|
+
var MAX_CHARACTER_CLASS_NESTING_DEPTH = 256;
|
|
1164
|
+
var create_character_class_operand = () => ({
|
|
1165
|
+
fragment: "",
|
|
1166
|
+
alternatives: [],
|
|
1167
|
+
tail: null,
|
|
1168
|
+
contains_complex_set: false
|
|
1169
|
+
});
|
|
1170
|
+
var throw_character_class_range_error = (index) => {
|
|
1171
|
+
throw new SyntaxError(`Unsupported range with a set-valued character-class operand at index ${index}`);
|
|
1172
|
+
};
|
|
1173
|
+
var add_character_class_atom = (operand, atom, index, contains_complex_set = true) => {
|
|
1174
|
+
if (operand.tail === "range")
|
|
1175
|
+
throw_character_class_range_error(index);
|
|
1176
|
+
operand.alternatives.push(atom);
|
|
1177
|
+
operand.tail = "set";
|
|
1178
|
+
operand.contains_complex_set = operand.contains_complex_set || contains_complex_set;
|
|
1179
|
+
};
|
|
1180
|
+
var append_character_class_fragment = (operand, fragment, set_valued = false, index = -1) => {
|
|
1181
|
+
if (set_valued && operand.tail === "range") {
|
|
1182
|
+
throw_character_class_range_error(index);
|
|
1183
|
+
}
|
|
1184
|
+
if (operand.tail === "range") {
|
|
1185
|
+
operand.fragment += fragment;
|
|
1186
|
+
operand.tail = "complete_range";
|
|
1187
|
+
return;
|
|
1188
|
+
}
|
|
1189
|
+
operand.fragment += fragment;
|
|
1190
|
+
operand.tail = set_valued ? "set" : "scalar";
|
|
1191
|
+
operand.contains_complex_set = operand.contains_complex_set || set_valued;
|
|
1192
|
+
};
|
|
1193
|
+
var rewrite_character_class_escape = (regex, index, operand) => {
|
|
1194
|
+
const braced = BRACED_ESCAPE_RE.exec(regex.slice(index));
|
|
1195
|
+
if (braced) {
|
|
1196
|
+
const [text, kind, body] = braced;
|
|
1197
|
+
if (kind === "P" && body === "Word") {
|
|
1198
|
+
add_character_class_atom(operand, `[^${UNICODE_WORD_CHARS_IN_CLASS}]`, index);
|
|
1199
|
+
} else {
|
|
1200
|
+
const replacement2 = kind === "x" ? `\\u{${body}}` : kind === "p" && body === "Word" ? UNICODE_WORD_CHARS_IN_CLASS : text;
|
|
1201
|
+
append_character_class_fragment(operand, replacement2, kind === "p" || kind === "P", index);
|
|
1202
|
+
}
|
|
1203
|
+
return index + text.length;
|
|
1204
|
+
}
|
|
1205
|
+
const fixed_width = FIXED_WIDTH_ESCAPE_RE.exec(regex.slice(index));
|
|
1206
|
+
if (fixed_width) {
|
|
1207
|
+
append_character_class_fragment(operand, fixed_width[0]);
|
|
1208
|
+
return index + fixed_width[0].length;
|
|
1209
|
+
}
|
|
1210
|
+
if (index + 1 >= regex.length) {
|
|
1211
|
+
throw new SyntaxError(`Unterminated escape in character class at index ${index}`);
|
|
1212
|
+
}
|
|
1213
|
+
const next = character_at(regex, index + 1);
|
|
1214
|
+
const next_end = index + 1 + next.length;
|
|
1215
|
+
const raw_whitespace = RAW_WHITESPACE_ESCAPES.get(next);
|
|
1216
|
+
if (raw_whitespace !== undefined) {
|
|
1217
|
+
append_character_class_fragment(operand, raw_whitespace);
|
|
1218
|
+
return next_end;
|
|
1219
|
+
}
|
|
1220
|
+
const complement = CLASS_COMPLEMENT_ALTERNATIVES.get(next);
|
|
1221
|
+
if (complement !== undefined) {
|
|
1222
|
+
add_character_class_atom(operand, complement, index);
|
|
1223
|
+
return next_end;
|
|
1224
|
+
}
|
|
1225
|
+
const rewrite = CLASS_ESCAPE_REWRITES.get(next);
|
|
1226
|
+
let replacement;
|
|
1227
|
+
let set_valued = false;
|
|
1228
|
+
if (rewrite !== undefined) {
|
|
1229
|
+
replacement = rewrite;
|
|
1230
|
+
set_valued = next !== "a" && next !== "e";
|
|
1231
|
+
} else if (/[A-Za-z0-9]/.test(next)) {
|
|
1232
|
+
replacement = `\\${next}`;
|
|
1233
|
+
} else if (JS_SYNTAX_CHARS.includes(next) || next === "-") {
|
|
1234
|
+
replacement = `\\${next}`;
|
|
1235
|
+
} else {
|
|
1236
|
+
replacement = next;
|
|
1237
|
+
}
|
|
1238
|
+
append_character_class_fragment(operand, replacement, set_valued, index);
|
|
1239
|
+
return next_end;
|
|
1240
|
+
};
|
|
1241
|
+
var compile_character_set_union = (pieces) => {
|
|
1242
|
+
if (pieces.length === 1)
|
|
1243
|
+
return pieces[0];
|
|
1244
|
+
return `(?:(?=(?:${pieces.join("|")}))${ANY_CODE_POINT})`;
|
|
1245
|
+
};
|
|
1246
|
+
var compile_character_class_operand = (operand) => {
|
|
1247
|
+
const pieces = operand.fragment.length === 0 ? operand.alternatives : [`[${operand.fragment}]`, ...operand.alternatives];
|
|
1248
|
+
return compile_character_set_union(pieces);
|
|
1249
|
+
};
|
|
1250
|
+
var get_ascii_fold_additions = (positive_atom) => {
|
|
1251
|
+
const membership = compile_unicode_regexp(`^(?:${positive_atom})$`, "u");
|
|
1252
|
+
let additions = "";
|
|
1253
|
+
for (let offset = 0;offset < 26; ++offset) {
|
|
1254
|
+
const upper = String.fromCharCode(65 + offset);
|
|
1255
|
+
const lower = String.fromCharCode(97 + offset);
|
|
1256
|
+
const has_upper = membership.test(upper);
|
|
1257
|
+
const has_lower = membership.test(lower);
|
|
1258
|
+
if (has_upper !== has_lower)
|
|
1259
|
+
additions += has_upper ? lower : upper;
|
|
1260
|
+
}
|
|
1261
|
+
return additions;
|
|
1262
|
+
};
|
|
1263
|
+
var parse_character_class = (regex, start, ascii_fold, apply_ascii_fold = true, nesting_depth = 1) => {
|
|
1264
|
+
if (nesting_depth > MAX_CHARACTER_CLASS_NESTING_DEPTH) {
|
|
1265
|
+
throw new SyntaxError(`Maximum character-class nesting depth of ${MAX_CHARACTER_CLASS_NESTING_DEPTH} exceeded at index ${start}`);
|
|
1266
|
+
}
|
|
1267
|
+
let i = start + 1;
|
|
1268
|
+
const negated = regex[i] === "^";
|
|
1269
|
+
if (negated)
|
|
1270
|
+
++i;
|
|
1271
|
+
const first_content_index = i;
|
|
1272
|
+
const operands = [create_character_class_operand()];
|
|
1273
|
+
let operand = operands[0];
|
|
1274
|
+
let contains_nested_negated_complex_set = false;
|
|
1275
|
+
while (i < regex.length) {
|
|
1276
|
+
const char = character_at(regex, i);
|
|
1277
|
+
if (char === "\\") {
|
|
1278
|
+
i = rewrite_character_class_escape(regex, i, operand);
|
|
1279
|
+
continue;
|
|
1280
|
+
}
|
|
1281
|
+
if (char === "]") {
|
|
1282
|
+
if (i === first_content_index) {
|
|
1283
|
+
append_character_class_fragment(operand, "\\]");
|
|
1284
|
+
++i;
|
|
1285
|
+
continue;
|
|
1286
|
+
}
|
|
1287
|
+
if (operand.tail === null) {
|
|
1288
|
+
if (operands.length > 1) {
|
|
1289
|
+
throw new SyntaxError(`Malformed character-class intersection with an empty operand at index ${i}`);
|
|
1290
|
+
}
|
|
1291
|
+
throw new SyntaxError(`Empty character class at index ${start}`);
|
|
1292
|
+
}
|
|
1293
|
+
const contains_complex_set = operands.some((candidate) => candidate.contains_complex_set);
|
|
1294
|
+
if (negated && operands.length > 1 && contains_nested_negated_complex_set) {
|
|
1295
|
+
throw new SyntaxError(`Unsupported outer-negated character-class intersection with a nested negated class containing a Unicode property, POSIX class, or shorthand at index ${start}`);
|
|
1296
|
+
}
|
|
1297
|
+
const first_atom = compile_character_class_operand(operands[0]);
|
|
1298
|
+
let positive_atom = first_atom;
|
|
1299
|
+
if (operands.length > 1) {
|
|
1300
|
+
let lookaheads = "";
|
|
1301
|
+
for (let j = 1;j < operands.length; ++j) {
|
|
1302
|
+
lookaheads += `(?=${compile_character_class_operand(operands[j])})`;
|
|
1303
|
+
}
|
|
1304
|
+
positive_atom = `(?:${lookaheads}${first_atom})`;
|
|
1305
|
+
}
|
|
1306
|
+
const is_direct_class = operands.length === 1 && operand.alternatives.length === 0;
|
|
1307
|
+
let direct_fragment = operand.fragment;
|
|
1308
|
+
if (ascii_fold && apply_ascii_fold) {
|
|
1309
|
+
const additions = get_ascii_fold_additions(positive_atom);
|
|
1310
|
+
if (additions.length > 0) {
|
|
1311
|
+
if (is_direct_class) {
|
|
1312
|
+
direct_fragment += additions;
|
|
1313
|
+
positive_atom = `[${direct_fragment}]`;
|
|
1314
|
+
} else {
|
|
1315
|
+
positive_atom = compile_character_set_union([
|
|
1316
|
+
positive_atom,
|
|
1317
|
+
`[${additions}]`
|
|
1318
|
+
]);
|
|
1319
|
+
}
|
|
1320
|
+
}
|
|
1321
|
+
}
|
|
1322
|
+
const atom = negated ? is_direct_class ? `[^${direct_fragment}]` : `(?:(?!${positive_atom})${ANY_CODE_POINT})` : positive_atom;
|
|
1323
|
+
return {
|
|
1324
|
+
atom,
|
|
1325
|
+
end: i + 1,
|
|
1326
|
+
negated,
|
|
1327
|
+
contains_complex_set,
|
|
1328
|
+
contains_nested_negated_complex_set
|
|
1329
|
+
};
|
|
1330
|
+
}
|
|
1331
|
+
if (regex.startsWith("&&", i)) {
|
|
1332
|
+
if (operand.tail === null) {
|
|
1333
|
+
throw new SyntaxError(`Malformed character-class intersection with an empty operand at index ${i}`);
|
|
1334
|
+
}
|
|
1335
|
+
operand = create_character_class_operand();
|
|
1336
|
+
operands.push(operand);
|
|
1337
|
+
i += 2;
|
|
1338
|
+
continue;
|
|
1339
|
+
}
|
|
1340
|
+
if (char === "[") {
|
|
1341
|
+
const suffix = regex.slice(i);
|
|
1342
|
+
if (EMPTY_NEGATED_POSIX_BRACKET_RE.test(suffix)) {
|
|
1343
|
+
throw new SyntaxError(`Malformed empty negated POSIX character class at index ${i}`);
|
|
1344
|
+
}
|
|
1345
|
+
const posix = POSIX_BRACKET_RE.exec(suffix);
|
|
1346
|
+
if (posix) {
|
|
1347
|
+
const [, posix_negated, name] = posix;
|
|
1348
|
+
const fragment = POSIX_CLASS_FRAGMENTS.get(name);
|
|
1349
|
+
if (fragment === undefined) {
|
|
1350
|
+
throw new SyntaxError(`Unsupported POSIX character class "${name}" at index ${i}`);
|
|
1351
|
+
}
|
|
1352
|
+
if (ascii_fold && posix_negated && (name === "lower" || name === "upper")) {
|
|
1353
|
+
throw new SyntaxError(`Unsupported negated POSIX ${name} class inside an inline case-insensitive group`);
|
|
1354
|
+
}
|
|
1355
|
+
if (posix_negated) {
|
|
1356
|
+
add_character_class_atom(operand, `[^${fragment}]`, i);
|
|
1357
|
+
} else {
|
|
1358
|
+
append_character_class_fragment(operand, fragment, true, i);
|
|
1359
|
+
}
|
|
1360
|
+
i += posix[0].length;
|
|
1361
|
+
continue;
|
|
1362
|
+
}
|
|
1363
|
+
if (UNSUPPORTED_POSIX_BRACKET_RE.test(suffix)) {
|
|
1364
|
+
throw new SyntaxError(`Unsupported POSIX collating or equivalence bracket expression at index ${i}`);
|
|
1365
|
+
}
|
|
1366
|
+
const nested = parse_character_class(regex, i, ascii_fold, false, nesting_depth + 1);
|
|
1367
|
+
add_character_class_atom(operand, nested.atom, i, nested.contains_complex_set);
|
|
1368
|
+
contains_nested_negated_complex_set ||= nested.contains_nested_negated_complex_set || nested.negated && nested.contains_complex_set;
|
|
1369
|
+
i = nested.end;
|
|
1370
|
+
continue;
|
|
1371
|
+
}
|
|
1372
|
+
if (char === "-") {
|
|
1373
|
+
const is_terminal_literal = regex[i + 1] === "]" || regex.startsWith("&&", i + 1);
|
|
1374
|
+
if (operand.tail === "set" && !is_terminal_literal) {
|
|
1375
|
+
throw_character_class_range_error(i);
|
|
1376
|
+
}
|
|
1377
|
+
if (operand.tail === null || operand.tail === "range" || operand.tail === "complete_range" || is_terminal_literal) {
|
|
1378
|
+
append_character_class_fragment(operand, "\\-");
|
|
1379
|
+
} else {
|
|
1380
|
+
operand.fragment += "-";
|
|
1381
|
+
operand.tail = "range";
|
|
1382
|
+
}
|
|
1383
|
+
++i;
|
|
1384
|
+
continue;
|
|
1385
|
+
}
|
|
1386
|
+
append_character_class_fragment(operand, char === "^" && operand.fragment.length === 0 ? "\\^" : char);
|
|
1387
|
+
i += char.length;
|
|
1388
|
+
}
|
|
1389
|
+
throw new SyntaxError(`${operands.length > 1 ? "Unterminated character-class intersection" : "Unterminated character class"} at index ${start}`);
|
|
1390
|
+
};
|
|
1391
|
+
var rewrite_oniguruma_to_js = (regex) => {
|
|
1392
|
+
let out = "";
|
|
1393
|
+
let atom_start = -1;
|
|
1394
|
+
let last_was_quantifier = false;
|
|
1395
|
+
let ascii_fold = false;
|
|
1396
|
+
const group_states = [];
|
|
1397
|
+
const emit_atom = (text) => {
|
|
1398
|
+
atom_start = out.length;
|
|
1399
|
+
out += text;
|
|
1400
|
+
last_was_quantifier = false;
|
|
1401
|
+
};
|
|
1402
|
+
for (let i = 0;i < regex.length; ) {
|
|
1403
|
+
const char = character_at(regex, i);
|
|
1404
|
+
if (char === "\\") {
|
|
1405
|
+
const braced = BRACED_ESCAPE_RE.exec(regex.slice(i));
|
|
1406
|
+
if (braced) {
|
|
1407
|
+
const [text, kind, body] = braced;
|
|
1408
|
+
let replacement2 = text;
|
|
1409
|
+
if (kind === "x") {
|
|
1410
|
+
const code_point_escape = `\\u{${body}}`;
|
|
1411
|
+
replacement2 = ascii_fold ? get_ascii_folded_hex_atom(body) ?? code_point_escape : code_point_escape;
|
|
1412
|
+
} else if (body === "Word") {
|
|
1413
|
+
replacement2 = kind === "p" ? UNICODE_WORD_CLASS : UNICODE_NON_WORD_CLASS;
|
|
1414
|
+
}
|
|
1415
|
+
emit_atom(replacement2);
|
|
1416
|
+
i += text.length;
|
|
1417
|
+
continue;
|
|
1418
|
+
}
|
|
1419
|
+
const fixed_width = FIXED_WIDTH_ESCAPE_RE.exec(regex.slice(i));
|
|
1420
|
+
if (fixed_width) {
|
|
1421
|
+
const text = fixed_width[0];
|
|
1422
|
+
const replacement2 = ascii_fold && text[1] !== "c" ? get_ascii_folded_hex_atom(text.slice(2)) ?? text : text;
|
|
1423
|
+
emit_atom(replacement2);
|
|
1424
|
+
i += text.length;
|
|
1425
|
+
continue;
|
|
1426
|
+
}
|
|
1427
|
+
if (i + 1 >= regex.length) {
|
|
1428
|
+
out += char;
|
|
1429
|
+
break;
|
|
1430
|
+
}
|
|
1431
|
+
const next = character_at(regex, i + 1);
|
|
1432
|
+
i += 1 + next.length;
|
|
1433
|
+
if (next === "G") {
|
|
1434
|
+
continue;
|
|
1435
|
+
}
|
|
1436
|
+
const raw_whitespace = RAW_WHITESPACE_ESCAPES.get(next);
|
|
1437
|
+
if (raw_whitespace !== undefined) {
|
|
1438
|
+
emit_atom(raw_whitespace);
|
|
1439
|
+
continue;
|
|
1440
|
+
}
|
|
1441
|
+
const rewrite = ESCAPE_REWRITES.get(next);
|
|
1442
|
+
let replacement;
|
|
1443
|
+
if (rewrite !== undefined) {
|
|
1444
|
+
replacement = rewrite;
|
|
1445
|
+
} else if (/[A-Za-z0-9]/.test(next)) {
|
|
1446
|
+
replacement = `\\${next}`;
|
|
1447
|
+
} else if (JS_SYNTAX_CHARS.includes(next)) {
|
|
1448
|
+
replacement = `\\${next}`;
|
|
1449
|
+
} else {
|
|
1450
|
+
replacement = next;
|
|
1451
|
+
}
|
|
1452
|
+
emit_atom(replacement);
|
|
1453
|
+
continue;
|
|
1454
|
+
}
|
|
1455
|
+
switch (char) {
|
|
1456
|
+
case "[": {
|
|
1457
|
+
const parsed = parse_character_class(regex, i, ascii_fold);
|
|
1458
|
+
emit_atom(parsed.atom);
|
|
1459
|
+
i = parsed.end;
|
|
1460
|
+
continue;
|
|
1461
|
+
}
|
|
1462
|
+
case "]":
|
|
1463
|
+
emit_atom("\\]");
|
|
1464
|
+
++i;
|
|
1465
|
+
continue;
|
|
1466
|
+
case ".":
|
|
1467
|
+
emit_atom("[^\\n]");
|
|
1468
|
+
++i;
|
|
1469
|
+
continue;
|
|
1470
|
+
case "^":
|
|
1471
|
+
emit_atom(LINE_START_ANCHOR);
|
|
1472
|
+
++i;
|
|
1473
|
+
continue;
|
|
1474
|
+
case "$":
|
|
1475
|
+
emit_atom(LINE_END_ANCHOR);
|
|
1476
|
+
++i;
|
|
1477
|
+
continue;
|
|
1478
|
+
case "(": {
|
|
1479
|
+
const inline_case_insensitive = regex.startsWith("(?i:", i);
|
|
1480
|
+
const source_prefix = inline_case_insensitive ? "(?i:" : GROUP_PREFIX_RE.exec(regex.slice(i))?.[0] ?? "(";
|
|
1481
|
+
const output_prefix = inline_case_insensitive ? "(?:" : source_prefix === "(?>" ? "(?:" : source_prefix;
|
|
1482
|
+
group_states.push([out.length, ascii_fold]);
|
|
1483
|
+
if (inline_case_insensitive)
|
|
1484
|
+
ascii_fold = true;
|
|
1485
|
+
out += output_prefix;
|
|
1486
|
+
last_was_quantifier = false;
|
|
1487
|
+
i += source_prefix.length;
|
|
1488
|
+
continue;
|
|
1489
|
+
}
|
|
1490
|
+
case ")":
|
|
1491
|
+
out += char;
|
|
1492
|
+
[atom_start, ascii_fold] = group_states.pop() ?? [-1, false];
|
|
1493
|
+
last_was_quantifier = false;
|
|
1494
|
+
++i;
|
|
1495
|
+
continue;
|
|
1496
|
+
case "|":
|
|
1497
|
+
out += char;
|
|
1498
|
+
atom_start = -1;
|
|
1499
|
+
last_was_quantifier = false;
|
|
1500
|
+
++i;
|
|
1501
|
+
continue;
|
|
1502
|
+
case "{": {
|
|
1503
|
+
const quant = QUANTIFIER_BRACE_RE.exec(regex.slice(i));
|
|
1504
|
+
if (!quant || atom_start < 0) {
|
|
1505
|
+
emit_atom("\\{");
|
|
1506
|
+
++i;
|
|
1507
|
+
continue;
|
|
1508
|
+
}
|
|
1509
|
+
const body = quant[1].startsWith(",") ? `0${quant[1]}` : quant[1];
|
|
1510
|
+
i += quant[0].length;
|
|
1511
|
+
const following = regex[i];
|
|
1512
|
+
if (following === "+" || following === "*") {
|
|
1513
|
+
out = `${out.slice(0, atom_start)}(?:${out.slice(atom_start)}{${body}})${following}`;
|
|
1514
|
+
++i;
|
|
1515
|
+
} else {
|
|
1516
|
+
out += `{${body}}`;
|
|
1517
|
+
}
|
|
1518
|
+
last_was_quantifier = true;
|
|
1519
|
+
continue;
|
|
1520
|
+
}
|
|
1521
|
+
case "}":
|
|
1522
|
+
emit_atom("\\}");
|
|
1523
|
+
++i;
|
|
1524
|
+
continue;
|
|
1525
|
+
case "+":
|
|
1526
|
+
if (last_was_quantifier) {
|
|
1527
|
+
++i;
|
|
1528
|
+
continue;
|
|
1529
|
+
}
|
|
1530
|
+
out += char;
|
|
1531
|
+
last_was_quantifier = true;
|
|
1532
|
+
++i;
|
|
1533
|
+
continue;
|
|
1534
|
+
case "*":
|
|
1535
|
+
case "?":
|
|
1536
|
+
out += char;
|
|
1537
|
+
last_was_quantifier = true;
|
|
1538
|
+
++i;
|
|
1539
|
+
continue;
|
|
1540
|
+
default:
|
|
1541
|
+
emit_atom(ascii_fold && is_ascii_letter(char) ? `[${char.toLowerCase()}${char.toUpperCase()}]` : char);
|
|
1542
|
+
i += char.length;
|
|
1543
|
+
continue;
|
|
1544
|
+
}
|
|
1545
|
+
}
|
|
1546
|
+
return out;
|
|
1547
|
+
};
|
|
1107
1548
|
var escape_reg_exp = (string) => string.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
1108
1549
|
var fuse_unk = (arr, tokens_to_ids, unk_token_id) => {
|
|
1109
1550
|
const fused = [];
|
|
@@ -1279,9 +1720,12 @@ var Sequence = class extends Normalizer_default {
|
|
|
1279
1720
|
};
|
|
1280
1721
|
var Sequence_default = Sequence;
|
|
1281
1722
|
var Replace = class extends Normalizer_default {
|
|
1723
|
+
constructor(config) {
|
|
1724
|
+
super(config);
|
|
1725
|
+
this.pattern = create_pattern(this.config.pattern ?? {});
|
|
1726
|
+
}
|
|
1282
1727
|
normalize(text) {
|
|
1283
|
-
|
|
1284
|
-
return pattern === null ? text : text.replaceAll(pattern, this.config.content ?? "");
|
|
1728
|
+
return this.pattern === null ? text : text.replaceAll(this.pattern, this.config.content ?? "");
|
|
1285
1729
|
}
|
|
1286
1730
|
};
|
|
1287
1731
|
var Replace_default = Replace;
|
|
@@ -1401,6 +1845,27 @@ var PreTokenizer = class extends Callable_default {
|
|
|
1401
1845
|
}
|
|
1402
1846
|
};
|
|
1403
1847
|
var PreTokenizer_default = PreTokenizer;
|
|
1848
|
+
var BYTES_TO_UNICODE = (() => {
|
|
1849
|
+
const bs = [
|
|
1850
|
+
...Array.from({ length: 126 - 33 + 1 }, (_, i) => i + 33),
|
|
1851
|
+
...Array.from({ length: "¬".charCodeAt(0) - "¡".charCodeAt(0) + 1 }, (_, i) => i + "¡".charCodeAt(0)),
|
|
1852
|
+
...Array.from({ length: "ÿ".charCodeAt(0) - "®".charCodeAt(0) + 1 }, (_, i) => i + "®".charCodeAt(0))
|
|
1853
|
+
];
|
|
1854
|
+
const cs = bs.slice();
|
|
1855
|
+
let n = 0;
|
|
1856
|
+
for (let b = 0;b < 256; ++b) {
|
|
1857
|
+
if (!bs.includes(b)) {
|
|
1858
|
+
bs.push(b);
|
|
1859
|
+
cs.push(256 + n);
|
|
1860
|
+
n += 1;
|
|
1861
|
+
}
|
|
1862
|
+
}
|
|
1863
|
+
const ccs = cs.map((n2) => String.fromCharCode(n2));
|
|
1864
|
+
return Object.fromEntries(bs.map((b, i) => [b, ccs[i]]));
|
|
1865
|
+
})();
|
|
1866
|
+
var reverse_dictionary = (data) => Object.fromEntries(Object.entries(data).map(([key, value]) => [value, key]));
|
|
1867
|
+
var UNICODE_TO_BYTES = reverse_dictionary(BYTES_TO_UNICODE);
|
|
1868
|
+
var PUNCTUATION_REGEX = "\\p{P}\\u0021-\\u002F\\u003A-\\u0040\\u005B-\\u0060\\u007B-\\u007E";
|
|
1404
1869
|
var ByteLevel = class extends PreTokenizer_default {
|
|
1405
1870
|
constructor(config) {
|
|
1406
1871
|
super();
|
|
@@ -1455,7 +1920,7 @@ var Split = class extends PreTokenizer_default {
|
|
|
1455
1920
|
return [];
|
|
1456
1921
|
}
|
|
1457
1922
|
if (this.config.invert) {
|
|
1458
|
-
return text.match(this.pattern) || [];
|
|
1923
|
+
return (text.match(this.pattern) || []).filter((x) => x);
|
|
1459
1924
|
} else if (this.config.behavior?.toLowerCase() === "removed") {
|
|
1460
1925
|
return text.split(this.pattern).filter((x) => x);
|
|
1461
1926
|
} else {
|
|
@@ -1681,12 +2146,13 @@ var CharTrie = class {
|
|
|
1681
2146
|
}
|
|
1682
2147
|
node.is_leaf = true;
|
|
1683
2148
|
}
|
|
1684
|
-
*common_prefix_search(
|
|
2149
|
+
*common_prefix_search(chars, start = 0) {
|
|
1685
2150
|
let node = this.root;
|
|
1686
2151
|
if (node === undefined)
|
|
1687
2152
|
return;
|
|
1688
2153
|
let prefix = "";
|
|
1689
|
-
for (
|
|
2154
|
+
for (let i = start;i < chars.length; ++i) {
|
|
2155
|
+
const ch = chars[i];
|
|
1690
2156
|
prefix += ch;
|
|
1691
2157
|
node = node.children.get(ch);
|
|
1692
2158
|
if (node === undefined)
|
|
@@ -1836,11 +2302,8 @@ var Unigram = class extends TokenizerModel_default {
|
|
|
1836
2302
|
let begin_pos = 0;
|
|
1837
2303
|
while (begin_pos < chars.length) {
|
|
1838
2304
|
let has_single_node = false;
|
|
1839
|
-
const
|
|
1840
|
-
const sliced = chars.slice(begin_pos).join("");
|
|
1841
|
-
const prefixed_tokens = this.trie.common_prefix_search(sliced);
|
|
2305
|
+
const prefixed_tokens = this.trie.common_prefix_search(chars, begin_pos);
|
|
1842
2306
|
for (const token of prefixed_tokens) {
|
|
1843
|
-
tokens.push(token);
|
|
1844
2307
|
const token_id = this.tokens_to_ids.get(token);
|
|
1845
2308
|
const token_score = this.scores[token_id];
|
|
1846
2309
|
const n = len(token);
|
|
@@ -2445,9 +2908,13 @@ var Sequence4 = class extends Decoder_default {
|
|
|
2445
2908
|
};
|
|
2446
2909
|
var Sequence_default4 = Sequence4;
|
|
2447
2910
|
var Replace3 = class extends Decoder_default {
|
|
2911
|
+
constructor(config) {
|
|
2912
|
+
super(config);
|
|
2913
|
+
this.pattern = create_pattern(this.config.pattern);
|
|
2914
|
+
}
|
|
2448
2915
|
decode_chain(tokens) {
|
|
2449
|
-
const pattern = create_pattern(this.config.pattern);
|
|
2450
2916
|
const content = this.config.content ?? "";
|
|
2917
|
+
const pattern = this.pattern;
|
|
2451
2918
|
return pattern === null ? tokens : tokens.map((token) => token.replaceAll(pattern, content));
|
|
2452
2919
|
}
|
|
2453
2920
|
};
|
|
@@ -2751,6 +3218,12 @@ var Tokenizer = class {
|
|
|
2751
3218
|
}
|
|
2752
3219
|
};
|
|
2753
3220
|
var Tokenizer_default = Tokenizer;
|
|
3221
|
+
var __defProp2 = Object.defineProperty;
|
|
3222
|
+
var __defNormalProp = (obj, key, value) => (key in obj) ? __defProp2(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
|
|
3223
|
+
var __publicField = (obj, key, value) => {
|
|
3224
|
+
__defNormalProp(obj, typeof key !== "symbol" ? key + "" : key, value);
|
|
3225
|
+
return value;
|
|
3226
|
+
};
|
|
2754
3227
|
var TOKEN_TYPES = Object.freeze({
|
|
2755
3228
|
Text: "Text",
|
|
2756
3229
|
NumericLiteral: "NumericLiteral",
|
|
@@ -2774,6 +3247,7 @@ var TOKEN_TYPES = Object.freeze({
|
|
|
2774
3247
|
CallOperator: "CallOperator",
|
|
2775
3248
|
AdditiveBinaryOperator: "AdditiveBinaryOperator",
|
|
2776
3249
|
MultiplicativeBinaryOperator: "MultiplicativeBinaryOperator",
|
|
3250
|
+
ExponentiationBinaryOperator: "ExponentiationBinaryOperator",
|
|
2777
3251
|
ComparisonBinaryOperator: "ComparisonBinaryOperator",
|
|
2778
3252
|
UnaryOperator: "UnaryOperator",
|
|
2779
3253
|
Comment: "Comment"
|
|
@@ -2817,7 +3291,9 @@ var ORDERED_MAPPING_TABLE = [
|
|
|
2817
3291
|
["+", TOKEN_TYPES.AdditiveBinaryOperator],
|
|
2818
3292
|
["-", TOKEN_TYPES.AdditiveBinaryOperator],
|
|
2819
3293
|
["~", TOKEN_TYPES.AdditiveBinaryOperator],
|
|
3294
|
+
["**", TOKEN_TYPES.ExponentiationBinaryOperator],
|
|
2820
3295
|
["*", TOKEN_TYPES.MultiplicativeBinaryOperator],
|
|
3296
|
+
["//", TOKEN_TYPES.MultiplicativeBinaryOperator],
|
|
2821
3297
|
["/", TOKEN_TYPES.MultiplicativeBinaryOperator],
|
|
2822
3298
|
["%", TOKEN_TYPES.MultiplicativeBinaryOperator],
|
|
2823
3299
|
["=", TOKEN_TYPES.Equals]
|
|
@@ -2845,7 +3321,7 @@ function preprocess(template, options = {}) {
|
|
|
2845
3321
|
if (options.trim_blocks) {
|
|
2846
3322
|
template = template.replace(/([#%-]})\n/g, "$1");
|
|
2847
3323
|
}
|
|
2848
|
-
return template.replace(/{
|
|
3324
|
+
return template.replace(/(\s*){%(-?)\s*(?:end)?generation\s*(-?)%}(\s*)/gs, (_, before, lstrip, rstrip, after) => (lstrip ? "" : before) + (rstrip ? "" : after));
|
|
2849
3325
|
}
|
|
2850
3326
|
function tokenize(source, options = {}) {
|
|
2851
3327
|
const tokens = [];
|
|
@@ -2857,8 +3333,9 @@ function tokenize(source, options = {}) {
|
|
|
2857
3333
|
while (predicate(src[cursorPosition])) {
|
|
2858
3334
|
if (src[cursorPosition] === "\\") {
|
|
2859
3335
|
++cursorPosition;
|
|
2860
|
-
if (cursorPosition >= src.length)
|
|
3336
|
+
if (cursorPosition >= src.length) {
|
|
2861
3337
|
throw new SyntaxError("Unexpected end of input");
|
|
3338
|
+
}
|
|
2862
3339
|
const escaped = src[cursorPosition++];
|
|
2863
3340
|
const unescaped = ESCAPE_CHARACTERS.get(escaped);
|
|
2864
3341
|
if (unescaped === undefined) {
|
|
@@ -2868,8 +3345,9 @@ function tokenize(source, options = {}) {
|
|
|
2868
3345
|
continue;
|
|
2869
3346
|
}
|
|
2870
3347
|
str += src[cursorPosition++];
|
|
2871
|
-
if (cursorPosition >= src.length)
|
|
3348
|
+
if (cursorPosition >= src.length) {
|
|
2872
3349
|
throw new SyntaxError("Unexpected end of input");
|
|
3350
|
+
}
|
|
2873
3351
|
}
|
|
2874
3352
|
return str;
|
|
2875
3353
|
};
|
|
@@ -2968,7 +3446,12 @@ function tokenize(source, options = {}) {
|
|
|
2968
3446
|
break;
|
|
2969
3447
|
default: {
|
|
2970
3448
|
++cursorPosition;
|
|
2971
|
-
|
|
3449
|
+
let num = consumeWhile(isInteger);
|
|
3450
|
+
if (num.length > 0 && src[cursorPosition] === "." && isInteger(src[cursorPosition + 1])) {
|
|
3451
|
+
++cursorPosition;
|
|
3452
|
+
const frac = consumeWhile(isInteger);
|
|
3453
|
+
num = `${num}.${frac}`;
|
|
3454
|
+
}
|
|
2972
3455
|
tokens.push(new Token(`${char}${num}`, num.length > 0 ? TOKEN_TYPES.NumericLiteral : TOKEN_TYPES.UnaryOperator));
|
|
2973
3456
|
continue;
|
|
2974
3457
|
}
|
|
@@ -3001,7 +3484,7 @@ function tokenize(source, options = {}) {
|
|
|
3001
3484
|
}
|
|
3002
3485
|
if (isInteger(char)) {
|
|
3003
3486
|
let num = consumeWhile(isInteger);
|
|
3004
|
-
if (src[cursorPosition] === "." && isInteger(src[cursorPosition + 1])) {
|
|
3487
|
+
if (tokens.at(-1)?.type !== TOKEN_TYPES.Dot && src[cursorPosition] === "." && isInteger(src[cursorPosition + 1])) {
|
|
3005
3488
|
++cursorPosition;
|
|
3006
3489
|
const frac = consumeWhile(isInteger);
|
|
3007
3490
|
num = `${num}.${frac}`;
|
|
@@ -3204,6 +3687,13 @@ var SpreadExpression = class extends Expression {
|
|
|
3204
3687
|
}
|
|
3205
3688
|
type = "SpreadExpression";
|
|
3206
3689
|
};
|
|
3690
|
+
var KeywordSpreadExpression = class extends Expression {
|
|
3691
|
+
constructor(argument) {
|
|
3692
|
+
super();
|
|
3693
|
+
this.argument = argument;
|
|
3694
|
+
}
|
|
3695
|
+
type = "KeywordSpreadExpression";
|
|
3696
|
+
};
|
|
3207
3697
|
var CallStatement = class extends Statement {
|
|
3208
3698
|
constructor(call, callerArgs, body) {
|
|
3209
3699
|
super();
|
|
@@ -3301,13 +3791,13 @@ function parse(tokens) {
|
|
|
3301
3791
|
++current;
|
|
3302
3792
|
let callerArgs = null;
|
|
3303
3793
|
if (is(TOKEN_TYPES.OpenParen)) {
|
|
3304
|
-
callerArgs = parseArgs();
|
|
3794
|
+
callerArgs = parseArgs("parameters");
|
|
3305
3795
|
}
|
|
3306
3796
|
const callee = parsePrimaryExpression();
|
|
3307
3797
|
if (callee.type !== "Identifier") {
|
|
3308
3798
|
throw new SyntaxError(`Expected identifier following call statement`);
|
|
3309
3799
|
}
|
|
3310
|
-
const callArgs = parseArgs();
|
|
3800
|
+
const callArgs = parseArgs("call");
|
|
3311
3801
|
expect(TOKEN_TYPES.CloseStatement, "Expected closing statement token");
|
|
3312
3802
|
const body = [];
|
|
3313
3803
|
while (!isStatement("endcall")) {
|
|
@@ -3404,7 +3894,7 @@ function parse(tokens) {
|
|
|
3404
3894
|
if (name.type !== "Identifier") {
|
|
3405
3895
|
throw new SyntaxError(`Expected identifier following macro statement`);
|
|
3406
3896
|
}
|
|
3407
|
-
const args = parseArgs();
|
|
3897
|
+
const args = parseArgs("parameters");
|
|
3408
3898
|
expect(TOKEN_TYPES.CloseStatement, "Expected closing statement token");
|
|
3409
3899
|
const body = [];
|
|
3410
3900
|
while (!isStatement("endmacro")) {
|
|
@@ -3536,24 +4026,47 @@ function parse(tokens) {
|
|
|
3536
4026
|
return member;
|
|
3537
4027
|
}
|
|
3538
4028
|
function parseCallExpression(callee) {
|
|
3539
|
-
let expression = new CallExpression(callee, parseArgs());
|
|
4029
|
+
let expression = new CallExpression(callee, parseArgs("call"));
|
|
3540
4030
|
expression = parseMemberExpression(expression);
|
|
3541
4031
|
if (is(TOKEN_TYPES.OpenParen)) {
|
|
3542
4032
|
expression = parseCallExpression(expression);
|
|
3543
4033
|
}
|
|
3544
4034
|
return expression;
|
|
3545
4035
|
}
|
|
3546
|
-
function parseArgs() {
|
|
4036
|
+
function parseArgs(mode) {
|
|
3547
4037
|
expect(TOKEN_TYPES.OpenParen, "Expected opening parenthesis for arguments list");
|
|
3548
|
-
const args = parseArgumentsList();
|
|
4038
|
+
const args = parseArgumentsList(mode);
|
|
3549
4039
|
expect(TOKEN_TYPES.CloseParen, "Expected closing parenthesis for arguments list");
|
|
3550
4040
|
return args;
|
|
3551
4041
|
}
|
|
3552
|
-
function parseArgumentsList() {
|
|
4042
|
+
function parseArgumentsList(mode) {
|
|
3553
4043
|
const args = [];
|
|
4044
|
+
const parameterNames = /* @__PURE__ */ new Set;
|
|
4045
|
+
let sawKeywordOrDefault = false;
|
|
4046
|
+
let sawSpreadArgument = false;
|
|
3554
4047
|
while (!is(TOKEN_TYPES.CloseParen)) {
|
|
4048
|
+
const isKeywordSpread = is(TOKEN_TYPES.ExponentiationBinaryOperator);
|
|
4049
|
+
const isPositionalSpread = is(TOKEN_TYPES.MultiplicativeBinaryOperator) && tokens[current].value === "*";
|
|
4050
|
+
if (mode === "parameters" && (isKeywordSpread || isPositionalSpread)) {
|
|
4051
|
+
throw new SyntaxError("Argument unpacking is not allowed in parameter declarations");
|
|
4052
|
+
}
|
|
4053
|
+
if (isKeywordSpread) {
|
|
4054
|
+
++current;
|
|
4055
|
+
args.push(new KeywordSpreadExpression(parseExpression()));
|
|
4056
|
+
if (is(TOKEN_TYPES.Comma)) {
|
|
4057
|
+
++current;
|
|
4058
|
+
}
|
|
4059
|
+
if (!is(TOKEN_TYPES.CloseParen)) {
|
|
4060
|
+
throw new SyntaxError("Expected closing parenthesis: `**` must be applied to the final argument");
|
|
4061
|
+
}
|
|
4062
|
+
break;
|
|
4063
|
+
}
|
|
3555
4064
|
let argument;
|
|
3556
|
-
if (
|
|
4065
|
+
if (isPositionalSpread) {
|
|
4066
|
+
if (sawSpreadArgument) {
|
|
4067
|
+
throw new SyntaxError("Only one `*` argument unpacking is allowed");
|
|
4068
|
+
}
|
|
4069
|
+
sawSpreadArgument = true;
|
|
3557
4070
|
++current;
|
|
3558
4071
|
const expr = parseExpression();
|
|
3559
4072
|
argument = new SpreadExpression(expr);
|
|
@@ -3562,11 +4075,28 @@ function parse(tokens) {
|
|
|
3562
4075
|
if (is(TOKEN_TYPES.Equals)) {
|
|
3563
4076
|
++current;
|
|
3564
4077
|
if (!(argument instanceof Identifier)) {
|
|
3565
|
-
throw new SyntaxError(
|
|
4078
|
+
throw new SyntaxError(mode === "parameters" ? "Expected identifier for parameter declaration" : "Expected identifier for keyword argument");
|
|
3566
4079
|
}
|
|
3567
4080
|
const value = parseExpression();
|
|
3568
4081
|
argument = new KeywordArgumentExpression(argument, value);
|
|
3569
4082
|
}
|
|
4083
|
+
if (mode === "parameters") {
|
|
4084
|
+
if (!(argument instanceof Identifier || argument instanceof KeywordArgumentExpression)) {
|
|
4085
|
+
throw new SyntaxError("Expected identifier for parameter declaration");
|
|
4086
|
+
}
|
|
4087
|
+
const parameterName = argument instanceof Identifier ? argument.value : argument.key.value;
|
|
4088
|
+
if (parameterNames.has(parameterName)) {
|
|
4089
|
+
throw new SyntaxError(`Duplicate parameter name: ${parameterName}`);
|
|
4090
|
+
}
|
|
4091
|
+
parameterNames.add(parameterName);
|
|
4092
|
+
}
|
|
4093
|
+
if (argument instanceof KeywordArgumentExpression) {
|
|
4094
|
+
sawKeywordOrDefault = true;
|
|
4095
|
+
} else if (sawKeywordOrDefault) {
|
|
4096
|
+
throw new SyntaxError(mode === "call" ? "Positional arguments must come before keyword arguments" : "Non-default argument follows default argument");
|
|
4097
|
+
} else if (sawSpreadArgument) {
|
|
4098
|
+
throw new SyntaxError("Positional arguments must not follow `*` argument unpacking");
|
|
4099
|
+
}
|
|
3570
4100
|
}
|
|
3571
4101
|
args.push(argument);
|
|
3572
4102
|
if (is(TOKEN_TYPES.Comma)) {
|
|
@@ -3613,8 +4143,8 @@ function parse(tokens) {
|
|
|
3613
4143
|
expect(TOKEN_TYPES.CloseSquareBracket, "Expected closing square bracket");
|
|
3614
4144
|
} else {
|
|
3615
4145
|
property = parsePrimaryExpression();
|
|
3616
|
-
if (property.type !== "Identifier") {
|
|
3617
|
-
throw new SyntaxError(`Expected identifier following dot operator`);
|
|
4146
|
+
if (property.type !== "Identifier" && property.type !== "IntegerLiteral") {
|
|
4147
|
+
throw new SyntaxError(`Expected identifier or integer following dot operator`);
|
|
3618
4148
|
}
|
|
3619
4149
|
}
|
|
3620
4150
|
object = new MemberExpression(object, property, computed);
|
|
@@ -3622,8 +4152,17 @@ function parse(tokens) {
|
|
|
3622
4152
|
return object;
|
|
3623
4153
|
}
|
|
3624
4154
|
function parseMultiplicativeExpression() {
|
|
3625
|
-
let left =
|
|
4155
|
+
let left = parsePowerExpression();
|
|
3626
4156
|
while (is(TOKEN_TYPES.MultiplicativeBinaryOperator)) {
|
|
4157
|
+
const operator = tokens[current++];
|
|
4158
|
+
const right = parsePowerExpression();
|
|
4159
|
+
left = new BinaryExpression(operator, left, right);
|
|
4160
|
+
}
|
|
4161
|
+
return left;
|
|
4162
|
+
}
|
|
4163
|
+
function parsePowerExpression() {
|
|
4164
|
+
let left = parseTestExpression();
|
|
4165
|
+
while (is(TOKEN_TYPES.ExponentiationBinaryOperator)) {
|
|
3627
4166
|
const operator = tokens[current++];
|
|
3628
4167
|
const right = parseTestExpression();
|
|
3629
4168
|
left = new BinaryExpression(operator, left, right);
|
|
@@ -3647,7 +4186,7 @@ function parse(tokens) {
|
|
|
3647
4186
|
return operand;
|
|
3648
4187
|
}
|
|
3649
4188
|
function parseFilterExpression() {
|
|
3650
|
-
let operand =
|
|
4189
|
+
let operand = parseUnarySignExpression();
|
|
3651
4190
|
while (is(TOKEN_TYPES.Pipe)) {
|
|
3652
4191
|
++current;
|
|
3653
4192
|
let filter = parsePrimaryExpression();
|
|
@@ -3661,6 +4200,15 @@ function parse(tokens) {
|
|
|
3661
4200
|
}
|
|
3662
4201
|
return operand;
|
|
3663
4202
|
}
|
|
4203
|
+
function parseUnarySignExpression() {
|
|
4204
|
+
const token = tokens[current];
|
|
4205
|
+
if (token && (token.type === TOKEN_TYPES.UnaryOperator || token.type === TOKEN_TYPES.AdditiveBinaryOperator) && (token.value === "-" || token.value === "+")) {
|
|
4206
|
+
const operator = tokens[current++];
|
|
4207
|
+
const argument = parseUnarySignExpression();
|
|
4208
|
+
return new UnaryExpression(operator, argument);
|
|
4209
|
+
}
|
|
4210
|
+
return parseCallMemberExpression();
|
|
4211
|
+
}
|
|
3664
4212
|
function parsePrimaryExpression() {
|
|
3665
4213
|
const token = tokens[current++];
|
|
3666
4214
|
switch (token.type) {
|
|
@@ -3788,8 +4336,9 @@ function escapeRegExp(s) {
|
|
|
3788
4336
|
return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
3789
4337
|
}
|
|
3790
4338
|
function replace(str, oldvalue, newvalue, count2) {
|
|
3791
|
-
if (count2 === 0)
|
|
4339
|
+
if (count2 === 0) {
|
|
3792
4340
|
return str;
|
|
4341
|
+
}
|
|
3793
4342
|
let remaining = count2 == null || count2 < 0 ? Infinity : count2;
|
|
3794
4343
|
const pattern = oldvalue.length === 0 ? new RegExp("(?=)", "gu") : new RegExp(escapeRegExp(oldvalue), "gu");
|
|
3795
4344
|
return str.replaceAll(pattern, (match) => {
|
|
@@ -3804,10 +4353,13 @@ var BreakControl = class extends Error {
|
|
|
3804
4353
|
};
|
|
3805
4354
|
var ContinueControl = class extends Error {
|
|
3806
4355
|
};
|
|
4356
|
+
var EMPTY_BUILTINS = /* @__PURE__ */ new Map;
|
|
3807
4357
|
var RuntimeValue = class {
|
|
3808
4358
|
type = "RuntimeValue";
|
|
3809
4359
|
value;
|
|
3810
|
-
builtins
|
|
4360
|
+
get builtins() {
|
|
4361
|
+
return EMPTY_BUILTINS;
|
|
4362
|
+
}
|
|
3811
4363
|
constructor(value = undefined) {
|
|
3812
4364
|
this.value = value;
|
|
3813
4365
|
}
|
|
@@ -3824,162 +4376,168 @@ var IntegerValue = class extends RuntimeValue {
|
|
|
3824
4376
|
var FloatValue = class extends RuntimeValue {
|
|
3825
4377
|
type = "FloatValue";
|
|
3826
4378
|
toString() {
|
|
4379
|
+
if (Object.is(this.value, -0)) {
|
|
4380
|
+
return "-0.0";
|
|
4381
|
+
}
|
|
3827
4382
|
return this.value % 1 === 0 ? this.value.toFixed(1) : this.value.toString();
|
|
3828
4383
|
}
|
|
3829
4384
|
};
|
|
3830
4385
|
var StringValue = class extends RuntimeValue {
|
|
3831
4386
|
type = "StringValue";
|
|
3832
|
-
|
|
3833
|
-
|
|
3834
|
-
|
|
3835
|
-
|
|
3836
|
-
|
|
3837
|
-
|
|
3838
|
-
|
|
3839
|
-
|
|
3840
|
-
|
|
3841
|
-
|
|
3842
|
-
|
|
3843
|
-
|
|
3844
|
-
|
|
3845
|
-
|
|
3846
|
-
|
|
3847
|
-
|
|
3848
|
-
|
|
3849
|
-
|
|
3850
|
-
|
|
3851
|
-
|
|
3852
|
-
|
|
3853
|
-
|
|
3854
|
-
|
|
3855
|
-
|
|
3856
|
-
|
|
3857
|
-
|
|
3858
|
-
|
|
3859
|
-
|
|
3860
|
-
|
|
3861
|
-
|
|
3862
|
-
|
|
3863
|
-
|
|
3864
|
-
|
|
3865
|
-
"
|
|
3866
|
-
|
|
3867
|
-
|
|
3868
|
-
|
|
3869
|
-
|
|
3870
|
-
|
|
3871
|
-
|
|
3872
|
-
|
|
3873
|
-
|
|
3874
|
-
|
|
3875
|
-
|
|
3876
|
-
|
|
3877
|
-
|
|
3878
|
-
|
|
3879
|
-
|
|
3880
|
-
|
|
3881
|
-
|
|
3882
|
-
|
|
3883
|
-
|
|
3884
|
-
|
|
3885
|
-
|
|
3886
|
-
|
|
3887
|
-
|
|
3888
|
-
|
|
4387
|
+
_builtins;
|
|
4388
|
+
get builtins() {
|
|
4389
|
+
return this._builtins ??= /* @__PURE__ */ new Map([
|
|
4390
|
+
[
|
|
4391
|
+
"upper",
|
|
4392
|
+
new FunctionValue(() => {
|
|
4393
|
+
return new StringValue(this.value.toUpperCase());
|
|
4394
|
+
})
|
|
4395
|
+
],
|
|
4396
|
+
[
|
|
4397
|
+
"lower",
|
|
4398
|
+
new FunctionValue(() => {
|
|
4399
|
+
return new StringValue(this.value.toLowerCase());
|
|
4400
|
+
})
|
|
4401
|
+
],
|
|
4402
|
+
[
|
|
4403
|
+
"strip",
|
|
4404
|
+
new FunctionValue(() => {
|
|
4405
|
+
return new StringValue(this.value.trim());
|
|
4406
|
+
})
|
|
4407
|
+
],
|
|
4408
|
+
[
|
|
4409
|
+
"title",
|
|
4410
|
+
new FunctionValue(() => {
|
|
4411
|
+
return new StringValue(titleCase(this.value));
|
|
4412
|
+
})
|
|
4413
|
+
],
|
|
4414
|
+
[
|
|
4415
|
+
"capitalize",
|
|
4416
|
+
new FunctionValue(() => {
|
|
4417
|
+
return new StringValue(this.value.charAt(0).toUpperCase() + this.value.slice(1));
|
|
4418
|
+
})
|
|
4419
|
+
],
|
|
4420
|
+
["length", new IntegerValue(this.value.length)],
|
|
4421
|
+
[
|
|
4422
|
+
"rstrip",
|
|
4423
|
+
new FunctionValue(() => {
|
|
4424
|
+
return new StringValue(this.value.trimEnd());
|
|
4425
|
+
})
|
|
4426
|
+
],
|
|
4427
|
+
[
|
|
4428
|
+
"lstrip",
|
|
4429
|
+
new FunctionValue(() => {
|
|
4430
|
+
return new StringValue(this.value.trimStart());
|
|
4431
|
+
})
|
|
4432
|
+
],
|
|
4433
|
+
[
|
|
4434
|
+
"startswith",
|
|
4435
|
+
new FunctionValue((args) => {
|
|
4436
|
+
if (args.length === 0) {
|
|
4437
|
+
throw new Error("startswith() requires at least one argument");
|
|
4438
|
+
}
|
|
4439
|
+
const pattern = args[0];
|
|
4440
|
+
if (pattern instanceof StringValue) {
|
|
4441
|
+
return new BooleanValue(this.value.startsWith(pattern.value));
|
|
4442
|
+
} else if (pattern instanceof ArrayValue) {
|
|
4443
|
+
for (const item of pattern.value) {
|
|
4444
|
+
if (!(item instanceof StringValue)) {
|
|
4445
|
+
throw new Error("startswith() tuple elements must be strings");
|
|
4446
|
+
}
|
|
4447
|
+
if (this.value.startsWith(item.value)) {
|
|
4448
|
+
return new BooleanValue(true);
|
|
4449
|
+
}
|
|
3889
4450
|
}
|
|
3890
|
-
|
|
3891
|
-
|
|
4451
|
+
return new BooleanValue(false);
|
|
4452
|
+
}
|
|
4453
|
+
throw new Error("startswith() argument must be a string or tuple of strings");
|
|
4454
|
+
})
|
|
4455
|
+
],
|
|
4456
|
+
[
|
|
4457
|
+
"endswith",
|
|
4458
|
+
new FunctionValue((args) => {
|
|
4459
|
+
if (args.length === 0) {
|
|
4460
|
+
throw new Error("endswith() requires at least one argument");
|
|
4461
|
+
}
|
|
4462
|
+
const pattern = args[0];
|
|
4463
|
+
if (pattern instanceof StringValue) {
|
|
4464
|
+
return new BooleanValue(this.value.endsWith(pattern.value));
|
|
4465
|
+
} else if (pattern instanceof ArrayValue) {
|
|
4466
|
+
for (const item of pattern.value) {
|
|
4467
|
+
if (!(item instanceof StringValue)) {
|
|
4468
|
+
throw new Error("endswith() tuple elements must be strings");
|
|
4469
|
+
}
|
|
4470
|
+
if (this.value.endsWith(item.value)) {
|
|
4471
|
+
return new BooleanValue(true);
|
|
4472
|
+
}
|
|
3892
4473
|
}
|
|
4474
|
+
return new BooleanValue(false);
|
|
3893
4475
|
}
|
|
3894
|
-
|
|
3895
|
-
}
|
|
3896
|
-
|
|
3897
|
-
|
|
3898
|
-
|
|
3899
|
-
|
|
3900
|
-
|
|
3901
|
-
|
|
3902
|
-
|
|
3903
|
-
|
|
3904
|
-
|
|
3905
|
-
|
|
3906
|
-
|
|
3907
|
-
|
|
3908
|
-
|
|
3909
|
-
|
|
3910
|
-
|
|
3911
|
-
|
|
4476
|
+
throw new Error("endswith() argument must be a string or tuple of strings");
|
|
4477
|
+
})
|
|
4478
|
+
],
|
|
4479
|
+
[
|
|
4480
|
+
"split",
|
|
4481
|
+
new FunctionValue((args) => {
|
|
4482
|
+
const sep = args[0] ?? new NullValue;
|
|
4483
|
+
if (!(sep instanceof StringValue || sep instanceof NullValue)) {
|
|
4484
|
+
throw new Error("sep argument must be a string or null");
|
|
4485
|
+
}
|
|
4486
|
+
const maxsplit = args[1] ?? new IntegerValue(-1);
|
|
4487
|
+
if (!(maxsplit instanceof IntegerValue)) {
|
|
4488
|
+
throw new Error("maxsplit argument must be a number");
|
|
4489
|
+
}
|
|
4490
|
+
let result = [];
|
|
4491
|
+
if (sep instanceof NullValue) {
|
|
4492
|
+
const text = this.value.trimStart();
|
|
4493
|
+
for (const { 0: match, index } of text.matchAll(/\S+/g)) {
|
|
4494
|
+
if (maxsplit.value !== -1 && result.length >= maxsplit.value && index !== undefined) {
|
|
4495
|
+
result.push(match + text.slice(index + match.length));
|
|
4496
|
+
break;
|
|
4497
|
+
}
|
|
4498
|
+
result.push(match);
|
|
3912
4499
|
}
|
|
3913
|
-
|
|
3914
|
-
|
|
4500
|
+
} else {
|
|
4501
|
+
if (sep.value === "") {
|
|
4502
|
+
throw new Error("empty separator");
|
|
3915
4503
|
}
|
|
3916
|
-
|
|
3917
|
-
|
|
3918
|
-
|
|
3919
|
-
throw new Error("endswith() argument must be a string or tuple of strings");
|
|
3920
|
-
})
|
|
3921
|
-
],
|
|
3922
|
-
[
|
|
3923
|
-
"split",
|
|
3924
|
-
new FunctionValue((args) => {
|
|
3925
|
-
const sep = args[0] ?? new NullValue;
|
|
3926
|
-
if (!(sep instanceof StringValue || sep instanceof NullValue)) {
|
|
3927
|
-
throw new Error("sep argument must be a string or null");
|
|
3928
|
-
}
|
|
3929
|
-
const maxsplit = args[1] ?? new IntegerValue(-1);
|
|
3930
|
-
if (!(maxsplit instanceof IntegerValue)) {
|
|
3931
|
-
throw new Error("maxsplit argument must be a number");
|
|
3932
|
-
}
|
|
3933
|
-
let result = [];
|
|
3934
|
-
if (sep instanceof NullValue) {
|
|
3935
|
-
const text = this.value.trimStart();
|
|
3936
|
-
for (const { 0: match, index } of text.matchAll(/\S+/g)) {
|
|
3937
|
-
if (maxsplit.value !== -1 && result.length >= maxsplit.value && index !== undefined) {
|
|
3938
|
-
result.push(match + text.slice(index + match.length));
|
|
3939
|
-
break;
|
|
4504
|
+
result = this.value.split(sep.value);
|
|
4505
|
+
if (maxsplit.value !== -1 && result.length > maxsplit.value) {
|
|
4506
|
+
result.push(result.splice(maxsplit.value).join(sep.value));
|
|
3940
4507
|
}
|
|
3941
|
-
result.push(match);
|
|
3942
4508
|
}
|
|
3943
|
-
|
|
3944
|
-
|
|
3945
|
-
|
|
4509
|
+
return new ArrayValue(result.map((part) => new StringValue(part)));
|
|
4510
|
+
})
|
|
4511
|
+
],
|
|
4512
|
+
[
|
|
4513
|
+
"replace",
|
|
4514
|
+
new FunctionValue((args) => {
|
|
4515
|
+
if (args.length < 2) {
|
|
4516
|
+
throw new Error("replace() requires at least two arguments");
|
|
3946
4517
|
}
|
|
3947
|
-
|
|
3948
|
-
|
|
3949
|
-
|
|
4518
|
+
const oldValue = args[0];
|
|
4519
|
+
const newValue = args[1];
|
|
4520
|
+
if (!(oldValue instanceof StringValue && newValue instanceof StringValue)) {
|
|
4521
|
+
throw new Error("replace() arguments must be strings");
|
|
3950
4522
|
}
|
|
3951
|
-
|
|
3952
|
-
|
|
3953
|
-
|
|
3954
|
-
|
|
3955
|
-
|
|
3956
|
-
|
|
3957
|
-
|
|
3958
|
-
if (args.length < 2) {
|
|
3959
|
-
throw new Error("replace() requires at least two arguments");
|
|
3960
|
-
}
|
|
3961
|
-
const oldValue = args[0];
|
|
3962
|
-
const newValue = args[1];
|
|
3963
|
-
if (!(oldValue instanceof StringValue && newValue instanceof StringValue)) {
|
|
3964
|
-
throw new Error("replace() arguments must be strings");
|
|
3965
|
-
}
|
|
3966
|
-
let count2;
|
|
3967
|
-
if (args.length > 2) {
|
|
3968
|
-
if (args[2].type === "KeywordArgumentsValue") {
|
|
3969
|
-
count2 = args[2].value.get("count") ?? new NullValue;
|
|
4523
|
+
let count2;
|
|
4524
|
+
if (args.length > 2) {
|
|
4525
|
+
if (args[2].type === "KeywordArgumentsValue") {
|
|
4526
|
+
count2 = args[2].value.get("count") ?? new NullValue;
|
|
4527
|
+
} else {
|
|
4528
|
+
count2 = args[2];
|
|
4529
|
+
}
|
|
3970
4530
|
} else {
|
|
3971
|
-
count2 =
|
|
4531
|
+
count2 = new NullValue;
|
|
3972
4532
|
}
|
|
3973
|
-
|
|
3974
|
-
|
|
3975
|
-
|
|
3976
|
-
|
|
3977
|
-
|
|
3978
|
-
|
|
3979
|
-
|
|
3980
|
-
|
|
3981
|
-
]
|
|
3982
|
-
]);
|
|
4533
|
+
if (!(count2 instanceof IntegerValue || count2 instanceof NullValue)) {
|
|
4534
|
+
throw new Error("replace() count argument must be a number or null");
|
|
4535
|
+
}
|
|
4536
|
+
return new StringValue(replace(this.value, oldValue.value, newValue.value, count2.value));
|
|
4537
|
+
})
|
|
4538
|
+
]
|
|
4539
|
+
]);
|
|
4540
|
+
}
|
|
3983
4541
|
};
|
|
3984
4542
|
var BooleanValue = class extends RuntimeValue {
|
|
3985
4543
|
type = "BooleanValue";
|
|
@@ -4018,6 +4576,7 @@ function toJSON(input, options = {}, depth = 0, convertUndefinedToNull = true) {
|
|
|
4018
4576
|
return result;
|
|
4019
4577
|
}
|
|
4020
4578
|
case "ArrayValue":
|
|
4579
|
+
case "NamespaceValue":
|
|
4021
4580
|
case "ObjectValue": {
|
|
4022
4581
|
const indentValue = indent ? " ".repeat(indent) : "";
|
|
4023
4582
|
const basePadding = `
|
|
@@ -4048,59 +4607,62 @@ function toJSON(input, options = {}, depth = 0, convertUndefinedToNull = true) {
|
|
|
4048
4607
|
}
|
|
4049
4608
|
var ObjectValue = class extends RuntimeValue {
|
|
4050
4609
|
type = "ObjectValue";
|
|
4610
|
+
_builtins;
|
|
4051
4611
|
__bool__() {
|
|
4052
4612
|
return new BooleanValue(this.value.size > 0);
|
|
4053
4613
|
}
|
|
4054
|
-
builtins
|
|
4055
|
-
[
|
|
4056
|
-
|
|
4057
|
-
|
|
4058
|
-
|
|
4059
|
-
|
|
4060
|
-
|
|
4061
|
-
return this.value.get(key.value) ?? defaultValue ?? new NullValue;
|
|
4062
|
-
})
|
|
4063
|
-
],
|
|
4064
|
-
["items", new FunctionValue(() => this.items())],
|
|
4065
|
-
["keys", new FunctionValue(() => this.keys())],
|
|
4066
|
-
["values", new FunctionValue(() => this.values())],
|
|
4067
|
-
[
|
|
4068
|
-
"dictsort",
|
|
4069
|
-
new FunctionValue((args) => {
|
|
4070
|
-
let kwargs = /* @__PURE__ */ new Map;
|
|
4071
|
-
const positionalArgs = args.filter((arg) => {
|
|
4072
|
-
if (arg instanceof KeywordArgumentsValue) {
|
|
4073
|
-
kwargs = arg.value;
|
|
4074
|
-
return false;
|
|
4614
|
+
get builtins() {
|
|
4615
|
+
return this._builtins ??= /* @__PURE__ */ new Map([
|
|
4616
|
+
[
|
|
4617
|
+
"get",
|
|
4618
|
+
new FunctionValue(([key, defaultValue]) => {
|
|
4619
|
+
if (!(key instanceof StringValue)) {
|
|
4620
|
+
throw new Error(`Object key must be a string: got ${key.type}`);
|
|
4075
4621
|
}
|
|
4076
|
-
return
|
|
4077
|
-
})
|
|
4078
|
-
|
|
4079
|
-
|
|
4080
|
-
|
|
4081
|
-
|
|
4082
|
-
|
|
4083
|
-
|
|
4084
|
-
|
|
4085
|
-
|
|
4086
|
-
|
|
4087
|
-
|
|
4088
|
-
|
|
4089
|
-
|
|
4090
|
-
|
|
4091
|
-
|
|
4092
|
-
|
|
4093
|
-
|
|
4094
|
-
|
|
4095
|
-
|
|
4096
|
-
|
|
4097
|
-
const
|
|
4098
|
-
|
|
4099
|
-
|
|
4100
|
-
|
|
4101
|
-
|
|
4102
|
-
|
|
4103
|
-
|
|
4622
|
+
return this.value.get(key.value) ?? defaultValue ?? new NullValue;
|
|
4623
|
+
})
|
|
4624
|
+
],
|
|
4625
|
+
["items", new FunctionValue(() => this.items())],
|
|
4626
|
+
["keys", new FunctionValue(() => this.keys())],
|
|
4627
|
+
["values", new FunctionValue(() => this.values())],
|
|
4628
|
+
[
|
|
4629
|
+
"dictsort",
|
|
4630
|
+
new FunctionValue((args) => {
|
|
4631
|
+
let kwargs = /* @__PURE__ */ new Map;
|
|
4632
|
+
const positionalArgs = args.filter((arg) => {
|
|
4633
|
+
if (arg instanceof KeywordArgumentsValue) {
|
|
4634
|
+
kwargs = arg.value;
|
|
4635
|
+
return false;
|
|
4636
|
+
}
|
|
4637
|
+
return true;
|
|
4638
|
+
});
|
|
4639
|
+
const caseSensitive = positionalArgs.at(0) ?? kwargs.get("case_sensitive") ?? new BooleanValue(false);
|
|
4640
|
+
if (!(caseSensitive instanceof BooleanValue)) {
|
|
4641
|
+
throw new Error("case_sensitive must be a boolean");
|
|
4642
|
+
}
|
|
4643
|
+
const by = positionalArgs.at(1) ?? kwargs.get("by") ?? new StringValue("key");
|
|
4644
|
+
if (!(by instanceof StringValue)) {
|
|
4645
|
+
throw new Error("by must be a string");
|
|
4646
|
+
}
|
|
4647
|
+
if (!["key", "value"].includes(by.value)) {
|
|
4648
|
+
throw new Error("by must be either 'key' or 'value'");
|
|
4649
|
+
}
|
|
4650
|
+
const reverse = positionalArgs.at(2) ?? kwargs.get("reverse") ?? new BooleanValue(false);
|
|
4651
|
+
if (!(reverse instanceof BooleanValue)) {
|
|
4652
|
+
throw new Error("reverse must be a boolean");
|
|
4653
|
+
}
|
|
4654
|
+
const items = Array.from(this.value.entries()).map(([key, value]) => new ArrayValue([new StringValue(key), value])).sort((a, b) => {
|
|
4655
|
+
const index = by.value === "key" ? 0 : 1;
|
|
4656
|
+
const aVal = a.value[index];
|
|
4657
|
+
const bVal = b.value[index];
|
|
4658
|
+
const result = compareRuntimeValues(aVal, bVal, caseSensitive.value);
|
|
4659
|
+
return reverse.value ? -result : result;
|
|
4660
|
+
});
|
|
4661
|
+
return new ArrayValue(items);
|
|
4662
|
+
})
|
|
4663
|
+
]
|
|
4664
|
+
]);
|
|
4665
|
+
}
|
|
4104
4666
|
items() {
|
|
4105
4667
|
return new ArrayValue(Array.from(this.value.entries()).map(([key, value]) => new ArrayValue([new StringValue(key), value])));
|
|
4106
4668
|
}
|
|
@@ -4117,9 +4679,18 @@ var ObjectValue = class extends RuntimeValue {
|
|
|
4117
4679
|
var KeywordArgumentsValue = class extends ObjectValue {
|
|
4118
4680
|
type = "KeywordArgumentsValue";
|
|
4119
4681
|
};
|
|
4682
|
+
var NamespaceValue = class extends RuntimeValue {
|
|
4683
|
+
type = "NamespaceValue";
|
|
4684
|
+
toString() {
|
|
4685
|
+
return toJSON(this, {}, 0, false);
|
|
4686
|
+
}
|
|
4687
|
+
};
|
|
4120
4688
|
var ArrayValue = class extends RuntimeValue {
|
|
4121
4689
|
type = "ArrayValue";
|
|
4122
|
-
|
|
4690
|
+
_builtins;
|
|
4691
|
+
get builtins() {
|
|
4692
|
+
return this._builtins ??= /* @__PURE__ */ new Map([["length", new IntegerValue(this.value.length)]]);
|
|
4693
|
+
}
|
|
4123
4694
|
__bool__() {
|
|
4124
4695
|
return new BooleanValue(this.value.length > 0);
|
|
4125
4696
|
}
|
|
@@ -4139,7 +4710,23 @@ var NullValue = class extends RuntimeValue {
|
|
|
4139
4710
|
var UndefinedValue = class extends RuntimeValue {
|
|
4140
4711
|
type = "UndefinedValue";
|
|
4141
4712
|
};
|
|
4142
|
-
|
|
4713
|
+
function normalizeNamespaceEntry(pair) {
|
|
4714
|
+
let values;
|
|
4715
|
+
if (pair instanceof ArrayValue) {
|
|
4716
|
+
values = pair.value;
|
|
4717
|
+
} else if (pair instanceof StringValue) {
|
|
4718
|
+
values = Array.from(pair.value, (char) => new StringValue(char));
|
|
4719
|
+
}
|
|
4720
|
+
if (!values || values.length !== 2) {
|
|
4721
|
+
throw new Error("namespace expected an object or an iterable of [key, value] pairs");
|
|
4722
|
+
}
|
|
4723
|
+
const [key, value] = values;
|
|
4724
|
+
if (!(key instanceof StringValue)) {
|
|
4725
|
+
throw new Error("namespace keys must be strings");
|
|
4726
|
+
}
|
|
4727
|
+
return [key, value];
|
|
4728
|
+
}
|
|
4729
|
+
var _Environment = class {
|
|
4143
4730
|
constructor(parent) {
|
|
4144
4731
|
this.parent = parent;
|
|
4145
4732
|
}
|
|
@@ -4147,69 +4734,40 @@ var Environment = class {
|
|
|
4147
4734
|
[
|
|
4148
4735
|
"namespace",
|
|
4149
4736
|
new FunctionValue((args) => {
|
|
4150
|
-
|
|
4151
|
-
|
|
4737
|
+
const positional = args.slice();
|
|
4738
|
+
let kwargs;
|
|
4739
|
+
if (positional.at(-1) instanceof KeywordArgumentsValue) {
|
|
4740
|
+
kwargs = positional.pop();
|
|
4152
4741
|
}
|
|
4153
|
-
if (
|
|
4154
|
-
throw new Error(
|
|
4742
|
+
if (positional.length > 1) {
|
|
4743
|
+
throw new Error(`namespace expected at most 1 argument, got ${positional.length}`);
|
|
4155
4744
|
}
|
|
4156
|
-
|
|
4745
|
+
const entries = /* @__PURE__ */ new Map;
|
|
4746
|
+
if (positional.length === 1) {
|
|
4747
|
+
const source = positional[0];
|
|
4748
|
+
if (source instanceof ObjectValue) {
|
|
4749
|
+
for (const [key, value] of source.value) {
|
|
4750
|
+
entries.set(key, value);
|
|
4751
|
+
}
|
|
4752
|
+
} else if (source instanceof ArrayValue) {
|
|
4753
|
+
for (const pair of source.value) {
|
|
4754
|
+
const [key, value] = normalizeNamespaceEntry(pair);
|
|
4755
|
+
entries.set(key.value, value);
|
|
4756
|
+
}
|
|
4757
|
+
} else {
|
|
4758
|
+
throw new Error(`'${source.type}' object is not iterable`);
|
|
4759
|
+
}
|
|
4760
|
+
}
|
|
4761
|
+
if (kwargs) {
|
|
4762
|
+
for (const [key, value] of kwargs.value) {
|
|
4763
|
+
entries.set(key, value);
|
|
4764
|
+
}
|
|
4765
|
+
}
|
|
4766
|
+
return new NamespaceValue(entries);
|
|
4157
4767
|
})
|
|
4158
4768
|
]
|
|
4159
4769
|
]);
|
|
4160
|
-
tests =
|
|
4161
|
-
["boolean", (operand) => operand.type === "BooleanValue"],
|
|
4162
|
-
["callable", (operand) => operand instanceof FunctionValue],
|
|
4163
|
-
[
|
|
4164
|
-
"odd",
|
|
4165
|
-
(operand) => {
|
|
4166
|
-
if (!(operand instanceof IntegerValue)) {
|
|
4167
|
-
throw new Error(`cannot odd on ${operand.type}`);
|
|
4168
|
-
}
|
|
4169
|
-
return operand.value % 2 !== 0;
|
|
4170
|
-
}
|
|
4171
|
-
],
|
|
4172
|
-
[
|
|
4173
|
-
"even",
|
|
4174
|
-
(operand) => {
|
|
4175
|
-
if (!(operand instanceof IntegerValue)) {
|
|
4176
|
-
throw new Error(`cannot even on ${operand.type}`);
|
|
4177
|
-
}
|
|
4178
|
-
return operand.value % 2 === 0;
|
|
4179
|
-
}
|
|
4180
|
-
],
|
|
4181
|
-
["false", (operand) => operand.type === "BooleanValue" && !operand.value],
|
|
4182
|
-
["true", (operand) => operand.type === "BooleanValue" && operand.value],
|
|
4183
|
-
["none", (operand) => operand.type === "NullValue"],
|
|
4184
|
-
["string", (operand) => operand.type === "StringValue"],
|
|
4185
|
-
["number", (operand) => operand instanceof IntegerValue || operand instanceof FloatValue],
|
|
4186
|
-
["integer", (operand) => operand instanceof IntegerValue],
|
|
4187
|
-
["iterable", (operand) => operand.type === "ArrayValue" || operand.type === "StringValue"],
|
|
4188
|
-
["mapping", (operand) => operand instanceof ObjectValue],
|
|
4189
|
-
[
|
|
4190
|
-
"sequence",
|
|
4191
|
-
(operand) => operand instanceof ArrayValue || operand instanceof ObjectValue || operand instanceof StringValue
|
|
4192
|
-
],
|
|
4193
|
-
[
|
|
4194
|
-
"lower",
|
|
4195
|
-
(operand) => {
|
|
4196
|
-
const str = operand.value;
|
|
4197
|
-
return operand.type === "StringValue" && str === str.toLowerCase();
|
|
4198
|
-
}
|
|
4199
|
-
],
|
|
4200
|
-
[
|
|
4201
|
-
"upper",
|
|
4202
|
-
(operand) => {
|
|
4203
|
-
const str = operand.value;
|
|
4204
|
-
return operand.type === "StringValue" && str === str.toUpperCase();
|
|
4205
|
-
}
|
|
4206
|
-
],
|
|
4207
|
-
["none", (operand) => operand.type === "NullValue"],
|
|
4208
|
-
["defined", (operand) => operand.type !== "UndefinedValue"],
|
|
4209
|
-
["undefined", (operand) => operand.type === "UndefinedValue"],
|
|
4210
|
-
["equalto", (a, b) => a.value === b.value],
|
|
4211
|
-
["eq", (a, b) => a.value === b.value]
|
|
4212
|
-
]);
|
|
4770
|
+
tests = _Environment.TESTS;
|
|
4213
4771
|
set(name, value) {
|
|
4214
4772
|
return this.declareVariable(name, convertToRuntimeValues(value));
|
|
4215
4773
|
}
|
|
@@ -4241,6 +4799,60 @@ var Environment = class {
|
|
|
4241
4799
|
}
|
|
4242
4800
|
}
|
|
4243
4801
|
};
|
|
4802
|
+
var Environment = _Environment;
|
|
4803
|
+
__publicField(Environment, "TESTS", /* @__PURE__ */ new Map([
|
|
4804
|
+
["boolean", (operand) => operand.type === "BooleanValue"],
|
|
4805
|
+
["callable", (operand) => operand instanceof FunctionValue],
|
|
4806
|
+
[
|
|
4807
|
+
"odd",
|
|
4808
|
+
(operand) => {
|
|
4809
|
+
if (!(operand instanceof IntegerValue)) {
|
|
4810
|
+
throw new Error(`cannot odd on ${operand.type}`);
|
|
4811
|
+
}
|
|
4812
|
+
return operand.value % 2 !== 0;
|
|
4813
|
+
}
|
|
4814
|
+
],
|
|
4815
|
+
[
|
|
4816
|
+
"even",
|
|
4817
|
+
(operand) => {
|
|
4818
|
+
if (!(operand instanceof IntegerValue)) {
|
|
4819
|
+
throw new Error(`cannot even on ${operand.type}`);
|
|
4820
|
+
}
|
|
4821
|
+
return operand.value % 2 === 0;
|
|
4822
|
+
}
|
|
4823
|
+
],
|
|
4824
|
+
["false", (operand) => operand.type === "BooleanValue" && !operand.value],
|
|
4825
|
+
["true", (operand) => operand.type === "BooleanValue" && operand.value],
|
|
4826
|
+
["none", (operand) => operand.type === "NullValue"],
|
|
4827
|
+
["string", (operand) => operand.type === "StringValue"],
|
|
4828
|
+
["number", (operand) => operand instanceof IntegerValue || operand instanceof FloatValue],
|
|
4829
|
+
["integer", (operand) => operand instanceof IntegerValue],
|
|
4830
|
+
["iterable", (operand) => operand.type === "ArrayValue" || operand.type === "StringValue"],
|
|
4831
|
+
["mapping", (operand) => operand instanceof ObjectValue],
|
|
4832
|
+
[
|
|
4833
|
+
"sequence",
|
|
4834
|
+
(operand) => operand instanceof ArrayValue || operand instanceof ObjectValue || operand instanceof StringValue
|
|
4835
|
+
],
|
|
4836
|
+
[
|
|
4837
|
+
"lower",
|
|
4838
|
+
(operand) => {
|
|
4839
|
+
const str = operand.value;
|
|
4840
|
+
return operand.type === "StringValue" && str === str.toLowerCase();
|
|
4841
|
+
}
|
|
4842
|
+
],
|
|
4843
|
+
[
|
|
4844
|
+
"upper",
|
|
4845
|
+
(operand) => {
|
|
4846
|
+
const str = operand.value;
|
|
4847
|
+
return operand.type === "StringValue" && str === str.toUpperCase();
|
|
4848
|
+
}
|
|
4849
|
+
],
|
|
4850
|
+
["none", (operand) => operand.type === "NullValue"],
|
|
4851
|
+
["defined", (operand) => operand.type !== "UndefinedValue"],
|
|
4852
|
+
["undefined", (operand) => operand.type === "UndefinedValue"],
|
|
4853
|
+
["equalto", (a, b) => a.value === b.value],
|
|
4854
|
+
["eq", (a, b) => a.value === b.value]
|
|
4855
|
+
]));
|
|
4244
4856
|
function setupGlobals(env2) {
|
|
4245
4857
|
env2.set("false", false);
|
|
4246
4858
|
env2.set("true", true);
|
|
@@ -4254,11 +4866,20 @@ function setupGlobals(env2) {
|
|
|
4254
4866
|
env2.set("False", false);
|
|
4255
4867
|
env2.set("None", null);
|
|
4256
4868
|
}
|
|
4869
|
+
function isAttributeContainer(value) {
|
|
4870
|
+
return value instanceof ObjectValue || value instanceof NamespaceValue;
|
|
4871
|
+
}
|
|
4872
|
+
function isNumericLikeValue(value) {
|
|
4873
|
+
return value instanceof IntegerValue || value instanceof FloatValue || value instanceof BooleanValue;
|
|
4874
|
+
}
|
|
4875
|
+
function getNumericValue(value) {
|
|
4876
|
+
return value instanceof BooleanValue ? Number(value.value) : value.value;
|
|
4877
|
+
}
|
|
4257
4878
|
function getAttributeValue(item, attributePath) {
|
|
4258
4879
|
const parts = attributePath.split(".");
|
|
4259
4880
|
let value = item;
|
|
4260
4881
|
for (const part of parts) {
|
|
4261
|
-
if (value
|
|
4882
|
+
if (isAttributeContainer(value)) {
|
|
4262
4883
|
value = value.value.get(part) ?? new UndefinedValue;
|
|
4263
4884
|
} else if (value instanceof ArrayValue) {
|
|
4264
4885
|
const index = parseInt(part, 10);
|
|
@@ -4286,14 +4907,7 @@ function compareRuntimeValues(a, b, caseSensitive = false) {
|
|
|
4286
4907
|
if (a instanceof UndefinedValue || b instanceof UndefinedValue) {
|
|
4287
4908
|
throw new Error(`Cannot compare ${a.type} with ${b.type}`);
|
|
4288
4909
|
}
|
|
4289
|
-
|
|
4290
|
-
const getNumericValue = (v) => {
|
|
4291
|
-
if (v instanceof BooleanValue) {
|
|
4292
|
-
return v.value ? 1 : 0;
|
|
4293
|
-
}
|
|
4294
|
-
return v.value;
|
|
4295
|
-
};
|
|
4296
|
-
if (isNumericLike(a) && isNumericLike(b)) {
|
|
4910
|
+
if (isNumericLikeValue(a) && isNumericLikeValue(b)) {
|
|
4297
4911
|
const aNum = getNumericValue(a);
|
|
4298
4912
|
const bNum = getNumericValue(b);
|
|
4299
4913
|
return aNum < bNum ? -1 : aNum > bNum ? 1 : 0;
|
|
@@ -4309,11 +4923,165 @@ function compareRuntimeValues(a, b, caseSensitive = false) {
|
|
|
4309
4923
|
aStr = aStr.toLowerCase();
|
|
4310
4924
|
bStr = bStr.toLowerCase();
|
|
4311
4925
|
}
|
|
4312
|
-
return aStr < bStr ? -1 : aStr > bStr ? 1 : 0;
|
|
4926
|
+
return aStr < bStr ? -1 : aStr > bStr ? 1 : 0;
|
|
4927
|
+
}
|
|
4928
|
+
default:
|
|
4929
|
+
throw new Error(`Cannot compare type: ${a.type}`);
|
|
4930
|
+
}
|
|
4931
|
+
}
|
|
4932
|
+
function getParameterDescriptor(argument) {
|
|
4933
|
+
if (argument.type === "Identifier") {
|
|
4934
|
+
return { name: argument.value };
|
|
4935
|
+
}
|
|
4936
|
+
const keywordArgument = argument;
|
|
4937
|
+
return { name: keywordArgument.key.value, defaultValue: keywordArgument.value };
|
|
4938
|
+
}
|
|
4939
|
+
function isSpecialMacroArgument(name) {
|
|
4940
|
+
return name === "kwargs" || name === "varargs";
|
|
4941
|
+
}
|
|
4942
|
+
function findAccessedSpecialMacroArguments(parameters, body) {
|
|
4943
|
+
const declared = /* @__PURE__ */ new Set;
|
|
4944
|
+
const accessed = /* @__PURE__ */ new Set;
|
|
4945
|
+
const declareName = (name) => {
|
|
4946
|
+
if (isSpecialMacroArgument(name)) {
|
|
4947
|
+
declared.add(name);
|
|
4948
|
+
}
|
|
4949
|
+
};
|
|
4950
|
+
const declareTarget = (target) => {
|
|
4951
|
+
if (target.type === "Identifier") {
|
|
4952
|
+
declareName(target.value);
|
|
4953
|
+
} else if (target.type === "TupleLiteral") {
|
|
4954
|
+
for (const item of target.value) {
|
|
4955
|
+
declareTarget(item);
|
|
4956
|
+
}
|
|
4957
|
+
}
|
|
4958
|
+
};
|
|
4959
|
+
const declareArgument = (argument) => {
|
|
4960
|
+
declareName(getParameterDescriptor(argument).name);
|
|
4961
|
+
};
|
|
4962
|
+
const visitArgumentDefaults = (args) => {
|
|
4963
|
+
for (const argument of args) {
|
|
4964
|
+
const parameter = getParameterDescriptor(argument);
|
|
4965
|
+
if (parameter.defaultValue) {
|
|
4966
|
+
visit(parameter.defaultValue);
|
|
4967
|
+
}
|
|
4968
|
+
}
|
|
4969
|
+
};
|
|
4970
|
+
const visitFilter = (filter) => {
|
|
4971
|
+
if (filter.type === "CallExpression") {
|
|
4972
|
+
visit(filter.args);
|
|
4973
|
+
}
|
|
4974
|
+
};
|
|
4975
|
+
const visit = (node) => {
|
|
4976
|
+
if (Array.isArray(node)) {
|
|
4977
|
+
for (const item of node) {
|
|
4978
|
+
visit(item);
|
|
4979
|
+
}
|
|
4980
|
+
return;
|
|
4981
|
+
}
|
|
4982
|
+
if (node instanceof Map) {
|
|
4983
|
+
for (const [key, value] of node) {
|
|
4984
|
+
visit(key);
|
|
4985
|
+
visit(value);
|
|
4986
|
+
}
|
|
4987
|
+
return;
|
|
4988
|
+
}
|
|
4989
|
+
if (!(node instanceof Statement)) {
|
|
4990
|
+
return;
|
|
4991
|
+
}
|
|
4992
|
+
switch (node.type) {
|
|
4993
|
+
case "For": {
|
|
4994
|
+
const statement = node;
|
|
4995
|
+
declareTarget(statement.loopvar);
|
|
4996
|
+
if (statement.iterable.type === "SelectExpression") {
|
|
4997
|
+
const iterable = statement.iterable;
|
|
4998
|
+
visit(iterable.lhs);
|
|
4999
|
+
visit(statement.body);
|
|
5000
|
+
visit(statement.defaultBlock);
|
|
5001
|
+
visit(iterable.test);
|
|
5002
|
+
return;
|
|
5003
|
+
}
|
|
5004
|
+
visit(statement.iterable);
|
|
5005
|
+
visit(statement.body);
|
|
5006
|
+
visit(statement.defaultBlock);
|
|
5007
|
+
return;
|
|
5008
|
+
}
|
|
5009
|
+
case "Set": {
|
|
5010
|
+
const statement = node;
|
|
5011
|
+
declareTarget(statement.assignee);
|
|
5012
|
+
visit(statement.value);
|
|
5013
|
+
visit(statement.body);
|
|
5014
|
+
return;
|
|
5015
|
+
}
|
|
5016
|
+
case "Macro": {
|
|
5017
|
+
const statement = node;
|
|
5018
|
+
for (const argument of statement.args) {
|
|
5019
|
+
declareArgument(argument);
|
|
5020
|
+
}
|
|
5021
|
+
visitArgumentDefaults(statement.args);
|
|
5022
|
+
visit(statement.body);
|
|
5023
|
+
return;
|
|
5024
|
+
}
|
|
5025
|
+
case "CallStatement": {
|
|
5026
|
+
const statement = node;
|
|
5027
|
+
visit(statement.call);
|
|
5028
|
+
for (const argument of statement.callerArgs ?? []) {
|
|
5029
|
+
declareArgument(argument);
|
|
5030
|
+
}
|
|
5031
|
+
visitArgumentDefaults(statement.callerArgs ?? []);
|
|
5032
|
+
visit(statement.body);
|
|
5033
|
+
return;
|
|
5034
|
+
}
|
|
5035
|
+
case "FilterStatement": {
|
|
5036
|
+
const statement = node;
|
|
5037
|
+
visit(statement.body);
|
|
5038
|
+
visitFilter(statement.filter);
|
|
5039
|
+
return;
|
|
5040
|
+
}
|
|
5041
|
+
case "Identifier": {
|
|
5042
|
+
const name = node.value;
|
|
5043
|
+
if (isSpecialMacroArgument(name) && !declared.has(name)) {
|
|
5044
|
+
accessed.add(name);
|
|
5045
|
+
}
|
|
5046
|
+
return;
|
|
5047
|
+
}
|
|
5048
|
+
case "MemberExpression": {
|
|
5049
|
+
const expression = node;
|
|
5050
|
+
visit(expression.object);
|
|
5051
|
+
if (expression.computed) {
|
|
5052
|
+
visit(expression.property);
|
|
5053
|
+
}
|
|
5054
|
+
return;
|
|
5055
|
+
}
|
|
5056
|
+
case "FilterExpression": {
|
|
5057
|
+
const expression = node;
|
|
5058
|
+
visit(expression.operand);
|
|
5059
|
+
visitFilter(expression.filter);
|
|
5060
|
+
return;
|
|
5061
|
+
}
|
|
5062
|
+
case "TestExpression":
|
|
5063
|
+
visit(node.operand);
|
|
5064
|
+
return;
|
|
5065
|
+
case "SelectExpression": {
|
|
5066
|
+
const expression = node;
|
|
5067
|
+
visit(expression.test);
|
|
5068
|
+
visit(expression.lhs);
|
|
5069
|
+
return;
|
|
5070
|
+
}
|
|
5071
|
+
case "KeywordArgumentExpression":
|
|
5072
|
+
visit(node.value);
|
|
5073
|
+
return;
|
|
5074
|
+
default:
|
|
5075
|
+
for (const child of Object.values(node)) {
|
|
5076
|
+
visit(child);
|
|
5077
|
+
}
|
|
4313
5078
|
}
|
|
4314
|
-
|
|
4315
|
-
|
|
5079
|
+
};
|
|
5080
|
+
for (const parameter of parameters) {
|
|
5081
|
+
declareArgument(parameter);
|
|
4316
5082
|
}
|
|
5083
|
+
visit(body);
|
|
5084
|
+
return accessed;
|
|
4317
5085
|
}
|
|
4318
5086
|
var Interpreter = class {
|
|
4319
5087
|
global;
|
|
@@ -4347,6 +5115,18 @@ var Interpreter = class {
|
|
|
4347
5115
|
throw new Error("Cannot perform operation on null values");
|
|
4348
5116
|
} else if (node.operator.value === "~") {
|
|
4349
5117
|
return new StringValue(left.value.toString() + right.value.toString());
|
|
5118
|
+
} else if (node.operator.value === "**" && isNumericLikeValue(left) && isNumericLikeValue(right)) {
|
|
5119
|
+
const a = getNumericValue(left);
|
|
5120
|
+
const b = getNumericValue(right);
|
|
5121
|
+
if (a === 0 && b < 0) {
|
|
5122
|
+
throw new Error("0.0 cannot be raised to a negative power");
|
|
5123
|
+
}
|
|
5124
|
+
const result = a ** b;
|
|
5125
|
+
if (!Number.isFinite(result)) {
|
|
5126
|
+
throw new Error("Exponentiation result is not a finite real number");
|
|
5127
|
+
}
|
|
5128
|
+
const isFloat = left instanceof FloatValue || right instanceof FloatValue || b < 0;
|
|
5129
|
+
return isFloat ? new FloatValue(result) : new IntegerValue(result);
|
|
4350
5130
|
} else if ((left instanceof IntegerValue || left instanceof FloatValue) && (right instanceof IntegerValue || right instanceof FloatValue)) {
|
|
4351
5131
|
const a = left.value, b = right.value;
|
|
4352
5132
|
switch (node.operator.value) {
|
|
@@ -4359,6 +5139,11 @@ var Interpreter = class {
|
|
|
4359
5139
|
}
|
|
4360
5140
|
case "/":
|
|
4361
5141
|
return new FloatValue(a / b);
|
|
5142
|
+
case "//": {
|
|
5143
|
+
const res = Math.floor(a / b);
|
|
5144
|
+
const isFloat = left instanceof FloatValue || right instanceof FloatValue;
|
|
5145
|
+
return isFloat ? new FloatValue(res) : new IntegerValue(res);
|
|
5146
|
+
}
|
|
4362
5147
|
case "%": {
|
|
4363
5148
|
const rem = a % b;
|
|
4364
5149
|
const isFloat = left instanceof FloatValue || right instanceof FloatValue;
|
|
@@ -4414,6 +5199,12 @@ var Interpreter = class {
|
|
|
4414
5199
|
evaluateArguments(args, environment) {
|
|
4415
5200
|
const positionalArguments = [];
|
|
4416
5201
|
const keywordArguments = /* @__PURE__ */ new Map;
|
|
5202
|
+
const addKeywordArgument = (key, value) => {
|
|
5203
|
+
if (keywordArguments.has(key)) {
|
|
5204
|
+
throw new Error(`Got multiple values for keyword argument '${key}'`);
|
|
5205
|
+
}
|
|
5206
|
+
keywordArguments.set(key, value);
|
|
5207
|
+
};
|
|
4417
5208
|
for (const argument of args) {
|
|
4418
5209
|
if (argument.type === "SpreadExpression") {
|
|
4419
5210
|
const spreadNode = argument;
|
|
@@ -4424,14 +5215,23 @@ var Interpreter = class {
|
|
|
4424
5215
|
for (const item of val.value) {
|
|
4425
5216
|
positionalArguments.push(item);
|
|
4426
5217
|
}
|
|
4427
|
-
} else if (argument.type
|
|
5218
|
+
} else if (argument.type !== "KeywordArgumentExpression" && argument.type !== "KeywordSpreadExpression") {
|
|
5219
|
+
positionalArguments.push(this.evaluate(argument, environment));
|
|
5220
|
+
}
|
|
5221
|
+
}
|
|
5222
|
+
for (const argument of args) {
|
|
5223
|
+
if (argument.type === "KeywordArgumentExpression") {
|
|
4428
5224
|
const kwarg = argument;
|
|
4429
|
-
|
|
4430
|
-
} else {
|
|
4431
|
-
|
|
4432
|
-
|
|
5225
|
+
addKeywordArgument(kwarg.key.value, this.evaluate(kwarg.value, environment));
|
|
5226
|
+
} else if (argument.type === "KeywordSpreadExpression") {
|
|
5227
|
+
const spreadNode = argument;
|
|
5228
|
+
const val = this.evaluate(spreadNode.argument, environment);
|
|
5229
|
+
if (!(val instanceof ObjectValue)) {
|
|
5230
|
+
throw new Error(`Argument after ** must be a mapping, not ${val.type}`);
|
|
5231
|
+
}
|
|
5232
|
+
for (const [key, value] of val.value) {
|
|
5233
|
+
addKeywordArgument(key, value);
|
|
4433
5234
|
}
|
|
4434
|
-
positionalArguments.push(this.evaluate(argument, environment));
|
|
4435
5235
|
}
|
|
4436
5236
|
}
|
|
4437
5237
|
return [positionalArguments, keywordArguments];
|
|
@@ -4672,7 +5472,7 @@ var Interpreter = class {
|
|
|
4672
5472
|
case "selectattr":
|
|
4673
5473
|
case "rejectattr": {
|
|
4674
5474
|
const select = filterName === "selectattr";
|
|
4675
|
-
if (operand.value.some((
|
|
5475
|
+
if (operand.value.some((item) => !isAttributeContainer(item))) {
|
|
4676
5476
|
throw new Error(`\`${filterName}\` can only be applied to array of objects`);
|
|
4677
5477
|
}
|
|
4678
5478
|
if (filter.args.some((x) => x.type !== "StringLiteral")) {
|
|
@@ -4705,7 +5505,7 @@ var Interpreter = class {
|
|
|
4705
5505
|
}
|
|
4706
5506
|
const defaultValue = kwargs.get("default");
|
|
4707
5507
|
const mapped = operand.value.map((item) => {
|
|
4708
|
-
if (!(item
|
|
5508
|
+
if (!isAttributeContainer(item)) {
|
|
4709
5509
|
throw new Error("items in map must be an object");
|
|
4710
5510
|
}
|
|
4711
5511
|
const value = getAttributeValue(item, attr.value);
|
|
@@ -4786,6 +5586,16 @@ var Interpreter = class {
|
|
|
4786
5586
|
switch (node.operator.value) {
|
|
4787
5587
|
case "not":
|
|
4788
5588
|
return new BooleanValue(!argument.value);
|
|
5589
|
+
case "+":
|
|
5590
|
+
case "-": {
|
|
5591
|
+
const sign = node.operator.value === "-" ? -1 : 1;
|
|
5592
|
+
if (argument instanceof IntegerValue || argument instanceof FloatValue || argument instanceof BooleanValue) {
|
|
5593
|
+
const value = argument instanceof BooleanValue ? argument.value ? 1 : 0 : argument.value;
|
|
5594
|
+
const result = sign * value;
|
|
5595
|
+
return argument instanceof FloatValue ? new FloatValue(result) : new IntegerValue(result);
|
|
5596
|
+
}
|
|
5597
|
+
throw new SyntaxError(`Unknown operator "${node.operator.value}" for ${argument.type}`);
|
|
5598
|
+
}
|
|
4789
5599
|
default:
|
|
4790
5600
|
throw new SyntaxError(`Unknown operator: ${node.operator.value}`);
|
|
4791
5601
|
}
|
|
@@ -4852,15 +5662,17 @@ var Interpreter = class {
|
|
|
4852
5662
|
} else {
|
|
4853
5663
|
property = this.evaluate(expr.property, environment);
|
|
4854
5664
|
}
|
|
5665
|
+
} else if (expr.property.type === "IntegerLiteral") {
|
|
5666
|
+
property = new IntegerValue(expr.property.value);
|
|
4855
5667
|
} else {
|
|
4856
5668
|
property = new StringValue(expr.property.value);
|
|
4857
5669
|
}
|
|
4858
5670
|
let value;
|
|
4859
|
-
if (object
|
|
5671
|
+
if (isAttributeContainer(object)) {
|
|
4860
5672
|
if (!(property instanceof StringValue)) {
|
|
4861
5673
|
throw new Error(`Cannot access property with non-string: got ${property.type}`);
|
|
4862
5674
|
}
|
|
4863
|
-
value = object.value.get(property.value) ?? object.builtins.get(property.value);
|
|
5675
|
+
value = object.value.get(property.value) ?? (object instanceof ObjectValue ? object.builtins.get(property.value) : undefined);
|
|
4864
5676
|
} else if (object instanceof ArrayValue || object instanceof StringValue) {
|
|
4865
5677
|
if (property instanceof IntegerValue) {
|
|
4866
5678
|
value = object.value.at(property.value);
|
|
@@ -4904,8 +5716,8 @@ var Interpreter = class {
|
|
|
4904
5716
|
} else if (node.assignee.type === "MemberExpression") {
|
|
4905
5717
|
const member = node.assignee;
|
|
4906
5718
|
const object = this.evaluate(member.object, environment);
|
|
4907
|
-
if (!(object instanceof
|
|
4908
|
-
throw new Error("
|
|
5719
|
+
if (!(object instanceof NamespaceValue)) {
|
|
5720
|
+
throw new Error("cannot assign attribute on non-namespace object");
|
|
4909
5721
|
}
|
|
4910
5722
|
if (member.property.type !== "Identifier") {
|
|
4911
5723
|
throw new Error("Cannot assign to member with non-identifier property");
|
|
@@ -5010,47 +5822,54 @@ var Interpreter = class {
|
|
|
5010
5822
|
}
|
|
5011
5823
|
return new StringValue(result);
|
|
5012
5824
|
}
|
|
5825
|
+
bindMacroArguments(displayName, parameters, specialArguments, args, scope) {
|
|
5826
|
+
const positionalArguments = args.slice();
|
|
5827
|
+
let keywordArguments = /* @__PURE__ */ new Map;
|
|
5828
|
+
if (positionalArguments.at(-1) instanceof KeywordArgumentsValue) {
|
|
5829
|
+
keywordArguments = new Map(positionalArguments.pop().value);
|
|
5830
|
+
}
|
|
5831
|
+
const pendingDefaults = [];
|
|
5832
|
+
for (let i = 0;i < parameters.length; ++i) {
|
|
5833
|
+
const { name, defaultValue } = getParameterDescriptor(parameters[i]);
|
|
5834
|
+
let value = positionalArguments[i];
|
|
5835
|
+
if (value === undefined && keywordArguments.has(name)) {
|
|
5836
|
+
value = keywordArguments.get(name);
|
|
5837
|
+
keywordArguments.delete(name);
|
|
5838
|
+
}
|
|
5839
|
+
if (value === undefined && defaultValue !== undefined) {
|
|
5840
|
+
pendingDefaults.push([name, defaultValue]);
|
|
5841
|
+
}
|
|
5842
|
+
scope.setVariable(name, value ?? new UndefinedValue);
|
|
5843
|
+
}
|
|
5844
|
+
if (specialArguments.has("kwargs")) {
|
|
5845
|
+
scope.setVariable("kwargs", new ObjectValue(keywordArguments));
|
|
5846
|
+
} else if (keywordArguments.size > 0) {
|
|
5847
|
+
throw new Error(`macro ${displayName} takes no keyword argument '${keywordArguments.keys().next().value}'`);
|
|
5848
|
+
}
|
|
5849
|
+
if (specialArguments.has("varargs")) {
|
|
5850
|
+
scope.setVariable("varargs", new ArrayValue(positionalArguments.slice(parameters.length)));
|
|
5851
|
+
} else if (positionalArguments.length > parameters.length) {
|
|
5852
|
+
throw new Error(`macro ${displayName} takes not more than ${parameters.length} argument(s)`);
|
|
5853
|
+
}
|
|
5854
|
+
for (const [name, defaultValue] of pendingDefaults) {
|
|
5855
|
+
scope.setVariable(name, this.evaluate(defaultValue, scope));
|
|
5856
|
+
}
|
|
5857
|
+
}
|
|
5013
5858
|
evaluateMacro(node, environment) {
|
|
5859
|
+
const specialArguments = findAccessedSpecialMacroArguments(node.args, node.body);
|
|
5014
5860
|
environment.setVariable(node.name.value, new FunctionValue((args, scope) => {
|
|
5015
5861
|
const macroScope = new Environment(scope);
|
|
5016
|
-
args
|
|
5017
|
-
let kwargs;
|
|
5018
|
-
if (args.at(-1)?.type === "KeywordArgumentsValue") {
|
|
5019
|
-
kwargs = args.pop();
|
|
5020
|
-
}
|
|
5021
|
-
for (let i = 0;i < node.args.length; ++i) {
|
|
5022
|
-
const nodeArg = node.args[i];
|
|
5023
|
-
const passedArg = args[i];
|
|
5024
|
-
if (nodeArg.type === "Identifier") {
|
|
5025
|
-
const identifier = nodeArg;
|
|
5026
|
-
if (!passedArg) {
|
|
5027
|
-
throw new Error(`Missing positional argument: ${identifier.value}`);
|
|
5028
|
-
}
|
|
5029
|
-
macroScope.setVariable(identifier.value, passedArg);
|
|
5030
|
-
} else if (nodeArg.type === "KeywordArgumentExpression") {
|
|
5031
|
-
const kwarg = nodeArg;
|
|
5032
|
-
const value = passedArg ?? kwargs?.value.get(kwarg.key.value) ?? this.evaluate(kwarg.value, macroScope);
|
|
5033
|
-
macroScope.setVariable(kwarg.key.value, value);
|
|
5034
|
-
} else {
|
|
5035
|
-
throw new Error(`Unknown argument type: ${nodeArg.type}`);
|
|
5036
|
-
}
|
|
5037
|
-
}
|
|
5862
|
+
this.bindMacroArguments(`'${node.name.value}'`, node.args, specialArguments, args, macroScope);
|
|
5038
5863
|
return this.evaluateBlock(node.body, macroScope);
|
|
5039
5864
|
}));
|
|
5040
5865
|
return new NullValue;
|
|
5041
5866
|
}
|
|
5042
5867
|
evaluateCallStatement(node, environment) {
|
|
5043
|
-
const
|
|
5044
|
-
|
|
5045
|
-
|
|
5046
|
-
|
|
5047
|
-
|
|
5048
|
-
if (param.type !== "Identifier") {
|
|
5049
|
-
throw new Error(`Caller parameter must be an identifier, got ${param.type}`);
|
|
5050
|
-
}
|
|
5051
|
-
callBlockEnv.setVariable(param.value, callerArgs[i] ?? new UndefinedValue);
|
|
5052
|
-
}
|
|
5053
|
-
}
|
|
5868
|
+
const parameters = node.callerArgs ?? [];
|
|
5869
|
+
const specialArguments = findAccessedSpecialMacroArguments(parameters, node.body);
|
|
5870
|
+
const callerFn = new FunctionValue((callerArguments) => {
|
|
5871
|
+
const callBlockEnv = new Environment(environment);
|
|
5872
|
+
this.bindMacroArguments("None", parameters, specialArguments, callerArguments, callBlockEnv);
|
|
5054
5873
|
return this.evaluateBlock(node.body, callBlockEnv);
|
|
5055
5874
|
});
|
|
5056
5875
|
const [macroArgs, macroKwargs] = this.evaluateArguments(node.call.args, environment);
|
|
@@ -5068,8 +5887,9 @@ var Interpreter = class {
|
|
|
5068
5887
|
return this.applyFilter(rendered, node.filter, environment);
|
|
5069
5888
|
}
|
|
5070
5889
|
evaluate(statement, environment) {
|
|
5071
|
-
if (!statement)
|
|
5890
|
+
if (!statement) {
|
|
5072
5891
|
return new UndefinedValue;
|
|
5892
|
+
}
|
|
5073
5893
|
switch (statement.type) {
|
|
5074
5894
|
case "Program":
|
|
5075
5895
|
return this.evalProgram(statement, environment);
|
|
@@ -5166,22 +5986,61 @@ var NEWLINE = `
|
|
|
5166
5986
|
`;
|
|
5167
5987
|
var OPEN_STATEMENT = "{%- ";
|
|
5168
5988
|
var CLOSE_STATEMENT = " -%}";
|
|
5989
|
+
var PRECEDENCE = Object.freeze({
|
|
5990
|
+
CONDITIONAL: 0,
|
|
5991
|
+
LOGICAL_OR: 1,
|
|
5992
|
+
LOGICAL_AND: 2,
|
|
5993
|
+
LOGICAL_NOT: 3,
|
|
5994
|
+
COMPARISON: 4,
|
|
5995
|
+
ADDITIVE: 5,
|
|
5996
|
+
MULTIPLICATIVE: 6,
|
|
5997
|
+
EXPONENTIATION: 7,
|
|
5998
|
+
TEST: 8,
|
|
5999
|
+
FILTER: 9,
|
|
6000
|
+
UNARY_SIGN: 10,
|
|
6001
|
+
ATOM: 11
|
|
6002
|
+
});
|
|
5169
6003
|
function getBinaryOperatorPrecedence(expr) {
|
|
5170
6004
|
switch (expr.operator.type) {
|
|
6005
|
+
case "ExponentiationBinaryOperator":
|
|
6006
|
+
return PRECEDENCE.EXPONENTIATION;
|
|
5171
6007
|
case "MultiplicativeBinaryOperator":
|
|
5172
|
-
return
|
|
6008
|
+
return PRECEDENCE.MULTIPLICATIVE;
|
|
5173
6009
|
case "AdditiveBinaryOperator":
|
|
5174
|
-
return
|
|
6010
|
+
return PRECEDENCE.ADDITIVE;
|
|
5175
6011
|
case "ComparisonBinaryOperator":
|
|
5176
|
-
return
|
|
6012
|
+
return PRECEDENCE.COMPARISON;
|
|
5177
6013
|
case "Identifier":
|
|
5178
|
-
if (expr.operator.value === "and")
|
|
5179
|
-
return
|
|
5180
|
-
|
|
5181
|
-
|
|
5182
|
-
|
|
6014
|
+
if (expr.operator.value === "and") {
|
|
6015
|
+
return PRECEDENCE.LOGICAL_AND;
|
|
6016
|
+
}
|
|
6017
|
+
if (expr.operator.value === "in" || expr.operator.value === "not in") {
|
|
6018
|
+
return PRECEDENCE.COMPARISON;
|
|
6019
|
+
}
|
|
6020
|
+
return PRECEDENCE.LOGICAL_OR;
|
|
6021
|
+
}
|
|
6022
|
+
return PRECEDENCE.LOGICAL_OR;
|
|
6023
|
+
}
|
|
6024
|
+
function getPrecedence(node) {
|
|
6025
|
+
switch (node.type) {
|
|
6026
|
+
case "SelectExpression":
|
|
6027
|
+
case "Ternary":
|
|
6028
|
+
return PRECEDENCE.CONDITIONAL;
|
|
6029
|
+
case "BinaryExpression":
|
|
6030
|
+
return getBinaryOperatorPrecedence(node);
|
|
6031
|
+
case "UnaryExpression":
|
|
6032
|
+
return node.operator.value === "not" ? PRECEDENCE.LOGICAL_NOT : PRECEDENCE.UNARY_SIGN;
|
|
6033
|
+
case "TestExpression":
|
|
6034
|
+
return PRECEDENCE.TEST;
|
|
6035
|
+
case "FilterExpression":
|
|
6036
|
+
return PRECEDENCE.FILTER;
|
|
6037
|
+
default:
|
|
6038
|
+
return PRECEDENCE.ATOM;
|
|
5183
6039
|
}
|
|
5184
|
-
|
|
6040
|
+
}
|
|
6041
|
+
function formatOperand(node, minPrecedence) {
|
|
6042
|
+
const expr = formatExpression(node);
|
|
6043
|
+
return getPrecedence(node) < minPrecedence ? `(${expr})` : expr;
|
|
5185
6044
|
}
|
|
5186
6045
|
function format(program, indent = "\t") {
|
|
5187
6046
|
const indentStr = typeof indent === "number" ? " ".repeat(indent) : indent;
|
|
@@ -5194,6 +6053,9 @@ function createStatement(...text) {
|
|
|
5194
6053
|
function formatStatements(stmts, depth, indentStr) {
|
|
5195
6054
|
return stmts.map((stmt) => formatStatement(stmt, depth, indentStr)).join(NEWLINE);
|
|
5196
6055
|
}
|
|
6056
|
+
function formatExpressionList(expressions) {
|
|
6057
|
+
return expressions.map((expression) => formatExpression(expression)).join(", ");
|
|
6058
|
+
}
|
|
5197
6059
|
function formatStatement(node, depth, indentStr) {
|
|
5198
6060
|
const pad = indentStr.repeat(depth);
|
|
5199
6061
|
switch (node.type) {
|
|
@@ -5271,12 +6133,12 @@ function formatSet(node, depth, indentStr) {
|
|
|
5271
6133
|
}
|
|
5272
6134
|
function formatMacro(node, depth, indentStr) {
|
|
5273
6135
|
const pad = indentStr.repeat(depth);
|
|
5274
|
-
const args = node.args
|
|
6136
|
+
const args = formatExpressionList(node.args);
|
|
5275
6137
|
return pad + createStatement("macro", `${node.name.value}(${args})`) + NEWLINE + formatStatements(node.body, depth + 1, indentStr) + NEWLINE + pad + createStatement("endmacro");
|
|
5276
6138
|
}
|
|
5277
6139
|
function formatCallStatement(node, depth, indentStr) {
|
|
5278
6140
|
const pad = indentStr.repeat(depth);
|
|
5279
|
-
const params = node.callerArgs && node.callerArgs.length > 0 ? `(${node.callerArgs
|
|
6141
|
+
const params = node.callerArgs && node.callerArgs.length > 0 ? `(${formatExpressionList(node.callerArgs)})` : "";
|
|
5280
6142
|
const callExpr = formatExpression(node.call);
|
|
5281
6143
|
let out = pad + createStatement(`call${params}`, callExpr) + NEWLINE;
|
|
5282
6144
|
out += formatStatements(node.body, depth + 1, indentStr) + NEWLINE;
|
|
@@ -5291,63 +6153,58 @@ function formatFilterStatement(node, depth, indentStr) {
|
|
|
5291
6153
|
out += pad + createStatement("endfilter");
|
|
5292
6154
|
return out;
|
|
5293
6155
|
}
|
|
5294
|
-
function formatExpression(node
|
|
6156
|
+
function formatExpression(node) {
|
|
5295
6157
|
switch (node.type) {
|
|
5296
6158
|
case "SpreadExpression": {
|
|
5297
6159
|
const n = node;
|
|
5298
6160
|
return `*${formatExpression(n.argument)}`;
|
|
5299
6161
|
}
|
|
6162
|
+
case "KeywordSpreadExpression": {
|
|
6163
|
+
const n = node;
|
|
6164
|
+
return `**${formatExpression(n.argument)}`;
|
|
6165
|
+
}
|
|
5300
6166
|
case "Identifier":
|
|
5301
6167
|
return node.value;
|
|
5302
6168
|
case "IntegerLiteral":
|
|
5303
6169
|
return `${node.value}`;
|
|
5304
|
-
case "FloatLiteral":
|
|
5305
|
-
|
|
6170
|
+
case "FloatLiteral": {
|
|
6171
|
+
const value = node.value;
|
|
6172
|
+
if (Object.is(value, -0)) {
|
|
6173
|
+
return "-0.0";
|
|
6174
|
+
}
|
|
6175
|
+
return value % 1 === 0 ? value.toFixed(1) : value.toString();
|
|
6176
|
+
}
|
|
5306
6177
|
case "StringLiteral":
|
|
5307
6178
|
return JSON.stringify(node.value);
|
|
5308
6179
|
case "BinaryExpression": {
|
|
5309
6180
|
const n = node;
|
|
5310
6181
|
const thisPrecedence = getBinaryOperatorPrecedence(n);
|
|
5311
|
-
const left =
|
|
5312
|
-
const right =
|
|
5313
|
-
|
|
5314
|
-
return thisPrecedence < parentPrec ? `(${expr})` : expr;
|
|
6182
|
+
const left = formatOperand(n.left, thisPrecedence);
|
|
6183
|
+
const right = formatOperand(n.right, thisPrecedence + 1);
|
|
6184
|
+
return `${left} ${n.operator.value} ${right}`;
|
|
5315
6185
|
}
|
|
5316
6186
|
case "UnaryExpression": {
|
|
5317
6187
|
const n = node;
|
|
5318
|
-
const
|
|
5319
|
-
return
|
|
6188
|
+
const operandPrecedence = n.argument.type === "UnaryExpression" ? getPrecedence(n) : PRECEDENCE.ATOM;
|
|
6189
|
+
return n.operator.value + (n.operator.value === "not" ? " " : "") + formatOperand(n.argument, operandPrecedence);
|
|
5320
6190
|
}
|
|
5321
6191
|
case "CallExpression": {
|
|
5322
6192
|
const n = node;
|
|
5323
|
-
const args = n.args
|
|
6193
|
+
const args = formatExpressionList(n.args);
|
|
5324
6194
|
return `${formatExpression(n.callee)}(${args})`;
|
|
5325
6195
|
}
|
|
5326
6196
|
case "MemberExpression": {
|
|
5327
6197
|
const n = node;
|
|
5328
|
-
|
|
5329
|
-
if (![
|
|
5330
|
-
"Identifier",
|
|
5331
|
-
"MemberExpression",
|
|
5332
|
-
"CallExpression",
|
|
5333
|
-
"StringLiteral",
|
|
5334
|
-
"IntegerLiteral",
|
|
5335
|
-
"FloatLiteral",
|
|
5336
|
-
"ArrayLiteral",
|
|
5337
|
-
"TupleLiteral",
|
|
5338
|
-
"ObjectLiteral"
|
|
5339
|
-
].includes(n.object.type)) {
|
|
5340
|
-
obj = `(${obj})`;
|
|
5341
|
-
}
|
|
6198
|
+
const obj = formatOperand(n.object, PRECEDENCE.ATOM);
|
|
5342
6199
|
let prop = formatExpression(n.property);
|
|
5343
|
-
if (!n.computed && n.property.type !== "Identifier") {
|
|
6200
|
+
if (!n.computed && n.property.type !== "Identifier" && n.property.type !== "IntegerLiteral") {
|
|
5344
6201
|
prop = `(${prop})`;
|
|
5345
6202
|
}
|
|
5346
6203
|
return n.computed ? `${obj}[${prop}]` : `${obj}.${prop}`;
|
|
5347
6204
|
}
|
|
5348
6205
|
case "FilterExpression": {
|
|
5349
6206
|
const n = node;
|
|
5350
|
-
const operand =
|
|
6207
|
+
const operand = formatOperand(n.operand, PRECEDENCE.FILTER);
|
|
5351
6208
|
if (n.filter.type === "CallExpression") {
|
|
5352
6209
|
return `${operand} | ${formatExpression(n.filter)}`;
|
|
5353
6210
|
}
|
|
@@ -5355,17 +6212,17 @@ function formatExpression(node, parentPrec = -1) {
|
|
|
5355
6212
|
}
|
|
5356
6213
|
case "SelectExpression": {
|
|
5357
6214
|
const n = node;
|
|
5358
|
-
return `${
|
|
6215
|
+
return `${formatOperand(n.lhs, PRECEDENCE.LOGICAL_OR)} if ${formatOperand(n.test, PRECEDENCE.LOGICAL_OR)}`;
|
|
5359
6216
|
}
|
|
5360
6217
|
case "TestExpression": {
|
|
5361
6218
|
const n = node;
|
|
5362
|
-
return `${
|
|
6219
|
+
return `${formatOperand(n.operand, PRECEDENCE.TEST)} is${n.negate ? " not" : ""} ${n.test.value}`;
|
|
5363
6220
|
}
|
|
5364
6221
|
case "ArrayLiteral":
|
|
5365
6222
|
case "TupleLiteral": {
|
|
5366
|
-
const elems = node.value
|
|
6223
|
+
const elems = formatExpressionList(node.value);
|
|
5367
6224
|
const brackets = node.type === "ArrayLiteral" ? "[]" : "()";
|
|
5368
|
-
return `${brackets[0]}${elems
|
|
6225
|
+
return `${brackets[0]}${elems}${brackets[1]}`;
|
|
5369
6226
|
}
|
|
5370
6227
|
case "ObjectLiteral": {
|
|
5371
6228
|
const entries = Array.from(node.value.entries()).map(([k, v]) => `${formatExpression(k)}: ${formatExpression(v)}`);
|
|
@@ -5384,8 +6241,7 @@ function formatExpression(node, parentPrec = -1) {
|
|
|
5384
6241
|
}
|
|
5385
6242
|
case "Ternary": {
|
|
5386
6243
|
const n = node;
|
|
5387
|
-
|
|
5388
|
-
return parentPrec > -1 ? `(${expr})` : expr;
|
|
6244
|
+
return `${formatOperand(n.trueExpr, PRECEDENCE.LOGICAL_OR)} if ${formatOperand(n.condition, PRECEDENCE.LOGICAL_OR)} else ${formatOperand(n.falseExpr, PRECEDENCE.CONDITIONAL)}`;
|
|
5389
6245
|
}
|
|
5390
6246
|
default:
|
|
5391
6247
|
throw new Error(`Unknown expression type: ${node.type}`);
|
|
@@ -5712,12 +6568,33 @@ function isValidHfModelId(string) {
|
|
|
5712
6568
|
return false;
|
|
5713
6569
|
return true;
|
|
5714
6570
|
}
|
|
6571
|
+
function makePretrainedOptionsKey(model_id, options = {}, ...parts) {
|
|
6572
|
+
return JSON.stringify([
|
|
6573
|
+
model_id,
|
|
6574
|
+
options.revision ?? "main",
|
|
6575
|
+
options.cache_dir ?? null,
|
|
6576
|
+
options.local_files_only ?? false,
|
|
6577
|
+
...parts
|
|
6578
|
+
]);
|
|
6579
|
+
}
|
|
6580
|
+
var ModelFileNotFoundError = class extends Error {
|
|
6581
|
+
constructor(message, { status = null } = {}) {
|
|
6582
|
+
super(message);
|
|
6583
|
+
this.name = "ModelFileNotFoundError";
|
|
6584
|
+
this.status = status;
|
|
6585
|
+
}
|
|
6586
|
+
};
|
|
6587
|
+
var NOT_FOUND_STATUSES = /* @__PURE__ */ new Set([401, 403, 404]);
|
|
5715
6588
|
function handleError(status, remoteURL, fatal) {
|
|
5716
6589
|
if (!fatal) {
|
|
5717
6590
|
return null;
|
|
5718
6591
|
}
|
|
5719
6592
|
const message = ERROR_MAPPING[status] ?? `Error (${status}) occurred while trying to load file`;
|
|
5720
|
-
|
|
6593
|
+
const fullMessage = `${message}: "${remoteURL}".`;
|
|
6594
|
+
if (NOT_FOUND_STATUSES.has(status)) {
|
|
6595
|
+
throw new ModelFileNotFoundError(fullMessage, { status });
|
|
6596
|
+
}
|
|
6597
|
+
throw Error(fullMessage);
|
|
5721
6598
|
}
|
|
5722
6599
|
async function readResponse(response, progress_callback, expectedSize) {
|
|
5723
6600
|
const contentLength = response.headers.get("Content-Length");
|
|
@@ -5751,7 +6628,7 @@ async function readResponse(response, progress_callback, expectedSize) {
|
|
|
5751
6628
|
function isBlobURL(url) {
|
|
5752
6629
|
return isValidUrl(url, ["blob:"]);
|
|
5753
6630
|
}
|
|
5754
|
-
function toAbsoluteURL(url) {
|
|
6631
|
+
function toAbsoluteURL(url, { allowUnresolved = false } = {}) {
|
|
5755
6632
|
let baseURL;
|
|
5756
6633
|
if (typeof location !== "undefined" && location.href) {
|
|
5757
6634
|
baseURL = location.href;
|
|
@@ -5760,7 +6637,14 @@ function toAbsoluteURL(url) {
|
|
|
5760
6637
|
} else {
|
|
5761
6638
|
return url;
|
|
5762
6639
|
}
|
|
5763
|
-
|
|
6640
|
+
try {
|
|
6641
|
+
return new URL(url, baseURL).href;
|
|
6642
|
+
} catch (error) {
|
|
6643
|
+
if (!allowUnresolved) {
|
|
6644
|
+
throw error;
|
|
6645
|
+
}
|
|
6646
|
+
return url;
|
|
6647
|
+
}
|
|
5764
6648
|
}
|
|
5765
6649
|
var HASH_ALGORITHM = "SHA-256";
|
|
5766
6650
|
var HASH_CACHE_NAME = "experimental_transformers-hash-cache";
|
|
@@ -5778,7 +6662,7 @@ var CrossOriginStorage = class {
|
|
|
5778
6662
|
return;
|
|
5779
6663
|
}
|
|
5780
6664
|
try {
|
|
5781
|
-
const
|
|
6665
|
+
const handle = await navigator.crossOriginStorage.requestFileHandle(makeHashDescriptor(hashValue));
|
|
5782
6666
|
const blob = await handle.getFile();
|
|
5783
6667
|
return new Response(blob, {
|
|
5784
6668
|
headers: {
|
|
@@ -5799,8 +6683,9 @@ var CrossOriginStorage = class {
|
|
|
5799
6683
|
}
|
|
5800
6684
|
};
|
|
5801
6685
|
_storeBlobInCOS = async (blob, hashHex) => {
|
|
5802
|
-
const
|
|
5803
|
-
create: true
|
|
6686
|
+
const handle = await navigator.crossOriginStorage.requestFileHandle(makeHashDescriptor(hashHex), {
|
|
6687
|
+
create: true,
|
|
6688
|
+
origins: "*"
|
|
5804
6689
|
});
|
|
5805
6690
|
const writableStream = await handle.createWritable();
|
|
5806
6691
|
await writableStream.write(blob);
|
|
@@ -5964,13 +6849,7 @@ async function fetch_file_head(urlOrPath) {
|
|
|
5964
6849
|
return env3.fetch(urlOrPath, { method: "GET", headers, cache: "no-store" });
|
|
5965
6850
|
}
|
|
5966
6851
|
function get_file_metadata(path_or_repo_id, filename, options = {}) {
|
|
5967
|
-
const key =
|
|
5968
|
-
path_or_repo_id,
|
|
5969
|
-
filename,
|
|
5970
|
-
options?.revision,
|
|
5971
|
-
options?.cache_dir,
|
|
5972
|
-
options?.local_files_only
|
|
5973
|
-
]);
|
|
6852
|
+
const key = makePretrainedOptionsKey(path_or_repo_id, options, filename);
|
|
5974
6853
|
return memoizePromise(key, () => _get_file_metadata(path_or_repo_id, filename, options));
|
|
5975
6854
|
}
|
|
5976
6855
|
async function _get_file_metadata(path_or_repo_id, filename, options) {
|
|
@@ -6006,37 +6885,36 @@ async function _get_file_metadata(path_or_repo_id, filename, options) {
|
|
|
6006
6885
|
}
|
|
6007
6886
|
}
|
|
6008
6887
|
if (env3.allowRemoteModels && !options.local_files_only && validModelId) {
|
|
6009
|
-
|
|
6010
|
-
|
|
6011
|
-
|
|
6012
|
-
|
|
6013
|
-
|
|
6014
|
-
|
|
6015
|
-
|
|
6016
|
-
|
|
6017
|
-
|
|
6018
|
-
|
|
6019
|
-
size = parseInt(match[1], 10);
|
|
6020
|
-
}
|
|
6888
|
+
const rangeResponse = await fetch_file_head(remoteURL);
|
|
6889
|
+
if (rangeResponse && rangeResponse.status >= 200 && rangeResponse.status < 300) {
|
|
6890
|
+
let size;
|
|
6891
|
+
const contentType = rangeResponse.headers.get("content-type");
|
|
6892
|
+
if (rangeResponse.status === 206) {
|
|
6893
|
+
const contentRange = rangeResponse.headers.get("content-range");
|
|
6894
|
+
if (contentRange) {
|
|
6895
|
+
const match = contentRange.match(/bytes \d+-\d+\/(\d+)/);
|
|
6896
|
+
if (match) {
|
|
6897
|
+
size = parseInt(match[1], 10);
|
|
6021
6898
|
}
|
|
6022
|
-
} else if (rangeResponse.status === 200) {
|
|
6023
|
-
try {
|
|
6024
|
-
await rangeResponse.body?.cancel();
|
|
6025
|
-
} catch (cancelError) {}
|
|
6026
|
-
}
|
|
6027
|
-
if (size === undefined) {
|
|
6028
|
-
const contentLength = rangeResponse.headers.get("content-length");
|
|
6029
|
-
size = contentLength ? parseInt(contentLength, 10) : undefined;
|
|
6030
6899
|
}
|
|
6031
|
-
|
|
6032
|
-
|
|
6033
|
-
|
|
6034
|
-
|
|
6035
|
-
fromCache: false
|
|
6036
|
-
};
|
|
6900
|
+
} else if (rangeResponse.status === 200) {
|
|
6901
|
+
try {
|
|
6902
|
+
await rangeResponse.body?.cancel();
|
|
6903
|
+
} catch (cancelError) {}
|
|
6037
6904
|
}
|
|
6038
|
-
|
|
6039
|
-
|
|
6905
|
+
if (size === undefined) {
|
|
6906
|
+
const contentLength = rangeResponse.headers.get("content-length");
|
|
6907
|
+
size = contentLength ? parseInt(contentLength, 10) : undefined;
|
|
6908
|
+
}
|
|
6909
|
+
return {
|
|
6910
|
+
exists: true,
|
|
6911
|
+
size,
|
|
6912
|
+
contentType: contentType || undefined,
|
|
6913
|
+
fromCache: false
|
|
6914
|
+
};
|
|
6915
|
+
}
|
|
6916
|
+
if (rangeResponse && rangeResponse.status !== 404 && rangeResponse.status !== 416) {
|
|
6917
|
+
handleError(rangeResponse.status, remoteURL, true);
|
|
6040
6918
|
}
|
|
6041
6919
|
}
|
|
6042
6920
|
return { exists: false, fromCache: false };
|
|
@@ -6092,6 +6970,9 @@ async function storeCachedResource(path_or_repo_id, filename, cache2, cacheKey,
|
|
|
6092
6970
|
if (await cache2.match(cacheKey) !== undefined) {
|
|
6093
6971
|
return;
|
|
6094
6972
|
}
|
|
6973
|
+
if (typeof Cache !== "undefined" && cache2 instanceof Cache && !isValidUrl(toAbsoluteURL(cacheKey, { allowUnresolved: true }), ["http:", "https:"])) {
|
|
6974
|
+
return;
|
|
6975
|
+
}
|
|
6095
6976
|
if (!result) {
|
|
6096
6977
|
const wrapped_progress = options.progress_callback ? (data) => dispatchCallback(options.progress_callback, {
|
|
6097
6978
|
status: "progress",
|
|
@@ -6138,13 +7019,13 @@ async function loadResourceFile(path_or_repo_id, filename, fatal = true, options
|
|
|
6138
7019
|
if (response === undefined || typeof response !== "string" && response.status === 404) {
|
|
6139
7020
|
if (options.local_files_only || !env3.allowRemoteModels) {
|
|
6140
7021
|
if (fatal) {
|
|
6141
|
-
throw
|
|
7022
|
+
throw new ModelFileNotFoundError(`\`local_files_only=true\` or \`env.allowRemoteModels=false\` and file was not found locally at "${localPath}".`);
|
|
6142
7023
|
} else {
|
|
6143
7024
|
return null;
|
|
6144
7025
|
}
|
|
6145
7026
|
}
|
|
6146
7027
|
if (!validModelId) {
|
|
6147
|
-
throw
|
|
7028
|
+
throw new ModelFileNotFoundError(`Local file missing at "${localPath}" and download aborted due to invalid model ID "${path_or_repo_id}".`);
|
|
6148
7029
|
}
|
|
6149
7030
|
response = await getFile(remoteURL);
|
|
6150
7031
|
if (response.status !== 200) {
|
|
@@ -6247,23 +7128,21 @@ async function getModelFile(path_or_repo_id, filename, fatal = true, options = {
|
|
|
6247
7128
|
throw Error("Invalid configuration detected: both local and remote models are disabled. Fix by setting `env.allowLocalModels` or `env.allowRemoteModels` to `true`.");
|
|
6248
7129
|
}
|
|
6249
7130
|
}
|
|
6250
|
-
|
|
6251
|
-
|
|
6252
|
-
|
|
6253
|
-
|
|
6254
|
-
});
|
|
6255
|
-
const key = `${path_or_repo_id}::${filename}`;
|
|
6256
|
-
let pending = INFLIGHT_LOADS.get(key);
|
|
7131
|
+
const key = makePretrainedOptionsKey(path_or_repo_id, options, filename, fatal, return_path);
|
|
7132
|
+
const { progress_callback } = options;
|
|
7133
|
+
const loads = progress_callback instanceof DefaultProgressCallback ? progress_callback.loads : INFLIGHT_LOADS;
|
|
7134
|
+
let pending = loads.get(key);
|
|
6257
7135
|
if (!pending) {
|
|
6258
|
-
|
|
6259
|
-
|
|
6260
|
-
|
|
6261
|
-
|
|
6262
|
-
}, (err) => {
|
|
6263
|
-
INFLIGHT_LOADS.delete(key);
|
|
6264
|
-
throw err;
|
|
7136
|
+
dispatchCallback(progress_callback, {
|
|
7137
|
+
status: "initiate",
|
|
7138
|
+
name: path_or_repo_id,
|
|
7139
|
+
file: filename
|
|
6265
7140
|
});
|
|
6266
|
-
|
|
7141
|
+
pending = getCache(options.cache_dir).then((cache2) => loadResourceFile(path_or_repo_id, filename, fatal, options, return_path, cache2));
|
|
7142
|
+
if (loads === INFLIGHT_LOADS) {
|
|
7143
|
+
pending = pending.finally(() => INFLIGHT_LOADS.delete(key));
|
|
7144
|
+
}
|
|
7145
|
+
loads.set(key, pending);
|
|
6267
7146
|
}
|
|
6268
7147
|
return await pending;
|
|
6269
7148
|
}
|
|
@@ -6915,11 +7794,7 @@ var uint16_to_float32 = /* @__PURE__ */ function() {
|
|
|
6915
7794
|
return out;
|
|
6916
7795
|
};
|
|
6917
7796
|
}();
|
|
6918
|
-
var
|
|
6919
|
-
__export(onnxruntime_node_exports, {
|
|
6920
|
-
default: () => onnxruntime_node_default
|
|
6921
|
-
});
|
|
6922
|
-
var onnxruntime_node_default = {};
|
|
7797
|
+
var onnx_node_default = {};
|
|
6923
7798
|
async function loadAndCacheFile(url) {
|
|
6924
7799
|
const fileName = url.split("/").pop();
|
|
6925
7800
|
let cache2;
|
|
@@ -7016,7 +7891,7 @@ var ORT_SYMBOL = /* @__PURE__ */ Symbol.for("onnxruntime");
|
|
|
7016
7891
|
if (ORT_SYMBOL in globalThis) {
|
|
7017
7892
|
ONNX = globalThis[ORT_SYMBOL];
|
|
7018
7893
|
} else if (apis.IS_NODE_ENV) {
|
|
7019
|
-
ONNX =
|
|
7894
|
+
ONNX = onnx_node_default;
|
|
7020
7895
|
switch (process.platform) {
|
|
7021
7896
|
case "win32":
|
|
7022
7897
|
supportedDevices.push("dml");
|
|
@@ -7136,12 +8011,13 @@ if (ONNX_ENV) {
|
|
|
7136
8011
|
if (ONNX_ENV.wasm) {
|
|
7137
8012
|
if (!(typeof ServiceWorkerGlobalScope !== "undefined" && self instanceof ServiceWorkerGlobalScope) && ONNX_ENV.versions?.web && !ONNX_ENV.wasm.wasmPaths) {
|
|
7138
8013
|
const wasmPathPrefix = `https://cdn.jsdelivr.net/npm/onnxruntime-web@${ONNX_ENV.versions.web}/dist/`;
|
|
7139
|
-
|
|
7140
|
-
|
|
7141
|
-
|
|
7142
|
-
}
|
|
7143
|
-
|
|
7144
|
-
|
|
8014
|
+
let wasmPathSuffix = ".asyncify";
|
|
8015
|
+
if (apis.IS_SAFARI_BELOW_26 && !apis.IS_WEBGPU_AVAILABLE) {
|
|
8016
|
+
wasmPathSuffix = "";
|
|
8017
|
+
}
|
|
8018
|
+
ONNX_ENV.wasm.wasmPaths = {
|
|
8019
|
+
mjs: `${wasmPathPrefix}ort-wasm-simd-threaded${wasmPathSuffix}.mjs`,
|
|
8020
|
+
wasm: `${wasmPathPrefix}ort-wasm-simd-threaded${wasmPathSuffix}.wasm`
|
|
7145
8021
|
};
|
|
7146
8022
|
}
|
|
7147
8023
|
ONNX_ENV.wasm.proxy = false;
|
|
@@ -7850,10 +8726,8 @@ var TensorOpRegistry = class {
|
|
|
7850
8726
|
this._rfft = wrap([
|
|
7851
8727
|
8,
|
|
7852
8728
|
9,
|
|
7853
|
-
18,
|
|
7854
|
-
0,
|
|
7855
8729
|
58,
|
|
7856
|
-
|
|
8730
|
+
93,
|
|
7857
8731
|
10,
|
|
7858
8732
|
33,
|
|
7859
8733
|
10,
|
|
@@ -7893,28 +8767,26 @@ var TensorOpRegistry = class {
|
|
|
7893
8767
|
1,
|
|
7894
8768
|
100,
|
|
7895
8769
|
90,
|
|
7896
|
-
|
|
8770
|
+
19,
|
|
7897
8771
|
10,
|
|
7898
8772
|
1,
|
|
7899
8773
|
120,
|
|
7900
8774
|
18,
|
|
7901
|
-
16,
|
|
7902
|
-
10,
|
|
7903
8775
|
14,
|
|
8776
|
+
10,
|
|
8777
|
+
12,
|
|
7904
8778
|
8,
|
|
7905
8779
|
1,
|
|
7906
8780
|
18,
|
|
8781
|
+
8,
|
|
7907
8782
|
10,
|
|
8783
|
+
0,
|
|
7908
8784
|
10,
|
|
7909
|
-
|
|
7910
|
-
18,
|
|
7911
|
-
1,
|
|
7912
|
-
115,
|
|
8785
|
+
0,
|
|
7913
8786
|
10,
|
|
7914
|
-
|
|
7915
|
-
|
|
8787
|
+
2,
|
|
8788
|
+
8,
|
|
7916
8789
|
1,
|
|
7917
|
-
99,
|
|
7918
8790
|
90,
|
|
7919
8791
|
11,
|
|
7920
8792
|
10,
|
|
@@ -7929,28 +8801,26 @@ var TensorOpRegistry = class {
|
|
|
7929
8801
|
18,
|
|
7930
8802
|
0,
|
|
7931
8803
|
98,
|
|
7932
|
-
|
|
8804
|
+
19,
|
|
7933
8805
|
10,
|
|
7934
8806
|
1,
|
|
7935
8807
|
121,
|
|
7936
8808
|
18,
|
|
7937
|
-
16,
|
|
7938
|
-
10,
|
|
7939
8809
|
14,
|
|
8810
|
+
10,
|
|
8811
|
+
12,
|
|
7940
8812
|
8,
|
|
7941
8813
|
1,
|
|
7942
8814
|
18,
|
|
8815
|
+
8,
|
|
7943
8816
|
10,
|
|
8817
|
+
0,
|
|
7944
8818
|
10,
|
|
7945
|
-
|
|
7946
|
-
18,
|
|
7947
|
-
1,
|
|
7948
|
-
115,
|
|
8819
|
+
0,
|
|
7949
8820
|
10,
|
|
7950
|
-
|
|
7951
|
-
|
|
7952
|
-
|
|
7953
|
-
99,
|
|
8821
|
+
2,
|
|
8822
|
+
8,
|
|
8823
|
+
2,
|
|
7954
8824
|
66,
|
|
7955
8825
|
2,
|
|
7956
8826
|
16,
|
|
@@ -8358,15 +9228,6 @@ var Tensor22 = class _Tensor {
|
|
|
8358
9228
|
return new _Tensor(this.type, [this.data[index]], iterDims);
|
|
8359
9229
|
}
|
|
8360
9230
|
}
|
|
8361
|
-
indexOf(item) {
|
|
8362
|
-
const this_data = this.data;
|
|
8363
|
-
for (let index = 0;index < this_data.length; ++index) {
|
|
8364
|
-
if (this_data[index] == item) {
|
|
8365
|
-
return index;
|
|
8366
|
-
}
|
|
8367
|
-
}
|
|
8368
|
-
return -1;
|
|
8369
|
-
}
|
|
8370
9231
|
_subarray(index, iterSize, iterDims) {
|
|
8371
9232
|
const o1 = index * iterSize;
|
|
8372
9233
|
const o2 = (index + 1) * iterSize;
|
|
@@ -8443,6 +9304,26 @@ var Tensor22 = class _Tensor {
|
|
|
8443
9304
|
}
|
|
8444
9305
|
return this;
|
|
8445
9306
|
}
|
|
9307
|
+
remainder(val) {
|
|
9308
|
+
return this.clone().remainder_(val);
|
|
9309
|
+
}
|
|
9310
|
+
remainder_(val) {
|
|
9311
|
+
const this_data = this.data;
|
|
9312
|
+
const is_bigint = this_data instanceof BigInt64Array || this_data instanceof BigUint64Array;
|
|
9313
|
+
const divisor = is_bigint ? BigInt(val) : Number(val);
|
|
9314
|
+
if ((divisor === 0 || divisor === 0n) && (this.type.includes("int") || this.type === "bool")) {
|
|
9315
|
+
throw new RangeError("Division by zero");
|
|
9316
|
+
}
|
|
9317
|
+
if (divisor < 0 && (this.type.startsWith("uint") || this.type === "bool")) {
|
|
9318
|
+
throw new RangeError("Negative divisors are not supported for unsigned or boolean tensors");
|
|
9319
|
+
}
|
|
9320
|
+
for (let i = 0;i < this_data.length; ++i) {
|
|
9321
|
+
const remainder = this_data[i] % divisor;
|
|
9322
|
+
const needs_shift = remainder < 0 && divisor > 0 || remainder > 0 && divisor < 0;
|
|
9323
|
+
this_data[i] = needs_shift ? remainder + divisor : remainder;
|
|
9324
|
+
}
|
|
9325
|
+
return this;
|
|
9326
|
+
}
|
|
8446
9327
|
clone() {
|
|
8447
9328
|
return new _Tensor(this.type, this.data.slice(), this.dims.slice());
|
|
8448
9329
|
}
|
|
@@ -8766,7 +9647,7 @@ var Tensor22 = class _Tensor {
|
|
|
8766
9647
|
};
|
|
8767
9648
|
function reshape(data, dimensions) {
|
|
8768
9649
|
const totalElements = data.length;
|
|
8769
|
-
const dimensionSize = dimensions.reduce((a, b) => a * b);
|
|
9650
|
+
const dimensionSize = dimensions.length === 0 ? 1 : dimensions.reduce((a, b) => a * b);
|
|
8770
9651
|
if (totalElements !== dimensionSize) {
|
|
8771
9652
|
throw Error(`cannot reshape array of size ${totalElements} into shape (${dimensions})`);
|
|
8772
9653
|
}
|
|
@@ -8830,8 +9711,9 @@ async function matmul(a, b) {
|
|
|
8830
9711
|
return await op({ a, b });
|
|
8831
9712
|
}
|
|
8832
9713
|
async function rfft(x, a) {
|
|
9714
|
+
const axis = safeIndex(Number(a.item()), x.dims.length);
|
|
8833
9715
|
const op = await TensorOpRegistry.rfft;
|
|
8834
|
-
return await op({ x, a });
|
|
9716
|
+
return await op({ x: x.unsqueeze(-1), a: new Tensor22("int64", [BigInt(axis)], []) });
|
|
8835
9717
|
}
|
|
8836
9718
|
async function topk(x, k) {
|
|
8837
9719
|
const op = await TensorOpRegistry.top_k;
|
|
@@ -9495,10 +10377,13 @@ function _build_translation_inputs(self2, raw_inputs, tokenizer_options, generat
|
|
|
9495
10377
|
if (!self2.language_codes.includes(src_lang_token)) {
|
|
9496
10378
|
throw new Error(`Source language code "${src_lang_token}" is not valid. Must be one of: {${self2.language_codes.join(", ")}}`);
|
|
9497
10379
|
}
|
|
9498
|
-
|
|
9499
|
-
|
|
9500
|
-
|
|
9501
|
-
|
|
10380
|
+
const post_processor_config = self2._tokenizer.post_processor?.config;
|
|
10381
|
+
if (post_processor_config && "single" in post_processor_config) {
|
|
10382
|
+
for (const item of post_processor_config.single) {
|
|
10383
|
+
if ("SpecialToken" in item && self2.languageRegex.test(item.SpecialToken.id)) {
|
|
10384
|
+
item.SpecialToken.id = self2.lang_to_token(src_lang_token);
|
|
10385
|
+
break;
|
|
10386
|
+
}
|
|
9502
10387
|
}
|
|
9503
10388
|
}
|
|
9504
10389
|
}
|
|
@@ -10263,7 +11148,7 @@ var AutoTokenizer = class {
|
|
|
10263
11148
|
};
|
|
10264
11149
|
var GITHUB_ISSUE_URL = "https://github.com/huggingface/transformers.js/issues/new/choose";
|
|
10265
11150
|
var FEATURE_EXTRACTOR_NAME = "preprocessor_config.json";
|
|
10266
|
-
var IMAGE_PROCESSOR_NAME =
|
|
11151
|
+
var IMAGE_PROCESSOR_NAME = "preprocessor_config.json";
|
|
10267
11152
|
var PROCESSOR_NAME = "processor_config.json";
|
|
10268
11153
|
var CHAT_TEMPLATE_NAME = "chat_template.jinja";
|
|
10269
11154
|
var Processor = class extends Callable {
|
|
@@ -10378,7 +11263,7 @@ var FeatureExtractor = class extends Callable {
|
|
|
10378
11263
|
};
|
|
10379
11264
|
function validate_audio_inputs(audio, feature_extractor) {
|
|
10380
11265
|
if (!(audio instanceof Float32Array || audio instanceof Float64Array)) {
|
|
10381
|
-
throw new Error(`${feature_extractor} expects input to be a Float32Array or a Float64Array, but got ${audio?.constructor?.name ?? typeof audio} instead. If using the feature extractor directly, remember to use \`
|
|
11266
|
+
throw new Error(`${feature_extractor} expects input to be a Float32Array or a Float64Array, but got ${audio?.constructor?.name ?? typeof audio} instead. If using the feature extractor directly, remember to use \`load_audio(url, sampling_rate)\` to obtain the raw audio data of the file/url.`);
|
|
10382
11267
|
}
|
|
10383
11268
|
}
|
|
10384
11269
|
var feature_extractors_exports = {};
|
|
@@ -10453,7 +11338,9 @@ async function load_audio(url, sampling_rate) {
|
|
|
10453
11338
|
}
|
|
10454
11339
|
return audio;
|
|
10455
11340
|
}
|
|
10456
|
-
|
|
11341
|
+
async function read_audio(url, sampling_rate) {
|
|
11342
|
+
return await load_audio(url, sampling_rate);
|
|
11343
|
+
}
|
|
10457
11344
|
function generalized_cosine_window(M, a_0) {
|
|
10458
11345
|
if (M < 1) {
|
|
10459
11346
|
return new Float64Array;
|
|
@@ -10833,9 +11720,10 @@ function encodeWAV(chunks, rate) {
|
|
|
10833
11720
|
view.setUint16(34, 32, true);
|
|
10834
11721
|
writeString(view, 36, "data");
|
|
10835
11722
|
view.setUint32(40, totalLength * 4, true);
|
|
10836
|
-
return new Blob([
|
|
10837
|
-
|
|
10838
|
-
|
|
11723
|
+
return new Blob([
|
|
11724
|
+
buffer,
|
|
11725
|
+
...chunks.map((chunk2) => chunk2.buffer instanceof ArrayBuffer ? new Uint8Array(chunk2.buffer, chunk2.byteOffset, chunk2.byteLength) : chunk2.slice())
|
|
11726
|
+
], { type: "audio/wav" });
|
|
10839
11727
|
}
|
|
10840
11728
|
function writeString(view, offset, string) {
|
|
10841
11729
|
for (let i = 0;i < string.length; ++i) {
|
|
@@ -11258,7 +12146,7 @@ var GraniteSpeechFeatureExtractor = class extends FeatureExtractor {
|
|
|
11258
12146
|
async _call(audio) {
|
|
11259
12147
|
validate_audio_inputs(audio, "GraniteSpeechFeatureExtractor");
|
|
11260
12148
|
const { n_fft, hop_length, n_mels } = this.config.melspec_kwargs;
|
|
11261
|
-
const num_frames =
|
|
12149
|
+
const num_frames = Math.floor(audio.length / hop_length) + 1;
|
|
11262
12150
|
const max_num_frames = num_frames - num_frames % 2;
|
|
11263
12151
|
const mel = await spectrogram(audio, this.window, n_fft, hop_length, {
|
|
11264
12152
|
power: 2,
|
|
@@ -11710,7 +12598,7 @@ var RawImage = class _RawImage {
|
|
|
11710
12598
|
throw new Error(`Tensor should have 3 dimensions, but has ${tensor.dims.length} dimensions.`);
|
|
11711
12599
|
}
|
|
11712
12600
|
if (channel_format === "CHW") {
|
|
11713
|
-
tensor = tensor.
|
|
12601
|
+
tensor = tensor.permute(1, 2, 0);
|
|
11714
12602
|
} else if (channel_format === "HWC") {} else {
|
|
11715
12603
|
throw new Error(`Unsupported channel format: ${channel_format}`);
|
|
11716
12604
|
}
|
|
@@ -14942,11 +15830,26 @@ function getNormalizedConfig(config) {
|
|
|
14942
15830
|
break;
|
|
14943
15831
|
case "youtu":
|
|
14944
15832
|
case "deepseek_v3":
|
|
15833
|
+
case "deepseek_v4":
|
|
14945
15834
|
case "glm_moe_dsa":
|
|
14946
15835
|
case "mistral4":
|
|
14947
15836
|
mapping["num_heads"] = "num_key_value_heads";
|
|
14948
15837
|
mapping["num_layers"] = "num_hidden_layers";
|
|
14949
|
-
mapping["dim_kv"] = "qk_head_dim";
|
|
15838
|
+
mapping["dim_kv"] = config.model_type === "deepseek_v4" ? "head_dim" : "qk_head_dim";
|
|
15839
|
+
mapping["num_attention_heads"] = "num_attention_heads";
|
|
15840
|
+
break;
|
|
15841
|
+
case "zaya":
|
|
15842
|
+
mapping["num_heads"] = "num_key_value_heads";
|
|
15843
|
+
mapping["num_layers"] = "num_hidden_layers";
|
|
15844
|
+
mapping["hidden_size"] = "hidden_size";
|
|
15845
|
+
mapping["dim_kv"] = "head_dim";
|
|
15846
|
+
mapping["num_attention_heads"] = "num_attention_heads";
|
|
15847
|
+
break;
|
|
15848
|
+
case "hrm_text":
|
|
15849
|
+
mapping["num_heads"] = "num_key_value_heads";
|
|
15850
|
+
mapping["num_layers"] = "num_hidden_layers";
|
|
15851
|
+
mapping["hidden_size"] = "hidden_size";
|
|
15852
|
+
mapping["dim_kv"] = "head_dim";
|
|
14950
15853
|
mapping["num_attention_heads"] = "num_attention_heads";
|
|
14951
15854
|
break;
|
|
14952
15855
|
case "t5":
|
|
@@ -15049,7 +15952,7 @@ function getCacheNames(config, options) {
|
|
|
15049
15952
|
config = new PretrainedConfig(config);
|
|
15050
15953
|
}
|
|
15051
15954
|
const pkv_prefix = options?.prefix ?? "past_key_values";
|
|
15052
|
-
const
|
|
15955
|
+
const cache_prefix = pkv_prefix === "present" ? "present" : "past";
|
|
15053
15956
|
const names = /* @__PURE__ */ new Set;
|
|
15054
15957
|
if (["lfm2", "lfm2_moe"].includes(config.model_type)) {
|
|
15055
15958
|
const { layer_types } = config;
|
|
@@ -15058,7 +15961,7 @@ function getCacheNames(config, options) {
|
|
|
15058
15961
|
names.add(`${pkv_prefix}.${i}.key`);
|
|
15059
15962
|
names.add(`${pkv_prefix}.${i}.value`);
|
|
15060
15963
|
} else if (layer_types[i] === "conv") {
|
|
15061
|
-
names.add(`${
|
|
15964
|
+
names.add(`${cache_prefix}_conv.${i}`);
|
|
15062
15965
|
} else {
|
|
15063
15966
|
throw new Error(`Unsupported layer type: ${layer_types[i]}`);
|
|
15064
15967
|
}
|
|
@@ -15070,8 +15973,8 @@ function getCacheNames(config, options) {
|
|
|
15070
15973
|
const num_layers = c.num_hidden_layers ?? layer_types?.length;
|
|
15071
15974
|
for (let i = 0;i < num_layers; ++i) {
|
|
15072
15975
|
if (!layer_types || layer_types[i] === "mamba") {
|
|
15073
|
-
names.add(`${
|
|
15074
|
-
names.add(`${
|
|
15976
|
+
names.add(`${cache_prefix}_conv.${i}`);
|
|
15977
|
+
names.add(`${cache_prefix}_ssm.${i}`);
|
|
15075
15978
|
}
|
|
15076
15979
|
if (!layer_types || layer_types[i] === "attention") {
|
|
15077
15980
|
names.add(`${pkv_prefix}.${i}.key`);
|
|
@@ -15087,13 +15990,13 @@ function getCacheNames(config, options) {
|
|
|
15087
15990
|
names.add(`${pkv_prefix}.${i}.value`);
|
|
15088
15991
|
} else if (layer_types[i] === "linear_attention") {
|
|
15089
15992
|
if (config.model_type === "olmo_hybrid") {
|
|
15090
|
-
names.add(`${
|
|
15091
|
-
names.add(`${
|
|
15092
|
-
names.add(`${
|
|
15993
|
+
names.add(`${cache_prefix}_conv.${i}.key`);
|
|
15994
|
+
names.add(`${cache_prefix}_conv.${i}.value`);
|
|
15995
|
+
names.add(`${cache_prefix}_conv.${i}.query`);
|
|
15093
15996
|
} else {
|
|
15094
|
-
names.add(`${
|
|
15997
|
+
names.add(`${cache_prefix}_conv.${i}`);
|
|
15095
15998
|
}
|
|
15096
|
-
names.add(`${
|
|
15999
|
+
names.add(`${cache_prefix}_recurrent.${i}`);
|
|
15097
16000
|
} else {
|
|
15098
16001
|
throw new Error(`Unsupported layer type: ${layer_types[i]}`);
|
|
15099
16002
|
}
|
|
@@ -15109,6 +16012,35 @@ function getCacheNames(config, options) {
|
|
|
15109
16012
|
names.add(`${pkv_prefix}.${i}.value`);
|
|
15110
16013
|
}
|
|
15111
16014
|
return names;
|
|
16015
|
+
} else if (config.model_type === "deepseek_v4") {
|
|
16016
|
+
const { layer_types, num_hidden_layers } = config;
|
|
16017
|
+
for (let i = 0;i < num_hidden_layers; ++i) {
|
|
16018
|
+
names.add(`${pkv_prefix}.${i}.key`);
|
|
16019
|
+
names.add(`${pkv_prefix}.${i}.value`);
|
|
16020
|
+
const layer_type = layer_types[i];
|
|
16021
|
+
if (layer_type === "compressed_sparse_attention") {
|
|
16022
|
+
names.add(`${cache_prefix}_compressor.${i}.kv`);
|
|
16023
|
+
names.add(`${cache_prefix}_compressor.${i}.gate`);
|
|
16024
|
+
names.add(`${cache_prefix}_indexer.${i}.kv`);
|
|
16025
|
+
names.add(`${cache_prefix}_indexer.${i}.gate`);
|
|
16026
|
+
} else if (layer_type === "heavily_compressed_attention") {
|
|
16027
|
+
names.add(`${cache_prefix}_compressor.${i}.kv`);
|
|
16028
|
+
names.add(`${cache_prefix}_compressor.${i}.gate`);
|
|
16029
|
+
} else if (layer_type && layer_type !== "sliding_attention") {
|
|
16030
|
+
throw new Error(`Unsupported layer type: ${layer_type}`);
|
|
16031
|
+
}
|
|
16032
|
+
}
|
|
16033
|
+
return names;
|
|
16034
|
+
} else if (config.model_type === "zaya") {
|
|
16035
|
+
const { num_hidden_layers, cca_time1 } = config;
|
|
16036
|
+
const stride = cca_time1 ?? 1;
|
|
16037
|
+
for (let i = 0;i < num_hidden_layers; i += stride) {
|
|
16038
|
+
names.add(`${pkv_prefix}.${i}.key`);
|
|
16039
|
+
names.add(`${pkv_prefix}.${i}.value`);
|
|
16040
|
+
names.add(`${pkv_prefix}.${i}.conv_state`);
|
|
16041
|
+
names.add(`${pkv_prefix}.${i}.shift_state`);
|
|
16042
|
+
}
|
|
16043
|
+
return names;
|
|
15112
16044
|
} else if (["lfm2_vl", "qwen3_5", "qwen3_5_moe", "voxtral_realtime"].includes(config.model_type)) {
|
|
15113
16045
|
let subConfig;
|
|
15114
16046
|
if (config.model_type === "voxtral_realtime" && options?.session_name === "audio_encoder") {
|
|
@@ -15352,6 +16284,14 @@ function validateInputs(session, inputs) {
|
|
|
15352
16284
|
}
|
|
15353
16285
|
var ModelOutput = class {
|
|
15354
16286
|
};
|
|
16287
|
+
var BaseModelOutput = class extends ModelOutput {
|
|
16288
|
+
constructor({ last_hidden_state, hidden_states = null, attentions = null }) {
|
|
16289
|
+
super();
|
|
16290
|
+
this.last_hidden_state = last_hidden_state;
|
|
16291
|
+
this.hidden_states = hidden_states;
|
|
16292
|
+
this.attentions = attentions;
|
|
16293
|
+
}
|
|
16294
|
+
};
|
|
15355
16295
|
var SequenceClassifierOutput = class extends ModelOutput {
|
|
15356
16296
|
constructor({ logits, ...attentions }) {
|
|
15357
16297
|
super();
|
|
@@ -15387,6 +16327,23 @@ var CausalLMOutput = class extends ModelOutput {
|
|
|
15387
16327
|
this.logits = logits;
|
|
15388
16328
|
}
|
|
15389
16329
|
};
|
|
16330
|
+
var CausalLMOutputWithPast = class extends ModelOutput {
|
|
16331
|
+
constructor({ logits, past_key_values }) {
|
|
16332
|
+
super();
|
|
16333
|
+
this.logits = logits;
|
|
16334
|
+
this.past_key_values = past_key_values;
|
|
16335
|
+
}
|
|
16336
|
+
};
|
|
16337
|
+
var Seq2SeqLMOutput = class extends ModelOutput {
|
|
16338
|
+
constructor({ logits, past_key_values, encoder_outputs, decoder_attentions = null, cross_attentions = null }) {
|
|
16339
|
+
super();
|
|
16340
|
+
this.logits = logits;
|
|
16341
|
+
this.past_key_values = past_key_values;
|
|
16342
|
+
this.encoder_outputs = encoder_outputs;
|
|
16343
|
+
this.decoder_attentions = decoder_attentions;
|
|
16344
|
+
this.cross_attentions = cross_attentions;
|
|
16345
|
+
}
|
|
16346
|
+
};
|
|
15390
16347
|
var ImageMattingOutput = class extends ModelOutput {
|
|
15391
16348
|
constructor({ alphas }) {
|
|
15392
16349
|
super();
|
|
@@ -15678,7 +16635,7 @@ var ClassifierFreeGuidanceLogitsProcessor = class extends LogitsProcessor {
|
|
|
15678
16635
|
constructor(guidance_scale) {
|
|
15679
16636
|
super();
|
|
15680
16637
|
if (guidance_scale <= 1) {
|
|
15681
|
-
throw new Error(`Require guidance scale >1 to use the classifier
|
|
16638
|
+
throw new Error(`Require guidance scale >1 to use the classifier-free guidance processor, got guidance scale ${guidance_scale}.`);
|
|
15682
16639
|
}
|
|
15683
16640
|
this.guidance_scale = guidance_scale;
|
|
15684
16641
|
}
|
|
@@ -16077,6 +17034,7 @@ var MODEL_SESSION_CONFIG = {
|
|
|
16077
17034
|
}
|
|
16078
17035
|
return s;
|
|
16079
17036
|
},
|
|
17037
|
+
cache_sessions: { decoder_model_merged: true },
|
|
16080
17038
|
optional_configs: { generation_config: "generation_config.json" }
|
|
16081
17039
|
},
|
|
16082
17040
|
[MODEL_TYPES.Phi3V]: {
|
|
@@ -16179,7 +17137,7 @@ function get_config(modelId, { config = null, cache_dir = null, local_files_only
|
|
|
16179
17137
|
if (config !== null) {
|
|
16180
17138
|
return AutoConfig.from_pretrained(modelId, { config, cache_dir, local_files_only, revision });
|
|
16181
17139
|
}
|
|
16182
|
-
const key =
|
|
17140
|
+
const key = makePretrainedOptionsKey(modelId, { cache_dir, local_files_only, revision });
|
|
16183
17141
|
return memoizePromise(key, () => AutoConfig.from_pretrained(modelId, { config, cache_dir, local_files_only, revision }));
|
|
16184
17142
|
}
|
|
16185
17143
|
async function get_model_files(modelId, { config = null, dtype: overrideDtype = null, device: overrideDevice = null, model_file_name = null } = {}) {
|
|
@@ -16616,7 +17574,7 @@ var PreTrainedModel = class extends Callable {
|
|
|
16616
17574
|
}, () => [decoder_start_token_id]);
|
|
16617
17575
|
} else if (Array.isArray(decoder_start_token_id)) {
|
|
16618
17576
|
if (decoder_start_token_id.length !== batch_size) {
|
|
16619
|
-
throw new Error(`\`decoder_start_token_id\`
|
|
17577
|
+
throw new Error(`\`decoder_start_token_id\` expected to have length ${batch_size} but got ${decoder_start_token_id.length}`);
|
|
16620
17578
|
}
|
|
16621
17579
|
decoder_input_ids = decoder_start_token_id;
|
|
16622
17580
|
} else {
|
|
@@ -16822,7 +17780,7 @@ function getPastKeyValues(decoderResults, pastKeyValues) {
|
|
|
16822
17780
|
const pkvs = /* @__PURE__ */ Object.create(null);
|
|
16823
17781
|
for (const name in decoderResults) {
|
|
16824
17782
|
if (name.startsWith("present")) {
|
|
16825
|
-
const newName = name.replace("present_ssm", "past_ssm").replace("present_conv", "past_conv").replace("present_recurrent", "past_recurrent").replace("present", "past_key_values");
|
|
17783
|
+
const newName = name.replace("present_ssm", "past_ssm").replace("present_conv", "past_conv").replace("present_recurrent", "past_recurrent").replace("present_compressor", "past_compressor").replace("present_indexer", "past_indexer").replace("present", "past_key_values");
|
|
16826
17784
|
const is_encoder_pkv = name.includes("encoder");
|
|
16827
17785
|
if (is_encoder_pkv && pastKeyValues) {
|
|
16828
17786
|
pkvs[newName] = pastKeyValues[newName];
|
|
@@ -16888,6 +17846,14 @@ function addPastKeyValues(self2, decoderFeeds, pastKeyValues) {
|
|
|
16888
17846
|
}
|
|
16889
17847
|
return new DynamicCache(entries);
|
|
16890
17848
|
}
|
|
17849
|
+
function setNumLogitsToKeep(self2, model_inputs, value) {
|
|
17850
|
+
if (model_inputs.num_logits_to_keep)
|
|
17851
|
+
return;
|
|
17852
|
+
const session = self2.sessions["decoder_model_merged"] ?? self2.sessions["model"];
|
|
17853
|
+
if (session?.inputNames.includes("num_logits_to_keep")) {
|
|
17854
|
+
model_inputs.num_logits_to_keep = new Tensor22("int64", [value], []);
|
|
17855
|
+
}
|
|
17856
|
+
}
|
|
16891
17857
|
async function decoder_forward(self2, model_inputs, is_encoder_decoder = false) {
|
|
16892
17858
|
const session = self2.sessions[is_encoder_decoder ? "decoder_model_merged" : "model"];
|
|
16893
17859
|
const { past_key_values, ...new_model_inputs } = model_inputs;
|
|
@@ -16898,9 +17864,7 @@ async function decoder_forward(self2, model_inputs, is_encoder_decoder = false)
|
|
|
16898
17864
|
const start_index = ["paligemma", "gemma3_text", "gemma3"].includes(self2.config.model_type) ? 1 : 0;
|
|
16899
17865
|
new_model_inputs.position_ids = create_position_ids(new_model_inputs, past_key_values, start_index);
|
|
16900
17866
|
}
|
|
16901
|
-
|
|
16902
|
-
new_model_inputs.num_logits_to_keep = new Tensor22("int64", [0n], []);
|
|
16903
|
-
}
|
|
17867
|
+
setNumLogitsToKeep(self2, new_model_inputs, 0n);
|
|
16904
17868
|
addPastKeyValues(self2, new_model_inputs, past_key_values);
|
|
16905
17869
|
const fixed = pick(new_model_inputs, session.inputNames);
|
|
16906
17870
|
return await sessionRun(session, fixed);
|
|
@@ -16917,6 +17881,7 @@ async function generic_text_to_text_forward(self2, {
|
|
|
16917
17881
|
past_key_values = null,
|
|
16918
17882
|
generation_config = null,
|
|
16919
17883
|
logits_processor = null,
|
|
17884
|
+
num_logits_to_keep = null,
|
|
16920
17885
|
...kwargs
|
|
16921
17886
|
}) {
|
|
16922
17887
|
if (!inputs_embeds) {
|
|
@@ -16971,7 +17936,8 @@ async function generic_text_to_text_forward(self2, {
|
|
|
16971
17936
|
attention_mask,
|
|
16972
17937
|
position_ids,
|
|
16973
17938
|
generation_config,
|
|
16974
|
-
logits_processor
|
|
17939
|
+
logits_processor,
|
|
17940
|
+
num_logits_to_keep
|
|
16975
17941
|
}, true);
|
|
16976
17942
|
return outputs;
|
|
16977
17943
|
}
|
|
@@ -17024,10 +17990,7 @@ function create_position_ids(model_inputs, past_key_values = null, start_index =
|
|
|
17024
17990
|
}
|
|
17025
17991
|
function decoder_prepare_inputs_for_generation(self2, input_ids, model_inputs, generation_config) {
|
|
17026
17992
|
const past_length = model_inputs.past_key_values ? model_inputs.past_key_values.get_seq_length() : 0;
|
|
17027
|
-
|
|
17028
|
-
if (session?.inputNames.includes("num_logits_to_keep") && !model_inputs.num_logits_to_keep) {
|
|
17029
|
-
model_inputs.num_logits_to_keep = new Tensor22("int64", [1n], []);
|
|
17030
|
-
}
|
|
17993
|
+
setNumLogitsToKeep(self2, model_inputs, 1n);
|
|
17031
17994
|
if (!model_inputs.attention_mask) {
|
|
17032
17995
|
let dims;
|
|
17033
17996
|
for (const key of ["input_ids", "inputs_embeds", "position_ids"]) {
|
|
@@ -17053,6 +18016,7 @@ function encoder_decoder_prepare_inputs_for_generation(self2, input_ids, model_i
|
|
|
17053
18016
|
if (model_inputs.past_key_values) {
|
|
17054
18017
|
input_ids = input_ids.map((x) => [x.at(-1)]);
|
|
17055
18018
|
}
|
|
18019
|
+
setNumLogitsToKeep(self2, model_inputs, 1n);
|
|
17056
18020
|
return {
|
|
17057
18021
|
...model_inputs,
|
|
17058
18022
|
decoder_input_ids: toI64Tensor(input_ids)
|
|
@@ -17151,6 +18115,7 @@ __export(models_exports, {
|
|
|
17151
18115
|
BartForSequenceClassification: () => BartForSequenceClassification,
|
|
17152
18116
|
BartModel: () => BartModel,
|
|
17153
18117
|
BartPretrainedModel: () => BartPretrainedModel,
|
|
18118
|
+
BaseModelOutput: () => BaseModelOutput,
|
|
17154
18119
|
BeitForImageClassification: () => BeitForImageClassification,
|
|
17155
18120
|
BeitModel: () => BeitModel,
|
|
17156
18121
|
BeitPreTrainedModel: () => BeitPreTrainedModel,
|
|
@@ -17186,6 +18151,8 @@ __export(models_exports, {
|
|
|
17186
18151
|
CamembertForTokenClassification: () => CamembertForTokenClassification,
|
|
17187
18152
|
CamembertModel: () => CamembertModel,
|
|
17188
18153
|
CamembertPreTrainedModel: () => CamembertPreTrainedModel,
|
|
18154
|
+
CausalLMOutput: () => CausalLMOutput,
|
|
18155
|
+
CausalLMOutputWithPast: () => CausalLMOutputWithPast,
|
|
17189
18156
|
ChatterboxModel: () => ChatterboxModel,
|
|
17190
18157
|
ChatterboxPreTrainedModel: () => ChatterboxPreTrainedModel,
|
|
17191
18158
|
ChineseCLIPModel: () => ChineseCLIPModel,
|
|
@@ -17251,6 +18218,9 @@ __export(models_exports, {
|
|
|
17251
18218
|
DeepseekV3ForCausalLM: () => DeepseekV3ForCausalLM,
|
|
17252
18219
|
DeepseekV3Model: () => DeepseekV3Model,
|
|
17253
18220
|
DeepseekV3PreTrainedModel: () => DeepseekV3PreTrainedModel,
|
|
18221
|
+
DeepseekV4ForCausalLM: () => DeepseekV4ForCausalLM,
|
|
18222
|
+
DeepseekV4Model: () => DeepseekV4Model,
|
|
18223
|
+
DeepseekV4PreTrainedModel: () => DeepseekV4PreTrainedModel,
|
|
17254
18224
|
DeiTForImageClassification: () => DeiTForImageClassification,
|
|
17255
18225
|
DeiTModel: () => DeiTModel,
|
|
17256
18226
|
DeiTPreTrainedModel: () => DeiTPreTrainedModel,
|
|
@@ -17375,6 +18345,9 @@ __export(models_exports, {
|
|
|
17375
18345
|
HieraForImageClassification: () => HieraForImageClassification,
|
|
17376
18346
|
HieraModel: () => HieraModel,
|
|
17377
18347
|
HieraPreTrainedModel: () => HieraPreTrainedModel,
|
|
18348
|
+
HrmTextForCausalLM: () => HrmTextForCausalLM,
|
|
18349
|
+
HrmTextModel: () => HrmTextModel,
|
|
18350
|
+
HrmTextPreTrainedModel: () => HrmTextPreTrainedModel,
|
|
17378
18351
|
HubertForCTC: () => HubertForCTC,
|
|
17379
18352
|
HubertForSequenceClassification: () => HubertForSequenceClassification,
|
|
17380
18353
|
HubertModel: () => HubertModel,
|
|
@@ -17386,6 +18359,7 @@ __export(models_exports, {
|
|
|
17386
18359
|
IJepaModel: () => IJepaModel,
|
|
17387
18360
|
IJepaPreTrainedModel: () => IJepaPreTrainedModel,
|
|
17388
18361
|
Idefics3ForConditionalGeneration: () => Idefics3ForConditionalGeneration,
|
|
18362
|
+
ImageMattingOutput: () => ImageMattingOutput,
|
|
17389
18363
|
JAISLMHeadModel: () => JAISLMHeadModel,
|
|
17390
18364
|
JAISModel: () => JAISModel,
|
|
17391
18365
|
JAISPreTrainedModel: () => JAISPreTrainedModel,
|
|
@@ -17437,6 +18411,7 @@ __export(models_exports, {
|
|
|
17437
18411
|
MaskFormerForInstanceSegmentation: () => MaskFormerForInstanceSegmentation,
|
|
17438
18412
|
MaskFormerModel: () => MaskFormerModel,
|
|
17439
18413
|
MaskFormerPreTrainedModel: () => MaskFormerPreTrainedModel,
|
|
18414
|
+
MaskedLMOutput: () => MaskedLMOutput,
|
|
17440
18415
|
Metric3DForDepthEstimation: () => Metric3DForDepthEstimation,
|
|
17441
18416
|
Metric3DPreTrainedModel: () => Metric3DPreTrainedModel,
|
|
17442
18417
|
Metric3Dv2ForDepthEstimation: () => Metric3Dv2ForDepthEstimation,
|
|
@@ -17450,6 +18425,13 @@ __export(models_exports, {
|
|
|
17450
18425
|
MimiEncoderOutput: () => MimiEncoderOutput,
|
|
17451
18426
|
MimiModel: () => MimiModel,
|
|
17452
18427
|
MimiPreTrainedModel: () => MimiPreTrainedModel,
|
|
18428
|
+
Ministral3ForCausalLM: () => Ministral3ForCausalLM,
|
|
18429
|
+
Ministral3Model: () => Ministral3Model,
|
|
18430
|
+
Ministral3PreTrainedModel: () => Ministral3PreTrainedModel,
|
|
18431
|
+
MinistralForCausalLM: () => MinistralForCausalLM,
|
|
18432
|
+
MinistralModel: () => MinistralModel,
|
|
18433
|
+
MinistralPreTrainedModel: () => MinistralPreTrainedModel,
|
|
18434
|
+
Mistral3ForConditionalGeneration: () => Mistral3ForConditionalGeneration,
|
|
17453
18435
|
Mistral4ForCausalLM: () => Mistral4ForCausalLM,
|
|
17454
18436
|
Mistral4Model: () => Mistral4Model,
|
|
17455
18437
|
Mistral4PreTrainedModel: () => Mistral4PreTrainedModel,
|
|
@@ -17486,6 +18468,7 @@ __export(models_exports, {
|
|
|
17486
18468
|
MobileViTV2ForImageClassification: () => MobileViTV2ForImageClassification,
|
|
17487
18469
|
MobileViTV2Model: () => MobileViTV2Model,
|
|
17488
18470
|
MobileViTV2PreTrainedModel: () => MobileViTV2PreTrainedModel,
|
|
18471
|
+
ModelOutput: () => ModelOutput,
|
|
17489
18472
|
ModernBertDecoderForCausalLM: () => ModernBertDecoderForCausalLM,
|
|
17490
18473
|
ModernBertDecoderModel: () => ModernBertDecoderModel,
|
|
17491
18474
|
ModernBertDecoderPreTrainedModel: () => ModernBertDecoderPreTrainedModel,
|
|
@@ -17572,6 +18555,7 @@ __export(models_exports, {
|
|
|
17572
18555
|
PyAnnoteForAudioFrameClassification: () => PyAnnoteForAudioFrameClassification,
|
|
17573
18556
|
PyAnnoteModel: () => PyAnnoteModel,
|
|
17574
18557
|
PyAnnotePreTrainedModel: () => PyAnnotePreTrainedModel,
|
|
18558
|
+
QuestionAnsweringModelOutput: () => QuestionAnsweringModelOutput,
|
|
17575
18559
|
Qwen2ForCausalLM: () => Qwen2ForCausalLM,
|
|
17576
18560
|
Qwen2Model: () => Qwen2Model,
|
|
17577
18561
|
Qwen2MoeForCausalLM: () => Qwen2MoeForCausalLM,
|
|
@@ -17642,6 +18626,8 @@ __export(models_exports, {
|
|
|
17642
18626
|
SegformerForSemanticSegmentation: () => SegformerForSemanticSegmentation,
|
|
17643
18627
|
SegformerModel: () => SegformerModel,
|
|
17644
18628
|
SegformerPreTrainedModel: () => SegformerPreTrainedModel,
|
|
18629
|
+
Seq2SeqLMOutput: () => Seq2SeqLMOutput,
|
|
18630
|
+
SequenceClassifierOutput: () => SequenceClassifierOutput,
|
|
17645
18631
|
SiglipModel: () => SiglipModel,
|
|
17646
18632
|
SiglipPreTrainedModel: () => SiglipPreTrainedModel,
|
|
17647
18633
|
SiglipTextModel: () => SiglipTextModel,
|
|
@@ -17691,6 +18677,7 @@ __export(models_exports, {
|
|
|
17691
18677
|
TableTransformerModel: () => TableTransformerModel,
|
|
17692
18678
|
TableTransformerObjectDetectionOutput: () => TableTransformerObjectDetectionOutput,
|
|
17693
18679
|
TableTransformerPreTrainedModel: () => TableTransformerPreTrainedModel,
|
|
18680
|
+
TokenClassifierOutput: () => TokenClassifierOutput,
|
|
17694
18681
|
TrOCRForCausalLM: () => TrOCRForCausalLM,
|
|
17695
18682
|
TrOCRPreTrainedModel: () => TrOCRPreTrainedModel,
|
|
17696
18683
|
UltravoxModel: () => UltravoxModel,
|
|
@@ -17765,7 +18752,10 @@ __export(models_exports, {
|
|
|
17765
18752
|
YolosPreTrainedModel: () => YolosPreTrainedModel,
|
|
17766
18753
|
YoutuForCausalLM: () => YoutuForCausalLM,
|
|
17767
18754
|
YoutuModel: () => YoutuModel,
|
|
17768
|
-
YoutuPreTrainedModel: () => YoutuPreTrainedModel
|
|
18755
|
+
YoutuPreTrainedModel: () => YoutuPreTrainedModel,
|
|
18756
|
+
ZayaForCausalLM: () => ZayaForCausalLM,
|
|
18757
|
+
ZayaModel: () => ZayaModel,
|
|
18758
|
+
ZayaPreTrainedModel: () => ZayaPreTrainedModel
|
|
17769
18759
|
});
|
|
17770
18760
|
var AlbertPreTrainedModel = class extends PreTrainedModel {
|
|
17771
18761
|
};
|
|
@@ -17931,6 +18921,7 @@ var ChatterboxModel = class extends ChatterboxPreTrainedModel {
|
|
|
17931
18921
|
past_key_values = null,
|
|
17932
18922
|
generation_config = null,
|
|
17933
18923
|
logits_processor = null,
|
|
18924
|
+
num_logits_to_keep = null,
|
|
17934
18925
|
audio_features = null,
|
|
17935
18926
|
audio_tokens = null,
|
|
17936
18927
|
speaker_embeddings = null,
|
|
@@ -17981,7 +18972,8 @@ var ChatterboxModel = class extends ChatterboxPreTrainedModel {
|
|
|
17981
18972
|
past_key_values,
|
|
17982
18973
|
attention_mask,
|
|
17983
18974
|
generation_config,
|
|
17984
|
-
logits_processor
|
|
18975
|
+
logits_processor,
|
|
18976
|
+
num_logits_to_keep
|
|
17985
18977
|
}, false);
|
|
17986
18978
|
return {
|
|
17987
18979
|
...outputs,
|
|
@@ -18014,22 +19006,29 @@ var ChatterboxModel = class extends ChatterboxPreTrainedModel {
|
|
|
18014
19006
|
return decoder_prepare_inputs_for_generation(this, input_ids, model_inputs, generation_config);
|
|
18015
19007
|
}
|
|
18016
19008
|
async generate(params) {
|
|
18017
|
-
const
|
|
19009
|
+
const supplied_past_key_values = params.past_key_values;
|
|
19010
|
+
const { sequences, audio_tokens, speaker_embeddings, speaker_features, past_key_values } = await super.generate({
|
|
18018
19011
|
...params,
|
|
18019
19012
|
return_dict_in_generate: true
|
|
18020
19013
|
});
|
|
18021
|
-
|
|
18022
|
-
|
|
18023
|
-
|
|
18024
|
-
|
|
18025
|
-
|
|
18026
|
-
|
|
18027
|
-
|
|
18028
|
-
|
|
18029
|
-
|
|
18030
|
-
|
|
18031
|
-
|
|
18032
|
-
|
|
19014
|
+
try {
|
|
19015
|
+
const new_tokens = sequences.slice(null, [
|
|
19016
|
+
params.input_ids.dims[1],
|
|
19017
|
+
-1
|
|
19018
|
+
]);
|
|
19019
|
+
const silence_tokens = full([new_tokens.dims[0], 3], SILENCE_TOKEN);
|
|
19020
|
+
const speech_tokens = cat([audio_tokens, new_tokens, silence_tokens], 1);
|
|
19021
|
+
const { waveform } = await sessionRun(this.sessions["conditional_decoder"], {
|
|
19022
|
+
speech_tokens,
|
|
19023
|
+
speaker_features,
|
|
19024
|
+
speaker_embeddings
|
|
19025
|
+
});
|
|
19026
|
+
return waveform;
|
|
19027
|
+
} finally {
|
|
19028
|
+
if (past_key_values !== supplied_past_key_values) {
|
|
19029
|
+
await past_key_values?.dispose();
|
|
19030
|
+
}
|
|
19031
|
+
}
|
|
18033
19032
|
}
|
|
18034
19033
|
};
|
|
18035
19034
|
var ChineseCLIPPreTrainedModel = class extends PreTrainedModel {
|
|
@@ -18266,6 +19265,12 @@ var DeepseekV3Model = class extends DeepseekV3PreTrainedModel {
|
|
|
18266
19265
|
};
|
|
18267
19266
|
var DeepseekV3ForCausalLM = class extends DeepseekV3PreTrainedModel {
|
|
18268
19267
|
};
|
|
19268
|
+
var DeepseekV4PreTrainedModel = class extends PreTrainedModel {
|
|
19269
|
+
};
|
|
19270
|
+
var DeepseekV4Model = class extends DeepseekV4PreTrainedModel {
|
|
19271
|
+
};
|
|
19272
|
+
var DeepseekV4ForCausalLM = class extends DeepseekV4PreTrainedModel {
|
|
19273
|
+
};
|
|
18269
19274
|
var DebertaV2PreTrainedModel = class extends PreTrainedModel {
|
|
18270
19275
|
};
|
|
18271
19276
|
var DebertaV2Model = class extends DebertaV2PreTrainedModel {
|
|
@@ -18563,7 +19568,8 @@ var Florence2ForConditionalGeneration = class extends Florence2PreTrainedModel {
|
|
|
18563
19568
|
encoder_outputs,
|
|
18564
19569
|
past_key_values,
|
|
18565
19570
|
inputs_embeds,
|
|
18566
|
-
decoder_inputs_embeds
|
|
19571
|
+
decoder_inputs_embeds,
|
|
19572
|
+
num_logits_to_keep = null
|
|
18567
19573
|
}) {
|
|
18568
19574
|
if (!inputs_embeds) {
|
|
18569
19575
|
({ inputs_embeds, attention_mask } = await this._prepare_inputs_embeds({
|
|
@@ -18588,7 +19594,8 @@ var Florence2ForConditionalGeneration = class extends Florence2PreTrainedModel {
|
|
|
18588
19594
|
attention_mask: decoder_attention_mask,
|
|
18589
19595
|
encoder_attention_mask: attention_mask,
|
|
18590
19596
|
encoder_hidden_states: encoder_outputs,
|
|
18591
|
-
past_key_values
|
|
19597
|
+
past_key_values,
|
|
19598
|
+
num_logits_to_keep
|
|
18592
19599
|
};
|
|
18593
19600
|
return await decoder_forward(this, decoderFeeds, true);
|
|
18594
19601
|
}
|
|
@@ -18657,6 +19664,7 @@ var Gemma3nForConditionalGeneration = class extends Gemma3nPreTrainedModel {
|
|
|
18657
19664
|
past_key_values = null,
|
|
18658
19665
|
generation_config = null,
|
|
18659
19666
|
logits_processor = null,
|
|
19667
|
+
num_logits_to_keep = null,
|
|
18660
19668
|
...kwargs
|
|
18661
19669
|
}) {
|
|
18662
19670
|
if (!inputs_embeds || !per_layer_inputs) {
|
|
@@ -18694,7 +19702,8 @@ var Gemma3nForConditionalGeneration = class extends Gemma3nPreTrainedModel {
|
|
|
18694
19702
|
attention_mask,
|
|
18695
19703
|
position_ids,
|
|
18696
19704
|
generation_config,
|
|
18697
|
-
logits_processor
|
|
19705
|
+
logits_processor,
|
|
19706
|
+
num_logits_to_keep
|
|
18698
19707
|
}, true);
|
|
18699
19708
|
return outputs;
|
|
18700
19709
|
}
|
|
@@ -18916,6 +19925,7 @@ var Qwen2VLForConditionalGeneration = class extends Qwen2VLPreTrainedModel {
|
|
|
18916
19925
|
});
|
|
18917
19926
|
}
|
|
18918
19927
|
prepare_inputs_for_generation(input_ids, model_inputs, generation_config) {
|
|
19928
|
+
setNumLogitsToKeep(this, model_inputs, 1n);
|
|
18919
19929
|
if (!model_inputs.attention_mask || model_inputs.position_ids) {
|
|
18920
19930
|
return model_inputs;
|
|
18921
19931
|
}
|
|
@@ -19094,6 +20104,18 @@ var HieraForImageClassification = class extends HieraPreTrainedModel {
|
|
|
19094
20104
|
return new SequenceClassifierOutput(await super._call(model_inputs));
|
|
19095
20105
|
}
|
|
19096
20106
|
};
|
|
20107
|
+
var HrmTextPreTrainedModel = class extends PreTrainedModel {
|
|
20108
|
+
};
|
|
20109
|
+
var HrmTextModel = class extends HrmTextPreTrainedModel {
|
|
20110
|
+
};
|
|
20111
|
+
var HrmTextForCausalLM = class extends HrmTextPreTrainedModel {
|
|
20112
|
+
forward_params = ["input_ids", "attention_mask", "token_type_ids", "past_key_values"];
|
|
20113
|
+
prepare_inputs_for_generation(input_ids, model_inputs, generation_config) {
|
|
20114
|
+
const prepared = decoder_prepare_inputs_for_generation(this, input_ids, model_inputs, generation_config);
|
|
20115
|
+
prepared.token_type_ids = (prepared.past_key_values ? zeros_like : ones_like)(prepared.input_ids);
|
|
20116
|
+
return prepared;
|
|
20117
|
+
}
|
|
20118
|
+
};
|
|
19097
20119
|
var Wav2Vec2PreTrainedModel = class extends PreTrainedModel {
|
|
19098
20120
|
};
|
|
19099
20121
|
var Wav2Vec2Model = class extends Wav2Vec2PreTrainedModel {
|
|
@@ -19341,12 +20363,26 @@ var MimiDecoderModel = class extends MimiPreTrainedModel {
|
|
|
19341
20363
|
});
|
|
19342
20364
|
}
|
|
19343
20365
|
};
|
|
20366
|
+
var MinistralPreTrainedModel = class extends PreTrainedModel {
|
|
20367
|
+
};
|
|
20368
|
+
var MinistralModel = class extends MinistralPreTrainedModel {
|
|
20369
|
+
};
|
|
20370
|
+
var MinistralForCausalLM = class extends MinistralPreTrainedModel {
|
|
20371
|
+
};
|
|
20372
|
+
var Ministral3PreTrainedModel = class extends PreTrainedModel {
|
|
20373
|
+
};
|
|
20374
|
+
var Ministral3Model = class extends Ministral3PreTrainedModel {
|
|
20375
|
+
};
|
|
20376
|
+
var Ministral3ForCausalLM = class extends Ministral3PreTrainedModel {
|
|
20377
|
+
};
|
|
19344
20378
|
var MistralPreTrainedModel = class extends PreTrainedModel {
|
|
19345
20379
|
};
|
|
19346
20380
|
var MistralModel = class extends MistralPreTrainedModel {
|
|
19347
20381
|
};
|
|
19348
20382
|
var MistralForCausalLM = class extends MistralPreTrainedModel {
|
|
19349
20383
|
};
|
|
20384
|
+
var Mistral3ForConditionalGeneration = class extends LlavaForConditionalGeneration {
|
|
20385
|
+
};
|
|
19350
20386
|
var Mistral4PreTrainedModel = class extends PreTrainedModel {
|
|
19351
20387
|
};
|
|
19352
20388
|
var Mistral4Model = class extends Mistral4PreTrainedModel {
|
|
@@ -19555,6 +20591,7 @@ var MultiModalityCausalLM = class extends MultiModalityPreTrainedModel {
|
|
|
19555
20591
|
}
|
|
19556
20592
|
prepare_inputs_for_generation(input_ids, model_inputs, generation_config) {
|
|
19557
20593
|
const has_past_key_values = !!model_inputs.past_key_values;
|
|
20594
|
+
setNumLogitsToKeep(this, model_inputs, 1n);
|
|
19558
20595
|
if (generation_config.guidance_scale !== null && generation_config.guidance_scale > 1) {
|
|
19559
20596
|
if (has_past_key_values) {
|
|
19560
20597
|
model_inputs.input_ids = cat([model_inputs.input_ids, model_inputs.input_ids], 0);
|
|
@@ -19811,6 +20848,7 @@ var Phi3VForCausalLM = class extends Phi3VPreTrainedModel {
|
|
|
19811
20848
|
past_key_values = null,
|
|
19812
20849
|
generation_config = null,
|
|
19813
20850
|
logits_processor = null,
|
|
20851
|
+
num_logits_to_keep = null,
|
|
19814
20852
|
...kwargs
|
|
19815
20853
|
}) {
|
|
19816
20854
|
if (!inputs_embeds) {
|
|
@@ -19838,7 +20876,8 @@ var Phi3VForCausalLM = class extends Phi3VPreTrainedModel {
|
|
|
19838
20876
|
attention_mask,
|
|
19839
20877
|
position_ids,
|
|
19840
20878
|
generation_config,
|
|
19841
|
-
logits_processor
|
|
20879
|
+
logits_processor,
|
|
20880
|
+
num_logits_to_keep
|
|
19842
20881
|
}, false);
|
|
19843
20882
|
return outputs;
|
|
19844
20883
|
}
|
|
@@ -20481,7 +21520,11 @@ function createEncoderState(model, input_features) {
|
|
|
20481
21520
|
enc_kv_cache[meta.name] = new Tensor22(meta.type, new cls(size), shape);
|
|
20482
21521
|
}
|
|
20483
21522
|
const padding_cls = DataTypeMap[padding_type];
|
|
20484
|
-
const enc_padding_cache = new Tensor22(padding_type, new padding_cls(PADDING_CACHE_CHANNELS * CONV1_LEFT_PAD), [
|
|
21523
|
+
const enc_padding_cache = new Tensor22(padding_type, new padding_cls(PADDING_CACHE_CHANNELS * CONV1_LEFT_PAD), [
|
|
21524
|
+
1,
|
|
21525
|
+
PADDING_CACHE_CHANNELS,
|
|
21526
|
+
CONV1_LEFT_PAD
|
|
21527
|
+
]);
|
|
20485
21528
|
const chunks_iter = input_features[Symbol.asyncIterator]?.() ?? input_features[Symbol.iterator]?.();
|
|
20486
21529
|
if (!chunks_iter) {
|
|
20487
21530
|
throw new Error("input_features must be iterable or async iterable");
|
|
@@ -20889,7 +21932,7 @@ var WhisperForConditionalGeneration = class extends WhisperPreTrainedModel {
|
|
|
20889
21932
|
throw new Error(`Layer index ${l} is out of bounds for cross attentions (length ${cross_attentions.length}).`);
|
|
20890
21933
|
}
|
|
20891
21934
|
return num_frames ? cross_attentions[l].slice(null, h, null, [0, num_frames]) : cross_attentions[l].slice(null, h);
|
|
20892
|
-
})).
|
|
21935
|
+
})).permute(1, 0, 2, 3);
|
|
20893
21936
|
const [std, calculatedMean] = std_mean(weights, -2, 0, true);
|
|
20894
21937
|
const smoothedWeights = weights.clone();
|
|
20895
21938
|
for (let a = 0;a < smoothedWeights.dims[0]; ++a) {
|
|
@@ -21004,6 +22047,12 @@ var YoutuModel = class extends YoutuPreTrainedModel {
|
|
|
21004
22047
|
};
|
|
21005
22048
|
var YoutuForCausalLM = class extends YoutuPreTrainedModel {
|
|
21006
22049
|
};
|
|
22050
|
+
var ZayaPreTrainedModel = class extends PreTrainedModel {
|
|
22051
|
+
};
|
|
22052
|
+
var ZayaModel = class extends ZayaPreTrainedModel {
|
|
22053
|
+
};
|
|
22054
|
+
var ZayaForCausalLM = class extends ZayaPreTrainedModel {
|
|
22055
|
+
};
|
|
21007
22056
|
var MODEL_MAPPING_NAMES_ENCODER_ONLY = /* @__PURE__ */ new Map([
|
|
21008
22057
|
["bert", "BertModel"],
|
|
21009
22058
|
["eurobert", "EuroBertModel"],
|
|
@@ -21158,6 +22207,7 @@ var MODEL_MAPPING_NAMES_DECODER_ONLY = /* @__PURE__ */ new Map([
|
|
|
21158
22207
|
["ernie4_5", "Ernie4_5ForCausalLM"],
|
|
21159
22208
|
["starcoder2", "Starcoder2Model"],
|
|
21160
22209
|
["deepseek_v3", "DeepseekV3Model"],
|
|
22210
|
+
["deepseek_v4", "DeepseekV4Model"],
|
|
21161
22211
|
["falcon", "FalconModel"],
|
|
21162
22212
|
["falcon_h1", "FalconH1Model"],
|
|
21163
22213
|
["nemotron_h", "NemotronHModel"],
|
|
@@ -21165,7 +22215,9 @@ var MODEL_MAPPING_NAMES_DECODER_ONLY = /* @__PURE__ */ new Map([
|
|
|
21165
22215
|
["stablelm", "StableLmModel"],
|
|
21166
22216
|
["modernbert-decoder", "ModernBertDecoderModel"],
|
|
21167
22217
|
["hunyuan_v1_dense", "HunYuanDenseV1Model"],
|
|
21168
|
-
["youtu", "YoutuModel"]
|
|
22218
|
+
["youtu", "YoutuModel"],
|
|
22219
|
+
["zaya", "ZayaModel"],
|
|
22220
|
+
["hrm_text", "HrmTextModel"]
|
|
21169
22221
|
]);
|
|
21170
22222
|
var MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES = /* @__PURE__ */ new Map([
|
|
21171
22223
|
["speecht5", "SpeechT5ForSpeechToText"],
|
|
@@ -21297,6 +22349,7 @@ var MODEL_FOR_CAUSAL_LM_MAPPING_NAMES = /* @__PURE__ */ new Map([
|
|
|
21297
22349
|
["ernie4_5", "Ernie4_5ForCausalLM"],
|
|
21298
22350
|
["starcoder2", "Starcoder2ForCausalLM"],
|
|
21299
22351
|
["deepseek_v3", "DeepseekV3ForCausalLM"],
|
|
22352
|
+
["deepseek_v4", "DeepseekV4ForCausalLM"],
|
|
21300
22353
|
["falcon", "FalconForCausalLM"],
|
|
21301
22354
|
["falcon_h1", "FalconH1ForCausalLM"],
|
|
21302
22355
|
["nemotron_h", "NemotronHForCausalLM"],
|
|
@@ -21306,6 +22359,8 @@ var MODEL_FOR_CAUSAL_LM_MAPPING_NAMES = /* @__PURE__ */ new Map([
|
|
|
21306
22359
|
["modernbert-decoder", "ModernBertDecoderForCausalLM"],
|
|
21307
22360
|
["hunyuan_v1_dense", "HunYuanDenseV1ForCausalLM"],
|
|
21308
22361
|
["youtu", "YoutuForCausalLM"],
|
|
22362
|
+
["zaya", "ZayaForCausalLM"],
|
|
22363
|
+
["hrm_text", "HrmTextForCausalLM"],
|
|
21309
22364
|
["phi3_v", "Phi3VForCausalLM"]
|
|
21310
22365
|
]);
|
|
21311
22366
|
var MODEL_FOR_MULTIMODALITY_MAPPING_NAMES = /* @__PURE__ */ new Map([["multi_modality", "MultiModalityCausalLM"]]);
|
|
@@ -22386,7 +23441,7 @@ var AutomaticSpeechRecognitionPipeline = class extends Pipeline {
|
|
|
22386
23441
|
const inputs = await this.processor(aud);
|
|
22387
23442
|
const max_new_tokens = Math.floor(aud.length / sampling_rate) * 6;
|
|
22388
23443
|
const outputs = await this.model.generate({ max_new_tokens, ...kwargs, ...inputs });
|
|
22389
|
-
const text = this.
|
|
23444
|
+
const text = this.tokenizer.batch_decode(outputs, { skip_special_tokens: true })[0];
|
|
22390
23445
|
toReturn.push({ text });
|
|
22391
23446
|
}
|
|
22392
23447
|
return single ? toReturn[0] : toReturn;
|
|
@@ -23176,15 +24231,19 @@ async function pipeline2(task, model = null, {
|
|
|
23176
24231
|
});
|
|
23177
24232
|
let files_loading = {};
|
|
23178
24233
|
if (progress_callback) {
|
|
23179
|
-
|
|
23180
|
-
|
|
23181
|
-
|
|
23182
|
-
|
|
23183
|
-
|
|
23184
|
-
|
|
23185
|
-
|
|
23186
|
-
|
|
23187
|
-
|
|
24234
|
+
try {
|
|
24235
|
+
const metadata = await Promise.all(expected_files.map(async (file) => get_file_metadata(model, file)));
|
|
24236
|
+
metadata.forEach((m, i) => {
|
|
24237
|
+
if (m.exists) {
|
|
24238
|
+
files_loading[expected_files[i]] = {
|
|
24239
|
+
loaded: 0,
|
|
24240
|
+
total: m.size ?? 0
|
|
24241
|
+
};
|
|
24242
|
+
}
|
|
24243
|
+
});
|
|
24244
|
+
} catch (e) {
|
|
24245
|
+
logger.warn(`Unable to fetch model file metadata for total progress tracking: ${e}`);
|
|
24246
|
+
}
|
|
23188
24247
|
}
|
|
23189
24248
|
const pretrainedOptions = {
|
|
23190
24249
|
progress_callback: progress_callback ? new DefaultProgressCallback(progress_callback, files_loading) : undefined,
|
|
@@ -23631,6 +24690,19 @@ var ModelRegistry = class {
|
|
|
23631
24690
|
return clear_pipeline_cache(task, modelId, options);
|
|
23632
24691
|
}
|
|
23633
24692
|
};
|
|
24693
|
+
|
|
24694
|
+
// ../plugin/src/features/magic-context/memory/transformers-remote-host.ts
|
|
24695
|
+
var DEFAULT_TRANSFORMERS_REMOTE_HOST = "https://huggingface.co/";
|
|
24696
|
+
function readHuggingFaceEndpoint() {
|
|
24697
|
+
return globalThis.process?.env?.HF_ENDPOINT;
|
|
24698
|
+
}
|
|
24699
|
+
function configureTransformersRemoteHost(env, endpoint = readHuggingFaceEndpoint()) {
|
|
24700
|
+
const normalized = endpoint?.trim().replace(/\/+$/, "");
|
|
24701
|
+
env.remoteHost = normalized ? `${normalized}/` : DEFAULT_TRANSFORMERS_REMOTE_HOST;
|
|
24702
|
+
}
|
|
24703
|
+
|
|
24704
|
+
// ../plugin/src/features/magic-context/memory/transformers-web-entry.ts
|
|
24705
|
+
configureTransformersRemoteHost(env3);
|
|
23634
24706
|
export {
|
|
23635
24707
|
ASTFeatureExtractor,
|
|
23636
24708
|
ASTForAudioClassification,
|
|
@@ -23695,6 +24767,7 @@ export {
|
|
|
23695
24767
|
BartModel,
|
|
23696
24768
|
BartPretrainedModel,
|
|
23697
24769
|
BartTokenizer,
|
|
24770
|
+
BaseModelOutput,
|
|
23698
24771
|
BaseStreamer,
|
|
23699
24772
|
BeitFeatureExtractor,
|
|
23700
24773
|
BeitForImageClassification,
|
|
@@ -23742,6 +24815,8 @@ export {
|
|
|
23742
24815
|
CamembertModel,
|
|
23743
24816
|
CamembertPreTrainedModel,
|
|
23744
24817
|
CamembertTokenizer,
|
|
24818
|
+
CausalLMOutput,
|
|
24819
|
+
CausalLMOutputWithPast,
|
|
23745
24820
|
ChatterboxFeatureExtractor,
|
|
23746
24821
|
ChatterboxModel,
|
|
23747
24822
|
ChatterboxPreTrainedModel,
|
|
@@ -23827,6 +24902,9 @@ export {
|
|
|
23827
24902
|
DeepseekV3ForCausalLM,
|
|
23828
24903
|
DeepseekV3Model,
|
|
23829
24904
|
DeepseekV3PreTrainedModel,
|
|
24905
|
+
DeepseekV4ForCausalLM,
|
|
24906
|
+
DeepseekV4Model,
|
|
24907
|
+
DeepseekV4PreTrainedModel,
|
|
23830
24908
|
DeiTFeatureExtractor,
|
|
23831
24909
|
DeiTForImageClassification,
|
|
23832
24910
|
DeiTImageProcessor,
|
|
@@ -23957,6 +25035,7 @@ export {
|
|
|
23957
25035
|
GemmaModel,
|
|
23958
25036
|
GemmaPreTrainedModel,
|
|
23959
25037
|
GemmaTokenizer,
|
|
25038
|
+
GenerationConfig,
|
|
23960
25039
|
Glm46VImageProcessor,
|
|
23961
25040
|
Glm46VProcessor,
|
|
23962
25041
|
GlmForCausalLM,
|
|
@@ -23991,6 +25070,9 @@ export {
|
|
|
23991
25070
|
HieraForImageClassification,
|
|
23992
25071
|
HieraModel,
|
|
23993
25072
|
HieraPreTrainedModel,
|
|
25073
|
+
HrmTextForCausalLM,
|
|
25074
|
+
HrmTextModel,
|
|
25075
|
+
HrmTextPreTrainedModel,
|
|
23994
25076
|
HubertForCTC,
|
|
23995
25077
|
HubertForSequenceClassification,
|
|
23996
25078
|
HubertModel,
|
|
@@ -24007,6 +25089,7 @@ export {
|
|
|
24007
25089
|
ImageClassificationPipeline,
|
|
24008
25090
|
ImageFeatureExtractionPipeline,
|
|
24009
25091
|
ImageProcessor as ImageFeatureExtractor,
|
|
25092
|
+
ImageMattingOutput,
|
|
24010
25093
|
ImageProcessor,
|
|
24011
25094
|
ImageSegmentationPipeline,
|
|
24012
25095
|
ImageToImagePipeline,
|
|
@@ -24082,6 +25165,7 @@ export {
|
|
|
24082
25165
|
MaskFormerImageProcessor,
|
|
24083
25166
|
MaskFormerModel,
|
|
24084
25167
|
MaskFormerPreTrainedModel,
|
|
25168
|
+
MaskedLMOutput,
|
|
24085
25169
|
MaxLengthCriteria,
|
|
24086
25170
|
Metric3DForDepthEstimation,
|
|
24087
25171
|
Metric3DPreTrainedModel,
|
|
@@ -24100,6 +25184,13 @@ export {
|
|
|
24100
25184
|
MimiPreTrainedModel,
|
|
24101
25185
|
MinLengthLogitsProcessor,
|
|
24102
25186
|
MinNewTokensLengthLogitsProcessor,
|
|
25187
|
+
Ministral3ForCausalLM,
|
|
25188
|
+
Ministral3Model,
|
|
25189
|
+
Ministral3PreTrainedModel,
|
|
25190
|
+
MinistralForCausalLM,
|
|
25191
|
+
MinistralModel,
|
|
25192
|
+
MinistralPreTrainedModel,
|
|
25193
|
+
Mistral3ForConditionalGeneration,
|
|
24103
25194
|
Mistral4ForCausalLM,
|
|
24104
25195
|
Mistral4Model,
|
|
24105
25196
|
Mistral4PreTrainedModel,
|
|
@@ -24147,6 +25238,8 @@ export {
|
|
|
24147
25238
|
MobileViTV2ForImageClassification,
|
|
24148
25239
|
MobileViTV2Model,
|
|
24149
25240
|
MobileViTV2PreTrainedModel,
|
|
25241
|
+
ModelFileNotFoundError,
|
|
25242
|
+
ModelOutput,
|
|
24150
25243
|
ModelRegistry,
|
|
24151
25244
|
ModernBertDecoderForCausalLM,
|
|
24152
25245
|
ModernBertDecoderModel,
|
|
@@ -24258,6 +25351,7 @@ export {
|
|
|
24258
25351
|
PyAnnoteModel,
|
|
24259
25352
|
PyAnnotePreTrainedModel,
|
|
24260
25353
|
PyAnnoteProcessor,
|
|
25354
|
+
QuestionAnsweringModelOutput,
|
|
24261
25355
|
QuestionAnsweringPipeline,
|
|
24262
25356
|
Qwen2ForCausalLM,
|
|
24263
25357
|
Qwen2Model,
|
|
@@ -24353,6 +25447,8 @@ export {
|
|
|
24353
25447
|
SegformerImageProcessor,
|
|
24354
25448
|
SegformerModel,
|
|
24355
25449
|
SegformerPreTrainedModel,
|
|
25450
|
+
Seq2SeqLMOutput,
|
|
25451
|
+
SequenceClassifierOutput,
|
|
24356
25452
|
SiglipImageProcessor,
|
|
24357
25453
|
SiglipModel,
|
|
24358
25454
|
SiglipPreTrainedModel,
|
|
@@ -24426,6 +25522,7 @@ export {
|
|
|
24426
25522
|
TextStreamer,
|
|
24427
25523
|
TextToAudioPipeline,
|
|
24428
25524
|
TokenClassificationPipeline,
|
|
25525
|
+
TokenClassifierOutput,
|
|
24429
25526
|
PreTrainedTokenizer as TokenizersBackend,
|
|
24430
25527
|
TopKLogitsWarper,
|
|
24431
25528
|
TopPLogitsWarper,
|
|
@@ -24530,6 +25627,9 @@ export {
|
|
|
24530
25627
|
YoutuForCausalLM,
|
|
24531
25628
|
YoutuModel,
|
|
24532
25629
|
YoutuPreTrainedModel,
|
|
25630
|
+
ZayaForCausalLM,
|
|
25631
|
+
ZayaModel,
|
|
25632
|
+
ZayaPreTrainedModel,
|
|
24533
25633
|
ZeroShotAudioClassificationPipeline,
|
|
24534
25634
|
ZeroShotClassificationPipeline,
|
|
24535
25635
|
ZeroShotImageClassificationPipeline,
|