@huggingface/transformers 4.0.0-next.2 → 4.0.0-next.3

package/dist/transformers.node.mjs

@@ -14,7 +14,7 @@ var __export = (target, all) => {
 import fs from "fs";
 import path from "path";
 import url from "url";
-var VERSION = "4.0.0-next.2";
+var VERSION = "4.0.0-next.3";
 var IS_PROCESS_AVAILABLE = typeof process !== "undefined";
 var IS_NODE_ENV = IS_PROCESS_AVAILABLE && process?.release?.name === "node";
 var IS_FS_AVAILABLE = !isEmpty(fs);
@@ -6759,7 +6759,7 @@ var uint16_to_float32 = /* @__PURE__ */ (function() {
 // src/backends/onnx.js
 import * as ONNX_NODE from "onnxruntime-node";
 
-// ../../node_modules/.pnpm/onnxruntime-web@1.25.0-dev.20260209-a3749f1353/node_modules/onnxruntime-web/dist/ort.webgpu.bundle.min.mjs
+// ../../node_modules/.pnpm/onnxruntime-web@1.25.0-dev.20260212-1a71a5f46e/node_modules/onnxruntime-web/dist/ort.webgpu.bundle.min.mjs
 var ort_webgpu_bundle_min_exports = {};
 __export(ort_webgpu_bundle_min_exports, {
   InferenceSession: () => nc,
@@ -8728,7 +8728,7 @@ async function is(a = {}) {
         we(`invalid type for getValue: ${t}`);
     }
   }, r.UTF8ToString = ct, r.stringToUTF8 = Pe, r.lengthBytesUTF8 = _e;
-  var bo, wo, Rr, $t, Te, mt, go, To, vo, Eo, So, Ao, Io, xo, Lo, Oo, Bo, Nr, Wr, kr, Fr, Et, Gr, Mo, $r, Uo, Co, Do, zr, Po, _o, Vr, k, St, Ro, C, zt, D, No, Hr, Wo, ko, Fo, jr, Go, $o, zo, Vo, Ho, jo, Yo, qo, Jo, Xo, Zo, Ko, Qo, ea, ta, ra, na, oa, aa, sa, ia, ua, fa, ca, da, la, pa, ma, ha, ya, ba, wa, ga, Ta, va, Ea, Sa, Aa, Ia, xa, Ne, df = [Ye, gr, In, Mn, Un, Cn, Dn, Pn, _n, Rn, Nn, Wn, kn, Fn, Gn, $n, eo, to, ro, co, lo, po, mo, ho, yo], Yr = { 1116828: (e, t, n, o, u) => {
+  var bo, wo, Rr, $t, Te, mt, go, To, vo, Eo, So, Ao, Io, xo, Lo, Oo, Bo, Nr, Wr, kr, Fr, Et, Gr, Mo, $r, Uo, Co, Do, zr, Po, _o, Vr, k, St, Ro, C, zt, D, No, Hr, Wo, ko, Fo, jr, Go, $o, zo, Vo, Ho, jo, Yo, qo, Jo, Xo, Zo, Ko, Qo, ea, ta, ra, na, oa, aa, sa, ia, ua, fa, ca, da, la, pa, ma, ha, ya, ba, wa, ga, Ta, va, Ea, Sa, Aa, Ia, xa, Ne, df = [Ye, gr, In, Mn, Un, Cn, Dn, Pn, _n, Rn, Nn, Wn, kn, Fn, Gn, $n, eo, to, ro, co, lo, po, mo, ho, yo], Yr = { 1117404: (e, t, n, o, u) => {
     if (r === void 0 || !r.Zc) return 1;
     if ((e = ct(Number(e >>> 0))).startsWith("./") && (e = e.substring(2)), !(e = r.Zc.get(e))) return 2;
     if (t = Number(t >>> 0), n = Number(n >>> 0), o = Number(o >>> 0), t + n > e.byteLength) return 3;
@@ -8748,11 +8748,11 @@ async function is(a = {}) {
     } catch {
       return 4;
     }
-  }, 1117652: (e, t, n) => {
+  }, 1118228: (e, t, n) => {
     r.ke(e, (l(), J).subarray(t >>> 0, t + n >>> 0));
-  }, 1117716: () => r.Ie(), 1117758: (e) => {
+  }, 1118292: () => r.Ie(), 1118334: (e) => {
     r.je(e);
-  }, 1117795: () => typeof wasmOffsetConverter < "u" };
+  }, 1118371: () => typeof wasmOffsetConverter < "u" };
   function lf() {
     return typeof wasmOffsetConverter < "u";
   }
@@ -10680,7 +10680,7 @@ var qs = F(() => {
 ze();
 ze();
 ze();
-var rs = "1.25.0-dev.20260209-a3749f1353";
+var rs = "1.25.0-dev.20260212-1a71a5f46e";
 var Ol = tn;
 {
   let a = (qs(), Ht(Ys)).wasmBackend;
@@ -22314,7 +22314,7 @@ var PreTrainedModel = class extends Callable2 {
   /**
    *
    * @param {GenerationConfig} generation_config
-   * @param {StoppingCriteriaList} [stopping_criteria=null]
+   * @param {import('../generation/stopping_criteria.js').StoppingCriteria|import('../generation/stopping_criteria.js').StoppingCriteria[]|StoppingCriteriaList} [stopping_criteria=null]
    */
   _get_stopping_criteria(generation_config, stopping_criteria = null) {
     const criteria = new StoppingCriteriaList();
@@ -30123,6 +30123,7 @@ var TextStreamer = class extends BaseStreamer {
     this.token_cache = [];
     this.print_len = 0;
     this.next_tokens_are_prompt = true;
+    this.special_ids = new Set(this.tokenizer.all_special_ids.map(BigInt));
   }
   /**
    * Receives tokens, decodes them, and prints them to stdout as soon as they form entire words.
@@ -30139,6 +30140,19 @@ var TextStreamer = class extends BaseStreamer {
     }
     const tokens = value[0];
     this.token_callback_function?.(tokens);
+    if (tokens.length === 1 && this.special_ids.has(tokens[0])) {
+      if (this.decode_kwargs.skip_special_tokens) return;
+      if (this.token_cache.length > 0) {
+        const text2 = this.tokenizer.decode(this.token_cache, this.decode_kwargs);
+        const printable_text2 = text2.slice(this.print_len);
+        this.on_finalized_text(printable_text2, false);
+        this.token_cache = [];
+        this.print_len = 0;
+      }
+      const special_text = this.tokenizer.decode(tokens, this.decode_kwargs);
+      this.on_finalized_text(special_text, false);
+      return;
+    }
     this.token_cache = mergeArrays(this.token_cache, tokens);
     const text = this.tokenizer.decode(this.token_cache, this.decode_kwargs);
     let printable_text;
@@ -31197,7 +31211,7 @@ export {
 
 onnxruntime-web/dist/ort.webgpu.bundle.min.mjs:
   (*!
-   * ONNX Runtime Web v1.25.0-dev.20260209-a3749f1353
+   * ONNX Runtime Web v1.25.0-dev.20260212-1a71a5f46e
    * Copyright (c) Microsoft Corporation. All rights reserved.
    * Licensed under the MIT License.
    *)

package/dist/transformers.web.js

@@ -17,7 +17,7 @@ var emptyObj3 = {};
 var node_url_default = emptyObj3;
 
 // src/env.js
-var VERSION = "4.0.0-next.2";
+var VERSION = "4.0.0-next.3";
 var IS_PROCESS_AVAILABLE = typeof process !== "undefined";
 var IS_NODE_ENV = IS_PROCESS_AVAILABLE && process?.release?.name === "node";
 var IS_FS_AVAILABLE = !isEmpty(node_fs_default);
@@ -18414,7 +18414,7 @@ var PreTrainedModel = class extends Callable2 {
   /**
    *
    * @param {GenerationConfig} generation_config
-   * @param {StoppingCriteriaList} [stopping_criteria=null]
+   * @param {import('../generation/stopping_criteria.js').StoppingCriteria|import('../generation/stopping_criteria.js').StoppingCriteria[]|StoppingCriteriaList} [stopping_criteria=null]
    */
   _get_stopping_criteria(generation_config, stopping_criteria = null) {
     const criteria = new StoppingCriteriaList();
@@ -26223,6 +26223,7 @@ var TextStreamer = class extends BaseStreamer {
     this.token_cache = [];
     this.print_len = 0;
     this.next_tokens_are_prompt = true;
+    this.special_ids = new Set(this.tokenizer.all_special_ids.map(BigInt));
   }
   /**
    * Receives tokens, decodes them, and prints them to stdout as soon as they form entire words.
@@ -26239,6 +26240,19 @@ var TextStreamer = class extends BaseStreamer {
     }
     const tokens = value[0];
     this.token_callback_function?.(tokens);
+    if (tokens.length === 1 && this.special_ids.has(tokens[0])) {
+      if (this.decode_kwargs.skip_special_tokens) return;
+      if (this.token_cache.length > 0) {
+        const text2 = this.tokenizer.decode(this.token_cache, this.decode_kwargs);
+        const printable_text2 = text2.slice(this.print_len);
+        this.on_finalized_text(printable_text2, false);
+        this.token_cache = [];
+        this.print_len = 0;
+      }
+      const special_text = this.tokenizer.decode(tokens, this.decode_kwargs);
+      this.on_finalized_text(special_text, false);
+      return;
+    }
     this.token_cache = mergeArrays(this.token_cache, tokens);
     const text = this.tokenizer.decode(this.token_cache, this.decode_kwargs);
     let printable_text;