@genai-fi/nanogpt 0.19.1 → 0.20.1

package/dist/tokeniser/BaseTokeniser.js

@@ -1,124 +1,2 @@
-import { E as d } from "../index-DvYrXKkX.js";
-const u = [
-  "<eos>",
-  "<bos>",
-  "",
-  "<pad>",
-  "<|user_start|>",
-  "<|user_end|>",
-  "<|assistant_start|>",
-  "<|assistant_end|>",
-  "<|system_start|>",
-  "<|system_end|>"
-];
-class T extends d {
-  id = "untrained";
-  datasetID;
-  specialTokens = /* @__PURE__ */ new Map();
-  specialTokenSet = /* @__PURE__ */ new Set();
-  isSpecialToken(t) {
-    return this.specialTokenSet.has(t);
-  }
-  addSpecialTokens() {
-    u.forEach((t, n) => {
-      this.addToken(t, n), this.specialTokens.set(t, n), this.specialTokenSet.add(n);
-    });
-  }
-  addSpecialToken(t, n) {
-    this.specialTokens.set(t, n), this.specialTokenSet.add(n);
-  }
-  generateID() {
-    const t = this.getVocab();
-    let n = 2166136261, e = 2654435769;
-    if (t.length === 0) {
-      this.id = "untrained";
-      return;
-    }
-    for (let a = 0; a < t.length; a++) {
-      const i = t[a];
-      n ^= i.length, n = Math.imul(n, 16777619), e ^= a, e = Math.imul(e, 2246822507);
-      for (let h = 0; h < i.length; h++) {
-        const c = i.charCodeAt(h);
-        n ^= c, n = Math.imul(n, 16777619), e ^= c, e = Math.imul(e, 3266489909);
-      }
-    }
-    const o = (n >>> 0).toString(36), s = (e >>> 0).toString(36);
-    this.id = "tokeniser_" + o + "_" + s;
-  }
-  encodeSequence(t) {
-    const n = this.encode(t);
-    return [this.bosToken, ...n, this.eosToken];
-  }
-  encodeAsSequence(t, n) {
-    const e = t.flatMap((o) => this.encode(o.content));
-    return n ? [this.bosToken, ...e, this.eosToken, this.bosToken] : [this.bosToken, ...e, this.eosToken];
-  }
-  encodeConversation(t, n, e) {
-    const o = [[this.bosToken]];
-    let s;
-    e && (s = [[!1]]);
-    const a = [
-      this.getSpecialTokenIndex("<|user_start|>"),
-      this.getSpecialTokenIndex("<|assistant_start|>"),
-      this.getSpecialTokenIndex("<|system_start|>")
-    ], i = [
-      this.getSpecialTokenIndex("<|user_end|>"),
-      this.getSpecialTokenIndex("<|assistant_end|>"),
-      this.getSpecialTokenIndex("<|system_end|>")
-    ];
-    for (const c of t) {
-      let l = !1;
-      const r = this.encode(c.content);
-      switch (c.role) {
-        case "user":
-          o.push([a[0]]), l = !0;
-          break;
-        case "assistant":
-          o.push([a[1]]);
-          break;
-        case "system":
-          o.push([a[2]]), l = !0;
-          break;
-      }
-      switch (o.push(r), c.role) {
-        case "user":
-          o.push([i[0]]);
-          break;
-        case "assistant":
-          o.push([i[1]]);
-          break;
-        case "system":
-          o.push([i[2]]);
-          break;
-      }
-      e && s && l ? (s.push([!1]), s.push(r.map(() => !1)), s.push([!1])) : e && s && (s.push([!1]), s.push(r.map(() => !0)), s.push([!0]));
-    }
-    const h = o.flat();
-    return n ? (h.push(a[1]), e && s && s.push([!1])) : (h.push(this.eosToken), e && s && s.push([!0])), e && s ? { tokens: h, mask: s.flat() } : h;
-  }
-  decodeConversation(t) {
-    const n = [];
-    let e = 0;
-    for (; e < t.length; ) {
-      const o = t[e];
-      let s = null;
-      if (o === this.getSpecialTokenIndex("<|user_start|>") ? s = "user" : o === this.getSpecialTokenIndex("<|assistant_start|>") ? s = "assistant" : o === this.getSpecialTokenIndex("<|system_start|>") ? s = "system" : o === this.bosToken || (o === this.eosToken ? s = null : (s = "text", e--)), s) {
-        e++;
-        const a = [];
-        for (; e < t.length && t[e] !== this.getSpecialTokenIndex(`<|${s}_end|>`) && t[e] !== this.eosToken; )
-          a.push(t[e]), e++;
-        const i = this.decode(a);
-        n.push({ role: s, content: i });
-      }
-      e++;
-    }
-    return n;
-  }
-  getSpecialTokenIndex(t) {
-    return this.specialTokens.get(t);
-  }
-}
-export {
-  u as SPECIALS,
-  T as default
-};
+import { n as e, t } from "../BaseTokeniser-DSg9zcYq.js";
+export { e as SPECIALS, t as default };

package/dist/tokeniser/CharTokeniser.js

@@ -1,107 +1,92 @@
-import { yieldIfNeeded as u } from "../utilities/yielder.js";
-import b, { SPECIALS as T } from "./BaseTokeniser.js";
-const l = ["<eos>", "<unk>"];
-class x extends b {
-  vocabSize = 0;
-  eosToken = 0;
-  bosToken = 0;
-  unkToken = 0;
-  vocab = [];
-  cache = /* @__PURE__ */ new Map();
-  _trained = !1;
-  constructor(t) {
-    if (super(), Array.isArray(t)) {
-      if (this.vocab = t, this.vocab.length > 0)
-        this.vocabSize = this.vocab.length, T.forEach((i) => {
-          const e = this.vocab.indexOf(i);
-          e !== -1 && this.addSpecialToken(i, e);
-        }), this.eosToken = this.getSpecialTokenIndex("<eos>"), this.bosToken = this.getSpecialTokenIndex("<bos>") ?? this.eosToken, this.unkToken = this.getSpecialTokenIndex("") ?? -1, this.unkToken === -1 && (this.unkToken = this.vocab.indexOf("<unk>")), this.unkToken === -1 && (this.unkToken = this.vocab.indexOf("<pad>")), this.unkToken === -1 && (this.unkToken = this.vocab.indexOf("_")), this.unkToken === -1 && (this.unkToken = this.vocab.indexOf(" ")), this.unkToken === -1 && (this.unkToken = this.eosToken), this.vocab = this.vocab.map((i) => i === "<pad>" ? "" : i), this.vocab.forEach((i, e) => {
-          this.cache.set(i, e);
-        });
-      else
-        throw new Error("Vocab cannot be empty");
-      this._trained = !0;
-    } else
-      this.vocabSize = t, this.vocab = new Array(this.vocabSize).fill(""), this.addSpecialTokens(), this.eosToken = this.getSpecialTokenIndex("<eos>"), this.bosToken = this.getSpecialTokenIndex("<bos>") ?? this.eosToken, this.unkToken = this.getSpecialTokenIndex(""), this.vocab.forEach((i, e) => {
-        this.cache.set(i, e);
-      }), this.cache.set("", this.unkToken);
-  }
-  addToken(t, i) {
-    if (this.cache.has(t))
-      return this.cache.get(t);
-    let e;
-    if (i !== void 0 ? e = i : (e = this.vocab.indexOf("", this.unkToken + 1), e === -1 && (e = this.vocabSize)), e >= this.vocabSize)
-      throw new Error("Vocab size exceeded");
-    return this.vocab[e] = t, this.cache.set(t, e), e;
-  }
-  get trained() {
-    return this.vocab.length === this.vocabSize && this._trained;
-  }
-  destroy() {
-    this.cache.clear(), this.vocab = [];
-  }
-  async train(t, i, e) {
-    this.datasetID = e;
-    const a = /* @__PURE__ */ new Set();
-    let h = performance.now();
-    for (const n of t)
-      n.forEach((o) => {
-        for (const r of o.content)
-          a.add(r);
-      }), h = await u(h, i, 0);
-    const s = Array.from(a), k = this.vocab.indexOf("", this.unkToken + 1), d = this.vocabSize - l.length;
-    if (k === -1)
-      return this.generateID(), this.vocabSize;
-    if (this._trained = !0, s.length > d) {
-      const n = /* @__PURE__ */ new Map();
-      t.forEach((o) => {
-        o.forEach((r) => {
-          for (const f of r.content)
-            n.set(f, (n.get(f) || 0) + 1);
-        });
-      }), s.sort((o, r) => (n.get(o) || 0) - (n.get(r) || 0)), s.splice(0, s.length - d);
-    }
-    let c = k;
-    if (c !== -1) {
-      const n = new Set(this.vocab);
-      for (const o of s)
-        if (!n.has(o) && (this.vocab[c] = o, n.add(o), c = this.vocab.indexOf("", c + 1), c === -1))
-          break;
-    }
-    return this.cache.clear(), this.vocab.forEach((n, o) => {
-      this.cache.set(n, o);
-    }), this.generateID(), this.emit("trainStatus", "trained"), this.vocabSize;
-  }
-  tokenise(t, i) {
-    if (!this.trained)
-      throw new Error("Tokeniser not trained");
-    return t.map((a) => i ? a.split("").map((h) => this.cache.get(h) ?? this.unkToken) : a.split("").map((h) => {
-      const s = this.cache.get(h);
-      return s !== void 0 ? this.vocab[s] : "";
-    }));
-  }
-  detokenise(t) {
-    return t.map((e) => Array.from(e).map((a) => this.vocab[a] || "").join(""));
-  }
-  encode(t) {
-    return this.tokenise([t], !0)[0];
-  }
-  decode(t) {
-    return this.detokenise([t])[0];
-  }
-  getVocab() {
-    return this.vocab;
-  }
-  getMerges() {
-    return [];
-  }
-  async createTrainingData(t, i = 5) {
-    const e = await this.tokenise(t, !0), a = [], h = [];
-    for (let s = 0; s < e.length - i; s++)
-      a.push(...e[s].slice(0, i)), h.push(e[s + 1][0]);
-    return [a, h];
-  }
-}
-export {
-  x as default
+import { yieldIfNeeded as e } from "../utilities/yielder.js";
+import { n as t, t as n } from "../BaseTokeniser-DSg9zcYq.js";
+//#region lib/tokeniser/CharTokeniser.ts
+var r = ["<eos>", "<unk>"], i = class extends n {
+	vocabSize = 0;
+	eosToken = 0;
+	bosToken = 0;
+	unkToken = 0;
+	vocab = [];
+	cache = /* @__PURE__ */ new Map();
+	_trained = !1;
+	constructor(e) {
+		if (super(), Array.isArray(e)) {
+			if (this.vocab = e, this.vocab.length > 0) this.vocabSize = this.vocab.length, t.forEach((e) => {
+				let t = this.vocab.indexOf(e);
+				t !== -1 && this.addSpecialToken(e, t);
+			}), this.eosToken = this.getSpecialTokenIndex("<eos>"), this.bosToken = this.getSpecialTokenIndex("<bos>") ?? this.eosToken, this.unkToken = this.getSpecialTokenIndex("") ?? -1, this.unkToken === -1 && (this.unkToken = this.vocab.indexOf("<unk>")), this.unkToken === -1 && (this.unkToken = this.vocab.indexOf("<pad>")), this.unkToken === -1 && (this.unkToken = this.vocab.indexOf("_")), this.unkToken === -1 && (this.unkToken = this.vocab.indexOf(" ")), this.unkToken === -1 && (this.unkToken = this.eosToken), this.vocab = this.vocab.map((e) => e === "<pad>" ? "" : e), this.vocab.forEach((e, t) => {
+				this.cache.set(e, t);
+			});
+			else throw Error("Vocab cannot be empty");
+			this._trained = !0;
+		} else this.vocabSize = e, this.vocab = Array(this.vocabSize).fill(""), this.addSpecialTokens(), this.eosToken = this.getSpecialTokenIndex("<eos>"), this.bosToken = this.getSpecialTokenIndex("<bos>") ?? this.eosToken, this.unkToken = this.getSpecialTokenIndex(""), this.vocab.forEach((e, t) => {
+			this.cache.set(e, t);
+		}), this.cache.set("", this.unkToken);
+	}
+	addToken(e, t) {
+		if (this.cache.has(e)) return this.cache.get(e);
+		let n;
+		if (t === void 0 ? (n = this.vocab.indexOf("", this.unkToken + 1), n === -1 && (n = this.vocabSize)) : n = t, n >= this.vocabSize) throw Error("Vocab size exceeded");
+		return this.vocab[n] = e, this.cache.set(e, n), n;
+	}
+	get trained() {
+		return this.vocab.length === this.vocabSize && this._trained;
+	}
+	destroy() {
+		this.cache.clear(), this.vocab = [];
+	}
+	async train(t, n, i) {
+		this.datasetID = i;
+		let a = /* @__PURE__ */ new Set(), o = performance.now();
+		for (let r of t) r.forEach((e) => {
+			for (let t of e.content) a.add(t);
+		}), o = await e(o, n, 0);
+		let s = Array.from(a), c = this.vocab.indexOf("", this.unkToken + 1), l = this.vocabSize - r.length;
+		if (c === -1) return this.generateID(), this.vocabSize;
+		if (this._trained = !0, s.length > l) {
+			let e = /* @__PURE__ */ new Map();
+			t.forEach((t) => {
+				t.forEach((t) => {
+					for (let n of t.content) e.set(n, (e.get(n) || 0) + 1);
+				});
+			}), s.sort((t, n) => (e.get(t) || 0) - (e.get(n) || 0)), s.splice(0, s.length - l);
+		}
+		let u = c;
+		if (u !== -1) {
+			let e = new Set(this.vocab);
+			for (let t of s) if (!e.has(t) && (this.vocab[u] = t, e.add(t), u = this.vocab.indexOf("", u + 1), u === -1)) break;
+		}
+		return this.cache.clear(), this.vocab.forEach((e, t) => {
+			this.cache.set(e, t);
+		}), this.generateID(), this.emit("trainStatus", "trained"), this.vocabSize;
+	}
+	tokenise(e, t) {
+		if (!this.trained) throw Error("Tokeniser not trained");
+		return e.map((e) => t ? e.split("").map((e) => this.cache.get(e) ?? this.unkToken) : e.split("").map((e) => {
+			let t = this.cache.get(e);
+			return t === void 0 ? "" : this.vocab[t];
+		}));
+	}
+	detokenise(e) {
+		return e.map((e) => Array.from(e).map((e) => this.vocab[e] || "").join(""));
+	}
+	encode(e) {
+		return this.tokenise([e], !0)[0];
+	}
+	decode(e) {
+		return this.detokenise([e])[0];
+	}
+	getVocab() {
+		return this.vocab;
+	}
+	getMerges() {
+		return [];
+	}
+	async createTrainingData(e, t = 5) {
+		let n = await this.tokenise(e, !0), r = [], i = [];
+		for (let e = 0; e < n.length - t; e++) r.push(...n[e].slice(0, t)), i.push(n[e + 1][0]);
+		return [r, i];
+	}
 };
+//#endregion
+export { i as default };