@genai-fi/nanogpt 0.19.1 → 0.20.1

package/dist/BaseTokeniser-DSg9zcYq.js

@@ -0,0 +1,221 @@
+import { a as e, t } from "./chunk-BPntVaq0.js";
+var n = (/* @__PURE__ */ e((/* @__PURE__ */ t(((e, t) => {
+	var n = Object.prototype.hasOwnProperty, r = "~";
+	function i() {}
+	Object.create && (i.prototype = Object.create(null), new i().__proto__ || (r = !1));
+	function a(e, t, n) {
+		this.fn = e, this.context = t, this.once = n || !1;
+	}
+	function o(e, t, n, i, o) {
+		if (typeof n != "function") throw TypeError("The listener must be a function");
+		var s = new a(n, i || e, o), c = r ? r + t : t;
+		return e._events[c] ? e._events[c].fn ? e._events[c] = [e._events[c], s] : e._events[c].push(s) : (e._events[c] = s, e._eventsCount++), e;
+	}
+	function s(e, t) {
+		--e._eventsCount === 0 ? e._events = new i() : delete e._events[t];
+	}
+	function c() {
+		this._events = new i(), this._eventsCount = 0;
+	}
+	c.prototype.eventNames = function() {
+		var e = [], t, i;
+		if (this._eventsCount === 0) return e;
+		for (i in t = this._events) n.call(t, i) && e.push(r ? i.slice(1) : i);
+		return Object.getOwnPropertySymbols ? e.concat(Object.getOwnPropertySymbols(t)) : e;
+	}, c.prototype.listeners = function(e) {
+		var t = r ? r + e : e, n = this._events[t];
+		if (!n) return [];
+		if (n.fn) return [n.fn];
+		for (var i = 0, a = n.length, o = Array(a); i < a; i++) o[i] = n[i].fn;
+		return o;
+	}, c.prototype.listenerCount = function(e) {
+		var t = r ? r + e : e, n = this._events[t];
+		return n ? n.fn ? 1 : n.length : 0;
+	}, c.prototype.emit = function(e, t, n, i, a, o) {
+		var s = r ? r + e : e;
+		if (!this._events[s]) return !1;
+		var c = this._events[s], l = arguments.length, u, d;
+		if (c.fn) {
+			switch (c.once && this.removeListener(e, c.fn, void 0, !0), l) {
+				case 1: return c.fn.call(c.context), !0;
+				case 2: return c.fn.call(c.context, t), !0;
+				case 3: return c.fn.call(c.context, t, n), !0;
+				case 4: return c.fn.call(c.context, t, n, i), !0;
+				case 5: return c.fn.call(c.context, t, n, i, a), !0;
+				case 6: return c.fn.call(c.context, t, n, i, a, o), !0;
+			}
+			for (d = 1, u = Array(l - 1); d < l; d++) u[d - 1] = arguments[d];
+			c.fn.apply(c.context, u);
+		} else {
+			var f = c.length, p;
+			for (d = 0; d < f; d++) switch (c[d].once && this.removeListener(e, c[d].fn, void 0, !0), l) {
+				case 1:
+					c[d].fn.call(c[d].context);
+					break;
+				case 2:
+					c[d].fn.call(c[d].context, t);
+					break;
+				case 3:
+					c[d].fn.call(c[d].context, t, n);
+					break;
+				case 4:
+					c[d].fn.call(c[d].context, t, n, i);
+					break;
+				default:
+					if (!u) for (p = 1, u = Array(l - 1); p < l; p++) u[p - 1] = arguments[p];
+					c[d].fn.apply(c[d].context, u);
+			}
+		}
+		return !0;
+	}, c.prototype.on = function(e, t, n) {
+		return o(this, e, t, n, !1);
+	}, c.prototype.once = function(e, t, n) {
+		return o(this, e, t, n, !0);
+	}, c.prototype.removeListener = function(e, t, n, i) {
+		var a = r ? r + e : e;
+		if (!this._events[a]) return this;
+		if (!t) return s(this, a), this;
+		var o = this._events[a];
+		if (o.fn) o.fn === t && (!i || o.once) && (!n || o.context === n) && s(this, a);
+		else {
+			for (var c = 0, l = [], u = o.length; c < u; c++) (o[c].fn !== t || i && !o[c].once || n && o[c].context !== n) && l.push(o[c]);
+			l.length ? this._events[a] = l.length === 1 ? l[0] : l : s(this, a);
+		}
+		return this;
+	}, c.prototype.removeAllListeners = function(e) {
+		var t;
+		return e ? (t = r ? r + e : e, this._events[t] && s(this, t)) : (this._events = new i(), this._eventsCount = 0), this;
+	}, c.prototype.off = c.prototype.removeListener, c.prototype.addListener = c.prototype.on, c.prefixed = r, c.EventEmitter = c, t !== void 0 && (t.exports = c);
+})))(), 1)).default, r = [
+	"<eos>",
+	"<bos>",
+	"",
+	"<pad>",
+	"<|user_start|>",
+	"<|user_end|>",
+	"<|assistant_start|>",
+	"<|assistant_end|>",
+	"<|system_start|>",
+	"<|system_end|>"
+], i = class extends n {
+	id = "untrained";
+	datasetID;
+	specialTokens = /* @__PURE__ */ new Map();
+	specialTokenSet = /* @__PURE__ */ new Set();
+	isSpecialToken(e) {
+		return this.specialTokenSet.has(e);
+	}
+	addSpecialTokens() {
+		r.forEach((e, t) => {
+			this.addToken(e, t), this.specialTokens.set(e, t), this.specialTokenSet.add(t);
+		});
+	}
+	addSpecialToken(e, t) {
+		this.specialTokens.set(e, t), this.specialTokenSet.add(t);
+	}
+	generateID() {
+		let e = this.getVocab(), t = 2166136261, n = 2654435769;
+		if (e.length === 0) {
+			this.id = "untrained";
+			return;
+		}
+		for (let r = 0; r < e.length; r++) {
+			let i = e[r];
+			t ^= i.length, t = Math.imul(t, 16777619), n ^= r, n = Math.imul(n, 2246822507);
+			for (let e = 0; e < i.length; e++) {
+				let r = i.charCodeAt(e);
+				t ^= r, t = Math.imul(t, 16777619), n ^= r, n = Math.imul(n, 3266489909);
+			}
+		}
+		let r = (t >>> 0).toString(36), i = (n >>> 0).toString(36);
+		this.id = "tokeniser_" + r + "_" + i;
+	}
+	encodeSequence(e) {
+		let t = this.encode(e);
+		return [
+			this.bosToken,
+			...t,
+			this.eosToken
+		];
+	}
+	encodeAsSequence(e, t) {
+		let n = e.flatMap((e) => this.encode(e.content));
+		return t ? [
+			this.bosToken,
+			...n,
+			this.eosToken,
+			this.bosToken
+		] : [
+			this.bosToken,
+			...n,
+			this.eosToken
+		];
+	}
+	encodeConversation(e, t, n) {
+		let r = [[this.bosToken]], i;
+		n && (i = [[!1]]);
+		let a = [
+			this.getSpecialTokenIndex("<|user_start|>"),
+			this.getSpecialTokenIndex("<|assistant_start|>"),
+			this.getSpecialTokenIndex("<|system_start|>")
+		], o = [
+			this.getSpecialTokenIndex("<|user_end|>"),
+			this.getSpecialTokenIndex("<|assistant_end|>"),
+			this.getSpecialTokenIndex("<|system_end|>")
+		];
+		for (let t of e) {
+			let e = !1, s = this.encode(t.content);
+			switch (t.role) {
+				case "user":
+					r.push([a[0]]), e = !0;
+					break;
+				case "assistant":
+					r.push([a[1]]);
+					break;
+				case "system":
+					r.push([a[2]]), e = !0;
+					break;
+			}
+			switch (r.push(s), t.role) {
+				case "user":
+					r.push([o[0]]);
+					break;
+				case "assistant":
+					r.push([o[1]]);
+					break;
+				case "system":
+					r.push([o[2]]);
+					break;
+			}
+			n && i && e ? (i.push([!1]), i.push(s.map(() => !1)), i.push([!1])) : n && i && (i.push([!1]), i.push(s.map(() => !0)), i.push([!0]));
+		}
+		let s = r.flat();
+		return t ? (s.push(a[1]), n && i && i.push([!1])) : (s.push(this.eosToken), n && i && i.push([!0])), n && i ? {
+			tokens: s,
+			mask: i.flat()
+		} : s;
+	}
+	decodeConversation(e) {
+		let t = [], n = 0;
+		for (; n < e.length;) {
+			let r = e[n], i = null;
+			if (r === this.getSpecialTokenIndex("<|user_start|>") ? i = "user" : r === this.getSpecialTokenIndex("<|assistant_start|>") ? i = "assistant" : r === this.getSpecialTokenIndex("<|system_start|>") ? i = "system" : r === this.bosToken || (r === this.eosToken ? i = null : (i = "text", n--)), i) {
+				n++;
+				let r = [];
+				for (; n < e.length && e[n] !== this.getSpecialTokenIndex(`<|${i}_end|>`) && e[n] !== this.eosToken;) r.push(e[n]), n++;
+				let a = this.decode(r);
+				t.push({
+					role: i,
+					content: a
+				});
+			}
+			n++;
+		}
+		return t;
+	}
+	getSpecialTokenIndex(e) {
+		return this.specialTokens.get(e);
+	}
+};
+//#endregion
+export { r as n, n as r, i as t };