@genai-fi/nanogpt 0.20.0 → 0.20.2

package/dist/tfjs_backend-BjuQ5FqB.js

@@ -0,0 +1,614 @@
+import { $ as e, An as t, Gr as n, In as r, L as i, Mi as a, Ps as o, Q as s, Ur as c, Wr as l, Wt as u, X as d, Yr as f, Z as p, _i as m, _r as h, an as g, b as _, br as v, cn as y, di as b, er as x, gr as S, ir as C, js as w, mi as T, mr as E, nr as D, oc as O, pi as k, pt as A, qr as j, rr as M, ti as N, tr as P, w as F, xi as I } from "./dist-BewPQWjc.js";
+import { a as ee, i as te, o as ne, r as re } from "./selu_util-zkAx5doH.js";
+//#region node_modules/@tensorflow/tfjs-core/dist/ops/fused/mat_mul.js
+function ie({ a: e, b: t, transposeA: n = !1, transposeB: i = !1, bias: a, activation: s = "linear", preluActivationWeights: c, leakyreluAlpha: l = .2 }) {
+	if (ne(m.state.gradientDepth, s) === !1) {
+		let r = S(e, t, n, i);
+		return a != null && (r = j(r, a)), re(r, s, c, l);
+	}
+	let d = T(e, "a", "fused matMul"), f = T(t, "b", "fused matMul");
+	[d, f] = I(d, f);
+	let p = n ? d.shape[d.rank - 2] : d.shape[d.rank - 1], h = i ? f.shape[f.rank - 1] : f.shape[f.rank - 2], g = n ? d.shape[d.rank - 1] : d.shape[d.rank - 2], _ = i ? f.shape[f.rank - 2] : f.shape[f.rank - 1], y = d.shape.slice(0, -2), b = f.shape.slice(0, -2), x = O(y), C = O(b);
+	o(p === h, () => `Error in fused matMul: inner shapes (${p}) and (${h}) of Tensors with shapes ${d.shape} and ${f.shape} and transposeA=${n} and transposeB=${i} must match.`);
+	let E = r(d.shape.slice(0, -2), f.shape.slice(0, -2)).concat([g, _]), D = n ? v(d, [
+		x,
+		p,
+		g
+	]) : v(d, [
+		x,
+		g,
+		p
+	]), k = i ? v(f, [
+		C,
+		_,
+		h
+	]) : v(f, [
+		C,
+		h,
+		_
+	]), A;
+	a != null && (A = T(a, "bias", "fused matMul"), [A] = I(A, d), r(E, A.shape));
+	let M;
+	c != null && (M = T(c, "prelu weights", "fused matMul"));
+	let N = (e, t) => {
+		let [r, o, c, l] = t, u = ee(v(e, c.shape), c, s), d, f;
+		if (!n && !i ? (d = S(u, o, !1, !0), f = S(r, u, !0, !1)) : !n && i ? (d = S(u, o, !1, !1), f = S(u, r, !0, !1)) : n && !i ? (d = S(o, u, !1, !0), f = S(r, u, !1, !1)) : (d = S(o, u, !0, !0), f = S(u, r, !0, !0)), a != null) {
+			let e = te(l, u);
+			return [
+				d,
+				f,
+				e
+			];
+		} else return [d, f];
+	}, P = {
+		a: D,
+		b: k,
+		bias: A,
+		preluActivationWeights: M
+	}, F = {
+		transposeA: n,
+		transposeB: i,
+		activation: s,
+		leakyreluAlpha: l
+	};
+	return a == null ? u((e, t, n) => {
+		let r = m.runKernel(w, P, F);
+		return n([
+			e,
+			t,
+			r
+		]), {
+			value: v(r, E),
+			gradFunc: N
+		};
+	})(D, k) : u((e, t, n, r) => {
+		let i = m.runKernel(w, P, F);
+		return r([
+			e,
+			t,
+			i,
+			n
+		]), {
+			value: v(i, E),
+			gradFunc: N
+		};
+	})(D, k, A);
+}
+var L = /* @__PURE__ */ k({ fusedMatMul_: ie }), ae = class e extends Error {
+	constructor(t) {
+		super(t), Object.setPrototypeOf(this, e.prototype);
+	}
+}, oe = class e extends Error {
+	constructor(t) {
+		super(t), Object.setPrototypeOf(this, e.prototype);
+	}
+}, R = class e extends Error {
+	constructor(t) {
+		super(t), Object.setPrototypeOf(this, e.prototype);
+	}
+}, z = class e extends Error {
+	constructor(t) {
+		super(t), Object.setPrototypeOf(this, e.prototype);
+	}
+}, se = class e extends Error {
+	constructor(t) {
+		super(t), Object.setPrototypeOf(this, e.prototype);
+	}
+};
+//#endregion
+//#region node_modules/@tensorflow/tfjs-layers/dist/utils/generic_utils.js
+function ce(e, t) {
+	if (Array.isArray(e)) {
+		let n = [];
+		for (let r = 0; r < t; r++) n = n.concat(e);
+		return n;
+	} else {
+		let n = Array(t);
+		return n.fill(e), n;
+	}
+}
+function B(e, t) {
+	if (!e) throw new se(t);
+}
+function le(e, t) {
+	let n = 0;
+	for (let r of e) r === t && n++;
+	return n;
+}
+function ue(e) {
+	return e.length === 1 ? e[0] : e;
+}
+function de(e) {
+	return Array.isArray(e) ? e : [e];
+}
+function fe(e) {
+	let t = e.replace(/(.)([A-Z][a-z0-9]+)/g, "$1_$2").replace(/([a-z])([A-Z])/g, "$1_$2").toLowerCase();
+	return t[0] === "_" ? "private" + t : t;
+}
+function pe(e) {
+	return e.length <= 1 || e.indexOf("_") === -1 ? e : e.replace(/[_]+(\w|$)/g, (e, t) => t.toUpperCase());
+}
+var V = {};
+function me(e) {
+	if (e == null) return null;
+	let t = {};
+	return t.className = e.getClassName(), t.config = e.getConfig(), t;
+}
+function H(e) {
+	if (!(typeof e != "object" || !e)) if (Array.isArray(e)) e.forEach((e) => H(e));
+	else {
+		let t = Object.keys(e);
+		for (let n of t) {
+			let t = e[n];
+			typeof t == "object" && t && (!Array.isArray(t) && t.type === "ndarray" && typeof t.value == "number" ? e[n] = t.value : H(t));
+		}
+	}
+}
+function he(e, t = {}, n = {}, r = "object", i = !1) {
+	if (typeof e == "string") {
+		let i = e, a;
+		if (i in n) a = n[i];
+		else if (i in V) a = V[i];
+		else if (a = t[i], a == null) throw new R(`Unknown ${r}: ${e}. This may be due to one of the following reasons:\n1. The ${r} is defined in Python, in which case it needs to be ported to TensorFlow.js or your JavaScript code.\n2. The custom ${r} is defined in JavaScript, but is not registered properly with tf.serialization.registerClass().`);
+		return a;
+	} else {
+		let a = e;
+		if (a.className == null || a.config == null) throw new R(`${r}: Improper config format: ${JSON.stringify(a)}.\n'className' and 'config' must set.`);
+		let o = a.className, s, c;
+		if (o in n ? [s, c] = n[o] : o in V ? [s, c] = V.className : o in t && ([s, c] = t[o]), s == null) throw new R(`Unknown ${r}: ${o}. This may be due to one of the following reasons:\n1. The ${r} is defined in Python, in which case it needs to be ported to TensorFlow.js or your JavaScript code.\n2. The custom ${r} is defined in JavaScript, but is not registered properly with tf.serialization.registerClass().`);
+		if (c != null) {
+			let e = {};
+			for (let t of Object.keys(V)) e[t] = V[t];
+			for (let t of Object.keys(n)) e[t] = n[t];
+			let t = a.config;
+			t.customObjects = e;
+			let r = Object.assign({}, V);
+			for (let e of Object.keys(n)) V[e] = n[e];
+			H(a.config);
+			let o = c(s, a.config, n, i);
+			return V = Object.assign({}, r), o;
+		} else {
+			let e = Object.assign({}, V);
+			for (let e of Object.keys(n)) V[e] = n[e];
+			let t = new s(a.config);
+			return V = Object.assign({}, e), t;
+		}
+	}
+}
+function ge(e, t) {
+	return e < t ? -1 : +(e > t);
+}
+function _e(e, t) {
+	return -1 * ge(e, t);
+}
+function ve(e) {
+	if (e == null) return e;
+	let t = [];
+	for (let n of e) t.indexOf(n) === -1 && t.push(n);
+	return t;
+}
+function ye(e) {
+	if (e == null) throw new R(`Invalid value in obj: ${JSON.stringify(e)}`);
+	for (let t in e) if (e.hasOwnProperty(t)) return !1;
+	return !0;
+}
+function U(e, t, n) {
+	if (n != null && e.indexOf(n) < 0) throw new R(`${n} is not a valid ${t}.  Valid values are ${e} or null/undefined.`);
+}
+function be(e, t, n = 0, r = Infinity) {
+	return B(n >= 0), B(r >= n), Array.isArray(e) && e.length >= n && e.length <= r && e.every((e) => typeof e === t);
+}
+function W(e, t) {
+	Array.isArray(e) ? (o(e.length > 0, () => `${t} is unexpectedly an empty array.`), e.forEach((e, n) => W(e, `element ${n + 1} of ${t}`))) : o(Number.isInteger(e) && e > 0, () => `Expected ${t} to be a positive integer, but got ${G(e)}.`);
+}
+function G(e) {
+	return e === null ? "null" : Array.isArray(e) ? "[" + e.map((e) => G(e)).join(",") + "]" : typeof e == "string" ? `"${e}"` : `${e}`;
+}
+function xe(e, t, n) {
+	let r = n == null ? a() : n(), i;
+	return (...o) => {
+		let s = n == null ? a() : n();
+		return s - r < t ? i : (r = s, i = e(...o), i);
+	};
+}
+function Se(e) {
+	return e === "relu" ? "relu" : e === "linear" ? "linear" : e === "elu" ? "elu" : null;
+}
+//#endregion
+//#region node_modules/@tensorflow/tfjs-layers/dist/keras_format/common.js
+var Ce = ["channelsFirst", "channelsLast"], we = ["nearest", "bilinear"], Te = [
+	"valid",
+	"same",
+	"causal"
+], Ee = ["max", "avg"], De = [
+	"sum",
+	"mul",
+	"concat",
+	"ave"
+], K = /* @__PURE__ */ new Map();
+function q(e) {
+	U(Ce, "DataFormat", e);
+}
+function Oe(e) {
+	U(we, "InterpolationFormat", e);
+}
+function ke(e) {
+	U(Te, "PaddingMode", e);
+}
+function Ae(e) {
+	U(Ee, "PoolMode", e);
+}
+var J = [], je = "/";
+function Me(e, t) {
+	J.push(e);
+	try {
+		let e = t();
+		return J.pop(), e;
+	} catch (e) {
+		throw J.pop(), e;
+	}
+}
+function Ne() {
+	return J.length === 0 ? "" : J.join(je) + je;
+}
+function Pe(e) {
+	if (!Le(e)) throw Error("Not a valid tensor name: '" + e + "'");
+	return Ne() + e;
+}
+function Fe(e) {
+	if (!Le(e)) throw Error("Not a valid tensor name: '" + e + "'");
+	K.has(e) || K.set(e, 0);
+	let t = K.get(e);
+	if (K.set(e, K.get(e) + 1), t > 0) {
+		let n = `${e}_${t}`;
+		return K.set(n, 1), n;
+	} else return e;
+}
+var Ie = /* @__PURE__ */ new RegExp(/^[A-Za-z0-9][-A-Za-z0-9\._\/]*$/);
+function Le(e) {
+	return !!e.match(Ie);
+}
+//#endregion
+//#region node_modules/@tensorflow/tfjs-layers/dist/utils/math_utils.js
+function Re(e) {
+	return e === parseInt(e.toString(), 10);
+}
+function Y(e, t, n) {
+	t ??= 0, n ??= e.length;
+	let r = 1;
+	for (let i = t; i < n; ++i) r *= e[i];
+	return r;
+}
+function ze(e) {
+	if (e.length === 0) return NaN;
+	let t = Infinity;
+	for (let n = 0; n < e.length; n++) {
+		let r = e[n];
+		r < t && (t = r);
+	}
+	return t;
+}
+function Be(e) {
+	if (e.length === 0) return NaN;
+	let t = -Infinity;
+	for (let n = 0; n < e.length; n++) {
+		let r = e[n];
+		r > t && (t = r);
+	}
+	return t;
+}
+function Ve(e, t) {
+	if (t < e) throw new R(`end (${t}) < begin (${e}) is forbidden.`);
+	let n = [];
+	for (let r = e; r < t; ++r) n.push(r);
+	return n;
+}
+//#endregion
+//#region node_modules/@tensorflow/tfjs-layers/dist/backend/common.js
+var He;
+function Ue() {
+	return He ??= N().epsilon(), He;
+}
+function X() {
+	return "channelsLast";
+}
+//#endregion
+//#region node_modules/@tensorflow/tfjs-layers/dist/backend/tfjs_backend.js
+function We(e, t) {
+	return f(e, t);
+}
+function Ge(e, t = -1) {
+	let n = e.shape.slice();
+	return t < 0 && (t = n.length + t + 1), n.splice(t, 0, 1), v(e, n);
+}
+function Ke(e, t) {
+	return b(() => {
+		if (e.shape.length !== 2) throw new R(`repeat() expects a rank-2 tensor, but received a rank-${e.shape.length} tensor.`);
+		return Qe(Ge(e, 1), [
+			1,
+			t,
+			1
+		]);
+	});
+}
+function qe(e) {
+	return v(e, [Y(e.shape)]);
+}
+function Je(e) {
+	if (e.rank <= 1) throw new R(`batchFlatten requires a minimum rank of 2. Got rank: ${e.rank}.`);
+	return v(e, [e.shape[0], Y(e.shape, 1)]);
+}
+function Z(t, n, r) {
+	return b(() => {
+		switch (t.rank) {
+			case 1: return e(t, n, r);
+			case 2: return s(t, [n, 0], [r, t.shape[1]]);
+			case 3: return p(t, [
+				n,
+				0,
+				0
+			], [
+				r,
+				t.shape[1],
+				t.shape[2]
+			]);
+			case 4: return d(t, [
+				n,
+				0,
+				0,
+				0
+			], [
+				r,
+				t.shape[1],
+				t.shape[2],
+				t.shape[3]
+			]);
+			case 5: return E(t, [
+				n,
+				0,
+				0,
+				0,
+				0
+			], [
+				r,
+				t.shape[1],
+				t.shape[2],
+				t.shape[3],
+				t.shape[4]
+			]);
+			case 6: return E(t, [
+				n,
+				0,
+				0,
+				0,
+				0,
+				0
+			], [
+				r,
+				t.shape[1],
+				t.shape[2],
+				t.shape[3],
+				t.shape[4],
+				t.shape[5]
+			]);
+			default: throw new R(`sliceAlongFirstAxis() received an unsupported tensor rank: ${t.rank}`);
+		}
+	});
+}
+function Q(t, n, r) {
+	return b(() => {
+		switch (t.rank) {
+			case 1: return e(t, n, r);
+			case 2: return s(t, [0, n], [t.shape[0], r]);
+			case 3: return p(t, [
+				0,
+				0,
+				n
+			], [
+				t.shape[0],
+				t.shape[1],
+				r
+			]);
+			case 4: return d(t, [
+				0,
+				0,
+				0,
+				n
+			], [
+				t.shape[0],
+				t.shape[1],
+				t.shape[2],
+				r
+			]);
+			default: throw new R(`sliceAlongLastAxis() received an unsupported tensor rank: ${t.rank}`);
+		}
+	});
+}
+function Ye(t, n, r, i) {
+	return b(() => {
+		switch (t.rank) {
+			case 1: return e(t, n, r);
+			case 2: switch (i) {
+				case 1: return Z(t, n, r);
+				case 2: return Q(t, n, r);
+				default: throw new R(`The axis is not within the rank of the tensor ${i}`);
+			}
+			case 3: switch (i) {
+				case 1: return Z(t, n, r);
+				case 2: return p(t, [
+					0,
+					n,
+					0
+				], [
+					t.shape[0],
+					r,
+					t.shape[2]
+				]);
+				case 3: return Q(t, n, r);
+				default: throw new R(`The axis is not within the rank of the tensor ${i}`);
+			}
+			case 4: switch (i) {
+				case 1: return Z(t, n, r);
+				case 2: return d(t, [
+					0,
+					n,
+					0,
+					0
+				], [
+					t.shape[0],
+					r,
+					t.shape[2],
+					t.shape[3]
+				]);
+				case 3: return d(t, [
+					0,
+					0,
+					n,
+					0
+				], [
+					t.shape[0],
+					t.shape[1],
+					r,
+					t.shape[3]
+				]);
+				case 4: return Q(t, n, r);
+				default: throw new R(`The axis is not within the rank of the tensor ${i}`);
+			}
+			default: throw new R(`sliceAlongLastAxis() received an unsupported tensor rank: ${t.rank}`);
+		}
+	});
+}
+function Xe(e, t = -1) {
+	let n;
+	return t < 0 && (n = e[0].rank, t = n === 0 ? 0 : n), t === e[0].rank && (t = -1), h(e, t);
+}
+function Ze(e, t) {
+	switch (e.rank) {
+		case 1: return M([e, t]);
+		case 2: return D([e, t], 0);
+		case 3: return P([e, t], 0);
+		case 4: return x([e, t], 0);
+		default: throw new R(`concatAlongFirstAxis() received an unsupported tensor rank: ${e.rank}`);
+	}
+}
+function Qe(e, t) {
+	if (Array.isArray(t) || (t = [t]), e.rank !== t.length) throw new R(`The length of input n (${t.length}) does not match the number of dimensions in input x (${e.rank})`);
+	return y(e, t);
+}
+function $e(e, t = 0, n = 1, r, i) {
+	return A(e, t, n, r, i);
+}
+function et(e, t, n, r) {
+	if (e.rank < 2 || t.rank < 2) throw new z(`dot requires both inputs to be rank >= 2 but got x shape = ${e.shape} and y shape = ${t.shape}`);
+	if (t.rank >= 3 && e.shape.slice(-1)[0] !== t.shape.slice(-2)[0]) throw new z(`If rank y >= 3, then the second last dim of y must equal the last dim of x but got x shape = ${e.shape} and  y shape = ${t.shape}`);
+	if (e.rank === 2 && t.rank === 2) return L({
+		a: e,
+		b: t,
+		transposeA: !1,
+		transposeB: !1,
+		bias: r ? $(e.rank, r, X()) : null,
+		activation: n
+	});
+	{
+		let i = e.shape.slice(), a = i.pop();
+		e = v(e, [-1, a]);
+		let o = t.shape.slice(), s = o.pop(), c = o.pop(), l = [...o, s], u = Array.from({ length: t.rank }, (e, n) => n === 0 ? t.rank - 2 : n <= t.rank - 2 ? n - 1 : n);
+		t = v(F(t, u), [c, -1]);
+		let d = [...i, ...l];
+		return v(L({
+			a: e,
+			b: t,
+			transposeA: !1,
+			transposeB: !1,
+			bias: r ? $(e.rank, r, X()) : null,
+			activation: n
+		}), d);
+	}
+}
+function tt(e, t, n) {
+	return b(() => (t = Array.isArray(t) ? i(t, "int32") : f(t, "int32"), g(e, t, n)));
+}
+function nt(e) {
+	return l(e, e);
+}
+function $(e, t, n) {
+	let r = t.shape;
+	if (t.rank !== 1 && t.rank !== e) throw new R(`Unexpected bias dimensions: ${t.rank}; expected it to be 1 or ${e}`);
+	if (e === 5) {
+		if (n === "channelsFirst") return r.length === 1 ? v(t, [
+			1,
+			r[0],
+			1,
+			1,
+			1
+		]) : v(t, [
+			1,
+			r[3],
+			r[0],
+			r[1],
+			r[2]
+		]);
+		if (n === "channelsLast") return r.length === 1 ? v(t, [
+			1,
+			1,
+			1,
+			1,
+			r[0]
+		]) : v(t, [1].concat(r));
+	} else if (e === 4) {
+		if (n === "channelsFirst") return r.length === 1 ? v(t, [
+			1,
+			r[0],
+			1,
+			1
+		]) : v(t, [
+			1,
+			r[2],
+			r[0],
+			r[1]
+		]);
+		if (n === "channelsLast") return r.length === 1 ? v(t, [
+			1,
+			1,
+			1,
+			r[0]
+		]) : v(t, [1].concat(r));
+	} else if (e === 3) {
+		if (n === "channelsFirst") return r.length === 1 ? v(t, [
+			1,
+			r[0],
+			1
+		]) : v(t, [
+			1,
+			r[1],
+			r[0]
+		]);
+		if (n === "channelsLast") return r.length === 1 ? v(t, [
+			1,
+			1,
+			r[0]
+		]) : v(t, [1].concat(r));
+	} else if (e < 3) return t;
+	throw new R(`Unsupported input rank by biasAdd: ${t.rank}`);
+}
+function rt(e, t, n) {
+	return b(() => (n ??= X(), q(n), j(e, $(e.rank, t, n))));
+}
+function it(e, n = 1) {
+	if (n !== 1) throw new z(`Support for alpha values other than 1 (${n}) is not implemented yet.`);
+	return t(e);
+}
+function at(e) {
+	return b(() => n(e, j(c(e), 1)));
+}
+function ot(e, t, n, r) {
+	return b(() => _(e, t, n, r));
+}
+function st(e) {
+	return b(() => C(j(.5, l(.2, e)), 0, 1));
+}
+function ct(e, t, n = !1) {
+	return n ? e() : t();
+}
+//#endregion
+export { ae as $, ke as A, le as B, Y as C, Ve as D, ze as E, De as F, ce as G, he as H, B as I, ue as J, _e as K, W as L, Pe as M, Fe as N, q as O, Me as P, ve as Q, be as R, X as S, Be as T, ye as U, xe as V, Se as W, de as X, pe as Y, fe as Z, Z as _, Xe as a, Qe as b, it as c, tt as d, z as et, st as f, Ye as g, Ke as h, Ze as i, Ae as j, Oe as k, Ge as l, $e as m, rt as n, R as nt, et as o, ct as p, me as q, We as r, ot as s, Je as t, oe as tt, qe as u, at as v, Re as w, Ue as x, nt as y, U as z };

package/dist/tokeniser/BaseTokeniser.d.ts

@@ -0,0 +1,33 @@
+import { Conversation, ITokeniser } from './type';
+import { default as EE } from 'eventemitter3';
+export declare const SPECIALS: string[];
+export default abstract class BaseTokeniser extends EE<'trainStatus'> implements ITokeniser {
+    id: string;
+    datasetID?: string;
+    protected specialTokens: Map<string, number>;
+    protected specialTokenSet: Set<number>;
+    abstract vocabSize: number;
+    abstract eosToken: number;
+    abstract bosToken: number;
+    abstract trained: boolean;
+    abstract addToken(token: string, index?: number): number;
+    isSpecialToken(index: number): boolean;
+    protected addSpecialTokens(): void;
+    protected addSpecialToken(token: string, index: number): void;
+    generateID(): void;
+    abstract train(text: Conversation[][], cb?: (vocab: number) => void, datasetID?: string): Promise<number>;
+    abstract getVocab(): string[];
+    abstract getMerges(): [string, string][];
+    abstract destroy(): void;
+    abstract encode(text: string): number[];
+    encodeSequence(text: string): number[];
+    encodeAsSequence(conversation: Conversation[], completion?: boolean): number[];
+    encodeConversation(conversation: Conversation[], completion?: boolean): number[];
+    encodeConversation(conversation: Conversation[], completion: boolean, masking: boolean): {
+        tokens: number[];
+        mask: boolean[];
+    };
+    abstract decode(tokens: number[]): string;
+    decodeConversation(tokens: number[] | Uint16Array): Conversation[];
+    getSpecialTokenIndex(token: string): number | undefined;
+}

package/dist/tokeniser/BaseTokeniser.js

@@ -0,0 +1,2 @@
+import { n as e, t } from "../BaseTokeniser-DSg9zcYq.js";
+export { e as SPECIALS, t as default };

package/dist/tokeniser/CharTokeniser.d.ts

@@ -0,0 +1,24 @@
+import { default as BaseTokeniser } from './BaseTokeniser';
+import { Conversation } from './type';
+export default class CharTokeniser extends BaseTokeniser {
+    vocabSize: number;
+    eosToken: number;
+    bosToken: number;
+    unkToken: number;
+    vocab: string[];
+    private cache;
+    private _trained;
+    constructor(vocabSizeOrVocab: number | string[]);
+    addToken(token: string, index?: number): number;
+    get trained(): boolean;
+    destroy(): void;
+    train(text: Conversation[][], cb?: (vocab: number) => void, datasetID?: string): Promise<number>;
+    tokenise(text: string[], numeric: true): number[][];
+    tokenise(text: string[]): string[][];
+    detokenise(tokens: (number[] | Uint16Array)[]): string[];
+    encode(text: string): number[];
+    decode(tokens: number[] | Uint16Array): string;
+    getVocab(): string[];
+    getMerges(): [string, string][];
+    createTrainingData(text: string[], windowSize?: number): Promise<[number[], number[]]>;
+}