@genai-fi/nanogpt 0.19.1 → 0.20.1

package/dist/DatasetBuilder-DgURD85T.js

@@ -0,0 +1,712 @@
+import { a as e } from "./chunk-BPntVaq0.js";
+import { Mi as t, Ms as n, Ps as r, V as i, Zr as a, bi as o, di as s, fi as c, ji as l, mt as ee, ni as u, wi as d, yi as f } from "./dist-BewPQWjc.js";
+import { n as p } from "./dist-DXwIvKxl.js";
+//#region node_modules/@tensorflow/tfjs-data/dist/util/deep_map.js
+var m = /* @__PURE__ */ e(ee());
+function h(e, t) {
+	return g(e, t);
+}
+function g(e, t, n = /* @__PURE__ */ new Map(), r = /* @__PURE__ */ new Set()) {
+	if (e == null) return null;
+	if (typeof Blob == "function" && e instanceof Blob) return e.slice();
+	if (r.has(e)) throw Error("Circular references are not supported.");
+	if (n.has(e)) return n.get(e);
+	let i = t(e);
+	if (i.recurse && i.value !== null) throw Error("A deep map function may not return both a value and recurse=true.");
+	if (!i.recurse) return n.set(e, i.value), i.value;
+	if (b(e)) {
+		let i = Array.isArray(e) ? [] : {};
+		r.add(e);
+		for (let a in e) {
+			let o = e[a];
+			i[a] = g(o, t, n, r);
+		}
+		return r.delete(e), e.__proto__ && (i.__proto__ = e.__proto__), i;
+	} else throw Error(`Can't recurse into non-iterable type: ${e}`);
+}
+function _(e, t = y) {
+	return v(e, t);
+}
+function v(e, t, n = /* @__PURE__ */ new Set()) {
+	let r = e[0];
+	if (n.has(r)) throw Error("Circular references are not supported.");
+	let i = t(e);
+	if (i.recurse && i.value !== null) throw Error("A deep zip function may not return both a value and recurse=true.");
+	if (!i.recurse) return i.value;
+	if (b(r)) {
+		let i = Array.isArray(r) ? [] : {};
+		n.add(r);
+		for (let a in r) i[a] = v(e.map((e) => e[a]), t, n);
+		return n.delete(r), i;
+	} else throw Error(`Can't recurse into non-iterable type: ${r}`);
+}
+function y(e) {
+	return e === null ? null : b(e[0]) ? {
+		value: null,
+		recurse: !0
+	} : {
+		value: e,
+		recurse: !1
+	};
+}
+function b(e) {
+	let t = !1;
+	if (n().get("IS_BROWSER")) t = e instanceof TextDecoder;
+	else {
+		let { StringDecoder: n } = a();
+		t = e instanceof n;
+	}
+	return e != null && !ArrayBuffer.isView(e) && (Array.isArray(e) || typeof e == "object" && !(e instanceof d) && !(e instanceof Promise) && !t);
+}
+function x(e) {
+	return e == null || S(e) || Array.isArray(e) || typeof e == "object" && e instanceof d || l(e);
+}
+function S(e) {
+	return e === null || typeof e != "object" && typeof e != "function";
+}
+//#endregion
+//#region node_modules/@tensorflow/tfjs-data/dist/util/deep_clone.js
+function C(e) {
+	return h(e, w);
+}
+function w(e) {
+	return e instanceof d ? {
+		value: e.clone(),
+		recurse: !1
+	} : b(e) ? {
+		value: null,
+		recurse: !0
+	} : {
+		value: e,
+		recurse: !1
+	};
+}
+//#endregion
+//#region node_modules/@tensorflow/tfjs-data/dist/util/ring_buffer.js
+var T = class {
+	constructor(e) {
+		if (this.capacity = e, this.begin = 0, this.end = 0, e == null) throw RangeError("Can't create a ring buffer of unknown capacity.");
+		if (e < 1) throw RangeError("Can't create ring buffer of capacity < 1.");
+		this.data = Array(e), this.doubledCapacity = 2 * e;
+	}
+	wrap(e) {
+		for (; e < 0;) e += this.doubledCapacity;
+		return e % this.doubledCapacity;
+	}
+	get(e) {
+		if (e < 0) throw RangeError("Can't get item at a negative index.");
+		return this.data[e % this.capacity];
+	}
+	set(e, t) {
+		if (e < 0) throw RangeError("Can't set item at a negative index.");
+		this.data[e % this.capacity] = t;
+	}
+	length() {
+		let e = this.end - this.begin;
+		return e < 0 && (e = this.doubledCapacity + e), e;
+	}
+	isFull() {
+		return this.length() === this.capacity;
+	}
+	isEmpty() {
+		return this.length() === 0;
+	}
+	push(e) {
+		if (this.isFull()) throw RangeError("Ring buffer is full.");
+		this.set(this.end, e), this.end = this.wrap(this.end + 1);
+	}
+	pushAll(e) {
+		for (let t of e) this.push(t);
+	}
+	pop() {
+		if (this.isEmpty()) throw RangeError("Ring buffer is empty.");
+		this.end = this.wrap(this.end - 1);
+		let e = this.get(this.end);
+		return this.set(this.end, void 0), e;
+	}
+	unshift(e) {
+		if (this.isFull()) throw RangeError("Ring buffer is full.");
+		this.begin = this.wrap(this.begin - 1), this.set(this.begin, e);
+	}
+	shift() {
+		if (this.isEmpty()) throw RangeError("Ring buffer is empty.");
+		let e = this.get(this.begin);
+		return this.set(this.begin, void 0), this.begin = this.wrap(this.begin + 1), e;
+	}
+	shuffleExcise(e) {
+		if (this.isEmpty()) throw RangeError("Ring buffer is empty.");
+		let t = this.wrap(this.begin + e), n = this.get(t);
+		return this.set(t, this.pop()), n;
+	}
+}, E = class e extends T {
+	constructor() {
+		super(e.INITIAL_CAPACITY);
+	}
+	isFull() {
+		return !1;
+	}
+	push(e) {
+		super.isFull() && this.expand(), super.push(e);
+	}
+	unshift(e) {
+		super.isFull() && this.expand(), super.unshift(e);
+	}
+	expand() {
+		let e = this.capacity * 2, t = Array(e), n = this.length();
+		for (let e = 0; e < n; e++) t[e] = this.get(this.wrap(this.begin + e));
+		this.data = t, this.capacity = e, this.doubledCapacity = 2 * this.capacity, this.begin = 0, this.end = n;
+	}
+};
+E.INITIAL_CAPACITY = 32;
+//#endregion
+//#region node_modules/@tensorflow/tfjs-data/dist/iterators/lazy_iterator.js
+function D(e) {
+	return new j(e);
+}
+function O(e) {
+	return new M(e);
+}
+function k(e, t) {
+	return new U(e, t);
+}
+var A = class {
+	async toArray() {
+		let e = [], t = await this.next();
+		for (; !t.done;) e.push(t.value), t = await this.next();
+		return e;
+	}
+	async toArrayForTest() {
+		let e = this.prefetch(100), t = [], n = await e.next();
+		for (; !n.done;) t.push(n.value), n = await e.next();
+		return t;
+	}
+	async resolveFully() {
+		let e = await this.next();
+		for (; !e.done;) e = await this.next();
+	}
+	async resolveWhile(e) {
+		let t = await this.next(), n = e(t.value);
+		for (; !t.done && n;) t = await this.next(), n = e(t.value);
+	}
+	handleErrors(e) {
+		return new z(this, e);
+	}
+	filter(e) {
+		return new L(this, e);
+	}
+	map(e) {
+		return new R(this, e);
+	}
+	mapAsync(e) {
+		return new B(this, e);
+	}
+	serialMapAsync(e) {
+		return new B(this, e).serial();
+	}
+	flatmap(e) {
+		return new H(this, e);
+	}
+	async forEachAsync(e) {
+		return this.map(e).resolveFully();
+	}
+	async serialForEach(e) {
+		return this.serialMapAsync(e).resolveWhile((e) => e === !0);
+	}
+	rowMajorBatch(e, t = !0) {
+		return new I(this, e, t);
+	}
+	columnMajorBatch(e, t = !0, n = y) {
+		return this.rowMajorBatch(e, t).map((e) => _(e, n));
+	}
+	concatenate(e, t) {
+		return new U(D([this, e]), t);
+	}
+	take(e) {
+		return e < 0 || e == null ? this : new F(this, e);
+	}
+	skip(e) {
+		return e < 0 || e == null ? this : new P(this, e);
+	}
+	prefetch(e) {
+		return new G(this, e);
+	}
+	shuffle(e, t) {
+		return new K(this, e, t);
+	}
+	serial() {
+		return new N(this);
+	}
+}, j = class extends A {
+	constructor(e) {
+		super(), this.items = e, this.trav = 0;
+	}
+	summary() {
+		return `Array of ${this.items.length} items`;
+	}
+	async next() {
+		if (this.trav >= this.items.length) return {
+			value: null,
+			done: !0
+		};
+		let e = this.items[this.trav];
+		return this.trav++, {
+			value: C(e),
+			done: !1
+		};
+	}
+}, M = class extends A {
+	constructor(e) {
+		super(), this.nextFn = e;
+	}
+	summary() {
+		return "Function call";
+	}
+	async next() {
+		try {
+			return this.nextFn();
+		} catch (e) {
+			throw e.message = `Error thrown while iterating through a dataset: ${e.message}`, e;
+		}
+	}
+}, N = class extends A {
+	constructor(e) {
+		super(), this.upstream = e, this.lastRead = Promise.resolve({
+			value: null,
+			done: !1
+		});
+	}
+	summary() {
+		return `${this.upstream.summary()} -> Serial`;
+	}
+	async next() {
+		return this.lastRead = this.lastRead.then(() => this.serialNext()), this.lastRead;
+	}
+	async serialNext() {
+		return this.upstream.next();
+	}
+}, P = class extends A {
+	constructor(e, t) {
+		super(), this.upstream = e, this.maxCount = t, this.count = 0, this.lastRead = Promise.resolve({
+			value: null,
+			done: !1
+		});
+	}
+	summary() {
+		return `${this.upstream.summary()} -> Skip`;
+	}
+	async next() {
+		return this.lastRead = this.lastRead.then(() => this.serialNext()), this.lastRead;
+	}
+	async serialNext() {
+		for (; this.count++ < this.maxCount;) {
+			let e = await this.upstream.next();
+			if (e.done) return e;
+			u(e.value);
+		}
+		return this.upstream.next();
+	}
+}, F = class extends A {
+	constructor(e, t) {
+		super(), this.upstream = e, this.maxCount = t, this.count = 0;
+	}
+	summary() {
+		return `${this.upstream.summary()} -> Take`;
+	}
+	async next() {
+		return this.count++ >= this.maxCount ? {
+			value: null,
+			done: !0
+		} : this.upstream.next();
+	}
+}, I = class extends A {
+	constructor(e, t, n = !0) {
+		super(), this.upstream = e, this.batchSize = t, this.enableSmallLastBatch = n, this.lastRead = Promise.resolve({
+			value: null,
+			done: !1
+		});
+	}
+	summary() {
+		return `${this.upstream.summary()} -> RowMajorBatch`;
+	}
+	async next() {
+		return this.lastRead = this.lastRead.then(() => this.serialNext()), this.lastRead;
+	}
+	async serialNext() {
+		let e = [];
+		for (; e.length < this.batchSize;) {
+			let t = await this.upstream.next();
+			if (t.done) return this.enableSmallLastBatch && e.length > 0 ? {
+				value: e,
+				done: !1
+			} : {
+				value: null,
+				done: !0
+			};
+			e.push(t.value);
+		}
+		return {
+			value: e,
+			done: !1
+		};
+	}
+}, L = class extends A {
+	constructor(e, t) {
+		super(), this.upstream = e, this.predicate = t, this.lastRead = Promise.resolve({
+			value: null,
+			done: !1
+		});
+	}
+	summary() {
+		return `${this.upstream.summary()} -> Filter`;
+	}
+	async next() {
+		return this.lastRead = this.lastRead.then(() => this.serialNext()), this.lastRead;
+	}
+	async serialNext() {
+		for (;;) {
+			let e = await this.upstream.next();
+			if (e.done || this.predicate(e.value)) return e;
+			u(e.value);
+		}
+	}
+}, R = class extends A {
+	constructor(e, t) {
+		super(), this.upstream = e, this.transform = t;
+	}
+	summary() {
+		return `${this.upstream.summary()} -> Map`;
+	}
+	async next() {
+		let e = await this.upstream.next();
+		if (e.done) return {
+			value: null,
+			done: !0
+		};
+		let t = f(e.value), n = this.transform(e.value), r = f(n);
+		for (let e of t) o(e, r) || e.dispose();
+		return {
+			value: n,
+			done: !1
+		};
+	}
+}, z = class extends A {
+	constructor(e, t) {
+		super(), this.upstream = e, this.handler = t, this.count = 0, this.lastRead = Promise.resolve({
+			value: null,
+			done: !1
+		});
+	}
+	summary() {
+		return `${this.upstream.summary()} -> handleErrors`;
+	}
+	async next() {
+		return this.lastRead = this.lastRead.then(() => this.serialNext()), this.lastRead;
+	}
+	async serialNext() {
+		for (;;) try {
+			return await this.upstream.next();
+		} catch (e) {
+			if (!this.handler(e)) return {
+				value: null,
+				done: !0
+			};
+		}
+	}
+}, B = class extends A {
+	constructor(e, t) {
+		super(), this.upstream = e, this.transform = t;
+	}
+	summary() {
+		return `${this.upstream.summary()} -> AsyncMap`;
+	}
+	async next() {
+		let e = await this.upstream.next();
+		if (e.done) return {
+			value: null,
+			done: !0
+		};
+		let t = f(e.value), n = await this.transform(e.value), r = f(n);
+		for (let e of t) o(e, r) || e.dispose();
+		return {
+			value: n,
+			done: !1
+		};
+	}
+}, V = class extends A {
+	constructor() {
+		super(), this.outputQueue = new E(), this.lastRead = Promise.resolve({
+			value: null,
+			done: !1
+		});
+	}
+	async next() {
+		return this.lastRead = this.lastRead.then(() => this.serialNext()), this.lastRead;
+	}
+	async serialNext() {
+		for (; this.outputQueue.length() === 0;) if (!await this.pump()) return {
+			value: null,
+			done: !0
+		};
+		return {
+			value: this.outputQueue.shift(),
+			done: !1
+		};
+	}
+}, H = class extends V {
+	constructor(e, t) {
+		super(), this.upstream = e, this.transform = t;
+	}
+	summary() {
+		return `${this.upstream.summary()} -> Flatmap`;
+	}
+	async pump() {
+		let e = await this.upstream.next();
+		if (e.done) return !1;
+		let t = f(e.value), n = this.transform(e.value), r = f(n);
+		this.outputQueue.pushAll(n);
+		for (let e of t) o(e, r) || e.dispose();
+		return !0;
+	}
+}, U = class extends A {
+	constructor(e, t) {
+		super(), this.baseErrorHandler = t, this.lastRead = null, this.iterator = null, this.moreIterators = e;
+	}
+	summary() {
+		return "TODO: fill in upstream of chained summaries -> Chained";
+	}
+	async next() {
+		return this.lastRead = this.readFromChain(this.lastRead), this.lastRead;
+	}
+	async readFromChain(e) {
+		if (await e, this.iterator == null) {
+			let e = await this.moreIterators.next();
+			if (e.done) return {
+				value: null,
+				done: !0
+			};
+			this.iterator = e.value, this.baseErrorHandler != null && (this.iterator = this.iterator.handleErrors(this.baseErrorHandler));
+		}
+		let t = await this.iterator.next();
+		return t.done ? (this.iterator = null, this.readFromChain(e)) : t;
+	}
+}, W;
+(function(e) {
+	e[e.FAIL = 0] = "FAIL", e[e.SHORTEST = 1] = "SHORTEST", e[e.LONGEST = 2] = "LONGEST";
+})(W ||= {});
+var G = class extends A {
+	constructor(e, t) {
+		super(), this.upstream = e, this.bufferSize = t, this.buffer = new T(t);
+	}
+	summary() {
+		return `${this.upstream.summary()} -> Prefetch`;
+	}
+	refill() {
+		for (; !this.buffer.isFull();) {
+			let e = this.upstream.next();
+			this.buffer.push(e);
+		}
+	}
+	next() {
+		return this.refill(), this.buffer.shift();
+	}
+}, K = class extends G {
+	constructor(e, n, r) {
+		super(e, n), this.upstream = e, this.windowSize = n, this.upstreamExhausted = !1, this.random = m.alea(r || t().toString()), this.lastRead = Promise.resolve({
+			value: null,
+			done: !1
+		});
+	}
+	async next() {
+		return this.lastRead = this.lastRead.then(() => this.serialNext()), this.lastRead;
+	}
+	randomInt(e) {
+		return Math.floor(this.random() * e);
+	}
+	chooseIndex() {
+		return this.randomInt(this.buffer.length());
+	}
+	async serialNext() {
+		for (this.upstreamExhausted || this.refill(); !this.buffer.isEmpty();) {
+			let e = this.chooseIndex(), t = await this.buffer.shuffleExcise(e);
+			if (t.done) this.upstreamExhausted = !0;
+			else return this.refill(), t;
+		}
+		return {
+			value: null,
+			done: !0
+		};
+	}
+}, q = class {
+	constructor() {
+		this.size = null;
+	}
+	batch(e, t = !0) {
+		let n = this;
+		r(e > 0, () => `batchSize needs to be positive, but it is
+      ${e}`);
+		let i;
+		return i = this.size === Infinity || this.size == null ? this.size : t ? Math.ceil(this.size / e) : Math.floor(this.size / e), J(async () => (await n.iterator()).columnMajorBatch(e, t, Y), i);
+	}
+	concatenate(e) {
+		let t = this, n;
+		return n = this.size === Infinity || e.size === Infinity ? Infinity : this.size != null && e.size != null ? this.size + e.size : null, J(async () => (await t.iterator()).concatenate(await e.iterator()), n);
+	}
+	filter(e) {
+		let t = this, n;
+		return n = this.size === Infinity ? Infinity : null, J(async () => (await t.iterator()).filter((t) => s(() => e(t))), n);
+	}
+	async forEachAsync(e) {
+		return (await this.iterator()).forEachAsync(e);
+	}
+	map(e) {
+		let t = this;
+		return J(async () => (await t.iterator()).map((t) => s(() => e(t))), this.size);
+	}
+	mapAsync(e) {
+		let t = this;
+		return J(async () => (await t.iterator()).mapAsync(e), this.size);
+	}
+	prefetch(e) {
+		if (e == null) throw RangeError("`Dataset.prefetch()` requires bufferSize to be specified.");
+		let t = this;
+		return J(async () => (await t.iterator()).prefetch(e), this.size);
+	}
+	repeat(e) {
+		let t = this, n;
+		return n = this.size != null && e > 0 ? this.size * e : e === 0 ? 0 : this.size != null && (e === void 0 || e < 0) ? Infinity : null, J(async () => k(O(async () => ({
+			value: await t.iterator(),
+			done: !1
+		})).take(e)), n);
+	}
+	skip(e) {
+		let t = this, n;
+		return n = this.size != null && e >= 0 && this.size >= e ? this.size - e : this.size != null && (this.size < e || e === void 0 || e < 0) ? 0 : null, J(async () => (await t.iterator()).skip(e), n);
+	}
+	shuffle(e, n, r = !0) {
+		if (e == null || e < 0) throw this.size == null ? RangeError("`Dataset.shuffle()` requires bufferSize to be specified.") : RangeError(`\`Dataset.shuffle()\` requires bufferSize to be specified.  If your data fits in main memory (for regular JS objects), and/or GPU memory (for \`tf.Tensor\`s), consider setting bufferSize to the dataset size (${this.size} elements)`);
+		let i = this, a = m.alea(n || t().toString());
+		return J(async () => {
+			let t = a.int32();
+			return r && (t += a.int32()), (await i.iterator()).shuffle(e, t.toString());
+		}, this.size);
+	}
+	take(e) {
+		let t = this, n;
+		return n = this.size != null && this.size > e ? e : this.size != null && this.size <= e ? this.size : null, J(async () => (await t.iterator()).take(e), n);
+	}
+	async toArray() {
+		if (this.size === Infinity) throw Error("Can not convert infinite data stream to array.");
+		return (await this.iterator()).toArray();
+	}
+	async toArrayForTest() {
+		if (this.size === Infinity) throw Error("Can not convert infinite data stream to array.");
+		return (await this.iterator()).toArrayForTest();
+	}
+};
+q.MAX_BUFFER_SIZE = 1e4;
+function J(e, t = null) {
+	return new class extends q {
+		constructor() {
+			super(...arguments), this.size = t;
+		}
+		async iterator() {
+			return e();
+		}
+	}();
+}
+function Y(e) {
+	if (e === null) return null;
+	let t = e[0];
+	return x(t) ? {
+		value: X(e),
+		recurse: !1
+	} : {
+		value: null,
+		recurse: !0
+	};
+}
+function X(e) {
+	if (e.length === 0) throw Error("Can't make a batch of zero elements.");
+	return e[0] instanceof d ? i(e) : c(e);
+}
+//#endregion
+//#region node_modules/@tensorflow/tfjs-data/dist/iterators/string_iterator.js
+p();
+//#endregion
+//#region node_modules/@tensorflow/tfjs-data/dist/readers.js
+function Z(e) {
+	return J(async () => {
+		let t = await e();
+		return O(() => t.next());
+	});
+}
+//#endregion
+//#region lib/training/DatasetBuilder.ts
+var Q = 8;
+function te(e, t) {
+	let n = e.map((e) => t.encodeConversation(e)).flat();
+	return new Uint16Array(n);
+}
+function ne(e, t) {
+	let n = e.map((e) => t.encodeConversation(e, !1, !0));
+	console.log("Tokenised Texts with Mask:", n);
+	let r = n.map((e) => e.tokens).flat(), i = n.map((e) => e.mask).flat();
+	return {
+		tokens: new Uint16Array(r),
+		mask: new Uint8Array(i.map((e) => +!!e))
+	};
+}
+function $(e) {
+	for (let t = e.length - 1; t > 0; t--) {
+		let n = Math.floor(Math.random() * (t + 1));
+		[e[t], e[n]] = [e[n], e[t]];
+	}
+	return e;
+}
+var re = class {
+	tokenizer;
+	blockSize;
+	constructor(e, t = 128) {
+		this.tokenizer = e, this.blockSize = t;
+	}
+	async createTextDataset(e, t = 32, n, r, i = 65535) {
+		if (e.length < this.blockSize + 1) throw Error(`Not enough tokens (${e.length}) for block size ${this.blockSize}`);
+		let a = {
+			shuffledIndexes: new Uint32Array(e.length),
+			step: 0
+		};
+		if (n) a.shuffledIndexes = n;
+		else {
+			a.shuffledIndexes = new Uint32Array(e.length);
+			for (let t = 0; t < e.length; t++) a.shuffledIndexes[t] = t;
+			$(a.shuffledIndexes);
+		}
+		return {
+			dataset: Z(function* () {
+				for (;;) {
+					let t = a.shuffledIndexes[a.step++];
+					if (a.step >= a.shuffledIndexes.length && (a.step = 0, $(a.shuffledIndexes)), t + this.blockSize + 1 > e.length) continue;
+					let n = new Int32Array(e.subarray(t, t + this.blockSize)), o = e.subarray(t + 1, t + this.blockSize + 1), s = new Int32Array(o);
+					if (r) {
+						let e = 0;
+						for (let n = 0; n < s.length; n++) r[t + 1 + n] === 0 && (s[n] = i, e++);
+						if (e === s.length) continue;
+					}
+					yield {
+						xs: n,
+						ys: s
+					};
+				}
+			}.bind(this)).batch(t).map((e) => {
+				let t = e;
+				return s(() => ({
+					xs: t.xs.cast("int32"),
+					ys: t.ys.cast("int32")
+				}));
+			}).prefetch(2),
+			state: a
+		};
+	}
+};
+//#endregion
+export { $ as a, ne as i, Q as n, te as r, re as t };