@genai-fi/nanogpt 0.19.1 → 0.20.1

package/dist/training/tasks/StartSentenceTask.d.ts

@@ -1,4 +1,4 @@
-import { Conversation, ITokeniser } from '../../main';
+import { Conversation, ITokeniser } from '../../../main';
 import { Task } from './Task';
 export default class StartSentenceTask extends Task {
     private rawText;

package/dist/training/tasks/StartSentenceTask.js

@@ -1,49 +1,45 @@
-import { Task as r } from "./Task.js";
-class a extends r {
-  rawText;
-  index = 0;
-  get length() {
-    return this.rawText.length;
-  }
-  constructor(t) {
-    super(), this.rawText = t;
-  }
-  hasMoreConversations() {
-    return this.index < this.rawText.length;
-  }
-  nextConversation() {
-    if (this.index >= this.rawText.length)
-      return null;
-    const t = this.rawText[this.index];
-    return this.index++, this.conversationFromString(t);
-  }
-  nextTokens(t, n) {
-    const e = this.nextConversation();
-    return e ? t.encodeConversation(e, !1, n) : null;
-  }
-  shuffle() {
-    this.index = 0;
-  }
-  conversationFromString(t) {
-    const n = t.indexOf(".");
-    return n === -1 ? [{
-      role: "assistant",
-      content: this.rawText[this.index]
-    }] : [
-      {
-        role: "user",
-        content: t.slice(0, n + 1).trim()
-      },
-      {
-        role: "assistant",
-        content: t.slice(n + 1).trim()
-      }
-    ];
-  }
-  async estimateTokens(t) {
-    return (await t.encodeConversation(this.conversationFromString(this.rawText[0]))).length * this.length;
-  }
-}
-export {
-  a as default
+import { Task as e } from "./Task.js";
+//#region lib/training/tasks/StartSentenceTask.ts
+var t = class extends e {
+	rawText;
+	index = 0;
+	get length() {
+		return this.rawText.length;
+	}
+	constructor(e) {
+		super(), this.rawText = e;
+	}
+	hasMoreConversations() {
+		return this.index < this.rawText.length;
+	}
+	nextConversation() {
+		if (this.index >= this.rawText.length) return null;
+		let e = this.rawText[this.index];
+		return this.index++, this.conversationFromString(e);
+	}
+	nextTokens(e, t) {
+		let n = this.nextConversation();
+		return n ? e.encodeConversation(n, !1, t) : null;
+	}
+	shuffle() {
+		this.index = 0;
+	}
+	conversationFromString(e) {
+		let t = e.indexOf(".");
+		return t === -1 ? [{
+			role: "assistant",
+			content: this.rawText[this.index]
+		}] : [{
+			role: "user",
+			content: e.slice(0, t + 1).trim()
+		}, {
+			role: "assistant",
+			content: e.slice(t + 1).trim()
+		}];
+	}
+	async estimateTokens(e) {
+		return (await e.encodeConversation(this.conversationFromString(this.rawText[0]))).length * this.length;
+	}
 };
+//#endregion
+export { t as default };

package/dist/training/tasks/Task.d.ts

@@ -1,4 +1,4 @@
-import { Conversation, ITokeniser } from '../../main';
+import { Conversation, ITokeniser } from '../../../main';
 export declare abstract class Task {
     abstract get length(): number;
     abstract hasMoreConversations(): boolean;

package/dist/training/tasks/Task.js

@@ -1,68 +1,55 @@
-import { yieldIfNeeded as k } from "../../utilities/yielder.js";
-class w {
+import { yieldIfNeeded as e } from "../../utilities/yielder.js";
+//#region lib/training/tasks/Task.ts
+var t = class {};
+function n(e, t, n, r, i, a) {
+	for (let o of e) {
+		let e = o.nextTokens(n, a ? !0 : void 0);
+		if (e) {
+			let n = Array.isArray(e) ? e : e.tokens;
+			r.total += n.length;
+			let o = t[t.length - 1], s = a ? a[a.length - 1] : null;
+			if (r.offset + n.length > o.length) {
+				let c = o.length - r.offset;
+				o.set(n.slice(0, c), r.offset);
+				let l = new Uint16Array(Math.floor(i * .1) + 100);
+				if (l.set(n.slice(c), 0), t.push(l), a && s && !Array.isArray(e)) {
+					s.set(e.mask.slice(0, c).map((e) => +!!e), r.offset);
+					let t = new Uint8Array(l.length);
+					t.set(e.mask.slice(c).map((e) => +!!e), 0), a.push(t);
+				}
+				r.offset = n.length - c;
+			} else o.set(n, r.offset), s && !Array.isArray(e) && s.set(e.mask.map((e) => +!!e), r.offset), r.offset += n.length;
+		}
+	}
 }
-function m(y, i, u, o, g, n) {
-  for (const a of y) {
-    const e = a.nextTokens(u, n ? !0 : void 0);
-    if (e) {
-      const f = Array.isArray(e) ? e : e.tokens;
-      o.total += f.length;
-      const c = i[i.length - 1], l = n ? n[n.length - 1] : null;
-      if (o.offset + f.length > c.length) {
-        const s = c.length - o.offset;
-        c.set(f.slice(0, s), o.offset);
-        const t = new Uint16Array(Math.floor(g * 0.1) + 100);
-        if (t.set(f.slice(s), 0), i.push(t), n && l && !Array.isArray(e)) {
-          l.set(
-            e.mask.slice(0, s).map((h) => h ? 1 : 0),
-            o.offset
-          );
-          const r = new Uint8Array(t.length);
-          r.set(
-            e.mask.slice(s).map((h) => h ? 1 : 0),
-            0
-          ), n.push(r);
-        }
-        o.offset = f.length - s;
-      } else
-        c.set(f, o.offset), l && !Array.isArray(e) && l.set(
-          e.mask.map((s) => s ? 1 : 0),
-          o.offset
-        ), o.offset += f.length;
-    }
-  }
+async function r(t, r, i, a) {
+	let o = Math.min((await Promise.all(t.map((e) => e.estimateTokens(r)))).reduce((e, t) => e + t, 0), r.vocabSize * 1e4), s = [new Uint16Array(o)], c = a ? [new Uint8Array(o)] : null, l = {
+		offset: 0,
+		total: 0
+	}, u = performance.now();
+	for (; l.offset < o && (n(t, s, r, l, o, c || void 0), !t.every((e) => !e.hasMoreConversations()));) u = await e(u, i, l.total);
+	if (s.length === 1) return c ? {
+		tokens: s[0].subarray(0, l.offset),
+		mask: c[0].subarray(0, l.offset)
+	} : s[0].subarray(0, l.offset);
+	let d = s.reduce((e, t) => e + t.length, 0) - (s[s.length - 1].length - l.offset), f = new Uint16Array(d), p = 0;
+	for (let e = 0; e < s.length; e++) {
+		let t = s[e];
+		e === s.length - 1 ? (f.set(t.subarray(0, l.offset), p), p += l.offset) : (f.set(t, p), p += t.length);
+	}
+	if (c) {
+		let e = new Uint8Array(d);
+		p = 0;
+		for (let t = 0; t < c.length; t++) {
+			let n = c[t];
+			t === c.length - 1 ? (e.set(n.subarray(0, l.offset), p), p += l.offset) : (e.set(n, p), p += n.length);
+		}
+		return {
+			tokens: f,
+			mask: e
+		};
+	}
+	return f;
 }
-async function p(y, i, u, o) {
-  const g = Math.min(
-    (await Promise.all(y.map((t) => t.estimateTokens(i)))).reduce((t, r) => t + r, 0),
-    i.vocabSize * 1e4
-  ), n = [new Uint16Array(g)], a = o ? [new Uint8Array(g)] : null, e = {
-    offset: 0,
-    total: 0
-  };
-  let f = performance.now();
-  for (; e.offset < g && (m(y, n, i, e, g, a || void 0), !y.every((t) => !t.hasMoreConversations())); )
-    f = await k(f, u, e.total);
-  if (n.length === 1)
-    return a ? { tokens: n[0].subarray(0, e.offset), mask: a[0].subarray(0, e.offset) } : n[0].subarray(0, e.offset);
-  const c = n.reduce((t, r) => t + r.length, 0) - (n[n.length - 1].length - e.offset), l = new Uint16Array(c);
-  let s = 0;
-  for (let t = 0; t < n.length; t++) {
-    const r = n[t];
-    t === n.length - 1 ? (l.set(r.subarray(0, e.offset), s), s += e.offset) : (l.set(r, s), s += r.length);
-  }
-  if (a) {
-    const t = new Uint8Array(c);
-    s = 0;
-    for (let r = 0; r < a.length; r++) {
-      const h = a[r];
-      r === a.length - 1 ? (t.set(h.subarray(0, e.offset), s), s += e.offset) : (t.set(h, s), s += h.length);
-    }
-    return { tokens: l, mask: t };
-  }
-  return l;
-}
-export {
-  w as Task,
-  p as tokensFromTasks
-};
+//#endregion
+export { t as Task, r as tokensFromTasks };

package/dist/training/tasks/splitter.js

@@ -1,21 +1,18 @@
-import s from "./ConversationTask.js";
-function f(e, o) {
-  if (o <= 0 || o >= 1)
-    throw new Error("validationSplit must be between 0 and 1");
-  e.forEach((n) => n.shuffle());
-  const r = [], a = [];
-  for (const n of e)
-    for (; n.hasMoreConversations(); ) {
-      const t = n.nextConversation();
-      if (!t)
-        break;
-      Math.random() < o ? a.push(t) : r.push(t);
-    }
-  return {
-    training: new s(r),
-    validation: new s(a)
-  };
+import e from "./ConversationTask.js";
+//#region lib/training/tasks/splitter.ts
+function t(t, n) {
+	if (n <= 0 || n >= 1) throw Error("validationSplit must be between 0 and 1");
+	t.forEach((e) => e.shuffle());
+	let r = [], i = [];
+	for (let e of t) for (; e.hasMoreConversations();) {
+		let t = e.nextConversation();
+		if (!t) break;
+		Math.random() < n ? i.push(t) : r.push(t);
+	}
+	return {
+		training: new e(r),
+		validation: new e(i)
+	};
 }
-export {
-  f as default
-};
+//#endregion
+export { t as default };

package/dist/training/types.d.ts

@@ -1,5 +1,5 @@
-import { TensorStatistics } from '../checks/weights';
-import { LoRAConfig } from '../models/config';
+import { TensorStatistics } from '../../checks/weights';
+import { LoRAConfig } from '../../models/config';
 import { NamedTensorMap, Tensor } from '@tensorflow/tfjs-core';
 export interface Metrics {
     accuracy?: number;

package/dist/training/types.js

@@ -1 +0,0 @@
-

package/dist/training/validation.d.ts

@@ -1,4 +1,4 @@
-import { ITokeniser, Task } from '../main';
+import { ITokeniser, Task } from '../../main';
 import { Tensor } from '@tensorflow/tfjs-core';
 import { Dataset } from '@tensorflow/tfjs-data';
 import { DatasetBuilder, DatasetState } from './DatasetBuilder';

package/dist/training/validation.js

@@ -1,84 +1,2 @@
-import "../index-CUXkjxiT.js";
-import "../random_width-DN5ZtQkM.js";
-import "../zeros_like-CWjDdwr-.js";
-import "../Generator.js";
-import "../index-Cp39cXWe.js";
-import "../dataset-CGGp1z9P.js";
-import "../ops/cpu/attentionMask.js";
-import "../ops/webgl/attentionMask.js";
-import "../ops/grads/attentionMask.js";
-import "../ops/cpu/rope.js";
-import "../ops/webgl/rope.js";
-import "../rope-DcrZM_e6.js";
-import "../ops/cpu/appendCache.js";
-import "../ops/webgl/appendCache.js";
-import "../ops/grads/softmax16.js";
-import "../matMul16-BcVC_E62.js";
-import "../ops/webgl/matMul16.js";
-import "../ops/cpu/matMul16.js";
-import "../pack16-bqltoUlR.js";
-import "../ops/transpose16.js";
-import "../ops/reshape16.js";
-import "../ops/cpu/qkv.js";
-import "../ops/webgl/qkv.js";
-import "../ops/grads/qkv.js";
-import "../ops/cpu/normRMS.js";
-import "../ops/webgl/normRMS.js";
-import "../ops/grads/normRMS.js";
-import "../ops/dropout16.js";
-import "../ops/webgl/dropout16.js";
-import "../ops/grads/add16.js";
-import "../jszip.min-BZhlzntC.js";
-import "../index-DvYrXKkX.js";
-import "../ops/cpu/adamAdjust.js";
-import "../ops/webgl/adamAdjust.js";
-import "../ops/cpu/adamMoments.js";
-import "../ops/webgl/adamMoments.js";
-import "../ops/cpu/gatherSub.js";
-import "../ops/webgl/gatherSub.js";
-import "../ops/cpu/scatterSub.js";
-import "../ops/webgl/scatterSub.js";
-import { PAGE_FACTOR as a, shuffle as k } from "./DatasetBuilder.js";
-import "../papaparse.min-C0cScC2i.js";
-import { tokensFromTasks as y } from "./tasks/Task.js";
-import "../ops/cpu/matMulGelu.js";
-import "../matMulGelu-JNLZqKQp.js";
-import "../ops/grads/matMulGelu.js";
-import "../ops/cpu/gelu.js";
-import "../ops/webgl/gelu.js";
-import "../gelu-B220X1Go.js";
-import "../ops/webgl/log.js";
-import "../checks/normRMS.js";
-import "../checks/normRMSGrad.js";
-async function Mt(n, x, o, l, c = 0.1, d) {
-  const r = n instanceof Uint16Array ? n : await y(n, x, void 0, d), i = r instanceof Uint16Array ? r : r.tokens, f = r instanceof Uint16Array ? void 0 : r.mask, m = /* @__PURE__ */ new Set();
-  if (c > 0) {
-    const t = Math.floor(i.length / (o.blockSize * a)), s = Math.max(1, Math.floor(t * c));
-    for (; m.size < s; ) {
-      const M = Math.floor(Math.random() * t);
-      m.add(M);
-    }
-  }
-  const p = new Uint32Array(
-    i.length - m.size * o.blockSize * a
-  ), e = new Uint32Array(m.size * o.blockSize * a);
-  let h = 0, g = 0;
-  for (let t = 0; t < i.length; t++) {
-    const s = Math.floor(t / (o.blockSize * a));
-    m.has(s) ? g < e.length && (e[g++] = t) : h < p.length && (p[h++] = t);
-  }
-  const { dataset: v, state: w } = await o.createTextDataset(
-    i,
-    l,
-    k(p),
-    f || void 0
-  ), { dataset: z, state: A } = await o.createTextDataset(
-    i,
-    l,
-    k(e)
-  );
-  return { trainDataset: v, validationDataset: z, size: i.length, validationState: A, trainState: w };
-}
-export {
-  Mt as createTrainValidationSplit
-};
+import { o as e } from "../main-CPjeMv0G.js";
+export { e as createTrainValidationSplit };

package/dist/utilities/arrayClose.js

@@ -1,20 +1,16 @@
-function i(r, e) {
-  let t = 0;
-  if ((Array.isArray(r) || r instanceof Float32Array) && (Array.isArray(e) || e instanceof Float32Array)) {
-    if (r.length !== e.length) return Number.POSITIVE_INFINITY;
-    for (let n = 0; n < r.length; ++n)
-      t = Math.max(t, i(r[n], e[n]));
-    return t;
-  } else if (typeof r == "number" && typeof e == "number") {
-    if (isNaN(r) && isNaN(e))
-      return 0;
-    if (!isFinite(r) || !isFinite(e))
-      return r === e ? 0 : Number.POSITIVE_INFINITY;
-    const n = Math.abs(r - e);
-    return t = Math.max(t, n), t;
-  } else
-    return Number.POSITIVE_INFINITY;
+//#region lib/utilities/arrayClose.ts
+function e(t, n) {
+	let r = 0;
+	if ((Array.isArray(t) || t instanceof Float32Array) && (Array.isArray(n) || n instanceof Float32Array)) {
+		if (t.length !== n.length) return Infinity;
+		for (let i = 0; i < t.length; ++i) r = Math.max(r, e(t[i], n[i]));
+		return r;
+	} else if (typeof t == "number" && typeof n == "number") {
+		if (isNaN(t) && isNaN(n)) return 0;
+		if (!isFinite(t) || !isFinite(n)) return t === n ? 0 : Infinity;
+		let e = Math.abs(t - n);
+		return r = Math.max(r, e), r;
+	} else return Infinity;
 }
-export {
-  i as arraysClose
-};
+//#endregion
+export { e as arraysClose };

package/dist/utilities/datasetID.js

@@ -1,21 +1,18 @@
-function h(s) {
-  const c = s.map((n) => String(n.id)).sort();
-  let e = 2166136261, o = 2654435769;
-  const t = (n) => {
-    e ^= n & 255, e = Math.imul(e, 16777619), o ^= n & 255, o = Math.imul(o, 2246822507);
-  }, g = (n) => {
-    const r = n.length >>> 0;
-    t(r & 255), t(r >>> 8 & 255), t(r >>> 16 & 255), t(r >>> 24 & 255);
-    for (let a = 0; a < n.length; a++) {
-      const l = n.charCodeAt(a);
-      t(l & 255), t(l >>> 8 & 255);
-    }
-  }, i = c.length >>> 0;
-  t(i & 255), t(i >>> 8 & 255), t(i >>> 16 & 255), t(i >>> 24 & 255);
-  for (const n of c)
-    g(n);
-  return "dataset__" + (e >>> 0).toString(36) + "_" + (o >>> 0).toString(36);
+//#region lib/utilities/datasetID.ts
+function e(e) {
+	let t = e.map((e) => String(e.id)).sort(), n = 2166136261, r = 2654435769, i = (e) => {
+		n ^= e & 255, n = Math.imul(n, 16777619), r ^= e & 255, r = Math.imul(r, 2246822507);
+	}, a = (e) => {
+		let t = e.length >>> 0;
+		i(t & 255), i(t >>> 8 & 255), i(t >>> 16 & 255), i(t >>> 24 & 255);
+		for (let t = 0; t < e.length; t++) {
+			let n = e.charCodeAt(t);
+			i(n & 255), i(n >>> 8 & 255);
+		}
+	}, o = t.length >>> 0;
+	i(o & 255), i(o >>> 8 & 255), i(o >>> 16 & 255), i(o >>> 24 & 255);
+	for (let e of t) a(e);
+	return "dataset__" + (n >>> 0).toString(36) + "_" + (r >>> 0).toString(36);
 }
-export {
-  h as default
-};
+//#endregion
+export { e as default };

package/dist/utilities/dummy.d.ts

@@ -1,4 +1,4 @@
-import { default as Model, ModelForwardAttributes } from '../models/model';
+import { default as Model, ModelForwardAttributes } from '../../models/model';
 export declare function dummyPassAsync(model: Model<ModelForwardAttributes>): Promise<void>;
 export interface MemoryRequirements {
     perBatch: number;

package/dist/utilities/dummy.js

@@ -1,43 +1,36 @@
-import { b as y, h as I, v as P } from "../index-CUXkjxiT.js";
-import { z as c } from "../zeros-DvZpK8s6.js";
-async function w(s) {
-  const t = c([1, s.config.blockSize], "int32"), n = s.forward({ training: !1 }, t);
-  await n.data(), n.dispose(), t.dispose();
+import { ii as e, kt as t, qt as n, si as r } from "../dist-BewPQWjc.js";
+//#region lib/utilities/dummy.ts
+async function i(e) {
+	let n = t([1, e.config.blockSize], "int32"), r = e.forward({ training: !1 }, n);
+	await r.data(), r.dispose(), n.dispose();
 }
-async function b(s) {
-  const t = y(), n = t.numBytesInGPUAllocated ?? t.numBytesAllocatedInGPU ?? t.numBytes;
-  await w(s);
-  const a = c([1, s.config.blockSize], "int32"), e = {
-    perBatch: 0,
-    tapeSize: 0,
-    gradients: s.getNumParams() * 4
-  };
-  try {
-    const i = () => {
-      const o = s.forward({ training: !0 }, a), u = I().state.activeTape;
-      let p = 0;
-      if (u)
-        for (const g of u)
-          p += g.saved?.reduce((B, z) => B + z.size * 4, 0) || 0;
-      e.tapeSize = p;
-      const l = o.mean();
-      return o.dispose(), l;
-    }, { value: d, grads: m } = P(i), r = y(), f = r.numBytesInGPUAllocated ?? r.numBytesAllocatedInGPU ?? r.numBytes;
-    e.perBatch = f - n - e.gradients, await d.data(), d.dispose();
-    for (const o in m)
-      m[o].dispose();
-    a.dispose();
-  } catch (i) {
-    console.error("Error during dummy training pass:", i), a.dispose();
-  }
-  return e;
+async function a(a) {
+	let o = r(), s = o.numBytesInGPUAllocated ?? o.numBytesAllocatedInGPU ?? o.numBytes;
+	await i(a);
+	let c = t([1, a.config.blockSize], "int32"), l = {
+		perBatch: 0,
+		tapeSize: 0,
+		gradients: a.getNumParams() * 4
+	};
+	try {
+		let { value: t, grads: i } = n(() => {
+			let t = a.forward({ training: !0 }, c), n = e().state.activeTape, r = 0;
+			if (n) for (let e of n) r += e.saved?.reduce((e, t) => e + t.size * 4, 0) || 0;
+			l.tapeSize = r;
+			let i = t.mean();
+			return t.dispose(), i;
+		}), o = r();
+		l.perBatch = (o.numBytesInGPUAllocated ?? o.numBytesAllocatedInGPU ?? o.numBytes) - s - l.gradients, await t.data(), t.dispose();
+		for (let e in i) i[e].dispose();
+		c.dispose();
+	} catch (e) {
+		console.error("Error during dummy training pass:", e), c.dispose();
+	}
+	return l;
 }
-function v(s) {
-  const t = c([1, s.config.blockSize], "int32");
-  s.forward({ training: !1 }, t).dispose(), t.dispose();
+function o(e) {
+	let n = t([1, e.config.blockSize], "int32");
+	e.forward({ training: !1 }, n).dispose(), n.dispose();
 }
-export {
-  v as dummyPass,
-  w as dummyPassAsync,
-  b as dummyPassTrainAsync
-};
+//#endregion
+export { o as dummyPass, i as dummyPassAsync, a as dummyPassTrainAsync };

package/dist/utilities/multinomialCPU.js

@@ -1,13 +1,9 @@
-import "../index-CUXkjxiT.js";
-import { t as e } from "../tensor2d-BnXMKScO.js";
-function f(n, i) {
-  let r = 0;
-  const o = i ?? Math.random();
-  for (let t = 0; t < n.length; t++)
-    if (r += n[t], o < r)
-      return e([[t]], [1, 1], "int32");
-  return e([[n.length - 1]], [1, 1], "int32");
+import { I as e } from "../dist-BewPQWjc.js";
+//#region lib/utilities/multinomialCPU.ts
+function t(t, n) {
+	let r = 0, i = n ?? Math.random();
+	for (let n = 0; n < t.length; n++) if (r += t[n], i < r) return e([[n]], [1, 1], "int32");
+	return e([[t.length - 1]], [1, 1], "int32");
 }
-export {
-  f as default
-};
+//#endregion
+export { t as default };

package/dist/utilities/naming.js

@@ -1 +0,0 @@
-

package/dist/utilities/packed.js

@@ -1,15 +1,13 @@
-import { h as n } from "../index-CUXkjxiT.js";
-function o() {
-  return n().backendName === "webgpu";
+import { ii as e } from "../dist-BewPQWjc.js";
+//#region lib/utilities/packed.ts
+function t() {
+	return e().backendName === "webgpu";
 }
-function r(e) {
-  return e.dtype === "packedF16";
+function n(e) {
+	return e.dtype === "packedF16";
 }
-function a(e) {
-  return r(e);
+function r(e) {
+	return n(e);
 }
-export {
-  r as isPackableTensor,
-  a as isPackedTensor,
-  o as packingSupported
-};
+//#endregion
+export { n as isPackableTensor, r as isPackedTensor, t as packingSupported };

package/dist/utilities/parameters.d.ts

@@ -1,4 +1,4 @@
-import { GPTConfig } from '../models/config';
+import { GPTConfig } from '../../models/config';
 export declare function estimateLayerParameters(config: GPTConfig): number;
 export declare function estimateParameterCount(config: GPTConfig): number;
 export declare function estimateMemoryUsage(config: GPTConfig): number;