@genai-fi/nanogpt 0.12.0 → 0.12.2

package/dist/Trainer.d.ts

@@ -32,7 +32,7 @@ export default class Trainer extends EE<'start' | 'stop' | 'log'> {
     stop(): void;
     reset(): void;
     getTotalSamples(): number;
-    prepare(tasks: Task[], options?: ITrainerOptions): Promise<void>;
+    prepare(tasks?: Task[] | Uint16Array, options?: ITrainerOptions): Promise<void>;
     train(options?: ITrainerOptions): Promise<void>;
     step(options?: ITrainerOptions): Promise<void>;
     getLog(): TrainingLogEntry[];

package/dist/Trainer.js

@@ -20,7 +20,7 @@ class p extends n {
   getTotalSamples() {
     return this.totalSamples;
   }
-  async prepare(t, e) {
+  async prepare(t = [], e) {
     const { trainDataset: a, validationDataset: s, size: i } = await this.trainer.createTrainValidationSplit(
       t,
       e?.batchSize || 32,

package/dist/main.d.ts

@@ -22,7 +22,7 @@ export type { TrainingProgress, TrainingLogEntry } from './training/Trainer';
 export type { GPTConfig } from './models/config';
 export { estimateParameterCount, estimateMemoryUsage, estimateTrainingMemoryUsage, estimateResources, validateConfig, } from './utilities/parameters';
 export { default as topP } from './utilities/topP';
-export { Task } from './training/tasks/Task';
+export { Task, tokensFromTasks } from './training/tasks/Task';
 export declare const tasks: {
     PretrainingTask: typeof PretrainingTask;
     StartSentenceTask: typeof StartSentenceTask;

package/dist/main.js

@@ -13,7 +13,7 @@ import { default as Co } from "./data/textLoader.js";
 import { default as Eo } from "./models/model.js";
 import { estimateMemoryUsage as Bo, estimateParameterCount as bo, estimateResources as yo, estimateTrainingMemoryUsage as Lo, validateConfig as So } from "./utilities/parameters.js";
 import { default as Ao } from "./utilities/topP.js";
-import { Task as Ro } from "./training/tasks/Task.js";
+import { Task as Go, tokensFromTasks as Ro } from "./training/tasks/Task.js";
 import o from "./training/tasks/PretrainingTask.js";
 import r from "./training/tasks/StartSentenceTask.js";
 import t from "./training/tasks/ConversationTask.js";
@@ -53,14 +53,14 @@ import "./ops/webgl/matMul16.js";
 import "./ops/cpu/matMul16.js";
 import "./ops/transpose16.js";
 import { selectBackend as wo } from "./backend.js";
-import { default as Fo } from "./utilities/performance.js";
+import { default as No } from "./utilities/performance.js";
 import p from "./layers/CausalSelfAttention.js";
 import a from "./layers/MLP.js";
 import i from "./layers/TransformerBlock.js";
 import s from "./layers/RoPECache.js";
-import { default as jo } from "./training/AdamExt.js";
-import { default as zo } from "./checks/index.js";
-import { sentenceEmbeddings as Io, sentenceEmbeddingsTensor as Jo } from "./utilities/sentences.js";
+import { default as qo } from "./training/AdamExt.js";
+import { default as Ho } from "./checks/index.js";
+import { sentenceEmbeddings as Jo, sentenceEmbeddingsTensor as Ko } from "./utilities/sentences.js";
 const to = {
   PretrainingTask: o,
   StartSentenceTask: r,
@@ -75,15 +75,15 @@ const to = {
   RoPECache: s
 };
 export {
-  jo as AdamExt,
+  qo as AdamExt,
   ko as BPETokeniser,
   co as CharTokeniser,
   io as Generator,
   Eo as Model,
   fo as NanoGPT,
-  Ro as Task,
+  Go as Task,
   lo as TeachableLLM,
-  zo as checks,
+  Ho as checks,
   Bo as estimateMemoryUsage,
   bo as estimateParameterCount,
   yo as estimateResources,
@@ -91,11 +91,12 @@ export {
   mo as layers,
   Co as loadTextData,
   eo as ops,
-  Fo as performanceTest,
+  No as performanceTest,
   wo as selectBackend,
-  Io as sentenceEmbeddings,
-  Jo as sentenceEmbeddingsTensor,
+  Jo as sentenceEmbeddings,
+  Ko as sentenceEmbeddingsTensor,
   to as tasks,
+  Ro as tokensFromTasks,
   Ao as topP,
   So as validateConfig,
   go as waitForModel

package/dist/tokeniser/bpe.js

@@ -1,5 +1,5 @@
-import l from "../utilities/tokenParse.js";
-import d, { SPECIALS as f } from "./BaseTokeniser.js";
+import p from "../utilities/tokenParse.js";
+import f, { SPECIALS as g } from "./BaseTokeniser.js";
 function u(o, e) {
   return `${o}-::-${e}`;
 }
@@ -53,7 +53,7 @@ function v(o, e) {
     o.tokens[s] = n;
   }), o.pairs.delete(u(e.a, e.b));
 }
-class x extends d {
+class x extends f {
   targetSize;
   vocab = /* @__PURE__ */ new Set();
   vocabIndex = /* @__PURE__ */ new Map();
@@ -62,7 +62,7 @@ class x extends d {
   constructor(e, s) {
     super(), Array.isArray(e) ? (e.forEach((t, n) => {
       this.vocab.add(t), this.vocabIndex.set(t, n);
-    }), s && (this.merges = s), this.targetSize = e.length, f.forEach((t) => {
+    }), s && (this.merges = s), this.targetSize = e.length, g.forEach((t) => {
       const n = e.indexOf(t);
       n !== -1 && this.addSpecialToken(t, n);
     })) : (this.addSpecialTokens(), this.targetSize = e);
@@ -80,7 +80,7 @@ class x extends d {
     this.vocab.clear(), this.vocabIndex.clear(), this.merges = [], this.pretokenMap.clear();
   }
   get trained() {
-    return this.vocab.size === this.targetSize && this.merges.length > 0;
+    return this.vocab.size > g.length && this.vocab.size <= this.targetSize && this.merges.length > 0;
   }
   get vocabSize() {
     return this.vocab.size;
@@ -95,7 +95,7 @@ class x extends d {
     return this.vocabIndex.get("") ?? 1;
   }
   async train(e) {
-    const s = e.map((a) => l(a)).flat(1), t = new Set(s);
+    const s = e.map((a) => p(a)).flat(1), t = new Set(s);
     this.vocab = /* @__PURE__ */ new Set(), this.pretokenMap.clear(), this.merges = [], this.addSpecialTokens();
     const n = Array.from(t), r = n.map((a) => Array.from(a).map((c) => (this.vocab.add(c), c))), i = b(r);
     for (; this.vocab.size < this.targetSize && this.merges.length < this.targetSize; ) {
@@ -104,13 +104,13 @@ class x extends d {
         break;
       this.merges.push([a.a, a.b]), this.vocab.add(a.a + a.b), v(i, a);
     }
-    n.forEach((a, p) => {
-      const c = r[p];
+    n.forEach((a, l) => {
+      const c = r[l];
       this.pretokenMap.set(a, c);
     }), this.vocabIndex.clear();
-    let g = 0;
+    let d = 0;
     for (const a of this.vocab.keys())
-      this.vocabIndex.set(a, g++);
+      this.vocabIndex.set(a, d++);
     return this.emit("trainStatus", "trained"), this.vocab.size;
   }
   getVocab() {
@@ -126,7 +126,7 @@ class x extends d {
     }), this.pretokenMap.set(e, s), s;
   }
   tokeniseStrings(e) {
-    return e.map((s) => l(s).map((r) => this.pretokenMap.has(r) ? this.pretokenMap.get(r) : this.tokeniseWord(r)).flat(1));
+    return e.map((s) => p(s).map((r) => this.pretokenMap.has(r) ? this.pretokenMap.get(r) : this.tokeniseWord(r)).flat(1));
   }
   tokenise(e, s) {
     const t = this.tokeniseStrings(e);

package/dist/training/Trainer.d.ts

@@ -94,7 +94,7 @@ export default abstract class GPTTrainer {
         log: TrainingLogEntry;
         progress: TrainingProgress;
     }>;
-    createTrainValidationSplit(tasks: Task[], batchSize?: number, validationSplit?: number): Promise<{
+    createTrainValidationSplit(tasks: Task[] | Uint16Array, batchSize?: number, validationSplit?: number): Promise<{
         trainDataset: Dataset<{
             xs: Tensor;
             ys: Tensor;

package/dist/training/Trainer.js

@@ -1,11 +1,11 @@
-import { DatasetBuilder as u, PAGE_FACTOR as f } from "./DatasetBuilder.js";
-import z from "./AdamExt.js";
-import { t as S, v as y, k, d as h, b as p } from "../index-Duu1Lvvv.js";
+import { DatasetBuilder as f, PAGE_FACTOR as u } from "./DatasetBuilder.js";
+import y from "./AdamExt.js";
+import { t as z, v as S, k, d as h, b as p } from "../index-Duu1Lvvv.js";
 import { tokensFromTasks as x } from "./tasks/Task.js";
 import { z as m } from "../zeros-5YROwwUH.js";
 class B {
-  constructor(t, e, s = 1e-3) {
-    this.tokenizer = e, this.model = t, this.lossScaling = t.lossScaling, this.learningRate = s, this.resetOptimizer(), this.datasetBuilder = new u(e, t.config.blockSize);
+  constructor(t, e, i = 1e-3) {
+    this.tokenizer = e, this.model = t, this.lossScaling = t.lossScaling, this.learningRate = i, this.resetOptimizer(), this.datasetBuilder = new f(e, t.config.blockSize);
   }
   model;
   optimizer;
@@ -36,7 +36,7 @@ class B {
   }
   resetOptimizer(t = { learningRateFactor: 1, beta1: 0.9, beta2: 0.99, epsilon: 1e-8 }) {
     this.optimizer && this.optimizer.dispose();
-    const e = new z(
+    const e = new y(
       t.learningRateFactor * this.learningRate,
       t.beta1,
       t.beta2,
@@ -51,11 +51,11 @@ class B {
     );
     this.optimizer = e;
   }
-  trainStep(t, e, s = !1, i = !1) {
-    return S(() => {
+  trainStep(t, e, i = !1, s = !1) {
+    return z(() => {
       this.model.getProfiler()?.startMemory();
-      const { xs: a, ys: l } = e, d = () => {
-        const [o, c] = this.model.forward(
+      const { xs: a, ys: l } = e, c = () => {
+        const [o, d] = this.model.forward(
           {
             training: !0,
             checkpointing: this._gradientCheckpointing,
@@ -65,47 +65,47 @@ class B {
           l
         );
         o.dispose();
-        const g = c.mul(p(this.lossScaling));
-        return c.dispose(), g;
-      }, { value: n, grads: r } = y(d);
-      return s ? this.model.getProfiler()?.endMemory("Training") : (this.optimizer.applyGradients(r), this.model.getProfiler()?.endMemory("Training"), i ? (t.gradients = r, Object.values(r).forEach((o) => k(o))) : h(r)), n.mul(p(1 / this.lossScaling));
+        const g = d.mul(p(this.lossScaling));
+        return d.dispose(), g;
+      }, { value: n, grads: r } = S(c);
+      return i ? this.model.getProfiler()?.endMemory("Training") : (this.optimizer.applyGradients(r), this.model.getProfiler()?.endMemory("Training"), s ? (t.gradients = r, Object.values(r).forEach((o) => k(o))) : h(r)), n.mul(p(1 / this.lossScaling));
     });
   }
   async dummyPass() {
     const t = m([1, this.model.config.blockSize], "int32"), e = m([1, this.model.config.blockSize], "int32");
     try {
-      const s = this.trainStep({}, { xs: t, ys: e }, !0);
-      await s.data(), s.dispose();
-    } catch (s) {
-      console.error("Error during dummy pass:", s);
+      const i = this.trainStep({}, { xs: t, ys: e }, !0);
+      await i.data(), i.dispose();
+    } catch (i) {
+      console.error("Error during dummy pass:", i);
     } finally {
       t.dispose(), e.dispose();
     }
   }
-  trainBatch(t, e, s = !1) {
+  trainBatch(t, e, i = !1) {
     try {
-      const i = this.trainStep(t, e, !1, s);
-      return e.xs.dispose(), e.ys.dispose(), t.step++, t.totalSteps++, i;
-    } catch (i) {
-      throw console.error(`Error processing batch at step ${t.step}:`, i), h(), i;
+      const s = this.trainStep(t, e, !1, i);
+      return e.xs.dispose(), e.ys.dispose(), t.step++, t.totalSteps++, s;
+    } catch (s) {
+      throw console.error(`Error processing batch at step ${t.step}:`, s), h(), s;
     }
   }
-  async createTrainValidationSplit(t, e = 32, s = 0.1) {
-    const i = await x(t, this.tokenizer), a = /* @__PURE__ */ new Set();
-    if (s > 0) {
-      const n = Math.floor(i.length / (this.datasetBuilder.blockSize * f)), r = Math.max(1, Math.floor(n * s));
+  async createTrainValidationSplit(t, e = 32, i = 0.1) {
+    const s = t instanceof Uint16Array ? t : await x(t, this.tokenizer), a = /* @__PURE__ */ new Set();
+    if (i > 0) {
+      const n = Math.floor(s.length / (this.datasetBuilder.blockSize * u)), r = Math.max(1, Math.floor(n * i));
       for (; a.size < r; ) {
         const o = Math.floor(Math.random() * n);
         a.add(o);
       }
     }
-    const l = await this.datasetBuilder.createTextDataset(i, e, a, !1), d = await this.datasetBuilder.createTextDataset(
-      i,
+    const l = await this.datasetBuilder.createTextDataset(s, e, a, !1), c = await this.datasetBuilder.createTextDataset(
+      s,
       e,
       a,
       !0
     );
-    return { trainDataset: l, validationDataset: d, size: i.length };
+    return { trainDataset: l, validationDataset: c, size: s.length };
   }
   dispose() {
     this.optimizer && this.optimizer.dispose();

package/dist/training/tasks/Task.d.ts

@@ -5,4 +5,4 @@ export declare abstract class Task {
     abstract nextConversation(): Conversation[] | null;
     abstract estimateTokens(tokeniser: ITokeniser): Promise<number>;
 }
-export declare function tokensFromTasks(tasks: Task[], tokenizer: ITokeniser): Promise<Uint16Array>;
+export declare function tokensFromTasks(tasks: Task[], tokenizer: ITokeniser, cb?: (tokens: number) => void): Promise<Uint16Array>;

package/dist/training/tasks/Task.js

@@ -1,41 +1,44 @@
-class g {
+class w {
 }
-function h(f, a, l, e, r) {
-  for (let i = 0; i < f.length; i++) {
-    const c = f[i].nextConversation();
+function g(l, i, h, n, t) {
+  for (let o = 0; o < l.length; o++) {
+    const c = l[o].nextConversation();
     if (c) {
-      const o = l.encodeConversation(c), s = a[a.length - 1];
-      if (e.offset + o.length > s.length) {
-        const n = s.length - e.offset;
-        s.set(o.slice(0, n), e.offset);
-        const t = new Uint16Array(Math.floor(r * 0.1) + 100);
-        t.set(o.slice(n), 0), a.push(t), e.offset = o.length - n;
+      const s = h.encodeConversation(c);
+      n.total += s.length;
+      const f = i[i.length - 1];
+      if (n.offset + s.length > f.length) {
+        const a = f.length - n.offset;
+        f.set(s.slice(0, a), n.offset);
+        const e = new Uint16Array(Math.floor(t * 0.1) + 100);
+        e.set(s.slice(a), 0), i.push(e), n.offset = s.length - a;
       } else
-        s.set(o, e.offset), e.offset += o.length;
+        f.set(s, n.offset), n.offset += s.length;
     }
   }
 }
-async function w(f, a) {
-  const l = (await Promise.all(f.map((n) => n.estimateTokens(a)))).reduce(
-    (n, t) => n + t,
+async function m(l, i, h) {
+  const n = (await Promise.all(l.map((e) => e.estimateTokens(i)))).reduce(
+    (e, r) => e + r,
     0
-  ), e = [new Uint16Array(l)], r = {
-    offset: 0
+  ), t = [new Uint16Array(n)], o = {
+    offset: 0,
+    total: 0
   };
-  let i = performance.now();
-  for (; r.offset < l && (h(f, e, a, r, l), !f.every((t) => !t.hasMoreConversations())); )
-    performance.now() - i > 40 && (await new Promise(requestAnimationFrame), i = performance.now());
-  if (e.length === 1)
-    return e[0].subarray(0, r.offset);
-  const c = e.reduce((n, t) => n + t.length, 0) - (e[e.length - 1].length - r.offset), o = new Uint16Array(c);
-  let s = 0;
-  for (let n = 0; n < e.length; n++) {
-    const t = e[n];
-    n === e.length - 1 ? (o.set(t.subarray(0, r.offset), s), s += r.offset) : (o.set(t, s), s += t.length);
+  let c = performance.now();
+  for (; o.offset < n && (g(l, t, i, o, n), !l.every((r) => !r.hasMoreConversations())); )
+    performance.now() - c > 40 && (await new Promise(requestAnimationFrame), c = performance.now()), h && h(o.total);
+  if (t.length === 1)
+    return t[0].subarray(0, o.offset);
+  const s = t.reduce((e, r) => e + r.length, 0) - (t[t.length - 1].length - o.offset), f = new Uint16Array(s);
+  let a = 0;
+  for (let e = 0; e < t.length; e++) {
+    const r = t[e];
+    e === t.length - 1 ? (f.set(r.subarray(0, o.offset), a), a += o.offset) : (f.set(r, a), a += r.length);
   }
-  return o;
+  return f;
 }
 export {
-  g as Task,
-  w as tokensFromTasks
+  w as Task,
+  m as tokensFromTasks
 };

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@genai-fi/nanogpt",
-    "version": "0.12.0",
+    "version": "0.12.2",
     "type": "module",
     "main": "dist/main.js",
     "types": "dist/main.d.ts",