@genai-fi/nanogpt 0.10.3 → 0.12.0

package/dist/tokeniser/CharTokeniser.js

@@ -1,77 +1,91 @@
-import { E as k } from "../index-DvYrXKkX.js";
+import k, { SPECIALS as d } from "./BaseTokeniser.js";
 const u = ["<eos>", "<unk>"];
 class b extends k {
   vocabSize = 0;
   eosToken = 0;
+  bosToken = 0;
   unkToken = 0;
   vocab = [];
   cache = /* @__PURE__ */ new Map();
   _trained = !1;
-  constructor(t) {
-    if (super(), Array.isArray(t)) {
-      if (this.vocab = t, this.vocab.length > 0)
-        this.vocabSize = this.vocab.length, this.eosToken = this.vocab.indexOf("<eos>"), this.unkToken = this.vocab.indexOf(""), this.unkToken === -1 && (this.unkToken = this.vocab.indexOf("<unk>")), this.unkToken === -1 && (this.unkToken = this.vocab.indexOf("<pad>")), this.unkToken === -1 && (this.unkToken = this.vocab.indexOf("_")), this.unkToken === -1 && (this.unkToken = this.vocab.indexOf(" ")), this.unkToken === -1 && (this.unkToken = this.eosToken), this.vocab = this.vocab.map((e) => e === "<pad>" ? "" : e), this.vocab.forEach((e, n) => {
-          this.cache.set(e, n);
+  constructor(i) {
+    if (super(), Array.isArray(i)) {
+      if (this.vocab = i, this.vocab.length > 0)
+        this.vocabSize = this.vocab.length, d.forEach((t) => {
+          const e = this.vocab.indexOf(t);
+          e !== -1 && this.addSpecialToken(t, e);
+        }), this.eosToken = this.getSpecialTokenIndex("<eos>"), this.bosToken = this.getSpecialTokenIndex("<bos>") ?? this.eosToken, this.unkToken = this.getSpecialTokenIndex("") ?? -1, this.unkToken === -1 && (this.unkToken = this.vocab.indexOf("<unk>")), this.unkToken === -1 && (this.unkToken = this.vocab.indexOf("<pad>")), this.unkToken === -1 && (this.unkToken = this.vocab.indexOf("_")), this.unkToken === -1 && (this.unkToken = this.vocab.indexOf(" ")), this.unkToken === -1 && (this.unkToken = this.eosToken), this.vocab = this.vocab.map((t) => t === "<pad>" ? "" : t), this.vocab.forEach((t, e) => {
+          this.cache.set(t, e);
         });
       else
         throw new Error("Vocab cannot be empty");
       this._trained = !0;
     } else
-      this.vocabSize = t, this.vocab = new Array(this.vocabSize).fill(""), this.vocab[0] = "<eos>", this.vocab[1] = "", this.eosToken = 0, this.unkToken = 1, this.cache.set("<eos>", 0), this.cache.set("", 1);
+      this.vocabSize = i, this.vocab = new Array(this.vocabSize).fill(""), this.addSpecialTokens(), this.eosToken = this.getSpecialTokenIndex("<eos>"), this.bosToken = this.getSpecialTokenIndex("<bos>") ?? this.eosToken, this.unkToken = this.getSpecialTokenIndex(""), this.vocab.forEach((t, e) => {
+        this.cache.set(t, e);
+      }), this.cache.set("", this.unkToken);
+  }
+  addToken(i, t) {
+    if (this.cache.has(i))
+      return this.cache.get(i);
+    let e;
+    if (t !== void 0 ? e = t : (e = this.vocab.indexOf("", this.unkToken + 1), e === -1 && (e = this.vocabSize)), e >= this.vocabSize)
+      throw new Error("Vocab size exceeded");
+    return this.vocab[e] = i, this.cache.set(i, e), e;
   }
   get trained() {
     return this.vocab.length === this.vocabSize && this._trained;
   }
   destroy() {
   }
-  async train(t) {
-    const e = t.map((i) => i.split("")).flat(), n = new Set(e), s = Array.from(n), h = this.vocab.indexOf("", this.unkToken + 1), o = this.vocabSize - u.length;
+  async train(i) {
+    const t = i.map((n) => n.split("")).flat(), e = new Set(t), s = Array.from(e), h = this.vocab.indexOf("", this.unkToken + 1), o = this.vocabSize - u.length;
     if (h === -1)
       return this.vocabSize;
     if (this._trained = !0, s.length > o) {
-      const i = /* @__PURE__ */ new Map();
-      e.forEach((a) => {
-        i.set(a, (i.get(a) || 0) + 1);
-      }), s.sort((a, r) => (i.get(a) || 0) - (i.get(r) || 0)), s.splice(0, s.length - o);
+      const n = /* @__PURE__ */ new Map();
+      t.forEach((a) => {
+        n.set(a, (n.get(a) || 0) + 1);
+      }), s.sort((a, r) => (n.get(a) || 0) - (n.get(r) || 0)), s.splice(0, s.length - o);
     }
     let c = h;
     if (c !== -1) {
-      const i = new Set(this.vocab);
+      const n = new Set(this.vocab);
       for (const a of s)
-        if (!i.has(a) && (this.vocab[c] = a, i.add(a), c = this.vocab.indexOf("", c + 1), c === -1))
+        if (!n.has(a) && (this.vocab[c] = a, n.add(a), c = this.vocab.indexOf("", c + 1), c === -1))
           break;
     }
-    return this.cache.clear(), this.vocab.forEach((i, a) => {
-      this.cache.set(i, a);
+    return this.cache.clear(), this.vocab.forEach((n, a) => {
+      this.cache.set(n, a);
     }), this.emit("trainStatus", "trained"), this.vocabSize;
   }
-  async tokenise(t, e) {
+  tokenise(i, t) {
     if (!this.trained)
       throw new Error("Tokeniser not trained");
-    return t.map((s) => e ? s.split("").map((h) => this.cache.get(h) ?? this.unkToken) : s.split("").map((h) => {
+    return i.map((s) => t ? s.split("").map((h) => this.cache.get(h) ?? this.unkToken) : s.split("").map((h) => {
       const o = this.cache.get(h);
       return o !== void 0 ? this.vocab[o] : "";
     }));
   }
-  async detokenise(t) {
-    return t.map((n) => n.map((s) => this.vocab[s]).join(""));
+  detokenise(i) {
+    return i.map((e) => Array.from(e).map((s) => this.vocab[s] || "").join(""));
   }
-  async encode(t) {
-    return (await this.tokenise([t], !0))[0];
+  encode(i) {
+    return this.tokenise([i], !0)[0];
   }
-  async decode(t) {
-    return (await this.detokenise([t]))[0];
+  decode(i) {
+    return this.detokenise([i])[0];
   }
   getVocab() {
     return this.vocab;
   }
-  async getMerges() {
+  getMerges() {
     return [];
   }
-  async createTrainingData(t, e = 5) {
-    const n = await this.tokenise(t, !0), s = [], h = [];
-    for (let o = 0; o < n.length - e; o++)
-      s.push(...n[o].slice(0, e)), h.push(n[o + 1][0]);
+  async createTrainingData(i, t = 5) {
+    const e = await this.tokenise(i, !0), s = [], h = [];
+    for (let o = 0; o < e.length - t; o++)
+      s.push(...e[o].slice(0, t)), h.push(e[o + 1][0]);
     return [s, h];
   }
 }

package/dist/tokeniser/bpe.d.ts

@@ -1,6 +1,5 @@
-import { default as EE } from 'eventemitter3';
-import { ITokeniser } from './type';
-export default class BPETokeniser extends EE<'trainStatus'> implements ITokeniser {
+import { default as BaseTokeniser } from './BaseTokeniser';
+export default class BPETokeniser extends BaseTokeniser {
     private targetSize;
     private vocab;
     private vocabIndex;
@@ -8,19 +7,21 @@ export default class BPETokeniser extends EE<'trainStatus'> implements ITokenise
     private pretokenMap;
     constructor(vocabSize: number);
     constructor(vocab: string[], merges?: [string, string][]);
+    addToken(token: string, index?: number): number;
     destroy(): void;
     get trained(): boolean;
     get vocabSize(): number;
     get eosToken(): number;
+    get bosToken(): number;
     get unkToken(): number;
     train(text: string[]): Promise<number>;
     getVocab(): string[];
-    getMerges(): Promise<[string, string][]>;
+    getMerges(): [string, string][];
     private tokeniseWord;
     private tokeniseStrings;
-    tokenise(text: string[], numeric: true): Promise<number[][]>;
-    tokenise(text: string[]): Promise<string[][]>;
-    detokenise(tokens: number[][]): Promise<string[]>;
-    encode(text: string): Promise<number[]>;
-    decode(tokens: number[]): Promise<string>;
+    tokenise(text: string[], numeric: true): number[][];
+    tokenise(text: string[]): string[][];
+    detokenise(tokens: number[][]): string[];
+    encode(text: string): number[];
+    decode(tokens: number[]): string;
 }

package/dist/tokeniser/bpe.js

@@ -1,68 +1,80 @@
 import l from "../utilities/tokenParse.js";
-import { E as f } from "../index-DvYrXKkX.js";
+import d, { SPECIALS as f } from "./BaseTokeniser.js";
 function u(o, e) {
   return `${o}-::-${e}`;
 }
-function k(o) {
+function b(o) {
   const e = /* @__PURE__ */ new Map();
   for (let s = 0; s < o.length; s++) {
     const t = o[s];
-    for (let r = 0; r < t.length - 1; r++) {
-      const n = u(t[r], t[r + 1]), a = e.get(n) || {
-        a: t[r],
-        b: t[r + 1],
+    for (let n = 0; n < t.length - 1; n++) {
+      const r = u(t[n], t[n + 1]), i = e.get(r) || {
+        a: t[n],
+        b: t[n + 1],
         count: 0,
         instances: /* @__PURE__ */ new Set()
       };
-      a.count += 1, a.instances.add(s), e.set(n, a);
+      i.count += 1, i.instances.add(s), e.set(r, i);
     }
   }
   return { pairs: e, tokens: o };
 }
-function h(o, e, s, t, r) {
-  const n = u(e, s);
-  if (o.pairs.has(n)) {
-    const a = o.pairs.get(n);
-    a.count += r, r > 0 ? a.instances.add(t) : a.count <= 0 ? o.pairs.delete(n) : a.instances.delete(t);
+function h(o, e, s, t, n) {
+  const r = u(e, s);
+  if (o.pairs.has(r)) {
+    const i = o.pairs.get(r);
+    i.count += n, n > 0 ? i.instances.add(t) : i.count <= 0 ? o.pairs.delete(r) : i.instances.delete(t);
   } else
-    o.pairs.set(n, { a: e, b: s, count: r, instances: /* @__PURE__ */ new Set([t]) });
+    o.pairs.set(r, { a: e, b: s, count: n, instances: /* @__PURE__ */ new Set([t]) });
 }
-function b(o) {
+function k(o) {
   let e = null, s = 0;
   for (const t of o.pairs.values())
     t.count > s && (s = t.count, e = t);
   return e;
 }
-function d(o, e) {
+function m(o, e) {
   return o.map((s) => {
     const t = [];
-    for (let r = 0; r < s.length; r++)
-      r < s.length - 1 && s[r] === e[0] && s[r + 1] === e[1] ? (t.push(e[0] + e[1]), r++) : t.push(s[r]);
+    for (let n = 0; n < s.length; n++)
+      n < s.length - 1 && s[n] === e[0] && s[n + 1] === e[1] ? (t.push(e[0] + e[1]), n++) : t.push(s[n]);
     return t;
   });
 }
-function m(o, e) {
+function v(o, e) {
   e.instances.forEach((s) => {
-    const t = o.tokens[s], r = [];
-    for (let n = 0; n < t.length; n++)
-      if (n < t.length - 1 && t[n] === e.a && t[n + 1] === e.b) {
-        const a = e.a + e.b;
-        r.push(a), n > 0 && (h(o, t[n - 1], e.a, s, -1), h(o, t[n - 1], a, s, 1)), n++, n < t.length - 1 && (h(o, e.b, t[n + 1], s, -1), h(o, a, t[n + 1], s, 1));
+    const t = o.tokens[s], n = [];
+    for (let r = 0; r < t.length; r++)
+      if (r < t.length - 1 && t[r] === e.a && t[r + 1] === e.b) {
+        const i = e.a + e.b;
+        n.push(i), r > 0 && (h(o, t[r - 1], e.a, s, -1), h(o, t[r - 1], i, s, 1)), r++, r < t.length - 1 && (h(o, e.b, t[r + 1], s, -1), h(o, i, t[r + 1], s, 1));
       } else
-        r.push(t[n]);
-    o.tokens[s] = r;
+        n.push(t[r]);
+    o.tokens[s] = n;
   }), o.pairs.delete(u(e.a, e.b));
 }
-class S extends f {
+class x extends d {
   targetSize;
   vocab = /* @__PURE__ */ new Set();
   vocabIndex = /* @__PURE__ */ new Map();
   merges = [];
   pretokenMap = /* @__PURE__ */ new Map();
   constructor(e, s) {
-    super(), Array.isArray(e) ? (e.forEach((t, r) => {
-      this.vocab.add(t), this.vocabIndex.set(t, r);
-    }), s && (this.merges = s), this.targetSize = e.length) : (this.vocab.add("<eos>"), this.vocab.add(""), this.targetSize = e);
+    super(), Array.isArray(e) ? (e.forEach((t, n) => {
+      this.vocab.add(t), this.vocabIndex.set(t, n);
+    }), s && (this.merges = s), this.targetSize = e.length, f.forEach((t) => {
+      const n = e.indexOf(t);
+      n !== -1 && this.addSpecialToken(t, n);
+    })) : (this.addSpecialTokens(), this.targetSize = e);
+  }
+  addToken(e, s) {
+    if (this.vocab.has(e))
+      return this.vocabIndex.get(e);
+    {
+      this.vocab.add(e);
+      const t = s !== void 0 ? s : this.vocab.size - 1;
+      return this.vocabIndex.set(e, t), t;
+    }
   }
   destroy() {
     this.vocab.clear(), this.vocabIndex.clear(), this.merges = [], this.pretokenMap.clear();
@@ -76,58 +88,61 @@ class S extends f {
   get eosToken() {
     return this.vocabIndex.get("<eos>") ?? 0;
   }
+  get bosToken() {
+    return this.vocabIndex.get("<bos>") ?? 0;
+  }
   get unkToken() {
     return this.vocabIndex.get("") ?? 1;
   }
   async train(e) {
-    const s = e.map((i) => l(i)).flat(1), t = new Set(s);
-    this.vocab = /* @__PURE__ */ new Set(), this.pretokenMap.clear(), this.merges = [], this.vocab.add("<eos>"), this.vocab.add("");
-    const r = Array.from(t), n = r.map((i) => Array.from(i).map((c) => (this.vocab.add(c), c))), a = k(n);
+    const s = e.map((a) => l(a)).flat(1), t = new Set(s);
+    this.vocab = /* @__PURE__ */ new Set(), this.pretokenMap.clear(), this.merges = [], this.addSpecialTokens();
+    const n = Array.from(t), r = n.map((a) => Array.from(a).map((c) => (this.vocab.add(c), c))), i = b(r);
     for (; this.vocab.size < this.targetSize && this.merges.length < this.targetSize; ) {
-      const i = b(a);
-      if (!i)
+      const a = k(i);
+      if (!a)
         break;
-      this.merges.push([i.a, i.b]), this.vocab.add(i.a + i.b), m(a, i);
+      this.merges.push([a.a, a.b]), this.vocab.add(a.a + a.b), v(i, a);
     }
-    r.forEach((i, p) => {
-      const c = n[p];
-      this.pretokenMap.set(i, c);
+    n.forEach((a, p) => {
+      const c = r[p];
+      this.pretokenMap.set(a, c);
     }), this.vocabIndex.clear();
     let g = 0;
-    for (const i of this.vocab.keys())
-      this.vocabIndex.set(i, g++);
+    for (const a of this.vocab.keys())
+      this.vocabIndex.set(a, g++);
     return this.emit("trainStatus", "trained"), this.vocab.size;
   }
   getVocab() {
     return Array.from(this.vocab);
   }
-  async getMerges() {
+  getMerges() {
     return this.merges;
   }
   tokeniseWord(e) {
     let s = Array.from(e);
     return this.merges.forEach((t) => {
-      s = d([s], t)[0];
+      s = m([s], t)[0];
     }), this.pretokenMap.set(e, s), s;
   }
   tokeniseStrings(e) {
-    return e.map((s) => l(s).map((n) => this.pretokenMap.has(n) ? this.pretokenMap.get(n) : this.tokeniseWord(n)).flat(1));
+    return e.map((s) => l(s).map((r) => this.pretokenMap.has(r) ? this.pretokenMap.get(r) : this.tokeniseWord(r)).flat(1));
   }
-  async tokenise(e, s) {
+  tokenise(e, s) {
     const t = this.tokeniseStrings(e);
-    return s ? t.map((r) => r.map((n) => this.vocabIndex.get(n) ?? this.unkToken)) : t.map((r) => r.map((n) => this.vocab.has(n) ? n : ""));
+    return s ? t.map((n) => n.map((r) => this.vocabIndex.get(r) ?? this.unkToken)) : t.map((n) => n.map((r) => this.vocab.has(r) ? r : ""));
   }
-  async detokenise(e) {
+  detokenise(e) {
     const s = this.getVocab();
-    return e.map((r) => r.map((n) => s[n]).join(""));
+    return e.map((n) => n.map((r) => s[r]).join(""));
   }
-  async encode(e) {
-    return (await this.tokenise([e], !0))[0];
+  encode(e) {
+    return this.tokenise([e], !0)[0];
   }
-  async decode(e) {
-    return (await this.detokenise([e]))[0];
+  decode(e) {
+    return this.detokenise([e])[0];
   }
 }
 export {
-  S as default
+  x as default
 };

package/dist/tokeniser/type.d.ts

@@ -1,14 +1,23 @@
 import { default as EE } from 'eventemitter3';
+export type Roles = 'user' | 'assistant' | 'system';
+export interface Conversation {
+    role: Roles;
+    content: string;
+}
 export interface ITokeniser extends EE<'trainStatus'> {
     train(text: string[]): Promise<number>;
-    tokenise(text: string[], numeric?: boolean): Promise<string[][] | number[][]>;
-    detokenise(tokens: string[][] | number[][]): Promise<string[]>;
     getVocab(): string[];
-    getMerges(): Promise<[string, string][]>;
+    getMerges(): [string, string][];
     destroy(): void;
-    encode(text: string): Promise<number[]>;
-    decode(tokens: number[]): Promise<string>;
+    encode(text: string): number[];
+    encodeConversation(conversation: Conversation[], completion?: boolean): number[];
+    encodeSequence(text: string): number[];
+    decode(tokens: number[] | Uint16Array): string;
+    decodeConversation(tokens: number[] | Uint16Array): Conversation[];
     vocabSize: number;
     eosToken: number;
+    bosToken: number;
     trained: boolean;
+    getSpecialTokenIndex(token: string): number | undefined;
+    isSpecialToken(index: number): boolean;
 }

package/dist/training/Adam.js

@@ -1,7 +1,7 @@
 import { adamAdjust as b } from "../ops/adamAdjust.js";
 import { adamMoments as d } from "../ops/adamMoments.js";
-import { O as g, e as h, t as o, d as B } from "../index-DOvlwCh-.js";
-import { z as M } from "../zeros-KnWaWf-X.js";
+import { O as g, e as h, t as o, d as B } from "../index-Duu1Lvvv.js";
+import { z as M } from "../zeros-5YROwwUH.js";
 class R extends g {
   constructor(t, a, e, s, i = null) {
     super(), this.learningRate = t, this.beta1 = a, this.beta2 = e, this.lossScaling = s, this.epsilon = i, this.accBeta1 = a, this.accBeta2 = e, i === null && (this.epsilon = h().backend.epsilon());

package/dist/training/AdamExt.js

@@ -1,4 +1,4 @@
-import { m as r, b as c, c as h, e as o } from "../index-DOvlwCh-.js";
+import { m as r, b as c, c as h, e as o } from "../index-Duu1Lvvv.js";
 import { AdamOptimizer as g } from "./Adam.js";
 class y extends g {
   constructor(t, e, s, i, a) {

package/dist/training/DatasetBuilder.d.ts

@@ -1,14 +1,14 @@
 import { Tensor } from '@tensorflow/tfjs-core';
-import { ITokeniser } from '../tokeniser/type';
+import { Conversation, ITokeniser } from '../tokeniser/type';
 import { Dataset } from '@tensorflow/tfjs-data';
 export declare const PAGE_FACTOR = 8;
-export declare function flattenTokens(textData: string[], tokenizer: ITokeniser): Promise<number[]>;
+export declare function flattenTokens(textData: Conversation[][], tokenizer: ITokeniser): Promise<number[]>;
 export declare class DatasetBuilder {
     tokenizer: ITokeniser;
     blockSize: number;
     private pageSize;
     constructor(tokenizer: ITokeniser, blockSize?: number);
-    createTextDataset(flatTokens: number[], batchSize?: number, masked?: Set<number>, invertMask?: boolean): Promise<Dataset<{
+    createTextDataset(flatTokens: Uint16Array, batchSize?: number, masked?: Set<number>, invertMask?: boolean): Promise<Dataset<{
         xs: Tensor;
         ys: Tensor;
     }>>;

package/dist/training/DatasetBuilder.js

@@ -1,67 +1,63 @@
-import { t as g } from "../index-DOvlwCh-.js";
-import { d as u, i as d } from "../dataset-BcwmTGYc.js";
+import { t as y } from "../index-Duu1Lvvv.js";
+import { d as g, i as z } from "../dataset-CVIJu7Xa.js";
 import "../index-Cp39cXWe.js";
-function z(r) {
-  return u(async () => {
-    const t = await r();
-    return d(() => t.next());
+function b(a) {
+  return g(async () => {
+    const t = await a();
+    return z(() => t.next());
   });
 }
-const S = 8;
-async function y(r, t) {
-  const s = await Promise.all(r.map((e) => t.encode(e))), o = t.eosToken >= 0, a = s.map((e) => o ? [...e, t.eosToken] : e).flat();
-  for (const e of a)
-    if (e < 0 || e >= t.vocabSize)
-      throw new Error(`Invalid token index ${e} found in tokenised data`);
-  return a;
+const f = 8;
+async function w(a, t) {
+  return (await Promise.all(a.map((s) => t.encodeConversation(s)))).flat();
 }
-class w {
+class m {
   tokenizer;
   blockSize;
   pageSize;
-  constructor(t, s = 128) {
-    this.tokenizer = t, this.blockSize = s, this.pageSize = s * S;
+  constructor(t, r = 128) {
+    this.tokenizer = t, this.blockSize = r, this.pageSize = r * f;
   }
   // Create dataset from text files
-  async createTextDataset(t, s = 32, o, a) {
+  async createTextDataset(t, r = 32, i, s) {
     if (t.length < this.blockSize + 1)
       throw new Error(`Not enough tokens (${t.length}) for block size ${this.blockSize}`);
-    if (o && o.size > t.length / this.pageSize / 2)
+    if (i && i.size > t.length / this.pageSize / 2)
       throw new Error("Too many masked pages - would leave insufficient training data");
-    const e = (function* () {
-      if (o && a) {
-        const i = Array.from(o);
+    const l = (function* () {
+      if (i && s) {
+        const e = Array.from(i);
         for (; ; ) {
-          const c = Math.floor(Math.random() * i.length), l = Math.floor(Math.random() * this.pageSize), n = i[c] * this.pageSize + l;
+          const o = Math.floor(Math.random() * e.length), h = Math.floor(Math.random() * this.pageSize), n = e[o] * this.pageSize + h;
           if (n + this.blockSize + 1 > t.length)
             continue;
-          const h = t.slice(n, n + this.blockSize), f = t.slice(n + 1, n + this.blockSize + 1);
-          yield { xs: h, ys: f };
+          const c = new Int32Array(t.subarray(n, n + this.blockSize)), u = new Int32Array(t.subarray(n + 1, n + this.blockSize + 1));
+          yield { xs: c, ys: u };
         }
       } else
         for (; ; ) {
-          const i = Math.floor(Math.random() * (t.length - this.blockSize - 1));
-          if (o) {
-            const n = Math.floor(i / this.pageSize), h = o.has(n);
-            if (h && !a || !h && a)
+          const e = Math.floor(Math.random() * (t.length - this.blockSize - 1));
+          if (i) {
+            const n = Math.floor(e / this.pageSize), c = i.has(n);
+            if (c && !s || !c && s)
               continue;
           }
-          const c = t.slice(i, i + this.blockSize), l = t.slice(i + 1, i + this.blockSize + 1);
-          yield { xs: c, ys: l };
+          const o = new Int32Array(t.subarray(e, e + this.blockSize)), h = new Int32Array(t.subarray(e + 1, e + this.blockSize + 1));
+          yield { xs: o, ys: h };
         }
     }).bind(this);
-    return z(e).batch(s).map((i) => {
-      const c = i;
-      return g(() => ({
-        xs: c.xs.cast("int32"),
-        ys: c.ys.cast("int32")
+    return b(l).batch(r).map((e) => {
+      const o = e;
+      return y(() => ({
+        xs: o.xs.cast("int32"),
+        ys: o.ys.cast("int32")
         // this.tf.oneHot(batchData.ys.cast('int32'), this.tokenizer.vocabSize),
       }));
     }).prefetch(2);
   }
 }
 export {
-  w as DatasetBuilder,
-  S as PAGE_FACTOR,
-  y as flattenTokens
+  m as DatasetBuilder,
+  f as PAGE_FACTOR,
+  w as flattenTokens
 };

package/dist/training/FullTrainer.js

@@ -1,6 +1,6 @@
 import b from "./Trainer.js";
 import L from "./Evaluator.js";
-import { d as w } from "../index-DOvlwCh-.js";
+import { d as w } from "../index-Duu1Lvvv.js";
 import y from "../utilities/profile.js";
 import { createTensorStatistics as D } from "../checks/weights.js";
 const T = {

package/dist/training/Trainer.d.ts

@@ -1,11 +1,12 @@
 import { ITokeniser } from '../tokeniser/type';
 import { DatasetBuilder } from './DatasetBuilder';
 import { default as AdamExt } from './AdamExt';
-import { NamedTensorMap, TensorContainer } from '@tensorflow/tfjs-core/dist/tensor_types';
+import { NamedTensorMap } from '@tensorflow/tfjs-core/dist/tensor_types';
 import { Scalar, Tensor } from '@tensorflow/tfjs-core';
 import { Dataset } from '@tensorflow/tfjs-data';
 import { default as Model, ModelForwardAttributes } from '../models/model';
 import { TensorStatistics } from '../checks/weights';
+import { Task } from './tasks/Task';
 export interface TrainingLogEntry {
     loss: number;
     valLoss?: number;
@@ -93,7 +94,7 @@ export default abstract class GPTTrainer {
         log: TrainingLogEntry;
         progress: TrainingProgress;
     }>;
-    createTrainValidationSplit(textData: string[], batchSize?: number, validationSplit?: number): Promise<{
+    createTrainValidationSplit(tasks: Task[], batchSize?: number, validationSplit?: number): Promise<{
         trainDataset: Dataset<{
             xs: Tensor;
             ys: Tensor;
@@ -102,7 +103,7 @@ export default abstract class GPTTrainer {
             xs: Tensor;
             ys: Tensor;
         }>;
+        size: number;
     }>;
-    createDataset(textData: string[], batchSize?: number): Promise<Dataset<TensorContainer>>;
     dispose(): void;
 }