@genai-fi/nanogpt 0.10.2 → 0.11.0

package/dist/tile-CvN_LyVr.js

@@ -0,0 +1,11 @@
+import { q as e, u as a, y as i, E as c, T as u } from "./index-D0RBWjq8.js";
+function l(r, t) {
+  const n = a(r, "x", "tile", "string_or_numeric");
+  i(n.rank === t.length, () => `Error in transpose: rank of input ${n.rank} must match length of reps ${t}.`);
+  const s = { x: n }, o = { reps: t };
+  return c.runKernel(u, s, o);
+}
+const p = /* @__PURE__ */ e({ tile_: l });
+export {
+  p as t
+};

package/dist/tokeniser/BaseTokeniser.d.ts

@@ -0,0 +1,27 @@
+import { Conversation, ITokeniser } from './type';
+import { default as EE } from 'eventemitter3';
+export declare const SPECIALS: string[];
+export default abstract class BaseTokeniser extends EE<'trainStatus'> implements ITokeniser {
+    protected specialTokens: Map<string, number>;
+    protected specialTokenSet: Set<number>;
+    abstract vocabSize: number;
+    abstract eosToken: number;
+    abstract bosToken: number;
+    abstract trained: boolean;
+    abstract addToken(token: string, index?: number): number;
+    isSpecialToken(index: number): boolean;
+    protected addSpecialTokens(): void;
+    protected addSpecialToken(token: string, index: number): void;
+    abstract train(text: string[]): Promise<number>;
+    abstract tokenise(text: string[], numeric?: boolean): Promise<string[][] | number[][]>;
+    abstract detokenise(tokens: string[][] | number[][]): Promise<string[]>;
+    abstract getVocab(): string[];
+    abstract getMerges(): Promise<[string, string][]>;
+    abstract destroy(): void;
+    abstract encode(text: string): Promise<number[]>;
+    encodeSequence(text: string): Promise<number[]>;
+    encodeConversation(conversation: Conversation[], completion?: boolean): Promise<number[]>;
+    abstract decode(tokens: number[]): Promise<string>;
+    decodeConversation(tokens: number[]): Promise<Conversation[]>;
+    getSpecialTokenIndex(token: string): number | undefined;
+}

package/dist/tokeniser/BaseTokeniser.js

@@ -0,0 +1,94 @@
+import { E as r } from "../index-DvYrXKkX.js";
+const h = [
+  "<eos>",
+  "<bos>",
+  "",
+  "<|user_start|>",
+  "<|user_end|>",
+  "<|assistant_start|>",
+  "<|assistant_end|>",
+  "<|system_start|>",
+  "<|system_end|>"
+];
+class k extends r {
+  specialTokens = /* @__PURE__ */ new Map();
+  specialTokenSet = /* @__PURE__ */ new Set();
+  isSpecialToken(e) {
+    return this.specialTokenSet.has(e);
+  }
+  addSpecialTokens() {
+    h.forEach((e, t) => {
+      this.addToken(e, t), this.specialTokens.set(e, t), this.specialTokenSet.add(t);
+    });
+  }
+  addSpecialToken(e, t) {
+    this.specialTokens.set(e, t), this.specialTokenSet.add(t);
+  }
+  async encodeSequence(e) {
+    const t = await this.encode(e);
+    return [this.bosToken, ...t, this.eosToken];
+  }
+  async encodeConversation(e, t) {
+    const s = [[this.bosToken]], a = [
+      this.getSpecialTokenIndex("<|user_start|>"),
+      this.getSpecialTokenIndex("<|assistant_start|>"),
+      this.getSpecialTokenIndex("<|system_start|>")
+    ], n = [
+      this.getSpecialTokenIndex("<|user_end|>"),
+      this.getSpecialTokenIndex("<|assistant_end|>"),
+      this.getSpecialTokenIndex("<|system_end|>")
+    ];
+    for (const i of e) {
+      const c = await this.encode(i.content);
+      switch (i.role) {
+        case "user":
+          s.push([a[0]]);
+          break;
+        case "assistant":
+          s.push([a[1]]);
+          break;
+        case "system":
+          s.push([a[2]]);
+          break;
+      }
+      switch (s.push(c), i.role) {
+        case "user":
+          s.push([n[0]]);
+          break;
+        case "assistant":
+          s.push([n[1]]);
+          break;
+        case "system":
+          s.push([n[2]]);
+          break;
+      }
+    }
+    const o = s.flat();
+    return t ? o.push(a[1]) : o.push(this.eosToken), o;
+  }
+  async decodeConversation(e) {
+    const t = [];
+    let s = 0;
+    for (; s < e.length; ) {
+      const a = e[s];
+      let n = null;
+      if (a === this.getSpecialTokenIndex("<|user_start|>") ? n = "user" : a === this.getSpecialTokenIndex("<|assistant_start|>") ? n = "assistant" : a === this.getSpecialTokenIndex("<|system_start|>") && (n = "system"), n) {
+        s++;
+        const o = [];
+        for (; s < e.length && e[s] !== this.getSpecialTokenIndex(`<|${n}_end|>`); )
+          o.push(e[s]), s++;
+        const i = await this.decode(o);
+        t.push({ role: n, content: i });
+      }
+      s++;
+    }
+    return t;
+  }
+  getSpecialTokenIndex(e) {
+    return this.specialTokens.get(e);
+  }
+}
+export {
+  h as SPECIALS,
+  k as default
+};

package/dist/tokeniser/CharTokeniser.d.ts

@@ -1,14 +1,15 @@
-import { default as EE } from 'eventemitter3';
-import { ITokeniser } from './type';
-export default class CharTokeniser extends EE<'trainStatus'> implements ITokeniser {
+import { default as BaseTokeniser } from './BaseTokeniser';
+export default class CharTokeniser extends BaseTokeniser {
     vocabSize: number;
     eosToken: number;
+    bosToken: number;
     unkToken: number;
     vocab: string[];
     private cache;
     private _trained;
     constructor(vocabSize: number);
     constructor(vocab: string[]);
+    addToken(token: string, index?: number): number;
     get trained(): boolean;
     destroy(): void;
     train(text: string[]): Promise<number>;

package/dist/tokeniser/CharTokeniser.js

@@ -1,66 +1,80 @@
-import { E as k } from "../index-DvYrXKkX.js";
+import k, { SPECIALS as d } from "./BaseTokeniser.js";
 const u = ["<eos>", "<unk>"];
 class b extends k {
   vocabSize = 0;
   eosToken = 0;
+  bosToken = 0;
   unkToken = 0;
   vocab = [];
   cache = /* @__PURE__ */ new Map();
   _trained = !1;
-  constructor(t) {
-    if (super(), Array.isArray(t)) {
-      if (this.vocab = t, this.vocab.length > 0)
-        this.vocabSize = this.vocab.length, this.eosToken = this.vocab.indexOf("<eos>"), this.unkToken = this.vocab.indexOf(""), this.unkToken === -1 && (this.unkToken = this.vocab.indexOf("<unk>")), this.unkToken === -1 && (this.unkToken = this.vocab.indexOf("<pad>")), this.unkToken === -1 && (this.unkToken = this.vocab.indexOf("_")), this.unkToken === -1 && (this.unkToken = this.vocab.indexOf(" ")), this.unkToken === -1 && (this.unkToken = this.eosToken), this.vocab = this.vocab.map((e) => e === "<pad>" ? "" : e), this.vocab.forEach((e, n) => {
-          this.cache.set(e, n);
+  constructor(s) {
+    if (super(), Array.isArray(s)) {
+      if (this.vocab = s, this.vocab.length > 0)
+        this.vocabSize = this.vocab.length, d.forEach((t) => {
+          const e = this.vocab.indexOf(t);
+          e !== -1 && this.addSpecialToken(t, e);
+        }), this.eosToken = this.getSpecialTokenIndex("<eos>"), this.bosToken = this.getSpecialTokenIndex("<bos>") ?? this.eosToken, this.unkToken = this.getSpecialTokenIndex("") ?? -1, this.unkToken === -1 && (this.unkToken = this.vocab.indexOf("<unk>")), this.unkToken === -1 && (this.unkToken = this.vocab.indexOf("<pad>")), this.unkToken === -1 && (this.unkToken = this.vocab.indexOf("_")), this.unkToken === -1 && (this.unkToken = this.vocab.indexOf(" ")), this.unkToken === -1 && (this.unkToken = this.eosToken), this.vocab = this.vocab.map((t) => t === "<pad>" ? "" : t), this.vocab.forEach((t, e) => {
+          this.cache.set(t, e);
         });
       else
         throw new Error("Vocab cannot be empty");
       this._trained = !0;
     } else
-      this.vocabSize = t, this.vocab = new Array(this.vocabSize).fill(""), this.vocab[0] = "<eos>", this.vocab[1] = "", this.eosToken = 0, this.unkToken = 1, this.cache.set("<eos>", 0), this.cache.set("", 1);
+      this.vocabSize = s, this.vocab = new Array(this.vocabSize).fill(""), this.addSpecialTokens(), this.eosToken = this.getSpecialTokenIndex("<eos>"), this.bosToken = this.getSpecialTokenIndex("<bos>") ?? this.eosToken, this.unkToken = this.getSpecialTokenIndex(""), this.vocab.forEach((t, e) => {
+        this.cache.set(t, e);
+      }), this.cache.set("", this.unkToken);
+  }
+  addToken(s, t) {
+    if (this.cache.has(s))
+      return this.cache.get(s);
+    let e;
+    if (t !== void 0 ? e = t : (e = this.vocab.indexOf("", this.unkToken + 1), e === -1 && (e = this.vocabSize)), e >= this.vocabSize)
+      throw new Error("Vocab size exceeded");
+    return this.vocab[e] = s, this.cache.set(s, e), e;
   }
   get trained() {
     return this.vocab.length === this.vocabSize && this._trained;
   }
   destroy() {
   }
-  async train(t) {
-    const e = t.map((i) => i.split("")).flat(), n = new Set(e), s = Array.from(n), h = this.vocab.indexOf("", this.unkToken + 1), o = this.vocabSize - u.length;
+  async train(s) {
+    const t = s.map((n) => n.split("")).flat(), e = new Set(t), i = Array.from(e), h = this.vocab.indexOf("", this.unkToken + 1), o = this.vocabSize - u.length;
     if (h === -1)
       return this.vocabSize;
-    if (this._trained = !0, s.length > o) {
-      const i = /* @__PURE__ */ new Map();
-      e.forEach((a) => {
-        i.set(a, (i.get(a) || 0) + 1);
-      }), s.sort((a, r) => (i.get(a) || 0) - (i.get(r) || 0)), s.splice(0, s.length - o);
+    if (this._trained = !0, i.length > o) {
+      const n = /* @__PURE__ */ new Map();
+      t.forEach((a) => {
+        n.set(a, (n.get(a) || 0) + 1);
+      }), i.sort((a, r) => (n.get(a) || 0) - (n.get(r) || 0)), i.splice(0, i.length - o);
     }
     let c = h;
     if (c !== -1) {
-      const i = new Set(this.vocab);
-      for (const a of s)
-        if (!i.has(a) && (this.vocab[c] = a, i.add(a), c = this.vocab.indexOf("", c + 1), c === -1))
+      const n = new Set(this.vocab);
+      for (const a of i)
+        if (!n.has(a) && (this.vocab[c] = a, n.add(a), c = this.vocab.indexOf("", c + 1), c === -1))
           break;
     }
-    return this.cache.clear(), this.vocab.forEach((i, a) => {
-      this.cache.set(i, a);
+    return this.cache.clear(), this.vocab.forEach((n, a) => {
+      this.cache.set(n, a);
     }), this.emit("trainStatus", "trained"), this.vocabSize;
   }
-  async tokenise(t, e) {
+  async tokenise(s, t) {
     if (!this.trained)
       throw new Error("Tokeniser not trained");
-    return t.map((s) => e ? s.split("").map((h) => this.cache.get(h) ?? this.unkToken) : s.split("").map((h) => {
+    return s.map((i) => t ? i.split("").map((h) => this.cache.get(h) ?? this.unkToken) : i.split("").map((h) => {
       const o = this.cache.get(h);
       return o !== void 0 ? this.vocab[o] : "";
     }));
   }
-  async detokenise(t) {
-    return t.map((n) => n.map((s) => this.vocab[s]).join(""));
+  async detokenise(s) {
+    return s.map((e) => e.map((i) => this.vocab[i]).join(""));
   }
-  async encode(t) {
-    return (await this.tokenise([t], !0))[0];
+  async encode(s) {
+    return (await this.tokenise([s], !0))[0];
   }
-  async decode(t) {
-    return (await this.detokenise([t]))[0];
+  async decode(s) {
+    return (await this.detokenise([s]))[0];
   }
   getVocab() {
     return this.vocab;
@@ -68,11 +82,11 @@ class b extends k {
   async getMerges() {
     return [];
   }
-  async createTrainingData(t, e = 5) {
-    const n = await this.tokenise(t, !0), s = [], h = [];
-    for (let o = 0; o < n.length - e; o++)
-      s.push(...n[o].slice(0, e)), h.push(n[o + 1][0]);
-    return [s, h];
+  async createTrainingData(s, t = 5) {
+    const e = await this.tokenise(s, !0), i = [], h = [];
+    for (let o = 0; o < e.length - t; o++)
+      i.push(...e[o].slice(0, t)), h.push(e[o + 1][0]);
+    return [i, h];
   }
 }
 export {

package/dist/tokeniser/bpe.d.ts

@@ -1,6 +1,5 @@
-import { default as EE } from 'eventemitter3';
-import { ITokeniser } from './type';
-export default class BPETokeniser extends EE<'trainStatus'> implements ITokeniser {
+import { default as BaseTokeniser } from './BaseTokeniser';
+export default class BPETokeniser extends BaseTokeniser {
     private targetSize;
     private vocab;
     private vocabIndex;
@@ -8,10 +7,12 @@ export default class BPETokeniser extends EE<'trainStatus'> implements ITokenise
     private pretokenMap;
     constructor(vocabSize: number);
     constructor(vocab: string[], merges?: [string, string][]);
+    addToken(token: string, index?: number): number;
     destroy(): void;
     get trained(): boolean;
     get vocabSize(): number;
     get eosToken(): number;
+    get bosToken(): number;
     get unkToken(): number;
     train(text: string[]): Promise<number>;
     getVocab(): string[];

package/dist/tokeniser/bpe.js

@@ -1,68 +1,80 @@
 import l from "../utilities/tokenParse.js";
-import { E as f } from "../index-DvYrXKkX.js";
+import d, { SPECIALS as f } from "./BaseTokeniser.js";
 function u(o, e) {
   return `${o}-::-${e}`;
 }
-function k(o) {
+function b(o) {
   const e = /* @__PURE__ */ new Map();
   for (let s = 0; s < o.length; s++) {
     const t = o[s];
-    for (let r = 0; r < t.length - 1; r++) {
-      const n = u(t[r], t[r + 1]), a = e.get(n) || {
-        a: t[r],
-        b: t[r + 1],
+    for (let n = 0; n < t.length - 1; n++) {
+      const r = u(t[n], t[n + 1]), i = e.get(r) || {
+        a: t[n],
+        b: t[n + 1],
         count: 0,
         instances: /* @__PURE__ */ new Set()
       };
-      a.count += 1, a.instances.add(s), e.set(n, a);
+      i.count += 1, i.instances.add(s), e.set(r, i);
     }
   }
   return { pairs: e, tokens: o };
 }
-function h(o, e, s, t, r) {
-  const n = u(e, s);
-  if (o.pairs.has(n)) {
-    const a = o.pairs.get(n);
-    a.count += r, r > 0 ? a.instances.add(t) : a.count <= 0 ? o.pairs.delete(n) : a.instances.delete(t);
+function h(o, e, s, t, n) {
+  const r = u(e, s);
+  if (o.pairs.has(r)) {
+    const i = o.pairs.get(r);
+    i.count += n, n > 0 ? i.instances.add(t) : i.count <= 0 ? o.pairs.delete(r) : i.instances.delete(t);
   } else
-    o.pairs.set(n, { a: e, b: s, count: r, instances: /* @__PURE__ */ new Set([t]) });
+    o.pairs.set(r, { a: e, b: s, count: n, instances: /* @__PURE__ */ new Set([t]) });
 }
-function b(o) {
+function k(o) {
   let e = null, s = 0;
   for (const t of o.pairs.values())
     t.count > s && (s = t.count, e = t);
   return e;
 }
-function d(o, e) {
+function m(o, e) {
   return o.map((s) => {
     const t = [];
-    for (let r = 0; r < s.length; r++)
-      r < s.length - 1 && s[r] === e[0] && s[r + 1] === e[1] ? (t.push(e[0] + e[1]), r++) : t.push(s[r]);
+    for (let n = 0; n < s.length; n++)
+      n < s.length - 1 && s[n] === e[0] && s[n + 1] === e[1] ? (t.push(e[0] + e[1]), n++) : t.push(s[n]);
     return t;
   });
 }
-function m(o, e) {
+function v(o, e) {
   e.instances.forEach((s) => {
-    const t = o.tokens[s], r = [];
-    for (let n = 0; n < t.length; n++)
-      if (n < t.length - 1 && t[n] === e.a && t[n + 1] === e.b) {
-        const a = e.a + e.b;
-        r.push(a), n > 0 && (h(o, t[n - 1], e.a, s, -1), h(o, t[n - 1], a, s, 1)), n++, n < t.length - 1 && (h(o, e.b, t[n + 1], s, -1), h(o, a, t[n + 1], s, 1));
+    const t = o.tokens[s], n = [];
+    for (let r = 0; r < t.length; r++)
+      if (r < t.length - 1 && t[r] === e.a && t[r + 1] === e.b) {
+        const i = e.a + e.b;
+        n.push(i), r > 0 && (h(o, t[r - 1], e.a, s, -1), h(o, t[r - 1], i, s, 1)), r++, r < t.length - 1 && (h(o, e.b, t[r + 1], s, -1), h(o, i, t[r + 1], s, 1));
       } else
-        r.push(t[n]);
-    o.tokens[s] = r;
+        n.push(t[r]);
+    o.tokens[s] = n;
   }), o.pairs.delete(u(e.a, e.b));
 }
-class S extends f {
+class T extends d {
   targetSize;
   vocab = /* @__PURE__ */ new Set();
   vocabIndex = /* @__PURE__ */ new Map();
   merges = [];
   pretokenMap = /* @__PURE__ */ new Map();
   constructor(e, s) {
-    super(), Array.isArray(e) ? (e.forEach((t, r) => {
-      this.vocab.add(t), this.vocabIndex.set(t, r);
-    }), s && (this.merges = s), this.targetSize = e.length) : (this.vocab.add("<eos>"), this.vocab.add(""), this.targetSize = e);
+    super(), Array.isArray(e) ? (e.forEach((t, n) => {
+      this.vocab.add(t), this.vocabIndex.set(t, n);
+    }), s && (this.merges = s), this.targetSize = e.length, f.forEach((t) => {
+      const n = e.indexOf(t);
+      n !== -1 && this.addSpecialToken(t, n);
+    })) : (this.addSpecialTokens(), this.targetSize = e);
+  }
+  addToken(e, s) {
+    if (this.vocab.has(e))
+      return this.vocabIndex.get(e);
+    {
+      this.vocab.add(e);
+      const t = s !== void 0 ? s : this.vocab.size - 1;
+      return this.vocabIndex.set(e, t), t;
+    }
   }
   destroy() {
     this.vocab.clear(), this.vocabIndex.clear(), this.merges = [], this.pretokenMap.clear();
@@ -76,26 +88,29 @@ class S extends f {
   get eosToken() {
     return this.vocabIndex.get("<eos>") ?? 0;
   }
+  get bosToken() {
+    return this.vocabIndex.get("<bos>") ?? 0;
+  }
   get unkToken() {
     return this.vocabIndex.get("") ?? 1;
   }
   async train(e) {
-    const s = e.map((i) => l(i)).flat(1), t = new Set(s);
-    this.vocab = /* @__PURE__ */ new Set(), this.pretokenMap.clear(), this.merges = [], this.vocab.add("<eos>"), this.vocab.add("");
-    const r = Array.from(t), n = r.map((i) => Array.from(i).map((c) => (this.vocab.add(c), c))), a = k(n);
+    const s = e.map((a) => l(a)).flat(1), t = new Set(s);
+    this.vocab = /* @__PURE__ */ new Set(), this.pretokenMap.clear(), this.merges = [], this.addSpecialTokens();
+    const n = Array.from(t), r = n.map((a) => Array.from(a).map((c) => (this.vocab.add(c), c))), i = b(r);
     for (; this.vocab.size < this.targetSize && this.merges.length < this.targetSize; ) {
-      const i = b(a);
-      if (!i)
+      const a = k(i);
+      if (!a)
         break;
-      this.merges.push([i.a, i.b]), this.vocab.add(i.a + i.b), m(a, i);
+      this.merges.push([a.a, a.b]), this.vocab.add(a.a + a.b), v(i, a);
     }
-    r.forEach((i, p) => {
-      const c = n[p];
-      this.pretokenMap.set(i, c);
+    n.forEach((a, p) => {
+      const c = r[p];
+      this.pretokenMap.set(a, c);
     }), this.vocabIndex.clear();
     let g = 0;
-    for (const i of this.vocab.keys())
-      this.vocabIndex.set(i, g++);
+    for (const a of this.vocab.keys())
+      this.vocabIndex.set(a, g++);
     return this.emit("trainStatus", "trained"), this.vocab.size;
   }
   getVocab() {
@@ -107,19 +122,19 @@ class S extends f {
   tokeniseWord(e) {
     let s = Array.from(e);
     return this.merges.forEach((t) => {
-      s = d([s], t)[0];
+      s = m([s], t)[0];
     }), this.pretokenMap.set(e, s), s;
   }
   tokeniseStrings(e) {
-    return e.map((s) => l(s).map((n) => this.pretokenMap.has(n) ? this.pretokenMap.get(n) : this.tokeniseWord(n)).flat(1));
+    return e.map((s) => l(s).map((r) => this.pretokenMap.has(r) ? this.pretokenMap.get(r) : this.tokeniseWord(r)).flat(1));
   }
   async tokenise(e, s) {
     const t = this.tokeniseStrings(e);
-    return s ? t.map((r) => r.map((n) => this.vocabIndex.get(n) ?? this.unkToken)) : t.map((r) => r.map((n) => this.vocab.has(n) ? n : ""));
+    return s ? t.map((n) => n.map((r) => this.vocabIndex.get(r) ?? this.unkToken)) : t.map((n) => n.map((r) => this.vocab.has(r) ? r : ""));
   }
   async detokenise(e) {
     const s = this.getVocab();
-    return e.map((r) => r.map((n) => s[n]).join(""));
+    return e.map((n) => n.map((r) => s[r]).join(""));
   }
   async encode(e) {
     return (await this.tokenise([e], !0))[0];
@@ -129,5 +144,5 @@ class S extends f {
   }
 }
 export {
-  S as default
+  T as default
 };

package/dist/tokeniser/type.d.ts

@@ -1,4 +1,9 @@
 import { default as EE } from 'eventemitter3';
+export type Roles = 'user' | 'assistant' | 'system';
+export interface Conversation {
+    role: Roles;
+    content: string;
+}
 export interface ITokeniser extends EE<'trainStatus'> {
     train(text: string[]): Promise<number>;
     tokenise(text: string[], numeric?: boolean): Promise<string[][] | number[][]>;
@@ -7,8 +12,14 @@ export interface ITokeniser extends EE<'trainStatus'> {
     getMerges(): Promise<[string, string][]>;
     destroy(): void;
     encode(text: string): Promise<number[]>;
+    encodeConversation(conversation: Conversation[], completion?: boolean): Promise<number[]>;
+    encodeSequence(text: string): Promise<number[]>;
     decode(tokens: number[]): Promise<string>;
+    decodeConversation(tokens: number[]): Promise<Conversation[]>;
     vocabSize: number;
     eosToken: number;
+    bosToken: number;
     trained: boolean;
+    getSpecialTokenIndex(token: string): number | undefined;
+    isSpecialToken(index: number): boolean;
 }

package/dist/training/Adam.js

@@ -1,7 +1,7 @@
 import { adamAdjust as b } from "../ops/adamAdjust.js";
 import { adamMoments as d } from "../ops/adamMoments.js";
-import { O as g, e as h, t as o, d as B } from "../index-D6Q1lPZO.js";
-import { z as M } from "../zeros-DBFVbpv5.js";
+import { O as g, e as h, t as o, d as B } from "../index-D0RBWjq8.js";
+import { z as M } from "../zeros-DeiE2zTa.js";
 class R extends g {
   constructor(t, a, e, s, i = null) {
     super(), this.learningRate = t, this.beta1 = a, this.beta2 = e, this.lossScaling = s, this.epsilon = i, this.accBeta1 = a, this.accBeta2 = e, i === null && (this.epsilon = h().backend.epsilon());

package/dist/training/AdamExt.js

@@ -1,4 +1,4 @@
-import { m as r, b as c, c as h, e as o } from "../index-D6Q1lPZO.js";
+import { m as r, b as c, c as h, e as o } from "../index-D0RBWjq8.js";
 import { AdamOptimizer as g } from "./Adam.js";
 class y extends g {
   constructor(t, e, s, i, a) {

package/dist/training/DatasetBuilder.d.ts

@@ -1,8 +1,8 @@
 import { Tensor } from '@tensorflow/tfjs-core';
-import { ITokeniser } from '../tokeniser/type';
+import { Conversation, ITokeniser } from '../tokeniser/type';
 import { Dataset } from '@tensorflow/tfjs-data';
 export declare const PAGE_FACTOR = 8;
-export declare function flattenTokens(textData: string[], tokenizer: ITokeniser): Promise<number[]>;
+export declare function flattenTokens(textData: Conversation[][], tokenizer: ITokeniser): Promise<number[]>;
 export declare class DatasetBuilder {
     tokenizer: ITokeniser;
     blockSize: number;

package/dist/training/DatasetBuilder.js

@@ -1,67 +1,63 @@
-import { t as g } from "../index-D6Q1lPZO.js";
-import { d as u, i as d } from "../dataset-D2P7rHAw.js";
+import { t as z } from "../index-D0RBWjq8.js";
+import { d as u, i as f } from "../dataset-DcjWqUVQ.js";
 import "../index-Cp39cXWe.js";
-function z(r) {
+function S(a) {
   return u(async () => {
-    const t = await r();
-    return d(() => t.next());
+    const t = await a();
+    return f(() => t.next());
   });
 }
-const S = 8;
-async function y(r, t) {
-  const s = await Promise.all(r.map((e) => t.encode(e))), o = t.eosToken >= 0, a = s.map((e) => o ? [...e, t.eosToken] : e).flat();
-  for (const e of a)
-    if (e < 0 || e >= t.vocabSize)
-      throw new Error(`Invalid token index ${e} found in tokenised data`);
-  return a;
+const b = 8;
+async function y(a, t) {
+  return (await Promise.all(a.map((r) => t.encodeConversation(r)))).flat();
 }
-class w {
+class x {
   tokenizer;
   blockSize;
   pageSize;
   constructor(t, s = 128) {
-    this.tokenizer = t, this.blockSize = s, this.pageSize = s * S;
+    this.tokenizer = t, this.blockSize = s, this.pageSize = s * b;
   }
   // Create dataset from text files
-  async createTextDataset(t, s = 32, o, a) {
+  async createTextDataset(t, s = 32, i, r) {
     if (t.length < this.blockSize + 1)
       throw new Error(`Not enough tokens (${t.length}) for block size ${this.blockSize}`);
-    if (o && o.size > t.length / this.pageSize / 2)
+    if (i && i.size > t.length / this.pageSize / 2)
       throw new Error("Too many masked pages - would leave insufficient training data");
-    const e = (function* () {
-      if (o && a) {
-        const i = Array.from(o);
+    const l = (function* () {
+      if (i && r) {
+        const e = Array.from(i);
         for (; ; ) {
-          const c = Math.floor(Math.random() * i.length), l = Math.floor(Math.random() * this.pageSize), n = i[c] * this.pageSize + l;
-          if (n + this.blockSize + 1 > t.length)
+          const n = Math.floor(Math.random() * e.length), h = Math.floor(Math.random() * this.pageSize), o = e[n] * this.pageSize + h;
+          if (o + this.blockSize + 1 > t.length)
             continue;
-          const h = t.slice(n, n + this.blockSize), f = t.slice(n + 1, n + this.blockSize + 1);
-          yield { xs: h, ys: f };
+          const c = t.slice(o, o + this.blockSize), g = t.slice(o + 1, o + this.blockSize + 1);
+          yield { xs: c, ys: g };
         }
       } else
         for (; ; ) {
-          const i = Math.floor(Math.random() * (t.length - this.blockSize - 1));
-          if (o) {
-            const n = Math.floor(i / this.pageSize), h = o.has(n);
-            if (h && !a || !h && a)
+          const e = Math.floor(Math.random() * (t.length - this.blockSize - 1));
+          if (i) {
+            const o = Math.floor(e / this.pageSize), c = i.has(o);
+            if (c && !r || !c && r)
               continue;
           }
-          const c = t.slice(i, i + this.blockSize), l = t.slice(i + 1, i + this.blockSize + 1);
-          yield { xs: c, ys: l };
+          const n = t.slice(e, e + this.blockSize), h = t.slice(e + 1, e + this.blockSize + 1);
+          yield { xs: n, ys: h };
         }
     }).bind(this);
-    return z(e).batch(s).map((i) => {
-      const c = i;
-      return g(() => ({
-        xs: c.xs.cast("int32"),
-        ys: c.ys.cast("int32")
+    return S(l).batch(s).map((e) => {
+      const n = e;
+      return z(() => ({
+        xs: n.xs.cast("int32"),
+        ys: n.ys.cast("int32")
         // this.tf.oneHot(batchData.ys.cast('int32'), this.tokenizer.vocabSize),
       }));
     }).prefetch(2);
   }
 }
 export {
-  w as DatasetBuilder,
-  S as PAGE_FACTOR,
+  x as DatasetBuilder,
+  b as PAGE_FACTOR,
   y as flattenTokens
 };

package/dist/training/FullTrainer.js

@@ -1,6 +1,6 @@
 import b from "./Trainer.js";
 import L from "./Evaluator.js";
-import { d as w } from "../index-D6Q1lPZO.js";
+import { d as w } from "../index-D0RBWjq8.js";
 import y from "../utilities/profile.js";
 import { createTensorStatistics as D } from "../checks/weights.js";
 const T = {