@genai-fi/nanogpt 0.18.0 → 0.18.1

package/dist/loader/loadTransformers.js

@@ -28,15 +28,15 @@ function h(e) {
     windowSize: e.windowSize
   }, a;
 }
-async function G(e, a, t, n) {
-  const m = h(e), r = (a.type ?? "char") === "char" ? new p(a.vocab) : new _(a.vocab, a.merges);
-  a.datasetID && (r.datasetID = a.datasetID);
-  const i = await b(n), s = /* @__PURE__ */ new Map();
-  for (const [l, d] of Object.entries(i))
-    s.set(l, [d]);
+async function G(e, a, r, n) {
+  const m = h(e), o = (a.type ?? "char") === "char" ? new p(a.vocab) : new _(a.vocab, a.merges);
+  a.datasetID && (o.datasetID = a.datasetID), a.id ? o.id = a.id : o.generateID();
+  const d = await b(n), s = /* @__PURE__ */ new Map();
+  for (const [i, l] of Object.entries(d))
+    s.set(i, [l]);
   c();
-  const o = y(m);
-  return o.metaData = t, await u(o), o.weightStore.loadWeights(s, !!t.url), { model: o, tokeniser: r, metaData: t };
+  const t = y(m);
+  return t.metaData = r, await u(t), t.weightStore.loadWeights(s, !!r.url), { model: t, tokeniser: o, metaData: r };
 }
 export {
   G as default,

package/dist/loader/save.js

@@ -2,13 +2,13 @@ import { z as y } from "../jszip.min-BZhlzntC.js";
 import b from "../tokeniser/CharTokeniser.js";
 import { save_safetensors as _ } from "../utilities/safetensors.js";
 import { VERSION as m } from "./load.js";
-async function d(i, e, o, t) {
+async function d(i, a, o, t) {
   const g = /* @__PURE__ */ new Map();
   i.weightStore.saveWeights(g);
   const n = new y();
   if (t?.optimizer) {
-    const a = await t.optimizer.saveMoments();
-    n.file("optimizer.safetensors", a, { binary: !0 }), n.file("optimizer_config.json", JSON.stringify(t.optimizer.serializeConfig()), {
+    const e = await t.optimizer.saveMoments();
+    n.file("optimizer.safetensors", e, { binary: !0 }), n.file("optimizer_config.json", JSON.stringify(t.optimizer.serializeConfig()), {
       binary: !1
     });
   }
@@ -16,8 +16,8 @@ async function d(i, e, o, t) {
     binary: !1
   });
   const s = {};
-  g.forEach((a, f) => {
-    a.length === 1 && (s[f] = a[0]);
+  g.forEach((e, f) => {
+    e.length === 1 && (s[f] = e[0]);
   });
   const c = await _(s);
   n.file("model.safetensors", c, { binary: !0 });
@@ -25,7 +25,7 @@ async function d(i, e, o, t) {
   let r;
   p === "GenAI_NanoGPT_v1" ? r = {
     model_type: "GenAI_NanoGPT_v1",
-    vocab_size: e.getVocab().length,
+    vocab_size: a.getVocab().length,
     hidden_size: i.config.nEmbed,
     num_hidden_layers: i.config.nLayer,
     num_attention_heads: i.config.nHead,
@@ -34,7 +34,7 @@ async function d(i, e, o, t) {
     useRope: i.config.useRope
   } : r = {
     model_type: "GenAI_NanoGPT_v2",
-    vocab_size: e.getVocab().length,
+    vocab_size: a.getVocab().length,
     hidden_size: i.config.nEmbed,
     num_hidden_layers: i.config.nLayer,
     num_attention_heads: i.config.nHead,
@@ -64,17 +64,18 @@ async function d(i, e, o, t) {
   }), n.file(
     "tokeniser.json",
     JSON.stringify({
-      type: e instanceof b ? "char" : "bpe",
-      vocab: e.getVocab(),
-      merges: e.getMerges(),
-      datasetID: e.datasetID
+      type: a instanceof b ? "char" : "bpe",
+      vocab: a.getVocab(),
+      merges: a.getMerges(),
+      datasetID: a.datasetID,
+      id: a.id
     }),
     {
       binary: !1
     }
   ), o?.files)
-    for (const [a, f] of Object.entries(o.files))
-      n.file(a, JSON.stringify(f), { binary: !1 });
+    for (const [e, f] of Object.entries(o.files))
+      n.file(e, JSON.stringify(f), { binary: !1 });
   return n.generateAsync({ type: "blob" });
 }
 export {

package/dist/loader/types.d.ts

@@ -29,6 +29,7 @@ export interface TransformersTokeniser {
     vocab: string[];
     merges: [string, string][];
     datasetID?: string;
+    id?: string;
 }
 export type ModelPhase = 'untrained' | 'pretrained' | 'finetuned';
 export interface DatasetMetadata {

package/dist/tokeniser/BaseTokeniser.d.ts

@@ -14,7 +14,7 @@ export default abstract class BaseTokeniser extends EE<'trainStatus'> implements
     isSpecialToken(index: number): boolean;
     protected addSpecialTokens(): void;
     protected addSpecialToken(token: string, index: number): void;
-    protected generateID(): void;
+    generateID(): void;
     abstract train(text: Conversation[][], cb?: (vocab: number) => void, datasetID?: string): Promise<number>;
     abstract getVocab(): string[];
     abstract getMerges(): [string, string][];

package/dist/tokeniser/BaseTokeniser.js

@@ -30,11 +30,15 @@ class T extends r {
   generateID() {
     const s = this.getVocab();
     let t = 2166136261, e = 2654435769;
-    for (let a = 0; a < s.length; a++) {
-      const i = s[a];
-      t ^= i.length, t = Math.imul(t, 16777619), e ^= a, e = Math.imul(e, 2246822507);
-      for (let c = 0; c < i.length; c++) {
-        const h = i.charCodeAt(c);
+    if (s.length === 0) {
+      this.id = "untrained";
+      return;
+    }
+    for (let i = 0; i < s.length; i++) {
+      const a = s[i];
+      t ^= a.length, t = Math.imul(t, 16777619), e ^= i, e = Math.imul(e, 2246822507);
+      for (let c = 0; c < a.length; c++) {
+        const h = a.charCodeAt(c);
         t ^= h, t = Math.imul(t, 16777619), e ^= h, e = Math.imul(e, 3266489909);
       }
     }
@@ -59,9 +63,9 @@ class T extends r {
       this.getSpecialTokenIndex("<|assistant_end|>"),
       this.getSpecialTokenIndex("<|system_end|>")
     ];
-    for (const i of s) {
-      const c = this.encode(i.content);
-      switch (i.role) {
+    for (const a of s) {
+      const c = this.encode(a.content);
+      switch (a.role) {
         case "user":
           e.push([o[0]]);
           break;
@@ -72,7 +76,7 @@ class T extends r {
           e.push([o[2]]);
           break;
       }
-      switch (e.push(c), i.role) {
+      switch (e.push(c), a.role) {
         case "user":
           e.push([n[0]]);
           break;
@@ -84,8 +88,8 @@ class T extends r {
           break;
       }
     }
-    const a = e.flat();
-    return t ? a.push(o[1]) : a.push(this.eosToken), a;
+    const i = e.flat();
+    return t ? i.push(o[1]) : i.push(this.eosToken), i;
   }
   decodeConversation(s) {
     const t = [];
@@ -95,11 +99,11 @@ class T extends r {
       let n = null;
       if (o === this.getSpecialTokenIndex("<|user_start|>") ? n = "user" : o === this.getSpecialTokenIndex("<|assistant_start|>") ? n = "assistant" : o === this.getSpecialTokenIndex("<|system_start|>") ? n = "system" : o === this.bosToken || (o === this.eosToken ? n = null : (n = "text", e--)), n) {
         e++;
-        const a = [];
+        const i = [];
         for (; e < s.length && s[e] !== this.getSpecialTokenIndex(`<|${n}_end|>`) && s[e] !== this.eosToken; )
-          a.push(s[e]), e++;
-        const i = this.decode(a);
-        t.push({ role: n, content: i });
+          i.push(s[e]), e++;
+        const a = this.decode(i);
+        t.push({ role: n, content: a });
       }
       e++;
     }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@genai-fi/nanogpt",
-    "version": "0.18.0",
+    "version": "0.18.1",
     "type": "module",
     "main": "dist/main.js",
     "types": "dist/main.d.ts",