@genai-fi/nanogpt 0.6.0 → 0.6.2

package/dist/utilities/load.js

@@ -1,99 +0,0 @@
-import { j as v } from "../jszip.min-CjP2V1VV.js";
-import { importWeights as F } from "./weights.js";
-import h from "../tokeniser/CharTokeniser.js";
-import b from "../NanoGPTModel.js";
-import { dummyPassAsync as u } from "./dummy.js";
-import { d as k } from "../index-bMBtI-WR.js";
-import j from "../tokeniser/bpe.js";
-import { load_safetensors as N } from "./safetensors.js";
-const I = 2;
-async function O(t) {
-  const s = await fetch(t);
-  if (!s.ok)
-    throw new Error(`Failed to fetch ${t}: ${s.statusText}`);
-  return s.arrayBuffer();
-}
-async function S(t) {
-  const s = /* @__PURE__ */ new Map(), r = await t.file("manifest.json")?.async("string");
-  if (!r)
-    throw new Error("Manifest file not found in the zip archive");
-  const p = JSON.parse(r);
-  for (const [o, a] of Object.entries(p.weightSpec))
-    s.set(o, { spec: a, data: new Float32Array() });
-  const e = await t.file("tokeniser.json")?.async("string");
-  if (!e)
-    throw new Error("Tokeniser file not found in the zip archive");
-  const i = JSON.parse(e), c = (i.type ?? "char") === "char" ? new h(i.vocab) : new j(i.vocab, i.merges), d = /* @__PURE__ */ new Map();
-  for (const o of Object.keys(t.files))
-    if (o.endsWith(".bin")) {
-      const a = o.replace(".bin", ""), w = await t.file(o).async("arraybuffer"), g = new Float32Array(w), l = s.get(a) || { spec: [], data: new Float32Array() };
-      l.data = g, s.set(a, l);
-      const n = await F(l);
-      d.set(a, n);
-    }
-  k();
-  const f = new b(p.config);
-  await u(f), f.loadWeights(d);
-  const m = await t.file("log.json")?.async("string");
-  if (m)
-    try {
-      const o = JSON.parse(m);
-      f.log = o;
-    } catch (o) {
-      throw console.error("Error parsing training log:", o), new Error(`Failed to parse training log: ${o}`);
-    }
-  return { model: f, tokeniser: c };
-}
-async function R(t) {
-  const s = typeof t == "string" ? await O(t) : t, r = await v.loadAsync(s);
-  if (r.file("manifest.json"))
-    return S(r);
-  {
-    const p = await r.file("config.json")?.async("string");
-    if (!p)
-      throw new Error("Config file not found in the zip archive");
-    const e = JSON.parse(p), i = {
-      vocabSize: e.vocab_size,
-      blockSize: e.block_size,
-      nLayer: e.num_hidden_layers,
-      nHead: e.num_attention_heads,
-      nEmbed: e.hidden_size,
-      dropout: e.dropout,
-      biasInLinear: e.biasInLinear,
-      biasInLayerNorm: e.biasInLayerNorm,
-      mlpFactor: e.mlpFactor,
-      useRope: e.useRope
-    }, y = await r.file("tokeniser.json")?.async("string");
-    if (!y)
-      throw new Error("Tokeniser file not found in the zip archive");
-    const c = JSON.parse(y), f = (c.type ?? "char") === "char" ? new h(c.vocab) : new j(c.vocab, c.merges), m = await N(await r.file("model.safetensors").async("arraybuffer")), o = /* @__PURE__ */ new Map();
-    for (const [n, E] of Object.entries(m))
-      o.set(n, [E]);
-    k();
-    const a = new b(i);
-    await u(a), a.loadWeights(o);
-    const w = await r.file("meta.json")?.async("string");
-    let g;
-    if (w)
-      try {
-        const n = JSON.parse(w);
-        n.name && (g = n.name);
-      } catch (n) {
-        console.error("Error parsing meta file:", n);
-      }
-    const l = await r.file("log.json")?.async("string");
-    if (l)
-      try {
-        const n = JSON.parse(l);
-        a.log = n;
-      } catch (n) {
-        throw console.error("Error parsing training log:", n), new Error(`Failed to parse training log: ${n}`);
-      }
-    return { model: a, tokeniser: f, name: g };
-  }
-}
-export {
-  I as VERSION,
-  R as loadModel,
-  S as loadOldModel
-};