@genai-fi/nanogpt 0.19.1 → 0.20.1

package/dist/TeachableLLM.js

@@ -1,273 +1,2 @@
-import { validateConfig as _ } from "./models/config.js";
-import { saveModel as p } from "./loader/save.js";
-import { loadModel as u } from "./loader/load.js";
-import f from "./Generator.js";
-import h from "./Trainer.js";
-import { E as c } from "./index-DvYrXKkX.js";
-import { dummyPassTrainAsync as d } from "./utilities/dummy.js";
-import "./index-CUXkjxiT.js";
-import "./random_width-DN5ZtQkM.js";
-import "./zeros_like-CWjDdwr-.js";
-import "./index-Cp39cXWe.js";
-import "./dataset-CGGp1z9P.js";
-import "./ops/cpu/attentionMask.js";
-import "./ops/webgl/attentionMask.js";
-import "./ops/grads/attentionMask.js";
-import "./ops/cpu/rope.js";
-import "./ops/webgl/rope.js";
-import "./rope-DcrZM_e6.js";
-import "./ops/cpu/appendCache.js";
-import "./ops/webgl/appendCache.js";
-import "./ops/grads/softmax16.js";
-import "./matMul16-BcVC_E62.js";
-import "./ops/webgl/matMul16.js";
-import "./ops/cpu/matMul16.js";
-import "./pack16-bqltoUlR.js";
-import "./ops/transpose16.js";
-import "./ops/reshape16.js";
-import "./ops/cpu/qkv.js";
-import "./ops/webgl/qkv.js";
-import "./ops/grads/qkv.js";
-import "./ops/cpu/normRMS.js";
-import "./ops/webgl/normRMS.js";
-import "./ops/grads/normRMS.js";
-import "./ops/dropout16.js";
-import "./ops/webgl/dropout16.js";
-import "./ops/grads/add16.js";
-import g from "./tokeniser/CharTokeniser.js";
-import w from "./tokeniser/bpe.js";
-import "./papaparse.min-C0cScC2i.js";
-import "./jszip.min-BZhlzntC.js";
-import "./ops/cpu/gatherSub.js";
-import "./ops/webgl/gatherSub.js";
-import "./ops/cpu/scatterSub.js";
-import "./ops/webgl/scatterSub.js";
-import "./ops/cpu/matMulGelu.js";
-import "./matMulGelu-JNLZqKQp.js";
-import "./ops/grads/matMulGelu.js";
-import "./ops/cpu/gelu.js";
-import "./ops/webgl/gelu.js";
-import "./gelu-B220X1Go.js";
-import "./ops/webgl/log.js";
-import "./ops/cpu/adamMoments.js";
-import "./ops/webgl/adamMoments.js";
-import "./ops/cpu/adamAdjust.js";
-import "./ops/webgl/adamAdjust.js";
-import "./checks/normRMS.js";
-import "./checks/normRMSGrad.js";
-import k from "./utilities/profile.js";
-import y from "./models/factory.js";
-class l {
-  ee = new c();
-  _config;
-  _model;
-  _tokeniser;
-  _status = "loading";
-  _memoryRequirements;
-  meta = {
-    version: 2,
-    application: "@genai-fi/nanogpt"
-  };
-  _trainer = null;
-  constructor(t, i) {
-    this._config = i?.config, this._tokeniser = t, this._model = i, i?.metaData && (this.meta = i.metaData);
-  }
-  get currentTrainer() {
-    return this._trainer;
-  }
-  get vocab() {
-    return this._tokeniser?.getVocab() || [];
-  }
-  get phase() {
-    return this._model?.metaData?.phase ?? "untrained";
-  }
-  set phase(t) {
-    if (!this._model)
-      throw new Error("model_not_initialized.");
-    this._model.metaData.phase = t, this.ee.emit("phase", t);
-  }
-  /** Model is fully loaded */
-  get loaded() {
-    return !!this._model && !!this._tokeniser && !!this._config;
-  }
-  get config() {
-    if (!this._config)
-      throw new Error("configuration_not_initialized.");
-    return this._config;
-  }
-  get model() {
-    if (!this._model)
-      throw new Error("model_not_initialized.");
-    return this._model;
-  }
-  get tokeniser() {
-    if (!this._tokeniser)
-      throw new Error("tokeniser_not_initialized.");
-    return this._tokeniser;
-  }
-  get status() {
-    return this._status;
-  }
-  /** Model is both ready and not busy */
-  get ready() {
-    return this._status === "ready" && !!this._model && !!this._tokeniser;
-  }
-  get busy() {
-    return this._status === "busy" || this._status === "training";
-  }
-  createLoRA(t, i) {
-    if (!this._model)
-      throw new Error("model_not_initialized.");
-    this._model.createLoRA(t, i), this.ee.emit("changeLoRA");
-  }
-  deleteLoRA(t) {
-    if (!this._model)
-      throw new Error("model_not_initialized.");
-    this._model.deleteLoRA(t), this.ee.emit("changeLoRA");
-  }
-  renameLoRA(t, i) {
-    if (!this._model)
-      throw new Error("model_not_initialized.");
-    this._model.renameLoRA(t, i), this.ee.emit("changeLoRA");
-  }
-  attachLoRA(t) {
-    if (!this._model)
-      throw new Error("model_not_initialized.");
-    this.model.lora?.name !== t && (this._trainer && (this._trainer.dispose(), this._trainer = null), this._model.attachLoRA(t), this.ee.emit("changeLoRA"));
-  }
-  detachLoRA() {
-    if (!this._model)
-      throw new Error("model_not_initialized.");
-    this._model.detachLoRA(), this.ee.emit("changeLoRA");
-  }
-  hasLoRA(t) {
-    if (!this._model)
-      throw new Error("model_not_initialized.");
-    return this._model.hasLoRA(t);
-  }
-  listLoRAs() {
-    if (!this._model)
-      throw new Error("model_not_initialized.");
-    return this._model.listLoRAs();
-  }
-  estimateTrainingMemoryUsage(t) {
-    const i = this._memoryRequirements ?? { perBatch: 0, gradients: 0 }, e = i.perBatch * t, o = i.gradients;
-    return e * 0.66 + o * 4;
-  }
-  setStatus(t) {
-    this._status !== t && (this._status = t, this.ee.emit("status", t));
-  }
-  saveModel(t) {
-    if (!this._model || !this._tokeniser)
-      throw new Error("model_or_tokeniser_not_initialized.");
-    return p(
-      this._model,
-      this._tokeniser,
-      {
-        ...t,
-        name: t?.name || this.meta.name
-      },
-      t?.includeOptimizer && this._trainer?.trainingType === "pretraining" ? { optimizer: this._trainer.optimizer, trainingLog: this._trainer.log } : void 0
-    );
-  }
-  static loadModel(t, i) {
-    const e = new l();
-    return u(t, i).then(({ model: o, tokeniser: s, metaData: r, optimizer: n, log: a }) => {
-      _(o.config), e._model = o, e._tokeniser = s, e._config = o.config, r && (e.meta = r), e.setStatus("warmup"), d(o).then((m) => {
-        e._memoryRequirements = m, n && (e._trainer = new h(
-          o,
-          s,
-          "pretraining",
-          o.metaData.pretrainingSettings,
-          n
-        ), a && (e._trainer.log = a)), e.setStatus("ready"), e.ee.emit("loaded"), e.ee.emit("phase", e.phase);
-      }).catch((m) => {
-        e.setStatus("error"), e.ee.emit("error", m), console.error("Error during warmup:", m);
-      });
-    }).catch((o) => {
-      e.setStatus("error"), e.ee.emit("error", o), console.error("Error loading model:", o);
-    }), e;
-  }
-  static create(t, i) {
-    _(i);
-    const e = i, o = t === "char" ? new g(e.vocabSize) : new w(e.vocabSize), s = y(e), r = new l(o, s);
-    return r.setStatus("warmup"), d(s).then((n) => {
-      r._memoryRequirements = n, r.tokeniser.trained ? (r.setStatus("ready"), r.ee.emit("loaded"), r.ee.emit("phase", r.phase)) : (r.setStatus("awaitingTokens"), r.ee.emit("loaded"), r.ee.emit("phase", r.phase), r.tokeniser.once("trainStatus", (a) => {
-        a === "trained" && r.setStatus("ready");
-      }));
-    }).catch((n) => {
-      r.setStatus("error"), r.ee.emit("error", n), console.error("Error during warmup:", n);
-    }), r;
-  }
-  getProfiler() {
-    return this._model?.getProfiler();
-  }
-  get enableProfiler() {
-    return !!this._model?.getProfiler();
-  }
-  set enableProfiler(t) {
-    if (t) {
-      if (!this._config)
-        return;
-      this.model.getProfiler() || this.model.setProfiler(new k());
-    } else
-      this.model.getProfiler() && this.model.setProfiler(null);
-  }
-  getNumParams() {
-    return this._model ? this._model.getNumParams() : 0;
-  }
-  trainer(t, i) {
-    if (!this._model || !this._tokeniser)
-      throw new Error("model_or_tokeniser_not_initialized.");
-    this._trainer && t && this._trainer.trainingType !== t && (this._trainer.dispose(), this._trainer = null);
-    const e = this._trainer === null ? new h(this._model, this._tokeniser, t, i) : new h(this._trainer, i);
-    return e.on("start", () => {
-      this.setStatus("training"), this.phase = t === "sft" ? "finetuned" : "pretrained";
-    }), e.on("stop", () => this.setStatus("ready")), e.on("log", async (o) => {
-      const s = this.ee.listeners("trainStep");
-      for (const r of s)
-        await r(o);
-    }), this._trainer && this._trainer !== e && this._trainer.removeAllListeners(), this._trainer = e, e;
-  }
-  async train(t, i, e) {
-    const o = this.trainer(e, i);
-    await o.prepare(t), await o.train();
-  }
-  async trainTokeniser(t) {
-    if (!this._tokeniser)
-      throw new Error("tokeniser_not_initialized.");
-    const i = await this._tokeniser.train(t);
-    return this._status === "awaitingTokens" && this.setStatus("ready"), i;
-  }
-  generator() {
-    if (!this._model || !this._tokeniser)
-      throw new Error("model_or_tokeniser_not_initialized.");
-    const t = new f(this._model, this._tokeniser);
-    return t.on("start", () => {
-      this.status === "ready" && this.setStatus("busy");
-    }), t.on("stop", () => {
-      this.status === "busy" && this.setStatus("ready");
-    }), t;
-  }
-  generateText(t, i) {
-    return Array.isArray(t) ? this.generator().generate(t, i) : this.generator().generate([], i);
-  }
-  dispose() {
-    this._trainer && (this._trainer.dispose(), this._trainer = null), this._model?.dispose(), this.ee.removeAllListeners();
-  }
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  on(t, i) {
-    if (t === "loaded" && this.loaded) {
-      setTimeout(() => i(), 0);
-      return;
-    }
-    this.ee.on(t, i);
-  }
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  off(t, i) {
-    this.ee.off(t, i);
-  }
-}
-export {
-  l as default
-};
+import { i as e } from "./main-CPjeMv0G.js";
+export { e as default };

package/dist/Trainer.js

@@ -1,244 +1,2 @@
-import { E as w } from "./index-DvYrXKkX.js";
-import c from "./training/PreTrainer.js";
-import { createTrainValidationSplit as f } from "./training/validation.js";
-import g from "./training/SFTTrainer.js";
-const s = [];
-for (let n = 0; n < 256; ++n)
-  s.push((n + 256).toString(16).slice(1));
-function T(n, t = 0) {
-  return (s[n[t + 0]] + s[n[t + 1]] + s[n[t + 2]] + s[n[t + 3]] + "-" + s[n[t + 4]] + s[n[t + 5]] + "-" + s[n[t + 6]] + s[n[t + 7]] + "-" + s[n[t + 8]] + s[n[t + 9]] + "-" + s[n[t + 10]] + s[n[t + 11]] + s[n[t + 12]] + s[n[t + 13]] + s[n[t + 14]] + s[n[t + 15]]).toLowerCase();
-}
-const p = new Uint8Array(16);
-function k() {
-  return crypto.getRandomValues(p);
-}
-function m(n, t, e) {
-  return crypto.randomUUID ? crypto.randomUUID() : L(n);
-}
-function L(n, t, e) {
-  n = n || {};
-  const i = n.random ?? n.rng?.() ?? k();
-  if (i.length < 16)
-    throw new Error("Random bytes length must be >= 16");
-  return i[6] = i[6] & 15 | 64, i[8] = i[8] & 63 | 128, T(i);
-}
-class u extends w {
-  trainer;
-  trainingType = "pretraining";
-  hasTrained = !1;
-  trainDataset;
-  validationDataset;
-  totalTokens = 0;
-  tokensProcessed = 0;
-  log = [];
-  progress = null;
-  options = {
-    batchSize: 32,
-    sftMode: "full",
-    logInterval: 10
-  };
-  tokenizer;
-  constructor(t, e, i, a, o) {
-    if (super(), t instanceof u) {
-      const r = e || t.options, h = t.options;
-      let l = !1;
-      if (t.trainingType === "sft" && r.sftMode !== h.sftMode && (l = !0), t.trainer instanceof g && t.trainer.loraName && r.loraName !== t.trainer.loraName && (l = !0), i !== void 0 && i !== t.trainingType && (l = !0), l) {
-        if (t.trainingType === "sft") {
-          const d = new g(t.model, t.tokenizer, r);
-          this.trainer = d, d.loraName = r.loraName;
-        } else
-          this.trainer = new c(t.model, t.tokenizer, r);
-        this.trainingType = i || t.trainingType, this.options = r, this.tokenizer = t.tokenizer;
-      } else
-        this.trainer = t.trainer, this.trainingType = i || t.trainingType, this.options = r, this.trainer.updateOptimizer(this.options), this.log = t.log, this.progress = t.progress, this.totalTokens = t.totalTokens, this.tokenizer = t.tokenizer, r.batchSize === h.batchSize && (this.trainDataset = t.trainDataset, this.validationDataset = t.validationDataset);
-      return;
-    }
-    if (!e)
-      throw new Error("Tokeniser must be provided when initializing Trainer with a model");
-    if (!t)
-      throw new Error("Model must be provided when initializing Trainer");
-    if (this.options = a || {
-      batchSize: 32,
-      sftMode: "full"
-    }, i === "sft") {
-      const r = new g(t, e, a, o);
-      this.trainer = r, r.loraName = a?.loraName;
-    } else
-      this.trainer = new c(t, e, a, o);
-    this.trainingType = i || "pretraining", this.tokenizer = e;
-  }
-  get model() {
-    return this.trainer.model;
-  }
-  get optimizer() {
-    return this.trainer.optimizer;
-  }
-  stop() {
-    this.trainer.stop();
-  }
-  reset() {
-    this.hasTrained = !1, this.log = [], this.trainer.reset();
-  }
-  dispose() {
-    this.trainer.dispose(), this.removeAllListeners();
-  }
-  getTotalTokens() {
-    return this.totalTokens;
-  }
-  setOptions(t) {
-    const e = new Set(
-      Object.keys(t).filter(
-        (i) => t[i] !== this.options[i]
-      )
-    );
-    if (this.trainer.isRunning) {
-      if (e.has("batchSize"))
-        throw new Error("Cannot change batch size during training");
-      if (e.has("sftMode"))
-        throw new Error("Cannot change SFT mode during training");
-      if (e.has("loraConfig"))
-        throw new Error("Cannot change LoRA configuration during training");
-      if (e.has("validationSplit"))
-        throw new Error("Cannot change validation split during training");
-      if (e.has("trainableWeights"))
-        throw new Error("Cannot change trainable weights during training");
-      if (e.has("mixedPrecision"))
-        throw new Error("Cannot change mixed precision setting during training");
-      if (e.has("gradientCheckpointing"))
-        throw new Error("Cannot change gradient checkpointing setting during training");
-    }
-    this.options = {
-      ...this.options,
-      ...t
-    }, this.trainer.updateOptimizer(this.options), e.has("metrics") && this.trainer.setMetrics(t.metrics || []);
-  }
-  async prepare(t = [], e) {
-    const i = this.options;
-    if (e && (this.model.metaData.pretrainingData = e.map((a) => ({
-      id: a.id,
-      name: a.name
-    }))), this.trainingType === "pretraining" && this.trainer instanceof c) {
-      const { trainDataset: a, validationDataset: o, size: r } = await f(
-        t,
-        this.trainer.tokenizer,
-        this.trainer.datasetBuilder,
-        i?.batchSize || 32,
-        i?.validationSplit || 0.1,
-        i?.maskedLoss ?? !1
-      ), h = r * (1 - (i?.validationSplit || 0));
-      this.trainDataset = a, this.validationDataset = o, this.totalTokens = h, this.options.epochSteps = Math.ceil(
-        this.totalTokens / ((i?.batchSize || 32) * this.model.config.blockSize)
-      ), this.trainer.updateOptimizer(this.options);
-    } else if (this.trainingType === "sft" && this.trainer instanceof g) {
-      if (t instanceof Uint16Array)
-        throw new Error("SFT training requires Task[] input");
-      const { trainDataset: a, validationDataset: o, size: r } = await f(
-        t,
-        this.trainer.tokenizer,
-        this.trainer.datasetBuilder,
-        i?.batchSize || 32,
-        i?.validationSplit || 0.1,
-        i?.maskedLoss ?? !0
-      ), h = r * (1 - (i?.validationSplit || 0));
-      this.trainDataset = a, this.validationDataset = o, this.totalTokens = h, this.options.epochSteps = Math.ceil(
-        this.totalTokens / ((i?.batchSize || 32) * this.model.config.blockSize)
-      ), this.trainer.updateOptimizer(this.options);
-    }
-  }
-  configureModel(t) {
-    const e = t?.sftMode || "full";
-    if (this.trainingType === "pretraining" && (this.trainer.model.hasLoRA() && this.trainer.model.detachLoRA(), this.trainer.model.weightStore.setTrainable(["*"])), this.trainingType === "sft") {
-      if (e === "lora") {
-        const i = this.trainer.model;
-        if (console.log("Configuring model for LoRA fine-tuning with options:", t), t?.loraName)
-          if (i.hasLoRA(t.loraName)) {
-            if (i.attachLoRA(t.loraName), t.loraConfig) {
-              const a = i.lora;
-              (a.alpha !== t.loraConfig.alpha || a.rank !== t.loraConfig.rank) && (i.detachLoRA(), i.deleteLoRA(t.loraName), i.createLoRA(t.loraName, t.loraConfig), i.attachLoRA(t.loraName), console.warn("Resetting LoRA with new configuration."));
-            }
-          } else if (t.loraConfig)
-            i.createLoRA(t.loraName, t.loraConfig), i.attachLoRA(t.loraName);
-          else
-            throw new Error(
-              `LoRA configuration must be provided to create LoRA with name ${t.loraName}`
-            );
-        else if (t?.loraConfig)
-          if (i.hasLoRA()) {
-            const a = i.lora;
-            if (a.alpha !== t.loraConfig.alpha || a.rank !== t.loraConfig.rank) {
-              i.detachLoRA();
-              const o = t.loraName || m();
-              i.createLoRA(o, t.loraConfig), i.attachLoRA(o);
-            }
-          } else {
-            const a = t.loraName || m();
-            i.createLoRA(a, t.loraConfig), i.attachLoRA(a);
-          }
-        else if (!i.hasLoRA()) throw new Error("LoRA configuration must be provided for lora SFT mode");
-      } else
-        this.trainer.model.hasLoRA() && this.trainer.model.detachLoRA();
-      e === "last-layer" ? this.trainer.model.weightStore.setTrainable([
-        `block_${this.trainer.model.config.nLayer - 1}_*`,
-        "token_embedding"
-      ]) : e === "full" && this.trainer.model.weightStore.setTrainable(["*"]);
-    }
-    t?.trainableWeights && this.trainer.model.weightStore.setTrainable(t.trainableWeights);
-  }
-  async train() {
-    const t = this.options;
-    if (!this.trainDataset)
-      throw new Error("Dataset not prepared");
-    this.hasTrained || this.trainer.setLearningRate(t?.learningRate || 1e-3), this.hasTrained = !0, this.emit("start"), this.model.metaData.pretrainingSettings = t;
-    const e = Date.now();
-    this.log.length > 0 && this.trainer.resumeFromLog(this.log[this.log.length - 1]), this.trainer.setGradientCheckpointing(t?.gradientCheckpointing || !1), this.trainer.setMixedPrecision(t?.mixedPrecision || !1), this.trainer.setLabelSmoothing(t?.labelSmoothing || 0), this.trainer.setDropout(t?.dropout || 0), this.trainer.setLayerDrop(t?.layerDrop || 0), this.configureModel(t), await this.trainer.trainOnDataset(
-      this.trainDataset,
-      {
-        ...t,
-        onStep: async (a) => {
-          this.log.push(a), this.progress = {
-            lastLog: a,
-            progress: a.totalTokens / this.totalTokens,
-            remaining: Math.max(0, (this.totalTokens - a.totalTokens) / a.totalTokens * a.duration)
-          }, this.tokensProcessed = a.totalTokens;
-          const o = this.listeners("log");
-          for (const r of o)
-            await r(a, this.progress);
-        }
-      },
-      this.validationDataset
-    ), this.model.metaData.actionLog = this.model.metaData.actionLog || [];
-    const i = Date.now();
-    this.model.metaData.actionLog.push({
-      action: "pretrain",
-      timestamp: i,
-      duration: i - e,
-      tokensProcessed: this.tokensProcessed,
-      options: t
-    }), this.emit("stop");
-  }
-  async step(t) {
-    if (!this.trainDataset)
-      throw new Error("Dataset not prepared");
-    this.hasTrained || this.trainer.setLearningRate(t?.learningRate || 1e-3), this.hasTrained = !0, this.emit("start");
-    const { log: e } = await this.trainer.stepDataset(this.trainDataset, t || {}, this.validationDataset), i = this.listeners("log");
-    for (const a of i)
-      await a(e, {
-        lastLog: e,
-        progress: e.totalTokens / this.totalTokens,
-        remaining: Math.max(0, (this.totalTokens - e.totalTokens) / e.totalTokens * e.duration)
-      });
-    this.emit("stop");
-  }
-  getLog() {
-    return this.log;
-  }
-  getProgress() {
-    return this.progress;
-  }
-  isPrepared() {
-    return this.trainDataset !== void 0 && this.validationDataset !== void 0;
-  }
-}
-export {
-  u as default
-};
+import { a as e } from "./main-CPjeMv0G.js";
+export { e as default };

package/dist/backend.js

@@ -1,13 +1,13 @@
-import { g as o, s as e, r as s } from "./index-CUXkjxiT.js";
-async function c(t, a) {
-  if (o() !== t) {
-    if (t === "webgpu") {
-      const { registerWebGPUBackend: i } = await import("./patches/webgpu_base.js");
-      i(a), await import("./index-CjOWnMXP.js"), await import("./ops/webgpu/index.js");
-    }
-    await e(t), await s(), console.log(`Backend set to ${t}`);
-  }
+import { ai as e, ci as t, ui as n } from "./dist-BewPQWjc.js";
+//#region lib/backend.ts
+async function r(r, i) {
+	if (e() !== r) {
+		if (r === "webgpu") {
+			let { registerWebGPUBackend: e } = await import("./patches/webgpu_base.js");
+			e(i), await import("./dist-DVmq73nz.js"), await import("./ops/webgpu/index.js");
+		}
+		await n(r), await t(), console.log(`Backend set to ${r}`);
+	}
 }
-export {
-  c as selectBackend
-};
+//#endregion
+export { r as selectBackend };