@genai-fi/nanogpt 0.3.2 → 0.4.1

package/dist/training/LayerTrainer.js

@@ -1,95 +0,0 @@
-import { generateText as u } from "../utilities/generate.js";
-import v from "./Trainer.js";
-import { schedule as w } from "./lwSchedule.js";
-import T from "./Evaluator.js";
-import { a as m } from "../index-pWA4_lUh.js";
-const x = {
-  desiredLoss: 0.01,
-  logInterval: 1,
-  stepsPerLayer: 400,
-  maxPasses: 3,
-  maxSteps: 1e3
-};
-class E extends v {
-  trainingPattern = [];
-  startPass = 0;
-  startLayer = 0;
-  constructor(a, r, e = 3e-4) {
-    if (super(a, r, e), this.trainingPattern = w[a.config.nLayer - 1] || [], a.log.length > 0) {
-      const i = a.log[a.log.length - 1];
-      i.pass !== void 0 && i.layer !== void 0 && (this.startPass = i.pass, this.startLayer = i.layer, console.log(`Resuming training from pass ${this.startPass}, layer ${this.startLayer}`));
-    }
-  }
-  applyTrainingPattern(a) {
-    const r = a < this.trainingPattern.length ? a : this.trainingPattern.length - 1, e = this.trainingPattern[r];
-    this.model.setSkipMask(e.skip), this.model.setTrainableMask(e.trainable), this.resetOptimizer(e.adam), console.log("Applied training pattern:", r, e);
-  }
-  // Train for multiple epochs using Dataset API - FIXED memory leaks
-  async trainOnDataset(a, r, e) {
-    const { desiredLoss: i, logInterval: L, stepsPerLayer: d, onLayerChange: l, onPassComplete: p, onStep: h, prompt: c } = {
-      ...x,
-      ...r
-    }, t = {
-      pass: 0,
-      layerStep: 0,
-      step: 0,
-      stepSinceLayerChange: 0,
-      lastLoss: 1e6,
-      totalSteps: 0,
-      losses: [],
-      validationLosses: [],
-      trainingDuration: 0
-    };
-    this.dummyPass();
-    const S = Date.now();
-    this.startPass = 0, this.startLayer = 0;
-    const g = e ? new T(this.model, e) : void 0, f = await a.iterator();
-    this.applyTrainingPattern(t.layerStep % this.trainingPattern.length);
-    try {
-      for (; !(t.lastLoss < i); ) {
-        const o = await f.next();
-        if (o.done) break;
-        const y = o.value, P = this.trainBatch(t, y);
-        t.stepSinceLayerChange++;
-        const n = {
-          loss: t.lastLoss,
-          step: t.step,
-          time: Date.now() - S,
-          batchSize: y.xs.shape[0],
-          pass: t.pass,
-          layer: t.layerStep % this.model.config.nLayer
-        };
-        if (this.model.log.push(n), t.step % L === 0) {
-          if (await P, g)
-            try {
-              const s = await g.evaluate(5);
-              t.validationLosses.push(s), n.valLoss = s;
-            } catch (s) {
-              console.error("Validation error:", s);
-            }
-          if (h) {
-            if (c) {
-              const s = await u(this.tokenizer, this.model, c, 100, {
-                temperature: 0.8,
-                topK: 10
-              });
-              n.example = s;
-            }
-            await h(n, {
-              duration: t.trainingDuration,
-              totalSamples: t.totalSteps * n.batchSize,
-              samplesPerSecond: t.totalSteps * n.batchSize / (t.trainingDuration / 1e3)
-            });
-          }
-        }
-        t.stepSinceLayerChange >= d && (t.layerStep++, t.layerStep % this.model.config.nLayer === 0 ? (l && await l(t.layerStep, t.pass), p && await p(t.pass), t.pass++) : l && await l(t.layerStep, t.pass), t.stepSinceLayerChange = 0, this.applyTrainingPattern(t.layerStep % this.trainingPattern.length));
-      }
-    } catch (o) {
-      throw console.error("Training error:", o), m(), o;
-    }
-    return m(), { losses: t.losses, validationLosses: t.validationLosses };
-  }
-}
-export {
-  E as default
-};

package/dist/training/lwSchedule.d.ts

@@ -1,7 +0,0 @@
-import { AdamConfig } from './Trainer';
-export interface LWSchedule {
-    adam: AdamConfig;
-    skip: boolean[];
-    trainable: boolean[];
-}
-export declare const schedule: LWSchedule[][];

package/dist/training/lwSchedule.js

@@ -1,162 +0,0 @@
-const e = [
-  [
-    {
-      adam: {
-        learningRateFactor: 1,
-        beta1: 0.9,
-        beta2: 0.999,
-        epsilon: 1e-8
-      },
-      skip: [!1],
-      trainable: [!0]
-    }
-  ],
-  [
-    {
-      adam: {
-        learningRateFactor: 1,
-        beta1: 0.9,
-        beta2: 0.999,
-        epsilon: 1e-8
-      },
-      skip: [!0, !1],
-      trainable: [!1, !0]
-    },
-    {
-      adam: {
-        learningRateFactor: 1,
-        beta1: 0.9,
-        beta2: 0.999,
-        epsilon: 1e-8
-      },
-      skip: [!1, !1],
-      trainable: [!0, !1]
-    },
-    {
-      adam: {
-        learningRateFactor: 0.3333333333333333,
-        beta1: 0.95,
-        beta2: 0.999,
-        epsilon: 1e-8
-      },
-      skip: [!1, !1],
-      trainable: [!0, !0]
-    }
-  ],
-  [],
-  [
-    {
-      adam: {
-        learningRateFactor: 1,
-        beta1: 0.9,
-        beta2: 0.999,
-        epsilon: 1e-8
-      },
-      skip: [!0, !0, !0, !1],
-      trainable: [!1, !1, !1, !0]
-    },
-    {
-      adam: {
-        learningRateFactor: 1,
-        beta1: 0.9,
-        beta2: 0.999,
-        epsilon: 1e-8
-      },
-      skip: [!0, !0, !1, !1],
-      trainable: [!1, !1, !0, !1]
-    },
-    {
-      adam: {
-        learningRateFactor: 0.3333333333333333,
-        beta1: 0.95,
-        beta2: 0.999,
-        epsilon: 1e-8
-      },
-      skip: [!0, !0, !1, !1],
-      trainable: [!1, !1, !1, !0]
-    },
-    {
-      adam: {
-        learningRateFactor: 1,
-        beta1: 0.9,
-        beta2: 0.999,
-        epsilon: 1e-8
-      },
-      skip: [!0, !1, !1, !1],
-      trainable: [!1, !0, !1, !1]
-    },
-    {
-      adam: {
-        learningRateFactor: 0.3333333333333333,
-        beta1: 0.95,
-        beta2: 0.999,
-        epsilon: 1e-8
-      },
-      skip: [!0, !1, !1, !1],
-      trainable: [!1, !1, !0, !1]
-    },
-    {
-      adam: {
-        learningRateFactor: 0.16666666666666666,
-        beta1: 0.98,
-        beta2: 0.9999,
-        epsilon: 1e-8
-      },
-      skip: [!0, !1, !1, !1],
-      trainable: [!1, !1, !1, !0]
-    },
-    {
-      adam: {
-        learningRateFactor: 1,
-        beta1: 0.9,
-        beta2: 0.999,
-        epsilon: 1e-8
-      },
-      skip: [!1, !1, !1, !1],
-      trainable: [!0, !1, !1, !1]
-    },
-    {
-      adam: {
-        learningRateFactor: 0.3333333333333333,
-        beta1: 0.95,
-        beta2: 0.999,
-        epsilon: 1e-8
-      },
-      skip: [!1, !1, !1, !1],
-      trainable: [!1, !0, !1, !1]
-    },
-    {
-      adam: {
-        learningRateFactor: 0.16666666666666666,
-        beta1: 0.98,
-        beta2: 0.9999,
-        epsilon: 1e-8
-      },
-      skip: [!1, !1, !1, !1],
-      trainable: [!1, !1, !0, !1]
-    },
-    {
-      adam: {
-        learningRateFactor: 0.16666666666666666,
-        beta1: 0.98,
-        beta2: 0.9999,
-        epsilon: 1e-8
-      },
-      skip: [!1, !1, !1, !1],
-      trainable: [!1, !1, !1, !0]
-    },
-    {
-      adam: {
-        learningRateFactor: 0.16666666666666666,
-        beta1: 0.98,
-        beta2: 0.9999,
-        epsilon: 1e-8
-      },
-      skip: [!1, !1, !1, !1],
-      trainable: [!0, !0, !0, !0]
-    }
-  ]
-];
-export {
-  e as schedule
-};