@genai-fi/nanogpt 0.3.1 → 0.4.0

package/dist/softmax-Be_lsqUc.js

@@ -1,105 +0,0 @@
-import { o as r, h as e, E as c, _ as $, $ as h, a0 as E, s as S, Z as _, a1 as d } from "./index-pWA4_lUh.js";
-import { m as k, s as w, e as K } from "./sum-C7Mgy9Bw.js";
-import { r as l } from "./reshape-C8CR_Bad.js";
-/**
- * @license
- * Copyright 2018 Google LLC. All Rights Reserved.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * =============================================================================
- */
-function T(n) {
-  const s = { x: e(n, "x", "exp") };
-  return c.runKernel($, s);
-}
-const b = /* @__PURE__ */ r({ exp_: T });
-/**
- * @license
- * Copyright 2018 Google LLC. All Rights Reserved.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * =============================================================================
- */
-function L(n) {
-  const s = { x: e(n, "x", "log", "float32") };
-  return c.runKernel(h, s);
-}
-const N = /* @__PURE__ */ r({ log_: L });
-/**
- * @license
- * Copyright 2020 Google LLC. All Rights Reserved.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * =============================================================================
- */
-function v(n, o = null, s = !1) {
-  const t = e(n, "x", "logSumExp"), a = E(o, t.shape), p = k(
-    t,
-    a,
-    !0
-    /* keepDims */
-  ), i = S(t, p), f = b(i), m = w(f, a), u = N(m), x = _(l(p, u.shape), u);
-  if (s) {
-    const g = K(x.shape, a);
-    return l(x, g);
-  }
-  return x;
-}
-const I = /* @__PURE__ */ r({ logSumExp_: v });
-/**
- * @license
- * Copyright 2018 Google LLC. All Rights Reserved.
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * =============================================================================
- */
-function y(n, o = -1) {
-  const s = e(n, "logits", "softmax", "float32");
-  if (o === -1 && (o = s.rank - 1), o !== s.rank - 1)
-    throw Error(`Softmax along a non-last dimension is not yet supported. Logits was rank ${s.rank} and dim was ${o}`);
-  const t = { logits: s }, a = { dim: o };
-  return c.runKernel(d, t, a);
-}
-const M = /* @__PURE__ */ r({ softmax_: y });
-export {
-  N as a,
-  b as e,
-  I as l,
-  M as s
-};

package/dist/training/LayerTrainer.d.ts

@@ -1,29 +0,0 @@
-import { ITokeniser } from '../tokeniser/type';
-import { default as NanoGPT } from '../NanoGPTModel';
-import { default as GPTTrainer, TrainingOptions } from './Trainer';
-import { Tensor } from '@tensorflow/tfjs-core';
-import { Dataset } from '@tensorflow/tfjs-data';
-interface LayerTrainingOptions extends TrainingOptions {
-    stepsPerLayer: number;
-    maxPasses: number;
-    onLayerChange?: (layer: number, pass: number, valLoss?: number) => Promise<void> | void;
-    onPassComplete?: (pass: number) => Promise<void> | void;
-}
-export default class LayerTrainer extends GPTTrainer {
-    private trainingPattern;
-    private startPass;
-    private startLayer;
-    constructor(model: NanoGPT, tokenizer: ITokeniser, learningRate?: number);
-    private applyTrainingPattern;
-    trainOnDataset(dataset: Dataset<{
-        xs: Tensor;
-        ys: Tensor;
-    }>, options: Partial<LayerTrainingOptions>, validationDataset?: Dataset<{
-        xs: Tensor;
-        ys: Tensor;
-    }>): Promise<{
-        losses: number[];
-        validationLosses: number[];
-    }>;
-}
-export {};

package/dist/training/LayerTrainer.js

@@ -1,90 +0,0 @@
-import { generateText as u } from "../utilities/generate.js";
-import v from "./Trainer.js";
-import { schedule as w } from "./lwSchedule.js";
-import T from "./Evaluator.js";
-import { a as m } from "../index-pWA4_lUh.js";
-const x = {
-  desiredLoss: 0.01,
-  logInterval: 1,
-  stepsPerLayer: 400,
-  maxPasses: 3,
-  maxSteps: 1e3
-};
-class E extends v {
-  trainingPattern = [];
-  startPass = 0;
-  startLayer = 0;
-  constructor(s, r, e = 3e-4) {
-    if (super(s, r, e), this.trainingPattern = w[s.config.nLayer - 1] || [], s.log.length > 0) {
-      const i = s.log[s.log.length - 1];
-      i.pass !== void 0 && i.layer !== void 0 && (this.startPass = i.pass, this.startLayer = i.layer, console.log(`Resuming training from pass ${this.startPass}, layer ${this.startLayer}`));
-    }
-  }
-  applyTrainingPattern(s) {
-    const r = s < this.trainingPattern.length ? s : this.trainingPattern.length - 1, e = this.trainingPattern[r];
-    this.model.setSkipMask(e.skip), this.model.setTrainableMask(e.trainable), this.resetOptimizer(e.adam), console.log("Applied training pattern:", r, e);
-  }
-  // Train for multiple epochs using Dataset API - FIXED memory leaks
-  async trainOnDataset(s, r, e) {
-    const { desiredLoss: i, logInterval: L, stepsPerLayer: f, onLayerChange: o, onPassComplete: p, onStep: h, prompt: c } = {
-      ...x,
-      ...r
-    }, t = {
-      pass: 0,
-      layerStep: 0,
-      step: 0,
-      stepSinceLayerChange: 0,
-      lastLoss: 1e6,
-      totalSteps: 0,
-      losses: [],
-      validationLosses: []
-    };
-    this.dummyPass();
-    const d = Date.now();
-    this.startPass = 0, this.startLayer = 0;
-    const g = e ? new T(this.model, e) : void 0, P = await s.iterator();
-    this.applyTrainingPattern(t.layerStep % this.trainingPattern.length);
-    try {
-      for (; !(t.lastLoss < i); ) {
-        const n = await P.next();
-        if (n.done) break;
-        const y = n.value, S = this.trainBatch(t, y);
-        t.stepSinceLayerChange++;
-        const l = {
-          loss: t.lastLoss,
-          step: t.step,
-          time: Date.now() - d,
-          batchSize: y.xs.shape[0],
-          pass: t.pass,
-          layer: t.layerStep % this.model.config.nLayer
-        };
-        if (this.model.log.push(l), t.step % L === 0) {
-          if (await S, g)
-            try {
-              const a = await g.evaluate(5);
-              t.validationLosses.push(a), l.valLoss = a;
-            } catch (a) {
-              console.error("Validation error:", a);
-            }
-          if (h) {
-            if (c) {
-              const a = await u(this.tokenizer, this.model, c, 100, {
-                temperature: 0.8,
-                topK: 10
-              });
-              l.example = a;
-            }
-            await h(l);
-          }
-        }
-        t.stepSinceLayerChange >= f && (t.layerStep++, t.layerStep % this.model.config.nLayer === 0 ? (o && await o(t.layerStep, t.pass), p && await p(t.pass), t.pass++) : o && await o(t.layerStep, t.pass), t.stepSinceLayerChange = 0, this.applyTrainingPattern(t.layerStep % this.trainingPattern.length));
-      }
-    } catch (n) {
-      throw console.error("Training error:", n), m(), n;
-    }
-    return m(), { losses: t.losses, validationLosses: t.validationLosses };
-  }
-}
-export {
-  E as default
-};

package/dist/training/lwSchedule.d.ts

@@ -1,7 +0,0 @@
-import { AdamConfig } from './Trainer';
-export interface LWSchedule {
-    adam: AdamConfig;
-    skip: boolean[];
-    trainable: boolean[];
-}
-export declare const schedule: LWSchedule[][];

package/dist/training/lwSchedule.js

@@ -1,162 +0,0 @@
-const e = [
-  [
-    {
-      adam: {
-        learningRateFactor: 1,
-        beta1: 0.9,
-        beta2: 0.999,
-        epsilon: 1e-8
-      },
-      skip: [!1],
-      trainable: [!0]
-    }
-  ],
-  [
-    {
-      adam: {
-        learningRateFactor: 1,
-        beta1: 0.9,
-        beta2: 0.999,
-        epsilon: 1e-8
-      },
-      skip: [!0, !1],
-      trainable: [!1, !0]
-    },
-    {
-      adam: {
-        learningRateFactor: 1,
-        beta1: 0.9,
-        beta2: 0.999,
-        epsilon: 1e-8
-      },
-      skip: [!1, !1],
-      trainable: [!0, !1]
-    },
-    {
-      adam: {
-        learningRateFactor: 0.3333333333333333,
-        beta1: 0.95,
-        beta2: 0.999,
-        epsilon: 1e-8
-      },
-      skip: [!1, !1],
-      trainable: [!0, !0]
-    }
-  ],
-  [],
-  [
-    {
-      adam: {
-        learningRateFactor: 1,
-        beta1: 0.9,
-        beta2: 0.999,
-        epsilon: 1e-8
-      },
-      skip: [!0, !0, !0, !1],
-      trainable: [!1, !1, !1, !0]
-    },
-    {
-      adam: {
-        learningRateFactor: 1,
-        beta1: 0.9,
-        beta2: 0.999,
-        epsilon: 1e-8
-      },
-      skip: [!0, !0, !1, !1],
-      trainable: [!1, !1, !0, !1]
-    },
-    {
-      adam: {
-        learningRateFactor: 0.3333333333333333,
-        beta1: 0.95,
-        beta2: 0.999,
-        epsilon: 1e-8
-      },
-      skip: [!0, !0, !1, !1],
-      trainable: [!1, !1, !1, !0]
-    },
-    {
-      adam: {
-        learningRateFactor: 1,
-        beta1: 0.9,
-        beta2: 0.999,
-        epsilon: 1e-8
-      },
-      skip: [!0, !1, !1, !1],
-      trainable: [!1, !0, !1, !1]
-    },
-    {
-      adam: {
-        learningRateFactor: 0.3333333333333333,
-        beta1: 0.95,
-        beta2: 0.999,
-        epsilon: 1e-8
-      },
-      skip: [!0, !1, !1, !1],
-      trainable: [!1, !1, !0, !1]
-    },
-    {
-      adam: {
-        learningRateFactor: 0.16666666666666666,
-        beta1: 0.98,
-        beta2: 0.9999,
-        epsilon: 1e-8
-      },
-      skip: [!0, !1, !1, !1],
-      trainable: [!1, !1, !1, !0]
-    },
-    {
-      adam: {
-        learningRateFactor: 1,
-        beta1: 0.9,
-        beta2: 0.999,
-        epsilon: 1e-8
-      },
-      skip: [!1, !1, !1, !1],
-      trainable: [!0, !1, !1, !1]
-    },
-    {
-      adam: {
-        learningRateFactor: 0.3333333333333333,
-        beta1: 0.95,
-        beta2: 0.999,
-        epsilon: 1e-8
-      },
-      skip: [!1, !1, !1, !1],
-      trainable: [!1, !0, !1, !1]
-    },
-    {
-      adam: {
-        learningRateFactor: 0.16666666666666666,
-        beta1: 0.98,
-        beta2: 0.9999,
-        epsilon: 1e-8
-      },
-      skip: [!1, !1, !1, !1],
-      trainable: [!1, !1, !0, !1]
-    },
-    {
-      adam: {
-        learningRateFactor: 0.16666666666666666,
-        beta1: 0.98,
-        beta2: 0.9999,
-        epsilon: 1e-8
-      },
-      skip: [!1, !1, !1, !1],
-      trainable: [!1, !1, !1, !0]
-    },
-    {
-      adam: {
-        learningRateFactor: 0.16666666666666666,
-        beta1: 0.98,
-        beta2: 0.9999,
-        epsilon: 1e-8
-      },
-      skip: [!1, !1, !1, !1],
-      trainable: [!0, !0, !0, !0]
-    }
-  ]
-];
-export {
-  e as schedule
-};