@genai-fi/nanogpt 0.19.1 → 0.20.0

package/dist/training/PreTrainer.d.ts

@@ -1,11 +0,0 @@
-import { default as Model, ModelForwardAttributes } from '../models/model';
-import { default as BasicTrainer } from './BasicTrainer';
-import { ITokeniser } from '../tokeniser/type';
-import { DatasetBuilder } from './DatasetBuilder';
-import { AdamWOptimizer } from './AdamW';
-import { AdamWOptimizerConfig } from './types';
-export default class PreTrainer extends BasicTrainer {
-    tokenizer: ITokeniser;
-    datasetBuilder: DatasetBuilder;
-    constructor(model: Model<ModelForwardAttributes>, tokenizer: ITokeniser, optConfig?: Partial<AdamWOptimizerConfig>, optimizer?: AdamWOptimizer);
-}

package/dist/training/PreTrainer.js

@@ -1,20 +0,0 @@
-import r from "./BasicTrainer.js";
-import { DatasetBuilder as n } from "./DatasetBuilder.js";
-const s = {
-  decayEpochs: 100,
-  epochSteps: 1e4,
-  warmupSteps: 1e3,
-  minLearningRate: 3e-5,
-  weightDecay: 0.1,
-  learningRate: 3e-4
-  // clipNorm: 1.0,
-};
-class m extends r {
-  constructor(i, e, t, a) {
-    super(i, e, { ...s, ...t }, a), this.tokenizer = e, this.optimizerConfig.minLearningRate = t?.minLearningRate ?? this.optimizerConfig.learningRate / 20, this.updateOptimizer(), this.datasetBuilder = new n(e, i.config.blockSize);
-  }
-  datasetBuilder;
-}
-export {
-  m as default
-};

package/dist/training/SFTTrainer.d.ts

@@ -1,12 +0,0 @@
-import { default as Model, ModelForwardAttributes } from '../models/model';
-import { default as BasicTrainer } from './BasicTrainer';
-import { ITokeniser } from '../tokeniser/type';
-import { AdamWOptimizer } from './AdamW';
-import { AdamWOptimizerConfig } from './types';
-import { DatasetBuilder } from './DatasetBuilder';
-export default class SFTTrainer extends BasicTrainer {
-    tokenizer: ITokeniser;
-    datasetBuilder: DatasetBuilder;
-    loraName?: string;
-    constructor(model: Model<ModelForwardAttributes>, tokenizer: ITokeniser, optConfig?: Partial<AdamWOptimizerConfig>, optimizer?: AdamWOptimizer);
-}

package/dist/training/SFTTrainer.js

@@ -1,22 +0,0 @@
-import r from "./BasicTrainer.js";
-import { DatasetBuilder as s } from "./DatasetBuilder.js";
-const n = {
-  decayEpochs: 100,
-  epochSteps: 1e4,
-  warmupSteps: 100,
-  minLearningRate: 1e-5,
-  weightDecay: 0.1,
-  beta2: 0.95,
-  learningRate: 3e-4
-  // clipNorm: 1.0,
-};
-class c extends r {
-  constructor(t, e, i, a) {
-    super(t, e, { ...n, ...i }, a), this.tokenizer = e, this.optimizerConfig.minLearningRate = i?.minLearningRate ?? this.optimizerConfig.learningRate / 20, this.updateOptimizer(), this.datasetBuilder = new s(e, t.config.blockSize), this.maskedLoss = !0;
-  }
-  datasetBuilder;
-  loraName;
-}
-export {
-  c as default
-};

package/dist/training/loss.d.ts

@@ -1,3 +0,0 @@
-import { Tensor } from '@tensorflow/tfjs-core';
-export declare function calculateLoss(logits: Tensor, targets: Tensor, masked?: boolean, keepBatch?: boolean, labelSmoothing?: number): Tensor;
-export declare function calculateAccuracy(logits: Tensor, targets: Tensor): Tensor;

package/dist/training/loss.js

@@ -1,24 +0,0 @@
-import { createSoftmaxCrossEntropyWithGrad as n } from "./sparseCrossEntropy.js";
-function i(t, s, r, c, o) {
-  try {
-    return n(
-      r,
-      c,
-      o && o > 0 ? o : void 0
-    )(t, s);
-  } catch (a) {
-    throw console.error("Error computing loss:", a), new Error(`Loss computation failed: ${a}`);
-  }
-}
-function l(t, s) {
-  try {
-    const r = t.argMax(-1), c = r.equal(s).cast("float32"), o = c.mean();
-    return r.dispose(), c.dispose(), o;
-  } catch (r) {
-    throw console.error("Error computing accuracy:", r), new Error(`Accuracy computation failed: ${r}`);
-  }
-}
-export {
-  l as calculateAccuracy,
-  i as calculateLoss
-};

package/dist/training/orthoGrad.d.ts

@@ -1,2 +0,0 @@
-import { Tensor } from '@tensorflow/tfjs-core';
-export declare function orthogonalizeGradient(weight: Tensor, gradient: Tensor, epsilon: number): Tensor;

package/dist/training/orthoGrad.js

@@ -1,10 +0,0 @@
-import { t as d } from "../index-CUXkjxiT.js";
-function l(m, t, s) {
-  return d(() => {
-    const r = m.reshape([-1]), o = t.reshape([-1]), e = r.mul(r).sum().add(s), c = r.mul(o).sum().div(e), n = o.sub(r.mul(c)), a = o.norm(), u = n.norm().add(s);
-    return n.mul(a.div(u)).reshape(t.shape);
-  });
-}
-export {
-  l as orthogonalizeGradient
-};

package/dist/training/sparseCrossEntropy.d.ts

@@ -1,7 +0,0 @@
-import * as tf from '@tensorflow/tfjs-core';
-/**
- * Numerically stable sparse cross-entropy with gradient support
- * This version handles potential numerical issues better
- */
-export declare function sparseSoftmaxCrossEntropy(logits: tf.Tensor, labels: tf.Tensor, validMask?: tf.Tensor, keepBatch?: boolean, originalBatchShape?: number[], labelSmoothing?: number): tf.Tensor;
-export declare function createSoftmaxCrossEntropyWithGrad(masked?: boolean, keepBatch?: boolean, labelSmoothing?: number): (...args: tf.Tensor[]) => tf.Tensor<tf.Rank>;

package/dist/training/sparseCrossEntropy.js

@@ -1,69 +0,0 @@
-import { gatherSub as $ } from "../ops/gatherSub.js";
-import { scatterSub as T } from "../ops/scatterSub.js";
-import { t as F, a1 as G, j as A, m as x, i as U, a2 as D, a as j, a3 as q } from "../index-CUXkjxiT.js";
-import { m as J, l as K, a as E, n as N, w as O } from "../not_equal-hurPF26l.js";
-import { s as m } from "../sum-BpiwSWvg.js";
-import { s as Q } from "../softmax-D4q1LJN7.js";
-function R(e, l, a, g, v, t = 0) {
-  return F(() => {
-    const S = e.shape[e.shape.length - 1], i = v || e.shape.slice(0, -1), f = i.reduce((c, d) => c * d, 1), h = e.shape.length > 2 ? e.reshape([f, S]) : e, y = l.shape.length > 1 ? l.reshape([f]).cast("int32") : l.cast("int32"), u = J(h, -1, !0), o = G(h, u), n = K(o, -1), p = $(n, y, o);
-    let s;
-    if (t > 0) {
-      const c = G(n, E(o, -1));
-      s = A(x(p, 1 - t), x(c, t));
-    } else
-      s = p;
-    if (a)
-      if (s = x(s, a), g) {
-        const c = m(a.reshape(i), -1);
-        s = U(m(s.reshape(i), -1), c);
-      } else {
-        const c = m(a);
-        s = U(m(s), c);
-      }
-    else
-      g ? s = E(s.reshape(i), -1) : s = E(s);
-    return s;
-  });
-}
-function as(e, l, a = 0) {
-  return D(
-    // @ts-expect-error Invalid params
-    (t, S, i) => {
-      const f = t.shape[t.shape.length - 1], h = t.shape.slice(0, -1), y = h.reduce((d, r) => d * r, 1), u = t.reshape([y, f]), o = S.reshape([y]).cast("int32");
-      let n, p = null;
-      if (e) {
-        const d = j(65535, "int32"), r = N(o, d);
-        p = r.cast("float32"), n = O(r, o, q(o)), d.dispose(), r.dispose();
-      } else
-        n = o;
-      const s = R(
-        u,
-        n,
-        p || void 0,
-        l,
-        h,
-        a
-      );
-      return i(p ? [u, n, p] : [u, n]), u.dispose(), o.dispose(), { value: s, gradFunc: (d, r) => F(() => {
-        const C = r[0], z = r[1], b = e ? r[2] : void 0, w = Q(C), H = b ? m(b) : j(C.shape[0], "float32"), I = d.div(H).broadcastTo([C.shape[0]]), M = b && e ? x(I, b) : I;
-        let L;
-        if (a > 0) {
-          const V = a / f, W = G(w, V);
-          L = T(
-            W,
-            z,
-            x(M, 1 - a)
-          );
-        } else
-          L = T(w, z, M);
-        const P = q(S);
-        return [L.reshape(t.shape), P];
-      }) };
-    }
-  );
-}
-export {
-  as as createSoftmaxCrossEntropyWithGrad,
-  R as sparseSoftmaxCrossEntropy
-};

package/dist/training/tasks/ConversationTask.d.ts

@@ -1,18 +0,0 @@
-import { Conversation, ITokeniser } from '../../main';
-import { Task } from './Task';
-export default class ConversationTask extends Task {
-    private rawConvo;
-    private shuffledIndices;
-    private index;
-    get length(): number;
-    constructor(conversations: Conversation[][]);
-    hasMoreConversations(): boolean;
-    nextConversation(): Conversation[] | null;
-    nextTokens(tokeniser: ITokeniser): number[] | null;
-    nextTokens(tokeniser: ITokeniser, masking: boolean): {
-        tokens: number[];
-        mask: boolean[];
-    } | null;
-    shuffle(): void;
-    estimateTokens(tokeniser: ITokeniser): Promise<number>;
-}

package/dist/training/tasks/ConversationTask.js

@@ -1,40 +0,0 @@
-import { Task as t } from "./Task.js";
-import { shuffle as i } from "../DatasetBuilder.js";
-class f extends t {
-  rawConvo;
-  shuffledIndices = null;
-  index = 0;
-  get length() {
-    return this.rawConvo.length;
-  }
-  constructor(n) {
-    super(), this.rawConvo = n;
-  }
-  hasMoreConversations() {
-    return this.index < this.rawConvo.length;
-  }
-  nextConversation() {
-    if (this.index >= this.rawConvo.length)
-      return null;
-    const n = this.rawConvo[this.shuffledIndices ? this.shuffledIndices[this.index] : this.index];
-    return this.index++, n;
-  }
-  nextTokens(n, s) {
-    const e = this.nextConversation();
-    return e ? n.encodeConversation(e, !1, s) : null;
-  }
-  shuffle() {
-    if (!this.shuffledIndices) {
-      this.shuffledIndices = new Uint32Array(this.rawConvo.length);
-      for (let n = 0; n < this.rawConvo.length; n++)
-        this.shuffledIndices[n] = n;
-    }
-    i(this.shuffledIndices), this.index = 0;
-  }
-  async estimateTokens(n) {
-    return n.encodeConversation(this.rawConvo[0]).length * this.length;
-  }
-}
-export {
-  f as default
-};

package/dist/training/tasks/PretrainingTask.d.ts

@@ -1,17 +0,0 @@
-import { Conversation, ITokeniser } from '../../main';
-import { Task } from './Task';
-export default class PretrainingTask extends Task {
-    private rawText;
-    private index;
-    get length(): number;
-    constructor(texts: string[]);
-    hasMoreConversations(): boolean;
-    nextConversation(): Conversation[] | null;
-    nextTokens(tokeniser: ITokeniser): number[] | null;
-    nextTokens(tokeniser: ITokeniser, masking: boolean): {
-        tokens: number[];
-        mask: boolean[];
-    } | null;
-    shuffle(): void;
-    estimateTokens(tokeniser: ITokeniser): Promise<number>;
-}

package/dist/training/tasks/PretrainingTask.js

@@ -1,47 +0,0 @@
-import { Task as r } from "./Task.js";
-class h extends r {
-  rawText;
-  index = 0;
-  get length() {
-    return this.rawText.length;
-  }
-  constructor(t) {
-    super(), this.rawText = t;
-  }
-  hasMoreConversations() {
-    return this.index < this.rawText.length;
-  }
-  nextConversation() {
-    if (this.index >= this.rawText.length)
-      return null;
-    const t = {
-      role: "assistant",
-      content: this.rawText[this.index]
-    };
-    return this.index++, [t];
-  }
-  nextTokens(t, n) {
-    if (this.index >= this.rawText.length)
-      return null;
-    const e = t.encodeSequence(this.rawText[this.index]);
-    if (this.index++, n) {
-      const s = new Array(e.length).fill(!0);
-      return { tokens: e, mask: s };
-    }
-    return e;
-  }
-  shuffle() {
-    this.index = 0;
-  }
-  async estimateTokens(t) {
-    return t.encodeConversation([
-      {
-        role: "assistant",
-        content: this.rawText[0]
-      }
-    ]).length * this.length;
-  }
-}
-export {
-  h as default
-};

package/dist/training/tasks/StartSentenceTask.d.ts

@@ -1,18 +0,0 @@
-import { Conversation, ITokeniser } from '../../main';
-import { Task } from './Task';
-export default class StartSentenceTask extends Task {
-    private rawText;
-    private index;
-    get length(): number;
-    constructor(texts: string[]);
-    hasMoreConversations(): boolean;
-    nextConversation(): Conversation[] | null;
-    nextTokens(tokeniser: ITokeniser): number[] | null;
-    nextTokens(tokeniser: ITokeniser, masking: boolean): {
-        tokens: number[];
-        mask: boolean[];
-    } | null;
-    shuffle(): void;
-    private conversationFromString;
-    estimateTokens(tokeniser: ITokeniser): Promise<number>;
-}

package/dist/training/tasks/StartSentenceTask.js

@@ -1,49 +0,0 @@
-import { Task as r } from "./Task.js";
-class a extends r {
-  rawText;
-  index = 0;
-  get length() {
-    return this.rawText.length;
-  }
-  constructor(t) {
-    super(), this.rawText = t;
-  }
-  hasMoreConversations() {
-    return this.index < this.rawText.length;
-  }
-  nextConversation() {
-    if (this.index >= this.rawText.length)
-      return null;
-    const t = this.rawText[this.index];
-    return this.index++, this.conversationFromString(t);
-  }
-  nextTokens(t, n) {
-    const e = this.nextConversation();
-    return e ? t.encodeConversation(e, !1, n) : null;
-  }
-  shuffle() {
-    this.index = 0;
-  }
-  conversationFromString(t) {
-    const n = t.indexOf(".");
-    return n === -1 ? [{
-      role: "assistant",
-      content: this.rawText[this.index]
-    }] : [
-      {
-        role: "user",
-        content: t.slice(0, n + 1).trim()
-      },
-      {
-        role: "assistant",
-        content: t.slice(n + 1).trim()
-      }
-    ];
-  }
-  async estimateTokens(t) {
-    return (await t.encodeConversation(this.conversationFromString(this.rawText[0]))).length * this.length;
-  }
-}
-export {
-  a as default
-};

package/dist/training/tasks/Task.d.ts

@@ -1,22 +0,0 @@
-import { Conversation, ITokeniser } from '../../main';
-export declare abstract class Task {
-    abstract get length(): number;
-    abstract hasMoreConversations(): boolean;
-    abstract nextConversation(): Conversation[] | null;
-    abstract nextTokens(tokeniser: ITokeniser): number[] | null;
-    abstract nextTokens(tokeniser: ITokeniser, masking: boolean): {
-        tokens: number[];
-        mask: boolean[];
-    } | null;
-    abstract nextTokens(tokeniser: ITokeniser, masking?: boolean): number[] | {
-        tokens: number[];
-        mask: boolean[];
-    } | null;
-    abstract estimateTokens(tokeniser: ITokeniser): Promise<number>;
-    abstract shuffle(): void;
-}
-export declare function tokensFromTasks(tasks: Task[], tokenizer: ITokeniser, cb?: (tokens: number) => void): Promise<Uint16Array>;
-export declare function tokensFromTasks(tasks: Task[], tokenizer: ITokeniser, cb?: (tokens: number) => void, masking?: boolean): Promise<{
-    tokens: Uint16Array;
-    mask: Uint8Array;
-}>;

package/dist/training/tasks/Task.js

@@ -1,68 +0,0 @@
-import { yieldIfNeeded as k } from "../../utilities/yielder.js";
-class w {
-}
-function m(y, i, u, o, g, n) {
-  for (const a of y) {
-    const e = a.nextTokens(u, n ? !0 : void 0);
-    if (e) {
-      const f = Array.isArray(e) ? e : e.tokens;
-      o.total += f.length;
-      const c = i[i.length - 1], l = n ? n[n.length - 1] : null;
-      if (o.offset + f.length > c.length) {
-        const s = c.length - o.offset;
-        c.set(f.slice(0, s), o.offset);
-        const t = new Uint16Array(Math.floor(g * 0.1) + 100);
-        if (t.set(f.slice(s), 0), i.push(t), n && l && !Array.isArray(e)) {
-          l.set(
-            e.mask.slice(0, s).map((h) => h ? 1 : 0),
-            o.offset
-          );
-          const r = new Uint8Array(t.length);
-          r.set(
-            e.mask.slice(s).map((h) => h ? 1 : 0),
-            0
-          ), n.push(r);
-        }
-        o.offset = f.length - s;
-      } else
-        c.set(f, o.offset), l && !Array.isArray(e) && l.set(
-          e.mask.map((s) => s ? 1 : 0),
-          o.offset
-        ), o.offset += f.length;
-    }
-  }
-}
-async function p(y, i, u, o) {
-  const g = Math.min(
-    (await Promise.all(y.map((t) => t.estimateTokens(i)))).reduce((t, r) => t + r, 0),
-    i.vocabSize * 1e4
-  ), n = [new Uint16Array(g)], a = o ? [new Uint8Array(g)] : null, e = {
-    offset: 0,
-    total: 0
-  };
-  let f = performance.now();
-  for (; e.offset < g && (m(y, n, i, e, g, a || void 0), !y.every((t) => !t.hasMoreConversations())); )
-    f = await k(f, u, e.total);
-  if (n.length === 1)
-    return a ? { tokens: n[0].subarray(0, e.offset), mask: a[0].subarray(0, e.offset) } : n[0].subarray(0, e.offset);
-  const c = n.reduce((t, r) => t + r.length, 0) - (n[n.length - 1].length - e.offset), l = new Uint16Array(c);
-  let s = 0;
-  for (let t = 0; t < n.length; t++) {
-    const r = n[t];
-    t === n.length - 1 ? (l.set(r.subarray(0, e.offset), s), s += e.offset) : (l.set(r, s), s += r.length);
-  }
-  if (a) {
-    const t = new Uint8Array(c);
-    s = 0;
-    for (let r = 0; r < a.length; r++) {
-      const h = a[r];
-      r === a.length - 1 ? (t.set(h.subarray(0, e.offset), s), s += e.offset) : (t.set(h, s), s += h.length);
-    }
-    return { tokens: l, mask: t };
-  }
-  return l;
-}
-export {
-  w as Task,
-  p as tokensFromTasks
-};

package/dist/training/tasks/splitter.d.ts

@@ -1,5 +0,0 @@
-import { Task } from './Task';
-export default function splitValidation(tasks: Task[], validationSplit: number): {
-    training: Task;
-    validation: Task;
-};

package/dist/training/tasks/splitter.js

@@ -1,21 +0,0 @@
-import s from "./ConversationTask.js";
-function f(e, o) {
-  if (o <= 0 || o >= 1)
-    throw new Error("validationSplit must be between 0 and 1");
-  e.forEach((n) => n.shuffle());
-  const r = [], a = [];
-  for (const n of e)
-    for (; n.hasMoreConversations(); ) {
-      const t = n.nextConversation();
-      if (!t)
-        break;
-      Math.random() < o ? a.push(t) : r.push(t);
-    }
-  return {
-    training: new s(r),
-    validation: new s(a)
-  };
-}
-export {
-  f as default
-};

package/dist/training/types.d.ts

@@ -1,78 +0,0 @@
-import { TensorStatistics } from '../checks/weights';
-import { LoRAConfig } from '../models/config';
-import { NamedTensorMap, Tensor } from '@tensorflow/tfjs-core';
-export interface Metrics {
-    accuracy?: number;
-    perplexity?: number;
-    loss: number;
-}
-export interface TrainingLogEntry {
-    trainingMetrics: Metrics;
-    validationMetrics?: Metrics;
-    step: number;
-    time: number;
-    example?: string;
-    batchSize: number;
-    gradientMetrics?: Map<string, TensorStatistics>;
-    learningRate?: number;
-    gradientNorm?: number;
-    weightNorm?: number;
-    weightStatistics?: Map<string, TensorStatistics>;
-    memoryUsage?: number;
-    tokensPerSecond: number;
-    duration: number;
-    totalTokens: number;
-}
-export interface LRSchedulerConfig {
-    warmupSteps: number;
-    decayEpochs: number;
-    minLearningRate: number;
-    epochSteps: number;
-    step?: number;
-}
-export interface AdamWOptimizerConfig extends LRSchedulerConfig {
-    learningRate: number;
-    beta1: number;
-    beta2: number;
-    accBeta1?: number;
-    accBeta2?: number;
-    epsilon?: number;
-    weightDecay: number;
-    lossScaling: number;
-    clipNorm?: number;
-    orthoGrad?: boolean;
-}
-export type TrainingMetrics = 'accuracy' | 'perplexity' | 'gradientNorm' | 'gradientStatistics' | 'weightNorm' | 'weightStatistics' | 'memoryUsage' | 'tokensPerSecond' | 'learningRate';
-export interface TrainingOptions extends Partial<AdamWOptimizerConfig> {
-    batchSize: number;
-    maxEpochs?: number;
-    logInterval?: number;
-    prompt?: string;
-    validationSplit?: number;
-    gradientCheckpointing?: boolean;
-    mixedPrecision?: boolean;
-    trainableWeights?: string[];
-    loraConfig?: LoRAConfig;
-    loraName?: string;
-    sftMode: 'full' | 'lora' | 'last-layer';
-    maskedLoss?: boolean;
-    metrics?: TrainingMetrics[];
-    contextScaling?: number;
-    labelSmoothing?: number;
-    dropout?: number;
-    layerDrop?: number;
-    debug?: boolean;
-    onStep?: (log: TrainingLogEntry) => void;
-}
-export interface TrainingState {
-    step: number;
-    lastLoss: number;
-    totalSteps: number;
-    losses: number[];
-    validationLosses: number[];
-    logStartTime: number;
-    trainingDuration: number;
-    gradients?: NamedTensorMap;
-    gradientNorm?: Tensor;
-    accuracy?: Tensor;
-}

package/dist/training/types.js

@@ -1 +0,0 @@
-

package/dist/training/validation.d.ts

@@ -1,17 +0,0 @@
-import { ITokeniser, Task } from '../main';
-import { Tensor } from '@tensorflow/tfjs-core';
-import { Dataset } from '@tensorflow/tfjs-data';
-import { DatasetBuilder, DatasetState } from './DatasetBuilder';
-export declare function createTrainValidationSplit(tasks: Task[] | Uint16Array, tokeniser: ITokeniser, datasetBuilder: DatasetBuilder, batchSize: number, validationSplit?: number, masking?: boolean): Promise<{
-    trainDataset: Dataset<{
-        xs: Tensor;
-        ys: Tensor;
-    }>;
-    validationDataset: Dataset<{
-        xs: Tensor;
-        ys: Tensor;
-    }>;
-    size: number;
-    validationState: DatasetState;
-    trainState: DatasetState;
-}>;