@genai-fi/nanogpt 0.0.1

package/LICENSE

@@ -0,0 +1,7 @@
+Copyright 2024 Nicolas Pope
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package/README.md

@@ -0,0 +1,20 @@
+# Introduction 
+TODO: Give a short introduction of your project. Let this section explain the objectives or the motivation behind this project. 
+
+# Getting Started
+TODO: Guide users through getting your code up and running on their own system. In this section you can talk about:
+1.	Installation process
+2.	Software dependencies
+3.	Latest releases
+4.	API references
+
+# Build and Test
+TODO: Describe and show how to build your code and run the tests. 
+
+# Contribute
+TODO: Explain how other users and developers can contribute to make your code better. 
+
+If you want to learn more about creating good readme files then refer the following [guidelines](https://docs.microsoft.com/en-us/azure/devops/repos/git/create-a-readme?view=azure-devops). You can also seek inspiration from the below readme files:
+- [ASP.NET Core](https://github.com/aspnet/Home)
+- [Visual Studio Code](https://github.com/Microsoft/vscode)
+- [Chakra Core](https://github.com/Microsoft/ChakraCore)

package/dist/Generator.d.ts

@@ -0,0 +1,14 @@
+import { default as NanoGPT } from './NanoGPTModel';
+import { ITokeniser } from './tokeniser/type';
+import { default as EE } from 'eventemitter3';
+export interface IGenerateOptions {
+    maxLength?: number;
+    temperature?: number;
+}
+export default class Generator extends EE<'start' | 'stop' | 'tokens'> {
+    private readonly model;
+    private readonly tokeniser;
+    constructor(model: NanoGPT, tokeniser: ITokeniser);
+    private generateBlockOfTokens;
+    generate(prompt?: string, options?: IGenerateOptions): Promise<string>;
+}

package/dist/Generator.js

@@ -0,0 +1,39 @@
+import { E as f } from "./index-SOhdqzHq.js";
+const u = 4;
+class p extends f {
+  constructor(s, t) {
+    super(), this.model = s, this.tokeniser = t;
+  }
+  generateBlockOfTokens(s, t) {
+    const r = t?.temperature ?? 1;
+    let e = s;
+    for (let n = 0; n < u; n++) {
+      const i = this.model.generate(e, r), a = e;
+      e = this.model.tf.concat([e, i], 1), a.dispose(), i.dispose();
+    }
+    return e;
+  }
+  async generate(s, t) {
+    const r = s ? await this.tokeniser.tokenise([s], !0) : [[this.tokeniser.eosToken]];
+    let e = this.model.tf.tensor2d(r, [1, r[0].length], "int32");
+    this.emit("start");
+    let n = s || "";
+    for (; ; ) {
+      const i = this.generateBlockOfTokens(e, t), a = e;
+      e = i;
+      const l = i.slice([0, a.shape[1]], [1, u]), o = (await l.array())[0];
+      let h = !1, c = !1;
+      const d = o.indexOf(this.tokeniser.eosToken);
+      d !== -1 && (h = !0, o.splice(d)), o.length + n.length >= (t?.maxLength ?? 1e3) && (c = !0, o.splice(
+        t?.maxLength ? t.maxLength - n.length : o.length
+      ));
+      const k = await this.tokeniser.decode(o);
+      if (n += k, this.emit("tokens", o, k), a.dispose(), l.dispose(), h || c)
+        break;
+    }
+    return e.dispose(), this.emit("stop"), n;
+  }
+}
+export {
+  p as default
+};

package/dist/NanoGPTModel.d.ts

@@ -0,0 +1,35 @@
+import { default as TF } from '@tensorflow/tfjs';
+import { GPTConfig } from './config';
+export interface TrainingLogEntry {
+    epoch: number;
+    loss: number;
+    valLoss?: number;
+    step: number;
+    time: number;
+    example?: string;
+    batchSize: number;
+}
+export default class NanoGPT {
+    readonly config: GPTConfig;
+    private wte;
+    private wpe;
+    private drop;
+    private blocks;
+    private lnF;
+    readonly tf: typeof TF;
+    log: TrainingLogEntry[];
+    constructor(tf: typeof TF, config?: Partial<GPTConfig>);
+    get variables(): TF.Variable[];
+    saveWeights(): Map<string, TF.Tensor[]>;
+    loadWeights(weights: Map<string, TF.Tensor[]>): void;
+    private inputPhase;
+    setSkipMask(mask: boolean[]): void;
+    setTrainableMask(mask: boolean[]): void;
+    set trainable(value: boolean);
+    forward(idx: TF.Tensor, targets?: TF.Tensor, training?: boolean): {
+        logits: TF.Tensor;
+        loss?: TF.Tensor;
+    };
+    generate(idx: TF.Tensor, temperature?: number, topK?: number): TF.Tensor;
+    getNumParams(): number;
+}

package/dist/NanoGPTModel.js

@@ -0,0 +1,129 @@
+import { defaultConfig as b } from "./config.js";
+import p from "./layers/TransformerBlock.js";
+import m from "./layers/TiedEmbedding.js";
+import d from "./layers/LayerNorm.js";
+class S {
+  config;
+  wte;
+  // Token embeddings
+  wpe;
+  // Position embeddings
+  drop;
+  // Dropout
+  blocks;
+  lnF;
+  // Final layer norm
+  tf;
+  log = [];
+  // Training log
+  constructor(t, e = {}) {
+    this.tf = t, this.config = { ...b, ...e }, this.wte = new m(t, {
+      vocabSize: this.config.vocabSize,
+      embedDim: this.config.nEmbed,
+      name: "token_embedding"
+    }), this.wpe = this.tf.layers.embedding({
+      inputDim: this.config.blockSize,
+      outputDim: this.config.nEmbed,
+      name: "positional_embedding",
+      embeddingsInitializer: this.tf.initializers.randomNormal({ mean: 0, stddev: 0.02 })
+    }), this.drop = this.tf.layers.dropout({ rate: this.config.dropout }), this.blocks = [];
+    for (let s = 0; s < this.config.nLayer; s++)
+      this.blocks.push(new p(this.tf, s, this.config));
+    this.lnF = new d(t, [this.config.nEmbed], 1e-5, "final_layer_norm");
+  }
+  get variables() {
+    return [
+      ...this.wpe.trainableWeights.map((t) => t.read()),
+      ...this.blocks.flatMap((t) => t.variables),
+      ...this.lnF.trainableWeights.map((t) => t),
+      ...this.wte.variables
+    ];
+  }
+  saveWeights() {
+    const t = /* @__PURE__ */ new Map();
+    t.set("token_embedding", this.wte.getWeights()), t.set("positional_embedding", this.wpe.getWeights());
+    for (let e = 0; e < this.blocks.length; e++)
+      this.blocks[e].saveWeights(t);
+    return t.set("final_layer_norm", this.lnF.getWeights()), t;
+  }
+  loadWeights(t) {
+    this.wte.setWeights(t.get("token_embedding") || []), this.wpe.setWeights(t.get("positional_embedding") || []);
+    for (let e = 0; e < this.blocks.length; e++)
+      this.blocks[e].loadWeights(t);
+    this.lnF.setWeights(t.get("final_layer_norm") || []);
+  }
+  inputPhase(t, e = !1) {
+    return this.tf.tidy(() => {
+      const [, s] = t.shape, i = this.wte.embed(t), o = this.tf.range(0, s, 1, "int32"), r = this.wpe.apply(o), n = i.add(r);
+      return this.drop.apply(n, { training: e });
+    });
+  }
+  setSkipMask(t) {
+    if (t.length !== this.blocks.length)
+      throw new Error(`Mask length ${t.length} does not match number of blocks ${this.blocks.length}`);
+    for (let e = 0; e < this.blocks.length; e++)
+      this.blocks[e].skipped = t[e];
+  }
+  setTrainableMask(t) {
+    if (t.length !== this.blocks.length)
+      throw new Error(`Mask length ${t.length} does not match number of blocks ${this.blocks.length}`);
+    for (let e = 0; e < this.blocks.length; e++)
+      this.blocks[e].trainable = t[e];
+  }
+  set trainable(t) {
+    for (const e of this.blocks)
+      e.trainable = t;
+    this.wpe.trainable = t, this.lnF.trainable = t;
+  }
+  forward(t, e, s = !1) {
+    if (t.shape.length !== 2)
+      throw new Error(`Invalid input shape: expected [batch_size, sequence_length], got ${t.shape}`);
+    if (t.shape[1] > this.config.blockSize)
+      throw new Error(`Input sequence length ${t.shape[1]} isn't block size ${this.config.blockSize}`);
+    if (t.dtype !== "int32")
+      throw new Error(`Input tensor must be of type int32, got ${t.dtype}`);
+    return this.tf.tidy(() => {
+      const [, i] = t.shape;
+      if (i > this.config.blockSize)
+        throw new Error(`Cannot forward sequence of length ${i}, block size is only ${this.config.blockSize}`);
+      let o = this.inputPhase(t, s);
+      for (const h of this.blocks)
+        o = h.call(o);
+      o = this.lnF.apply(o);
+      const r = this.wte.project(o);
+      let n;
+      if (e)
+        try {
+          n = this.tf.losses.softmaxCrossEntropy(e, r, this.tf.Reduction.MEAN);
+        } catch (h) {
+          throw console.error("Error computing loss:", h), new Error(`Loss computation failed: ${h}`);
+        }
+      return { logits: r, loss: n };
+    });
+  }
+  generate(t, e = 1, s) {
+    return this.tf.tidy(() => {
+      const i = t, o = i.shape[1], r = o <= this.config.blockSize ? i : i.slice(
+        [0, o - this.config.blockSize],
+        [i.shape[0], this.config.blockSize]
+      ), { logits: n } = this.forward(r, void 0, !1), h = n.shape[1] - 1, a = n.slice([0, h, 0], [n.shape[0], 1, n.shape[2]]).div(e);
+      let l;
+      if (s) {
+        const { values: c, indices: g } = this.tf.topk(a, s), f = this.tf.multinomial(c.squeeze([1]), 1);
+        l = this.tf.gather(g.squeeze([1]), f, 1);
+      } else
+        l = this.tf.multinomial(a.squeeze([1]), 1);
+      return l = l.reshape([1, 1]), l;
+    });
+  }
+  // Get number of parameters
+  getNumParams() {
+    const t = this.config.vocabSize * this.config.nEmbed + this.config.blockSize * this.config.nEmbed, e = this.config.nLayer * (4 * this.config.nEmbed * this.config.nEmbed + // qkv + proj
+    2 * this.config.nEmbed), s = this.config.nLayer * (4 * this.config.nEmbed * this.config.nEmbed + // fc
+    this.config.nEmbed * 4 * this.config.nEmbed), i = this.config.nEmbed + this.config.vocabSize * this.config.nEmbed;
+    return t + e + s + i;
+  }
+}
+export {
+  S as default
+};

package/dist/TeachableLLM.d.ts

@@ -0,0 +1,21 @@
+import { default as TF } from '@tensorflow/tfjs';
+import { GPTConfig } from './config';
+import { ITokeniser } from './tokeniser/type';
+import { default as NanoGPT } from './NanoGPTModel';
+import { default as Generator, IGenerateOptions } from './Generator';
+import { default as Trainer, ITrainerOptions } from './Trainer';
+export default class TeachableLLM {
+    readonly config: GPTConfig;
+    readonly model: NanoGPT;
+    readonly tf: typeof TF;
+    readonly tokeniser: ITokeniser;
+    constructor(tf: typeof TF, tokeniser: ITokeniser, model: NanoGPT);
+    saveModel(): Promise<Blob>;
+    static loadModel(tf: typeof TF, data: Blob | Buffer | string): Promise<TeachableLLM>;
+    static create(tf: typeof TF, tokeniser: ITokeniser, config?: Partial<GPTConfig>): TeachableLLM;
+    getNumParams(): number;
+    trainer(): Trainer;
+    train(text: string[], options?: ITrainerOptions): Promise<void>;
+    generator(): Generator;
+    generateText(prompt?: string, options?: IGenerateOptions): Promise<string>;
+}

package/dist/TeachableLLM.js

@@ -0,0 +1,47 @@
+import s from "./NanoGPTModel.js";
+import { defaultConfig as a } from "./config.js";
+import { saveModel as m } from "./utilities/save.js";
+import { loadModel as l } from "./utilities/load.js";
+import d from "./Generator.js";
+import c from "./Trainer.js";
+import "./tokeniser/CharTokeniser.js";
+class i {
+  config;
+  model;
+  tf;
+  tokeniser;
+  constructor(e, t, r) {
+    this.tf = e, this.config = r.config, this.tokeniser = t, this.model = r;
+  }
+  saveModel() {
+    return m(this.model, this.tokeniser);
+  }
+  static async loadModel(e, t) {
+    const { model: r, tokeniser: o } = await l(e, t);
+    return new i(e, o, r);
+  }
+  static create(e, t, r = {}) {
+    const o = { ...a, ...r };
+    o.vocabSize = t.vocabSize;
+    const n = new s(e, o);
+    return new i(e, t, n);
+  }
+  getNumParams() {
+    return this.model.getNumParams();
+  }
+  trainer() {
+    return new c(this.model, this.tokeniser);
+  }
+  train(e, t) {
+    return this.trainer().train(e, t);
+  }
+  generator() {
+    return new d(this.model, this.tokeniser);
+  }
+  generateText(e, t) {
+    return this.generator().generate(e, t);
+  }
+}
+export {
+  i as default
+};

package/dist/Trainer.d.ts

@@ -0,0 +1,19 @@
+import { default as NanoGPT } from './NanoGPTModel';
+import { ITokeniser } from './tokeniser/type';
+import { default as EE } from 'eventemitter3';
+export interface ITrainerOptions {
+    epochs?: number;
+    batchSize?: number;
+    learningRate?: number;
+    maxSteps?: number;
+    desiredLoss?: number;
+    logInterval?: number;
+    prompt?: string;
+    validationSplit?: number;
+}
+export default class Trainer extends EE<'start' | 'stop' | 'log'> {
+    private trainer;
+    constructor(model: NanoGPT, tokeniser: ITokeniser);
+    stop(): void;
+    train(text: string[], options?: ITrainerOptions): Promise<void>;
+}

package/dist/Trainer.js

@@ -0,0 +1,34 @@
+import { E as s } from "./index-SOhdqzHq.js";
+import n from "./training/FullTrainer.js";
+class o extends s {
+  trainer;
+  constructor(a, t) {
+    super(), this.trainer = new n(a.tf, a, t, 1e-3);
+  }
+  stop() {
+  }
+  async train(a, t) {
+    const { trainDataset: e, validationDataset: r } = await this.trainer.createTrainValidationSplit(
+      a,
+      t?.batchSize || 32,
+      t?.validationSplit || 0.1
+    );
+    this.emit("start"), await this.trainer.trainOnDataset(
+      e,
+      {
+        epochs: t?.epochs || 2,
+        prompt: t?.prompt,
+        stepsPerEpoch: t?.maxSteps || 100,
+        logInterval: t?.logInterval || 10,
+        desiredLoss: t?.desiredLoss || 0.01,
+        onStep: async (i) => {
+          this.emit("log", i);
+        }
+      },
+      r
+    ), this.emit("stop");
+  }
+}
+export {
+  o as default
+};

package/dist/_commonjsHelpers-DaMA6jEr.js

@@ -0,0 +1,8 @@
+var o = typeof globalThis < "u" ? globalThis : typeof window < "u" ? window : typeof global < "u" ? global : typeof self < "u" ? self : {};
+function l(e) {
+  return e && e.__esModule && Object.prototype.hasOwnProperty.call(e, "default") ? e.default : e;
+}
+export {
+  o as c,
+  l as g
+};

package/dist/assets/worker-BYeSPNkq.js

@@ -0,0 +1 @@
+(function(){"use strict";function u(n,t){const e=n,s=[];let a="";for(let o=0;o<e.length;o++){const c=e[o];switch(c){case"0":case"1":case"2":case"3":case"4":case"5":case"6":case"7":case"8":case"9":case":":case";":case",":case".":case"?":case"!":case'"':case"'":case"`":case"(":case")":case"[":case"]":case"{":case"}":case"-":case"_":case"/":case"\\":case"%":case"<":case">":case"=":case"+":case"*":case"&":case"^":case"|":case"~":case"@":case"#":case"$":s.push(a),s.push(c),a="";break;case" ":s.push(a),a=c;break;default:a+=c;break}}return a.length>0&&s.push(a),s}function k(n){const t=new Map;for(let e=0;e<n.length;e++){const s=n[e];for(let a=0;a<s.length-1;a++){const o=`${s[a]}${s[a+1]}`,c=t.get(o)||{a:s[a],b:s[a+1],count:0,instances:new Set};c.count+=1,c.instances.add(e),t.set(o,c)}}return{pairs:t,tokens:n}}function p(n,t,e,s,a){const o=`${t}${e}`;if(n.pairs.has(o)){const c=n.pairs.get(o);c.count+=a,c.instances.add(s)}else n.pairs.set(o,{a:t,b:e,count:a,instances:new Set([s])})}function b(n){let t=null,e=0;for(const s of n.pairs.values())s.count>e&&(e=s.count,t=s);return t}function m(n,t){return n.map(e=>{const s=[];for(let a=0;a<e.length;a++)a<e.length-1&&e[a]===t[0]&&e[a+1]===t[1]?(s.push(t[0]+t[1]),a++):s.push(e[a]);return s})}function y(n,t){t.instances.forEach(e=>{const s=n.tokens[e],a=[];for(let o=0;o<s.length;o++)if(o<s.length-1&&s[o]===t.a&&s[o+1]===t.b){const c=t.a+t.b;a.push(c),o>0&&(p(n,s[o-1],t.a,e,-1),p(n,s[o-1],c,e,1)),o++,o<s.length-1&&(p(n,t.b,s[o+1],e,-1),p(n,c,s[o+1],e,1))}else a.push(s[o]);n.tokens[e]=a}),n.pairs.delete(`${t.a}${t.b}`)}class l{vocab=new Set;vocabIndex=new Map;merges=[];pretokenMap=new Map;constructor(t,e){t&&t.forEach((s,a)=>{this.vocab.add(s),this.vocabIndex.set(s,a)}),e&&(this.merges=e)}train(t,e,s){const a=t.map(i=>u(i)).flat(1),o=new Set(a);this.vocab=new Set,this.pretokenMap.clear(),this.merges=[],this.vocab.add("<eos>");const c=Array.from(o),d=c.map(i=>i.split("").map(h=>(this.vocab.add(h),h))),f=k(d);for(;this.vocab.size<e&&this.merges.length<e;){const i=b(f);if(!i)break;this.merges.push([i.a,i.b]),this.vocab.add(i.a+i.b),y(f,i),s&&this.vocab.size%100===0&&s(this.vocab.size/e,this.vocab.size)}c.forEach((i,g)=>{const h=d[g];this.pretokenMap.set(i,h)}),this.vocabIndex.clear();let M=0;for(const i of this.vocab.keys())this.vocabIndex.set(i,M++)}getVocab(){return Array.from(this.vocab)}getMerges(){return this.merges}tokeniseWord(t){let e=t.split("");return this.merges.forEach(s=>{e=m([e],s)[0]}),this.pretokenMap.set(t,e),e}tokeniseStrings(t){return t.map(e=>u(e).map(o=>this.pretokenMap.has(o)?this.pretokenMap.get(o):this.tokeniseWord(o)).flat(1))}tokenise(t,e){const s=this.tokeniseStrings(t);return e?s.map(a=>a.map(o=>this.vocabIndex.get(o)??-1)):s}}let r=new l;onmessage=async n=>{if(n.data.type==="tokenise")if(n.data.numeric){const t=r.tokenise(n.data.text,!0),e={type:"tokeniseResponse",id:n.data.id,tokens:t,numeric:!0};postMessage(e)}else{const t=r.tokenise(n.data.text),e={type:"tokeniseResponse",id:n.data.id,tokens:t,numeric:!1};postMessage(e)}else if(n.data.type==="detokenise"){const t=r.getVocab(),e=n.data.tokens.map(a=>a.map(o=>t[o]).join("")),s={type:"detokeniseResponse",id:n.data.id,text:e};postMessage(s)}else if(n.data.type==="train"){r=new l,r.train(n.data.text,n.data.vocabSize??100,(e,s)=>{const a={type:"trainStatus",id:n.data.id,progress:e,vocabSize:s};postMessage(a)});const t={type:"trainResponse",id:n.data.id,vocabSize:r.getVocab().length};postMessage(t)}else if(n.data.type==="tokens"){const t=r.getVocab(),e={type:"tokensResponse",id:n.data.id,tokens:t};postMessage(e)}}})();

package/dist/config.d.ts

@@ -0,0 +1,11 @@
+export interface GPTConfig {
+    vocabSize: number;
+    blockSize: number;
+    nLayer: number;
+    nHead: number;
+    nEmbed: number;
+    dropout: number;
+    biasInLinear: boolean;
+    biasInLayerNorm: boolean;
+}
+export declare const defaultConfig: GPTConfig;

package/dist/config.js

@@ -0,0 +1,19 @@
+const e = {
+  vocabSize: 50304,
+  // GPT-2 vocab size
+  blockSize: 1024,
+  // Maximum sequence length
+  nLayer: 12,
+  // Number of transformer layers
+  nHead: 12,
+  // Number of attention heads
+  nEmbed: 768,
+  // Embedding dimension
+  dropout: 0,
+  // Dropout probability
+  biasInLinear: !1,
+  biasInLayerNorm: !1
+};
+export {
+  e as defaultConfig
+};