@genai-fi/nanogpt 0.0.1

package/dist/layers/TiedEmbedding.js

@@ -0,0 +1,532 @@
+import { o as p, c as a, b as V, E as u, C as X, R as Y, d as A, B as Z, S as ee, f as te, g as se, h as ne, I as re, L as ae, N as ue, P as ie, i as oe, j as le, k as ce, l as pe, n as B, t as he, T as L, m as fe, p as me, q as de, r as q, u as P, v as U, _ as H } from "../index-B8nyc6IR.js";
+/**
+ * @license
+ * Copyright 2020 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+function ge(t, e) {
+  const s = a(t, "real", "complex"), n = a(e, "imag", "complex");
+  V(s.shape, n.shape, `real and imag shapes, ${s.shape} and ${n.shape}, must match in call to tf.complex().`);
+  const r = { real: s, imag: n };
+  return u.runKernel(X, r);
+}
+const $e = /* @__PURE__ */ p({ complex_: ge });
+/**
+ * @license
+ * Copyright 2020 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+function xe(t, e) {
+  const n = { x: a(t, "x", "reshape", "string_or_numeric") }, r = { shape: e };
+  return u.runKernel(Y, n, r);
+}
+const f = /* @__PURE__ */ p({ reshape_: xe });
+/**
+ * @license
+ * Copyright 2020 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+function ke(t, e, s = !1, n = !1) {
+  let r = a(t, "a", "matMul"), i = a(e, "b", "matMul");
+  [r, i] = A(r, i);
+  const l = { a: r, b: i }, h = { transposeA: s, transposeB: n };
+  return u.runKernel(Z, l, h);
+}
+const d = /* @__PURE__ */ p({ matMul_: ke });
+/**
+ * @license
+ * Copyright 2018 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+function be(t) {
+  const s = { x: a(t, "x", "sigmoid", "float32") };
+  return u.runKernel(ee, s);
+}
+const De = /* @__PURE__ */ p({ sigmoid_: be });
+/**
+ * @license
+ * Copyright 2020 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+function Se(t) {
+  const s = { x: a(t, "x", "elu", "float32") };
+  return u.runKernel(te, s);
+}
+const ye = /* @__PURE__ */ p({ elu_: Se });
+/**
+ * @license
+ * Copyright 2018 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+function _e(t, e = null, s = !1) {
+  let n = a(t, "x", "sum");
+  n.dtype === "bool" && (n = se(n, "int32"));
+  const r = { x: n }, i = { axis: e, keepDims: s };
+  return u.runKernel(ne, r, i);
+}
+const Me = /* @__PURE__ */ p({ sum_: _e });
+/**
+ * @license
+ * Copyright 2020 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+function Ke(t) {
+  const s = { input: a(t, "input", "imag") };
+  return u.runKernel(re, s);
+}
+const we = /* @__PURE__ */ p({ imag_: Ke });
+/**
+ * @license
+ * Copyright 2020 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+function ze(t, e = 0.2) {
+  const n = { x: a(t, "x", "leakyRelu") }, r = { alpha: e };
+  return u.runKernel(ae, n, r);
+}
+const Ee = /* @__PURE__ */ p({ leakyRelu_: ze });
+/**
+ * @license
+ * Copyright 2018 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+function We(t) {
+  const s = { x: a(t, "x", "neg") };
+  return u.runKernel(ue, s);
+}
+const Oe = /* @__PURE__ */ p({ neg_: We });
+/**
+ * @license
+ * Copyright 2020 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+function Re(t, e) {
+  const s = a(t, "x", "prelu"), n = a(e, "alpha", "prelu"), r = { x: s, alpha: n };
+  return u.runKernel(ie, r);
+}
+const Fe = /* @__PURE__ */ p({ prelu_: Re });
+/**
+ * @license
+ * Copyright 2020 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+function Le(t) {
+  const s = { input: a(t, "input", "real") };
+  return u.runKernel(oe, s);
+}
+const Ae = /* @__PURE__ */ p({ real_: Le });
+/**
+ * @license
+ * Copyright 2020 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+function Be(t) {
+  const s = { x: a(t, "x", "relu") };
+  return u.runKernel(le, s);
+}
+const Te = /* @__PURE__ */ p({ relu_: Be });
+/**
+ * @license
+ * Copyright 2020 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+function Ne(t) {
+  const s = { x: a(t, "x", "relu6") };
+  return u.runKernel(ce, s);
+}
+const ve = /* @__PURE__ */ p({ relu6_: Ne });
+/**
+ * @license
+ * Copyright 2018 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+function Ce(t, e = 0) {
+  const n = { x: a(t, "x", "step") }, r = { alpha: e };
+  return u.runKernel(pe, n, r);
+}
+const Ge = /* @__PURE__ */ p({ step_: Ce });
+/**
+ * @license
+ * Copyright 2018 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+function Ie(t, e, s) {
+  const n = a(t, "x", "transpose");
+  if (e == null && (e = n.shape.map((l, h) => h).reverse()), B(n.rank === e.length, () => `Error in transpose: rank of input ${n.rank} must match length of perm ${e}.`), e.forEach((l) => {
+    B(l >= 0 && l < n.rank, () => `All entries in 'perm' must be between 0 and ${n.rank - 1} but got ${e}`);
+  }), n.rank <= 1)
+    return n.clone();
+  const r = { x: n }, i = { perm: e };
+  return n.dtype === "complex64" ? he(() => {
+    let l = Ae(n), h = we(n);
+    return l = u.runKernel(L, { x: l }, i), h = u.runKernel(L, { x: h }, i), s && (h = Oe(h)), $e(l, h);
+  }) : u.runKernel(L, r, i);
+}
+const je = /* @__PURE__ */ p({ transpose_: Ie });
+/**
+ * @license
+ * Copyright 2019 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+function qe(t, e, s) {
+  if (s == null || s === "linear")
+    return t;
+  if (s === "relu")
+    return fe(t, Ge(e));
+  throw new Error(`Cannot compute gradient for fused activation ${s}.`);
+}
+function Pe(t, e) {
+  let s = e;
+  const n = me(t.shape, e.shape);
+  return n.length > 0 && (s = Me(s, n)), f(s, t.shape);
+}
+function Ue(t, e, s, n) {
+  if (e === "linear")
+    return t;
+  if (e === "relu")
+    return Te(t);
+  if (e === "elu")
+    return ye(t);
+  if (e === "relu6")
+    return ve(t);
+  if (e === "prelu")
+    return Fe(t, s);
+  if (e === "leakyrelu")
+    return Ee(t, n);
+  if (e === "sigmoid")
+    return De(t);
+  throw new Error(`Unknown fused activation ${e}.`);
+}
+const He = (t, e) => !(t > 0) || e === "linear";
+/**
+ * @license
+ * Copyright 2019 Google LLC. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+function Je({ a: t, b: e, transposeA: s = !1, transposeB: n = !1, bias: r, activation: i = "linear", preluActivationWeights: l, leakyreluAlpha: h = 0.2 }) {
+  if (He(u.state.gradientDepth, i) === !1) {
+    let x = d(t, e, s, n);
+    return r != null && (x = de(x, r)), Ue(x, i, l, h);
+  }
+  let o = a(t, "a", "fused matMul"), c = a(e, "b", "fused matMul");
+  [o, c] = A(o, c);
+  const b = s ? o.shape[o.rank - 2] : o.shape[o.rank - 1], D = n ? c.shape[c.rank - 1] : c.shape[c.rank - 2], w = s ? o.shape[o.rank - 1] : o.shape[o.rank - 2], z = n ? c.shape[c.rank - 2] : c.shape[c.rank - 1], T = o.shape.slice(0, -2), S = c.shape.slice(0, -2), N = q(T), v = q(S);
+  B(b === D, () => `Error in fused matMul: inner shapes (${b}) and (${D}) of Tensors with shapes ${o.shape} and ${c.shape} and transposeA=${s} and transposeB=${n} must match.`);
+  const O = P(o.shape.slice(0, -2), c.shape.slice(0, -2)).concat([w, z]), R = s ? f(o, [N, b, w]) : f(o, [N, w, b]), F = n ? f(c, [v, z, D]) : f(c, [v, D, z]);
+  let y;
+  r != null && (y = a(r, "bias", "fused matMul"), [y] = A(y, o), P(O, y.shape));
+  let C;
+  l != null && (C = a(l, "prelu weights", "fused matMul"));
+  const G = (x, K) => {
+    const [g, $, k, E] = K, m = qe(f(x, k.shape), k, i);
+    let _, M;
+    if (!s && !n ? (_ = d(m, $, !1, !0), M = d(g, m, !0, !1)) : !s && n ? (_ = d(m, $, !1, !1), M = d(m, g, !0, !1)) : s && !n ? (_ = d($, m, !1, !0), M = d(g, m, !1, !1)) : (_ = d($, m, !0, !0), M = d(m, g, !0, !0)), r != null) {
+      const Q = Pe(E, m);
+      return [_, M, Q];
+    } else
+      return [_, M];
+  }, I = {
+    a: R,
+    b: F,
+    bias: y,
+    preluActivationWeights: C
+  }, j = { transposeA: s, transposeB: n, activation: i, leakyreluAlpha: h };
+  return r == null ? U((K, g, $) => {
+    const k = (
+      // tslint:disable-next-line: no-unnecessary-type-assertion
+      u.runKernel(H, I, j)
+    );
+    return $([K, g, k]), { value: f(k, O), gradFunc: G };
+  })(R, F) : U((K, g, $, k) => {
+    const E = (
+      // tslint:disable-next-line: no-unnecessary-type-assertion
+      u.runKernel(H, I, j)
+    );
+    return k([K, g, E, $]), { value: f(E, O), gradFunc: G };
+  })(R, F, y);
+}
+const J = /* @__PURE__ */ p({ fusedMatMul_: Je });
+/**
+ * @license
+ * Copyright 2018 Google LLC
+ *
+ * Use of this source code is governed by an MIT-style
+ * license that can be found in the LICENSE file or at
+ * https://opensource.org/licenses/MIT.
+ * =============================================================================
+ */
+class W extends Error {
+  constructor(e) {
+    super(e), Object.setPrototypeOf(this, W.prototype);
+  }
+}
+/**
+ * @license
+ * Copyright 2018 Google LLC
+ *
+ * Use of this source code is governed by an MIT-style
+ * license that can be found in the LICENSE file or at
+ * https://opensource.org/licenses/MIT.
+ * =============================================================================
+ */
+function Qe(t, e, s, n) {
+  if (t.rank < 2 || e.rank < 2)
+    throw new W(`dot requires both inputs to be rank >= 2 but got x shape = ${t.shape} and y shape = ${e.shape}`);
+  if (e.rank >= 3) {
+    const r = t.shape.slice(-1)[0], i = e.shape.slice(-2)[0];
+    if (r !== i)
+      throw new W(`If rank y >= 3, then the second last dim of y must equal the last dim of x but got x shape = ${t.shape} and  y shape = ${e.shape}`);
+  }
+  if (t.rank === 2 && e.rank === 2)
+    return J({
+      a: t,
+      b: e,
+      transposeA: !1,
+      transposeB: !1,
+      bias: null,
+      activation: s
+    });
+  {
+    const r = t.shape.slice(), i = r.pop();
+    t = f(t, [-1, i]);
+    const l = e.shape.slice(), h = l.pop(), o = l.pop(), c = [...l, h], b = Array.from({ length: e.rank }, (T, S) => S === 0 ? e.rank - 2 : S <= e.rank - 2 ? S - 1 : S);
+    e = f(je(e, b), [o, -1]);
+    const D = [...r, ...c];
+    return f(J({
+      a: t,
+      b: e,
+      transposeA: !1,
+      transposeB: !1,
+      bias: null,
+      activation: s
+    }), D);
+  }
+}
+class Ye {
+  vocabSize;
+  embedDim;
+  tf;
+  tiedWeights;
+  initializer;
+  constructor(e, s, n) {
+    this.vocabSize = s.vocabSize, this.embedDim = s.embedDim, this.tf = e, this.initializer = this.tf.initializers.randomNormal({
+      mean: 0,
+      stddev: 0.02
+    }), this.tiedWeights = this.tf.variable(
+      this.initializer.apply([this.vocabSize, this.embedDim]),
+      !0,
+      n || "tied_embedding"
+    );
+  }
+  get variables() {
+    return [this.tiedWeights];
+  }
+  embed(e) {
+    return this.tf.gather(this.tiedWeights, e, 0);
+  }
+  project(e) {
+    return Qe(e, this.tiedWeights.transpose());
+  }
+  getWeights() {
+    return [this.tiedWeights];
+  }
+  setWeights(e) {
+    this.tiedWeights.assign(e[0]);
+  }
+  getConfig() {
+    return {
+      vocabSize: this.vocabSize,
+      embedDim: this.embedDim
+    };
+  }
+}
+export {
+  Ye as default
+};

package/dist/layers/TransformerBlock.d.ts

@@ -0,0 +1,19 @@
+import { default as TF } from '@tensorflow/tfjs';
+import { GPTConfig } from '../config';
+export default class Block {
+    private ln1;
+    private attn;
+    private ln2;
+    private mlp;
+    private tf;
+    private index;
+    private _trainable;
+    skipped: boolean;
+    constructor(tf: typeof TF, index: number, config: GPTConfig);
+    get variables(): TF.Variable[];
+    get trainable(): boolean;
+    set trainable(value: boolean);
+    saveWeights(map: Map<string, TF.Tensor[]>): void;
+    loadWeights(weights: Map<string, TF.Tensor[]>): void;
+    call(x: TF.Tensor, training?: boolean): TF.Tensor;
+}

package/dist/layers/TransformerBlock.js

@@ -0,0 +1,47 @@
+import r from "./CausalSelfAttention.js";
+import d from "./MLP.js";
+import n from "./LayerNorm.js";
+class _ {
+  ln1;
+  attn;
+  ln2;
+  mlp;
+  tf;
+  index;
+  _trainable = !0;
+  skipped = !1;
+  constructor(t, s, i) {
+    this.tf = t, this.index = s, this.ln1 = new n(t, [i.nEmbed], 1e-5, `block_${this.index}_ln1`), this.attn = new r(this.tf, this.index, i), this.ln2 = new n(t, [i.nEmbed], 1e-5, `block_${this.index}_ln2`), this.mlp = new d(this.tf, this.index, i);
+  }
+  get variables() {
+    return [
+      ...this.ln1.trainableWeights.map((t) => t),
+      ...this.attn.variables,
+      ...this.ln2.trainableWeights.map((t) => t),
+      ...this.mlp.variables
+    ];
+  }
+  get trainable() {
+    return this._trainable;
+  }
+  set trainable(t) {
+    this._trainable = t, this.ln1.trainable = t, this.ln2.trainable = t, this.attn.trainable = t, this.mlp.trainable = t;
+  }
+  saveWeights(t) {
+    this.attn.saveWeights(t), this.mlp.saveWeights(t), t.set(`block_${this.index}_ln1`, this.ln1.getWeights()), t.set(`block_${this.index}_ln2`, this.ln2.getWeights());
+  }
+  loadWeights(t) {
+    this.attn.loadWeights(t), this.mlp.loadWeights(t), this.ln1.setWeights(t.get(`block_${this.index}_ln1`) || []), this.ln2.setWeights(t.get(`block_${this.index}_ln2`) || []);
+  }
+  call(t, s = !1) {
+    return this.tf.tidy(() => {
+      if (this.skipped)
+        return t;
+      const i = this.ln1.apply(t), l = this.attn.call(i, s), e = t.add(l), a = this.ln2.apply(e), h = this.mlp.call(a, s);
+      return e.add(h);
+    });
+  }
+}
+export {
+  _ as default
+};

package/dist/main.d.ts

@@ -0,0 +1,6 @@
+export { default as NanoGPT } from './NanoGPTModel';
+export { default as TeachableLLM } from './TeachableLLM';
+export { default as CharTokeniser } from './tokeniser/CharTokeniser';
+export type { ITrainerOptions } from './Trainer';
+export type { IGenerateOptions } from './Generator';
+export type { TrainingLogEntry } from './NanoGPTModel';

package/dist/main.js

@@ -0,0 +1,8 @@
+import { default as o } from "./NanoGPTModel.js";
+import { default as f } from "./TeachableLLM.js";
+import { default as l } from "./tokeniser/CharTokeniser.js";
+export {
+  l as CharTokeniser,
+  o as NanoGPT,
+  f as TeachableLLM
+};

package/dist/tokeniser/CharTokeniser.d.ts

@@ -0,0 +1,20 @@
+import { default as EE } from 'eventemitter3';
+import { ITokeniser } from './type';
+export default class CharTokeniser extends EE<'trainStatus'> implements ITokeniser {
+    vocabSize: number;
+    eosToken: number;
+    vocab: string[];
+    private cache;
+    constructor(vocab?: string[]);
+    get trained(): boolean;
+    destroy(): void;
+    train(text: string[]): Promise<number>;
+    tokenise(text: string[], numeric: true): Promise<number[][]>;
+    tokenise(text: string[]): Promise<string[][]>;
+    detokenise(tokens: number[][]): Promise<string[]>;
+    encode(text: string): Promise<number[]>;
+    decode(tokens: number[]): Promise<string>;
+    getVocab(): string[];
+    getMerges(): Promise<[string, string][]>;
+    createTrainingData(text: string[], windowSize?: number): Promise<[number[], number[]]>;
+}

package/dist/tokeniser/CharTokeniser.js

@@ -0,0 +1,52 @@
+import { E as r } from "../index-SOhdqzHq.js";
+class h extends r {
+  vocabSize = 0;
+  eosToken = 0;
+  vocab = [];
+  cache = /* @__PURE__ */ new Map();
+  constructor(t) {
+    super(), this.vocab = t || [], this.vocab.length > 0 && (this.vocabSize = this.vocab.length, this.eosToken = this.vocab.indexOf("<eos>"), this.vocab.forEach((a, s) => {
+      this.cache.set(a, s);
+    }));
+  }
+  get trained() {
+    return this.vocabSize > 0;
+  }
+  destroy() {
+  }
+  async train(t) {
+    const a = new Set(t.map((e) => e.split("")).flat()), s = Array.from(a);
+    return s.sort((e, o) => e.charCodeAt(0) - o.charCodeAt(0)), this.vocab = [...s, "<eos>"], this.eosToken = this.vocab.indexOf("<eos>"), this.vocabSize = this.vocab.length, this.vocab.forEach((e, o) => {
+      this.cache.set(e, o);
+    }), this.vocabSize;
+  }
+  async tokenise(t, a) {
+    if (!this.trained)
+      throw new Error("Tokeniser not trained");
+    return t.map((e) => a ? e.split("").map((o) => this.cache.get(o) ?? -1) : e.split(""));
+  }
+  async detokenise(t) {
+    return t.map((s) => s.map((e) => this.vocab[e]).join(""));
+  }
+  async encode(t) {
+    return (await this.tokenise([t], !0))[0];
+  }
+  async decode(t) {
+    return (await this.detokenise([t]))[0];
+  }
+  getVocab() {
+    return this.vocab;
+  }
+  async getMerges() {
+    return [];
+  }
+  async createTrainingData(t, a = 5) {
+    const s = await this.tokenise(t, !0), e = [], o = [];
+    for (let i = 0; i < s.length - a; i++)
+      e.push(...s[i].slice(0, a)), o.push(s[i + 1][0]);
+    return [e, o];
+  }
+}
+export {
+  h as default
+};