@genai-fi/nanogpt 0.19.0 → 0.20.0

package/dist/tokeniser/BaseTokeniser.js

@@ -1,124 +0,0 @@
-import { E as d } from "../index-DvYrXKkX.js";
-const u = [
-  "<eos>",
-  "<bos>",
-  "",
-  "<pad>",
-  "<|user_start|>",
-  "<|user_end|>",
-  "<|assistant_start|>",
-  "<|assistant_end|>",
-  "<|system_start|>",
-  "<|system_end|>"
-];
-class T extends d {
-  id = "untrained";
-  datasetID;
-  specialTokens = /* @__PURE__ */ new Map();
-  specialTokenSet = /* @__PURE__ */ new Set();
-  isSpecialToken(t) {
-    return this.specialTokenSet.has(t);
-  }
-  addSpecialTokens() {
-    u.forEach((t, n) => {
-      this.addToken(t, n), this.specialTokens.set(t, n), this.specialTokenSet.add(n);
-    });
-  }
-  addSpecialToken(t, n) {
-    this.specialTokens.set(t, n), this.specialTokenSet.add(n);
-  }
-  generateID() {
-    const t = this.getVocab();
-    let n = 2166136261, e = 2654435769;
-    if (t.length === 0) {
-      this.id = "untrained";
-      return;
-    }
-    for (let a = 0; a < t.length; a++) {
-      const i = t[a];
-      n ^= i.length, n = Math.imul(n, 16777619), e ^= a, e = Math.imul(e, 2246822507);
-      for (let h = 0; h < i.length; h++) {
-        const c = i.charCodeAt(h);
-        n ^= c, n = Math.imul(n, 16777619), e ^= c, e = Math.imul(e, 3266489909);
-      }
-    }
-    const o = (n >>> 0).toString(36), s = (e >>> 0).toString(36);
-    this.id = "tokeniser_" + o + "_" + s;
-  }
-  encodeSequence(t) {
-    const n = this.encode(t);
-    return [this.bosToken, ...n, this.eosToken];
-  }
-  encodeAsSequence(t, n) {
-    const e = t.flatMap((o) => this.encode(o.content));
-    return n ? [this.bosToken, ...e, this.eosToken, this.bosToken] : [this.bosToken, ...e, this.eosToken];
-  }
-  encodeConversation(t, n, e) {
-    const o = [[this.bosToken]];
-    let s;
-    e && (s = [[!1]]);
-    const a = [
-      this.getSpecialTokenIndex("<|user_start|>"),
-      this.getSpecialTokenIndex("<|assistant_start|>"),
-      this.getSpecialTokenIndex("<|system_start|>")
-    ], i = [
-      this.getSpecialTokenIndex("<|user_end|>"),
-      this.getSpecialTokenIndex("<|assistant_end|>"),
-      this.getSpecialTokenIndex("<|system_end|>")
-    ];
-    for (const c of t) {
-      let l = !1;
-      const r = this.encode(c.content);
-      switch (c.role) {
-        case "user":
-          o.push([a[0]]), l = !0;
-          break;
-        case "assistant":
-          o.push([a[1]]);
-          break;
-        case "system":
-          o.push([a[2]]), l = !0;
-          break;
-      }
-      switch (o.push(r), c.role) {
-        case "user":
-          o.push([i[0]]);
-          break;
-        case "assistant":
-          o.push([i[1]]);
-          break;
-        case "system":
-          o.push([i[2]]);
-          break;
-      }
-      e && s && l ? (s.push([!1]), s.push(r.map(() => !1)), s.push([!1])) : e && s && (s.push([!1]), s.push(r.map(() => !0)), s.push([!0]));
-    }
-    const h = o.flat();
-    return n ? (h.push(a[1]), e && s && s.push([!1])) : (h.push(this.eosToken), e && s && s.push([!0])), e && s ? { tokens: h, mask: s.flat() } : h;
-  }
-  decodeConversation(t) {
-    const n = [];
-    let e = 0;
-    for (; e < t.length; ) {
-      const o = t[e];
-      let s = null;
-      if (o === this.getSpecialTokenIndex("<|user_start|>") ? s = "user" : o === this.getSpecialTokenIndex("<|assistant_start|>") ? s = "assistant" : o === this.getSpecialTokenIndex("<|system_start|>") ? s = "system" : o === this.bosToken || (o === this.eosToken ? s = null : (s = "text", e--)), s) {
-        e++;
-        const a = [];
-        for (; e < t.length && t[e] !== this.getSpecialTokenIndex(`<|${s}_end|>`) && t[e] !== this.eosToken; )
-          a.push(t[e]), e++;
-        const i = this.decode(a);
-        n.push({ role: s, content: i });
-      }
-      e++;
-    }
-    return n;
-  }
-  getSpecialTokenIndex(t) {
-    return this.specialTokens.get(t);
-  }
-}
-export {
-  u as SPECIALS,
-  T as default
-};

package/dist/tokeniser/CharTokeniser.d.ts

@@ -1,24 +0,0 @@
-import { default as BaseTokeniser } from './BaseTokeniser';
-import { Conversation } from './type';
-export default class CharTokeniser extends BaseTokeniser {
-    vocabSize: number;
-    eosToken: number;
-    bosToken: number;
-    unkToken: number;
-    vocab: string[];
-    private cache;
-    private _trained;
-    constructor(vocabSizeOrVocab: number | string[]);
-    addToken(token: string, index?: number): number;
-    get trained(): boolean;
-    destroy(): void;
-    train(text: Conversation[][], cb?: (vocab: number) => void, datasetID?: string): Promise<number>;
-    tokenise(text: string[], numeric: true): number[][];
-    tokenise(text: string[]): string[][];
-    detokenise(tokens: (number[] | Uint16Array)[]): string[];
-    encode(text: string): number[];
-    decode(tokens: number[] | Uint16Array): string;
-    getVocab(): string[];
-    getMerges(): [string, string][];
-    createTrainingData(text: string[], windowSize?: number): Promise<[number[], number[]]>;
-}

package/dist/tokeniser/CharTokeniser.js

@@ -1,107 +0,0 @@
-import { yieldIfNeeded as u } from "../utilities/yielder.js";
-import b, { SPECIALS as T } from "./BaseTokeniser.js";
-const l = ["<eos>", "<unk>"];
-class x extends b {
-  vocabSize = 0;
-  eosToken = 0;
-  bosToken = 0;
-  unkToken = 0;
-  vocab = [];
-  cache = /* @__PURE__ */ new Map();
-  _trained = !1;
-  constructor(t) {
-    if (super(), Array.isArray(t)) {
-      if (this.vocab = t, this.vocab.length > 0)
-        this.vocabSize = this.vocab.length, T.forEach((i) => {
-          const e = this.vocab.indexOf(i);
-          e !== -1 && this.addSpecialToken(i, e);
-        }), this.eosToken = this.getSpecialTokenIndex("<eos>"), this.bosToken = this.getSpecialTokenIndex("<bos>") ?? this.eosToken, this.unkToken = this.getSpecialTokenIndex("") ?? -1, this.unkToken === -1 && (this.unkToken = this.vocab.indexOf("<unk>")), this.unkToken === -1 && (this.unkToken = this.vocab.indexOf("<pad>")), this.unkToken === -1 && (this.unkToken = this.vocab.indexOf("_")), this.unkToken === -1 && (this.unkToken = this.vocab.indexOf(" ")), this.unkToken === -1 && (this.unkToken = this.eosToken), this.vocab = this.vocab.map((i) => i === "<pad>" ? "" : i), this.vocab.forEach((i, e) => {
-          this.cache.set(i, e);
-        });
-      else
-        throw new Error("Vocab cannot be empty");
-      this._trained = !0;
-    } else
-      this.vocabSize = t, this.vocab = new Array(this.vocabSize).fill(""), this.addSpecialTokens(), this.eosToken = this.getSpecialTokenIndex("<eos>"), this.bosToken = this.getSpecialTokenIndex("<bos>") ?? this.eosToken, this.unkToken = this.getSpecialTokenIndex(""), this.vocab.forEach((i, e) => {
-        this.cache.set(i, e);
-      }), this.cache.set("", this.unkToken);
-  }
-  addToken(t, i) {
-    if (this.cache.has(t))
-      return this.cache.get(t);
-    let e;
-    if (i !== void 0 ? e = i : (e = this.vocab.indexOf("", this.unkToken + 1), e === -1 && (e = this.vocabSize)), e >= this.vocabSize)
-      throw new Error("Vocab size exceeded");
-    return this.vocab[e] = t, this.cache.set(t, e), e;
-  }
-  get trained() {
-    return this.vocab.length === this.vocabSize && this._trained;
-  }
-  destroy() {
-    this.cache.clear(), this.vocab = [];
-  }
-  async train(t, i, e) {
-    this.datasetID = e;
-    const a = /* @__PURE__ */ new Set();
-    let h = performance.now();
-    for (const n of t)
-      n.forEach((o) => {
-        for (const r of o.content)
-          a.add(r);
-      }), h = await u(h, i, 0);
-    const s = Array.from(a), k = this.vocab.indexOf("", this.unkToken + 1), d = this.vocabSize - l.length;
-    if (k === -1)
-      return this.generateID(), this.vocabSize;
-    if (this._trained = !0, s.length > d) {
-      const n = /* @__PURE__ */ new Map();
-      t.forEach((o) => {
-        o.forEach((r) => {
-          for (const f of r.content)
-            n.set(f, (n.get(f) || 0) + 1);
-        });
-      }), s.sort((o, r) => (n.get(o) || 0) - (n.get(r) || 0)), s.splice(0, s.length - d);
-    }
-    let c = k;
-    if (c !== -1) {
-      const n = new Set(this.vocab);
-      for (const o of s)
-        if (!n.has(o) && (this.vocab[c] = o, n.add(o), c = this.vocab.indexOf("", c + 1), c === -1))
-          break;
-    }
-    return this.cache.clear(), this.vocab.forEach((n, o) => {
-      this.cache.set(n, o);
-    }), this.generateID(), this.emit("trainStatus", "trained"), this.vocabSize;
-  }
-  tokenise(t, i) {
-    if (!this.trained)
-      throw new Error("Tokeniser not trained");
-    return t.map((a) => i ? a.split("").map((h) => this.cache.get(h) ?? this.unkToken) : a.split("").map((h) => {
-      const s = this.cache.get(h);
-      return s !== void 0 ? this.vocab[s] : "";
-    }));
-  }
-  detokenise(t) {
-    return t.map((e) => Array.from(e).map((a) => this.vocab[a] || "").join(""));
-  }
-  encode(t) {
-    return this.tokenise([t], !0)[0];
-  }
-  decode(t) {
-    return this.detokenise([t])[0];
-  }
-  getVocab() {
-    return this.vocab;
-  }
-  getMerges() {
-    return [];
-  }
-  async createTrainingData(t, i = 5) {
-    const e = await this.tokenise(t, !0), a = [], h = [];
-    for (let s = 0; s < e.length - i; s++)
-      a.push(...e[s].slice(0, i)), h.push(e[s + 1][0]);
-    return [a, h];
-  }
-}
-export {
-  x as default
-};

package/dist/tokeniser/bpe.d.ts

@@ -1,28 +0,0 @@
-import { default as BaseTokeniser } from './BaseTokeniser';
-import { Conversation } from './type';
-export default class BPETokeniser extends BaseTokeniser {
-    private targetSize;
-    private vocab;
-    private vocabIndex;
-    private merges;
-    private pretokenMap;
-    constructor(vocabSize: number);
-    constructor(vocab: string[], merges?: [string, string][]);
-    addToken(token: string, index?: number): number;
-    destroy(): void;
-    get trained(): boolean;
-    get vocabSize(): number;
-    get eosToken(): number;
-    get bosToken(): number;
-    get unkToken(): number;
-    train(text?: Conversation[][], cb?: (vocab: number) => void, datasetID?: string): Promise<number>;
-    getVocab(): string[];
-    getMerges(): [string, string][];
-    private tokeniseWord;
-    private tokeniseStrings;
-    tokenise(text: string[], numeric: true): number[][];
-    tokenise(text: string[]): string[][];
-    detokenise(tokens: number[][]): string[];
-    encode(text: string): number[];
-    decode(tokens: number[]): string;
-}

package/dist/tokeniser/bpe.js

@@ -1,173 +0,0 @@
-import { yieldIfNeeded as p } from "../utilities/yielder.js";
-import m from "../utilities/tokenParse.js";
-import T, { SPECIALS as S } from "./BaseTokeniser.js";
-function g(o, e) {
-  return `${o}-::-${e}`;
-}
-function y(o) {
-  const e = /* @__PURE__ */ new Map();
-  for (let s = 0; s < o.length; s++) {
-    const t = o[s];
-    for (let n = 0; n < t.length - 1; n++) {
-      const r = g(t[n], t[n + 1]), a = e.get(r) || {
-        a: t[n],
-        b: t[n + 1],
-        count: 0,
-        instances: /* @__PURE__ */ new Set()
-      };
-      a.count += 1, a.instances.add(s), e.set(r, a);
-    }
-  }
-  return { pairs: e, tokens: o };
-}
-function f(o, e, s, t, n) {
-  const r = g(e, s);
-  if (o.pairs.has(r)) {
-    const a = o.pairs.get(r);
-    a.count += n, n > 0 ? a.instances.add(t) : a.count <= 0 ? o.pairs.delete(r) : a.instances.delete(t);
-  } else
-    o.pairs.set(r, { a: e, b: s, count: n, instances: /* @__PURE__ */ new Set([t]) });
-}
-function I(o) {
-  let e = null, s = 0;
-  for (const t of o.pairs.values())
-    t.count > s && (s = t.count, e = t);
-  return e;
-}
-function x(o, e) {
-  return o.map((s) => {
-    const t = [];
-    for (let n = 0; n < s.length; n++)
-      n < s.length - 1 && s[n] === e[0] && s[n + 1] === e[1] ? (t.push(e[0] + e[1]), n++) : t.push(s[n]);
-    return t;
-  });
-}
-function A(o, e) {
-  e.instances.forEach((s) => {
-    const t = o.tokens[s], n = [];
-    for (let r = 0; r < t.length; r++)
-      if (r < t.length - 1 && t[r] === e.a && t[r + 1] === e.b) {
-        const a = e.a + e.b;
-        n.push(a), r > 0 && (f(o, t[r - 1], e.a, s, -1), f(o, t[r - 1], a, s, 1)), r++, r < t.length - 1 && (f(o, e.b, t[r + 1], s, -1), f(o, a, t[r + 1], s, 1));
-      } else
-        n.push(t[r]);
-    o.tokens[s] = n;
-  }), o.pairs.delete(g(e.a, e.b));
-}
-class P extends T {
-  targetSize;
-  vocab = /* @__PURE__ */ new Set();
-  vocabIndex = /* @__PURE__ */ new Map();
-  merges = [];
-  pretokenMap = /* @__PURE__ */ new Map();
-  constructor(e, s) {
-    super(), Array.isArray(e) ? (e.forEach((t, n) => {
-      this.vocab.add(t), this.vocabIndex.set(t, n);
-    }), s && (this.merges = s), this.targetSize = e.length, S.forEach((t) => {
-      const n = e.indexOf(t);
-      n !== -1 && this.addSpecialToken(t, n);
-    })) : (this.addSpecialTokens(), this.targetSize = e);
-  }
-  addToken(e, s) {
-    if (this.vocab.has(e))
-      return this.vocabIndex.get(e);
-    {
-      this.vocab.add(e);
-      const t = s !== void 0 ? s : this.vocab.size - 1;
-      return this.vocabIndex.set(e, t), t;
-    }
-  }
-  destroy() {
-    this.vocab.clear(), this.vocabIndex.clear(), this.merges = [], this.pretokenMap.clear();
-  }
-  get trained() {
-    return this.vocab.size > S.length && this.vocab.size <= this.targetSize;
-  }
-  get vocabSize() {
-    return this.vocab.size;
-  }
-  get eosToken() {
-    return this.vocabIndex.get("<eos>") ?? 0;
-  }
-  get bosToken() {
-    return this.vocabIndex.get("<bos>") ?? 0;
-  }
-  get unkToken() {
-    return this.vocabIndex.get("") ?? 1;
-  }
-  async train(e = [], s, t) {
-    this.datasetID = t;
-    let n = performance.now();
-    const r = new Array(e.length);
-    for (let i = 0; i < e.length; i++) {
-      const h = e[i], l = new Array(h.length);
-      for (let d = 0; d < h.length; d++)
-        l[d] = m(h[d].content);
-      n = await p(n, s, this.vocab.size), r[i] = l;
-    }
-    const a = r.flat(2), z = new Set(a);
-    this.vocab = /* @__PURE__ */ new Set(), this.pretokenMap.clear(), this.merges = [], this.addSpecialTokens();
-    const b = Array.from(z), v = b.map((i) => Array.from(i).map((l) => (this.vocab.add(l), l))), k = y(v);
-    if (n = await p(n, s, this.vocab.size), this.vocab.size >= this.targetSize) {
-      console.warn("Initial vocab size is greater than or equal to target size. No merges will be performed.");
-      const i = /* @__PURE__ */ new Map();
-      a.forEach((c) => {
-        Array.from(c).forEach((u) => {
-          i.set(u, (i.get(u) || 0) + 1);
-        });
-      });
-      const h = Array.from(i.entries()).sort((c, u) => u[1] - c[1]);
-      this.vocab = /* @__PURE__ */ new Set(), this.addSpecialTokens(), h.slice(0, this.targetSize - this.vocab.size).map(([c]) => c).forEach((c) => this.vocab.add(c)), this.vocabIndex.clear();
-      let d = 0;
-      for (const c of this.vocab.keys())
-        this.vocabIndex.set(c, d++);
-      return this.generateID(), this.emit("trainStatus", "trained"), this.vocab.size;
-    }
-    for (; this.vocab.size < this.targetSize && this.merges.length < this.targetSize; ) {
-      const i = I(k);
-      if (!i)
-        break;
-      this.merges.push([i.a, i.b]), this.vocab.add(i.a + i.b), A(k, i), n = await p(n, s, this.vocab.size);
-    }
-    b.forEach((i, h) => {
-      const l = v[h];
-      this.pretokenMap.set(i, l);
-    }), this.vocabIndex.clear();
-    let w = 0;
-    for (const i of this.vocab.keys())
-      this.vocabIndex.set(i, w++);
-    return this.generateID(), this.emit("trainStatus", "trained"), this.vocab.size;
-  }
-  getVocab() {
-    return Array.from(this.vocab);
-  }
-  getMerges() {
-    return this.merges;
-  }
-  tokeniseWord(e) {
-    let s = Array.from(e);
-    return this.merges.forEach((t) => {
-      s = x([s], t)[0];
-    }), this.pretokenMap.set(e, s), s;
-  }
-  tokeniseStrings(e) {
-    return e.map((s) => m(s).map((r) => this.pretokenMap.has(r) ? this.pretokenMap.get(r) : this.tokeniseWord(r)).flat(1));
-  }
-  tokenise(e, s) {
-    const t = this.tokeniseStrings(e);
-    return s ? t.map((n) => n.map((r) => this.vocabIndex.get(r) ?? this.unkToken)) : t.map((n) => n.map((r) => this.vocab.has(r) ? r : ""));
-  }
-  detokenise(e) {
-    const s = this.getVocab();
-    return e.map((n) => n.map((r) => s[r]).join(""));
-  }
-  encode(e) {
-    return this.tokenise([e], !0)[0];
-  }
-  decode(e) {
-    return this.detokenise([e])[0];
-  }
-}
-export {
-  P as default
-};

package/dist/tokeniser/messages.d.ts

@@ -1,61 +0,0 @@
-interface TrainMessage {
-    type: 'train';
-    id: number;
-    text: string[];
-    vocabSize: number;
-}
-interface TrainResponse {
-    type: 'trainResponse';
-    id: number;
-    vocabSize: number;
-}
-interface TrainStatusMessage {
-    type: 'trainStatus';
-    id: number;
-    progress: number;
-    vocabSize: number;
-}
-interface TokeniseMessage {
-    type: 'tokenise';
-    id: number;
-    numeric?: boolean;
-    text: string[];
-}
-interface TokeniseResponse {
-    type: 'tokeniseResponse';
-    id: number;
-    numeric: boolean;
-    tokens: string[][] | number[][];
-}
-interface DetokeniseMessage {
-    type: 'detokenise';
-    id: number;
-    tokens: number[][];
-}
-interface DetokeniseResponse {
-    type: 'detokeniseResponse';
-    id: number;
-    text: string[];
-}
-interface TokensMessage {
-    type: 'tokens';
-    id: number;
-}
-interface TokensResponse {
-    type: 'tokensResponse';
-    id: number;
-    tokens: string[];
-}
-interface BuildTrainingDataMessage {
-    type: 'buildTrainingData';
-    id: number;
-    text: string[];
-    windowSize: number;
-}
-interface BuildTrainingDataResponse {
-    type: 'buildTrainingDataResponse';
-    id: number;
-    trainingData: [number[], number[]];
-}
-export type TokeniserMessage = TrainMessage | TrainResponse | TrainStatusMessage | TokeniseMessage | DetokeniseMessage | TokeniseResponse | DetokeniseResponse | TokensMessage | TokensResponse | BuildTrainingDataMessage | BuildTrainingDataResponse;
-export {};

package/dist/tokeniser/messages.js

@@ -1 +0,0 @@
-

package/dist/tokeniser/type.d.ts

@@ -1,34 +0,0 @@
-import { default as EE } from 'eventemitter3';
-export type Roles = 'user' | 'assistant' | 'system' | 'text';
-export interface Conversation {
-    role: Roles;
-    content: string;
-}
-export interface ITokeniser extends EE<'trainStatus'> {
-    id: string;
-    datasetID?: string;
-    train(text: Conversation[][], cb?: (vocab: number) => void, datasetID?: string): Promise<number>;
-    getVocab(): string[];
-    getMerges(): [string, string][];
-    destroy(): void;
-    encode(text: string): number[];
-    encodeConversation(conversation: Conversation[], completion?: boolean): number[];
-    encodeConversation(conversation: Conversation[], completion: boolean, masking: boolean): {
-        tokens: number[];
-        mask: boolean[];
-    };
-    encodeConversation(conversation: Conversation[], completion?: boolean, masking?: boolean): number[] | {
-        tokens: number[];
-        mask: boolean[];
-    };
-    encodeSequence(text: string): number[];
-    encodeAsSequence(conversation: Conversation[], completion?: boolean): number[];
-    decode(tokens: number[] | Uint16Array): string;
-    decodeConversation(tokens: number[] | Uint16Array): Conversation[];
-    vocabSize: number;
-    eosToken: number;
-    bosToken: number;
-    trained: boolean;
-    getSpecialTokenIndex(token: string): number | undefined;
-    isSpecialToken(index: number): boolean;
-}

package/dist/tokeniser/type.js

@@ -1 +0,0 @@
-

package/dist/training/AdamW.d.ts

@@ -1,36 +0,0 @@
-import { Optimizer, Tensor } from '@tensorflow/tfjs-core';
-import { ConfigDict, Serializable, SerializableConstructor } from '@tensorflow/tfjs-core/dist/serialization';
-import { NamedTensor, NamedVariableMap } from '@tensorflow/tfjs-core/dist/tensor_types';
-import { default as LRScheduler } from './LRScheduler';
-import { AdamWOptimizerConfig } from './types';
-export declare class AdamWOptimizer extends Optimizer {
-    private config;
-    readonly className = "AdamW";
-    private accBeta1;
-    private accBeta2;
-    private accumulatedMoments;
-    protected learningRate: number;
-    protected beta1: number;
-    protected beta2: number;
-    protected lossScaling: number;
-    protected weightDecay: number;
-    protected epsilon: number | null;
-    protected lrScheduler: LRScheduler;
-    protected clipNorm?: number;
-    protected orthGradEpsilon: number;
-    protected orthGrad: boolean;
-    constructor(config: AdamWOptimizerConfig);
-    get lr(): number;
-    saveMoments(): Promise<ArrayBuffer>;
-    loadMoments(momentData: ArrayBuffer): Promise<void>;
-    serializeConfig(): AdamWOptimizerConfig;
-    private orthogonalizeGradient;
-    updateConfig(newConfig: Partial<AdamWOptimizerConfig>): void;
-    applyGradients(variableGradients: NamedVariableMap | NamedTensor[]): Tensor;
-    dispose(): void;
-    getWeights(): Promise<NamedTensor[]>;
-    setWeights(weightValues: NamedTensor[]): Promise<void>;
-    getConfig(): ConfigDict;
-    /** @nocollapse */
-    static fromConfig<T extends Serializable>(cls: SerializableConstructor<T>, config: ConfigDict): T;
-}