@genai-fi/nanogpt 0.21.0 → 0.23.0

package/dist/main.d.ts

@@ -14,6 +14,8 @@ export { default as BPETokeniser } from './tokeniser/bpe';
 export { default as waitForModel } from './utilities/waitForModel';
 export { default as generateDatasetID } from './utilities/datasetID';
 export { default as loadTextData } from './data/textLoader';
+export type { ConversationStream, ConversationCursor } from './data/stream';
+export { MemoryConversationStream } from './data/stream';
 export type { DatasetMetadata, ModelMode } from './loader/types';
 export { default as Generator, type IGenerator } from './Generator';
 export { default as Evaluator } from './training/Evaluator';
@@ -25,6 +27,8 @@ export type { TrainingOptions, TrainingLogEntry } from './training/types';
 export type { GPTConfig } from './models/config';
 export { estimateParameterCount, estimateMemoryUsage, estimateTrainingMemoryUsage, estimateResources, validateConfig, } from './utilities/parameters';
 export { default as topP } from './utilities/topP';
+export { sliceUint16Shards, sliceUint8Shards } from './utilities/tokens';
+export { TokenStore, createTokenStore } from './training/tasks/TokenStore';
 export { Task, tokensFromTasks } from './training/tasks/Task';
 export declare const tasks: {
     PretrainingTask: typeof PretrainingTask;

package/dist/main.js

@@ -1,16 +1,20 @@
-import { a as e, b as t, i as n, n as r, r as i, s as a, t as o, y as s } from "./main-DdsoTJAW.js";
-import c from "./tokeniser/CharTokeniser.js";
-import l from "./tokeniser/bpe.js";
-import { AdamWOptimizer as u } from "./training/AdamW.js";
-import d from "./utilities/topP.js";
-import f from "./training/Evaluator.js";
-import p from "./utilities/waitForModel.js";
-import m from "./utilities/datasetID.js";
-import h from "./data/textLoader.js";
-import { estimateMemoryUsage as g, estimateParameterCount as _, estimateResources as v, estimateTrainingMemoryUsage as y, validateConfig as b } from "./utilities/parameters.js";
-import { Task as x, tokensFromTasks as S } from "./training/tasks/Task.js";
-import { selectBackend as C } from "./backend.js";
-import w from "./utilities/performance.js";
-import T from "./checks/index.js";
-import { sentenceEmbeddings as E, sentenceEmbeddingsTensor as D } from "./utilities/sentences.js";
-export { u as AdamWOptimizer, l as BPETokeniser, c as CharTokeniser, f as Evaluator, a as Generator, t as Model, s as NanoGPT, x as Task, n as TeachableLLM, e as Trainer, T as checks, g as estimateMemoryUsage, _ as estimateParameterCount, v as estimateResources, y as estimateTrainingMemoryUsage, m as generateDatasetID, o as layers, h as loadTextData, r as ops, w as performanceTest, C as selectBackend, E as sentenceEmbeddings, D as sentenceEmbeddingsTensor, i as tasks, S as tokensFromTasks, d as topP, b as validateConfig, p as waitForModel };
+import { _ as e, a as t, i as n, n as r, r as i, t as a, v as o } from "./main-BSaDGH7I.js";
+import s from "./tokeniser/CharTokeniser.js";
+import c from "./tokeniser/bpe.js";
+import { AdamWOptimizer as l } from "./training/AdamW.js";
+import u from "./utilities/topP.js";
+import d from "./training/Evaluator.js";
+import { TokenStore as f, createTokenStore as p } from "./training/tasks/TokenStore.js";
+import { Task as m, tokensFromTasks as h } from "./training/tasks/Task.js";
+import { t as g } from "./Trainer-DBsyWJ4s.js";
+import _ from "./utilities/waitForModel.js";
+import v from "./utilities/datasetID.js";
+import { n as y } from "./stream-BjdpSNqB.js";
+import b from "./data/textLoader.js";
+import { estimateMemoryUsage as x, estimateParameterCount as S, estimateResources as C, estimateTrainingMemoryUsage as w, validateConfig as T } from "./utilities/parameters.js";
+import { sliceUint16Shards as E, sliceUint8Shards as D } from "./utilities/tokens.js";
+import { selectBackend as O } from "./backend.js";
+import k from "./utilities/performance.js";
+import A from "./checks/index.js";
+import { sentenceEmbeddings as j, sentenceEmbeddingsTensor as M } from "./utilities/sentences.js";
+export { l as AdamWOptimizer, c as BPETokeniser, s as CharTokeniser, d as Evaluator, t as Generator, y as MemoryConversationStream, o as Model, e as NanoGPT, m as Task, n as TeachableLLM, f as TokenStore, g as Trainer, A as checks, p as createTokenStore, x as estimateMemoryUsage, S as estimateParameterCount, C as estimateResources, w as estimateTrainingMemoryUsage, v as generateDatasetID, a as layers, b as loadTextData, r as ops, k as performanceTest, O as selectBackend, j as sentenceEmbeddings, M as sentenceEmbeddingsTensor, E as sliceUint16Shards, D as sliceUint8Shards, i as tasks, h as tokensFromTasks, u as topP, T as validateConfig, _ as waitForModel };

package/dist/models/NanoGPTV1.js

@@ -1,2 +1,2 @@
-import { y as e } from "../main-DdsoTJAW.js";
+import { _ as e } from "../main-BSaDGH7I.js";
 export { e as default };

package/dist/models/NanoGPTV2.js

@@ -1,2 +1,2 @@
-import { v as e } from "../main-DdsoTJAW.js";
+import { g as e } from "../main-BSaDGH7I.js";
 export { e as default };

package/dist/models/factory.js

@@ -1,2 +1,2 @@
-import { _ as e } from "../main-DdsoTJAW.js";
+import { h as e } from "../main-BSaDGH7I.js";
 export { e as default };

package/dist/models/model.js

@@ -1,2 +1,2 @@
-import { b as e } from "../main-DdsoTJAW.js";
+import { v as e } from "../main-BSaDGH7I.js";
 export { e as default };