@genai-fi/nanogpt 0.22.0 → 0.24.0

package/dist/main.d.ts

@@ -1,6 +1,3 @@
-import { default as PretrainingTask } from './training/tasks/PretrainingTask';
-import { default as StartSentenceTask } from './training/tasks/StartSentenceTask';
-import { default as ConversationTask } from './training/tasks/ConversationTask';
 import { pack16 } from './ops/pack16';
 import { unpack16 } from './ops/unpack16';
 import { default as CausalSelfAttention } from './layers/CausalSelfAttention';
@@ -14,7 +11,7 @@ export { default as BPETokeniser } from './tokeniser/bpe';
 export { default as waitForModel } from './utilities/waitForModel';
 export { default as generateDatasetID } from './utilities/datasetID';
 export { default as loadTextData } from './data/textLoader';
-export type { ConversationStream, ConversationCursor } from './data/stream';
+export type { ConversationStream } from './data/stream';
 export { MemoryConversationStream } from './data/stream';
 export type { DatasetMetadata, ModelMode } from './loader/types';
 export { default as Generator, type IGenerator } from './Generator';
@@ -28,12 +25,8 @@ export type { GPTConfig } from './models/config';
 export { estimateParameterCount, estimateMemoryUsage, estimateTrainingMemoryUsage, estimateResources, validateConfig, } from './utilities/parameters';
 export { default as topP } from './utilities/topP';
 export { sliceUint16Shards, sliceUint8Shards } from './utilities/tokens';
-export { Task, tokensFromTasks } from './training/tasks/Task';
-export declare const tasks: {
-    PretrainingTask: typeof PretrainingTask;
-    StartSentenceTask: typeof StartSentenceTask;
-    ConversationTask: typeof ConversationTask;
-};
+export { TokenStore, createTokenStore } from './training/tasks/TokenStore';
+export { tokensFromStreams } from './training/tasks/tokenStream';
 declare const ops: {
     pack16: typeof pack16;
     unpack16: typeof unpack16;

package/dist/main.js

@@ -1,18 +1,20 @@
-import { a as e, b as t, i as n, n as r, r as i, s as a, t as o, y as s } from "./main-Bgc7_9kb.js";
-import c from "./tokeniser/CharTokeniser.js";
-import l from "./tokeniser/bpe.js";
-import { AdamWOptimizer as u } from "./training/AdamW.js";
-import d from "./utilities/topP.js";
-import f from "./training/Evaluator.js";
-import { sliceUint16Shards as p, sliceUint8Shards as m } from "./utilities/tokens.js";
+import { _ as e, g as t, i as n, n as r, r as i, t as a } from "./main-Dz72vadm.js";
+import o from "./tokeniser/CharTokeniser.js";
+import s from "./tokeniser/bpe.js";
+import { AdamWOptimizer as c } from "./training/AdamW.js";
+import l from "./utilities/topP.js";
+import u from "./training/Evaluator.js";
+import { TokenStore as d, createTokenStore as f } from "./training/tasks/TokenStore.js";
+import { tokensFromStreams as p } from "./training/tasks/tokenStream.js";
+import { t as m } from "./Trainer-Cr7csbTD.js";
 import h from "./utilities/waitForModel.js";
 import g from "./utilities/datasetID.js";
-import { n as _ } from "./stream-DKl3GTDL.js";
+import { n as _ } from "./stream-BpAwcvHz.js";
 import v from "./data/textLoader.js";
 import { estimateMemoryUsage as y, estimateParameterCount as b, estimateResources as x, estimateTrainingMemoryUsage as S, validateConfig as C } from "./utilities/parameters.js";
-import { Task as w, tokensFromTasks as T } from "./training/tasks/Task.js";
+import { sliceUint16Shards as w, sliceUint8Shards as T } from "./utilities/tokens.js";
 import { selectBackend as E } from "./backend.js";
 import D from "./utilities/performance.js";
 import O from "./checks/index.js";
 import { sentenceEmbeddings as k, sentenceEmbeddingsTensor as A } from "./utilities/sentences.js";
-export { u as AdamWOptimizer, l as BPETokeniser, c as CharTokeniser, f as Evaluator, a as Generator, _ as MemoryConversationStream, t as Model, s as NanoGPT, w as Task, n as TeachableLLM, e as Trainer, O as checks, y as estimateMemoryUsage, b as estimateParameterCount, x as estimateResources, S as estimateTrainingMemoryUsage, g as generateDatasetID, o as layers, v as loadTextData, r as ops, D as performanceTest, E as selectBackend, k as sentenceEmbeddings, A as sentenceEmbeddingsTensor, p as sliceUint16Shards, m as sliceUint8Shards, i as tasks, T as tokensFromTasks, d as topP, C as validateConfig, h as waitForModel };
+export { c as AdamWOptimizer, s as BPETokeniser, o as CharTokeniser, u as Evaluator, n as Generator, _ as MemoryConversationStream, e as Model, t as NanoGPT, i as TeachableLLM, d as TokenStore, m as Trainer, O as checks, f as createTokenStore, y as estimateMemoryUsage, b as estimateParameterCount, x as estimateResources, S as estimateTrainingMemoryUsage, g as generateDatasetID, a as layers, v as loadTextData, r as ops, D as performanceTest, E as selectBackend, k as sentenceEmbeddings, A as sentenceEmbeddingsTensor, w as sliceUint16Shards, T as sliceUint8Shards, p as tokensFromStreams, l as topP, C as validateConfig, h as waitForModel };

package/dist/models/NanoGPTV1.js

@@ -1,2 +1,2 @@
-import { y as e } from "../main-Bgc7_9kb.js";
+import { g as e } from "../main-Dz72vadm.js";
 export { e as default };

package/dist/models/NanoGPTV2.js

@@ -1,2 +1,2 @@
-import { v as e } from "../main-Bgc7_9kb.js";
+import { h as e } from "../main-Dz72vadm.js";
 export { e as default };

package/dist/models/factory.js

@@ -1,2 +1,2 @@
-import { _ as e } from "../main-Bgc7_9kb.js";
+import { m as e } from "../main-Dz72vadm.js";
 export { e as default };

package/dist/models/model.js

@@ -1,2 +1,2 @@
-import { b as e } from "../main-Bgc7_9kb.js";
+import { _ as e } from "../main-Dz72vadm.js";
 export { e as default };