@genai-fi/nanogpt 0.23.0 → 1.0.0

package/dist/loader/types.d.ts

@@ -1,4 +1,4 @@
-import { GenerateOptions } from '../../inference/types';
+import { IGenerateOptions } from '../../inference/types';
 import { LoRAConfig } from '../../models/config';
 import { default as Model, ModelForwardAttributes, TrainingState } from '../../models/model';
 import { ITokeniser } from '../../tokeniser/type';
@@ -42,7 +42,7 @@ export interface ActionLogEntry {
     timestamp: number;
     duration: number;
     tokensProcessed: number;
-    options: TrainingOptions | GenerateOptions;
+    options: TrainingOptions | IGenerateOptions;
 }
 export interface TransformersMetadata {
     name?: string;
@@ -55,7 +55,7 @@ export interface TransformersMetadata {
     mode?: ModelMode;
     pretrainingData?: DatasetMetadata[];
     pretrainingSettings?: TrainingOptions;
-    generationSettings?: GenerateOptions;
+    generationSettings?: IGenerateOptions;
     actionLog?: ActionLogEntry[];
     [key: string]: unknown;
 }

package/dist/main.d.ts

@@ -1,54 +1,45 @@
-import { default as PretrainingTask } from './training/tasks/PretrainingTask';
-import { default as StartSentenceTask } from './training/tasks/StartSentenceTask';
-import { default as ConversationTask } from './training/tasks/ConversationTask';
+import { default as Evaluator } from './training/Evaluator';
+import { AdamWOptimizer } from './training/AdamW';
+import { estimateParameterCount, estimateMemoryUsage, estimateTrainingMemoryUsage, estimateResources, validateConfig } from './utilities/parameters';
+import { default as topP } from './utilities/topP';
+import { sliceUint16Shards, sliceUint8Shards } from './utilities/tokens';
+import { default as performanceTest } from './utilities/performance';
+import { sentenceEmbeddings, sentenceEmbeddingsTensor } from './utilities/sentences';
 import { pack16 } from './ops/pack16';
 import { unpack16 } from './ops/unpack16';
-import { default as CausalSelfAttention } from './layers/CausalSelfAttention';
-import { default as MLP } from './layers/MLP';
-import { default as TransformerBlock } from './layers/TransformerBlock';
-import { default as RoPECache } from './layers/RoPECache';
-export { default as NanoGPT } from './models/NanoGPTV1';
 export { default as TeachableLLM } from './TeachableLLM';
-export { default as CharTokeniser } from './tokeniser/CharTokeniser';
-export { default as BPETokeniser } from './tokeniser/bpe';
-export { default as waitForModel } from './utilities/waitForModel';
-export { default as generateDatasetID } from './utilities/datasetID';
-export { default as loadTextData } from './data/textLoader';
-export type { ConversationStream, ConversationCursor } from './data/stream';
-export { MemoryConversationStream } from './data/stream';
-export type { DatasetMetadata, ModelMode } from './loader/types';
-export { default as Generator, type IGenerator } from './Generator';
-export { default as Evaluator } from './training/Evaluator';
-export { default as Trainer } from './Trainer';
-export type { IGenerateOptions } from './Generator';
-export { type ModelForwardAttributes, default as Model } from './models/model';
+export * as tokenise from './tokenise';
 export type { ITokeniser, Conversation, Roles } from './tokeniser/type';
-export type { TrainingOptions, TrainingLogEntry } from './training/types';
+export * as data from './data';
+export type { ConversationStream } from './data/stream';
+export type { DatasetMetadata, ModelMode } from './loader/types';
+export * as models from './models';
 export type { GPTConfig } from './models/config';
-export { estimateParameterCount, estimateMemoryUsage, estimateTrainingMemoryUsage, estimateResources, validateConfig, } from './utilities/parameters';
-export { default as topP } from './utilities/topP';
-export { sliceUint16Shards, sliceUint8Shards } from './utilities/tokens';
-export { TokenStore, createTokenStore } from './training/tasks/TokenStore';
-export { Task, tokensFromTasks } from './training/tasks/Task';
-export declare const tasks: {
-    PretrainingTask: typeof PretrainingTask;
-    StartSentenceTask: typeof StartSentenceTask;
-    ConversationTask: typeof ConversationTask;
+export type { ModelForwardAttributes } from './models/model';
+export type { IGenerateOptions, IGeneratorResponse, IGeneratorOutput, GeneratorConversation } from './inference/types';
+export type { TrainingOptions, TrainingLogEntry } from './training/types';
+export type { ITrainingJob } from './api/training';
+export declare const training: {
+    Evaluator: typeof Evaluator;
+    AdamWOptimizer: typeof AdamWOptimizer;
+};
+export declare const utilities: {
+    estimateParameterCount: typeof estimateParameterCount;
+    estimateMemoryUsage: typeof estimateMemoryUsage;
+    estimateTrainingMemoryUsage: typeof estimateTrainingMemoryUsage;
+    estimateResources: typeof estimateResources;
+    validateConfig: typeof validateConfig;
+    topP: typeof topP;
+    sliceUint16Shards: typeof sliceUint16Shards;
+    sliceUint8Shards: typeof sliceUint8Shards;
+    performanceTest: typeof performanceTest;
+    sentenceEmbeddings: typeof sentenceEmbeddings;
+    sentenceEmbeddingsTensor: typeof sentenceEmbeddingsTensor;
 };
-declare const ops: {
+export declare const ops: {
     pack16: typeof pack16;
     unpack16: typeof unpack16;
 };
-export { ops };
-export { selectBackend } from './backend';
-export { default as performanceTest } from './utilities/performance';
-export declare const layers: {
-    CausalSelfAttention: typeof CausalSelfAttention;
-    MLP: typeof MLP;
-    TransformerBlock: typeof TransformerBlock;
-    RoPECache: typeof RoPECache;
-};
-export { AdamWOptimizer } from './training/AdamW';
+export * as layers from './layers';
 export { default as checks } from './checks';
 export type { TensorStatistics } from './checks/weights';
-export { sentenceEmbeddings, sentenceEmbeddingsTensor } from './utilities/sentences';