@genai-fi/nanogpt 0.24.0 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/README.md +78 -281
  2. package/dist/{RealDiv-CNsvC4AU.js → RealDiv-CSnvtN2E.js} +20 -20
  3. package/dist/{Reshape-dnm9bO3B.js → Reshape-BlylqwWy.js} +12 -12
  4. package/dist/TeachableLLM.d.ts +10 -14
  5. package/dist/TeachableLLM.js +201 -2
  6. package/dist/api/responses.d.ts +81 -0
  7. package/dist/api/responses.js +169 -0
  8. package/dist/api/training.d.ts +70 -0
  9. package/dist/api/training.js +205 -0
  10. package/dist/data/docx.js +9 -3036
  11. package/dist/data/stream.d.ts +2 -2
  12. package/dist/data/textLoader.d.ts +1 -1
  13. package/dist/data/textLoader.js +1 -1
  14. package/dist/data.d.ts +3 -0
  15. package/dist/data.js +12 -0
  16. package/dist/{dist-BqAU9-yi.js → dist-CwK5S7Ls.js} +2168 -2168
  17. package/dist/{gpgpu_math-DBYEAAdI.js → gpgpu_math-20tPK8LM.js} +458 -458
  18. package/dist/{Generator.d.ts → inference/Generator.d.ts} +16 -44
  19. package/dist/inference/Generator.js +271 -0
  20. package/dist/inference/tokenisePrompt.d.ts +4 -0
  21. package/dist/inference/tokenisePrompt.js +13 -0
  22. package/dist/inference/types.d.ts +44 -8
  23. package/dist/inference/utilities.d.ts +9 -0
  24. package/dist/inference/utilities.js +20 -0
  25. package/dist/jszip.min-DKa1Rjyn.js +3033 -0
  26. package/dist/{kernel_funcs_utils-D-mATnGR.js → kernel_funcs_utils-ql8Y8qPn.js} +96 -93
  27. package/dist/layers/MLP.d.ts +1 -1
  28. package/dist/layers/PositionEmbedding.d.ts +2 -1
  29. package/dist/layers/PositionEmbedding.js +1 -1
  30. package/dist/layers/RMSNorm.d.ts +1 -1
  31. package/dist/layers/TiedEmbedding.js +1 -1
  32. package/dist/layers.d.ts +4 -0
  33. package/dist/layers.js +14 -0
  34. package/dist/loader/load.js +58 -2
  35. package/dist/loader/loadHF.d.ts +1 -1
  36. package/dist/loader/loadHF.js +17 -2
  37. package/dist/loader/loadTransformers.js +46 -2
  38. package/dist/loader/newZipLoad.js +25 -2
  39. package/dist/loader/oldZipLoad.d.ts +1 -1
  40. package/dist/loader/oldZipLoad.js +37 -2
  41. package/dist/loader/save.js +75 -2
  42. package/dist/loader/types.d.ts +3 -3
  43. package/dist/main.d.ts +34 -35
  44. package/dist/main.js +12327 -20
  45. package/dist/{matMulGelu-BAIgQaRx.js → matMulGelu-CBoqTZM7.js} +2 -2
  46. package/dist/models/NanoGPTV1.js +95 -2
  47. package/dist/models/NanoGPTV2.js +86 -2
  48. package/dist/models/factory.js +13 -2
  49. package/dist/models/model.js +76 -2
  50. package/dist/models.d.ts +4 -0
  51. package/dist/models.js +14 -0
  52. package/dist/ops/dot16.js +1 -1
  53. package/dist/ops/matMulGelu.js +1 -1
  54. package/dist/ops/webgl/adamAdjust.js +1 -1
  55. package/dist/ops/webgl/fusedSoftmax.js +2 -2
  56. package/dist/ops/webgl/gelu.js +2 -2
  57. package/dist/ops/webgl/log.js +5 -5
  58. package/dist/ops/webgl/matMulGelu.js +1 -1
  59. package/dist/ops/webgl/matMulMul.js +1 -1
  60. package/dist/{tfjs_backend-CydPRQTc.js → tfjs_backend-h5weiy1O.js} +36 -36
  61. package/dist/tokenise.d.ts +4 -0
  62. package/dist/tokenise.js +15 -0
  63. package/dist/training/BasicTrainer.d.ts +5 -10
  64. package/dist/training/BasicTrainer.js +80 -88
  65. package/dist/training/configure.d.ts +3 -0
  66. package/dist/training/configure.js +32 -0
  67. package/dist/training/factory.d.ts +6 -0
  68. package/dist/training/factory.js +8 -0
  69. package/dist/training/prepareData.d.ts +22 -0
  70. package/dist/training/prepareData.js +49 -0
  71. package/dist/training/tasks/tokenStream.d.ts +2 -1
  72. package/dist/training/types.d.ts +14 -1
  73. package/dist/training/validateOptions.d.ts +2 -0
  74. package/dist/training/validateOptions.js +19 -0
  75. package/dist/utilities/arrayShape.d.ts +1 -0
  76. package/dist/utilities/arrayShape.js +8 -0
  77. package/dist/utilities/waitForModel.d.ts +1 -1
  78. package/dist/v4-BK7K-jy_.js +30 -0
  79. package/package.json +8 -2
  80. package/dist/Generator.js +0 -2
  81. package/dist/Trainer-Cr7csbTD.js +0 -228
  82. package/dist/Trainer.d.ts +0 -45
  83. package/dist/Trainer.js +0 -2
  84. package/dist/main-Dz72vadm.js +0 -13267
@@ -16,9 +16,9 @@ declare class JSONLFromReadableStream implements ConversationStream {
16
16
  begin(cb: (conv: Conversation[]) => void, yieldCb?: () => void): Promise<void>;
17
17
  }
18
18
  export declare class JSONLConversationStream extends JSONLFromReadableStream {
19
- constructor(file: File);
19
+ constructor(file: Blob);
20
20
  }
21
21
  export declare class ZipJSONLConversationStream extends JSONLFromReadableStream {
22
- constructor(file: File, preferredEntryName?: string);
22
+ constructor(file: Blob, preferredEntryName?: string);
23
23
  }
24
24
  export {};
@@ -4,4 +4,4 @@ export interface DataOptions {
4
4
  column?: string;
5
5
  hasHeader?: boolean;
6
6
  }
7
- export default function loadTextData(file: File, options?: DataOptions): Promise<ConversationStream>;
7
+ export default function loadTextData(file: Blob | File, options?: DataOptions): Promise<ConversationStream>;
@@ -522,7 +522,7 @@ function d(e) {
522
522
  }
523
523
  }
524
524
  async function f(e, t) {
525
- let u = e.type === "" ? d(e.name) : e.type;
525
+ let u = e.type === "" ? e instanceof File ? d(e.name) : "application/zip" : e.type;
526
526
  if (u === "application/parquet") throw Error("Parquet loading is not currently supported in the browser. Please convert your data to JSONL format.");
527
527
  if (u === "application/pdf") return new i(await n(e, t?.maxSize));
528
528
  if (u === "application/vnd.openxmlformats-officedocument.wordprocessingml.document") return new i(await r(e));
package/dist/data.d.ts ADDED
@@ -0,0 +1,3 @@
1
+ export { default as loadTextData } from './data/textLoader';
2
+ export { MemoryConversationStream } from './data/stream';
3
+ export { default as generateDatasetID } from './utilities/datasetID';
package/dist/data.js ADDED
@@ -0,0 +1,12 @@
1
+ import { n as e } from "./chunk-CWhphoD1.js";
2
+ import t from "./utilities/datasetID.js";
3
+ import { n } from "./stream-BpAwcvHz.js";
4
+ import r from "./data/textLoader.js";
5
+ //#region lib/data.ts
6
+ var i = /* @__PURE__ */ e({
7
+ MemoryConversationStream: () => n,
8
+ generateDatasetID: () => t,
9
+ loadTextData: () => r
10
+ });
11
+ //#endregion
12
+ export { n as MemoryConversationStream, t as generateDatasetID, r as loadTextData, i as t };