@genai-fi/nanogpt 0.10.3 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/Generator.d.ts +10 -5
- package/dist/Generator.js +1789 -1765
- package/dist/{RealDiv-KAPDe8zB.js → RealDiv-C8neBwFi.js} +15 -15
- package/dist/{Reshape-BYkmUnAv.js → Reshape-Bd4V_4X7.js} +1 -1
- package/dist/{Reshape-Zt6eb7yh.js → Reshape-Ck29jQSY.js} +5 -5
- package/dist/TeachableLLM.d.ts +5 -3
- package/dist/TeachableLLM.js +14 -14
- package/dist/Trainer.d.ts +3 -1
- package/dist/Trainer.js +11 -8
- package/dist/{axis_util-BaG7mf5A.js → axis_util-DGqbT-FX.js} +3 -3
- package/dist/backend.js +2 -2
- package/dist/{backend_util-RCe-rHaj.js → backend_util-DC3rBo_H.js} +18 -18
- package/dist/{backend_webgpu-DE3ACOLx.js → backend_webgpu-mbhNnlx9.js} +3 -3
- package/dist/{broadcast_to-B3eYlZm7.js → broadcast_to-D1Dmg2Oz.js} +2 -2
- package/dist/checks/appendCache.js +2 -2
- package/dist/checks/attentionMask.js +3 -3
- package/dist/checks/gelu.js +2 -2
- package/dist/checks/matMulGelu.js +2 -2
- package/dist/checks/normRMS.js +4 -4
- package/dist/checks/normRMSGrad.js +3 -3
- package/dist/checks/packUnpack.js +2 -2
- package/dist/checks/qkv.js +4 -4
- package/dist/checks/rope.js +2 -2
- package/dist/{clip_by_value-BnO7-a88.js → clip_by_value-fg2aKzUy.js} +5 -5
- package/dist/complex-Cyg-eQeZ.js +11 -0
- package/dist/concat-CSm2rMwe.js +17 -0
- package/dist/{concat_util-DpW8mL_l.js → concat_util-D0je5Ppu.js} +1 -1
- package/dist/{dataset-BcwmTGYc.js → dataset-CVIJu7Xa.js} +7 -7
- package/dist/{dropout-BcvN9JYi.js → dropout-DLhSMNTZ.js} +9 -9
- package/dist/expand_dims-ChkuOp6I.js +11 -0
- package/dist/{exports_initializers-Hta_rEnm.js → exports_initializers-1KWPiStI.js} +1 -1
- package/dist/{floor-D5QdR_le.js → floor-BRMPgeIs.js} +1 -1
- package/dist/{gather-D3JcZUaI.js → gather-BSULDalH.js} +1 -1
- package/dist/{gelu-CjNPL4OH.js → gelu-BK1k-n1i.js} +1 -1
- package/dist/{gpgpu_math-DAOmgtXR.js → gpgpu_math-BJSTk_mW.js} +25 -25
- package/dist/{index-BwexR4lA.js → index-BBVLAXZD.js} +89 -89
- package/dist/{index-DOvlwCh-.js → index-Duu1Lvvv.js} +53 -53
- package/dist/{kernel_funcs_utils-CCzYdUZg.js → kernel_funcs_utils-BtYrPoJu.js} +6 -6
- package/dist/layers/BaseLayer.js +2 -2
- package/dist/layers/CausalSelfAttention.js +6 -6
- package/dist/layers/MLP.js +4 -4
- package/dist/layers/PositionEmbedding.js +5 -5
- package/dist/layers/RMSNorm.js +3 -3
- package/dist/layers/RoPECache.js +4 -4
- package/dist/layers/TiedEmbedding.js +6 -6
- package/dist/layers/TransformerBlock.js +1 -1
- package/dist/loader/loadTransformers.js +1 -1
- package/dist/loader/oldZipLoad.js +9 -9
- package/dist/log_sum_exp-CVqLsVLl.js +39 -0
- package/dist/main.d.ts +10 -1
- package/dist/main.js +68 -58
- package/dist/{matMul16-BWRSOCWB.js → matMul16-xswmhSuF.js} +3 -3
- package/dist/{matMulGelu-CzfgT6Wq.js → matMulGelu-BpvgnYG8.js} +14 -14
- package/dist/mat_mul-Bn2BDpT4.js +11 -0
- package/dist/{mod-AnXEvvpo.js → mod-B4AUd1Np.js} +1 -1
- package/dist/models/NanoGPTV1.js +2 -2
- package/dist/models/model.js +9 -9
- package/dist/{ones-D2rT0xk2.js → ones-CBI1AQjb.js} +3 -3
- package/dist/ops/adamAdjust.js +1 -1
- package/dist/ops/adamMoments.js +1 -1
- package/dist/ops/add16.js +1 -1
- package/dist/ops/appendCache.js +3 -3
- package/dist/ops/attentionMask.js +1 -1
- package/dist/ops/concat16.js +2 -2
- package/dist/ops/cpu/adamAdjust.js +9 -9
- package/dist/ops/cpu/adamMoments.js +5 -5
- package/dist/ops/cpu/appendCache.js +6 -6
- package/dist/ops/cpu/attentionMask.js +10 -10
- package/dist/ops/cpu/fusedSoftmax.js +5 -5
- package/dist/ops/cpu/gatherSub.js +9 -9
- package/dist/ops/cpu/gelu.js +5 -5
- package/dist/ops/cpu/matMul16.js +2 -2
- package/dist/ops/cpu/matMulGelu.js +3 -3
- package/dist/ops/cpu/matMulMul.js +5 -5
- package/dist/ops/cpu/mulDropout.js +1 -1
- package/dist/ops/cpu/normRMS.js +7 -7
- package/dist/ops/cpu/qkv.js +3 -3
- package/dist/ops/cpu/rope.js +5 -5
- package/dist/ops/cpu/scatterSub.js +11 -11
- package/dist/ops/dot16.js +2 -2
- package/dist/ops/gatherSub.js +1 -1
- package/dist/ops/gelu.js +2 -2
- package/dist/ops/grads/add16.js +4 -4
- package/dist/ops/grads/attentionMask.js +2 -2
- package/dist/ops/grads/gelu.js +2 -2
- package/dist/ops/grads/matMul16.js +3 -3
- package/dist/ops/grads/matMulGelu.js +6 -6
- package/dist/ops/grads/normRMS.js +4 -4
- package/dist/ops/grads/pack16.js +3 -3
- package/dist/ops/grads/qkv.js +10 -10
- package/dist/ops/grads/rope.js +2 -2
- package/dist/ops/grads/softmax16.js +1 -1
- package/dist/ops/grads/unpack16.js +2 -2
- package/dist/ops/matMul16.js +3 -3
- package/dist/ops/matMulGelu.js +2 -2
- package/dist/ops/matMulMul.js +1 -1
- package/dist/ops/mul16.js +1 -1
- package/dist/ops/mulDrop.js +1 -1
- package/dist/ops/normRMS.js +1 -1
- package/dist/ops/pack16.js +2 -2
- package/dist/ops/qkv.js +1 -1
- package/dist/ops/reshape16.js +2 -2
- package/dist/ops/rope.js +2 -2
- package/dist/ops/scatterSub.js +1 -1
- package/dist/ops/slice16.js +2 -2
- package/dist/ops/softmax16.js +1 -1
- package/dist/ops/sub16.js +1 -1
- package/dist/ops/sum16.js +2 -2
- package/dist/ops/transpose16.js +6 -6
- package/dist/ops/unpack16.js +2 -2
- package/dist/ops/webgl/adamAdjust.js +2 -2
- package/dist/ops/webgl/adamMoments.js +1 -1
- package/dist/ops/webgl/appendCache.js +1 -1
- package/dist/ops/webgl/attentionMask.js +1 -1
- package/dist/ops/webgl/fusedSoftmax.js +4 -4
- package/dist/ops/webgl/gatherSub.js +1 -1
- package/dist/ops/webgl/gelu.js +2 -2
- package/dist/ops/webgl/log.js +3 -3
- package/dist/ops/webgl/matMul16.js +8 -8
- package/dist/ops/webgl/matMulGelu.js +4 -4
- package/dist/ops/webgl/matMulMul.js +7 -7
- package/dist/ops/webgl/mulDropout.js +1 -1
- package/dist/ops/webgl/normRMS.js +7 -7
- package/dist/ops/webgl/qkv.js +1 -1
- package/dist/ops/webgl/rope.js +1 -1
- package/dist/ops/webgl/scatterSub.js +1 -1
- package/dist/ops/webgpu/adamAdjust.js +3 -3
- package/dist/ops/webgpu/adamMoments.js +5 -5
- package/dist/ops/webgpu/add16.js +1 -1
- package/dist/ops/webgpu/appendCache.js +3 -3
- package/dist/ops/webgpu/attentionMask.js +2 -2
- package/dist/ops/webgpu/attentionMask32_program.js +2 -2
- package/dist/ops/webgpu/concat16.js +5 -5
- package/dist/ops/webgpu/gatherSub.js +5 -5
- package/dist/ops/webgpu/gelu.js +3 -3
- package/dist/ops/webgpu/matMul16.js +19 -19
- package/dist/ops/webgpu/matMul16_program.js +2 -2
- package/dist/ops/webgpu/mul16.js +4 -4
- package/dist/ops/webgpu/normRMS.js +6 -6
- package/dist/ops/webgpu/normRMSGrad.js +4 -4
- package/dist/ops/webgpu/pack16.js +3 -3
- package/dist/ops/webgpu/pack16_program.js +2 -2
- package/dist/ops/webgpu/qkv.js +8 -8
- package/dist/ops/webgpu/rope.js +3 -3
- package/dist/ops/webgpu/scatterSub.js +3 -3
- package/dist/ops/webgpu/slice16.js +4 -4
- package/dist/ops/webgpu/softmax16.js +4 -4
- package/dist/ops/webgpu/softmax16_program.js +2 -2
- package/dist/ops/webgpu/softmax16_subgroup_program.js +2 -2
- package/dist/ops/webgpu/softmax16grad.js +1 -1
- package/dist/ops/webgpu/sub16.js +4 -4
- package/dist/ops/webgpu/sum16.js +5 -5
- package/dist/ops/webgpu/transpose16.js +2 -2
- package/dist/ops/webgpu/transpose16_program.js +2 -2
- package/dist/ops/webgpu/transpose16_shared_program.js +3 -3
- package/dist/ops/webgpu/unpack16.js +5 -5
- package/dist/ops/webgpu/utils/binary_op.js +3 -3
- package/dist/ops/webgpu/utils/reductions.js +4 -4
- package/dist/{ops-B5yanEdW.js → ops-C2_OXuZ4.js} +69 -69
- package/dist/{pack16-nQ6JaLo-.js → pack16-atD0eYRm.js} +9 -9
- package/dist/patches/webgpu_backend.js +6 -6
- package/dist/patches/webgpu_base.js +1 -1
- package/dist/patches/webgpu_program.js +8 -8
- package/dist/{random_width-or-CEftb.js → random_width-BN4wGJaW.js} +33 -33
- package/dist/range-DKmP1-OQ.js +10 -0
- package/dist/relu-BsXmGzzu.js +9 -0
- package/dist/{reshape-ByE68wS9.js → reshape-BI0yzp1T.js} +1 -1
- package/dist/{resize_nearest_neighbor-B19mCEg2.js → resize_nearest_neighbor-BA_BX-ub.js} +26 -26
- package/dist/{rope-Ir4mTyD1.js → rope-DJ7Y7c-u.js} +1 -1
- package/dist/{scatter_nd_util-lvSiX8q4.js → scatter_nd_util-k9MUVUkn.js} +1 -1
- package/dist/{selu_util-kbhpTdYD.js → selu_util-DyW0X1WG.js} +5 -5
- package/dist/{shared-DT1TkE6w.js → shared-Q3BS6T03.js} +1 -1
- package/dist/{shared-dntlHIDQ.js → shared-nnSWpC3u.js} +86 -86
- package/dist/{slice-BfEGSH82.js → slice-wBNvzVyz.js} +1 -1
- package/dist/{slice_util-uTKwiEpW.js → slice_util-zN8KFC5I.js} +1 -1
- package/dist/{softmax-CA5jFsLR.js → softmax-DfuYyjMh.js} +1 -1
- package/dist/split-BYrLboMq.js +9 -0
- package/dist/squeeze-Bk8Brcct.js +10 -0
- package/dist/{stack-Cf4n9h0N.js → stack-CDWShFHF.js} +1 -1
- package/dist/{step-CINUs5QB.js → step-BS5JXRR6.js} +23 -23
- package/dist/{sum-DWAtNGez.js → sum-BPUfDB2X.js} +3 -3
- package/dist/tensor-CEt9Nm2s.js +8 -0
- package/dist/tensor1d-Cc_KCIDg.js +11 -0
- package/dist/{tensor2d-Bs9wZRc7.js → tensor2d-BN97fF71.js} +3 -3
- package/dist/{tensor4d-BARPdTaS.js → tensor4d-vuDDgdUI.js} +1 -1
- package/dist/{tfjs_backend-y1cvNhLA.js → tfjs_backend-806hyYve.js} +49 -49
- package/dist/{tile-mbfagpsB.js → tile-OWUvpIVt.js} +3 -3
- package/dist/tokeniser/BaseTokeniser.d.ts +25 -0
- package/dist/tokeniser/BaseTokeniser.js +94 -0
- package/dist/tokeniser/CharTokeniser.d.ts +10 -9
- package/dist/tokeniser/CharTokeniser.js +44 -30
- package/dist/tokeniser/bpe.d.ts +10 -9
- package/dist/tokeniser/bpe.js +67 -52
- package/dist/tokeniser/type.d.ts +14 -5
- package/dist/training/Adam.js +2 -2
- package/dist/training/AdamExt.js +1 -1
- package/dist/training/DatasetBuilder.d.ts +3 -3
- package/dist/training/DatasetBuilder.js +34 -38
- package/dist/training/FullTrainer.js +1 -1
- package/dist/training/Trainer.d.ts +4 -3
- package/dist/training/Trainer.js +22 -25
- package/dist/training/sparseCrossEntropy.js +3 -3
- package/dist/training/tasks/ConversationTask.d.ts +11 -0
- package/dist/training/tasks/ConversationTask.js +26 -0
- package/dist/training/tasks/PretrainingTask.d.ts +11 -0
- package/dist/training/tasks/PretrainingTask.js +34 -0
- package/dist/training/tasks/StartSentenceTask.d.ts +12 -0
- package/dist/training/tasks/StartSentenceTask.js +42 -0
- package/dist/training/tasks/Task.d.ts +8 -0
- package/dist/training/tasks/Task.js +41 -0
- package/dist/{transpose-ClWiBS_b.js → transpose-BUkQCJp9.js} +6 -6
- package/dist/{unsorted_segment_sum-BDDhB_E6.js → unsorted_segment_sum-BljxHhCY.js} +5 -5
- package/dist/utilities/dummy.js +3 -3
- package/dist/utilities/multinomialCPU.js +2 -2
- package/dist/utilities/packed.js +1 -1
- package/dist/utilities/performance.js +1 -1
- package/dist/utilities/profile.js +1 -1
- package/dist/utilities/safetensors.js +2 -2
- package/dist/utilities/sentences.d.ts +1 -1
- package/dist/utilities/sentences.js +11 -11
- package/dist/utilities/weights.js +2 -2
- package/dist/{variable-WawDEaAb.js → variable-DPt_Iuog.js} +1 -1
- package/dist/{webgpu_program-DuOXPQol.js → webgpu_program-BpWRlghH.js} +3 -3
- package/dist/{webgpu_util-RxEF33Rj.js → webgpu_util-DMiKzzQM.js} +7 -7
- package/dist/{zeros-KnWaWf-X.js → zeros-5YROwwUH.js} +2 -2
- package/dist/{zeros_like-DvE73F4e.js → zeros_like-De4n1C3m.js} +71 -71
- package/package.json +1 -1
- package/dist/complex-DjxcVmoX.js +0 -11
- package/dist/concat-BV8bt5H-.js +0 -17
- package/dist/expand_dims-DT4tEPwA.js +0 -11
- package/dist/log_sum_exp-ngO0-4pK.js +0 -39
- package/dist/mat_mul-SjpJRLyL.js +0 -11
- package/dist/range-BklejeeW.js +0 -10
- package/dist/relu-CP0ZcxWO.js +0 -9
- package/dist/split-CVLc0w--.js +0 -9
- package/dist/squeeze-C7Z2srUo.js +0 -10
- package/dist/tensor-DJoc7gJU.js +0 -8
- package/dist/tensor1d-D11P_7Dp.js +0 -11
package/dist/training/Trainer.js
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
import { DatasetBuilder as
|
|
1
|
+
import { DatasetBuilder as u, PAGE_FACTOR as f } from "./DatasetBuilder.js";
|
|
2
2
|
import z from "./AdamExt.js";
|
|
3
|
-
import { t as S, v as
|
|
4
|
-
import {
|
|
5
|
-
|
|
3
|
+
import { t as S, v as y, k, d as h, b as p } from "../index-Duu1Lvvv.js";
|
|
4
|
+
import { tokensFromTasks as x } from "./tasks/Task.js";
|
|
5
|
+
import { z as m } from "../zeros-5YROwwUH.js";
|
|
6
|
+
class B {
|
|
6
7
|
constructor(t, e, s = 1e-3) {
|
|
7
|
-
this.tokenizer = e, this.model = t, this.lossScaling = t.lossScaling, this.learningRate = s, this.resetOptimizer(), this.datasetBuilder = new
|
|
8
|
+
this.tokenizer = e, this.model = t, this.lossScaling = t.lossScaling, this.learningRate = s, this.resetOptimizer(), this.datasetBuilder = new u(e, t.config.blockSize);
|
|
8
9
|
}
|
|
9
10
|
model;
|
|
10
11
|
optimizer;
|
|
@@ -53,8 +54,8 @@ class M {
|
|
|
53
54
|
trainStep(t, e, s = !1, i = !1) {
|
|
54
55
|
return S(() => {
|
|
55
56
|
this.model.getProfiler()?.startMemory();
|
|
56
|
-
const { xs: a, ys: l } = e,
|
|
57
|
-
const [
|
|
57
|
+
const { xs: a, ys: l } = e, d = () => {
|
|
58
|
+
const [o, c] = this.model.forward(
|
|
58
59
|
{
|
|
59
60
|
training: !0,
|
|
60
61
|
checkpointing: this._gradientCheckpointing,
|
|
@@ -63,15 +64,15 @@ class M {
|
|
|
63
64
|
a,
|
|
64
65
|
l
|
|
65
66
|
);
|
|
66
|
-
|
|
67
|
-
const
|
|
68
|
-
return
|
|
69
|
-
}, { value:
|
|
70
|
-
return s ? this.model.getProfiler()?.endMemory("Training") : (this.optimizer.applyGradients(r), this.model.getProfiler()?.endMemory("Training"), i ? (t.gradients = r, Object.values(r).forEach((
|
|
67
|
+
o.dispose();
|
|
68
|
+
const g = c.mul(p(this.lossScaling));
|
|
69
|
+
return c.dispose(), g;
|
|
70
|
+
}, { value: n, grads: r } = y(d);
|
|
71
|
+
return s ? this.model.getProfiler()?.endMemory("Training") : (this.optimizer.applyGradients(r), this.model.getProfiler()?.endMemory("Training"), i ? (t.gradients = r, Object.values(r).forEach((o) => k(o))) : h(r)), n.mul(p(1 / this.lossScaling));
|
|
71
72
|
});
|
|
72
73
|
}
|
|
73
74
|
async dummyPass() {
|
|
74
|
-
const t =
|
|
75
|
+
const t = m([1, this.model.config.blockSize], "int32"), e = m([1, this.model.config.blockSize], "int32");
|
|
75
76
|
try {
|
|
76
77
|
const s = this.trainStep({}, { xs: t, ys: e }, !0);
|
|
77
78
|
await s.data(), s.dispose();
|
|
@@ -86,34 +87,30 @@ class M {
|
|
|
86
87
|
const i = this.trainStep(t, e, !1, s);
|
|
87
88
|
return e.xs.dispose(), e.ys.dispose(), t.step++, t.totalSteps++, i;
|
|
88
89
|
} catch (i) {
|
|
89
|
-
throw console.error(`Error processing batch at step ${t.step}:`, i),
|
|
90
|
+
throw console.error(`Error processing batch at step ${t.step}:`, i), h(), i;
|
|
90
91
|
}
|
|
91
92
|
}
|
|
92
93
|
async createTrainValidationSplit(t, e = 32, s = 0.1) {
|
|
93
|
-
const i = await
|
|
94
|
+
const i = await x(t, this.tokenizer), a = /* @__PURE__ */ new Set();
|
|
94
95
|
if (s > 0) {
|
|
95
|
-
const
|
|
96
|
+
const n = Math.floor(i.length / (this.datasetBuilder.blockSize * f)), r = Math.max(1, Math.floor(n * s));
|
|
96
97
|
for (; a.size < r; ) {
|
|
97
|
-
const
|
|
98
|
-
a.add(
|
|
98
|
+
const o = Math.floor(Math.random() * n);
|
|
99
|
+
a.add(o);
|
|
99
100
|
}
|
|
100
101
|
}
|
|
101
|
-
const l = await this.datasetBuilder.createTextDataset(i, e, a, !1),
|
|
102
|
+
const l = await this.datasetBuilder.createTextDataset(i, e, a, !1), d = await this.datasetBuilder.createTextDataset(
|
|
102
103
|
i,
|
|
103
104
|
e,
|
|
104
105
|
a,
|
|
105
106
|
!0
|
|
106
107
|
);
|
|
107
|
-
return { trainDataset: l, validationDataset:
|
|
108
|
-
}
|
|
109
|
-
async createDataset(t, e = 32) {
|
|
110
|
-
const s = await h(t, this.tokenizer);
|
|
111
|
-
return await this.datasetBuilder.createTextDataset(s, e);
|
|
108
|
+
return { trainDataset: l, validationDataset: d, size: i.length };
|
|
112
109
|
}
|
|
113
110
|
dispose() {
|
|
114
111
|
this.optimizer && this.optimizer.dispose();
|
|
115
112
|
}
|
|
116
113
|
}
|
|
117
114
|
export {
|
|
118
|
-
|
|
115
|
+
B as default
|
|
119
116
|
};
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { gatherSub as x } from "../ops/gatherSub.js";
|
|
2
2
|
import { scatterSub as L } from "../ops/scatterSub.js";
|
|
3
|
-
import {
|
|
4
|
-
import { s as y } from "../softmax-
|
|
5
|
-
import { m as z, l as v } from "../log_sum_exp-
|
|
3
|
+
import { a1 as C, t as u, a2 as E, c as G } from "../index-Duu1Lvvv.js";
|
|
4
|
+
import { s as y } from "../softmax-DfuYyjMh.js";
|
|
5
|
+
import { m as z, l as v } from "../log_sum_exp-CVqLsVLl.js";
|
|
6
6
|
function k(t, s) {
|
|
7
7
|
return u(() => {
|
|
8
8
|
const n = t.shape[t.shape.length - 1], c = t.shape.slice(0, -1).reduce((o, e) => o * e, 1), h = t.shape.length > 2 ? t.reshape([c, n]) : t, p = s.shape.length > 1 ? s.reshape([c]).cast("int32") : s.cast("int32"), r = z(h, -1, !0), a = G(h, r), d = v(a, -1);
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { Conversation, ITokeniser } from '../../main';
|
|
2
|
+
import { Task } from './Task';
|
|
3
|
+
export default class ConversationTask extends Task {
|
|
4
|
+
private rawConvo;
|
|
5
|
+
private index;
|
|
6
|
+
get length(): number;
|
|
7
|
+
constructor(conversations: Conversation[][]);
|
|
8
|
+
hasMoreConversations(): boolean;
|
|
9
|
+
nextConversation(): Conversation[] | null;
|
|
10
|
+
estimateTokens(tokeniser: ITokeniser): Promise<number>;
|
|
11
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { Task as t } from "./Task.js";
|
|
2
|
+
class s extends t {
|
|
3
|
+
rawConvo;
|
|
4
|
+
index = 0;
|
|
5
|
+
get length() {
|
|
6
|
+
return this.rawConvo.length;
|
|
7
|
+
}
|
|
8
|
+
constructor(n) {
|
|
9
|
+
super(), this.rawConvo = n;
|
|
10
|
+
}
|
|
11
|
+
hasMoreConversations() {
|
|
12
|
+
return this.index < this.rawConvo.length;
|
|
13
|
+
}
|
|
14
|
+
nextConversation() {
|
|
15
|
+
if (this.index >= this.rawConvo.length)
|
|
16
|
+
return null;
|
|
17
|
+
const n = this.rawConvo[this.index];
|
|
18
|
+
return this.index++, n;
|
|
19
|
+
}
|
|
20
|
+
async estimateTokens(n) {
|
|
21
|
+
return (await n.encodeConversation(this.rawConvo[0])).length * this.length;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
export {
|
|
25
|
+
s as default
|
|
26
|
+
};
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { Conversation, ITokeniser } from '../../main';
|
|
2
|
+
import { Task } from './Task';
|
|
3
|
+
export default class PretrainingTask extends Task {
|
|
4
|
+
private rawText;
|
|
5
|
+
private index;
|
|
6
|
+
get length(): number;
|
|
7
|
+
constructor(texts: string[]);
|
|
8
|
+
hasMoreConversations(): boolean;
|
|
9
|
+
nextConversation(): Conversation[] | null;
|
|
10
|
+
estimateTokens(tokeniser: ITokeniser): Promise<number>;
|
|
11
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { Task as e } from "./Task.js";
|
|
2
|
+
class r extends e {
|
|
3
|
+
rawText;
|
|
4
|
+
index = 0;
|
|
5
|
+
get length() {
|
|
6
|
+
return this.rawText.length;
|
|
7
|
+
}
|
|
8
|
+
constructor(t) {
|
|
9
|
+
super(), this.rawText = t;
|
|
10
|
+
}
|
|
11
|
+
hasMoreConversations() {
|
|
12
|
+
return this.index < this.rawText.length;
|
|
13
|
+
}
|
|
14
|
+
nextConversation() {
|
|
15
|
+
if (this.index >= this.rawText.length)
|
|
16
|
+
return null;
|
|
17
|
+
const t = {
|
|
18
|
+
role: "assistant",
|
|
19
|
+
content: this.rawText[this.index]
|
|
20
|
+
};
|
|
21
|
+
return this.index++, [t];
|
|
22
|
+
}
|
|
23
|
+
async estimateTokens(t) {
|
|
24
|
+
return (await t.encodeConversation([
|
|
25
|
+
{
|
|
26
|
+
role: "assistant",
|
|
27
|
+
content: this.rawText[0]
|
|
28
|
+
}
|
|
29
|
+
])).length * this.length;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
export {
|
|
33
|
+
r as default
|
|
34
|
+
};
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { Conversation, ITokeniser } from '../../main';
|
|
2
|
+
import { Task } from './Task';
|
|
3
|
+
export default class StartSentenceTask extends Task {
|
|
4
|
+
private rawText;
|
|
5
|
+
private index;
|
|
6
|
+
get length(): number;
|
|
7
|
+
constructor(texts: string[]);
|
|
8
|
+
hasMoreConversations(): boolean;
|
|
9
|
+
nextConversation(): Conversation[] | null;
|
|
10
|
+
private conversationFromString;
|
|
11
|
+
estimateTokens(tokeniser: ITokeniser): Promise<number>;
|
|
12
|
+
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { Task as e } from "./Task.js";
|
|
2
|
+
class a extends e {
|
|
3
|
+
rawText;
|
|
4
|
+
index = 0;
|
|
5
|
+
get length() {
|
|
6
|
+
return this.rawText.length;
|
|
7
|
+
}
|
|
8
|
+
constructor(t) {
|
|
9
|
+
super(), this.rawText = t;
|
|
10
|
+
}
|
|
11
|
+
hasMoreConversations() {
|
|
12
|
+
return this.index < this.rawText.length;
|
|
13
|
+
}
|
|
14
|
+
nextConversation() {
|
|
15
|
+
if (this.index >= this.rawText.length)
|
|
16
|
+
return null;
|
|
17
|
+
const t = this.rawText[this.index];
|
|
18
|
+
return this.index++, this.conversationFromString(t);
|
|
19
|
+
}
|
|
20
|
+
conversationFromString(t) {
|
|
21
|
+
const n = t.indexOf(".");
|
|
22
|
+
return n === -1 ? [{
|
|
23
|
+
role: "assistant",
|
|
24
|
+
content: this.rawText[this.index]
|
|
25
|
+
}] : [
|
|
26
|
+
{
|
|
27
|
+
role: "user",
|
|
28
|
+
content: t.slice(0, n + 1).trim()
|
|
29
|
+
},
|
|
30
|
+
{
|
|
31
|
+
role: "assistant",
|
|
32
|
+
content: t.slice(n + 1).trim()
|
|
33
|
+
}
|
|
34
|
+
];
|
|
35
|
+
}
|
|
36
|
+
async estimateTokens(t) {
|
|
37
|
+
return (await t.encodeConversation(this.conversationFromString(this.rawText[0]))).length * this.length;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
export {
|
|
41
|
+
a as default
|
|
42
|
+
};
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { Conversation, ITokeniser } from '../../main';
|
|
2
|
+
export declare abstract class Task {
|
|
3
|
+
abstract get length(): number;
|
|
4
|
+
abstract hasMoreConversations(): boolean;
|
|
5
|
+
abstract nextConversation(): Conversation[] | null;
|
|
6
|
+
abstract estimateTokens(tokeniser: ITokeniser): Promise<number>;
|
|
7
|
+
}
|
|
8
|
+
export declare function tokensFromTasks(tasks: Task[], tokenizer: ITokeniser): Promise<Uint16Array>;
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
class g {
|
|
2
|
+
}
|
|
3
|
+
function h(f, a, l, e, r) {
|
|
4
|
+
for (let i = 0; i < f.length; i++) {
|
|
5
|
+
const c = f[i].nextConversation();
|
|
6
|
+
if (c) {
|
|
7
|
+
const o = l.encodeConversation(c), s = a[a.length - 1];
|
|
8
|
+
if (e.offset + o.length > s.length) {
|
|
9
|
+
const n = s.length - e.offset;
|
|
10
|
+
s.set(o.slice(0, n), e.offset);
|
|
11
|
+
const t = new Uint16Array(Math.floor(r * 0.1) + 100);
|
|
12
|
+
t.set(o.slice(n), 0), a.push(t), e.offset = o.length - n;
|
|
13
|
+
} else
|
|
14
|
+
s.set(o, e.offset), e.offset += o.length;
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
async function w(f, a) {
|
|
19
|
+
const l = (await Promise.all(f.map((n) => n.estimateTokens(a)))).reduce(
|
|
20
|
+
(n, t) => n + t,
|
|
21
|
+
0
|
|
22
|
+
), e = [new Uint16Array(l)], r = {
|
|
23
|
+
offset: 0
|
|
24
|
+
};
|
|
25
|
+
let i = performance.now();
|
|
26
|
+
for (; r.offset < l && (h(f, e, a, r, l), !f.every((t) => !t.hasMoreConversations())); )
|
|
27
|
+
performance.now() - i > 40 && (await new Promise(requestAnimationFrame), i = performance.now());
|
|
28
|
+
if (e.length === 1)
|
|
29
|
+
return e[0].subarray(0, r.offset);
|
|
30
|
+
const c = e.reduce((n, t) => n + t.length, 0) - (e[e.length - 1].length - r.offset), o = new Uint16Array(c);
|
|
31
|
+
let s = 0;
|
|
32
|
+
for (let n = 0; n < e.length; n++) {
|
|
33
|
+
const t = e[n];
|
|
34
|
+
n === e.length - 1 ? (o.set(t.subarray(0, r.offset), s), s += r.offset) : (o.set(t, s), s += t.length);
|
|
35
|
+
}
|
|
36
|
+
return o;
|
|
37
|
+
}
|
|
38
|
+
export {
|
|
39
|
+
g as Task,
|
|
40
|
+
w as tokensFromTasks
|
|
41
|
+
};
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { c as k } from "./complex-
|
|
1
|
+
import { o as u, q as i, E as o, ap as $, aq as g, ar as x, x as l, t as m, as as p } from "./index-Duu1Lvvv.js";
|
|
2
|
+
import { c as k } from "./complex-Cyg-eQeZ.js";
|
|
3
3
|
function K(r) {
|
|
4
4
|
const e = { input: i(r, "input", "imag") };
|
|
5
5
|
return o.runKernel($, e);
|
|
@@ -12,7 +12,7 @@ function E(r) {
|
|
|
12
12
|
const _ = /* @__PURE__ */ u({ neg_: E });
|
|
13
13
|
function b(r) {
|
|
14
14
|
const e = { input: i(r, "input", "real") };
|
|
15
|
-
return o.runKernel(
|
|
15
|
+
return o.runKernel(x, e);
|
|
16
16
|
}
|
|
17
17
|
const d = /* @__PURE__ */ u({ real_: b });
|
|
18
18
|
function N(r, t, e) {
|
|
@@ -22,15 +22,15 @@ function N(r, t, e) {
|
|
|
22
22
|
}), n.rank <= 1)
|
|
23
23
|
return n.clone();
|
|
24
24
|
const f = { x: n }, c = { perm: t };
|
|
25
|
-
return n.dtype === "complex64" ?
|
|
25
|
+
return n.dtype === "complex64" ? m(() => {
|
|
26
26
|
let s = d(n), a = h(n);
|
|
27
27
|
return s = o.runKernel(p, { x: s }, c), a = o.runKernel(p, { x: a }, c), e && (a = _(a)), k(s, a);
|
|
28
28
|
}) : o.runKernel(p, f, c);
|
|
29
29
|
}
|
|
30
|
-
const
|
|
30
|
+
const v = /* @__PURE__ */ u({ transpose_: N });
|
|
31
31
|
export {
|
|
32
32
|
h as i,
|
|
33
33
|
_ as n,
|
|
34
34
|
d as r,
|
|
35
|
-
|
|
35
|
+
v as t
|
|
36
36
|
};
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { k as C, c as g, m as D } from "./step-
|
|
3
|
-
import { r as b } from "./reshape-
|
|
4
|
-
import { m as pn, a as hn, e as w } from "./log_sum_exp-
|
|
5
|
-
import { s as K } from "./sum-
|
|
1
|
+
import { o as h, q as c, E as d, bo as T, bp as q, bq as H, x as l, br as P, L as _, bs as y, bt as B, bu as I, bv as W, bw as A, bx as G, by as L, bz as O, bA as z, bB as F, B as M, _ as j, bC as J, bD as Q, bE as U, a1 as V, c as N, m as X, bF as Y, bG as Z, bH as R, bI as nn, bJ as tn, bK as sn, bL as en, bM as rn, bN as on, bO as an, bP as un, aG as cn, bQ as ln } from "./index-Duu1Lvvv.js";
|
|
2
|
+
import { k as C, c as g, m as D } from "./step-BS5JXRR6.js";
|
|
3
|
+
import { r as b } from "./reshape-BI0yzp1T.js";
|
|
4
|
+
import { m as pn, a as hn, e as w } from "./log_sum_exp-CVqLsVLl.js";
|
|
5
|
+
import { s as K } from "./sum-BPUfDB2X.js";
|
|
6
6
|
function fn(s, n = null, t = !1) {
|
|
7
7
|
const i = { x: c(s, "x", "all", "bool") }, o = { axis: n, keepDims: t };
|
|
8
8
|
return d.runKernel(T, i, o);
|
package/dist/utilities/dummy.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { a as y, e as S, v as w } from "../index-
|
|
2
|
-
import { z as m } from "../zeros-
|
|
3
|
-
import { o as P } from "../ones-
|
|
1
|
+
import { a as y, e as S, v as w } from "../index-Duu1Lvvv.js";
|
|
2
|
+
import { z as m } from "../zeros-5YROwwUH.js";
|
|
3
|
+
import { o as P } from "../ones-CBI1AQjb.js";
|
|
4
4
|
async function b(s) {
|
|
5
5
|
const t = m([1, s.config.blockSize], "int32"), [n, o] = s.forward({ training: !1 }, t);
|
|
6
6
|
await n.data(), n.dispose(), o && o.dispose(), t.dispose();
|
package/dist/utilities/packed.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { default as TeachableLLM } from '../TeachableLLM';
|
|
2
2
|
import { Tensor2D, Tensor3D } from '@tensorflow/tfjs-core';
|
|
3
3
|
export declare function meanPooling(embeddings: Tensor3D, attentionMask?: Tensor2D): Tensor2D;
|
|
4
|
-
export declare function sentenceEmbeddingsTensor(model: TeachableLLM, sentences: string[], batchSize?: number):
|
|
4
|
+
export declare function sentenceEmbeddingsTensor(model: TeachableLLM, sentences: string[], batchSize?: number): Tensor2D;
|
|
5
5
|
export declare function sentenceEmbeddings(model: TeachableLLM, sentences: string[], batchSize?: number): Promise<number[][]>;
|
|
@@ -1,26 +1,26 @@
|
|
|
1
|
-
import { m as w } from "../index-
|
|
2
|
-
import { t as
|
|
3
|
-
import { e as y } from "../expand_dims-
|
|
4
|
-
import { s as
|
|
5
|
-
import { c as T } from "../concat-
|
|
1
|
+
import { m as w } from "../index-Duu1Lvvv.js";
|
|
2
|
+
import { t as f } from "../tensor2d-BN97fF71.js";
|
|
3
|
+
import { e as y } from "../expand_dims-ChkuOp6I.js";
|
|
4
|
+
import { s as g } from "../sum-BPUfDB2X.js";
|
|
5
|
+
import { c as T } from "../concat-CSm2rMwe.js";
|
|
6
6
|
const p = 16;
|
|
7
7
|
function A(o, t) {
|
|
8
8
|
if (!t)
|
|
9
9
|
return o.mean(1);
|
|
10
|
-
const r = y(t, 2), i = w(o, r), e =
|
|
10
|
+
const r = y(t, 2), i = w(o, r), e = g(i, 1), s = g(t, 1, !0), c = e.div(s.maximum(1e-9));
|
|
11
11
|
return r.dispose(), i.dispose(), e.dispose(), s.dispose(), c;
|
|
12
12
|
}
|
|
13
|
-
|
|
13
|
+
function E(o, t, r = p) {
|
|
14
14
|
const i = o.tokeniser, e = o.config.blockSize;
|
|
15
15
|
let s = null, c = 0;
|
|
16
16
|
for (; c < t.length; ) {
|
|
17
|
-
const
|
|
18
|
-
for (const n of
|
|
17
|
+
const b = t.slice(c, c + p).map((n) => i.encode(n)), l = [], d = [];
|
|
18
|
+
for (const n of b)
|
|
19
19
|
n.length > e ? (l.push(n.slice(n.length - e, n.length)), d.push(new Array(e).fill(1))) : n.length < e ? (l.push(n.concat(new Array(e - n.length).fill(0))), d.push(
|
|
20
20
|
new Array(n.length).fill(1).concat(new Array(e - n.length).fill(0))
|
|
21
21
|
)) : (l.push(n), d.push(new Array(e).fill(1)));
|
|
22
|
-
const
|
|
23
|
-
if (
|
|
22
|
+
const k = f(l, [l.length, e], "int32"), m = f(d, [d.length, e], "float32"), u = o.model.forward({ skipLogits: !0, training: !1 }, k)[0], a = A(u, m);
|
|
23
|
+
if (m.dispose(), u.dispose(), s === null)
|
|
24
24
|
s = a;
|
|
25
25
|
else {
|
|
26
26
|
const n = s;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import "../index-
|
|
2
|
-
import { t as p } from "../tensor-
|
|
1
|
+
import "../index-Duu1Lvvv.js";
|
|
2
|
+
import { t as p } from "../tensor-CEt9Nm2s.js";
|
|
3
3
|
function h(n) {
|
|
4
4
|
const e = n.reduce((s, o) => s + o.length, 0), a = new Float32Array(e);
|
|
5
5
|
let t = 0;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { ac as z, ab as F, aa as E, a8 as j, x as A } from "./index-Duu1Lvvv.js";
|
|
2
2
|
function L(t, s) {
|
|
3
3
|
if (Math.max(...t) > 5)
|
|
4
4
|
throw new Error("Cannot symbolically compute strides for rank > 6 tensor.");
|
|
@@ -27,7 +27,7 @@ var w;
|
|
|
27
27
|
})(w || (w = {}));
|
|
28
28
|
const H = (t, s, e, o, i) => {
|
|
29
29
|
const u = { dtype: o.dtype, shape: o.shape }, n = D(e, u, s), r = t.createShaderModule({ code: n, label: s.constructor.name });
|
|
30
|
-
let d =
|
|
30
|
+
let d = E().get("WEBGPU_PRINT_SHADER");
|
|
31
31
|
if (d !== "") {
|
|
32
32
|
d = d.toLowerCase();
|
|
33
33
|
const p = d.split(",");
|
|
@@ -281,7 +281,7 @@ function y(t, s = "") {
|
|
|
281
281
|
const e = t.length, o = s !== "" ? `get${s.charAt(0).toUpperCase() + s.slice(1)}CoordsFromIndex` : "getCoordsFromIndex", i = s !== "" ? `${s.charAt(0).toLowerCase() + s.slice(1)}ShapeStrides` : "outShapeStrides";
|
|
282
282
|
if (e <= 1)
|
|
283
283
|
return `fn ${o}(index : i32) -> i32 { return index; }`;
|
|
284
|
-
const u =
|
|
284
|
+
const u = j(t), n = g(e), r = [];
|
|
285
285
|
for (let p = 0; p < e; p++)
|
|
286
286
|
r.push(`d${p}`);
|
|
287
287
|
if (u.length === 1)
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { x as u } from "./index-Duu1Lvvv.js";
|
|
2
2
|
const c = (r) => {
|
|
3
3
|
let t = 1;
|
|
4
4
|
for (let n = 0; n < r.length; n++)
|
|
@@ -23,16 +23,16 @@ function p(r, t, n = !1) {
|
|
|
23
23
|
const a = c(r.x.map((i) => t[i])), o = c(r.y.map((i) => t[i]));
|
|
24
24
|
return a <= 4 ? [4, 16, 1] : o <= 4 ? [16, 4, 1] : [16, 16, 1];
|
|
25
25
|
}
|
|
26
|
-
function
|
|
26
|
+
function x(r, t, n = !1) {
|
|
27
27
|
if (n)
|
|
28
28
|
return [4, 4, 1];
|
|
29
29
|
const a = c(r.x.map((i) => t[i])), o = c(r.y.map((i) => t[i]));
|
|
30
30
|
return a <= 4 ? [1, 2, 1] : o <= 4 ? [2, 1, 1] : [2, 2, 1];
|
|
31
31
|
}
|
|
32
|
-
function
|
|
32
|
+
function M(r) {
|
|
33
33
|
return { x: r.map((t, n) => n) };
|
|
34
34
|
}
|
|
35
|
-
function
|
|
35
|
+
function h(r) {
|
|
36
36
|
if (r === "float32" || r === "int32" || r === "bool" || r === "string" || r === "packedF16")
|
|
37
37
|
return 4;
|
|
38
38
|
if (r === "complex64")
|
|
@@ -52,13 +52,13 @@ var s;
|
|
|
52
52
|
r[r.MatMulReduceProgram = 0] = "MatMulReduceProgram", r[r.MatMulSplitKProgram = 1] = "MatMulSplitKProgram", r[r.MatMulSmallOutputSizeProgram = 2] = "MatMulSmallOutputSizeProgram", r[r.MatMulPackedProgram = 3] = "MatMulPackedProgram", r[r.MatMulMax = 4] = "MatMulMax";
|
|
53
53
|
})(s || (s = {}));
|
|
54
54
|
export {
|
|
55
|
-
|
|
55
|
+
h as G,
|
|
56
56
|
s as M,
|
|
57
57
|
d as a,
|
|
58
58
|
b,
|
|
59
59
|
m as c,
|
|
60
60
|
p as d,
|
|
61
|
-
|
|
62
|
-
|
|
61
|
+
x as e,
|
|
62
|
+
M as f,
|
|
63
63
|
g as i
|
|
64
64
|
};
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { c as f } from "./complex-
|
|
1
|
+
import { u as n, Q as m, U as i, E as c } from "./index-Duu1Lvvv.js";
|
|
2
|
+
import { c as f } from "./complex-Cyg-eQeZ.js";
|
|
3
3
|
function e(o, r = "float32") {
|
|
4
4
|
if (n(o), r === "complex64") {
|
|
5
5
|
const s = e(o, "float32"), t = e(o, "float32");
|