catniff 0.6.6 → 0.6.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/nn.js +5 -5
- package/package.json +1 -1
package/dist/nn.js
CHANGED
|
@@ -191,7 +191,7 @@ class LayerNorm {
|
|
|
191
191
|
class Embedding {
|
|
192
192
|
weight;
|
|
193
193
|
constructor(numEmbeddings, embeddingDim, device) {
|
|
194
|
-
this.weight = core_1.Tensor.randn([numEmbeddings, embeddingDim], { device });
|
|
194
|
+
this.weight = core_1.Tensor.randn([numEmbeddings, embeddingDim], { requiresGrad: true, device });
|
|
195
195
|
}
|
|
196
196
|
forward(input) {
|
|
197
197
|
return this.weight.index(input);
|
|
@@ -207,10 +207,10 @@ class MultiheadAttention {
|
|
|
207
207
|
headDim;
|
|
208
208
|
dropout;
|
|
209
209
|
constructor(embedDim, numHeads, dropout = 0, bias = true, device) {
|
|
210
|
-
this.qProjection = new
|
|
211
|
-
this.kProjection = new
|
|
212
|
-
this.vProjection = new
|
|
213
|
-
this.oProjection = new
|
|
210
|
+
this.qProjection = new Linear(embedDim, embedDim, bias, device);
|
|
211
|
+
this.kProjection = new Linear(embedDim, embedDim, bias, device);
|
|
212
|
+
this.vProjection = new Linear(embedDim, embedDim, bias, device);
|
|
213
|
+
this.oProjection = new Linear(embedDim, embedDim, bias, device);
|
|
214
214
|
this.embedDim = embedDim;
|
|
215
215
|
this.numHeads = numHeads;
|
|
216
216
|
this.headDim = Math.floor(embedDim / numHeads);
|