@mineproness/tensorloop 2.1.64 → 2.1.65

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (7) hide show
  1. package/package.json +1 -1
  2. package/src/Softmax/index.js +1 -1
  3. package/src/shaders/ComputeDInput.wgsl +1 -1
  4. package/src/shaders/LB.wgsl +2 -2
  5. package/src/shaders/RB.wgsl +1 -1
  6. package/src/shaders/RF.wgsl +1 -1
  7. package/src/shaders/SB.wgsl +4 -0
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@mineproness/tensorloop",
3
- "version": "2.1.64",
3
+ "version": "2.1.65",
4
4
  "description": "Raw-JAVASCRIPT Based Matirx and AI LLM Package called Tensorloop.",
5
5
  "keywords": [
6
6
  "ai",
package/src/Softmax/index.js CHANGED
@@ -123,7 +123,7 @@ export class Softmax {
123
123
  const targetBuffer = await this.gpu.createBuffer(target.length * Float32Array.BYTES_PER_ELEMENT)
124
124
  const loss = await this.gpu.createBuffer(4 * tokenCount)
125
125
  const acc = await this.gpu.createBuffer(4 * 2)
126
- await this.gpu.WriteBuffer(targetBuffer, new Float32Array(target))
126
+ await this.gpu.WriteBuffer(targetBuffer, new Uint32Array(target))
127
127
  const bind = await this.device.createBindGroup({
128
128
  layout: await this.lossPipline.getBindGroupLayout(0),
129
129
  entries: [
package/src/shaders/ComputeDInput.wgsl CHANGED
@@ -28,5 +28,5 @@ fn main(@builtin(global_invocation_id) id: vec3<u32>) {
28
28
  let grad = outGradient[token * u32(params.vocabSize) + i];
29
29
  gradInput += grad * w;
30
30
  }
31
- dInput[index] = max(-7.0, min(7.0,gradInput));
31
+ dInput[index] = gradInput;
32
32
  }
package/src/shaders/LB.wgsl CHANGED
@@ -44,14 +44,14 @@ fn main(@builtin(global_invocation_id) id: vec3<u32>) {
44
44
  let dW = inputValue * gradient;
45
45
  dWeights += f32(dW);
46
46
  }
47
- let vClap = weight[index] - params.lr * max(- 7.0, min(7.0, dWeights));
47
+ let vClap = weight[index] - params.lr * max(- 15.0, min(15.0, dWeights));
48
48
  weight[index] = vClap;
49
49
  if (inputIndex == 0u) {
50
50
  var dBias = 0.0;
51
51
  for (var token = 0u; token < u32(params.tokenCount); token++) {
52
52
  dBias += f32(outGradient[token * u32(params.vocabSize) + outputIndex]);
53
53
  }
54
- bias[outputIndex] = bias[outputIndex] - params.lr * max(- 7.0, min(7.0, dBias));
54
+ bias[outputIndex] = bias[outputIndex] - params.lr * max(- 15.0, min(15.0, dBias));
55
55
 
56
56
  }
57
57
  }
package/src/shaders/RB.wgsl CHANGED
@@ -32,6 +32,6 @@ fn main(@builtin(global_invocation_id) id: vec3<u32>) {
32
32
  output[index] = grad;
33
33
  }
34
34
  else {
35
- output[index] = grad * 0.02;
35
+ output[index] = 0;
36
36
  }
37
37
  }
package/src/shaders/RF.wgsl CHANGED
@@ -28,6 +28,6 @@ fn main(@builtin(global_invocation_id) id: vec3<u32>) {
28
28
  output[index] = x;
29
29
  }
30
30
  else {
31
- output[index] = x * 0.02;
31
+ output[index] = 0;
32
32
  }
33
33
  }
package/src/shaders/SB.wgsl CHANGED
@@ -27,6 +27,10 @@ fn main(@builtin(global_invocation_id) id: vec3<u32>) {
27
27
  let vocabIndex = index % params.vocabSize;
28
28
  let start = token * params.vocabSize;
29
29
  let end = start + params.vocabSize;
30
+ let total = params.tokenCount * params.vocabSize;
31
+ if (index >= total) {
32
+ return;
33
+ }
30
34
  var targetIndex = targetToken[token];
31
35
  let total: u32 = params.tokenCount * params.vocabSize;
32
36
  output[index] = probs[index];