@mineproness/tensorloop 1.5.5 → 1.5.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/package.json +1 -1
  2. package/src/GPU.js +34 -61
  3. package/src/Linear/index.js +2 -4
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@mineproness/tensorloop",
3
- "version": "1.5.5",
3
+ "version": "1.5.6",
4
4
  "description": "Raw-JAVASCRIPT Based Matirx and AI LLM Package called Tensorloop.",
5
5
  "keywords": [
6
6
  "ai",
package/src/GPU.js CHANGED
@@ -23,8 +23,14 @@ export class GPULinear {
23
23
  cacheLBIas = [];
24
24
  cacheLBWeights = [];
25
25
  cacheLBInputGradent = [];
26
- InitKernel(weights, size) {
26
+ // texures
27
+ Bias = [];
28
+ Weights = []
29
+ InitKernel(weights, bias, size) {
27
30
  const splitedWeights = splitMatrix(weights)
31
+ const splitedBias = splitMatrix(bias)
32
+ this.Weights = splitedWeights.data
33
+ this.Bias = splitedBias.data
28
34
  for (let a = 0; a < splitedWeights.data.length; a++) {
29
35
  this.cacheLF[a] = gpu.createKernel(function (input, bias, weights) {
30
36
  let sum = bias[this.thread.x]
@@ -50,8 +56,11 @@ export class GPULinear {
50
56
  )
51
57
 
52
58
  return bias[this.thread.x] - learningRate * grad
59
+ }, {
60
+ pipeline: true,
61
+ immutable: true,
62
+ output: [splitedWeights.data[a].length]
53
63
  })
54
- .setOutput([splitedWeights.data[a].length])
55
64
  this.cacheLBInputGradent[a] = gpu.createKernel(function (
56
65
  weights,
57
66
  outputGradient
@@ -68,11 +77,11 @@ export class GPULinear {
68
77
  }
69
78
 
70
79
  return sum
80
+ }, {
81
+ constants: { outputSize: splitedWeights.data[a].length },
82
+ output: [size]
71
83
  })
72
- .setConstants({
73
- outputSize: splitedWeights.data[a].length
74
- })
75
- .setOutput([size])
84
+
76
85
  this.cacheLBWeights[a] = gpu.createKernel(function (
77
86
  weights,
78
87
  outputGradient,
@@ -86,74 +95,38 @@ export class GPULinear {
86
95
 
87
96
  return weights[this.thread.y][this.thread.x]
88
97
  - learningRate * input[this.thread.x] * grad
98
+ }, {
99
+ pipeline: true,
100
+ immutable: true,
101
+ output: [size, splitedWeights.data[a].length]
89
102
  })
90
- .setOutput([size, splitedWeights.data[a].length])
91
103
  }
92
104
 
93
105
 
94
106
 
95
107
 
96
108
  }
97
- CalculateLinearForward(weights, bias, input) {
98
- // console.log(input)
99
- const splitedBias = splitMatrix(bias)
100
- const splitedWeights = splitMatrix(weights)
101
- const result = new Float32Array(weights.length)
109
+ CalculateLinearForward(input) {
110
+ const result = []
102
111
  let offset = 0;
103
- for (let a = 0; a < splitedBias.data.length; a++) {
104
- const output = this.cacheLF[a](input, splitedBias.data[a], splitedWeights.data[a])
105
- result.set(output, offset)
106
- offset += output.length
112
+ for (let a = 0; a < this.Weights.length; a++) {
113
+ const output = this.cacheLF[a](input, this.Bias[a], this.Weights[a])
114
+ result.push(...output)
115
+
107
116
  }
108
117
  return result
109
118
  }
110
- CalculateBackward(weights, bias, out, learningRate, input) {
111
- const splitedBias = splitMatrix(bias)
112
- const splitedWeights = splitMatrix(weights)
113
- const inputGradient = new Float32Array(input.length)
114
- const biasOut = new Float32Array(bias.length)
115
- const weightsOut = []
116
- let offsetBias = 0
117
- let offsetWeights = 0
118
- let gradientOffset = 0
119
-
120
- for (let a = 0; a < splitedWeights.data.length; a++) {
121
- const weightChunk = splitedWeights.data[a]
122
- const biasChunk = splitedBias.data[a]
123
- const chunkOutputSize = weightChunk.length
124
- const gradientChunk = out.slice(
125
- gradientOffset,
126
- gradientOffset + chunkOutputSize
127
- )
128
- gradientOffset += chunkOutputSize
129
- const newBias = this.cacheLBIas[a](
130
- biasChunk,
131
- gradientChunk,
132
- learningRate
133
- )
134
- biasOut.set(newBias, offsetBias)
135
- offsetBias += newBias.length
136
- const outputGra = this.cacheLBInputGradent[a](
137
- weightChunk,
138
- gradientChunk
139
- )
140
- for (let i = 0; i < outputGra.length; i++) {
141
- inputGradient[i] += outputGra[i]
119
+ CalculateBackward(out, learningRate, input) {
120
+ let inputGradient = new Float32Array(input.length).fill(0)
121
+ for (let a = 0; a < this.Weights.length; a++) {
122
+ const outGradient = this.cacheLBInputGradent[a](this.Weights[a], out)
123
+ for (let i = 0; i < outGradient.length; i++) {
124
+ inputGradient[i] += outGradient[i]
142
125
  }
143
- const newWeights = this.cacheLBWeights[a](
144
- weightChunk,
145
- gradientChunk,
146
- input,
147
- learningRate
148
- )
149
- weightsOut.push(...newWeights)
150
-
151
- }
126
+ this.Weights[a] = this.cacheLBWeights[a](this.Weights[a], out, input, learningRate)
127
+ this.Bias[a] = this.cacheLBIas[a](this.Bias[a], out, learningRate)
152
128
 
153
- return {
154
- inputGradient,
155
- biasOut,
156
- weightsOut
157
129
  }
130
+ return inputGradient
158
131
  }
159
132
  }
package/src/Linear/index.js CHANGED
@@ -41,7 +41,7 @@ export class Linear {
41
41
  this.Bias = Bias;
42
42
  }
43
43
  this.gpu = new GPULinear()
44
- this.gpu.InitKernel(this.Weights, embeddingSize)
44
+ this.gpu.InitKernel(this.Weights, this.Bias, embeddingSize)
45
45
  }
46
46
  Save() {
47
47
  SaveVectors(this.Weights, this.configs.save.filename[0])
@@ -54,9 +54,7 @@ export class Linear {
54
54
  backward(outputGradient, learningRate, idx) {
55
55
  const input = this.input[idx];
56
56
  const weights = this.Weights
57
- const {biasOut,inputGradient, weightsOut} = this.gpu.CalculateBackward(weights, this.Bias, outputGradient, learningRate, input)
58
- this.Weights = weightsOut;
59
- this.Bias = biasOut
57
+ const inputGradient = this.gpu.CalculateBackward(outputGradient, learningRate, input)
60
58
  return inputGradient
61
59
  }
62
60
  ClearInputCache(){