@mineproness/tensorloop 1.3.4 → 1.3.7

package/index.js

@@ -7,6 +7,7 @@ import {tools} from './src/tools/index.js'
 import {ReLU} from './src/ReLU/index.js'
 import {PositionEmbedding as PE} from './src/PositionEmbedding/index.js'
 import {SelfAttention as ST} from './src/SelfAttention/index.js'
+import {GELU as GU} from './src/GELU/index.js'
 export const Tokenizer = tokenizer
 export const Embedding = embeddding2
 export const Linear = li2
@@ -15,4 +16,5 @@ export const CrossEntropy = crossEntropy
 export const Tools = tools
 export const ReLu = ReLU
 export const PositionEmbedding = PE
-export const SelfAttention = ST
+export const SelfAttention = ST
+export const GELU = GU

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mineproness/tensorloop",
-  "version": "1.3.4",
+  "version": "1.3.7",
   "description": "The most powerfull Machine Learning Library for AI.",
   "keywords": [
     "ai",

package/readme.md

@@ -8,7 +8,7 @@ A Library that Supply AI Tools and Large Language model Development tools like E
 - Linear Weights and Bias. 
 - Softmax forward and Backward.
 - CrossEntroy for Loss Calculation.
-- Relu for Stable Layer.
+- Relu and GELU for Stable Layer.
 - Tokenizer for convert into tokens.
 - Self attention 
 - Position Embedding

package/src/GELU/index.d.ts

@@ -0,0 +1,6 @@
+export class GELU{
+    input : Float32Array[];
+    forward(input: Float32Array, index: number) : Float32Array;
+    backward(Doutput: Float32Array,  index: number) : Float32Array;
+    ClearInputCache() : void;
+}

package/src/GELU/index.js

@@ -0,0 +1,43 @@
+export default class GELU {
+    constructor() {
+        this.inputs = []
+    }
+
+    forward(input, index = 0) {
+        this.inputs[index] = new Float32Array(input)
+        const output = new Float32Array(input.length)
+        const k = Math.sqrt(2 / Math.PI)
+        for (let i = 0; i < input.length; i++) {
+            const x = input[i]
+            const u = k * (x + 0.044715 * x * x * x)
+            output[i] = 0.5 * x * (1 + Math.tanh(u))
+        }
+        return output
+    }
+
+    backward(dOutput, index = 0) {
+        const input = this.inputs[index]
+        if (!input) throw new Error(`No GELU cache for index ${index}`);
+        const grad = new Float32Array(dOutput.length)
+        const k = Math.sqrt(2 / Math.PI)
+        for (let i = 0; i < dOutput.length; i++) {
+            const x = input[i]
+            const x3 = x * x * x
+            const u = k * (x + 0.044715 * x3)
+            const t = Math.tanh(u)
+            const sech2 = 1 - t * t
+            const duDx = k * (1 + 3 * 0.044715 * x * x)
+            const geluGrad =
+                0.5 * (1 + t) +
+                0.5 * x * sech2 * duDx
+
+            grad[i] = dOutput[i] * geluGrad
+        }
+
+        return grad
+    }
+
+    ClearInputCache() {
+        this.inputs = []
+    }
+}

package/src/SelfAttention/index.d.ts

@@ -4,9 +4,8 @@ export class SelfAttention{
     k : Float32Array[]
     input;
     constructor(embeddingSize : number, configs: {
-        save: {
             dirname: string
-        }
+        
     })
     weights : Float32Array[]
     score : number