flockml 1.0.0

package/src/coordinator.ts

@@ -0,0 +1,137 @@
+import { NeuralNetwork } from './network';
+import { Quantizer, QuantizedPayload } from './quantization';
+import { flockml } from './messages';
+import * as tf from '@tensorflow/tfjs';
+
+/**
+ * The Central Brain: FedAvg Coordinator.
+ * 
+ * Powered by TFJS. Averages massive incoming weight tensors continuously
+ * using hardware acceleration.
+ */
+export class Coordinator {
+  globalModel: NeuralNetwork;
+  clientUpdates: {
+    weights_ih: tf.Tensor2D;
+    weights_ho: tf.Tensor2D;
+    bias_h: tf.Tensor2D;
+    bias_o: tf.Tensor2D;
+  }[];
+
+  constructor(inputNodes: number, hiddenNodes: number, outputNodes: number) {
+    this.globalModel = new NeuralNetwork(inputNodes, hiddenNodes, outputNodes);
+    this.clientUpdates = [];
+  }
+
+  private mapFromProto(protoPayload: flockml.IQuantizedMatrix | null | undefined): QuantizedPayload {
+    if (!protoPayload) throw new Error("Missing payload section");
+    return {
+      data: protoPayload.data || new Uint8Array(),
+      min: protoPayload.min || 0,
+      max: protoPayload.max || 0,
+      rows: protoPayload.rows || 0,
+      cols: protoPayload.cols || 0
+    };
+  }
+
+  /**
+   * Receives a binary protobuf payload from a browser client.
+   */
+  receiveUpdate(binaryPayload: Uint8Array): void {
+    const message = flockml.GradientUpdate.decode(binaryPayload);
+
+    const weights_ih = Quantizer.dequantize(this.mapFromProto(message.weightsIh));
+    const weights_ho = Quantizer.dequantize(this.mapFromProto(message.weightsHo));
+    const bias_h = Quantizer.dequantize(this.mapFromProto(message.biasH));
+    const bias_o = Quantizer.dequantize(this.mapFromProto(message.biasO));
+
+    this.clientUpdates.push({
+      weights_ih,
+      weights_ho,
+      bias_h,
+      bias_o
+    });
+  }
+
+  /**
+   * The Federated Averaging (FedAvg) Algorithm with Anomaly Detection.
+   * 
+   * Averages all the incoming matrices. Because of Differential Privacy, 
+   * the Laplacian noise mathematically cancels out to 0 here, leaving only
+   * the pure, learned signal from the crowdsourced devices.
+   */
+  aggregate(): void {
+    if (this.clientUpdates.length === 0) return;
+
+    // 1. Anomaly Detection (Anti-Sybil / Data Poisoning)
+    // Runs mathematically on the GPU without downloading large buffers to CPU
+    const safeUpdates = this.clientUpdates.filter(update => {
+      return tf.tidy(() => {
+        const checkTensor = (t: tf.Tensor2D) => {
+          const hasNaN = tf.sum(tf.cast(tf.isNaN(t), 'int32')).dataSync()[0] > 0;
+          const maxAbs = tf.max(tf.abs(t)).dataSync()[0];
+          return !hasNaN && maxAbs <= 1000;
+        };
+        
+        return checkTensor(update.weights_ih) && 
+               checkTensor(update.weights_ho) && 
+               checkTensor(update.bias_h) && 
+               checkTensor(update.bias_o);
+      });
+    });
+
+    if (safeUpdates.length === 0) {
+      console.warn("[Coordinator] All received updates were flagged as anomalous and dropped.");
+      // Dispose the tensors since we are dropping them
+      for (const update of this.clientUpdates) {
+        update.weights_ih.dispose();
+        update.weights_ho.dispose();
+        update.bias_h.dispose();
+        update.bias_o.dispose();
+      }
+      this.clientUpdates = [];
+      return;
+    }
+
+    if (safeUpdates.length < this.clientUpdates.length) {
+      console.warn(`[Coordinator] Dropped ${this.clientUpdates.length - safeUpdates.length} poisoned payloads.`);
+    }
+
+    // 2. Federated Averaging
+    const averageTensor = (matrixKey: 'weights_ih' | 'weights_ho' | 'bias_h' | 'bias_o', targetVar: tf.Variable) => {
+      tf.tidy(() => {
+        const tensors = safeUpdates.map(u => u[matrixKey]);
+        const stacked = tf.stack(tensors);
+        const mean = tf.mean(stacked, 0);
+        targetVar.assign(mean);
+      });
+    };
+
+    averageTensor('weights_ih', this.globalModel.weights_ih);
+    averageTensor('weights_ho', this.globalModel.weights_ho);
+    averageTensor('bias_h', this.globalModel.bias_h);
+    averageTensor('bias_o', this.globalModel.bias_o);
+
+    // Dispose all client tensors after aggregation to prevent memory leaks
+    for (const update of this.clientUpdates) {
+      update.weights_ih.dispose();
+      update.weights_ho.dispose();
+      update.bias_h.dispose();
+      update.bias_o.dispose();
+    }
+    
+    this.clientUpdates = [];
+  }
+
+  /**
+   * Broadcasts the current global model weights to all new clients.
+   */
+  getGlobalWeightsForBroadcast() {
+    return {
+      weights_ih: Quantizer.quantize(this.globalModel.weights_ih as tf.Tensor2D),
+      weights_ho: Quantizer.quantize(this.globalModel.weights_ho as tf.Tensor2D),
+      bias_h: Quantizer.quantize(this.globalModel.bias_h as tf.Tensor2D),
+      bias_o: Quantizer.quantize(this.globalModel.bias_o as tf.Tensor2D)
+    };
+  }
+}

package/src/index.ts

@@ -0,0 +1,5 @@
+export { NeuralNetwork } from './network';
+export { Quantizer, type QuantizedPayload } from './quantization';
+export { DifferentialPrivacy } from './privacy';
+export { Coordinator } from './coordinator';
+export { FlockNode as FlockNode } from './client-node';

package/src/messages.d.ts

@@ -0,0 +1,429 @@
+import * as $protobuf from "protobufjs";
+import Long = require("long");
+
+/** Namespace flockml. */
+export namespace flockml {
+
+    /**
+     * Properties of a QuantizedMatrix.
+     * @deprecated Use flockml.QuantizedMatrix.$Properties instead.
+     */
+    interface IQuantizedMatrix extends flockml.QuantizedMatrix.$Properties {
+    }
+
+    /** Represents a QuantizedMatrix. */
+    class QuantizedMatrix {
+
+        /**
+         * Constructs a new QuantizedMatrix.
+         * @param [properties] Properties to set
+         */
+        constructor(properties?: flockml.QuantizedMatrix.$Properties);
+
+        /** Unknown fields preserved while decoding when enabled */
+        $unknowns?: Uint8Array[];
+
+        /** QuantizedMatrix data. */
+        data: Uint8Array;
+
+        /** QuantizedMatrix min. */
+        min: number;
+
+        /** QuantizedMatrix max. */
+        max: number;
+
+        /** QuantizedMatrix rows. */
+        rows: number;
+
+        /** QuantizedMatrix cols. */
+        cols: number;
+
+        /**
+         * Creates a new QuantizedMatrix instance using the specified properties.
+         * @param [properties] Properties to set
+         * @returns QuantizedMatrix instance
+         */
+        static create(properties: flockml.QuantizedMatrix.$Shape): flockml.QuantizedMatrix & flockml.QuantizedMatrix.$Shape;
+        static create(properties?: flockml.QuantizedMatrix.$Properties): flockml.QuantizedMatrix;
+
+        /**
+         * Encodes the specified QuantizedMatrix message. Does not implicitly {@link flockml.QuantizedMatrix.verify|verify} messages.
+         * @param message QuantizedMatrix message or plain object to encode
+         * @param [writer] Writer to encode to
+         * @returns Writer
+         */
+        static encode(message: flockml.QuantizedMatrix.$Properties, writer?: $protobuf.Writer): $protobuf.Writer;
+
+        /**
+         * Encodes the specified QuantizedMatrix message, length delimited. Does not implicitly {@link flockml.QuantizedMatrix.verify|verify} messages.
+         * @param message QuantizedMatrix message or plain object to encode
+         * @param [writer] Writer to encode to
+         * @returns Writer
+         */
+        static encodeDelimited(message: flockml.QuantizedMatrix.$Properties, writer?: $protobuf.Writer): $protobuf.Writer;
+
+        /**
+         * Decodes a QuantizedMatrix message from the specified reader or buffer.
+         * @param reader Reader or buffer to decode from
+         * @param [length] Message length if known beforehand
+         * @returns {flockml.QuantizedMatrix & flockml.QuantizedMatrix.$Shape} QuantizedMatrix
+         * @throws {Error} If the payload is not a reader or valid buffer
+         * @throws {$protobuf.util.ProtocolError} If required fields are missing
+         */
+        static decode(reader: ($protobuf.Reader|Uint8Array), length?: number): flockml.QuantizedMatrix & flockml.QuantizedMatrix.$Shape;
+
+        /**
+         * Decodes a QuantizedMatrix message from the specified reader or buffer, length delimited.
+         * @param reader Reader or buffer to decode from
+         * @returns {flockml.QuantizedMatrix & flockml.QuantizedMatrix.$Shape} QuantizedMatrix
+         * @throws {Error} If the payload is not a reader or valid buffer
+         * @throws {$protobuf.util.ProtocolError} If required fields are missing
+         */
+        static decodeDelimited(reader: ($protobuf.Reader|Uint8Array)): flockml.QuantizedMatrix & flockml.QuantizedMatrix.$Shape;
+
+        /**
+         * Verifies a QuantizedMatrix message.
+         * @param message Plain object to verify
+         * @returns `null` if valid, otherwise the reason why it is not
+         */
+        static verify(message: { [k: string]: any }): (string|null);
+
+        /**
+         * Creates a QuantizedMatrix message from a plain object. Also converts values to their respective internal types.
+         * @param object Plain object
+         * @returns QuantizedMatrix
+         */
+        static fromObject(object: { [k: string]: any }): flockml.QuantizedMatrix;
+
+        /**
+         * Creates a plain object from a QuantizedMatrix message. Also converts values to other types if specified.
+         * @param message QuantizedMatrix
+         * @param [options] Conversion options
+         * @returns Plain object
+         */
+        static toObject(message: flockml.QuantizedMatrix, options?: $protobuf.IConversionOptions): { [k: string]: any };
+
+        /**
+         * Converts this QuantizedMatrix to JSON.
+         * @returns JSON object
+         */
+        toJSON(): { [k: string]: any };
+
+        /**
+         * Gets the type url for QuantizedMatrix
+         * @param [prefix] Custom type url prefix, defaults to `"type.googleapis.com"`
+         * @returns The type url
+         */
+        static getTypeUrl(prefix?: string): string;
+    }
+
+    namespace QuantizedMatrix {
+
+        /** Properties of a QuantizedMatrix. */
+        interface $Properties {
+
+            /** QuantizedMatrix data */
+            data?: (Uint8Array|null);
+
+            /** QuantizedMatrix min */
+            min?: (number|null);
+
+            /** QuantizedMatrix max */
+            max?: (number|null);
+
+            /** QuantizedMatrix rows */
+            rows?: (number|null);
+
+            /** QuantizedMatrix cols */
+            cols?: (number|null);
+
+            /** Unknown fields preserved while decoding when enabled */
+            $unknowns?: Uint8Array[];
+        }
+
+        /** Shape of a QuantizedMatrix. */
+        type $Shape = flockml.QuantizedMatrix.$Properties;
+    }
+
+    /**
+     * Properties of a GradientUpdate.
+     * @deprecated Use flockml.GradientUpdate.$Properties instead.
+     */
+    interface IGradientUpdate extends flockml.GradientUpdate.$Properties {
+    }
+
+    /** Represents a GradientUpdate. */
+    class GradientUpdate {
+
+        /**
+         * Constructs a new GradientUpdate.
+         * @param [properties] Properties to set
+         */
+        constructor(properties?: flockml.GradientUpdate.$Properties);
+
+        /** Unknown fields preserved while decoding when enabled */
+        $unknowns?: Uint8Array[];
+
+        /** GradientUpdate weightsIh. */
+        weightsIh?: (flockml.QuantizedMatrix.$Properties|null);
+
+        /** GradientUpdate weightsHo. */
+        weightsHo?: (flockml.QuantizedMatrix.$Properties|null);
+
+        /** GradientUpdate biasH. */
+        biasH?: (flockml.QuantizedMatrix.$Properties|null);
+
+        /** GradientUpdate biasO. */
+        biasO?: (flockml.QuantizedMatrix.$Properties|null);
+
+        /** GradientUpdate batchSize. */
+        batchSize: number;
+
+        /**
+         * Creates a new GradientUpdate instance using the specified properties.
+         * @param [properties] Properties to set
+         * @returns GradientUpdate instance
+         */
+        static create(properties: flockml.GradientUpdate.$Shape): flockml.GradientUpdate & flockml.GradientUpdate.$Shape;
+        static create(properties?: flockml.GradientUpdate.$Properties): flockml.GradientUpdate;
+
+        /**
+         * Encodes the specified GradientUpdate message. Does not implicitly {@link flockml.GradientUpdate.verify|verify} messages.
+         * @param message GradientUpdate message or plain object to encode
+         * @param [writer] Writer to encode to
+         * @returns Writer
+         */
+        static encode(message: flockml.GradientUpdate.$Properties, writer?: $protobuf.Writer): $protobuf.Writer;
+
+        /**
+         * Encodes the specified GradientUpdate message, length delimited. Does not implicitly {@link flockml.GradientUpdate.verify|verify} messages.
+         * @param message GradientUpdate message or plain object to encode
+         * @param [writer] Writer to encode to
+         * @returns Writer
+         */
+        static encodeDelimited(message: flockml.GradientUpdate.$Properties, writer?: $protobuf.Writer): $protobuf.Writer;
+
+        /**
+         * Decodes a GradientUpdate message from the specified reader or buffer.
+         * @param reader Reader or buffer to decode from
+         * @param [length] Message length if known beforehand
+         * @returns {flockml.GradientUpdate & flockml.GradientUpdate.$Shape} GradientUpdate
+         * @throws {Error} If the payload is not a reader or valid buffer
+         * @throws {$protobuf.util.ProtocolError} If required fields are missing
+         */
+        static decode(reader: ($protobuf.Reader|Uint8Array), length?: number): flockml.GradientUpdate & flockml.GradientUpdate.$Shape;
+
+        /**
+         * Decodes a GradientUpdate message from the specified reader or buffer, length delimited.
+         * @param reader Reader or buffer to decode from
+         * @returns {flockml.GradientUpdate & flockml.GradientUpdate.$Shape} GradientUpdate
+         * @throws {Error} If the payload is not a reader or valid buffer
+         * @throws {$protobuf.util.ProtocolError} If required fields are missing
+         */
+        static decodeDelimited(reader: ($protobuf.Reader|Uint8Array)): flockml.GradientUpdate & flockml.GradientUpdate.$Shape;
+
+        /**
+         * Verifies a GradientUpdate message.
+         * @param message Plain object to verify
+         * @returns `null` if valid, otherwise the reason why it is not
+         */
+        static verify(message: { [k: string]: any }): (string|null);
+
+        /**
+         * Creates a GradientUpdate message from a plain object. Also converts values to their respective internal types.
+         * @param object Plain object
+         * @returns GradientUpdate
+         */
+        static fromObject(object: { [k: string]: any }): flockml.GradientUpdate;
+
+        /**
+         * Creates a plain object from a GradientUpdate message. Also converts values to other types if specified.
+         * @param message GradientUpdate
+         * @param [options] Conversion options
+         * @returns Plain object
+         */
+        static toObject(message: flockml.GradientUpdate, options?: $protobuf.IConversionOptions): { [k: string]: any };
+
+        /**
+         * Converts this GradientUpdate to JSON.
+         * @returns JSON object
+         */
+        toJSON(): { [k: string]: any };
+
+        /**
+         * Gets the type url for GradientUpdate
+         * @param [prefix] Custom type url prefix, defaults to `"type.googleapis.com"`
+         * @returns The type url
+         */
+        static getTypeUrl(prefix?: string): string;
+    }
+
+    namespace GradientUpdate {
+
+        /** Properties of a GradientUpdate. */
+        interface $Properties {
+
+            /** GradientUpdate weightsIh */
+            weightsIh?: (flockml.QuantizedMatrix.$Properties|null);
+
+            /** GradientUpdate weightsHo */
+            weightsHo?: (flockml.QuantizedMatrix.$Properties|null);
+
+            /** GradientUpdate biasH */
+            biasH?: (flockml.QuantizedMatrix.$Properties|null);
+
+            /** GradientUpdate biasO */
+            biasO?: (flockml.QuantizedMatrix.$Properties|null);
+
+            /** GradientUpdate batchSize */
+            batchSize?: (number|null);
+
+            /** Unknown fields preserved while decoding when enabled */
+            $unknowns?: Uint8Array[];
+        }
+
+        /** Shape of a GradientUpdate. */
+        type $Shape = flockml.GradientUpdate.$Properties;
+    }
+
+    /**
+     * Properties of a GlobalModel.
+     * @deprecated Use flockml.GlobalModel.$Properties instead.
+     */
+    interface IGlobalModel extends flockml.GlobalModel.$Properties {
+    }
+
+    /** Represents a GlobalModel. */
+    class GlobalModel {
+
+        /**
+         * Constructs a new GlobalModel.
+         * @param [properties] Properties to set
+         */
+        constructor(properties?: flockml.GlobalModel.$Properties);
+
+        /** Unknown fields preserved while decoding when enabled */
+        $unknowns?: Uint8Array[];
+
+        /** GlobalModel weightsIh. */
+        weightsIh?: (flockml.QuantizedMatrix.$Properties|null);
+
+        /** GlobalModel weightsHo. */
+        weightsHo?: (flockml.QuantizedMatrix.$Properties|null);
+
+        /** GlobalModel biasH. */
+        biasH?: (flockml.QuantizedMatrix.$Properties|null);
+
+        /** GlobalModel biasO. */
+        biasO?: (flockml.QuantizedMatrix.$Properties|null);
+
+        /** GlobalModel version. */
+        version: number;
+
+        /**
+         * Creates a new GlobalModel instance using the specified properties.
+         * @param [properties] Properties to set
+         * @returns GlobalModel instance
+         */
+        static create(properties: flockml.GlobalModel.$Shape): flockml.GlobalModel & flockml.GlobalModel.$Shape;
+        static create(properties?: flockml.GlobalModel.$Properties): flockml.GlobalModel;
+
+        /**
+         * Encodes the specified GlobalModel message. Does not implicitly {@link flockml.GlobalModel.verify|verify} messages.
+         * @param message GlobalModel message or plain object to encode
+         * @param [writer] Writer to encode to
+         * @returns Writer
+         */
+        static encode(message: flockml.GlobalModel.$Properties, writer?: $protobuf.Writer): $protobuf.Writer;
+
+        /**
+         * Encodes the specified GlobalModel message, length delimited. Does not implicitly {@link flockml.GlobalModel.verify|verify} messages.
+         * @param message GlobalModel message or plain object to encode
+         * @param [writer] Writer to encode to
+         * @returns Writer
+         */
+        static encodeDelimited(message: flockml.GlobalModel.$Properties, writer?: $protobuf.Writer): $protobuf.Writer;
+
+        /**
+         * Decodes a GlobalModel message from the specified reader or buffer.
+         * @param reader Reader or buffer to decode from
+         * @param [length] Message length if known beforehand
+         * @returns {flockml.GlobalModel & flockml.GlobalModel.$Shape} GlobalModel
+         * @throws {Error} If the payload is not a reader or valid buffer
+         * @throws {$protobuf.util.ProtocolError} If required fields are missing
+         */
+        static decode(reader: ($protobuf.Reader|Uint8Array), length?: number): flockml.GlobalModel & flockml.GlobalModel.$Shape;
+
+        /**
+         * Decodes a GlobalModel message from the specified reader or buffer, length delimited.
+         * @param reader Reader or buffer to decode from
+         * @returns {flockml.GlobalModel & flockml.GlobalModel.$Shape} GlobalModel
+         * @throws {Error} If the payload is not a reader or valid buffer
+         * @throws {$protobuf.util.ProtocolError} If required fields are missing
+         */
+        static decodeDelimited(reader: ($protobuf.Reader|Uint8Array)): flockml.GlobalModel & flockml.GlobalModel.$Shape;
+
+        /**
+         * Verifies a GlobalModel message.
+         * @param message Plain object to verify
+         * @returns `null` if valid, otherwise the reason why it is not
+         */
+        static verify(message: { [k: string]: any }): (string|null);
+
+        /**
+         * Creates a GlobalModel message from a plain object. Also converts values to their respective internal types.
+         * @param object Plain object
+         * @returns GlobalModel
+         */
+        static fromObject(object: { [k: string]: any }): flockml.GlobalModel;
+
+        /**
+         * Creates a plain object from a GlobalModel message. Also converts values to other types if specified.
+         * @param message GlobalModel
+         * @param [options] Conversion options
+         * @returns Plain object
+         */
+        static toObject(message: flockml.GlobalModel, options?: $protobuf.IConversionOptions): { [k: string]: any };
+
+        /**
+         * Converts this GlobalModel to JSON.
+         * @returns JSON object
+         */
+        toJSON(): { [k: string]: any };
+
+        /**
+         * Gets the type url for GlobalModel
+         * @param [prefix] Custom type url prefix, defaults to `"type.googleapis.com"`
+         * @returns The type url
+         */
+        static getTypeUrl(prefix?: string): string;
+    }
+
+    namespace GlobalModel {
+
+        /** Properties of a GlobalModel. */
+        interface $Properties {
+
+            /** GlobalModel weightsIh */
+            weightsIh?: (flockml.QuantizedMatrix.$Properties|null);
+
+            /** GlobalModel weightsHo */
+            weightsHo?: (flockml.QuantizedMatrix.$Properties|null);
+
+            /** GlobalModel biasH */
+            biasH?: (flockml.QuantizedMatrix.$Properties|null);
+
+            /** GlobalModel biasO */
+            biasO?: (flockml.QuantizedMatrix.$Properties|null);
+
+            /** GlobalModel version */
+            version?: (number|null);
+
+            /** Unknown fields preserved while decoding when enabled */
+            $unknowns?: Uint8Array[];
+        }
+
+        /** Shape of a GlobalModel. */
+        type $Shape = flockml.GlobalModel.$Properties;
+    }
+}