@epfml/discojs 3.0.1-p20241014092014.0 → 3.0.1-p20241024094708.0

package/dist/client/client.js

@@ -1,5 +1,4 @@
 import createDebug from "debug";
-import axios from 'axios';
 import { serialization } from '../index.js';
 import { EventEmitter } from '../utils/event_emitter.js';
 import { type } from "./messages.js";
@@ -149,8 +148,9 @@ export class Client extends EventEmitter {
             url.pathname += '/';
         }
         url.pathname += `tasks/${this.task.id}/model.json`;
-        const response = await axios.get(url.href, { responseType: 'arraybuffer' });
-        return await serialization.model.decode(new Uint8Array(response.data));
+        const response = await fetch(url);
+        const encoded = new Uint8Array(await response.arrayBuffer());
+        return await serialization.model.decode(encoded);
     }
     get ownId() {
         if (this._ownId === undefined) {

package/dist/client/decentralized/messages.d.ts

@@ -1,4 +1,4 @@
-import { weights } from '../../serialization/index.js';
+import { serialization } from "../../index.js";
 import { type SignalData } from './peer.js';
 import { type NodeID } from '../types.js';
 import { type } from '../messages.js';
@@ -29,7 +29,7 @@ export interface Payload {
     peer: NodeID;
     aggregationRound: number;
     communicationRound: number;
-    payload: weights.Encoded;
+    payload: serialization.Encoded;
 }
 export type MessageFromServer = NewDecentralizedNodeInfo | SignalForPeer | PeersForRound | WaitingForMoreParticipants | EnoughParticipants;
 export type MessageToServer = ClientConnected | SignalForPeer | PeerIsReady | JoinRound;

package/dist/client/decentralized/messages.js

@@ -1,4 +1,4 @@
-import { weights } from '../../serialization/index.js';
+import { serialization } from "../../index.js";
 import { isNodeID } from '../types.js';
 import { type, hasMessageType } from '../messages.js';
 export function isMessageFromServer(o) {
@@ -41,7 +41,7 @@ export function isPeerMessage(o) {
     switch (o.type) {
         case type.Payload:
             return ('peer' in o && isNodeID(o.peer) &&
-                'payload' in o && weights.isEncoded(o.payload));
+                'payload' in o && serialization.isEncoded(o.payload));
     }
     return false;
 }

package/dist/client/event_connection.js

@@ -1,6 +1,6 @@
 import createDebug from "debug";
 import WebSocket from "isomorphic-ws";
-import msgpack from "msgpack-lite";
+import * as msgpack from "@msgpack/msgpack";
 import * as decentralizedMessages from './decentralized/messages.js';
 import { type } from './messages.js';
 import { timeout } from './utils.js';
@@ -57,7 +57,7 @@ export class PeerConnection extends EventEmitter {
         if (!decentralizedMessages.isPeerMessage(msg)) {
             throw new Error(`can't send this type of message: ${JSON.stringify(msg)}`);
         }
-        this.peer.send(msgpack.encode(msg));
+        this.peer.send(Buffer.from(msgpack.encode(msg)));
     }
     async disconnect() {
         await this.peer.destroy();

package/dist/client/federated/messages.d.ts

@@ -1,4 +1,4 @@
-import { type weights } from '../../serialization/index.js';
+import type { serialization } from "../../index.js";
 import { type NodeID } from '..//types.js';
 import { type } from '../messages.js';
 import type { ClientConnected, WaitingForMoreParticipants, EnoughParticipants } from '../messages.js';
@@ -7,18 +7,18 @@ export interface NewFederatedNodeInfo {
     type: type.NewFederatedNodeInfo;
     id: NodeID;
     waitForMoreParticipants: boolean;
-    payload: weights.Encoded;
+    payload: serialization.Encoded;
     round: number;
     nbOfParticipants: number;
 }
 export interface SendPayload {
     type: type.SendPayload;
-    payload: weights.Encoded;
+    payload: serialization.Encoded;
     round: number;
 }
 export interface ReceiveServerPayload {
     type: type.ReceiveServerPayload;
-    payload: weights.Encoded;
+    payload: serialization.Encoded;
     round: number;
     nbOfParticipants: number;
 }

package/dist/index.d.ts

@@ -1,7 +1,5 @@
 export * as data from './dataset/index.js';
 export * as serialization from './serialization/index.js';
-export { Encoded as EncodedModel } from './serialization/model.js';
-export { Encoded as EncodedWeights } from './serialization/weights.js';
 export * as training from './training/index.js';
 export * as privacy from './privacy.js';
 export * as client from './client/index.js';

package/dist/serialization/coder.d.ts

@@ -0,0 +1,4 @@
+export type Encoded = Uint8Array;
+export declare function isEncoded(raw: unknown): raw is Encoded;
+export declare function encode(serialized: unknown): Encoded;
+export declare function decode(encoded: Encoded): unknown;

package/dist/serialization/coder.js

@@ -0,0 +1,51 @@
+import * as msgpack from "@msgpack/msgpack";
+export function isEncoded(raw) {
+    if (!(raw instanceof Uint8Array))
+        return false;
+    const _ = raw;
+    return true;
+}
+// create a new buffer instead of referencing the backing one
+function copy(arr) {
+    // `Buffer.slice` (subclass of Uint8Array on Node) doesn't copy
+    // thus doesn't respect Liskov substitution principle
+    // https://nodejs.org/api/buffer.html#bufslicestart-end
+    // here we call the correct implementation
+    return Uint8Array.prototype.slice.call(arr);
+}
+// to avoid mapping every ArrayBuffer to Uint8Array,
+// we register our own convertors for the type we know are needed
+// type id are arbitrally taken from msgpack-lite
+// https://www.npmjs.com/package/msgpack-lite#extension-types
+const CODEC = new msgpack.ExtensionCodec();
+// used by TFJS's weights
+CODEC.register({
+    type: 0x17,
+    encode(obj) {
+        if (!(obj instanceof Float32Array))
+            return null;
+        return new Uint8Array(obj.buffer, obj.byteOffset, obj.byteLength);
+    },
+    decode: (raw) => 
+    // to reinterpred uint8 into float32, it needs to be 4-bytes aligned
+    // but the given buffer might not be so we need to copy it.
+    new Float32Array(copy(raw).buffer),
+});
+// used by TFJS's saved model
+CODEC.register({
+    type: 0x1a,
+    encode(obj) {
+        if (!(obj instanceof ArrayBuffer))
+            return null;
+        return new Uint8Array(obj);
+    },
+    decode: (raw) => 
+    // need to copy as backing ArrayBuffer might be larger
+    copy(raw),
+});
+export function encode(serialized) {
+    return msgpack.encode(serialized, { extensionCodec: CODEC });
+}
+export function decode(encoded) {
+    return msgpack.decode(encoded, { extensionCodec: CODEC });
+}

package/dist/serialization/index.d.ts

@@ -1,2 +1,4 @@
 export * as model from './model.js';
 export * as weights from './weights.js';
+export type { Encoded } from "./coder.js";
+export { isEncoded } from "./coder.js";

package/dist/serialization/index.js

@@ -1,2 +1,3 @@
 export * as model from './model.js';
 export * as weights from './weights.js';
+export { isEncoded } from "./coder.js";

package/dist/serialization/model.d.ts

@@ -1,5 +1,4 @@
 import type { Model } from '../index.js';
-export type Encoded = Uint8Array;
-export declare function isEncoded(raw: unknown): raw is Encoded;
+import { Encoded } from "./coder.js";
 export declare function encode(model: Model): Promise<Encoded>;
 export declare function decode(encoded: unknown): Promise<Model>;

package/dist/serialization/model.js

@@ -1,38 +1,29 @@
-import msgpack from 'msgpack-lite';
 import { models, serialization } from '../index.js';
+import * as coder from "./coder.js";
+import { isEncoded } from "./coder.js";
 const Type = {
     TFJS: 0,
     GPT: 1
 };
-export function isEncoded(raw) {
-    return raw instanceof Uint8Array;
-}
 export async function encode(model) {
-    let encoded;
     switch (true) {
         case model instanceof models.TFJS: {
             const serialized = await model.serialize();
-            encoded = msgpack.encode([Type.TFJS, serialized]);
-            break;
+            return coder.encode([Type.TFJS, serialized]);
         }
         case model instanceof models.GPT: {
             const { weights, config } = model.serialize();
             const serializedWeights = await serialization.weights.encode(weights);
-            encoded = msgpack.encode([Type.GPT, serializedWeights, config]);
-            break;
+            return coder.encode([Type.GPT, serializedWeights, config]);
         }
         default:
             throw new Error("unknown model type");
     }
-    // Node's Buffer extends Node's Uint8Array, which might not be the same
-    // as the browser's Uint8Array. we ensure here that it is.
-    return new Uint8Array(encoded);
 }
 export async function decode(encoded) {
-    if (!isEncoded(encoded)) {
+    if (!isEncoded(encoded))
         throw new Error("Invalid encoding, raw encoding isn't an instance of Uint8Array");
-    }
-    const raw = msgpack.decode(encoded);
+    const raw = coder.decode(encoded);
     if (!Array.isArray(raw) || raw.length < 2) {
         throw new Error("invalid encoding, encoding isn't an array or doesn't contain enough values");
     }
@@ -59,15 +50,9 @@ export async function decode(encoded) {
             else {
                 throw new Error('invalid encoding, gpt-tfjs model encoding should be an array of length 2 or 3');
             }
-            if (!Array.isArray(rawModel)) {
-                throw new Error('invalid encoding, gpt-tfjs model weights should be an array');
-            }
-            const arr = rawModel;
-            if (arr.some((r) => typeof r !== 'number')) {
-                throw new Error("invalid encoding, gpt-tfjs weights should be numbers");
-            }
-            const nums = arr;
-            const weights = serialization.weights.decode(nums);
+            if (!isEncoded(rawModel))
+                throw new Error("invalid encoding, gpt-tfjs model weights should be an encoding of its weights");
+            const weights = serialization.weights.decode(rawModel);
             return models.GPT.deserialize({ weights, config });
         }
         default:

package/dist/serialization/weights.d.ts

@@ -1,5 +1,4 @@
-import { WeightsContainer } from '../index.js';
-export type Encoded = number[];
-export declare function isEncoded(raw: unknown): raw is Encoded;
+import { WeightsContainer } from "../index.js";
+import { Encoded } from "./coder.js";
 export declare function encode(weights: WeightsContainer): Promise<Encoded>;
 export declare function decode(encoded: Encoded): WeightsContainer;

package/dist/serialization/weights.js

@@ -1,37 +1,26 @@
-import * as msgpack from 'msgpack-lite';
-import * as tf from '@tensorflow/tfjs';
-import { WeightsContainer } from '../index.js';
+import * as tf from "@tensorflow/tfjs";
+import { WeightsContainer } from "../index.js";
+import * as coder from "./coder.js";
 function isSerialized(raw) {
-    if (typeof raw !== 'object' || raw === null) {
+    if (typeof raw !== "object" || raw === null)
         return false;
-    }
     const { shape, data } = raw;
-    if (!(Array.isArray(shape) && shape.every((e) => typeof e === 'number')) ||
-        !(Array.isArray(data) && data.every((e) => typeof e === 'number'))) {
+    if (!(Array.isArray(shape) && shape.every((e) => typeof e === "number")) ||
+        !(data instanceof Float32Array))
         return false;
-    }
-    const _ = {
-        shape: shape,
-        data: data,
-    };
+    const _ = { shape, data };
     return true;
 }
-export function isEncoded(raw) {
-    return Array.isArray(raw) && raw.every((e) => typeof e === 'number');
-}
 export async function encode(weights) {
-    const serialized = await Promise.all(weights.weights.map(async (t) => {
-        return {
-            shape: t.shape,
-            data: [...await t.data()]
-        };
-    }));
-    return [...msgpack.encode(serialized).values()];
+    const serialized = await Promise.all(weights.weights.map(async (t) => ({
+        shape: t.shape,
+        data: await t.data(),
+    })));
+    return coder.encode(serialized);
 }
 export function decode(encoded) {
-    const raw = msgpack.decode(encoded);
-    if (!(Array.isArray(raw) && raw.every(isSerialized))) {
-        throw new Error('expected to decode an array of serialized weights');
-    }
+    const raw = coder.decode(encoded);
+    if (!(Array.isArray(raw) && raw.every(isSerialized)))
+        throw new Error("expected to decode an array of serialized weights");
     return new WeightsContainer(raw.map((w) => tf.tensor(w.data, w.shape)));
 }

package/dist/task/task_handler.d.ts

@@ -1,5 +1,5 @@
-import { Map } from 'immutable';
-import type { Model } from '../index.js';
-import type { Task, TaskID } from './task.js';
-export declare function pushTask(url: URL, task: Task, model: Model): Promise<void>;
-export declare function fetchTasks(url: URL): Promise<Map<TaskID, Task>>;
+import { Map } from "immutable";
+import type { Model } from "../index.js";
+import type { Task, TaskID } from "./task.js";
+export declare function pushTask(base: URL, task: Task, model: Model): Promise<void>;
+export declare function fetchTasks(base: URL): Promise<Map<TaskID, Task>>;

package/dist/task/task_handler.js

@@ -1,22 +1,28 @@
-import axios from 'axios';
 import createDebug from "debug";
-import { Map } from 'immutable';
-import { serialization } from '../index.js';
-import { isTask } from './task.js';
+import { Map } from "immutable";
+import { serialization } from "../index.js";
+import { isTask } from "./task.js";
 const debug = createDebug("discojs:task:handlers");
-const TASK_ENDPOINT = 'tasks';
-export async function pushTask(url, task, model) {
-    await axios.post(url.href + TASK_ENDPOINT, {
-        task,
-        model: await serialization.model.encode(model),
-        weights: await serialization.weights.encode(model.weights)
+function urlToTasks(base) {
+    const ret = new URL(base);
+    ret.pathname += "tasks";
+    return ret;
+}
+export async function pushTask(base, task, model) {
+    await fetch(urlToTasks(base), {
+        method: "POST",
+        body: JSON.stringify({
+            task,
+            model: await serialization.model.encode(model),
+            weights: await serialization.weights.encode(model.weights),
+        }),
     });
 }
-export async function fetchTasks(url) {
-    const response = await axios.get(new URL(TASK_ENDPOINT, url).href);
-    const tasks = response.data;
+export async function fetchTasks(base) {
+    const response = await fetch(urlToTasks(base));
+    const tasks = await response.json();
     if (!Array.isArray(tasks)) {
-        throw new Error('Expected to receive an array of Tasks when fetching tasks');
+        throw new Error("Expected to receive an array of Tasks when fetching tasks");
     }
     else if (!tasks.every(isTask)) {
         for (const task of tasks) {
@@ -24,7 +30,7 @@ export async function fetchTasks(url) {
                 debug("task has invalid format: :O", task);
             }
         }
-        throw new Error('invalid tasks response, the task object received is not well formatted');
+        throw new Error("invalid tasks response, the task object received is not well formatted");
     }
     return Map(tasks.map((t) => [t.id, t]));
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@epfml/discojs",
-  "version": "3.0.1-p20241014092014.0",
+  "version": "3.0.1-p20241024094708.0",
   "type": "module",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -19,13 +19,12 @@
   },
   "homepage": "https://github.com/epfml/disco#readme",
   "dependencies": {
+    "@msgpack/msgpack": "^3.0.0-beta2",
     "@tensorflow/tfjs": "4",
     "@xenova/transformers": "2",
-    "axios": "1",
     "immutable": "4",
     "isomorphic-wrtc": "1",
     "isomorphic-ws": "5",
-    "msgpack-lite": "0.1",
     "simple-peer": "9",
     "tslib": "2",
     "ws": "8"