@stellarapp/tfjs-stellar 1.0.3 → 1.0.5

package/dist/src/layers/transformer_encoder.js

@@ -1,175 +0,0 @@
-import * as tf from "@tensorflow/tfjs";
-import { MultiHeadAttention } from "@/layers/multihead_attention";
-/**
- * This class implements the transformer encoder architecture from the 2017 paper
- * Attention Is All You Need.
- *
- * This layer accepts exactly one tensor input with the shape
- * `[ batch, sequences, embedding dims ]`.
- *
- * @param numHeads number of attention heads to use
- * @param embedDim the embedding size of the input (input embeddings, typically the last dimension)
- * @param causal use causal masking, default `false` for encoders
- * @param dropout use dropout during the attention calculations, default `0.1`
- * @param activation the activation of the intermediate feed forward layer, default `relu`
- * @param dimsFeedForward the size of the intermediate feed forward layer, default `2048`
- * @param useBias use bias for the dense sublayers and multiHead attention's dense sublayers, default `true`
- */
-export class TransformerEncoder extends tf.layers.Layer {
-    static className = "TransformerEncoder";
-    selfAttention;
-    selfAttentionDropout;
-    selfAttentionNorm;
-    reluLayer;
-    linearLayer;
-    feedForwardDropout;
-    feedFowardNorm;
-    numHeads;
-    embedDim;
-    causal;
-    useBias;
-    dropout;
-    activation;
-    dimsFeedForward;
-    constructor({ numHeads, embedDim, causal, useBias, dropout, activation, dimsFeedForward, ...args }) {
-        super(args);
-        this.numHeads = numHeads;
-        this.embedDim = embedDim;
-        this.causal = causal ?? false;
-        this.useBias = useBias ?? true;
-        this.dropout = dropout ?? 0.1;
-        this.activation = activation ?? "relu";
-        this.dimsFeedForward = dimsFeedForward ?? 2048;
-        if (this.dropout >= 1) {
-            throw Error(`${this.getClassName()}::constructor dropout must be within [0, 1)`);
-        }
-        // self attention sub-block
-        this.selfAttention = new MultiHeadAttention({
-            numHeads: this.numHeads, embedDim: this.embedDim, useBias: this.useBias,
-            dropout: this.dropout, causal: this.causal
-        });
-        this.selfAttentionDropout = tf.layers.dropout({ rate: this.dropout });
-        this.selfAttentionNorm = tf.layers.layerNormalization({ epsilon: 1e-6 });
-        // feed forward sub-block
-        this.reluLayer = tf.layers.dense({
-            units: this.dimsFeedForward, activation: this.activation,
-            useBias: this.useBias
-        });
-        this.linearLayer = tf.layers.dense({
-            units: this.embedDim, activation: "linear",
-            useBias: this.useBias
-        });
-        this.feedForwardDropout = tf.layers.dropout({ rate: this.dropout });
-        this.feedFowardNorm = tf.layers.layerNormalization({ epsilon: 1e-6 });
-    }
-    /**
-     * Forward propagation
-     */
-    call(inputs, kwargs) {
-        // validate the input tensors
-        let input;
-        if (Array.isArray(inputs)) {
-            if (inputs.length != 1) {
-                throw Error(`${this.getClassName}::call ${this.name} expects exactly 1 tensor` +
-                    ` input, got ${inputs.length} inputs instead.`);
-            }
-            input = inputs[0];
-        }
-        else {
-            input = inputs;
-        }
-        // perform forward propagation
-        return tf.tidy(() => {
-            const attention = this.selfAttentionBlock(input, kwargs);
-            const feedforward = this.feedForwardBlock(attention, kwargs);
-            return feedforward;
-        });
-    }
-    selfAttentionBlock(x, kwargs) {
-        return tf.tidy(() => {
-            const residual = x;
-            let attention = this.selfAttention.apply(x, kwargs);
-            attention = this.selfAttentionDropout.apply(attention, kwargs);
-            attention = tf.add(attention, residual);
-            attention = this.selfAttentionNorm.apply(attention);
-            return attention;
-        });
-    }
-    feedForwardBlock(x, kwargs) {
-        return tf.tidy(() => {
-            const residual = x;
-            let feedForward = this.reluLayer.apply(x);
-            feedForward = this.linearLayer.apply(feedForward);
-            feedForward = this.feedForwardDropout.apply(feedForward, kwargs);
-            feedForward = tf.add(feedForward, residual);
-            feedForward = this.feedFowardNorm.apply(feedForward);
-            return feedForward;
-        });
-    }
-    /**
-     * Initialize the sublayers' weights and track them to enable backpropagation.
-     */
-    build(inputShape) {
-        let input_shapes = [];
-        if (Array.isArray(inputShape) && Array.isArray(inputShape[0])) {
-            // input is an array of shapes
-            input_shapes = inputShape;
-        }
-        else if (inputShape.length != 0) {
-            // input is a single shape
-            input_shapes = [inputShape];
-        }
-        // expects only 1 rank 3 tensor input
-        if (input_shapes.length != 1 || input_shapes[0].length != 3) {
-            throw Error(`${this.getClassName()}::build ${this.name} expects a single input shape of [batch, seq, embed_dim], got ${JSON.stringify(inputShape)}`);
-        }
-        // initialize self attention sub-block's weights
-        this.selfAttention.build(inputShape);
-        this.selfAttentionNorm.build(inputShape);
-        // inintialize feedforward sub-block's weights
-        const reluLayerOutputShape = this.reluLayer.computeOutputShape(inputShape);
-        const linearLayerOutputShape = this.linearLayer.computeOutputShape(reluLayerOutputShape);
-        this.reluLayer.build(inputShape);
-        this.linearLayer.build(reluLayerOutputShape);
-        this.feedFowardNorm.build(linearLayerOutputShape);
-        // track sublayers' weights
-        this.trainableWeights = [
-            ...this.selfAttention.trainableWeights,
-            ...this.selfAttentionDropout.trainableWeights,
-            ...this.selfAttentionNorm.trainableWeights,
-            ...this.reluLayer.trainableWeights,
-            ...this.linearLayer.trainableWeights,
-            ...this.feedForwardDropout.trainableWeights,
-            ...this.feedFowardNorm.trainableWeights
-        ];
-        // rename the weights otherwise they'll take on the default naming and overlap
-        // each other which breaks model loading due to duplicate weight names
-        let indexing = 0;
-        for (const weight of this.trainableWeights) {
-            const unique_name = `${this.getClassName()}_${indexing}`;
-            weight.name += unique_name;
-            weight.originalName += unique_name;
-            indexing++;
-        }
-        super.build(inputShape);
-    }
-    /**
-     * Save the layer's hyperparameters for serialization
-     */
-    getConfig() {
-        const base_config = super.getConfig();
-        const config = {
-            numHeads: this.numHeads,
-            embedDim: this.embedDim,
-            causal: this.causal,
-            useBias: this.useBias,
-            dropout: this.dropout,
-            activation: this.activation,
-            dimsFeedForward: this.dimsFeedForward
-        };
-        Object.assign(config, base_config);
-        return config;
-    }
-}
-tf.serialization.registerClass(TransformerEncoder);
-//# sourceMappingURL=transformer_encoder.js.map

package/dist/src/layers/transformer_encoder.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"transformer_encoder.js","sourceRoot":"","sources":["../../../src/layers/transformer_encoder.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAIvC,OAAO,EAAE,kBAAkB,EAA+B,MAAM,8BAA8B,CAAC;AAS/F;;;;;;;;;;;;;;GAcG;AACH,MAAM,OAAO,kBAAmB,SAAQ,EAAE,CAAC,MAAM,CAAC,KAAK;IACnD,MAAM,CAAC,SAAS,GAAG,oBAAoB,CAAC;IAEvB,aAAa,CAAkB;IAC/B,oBAAoB,CAAkB;IACtC,iBAAiB,CAAkB;IAEnC,SAAS,CAAkB;IAC3B,WAAW,CAAkB;IAC7B,kBAAkB,CAAkB;IACpC,cAAc,CAAkB;IAEhC,QAAQ,CAAS;IACjB,QAAQ,CAAS;IACjB,MAAM,CAAU;IAChB,OAAO,CAAU;IACjB,OAAO,CAAS;IAChB,UAAU,CAAuB;IACjC,eAAe,CAAS;IAGzC,YAAY,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,eAAe,EAAE,GAAG,IAAI,EAA0B;QACtH,KAAK,CAAC,IAAI,CAAC,CAAC;QAEZ,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,KAAK,CAAC;QAC9B,IAAI,CAAC,OAAO,GAAG,OAAO,IAAI,IAAI,CAAC;QAC/B,IAAI,CAAC,OAAO,GAAG,OAAO,IAAI,GAAG,CAAC;QAC9B,IAAI,CAAC,UAAU,GAAG,UAAU,IAAI,MAAM,CAAC;QACvC,IAAI,CAAC,eAAe,GAAG,eAAe,IAAI,IAAI,CAAC;QAE/C,IAAI,IAAI,CAAC,OAAO,IAAI,CAAC,EAAE,CAAC;YACpB,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,YAAY,EAAE,6CAA6C,CAAC,CAAC;QACrF,CAAC;QAED,2BAA2B;QAC3B,IAAI,CAAC,aAAa,GAAG,IAAI,kBAAkB,CAAC;YACxC,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO;YACvE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM;SAC7C,CAAC,CAAC;QACH,IAAI,CAAC,oBAAoB,GAAG,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAA;QACrE,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC,MAAM,CAAC,kBAAkB,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QAEzE,yBAAyB;QACzB,IAAI,CAAC,SAAS,GAAG,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC;YAC7B,KAAK,EAAE,IAAI,CAAC,eAAe,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU;YACxD,OAAO,EAAE,IAAI,CAAC,OAAO;SACxB,CAAC,CAAC;QACH,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC;YAC/B,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE,UAAU,EAAE,QAAQ;YAC1C,OAAO,EAAE,IAAI,CAAC,OAAO;SACxB,CAAC,CAAC;QACH,IAAI,CAAC,kBAAkB,GAAG,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QACpE,IAAI,CAAC,cAAc,GAAG,EAAE,CAAC,MAAM,CAAC,kBAAkB,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1E,CAAC;IAGD;;OAEG;IACM,IAAI,CAAC,MAA+B,EAAE,MAAc;QACzD,6BAA6B;QAC7B,IAAI,KAAgB,CAAC;QAErB,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YACxB,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBACrB,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,YAAY,UAAU,IAAI,CAAC,IAAI,2BAA2B;oBAC1E,eAAe,MAAM,CAAC,MAAM,kBAAkB,CAAC,CAAC;YACxD,CAAC;YAED,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QACtB,CAAC;aAAM,CAAC;YACJ,KAAK,GAAG,MAAM,CAAC;QACnB,CAAC;QAED,8BAA8B;QAC9B,OAAO,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE;YAChB,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;YACzD,MAAM,WAAW,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;YAE7D,OAAO,WAAW,CAAC;QACvB,CAAC,CAAC,CAAC;IACP,CAAC;IAGO,kBAAkB,CAAC,CAAY,EAAE,MAAc;QACnD,OAAO,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE;YAChB,MAAM,QAAQ,GAAG,CAAC,CAAC;YAEnB,IAAI,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAc,CAAC;YACjE,SAAS,GAAG,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC,SAAS,EAAE,MAAM,CAAc,CAAC;YAC5E,SAAS,GAAG,EAAE,CAAC,GAAG,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;YACxC,SAAS,GAAG,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,SAAS,CAAc,CAAC;YAEjE,OAAO,SAAS,CAAC;QACrB,CAAC,CAAC,CAAC;IACP,CAAC;IAGO,gBAAgB,CAAC,CAAY,EAAE,MAAc;QACjD,OAAO,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE;YAChB,MAAM,QAAQ,GAAG,CAAC,CAAC;YAEnB,IAAI,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAC1C,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;YAClD,WAAW,GAAG,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC,WAAW,EAAE,MAAM,CAAc,CAAC;YAC9E,WAAW,GAAG,EAAE,CAAC,GAAG,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;YAC5C,WAAW,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,WAAW,CAAc,CAAC;YAElE,OAAO,WAAW,CAAC;QACvB,CAAC,CAAC,CAAC;IACP,CAAC;IAGD;;OAEG;IACM,KAAK,CAAC,UAAiC;QAC5C,IAAI,YAAY,GAAe,EAAE,CAAC;QAElC,IAAI,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5D,8BAA8B;YAC9B,YAAY,GAAG,UAAwB,CAAC;QAC5C,CAAC;aAAM,IAAI,UAAU,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YAChC,0BAA0B;YAC1B,YAAY,GAAG,CAAC,UAAsB,CAAC,CAAC;QAC5C,CAAC;QAED,qCAAqC;QACrC,IAAI,YAAY,CAAC,MAAM,IAAI,CAAC,IAAI,YAAY,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YAC1D,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,YAAY,EAAE,WAAW,IAAI,CAAC,IAAI,iEAAiE,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,EAAE,CAAC,CAAA;QACxJ,CAAC;QAED,gDAAgD;QAChD,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QACrC,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QAEzC,8CAA8C;QAC9C,MAAM,oBAAoB,GAAG,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAAC,UAAU,CAAC,CAAC;QAC3E,MAAM,sBAAsB,GAAG,IAAI,CAAC,WAAW,CAAC,kBAAkB,CAAC,oBAAoB,CAAC,CAAC;QAEzF,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QACjC,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;QAC7C,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;QAElD,2BAA2B;QAC3B,IAAI,CAAC,gBAAgB,GAAG;YACpB,GAAG,IAAI,CAAC,aAAa,CAAC,gBAAgB;YACtC,GAAG,IAAI,CAAC,oBAAoB,CAAC,gBAAgB;YAC7C,GAAG,IAAI,CAAC,iBAAiB,CAAC,gBAAgB;YAC1C,GAAG,IAAI,CAAC,SAAS,CAAC,gBAAgB;YAClC,GAAG,IAAI,CAAC,WAAW,CAAC,gBAAgB;YACpC,GAAG,IAAI,CAAC,kBAAkB,CAAC,gBAAgB;YAC3C,GAAG,IAAI,CAAC,cAAc,CAAC,gBAAgB;SAC1C,CAAC;QAEF,8EAA8E;QAC9E,sEAAsE;QACtE,IAAI,QAAQ,GAAG,CAAC,CAAC;QAEjB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACzC,MAAM,WAAW,GAAG,GAAG,IAAI,CAAC,YAAY,EAAE,IAAI,QAAQ,EAAE,CAAC;YACxD,MAAc,CAAC,IAAI,IAAI,WAAW,CAAC;YACnC,MAAc,CAAC,YAAY,IAAI,WAAW,CAAC;YAC5C,QAAQ,EAAE,CAAC;QACf,CAAC;QAED,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IAC5B,CAAC;IAGD;;OAEG;IACM,SAAS;QACd,MAAM,WAAW,GAAG,KAAK,CAAC,SAAS,EAAE,CAAC;QAEtC,MAAM,MAAM,GAAG;YACX,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,eAAe,EAAE,IAAI,CAAC,eAAe;SACxC,CAAC;QAEF,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;QAEnC,OAAO,MAAM,CAAC;IAClB,CAAC;;AAIL,EAAE,CAAC,aAAa,CAAC,aAAa,CAAC,kBAAkB,CAAC,CAAC"}

package/dist/src/layers/transformer_encoder.test.d.ts

@@ -1,2 +0,0 @@
-export {};
-//# sourceMappingURL=transformer_encoder.test.d.ts.map

package/dist/src/layers/transformer_encoder.test.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"transformer_encoder.test.d.ts","sourceRoot":"","sources":["../../../src/layers/transformer_encoder.test.ts"],"names":[],"mappings":""}

package/dist/src/layers/transformer_encoder.test.js

@@ -1,58 +0,0 @@
-import * as tf from "@tensorflow/tfjs";
-import { TransformerEncoder } from "@/layers/transformer_encoder";
-// disables warning for using the faster node backend,
-// https://github.com/tensorflow/tfjs/issues/5349#issuecomment-885170504
-tf.env().set('IS_NODE', false);
-describe("TransformerEncoder tests", () => {
-    it("should return an output with the same shape as the input", () => {
-        const input = tf.randomUniform([2, 3, 10]);
-        const decoder = new TransformerEncoder({
-            numHeads: 2, embedDim: input.shape.at(-1),
-            dropout: 0.5, activation: "gelu", dimsFeedForward: 512, useBias: true
-        });
-        const output = decoder.apply(input);
-        expect(output.shape.length).toBe(input.shape.length);
-    });
-    test("correct forward calls", () => {
-        const input = tf.randomUniform([2, 3, 10]);
-        const encoder = new TransformerEncoder({ numHeads: 2, embedDim: input.shape.at(-1) });
-        expect(() => encoder.apply(input)).not.toThrow();
-        expect(() => encoder.apply([input])).not.toThrow();
-        const causal = new TransformerEncoder({ numHeads: 2, embedDim: input.shape.at(-1), causal: true });
-        expect(() => causal.apply(input)).not.toThrow();
-        expect(() => causal.apply([input])).not.toThrow();
-    });
-    it("should fail to instantiate a layer if heads count is not divisible by the input's embedding dimension", () => {
-        const input = tf.randomUniform([2, 3, 10]);
-        expect(() => new TransformerEncoder({ numHeads: 3, embedDim: input.shape.at(-1) })).toThrow();
-        expect(() => new TransformerEncoder({ numHeads: 5, embedDim: input.shape.at(-1) })).not.toThrow();
-    });
-    it("should not accept non-rank 3 tensor inputs", () => {
-        const incorrect_input = tf.randomUniform([2, 3, 10, 10]);
-        const incorrect_input2 = tf.randomUniform([2, 3]);
-        const correct_input = tf.randomUniform([2, 3, 10]);
-        const encoder = new TransformerEncoder({ numHeads: 2, embedDim: incorrect_input.shape.at(-1) });
-        expect(() => encoder.apply([correct_input, correct_input])).toThrow();
-        expect(() => encoder.apply(incorrect_input)).toThrow();
-        expect(() => encoder.apply(incorrect_input2)).toThrow();
-        expect(() => encoder.apply([correct_input, incorrect_input])).toThrow();
-        expect(() => encoder.apply([incorrect_input, correct_input])).toThrow();
-        expect(() => encoder.apply([correct_input, incorrect_input2])).toThrow();
-        expect(() => encoder.apply([incorrect_input2, correct_input])).toThrow();
-    });
-    it("should accept exactly one input", () => {
-        const input = tf.randomUniform([2, 3, 10]);
-        const encoder = new TransformerEncoder({ numHeads: 1, embedDim: input.shape.at(-1) });
-        expect(() => encoder.apply(input)).not.toThrow();
-        expect(() => encoder.apply([input])).not.toThrow();
-        expect(() => encoder.apply([])).toThrow();
-        expect(() => encoder.apply([input, input])).toThrow();
-        expect(() => encoder.apply([input, input, input])).toThrow();
-    });
-    it("should return a non-empty config dict", () => {
-        const input = tf.randomUniform([2, 3, 10]);
-        const encoder = new TransformerEncoder({ numHeads: 1, embedDim: input.shape.at(-1) });
-        expect(Object.keys(encoder.getConfig())).not.toBe(0);
-    });
-});
-//# sourceMappingURL=transformer_encoder.test.js.map

package/dist/src/layers/transformer_encoder.test.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"transformer_encoder.test.js","sourceRoot":"","sources":["../../../src/layers/transformer_encoder.test.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAEvC,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAC;AAElE,sDAAsD;AACtD,wEAAwE;AACxE,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;AAG/B,QAAQ,CAAC,0BAA0B,EAAE,GAAG,EAAE;IACtC,EAAE,CAAC,0DAA0D,EAAE,GAAG,EAAE;QAChE,MAAM,KAAK,GAAG,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QAE3C,MAAM,OAAO,GAAG,IAAI,kBAAkB,CAAC;YACnC,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE;YAC1C,OAAO,EAAE,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,eAAe,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI;SACxE,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,CAAc,CAAC;QAEjD,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACzD,CAAC,CAAC,CAAA;IAGF,IAAI,CAAC,uBAAuB,EAAE,GAAG,EAAE;QAC/B,MAAM,KAAK,GAAG,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QAE3C,MAAM,OAAO,GAAG,IAAI,kBAAkB,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,EAAE,CAAC,CAAC;QACvF,MAAM,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;QACjD,MAAM,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;QAEnD,MAAM,MAAM,GAAG,IAAI,kBAAkB,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;QACpG,MAAM,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;QAChD,MAAM,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;IACtD,CAAC,CAAC,CAAA;IAGF,EAAE,CAAC,uGAAuG,EAAE,GAAG,EAAE;QAC7G,MAAM,KAAK,GAAG,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QAE3C,MAAM,CAAC,GAAG,EAAE,CAAC,IAAI,kBAAkB,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,EAAE,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;QAC/F,MAAM,CAAC,GAAG,EAAE,CAAC,IAAI,kBAAkB,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;IACvG,CAAC,CAAC,CAAA;IAGF,EAAE,CAAC,4CAA4C,EAAE,GAAG,EAAE;QAClD,MAAM,eAAe,GAAG,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;QACzD,MAAM,gBAAgB,GAAG,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAClD,MAAM,aAAa,GAAG,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QAGnD,MAAM,OAAO,GAAG,IAAI,kBAAkB,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,eAAe,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,EAAE,CAAC,CAAC;QACjG,MAAM,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,aAAa,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;QAEtE,MAAM,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;QACvD,MAAM,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;QAExD,MAAM,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,aAAa,EAAE,eAAe,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;QACxE,MAAM,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,eAAe,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;QAExE,MAAM,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,aAAa,EAAE,gBAAgB,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;QACzE,MAAM,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,gBAAgB,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;IAC7E,CAAC,CAAC,CAAA;IAGF,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;QACvC,MAAM,KAAK,GAAG,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QAE3C,MAAM,OAAO,GAAG,IAAI,kBAAkB,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,EAAE,CAAC,CAAC;QACvF,MAAM,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;QACjD,MAAM,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;QAEnD,MAAM,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;QAC1C,MAAM,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;QACtD,MAAM,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAA;IAChE,CAAC,CAAC,CAAA;IAGF,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;QAC7C,MAAM,KAAK,GAAG,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QAE3C,MAAM,OAAO,GAAG,IAAI,kBAAkB,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,EAAE,CAAC,CAAC;QACvF,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACzD,CAAC,CAAC,CAAA;AACN,CAAC,CAAC,CAAA"}

package/dist/src/losses/dice.d.ts

@@ -1,30 +0,0 @@
-import * as tf from "@tensorflow/tfjs";
-export declare function diceBinaryStandard(y_true: tf.Tensor, y_pred: tf.Tensor): tf.Tensor;
-export declare function diceBinaryGlobal(y_true: tf.Tensor, y_pred: tf.Tensor): tf.Tensor;
-export declare function diceCategoricalStandard(y_true: tf.Tensor, y_pred: tf.Tensor): tf.Tensor;
-export declare function diceCategoricalGeneralized(y_true: tf.Tensor, y_pred: tf.Tensor): tf.Tensor;
-export declare function diceCategoricalGlobal(y_true: tf.Tensor, y_pred: tf.Tensor): tf.Tensor;
-/**
- * Calculates the Sorensen-Dice coefficient and the binary cross entropy losses.
- * Both have equal weight.
- *
- * @param y_true the label tensor
- * @param y_pred the prediction tensor (not sparse)
- * @returns a tensor of shape `[ batch ]`
- */
-export declare function diceBinaryCrossentropy(y_true: tf.Tensor, y_pred: tf.Tensor): tf.Tensor;
-/**
- * Calculates the Sorensen-Dice coefficient and the categorical cross entropy losses.
- * Both have equal weight. Expects dense (non-sparse) label tensors.
- *
- * This does not support sparse tensors because TFJS's
- * sparseCategoricalCrossentropy loss onehots the label
- * and calls categoricalCrossentropy. See
- * https://github.com/tensorflow/tfjs/blob/0fc04d958ea592f3b8db79a8b3b497b5c8904097/tfjs-layers/src/losses.ts#L143-L146
- *
- * @param y_true the label
- * @param y_pred the prediction tensor (not sparse)
- * @returns a tensor of shape `[ batch ]`
- */
-export declare function diceCategoricalCrossentropy(y_true: tf.Tensor, y_pred: tf.Tensor): tf.Tensor;
-//# sourceMappingURL=dice.d.ts.map

package/dist/src/losses/dice.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"dice.d.ts","sourceRoot":"","sources":["../../../src/losses/dice.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAWvC,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC,MAAM,GAAG,EAAE,CAAC,MAAM,CAclF;AAQD,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC,MAAM,GAAG,EAAE,CAAC,MAAM,CAahF;AAOD,wBAAgB,uBAAuB,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC,MAAM,GAAG,EAAE,CAAC,MAAM,CAUvF;AAOD,wBAAgB,0BAA0B,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC,MAAM,GAAG,EAAE,CAAC,MAAM,CAgB1F;AAOD,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC,MAAM,GAAG,EAAE,CAAC,MAAM,CAWrF;AAOD;;;;;;;GAOG;AACH,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC,MAAM,GAAG,EAAE,CAAC,MAAM,CAMtF;AAOD;;;;;;;;;;;;GAYG;AACH,wBAAgB,2BAA2B,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC,MAAM,GAAG,EAAE,CAAC,MAAM,CAM3F"}

package/dist/src/losses/dice.js

@@ -1,93 +0,0 @@
-import * as tf from "@tensorflow/tfjs";
-import { categoricalCrossentropy, binaryCrossentropy } from "@tensorflow/tfjs-layers/dist/losses";
-const epsilon = 1e-7;
-const REDUCE_HW = [1, 2]; // reduce over width and height
-const REDUCE_BHW = [0, 1, 2]; // reduce over batch, width, height
-const REDUCE_BHWC = [0, 1, 2, 3]; // reduce all dimensions
-// Standard (Sorensen) Dice Loss
-export function diceBinaryStandard(y_true, y_pred) {
-    const y_true_flat = tf.reshape(y_true, [y_true.shape[0], -1]);
-    const y_pred_flat = tf.reshape(y_pred, [y_pred.shape[0], -1]);
-    const intersection = tf.sum(tf.mul(y_true_flat, y_pred_flat), 1);
-    const union = tf.add(tf.sum(y_true_flat, 1), tf.sum(y_pred_flat, 1));
-    const dice = tf.div(intersection.mul(2).add(epsilon), union.add(epsilon));
-    return tf.scalar(1).sub(dice);
-}
-// prevents minification of function name which TFJS relies on
-Object.defineProperty(diceBinaryStandard, "name", { value: "diceBinaryStandard", configurable: false });
-// https://github.com/keras-team/keras/blob/v3.3.3/keras/src/losses/losses.py#L1983-L2010
-export function diceBinaryGlobal(y_true, y_pred) {
-    const y_true_flat = tf.reshape(y_true, [-1]);
-    const y_pred_flat = tf.reshape(y_pred, [-1]);
-    const intersection = tf.sum(tf.mul(y_true_flat, y_pred_flat));
-    const union = tf.add(tf.sum(y_true_flat), tf.sum(y_pred_flat));
-    const dice = tf.div(intersection.mul(2).add(epsilon), union.add(epsilon));
-    return tf.scalar(1).sub(dice);
-}
-// prevents minification of function name which TFJS relies on
-Object.defineProperty(diceBinaryGlobal, "name", { value: "diceBinaryGlobal", configurable: false });
-export function diceCategoricalStandard(y_true, y_pred) {
-    const intersection = tf.sum(tf.mul(y_true, y_pred), REDUCE_HW);
-    const union = tf.add(y_true, y_pred).sum(REDUCE_HW);
-    const dice = tf.div(intersection.mul(2).add(epsilon), union.add(epsilon));
-    return tf.scalar(1).sub(tf.mean(dice, -1));
-}
-// prevents minification of function name which TFJS relies on
-Object.defineProperty(diceCategoricalStandard, "name", { value: "diceCategoricalStandard", configurable: false });
-export function diceCategoricalGeneralized(y_true, y_pred) {
-    // this is done twice so we calculate it once
-    const y_true_sum = y_true.sum(REDUCE_BHW);
-    const weighting = tf.div(1, y_true_sum.square().add(epsilon));
-    const intersection = tf.sum(tf.mul(y_true, y_pred), REDUCE_BHW).mul(weighting).sum();
-    const union = tf.add(y_true_sum, y_pred.sum(REDUCE_BHW)).mul(weighting).sum();
-    const dice = tf.div(intersection.mul(2).add(epsilon), union.add(epsilon));
-    return tf.scalar(1).sub(dice);
-}
-// prevents minification of function name which TFJS relies on
-Object.defineProperty(diceCategoricalGeneralized, "name", { value: "diceCategoricalGeneralized", configurable: false });
-export function diceCategoricalGlobal(y_true, y_pred) {
-    const intersection = tf.sum(tf.mul(y_true, y_pred), REDUCE_BHWC);
-    const union = tf.add(tf.sum(y_true, REDUCE_BHWC), tf.sum(y_pred, REDUCE_BHWC));
-    const dice = tf.div(intersection.mul(2).add(epsilon), union.add(epsilon));
-    return tf.scalar(1).sub(dice);
-}
-// prevents minification of function name which TFJS relies on
-Object.defineProperty(diceCategoricalGlobal, "name", { value: "diceCategoricalGlobal", configurable: false });
-/**
- * Calculates the Sorensen-Dice coefficient and the binary cross entropy losses.
- * Both have equal weight.
- *
- * @param y_true the label tensor
- * @param y_pred the prediction tensor (not sparse)
- * @returns a tensor of shape `[ batch ]`
- */
-export function diceBinaryCrossentropy(y_true, y_pred) {
-    // reduce cross entropy shape from [B, H, W] to [B] to match dice
-    const bce = binaryCrossentropy(y_true, y_pred).mean(REDUCE_HW);
-    const dice = diceBinaryStandard(y_true, y_pred);
-    return tf.add(bce.mul(0.5), dice.mul(0.5));
-}
-// prevents minification of function name which TFJS relies on
-Object.defineProperty(diceBinaryCrossentropy, "name", { value: "diceBinaryCrossentropy", configurable: false });
-/**
- * Calculates the Sorensen-Dice coefficient and the categorical cross entropy losses.
- * Both have equal weight. Expects dense (non-sparse) label tensors.
- *
- * This does not support sparse tensors because TFJS's
- * sparseCategoricalCrossentropy loss onehots the label
- * and calls categoricalCrossentropy. See
- * https://github.com/tensorflow/tfjs/blob/0fc04d958ea592f3b8db79a8b3b497b5c8904097/tfjs-layers/src/losses.ts#L143-L146
- *
- * @param y_true the label
- * @param y_pred the prediction tensor (not sparse)
- * @returns a tensor of shape `[ batch ]`
- */
-export function diceCategoricalCrossentropy(y_true, y_pred) {
-    // reduce cross entropy shape from [B, H, W] to [B] to match dice
-    const cce = categoricalCrossentropy(y_true, y_pred).mean(REDUCE_HW);
-    const dice = diceCategoricalStandard(y_true, y_pred);
-    return tf.add(cce.mul(0.5), dice.mul(0.5));
-}
-// prevents minification of function name which TFJS relies on
-Object.defineProperty(diceCategoricalCrossentropy, "name", { value: "diceCategoricalCrossentropy", configurable: false });
-//# sourceMappingURL=dice.js.map

package/dist/src/losses/dice.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"dice.js","sourceRoot":"","sources":["../../../src/losses/dice.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACvC,OAAO,EAAE,uBAAuB,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AAElG,MAAM,OAAO,GAAG,IAAI,CAAC;AAErB,MAAM,SAAS,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,+BAA+B;AACzD,MAAM,UAAU,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,mCAAmC;AACjE,MAAM,WAAW,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,wBAAwB;AAG1D,gCAAgC;AAChC,MAAM,UAAU,kBAAkB,CAAC,MAAiB,EAAE,MAAiB;IAEnE,MAAM,WAAW,GAAG,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9D,MAAM,WAAW,GAAG,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAE9D,MAAM,YAAY,GAAG,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,WAAW,EAAE,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC;IACjE,MAAM,KAAK,GAAG,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC;IAErE,MAAM,IAAI,GAAG,EAAE,CAAC,GAAG,CACf,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,EAChC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CACrB,CAAC;IAEF,OAAO,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;AAClC,CAAC;AAGD,8DAA8D;AAC9D,MAAM,CAAC,cAAc,CAAC,kBAAkB,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,oBAAoB,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC;AAGxG,yFAAyF;AACzF,MAAM,UAAU,gBAAgB,CAAC,MAAiB,EAAE,MAAiB;IACjE,MAAM,WAAW,GAAG,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7C,MAAM,WAAW,GAAG,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAE7C,MAAM,YAAY,GAAG,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC,CAAC;IAC9D,MAAM,KAAK,GAAG,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC;IAE/D,MAAM,IAAI,GAAG,EAAE,CAAC,GAAG,CACf,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,EAChC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CACrB,CAAC;IAEF,OAAO,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;AAClC,CAAC;AAGD,8DAA8D;AAC9D,MAAM,CAAC,cAAc,CAAC,gBAAgB,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,kBAAkB,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC;AAGpG,MAAM,UAAU,uBAAuB,CAAC,MAAiB,EAAE,MAAiB;IACxE,MAAM,YAAY,GAAG,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,SAAS,CAAC,CAAC;IAC/D,MAAM,KAAK,GAAG,EAAE,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IAEpD,MAAM,IAAI,GAAG,EAAE,CAAC,GAAG,CACf,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,EAChC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CACrB,CAAC;IAEF,OAAO,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;AAC/C,CAAC;AAGD,8DAA8D;AAC9D,MAAM,CAAC,cAAc,CAAC,uBAAuB,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,yBAAyB,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC;AAGlH,MAAM,UAAU,0BAA0B,CAAC,MAAiB,EAAE,MAAiB;IAE3E,6CAA6C;IAC7C,MAAM,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IAE1C,MAAM,SAAS,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;IAE9D,MAAM,YAAY,GAAG,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,UAAU,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,GAAG,EAAE,CAAC;IACrF,MAAM,KAAK,GAAG,EAAE,CAAC,GAAG,CAAC,UAAU,EAAE,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,GAAG,EAAE,CAAC;IAE9E,MAAM,IAAI,GAAG,EAAE,CAAC,GAAG,CACf,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,EAChC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CACrB,CAAC;IAEF,OAAO,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;AAClC,CAAC;AAGD,8DAA8D;AAC9D,MAAM,CAAC,cAAc,CAAC,0BAA0B,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,4BAA4B,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC;AAGxH,MAAM,UAAU,qBAAqB,CAAC,MAAiB,EAAE,MAAiB;IAEtE,MAAM,YAAY,GAAG,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,CAAC,CAAC;IACjE,MAAM,KAAK,GAAG,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,MAAM,EAAE,WAAW,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC;IAE/E,MAAM,IAAI,GAAG,EAAE,CAAC,GAAG,CACf,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,EAChC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CACrB,CAAC;IAEF,OAAO,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;AAClC,CAAC;AAGD,8DAA8D;AAC9D,MAAM,CAAC,cAAc,CAAC,qBAAqB,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,uBAAuB,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC;AAG9G;;;;;;;GAOG;AACH,MAAM,UAAU,sBAAsB,CAAC,MAAiB,EAAE,MAAiB;IACvE,iEAAiE;IACjE,MAAM,GAAG,GAAG,kBAAkB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC/D,MAAM,IAAI,GAAG,kBAAkB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEhD,OAAO,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;AAC/C,CAAC;AAGD,8DAA8D;AAC9D,MAAM,CAAC,cAAc,CAAC,sBAAsB,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,wBAAwB,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC;AAGhH;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,2BAA2B,CAAC,MAAiB,EAAE,MAAiB;IAC5E,iEAAiE;IACjE,MAAM,GAAG,GAAG,uBAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACpE,MAAM,IAAI,GAAG,uBAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAErD,OAAO,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;AAC/C,CAAC;AAGD,8DAA8D;AAC9D,MAAM,CAAC,cAAc,CAAC,2BAA2B,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,6BAA6B,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC"}

package/dist/src/losses/index.d.ts

@@ -1,2 +0,0 @@
-export * from "./dice";
-//# sourceMappingURL=index.d.ts.map

package/dist/src/losses/index.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/losses/index.ts"],"names":[],"mappings":"AAAA,cAAc,QAAQ,CAAC"}

package/dist/src/losses/index.js

@@ -1,2 +0,0 @@
-export * from "./dice";
-//# sourceMappingURL=index.js.map

package/dist/src/losses/index.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/losses/index.ts"],"names":[],"mappings":"AAAA,cAAc,QAAQ,CAAC"}

package/dist/src/masks.d.ts

@@ -1,20 +0,0 @@
-import * as tf from "@tensorflow/tfjs";
-/**
- * Generate a causal mask used in self-attention to prevent tokens from looking
- * ahead. The values in the upper right portion of the mask matrix are set to
- * -1e7 so that they have no impact during scaled dot product attention.
- */
-export declare function causal(query_seq_length: number, key_seq_length: number): tf.Tensor<tf.Rank>;
-/**
- * Generate a self-attention mask that prevents packed sequences from cross document
- * boundaries and attending to each other. The result is a tensor of diagonally
- * positioned blocks of zeroes (allow attention) and -1e7 values (prevent attention).
- * The latter is scored zero during the scaled dot product attention's softmax operation.
- *
- * @param boundaries an array of ones (denotes start of a new sample or docment) and zeroes
- *
- * Example boundary of 3 samples that are packed into one:
- * `[1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0]`
- */
-export declare function packing(boundaries: Int32Array): tf.Tensor<tf.Rank>;
-//# sourceMappingURL=masks.d.ts.map

package/dist/src/masks.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"masks.d.ts","sourceRoot":"","sources":["../../src/masks.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAGvC;;;;GAIG;AACH,wBAAgB,MAAM,CAAC,gBAAgB,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,sBAItE;AAGD;;;;;;;;;;GAUG;AACH,wBAAgB,OAAO,CAAC,UAAU,EAAE,UAAU,sBAc7C"}

package/dist/src/masks.js

@@ -1,37 +0,0 @@
-import * as tf from "@tensorflow/tfjs";
-/**
- * Generate a causal mask used in self-attention to prevent tokens from looking
- * ahead. The values in the upper right portion of the mask matrix are set to
- * -1e7 so that they have no impact during scaled dot product attention.
- */
-export function causal(query_seq_length, key_seq_length) {
-    return tf.linalg.bandPart(tf.ones([query_seq_length, key_seq_length]), -1, 0)
-        .sub(1)
-        .mul(1e7);
-}
-/**
- * Generate a self-attention mask that prevents packed sequences from cross document
- * boundaries and attending to each other. The result is a tensor of diagonally
- * positioned blocks of zeroes (allow attention) and -1e7 values (prevent attention).
- * The latter is scored zero during the scaled dot product attention's softmax operation.
- *
- * @param boundaries an array of ones (denotes start of a new sample or docment) and zeroes
- *
- * Example boundary of 3 samples that are packed into one:
- * `[1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0]`
- */
-export function packing(boundaries) {
-    // see images at
-    // https://reddit.com/r/LocalLLaMA/comments/197efaz/training_llama_mistral_and_mixtralmoe_faster_with/
-    return tf.tidy(() => {
-        // cumsum transforms the tensor such that each sequence in the pack gets its own id,
-        const partitions = tf.tensor1d(boundaries).cumsum();
-        return partitions.expandDims(1)
-            .equal(partitions.expandDims(0))
-            .sub(1)
-            .mul(1e7)
-            // introduce a head dimension so it can be broadcasted
-            .expandDims(0);
-    });
-}
-//# sourceMappingURL=masks.js.map

package/dist/src/masks.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"masks.js","sourceRoot":"","sources":["../../src/masks.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAGvC;;;;GAIG;AACH,MAAM,UAAU,MAAM,CAAC,gBAAwB,EAAE,cAAsB;IACnE,OAAO,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,gBAAgB,EAAE,cAAc,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;SACxE,GAAG,CAAC,CAAC,CAAC;SACN,GAAG,CAAC,GAAG,CAAC,CAAC;AAClB,CAAC;AAGD;;;;;;;;;;GAUG;AACH,MAAM,UAAU,OAAO,CAAC,UAAsB;IAC1C,gBAAgB;IAChB,sGAAsG;IACtG,OAAO,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE;QAChB,oFAAoF;QACpF,MAAM,UAAU,GAAG,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,MAAM,EAAE,CAAC;QAEpD,OAAO,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC;aAC1B,KAAK,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;aAC/B,GAAG,CAAC,CAAC,CAAC;aACN,GAAG,CAAC,GAAG,CAAC;YACT,sDAAsD;aACrD,UAAU,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC,CAAC,CAAA;AACN,CAAC"}

package/dist/src/metrics.d.ts

@@ -1,20 +0,0 @@
-import { Tensor } from "@tensorflow/tfjs";
-/**
- * Applies the recall metric with the prediction rounded based on a threshold
- *
- * @param y_true the label tensor
- * @param y_pred the prediction tensor
- * @param threshold threshold value to be considered a positive prediction, defaults to `0.5`
- * @returns
- */
-export declare function recall(y_true: Tensor, y_pred: Tensor, threshold?: number): Tensor<import("@tensorflow/tfjs-core").Rank>;
-/**
- * Applies the precision metric with the prediction rounded based on a threshold
- *
- * @param y_true the label tensor
- * @param y_pred the prediction tensor
- * @param threshold threshold value to be considered a positive prediction, defaults to `0.5`
- * @returns
- */
-export declare function precision(y_true: Tensor, y_pred: Tensor, threshold?: number): Tensor<import("@tensorflow/tfjs-core").Rank>;
-//# sourceMappingURL=metrics.d.ts.map

package/dist/src/metrics.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../../src/metrics.ts"],"names":[],"mappings":"AAAA,OAAO,EAAW,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAGnD;;;;;;;GAOG;AACH,wBAAgB,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,GAAE,MAAY,gDAE7E;AAKD;;;;;;;GAOG;AACH,wBAAgB,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,GAAE,MAAY,gDAEhF"}

package/dist/src/metrics.js

@@ -1,28 +0,0 @@
-import { metrics } from "@tensorflow/tfjs";
-/**
- * Applies the recall metric with the prediction rounded based on a threshold
- *
- * @param y_true the label tensor
- * @param y_pred the prediction tensor
- * @param threshold threshold value to be considered a positive prediction, defaults to `0.5`
- * @returns
- */
-export function recall(y_true, y_pred, threshold = 0.5) {
-    return metrics.recall(y_true, y_pred.greaterEqual(threshold));
-}
-// prevents minification of function name which TFJS relies on
-Object.defineProperty(recall, "name", { value: "recall", configurable: false });
-/**
- * Applies the precision metric with the prediction rounded based on a threshold
- *
- * @param y_true the label tensor
- * @param y_pred the prediction tensor
- * @param threshold threshold value to be considered a positive prediction, defaults to `0.5`
- * @returns
- */
-export function precision(y_true, y_pred, threshold = 0.5) {
-    return metrics.precision(y_true, y_pred.greaterEqual(threshold));
-}
-// prevents minification of function name which TFJS relies on
-Object.defineProperty(precision, "name", { value: "precision", configurable: false });
-//# sourceMappingURL=metrics.js.map

package/dist/src/metrics.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../../src/metrics.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAU,MAAM,kBAAkB,CAAC;AAGnD;;;;;;;GAOG;AACH,MAAM,UAAU,MAAM,CAAC,MAAc,EAAE,MAAc,EAAE,YAAoB,GAAG;IAC1E,OAAO,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC;AAClE,CAAC;AAED,8DAA8D;AAC9D,MAAM,CAAC,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC;AAEhF;;;;;;;GAOG;AACH,MAAM,UAAU,SAAS,CAAC,MAAc,EAAE,MAAc,EAAE,YAAoB,GAAG;IAC7E,OAAO,OAAO,CAAC,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC;AACrE,CAAC;AAED,8DAA8D;AAC9D,MAAM,CAAC,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC"}

package/dist/src/models/gpt_model.d.ts

@@ -1,94 +0,0 @@
-import * as tf from "@tensorflow/tfjs";
-import { type LossOrMetricFn } from "@/tfjs_types";
-import { LlmModel, type LlmModelArgs } from "@/models/llm_model";
-import { KvCacheContainer } from "@/kv_cache";
-import { type DisposeResult } from "@tensorflow/tfjs-layers/dist/engine/topology";
-export interface GptModelArgs extends LlmModelArgs {
-    /**
-     * Number of heads per attention layer.
-     */
-    numHeads: number;
-    /**
-     * Number of GPT decoder blocks.
-     */
-    numLayers: number;
-    /**
-     * The embedding size of each token.
-     */
-    embedDim: number;
-    /**
-     * The vocabulary size of the embedding layer and number of units of the output
-     * layer. This is also the tokenizer vocabulary size.
-     */
-    vocabSize: number;
-    /**
-     * Pad the embeddings' vocab size and output layer's units to the next nearest
-     * multiple of 64 to optimize hardware efficiency. Defaults to `true`.
-     *
-     * For example: if a tokenizer has 50,257 tokens, the model uses 50,304 for the
-     * vocab size and output units count.
-     */
-    padToMultipleOf64?: boolean;
-}
-/**
- * This is a subclass of tf.Sequential that creating a GPT-like model and
- * automatically handles padding (and masking) the vocab size for hardware
- * efficiency.
- *
- * Example:
- *
- * ```javascript
- *
- * const model = new GptModel({ numLayers: 1, numHeads: 1, embedDim: 16, vocabSize: 64 });
- * model.compile({ loss: "sparseCategoricalCrossentropy", optimizer: "adam" });
- *
- * // use fitDataset() instead of fit for masking support
- * model.fitDataset(your_batched_generator_dataset, { epochs: 1 });
- *
- * const kv_cache = new KvCacheContainer(your_preferred_max_sequence_length);
- *
- * // use generate() and predictNextToken() instead of predict() for masking and auto memory cleanup
- * model.generate(tokenized_tensor1d_input, kv_cache, onPredict_callback)
- *
- *
- * ```
- */
-export declare class GptModel extends LlmModel {
-    static className: string;
-    protected readonly numHeads: number;
-    protected readonly numLayers: number;
-    protected readonly embedDim: number;
-    protected readonly vocabSize: number;
-    protected readonly padToMultipleOf64: boolean;
-    protected readonly vocabSizePadded: number;
-    protected vocab_padding_mask?: tf.Tensor1D;
-    /**
-     * DO NOT add layers in the constructor or it will break tf.loadLayersModel().
-     * It should be done in build() instead.
-     */
-    constructor(args: GptModelArgs);
-    protected fitBatch(xs: tf.Tensor, ys: tf.Tensor, loss_mask: tf.Tensor | undefined, loss_function: LossOrMetricFn, other_masks?: {
-        [key: string]: tf.Tensor | undefined;
-    }): {
-        y_pred: tf.Tensor<tf.Rank>;
-        loss: tf.Scalar;
-    };
-    /**
-     * Overrides LlmModel.predictNextToken to add softmax before argMax because the final
-     * dense layer doesn't have an activation.
-     *
-     * TODO: implement temperature and multinomial sampling so that the model has varied outputs
-     */
-    predictNextToken(input: tf.Tensor2D, kv_cache: KvCacheContainer): tf.Tensor2D;
-    build(inputShape?: tf.Shape | tf.Shape[]): void;
-    dispose(): DisposeResult;
-    getConfig(): {
-        numHeads: number;
-        numLayers: number;
-        embedDim: number;
-        vocabSize: number;
-        vocabSizePadded: number;
-        padToMultipleOf64: boolean;
-    };
-}
-//# sourceMappingURL=gpt_model.d.ts.map