@scrypted/libav 1.0.166 → 1.0.168

package/opus/dnn/training_tf2/keraslayerdump.py

@@ -0,0 +1,189 @@
+'''Copyright (c) 2017-2018 Mozilla
+
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions
+   are met:
+
+   - Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+
+   - Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+   ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+   A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR
+   CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+   EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+   PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+   PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+   LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+'''
+
+""" helper functions for dumping some Keras layers to C files """
+
+import numpy as np
+
+
+def printVector(f, vector, name, dtype='float', dotp=False, static=True):
+    """ prints vector as one-dimensional C array """
+    if dotp:
+        vector = vector.reshape((vector.shape[0]//4, 4, vector.shape[1]//8, 8))
+        vector = vector.transpose((2, 0, 3, 1))
+    v = np.reshape(vector, (-1))
+    if static:
+        f.write('static const {} {}[{}] = {{\n   '.format(dtype, name, len(v)))
+    else:
+        f.write('const {} {}[{}] = {{\n   '.format(dtype, name, len(v)))
+    for i in range(0, len(v)):
+        f.write('{}'.format(v[i]))
+        if (i!=len(v)-1):
+            f.write(',')
+        else:
+            break;
+        if (i%8==7):
+            f.write("\n   ")
+        else:
+            f.write(" ")
+    f.write('\n};\n\n')
+    return vector
+
+def printSparseVector(f, A, name, have_diag=True):
+    N = A.shape[0]
+    M = A.shape[1]
+    W = np.zeros((0,), dtype='int')
+    W0 = np.zeros((0,))
+    if have_diag:
+        diag = np.concatenate([np.diag(A[:,:N]), np.diag(A[:,N:2*N]), np.diag(A[:,2*N:])])
+        A[:,:N] = A[:,:N] - np.diag(np.diag(A[:,:N]))
+        A[:,N:2*N] = A[:,N:2*N] - np.diag(np.diag(A[:,N:2*N]))
+        A[:,2*N:] = A[:,2*N:] - np.diag(np.diag(A[:,2*N:]))
+        printVector(f, diag, name + '_diag')
+    AQ = np.minimum(127, np.maximum(-128, np.round(A*128))).astype('int')
+    idx = np.zeros((0,), dtype='int')
+    for i in range(M//8):
+        pos = idx.shape[0]
+        idx = np.append(idx, -1)
+        nb_nonzero = 0
+        for j in range(N//4):
+            block = A[j*4:(j+1)*4, i*8:(i+1)*8]
+            qblock = AQ[j*4:(j+1)*4, i*8:(i+1)*8]
+            if np.sum(np.abs(block)) > 1e-10:
+                nb_nonzero = nb_nonzero + 1
+                idx = np.append(idx, j*4)
+                vblock = qblock.transpose((1,0)).reshape((-1,))
+                W0 = np.concatenate([W0, block.reshape((-1,))])
+                W = np.concatenate([W, vblock])
+        idx[pos] = nb_nonzero
+    f.write('#ifdef DOT_PROD\n')
+    printVector(f, W, name, dtype='qweight')
+    f.write('#else /*DOT_PROD*/\n')
+    printVector(f, W0, name, dtype='qweight')
+    f.write('#endif /*DOT_PROD*/\n')
+    printVector(f, idx, name + '_idx', dtype='int')
+    return AQ
+
+def dump_sparse_gru(self, f, hf):
+    name = 'sparse_' + self.name
+    print("printing layer " + name + " of type sparse " + self.__class__.__name__)
+    weights = self.get_weights()
+    qweights = printSparseVector(f, weights[1], name + '_recurrent_weights')
+    printVector(f, weights[-1], name + '_bias')
+    subias = weights[-1].copy()
+    subias[1,:] = subias[1,:] - np.sum(qweights*(1./128),axis=0)
+    printVector(f, subias, name + '_subias')
+    if hasattr(self, 'activation'):
+        activation = self.activation.__name__.upper()
+    else:
+        activation = 'TANH'
+    if hasattr(self, 'reset_after') and not self.reset_after:
+        reset_after = 0
+    else:
+        reset_after = 1
+    neurons = weights[0].shape[1]//3
+    max_rnn_neurons = neurons
+    f.write('const SparseGRULayer {} = {{\n   {}_bias,\n   {}_subias,\n   {}_recurrent_weights_diag,\n   {}_recurrent_weights,\n   {}_recurrent_weights_idx,\n   {}, ACTIVATION_{}, {}\n}};\n\n'
+            .format(name, name, name, name, name, name, weights[0].shape[1]//3, activation, reset_after))
+    hf.write('#define {}_OUT_SIZE {}\n'.format(name.upper(), weights[0].shape[1]//3))
+    hf.write('#define {}_STATE_SIZE {}\n'.format(name.upper(), weights[0].shape[1]//3))
+    hf.write('extern const SparseGRULayer {};\n\n'.format(name));
+    return max_rnn_neurons
+
+def dump_gru_layer(self, f, hf, dotp=False, sparse=False):
+    name = self.name
+    print("printing layer " + name + " of type " + self.__class__.__name__)
+    weights = self.get_weights()
+    if sparse:
+        qweight = printSparseVector(f, weights[0], name + '_weights', have_diag=False)
+    else:
+        qweight = printVector(f, weights[0], name + '_weights')
+
+    if dotp:
+        f.write('#ifdef DOT_PROD\n')
+        qweight2 = np.clip(np.round(128.*weights[1]).astype('int'), -128, 127)
+        printVector(f, qweight2, name + '_recurrent_weights', dotp=True, dtype='qweight')
+        f.write('#else /*DOT_PROD*/\n')
+    else:
+        qweight2 = weights[1]
+
+    printVector(f, weights[1], name + '_recurrent_weights')
+    if dotp:
+        f.write('#endif /*DOT_PROD*/\n')
+
+    printVector(f, weights[-1], name + '_bias')
+    subias = weights[-1].copy()
+    subias[0,:] = subias[0,:] - np.sum(qweight*(1./128.),axis=0)
+    subias[1,:] = subias[1,:] - np.sum(qweight2*(1./128.),axis=0)
+    printVector(f, subias, name + '_subias')
+    if hasattr(self, 'activation'):
+        activation = self.activation.__name__.upper()
+    else:
+        activation = 'TANH'
+    if hasattr(self, 'reset_after') and not self.reset_after:
+        reset_after = 0
+    else:
+        reset_after = 1
+    neurons = weights[0].shape[1]//3
+    max_rnn_neurons = neurons
+    f.write('const GRULayer {} = {{\n   {}_bias,\n   {}_subias,\n   {}_weights,\n   {},\n   {}_recurrent_weights,\n   {}, {}, ACTIVATION_{}, {}\n}};\n\n'
+            .format(name, name, name, name, name + "_weights_idx" if sparse else "NULL", name, weights[0].shape[0], weights[0].shape[1]//3, activation, reset_after))
+    hf.write('#define {}_OUT_SIZE {}\n'.format(name.upper(), weights[0].shape[1]//3))
+    hf.write('#define {}_STATE_SIZE {}\n'.format(name.upper(), weights[0].shape[1]//3))
+    hf.write('extern const GRULayer {};\n\n'.format(name));
+    return max_rnn_neurons
+
+def dump_dense_layer_impl(name, weights, bias, activation, f, hf):
+    printVector(f, weights, name + '_weights')
+    printVector(f, bias, name + '_bias')
+    f.write('const DenseLayer {} = {{\n   {}_bias,\n   {}_weights,\n   {}, {}, ACTIVATION_{}\n}};\n\n'
+            .format(name, name, name, weights.shape[0], weights.shape[1], activation))
+    hf.write('#define {}_OUT_SIZE {}\n'.format(name.upper(), weights.shape[1]))
+    hf.write('extern const DenseLayer {};\n\n'.format(name));
+
+def dump_dense_layer(self, f, hf):
+    name = self.name
+    print("printing layer " + name + " of type " + self.__class__.__name__)
+    weights = self.get_weights()
+    activation = self.activation.__name__.upper()
+    dump_dense_layer_impl(name, weights[0], weights[1], activation, f, hf)
+    return False
+
+def dump_conv1d_layer(self, f, hf):
+    name = self.name
+    print("printing layer " + name + " of type " + self.__class__.__name__)
+    weights = self.get_weights()
+    printVector(f, weights[0], name + '_weights')
+    printVector(f, weights[-1], name + '_bias')
+    activation = self.activation.__name__.upper()
+    max_conv_inputs = weights[0].shape[1]*weights[0].shape[0]
+    f.write('const Conv1DLayer {} = {{\n   {}_bias,\n   {}_weights,\n   {}, {}, {}, ACTIVATION_{}\n}};\n\n'
+            .format(name, name, name, weights[0].shape[1], weights[0].shape[0], weights[0].shape[2], activation))
+    hf.write('#define {}_OUT_SIZE {}\n'.format(name.upper(), weights[0].shape[2]))
+    hf.write('#define {}_STATE_SIZE ({}*{})\n'.format(name.upper(), weights[0].shape[1], (weights[0].shape[0]-1)))
+    hf.write('#define {}_DELAY {}\n'.format(name.upper(), (weights[0].shape[0]-1)//2))
+    hf.write('extern const Conv1DLayer {};\n\n'.format(name));
+    return max_conv_inputs

package/opus/dnn/training_tf2/lossfuncs.py

@@ -0,0 +1,99 @@
+"""
+Custom Loss functions and metrics for training/analysis
+"""
+
+from tf_funcs import *
+import tensorflow as tf
+
+# The following loss functions all expect the lpcnet model to output the lpc prediction
+
+# Computing the excitation by subtracting the lpc prediction from the target, followed by minimizing the cross entropy
+def res_from_sigloss():
+    def loss(y_true,y_pred):
+        p = y_pred[:,:,0:1]
+        model_out = y_pred[:,:,2:]
+        e_gt = tf_l2u(y_true - p)
+        e_gt = tf.round(e_gt)
+        e_gt = tf.cast(e_gt,'int32')
+        sparse_cel = tf.keras.losses.SparseCategoricalCrossentropy(reduction=tf.keras.losses.Reduction.NONE)(e_gt,model_out)
+        return sparse_cel
+    return loss
+
+# Interpolated and Compensated Loss (In case of end to end lpcnet)
+# Interpolates between adjacent embeddings based on the fractional value of the excitation computed (similar to the embedding interpolation)
+# Also adds a probability compensation (to account for matching cross entropy in the linear domain), weighted by gamma
+def interp_mulaw(gamma = 1):
+    def loss(y_true,y_pred):
+        y_true = tf.cast(y_true, 'float32')
+        p = y_pred[:,:,0:1]
+        real_p = y_pred[:,:,1:2]
+        model_out = y_pred[:,:,2:]
+        e_gt = tf_l2u(y_true - p)
+        exc_gt = tf_l2u(y_true - real_p)
+        prob_compensation = tf.squeeze((K.abs(e_gt - 128)/128.0)*K.log(256.0))
+        regularization = tf.squeeze((K.abs(exc_gt - 128)/128.0)*K.log(256.0))
+        alpha = e_gt - tf.math.floor(e_gt)
+        alpha = tf.tile(alpha,[1,1,256])
+        e_gt = tf.cast(e_gt,'int32')
+        e_gt = tf.clip_by_value(e_gt,0,254)
+        interp_probab = (1 - alpha)*model_out + alpha*tf.roll(model_out,shift = -1,axis = -1)
+        sparse_cel = tf.keras.losses.SparseCategoricalCrossentropy(reduction=tf.keras.losses.Reduction.NONE)(e_gt,interp_probab)
+        loss_mod = sparse_cel + prob_compensation + gamma*regularization
+        return loss_mod
+    return loss
+
+# Same as above, except a metric
+def metric_oginterploss(y_true,y_pred):
+    p = y_pred[:,:,0:1]
+    model_out = y_pred[:,:,2:]
+    e_gt = tf_l2u(y_true - p)
+    prob_compensation = tf.squeeze((K.abs(e_gt - 128)/128.0)*K.log(256.0))
+    alpha = e_gt - tf.math.floor(e_gt)
+    alpha = tf.tile(alpha,[1,1,256])
+    e_gt = tf.cast(e_gt,'int32')
+    e_gt = tf.clip_by_value(e_gt,0,254)
+    interp_probab = (1 - alpha)*model_out + alpha*tf.roll(model_out,shift = -1,axis = -1)
+    sparse_cel = tf.keras.losses.SparseCategoricalCrossentropy(reduction=tf.keras.losses.Reduction.NONE)(e_gt,interp_probab)
+    loss_mod = sparse_cel + prob_compensation
+    return loss_mod
+
+# Interpolated cross entropy loss metric
+def metric_icel(y_true, y_pred):
+    p = y_pred[:,:,0:1]
+    model_out = y_pred[:,:,2:]
+    e_gt = tf_l2u(y_true - p)
+    alpha = e_gt - tf.math.floor(e_gt)
+    alpha = tf.tile(alpha,[1,1,256])
+    e_gt = tf.cast(e_gt,'int32')
+    e_gt = tf.clip_by_value(e_gt,0,254) #Check direction
+    interp_probab = (1 - alpha)*model_out + alpha*tf.roll(model_out,shift = -1,axis = -1)
+    sparse_cel = tf.keras.losses.SparseCategoricalCrossentropy(reduction=tf.keras.losses.Reduction.NONE)(e_gt,interp_probab)
+    return sparse_cel
+
+# Non-interpolated (rounded) cross entropy loss metric
+def metric_cel(y_true, y_pred):
+    y_true = tf.cast(y_true, 'float32')
+    p = y_pred[:,:,0:1]
+    model_out = y_pred[:,:,2:]
+    e_gt = tf_l2u(y_true - p)
+    e_gt = tf.round(e_gt)
+    e_gt = tf.cast(e_gt,'int32')
+    e_gt = tf.clip_by_value(e_gt,0,255)
+    sparse_cel = tf.keras.losses.SparseCategoricalCrossentropy(reduction=tf.keras.losses.Reduction.NONE)(e_gt,model_out)
+    return sparse_cel
+
+# Variance metric of the output excitation
+def metric_exc_sd(y_true,y_pred):
+    p = y_pred[:,:,0:1]
+    e_gt = tf_l2u(y_true - p)
+    sd_egt = tf.keras.losses.MeanSquaredError(reduction=tf.keras.losses.Reduction.NONE)(e_gt,128)
+    return sd_egt
+
+def loss_matchlar():
+    def loss(y_true,y_pred):
+        model_rc = y_pred[:,:,:16]
+        #y_true = lpc2rc(y_true)
+        loss_lar_diff = K.log((1.01 + model_rc)/(1.01 - model_rc)) - K.log((1.01 + y_true)/(1.01 - y_true))
+        loss_lar_diff = tf.square(loss_lar_diff)
+        return tf.reduce_mean(loss_lar_diff, axis=-1)
+    return loss

package/opus/dnn/training_tf2/lpcnet.py

@@ -0,0 +1,339 @@
+#!/usr/bin/python3
+'''Copyright (c) 2018 Mozilla
+
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions
+   are met:
+
+   - Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+
+   - Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+   ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+   A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR
+   CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+   EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+   PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+   PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+   LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+'''
+
+import math
+import tensorflow as tf
+from tensorflow.keras.models import Model
+from tensorflow.keras.layers import Input, GRU, Dense, Embedding, Reshape, Concatenate, Lambda, Conv1D, Multiply, Add, Bidirectional, MaxPooling1D, Activation, GaussianNoise
+from tensorflow.compat.v1.keras.layers import CuDNNGRU
+from tensorflow.keras import backend as K
+from tensorflow.keras.constraints import Constraint
+from tensorflow.keras.initializers import Initializer
+from tensorflow.keras.callbacks import Callback
+from mdense import MDense
+import numpy as np
+import h5py
+import sys
+from tf_funcs import *
+from diffembed import diff_Embed
+from parameters import set_parameter
+
+frame_size = 160
+pcm_bits = 8
+embed_size = 128
+pcm_levels = 2**pcm_bits
+
+def interleave(p, samples):
+    p2=tf.expand_dims(p, 3)
+    nb_repeats = pcm_levels//(2*p.shape[2])
+    p3 = tf.reshape(tf.repeat(tf.concat([1-p2, p2], 3), nb_repeats), (-1, samples, pcm_levels))
+    return p3
+
+def tree_to_pdf(p, samples):
+    return interleave(p[:,:,1:2], samples) * interleave(p[:,:,2:4], samples) * interleave(p[:,:,4:8], samples) * interleave(p[:,:,8:16], samples) \
+         * interleave(p[:,:,16:32], samples) * interleave(p[:,:,32:64], samples) * interleave(p[:,:,64:128], samples) * interleave(p[:,:,128:256], samples)
+
+def tree_to_pdf_train(p):
+    #FIXME: try not to hardcode the 2400 samples (15 frames * 160 samples/frame)
+    return tree_to_pdf(p, 2400)
+
+def tree_to_pdf_infer(p):
+    return tree_to_pdf(p, 1)
+
+def quant_regularizer(x):
+    Q = 128
+    Q_1 = 1./Q
+    #return .01 * tf.reduce_mean(1 - tf.math.cos(2*3.1415926535897931*(Q*x-tf.round(Q*x))))
+    return .01 * tf.reduce_mean(K.sqrt(K.sqrt(1.0001 - tf.math.cos(2*3.1415926535897931*(Q*x-tf.round(Q*x))))))
+
+class Sparsify(Callback):
+    def __init__(self, t_start, t_end, interval, density, quantize=False):
+        super(Sparsify, self).__init__()
+        self.batch = 0
+        self.t_start = t_start
+        self.t_end = t_end
+        self.interval = interval
+        self.final_density = density
+        self.quantize = quantize
+
+    def on_batch_end(self, batch, logs=None):
+        #print("batch number", self.batch)
+        self.batch += 1
+        if self.quantize or (self.batch > self.t_start and (self.batch-self.t_start) % self.interval == 0) or self.batch >= self.t_end:
+            #print("constrain");
+            layer = self.model.get_layer('gru_a')
+            w = layer.get_weights()
+            p = w[1]
+            nb = p.shape[1]//p.shape[0]
+            N = p.shape[0]
+            #print("nb = ", nb, ", N = ", N);
+            #print(p.shape)
+            #print ("density = ", density)
+            for k in range(nb):
+                density = self.final_density[k]
+                if self.batch < self.t_end and not self.quantize:
+                    r = 1 - (self.batch-self.t_start)/(self.t_end - self.t_start)
+                    density = 1 - (1-self.final_density[k])*(1 - r*r*r)
+                A = p[:, k*N:(k+1)*N]
+                A = A - np.diag(np.diag(A))
+                #This is needed because of the CuDNNGRU strange weight ordering
+                A = np.transpose(A, (1, 0))
+                L=np.reshape(A, (N//4, 4, N//8, 8))
+                S=np.sum(L*L, axis=-1)
+                S=np.sum(S, axis=1)
+                SS=np.sort(np.reshape(S, (-1,)))
+                thresh = SS[round(N*N//32*(1-density))]
+                mask = (S>=thresh).astype('float32')
+                mask = np.repeat(mask, 4, axis=0)
+                mask = np.repeat(mask, 8, axis=1)
+                mask = np.minimum(1, mask + np.diag(np.ones((N,))))
+                #This is needed because of the CuDNNGRU strange weight ordering
+                mask = np.transpose(mask, (1, 0))
+                p[:, k*N:(k+1)*N] = p[:, k*N:(k+1)*N]*mask
+                #print(thresh, np.mean(mask))
+            if self.quantize and ((self.batch > self.t_start and (self.batch-self.t_start) % self.interval == 0) or self.batch >= self.t_end):
+                if self.batch < self.t_end:
+                    threshold = .5*(self.batch - self.t_start)/(self.t_end - self.t_start)
+                else:
+                    threshold = .5
+                quant = np.round(p*128.)
+                res = p*128.-quant
+                mask = (np.abs(res) <= threshold).astype('float32')
+                p = mask/128.*quant + (1-mask)*p
+
+            w[1] = p
+            layer.set_weights(w)
+
+class SparsifyGRUB(Callback):
+    def __init__(self, t_start, t_end, interval, grua_units, density, quantize=False):
+        super(SparsifyGRUB, self).__init__()
+        self.batch = 0
+        self.t_start = t_start
+        self.t_end = t_end
+        self.interval = interval
+        self.final_density = density
+        self.grua_units = grua_units
+        self.quantize = quantize
+
+    def on_batch_end(self, batch, logs=None):
+        #print("batch number", self.batch)
+        self.batch += 1
+        if self.quantize or (self.batch > self.t_start and (self.batch-self.t_start) % self.interval == 0) or self.batch >= self.t_end:
+            #print("constrain");
+            layer = self.model.get_layer('gru_b')
+            w = layer.get_weights()
+            p = w[0]
+            N = p.shape[0]
+            M = p.shape[1]//3
+            for k in range(3):
+                density = self.final_density[k]
+                if self.batch < self.t_end and not self.quantize:
+                    r = 1 - (self.batch-self.t_start)/(self.t_end - self.t_start)
+                    density = 1 - (1-self.final_density[k])*(1 - r*r*r)
+                A = p[:, k*M:(k+1)*M]
+                #This is needed because of the CuDNNGRU strange weight ordering
+                A = np.reshape(A, (M, N))
+                A = np.transpose(A, (1, 0))
+                N2 = self.grua_units
+                A2 = A[:N2, :]
+                L=np.reshape(A2, (N2//4, 4, M//8, 8))
+                S=np.sum(L*L, axis=-1)
+                S=np.sum(S, axis=1)
+                SS=np.sort(np.reshape(S, (-1,)))
+                thresh = SS[round(M*N2//32*(1-density))]
+                mask = (S>=thresh).astype('float32')
+                mask = np.repeat(mask, 4, axis=0)
+                mask = np.repeat(mask, 8, axis=1)
+                A = np.concatenate([A2*mask, A[N2:,:]], axis=0)
+                #This is needed because of the CuDNNGRU strange weight ordering
+                A = np.transpose(A, (1, 0))
+                A = np.reshape(A, (N, M))
+                p[:, k*M:(k+1)*M] = A
+                #print(thresh, np.mean(mask))
+            if self.quantize and ((self.batch > self.t_start and (self.batch-self.t_start) % self.interval == 0) or self.batch >= self.t_end):
+                if self.batch < self.t_end:
+                    threshold = .5*(self.batch - self.t_start)/(self.t_end - self.t_start)
+                else:
+                    threshold = .5
+                quant = np.round(p*128.)
+                res = p*128.-quant
+                mask = (np.abs(res) <= threshold).astype('float32')
+                p = mask/128.*quant + (1-mask)*p
+
+            w[0] = p
+            layer.set_weights(w)
+
+
+class PCMInit(Initializer):
+    def __init__(self, gain=.1, seed=None):
+        self.gain = gain
+        self.seed = seed
+
+    def __call__(self, shape, dtype=None):
+        num_rows = 1
+        for dim in shape[:-1]:
+            num_rows *= dim
+        num_cols = shape[-1]
+        flat_shape = (num_rows, num_cols)
+        if self.seed is not None:
+            np.random.seed(self.seed)
+        a = np.random.uniform(-1.7321, 1.7321, flat_shape)
+        #a[:,0] = math.sqrt(12)*np.arange(-.5*num_rows+.5,.5*num_rows-.4)/num_rows
+        #a[:,1] = .5*a[:,0]*a[:,0]*a[:,0]
+        a = a + np.reshape(math.sqrt(12)*np.arange(-.5*num_rows+.5,.5*num_rows-.4)/num_rows, (num_rows, 1))
+        return self.gain * a.astype("float32")
+
+    def get_config(self):
+        return {
+            'gain': self.gain,
+            'seed': self.seed
+        }
+
+class WeightClip(Constraint):
+    '''Clips the weights incident to each hidden unit to be inside a range
+    '''
+    def __init__(self, c=2):
+        self.c = c
+
+    def __call__(self, p):
+        # Ensure that abs of adjacent weights don't sum to more than 127. Otherwise there's a risk of
+        # saturation when implementing dot products with SSSE3 or AVX2.
+        return self.c*p/tf.maximum(self.c, tf.repeat(tf.abs(p[:, 1::2])+tf.abs(p[:, 0::2]), 2, axis=1))
+        #return K.clip(p, -self.c, self.c)
+
+    def get_config(self):
+        return {'name': self.__class__.__name__,
+            'c': self.c}
+
+constraint = WeightClip(0.992)
+
+def new_lpcnet_model(rnn_units1=384, rnn_units2=16, nb_used_features=20, batch_size=128, training=False, adaptation=False, quantize=False, flag_e2e = False, cond_size=128, lpc_order=16, lpc_gamma=1., lookahead=2):
+    pcm = Input(shape=(None, 1), batch_size=batch_size)
+    dpcm = Input(shape=(None, 3), batch_size=batch_size)
+    feat = Input(shape=(None, nb_used_features), batch_size=batch_size)
+    pitch = Input(shape=(None, 1), batch_size=batch_size)
+    dec_feat = Input(shape=(None, cond_size))
+    dec_state1 = Input(shape=(rnn_units1,))
+    dec_state2 = Input(shape=(rnn_units2,))
+
+    padding = 'valid' if training else 'same'
+    fconv1 = Conv1D(cond_size, 3, padding=padding, activation='tanh', name='feature_conv1')
+    fconv2 = Conv1D(cond_size, 3, padding=padding, activation='tanh', name='feature_conv2')
+    pembed = Embedding(256, 64, name='embed_pitch')
+    cat_feat = Concatenate()([feat, Reshape((-1, 64))(pembed(pitch))])
+
+    cfeat = fconv2(fconv1(cat_feat))
+
+    fdense1 = Dense(cond_size, activation='tanh', name='feature_dense1')
+    fdense2 = Dense(cond_size, activation='tanh', name='feature_dense2')
+
+    if flag_e2e and quantize:
+        fconv1.trainable = False
+        fconv2.trainable = False
+        fdense1.trainable = False
+        fdense2.trainable = False
+
+    cfeat = fdense2(fdense1(cfeat))
+
+    error_calc = Lambda(lambda x: tf_l2u(x[0] - tf.roll(x[1],1,axis = 1)))
+    if flag_e2e:
+        lpcoeffs = diff_rc2lpc(name = "rc2lpc")(cfeat)
+    else:
+        lpcoeffs = Input(shape=(None, lpc_order), batch_size=batch_size)
+
+    real_preds = diff_pred(name = "real_lpc2preds")([pcm,lpcoeffs])
+    weighting = lpc_gamma ** np.arange(1, 17).astype('float32')
+    weighted_lpcoeffs = Lambda(lambda x: x[0]*x[1])([lpcoeffs, weighting])
+    tensor_preds = diff_pred(name = "lpc2preds")([pcm,weighted_lpcoeffs])
+    past_errors = error_calc([pcm,tensor_preds])
+
+    embed = diff_Embed(name='embed_sig',initializer = PCMInit())
+    cpcm = Concatenate()([tf_l2u(pcm),tf_l2u(tensor_preds),past_errors])
+    cpcm = GaussianNoise(.3)(cpcm)
+    cpcm = Reshape((-1, embed_size*3))(embed(cpcm))
+    cpcm_decoder = Reshape((-1, embed_size*3))(embed(dpcm))
+
+
+    rep = Lambda(lambda x: K.repeat_elements(x, frame_size, 1))
+
+    quant = quant_regularizer if quantize else None
+
+    if training:
+        rnn = CuDNNGRU(rnn_units1, return_sequences=True, return_state=True, name='gru_a', stateful=True,
+              recurrent_constraint = constraint, recurrent_regularizer=quant)
+        rnn2 = CuDNNGRU(rnn_units2, return_sequences=True, return_state=True, name='gru_b', stateful=True,
+               kernel_constraint=constraint, recurrent_constraint = constraint, kernel_regularizer=quant, recurrent_regularizer=quant)
+    else:
+        rnn = GRU(rnn_units1, return_sequences=True, return_state=True, recurrent_activation="sigmoid", reset_after='true', name='gru_a', stateful=True,
+              recurrent_constraint = constraint, recurrent_regularizer=quant)
+        rnn2 = GRU(rnn_units2, return_sequences=True, return_state=True, recurrent_activation="sigmoid", reset_after='true', name='gru_b', stateful=True,
+               kernel_constraint=constraint, recurrent_constraint = constraint, kernel_regularizer=quant, recurrent_regularizer=quant)
+
+    rnn_in = Concatenate()([cpcm, rep(cfeat)])
+    md = MDense(pcm_levels, activation='sigmoid', name='dual_fc')
+    gru_out1, _ = rnn(rnn_in)
+    gru_out1 = GaussianNoise(.005)(gru_out1)
+    gru_out2, _ = rnn2(Concatenate()([gru_out1, rep(cfeat)]))
+    ulaw_prob = Lambda(tree_to_pdf_train)(md(gru_out2))
+
+    if adaptation:
+        rnn.trainable=False
+        rnn2.trainable=False
+        md.trainable=False
+        embed.Trainable=False
+
+    m_out = Concatenate(name='pdf')([tensor_preds,real_preds,ulaw_prob])
+    if not flag_e2e:
+        model = Model([pcm, feat, pitch, lpcoeffs], m_out)
+    else:
+        model = Model([pcm, feat, pitch], [m_out, cfeat])
+    model.rnn_units1 = rnn_units1
+    model.rnn_units2 = rnn_units2
+    model.nb_used_features = nb_used_features
+    model.frame_size = frame_size
+
+    if not flag_e2e:
+        encoder = Model([feat, pitch], cfeat)
+        dec_rnn_in = Concatenate()([cpcm_decoder, dec_feat])
+    else:
+        encoder = Model([feat, pitch], [cfeat,lpcoeffs])
+        dec_rnn_in = Concatenate()([cpcm_decoder, dec_feat])
+    dec_gru_out1, state1 = rnn(dec_rnn_in, initial_state=dec_state1)
+    dec_gru_out2, state2 = rnn2(Concatenate()([dec_gru_out1, dec_feat]), initial_state=dec_state2)
+    dec_ulaw_prob = Lambda(tree_to_pdf_infer)(md(dec_gru_out2))
+
+    if flag_e2e:
+        decoder = Model([dpcm, dec_feat, dec_state1, dec_state2], [dec_ulaw_prob, state1, state2])
+    else:
+        decoder = Model([dpcm, dec_feat, dec_state1, dec_state2], [dec_ulaw_prob, state1, state2])
+
+    # add parameters to model
+    set_parameter(model, 'lpc_gamma', lpc_gamma, dtype='float64')
+    set_parameter(model, 'flag_e2e', flag_e2e, dtype='bool')
+    set_parameter(model, 'lookahead', lookahead, dtype='int32')
+
+    return model, encoder, decoder