@scrypted/libav 1.0.170 → 1.0.171

package/opus/dnn/training_tf2/keraslayerdump.py

@@ -1,189 +0,0 @@
-'''Copyright (c) 2017-2018 Mozilla
-
-   Redistribution and use in source and binary forms, with or without
-   modification, are permitted provided that the following conditions
-   are met:
-
-   - Redistributions of source code must retain the above copyright
-   notice, this list of conditions and the following disclaimer.
-
-   - Redistributions in binary form must reproduce the above copyright
-   notice, this list of conditions and the following disclaimer in the
-   documentation and/or other materials provided with the distribution.
-
-   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-   ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-   A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR
-   CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
-   EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-   PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
-   PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
-   LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
-   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-'''
-
-""" helper functions for dumping some Keras layers to C files """
-
-import numpy as np
-
-
-def printVector(f, vector, name, dtype='float', dotp=False, static=True):
-    """ prints vector as one-dimensional C array """
-    if dotp:
-        vector = vector.reshape((vector.shape[0]//4, 4, vector.shape[1]//8, 8))
-        vector = vector.transpose((2, 0, 3, 1))
-    v = np.reshape(vector, (-1))
-    if static:
-        f.write('static const {} {}[{}] = {{\n   '.format(dtype, name, len(v)))
-    else:
-        f.write('const {} {}[{}] = {{\n   '.format(dtype, name, len(v)))
-    for i in range(0, len(v)):
-        f.write('{}'.format(v[i]))
-        if (i!=len(v)-1):
-            f.write(',')
-        else:
-            break;
-        if (i%8==7):
-            f.write("\n   ")
-        else:
-            f.write(" ")
-    f.write('\n};\n\n')
-    return vector
-
-def printSparseVector(f, A, name, have_diag=True):
-    N = A.shape[0]
-    M = A.shape[1]
-    W = np.zeros((0,), dtype='int')
-    W0 = np.zeros((0,))
-    if have_diag:
-        diag = np.concatenate([np.diag(A[:,:N]), np.diag(A[:,N:2*N]), np.diag(A[:,2*N:])])
-        A[:,:N] = A[:,:N] - np.diag(np.diag(A[:,:N]))
-        A[:,N:2*N] = A[:,N:2*N] - np.diag(np.diag(A[:,N:2*N]))
-        A[:,2*N:] = A[:,2*N:] - np.diag(np.diag(A[:,2*N:]))
-        printVector(f, diag, name + '_diag')
-    AQ = np.minimum(127, np.maximum(-128, np.round(A*128))).astype('int')
-    idx = np.zeros((0,), dtype='int')
-    for i in range(M//8):
-        pos = idx.shape[0]
-        idx = np.append(idx, -1)
-        nb_nonzero = 0
-        for j in range(N//4):
-            block = A[j*4:(j+1)*4, i*8:(i+1)*8]
-            qblock = AQ[j*4:(j+1)*4, i*8:(i+1)*8]
-            if np.sum(np.abs(block)) > 1e-10:
-                nb_nonzero = nb_nonzero + 1
-                idx = np.append(idx, j*4)
-                vblock = qblock.transpose((1,0)).reshape((-1,))
-                W0 = np.concatenate([W0, block.reshape((-1,))])
-                W = np.concatenate([W, vblock])
-        idx[pos] = nb_nonzero
-    f.write('#ifdef DOT_PROD\n')
-    printVector(f, W, name, dtype='qweight')
-    f.write('#else /*DOT_PROD*/\n')
-    printVector(f, W0, name, dtype='qweight')
-    f.write('#endif /*DOT_PROD*/\n')
-    printVector(f, idx, name + '_idx', dtype='int')
-    return AQ
-
-def dump_sparse_gru(self, f, hf):
-    name = 'sparse_' + self.name
-    print("printing layer " + name + " of type sparse " + self.__class__.__name__)
-    weights = self.get_weights()
-    qweights = printSparseVector(f, weights[1], name + '_recurrent_weights')
-    printVector(f, weights[-1], name + '_bias')
-    subias = weights[-1].copy()
-    subias[1,:] = subias[1,:] - np.sum(qweights*(1./128),axis=0)
-    printVector(f, subias, name + '_subias')
-    if hasattr(self, 'activation'):
-        activation = self.activation.__name__.upper()
-    else:
-        activation = 'TANH'
-    if hasattr(self, 'reset_after') and not self.reset_after:
-        reset_after = 0
-    else:
-        reset_after = 1
-    neurons = weights[0].shape[1]//3
-    max_rnn_neurons = neurons
-    f.write('const SparseGRULayer {} = {{\n   {}_bias,\n   {}_subias,\n   {}_recurrent_weights_diag,\n   {}_recurrent_weights,\n   {}_recurrent_weights_idx,\n   {}, ACTIVATION_{}, {}\n}};\n\n'
-            .format(name, name, name, name, name, name, weights[0].shape[1]//3, activation, reset_after))
-    hf.write('#define {}_OUT_SIZE {}\n'.format(name.upper(), weights[0].shape[1]//3))
-    hf.write('#define {}_STATE_SIZE {}\n'.format(name.upper(), weights[0].shape[1]//3))
-    hf.write('extern const SparseGRULayer {};\n\n'.format(name));
-    return max_rnn_neurons
-
-def dump_gru_layer(self, f, hf, dotp=False, sparse=False):
-    name = self.name
-    print("printing layer " + name + " of type " + self.__class__.__name__)
-    weights = self.get_weights()
-    if sparse:
-        qweight = printSparseVector(f, weights[0], name + '_weights', have_diag=False)
-    else:
-        qweight = printVector(f, weights[0], name + '_weights')
-
-    if dotp:
-        f.write('#ifdef DOT_PROD\n')
-        qweight2 = np.clip(np.round(128.*weights[1]).astype('int'), -128, 127)
-        printVector(f, qweight2, name + '_recurrent_weights', dotp=True, dtype='qweight')
-        f.write('#else /*DOT_PROD*/\n')
-    else:
-        qweight2 = weights[1]
-
-    printVector(f, weights[1], name + '_recurrent_weights')
-    if dotp:
-        f.write('#endif /*DOT_PROD*/\n')
-
-    printVector(f, weights[-1], name + '_bias')
-    subias = weights[-1].copy()
-    subias[0,:] = subias[0,:] - np.sum(qweight*(1./128.),axis=0)
-    subias[1,:] = subias[1,:] - np.sum(qweight2*(1./128.),axis=0)
-    printVector(f, subias, name + '_subias')
-    if hasattr(self, 'activation'):
-        activation = self.activation.__name__.upper()
-    else:
-        activation = 'TANH'
-    if hasattr(self, 'reset_after') and not self.reset_after:
-        reset_after = 0
-    else:
-        reset_after = 1
-    neurons = weights[0].shape[1]//3
-    max_rnn_neurons = neurons
-    f.write('const GRULayer {} = {{\n   {}_bias,\n   {}_subias,\n   {}_weights,\n   {},\n   {}_recurrent_weights,\n   {}, {}, ACTIVATION_{}, {}\n}};\n\n'
-            .format(name, name, name, name, name + "_weights_idx" if sparse else "NULL", name, weights[0].shape[0], weights[0].shape[1]//3, activation, reset_after))
-    hf.write('#define {}_OUT_SIZE {}\n'.format(name.upper(), weights[0].shape[1]//3))
-    hf.write('#define {}_STATE_SIZE {}\n'.format(name.upper(), weights[0].shape[1]//3))
-    hf.write('extern const GRULayer {};\n\n'.format(name));
-    return max_rnn_neurons
-
-def dump_dense_layer_impl(name, weights, bias, activation, f, hf):
-    printVector(f, weights, name + '_weights')
-    printVector(f, bias, name + '_bias')
-    f.write('const DenseLayer {} = {{\n   {}_bias,\n   {}_weights,\n   {}, {}, ACTIVATION_{}\n}};\n\n'
-            .format(name, name, name, weights.shape[0], weights.shape[1], activation))
-    hf.write('#define {}_OUT_SIZE {}\n'.format(name.upper(), weights.shape[1]))
-    hf.write('extern const DenseLayer {};\n\n'.format(name));
-
-def dump_dense_layer(self, f, hf):
-    name = self.name
-    print("printing layer " + name + " of type " + self.__class__.__name__)
-    weights = self.get_weights()
-    activation = self.activation.__name__.upper()
-    dump_dense_layer_impl(name, weights[0], weights[1], activation, f, hf)
-    return False
-
-def dump_conv1d_layer(self, f, hf):
-    name = self.name
-    print("printing layer " + name + " of type " + self.__class__.__name__)
-    weights = self.get_weights()
-    printVector(f, weights[0], name + '_weights')
-    printVector(f, weights[-1], name + '_bias')
-    activation = self.activation.__name__.upper()
-    max_conv_inputs = weights[0].shape[1]*weights[0].shape[0]
-    f.write('const Conv1DLayer {} = {{\n   {}_bias,\n   {}_weights,\n   {}, {}, {}, ACTIVATION_{}\n}};\n\n'
-            .format(name, name, name, weights[0].shape[1], weights[0].shape[0], weights[0].shape[2], activation))
-    hf.write('#define {}_OUT_SIZE {}\n'.format(name.upper(), weights[0].shape[2]))
-    hf.write('#define {}_STATE_SIZE ({}*{})\n'.format(name.upper(), weights[0].shape[1], (weights[0].shape[0]-1)))
-    hf.write('#define {}_DELAY {}\n'.format(name.upper(), (weights[0].shape[0]-1)//2))
-    hf.write('extern const Conv1DLayer {};\n\n'.format(name));
-    return max_conv_inputs

package/opus/dnn/training_tf2/lossfuncs.py

@@ -1,99 +0,0 @@
-"""
-Custom Loss functions and metrics for training/analysis
-"""
-
-from tf_funcs import *
-import tensorflow as tf
-
-# The following loss functions all expect the lpcnet model to output the lpc prediction
-
-# Computing the excitation by subtracting the lpc prediction from the target, followed by minimizing the cross entropy
-def res_from_sigloss():
-    def loss(y_true,y_pred):
-        p = y_pred[:,:,0:1]
-        model_out = y_pred[:,:,2:]
-        e_gt = tf_l2u(y_true - p)
-        e_gt = tf.round(e_gt)
-        e_gt = tf.cast(e_gt,'int32')
-        sparse_cel = tf.keras.losses.SparseCategoricalCrossentropy(reduction=tf.keras.losses.Reduction.NONE)(e_gt,model_out)
-        return sparse_cel
-    return loss
-
-# Interpolated and Compensated Loss (In case of end to end lpcnet)
-# Interpolates between adjacent embeddings based on the fractional value of the excitation computed (similar to the embedding interpolation)
-# Also adds a probability compensation (to account for matching cross entropy in the linear domain), weighted by gamma
-def interp_mulaw(gamma = 1):
-    def loss(y_true,y_pred):
-        y_true = tf.cast(y_true, 'float32')
-        p = y_pred[:,:,0:1]
-        real_p = y_pred[:,:,1:2]
-        model_out = y_pred[:,:,2:]
-        e_gt = tf_l2u(y_true - p)
-        exc_gt = tf_l2u(y_true - real_p)
-        prob_compensation = tf.squeeze((K.abs(e_gt - 128)/128.0)*K.log(256.0))
-        regularization = tf.squeeze((K.abs(exc_gt - 128)/128.0)*K.log(256.0))
-        alpha = e_gt - tf.math.floor(e_gt)
-        alpha = tf.tile(alpha,[1,1,256])
-        e_gt = tf.cast(e_gt,'int32')
-        e_gt = tf.clip_by_value(e_gt,0,254)
-        interp_probab = (1 - alpha)*model_out + alpha*tf.roll(model_out,shift = -1,axis = -1)
-        sparse_cel = tf.keras.losses.SparseCategoricalCrossentropy(reduction=tf.keras.losses.Reduction.NONE)(e_gt,interp_probab)
-        loss_mod = sparse_cel + prob_compensation + gamma*regularization
-        return loss_mod
-    return loss
-
-# Same as above, except a metric
-def metric_oginterploss(y_true,y_pred):
-    p = y_pred[:,:,0:1]
-    model_out = y_pred[:,:,2:]
-    e_gt = tf_l2u(y_true - p)
-    prob_compensation = tf.squeeze((K.abs(e_gt - 128)/128.0)*K.log(256.0))
-    alpha = e_gt - tf.math.floor(e_gt)
-    alpha = tf.tile(alpha,[1,1,256])
-    e_gt = tf.cast(e_gt,'int32')
-    e_gt = tf.clip_by_value(e_gt,0,254)
-    interp_probab = (1 - alpha)*model_out + alpha*tf.roll(model_out,shift = -1,axis = -1)
-    sparse_cel = tf.keras.losses.SparseCategoricalCrossentropy(reduction=tf.keras.losses.Reduction.NONE)(e_gt,interp_probab)
-    loss_mod = sparse_cel + prob_compensation
-    return loss_mod
-
-# Interpolated cross entropy loss metric
-def metric_icel(y_true, y_pred):
-    p = y_pred[:,:,0:1]
-    model_out = y_pred[:,:,2:]
-    e_gt = tf_l2u(y_true - p)
-    alpha = e_gt - tf.math.floor(e_gt)
-    alpha = tf.tile(alpha,[1,1,256])
-    e_gt = tf.cast(e_gt,'int32')
-    e_gt = tf.clip_by_value(e_gt,0,254) #Check direction
-    interp_probab = (1 - alpha)*model_out + alpha*tf.roll(model_out,shift = -1,axis = -1)
-    sparse_cel = tf.keras.losses.SparseCategoricalCrossentropy(reduction=tf.keras.losses.Reduction.NONE)(e_gt,interp_probab)
-    return sparse_cel
-
-# Non-interpolated (rounded) cross entropy loss metric
-def metric_cel(y_true, y_pred):
-    y_true = tf.cast(y_true, 'float32')
-    p = y_pred[:,:,0:1]
-    model_out = y_pred[:,:,2:]
-    e_gt = tf_l2u(y_true - p)
-    e_gt = tf.round(e_gt)
-    e_gt = tf.cast(e_gt,'int32')
-    e_gt = tf.clip_by_value(e_gt,0,255)
-    sparse_cel = tf.keras.losses.SparseCategoricalCrossentropy(reduction=tf.keras.losses.Reduction.NONE)(e_gt,model_out)
-    return sparse_cel
-
-# Variance metric of the output excitation
-def metric_exc_sd(y_true,y_pred):
-    p = y_pred[:,:,0:1]
-    e_gt = tf_l2u(y_true - p)
-    sd_egt = tf.keras.losses.MeanSquaredError(reduction=tf.keras.losses.Reduction.NONE)(e_gt,128)
-    return sd_egt
-
-def loss_matchlar():
-    def loss(y_true,y_pred):
-        model_rc = y_pred[:,:,:16]
-        #y_true = lpc2rc(y_true)
-        loss_lar_diff = K.log((1.01 + model_rc)/(1.01 - model_rc)) - K.log((1.01 + y_true)/(1.01 - y_true))
-        loss_lar_diff = tf.square(loss_lar_diff)
-        return tf.reduce_mean(loss_lar_diff, axis=-1)
-    return loss

package/opus/dnn/training_tf2/lpcnet.py

@@ -1,339 +0,0 @@
-#!/usr/bin/python3
-'''Copyright (c) 2018 Mozilla
-
-   Redistribution and use in source and binary forms, with or without
-   modification, are permitted provided that the following conditions
-   are met:
-
-   - Redistributions of source code must retain the above copyright
-   notice, this list of conditions and the following disclaimer.
-
-   - Redistributions in binary form must reproduce the above copyright
-   notice, this list of conditions and the following disclaimer in the
-   documentation and/or other materials provided with the distribution.
-
-   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-   ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-   A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR
-   CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
-   EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-   PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
-   PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
-   LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
-   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-'''
-
-import math
-import tensorflow as tf
-from tensorflow.keras.models import Model
-from tensorflow.keras.layers import Input, GRU, Dense, Embedding, Reshape, Concatenate, Lambda, Conv1D, Multiply, Add, Bidirectional, MaxPooling1D, Activation, GaussianNoise
-from tensorflow.compat.v1.keras.layers import CuDNNGRU
-from tensorflow.keras import backend as K
-from tensorflow.keras.constraints import Constraint
-from tensorflow.keras.initializers import Initializer
-from tensorflow.keras.callbacks import Callback
-from mdense import MDense
-import numpy as np
-import h5py
-import sys
-from tf_funcs import *
-from diffembed import diff_Embed
-from parameters import set_parameter
-
-frame_size = 160
-pcm_bits = 8
-embed_size = 128
-pcm_levels = 2**pcm_bits
-
-def interleave(p, samples):
-    p2=tf.expand_dims(p, 3)
-    nb_repeats = pcm_levels//(2*p.shape[2])
-    p3 = tf.reshape(tf.repeat(tf.concat([1-p2, p2], 3), nb_repeats), (-1, samples, pcm_levels))
-    return p3
-
-def tree_to_pdf(p, samples):
-    return interleave(p[:,:,1:2], samples) * interleave(p[:,:,2:4], samples) * interleave(p[:,:,4:8], samples) * interleave(p[:,:,8:16], samples) \
-         * interleave(p[:,:,16:32], samples) * interleave(p[:,:,32:64], samples) * interleave(p[:,:,64:128], samples) * interleave(p[:,:,128:256], samples)
-
-def tree_to_pdf_train(p):
-    #FIXME: try not to hardcode the 2400 samples (15 frames * 160 samples/frame)
-    return tree_to_pdf(p, 2400)
-
-def tree_to_pdf_infer(p):
-    return tree_to_pdf(p, 1)
-
-def quant_regularizer(x):
-    Q = 128
-    Q_1 = 1./Q
-    #return .01 * tf.reduce_mean(1 - tf.math.cos(2*3.1415926535897931*(Q*x-tf.round(Q*x))))
-    return .01 * tf.reduce_mean(K.sqrt(K.sqrt(1.0001 - tf.math.cos(2*3.1415926535897931*(Q*x-tf.round(Q*x))))))
-
-class Sparsify(Callback):
-    def __init__(self, t_start, t_end, interval, density, quantize=False):
-        super(Sparsify, self).__init__()
-        self.batch = 0
-        self.t_start = t_start
-        self.t_end = t_end
-        self.interval = interval
-        self.final_density = density
-        self.quantize = quantize
-
-    def on_batch_end(self, batch, logs=None):
-        #print("batch number", self.batch)
-        self.batch += 1
-        if self.quantize or (self.batch > self.t_start and (self.batch-self.t_start) % self.interval == 0) or self.batch >= self.t_end:
-            #print("constrain");
-            layer = self.model.get_layer('gru_a')
-            w = layer.get_weights()
-            p = w[1]
-            nb = p.shape[1]//p.shape[0]
-            N = p.shape[0]
-            #print("nb = ", nb, ", N = ", N);
-            #print(p.shape)
-            #print ("density = ", density)
-            for k in range(nb):
-                density = self.final_density[k]
-                if self.batch < self.t_end and not self.quantize:
-                    r = 1 - (self.batch-self.t_start)/(self.t_end - self.t_start)
-                    density = 1 - (1-self.final_density[k])*(1 - r*r*r)
-                A = p[:, k*N:(k+1)*N]
-                A = A - np.diag(np.diag(A))
-                #This is needed because of the CuDNNGRU strange weight ordering
-                A = np.transpose(A, (1, 0))
-                L=np.reshape(A, (N//4, 4, N//8, 8))
-                S=np.sum(L*L, axis=-1)
-                S=np.sum(S, axis=1)
-                SS=np.sort(np.reshape(S, (-1,)))
-                thresh = SS[round(N*N//32*(1-density))]
-                mask = (S>=thresh).astype('float32')
-                mask = np.repeat(mask, 4, axis=0)
-                mask = np.repeat(mask, 8, axis=1)
-                mask = np.minimum(1, mask + np.diag(np.ones((N,))))
-                #This is needed because of the CuDNNGRU strange weight ordering
-                mask = np.transpose(mask, (1, 0))
-                p[:, k*N:(k+1)*N] = p[:, k*N:(k+1)*N]*mask
-                #print(thresh, np.mean(mask))
-            if self.quantize and ((self.batch > self.t_start and (self.batch-self.t_start) % self.interval == 0) or self.batch >= self.t_end):
-                if self.batch < self.t_end:
-                    threshold = .5*(self.batch - self.t_start)/(self.t_end - self.t_start)
-                else:
-                    threshold = .5
-                quant = np.round(p*128.)
-                res = p*128.-quant
-                mask = (np.abs(res) <= threshold).astype('float32')
-                p = mask/128.*quant + (1-mask)*p
-
-            w[1] = p
-            layer.set_weights(w)
-
-class SparsifyGRUB(Callback):
-    def __init__(self, t_start, t_end, interval, grua_units, density, quantize=False):
-        super(SparsifyGRUB, self).__init__()
-        self.batch = 0
-        self.t_start = t_start
-        self.t_end = t_end
-        self.interval = interval
-        self.final_density = density
-        self.grua_units = grua_units
-        self.quantize = quantize
-
-    def on_batch_end(self, batch, logs=None):
-        #print("batch number", self.batch)
-        self.batch += 1
-        if self.quantize or (self.batch > self.t_start and (self.batch-self.t_start) % self.interval == 0) or self.batch >= self.t_end:
-            #print("constrain");
-            layer = self.model.get_layer('gru_b')
-            w = layer.get_weights()
-            p = w[0]
-            N = p.shape[0]
-            M = p.shape[1]//3
-            for k in range(3):
-                density = self.final_density[k]
-                if self.batch < self.t_end and not self.quantize:
-                    r = 1 - (self.batch-self.t_start)/(self.t_end - self.t_start)
-                    density = 1 - (1-self.final_density[k])*(1 - r*r*r)
-                A = p[:, k*M:(k+1)*M]
-                #This is needed because of the CuDNNGRU strange weight ordering
-                A = np.reshape(A, (M, N))
-                A = np.transpose(A, (1, 0))
-                N2 = self.grua_units
-                A2 = A[:N2, :]
-                L=np.reshape(A2, (N2//4, 4, M//8, 8))
-                S=np.sum(L*L, axis=-1)
-                S=np.sum(S, axis=1)
-                SS=np.sort(np.reshape(S, (-1,)))
-                thresh = SS[round(M*N2//32*(1-density))]
-                mask = (S>=thresh).astype('float32')
-                mask = np.repeat(mask, 4, axis=0)
-                mask = np.repeat(mask, 8, axis=1)
-                A = np.concatenate([A2*mask, A[N2:,:]], axis=0)
-                #This is needed because of the CuDNNGRU strange weight ordering
-                A = np.transpose(A, (1, 0))
-                A = np.reshape(A, (N, M))
-                p[:, k*M:(k+1)*M] = A
-                #print(thresh, np.mean(mask))
-            if self.quantize and ((self.batch > self.t_start and (self.batch-self.t_start) % self.interval == 0) or self.batch >= self.t_end):
-                if self.batch < self.t_end:
-                    threshold = .5*(self.batch - self.t_start)/(self.t_end - self.t_start)
-                else:
-                    threshold = .5
-                quant = np.round(p*128.)
-                res = p*128.-quant
-                mask = (np.abs(res) <= threshold).astype('float32')
-                p = mask/128.*quant + (1-mask)*p
-
-            w[0] = p
-            layer.set_weights(w)
-
-
-class PCMInit(Initializer):
-    def __init__(self, gain=.1, seed=None):
-        self.gain = gain
-        self.seed = seed
-
-    def __call__(self, shape, dtype=None):
-        num_rows = 1
-        for dim in shape[:-1]:
-            num_rows *= dim
-        num_cols = shape[-1]
-        flat_shape = (num_rows, num_cols)
-        if self.seed is not None:
-            np.random.seed(self.seed)
-        a = np.random.uniform(-1.7321, 1.7321, flat_shape)
-        #a[:,0] = math.sqrt(12)*np.arange(-.5*num_rows+.5,.5*num_rows-.4)/num_rows
-        #a[:,1] = .5*a[:,0]*a[:,0]*a[:,0]
-        a = a + np.reshape(math.sqrt(12)*np.arange(-.5*num_rows+.5,.5*num_rows-.4)/num_rows, (num_rows, 1))
-        return self.gain * a.astype("float32")
-
-    def get_config(self):
-        return {
-            'gain': self.gain,
-            'seed': self.seed
-        }
-
-class WeightClip(Constraint):
-    '''Clips the weights incident to each hidden unit to be inside a range
-    '''
-    def __init__(self, c=2):
-        self.c = c
-
-    def __call__(self, p):
-        # Ensure that abs of adjacent weights don't sum to more than 127. Otherwise there's a risk of
-        # saturation when implementing dot products with SSSE3 or AVX2.
-        return self.c*p/tf.maximum(self.c, tf.repeat(tf.abs(p[:, 1::2])+tf.abs(p[:, 0::2]), 2, axis=1))
-        #return K.clip(p, -self.c, self.c)
-
-    def get_config(self):
-        return {'name': self.__class__.__name__,
-            'c': self.c}
-
-constraint = WeightClip(0.992)
-
-def new_lpcnet_model(rnn_units1=384, rnn_units2=16, nb_used_features=20, batch_size=128, training=False, adaptation=False, quantize=False, flag_e2e = False, cond_size=128, lpc_order=16, lpc_gamma=1., lookahead=2):
-    pcm = Input(shape=(None, 1), batch_size=batch_size)
-    dpcm = Input(shape=(None, 3), batch_size=batch_size)
-    feat = Input(shape=(None, nb_used_features), batch_size=batch_size)
-    pitch = Input(shape=(None, 1), batch_size=batch_size)
-    dec_feat = Input(shape=(None, cond_size))
-    dec_state1 = Input(shape=(rnn_units1,))
-    dec_state2 = Input(shape=(rnn_units2,))
-
-    padding = 'valid' if training else 'same'
-    fconv1 = Conv1D(cond_size, 3, padding=padding, activation='tanh', name='feature_conv1')
-    fconv2 = Conv1D(cond_size, 3, padding=padding, activation='tanh', name='feature_conv2')
-    pembed = Embedding(256, 64, name='embed_pitch')
-    cat_feat = Concatenate()([feat, Reshape((-1, 64))(pembed(pitch))])
-
-    cfeat = fconv2(fconv1(cat_feat))
-
-    fdense1 = Dense(cond_size, activation='tanh', name='feature_dense1')
-    fdense2 = Dense(cond_size, activation='tanh', name='feature_dense2')
-
-    if flag_e2e and quantize:
-        fconv1.trainable = False
-        fconv2.trainable = False
-        fdense1.trainable = False
-        fdense2.trainable = False
-
-    cfeat = fdense2(fdense1(cfeat))
-
-    error_calc = Lambda(lambda x: tf_l2u(x[0] - tf.roll(x[1],1,axis = 1)))
-    if flag_e2e:
-        lpcoeffs = diff_rc2lpc(name = "rc2lpc")(cfeat)
-    else:
-        lpcoeffs = Input(shape=(None, lpc_order), batch_size=batch_size)
-
-    real_preds = diff_pred(name = "real_lpc2preds")([pcm,lpcoeffs])
-    weighting = lpc_gamma ** np.arange(1, 17).astype('float32')
-    weighted_lpcoeffs = Lambda(lambda x: x[0]*x[1])([lpcoeffs, weighting])
-    tensor_preds = diff_pred(name = "lpc2preds")([pcm,weighted_lpcoeffs])
-    past_errors = error_calc([pcm,tensor_preds])
-
-    embed = diff_Embed(name='embed_sig',initializer = PCMInit())
-    cpcm = Concatenate()([tf_l2u(pcm),tf_l2u(tensor_preds),past_errors])
-    cpcm = GaussianNoise(.3)(cpcm)
-    cpcm = Reshape((-1, embed_size*3))(embed(cpcm))
-    cpcm_decoder = Reshape((-1, embed_size*3))(embed(dpcm))
-
-
-    rep = Lambda(lambda x: K.repeat_elements(x, frame_size, 1))
-
-    quant = quant_regularizer if quantize else None
-
-    if training:
-        rnn = CuDNNGRU(rnn_units1, return_sequences=True, return_state=True, name='gru_a', stateful=True,
-              recurrent_constraint = constraint, recurrent_regularizer=quant)
-        rnn2 = CuDNNGRU(rnn_units2, return_sequences=True, return_state=True, name='gru_b', stateful=True,
-               kernel_constraint=constraint, recurrent_constraint = constraint, kernel_regularizer=quant, recurrent_regularizer=quant)
-    else:
-        rnn = GRU(rnn_units1, return_sequences=True, return_state=True, recurrent_activation="sigmoid", reset_after='true', name='gru_a', stateful=True,
-              recurrent_constraint = constraint, recurrent_regularizer=quant)
-        rnn2 = GRU(rnn_units2, return_sequences=True, return_state=True, recurrent_activation="sigmoid", reset_after='true', name='gru_b', stateful=True,
-               kernel_constraint=constraint, recurrent_constraint = constraint, kernel_regularizer=quant, recurrent_regularizer=quant)
-
-    rnn_in = Concatenate()([cpcm, rep(cfeat)])
-    md = MDense(pcm_levels, activation='sigmoid', name='dual_fc')
-    gru_out1, _ = rnn(rnn_in)
-    gru_out1 = GaussianNoise(.005)(gru_out1)
-    gru_out2, _ = rnn2(Concatenate()([gru_out1, rep(cfeat)]))
-    ulaw_prob = Lambda(tree_to_pdf_train)(md(gru_out2))
-
-    if adaptation:
-        rnn.trainable=False
-        rnn2.trainable=False
-        md.trainable=False
-        embed.Trainable=False
-
-    m_out = Concatenate(name='pdf')([tensor_preds,real_preds,ulaw_prob])
-    if not flag_e2e:
-        model = Model([pcm, feat, pitch, lpcoeffs], m_out)
-    else:
-        model = Model([pcm, feat, pitch], [m_out, cfeat])
-    model.rnn_units1 = rnn_units1
-    model.rnn_units2 = rnn_units2
-    model.nb_used_features = nb_used_features
-    model.frame_size = frame_size
-
-    if not flag_e2e:
-        encoder = Model([feat, pitch], cfeat)
-        dec_rnn_in = Concatenate()([cpcm_decoder, dec_feat])
-    else:
-        encoder = Model([feat, pitch], [cfeat,lpcoeffs])
-        dec_rnn_in = Concatenate()([cpcm_decoder, dec_feat])
-    dec_gru_out1, state1 = rnn(dec_rnn_in, initial_state=dec_state1)
-    dec_gru_out2, state2 = rnn2(Concatenate()([dec_gru_out1, dec_feat]), initial_state=dec_state2)
-    dec_ulaw_prob = Lambda(tree_to_pdf_infer)(md(dec_gru_out2))
-
-    if flag_e2e:
-        decoder = Model([dpcm, dec_feat, dec_state1, dec_state2], [dec_ulaw_prob, state1, state2])
-    else:
-        decoder = Model([dpcm, dec_feat, dec_state1, dec_state2], [dec_ulaw_prob, state1, state2])
-
-    # add parameters to model
-    set_parameter(model, 'lpc_gamma', lpc_gamma, dtype='float64')
-    set_parameter(model, 'flag_e2e', flag_e2e, dtype='bool')
-    set_parameter(model, 'lookahead', lookahead, dtype='int32')
-
-    return model, encoder, decoder