@scrypted/libav 1.0.166 → 1.0.168

package/opus/dnn/torch/fargan/test_fargan.py

@@ -0,0 +1,128 @@
+import os
+import argparse
+import numpy as np
+
+import torch
+from torch import nn
+import torch.nn.functional as F
+import tqdm
+
+import fargan
+from dataset import FARGANDataset
+
+nb_features = 36
+nb_used_features = 20
+
+parser = argparse.ArgumentParser()
+
+parser.add_argument('model', type=str, help='CELPNet model')
+parser.add_argument('features', type=str, help='path to feature file in .f32 format')
+parser.add_argument('output', type=str, help='path to output file (16-bit PCM)')
+
+parser.add_argument('--cuda-visible-devices', type=str, help="comma separates list of cuda visible device indices, default: CUDA_VISIBLE_DEVICES", default=None)
+
+
+model_group = parser.add_argument_group(title="model parameters")
+model_group.add_argument('--cond-size', type=int, help="first conditioning size, default: 256", default=256)
+
+args = parser.parse_args()
+
+if args.cuda_visible_devices != None:
+    os.environ['CUDA_VISIBLE_DEVICES'] = args.cuda_visible_devices
+
+
+features_file = args.features
+signal_file = args.output
+
+
+
+device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
+
+checkpoint = torch.load(args.model, map_location='cpu')
+
+model = fargan.FARGAN(*checkpoint['model_args'], **checkpoint['model_kwargs'])
+
+
+model.load_state_dict(checkpoint['state_dict'], strict=False)
+
+features = np.reshape(np.memmap(features_file, dtype='float32', mode='r'), (1, -1, nb_features))
+lpc = features[:,4-1:-1,nb_used_features:]
+features = features[:, :, :nb_used_features]
+#periods = np.round(50*features[:,:,nb_used_features-2]+100).astype('int')
+periods = np.round(np.clip(256./2**(features[:,:,nb_used_features-2]+1.5), 32, 255)).astype('int')
+
+
+nb_frames = features.shape[1]
+#nb_frames = 1000
+gamma = checkpoint['model_kwargs']['gamma']
+
+def lpc_synthesis_one_frame(frame, filt, buffer, weighting_vector=np.ones(16)):
+
+    out = np.zeros_like(frame)
+    filt = np.flip(filt)
+
+    inp = frame[:]
+
+
+    for i in range(0, inp.shape[0]):
+
+        s = inp[i] - np.dot(buffer*weighting_vector, filt)
+
+        buffer[0] = s
+
+        buffer = np.roll(buffer, -1)
+
+        out[i] = s
+
+    return out
+
+def inverse_perceptual_weighting (pw_signal, filters, weighting_vector):
+
+    #inverse perceptual weighting= H_preemph / W(z/gamma)
+
+    signal = np.zeros_like(pw_signal)
+    buffer = np.zeros(16)
+    num_frames = pw_signal.shape[0] //160
+    assert num_frames == filters.shape[0]
+    for frame_idx in range(0, num_frames):
+
+        in_frame = pw_signal[frame_idx*160: (frame_idx+1)*160][:]
+        out_sig_frame = lpc_synthesis_one_frame(in_frame, filters[frame_idx, :], buffer, weighting_vector)
+        signal[frame_idx*160: (frame_idx+1)*160] = out_sig_frame[:]
+        buffer[:] = out_sig_frame[-16:]
+    return signal
+
+def inverse_perceptual_weighting40 (pw_signal, filters):
+
+    #inverse perceptual weighting= H_preemph / W(z/gamma)
+
+    signal = np.zeros_like(pw_signal)
+    buffer = np.zeros(16)
+    num_frames = pw_signal.shape[0] //40
+    assert num_frames == filters.shape[0]
+    for frame_idx in range(0, num_frames):
+        in_frame = pw_signal[frame_idx*40: (frame_idx+1)*40][:]
+        out_sig_frame = lpc_synthesis_one_frame(in_frame, filters[frame_idx, :], buffer)
+        signal[frame_idx*40: (frame_idx+1)*40] = out_sig_frame[:]
+        buffer[:] = out_sig_frame[-16:]
+    return signal
+
+from scipy.signal import lfilter
+
+if __name__ == '__main__':
+    model.to(device)
+    features = torch.tensor(features).to(device)
+    #lpc = torch.tensor(lpc).to(device)
+    periods = torch.tensor(periods).to(device)
+    weighting = gamma**np.arange(1, 17)
+    lpc = lpc*weighting
+    lpc = fargan.interp_lpc(torch.tensor(lpc), 4).numpy()
+
+    sig, _ = model(features, periods, nb_frames - 4)
+    #weighting_vector = np.array([gamma**i for i in range(16,0,-1)])
+    sig = sig.detach().numpy().flatten()
+    sig = lfilter(np.array([1.]), np.array([1., -.85]), sig)
+    #sig = inverse_perceptual_weighting40(sig, lpc[0,:,:])
+
+    pcm = np.round(32768*np.clip(sig, a_max=.99, a_min=-.99)).astype('int16')
+    pcm.tofile(signal_file)

package/opus/dnn/torch/fargan/train_fargan.py

@@ -0,0 +1,168 @@
+import os
+import argparse
+import random
+import numpy as np
+
+import torch
+from torch import nn
+import torch.nn.functional as F
+import tqdm
+
+import fargan
+from dataset import FARGANDataset
+from stft_loss import *
+
+parser = argparse.ArgumentParser()
+
+parser.add_argument('features', type=str, help='path to feature file in .f32 format')
+parser.add_argument('signal', type=str, help='path to signal file in .s16 format')
+parser.add_argument('output', type=str, help='path to output folder')
+
+parser.add_argument('--suffix', type=str, help="model name suffix", default="")
+parser.add_argument('--cuda-visible-devices', type=str, help="comma separates list of cuda visible device indices, default: CUDA_VISIBLE_DEVICES", default=None)
+
+
+model_group = parser.add_argument_group(title="model parameters")
+model_group.add_argument('--cond-size', type=int, help="first conditioning size, default: 256", default=256)
+model_group.add_argument('--gamma', type=float, help="Use A(z/gamma), default: 0.9", default=0.9)
+
+training_group = parser.add_argument_group(title="training parameters")
+training_group.add_argument('--batch-size', type=int, help="batch size, default: 512", default=512)
+training_group.add_argument('--lr', type=float, help='learning rate, default: 1e-3', default=1e-3)
+training_group.add_argument('--epochs', type=int, help='number of training epochs, default: 20', default=20)
+training_group.add_argument('--sequence-length', type=int, help='sequence length, default: 15', default=15)
+training_group.add_argument('--lr-decay', type=float, help='learning rate decay factor, default: 1e-4', default=1e-4)
+training_group.add_argument('--initial-checkpoint', type=str, help='initial checkpoint to start training from, default: None', default=None)
+
+args = parser.parse_args()
+
+if args.cuda_visible_devices != None:
+    os.environ['CUDA_VISIBLE_DEVICES'] = args.cuda_visible_devices
+
+# checkpoints
+checkpoint_dir = os.path.join(args.output, 'checkpoints')
+checkpoint = dict()
+os.makedirs(checkpoint_dir, exist_ok=True)
+
+
+# training parameters
+batch_size = args.batch_size
+lr = args.lr
+epochs = args.epochs
+sequence_length = args.sequence_length
+lr_decay = args.lr_decay
+
+adam_betas = [0.8, 0.95]
+adam_eps = 1e-8
+features_file = args.features
+signal_file = args.signal
+
+# model parameters
+cond_size  = args.cond_size
+
+
+checkpoint['batch_size'] = batch_size
+checkpoint['lr'] = lr
+checkpoint['lr_decay'] = lr_decay
+checkpoint['epochs'] = epochs
+checkpoint['sequence_length'] = sequence_length
+checkpoint['adam_betas'] = adam_betas
+
+
+device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
+
+checkpoint['model_args']    = ()
+checkpoint['model_kwargs']  = {'cond_size': cond_size, 'gamma': args.gamma}
+print(checkpoint['model_kwargs'])
+model = fargan.FARGAN(*checkpoint['model_args'], **checkpoint['model_kwargs'])
+
+#model = fargan.FARGAN()
+#model = nn.DataParallel(model)
+
+if type(args.initial_checkpoint) != type(None):
+    checkpoint = torch.load(args.initial_checkpoint, map_location='cpu')
+    model.load_state_dict(checkpoint['state_dict'], strict=False)
+
+checkpoint['state_dict']    = model.state_dict()
+
+
+dataset = FARGANDataset(features_file, signal_file, sequence_length=sequence_length)
+dataloader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, shuffle=True, drop_last=True, num_workers=4)
+
+
+optimizer = torch.optim.AdamW(model.parameters(), lr=lr, betas=adam_betas, eps=adam_eps)
+
+
+# learning rate scheduler
+scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer=optimizer, lr_lambda=lambda x : 1 / (1 + lr_decay * x))
+
+states = None
+
+spect_loss =  MultiResolutionSTFTLoss(device).to(device)
+
+if __name__ == '__main__':
+    model.to(device)
+
+    for epoch in range(1, epochs + 1):
+
+        running_specc = 0
+        running_cont_loss = 0
+        running_loss = 0
+
+        print(f"training epoch {epoch}...")
+        with tqdm.tqdm(dataloader, unit='batch') as tepoch:
+            for i, (features, periods, target, lpc) in enumerate(tepoch):
+                optimizer.zero_grad()
+                features = features.to(device)
+                #lpc = torch.tensor(fargan.interp_lpc(lpc.numpy(), 4))
+                #print("interp size", lpc.shape)
+                #lpc = lpc.to(device)
+                #lpc = lpc*(args.gamma**torch.arange(1,17, device=device))
+                #lpc = fargan.interp_lpc(lpc, 4)
+                periods = periods.to(device)
+                if (np.random.rand() > 0.1):
+                    target = target[:, :sequence_length*160]
+                    #lpc = lpc[:,:sequence_length*4,:]
+                    features = features[:,:sequence_length+4,:]
+                    periods = periods[:,:sequence_length+4]
+                else:
+                    target=target[::2, :]
+                    #lpc=lpc[::2,:]
+                    features=features[::2,:]
+                    periods=periods[::2,:]
+                target = target.to(device)
+                #print(target.shape, lpc.shape)
+                #target = fargan.analysis_filter(target, lpc[:,:,:], nb_subframes=1, gamma=args.gamma)
+
+                #nb_pre = random.randrange(1, 6)
+                nb_pre = 2
+                pre = target[:, :nb_pre*160]
+                sig, states = model(features, periods, target.size(1)//160 - nb_pre, pre=pre, states=None)
+                sig = torch.cat([pre, sig], -1)
+
+                cont_loss = fargan.sig_loss(target[:, nb_pre*160:nb_pre*160+160], sig[:, nb_pre*160:nb_pre*160+160])
+                specc_loss = spect_loss(sig, target.detach())
+                loss = .03*cont_loss + specc_loss
+
+                loss.backward()
+                optimizer.step()
+
+                #model.clip_weights()
+
+                scheduler.step()
+
+                running_specc += specc_loss.detach().cpu().item()
+                running_cont_loss += cont_loss.detach().cpu().item()
+
+                running_loss += loss.detach().cpu().item()
+                tepoch.set_postfix(loss=f"{running_loss/(i+1):8.5f}",
+                                   cont_loss=f"{running_cont_loss/(i+1):8.5f}",
+                                   specc=f"{running_specc/(i+1):8.5f}",
+                                   )
+
+        # save checkpoint
+        checkpoint_path = os.path.join(checkpoint_dir, f'fargan{args.suffix}_{epoch}.pth')
+        checkpoint['state_dict'] = model.state_dict()
+        checkpoint['loss'] = running_loss / len(dataloader)
+        checkpoint['epoch'] = epoch
+        torch.save(checkpoint, checkpoint_path)

package/opus/dnn/torch/fwgan/dump_model_weights.py

@@ -0,0 +1,88 @@
+import os
+import sys
+import argparse
+
+import torch
+from torch import nn
+
+
+sys.path.append(os.path.join(os.path.split(__file__)[0], '../weight-exchange'))
+import wexchange.torch
+
+from models import model_dict
+
+unquantized = [
+    'bfcc_with_corr_upsampler.fc',
+    'cont_net.0',
+    'fwc6.cont_fc.0',
+    'fwc6.fc.0',
+    'fwc6.fc.1.gate',
+    'fwc7.cont_fc.0',
+    'fwc7.fc.0',
+    'fwc7.fc.1.gate'
+]
+
+description=f"""
+This is an unsafe dumping script for FWGAN models. It assumes that all weights are included in Linear, Conv1d or GRU layer
+and will fail to export any other weights.
+
+Furthermore, the quanitze option relies on the following explicit list of layers to be excluded:
+{unquantized}.
+
+Modify this script manually if adjustments are needed.
+"""
+
+parser = argparse.ArgumentParser(description=description)
+parser.add_argument('model', choices=['fwgan400', 'fwgan500'], help='model name')
+parser.add_argument('weightfile', type=str, help='weight file path')
+parser.add_argument('export_folder', type=str)
+parser.add_argument('--export-filename', type=str, default='fwgan_data', help='filename for source and header file (.c and .h will be added), defaults to fwgan_data')
+parser.add_argument('--struct-name', type=str, default='FWGAN', help='name for C struct, defaults to FWGAN')
+parser.add_argument('--quantize', action='store_true', help='apply quantization')
+
+if __name__ == "__main__":
+    args = parser.parse_args()
+
+    model = model_dict[args.model]()
+
+    print(f"loading weights from {args.weightfile}...")
+    saved_gen= torch.load(args.weightfile, map_location='cpu')
+    model.load_state_dict(saved_gen)
+    def _remove_weight_norm(m):
+        try:
+            torch.nn.utils.remove_weight_norm(m)
+        except ValueError:  # this module didn't have weight norm
+            return
+    model.apply(_remove_weight_norm)
+
+
+    print("dumping model...")
+    quantize_model=args.quantize
+
+    output_folder = args.export_folder
+    os.makedirs(output_folder, exist_ok=True)
+
+    writer = wexchange.c_export.c_writer.CWriter(os.path.join(output_folder, args.export_filename), model_struct_name=args.struct_name)
+
+    for name, module in model.named_modules():
+
+        if quantize_model:
+            quantize=name not in unquantized
+            scale = None if quantize else 1/128
+        else:
+            quantize=False
+            scale=1/128
+
+        if isinstance(module, nn.Linear):
+            print(f"dumping linear layer {name}...")
+            wexchange.torch.dump_torch_dense_weights(writer, module, name.replace('.', '_'), quantize=quantize, scale=scale)
+
+        if isinstance(module, nn.Conv1d):
+            print(f"dumping conv1d layer {name}...")
+            wexchange.torch.dump_torch_conv1d_weights(writer, module, name.replace('.', '_'), quantize=quantize, scale=scale)
+
+        if isinstance(module, nn.GRU):
+            print(f"dumping GRU layer {name}...")
+            wexchange.torch.dump_torch_gru_weights(writer, module, name.replace('.', '_'), quantize=quantize, scale=scale, recurrent_scale=scale)
+
+    writer.close()

package/opus/dnn/torch/fwgan/inference.py

@@ -0,0 +1,141 @@
+import os
+import time
+import torch
+import numpy as np
+from scipy import signal as si
+from scipy.io import wavfile
+import argparse
+
+from models import model_dict
+
+parser = argparse.ArgumentParser()
+parser.add_argument('model', choices=['fwgan400', 'fwgan500'], help='model name')
+parser.add_argument('weightfile', type=str, help='weight file')
+parser.add_argument('input', type=str, help='input: feature file or folder with feature files')
+parser.add_argument('output', type=str, help='output: wav file name or folder name, depending on input')
+
+
+########################### Signal Processing Layers ###########################
+
+def preemphasis(x, coef= -0.85):
+
+    return si.lfilter(np.array([1.0, coef]), np.array([1.0]), x).astype('float32')
+
+def deemphasis(x, coef= -0.85):
+
+    return si.lfilter(np.array([1.0]), np.array([1.0, coef]), x).astype('float32')
+
+gamma = 0.92
+weighting_vector = np.array([gamma**i for i in range(16,0,-1)])
+
+
+def lpc_synthesis_one_frame(frame, filt, buffer, weighting_vector=np.ones(16)):
+
+    out = np.zeros_like(frame)
+
+    filt = np.flip(filt)
+
+    inp = frame[:]
+
+
+    for i in range(0, inp.shape[0]):
+
+        s = inp[i] - np.dot(buffer*weighting_vector, filt)
+
+        buffer[0] = s
+
+        buffer = np.roll(buffer, -1)
+
+        out[i] = s
+
+    return out
+
+def inverse_perceptual_weighting (pw_signal, filters, weighting_vector):
+
+    #inverse perceptual weighting= H_preemph / W(z/gamma)
+
+    pw_signal = preemphasis(pw_signal)
+
+    signal = np.zeros_like(pw_signal)
+    buffer = np.zeros(16)
+    num_frames = pw_signal.shape[0] //160
+    assert num_frames == filters.shape[0]
+
+    for frame_idx in range(0, num_frames):
+
+        in_frame = pw_signal[frame_idx*160: (frame_idx+1)*160][:]
+        out_sig_frame = lpc_synthesis_one_frame(in_frame, filters[frame_idx, :], buffer, weighting_vector)
+        signal[frame_idx*160: (frame_idx+1)*160] = out_sig_frame[:]
+        buffer[:] = out_sig_frame[-16:]
+
+    return signal
+
+
+def process_item(generator, feature_filename, output_filename, verbose=False):
+
+    feat = np.memmap(feature_filename, dtype='float32', mode='r')
+
+    num_feat_frames = len(feat) // 36
+    feat = np.reshape(feat, (num_feat_frames, 36))
+
+    bfcc = np.copy(feat[:, :18])
+    corr = np.copy(feat[:, 19:20]) + 0.5
+    bfcc_with_corr =  torch.from_numpy(np.hstack((bfcc, corr))).type(torch.FloatTensor).unsqueeze(0)#.to(device)
+
+    period = torch.from_numpy((0.1 + 50 * np.copy(feat[:, 18:19]) + 100)\
+                            .astype('int32')).type(torch.long).view(1,-1)#.to(device)
+
+    lpc_filters = np.copy(feat[:, -16:])
+
+    start_time = time.time()
+    x1 = generator(period, bfcc_with_corr, torch.zeros(1,320)) #this means the vocoder runs in complete synthesis mode with zero history audio frames
+    end_time = time.time()
+    total_time = end_time - start_time
+    x1 = x1.squeeze(1).squeeze(0).detach().cpu().numpy()
+    gen_seconds = len(x1)/16000
+    out = deemphasis(inverse_perceptual_weighting(x1, lpc_filters, weighting_vector))
+    if verbose:
+        print(f"Took {total_time:.3f}s to generate {len(x1)}  samples ({gen_seconds}s) -> {gen_seconds/total_time:.2f}x real time")
+
+    out = np.clip(np.round(2**15 * out), -2**15, 2**15 -1).astype(np.int16)
+    wavfile.write(output_filename, 16000, out)
+
+
+########################### The inference loop over folder containing lpcnet feature files #################################
+if __name__ == "__main__":
+
+    args = parser.parse_args()
+
+    generator = model_dict[args.model]()
+
+
+    #Load the FWGAN500Hz Checkpoint
+    saved_gen= torch.load(args.weightfile, map_location='cpu')
+    generator.load_state_dict(saved_gen)
+
+    #this is just to remove the weight_norm from the model layers as it's no longer needed
+    def _remove_weight_norm(m):
+        try:
+            torch.nn.utils.remove_weight_norm(m)
+        except ValueError:  # this module didn't have weight norm
+            return
+    generator.apply(_remove_weight_norm)
+
+    #enable inference mode
+    generator = generator.eval()
+
+    print('Successfully loaded the generator model ... start generation:')
+
+    if os.path.isdir(args.input):
+
+        os.makedirs(args.output, exist_ok=True)
+
+        for fn in os.listdir(args.input):
+            print(f"processing input {fn}...")
+            feature_filename = os.path.join(args.input, fn)
+            output_filename = os.path.join(args.output, os.path.splitext(fn)[0] + f"_{args.model}.wav")
+            process_item(generator, feature_filename, output_filename)
+    else:
+        process_item(generator, args.input, args.output)
+
+    print("Finished!")

package/opus/dnn/torch/fwgan/models/__init__.py

@@ -0,0 +1,7 @@
+from .fwgan400 import FWGAN400ContLarge
+from .fwgan500 import FWGAN500Cont
+
+model_dict = {
+    'fwgan400': FWGAN400ContLarge,
+    'fwgan500': FWGAN500Cont
+}