@scrypted/libav 1.0.166 → 1.0.168

package/opus/dnn/torch/dnntools/dnntools/sparsification/linear_sparsifier.py

@@ -0,0 +1,128 @@
+"""
+/* Copyright (c) 2023 Amazon
+   Written by Jan Buethe */
+/*
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions
+   are met:
+
+   - Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+
+   - Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+   ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+   OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+   EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+   PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+   PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+   LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+"""
+
+import torch
+
+from .base_sparsifier import BaseSparsifier
+from .common import sparsify_matrix
+
+
+class LinearSparsifier(BaseSparsifier):
+    def __init__(self, task_list, start, stop, interval, exponent=3):
+        """ Sparsifier for torch.nn.GRUs
+
+            Parameters:
+            -----------
+            task_list : list
+                task_list contains a list of tuples (linear, params), where linear is an instance
+                of torch.nn.Linear and params is a tuple (density, [m, n]),
+                where density is the target density in [0, 1], [m, n] is the shape sub-blocks to which
+                sparsification is applied.
+
+            start : int
+                training step after which sparsification will be started.
+
+            stop : int
+                training step after which sparsification will be completed.
+
+            interval : int
+                sparsification interval for steps between start and stop. After stop sparsification will be
+                carried out after every call to GRUSparsifier.step()
+
+            exponent : float
+                Interpolation exponent for sparsification interval. In step i sparsification will be carried out
+                with density (alpha + target_density * (1 * alpha)), where
+                alpha = ((stop - i) / (start - stop)) ** exponent
+
+            Example:
+            --------
+            >>> import torch
+            >>> linear = torch.nn.Linear(8, 16)
+            >>> params = (0.2, [8, 4])
+            >>> sparsifier = LinearSparsifier([(linear, params)], 0, 100, 50)
+            >>> for i in range(100):
+            ...         sparsifier.step()
+        """
+
+        super().__init__(task_list, start, stop, interval, exponent=3)
+
+        self.last_mask = None
+
+    def sparsify(self, alpha, verbose=False):
+        """ carries out sparsification step
+
+            Call this function after optimizer.step in your
+            training loop.
+
+            Parameters:
+            ----------
+            alpha : float
+                density interpolation parameter (1: dense, 0: target density)
+            verbose : bool
+                if true, densities are printed out
+
+            Returns:
+            --------
+            None
+
+        """
+
+        with torch.no_grad():
+            for linear, params in self.task_list:
+                if hasattr(linear, 'weight_v'):
+                    weight = linear.weight_v
+                else:
+                    weight = linear.weight
+                target_density, block_size = params
+                density = alpha + (1 - alpha) * target_density
+                weight[:], new_mask = sparsify_matrix(weight, density, block_size, return_mask=True)
+
+                if self.last_mask is not None:
+                    if not torch.all(self.last_mask * new_mask == new_mask) and debug:
+                        print("weight resurrection in conv.weight")
+
+                self.last_mask = new_mask
+
+                if verbose:
+                    print(f"linear_sparsifier[{self.step_counter}]: {density=}")
+
+
+if __name__ == "__main__":
+    print("Testing sparsifier")
+
+    import torch
+    linear = torch.nn.Linear(8, 16)
+    params = (0.2, [4, 2])
+
+    sparsifier = LinearSparsifier([(linear, params)], 0, 100, 5)
+
+    for i in range(100):
+            sparsifier.step(verbose=True)
+
+    print(linear.weight)

package/opus/dnn/torch/dnntools/dnntools/sparsification/utils.py

@@ -0,0 +1,64 @@
+import torch
+
+from dnntools.sparsification import GRUSparsifier, LinearSparsifier, Conv1dSparsifier, ConvTranspose1dSparsifier
+
+def mark_for_sparsification(module, params):
+    setattr(module, 'sparsify', True)
+    setattr(module, 'sparsification_params', params)
+    return module
+
+def create_sparsifier(module, start, stop, interval):
+    sparsifier_list = []
+    for m in module.modules():
+        if hasattr(m, 'sparsify'):
+            if isinstance(m, torch.nn.GRU):
+                sparsifier_list.append(
+                    GRUSparsifier([(m, m.sparsification_params)], start, stop, interval)
+                )
+            elif isinstance(m, torch.nn.Linear):
+                sparsifier_list.append(
+                    LinearSparsifier([(m, m.sparsification_params)], start, stop, interval)
+                )
+            elif isinstance(m, torch.nn.Conv1d):
+                sparsifier_list.append(
+                    Conv1dSparsifier([(m, m.sparsification_params)], start, stop, interval)
+                )
+            elif isinstance(m, torch.nn.ConvTranspose1d):
+                sparsifier_list.append(
+                    ConvTranspose1dSparsifier([(m, m.sparsification_params)], start, stop, interval)
+                )
+            else:
+                print(f"[create_sparsifier] warning: module {m} marked for sparsification but no suitable sparsifier exists.")
+
+    def sparsify(verbose=False):
+        for sparsifier in sparsifier_list:
+            sparsifier.step(verbose)
+
+    return sparsify
+
+
+def count_parameters(model, verbose=False):
+    total = 0
+    for name, p in model.named_parameters():
+        count = torch.ones_like(p).sum().item()
+
+        if verbose:
+            print(f"{name}: {count} parameters")
+
+        total += count
+
+    return total
+
+def estimate_nonzero_parameters(module):
+    num_zero_parameters = 0
+    if hasattr(module, 'sparsify'):
+        params = module.sparsification_params
+        if isinstance(module, torch.nn.Conv1d) or isinstance(module, torch.nn.ConvTranspose1d):
+            num_zero_parameters = torch.ones_like(module.weight).sum().item() * (1 - params[0])
+        elif isinstance(module, torch.nn.GRU):
+            num_zero_parameters = module.input_size * module.hidden_size * (3 - params['W_ir'][0] - params['W_iz'][0] - params['W_in'][0])
+            num_zero_parameters += module.hidden_size * module.hidden_size * (3 - params['W_hr'][0] - params['W_hz'][0] - params['W_hn'][0])
+        elif isinstance(module, torch.nn.Linear):
+            num_zero_parameters = module.in_features * module.out_features * params[0]
+        else:
+            raise ValueError(f'unknown sparsification method for module of type {type(module)}')

package/opus/dnn/torch/dnntools/requirements.txt

@@ -0,0 +1 @@
+torch

package/opus/dnn/torch/dnntools/setup.py

@@ -0,0 +1,48 @@
+"""
+/* Copyright (c) 2023 Amazon
+   Written by Jan Buethe */
+/*
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions
+   are met:
+
+   - Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+
+   - Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+   ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+   OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+   EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+   PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+   PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+   LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+"""
+
+#!/usr/bin/env/python
+import os
+from setuptools import setup
+
+lib_folder = os.path.dirname(os.path.realpath(__file__))
+
+with open(os.path.join(lib_folder, 'requirements.txt'), 'r') as f:
+    install_requires = list(f.read().splitlines())
+
+print(install_requires)
+
+setup(name='dnntools',
+      version='1.0',
+      author='Jan Buethe',
+      author_email='jbuethe@amazon.de',
+      description='Non-Standard tools for deep neural network training with PyTorch',
+      packages=['dnntools', 'dnntools.sparsification', 'dnntools.quantization'],
+      install_requires=install_requires
+      )

package/opus/dnn/torch/fargan/README.md

@@ -0,0 +1,54 @@
+# Framewise Auto-Regressive GAN (FARGAN)
+
+Implementation of FARGAN, a low-complexity neural vocoder. Pre-trained models
+are provided as C code in the dnn/ directory with the corresponding model in
+dnn/models/ directory (name starts with fargan_). If you don't want to train
+a new FARGAN model, you can skip straight to the Inference section.
+
+## Data preparation
+
+For data preparation you need to build Opus as detailed in the top-level README.
+You will need to use the --enable-deep-plc configure option.
+The build will produce an executable named "dump_data".
+To prepare the training data, run:
+```
+./dump_data -train in_speech.pcm out_features.f32 out_speech.pcm
+```
+Where the in_speech.pcm speech file is a raw 16-bit PCM file sampled at 16 kHz.
+The speech data used for training the model can be found at:
+https://media.xiph.org/lpcnet/speech/tts_speech_negative_16k.sw
+
+## Training
+
+To perform pre-training, run the following command:
+```
+python ./train_fargan.py out_features.f32 out_speech.pcm output_dir --epochs 400 --batch-size 4096 --lr 0.002 --cuda-visible-devices 0
+```
+Once pre-training is complete, run adversarial training using:
+```
+python adv_train_fargan.py out_features.f32 out_speech.pcm output_dir --lr 0.000002 --reg-weight 5 --batch-size 160 --cuda-visible-devices 0 --initial-checkpoint output_dir/checkpoints/fargan_400.pth
+```
+The final model will be in output_dir/checkpoints/fargan_adv_50.pth.
+
+The model can optionally be converted to C using:
+```
+python dump_fargan_weights.py output_dir/checkpoints/fargan_adv_50.pth fargan_c_dir
+```
+which will create a fargan_data.c and a fargan_data.h file in the fargan_c_dir directory.
+Copy these files to the opus/dnn/ directory (replacing the existing ones) and recompile Opus.
+
+## Inference
+
+To run the inference, start by generating the features from the audio using:
+```
+./fargan_demo -features test_speech.pcm test_features.f32
+```
+Synthesis can be achieved either using the PyTorch code or the C code.
+To synthesize from PyTorch, run:
+```
+python test_fargan.py output_dir/checkpoints/fargan_adv_50.pth test_features.f32 output_speech.pcm
+```
+To synthesize from the C code, run:
+```
+./fargan_demo -fargan-synthesis test_features.f32 output_speech.pcm
+```

package/opus/dnn/torch/fargan/adv_train_fargan.py

@@ -0,0 +1,277 @@
+import os
+import argparse
+import random
+import numpy as np
+import sys
+import math as m
+
+import torch
+from torch import nn
+import torch.nn.functional as F
+import tqdm
+
+import fargan
+from dataset import FARGANDataset
+from stft_loss import *
+
+source_dir = os.path.split(os.path.abspath(__file__))[0]
+sys.path.append(os.path.join(source_dir, "../osce/"))
+
+import models as osce_models
+
+
+def fmap_loss(scores_real, scores_gen):
+    num_discs = len(scores_real)
+    loss_feat = 0
+    for k in range(num_discs):
+        num_layers = len(scores_gen[k]) - 1
+        f = 4 / num_discs / num_layers
+        for l in range(num_layers):
+            loss_feat += f * F.l1_loss(scores_gen[k][l], scores_real[k][l].detach())
+
+    return loss_feat
+
+parser = argparse.ArgumentParser()
+
+parser.add_argument('features', type=str, help='path to feature file in .f32 format')
+parser.add_argument('signal', type=str, help='path to signal file in .s16 format')
+parser.add_argument('output', type=str, help='path to output folder')
+
+parser.add_argument('--suffix', type=str, help="model name suffix", default="")
+parser.add_argument('--cuda-visible-devices', type=str, help="comma separates list of cuda visible device indices, default: CUDA_VISIBLE_DEVICES", default=None)
+
+
+model_group = parser.add_argument_group(title="model parameters")
+model_group.add_argument('--cond-size', type=int, help="first conditioning size, default: 256", default=256)
+model_group.add_argument('--gamma', type=float, help="Use A(z/gamma), default: 0.9", default=0.9)
+
+training_group = parser.add_argument_group(title="training parameters")
+training_group.add_argument('--batch-size', type=int, help="batch size, default: 128", default=128)
+training_group.add_argument('--lr', type=float, help='learning rate, default: 5e-4', default=5e-4)
+training_group.add_argument('--epochs', type=int, help='number of training epochs, default: 50', default=50)
+training_group.add_argument('--sequence-length', type=int, help='sequence length, default: 60', default=60)
+training_group.add_argument('--lr-decay', type=float, help='learning rate decay factor, default: 0.0', default=0.0)
+training_group.add_argument('--initial-checkpoint', type=str, help='initial checkpoint to start training from, default: None', default=None)
+training_group.add_argument('--reg-weight', type=float, help='regression loss weight, default: 1.0', default=1.0)
+training_group.add_argument('--fmap-weight', type=float, help='feature matchin loss weight, default: 1.0', default=1.)
+
+args = parser.parse_args()
+
+if args.cuda_visible_devices != None:
+    os.environ['CUDA_VISIBLE_DEVICES'] = args.cuda_visible_devices
+
+# checkpoints
+checkpoint_dir = os.path.join(args.output, 'checkpoints')
+checkpoint = dict()
+os.makedirs(checkpoint_dir, exist_ok=True)
+
+
+# training parameters
+batch_size = args.batch_size
+lr = args.lr
+epochs = args.epochs
+sequence_length = args.sequence_length
+lr_decay = args.lr_decay
+
+adam_betas = [0.8, 0.99]
+adam_eps = 1e-8
+features_file = args.features
+signal_file = args.signal
+
+# model parameters
+cond_size  = args.cond_size
+
+
+checkpoint['batch_size'] = batch_size
+checkpoint['lr'] = lr
+checkpoint['lr_decay'] = lr_decay
+checkpoint['epochs'] = epochs
+checkpoint['sequence_length'] = sequence_length
+checkpoint['adam_betas'] = adam_betas
+
+
+device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
+
+checkpoint['model_args']    = ()
+checkpoint['model_kwargs']  = {'cond_size': cond_size, 'gamma': args.gamma}
+print(checkpoint['model_kwargs'])
+model = fargan.FARGAN(*checkpoint['model_args'], **checkpoint['model_kwargs'])
+
+
+#discriminator
+disc_name = 'fdmresdisc'
+disc = osce_models.model_dict[disc_name](
+    architecture='free',
+    design='f_down',
+    fft_sizes_16k=[2**n for n in range(6, 12)],
+    freq_roi=[0, 7400],
+    max_channels=256,
+    noise_gain=0.0
+)
+
+if type(args.initial_checkpoint) != type(None):
+    checkpoint = torch.load(args.initial_checkpoint, map_location='cpu')
+    model.load_state_dict(checkpoint['state_dict'], strict=False)
+
+checkpoint['state_dict']    = model.state_dict()
+
+
+dataset = FARGANDataset(features_file, signal_file, sequence_length=sequence_length)
+dataloader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, shuffle=True, drop_last=True, num_workers=4)
+
+
+optimizer = torch.optim.AdamW(model.parameters(), lr=lr, betas=adam_betas, eps=adam_eps)
+optimizer_disc = torch.optim.AdamW([p for p in disc.parameters() if p.requires_grad], lr=lr, betas=adam_betas, eps=adam_eps)
+
+
+# learning rate scheduler
+scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer=optimizer, lr_lambda=lambda x : 1 / (1 + lr_decay * x))
+scheduler_disc = torch.optim.lr_scheduler.LambdaLR(optimizer=optimizer_disc, lr_lambda=lambda x : 1 / (1 + lr_decay * x))
+
+states = None
+
+spect_loss =  MultiResolutionSTFTLoss(device).to(device)
+
+for param in model.parameters():
+    param.requires_grad = False
+
+batch_count = 0
+if __name__ == '__main__':
+    model.to(device)
+    disc.to(device)
+
+    for epoch in range(1, epochs + 1):
+
+        m_r = 0
+        m_f = 0
+        s_r = 1
+        s_f = 1
+
+        running_cont_loss = 0
+        running_disc_loss = 0
+        running_gen_loss = 0
+        running_fmap_loss = 0
+        running_reg_loss = 0
+        running_wc = 0
+
+        print(f"training epoch {epoch}...")
+        with tqdm.tqdm(dataloader, unit='batch') as tepoch:
+            for i, (features, periods, target, lpc) in enumerate(tepoch):
+                if epoch == 1 and i == 400:
+                    for param in model.parameters():
+                        param.requires_grad = True
+                    for param in model.cond_net.parameters():
+                        param.requires_grad = False
+                    for param in model.sig_net.cond_gain_dense.parameters():
+                        param.requires_grad = False
+
+                optimizer.zero_grad()
+                features = features.to(device)
+                #lpc = lpc.to(device)
+                #lpc = lpc*(args.gamma**torch.arange(1,17, device=device))
+                #lpc = fargan.interp_lpc(lpc, 4)
+                periods = periods.to(device)
+                if True:
+                    target = target[:, :sequence_length*160]
+                    #lpc = lpc[:,:sequence_length*4,:]
+                    features = features[:,:sequence_length+4,:]
+                    periods = periods[:,:sequence_length+4]
+                else:
+                    target=target[::2, :]
+                    #lpc=lpc[::2,:]
+                    features=features[::2,:]
+                    periods=periods[::2,:]
+                target = target.to(device)
+                #target = fargan.analysis_filter(target, lpc[:,:,:], nb_subframes=1, gamma=args.gamma)
+
+                #nb_pre = random.randrange(1, 6)
+                nb_pre = 2
+                pre = target[:, :nb_pre*160]
+                output, _ = model(features, periods, target.size(1)//160 - nb_pre, pre=pre, states=None)
+                output = torch.cat([pre, output], -1)
+
+
+                # discriminator update
+                scores_gen = disc(output.detach().unsqueeze(1))
+                scores_real = disc(target.unsqueeze(1))
+
+                disc_loss = 0
+                for scale in scores_gen:
+                    disc_loss += ((scale[-1]) ** 2).mean()
+                    m_f = 0.9 * m_f + 0.1 * scale[-1].detach().mean().cpu().item()
+                    s_f = 0.9 * s_f + 0.1 * scale[-1].detach().std().cpu().item()
+
+                for scale in scores_real:
+                    disc_loss += ((1 - scale[-1]) ** 2).mean()
+                    m_r = 0.9 * m_r + 0.1 * scale[-1].detach().mean().cpu().item()
+                    s_r = 0.9 * s_r + 0.1 * scale[-1].detach().std().cpu().item()
+
+                disc_loss = 0.5 * disc_loss / len(scores_gen)
+                winning_chance = 0.5 * m.erfc( (m_r - m_f) / m.sqrt(2 * (s_f**2 + s_r**2)) )
+                running_wc += winning_chance
+
+                disc.zero_grad()
+                disc_loss.backward()
+                optimizer_disc.step()
+
+                # model update
+                scores_gen = disc(output.unsqueeze(1))
+                if False: # todo: check whether that makes a difference
+                    with torch.no_grad():
+                        scores_real = disc(target.unsqueeze(1))
+
+                cont_loss = fargan.sig_loss(target[:, nb_pre*160:nb_pre*160+80], output[:, nb_pre*160:nb_pre*160+80])
+                specc_loss = spect_loss(output, target.detach())
+                reg_loss = (.00*cont_loss + specc_loss)
+
+                loss_gen = 0
+                for scale in scores_gen:
+                    loss_gen += ((1 - scale[-1]) ** 2).mean() / len(scores_gen)
+
+                feat_loss = args.fmap_weight * fmap_loss(scores_real, scores_gen)
+
+                reg_weight = args.reg_weight# + 15./(1 + (batch_count/7600.))
+                gen_loss = reg_weight * reg_loss +  feat_loss + loss_gen
+
+                model.zero_grad()
+
+
+                gen_loss.backward()
+                optimizer.step()
+
+                #model.clip_weights()
+
+                scheduler.step()
+                scheduler_disc.step()
+
+                running_cont_loss += cont_loss.detach().cpu().item()
+                running_gen_loss += loss_gen.detach().cpu().item()
+                running_disc_loss += disc_loss.detach().cpu().item()
+                running_fmap_loss += feat_loss.detach().cpu().item()
+                running_reg_loss += reg_loss.detach().cpu().item()
+
+
+
+                tepoch.set_postfix(cont_loss=f"{running_cont_loss/(i+1):8.5f}",
+                                   reg_weight=f"{reg_weight:8.5f}",
+                                   gen_loss=f"{running_gen_loss/(i+1):8.5f}",
+                                   disc_loss=f"{running_disc_loss/(i+1):8.5f}",
+                                   fmap_loss=f"{running_fmap_loss/(i+1):8.5f}",
+                                   reg_loss=f"{running_reg_loss/(i+1):8.5f}",
+                                   wc = f"{running_wc/(i+1):8.5f}",
+                                   )
+                batch_count = batch_count + 1
+
+        # save checkpoint
+        checkpoint_path = os.path.join(checkpoint_dir, f'fargan{args.suffix}_adv_{epoch}.pth')
+        checkpoint['state_dict'] = model.state_dict()
+        checkpoint['disc_sate_dict'] = disc.state_dict()
+        checkpoint['loss'] = {
+            'cont': running_cont_loss / len(dataloader),
+            'gen': running_gen_loss / len(dataloader),
+            'disc': running_disc_loss / len(dataloader),
+            'fmap': running_fmap_loss / len(dataloader),
+            'reg': running_reg_loss / len(dataloader)
+        }
+        checkpoint['epoch'] = epoch
+        torch.save(checkpoint, checkpoint_path)

package/opus/dnn/torch/fargan/dataset.py

@@ -0,0 +1,61 @@
+import torch
+import numpy as np
+import fargan
+
+class FARGANDataset(torch.utils.data.Dataset):
+    def __init__(self,
+                feature_file,
+                signal_file,
+                frame_size=160,
+                sequence_length=15,
+                lookahead=1,
+                nb_used_features=20,
+                nb_features=36):
+
+        self.frame_size = frame_size
+        self.sequence_length = sequence_length
+        self.lookahead = lookahead
+        self.nb_features = nb_features
+        self.nb_used_features = nb_used_features
+        pcm_chunk_size = self.frame_size*self.sequence_length
+
+        self.data = np.memmap(signal_file, dtype='int16', mode='r')
+        #self.data = self.data[1::2]
+        self.nb_sequences = len(self.data)//(pcm_chunk_size)-4
+        self.data = self.data[(4-self.lookahead)*self.frame_size:]
+        self.data = self.data[:self.nb_sequences*pcm_chunk_size]
+
+
+        #self.data = np.reshape(self.data, (self.nb_sequences, pcm_chunk_size))
+        sizeof = self.data.strides[-1]
+        self.data = np.lib.stride_tricks.as_strided(self.data, shape=(self.nb_sequences, pcm_chunk_size*2),
+                                           strides=(pcm_chunk_size*sizeof, sizeof))
+
+        self.features = np.reshape(np.memmap(feature_file, dtype='float32', mode='r'), (-1, nb_features))
+        sizeof = self.features.strides[-1]
+        self.features = np.lib.stride_tricks.as_strided(self.features, shape=(self.nb_sequences, self.sequence_length*2+4, nb_features),
+                                           strides=(self.sequence_length*self.nb_features*sizeof, self.nb_features*sizeof, sizeof))
+        #self.periods = np.round(50*self.features[:,:,self.nb_used_features-2]+100).astype('int')
+        self.periods = np.round(np.clip(256./2**(self.features[:,:,self.nb_used_features-2]+1.5), 32, 255)).astype('int')
+
+        self.lpc = self.features[:, :, self.nb_used_features:]
+        self.features = self.features[:, :, :self.nb_used_features]
+        print("lpc_size:", self.lpc.shape)
+
+    def __len__(self):
+        return self.nb_sequences
+
+    def __getitem__(self, index):
+        features = self.features[index, :, :].copy()
+        if self.lookahead != 0:
+            lpc = self.lpc[index, 4-self.lookahead:-self.lookahead, :].copy()
+        else:
+            lpc = self.lpc[index, 4:, :].copy()
+        data = self.data[index, :].copy().astype(np.float32) / 2**15
+        periods = self.periods[index, :].copy()
+        #lpc = lpc*(self.gamma**np.arange(1,17))
+        #lpc=lpc[None,:,:]
+        #lpc = fargan.interp_lpc(lpc, 4)
+        #lpc=lpc[0,:,:]
+
+        return features, periods, data, lpc