@scrypted/libav 1.0.169 → 1.0.171

package/opus/dnn/torch/lpcnet/scripts/loop_run.sh

@@ -1,52 +0,0 @@
-#!/bin/bash
-
-
-case $# in
-    9) SETUP=$1; OUTDIR=$2; NAME=$3; DEVICE=$4; ROUNDS=$5; LPCNEXT=$6; LPCNET=$7; TESTSUITE=$8; TESTITEMS=$9;;
-    *) echo "loop_run.sh setup outdir name device rounds lpcnext_repo lpcnet_repo testsuite_repo testitems"; exit;;
-esac
-
-
-PYTHON="/home/ubuntu/opt/miniconda3/envs/torch/bin/python"
-TESTFEATURES=${LPCNEXT}/testitems/features/all_0_orig_features.f32
-WARPQREFERENCE=${LPCNEXT}/testitems/wav/all_0_orig.wav
-METRICS="warpq,pesq,pitch_error,voicing_error"
-LPCNETDEMO=${LPCNET}/lpcnet_demo
-
-for ((round = 1; round <= $ROUNDS; round++))
-do
-    echo
-    echo round $round
-
-    UUID=$(uuidgen)
-    TRAINOUT=${OUTDIR}/${UUID}/training
-    TESTOUT=${OUTDIR}/${UUID}/testing
-    CHECKPOINT=${TRAINOUT}/checkpoints/checkpoint_last.pth
-    FINALCHECKPOINT=${TRAINOUT}/checkpoints/checkpoint_finalize_last.pth
-
-    # run training
-    echo "starting training..."
-    $PYTHON $LPCNEXT/train_lpcnet.py $SETUP $TRAINOUT --device $DEVICE --test-features $TESTFEATURES --warpq-reference $WARPQREFERENCE
-
-    # run finalization
-    echo "starting finalization..."
-    $PYTHON $LPCNEXT/train_lpcnet.py $SETUP $TRAINOUT \
-            --device $DEVICE --test-features $TESTFEATURES \
-            --warpq-reference $WARPQREFERENCE \
-            --finalize --initial-checkpoint $CHECKPOINT
-
-    # create test configs
-    $PYTHON $LPCNEXT/make_test_config.py ${OUTDIR}/${UUID}/testconfig.yml "$NAME $UUID" $CHECKPOINT --lpcnet-demo $LPCNETDEMO
-    $PYTHON $LPCNEXT/make_test_config.py ${OUTDIR}/${UUID}/testconfig_finalize.yml "$NAME $UUID finalized" $FINALCHECKPOINT --lpcnet-demo $LPCNETDEMO
-
-    # run tests
-    echo "starting test 1 (no finalization)..."
-    $PYTHON $TESTSUITE/run_test.py ${OUTDIR}/${UUID}/testconfig.yml  \
-            $TESTITEMS ${TESTOUT}/prefinal --num-workers 8 \
-            --num-testitems 400 --metrics $METRICS
-
-    echo "starting test 2 (after finalization)..."
-    $PYTHON $TESTSUITE/run_test.py ${OUTDIR}/${UUID}/testconfig_finalize.yml  \
-            $TESTITEMS ${TESTOUT}/final --num-workers 8 \
-            --num-testitems 400 --metrics $METRICS
-done

package/opus/dnn/torch/lpcnet/scripts/make_animation.py

@@ -1,67 +0,0 @@
-"""
-/* Copyright (c) 2023 Amazon
-   Written by Jan Buethe */
-/*
-   Redistribution and use in source and binary forms, with or without
-   modification, are permitted provided that the following conditions
-   are met:
-
-   - Redistributions of source code must retain the above copyright
-   notice, this list of conditions and the following disclaimer.
-
-   - Redistributions in binary form must reproduce the above copyright
-   notice, this list of conditions and the following disclaimer in the
-   documentation and/or other materials provided with the distribution.
-
-   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-   ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
-   OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
-   EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-   PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
-   PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
-   LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
-   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
-"""
-
-
-""" script for creating animations from debug data
-
-"""
-
-
-import argparse
-
-
-import sys
-sys.path.append('./')
-
-from utils.endoscopy import make_animation, read_data
-
-parser = argparse.ArgumentParser()
-
-parser.add_argument('folder', type=str, help='endoscopy folder with debug output')
-parser.add_argument('output', type=str, help='output file (will be auto-extended with .mp4)')
-
-parser.add_argument('--start-index', type=int, help='index of first sample to be considered', default=0)
-parser.add_argument('--stop-index', type=int, help='index of last sample to be considered', default=-1)
-parser.add_argument('--interval', type=int, help='interval between frames in ms', default=20)
-parser.add_argument('--half-window-length', type=int, help='half size of window for displaying signals', default=80)
-
-
-if __name__ == "__main__":
-    args = parser.parse_args()
-
-    filename = args.output if args.output.endswith('.mp4') else args.output + '.mp4'
-    data = read_data(args.folder)
-
-    make_animation(
-        data,
-        filename,
-        start_index=args.start_index,
-        stop_index = args.stop_index,
-        half_signal_window_length=args.half_window_length
-    )

package/opus/dnn/torch/lpcnet/scripts/modify_dataset_target.py

@@ -1,17 +0,0 @@
-import argparse
-
-import numpy as np
-
-
-parser = argparse.ArgumentParser(description="sets s_t to augmented_s_t")
-
-parser.add_argument('datafile', type=str, help='data.s16 file path')
-
-args = parser.parse_args()
-
-data = np.memmap(args.datafile, dtype='int16', mode='readwrite')
-
-# signal is in data[1::2]
-# last augmented signal is in data[0::2]
-
-data[1 : - 1 : 2] = data[2 : : 2]

package/opus/dnn/torch/lpcnet/scripts/multi_run.sh

@@ -1,17 +0,0 @@
-#!/bin/bash
-
-case $# in
-    9) SETUP=$1; OUTDIR=$2; NAME=$3; NUMDEVICES=$4; ROUNDS=$5; LPCNEXT=$6; LPCNET=$7; TESTSUITE=$8; TESTITEMS=$9;;
-    *) echo "multi_run.sh setup outdir name num_devices rounds_per_device lpcnext_repo lpcnet_repo testsuite_repo testitems"; exit;;
-esac
-
-
-LOOPRUN=${LPCNEXT}/loop_run.sh
-
-mkdir -p $OUTDIR
-
-for ((i = 0; i < $NUMDEVICES; i++))
-do
-    echo "launching job queue for device $i"
-    nohup bash $LOOPRUN $SETUP $OUTDIR "$NAME" "cuda:$i" $ROUNDS $LPCNEXT $LPCNET $TESTSUITE $TESTITEMS > $OUTDIR/job_${i}_out.txt &
-done

package/opus/dnn/torch/lpcnet/scripts/run_inference_test.sh

@@ -1,22 +0,0 @@
-#!/bin/bash
-
-
-case $# in
-    3) FEATURES=$1; FOLDER=$2; PYTHON=$3;;
-    *) echo "run_inference_test.sh <features file> <output folder> <python path>"; exit;;
-esac
-
-
-SCRIPTFOLDER=$(dirname "$0")
-
-mkdir -p $FOLDER/inference_test
-
-# update checkpoints
-for fn in $(find $FOLDER -type f -name "checkpoint*.pth")
-do
-    tmp=$(basename $fn)
-    tmp=${tmp%.pth}
-    epoch=${tmp#checkpoint_epoch_}
-    echo "running inference with checkpoint $fn..."
-    $PYTHON $SCRIPTFOLDER/../test_lpcnet.py $FEATURES $fn $FOLDER/inference_test/output_epoch_${epoch}.wav
-done

package/opus/dnn/torch/lpcnet/scripts/update_checkpoints.py

@@ -1,54 +0,0 @@
-"""
-/* Copyright (c) 2023 Amazon
-   Written by Jan Buethe */
-/*
-   Redistribution and use in source and binary forms, with or without
-   modification, are permitted provided that the following conditions
-   are met:
-
-   - Redistributions of source code must retain the above copyright
-   notice, this list of conditions and the following disclaimer.
-
-   - Redistributions in binary form must reproduce the above copyright
-   notice, this list of conditions and the following disclaimer in the
-   documentation and/or other materials provided with the distribution.
-
-   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-   ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
-   OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
-   EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-   PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
-   PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
-   LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
-   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
-"""
-
-""" script for updating checkpoints with new setup entries
-
-    Use this script to update older outputs with newly introduced
-    parameters. (Saves us the trouble of backward compatibility)
-"""
-
-
-import argparse
-
-import torch
-
-parser = argparse.ArgumentParser()
-
-parser.add_argument('checkpoint_file', type=str, help='checkpoint to be updated')
-parser.add_argument('--model', type=str, help='model update', default=None)
-
-args = parser.parse_args()
-
-checkpoint = torch.load(args.checkpoint_file, map_location='cpu')
-
-# update model entry
-if type(args.model) != type(None):
-    checkpoint['setup']['lpcnet']['model'] = args.model
-
-torch.save(checkpoint, args.checkpoint_file)

package/opus/dnn/torch/lpcnet/scripts/update_output_folder.sh

@@ -1,22 +0,0 @@
-#!/bin/bash
-
-
-case $# in
-    3) FOLDER=$1; MODEL=$2; PYTHON=$3;;
-    *) echo "update_output_folder.sh folder model python"; exit;;
-esac
-
-
-SCRIPTFOLDER=$(dirname "$0")
-
-
-# update setup
-echo "updating $FOLDER/setup.py..."
-$PYTHON $SCRIPTFOLDER/update_setups.py $FOLDER/setup.yml --model $MODEL
-
-# update checkpoints
-for fn in $(find $FOLDER -type f -name "checkpoint*.pth")
-do
-    echo "updating $fn..."
-    $PYTHON $SCRIPTFOLDER/update_checkpoints.py $fn --model $MODEL
-done

package/opus/dnn/torch/lpcnet/scripts/update_setups.py

@@ -1,57 +0,0 @@
-"""
-/* Copyright (c) 2023 Amazon
-   Written by Jan Buethe */
-/*
-   Redistribution and use in source and binary forms, with or without
-   modification, are permitted provided that the following conditions
-   are met:
-
-   - Redistributions of source code must retain the above copyright
-   notice, this list of conditions and the following disclaimer.
-
-   - Redistributions in binary form must reproduce the above copyright
-   notice, this list of conditions and the following disclaimer in the
-   documentation and/or other materials provided with the distribution.
-
-   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-   ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
-   OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
-   EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-   PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
-   PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
-   LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
-   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
-"""
-
-""" script for updating setup files with new setup entries
-
-    Use this script to update older outputs with newly introduced
-    parameters. (Saves us the trouble of backward compatibility)
-"""
-
-import argparse
-
-import yaml
-
-parser = argparse.ArgumentParser()
-
-parser.add_argument('setup_file', type=str, help='setup to be updated')
-parser.add_argument('--model', type=str, help='model update', default=None)
-
-args = parser.parse_args()
-
-# load setup
-with open(args.setup_file, 'r') as f:
-    setup = yaml.load(f.read(), yaml.FullLoader)
-
-# update model entry
-if type(args.model) != type(None):
-    setup['lpcnet']['model'] = args.model
-
-# dump result
-with open(args.setup_file, 'w') as f:
-    yaml.dump(setup, f)

package/opus/dnn/torch/lpcnet/test_lpcnet.py

@@ -1,89 +0,0 @@
-"""
-/* Copyright (c) 2023 Amazon
-   Written by Jan Buethe */
-/*
-   Redistribution and use in source and binary forms, with or without
-   modification, are permitted provided that the following conditions
-   are met:
-
-   - Redistributions of source code must retain the above copyright
-   notice, this list of conditions and the following disclaimer.
-
-   - Redistributions in binary form must reproduce the above copyright
-   notice, this list of conditions and the following disclaimer in the
-   documentation and/or other materials provided with the distribution.
-
-   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-   ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
-   OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
-   EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-   PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
-   PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
-   LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
-   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
-"""
-
-import argparse
-
-import torch
-import numpy as np
-
-
-from models import model_dict
-from utils.data import load_features
-from utils.wav import wavwrite16
-
-debug = False
-if debug:
-    args = type('dummy', (object,),
-    {
-        'features'      : 'features.f32',
-        'checkpoint'    : 'checkpoint.pth',
-        'output'        : 'out.wav',
-        'version'       : 2
-    })()
-else:
-    parser = argparse.ArgumentParser()
-
-    parser.add_argument('features', type=str, help='feature file')
-    parser.add_argument('checkpoint', type=str, help='checkpoint file')
-    parser.add_argument('output', type=str, help='output file')
-    parser.add_argument('--version', type=int, help='feature version', default=2)
-
-    args = parser.parse_args()
-
-
-torch.set_num_threads(2)
-
-version = args.version
-feature_file = args.features
-checkpoint_file = args.checkpoint
-
-
-
-output_file = args.output
-if not output_file.endswith('.wav'):
-    output_file += '.wav'
-
-checkpoint = torch.load(checkpoint_file, map_location="cpu")
-
-# check model
-if not 'model' in checkpoint['setup']['lpcnet']:
-    print(f'warning: did not find model entry in setup, using default lpcnet')
-    model_name = 'lpcnet'
-else:
-    model_name = checkpoint['setup']['lpcnet']['model']
-
-model = model_dict[model_name](checkpoint['setup']['lpcnet']['config'])
-
-model.load_state_dict(checkpoint['state_dict'])
-
-data = load_features(feature_file)
-
-output = model.generate(data['features'], data['periods'], data['lpcs'])
-
-wavwrite16(output_file, output.numpy(), 16000)

package/opus/dnn/torch/lpcnet/train_lpcnet.py

@@ -1,272 +0,0 @@
-"""
-/* Copyright (c) 2023 Amazon
-   Written by Jan Buethe */
-/*
-   Redistribution and use in source and binary forms, with or without
-   modification, are permitted provided that the following conditions
-   are met:
-
-   - Redistributions of source code must retain the above copyright
-   notice, this list of conditions and the following disclaimer.
-
-   - Redistributions in binary form must reproduce the above copyright
-   notice, this list of conditions and the following disclaimer in the
-   documentation and/or other materials provided with the distribution.
-
-   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-   ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
-   OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
-   EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-   PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
-   PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
-   LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
-   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
-"""
-
-import os
-import argparse
-import sys
-
-try:
-    import git
-    has_git = True
-except:
-    has_git = False
-
-import yaml
-
-
-import torch
-from torch.optim.lr_scheduler import LambdaLR
-
-from data import LPCNetDataset
-from models import model_dict
-from engine.lpcnet_engine import train_one_epoch, evaluate
-from utils.data import load_features
-from utils.wav import wavwrite16
-
-
-debug = False
-if debug:
-    args = type('dummy', (object,),
-    {
-        'setup' : 'setup.yml',
-        'output' : 'testout',
-        'device' : None,
-        'test_features' : None,
-        'finalize': False,
-        'initial_checkpoint': None,
-        'no-redirect': False
-    })()
-else:
-    parser = argparse.ArgumentParser("train_lpcnet.py")
-    parser.add_argument('setup', type=str, help='setup yaml file')
-    parser.add_argument('output', type=str, help='output path')
-    parser.add_argument('--device', type=str, help='compute device', default=None)
-    parser.add_argument('--test-features', type=str, help='test feature file in v2 format', default=None)
-    parser.add_argument('--finalize', action='store_true', help='run single training round with lr=1e-5')
-    parser.add_argument('--initial-checkpoint', type=str, help='initial checkpoint', default=None)
-    parser.add_argument('--no-redirect', action='store_true', help='disables re-direction of output')
-
-    args = parser.parse_args()
-
-
-torch.set_num_threads(4)
-
-with open(args.setup, 'r') as f:
-    setup = yaml.load(f.read(), yaml.FullLoader)
-
-if args.finalize:
-    if args.initial_checkpoint is None:
-        raise ValueError('finalization requires initial checkpoint')
-
-    if 'sparsification' in setup['lpcnet']['config']:
-        for sp_job in setup['lpcnet']['config']['sparsification'].values():
-            sp_job['start'], sp_job['stop'] = 0, 0
-
-    setup['training']['lr'] = 1.0e-5
-    setup['training']['lr_decay_factor'] = 0.0
-    setup['training']['epochs'] = 1
-
-    checkpoint_prefix = 'checkpoint_finalize'
-    output_prefix = 'output_finalize'
-    setup_name = 'setup_finalize.yml'
-    output_file='out_finalize.txt'
-else:
-    checkpoint_prefix = 'checkpoint'
-    output_prefix = 'output'
-    setup_name = 'setup.yml'
-    output_file='out.txt'
-
-
-# check model
-if not 'model' in setup['lpcnet']:
-    print(f'warning: did not find model entry in setup, using default lpcnet')
-    model_name = 'lpcnet'
-else:
-    model_name = setup['lpcnet']['model']
-
-# prepare output folder
-if os.path.exists(args.output) and not debug and not args.finalize:
-    print("warning: output folder exists")
-
-    reply = input('continue? (y/n): ')
-    while reply not in {'y', 'n'}:
-        reply = input('continue? (y/n): ')
-
-    if reply == 'n':
-        os._exit()
-else:
-    os.makedirs(args.output, exist_ok=True)
-
-checkpoint_dir = os.path.join(args.output, 'checkpoints')
-os.makedirs(checkpoint_dir, exist_ok=True)
-
-
-# add repo info to setup
-if has_git:
-    working_dir = os.path.split(__file__)[0]
-    try:
-        repo = git.Repo(working_dir)
-        setup['repo'] = dict()
-        hash = repo.head.object.hexsha
-        urls = list(repo.remote().urls)
-        is_dirty = repo.is_dirty()
-
-        if is_dirty:
-            print("warning: repo is dirty")
-
-        setup['repo']['hash'] = hash
-        setup['repo']['urls'] = urls
-        setup['repo']['dirty'] = is_dirty
-    except:
-        has_git = False
-
-# dump setup
-with open(os.path.join(args.output, setup_name), 'w') as f:
-    yaml.dump(setup, f)
-
-# prepare inference test if wanted
-run_inference_test = False
-if type(args.test_features) != type(None):
-    test_features = load_features(args.test_features)
-    inference_test_dir = os.path.join(args.output, 'inference_test')
-    os.makedirs(inference_test_dir, exist_ok=True)
-    run_inference_test = True
-
-# training parameters
-batch_size      = setup['training']['batch_size']
-epochs          = setup['training']['epochs']
-lr              = setup['training']['lr']
-lr_decay_factor = setup['training']['lr_decay_factor']
-
-# load training dataset
-lpcnet_config = setup['lpcnet']['config']
-data = LPCNetDataset(   setup['dataset'],
-                        features=lpcnet_config['features'],
-                        input_signals=lpcnet_config['signals'],
-                        target=lpcnet_config['target'],
-                        frames_per_sample=setup['training']['frames_per_sample'],
-                        feature_history=lpcnet_config['feature_history'],
-                        feature_lookahead=lpcnet_config['feature_lookahead'],
-                        lpc_gamma=lpcnet_config.get('lpc_gamma', 1))
-
-# load validation dataset if given
-if 'validation_dataset' in setup:
-    validation_data = LPCNetDataset(   setup['validation_dataset'],
-                        features=lpcnet_config['features'],
-                        input_signals=lpcnet_config['signals'],
-                        target=lpcnet_config['target'],
-                        frames_per_sample=setup['training']['frames_per_sample'],
-                        feature_history=lpcnet_config['feature_history'],
-                        feature_lookahead=lpcnet_config['feature_lookahead'],
-                        lpc_gamma=lpcnet_config.get('lpc_gamma', 1))
-
-    validation_dataloader = torch.utils.data.DataLoader(validation_data, batch_size=batch_size, drop_last=True, num_workers=4)
-
-    run_validation = True
-else:
-    run_validation = False
-
-# create model
-model = model_dict[model_name](setup['lpcnet']['config'])
-
-if args.initial_checkpoint is not None:
-    print(f"loading state dict from {args.initial_checkpoint}...")
-    chkpt = torch.load(args.initial_checkpoint, map_location='cpu')
-    model.load_state_dict(chkpt['state_dict'])
-
-# set compute device
-if type(args.device) == type(None):
-    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-else:
-    device = torch.device(args.device)
-
-# push model to device
-model.to(device)
-
-# dataloader
-dataloader = torch.utils.data.DataLoader(data, batch_size=batch_size, drop_last=True, shuffle=True, num_workers=4)
-
-# optimizer is introduced to trainable parameters
-parameters = [p for p in model.parameters() if p.requires_grad]
-optimizer = torch.optim.Adam(parameters, lr=lr)
-
-# learning rate scheduler
-scheduler = LambdaLR(optimizer=optimizer, lr_lambda=lambda x : 1 / (1 + lr_decay_factor * x))
-
-# loss
-criterion = torch.nn.NLLLoss()
-
-# model checkpoint
-checkpoint = {
-    'setup'         : setup,
-    'state_dict'    : model.state_dict(),
-    'loss'          : -1
-}
-
-if not args.no_redirect:
-    print(f"re-directing output to {os.path.join(args.output, output_file)}")
-    sys.stdout = open(os.path.join(args.output, output_file), "w")
-
-best_loss = 1e9
-
-for ep in range(1, epochs + 1):
-    print(f"training epoch {ep}...")
-    new_loss = train_one_epoch(model, criterion, optimizer, dataloader, device, scheduler)
-
-
-    # save checkpoint
-    checkpoint['state_dict'] = model.state_dict()
-    checkpoint['loss']       = new_loss
-
-    if run_validation:
-        print("running validation...")
-        validation_loss = evaluate(model, criterion, validation_dataloader, device)
-        checkpoint['validation_loss'] = validation_loss
-
-        if validation_loss < best_loss:
-            torch.save(checkpoint, os.path.join(checkpoint_dir, checkpoint_prefix + f'_best.pth'))
-            best_loss = validation_loss
-
-    torch.save(checkpoint, os.path.join(checkpoint_dir, checkpoint_prefix + f'_epoch_{ep}.pth'))
-    torch.save(checkpoint, os.path.join(checkpoint_dir, checkpoint_prefix + f'_last.pth'))
-
-    # run inference test
-    if run_inference_test:
-        model.to("cpu")
-        print("running inference test...")
-
-        output = model.generate(test_features['features'], test_features['periods'], test_features['lpcs'])
-
-        testfilename = os.path.join(inference_test_dir, output_prefix + f'_epoch_{ep}.wav')
-
-        wavwrite16(testfilename, output.numpy(), 16000)
-
-        model.to(device)
-
-    print()

package/opus/dnn/torch/lpcnet/utils/__init__.py

@@ -1,4 +0,0 @@
-from . import sparsification
-from . import data
-from . import pcm
-from . import sample