@scrypted/libav 1.0.167 → 1.0.169

package/opus/dnn/torch/osce/stndrd/evaluation/run_nomad.py

@@ -0,0 +1,138 @@
+"""
+/* Copyright (c) 2023 Amazon
+   Written by Jan Buethe */
+/*
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions
+   are met:
+
+   - Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+
+   - Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+   ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+   OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+   EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+   PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+   PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+   LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+"""
+
+import os
+import argparse
+import tempfile
+import shutil
+
+import pandas as pd
+from scipy.spatial.distance import cdist
+from scipy.io import wavfile
+import numpy as np
+
+from nomad_audio.nomad import Nomad
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument('folder', type=str, help='folder with processed items')
+parser.add_argument('--full-reference', action='store_true', help='use NOMAD as full-reference metric')
+parser.add_argument('--device', type=str, default=None, help='device for Nomad')
+
+
+def get_bitrates(folder):
+    with open(os.path.join(folder, 'bitrates.txt')) as f:
+        x = f.read()
+
+    bitrates = [int(y) for y in x.rstrip('\n').split()]
+
+    return bitrates
+
+def get_itemlist(folder):
+    with open(os.path.join(folder, 'items.txt')) as f:
+        lines = f.readlines()
+
+    items = [x.split()[0] for x in lines]
+
+    return items
+
+
+def nomad_wrapper(ref_folder, deg_folder, full_reference=False, ref_embeddings=None, device=None):
+    model = Nomad(device=device)
+    if not full_reference:
+        results = model.predict(nmr=ref_folder, deg=deg_folder)[0].to_dict()['NOMAD']
+        return results, None
+    else:
+        if ref_embeddings is None:
+            print(f"Computing reference embeddings from {ref_folder}")
+            ref_data = pd.DataFrame(sorted(os.listdir(ref_folder)))
+            ref_data.columns = ['filename']
+            ref_data['filename'] = [os.path.join(ref_folder, x) for x in ref_data['filename']]
+            ref_embeddings = model.get_embeddings_csv(model.model, ref_data).set_index('filename')
+
+        print(f"Computing degraded embeddings from {deg_folder}")
+        deg_data = pd.DataFrame(sorted(os.listdir(deg_folder)))
+        deg_data.columns = ['filename']
+        deg_data['filename'] = [os.path.join(deg_folder, x) for x in deg_data['filename']]
+        deg_embeddings = model.get_embeddings_csv(model.model, deg_data).set_index('filename')
+
+        dist = np.diag(cdist(ref_embeddings, deg_embeddings)) # wasteful
+        test_files = [x.split('/')[-1].split('.')[0] for x in deg_embeddings.index]
+
+        results = dict(zip(test_files, dist))
+
+        return results, ref_embeddings
+
+
+
+
+def nomad_process_all(folder, full_reference=False, device=None):
+    bitrates = get_bitrates(folder)
+    items = get_itemlist(folder)
+    with tempfile.TemporaryDirectory() as dir:
+        cleandir  = os.path.join(dir, 'clean')
+        opusdir   = os.path.join(dir, 'opus')
+        lacedir   = os.path.join(dir, 'lace')
+        nolacedir = os.path.join(dir, 'nolace')
+
+        # prepare files
+        for d in [cleandir, opusdir, lacedir, nolacedir]: os.makedirs(d)
+        for br in bitrates:
+            for item in items:
+                for cond in ['clean', 'opus', 'lace', 'nolace']:
+                    shutil.copyfile(os.path.join(folder, cond, f"{item}_{br}_{cond}.wav"), os.path.join(dir, cond, f"{item}_{br}.wav"))
+
+        nomad_opus, ref_embeddings   = nomad_wrapper(cleandir, opusdir, full_reference=full_reference, ref_embeddings=None)
+        nomad_lace, ref_embeddings   = nomad_wrapper(cleandir, lacedir, full_reference=full_reference, ref_embeddings=ref_embeddings)
+        nomad_nolace, ref_embeddings = nomad_wrapper(cleandir, nolacedir, full_reference=full_reference, ref_embeddings=ref_embeddings)
+
+    results = dict()
+    for br in bitrates:
+        results[br] = np.zeros((len(items), 3))
+        for i, item in enumerate(items):
+            key = f"{item}_{br}"
+            results[br][i, 0] = nomad_opus[key]
+            results[br][i, 1] = nomad_lace[key]
+            results[br][i, 2] = nomad_nolace[key]
+
+    return results
+
+
+
+if __name__ == "__main__":
+    args = parser.parse_args()
+
+    items = get_itemlist(args.folder)
+    bitrates = get_bitrates(args.folder)
+
+    results = nomad_process_all(args.folder, full_reference=args.full_reference, device=args.device)
+
+    np.save(os.path.join(args.folder, f'results_nomad.npy'), results)
+
+    print("Done.")

package/opus/dnn/torch/osce/stndrd/evaluation/run_osce_test.py

@@ -0,0 +1,196 @@
+import os
+import argparse
+import yaml
+import subprocess
+
+import numpy as np
+
+from moc2 import compare as moc
+
+DEBUG=False
+
+parser = argparse.ArgumentParser()
+
+parser.add_argument('inputdir', type=str, help='Input folder with test items')
+parser.add_argument('outputdir', type=str, help='Output folder')
+parser.add_argument('bitrate', type=int, help='bitrate to test')
+parser.add_argument('--reference_opus_demo', type=str, default='./opus_demo', help='reference opus_demo binary for generating bitstreams and reference output')
+parser.add_argument('--encoder_options', type=str, default="", help='encoder options (e.g. -complexity 5)')
+parser.add_argument('--test_opus_demo', type=str, default='./opus_demo', help='opus_demo binary under test')
+parser.add_argument('--test_opus_demo_options', type=str, default='-dec_complexity 7', help='options for test opus_demo (e.g. "-dec_complexity 7")')
+parser.add_argument('--verbose', type=int, default=0, help='verbosity level: 0 for quiet (default), 1 for reporting individual test results, 2 for reporting per-item scores in failed tests')
+
+def run_opus_encoder(opus_demo_path, input_pcm_path, bitstream_path, application, fs, num_channels, bitrate, options=[], verbose=False):
+
+    call_args = [
+        opus_demo_path,
+        "-e",
+        application,
+        str(fs),
+        str(num_channels),
+        str(bitrate),
+        "-bandwidth",
+        "WB"
+    ]
+
+    call_args += options
+
+    call_args += [
+        input_pcm_path,
+        bitstream_path
+    ]
+
+    try:
+        if verbose:
+            print(f"running {call_args}...")
+            subprocess.run(call_args)
+        else:
+            subprocess.run(call_args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+    except:
+        return 1
+
+    return 0
+
+
+def run_opus_decoder(opus_demo_path, bitstream_path, output_pcm_path, fs, num_channels, options=[], verbose=False):
+
+    call_args = [
+        opus_demo_path,
+        "-d",
+        str(fs),
+        str(num_channels)
+    ]
+
+    call_args += options
+
+    call_args += [
+        bitstream_path,
+        output_pcm_path
+    ]
+
+    try:
+        if verbose:
+            print(f"running {call_args}...")
+            subprocess.run(call_args)
+        else:
+            subprocess.run(call_args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+    except:
+        return 1
+
+    return 0
+
+def compute_moc_score(reference_pcm, test_pcm, delay=91):
+    x_ref = np.fromfile(reference_pcm, dtype=np.int16).astype(np.float32) / (2 ** 15)
+    x_cut = np.fromfile(test_pcm, dtype=np.int16).astype(np.float32) / (2 ** 15)
+
+    moc_score = moc(x_ref, x_cut[delay:])
+
+    return moc_score
+
+def sox(*call_args):
+    try:
+        call_args = ["sox"] + list(call_args)
+        subprocess.run(call_args)
+        return 0
+    except:
+        return 1
+
+def process_clip_factory(ref_opus_demo, test_opus_demo, enc_options, test_options):
+    def process_clip(clip_path, processdir, bitrate):
+        # derive paths
+        clipname = os.path.splitext(os.path.split(clip_path)[1])[0]
+        pcm_path = os.path.join(processdir, clipname + ".raw")
+        bitstream_path = os.path.join(processdir, clipname + ".bin")
+        ref_path = os.path.join(processdir, clipname + "_ref.raw")
+        test_path = os.path.join(processdir, clipname + "_test.raw")
+
+        # run sox
+        sox(clip_path, pcm_path)
+
+        # run encoder
+        run_opus_encoder(ref_opus_demo, pcm_path, bitstream_path, "voip", 16000, 1, bitrate, enc_options)
+
+        # run decoder
+        run_opus_decoder(ref_opus_demo, bitstream_path, ref_path, 16000, 1)
+        run_opus_decoder(test_opus_demo, bitstream_path, test_path, 16000, 1, options=test_options)
+
+        d_ref  = compute_moc_score(pcm_path, ref_path)
+        d_test = compute_moc_score(pcm_path, test_path)
+
+        return d_ref, d_test
+
+
+    return process_clip
+
+def main(inputdir, outputdir, bitrate, reference_opus_demo, test_opus_demo, enc_option_string, test_option_string, verbose):
+
+    # load clips list
+    with open(os.path.join(inputdir, 'clips.yml'), "r") as f:
+        clips = yaml.safe_load(f)
+
+    # parse test options
+    enc_options = enc_option_string.split()
+    test_options = test_option_string.split()
+
+    process_clip = process_clip_factory(reference_opus_demo, test_opus_demo, enc_options, test_options)
+
+    os.makedirs(outputdir, exist_ok=True)
+    processdir = os.path.join(outputdir, 'process')
+    os.makedirs(processdir, exist_ok=True)
+
+    num_passed = 0
+    results = dict()
+    min_rel_diff = 1000
+    min_mean = 1000
+    worst_clip = None
+    worst_lang = None
+    for lang, lang_clips in clips.items():
+        if verbose > 0: print(f"processing language {lang}...")
+        results[lang] = np.zeros((len(lang_clips), 2))
+        for i, clip in enumerate(lang_clips):
+            clip_path = os.path.join(inputdir, clip)
+            d_ref, d_test = process_clip(clip_path, processdir, bitrate)
+            results[lang][i, 0] = d_ref
+            results[lang][i, 1] = d_test
+
+        alpha = 0.5
+        rel_diff = ((results[lang][:, 0] ** alpha - results[lang][:, 1] ** alpha) /(results[lang][:, 0] ** alpha))
+
+        min_idx = np.argmin(rel_diff).item()
+        if rel_diff[min_idx] < min_rel_diff:
+            min_rel_diff = rel_diff[min_idx]
+            worst_clip = lang_clips[min_idx]
+
+        if np.mean(rel_diff) < min_mean:
+            min_mean = np.mean(rel_diff).item()
+            worst_lang = lang
+
+        if np.min(rel_diff) < -0.1 or np.mean(rel_diff) < -0.025:
+            if verbose > 0: print(f"FAIL ({np.mean(results[lang], axis=0)} {np.mean(rel_diff)} {np.min(rel_diff)})")
+            if verbose > 1:
+                for i, c in enumerate(lang_clips):
+                    print(f"    {c:50s} {results[lang][i]} {rel_diff[i]}")
+        else:
+            if verbose > 0: print(f"PASS ({np.mean(results[lang], axis=0)} {np.mean(rel_diff)} {np.min(rel_diff)})")
+            num_passed += 1
+
+    print(f"{num_passed}/{len(clips)} tests passed!")
+
+    print(f"worst case occured at clip {worst_clip} with relative difference of {min_rel_diff}")
+    print(f"worst mean relative difference was {min_mean} for test {worst_lang}")
+
+    np.save(os.path.join(outputdir, f'results_' + "_".join(test_options) + f"_{bitrate}.npy"), results, allow_pickle=True)
+
+
+
+if __name__ == "__main__":
+    args = parser.parse_args()
+
+    main(args.inputdir,
+         args.outputdir,
+         args.bitrate,
+         args.reference_opus_demo,
+         args.test_opus_demo,
+         args.encoder_options,
+         args.test_opus_demo_options,
+         args.verbose)

package/opus/dnn/torch/osce/stndrd/presentation/endoscopy.py

@@ -0,0 +1,205 @@
+""" module for inspecting models during inference """
+
+import os
+
+import yaml
+import matplotlib.pyplot as plt
+import matplotlib.animation as animation
+
+import torch
+import numpy as np
+
+# stores entries {key : {'fid' : fid, 'fs' : fs, 'dim' : dim, 'dtype' : dtype}}
+_state = dict()
+_folder = 'endoscopy'
+
+def get_gru_gates(gru, input, state):
+    hidden_size = gru.hidden_size
+
+    direct = torch.matmul(gru.weight_ih_l0, input.squeeze())
+    recurrent = torch.matmul(gru.weight_hh_l0, state.squeeze())
+
+    # reset gate
+    start, stop = 0 * hidden_size, 1 * hidden_size
+    reset_gate = torch.sigmoid(direct[start : stop] + gru.bias_ih_l0[start : stop] + recurrent[start : stop] + gru.bias_hh_l0[start : stop])
+
+    # update gate
+    start, stop = 1 * hidden_size, 2 * hidden_size
+    update_gate = torch.sigmoid(direct[start : stop] + gru.bias_ih_l0[start : stop] + recurrent[start : stop] + gru.bias_hh_l0[start : stop])
+
+    # new gate
+    start, stop = 2 * hidden_size, 3 * hidden_size
+    new_gate = torch.tanh(direct[start : stop] + gru.bias_ih_l0[start : stop] + reset_gate * (recurrent[start : stop] +  gru.bias_hh_l0[start : stop]))
+
+    return {'reset_gate' : reset_gate, 'update_gate' : update_gate, 'new_gate' : new_gate}
+
+
+def init(folder='endoscopy'):
+    """ sets up output folder for endoscopy data """
+
+    global _folder
+    _folder = folder
+
+    if not os.path.exists(folder):
+        os.makedirs(folder)
+    else:
+        print(f"warning: endoscopy folder {folder} exists. Content may be lost or inconsistent results may occur.")
+
+def write_data(key, data, fs):
+    """ appends data to previous data written under key """
+
+    global _state
+
+    # convert to numpy if torch.Tensor is given
+    if isinstance(data, torch.Tensor):
+        data = data.detach().numpy()
+
+    if not key in _state:
+        _state[key] = {
+            'fid'   : open(os.path.join(_folder, key + '.bin'), 'wb'),
+            'fs'    : fs,
+            'dim'   : tuple(data.shape),
+            'dtype' : str(data.dtype)
+        }
+
+        with open(os.path.join(_folder, key + '.yml'), 'w') as f:
+            f.write(yaml.dump({'fs' : fs, 'dim' : tuple(data.shape), 'dtype' : str(data.dtype).split('.')[-1]}))
+    else:
+        if _state[key]['fs'] != fs:
+            raise ValueError(f"fs changed for key {key}: {_state[key]['fs']} vs. {fs}")
+        if _state[key]['dtype'] != str(data.dtype):
+            raise ValueError(f"dtype changed for key {key}: {_state[key]['dtype']} vs. {str(data.dtype)}")
+        if _state[key]['dim'] != tuple(data.shape):
+            raise ValueError(f"dim changed for key {key}: {_state[key]['dim']} vs. {tuple(data.shape)}")
+
+    _state[key]['fid'].write(data.tobytes())
+
+def close(folder='endoscopy'):
+    """ clean up """
+    for key in _state.keys():
+        _state[key]['fid'].close()
+
+
+def read_data(folder='endoscopy'):
+    """ retrieves written data as numpy arrays """
+
+
+    keys = [name[:-4] for name in os.listdir(folder) if name.endswith('.yml')]
+
+    return_dict = dict()
+
+    for key in keys:
+        with open(os.path.join(folder, key + '.yml'), 'r') as f:
+            value = yaml.load(f.read(), yaml.FullLoader)
+
+        with open(os.path.join(folder, key + '.bin'), 'rb') as f:
+            data = np.frombuffer(f.read(), dtype=value['dtype'])
+
+        value['data'] = data.reshape((-1,) + value['dim'])
+
+        return_dict[key] = value
+
+    return return_dict
+
+def get_best_reshape(shape, target_ratio=1):
+    """ calculated the best 2d reshape of shape given the target ratio (rows/cols)"""
+
+    if len(shape) > 1:
+        pixel_count = 1
+        for s in shape:
+            pixel_count *= s
+    else:
+        pixel_count = shape[0]
+
+    if pixel_count == 1:
+        return (1,)
+
+    num_columns = int((pixel_count / target_ratio)**.5)
+
+    while (pixel_count % num_columns):
+        num_columns -= 1
+
+    num_rows = pixel_count // num_columns
+
+    return (num_rows, num_columns)
+
+def get_type_and_shape(shape):
+
+    # can happen if data is one dimensional
+    if len(shape) == 0:
+        shape = (1,)
+
+    # calculate pixel count
+    if len(shape) > 1:
+        pixel_count = 1
+        for s in shape:
+            pixel_count *= s
+    else:
+        pixel_count = shape[0]
+
+    if pixel_count == 1:
+        return 'plot', (1, )
+
+    # stay with shape if already 2-dimensional
+    if len(shape) == 2:
+        if (shape[0] != pixel_count) or (shape[1] != pixel_count):
+            return 'image', shape
+
+    return 'image', get_best_reshape(shape)
+
+def make_animation(data, filename, start_index=80, stop_index=-80, interval=20, half_signal_window_length=80):
+
+    # determine plot setup
+    num_keys = len(data.keys())
+
+    num_rows = int((num_keys * 3/4) ** .5)
+
+    num_cols = (num_keys + num_rows - 1) // num_rows
+
+    fig, axs = plt.subplots(num_rows, num_cols)
+    fig.set_size_inches(num_cols * 5, num_rows * 5)
+
+    display = dict()
+
+    fs_max = max([val['fs'] for val in data.values()])
+
+    num_samples = max([val['data'].shape[0] for val in data.values()])
+
+    keys = sorted(data.keys())
+
+    # inspect data
+    for i, key in enumerate(keys):
+        axs[i // num_cols, i % num_cols].title.set_text(key)
+
+        display[key] = dict()
+
+        display[key]['type'], display[key]['shape'] = get_type_and_shape(data[key]['dim'])
+        display[key]['down_factor'] = data[key]['fs'] / fs_max
+
+    start_index = max(start_index, half_signal_window_length)
+    while stop_index < 0:
+        stop_index += num_samples
+
+    stop_index = min(stop_index, num_samples - half_signal_window_length)
+
+    # actual plotting
+    frames = []
+    for index in range(start_index, stop_index):
+        ims = []
+        for i, key in enumerate(keys):
+            feature_index = int(round(index * display[key]['down_factor']))
+
+            if display[key]['type'] == 'plot':
+                ims.append(axs[i // num_cols, i % num_cols].plot(data[key]['data'][index - half_signal_window_length : index + half_signal_window_length], marker='P', markevery=[half_signal_window_length], animated=True, color='blue')[0])
+
+            elif display[key]['type'] == 'image':
+                ims.append(axs[i // num_cols, i % num_cols].imshow(data[key]['data'][index].reshape(display[key]['shape']), animated=True))
+
+        frames.append(ims)
+
+    ani = animation.ArtistAnimation(fig, frames, interval=interval, blit=True, repeat_delay=1000)
+
+    if not filename.endswith('.mp4'):
+        filename += '.mp4'
+
+    ani.save(filename)