@scrypted/libav 1.0.167 → 1.0.169

package/opus/dnn/torch/neural-pitch/data_augmentation.py

@@ -0,0 +1,149 @@
+"""
+Perform Data Augmentation (Gain, Additive Noise, Random Filtering) on Input TTS Data
+1. Read in chunks and compute clean pitch first
+2. Then add in augmentation (Noise/Level/Response)
+    - Adds filtered noise from the "Demand" dataset, https://zenodo.org/record/1227121#.XRKKxYhKiUk
+    - When using the Demand Dataset, consider each channel as a possible noise input, and keep the first 4 minutes of noise for training
+3. Use this "augmented" audio for feature computation, and compute pitch using CREPE on the clean input
+
+Notes: To ensure consistency with the discovered CREPE offset, we do the following
+- We pad the input audio to the zero-centered CREPE estimator with 80 zeros
+- We pad the input audio to our feature computation with 160 zeros to center them
+"""
+
+import argparse
+parser = argparse.ArgumentParser()
+
+parser.add_argument('data', type=str, help='input raw audio data')
+parser.add_argument('output', type=str, help='output directory')
+parser.add_argument('path_lpcnet_extractor', type=str, help='path to LPCNet extractor object file (generated on compilation)')
+parser.add_argument('noise_dataset', type=str, help='Location of the Demand Datset')
+parser.add_argument('--flag_xcorr', type=bool, help='Flag to additionally dump xcorr features',choices=[True,False],default = False,required = False)
+parser.add_argument('--fraction_input_use', type=float, help='Fraction of input data to consider',default = 0.3,required = False)
+parser.add_argument('--gpu_index', type=int, help='GPU index to use if multiple GPUs',default = 0,required = False)
+parser.add_argument('--choice_augment', type=str, help='Choice of noise augmentation, either use additive synthetic noise or add noise from the demand dataset',choices = ['demand','synthetic'],default = "demand",required = False)
+parser.add_argument('--fraction_clean', type=float, help='Fraction of data to keep clean (that is not augment with anything)',default = 0.2,required = False)
+parser.add_argument('--chunk_size', type=int, help='Number of samples to augment with for each iteration',default = 80000,required = False)
+parser.add_argument('--N', type=int, help='STFT window size',default = 320,required = False)
+parser.add_argument('--H', type=int, help='STFT Hop size',default = 160,required = False)
+parser.add_argument('--freq_keep', type=int, help='Number of Frequencies to keep',default = 30,required = False)
+
+args = parser.parse_args()
+
+import os
+os.environ["CUDA_VISIBLE_DEVICES"] = str(args.gpu_index)
+
+from utils import stft, random_filter
+
+import numpy as np
+import tqdm
+import crepe
+import random
+import glob
+import subprocess
+
+data_full = np.memmap(args.data, dtype=np.int16,mode = 'r')
+data = data_full[:(int)(args.fraction_input_use*data_full.shape[0])]
+
+# list_features = []
+list_cents = []
+list_confidences = []
+
+N = args.N
+H = args.H
+freq_keep = args.freq_keep
+# Minimum/Maximum periods, decided by LPCNet
+min_period = 32
+max_period = 256
+f_ref = 16000/max_period
+chunk_size = args.chunk_size
+num_frames_chunk = chunk_size//H
+list_indices_keep = np.concatenate([np.arange(freq_keep), (N//2 + 1) + np.arange(freq_keep), 2*(N//2 + 1) + np.arange(freq_keep)])
+
+output_IF  = np.memmap(args.output + '_iffeat.f32', dtype=np.float32, shape=(((data.shape[0]//chunk_size - 1)//1)*num_frames_chunk,list_indices_keep.shape[0]), mode='w+')
+if args.flag_xcorr:
+    output_xcorr  = np.memmap(args.output + '_xcorr.f32', dtype=np.float32, shape=(((data.shape[0]//chunk_size - 1)//1)*num_frames_chunk,257), mode='w+')
+
+fraction_clean = args.fraction_clean
+
+noise_dataset = args.noise_dataset
+
+for i in tqdm.trange((data.shape[0]//chunk_size - 1)//1):
+    chunk = data[i*chunk_size:(i + 1)*chunk_size]/(2**15 - 1)
+
+    # Clean Pitch/Confidence Estimate
+    # Padding input to CREPE by 80 samples to ensure it aligns
+    _, pitch, confidence, _ = crepe.predict(np.concatenate([np.zeros(80),chunk]), 16000, center=True, viterbi=True,verbose=0)
+    cent = 1200*np.log2(np.divide(pitch, f_ref, out=np.zeros_like(pitch), where=pitch!=0) + 1.0e-8)
+
+    # Filter out of range pitches/confidences
+    confidence[pitch < 16000/max_period] = 0
+    confidence[pitch > 16000/min_period] = 0
+
+    # Keep fraction of data clean, augment only 1 minus the fraction
+    if (np.random.rand() > fraction_clean):
+        # Response, generate controlled/random 2nd order IIR filter and filter chunk
+        chunk = random_filter(chunk)
+
+        # Level/Gain response {scale by random gain between 1.0e-3 and 10}
+        # Generate random gain in dB and then convert to scale
+        g_dB = np.random.uniform(low =  -60, high = 20, size = 1)
+        # g_dB = 0
+        g = 10**(g_dB/20)
+
+        # Noise Addition {Add random SNR 2nd order randomly colored noise}
+        # Generate noise SNR value and add corresponding noise
+        snr_dB = np.random.uniform(low =  -20, high = 30, size = 1)
+
+        if args.choice_augment == 'synthetic':
+            n = np.random.randn(chunk_size)
+        else:
+            list_noisefiles = noise_dataset + '*.wav'
+            noise_file = random.choice(glob.glob(list_noisefiles))
+            n = np.memmap(noise_file, dtype=np.int16,mode = 'r')/(2**15 - 1)
+            rand_range = np.random.randint(low = 0, high = (n.shape[0] - 16000*60 - chunk.shape[0])) # 16000 is subtracted because we will use the last 1 minutes of noise for testing
+            n = n[rand_range:rand_range + chunk.shape[0]]
+
+        # Randomly filter the sampled noise as well
+        n = random_filter(n)
+        # generate random prime number between 0,500 and make those samples of noise 0 (to prevent GRU from picking up temporal patterns)
+        Nprime = random.choice([2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131, 137, 139, 149, 151, 157, 163, 167, 173, 179, 181, 191, 193, 197, 199, 211, 223, 227, 229, 233, 239, 241, 251, 257, 263, 269, 271, 277, 281, 283, 293, 307, 311, 313, 317, 331, 337, 347, 349, 353, 359, 367, 373, 379, 383, 389, 397, 401, 409, 419, 421, 431, 433, 439, 443, 449, 457, 461, 463, 467, 479, 487, 491, 499, 503, 509, 521, 523, 541])
+        n[chunk_size - Nprime:] = np.zeros(Nprime)
+        snr_multiplier = np.sqrt((np.sum(np.abs(chunk)**2)/np.sum(np.abs(n)**2))*10**(-snr_dB/10))
+
+        chunk = g*(chunk + snr_multiplier*n)
+
+    # Zero pad input audio by 160 to center the frames
+    spec = stft(x = np.concatenate([np.zeros(160),chunk]), w = 'boxcar', N = N, H = H).T
+    phase_diff = spec*np.conj(np.roll(spec,1,axis = -1))
+    phase_diff = phase_diff/(np.abs(phase_diff) + 1.0e-8)
+    feature = np.concatenate([np.log(np.abs(spec) + 1.0e-8),np.real(phase_diff),np.imag(phase_diff)],axis = 0).T
+    feature = feature[:,list_indices_keep]
+
+    if args.flag_xcorr:
+        # Dump noisy audio into temp file
+        data_temp = np.memmap('./temp_augment.raw', dtype=np.int16, shape=(chunk.shape[0]), mode='w+')
+        # data_temp[:chunk.shape[0]] = (chunk/(np.max(np.abs(chunk)))*(2**15 - 1)).astype(np.int16)
+        data_temp[:chunk.shape[0]] = ((chunk)*(2**15 - 1)).astype(np.int16)
+
+        subprocess.run([args.path_lpcnet_extractor, './temp_augment.raw', './temp_augment_xcorr.f32'])
+        feature_xcorr = np.flip(np.fromfile('./temp_augment_xcorr.f32', dtype='float32').reshape((-1,256),order = 'C'),axis = 1)
+        ones_zero_lag = np.expand_dims(np.ones(feature_xcorr.shape[0]),-1)
+        feature_xcorr = np.concatenate([ones_zero_lag,feature_xcorr],axis = -1)
+
+        os.remove('./temp_augment.raw')
+        os.remove('./temp_augment_xcorr.f32')
+    num_frames = min(cent.shape[0],feature.shape[0],feature_xcorr.shape[0],num_frames_chunk)
+    feature = feature[:num_frames,:]
+    cent = cent[:num_frames]
+    confidence = confidence[:num_frames]
+    feature_xcorr = feature_xcorr[:num_frames]
+    output_IF[i*num_frames_chunk:(i + 1)*num_frames_chunk,:] = feature
+    output_xcorr[i*num_frames_chunk:(i + 1)*num_frames_chunk,:] = feature_xcorr
+    list_cents.append(cent)
+    list_confidences.append(confidence)
+
+list_cents = np.hstack(list_cents)
+list_confidences = np.hstack(list_confidences)
+
+np.save(args.output + '_pitches',np.vstack([list_cents,list_confidences]))

package/opus/dnn/torch/neural-pitch/download_demand.sh

@@ -0,0 +1,43 @@
+wget https://zenodo.org/record/1227121/files/DKITCHEN_16k.zip
+
+wget https://zenodo.org/record/1227121/files/DLIVING_16k.zip
+
+wget https://zenodo.org/record/1227121/files/DWASHING_16k.zip
+
+wget https://zenodo.org/record/1227121/files/NFIELD_16k.zip
+
+wget https://zenodo.org/record/1227121/files/NPARK_16k.zip
+
+wget https://zenodo.org/record/1227121/files/NRIVER_16k.zip
+
+wget https://zenodo.org/record/1227121/files/OHALLWAY_16k.zip
+
+wget https://zenodo.org/record/1227121/files/OMEETING_16k.zip
+
+wget https://zenodo.org/record/1227121/files/OOFFICE_16k.zip
+
+wget https://zenodo.org/record/1227121/files/PCAFETER_16k.zip
+
+wget https://zenodo.org/record/1227121/files/PRESTO_16k.zip
+
+wget https://zenodo.org/record/1227121/files/PSTATION_16k.zip
+
+wget https://zenodo.org/record/1227121/files/TMETRO_16k.zip
+
+wget https://zenodo.org/record/1227121/files/TCAR_16k.zip
+
+wget https://zenodo.org/record/1227121/files/TBUS_16k.zip
+
+wget https://zenodo.org/record/1227121/files/STRAFFIC_16k.zip
+
+wget https://zenodo.org/record/1227121/files/SPSQUARE_16k.zip
+
+unzip '*.zip'
+
+mkdir -p ./combined_demand_channels/
+for file in */*.wav; do
+parentdir="$(dirname "$file")"
+echo $parentdir
+fname="$(basename "$file")"
+cp $file ./combined_demand_channels/$parentdir+$fname
+done

package/opus/dnn/torch/neural-pitch/evaluation.py

@@ -0,0 +1,349 @@
+"""
+Evaluation script to compute the Raw Pitch Accuracy
+Procedure:
+    - Look at all voiced frames in file
+    - Compute number of pitches in those frames that lie within a 50 cent threshold
+    RPA = (Total number of pitches within threshold summed across all files)/(Total number of voiced frames summed accross all files)
+"""
+
+import os
+os.environ["CUDA_VISIBLE_DEVICES"] = "0"
+
+from prettytable import PrettyTable
+import numpy as np
+import glob
+import random
+import tqdm
+import torch
+import librosa
+import json
+from utils import stft, random_filter, feature_xform
+import subprocess
+import crepe
+
+from models import PitchDNN, PitchDNNIF, PitchDNNXcorr
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+def rca(reference,input,voicing,thresh = 25):
+    idx_voiced = np.where(voicing != 0)[0]
+    acc = np.where(np.abs(reference - input)[idx_voiced] < thresh)[0]
+    return acc.shape[0]
+
+def sweep_rca(reference,input,voicing,thresh = 25,ind_arr = np.arange(-10,10)):
+    l = []
+    for i in ind_arr:
+        l.append(rca(reference,np.roll(input,i),voicing,thresh))
+    l = np.array(l)
+
+    return np.max(l)
+
+def rpa(model,device = 'cpu',data_format = 'if'):
+    list_files = glob.glob('/home/ubuntu/Code/Datasets/SPEECH DATA/combined_mic_16k_raw/*.raw')
+    dir_f0 = '/home/ubuntu/Code/Datasets/SPEECH DATA/combine_f0_ptdb/'
+    # random_shuffle = list(np.random.permutation(len(list_files)))
+    random.shuffle(list_files)
+    list_files = list_files[:1000]
+
+    C_all = 0
+    C_all_m = 0
+    C_all_f = 0
+    list_rca_model_all = []
+    list_rca_male_all = []
+    list_rca_female_all = []
+
+    thresh = 50
+    N = 320
+    H = 160
+    freq_keep = 30
+
+    for idx in tqdm.trange(len(list_files)):
+        audio_file = list_files[idx]
+        file_name = os.path.basename(list_files[idx])[:-4]
+
+        audio = np.memmap(list_files[idx], dtype=np.int16)/(2**15 - 1)
+        offset = 432
+        audio = audio[offset:]
+        rmse = np.squeeze(librosa.feature.rms(y = audio,frame_length = 320,hop_length = 160))
+
+        spec = stft(x = np.concatenate([np.zeros(160),audio]), w = 'boxcar', N = N, H = H).T
+        phase_diff = spec*np.conj(np.roll(spec,1,axis = -1))
+        phase_diff = phase_diff/(np.abs(phase_diff) + 1.0e-8)
+        idx_save = np.concatenate([np.arange(freq_keep),(N//2 + 1) + np.arange(freq_keep),2*(N//2 + 1) + np.arange(freq_keep)])
+        feature = np.concatenate([np.log(np.abs(spec) + 1.0e-8),np.real(phase_diff),np.imag(phase_diff)],axis = 0).T
+        feature_if = feature[:,idx_save]
+
+        data_temp = np.memmap('./temp.raw', dtype=np.int16, shape=(audio.shape[0]), mode='w+')
+        data_temp[:audio.shape[0]] = (audio/(np.max(np.abs(audio)))*(2**15 - 1)).astype(np.int16)
+
+        subprocess.run(["../../../lpcnet_xcorr_extractor", './temp.raw', './temp_xcorr.f32'])
+        feature_xcorr = np.flip(np.fromfile('./temp_xcorr.f32', dtype='float32').reshape((-1,256),order = 'C'),axis = 1)
+        ones_zero_lag = np.expand_dims(np.ones(feature_xcorr.shape[0]),-1)
+        feature_xcorr = np.concatenate([ones_zero_lag,feature_xcorr],axis = -1)
+        # feature_xcorr = feature_xform(feature_xcorr)
+
+        os.remove('./temp.raw')
+        os.remove('./temp_xcorr.f32')
+
+        if data_format == 'if':
+            feature = feature_if
+        elif data_format == 'xcorr':
+            feature = feature_xcorr
+        else:
+            indmin = min(feature_if.shape[0],feature_xcorr.shape[0])
+            feature = np.concatenate([feature_xcorr[:indmin,:],feature_if[:indmin,:]],-1)
+
+
+        pitch_file_name = dir_f0 + "ref" + os.path.basename(list_files[idx])[3:-4] + ".f0"
+        pitch = np.loadtxt(pitch_file_name)[:,0]
+        voicing = np.loadtxt(pitch_file_name)[:,1]
+        indmin = min(voicing.shape[0],rmse.shape[0],pitch.shape[0])
+        pitch = pitch[:indmin]
+        voicing = voicing[:indmin]
+        rmse = rmse[:indmin]
+        voicing = voicing*(rmse > 0.05*np.max(rmse))
+        if "mic_F" in audio_file:
+            idx_correct = np.where(pitch < 125)
+            voicing[idx_correct] = 0
+
+        cent = np.rint(1200*np.log2(np.divide(pitch, (16000/256), out=np.zeros_like(pitch), where=pitch!=0) + 1.0e-8)).astype('int')
+
+
+        model_cents = model(torch.from_numpy(np.copy(np.expand_dims(feature,0))).float().to(device))
+        model_cents = 20*model_cents.argmax(dim=1).cpu().detach().squeeze().numpy()
+
+        num_frames = min(cent.shape[0],model_cents.shape[0])
+        pitch = pitch[:num_frames]
+        cent = cent[:num_frames]
+        voicing = voicing[:num_frames]
+        model_cents = model_cents[:num_frames]
+
+        voicing_all = np.copy(voicing)
+        # Forcefully make regions where pitch is <65 or greater than 500 unvoiced for relevant accurate pitch comparisons for our model
+        force_out_of_pitch = np.where(np.logical_or(pitch < 65,pitch > 500)==True)
+        voicing_all[force_out_of_pitch] = 0
+        C_all = C_all + np.where(voicing_all != 0)[0].shape[0]
+
+        list_rca_model_all.append(rca(cent,model_cents,voicing_all,thresh))
+
+        if "mic_M" in audio_file:
+            list_rca_male_all.append(rca(cent,model_cents,voicing_all,thresh))
+            C_all_m = C_all_m + np.where(voicing_all != 0)[0].shape[0]
+        else:
+            list_rca_female_all.append(rca(cent,model_cents,voicing_all,thresh))
+            C_all_f = C_all_f + np.where(voicing_all != 0)[0].shape[0]
+
+    list_rca_model_all = np.array(list_rca_model_all)
+    list_rca_male_all = np.array(list_rca_male_all)
+    list_rca_female_all = np.array(list_rca_female_all)
+
+
+    x = PrettyTable()
+
+    x.field_names = ["Experiment", "Mean RPA"]
+    x.add_row(["Both all pitches", np.sum(list_rca_model_all)/C_all])
+
+    x.add_row(["Male all pitches", np.sum(list_rca_male_all)/C_all_m])
+
+    x.add_row(["Female all pitches", np.sum(list_rca_female_all)/C_all_f])
+
+    print(x)
+
+    return None
+
+def cycle_eval(checkpoint_list, noise_type = 'synthetic', noise_dataset = None, list_snr = [-20,-15,-10,-5,0,5,10,15,20], ptdb_dataset_path = None,fraction = 0.1,thresh = 50):
+    """
+    Cycle through SNR evaluation for list of checkpoints
+    """
+    list_files = glob.glob(ptdb_dataset_path + 'combined_mic_16k/*.raw')
+    dir_f0 = ptdb_dataset_path + 'combined_reference_f0/'
+    random.shuffle(list_files)
+    list_files = list_files[:(int)(fraction*len(list_files))]
+
+    dict_models = {}
+    list_snr.append(np.inf)
+
+    for f in checkpoint_list:
+        if (f!='crepe') and (f!='lpcnet'):
+
+            checkpoint = torch.load(f, map_location='cpu')
+            dict_params = checkpoint['config']
+            if dict_params['data_format'] == 'if':
+                from models import large_if_ccode as model
+                pitch_nn = PitchDNNIF(dict_params['freq_keep']*3,dict_params['gru_dim'],dict_params['output_dim'])
+            elif dict_params['data_format'] == 'xcorr':
+                from models import large_xcorr as model
+                pitch_nn = PitchDNNXcorr(dict_params['xcorr_dim'],dict_params['gru_dim'],dict_params['output_dim'])
+            else:
+                from models import large_joint as model
+                pitch_nn = PitchDNN(dict_params['freq_keep']*3,dict_params['xcorr_dim'],dict_params['gru_dim'],dict_params['output_dim'])
+
+            pitch_nn.load_state_dict(checkpoint['state_dict'])
+
+            N = dict_params['window_size']
+            H = dict_params['hop_factor']
+            freq_keep = dict_params['freq_keep']
+
+            list_mean = []
+            list_std = []
+            for snr_dB in list_snr:
+                C_all = 0
+                C_correct = 0
+                for idx in tqdm.trange(len(list_files)):
+                    audio_file = list_files[idx]
+                    file_name = os.path.basename(list_files[idx])[:-4]
+
+                    audio = np.memmap(list_files[idx], dtype=np.int16)/(2**15 - 1)
+                    offset = 432
+                    audio = audio[offset:]
+                    rmse = np.squeeze(librosa.feature.rms(y = audio,frame_length = N,hop_length = H))
+
+                    if noise_type != 'synthetic':
+                        list_noisefiles = noise_dataset + '*.wav'
+                        noise_file = random.choice(glob.glob(list_noisefiles))
+                        n = np.memmap(noise_file, dtype=np.int16,mode = 'r')/(2**15 - 1)
+                        rand_range = np.random.randint(low = 0, high = (16000*60*5 - audio.shape[0])) # Last 1 minute of noise used for testing
+                        n = n[rand_range:rand_range + audio.shape[0]]
+                    else:
+                        n = np.random.randn(audio.shape[0])
+                        n = random_filter(n)
+
+                    snr_multiplier = np.sqrt((np.sum(np.abs(audio)**2)/np.sum(np.abs(n)**2))*10**(-snr_dB/10))
+                    audio = audio + snr_multiplier*n
+
+                    spec = stft(x = np.concatenate([np.zeros(160),audio]), w = 'boxcar', N = N, H = H).T
+                    phase_diff = spec*np.conj(np.roll(spec,1,axis = -1))
+                    phase_diff = phase_diff/(np.abs(phase_diff) + 1.0e-8)
+                    idx_save = np.concatenate([np.arange(freq_keep),(N//2 + 1) + np.arange(freq_keep),2*(N//2 + 1) + np.arange(freq_keep)])
+                    feature = np.concatenate([np.log(np.abs(spec) + 1.0e-8),np.real(phase_diff),np.imag(phase_diff)],axis = 0).T
+                    feature_if = feature[:,idx_save]
+
+                    data_temp = np.memmap('./temp.raw', dtype=np.int16, shape=(audio.shape[0]), mode='w+')
+                    # data_temp[:audio.shape[0]] = (audio/(np.max(np.abs(audio)))*(2**15 - 1)).astype(np.int16)
+                    data_temp[:audio.shape[0]] = ((audio)*(2**15 - 1)).astype(np.int16)
+
+                    subprocess.run(["../../../lpcnet_xcorr_extractor", './temp.raw', './temp_xcorr.f32'])
+                    feature_xcorr = np.flip(np.fromfile('./temp_xcorr.f32', dtype='float32').reshape((-1,256),order = 'C'),axis = 1)
+                    ones_zero_lag = np.expand_dims(np.ones(feature_xcorr.shape[0]),-1)
+                    feature_xcorr = np.concatenate([ones_zero_lag,feature_xcorr],axis = -1)
+
+                    os.remove('./temp.raw')
+                    os.remove('./temp_xcorr.f32')
+
+                    if dict_params['data_format'] == 'if':
+                        feature = feature_if
+                    elif dict_params['data_format'] == 'xcorr':
+                        feature = feature_xcorr
+                    else:
+                        indmin = min(feature_if.shape[0],feature_xcorr.shape[0])
+                        feature = np.concatenate([feature_xcorr[:indmin,:],feature_if[:indmin,:]],-1)
+
+                    pitch_file_name = dir_f0 + "ref" + os.path.basename(list_files[idx])[3:-4] + ".f0"
+                    pitch = np.loadtxt(pitch_file_name)[:,0]
+                    voicing = np.loadtxt(pitch_file_name)[:,1]
+                    indmin = min(voicing.shape[0],rmse.shape[0],pitch.shape[0])
+                    pitch = pitch[:indmin]
+                    voicing = voicing[:indmin]
+                    rmse = rmse[:indmin]
+                    voicing = voicing*(rmse > 0.05*np.max(rmse))
+                    if "mic_F" in audio_file:
+                        idx_correct = np.where(pitch < 125)
+                        voicing[idx_correct] = 0
+
+                    cent = np.rint(1200*np.log2(np.divide(pitch, (16000/256), out=np.zeros_like(pitch), where=pitch!=0) + 1.0e-8)).astype('int')
+
+                    model_cents = pitch_nn(torch.from_numpy(np.copy(np.expand_dims(feature,0))).float().to(device))
+                    model_cents = 20*model_cents.argmax(dim=1).cpu().detach().squeeze().numpy()
+
+                    num_frames = min(cent.shape[0],model_cents.shape[0])
+                    pitch = pitch[:num_frames]
+                    cent = cent[:num_frames]
+                    voicing = voicing[:num_frames]
+                    model_cents = model_cents[:num_frames]
+
+                    voicing_all = np.copy(voicing)
+                    # Forcefully make regions where pitch is <65 or greater than 500 unvoiced for relevant accurate pitch comparisons for our model
+                    force_out_of_pitch = np.where(np.logical_or(pitch < 65,pitch > 500)==True)
+                    voicing_all[force_out_of_pitch] = 0
+                    C_all = C_all + np.where(voicing_all != 0)[0].shape[0]
+
+                    C_correct = C_correct + rca(cent,model_cents,voicing_all,thresh)
+                list_mean.append(C_correct/C_all)
+        else:
+            fname = f
+            list_mean = []
+            list_std = []
+            for snr_dB in list_snr:
+                C_all = 0
+                C_correct = 0
+                for idx in tqdm.trange(len(list_files)):
+                    audio_file = list_files[idx]
+                    file_name = os.path.basename(list_files[idx])[:-4]
+
+                    audio = np.memmap(list_files[idx], dtype=np.int16)/(2**15 - 1)
+                    offset = 432
+                    audio = audio[offset:]
+                    rmse = np.squeeze(librosa.feature.rms(y = audio,frame_length = 320,hop_length = 160))
+
+                    if noise_type != 'synthetic':
+                        list_noisefiles = noise_dataset + '*.wav'
+                        noise_file = random.choice(glob.glob(list_noisefiles))
+                        n = np.memmap(noise_file, dtype=np.int16,mode = 'r')/(2**15 - 1)
+                        rand_range = np.random.randint(low = 0, high = (16000*60*5 - audio.shape[0])) # Last 1 minute of noise used for testing
+                        n = n[rand_range:rand_range + audio.shape[0]]
+                    else:
+                        n = np.random.randn(audio.shape[0])
+                        n = random_filter(n)
+
+                    snr_multiplier = np.sqrt((np.sum(np.abs(audio)**2)/np.sum(np.abs(n)**2))*10**(-snr_dB/10))
+                    audio = audio + snr_multiplier*n
+
+                    if (f == 'crepe'):
+                        _, model_frequency, _, _ = crepe.predict(np.concatenate([np.zeros(80),audio]), 16000, viterbi=True,center=True,verbose=0)
+                        model_cents = 1200*np.log2(model_frequency/(16000/256) + 1.0e-8)
+                    else:
+                        data_temp = np.memmap('./temp.raw', dtype=np.int16, shape=(audio.shape[0]), mode='w+')
+                        # data_temp[:audio.shape[0]] = (audio/(np.max(np.abs(audio)))*(2**15 - 1)).astype(np.int16)
+                        data_temp[:audio.shape[0]] = ((audio)*(2**15 - 1)).astype(np.int16)
+
+                        subprocess.run(["../../../lpcnet_xcorr_extractor", './temp.raw', './temp_xcorr.f32', './temp_period.f32'])
+                        feature_xcorr = np.fromfile('./temp_period.f32', dtype='float32')
+                        model_cents = 1200*np.log2((256/feature_xcorr +  1.0e-8) + 1.0e-8)
+
+                        os.remove('./temp.raw')
+                        os.remove('./temp_xcorr.f32')
+                        os.remove('./temp_period.f32')
+
+
+                    pitch_file_name = dir_f0 + "ref" + os.path.basename(list_files[idx])[3:-4] + ".f0"
+                    pitch = np.loadtxt(pitch_file_name)[:,0]
+                    voicing = np.loadtxt(pitch_file_name)[:,1]
+                    indmin = min(voicing.shape[0],rmse.shape[0],pitch.shape[0])
+                    pitch = pitch[:indmin]
+                    voicing = voicing[:indmin]
+                    rmse = rmse[:indmin]
+                    voicing = voicing*(rmse > 0.05*np.max(rmse))
+                    if "mic_F" in audio_file:
+                        idx_correct = np.where(pitch < 125)
+                        voicing[idx_correct] = 0
+
+                    cent = np.rint(1200*np.log2(np.divide(pitch, (16000/256), out=np.zeros_like(pitch), where=pitch!=0) + 1.0e-8)).astype('int')
+                    num_frames = min(cent.shape[0],model_cents.shape[0])
+                    pitch = pitch[:num_frames]
+                    cent = cent[:num_frames]
+                    voicing = voicing[:num_frames]
+                    model_cents = model_cents[:num_frames]
+
+                    voicing_all = np.copy(voicing)
+                    # Forcefully make regions where pitch is <65 or greater than 500 unvoiced for relevant accurate pitch comparisons for our model
+                    force_out_of_pitch = np.where(np.logical_or(pitch < 65,pitch > 500)==True)
+                    voicing_all[force_out_of_pitch] = 0
+                    C_all = C_all + np.where(voicing_all != 0)[0].shape[0]
+
+                    C_correct = C_correct + rca(cent,model_cents,voicing_all,thresh)
+                list_mean.append(C_correct/C_all)
+        dict_models[fname] = {}
+        dict_models[fname]['list_SNR'] = list_mean[:-1]
+        dict_models[fname]['inf'] = list_mean[-1]
+
+    return dict_models

package/opus/dnn/torch/neural-pitch/experiments.py

@@ -0,0 +1,38 @@
+"""
+Running the experiments;
+    1. RCA vs SNR for our models, CREPE, LPCNet
+"""
+
+import argparse
+parser = argparse.ArgumentParser()
+
+parser.add_argument('ptdb_root', type=str, help='Root Directory for PTDB generated by running ptdb_process.sh ')
+parser.add_argument('output', type=str, help='Output dump file name')
+parser.add_argument('method', type=str, help='Output Directory to save experiment dumps',choices=['model','lpcnet','crepe'])
+parser.add_argument('--noise_dataset', type=str, help='Location of the Demand Datset',default = './',required=False)
+parser.add_argument('--noise_type', type=str, help='Type of additive noise',default = 'synthetic',choices=['synthetic','demand'],required=False)
+parser.add_argument('--pth_file', type=str, help='.pth file to analyze',default = './',required = False)
+parser.add_argument('--fraction_files_analyze', type=float, help='Fraction of PTDB dataset to test on',default = 1,required = False)
+parser.add_argument('--threshold_rca', type=float, help='Cent threshold when computing RCA',default = 50,required = False)
+parser.add_argument('--gpu_index', type=int, help='GPU index to use if multiple GPUs',default = 0,required = False)
+
+args = parser.parse_args()
+
+import os
+os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
+os.environ["CUDA_VISIBLE_DEVICES"] = str(args.gpu_index)
+
+import json
+from evaluation import cycle_eval
+
+if args.method == 'model':
+    dict_store = cycle_eval([args.pth_file], noise_type = args.noise_type, noise_dataset = args.noise_dataset, list_snr = [-20,-15,-10,-5,0,5,10,15,20], ptdb_dataset_path = args.ptdb_root,fraction = args.fraction_files_analyze,thresh = args.threshold_rca)
+else:
+    dict_store = cycle_eval([args.method], noise_type = args.noise_type, noise_dataset = args.noise_dataset, list_snr = [-20,-15,-10,-5,0,5,10,15,20], ptdb_dataset_path = args.ptdb_root,fraction = args.fraction_files_analyze,thresh = args.threshold_rca)
+
+dict_store["method"] = args.method
+if args.method == 'model':
+    dict_store['pth'] = args.pth_file
+
+with open(args.output, 'w') as fp:
+    json.dump(dict_store, fp)