rababa 0.1.0 → 0.1.1

data/python/config/cbhg.yml

@@ -16,7 +16,7 @@ diacritics_separator: '*'  # Required if the data already processed
 text_encoder: ArabicEncoderWithStartSymbol
 text_cleaner: valid_arabic_cleaners # a white list that uses only Arabic letters, punctuations, and a space
 max_len: 600 # sentences larger than this size will not be used
-
+reconcile: true
 
 max_steps: 2_000_000
 learning_rate: 0.001

data/python/config/test_cbhg.yml

@@ -0,0 +1,51 @@
+session_name: base
+
+data_directory: "data"
+data_type: "CA_MSA"
+log_directory: "log_dir"
+load_training_data: true
+load_test_data: false
+load_validation_data: true
+n_training_examples: null # null load all training examples, good for fast loading
+n_test_examples: null  # null load all test examples
+n_validation_examples: null # null load all validation examples
+test_file_name: "test.csv"
+is_data_preprocessed: false # The data file is organized as (original text | text | diacritics)
+data_separator: '|' # Required if the data already processed
+diacritics_separator: '*'  # Required if the data already processed
+text_encoder: ArabicEncoderWithStartSymbol
+text_cleaner: valid_arabic_cleaners # a white list that uses only Arabic letters, punctuations, and a space
+max_len: 600 # sentences larger than this size will not be used
+reconcile: true
+
+max_steps: 50
+learning_rate: 0.001
+batch_size: 32
+adam_beta1: 0.9
+adam_beta2: 0.999
+use_decay: true
+weight_decay: 0.0
+embedding_dim: 256
+use_prenet: false
+prenet_sizes: [512, 256]
+cbhg_projections: [128, 256]
+cbhg_filters: 16
+cbhg_gru_units: 256
+post_cbhg_layers_units: [256, 256]
+post_cbhg_use_batch_norm: true
+
+use_mixed_precision: false
+optimizer_type: Adam
+device: cuda
+
+# LOGGING
+evaluate_frequency: 5000
+evaluate_with_error_rates_frequency: 5000
+n_predicted_text_tensorboard: 10 # To be written to the tensorboard
+model_save_frequency: 5000
+train_plotting_frequency: 50000000 # No plotting for this model
+n_steps_avg_losses: [100, 500, 1_000, 5_000] # command line display of average loss values for the last n steps
+error_rates_n_batches: 10000 # if calculating error rate is slow, then you can specify the number of batches to be calculated
+
+test_model_path: null # load the last saved model
+train_resume_model_path: null # load last saved model

data/python/dataset.py

@@ -4,10 +4,13 @@ Loading the diacritization dataset
 
 import os
 
-from diacritization_evaluation import util
+import util.text_cleaners as cleaners
 import pandas as pd
 import torch
 import random
+import warnings
+from diacritization_evaluation import util
+
 from torch.utils.data import DataLoader, Dataset
 
 from config_manager import ConfigManager
@@ -15,7 +18,7 @@ from config_manager import ConfigManager
 
 class DiacritizationDataset(Dataset):
     """
-    The diacritization dataset
+    The datasets for preprocessing for diacritization
     """
 
     def __init__(self, config_manager: ConfigManager, list_ids, data):
@@ -24,6 +27,7 @@ class DiacritizationDataset(Dataset):
         self.data = data
         self.text_encoder = config_manager.text_encoder
         self.config = config_manager.config
+        # print('config:: ', self.config)
 
     def __len__(self):
         "Denotes the total number of samples"
@@ -33,35 +37,22 @@ class DiacritizationDataset(Dataset):
         "Generates one sample of data"
         # Select sample
         id = self.list_ids[index]
-        if self.config["is_data_preprocessed"]:
-            data = self.data.iloc[id]
-            inputs = torch.Tensor(self.text_encoder.input_to_sequence(data[1]))
-            targets = torch.Tensor(
-                self.text_encoder.target_to_sequence(
-                    data[2].split(self.config["diacritics_separator"])
-                )
-            )
-            return inputs, targets, data[0]
-        
-        encoding_failed = True
-        while encoding_failed:
-            try:
-                data = self.data[id]
-                data = self.text_encoder.clean(data)
-                text, inputs, diacritics = util.extract_haraqat(data)
-                encoding_failed = False
-            except:
-                print('dataset.py :: error with that data')
-                print('id: ', id)
-                print('data: ', data)
-                # text, inputs, diacritics = util.extract_haraqat(data[0])
-                id = random.randint(0, len(data)) 
-
-        inputs = torch.Tensor(self.text_encoder.input_to_sequence("".join(inputs)))
-        diacritics = torch.Tensor(self.text_encoder.target_to_sequence(diacritics))
-
-        return inputs, diacritics, text
+        data_orig = self.data[id].strip()
+        text, inputs, diacritics = cleaners.extract_haraqat(
+                                        self.text_encoder.clean(data_orig))
+
+        inputs = torch.Tensor(
+                    self.text_encoder.input_to_sequence("".join(inputs)))
+        diacritics = torch.Tensor(
+                    self.text_encoder.target_to_sequence(diacritics))
+
+        return inputs, diacritics, data_orig
 
+#data = self.data[id]
+#data = self.text_encoder.clean(data)
+#text, inputs, diacritics = util.extract_haraqat(data)
+#inputs = torch.Tensor(self.text_encoder.input_to_sequence("".join(inputs)))
+#diacritics = torch.Tensor(self.text_encoder.target_to_sequence(diacritics))
 
 def collate_fn(data):
     """
@@ -164,7 +155,8 @@ def load_test_data(config_manager: ConfigManager, loader_parameters):
             config_manager, [idx for idx in range(len(test_data))], test_data
         )
 
-    test_iterator = DataLoader(test_dataset, collate_fn=collate_fn, **loader_parameters)
+    test_iterator = DataLoader(test_dataset, collate_fn=collate_fn,
+                               **loader_parameters)
 
     print(f"Length of test iterator = {len(test_iterator)}")
     return test_iterator

data/python/diacritization_model_to_onnx.py

@@ -3,21 +3,21 @@ import pickle
 
 import numpy as np
 
-from diacritizer import CBHGDiacritizer
+from diacritizer import Diacritizer
 
 
 """
     Key Params:
-        max_len: 
+        max_len:
             is the max length for the arabic strings to be diacritized
-        batch size: 
+        batch size:
             has to do with the model training and usage
 """
-max_len = 300 # 600 for the original length
+max_len = 200 # 600 for the original length
 batch_size = 32
 
 
-""" 
+"""
     example and mock data:
     we found that populating all the data, removing the zeros gives better results.
 """
@@ -25,7 +25,7 @@ src = torch.Tensor([[1 for i in range(max_len)]
                     for i in range(batch_size)]).long()
 lengths = torch.Tensor([max_len for i in range(batch_size)]).long()
 # example data
-batch_data = pickle.load( open('../models-data/batch_data.pkl', 'rb') )
+batch_data = pickle.load( open('../models-data/batch_example_data.pkl', 'rb') )
 
 #target = batch_data['target']
 
@@ -37,7 +37,7 @@ model_kind_str = 'cbhg'
 config_str = 'config/cbhg.yml'
 load_model = True
 
-dia = CBHGDiacritizer(config_str, model_kind_str, load_model)
+dia = Diacritizer(config_str, model_kind_str, load_model)
 
 # set model to inference mode
 dia.model.to(dia.device)
@@ -58,13 +58,22 @@ import onnxruntime
 onnx_model_filename = '../models-data/diacritization_model.onnx'
 
 
+print(src.shape)
+
+#exit()
 # export model
-torch.onnx.export(dia.model, 
-                  (src, lengths), 
-                  onnx_model_filename, 
-                  verbose=False, 
-                  opset_version=11, 
-                  input_names=['src', 'lengths'])
+torch.onnx.export(dia.model,
+                  (src, lengths),
+                  onnx_model_filename,
+                  verbose=False,
+                  opset_version=11,
+                  input_names=['src', 'lengths'],
+                  output_names=['output'],
+                  dynamic_axes = {'src': [1], #[0,1,2], #[0,1,2],
+                  #'input_2':{0:'batch'},
+                  'output': [1]
+                  })
+
 print('Model printed in rel. path:', onnx_model_filename)
 
 
@@ -94,10 +103,202 @@ ort_inputs = {ort_session.get_inputs()[0].name: src.detach().numpy().astype(np.i
 # run onnx model
 ort_outs = ort_session.run(None, ort_inputs)
 
+print('outs:: ', ort_outs)
 
-for i in range(batch_size):
-    np.testing.assert_allclose(torch_out['diacritics'][i].detach().numpy(), ort_outs[0][i], rtol=1e-03, atol=1e-03)
 
+print('src:: ', src.detach().numpy().astype(np.int64))
+print('lengths: ',lengths.detach().numpy().astype(np.int64))
+
+#exit()
+
+for i in range(batch_size):
+    np.testing.assert_allclose(torch_out['diacritics'][i].detach().numpy(),
+                               ort_outs[0][i], rtol=1e-03, atol=1e-03)
 
 print("\n!!!Exported model has been tested with ONNXRuntime, result looks good within given tolerance!!!")
 
+
+
+vec = [[41, 12, 40] for i in range(batch_size)]
+src = torch.Tensor(vec).long()
+
+lengths = torch.Tensor([3 for i in range(batch_size)]).long()
+
+ort_inputs = {ort_session.get_inputs()[0].name: src.detach().numpy().astype(np.int64),
+              ort_session.get_inputs()[1].name: lengths.detach().numpy().astype(np.int64)}
+
+
+#print('12345678910')
+#print(ort_session.get_inputs()[0].name)
+#print(ort_session.get_inputs()[1].name)
+
+print('run 3')
+ort_outs = ort_session.run(None, ort_inputs)
+print('outs:: ', ort_outs[0].shape)
+print('outs:: ', ort_outs[0][0][0])
+print('outs:: ', ort_outs[0][0][1])
+print('outs:: ', ort_outs[0][0][2])
+
+torch_out = dia.model(src, lengths)
+
+#print(torch_out['diacritics'][0])
+for i in range(batch_size):
+    np.testing.assert_allclose(torch_out['diacritics'][i].detach().numpy(), \
+                               ort_outs[0][i], rtol=1e-03, atol=1e-03)
+
+print('12345678910')
+print(ort_session.get_inputs()[0].name)
+print(ort_session.get_inputs()[1].name)
+
+#exit()
+
+
+
+"""
+    Test ONNX model on randomized data
+"""
+
+import random
+test_id = 0
+
+print('***** Test MAX size :: Random Boolean vectors: *****')
+print(max_len)
+
+for test_run in range(3):
+
+    vec = [[random.randint(0,1) for i in range(max_len)]
+            for i in range(batch_size)]
+    src = torch.Tensor(vec).long()
+    lengths = torch.Tensor([max_len for i in range(batch_size)]).long()
+
+    """
+    with open('test_data/test'+str(test_id)+'.txt', 'w') as f:
+        for ll in src.detach().tolist():
+            for item in ll:
+                f.write("%s " % item)
+            f.write("\n")
+    f.close()
+    """
+    torch_out = dia.model(src, lengths)
+    """
+    my_list = torch_out['diacritics'].detach().numpy().tolist()
+    with open('test_data/test'+str(test_id)+'_torch.txt', 'w') as f:
+        for ll in my_list:
+            for item in ll:
+                for l in item:
+                    f.write("%s " % l)
+                f.write("\n")
+    f.close()
+    test_id+=1
+    """
+    # prepare onnx input
+    ort_inputs = {ort_session.get_inputs()[0].name: src.detach().numpy().astype(np.int64),
+                  ort_session.get_inputs()[1].name: lengths.detach().numpy().astype(np.int64)}
+
+    # run onnx model
+    ort_outs = ort_session.run(None, ort_inputs)
+
+    for i in range(batch_size):
+        np.testing.assert_allclose(torch_out['diacritics'][i].detach().numpy(), \
+                                   ort_outs[0][i], rtol=1e-03, atol=1e-03)
+
+    print('test :: ', test_run)
+    print("Result looks good within given tolerance!!!")
+
+
+print('***** Test MAX size :: Random float, vectors within 0:16 *****')
+print(max_len)
+
+for test_run in range(3):
+
+    vec = [[random.randint(0, 17) for i in range(max_len)]
+            for i in range(batch_size)]
+    src = torch.Tensor(vec).long()
+    """
+    with open('test_data/test'+str(test_id)+'.txt', 'w') as f:
+        for ll in src.detach().tolist():
+            for item in ll:
+                f.write("%s " % item)
+            f.write("\n")
+    f.close()
+    """
+    torch_out = dia.model(src, lengths)
+
+    #my_list = torch_out['diacritics'].detach().numpy().tolist()
+    """
+    with open('test_data/test'+str(test_id)+'_torch.txt', 'w') as f:
+        for ll in my_list:
+            for item in ll:
+                for l in item:
+                    f.write("%s " % l)
+                f.write("\n")
+    f.close()
+    test_id+=1
+    """
+    # prepare onnx input
+    ort_inputs = {ort_session.get_inputs()[0].name: src.detach().numpy().astype(np.int64),
+                  ort_session.get_inputs()[1].name: lengths.detach().numpy().astype(np.int64)}
+
+    # run onnx model
+    ort_outs = ort_session.run(None, ort_inputs)
+
+    for i in range(batch_size):
+        np.testing.assert_allclose(torch_out['diacritics'][i].detach().numpy(), \
+                                   ort_outs[0][i], rtol=1, atol=1)
+
+    print('test :: ', test_run)
+    print("Result looks good within given tolerance!!!")
+
+
+print('***** Test Dynamical sizes :: Random Boolean vectors: *****')
+
+for l in [2, 10, 40, 100, 150]:
+
+    print('length:: ', l)
+
+    vec = [[1 for i in range(l)] # random.randint(0,1)
+            for i in range(batch_size)]
+    src = torch.Tensor(vec).long()
+    lengths = torch.Tensor([l for i in range(batch_size)]).long()
+
+    torch_out = dia.model(src, lengths)
+
+    # prepare onnx input
+    ort_inputs = {ort_session.get_inputs()[0].name: src.detach().numpy().astype(np.int64),
+                  ort_session.get_inputs()[1].name: lengths.detach().numpy().astype(np.int64)}
+
+    # run onnx model
+    ort_outs = ort_session.run(None, ort_inputs)
+
+    for i in range(batch_size):
+        np.testing.assert_allclose(torch_out['diacritics'][i].detach().numpy(), \
+                                   ort_outs[0][i], rtol=1e-03, atol=1e-03)
+
+    print('test :: ', l)
+    print("Result looks good within given tolerance!!!")
+
+
+print('***** Test Dynamical sizes :: Random float, vectors within 0:16 *****')
+
+for l in [2, 10, 40, 100, 150]:
+
+    vec = [[random.randint(0, 17) for i in range(l)]
+            for i in range(batch_size)]
+    src = torch.Tensor(vec).long()
+    lengths = torch.Tensor([l for i in range(batch_size)]).long()
+
+    torch_out = dia.model(src, lengths)
+
+    # prepare onnx input
+    ort_inputs = {ort_session.get_inputs()[0].name: src.detach().numpy().astype(np.int64),
+                  ort_session.get_inputs()[1].name: lengths.detach().numpy().astype(np.int64)}
+
+    # run onnx model
+    ort_outs = ort_session.run(None, ort_inputs)
+
+    for i in range(batch_size):
+        np.testing.assert_allclose(torch_out['diacritics'][i].detach().numpy(), \
+                                   ort_outs[0][i], rtol=1, atol=1)
+
+    print('test :: ', l)
+    print("Result looks good within given tolerance!!!")

data/python/diacritizer.py

@@ -1,6 +1,8 @@
 from typing import Dict
 import torch
 import tqdm
+import pandas as pd
+import numpy as np
 from config_manager import ConfigManager
 from dataset import (DiacritizationDataset,
                      collate_fn)
@@ -9,7 +11,6 @@ from torch.utils.data import (DataLoader,
 import util.reconcile_original_plus_diacritized as reconcile
 
 
-
 class Diacritizer:
     def __init__(
         self, config_path: str, model_kind: str, load_model: bool = False
@@ -35,6 +36,7 @@ class Diacritizer:
 
     def diacritize_text(self, text: str):
         # convert string into indices
+        text = text.strip()
         seq = self.text_encoder.input_to_sequence(text)
         # transform indices into "batch data"
         batch_data = {'original': [text],
@@ -48,25 +50,31 @@ class Diacritizer:
         loader_params = {"batch_size": self.config_manager.config["batch_size"],
                          "shuffle": False,
                          "num_workers": 2}
-        # data processed or not, specs in config file
-        if self.config_manager.config["is_data_preprocessed"]:
-            data = pd.read_csv(path,
-                               encoding="utf-8",
-                               sep=self.config_manager.config["data_separator"],
-                               nrows=self.config_manager.config["n_validation_examples"],
-                               header=None)
-
-            # data = data[data[0] <= config_manager.config["max_len"]]
-            dataset = DiacritizationDataset(self.config_manager, data.index, data)
-        else:
-            with open(path, encoding="utf8") as file:
-                data = file.readlines()
-            data = [text for text in data if len(text) <= self.config_manager.config["max_len"]]
-            dataset = DiacritizationDataset(
-                self.config_manager, [idx for idx in range(len(data))], data)
-
-        data_iterator = DataLoader(dataset, collate_fn=collate_fn, **loader_params)
-        # print(f"Length of data iterator = {len(valid_iterator)}")
+
+        data_tmp = pd.read_csv(path,
+                           encoding="utf-8",
+                           sep=self.config_manager.config["data_separator"],
+                           header=None)
+
+        data = []
+        max_len = self.config_manager.config["max_len"]
+        for txt in [d[0] for d in data_tmp.values.tolist()]:
+            if len(txt) > max_len:
+                txt = txt[:max_len]
+                warnings.warn('Warning: text length cut for sentence: \n'+text)
+            data.append(txt)
+
+        list_ids = [idx for idx in range(len(data))]
+        dataset = DiacritizationDataset(self.config_manager,
+                                        list_ids,
+                                        data)
+
+        data_iterator = DataLoader(dataset,
+                                   collate_fn=collate_fn,
+                                   # **loader_params,
+                                   shuffle=False)
+
+        # print(f"Length of data iterator = {len(data_iterator)}")
         return data_iterator
 
     def diacritize_file(self, path: str):
@@ -75,6 +83,7 @@ class Diacritizer:
         diacritized_data = []
         for batch_inputs in tqdm.tqdm(data_iterator):
 
+            #batch_inputs["original"] = batch_inputs["original"].to(self.device)
             batch_inputs["src"] = batch_inputs["src"].to(self.device)
             batch_inputs["lengths"] = batch_inputs["lengths"].to('cpu')
             batch_inputs["target"] = batch_inputs["target"].to(self.device)
@@ -85,7 +94,7 @@ class Diacritizer:
         return diacritized_data
 
     def diacritize_batch(self, batch):
-        #print('batch: ',batch)
+        # print('batch: ',batch)
         self.model.eval()
         originals = batch['original']
         inputs = batch["src"]
@@ -93,25 +102,20 @@ class Diacritizer:
         outputs = self.model(inputs.to(self.device), lengths.to("cpu"))
         diacritics = outputs["diacritics"]
         predictions = torch.max(diacritics, 2).indices
-        sentences = []
 
+        sentences = []
         for src, prediction, original in zip(inputs, predictions, originals):
             sentence = self.text_encoder.combine_text_and_haraqat(
-                list(src.detach().cpu().numpy()),
-                list(prediction.detach().cpu().numpy()),
-            )
+                    list(src.detach().cpu().numpy()),
+                    list(prediction.detach().cpu().numpy()))
             # Diacritized strings, sentence have to be "reconciled"
             # with original strings, because the non arabic strings are removed
             # before being processed in nnet
-            sentence = reconcile.reconcile_strings(original, sentence)
+            if self.config['reconcile']:
+                sentence = reconcile.reconcile_strings(original, sentence)
             sentences.append(sentence)
 
         return sentences
 
     def diacritize_iterators(self, iterator):
         pass
-
-""" not needed
-class CBHGDiacritizer(Diacritizer):
-class Seq2SeqDiacritizer(Diacritizer):
-"""