rababa 0.1.0 → 0.1.1

data/exe/rababa

@@ -10,7 +10,7 @@ parser = Rababa.parser
 
 config_path = parser.has_key?(:config) ? parser[:config] : "config/model.yml"
 
-diacritizer = Rababa::Diacritizer.new(parser[:model_path], config_path)
+diacritizer = Rababa::Diacritizer.new(parser[:model_path], YAML.load_file(config_path))
 
 if parser.has_key?(:text)
   # run diacritization text if has :text

data/lib/README.adoc

@@ -0,0 +1,95 @@
+= Arabic Diacritization in Ruby with Rababa
+
+== Try out Rababa
+
+* Install the Gems listed below
+* Download a Ruby model on https://github.com/secryst/rababa-models[releases]
+
+== Usage
+
+=== Install
+
+[source,sh]
+----
+gem install rababa
+----
+
+=== Download the ONNX model
+
+Please download the `diacritization_model_max_len_200.onnx` model file
+from https://github.com/secryst/rababa-models/releases/tag/0.1.
+
+
+=== Running examples
+
+One can diacritize either single strings:
+
+[source,sh]
+----
+rababa -t 'قطر' -m diacritization_model_max_len_200.onnx
+# or when inside the gem directory during development
+bundle exec exe/rababa -t 'قطر' -m diacritization_model_max_len_200.onnx
+----
+
+Or files as `data/examples.txt` or your own Arabic file (the max string length
+is specified in the model and has to match the `max_len` parameter in
+`config/models.yaml`):
+
+[source,sh]
+----
+rababa -f data/example.txt -m diacritization_model_max_len_200.onnx
+# or when inside the gem directory during development
+bundle exec exe/rababa -f data/example.txt -m diacritization_model_max_len_200.onnx
+----
+
+One would have to preprocess generic arabic texts for running Rababa in general.
+This can be done on sentences beginnings running for instance
+https://github.com/Hamza5/Pipeline-diacritizer[Hamza5]:
+
+----
+python __main__.py preprocess source destination
+----
+
+== Training
+
+=== ONNX Models
+
+They can either be built in the `/python` repository or downloaded from the
+https://github.com/secryst/rababa-models[releases].
+
+Or ONNX model can be generated running the Python
+https://github.com/interscript/rababa/blob/main/python/diacritization_model_to_onnx.py[code]
+in this library.
+
+It requires to go through some of the steps described in the link above.
+
+=== Parameters
+
+* text to diacritize: "**-t**TEXT", "--text=TEXT",
+* path to file to diacritize: "**-f**FILE", "--text_filename=FILE",
+* path to ONNX model **Mandatory**: "-mMODEL", "--model_file=MODEL",
+* path to config file **Default:config/model.yml**: "-cCONFIG", "--config=CONFIG"
+
+=== Config
+
+==== Players
+
+* `max_len`: `200` -- `600`
+
+** Parameter that has to match the ONNX model built using the
+  https://github.com/interscript/rababa/blob/main/python/diacritization_model_to_onnx.py[code]
+  and following the python/README.adoc.
+
+** Longer sentences will need to be preprocessed, which can be done for
+  instance using https://github.com/Hamza5[Hamza5]
+  https://github.com/Hamza5/Pipeline-diacritizer/blob/master/pipeline_diacritizer/pipeline_diacritizer.py[code].
+
+** the smaller the faster the NNets code.
+
+* text_encoder corresponding to the https://github.com/interscript/rababa/blob/main/python/util/text_encoders.py[rules]:
+** `BasicArabicEncoder`
+** `ArabicEncoderWithStartSymbol`
+
+* text_cleaner corresponding to https://github.com/interscript/rababa/blob/main/python/util/text_cleaners.py[logics]:
+** `basic_cleaners`: remove redundancy in whitespaces and strip string
+** `valid_arabic_cleaners`: basic+filter of only Arabic words

data/lib/rababa/diacritizer.rb

@@ -1,6 +1,6 @@
 
 # this refers to:
-#   https://github.com/interscript/rababa/blob/master/python/diacritizer.py
+#   https://github.com/interscript/rababa/blob/main/python/diacritizer.py
 # as well a drastic simplification of
 #   https://github.com/almodhfer/Arabic_Diacritization/blob/master/config_manager.py
 
@@ -16,27 +16,33 @@ module Rababa
         include Rababa::Harakats
         include Rababa::Reconcile
 
-        def initialize(onnx_model_path, config_path)
+        def initialize(onnx_model_path, config)
 
             # load inference model from model_path
             @onnx_session = OnnxRuntime::InferenceSession.new(onnx_model_path)
 
             # load config
-            @config = YAML.load_file(config_path)
+            @config = config
             @max_length = @config['max_len']
             @batch_size = @config['batch_size']
 
             # instantiate encoder's class
-            @encoder = get_text_encoder()
+            @encoder = get_text_encoder
             @start_symbol_id = @encoder.start_symbol_id
 
         end
 
         # preprocess text into indices
         def preprocess_text(text)
-            #if (text.length > @max_length)
-            #    raise ValueError.new('text length larger than max_length')
-            #end
+            # if (text.length > @max_length)
+            #     raise ValueError.new('text length larger than max_length')
+            # end
+            # hack in absence of preprocessing!
+            if text.length > @max_length
+                text = text[0..@max_length]
+                warn('WARNING:: string cut length > #{@max_length},\n')
+                warn('text:: '+text)
+            end
 
             text = @encoder.clean(text)
             text = remove_diacritics(text)
@@ -47,6 +53,8 @@ module Rababa
 
         # Diacritize single arabic strings
         def diacritize_text(text)
+            """Diacritize single arabic strings"""
+            text = text.strip()
             seq = preprocess_text(text)
 
             # initialize onnx computation
@@ -66,7 +74,7 @@ module Rababa
         def diacritize_file(path)
             texts = []
             File.open(path).each do |line|
-                texts.push(line.chomp)
+                texts.push(line.chomp.strip())
             end
 
             # process batches

data/lib/rababa/encoders.rb

@@ -1,8 +1,8 @@
 """
 corresponds to:
-https://github.com/interscript/rababa/blob/master/python/util/text_encoders.py
+https://github.com/interscript/rababa/blob/main/python/util/text_encoders.py
 and
-https://github.com/interscript/rababa/blob/master/python/util/text_cleaners.py
+https://github.com/interscript/rababa/blob/main/python/util/text_cleaners.py
 """
 
 require_relative "arabic_constants"

data/lib/rababa/harakats.rb

@@ -14,7 +14,7 @@ module Rababa::Harakats
         char_haraqat = []
 
         while stack.length != 0
-            char_haraqat.append(stack.pop)
+            char_haraqat << stack.pop
         end
 
         full_haraqah = char_haraqat.join("")

data/lib/rababa/reconcile.rb

@@ -31,7 +31,7 @@ module Rababa::Reconcile
           (idx_ori..d_original.length).each {|i|
               if (c_dia == d_original[i])
                   idx_ori = i
-                  l_map.append([idx_dia, idx_ori])
+                  l_map << [idx_dia, idx_ori]
                   break
               end
           }
@@ -99,35 +99,3 @@ module Rababa::Reconcile
   end
 
 end
-
-
-"""TESTS
-TODO: MOVE TO RSPEC
-d_tests = [{'original' => '# گيله پسمير الجديد 34',
-            'diacritized' => 'يَلِهُ سُمِيْرٌ الجَدِيدُ',
-            'reconciled' => '# گيَلِهُ پسُمِيْرٌ الجَدِيدُ 34' },
-
-           {'original' => 'abc',
-            'diacritized' => '',
-            'reconciled' => 'abc'},
-
-           {'original' => '‘Iz. Ibrāhīm as-Sa‘danī',
-            'diacritized' => '',
-            'reconciled' => '‘Iz. Ibrāhīm as-Sa‘danī'},
-
-           {'original' => '26 سبتمبر العقبة',
-            'diacritized' => 'سَبْتَمْبَرِ العَقَبَة',
-            'reconciled' => '26 سَبْتَمْبَرِ العَقَبَة'}]
-
-d_tests.each {|d| \
-    if not d['reconciled']==reconcile_strings(d['original'], d['diacritized'])
-        raise Exception.new('reconcile string not matched')
-    end
-}
-
-or:
-for s in '# گيله پسمير الجديد 34' 'abc'  '‘Iz. Ibrāhīm as-Sa‘danī' '26 سبتمبر العقبة'
-do;
-    ruby rababa.rb -t $s -m '../models-data/diacritization_model.onnx'
-done
-"""

data/lib/rababa/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Rababa
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

data/models-data/README.adoc

@@ -0,0 +1,6 @@
+=== Model data dir
+
+Contains:
+
+* ONNX data
+* Pickle sample data

data/python/README.adoc

@@ -0,0 +1,211 @@
+= Rababa Python for diacritization
+
+== Purpose
+
+Rababa Python is used for both:
+
+* Training of the Rababa diacriticization models
+* Conversion of non-diacriticized Arabic into diacriticized Arabic
+  (i.e. running of the Rababa diacriticization models)
+
+== Introduction
+
+Rababa uses deep Learning models for recovering Arabic language diacritics.
+
+Rababa implements the models described in the paper
+https://ieeexplore.ieee.org/document/9274427[Effective Deep Learning Models for Automatic Diacritization of Arabic Text] and refers to the implementation models from
+https://github.com/almodhfer/Arabic_Diacritization[almodhfer],
+which we have selected for this project from a list of alternatives listed in
+the README.
+
+Out of the four models that https://github.com/almodhfer[almodhfer] has
+implemented, we selected the simplest and most performant ones:
+
+* The baseline model (`baseline`): consists of 3 bidirectional LSTM layers with
+  optional batch norm layers.
+
+* The CBHG model (`cbhg`): uses only the encoder of the Tacotron-based model
+  with optional post LSTM, and batch norm layers.
+
+
+== Usage
+
+=== Prerequisites
+
+Python version: 3.6+.
+
+Setup dependencies with:
+
+[source,bash]
+----
+pip install -r requirement.txt
+----
+
+
+=== Quickstart
+
+. Setup prerequisites
+
+. Download the released model
+  https://github.com/secryst/rababa-models/releases/download/0.1/2000000-snapshot.pt[here]
+  and place under `python/log_dir/CA_MSA.base.cbhg/models/2000000-snapshot.pt`
+
+. Single sentences and text can now be diacritized as below:
+
+[source,bash]
+----
+python diacritize.py --model_kind "cbhg" --config config/cbhg.yml --text 'قطر'
+python diacritize.py --model_kind "cbhg" --config config/cbhg.yml --text_file relative_path_to_text_file
+----
+
+The maximal string length is set in configs at `600`.
+
+Longer lines need to be broken down, for instance using the library
+introduced in the link:../lib/README.adoc[Ruby quickstart] section.
+
+
+== Training
+
+=== Datasets
+
+* We have chosen the "Tashkeela processed" corpus ~2,800,000 sentences:
+** https://github.com/interscript/rababa-tashkeela
+
+Other datasets are discussed in the reviewed literature or in the article
+referenced above.
+
+For training, data needs to be stored in the `data/CA_MSA` directory in such a
+format:
+
+[source,bash]
+----
+> ls data/CA_MSA/*
+--> data/CA_MSA/eval.csv  data/CA_MSA/train.csv  data/CA_MSA/test.csv
+----
+
+For instance:
+
+[source,bash]
+----
+mkdir -p data/CA_MSA
+cd data
+curl -sSL https://github.com/interscript/rababa-tashkeela/archive/refs/tags/v1.0.zip -o tashkeela.zip
+unzip tashkeela.zip
+for d in `ls rababa-tashkeela-1.0/tashkeela_val/*`; do cat $d >> CA_MSA/eval.csv; done
+for d in `ls rababa-tashkeela-1.0/tashkeela_train/*`; do cat $d >> CA_MSA/train.csv; done
+for d in `ls rababa-tashkeela-1.0/tashkeela_test/*`; do cat $d >> CA_MSA/test.csv; done
+----
+
+Alternatively, the dataset can be downloaded at
+[rababa-tashkeela](https://github.com/interscript/rababa-tashkeela).
+
+=== Load Model
+
+Alternatively, trained CBHG models are available under
+https://github.com/secryst/rababa-models[releases].
+
+Models are to be copied as specified in the link just above under:
+
+[source,bash]
+----
+> log_dir/CA_MSA.base.cbhg/models/2000000-snapshot.pt
+----
+
+
+=== Config Files
+
+One can adjust the model configurations in the `/config` repository.
+
+The model configurations are about the layers but also the dataset to be used
+and various other options.
+
+The configuration files are called explicitly in the below applications.
+
+=== Data Preprocessing
+
+The original work cited above allow for both raw and preprocessed.
+
+We go for the simplest raw version here:
+- As mentioned above, corpus must have `test.csv`,
+  `train.csv`, and `valid.csv`.
+
+- Specify that the data is not preprocessed in the config.
+  In that case, each batch will be processed and the text and diacritics
+  will be extracted from the original text.
+
+- You also have to specify the text encoder and the cleaner functions.
+  Two text encoders were included: `BasicArabicEncoder`,
+  `ArabicEncoderWithStartSymbol`.
+
+  Moreover, we have one cleaning function: `valid_arabic_cleaners`, which clean
+  all characters except valid Arabic characters, which include Arabic letters,
+  punctuations, and diacritics.
+
+=== Training
+
+All models config are placed in the config directory.
+
+[source,bash]
+----
+python train.py --model "cbhg" --config config/cbhg.yml
+----
+
+The model will report the WER and DER while training using the
+`diacritization_evaluation` package. The frequency of calculating WER and
+DER can be specified in the config file.
+
+=== Testing
+
+The testing is done in the same way as the training,
+For instance, with the CBHG model on the data in `/data/CA_MSA/test.csv`:
+
+[source,bash]
+----
+python test.py --model 'cbhg' --config config/cbhg.yml
+----
+
+The model will load the last saved model unless you specified it in the config:
+`test_data_path`. The test file is expected to have the correct diacritization!
+
+If the test file name is different than `test.csv`, you
+can add it to the `config: test_file_name`.
+
+=== Diacritize text or files
+
+Single sentences or files can be processed. The code outputs is the diacritized
+text or lines.
+
+[source,bash]
+----
+python diacritize.py --model_kind "cbhg" --config config/cbhg.yml --text 'قطر'
+python diacritize.py --model_kind "cbhg" --config config/cbhg.yml --text_file relative_path_to_text_file
+----
+
+=== Convert CBHG, Python model to ONNX
+
+The last model stored during training is automatically chosen and the ONNX model
+is saved into a hardcoded location:
+
+* `../models-data/diacritization_model.onnx`
+
+==== Run
+
+[source,bash]
+----
+python diacritization_model_to_onnx.py
+----
+
+==== Important parameters
+
+They are hardcoded in the beginning of the script:
+
+* `max_len`:
+** match string length, initial model value is given in config.
+** this param allows tuning the model speed and size!
+** the Ruby ../lib/README.md points to resources for preprocessing
+
+* batch_size:
+** the value is given by the original model and its training.
+** this constrain how the ONNX model can be put in production:
+... if > 1, single lines involve redundant computations
+... if > 1, files are processed in batches.