subtitle 0.1.8 → 0.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA256:
-  metadata.gz: aba6a9f70a40bf96fd797a6a68816ca2e3070f93573d26dbbb1df4b12a47d691
-  data.tar.gz: 1f61c3bb5a4ec42ca6fe694c9e1bac4d805d21d7f057b6f45a4c51ac0671e287
+SHA1:
+  metadata.gz: f603ac76acbb145807944c0f948d6550eee197cc
+  data.tar.gz: 7d06b0e8ee047ab1790237fca1c478da03725541
 SHA512:
-  metadata.gz: e996202bdd3ee2c8860b51aa6c5e414b9eea4735355c5b8f9d58f90ce67105fb1fafd781ae3e7375215ae8d364c9dfd0caa6dc3d73fd9f0257bb2748b945badc
-  data.tar.gz: 5bfdf76e39dfc65a4ef5e5e288b74aa3e098fe358c402214300710b7b614d8c444053ed7dcdc3e012327d6c7b33d6d91db8701e06c39d6232dfdde42197d3e17
+  metadata.gz: 204c3af4231e25e6caaa198e9a8b7d46b4f917afcc8abdbce27bccbb94908d28d21c1b0318aa36a1ba63c83c064963e24c7398a99a0cd183c3ca10568fb6fe34
+  data.tar.gz: fb867912d76f039abf21fd1495c0f9b9139a594cde028f1d50184bd15d007a59113366507b7a85f1039ce0c638d423fbd7cff99d22792f83af59ccede48167dc

data/lib/allfather.rb

@@ -0,0 +1,83 @@
+# 
+# A Module that kind of acts as an interface where the generic methods
+# that applies to each caption type can be defined
+#
+# To use for a new caption type, simply include this module and provide
+# caption specific implementations
+#
+module AllFather
+  
+  # 
+  # Valid file extensions that we support; Keep expanding as we grow
+  #
+  VALID_FILES = [".scc", ".srt", ".vtt", ".ttml", ".dfxp"]
+
+  # 
+  # Generic exception class that is raised for validation errors
+  #
+  class InvalidInputException < StandardError; end
+
+  #
+  # Lang inference failure exception
+  #
+  class LangDetectionFailureException < StandardError; end
+
+  #
+  # Method to do basic validations like is this a valid file to even
+  # accept for any future transactions 
+  #
+  # ==== Returns:
+  # true if the file is valid and false otherwise
+  #
+  def is_valid?
+    raise "Not Implemented. Class #{self.class.name} doesn't implement is_valid?"
+  end
+
+  #
+  # Method to infer the language(s) of the caption by inspecting the file
+  # depending on the type of the caption file
+  #
+  # ==== Returns
+  #
+  # * The ISO 639-1 Letter Language codes
+  # 
+  def infer_languages
+    raise "Not Implemented. Class #{self.class.name} doesn't implement infer_languages"
+  end
+
+  #
+  # Method to translate the caption from one language to another
+  #
+  # :args: src_lang, target_lang, output_file
+  #
+  # * +input_caption+   - A Valid input caption file. Refer to #is_valid?
+  # * +src_lang+        - can be inferred using #infer_language method
+  # * +target_lang+     - Target 2 letter ISO language code to which the source needs to be translated in to.
+  # * +output_file+     - Output file. Can be a fully qualified path or just file name
+  #
+  # ==== Raises
+  # 
+  # InvalidInputException shall be raised if
+  # 1. The input file doesn't exist or is unreadable or is invalid caption
+  # 2. The output file can't be written
+  # 3. The target_lang is not a valid ISO 639-1 Letter Language code
+  #
+  def translate(src_lang, target_lang, output_file)
+    # Check if a non empty output file is present and error out to avoid
+    # the danger or overwriting some important file !!
+    if File.exists?(output_file) && File.size(output_file) > 0
+      raise InvalidInputException.new("Output file #{output_file} is not empty.")
+    else
+      # Just open the file in writable mode and close it just to ensure that
+      # we can write the output file
+      File.open(output_file, "w") {|f|
+      }
+    end
+    # Check if the file is writable ?
+    unless File.writable?(output_file)
+      raise InvalidInputException.new("Output file #{output_file} not writable.")
+    end
+    # Further checks can be done only in caption specific implementations
+    # or translation engine specific implementation
+  end
+end

data/lib/dfxp.rb

@@ -0,0 +1,30 @@
+require_relative "engines/translator"
+require_relative "allfather"
+require_relative "ttml"
+
+#
+# Library to handle DFXP Files
+#
+# Uses the translator available to do the necessary language operations
+# as defined by the AllFather
+#
+class DFXP < TTML
+
+	def initialize(cc_file, translator, opts={})
+    @cc_file = cc_file
+    @translator = translator
+    @force_detect = opts[:force_detect] || false
+    raise "Invalid TTML file provided" unless is_valid?
+  end
+
+  def is_valid?
+    # Do any VTT specific validations here
+    if @cc_file =~ /^.*\.(dfxp)$/
+      return true
+    end
+    # TODO: Check if it's required to do a File read to see if this
+    # a well-formed XML. Another is to see if lang is available in each div
+    return false
+  end
+  
+end

data/lib/engines/aws.rb

@@ -0,0 +1,102 @@
+require 'aws-sdk'
+require 'aws-sdk'
+require_relative 'translator'
+
+# 
+# Provides Language services using Amazon Translate
+#
+# Module can be intialized using multiple options
+#
+# == Credential Referencing Order
+#
+# * [Arguments]             - Pass the credentials access_key_id and secret_access_key as arguments
+# * [Environment route]     - AWS_ACCESS_KEY_ID & AWS_SECRET_ACCESS_KEY can be exposed as 
+# environment variables
+# * [Profile Name]          - The application uses the credentials of the system and picks the
+# credentials referred to by the profile
+#
+class AwsEngine
+  include Translator
+
+  DEFAULT_REGION = ENV["AWS_DEFAULT_REGION"] || "us-east-1"
+
+  # 
+  # :args: options
+  #
+  # ==== Arguments
+  # options can carry the following details
+  #
+  # * [:access_key_id]      - access key id
+  # * [:secret_access_key]  - Secret access key
+  # * [:env]                - true for using credentials from environment variables
+  # * [:profile]            - profile name for using shared credentials setup
+  # * [:region]             - If not provided defaults to us-east-1
+  #
+  # ==== raises                 
+  #
+  # * EngineInitializationException if credentials cannot be setup due to lack of details
+  # * Aws Exceptions if profile name is invalid or invalid credentials are passed
+  # 
+  def initialize(options)
+    access_key_id = nil
+    secret_access_key = nil
+    @region = options[:region] || DEFAULT_REGION
+    if options[:env]
+      access_key_id = ENV["AWS_ACCESS_KEY_ID"]
+      secret_access_key = ENV["AWS_SECRET_ACCESS_KEY"]
+    elsif options[:access_key_id] && options[:secret_access_key]
+      access_key_id = options[:access_key_id]
+      secret_access_key = options[:secret_access_key]
+    end
+    if access_key_id && secret_access_key
+      Aws.config.update({
+        region: options[:region] || DEFAULT_REGION,
+        credentials: Aws::Credentials.new(access_key_id, secret_access_key)
+      })
+    elsif options[:profile]
+      credentials = Aws::SharedCredentials.new(profile_name: options[:profile])
+      Aws.config.update({
+        region: @region,
+        credentials: credentials.credentials
+      })
+    else
+      raise Translator::EngineInitializationException.new(
+        "Failed to initialize Aws Engine. Credentials are missing / not provided")
+    end
+    @translate_service  = Aws::Translate::Client.new(region: @region)
+    @comprehend_service = Aws::Comprehend::Client.new(region: @region)
+  end
+
+  # 
+  # Invokes the language detection API of AWS and returns only the language
+  # of the highest score and returns the ISO 639-1 code
+  #
+  # :args: text
+  # 
+  # ===== Arguments 
+  # * +text+ - The text for which the language is to be inferred
+  # 
+  def infer_language(text)
+    response = @comprehend_service.detect_dominant_language({ text: "#{text}" })
+    response[:languages][0][:language_code]
+  end
+
+  # 
+  # Invokes the translation API of AWS and returns the translated text
+  # as per the arguments provided
+  # Will Raise exception if a translation cannot be made between the source
+  # and target language codes or if the lang code is invalid
+  #
+  # :args: input_text, src_lang, target_lang
+  #
+  # * +input_text+      - The text that needs to be translated
+  # * +src_lang+        - The source language of the text
+  # * +target_lang+     - The target language to which the input_text needs to be translated to
+  #
+  def translate(input_text, src_lang, target_lang)
+    response = @translate_service.translate_text({ :text => "#{input_text}" , 
+      :source_language_code => "#{src_lang}", :target_language_code => "#{target_lang}"})
+    response.translated_text
+  end
+end
+

data/lib/engines/translator.rb

@@ -0,0 +1,58 @@
+# 
+# A Module that kind of acts as an interface where the methods
+# expected out of each vendor is encapsulated into
+#
+# To use for a new vendor, simply include this module and provide
+# caption specific implementations
+#
+module Translator
+
+  # 
+  # Constants For Engines
+  ENGINE_AWS = 1
+  ENGINE_GCP = 2
+
+  #
+  # Keys for each Engine
+  AWS_KEYS = [:access_key_id, :secret_access_key, :profile]
+  GCP_KEYS = [:api_key, :project_id, :creds_path]
+  
+  ENGINE_KEYS = {ENGINE_AWS => AWS_KEYS, ENGINE_GCP => GCP_KEYS}
+  #
+  # This exception shall be raised when we fail to initialize an
+  # engine for the purposes of language detection / translation
+  #
+  # ==== Example
+  # * When credentials are not passed
+  #
+  class EngineInitializationException < StandardError; end
+
+  #
+  # Method to infer the language by inspecting the text
+  # passed as argument
+  #
+  # :args: text
+  #
+  # * +text+ - String whose language needs to be inferred
+  #
+  # ==== Returns
+  #
+  # * The ISO 639-1 Letter Language code
+  # 
+  def infer_language(text)
+    raise "Not Implemented. Class #{self.class.name} doesn't implement infer_language"
+  end
+
+  #
+  # Method to translate from given language to another
+  #
+  # :args: input_text, src_lang, target_lang, output_file
+  #
+  # * +input_text+      - Text which needs to be translated
+  # * +src_lang+        - can be inferred using #infer_language method
+  # * +target_lang+     - Target 2 letter ISO language code to which the source needs to be translated in to.
+  # 
+  def translate(input_text, src_lang, target_lang)
+    raise "Not Implemented. Class #{self.class.name} doesn't implement translate"
+  end
+end

data/lib/scc.rb

@@ -1,13 +1,47 @@
-require 'aws-sdk-translate'
-require 'aws-sdk-comprehend'
+require_relative "engines/translator"
+require_relative "allfather"
 
+#
+# Library to handle SCC Files
+#
+# Uses the translator available to do the necessary language operations
+# as defined by the AllFather
+#
 class SCC
 
-  def initialize(awskey, awssecret)
-    @translate = Aws::Translate::Client.new(:access_key_id => "#{awskey}", :secret_access_key => "#{awssecret}")
-    @comp = Aws::Comprehend::Client.new(:access_key_id => "#{awskey}", :secret_access_key => "#{awssecret}")
+  include AllFather
+
+  def initialize(cc_file, translator)
+    @cc_file = cc_file
+    @translator = translator
+    raise "Invalid SCC file provided" unless is_valid?
+  end
+
+  def is_valid?
+    # Do any SCC specific validations here
+    if @cc_file =~ /^.*\.(scc)$/
+      return true
+    end
+    return false
+  end
+
+  def infer_languages
+    lang = nil
+    begin
+      sample_text = get_text(@cc_file, 100)
+      lang = @translator.infer_language(sample_text)
+    rescue StandardError => e
+      puts "Error while detecting the language due to #{e.message}"
+    end
+    lang
   end
 
+  def translate(src_lang, dest_lang, out_file)
+    raise "Not Implemented. Class #{self.class.name} doesn't implement translate yet !!"
+  end
+
+  private
+
   def get_text(srt_file, num_chars)
     ccfile = File.open(srt_file, 'r:UTF-8', &:read)
     text_sample = ""
@@ -15,12 +49,12 @@ class SCC
       if line =~ /^\d\d:\d\d:\d\d:\d\d\s/
         scc_text_code = line.gsub(/^\d\d:\d\d:\d\d:\d\d\s/, '')
         text_sample << decode(scc_text_code)
-        if text_sample.length > (num_chars+1)
+        if text_sample.length > (num_chars + 1)
           break
         end
       end
     end
-    return text_sample[0,num_chars]
+    return text_sample[0, num_chars]
   end
 
   def decode(scc_code_text)
@@ -31,7 +65,7 @@ class SCC
     hex_codes.each do | code |
       if ["94", "91", "92", "97", "15", "16", "10", "13"].include?(code)
         skip_next = true
-        skip_count = skip_count +1
+        skip_count = skip_count + 1
         next
       end
       if skip_count == 1 && skip_next
@@ -60,18 +94,4 @@ class SCC
     end
     encoded_str
   end
-
-  def detect_lang(scc_file)
-    lang = nil
-    begin
-      sample_text = get_text(scc_file, 100)
-      response = @comp.detect_dominant_language( {
-                                                     text: "#{sample_text}"
-                                                 })
-      lang = response[:languages][0][:language_code] rescue nil
-    rescue => error
-      puts "Error while detecting the language!!"
-    end
-    lang
-  end
-end
+end

data/lib/srt.rb

@@ -1,81 +1,106 @@
-require 'aws-sdk-translate'
-require 'aws-sdk-comprehend'
+require_relative "engines/translator"
+require_relative "allfather"
 
+#
+# Library to handle SRT Files
+#
+# Uses the translator available to do the necessary language operations
+# as defined by the AllFather
+#
 class SRT
-  def initialize(awskey, awssecret)
-    @translate = Aws::Translate::Client.new(:access_key_id => "#{awskey}", :secret_access_key => "#{awssecret}")
-    @comp = Aws::Comprehend::Client.new(:access_key_id => "#{awskey}", :secret_access_key => "#{awssecret}")
+
+  include AllFather
+
+  def initialize(cc_file, translator)
+    @cc_file = cc_file
+    @translator = translator
+    raise "Invalid SRT file provided" unless is_valid?
+  end
+
+  def is_valid?
+    # Do any SRT specific validations here
+    if @cc_file =~ /^.*\.(srt)$/
+      return true
+    end
+    return false
   end
 
-  def translate_text(srt_file, src_lang, dest_lang, out_file)
-    ccfile = File.open(srt_file, 'r:UTF-8', &:read)
-    outfile = File.open(out_file, "w")
-    text_collection = false
-    text_sample = ""
-    ccfile.each_line do | line |
-      if line =~ /^(\d\d:)\d\d:\d\d[,.]\d\d\d.*-->.*(\d\d:)\d\d:\d\d[,.]\d\d\d/
-        text_collection = true
-        outfile.puts line
-      elsif line.strip.empty? && !text_sample.empty?
-        json_text = JSON.parse(text_sample) rescue nil
-        if json_text.nil?
-          trans_resp = @translate.translate_text({ :text => "#{text_sample}" , :source_language_code => "#{src_lang}", :target_language_code => "#{dest_lang}"})
-          outfile.puts trans_resp.translated_text
+  def translate(src_lang, dest_lang, out_file)
+    super(src_lang, dest_lang, out_file)
+    begin
+      ccfile = File.open(@cc_file, 'r:UTF-8', &:read)
+      outfile = File.open(out_file, "w")
+      text_collection = false
+      text_sample = ""
+      ccfile.each_line do | line |
+        if line =~ /^(\d\d:)\d\d:\d\d[,.]\d\d\d.*-->.*(\d\d:)\d\d:\d\d[,.]\d\d\d/
+          text_collection = true
+          outfile.puts line
+        elsif line.strip.empty? && !text_sample.empty?
+          json_text = JSON.parse(text_sample) rescue nil
+          if json_text.nil?
+            trans_resp = @translator.translate(text_sample, src_lang, dest_lang)
+            outfile.puts trans_resp
+          else
+            outfile.puts text_sample
+          end
           outfile.puts
+          text_sample = ""
+          text_collection = false
+        elsif text_collection
+          text_sample << line
         else
-          outfile.puts text_sample
-          outfile.puts
+          outfile.puts line
         end
-        text_sample = ""
-        text_collection = false
-      elsif text_collection
-        text_sample << line
-      else
-        outfile.puts line
       end
-      next
-    end
 
-    if !text_sample.empty?
-      trans_resp = @translate.translate_text({ :text => "#{text_sample}" , :source_language_code => "#{src_lang}", :target_language_code => "#{dest_lang}"})
-      outfile.puts trans_resp.translated_text
-      outfile.puts
+      if !text_sample.empty?
+        trans_resp = @translator.translate(text_sample, src_lang, dest_lang)
+        outfile.puts trans_resp
+        outfile.puts
+      end
+    ensure
+      ccfile.close rescue nil
       outfile.close
     end
   end
 
-
-  def get_text(srt_file, num_chars)
-    ccfile = File.open(srt_file, 'r:UTF-8', &:read)
-    text_collection = false
-    text_sample = ""
-    ccfile.each_line do | line |
-      line = line
-      if line =~ /^(\d\d:)\d\d:\d\d[,.]\d\d\d.*-->.*(\d\d:)\d\d:\d\d[,.]\d\d\d/
-        text_collection = true
-      elsif line.strip.empty?
-        text_collection = false
-      elsif text_collection && text_sample.length < (num_chars+1)
-        text_sample << line
-      end
-      break if text_sample.length > (num_chars+1)
-      next
+  def infer_languages
+    lang = nil
+    begin
+      sample_text = get_text(@cc_file, 100)
+      lang = @translator.infer_language(sample_text)
+    rescue StandardError => e
+      puts "Error while detecting the language due to #{e.message}"
     end
-    return text_sample[0,num_chars]
+    [lang]
   end
 
-  def detect_lang(srt_file)
-    lang = nil
+  private 
+
+  # 
+  # Method to get a minimal amount of key text that excludes any tags
+  # or control information for the engine to meaninfully and 
+  # correctly infer the language being referred to in ths VTT
+  #
+  def get_text(srt_file, num_chars)
     begin
-    sample_text = get_text(srt_file, 100)
-    response = @comp.detect_dominant_language( {
-        text: "#{sample_text}"
-                                    })
-    lang = response[:languages][0][:language_code] rescue nil
-    rescue => error
-      puts "Error while detecting the language!!"
+      ccfile = File.open(srt_file, 'r:UTF-8', &:read)
+      text_collection = false
+      text_sample = ""
+      ccfile.each_line do |line|
+        if line =~ /^(\d\d:)\d\d:\d\d[,.]\d\d\d.*-->.*(\d\d:)\d\d:\d\d[,.]\d\d\d/
+          text_collection = true
+        elsif line.strip.empty?
+          text_collection = false
+        elsif text_collection && text_sample.length < (num_chars + 1)
+          text_sample << line
+        end
+        break if text_sample.length > (num_chars + 1)
+      end
+    ensure
+      ccfile.close rescue nil
     end
-    lang
+    return text_sample[0, num_chars]
   end
-
 end

data/lib/subtitle.rb

@@ -1,43 +1,88 @@
-require "srt"
+require_relative "srt"
+require_relative "vtt"
+require_relative "scc"
+require_relative "ttml"
+require_relative "dfxp"
+require_relative "allfather"
+require_relative "engines/translator"
+require_relative "engines/aws"
+
 
 class Subtitle
-  def initialize(awskey, awssecret, ccfile)
-    if awskey.nil? || awssecret.nil? || ccfile.nil?
-      raise "Invalid Arguments, please check"
-    end
-    @ccfile = ccfile
-    unless file_valid
-      raise "Incorrect File extension"
-    end
-    begin
-      @srt_parser = SRT.new(awskey, awssecret)
-    rescue
-      raise "Could not initialize Parser!!. Check the Keys supplied."
-    end
+  def initialize(options={})
+    # Infer the caption handler from the extension
+    @cc_file = options[:cc_file]
+    raise "Input caption not provided. Please provide the same in :cc_file option" if @cc_file.nil?
+    translator = get_translator(options)
+    @handler = get_caption_handler(options, translator)
   end
 
   def detect_language
-    detected_lang = @srt_parser.detect_lang(@ccfile)
-    detected_lang
+    @handler.infer_languages
   end
 
-  def translate_cc( dest_lang, src_lang = nil, outfile = nil)
+  def translate(dest_lang, src_lang = nil, outfile = nil)
     if outfile.nil?
-      outfile = "#{@ccfile}_#{dest_lang}"
+      outfile = "#{@cc_file}_#{dest_lang}"
     end
     if src_lang.nil?
-      src_lang = detect_language
-      raise "could not detect Source Language!!"  if src_lang.nil?
+      src_lang = detect_language[0] rescue nil
+      raise "Could not detect Source Language!!" if src_lang.nil?
     end
-    @srt_parser.translate_text(@ccfile, src_lang, dest_lang, outfile)
+    @handler.translate(src_lang, dest_lang, outfile)
     outfile
   end
 
-  def file_valid
-    valid = false
-    if @ccfile =~ /^.*\.(srt|vtt)$/
-      valid = true
+  private
+
+  def get_translator(options)
+    translator = nil
+    # Try to infer the engine based on the passed options
+    engine = options[:engine]
+    unless engine
+      engine_props = Translator::ENGINE_KEYS
+      engine_props.each do |k, values|
+        original_size = values.size 
+        diff = values - options.keys
+        if diff.size < original_size
+          # We have some keys for this engine in options
+          engine = k
+          break
+        end
+      end
+    end
+    case engine 
+    when Translator::ENGINE_AWS
+      translator = AwsEngine.new(options)
+    when Translator::ENGINE_GCP
+      raise "GCP is yet to be implemented"
+    else
+      raise "Unable to infer the Translation Engine. Options missing key credential params"
+    end
+    translator
+  end
+
+  def get_caption_handler(options, translator)
+    caption_file = options[:cc_file]
+    extension = File.extname(caption_file)
+    unless AllFather::VALID_FILES.include?(extension)
+      raise "Caption support for #{caption_file} of type #{extension} is not supported yet" 
+    end
+    handler = nil
+    case extension.downcase
+    when ".scc"
+      handler = SCC.new(caption_file, translator)
+    when ".srt"
+      handler = SRT.new(caption_file, translator)
+    when ".vtt"
+      handler = VTT.new(caption_file, translator)
+    when ".ttml"
+      handler = TTML.new(caption_file, translator, {:force_detect => options[:force_detect]})
+    when ".dfxp"
+      handler = DFXP.new(caption_file, translator, {:force_detect => options[:force_detect]})
+    else
+      raise "Cannot handle file type .#{extension}"
     end
-    valid
+    handler
   end
-end
+end

data/lib/ttml.rb

@@ -0,0 +1,180 @@
+require_relative "engines/translator"
+require_relative "allfather"
+
+require "nokogiri"
+
+#
+# Library to handle TTML Files
+#
+# Uses the translator available to do the necessary language operations
+# as defined by the AllFather
+#
+class TTML
+
+  include AllFather
+
+  def initialize(cc_file, translator, opts={})
+    @cc_file = cc_file
+    @translator = translator
+    @force_detect = opts[:force_detect] || false
+    raise "Invalid TTML file provided" unless is_valid?
+  end
+
+  def is_valid?
+    # Do any VTT specific validations here
+    if @cc_file =~ /^.*\.(ttml)$/
+      return true
+    end
+    # TODO: Check if it's required to do a File read to see if this
+    # a well-formed XML. Another is to see if lang is available in each div
+    return false
+  end
+
+  def infer_languages
+    lang = []
+    begin
+      xml_file = File.open(@cc_file)
+      xml_doc  = Nokogiri::XML(xml_file)
+      div_objects = xml_doc.css("/tt/body/div")
+      div_objects.each_with_index do |div, index|
+        # By default, return the lang if specified in the div and 
+        # force detect is false
+        inferred_lang = div.attributes['lang'].value rescue nil
+        if inferred_lang.nil?
+          # If lang is not provided in the caption, then override
+          # force detect for inferrence
+          @force_detect = true
+        end
+        if @force_detect
+          sample_text = get_text(div, 100)
+          inferred_lang = @translator.infer_language(sample_text) rescue nil
+          if inferred_lang.nil?
+            err_msg = "Failed to detect lang for div block number #{index + 1}"
+            unless lang.empty?
+              err_msg += "; Detected languages before failure are #{lang}"
+            end
+            raise AllFather::LangDetectionFailureException.new(err_msg)
+          end
+        end
+        lang << inferred_lang
+      end
+    rescue StandardError => e
+      puts "Error while detecting the language due to #{e.message}"
+    ensure
+      xml_file.close rescue nil
+    end
+    return nil if lang.empty?
+    lang
+  end
+
+  def translate(src_lang, dest_lang, out_file)
+    super(src_lang, dest_lang, out_file)
+    xml_file = File.open(@cc_file, 'r:UTF-8', &:read)
+    xml_doc  = Nokogiri::XML(xml_file)
+    div_objects = xml_doc.css("/tt/body/div")
+    # Irrespective of what lang the div xml:lang says, infer the lang and then
+    # check to see if it matches src_lang
+    matched_div = nil
+    div_objects.each do |div|
+      sample_text = get_text(div, 100)
+      inferred_lang = @translator.infer_language(sample_text) rescue nil
+      next if inferred_lang.nil?
+      if inferred_lang.eql?(src_lang)
+        matched_div = div 
+        break 
+      end
+    end
+    if matched_div.nil?
+      FileUtils.remove_file(out_file)
+      raise AllFather::InvalidInputException.new("Unable to find #{src_lang} language section in TTML")
+    end
+    # Update the Lang in the Div
+    matched_div.lang = dest_lang
+
+    blocks = matched_div.css("p")
+    blocks.each do |block|
+      # Multiple spaces being stripped off
+      text = block.inner_html.strip.gsub(/(\s){2,}/, '')
+      text_blocks = get_block_text(text)
+      translated_text = ""
+      text_blocks.each do |text_block|
+        if text_block.start_with?('<') || text_block.empty?
+          translated_text << text_block
+          next
+        end
+        translated_resp = @translator.translate(text_block, src_lang, dest_lang)
+        translated_text << translated_resp
+      end
+      block.inner_html = translated_text
+    end
+    xml_file.close rescue nil
+    File.write(out_file, xml_doc)
+    out_file
+  end
+
+  private
+
+  #
+  # Method to segregate the data from markups as markups don't need
+  # translations.
+  # For example, if the cue block is of the form
+  # This is a test caption with <span id="1">a test span </span> within a block
+  # This method returns
+  # ["This is a test caption with ", "<span id=\"1\">", "a test span ", "</span>", " within a block"]
+  # as we can infer the markups can be retained as is to avoid translation
+  #
+  def get_block_text(text)
+    data = []
+    tag_start = tag_end = false
+    str_length = text.size
+    text_block = ""
+    markup_block = ""
+    for i in 0...text.size do
+      if text[i] == '<'
+        tag_end = false
+        tag_start = true
+        markup_block << text[i]
+        data << text_block
+        text_block = ""
+        next 
+      elsif text[i] == '>'
+        tag_end = true
+        tag_start = false
+        markup_block << text[i]
+        data << markup_block
+        markup_block = ""
+        next
+      end
+      if tag_start && !tag_end
+        markup_block << text[i]
+      else
+        text_block << text[i]
+      end
+    end
+    unless text_block.empty?
+      data << text_block
+    end
+    data
+  end
+
+  # 
+  # Method to get a minimal amount of key text that excludes any tags
+  # or control information for the engine to meaninfully and 
+  # correctly infer the language being referred to in ths TTML
+  #
+  def get_text(div, num_chars)
+    text_sample = ""
+    blocks = div.css("p")
+    blocks.each do |block|
+      # Multiple spaces being stripped off
+      text = block.inner_html.strip.gsub(/(\s){2,}/, '')
+      # Strip off html tags (if any)
+      text = text.gsub(/(<.*?>)/, ' ')
+      text_sample << text
+      if text_sample.length > (num_chars + 1)
+        break
+      end
+    end
+    return text_sample[0, num_chars]
+  end
+end

data/lib/vtt.rb

@@ -1,81 +1,115 @@
-require 'aws-sdk-translate'
-require 'aws-sdk-comprehend'
+require_relative "engines/translator"
+require_relative "allfather"
 
+#
+# Library to handle VTT Files
+#
+# Uses the translator available to do the necessary language operations
+# as defined by the AllFather
+#
 class VTT
-  def initialize(awskey, awssecret)
-    @translate = Aws::Translate::Client.new(:access_key_id => "#{awskey}", :secret_access_key => "#{awssecret}")
-    @comp = Aws::Comprehend::Client.new(:access_key_id => "#{awskey}", :secret_access_key => "#{awssecret}")
+
+  include AllFather
+
+  def initialize(cc_file, translator)
+    @cc_file = cc_file
+    @translator = translator
+    raise "Invalid VTT file provided" unless is_valid?
   end
 
-  def translate_text(srt_file, src_lang, dest_lang, out_file)
-    ccfile = File.open(srt_file, 'r:UTF-8', &:read)
-    outfile = File.open(out_file, "w")
-    text_collection = false
-    text_sample = ""
-    ccfile.each_line do | line |
-      if line =~ /^(\d\d:)\d\d:\d\d[,.]\d\d\d.*-->.*(\d\d:)\d\d:\d\d[,.]\d\d\d/
-        text_collection = true
-        outfile.puts line
-      elsif line.strip.empty? && !text_sample.empty?
-        json_text = JSON.parse(text_sample) rescue nil
-        if json_text.nil?
-          trans_resp = @translate.translate_text({ :text => "#{text_sample}" , :source_language_code => "#{src_lang}", :target_language_code => "#{dest_lang}"})
-          outfile.puts trans_resp.translated_text
-          outfile.puts
+  def translate(src_lang, dest_lang, out_file)
+    super(src_lang, dest_lang, out_file)
+    begin
+      ccfile = File.open(@cc_file, 'r:UTF-8', &:read)
+      outfile = File.open(out_file, "w")
+      text_collection = false
+      text_sample = ""
+      ccfile.each_line do | line |
+        if line =~ /^(\d\d:)\d\d:\d\d[,.]\d\d\d.*-->.*(\d\d:)\d\d:\d\d[,.]\d\d\d/
+          text_collection = true
+          outfile.puts line
+        elsif line.strip.empty? && !text_sample.empty?
+          json_text = JSON.parse(text_sample) rescue nil
+          if json_text.nil?
+            trans_resp = @translator.translate(text_sample, src_lang, dest_lang)
+            outfile.puts trans_resp
+            outfile.puts
+          else
+            outfile.puts text_sample
+            outfile.puts
+          end
+          text_sample = ""
+          text_collection = false
+        elsif text_collection
+          text_sample << line
         else
-          outfile.puts text_sample
-          outfile.puts
+          outfile.puts line
         end
-        text_sample = ""
-        text_collection = false
-      elsif text_collection
-        text_sample << line
-      else
-        outfile.puts line
       end
-      next
-    end
 
-    if !text_sample.empty?
-      trans_resp = @translate.translate_text({ :text => "#{text_sample}" , :source_language_code => "#{src_lang}", :target_language_code => "#{dest_lang}"})
-      outfile.puts trans_resp.translated_text
-      outfile.puts
+      if !text_sample.empty?
+        trans_resp = @translator.translate(text_sample, src_lang, dest_lang)
+        outfile.puts trans_resp
+        outfile.puts
+      end
+    ensure
+      ccfile.close rescue nil
       outfile.close
     end
   end
 
+  #
+  # Returns the inferred language in an array
+  #
+  def infer_languages
+    lang = nil
+    begin
+      sample_text = get_text(@cc_file, 100)
+      lang = @translator.infer_language(sample_text)
+    rescue StandardError => e
+      puts "Error while detecting the language due to #{e.message}"
+    end
+    [lang]
+  end
 
-  def get_text(srt_file, num_chars)
-    ccfile = File.open(srt_file, 'r:UTF-8', &:read)
-    text_collection = false
-    text_sample = ""
-    ccfile.each_line do | line |
-      line = line
-      if line =~ /^(\d\d:)\d\d:\d\d[,.]\d\d\d.*-->.*(\d\d:)\d\d:\d\d[,.]\d\d\d/
-        text_collection = true
-      elsif line.strip.empty?
-        text_collection = false
-      elsif text_collection && text_sample.length < (num_chars+1)
-        text_sample << line
-      end
-      break if text_sample.length > (num_chars+1)
-      next
+  # 
+  # Method to add required set of validations specific to caption type
+  #
+  def is_valid?
+    # Do any VTT specific validations here
+    if @cc_file =~ /^.*\.(vtt)$/
+      return true
     end
-    return text_sample[0,num_chars]
+    # TODO: Check if it's required to do a File read to see if the 1st line is WEBVTT
+    # to handle cases where invalid file is named with vtt extension
+    return false
   end
 
-  def detect_lang(srt_file)
-    lang = nil
+  private 
+
+  # 
+  # Method to get a minimal amount of key text that excludes any tags
+  # or control information for the engine to meaninfully and 
+  # correctly infer the language being referred to in ths VTT
+  #
+  def get_text(vtt_file, num_chars)
     begin
-      sample_text = get_text(srt_file, 100)
-      response = @comp.detect_dominant_language( {
-                                                     text: "#{sample_text}"
-                                                 })
-      lang = response[:languages][0][:language_code] rescue nil
-    rescue => error
-      puts "Error while detecting the language!!"
+      ccfile = File.open(vtt_file, 'r:UTF-8', &:read)
+      text_collection = false
+      text_sample = ""
+      ccfile.each_line do |line|
+        if line =~ /^(\d\d:)\d\d:\d\d[,.]\d\d\d.*-->.*(\d\d:)\d\d:\d\d[,.]\d\d\d/
+          text_collection = true
+        elsif line.strip.empty?
+          text_collection = false
+        elsif text_collection && text_sample.length < (num_chars + 1)
+          text_sample << line
+        end
+        break if text_sample.length > (num_chars + 1)
+      end
+    ensure
+      ccfile.close rescue nil
     end
-    lang
+    return text_sample[0, num_chars]
   end
-
 end

metadata

@@ -1,14 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: subtitle
 version: !ruby/object:Gem::Version
-  version: 0.1.8
+  version: 0.2.4
 platform: ruby
 authors:
 - Maheshwaran G
+- Arunjeyaprasad A J
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-10-21 00:00:00.000000000 Z
+date: 2019-10-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -25,43 +26,37 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.0'
 - !ruby/object:Gem::Dependency
-  name: aws-sdk-comprehend
+  name: aws-sdk
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: aws-sdk-translate
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
+        version: '2.11'
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-description: subtitle gem to detect and translate closed caption for SubRip and WebVTT
+        version: '2.11'
+description: Subtitle gem helps you to detect language and translate closed caption
+  to required language.
 email:
 - pgmaheshwaran@gmail.com
+- arunjeyaprasad@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- lib/allfather.rb
+- lib/dfxp.rb
+- lib/engines/aws.rb
+- lib/engines/gcp.rb
+- lib/engines/translator.rb
 - lib/scc.rb
 - lib/srt.rb
 - lib/subtitle.rb
+- lib/ttml.rb
 - lib/vtt.rb
 homepage: https://github.com/cloudaffair/subtitle
 licenses:
@@ -85,9 +80,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.3
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
-summary: subtitle helps you to detect language and translate closed caption to required
-  language
+summary: Subtitle gem helps you to detect language and translate closed caption to
+  required language
 test_files: []