terrestrial-cli 0.1.0

data/lib/terrestrial/cli/parser/android_xml.rb

@@ -0,0 +1,82 @@
+module Terrestrial
+  module Cli
+    module Parser
+      class AndroidXML < BaseParser
+        LANGUAGE = :android_xml
+
+        def self.find_strings(file)
+          self.new(file).find_strings
+        end
+
+        def self.find_api_calls(file)
+          self.new(file).find_api_calls
+        end
+
+        def initialize(file)
+          @path = file
+          @file = File.new(file)
+          @document = REXML::Document.new(@file)
+        end
+
+        def find_strings
+          result = []
+          REXML::XPath.each(@document, "//resources/string") do |node|
+            result << build_new_string_entry(node)
+          end
+          result
+        end
+
+        def find_api_calls
+          result = []
+          REXML::XPath.each(@document, "//resources/string[@terrestrial=\"true\"]") do |node|
+            result << build_registry_entry_hash(node)
+          end
+          result
+        end
+
+        def build_new_string_entry(node)
+          Hash.new.tap do |entry|
+            entry["language"] = LANGUAGE
+            entry["file"] = @path
+            entry["string"] = get_string_from_node(node)
+            entry["type"] = "android-strings-xml"
+            entry["line_number"] = nil
+            # entry.variables = get_variables_from_string(entry.string)
+            entry["identifier"] = node.attributes["name"]
+          end
+        end
+
+        def build_registry_entry_hash(node)
+          Hash.new.tap do |entry|
+            entry["string"] = get_string_from_node(node)
+            entry["context"] = node.attributes["context"] || ""
+            entry["file"] = @path
+            entry["line_number"] = nil
+            entry["type"] = "android-strings-xml"
+            entry["id"] = node.attributes["name"]
+          end
+        end
+
+        def get_string_from_node(node)
+          # Why could the text be nil?
+          #  - If it contains valid XML!
+          #
+          # We assume anything inside the string tag is actually 
+          # what should be shown in the UI, so we just parse it 
+          # as a string if we realise that the parser thinks it
+          # is XML.
+
+          if !node.get_text.nil?
+            node.get_text.value
+          else
+            node.children.first.to_s
+          end
+        end
+
+        def get_variables_from_string(string)
+          string.scan(/(\%\d\$[dsf])/).map {|match| match[0] }
+        end
+      end
+    end
+  end
+end

data/lib/terrestrial/cli/parser/base_parser.rb

@@ -0,0 +1,42 @@
+module Terrestrial
+  module Cli
+    module Parser
+      class BaseParser
+
+        # Interface for finding locations in files where the Terrestrial
+        # API will be accessing strings
+        #
+        #   file - path to source file
+        #
+        #   Expected to return an array of
+        #   Bootstrapper::NewStringEntry
+        #   objects
+        def self.find_api_calls(file)
+          raise "Not implemented"
+        end
+
+        # Interface for finding strings in a source file
+        #
+        #   file - path to source file
+        #
+        #   Expected to return an array of
+        #   hashes TODO: make return an object
+        def self.find_string(file)
+          raise "Not implemented"
+        end
+
+        def self.find_nslocalizedstrings(file)
+          raise "Not implemented"
+        end
+        
+
+
+        def self.scan_lines(path)
+          File.readlines(file).each_with_index do |line, index|
+            yield line, index
+          end
+        end
+      end
+    end
+  end
+end

data/lib/terrestrial/cli/parser/objc.rb

@@ -0,0 +1,127 @@
+module Terrestrial
+  module Cli
+    module Parser
+      class ObjC < BaseParser
+        LANGUAGE = :objc
+
+        STRING_REGEX = /@"(.*?)"/
+        NSLOCALIZEDSTRING_REGEX = /NSLocalizedString\(.*@"(.*)".*\)/
+        DOT_TRANSLATED_REGEX = /@"([^"]*)".translated\W/
+        TRANSLATED_WITH_CONTEXT_REGEX = /@"([^"]*)"\stranslatedWithContext:/
+
+        def self.find_strings(file)
+          results = []
+          if is_view_controller?(file)
+            File.readlines(file, :encoding => "UTF-8").each_with_index do |line, index|
+              line.encode!('UTF-16', :undef => :replace, :invalid => :replace, :replace => "")
+              line.encode!('UTF-8')
+              results.concat(analyse_line_for_strings(line,index, file))
+            end
+          end
+          results
+        end
+
+        def self.analyse_line_for_strings(line, index, file_path)
+          results = []
+          line.scan(STRING_REGEX).each do |match|
+            unless looks_suspicious(line)
+              results.push(Hash.new.tap do |entry|
+                entry["language"] = LANGUAGE
+                entry["file"] = file_path
+                entry["line_number"] = index + 1
+                entry["string"] = match[0]
+                entry["type"] = guess_type(line)
+               # entry.variables = get_variable_names(line) if entry.type == "stringWithFormat"
+
+              end)
+            end
+          end
+          results
+        end
+
+        def self.find_api_calls(file)
+          results = []
+          File.readlines(file).each_with_index do |line, index|
+            results.concat(analyse_line_for_dot_translated(line, index, file))
+            results.concat(analyse_line_for_translatedWithContext(line, index, file))
+          end
+          results
+        end
+
+        def self.analyse_line_for_translatedWithContext(line, index, file_path)
+          results = []
+          line.scan(TRANSLATED_WITH_CONTEXT_REGEX).each do |match|
+            results.push(Hash.new.tap do |h|
+              h["file"] = file_path
+              h["line_number"] = index + 1
+              h["string"] = match[0]
+              h["type"] = "translatedWithContext"
+              h["context"] = get_context(line, h["string"])
+            end)
+          end
+          results
+        end
+
+        def self.analyse_line_for_dot_translated(line, index, file_path)
+          results = []
+          line.scan(DOT_TRANSLATED_REGEX).each do |match|
+            results.push(Hash.new.tap do |h|
+              h["file"] = file_path
+              h["line_number"] = index + 1
+              h["string"] = match[0]
+              h["type"] = ".translated"
+              h["context"] = ""
+            end)
+          end
+          results
+        end
+
+        def self.get_context(line, match)
+          line.match(/"#{match}" translatedWithContext:\s?@"([^"]*)"/)[1]
+        end
+
+        def self.guess_type(line)
+          if line.include? "stringWithFormat"
+            "stringWithFormat"
+          else
+            "unknown"
+          end
+        end
+
+        def self.get_variable_names(line)
+          line
+            .scan(/stringWithFormat:\s?@"[^"]+",\s?(.*?)\][^\s*,]/)
+            .first.first # Array of arrays Yo.
+            .split(",")
+            .map {|var| var.gsub(/\s+/, "")}
+        end
+
+        def self.looks_suspicious(line)
+          without_strings = line.gsub(STRING_REGEX, "")
+          without_strings.include?("_LOG") || 
+          without_strings.include?("DLog") || 
+          without_strings.include?("NSLog") || 
+          without_strings.include?("NSAssert") ||
+          without_strings.downcase.include?("uistoryboard") ||
+          without_strings.downcase.include?("instantiateviewcontrollerwithidentifier") ||
+          without_strings.downcase.include?("uiimage") ||
+          without_strings.downcase.include?("nsentitydescription") ||
+          without_strings.downcase.include?("nspredicate") ||
+          without_strings.downcase.include?("dateformat") ||
+          without_strings.downcase.include?("datefromstring") ||
+          without_strings.downcase.include?("==") ||
+          without_strings.downcase.include?("isequaltostring") ||
+          without_strings.downcase.include?("valueforkey") ||
+          without_strings.downcase.include?("cellidentifier") ||
+          without_strings.downcase.include?("uifont") ||
+          without_strings.downcase.include?("static ") ||
+          without_strings.downcase.include?("print(")
+        end
+
+        def self.is_view_controller?(file)
+          !file.match(/ViewController/).nil?
+        end
+      end
+    end
+  end
+end

data/lib/terrestrial/cli/parser/storyboard.rb

@@ -0,0 +1,166 @@
+require 'rexml/document'
+
+module Terrestrial
+  module Cli
+    module Parser
+      class Storyboard
+        LANGUAGE = :ios_storyboard
+        attr_reader  :result
+        
+        include REXML
+        QUERIES = {
+          "storyboard-label" => "//label",
+          "storyboard-text-field" => "//textField",
+          "storyboard-button" => "//button/state",
+          "storyboard-bar-button-item" => "//barButtonItem",
+          "storyboard-navbar-item" => "//navigationItem",
+          "storyboard-text-view" => "//textView"
+        }
+
+        TEXT_ATTRIBUTE = {
+          "storyboard-label" => "text",
+          "storyboard-text-field" => "placeholder",
+          "storyboard-button" => "title",
+          "storyboard-bar-button-item" => "title",
+          "storyboard-navbar-item" => "title",
+          "storyboard-text-view" => "text"
+        }
+
+        TYPES = QUERIES.keys
+
+        def initialize(file)
+          @path = file
+          @file = File.new(file)
+          @document = Document.new(@file)
+          @result = []
+        end
+
+        def self.find_strings(file)
+          self.new(file).find_strings
+        end
+
+        def self.find_api_calls(file)
+          self.new(file).find_api_calls
+        end
+
+        def find_api_calls
+          labels = []
+          XPath.each(@document, api_calls_query) do |node|
+            type    = type_for(node.name)
+            string  = get_string(node)
+            context = get_context(node)
+
+            labels << build_registry_entry_hash(string, context, type)
+          end
+          @result = labels
+          @result
+        end
+
+        def find_strings
+          TYPES.each do |type|
+            @result.concat(find_entries_for_type(type))
+          end
+          @result
+        end
+
+        
+
+        def type_for(name)
+          {
+            "label"          => "storyboard-label",
+            "textField"      => "storyboard-text-field",
+            "button"         => "storyboard-button",
+            "barButtonItem"  => "storyboard-bar-button-item",
+            "navigationItem" => "storyboard-navbar-item",
+            "textView"       => "storyboard-text-view",
+          }[name]
+        end
+
+        def find_entries_for_type(type)
+          labels = []
+          XPath.each(@document, QUERIES[type]) do |node|
+            labels << new_entry(node.attributes[TEXT_ATTRIBUTE[type]], 
+                                    type: type, 
+                                      id: get_id(node))
+          end
+          labels
+        end
+
+        def get_id(node)
+          # Why? Because the button's text is not in the
+          # button, but in a child element, that doesn't have
+          # an ID. So for most situations you'll just pick the
+          # ID off the element, but sometimes we'll have to 
+          # traverse back up to get the ID. If that element
+          # doesn't have an ID, it's a new situation and
+          # we should get an exception down the line.
+
+          target = node
+          if target.attributes["id"].nil?
+            target.parent.attributes["id"]
+          else
+            target.attributes["id"]
+          end
+        end
+
+        def get_context(node)
+          # //textView/userDefinedRuntimeAttributes/userDefinedRuntimeAttribute[@keyPath="contextInfo"]
+          context = ""
+          attributes = node.elements.select {|e| e.name == "userDefinedRuntimeAttributes"}.first
+          attributes.each_element_with_attribute("keyPath", "contextInfo") do |e| 
+            context = e.attributes["value"]
+          end
+          context
+        end
+
+        def get_string(node)
+          type = type_for(node.name)
+          if type == "storyboard-button"
+            node.elements.select {|e| e.name == "state"}.first.attributes[TEXT_ATTRIBUTE[type]]
+          else
+            node.attributes[TEXT_ATTRIBUTE[type]]
+          end
+        end
+
+        def api_calls_query
+          # Finds all the attributes that say that an element is 
+          # translated by Terrestrial, and we then traverse
+          # two parents up:
+          #
+          # <*targetElement*>
+          #   <userDefinedRuntimeAttributes>
+          #     <userDefinedRuntimeAttribute ... />   <- these are what we find
+
+          '//userDefinedRuntimeAttribute[@type="boolean" and @value="YES"]/../..'
+        end
+        
+        def new_entry(string, opts)
+          defaults = { type: "storyboard" }
+          values = defaults.merge(opts)
+
+          Hash.new.tap do |entry|
+            entry["file"] = @path
+            entry["language"] = LANGUAGE
+            entry["string"] = string.to_s
+            entry["type"] = values.fetch(:type)
+            entry["line_number"] = nil
+            entry["metadata"] = {
+              "storyboard_element_id" => values.fetch(:id)
+            }
+          end
+        end
+
+        def build_registry_entry_hash(string, context, type)
+          Hash.new.tap do |entry|
+            entry["string"] = string.to_s
+            entry["language"] = LANGUAGE
+            entry["context"] = context || ""
+            entry["file"] = @path
+            entry["line_number"] = nil
+            entry["type"] = type
+          end
+        end
+      end
+    end
+  end
+end

data/lib/terrestrial/cli/parser/string_analyser.rb

@@ -0,0 +1,115 @@
+module Terrestrial
+  module Cli
+    module Parser
+      class StringAnalyser
+
+        def self.is_string_for_humans?(string, language, variables = [])
+          self.new(string, language, variables).decide
+        end
+
+        def initialize(string, language, variables = [])
+          @string    = string
+          @variables = variables || [] # TODO: Find out what was passing in variables as nil instead of empty array
+          @language  = language
+        end
+        
+        def decide
+          if @variables.any?
+            looks_like_string_without_variables?
+          else
+            if has_camel_case_words? || looks_like_sql? || is_number? || has_snake_case_words?
+              false
+            elsif number_of_words > 1 && percentage_of_none_alphanumeric < 0.15
+              true
+            elsif number_of_words == 1 && is_capitalised? && percentage_of_none_alphanumeric < 0.1
+              true
+            else
+              false
+            end
+          end
+        end
+
+        def is_number?
+          !(@string =~ /\A[-+]?[0-9]*\.?[0-9]+\Z/).nil?
+        end
+
+        def number_of_words
+          @string.split(" ").length
+        end
+
+        def percentage_of_none_alphanumeric
+          total = @string.split("").length.to_f
+          non_alphanumeric = @string
+                              .split("")
+                              .select {|c| /[0-9a-zA-Z i\s]/.match(c).nil? }
+                              .length
+                              .to_f
+
+          non_alphanumeric / total
+        end
+
+        def looks_like_sql?
+          # Handle SQL with clever regex
+          !@string.match(/(ALTER|CREATE|DROP) TABLE/).nil? ||
+          !@string.match(/(DELETE|SELECT|INSERT|UPDATE).+(FROM|INTO|SET)/).nil? ||
+          !@string.match(/(delete|select|insert|update).+(from|into|set)/).nil?
+        end
+
+        def has_weird_characters?
+          (@string.split("") & ["<",">","\\", "/","*"]).length > 0
+        end
+
+        def has_punctuation?
+          (@string.split("") & [".",",","=","&"]).length > 0
+        end
+
+        def has_camel_case_words?
+          @string.split(" ")
+            .select {|word| !word.match(/([a-zA-Z][a-z]+[A-Z][a-zA-Z]+)/).nil? }
+            .any?
+        end
+
+        def has_camel_case_words?
+          @string.split(" ")
+            .select {|word| !word.match(/([a-zA-Z][a-z]+[A-Z][a-zA-Z]+)/).nil? }
+            .any?
+        end
+
+        def has_snake_case_words?
+          @string.split(" ")
+            .select {|word| !word.match(/\b\w*(_\w*)+\b/).nil? }
+            .any?
+        end
+
+        def is_capitalised?
+          @string == @string.capitalize
+        end
+
+        def looks_like_string_without_variables?
+          # Strip away the variables, remove extra whitespace,
+          # and feed that string back into the system to see
+          # if it now looks human readable or not.
+
+          if @language == ObjC::LANGUAGE
+            new_string = @string
+                           .gsub(/(%@)|(%d)/, "")
+                           .gsub(/\s\s/, " ")
+          elsif @language == Swift::LANGUAGE
+            new_string = @string
+                           .gsub(/\\\(.*\)/, "")
+                           .gsub(/\s\s/, " ")
+          elsif @language == AndroidXML::LANGUAGE
+            new_string = @string
+
+            @variables.each do |v|
+              new_string = new_string.gsub(v, "")
+              new_string = new_string.gsub(/\s\s/, " ")
+            end
+          end
+
+          self.class.new(new_string, @language).decide
+        end
+      end
+    end
+  end
+end

data/lib/terrestrial/cli/parser/swift.rb

@@ -0,0 +1,102 @@
+module Terrestrial
+  module Cli
+    module Parser
+      class Swift
+        LANGUAGE = :swift
+
+        NSLOCALIZEDSTRING_REGEX = /NSLocalizedString\(.*"(.*)".*\)/
+        STRING_REGEX = /"([^"]*)"/
+        DOT_TRANSLATED_REGEX = /"([^"]*)".translated\W/
+        TRANSLATED_WITH_CONTEXT_REGEX = /"([^"]*)".translatedWithContext/
+        VARIABLE_REGEX = /\\\((.*?)\)/
+
+        def self.find_strings(file)
+          results = []
+          if is_view_controller?(file)
+            File.readlines(file).each_with_index do |line, index|
+              results.concat analyse_line_for_strings(line, index, file)
+            end
+          end
+          results
+        end
+
+        def self.analyse_line_for_strings(line, index, file_path)
+          results = []
+          line.scan(STRING_REGEX).each do |match|
+            unless looks_suspicious(line)
+              results.push(Hash.new.tap do |entry|
+                entry["language"] = LANGUAGE
+                entry["file"] = file_path
+                entry["line_number"] = index + 1
+                entry["string"] = match[0]
+                entry["type"] = find_variables(match[0]).any? ? "stringWithFormat" : "unknown"
+                # entry.variables = find_variables(match[0])
+              end)
+            end
+          end
+          results
+        end
+
+        def self.find_api_calls(file)
+          results = []
+          File.readlines(file).each_with_index do |line, index|
+            line.scan(DOT_TRANSLATED_REGEX).each do |match|
+              results.push(Hash.new.tap do |h|
+                h["file"] = file
+                h["line_number"] = index + 1
+                h["string"] = match[0]
+                h["type"] = ".translated"
+                h["context"] = ""
+              end)
+            end
+
+            line.scan(TRANSLATED_WITH_CONTEXT_REGEX).each do |match|
+              results.push(Hash.new.tap do |h|
+                h["file"] = file
+                h["line_number"] = index + 1
+                h["string"] = match[0]
+                h["type"] = "translatedWithContext"
+                h["context"] = get_context(line, h["string"])
+              end)
+            end
+          end
+          results
+        end
+
+        def self.get_context(line, match)
+          line.match(/"#{match}"\.translatedWithContext\("([^"]*)"\)/)[1]
+        end
+
+        def self.find_variables(string)
+          # tries to find \(asd) inside the string itself
+          string.scan(VARIABLE_REGEX).map {|matches| matches[0]}
+        end
+
+        def self.is_view_controller?(file)
+          !file.match(/ViewController/).nil?
+        end
+
+        def self.looks_suspicious(line)
+          without_strings = line.gsub(STRING_REGEX, "")
+          without_strings.include?("_LOG") || 
+          without_strings.include?("DLog") || 
+          without_strings.include?("NSLog") || 
+          without_strings.include?("NSAssert") ||
+          without_strings.downcase.include?("uistoryboard") ||
+          without_strings.downcase.include?("instantiateviewcontrollerwithidentifier") ||
+          without_strings.downcase.include?("uiimage") ||
+          without_strings.downcase.include?("nsentitydescription") ||
+          without_strings.downcase.include?("nspredicate") ||
+          without_strings.downcase.include?("dateformat") ||
+          without_strings.downcase.include?("datefromstring") ||
+          without_strings.downcase.include?("==") ||
+          without_strings.downcase.include?("isequaltostring") ||
+          without_strings.downcase.include?("valueforkey") ||
+          without_strings.downcase.include?("cellidentifier") ||
+          without_strings.downcase.include?("uifont") ||
+          without_strings.downcase.include?("print(")
+        end
+      end
+    end
+  end
+end

data/lib/terrestrial/cli/parser.rb

@@ -0,0 +1,25 @@
+require 'terrestrial/cli/parser/base_parser'
+require 'terrestrial/cli/parser/objc'
+require 'terrestrial/cli/parser/swift'
+require 'terrestrial/cli/parser/storyboard'
+require 'terrestrial/cli/parser/android_xml'
+require 'terrestrial/cli/parser/string_analyser'
+
+module Terrestrial
+  module Cli
+    module Parser
+
+      def self.find_strings(file)
+        EngineMapper.parser_for(File.extname(file)).find_strings(file)
+      end
+
+      def self.find_api_calls(file)
+        EngineMapper.parser_for(File.extname(file)).find_api_calls(file)
+      end
+
+      def self.find_nslocalizedstrings(file)
+        EngineMapper.parser_for(File.extname(file)).find_nslocalizedstrings(file)
+      end
+    end
+  end
+end