rosette-core 1.0.1

data/lib/rosette/core/error_reporters/raising_error_reporter.rb

@@ -0,0 +1,27 @@
+# encoding: UTF-8
+
+module Rosette
+  module Core
+
+    # Raises errors instead of printing or logging them.
+    class RaisingErrorReporter < ErrorReporter
+      # Raises an error.
+      #
+      # @param [Exception] error The error to raise.
+      # @param [Hash] options A hash of associated options.
+      # @return [void]
+      def report_error(error, options = {})
+        raise error
+      end
+
+      # Does nothing.
+      #
+      # @param [Exception] error An error, but nothing is done with it.
+      # @param [Hash] options A hash of associated options.
+      # @return [void]
+      def report_warning(error, options = {})
+      end
+    end
+
+  end
+end

data/lib/rosette/core/errors.rb

@@ -0,0 +1,93 @@
+# encoding: UTF-8
+
+module Rosette
+  module Core
+
+    # Represents a generic syntax error that probably occurred when attempting
+    # to parse a bit of source code or other well-defined text format.
+    #
+    # @!attribute [r] message
+    #   @return [String] the error message.
+    # @!attribute [r] original_exception
+    #   @return [Exception] the original exception, probably from a parser of
+    #     some kind.
+    # @!attribute [r] language
+    #   @return [Symbol] the language that the text the parser attempted to
+    #     parse was in when this error was generated.
+    class SyntaxError < StandardError
+      attr_reader :message, :original_exception, :language
+
+      # Creates a new syntax error object.
+      #
+      # @param [String] msg The error message text.
+      # @param [Exception] original_exception The original error from the
+      #   parser or some such.
+      # @param [Symbol] language The source code language.
+      def initialize(msg, original_exception, language)
+        super(msg)
+        @message = msg
+        @original_exception = original_exception
+        @language = language
+      end
+
+      # A string representation of this error which combines both the original
+      # message as well as the language.
+      #
+      # @return [String]
+      def message
+        "#{@message} (#{language})"
+      end
+
+      alias :to_s :message
+    end
+
+    # Represents a generic syntax error generated by one of Rosette's
+    # extractors. Since extractors parse source code and other
+    # well-defined text formats, there's always the possibility that these formats
+    # will contain syntax errors.
+    #
+    # @!attribute [r] original_exception
+    #   @return [Exception] the original error from the parser or some such.
+    # @!attribute [r] language
+    #   @return [Symbol] the source code language.
+    # @!attribute [r] file
+    #   @return [String] the file that was being parsed when the error was
+    #     generated.
+    # @!attribute [r] commit_id
+    #   @return [String] the commit id that contained the file that was being
+    #     parsed when the error was generated.
+    class ExtractionSyntaxError < StandardError
+      attr_reader :original_exception, :language, :file, :commit_id
+
+      # Creates a new syntax error object.
+      #
+      # @param [String] msg The error message text.
+      # @param [Exception] original_exception The original error from the
+      #   parser or some such.
+      # @param [Symbol] language The source code language.
+      # @param [String] file The file that was being parsed when the error
+      #   was generated.
+      # @param [String] commit_id The commit id that contained the file that
+      #   was being parsed when the error was generated.
+      def initialize(msg, original_exception, language, file, commit_id)
+        super(msg)
+        @message = msg
+        @original_exception = original_exception
+        @language = language
+        @file = file
+        @commit_id = commit_id
+      end
+
+      # A string representation of this error which includes the original
+      # message, the source language, the file, and the commit id.
+      #
+      # @return [String]
+      def message
+        "#{@message}: #{original_exception.message} (#{language}) in #{file} at #{commit_id}"
+      end
+
+      alias :to_s :message
+    end
+
+  end
+end

data/lib/rosette/core/extractor/commit_log.rb

@@ -0,0 +1,33 @@
+# encoding: UTF-8
+
+module Rosette
+  module Core
+
+    # Represents a commit log. Commit logs track the status of individual
+    # commits, what time the commit occurred, and how many phrases it contains.
+    #
+    # @!attribute [rw] repo_name
+    #   @return [String] the name of the repo the commit was found in.
+    # @!attribute [rw] commit_id
+    #   @return [String] the git commit id.
+    # @!attribute [rw] phrase_count
+    #   @return [Fixnum] the number of phrases found in this commit.
+    # @!attribute [rw] status
+    #   @return [String] the [PhraseStatus] of this commit.
+    # @!attribute [rw] commit_datetime
+    #   @return [DateTime] the time this commit was made.
+    # @!attribute [rw] branch_name
+    #   @return [String] the name of the branch that contains this commit.
+    class CommitLog
+      def initialize(repo_name, commit_id, phrase_count = nil, status = nil, commit_datetime = nil, branch_name = nil)
+        @repo_name = repo_name
+        @commit_id = commit_id
+        @phrase_count = phrase_count
+        @status = status
+        @commit_datetime = commit_datetime
+        @branch_name = branch_name
+      end
+    end
+
+  end
+end

data/lib/rosette/core/extractor/commit_log_status.rb

@@ -0,0 +1,57 @@
+# encoding: UTF-8
+
+require 'aasm'
+
+module Rosette
+  module Core
+
+    # Provides a state machine for transitioning between the possible states of
+    # a commit log.
+    module CommitLogStatus
+      # aasm requires all states to be symbols
+      Rosette::DataStores::PhraseStatus.all.each do |status|
+        self.const_set(status, status.to_sym)
+      end
+
+      def self.included(base)
+        base.class_eval do
+          include AASM
+
+          aasm do
+            # define states from PhraseStatus constants
+            Rosette::DataStores::PhraseStatus.all.each do |status|
+              state status.to_sym
+            end
+
+            attribute_name :status
+            initial_state NOT_SEEN
+
+            event :fetch do
+              transitions from: NOT_SEEN, to: FETCHED
+            end
+
+            event :extract do
+              transitions from: FETCHED, to: EXTRACTED
+            end
+
+            event :push do
+              transitions from: [EXTRACTED, PUSHED], to: PUSHED
+            end
+
+            event :finalize do
+              transitions from: [EXTRACTED, PUSHED, FINALIZED], to: FINALIZED
+            end
+
+            # called when jgit can't find the commit
+            event :missing do
+              transitions(
+                from: Rosette::DataStores::PhraseStatus.statuses,
+                to: MISSING
+              )
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rosette/core/extractor/commit_processor.rb

@@ -0,0 +1,109 @@
+# encoding: UTF-8
+
+java_import 'org.eclipse.jgit.revwalk.RevWalk'
+
+module Rosette
+  module Core
+
+    # Extracts phrases from a git commit. Should be thread-safe.
+    #
+    # @!attribute [r] config
+    #   @return [Configurator] the Rosette config to use.
+    # @!attribute [r] error_reporter
+    #   @return [ErrorReporter] the error reporter to report syntax
+    #     errors, etc to.
+    #
+    # @example
+    #   processor = CommitProcessor.new(configuration)
+    #   processor.process_each_phrase('my_repo', 'master') do |phrase|
+    #     puts phrase.key
+    #   end
+    class CommitProcessor
+      attr_reader :config, :error_reporter
+
+      # Creates a new processor.
+      #
+      # @param [Configurator] config The Rosette config to use.
+      # @param [ErrorReporter] error_reporter The error reporter to report
+      #   syntax errors, etc to.
+      def initialize(config, error_reporter = NilErrorReporter.instance)
+        @config = config
+        @error_reporter = error_reporter
+      end
+
+      # Extracts translatable phrases from the given ref and yields them
+      # sequentially to the given block. If no block is given, this method
+      # returns an +Enumerator+.
+      #
+      # @param [String] repo_name The name of the repository to extract
+      #   translatable phrases from. Must be configured in +config+.
+      # @param [String] commit_ref The git ref or commit id to extract
+      #   translatable phrases from.
+      # @raise [Java::OrgEclipseJgitErrors::MissingObjectException]
+      # @return [void, Enumerator] either nothing if a block is given or
+      #   an instance of +Enumerator+ if no block is given.
+      # @yield [phrase] a single extracted phrase.
+      # @yieldparam phrase [Phrase]
+      def process_each_phrase(repo_name, commit_ref)
+        if block_given?
+          repo_config = config.get_repo(repo_name)
+          rev_walk = RevWalk.new(repo_config.repo.jgit_repo)
+          diff_finder = DiffFinder.new(repo_config.repo.jgit_repo, rev_walk)
+          commit = repo_config.repo.get_rev_commit(commit_ref, rev_walk)
+
+          diff_finder.diff_with_parents(commit).each_pair do |_, diff_entries|
+            diff_entries.each do |diff_entry|
+              if diff_entry.getNewPath != '/dev/null'
+                process_diff_entry(diff_entry, repo_config, commit) do |phrase|
+                  yield phrase
+                end
+              end
+            end
+          end
+        else
+          to_enum(__method__, repo_name, commit_ref)
+        end
+      end
+
+      protected
+
+      def process_diff_entry(diff_entry, repo_config, commit)
+        repo_config.get_extractor_configs(diff_entry.getNewPath).each do |extractor_config|
+          source_code = read_object_from_entry(diff_entry, repo_config, extractor_config)
+          line_numbers_to_author = repo_config.repo.blame(diff_entry.getNewPath, commit.getId.name)
+
+          begin
+            extractor_config.extractor.extract_each_from(source_code) do |phrase, line_number|
+              phrase.file = diff_entry.getNewPath
+              phrase.commit_id = commit.getId.name
+
+              if extractor_config.extractor.supports_line_numbers?
+                if author_identity = line_numbers_to_author[line_number - 1]
+                  phrase.author_name = author_identity.getName
+                  phrase.author_email = author_identity.getEmailAddress
+                  phrase.line_number = line_number
+                end
+              end
+
+              yield phrase
+            end
+          rescue SyntaxError => e
+            error_reporter.report_error(
+              ExtractionSyntaxError.new(
+                e.message, e.original_exception, e.language,
+                diff_entry.getNewPath, commit.getId.name
+              )
+            )
+          end
+        end
+      end
+
+      def read_object_from_entry(diff_entry, repo_config, extractor_config)
+        object_reader = repo_config.repo.jgit_repo.newObjectReader
+        bytes = object_reader.open(diff_entry.getNewId.toObjectId).getBytes
+        Java::JavaLang::String.new(bytes, extractor_config.encoding.to_s).to_s
+      end
+    end
+
+  end
+end

data/lib/rosette/core/extractor/extractor.rb

@@ -0,0 +1,72 @@
+# encoding: UTF-8
+
+module Rosette
+  module Core
+
+    # Base class for extractors that extract phrases from source code,
+    # eg. Ruby, JavaScript, HAML, etc.
+    #
+    # @!attribute [r] config
+    #   @return [Configurator] the Rosette config to use.
+    class Extractor
+      attr_reader :config
+
+      # Creates a new extractor.
+      #
+      # @param [Configurator] config The Rosette config to use.
+      def initialize(config = nil)
+        @config = config
+      end
+
+      # Extracts each translatable phrase from the given source code.
+      # Derived classes must implement the +#each_function_call+ method
+      # for this to work.
+      #
+      # @param [String] source_code The source code to extract phrases
+      #   from.
+      # @return [void, Enumerator] If passed a block, this method yields
+      #   each consecutive phrase found in +source_code+. If no block is
+      #   passed, it returns an +Enumerator+.
+      # @yield [phrase] a single extracted phrase.
+      # @yieldparam phrase [Phrase]
+      def extract_each_from(source_code)
+        if block_given?
+          each_function_call(source_code) do |node, line_number|
+            if valid_name?(node) && valid_args?(node)
+              yield make_phrase(get_key(node), get_meta_key(node)), line_number
+            end
+          end
+        else
+          to_enum(__method__, source_code)
+        end
+      end
+
+      protected
+
+      def each_function_call(source_code)
+        raise NotImplementedError, "#{__method__} must be implemented by derived classes."
+      end
+
+      def valid_name?(node)
+        raise NotImplementedError, "#{__method__} must be implemented by derived classes."
+      end
+
+      def valid_args?(node)
+        raise NotImplementedError, "#{__method__} must be implemented by derived classes."
+      end
+
+      def get_key(node)
+        raise NotImplementedError, "#{__method__} must be implemented by derived classes."
+      end
+
+      def get_meta_key(node)
+        nil
+      end
+
+      def make_phrase(key, meta_key = nil, file = nil)
+        Phrase.new(key, meta_key, file)
+      end
+    end
+
+  end
+end

data/lib/rosette/core/extractor/extractor_config.rb

@@ -0,0 +1,74 @@
+# encoding: UTF-8
+
+module Rosette
+  module Core
+
+    # Configuration for an extractor.
+    #
+    # @example
+    #   ex = ExtractorConfig.new(JsonExtractor::KeyValueExtractor)
+    #     .set_encoding(Encoding::UTF_8)
+    #     .set_conditions do |root|
+    #       root.match_file_extension('.json').and(
+    #         root.match_path('config/locales')
+    #       )
+    #     end
+    #
+    # @!attribute [r] extractor
+    #   @return [Extractor] the extractor instance that will be used to
+    #     extract phrases.
+    # @!attribute [r] encoding
+    #   @return [String, Encoding] the encoding to expect the contents of
+    #     source files to be in.
+    # @!attribute [r] root
+    #   @return [PathMatcherFactory::Node] the root of the
+    #     conditions tree. Only files that are matched by the conditions in
+    #     this tree will have their phrases extracted.
+    class ExtractorConfig
+      attr_reader :extractor_id, :extractor, :encoding, :root
+
+      # Creates a new extractor configuration.
+      #
+      # @param [String] extractor_id The extractor id of +extractor_class+.
+      # @param [Class] extractor_class The extractor to use.
+      def initialize(extractor_id, extractor_class)
+        @extractor_id = extractor_id
+        @extractor = extractor_class.new(self)
+        @root = PathMatcherFactory.create_root
+        @encoding = Rosette::Core::DEFAULT_ENCODING
+      end
+
+      # Sets the encoding to expect the contents of source files to be in.
+      #
+      # @param [String, Encoding] new_encoding the encoding to use.
+      # @return [self]
+      def set_encoding(new_encoding)
+        @encoding = new_encoding
+        self
+      end
+
+      # Determines if the given path matches all the conditions in the
+      # conditions tree.
+      #
+      # @param [String] path The path to match.
+      # @return [Boolean] true if the path matches, false otherwise.
+      def matches?(path)
+        root.matches?(path)
+      end
+
+      # Creates and yields a node that represents the root of a conditions
+      # tree. Callers should use the yielded root to build up a set of
+      # conditions that will be used to match paths in the repository.
+      # Matching paths will be processed by the extractor (i.e. their
+      # translatable phrases will be identified and stored).
+      #
+      # @return [self]
+      # @yield [root] the root of the conditions tree
+      # @yieldparam root [PathMatcherFactory::Node]
+      def set_conditions
+        @root = yield PathMatcherFactory.create_root
+        self
+      end
+    end
+  end
+end