rosette-core 1.0.1

data/lib/rosette/core/commands/errors.rb

@@ -0,0 +1,17 @@
+# encoding: UTF-8
+
+module Rosette
+  module Core
+    module Commands
+
+      # Custom errors that can be raised during command execution.
+      module Errors
+        # Raised whenever Rosette is asked to return information about
+        # a commit that has not been processed (i.e. committed) yet.
+        # Rosette knows nothing about commits it hasn't seen yet.
+        class UnprocessedCommitError < StandardError; end
+      end
+
+    end
+  end
+end

data/lib/rosette/core/commands/git/commit_command.rb

@@ -0,0 +1,65 @@
+# encoding: UTF-8
+
+module Rosette
+  module Core
+    module Commands
+
+      # Inspects the given commit and extracts translatable phrases using the configured
+      # extractors. By design, {CommitCommand} (and the rest of Rosette) will not process
+      # merge commits. Generally, a snapshot of the repository will be necessary to
+      # find the commits where individual files were last changed (see the
+      # {RepoSnapshotCommand} or {SnapshotFactory} class to take snapshots).
+      #
+      # @see Rosette::Core::Commands::RepoSnapshotCommand
+      # @see Rosette::Core::SnapshotFactory
+      #
+      # @example
+      #   CommitCommand.new(configuration)
+      #     .set_repo_name('my_repo')
+      #     .set_ref('master')
+      #     .execute
+      #
+      # @example
+      #   CommitCommand.new(configuration)
+      #     .set_repo_name('my_repo')
+      #     .set_commit_id('67f0e9a60dfe39430b346086f965e6c94a8ddd24')
+      #     .execute
+      class CommitCommand < GitCommand
+        include WithRepoName
+        include WithRef
+
+        # Executes the command. Causes phrases to be extracted from the given git ref or
+        # commit id and written to the configured data store, triggering hooks in the process.
+        #
+        # @return [void]
+        def execute
+          commit_processor.process_each_phrase(repo_name, commit_id) do |phrase|
+            begin
+              datastore.store_phrase(repo_name, phrase)
+            rescue ActiveRecord::RecordNotUnique => e
+              configuration.error_reporter.report_warning(e)
+            end
+          end
+
+          trigger_hooks(:after)
+        end
+
+        private
+
+        def commit_processor
+          @commit_processor ||= Rosette::Core::CommitProcessor.new(
+            configuration, configuration.error_reporter
+          )
+        end
+
+        def trigger_hooks(stage)
+          repo_config = get_repo(repo_name)
+          repo_config.hooks.fetch(stage, {}).fetch(:commit, []).each do |hook_proc|
+            hook_proc.call(configuration, repo_config, commit_id)
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/rosette/core/commands/git/diff_base_command.rb

@@ -0,0 +1,301 @@
+# encoding: UTF-8
+
+module Rosette
+  module Core
+    module Commands
+
+      # The base class for commands that need to calculate diffs.
+      class DiffBaseCommand < GitCommand
+        attr_reader :strict
+        alias_method :strict?, :strict
+
+        def initialize(*args)
+          super
+          @strict = true
+        end
+
+        # Sets a boolean value that determines if the diff should be made
+        # against true parents of the given commit or the most recently
+        # processed parents. If set to +true+, the diff will be computed
+        # between the given commit and its actual (true) parents, and an error
+        # will be raised if any of the parents have not been processed by
+        # Rosette (i.e. don't exist in the commit log). If set to +false+, the
+        # diff will be computed between the given commit and the most recent
+        # parents that have been processed. For each parent of the given commit,
+        # +ShowCommand+ will traverse the commit history looking for a processed
+        # commit and use it in place of the true parent (assuming the true
+        # parent hasn't been processed yet). It's worth noting that setting
+        # +strict+ to +false+ may potentially produce results that contain
+        # phrases that were not introduced exclusively in the given commit.
+        # Since such behavior is different from what git clients return, and
+        # therefore possibly unexpected, strictness is enabled (set to +true+)
+        # by default.
+        #
+        # @param [Boolean] strict
+        # @return [self]
+        def set_strict(strict)
+          @strict = strict
+          self
+        end
+
+        protected
+
+        include WithSnapshots
+
+        def diff_between(commit_id, parent_commit_ids, paths = nil)
+          # handle the case where this is the first commit (or there are no
+          # processed parents)
+          if parent_commit_ids.empty?
+            entries = retrieve_child_phrases(commit_id, []).map do |p|
+              DiffEntry.new(p, :added)
+            end
+
+            { added: entries, removed: [], modified: [] }
+          else
+            calculate_phrase_diff_against(commit_id, parent_commit_ids, paths)
+          end
+        end
+
+        def ensure_commits_have_been_processed(commits)
+          commits.uniq.each do |commit_id|
+            if commit_id
+              unless commit_exists?(repo_name, commit_id)
+                raise Errors::UnprocessedCommitError,
+                  "Commit #{commit_id} has not been processed yet."
+              end
+            end
+          end
+        end
+
+        def commit_exists?(repo_name, commit_id)
+          datastore.commit_log_exists?(repo_name, commit_id)
+        end
+
+        def compare(head_phrases, diff_point_phrases)
+          partitioned_head_phrases = partition_phrases(head_phrases)
+          partitioned_diff_point_phrases = partition_phrases(diff_point_phrases)
+          join_diffs(
+            key_diff(partitioned_head_phrases, partitioned_diff_point_phrases),
+            meta_key_diff(partitioned_head_phrases, partitioned_diff_point_phrases)
+          )
+        end
+
+        def join_diffs(diff1, diff2)
+          # we don't care about :unmodified, so leave it out
+          [:added, :removed, :modified].each_with_object({}) do |state, ret|
+            ret[state] = diff1[state] + diff2[state]
+          end
+        end
+
+        def key_diff(partitioned_head_phrases, partitioned_diff_point_phrases)
+          diff = Hash.new { |hash, key| hash[key] = [] }
+
+          key_head_to_diff_point(
+            partitioned_head_phrases.first,
+            partitioned_diff_point_phrases.first
+          ) do |phrase, state, old_phrase|
+            diff[state] << DiffEntry.new(phrase, state, old_phrase)
+          end
+
+          key_diff_point_to_head(
+            partitioned_head_phrases.first,
+            partitioned_diff_point_phrases.first
+          ) do |phrase, state, old_phrase|
+            diff[state] << DiffEntry.new(phrase, state, old_phrase)
+          end
+
+          diff
+        end
+
+        def key_head_to_diff_point(head_phrases, diff_point_phrases)
+          if block_given?
+            head_phrases.each do |head_phrase|
+              phrase = diff_point_phrases.find do |diff_point_phrase|
+                diff_point_phrase.key == head_phrase.key &&
+                  diff_point_phrase.file == head_phrase.file
+              end
+
+              state = phrase ? :unmodified : :added
+              yield head_phrase, state
+            end
+          else
+            to_enum(__method__, head_phrases, diff_point_phrases)
+          end
+        end
+
+        def key_diff_point_to_head(head_phrases, diff_point_phrases)
+          if block_given?
+            diff_point_phrases.each do |diff_point_phrase|
+              phrase = head_phrases.find do |head_phrase|
+                head_phrase.key == diff_point_phrase.key &&
+                  head_phrase.file == diff_point_phrase.file
+              end
+
+              unless phrase
+                yield diff_point_phrase, :removed
+              end
+            end
+          else
+            to_enum(__method__, head_phrases, diff_point_phrases)
+          end
+        end
+
+        def meta_key_diff(partitioned_head_phrases, partitioned_diff_point_phrases)
+          diff = Hash.new { |hash, key| hash[key] = [] }
+
+          meta_key_head_to_diff_point(
+            partitioned_head_phrases.last,
+            partitioned_diff_point_phrases.last
+          ) do |phrase, state, old_phrase|
+            diff[state] << DiffEntry.new(phrase, state, old_phrase)
+          end
+
+          meta_key_diff_point_to_head(
+            partitioned_head_phrases.last,
+            partitioned_diff_point_phrases.last
+          ) do |phrase, state, old_phrase|
+            diff[state] << DiffEntry.new(phrase, state, old_phrase)
+          end
+
+          diff
+        end
+
+        # identifies phrases in head that:
+        #   are not in diff point ('added')
+        #   have the same meta key but different keys as a phrase in diff point ('modified')
+        #   are identical to a phrase in diff point ('unmodified')
+        def meta_key_head_to_diff_point(head_phrases, diff_point_phrases)
+          if block_given?
+            # iterate over all head phrases that have meta keys
+            head_phrases.each do |head_phrase|
+              idx = diff_point_phrases.find_index do |diff_point_phrase|
+                diff_point_phrase.meta_key == head_phrase.meta_key &&
+                  diff_point_phrase.file == head_phrase.file
+              end
+
+              state = if idx
+                if diff_point_phrases[idx].key == head_phrase.key
+                  :unmodified
+                else
+                  :modified
+                end
+              else
+                :added
+              end
+
+              if state == :modified
+                yield head_phrase, state, diff_point_phrases[idx]
+              else
+                yield head_phrase, state
+              end
+            end
+          else
+            to_enum(__method__, head_phrases, diff_point_phrases)
+          end
+        end
+
+        # identifies phrases in diff point that are not in head ('removed')
+        def meta_key_diff_point_to_head(head_phrases, diff_point_phrases)
+          if block_given?
+            diff_point_phrases.each do |diff_point_phrase|
+              idx = head_phrases.find_index do |head_phrase|
+                head_phrase.meta_key == diff_point_phrase.meta_key &&
+                  head_phrase.file == diff_point_phrase.file
+              end
+
+              unless idx
+                yield diff_point_phrase, :removed
+              end
+            end
+          else
+            to_enum(__method__, head_phrases, diff_point_phrases)
+          end
+        end
+
+        def partition_phrases(phrases)
+          phrases.partition { |ph| ph.index_key == :key }
+        end
+
+        # if parent_commit_id is processed, it will get returned (i.e. it will
+        # start looking at parent_commit_id, not parent_commit_id - 1).
+        def get_closest_processed_parent(parent_commit_id)
+          parent = repo.each_commit_starting_at(parent_commit_id).find do |rev|
+            commit_exists?(repo_name, rev.getId.name)
+          end
+
+          parent.getId.name if parent
+        end
+
+        def calculate_phrase_diff_against(commit_id, parent_commit_ids, paths)
+          { added: [], modified: [], removed: [] }.tap do |final_diff|
+            parent_commit_ids.each do |parent_commit_id|
+              git_diff = compute_git_diff_between(commit_id, parent_commit_id)
+              paths ||= compute_paths(git_diff)
+              child_phrases = retrieve_child_phrases(commit_id, paths)
+              parent_phrases = retrieve_parent_phrases(parent_commit_id, paths)
+              phrase_diff = compare(child_phrases, parent_phrases)
+
+              phrase_diff.each_pair do |state, phrases|
+                final_diff[state].concat(phrases)
+              end
+            end
+          end
+        end
+
+        def compute_git_diff_between(commit_id, parent_commit_id)
+          rev_walker = RevWalk.new(repo.jgit_repo)
+          diff_finder = DiffFinder.new(repo.jgit_repo, rev_walker)
+          repo.diff([parent_commit_id], commit_id, [], diff_finder)
+        end
+
+        def compute_paths(diff)
+          child_paths = []
+
+          diff.each_with_object([]) do |(_, diff_entries), ret|
+            diff_entries.each do |diff_entry|
+              path = get_path(diff_entry)
+
+              if repo_config.extractor_configs.any? { |ext| ext.matches?(path) }
+                ret << path
+              end
+            end
+          end
+        end
+
+        def get_path(diff_entry)
+          new_path = diff_entry.getNewPath
+
+          if new_path == '/dev/null'
+            diff_entry.getOldPath
+          else
+            new_path
+          end
+        end
+
+        def retrieve_child_phrases(commit_id, paths)
+          snapshot = take_snapshot(repo_config, commit_id, paths)
+          datastore.phrases_by_commits(repo_name, snapshot).to_a
+        end
+
+        def retrieve_parent_phrases(parent_commit_id, paths)
+          ensure_commits_have_been_processed([parent_commit_id])
+
+          parent_snapshot = take_snapshot(
+            repo_config, parent_commit_id, paths
+          )
+
+          datastore.phrases_by_commits(repo_name, parent_snapshot).to_a
+        end
+
+        def repo_config
+          get_repo(repo_name)
+        end
+
+        def repo
+          repo_config.repo
+        end
+      end
+
+    end
+  end
+end

data/lib/rosette/core/commands/git/diff_command.rb

@@ -0,0 +1,188 @@
+# encoding: UTF-8
+
+module Rosette
+  module Core
+    module Commands
+
+      # Detects phrase changes between two git refs or commit ids. Identifies phrases
+      # that have been added, removed, or changed. The refs used in the comparison
+      # are referred to as a "head" and a "diff point". In Rosette/git parlance, a "head"
+      # generally means a git ref that is currently being modified, i.e. a branch. A
+      # "diff point" is some common ref to compare "head" to, often the repository's
+      # "master" branch.
+      #
+      # Perhaps an easier way to visualize these concepts is through a git command-line
+      # example. Imagine you're developing a feature to add widget support to your app.
+      # You start work by switching to a new git branch (i.e. +git+ +checkout+ +-b+ +add_widgets+).
+      # After adding a few files, changing some others, and deleting old code, you want
+      # to see a complete set of all your changes. To do this, you run +git+ +diff+. At
+      # this point, your "head" is your branch "add_widgets" and your "diff point" is
+      # "master" (or whichever branch you were on when you ran +git+ +checkout+). You
+      # could also have run +git+ +diff+ +master+ or +git+ +diff+ +master+ +HEAD+ to
+      # get the same result. You can think of {DiffCommand} like a +git+ +diff+ command
+      # of the form +git+ +diff+ +<diff+ +point>+ +<head>+.
+      #
+      # @see Rosette::Core::DiffFinder
+      #
+      # @example
+      #   cmd = DiffCommand.new(configuration)
+      #     .set_repo_name('my_repo')
+      #     .set_head_ref('my_branch')
+      #     .set_diff_point_ref('master')
+      #     .set_paths(['config/locales/en.yml'])
+      #
+      #   cmd.execute
+      #   # {
+      #   #   added: [
+      #   #     { key: 'Foo', ... },
+      #   #   ],
+      #   #   removed: [
+      #   #     { key: 'I got deleted', ... }
+      #   #   ],
+      #   #   modified: [
+      #   #     { key: 'New value', old_key: 'Old value', ... }
+      #   #   ]
+      #   # }
+      #
+      # @!attribute [r] head_commit_str
+      #   @return [String] the raw head ref or commit id as set via {#set_head_ref}
+      #     or {#set_head_commit_id}.
+      #
+      # @!attribute [r] diff_point_commit_str
+      #   @return [String] the raw diff point ref or commit id as set via
+      #     {#set_diff_point_ref} or {#set_diff_point_commit_id}.
+      #
+      # @!attribute [r] paths
+      #   @return [Array] the list of paths to include in the diff. The diff will not
+      #     contain phrases that were added/removed/modified in files that are not
+      #     contained within this list.
+      class DiffCommand < DiffBaseCommand
+        attr_reader :head_commit_str, :diff_point_commit_str, :paths
+
+        include WithRepoName
+
+        validate :head_commit_str, type: :commit
+        validate :diff_point_commit_str, type: :commit
+
+        # Set the head commit id. Calling this method after {#set_head_ref} will
+        # overwrite the head ref value. In other words, it's generally a good idea
+        # to only call one of {#set_head_commit_id} or {#set_head_ref} but not both.
+        #
+        # @param [String] head_commit_id The head commit id.
+        # @return [self]
+        def set_head_commit_id(head_commit_id)
+          @head_commit_str = head_commit_id
+          self
+        end
+
+        # Set the head ref (i.e. a branch name). Calling this method after
+        # {#set_head_commit_id} will overwrite the head commit id value. In other
+        # words, it's generally a good idea to only call one of {#set_head_commit_id}
+        # or {#set_head_ref} but not both.
+        #
+        # @param [String] head_ref The head ref.
+        # @return [self]
+        def set_head_ref(head_ref)
+          @head_commit_str = head_ref
+          self
+        end
+
+        # Set the diff point commit id. Calling this method after {#set_head_ref}
+        # will overwrite the head ref value. In other words, it's generally a good
+        # idea to only call one of {#set_head_commit_id} or {#set_head_ref} but
+        # not both.
+        #
+        # @param [String] diff_point_commit_id The diff point commit id.
+        # @return [self]
+        def set_diff_point_commit_id(diff_point_commit_id)
+          @diff_point_commit_str = diff_point_commit_id
+          self
+        end
+
+        # Set the diff point ref (i.e. master). Calling this method after
+        # {#set_diff_point_commit_id} will overwrite the diff point commit id value.
+        # In other words, it's generally a good idea to only call one of
+        # {#set_diff_point_commit_id} or {#set_diff_point_ref} but not both.
+        #
+        # @param [String] diff_point_ref The diff point ref.
+        # @return [self]
+        def set_diff_point_ref(diff_point_ref)
+          @diff_point_commit_str = diff_point_ref
+          self
+        end
+
+        # Set the list of paths to consider when computing the diff. Any paths not
+        # in this list will not appear in the diff.
+        #
+        # @param [Array] paths The list of paths.
+        # @return [self]
+        def set_paths(paths)
+          @paths = paths
+          self
+        end
+
+        # Resolves the given head git ref or commit id and returns the corresponding
+        # commit id. If {#set_head_ref} was used to set a git ref (i.e. branch name),
+        # this method looks up and returns the corresponding commit id. If
+        # {#set_head_commit_id} was used to set a commit id, then that commit id is
+        # validated and returned.
+        #
+        # @return [String] The commit id set via either {#set_head_ref} or
+        #   {#set_head_commit_id}.
+        #
+        # @raise [Java::OrgEclipseJgitErrors::MissingObjectException, Java::JavaLang::IllegalArgumentException]
+        #   If either the commit id doesn't exist or the ref can't be found.
+        def head_commit_id
+          @head_commit_id ||= get_repo(repo_name)
+            .repo.get_rev_commit(head_commit_str)
+            .getId
+            .name
+        end
+
+        # Resolves the given diff point git ref or commit id and returns the corresponding
+        # commit id. If {#set_diff_point_ref} was used to set a git ref (i.e. branch name),
+        # this method looks up and returns the corresponding commit id. If
+        # {#set_diff_point_commit_id} was used to set a commit id, then that commit id is
+        # validated and returned.
+        #
+        # @return [String] The commit id set via either {#set_diff_point_ref} or
+        #   {#set_diff_point_commit_id}.
+        #
+        # @raise [Java::OrgEclipseJgitErrors::MissingObjectException, Java::JavaLang::IllegalArgumentException]
+        #   If either the commit id doesn't exist or the ref can't be found.
+        def diff_point_commit_id
+          @diff_point_commit_id ||= get_repo(repo_name)
+            .repo.get_rev_commit(diff_point_commit_str)
+            .getId
+            .name
+        end
+
+        # Computes the diff.
+        # @return [Hash] a hash of differences, grouped by added/removed/modified keys. Each
+        #   value is an array of phrases. For added and removed phrases, the phrase hashes
+        #   will contain normal phrase attributes. For changed phrases, the phrase hashes
+        #   will contain normal phrase attributes plus a special +old_key+ attribute that
+        #   contains the previous key of the phrase. See the example above for a visual
+        #   representation of the diff hash.
+        def execute
+          diff_between(head_commit_id, Array(parent_commit_id))
+        end
+
+        protected
+
+        def diff_point_exists?
+          commit_exists?(repo_name, diff_point_commit_id)
+        end
+
+        def parent_commit_id
+          parent_commit_id = if strict? || diff_point_exists?
+            diff_point_commit_id
+          else
+            get_closest_processed_parent(diff_point_commit_id)
+          end
+        end
+      end
+
+    end
+  end
+end