rosette-core 1.0.1

data/lib/rosette/queuing/commits/commits_queue_configurator.rb

@@ -0,0 +1,60 @@
+# encoding: UTF-8
+
+module Rosette
+  module Queuing
+    module Commits
+
+      # Configuration specific to the "commits" queue that processes commits.
+      #
+      # @!attribute [r] name
+      #   @return [String]
+      # @!attribute [r] diff_point
+      #   @return [String] the diff point to use when computing phrase diffs.
+      #     Defaults to "master". Used only when +phrase_storage_granularity+
+      #     is set to [Rosette::Queuing::Commits::PhraseStorageGranularity::BRANCH].
+      # @!attribute [r] phrase_storage_granularity
+      #   @return [String] determines which set of phrases to push to the TMS.
+      #     Must be one of the constant values in
+      #     [Rosette::Queuing::Commits::PhraseStorageGranularity]
+      #
+      # @see Rosette::Queuing::Commits::PhraseStorageGranularity
+      class CommitsQueueConfigurator
+        DEFAULT_DIFF_POINT = 'master'
+        DEFAULT_PHRASE_GRANULARITY = PhraseStorageGranularity::COMMIT
+
+        attr_reader :name, :diff_point, :phrase_storage_granularity
+
+        # Creates a new configurator and sets up a few defaults.
+        #
+        # @param [String] name The name of the queue.
+        # @return [CommitsQueueConfigurator]
+        def initialize(name)
+          @name = name
+          @diff_point = DEFAULT_DIFF_POINT
+          @phrase_storage_granularity = DEFAULT_PHRASE_GRANULARITY
+        end
+
+        # Sets the phrase storage granularity, i.e. the method used to determine
+        # which set of phrases should get uploaded to the TMS.
+        #
+        # @param [String] granularity One of the constant values in
+        #   [Rosette::Queuing::Commits::PhraseStorageGranularity].
+        # @return [void]
+        def set_phrase_storage_granularity(granularity)
+          @phrase_storage_granularity = granularity
+        end
+
+        # Sets the diff point to use when computing phrase diffs. Note that this
+        # value is only important when +phrase_storage_granularity+ is set to
+        # [Rosette::Queuing::Commits::PhraseStorageGranularity::BRANCH].
+        #
+        # @param [String] new_diff_point The diff point to set.
+        # @return [void]
+        def set_diff_point(new_diff_point)
+          @diff_point = new_diff_point
+        end
+      end
+
+    end
+  end
+end

data/lib/rosette/queuing/commits/extract_stage.rb

@@ -0,0 +1,46 @@
+# encoding: UTF-8
+
+module Rosette
+  module Queuing
+    module Commits
+
+      # Extracts phrases from the given commit and stores them in the datastore.
+      #
+      # @see RepoConfig
+      class ExtractStage < Stage
+        accepts PhraseStatus::FETCHED
+
+        # Executes this stage and updates the commit log with an +EXTRACTED+
+        # status. Uses [Rosette::Core::Commands::CommitCommand] under the hood.
+        # If the commit no longer exists in the git repository, the commit log
+        # will be updated with a status of +MISSING+.
+        #
+        # @return [void]
+        def execute!
+          logger.info("Extracting phrases for #{commit_log.commit_id}")
+
+          Rosette::Core::Commands::CommitCommand.new(rosette_config)
+            .set_repo_name(repo_config.name)
+            .set_ref(commit_log.commit_id)
+            .execute
+
+          commit_log.extract
+          commit_log.commit_datetime = Time.at(rev_commit.getCommitTime)
+
+          logger.info("Finished extracting phrases for #{commit_log.commit_id}")
+        rescue Java::OrgEclipseJgitErrors::MissingObjectException => e
+          commit_log.missing
+        ensure
+          save_commit_log
+        end
+
+        protected
+
+        def rev_commit
+          @rev_commit ||= repo_config.repo.get_rev_commit(commit_log.commit_id)
+        end
+      end
+
+    end
+  end
+end

data/lib/rosette/queuing/commits/fetch_stage.rb

@@ -0,0 +1,51 @@
+# encoding: UTF-8
+
+java_import 'org.eclipse.jgit.api.Git'
+
+module Rosette
+  module Queuing
+    module Commits
+
+      # Fetches the repository. This should be the first commit processing stage
+      # since it ensures the given commit is available in the repository for
+      # processing.
+      #
+      # @see RepoConfig
+      class FetchStage < Stage
+        accepts PhraseStatus::NOT_SEEN, PhraseStatus::NOT_FOUND
+
+        # Executes this stage and updates the commit log with a +FETCHED+
+        # status. Performs a git fetch.
+        #
+        # @return [void]
+        def execute!
+          logger.info("Fetching git repository #{repo_config.name}")
+
+          git.fetch
+            .setRemote('origin')
+            .setRemoveDeletedRefs(true)
+            .call
+
+          commit_log.fetch
+
+          # git won't know about the commit before a fetch, which is why branch
+          # name is set in this stage and not when the commit is first enqueued
+          commit_log.branch_name = Rosette::Core::BranchUtils.derive_branch_name(
+            commit_log.commit_id, repo_config.repo
+          )
+
+          save_commit_log
+
+          logger.info("Finished fetching git repository #{repo_config.name}")
+        end
+
+        private
+
+        def git
+          @git ||= Git.new(repo_config.repo.jgit_repo)
+        end
+      end
+
+    end
+  end
+end

data/lib/rosette/queuing/commits/finalize_stage.rb

@@ -0,0 +1,76 @@
+# encoding: UTF-8
+
+module Rosette
+  module Queuing
+    module Commits
+
+      # Performs cleanup tasks for the given commit as defined by the configured
+      # translation management system.
+      #
+      # @see RepoConfig
+      class FinalizeStage < Stage
+        accepts PhraseStatus::PUSHED
+
+        # The number of seconds to wait in between consecutive pulls. This value
+        # will be passed to the queue implementation, as delay is handled at the
+        # queue layer.
+        CONSECUTIVE_FINALIZE_DELAY_MIN = 10 * 60  # 10 minutes
+        CONSECUTIVE_FINALIZE_DELAY_MAX = 45 * 60  # 45 minutes
+
+        # Executes this stage and updates the commit log. If the commit has been
+        # fully translated, the commit log will be updated with a +FINALIZED+
+        # status, and the +finalize+ method will be called on the translation
+        # management system. If the commit has not been fully translated, the
+        # commit log's status won't be updated and +finalize+ will not be
+        # called. In both cases, commit log locale entries will be updated to
+        # track translation progress.
+        #
+        # @return [void]
+        def execute!
+          logger.info("Finalizing commit #{commit_log.commit_id}")
+
+          status = repo_config.tms.status(commit_log.commit_id)
+
+          repo_config.locales.each do |locale|
+            locale_code = locale.code
+
+            rosette_config.datastore.add_or_update_commit_log_locale(
+              commit_log.commit_id, locale_code, status.locale_count(locale_code)
+            )
+          end
+
+          if status.fully_translated?
+            repo_config.tms.finalize(commit_log.commit_id)
+            commit_log.finalize
+            save_commit_log
+          end
+
+          logger.info("Finished finalizing commit #{commit_log.commit_id}")
+        end
+
+        # Converts this stage to a job that can be enqueued. This method should
+        # be called after +#execute!+, meaning the commit log has been updated
+        # to the next status in the pipeline. If that next status also happens
+        # to be PUSHED, this method adds a delay to avoid finalizing too often.
+        # If the chosen queue implementation does not support delays, setting
+        # this value should be a safe no-op (i.e. have no adverse side-effects).
+        #
+        # @return [CommitJob]
+        def to_job
+          super.tap do |job|
+            if commit_log.status == PhraseStatus::PUSHED
+              job.set_delay(random_delay)
+            end
+          end
+        end
+
+        protected
+
+        def random_delay
+          rand(CONSECUTIVE_FINALIZE_DELAY_MIN..CONSECUTIVE_FINALIZE_DELAY_MAX)
+        end
+      end
+
+    end
+  end
+end

data/lib/rosette/queuing/commits/phrase_storage_granularity.rb

@@ -0,0 +1,20 @@
+# encoding: UTF-8
+
+module Rosette
+  module Queuing
+    module Commits
+
+      # Provides a set of constants that specify how to determine the set of
+      # phrases to push to a translation management system (TMS).
+      class PhraseStorageGranularity
+        # Push only the phrases that were added or changed in single commits.
+        COMMIT = 'COMMIT'
+
+        # Push all the phrases contained by each git branch, regardless of the
+        # order of commits.
+        BRANCH = 'BRANCH'
+      end
+
+    end
+  end
+end

data/lib/rosette/queuing/commits/push_stage.rb

@@ -0,0 +1,91 @@
+# encoding: UTF-8
+
+module Rosette
+  module Queuing
+    module Commits
+
+      # Saves (or "pushes") a set of phrases from the given commit to the
+      # configured translation management system (TMS).
+      #
+      # @see RepoConfig
+      class PushStage < Stage
+        accepts PhraseStatus::EXTRACTED
+
+        # Executes this stage and updates the commit log. If the commit queue
+        # phrase storage granularity is set to +COMMIT+, only the phrases added
+        # or modified in the given commit will get pushed to the TMS. If the
+        # granularity is instead set to +BRANCH+, the full phrase diff between
+        # the commit's branch and the configured diff point (usually master)
+        # will get pushed to the TMS. If the commit or diff contains no phrases,
+        # this method doesn't push anything but will still update the commit log
+        # with a +FINALIZED+ status. If the commit no longer exists in the git
+        # repository, the commit log will be updated with a status of +MISSING+.
+        #
+        # @return [void]
+        def execute!
+          logger.info("Pushing commit #{commit_log.commit_id}")
+
+          if phrases.size > 0
+            commit_log.phrase_count = phrases.size
+            repo_config.tms.store_phrases(phrases, commit_log.commit_id)
+            commit_log.push
+          else
+            commit_log.finalize!
+          end
+
+          logger.info("Finished pushing commit #{commit_log.commit_id}")
+        rescue Java::OrgEclipseJgitErrors::MissingObjectException => ex
+          commit_log.missing
+        ensure
+          save_commit_log
+        end
+
+        protected
+
+        def granularity
+          case queue_config.phrase_storage_granularity
+            when PhraseStorageGranularity::BRANCH
+              if commit_log.branch_name
+                PhraseStorageGranularity::BRANCH
+              else
+                PhraseStorageGranularity::COMMIT
+              end
+            else
+              queue_config.phrase_storage_granularity
+          end
+        end
+
+        def phrases
+          @phrases ||= case granularity
+            when PhraseStorageGranularity::COMMIT
+              phrases_from(diff_for_commit)
+            when PhraseStorageGranularity::BRANCH
+              phrases_from(diff_for_branch)
+          end
+        end
+
+        def diff_for_commit
+          Rosette::Core::Commands::ShowCommand.new(rosette_config)
+            .set_repo_name(repo_config.name)
+            .set_commit_id(commit_log.commit_id)
+            .set_strict(false)
+            .execute
+        end
+
+        def diff_for_branch
+          Rosette::Core::Commands::DiffCommand.new(rosette_config)
+            .set_repo_name(repo_config.name)
+            .set_head_ref(commit_log.branch_name)
+            .set_diff_point_ref(queue_config.diff_point)
+            .set_strict(false)
+            .execute
+        end
+
+        def phrases_from(diff)
+          (diff[:added] + diff[:modified]).map(&:phrase)
+        end
+      end
+
+    end
+  end
+end

data/lib/rosette/queuing/commits/stage.rb

@@ -0,0 +1,96 @@
+# encoding: UTF-8
+
+module Rosette
+  module Queuing
+    module Commits
+
+      # Base class for stages of the commit processing pipeline.
+      #
+      # @!attribute [r] rosette_config
+      #   @return [Configurator] the Rosette config to use.
+      # @!attribute [r] repo_config
+      #   @return [RepoConfig] the repo config to use.
+      # @!attribute [r] logger
+      #   @return [Logger] the logger to log messages to.
+      # @!attribute [r] commit_log
+      #   @return [CommitLog] the commit log to process.
+      class Stage
+        PhraseStatus = Rosette::DataStores::PhraseStatus
+
+        class << self
+          # Sets the +PhraseStatus+es this stage can handle.
+          #
+          # @param [Array<String>] statuses A splatted list of statuses.
+          # @return [void]
+          def accepts(*statuses)
+            @_accepts_statuses = statuses
+          end
+
+          # Returns true if this stage accepts the given commit log (i.e. can
+          # process it), false otherwise.
+          #
+          # @param [CommitLog] commit_log The commit log to check.
+          # @return [Boolean]
+          def accepts?(commit_log)
+            @_accepts_statuses.include?(commit_log.status)
+          end
+
+          # If given a commit log that this stage accepts, returns an instance
+          # of this stage class. If the stage does not accept the commit log,
+          # returns +nil+.
+          #
+          # @param [CommitLog] commit_log The commit log to wrap.
+          # @param [Configurator] rosette_config The Rosette config to pass to
+          #   the stage instance.
+          # @param [Logger] logger The logger to pass to the stage instance.
+          # @return [Stage] an instance of this stage or +nil+.
+          def for_commit_log(commit_log, rosette_config, logger)
+            if accepts?(commit_log)
+              repo_config = rosette_config.get_repo(commit_log.repo_name)
+              new(rosette_config, repo_config, logger, commit_log)
+            end
+          end
+        end
+
+        attr_reader :rosette_config, :repo_config, :logger, :commit_log
+
+        # Creates a new instance of this stage.
+        #
+        # @param [Configurator] rosette_config The Rosette config to use.
+        # @param [RepoConfig] repo_config The repo config to use.
+        # @param [Logger] logger The logger to log messages to.
+        # @param [CommitLog] commit_log The commit log to process.
+        # @return [Stage]
+        def initialize(rosette_config, repo_config, logger, commit_log)
+          @rosette_config = rosette_config
+          @repo_config = repo_config
+          @logger = logger
+          @commit_log = commit_log
+        end
+
+        # Converts this stage to a job that can be enqueued.
+        #
+        # @return [CommitJob]
+        def to_job
+          CommitJob.from_stage(self)
+        end
+
+        protected
+
+        def save_commit_log
+          rosette_config.datastore.add_or_update_commit_log(
+            commit_log.repo_name, commit_log.commit_id,
+            commit_log.commit_datetime, commit_log.status,
+            commit_log.phrase_count, commit_log.branch_name
+          )
+        end
+
+        def queue_config
+          @queue_config ||=
+            rosette_config.queue.configurator.get_queue_config('commits')
+        end
+      end
+
+    end
+  end
+end