rosette-core 1.0.1

data/lib/rosette/core/commands/git/with_refs.rb

@@ -0,0 +1,92 @@
+# encoding: UTF-8
+
+require 'thread'
+
+module Rosette
+  module Core
+    module Commands
+
+      # Mixin that handles configuration and validation of a set of git refs or
+      # commit ids. Meant to be mixed into the classes in {Rosette::Core::Commands}.
+      # Required to be used in combination with {Rosette::Core::Commands::WithRepoName}.
+      #
+      # @see Rosette::Core::Commands::WithRepoName
+      #
+      # @example
+      #   class MyCommand < Rosette::Core::Commands::Command
+      #     include WithRepoName
+      #     include WithRefs
+      #   end
+      #
+      #   cmd = MyCommand.new
+      #     .set_repo_name('my_repo')
+      #     .set_refs(['master'])
+      #
+      #   cmd.commit_strs  # => ["master"]
+      #   cmd.commit_ids   # => ["67f0e9a60dfe39430b346086f965e6c94a8ddd24"]
+      #   cmd.valid?         # => true
+      #
+      #   cmd.set_refs(['non_existent_ref'])
+      #   cmd.valid?    # => false
+      #   cmd.messages  # => { commit_str: ["Unable to find commit 'non_existent_ref'"] }
+      #
+      # @!attribute [r] commit_str
+      #   @return [String] the raw value as set by either {#set_ref} or {#set_commit_id}.
+      module WithRefs
+        attr_reader :commit_strs
+
+        # Set the git refs (i.e. a branch names). Calling this method after {#set_commit_ids}
+        # will overwrite the commit id values. In other words, it's generally a good idea to
+        # only call one of {#set_commit_ids} or {#set_refs} but not both.
+        #
+        # @param [Array<String>] ref_strs The git refs.
+        # @return [self]
+        def set_refs(ref_strs)
+          @commit_strs = ref_strs
+          self
+        end
+
+        # Set the git commit ids. Calling this method after {#set_refs} will overwrite the ref values.
+        # In other words, it's generally a good idea to only call one of {#set_commit_ids} or {#set_refs}
+        # but not both.
+        #
+        # @param [Array<String>] commit_ids The commit ids.
+        # @return [self]
+        def set_commit_ids(commit_ids)
+          @commit_strs = commit_ids
+          self
+        end
+
+        # Resolves the git ref or commit id at +index+ and returns the corresponding commit id.
+        # If {#set_refs} was used to set git refs (i.e. branch names), this method looks up
+        # and returns the corresponding commit id. If {#set_commit_ids} was used to set
+        # commit ids, then that commit id is validated and returned.
+        #
+        # @param [Fixnum] index The index of the ref or commit id to return.
+        # @return [String] The commit id set via either {#set_refs} or {#set_commit_ids}.
+        # @raise [Java::OrgEclipseJgitErrors::MissingObjectException, Java::JavaLang::IllegalArgumentException]
+        #   If either the commit id doesn't exist or the ref can't be found.
+        def commit_ids
+          @commit_ids ||= @commit_strs.map do |commit_str|
+            REV_COMMIT_MUTEX.synchronize do
+              get_repo(repo_name)
+                .repo.get_rev_commit(commit_str)
+                .getId.name
+            end
+          end
+        end
+
+        private
+
+        REV_COMMIT_MUTEX = Mutex.new
+
+        def self.included(base)
+          if base.respond_to?(:validate)
+            base.validate :commit_strs, type: :commits
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/rosette/core/commands/git/with_repo_name.rb

@@ -0,0 +1,50 @@
+# encoding: UTF-8
+
+module Rosette
+  module Core
+    module Commands
+
+      # Mixin that handles configuration and validation of a repository name.
+      # Meant to be mixed into the classes in {Rosette::Core::Commands}.
+      #
+      # @example
+      #   class MyCommand < Rosette::Core::Commands::Command
+      #     include WithRepoName
+      #   end
+      #
+      #   cmd = MyCommand.new
+      #     .set_repo_name('my_repo')
+      #
+      #   cmd.repo_name  # => "my_repo"
+      #   cmd.valid?     # => true
+      #
+      #   cmd.set_repo_name('non_existent_repo')
+      #   cmd.valid?     # => false
+      #   cmd.messages   # => { repo_name: ["Unable to find repo 'non_existent_repo'."] }
+      #
+      # @!attribute [r] repo_name
+      #   @return [String] the repository name.
+      module WithRepoName
+        attr_reader :repo_name
+
+        # Set the repository name.
+        #
+        # @param [String] repo_name The repository name.
+        # @return [self]
+        def set_repo_name(repo_name)
+          @repo_name = repo_name
+          self
+        end
+
+        protected
+
+        def self.included(base)
+          if base.respond_to?(:validate)
+            base.validate :repo_name, type: :repo
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/rosette/core/commands/git/with_snapshots.rb

@@ -0,0 +1,45 @@
+# encoding: UTF-8
+
+module Rosette
+  module Core
+    module Commands
+
+      # Mixin capable of taking snapshots of git repositories. Meant to be mixed
+      # into the classes in {Rosette::Core::Commands}.
+      #
+      # @see Rosette::Core::SnapshotFactory
+      #
+      # @example
+      #   class MyCommand < Rosette::Core::Commands::Command
+      #     include WithSnapshots
+      #   end
+      #
+      #   cmd = MyCommand.new
+      #   snap = cmd.take_snapshot(repo_config, commit_id, paths)
+      #   snap  # => { 'path/to/file.rb' => '67f0e9a60dfe39430b346086f965e6c94a8ddd24', ... }
+      module WithSnapshots
+        # Takes and returns a snapshot hash for the given repo and commit id. Limits
+        # the paths returned via the +paths+ argument. If no paths are passed,
+        # {#take_snapshot} returns a snapshot containing all the paths in the repository.
+        #
+        # @param [Rosette::Core::RepoConfig] repo_config The repository configuration
+        #   to take the snapshot from.
+        # @param [String] commit_id The git commit id to take the snapshot of.
+        # @param [Array] paths The list of paths to consider when taking the snapshot.
+        #   Only those paths included in this list will appear in the snapshot hash.
+        # @return [Hash] the snapshot hash (path to commit id pairs).
+        def take_snapshot(repo_config, commit_id, paths = [])
+          __snapshot_factory__.take_snapshot(repo_config, commit_id, paths)
+        end
+
+        private
+
+        def __snapshot_factory__
+          @@__snapshot_factory__ ||=
+            CachedSnapshotFactory.new(configuration.cache)
+        end
+      end
+
+    end
+  end
+end

data/lib/rosette/core/commands/queuing/enqueue_commit_command.rb

@@ -0,0 +1,37 @@
+# encoding: UTF-8
+
+module Rosette
+  module Core
+    module Commands
+
+      # Enqueues a commit for processing on Rosette's configured queue.
+      #
+      # @see Rosette::Queuing
+      #
+      # @example
+      #   EnqueueCommitCommand.new(configuration)
+      #     .set_repo_name('my_repo')
+      #     .set_ref('master')
+      #     .execute
+      #
+      # @example
+      #   EnqueueCommitCommand.new(configuration)
+      #     .set_repo_name('my_repo')
+      #     .set_commit_id('67f0e9a60dfe39430b346086f965e6c94a8ddd24')
+      #     .execute
+      class EnqueueCommitCommand < GitCommand
+        include WithRepoName
+        include WithRef
+
+        def execute
+          conductor = Rosette::Queuing::Commits::CommitConductor.new(
+            configuration, repo_name, Rosette.logger
+          )
+
+          conductor.enqueue(commit_id)
+        end
+      end
+
+    end
+  end
+end

data/lib/rosette/core/commands/queuing/requeue_commit_command.rb

@@ -0,0 +1,46 @@
+# encoding: UTF-8
+
+module Rosette
+  module Core
+    module Commands
+
+      # Sets the commit's status to NOT_FOUND, then enqueues it for processing
+      # on Rosette's configured queue. In other words, this command will
+      # re-process the commit, which should be an idempotent operation.
+      #
+      # @see Rosette::Queuing
+      #
+      # @example
+      #   RequeueCommitCommand.new(configuration)
+      #     .set_repo_name('my_repo')
+      #     .set_ref('master')
+      #     .execute
+      #
+      # @example
+      #   RequeueCommitCommand.new(configuration)
+      #     .set_repo_name('my_repo')
+      #     .set_commit_id('67f0e9a60dfe39430b346086f965e6c94a8ddd24')
+      #     .execute
+      class RequeueCommitCommand < GitCommand
+        include WithRepoName
+        include WithRef
+
+        def execute
+          commit_log = datastore.lookup_commit_log(repo_name, commit_id)
+
+          datastore.add_or_update_commit_log(
+            commit_log.repo_name, commit_log.commit_id,
+            commit_log.commit_datetime, PhraseStatus::NOT_FOUND,
+            commit_log.phrase_count, commit_log.branch_name
+          )
+
+          EnqueueCommitCommand.new(configuration)
+            .set_repo_name(repo_name)
+            .set_commit_id(commit_id)
+            .execute
+        end
+      end
+
+    end
+  end
+end

data/lib/rosette/core/commands/translations/export_command.rb

@@ -0,0 +1,257 @@
+# encoding: UTF-8
+
+require 'stringio'
+require 'base64'
+require 'digest/md5'
+
+module Rosette
+  module Core
+    module Commands
+
+      # Finds, encodes, and serializes the translations identified by a
+      # snapshot of the given git ref or commit id. In other words, this
+      # command exports the translations for a git branch or commit. This
+      # command also applies any configured pre-processors to the
+      # translations before serializing them. As a better visualization,
+      # here's the pipeline translations go through when exported:
+      #
+      # preprocessed -> serialized/encoded -> base 64 encoded (if
+      # requested) -> returned
+      #
+      # @!attribute [r] locale
+      #   @return [String] the locale to export translations for.
+      # @!attribute [r] serializer
+      #   @return [String] the serializer to use when exporting the
+      #     translations. Must be recognizable as a serializer id, eg.
+      #     'yaml/rails' or 'json/key-value'.
+      # @!attribute [r] base_64_encode
+      #   @return [Boolean] whether or not the serialized translations
+      #     should be returned encoded in base 64.
+      # @!attribute [r] encoding
+      #   @return [String, Encoding] the encoding translations are
+      #     expected to be in. This attribute refers to string encoding
+      #     and is distinct from base 64 encoding.
+      # @!attribute [r] include_snapshot
+      #   @return [Boolean] whether or not the snapshot used to identify
+      #     translations is returned alongside the serialized phrases.
+      # @!attribute [r] include_checksum
+      #   @return [Boolean] whether or not the checksum of translations
+      #     is returned alongside the serialized phrases.
+      # @!attribute [r] paths
+      #   @return [Array<String>] the list of paths to export translations
+      #     for. Any translations that belong to phrases that did not come
+      #     from a path in this list will not be included in the export.
+      #
+      # @example
+      #   cmd = ExportCommand.new(configuration)
+      #     .set_repo_name('my_repo')
+      #     .set_ref('master')
+      #     .set_locale('pt-BR')
+      #     .set_serializer('json/key-value')
+      #     .set_base_64_encode(true)
+      #     .set_encoding(Encoding::UTF_8)
+      #     .set_include_snapshot(false)
+      #
+      #   cmd.execute
+      #   # =>
+      #   # {
+      #   #   payload: "<base 64 encoded string>",
+      #   #   encoding: "UTF_8"
+      #   #   translation_count: 105,
+      #   #   base_64_encoded: true
+      #   #   locale: "pt-BR"
+      #   # }
+      class ExportCommand < GitCommand
+        attr_reader :locale, :serializer, :base_64_encode
+        attr_reader :encoding, :include_snapshot, :include_checksum
+        attr_reader :paths
+
+        include WithRepoName
+        include WithRef
+        include WithLocale
+
+        include WithSnapshots
+
+        validate :serializer, type: :serializer
+        validate :encoding, type: :encoding
+
+        def initialize(*args)
+          super
+          @paths = []
+          @encoding = Encoding::UTF_8
+          @base_64_encode = false
+          @include_snapshot = false
+          @include_checksum = false
+        end
+
+        # Sets the serializer used to export translations. Must be recognizable
+        # as a serializer id, eg. 'yaml/rails' or 'json/key-value'.
+        #
+        # @param [String] serializer The serializer to use.
+        # @return [self]
+        def set_serializer(serializer)
+          @serializer = serializer
+          self
+        end
+
+        # Sets whether or not the serialized translations should be returned
+        # encoded in base 64.
+        #
+        # @param [Boolean] should_encode To encode or not encode, that is
+        #   the question.
+        # @return [self]
+        def set_base_64_encode(should_encode)
+          @base_64_encode = should_encode
+          self
+        end
+
+        # Sets the encoding translations are expected to be in. Not to be
+        # confused with base 64 encoding.
+        #
+        # @param [String, Encoding] encoding The encoding to use. Can be
+        #   either a +String+ or a Ruby +Encoding+, eg. +Encoding::UTF_8+.
+        # @return [self]
+        def set_encoding(encoding)
+          @encoding = encoding
+          self
+        end
+
+        # Sets whether or not to include the snapshot in the return value.
+        #
+        # @param [Boolean] should_include_snapshot whether or not to
+        #   return the snapshot.
+        # @return [self]
+        def set_include_snapshot(should_include_snapshot)
+          @include_snapshot = should_include_snapshot
+          self
+        end
+
+        # Sets whether or not to include a checksum of the phrases in the
+        # return value.
+        #
+        # @param [Boolean] should_include_checksum whether or not to include
+        #   the checksum.
+        # @return [self]
+        def set_include_checksum(should_include_checksum)
+          @include_checksum = should_include_checksum
+          self
+        end
+
+        # A list of files or paths to filter translations by. Only translations
+        # matching these paths will be included in the export payload.
+        def set_paths(paths)
+          @paths = Array(paths)
+          self
+        end
+
+        # Perform the export.
+        #
+        # @return [Hash] containing the following attributes:
+        #   * +payload+: The serialized +String+ blob of all the translations.
+        #   * +encoding+: The encoding of the strings in +payload+.
+        #   * +translation_count+: The number of translations in +payload+.
+        #   * +base_64_encoded+: A boolean indicating if +payload+ is base
+        #     64 encoded.
+        #   * +locale+: The locale the translations in +payload+ are written in.
+        #   * +snapshot+: The snapshot used to identify the translations in
+        def execute
+          stream = StringIO.new
+          snapshot = take_snapshot(repo_config, commit_id, paths)
+          translation_count = 0
+          checksum_list = []
+
+          serializer_instance = serializer_config.klass.new(
+            stream, locale_obj, encoding
+          )
+
+          write_translations_for(snapshot, serializer_instance) do |trans|
+            translation_count += 1
+
+            if include_checksum
+              checksum_list << "#{trans.phrase.index_value}#{trans.translation}"
+            end
+          end
+
+          params = {
+            payload: encode(stream.string),
+            encoding: serializer_instance.encoding.to_s,
+            translation_count: translation_count,
+            base_64_encoded: base_64_encode,
+            locale: locale,
+            paths: paths
+          }
+
+          if include_snapshot
+            params.merge!(snapshot: snapshot)
+          end
+
+          if include_checksum
+            params.merge!(checksum: checksum_for(checksum_list))
+          end
+
+          params
+        end
+
+        private
+
+        def write_translations_for(snapshot, serializer_instance)
+          each_translation(snapshot) do |trans|
+            next unless include_trans?(trans)
+            trans = apply_preprocessors(trans, serializer_config)
+            yield trans if block_given?
+
+            serializer_instance.write_key_value(
+              trans.phrase.index_value, trans.translation
+            )
+          end
+
+          serializer_instance.flush
+        end
+
+        def include_trans?(trans)
+          paths.size == 0 || paths.include?(trans.phrase.file)
+        end
+
+        def checksum_for(list)
+          Digest::MD5.hexdigest(list.sort.join)
+        end
+
+        def locale_obj
+          @locale_obj ||= repo_config.get_locale(locale)
+        end
+
+        def apply_preprocessors(translation, serializer_config)
+          serializer_config.preprocessors.inject(translation) do |trans, preprocessor|
+            preprocessor.process(trans)
+          end
+        end
+
+        def encode(string)
+          if base_64_encode
+            Base64.encode64(string)
+          else
+            string
+          end
+        end
+
+        def serializer_config
+          @serializer_config ||= repo_config.get_serializer_config(serializer)
+        end
+
+        def repo_config
+          @repo_config ||= get_repo(repo_name)
+        end
+
+        def each_translation(snapshot)
+          datastore.phrases_by_commits(repo_name, snapshot) do |phrase|
+            # only yield if a translation exists
+            if text = repo_config.tms.lookup_translation(locale_obj, phrase)
+              yield Translation.new(phrase, locale, text)
+            end
+          end
+        end
+      end
+
+    end
+  end
+end