puttext 0.1.0

data/lib/puttext/parser/base.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require_relative '../po_entry'
+
+module PutText
+  module Parser
+    # Thrown when any error parsing a file occurs
+    class ParseError < StandardError; end
+
+    class Base
+      # Parse gettext strings from a file in the path.
+      # @param [String] path the path of the file to parse.
+      # @return [Array<POEntry>] an array of POEntry objects extracted
+      #   from the given file.
+      def strings_from_file(path)
+        strings_from_source(File.read(path), filename: path)
+      end
+
+      # @abstract Subclass is expected to implement #strings_from_source
+      # @!method strings_from_source(source, opts)
+      #   Parse gettext strings from a given snippet of source code.
+      #   @param [String] source the snippet of source code to parse.
+      #   @param [Hash] opts
+      #   @option opts [String] :filename path of the file being parsed.
+      #     Defaults to "(string)".
+      #   @option opts [Integer] :first_line number of the first line being
+      #     parsed. Defaults to 1.
+      #   @return [Array<POEntry>] an array of POEntry objects
+      #     extracted from the given source code.
+    end
+  end
+end

data/lib/puttext/parser/ruby.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+require_relative 'base'
+require_relative '../po_entry'
+
+require 'parser/current'
+
+# opt-in to most recent Parser AST format:
+Parser::Builders::Default.emit_lambda = true
+Parser::Builders::Default.emit_procarg0 = true
+
+module PutText
+  module Parser
+    class Ruby < Base
+      METHODS = {
+        gettext:   :regular,
+        _:         :regular,
+        ngettext:  :plural,
+        n_:        :plural,
+        sgettext:  :context_sep,
+        s_:        :context_sep,
+        nsgettext: :context_sep_plural,
+        ns_:       :context_sep_plural,
+        pgettext:  :context,
+        p_:        :context,
+        npgettext: :context_plural,
+        np_:       :context_plural
+      }.freeze
+
+      PARAMS = {
+        regular:            %i(msgid),
+        plural:             %i(msgid msgid_plural),
+        context:            %i(msgctxt msgid),
+        context_plural:     %i(msgctxt msgid msgid_plural),
+        context_sep:        %i(msgid separator),
+        context_sep_plural: %i(msgid msgid_plural _ separator)
+      }.freeze
+
+      def initialize
+        @ruby_parser = ::Parser::CurrentRuby.new
+      end
+
+      def strings_from_source(source, filename: '(string)', first_line: 1)
+        buffer = ::Parser::Source::Buffer.new(filename, first_line)
+        buffer.source = source
+
+        @ruby_parser.reset
+        ast = @ruby_parser.parse(buffer)
+
+        if ast.is_a? ::Parser::AST::Node
+          find_strings_in_ast(ast)
+        else
+          []
+        end
+      end
+
+      private
+
+      def string_from_ast_node(ast_node)
+        return if ast_node.nil?
+
+        case ast_node.type
+        when :str
+          ast_node.children[0]
+        else
+          raise ParseError,
+                format('unsupported AST node type: %s', ast_node.type)
+        end
+      end
+
+      def po_entry_from_ast_node(ast_node, type)
+        filename = ast_node.location.expression.source_buffer.name
+        line     = ast_node.location.line
+
+        entry_attrs = { references: ["#{filename}:#{line}"] }
+
+        PARAMS[type].each_with_index do |name, index|
+          next if name == :_ # skip parameters named _
+
+          param = string_from_ast_node(ast_node.children[index + 2])
+          entry_attrs[name] = param if param
+        end
+
+        PutText::POEntry.new(entry_attrs)
+      end
+
+      def find_strings_in_ast(ast_node)
+        entries = []
+
+        if ast_node.type == :send && METHODS[ast_node.children[1]]
+          entries << po_entry_from_ast_node(
+            ast_node,
+            METHODS[ast_node.children[1]]
+          )
+        else
+          entries += find_strings_in_each_ast(ast_node.children)
+        end
+
+        entries
+      end
+
+      def find_strings_in_each_ast(ast_nodes)
+        entries = []
+
+        ast_nodes.each do |node|
+          next unless node.is_a? ::Parser::AST::Node
+          entries += find_strings_in_ast(node)
+        end
+
+        entries
+      end
+    end
+  end
+end

data/lib/puttext/parser/slim.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require_relative 'base'
+require_relative 'ruby'
+
+begin
+  require 'slim'
+
+  module PutText
+    module Parser
+      class Slim < Base
+        def initialize
+          @ruby_parser = PutText::Parser::Ruby.new
+          @slim_engine = Engine.new
+        end
+
+        def strings_from_source(source, filename: '(string)', first_line: 1)
+          slim_ruby_code = @slim_engine.call(source)
+
+          @ruby_parser.strings_from_source(
+            slim_ruby_code,
+            filename: filename,
+            first_line: first_line
+          )
+        end
+
+        class IgnoreEmbedded < ::Slim::Filter
+          def on_slim_embedded(_name, body)
+            newlines = count_newlines(body)
+
+            node = [:multi]
+            newlines.times { node.push [:newline] }
+            node
+          end
+
+          private
+
+          def count_newlines(body)
+            newlines = 0
+            newlines += 1 if body.first == :newline
+
+            body.each do |el|
+              newlines += count_newlines(el) if el.is_a?(Array)
+            end
+
+            newlines
+          end
+        end
+
+        class Engine < ::Slim::Engine
+          replace ::Slim::Embedded, IgnoreEmbedded
+        end
+      end
+    end
+  end
+
+# rubocop:disable Lint/HandleExceptions
+rescue LoadError
+  # Optional dependency, do not fail if not found
+end
+# rubocop:enable Lint/HandleExceptions

data/lib/puttext/po_entry.rb

@@ -0,0 +1,181 @@
+# frozen_string_literal: true
+
+module PutText
+  class POEntry
+    NS_SEPARATOR = '|'.freeze
+
+    PO_C_STYLE_ESCAPES = {
+      "\n" => '\\n',
+      "\r" => '\\r',
+      "\t" => '\\t',
+      '\\' => '\\\\',
+      '"' => '\\"'
+    }.freeze
+
+    attr_reader :msgid
+    attr_reader :msgid_plural
+    attr_reader :msgctxt
+    attr_reader :references
+
+    # Create a new POEntry
+    #
+    # @param [Hash] attrs
+    # @option attrs [String] :msgid the id of the string (the string that needs
+    #   to be translated). Can include a context, separated from the id by
+    #   {NS_SEPARATOR} or by the specified :separator.
+    # @option attrs [String] :msgid_plural the pluralized id of the string (the
+    #   pluralized string that needs to be translated).
+    # @option attrs [String] :msgctxt the context of the string.
+    # @option attrs [Array<String>] :references a list of files with line
+    #   numbers, pointing to where the string was found.
+    # @option attrs [String] :separator the separator of context from id in
+    #   :msgid.
+    def initialize(attrs)
+      id, ctx = extract_context(
+        attrs[:msgid], attrs[:separator] || NS_SEPARATOR
+      )
+
+      @msgid        = id
+      @msgctxt      = attrs[:msgctxt] || ctx
+      @msgid_plural = attrs[:msgid_plural]
+      @references   = attrs[:references] || []
+    end
+
+    # Convert the entry to a string representation, to be written to a .po file
+    # @return [String] a string representation of the entry.
+    def to_s
+      str = String.new('')
+
+      # Add comments
+      str = add_comment(str, ':', @references.join(' ')) if references?
+
+      # Add id and context
+      str = add_string(str, 'msgctxt', @msgctxt) if @msgctxt
+      str = add_string(str, 'msgid', @msgid)
+      str = add_string(str, 'msgid_plural', @msgid_plural) if plural?
+      str = add_translations(str)
+
+      str
+    end
+
+    # Check if the entry has any references.
+    # @return [Boolean] whether the entry has any references.
+    def references?
+      !@references.empty?
+    end
+
+    # Check if the entry has a plural form.
+    # @return [Boolean] whether the entry has a plural form.
+    def plural?
+      !@msgid_plural.nil?
+    end
+
+    # Return an object uniquely identifying this entry. The returned object can
+    # be used to find duplicate entries.
+    # @return an object uniquely identifying this entry.
+    def unique_key
+      [@msgid, @msgctxt]
+    end
+
+    # Merge this entry with another entry. Modifies the current entry in place.
+    # Currently, merges only the references, and leaves other attributes of the
+    # current entry untouched.
+    #
+    # @param [POEntry] other_entry the entry to merge with.
+    # @return [POEntry] the merged entry.
+    def merge(other_entry)
+      @references += other_entry.references
+      self
+    end
+
+    def ==(other)
+      @msgid == other.msgid &&
+        @msgid_plural == other.msgid_plural &&
+        @msgctxt == other.msgctxt &&
+        @references == other.references
+    end
+
+    private
+
+    def extract_context(str, separator)
+      parts = str.rpartition(separator)
+      [parts[2], parts[0] == '' ? nil : parts[0]]
+    end
+
+    def add_comment(str, comment_type, value)
+      value.each_line do |line|
+        str << '#'
+        str << comment_type
+        str << ' '
+        str << line
+        str << "\n"
+      end
+
+      str
+    end
+
+    def add_string(str, id, value)
+      str << id
+      str << ' '
+      str << string_to_po(value)
+      str << "\n"
+    end
+
+    def add_translations(str)
+      if plural?
+        add_string(str, 'msgstr[0]', '')
+        add_string(str, 'msgstr[1]', '')
+      else
+        add_string(str, 'msgstr', '')
+      end
+
+      str
+    end
+
+    def string_to_po(str)
+      lines = str.split("\n", -1)
+
+      if lines.empty?
+        '""'
+      elsif lines.length == 1
+        "\"#{po_escape_string(lines[0])}\""
+      else
+        multiline_string_to_po(lines)
+      end
+    end
+
+    def multiline_string_to_po(str_lines)
+      po_str = String.new('""')
+
+      str_lines.each_with_index do |line, index|
+        last = index == str_lines.length - 1
+        add_multiline_str_part(po_str, line, last)
+      end
+
+      po_str
+    end
+
+    def add_multiline_str_part(str, part, last)
+      return if last && part.empty?
+
+      str << "\n\""
+      str << po_escape_string(part)
+      str << '\\n' unless last
+      str << '"'
+    end
+
+    def po_escape_string(str)
+      encoded = String.new('')
+
+      str.each_char do |char|
+        encoded << if PO_C_STYLE_ESCAPES[char]
+                     PO_C_STYLE_ESCAPES[char]
+                   else
+                     char
+                   end
+      end
+
+      encoded
+    end
+  end
+end

data/lib/puttext/po_file.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module PutText
+  class POFile
+    attr_accessor :entries
+
+    # Create a new POFile
+    # @param [Array<POEntry>] entries an array of POEntry objects, that should
+    #   be placed in this file.
+    def initialize(entries)
+      @entries = entries
+    end
+
+    def to_s
+      str_io = StringIO.new
+      write_to(str_io)
+      str_io.string
+    end
+
+    # Write the contents of this file to the specified IO object.
+    # @param [IO] io the IO object to write the contents of the file to.
+    def write_to(io)
+      deduplicate
+
+      @entries.each_with_index do |entry, index|
+        io.write("\n") unless index == 0
+        io.write(entry.to_s)
+      end
+    end
+
+    def ==(other)
+      @entries.sort == other.entries.sort
+    end
+
+    private
+
+    def deduplicate
+      uniq_entries = {}
+
+      @entries.each do |entry|
+        key = entry.unique_key
+
+        if uniq_entries[key]
+          uniq_entries[key].merge(entry)
+        else
+          uniq_entries[key] = entry
+        end
+      end
+
+      @entries = uniq_entries.values
+    end
+  end
+end

data/puttext.gemspec

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require 'date'
+
+Gem::Specification.new do |s|
+  s.name     = 'puttext'
+  s.version  = '0.1.0'
+  s.date     = Date.today.to_s
+  s.summary  = 'Extract gettext strings from Ruby source'
+  s.authors  = ['Mantas Norvaiša']
+  s.email    = 'mntnorv@gmail.com'
+  s.homepage = 'https://github.com/mntnorv/puttext'
+  s.license  = 'MIT'
+
+  s.files         = `git ls-files`.split
+  s.executables   = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  s.require_paths = ['lib']
+
+  s.required_ruby_version = '>= 2.0.0'
+
+  s.add_runtime_dependency('parser', '>= 2.4.0.0', '< 3.0')
+
+  # Optional dependencies at runtime, required for development
+  s.add_development_dependency('slim', '~> 3.0')
+
+  # Tools
+  s.add_development_dependency('rake')
+
+  # Testing
+  s.add_development_dependency('rspec', '~> 3.5')
+  s.add_development_dependency('unindent')
+
+  # Linters and code policies
+  s.add_development_dependency('rubocop', '~> 0.46.0')
+  s.add_development_dependency('simplecov')
+  s.add_development_dependency('codeclimate-test-reporter', '~> 1.0.0')
+end