simple_po_parser 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 5af2707fffbad814707531a12d5c255cf0be4912
+  data.tar.gz: 1a0bc546529532459b6059dc3cf858815cf7f6b5
+SHA512:
+  metadata.gz: d7b3fe6a2abff7824a0fbb57f4671116370e6b7a84a83d8c871de8b2fdb75f832ecaf4b2f6fe09b6c5794732479685b4f3d130543be23f59a36614cd4e19eaca
+  data.tar.gz: 480e973695ec4c9ea10b51e7971306862df1b7fb7109ad7ac8fc70a03448d87e1a72b87921dc80f16ef5689cfba3013753959909c4b63ba0babfe9c306020558

data/.gitignore

@@ -0,0 +1,8 @@
+*.gem
+.bundle
+.ruby-version
+Gemfile.lock
+coverage
+pkg
+test/tmp
+tmp

data/.rspec

@@ -0,0 +1,3 @@
+--color
+--format doc
+--require spec_helper

data/.travis.yml

@@ -0,0 +1,8 @@
+bundler_args: --without development
+language: ruby
+sudo: false
+rvm:
+  - 2.1.9
+  - 2.2.6
+  - 2.3.1
+script: bundle exec rspec spec

data/Gemfile

@@ -0,0 +1,9 @@
+source 'https://rubygems.org'
+
+group :test do
+  gem 'coveralls', :require => false
+  gem 'rspec', '~> 3.5.0'
+  gem 'awesome_print'
+end
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,58 @@
+PATH
+  remote: .
+  specs:
+    simple_po_parser (0.0.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    awesome_print (1.7.0)
+    coveralls (0.8.16)
+      json (>= 1.8, < 3)
+      simplecov (~> 0.12.0)
+      term-ansicolor (~> 1.3.0)
+      thor (~> 0.19.1)
+      tins (>= 1.6.0, < 2)
+    diff-lcs (1.2.5)
+    docile (1.1.5)
+    geminabox-release (0.2.1)
+      bundler (>= 1.0.14)
+    json (2.0.2)
+    rake (12.0.0)
+    rspec (3.5.0)
+      rspec-core (~> 3.5.0)
+      rspec-expectations (~> 3.5.0)
+      rspec-mocks (~> 3.5.0)
+    rspec-core (3.5.4)
+      rspec-support (~> 3.5.0)
+    rspec-expectations (3.5.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.5.0)
+    rspec-mocks (3.5.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.5.0)
+    rspec-support (3.5.0)
+    simplecov (0.12.0)
+      docile (~> 1.1.0)
+      json (>= 1.8, < 3)
+      simplecov-html (~> 0.10.0)
+    simplecov-html (0.10.0)
+    term-ansicolor (1.3.2)
+      tins (~> 1.0)
+    thor (0.19.4)
+    tins (1.13.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  awesome_print
+  bundler
+  coveralls
+  geminabox-release (= 0.2.1)
+  rake
+  rspec (~> 3.5.0)
+  simple_po_parser!
+
+BUNDLED WITH
+   1.13.6

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2017 Dennis-Florian Herr @ Experteer GmbH
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,30 @@
+# Simple Po Parser
+
+This is a simple PO file to ruby hash parser, which complies with [GNU PO file specification](https://www.gnu.org/software/gettext/manual/html_node/PO-Files.html). Tested with the msgcat (GNU gettext-tools) 0.18.3 tool.
+
+## Hash format
+
+A PO message is parsed into a hash with meaningful keys for each type of line.
+The values are always arrays of strings.
+Each string is representing one line of content in the PO file.
+
+```ruby
+{
+  :translator_comment => [""],
+  :extracted_comment => [""],
+  :reference => [""],
+  :flag => [""],
+  :previous_msgctxt => [""], # msgctxt of the message used for the fuzzy translation
+  :previous_msgid => [""], # msgid of the messaged used for the fuzzy translation
+  :previous_msgid_plural => [""],
+  :msgctxt => [""],
+  :msgid => [""],
+  :msgid_plural => [""],
+  :msgstr => [""], # for singular messages
+  "msgstr[N]" => [""] # for plural messages, there N is the plural number starting from 0
+}
+```
+
+### License
+
+License: [MIT](LICENSE.txt) - Copyright (c) 2017 Dennis-Florian Herr @Experteer GmbH

data/Rakefile

@@ -0,0 +1,68 @@
+require 'rubygems'
+require 'rake'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+task :default => :spec
+
+require 'geminabox-release'
+GeminaboxRelease.patch(:use_config => true, :remove_release => false)
+
+desc "Generate a random po file to \"test/benchmark.po\". Takes optional rake args for number of entries"
+task 'generate_random_pofile', :messages, :obsoletes do |t, args|
+  args.with_defaults(:messages => "200", :obsoletes => "10")
+  require_relative 'spec/utils/random_pofile_generator'
+  PoParser::RandomPoFileGenerator.generate_file(
+    File.expand_path("test/benchmark.po", __dir__), args[:messages].to_i, args[:obsoletes].to_i
+  )
+end
+
+namespace :parser do
+  require 'benchmark'
+  require 'simple_po_parser'
+
+  desc "Benchmark of 10 full PoParser runs of test/benchmark.po"
+  task "benchmark" do
+    pofile = File.expand_path("test/benchmark.po", __dir__)
+    Benchmark.bmbm do |x|
+      x.report("Parser:") {10.times { SimplePoParser.parse(pofile) }}
+    end
+  end
+
+  desc "Generate 5 random PO files with 100 to 500 messages and benchmark each full PoParser run"
+  task 'five_random_po_full' do
+    include Benchmark
+    require_relative 'spec/utils/random_pofile_generator'
+    pofile = File.expand_path("test/benchmark.po.tmp", __dir__)
+    Benchmark.benchmark(CAPTION, 6, FORMAT, "total:") do |x|
+      total = nil
+      total_length = 0
+      for i in 0..5 do
+        length = (Random.new.rand * 400.0 + 100).to_i
+        total_length += length
+        puts "Benchmarking file of length #{length}"
+        SimplePoParser::RandomPoFileGenerator.generate_file(pofile, length)
+        t = x.report("try#{i}:") {SimplePoParser.parse(pofile)}
+        File.unlink(pofile)
+        total = total ? total+t : t
+      end
+      puts "Total message length #{total_length}"
+      [total]
+    end
+  end
+
+  desc "Show ruby-prof profiler for spec/fixtures/complex_entry.po"
+  task "profile_parser" do
+    require 'ruby-prof'
+    RubyProf.start
+    po_message = File.read(File.expand_path("spec/simple_po_parser/fixtures/complex_entry.po", __dir__))
+    SimplePoParser.parse_message(po_message)
+    result = RubyProf.stop
+
+    printer = RubyProf::FlatPrinter.new(result)
+    printer.print(STDOUT)
+  end
+
+
+
+end

data/lib/simple_po_parser/error.rb

@@ -0,0 +1,13 @@
+module SimplePoParser
+	class ParserError < StandardError
+	end
+
+	class PoSyntaxError < ParserError
+		@msg = ""
+		def initialize(msg="Invalid po syntax")
+			@msg = msg
+			super(msg)
+		end
+
+	end
+end

data/lib/simple_po_parser/parser.rb

@@ -0,0 +1,352 @@
+module SimplePoParser
+  # FastParser directly using Rubys powerful StringScanner (strscan)
+  #
+  # Important notes about StringScanner.scan:
+  # * scan will return nil if there is no match. Using the regex * (zero or more) quantifier will
+  #  let scan return an empty string if there is "no match" as the empty string qualifies as
+  #  a match of the regex (zero times). We make use of this "trick"
+  # * the start of line anchor ^ is obsolete as scan will only match start of line.
+  # * rubys regex is by default in single-line mode, therefore scan will only match until
+  #  the next newline is hit (unless multi-line mode is explicitly enabled)
+  module Parser
+    require_relative 'error'
+    require 'strscan'
+    extend self
+
+    # parse a single message of the PO format.
+    #
+    # @param message a single PO message in String format without leading or trailing whitespace
+    # @return [Hash] parsed PO message information in Hash format
+    def parse(message)
+      @result = {}
+      @scanner = StringScanner.new(message.strip)
+      begin
+        lines
+      rescue ParserError => pe
+        puts "Parsing error!"
+        puts "#{pe.message}"
+        puts "Backtrace:\n#{pe.backtrace.select{|i| i =~ /lib\/poparser/}.join("\n")}"
+        puts "\nResult up to error: '#{@result}'"
+        exit
+      end
+      @result
+    end
+
+    private
+
+    #########################################
+    ###            branching              ###
+    #########################################
+
+    # arbitary line of a PO message. Can be comment or message
+    # message parsing is always started with checking for msgctxt as content is expected in
+    # msgctxt -> msgid -> msgid_plural -> msgstr order
+    def lines
+      begin
+        if @scanner.scan(/#/)
+          comment
+        else
+          msgctxt
+        end
+      rescue PoSyntaxError => pe
+        # throw a normal ParserError to break the recursion
+        raise ParserError, "Syntax error in lines\n" + pe.message, pe.backtrace
+      end
+    end
+
+    # match a comment line. called on lines starting with '#'.
+    # Recalls line when the comment line was parsed
+    def comment
+      begin
+        case @scanner.getch
+        when ' '
+          skip_whitespace
+          add_result(:translator_comment, comment_text)
+          lines
+        when '.'
+          skip_whitespace
+          add_result(:extracted_comment, comment_text)
+          lines
+        when ':'
+          skip_whitespace
+          add_result(:reference, comment_text)
+          lines
+        when ','
+          skip_whitespace
+          add_result(:flag, comment_text)
+          lines
+        when '|'
+          skip_whitespace
+          previous_comments
+          lines
+        when "\n"
+          add_result(:translator_comment, "") # empty comment line
+          lines
+        when '~'
+          if @result[:previous_msgctxt] || @result[:previous_msgid] || @result[:previous_msgid_plural]
+            raise PoSyntaxError, "Previous comment entries need to be marked obsolete too in obsolete message entries. But already got: #{@result}"
+          end
+          skip_whitespace
+          add_result(:obsolete, comment_text)
+          obsoletes
+        else
+          @scanner.pos = @scanner.pos - 2
+          raise PoSyntaxError, "Unknown comment type #{@scanner.peek(10).inspect}"
+        end
+      rescue PoSyntaxError => pe
+        raise PoSyntaxError, "Syntax error in comment\n" + pe.message, pe.backtrace
+      end
+    end
+
+    # matches the msgctxt line and will continue to check for msgid afterwards
+    #
+    # msgctxt is optional
+    def msgctxt
+      begin
+        if @scanner.scan(/msgctxt/)
+          skip_whitespace
+          text = message_line
+          add_result(:msgctxt, text)
+          message_multiline(:msgctxt) if text.empty?
+        end
+        msgid
+      rescue PoSyntaxError => pe
+        raise PoSyntaxError, "Syntax error in msgctxt\n" + pe.message, pe.backtrace
+      end
+    end
+
+    # matches the msgid line. Will check for optional msgid_plural.
+    # Will advance to msgstr or msgstr_plural based on msgid_plural
+    #
+    # msgid is required
+    def msgid
+      begin
+        if @scanner.scan(/msgid/)
+          skip_whitespace
+          text = message_line
+          add_result(:msgid, text)
+          message_multiline(:msgid) if text.empty?
+          if msgid_plural
+            msgstr_plural
+          else
+            msgstr
+          end
+        else
+          err_msg = "Message without msgid is not allowed."
+          err_msg += "The Line started unexpectedly with #{@scanner.peek(10).inspect}."
+          raise PoSyntaxError, err_msg
+        end
+      rescue PoSyntaxError => pe
+        raise PoSyntaxError, "Syntax error in msgid\n" + pe.message, pe.backtrace
+      end
+
+    end
+
+    # matches the msgid_plural line.
+    #
+    # msgid_plural is optional
+    #
+    # @return [boolean] true if msgid_plural is present, false otherwise
+    def msgid_plural
+      begin
+        if @scanner.scan(/msgid_plural/)
+          skip_whitespace
+          text = message_line
+          add_result(:msgid_plural, text)
+          message_multiline(:msgid_plural) if text.empty?
+          true
+        else
+          false
+        end
+      rescue PoSyntaxError => pe
+        raise PoSyntaxError, "Syntax error in msgid\n" + pe.message, pe.backtrace
+      end
+    end
+
+    # matches the msgstr singular line
+    #
+    # msgstr is required in singular translations
+    def msgstr
+      begin
+        if @scanner.scan(/msgstr/)
+          skip_whitespace
+          text = message_line
+          add_result(:msgstr, text)
+          message_multiline(:msgstr) if text.empty?
+          skip_whitespace
+          raise PoSyntaxError, "Unexpected content after expected message end #{@scanner.peek(10).inspect}" unless @scanner.eos?
+        else
+         raise PoSyntaxError, "Singular message without msgstr is not allowed. Line started unexpectedly with #{@scanner.peek(10).inspect}."
+        end
+      rescue PoSyntaxError => pe
+        raise PoSyntaxError, "Syntax error in msgstr\n" + pe.message, pe.backtrace
+      end
+    end
+
+
+    def msgstr_plural(num = 0)
+      begin
+        msgstr_key = @scanner.scan(/msgstr\[\d\]/) # matches 'msgstr[0]' to 'msgstr[9]'
+        if msgstr_key
+          # msgstr plurals must come in 0-based index in order
+          msgstr_num = msgstr_key.match(/\d/)[0].to_i
+          raise PoSyntaxError, "Bad 'msgstr[index]' index." if msgstr_num != num
+          skip_whitespace
+          text = message_line
+          add_result(msgstr_key, text)
+          message_multiline(msgstr_key) if text.empty?
+          msgstr_plural(num+1)
+        elsif num == 0 # and msgstr_key was false
+          raise PoSyntaxError, "Plural message without msgstr[0] is not allowed. Line started unexpectedly with #{@scanner.peek(10).inspect}."
+        else
+          raise PoSyntaxError, "End of message was expected, but line started unexpectedly with #{@scanner.peek(10).inspect}" unless @scanner.eos?
+        end
+      rescue PoSyntaxError => pe
+        raise PoSyntaxError, "Syntax error in msgstr_plural\n" + pe.message, pe.backtrace
+      end
+    end
+
+    def previous_comments
+      begin
+        # next part must be msgctxt, msgid or msgid_plural
+        if @scanner.scan(/msg/)
+          if @scanner.scan(/id/)
+            if @scanner.scan(/_plural/)
+              key = :previous_msgid_plural
+            else
+              key = :previous_msgid
+            end
+          elsif @scanner.scan(/ctxt/)
+            key = :previous_msgctxt
+          else
+            raise PoSyntaxError, "Previous comment type #{("msg" + @scanner.peek(10)).inspect} unknown."
+          end
+          skip_whitespace
+          text = message_line
+          add_result(key, text)
+          previous_multiline(key) if text.empty?
+        else
+          raise PoSyntaxError, "Previous comments must start with '#| msg'. #{@scanner.peek(10).inspect} unknown."
+        end
+      rescue PoSyntaxError => pe
+        raise PoSyntaxError, "Syntax error in previous_comments\n" + pe.message, pe.backtrace
+      end
+    end
+
+    def previous_multiline(key)
+      begin
+        # scan multilines until no further multiline is hit
+        # /#\|\p{Blank}"/ needs to catch the double quote to ensure it hits a previous
+        # multiline and not another line type.
+        if @scanner.scan(/#\|\p{Blank}*"/)
+          @scanner.pos = @scanner.pos - 1 # go one character back, so we can reuse the "message line" method
+          add_result(key, message_line)
+          previous_multiline(key) # go on until we no longer hit a multiline line
+        end
+      rescue PoSyntaxError => pe
+        raise PoSyntaxError, "Syntax error in previous_multiline\n" + pe.message, pe.backtrace
+      end
+    end
+
+    def message_multiline(key)
+      begin
+        skip_whitespace
+        if @scanner.check(/"/)
+          add_result(key, message_line)
+          message_multiline(key)
+        end
+      rescue PoSyntaxError => pe
+        raise PoSyntaxError, "Syntax error in message_multiline with key '#{key}'\n" + pe.message, pe.backtrace
+      end
+    end
+
+    # identifies a message line and returns it's text or raises an error
+    #
+    # @return [String] message_text
+    def message_line
+      begin
+        if @scanner.getch == '"'
+          text = message_text
+          unless @scanner.getch == '"'
+            err_msg = "The message text '#{text}' must be finished with the double quote character '\"'."
+            raise PoSyntaxError, err_msg
+          end
+          skip_whitespace
+          unless end_of_line
+            err_msg = "There should be only whitespace until the end of line"
+            err_msg += "after the double quote character of a message text."
+            raise PoSyntaxError.new(err_msg)
+          end
+          text
+        else
+          @scanner.pos = @scanner.pos - 1
+          err_msg = "A message text needs to start with the double quote character '\"',"
+          err_msg += " but this was found: #{@scanner.peek(10).inspect}"
+          raise PoSyntaxError, err_msg
+        end
+      rescue PoSyntaxError => pe
+        raise PoSyntaxError, "Syntax error in message_line\n" + pe.message, pe.backtrace
+      end
+    end
+
+    # used to parse all obsolete lines. An obsolete message may only contain obsolete entries
+    def obsoletes
+      if @scanner.scan(/#~/)
+        skip_whitespace
+        add_result(:obsolete, comment_text)
+        obsoletes
+      else
+        raise PoSyntaxError, "All lines must be obsolete after the first obsolete line, but got #{@scanner.peek(10).inspect}." unless @scanner.eos?
+      end
+    end
+
+    #########################################
+    ###             scanning              ###
+    #########################################
+
+    # returns the text of a comment
+    #
+    # @return [String] text
+    def comment_text
+      begin
+        text = @scanner.scan(/.*/) # everything until newline
+        text.rstrip! # benchmarked faster too rstrip the string in place
+        raise PoSyntaxError, "Comment text should advance to next line or stop at eos" unless end_of_line
+        text
+      rescue PoSyntaxError => pe
+        raise PoSyntaxError, "Syntax error in commtent_text\n" + pe.message, pe.backtrace
+      end
+    end
+
+    # returns the text of a message line
+    #
+    # @return [String] text
+    def message_text
+      @scanner.scan_until(/(\\(\\|")|[^"])*/) # this parses anything until an unescaped quote is hit
+    end
+
+    # advances the scanner until the next non whitespace position.
+    # Does not match newlines. See WHITESPACE_REGEX constant
+    def skip_whitespace
+      @scanner.skip(/\p{Blank}+/)
+    end
+
+    # returns true if the scanner is at beginning of next line or end of string
+    def end_of_line
+      @scanner.scan(/\n/)
+      @scanner.eos? || @scanner.bol?
+    end
+
+    # adds text to the given key in results
+    # creates an array if the given key already has a result
+    def add_result(key, text)
+      if @result[key]
+        @result[key].push(text)
+      else
+        @result[key] = [text]
+      end
+    end
+
+  end
+
+end

data/lib/simple_po_parser/tokenizer.rb

@@ -0,0 +1,21 @@
+module SimplePoParser
+  # Split a PO file into single PO message entities (a message is seperated by two newline)
+  class Tokenizer
+    def initialize
+      @messages = []
+    end
+
+    def parse_file(path)
+      File.open(path, 'r').each_line("\n\n") do |block|
+        block.strip! # dont parse empty blocks
+        @messages << parse_block(block) if block != ''
+      end
+      @messages
+    end
+
+  private
+    def parse_block(block)
+      Parser.parse(block)
+    end
+  end
+end

data/lib/simple_po_parser/version.rb

@@ -0,0 +1,3 @@
+module SimplePoParser
+  VERSION = "1.0.0"
+end

data/lib/simple_po_parser.rb

@@ -0,0 +1,21 @@
+require 'simple_po_parser/error'
+require 'simple_po_parser/parser'
+require 'simple_po_parser/tokenizer'
+require 'simple_po_parser/version'
+
+module SimplePoParser
+  class << self
+    # parse po file
+    #
+    # returns an array of po messages as hashes
+    def parse(path)
+      Tokenizer.new.parse_file(path)
+    end
+
+    # parses a single message.
+    def parse_message(message)
+      Parser.parse(message)
+    end
+  end
+
+end

data/simple_po_parser.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'simple_po_parser/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "simple_po_parser"
+  spec.version       = SimplePoParser::VERSION
+  spec.authors       = ["Dennis-Florian Herr"]
+  spec.email         = ["dennis.herr@experteer.com"]
+  spec.summary       = %q{A simple PO file to ruby hash parser}
+  spec.description   = %q{A simple PO file to ruby hash parser . PO files are translation files generated by GNU/Gettext tool.}
+  spec.homepage      = "http://github.com/experteer/simple_po_parser"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^spec/})
+  spec.require_paths = ["lib"]
+
+  # Development deps
+  spec.add_development_dependency "bundler", ">= 0"
+  spec.add_development_dependency "rake", ">= 0"
+  # geminabox release
+  spec.add_development_dependency 'geminabox-release', "0.2.1"
+end

data/spec/simple_po_parser/fixtures/complex_entry.po

@@ -0,0 +1,21 @@
+# translator-comment
+#
+#. extract
+#: reference1
+#: reference2
+#, flag
+#| msgctxt "previous context"
+#| msgid ""
+#|  "multiline\n"
+#|"previous messageid"
+#| msgid_plural "previous msgid_plural"
+msgctxt "Context"
+msgid "msgid"
+msgid_plural ""
+"multiline msgid_plural\n"
+""
+msgstr[0] "msgstr 0"
+msgstr[1] ""
+"msgstr 1 multiline 1\n"
+"msgstr 1 line 2\n"
+msgstr[2] "msgstr 2"