PoParser 2.0.1 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4001750c5298956f310f421c8f9cfe8d8c64283c
-  data.tar.gz: bff6cd76882df6d406cc2f993a010a48b41320b9
+  metadata.gz: 32f20604e8c93db11ec6985ba1080532234efb33
+  data.tar.gz: 43ffda461e75249098e7e4b3fd2fc5cbe8918359
 SHA512:
-  metadata.gz: a67f9b02929802776b34717d0f4d3f3144bf500a9df17a4f9811f43446f15b64f964ccb0736486e44c73bc22760a3687a98fe0414c92e70843ef5247b7b8821e
-  data.tar.gz: 2ffcdb67d60a39d43d49f55222e1dbae130a852522105a1452adebe57285332c68741ac60aefa8c5fbcedeed61a34cc66c183b7bace7588f62993296a7182db4
+  metadata.gz: e038dce601f0c92ce204a5bfb79f3298c6f1d048447515a373d9fb4fce007a548b1701b4d9f45ab9303f8f2f4d953772baa55baa2eabfa99c2f280bafee1d9c1
+  data.tar.gz: 02c28740f60996c1b1b3d1989a99f14cf97e50bf93aac8ff22df3691c3c43ee40ae1aa3ad408461d2704ef752ed93fe135b3bcfb0e634be3e364674ef74672fa

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+3.0.0 / 2017-03-03
+==================
+  * replaced parslet with simple_po_parser. Speedup parsing by up to 500x.
+    Courtesy of @dfherr
 
 2.0.1 / 2016-12-29
 ==================

data/Gemfile

@@ -2,14 +2,17 @@ source 'https://rubygems.org'
 
 group :test do
   gem 'coveralls', :require => false
-  gem 'pry-byebug', :platforms => :mri
   gem 'rspec', '~> 3.5.0'
   gem 'awesome_print'
 end
 
 group :development do
-  gem 'pry-byebug', :platforms => :mri
   gem 'guard-rspec'
+  gem 'ruby-prof'
+end
+
+group :test, :development do
+  gem 'pry-byebug', :platforms => :mri
 end
 
 # Specify your gem's dependencies in poparser.gemspec

data/README.md

@@ -103,7 +103,9 @@ translator_comment
 reference
 extracted_comment
 flag
-previous_untraslated_string
+previous_msgctxt
+previous_msgid
+previous_msgid_plural
 cached # obsolete entries
 msgid
 msgid_plural
@@ -140,6 +142,16 @@ entry.msgstr.to_s
 #=> "This entry is translated"
 ```
 
+But be careful with plural messages, there msgstr is an array
+```ruby
+if entry.plural?
+  entry.msgstr.each do |msgstr|
+    msgstr.to_s
+    #=> This is one of the plural translations
+  end  
+end  
+```
+
 You can mark an entry as fuzzy:
 
 ```ruby
@@ -220,6 +232,16 @@ You can get and set following variables from `header`:
 * Update header after changing/saving po
 * add `before_save` and `after_save` callbacks
 
+##Authors
+
+[Arash Mousavi](https://github.com/arashm)
+
+### [Contributors](https://github.com/arashm/PoParser/contributors)
+ - [Dennis-Florian Herr](https://github.com/dfherr)
+ - [Roland Gropmair](https://github.com/roland9)
+
+License: [MIT](https://github.com/arashm/PoParser/blob/master/LICENSE.txt)
+
 ## Contributing
 
 1. Fork it ( http://github.com/arashm/poparser/fork )

data/Rakefile

@@ -1 +1,65 @@
-require "bundler/gem_tasks"
+require 'bundler/gem_tasks'
+require 'rubygems'
+require 'rake'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec
+
+desc "Generate a random po file. Takes optional rake args for number of entries"
+task 'generate_random_pofile', :messages, :obsoletes do |t, args|
+  args.with_defaults(:messages => "200", :obsoletes => "10")
+  require_relative 'spec/utils/random_pofile_generator'
+  PoParser::RandomPoFileGenerator.generate_file(
+    File.expand_path("test/benchmark.po", __dir__), args[:messages].to_i, args[:obsoletes].to_i
+  )
+end
+
+namespace :debug do
+  require 'benchmark'
+  require_relative 'lib/poparser'
+
+  desc "Benchmark of 10 full PoParser runs of test/benchmark.po"
+  task "benchmark" do
+    pofile = File.expand_path("test/benchmark.po", __dir__)
+    Benchmark.bmbm do |x|
+      x.report("Parser:") {10.times { PoParser.new.parse(pofile) }}
+    end
+  end
+
+  desc "Generate 5 random PO files with 100 to 500 messages and benchmark each full PoParser run"
+  task 'five_random_po_full' do
+    include Benchmark
+    require_relative 'spec/utils/random_pofile_generator'
+    pofile = File.expand_path("test/benchmark.po.tmp", __dir__)
+    Benchmark.benchmark(CAPTION, 6, FORMAT, "total:") do |x|
+      total = nil
+      total_length = 0
+      for i in 0..5 do
+        length = (Random.new.rand * 400.0 + 100).to_i
+        total_length += length
+        puts "Benchmarking file of length #{length}"
+        SimplePoParser::RandomPoFileGenerator.generate_file(pofile, length)
+        t = x.report("try#{i}:") {PoParser.new.parse(pofile)}
+        File.unlink(pofile)
+        total = total ? total+t : t
+      end
+      puts "Total message length #{total_length}"
+      [total]
+    end
+  end
+
+  desc "Show ruby-prof profiler for spec/fixtures/complex_entry.po"
+  task "profile_parser" do
+    require 'ruby-prof'
+    RubyProf.start
+    po_message = File.read(File.expand_path("spec/simple_po_parser/fixtures/complex_entry.po", __dir__))
+    PoParser.new.parse(po_message)
+    result = RubyProf.stop
+
+    printer = RubyProf::FlatPrinter.new(result)
+    printer.print(STDOUT)
+  end
+
+end

data/lib/poparser/comment.rb

@@ -10,23 +10,50 @@ module PoParser
     def to_s(with_label = false)
       return to_str unless with_label
       if @value.is_a? Array
-        string = []
-        @value.each do |str|
-          string << "#{COMMENTS_LABELS[@type]} #{str}\n".gsub(/[^\S\n]+$/, '')
+        if @type.to_s =~ /^previous_/ # these behave more like messages
+          remove_empty_line
+          string = ["#{COMMENTS_LABELS[@type]} \"\"\n"]
+          @value.each do |str|
+            string << "#| \"#{str}\"\n".gsub(/[\p{Blank}]+$/, '')
+          end
+        else
+          string = []
+          @value.each do |str|
+            string << "#{COMMENTS_LABELS[@type]} #{str}\n".gsub(/[\p{Blank}]+$/, '')
+          end
         end
         return string.join
       else
-        # removes the space but not newline at the end
-        "#{COMMENTS_LABELS[@type]} #{@value}\n".gsub(/[^\S\n]+$/, '')
+        if @type.to_s =~ /^previous_/ # these behave more like messages
+          "#{COMMENTS_LABELS[@type]} \"#{@value}\"\n".gsub(/[\p{Blank}]+$/, '')
+        else
+          # removes the space but not newline at the end
+          "#{COMMENTS_LABELS[@type]} #{@value}\n".gsub(/[\p{Blank}]+$/, '')
+        end
       end
     end
 
     def to_str
-      @value.is_a?(Array) ? @value.join : @value
+      if @value.is_a?(Array)
+        if @type.to_s =~ /^previous_/ # these behave more like messages
+          @value.join
+        else
+          @value.join("\n")
+        end
+      else
+        @value
+      end
     end
 
     def inspect
       @value
     end
+
+  private
+    def remove_empty_line
+      if @value.is_a? Array
+        @value.shift if @value.first == ''
+      end
+    end
   end
 end

data/lib/poparser/constants.rb

@@ -4,8 +4,10 @@ module PoParser
     :extracted_comment => '#.',
     :reference => '#:',
     :flag => '#,',
-    :previous_untraslated_string => '#|',
-    :cached => '#~'
+    :previous_msgctxt => '#| msgctxt',
+    :previous_msgid => '#| msgid',
+    :previous_msgid_plural => '#| msgid_plural',
+    :obsolete => '#~'
   }
 
   ENTRIES_LABELS = {

data/lib/poparser/entry.rb

@@ -18,27 +18,27 @@ module PoParser
       define_reader_methods
 
       self.class.send(:alias_method, :translate, :msgstr=)
-      self.class.send(:alias_method, :obsolete, :cached)
-      self.class.send(:alias_method, :obsolete=, :cached=)
+      self.class.send(:alias_method, :cached, :obsolete)
+      self.class.send(:alias_method, :cached=, :obsolete=)
       # alias for backward compatibility of this typo
       self.class.send(:alias_method, :refrence, :reference)
-      self.class.send(:alias_method, :refrence=, :reference)
+      self.class.send(:alias_method, :refrence=, :reference=)
     end
 
-    # If entry doesn't have any msgid, it's probably a cached entry that is
+    # If entry doesn't have any msgid, it's probably a obsolete entry that is
     # kept by the program for later use. These entries will usually start with: #~
     #
     # @return [Boolean]
-    def cached?
-      !@cached.nil?
+    def obsolete?
+      !@obsolete.nil?
     end
-    alias_method :obsolete?, :cached?
+    alias_method :cached?, :obsolete?
 
     # Checks if the entry is untraslated
     #
     # @return [Boolean]
     def untranslated?
-      return false if cached? || fuzzy?
+      return false if obsolete? || fuzzy?
       if @msgstr.is_a? Array
         return @msgstr.map {|ms| ms.str}.join.empty?
       end
@@ -50,7 +50,7 @@ module PoParser
     #
     # @return [Boolean]
     def translated?
-      return false if cached? || fuzzy?
+      return false if obsolete? || fuzzy?
       not untranslated?
     end
     alias_method :complete? , :translated?
@@ -66,7 +66,7 @@ module PoParser
     #
     # @return [Boolean]
     def fuzzy?
-      return false if cached?
+      return false if obsolete?
       @flag.to_s.match('fuzzy') ? true : false
     end
 

data/lib/poparser/header.rb

@@ -70,6 +70,26 @@ module PoParser
       string.join("\n")
     end
 
+    def inspect
+      string = []
+      if @comments.is_a?(Array)
+        @comments.each do |comment|
+          string << "# #{comment}".strip
+        end
+      else
+        string << "# #{@comments}".strip
+      end
+      string << "#, #{@flag.to_s}" if @flag
+      string << "msgid \"\"\nmsgstr \"\""
+      configs.each do |k, v|
+        if v.nil? || v.empty?
+          next
+        end
+        string << "#{k}: #{v}\n".dump
+      end
+      string.join("\n")
+    end
+
   private
     def convert_msgstr_to_hash(msgstr)
       options_array = msgstr.value.map do |options|

data/lib/poparser/po.rb

@@ -38,14 +38,14 @@ module PoParser
 
     # Returns an array of all entries in po file
     #
-    # @param include_cached [Boolean] Whether include cached entries or not
+    # @param include_obsolete [Boolean] Whether include obsolete entries or not
     # @return [Array]
-    def entries(include_cached=false)
-      if include_cached
+    def entries(include_obsolete=false)
+      if include_obsolete
         @entries
       else
         find_all do |entry|
-          !entry.cached?
+          !entry.obsolete?
         end
       end
     end
@@ -78,17 +78,17 @@ module PoParser
       end
     end
 
-    # Finds all cached entries
+    # Finds all obsolete entries
     #
-    # @return [Array] an array of cached entries
-    def cached
+    # @return [Array] an array of obsolete entries
+    def obsolete
       find_all do |entry|
-        entry.cached?
+        entry.obsolete?
       end
     end
-    alias_method :obsolete, :cached
+    alias_method :obsolete, :obsolete
 
-    # Count of all entries without counting cached entries
+    # Count of all entries without counting obsolete entries
     #
     # @return [String]
     def size

data/lib/poparser/tokenizer.rb

@@ -2,33 +2,21 @@ module PoParser
   # Feed each block of PO file to Parser.
   class Tokenizer
     def initialize
-      @parser = Parser.new
-      @po     = Po.new
+      @po = Po.new
     end
 
     def extract_entries(path)
       @po.path = path
-      block = ''
-      File.open(path, 'r') do |f|
-        f.each_line do |line|
-          if line.match(/^\n$/)
-            @po << parse_block(block) if block != ''
-            block = ''
-          elsif f.eof?
-            block += line
-            @po << parse_block(block)
-          else
-            block += line
-          end
-        end
+      File.open(path, 'r').each_line("\n\n") do |block|
+        block.strip!
+        @po << parse_block(block) if block != ''
       end
       @po
     end
 
   private
     def parse_block(block)
-      parsed_hash = @parser.parse(block)
-      Transformer.new.transform(parsed_hash)
+      hash = SimplePoParser.parse_message(block)
     end
   end
 end

data/lib/poparser/version.rb

@@ -1,3 +1,3 @@
 module PoParser
-  VERSION = "2.0.1"
+  VERSION = "3.0.0"
 end

data/lib/poparser.rb

@@ -1,17 +1,15 @@
 # External Libs
-require 'parslet'
+require 'simple_po_parser'
 
 # Local files
-require 'poparser/constants'
-require 'poparser/parser'
-require 'poparser/transformer'
-require 'poparser/tokenizer'
-require 'poparser/comment'
-require 'poparser/message'
-require 'poparser/header'
-require 'poparser/entry'
-require 'poparser/po'
-require 'poparser/version'
+require_relative 'poparser/constants'
+require_relative 'poparser/tokenizer'
+require_relative 'poparser/comment'
+require_relative 'poparser/message'
+require_relative 'poparser/header'
+require_relative 'poparser/entry'
+require_relative 'poparser/po'
+require_relative 'poparser/version'
 
 module PoParser
   class << self

data/poparser.gemspec

@@ -19,7 +19,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   # Runtime deps
-  spec.add_runtime_dependency "parslet", "~> 1.7"
+  spec.add_runtime_dependency "simple_po_parser", "~> 1.1"
 
   # Development deps
   spec.add_development_dependency "bundler", ">= 0"

data/spec/poparser/comment_spec.rb

@@ -10,7 +10,9 @@ describe PoParser::Comment do
 
   it 'converts array of same comment to string' do
     comment = PoParser::Comment.new(:translator_comment, ["this is a line", "this is another line"])
-    result = "# this is a line\n# this is another line\n"
-    expect(comment.to_s(true)).to eq(result)
+    result = "this is a line\nthis is another line"
+    result_with_label = "# this is a line\n# this is another line\n"
+    expect(comment.to_s).to eq(result)
+    expect(comment.to_s(true)).to eq(result_with_label)
   end
 end

data/spec/poparser/entry_spec.rb

@@ -8,8 +8,7 @@ describe PoParser::Entry do
   end
 
   let(:labels) do
-    [:reference, :refrence, :extracted_comment, :flag, :previous_untraslated_string,
-      :translator_comment, :msgid, :msgid_plural, :msgstr, :msgctxt]
+    PoParser::LABELS + [:refrence]  # backward typos
   end
 
   it 'should respond to labels' do
@@ -90,16 +89,34 @@ describe PoParser::Entry do
     end
   end
 
-  context 'Cached' do
+  context 'Previous' do
+    it 'should be able to show content of previous_msgid' do
+      @entry.previous_msgid = 'Hello'
+      result = "Hello"
+      result_with_label = "#| msgid \"Hello\"\n"
+      expect(@entry.previous_msgid.to_s).to eq result
+      expect(@entry.previous_msgid.to_s(true)).to eq result_with_label
+    end
+
+    it 'convert multiline entries to string' do
+      @entry.previous_msgid = ['first line\n', 'second line']
+      result = "first line\\nsecond line"
+      result_with_label = "#| msgid \"\"\n#| \"first line\\n\"\n#| \"second line\"\n"
+      expect(@entry.previous_msgid.to_s).to eq result
+      expect(@entry.previous_msgid.to_s(true)).to eq result_with_label
+    end
+  end
+
+  context 'obsolete' do
     before do
       @entry = PoParser::Entry.new
-      @entry.cached = '#~ msgid "a cached entry"'
+      @entry.obsolete = '#~ msgid "a obsolete entry"'
       @entry.flag = 'Fuzzy'
     end
 
-    it 'checks for chached entries' do
-      expect(@entry.cached?).to be_truthy
+    it 'checks for obsolete entries' do
       expect(@entry.obsolete?).to be_truthy
+      expect(@entry.cached?).to be_truthy
     end
 
     it 'shouldn be counted as untranslated' do

data/spec/poparser/po_spec.rb

@@ -39,11 +39,11 @@ describe PoParser::Po do
     expect(@po.untranslated.size).to eq 2
   end
 
-  it 'returns all cached strings' do
+  it 'returns all obsolete strings' do
     entry2, entry3 = entry.dup, entry.dup
-    [entry2, entry3].each { |en| en[:cached] = 'test' }
+    [entry2, entry3].each { |en| en[:obsolete] = 'test' }
     @po << [entry, entry2, entry3]
-    expect(@po.cached.size).to eq 2
+    expect(@po.obsolete.size).to eq 2
   end
 
   it 'shows stats' do
@@ -58,10 +58,10 @@ describe PoParser::Po do
     expect(result[:fuzzy]).to eq 25
   end
 
-  it 'shouldn\'t count cached entries' do
+  it 'shouldn\'t count obsolete entries' do
     @po << entry
-    cached = { cached: 'sth', flag: 'Fuzzy' }
-    @po << cached
+    obsolete = { obsolete: 'sth', flag: 'Fuzzy' }
+    @po << obsolete
     expect(@po.size).to eq(1)
   end
 

data/spec/poparser/poparser_spec.rb

@@ -3,8 +3,134 @@ require "spec_helper"
 
 describe PoParser do
   let(:po_file) { Pathname.new('spec/poparser/fixtures/tokenizer.po').realpath }
+  let(:header_fixture) { Pathname.new('spec/poparser/fixtures/header.po').realpath }
+  let(:multiline_fixture) { Pathname.new('spec/poparser/fixtures/multiline.po').realpath }
+  let(:plural_fixture) { Pathname.new('spec/poparser/fixtures/plural.po').realpath }
+  let(:test_fixture) { Pathname.new('spec/poparser/fixtures/test.po').realpath }
 
   it 'parses a file' do
     expect(PoParser.parse(po_file)).to be_a_kind_of PoParser::Po
   end
+
+  it 'correclty parses header fixture' do
+    entry = PoParser::Entry.new(
+    {
+      :translator_comment => [
+        "Arash Mousavi <mousavi.arash@gmail.com>, 2014.",
+        ""],
+      :flag => "fuzzy",
+      :msgid => "",
+      :msgstr => [
+        "",
+        "Project-Id-Version: damned-lies master\\n",
+        "Report-Msgid-Bugs-To: \\n"]
+      })
+    expected_result = PoParser::Header.new(entry)
+    expect(PoParser.parse(header_fixture).header.inspect).to eql(expected_result.inspect)
+  end
+
+  it 'correclty parses multiline fixture' do
+    expected_result = PoParser::Entry.new(
+    {
+      :msgid => [
+        "",
+        "first",
+        "second"],
+      :msgstr => [
+        "",
+        "aval",
+        "dovom"]
+      })
+    expect(PoParser.parse(multiline_fixture).entries[0].inspect).to eq(expected_result.inspect)
+  end
+
+  it 'correclty parses plural fixture' do
+    expected_result = PoParser::Entry.new(
+    {
+      :msgid => " including <a href=\\\"%(img_url)s\\\">%(stats)s image</a>",
+      :msgid_plural => " including <a href=\\\"%(img_url)s\\\">%(stats)s images</a>",
+      "msgstr[0]" => [
+        "",
+        "sad ads fdsaf ds fdfs dsa "
+      ],
+      "msgstr[1]" => [
+        "",
+        "sad ads fdsaf ds fdfs dsa "
+      ]
+      })
+    expect(PoParser.parse(plural_fixture).entries[0].inspect).to eq(expected_result.inspect)
+  end
+
+  it 'correclty parses test fixture' do
+    expected_result = PoParser::Po.new
+    expected_result << { :translator_comment => [
+        "Persian translation for damned-lies.",
+        "Copyright (C) 2012 damned-lies's COPYRIGHT HOLDER",
+        "This file is distributed under the same license as the damned-lies package.",
+        "Arash Mousavi <mousavi.arash@gmail.com>, 2014.",
+        ""
+      ],
+      :msgid => "",
+      :msgstr => [
+        "",
+        "Project-Id-Version: damned-lies master\\n",
+        "Report-Msgid-Bugs-To: \\n",
+        "POT-Creation-Date: 2012-05-04 12:56+0000\\n",
+        "PO-Revision-Date: 2014-05-15 22:24+0330\\n",
+        "Last-Translator: Arash Mousavi <mousavi.arash@gmail.com>\\n",
+        "Language-Team: Persian <fa@li.org>\\n",
+        "MIME-Version: 1.0\\n",
+        "Content-Type: text/plain; charset=UTF-8\\n",
+        "Content-Transfer-Encoding: 8bit\\n",
+        "Plural-Forms: nplurals=1; plural=0;\\n",
+        "X-Generator: Poedit 1.6.4\\n"
+      ]
+    }
+    expected_result << {
+      :reference => "database-content.py:1 database-content.py:129 settings.py:52",
+      :msgid => "Afrikaans",
+      :msgstr => "آفریقایی"
+    }
+    expected_result << { :reference => "templates/vertimus/vertimus_detail.html:105",
+      :flag => "python-format",
+      :msgid => " including <a href=\\\"%(img_url)s\\\">%(stats)s image</a>",
+      :msgid_plural => " including <a href=\\\"%(img_url)s\\\">%(stats)s images</a>",
+      "msgstr[0]" => "",
+      "msgstr[1]" => ""
+    }
+    expected_result << {
+      :reference => "templates/vertimus/vertimus_detail.html:136 vertimus/forms.py:79",
+      :msgid => "Invalid action. Someone probably posted another action just before you.",
+      :msgstr => [
+        "",
+        "فعالیت نامعتبر. شاید یک نفر دیگر دقیقا قبل از شما یک فعالیت دیگر ارسال کرده ",
+        "است."
+      ]
+    }
+    expected_result << { :reference => "vertimus/models.py:470",
+      :flag => "python-format",
+      :previous_msgid => [
+        "",
+        "Hello,\\n",
+        "\\n",
+        "The new state of %(module)s - %(branch)s - %(domain)s (%(language)s) is ",
+        "now '%(new_state)s'.\\n",
+        "%(url)s\\n",
+        "\\n"
+      ],
+      :msgid => [
+        "",
+        "The new state of %(module)s - %(branch)s - %(domain)s (%(language)s) is now ",
+        "'%(new_state)s'."
+      ],
+      :msgstr => [
+        "",
+        "وضعیت جدید %(module)s - %(branch)s - %(domain)s (%(language)s) هم‌اکنون ",
+        "«%(new_state)s» است."
+      ]
+    }
+    allow_any_instance_of(PoParser::Header).to receive(:puts)
+    expect(PoParser.parse(test_fixture).to_s).to eq(expected_result.to_s)
+  end
+
 end