simple_po_parser 1.1.3 → 1.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 145b9c1db6de2e03d5eab7a4b04297cb418af047
-  data.tar.gz: f6cbc4211bc7bb482794c7d620585f8cab1dddba
+SHA256:
+  metadata.gz: 7b32bc4f19bd4e4967e9395bb2869e377e75f63bac97a0820ced810d03191b73
+  data.tar.gz: 24bb37816cc545d65b2840ad96628bc17b4f7c994eeabe8d0cb43307302c89d1
 SHA512:
-  metadata.gz: 60479f02418e32487fa9151b8c4a44cc38e3a701174d4201d4335668e4fb08674810ad6a759e2e3aed0979e457a17ed430ea90ed12ef3386e34069539e51407f
-  data.tar.gz: 16984994b33283f09fe363d204f7830351ccd11fe9293ab32c380c534970d7c76d4d36bc69a3fe917c788c3268aaeeef5408b7a348e0a047cfd43a3c5525ece2
+  metadata.gz: 7469e3b963f89d2f35a29cef3d3c7f4fe3bb0e78e9cbbad9fd6e8f9a800a44b804c1193abfefa2685027d773fdf0c86faaa6abab1d0d84df35ca068e6b0eb209
+  data.tar.gz: 1283c455623d2e8c387e8b57d963dffd11a244dfdd4c4049f10444c4cf1fffa3adc5c9c59b83c8c463ac0167c1bd410741a1c17bd06b70f50e7822d8e8fb3bf2

data/.gitattributes

@@ -0,0 +1 @@
+spec/simple_po_parser/fixtures/crlf_encoded.po binary

data/CHANGELOG.md

@@ -1,15 +1,35 @@
-Version 1.1.0
+Changelog
 =============
 
+## Version 1.1.6
+  * fixes parsing of multiline strings that have content on the first line. Fixes issue #3, added via PR #4 by @mgruner
+
+## Version 1.1.5
+ * added support for windows CRLF line endings. Fixes issue #2
+ * Note: CRLF support is enabled if the first line ends in a CRLF and reduces performance by about 50%. Performance for files only using \n is not affected. Files not using \r\n in the first line but somewhere else in the file might trigger errors.
+
+## Version 1.1.4
+  * see 1.1.5. **Shouldn't be used**.
+
+## Version 1.1.3
+
+  * merged PR#1 from @christopherstat for UTF-8 support of legacy ruby versions
+
+## Version 1.1.2
+  * Made the parser thread-safe
+
+## Version 1.1.2
+  * Made the parser thread-safe
+
+## Version 1.1.0
+
   * for line types only parsed once the parser returns a string instead of an array with one string
 
-Version 1.0.1
-=============
+## Version 1.0.1
 
   * added specs
   * fixed minor parser errors
 
-Version 1.0.0
-=============
+## Version 1.0.0
 
   * initial release

data/Rakefile

@@ -20,15 +20,15 @@ namespace :parser do
   require 'benchmark'
   require 'simple_po_parser'
 
-  desc "Benchmark of 10 full PoParser runs of test/benchmark.po"
+  desc "Benchmark of 1000 full PoParser runs of test/benchmark.po"
   task "benchmark" do
     pofile = File.expand_path("test/benchmark.po", __dir__)
     Benchmark.bmbm do |x|
-      x.report("Parser:") {10.times { SimplePoParser.parse(pofile) }}
+      x.report("Parser:") {1000.times { SimplePoParser.parse(pofile) }}
     end
   end
 
-  desc "Generate 5 random PO files with 100 to 500 messages and benchmark each full PoParser run"
+  desc "Generate 5 random PO files with 1000 to 5000 messages and benchmark each full PoParser run"
   task 'five_random_po_full' do
     include Benchmark
     require_relative 'spec/utils/random_pofile_generator'
@@ -37,7 +37,7 @@ namespace :parser do
       total = nil
       total_length = 0
       for i in 0..5 do
-        length = (Random.new.rand * 400.0 + 100).to_i
+        length = (Random.new.rand * 4000.0 + 1000).to_i
         total_length += length
         puts "Benchmarking file of length #{length}"
         SimplePoParser::RandomPoFileGenerator.generate_file(pofile, length)

data/lib/simple_po_parser/parser.rb

@@ -109,7 +109,7 @@ module SimplePoParser
           skip_whitespace
           text = message_line
           add_result(:msgctxt, text)
-          message_multiline(:msgctxt) if text.empty?
+          message_multiline(:msgctxt) if @scanner.peek(1) == '"'
         end
         msgid
       rescue PoSyntaxError => pe
@@ -127,7 +127,7 @@ module SimplePoParser
           skip_whitespace
           text = message_line
           add_result(:msgid, text)
-          message_multiline(:msgid) if text.empty?
+          message_multiline(:msgid) if @scanner.peek(1) == '"'
           if msgid_plural
             msgstr_plural
           else
@@ -155,7 +155,7 @@ module SimplePoParser
           skip_whitespace
           text = message_line
           add_result(:msgid_plural, text)
-          message_multiline(:msgid_plural) if text.empty?
+          message_multiline(:msgid_plural) if @scanner.peek(1) == '"'
           true
         else
           false
@@ -174,7 +174,7 @@ module SimplePoParser
           skip_whitespace
           text = message_line
           add_result(:msgstr, text)
-          message_multiline(:msgstr) if text.empty?
+          message_multiline(:msgstr) if @scanner.peek(1) == '"'
           skip_whitespace
           raise PoSyntaxError, "Unexpected content after expected message end #{@scanner.peek(10).inspect}" unless @scanner.eos?
         else
@@ -202,7 +202,7 @@ module SimplePoParser
           skip_whitespace
           text = message_line
           add_result(msgstr_key, text)
-          message_multiline(msgstr_key) if text.empty?
+          message_multiline(msgstr_key) if @scanner.peek(1) == '"'
           msgstr_plural(num+1)
         elsif num == 0 # and msgstr_key was false
           raise PoSyntaxError, "Plural message without msgstr[0] is not allowed. Line started unexpectedly with #{@scanner.peek(10).inspect}."
@@ -238,7 +238,7 @@ module SimplePoParser
           skip_whitespace
           text = message_line
           add_result(key, text)
-          previous_multiline(key) if text.empty?
+          previous_multiline(key) if @scanner.match?(/#\|\p{Blank}*"/)
         else
           raise PoSyntaxError, "Previous comments must start with '#| msg'. #{@scanner.peek(10).inspect} unknown."
         end
@@ -265,8 +265,10 @@ module SimplePoParser
 
     # parses a multiline message
     #
-    # multiline messages are indicated by an empty content as first line and the next line
-    # starting with the double quote character
+    # Multiline messages are usually indicated by an empty string as the first line,
+    # followed by more lines starting with the double quote character.
+    #
+    # However, according to the PO file standard, the first line can also contain content.
     def message_multiline(key)
       begin
         skip_whitespace

data/lib/simple_po_parser/tokenizer.rb

@@ -8,7 +8,15 @@ module SimplePoParser
     end
 
     def parse_file(path)
-      File.open(path, 'r').each_line("\n\n") do |block|
+      file = File.open(path, "r")
+      if(file.gets =~ /\r$/)
+        # detected windows line ending
+        file.close
+        file = File.open(path, "rt")
+      else
+        file.rewind
+      end
+      file.each_line("\n\n") do |block|
         block.strip! # dont parse empty blocks
         @messages << parse_block(block) if block != ''
       end

data/lib/simple_po_parser/version.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
 
 module SimplePoParser
-  VERSION = "1.1.3"
+  VERSION = "1.1.6"
 end

data/spec/simple_po_parser/fixtures/crlf_encoded.po

@@ -0,0 +1,51 @@
+# PO Header entry
+#
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: simple_po_parser 1\n"
+"Report-Msgid-Bugs-To: me\n"
+"POT-Creation-Date: 2012-05-04 12:56+0000\n"
+"PO-Revision-Date: 2014-05-15 22:24+0330\n"
+"Last-Translator: Dennis-Florian Herr <dennis.herr@experteer.com>\n"
+"Language-Team: English\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Plural-Forms: nplurals=1; plural=0;\n"
+
+# translator-comment
+#. extract
+#: reference1
+msgctxt "Context"
+msgid "msgid"
+msgstr "translated"
+
+# translator-comment
+#
+#. extract
+#: reference1
+#: reference2
+#, flag
+#| msgctxt "previous context"
+#| msgid ""
+#|  "multiline\n"
+#|"previous messageid"
+#| msgid_plural "previous msgid_plural"
+msgctxt "Context"
+msgid "msgid"
+msgid_plural ""
+"multiline msgid_plural\n"
+""
+msgstr[0] "msgstr 0"
+msgstr[1] ""
+"msgstr 1 multiline 1\n"
+"msgstr 1 line 2\n"
+msgstr[2] "msgstr 2"
+
+# translator-comment
+#. extract
+#: reference1
+#~ msgctxt "Context"
+#~ msgid "msgid"
+#~ msgstr "translated"

data/spec/simple_po_parser/fixtures/multiline.po

@@ -0,0 +1,10 @@
+#| msgid "multiline\n"
+#|"previous messageid"
+#|"with non-empty first line"
+msgid ""
+"multiline string "
+"with empty first line "
+"and trailing spaces"
+msgstr "multiline string"
+"with non-empty first line"
+"and no trailing spaces"

data/spec/simple_po_parser/parser_spec.rb

@@ -4,6 +4,7 @@ describe SimplePoParser::Parser do
   let (:po_header) { File.read(File.expand_path("fixtures/header.po", __dir__))}
   let(:po_complex_message) { File.read(File.expand_path("fixtures/complex_entry.po", __dir__))}
   let(:po_simple_message) { File.read(File.expand_path("fixtures/simple_entry.po", __dir__))}
+  let(:po_multiline_message) { File.read(File.expand_path("fixtures/multiline.po", __dir__))}
 
   it "parses the PO header" do
     expected_result = {
@@ -27,6 +28,15 @@ describe SimplePoParser::Parser do
     expect(SimplePoParser::Parser.new.parse(po_simple_message)).to eq(expected_result)
   end
 
+  it "parses the multiline entry as expected" do
+    expected_result = {
+      :msgid => ["", "multiline string ", "with empty first line ", "and trailing spaces"],
+      :msgstr => ["multiline string", "with non-empty first line", "and no trailing spaces"],
+      :previous_msgid => ["multiline\\n", "previous messageid", "with non-empty first line"],
+    }
+    expect(SimplePoParser::Parser.new.parse(po_multiline_message)).to eq(expected_result)
+  end
+
   it "parses the complex entry as expected" do
     expected_result = {
       :translator_comment => ["translator-comment", ""],

data/spec/simple_po_parser/simple_po_parser_spec.rb

@@ -2,12 +2,16 @@ require "spec_helper"
 
 describe SimplePoParser do
   let(:po_file) { Pathname.new('spec/simple_po_parser/fixtures/complete_file.po').realpath }
+  let(:crlf_file) { Pathname.new('spec/simple_po_parser/fixtures/crlf_encoded.po').realpath }
   let(:non_ascii_file) { Pathname.new('spec/simple_po_parser/fixtures/non_ascii_file.po').realpath }
   let(:po_complex_message) { File.read(File.expand_path("fixtures/complex_entry.po", __dir__))}
 
   it "parses a po file" do
     expect(SimplePoParser.parse(po_file)).to be_a_kind_of Array
   end
+  it "parses crlf encoded files" do
+    expect(SimplePoParser.parse(crlf_file)).to be_a_kind_of Array
+  end
 
   it "parses a non ascii po file" do
     expect(SimplePoParser.parse(non_ascii_file)).to be_a_kind_of Array

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: simple_po_parser
 version: !ruby/object:Gem::Version
-  version: 1.1.3
+  version: 1.1.6
 platform: ruby
 authors:
 - Dennis-Florian Herr
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-08-15 00:00:00.000000000 Z
+date: 2022-03-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -46,6 +46,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".gitattributes"
 - ".gitignore"
 - ".rspec"
 - ".travis.yml"
@@ -62,7 +63,9 @@ files:
 - simple_po_parser.gemspec
 - spec/simple_po_parser/fixtures/complete_file.po
 - spec/simple_po_parser/fixtures/complex_entry.po
+- spec/simple_po_parser/fixtures/crlf_encoded.po
 - spec/simple_po_parser/fixtures/header.po
+- spec/simple_po_parser/fixtures/multiline.po
 - spec/simple_po_parser/fixtures/non_ascii_file.po
 - spec/simple_po_parser/fixtures/simple_entry.po
 - spec/simple_po_parser/parser_spec.rb
@@ -74,7 +77,7 @@ homepage: http://github.com/experteer/simple_po_parser
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -89,15 +92,16 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.11
-signing_key: 
+rubygems_version: 3.2.15
+signing_key:
 specification_version: 4
 summary: A simple PO file to ruby hash parser
 test_files:
 - spec/simple_po_parser/fixtures/complete_file.po
 - spec/simple_po_parser/fixtures/complex_entry.po
+- spec/simple_po_parser/fixtures/crlf_encoded.po
 - spec/simple_po_parser/fixtures/header.po
+- spec/simple_po_parser/fixtures/multiline.po
 - spec/simple_po_parser/fixtures/non_ascii_file.po
 - spec/simple_po_parser/fixtures/simple_entry.po
 - spec/simple_po_parser/parser_spec.rb