PoParser 0.1.0

data/lib/poparser/tokenizer.rb

@@ -0,0 +1,36 @@
+require 'ap'
+
+module PoParser
+  # Feed each block of PO file to Parser.
+  class Tokenizer
+    def initialize
+      @parser = Parser.new
+      @po     = Po.new
+    end
+
+    def extract_entries(path)
+      @po.path = path
+      block = ''
+      File.open(path, 'r') do |f|
+        f.each_line do |line|
+          if line.match(/^\n$/)
+            @po << parse_block(block)
+            block = ''
+          elsif f.eof?
+            block += line
+            @po << parse_block(block)
+          else
+            block += line
+          end
+        end
+      end
+      @po
+    end
+
+  private
+    def parse_block(block)
+      parsed_hash = @parser.parse(block)
+      Transformer.new.transform(parsed_hash)
+    end
+  end
+end

data/lib/poparser/transformer.rb

@@ -0,0 +1,59 @@
+module PoParser
+  # Converts the array returned from {Parser} to a useable hash
+  class Transformer
+    def initialize
+      @hash = {}
+      super
+    end
+
+    def transform(obj)
+      apply_transforms(obj).each do |hash|
+        merge(hash)
+      end
+      @hash
+    end
+
+  private
+    # @Note: There was a problem applying all rules together. I don't know
+    #   in what order Parslet run rules, but it's not in order. I end up
+    #   making to seperate transform and feed one output to the other.
+    def first_transform
+      Parslet::Transform.new do
+        rule(:msgstr_plural => subtree(:plural)) do
+          if plural.is_a? Array
+            { "msgstr\[#{plural[0][:plural_id]}\]".to_sym => plural }
+          else
+            { "msgstr\[#{plural[:plural_id]}\]".to_sym => plural }
+          end
+        end
+
+        rule(:text => simple(:txt)) { txt.to_s.chomp }
+      end
+    end
+
+    def second_transform
+      Parslet::Transform.new do
+        rule(:plural_id => simple(:id), :text => simple(:txt)) { txt }
+      end
+    end
+
+    def apply_transforms(hash)
+      first  = first_transform.apply(hash)
+      second_transform.apply(first)
+    end
+
+    # Merges two hashed together. If both hashes have common keys it
+    # will create an array of them
+    # 
+    # @return [Hash]
+    def merge(newh)
+      @hash.merge!(newh) do |key, oldval, newval|
+        if oldval.is_a? Array
+          oldval << newval
+        else
+          Array.new [oldval, newval]
+        end
+      end
+    end
+  end
+end

data/lib/poparser/version.rb

@@ -0,0 +1,3 @@
+module PoParser
+  VERSION = "0.1.0"
+end

data/lib/poparser.rb

@@ -0,0 +1,21 @@
+# External Libs
+require 'parslet'
+
+# Local files
+require 'poparser/constants'
+require 'poparser/parser'
+require 'poparser/transformer'
+require 'poparser/tokenizer'
+require 'poparser/comment'
+require 'poparser/message'
+require 'poparser/entry'
+require 'poparser/po'
+require 'poparser/version'
+
+module PoParser
+  class << self
+    def parse(path)
+      Tokenizer.new.extract_entries(path)
+    end
+  end
+end

data/poparser.gemspec

@@ -0,0 +1,31 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'poparser/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "PoParser"
+  spec.version       = PoParser::VERSION
+  spec.authors       = ["Arash Mousavi"]
+  spec.email         = ["mousavi.arash@gmail.com"]
+  spec.summary       = %q{A PO file parser, editor and generator.}
+  spec.description   = %q{A PO file parser, editor and generator. PO files are translation files generated by GNU/Gettext tool.}
+  spec.homepage      = "http://github.com/arashm/poparser"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^spec/})
+  spec.require_paths = ["lib"]
+
+  # Runtime deps
+  spec.add_runtime_dependency "parslet"
+
+  # Development deps
+  spec.add_development_dependency "bundler", "~> 1.5"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec", "~> 2.14"
+  spec.add_development_dependency "guard-rspec"
+  spec.add_development_dependency "pry-debugger"
+  spec.add_development_dependency "awesome_print"
+end

data/spec/poparser/comment_spec.rb

@@ -0,0 +1,16 @@
+# encoding: utf-8
+require 'spec_helper'
+
+describe PoParser::Comment do
+  it 'converts the comment to string' do
+    comment = PoParser::Comment.new(:translator_comment, "this is a line")
+    result = "# this is a line\n"
+    expect(comment.to_s(true)).to eq(result)
+  end
+
+  it 'converts array of same comment to string' do
+    comment = PoParser::Comment.new(:translator_comment, ["this is a line", "this is another line"])
+    result = "# this is a line\n# this is another line\n"
+    expect(comment.to_s(true)).to eq(result)
+  end
+end

data/spec/poparser/entry_spec.rb

@@ -0,0 +1,85 @@
+# encoding: utf-8
+require 'spec_helper'
+
+describe PoParser::Entry do
+  before(:each) do
+    @entry = PoParser::Entry.new
+  end
+
+  let(:labels) do
+    [:refrence, :extracted_comment, :flag, :previous_untraslated_string,
+      :translator_comment, :msgid, :msgid_plural, :msgstr, :msgctxt]
+  end
+
+  it 'should respond to labels' do
+    labels.each do |label|
+      @entry.should respond_to label
+    end
+  end
+
+  it 'should show a hash presentation of a entry' do
+    @entry.msgid = 'string'
+    @entry.msgstr = 'reshte'
+    expect(@entry.to_h).to eq({:msgid=>"string", :msgstr=>"reshte"})
+  end
+
+  it 'should translate the entry' do
+    @entry.translate ('this entry is translated')
+    expect(@entry.msgstr.to_s).to eq 'this entry is translated'
+  end
+
+  it 'checks if the entry is translated' do
+    expect(@entry.translated?).to be_false
+    @entry.translate ''
+    expect(@entry.translated?).to be_false
+    @entry.translate 'translated'
+    expect(@entry.complete?).to be_true
+  end
+
+  context 'Plural' do
+    it 'returns false if it\'s not plural' do
+      expect(@entry.plural?).to be_false
+    end
+
+    it 'returns true if it\'s plural' do
+      @entry.msgid_plural = 'sth'
+      expect(@entry.plural?).to be_true
+    end
+  end
+
+  context 'Flags' do
+    it 'should check if a entry is fuzzy' do
+      expect(@entry.fuzzy?).to be_false
+      @entry.flag = 'fuzzy'
+      expect(@entry.fuzzy?).to be_true
+    end
+
+    it 'should flag a entry as fuzzy' do
+      expect(@entry.flag_as_fuzzy).to be_true
+      expect(@entry.flag).to eq('fuzzy')
+    end
+
+    it 'should be able to set a custome flag' do
+      expect(@entry.flag_as 'python-format').to be_true
+      expect(@entry.flag).to eq('python-format')
+    end
+  end
+
+  context 'Convertion to string' do
+    it 'should be able to show string representaion of entries' do
+      @entry.flag = 'fuzzy'
+      @entry.msgid = 'string'
+      @entry.msgstr = 'reshte'
+      result = "#, fuzzy\nmsgid \"string\"\nmsgstr \"reshte\"\n"
+      expect(@entry.to_s).to eq result
+    end
+
+    it 'convert multiline entries to string' do
+      @entry.flag = 'fuzzy'
+      @entry.msgid = ['first line', 'second line']
+      @entry.msgstr = ['first line', 'second line']
+      result = "#, fuzzy\nmsgid \"\"\n\"first line\"\n\"second line\"\nmsgstr \"\"\n\"first line\"\n\"second line\"\n"
+      expect(@entry.to_s).to eq(result)
+    end
+  end
+end

data/spec/poparser/fixtures/multiline.po

@@ -0,0 +1,6 @@
+msgid ""
+"first"
+"second"
+msgstr ""
+"aval"
+"dovom"

data/spec/poparser/fixtures/plural.po

@@ -0,0 +1,6 @@
+msgid " including <a href=\"%(img_url)s\">%(stats)s image</a>"
+msgid_plural " including <a href=\"%(img_url)s\">%(stats)s images</a>"
+msgstr[0] ""
+"sad ads fdsaf ds fdfs dsa "
+msgstr[1] ""
+"sad ads fdsaf ds fdfs dsa "

data/spec/poparser/fixtures/tokenizer.po

@@ -0,0 +1,7 @@
+#: templates:105
+msgid "Afrikaans"
+msgstr "آفریقایی"
+
+#, fuzzy
+msgid "Afrikaans"
+msgstr "آفریقایی"

data/spec/poparser/message_spec.rb

@@ -0,0 +1,34 @@
+# encoding: utf-8
+require 'spec_helper'
+
+describe PoParser::Message do
+  it 'converts the message to string' do
+    message = PoParser::Message.new(:msgid, "this is a line")
+    result = "msgid \"this is a line\"\n"
+    expect(message.to_s(true)).to eq(result)
+  end
+
+  it 'converts array of same message to string' do
+    message = PoParser::Message.new(:msgid, ["this is a line", "this is another line"])
+    result = "msgid \"\"\n\"this is a line\"\n\"this is another line\"\n"
+    expect(message.to_s(true)).to eq(result)
+  end
+
+  it 'shows one line string for multiline entries' do
+    message = PoParser::Message.new(:msgid, ["", "this is a line ", "this is another line"])
+    result = "this is a line this is another line"
+    expect(message.str).to eq result
+  end
+
+  it 'converts plural msgstr correctly' do
+    message = PoParser::Message.new(:"msgstr[0]", "this is a line")
+    result = "msgstr[0] \"this is a line\"\n"
+    expect(message.to_s(true)).to eq(result)
+  end
+
+  it 'converts multiline plural msgstr correctly' do
+    message = PoParser::Message.new(:"msgstr[0]", ["this is a line", "this is another line"])
+    result = "msgstr[0] \"\"\n\"this is a line\"\n\"this is another line\"\n"
+    expect(message.to_s(true)).to eq(result)
+  end
+end

data/spec/poparser/parser_spec.rb

@@ -0,0 +1,69 @@
+#encoding: utf-8
+require "spec_helper"
+
+describe PoParser::Parser do
+  let(:po) { PoParser::Parser.new }
+
+  context(:comments) do
+
+    let(:tc)  { po.translator_comment }
+    let(:rc)  { po.refrence }
+    let(:ec)  { po.extracted_comment }
+    let(:fc)  { po.flag }
+    let(:pusc){ po.previous_untraslated_string }
+
+    it 'parses the translator comment' do
+      tc.should parse("# Persian translation for damned-lies 123123\n")
+      tc.should parse("# Copyright (C) 2012 damned-lies's COPYRIGHT HOLDER\n")
+      tc.should parse("# Arash Mousavi <mousavi.arash@gmail.com>, 2014.\n")
+    end
+
+    it 'parses refrence comment' do
+      rc.should parse("#: database-content.py:1 database-content.py:129 settings.py:52\n")
+    end
+
+    it 'parses extracted_comment' do
+      ec.should parse("#. database-content.py:1 database-content.py:129 settings.py:52\n")
+    end
+
+    it 'parses flag_comment' do
+      fc.should parse("#, python-format\n")
+    end
+
+    it 'parses previous_untraslated_string' do
+      pusc.should parse("#| msgid \"\"\n")
+      pusc.should parse("#| \"Hello,\\n\"\n")
+      pusc.should parse("#| \"The new state of %(module)s - %(branch)s - %(domain)s (%(language)s) is \"\n")
+    end
+
+  end
+
+  context 'Entries' do
+    let(:msgid) { po.msgid }
+    let(:msgstr){ po.msgstr }
+    let(:pofile){ Pathname.new('spec/poparser/fixtures/multiline.po').realpath }
+
+    it 'parses msgid' do
+      msgid.should parse "msgid \"The new state of %(module)s - %(branch)s - %(domain)s (%(language)s) is now \"\n"
+      msgid.should parse "msgid \"The new \"state\" of %(module)s - %(branch)s - %(domain)s (%(language)s) is now \"\n"
+    end
+
+    it 'parses msgstr' do
+      msgstr.should parse "msgstr \"The new state of %(module)s - %(branch)s - %(domain)s (%(language)s) is now \"\n"
+      msgstr.should parse "msgstr \"فعالیت نامعتبر. شاید یک نفر دیگر دقیقا قبل از شما یک فعالیت دیگر ارسال کرده ۱۲۳۱۲۳۱safda \"\n"
+    end
+
+    it 'parses multiline entries' do
+      data = pofile.read
+      result = [{:msgid=>[{:text=>""}, {:text=>"first"}, {:text=>"second"}]}, {:msgstr=>[{:text=>""}, {:text=>"aval"}, {:text=>"dovom"}]}]
+      expect(po.parse data).to eq(result)
+    end
+
+    it 'parses plural msgstr entries' do
+      str = "msgstr[0] \"\""
+      result = [{:msgstr_plural=>{:plural_id=>"0", :text=>""}}]
+      expect(po.parse(str)).to eq(result)
+    end
+  end
+
+end

data/spec/poparser/po_spec.rb

@@ -0,0 +1,48 @@
+# encoding: utf-8
+require 'spec_helper'
+
+describe PoParser::Po do
+  let (:entry) do 
+    {
+      translator_comment: 'comment',
+      refrence: 'refrence comment',
+      msgid: 'untranslated',
+      msgstr: 'translated string'
+    }
+  end
+  
+  before(:each) do
+    @po = PoParser::Po.new
+  end
+
+  it 'should be able to add an entry to Po' do
+    # << is an alias for Po#add_entry
+    expect(@po << entry).to be_a_kind_of PoParser::Entry
+  end
+
+  it 'should be able to add multiple entries' do
+    entries = [entry, entry.dup]
+    expect(@po << entries).to be_a_kind_of Array
+  end
+
+  it 'returns all fuzzy entries' do
+    entry2, entry3 = entry.dup, entry.dup
+    [entry2, entry3].each { |en| en[:flag] = 'fuzzy' }
+    @po << [entry, entry2, entry3]
+    expect(@po.fuzzy.size).to eq 2
+  end
+
+  it 'returns all untraslated strings' do
+    entry2, entry3 = entry.dup, entry.dup
+    [entry2, entry3].each { |en| en[:msgstr] = '' }
+    @po << [entry, entry2, entry3]
+    expect(@po.untranslated.size).to eq 2
+  end
+
+  it 'shows stats' do
+    entry2, entry3 = entry.dup, entry.dup
+    [entry2, entry3].each { |en| en[:msgstr] = '' }
+    @po << [entry, entry2, entry3]
+    ap @po.stats
+  end
+end

data/spec/poparser/poparser_spec.rb

@@ -0,0 +1,10 @@
+# encoding: utf-8
+require "spec_helper"
+
+describe PoParser do
+  let(:po_file) { Pathname.new('spec/poparser/fixtures/tokenizer.po').realpath }
+
+  it 'parses a file' do
+    expect(PoParser.parse(po_file)).to be_a_kind_of PoParser::Po
+  end
+end

data/spec/poparser/test.po

@@ -0,0 +1,51 @@
+# Persian translation for damned-lies.
+# Copyright (C) 2012 damned-lies's COPYRIGHT HOLDER
+# This file is distributed under the same license as the damned-lies package.
+# Arash Mousavi <mousavi.arash@gmail.com>, 2014.
+#
+msgid ""
+msgstr ""
+"Project-Id-Version: damned-lies master\n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2012-05-04 12:56+0000\n"
+"PO-Revision-Date: 2014-05-15 22:24+0330\n"
+"Last-Translator: Arash Mousavi <mousavi.arash@gmail.com>\n"
+"Language-Team: Persian <fa@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Plural-Forms: nplurals=1; plural=0;\n"
+"X-Generator: Poedit 1.6.4\n"
+
+#: database-content.py:1 database-content.py:129 settings.py:52
+msgid "Afrikaans"
+msgstr "آفریقایی"
+
+#: templates/vertimus/vertimus_detail.html:105
+#, python-format
+msgid " including <a href=\"%(img_url)s\">%(stats)s image</a>"
+msgid_plural " including <a href=\"%(img_url)s\">%(stats)s images</a>"
+msgstr[0] ""
+msgstr[1] ""
+
+#: templates/vertimus/vertimus_detail.html:136 vertimus/forms.py:79
+msgid "Invalid action. Someone probably posted another action just before you."
+msgstr ""
+"فعالیت نامعتبر. شاید یک نفر دیگر دقیقا قبل از شما یک فعالیت دیگر ارسال کرده "
+"است."
+
+#: vertimus/models.py:470
+#, python-format
+#| msgid ""
+#| "Hello,\n"
+#| "\n"
+#| "The new state of %(module)s - %(branch)s - %(domain)s (%(language)s) is "
+#| "now '%(new_state)s'.\n"
+#| "%(url)s\n"
+#| "\n"
+msgid ""
+"The new state of %(module)s - %(branch)s - %(domain)s (%(language)s) is now "
+"'%(new_state)s'."
+msgstr ""
+"وضعیت جدید %(module)s - %(branch)s - %(domain)s (%(language)s) هم‌اکنون "
+"«%(new_state)s» است."

data/spec/poparser/tokenizer_spec.rb

@@ -0,0 +1,14 @@
+# encoding: utf-8
+require "spec_helper.rb"
+
+describe PoParser::Tokenizer do
+  let(:token)  { PoParser::Tokenizer.new }
+  let(:po_file){ Pathname.new('spec/poparser/fixtures/tokenizer.po').realpath }
+  let(:result) { [{:refrence=>"templates:105", :msgid=>"Afrikaans", :msgstr=>"آفریقایی"}, {:flag=>"fuzzy", :msgid=>"Afrikaans", :msgstr=>"آفریقایی" }] }
+
+  it 'should be able to extracts entries' do
+    expect(
+      token.extract_entries(po_file).to_h
+    ).to eq(result)
+  end
+end

data/spec/poparser/transformer_spec.rb

@@ -0,0 +1,24 @@
+# encoding: utf-8
+require "spec_helper"
+
+describe PoParser::Transformer do
+  let(:trans){ PoParser::Transformer.new }
+
+  it 'transforms the returned array from parslet to a usable hash' do
+    parslet_array = [{:translator_comment=>"Persian translation\n"}, {:translator_comment=>"Copyright\n"}, {:msgid=>"\"test\"\n"}]
+    transformed_hash = {:translator_comment=>["Persian translation\n", "Copyright\n"], :msgid=>"\"test\"\n"}
+    expect(trans.transform(parslet_array)).to eq(transformed_hash)
+  end
+
+  it 'transforms plural msgstr forms correctly' do
+    data = [{:msgstr_plural=>{:plural_id=>"0", :text=>"this is a txt"}}]
+    result = { :'msgstr[0]' => "this is a txt" }
+    expect(trans.transform(data)).to eq(result)
+  end
+
+  it 'transforms multiline plural msgstr forms correctly' do
+    data = [{:msgstr_plural=>[{:plural_id=>"0", :text=>"this is a txt"}, {:text => 'some text'}]}]
+    result = { :'msgstr[0]' => ["this is a txt", "some text"] }
+    expect(trans.transform(data)).to eq(result)
+  end
+end

data/spec/poparser/version_spec.rb

@@ -0,0 +1,8 @@
+require 'spec_helper'
+
+# Just as test to ensure that test suit is working correctly
+describe 'Version' do
+  it 'shows the version correctly' do
+    expect(PoParser::VERSION).to eq('0.0.1')
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,22 @@
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# Require this file using `require "spec_helper"` to ensure that it is only
+# loaded once.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+require 'poparser'
+require 'parslet/rig/rspec'
+require 'coveralls'
+Coveralls.wear!
+
+RSpec.configure do |config|
+  config.treat_symbols_as_metadata_keys_with_true_values = true
+  config.run_all_when_everything_filtered = true
+  config.filter_run :focus
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = 'random'
+end