patinfo2csv 1.0.4 → 1.0.5

data/History.txt

@@ -1,3 +1,7 @@
+=== 1.0.5 / 26.03.2012
+
+* Refactor character replacing and chapter choosing
+
 === 1.0.4 / 23.03.2012
 
 * Deleted last <br /> tag in chapter

data/lib/patinfo2csv.rb

@@ -3,13 +3,14 @@
 
 require 'version'
 require 'patinfo2csv/cli'
+require 'patinfo2csv/loader'
 require 'patinfo2csv/converter'
 
 module Patinfo2csv
-  # FIXME
+  # NOTE
   # escaped backslash and utf-8 code map
   # for patinfo.yaml(ASCII)
-  ESCAPED_STR_CODE_MAP = {
+  ESCAPED_CHAR_CODE_MAP = {
     /\\x24/      => "$",
     /\\x25/      => "%",
     /\\x26/      => "&",
@@ -26,67 +27,67 @@ module Patinfo2csv
     /\\xC2\\xBE/ => "¾",
     /\\xC2\\x[A-z0-9]{2}/ => "",
     /\\xC3\\x82/ => "Â",
-    /\\xC3\\x83/ => "Ã", 
-    /\\xC3\\x84/ => "Ä", 
-    /\\xC3\\x85/ => "Å", 
-    /\\xC3\\x86/ => "Æ", 
-    /\\xC3\\x87/ => "Ç", 
-    /\\xC3\\x88/ => "È", 
-    /\\xC3\\x89/ => "É", 
-    /\\xC3\\x8A/ => "Ê", 
-    /\\xC3\\x8B/ => "Ë", 
-    /\\xC3\\x8C/ => "Ì", 
-    /\\xC3\\x8D/ => "Í", 
-    /\\xC3\\x8E/ => "Î", 
-    /\\xC3\\x8F/ => "Ï", 
+#     /\\xC3\\x83/ => "Ã",
+    /\\xC3\\x84/ => "Ä",
+#     /\\xC3\\x85/ => "Å",
+#     /\\xC3\\x86/ => "Æ",
+    /\\xC3\\x87/ => "Ç",
+    /\\xC3\\x88/ => "È",
+    /\\xC3\\x89/ => "É",
+    /\\xC3\\x8A/ => "Ê",
+    /\\xC3\\x8B/ => "Ë",
+    /\\xC3\\x8C/ => "Ì",
+    /\\xC3\\x8D/ => "Í",
+    /\\xC3\\x8E/ => "Î",
+    /\\xC3\\x8F/ => "Ï",
     /\\xC3\\x90/ => "Ð",
-    /\\xC3\\x91/ => "Ñ", 
-    /\\xC3\\x92/ => "Ò", 
-    /\\xC3\\x93/ => "Ó", 
-    /\\xC3\\x94/ => "Ô", 
-    /\\xC3\\x95/ => "Õ", 
-    /\\xC3\\x96/ => "Ö", 
-    /\\xC3\\x97/ => "×", 
-    /\\xC3\\x98/ => "Ø", 
-    /\\xC3\\x99/ => "Ù", 
-    /\\xC3\\x9A/ => "Ú", 
-    /\\xC3\\x9B/ => "Û", 
-    /\\xC3\\x9C/ => "Ü", 
-    /\\xC3\\x9D/ => "Ý", 
-    /\\xC3\\x9E/ => "Þ", 
-    /\\xC3\\x9F/ => "ß", 
-    /\\xC3\\xA0/ => "à", 
-    /\\xC3\\xA1/ => "á", 
-    /\\xC3\\xA2/ => "â", 
-    /\\xC3\\xA3/ => "ã", 
-    /\\xC3\\xA4/ => "ä", 
-    /\\xC3\\xA5/ => "å", 
-    /\\xC3\\xA6/ => "æ", 
-    /\\xC3\\xA7/ => "ç", 
-    /\\xC3\\xA8/ => "è", 
-    /\\xC3\\xA9/ => "é", 
-    /\\xC3\\xAA/ => "ê", 
-    /\\xC3\\xAB/ => "ë", 
-    /\\xC3\\xAC/ => "ì", 
-    /\\xC3\\xAD/ => "í", 
-    /\\xC3\\xAE/ => "î", 
-    /\\xC3\\xAF/ => "ï", 
-    /\\xC3\\xB0/ => "ð", 
-    /\\xC3\\xB1/ => "ñ", 
-    /\\xC3\\xB2/ => "ò", 
-    /\\xC3\\xB3/ => "ó", 
-    /\\xC3\\xB4/ => "ô", 
-    /\\xC3\\xB5/ => "õ", 
-    /\\xC3\\xB6/ => "ö", 
-    /\\xC3\\xB7/ => "÷", 
-    /\\xC3\\xB8/ => "ø", 
-    /\\xC3\\xB9/ => "ù", 
-    /\\xC3\\xBA/ => "ú", 
-    /\\xC3\\xBB/ => "û", 
-    /\\xC3\\xBC/ => "ü", 
-    /\\xC3\\xBD/ => "ý", 
-    /\\xC3\\xBE/ => "þ", 
-    /\\xC3\\xBF/ => "ÿ", 
+#    /\\xC3\\x91/ => "Ñ",
+    /\\xC3\\x92/ => "Ò",
+    /\\xC3\\x93/ => "Ó",
+    /\\xC3\\x94/ => "Ô",
+#    /\\xC3\\x95/ => "Õ",
+    /\\xC3\\x96/ => "Ö",
+    /\\xC3\\x97/ => "×",
+#    /\\xC3\\x98/ => "Ø",
+    /\\xC3\\x99/ => "Ù",
+    /\\xC3\\x9A/ => "Ú",
+    /\\xC3\\x9B/ => "Û",
+    /\\xC3\\x9C/ => "Ü",
+    /\\xC3\\x9D/ => "Ý",
+#    /\\xC3\\x9E/ => "Þ",
+    /\\xC3\\x9F/ => "ß",
+    /\\xC3\\xA0/ => "à",
+    /\\xC3\\xA1/ => "á",
+    /\\xC3\\xA2/ => "â",
+#    /\\xC3\\xA3/ => "ã",
+    /\\xC3\\xA4/ => "ä",
+#    /\\xC3\\xA5/ => "å",
+#    /\\xC3\\xA6/ => "æ",
+    /\\xC3\\xA7/ => "ç",
+    /\\xC3\\xA8/ => "è",
+    /\\xC3\\xA9/ => "é",
+    /\\xC3\\xAA/ => "ê",
+    /\\xC3\\xAB/ => "ë",
+    /\\xC3\\xAC/ => "ì",
+    /\\xC3\\xAD/ => "í",
+    /\\xC3\\xAE/ => "î",
+    /\\xC3\\xAF/ => "ï",
+#    /\\xC3\\xB0/ => "ð",
+#    /\\xC3\\xB1/ => "ñ", 
+    /\\xC3\\xB2/ => "ò",
+    /\\xC3\\xB3/ => "ó",
+    /\\xC3\\xB4/ => "ô",
+#    /\\xC3\\xB5/ => "õ",
+    /\\xC3\\xB6/ => "ö",
+    /\\xC3\\xB7/ => "÷",
+#    /\\xC3\\xB8/ => "ø",
+    /\\xC3\\xB9/ => "ù",
+    /\\xC3\\xBA/ => "ú",
+    /\\xC3\\xBB/ => "û",
+    /\\xC3\\xBC/ => "ü",
+    /\\xC3\\xBD/ => "ý",
+#    /\\xC3\\xBE/ => "þ",
+    /\\xC3\\xBF/ => "ÿ",
     /\\xC3\\x[A-z0-9]{2}/ => "",
     /\\xE2\\x80\\x90/ => "‐",
     /\\xE2\\x80\\x91/ => "‑",
@@ -103,7 +104,7 @@ module Patinfo2csv
     /\\xE2\\x80\\x9E/ => "„",
     /\\xE2\\x80\\x9F/ => "‟",
     /\\xE2\\x80\\xA6/ => "…",
-    /\\xE2\\x80\\xB0/ => "‰", 
+    /\\xE2\\x80\\xB0/ => "‰",
     /\\xE2\\x80\\xB1/ => "‱",
     /\\xE2\\x80\\xB2/ => "′",
     /\\xE2\\x80\\xB3/ => "″",

data/lib/patinfo2csv/cli.rb

@@ -12,32 +12,21 @@ module Patinfo2csv
       # Param:: output_file  patinfo.csv
       # Param:: lang         (de|fr)
       def run(patinfo_yaml, code_txt, output_file, lang)
-        cv = Patinfo2csv::Converter.new()
-        cv.lang     = lang
-        cv.patinfos = self.load_yaml(patinfo_yaml)
-        cv.codes    = self.parse_txt(code_txt)
-        rows = cv.to_csv
+        start = Time.now
+        loader    = Patinfo2csv::Loader.new()
+        converter = Patinfo2csv::Converter.new()
+        converter.patinfos = loader.load_yaml(patinfo_yaml)
+        converter.lang     = lang
+        converter.codes    = self.parse_txt(code_txt)
+        rows = converter.to_csv
         self.output_rows(rows, output_file)
-        report(rows, cv.codes, lang)
-      end
-      def load_yaml(patinfo_yaml)
-        # before yaml loading, replace escaped chars
-        file = ''
-        File.open(patinfo_yaml, "r:ascii:utf-8") { |f|
-          while line = f.gets
-            ESCAPED_STR_CODE_MAP.each do |code, char|
-              line.gsub!(code, char)
-            end
-            file << line
-          end
-        }
-        fh = StringIO.new file
-        YAML.load_documents(fh) # utf-8
+        report(rows, converter.codes, lang)
+        puts "#{Time.now - start} sec."
       end
       def parse_txt(code_txt)
         codes = []
         File.open(code_txt, "r:utf-8") do |input|
-          while line=input.gets
+          while line = input.gets
             codes << line.gsub(/[^\d]/, '').chomp
           end
         end

data/lib/patinfo2csv/converter.rb

@@ -1,6 +1,8 @@
 #!/usr/bin/env ruby
 # encoding: utf-8
 
+require 'thread'
+
 module Patinfo2csv
   class Converter
     attr_accessor :patinfos, :codes, :lang
@@ -13,33 +15,48 @@ module Patinfo2csv
       :composition
     ]
     DELIMITER = ';'
+    WORKERS = 2
     def initialize
+      @mutex = Mutex.new
+      @q     = Queue.new
       @lang     = "de"
       @patinfos = []
       @codes    = []
       @rows     = []
     end
     def to_csv
-      data = []
-      @patinfos.each_with_index do |row, index|
-        target = []
-        next if row['article_codes'].nil?
-        row['article_codes'].each do |article|
-          if @codes.include?(article[:article_ean13]) # EAN
-            target << article[:article_ean13]
-            @codes.delete(article[:article_ean13])
-          #elsif @code.include?(code[:article_pcode]) # Pharmacode
-          #  target << code[:article_pcode]
-          #  @code.delete(code[:article_pcode])
+      @patinfos.each do |row|
+        @q << row
+      end
+      Array.new(WORKERS) do
+        @q << nil
+        Thread.new do
+          while row = @q.pop
+            target = []
+            next if row['article_codes'].nil?
+            row['article_codes'].each do |article|
+              @mutex.synchronize do
+                if @codes.include?(article[:article_ean13]) # EAN
+                  target << article[:article_ean13]
+                  @codes.delete(article[:article_ean13])
+                end
+                #elsif @code.include?(code[:article_pcode]) # Pharmacode
+                #  target << code[:article_pcode]
+                #  @code.delete(code[:article_pcode])
+                #end
+              end
+            end
+            next if target.empty?
+            chapters = extract_chapters(row)
+            next if chapters.empty?
+            target.each do |code|
+              @mutex.synchronize do
+                @rows << [%Q!"#{code}"!, chapters].flatten.join(DELIMITER)
+              end
+            end
           end
         end
-        next if target.empty?
-        chapters = extract_chapters(row)
-        next if chapters.empty?
-        target.each do |code|
-          @rows << [%Q!"#{code}"!, chapters].flatten.join(DELIMITER)
-        end
-      end
+      end.map(&:join)
       unless @rows.empty? #header
         @rows.unshift([
           "EAN",

data/lib/version.rb

@@ -1,3 +1,3 @@
 module Patinfo2csv
-  VERSION = "1.0.4"
+  VERSION = "1.0.5"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: patinfo2csv
 version: !ruby/object:Gem::Version
-  version: 1.0.4
+  version: 1.0.5
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-03-23 00:00:00.000000000 Z
+date: 2012-03-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rdoc
-  requirement: &7785560 !ruby/object:Gem::Requirement
+  requirement: &21274100 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: '3.10'
   type: :development
   prerelease: false
-  version_requirements: *7785560
+  version_requirements: *21274100
 - !ruby/object:Gem::Dependency
   name: hoe
-  requirement: &7785140 !ruby/object:Gem::Requirement
+  requirement: &21273660 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,7 +32,7 @@ dependencies:
         version: '2.13'
   type: :development
   prerelease: false
-  version_requirements: *7785140
+  version_requirements: *21273660
 description: ! 'Patinfo2csv extracts and converts patinfo.yaml to patinfo.csv;
 
   only 4 chapters(de, text) in patinfo.yaml are extracted.