embulk-parser-mysqldump_tab 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 540119b437ab3189dafba2b577bd0ef444c72583
-  data.tar.gz: ac15037cbdaab0a87a50a392837f28b14e742e65
+  metadata.gz: 910ce8115b6c62894610139f06e36fa86fb9e283
+  data.tar.gz: d4d008238a8b8753710b87617a39474493e9a785
 SHA512:
-  metadata.gz: a5f756f601edcb1bd9f2eec7845cb9775b165c1c65ce99426160542d24f0b19ee604487364bdbe78614c7c6bfb129ba045ce211780697ac7eda13df0ead88653
-  data.tar.gz: 53ea4300b9ed686a312d4ce8a8da9d82a843c14eeaaa59274a669d12c209310fe0399088ec43126c87c264e5ba289f1ef4b78518d02dc3c5a73e578415c497ca
+  metadata.gz: 34792cf6ea893eaff9d82d1e8c276e8769e53db621df45df597d0d096e07f7a533954f50004c95ee88261510f7c23f4c64c97cbe943870c46e12d6a553e52f5a
+  data.tar.gz: 2fba37e57d21e33a1bcfbf67bf53c73ea01b43864e034e99441f264e261b307a5b5bfe1792a9d2f97d1d5b93b3f0abe6cb20e240f64372f231a1f9962964cce4

data/README.md

@@ -22,6 +22,7 @@ in:
     - {name: id, type: long}
     - {name: name, type: string}
     - {name: email, type: string}
+    - {name: created, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
 out:
   type: stdout
 ```

data/embulk-parser-mysqldump_tab.gemspec

@@ -1,7 +1,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-parser-mysqldump_tab"
-  spec.version       = "0.1.0"
+  spec.version       = "0.1.1"
   spec.authors       = ["inouet"]
   spec.summary       = "Mysqldump Tab parser plugin for Embulk"
   spec.description   = "Embulk parser plugin for mysqldump file that dumped with the --tab option."

data/lib/embulk/parser/mysqldump_tab.rb

@@ -13,17 +13,13 @@ module Embulk
 
       def self.transaction(config, &control)
         # configuration code:
-
         parser_task = config.load_config(Java::LineDecoder::DecoderTask)
 
         task = {
           "decoder_task" => DataSource.from_java(parser_task.dump)
-          # "option1" => config.param("option1", :integer),                     # integer, required
-          # "option2" => config.param("option2", :string, default: "myvalue"),  # string, optional
-          # "option3" => config.param("option3", :string, default: nil),        # string, optional
         }
 
-        # https://github.com/treasure-data/embulk-input-jira/blob/master/lib/embulk/input/jira.rb#L22
+        # see https://github.com/treasure-data/embulk-input-jira/blob/master/lib/embulk/input/jira.rb#L22
         attributes = {}
         columns = config.param(:columns, :array).map do |column|
           name = column["name"]
@@ -33,20 +29,13 @@ module Embulk
         end
 
         task[:attributes] = attributes
-
-        # parser option
-        # task[:option1] = config['option1']
-        # task[:option1] = config.param(:option1, :integer, default: 5)
+        task[:columns] = columns
 
         yield(task, columns)
       end
 
       def init
         # initialization code:
-        # @option1 = task["option1"]
-        # @option2 = task["option2"]
-        # @option3 = task["option3"]
-
         @decoder_task = task.param("decoder_task", :hash).load_task(Java::LineDecoder::DecoderTask)
       end
 
@@ -76,6 +65,7 @@ module Embulk
         page_builder.finish
       end
 
+      private
       def parse_line(line)
         # Escape "escaped TAB" temporarily
         line = line.gsub(/\\#{FIELDS_TERMINATED_BY}/, DUMMY_STRING)
@@ -84,22 +74,69 @@ module Embulk
         cols = line.split(FIELDS_TERMINATED_BY)
         cols.map! { |item| item.gsub(/#{DUMMY_STRING}/, FIELDS_TERMINATED_BY) }
 
-        len = task[:attributes].length
-        cols = adjust_column(cols, len)
-        return cols
+        cols = make_record(cols)
       end
 
       def in_column?(line)
         /#{Regexp.escape(FIELDS_ESCAPED_BY)}$/.match(line) ? true : false # escaped new line
       end
 
-      # Adjust array length
-      def adjust_column(arr, len)
-        arr = arr.slice(0, len) # Truncate if more than len
-        arr.fill(0, len) { |i| arr[i] } # If it is less than len, fill it with nil
+      # see https://github.com/takumakanari/embulk-parser-json/blob/master/lib/embulk/parser/jsonpath.rb#L43
+      def make_record(arr)
+        columns = @task[:columns]
+        record = columns.map.with_index do |col, i|
+          val  = cast_value(arr[i], col)
+        end
       end
 
-    end
+      def cast_value(val, col)
+        type = col["type"]
+        fmt  = col["format"]
+
+        case type
+        when "string"
+          val
+        when "long"
+          val.to_i
+        when "double"
+          val.to_f
+        when "json"
+          val
+        when "boolean"
+          if kind_of_boolean?(val)
+            val
+          elsif val.nil? || val.empty?
+            nil
+          elsif val.kind_of?(String)
+            ["yes", "true", "1"].include?(val.downcase)
+          elsif val.kind_of?(Numeric)
+            !val.zero?
+          else
+            !!val
+          end
+        when "timestamp"
+          if val.nil? || val.empty?
+            nil
+          else
+            begin
+              Time.strptime(val, fmt)
+            rescue ArgumentError => e
+              #raise DataParseError.new e
+              nil
+            end
+          end
+        else
+          raise "Unsupported type #{type}"
+        end
+      end
+
+      def kind_of_boolean?(val)
+        val.kind_of?(TrueClass) || val.kind_of?(FalseClass)
+      end
 
+      class DataParseError < StandardError
+      end
+
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-parser-mysqldump_tab
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - inouet
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-10-06 00:00:00.000000000 Z
+date: 2017-10-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: embulk