embulk-parser-unpack 0.1.0 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 650827aaf0ff16b4ef922a62b7de00f80ad4ce9f
-  data.tar.gz: eb56b59f391e675e386d1b5051a9f37358a871b3
+  metadata.gz: 8189fb6c55eff53f51ba19551118505412203ea2
+  data.tar.gz: a88cef2596881d886ae9c7d94e773df70a8715c1
 SHA512:
-  metadata.gz: 6c9131cb80a84d918b854db39a01ae0fc7a9c3cdb9992c8d170e498adaba42621880b102280491e8c0286eb9e61ee09f3b51dd12136864a86bb47a7cc97735e6
-  data.tar.gz: 398f7585994badbadd0d5f3f7c1702d5ab3752f18561e18c500278b1e48ef5de01b28efc659f6de4232bde2fd1aaff091bb540b89587eea6ebf383f6fd7a5183
+  metadata.gz: 3803a0ca9e19602eaf552b7b2f89432aac1020007e04b5befb097dff0c1903d1ec39850681eb2083d3ec7f828d8f6072608d0a26ecaba76eeaa98456ca9f0a49
+  data.tar.gz: 671b3071171d5e74f81c9f87f5d76b4578ca7be1d4c08d86e6b996d06e91d7c612bb800f07969a74cd7e78b4d3149faef12f161b243c009cfc66c46995013f18

data/.gitignore

@@ -0,0 +1,5 @@
+*~
+/pkg/
+/tmp/
+/.bundle/
+/Gemfile.lock

data/.ruby-version

@@ -0,0 +1 @@
+jruby-9.1.5.0

data/.travis.yml

@@ -0,0 +1,12 @@
+language: ruby
+rvm:
+  - jruby-9.1.5.0
+jdk:
+  - oraclejdk8
+env:
+  global:
+    - JRUBY_OPTS="-Xcli.debug=true --debug"
+
+gemfile:
+  - gemfiles/embulk-latest
+

data/README.md

@@ -1,6 +1,9 @@
+[![Build Status](https://travis-ci.org/kakoni/embulk-parser-unpack.svg?branch=master)](https://travis-ci.org/kakoni/embulk-parser-unpack)
+
 # Unpack parser plugin for Embulk
 
-TODO: Write short description here and embulk-parser-unpack.gemspec file.
+Unpack parser. Useful for parsing fixed width format files.
+Can be used to transform `FirstSecond Third` line to `{key: "First", key2: "Second", key3: "Third"}`
 
 ## Overview
 
@@ -9,9 +12,9 @@ TODO: Write short description here and embulk-parser-unpack.gemspec file.
 
 ## Configuration
 
-- **option1**: description (integer, required)
-- **option2**: description (string, default: `"myvalue"`)
-- **option3**: description (string, default: `null`)
+- **format**: Unpack format string. [See String#unpack](http://apidock.com/ruby/String/unpack) (string, required)
+- **strip_whitespace**: Strip whitespace from parsed values. (bool, default: true)
+- **columns**: declares the list of columns, unpacked values will be assigned to these in order.
 
 ## Example
 
@@ -20,19 +23,16 @@ in:
   type: any file input plugin type
   parser:
     type: unpack
-    option1: example1
-    option2: example2
-```
-
-(If guess supported) you don't have to write `parser:` section in the configuration file. After writing `in:` section, you can let embulk guess `parser:` section using this command:
+    format: a2a5@10a4 #Extracts three values, first 2 chars, then 5 chars and lastly 4 chars from position 10.
+    columns:
+    - {name: first, type: string}
+    - {name: second, type: string}
+    - {name: third, type: string}
 
-```
-$ embulk gem install embulk-parser-unpack
-$ embulk guess -g unpack config.yml -o guessed.yml
 ```
 
-## Build
+## Install plugin
 
 ```
-$ rake
+$ embulk gem install embulk-parser-unpack
 ```

data/Rakefile

@@ -1,3 +1,8 @@
 require "bundler/gem_tasks"
 
-task default: :build
+task default: :test
+
+desc "Run tests"
+task :test do
+  ruby("test/run-test.rb", "--use-color=yes")
+end

data/embulk-parser-unpack.gemspec

@@ -1,10 +1,10 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-parser-unpack"
-  spec.version       = "0.1.0"
+  spec.version       = "0.1.5"
   spec.authors       = ["Karri Niemel\u{e4}"]
   spec.summary       = "Unpack parser plugin for Embulk"
-  spec.description   = "Parses files read by other file input plugins."
+  spec.description   = "Parses fixed width files read by other file input plugins."
   spec.email         = ["kakoni@gmail.com"]
   spec.licenses      = ["MIT"]
   spec.homepage      = "https://github.com/kakoni/embulk-parser-unpack"
@@ -16,4 +16,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'embulk', ['>= 0.8.9']
   spec.add_development_dependency 'bundler', ['>= 1.10.6']
   spec.add_development_dependency 'rake', ['>= 10.0']
+  spec.add_development_dependency 'test-unit'
+  spec.add_development_dependency 'test-unit-rr'
 end

data/gemfiles/embulk-latest

@@ -0,0 +1,4 @@
+source 'https://rubygems.org/'
+gemspec :path => '../'
+
+gem "embulk", "~> 0.8.13"

data/lib/embulk/parser/unpack.rb

@@ -11,6 +11,7 @@ module Embulk
         task = {
           "decoder" => DataSource.from_java(decoder_task.dump),
           "format" => config.param("format", :string),
+          "strip_whitespace" => config.param("strip_whitespace", :bool, default: true),
         }
 
         columns = []
@@ -18,7 +19,6 @@ module Embulk
         schema.each do |column|
           name = column["name"]
           type = column["type"].to_sym
-
           columns << Column.new(nil, name, type)
         end
 
@@ -28,7 +28,7 @@ module Embulk
       def init
         @format = task["format"]
         @decoder = task.param("decoder", :hash).load_task(Java::LineDecoder::DecoderTask)
-
+        @strip_whitespace = task["strip_whitespace"]
       end
 
 
@@ -48,6 +48,7 @@ module Embulk
 
       def process_line(line)
         values = line.unpack(@format)
+        values.map(&:strip!) if @strip_whitespace
         page_builder.add(values)
       end
 

data/test/capture_io.rb

@@ -0,0 +1,45 @@
+module CaptureIo
+  def capture(output = :out, &block)
+    _, out = swap_io(output, &block)
+    out
+  end
+
+  def silence(&block)
+    block_result = nil
+    swap_io(:out) do
+      block_result,_ = swap_io(:err, &block)
+    end
+    block_result
+  end
+
+  def swap_io(output = :out, &block)
+    java_import 'java.io.PrintStream'
+    java_import 'java.io.ByteArrayOutputStream'
+    java_import 'java.lang.System'
+
+    ruby_original_stream = output == :out ? $stdout.dup : $stderr.dup
+    java_original_stream = System.send(output) # :out or :err
+    ruby_buf = StringIO.new
+    java_buf = ByteArrayOutputStream.new
+
+    case output
+    when :out
+      $stdout = ruby_buf
+      System.setOut(PrintStream.new(java_buf))
+    when :err
+      $stderr = ruby_buf
+      System.setErr(PrintStream.new(java_buf))
+    end
+
+    [block.call, ruby_buf.string + java_buf.toString]
+  ensure
+    case output
+    when :out
+      $stdout = ruby_original_stream
+      System.setOut(java_original_stream)
+    when :err
+      $stderr = ruby_original_stream
+      System.setErr(java_original_stream)
+    end
+  end
+end

data/test/embulk/parser/test_unpack.rb

@@ -0,0 +1,52 @@
+require "prepare_embulk"
+require "embulk/parser/unpack"
+require "embulk/data_source"
+
+
+module Embulk
+  module Parser
+    class QueryStringTest < Test::Unit::TestCase
+
+      class TestProcessLine < self
+
+        def test_foo
+          mock(page_builder).add(["eka", "Toka", "Kolmas", "Neljas"])
+          line = "ekaTokaKolmas Neljas"
+          plugin.send(:process_line, line)
+        end
+
+        def plugin
+         @plugin ||= Unpack.new(DataSource[task], schema, page_builder)
+        end
+
+        def page_builder
+          @page_builder ||= Object.new
+        end
+
+        def task
+          {
+            "decoder" => {"Charset" => "UTF-8", "Newline" => "CRLF"},
+            "format" => "a3a4a6@14a6",
+            "columns" => columns,
+          }
+        end
+
+        def columns
+          [
+            {"name" => "foo", "type" => :string},
+            {"name" => "bar", "type" => :string},
+            {"name" => "baz", "type" => :string},
+            {"name" => "qux", "type" => :string},
+          ]
+        end
+
+        def schema
+          columns.map do |column|
+            Column.new(nil, column["name"], column["type"].to_sym)
+          end
+        end
+
+      end
+    end
+  end
+end

data/test/embulk_run_helper.rb

@@ -0,0 +1,19 @@
+require "capture_io"
+
+module EmbulkRunHelper
+  include CaptureIo
+
+  def embulk_guess(seed_path, dest_path)
+    silence do
+      embulk_exec(%W(guess -g query_string #{seed_path} -o #{dest_path}))
+    end
+  end
+
+  def embulk_run(yaml_path)
+    embulk_exec(%W(run #{yaml_path}))
+  end
+
+  def embulk_exec(cli_options = [])
+    Embulk.run(cli_options)
+  end
+end

data/test/prepare_embulk.rb

@@ -0,0 +1,5 @@
+require "embulk"
+
+Embulk.setup
+
+require "embulk/command/embulk_run"

data/test/run-test.rb

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+
+base_dir = File.expand_path(File.join(File.dirname(__FILE__), ".."))
+lib_dir = File.join(base_dir, "lib")
+test_dir = File.join(base_dir, "test")
+
+require "test-unit"
+require "test/unit/rr"
+
+$LOAD_PATH.unshift(lib_dir)
+$LOAD_PATH.unshift(test_dir)
+
+ENV["TEST_UNIT_MAX_DIFF_TARGET_STRING_SIZE"] ||= "5000"
+
+exit Test::Unit::AutoRunner.run(true, test_dir, ARGV + %w(--collector=dir))

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-parser-unpack
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.5
 platform: ruby
 authors:
 - Karri Niemelä
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-09-23 00:00:00.000000000 Z
+date: 2016-09-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -52,21 +52,58 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '10.0'
-description: Parses files read by other file input plugins.
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: test-unit
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: test-unit-rr
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Parses fixed width files read by other file input plugins.
 email:
 - kakoni@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".gitignore"
+- ".ruby-version"
+- ".travis.yml"
 - Gemfile
 - LICENSE
 - LICENSE.txt
 - README.md
 - Rakefile
 - embulk-parser-unpack.gemspec
+- gemfiles/embulk-latest
 - lib/embulk/guess/unpack.rb
 - lib/embulk/parser/unpack.rb
+- test/capture_io.rb
+- test/embulk/parser/test_unpack.rb
+- test/embulk_run_helper.rb
+- test/prepare_embulk.rb
+- test/run-test.rb
 homepage: https://github.com/kakoni/embulk-parser-unpack
 licenses:
 - MIT
@@ -91,4 +128,9 @@ rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: Unpack parser plugin for Embulk
-test_files: []
+test_files:
+- test/capture_io.rb
+- test/embulk/parser/test_unpack.rb
+- test/embulk_run_helper.rb
+- test/prepare_embulk.rb
+- test/run-test.rb