embulk-parser-query_string 0.0.3 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fb9918e68aa9b4a3569fefcbb2ffa477aba99378
-  data.tar.gz: cbf1b7c3d4d14ecc0180c9de70f8e64b59c185b3
+  metadata.gz: 482d53ac57ad5d31f0149507090deeb010743c45
+  data.tar.gz: 4fac1b68e078caf3f88e763ce7af0deceb0eeff6
 SHA512:
-  metadata.gz: c60a005d7be4dd2de37445d1b7e696043824ef69f8c05c4fb90304aea8de14f0a75507031f7667ddacdcf2750806bdd2a657d47ab27a6fbcd384431e939c18fc
-  data.tar.gz: c433316f33f2b8b3ad29b7ec052f14a393c0101ace98ac103cf0d965a5a3fd43915c1e93d2c0273393f7642f53b8793355c33515923ef6e5dc2c58f7cca92094
+  metadata.gz: f06bd83e24c40a98d38d343e4d3af7febfc761f39fab704538b9d8a45046219a1d1ffa2c0d49d09fe8bce07ba11f67c301405425921f191756333f80a6e96f4b
+  data.tar.gz: fbb0c6bd32fa340e4bb0bbb50c17a5f8a3265ea37af262679259f32ebe3571bbeb669420eae1ef47d16a7c279ef99059bcc71405b6bfc29d995f2a549d305396

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 0.1.0 - 2015-07-14
+* [enhancement] Add capture option [#11](https://github.com/treasure-data/embulk-parser-query_string/pull/11)
+
 ## 0.0.3 - 2015-07-08
 
 * [enhancement] Support embulk 0.6.10 (backward compatibility) [#9](https://github.com/treasure-data/embulk-parser-query_string/pull/9)

data/README.md

@@ -22,6 +22,7 @@ Required Embulk version >= 0.6.10.
 
 - **strip_quote**: If you have quoted lines file such as `"foo=FOO&bar=BAR"`, should be true for strip their quotes. (bool, default: true)
 - **strip_whitespace**: Strip whitespace before parsing lines for any indented line parse correctly such as '  foo=FOO'. (bool, default: true)
+- **capture**: Capture valuable text from each line using Regexp. Matched first pattern (a.k.a `$1`) will be used. See also [partial-config.yml](./partial-config.yml) (string, default: nil)
 
 ## Example
 

data/embulk-parser-query_string.gemspec

@@ -1,7 +1,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-parser-query_string"
-  spec.version       = "0.0.3"
+  spec.version       = "0.1.0"
   spec.authors       = ["yoshihara", "uu59"]
   spec.summary       = "Query String parser plugin for Embulk"
   spec.description   = "Parses Query String files read by other file input plugins."

data/lib/embulk/guess/query_string.rb

@@ -10,9 +10,11 @@ module Embulk
       def guess_lines(config, sample_lines)
         return {} unless config.fetch("parser", {}).fetch("type", "query_string") == "query_string"
 
+        parser_config = config.param("parser", :hash)
         options = {
-          strip_quote: config.param("strip_quote", :bool, default: true),
-          strip_whitespace: config.param("strip_whitespace", :bool, default: true)
+          strip_quote: parser_config.param("strip_quote", :bool, default: true),
+          strip_whitespace: parser_config.param("strip_whitespace", :bool, default: true),
+          capture: parser_config.param("capture", :string, default: nil)
         }
         records = sample_lines.map do |line|
           Parser::QueryString.parse(line, options) || {}

data/lib/embulk/parser/query_string.rb

@@ -49,6 +49,10 @@ module Embulk
       end
 
       def self.parse(line, options = {})
+        if options[:capture]
+          line = line.match(options[:capture]).to_a[1]
+          # TODO: detect incorrect regexp given
+        end
         line.chomp!
         line.strip! if options[:strip_whitespace]
         if options[:strip_quote]

data/partial-config.yml

@@ -1,8 +1,9 @@
 in:
   type: file
-  path_prefix: ./file
+  path_prefix: ./target_file
   parser:
     strip_quote: true
     strip_whitespace: true
+    capture: 'www01: (".*?")' # sample regexp to capture query string from such line -> www01: "uid=123&s=foo"
 exec: {}
 out: {type: stdout}

data/test/embulk/guess/test_query_string.rb

@@ -101,9 +101,11 @@ module Embulk
 
       def task
         {
-          strip_quote: true,
-          strip_whitespace: true,
-          schema: columns,
+          parser: {
+            strip_quote: true,
+            strip_whitespace: true,
+            schema: columns,
+          }
         }
       end
 

data/test/embulk/parser/test_query_string_plugin.rb

@@ -21,6 +21,16 @@ module Embulk
           assert_equal(expected, result)
         end
 
+        def test_with_capture
+          result = QueryString.parse(indented_line, capture: /^ *(.*)$/)
+          assert_equal(expected, result)
+        end
+
+        def test_with_capture_and_quote
+          result = QueryString.parse(complex_line, strip_quote: true, capture: /^[^"]*(".*?")$/)
+          assert_equal(expected, result)
+        end
+
         def test_with_invalid
           result = QueryString.parse(invalid_line)
           assert_nil(result)
@@ -44,6 +54,10 @@ module Embulk
           %Q(  #{line})
         end
 
+        def complex_line
+          %Q(Jul 11 11:22:33 ec2-instance-001 : "#{line}")
+        end
+
         def invalid_line
           "invalid=www=form"
         end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-parser-query_string
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.1.0
 platform: ruby
 authors:
 - yoshihara
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-07-08 00:00:00.000000000 Z
+date: 2015-07-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement