embulk-filter-query_string 0.1.2 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3221e8d17d5faad744c2fdc722adfccb54f871c5
-  data.tar.gz: a9c510e70a6ef266615b118690aa9c821dae488e
+  metadata.gz: 7259a5d22d2dcfe756e90863aa86e83ee47031cb
+  data.tar.gz: 0b77859eabb7effbc287629bdc0c0848317a1c81
 SHA512:
-  metadata.gz: 89607dde6b53d17258c9c6ce587f1b1e3604608c1f1d8a5c9de5074595a2ca8e8053a780b489bfb219f126fa91c9727f5985395fb81a812e6e395835891ca6d2
-  data.tar.gz: e0c5d54b140e0518f7ee3f1d85c41403ad144cb10dfe888a0581be5e7cb9c401d9a6e81299c919023a1938cd31f2192fe14a7cc9f9084fc7ddb257bd25ea0432
+  metadata.gz: bbc77a119f5acdd290d6cc4ff8c82d1db7504ff4802bcfba93aa607713e027a23eda94ea12e029efb7895cd432a5accf5b57c8803401d2b9faba025ee5cc802d
+  data.tar.gz: 8f4eaa31cc2edfc262ebf22a9b920455a2327585afb542e1821c444e8b7c844d58a2f817d7fd3beefc83eda213ded9da9b81158e5caea08204a5eadb375a053d

data/build.gradle

@@ -14,14 +14,14 @@ configurations {
     provided
 }
 
-version = "0.1.0"
+version = "0.2.0"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
 dependencies {
-    compile "org.embulk:embulk-core:0.8.4"
-    provided "org.embulk:embulk-core:0.8.4"
+    compile "org.embulk:embulk-core:0.8.13"
+    provided "org.embulk:embulk-core:0.8.13"
 
     // compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION"
     compile 'org.apache.httpcomponents:httpclient:4.5.1'
@@ -79,7 +79,7 @@ task gemspec {
         gemspecFile.write($/
 Gem::Specification.new do |spec|
   spec.name          = "embulk-filter-query_string"
-  spec.version       = "0.1.2"
+  spec.version       = "${version}"
   spec.authors       = ["Minnano Wedding Co., Ltd."]
   spec.summary       = %[Query String filter plugin for Embulk]
   spec.description   = %[The query string filter plugin parses the column contents as query string and insert columns from that field.]

data/src/main/java/org/embulk/filter/query_string/QueryStringFilterPlugin.java

@@ -40,10 +40,14 @@ import org.slf4j.Logger;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 public class QueryStringFilterPlugin
         implements FilterPlugin
 {
+    public static final Pattern QUERY_STRING_PATTERN = Pattern.compile("[^\\?]*\\?([a-z0-9=&;*._%+-]+)", Pattern.CASE_INSENSITIVE);
+
     @Override
     public void transaction(ConfigSource config, Schema inputSchema, FilterPlugin.Control control)
     {
@@ -173,16 +177,18 @@ public class QueryStringFilterPlugin
                     for (Column inputColumn : inputSchema.getColumns()) {
                         if (columnName.equals(inputColumn.getName())) {
                             String path = null;
-                            int pos = -1;
+                            String queryString = null;
 
                             if (!reader.isNull(inputColumn)) {
                                 path = reader.getString(inputColumn);
-                                pos = path.indexOf('?');
-                            }
 
-                            if (path != null && pos > 0) {
-                                String queryString = path.substring(pos + 1);
+                                Matcher matcher = QUERY_STRING_PATTERN.matcher(path);
+                                if (matcher.lookingAt()) {
+                                    queryString = matcher.group(1);
+                                }
+                            }
 
+                            if (queryString != null) {
                                 Map<String, Object> map = new HashMap<>();
                                 QueryStringParser.parseQueryString(queryString, callback, map);
                                 for (ColumnConfig config : task.getExpandedColumns()) {

data/src/test/java/org/embulk/filter/query_string/TestQueryStringFilterPlugin.java

@@ -45,6 +45,11 @@ import static org.junit.Assert.assertTrue;
 
 public class TestQueryStringFilterPlugin
 {
+    private static interface AssertionWithPage
+    {
+        void run(PageReader pageReader, TestPageBuilderReader.MockPageOutput pageOutput);
+    }
+
     @Rule
     public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
 
@@ -85,26 +90,39 @@ public class TestQueryStringFilterPlugin
                 .add("qs", STRING)
                 .add("qa", STRING)
                 .build();
-
-        final QueryStringFilterPlugin plugin = new QueryStringFilterPlugin();
-        plugin.transaction(configSource, inputSchema, new FilterPlugin.Control()
+        testQueryString(configSource, inputSchema, "/path?q1=1&q2=2#fragment", new AssertionWithPage()
         {
             @Override
-            public void run(TaskSource taskSource, Schema outputSchema)
+            public void run(PageReader pageReader, TestPageBuilderReader.MockPageOutput pageOutput)
             {
-                TestPageBuilderReader.MockPageOutput mockPageOutput = new TestPageBuilderReader.MockPageOutput();
-                PageOutput pageOutput = plugin.open(taskSource, inputSchema, outputSchema, mockPageOutput);
+                for (Page page : pageOutput.pages) {
+                    pageReader.setPage(page);
 
-                List<Page> pages = PageTestUtils.buildPage(runtime.getBufferAllocator(), inputSchema, "before", "/path?q1=one&q2=2", "after");
-                for (Page page : pages) {
-                    pageOutput.add(page);
+                    assertThat(pageReader.getString(0), is("before"));
+                    assertThat(pageReader.getString(1), is("1"));
+                    assertEquals(2L, pageReader.getLong(2));
+                    assertThat(pageReader.getString(3), is("after"));
                 }
+            }
+        });
+    }
 
-                pageOutput.finish();
-                pageOutput.close();
+    @Test
+    public void testOpenSuccessfullyWithHashbang()
+    {
+        ConfigSource configSource = loadConfigSource("testOpen.yml");
+        final Schema inputSchema = Schema.builder()
+                .add("qb", STRING)
+                .add("qs", STRING)
+                .add("qa", STRING)
+                .build();
 
-                PageReader pageReader = new PageReader(outputSchema);
-                for (Page page : mockPageOutput.pages) {
+        testQueryString(configSource, inputSchema, "/#!/path?q1=one&q2=2#fragment", new AssertionWithPage()
+        {
+            @Override
+            public void run(PageReader pageReader, TestPageBuilderReader.MockPageOutput pageOutput)
+            {
+                for (Page page : pageOutput.pages) {
                     pageReader.setPage(page);
 
                     assertThat(pageReader.getString(0), is("before"));
@@ -152,6 +170,32 @@ public class TestQueryStringFilterPlugin
         });
     }
 
+    private void testQueryString(ConfigSource configSource, final Schema inputSchema, final String path, final AssertionWithPage assertion)
+    {
+        final QueryStringFilterPlugin plugin = new QueryStringFilterPlugin();
+        plugin.transaction(configSource, inputSchema, new FilterPlugin.Control()
+        {
+            @Override
+            public void run(TaskSource taskSource, Schema outputSchema)
+            {
+                TestPageBuilderReader.MockPageOutput mockPageOutput = new TestPageBuilderReader.MockPageOutput();
+                PageOutput pageOutput = plugin.open(taskSource, inputSchema, outputSchema, mockPageOutput);
+
+                List<Page> pages = PageTestUtils.buildPage(runtime.getBufferAllocator(), inputSchema, "before", path, "after");
+                for (Page page : pages) {
+                    pageOutput.add(page);
+                }
+
+                pageOutput.finish();
+                pageOutput.close();
+
+                PageReader pageReader = new PageReader(outputSchema);
+                assertion.run(pageReader, mockPageOutput);
+                ;
+            }
+        });
+    }
+
     private ConfigSource loadConfigSource(String yamlPath)
     {
         try {

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-query_string
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.2.0
 platform: ruby
 authors:
 - Minnano Wedding Co., Ltd.
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-03-11 00:00:00.000000000 Z
+date: 2016-12-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -64,7 +64,7 @@ files:
 - src/test/resources/testTransaction.yml
 - classpath/commons-codec-1.9.jar
 - classpath/commons-logging-1.2.jar
-- classpath/embulk-filter-query_string-0.1.0.jar
+- classpath/embulk-filter-query_string-0.2.0.jar
 - classpath/fastutil-6.2.2.jar
 - classpath/httpclient-4.5.1.jar
 - classpath/httpcore-4.4.3.jar