embulk-filter-crawler 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4cb84672dc6d8f59c550c3faca25b051abe23209
-  data.tar.gz: 35fa411ca6985bf8815ef79f40c32865cc11b109
+  metadata.gz: 30a79fc0749463b0dce27c96cb1e32f6cdced9f2
+  data.tar.gz: 80db4bafe0d5b0418aa8cf18be52c8cfe66f2e95
 SHA512:
-  metadata.gz: a165c5351f4d4d929ec98f3b4d539cba692fbf6ed8173994a396381d58a02754ba9f36d9e36625ad4e0f2f55bfc31c1e547549fca195e7e726ede28cd7d8758e
-  data.tar.gz: c53e77f4f465f9da0d733a3b0db8b955e3b3c0c6ccf371a34f7487a4a98cf7052987f35974ada23824a1f72a31f1eebaa4351d9f32b54f9e4502e699e7625fba
+  metadata.gz: 7c7df4cb9d0e103a8dc17e34ee90be45da7c659f5a0a0fc06d28215854dc12b026de9e4eaf4b65354c3befcc191c0a60efc33bd3c0c39ba4ac77eb3a0d327760
+  data.tar.gz: fc890159a20819a203eae0ae61eb4cce3a9b8bc11d2e8903a61bd5a59ba1ccf451dd8f58e5e918eeb0d1f0f07fcff2138491c343c1eb7a1b52e01e219816614a

data/README.md

@@ -10,13 +10,14 @@ Write short description here and build.gradle file.
 
 - **target_key**: base_url column key name (string, require)
 - **max_depth_of_crawling**: max depth of crawling (integer, default: unlimited)
-- **seed_size**: seed_size (string, default: `"myvalue"`)
-- **number_of_crawlers**: parallelism (string, default: 1)
-- **max_pages_to_fetch**: max_pages_to_fetch (string, default: unlimited)
+- **number_of_crawlers**: parallelism (integer, default: 1)
+- **max_pages_to_fetch**: max_pages_to_fetch (integer, default: unlimited)
 - **crawl_storage_folder**: crawl_storage_folder (string, require)
 - **politeness_delay**: politeness_delay (integer, default: null)
 - **user_agent_string**: user_agent_string (string, default: null)
-- **keep_input**: keep_input (string, default: `"myvalue"`)
+- **output_prefix**: output_prefix (string, default: "")
+- **connection_timeout**: connection timeout millisecond (integer, default: 30000)
+- **socket_timeout**: socket timeout millisecond (integer, default: 20000)
 
 ## Example
 
@@ -33,7 +34,6 @@ filters:
   - type: crawler
     target_key: url
     number_of_crawlers: 10
-    seed_size: 100
     max_depth_of_crawling: 4
     politeness_delay: 100
     crawl_storage_folder: "/tmp/crawl/%s"

data/build.gradle

@@ -14,7 +14,7 @@ configurations {
     provided
 }
 
-version = "0.1.0"
+version = "0.1.1"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7

data/src/main/java/org/embulk/filter/crawler/CrawlerFilterPlugin.java

@@ -74,6 +74,14 @@ public class CrawlerFilterPlugin
         @Config("should_not_visit_pattern")
         @ConfigDefault("null")
         public Optional<String> getShouldNotVisitPattern();
+
+        @Config("connection_timeout")
+        @ConfigDefault("null")
+        public Optional<Integer> getConnectionTimeout();
+
+        @Config("socket_timeout")
+        @ConfigDefault("null")
+        public Optional<Integer> getSocketTimeout();
     }
 
     @Override
@@ -202,6 +210,12 @@ public class CrawlerFilterPlugin
                 if (task.getUserAgentString().isPresent()) {
                     config.setUserAgentString(task.getUserAgentString().get());
                 }
+                if (task.getSocketTimeout().isPresent()) {
+                    config.setSocketTimeout(task.getSocketTimeout().get());
+                }
+                if (task.getConnectionTimeout().isPresent()) {
+                    config.setConnectionTimeout(task.getConnectionTimeout().get());
+                }
 
                 PageFetcher pageFetcher = new PageFetcher(config);
                 RobotstxtConfig robotstxtConfig = new RobotstxtConfig();

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-crawler
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - toyama0919
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-03-25 00:00:00.000000000 Z
+date: 2016-03-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -73,7 +73,7 @@ files:
 - classpath/commons-logging-1.2.jar
 - classpath/crawler4j-4.2.jar
 - classpath/dom4j-1.6.1.jar
-- classpath/embulk-filter-crawler-0.1.0.jar
+- classpath/embulk-filter-crawler-0.1.1.jar
 - classpath/fontbox-1.8.4.jar
 - classpath/geronimo-stax-api_1.0_spec-1.0.1.jar
 - classpath/httpclient-4.4.jar