embulk-filter-crawler 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 30a79fc0749463b0dce27c96cb1e32f6cdced9f2
-  data.tar.gz: 80db4bafe0d5b0418aa8cf18be52c8cfe66f2e95
+  metadata.gz: 52c2e67425310d86c6695fecbd4eeec2e80bc1f7
+  data.tar.gz: 12615ac3b93eb0eef4dd7c057f2508b070770611
 SHA512:
-  metadata.gz: 7c7df4cb9d0e103a8dc17e34ee90be45da7c659f5a0a0fc06d28215854dc12b026de9e4eaf4b65354c3befcc191c0a60efc33bd3c0c39ba4ac77eb3a0d327760
-  data.tar.gz: fc890159a20819a203eae0ae61eb4cce3a9b8bc11d2e8903a61bd5a59ba1ccf451dd8f58e5e918eeb0d1f0f07fcff2138491c343c1eb7a1b52e01e219816614a
+  metadata.gz: 691bdc10a98944c09892392eab8187d14209cc00724d235c4fdab2d5d66f202254133da858a23a11371142941eb4bb060dda46960a529d5fb6e1091e4ef2ba9f
+  data.tar.gz: 4c5d5da29cbda4ba32f2450aeea928f3779179634565450214494d9ca9a29de41f63f1df536e15cac6f4ce98c3c1e32242396c2d13df740b00628d8f8b30219a

data/build.gradle

@@ -14,7 +14,7 @@ configurations {
     provided
 }
 
-version = "0.1.1"
+version = "0.1.2"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7

data/src/main/java/org/embulk/filter/crawler/CrawlerFilterPlugin.java

@@ -19,6 +19,7 @@ import org.embulk.spi.PageOutput;
 import org.embulk.spi.PageReader;
 import org.embulk.spi.Schema;
 import org.embulk.spi.type.Types;
+import org.slf4j.Logger;
 
 import com.google.common.base.Optional;
 import com.google.common.collect.ImmutableList;
@@ -34,6 +35,8 @@ import edu.uci.ics.crawler4j.robotstxt.RobotstxtServer;
 public class CrawlerFilterPlugin
         implements FilterPlugin
 {
+    private static final Logger logger = Exec.getLogger(CrawlerFilterPlugin.class);
+
     public interface PluginTask
             extends Task
     {
@@ -131,21 +134,10 @@ public class CrawlerFilterPlugin
         return new PageOutput() {
             private PageReader reader = new PageReader(inputSchema);
             private PageBuilder builder = new PageBuilder(Exec.getBufferAllocator(), outputSchema, output);
-            private CrawlController controller = getController();
 
             @Override
             public void finish()
             {
-                for (Object object : controller.getCrawlersLocalData()) {
-                    CrawlStat crawlStat = (CrawlStat) object;
-                    for (Map<String, Object> map : crawlStat.getPages()) {
-                        for (Column outputColumn : outputSchema.getColumns()) {
-                            final Object value = map.get(outputColumn.getName());
-                            setValue(value, outputColumn);
-                        }
-                        builder.addRecord();
-                    }
-                }
                 builder.finish();
             }
 
@@ -158,6 +150,7 @@ public class CrawlerFilterPlugin
             @Override
             public void add(Page page)
             {
+                CrawlController controller = getController();
                 reader.setPage(page);
                 while (reader.nextRecord()) {
                     controller.addSeed(reader.getString(keyNameColumn));
@@ -169,6 +162,16 @@ public class CrawlerFilterPlugin
                 }
                 controller.setCustomData(customData);
                 controller.start(EmbulkCrawler.class, task.getNumberOfCrawlers());
+                for (Object object : controller.getCrawlersLocalData()) {
+                    CrawlStat crawlStat = (CrawlStat) object;
+                    for (Map<String, Object> map : crawlStat.getPages()) {
+                        for (Column outputColumn : outputSchema.getColumns()) {
+                            final Object value = map.get(outputColumn.getName());
+                            setValue(value, outputColumn);
+                        }
+                        builder.addRecord();
+                    }
+                }
             }
 
             /**

data/src/main/java/org/embulk/filter/crawler/EmbulkCrawler.java

@@ -91,7 +91,7 @@ public class EmbulkCrawler extends WebCrawler
             BinaryParseData binaryParseData = (BinaryParseData) parseData;
             map.put(outputPrefix + Constants.HTML, binaryParseData.getHtml());
         }
-        logger.info("{}", webURL.getURL());
+        logger.debug("{}", webURL.getURL());
         myCrawlStat.pages.add(map);
     }
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-crawler
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - toyama0919
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-03-26 00:00:00.000000000 Z
+date: 2016-04-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -73,7 +73,7 @@ files:
 - classpath/commons-logging-1.2.jar
 - classpath/crawler4j-4.2.jar
 - classpath/dom4j-1.6.1.jar
-- classpath/embulk-filter-crawler-0.1.1.jar
+- classpath/embulk-filter-crawler-0.1.2.jar
 - classpath/fontbox-1.8.4.jar
 - classpath/geronimo-stax-api_1.0_spec-1.0.1.jar
 - classpath/httpclient-4.4.jar