embulk-parser-apache-log 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d77195ff8c816230eea533a5c60c384ced5d4585
-  data.tar.gz: 59d01cc0e3b78329d454eb2b053841bc9334d574
+  metadata.gz: 72fef5da469e9dea85e8a433f872dccb531e04c7
+  data.tar.gz: e12c5013eb3688e8e27a6d67f6949d042e2623c8
 SHA512:
-  metadata.gz: 4d92ba10fbc5100dd57a6edb67d37dc2c05313770b6c774bd9b1e48de5bab17667b05f02db0c4c37002cf87d053e840c969b070a79811a6c567c26178ba997db
-  data.tar.gz: b9b7b24e908af7913f0dfee7b43f4374a622a4b7dea2cc8c4fd9325ac648afb94a96fda24023749ffea30ff18d55470d7d9758e3835fb35a50162caa483b44ce
+  metadata.gz: f63c0461d2be7368e9bb6e3f3714100ec3b8541d0aa3e3186dbf2c5c6bc9d9a13d64b64c3f0bb84cd51e5e097a747701545cdfd56d0173c2f1b7df617211b9c2
+  data.tar.gz: 53e3c4f7859ab55ff5b13d3ea63a3db4deb5d2ce467356790012f38c2652927885c942ae97c43421de2c975e7d2755d2df27c5138a5e1903016d88a5d9feba36

data/CHANGES.md

@@ -1,3 +1,7 @@
+0.1.2 (2019-10-31)
+------------------
+* Introduce `stop_on_invalid_record` option. [@lafent]
+
 0.1.1 (2017-11-20)
 ------------------
 * Use traditional constructors of TimestampParser against Embulk core deprecation [@dmikurube]

data/README.md

@@ -10,6 +10,7 @@ Embulk parser plugin for apache log (common, combined)
 ## Configuration
 
 - **format**: log format(common,combined) (string, default: combined)
+- **stop_on_invalid_record**: ignore invalid log entries (true, false) (boolean, default: true)
 
 ## Example
 
@@ -19,6 +20,7 @@ in:
   parser:
     type: apache-log
     format: common
+    stop_on_invalid_record: true
 ```
 
 ```

data/build.gradle

@@ -1,6 +1,6 @@
 plugins {
     id "com.jfrog.bintray" version "1.1"
-    id "com.github.jruby-gradle.base" version "0.1.5"
+    id "com.github.jruby-gradle.base" version "1.5.0"
     id "java"
     id "checkstyle"
 }
@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.1.1"
+version = "0.1.2"
 
 sourceCompatibility = 1.7
 
@@ -50,14 +50,16 @@ task checkstyle(type: Checkstyle) {
     source = sourceSets.main.allJava + sourceSets.test.allJava
 }
 task gem(type: JRubyExec, dependsOn: ["gemspec", "classpath"]) {
-    jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
-    script "${project.name}.gemspec"
+    jrubyArgs "-S"
+    script "gem"
+    scriptArgs "build", "${project.name}.gemspec"
     doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
 }
 
 task gemPush(type: JRubyExec, dependsOn: ["gem"]) {
-    jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "push"
-    script "pkg/${project.name}-${project.version}.gem"
+    jrubyArgs "-S"
+    script "gem"
+    scriptArgs "push", "pkg/${project.name}-${project.version}.gem"
 }
 
 task "package"(dependsOn: ["gemspec", "classpath"]) << {

data/src/main/java/org/embulk/parser/ApacheLogParserPlugin.java

@@ -32,9 +32,13 @@ import java.util.regex.Pattern;
 
 import com.google.common.base.Throwables;
 
+import org.slf4j.Logger;
+
 public class ApacheLogParserPlugin
         implements ParserPlugin
 {
+    private static final Logger logger = Exec.getLogger(ApacheLogParserPlugin.class);
+
     public enum LogFormat
     {
          combined("combined"),
@@ -58,6 +62,9 @@ public class ApacheLogParserPlugin
         @ConfigDefault("\"combined\"")
         public LogFormat getFormat();
 
+        @Config("stop_on_invalid_record")
+        @ConfigDefault("true")
+        Boolean getStopOnInvalidRecord();
     }
 
     @Override
@@ -123,7 +130,12 @@ public class ApacheLogParserPlugin
               accessLogEntryMatcher = accessLogPattern.matcher(line);
 
               if(!accessLogEntryMatcher.matches()){
-                throw new RuntimeException("unmatched line" + line);
+                if (task.getStopOnInvalidRecord()) {
+                  throw new RuntimeException("unmatched line" + line);
+                } else {
+                  logger.info("unable to parse line: " + line);
+                  continue;
+                }
               }
 
               pageBuilder.setString(0,accessLogEntryMatcher.group(1));
@@ -132,7 +144,12 @@ public class ApacheLogParserPlugin
               try {
                   pageBuilder.setTimestamp(3,time_parser.parse(accessLogEntryMatcher.group(4)));
               } catch(TimestampParseException ex) {
-                throw Throwables.propagate(ex);
+                if (task.getStopOnInvalidRecord()) {
+                  throw Throwables.propagate(ex);
+                } else {
+                  logger.info("unable to parse time from line: " + line);
+                  continue;
+                }
               }
               pageBuilder.setString(4,accessLogEntryMatcher.group(5));
               pageBuilder.setString(5,accessLogEntryMatcher.group(6));

metadata

@@ -1,19 +1,19 @@
 --- !ruby/object:Gem::Specification
 name: embulk-parser-apache-log
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Hiroyuki Sato
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-11-20 00:00:00.000000000 Z
+date: 2019-10-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
   name: bundler
@@ -21,13 +21,13 @@ dependencies:
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '10.0'
   name: rake
@@ -35,7 +35,7 @@ dependencies:
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '10.0'
 description: Parses Apache Log files read by other file input plugins.
@@ -45,11 +45,12 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
+- ".gitignore"
 - CHANGES.md
 - LICENSE.txt
 - README.md
 - build.gradle
+- classpath/embulk-parser-apache-log-0.1.2.jar
 - config/checkstyle/checkstyle.xml
 - config/checkstyle/default.xml
 - example/config.yml
@@ -62,7 +63,6 @@ files:
 - lib/embulk/parser/apache-log.rb
 - src/main/java/org/embulk/parser/ApacheLogParserPlugin.java
 - src/test/java/org/embulk/parser/TestApacheLogParserPlugin.java
-- classpath/embulk-parser-apache-log-0.1.1.jar
 homepage: https://github.com/hiroyuki-sato/embulk-parser-apache-log
 licenses:
 - MIT
@@ -73,17 +73,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.1.9
+rubygems_version: 2.6.8
 signing_key:
 specification_version: 4
 summary: Apache Log parser plugin for Embulk