embulk-parser-apache-log 0.1.0 → 0.1.1

data/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,5 @@
-#Wed Feb 04 13:46:12 PST 2015
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-2.2.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.1-bin.zip

data/gradlew

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/usr/bin/env sh
 
 ##############################################################################
 ##
@@ -6,20 +6,38 @@
 ##
 ##############################################################################
 
-# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-DEFAULT_JVM_OPTS=""
+# Attempt to set APP_HOME
+# Resolve links: $0 may be a link
+PRG="$0"
+# Need this for relative symlinks.
+while [ -h "$PRG" ] ; do
+    ls=`ls -ld "$PRG"`
+    link=`expr "$ls" : '.*-> \(.*\)$'`
+    if expr "$link" : '/.*' > /dev/null; then
+        PRG="$link"
+    else
+        PRG=`dirname "$PRG"`"/$link"
+    fi
+done
+SAVED="`pwd`"
+cd "`dirname \"$PRG\"`/" >/dev/null
+APP_HOME="`pwd -P`"
+cd "$SAVED" >/dev/null
 
 APP_NAME="Gradle"
 APP_BASE_NAME=`basename "$0"`
 
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS=""
+
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD="maximum"
 
-warn ( ) {
+warn () {
     echo "$*"
 }
 
-die ( ) {
+die () {
     echo
     echo "$*"
     echo
@@ -30,6 +48,7 @@ die ( ) {
 cygwin=false
 msys=false
 darwin=false
+nonstop=false
 case "`uname`" in
   CYGWIN* )
     cygwin=true
@@ -40,31 +59,11 @@ case "`uname`" in
   MINGW* )
     msys=true
     ;;
+  NONSTOP* )
+    nonstop=true
+    ;;
 esac
 
-# For Cygwin, ensure paths are in UNIX format before anything is touched.
-if $cygwin ; then
-    [ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
-fi
-
-# Attempt to set APP_HOME
-# Resolve links: $0 may be a link
-PRG="$0"
-# Need this for relative symlinks.
-while [ -h "$PRG" ] ; do
-    ls=`ls -ld "$PRG"`
-    link=`expr "$ls" : '.*-> \(.*\)$'`
-    if expr "$link" : '/.*' > /dev/null; then
-        PRG="$link"
-    else
-        PRG=`dirname "$PRG"`"/$link"
-    fi
-done
-SAVED="`pwd`"
-cd "`dirname \"$PRG\"`/" >&-
-APP_HOME="`pwd -P`"
-cd "$SAVED" >&-
-
 CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
 
 # Determine the Java command to use to start the JVM.
@@ -90,7 +89,7 @@ location of your Java installation."
 fi
 
 # Increase the maximum file descriptors if we can.
-if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then
+if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
     MAX_FD_LIMIT=`ulimit -H -n`
     if [ $? -eq 0 ] ; then
         if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
@@ -114,6 +113,7 @@ fi
 if $cygwin ; then
     APP_HOME=`cygpath --path --mixed "$APP_HOME"`
     CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
+    JAVACMD=`cygpath --unix "$JAVACMD"`
 
     # We build the pattern for arguments to be converted via cygpath
     ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
@@ -154,11 +154,19 @@ if $cygwin ; then
     esac
 fi
 
-# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules
-function splitJvmOpts() {
-    JVM_OPTS=("$@")
+# Escape application args
+save () {
+    for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
+    echo " "
 }
-eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS
-JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME"
+APP_ARGS=$(save "$@")
+
+# Collect all arguments for the java command, following the shell quoting and substitution rules
+eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
+
+# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
+if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
+  cd "$(dirname "$0")"
+fi
 
-exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@"
+exec "$JAVACMD" "$@"

data/src/main/java/org/embulk/parser/ApacheLogParserPlugin.java

@@ -51,7 +51,7 @@ public class ApacheLogParserPlugin
          }
     }
     public interface PluginTask
-            extends Task, LineDecoder.DecoderTask, TimestampParser.ParserTask
+            extends Task, LineDecoder.DecoderTask, TimestampParser.Task
     {
 
         @Config("format")
@@ -67,26 +67,29 @@ public class ApacheLogParserPlugin
         ArrayList<ColumnConfig> columns = new ArrayList<ColumnConfig>();
         final LogFormat format = task.getFormat();
 
-        columns.add(new ColumnConfig("remote_host",STRING ,null));
-        columns.add(new ColumnConfig("identity_check",STRING ,null));
-        columns.add(new ColumnConfig("user",STRING ,null));
-        columns.add(new ColumnConfig("datetime",TIMESTAMP,null));
-        columns.add(new ColumnConfig("method",STRING ,null));
-        columns.add(new ColumnConfig("path",STRING ,null));
-        columns.add(new ColumnConfig("protocol",STRING ,null));
-        columns.add(new ColumnConfig("status",STRING ,null));
-        columns.add(new ColumnConfig("size",STRING ,null));
+        columns.add(new ColumnConfig("remote_host",STRING ,config));
+        columns.add(new ColumnConfig("identity_check",STRING ,config));
+        columns.add(new ColumnConfig("user",STRING ,config));
+        columns.add(new ColumnConfig("datetime",TIMESTAMP,config));
+        columns.add(new ColumnConfig("method",STRING ,config));
+        columns.add(new ColumnConfig("path",STRING ,config));
+        columns.add(new ColumnConfig("protocol",STRING ,config));
+        columns.add(new ColumnConfig("status",STRING ,config));
+        columns.add(new ColumnConfig("size",STRING ,config));
 
         // combined
         if( format == LogFormat.combined ){
-          columns.add(new ColumnConfig("referer",STRING ,null));
-          columns.add(new ColumnConfig("user_agent",STRING ,null));
+          columns.add(new ColumnConfig("referer",STRING ,config));
+          columns.add(new ColumnConfig("user_agent",STRING ,config));
         }
 
         Schema schema = new SchemaConfig(columns).toSchema();
         control.run(task.dump(), schema);
     }
 
+    private static interface ParserIntlTask extends Task, TimestampParser.Task {}
+    private static interface ParserIntlColumnOption extends Task, TimestampParser.TimestampColumnOption {}
+
     @Override
     public void run(TaskSource taskSource, Schema schema,
             FileInput input, PageOutput output)
@@ -101,7 +104,14 @@ public class ApacheLogParserPlugin
                                                      Pattern.CASE_INSENSITIVE
                                                    | Pattern.DOTALL);
         Matcher accessLogEntryMatcher;
-        final TimestampParser time_parser = new TimestampParser("%d/%b/%Y:%T %z",task);
+        // TODO: Switch to a newer TimestampParser constructor after a reasonable interval.
+        // Traditional constructor is used here for compatibility.
+        final ConfigSource configSource = Exec.newConfigSource();
+        configSource.set("format", "%d/%b/%Y:%T %z");
+        configSource.set("timezone", task.getDefaultTimeZone());
+        final TimestampParser time_parser = new TimestampParser(
+            Exec.newConfigSource().loadConfig(ParserIntlTask.class),
+            configSource.loadConfig(ParserIntlColumnOption.class));
 
         while( input.nextFile() ){
             while(true){
@@ -144,17 +154,18 @@ public class ApacheLogParserPlugin
         final String rexa = "(\\d+(?:\\.\\d+){3})";  // an IP address
         final String rexs = "(\\S+)";                // a single token (no spaces)
         final String rexdt = "\\[([^\\]]+)\\]";      // something between [ and ]
-        final String rexstr = "\"([^\"]*?)\"";       // a quoted string
+        final String rexstr = "\"(.*?)\"";       // a quoted string
         final String rexi = "(\\d+)";                // unsigned integer
-        final String rexp = "\"(\\S+)\\s(\\S+)\\s(\\S+)\""; // method, path, protocol
+        final String rexp = "\"(\\S+)\\s(.*?)\\s(HTTP\\/\\d+\\.\\d+)\""; // method, path, protocol
+
         String rex;
 
         if( type == LogFormat.combined ){
           rex = "^" + String.join( " ", rexa, rexs, rexs, rexdt, rexp,
-                             rexi, rexi, rexstr, rexstr) + "$";
+                             rexi, rexs, rexstr, rexstr) + "$";
         } else {
           rex = "^" + String.join( " ", rexa, rexs, rexs, rexdt, rexp,
-                             rexi, rexi) + "$";
+                             rexi, rexs) + "$";
         }
 
         return rex;

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-parser-apache-log
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Hiroyuki Sato
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-05-30 00:00:00.000000000 Z
+date: 2017-11-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -50,6 +50,10 @@ files:
 - LICENSE.txt
 - README.md
 - build.gradle
+- config/checkstyle/checkstyle.xml
+- config/checkstyle/default.xml
+- example/config.yml
+- example/sample.log
 - gradle/wrapper/gradle-wrapper.jar
 - gradle/wrapper/gradle-wrapper.properties
 - gradlew
@@ -58,7 +62,7 @@ files:
 - lib/embulk/parser/apache-log.rb
 - src/main/java/org/embulk/parser/ApacheLogParserPlugin.java
 - src/test/java/org/embulk/parser/TestApacheLogParserPlugin.java
-- classpath/embulk-parser-apache-log-0.1.0.jar
+- classpath/embulk-parser-apache-log-0.1.1.jar
 homepage: https://github.com/hiroyuki-sato/embulk-parser-apache-log
 licenses:
 - MIT