embulk-parser-apache-custom-log 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3ba30504cc0a6d376ddd430be6eb5e52d1268ec2
-  data.tar.gz: cf21f1c47b0f0fa4b15ed2f2f1bda56a7224e8f6
+  metadata.gz: 0e9175fc76d713c5e0cb3d94e76abfd1d5907381
+  data.tar.gz: b0b200ead1ca5153054ac23656bdef3257ef9ba9
 SHA512:
-  metadata.gz: a441b181d0243e8f00a97ccc2fbc4e7fe53a3918a4dac8574e6c04d85d6ae494390effa1d1b384a308f464cdb1cd32cbf04550d08f38cfd46ff0effc2669f75a
-  data.tar.gz: 6d3dd325b7168007af1bf5be476534d899f6ff061f0ea2ad918b220ea6612e97281680d91d534003fa3a95cc9198503d141231f41c16645b5f080a798e88cc72
+  metadata.gz: 9d31ad44accc1e444a1b27d5d0cb26e62c09d859ae1d25ab0fbcbabf1187810967ee22bc09347f1f0058e5172f4d942d46bf036bf4afdc726e920d30bb26f2a9
+  data.tar.gz: a8c7663f5b7084dfcc721b9b09ddb25e7a8f6365fe0db77536182dd9c914fe0028dc2efa27b9c36ed07d76aac6a7ec4430b1526807bfde2fc7c561225e24635f

data/README.md

@@ -1,5 +1,7 @@
 # Apache **CustomLog**  parser plugin for Embulk
 
+![circlci-badge](https://circleci.com/gh/jami-i/embulk-parser-apache-custom-log.svg?&style=shield&circle-token=7092d38cbe35149872178bbce455dfc9d5bc951b)
+
 Embulk parser plugin for apache **CustomLog**.
 
 Parser configuration based  [Apache HTTPD 2.2 CustomLogFormat](http://httpd.apache.org/docs/2.2/en/mod/mod_log_config.html#formats)
@@ -64,7 +66,7 @@ see: [LogFormats.java](https://github.com/jami-i/embulk-parser-apache-custom-log
 | q  | String    | request-query           |          |
 | r  | String    | request-line            |          |
 | s  | Long      | response-status         |          |
-| t  | Timestamp | request-time            | timestamp format defined in [strptime](http://docs.ruby-lang.org/en/2.0.0/DateTime.html#method-c-_strptime)|
+| t  | Timestamp | request-time            | timestamp format defined in [strptime](http://docs.ruby-lang.org/en/2.0.0/DateTime.html#method-c-_strptime). <br/>if format is not present, DateTimeFormatter class will be used. |
 | T  | Long      | request-process-time-s  |          |
 | u  | String    | request-user            |          |
 | U  | String    | request-path            |          |

data/build.gradle

@@ -12,7 +12,7 @@ configurations {
     provided
 }
 
-version = "0.3.0"
+version = "0.4.0"
 
 dependencies {
     compile  "org.embulk:embulk-core:0.7.4"

data/circle.yml

@@ -0,0 +1,3 @@
+machine:
+  java:
+    version: oraclejdk8

data/src/main/java/org/embulk/parser/apache/log/SimpleDateFormatTimestampLogElement.java

@@ -0,0 +1,42 @@
+package org.embulk.parser.apache.log;
+
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.time.TimestampParser;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
+
+import java.util.Locale;
+
+
+public class SimpleDateFormatTimestampLogElement extends TimestampLogElement {
+
+    static final DateTimeFormatter formatter =
+            DateTimeFormat
+                    .forPattern("dd/MMM/yyyy:HH:mm:ss Z")
+                    .withLocale(Locale.US);
+
+    public SimpleDateFormatTimestampLogElement(TimestampParser.Task task, String name) {
+        super(task, name, "\\[([^\\]]+)\\]", "");
+    }
+
+    @Override
+    public Timestamp parse(String s) {
+        try{
+            long epoch = formatter.parseDateTime(s).getMillis();
+            return Timestamp.ofEpochMilli(epoch);
+        }catch (Exception e){
+            return null;
+        }
+    }
+
+    @Override
+    public void setToPageBuilder(PageBuilder pageBuilder, int i, String value) {
+        Timestamp parse = parse(value);
+        if(parse != null){
+            pageBuilder.setTimestamp(i, parse);
+        }else{
+            pageBuilder.setNull(i);
+        }
+    }
+}

data/src/main/java/org/embulk/parser/apache/log/TimestampLogElementFactory.java

@@ -1,13 +1,15 @@
 package org.embulk.parser.apache.log;
 
-import java.util.regex.Pattern;
-import java.util.regex.Matcher;
 
 import org.apache.commons.lang3.StringUtils;
 import org.embulk.spi.time.TimestampParser;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class TimestampLogElementFactory implements LogElementFactory<TimestampLogElement>, Patterns {
 
+    private static final Logger logger = LoggerFactory.getLogger(TimestampLogElementFactory.class);
+
     private TimestampParser.Task task;
     private String name;
 
@@ -19,7 +21,8 @@ public class TimestampLogElementFactory implements LogElementFactory<TimestampLo
     @Override
     public TimestampLogElement create(String parameter) {
         if(StringUtils.isEmpty(parameter)){
-            return new TimestampLogElement(task, name, "\\[([^\\]]+)\\]");
+            logger.info("since format parameter is not given, use DateTimeFormatter.");
+            return new SimpleDateFormatTimestampLogElement(task, name);
         }else{
             String regex = toTimestampRegex(parameter);
             return new TimestampLogElement(task, name, regex, parameter);

data/src/test/java/org/embulk/parser/TestApacheLogParserPlugin.java

@@ -9,6 +9,7 @@ import java.io.File;
 import java.io.FileReader;
 import java.io.IOException;
 import java.net.URISyntaxException;
+import java.util.Arrays;
 import java.util.function.Consumer;
 
 import static org.hamcrest.CoreMatchers.is;
@@ -16,7 +17,7 @@ import static org.junit.Assert.assertThat;
 
 public class TestApacheLogParserPlugin {
 
-    private static EmbulkPluginTester tester = new EmbulkPluginTester(ParserPlugin.class, "apache-log", ApacheLogParserPlugin.class);
+    private static EmbulkPluginTester tester = new EmbulkPluginTester(ParserPlugin.class, "apache-log", ApacheCustomLogParserPlugin.class);
 
     @Test
     public void test_common() throws Exception {
@@ -52,6 +53,40 @@ public class TestApacheLogParserPlugin {
 
     }
 
+    @Test
+    public void test_custom_time_format() throws Exception {
+        tester.run("/yml/test_custom_time_format.yml");
+
+        assertResult(
+                "/temp/result_custom_time_format.000.00.tsv",
+                cols -> {
+                    String[] expected = new String[]{
+                            "remote-host",
+                            "remote-log-name",
+                            "request-user",
+                            "request-time",
+                            "request-line",
+                            "response-status",
+                            "response-bytes"
+                    };
+                    assertThat(cols, is(expected));
+                },
+                cols -> {
+                    String[] expected = new String[]{
+                            "127.0.0.1",
+                            "",
+                            "frank",
+                            "2015-11-20 13:55:36.000000 +0000",
+                            "GET /apache_pb.gif HTTP/1.0",
+                            "200",
+                            "2326"
+                    };
+                    assertThat(cols, is(expected));
+                }
+        );
+
+    }
+
     @Test
     public void test_combined() throws Exception {
         tester.run("/yml/test_combined.yml");

data/src/test/java/org/embulk/tester/DummyConfigSource.java

@@ -29,6 +29,11 @@ public class DummyConfigSource implements ConfigSource {
         return false;
     }
 
+    @Override
+    public boolean has(String attrName) {
+        return false;
+    }
+
     @Override
     public <E> E get(Class<E> type, String attrName) {
         return null;

data/src/test/java/org/embulk/tester/EmbulkPluginTester.java

@@ -8,7 +8,8 @@ import java.io.FileWriter;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import org.embulk.command.Runner;
+import org.embulk.EmbulkEmbed;
+import org.embulk.config.ConfigLoader;
 
 public class EmbulkPluginTester {
 
@@ -19,8 +20,14 @@ public class EmbulkPluginTester {
 
 	public void run(String ymlPath) throws Exception
 	{
-		Runner runner = new Runner("{}");
-		runner.run(convert(ymlPath));
+		EmbulkEmbed.Bootstrap bootstrap = new EmbulkEmbed.Bootstrap();
+
+		EmbulkEmbed embulk = bootstrap.initialize();
+
+		ConfigLoader configLoader = new ConfigLoader(embulk.getModelManager());
+		embulk.run(configLoader.fromYamlFile(new File(convert(ymlPath))));
+
+
 	}
 
 	private String convert(String yml) throws Exception

data/src/test/resources/data/access_log_custom_time_format

@@ -0,0 +1 @@
+127.0.0.1 - frank [2015.11.20 13:55:36] "GET /apache_pb.gif HTTP/1.0" 200 2326

data/src/test/resources/yml/test_custom_time_format.yml

@@ -0,0 +1,13 @@
+in:
+  type: file
+  path_prefix: 'data/access_log_custom_time_format'
+  parser:
+    type: apache-log
+    format: '%h %l %u %{[%Y.%m.%d %T]}t \"%r\" %>s %b'
+out:
+  type: file
+  path_prefix: '/temp/result_custom_time_format.'
+  file_ext: tsv
+  formatter:
+    type: csv
+    delimiter: "\t"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-parser-apache-custom-log
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Hiroyuki Sato
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-10-25 00:00:00.000000000 Z
+date: 2015-11-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -52,6 +52,7 @@ files:
 - LICENSE.txt
 - README.md
 - build.gradle
+- circle.yml
 - gradle/wrapper/gradle-wrapper.jar
 - gradle/wrapper/gradle-wrapper.properties
 - gradlew
@@ -66,6 +67,7 @@ files:
 - src/main/java/org/embulk/parser/apache/log/LongLogElementFactory.java
 - src/main/java/org/embulk/parser/apache/log/Patterns.java
 - src/main/java/org/embulk/parser/apache/log/Replacement.java
+- src/main/java/org/embulk/parser/apache/log/SimpleDateFormatTimestampLogElement.java
 - src/main/java/org/embulk/parser/apache/log/StringLogElement.java
 - src/main/java/org/embulk/parser/apache/log/StringLogElementFactory.java
 - src/main/java/org/embulk/parser/apache/log/TimestampLogElement.java
@@ -82,12 +84,14 @@ files:
 - src/test/resources/data/access_log_2_combined
 - src/test/resources/data/access_log_combined
 - src/test/resources/data/access_log_common
+- src/test/resources/data/access_log_custom_time_format
 - src/test/resources/resource.txt
 - src/test/resources/temp/dummy
 - src/test/resources/yml/test_combined.yml
 - src/test/resources/yml/test_combined2.yml
 - src/test/resources/yml/test_common.yml
-- classpath/embulk-parser-apache-custom-log-0.3.0.jar
+- src/test/resources/yml/test_custom_time_format.yml
+- classpath/embulk-parser-apache-custom-log-0.4.0.jar
 homepage: https://github.com/jami-i/embulk-parser-apache-custom-log
 licenses:
 - MIT