embulk-parser-apache-custom-log 0.2.0

data/src/main/java/org/embulk/parser/apache/log/Replacement.java

@@ -0,0 +1,27 @@
+package org.embulk.parser.apache.log;
+
+
+public class Replacement {
+    private final int start;
+    private final int end;
+    private final LogElement<?> logElement;
+
+    public Replacement(int start, int end, LogElement<?> logElement) {
+        this.logElement = logElement;
+        this.end = end;
+        this.start = start;
+    }
+
+    public int getStart() {
+        return start;
+    }
+
+    public int getEnd() {
+        return end;
+    }
+
+    public LogElement<?> getLogElement() {
+        return logElement;
+    }
+
+}

data/src/main/java/org/embulk/parser/apache/log/StringLogElement.java

@@ -0,0 +1,33 @@
+package org.embulk.parser.apache.log;
+
+
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.type.Types;
+
+public class StringLogElement extends LogElement<String> {
+
+    public StringLogElement(String name, String regex) {
+        super(name, regex, Types.STRING);
+    }
+
+    @Override
+    public String parse(String s) {
+        if("-".equals(s)){
+            return null;
+        }else{
+            return s;
+        }
+
+    }
+
+    @Override
+    public void setToPageBuilder(PageBuilder pageBuilder, int i, String value) {
+        String parsed = parse(value);
+        if(parsed != null){
+            pageBuilder.setString(i, parsed);
+        }else{
+            pageBuilder.setNull(i);
+        }
+
+    }
+}

data/src/main/java/org/embulk/parser/apache/log/StringLogElementFactory.java

@@ -0,0 +1,29 @@
+package org.embulk.parser.apache.log;
+
+
+import org.apache.commons.lang3.StringUtils;
+
+public class StringLogElementFactory implements LogElementFactory<StringLogElement>, Patterns {
+
+    private String name;
+    private String regexp;
+
+    public StringLogElementFactory(String name, String regexp) {
+        this.name = name;
+        this.regexp = regexp;
+    }
+
+    public StringLogElementFactory(String name) {
+        this.name = name;
+        this.regexp = ANY;
+    }
+
+    @Override
+    public StringLogElement create(String parameter) {
+        if(StringUtils.isEmpty(parameter)){
+            return new StringLogElement(name, regexp);
+        }else{
+            return new StringLogElement(name + "-" + parameter, regexp);
+        }
+    }
+}

data/src/main/java/org/embulk/parser/apache/log/TimestampLogElement.java

@@ -0,0 +1,42 @@
+package org.embulk.parser.apache.log;
+
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.time.TimestampParser;
+
+
+import static org.embulk.spi.type.Types.TIMESTAMP;
+
+
+public class TimestampLogElement extends LogElement<Timestamp> {
+
+    private final TimestampParser parser;
+
+    public TimestampLogElement(TimestampParser.Task task, String name, String regex) {
+        this(task, name, regex, "%d/%b/%Y:%T %z");
+    }
+
+    public TimestampLogElement(TimestampParser.Task task, String name, String regex, String pattern) {
+        super(name, regex, TIMESTAMP);
+        this.parser = new TimestampParser(task.getJRuby(), pattern, task.getDefaultTimeZone());
+    }
+
+    @Override
+    public Timestamp parse(String s) {
+        try{
+            return parser.parse(s);
+        }catch (Exception e){
+            return null;
+        }
+    }
+
+    @Override
+    public void setToPageBuilder(PageBuilder pageBuilder, int i, String value) {
+        Timestamp parse = parse(value);
+        if(parse != null){
+            pageBuilder.setTimestamp(i, parse);
+        }else{
+            pageBuilder.setNull(i);
+        }
+    }
+}

data/src/main/java/org/embulk/parser/apache/log/TimestampLogElementFactory.java

@@ -0,0 +1,24 @@
+package org.embulk.parser.apache.log;
+
+import org.apache.commons.lang3.StringUtils;
+import org.embulk.spi.time.TimestampParser;
+
+public class TimestampLogElementFactory implements LogElementFactory<TimestampLogElement>, Patterns {
+
+    private TimestampParser.Task task;
+    private String name;
+
+    public TimestampLogElementFactory(TimestampParser.Task task, String name) {
+        this.task = task;
+        this.name = name;
+    }
+
+    @Override
+    public TimestampLogElement create(String parameter) {
+        if(StringUtils.isEmpty(parameter)){
+            return new TimestampLogElement(task, name, "\\[([^\\]]+)\\]");
+        }else{
+            return new TimestampLogElement(task, name, parameter);
+        }
+    }
+}

data/src/test/java/org/embulk/parser/TestApacheLogParserPlugin.java

@@ -0,0 +1,162 @@
+package org.embulk.parser;
+
+import org.embulk.spi.ParserPlugin;
+import org.embulk.tester.EmbulkPluginTester;
+import org.junit.Test;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.util.function.Consumer;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+
+public class TestApacheLogParserPlugin {
+
+    private static EmbulkPluginTester tester = new EmbulkPluginTester(ParserPlugin.class, "apache-log", ApacheLogParserPlugin.class);
+
+    @Test
+    public void test_common() throws Exception {
+        tester.run("/yml/test_common.yml");
+
+        assertResult(
+                "/temp/result_common.000.00.tsv",
+                cols -> {
+                    String[] expected = new String[]{
+                            "remote-host",
+                            "remote-log-name",
+                            "request-user",
+                            "request-time",
+                            "request-line",
+                            "response-status",
+                            "response-bytes"
+                    };
+                    assertThat(cols, is(expected));
+                },
+                cols -> {
+                    String[] expected = new String[]{
+                            "127.0.0.1",
+                            "",
+                            "frank",
+                            "2000-10-10 20:55:36.000000 +0000",
+                            "GET /apache_pb.gif HTTP/1.0",
+                            "200",
+                            "2326"
+                    };
+                    assertThat(cols, is(expected));
+                }
+        );
+
+    }
+
+    @Test
+    public void test_combined() throws Exception {
+        tester.run("/yml/test_combined.yml");
+
+        assertResult(
+                "/temp/result_combined.000.00.tsv",
+                cols -> {
+                    String[] expected = new String[]{
+                            "remote-host",
+                            "remote-log-name",
+                            "request-user",
+                            "request-time",
+                            "request-line",
+                            "response-status",
+                            "response-bytes",
+                            "request-header-Referer",
+                            "request-header-User-agent"
+                    };
+                    assertThat(cols, is(expected));
+                },
+                cols -> {
+                    String[] expected = new String[]{
+                            "127.0.0.1",
+                            "",
+                            "frank",
+                            "2000-10-10 20:55:36.000000 +0000",
+                            "GET /apache_pb.gif HTTP/1.0",
+                            "200",
+                            "2326",
+                            "http://www.example.com/start.html",
+                            "Mozilla/4.08 [en] (Win98; I ;Nav)"
+                    };
+                    assertThat(cols, is(expected));
+                }
+        );
+
+    }
+
+    @Test
+    public void test_combined2() throws Exception {
+        tester.run("/yml/test_combined2.yml");
+
+        assertResult(
+                "/temp/result_2_combined.000.00.tsv",
+                cols -> {
+                    String[] expected = new String[]{
+                            "remote-host",
+                            "remote-log-name",
+                            "request-user",
+                            "request-time",
+                            "request-method",
+                            "request-path",
+                            "request-query",
+                            "request-protocol",
+                            "response-status",
+                            "response-bytes",
+                            "request-header-Referer",
+                            "request-header-User-agent"
+                    };
+                    assertThat(cols, is(expected));
+                },
+                cols -> {
+                    String[] expected = new String[]{
+                            "24.93.39.209",
+                            "",
+                            "",
+                            "2015-07-25 06:31:32.000000 +0000",
+                            "POST",
+                            "/search/",
+                            "?c=Computers",
+                            "HTTP/1.1",
+                            "200",
+                            "88",
+                            "/category/health",
+                            "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; WOW64; Trident/4.0; GTB6; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30618; .NET4.0C)"
+                    };
+                    assertThat(cols, is(expected));
+                }
+        );
+
+    }
+
+    @SafeVarargs
+    private final void assertResult(String path, Consumer<String[]> headerAssert, Consumer<String[]>... bodyHeadAsserts) throws URISyntaxException, IOException {
+
+        File resultFile = new File(TestApacheLogParserPlugin.class.getResource(path).toURI());
+
+        try (BufferedReader reader = new BufferedReader(new FileReader(resultFile))) {
+
+            String[] headerLine = reader.readLine().split("\t");
+
+            for (Consumer<String[]> bodyHeadAssert : bodyHeadAsserts) {
+                String[] bodyHeadLine = reader.readLine().split("\t");
+
+                assertThat("body column length mismatch.", bodyHeadLine.length, is(headerLine.length));
+
+                headerAssert.accept(headerLine);
+
+                bodyHeadAssert.accept(bodyHeadLine);
+            }
+
+        }
+
+
+    }
+
+
+}

data/src/test/java/org/embulk/parser/apache/log/LogFormatsTest.java

@@ -0,0 +1,39 @@
+package org.embulk.parser.apache.log;
+
+import junit.framework.TestCase;
+import org.embulk.spi.time.TimestampParser;
+import org.joda.time.DateTimeZone;
+import org.jruby.embed.ScriptingContainer;
+
+import java.util.regex.Pattern;
+
+public class LogFormatsTest extends TestCase {
+
+    public void testLogFormat2Regexp() throws Exception {
+
+        String format = "%!100<v %!100,200<v %100,200,300>v %!100,200,300>{hogeHoge}v %v %{X-Forwarded-For}i %t %{%D}t %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O %D";
+
+        LogFormats logFormats = new LogFormats(new TimestampParser.Task() {
+            @Override
+            public DateTimeZone getDefaultTimeZone() {
+                return DateTimeZone.UTC;
+            }
+
+            @Override
+            public String getDefaultTimestampFormat() {
+                return "\"%Y-%m-%d %H:%M:%S.%N %z\"";
+            }
+
+            @Override
+            public ScriptingContainer getJRuby() {
+                return new ScriptingContainer();
+            }
+        });
+
+        String s = logFormats.logFormat2RegexpString(format);
+
+        System.out.println(Pattern.quote(s));
+
+
+    }
+}

data/src/test/java/org/embulk/parser/apache/log/PatternsTest.java

@@ -0,0 +1,120 @@
+package org.embulk.parser.apache.log;
+
+import junit.framework.TestCase;
+import org.junit.Test;
+import org.junit.experimental.runners.Enclosed;
+import org.junit.runner.RunWith;
+
+import java.util.regex.Pattern;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+
+@RunWith(Enclosed.class)
+public class PatternsTest extends TestCase {
+
+    public static class Test_IP_ADDRESS {
+        Pattern pattern = Pattern.compile(Patterns.IP_ADDRESS);
+        @Test
+        public void test_match_with_ipv4(){
+            assertThat(pattern.matcher("127.0.0.1").matches(), is(true));
+            assertThat(pattern.matcher("255.255.255.255").matches(), is(true));
+        }
+    }
+
+    public static class Test_LONG {
+        Pattern pattern = Pattern.compile(Patterns.LONG);
+        @Test
+        public void test_match_with_long(){
+            assertThat(pattern.matcher("1").matches(), is(true));
+            assertThat(pattern.matcher("-1").matches(), is(true));
+            assertThat(pattern.matcher("-").matches(), is(true));
+
+            assertThat(pattern.matcher("a").matches(), is(false));
+        }
+    }
+
+    public static class Test_ANY {
+        Pattern pattern = Pattern.compile(Patterns.ANY);
+        @Test
+        public void test_match_with_long(){
+            assertThat(pattern.matcher("1").matches(), is(true));
+            assertThat(pattern.matcher("-1").matches(), is(true));
+            assertThat(pattern.matcher("-").matches(), is(true));
+
+            assertThat(pattern.matcher("a").matches(), is(true));
+            assertThat(pattern.matcher("").matches(), is(true));
+        }
+    }
+
+    public static class Test_PATH {
+        Pattern pattern = Pattern.compile(Patterns.PATH);
+        @Test
+        public void test_match_with_long(){
+            assertThat(pattern.matcher("/1").matches(), is(true));
+            assertThat(pattern.matcher("/abc/123").matches(), is(true));
+
+            assertThat(pattern.matcher("").matches(), is(false));
+        }
+    }
+
+    public static class Test_QUERY {
+        Pattern pattern = Pattern.compile(Patterns.QUERY);
+        @Test
+        public void test_match_with_long(){
+            assertThat(pattern.matcher("?1").matches(), is(true));
+            assertThat(pattern.matcher("?abc=123&p=v#hash").matches(), is(true));
+
+            assertThat(pattern.matcher("").matches(), is(true));
+        }
+    }
+
+    public static class Test_STATUS {
+        Pattern pattern = Pattern.compile(Patterns.STATUS);
+        @Test
+        public void test_match_with_long(){
+            assertThat(pattern.matcher("100").matches(), is(true));
+            assertThat(pattern.matcher("200").matches(), is(true));
+            assertThat(pattern.matcher("302").matches(), is(true));
+            assertThat(pattern.matcher("404").matches(), is(true));
+            assertThat(pattern.matcher("500").matches(), is(true));
+            assertThat(pattern.matcher("999").matches(), is(true));
+
+            assertThat(pattern.matcher("99").matches(), is(false));
+            assertThat(pattern.matcher("099").matches(), is(false));
+            assertThat(pattern.matcher("1000").matches(), is(false));
+        }
+    }
+
+    public static class Test_METHOD {
+        Pattern pattern = Pattern.compile(Patterns.METHOD);
+        @Test
+        public void test_match_with_long(){
+            assertThat(pattern.matcher("HEAD").matches(), is(true));
+            assertThat(pattern.matcher("GET").matches(), is(true));
+            assertThat(pattern.matcher("POST").matches(), is(true));
+            assertThat(pattern.matcher("PUT").matches(), is(true));
+            assertThat(pattern.matcher("OPTIONS").matches(), is(true));
+            assertThat(pattern.matcher("TRACE").matches(), is(true));
+            assertThat(pattern.matcher("CONNECT").matches(), is(true));
+
+            assertThat(pattern.matcher("").matches(), is(false));
+            assertThat(pattern.matcher("OTHER").matches(), is(false));
+        }
+    }
+
+    public static class Test_CONN_STATUS {
+        Pattern pattern = Pattern.compile(Patterns.CONN_STATUS);
+        @Test
+        public void test_match_with_long(){
+            assertThat(pattern.matcher("X").matches(), is(true));
+            assertThat(pattern.matcher("-").matches(), is(true));
+            assertThat(pattern.matcher("+").matches(), is(true));
+
+            assertThat(pattern.matcher("").matches(), is(false));
+            assertThat(pattern.matcher("foo").matches(), is(false));
+        }
+    }
+
+
+}