embulk-parser-apache-custom-log 0.2.0

data/lib/embulk/guess/apache-custom-log.rb

@@ -0,0 +1,61 @@
+module Embulk
+  module Guess
+
+    # TODO implement guess plugin to make this command work:
+    #      $ embulk guess -g "apache-log" partial-config.yml
+    #
+    #      Depending on the file format the plugin uses, you can use choose
+    #      one of binary guess (GuessPlugin), text guess (TextGuessPlugin),
+    #      or line guess (LineGuessPlugin).
+
+    #class ApacheCustomLogParserGuessPlugin < GuessPlugin
+    #  Plugin.register_guess("apache-log", self)
+    #
+    #  def guess(config, sample_buffer)
+    #    if sample_buffer[0,2] == GZIP_HEADER
+    #      guessed = {}
+    #      guessed["type"] = "apache-log"
+    #      guessed["property1"] = "guessed-value"
+    #      return {"parser" => guessed}
+    #    else
+    #      return {}
+    #    end
+    #  end
+    #end
+
+    #class ApacheLogParserGuessPlugin < TextGuessPlugin
+    #  Plugin.register_guess("apache-log", self)
+    #
+    #  def guess_text(config, sample_text)
+    #    js = JSON.parse(sample_text) rescue nil
+    #    if js && js["mykeyword"] == "keyword"
+    #      guessed = {}
+    #      guessed["type"] = "apache-log"
+    #      guessed["property1"] = "guessed-value"
+    #      return {"parser" => guessed}
+    #    else
+    #      return {}
+    #    end
+    #  end
+    #end
+
+    #class ApacheLogParserGuessPlugin < LineGuessPlugin
+    #  Plugin.register_guess("apache-log", self)
+    #
+    #  def guess_lines(config, sample_lines)
+    #    all_line_matched = sample_lines.all? do |line|
+    #      line =~ /mypattern/
+    #    end
+    #    if all_line_matched
+    #      guessed = {}
+    #      guessed["type"] = "apache-log"
+    #      guessed["property1"] = "guessed-value"
+    #      return {"parser" => guessed}
+    #    else
+    #      return {}
+    #    end
+    #  end
+    #end
+
+  end
+end

data/lib/embulk/parser/apache-custom-log.rb

@@ -0,0 +1,3 @@
+Embulk::JavaPlugin.register_parser(
+  "apache-custom-log", "org.embulk.parser.ApacheCustomLogParserPlugin",
+  File.expand_path('../../../../classpath', __FILE__))

data/src/main/java/org/embulk/parser/ApacheCustomLogParserPlugin.java

@@ -0,0 +1,109 @@
+package org.embulk.parser;
+
+import com.google.common.collect.Lists;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.Task;
+import org.embulk.config.TaskSource;
+import org.embulk.parser.apache.log.LogElement;
+import org.embulk.parser.apache.log.LogFormats;
+import org.embulk.parser.apache.log.Replacement;
+import org.embulk.spi.*;
+import org.embulk.spi.time.TimestampParser;
+import org.embulk.spi.util.LineDecoder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class ApacheCustomLogParserPlugin
+        implements ParserPlugin
+{
+
+    private static final Logger logger = LoggerFactory.getLogger(ApacheCustomLogParserPlugin.class);
+
+    public interface PluginTask
+            extends Task, LineDecoder.DecoderTask, TimestampParser.Task
+    {
+
+        @Config("format")
+        String getFormat();
+
+    }
+
+    @Override
+    public void transaction(ConfigSource config, ParserPlugin.Control control)
+    {
+        PluginTask task = config.loadConfig(PluginTask.class);
+        ArrayList<ColumnConfig> columns = Lists.newArrayList();
+        final String format = task.getFormat();
+
+        List<Replacement> replacements = new LogFormats(task).getReplacements(format);
+
+        for (Replacement replacement : replacements) {
+            LogElement<?> logElement = replacement.getLogElement();
+            columns.add(logElement.getColumnConfig(config));
+        }
+
+        Schema schema = new SchemaConfig(columns).toSchema();
+        control.run(task.dump(), schema);
+    }
+
+    @Override
+    public void run(TaskSource taskSource, Schema schema,
+            FileInput input, PageOutput output)
+    {
+        PluginTask task = taskSource.loadTask(PluginTask.class);
+        LineDecoder lineDecoder = new LineDecoder(input,task);
+        PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), schema, output);
+        String line;
+        final String format = task.getFormat();
+        LogFormats logFormats = new LogFormats(task);
+
+        List<Replacement> replacements = logFormats.getReplacements(format);
+
+        String regexp = logFormats.logFormat2RegexpString(format);
+
+        logger.info("LogFormat : " + format);
+        logger.info("RegExp    : " + regexp);
+
+        Pattern accessLogPattern = Pattern.compile("^" + regexp + "$", Pattern.CASE_INSENSITIVE | Pattern.DOTALL);
+        Matcher accessLogEntryMatcher;
+
+        int replacementSize = replacements.size();
+
+        logger.info("replacement : " + replacementSize);
+
+        while( input.nextFile() ){
+            while(true){
+                line = lineDecoder.poll();
+
+                if (line == null) {
+                  break;
+                }
+
+                accessLogEntryMatcher = accessLogPattern.matcher(line);
+
+                if(replacementSize != accessLogEntryMatcher.groupCount()){
+                    logger.warn("group count mismatch. + expected : " + replacementSize);
+                }
+
+                while(accessLogEntryMatcher.find()){
+                    for (int i = 0; i < replacementSize; i++) {
+                        LogElement<?> logElement = replacements.get(i).getLogElement();
+                        String value = accessLogEntryMatcher.group(i + 1);
+
+                        logElement.setToPageBuilder(pageBuilder, i, value);
+                    }
+                }
+
+                pageBuilder.addRecord();
+            }
+        }
+        pageBuilder.finish();
+    }
+
+}

data/src/main/java/org/embulk/parser/apache/log/LogElement.java

@@ -0,0 +1,41 @@
+package org.embulk.parser.apache.log;
+
+
+import org.embulk.config.ConfigSource;
+import org.embulk.spi.ColumnConfig;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.type.Type;
+
+public abstract class LogElement<T> {
+
+    protected String name;
+    protected String regexp;
+    protected final Type outputType;
+
+    public LogElement(String name, String regex, Type outputType){
+        this.name = name;
+        this.regexp = regex;
+        this.outputType = outputType;
+    }
+
+    public String getName(){
+        return name;
+    }
+
+    public String getRegexp() {
+        return regexp;
+    }
+
+    public Type getOutputType() {
+        return outputType;
+    }
+
+    public abstract T parse(String s);
+
+    public abstract void setToPageBuilder(PageBuilder pageBuilder, int i, String value);
+
+    public ColumnConfig getColumnConfig(ConfigSource config){
+        return new ColumnConfig(name, outputType, config);
+    }
+
+}

data/src/main/java/org/embulk/parser/apache/log/LogElementFactory.java

@@ -0,0 +1,6 @@
+package org.embulk.parser.apache.log;
+
+
+public interface LogElementFactory<T extends LogElement> {
+    T create(String parameter);
+}

data/src/main/java/org/embulk/parser/apache/log/LogFormats.java

@@ -0,0 +1,152 @@
+package org.embulk.parser.apache.log;
+
+import com.google.common.collect.Lists;
+import org.embulk.spi.time.TimestampParser;
+
+import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+
+public class LogFormats implements Patterns {
+
+    TimestampParser.Task task;
+
+    public LogFormats(TimestampParser.Task task) {
+        this.task = task;
+    }
+
+    public Map<String, LogElementFactory<? extends LogElement>> getLogElementMappings(){
+
+        Map<String, LogElementFactory<? extends LogElement>> mapping = new HashMap<>();
+
+        mapping.put("a", new StringLogElementFactory("remote-ip",     IP_ADDRESS));
+        mapping.put("A", new StringLogElementFactory("local-ip",      IP_ADDRESS));
+        mapping.put("b", new LongLogElementFactory("response-bytes"));
+        mapping.put("B", new LongLogElementFactory("response-bytes"));
+        mapping.put("C", new StringLogElementFactory("request-cookie"));
+        mapping.put("D", new LongLogElementFactory("request-process-time-us"));
+        mapping.put("e", new StringLogElementFactory("env"));
+        mapping.put("f", new StringLogElementFactory("file-name"));
+        mapping.put("h", new StringLogElementFactory("remote-host"));
+        mapping.put("H", new StringLogElementFactory("request-protocol", NON_SPACE));
+        mapping.put("i", new StringLogElementFactory("request-header"));
+        mapping.put("l", new StringLogElementFactory("remote-log-name", NON_SPACE));
+        mapping.put("m", new StringLogElementFactory("request-method", METHOD));
+
+        mapping.put("n", new StringLogElementFactory("module-note"));
+        mapping.put("o", new StringLogElementFactory("response-header"));
+
+        mapping.put("p", new LongLogElementFactory("request-port"));
+
+        mapping.put("P", new LongLogElementFactory("request-process"));
+
+        mapping.put("q", new StringLogElementFactory("request-query", QUERY));
+
+        mapping.put("r", new StringLogElementFactory("request-line"));
+        mapping.put("s", new LongLogElementFactory("response-status", STATUS));
+
+        mapping.put("t", new TimestampLogElementFactory(task, "request-time"));
+
+        mapping.put("T", new LongLogElementFactory("request-process-time-s"));
+
+        mapping.put("u", new StringLogElementFactory("request-user"));
+        mapping.put("U", new StringLogElementFactory("request-path", PATH));
+        mapping.put("v", new StringLogElementFactory("request-server-name", NON_SPACE));
+        mapping.put("V", new StringLogElementFactory("canonical-server-name", NON_SPACE));
+        mapping.put("X", new StringLogElementFactory("connection-status", CONN_STATUS));
+        mapping.put("I", new LongLogElementFactory("request-total-bytes"));
+        mapping.put("O", new LongLogElementFactory("response-total-bytes"));
+
+        mapping.put("%", new StringLogElementFactory("%", "(¥¥%)"));
+
+        return mapping;
+    }
+
+    /**
+     * RegExp pattern of extract log format key
+     *
+     * this pattern has 9 groups, which are described as below.
+     *
+     * (%((!)?(\d{3}(,\d{3})*))?(<|>)?(\{([^\}]+)\})?([A-z]))
+     * | ||   |     |           |     |  |           |- group(9) key
+     * | ||   |     |           |     |  |------------- group(8) optional parameter
+     * | ||   |     |           |     |---------------- group(7) optional parameter wrapper group
+     * | ||   |     |           |---------------------- group(6) logging timing parameter
+     * | ||   |     |---------------------------------- group(5) additional http status(es)
+     * | ||   |---------------------------------------- group(4) http status(es)
+     * | ||-------------------------------------------- group(3) inverse http status specifier
+     * | |--------------------------------------------- group(2) http status specifier
+     * |----------------------------------------------- group(0), group(1)
+     *
+     */
+    public static final Pattern logFormatExtractor =
+            Pattern.compile("(%((!)?(\\d{3}(,\\d{3})*))?(<|>)?(\\{([^\\}]+)\\})?([A-z]))",
+                    Pattern.DOTALL);
+
+    /**
+     * Convert logFormat String to Regexp String
+     * @param logFormat apache custom log format
+     * @return The pattern that matches CustomLog Configuration.
+     *
+     */
+    public String logFormat2RegexpString(String logFormat){
+        List<Replacement> replacements = getReplacements(logFormat);
+        return replace(logFormat, replacements);
+    }
+
+    private String replace(String logFormat, List<Replacement> replacements) {
+        int offset = 0;
+
+        for (Replacement replacement : replacements) {
+            String left  = logFormat.substring(0, offset + replacement.getStart());
+            String right = logFormat.substring(offset + replacement.getEnd(), logFormat.length());
+            int originalLength = logFormat.length() - left.length() - right.length();
+
+            String regexp = replacement.getLogElement().getRegexp();
+            logFormat = left + regexp + right;
+            offset += regexp.length() - originalLength;
+        }
+        return logFormat;
+    }
+
+    public List<Replacement> getReplacements(String logFormat) {
+        Matcher matcher = logFormatExtractor.matcher(logFormat);
+
+        List<Replacement> replacements = Lists.newArrayList();
+
+        while(matcher.find()){
+            if(matcher.groupCount() != 9){
+                throw new IllegalArgumentException("invalid regexp pattern");
+            }
+            String all = empty(matcher.group(1));
+
+            //TODO implement
+            //String ignoreStatus = empty(matcher.group(3));
+            //Object[] statuses = Arrays.stream(empty(matcher.group(4)).split(",")).toArray();
+            //String position = empty(matcher.group(6));
+
+            String parameter = matcher.group(8);
+            String key = empty(matcher.group(9));
+
+            LogElementFactory<? extends LogElement> factory = getLogElementMappings().get(key);
+
+            if(factory != null){
+                int start = matcher.start();
+                int end   = matcher.end();
+                LogElement logElement = factory.create(parameter);
+                replacements.add(new Replacement(start, end, logElement));
+            }else{
+                throw new IllegalStateException("unknown log format key " + all);
+            }
+
+        }
+        return replacements;
+    }
+
+    private String empty(String s){
+        return s == null ? "" : s;
+    }
+
+
+}

data/src/main/java/org/embulk/parser/apache/log/LongLogElement.java

@@ -0,0 +1,29 @@
+package org.embulk.parser.apache.log;
+
+
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.type.Types;
+
+public class LongLogElement extends LogElement<Long> {
+
+    public LongLogElement(String name, String regex) {
+        super(name, regex, Types.LONG);
+    }
+
+    @Override
+    public Long parse(String s) {
+        try{
+            if("-".equals(s)){
+                return 0L;
+            }
+            return Long.parseLong(s);
+        }catch (NumberFormatException e){
+            return 0L;
+        }
+    }
+
+    @Override
+    public void setToPageBuilder(PageBuilder pageBuilder, int i, String value) {
+        pageBuilder.setLong(i, parse(value));
+    }
+}

data/src/main/java/org/embulk/parser/apache/log/LongLogElementFactory.java

@@ -0,0 +1,30 @@
+package org.embulk.parser.apache.log;
+
+
+import org.apache.commons.lang3.StringUtils;
+
+public class LongLogElementFactory implements LogElementFactory<LongLogElement>, Patterns {
+
+    private String name;
+    private String regexp;
+
+    public LongLogElementFactory(String name, String regexp) {
+        this.name = name;
+        this.regexp = regexp;
+    }
+
+    public LongLogElementFactory(String name) {
+        this.name = name;
+        this.regexp = LONG;
+    }
+
+    @Override
+    public LongLogElement create(String parameter) {
+        if(StringUtils.isEmpty(parameter)){
+            return new LongLogElement(name, regexp);
+        }else {
+            return new LongLogElement(name + "-" + parameter, regexp);
+        }
+
+    }
+}

data/src/main/java/org/embulk/parser/apache/log/Patterns.java

@@ -0,0 +1,23 @@
+package org.embulk.parser.apache.log;
+
+public interface Patterns {
+
+    String NON_SPACE = "([^\\s]*)";
+
+    String IP_ADDRESS = "(\\d+(?:\\.\\d+){3})";
+
+    String LONG = "(-?\\d+|-)";
+
+    String ANY = "(.*)";
+
+    String PATH = "(/[^\\?]*)";
+
+    String QUERY = "(\\?.*)?";
+
+    String STATUS = "([1-9]\\d{2})";
+
+    String METHOD = "(GET|POST|PUT|DELETE|HEAD|OPTIONS|TRACE|CONNECT)";
+
+    String CONN_STATUS = "([X+\\-])";
+
+}