embulk-parser-grok 0.1.6

data/lib/embulk/guess/grok.rb

@@ -0,0 +1,3 @@
+Embulk::JavaPlugin.register_guess(
+  "grok", "org.embulk.parser.grok.GrokGuessPlugin",
+  File.expand_path('../../../../classpath', __FILE__))

data/lib/embulk/parser/grok.rb

@@ -0,0 +1,3 @@
+Embulk::JavaPlugin.register_parser(
+  "grok", "org.embulk.parser.grok.GrokParserPlugin",
+  File.expand_path('../../../../classpath', __FILE__))

data/pattern/grok-patterns

@@ -0,0 +1,105 @@
+# Forked from https://github.com/elasticsearch/logstash/tree/v1.4.0/patterns
+USERNAME [a-zA-Z0-9._-]+
+USER %{USERNAME:UNWANTED}
+INT (?:[+-]?(?:[0-9]+))
+BASE10NUM (?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+)))
+NUMBER (?:%{BASE10NUM:UNWANTED})
+BASE16NUM (?<![0-9A-Fa-f])(?:[+-]?(?:0x)?(?:[0-9A-Fa-f]+))
+BASE16FLOAT \b(?<![0-9A-Fa-f.])(?:[+-]?(?:0x)?(?:(?:[0-9A-Fa-f]+(?:\.[0-9A-Fa-f]*)?)|(?:\.[0-9A-Fa-f]+)))\b
+
+POSINT \b(?:[1-9][0-9]*)\b
+NONNEGINT \b(?:[0-9]+)\b
+WORD \b\w+\b
+WORDOREMPTY \b\w*\b
+NOTSPACE \S+
+SPACE \s*
+DATA .*?
+GREEDYDATA .*
+#QUOTEDSTRING (?:(?<!\\)(?:"(?:\\.|[^\\"])*"|(?:'(?:\\.|[^\\'])*')|(?:`(?:\\.|[^\\`])*`)))
+QUOTEDSTRING (?>(?<!\\)(?>"(?>\\.|[^\\"]+)+"|""|(?>'(?>\\.|[^\\']+)+')|''|(?>`(?>\\.|[^\\`]+)+`)|``))
+UUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}
+RESOURCEID \b[A-Za-z0-9_/-]*\b
+
+# Networking
+MAC (?:%{CISCOMAC:UNWANTED}|%{WINDOWSMAC:UNWANTED}|%{COMMONMAC:UNWANTED})
+CISCOMAC (?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4})
+WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2})
+COMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2})
+IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?
+IPV4 (?<![0-9])(?:(?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2}))(?![0-9])
+IP (?:%{IPV6:UNWANTED}|%{IPV4:UNWANTED})
+HOSTNAME \b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\.?|\b)
+HOST %{HOSTNAME:UNWANTED}
+IPORHOST (?:%{HOSTNAME:UNWANTED}|%{IP:UNWANTED})
+HOSTPORT (?:%{IPORHOST}:%{POSINT:PORT})
+
+# paths
+PATH (?:%{UNIXPATH}|%{WINPATH})
+UNIXPATH (?>/(?>[\w_%!$@:.,~-]+|\\.)*)+
+#UNIXPATH (?<![\w\/])(?:/[^\/\s?*]*)+
+TTY (?:/dev/(pts|tty([pq])?)(\w+)?/?(?:[0-9]+))
+WINPATH (?>[A-Za-z]+:|\\)(?:\\[^\\?*]*)+
+URIPROTO [A-Za-z]+(\+[A-Za-z+]+)?
+URIHOST %{IPORHOST}(?::%{POSINT:port})?
+# uripath comes loosely from RFC1738, but mostly from what Firefox
+# doesn't turn into %XX
+URIPATH (?:/[A-Za-z0-9$.+!*'(){},~:;=@#%_\-]*)+
+#URIPARAM \?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)?
+URIPARAM \?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]]*
+URIPATHPARAM %{URIPATH}(?:%{URIPARAM})?
+URI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})?
+
+# Months: January, Feb, 3, 03, 12, December
+MONTH \b(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\b
+MONTHNUM (?:0?[1-9]|1[0-2])
+MONTHNUM2 (?:0[1-9]|1[0-2])
+MONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9])
+
+# Days: Monday, Tue, Thu, etc...
+DAY (?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?)
+
+# Years?
+YEAR (?>\d\d){1,2}
+# Time: HH:MM:SS
+#TIME \d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?
+# I'm still on the fence about using grok to perform the time match,
+# since it's probably slower.
+# TIME %{POSINT<24}:%{POSINT<60}(?::%{POSINT<60}(?:\.%{POSINT})?)?
+HOUR (?:2[0123]|[01]?[0-9])
+MINUTE (?:[0-5][0-9])
+# '60' is a leap second in most time standards and thus is valid.
+SECOND (?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)
+TIME (?!<[0-9])%{HOUR:UNWANTED}:%{MINUTE:UNWANTED}(?::%{SECOND:UNWANTED})(?![0-9])
+# datestamp is YYYY/MM/DD-HH:MM:SS.UUUU (or something like it)
+DATE_US %{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR}
+DATE_EU %{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR}
+ISO8601_TIMEZONE (?:Z|[+-]%{HOUR:UNWANTED}(?::?%{MINUTE:UNWANTED}))
+ISO8601_SECOND (?:%{SECOND:UNWANTED}|60)
+TIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR:UNWANTED}:?%{MINUTE:UNWANTED}(?::?%{SECOND:UNWANTED})?%{ISO8601_TIMEZONE}?
+DATE %{DATE_US}|%{DATE_EU}
+DATESTAMP %{DATE}[- ]%{TIME}
+TZ (?:[PMCE][SD]T|UTC)
+DATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ}
+DATESTAMP_RFC2822 %{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE}
+DATESTAMP_OTHER %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR}
+DATESTAMP_EVENTLOG %{YEAR}%{MONTHNUM2}%{MONTHDAY}%{HOUR:UNWANTED}%{MINUTE:UNWANTED}%{SECOND:UNWANTED}
+
+# Syslog Dates: Month Day HH:MM:SS
+SYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME}
+PROG (?:[\w._/%-]+)
+SYSLOGPROG %{PROG:program}(?:\[%{POSINT:pid}\])?
+SYSLOGHOST %{IPORHOST}
+SYSLOGFACILITY <%{NONNEGINT:facility}.%{NONNEGINT:priority}>
+HTTPDATE %{MONTHDAY:UNWANTED}/%{MONTH:UNWANTED}/%{YEAR:UNWANTED}:%{TIME:UNWANTED} %{INT:UNWANTED}
+
+# Shortcuts
+QS %{QUOTEDSTRING:UNWANTED}
+
+# Log formats
+SYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}:
+
+MESSAGESLOG %{SYSLOGBASE} %{DATA}
+
+COMMONAPACHELOG %{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-)
+COMBINEDAPACHELOG %{COMMONAPACHELOG} %{QS:referrer} %{QS:agent}
+

data/pattern/my-patterns

@@ -0,0 +1,12 @@
+
+LOGLEVEL ([A|a]lert|ALERT|[T|t]race|TRACE|[D|d]ebug|DEBUG|[N|n]otice|NOTICE|[I|i]nfo|INFO|[W|w]arn?(?:ing)?|WARN?(?:ING)?|[E|e]rr?(?:or)?|ERR?(?:OR)?|[C|c]rit?(?:ical)?|CRIT?(?:ICAL)?|[F|f]atal|FATAL|[S|s]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?)
+
+TIMESTAMP_ISO8601_WITH_SPACE %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR:UNWANTED}:?%{MINUTE:UNWANTED}(?::?%{SECOND:UNWANTED})?( %{ISO8601_TIMEZONE})?
+
+MULTILINES (.*+\n)*.*
+MULTILINESTOTHEEND (.*+\n)*+
+
+MULTILINELOG_FIRSTLINE %{TIMESTAMP_ISO8601_WITH_SPACE:timestamp} \[%{LOGLEVEL:log_level}\] %{DATA:message}$
+MULTILINELOG %{TIMESTAMP_ISO8601_WITH_SPACE:timestamp} \[%{LOGLEVEL:log_level}\] %{DATA:message}(?:\n%{MULTILINESTOTHEEND:stack_trace})?$
+
+

data/sample/apache.yml

@@ -0,0 +1,28 @@
+in:
+  type: file
+  path_prefix: src/test/resources/apache.log
+  parser:
+    charset: UTF-8
+    newline: CRLF
+    type: grok
+    grok_pattern_files: [pattern/grok-patterns, pattern/my-patterns]
+    timestamp_parser: ruby
+    grok_pattern: '%{COMBINEDAPACHELOG}'
+    columns:
+    - {name: request, type: string}
+    - {name: agent, type: string}
+    - {name: COMMONAPACHELOG, type: string}
+    - {name: auth, type: string}
+    - {name: ident, type: string}
+    - {name: verb, type: string}
+    - {name: referrer, type: string}
+    - {name: bytes, type: long}
+    - {name: response, type: long}
+    - {name: clientip, type: string}
+    - {name: COMBINEDAPACHELOG, type: string}
+    - {name: httpversion, type: string}
+    - {name: rawrequest, type: string}
+    - {name: timestamp, format: '%d/%b/%Y:%T %z', type: timestamp}
+exec:
+  guess_plugins: [grok]
+out: {type: stdout}

data/sample/apache_stop.yml

@@ -0,0 +1,29 @@
+in:
+  type: file
+  path_prefix: src/test/resources/apache_with_error.log
+  parser:
+    charset: UTF-8
+    newline: CRLF
+    type: grok
+    grok_pattern_files: [pattern/grok-patterns, pattern/my-patterns]
+    timestamp_parser: ruby
+    grok_pattern: '%{COMBINEDAPACHELOG}'
+    stop_on_invalid_record: true
+    columns:
+    - {name: request, type: string}
+    - {name: agent, type: string}
+    - {name: COMMONAPACHELOG, type: string}
+    - {name: auth, type: string}
+    - {name: ident, type: string}
+    - {name: verb, type: string}
+    - {name: referrer, type: string}
+    - {name: bytes, type: long}
+    - {name: response, type: long}
+    - {name: clientip, type: string}
+    - {name: COMBINEDAPACHELOG, type: string}
+    - {name: httpversion, type: string}
+    - {name: rawrequest, type: string}
+    - {name: timestamp, format: '%d/%b/%Y:%T %z', type: timestamp}
+exec:
+  guess_plugins: [grok]
+out: {type: stdout}

data/sample/guess.yml

@@ -0,0 +1,19 @@
+in:
+  type: file
+  path_prefix: src/test/resources/apache.log
+  parser:
+    charset: UTF-8
+    newline: CRLF
+    type: grok
+    grok_pattern_files:
+      - pattern/grok-patterns
+      - pattern/my-patterns
+    guess_patterns:
+      - "%{COMBINEDAPACHELOG}"
+      - "%{COMMONAPACHELOG}"
+    timestamp_parser: ruby
+exec:
+  guess_plugins:
+    - "grok"
+out:
+  type: stdout

data/sample/multiline.yml

@@ -0,0 +1,19 @@
+in:
+  type: file
+  path_prefix: src/test/resources/multiline.log
+  parser:
+    charset: UTF-8
+    newline: CRLF
+    type: grok
+    grok_pattern_files: [pattern/grok-patterns, pattern/my-patterns]
+    timestamp_parser: ruby
+    first_line_pattern: '%{MULTILINELOG_FIRSTLINE}'
+    grok_pattern: '%{MULTILINELOG}'
+    columns:
+    - {name: timestamp, format: '%Y-%m-%d %H:%M:%S.%N %z', type: timestamp}
+    - {name: log_level, type: string}
+    - {name: message, type: string}
+    - {name: stack_trace, type: string}
+exec:
+  guess_plugins: [grok]
+out: {type: stdout}

data/src/main/java/org/embulk/parser/grok/DateParser.java

@@ -0,0 +1,8 @@
+package org.embulk.parser.grok;
+
+import org.embulk.spi.time.Timestamp;
+
+@FunctionalInterface
+public interface DateParser {
+    Timestamp parse(String date);
+}

data/src/main/java/org/embulk/parser/grok/GrokColumnVisitor.java

@@ -0,0 +1,98 @@
+package org.embulk.parser.grok;
+
+import com.google.common.collect.ImmutableSet;
+import org.embulk.spi.Column;
+import org.embulk.spi.ColumnVisitor;
+import org.embulk.spi.Exec;
+import org.embulk.spi.PageBuilder;
+import org.slf4j.Logger;
+
+import java.util.List;
+import java.util.Map;
+
+public class GrokColumnVisitor implements ColumnVisitor {
+
+    private Map<String, Object> record;
+    private PageBuilder pageBuilder;
+    private final List<DateParser> timestampParsers;
+    private final Logger logger = Exec.getLogger(GrokColumnVisitor.class.getName());
+
+    private static final ImmutableSet<String> TRUE_STRINGS =
+            ImmutableSet.of(
+                    "true", "True", "TRUE",
+                    "yes", "Yes", "YES",
+                    "t", "T", "y", "Y",
+                    "on", "On", "ON",
+                    "1");
+
+    public GrokColumnVisitor(Map<String, Object> record, PageBuilder pageBuilder, List<DateParser> timestampParsers) {
+        this.record = record;
+        this.pageBuilder = pageBuilder;
+        this.timestampParsers = timestampParsers;
+    }
+
+    @Override
+    public void booleanColumn(Column column) {
+        if (record.get(column.getName()) == null) {
+            pageBuilder.setNull(column);
+        } else {
+            pageBuilder.setBoolean(column, TRUE_STRINGS.contains(record.get(column.getName()).toString()));
+        }
+    }
+
+    @Override
+    public void longColumn(Column column) {
+        Object longNum = record.get(column.getName());
+        if (longNum == null || longNum.toString().equals("null")) {
+            pageBuilder.setNull(column);
+        } else {
+            try {
+                pageBuilder.setLong(column, Long.parseLong(longNum.toString()));
+            } catch (NumberFormatException e) {
+                logger.error("This column is not Long:" + longNum.toString(), e);
+                throw new GrokRecordValidateException(e);
+            }
+        }
+    }
+
+    @Override
+    public void doubleColumn(Column column) {
+        Object dbl = record.get(column.getName());
+        if (dbl == null) {
+            pageBuilder.setNull(column);
+        } else {
+            try {
+                pageBuilder.setDouble(column, Double.parseDouble(dbl.toString()));
+            } catch (NumberFormatException e) {
+                logger.error("This column is not Double:" + dbl.toString(), e);
+                throw new GrokRecordValidateException(e);
+            }
+        }
+    }
+
+    @Override
+    public void stringColumn(Column column) {
+        if (record.get(column.getName()) == null)
+            pageBuilder.setNull(column);
+        else {
+            pageBuilder.setString(column, record.get(column.getName()).toString());
+        }
+    }
+
+    @Override
+    public void timestampColumn(Column column) {
+        Object time = record.get(column.getName());
+
+        if (time == null) {
+            pageBuilder.setNull(column);
+        } else {
+            String timeString = time.toString();
+            try {
+                pageBuilder.setTimestamp(column, timestampParsers.get(column.getIndex()).parse(timeString));
+            } catch (RuntimeException e) {
+                logger.error("TimestampParseError:" + column.getName() + ", timeString:" + timeString + ", getIndex:" + column.getIndex(), e);
+                throw new GrokRecordValidateException(e);
+            }
+        }
+    }
+}

data/src/main/java/org/embulk/parser/grok/GrokGuessPlugin.java

@@ -0,0 +1,68 @@
+package org.embulk.parser.grok;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import oi.thekraken.grok.api.exception.GrokException;
+import org.embulk.config.*;
+import org.embulk.spi.Buffer;
+import org.embulk.spi.Exec;
+import org.embulk.spi.GuessPlugin;
+import org.embulk.spi.util.LineDecoder;
+import org.embulk.spi.util.ListFileInput;
+import org.slf4j.Logger;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class GrokGuessPlugin implements GuessPlugin {
+    public final Logger logger = Exec.getLogger(GrokGuessPlugin.class.getName());
+
+    public interface PluginTask
+            extends Task, LineDecoder.DecoderTask {
+
+        @Config("grok_pattern_files")
+        List<String> getGrokPatternFiles();
+
+        @Config("guess_patterns")
+        @ConfigDefault("[]")
+        List<String> getGuessPatterns();
+    }
+
+    @Override
+    public ConfigDiff guess(ConfigSource config, Buffer sample) {
+
+        GrokGuessPlugin.PluginTask task = config.getNested("parser").loadConfig(GrokGuessPlugin.PluginTask.class);
+
+        LineDecoder.DecoderTask decoderTask = config.loadConfig(LineDecoder.DecoderTask.class);
+        LineDecoder decoder = new LineDecoder(new ListFileInput(ImmutableList.of(ImmutableList.of((sample)))), decoderTask);
+
+        List<String> sampleLines = new ArrayList<>();
+        while (true) {
+            if (!decoder.nextFile()) {
+                break;
+            }
+            while (true) {
+                String line = decoder.poll();
+                if (line == null) {
+                    break;
+                }
+                sampleLines.add(line);
+            }
+        }
+
+        GrokGuesser guesser = new GrokGuesser(
+                task.getGuessPatterns(),
+                task.getGrokPatternFiles()
+        );
+        try {
+            String pattern = guesser.guessPattern(sampleLines);
+            List<Map<String, Object>> columns = guesser.guessColumns(sampleLines, pattern);
+            return Exec.newConfigDiff().set(
+                    "parser", ImmutableMap.of("grok_pattern", pattern, "columns", columns));
+        } catch (GrokException e) {
+            return Exec.newConfigDiff();
+        }
+
+    }
+}

data/src/main/java/org/embulk/parser/grok/GrokGuesser.java

@@ -0,0 +1,185 @@
+package org.embulk.parser.grok;
+
+import com.google.common.collect.ImmutableMap;
+import oi.thekraken.grok.api.Grok;
+import oi.thekraken.grok.api.Match;
+import oi.thekraken.grok.api.exception.GrokException;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.stream.Collectors;
+
+public class GrokGuesser {
+
+    private List<String> guessPatterns;
+    private List<String> patternFiles;
+
+    public GrokGuesser(List<String> guessPatterns, List<String> patternFiles) {
+        this.guessPatterns = guessPatterns;
+        this.patternFiles = patternFiles;
+    }
+
+    public String guessPattern(List<String> sampleLines) throws GrokException {
+        for (String guessPattern : guessPatterns) {
+            Grok grok = new Grok();
+            for (String file : patternFiles) {
+                grok.addPatternFromFile(file);
+            }
+            try {
+                grok.compile(guessPattern);
+            } catch (GrokException e) {
+                continue;
+            }
+
+            boolean allMatch = sampleLines.stream().allMatch(line -> {
+                Match m = grok.match(line);
+                m.captures();
+                return !m.isNull();
+            });
+            if (allMatch) {
+                return guessPattern;
+            }
+        }
+
+        throw new GrokException("Patterns not matched");
+    }
+
+    public List<Map<String, Object>> guessColumns(List<String> sampleLines, String pattern) throws GrokException {
+
+        Grok grok = new Grok();
+        for (String file : patternFiles) {
+            grok.addPatternFromFile(file);
+        }
+        grok.compile(pattern);
+
+        List<Map<String, Object>> records = sampleLines.stream().map(line -> {
+            Match m = grok.match(line);
+            m.captures();
+            return m.toMap();
+        }).collect(Collectors.toList());
+
+        return guessTypesFromRecords(records);
+    }
+
+
+    private List<Map<String, Object>> guessTypesFromRecords(List<Map<String, Object>> samples) {
+        Map<String, ColumnType> types = new HashMap<>();
+        for (Map<String, Object> record : samples) {
+            for (Map.Entry<String, Object> entry : record.entrySet()) {
+                ColumnType currentType = guessType(entry.getValue());
+                if (types.containsKey(entry.getKey())) {
+                    types.put(entry.getKey(), mergeType(currentType, types.get(entry.getKey())));
+                } else {
+                    types.put(entry.getKey(), currentType);
+                }
+            }
+        }
+        return types.entrySet().stream().map(entry -> {
+            Map<String, Object> val = new HashMap<>();
+            val.put("name", entry.getKey());
+            val.put("type", entry.getValue().getType());
+            if (entry.getValue().getType().equals("timestamp")) {
+                val.put("format", entry.getValue().getFormat());
+            }
+            return val;
+        }).collect(Collectors.toList());
+    }
+
+    private Map<String, SimpleDateFormat> timestampFormats = ImmutableMap.of(
+            "%d/%b/%Y:%T %z", new SimpleDateFormat("dd/MMM/yyyy:HH:mm:ss z", Locale.ENGLISH),
+            "%Y-%m-%d %H:%M:%S", new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"),
+            "%Y-%m-%d %H:%M:%S.%N", new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"),
+            "%Y-%m-%d %H:%M:%S.%N %z", new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS z"),
+            "%Y-%m-%dT%H:%M:%S.%N%z", new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSz")
+    );
+
+    private ColumnType guessType(Object value) {
+
+        if (value == null) {
+            return new ColumnType("string");
+        } else if (value instanceof Integer) {
+            return new ColumnType("long");
+        } else if (value instanceof Double) {
+            return new ColumnType("double");
+        } else {
+            Optional<String> dateFormat = timestampFormats.entrySet().stream().filter(e -> {
+                try {
+                    return e.getValue().parse(value.toString()) != null;
+                } catch (ParseException e1) {
+                    return false;
+                }
+            }).map(Map.Entry::getKey).findFirst();
+            if (dateFormat.isPresent()) {
+                return new ColumnType("timestamp", dateFormat.get());
+            } else {
+                return new ColumnType("string");
+            }
+        }
+    }
+
+    private ColumnType mergeType(ColumnType t1, ColumnType t2) {
+        if (t1.equals(t2)) {
+            return t1;
+        }
+
+        if (t1.getType().equals("string") || t2.getType().equals("string")) {
+            return new ColumnType("string");
+        }
+
+        if (t1.getType().equals("timestamp") || t2.getType().equals("timestamp")) {
+            return new ColumnType("string");
+        }
+
+        if ((t1.getType().equals("long") && t2.getType().equals("double"))
+                || (t1.getType().equals("double") && t2.getType().equals("long"))) {
+            return new ColumnType("double");
+        }
+
+        return new ColumnType("string");
+    }
+
+    static class ColumnType {
+        private String type;
+        private String format;
+
+        public ColumnType(String type) {
+            this.type = type;
+            this.format = null;
+        }
+
+        public ColumnType(String type, String format) {
+            this.type = type;
+            this.format = format;
+        }
+
+        public String getType() {
+            return type;
+        }
+
+        public void setType(String type) {
+            this.type = type;
+        }
+
+        public String getFormat() {
+            return format;
+        }
+
+        public void setFormat(String format) {
+            this.format = format;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+
+            ColumnType that = (ColumnType) o;
+
+            if (type != null ? !type.equals(that.type) : that.type != null) return false;
+            return format != null ? format.equals(that.format) : that.format == null;
+
+        }
+    }
+}
+