embulk-parser-grok 0.1.6

Sign up to get free protection for your applications and to get access to all the features.
Files changed (40) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +22 -0
  3. data/LICENSE.txt +21 -0
  4. data/README.md +113 -0
  5. data/build.gradle +95 -0
  6. data/gradle/wrapper/gradle-wrapper.jar +0 -0
  7. data/gradle/wrapper/gradle-wrapper.properties +6 -0
  8. data/gradlew +164 -0
  9. data/gradlew.bat +90 -0
  10. data/lib/embulk/guess/grok.rb +3 -0
  11. data/lib/embulk/parser/grok.rb +3 -0
  12. data/pattern/grok-patterns +105 -0
  13. data/pattern/my-patterns +12 -0
  14. data/sample/apache.yml +28 -0
  15. data/sample/apache_stop.yml +29 -0
  16. data/sample/guess.yml +19 -0
  17. data/sample/multiline.yml +19 -0
  18. data/src/main/java/org/embulk/parser/grok/DateParser.java +8 -0
  19. data/src/main/java/org/embulk/parser/grok/GrokColumnVisitor.java +98 -0
  20. data/src/main/java/org/embulk/parser/grok/GrokGuessPlugin.java +68 -0
  21. data/src/main/java/org/embulk/parser/grok/GrokGuesser.java +185 -0
  22. data/src/main/java/org/embulk/parser/grok/GrokParserPlugin.java +94 -0
  23. data/src/main/java/org/embulk/parser/grok/GrokRecordIterator.java +71 -0
  24. data/src/main/java/org/embulk/parser/grok/GrokRecordValidateException.java +11 -0
  25. data/src/main/java/org/embulk/parser/grok/MultipleLineDecoder.java +66 -0
  26. data/src/main/java/org/embulk/parser/grok/TimestampParserFactory.java +85 -0
  27. data/src/test/java/org/embulk/parser/TestGrokGuessPlugin.java +50 -0
  28. data/src/test/java/org/embulk/parser/TestGrokParserPlugin.java +55 -0
  29. data/src/test/java/org/embulk/parser/TestGrokPluginBase.java +69 -0
  30. data/src/test/java/org/embulk/util/EmbulkPluginTester.java +80 -0
  31. data/src/test/java/org/embulk/util/StreamUtil.java +30 -0
  32. data/src/test/resources/apache.log +101 -0
  33. data/src/test/resources/apache.yml +35 -0
  34. data/src/test/resources/apache_with_error.log +101 -0
  35. data/src/test/resources/expected_apache.csv +102 -0
  36. data/src/test/resources/expected_multiline.csv +45 -0
  37. data/src/test/resources/guess.yml +19 -0
  38. data/src/test/resources/multiline.log +44 -0
  39. data/src/test/resources/multiline.yml +25 -0
  40. metadata +115 -0
data/src/test/resources/expected_multiline.csv ADDED
@@ -0,0 +1,45 @@
1
+ timestamp|log_level|message|stack_trace
2
+ 2015-12-15 02:45:44.342000 +0000|INFO|(transaction): Loaded plugin embulk-parser-grok (0.1.5)|""
3
+ 2015-12-15 02:45:44.355000 +0000|INFO|(transaction): Listing local files at directory 'src/test/resources' filtering filename by prefix 'apache.log'|""
4
+ 2015-12-15 02:45:44.358000 +0000|INFO|(transaction): Loading files [src/test/resources/apache.log]|"org.embulk.exec.PartialExecutionException: org.embulk.config.ConfigException: com.fasterxml.jackson.core.JsonParseException: Unrecognized token 'ruby': was expecting ('true', 'false' or 'null')
5
+ at [Source: ruby; line: 1, column: 9]
6
+ at org.embulk.exec.BulkLoader$LoaderState.buildPartialExecuteException(org/embulk/exec/BulkLoader.java:363)
7
+ at org.embulk.exec.BulkLoader.doRun(org/embulk/exec/BulkLoader.java:572)
8
+ at org.embulk.exec.BulkLoader.access$000(org/embulk/exec/BulkLoader.java:33)
9
+ at org.embulk.exec.BulkLoader$1.run(org/embulk/exec/BulkLoader.java:374)
10
+ at org.embulk.exec.BulkLoader$1.run(org/embulk/exec/BulkLoader.java:370)
11
+ at org.embulk.spi.Exec.doWith(org/embulk/spi/Exec.java:25)
12
+ at org.embulk.exec.BulkLoader.run(org/embulk/exec/BulkLoader.java:370)
13
+ at org.embulk.EmbulkEmbed.run(org/embulk/EmbulkEmbed.java:180)
14
+ at java.lang.reflect.Method.invoke(java/lang/reflect/Method.java:497)
15
+ at RUBY.run(/home/ikezoe/.embulk/bin/embulk!/embulk/runner.rb:77)
16
+ at RUBY.run(/home/ikezoe/.embulk/bin/embulk!/embulk/command/embulk_run.rb:296)
17
+ at RUBY.<top>(/home/ikezoe/.embulk/bin/embulk!/embulk/command/embulk_main.rb:2)
18
+ at org.jruby.RubyKernel.require(org/jruby/RubyKernel.java:940)
19
+ at RUBY.(root)(uri:classloader:/META-INF/jruby.home/lib/ruby/stdlib/rubygems/core_ext/kernel_require.rb:1)
20
+ at home.ikezoe.$_dot_embulk.bin.embulk.embulk.command.embulk_bundle.<top>(file:/home/ikezoe/.embulk/bin/embulk!/embulk/command/embulk_bundle.rb:51)
21
+ at java.lang.invoke.MethodHandle.invokeWithArguments(java/lang/invoke/MethodHandle.java:627)
22
+ at org.embulk.cli.Main.main(org/embulk/cli/Main.java:23)
23
+ "
24
+ 2015-12-15 02:46:27.968000 +0000|INFO|(transaction): Loaded plugin embulk-parser-grok (0.1.5)|""
25
+ 2015-12-15 02:46:27.983000 +0000|INFO|(transaction): Listing local files at directory 'src/test/resources' filtering filename by prefix 'apache.log'|""
26
+ 2015-12-15 02:46:27.986000 +0000|INFO|(transaction): Loading files [src/test/resources/apache.log]|"org.embulk.exec.PartialExecutionException: org.embulk.config.ConfigException: com.fasterxml.jackson.core.JsonParseException: Unrecognized token 'ruby': was expecting ('true', 'false' or 'null')
27
+ at [Source: ruby; line: 1, column: 9]
28
+ at org.embulk.exec.BulkLoader$LoaderState.buildPartialExecuteException(org/embulk/exec/BulkLoader.java:363)
29
+ at org.embulk.exec.BulkLoader.doRun(org/embulk/exec/BulkLoader.java:572)
30
+ at org.embulk.exec.BulkLoader.access$000(org/embulk/exec/BulkLoader.java:33)
31
+ at org.embulk.exec.BulkLoader$1.run(org/embulk/exec/BulkLoader.java:374)
32
+ at org.embulk.exec.BulkLoader$1.run(org/embulk/exec/BulkLoader.java:370)
33
+ at org.embulk.spi.Exec.doWith(org/embulk/spi/Exec.java:25)
34
+ at org.embulk.exec.BulkLoader.run(org/embulk/exec/BulkLoader.java:370)
35
+ at org.embulk.EmbulkEmbed.run(org/embulk/EmbulkEmbed.java:180)
36
+ at java.lang.reflect.Method.invoke(java/lang/reflect/Method.java:497)
37
+ at RUBY.run(/home/ikezoe/.embulk/bin/embulk!/embulk/runner.rb:77)
38
+ at RUBY.run(/home/ikezoe/.embulk/bin/embulk!/embulk/command/embulk_run.rb:296)
39
+ at RUBY.<top>(/home/ikezoe/.embulk/bin/embulk!/embulk/command/embulk_main.rb:2)
40
+ at org.jruby.RubyKernel.require(org/jruby/RubyKernel.java:940)
41
+ at RUBY.(root)(uri:classloader:/META-INF/jruby.home/lib/ruby/stdlib/rubygems/core_ext/kernel_require.rb:1)
42
+ at home.ikezoe.$_dot_embulk.bin.embulk.embulk.command.embulk_bundle.<top>(file:/home/ikezoe/.embulk/bin/embulk!/embulk/command/embulk_bundle.rb:51)
43
+ at java.lang.invoke.MethodHandle.invokeWithArguments(java/lang/invoke/MethodHandle.java:627)
44
+ at org.embulk.cli.Main.main(org/embulk/cli/Main.java:23)
45
+ "
data/src/test/resources/guess.yml ADDED
@@ -0,0 +1,19 @@
1
+ in:
2
+ type: file
3
+ path_prefix: __INPUT_PATH__
4
+ parser:
5
+ charset: UTF-8
6
+ newline: CRLF
7
+ type: grok
8
+ grok_pattern_files:
9
+ - __PROJECT_DIR__/pattern/grok-patterns
10
+ - __PROJECT_DIR__/pattern/my-patterns
11
+ guess_patterns:
12
+ - "%{COMBINEDAPACHELOG}"
13
+ - "%{COMMONAPACHELOG}"
14
+ timestamp_parser: ruby
15
+ exec:
16
+ guess_plugins:
17
+ - "grok"
18
+ out:
19
+ type: stdout
data/src/test/resources/multiline.log ADDED
@@ -0,0 +1,44 @@
1
+ 2015-12-15 11:45:44.342 +0900 [INFO] (transaction): Loaded plugin embulk-parser-grok (0.1.5)
2
+ 2015-12-15 11:45:44.355 +0900 [INFO] (transaction): Listing local files at directory 'src/test/resources' filtering filename by prefix 'apache.log'
3
+ 2015-12-15 11:45:44.358 +0900 [INFO] (transaction): Loading files [src/test/resources/apache.log]
4
+ org.embulk.exec.PartialExecutionException: org.embulk.config.ConfigException: com.fasterxml.jackson.core.JsonParseException: Unrecognized token 'ruby': was expecting ('true', 'false' or 'null')
5
+ at [Source: ruby; line: 1, column: 9]
6
+ at org.embulk.exec.BulkLoader$LoaderState.buildPartialExecuteException(org/embulk/exec/BulkLoader.java:363)
7
+ at org.embulk.exec.BulkLoader.doRun(org/embulk/exec/BulkLoader.java:572)
8
+ at org.embulk.exec.BulkLoader.access$000(org/embulk/exec/BulkLoader.java:33)
9
+ at org.embulk.exec.BulkLoader$1.run(org/embulk/exec/BulkLoader.java:374)
10
+ at org.embulk.exec.BulkLoader$1.run(org/embulk/exec/BulkLoader.java:370)
11
+ at org.embulk.spi.Exec.doWith(org/embulk/spi/Exec.java:25)
12
+ at org.embulk.exec.BulkLoader.run(org/embulk/exec/BulkLoader.java:370)
13
+ at org.embulk.EmbulkEmbed.run(org/embulk/EmbulkEmbed.java:180)
14
+ at java.lang.reflect.Method.invoke(java/lang/reflect/Method.java:497)
15
+ at RUBY.run(/home/ikezoe/.embulk/bin/embulk!/embulk/runner.rb:77)
16
+ at RUBY.run(/home/ikezoe/.embulk/bin/embulk!/embulk/command/embulk_run.rb:296)
17
+ at RUBY.<top>(/home/ikezoe/.embulk/bin/embulk!/embulk/command/embulk_main.rb:2)
18
+ at org.jruby.RubyKernel.require(org/jruby/RubyKernel.java:940)
19
+ at RUBY.(root)(uri:classloader:/META-INF/jruby.home/lib/ruby/stdlib/rubygems/core_ext/kernel_require.rb:1)
20
+ at home.ikezoe.$_dot_embulk.bin.embulk.embulk.command.embulk_bundle.<top>(file:/home/ikezoe/.embulk/bin/embulk!/embulk/command/embulk_bundle.rb:51)
21
+ at java.lang.invoke.MethodHandle.invokeWithArguments(java/lang/invoke/MethodHandle.java:627)
22
+ at org.embulk.cli.Main.main(org/embulk/cli/Main.java:23)
23
+ 2015-12-15 11:46:27.968 +0900 [INFO] (transaction): Loaded plugin embulk-parser-grok (0.1.5)
24
+ 2015-12-15 11:46:27.983 +0900 [INFO] (transaction): Listing local files at directory 'src/test/resources' filtering filename by prefix 'apache.log'
25
+ 2015-12-15 11:46:27.986 +0900 [INFO] (transaction): Loading files [src/test/resources/apache.log]
26
+ org.embulk.exec.PartialExecutionException: org.embulk.config.ConfigException: com.fasterxml.jackson.core.JsonParseException: Unrecognized token 'ruby': was expecting ('true', 'false' or 'null')
27
+ at [Source: ruby; line: 1, column: 9]
28
+ at org.embulk.exec.BulkLoader$LoaderState.buildPartialExecuteException(org/embulk/exec/BulkLoader.java:363)
29
+ at org.embulk.exec.BulkLoader.doRun(org/embulk/exec/BulkLoader.java:572)
30
+ at org.embulk.exec.BulkLoader.access$000(org/embulk/exec/BulkLoader.java:33)
31
+ at org.embulk.exec.BulkLoader$1.run(org/embulk/exec/BulkLoader.java:374)
32
+ at org.embulk.exec.BulkLoader$1.run(org/embulk/exec/BulkLoader.java:370)
33
+ at org.embulk.spi.Exec.doWith(org/embulk/spi/Exec.java:25)
34
+ at org.embulk.exec.BulkLoader.run(org/embulk/exec/BulkLoader.java:370)
35
+ at org.embulk.EmbulkEmbed.run(org/embulk/EmbulkEmbed.java:180)
36
+ at java.lang.reflect.Method.invoke(java/lang/reflect/Method.java:497)
37
+ at RUBY.run(/home/ikezoe/.embulk/bin/embulk!/embulk/runner.rb:77)
38
+ at RUBY.run(/home/ikezoe/.embulk/bin/embulk!/embulk/command/embulk_run.rb:296)
39
+ at RUBY.<top>(/home/ikezoe/.embulk/bin/embulk!/embulk/command/embulk_main.rb:2)
40
+ at org.jruby.RubyKernel.require(org/jruby/RubyKernel.java:940)
41
+ at RUBY.(root)(uri:classloader:/META-INF/jruby.home/lib/ruby/stdlib/rubygems/core_ext/kernel_require.rb:1)
42
+ at home.ikezoe.$_dot_embulk.bin.embulk.embulk.command.embulk_bundle.<top>(file:/home/ikezoe/.embulk/bin/embulk!/embulk/command/embulk_bundle.rb:51)
43
+ at java.lang.invoke.MethodHandle.invokeWithArguments(java/lang/invoke/MethodHandle.java:627)
44
+ at org.embulk.cli.Main.main(org/embulk/cli/Main.java:23)
data/src/test/resources/multiline.yml ADDED
@@ -0,0 +1,25 @@
1
+ in:
2
+ type: file
3
+ path_prefix: __INPUT_PATH__
4
+ parser:
5
+ charset: UTF-8
6
+ newline: CRLF
7
+ type: grok
8
+ grok_pattern_files:
9
+ - __PROJECT_DIR__/pattern/grok-patterns
10
+ - __PROJECT_DIR__/pattern/my-patterns
11
+ first_line_pattern: '%{MULTILINELOG_FIRSTLINE}'
12
+ grok_pattern: '%{MULTILINELOG}'
13
+ columns:
14
+ - {name: timestamp, format: '%Y-%m-%d %H:%M:%S.%N %z', type: timestamp}
15
+ - {name: log_level, type: string}
16
+ - {name: message, type: string}
17
+ - {name: stack_trace, type: string}
18
+
19
+ out:
20
+ type: file
21
+ path_prefix: __OUTPUT_PATH__
22
+ file_ext: output.csv
23
+ formatter:
24
+ type: csv
25
+ delimiter: "|"
metadata ADDED
@@ -0,0 +1,115 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: embulk-parser-grok
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.6
5
+ platform: ruby
6
+ authors:
7
+ - Ikezoe Akihiro
8
+ - Rishi Kathera
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2015-12-15 00:00:00.000000000 Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ~>
18
+ - !ruby/object:Gem::Version
19
+ version: '1.0'
20
+ name: bundler
21
+ prerelease: false
22
+ type: :development
23
+ version_requirements: !ruby/object:Gem::Requirement
24
+ requirements:
25
+ - - ~>
26
+ - !ruby/object:Gem::Version
27
+ version: '1.0'
28
+ - !ruby/object:Gem::Dependency
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - '>='
32
+ - !ruby/object:Gem::Version
33
+ version: '10.0'
34
+ name: rake
35
+ prerelease: false
36
+ type: :development
37
+ version_requirements: !ruby/object:Gem::Requirement
38
+ requirements:
39
+ - - '>='
40
+ - !ruby/object:Gem::Version
41
+ version: '10.0'
42
+ description: Parses Grok files read by other file input plugins.
43
+ email:
44
+ - ikezoe_a@ariel-networks.com
45
+ executables: []
46
+ extensions: []
47
+ extra_rdoc_files: []
48
+ files:
49
+ - .gitignore
50
+ - LICENSE.txt
51
+ - README.md
52
+ - build.gradle
53
+ - gradle/wrapper/gradle-wrapper.jar
54
+ - gradle/wrapper/gradle-wrapper.properties
55
+ - gradlew
56
+ - gradlew.bat
57
+ - lib/embulk/guess/grok.rb
58
+ - lib/embulk/parser/grok.rb
59
+ - pattern/grok-patterns
60
+ - pattern/my-patterns
61
+ - sample/apache.yml
62
+ - sample/apache_stop.yml
63
+ - sample/guess.yml
64
+ - sample/multiline.yml
65
+ - src/main/java/org/embulk/parser/grok/DateParser.java
66
+ - src/main/java/org/embulk/parser/grok/GrokColumnVisitor.java
67
+ - src/main/java/org/embulk/parser/grok/GrokGuessPlugin.java
68
+ - src/main/java/org/embulk/parser/grok/GrokGuesser.java
69
+ - src/main/java/org/embulk/parser/grok/GrokParserPlugin.java
70
+ - src/main/java/org/embulk/parser/grok/GrokRecordIterator.java
71
+ - src/main/java/org/embulk/parser/grok/GrokRecordValidateException.java
72
+ - src/main/java/org/embulk/parser/grok/MultipleLineDecoder.java
73
+ - src/main/java/org/embulk/parser/grok/TimestampParserFactory.java
74
+ - src/test/java/org/embulk/parser/TestGrokGuessPlugin.java
75
+ - src/test/java/org/embulk/parser/TestGrokParserPlugin.java
76
+ - src/test/java/org/embulk/parser/TestGrokPluginBase.java
77
+ - src/test/java/org/embulk/util/EmbulkPluginTester.java
78
+ - src/test/java/org/embulk/util/StreamUtil.java
79
+ - src/test/resources/apache.log
80
+ - src/test/resources/apache.yml
81
+ - src/test/resources/apache_with_error.log
82
+ - src/test/resources/expected_apache.csv
83
+ - src/test/resources/expected_multiline.csv
84
+ - src/test/resources/guess.yml
85
+ - src/test/resources/multiline.log
86
+ - src/test/resources/multiline.yml
87
+ - classpath/embulk-parser-grok-0.1.6.jar
88
+ - classpath/grok-0.1.1.jar
89
+ - classpath/gson-2.2.2.jar
90
+ - classpath/named-regexp-0.2.3.jar
91
+ homepage: https://github.com/arielnetworks/embulk-parser-grok
92
+ licenses:
93
+ - MIT
94
+ metadata: {}
95
+ post_install_message:
96
+ rdoc_options: []
97
+ require_paths:
98
+ - lib
99
+ required_ruby_version: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - '>='
102
+ - !ruby/object:Gem::Version
103
+ version: '0'
104
+ required_rubygems_version: !ruby/object:Gem::Requirement
105
+ requirements:
106
+ - - '>='
107
+ - !ruby/object:Gem::Version
108
+ version: '0'
109
+ requirements: []
110
+ rubyforge_project:
111
+ rubygems_version: 2.1.9
112
+ signing_key:
113
+ specification_version: 4
114
+ summary: Grok parser plugin for Embulk
115
+ test_files: []