embulk-parser-grok 0.1.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +22 -0
  3. data/LICENSE.txt +21 -0
  4. data/README.md +113 -0
  5. data/build.gradle +95 -0
  6. data/gradle/wrapper/gradle-wrapper.jar +0 -0
  7. data/gradle/wrapper/gradle-wrapper.properties +6 -0
  8. data/gradlew +164 -0
  9. data/gradlew.bat +90 -0
  10. data/lib/embulk/guess/grok.rb +3 -0
  11. data/lib/embulk/parser/grok.rb +3 -0
  12. data/pattern/grok-patterns +105 -0
  13. data/pattern/my-patterns +12 -0
  14. data/sample/apache.yml +28 -0
  15. data/sample/apache_stop.yml +29 -0
  16. data/sample/guess.yml +19 -0
  17. data/sample/multiline.yml +19 -0
  18. data/src/main/java/org/embulk/parser/grok/DateParser.java +8 -0
  19. data/src/main/java/org/embulk/parser/grok/GrokColumnVisitor.java +98 -0
  20. data/src/main/java/org/embulk/parser/grok/GrokGuessPlugin.java +68 -0
  21. data/src/main/java/org/embulk/parser/grok/GrokGuesser.java +185 -0
  22. data/src/main/java/org/embulk/parser/grok/GrokParserPlugin.java +94 -0
  23. data/src/main/java/org/embulk/parser/grok/GrokRecordIterator.java +71 -0
  24. data/src/main/java/org/embulk/parser/grok/GrokRecordValidateException.java +11 -0
  25. data/src/main/java/org/embulk/parser/grok/MultipleLineDecoder.java +66 -0
  26. data/src/main/java/org/embulk/parser/grok/TimestampParserFactory.java +85 -0
  27. data/src/test/java/org/embulk/parser/TestGrokGuessPlugin.java +50 -0
  28. data/src/test/java/org/embulk/parser/TestGrokParserPlugin.java +55 -0
  29. data/src/test/java/org/embulk/parser/TestGrokPluginBase.java +69 -0
  30. data/src/test/java/org/embulk/util/EmbulkPluginTester.java +80 -0
  31. data/src/test/java/org/embulk/util/StreamUtil.java +30 -0
  32. data/src/test/resources/apache.log +101 -0
  33. data/src/test/resources/apache.yml +35 -0
  34. data/src/test/resources/apache_with_error.log +101 -0
  35. data/src/test/resources/expected_apache.csv +102 -0
  36. data/src/test/resources/expected_multiline.csv +45 -0
  37. data/src/test/resources/guess.yml +19 -0
  38. data/src/test/resources/multiline.log +44 -0
  39. data/src/test/resources/multiline.yml +25 -0
  40. metadata +115 -0
data/src/test/resources/expected_multiline.csv ADDED
@@ -0,0 +1,45 @@
1
+ timestamp|log_level|message|stack_trace
2
+ 2015-12-15 02:45:44.342000 +0000|INFO|(transaction): Loaded plugin embulk-parser-grok (0.1.5)|""
3
+ 2015-12-15 02:45:44.355000 +0000|INFO|(transaction): Listing local files at directory 'src/test/resources' filtering filename by prefix 'apache.log'|""
4
+ 2015-12-15 02:45:44.358000 +0000|INFO|(transaction): Loading files [src/test/resources/apache.log]|"org.embulk.exec.PartialExecutionException: org.embulk.config.ConfigException: com.fasterxml.jackson.core.JsonParseException: Unrecognized token 'ruby': was expecting ('true', 'false' or 'null')
5
+ at [Source: ruby; line: 1, column: 9]
6
+ at org.embulk.exec.BulkLoader$LoaderState.buildPartialExecuteException(org/embulk/exec/BulkLoader.java:363)
7
+ at org.embulk.exec.BulkLoader.doRun(org/embulk/exec/BulkLoader.java:572)
8
+ at org.embulk.exec.BulkLoader.access$000(org/embulk/exec/BulkLoader.java:33)
9
+ at org.embulk.exec.BulkLoader$1.run(org/embulk/exec/BulkLoader.java:374)
10
+ at org.embulk.exec.BulkLoader$1.run(org/embulk/exec/BulkLoader.java:370)
11
+ at org.embulk.spi.Exec.doWith(org/embulk/spi/Exec.java:25)
12
+ at org.embulk.exec.BulkLoader.run(org/embulk/exec/BulkLoader.java:370)
13
+ at org.embulk.EmbulkEmbed.run(org/embulk/EmbulkEmbed.java:180)
14
+ at java.lang.reflect.Method.invoke(java/lang/reflect/Method.java:497)
15
+ at RUBY.run(/home/ikezoe/.embulk/bin/embulk!/embulk/runner.rb:77)
16
+ at RUBY.run(/home/ikezoe/.embulk/bin/embulk!/embulk/command/embulk_run.rb:296)
17
+ at RUBY.<top>(/home/ikezoe/.embulk/bin/embulk!/embulk/command/embulk_main.rb:2)
18
+ at org.jruby.RubyKernel.require(org/jruby/RubyKernel.java:940)
19
+ at RUBY.(root)(uri:classloader:/META-INF/jruby.home/lib/ruby/stdlib/rubygems/core_ext/kernel_require.rb:1)
20
+ at home.ikezoe.$_dot_embulk.bin.embulk.embulk.command.embulk_bundle.<top>(file:/home/ikezoe/.embulk/bin/embulk!/embulk/command/embulk_bundle.rb:51)
21
+ at java.lang.invoke.MethodHandle.invokeWithArguments(java/lang/invoke/MethodHandle.java:627)
22
+ at org.embulk.cli.Main.main(org/embulk/cli/Main.java:23)
23
+ "
24
+ 2015-12-15 02:46:27.968000 +0000|INFO|(transaction): Loaded plugin embulk-parser-grok (0.1.5)|""
25
+ 2015-12-15 02:46:27.983000 +0000|INFO|(transaction): Listing local files at directory 'src/test/resources' filtering filename by prefix 'apache.log'|""
26
+ 2015-12-15 02:46:27.986000 +0000|INFO|(transaction): Loading files [src/test/resources/apache.log]|"org.embulk.exec.PartialExecutionException: org.embulk.config.ConfigException: com.fasterxml.jackson.core.JsonParseException: Unrecognized token 'ruby': was expecting ('true', 'false' or 'null')
27
+ at [Source: ruby; line: 1, column: 9]
28
+ at org.embulk.exec.BulkLoader$LoaderState.buildPartialExecuteException(org/embulk/exec/BulkLoader.java:363)
29
+ at org.embulk.exec.BulkLoader.doRun(org/embulk/exec/BulkLoader.java:572)
30
+ at org.embulk.exec.BulkLoader.access$000(org/embulk/exec/BulkLoader.java:33)
31
+ at org.embulk.exec.BulkLoader$1.run(org/embulk/exec/BulkLoader.java:374)
32
+ at org.embulk.exec.BulkLoader$1.run(org/embulk/exec/BulkLoader.java:370)
33
+ at org.embulk.spi.Exec.doWith(org/embulk/spi/Exec.java:25)
34
+ at org.embulk.exec.BulkLoader.run(org/embulk/exec/BulkLoader.java:370)
35
+ at org.embulk.EmbulkEmbed.run(org/embulk/EmbulkEmbed.java:180)
36
+ at java.lang.reflect.Method.invoke(java/lang/reflect/Method.java:497)
37
+ at RUBY.run(/home/ikezoe/.embulk/bin/embulk!/embulk/runner.rb:77)
38
+ at RUBY.run(/home/ikezoe/.embulk/bin/embulk!/embulk/command/embulk_run.rb:296)
39
+ at RUBY.<top>(/home/ikezoe/.embulk/bin/embulk!/embulk/command/embulk_main.rb:2)
40
+ at org.jruby.RubyKernel.require(org/jruby/RubyKernel.java:940)
41
+ at RUBY.(root)(uri:classloader:/META-INF/jruby.home/lib/ruby/stdlib/rubygems/core_ext/kernel_require.rb:1)
42
+ at home.ikezoe.$_dot_embulk.bin.embulk.embulk.command.embulk_bundle.<top>(file:/home/ikezoe/.embulk/bin/embulk!/embulk/command/embulk_bundle.rb:51)
43
+ at java.lang.invoke.MethodHandle.invokeWithArguments(java/lang/invoke/MethodHandle.java:627)
44
+ at org.embulk.cli.Main.main(org/embulk/cli/Main.java:23)
45
+ "
data/src/test/resources/guess.yml ADDED
@@ -0,0 +1,19 @@
1
+ in:
2
+ type: file
3
+ path_prefix: __INPUT_PATH__
4
+ parser:
5
+ charset: UTF-8
6
+ newline: CRLF
7
+ type: grok
8
+ grok_pattern_files:
9
+ - __PROJECT_DIR__/pattern/grok-patterns
10
+ - __PROJECT_DIR__/pattern/my-patterns
11
+ guess_patterns:
12
+ - "%{COMBINEDAPACHELOG}"
13
+ - "%{COMMONAPACHELOG}"
14
+ timestamp_parser: ruby
15
+ exec:
16
+ guess_plugins:
17
+ - "grok"
18
+ out:
19
+ type: stdout
data/src/test/resources/multiline.log ADDED
@@ -0,0 +1,44 @@
1
+ 2015-12-15 11:45:44.342 +0900 [INFO] (transaction): Loaded plugin embulk-parser-grok (0.1.5)
2
+ 2015-12-15 11:45:44.355 +0900 [INFO] (transaction): Listing local files at directory 'src/test/resources' filtering filename by prefix 'apache.log'
3
+ 2015-12-15 11:45:44.358 +0900 [INFO] (transaction): Loading files [src/test/resources/apache.log]
4
+ org.embulk.exec.PartialExecutionException: org.embulk.config.ConfigException: com.fasterxml.jackson.core.JsonParseException: Unrecognized token 'ruby': was expecting ('true', 'false' or 'null')
5
+ at [Source: ruby; line: 1, column: 9]
6
+ at org.embulk.exec.BulkLoader$LoaderState.buildPartialExecuteException(org/embulk/exec/BulkLoader.java:363)
7
+ at org.embulk.exec.BulkLoader.doRun(org/embulk/exec/BulkLoader.java:572)
8
+ at org.embulk.exec.BulkLoader.access$000(org/embulk/exec/BulkLoader.java:33)
9
+ at org.embulk.exec.BulkLoader$1.run(org/embulk/exec/BulkLoader.java:374)
10
+ at org.embulk.exec.BulkLoader$1.run(org/embulk/exec/BulkLoader.java:370)
11
+ at org.embulk.spi.Exec.doWith(org/embulk/spi/Exec.java:25)
12
+ at org.embulk.exec.BulkLoader.run(org/embulk/exec/BulkLoader.java:370)
13
+ at org.embulk.EmbulkEmbed.run(org/embulk/EmbulkEmbed.java:180)
14
+ at java.lang.reflect.Method.invoke(java/lang/reflect/Method.java:497)
15
+ at RUBY.run(/home/ikezoe/.embulk/bin/embulk!/embulk/runner.rb:77)
16
+ at RUBY.run(/home/ikezoe/.embulk/bin/embulk!/embulk/command/embulk_run.rb:296)
17
+ at RUBY.<top>(/home/ikezoe/.embulk/bin/embulk!/embulk/command/embulk_main.rb:2)
18
+ at org.jruby.RubyKernel.require(org/jruby/RubyKernel.java:940)
19
+ at RUBY.(root)(uri:classloader:/META-INF/jruby.home/lib/ruby/stdlib/rubygems/core_ext/kernel_require.rb:1)
20
+ at home.ikezoe.$_dot_embulk.bin.embulk.embulk.command.embulk_bundle.<top>(file:/home/ikezoe/.embulk/bin/embulk!/embulk/command/embulk_bundle.rb:51)
21
+ at java.lang.invoke.MethodHandle.invokeWithArguments(java/lang/invoke/MethodHandle.java:627)
22
+ at org.embulk.cli.Main.main(org/embulk/cli/Main.java:23)
23
+ 2015-12-15 11:46:27.968 +0900 [INFO] (transaction): Loaded plugin embulk-parser-grok (0.1.5)
24
+ 2015-12-15 11:46:27.983 +0900 [INFO] (transaction): Listing local files at directory 'src/test/resources' filtering filename by prefix 'apache.log'
25
+ 2015-12-15 11:46:27.986 +0900 [INFO] (transaction): Loading files [src/test/resources/apache.log]
26
+ org.embulk.exec.PartialExecutionException: org.embulk.config.ConfigException: com.fasterxml.jackson.core.JsonParseException: Unrecognized token 'ruby': was expecting ('true', 'false' or 'null')
27
+ at [Source: ruby; line: 1, column: 9]
28
+ at org.embulk.exec.BulkLoader$LoaderState.buildPartialExecuteException(org/embulk/exec/BulkLoader.java:363)
29
+ at org.embulk.exec.BulkLoader.doRun(org/embulk/exec/BulkLoader.java:572)
30
+ at org.embulk.exec.BulkLoader.access$000(org/embulk/exec/BulkLoader.java:33)
31
+ at org.embulk.exec.BulkLoader$1.run(org/embulk/exec/BulkLoader.java:374)
32
+ at org.embulk.exec.BulkLoader$1.run(org/embulk/exec/BulkLoader.java:370)
33
+ at org.embulk.spi.Exec.doWith(org/embulk/spi/Exec.java:25)
34
+ at org.embulk.exec.BulkLoader.run(org/embulk/exec/BulkLoader.java:370)
35
+ at org.embulk.EmbulkEmbed.run(org/embulk/EmbulkEmbed.java:180)
36
+ at java.lang.reflect.Method.invoke(java/lang/reflect/Method.java:497)
37
+ at RUBY.run(/home/ikezoe/.embulk/bin/embulk!/embulk/runner.rb:77)
38
+ at RUBY.run(/home/ikezoe/.embulk/bin/embulk!/embulk/command/embulk_run.rb:296)
39
+ at RUBY.<top>(/home/ikezoe/.embulk/bin/embulk!/embulk/command/embulk_main.rb:2)
40
+ at org.jruby.RubyKernel.require(org/jruby/RubyKernel.java:940)
41
+ at RUBY.(root)(uri:classloader:/META-INF/jruby.home/lib/ruby/stdlib/rubygems/core_ext/kernel_require.rb:1)
42
+ at home.ikezoe.$_dot_embulk.bin.embulk.embulk.command.embulk_bundle.<top>(file:/home/ikezoe/.embulk/bin/embulk!/embulk/command/embulk_bundle.rb:51)
43
+ at java.lang.invoke.MethodHandle.invokeWithArguments(java/lang/invoke/MethodHandle.java:627)
44
+ at org.embulk.cli.Main.main(org/embulk/cli/Main.java:23)
data/src/test/resources/multiline.yml ADDED
@@ -0,0 +1,25 @@
1
+ in:
2
+ type: file
3
+ path_prefix: __INPUT_PATH__
4
+ parser:
5
+ charset: UTF-8
6
+ newline: CRLF
7
+ type: grok
8
+ grok_pattern_files:
9
+ - __PROJECT_DIR__/pattern/grok-patterns
10
+ - __PROJECT_DIR__/pattern/my-patterns
11
+ first_line_pattern: '%{MULTILINELOG_FIRSTLINE}'
12
+ grok_pattern: '%{MULTILINELOG}'
13
+ columns:
14
+ - {name: timestamp, format: '%Y-%m-%d %H:%M:%S.%N %z', type: timestamp}
15
+ - {name: log_level, type: string}
16
+ - {name: message, type: string}
17
+ - {name: stack_trace, type: string}
18
+
19
+ out:
20
+ type: file
21
+ path_prefix: __OUTPUT_PATH__
22
+ file_ext: output.csv
23
+ formatter:
24
+ type: csv
25
+ delimiter: "|"
metadata ADDED
@@ -0,0 +1,115 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: embulk-parser-grok
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.6
5
+ platform: ruby
6
+ authors:
7
+ - Ikezoe Akihiro
8
+ - Rishi Kathera
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2015-12-15 00:00:00.000000000 Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ~>
18
+ - !ruby/object:Gem::Version
19
+ version: '1.0'
20
+ name: bundler
21
+ prerelease: false
22
+ type: :development
23
+ version_requirements: !ruby/object:Gem::Requirement
24
+ requirements:
25
+ - - ~>
26
+ - !ruby/object:Gem::Version
27
+ version: '1.0'
28
+ - !ruby/object:Gem::Dependency
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - '>='
32
+ - !ruby/object:Gem::Version
33
+ version: '10.0'
34
+ name: rake
35
+ prerelease: false
36
+ type: :development
37
+ version_requirements: !ruby/object:Gem::Requirement
38
+ requirements:
39
+ - - '>='
40
+ - !ruby/object:Gem::Version
41
+ version: '10.0'
42
+ description: Parses Grok files read by other file input plugins.
43
+ email:
44
+ - ikezoe_a@ariel-networks.com
45
+ executables: []
46
+ extensions: []
47
+ extra_rdoc_files: []
48
+ files:
49
+ - .gitignore
50
+ - LICENSE.txt
51
+ - README.md
52
+ - build.gradle
53
+ - gradle/wrapper/gradle-wrapper.jar
54
+ - gradle/wrapper/gradle-wrapper.properties
55
+ - gradlew
56
+ - gradlew.bat
57
+ - lib/embulk/guess/grok.rb
58
+ - lib/embulk/parser/grok.rb
59
+ - pattern/grok-patterns
60
+ - pattern/my-patterns
61
+ - sample/apache.yml
62
+ - sample/apache_stop.yml
63
+ - sample/guess.yml
64
+ - sample/multiline.yml
65
+ - src/main/java/org/embulk/parser/grok/DateParser.java
66
+ - src/main/java/org/embulk/parser/grok/GrokColumnVisitor.java
67
+ - src/main/java/org/embulk/parser/grok/GrokGuessPlugin.java
68
+ - src/main/java/org/embulk/parser/grok/GrokGuesser.java
69
+ - src/main/java/org/embulk/parser/grok/GrokParserPlugin.java
70
+ - src/main/java/org/embulk/parser/grok/GrokRecordIterator.java
71
+ - src/main/java/org/embulk/parser/grok/GrokRecordValidateException.java
72
+ - src/main/java/org/embulk/parser/grok/MultipleLineDecoder.java
73
+ - src/main/java/org/embulk/parser/grok/TimestampParserFactory.java
74
+ - src/test/java/org/embulk/parser/TestGrokGuessPlugin.java
75
+ - src/test/java/org/embulk/parser/TestGrokParserPlugin.java
76
+ - src/test/java/org/embulk/parser/TestGrokPluginBase.java
77
+ - src/test/java/org/embulk/util/EmbulkPluginTester.java
78
+ - src/test/java/org/embulk/util/StreamUtil.java
79
+ - src/test/resources/apache.log
80
+ - src/test/resources/apache.yml
81
+ - src/test/resources/apache_with_error.log
82
+ - src/test/resources/expected_apache.csv
83
+ - src/test/resources/expected_multiline.csv
84
+ - src/test/resources/guess.yml
85
+ - src/test/resources/multiline.log
86
+ - src/test/resources/multiline.yml
87
+ - classpath/embulk-parser-grok-0.1.6.jar
88
+ - classpath/grok-0.1.1.jar
89
+ - classpath/gson-2.2.2.jar
90
+ - classpath/named-regexp-0.2.3.jar
91
+ homepage: https://github.com/arielnetworks/embulk-parser-grok
92
+ licenses:
93
+ - MIT
94
+ metadata: {}
95
+ post_install_message:
96
+ rdoc_options: []
97
+ require_paths:
98
+ - lib
99
+ required_ruby_version: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - '>='
102
+ - !ruby/object:Gem::Version
103
+ version: '0'
104
+ required_rubygems_version: !ruby/object:Gem::Requirement
105
+ requirements:
106
+ - - '>='
107
+ - !ruby/object:Gem::Version
108
+ version: '0'
109
+ requirements: []
110
+ rubyforge_project:
111
+ rubygems_version: 2.1.9
112
+ signing_key:
113
+ specification_version: 4
114
+ summary: Grok parser plugin for Embulk
115
+ test_files: []