fluent-plugin-honeycomb 0.1.2 → 0.2.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (7) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +1 -0
  3. data/.travis.yml +10 -0
  4. data/README.md +11 -4
  5. data/fluent-plugin-honeycomb.gemspec +9 -5
  6. data/lib/fluent/plugin/out_honeycomb.rb +66 -9
  7. metadata +66 -10
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: ad45390b1dbaaa442c8a18d11a97697f88c57aaf
4
- data.tar.gz: 0a8132d267cfd025ac32af884495d0cf20ffa8ff
3
+ metadata.gz: 9e8b797012c9236ae425f1e7eedc6fa092ee3aba
4
+ data.tar.gz: 1ae8b3e70b777fdd3af766879f6be2c17d21cada
5
5
  SHA512:
6
- metadata.gz: 68b0efa95bbb025afe016de0b1b45769e04e8ee4afb7a035ffd3cca59a36131ae76f13bb69c1308983588507f842a5fc66dc221f34c5ad4db2799be5b38f78fb
7
- data.tar.gz: 61fe6988b6cfcad040db8110033b095d0674f9006bac2891d5e19b607874e96a4b63a3a32f01f70dd092ce22217874c4abb4a9b9d1dac2e93349ffd1b6ac8b7a
6
+ metadata.gz: d99adba6eb128a09a7596121536154bf36d08d9e5fe91cf7ab349ed53069106e79fce87f7e5c21c0b0dff2501dfa86f4b67d90650f9e4870cd4f0227c9e4dc16
7
+ data.tar.gz: 360a0037136d1f3c828406817853033858ee520756e28200a060c2b4ba6a10b457b34757fab02017892cf6e761b9b47606319a9725e4355cbeae164a1eb823f7
data/.gitignore CHANGED
@@ -1,2 +1,3 @@
1
1
  *.gem
2
2
  Gemfile.lock
3
+ coverage/
data/.travis.yml CHANGED
@@ -1,4 +1,14 @@
1
1
  language: ruby
2
2
 
3
3
  rvm:
4
+ - 2.1
4
5
  - 2.2
6
+ - 2.3.1
7
+
8
+ deploy:
9
+ provider: rubygems
10
+ api_key:
11
+ secure: XH0/YaToo890z8rZ4bqpYmzDw4KVzU1DJjVno6kG1z5ji3+yCd8lsHnb+5G096RBqPqFqlZdYxf5RX4Xb5NKufj5Ecn/5MAsTvnvUPVgFLIwGRy7c5HezrBJ3fciC2fGbnIi6QeI9KrGbfPhIQ1uuywwTOqnCd/8jguemejKrAGP32+OnrnHdD5I+bqTViNXbvoV3WVCEsXN7KCjNDgLJYmOzTk/0m79P7VOUD8VJsf2hT6TSnrIRemkg26UsVXzJ7au2QcRSCPsd/uPJ+AGlrhyGx0ua+hyIcRj/31n9cobsin+OsGOlACvC3CPIaE8hM7c99M7Xlp6YcPCS11huoPAXP/HAwBC1zgQOPd5XtUruHfwUcgDnxUmVo2treQ3hqcn6fZW2pEu8nCYh26jgAvTTS68a8VU8hgGcb5rHH5E9Vg6djAihlKrJPh5VYeoo+TkIaVOjFkyP1K1g9Opb5MJU59ECp7uItsFvzehXr1+0d+SUBwdFjMHSPUjyeWgt3yyfOwf2EnctOWnttgmxLLb0aivfuipZcFk+712fitBXAXoksTLl6/UjrYNrE0reVOgRY/xZ507ZGR5HpzjA9xcKQpK/A0Boqi1dCfV654um//wgzJESLkixrhGNh629uHamIdKLle53yn3htR5I5VEldhu3R4qb2wBtcJ0xhY=
12
+ gem: fluent-plugin-honeycomb
13
+ on:
14
+ tags: true
data/README.md CHANGED
@@ -1,3 +1,6 @@
1
+ [![Build Status](https://travis-ci.org/honeycombio/fluent-plugin-honeycomb.svg?branch=master)](https://travis-ci.org/honeycombio/fluent-plugin-honeycomb)
2
+
3
+ Send your logs to [Honeycomb](https://honeycomb.io).
1
4
 
2
5
  ## Getting started
3
6
 
@@ -29,10 +32,6 @@ Parameter | Type | Required? | Description
29
32
 
30
33
  `fluent-plugin-honeycomb` supports the [standard configuration options](http://docs.fluentd.org/v0.12/articles/buffer-plugin-overview) for buffered output plugins.
31
34
 
32
- ## Limitations
33
-
34
- `fluent-plugin-honeycomb` does not currently batch events. This functionality should be implemented in `libhoney-rb`.
35
-
36
35
  ## Development
37
36
  I recommend using [rbenv](https://github.com/rbenv/rbenv) for development.
38
37
 
@@ -50,3 +49,11 @@ fluentd -c ./fluent/fluent.conf -v
50
49
  ```
51
50
 
52
51
  A note about naming: This gem must be named `fluent-plugin-xxx` in order to automatically be included in Fluentd's plugin list. See http://www.fluentd.org/faqs.
52
+
53
+ ## Releasing a new version
54
+ Travis will automatically upload tagged releases to Rubygems. To release a new
55
+ version, run
56
+ ```
57
+ bump patch --tag # Or bump minor --tag, etc.
58
+ git push --tags
59
+ ```
data/fluent-plugin-honeycomb.gemspec CHANGED
@@ -1,10 +1,10 @@
1
1
  Gem::Specification.new do |spec|
2
2
  spec.name = 'fluent-plugin-honeycomb'
3
- spec.version = '0.1.2'
3
+ spec.version = '0.2.0'
4
4
  spec.date = '2017-02-07'
5
5
 
6
- spec.summary = "send data to Honeycomb"
7
- spec.description = "Ruby gem for sending data to Honeycomb"
6
+ spec.summary = "Fluentd output plugin for Honeycomb.io"
7
+ spec.description = "Fluentd output plugin for Honeycomb.io"
8
8
  spec.authors = ['The Honeycomb.io Team']
9
9
  spec.email = 'support@honeycomb.io'
10
10
  spec.files = []
@@ -16,13 +16,17 @@ Gem::Specification.new do |spec|
16
16
  end
17
17
  spec.require_paths = ["lib"]
18
18
 
19
- spec.required_ruby_version = '>= 2.2.0'
19
+ spec.required_ruby_version = '>= 2.1.0'
20
20
 
21
- spec.add_runtime_dependency "libhoney", "~> 1.0"
22
21
  spec.add_runtime_dependency "fluentd", "~> 0.12"
22
+ spec.add_runtime_dependency "http", "~> 2.0"
23
23
 
24
24
  spec.add_development_dependency "bundler", "~> 1.7"
25
25
  spec.add_development_dependency "rake", "~> 10.0"
26
26
  spec.add_development_dependency "webmock", "~> 2.1"
27
27
  spec.add_development_dependency "test-unit"
28
+ spec.add_development_dependency "simplecov"
29
+ spec.add_development_dependency "coveralls"
30
+ spec.add_development_dependency "bump"
31
+ spec.add_development_dependency "timecop"
28
32
  end
data/lib/fluent/plugin/out_honeycomb.rb CHANGED
@@ -1,5 +1,6 @@
1
+ require 'json'
2
+ require 'http'
1
3
  require 'fluent/output'
2
- require 'libhoney'
3
4
 
4
5
  module Fluent
5
6
  class HoneycombOutput < BufferedOutput
@@ -9,7 +10,7 @@ module Fluent
9
10
 
10
11
  # config_param defines a parameter. You can refer a parameter via @path instance variable
11
12
  # Without :default, a parameter is required.
12
- config_param :writekey, :string
13
+ config_param :writekey, :string, :secret => true
13
14
  config_param :dataset, :string
14
15
  config_param :sample_rate, :integer, :default => 1
15
16
  config_param :include_tag_key, :bool, :default => false
@@ -30,18 +31,12 @@ module Fluent
30
31
  # Open sockets or files here.
31
32
  def start
32
33
  super
33
- @client = Libhoney::Client.new(:writekey => @writekey,
34
- :dataset => @dataset,
35
- :sample_rate => @sample_rate,
36
- :api_host => @api_host)
37
34
  end
38
35
 
39
36
  # This method is called when shutting down.
40
37
  # Shutdown the thread and close sockets or files here.
41
38
  def shutdown
42
39
  super
43
- # Drain libhoney request queue before shutting down.
44
- @client.close(true)
45
40
  end
46
41
 
47
42
  def format(tag, time, record)
@@ -56,11 +51,73 @@ module Fluent
56
51
  #
57
52
  # NOTE! This method is called by internal thread, not Fluentd's main thread. So IO wait doesn't affect other plugins.
58
53
  def write(chunk)
54
+ batch = []
59
55
  chunk.msgpack_each do |(tag, time, record)|
56
+ if !record.is_a? Hash
57
+ log.debug "Skipping record #{record}"
58
+ next
59
+ end
60
60
  if @include_tag_key
61
61
  record[@tag_key] = tag
62
62
  end
63
- @client.send_now(record)
63
+ batch.push({
64
+ "data" => record,
65
+ "samplerate" => @sample_rate,
66
+ "time" => Time.at(time).utc.to_datetime.rfc3339
67
+ })
68
+ end
69
+
70
+ if batch.length == 0
71
+ return
72
+ end
73
+ log.info "publishing #{batch.length} records"
74
+ body = JSON.dump({ @dataset => batch })
75
+ resp = HTTP.headers(
76
+ "User-Agent" => "fluent-plugin-honeycomb",
77
+ "Content-Type" => "application/json",
78
+ "X-Honeycomb-Team" => @writekey)
79
+ .post(URI.join(@api_host, "/1/batch"), {
80
+ :body => body,
81
+ })
82
+ parse_response(resp)
83
+ end
84
+
85
+ def parse_response(resp)
86
+ if resp.status != 200
87
+ # Force retry
88
+ raise Exception.new("Error sending batch: #{resp.status}, #{resp.body}")
89
+ else
90
+ begin
91
+ results = JSON.parse(resp.body)
92
+ rescue JSON::ParserError => e
93
+ log.warn "Error parsing response as JSON: #{e}"
94
+ return
95
+ end
96
+ successes = 0
97
+ failures = []
98
+ results.each do |key, statuses|
99
+ if !statuses.is_a? Array
100
+ next
101
+ end
102
+
103
+ statuses.each do |s|
104
+ if !s.is_a? Hash
105
+ next
106
+ end
107
+
108
+ if s["status"] == 202
109
+ successes += 1
110
+ else
111
+ failures[r["status"]] += 1
112
+ end
113
+ end
114
+ end
115
+
116
+ if failures.size > 0
117
+ log.warn "Errors publishing records: #{failures}"
118
+ else
119
+ log.debug "Successfully published #{successes} records"
120
+ end
64
121
  end
65
122
  end
66
123
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-honeycomb
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.2
4
+ version: 0.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - The Honeycomb.io Team
@@ -11,33 +11,33 @@ cert_chain: []
11
11
  date: 2017-02-07 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
- name: libhoney
14
+ name: fluentd
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
17
  - - "~>"
18
18
  - !ruby/object:Gem::Version
19
- version: '1.0'
19
+ version: '0.12'
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - "~>"
25
25
  - !ruby/object:Gem::Version
26
- version: '1.0'
26
+ version: '0.12'
27
27
  - !ruby/object:Gem::Dependency
28
- name: fluentd
28
+ name: http
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
31
  - - "~>"
32
32
  - !ruby/object:Gem::Version
33
- version: '0.12'
33
+ version: '2.0'
34
34
  type: :runtime
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
38
  - - "~>"
39
39
  - !ruby/object:Gem::Version
40
- version: '0.12'
40
+ version: '2.0'
41
41
  - !ruby/object:Gem::Dependency
42
42
  name: bundler
43
43
  requirement: !ruby/object:Gem::Requirement
@@ -94,7 +94,63 @@ dependencies:
94
94
  - - ">="
95
95
  - !ruby/object:Gem::Version
96
96
  version: '0'
97
- description: Ruby gem for sending data to Honeycomb
97
+ - !ruby/object:Gem::Dependency
98
+ name: simplecov
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - ">="
102
+ - !ruby/object:Gem::Version
103
+ version: '0'
104
+ type: :development
105
+ prerelease: false
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - ">="
109
+ - !ruby/object:Gem::Version
110
+ version: '0'
111
+ - !ruby/object:Gem::Dependency
112
+ name: coveralls
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - ">="
116
+ - !ruby/object:Gem::Version
117
+ version: '0'
118
+ type: :development
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - ">="
123
+ - !ruby/object:Gem::Version
124
+ version: '0'
125
+ - !ruby/object:Gem::Dependency
126
+ name: bump
127
+ requirement: !ruby/object:Gem::Requirement
128
+ requirements:
129
+ - - ">="
130
+ - !ruby/object:Gem::Version
131
+ version: '0'
132
+ type: :development
133
+ prerelease: false
134
+ version_requirements: !ruby/object:Gem::Requirement
135
+ requirements:
136
+ - - ">="
137
+ - !ruby/object:Gem::Version
138
+ version: '0'
139
+ - !ruby/object:Gem::Dependency
140
+ name: timecop
141
+ requirement: !ruby/object:Gem::Requirement
142
+ requirements:
143
+ - - ">="
144
+ - !ruby/object:Gem::Version
145
+ version: '0'
146
+ type: :development
147
+ prerelease: false
148
+ version_requirements: !ruby/object:Gem::Requirement
149
+ requirements:
150
+ - - ">="
151
+ - !ruby/object:Gem::Version
152
+ version: '0'
153
+ description: Fluentd output plugin for Honeycomb.io
98
154
  email: support@honeycomb.io
99
155
  executables: []
100
156
  extensions: []
@@ -120,7 +176,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
120
176
  requirements:
121
177
  - - ">="
122
178
  - !ruby/object:Gem::Version
123
- version: 2.2.0
179
+ version: 2.1.0
124
180
  required_rubygems_version: !ruby/object:Gem::Requirement
125
181
  requirements:
126
182
  - - ">="
@@ -131,5 +187,5 @@ rubyforge_project:
131
187
  rubygems_version: 2.4.5
132
188
  signing_key:
133
189
  specification_version: 4
134
- summary: send data to Honeycomb
190
+ summary: Fluentd output plugin for Honeycomb.io
135
191
  test_files: []