fluent-plugin-honeycomb 0.1.2 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (7) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +1 -0
  3. data/.travis.yml +10 -0
  4. data/README.md +11 -4
  5. data/fluent-plugin-honeycomb.gemspec +9 -5
  6. data/lib/fluent/plugin/out_honeycomb.rb +66 -9
  7. metadata +66 -10
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: ad45390b1dbaaa442c8a18d11a97697f88c57aaf
4
- data.tar.gz: 0a8132d267cfd025ac32af884495d0cf20ffa8ff
3
+ metadata.gz: 9e8b797012c9236ae425f1e7eedc6fa092ee3aba
4
+ data.tar.gz: 1ae8b3e70b777fdd3af766879f6be2c17d21cada
5
5
  SHA512:
6
- metadata.gz: 68b0efa95bbb025afe016de0b1b45769e04e8ee4afb7a035ffd3cca59a36131ae76f13bb69c1308983588507f842a5fc66dc221f34c5ad4db2799be5b38f78fb
7
- data.tar.gz: 61fe6988b6cfcad040db8110033b095d0674f9006bac2891d5e19b607874e96a4b63a3a32f01f70dd092ce22217874c4abb4a9b9d1dac2e93349ffd1b6ac8b7a
6
+ metadata.gz: d99adba6eb128a09a7596121536154bf36d08d9e5fe91cf7ab349ed53069106e79fce87f7e5c21c0b0dff2501dfa86f4b67d90650f9e4870cd4f0227c9e4dc16
7
+ data.tar.gz: 360a0037136d1f3c828406817853033858ee520756e28200a060c2b4ba6a10b457b34757fab02017892cf6e761b9b47606319a9725e4355cbeae164a1eb823f7
data/.gitignore CHANGED
@@ -1,2 +1,3 @@
1
1
  *.gem
2
2
  Gemfile.lock
3
+ coverage/
data/.travis.yml CHANGED
@@ -1,4 +1,14 @@
1
1
  language: ruby
2
2
 
3
3
  rvm:
4
+ - 2.1
4
5
  - 2.2
6
+ - 2.3.1
7
+
8
+ deploy:
9
+ provider: rubygems
10
+ api_key:
11
+ secure: XH0/YaToo890z8rZ4bqpYmzDw4KVzU1DJjVno6kG1z5ji3+yCd8lsHnb+5G096RBqPqFqlZdYxf5RX4Xb5NKufj5Ecn/5MAsTvnvUPVgFLIwGRy7c5HezrBJ3fciC2fGbnIi6QeI9KrGbfPhIQ1uuywwTOqnCd/8jguemejKrAGP32+OnrnHdD5I+bqTViNXbvoV3WVCEsXN7KCjNDgLJYmOzTk/0m79P7VOUD8VJsf2hT6TSnrIRemkg26UsVXzJ7au2QcRSCPsd/uPJ+AGlrhyGx0ua+hyIcRj/31n9cobsin+OsGOlACvC3CPIaE8hM7c99M7Xlp6YcPCS11huoPAXP/HAwBC1zgQOPd5XtUruHfwUcgDnxUmVo2treQ3hqcn6fZW2pEu8nCYh26jgAvTTS68a8VU8hgGcb5rHH5E9Vg6djAihlKrJPh5VYeoo+TkIaVOjFkyP1K1g9Opb5MJU59ECp7uItsFvzehXr1+0d+SUBwdFjMHSPUjyeWgt3yyfOwf2EnctOWnttgmxLLb0aivfuipZcFk+712fitBXAXoksTLl6/UjrYNrE0reVOgRY/xZ507ZGR5HpzjA9xcKQpK/A0Boqi1dCfV654um//wgzJESLkixrhGNh629uHamIdKLle53yn3htR5I5VEldhu3R4qb2wBtcJ0xhY=
12
+ gem: fluent-plugin-honeycomb
13
+ on:
14
+ tags: true
data/README.md CHANGED
@@ -1,3 +1,6 @@
1
+ [![Build Status](https://travis-ci.org/honeycombio/fluent-plugin-honeycomb.svg?branch=master)](https://travis-ci.org/honeycombio/fluent-plugin-honeycomb)
2
+
3
+ Send your logs to [Honeycomb](https://honeycomb.io).
1
4
 
2
5
  ## Getting started
3
6
 
@@ -29,10 +32,6 @@ Parameter | Type | Required? | Description
29
32
 
30
33
  `fluent-plugin-honeycomb` supports the [standard configuration options](http://docs.fluentd.org/v0.12/articles/buffer-plugin-overview) for buffered output plugins.
31
34
 
32
- ## Limitations
33
-
34
- `fluent-plugin-honeycomb` does not currently batch events. This functionality should be implemented in `libhoney-rb`.
35
-
36
35
  ## Development
37
36
  I recommend using [rbenv](https://github.com/rbenv/rbenv) for development.
38
37
 
@@ -50,3 +49,11 @@ fluentd -c ./fluent/fluent.conf -v
50
49
  ```
51
50
 
52
51
  A note about naming: This gem must be named `fluent-plugin-xxx` in order to automatically be included in Fluentd's plugin list. See http://www.fluentd.org/faqs.
52
+
53
+ ## Releasing a new version
54
+ Travis will automatically upload tagged releases to Rubygems. To release a new
55
+ version, run
56
+ ```
57
+ bump patch --tag # Or bump minor --tag, etc.
58
+ git push --tags
59
+ ```
data/fluent-plugin-honeycomb.gemspec CHANGED
@@ -1,10 +1,10 @@
1
1
  Gem::Specification.new do |spec|
2
2
  spec.name = 'fluent-plugin-honeycomb'
3
- spec.version = '0.1.2'
3
+ spec.version = '0.2.0'
4
4
  spec.date = '2017-02-07'
5
5
 
6
- spec.summary = "send data to Honeycomb"
7
- spec.description = "Ruby gem for sending data to Honeycomb"
6
+ spec.summary = "Fluentd output plugin for Honeycomb.io"
7
+ spec.description = "Fluentd output plugin for Honeycomb.io"
8
8
  spec.authors = ['The Honeycomb.io Team']
9
9
  spec.email = 'support@honeycomb.io'
10
10
  spec.files = []
@@ -16,13 +16,17 @@ Gem::Specification.new do |spec|
16
16
  end
17
17
  spec.require_paths = ["lib"]
18
18
 
19
- spec.required_ruby_version = '>= 2.2.0'
19
+ spec.required_ruby_version = '>= 2.1.0'
20
20
 
21
- spec.add_runtime_dependency "libhoney", "~> 1.0"
22
21
  spec.add_runtime_dependency "fluentd", "~> 0.12"
22
+ spec.add_runtime_dependency "http", "~> 2.0"
23
23
 
24
24
  spec.add_development_dependency "bundler", "~> 1.7"
25
25
  spec.add_development_dependency "rake", "~> 10.0"
26
26
  spec.add_development_dependency "webmock", "~> 2.1"
27
27
  spec.add_development_dependency "test-unit"
28
+ spec.add_development_dependency "simplecov"
29
+ spec.add_development_dependency "coveralls"
30
+ spec.add_development_dependency "bump"
31
+ spec.add_development_dependency "timecop"
28
32
  end
data/lib/fluent/plugin/out_honeycomb.rb CHANGED
@@ -1,5 +1,6 @@
1
+ require 'json'
2
+ require 'http'
1
3
  require 'fluent/output'
2
- require 'libhoney'
3
4
 
4
5
  module Fluent
5
6
  class HoneycombOutput < BufferedOutput
@@ -9,7 +10,7 @@ module Fluent
9
10
 
10
11
  # config_param defines a parameter. You can refer a parameter via @path instance variable
11
12
  # Without :default, a parameter is required.
12
- config_param :writekey, :string
13
+ config_param :writekey, :string, :secret => true
13
14
  config_param :dataset, :string
14
15
  config_param :sample_rate, :integer, :default => 1
15
16
  config_param :include_tag_key, :bool, :default => false
@@ -30,18 +31,12 @@ module Fluent
30
31
  # Open sockets or files here.
31
32
  def start
32
33
  super
33
- @client = Libhoney::Client.new(:writekey => @writekey,
34
- :dataset => @dataset,
35
- :sample_rate => @sample_rate,
36
- :api_host => @api_host)
37
34
  end
38
35
 
39
36
  # This method is called when shutting down.
40
37
  # Shutdown the thread and close sockets or files here.
41
38
  def shutdown
42
39
  super
43
- # Drain libhoney request queue before shutting down.
44
- @client.close(true)
45
40
  end
46
41
 
47
42
  def format(tag, time, record)
@@ -56,11 +51,73 @@ module Fluent
56
51
  #
57
52
  # NOTE! This method is called by internal thread, not Fluentd's main thread. So IO wait doesn't affect other plugins.
58
53
  def write(chunk)
54
+ batch = []
59
55
  chunk.msgpack_each do |(tag, time, record)|
56
+ if !record.is_a? Hash
57
+ log.debug "Skipping record #{record}"
58
+ next
59
+ end
60
60
  if @include_tag_key
61
61
  record[@tag_key] = tag
62
62
  end
63
- @client.send_now(record)
63
+ batch.push({
64
+ "data" => record,
65
+ "samplerate" => @sample_rate,
66
+ "time" => Time.at(time).utc.to_datetime.rfc3339
67
+ })
68
+ end
69
+
70
+ if batch.length == 0
71
+ return
72
+ end
73
+ log.info "publishing #{batch.length} records"
74
+ body = JSON.dump({ @dataset => batch })
75
+ resp = HTTP.headers(
76
+ "User-Agent" => "fluent-plugin-honeycomb",
77
+ "Content-Type" => "application/json",
78
+ "X-Honeycomb-Team" => @writekey)
79
+ .post(URI.join(@api_host, "/1/batch"), {
80
+ :body => body,
81
+ })
82
+ parse_response(resp)
83
+ end
84
+
85
+ def parse_response(resp)
86
+ if resp.status != 200
87
+ # Force retry
88
+ raise Exception.new("Error sending batch: #{resp.status}, #{resp.body}")
89
+ else
90
+ begin
91
+ results = JSON.parse(resp.body)
92
+ rescue JSON::ParserError => e
93
+ log.warn "Error parsing response as JSON: #{e}"
94
+ return
95
+ end
96
+ successes = 0
97
+ failures = []
98
+ results.each do |key, statuses|
99
+ if !statuses.is_a? Array
100
+ next
101
+ end
102
+
103
+ statuses.each do |s|
104
+ if !s.is_a? Hash
105
+ next
106
+ end
107
+
108
+ if s["status"] == 202
109
+ successes += 1
110
+ else
111
+ failures[r["status"]] += 1
112
+ end
113
+ end
114
+ end
115
+
116
+ if failures.size > 0
117
+ log.warn "Errors publishing records: #{failures}"
118
+ else
119
+ log.debug "Successfully published #{successes} records"
120
+ end
64
121
  end
65
122
  end
66
123
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-honeycomb
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.2
4
+ version: 0.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - The Honeycomb.io Team
@@ -11,33 +11,33 @@ cert_chain: []
11
11
  date: 2017-02-07 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
- name: libhoney
14
+ name: fluentd
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
17
  - - "~>"
18
18
  - !ruby/object:Gem::Version
19
- version: '1.0'
19
+ version: '0.12'
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - "~>"
25
25
  - !ruby/object:Gem::Version
26
- version: '1.0'
26
+ version: '0.12'
27
27
  - !ruby/object:Gem::Dependency
28
- name: fluentd
28
+ name: http
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
31
  - - "~>"
32
32
  - !ruby/object:Gem::Version
33
- version: '0.12'
33
+ version: '2.0'
34
34
  type: :runtime
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
38
  - - "~>"
39
39
  - !ruby/object:Gem::Version
40
- version: '0.12'
40
+ version: '2.0'
41
41
  - !ruby/object:Gem::Dependency
42
42
  name: bundler
43
43
  requirement: !ruby/object:Gem::Requirement
@@ -94,7 +94,63 @@ dependencies:
94
94
  - - ">="
95
95
  - !ruby/object:Gem::Version
96
96
  version: '0'
97
- description: Ruby gem for sending data to Honeycomb
97
+ - !ruby/object:Gem::Dependency
98
+ name: simplecov
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - ">="
102
+ - !ruby/object:Gem::Version
103
+ version: '0'
104
+ type: :development
105
+ prerelease: false
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - ">="
109
+ - !ruby/object:Gem::Version
110
+ version: '0'
111
+ - !ruby/object:Gem::Dependency
112
+ name: coveralls
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - ">="
116
+ - !ruby/object:Gem::Version
117
+ version: '0'
118
+ type: :development
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - ">="
123
+ - !ruby/object:Gem::Version
124
+ version: '0'
125
+ - !ruby/object:Gem::Dependency
126
+ name: bump
127
+ requirement: !ruby/object:Gem::Requirement
128
+ requirements:
129
+ - - ">="
130
+ - !ruby/object:Gem::Version
131
+ version: '0'
132
+ type: :development
133
+ prerelease: false
134
+ version_requirements: !ruby/object:Gem::Requirement
135
+ requirements:
136
+ - - ">="
137
+ - !ruby/object:Gem::Version
138
+ version: '0'
139
+ - !ruby/object:Gem::Dependency
140
+ name: timecop
141
+ requirement: !ruby/object:Gem::Requirement
142
+ requirements:
143
+ - - ">="
144
+ - !ruby/object:Gem::Version
145
+ version: '0'
146
+ type: :development
147
+ prerelease: false
148
+ version_requirements: !ruby/object:Gem::Requirement
149
+ requirements:
150
+ - - ">="
151
+ - !ruby/object:Gem::Version
152
+ version: '0'
153
+ description: Fluentd output plugin for Honeycomb.io
98
154
  email: support@honeycomb.io
99
155
  executables: []
100
156
  extensions: []
@@ -120,7 +176,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
120
176
  requirements:
121
177
  - - ">="
122
178
  - !ruby/object:Gem::Version
123
- version: 2.2.0
179
+ version: 2.1.0
124
180
  required_rubygems_version: !ruby/object:Gem::Requirement
125
181
  requirements:
126
182
  - - ">="
@@ -131,5 +187,5 @@ rubyforge_project:
131
187
  rubygems_version: 2.4.5
132
188
  signing_key:
133
189
  specification_version: 4
134
- summary: send data to Honeycomb
190
+ summary: Fluentd output plugin for Honeycomb.io
135
191
  test_files: []