logstash-input-couchdb_changes 0.1.3 → 0.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 1cec1015bb085e679dc5033cc661e358a0c6fa3a
4
- data.tar.gz: 739401b666711fc39f05acd316a8642f719c4efb
3
+ metadata.gz: 622600b5454a2bdfdee79bf629696c8f33a35d25
4
+ data.tar.gz: 3d5d9889d441642874e4c14638fb3ea55642d4c3
5
5
  SHA512:
6
- metadata.gz: 3efdcb1b88cae4999b14a4142acc102c2b969af06799ffa7c1a1a3a81b6231b3401c99ca2081f00477afc2c3036662dacc7f87b94b479db30664b989c35b2b98
7
- data.tar.gz: 06fba2fdbe31fa7dcb46ebcb72612d6b2584a4b548d75dff7dc5c3c6b8404d86432b91cb410720ce470b8eeb07273791db824dc6248bef8df293929eeb2a8b9e
6
+ metadata.gz: 0358d2db602dca21f2f6b53a9d0f92fa200415c3a42c07c4f0a21fdabe1f303dad208424cf3931cc2cfd0d6a89b4bc819ce20ec6e08b2e90c4f15f4617e833a7
7
+ data.tar.gz: cbd99b5c738b723005aa7c7066c40c75615e095921f8737d3ef66eba832e7eae0d414d2b6bc326155c127a8b17911e047ebbe95e272d5532768d8d4f4b5510d0
@@ -5,13 +5,13 @@ require "logstash/namespace"
5
5
  require "net/http"
6
6
  require "uri"
7
7
 
8
- # This CouchDB input allows you to automatically stream events from the
8
+ # This CouchDB input allows you to automatically stream events from the
9
9
  # CouchDB http://guide.couchdb.org/draft/notifications.html[_changes] URI.
10
10
  # Moreover, any "future" changes will automatically be streamed as well making it easy to synchronize
11
11
  # your CouchDB data with any target destination
12
12
  #
13
13
  # ### Upsert and delete
14
- # You can use event metadata to allow for document deletion.
14
+ # You can use event metadata to allow for document deletion.
15
15
  # All non-delete operations are treated as upserts
16
16
  #
17
17
  # ### Starting at a Specific Sequence
@@ -33,21 +33,21 @@ class LogStash::Inputs::CouchDBChanges < LogStash::Inputs::Base
33
33
  # Connect to CouchDB's _changes feed securely (via https)
34
34
  # Default: false (via http)
35
35
  config :secure, :validate => :boolean, :default => false
36
-
36
+
37
37
  # Path to a CA certificate file, used to validate certificates
38
38
  config :ca_file, :validate => :path
39
39
 
40
- # Username, if authentication is needed to connect to
40
+ # Username, if authentication is needed to connect to
41
41
  # CouchDB
42
42
  config :username, :validate => :string, :default => nil
43
43
 
44
- # Password, if authentication is needed to connect to
44
+ # Password, if authentication is needed to connect to
45
45
  # CouchDB
46
46
  config :password, :validate => :password, :default => nil
47
-
47
+
48
48
  # Logstash connects to CouchDB's _changes with feed=continuous
49
49
  # The heartbeat is how often (in milliseconds) Logstash will ping
50
- # CouchDB to ensure the connection is maintained. Changing this
50
+ # CouchDB to ensure the connection is maintained. Changing this
51
51
  # setting is not recommended unless you know what you are doing.
52
52
  config :heartbeat, :validate => :number, :default => 1000
53
53
 
@@ -58,33 +58,33 @@ class LogStash::Inputs::CouchDBChanges < LogStash::Inputs::Base
58
58
  # If unspecified, Logstash will attempt to read the last sequence number
59
59
  # from the `sequence_path` file. If that is empty or non-existent, it will
60
60
  # begin with 0 (the beginning).
61
- #
62
- # If you specify this value, it is anticipated that you will
61
+ #
62
+ # If you specify this value, it is anticipated that you will
63
63
  # only be doing so for an initial read under special circumstances
64
64
  # and that you will unset this value afterwards.
65
65
  config :initial_sequence, :validate => :number
66
-
66
+
67
67
  # Preserve the CouchDB document revision "_rev" value in the
68
68
  # output.
69
69
  config :keep_revision, :validate => :boolean, :default => false
70
-
71
- # Future feature! Until implemented, changing this from the default
70
+
71
+ # Future feature! Until implemented, changing this from the default
72
72
  # will not do anything.
73
73
  #
74
74
  # Ignore attachments associated with CouchDB documents.
75
75
  config :ignore_attachments, :validate => :boolean, :default => true
76
-
76
+
77
77
  # Reconnect flag. When true, always try to reconnect after a failure
78
78
  config :always_reconnect, :validate => :boolean, :default => true
79
-
79
+
80
80
  # Reconnect delay: time between reconnect attempts, in seconds.
81
81
  config :reconnect_delay, :validate => :number, :default => 10
82
-
82
+
83
83
  # Timeout: Number of milliseconds to wait for new data before
84
84
  # terminating the connection. If a timeout is set it will disable
85
85
  # the heartbeat configuration option.
86
86
  config :timeout, :validate => :number
87
-
87
+
88
88
  # Declare these constants here.
89
89
  FEED = 'continuous'
90
90
  INCLUDEDOCS = 'true'
@@ -114,14 +114,8 @@ class LogStash::Inputs::CouchDBChanges < LogStash::Inputs::Base
114
114
 
115
115
  @sequence = @initial_sequence ? @initial_sequence : @sequencedb.read
116
116
 
117
- if @username && @password
118
- @userinfo = @username + ':' + @password.value
119
- else
120
- @userinfo = nil
121
- end
122
-
123
117
  end
124
-
118
+
125
119
  module SequenceDB
126
120
  class File
127
121
  def initialize(file)
@@ -138,14 +132,16 @@ class LogStash::Inputs::CouchDBChanges < LogStash::Inputs::Base
138
132
  end
139
133
  end
140
134
  end
141
-
135
+
142
136
  public
143
137
  def run(queue)
144
138
  buffer = FileWatch::BufferedTokenizer.new
145
139
  @logger.info("Connecting to CouchDB _changes stream at:", :host => @host.to_s, :port => @port.to_s, :db => @db)
146
140
  uri = build_uri
147
141
  Net::HTTP.start(@host, @port, :use_ssl => (@secure == true), :ca_file => @ca_file) do |http|
142
+
148
143
  request = Net::HTTP::Get.new(uri.request_uri)
144
+ request.basic_auth(@username, @password.value) if @username && @password
149
145
  http.request request do |response|
150
146
  raise ArgumentError, "Database not found!" if response.code == "404"
151
147
  response.read_body do |chunk|
@@ -154,11 +150,11 @@ class LogStash::Inputs::CouchDBChanges < LogStash::Inputs::Base
154
150
  # sent as a sort of keep-alive. We should ignore those.
155
151
  next if changes.chomp.empty?
156
152
  if event = build_event(changes)
157
- @logger.debug("event", :event => event.to_hash_with_metadata) if @logger.debug?
158
- decorate(event)
159
- queue << event
160
- @sequence = event['@metadata']['seq']
161
- @sequencedb.write(@sequence.to_s)
153
+ @logger.debug("event", :event => event.to_hash_with_metadata) if @logger.debug?
154
+ decorate(event)
155
+ queue << event
156
+ @sequence = event['@metadata']['seq']
157
+ @sequencedb.write(@sequence.to_s)
162
158
  end
163
159
  end
164
160
  end
@@ -175,12 +171,12 @@ class LogStash::Inputs::CouchDBChanges < LogStash::Inputs::Base
175
171
  @logger.error("Unable to connect to database", :db => @db, :error => e.to_s)
176
172
  retry if reconnect?
177
173
  end
178
-
174
+
179
175
  private
180
176
  def build_uri
181
177
  options = {:feed => FEED, :include_docs => INCLUDEDOCS, :since => @sequence}
182
178
  options = options.merge(@timeout ? {:timeout => @timeout} : {:heartbeat => @heartbeat})
183
- URI::HTTP.build(:scheme => @scheme, :userinfo => @userinfo, :host => @host, :port => @port, :path => @path, :query => URI.encode_www_form(options))
179
+ URI::HTTP.build(:scheme => @scheme, :host => @host, :port => @port, :path => @path, :query => URI.encode_www_form(options))
184
180
  end
185
181
 
186
182
  private
@@ -1,10 +1,10 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-input-couchdb_changes'
4
- s.version = '0.1.3'
4
+ s.version = '0.1.4'
5
5
  s.licenses = ['Apache License (2.0)']
6
6
  s.summary = "This input captures the _changes stream from a CouchDB instance"
7
- s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
7
+ s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
8
8
  s.authors = ["Elasticsearch"]
9
9
  s.email = 'info@elasticsearch.com'
10
10
  s.homepage = "http://www.elasticsearch.org/guide/en/logstash/current/index.html"
@@ -29,4 +29,3 @@ Gem::Specification.new do |s|
29
29
  s.add_development_dependency 'logstash-output-elasticsearch'
30
30
 
31
31
  end
32
-
@@ -5,6 +5,13 @@ require "logstash/json"
5
5
  require "logstash/inputs/couchdb_changes"
6
6
 
7
7
  module Helpers
8
+ USERNAME = "logstash"
9
+ PASSWORD = "logstash"
10
+
11
+ def auth
12
+ "#{USERNAME}:#{PASSWORD}@"
13
+ end
14
+
8
15
  def createdb
9
16
  ftw = FTW::Agent.new
10
17
  ftw.put!("http://127.0.0.1:5984/db")
@@ -66,15 +73,17 @@ module Helpers
66
73
 
67
74
  def createuser
68
75
  ftw = FTW::Agent.new
69
- ftw.put!("http://127.0.0.1:5984/_config/admins/logstash", :body => '"logstash"')
76
+ ftw.put!("http://127.0.0.1:5984/_config/admins/#{USERNAME}", :body => "\"#{PASSWORD}\"")
70
77
  end
71
78
 
72
79
  def deleteuser
73
- user = "logstash"
74
- pass = "logstash"
75
- auth = "#{user}:#{pass}@"
76
80
  ftw = FTW::Agent.new
77
- ftw.delete!("http://#{auth}127.0.0.1:5984/_config/admins/logstash")
81
+ ftw.delete!("http://#{auth}127.0.0.1:5984/_config/admins/#{USERNAME}")
82
+ end
83
+
84
+ def addmember
85
+ ftw = FTW::Agent.new
86
+ ftw.put!("http://#{auth}127.0.0.1:5984/db/_security", :body => "{\"members\":{ \"names\":[\"#{USERNAME}\"]}}")
78
87
  end
79
88
 
80
89
  def deleteindex
@@ -98,7 +107,7 @@ module Helpers
98
107
  File.delete(sequence) if File.exist?(sequence)
99
108
  end
100
109
  end
101
-
110
+
102
111
  describe "inputs/couchdb_changes", :elasticsearch => true, :couchdb => true do
103
112
  describe "Load couchdb documents", :elasticsearch => true, :couchdb => true do
104
113
  include Helpers
@@ -132,7 +141,7 @@ describe "inputs/couchdb_changes", :elasticsearch => true, :couchdb => true do
132
141
  }
133
142
  }
134
143
  CONFIG
135
-
144
+
136
145
  agent do
137
146
  # Verify the count
138
147
  ftw.post!("http://127.0.0.1:9200/#{index}/_refresh")
@@ -166,7 +175,7 @@ describe "inputs/couchdb_changes", :elasticsearch => true, :couchdb => true do
166
175
  teardown
167
176
  end
168
177
  end
169
-
178
+
170
179
  describe "Test document updates", :elasticsearch => true, :couchdb => true do
171
180
  include Helpers
172
181
  sequence = "/tmp/.couchdb_seq"
@@ -200,7 +209,7 @@ describe "inputs/couchdb_changes", :elasticsearch => true, :couchdb => true do
200
209
  }
201
210
  }
202
211
  CONFIG
203
-
212
+
204
213
  agent do
205
214
  # Verify the count (which should still be 10)
206
215
  ftw.post!("http://127.0.0.1:9200/#{index}/_refresh")
@@ -237,7 +246,7 @@ describe "inputs/couchdb_changes", :elasticsearch => true, :couchdb => true do
237
246
  include Helpers
238
247
  sequence = "/tmp/.couchdb_seq"
239
248
  index = "couchdb_test"
240
-
249
+
241
250
  ftw = FTW::Agent.new
242
251
 
243
252
  config <<-CONFIG
@@ -317,7 +326,7 @@ describe "inputs/couchdb_changes", :elasticsearch => true, :couchdb => true do
317
326
  buildup
318
327
  deletedoc # from CouchDB
319
328
  end
320
-
329
+
321
330
  ftw = FTW::Agent.new
322
331
 
323
332
  config <<-CONFIG
@@ -341,7 +350,7 @@ describe "inputs/couchdb_changes", :elasticsearch => true, :couchdb => true do
341
350
  }
342
351
  }
343
352
  CONFIG
344
-
353
+
345
354
  agent do
346
355
  # Verify the count (should now be 9)
347
356
  ftw.post!("http://127.0.0.1:9200/#{index}/_refresh")
@@ -358,7 +367,7 @@ describe "inputs/couchdb_changes", :elasticsearch => true, :couchdb => true do
358
367
  result = LogStash::Json.load(data)
359
368
  insist { result["hits"]["hits"] }.any? { |doc| doc["_id"] == "9" }
360
369
  end
361
-
370
+
362
371
  after do
363
372
  teardown
364
373
  end
@@ -367,14 +376,14 @@ describe "inputs/couchdb_changes", :elasticsearch => true, :couchdb => true do
367
376
 
368
377
  describe "Test authenticated connectivity", :elasticsearch => true, :couchdb => true do
369
378
  include Helpers
370
- user = "logstash"
371
- pass = "logstash"
372
379
  sequence = "/tmp/.couchdb_seq"
373
380
  index = "couchdb_test"
374
381
 
382
+
375
383
  before do
376
384
  buildup
377
385
  createuser
386
+ addmember
378
387
  end
379
388
 
380
389
  ftw = FTW::Agent.new
@@ -388,8 +397,8 @@ describe "inputs/couchdb_changes", :elasticsearch => true, :couchdb => true do
388
397
  always_reconnect => false
389
398
  sequence_path => "#{sequence}"
390
399
  type => "couchdb"
391
- username => "#{user}"
392
- password => "#{pass}"
400
+ username => "#{Helpers::USERNAME}"
401
+ password => "#{Helpers::PASSWORD}"
393
402
  }
394
403
  }
395
404
  output {
@@ -402,7 +411,7 @@ describe "inputs/couchdb_changes", :elasticsearch => true, :couchdb => true do
402
411
  }
403
412
  }
404
413
  CONFIG
405
-
414
+
406
415
  agent do
407
416
  # Verify the count
408
417
  ftw.post!("http://127.0.0.1:9200/#{index}/_refresh")
@@ -423,7 +432,7 @@ describe "inputs/couchdb_changes", :elasticsearch => true, :couchdb => true do
423
432
  # verify the 'name' field
424
433
  insist { doc3["_source"]["name"] } == "Captain America"
425
434
  end
426
-
435
+
427
436
  after do
428
437
  deleteuser
429
438
  teardown
@@ -466,7 +475,7 @@ describe "inputs/couchdb_changes", :elasticsearch => true, :couchdb => true do
466
475
  }
467
476
  }
468
477
  CONFIG
469
-
478
+
470
479
  agent do
471
480
  # Verify the count
472
481
  ftw.post!("http://127.0.0.1:9200/#{index}/_refresh")
@@ -492,7 +501,7 @@ describe "inputs/couchdb_changes", :elasticsearch => true, :couchdb => true do
492
501
  teardown
493
502
  end
494
503
  end
495
-
504
+
496
505
  end
497
506
 
498
507
 
metadata CHANGED
@@ -1,17 +1,18 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-input-couchdb_changes
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.3
4
+ version: 0.1.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elasticsearch
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-02-26 00:00:00.000000000 Z
11
+ date: 2015-03-15 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
- requirement: !ruby/object:Gem::Requirement
14
+ name: logstash-core
15
+ version_requirements: !ruby/object:Gem::Requirement
15
16
  requirements:
16
17
  - - '>='
17
18
  - !ruby/object:Gem::Version
@@ -19,10 +20,7 @@ dependencies:
19
20
  - - <
20
21
  - !ruby/object:Gem::Version
21
22
  version: 2.0.0
22
- name: logstash-core
23
- prerelease: false
24
- type: :runtime
25
- version_requirements: !ruby/object:Gem::Requirement
23
+ requirement: !ruby/object:Gem::Requirement
26
24
  requirements:
27
25
  - - '>='
28
26
  - !ruby/object:Gem::Version
@@ -30,76 +28,78 @@ dependencies:
30
28
  - - <
31
29
  - !ruby/object:Gem::Version
32
30
  version: 2.0.0
31
+ prerelease: false
32
+ type: :runtime
33
33
  - !ruby/object:Gem::Dependency
34
+ name: logstash-codec-plain
35
+ version_requirements: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - '>='
38
+ - !ruby/object:Gem::Version
39
+ version: '0'
34
40
  requirement: !ruby/object:Gem::Requirement
35
41
  requirements:
36
42
  - - '>='
37
43
  - !ruby/object:Gem::Version
38
44
  version: '0'
39
- name: logstash-codec-plain
40
45
  prerelease: false
41
46
  type: :runtime
47
+ - !ruby/object:Gem::Dependency
48
+ name: json
42
49
  version_requirements: !ruby/object:Gem::Requirement
43
50
  requirements:
44
51
  - - '>='
45
52
  - !ruby/object:Gem::Version
46
53
  version: '0'
47
- - !ruby/object:Gem::Dependency
48
54
  requirement: !ruby/object:Gem::Requirement
49
55
  requirements:
50
56
  - - '>='
51
57
  - !ruby/object:Gem::Version
52
58
  version: '0'
53
- name: json
54
59
  prerelease: false
55
60
  type: :runtime
61
+ - !ruby/object:Gem::Dependency
62
+ name: ftw
56
63
  version_requirements: !ruby/object:Gem::Requirement
57
64
  requirements:
58
- - - '>='
65
+ - - ~>
59
66
  - !ruby/object:Gem::Version
60
- version: '0'
61
- - !ruby/object:Gem::Dependency
67
+ version: 0.0.42
62
68
  requirement: !ruby/object:Gem::Requirement
63
69
  requirements:
64
70
  - - ~>
65
71
  - !ruby/object:Gem::Version
66
72
  version: 0.0.42
67
- name: ftw
68
73
  prerelease: false
69
74
  type: :development
75
+ - !ruby/object:Gem::Dependency
76
+ name: logstash-devutils
70
77
  version_requirements: !ruby/object:Gem::Requirement
71
78
  requirements:
72
- - - ~>
79
+ - - '>='
73
80
  - !ruby/object:Gem::Version
74
- version: 0.0.42
75
- - !ruby/object:Gem::Dependency
81
+ version: 0.0.6
76
82
  requirement: !ruby/object:Gem::Requirement
77
83
  requirements:
78
84
  - - '>='
79
85
  - !ruby/object:Gem::Version
80
86
  version: 0.0.6
81
- name: logstash-devutils
82
87
  prerelease: false
83
88
  type: :development
89
+ - !ruby/object:Gem::Dependency
90
+ name: logstash-output-elasticsearch
84
91
  version_requirements: !ruby/object:Gem::Requirement
85
92
  requirements:
86
93
  - - '>='
87
94
  - !ruby/object:Gem::Version
88
- version: 0.0.6
89
- - !ruby/object:Gem::Dependency
95
+ version: '0'
90
96
  requirement: !ruby/object:Gem::Requirement
91
97
  requirements:
92
98
  - - '>='
93
99
  - !ruby/object:Gem::Version
94
100
  version: '0'
95
- name: logstash-output-elasticsearch
96
101
  prerelease: false
97
102
  type: :development
98
- version_requirements: !ruby/object:Gem::Requirement
99
- requirements:
100
- - - '>='
101
- - !ruby/object:Gem::Version
102
- version: '0'
103
103
  description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
104
104
  email: info@elasticsearch.com
105
105
  executables: []
@@ -141,7 +141,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
141
141
  version: '0'
142
142
  requirements: []
143
143
  rubyforge_project:
144
- rubygems_version: 2.4.5
144
+ rubygems_version: 2.1.9
145
145
  signing_key:
146
146
  specification_version: 4
147
147
  summary: This input captures the _changes stream from a CouchDB instance