logstash-input-azure_blob_storage 0.11.0 → 0.11.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +5 -0
- data/lib/logstash/inputs/azure_blob_storage.rb +18 -17
- data/logstash-input-azure_blob_storage.gemspec +1 -1
- metadata +3 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5fb68f13f46e7a0455fe4ffd3f6c9e04b136611e01504310bd739bbc6813c6f6
|
4
|
+
data.tar.gz: 3f818813b0b45acac96edb34a4948d01c234946fb2580eefe5ece8e43240c0c1
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b596bbfc6a1e3400c33e54bbfa4adb753ea1c6593ae647da221368a089b25cd650856d4abb78c5f39ae39df67387b3de938962d63a90e06d2b54164599ced0a9
|
7
|
+
data.tar.gz: 6c0eb3959fa0f393f63c0697f26d49b01280604e2443b8c0a17342d768f4c1a9402e4c8f658462cdd1d76cc11fa7a654bb36052ff05fdbec775050ad33539a1c
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,8 @@
|
|
1
|
+
## 0.11.0
|
2
|
+
- Implemented start_fresh to skip all previous logs and start monitoring new entries
|
3
|
+
- Fixed the timer, now properly sleep the interval and check again
|
4
|
+
- Work around for a Faraday Middleware v.s. Azure Storage Account bug in follow_redirect
|
5
|
+
|
1
6
|
## 0.10.6
|
2
7
|
- Fixed the rootcause of the checking the codec. Now compare the classname.
|
3
8
|
|
@@ -91,8 +91,6 @@ def register
|
|
91
91
|
@processed = 0
|
92
92
|
@regsaved = @processed
|
93
93
|
|
94
|
-
#@buffer = FileWatch::BufferedTokenizer.new('\n')
|
95
|
-
|
96
94
|
# Try in this order to access the storageaccount
|
97
95
|
# 1. storageaccount / sas_token
|
98
96
|
# 2. connection_string
|
@@ -120,22 +118,22 @@ def register
|
|
120
118
|
|
121
119
|
@registry = Hash.new
|
122
120
|
if registry_create_policy == "resume"
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
121
|
+
@logger.info(@pipe_id+" resuming from registry")
|
122
|
+
for counter in 0..3
|
123
|
+
begin
|
124
|
+
@registry = Marshal.load(@blob_client.get_blob(container, registry_path)[1])
|
125
|
+
#[0] headers [1] responsebody
|
126
|
+
rescue Exception => e
|
127
|
+
@logger.error(@pipe_id+" caught: #{e.message}")
|
128
|
+
@registry.clear
|
129
|
+
@logger.error(@pipe_id+" loading registry failed, starting over")
|
130
|
+
end
|
130
131
|
end
|
131
132
|
end
|
132
133
|
# read filelist and set offsets to file length to mark all the old files as done
|
133
134
|
if registry_create_policy == "start_fresh"
|
134
135
|
@logger.info(@pipe_id+" starting fresh")
|
135
136
|
@registry = list_blobs(true)
|
136
|
-
#tempreg.each do |name, file|
|
137
|
-
# @registry.store(name, { :offset => file[:length], :length => file[:length] })
|
138
|
-
#end
|
139
137
|
end
|
140
138
|
|
141
139
|
@is_json = false
|
@@ -168,8 +166,8 @@ def run(queue)
|
|
168
166
|
# we can abort the loop if stop? becomes true
|
169
167
|
while !stop?
|
170
168
|
chrono = Time.now.to_i
|
171
|
-
# load
|
172
|
-
# TODO: sort by timestamp
|
169
|
+
# load the registry, compare it's offsets to file list, set offset to 0 for new files, process the whole list and if finished within the interval wait for next loop,
|
170
|
+
# TODO: sort by timestamp ?
|
173
171
|
#filelist.sort_by(|k,v|resource(k)[:date])
|
174
172
|
worklist.clear
|
175
173
|
filelist.clear
|
@@ -192,7 +190,7 @@ def run(queue)
|
|
192
190
|
# This would be ideal for threading since it's IO intensive, would be nice with a ruby native ThreadPool
|
193
191
|
worklist.each do |name, file|
|
194
192
|
#res = resource(name)
|
195
|
-
@logger.
|
193
|
+
@logger.debug(@pipe_id+" processing #{name} from #{file[:offset]} to #{file[:length]}")
|
196
194
|
size = 0
|
197
195
|
if file[:offset] == 0
|
198
196
|
chunk = full_read(name)
|
@@ -224,6 +222,8 @@ def run(queue)
|
|
224
222
|
@processed += counter
|
225
223
|
end
|
226
224
|
@registry.store(name, { :offset => size, :length => file[:length] })
|
225
|
+
# TODO add input plugin option to prevent connection cache
|
226
|
+
@blob_client.client.reset_agents!
|
227
227
|
#@logger.info(@pipe_id+" name #{name} size #{size} len #{file[:length]}")
|
228
228
|
# if stop? good moment to stop what we're doing
|
229
229
|
if stop?
|
@@ -246,7 +246,9 @@ end
|
|
246
246
|
def stop
|
247
247
|
save_registry(@registry)
|
248
248
|
end
|
249
|
-
|
249
|
+
def close
|
250
|
+
save_registry(@registry)
|
251
|
+
end
|
250
252
|
|
251
253
|
|
252
254
|
private
|
@@ -371,7 +373,6 @@ def learn_encapsulation
|
|
371
373
|
# From one file, read first block and last block to learn head and tail
|
372
374
|
# If the blobstorage can't be found, an error from farraday middleware will come with the text
|
373
375
|
# org.jruby.ext.set.RubySet cannot be cast to class org.jruby.RubyFixnum
|
374
|
-
# implement options ... prefix may ot exist!
|
375
376
|
blob = @blob_client.list_blobs(container, { maxresults: 1, prefix: @prefix }).first
|
376
377
|
return if blob.nil?
|
377
378
|
blocks = @blob_client.list_blob_blocks(container, blob.name)[:committed]
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-azure_blob_storage
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.11.
|
4
|
+
version: 0.11.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jan Geertsma
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-11-
|
11
|
+
date: 2019-11-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -112,8 +112,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
112
112
|
- !ruby/object:Gem::Version
|
113
113
|
version: '0'
|
114
114
|
requirements: []
|
115
|
-
|
116
|
-
rubygems_version: 2.7.9
|
115
|
+
rubygems_version: 3.0.6
|
117
116
|
signing_key:
|
118
117
|
specification_version: 4
|
119
118
|
summary: This logstash plugin reads and parses data from Azure Storage Blobs.
|