logstash-input-sqlite 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,15 @@
1
+ ---
2
+ !binary "U0hBMQ==":
3
+ metadata.gz: !binary |-
4
+ YmQ0ZTFlMzE0ZWRmNTFkY2U2MjY3NGExNzcxMDBhMjZhZDE1NTllMg==
5
+ data.tar.gz: !binary |-
6
+ NmNmMWY4ZmQ3NDBjMzA1ZTI5ZDY2OTEyOTYzZDE1YWMwOWI0ODhhMg==
7
+ SHA512:
8
+ metadata.gz: !binary |-
9
+ NmQyMGMyMDE2YzRiMzFhZmQ3Yzc3MGUwOGQyYjA1MThhM2VhNWM5MmM4NWRl
10
+ YjNiNTMzM2E0MmIzMWIyNGJhNTFlNmQwZDZhNzA1MzY5ODRmZWIyNTIzZGNk
11
+ Nzc2ZWFhZTg5YTUwZTRkYjJhODczZTEzOWI2NTdiZjNiMmNlOTY=
12
+ data.tar.gz: !binary |-
13
+ NDE3OTBjMzM3Y2U0OWJkZjM3MjFjZmRjYTUwOGE3ODY2NzU4Njc3Njg4MjVi
14
+ YWE0MzI5Y2E4MzBhYjhlYjViN2VjMjIyNmY0NmVhZmIzZGM2ZjFkZDNkNzgy
15
+ MzliYWVjMmE5YmE2M2I1MDliMWYxNGU2ZGQ0ZTRkNzNkY2FlNzA=
data/.gitignore ADDED
@@ -0,0 +1,4 @@
1
+ *.gem
2
+ Gemfile.lock
3
+ .bundle
4
+ vendor
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'http://rubygems.org'
2
+ gem 'rake'
3
+ gem 'gem_publisher'
4
+ gem 'archive-tar-minitar'
data/LICENSE ADDED
@@ -0,0 +1,13 @@
1
+ Copyright (c) 2012-2014 Elasticsearch <http://www.elasticsearch.org>
2
+
3
+ Licensed under the Apache License, Version 2.0 (the "License");
4
+ you may not use this file except in compliance with the License.
5
+ You may obtain a copy of the License at
6
+
7
+ http://www.apache.org/licenses/LICENSE-2.0
8
+
9
+ Unless required by applicable law or agreed to in writing, software
10
+ distributed under the License is distributed on an "AS IS" BASIS,
11
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ See the License for the specific language governing permissions and
13
+ limitations under the License.
data/Rakefile ADDED
@@ -0,0 +1,6 @@
1
+ @files=[]
2
+
3
+ task :default do
4
+ system("rake -T")
5
+ end
6
+
@@ -0,0 +1,186 @@
1
+ # encoding: utf-8
2
+ require "logstash/inputs/base"
3
+ require "logstash/namespace"
4
+ require "socket"
5
+
6
+
7
+ # Read rows from an sqlite database.
8
+ #
9
+ # This is most useful in cases where you are logging directly to a table.
10
+ # Any tables being watched must have an 'id' column that is monotonically
11
+ # increasing.
12
+ #
13
+ # All tables are read by default except:
14
+ # * ones matching 'sqlite_%' - these are internal/adminstrative tables for sqlite
15
+ # * 'since_table' - this is used by this plugin to track state.
16
+ #
17
+ # ## Example
18
+ #
19
+ # % sqlite /tmp/example.db
20
+ # sqlite> CREATE TABLE weblogs (
21
+ # id INTEGER PRIMARY KEY AUTOINCREMENT,
22
+ # ip STRING,
23
+ # request STRING,
24
+ # response INTEGER);
25
+ # sqlite> INSERT INTO weblogs (ip, request, response)
26
+ # VALUES ("1.2.3.4", "/index.html", 200);
27
+ #
28
+ # Then with this logstash config:
29
+ #
30
+ # input {
31
+ # sqlite {
32
+ # path => "/tmp/example.db"
33
+ # type => weblogs
34
+ # }
35
+ # }
36
+ # output {
37
+ # stdout {
38
+ # debug => true
39
+ # }
40
+ # }
41
+ #
42
+ # Sample output:
43
+ #
44
+ # {
45
+ # "@source" => "sqlite://sadness/tmp/x.db",
46
+ # "@tags" => [],
47
+ # "@fields" => {
48
+ # "ip" => "1.2.3.4",
49
+ # "request" => "/index.html",
50
+ # "response" => 200
51
+ # },
52
+ # "@timestamp" => "2013-05-29T06:16:30.850Z",
53
+ # "@source_host" => "sadness",
54
+ # "@source_path" => "/tmp/x.db",
55
+ # "@message" => "",
56
+ # "@type" => "foo"
57
+ # }
58
+ #
59
+ class LogStash::Inputs::Sqlite < LogStash::Inputs::Base
60
+ config_name "sqlite"
61
+ milestone 1
62
+
63
+ # The path to the sqlite database file.
64
+ config :path, :validate => :string, :required => true
65
+
66
+ # Any tables to exclude by name.
67
+ # By default all tables are followed.
68
+ config :exclude_tables, :validate => :array, :default => []
69
+
70
+ # How many rows to fetch at a time from each SELECT call.
71
+ config :batch, :validate => :number, :default => 5
72
+
73
+ SINCE_TABLE = :since_table
74
+
75
+ public
76
+ def init_placeholder_table(db)
77
+ begin
78
+ db.create_table SINCE_TABLE do
79
+ String :table
80
+ Int :place
81
+ end
82
+ rescue
83
+ @logger.debug("since tables already exists")
84
+ end
85
+ end
86
+
87
+ public
88
+ def get_placeholder(db, table)
89
+ since = db[SINCE_TABLE]
90
+ x = since.where(:table => "#{table}")
91
+ if x[:place].nil?
92
+ init_placeholder(db, table)
93
+ return 0
94
+ else
95
+ @logger.debug("placeholder already exists, it is #{x[:place]}")
96
+ return x[:place][:place]
97
+ end
98
+ end
99
+
100
+ public
101
+ def init_placeholder(db, table)
102
+ @logger.debug("init placeholder for #{table}")
103
+ since = db[SINCE_TABLE]
104
+ since.insert(:table => table, :place => 0)
105
+ end
106
+
107
+ public
108
+ def update_placeholder(db, table, place)
109
+ @logger.debug("set placeholder to #{place}")
110
+ since = db[SINCE_TABLE]
111
+ since.where(:table => table).update(:place => place)
112
+ end
113
+
114
+ public
115
+ def get_all_tables(db)
116
+ return db["SELECT * FROM sqlite_master WHERE type = 'table' AND tbl_name != '#{SINCE_TABLE}' AND tbl_name NOT LIKE 'sqlite_%'"].map { |t| t[:name] }.select { |n| !@exclude_tables.include?(n) }
117
+ end
118
+
119
+ public
120
+ def get_n_rows_from_table(db, table, offset, limit)
121
+ dataset = db["SELECT * FROM #{table}"]
122
+ return db["SELECT * FROM #{table} WHERE (id > #{offset}) ORDER BY 'id' LIMIT #{limit}"].map { |row| row }
123
+ end
124
+
125
+ public
126
+ def register
127
+ require "sequel"
128
+ require "jdbc/sqlite3"
129
+ @host = Socket.gethostname
130
+ @logger.info("Registering sqlite input", :database => @path)
131
+ @db = Sequel.connect("jdbc:sqlite:#{@path}")
132
+ @tables = get_all_tables(@db)
133
+ @table_data = {}
134
+ @tables.each do |table|
135
+ init_placeholder_table(@db)
136
+ last_place = get_placeholder(@db, table)
137
+ @table_data[table] = { :name => table, :place => last_place }
138
+ end
139
+ end # def register
140
+
141
+ public
142
+ def run(queue)
143
+ sleep_min = 0.01
144
+ sleep_max = 5
145
+ sleeptime = sleep_min
146
+
147
+ begin
148
+ @logger.debug("Tailing sqlite db", :path => @path)
149
+ loop do
150
+ count = 0
151
+ @table_data.each do |k, table|
152
+ table_name = table[:name]
153
+ offset = table[:place]
154
+ @logger.debug("offset is #{offset}", :k => k, :table => table_name)
155
+ rows = get_n_rows_from_table(@db, table_name, offset, @batch)
156
+ count += rows.count
157
+ rows.each do |row|
158
+ event = LogStash::Event.new("host" => @host, "db" => @db)
159
+ decorate(event)
160
+ # store each column as a field in the event.
161
+ row.each do |column, element|
162
+ next if column == :id
163
+ event[column.to_s] = element
164
+ end
165
+ queue << event
166
+ @table_data[k][:place] = row[:id]
167
+ end
168
+ # Store the last-seen row in the database
169
+ update_placeholder(@db, table_name, @table_data[k][:place])
170
+ end
171
+
172
+ if count == 0
173
+ # nothing found in that iteration
174
+ # sleep a bit
175
+ @logger.debug("No new rows. Sleeping.", :time => sleeptime)
176
+ sleeptime = [sleeptime * 2, sleep_max].min
177
+ sleep(sleeptime)
178
+ else
179
+ sleeptime = sleep_min
180
+ end
181
+ end # loop
182
+ end # begin/rescue
183
+ end #run
184
+
185
+ end # class Logtstash::Inputs::EventLog
186
+
@@ -0,0 +1,29 @@
1
+ Gem::Specification.new do |s|
2
+
3
+ s.name = 'logstash-input-sqlite'
4
+ s.version = '0.1.0'
5
+ s.licenses = ['Apache License (2.0)']
6
+ s.summary = "Read rows from an sqlite database."
7
+ s.description = "Read rows from an sqlite database."
8
+ s.authors = ["Elasticsearch"]
9
+ s.email = 'richard.pijnenburg@elasticsearch.com'
10
+ s.homepage = "http://logstash.net/"
11
+ s.require_paths = ["lib"]
12
+
13
+ # Files
14
+ s.files = `git ls-files`.split($\)+::Dir.glob('vendor/*')
15
+
16
+ # Tests
17
+ s.test_files = s.files.grep(%r{^(test|spec|features)/})
18
+
19
+ # Special flag to let us know this is actually a logstash plugin
20
+ s.metadata = { "logstash_plugin" => "true", "group" => "input" }
21
+
22
+ # Gem dependencies
23
+ s.add_runtime_dependency 'logstash', '>= 1.4.0', '< 2.0.0'
24
+
25
+ s.add_runtime_dependency 'sequel'
26
+ s.add_runtime_dependency 'jdbc-sqlite3'
27
+
28
+ end
29
+
@@ -0,0 +1,9 @@
1
+ require "gem_publisher"
2
+
3
+ desc "Publish gem to RubyGems.org"
4
+ task :publish_gem do |t|
5
+ gem_file = Dir.glob(File.expand_path('../*.gemspec',File.dirname(__FILE__))).first
6
+ gem = GemPublisher.publish_if_updated(gem_file, :rubygems)
7
+ puts "Published #{gem}" if gem
8
+ end
9
+
@@ -0,0 +1,169 @@
1
+ require "net/http"
2
+ require "uri"
3
+ require "digest/sha1"
4
+
5
+ def vendor(*args)
6
+ return File.join("vendor", *args)
7
+ end
8
+
9
+ directory "vendor/" => ["vendor"] do |task, args|
10
+ mkdir task.name
11
+ end
12
+
13
+ def fetch(url, sha1, output)
14
+
15
+ puts "Downloading #{url}"
16
+ actual_sha1 = download(url, output)
17
+
18
+ if actual_sha1 != sha1
19
+ fail "SHA1 does not match (expected '#{sha1}' but got '#{actual_sha1}')"
20
+ end
21
+ end # def fetch
22
+
23
+ def file_fetch(url, sha1)
24
+ filename = File.basename( URI(url).path )
25
+ output = "vendor/#{filename}"
26
+ task output => [ "vendor/" ] do
27
+ begin
28
+ actual_sha1 = file_sha1(output)
29
+ if actual_sha1 != sha1
30
+ fetch(url, sha1, output)
31
+ end
32
+ rescue Errno::ENOENT
33
+ fetch(url, sha1, output)
34
+ end
35
+ end.invoke
36
+
37
+ return output
38
+ end
39
+
40
+ def file_sha1(path)
41
+ digest = Digest::SHA1.new
42
+ fd = File.new(path, "r")
43
+ while true
44
+ begin
45
+ digest << fd.sysread(16384)
46
+ rescue EOFError
47
+ break
48
+ end
49
+ end
50
+ return digest.hexdigest
51
+ ensure
52
+ fd.close if fd
53
+ end
54
+
55
+ def download(url, output)
56
+ uri = URI(url)
57
+ digest = Digest::SHA1.new
58
+ tmp = "#{output}.tmp"
59
+ Net::HTTP.start(uri.host, uri.port, :use_ssl => (uri.scheme == "https")) do |http|
60
+ request = Net::HTTP::Get.new(uri.path)
61
+ http.request(request) do |response|
62
+ fail "HTTP fetch failed for #{url}. #{response}" if [200, 301].include?(response.code)
63
+ size = (response["content-length"].to_i || -1).to_f
64
+ count = 0
65
+ File.open(tmp, "w") do |fd|
66
+ response.read_body do |chunk|
67
+ fd.write(chunk)
68
+ digest << chunk
69
+ if size > 0 && $stdout.tty?
70
+ count += chunk.bytesize
71
+ $stdout.write(sprintf("\r%0.2f%%", count/size * 100))
72
+ end
73
+ end
74
+ end
75
+ $stdout.write("\r \r") if $stdout.tty?
76
+ end
77
+ end
78
+
79
+ File.rename(tmp, output)
80
+
81
+ return digest.hexdigest
82
+ rescue SocketError => e
83
+ puts "Failure while downloading #{url}: #{e}"
84
+ raise
85
+ ensure
86
+ File.unlink(tmp) if File.exist?(tmp)
87
+ end # def download
88
+
89
+ def untar(tarball, &block)
90
+ require "archive/tar/minitar"
91
+ tgz = Zlib::GzipReader.new(File.open(tarball))
92
+ # Pull out typesdb
93
+ tar = Archive::Tar::Minitar::Input.open(tgz)
94
+ tar.each do |entry|
95
+ path = block.call(entry)
96
+ next if path.nil?
97
+ parent = File.dirname(path)
98
+
99
+ mkdir_p parent unless File.directory?(parent)
100
+
101
+ # Skip this file if the output file is the same size
102
+ if entry.directory?
103
+ mkdir path unless File.directory?(path)
104
+ else
105
+ entry_mode = entry.instance_eval { @mode } & 0777
106
+ if File.exists?(path)
107
+ stat = File.stat(path)
108
+ # TODO(sissel): Submit a patch to archive-tar-minitar upstream to
109
+ # expose headers in the entry.
110
+ entry_size = entry.instance_eval { @size }
111
+ # If file sizes are same, skip writing.
112
+ next if stat.size == entry_size && (stat.mode & 0777) == entry_mode
113
+ end
114
+ puts "Extracting #{entry.full_name} from #{tarball} #{entry_mode.to_s(8)}"
115
+ File.open(path, "w") do |fd|
116
+ # eof? check lets us skip empty files. Necessary because the API provided by
117
+ # Archive::Tar::Minitar::Reader::EntryStream only mostly acts like an
118
+ # IO object. Something about empty files in this EntryStream causes
119
+ # IO.copy_stream to throw "can't convert nil into String" on JRuby
120
+ # TODO(sissel): File a bug about this.
121
+ while !entry.eof?
122
+ chunk = entry.read(16384)
123
+ fd.write(chunk)
124
+ end
125
+ #IO.copy_stream(entry, fd)
126
+ end
127
+ File.chmod(entry_mode, path)
128
+ end
129
+ end
130
+ tar.close
131
+ File.unlink(tarball) if File.file?(tarball)
132
+ end # def untar
133
+
134
+ def ungz(file)
135
+
136
+ outpath = file.gsub('.gz', '')
137
+ tgz = Zlib::GzipReader.new(File.open(file))
138
+ begin
139
+ File.open(outpath, "w") do |out|
140
+ IO::copy_stream(tgz, out)
141
+ end
142
+ File.unlink(file)
143
+ rescue
144
+ File.unlink(outpath) if File.file?(outpath)
145
+ raise
146
+ end
147
+ tgz.close
148
+ end
149
+
150
+ desc "Process any vendor files required for this plugin"
151
+ task "vendor" do |task, args|
152
+
153
+ @files.each do |file|
154
+ download = file_fetch(file['url'], file['sha1'])
155
+ if download =~ /.tar.gz/
156
+ prefix = download.gsub('.tar.gz', '').gsub('vendor/', '')
157
+ untar(download) do |entry|
158
+ if !file['files'].nil?
159
+ next unless file['files'].include?(entry.full_name.gsub(prefix, ''))
160
+ out = entry.full_name.split("/").last
161
+ end
162
+ File.join('vendor', out)
163
+ end
164
+ elsif download =~ /.gz/
165
+ ungz(download)
166
+ end
167
+ end
168
+
169
+ end
@@ -0,0 +1 @@
1
+ require 'spec_helper'
metadata ADDED
@@ -0,0 +1,103 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: logstash-input-sqlite
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - Elasticsearch
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2014-11-05 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: logstash
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ! '>='
18
+ - !ruby/object:Gem::Version
19
+ version: 1.4.0
20
+ - - <
21
+ - !ruby/object:Gem::Version
22
+ version: 2.0.0
23
+ type: :runtime
24
+ prerelease: false
25
+ version_requirements: !ruby/object:Gem::Requirement
26
+ requirements:
27
+ - - ! '>='
28
+ - !ruby/object:Gem::Version
29
+ version: 1.4.0
30
+ - - <
31
+ - !ruby/object:Gem::Version
32
+ version: 2.0.0
33
+ - !ruby/object:Gem::Dependency
34
+ name: sequel
35
+ requirement: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - ! '>='
38
+ - !ruby/object:Gem::Version
39
+ version: '0'
40
+ type: :runtime
41
+ prerelease: false
42
+ version_requirements: !ruby/object:Gem::Requirement
43
+ requirements:
44
+ - - ! '>='
45
+ - !ruby/object:Gem::Version
46
+ version: '0'
47
+ - !ruby/object:Gem::Dependency
48
+ name: jdbc-sqlite3
49
+ requirement: !ruby/object:Gem::Requirement
50
+ requirements:
51
+ - - ! '>='
52
+ - !ruby/object:Gem::Version
53
+ version: '0'
54
+ type: :runtime
55
+ prerelease: false
56
+ version_requirements: !ruby/object:Gem::Requirement
57
+ requirements:
58
+ - - ! '>='
59
+ - !ruby/object:Gem::Version
60
+ version: '0'
61
+ description: Read rows from an sqlite database.
62
+ email: richard.pijnenburg@elasticsearch.com
63
+ executables: []
64
+ extensions: []
65
+ extra_rdoc_files: []
66
+ files:
67
+ - .gitignore
68
+ - Gemfile
69
+ - LICENSE
70
+ - Rakefile
71
+ - lib/logstash/inputs/sqlite.rb
72
+ - logstash-input-sqlite.gemspec
73
+ - rakelib/publish.rake
74
+ - rakelib/vendor.rake
75
+ - spec/inputs/sqlite_spec.rb
76
+ homepage: http://logstash.net/
77
+ licenses:
78
+ - Apache License (2.0)
79
+ metadata:
80
+ logstash_plugin: 'true'
81
+ group: input
82
+ post_install_message:
83
+ rdoc_options: []
84
+ require_paths:
85
+ - lib
86
+ required_ruby_version: !ruby/object:Gem::Requirement
87
+ requirements:
88
+ - - ! '>='
89
+ - !ruby/object:Gem::Version
90
+ version: '0'
91
+ required_rubygems_version: !ruby/object:Gem::Requirement
92
+ requirements:
93
+ - - ! '>='
94
+ - !ruby/object:Gem::Version
95
+ version: '0'
96
+ requirements: []
97
+ rubyforge_project:
98
+ rubygems_version: 2.4.1
99
+ signing_key:
100
+ specification_version: 4
101
+ summary: Read rows from an sqlite database.
102
+ test_files:
103
+ - spec/inputs/sqlite_spec.rb