packageiq 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +11 -0
- data/.travis.yml +4 -0
- data/Gemfile +4 -0
- data/README.md +34 -0
- data/Rakefile +1 -0
- data/bin/console +14 -0
- data/bin/setup +7 -0
- data/exe/piq_indexer +77 -0
- data/exe/piq_rhel +34 -0
- data/lib/packageiq/cli.rb +32 -0
- data/lib/packageiq/command.rb +17 -0
- data/lib/packageiq/loader.rb +65 -0
- data/lib/packageiq/mappings/elasticsearch.rb +227 -0
- data/lib/packageiq/provider/rhel.rb +148 -0
- data/lib/packageiq/transport/rabbitmq.rb +73 -0
- data/lib/packageiq/utils.rb +100 -0
- data/lib/packageiq/version.rb +3 -0
- data/lib/packageiq.rb +5 -0
- data/packageiq.gemspec +36 -0
- metadata +150 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA1:
|
|
3
|
+
metadata.gz: 3b2705e163d8577a10f003c70520ad9381c7452c
|
|
4
|
+
data.tar.gz: 818b2c6b340c08c2f571162bb82a82b4997e0dcb
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: c395993cf419552922bc2ac4fa514a6cb7f8e676ceb8cc55277917533b63c1a03ba903fa77b0d6f495c30c59f34c82089713fc864543566f5e1c6aa39bebdb4a
|
|
7
|
+
data.tar.gz: 6265eb0c6f3f6b5bb345f1e09e7ca733f76a759c0c8d87ecf8ed4d862d2bebd4bf6265f2e8a1eb22e0f05868329af291316e4244d5bceba3cc439e12ba04bc1a
|
data/.gitignore
ADDED
data/.travis.yml
ADDED
data/Gemfile
ADDED
data/README.md
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# Packageiq
|
|
2
|
+
|
|
3
|
+
Package Intelligence Gathering System
|
|
4
|
+
|
|
5
|
+
## Installation
|
|
6
|
+
|
|
7
|
+
Add this line to your application's Gemfile:
|
|
8
|
+
|
|
9
|
+
```ruby
|
|
10
|
+
gem 'packageiq'
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
And then execute:
|
|
14
|
+
|
|
15
|
+
$ bundle
|
|
16
|
+
|
|
17
|
+
Or install it yourself as:
|
|
18
|
+
|
|
19
|
+
$ gem install packageiq
|
|
20
|
+
|
|
21
|
+
## Usage
|
|
22
|
+
|
|
23
|
+
TODO: Write usage instructions here
|
|
24
|
+
|
|
25
|
+
## Development
|
|
26
|
+
|
|
27
|
+
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake false` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
|
28
|
+
|
|
29
|
+
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
|
30
|
+
|
|
31
|
+
## Contributing
|
|
32
|
+
|
|
33
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/packageiq.
|
|
34
|
+
|
data/Rakefile
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
require "bundler/gem_tasks"
|
data/bin/console
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
|
|
3
|
+
require "bundler/setup"
|
|
4
|
+
require "packageiq"
|
|
5
|
+
|
|
6
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
|
7
|
+
# with your gem easier. You can also use a different console, if you like.
|
|
8
|
+
|
|
9
|
+
# (If you use this, don't forget to add pry to your Gemfile!)
|
|
10
|
+
# require "pry"
|
|
11
|
+
# Pry.start
|
|
12
|
+
|
|
13
|
+
require "irb"
|
|
14
|
+
IRB.start
|
data/bin/setup
ADDED
data/exe/piq_indexer
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
|
|
3
|
+
require 'packageiq/cli'
|
|
4
|
+
require 'packageiq/utils'
|
|
5
|
+
require 'packageiq/loader'
|
|
6
|
+
require 'packageiq/version'
|
|
7
|
+
require 'sneakers'
|
|
8
|
+
require 'sneakers/runner'
|
|
9
|
+
require 'elasticsearch'
|
|
10
|
+
require 'base64'
|
|
11
|
+
require 'json'
|
|
12
|
+
|
|
13
|
+
# read CLI arguments
|
|
14
|
+
options = Packageiq::CLI.read
|
|
15
|
+
|
|
16
|
+
# load settings from config files
|
|
17
|
+
loader = Packageiq::Loader.new(options)
|
|
18
|
+
config = loader.load
|
|
19
|
+
|
|
20
|
+
begin
|
|
21
|
+
ES_SETTINGS = config[:elasticsearch]
|
|
22
|
+
|
|
23
|
+
AMQP_SETTINGS = { amqp: Packageiq::Utils.amqp_uri(config[:rabbitmq]),
|
|
24
|
+
exchange: 'piq_indexer',
|
|
25
|
+
exchange_type: :direct,
|
|
26
|
+
exchange_options: { durable: false },
|
|
27
|
+
heartbeat: 20 }.merge(config[:rabbitmq])
|
|
28
|
+
|
|
29
|
+
INDEXER_SETTINGS = { daemonize: false,
|
|
30
|
+
start_worker_delay: 0.2,
|
|
31
|
+
workers: 2,
|
|
32
|
+
timeout_after_job: 5,
|
|
33
|
+
threads: 5,
|
|
34
|
+
prefetch: 5,
|
|
35
|
+
ack: true }.merge(config[:indexer])
|
|
36
|
+
rescue
|
|
37
|
+
raise 'Configuration Error. Check config files for rabbitmq, indexer, and elasticsearch settings.'
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# Configure Elasticsearch Client
|
|
41
|
+
$es_client = Elasticsearch::Client.new(ES_SETTINGS)
|
|
42
|
+
|
|
43
|
+
Sneakers.configure(AMQP_SETTINGS.merge(INDEXER_SETTINGS))
|
|
44
|
+
Sneakers.logger.level = Logger::INFO
|
|
45
|
+
|
|
46
|
+
# Sneakers worker class for indexing packages
|
|
47
|
+
class Indexer
|
|
48
|
+
include Sneakers::Worker
|
|
49
|
+
|
|
50
|
+
from_queue(AMQP_SETTINGS[:queue],
|
|
51
|
+
durable: false,
|
|
52
|
+
hooks: {
|
|
53
|
+
after_fork: lambda do
|
|
54
|
+
$es_client.transport.reload_connections!
|
|
55
|
+
end
|
|
56
|
+
})
|
|
57
|
+
|
|
58
|
+
# calculates document id for elasticsearch
|
|
59
|
+
def make_id(msg)
|
|
60
|
+
msg_hash = JSON.parse(msg)
|
|
61
|
+
hostname = msg_hash['server']['hostname']
|
|
62
|
+
package = msg_hash['package']['name']
|
|
63
|
+
arch = msg_hash['package']['arch']
|
|
64
|
+
Base64.encode64(hostname + package + arch)
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def work_with_params(msg, _delivery_info, metadata)
|
|
68
|
+
id = make_id(msg)
|
|
69
|
+
type = metadata[:type]
|
|
70
|
+
logger.info "Document ID: #{id} Type: #{type}"
|
|
71
|
+
$es_client.index index: ES_SETTINGS[:index], type: type, id: id, body: msg
|
|
72
|
+
ack!
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
sneakers = Sneakers::Runner.new([Indexer])
|
|
77
|
+
sneakers.run
|
data/exe/piq_rhel
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
|
|
3
|
+
require 'packageiq/cli'
|
|
4
|
+
require 'packageiq/loader'
|
|
5
|
+
require 'packageiq/provider/rhel'
|
|
6
|
+
require 'packageiq/transport/rabbitmq'
|
|
7
|
+
require 'packageiq/version'
|
|
8
|
+
|
|
9
|
+
# read CLI arguments
|
|
10
|
+
options = Packageiq::CLI.read
|
|
11
|
+
|
|
12
|
+
# load settings from config files
|
|
13
|
+
loader = Packageiq::Loader.new(options)
|
|
14
|
+
config = loader.load
|
|
15
|
+
|
|
16
|
+
# setup rabbitmq
|
|
17
|
+
rmq = Packageiq::Transport::Rabbitmq.new(config[:rabbitmq])
|
|
18
|
+
rmq.start
|
|
19
|
+
rmq.create_channel
|
|
20
|
+
queue_name = config[:rabbitmq][:queue]
|
|
21
|
+
queue = rmq.create_queue(queue_name)
|
|
22
|
+
|
|
23
|
+
# get package inventory
|
|
24
|
+
piq = Packageiq::Provider::RHEL.new
|
|
25
|
+
inventory = piq.build_inventory
|
|
26
|
+
|
|
27
|
+
inventory.each do |package_entry|
|
|
28
|
+
message = Packageiq::Transport::Rabbitmq.serialize(package_entry)
|
|
29
|
+
queue.publish(message,
|
|
30
|
+
routing_key: queue.name,
|
|
31
|
+
type: 'rhel',
|
|
32
|
+
content_type: 'application/json',
|
|
33
|
+
app_id: "piq_rhel #{Packageiq::VERSION}")
|
|
34
|
+
end
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
require 'optparse'
|
|
2
|
+
|
|
3
|
+
module Packageiq
|
|
4
|
+
# Parse CLI arguments
|
|
5
|
+
class CLI
|
|
6
|
+
def self.read(arguments = ARGV)
|
|
7
|
+
options = {}
|
|
8
|
+
optparse = OptionParser.new do |opts|
|
|
9
|
+
opts.on('-h', '--help', 'Display this message') do
|
|
10
|
+
puts opts
|
|
11
|
+
exit
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
opts.on('-V', '--version', 'Display Version') do
|
|
15
|
+
puts Packageiq::VERSION
|
|
16
|
+
exit
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
opts.on('-c', '--config FILE', 'PackageIQ config FILE') do |file|
|
|
20
|
+
options[:config_file] = file
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
opts.on('-d', '--config_dir DIR',
|
|
24
|
+
'DIR for PackageIQ config files') do |dir|
|
|
25
|
+
options[:config_dir] = dir
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
optparse.parse!(arguments)
|
|
29
|
+
options
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
require 'open3'
|
|
2
|
+
|
|
3
|
+
module Packageiq
|
|
4
|
+
# run system commands
|
|
5
|
+
class Command
|
|
6
|
+
attr_accessor :command
|
|
7
|
+
|
|
8
|
+
def initialize(args = {})
|
|
9
|
+
@command = args[:command] if args[:command]
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
attr_reader :stdout, :stderr, :status
|
|
13
|
+
def run
|
|
14
|
+
@stdout, @stderr, @status = Open3.capture3(@command)
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
end
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
require 'json'
|
|
2
|
+
|
|
3
|
+
module Packageiq
|
|
4
|
+
# load settings from config file
|
|
5
|
+
class Loader
|
|
6
|
+
CONFIG_DIR = '/etc/packageiq'
|
|
7
|
+
|
|
8
|
+
attr_accessor :config_dir, :config_file
|
|
9
|
+
attr_reader :settings
|
|
10
|
+
|
|
11
|
+
# can accept options hash from CLI class
|
|
12
|
+
def initialize(options = {})
|
|
13
|
+
@config_dir = options[:config_dir] || CONFIG_DIR
|
|
14
|
+
@config_file = options[:config_file] || nil
|
|
15
|
+
@settings = {}
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
# load config file settings
|
|
19
|
+
# return hash of all settings
|
|
20
|
+
def load
|
|
21
|
+
if config_file
|
|
22
|
+
full_path = full_path(config_dir, config_file)
|
|
23
|
+
load_file(full_path)
|
|
24
|
+
else
|
|
25
|
+
load_dir(config_dir)
|
|
26
|
+
end
|
|
27
|
+
settings
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
private
|
|
31
|
+
|
|
32
|
+
# joins to parts to form full path
|
|
33
|
+
# returns full path to file or directory
|
|
34
|
+
def full_path(part1, part2)
|
|
35
|
+
File.join(part1, part2)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# recursively load config files from directory
|
|
39
|
+
def load_dir(directory)
|
|
40
|
+
Dir.entries(directory).each do |entry|
|
|
41
|
+
next if entry == '..' || entry == '.'
|
|
42
|
+
full_path = full_path(directory, entry)
|
|
43
|
+
if File.file?(full_path)
|
|
44
|
+
load_file(full_path)
|
|
45
|
+
elsif File.directory?(full_path)
|
|
46
|
+
load_dir(full_path)
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# load file contents into settings hash
|
|
52
|
+
# only loads .json files. does not load dotfiles
|
|
53
|
+
def load_file(file_path)
|
|
54
|
+
return unless File.fnmatch('**.json', file_path)
|
|
55
|
+
handle = File.open(file_path)
|
|
56
|
+
contents = handle.read
|
|
57
|
+
config_hash = parse_contents(contents)
|
|
58
|
+
settings.merge!(config_hash)
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def parse_contents(contents)
|
|
62
|
+
JSON.parse(contents, symbolize_names: true)
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
end
|
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
module Packageiq
|
|
2
|
+
# this module contains elasticsearch document type mappings
|
|
3
|
+
module Mappings
|
|
4
|
+
RHEL = {
|
|
5
|
+
rhel: {
|
|
6
|
+
properties: {
|
|
7
|
+
update: {
|
|
8
|
+
type: 'object',
|
|
9
|
+
properties: {
|
|
10
|
+
available: {
|
|
11
|
+
type: 'string',
|
|
12
|
+
fields: {
|
|
13
|
+
raw: {
|
|
14
|
+
type: 'string',
|
|
15
|
+
index: 'not_analyzed'
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
},
|
|
19
|
+
version: {
|
|
20
|
+
type: 'string',
|
|
21
|
+
fields: {
|
|
22
|
+
raw: {
|
|
23
|
+
type: 'string',
|
|
24
|
+
index: 'not_analyzed'
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
},
|
|
28
|
+
repo: {
|
|
29
|
+
type: 'string',
|
|
30
|
+
fields: {
|
|
31
|
+
raw: {
|
|
32
|
+
type: 'string',
|
|
33
|
+
index: 'not_analyzed'
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
},
|
|
39
|
+
package: {
|
|
40
|
+
type: 'object',
|
|
41
|
+
properties: {
|
|
42
|
+
signature: {
|
|
43
|
+
type: 'string',
|
|
44
|
+
fields: {
|
|
45
|
+
raw: {
|
|
46
|
+
type: 'string',
|
|
47
|
+
index: 'not_analyzed'
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
},
|
|
51
|
+
arch: {
|
|
52
|
+
type: 'string',
|
|
53
|
+
fields: {
|
|
54
|
+
raw: {
|
|
55
|
+
type: 'string',
|
|
56
|
+
index: 'not_analyzed'
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
},
|
|
60
|
+
name: {
|
|
61
|
+
type: 'string',
|
|
62
|
+
fields: {
|
|
63
|
+
raw: {
|
|
64
|
+
type: 'string',
|
|
65
|
+
index: 'not_analyzed'
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
},
|
|
69
|
+
release: {
|
|
70
|
+
type: 'string',
|
|
71
|
+
fields: {
|
|
72
|
+
raw: {
|
|
73
|
+
type: 'string',
|
|
74
|
+
index: 'not_analyzed'
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
},
|
|
78
|
+
epoch: {
|
|
79
|
+
type: 'string',
|
|
80
|
+
fields: {
|
|
81
|
+
raw: {
|
|
82
|
+
type: 'string',
|
|
83
|
+
index: 'not_analyzed'
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
},
|
|
87
|
+
url: {
|
|
88
|
+
type: 'string',
|
|
89
|
+
fields: {
|
|
90
|
+
raw: {
|
|
91
|
+
type: 'string',
|
|
92
|
+
index: 'not_analyzed'
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
},
|
|
96
|
+
size: {
|
|
97
|
+
type: 'integer'
|
|
98
|
+
},
|
|
99
|
+
vendor: {
|
|
100
|
+
type: 'string',
|
|
101
|
+
fields: {
|
|
102
|
+
raw: {
|
|
103
|
+
type: 'string',
|
|
104
|
+
index: 'not_analyzed'
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
},
|
|
108
|
+
license: {
|
|
109
|
+
type: 'string',
|
|
110
|
+
fields: {
|
|
111
|
+
raw: {
|
|
112
|
+
type: 'string',
|
|
113
|
+
index: 'not_analyzed'
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
},
|
|
117
|
+
packager: {
|
|
118
|
+
type: 'string',
|
|
119
|
+
fields: {
|
|
120
|
+
raw: {
|
|
121
|
+
type: 'string',
|
|
122
|
+
index: 'not_analyzed'
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
},
|
|
126
|
+
version: {
|
|
127
|
+
type: 'string',
|
|
128
|
+
fields: {
|
|
129
|
+
raw: {
|
|
130
|
+
type: 'string',
|
|
131
|
+
index: 'not_analyzed'
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
},
|
|
135
|
+
source_rpm: {
|
|
136
|
+
type: 'string',
|
|
137
|
+
fields: {
|
|
138
|
+
raw: {
|
|
139
|
+
type: 'string',
|
|
140
|
+
index: 'not_analyzed'
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
},
|
|
144
|
+
group: {
|
|
145
|
+
type: 'string',
|
|
146
|
+
fields: {
|
|
147
|
+
raw: {
|
|
148
|
+
type: 'string',
|
|
149
|
+
index: 'not_analyzed'
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
},
|
|
153
|
+
install_date: {
|
|
154
|
+
type: 'string',
|
|
155
|
+
fields: {
|
|
156
|
+
raw: {
|
|
157
|
+
type: 'string',
|
|
158
|
+
index: 'not_analyzed'
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
},
|
|
162
|
+
build_date: {
|
|
163
|
+
type: 'string',
|
|
164
|
+
fields: {
|
|
165
|
+
raw: {
|
|
166
|
+
type: 'string',
|
|
167
|
+
index: 'not_analyzed'
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
},
|
|
171
|
+
build_host: {
|
|
172
|
+
type: 'string',
|
|
173
|
+
fields: {
|
|
174
|
+
raw: {
|
|
175
|
+
type: 'string',
|
|
176
|
+
index: 'not_analyzed'
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
},
|
|
180
|
+
summary: {
|
|
181
|
+
type: 'string',
|
|
182
|
+
fields: {
|
|
183
|
+
raw: {
|
|
184
|
+
type: 'string',
|
|
185
|
+
index: 'not_analyzed'
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
},
|
|
191
|
+
server: {
|
|
192
|
+
type: 'object',
|
|
193
|
+
properties: {
|
|
194
|
+
os_release: {
|
|
195
|
+
type: 'string',
|
|
196
|
+
fields: {
|
|
197
|
+
raw: {
|
|
198
|
+
type: 'string',
|
|
199
|
+
index: 'not_analyzed'
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
},
|
|
203
|
+
hostname: {
|
|
204
|
+
type: 'string',
|
|
205
|
+
fields: {
|
|
206
|
+
raw: {
|
|
207
|
+
type: 'string',
|
|
208
|
+
index: 'not_analyzed'
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
},
|
|
212
|
+
collection_time: {
|
|
213
|
+
type: 'string',
|
|
214
|
+
fields: {
|
|
215
|
+
raw: {
|
|
216
|
+
type: 'string',
|
|
217
|
+
index: 'not_analyzed'
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
end
|
|
227
|
+
end
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
require 'packageiq/command'
|
|
2
|
+
require 'socket'
|
|
3
|
+
|
|
4
|
+
module Packageiq
|
|
5
|
+
module Provider
|
|
6
|
+
# rhel based system package provider
|
|
7
|
+
class RHEL
|
|
8
|
+
# mapping for yum info field name to symbol
|
|
9
|
+
RPM_INFO_KEY = {
|
|
10
|
+
'Name' => :name,
|
|
11
|
+
'Epoch' => :epoch,
|
|
12
|
+
'Version' => :version,
|
|
13
|
+
'Release' => :release,
|
|
14
|
+
'Architecture' => :arch,
|
|
15
|
+
'Install Date' => :install_date,
|
|
16
|
+
'Group' => :group,
|
|
17
|
+
'Size' => :size,
|
|
18
|
+
'License' => :license,
|
|
19
|
+
'Signature' => :signature,
|
|
20
|
+
'Source RPM' => :source_rpm,
|
|
21
|
+
'Build Date' => :build_date,
|
|
22
|
+
'Build Host' => :build_host,
|
|
23
|
+
'Packager' => :packager,
|
|
24
|
+
'Vendor' => :vendor,
|
|
25
|
+
'URL' => :url,
|
|
26
|
+
'Summary' => :summary
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
attr_reader :command_handler, :hostname, :timestamp, :os_release
|
|
30
|
+
|
|
31
|
+
def initialize
|
|
32
|
+
@command_handler = Packageiq::Command.new
|
|
33
|
+
@hostname = Socket.gethostname
|
|
34
|
+
@timestamp = Time.new
|
|
35
|
+
@os_release = rhel_release.strip
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# returns array of installed packages
|
|
39
|
+
def installed
|
|
40
|
+
installed = run('rpm -qa')
|
|
41
|
+
installed.split("\n")
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# returns array of available yum update hashes
|
|
45
|
+
def updates
|
|
46
|
+
updates = run('yum list updates')
|
|
47
|
+
parse_list(updates)
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# returns hash of rpm info
|
|
51
|
+
def info(package)
|
|
52
|
+
info = run("rpm -qi #{package}")
|
|
53
|
+
parse_info(info)
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# returns redhat relase string
|
|
57
|
+
def rhel_release
|
|
58
|
+
run('cat /etc/redhat-release')
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# returns hash of server info
|
|
62
|
+
def server_info
|
|
63
|
+
{
|
|
64
|
+
server:
|
|
65
|
+
{
|
|
66
|
+
hostname: hostname,
|
|
67
|
+
os_release: os_release,
|
|
68
|
+
collection_time: timestamp
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# build full package inventory
|
|
74
|
+
# returns array of package_entry hashes
|
|
75
|
+
def build_inventory
|
|
76
|
+
inventory = []
|
|
77
|
+
updates_array = updates
|
|
78
|
+
installed.each do |package|
|
|
79
|
+
package_info = info(package)
|
|
80
|
+
package_entry = updateable(package_info, updates_array)
|
|
81
|
+
package_entry.merge!(server_info)
|
|
82
|
+
inventory << package_entry
|
|
83
|
+
end
|
|
84
|
+
inventory
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# adds available update info to package_info hash
|
|
88
|
+
def updateable(package_info, updates)
|
|
89
|
+
update_info = { update: { available: 'no', version: '-', repo: '-' } }
|
|
90
|
+
updates.each do |update|
|
|
91
|
+
next unless update[:name] == package_info[:package][:name]
|
|
92
|
+
update_info[:update][:available] = 'yes'
|
|
93
|
+
update_info[:update][:version] = update[:version]
|
|
94
|
+
update_info[:update][:repo] = update[:repo]
|
|
95
|
+
break
|
|
96
|
+
end
|
|
97
|
+
package_info.merge(update_info)
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
private
|
|
101
|
+
|
|
102
|
+
# use command handler to run command
|
|
103
|
+
def run(command)
|
|
104
|
+
command_handler.command = command
|
|
105
|
+
command_handler.run
|
|
106
|
+
command_handler.stdout
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
# normalize output so each package listing is on one line
|
|
110
|
+
def unwrap(output)
|
|
111
|
+
output.gsub(/\n/, '#').gsub(/#\s/, ' ').gsub(/#/, "\n")
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# parse list output to hash, return array of hashes
|
|
115
|
+
def parse_list(list)
|
|
116
|
+
list = unwrap(list)
|
|
117
|
+
lines = list.split("\n")
|
|
118
|
+
package_list = []
|
|
119
|
+
lines.each do |line|
|
|
120
|
+
package_info = {}
|
|
121
|
+
next if line =~ Regexp.new(/^(Installed|Updated) Packages$/)
|
|
122
|
+
parts = line.split
|
|
123
|
+
package = parts[0]
|
|
124
|
+
package_info[:name] = package.split('.')[0]
|
|
125
|
+
package_info[:arch] = package.split('.')[1]
|
|
126
|
+
package_info[:version] = parts[1]
|
|
127
|
+
package_info[:repo] = parts[2]
|
|
128
|
+
package_list << package_info
|
|
129
|
+
end
|
|
130
|
+
package_list
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
# parse rpm info into hash
|
|
134
|
+
def parse_info(info)
|
|
135
|
+
lines = info.split("\n")
|
|
136
|
+
info_hash = { package: {} }
|
|
137
|
+
lines.each do |line|
|
|
138
|
+
RPM_INFO_KEY.each do |field, symbol|
|
|
139
|
+
parse_regex = Regexp.new("^(#{field})+\s*:{1}\s+(.*)$")
|
|
140
|
+
parts = parse_regex.match(line)
|
|
141
|
+
info_hash[:package][symbol] = parts[2] if parts
|
|
142
|
+
end
|
|
143
|
+
end
|
|
144
|
+
info_hash
|
|
145
|
+
end
|
|
146
|
+
end
|
|
147
|
+
end
|
|
148
|
+
end
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
require 'bunny'
|
|
2
|
+
require 'json'
|
|
3
|
+
|
|
4
|
+
module Packageiq
|
|
5
|
+
module Transport
|
|
6
|
+
# rabbitmq transport
|
|
7
|
+
class Rabbitmq
|
|
8
|
+
attr_accessor :host, :port, :tls,
|
|
9
|
+
:tls_cert, :tls_key, :tls_ca_certificates,
|
|
10
|
+
:vhost, :user, :pass
|
|
11
|
+
|
|
12
|
+
def initialize(args = {})
|
|
13
|
+
@host = args[:host] || '127.0.0.1'
|
|
14
|
+
@port = args[:port] || 5672
|
|
15
|
+
@tls = args[:tls] || false
|
|
16
|
+
@tls_cert = args[:tls_cert] || ''
|
|
17
|
+
@tls_key = args[:tls_key] || ''
|
|
18
|
+
@tls_ca_certificates = args[:tls_ca_certificates] || []
|
|
19
|
+
@vhost = args[:vhost] || '/'
|
|
20
|
+
@user = args[:user] || 'guest'
|
|
21
|
+
@pass = args[:pass] || 'guest'
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# serialize message
|
|
25
|
+
def self.serialize(message)
|
|
26
|
+
message.to_json
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
attr_reader :conn
|
|
30
|
+
|
|
31
|
+
# initialize bunny instance and start connection
|
|
32
|
+
def start
|
|
33
|
+
@conn = Bunny.new(host: host, port: port,
|
|
34
|
+
tls: tls, tls_cert: tls_cert,
|
|
35
|
+
tls_key: tls_key,
|
|
36
|
+
tls_ca_certificates: tls_ca_certificates,
|
|
37
|
+
vhost: vhost, user: user, pass: pass)
|
|
38
|
+
@conn.start
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
attr_reader :channel
|
|
42
|
+
|
|
43
|
+
# create rabbitmq channel
|
|
44
|
+
def create_channel
|
|
45
|
+
@channel = conn.create_channel
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# close rabbitmq channel
|
|
49
|
+
def close_channel
|
|
50
|
+
@channel.close_channel
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
attr_reader :queue
|
|
54
|
+
|
|
55
|
+
# create rabbitmq queue
|
|
56
|
+
def create_queue(name, opts = {})
|
|
57
|
+
durable = opts[:durable] || false
|
|
58
|
+
auto_delete = opts[:auto_delete] || false
|
|
59
|
+
exclusive = opts[:exclusive] || false
|
|
60
|
+
@queue = channel.queue(name,
|
|
61
|
+
durable: durable,
|
|
62
|
+
auto_delete: auto_delete,
|
|
63
|
+
exclusive: exclusive)
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
# check if queue exists
|
|
67
|
+
# returns boolean
|
|
68
|
+
def queue_exists?(name)
|
|
69
|
+
conn.queue_exists?(name)
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
end
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
module Packageiq
|
|
2
|
+
module Utils
|
|
3
|
+
def self.amqp_uri(options = {})
|
|
4
|
+
vhost = options[:vhost] == '/' ? '/' : ''
|
|
5
|
+
scheme = options[:tls] ? 'amqps://' : 'amqp://'
|
|
6
|
+
userinfo = options[:user] + ':' + options[:pass]
|
|
7
|
+
authority = userinfo + '@' + options[:host] + ':' + options[:port].to_s
|
|
8
|
+
scheme + authority + vhost
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
def self.rpmvercmp(str1, str2)
|
|
12
|
+
return 0 if str1 == str2
|
|
13
|
+
|
|
14
|
+
front_strip_re = /^[^A-Za-z0-9~]+/
|
|
15
|
+
|
|
16
|
+
while str1.length > 0 or str2.length > 0
|
|
17
|
+
# trim anything that's in front_strip_re and != '~' off the beginning of each string
|
|
18
|
+
str1 = str1.gsub(front_strip_re, '')
|
|
19
|
+
str2 = str2.gsub(front_strip_re, '')
|
|
20
|
+
|
|
21
|
+
# "handle the tilde separator, it sorts before everything else"
|
|
22
|
+
if /^~/.match(str1) && /^~/.match(str2)
|
|
23
|
+
# if they both have ~, strip it
|
|
24
|
+
str1 = str1[1..-1]
|
|
25
|
+
str2 = str2[1..-1]
|
|
26
|
+
elsif /^~/.match(str1)
|
|
27
|
+
return -1
|
|
28
|
+
elsif /^~/.match(str2)
|
|
29
|
+
return 1
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
break if str1.length == 0 or str2.length == 0
|
|
33
|
+
|
|
34
|
+
# "grab first completely alpha or completely numeric segment"
|
|
35
|
+
isnum = false
|
|
36
|
+
# if the first char of str1 is a digit, grab the chunk of continuous digits from each string
|
|
37
|
+
if /^[0-9]+/.match(str1)
|
|
38
|
+
if str1 =~ /^[0-9]+/
|
|
39
|
+
segment1 = $~.to_s
|
|
40
|
+
str1 = $~.post_match
|
|
41
|
+
else
|
|
42
|
+
segment1 = ''
|
|
43
|
+
end
|
|
44
|
+
if str2 =~ /^[0-9]+/
|
|
45
|
+
segment2 = $~.to_s
|
|
46
|
+
str2 = $~.post_match
|
|
47
|
+
else
|
|
48
|
+
segment2 = ''
|
|
49
|
+
end
|
|
50
|
+
isnum = true
|
|
51
|
+
# else grab the chunk of continuous alphas from each string (which may be '')
|
|
52
|
+
else
|
|
53
|
+
if str1 =~ /^[A-Za-z]+/
|
|
54
|
+
segment1 = $~.to_s
|
|
55
|
+
str1 = $~.post_match
|
|
56
|
+
else
|
|
57
|
+
segment1 = ''
|
|
58
|
+
end
|
|
59
|
+
if str2 =~ /^[A-Za-z]+/
|
|
60
|
+
segment2 = $~.to_s
|
|
61
|
+
str2 = $~.post_match
|
|
62
|
+
else
|
|
63
|
+
segment2 = ''
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# if the segments we just grabbed from the strings are different types (i.e. one numeric one alpha),
|
|
68
|
+
# where alpha also includes ''; "numeric segments are always newer than alpha segments"
|
|
69
|
+
if segment2.length == 0
|
|
70
|
+
return 1 if isnum
|
|
71
|
+
return -1
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
if isnum
|
|
75
|
+
# "throw away any leading zeros - it's a number, right?"
|
|
76
|
+
segment1 = segment1.gsub(/^0+/, '')
|
|
77
|
+
segment2 = segment2.gsub(/^0+/, '')
|
|
78
|
+
# "whichever number has more digits wins"
|
|
79
|
+
return 1 if segment1.length > segment2.length
|
|
80
|
+
return -1 if segment1.length < segment2.length
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# "strcmp will return which one is greater - even if the two segments are alpha
|
|
84
|
+
# or if they are numeric. don't return if they are equal because there might
|
|
85
|
+
# be more segments to compare"
|
|
86
|
+
rc = segment1 <=> segment2
|
|
87
|
+
return rc if rc != 0
|
|
88
|
+
end #end while loop
|
|
89
|
+
|
|
90
|
+
# if we haven't returned anything yet, "whichever version still has characters left over wins"
|
|
91
|
+
if str1.length > str2.length
|
|
92
|
+
return 1
|
|
93
|
+
elsif str1.length < str2.length
|
|
94
|
+
return -1
|
|
95
|
+
else
|
|
96
|
+
return 0
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
end
|
data/lib/packageiq.rb
ADDED
data/packageiq.gemspec
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# coding: utf-8
|
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
|
+
require 'packageiq/version'
|
|
5
|
+
|
|
6
|
+
Gem::Specification.new do |spec|
|
|
7
|
+
spec.name = 'packageiq'
|
|
8
|
+
spec.version = Packageiq::VERSION
|
|
9
|
+
spec.authors = ['Eric Olsen']
|
|
10
|
+
spec.email = ['eric@ericolsen.net']
|
|
11
|
+
|
|
12
|
+
spec.summary = 'Package Intelligence Gathering System'
|
|
13
|
+
spec.description = 'Package Intelligence Gathering System'
|
|
14
|
+
spec.homepage = 'http://www.packageiq.io'
|
|
15
|
+
|
|
16
|
+
# Prevent pushing this gem to RubyGems.org by setting 'allowed_push_host', or
|
|
17
|
+
# delete this section to allow pushing this gem to any host.
|
|
18
|
+
if spec.respond_to?(:metadata)
|
|
19
|
+
spec.metadata['allowed_push_host'] = 'https://rubygems.org'
|
|
20
|
+
else
|
|
21
|
+
fail 'RubyGems 2.0 or newer is required to protect against public gem pushes.'
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
|
25
|
+
spec.bindir = 'exe'
|
|
26
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
|
27
|
+
spec.require_paths = ['lib']
|
|
28
|
+
|
|
29
|
+
spec.add_dependency 'bunny', '~> 2.2.2'
|
|
30
|
+
spec.add_dependency 'json', '~> 1.8.3'
|
|
31
|
+
spec.add_dependency 'sneakers', '~> 2.3.5'
|
|
32
|
+
spec.add_dependency 'elasticsearch', '~> 1.0.15'
|
|
33
|
+
|
|
34
|
+
spec.add_development_dependency 'bundler', '~> 1.10'
|
|
35
|
+
spec.add_development_dependency 'rake', '~> 10.0'
|
|
36
|
+
end
|
metadata
ADDED
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
|
2
|
+
name: packageiq
|
|
3
|
+
version: !ruby/object:Gem::Version
|
|
4
|
+
version: 0.1.0
|
|
5
|
+
platform: ruby
|
|
6
|
+
authors:
|
|
7
|
+
- Eric Olsen
|
|
8
|
+
autorequire:
|
|
9
|
+
bindir: exe
|
|
10
|
+
cert_chain: []
|
|
11
|
+
date: 2016-02-08 00:00:00.000000000 Z
|
|
12
|
+
dependencies:
|
|
13
|
+
- !ruby/object:Gem::Dependency
|
|
14
|
+
name: bunny
|
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
|
16
|
+
requirements:
|
|
17
|
+
- - "~>"
|
|
18
|
+
- !ruby/object:Gem::Version
|
|
19
|
+
version: 2.2.2
|
|
20
|
+
type: :runtime
|
|
21
|
+
prerelease: false
|
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
23
|
+
requirements:
|
|
24
|
+
- - "~>"
|
|
25
|
+
- !ruby/object:Gem::Version
|
|
26
|
+
version: 2.2.2
|
|
27
|
+
- !ruby/object:Gem::Dependency
|
|
28
|
+
name: json
|
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
|
30
|
+
requirements:
|
|
31
|
+
- - "~>"
|
|
32
|
+
- !ruby/object:Gem::Version
|
|
33
|
+
version: 1.8.3
|
|
34
|
+
type: :runtime
|
|
35
|
+
prerelease: false
|
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
37
|
+
requirements:
|
|
38
|
+
- - "~>"
|
|
39
|
+
- !ruby/object:Gem::Version
|
|
40
|
+
version: 1.8.3
|
|
41
|
+
- !ruby/object:Gem::Dependency
|
|
42
|
+
name: sneakers
|
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
|
44
|
+
requirements:
|
|
45
|
+
- - "~>"
|
|
46
|
+
- !ruby/object:Gem::Version
|
|
47
|
+
version: 2.3.5
|
|
48
|
+
type: :runtime
|
|
49
|
+
prerelease: false
|
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
51
|
+
requirements:
|
|
52
|
+
- - "~>"
|
|
53
|
+
- !ruby/object:Gem::Version
|
|
54
|
+
version: 2.3.5
|
|
55
|
+
- !ruby/object:Gem::Dependency
|
|
56
|
+
name: elasticsearch
|
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
|
58
|
+
requirements:
|
|
59
|
+
- - "~>"
|
|
60
|
+
- !ruby/object:Gem::Version
|
|
61
|
+
version: 1.0.15
|
|
62
|
+
type: :runtime
|
|
63
|
+
prerelease: false
|
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
65
|
+
requirements:
|
|
66
|
+
- - "~>"
|
|
67
|
+
- !ruby/object:Gem::Version
|
|
68
|
+
version: 1.0.15
|
|
69
|
+
- !ruby/object:Gem::Dependency
|
|
70
|
+
name: bundler
|
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
|
72
|
+
requirements:
|
|
73
|
+
- - "~>"
|
|
74
|
+
- !ruby/object:Gem::Version
|
|
75
|
+
version: '1.10'
|
|
76
|
+
type: :development
|
|
77
|
+
prerelease: false
|
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
79
|
+
requirements:
|
|
80
|
+
- - "~>"
|
|
81
|
+
- !ruby/object:Gem::Version
|
|
82
|
+
version: '1.10'
|
|
83
|
+
- !ruby/object:Gem::Dependency
|
|
84
|
+
name: rake
|
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
|
86
|
+
requirements:
|
|
87
|
+
- - "~>"
|
|
88
|
+
- !ruby/object:Gem::Version
|
|
89
|
+
version: '10.0'
|
|
90
|
+
type: :development
|
|
91
|
+
prerelease: false
|
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
93
|
+
requirements:
|
|
94
|
+
- - "~>"
|
|
95
|
+
- !ruby/object:Gem::Version
|
|
96
|
+
version: '10.0'
|
|
97
|
+
description: Package Intelligence Gathering System
|
|
98
|
+
email:
|
|
99
|
+
- eric@ericolsen.net
|
|
100
|
+
executables:
|
|
101
|
+
- piq_indexer
|
|
102
|
+
- piq_rhel
|
|
103
|
+
extensions: []
|
|
104
|
+
extra_rdoc_files: []
|
|
105
|
+
files:
|
|
106
|
+
- ".gitignore"
|
|
107
|
+
- ".travis.yml"
|
|
108
|
+
- Gemfile
|
|
109
|
+
- README.md
|
|
110
|
+
- Rakefile
|
|
111
|
+
- bin/console
|
|
112
|
+
- bin/setup
|
|
113
|
+
- exe/piq_indexer
|
|
114
|
+
- exe/piq_rhel
|
|
115
|
+
- lib/packageiq.rb
|
|
116
|
+
- lib/packageiq/cli.rb
|
|
117
|
+
- lib/packageiq/command.rb
|
|
118
|
+
- lib/packageiq/loader.rb
|
|
119
|
+
- lib/packageiq/mappings/elasticsearch.rb
|
|
120
|
+
- lib/packageiq/provider/rhel.rb
|
|
121
|
+
- lib/packageiq/transport/rabbitmq.rb
|
|
122
|
+
- lib/packageiq/utils.rb
|
|
123
|
+
- lib/packageiq/version.rb
|
|
124
|
+
- packageiq.gemspec
|
|
125
|
+
homepage: http://www.packageiq.io
|
|
126
|
+
licenses: []
|
|
127
|
+
metadata:
|
|
128
|
+
allowed_push_host: https://rubygems.org
|
|
129
|
+
post_install_message:
|
|
130
|
+
rdoc_options: []
|
|
131
|
+
require_paths:
|
|
132
|
+
- lib
|
|
133
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
|
134
|
+
requirements:
|
|
135
|
+
- - ">="
|
|
136
|
+
- !ruby/object:Gem::Version
|
|
137
|
+
version: '0'
|
|
138
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
139
|
+
requirements:
|
|
140
|
+
- - ">="
|
|
141
|
+
- !ruby/object:Gem::Version
|
|
142
|
+
version: '0'
|
|
143
|
+
requirements: []
|
|
144
|
+
rubyforge_project:
|
|
145
|
+
rubygems_version: 2.4.8
|
|
146
|
+
signing_key:
|
|
147
|
+
specification_version: 4
|
|
148
|
+
summary: Package Intelligence Gathering System
|
|
149
|
+
test_files: []
|
|
150
|
+
has_rdoc:
|