vault-tools 0.7.1 → 2.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.travis.yml +4 -3
- data/Gemfile +4 -3
- data/Gemfile.lock +65 -72
- data/README.md +0 -13
- data/lib/vault-tools.rb +15 -16
- data/lib/vault-tools/app.rb +3 -3
- data/lib/vault-tools/config.rb +6 -24
- data/lib/vault-tools/log.rb +4 -4
- data/lib/vault-tools/pipeline.rb +1 -1
- data/lib/vault-tools/s3.rb +9 -9
- data/lib/vault-tools/statement_store.rb +15 -9
- data/lib/vault-tools/time.rb +1 -1
- data/lib/vault-tools/user.rb +3 -3
- data/lib/vault-tools/version.rb +3 -1
- data/lib/vault-tools/web.rb +9 -1
- data/test/config_test.rb +7 -36
- data/test/defaults_test.rb +4 -4
- data/test/helper.rb +12 -46
- data/test/log_test.rb +5 -4
- data/test/s3_test.rb +16 -16
- data/test/statement_store_test.rb +9 -12
- data/test/web_test.rb +1 -0
- data/vault-tools.gemspec +10 -9
- metadata +17 -46
- data/lib/vault-tools/tracing.rb +0 -92
- data/lib/vault-tools/tracing/sidekiq_client.rb +0 -37
- data/lib/vault-tools/tracing/sidekiq_server.rb +0 -54
- data/test/tracing_test.rb +0 -86
@@ -2,8 +2,9 @@ module Vault
|
|
2
2
|
# The StatementStore knows how to save and retrieve invoices from S3
|
3
3
|
class StatementStore
|
4
4
|
def initialize(opts = {})
|
5
|
-
@
|
6
|
-
|
5
|
+
@credentials = Aws::Credentials.new(opts.fetch(:key_id, Config[:aws_access_key_id]),
|
6
|
+
opts.fetch(:key, Config[:aws_secret_access_key]))
|
7
|
+
@region = opts.fetch(:region, Config[:aws_region])
|
7
8
|
end
|
8
9
|
|
9
10
|
# Retrieve invoice JSON from S3
|
@@ -34,20 +35,24 @@ module Vault
|
|
34
35
|
|
35
36
|
# Retrieve the contents in a given format of a given file from S3
|
36
37
|
def retrieve(format, opts)
|
37
|
-
s3.
|
38
|
+
s3.get_object({
|
39
|
+
bucket: bucket_for(format, opts),
|
40
|
+
key: path_for(opts)
|
41
|
+
}).body.read
|
38
42
|
end
|
39
43
|
|
40
44
|
# Write the contents in the given format to S3
|
41
45
|
def write(format, opts)
|
42
|
-
|
43
|
-
|
44
|
-
|
46
|
+
s3.put_object({
|
47
|
+
bucket: bucket_for(format, opts),
|
48
|
+
key: path_for(opts),
|
49
|
+
body: opts[:contents]
|
50
|
+
})
|
45
51
|
end
|
46
52
|
|
47
53
|
# Get an instance of the S3 client to work with
|
48
54
|
def s3
|
49
|
-
@s3 ||=
|
50
|
-
use_ssl: true)
|
55
|
+
@s3 ||= Aws::S3::Client.new(credentials: @credentials, region: @region)
|
51
56
|
end
|
52
57
|
|
53
58
|
# Determine which bucket an invoice should live in
|
@@ -73,10 +78,11 @@ module Vault
|
|
73
78
|
private
|
74
79
|
|
75
80
|
def validate_path_opts(opts)
|
81
|
+
user = opts[:user_hid] || opts[:user_id]
|
82
|
+
|
76
83
|
fail(ArgumentError, 'start_time required!') unless opts[:start_time]
|
77
84
|
fail(ArgumentError, 'stop_time required!') unless opts[:stop_time]
|
78
85
|
fail(ArgumentError, 'version required!') unless opts[:version]
|
79
|
-
user = opts[:user_hid] || opts[:user_id]
|
80
86
|
fail(ArgumentError, 'user_hid or or user_id required!') unless user
|
81
87
|
end
|
82
88
|
end
|
data/lib/vault-tools/time.rb
CHANGED
data/lib/vault-tools/user.rb
CHANGED
@@ -5,7 +5,7 @@ module Vault
|
|
5
5
|
ID_CAPTURE = /^user(\d+)\@[\w\.]+com$/
|
6
6
|
# Convert a user ID into a Heroku user ID.
|
7
7
|
#
|
8
|
-
# @param user_id [
|
8
|
+
# @param user_id [Integer] A user ID.
|
9
9
|
# @return [String] A Heroku ID that uniquely represents the user.
|
10
10
|
def self.id_to_hid(user_id)
|
11
11
|
"user#{user_id}@heroku.com"
|
@@ -13,7 +13,7 @@ module Vault
|
|
13
13
|
|
14
14
|
# Convert a user ID into a v5 UUID.
|
15
15
|
#
|
16
|
-
# @param user_id [
|
16
|
+
# @param user_id [Integer] A user ID.
|
17
17
|
# @return [String] A v5 UUID that uniquely represents the user.
|
18
18
|
def self.id_to_uuid(user_id)
|
19
19
|
url = "https://vault.heroku.com/users/#{user_id}"
|
@@ -25,7 +25,7 @@ module Vault
|
|
25
25
|
# @param heroku_id [String] A Heroku user ID, such as
|
26
26
|
# `user1234@heroku.com`.
|
27
27
|
# @raise [ArgumentError] Raised if a malformed Heroku ID is provided.
|
28
|
-
# @return [
|
28
|
+
# @return [Integer] The core user ID that uniquely represents the user.
|
29
29
|
def self.hid_to_id(heroku_id)
|
30
30
|
if user_id = heroku_id.slice(ID_CAPTURE, 1)
|
31
31
|
user_id.to_i
|
data/lib/vault-tools/version.rb
CHANGED
data/lib/vault-tools/web.rb
CHANGED
@@ -100,11 +100,19 @@ module Vault
|
|
100
100
|
# Start timing the request.
|
101
101
|
before do
|
102
102
|
@start_request = Time.now
|
103
|
+
# if client sends content_type: application/json which no longer works
|
104
|
+
# https://github.com/puma/puma/compare/4.3.1...4.3.5#commitcomment-39478516
|
105
|
+
#
|
106
|
+
# fixing this here so that we do not break all client users at once
|
107
|
+
if request.env["CONTENT_TYPE"].nil? && request.env["HTTP_CONTENT_TYPE"]
|
108
|
+
request.env["CONTENT_TYPE"] = request.env["HTTP_CONTENT_TYPE"]
|
109
|
+
end
|
103
110
|
end
|
104
111
|
|
105
112
|
# Log details about the request including how long it took.
|
106
113
|
after do
|
107
|
-
|
114
|
+
@action ||= 'unknown'
|
115
|
+
Log.count_status(response.status, request_path: request.path_info)
|
108
116
|
Log.time("http.#{@action}", (Time.now - @start_request) * 1000)
|
109
117
|
end
|
110
118
|
|
data/test/config_test.rb
CHANGED
@@ -9,35 +9,6 @@ class ConfigTest < Vault::TestCase
|
|
9
9
|
Config.reset!
|
10
10
|
end
|
11
11
|
|
12
|
-
# Config.remote_env uses the Heroku API to read config vars from
|
13
|
-
# other apps.
|
14
|
-
def test_remote_env
|
15
|
-
api_mock = MiniTest::Mock.new
|
16
|
-
api_response = OpenStruct.new(body: {'DATABASE_URL' => 'postgres:///foo'})
|
17
|
-
Heroku::API.stub(:new, api_mock) do
|
18
|
-
api_mock.expect(:get_config_vars, api_response, ['app'])
|
19
|
-
assert_equal('postgres:///foo',
|
20
|
-
Config.remote_env('app', 'DATABASE_URL'))
|
21
|
-
end
|
22
|
-
end
|
23
|
-
|
24
|
-
# Config.remote_env uses the Heroku API to read config vars from
|
25
|
-
# other apps.
|
26
|
-
def test_shared_config_loads_shared_with_correct_precedence
|
27
|
-
set_env('CONFIG_APP', 'vault-config')
|
28
|
-
api_mock = MiniTest::Mock.new
|
29
|
-
api_response = OpenStruct.new(body: {'HELLO' => 'world'})
|
30
|
-
Heroku::API.stub(:new, api_mock) do
|
31
|
-
api_mock.expect(:get_config_vars, api_response, ['vault-config'])
|
32
|
-
assert_equal(nil, Config[:hello])
|
33
|
-
Config.default(:hello, 'foo')
|
34
|
-
assert_equal('foo', Config[:hello])
|
35
|
-
# this is how we'll call it in the code
|
36
|
-
Config.load_shared!(Config[:config_app])
|
37
|
-
assert_equal('world', Config[:hello])
|
38
|
-
end
|
39
|
-
end
|
40
|
-
|
41
12
|
# Config.env returns the value matching the specified environment
|
42
13
|
# variable name.
|
43
14
|
def test_env
|
@@ -55,7 +26,7 @@ class ConfigTest < Vault::TestCase
|
|
55
26
|
# Config.env return nil if an unknown environment variable is
|
56
27
|
# requested.
|
57
28
|
def test_env_with_unknown_name
|
58
|
-
|
29
|
+
assert_nil(Config.env('UNKNOWN'))
|
59
30
|
end
|
60
31
|
|
61
32
|
# Config.env! returns the value matching the specified environment
|
@@ -114,14 +85,14 @@ class ConfigTest < Vault::TestCase
|
|
114
85
|
# variable.
|
115
86
|
def test_app_name
|
116
87
|
set_env 'APP_NAME', "my-app"
|
117
|
-
Config.app_name
|
88
|
+
assert_equal(Config.app_name, 'my-app')
|
118
89
|
end
|
119
90
|
|
120
91
|
# Config.app_deploy returns the value of the APP_DEPLOY environment
|
121
92
|
# variable.
|
122
93
|
def test_app_deploy
|
123
94
|
set_env 'APP_DEPLOY', "test"
|
124
|
-
Config.app_deploy
|
95
|
+
assert_equal(Config.app_deploy, 'test')
|
125
96
|
end
|
126
97
|
|
127
98
|
# Config.port raises a RuntimeError if no `PORT` environment variable
|
@@ -132,7 +103,7 @@ class ConfigTest < Vault::TestCase
|
|
132
103
|
end
|
133
104
|
end
|
134
105
|
|
135
|
-
# Config.port converts the value from the environment to a
|
106
|
+
# Config.port converts the value from the environment to a Integer
|
136
107
|
def test_port_convert_to_int
|
137
108
|
set_env 'PORT', "3000"
|
138
109
|
assert_equal(3000, Config.port)
|
@@ -150,14 +121,14 @@ class ConfigTest < Vault::TestCase
|
|
150
121
|
|
151
122
|
# Config.int(VAR) returns nil or VAR as integer.
|
152
123
|
def test_int
|
153
|
-
|
124
|
+
assert_nil Config.int('FOO')
|
154
125
|
set_env 'FOO', "3000"
|
155
126
|
assert_equal(3000, Config.int('FOO'))
|
156
127
|
end
|
157
128
|
|
158
129
|
# Config.time returns nil or VAR as time
|
159
130
|
def test_time
|
160
|
-
|
131
|
+
assert_nil(Config.time('T'))
|
161
132
|
set_env 'T', '2000'
|
162
133
|
assert_equal(Time.utc(2000), Config.time(:t))
|
163
134
|
set_env 'T', '2000-2'
|
@@ -170,7 +141,7 @@ class ConfigTest < Vault::TestCase
|
|
170
141
|
|
171
142
|
# Config.time returns nil or VAR as URI
|
172
143
|
def test_uri
|
173
|
-
|
144
|
+
assert_nil(Config.uri('URL'))
|
174
145
|
set_env 'URL', 'http://user:password@the-web.com/path/to/greatness?foo=bar'
|
175
146
|
uri = Config.uri('URL')
|
176
147
|
assert_equal('http', uri.scheme)
|
data/test/defaults_test.rb
CHANGED
@@ -8,13 +8,13 @@ class DefaultsTest < Vault::TestCase
|
|
8
8
|
end
|
9
9
|
|
10
10
|
def test_default_when_no_value
|
11
|
-
|
11
|
+
assert_nil(Config[:max_connections])
|
12
12
|
Config.default(:max_connections, 10)
|
13
13
|
assert_equal(Config[:max_connections], 10)
|
14
14
|
end
|
15
15
|
|
16
16
|
def test_default_with_int
|
17
|
-
|
17
|
+
assert_nil(Config[:max_connections])
|
18
18
|
Config.default(:max_connections, '10')
|
19
19
|
assert_equal('10', Config[:max_connections])
|
20
20
|
assert_equal(10, Config.int(:max_connections))
|
@@ -23,7 +23,7 @@ class DefaultsTest < Vault::TestCase
|
|
23
23
|
end
|
24
24
|
|
25
25
|
def test_default_with_time
|
26
|
-
|
26
|
+
assert_nil(Config[:date])
|
27
27
|
Config.default(:date, '2013-01-01')
|
28
28
|
assert_equal('2013-01-01', Config[:date])
|
29
29
|
assert_equal(Time.utc(2013), Config.time(:date))
|
@@ -33,7 +33,7 @@ class DefaultsTest < Vault::TestCase
|
|
33
33
|
end
|
34
34
|
|
35
35
|
def test_default_with_array
|
36
|
-
|
36
|
+
assert_nil(Config[:array])
|
37
37
|
Config.default(:array, '10')
|
38
38
|
assert_equal('10', Config[:array])
|
39
39
|
assert_equal(['10'], Config.array(:array))
|
data/test/helper.rb
CHANGED
@@ -1,8 +1,11 @@
|
|
1
1
|
require 'vault-test-tools'
|
2
2
|
require 'vault-tools'
|
3
3
|
require 'rr'
|
4
|
+
require 'minitest/around/unit'
|
5
|
+
require 'pry'
|
4
6
|
|
5
7
|
ENV['RACK_ENV'] = 'test'
|
8
|
+
ENV['AWS_REGION'] = 'us-east-1'
|
6
9
|
|
7
10
|
module LoggedDataHelper
|
8
11
|
def logged_data
|
@@ -39,69 +42,32 @@ class Vault::TestCase
|
|
39
42
|
end
|
40
43
|
|
41
44
|
module StubbedS3
|
42
|
-
class FakeFile
|
43
|
-
def initialize(contents=nil)
|
44
|
-
@contents = contents
|
45
|
-
end
|
46
|
-
|
47
|
-
def write(contents)
|
48
|
-
@contents = contents
|
49
|
-
end
|
50
|
-
|
51
|
-
def read
|
52
|
-
@contents
|
53
|
-
end
|
54
|
-
end
|
55
|
-
|
56
|
-
class FakeBucket
|
57
|
-
def initialize
|
58
|
-
@files = {}
|
59
|
-
end
|
60
|
-
|
61
|
-
def [](file_name)
|
62
|
-
@files[file_name] ||= FakeFile.new
|
63
|
-
end
|
64
|
-
|
65
|
-
def write(file_name, contents)
|
66
|
-
@files[file_name].write(contents)
|
67
|
-
end
|
68
|
-
|
69
|
-
def objects
|
70
|
-
self
|
71
|
-
end
|
72
|
-
end
|
73
|
-
|
74
45
|
class FakeClient
|
75
46
|
def initialize
|
76
|
-
@
|
47
|
+
@files = {}
|
77
48
|
end
|
78
49
|
|
79
|
-
def
|
80
|
-
@
|
50
|
+
def put_object(opts)
|
51
|
+
@files["#{opts[:bucket]}::#{opts[:key]}"] = opts[:body]
|
81
52
|
end
|
82
53
|
|
83
|
-
def
|
84
|
-
|
54
|
+
def get_object(opts)
|
55
|
+
val = @files["#{opts[:bucket]}::#{opts[:key]}"]
|
56
|
+
OpenStruct.new(body: OpenStruct.new(read: val))
|
85
57
|
end
|
86
58
|
end
|
87
59
|
|
88
60
|
class << self
|
89
61
|
def seed(bucket, file, contents)
|
90
|
-
fake_client.
|
62
|
+
fake_client.put_object({bucket: bucket, key: file, body: contents})
|
91
63
|
end
|
92
64
|
|
93
65
|
def fake_client
|
94
66
|
@client ||= FakeClient.new
|
95
67
|
end
|
96
68
|
|
97
|
-
def enable!(
|
98
|
-
|
99
|
-
expected_aws_args = {
|
100
|
-
access_key_id: opts.fetch(:access_key_id, 'FAKE_ID'),
|
101
|
-
secret_access_key: opts.fetch(:secret_access_key, 'FAKE_KEY'),
|
102
|
-
use_ssl: opts.fetch(:use_ssl, true)
|
103
|
-
}
|
104
|
-
env.stub(AWS::S3).new(expected_aws_args) { fake_client }
|
69
|
+
def enable!(opts={}, &block)
|
70
|
+
Aws::S3::Client.stub(:new, fake_client) { yield }
|
105
71
|
end
|
106
72
|
end
|
107
73
|
end
|
data/test/log_test.rb
CHANGED
@@ -31,10 +31,11 @@ class LogTest < Vault::TestCase
|
|
31
31
|
|
32
32
|
# Vault::Log.count_status emits metrics to measure HTTP responses.
|
33
33
|
def test_count_status
|
34
|
-
Vault::Log.count_status(201)
|
35
|
-
assert_equal
|
36
|
-
assert_equal
|
37
|
-
assert_equal '
|
34
|
+
Vault::Log.count_status(201, request_path: '/some/request/path')
|
35
|
+
assert_equal 1, logdata['count#test-app.http.201']
|
36
|
+
assert_equal 1, logdata['count#test-app.http.2xx']
|
37
|
+
assert_equal '/some/request/path', logdata['request_path']
|
38
|
+
assert_equal 'test-deploy', logdata['source']
|
38
39
|
end
|
39
40
|
|
40
41
|
def test_measure
|
data/test/s3_test.rb
CHANGED
@@ -7,13 +7,14 @@ end
|
|
7
7
|
class S3Test < Vault::TestCase
|
8
8
|
include LoggedDataHelper
|
9
9
|
|
10
|
-
def
|
11
|
-
super
|
10
|
+
def around
|
12
11
|
set_env 'APP_DEPLOY', 'test'
|
13
12
|
set_env 'AWS_ACCESS_KEY_ID', 'fake access key id'
|
14
13
|
set_env 'AWS_SECRET_ACCESS_KEY', 'fake secret access key'
|
15
|
-
|
16
|
-
|
14
|
+
StubbedS3.enable! do
|
15
|
+
@consumer = SomeS3Consumer.new
|
16
|
+
yield
|
17
|
+
end
|
17
18
|
end
|
18
19
|
|
19
20
|
def log_output
|
@@ -23,33 +24,32 @@ class S3Test < Vault::TestCase
|
|
23
24
|
# S3 writes should be logged.
|
24
25
|
def test_write_logs
|
25
26
|
@consumer.write('fake bucket', 'fake key', 'fake value')
|
26
|
-
assert_match
|
27
|
+
assert_match(/fake key/, log_output)
|
27
28
|
end
|
28
29
|
|
29
30
|
# S3 reads should be logged.
|
30
31
|
def test_read_logs
|
31
32
|
@consumer.read('fake bucket', 'fake key')
|
32
|
-
assert_match
|
33
|
+
assert_match(/fake key/, log_output)
|
33
34
|
end
|
34
35
|
|
35
36
|
# Should use S3 to write to bucket
|
36
37
|
def test_writes_to_s3_bucket
|
37
|
-
mock(@consumer).s3.mock!.
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
38
|
+
mock(@consumer).s3.mock!.put_object({
|
39
|
+
bucket: 'fake bucket',
|
40
|
+
key: 'fake key',
|
41
|
+
body: 'fake value'
|
42
|
+
})
|
42
43
|
@consumer.write('fake bucket', 'fake key', 'fake value')
|
43
44
|
end
|
44
45
|
|
45
46
|
# Should use S3 to read from bucket
|
46
47
|
def test_reads_from_s3_bucket
|
47
48
|
#s3.buckets[bucket].objects[key].read
|
48
|
-
mock(@consumer).s3.mock!.
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
mock!.read
|
49
|
+
mock(@consumer).s3.mock!.get_object({
|
50
|
+
bucket: 'fake bucket',
|
51
|
+
key: 'fake key'
|
52
|
+
}).mock!.body.mock!.read
|
53
53
|
@consumer.read('fake bucket', 'fake key')
|
54
54
|
end
|
55
55
|
end
|
@@ -1,12 +1,12 @@
|
|
1
1
|
require 'helper'
|
2
2
|
|
3
3
|
class StatementStoreTest < Vault::TestCase
|
4
|
-
def
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
4
|
+
def around(&block)
|
5
|
+
StubbedS3.enable! do
|
6
|
+
StubbedS3.seed('vault-v2-json-invoice-test',
|
7
|
+
'2014-10-01/2014-11-01/user8@heroku.com_v2', '{"foo": 1}')
|
8
|
+
yield
|
9
|
+
end
|
10
10
|
end
|
11
11
|
|
12
12
|
def test_invoice_path_with_user_id
|
@@ -36,15 +36,12 @@ class StatementStoreTest < Vault::TestCase
|
|
36
36
|
# Nothing Before Write
|
37
37
|
doc = inv.get_json(start_time: '2014-10-01', stop_time: '2014-11-01',
|
38
38
|
user_hid: 'user9@heroku.com', version: 2)
|
39
|
-
|
40
|
-
assert_equal expected, doc
|
39
|
+
assert_nil doc
|
41
40
|
|
42
41
|
# Write to S3 updating expectation
|
43
42
|
expected = {"bar" => 3}
|
44
|
-
|
45
|
-
|
46
|
-
contents: expected)
|
47
|
-
assert obj.is_a? StubbedS3::FakeFile
|
43
|
+
inv.write_json(start_time: '2014-10-01', stop_time: '2014-11-01',
|
44
|
+
user_hid: 'user9@heroku.com', version: 2, contents: expected)
|
48
45
|
|
49
46
|
# There after write
|
50
47
|
doc = inv.get_json(start_time: '2014-10-01', stop_time: '2014-11-01',
|