fare 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +20 -0
- data/.rspec +4 -0
- data/.travis.yml +10 -0
- data/Gemfile +4 -0
- data/README.md +470 -0
- data/Rakefile +28 -0
- data/bin/fare +48 -0
- data/fare.gemspec +35 -0
- data/features/multiqueue.feature +60 -0
- data/features/multistack.feature +65 -0
- data/features/step_definitions/aruba.rb +1 -0
- data/features/step_definitions/fare_steps.rb +40 -0
- data/features/subscriber.feature +95 -0
- data/features/support/env.rb +34 -0
- data/lib/fare.rb +96 -0
- data/lib/fare/configuration.rb +57 -0
- data/lib/fare/configuration_dsl.rb +134 -0
- data/lib/fare/configuration_when_locked.rb +82 -0
- data/lib/fare/event.rb +26 -0
- data/lib/fare/generate_lock_file.rb +131 -0
- data/lib/fare/load_configuration_file.rb +45 -0
- data/lib/fare/middleware/logging.rb +46 -0
- data/lib/fare/middleware/newrelic.rb +35 -0
- data/lib/fare/middleware/raven.rb +47 -0
- data/lib/fare/publisher.rb +65 -0
- data/lib/fare/queue_adapter.rb +30 -0
- data/lib/fare/rspec.rb +85 -0
- data/lib/fare/subscriber.rb +35 -0
- data/lib/fare/subscriber_cli.rb +270 -0
- data/lib/fare/subscriber_stack.rb +39 -0
- data/lib/fare/test_mode.rb +204 -0
- data/lib/fare/topic.rb +25 -0
- data/lib/fare/topic_adapter.rb +13 -0
- data/lib/fare/update_cli.rb +41 -0
- data/lib/fare/version.rb +3 -0
- data/spec/logger_spec.rb +45 -0
- data/spec/raven_spec.rb +52 -0
- data/spec/rspec_integration_spec.rb +45 -0
- data/spec/spec_helper.rb +30 -0
- data/spec/stubbed_subscribing_spec.rb +66 -0
- metadata +264 -0
@@ -0,0 +1,57 @@
|
|
1
|
+
module Fare
|
2
|
+
class Configuration
|
3
|
+
|
4
|
+
attr_reader :publish_topics, :environments, :subscribers
|
5
|
+
attr_accessor :app_name, :app_version, :default_version
|
6
|
+
|
7
|
+
def initialize
|
8
|
+
@publish_topics = []
|
9
|
+
@environments = {}
|
10
|
+
@subscribers = {}
|
11
|
+
@default_version = "0.0"
|
12
|
+
@backup = false
|
13
|
+
end
|
14
|
+
|
15
|
+
def load_environment(environment)
|
16
|
+
environments.fetch(environment) { Proc.new {} }.call
|
17
|
+
end
|
18
|
+
|
19
|
+
def subscribe_topics
|
20
|
+
subscribers.map { |name, config| config.topics }.flatten
|
21
|
+
end
|
22
|
+
|
23
|
+
def subscriber(name)
|
24
|
+
subscribers[name.to_s] ||= SubscriberConfiguration.new(name.to_s)
|
25
|
+
end
|
26
|
+
|
27
|
+
def backup?
|
28
|
+
@backup
|
29
|
+
end
|
30
|
+
|
31
|
+
def backup!
|
32
|
+
@backup = true
|
33
|
+
end
|
34
|
+
|
35
|
+
class SubscriberConfiguration
|
36
|
+
|
37
|
+
attr_accessor :setup, :always_run, :stacks
|
38
|
+
|
39
|
+
def initialize(name)
|
40
|
+
@name = name
|
41
|
+
@stacks = []
|
42
|
+
@setup = Proc.new {}
|
43
|
+
@always_run = Proc.new {}
|
44
|
+
end
|
45
|
+
|
46
|
+
def topics
|
47
|
+
stacks.map(&:topics).flatten
|
48
|
+
end
|
49
|
+
|
50
|
+
def load_setup
|
51
|
+
@setup.call
|
52
|
+
end
|
53
|
+
|
54
|
+
end
|
55
|
+
|
56
|
+
end
|
57
|
+
end
|
@@ -0,0 +1,134 @@
|
|
1
|
+
module Fare
|
2
|
+
|
3
|
+
# This class is the context in which the fare config file is loaded.
|
4
|
+
# It provides a DSL that fills a configuration object.
|
5
|
+
class ConfigurationDSL
|
6
|
+
|
7
|
+
InvalidOption = Class.new(ArgumentError)
|
8
|
+
VALID_PART_OF_TOPIC_NAME = /\A\w+\z/
|
9
|
+
|
10
|
+
attr_reader :configuration
|
11
|
+
|
12
|
+
def initialize
|
13
|
+
@configuration = Configuration.new
|
14
|
+
end
|
15
|
+
|
16
|
+
def environment(name, &block)
|
17
|
+
validate! "Environment", name.to_s
|
18
|
+
configuration.environments[name.to_s] = block
|
19
|
+
end
|
20
|
+
|
21
|
+
def app_name(name)
|
22
|
+
validate! "App name", name
|
23
|
+
configuration.app_name = name
|
24
|
+
end
|
25
|
+
|
26
|
+
def default_version(version)
|
27
|
+
configuration.default_version = version
|
28
|
+
end
|
29
|
+
|
30
|
+
def app_version(version)
|
31
|
+
configuration.app_version = version
|
32
|
+
end
|
33
|
+
|
34
|
+
def backup!
|
35
|
+
configuration.backup!
|
36
|
+
end
|
37
|
+
|
38
|
+
def publishes(topic)
|
39
|
+
topic.each do |key, value|
|
40
|
+
case key
|
41
|
+
when :subject then validate! "Subject", value
|
42
|
+
when :action then validate! "Action", value
|
43
|
+
when :version then true
|
44
|
+
else
|
45
|
+
raise InvalidOption, "Unknown option: #{key}"
|
46
|
+
end
|
47
|
+
end
|
48
|
+
configuration.publish_topics << topic
|
49
|
+
end
|
50
|
+
|
51
|
+
def subscriber(name = configuration.app_name, &block)
|
52
|
+
config = configuration.subscriber(name)
|
53
|
+
SubscriberDSL.new(config).parse(&block)
|
54
|
+
end
|
55
|
+
|
56
|
+
private
|
57
|
+
|
58
|
+
def validate!(name, value)
|
59
|
+
if value !~ VALID_PART_OF_TOPIC_NAME
|
60
|
+
raise InvalidOption, "#{name} can only contain word characters: #{value.inspect}"
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
class SubscriberDSL
|
65
|
+
|
66
|
+
attr_reader :configuration
|
67
|
+
|
68
|
+
def initialize(configuration)
|
69
|
+
@configuration = configuration
|
70
|
+
end
|
71
|
+
|
72
|
+
def parse(&block)
|
73
|
+
instance_eval(&block)
|
74
|
+
end
|
75
|
+
|
76
|
+
def stack(&block)
|
77
|
+
stack_dsl = StackDSL.new(configuration)
|
78
|
+
stack_dsl.parse(&block)
|
79
|
+
stack_dsl.verify!
|
80
|
+
stack = stack_dsl.to_stack
|
81
|
+
configuration.stacks << stack
|
82
|
+
end
|
83
|
+
|
84
|
+
def setup(&block)
|
85
|
+
configuration.setup = block
|
86
|
+
end
|
87
|
+
|
88
|
+
def always_run(&block)
|
89
|
+
configuration.always_run = block
|
90
|
+
end
|
91
|
+
|
92
|
+
end
|
93
|
+
|
94
|
+
class StackDSL
|
95
|
+
|
96
|
+
attr_reader :configuration
|
97
|
+
|
98
|
+
def initialize(configuration)
|
99
|
+
@configuration = configuration
|
100
|
+
@topics = []
|
101
|
+
@run = :not_specified
|
102
|
+
end
|
103
|
+
|
104
|
+
def verify!
|
105
|
+
if @topics.empty?
|
106
|
+
raise "Stack without topics for stack on line #{@source_location.last} of #{@source_location.first}"
|
107
|
+
end
|
108
|
+
if @run == :not_specified
|
109
|
+
raise "No run list specified for stack on line #{@source_location.last} of #{@source_location.first}"
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
def to_stack
|
114
|
+
SubscriberStack.new(configuration, @topics, @run)
|
115
|
+
end
|
116
|
+
|
117
|
+
def parse(&block)
|
118
|
+
@source_location = block.source_location
|
119
|
+
instance_eval(&block)
|
120
|
+
end
|
121
|
+
|
122
|
+
def listen_to(topic)
|
123
|
+
@topics << Topic.new(topic)
|
124
|
+
end
|
125
|
+
|
126
|
+
def run(&block)
|
127
|
+
@run = block
|
128
|
+
end
|
129
|
+
|
130
|
+
end
|
131
|
+
|
132
|
+
end
|
133
|
+
|
134
|
+
end
|
@@ -0,0 +1,82 @@
|
|
1
|
+
module Fare
|
2
|
+
class ConfigurationWhenLocked
|
3
|
+
|
4
|
+
attr_reader :options
|
5
|
+
|
6
|
+
def initialize(options)
|
7
|
+
@options = options
|
8
|
+
validate_lockfile!
|
9
|
+
@configuration = loaded_configuration.load_configuration
|
10
|
+
@configuration.load_environment(environment)
|
11
|
+
end
|
12
|
+
|
13
|
+
def validate_subscriber!
|
14
|
+
configuration.setup.call
|
15
|
+
configuration.stacks.each(&:to_app)
|
16
|
+
end
|
17
|
+
|
18
|
+
def find_publishable_topic(options)
|
19
|
+
lock_file_contents.fetch("publishes").find { |t|
|
20
|
+
t.fetch("subject") == options.fetch(:subject) && t.fetch("action") == options.fetch(:action)
|
21
|
+
}
|
22
|
+
end
|
23
|
+
|
24
|
+
def app_name
|
25
|
+
configuration.app_name
|
26
|
+
end
|
27
|
+
|
28
|
+
def app_version
|
29
|
+
configuration.app_version || "unknown"
|
30
|
+
end
|
31
|
+
|
32
|
+
def fetch_subscriber(name)
|
33
|
+
configuration.subscribers.fetch(name)
|
34
|
+
end
|
35
|
+
|
36
|
+
def fetch_subscriber_queue(name)
|
37
|
+
Fare.queue_adapter.fetch(environment, name)
|
38
|
+
end
|
39
|
+
|
40
|
+
private
|
41
|
+
|
42
|
+
def subscribers
|
43
|
+
configuration.subscribers
|
44
|
+
end
|
45
|
+
|
46
|
+
attr_reader :configuration
|
47
|
+
|
48
|
+
def validate_lockfile!
|
49
|
+
unless File.exist?(lock_filename)
|
50
|
+
raise LockFileNotFoundError, "Lockfile doesn't exist at #{lock_filename}"
|
51
|
+
end
|
52
|
+
if loaded_configuration.checksum != lock_file_contents.fetch("checksum")
|
53
|
+
raise ChecksumNotMatchingError, "Checksum of lockfile doesn't match. Run `fare update` maybe?"
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
def lock_filename
|
58
|
+
loaded_configuration.lock_filename
|
59
|
+
end
|
60
|
+
|
61
|
+
def lock_file_contents
|
62
|
+
@lock_file_contents ||= YAML.load(File.open(loaded_configuration.lock_filename, "r:utf-8").read)
|
63
|
+
end
|
64
|
+
|
65
|
+
def loaded_configuration
|
66
|
+
@loaded_configuration ||= LoadConfigurationFile.new(filename: filename, environment: environment)
|
67
|
+
end
|
68
|
+
|
69
|
+
def environment
|
70
|
+
options.fetch(:environment) { find_environment }
|
71
|
+
end
|
72
|
+
|
73
|
+
def filename
|
74
|
+
options.fetch(:filename) { Fare.default_config_file }
|
75
|
+
end
|
76
|
+
|
77
|
+
def find_environment
|
78
|
+
Fare.default_environment || raise(NoEnvironmentFoundError, "No environment specified!")
|
79
|
+
end
|
80
|
+
|
81
|
+
end
|
82
|
+
end
|
data/lib/fare/event.rb
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
module Fare
|
2
|
+
class Event
|
3
|
+
include Virtus.model
|
4
|
+
|
5
|
+
attribute :id
|
6
|
+
attribute :subject
|
7
|
+
attribute :action
|
8
|
+
attribute :payload
|
9
|
+
attribute :source
|
10
|
+
attribute :version
|
11
|
+
attribute :sent_at, DateTime
|
12
|
+
|
13
|
+
def serialize
|
14
|
+
Base64.urlsafe_encode64(attributes.to_json)
|
15
|
+
end
|
16
|
+
|
17
|
+
def self.deserialize(message)
|
18
|
+
new JSON.parse(Base64.urlsafe_decode64(message))
|
19
|
+
end
|
20
|
+
|
21
|
+
def to_json(*)
|
22
|
+
attributes.to_json
|
23
|
+
end
|
24
|
+
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,131 @@
|
|
1
|
+
module Fare
|
2
|
+
class GenerateLockFile
|
3
|
+
extend Forwardable
|
4
|
+
|
5
|
+
def_delegators :@configuration, :environment, :lock_filename, :checksum
|
6
|
+
|
7
|
+
attr_reader :topics, :force, :queues
|
8
|
+
|
9
|
+
def initialize(options = {})
|
10
|
+
@force = options.delete(:force) { false }
|
11
|
+
@configuration = LoadConfigurationFile.new(options)
|
12
|
+
@topics = {}
|
13
|
+
@queues = {}
|
14
|
+
end
|
15
|
+
|
16
|
+
def configuration
|
17
|
+
@configuration.configuration
|
18
|
+
end
|
19
|
+
|
20
|
+
def call
|
21
|
+
return if skip_generating?
|
22
|
+
|
23
|
+
configuration.load_environment(environment)
|
24
|
+
|
25
|
+
find_publishing_topics
|
26
|
+
find_subscriber_queues
|
27
|
+
|
28
|
+
File.open(lock_filename, "w:utf-8") do |f|
|
29
|
+
f.write(data.to_yaml)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
private
|
34
|
+
|
35
|
+
def backup?
|
36
|
+
configuration.backup?
|
37
|
+
end
|
38
|
+
|
39
|
+
def skip_generating?
|
40
|
+
!force && checksum_matches?
|
41
|
+
end
|
42
|
+
|
43
|
+
def checksum_matches?
|
44
|
+
File.exist?(lock_filename) && YAML.load(File.open(lock_filename, "r:utf-8").read).fetch("checksum") == checksum
|
45
|
+
end
|
46
|
+
|
47
|
+
def find_publishing_topics
|
48
|
+
configuration.publish_topics.each do |topic_info|
|
49
|
+
topic = topic_for(topic_info)
|
50
|
+
subscribe topic, backup_queue if backup?
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
def find_subscriber_queues
|
55
|
+
configuration.subscribers.each do |subscriber_name, subscriber|
|
56
|
+
subscriber.topics.each do |topic_info|
|
57
|
+
topic = topic_for(topic_info)
|
58
|
+
subscribe topic, backup_queue if backup?
|
59
|
+
subscribe topic, queue_for(subscriber_name)
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
def data
|
65
|
+
data = {
|
66
|
+
"publishes" => configuration.publish_topics.map { |topic_info| serialize_topic(topic_info) },
|
67
|
+
"subscribes" => configuration.subscribe_topics.map { |topic_info| serialize_topic(topic_info) },
|
68
|
+
"queues" => Hash[ queues.map { |name, queue| [ name, queue.url ] }],
|
69
|
+
"checksum" => checksum,
|
70
|
+
}
|
71
|
+
data["backup_queue"] = backup_queue.url if backup?
|
72
|
+
data
|
73
|
+
end
|
74
|
+
|
75
|
+
def subscribe(topic, queue)
|
76
|
+
subscription = topic.subscribe(queue.arn)
|
77
|
+
subscription.raw_message_delivery = true
|
78
|
+
account_id = topic.owner
|
79
|
+
policy = {
|
80
|
+
"Version" => "2012-10-17",
|
81
|
+
"Id" => "Fare_Policy",
|
82
|
+
"Statement" => [
|
83
|
+
{
|
84
|
+
"Sid" => "1",
|
85
|
+
"Effect" => "Allow",
|
86
|
+
"Principal" => { "AWS" => "*" },
|
87
|
+
"Action" => "SQS:SendMessage",
|
88
|
+
"Resource" => queue.arn,
|
89
|
+
"Condition" => {
|
90
|
+
"ArnLike" => {
|
91
|
+
"aws:SourceArn" => "arn:aws:sns:*:#{account_id}:*"
|
92
|
+
}
|
93
|
+
}
|
94
|
+
},
|
95
|
+
]
|
96
|
+
}
|
97
|
+
queue.policy = policy
|
98
|
+
end
|
99
|
+
|
100
|
+
def serialize_topic(topic_info)
|
101
|
+
{
|
102
|
+
"subject" => topic_info.fetch(:subject),
|
103
|
+
"action" => topic_info.fetch(:action),
|
104
|
+
"version" => topic_info.fetch(:version) { configuration.default_version },
|
105
|
+
"arn" => topic_for(topic_info).arn,
|
106
|
+
}
|
107
|
+
end
|
108
|
+
|
109
|
+
def backup_queue
|
110
|
+
if backup?
|
111
|
+
@backup_queue ||= queue_for("backup")
|
112
|
+
else
|
113
|
+
fail "NO BACKUP FOR YOU!"
|
114
|
+
end
|
115
|
+
end
|
116
|
+
|
117
|
+
def app_name
|
118
|
+
configuration.app_name
|
119
|
+
end
|
120
|
+
|
121
|
+
def topic_for(options)
|
122
|
+
topic_key = [options[:subject], options[:action] ]
|
123
|
+
topics[topic_key] ||= Fare.topic_adapter.create("#{environment}-#{options[:subject]}-#{options[:action]}").tap { |topic| puts "Created topic #{topic.arn}" }
|
124
|
+
end
|
125
|
+
|
126
|
+
def queue_for(name)
|
127
|
+
queues[name] ||= Fare.queue_adapter.fetch(environment, name)
|
128
|
+
end
|
129
|
+
|
130
|
+
end
|
131
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
module Fare
|
2
|
+
class LoadConfigurationFile
|
3
|
+
|
4
|
+
attr_reader :filename, :environment
|
5
|
+
|
6
|
+
def initialize(options)
|
7
|
+
@filename = options.fetch(:filename) { Fare.default_config_file }
|
8
|
+
@environment = options.fetch(:environment) { Fare.default_environment }
|
9
|
+
verify_file!
|
10
|
+
end
|
11
|
+
|
12
|
+
def verify_file!
|
13
|
+
unless exists?
|
14
|
+
raise ConfigurationNotFoundError, "Configuration file doesn't exist at #{filename}"
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def configuration
|
19
|
+
@configuration ||= load_configuration
|
20
|
+
end
|
21
|
+
|
22
|
+
def load_configuration
|
23
|
+
dsl = ConfigurationDSL.new
|
24
|
+
dsl.instance_eval(code, filename)
|
25
|
+
dsl.configuration
|
26
|
+
end
|
27
|
+
|
28
|
+
def code
|
29
|
+
@code ||= File.open(filename.to_s, "r:utf-8").read
|
30
|
+
end
|
31
|
+
|
32
|
+
def lock_filename
|
33
|
+
File.expand_path("../fare.#{environment}.lock", filename)
|
34
|
+
end
|
35
|
+
|
36
|
+
def checksum
|
37
|
+
Digest::MD5.hexdigest(code)
|
38
|
+
end
|
39
|
+
|
40
|
+
def exists?
|
41
|
+
File.exist?(filename)
|
42
|
+
end
|
43
|
+
|
44
|
+
end
|
45
|
+
end
|