fare 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +20 -0
- data/.rspec +4 -0
- data/.travis.yml +10 -0
- data/Gemfile +4 -0
- data/README.md +470 -0
- data/Rakefile +28 -0
- data/bin/fare +48 -0
- data/fare.gemspec +35 -0
- data/features/multiqueue.feature +60 -0
- data/features/multistack.feature +65 -0
- data/features/step_definitions/aruba.rb +1 -0
- data/features/step_definitions/fare_steps.rb +40 -0
- data/features/subscriber.feature +95 -0
- data/features/support/env.rb +34 -0
- data/lib/fare.rb +96 -0
- data/lib/fare/configuration.rb +57 -0
- data/lib/fare/configuration_dsl.rb +134 -0
- data/lib/fare/configuration_when_locked.rb +82 -0
- data/lib/fare/event.rb +26 -0
- data/lib/fare/generate_lock_file.rb +131 -0
- data/lib/fare/load_configuration_file.rb +45 -0
- data/lib/fare/middleware/logging.rb +46 -0
- data/lib/fare/middleware/newrelic.rb +35 -0
- data/lib/fare/middleware/raven.rb +47 -0
- data/lib/fare/publisher.rb +65 -0
- data/lib/fare/queue_adapter.rb +30 -0
- data/lib/fare/rspec.rb +85 -0
- data/lib/fare/subscriber.rb +35 -0
- data/lib/fare/subscriber_cli.rb +270 -0
- data/lib/fare/subscriber_stack.rb +39 -0
- data/lib/fare/test_mode.rb +204 -0
- data/lib/fare/topic.rb +25 -0
- data/lib/fare/topic_adapter.rb +13 -0
- data/lib/fare/update_cli.rb +41 -0
- data/lib/fare/version.rb +3 -0
- data/spec/logger_spec.rb +45 -0
- data/spec/raven_spec.rb +52 -0
- data/spec/rspec_integration_spec.rb +45 -0
- data/spec/spec_helper.rb +30 -0
- data/spec/stubbed_subscribing_spec.rb +66 -0
- metadata +264 -0
@@ -0,0 +1,57 @@
|
|
1
|
+
module Fare
|
2
|
+
class Configuration
|
3
|
+
|
4
|
+
attr_reader :publish_topics, :environments, :subscribers
|
5
|
+
attr_accessor :app_name, :app_version, :default_version
|
6
|
+
|
7
|
+
def initialize
|
8
|
+
@publish_topics = []
|
9
|
+
@environments = {}
|
10
|
+
@subscribers = {}
|
11
|
+
@default_version = "0.0"
|
12
|
+
@backup = false
|
13
|
+
end
|
14
|
+
|
15
|
+
def load_environment(environment)
|
16
|
+
environments.fetch(environment) { Proc.new {} }.call
|
17
|
+
end
|
18
|
+
|
19
|
+
def subscribe_topics
|
20
|
+
subscribers.map { |name, config| config.topics }.flatten
|
21
|
+
end
|
22
|
+
|
23
|
+
def subscriber(name)
|
24
|
+
subscribers[name.to_s] ||= SubscriberConfiguration.new(name.to_s)
|
25
|
+
end
|
26
|
+
|
27
|
+
def backup?
|
28
|
+
@backup
|
29
|
+
end
|
30
|
+
|
31
|
+
def backup!
|
32
|
+
@backup = true
|
33
|
+
end
|
34
|
+
|
35
|
+
class SubscriberConfiguration
|
36
|
+
|
37
|
+
attr_accessor :setup, :always_run, :stacks
|
38
|
+
|
39
|
+
def initialize(name)
|
40
|
+
@name = name
|
41
|
+
@stacks = []
|
42
|
+
@setup = Proc.new {}
|
43
|
+
@always_run = Proc.new {}
|
44
|
+
end
|
45
|
+
|
46
|
+
def topics
|
47
|
+
stacks.map(&:topics).flatten
|
48
|
+
end
|
49
|
+
|
50
|
+
def load_setup
|
51
|
+
@setup.call
|
52
|
+
end
|
53
|
+
|
54
|
+
end
|
55
|
+
|
56
|
+
end
|
57
|
+
end
|
@@ -0,0 +1,134 @@
|
|
1
|
+
module Fare
|
2
|
+
|
3
|
+
# This class is the context in which the fare config file is loaded.
|
4
|
+
# It provides a DSL that fills a configuration object.
|
5
|
+
class ConfigurationDSL
|
6
|
+
|
7
|
+
InvalidOption = Class.new(ArgumentError)
|
8
|
+
VALID_PART_OF_TOPIC_NAME = /\A\w+\z/
|
9
|
+
|
10
|
+
attr_reader :configuration
|
11
|
+
|
12
|
+
def initialize
|
13
|
+
@configuration = Configuration.new
|
14
|
+
end
|
15
|
+
|
16
|
+
def environment(name, &block)
|
17
|
+
validate! "Environment", name.to_s
|
18
|
+
configuration.environments[name.to_s] = block
|
19
|
+
end
|
20
|
+
|
21
|
+
def app_name(name)
|
22
|
+
validate! "App name", name
|
23
|
+
configuration.app_name = name
|
24
|
+
end
|
25
|
+
|
26
|
+
def default_version(version)
|
27
|
+
configuration.default_version = version
|
28
|
+
end
|
29
|
+
|
30
|
+
def app_version(version)
|
31
|
+
configuration.app_version = version
|
32
|
+
end
|
33
|
+
|
34
|
+
def backup!
|
35
|
+
configuration.backup!
|
36
|
+
end
|
37
|
+
|
38
|
+
def publishes(topic)
|
39
|
+
topic.each do |key, value|
|
40
|
+
case key
|
41
|
+
when :subject then validate! "Subject", value
|
42
|
+
when :action then validate! "Action", value
|
43
|
+
when :version then true
|
44
|
+
else
|
45
|
+
raise InvalidOption, "Unknown option: #{key}"
|
46
|
+
end
|
47
|
+
end
|
48
|
+
configuration.publish_topics << topic
|
49
|
+
end
|
50
|
+
|
51
|
+
def subscriber(name = configuration.app_name, &block)
|
52
|
+
config = configuration.subscriber(name)
|
53
|
+
SubscriberDSL.new(config).parse(&block)
|
54
|
+
end
|
55
|
+
|
56
|
+
private
|
57
|
+
|
58
|
+
def validate!(name, value)
|
59
|
+
if value !~ VALID_PART_OF_TOPIC_NAME
|
60
|
+
raise InvalidOption, "#{name} can only contain word characters: #{value.inspect}"
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
class SubscriberDSL
|
65
|
+
|
66
|
+
attr_reader :configuration
|
67
|
+
|
68
|
+
def initialize(configuration)
|
69
|
+
@configuration = configuration
|
70
|
+
end
|
71
|
+
|
72
|
+
def parse(&block)
|
73
|
+
instance_eval(&block)
|
74
|
+
end
|
75
|
+
|
76
|
+
def stack(&block)
|
77
|
+
stack_dsl = StackDSL.new(configuration)
|
78
|
+
stack_dsl.parse(&block)
|
79
|
+
stack_dsl.verify!
|
80
|
+
stack = stack_dsl.to_stack
|
81
|
+
configuration.stacks << stack
|
82
|
+
end
|
83
|
+
|
84
|
+
def setup(&block)
|
85
|
+
configuration.setup = block
|
86
|
+
end
|
87
|
+
|
88
|
+
def always_run(&block)
|
89
|
+
configuration.always_run = block
|
90
|
+
end
|
91
|
+
|
92
|
+
end
|
93
|
+
|
94
|
+
class StackDSL
|
95
|
+
|
96
|
+
attr_reader :configuration
|
97
|
+
|
98
|
+
def initialize(configuration)
|
99
|
+
@configuration = configuration
|
100
|
+
@topics = []
|
101
|
+
@run = :not_specified
|
102
|
+
end
|
103
|
+
|
104
|
+
def verify!
|
105
|
+
if @topics.empty?
|
106
|
+
raise "Stack without topics for stack on line #{@source_location.last} of #{@source_location.first}"
|
107
|
+
end
|
108
|
+
if @run == :not_specified
|
109
|
+
raise "No run list specified for stack on line #{@source_location.last} of #{@source_location.first}"
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
def to_stack
|
114
|
+
SubscriberStack.new(configuration, @topics, @run)
|
115
|
+
end
|
116
|
+
|
117
|
+
def parse(&block)
|
118
|
+
@source_location = block.source_location
|
119
|
+
instance_eval(&block)
|
120
|
+
end
|
121
|
+
|
122
|
+
def listen_to(topic)
|
123
|
+
@topics << Topic.new(topic)
|
124
|
+
end
|
125
|
+
|
126
|
+
def run(&block)
|
127
|
+
@run = block
|
128
|
+
end
|
129
|
+
|
130
|
+
end
|
131
|
+
|
132
|
+
end
|
133
|
+
|
134
|
+
end
|
@@ -0,0 +1,82 @@
|
|
1
|
+
module Fare
|
2
|
+
class ConfigurationWhenLocked
|
3
|
+
|
4
|
+
attr_reader :options
|
5
|
+
|
6
|
+
def initialize(options)
|
7
|
+
@options = options
|
8
|
+
validate_lockfile!
|
9
|
+
@configuration = loaded_configuration.load_configuration
|
10
|
+
@configuration.load_environment(environment)
|
11
|
+
end
|
12
|
+
|
13
|
+
def validate_subscriber!
|
14
|
+
configuration.setup.call
|
15
|
+
configuration.stacks.each(&:to_app)
|
16
|
+
end
|
17
|
+
|
18
|
+
def find_publishable_topic(options)
|
19
|
+
lock_file_contents.fetch("publishes").find { |t|
|
20
|
+
t.fetch("subject") == options.fetch(:subject) && t.fetch("action") == options.fetch(:action)
|
21
|
+
}
|
22
|
+
end
|
23
|
+
|
24
|
+
def app_name
|
25
|
+
configuration.app_name
|
26
|
+
end
|
27
|
+
|
28
|
+
def app_version
|
29
|
+
configuration.app_version || "unknown"
|
30
|
+
end
|
31
|
+
|
32
|
+
def fetch_subscriber(name)
|
33
|
+
configuration.subscribers.fetch(name)
|
34
|
+
end
|
35
|
+
|
36
|
+
def fetch_subscriber_queue(name)
|
37
|
+
Fare.queue_adapter.fetch(environment, name)
|
38
|
+
end
|
39
|
+
|
40
|
+
private
|
41
|
+
|
42
|
+
def subscribers
|
43
|
+
configuration.subscribers
|
44
|
+
end
|
45
|
+
|
46
|
+
attr_reader :configuration
|
47
|
+
|
48
|
+
def validate_lockfile!
|
49
|
+
unless File.exist?(lock_filename)
|
50
|
+
raise LockFileNotFoundError, "Lockfile doesn't exist at #{lock_filename}"
|
51
|
+
end
|
52
|
+
if loaded_configuration.checksum != lock_file_contents.fetch("checksum")
|
53
|
+
raise ChecksumNotMatchingError, "Checksum of lockfile doesn't match. Run `fare update` maybe?"
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
def lock_filename
|
58
|
+
loaded_configuration.lock_filename
|
59
|
+
end
|
60
|
+
|
61
|
+
def lock_file_contents
|
62
|
+
@lock_file_contents ||= YAML.load(File.open(loaded_configuration.lock_filename, "r:utf-8").read)
|
63
|
+
end
|
64
|
+
|
65
|
+
def loaded_configuration
|
66
|
+
@loaded_configuration ||= LoadConfigurationFile.new(filename: filename, environment: environment)
|
67
|
+
end
|
68
|
+
|
69
|
+
def environment
|
70
|
+
options.fetch(:environment) { find_environment }
|
71
|
+
end
|
72
|
+
|
73
|
+
def filename
|
74
|
+
options.fetch(:filename) { Fare.default_config_file }
|
75
|
+
end
|
76
|
+
|
77
|
+
def find_environment
|
78
|
+
Fare.default_environment || raise(NoEnvironmentFoundError, "No environment specified!")
|
79
|
+
end
|
80
|
+
|
81
|
+
end
|
82
|
+
end
|
data/lib/fare/event.rb
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
module Fare
|
2
|
+
class Event
|
3
|
+
include Virtus.model
|
4
|
+
|
5
|
+
attribute :id
|
6
|
+
attribute :subject
|
7
|
+
attribute :action
|
8
|
+
attribute :payload
|
9
|
+
attribute :source
|
10
|
+
attribute :version
|
11
|
+
attribute :sent_at, DateTime
|
12
|
+
|
13
|
+
def serialize
|
14
|
+
Base64.urlsafe_encode64(attributes.to_json)
|
15
|
+
end
|
16
|
+
|
17
|
+
def self.deserialize(message)
|
18
|
+
new JSON.parse(Base64.urlsafe_decode64(message))
|
19
|
+
end
|
20
|
+
|
21
|
+
def to_json(*)
|
22
|
+
attributes.to_json
|
23
|
+
end
|
24
|
+
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,131 @@
|
|
1
|
+
module Fare
|
2
|
+
class GenerateLockFile
|
3
|
+
extend Forwardable
|
4
|
+
|
5
|
+
def_delegators :@configuration, :environment, :lock_filename, :checksum
|
6
|
+
|
7
|
+
attr_reader :topics, :force, :queues
|
8
|
+
|
9
|
+
def initialize(options = {})
|
10
|
+
@force = options.delete(:force) { false }
|
11
|
+
@configuration = LoadConfigurationFile.new(options)
|
12
|
+
@topics = {}
|
13
|
+
@queues = {}
|
14
|
+
end
|
15
|
+
|
16
|
+
def configuration
|
17
|
+
@configuration.configuration
|
18
|
+
end
|
19
|
+
|
20
|
+
def call
|
21
|
+
return if skip_generating?
|
22
|
+
|
23
|
+
configuration.load_environment(environment)
|
24
|
+
|
25
|
+
find_publishing_topics
|
26
|
+
find_subscriber_queues
|
27
|
+
|
28
|
+
File.open(lock_filename, "w:utf-8") do |f|
|
29
|
+
f.write(data.to_yaml)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
private
|
34
|
+
|
35
|
+
def backup?
|
36
|
+
configuration.backup?
|
37
|
+
end
|
38
|
+
|
39
|
+
def skip_generating?
|
40
|
+
!force && checksum_matches?
|
41
|
+
end
|
42
|
+
|
43
|
+
def checksum_matches?
|
44
|
+
File.exist?(lock_filename) && YAML.load(File.open(lock_filename, "r:utf-8").read).fetch("checksum") == checksum
|
45
|
+
end
|
46
|
+
|
47
|
+
def find_publishing_topics
|
48
|
+
configuration.publish_topics.each do |topic_info|
|
49
|
+
topic = topic_for(topic_info)
|
50
|
+
subscribe topic, backup_queue if backup?
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
def find_subscriber_queues
|
55
|
+
configuration.subscribers.each do |subscriber_name, subscriber|
|
56
|
+
subscriber.topics.each do |topic_info|
|
57
|
+
topic = topic_for(topic_info)
|
58
|
+
subscribe topic, backup_queue if backup?
|
59
|
+
subscribe topic, queue_for(subscriber_name)
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
def data
|
65
|
+
data = {
|
66
|
+
"publishes" => configuration.publish_topics.map { |topic_info| serialize_topic(topic_info) },
|
67
|
+
"subscribes" => configuration.subscribe_topics.map { |topic_info| serialize_topic(topic_info) },
|
68
|
+
"queues" => Hash[ queues.map { |name, queue| [ name, queue.url ] }],
|
69
|
+
"checksum" => checksum,
|
70
|
+
}
|
71
|
+
data["backup_queue"] = backup_queue.url if backup?
|
72
|
+
data
|
73
|
+
end
|
74
|
+
|
75
|
+
def subscribe(topic, queue)
|
76
|
+
subscription = topic.subscribe(queue.arn)
|
77
|
+
subscription.raw_message_delivery = true
|
78
|
+
account_id = topic.owner
|
79
|
+
policy = {
|
80
|
+
"Version" => "2012-10-17",
|
81
|
+
"Id" => "Fare_Policy",
|
82
|
+
"Statement" => [
|
83
|
+
{
|
84
|
+
"Sid" => "1",
|
85
|
+
"Effect" => "Allow",
|
86
|
+
"Principal" => { "AWS" => "*" },
|
87
|
+
"Action" => "SQS:SendMessage",
|
88
|
+
"Resource" => queue.arn,
|
89
|
+
"Condition" => {
|
90
|
+
"ArnLike" => {
|
91
|
+
"aws:SourceArn" => "arn:aws:sns:*:#{account_id}:*"
|
92
|
+
}
|
93
|
+
}
|
94
|
+
},
|
95
|
+
]
|
96
|
+
}
|
97
|
+
queue.policy = policy
|
98
|
+
end
|
99
|
+
|
100
|
+
def serialize_topic(topic_info)
|
101
|
+
{
|
102
|
+
"subject" => topic_info.fetch(:subject),
|
103
|
+
"action" => topic_info.fetch(:action),
|
104
|
+
"version" => topic_info.fetch(:version) { configuration.default_version },
|
105
|
+
"arn" => topic_for(topic_info).arn,
|
106
|
+
}
|
107
|
+
end
|
108
|
+
|
109
|
+
def backup_queue
|
110
|
+
if backup?
|
111
|
+
@backup_queue ||= queue_for("backup")
|
112
|
+
else
|
113
|
+
fail "NO BACKUP FOR YOU!"
|
114
|
+
end
|
115
|
+
end
|
116
|
+
|
117
|
+
def app_name
|
118
|
+
configuration.app_name
|
119
|
+
end
|
120
|
+
|
121
|
+
def topic_for(options)
|
122
|
+
topic_key = [options[:subject], options[:action] ]
|
123
|
+
topics[topic_key] ||= Fare.topic_adapter.create("#{environment}-#{options[:subject]}-#{options[:action]}").tap { |topic| puts "Created topic #{topic.arn}" }
|
124
|
+
end
|
125
|
+
|
126
|
+
def queue_for(name)
|
127
|
+
queues[name] ||= Fare.queue_adapter.fetch(environment, name)
|
128
|
+
end
|
129
|
+
|
130
|
+
end
|
131
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
module Fare
|
2
|
+
class LoadConfigurationFile
|
3
|
+
|
4
|
+
attr_reader :filename, :environment
|
5
|
+
|
6
|
+
def initialize(options)
|
7
|
+
@filename = options.fetch(:filename) { Fare.default_config_file }
|
8
|
+
@environment = options.fetch(:environment) { Fare.default_environment }
|
9
|
+
verify_file!
|
10
|
+
end
|
11
|
+
|
12
|
+
def verify_file!
|
13
|
+
unless exists?
|
14
|
+
raise ConfigurationNotFoundError, "Configuration file doesn't exist at #{filename}"
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def configuration
|
19
|
+
@configuration ||= load_configuration
|
20
|
+
end
|
21
|
+
|
22
|
+
def load_configuration
|
23
|
+
dsl = ConfigurationDSL.new
|
24
|
+
dsl.instance_eval(code, filename)
|
25
|
+
dsl.configuration
|
26
|
+
end
|
27
|
+
|
28
|
+
def code
|
29
|
+
@code ||= File.open(filename.to_s, "r:utf-8").read
|
30
|
+
end
|
31
|
+
|
32
|
+
def lock_filename
|
33
|
+
File.expand_path("../fare.#{environment}.lock", filename)
|
34
|
+
end
|
35
|
+
|
36
|
+
def checksum
|
37
|
+
Digest::MD5.hexdigest(code)
|
38
|
+
end
|
39
|
+
|
40
|
+
def exists?
|
41
|
+
File.exist?(filename)
|
42
|
+
end
|
43
|
+
|
44
|
+
end
|
45
|
+
end
|