izanami 0.14.0

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,133 @@
1
+ # encoding: UTF-8
2
+
3
+ module Izanami
4
+
5
+ # Open a communication channel between workers to handle the output
6
+ # of a command.
7
+ #
8
+ # A channel does two things: it publish and reads the data
9
+ # sent by a worker to a Redis pub/sub channel and stores that data when
10
+ # the channel is closed. That way, a process can read the output of
11
+ # a command when this is been executed or when is finished.
12
+ class Channel
13
+ EOC = '--EOC--' # End Of Channel
14
+ EOC.freeze
15
+
16
+ SEPARATOR = "\n"
17
+ SEPARATOR.freeze
18
+
19
+ # Handles the input written to the channel.
20
+ class Input
21
+
22
+ # @param [Izanami::Mapper::Command] mapper the instance which talks to redis.
23
+ # @param [String] id the command id.
24
+ def initialize(mapper, id)
25
+ @mapper = mapper
26
+ @id = id
27
+ @content = []
28
+ end
29
+
30
+ # Add new content to the channel.
31
+ #
32
+ # @param [String] payload the new content to add.
33
+ def <<(payload)
34
+ @mapper.publish(@id, payload)
35
+ @content << payload
36
+ end
37
+ alias_method :add, :<<
38
+
39
+ # Closes the channel.
40
+ #
41
+ # It stores all the content sent and sends an {EOC} signal
42
+ #
43
+ # @return [String] all the written content.
44
+ def close
45
+ string = to_s
46
+ @mapper.update(@id, 'output', string)
47
+ @mapper.publish(@id, EOC)
48
+
49
+ string
50
+ end
51
+
52
+ # Content that has been written.
53
+ #
54
+ # @note each line of the content is separated by {SEPARATOR}.
55
+ #
56
+ # @return [String]
57
+ def to_s
58
+ @content.join(SEPARATOR)
59
+ end
60
+ alias_method :read, :to_s
61
+ end
62
+
63
+ # @param [Izanami::Mapper::Command] mapper the instance which talks to redis.
64
+ def initialize(mapper)
65
+ @mapper = mapper
66
+ end
67
+
68
+ # Opens the channel to write.
69
+ #
70
+ # @note if a block is given, the input is closed when the block
71
+ # finish. If not, the client of the channel must close it.
72
+ #
73
+ # @param [String] id the command id.
74
+ #
75
+ # @yield [Izanami::Channel::Input] content handler.
76
+ #
77
+ # @return [Izanami::Channel::Input]
78
+ def write(id)
79
+ input = Input.new(@mapper, id)
80
+ if block_given?
81
+ yield input
82
+ input.close
83
+ end
84
+
85
+ input
86
+ end
87
+
88
+ # Opens the channel for reading.
89
+ #
90
+ # @param [String] id the command id.
91
+ #
92
+ # @yield [String] each of the lines read from the channel.
93
+ def read(id, &block)
94
+ command = @mapper.find(id) || {}
95
+ output = command['output']
96
+
97
+ if output
98
+ read_string(output, &block)
99
+ else
100
+ read_buffer(id, &block)
101
+ end
102
+ end
103
+
104
+ # Reads the content from a string.
105
+ #
106
+ # @param [String] output the stored command's output.
107
+ #
108
+ # @yield [String] each of the lines read from the stored output.
109
+ def read_string(output, &block)
110
+ output.split(SEPARATOR).each(&block)
111
+ end
112
+ protected :read_string
113
+
114
+ # Reads the content from Redis channel.
115
+ #
116
+ # @param [String] id the command id.
117
+ #
118
+ # @yield [String] each of the lines read from the Redis channel.
119
+ # It stops when a {EOC} flag is read.
120
+ def read_buffer(id)
121
+ @mapper.subscribe(id) do |on|
122
+ on.message do |_, line|
123
+ if line.match(/\A#{EOC}\z/)
124
+ @mapper.unsubscribe(id)
125
+ else
126
+ yield line
127
+ end
128
+ end
129
+ end
130
+ end
131
+ protected :read_buffer
132
+ end
133
+ end
@@ -0,0 +1,64 @@
1
+ require 'redis'
2
+ require 'redis/namespace'
3
+
4
+ module Izanami
5
+
6
+ # Proxy around a Redis client.
7
+ # It handles the configuration and the namespaces.
8
+ class Mapper
9
+ attr_reader :namespace, :options
10
+
11
+ # New Mapper client.
12
+ #
13
+ # @param [Hash] options
14
+ # @option options [String] :namespace the default namespace for all the keys.
15
+ # @option options [Hash] :redis Redis client options
16
+ def initialize(options = {})
17
+ @options = options.dup
18
+ @redis = @options.delete(:redis)
19
+ @namespace = build_namespace(@options.delete(:namespace))
20
+ end
21
+
22
+ # Generate the namespace based in a key.
23
+ #
24
+ # @param [String] key the default mapper key.
25
+ #
26
+ # @return [String] the namespace.
27
+ def build_namespace(key)
28
+ key || 'izanami'
29
+ end
30
+ protected :build_namespace
31
+
32
+ # Redis client, without the namespace.
33
+ #
34
+ # @return [Redis::Client]
35
+ def redis
36
+ @redis ||= initialize_redis
37
+ end
38
+
39
+ # Redis client, with the namespace.
40
+ #
41
+ # @return [Redis::Namespace]
42
+ def client
43
+ @client ||= initialize_client
44
+ end
45
+
46
+ def initialize_redis
47
+ Redis.new(@options)
48
+ end
49
+ protected :initialize_redis
50
+
51
+ def initialize_client
52
+ Redis::Namespace.new(@namespace, redis: redis)
53
+ end
54
+ protected :initialize_client
55
+
56
+ # Inspect the mapper
57
+ def to_s(*)
58
+ name = self.class.name
59
+ info = redis.inspect
60
+
61
+ "<#{name} connected to #{info} with @namespace=\"#{@namespace}\">"
62
+ end
63
+ end
64
+ end
@@ -0,0 +1,161 @@
1
+ require 'izanami/mapper'
2
+
3
+ module Izanami
4
+ module Mappers
5
+
6
+ # Mapper that handles the storage of `commands`.
7
+ class Command < Mapper
8
+
9
+ # Find a command.
10
+ #
11
+ # @param [String] id the command's id.
12
+ #
13
+ # @return [Hash, nil] the command or nil if is not found.
14
+ def find(id)
15
+ if exists?(id)
16
+ client.hgetall(id)
17
+ end
18
+ end
19
+
20
+ # Get the attribute of a command.
21
+ #
22
+ # @param [String] id the command's id.
23
+ # @param [String] attribute the command's attribute.
24
+ #
25
+ # @return [String, nil] the attribute or nil if is empty.
26
+ def get(id, attribute)
27
+ client.hget(id, attribute)
28
+ end
29
+
30
+ # The command exists?
31
+ #
32
+ # @param [String] id the command's id.
33
+ #
34
+ # @return [true, false]
35
+ def exists?(id)
36
+ client.exists(id)
37
+ end
38
+
39
+ # Count all commands.
40
+ #
41
+ # @return [Fixnum]
42
+ def count
43
+ client.scard('ids')
44
+ end
45
+
46
+ # Retrieve only the specified number of commands, sorted by id.
47
+ #
48
+ # @param [Fixnum, String] number the quantity of commands to retrieve.
49
+ #
50
+ # @return [Array] all the commands found.
51
+ def take(number)
52
+ options = { order: 'DESC', limit: [0, number] }
53
+ client.sort('ids', options).map { |id| find(id) }
54
+ end
55
+
56
+ # Delete the specified commands
57
+ #
58
+ # @param [Array] ids the commands' ids.
59
+ def delete(ids)
60
+ client.multi do
61
+ client.del(ids)
62
+ client.srem('ids', ids)
63
+ end
64
+ end
65
+
66
+ # Delete all commands
67
+ def delete_all
68
+ delete client.smembers('ids')
69
+ end
70
+
71
+ # Save the hash as a command.
72
+ #
73
+ # If the hash has an {'id'}, the command exists. If not,
74
+ # a new id is generated.
75
+ #
76
+ # @note all the commands has an expiration time of {#ttl} time.
77
+ #
78
+ # @return [Hash] the stored attributes.
79
+ def save(hash)
80
+ attributes = hash.dup
81
+ attributes['id'] ||= generate_id
82
+
83
+ client.multi do
84
+ client.hmset(attributes['id'], *attributes.to_a)
85
+ client.sadd('ids', attributes['id'])
86
+
87
+ # just keep the command enough time
88
+ client.expire(attributes['id'], ttl)
89
+ end
90
+
91
+ attributes
92
+ end
93
+
94
+ # Update one field.
95
+ #
96
+ # @param [String] id the command's id.
97
+ # @param [String] attribute the command's attribute name.
98
+ # @param [String] value the new value for the attribute.
99
+ #
100
+ # @return [Hash] the attributes stored.
101
+ def update(id, attribute, value)
102
+ save('id' => id, attribute => value)
103
+ end
104
+
105
+ # Generate a new unique id.
106
+ def generate_id
107
+ Time.now.utc.strftime("%Y%m%d%H%M%S")
108
+ end
109
+ protected :generate_id
110
+
111
+ # Default expiration time (defaults to 604800s == 7 days)
112
+ #
113
+ # @return [String, Fixnum]
114
+ def ttl
115
+ @options[:ttl] || 604800 # 7 days
116
+ end
117
+
118
+ # Publish to a Redis channel.
119
+ #
120
+ # @param [String] id the command's id.
121
+ # @param [String] payload what to publish.
122
+ def publish(id, payload)
123
+ client.publish(channel(id), payload)
124
+ end
125
+
126
+ # Subscribe to a Redis channel.
127
+ #
128
+ # @see {Redis::Client#subscribe}
129
+ #
130
+ # @param [String] id the command's id.
131
+ def subscribe(id, &block)
132
+ client.subscribe(channel(id), &block)
133
+ end
134
+
135
+ # Unsubscribe from a Redis channel.
136
+ #
137
+ # @see {Redis::Client#subscribe}
138
+ def unsubscribe(id)
139
+ client.unsubscribe(channel(id))
140
+ end
141
+
142
+ # Get the channel name based on an ID.
143
+ #
144
+ # @param [String] id the command's id.
145
+ #
146
+ # @return [String] the channel name.
147
+ def channel(id)
148
+ "#{id}:channel"
149
+ end
150
+ protected :channel
151
+
152
+ # Build the namespace for all the commands' keys.
153
+ #
154
+ # @return [String]
155
+ def build_namespace(key)
156
+ "#{super(key)}:commands"
157
+ end
158
+ protected :build_namespace
159
+ end
160
+ end
161
+ end
@@ -0,0 +1,3 @@
1
+ module Izanami
2
+ VERSION = "0.14.0"
3
+ end
@@ -0,0 +1,59 @@
1
+ # encoding: UTF-8
2
+
3
+ module Izanami
4
+
5
+ # Inject in a class or module all the {Izanami::Worker} modules.
6
+ #
7
+ # @param [Module, Class] target where to inject (include and extend) the modules.
8
+ def self.Worker(target)
9
+ target.send :extend, Worker::ClassMethods
10
+ target.send :include, Worker::InstanceMethods
11
+ end
12
+
13
+ # Basic interface to create {Izanami::Workers}
14
+ module Worker
15
+ module ClassMethods
16
+
17
+ # @see {#initialize} and {#defer}
18
+ def defer(*args)
19
+ new(*args).defer
20
+ end
21
+
22
+ # @see {#initialize} and {#call}
23
+ def call(*args)
24
+ new(*args).call
25
+ end
26
+
27
+ # @see {.call}
28
+ def run(*args)
29
+ new(*args).run
30
+ end
31
+ end
32
+
33
+ module InstanceMethods
34
+
35
+ # Executes a {#call} method in a different process, forking the
36
+ # current one.
37
+ #
38
+ # @note the process forked is detached.
39
+ #
40
+ # @return [String, Fixnum] the new process pid.
41
+ def defer
42
+ pid = Process.fork { call }
43
+ Process.detach(pid)
44
+
45
+ pid
46
+ end
47
+
48
+ # Main worker action. Must be defined in the concrete class.
49
+ def call
50
+ raise 'The method #call should be defined in the worker class'
51
+ end
52
+
53
+ # @see {#call}
54
+ def run
55
+ call
56
+ end
57
+ end
58
+ end
59
+ end
@@ -0,0 +1,82 @@
1
+ require 'open3'
2
+ require 'izanami/channel'
3
+ require 'izanami/worker'
4
+ require 'izanami/mappers/command'
5
+
6
+ module Izanami
7
+ module Workers
8
+ # Worker that executes the {cap} commands and stores the results.
9
+ #
10
+ # @param [Hash] redis_options Options for the redis mapper.
11
+ # @param [Hash] command the command to execute.
12
+ class Command < Struct.new(:redis_options, :command)
13
+ Izanami::Worker self
14
+
15
+ attr_writer :shell
16
+
17
+ # Executes the cap command with the correct environment and handles the
18
+ # IO streams.
19
+ def call
20
+ Open3.popen2e(shell_env, cmd, shell_options) do |stdin, stdout, thread|
21
+ handle_streams(stdin, stdout, thread)
22
+ end
23
+ end
24
+
25
+ # ENV vars for the {cap} command.
26
+ #
27
+ # This worker is executed inside a ruby process initiated by {bundler}.
28
+ # Because the {cap} process is executed in a sandbox with its own {bundler}
29
+ # ecosystem, we need to remove all the {bundler} environment variables to have
30
+ # a clean execution.
31
+ #
32
+ # @return [Hash]
33
+ def shell_env
34
+ bundler_keys = ENV.select { |var, _| var.to_s.match(/\ABUNDLE/) }.keys
35
+ bundler_keys.reduce({}) do |hash, (k,_)|
36
+ hash[k] = nil
37
+ hash
38
+ end
39
+ end
40
+
41
+ # Options for the new process.
42
+ #
43
+ # Here we define the directory for the {cap} process, which is the
44
+ # ENV['IZANAMI_SANDBOX'] configuration variable.
45
+ #
46
+ # @return [Hash]
47
+ def shell_options
48
+ { chdir: ENV['IZANAMI_SANDBOX'] }
49
+ end
50
+
51
+ # The {cap} command
52
+ #
53
+ # @return [String]
54
+ def cmd
55
+ "bundle exec cap #{command['tasks']}"
56
+ end
57
+
58
+ # Handle the process IO streams.
59
+ #
60
+ # It writes the process output to a {Izanami::Channel} and
61
+ # saves the status of the command.
62
+ #
63
+ # @see {Open3.popen2e}
64
+ def handle_streams(stdin, stdout, thread)
65
+ mapper = Izanami::Mappers::Command.new(redis_options)
66
+ channel = Izanami::Channel.new(mapper)
67
+
68
+ channel.write(command['id']) do |input|
69
+ while (line = stdout.gets)
70
+ input << line.chomp
71
+ end
72
+ end
73
+
74
+ status = thread.value.success? ? 'success' : 'fail'
75
+ mapper.update(command['id'], 'status', status)
76
+
77
+ stdin.close
78
+ stdout.close
79
+ end
80
+ end
81
+ end
82
+ end
@@ -0,0 +1,65 @@
1
+ require 'open3'
2
+ require 'izanami/worker'
3
+
4
+ module Izanami
5
+ module Workers
6
+
7
+ # Utility worker that watch the IZANAMI_SANDBOX repository
8
+ # and updates the content each IZANAMI_WATCHDOG_SLEEP_TIME time
9
+ # via {git}.
10
+ class Watchdog
11
+ Izanami::Worker self
12
+
13
+ # Main action. Updates the repo in a loop,
14
+ # waiting the defined time.
15
+ def call
16
+ loop do
17
+ update_repo
18
+ sleep wait_time
19
+ end
20
+ end
21
+
22
+ # Update the repo. Prints to $stdout the results of the {git} command.
23
+ def update_repo
24
+ Open3.popen2e(cmd, shell_options) do |stdin, stdout, thread|
25
+ while (line = stdout.gets)
26
+ $stdout.puts line.chomp
27
+ end
28
+
29
+ status = thread.value.success? ? 'success' : 'fail'
30
+ $stdout.puts "\"#{cmd}\" => #{status}"
31
+
32
+ stdin.close
33
+ stdout.close
34
+ end
35
+ end
36
+
37
+ # Options for the new process.
38
+ #
39
+ # Here we define the directory for the {cap} process, which is the
40
+ # ENV['IZANAMI_SANDBOX'] configuration variable.
41
+ #
42
+ # @return [Hash]
43
+ def shell_options
44
+ { chdir: ENV['IZANAMI_SANDBOX'] }
45
+ end
46
+
47
+ # {git} command for the updating.
48
+ #
49
+ # @return [String]
50
+ def cmd
51
+ 'git pull'
52
+ end
53
+
54
+ # Time to wait between each update
55
+ #
56
+ # Can be configured with the ENV['IZANAMI_WATCHDOG_SLEEP_TIME'] variable.
57
+ #
58
+ # @return [Fixnum] the time (default to 300 seconds).
59
+ def wait_time
60
+ time = ENV['IZANAMI_WATCHDOG_SLEEP_TIME'] || 300
61
+ time.to_i
62
+ end
63
+ end
64
+ end
65
+ end
data/lib/izanami.rb ADDED
@@ -0,0 +1,5 @@
1
+ require 'izanami/version'
2
+
3
+ module Izanami
4
+ # Your code goes here...
5
+ end
data/spec/helper.rb ADDED
@@ -0,0 +1,12 @@
1
+ ENV['RACK_ENV'] ||= 'test'
2
+ ENV['IZANAMI_USER'] = 'spec'
3
+ ENV['IZANAMI_PASSWORD'] = 'spec'
4
+ ENV['IZANAMI_SANDBOX'] = File.expand_path('..', __FILE__) + '/sandbox'
5
+ ENV['IZANAMI_REDIS_URL'] ||= 'redis://127.0.0.1:6379/0'
6
+
7
+ require 'bundler'
8
+ Bundler.setup(:default, :test)
9
+
10
+ require 'rspec'
11
+ require 'rspec/autorun'
12
+ require 'rack/test'