izanami 0.14.0

data/lib/izanami/channel.rb

@@ -0,0 +1,133 @@
+# encoding: UTF-8
+
+module Izanami
+
+  # Open a communication channel between workers to handle the output
+  # of a command.
+  #
+  # A channel does two things: it publish and reads the data
+  # sent by a worker to a Redis pub/sub channel and stores that data when
+  # the channel is closed. That way, a process can read the output of
+  # a command when this is been executed or when is finished.
+  class Channel
+    EOC = '--EOC--' # End Of Channel
+    EOC.freeze
+
+    SEPARATOR = "\n"
+    SEPARATOR.freeze
+
+    # Handles the input written to the channel.
+    class Input
+
+      # @param [Izanami::Mapper::Command] mapper the instance which talks to redis.
+      # @param [String] id the command id.
+      def initialize(mapper, id)
+        @mapper  = mapper
+        @id      = id
+        @content = []
+      end
+
+      # Add new content to the channel.
+      #
+      # @param [String] payload the new content to add.
+      def <<(payload)
+        @mapper.publish(@id, payload)
+        @content << payload
+      end
+      alias_method :add, :<<
+
+      # Closes the channel.
+      #
+      # It stores all the content sent and sends an {EOC} signal
+      #
+      # @return [String] all the written content.
+      def close
+        string = to_s
+        @mapper.update(@id, 'output', string)
+        @mapper.publish(@id, EOC)
+
+        string
+      end
+
+      # Content that has been written.
+      #
+      # @note each line of the content is separated by {SEPARATOR}.
+      #
+      # @return [String]
+      def to_s
+        @content.join(SEPARATOR)
+      end
+      alias_method :read, :to_s
+    end
+
+    # @param [Izanami::Mapper::Command] mapper the instance which talks to redis.
+    def initialize(mapper)
+      @mapper = mapper
+    end
+
+    # Opens the channel to write.
+    #
+    # @note if a block is given, the input is closed when the block
+    # finish. If not, the client of the channel must close it.
+    #
+    # @param [String] id the command id.
+    #
+    # @yield [Izanami::Channel::Input] content handler.
+    #
+    # @return [Izanami::Channel::Input]
+    def write(id)
+      input = Input.new(@mapper, id)
+      if block_given?
+        yield input
+        input.close
+      end
+
+      input
+    end
+
+    # Opens the channel for reading.
+    #
+    # @param [String] id the command id.
+    #
+    # @yield [String] each of the lines read from the channel.
+    def read(id, &block)
+      command = @mapper.find(id) || {}
+      output  = command['output']
+
+      if output
+        read_string(output, &block)
+      else
+        read_buffer(id, &block)
+      end
+    end
+
+    # Reads the content from a string.
+    #
+    # @param [String] output the stored command's output.
+    #
+    # @yield [String] each of the lines read from the stored output.
+    def read_string(output, &block)
+      output.split(SEPARATOR).each(&block)
+    end
+    protected :read_string
+
+    # Reads the content from Redis channel.
+    #
+    # @param [String] id the command id.
+    #
+    # @yield [String] each of the lines read from the Redis channel.
+    # It stops when a {EOC} flag is read.
+    def read_buffer(id)
+      @mapper.subscribe(id) do |on|
+        on.message do |_, line|
+          if line.match(/\A#{EOC}\z/)
+            @mapper.unsubscribe(id)
+          else
+            yield line
+          end
+        end
+      end
+    end
+    protected :read_buffer
+  end
+end

data/lib/izanami/mapper.rb

@@ -0,0 +1,64 @@
+require 'redis'
+require 'redis/namespace'
+
+module Izanami
+
+  # Proxy around a Redis client.
+  # It handles the configuration and the namespaces.
+  class Mapper
+    attr_reader :namespace, :options
+
+    # New Mapper client.
+    #
+    # @param [Hash] options
+    # @option options [String] :namespace the default namespace for all the keys.
+    # @option options [Hash] :redis Redis client options
+    def initialize(options = {})
+      @options   = options.dup
+      @redis     = @options.delete(:redis)
+      @namespace = build_namespace(@options.delete(:namespace))
+    end
+
+    # Generate the namespace based in a key.
+    #
+    # @param [String] key the default mapper key.
+    #
+    # @return [String] the namespace.
+    def build_namespace(key)
+      key || 'izanami'
+    end
+    protected :build_namespace
+
+    # Redis client, without the namespace.
+    #
+    # @return [Redis::Client]
+    def redis
+      @redis ||= initialize_redis
+    end
+
+    # Redis client, with the namespace.
+    #
+    # @return [Redis::Namespace]
+    def client
+      @client ||= initialize_client
+    end
+
+    def initialize_redis
+      Redis.new(@options)
+    end
+    protected :initialize_redis
+
+    def initialize_client
+      Redis::Namespace.new(@namespace, redis: redis)
+    end
+    protected :initialize_client
+
+    # Inspect the mapper
+    def to_s(*)
+      name = self.class.name
+      info = redis.inspect
+
+      "<#{name} connected to #{info} with @namespace=\"#{@namespace}\">"
+    end
+  end
+end

data/lib/izanami/mappers/command.rb

@@ -0,0 +1,161 @@
+require 'izanami/mapper'
+
+module Izanami
+  module Mappers
+
+    # Mapper that handles the storage of `commands`.
+    class Command < Mapper
+
+      # Find a command.
+      #
+      # @param [String] id the command's id.
+      #
+      # @return [Hash, nil] the command or nil if is not found.
+      def find(id)
+        if exists?(id)
+          client.hgetall(id)
+        end
+      end
+
+      # Get the attribute of a command.
+      #
+      # @param [String] id the command's id.
+      # @param [String] attribute the command's attribute.
+      #
+      # @return [String, nil] the attribute or nil if is empty.
+      def get(id, attribute)
+        client.hget(id, attribute)
+      end
+
+      # The command exists?
+      #
+      # @param [String] id the command's id.
+      #
+      # @return [true, false]
+      def exists?(id)
+        client.exists(id)
+      end
+
+      # Count all commands.
+      #
+      # @return [Fixnum]
+      def count
+        client.scard('ids')
+      end
+
+      # Retrieve only the specified number of commands, sorted by id.
+      #
+      # @param [Fixnum, String] number the quantity of commands to retrieve.
+      #
+      # @return [Array] all the commands found.
+      def take(number)
+        options = { order: 'DESC', limit: [0, number] }
+        client.sort('ids', options).map { |id| find(id) }
+      end
+
+      # Delete the specified commands
+      #
+      # @param [Array] ids the commands' ids.
+      def delete(ids)
+        client.multi do
+          client.del(ids)
+          client.srem('ids', ids)
+        end
+      end
+
+      # Delete all commands
+      def delete_all
+        delete client.smembers('ids')
+      end
+
+      # Save the hash as a command.
+      #
+      # If the hash has an {'id'}, the command exists. If not,
+      # a new id is generated.
+      #
+      # @note all the commands has an expiration time of {#ttl} time.
+      #
+      # @return [Hash] the stored attributes.
+      def save(hash)
+        attributes = hash.dup
+        attributes['id'] ||= generate_id
+
+        client.multi do
+          client.hmset(attributes['id'], *attributes.to_a)
+          client.sadd('ids', attributes['id'])
+
+          # just keep the command enough time
+          client.expire(attributes['id'], ttl)
+        end
+
+        attributes
+      end
+
+      # Update one field.
+      #
+      # @param [String] id the command's id.
+      # @param [String] attribute the command's attribute name.
+      # @param [String] value the new value for the attribute.
+      #
+      # @return [Hash] the attributes stored.
+      def update(id, attribute, value)
+        save('id' => id, attribute => value)
+      end
+
+      # Generate a new unique id.
+      def generate_id
+        Time.now.utc.strftime("%Y%m%d%H%M%S")
+      end
+      protected :generate_id
+
+      # Default expiration time (defaults to 604800s == 7 days)
+      #
+      # @return [String, Fixnum]
+      def ttl
+        @options[:ttl] || 604800 # 7 days
+      end
+
+      # Publish to a Redis channel.
+      #
+      # @param [String] id the command's id.
+      # @param [String] payload what to publish.
+      def publish(id, payload)
+        client.publish(channel(id), payload)
+      end
+
+      # Subscribe to a Redis channel.
+      #
+      # @see {Redis::Client#subscribe}
+      #
+      # @param [String] id the command's id.
+      def subscribe(id, &block)
+        client.subscribe(channel(id), &block)
+      end
+
+      # Unsubscribe from a Redis channel.
+      #
+      # @see {Redis::Client#subscribe}
+      def unsubscribe(id)
+        client.unsubscribe(channel(id))
+      end
+
+      # Get the channel name based on an ID.
+      #
+      # @param [String] id the command's id.
+      #
+      # @return [String] the channel name.
+      def channel(id)
+        "#{id}:channel"
+      end
+      protected :channel
+
+      # Build the namespace for all the commands' keys.
+      #
+      # @return [String]
+      def build_namespace(key)
+        "#{super(key)}:commands"
+      end
+      protected :build_namespace
+    end
+  end
+end

data/lib/izanami/version.rb

@@ -0,0 +1,3 @@
+module Izanami
+  VERSION = "0.14.0"
+end

data/lib/izanami/worker.rb

@@ -0,0 +1,59 @@
+# encoding: UTF-8
+
+module Izanami
+
+  # Inject in a class or module all the {Izanami::Worker} modules.
+  #
+  # @param [Module, Class] target where to inject (include and extend) the modules.
+  def self.Worker(target)
+    target.send :extend,  Worker::ClassMethods
+    target.send :include, Worker::InstanceMethods
+  end
+
+  # Basic interface to create {Izanami::Workers}
+  module Worker
+    module ClassMethods
+
+      # @see {#initialize} and {#defer}
+      def defer(*args)
+        new(*args).defer
+      end
+
+      # @see {#initialize} and {#call}
+      def call(*args)
+        new(*args).call
+      end
+
+      # @see {.call}
+      def run(*args)
+        new(*args).run
+      end
+    end
+
+    module InstanceMethods
+
+      # Executes a {#call} method in a different process, forking the
+      # current one.
+      #
+      # @note the process forked is detached.
+      #
+      # @return [String, Fixnum] the new process pid.
+      def defer
+        pid = Process.fork { call }
+        Process.detach(pid)
+
+        pid
+      end
+
+      # Main worker action. Must be defined in the concrete class.
+      def call
+        raise 'The method #call should be defined in the worker class'
+      end
+
+      # @see {#call}
+      def run
+        call
+      end
+    end
+  end
+end

data/lib/izanami/workers/command.rb

@@ -0,0 +1,82 @@
+require 'open3'
+require 'izanami/channel'
+require 'izanami/worker'
+require 'izanami/mappers/command'
+
+module Izanami
+  module Workers
+    # Worker that executes the {cap} commands and stores the results.
+    #
+    # @param [Hash] redis_options Options for the redis mapper.
+    # @param [Hash] command the command to execute.
+    class Command < Struct.new(:redis_options, :command)
+      Izanami::Worker self
+
+      attr_writer :shell
+
+      # Executes the cap command with the correct environment and handles the
+      # IO streams.
+      def call
+        Open3.popen2e(shell_env, cmd, shell_options) do |stdin, stdout, thread|
+          handle_streams(stdin, stdout, thread)
+        end
+      end
+
+      # ENV vars for the {cap} command.
+      #
+      # This worker is executed inside a ruby process initiated by {bundler}.
+      # Because the {cap} process is executed in a sandbox with its own {bundler}
+      # ecosystem, we need to remove all the {bundler} environment variables to have
+      # a clean execution.
+      #
+      # @return [Hash]
+      def shell_env
+        bundler_keys = ENV.select { |var, _| var.to_s.match(/\ABUNDLE/) }.keys
+        bundler_keys.reduce({}) do |hash, (k,_)|
+          hash[k] = nil
+          hash
+        end
+      end
+
+      # Options for the new process.
+      #
+      # Here we define the directory for the {cap} process, which is the
+      # ENV['IZANAMI_SANDBOX'] configuration variable.
+      #
+      # @return [Hash]
+      def shell_options
+        { chdir: ENV['IZANAMI_SANDBOX'] }
+      end
+
+      # The {cap} command
+      #
+      # @return [String]
+      def cmd
+        "bundle exec cap #{command['tasks']}"
+      end
+
+      # Handle the process IO streams.
+      #
+      # It writes the process output to a {Izanami::Channel} and
+      # saves the status of the command.
+      #
+      # @see {Open3.popen2e}
+      def handle_streams(stdin, stdout, thread)
+        mapper  = Izanami::Mappers::Command.new(redis_options)
+        channel = Izanami::Channel.new(mapper)
+
+        channel.write(command['id']) do |input|
+          while (line = stdout.gets)
+            input << line.chomp
+          end
+        end
+
+        status = thread.value.success? ? 'success' : 'fail'
+        mapper.update(command['id'], 'status', status)
+
+        stdin.close
+        stdout.close
+      end
+    end
+  end
+end

data/lib/izanami/workers/watchdog.rb

@@ -0,0 +1,65 @@
+require 'open3'
+require 'izanami/worker'
+
+module Izanami
+  module Workers
+
+    # Utility worker that watch the IZANAMI_SANDBOX repository
+    # and updates the content each IZANAMI_WATCHDOG_SLEEP_TIME time
+    # via {git}.
+    class Watchdog
+      Izanami::Worker self
+
+      # Main action. Updates the repo in a loop,
+      # waiting the defined time.
+      def call
+        loop do
+          update_repo
+          sleep wait_time
+        end
+      end
+
+      # Update the repo. Prints to $stdout the results of the {git} command.
+      def update_repo
+        Open3.popen2e(cmd, shell_options) do |stdin, stdout, thread|
+          while (line = stdout.gets)
+            $stdout.puts line.chomp
+          end
+
+          status = thread.value.success? ? 'success' : 'fail'
+          $stdout.puts "\"#{cmd}\" => #{status}"
+
+          stdin.close
+          stdout.close
+        end
+      end
+
+      # Options for the new process.
+      #
+      # Here we define the directory for the {cap} process, which is the
+      # ENV['IZANAMI_SANDBOX'] configuration variable.
+      #
+      # @return [Hash]
+      def shell_options
+        { chdir: ENV['IZANAMI_SANDBOX'] }
+      end
+
+      # {git} command for the updating.
+      #
+      # @return [String]
+      def cmd
+        'git pull'
+      end
+
+      # Time to wait between each update
+      #
+      # Can be configured with the ENV['IZANAMI_WATCHDOG_SLEEP_TIME'] variable.
+      #
+      # @return [Fixnum] the time (default to 300 seconds).
+      def wait_time
+        time = ENV['IZANAMI_WATCHDOG_SLEEP_TIME'] || 300
+        time.to_i
+      end
+    end
+  end
+end

data/lib/izanami.rb

@@ -0,0 +1,5 @@
+require 'izanami/version'
+
+module Izanami
+  # Your code goes here...
+end

data/spec/helper.rb

@@ -0,0 +1,12 @@
+ENV['RACK_ENV'] ||= 'test'
+ENV['IZANAMI_USER'] = 'spec'
+ENV['IZANAMI_PASSWORD'] = 'spec'
+ENV['IZANAMI_SANDBOX'] = File.expand_path('..', __FILE__) + '/sandbox'
+ENV['IZANAMI_REDIS_URL'] ||= 'redis://127.0.0.1:6379/0'
+
+require 'bundler'
+Bundler.setup(:default, :test)
+
+require 'rspec'
+require 'rspec/autorun'
+require 'rack/test'