rworkflow 0.6.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 0a21aaa1eb7353e45eb10cf8498ed9c8ae9cebfc
+  data.tar.gz: 05a018eb4b27041ed86515cb7db4e2c945708368
+SHA512:
+  metadata.gz: f304c504e1f537f0430f94739997c858b23d2308844796d00b2bc40fb74be22cdbd0f018d6b004e60e05538e4c3283ba969ac5572ef967257e91c3cd4d5f611d
+  data.tar.gz: d89cddce1e44a96ef6252ff43a528780b7f4c023ee9e83d93bf08d679448704341a61614dec068dc73a1d028279f9c94c44641078bdb40449611426c5b96164b

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright 2015 YOURNAME
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,117 @@
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+
+require 'rdoc/task'
+
+RDoc::Task.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'Rworkflow'
+  rdoc.options << '--line-numbers'
+  rdoc.rdoc_files.include('README.rdoc')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib'
+  t.libs << 'test'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = false
+end
+
+task default: :test
+
+namespace :cim do
+  desc 'Tags, updates README, and CHANGELOG and pushes to Github. Requires ruby-git'
+  task :release do
+    tasks = ['cim:assert_clean_repo', 'cim:git_fetch', 'cim:assert_version', 'cim:update_readme', 'cim:update_changelog', 'cim:commit_changes', 'cim:tag']
+    begin
+      tasks.each { |task| Rake::Task[task].invoke }
+      `git push && git push origin '#{Rworkflow::VERSION}'`
+    rescue => error
+      puts ">>> ERROR: #{error}; might want to reset your repository"
+    end
+  end
+
+  desc 'Fails if the current repository is not clean'
+  task :assert_clean_repo do
+    status = `git status -s`.chomp.strip
+    if status.empty?
+      status = `git log origin/master..HEAD`.chomp.strip # check if we have unpushed commits
+      if status.empty?
+        puts ">>> Repository is clean!"
+      else
+        puts ">>> Please push your committed changes before releasing!"
+        exit -1
+      end
+    else
+      puts ">>> Please stash or commit your changes before releasing!"
+      exit -1
+    end
+  end
+
+  desc 'Fetches latest tags/commits'
+  task :git_fetch do
+    puts '>>> Fetching latest git refs'
+    `git fetch --tags`
+  end
+
+  desc 'Fails if the current gem version is not greater than the last tag'
+  task :assert_version do
+    latest = `git describe --abbrev=0`.chomp.strip
+    current = Rworkflow::VERSION
+
+    unless Gem::Version.new(current) > Gem::Version.new(latest)
+      puts ">>> Latest tagged version is #{latest}; make sure gem version (#{current}) is greater!"
+      exit -1
+    end
+  end
+
+  desc 'Updates README with latest version'
+  task :update_readme do
+    puts '>>> Updating README.md'
+    replace = %([![GitHub release](https://img.shields.io/badge/release-#{Rworkflow::VERSION}-blue.png)](https://github.com/barcoo/rworkflow/releases/tag/#{Rworkflow::VERSION}))
+
+    `sed -i -u 's@^\\[\\!\\[GitHub release\\].*$@#{replace}@' README.md`
+  end
+
+  desc 'Updates CHANGELOG with commit log from last tag to this one'
+  task :update_changelog do
+    puts '>>> Updating CHANGELOG.md'
+    latest = `git describe --abbrev=0`.chomp.strip
+    log = `git log --pretty=format:'- [%h](https://github.com/barcoo/rworkflow/commit/%h) *%ad* __%s__ (%an)' --date=short '#{latest}'..HEAD`.chomp
+
+    changelog = File.open('.CHANGELOG.md', 'w')
+    changelog.write("# Changelog\n\n###{Rworkflow::VERSION}\n\n#{log}\n\n")
+    File.open('CHANGELOG.md', 'r') do |file|
+      file.readline # skip first two lines
+      file.readline
+      while buffer = file.read(2048)
+        changelog.write(buffer)
+      end
+    end
+
+    changelog.close
+    `mv '.CHANGELOG.md' 'CHANGELOG.md'`
+  end
+
+  desc 'Commits the README/CHANGELOG changes'
+  task :commit_changes do
+    puts '>>> Committing updates to README/CHANGELOG'
+    `git commit -am'Updated README.md and CHANGELOG.md on new release'`
+  end
+
+  desc 'Creates and pushes the tag to git'
+  task :tag do
+    puts '>>> Tagging'
+    message = STDOUT.print ">>> Please enter a tag message: "
+    input = STDIN.gets.strip.tr("'", "\'")
+    `git tag -a '#{Rworkflow::VERSION}' -m '#{input}'`
+  end
+end

data/lib/rworkflow/flow.rb

@@ -0,0 +1,450 @@
+module Rworkflow
+  class Flow
+    STATE_SUCCESSFUL = :successful
+    STATE_FAILED = :failed
+    STATES_TERMINAL = [STATE_FAILED, STATE_SUCCESSFUL].freeze
+    STATES_FAILED = [STATE_FAILED].freeze
+
+    REDIS_NS = 'flow'.freeze
+    WORKFLOW_REGISTRY = "#{REDIS_NS}:__registry".freeze
+
+    attr_accessor :id
+    attr_reader :lifecycle
+
+    def initialize(id)
+      @id = id
+      @redis_key = "#{REDIS_NS}:#{id}"
+
+      @storage = RedisRds::Hash.new(@redis_key)
+      @flow_data = RedisRds::Hash.new("#{@redis_key}__data")
+      @processing = RedisRds::Hash.new("#{@redis_key}__processing")
+
+      load_lifecycle
+    end
+
+    def load_lifecycle
+      serialized = @storage.get(:lifecycle)
+      unless serialized.nil?
+        raw = self.class.serializer.load(serialized)
+        @lifecycle = Rworkflow::Lifecycle.unserialize(raw) unless raw.nil?
+      end
+    rescue
+      @lifecycle = nil
+    end
+    private :load_lifecycle
+
+    def lifecycle=(new_lifecycle)
+      @lifecycle = new_lifecycle
+      @storage.set(:lifecycle, self.class.serializer.dump(@lifecycle.serialize))
+    end
+
+    def finished?
+      return false unless started?
+      total = get_counters.reduce(0) do |sum, pair|
+        self.class.terminal?(pair[0]) ? sum : (sum + pair[1].to_i)
+      end
+
+      return total == 0
+    end
+
+    def status
+      status = 'Running'
+      status = (successful?) ? 'Finished' : 'Failed' if finished?
+
+      return status
+    end
+
+    def created_at
+      return @created_at ||= begin Time.at(get(:created_at, 0)) end
+    end
+
+    def started?
+      return !get(:start_time).nil?
+    end
+
+    def name
+      return get(:name, @id)
+    end
+
+    def name=(name)
+      return set(:name, name)
+    end
+
+    def start_time
+      return Time.at(get(:start_time, 0))
+    end
+
+    def finish_time
+      return Time.at(get(:finish_time, 0))
+    end
+
+    def expected_duration
+      return Float::INFINITY
+    end
+
+    def valid?
+      return !@lifecycle.nil?
+    end
+
+    def count(state)
+      return get_list(state).size
+    end
+
+    def get_counters
+      counters = @storage.get(:counters)
+      if !counters.nil?
+        counters = begin
+          self.class.serializer.load(counters)
+        rescue => e
+          Rails.logger.error("Error loading stored flow counters: #{e.message}")
+          nil
+        end
+      end
+      return counters || get_counters!
+    end
+
+    # fetches counters atomically
+    def get_counters!
+      counters = { processing: 0 }
+
+      names = @lifecycle.states.keys
+      results = RedisRds::Object.connection.multi do
+        self.class::STATES_TERMINAL.each { |name| get_list(name).size }
+        names.each { |name| get_list(name).size }
+        @processing.getall
+      end
+
+      (self.class::STATES_TERMINAL + names).each do |name|
+        counters[name] = results.shift.to_i
+      end
+
+      counters[:processing] = results.shift.reduce(0) { |sum, pair| sum + pair.last.to_i }
+
+      return counters
+    end
+    private :get_counters!
+
+    def fetch(fetcher_id, state_name)
+      @processing.set(fetcher_id, 1)
+      list = get_state_list(state_name)
+      unless list.nil?
+        failed = []
+        cardinality = @lifecycle.states[state_name].cardinality
+        cardinality = get(:start_count).to_i if cardinality == Lifecycle::CARDINALITY_ALL_STARTED
+        force_list_complete = @lifecycle.states[state_name].policy == State::STATE_POLICY_WAIT
+        raw_objects = list.lpop(cardinality, force_list_complete)
+        unless raw_objects.empty?
+          objects = raw_objects.map do |raw_object|
+            begin
+              self.class.serializer.load(raw_object)
+            rescue StandardError => _
+              failed << raw_object
+              nil
+            end
+          end.compact
+          @processing.set(fetcher_id, objects.size)
+
+          unless failed.empty?
+            push(failed, STATE_FAILED)
+            Rails.logger.error("Failed to parse #{failed.size} in workflow #{@id} for fetcher id #{fetcher_id} at state #{state_name}")
+          end
+
+          yield(objects) if block_given?
+        end
+      end
+    ensure
+      @processing.remove(fetcher_id)
+      terminate if finished?
+    end
+
+    def list_objects(state_name, limit = -1)
+      list = get_list(state_name)
+      return list.get(0, limit).map {|object| self.class.serializer.load(object)}
+    end
+
+    def get_state_list(state_name)
+      list = nil
+      state = @lifecycle.states[state_name]
+
+      if !state.nil?
+        list = get_list(state_name)
+      else
+        Rails.logger.error("Tried accessing invalid state #{state_name} for workflow #{id}")
+      end
+      return list
+    end
+    private :get_state_list
+
+    def terminate
+      mutex = RedisRds::Mutex.new(self.id)
+      mutex.synchronize do
+        if !self.cleaned_up?
+          set(:finish_time, Time.now.to_i)
+          post_process
+
+          if self.public?
+            counters = get_counters!
+            counters[:processing] = 0 # Some worker might have increased the processing flag at that time even if there is no more jobs to be done
+            @storage.setnx(:counters, self.class.serializer.dump(counters))
+            states_cleanup
+          else
+            self.cleanup
+          end
+        end
+      end
+    end
+
+    def post_process
+    end
+    protected :post_process
+
+    def metadata_string
+      return "Rworkflow: #{self.name}"
+    end
+
+    def cleaned_up?
+      return states_list.all? { |name| !get_list(name).exists? }
+    end
+
+    def states_list
+      states = self.class::STATES_TERMINAL
+      states += @lifecycle.states.keys if valid?
+
+      return states
+    end
+
+    def transition(from_state, name, objects)
+      objects = Array.wrap(objects)
+      to_state = begin
+        lifecycle.transition(from_state, name)
+      rescue Rworkflow::StateError => e
+        Rails.logger.error("Error transitioning: #{e}")
+        nil
+      end
+
+      if !to_state.nil?
+        push(objects, to_state)
+        log(from_state, name, objects.size)
+      end
+    end
+
+    def logging?
+      return get(:logging, false)
+    end
+
+    def log(from_state, transition, num_objects)
+      logger.incrby("#{from_state}__#{transition}", num_objects.to_i) if logging?
+    end
+
+    def logger
+      return @logger ||= begin
+        RedisRds::Hash.new("#{@redis_key}__logger")
+      end
+    end
+
+    def logs
+      logs = {}
+      if valid? && logging?
+        state_transition_counters = logger.getall
+        state_transition_counters.each do |state_transition, counter|
+          state, transition = state_transition.split('__')
+          logs[state] = {} unless logs.key?(state)
+          logs[state][transition] = counter.to_i
+        end
+      end
+
+      return logs
+    end
+
+    def get_state_cardinality(state_name)
+      cardinality = @lifecycle.states[state_name].cardinality
+      cardinality = self.get(:start_count).to_i if cardinality == Rworkflow::Lifecycle::CARDINALITY_ALL_STARTED
+      return cardinality
+    end
+
+    def set(key, value)
+      @flow_data.set(key, self.class.serializer.dump(value))
+    end
+
+    def get(key, default = nil)
+      value = @flow_data.get(key)
+      value = if value.nil? then default else self.class.serializer.load(value) end
+
+      return value
+    end
+
+    def incr(key, value = 1)
+      return @flow_data.incrby(key, value)
+    end
+
+    def push(objects, state)
+      objects = Array.wrap(objects)
+
+      return 0 if objects.empty?
+
+      list = get_list(state)
+      list.rpush(objects.map { |object| self.class.serializer.dump(object) })
+
+      return objects.size
+    end
+    private :push
+
+    def get_list(name)
+      return RedisRds::List.new("#{@redis_key}:lists:#{name}")
+    end
+    private :get_list
+
+    def cleanup
+      return if Rails.env.test?
+      states_cleanup
+      logger.delete
+      @processing.delete
+      self.class.unregister(self)
+      @flow_data.delete
+      @storage.delete
+    end
+
+    def states_cleanup
+      return if Rails.env.test?
+      states_list.each { |name| get_list(name).delete }
+    end
+    protected :states_cleanup
+
+    def start(objects)
+      objects = Array.wrap(objects)
+      set(:start_time, Time.now.to_i)
+      set(:start_count, objects.size)
+      push(objects, lifecycle.initial)
+      log(lifecycle.initial, 'initial', objects.size)
+    end
+
+    def total_objects_processed(counters = nil)
+      return (counters || get_counters).reduce(0) do |sum, pair|
+        if self.class.terminal?(pair[0])
+          sum + pair[1]
+        else
+          sum
+        end
+      end
+    end
+
+    def total_objects(counters = nil)
+      return (counters || get_counters).reduce(0) { |sum, pair| sum + pair[1] }
+    end
+
+    def total_objects_failed(counters = nil)
+      return (counters || get_counters).reduce(0) do |sum, pair|
+        if self.class.failure?(pair[0])
+          sum + pair[1]
+        else
+          sum
+        end
+      end
+    end
+
+    def successful?
+      return false if !finished?
+      return !failed?
+    end
+
+    def failed?
+      return false if !finished?
+      return total_objects_failed > 0
+    end
+
+    def public?
+      return @public ||= begin get(:public, false) end
+    end
+
+    class << self
+      def create(lifecycle, name = '', options = {})
+        id = generate_id(name)
+        workflow = new(id)
+        workflow.name = name
+        workflow.lifecycle = lifecycle
+        workflow.set(:created_at, Time.now.to_i)
+        workflow.set(:public, options.fetch(:public, false))
+        workflow.set(:logging, options.fetch(:logging, true))
+
+        register(workflow)
+
+        return workflow
+      end
+
+      def generate_id(workflow_name)
+        now = Time.now.to_f
+        random = Random.new(now)
+        return "#{name}__#{workflow_name}__#{(Time.now.to_f * 1000).to_i}__#{random.rand(now).to_i}"
+      end
+      private :generate_id
+
+      def cleanup(id)
+        workflow = new(id)
+        workflow.cleanup
+      end
+
+      def get_public_workflows(options = {})
+        return registry.public_flows(options.reverse_merge(parent_class: self)).map { |id| load(id) }
+      end
+
+      def get_private_workflows(options = {})
+        return registry.private_flows(options.reverse_merge(parent_class: self)).map { |id| load(id) }
+      end
+
+      def all(options = {})
+        return registry.all(options.reverse_merge(parent_class: self)).map { |id| load(id) }
+      end
+
+      def load(id, klass = nil)
+        workflow = nil
+
+        klass = read_flow_class(id) if klass.nil?
+        workflow = klass.new(id) if klass.respond_to?(:new)
+        return workflow
+      end
+
+      def read_flow_class(id)
+        klass = nil
+        raw_class = id.split('__').first
+        klass = begin
+          raw_class.constantize
+        rescue NameError => _
+          Rails.logger.warn("Unknown flow class for workflow id #{id}")
+          nil
+        end if !raw_class.nil?
+
+        return klass
+      end
+
+      def registered?(workflow)
+        return registry.include?(workflow)
+      end
+
+      def register(workflow)
+        registry.add(workflow)
+      end
+
+      def unregister(workflow)
+        registry.remove(workflow)
+      end
+
+      def terminal?(state)
+        return self::STATES_TERMINAL.include?(state)
+      end
+
+      def failure?(state)
+        return self::STATES_FAILED.include?(state)
+      end
+
+      def registry
+        return @registry ||= begin
+          FlowRegistry.new(Rworkflow::VERSION.to_s)
+        end
+      end
+
+      def serializer
+        YAML
+      end
+    end
+  end
+end

data/lib/rworkflow/flow_registry.rb

@@ -0,0 +1,69 @@
+module Rworkflow
+  class FlowRegistry
+    REDIS_PREFIX = 'flow:__registry'.freeze
+
+    def initialize(prefix = nil)
+      @redis_key = [REDIS_PREFIX, prefix].compact.join(':')
+      @public = RedisRds::SortedSet.new("#{@redis_key}:public")
+      @private = RedisRds::SortedSet.new("#{@redis_key}:private")
+    end
+
+    # Warning: using parent_class forces us to load everything, make this potentially much slower as we have to do the
+    # pagination in the app, not in the db
+    def all(options = {})
+      return self.public_flows(options) + self.private_flows(options)
+    end
+
+    def public_flows(options = {})
+      return get(@public, **options)
+    end
+
+    def private_flows(options = {})
+      return get(@private, **options)
+    end
+
+    def add(flow)
+      key = flow.created_at.to_i
+
+      if flow.public?
+        @public.add(key, flow.id)
+      else
+        @private.add(key, flow.id)
+      end
+    end
+
+    def remove(flow)
+      if flow.public?
+        @public.remove(flow.id)
+      else
+        @private.remove(flow.id)
+      end
+    end
+
+    def include?(flow)
+      if flow.public?
+        @public.include?(flow.id)
+      else
+        @private.include?(flow.id)
+      end
+    end
+
+    def get(zset, parent_class: nil, from: nil, to: nil, order: :asc)
+      from = from.to_i
+      to = to.nil? ? -1 : to.to_i
+
+      ids = []
+      if parent_class.nil? || parent_class == Flow
+        ids = zset.range(from, to, order: order)
+      else
+        ids = zset.range(0, -1, order: order).select do |id|
+          klass = Flow.read_flow_class(id)
+          !klass.nil? && klass <= parent_class
+        end.slice(from..to)
+      end
+
+      return ids
+    end
+    private :get
+  end
+end