yawl 0.0.2

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in yawl.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Ricardo Chimal, Jr.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,37 @@
+# Yawl
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'yawl'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install yawl
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Example
+
+```
+bundle exec sequel -m migrations/ postgres://localhost/yawl_examples
+bundle exec ruby examples/cook.rb
+bundle exec ruby examples/cook_worker.rb
+```
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/examples/cook.rb

@@ -0,0 +1,11 @@
+$:.unshift(File.expand_path(File.join(File.dirname($0), "../lib")))
+
+require "sequel"
+
+Sequel.connect(ENV["DATABASE_URL"] || "postgres://localhost/yawl_examples")
+
+require "yawl"
+require File.dirname(File.expand_path(__FILE__)) + "/steps/scrambled_eggs"
+
+p = Yawl::Process.create(:desired_state => "scrambled_eggs")
+p.start

data/examples/cook_worker.rb

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+$:.unshift(File.expand_path(File.join(File.dirname($0), "../lib")))
+
+require "sequel"
+Sequel.connect(ENV["DATABASE_URL"] || "postgres://localhost/yawl_examples")
+
+require "yawl"
+require "yawl/worker"
+
+require File.dirname(File.expand_path(__FILE__)) + "/steps/scrambled_eggs"
+
+$stdout.sync = true
+Yawl::Worker.start

data/examples/steps/scrambled_eggs.rb

@@ -0,0 +1,38 @@
+# define a set of steps
+
+Yawl::Steps.set :scrambled_eggs do
+  step :buy_eggs do
+    def run
+      puts "bought a dozen eggs at the market"
+    end
+  end
+
+  step :put_pan_on_stove do
+    def run
+      puts "put a non-stick pan on the stove"
+    end
+  end
+
+  step :turn_on_heat_to_medium do
+    def run
+      puts "turn on heat to medium"
+    end
+  end
+
+  step :crack_eggs_and_scramble do
+    def run
+      puts "crack eggs on pan and scramble"
+    end
+  end
+
+  step :serve do
+    def run
+      puts "you just got served"
+    end
+  end
+end
+
+# Define a process definition and which set of steps should belong to the process
+Yawl::ProcessDefinitions.add(:scrambled_eggs) do |process|
+  Yawl::Steps.realize_set_on(:scrambled_eggs, process)
+end

data/lib/yawl/config.rb

@@ -0,0 +1,28 @@
+module Yawl
+  module Config
+    extend self
+
+    attr_writer :app
+    attr_accessor :deploy
+
+    def app
+      @app || "yawl"
+    end
+
+    def log_quiet?
+      env("LOG_QUIET") == "1"
+    end
+
+    def log_sequel?
+      env("LOG_SEQUEL") == "1"
+    end
+
+    def env(key)
+      ENV[key]
+    end
+
+    def env!(key)
+      ENV[key] or raise "Missing ENV[#{key}]"
+    end
+  end
+end

data/lib/yawl/log.rb

@@ -0,0 +1,37 @@
+require "scrolls"
+require "thread"
+
+module Yawl
+  module Log
+    extend self
+
+    def elapsed_since(t0)
+      "%dms" % (Time.now - t0)
+    end
+
+    def log_measure(data)
+      measure_base = "#{Config.app}.#{data[:ns]}.#{data[:fn]}"
+      begin
+        t0 = Time.now
+        log(data.merge(at: "start"))
+        log_control = OpenStruct.new(data.merge(finish_at: "finish"))
+        result = yield(log_control)
+        log(data.merge("measure##{measure_base}.#{log_control.finish_at}" => elapsed_since(t0), at: log_control.finish_at))
+        result
+      rescue => error
+        log(data.merge("measure##{measure_base}.exception" => elapsed_since(t0), at: "exception", "class" => error.class, message: error.message))
+        raise error
+      end
+    end
+
+    def log(data, &block)
+      if Config.log_quiet?
+        block.call if block
+      else
+        params = { app: Config.app }
+        params[:source] = Config.deploy if Config.deploy
+        Scrolls.log(params.merge(data), &block)
+      end
+    end
+  end
+end

data/lib/yawl/process.rb

@@ -0,0 +1,136 @@
+require "sequel"
+require "securerandom"
+
+require "yawl/log"
+
+module Yawl
+  class Process < Sequel::Model
+    plugin :validation_helpers
+    one_to_many :steps, :order => [:seq]
+
+    class ConcurrencyError < RuntimeError; end
+    class RequestIdAttributeMismatch < RuntimeError; end
+
+    def self.log(data, &block)
+      Log.log({ ns: "process" }.merge(data), &block)
+    end
+
+    def log(data, &block)
+      self.class.log({ process_id: id, desired_state: desired_state }.merge(data), &block)
+    end
+
+    def self.existing_by_request_id_and_attributes(request_id, attributes)
+      return nil unless request_id
+    end
+
+    def self.web_only_attributes
+      %w[config desired_state request_id]
+    end
+
+    def self.clean_attributes(attributes)
+      attributes.reject {|k, v| !web_only_attributes.include?(k.to_s) }
+    end
+
+    def before_validation
+      super
+      if new?
+        self.name = SecureRandom.uuid
+        self.created_at = Time.now
+      end
+    end
+
+    def validate
+      super
+      validates_presence :name
+      validates_includes ProcessDefinitions.all_names, :desired_state
+    end
+
+    def start
+      realize_process_definition
+      start_first_unfinished_step_or_complete
+    end
+
+    def running?
+      !%w[completed failed].include?(state)
+    end
+
+    def current?
+    end
+
+    def start_first_unfinished_step_or_complete
+      unless start_first_unfinished_step
+        log(at: "completed")
+        update(:state => "completed")
+        end_state_reached
+      end
+    end
+
+    def start_first_unfinished_step
+      if unfinished_step = unfinished_steps.first
+        update(:state => "executing")
+        unfinished_step.start
+        unfinished_step
+      end
+    end
+
+    def step_finished
+      start_first_unfinished_step_or_complete
+    end
+
+    def step_failed
+      log(at: "failed")
+      update(:state => "failed")
+      end_state_reached
+    end
+
+    def next_step_seq
+      steps_dataset.count == 0 ? 1 : steps_dataset.last.seq + 1
+    end
+
+    def add_step(data = {})
+      data[:seq] ||= next_step_seq
+      super(data)
+    end
+
+    def unfinished_steps
+      steps_dataset.where("state != 'completed'")
+    end
+
+    def end_state_reached
+    end
+
+    def callback_payload
+      {
+        "process" => to_public_h.merge("url" => Urls.process(self))
+      }
+    end
+
+    def merged_config
+      default_config.deep_merge(config || {})
+    end
+
+    def default_config
+      {
+        "ion_branch" => "master",
+        "manifesto_source_manifest" => "production",
+        "manifesto_releases" => {}
+      }
+    end
+
+    def realize_process_definition
+      log(fn: "realize_process_definition") do
+        ProcessDefinitions.realize_on(desired_state, self)
+        save # if realizing made any changes, such as to config
+      end
+    end
+
+    def to_public_h
+      {
+        "name" => name,
+        "desired_state" => desired_state,
+        "state" => state,
+        "config" => merged_config
+      }
+    end
+  end
+end

data/lib/yawl/process_definitions.rb

@@ -0,0 +1,27 @@
+module Yawl
+  module ProcessDefinitions
+    def self.[](name)
+      all[name.to_sym]
+    end
+
+    def self.add(name, &block)
+      all[name.to_sym] = block
+    end
+
+    def self.all
+      @all ||= {}
+    end
+
+    def self.all_names
+      all.keys.map {|k| k.to_s }
+    end
+
+    def self.realize_on(name, process)
+      unless definition = self[name]
+        raise "Can't find definition #{name}"
+      end
+
+      definition.call(process)
+    end
+  end
+end

data/lib/yawl/step.rb

@@ -0,0 +1,109 @@
+require "sequel"
+require "queue_classic"
+require "queue_classic-later"
+
+require "yawl/log"
+
+module Yawl
+  class Step < Sequel::Model
+    many_to_one :process
+    one_to_many :attempts, :class => "Yawl::StepAttempt", :order => :started_at
+
+    class Fatal < StandardError
+      def initialize
+        super("Fatal error in step")
+      end
+    end
+
+    class Tired < StandardError
+      def initialize
+        super("Step slept")
+      end
+    end
+
+    def self.log(data, &block)
+      Log.log({ ns: "step" }.merge(data), &block)
+    end
+
+    def log(data, &block)
+      self.class.log({ process_id: process.id, step_id: id, step_name: name }.merge(data), &block)
+    end
+
+    def self.execute(id)
+      self[id].execute
+    end
+
+    def self.restart_interrupted
+      where(:state => "interrupted").each do |step|
+        step.start
+      end
+    end
+
+    def duration
+      if attempts.any?
+        (attempts.last.completed_at || Time.now) - attempts.first.started_at
+      end
+    end
+
+    def start
+      log(fn: "start")
+      update(:state => "pending")
+      QC.enqueue("Yawl::Step.execute", id)
+    end
+
+    def start_in(seconds)
+      log(fn: "start_in", seconds: seconds)
+      QC.enqueue_in(seconds, "Yawl::Step.execute", id)
+    end
+
+    def start_after_delay
+      start_in(real_step.delay_between_attempts)
+    end
+
+    def real_step
+      @real_step ||= Steps.for(name, self)
+    end
+
+    def out_of_attempts?
+      attempts.count >= real_step.attempts
+    end
+
+    def execute
+      log(fn: "execute") do
+        begin
+          # TODO(dpiddy): transactions here?
+          update(:state => "executing")
+          attempt = add_attempt(:started_at => Time.now)
+          real_step.run_with_log
+          log(fn: "execute", at: "run_completed")
+          attempt.update(:output => real_step.output, :completed_at => Time.now)
+          update(:state => "completed")
+          process.step_finished
+        rescue Step::Fatal, Step::Tired, StandardError, SignalException => e
+          log(:fn => "execute", :at => "caught_exception", :class => e.class, :message => e.message)
+          attempt.update(:output => "#{real_step.output}\n\n---\nCAUGHT ERROR: #{e}\n#{e.backtrace.join("\n")}\n", :completed_at => Time.now)
+
+          if out_of_attempts? || e.is_a?(Step::Fatal)
+            update(:state => "failed")
+            process.step_failed
+            raise
+          elsif SignalException === e && e.signm == "SIGTERM" # we are shutting down
+            update(:state => "interrupted")
+            raise
+          else
+            update(:state => "pending")
+            start_after_delay
+          end
+        end
+      end
+    end
+
+    def to_public_h
+      {
+        "seq" => seq,
+        "name" => name,
+        "state" => state
+      }
+    end
+  end
+end

data/lib/yawl/step_attempt.rb

@@ -0,0 +1,11 @@
+require "sequel"
+
+module Yawl
+  class StepAttempt < Sequel::Model
+    many_to_one :step
+
+    def output
+      (super || "")
+    end
+  end
+end

data/lib/yawl/steps/base.rb

@@ -0,0 +1,136 @@
+require "stringio"
+require "forwardable"
+
+module Yawl
+  module Steps
+    class SetDefiner
+      attr_reader :step_names
+
+      def initialize(name, &block)
+        @name = name
+        @step_names = []
+        @block = block
+      end
+
+      def run
+        instance_eval(&@block)
+      end
+
+      def step(name, &block)
+        Steps.step(name, &block).tap do |k|
+          step_names.push(k.name)
+        end
+      end
+    end
+
+    def self.sets
+      @sets ||= {}
+    end
+
+    def self.set(name, &block)
+      name = name.to_sym
+      definer = SetDefiner.new(name, &block)
+      definer.run
+      sets[name] = definer.step_names
+    end
+
+    def self.step(name, &block)
+      name = name.to_sym
+      Class.new(Base, &block).tap do |klass|
+        klass.instance_eval "def name; #{name.inspect} end", __FILE__, __LINE__
+        Base.register(klass)
+      end
+    end
+
+    def self.realize_set_on(name, process)
+      unless set_step_names = sets[name.to_sym]
+        raise "Set #{name} not found"
+      end
+
+      set_step_names.each do |set_step_name|
+        process.add_step(:name => set_step_name)
+      end
+    end
+
+    def self.for(name, process_step)
+      name = name.to_sym
+      klass = Base.all_steps[name] || RealStepMissing
+      klass.new(process_step)
+    end
+
+    class Base
+      extend Forwardable
+
+      def self.all_steps
+        @all_steps ||= {}
+      end
+
+      def self.register(step)
+        all_steps[step.name] = step
+      end
+
+      attr_reader :output_io
+
+      def initialize(process_step)
+        @process_step = process_step
+        @output_io = StringIO.new("")
+      end
+
+      def puts(*a)
+        @output_io.puts(*a)
+      end
+
+      def output
+        @output_io.string
+      end
+
+      def attempts
+        3
+      end
+
+      def delay_between_attempts
+        10
+      end
+
+      def process
+        @process_step.process
+      end
+
+      def name
+        self.class.name
+      end
+
+      def run_with_log
+        log(fn: "run") do
+          run
+        end
+      end
+
+      def sleep
+        raise Step::Tired
+      end
+
+      def fatal!
+        raise Step::Fatal
+      end
+
+      def log(data, &block)
+        Log.log({ ns: "step_#{name}" }.merge(data), &block)
+      end
+    end
+
+    class RealStepMissing < Base
+      def self.name
+        "real_step_missing"
+      end
+
+      def run
+        raise "The real step is missing"
+      end
+
+      def attempts
+        0
+      end
+    end
+  end
+end

data/lib/yawl/version.rb

@@ -0,0 +1,3 @@
+module Yawl
+  VERSION = "0.0.2"
+end

data/lib/yawl/worker.rb

@@ -0,0 +1,16 @@
+require "queue_classic"
+
+module Yawl
+  class Worker < QC::Worker
+    def self.start
+      worker = new
+
+      trap("TERM") do
+        worker.stop
+        raise SignalException.new("SIGTERM")
+      end
+
+      worker.start
+    end
+  end
+end

data/lib/yawl.rb

@@ -0,0 +1,15 @@
+require "yawl/version"
+
+module Yawl
+  # Your code goes here...
+end
+
+require "yawl/config"
+require "yawl/log"
+
+require "yawl/process"
+require "yawl/process_definitions"
+require "yawl/step"
+require "yawl/step_attempt"
+
+require "yawl/steps/base"

data/migrations/01_setup_tables.rb

@@ -0,0 +1,34 @@
+Sequel.migration do
+  change do
+    create_table(:processes) do
+      primary_key :id
+      String :desired_state, :text=>true, :null=>false
+      String :state, :default=>"pending", :text=>true, :null=>false
+      DateTime :created_at
+      String :name, :text=>true
+      String :config
+      String :request_id, :text=>true
+      String :specified_attributes
+    end
+
+    create_table(:steps) do
+      primary_key :id
+      Integer :process_id, :null=>false
+      Integer :seq, :null=>false
+      String :name, :text=>true, :null=>false
+      String :state, :default=>"pending", :text=>true, :null=>false
+
+      index [:process_id]
+    end   
+    
+    create_table(:step_attempts) do
+      primary_key :id
+      Integer :step_id, :null=>false
+      File :output
+      DateTime :started_at
+      DateTime :completed_at
+
+      index [:step_id]
+    end
+  end
+end

data/migrations/02_setup_queue_classic.rb

@@ -0,0 +1,14 @@
+require "queue_classic"
+require "queue_classic/later"
+
+Sequel.migration do
+  down do
+    QC::Setup.drop
+    QC::Later::Setup.drop
+  end
+
+  up do
+    QC::Setup.create
+    QC::Later::Setup.create
+  end
+end

data/yawl.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'yawl/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "yawl"
+  spec.version       = Yawl::VERSION
+  spec.authors       = ["Ricardo Chimal, Jr."]
+  spec.email         = ["kiwi@null.cx"]
+  spec.description   = %q{Yet Another Workflow Library for Ruby}
+  spec.summary       = %q{Yet Another Workflow Library for Ruby}
+  spec.homepage      = "https://github.com/ricardochimal/yawl"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "sequel"
+  spec.add_dependency "scrolls"
+  spec.add_dependency "queue_classic"
+  spec.add_dependency "queue_classic-later"
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+end

metadata

@@ -0,0 +1,169 @@
+--- !ruby/object:Gem::Specification
+name: yawl
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+  prerelease: 
+platform: ruby
+authors:
+- Ricardo Chimal, Jr.
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-01-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: scrolls
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: queue_classic
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: queue_classic-later
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Yet Another Workflow Library for Ruby
+email:
+- kiwi@null.cx
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- examples/cook.rb
+- examples/cook_worker.rb
+- examples/steps/scrambled_eggs.rb
+- lib/yawl.rb
+- lib/yawl/config.rb
+- lib/yawl/log.rb
+- lib/yawl/process.rb
+- lib/yawl/process_definitions.rb
+- lib/yawl/step.rb
+- lib/yawl/step_attempt.rb
+- lib/yawl/steps/base.rb
+- lib/yawl/version.rb
+- lib/yawl/worker.rb
+- migrations/01_setup_tables.rb
+- migrations/02_setup_queue_classic.rb
+- yawl.gemspec
+homepage: https://github.com/ricardochimal/yawl
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -1330465653979215358
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -1330465653979215358
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Yet Another Workflow Library for Ruby
+test_files: []