gina-conveyor 0.0.2

data/lib/conveyor/queue.rb

@@ -0,0 +1,80 @@
+module Conveyor
+  class Queue
+    def initialize
+      @mutex = Mutex.new
+      @queue = []
+    end
+
+    def count
+      @queue.count
+    end
+
+    def touch(file)
+      push(file)
+    end
+
+    def push(file)
+      @mutex.synchronize do
+        i = @queue.find_index { |j| j[:file] == file }
+        if i.nil?
+          @queue.push({ :file => file, :updated_at => Time.now })
+        else
+          @queue[i][:updated_at] = Time.now
+        end
+        @queue.sort! { |x,y| x[:updated_at] <=> y[:updated_at] }
+      end
+    end
+
+    def reserve(file = nil)
+      @mutex.synchronize do
+        # find first non-reserved job
+        i = @queue.find_index { |j| !j[:reserved] }
+        @queue[i][:reserved] = true unless i.nil?
+        @queue[i] unless i.nil?
+      end
+    end
+    
+    def unreserve(file)
+      @mutex.synchronize do
+        i = find_index(file)
+        @queue[i][:reserved] = false unless i.nil?
+        @queue[i] unless i.nil?
+      end
+    end
+    
+    def find_index(file = nil)
+      if file.nil?
+        @queue.empty? ? nil : 0
+      else
+        @queue.find_index { |j| j[:file] == file }
+      end
+    end
+
+    def find(file)
+      i = find_index(file)
+      @queue[i] unless i.nil?
+    end
+
+    def unpop(job)
+      @mutex.synchronize do
+        unless find(job[:file])
+          @queue.push(job)
+          @queue.sort! { |x,y| x[:updated_at] <=> y[:updated_at] }
+        end
+      end
+    end
+
+    def pop(file = nil)
+      if file.nil?
+        @mutex.synchronize do
+          @queue.shift
+        end
+      else
+        @mutex.synchronize do
+          j = find(file)
+          @queue.delete(j)
+        end
+      end
+    end
+  end
+end

data/lib/conveyor/status.rb

@@ -0,0 +1,29 @@
+module Conveyor
+  class Status
+    attr_reader :success
+    attr_accessor :errors
+    attr_reader :path
+    
+    def initialize(path = nil)
+      reset!(path)
+    end
+    
+    def reset!(path = nil)
+      @path = path
+      @success = true
+      @errors = []
+    end
+    
+    def fail!(v = false)
+      @success &= v
+    end
+    
+    def success!(v=true)
+      @success &= v
+    end
+    
+    def success?
+      @success
+    end
+  end
+end

data/lib/conveyor/version.rb

@@ -0,0 +1,3 @@
+module Conveyor
+  VERSION = "0.0.2"
+end

data/lib/conveyor/websocket.rb

@@ -0,0 +1,37 @@
+module Conveyor
+  class Websocket
+    class << self
+      def start
+        if config[:disable]
+          fm.info "Websocket disabled"
+          return
+        end
+
+        fm.info "Starting websocket on #{config[:host]}:#{config[:port]}", :color => :green
+        
+        EventMachine::start_server(config[:host], config[:port],
+          EventMachine::WebSocket::Connection, config) do |ws|
+          ws.onopen {
+            sid = fm.channel.subscribe { |type,msg| ws.send msg }
+            fm.info "#{sid} connected to websocket!"
+            ws.onclose {
+              fm.channel.unsubscribe(sid)
+            }
+          }
+        end
+      end
+
+      def stop
+        # EM::WebSocket.stop unless config[:disable]
+      end
+
+      def fm
+        Conveyor::Foreman.instance
+      end
+
+      def config
+        fm.config[:websocket]
+      end
+    end
+  end
+end

data/lib/conveyor/worker.rb

@@ -0,0 +1,120 @@
+require 'rubygems'
+require 'active_support/core_ext'
+require 'open3'
+require 'conveyor/workers/syntax'
+
+module Conveyor
+  class Worker
+    include Conveyor::Output
+    include Conveyor::Workers::Syntax
+    
+    attr_accessor :filename
+    attr_reader :status
+    attr_reader :worker_def
+
+    def initialize(file, worker_def, log = MSGLVLS[:debug])
+      @filename = file
+      @loglvl = log
+      @worker_def = worker_def
+      @notify = []
+      # @glob = escape_glob(glob)
+    end
+
+    # Return name to be used for logging purposes
+    def name(value=nil)
+      @name = value unless value.nil?
+      @name ||= File.basename(worker_def)
+      @name
+    end
+
+    # Default log file to be based on worker def name
+    def logfile
+      dir = File.dirname(worker_def)
+      File.expand_path(File.basename(worker_def, '.worker') + '.log', dir)
+    end
+    
+    # Catch any calls to error and set the status fail flags
+    def error(*msg)
+      opts = msg.extract_options!
+      unless msg.flatten.empty?
+        @status.fail!
+        msg.unshift("Error encountered in #{worker_def}")
+        super(*msg, opts)
+      end
+      @status.success?
+    end
+
+    # Start the worker
+    def start
+      @status = Conveyor::Status.new(@filename)
+      info "Starting #{@filename}", :color => :green
+      @start = Time.now
+      begin
+        instance_eval(File.read(@worker_def), worker_def)
+      ensure
+        @elapsed = "%0.2f"%(Time.now - @start)
+
+        #Check status and send any errors we collected
+        if @status.success?
+          info "Completed #{@filename}, #{@elapsed}s elapsed", :color => :green
+        else
+          error "Error(s) encountered in #{@filename}", :color => :red
+        end
+        send_notifications
+      end
+    end
+  
+    protected
+    
+    def create_dirs_for_cmd(src, dest)
+      if src.is_a?(Array) || Array.wrap(src).count > 1 || dest.last == ?/
+        mkdir(dest)
+      else
+        mkdir(File.dirname(dest))
+      end
+    end
+    
+    def verified_cmd(cmd, src, dest, &block)
+      create_dirs_for_cmd(src, dest)
+      dest = File.expand_path(dest)
+      
+      Array.wrap(src).each do |s|
+        run "#{cmd} #{s} #{dest}"
+        sync
+        
+        if block_given?
+          result = yield(s, dest)
+          @status.fail! unless result
+        end
+      end
+    end
+    
+    def verified_move(src, dest)
+      verified_cmd(:mv, src, dest) do |src,dest|
+        verify_move(src,dest)
+      end
+    end
+        
+    def verified_copy(src, dest)
+      verified_cmd(:cp, src, dest) do |src,dest|
+        verify_copy(src,dest)
+      end
+    end
+    
+    def verify_copy(src, dest)
+      if File.directory? dest
+        File.exists?(File.join(dest, File.basename(src)))
+      else
+        File.exists?(dest)
+      end
+    end
+    
+    def verify_move(src, dest)
+      if File.directory? dest
+        File.exists?(File.join(dest, File.basename(src))) && !File.exists?(src)
+      else
+        File.exists?(dest) && !File.exists?(src)
+      end
+    end
+  end
+end

data/lib/conveyor/workers/syntax.rb

@@ -0,0 +1,127 @@
+module Conveyor
+  module Workers
+    module Syntax
+      
+      # Will return a string of any method returned that isn't handled
+      # ex: match extension txt => match(extension("foo"))
+      def method_missing(method, value = nil)
+        return method.to_s
+      end      
+      
+      # Returns a recursive file glob string for the passed in string
+      def file(glob)
+        "**/#{glob}"
+      end
+    
+      # Returns an extension glob string for passed in string
+      def extension(glob)
+        "*.#{glob}"
+      end
+
+      # Returns a glob string that will match any file
+      def any
+        '*'
+      end
+      
+      # Returns a list of files that have the same basename but different extension
+      # in the same directory
+      def like(name)
+        dir = File.dirname(name)
+        Dir.glob(File.join(dir, File.basename(name, '.*') + '.*'))
+      end
+
+      # Returns the filename of the file that triggered the job
+      def filename
+        @filename
+      end
+      
+      # Which directories to watch for file change events.
+      def watch(*args, &block)
+        yield
+      end
+
+      # Match string/glob for the files that should trigger file change events
+      def match(glob, &block)
+        yield @filename
+      end
+      
+      # Run the system sync command
+      def sync
+        run 'sync', :quiet => true
+      end
+    
+      # Change current working directory, optionally takes a block
+      # NOTE: Consider removing this as it can cause problems with threaded workers
+      def chdir(dir, &block)
+        Dir.chdir(File.expand_path(dir), &block)
+      end
+      
+      # Run the system command, and make sure that any errors are caught and returne
+      # up the status change
+      def run(*cmd)
+        opts = cmd.extract_options!
+        begin
+          info cmd.join(' ') unless opts[:quiet]
+          output,err,thr = Open3.capture3(Array.wrap(cmd).join(' '))
+          info output.chomp unless output.chomp.length == 0
+  				if thr.success?
+  					if err.chomp.length > 0
+  						warning "Error output recieved, but no error code recieved"
+  						warning err.chomp
+  					end
+  				else
+          	error "Error running: `#{cmd.join(' ')}`", err.chomp 
+          	@status.fail!
+  				end
+        
+          return thr.success?
+        rescue => e
+          error e.class, e.message, e.backtrace.join("\n")
+        end
+      end
+      
+      # Deletes passed in files
+      def delete(files)
+        # sync before we delete
+        sync
+        Array.wrap(files).each do |f|
+          info "removing #{f}"
+          FileUtils.rm(f)
+          error "#{f} wasn't removed" if File.exists?(f)
+        end
+      end
+    
+      # Create a new directory
+      def mkdir(dir)
+        FileUtils.mkdir_p(File.expand_path(dir))      
+        @status.fail! unless File.exists?(File.expand_path(dir))
+      end
+
+      # Copy files to destination
+      def copy(src = [], dest = nil)
+        destination = dest unless dest.nil?
+        source = src unless src.empty?
+      
+        if source && destination
+          verified_copy(source, destination)
+        end      
+      end
+    
+      # Move files to destination
+      def move(src=[], dest = nil)
+        destination = dest unless dest.nil?
+        source = src unless src.empty?
+      
+        if source && destination
+          verified_move(source, destination)
+        end
+      end
+  
+      # Scp files to destination
+      # See: man scp
+      def scp(src, dest)
+        run "scp #{Array.wrap(src).join(' ')} #{dest}"
+      end
+    end
+  end
+end

data/lib/conveyor.rb

@@ -0,0 +1,62 @@
+#!/usr/bin/env ruby
+$: << File.expand_path('../lib', File.dirname(__FILE__))
+
+require 'rubygems'
+require 'active_support/core_ext'
+require 'singleton'
+require 'listen'
+require 'yaml'
+require 'rainbow'
+require 'rb-readline'
+require 'fileutils'
+require 'eventmachine'
+require 'em-websocket'
+require "conveyor/version"
+
+module Conveyor
+  autoload :Output,     'conveyor/output'
+  autoload :Foreman,    'conveyor/foreman'
+  autoload :Belt,       'conveyor/belt'
+  autoload :Worker,     'conveyor/worker'
+  autoload :Status,     'conveyor/status'
+  autoload :Input, 		  'conveyor/input'
+  autoload :Queue,      'conveyor/queue'
+  autoload :Websocket,  'conveyor/websocket'
+
+  def self.stop
+    Foreman.instance.stop!
+    EventMachine.stop
+    exit
+  end
+
+  def self.fm
+    Foreman.instance
+  end
+
+  def self.start
+    EventMachine.run do
+      trap("TERM") { stop }
+      trap("INT") { stop }
+
+      EventMachine.threadpool_size = fm.config[:threadpool] || 20
+
+      fm.info "Starting Conveyor v#{Conveyor::VERSION}"
+      fm.start
+      fm.info "Waiting for files", :color => :green
+      fm.info "Press CTRL-C to stop"
+      Conveyor::Websocket.start
+
+      EventMachine::PeriodicTimer.new(1) do
+        fm.output_status
+      end
+
+      EventMachine::PeriodicTimer.new(1) do
+        fm.check
+      end
+    end
+  end
+end
+
+def watch(*args, &block)
+  fm.watch(*args, &block)
+end

data/webviewer/.gitignore

@@ -0,0 +1,15 @@
+# See http://help.github.com/ignore-files/ for more about ignoring files.
+#
+# If you find yourself ignoring temporary files generated by your text editor
+# or operating system, you probably want to add a global ignore instead:
+#   git config --global core.excludesfile ~/.gitignore_global
+
+# Ignore bundler config
+/.bundle
+
+# Ignore the default SQLite database.
+/db/*.sqlite3
+
+# Ignore all logfiles and tempfiles.
+/log/*.log
+/tmp

data/webviewer/Gemfile

@@ -0,0 +1,39 @@
+source 'https://rubygems.org'
+
+gem 'rails', '3.2.3'
+
+# Bundle edge Rails instead:
+# gem 'rails', :git => 'git://github.com/rails/rails.git'
+
+gem 'sqlite3'
+gem 'haml'
+gem 'bootstrap-sass', '~> 2.0.2'
+
+# Gems used only for assets and not required
+# in production environments by default.
+group :assets do
+  gem 'sass-rails',   '~> 3.2.3'
+  gem 'coffee-rails', '~> 3.2.1'
+
+  # See https://github.com/sstephenson/execjs#readme for more supported runtimes
+  gem 'therubyracer', :platform => :ruby
+
+  gem 'uglifier', '>= 1.0.3'
+end
+
+gem 'jquery-rails'
+
+# To use ActiveModel has_secure_password
+# gem 'bcrypt-ruby', '~> 3.0.0'
+
+# To use Jbuilder templates for JSON
+# gem 'jbuilder'
+
+# Use unicorn as the app server
+# gem 'unicorn'
+
+# Deploy with Capistrano
+# gem 'capistrano'
+
+# To use debugger
+# gem 'ruby-debug19', :require => 'ruby-debug'

data/webviewer/Gemfile.lock

@@ -0,0 +1,119 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    actionmailer (3.2.3)
+      actionpack (= 3.2.3)
+      mail (~> 2.4.4)
+    actionpack (3.2.3)
+      activemodel (= 3.2.3)
+      activesupport (= 3.2.3)
+      builder (~> 3.0.0)
+      erubis (~> 2.7.0)
+      journey (~> 1.0.1)
+      rack (~> 1.4.0)
+      rack-cache (~> 1.2)
+      rack-test (~> 0.6.1)
+      sprockets (~> 2.1.2)
+    activemodel (3.2.3)
+      activesupport (= 3.2.3)
+      builder (~> 3.0.0)
+    activerecord (3.2.3)
+      activemodel (= 3.2.3)
+      activesupport (= 3.2.3)
+      arel (~> 3.0.2)
+      tzinfo (~> 0.3.29)
+    activeresource (3.2.3)
+      activemodel (= 3.2.3)
+      activesupport (= 3.2.3)
+    activesupport (3.2.3)
+      i18n (~> 0.6)
+      multi_json (~> 1.0)
+    arel (3.0.2)
+    bootstrap-sass (2.0.2)
+    builder (3.0.0)
+    coffee-rails (3.2.2)
+      coffee-script (>= 2.2.0)
+      railties (~> 3.2.0)
+    coffee-script (2.2.0)
+      coffee-script-source
+      execjs
+    coffee-script-source (1.3.1)
+    erubis (2.7.0)
+    execjs (1.3.1)
+      multi_json (~> 1.0)
+    haml (3.1.4)
+    hike (1.2.1)
+    i18n (0.6.0)
+    journey (1.0.3)
+    jquery-rails (2.0.2)
+      railties (>= 3.2.0, < 5.0)
+      thor (~> 0.14)
+    json (1.7.0)
+    libv8 (3.3.10.4)
+    mail (2.4.4)
+      i18n (>= 0.4.0)
+      mime-types (~> 1.16)
+      treetop (~> 1.4.8)
+    mime-types (1.18)
+    multi_json (1.3.4)
+    polyglot (0.3.3)
+    rack (1.4.1)
+    rack-cache (1.2)
+      rack (>= 0.4)
+    rack-ssl (1.3.2)
+      rack
+    rack-test (0.6.1)
+      rack (>= 1.0)
+    rails (3.2.3)
+      actionmailer (= 3.2.3)
+      actionpack (= 3.2.3)
+      activerecord (= 3.2.3)
+      activeresource (= 3.2.3)
+      activesupport (= 3.2.3)
+      bundler (~> 1.0)
+      railties (= 3.2.3)
+    railties (3.2.3)
+      actionpack (= 3.2.3)
+      activesupport (= 3.2.3)
+      rack-ssl (~> 1.3.2)
+      rake (>= 0.8.7)
+      rdoc (~> 3.4)
+      thor (~> 0.14.6)
+    rake (0.9.2.2)
+    rdoc (3.12)
+      json (~> 1.4)
+    sass (3.1.16)
+    sass-rails (3.2.5)
+      railties (~> 3.2.0)
+      sass (>= 3.1.10)
+      tilt (~> 1.3)
+    sprockets (2.1.3)
+      hike (~> 1.2)
+      rack (~> 1.0)
+      tilt (~> 1.1, != 1.3.0)
+    sqlite3 (1.3.6)
+    therubyracer (0.10.1)
+      libv8 (~> 3.3.10)
+    thor (0.14.6)
+    tilt (1.3.3)
+    treetop (1.4.10)
+      polyglot
+      polyglot (>= 0.3.1)
+    tzinfo (0.3.33)
+    uglifier (1.2.4)
+      execjs (>= 0.3.0)
+      multi_json (>= 1.0.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bootstrap-sass (~> 2.0.2)
+  coffee-rails (~> 3.2.1)
+  haml
+  jquery-rails
+  rails (= 3.2.3)
+  sass-rails (~> 3.2.3)
+  sqlite3
+  therubyracer
+  uglifier (>= 1.0.3)