gina-conveyor 0.0.2

data/.gitignore

@@ -0,0 +1,9 @@
+conf.d/*.yml
+*.swp
+workers/*.worker
+log/*
+*.log
+.DS_Store
+.workers
+pkg
+.conveyor

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,49 @@
+PATH
+  remote: .
+  specs:
+    gina-conveyor (0.0.2)
+      activemodel (~> 3.2.0)
+      activesupport (~> 3.2.0)
+      em-websocket (~> 0.3.6)
+      eventmachine (~> 0.12.10)
+      listen (~> 0.4.2)
+      rainbow (~> 1.1.4)
+      rb-readline (~> 0.4.2)
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    activemodel (3.2.3)
+      activesupport (= 3.2.3)
+      builder (~> 3.0.0)
+    activesupport (3.2.3)
+      i18n (~> 0.6)
+      multi_json (~> 1.0)
+    addressable (2.2.8)
+    builder (3.0.0)
+    em-websocket (0.3.6)
+      addressable (>= 2.1.1)
+      eventmachine (>= 0.12.9)
+    eventmachine (0.12.10)
+    ffi (1.0.11)
+    i18n (0.6.0)
+    listen (0.4.2)
+      rb-fchange (~> 0.0.5)
+      rb-fsevent (~> 0.9.1)
+      rb-inotify (~> 0.8.8)
+    multi_json (1.3.5)
+    rainbow (1.1.4)
+    rake (0.9.2.2)
+    rb-fchange (0.0.5)
+      ffi
+    rb-fsevent (0.9.1)
+    rb-inotify (0.8.8)
+      ffi (>= 0.5.0)
+    rb-readline (0.4.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  gina-conveyor!
+  rake

data/LICENSE

@@ -0,0 +1,25 @@
+Copyright (c) 2011 University of Alaska. All rights reserved.
+
+Developed by: Will Fisher and Scott Macfarlane
+              Geographic Information Network of Alaska
+              University of Alaska Fairbanks
+              http://www.gina.alaska.edu
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,24 @@
+Conveyor
+========
+
+Start of a thought on some code to handle moving data around when it shows up on various systems
+
+Installation
+------------
+
+    git clone https://github.com/gina-alaska/conveyor.git
+    cd conveyor
+    bundle install
+ 
+Usage
+-----
+
+    thor list                # list available commands
+    thor help [command]      # show help for the given command
+    thor belt list           # list the currently active belts
+    thor worker watch        # tell the workers to start watching directories!
+
+License
+-------
+
+See LICENSE file for licensing and credits.  Think BSD/MIT.

data/Rakefile

@@ -0,0 +1,2 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"

data/bin/conveyor

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require 'conveyor'
+
+Conveyor.start

data/conveyor.gemspec

@@ -0,0 +1,30 @@
+# -*- encoding: utf-8 -*-
+
+lib = File.expand_path('../lib/', __FILE__)
+$:.unshift lib unless $:.include?(lib)
+
+require 'conveyor/version'
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Will Fisher"]
+  gem.email         = ["will@gina.alaska.edu"]
+  gem.summary       = %q{GINA Data Conveyor}
+  gem.description   = %q{Conveyor is used for shuffling data around}
+  gem.homepage      = "http://github.com/gina-alaska/conveyor.git"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = %w{conveyor}
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "gina-conveyor"
+  gem.require_paths = ["lib"]
+  gem.version       = Conveyor::VERSION
+
+  gem.add_development_dependency('rake')
+  gem.add_dependency('activemodel', '~> 3.2.0')
+  gem.add_dependency('activesupport', '~> 3.2.0')
+  gem.add_dependency('listen', '~> 0.4.2')
+  gem.add_dependency('rainbow', '~> 1.1.4')
+  gem.add_dependency('rb-readline', '~> 0.4.2')
+  gem.add_dependency('eventmachine', '~> 0.12.10')
+  gem.add_dependency('em-websocket', '~> 0.3.6')
+end

data/lib/conveyor/belt.rb

@@ -0,0 +1,59 @@
+module Conveyor
+  # Does not persist, do not use any class variables 
+  class Belt
+    SETTLE_TIME = 30
+
+    include Conveyor::Output
+
+    attr_reader :command_file
+    
+    def initialize(watch_path, command_file)#, opts = {})
+      @work_dir = watch_path
+      @command_file = command_file
+      @queue = Conveyor::Queue.new
+    end
+
+    def count
+      @queue.count
+    end
+
+    def name
+      File.basename(@command_file, '.worker')
+    end
+
+    def touch(files)
+      files.each do |f|
+        @queue.push(f)
+      end
+    end
+    
+    def check
+      job = @queue.reserve
+      # puts job.inspect
+      if job && (Time.now - job[:updated_at]) > SETTLE_TIME
+        Worker.new(job[:file], @command_file).start
+        # Worker done remove it from the queue
+        @queue.pop(job[:file])
+      else
+        # File not ready yet
+        @queue.unreserve(job[:file]) unless job.nil?
+      end
+    rescue => e
+      puts e.message
+      puts e.backtrace
+    end
+
+    private
+
+    def process(file)
+    end
+
+    def escape_glob(glob)
+      if glob.class == String 
+        Regexp.new(glob)
+      else 
+        glob
+      end
+    end  
+  end
+end

data/lib/conveyor/foreman.rb

@@ -0,0 +1,182 @@
+module Conveyor
+  # Handle the incoming watch requests and assign the workers
+  class Foreman
+    include Singleton
+    include Conveyor::Output
+
+    attr_accessor :workers, :channel, :config
+
+    def initialize
+      loglvl(:debug)
+      read_configs
+
+      @listeners = {}
+      @belts = {}
+    end
+
+    def logfile
+      @config[:logfile]
+    end
+
+    def name
+      'Foreman'
+    end
+
+    def channel 
+      @channel ||= EM::Channel.new
+    end
+  
+    def read_configs
+      @config = {
+        "worker_defs" => File.expand_path('.workers', Dir.pwd),
+        "logfile" => File.expand_path('log/conveyor.log', Dir.pwd),
+        "threadpool" => 5,
+        "websocket" => {
+          "disabled" => false,
+          "host" => "0.0.0.0",
+          "port" => 9876
+        }
+      }
+      
+      @config_file = '.conveyor'
+      if File.exists? @config_file
+        @config.merge! YAML.load(File.open(@config_file))
+      elsif File.exists?('~/.conveyor')
+        @config_file = '~/.conveyor'
+        @config.merge! YAML.load(File.open(@config_file))
+      else
+        write_config(@config)
+      end
+      
+      # New version of conveyor update config file with new params
+      if !@config['version'] || @config['version'] != Conveyor::VERSION
+        @config['version'] = Conveyor::VERSION
+        write_config(@config)
+      end
+
+      @config.symbolize_keys!
+      @config[:websocket].symbolize_keys!
+    end
+    
+    def write_config(config)
+      File.open(@config_file, 'w') { |fp| fp << config.to_yaml }
+    end
+
+    def watch(*args, &block)
+      @listener_opts = args.extract_options!
+      @listener_dir = File.expand_path(args.first)
+      raise "Directory #{@listener_dir} not found" unless File.directory? @listener_dir
+      
+      @listener_opts[:latency] ||= 1
+      # Set a large latency if we force polling, prevents high cpu usage
+      @listener_opts[:latency] = 1 if @listener_opts[:latency] < 1 and @listener_opts[:force_polling]
+      
+      yield
+    end
+    
+    def match(*args, &block)
+      opts = args.extract_options!
+      debug "Filters: #{args.inspect}"
+      
+      listener = Listen.to(@listener_dir)
+      if @listener_opts[:latency]
+      	listener.latency(@listener_opts[:latency])
+      else
+	      listener.latency(0.5)
+      end
+
+      listener.ignore(opts[:ignore]) if @listener_opts[:ignore]
+      if @listener_opts[:force_polling]
+        debug "Force polling"
+        listener.force_polling(true) 
+      end
+      listener.filter(*args)
+
+      b = @belts[@listener_dir] = Belt.new(@listener_dir, @current_worker)
+      callback = lambda do |modified, added, removed|
+        begin
+          files = modified + added
+          b.touch(files) unless files.empty?
+        rescue => e
+          puts "Error: " + e.message
+          puts e.backtrace
+        end
+      end
+
+      listener.change(&callback)
+      @listeners[@listener_dir] = listener
+    rescue => e
+      error "ERROR: #{e.message}"
+      error e.backtrace
+    end
+    
+    def file(glob)
+      /#{glob}$/
+    end
+    
+    def extension(glob)
+      /\.#{glob}$/
+    end
+
+    def any
+      '*'
+    end
+    
+    def notify_list
+      @notify_list.flatten!
+      @notify_list.uniq!
+      @notify_list
+    end
+
+    def stop!
+      @listeners.each { |dir,l| info "Stopping #{dir} listener"; l.stop }
+      @listeners = {}
+      @notify_list = []      
+    end
+    
+    def start
+      load!
+      @listeners.each do |k, listener| 
+        info "Watching #{k}"
+        listener.start(false) 
+      end
+    end
+
+    def output_status
+      status = @belts.collect { |dir, b| "#{b.name}: #{b.count}" }
+      print "\r#{status.join(', ')}"
+    end
+
+    def check
+      @belts.each do |dir, b|
+        EM.defer do
+          b.check
+        end
+      end      
+    end
+
+    def load!
+      stop!
+      
+      info "Loading workers from #{@config[:worker_defs]}"            
+      FileUtils.mkdir_p(@config[:worker_defs])
+      
+      Dir.glob(File.join(@config[:worker_defs], '*.worker')) do |file|
+        begin
+          @current_worker = File.expand_path(file)
+          instance_eval File.read(@current_worker)
+        rescue => e
+          error [
+            "Error loading #{@current_worker}, skipping", 
+            e.message, 
+            e.backtrace
+          ].flatten
+        end
+      end
+    end
+    
+    def method_missing(method, value = nil)
+      return method.to_s
+    end 
+  end
+end

data/lib/conveyor/input/commands.rb

@@ -0,0 +1,21 @@
+module Conveyor
+	module Input
+		class Commands
+			class << self
+				def stop(*opts)
+					Kernel.exit
+				end
+				alias_method :exit, :stop
+
+				def listeners(*opts)
+					puts "Watching: "
+					puts "\t" + Conveyor::Foreman.instance.workers.keys.join("\n\t")
+				end
+
+				def unknown(cmd = nil)
+					puts "Unknown command #{cmd}"
+				end
+			end
+		end
+	end
+end

data/lib/conveyor/input/console.rb

@@ -0,0 +1,39 @@
+module Conveyor
+	module Input
+		class Console
+	    include Singleton
+
+	    def initialize
+	    end
+
+			def listen
+				save_tty_settings
+	      while line = Readline.readline('> ', true)
+	        handle line
+	      end
+	    rescue Interrupt => e	
+	    	restore_tty_settings
+	    	exit
+	 		end
+
+			def handle(line)
+				cmd = line.split(/\s+/)
+        return if cmd.empty?
+
+				if Commands.respond_to? cmd.first
+					Commands.send(cmd.shift, cmd.join(' '))
+				else
+					Commands.unknown(*cmd)
+				end
+			end
+
+			def save_tty_settings
+	      @stty_save = `stty -g`.chomp
+			end
+
+			def restore_tty_settings
+				system('stty', @stty_save);
+			end
+		end
+	end
+end

data/lib/conveyor/input.rb

@@ -0,0 +1,12 @@
+require 'conveyor/input/commands'
+require 'conveyor/input/console'
+
+module Conveyor
+	module Input
+		class << self
+			def listen
+				Conveyor::Input::Console.instance.listen
+			end
+		end
+	end
+end

data/lib/conveyor/job_state.rb

@@ -0,0 +1,2 @@
+module Conveyor
+end

data/lib/conveyor/output/channel.rb

@@ -0,0 +1,22 @@
+module Conveyor
+  module Output
+    class Channel
+      include Singleton
+
+      def initialize
+      end
+
+      def write(name, msgtype, *msg)
+        @channel = Conveyor::Foreman.instance.channel
+        return false if @channel.nil?
+
+        options = msg.extract_options!
+        format = '[%s] [%s::%s] %s'
+
+        Array.new(msg).each do |m|
+          @channel.push [msgtype, sprintf(format, Time.now, name, msgtype, m)]
+        end
+      end
+    end
+  end
+end

data/lib/conveyor/output/console.rb

@@ -0,0 +1,39 @@
+module Conveyor
+  module Output
+    class Console
+      class << self
+        def output(*msg)
+          options = msg.extract_options!
+          options[:color] ||= :default
+          options[:tab] ||= 0
+
+          format = "\t"*options[:tab]
+          format << "\r[%s] %s"
+          if msg.class == Array
+            msg.each do |m|
+              puts sprintf(format, Time.now, m).color(options[:color])              
+            end
+          else
+            puts sprintf(format, Time.now, msg).color(options[:color])              
+          end
+        end
+        alias_method :info, :output
+        alias_method :debug, :output
+        
+        def warning(*msg)
+          options = msg.extract_options!
+          options[:color] ||= :yellow
+          msg.flatten!
+          output(*msg, options)
+        end
+                
+        def error(*msg)
+          options = msg.extract_options!
+          options[:color] ||= :red
+          msg.flatten!
+          output(*msg, options)
+        end
+      end
+    end
+  end
+end

data/lib/conveyor/output/email.rb

@@ -0,0 +1,36 @@
+module Conveyor
+  module Output
+    class Email
+      class << self
+        def say(*msg)
+          # Do nothing
+        end
+        alias_method :info, :say
+        alias_method :debug, :say
+
+        def reset!
+          @msg_queue = []
+        end
+        
+        def warning(*msg)
+          # Do nothing
+        end
+        
+        def error(*msg)
+          @msg_queue ||= []
+          options = msg.extract_options!
+
+          msg =  msg.join("\n") if msg.class == Array
+          @msg_queue << msg
+        end
+        
+        def mail
+          return if @msg_queue.nil? || @msg_queue.empty?
+          puts "Sending email to #{Conveyor::Foreman.instance.notify_list}"
+          puts @msg_queue
+          reset!
+        end
+      end
+    end
+  end
+end

data/lib/conveyor/output/logfile.rb

@@ -0,0 +1,38 @@
+module Conveyor
+  module Output
+    class Logfile
+      class << self
+        def write(logfile, name, msgtype, *msg)
+          return false if logfile.nil?
+          
+          options = msg.extract_options!
+
+          format = '[%s] [%s::%s] %s'
+
+          if msg.class == Array
+            msg.each do |m|
+              output logfile, 
+                      sprintf(format, Time.now, name, msgtype, m)
+            end
+          else
+            output logfile, 
+                      sprintf(format, Time.now, name, msgtype, msg)
+          end
+        end
+
+        def output(logfile, msg = nil, &block)
+          FileUtils.mkdir_p(File.dirname(logfile))
+          fp = File.open(logfile, 'a')
+
+          if block_given?
+            yield fp
+          elsif !msg.nil?
+            fp << msg + "\n"
+          end
+
+          fp.close
+        end
+      end
+    end
+  end
+end

data/lib/conveyor/output.rb

@@ -0,0 +1,76 @@
+require 'conveyor/output/console'
+require 'conveyor/output/email'
+require 'conveyor/output/logfile'
+require 'conveyor/output/channel'
+
+module Conveyor
+  module Output
+    MSGLVLS = {
+      :debug    => 1, 
+      :info     => 10, 
+      :warning  => 20, 
+      :error    => 30
+    }
+
+    def should_log?(lvl, maxlvl = nil)
+      if maxlvl.nil? && @loglvl.nil?
+        loglvl :debug
+        should_log? lvl
+      elsif maxlvl.nil?
+        MSGLVLS[lvl.to_sym] >= @loglvl
+      else
+        MSGLVLS[lvl.to_sym] >= MSGLVLS[maxlvl.to_sym]
+      end
+    end
+
+    def loglvl(lvl)
+      @loglvl = MSGLVLS[lvl.to_sym] || MSGLVLS[:debug]
+    end
+
+    # Overrite this method to control logfile used
+    def logfile
+      nil
+    end
+
+    # Override this method to control name used
+    def name
+      nil
+    end
+
+    def say(*msg)
+      output(:info, *msg)
+    end
+
+    def info(*msg)
+      output(:info, *msg)
+    end
+    
+    def warning(*msg)
+      output(:warning, *msg)
+    end
+    
+    def error(*msg)
+      output(:error, *msg)
+    end
+
+    def debug(*msg)
+      output(:debug, *msg)
+    end
+    
+    def output(msgtype, *msg)
+      return false unless should_log?(msgtype)
+      Console.send(msgtype, *msg)
+      Logfile.write(logfile, name, msgtype, *msg)
+      Email.send(msgtype, *msg) 
+      Channel.instance.write(name, msgtype, *msg)
+    end
+    
+    def send_notifications
+      Email.mail
+    end
+
+    def notify(*emails)
+      Conveyor::Foreman.instance.notify_list << emails
+    end
+  end
+end