rflow 0.0.5

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: c322bc6cf9c3b4ccd46b13cf56d3c2460dc5f0be
+  data.tar.gz: af2d6fb3e7051a074c56fa10b70b5a02b23bb0d7
+SHA512:
+  metadata.gz: 5a3cd46af3c815d2cb5840d48a0e38f7e28dbe911276fac75d3466bef9d05d7d49e38baa010b5ce419a67aea5829d2f227d2ff74aa5e689f6c1e6109d882ad81
+  data.tar.gz: 539c61aca94e84e1ccb00acba1ef87c7a6556dc65180f9f905561b09e147eab83ff0a409dd01b84a1113ba820cf89c6470ee64c1ace68736232423ba3a1d9668

data/.gitignore

@@ -0,0 +1,6 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+\#*
+.\#*

data/.rvmrc

@@ -0,0 +1 @@
+rvm 1.9.2@rflow-devel

data/Gemfile

@@ -0,0 +1,5 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in rflow.gemspec
+gemspec
+

data/NOTES

@@ -0,0 +1,187 @@
+RFlow Manager
+
+  Components
+    Input Ports
+    Output Ports
+
+  Connections
+    Input Ports
+    Output Ports
+
+rflow <config file>
+  figure out a work directory
+  make sure that it has the right subdirectories (can be overridden)
+    run tmp logs schemas components
+  Set up logging to logs/rflow.log
+  Load all schemas
+  Verify all component installation
+  Initialize components
+  Start components running and make sure that they "daemonize" correctly
+    - place pid files in deployment's run directory
+  Configure components via zmq
+  Daemonize self
+  
+
+
+class Component
+  def self.input_port
+  end 
+
+  def self.output_port
+  end
+
+  attr_accessor :state
+
+  def initialize(config, run_directory)
+    
+  end
+
+  def run
+    
+  end
+
+  def configure
+
+end
+
+
+
+class PassThrough < Component
+  input_port [:in]
+  input_port :another_in
+  output_port :out
+  output_port :another_out
+
+  def initialize(config, run_directory)
+    # This will initialize the ports
+    super
+    # Do stuff to initialize component
+  end   
+
+end
+
+
+Computation Requirements:
+  Initial startup with:
+    - management bus connection information
+    - group and instance UUID
+    - beacon interval
+    - run directory, containing 
+      - PID files
+      - log dir + logs
+      - computation-specific configuration (conf dir)
+  Needs to process the following messages from mgmt bus:
+    - CONFIGURE (ports)
+    - RUN
+    - SHUTDOWN
+  Needs to send the following messages to mgmt bus:
+    - LOG
+    - BEACON (state machine of the below submessages)
+      - STARTED
+      - CONFIGURED
+      - RUNNING
+      - STOPPING
+      - STOPPED
+      - ERROR
+  On startup:
+    - listen to mgmt bus
+    - publish BEACON + state to mgmt bus every (beacon interval) seconds (default to 1 sec)
+
+
+External Computations:
+  - Given (out-of-band) startup info (mgmt bus, UUIDs, beacon interval)
+  - 
+
+
+RFlow
+  - Will need a DB for config
+    - Initial startup will need to resolve all remaining outstanding items (ports, UUIDs, etc) and store in config DB
+    - MVC, Mongrel2-like?
+
+Translate
+  - Need to add <associated type="objtype" name="myname"> where name attr can be used in later XML templates
+      
+
+
+
+
+
+----------------
+Plugins:
+  an externally defined plugin needs access to all current data types, as well as being able to define its own and tell the system about that.
+    - necessary to tell system?
+    - need a protocol for defining schema transfer
+    - each message has attached schema
+  
+
+lib/rflow/message.rb
+
+RFlow::Config
+
+RFlow::Management
+  - Somewhere for external people to register new computations with running system
+  - computation says that its running and asks for Connection configuration
+    - how will it specify where in the workflow it wants to run????
+  
+RFlow::Message(complete on-the-wire Avro message format)
+  data_type, provenance, external_ids, empty, data (see below)
+
+RFlow::Data::(various message data blocks)
+
+RFlow::Computation
+  uuid, name, class, input_ports, output_ports
+
+
+RFlow::Connection
+  encapsulates link knowlege and provides an API for sending and receiving
+  each computation will have one for each port
+  each computation will call into the connection to send (possibly via a Port object) and recieve
+
+RFlow::Connection::AMQP
+  will manage connections to an AMQP server
+
+RFlow::Connection::ZMQ
+
+    
+
+
+computation_a.output_port -> (connection.incoming -> connection.outgoing) -> computation_b.input_port
+
+AMQP::Topic - responsible for setting up a topic -> queue binding
+  r.incoming = amqp connection, channel, vhost, login, password, topic
+  r.outgoing = amqp connection, channel, vhost, login, password, queue name
+  behavior -> n x m, "round-robin" among the connected outgoing
+    incoming behavior will need to set topic/key, uses the data type in the RFlow::Message
+    
+
+ZMQ::PubSub - device-less, responsible for assigning ip/port and assigning one client to bind the port
+  r.incoming = zmq connection string (tcp://ip:port), type pub
+  r.outgoing = zmq connection string (tcp://ip:port), type sub
+  behavior -> n x m, broadcast sending, 
+
+ZMQ::PushPull - device-less, responsible for assigning ip/port and assigning one client to bind the port
+  r.incoming = zmq connection string (tcp://ip:port), type push
+  r.outgoing = zmq connection string (tcp://ip:port), type pull
+
+
+Startup
+
+RFlow.run is the management process for the workflow
+
+computations = config.computations.map do |c|
+  instantiate_computation(c)
+  # Check for errors here, which would be evident if a computation couldn't be found/created
+  # Just creating single process ruby objects here to check for errors
+end
+
+computations.each do |c|
+  c.configure # with what????
+  # Still single ruby process to set and deconflict all the configuration parameters
+end
+
+computations.each do |c|
+  c.run
+end
+
+listen_for_management_events_from_old_computations
+listen_for_new_computation_registration

data/Rakefile

@@ -0,0 +1,16 @@
+require 'bundler'
+require 'rspec/core/rake_task'
+require 'rdoc/task'
+Bundler::GemHelper.install_tasks
+
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.verbose = true
+  t.rspec_opts = '--tty --color'
+end
+
+RDoc::Task.new do |rd|
+  rd.main = "README"
+  rd.rdoc_files.include("README", "lib/**/*.rb")
+  rd.rdoc_dir = File.join('doc', 'html')
+end
+

data/bin/rflow

@@ -0,0 +1,215 @@
+#!/usr/bin/env ruby
+
+# Will require rflow after option parsing to speed up a couple of
+# startup cases (version and help) that don't need it
+require 'optparse'
+
+options = {
+  :daemonize => true,
+  :startup_log_level => :INFO,
+  :extensions_file_paths => [],
+  :gems => []
+}
+
+option_parser = OptionParser.new do |opts|
+  opts.banner = <<EOB
+Usage: #{File.basename $0} [options] (start|stop|status|load)
+EOB
+
+  opts.on("-d", "--database DB", "Config database (sqlite) path (GENERALLY REQUIRED)") do |db|
+    options[:config_database_path] = File.expand_path(db)
+  end
+
+  opts.on("-c", "--config CONFIG", "Config file path (only valid for load)") do |config|
+    options[:config_file_path] = File.expand_path(config)
+  end
+
+  opts.on("-e", "--extensions FILE1[,FILE_N]", Array, "Extension file paths (will load)") do |extensions|
+    options[:extensions_file_paths] += extensions.map {|extension| File.expand_path(extension)}
+  end
+  
+  opts.on("-g", "--gems GEM1[,GEM_N]", Array, "Extension gems (will require)") do |gems|
+    options[:gems] += gems
+  end
+  
+  opts.on("-l", "--log LOGFILE", "Initial startup log file (in addition to stdout)") do |log|
+    options[:startup_log_file_path] = File.expand_path(log)
+  end
+
+  opts.on("-v", "--verbose [LEVEL]", [:DEBUG, :INFO, :WARN], "Control the startup log (and stdout) verbosity (DEBUG, INFO, WARN) defaults to INFO") do |level|
+    options[:startup_log_level] = level || :DEBUG
+  end
+
+  opts.on("-f", "Run in the foreground") do |f|
+    options[:daemonize] = false
+  end
+
+  opts.on_tail("--version", "Show RFlow version and exit") do
+    require 'rflow/version'
+    puts RFlow::VERSION
+    exit 0
+  end
+  
+  opts.on_tail("-h", "--help", "Show this message and exit") do
+    puts opts
+    exit 0
+  end
+  
+end
+
+begin
+  option_parser.parse!
+rescue Exception => e
+  STDERR.puts "Error processing arguments: #{e.class}: #{e.message}"
+  exit 1
+end
+
+# Now require rflow because the following parts of the startup require
+# pieces (usually RFlow::Configuration or RFlow.logger)
+require 'rflow'
+
+# Set up the startup logging, which is distinct from the runtime
+# logging that is defined in the config database.  The startup logging
+# will always go to STDOUT, as well as to the file specified with the
+# '-l' parameter
+startup_logger = Log4r::Logger.new 'startup'
+startup_logger.add Log4r::StdoutOutputter.new('startup_stdout', :formatter => RFlow::LOG_PATTERN_FORMATTER)
+startup_logger.level = Log4r::LNAMES.index options[:startup_log_level].to_s
+
+if options[:startup_log_file_path]
+  begin
+    startup_logger.add Log4r::FileOutputter.new('startup_file', :filename => options[:startup_log_file_path], :formatter => RFlow::LOG_PATTERN_FORMATTER)
+  rescue Exception => e
+    startup_logger.fatal "Log file '#{options[:startup_log_file_path]}' problem: #{e.message}"
+    exit 1
+  end
+end
+
+command = ARGV[0]
+unless ['start', 'stop', 'status', 'load'].include? command
+  startup_logger.fatal "Command needs to be one of [start|stop|status|load]\n#{option_parser.help}"
+  exit 1
+end
+
+if options[:config_file_path] && command != 'load'
+  startup_logger.fatal "Config file only valid for 'load' command"
+  exit 1
+end
+
+
+unless options[:config_database_path]
+  startup_logger.warn "Config database not specified, using default 'config.sqlite'"
+  options[:config_database_path] = File.expand_path(File.join(Dir.getwd, 'config.sqlite'))
+end
+
+
+# Set the standard logger to the startup one in the case that we need
+# to call into RFlow to check on or setup things, like the config
+# database.  We want those log messages to go to the startup log when
+# setting up.  The running log will transition to what is specified in
+# the config database
+RFlow.logger = startup_logger
+
+
+case command
+when 'load'
+  # Load the database with the config file, if it exists.  Will
+  # otherwise default values (not very useful)
+  if options[:config_file_path]
+    unless File.exist? options[:config_file_path]
+      startup_logger.fatal "Config file '#{options[:config_file_path]}' not found\n#{option_parser.help}"
+      exit 1
+    end
+    
+    unless File.readable? options[:config_file_path]
+      startup_logger.fatal "Config file '#{options[:config_file_path]}' not readable\n#{option_parser.help}"
+      exit 1
+    end
+  end
+  
+  if File.exist? options[:config_database_path]
+    startup_logger.fatal "Config database '#{options[:config_database_path]}' exists, exiting to prevent accidental overwrite from config file '#{options[:config_file_path]}'"
+    exit 1
+  end
+  
+  startup_logger.warn "Config database '#{options[:config_database_path]}' not found, creating"
+  begin
+    RFlow::Configuration::initialize_database(options[:config_database_path], options[:config_file_path])
+  rescue Exception => e
+    startup_logger.fatal "Error initializing configuration database: #{e.message}: #{e.backtrace.join "\n"}"
+    exit 1
+  end
+
+  startup_logger.warn "Successfully initialized database '#{options[:config_database_path]}' with '#{options[:config_file_path]}'"
+  exit 0
+end
+
+
+# Load the database config and start setting up environment
+begin
+  config = RFlow::Configuration.new(options[:config_database_path])
+rescue Exception => e
+  startup_logger.fatal "Error loading config database: #{e.class} - #{e.message}"
+  exit 1
+end
+
+Dir.chdir(File.dirname(options[:config_database_path]))
+Dir.chdir(config['rflow.application_directory_path'])
+pid = RFlow.running_pid_file_path?(config['rflow.pid_file_path'])
+
+case command
+when 'stop'
+  if pid
+    startup_logger.info "#{config['rflow.application_name']} running, process #{pid} found in #{File.expand_path(config['rflow.pid_file_path'])}, terminating"
+    # TODO: check if it actually shut down
+    Process.kill 'INT', pid
+  else
+    startup_logger.warn "#{config['rflow.application_name']} process not found in #{File.expand_path(config['rflow.pid_file_path'])}"
+    exit 1
+  end
+  exit 0
+
+when 'status'
+  unless pid
+    startup_logger.error "#{config['rflow.application_name']} process not found in #{File.expand_path(config['rflow.pid_file_path'])}"
+    exit 1
+  end
+  startup_logger.info "#{config['rflow.application_name']} running, process #{pid} found in #{File.expand_path(config['rflow.pid_file_path'])}"
+  exit 0
+
+when 'start'
+  if pid
+    startup_logger.error "#{config['rflow.application_name']} already running, process #{pid} found in #{File.expand_path(config['rflow.pid_file_path'])}"
+    exit 1
+  end
+end
+
+
+# We should have eliminated all commands but 'start' at this point
+
+# require all the gem extensions
+options[:gems].each do |extension_gem|
+  startup_logger.info "Requiring #{extension_gem}"
+  require extension_gem
+end
+
+
+# load all the file extensions
+options[:extensions_file_paths].each do |extensions_file_path|
+  startup_logger.info "Loading #{extensions_file_path}"
+  unless File.readable? extensions_file_path
+    startup_logger.fatal "Extensions file ('#{Dir.getwd}') '#{extensions_file_path}' not reabable\n#{option_parser.help}"
+    exit 1
+  end
+  load extensions_file_path
+end
+
+
+# Start the flow
+begin
+  RFlow.run options[:config_database_path], options[:daemonize]
+rescue Exception => e
+  startup_logger.fatal "Error running rflow: #{e.class}: #{e.message}"
+end
+
+__END__