fbp 0.1.0

data/lib/fbp/fpb-thread-pool.rb

@@ -0,0 +1,91 @@
+require "thread-pool/thread-pool.rb"
+
+module Fbp
+
+=begin rdoc
+=== Description
+The Node pool class is built upon Kim Burgestrand Pool class which provides thread pool management
+for Ruby.  The Node pool provides a standard set of module functions that allow for setting the
+number of threads in the Pool and to shutdown the Pool.
+=end
+	class Node_pool #:nodoc:
+
+		attr_reader :pool  #:nodoc:
+		attr_reader :num_threads  #:nodoc:
+
+		def initialize (num_threads = 10) #:nodoc:
+			@num_threads = num_threads
+			@pool = nil
+		end
+
+		def make_pool   #:nodoc:
+		  return false if !@pool.nil?
+			@pool = Pool.new(@num_threads)
+		end
+
+		def num_threads=(num_threads) #:nodoc:
+			@num_threads = num_threads if @pool.nil?
+		end
+
+		def shutdown   #:nodoc:
+			@pool.shutdown if !@pool.nil?
+			@pool = nil
+		end
+
+	end
+	
+	@@node_pool = Node_pool.new  #:nodoc:
+
+=begin rdoc
+The  num_threads= sets the number of threads that will be used by the thread pool
+that will be used for any Fbp network.  The default value is 10 threads.  This
+function <b>only</b> works <b>before></b>  calling Fbp::make_pool.  After that this
+function will do nothing
+=== Parameters
++num_threads+ - The number of threads that should be used in the Thread Pool
+=end
+  public
+	def self.num_threads= (num_threads)
+		@@node_pool.num_threads = num_threads
+	end
+
+=begin
+Returns that number of threads that are in the Pool
+=end
+  public
+	def self.num_threads
+		@@node_pool.num_threads
+	end
+
+=begin
+This function will create the thread pool that will be used by any Fbp network.
+This function <b>must</b> be called before executing any Fbp network.
+=end
+  public
+	def self.make_pool
+		@@node_pool.make_pool
+	end
+
+=begin
+This function will return back if Fbp::make_pool has been called and a Pool has
+been created.
+=end
+  public
+	def self.has_pool
+		!@@node_pool.pool.nil?
+	end
+
+=begin
+This function will stop all running threads in the Fbp pool.  Calling this function
+will kill all Fbp processing.
+=end
+  public
+	def self.shutdown
+		!@@node_pool.shutdown
+  end
+
+  protected
+  def self.schedule(&block)  #:nodoc:
+    @@node_pool.pool.schedule { block.call } if !@@node_pool.pool.nil?
+  end
+end

data/lib/fbp/selector-node.rb

@@ -0,0 +1,131 @@
+require "fbp/constants"
+
+module Fbp
+
+=begin rdoc
+<b>Selector_node</b>
+=== Description
+The Selector node class provides for comparing the value(s) in an IP and with set value(s) and the
+out come of the comparison will determine which output channel the IP will be sent to and thus splitting the
+incoming stream to  different downstream nodes.
+===
+The Selector_node class requires an IIP of the form:
+
+Selector
+	{:key => {:comparison => comparison, :value => value, :match => :output_name, :reject => :output_name}}
+
+	:key
+    This is the symbol that will be used to match against keys in an IP
+
+	:comparison
+    This is a constant define in constants.rb for comparison operations
+
+	:value
+    This is the value that will be used to check against the corresponding value in an IP
+
+	:match
+    The name of the output queue that will be written to for IPs that match the selector
+
+	:reject
+    The name of the output queue that will be written to for IPs that do not match the selector
+
+There may be multiple selectors with different key values that will match different keys in an IP
+
+When an IP arrives at a Selector_node, the do_node_work will match keys in the IP with keys in the
+selectors hash.  If there is a match, then the comparison stipulated in the selector will be done on
+the value in the IP against the value in the selector.  If the comparison is a match then the IP
+will be written to the output named in the :match item otherwise it will be written to the output
+named in the :reject item.
+
+If an IP does not have any keys that match any selectors the IP will be written to the output named :output
+=end
+	class Selector_node < Node
+=begin rdoc
+When creating a new Selector_node instance one can provide an
+array of selector records.
+=end
+		def initialize(selector = nil)
+			super()
+			@options[:selectors] = selector if !selector.nil?
+		end
+
+=begin rdoc
+Checks to see if this Selector_node instance has an array of Selector records.
+If it does have an array of Selector records then true will be returned
+otherwise false.  A Selector_node instance will not execute until it has
+an array of Selector records
+=end
+		def is_ready_to_run?
+			@options.has_key? :selectors
+		end
+
+=begin rdoc
+Each IP that is presented for processing will be compared to the array of
+selector records associated with this instance.  The selector records
+determine how the IP will be output.  If there is a match, then the IP
+will be written to the output channel defined for a match in the selector record.
+If it does not match the the IP will be written to the output channel defined
+for a reject in the selector record.
+
+If the incoming IP does not have any of the keys defined in the selector
+records the IP will be written to the output channel.
+=end
+		def do_node_work(args)
+			return false if args.has_key? :completed
+			return true if args.has_key? :start
+			key_match = false
+
+			selectors = @options[:selectors]
+
+
+			multi_ip =  args.has_key? :ips
+			ips = multi_ip ? args[:ips] : [args]
+			ips.each do |ip|
+
+				selectors.each do |selector_hash|
+
+					selector_hash.each  do |key, h|
+						if ip.has_key? key
+							key_match = true
+							value = ip[key]
+							compare_value = h[:value]
+							comparision = h[:comparison]
+
+							pass = case comparision
+								       when NOT_EQUAL_COMPARE
+									       compare_value != value
+								       when EQUAL_COMPARE
+									       compare_value == value
+								       when  GREATER_THAN
+									       value > compare_value
+								       when GREATER_THAN_OR_EQUAL
+									       value >= compare_value
+								       when  LESS_THAN
+									       value <  compare_value
+								       when LESS_THAN_OR_EQUAL
+									       value <= compare_value
+								       when CONTAINS
+									       value.to_s.include?(compare_value.to_s)
+								       when DOES_NOT_CONTAINS
+									       !value.to_s.include?(compare_value.to_s)
+								       when STARTS_WITH
+									       value.start_with?(compare_value.to_s)
+								       when ENDS_WITH
+									       value.end_with?(compare_value.to_s)
+								       when MATCHES
+									       !value.match(compare_value.to_s).nil?
+								       else
+									       false
+							       end
+							output_name = pass ? h[:match] : h[:reject]
+							write_to_output(ip, output_name)  if !output.nil?
+						end
+					end
+				end
+  		end
+			write_to_output(args, :output) if !key_match
+			true
+		end
+  end
+  
+end

data/lib/fbp/sort-node.rb

@@ -0,0 +1,68 @@
+module Fbp
+=begin
+=== Description
+The Sort node will take its input IP and sort that data according to a set of hash keys that
+specify which data should be sorted.
+
+=== Discussion
+This implementation requires that all of the data to be sorted be available.  This is NOT a scalable
+solution.
+
+I have read some of the literature on sorting data streams but currently I do not have a ready
+solution.  For now this will have to serve.
+=end
+  class Sort_node < Node
+=begin rdoc
+When creating a new Sort_node instance one can provide an array of
+symbols which would be used to match keys in an incoming IP. The
+order of the keys in the array determine the sort order.  The
+first key in the array will be used to sort the data then the
+subsequent keys if any.
+=end
+		def initialize(sort_keys = nil)
+			super()
+			@options[:sort_keys] = sort_keys
+      write_to_input({:begin_transaction => true})
+		end
+=begin rdoc
+Checks to see if this Sort_node instance has any sort keys set.
+True will be returned if any keys are set, false otherwise.
+=end
+		def is_ready_to_run?
+			@options.has_key? :sort_keys
+		end
+
+=begin rdoc
+The Sort_node assumes that will will receive an IP with a :ips key
+which contains all of the IPs that need to be sorted.  If the :ips key
+is not present then the IP is sent unchanged to the down stream node.
+
+The IPs will be sorted based upon the sort_keys parameter.  For each
+sort_key all of the IPs will be sorted based upon the value in the IP
+that matches the sort_key.  If an IP does not have the sort_key then
+that IP is excluded from the sort and is disregarded.
+
+Once the IPs have been sorted the IP has its :ips value replaced by
+the sorted data and the IP is then sent to the down stream node.
+=end
+		def do_node_work(args)
+      args.delete :completed
+			ip_array = args[:ips]
+      return super(args) if ip_array.nil?
+
+      sort_keys_array = @options[:sort_keys].respond_to?('each') ? @options[:sort_keys] : [@options[:sort_keys]]
+
+      data_to_sort = Array.new
+
+      ip_array.each {|h| sort_keys_array.each  {|k| data_to_sort << h if h.has_key?(k) && !data_to_sort.include?(h)}}
+      sort_keys_array.each {|k| data_to_sort.sort! {|a,b| a[k] <=> b[k]}}
+      data_to_sort.each {|a| write_to_output(a)}
+
+			# Update the input IP with the sorted list of IPs
+			args[:ips] = data_to_sort
+			super(args)
+      false
+		end
+	end
+
+end

data/lib/fbp/text_file_reader_node.rb

@@ -0,0 +1,80 @@
+module Fbp
+=begin
+=== Description
+The Text file reader node takes in a file path as an IIP and will open that
+file and read a line at a time and package that data as an IP and send that
+data to the downstream node.
+
+=== Discussion
+The Test_file_reader_node requires the file to be read exists before running this
+node.
+=end
+	class Test_file_reader_node < Node
+=begin rdoc
+When creating a new Test_file_reader_node instance one can provide
+a string which is the full path to the file to be read.
+=end
+		def initialize(file_name = nil)
+			super()
+      @options[:requires_input] = false
+			@options[:file_name] = file_name if !file_name.nil?
+			@file = nil
+		end
+
+=begin rdoc
+Checks to see if this Test_file_reader_node instance has
+a file_name set.  If so then true will be returned otherwise
+false.
+=end
+	  def is_ready_to_run?()
+			@options.has_key? :file_name
+		end
+
+=begin
+Once this instance is executed, it will open the file
+specified in the  :file_name option and read a line
+at a time.  A line is defined as a series of characters
+up to a return character.
+
+Once a line is read, an IP is made with that data
+keys with :data and sent to the down stream node.
+
+When the EOF is reached the file will be closed.
+=end
+		def do_node_work(args)
+
+      if args.has_key? :completed
+        @file.close  if !@file.nil?
+        @file = nil
+        return false
+      end
+
+      if args.has_key? :continue
+
+        file_line = nil
+        close_file = false
+        @file = File.new(@options[:file_name], 'r') if @file.nil?
+
+
+			  begin
+				  file_line = @file.gets
+          close_file = true if file_line.nil?
+          close_file = true if @file.eof?
+			  rescue
+				  close_file = true
+			  end
+
+			  write_to_output({:data => file_line}) if !file_line.nil?
+
+			  if close_file
+				  @file.close
+				  @file = nil
+          return false
+        end
+      end
+			
+			true
+		end
+
+	end
+end

data/lib/fbp/text_file_writer_node.rb

@@ -0,0 +1,65 @@
+module Fbp
+=begin
+=== Description
+The Text file writer node takes in a file path as an IIP and will open that
+file and will take in IPs and write the value of the :data item in the IP to
+the file.
+
+== Discussion
+The Text_file_writer_node does allow for setting the file open mode by having
+an IIP with a :file_open_mode key specifying the open mode.  Setting this value
+using an IIP allows for setting the open mode to a+ so appending instead of the
+the default which to overwrite the file using "w"
+=end
+	class Text_file_writer_node < Node
+=begin rdoc
+When creating a new Text_file_writer_node instance one can provide
+a string which is the full path to the file to be written.
+=end
+		def initialize(file_name = nil)
+			super()
+			@options[:file_name] = file_name if !file_name.nil?
+			@options[:file_open_mode] = 'w'
+			@file = nil
+		end
+=begin rdoc
+Checks to see if this Text_file_writer_node instance has
+a file_name and file_open_mode set.  If so then true will be
+returned otherwise false.
+=end
+		def is_ready_to_run?()
+			@options.has_key?(:file_name) && @options.has_key?(:file_open_mode)
+		end
+=begin rdoc
+When an IP is received the IP is checked to see if it has a :data key.
+If it does then that data is written to the file specified by the
+:file_name option.  This will continue until an IP is sent with the
+:completed  key at which will cause the file to be closed.
+=end
+		def do_node_work(args)
+
+      if args.has_key? :completed
+        @file.close if !@file.nil?
+        @file = nil
+        return false
+      end
+
+      if args.has_key? :data
+        data = args[:data]
+
+        if data.nil?
+          @file.close if !@file.nil?
+          @file = nil
+          return false
+        end
+
+        @file = File.new(@options[:file_name], @options[:file_open_mode]) if @file.nil?
+        @file.write(data)
+        @file.flush
+      end
+
+			true
+		end
+
+	end
+end

data/lib/fbp/version.rb

@@ -0,0 +1,3 @@
+module Fbp
+  VERSION = "0.1.0"  #:nodoc:
+end

data/lib/thread-pool/thread-pool.rb

@@ -0,0 +1,150 @@
+# Ruby Thread Pool
+# ================
+# A thread pool is useful when you wish to do some work in a thread, but do
+# not know how much work you will be doing in advance. Spawning one thread
+# for each task is potentially expensive, as threads are not free.
+#
+# In this case, it might be more beneficial to start a predefined set of
+# threads and then hand off work to them as it becomes available. This is
+# the pure essence of what a thread pool is: an array of threads, all just
+# waiting to do some work for you!
+#
+# Prerequisites
+# -------------
+
+# We need the [Queue](http://rdoc.info/stdlib/thread/1.9.2/Queue), as our
+# thread pool is largely dependent on it. Thanks to this, the implementation
+# becomes very simple!
+require 'thread'
+
+# Public Interface
+# ----------------
+
+# `Pool` is our thread pool class. It will allow us to do three operations:
+#
+# - `.new(size)` creates a thread pool of a given size
+# - `#schedule(*args, &job)` schedules a new job to be executed
+# - `#shutdown` shuts down all threads (after letting them finish working, of course)
+class Pool
+
+	# ### initialization, or `Pool.new(size)`
+	# Creating a new `Pool` involves a certain amount of work. First, however,
+	# we need to define its’ `size`. It defines how many threads we will have
+	# working internally.
+	#
+	# Which size is best for you is hard to answer. You do not want it to be
+	# too low, as then you won’t be able to do as many things concurrently.
+	# However, if you make it too high Ruby will spend too much time switching
+	# between threads, and that will also degrade performance!
+	def initialize(size)
+		# Before we do anything else, we need to store some information about
+		# our pool. `@size` is useful later, when we want to shut our pool down,
+		# and `@jobs` is the heart of our pool that allows us to schedule work.
+		@size = size
+		@jobs = Queue.new
+
+		# #### Creating our pool of threads
+		# Once preparation is done, it’s time to create our pool of threads.
+		# Each thread store its’ index in a thread-local variable, in case we
+		# need to know which thread a job is executing in later on.
+		@pool = Array.new(@size) do |i|
+			Thread.new do
+				Thread.current[:id] = i
+
+				# We start off by defining a `catch` around our worker loop. This
+				# way we’ve provided a method for graceful shutdown of our threads.
+				# Shutting down is merely a `#schedule { throw :exit }` away!
+				catch(:exit) do
+					# The worker thread life-cycle is very simple. We continuously wait
+					# for tasks to be put into our job `Queue`. If the `Queue` is empty,
+					# we will wait until it’s not.
+					loop do
+						# Once we have a piece of work to be done, we will pull out the
+						# information we need and get to work.
+						job, args = @jobs.pop
+						job.call(*args)
+					end
+				end
+			end
+		end
+	end
+
+	# ### Work scheduling
+
+	# To schedule a piece of work to be done is to say to the `Pool` that you
+	# want something done.
+	def schedule(*args, &block)
+		# Your given task will not be run immediately; rather, it will be put
+		# into the work `Queue` and executed once a thread is ready to work.
+		@jobs << [block, args]
+	end
+
+	# ### Graceful shutdown
+
+	# If you ever wish to close down your application, I took the liberty of
+	# making it easy for you to wait for any currently executing jobs to finish
+	# before you exit.
+	def shutdown
+		# A graceful shutdown involves threads exiting cleanly themselves, and
+		# since we’ve defined a `catch`-handler around the threads’ worker loop
+		# it is simply a matter of throwing `:exit`. Thus, if we throw one `:exit`
+		# for each thread in our pool, they will all exit eventually!
+		@size.times do
+			schedule { throw :exit }
+		end
+
+		# And now one final thing: wait for our `throw :exit` jobs to be run on
+		# all our worker threads. This call will not return until all worker threads
+		# have exited.
+		@pool.map(&:join)
+	end
+end
+
+# Demonstration
+# -------------
+# Running this file will display how the thread pool works.
+if $0 == __FILE__
+	# - First, we create a new thread pool with a size of 10. This number is
+	#   lower than our planned amount of work, to show that threads do not
+	#   exit once they have finished a task.
+	p = Pool.new(10)
+
+	# - Next we simulate some workload by scheduling a large amount of work
+	#   to be done. The actual time taken for each job is randomized. This
+	#   is to demonstrate that even if two tasks are scheduled approximately
+	#   at the same time, the one that takes less time to execute is likely
+	#   to finish before the other one.
+	20.times do |i|
+		p.schedule do
+			sleep rand(4) + 2
+			puts "Job #{i} finished by thread #{Thread.current[:id]}"
+		end
+	end
+
+	# - Finally, register an `at_exit`-hook that will wait for our thread pool
+	#   to properly shut down before allowing our script to completely exit.
+	at_exit { p.shutdown }
+end
+
+# License (X11 License)
+# =====================
+#
+# Copyright (c) 2012, Kim Burgestrand <kim@burgestrand.se>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.