fbp 0.1.0

data/lib/fbp/aggregator-node.rb

@@ -0,0 +1,31 @@
+module Fbp
+=begin
+=== Description
+The Aggregator node puts itself into a transaction which will cause all of
+the incoming IPs to be aggregated until the upstream node has completed then
+the aggregated IPs will be sent on to the down stream node(s)
+=end
+  class Aggregator_node < Node
+=begin rdoc
+When a Aggregator_node is created, it is immediately put into a transaction.
+This allows the  Aggregator_node to aggregate all of the IPs that come to it
+until the up stream node completes its work
+=end
+    def initialize()
+      super()
+      write_to_input({:begin_transaction => true})
+    end
+
+=begin rdoc
+Once the up stream node completes.  The transaction will end and
+all of the IPs that had been accumulated during the transaction will
+be sent to the down stream node.
+=end
+    def do_node_work(args)
+      args.delete :completed
+      write_to_output(args)
+      false
+    end
+
+  end
+end

data/lib/fbp/assign-node.rb

@@ -0,0 +1,41 @@
+module Fbp
+=begin rdoc
+== Description
+The Assign_node will take its incoming IPs and add items to the hash
+before passing it one to its down stream node(s)
+
+== Discussion
+The Assign_node takes an optional IIP of the form <tt>{:additional_args =>{}}</tt>
+
+The hash contains the items that will be added to every incoming
+IP sent to this node. The IIP is optional with this node.  If no
+additional arguments are given to this node then it will simply write all
+of its in coming IPs to its output.
+=end
+	class Assign_node < Node
+
+=begin rdoc
+When creating a new Assign_node instance one can optionally provide a hash
+containing the items to be added to every in coming IP
+=end
+		def initialize(additional_args = nil)
+			super()
+			@options[:additional_args] = additional_args if !additional_args.nil?
+		end
+
+=begin rdoc
+If an IIP has set the :additional_args key in options, then those items
+will be appended to every incoming IP.  If the :additional_args key has
+not been set then the IP will be passed downstream untouched.
+
+If the IP has an :ips key then each of those IP in the array of IPs
+will be modified as described above.
+=end
+		def  do_node_work(args)
+			ips =  args.has_key? :ips ? args[:ips] : [args]
+			ips.each {|ip| ip.merge!(@options[:additional_args]) if @options.has_key? :additional_args}
+			super(args)
+		end
+
+	end
+end

data/lib/fbp/concatenate-node.rb

@@ -0,0 +1,66 @@
+
+module Fbp
+
+=begin
+=== Description
+The Concatenate_node takes a list of keys as it IIP and as IPs are sent
+to this node, it will check to see if th IP has any of the keys specified
+in the IIP.  If there is a match then the IP is merged into a single IP.
+Once all of the keys that are specified in the IIP are in the merged IP,
+it is sent to the down stream node.
+
+An example of the usage of this node would be if the down stream node
+was an encryption node.  The encryption node might require an IIP with
+keys of key_data, salt, mode, and padding.  This concatenate node
+could be used to gather all of that dat into an IIP to send to the
+encryption node.
+=end
+	class Concatenate_node < Node
+
+=begin rdoc
+When creating a new Concatenate_node instance one can provide an array of
+keys that will be used to determine if all of the necessary data has
+been cached before creating a new IP to send down stream.
+=end
+		def initialize(keys = nil)
+			super()
+			@options[:keys] = keys if !keys.nil?
+			@output_ip = Hash.new
+		end
+
+=begin rdoc
+This is called to see if this node has been parameterized with the
+required keys needed for this node to do its work
+=end
+		def is_ready_to_run?
+			@options.has_key? :keys
+		end
+
+=begin rdoc
+This will do the work of determining if all of the required data
+has been acquired before creating an IP to send down stream.
+=end
+		def do_node_work(args)
+			# Merge the IP into Concatenate_node the output hash
+			ips = nil
+
+			if args.has_key? :ips
+				ips =  args[:ips]
+				temp = args.clone
+				temp.delete(:ips)
+				ips << temp if !temp.empty?
+			else
+				ips = [args]
+			end
+
+			ips.each {|ip| @output_ip.merge!(ip)}
+			# Check to see if the output_ip hash has all of the required keys.
+			# If it does, then write the output_ip hash to the output queue
+			if @options[:keys] == (@options[:keys] & @output_ip.keys)
+				write_to_output(@output_ip)
+				@output_ip.clear
+			end
+			true
+		end
+	end
+end

data/lib/fbp/constants.rb

@@ -0,0 +1,13 @@
+module Fbp
+	NOT_EQUAL_COMPARE = 0
+	EQUAL_COMPARE = 1
+	GREATER_THAN = 2
+	GREATER_THAN_OR_EQUAL = 3
+	LESS_THAN = 4
+	LESS_THAN_OR_EQUAL = 5
+	CONTAINS = 6
+	DOES_NOT_CONTAINS = 7
+	STARTS_WITH = 8
+	ENDS_WITH = 9
+	MATCHES = 10
+end

data/lib/fbp/counter-node.rb

@@ -0,0 +1,50 @@
+module Fbp
+=begin
+=== Description
+The counter node counts either incoming IPs or IPs that
+contain a certain key and then adds that count data to
+the incoming IPs
+=end
+	class Counter_node < Node
+
+=begin rdoc
+When creating a new Counter_node instance one can optionally provide a
+symbol which would be used to match a key in an incoming IP.  The IP
+would then only be counted if the IP contained that key.
+=end
+		def initialize(key = nil)
+			super()
+			@options[:key] = key if !key.nil?
+		end
+
+=begin rdoc
+When an IP is received if a key has been set then the IP will be
+checked to see if it contains that key.  If so then it is counted.
+
+If the incoming IP has the :ips key then the nested IPs will be
+search for the matching key and counted if found.
+
+In either a single IP or an IP with the :ips key is received and
+no key has been set, then all IPs will be counted.
+=end
+		def do_node_work(args)
+			counter = 0
+			ips =  args.has_key? :ips ? args[:ips] : [args]
+
+			key = nil
+			key = @options[:key] if @options.has_key?(:key)
+
+			ips.each do |ip|
+				if key.nil?
+					counter += 1
+				else
+					counter += 1 if ip.has_key?(key)
+				end
+			end
+
+			args[:counter] = counter
+			super(args)
+		end
+
+	end
+end

data/lib/fbp/decode-node.rb

@@ -0,0 +1,34 @@
+require 'json'
+
+module Fbp
+
+=begin rdoc
+=== Description
+The Decode node takes it incoming IP that is an JSON encoded IP and decodes it
+into a Ruby hash to be sent on to the downstream node.
+
+This node would be used when receiving an IP from another machine or another process
+on the same machine.
+=end
+	class Decode_node < Node
+
+=begin rdoc
+When an IP comes in it will be checked for the :json key.  If is exists
+then the data will be extracted and the reconstituted data will be sent
+to the down stream node.  If the IP does not have the :json key it will
+be sent to the down stream node as is.
+=end
+		def do_node_work(args)
+      return super(args) if !args.has_key?(:json)
+			decoded_obj = nil
+			begin
+				decoded_obj = JSON.parse(args)
+			rescue
+				decoded_obj = nil
+			end
+			super(decoded_obj)
+		end
+		
+	end
+
+end

data/lib/fbp/encode-node.rb

@@ -0,0 +1,31 @@
+require 'json'
+
+module Fbp
+=begin
+=== Description
+The Encode node takes it incoming IP and encodes the IP into a JSON string.  This data
+is then added to the incoming IP and sent to the downstream node.
+
+This node would be used when sending IPs to another machine or another process
+on the same machine.
+=end
+	class Encode_node < Node
+
+=begin rdoc
+When an IP comes in it will be converted into a JSON text object.
+That text object will then be append to the incoming IP with the
+:json key and sent to the down stream node.
+=end
+		def do_node_work(args)
+			encoded_obj = nil
+			begin
+				encoded_obj = args.to_json
+			rescue
+				encoded_obj = nil
+			end
+			args.merge!({:json => encoded_obj}) if !encoded_obj.nil?
+			super(args)
+		end
+	end
+
+end

data/lib/fbp/flow-node.rb

@@ -0,0 +1,369 @@
+
+module Fbp
+  private
+  def self.add_node_to_flow_desc(node, flow_desc) #:nodoc:
+    return if node.nil? || flow_desc.nil?
+
+    class_symbol =  node.class.name.to_sym
+
+    # Add the input node first
+    node_record = {:node => class_symbol, :options => node.options, :object_id => node.object_id, :input => nil}
+    flow_desc << node_record
+
+    # Recursively add all the nodes in the network of nodes
+    node.output.each_value do |channel|
+      next if channel.empty?
+      channel.each {|a_node| add_node_to_flow_desc(a_node, flow_desc)}
+    end
+  end
+
+  private
+  def self.set_input_array(node, lookup_data, flow_desc) #:nodoc:
+    return if node.nil? || lookup_data.nil? || flow_desc.nil?
+    node_index = lookup_data[node.object_id]
+
+    node.output.each_key do |key|
+      node_array = node.output[key]
+      next if node_array.nil?
+      node_array.each do |n|
+        n_index = lookup_data[n.object_id]
+        fr = flow_desc[n_index]
+        input_array = fr[:input]
+        input_array = Array.new if input_array.nil?
+        input_array << {node_index => key}
+        fr[:input] = input_array
+        set_input_array(n, lookup_data, flow_desc)
+      end
+    end
+  end
+
+=begin rdoc
+Given a Fbp node that is the beginning of a network of Fbp nodes, the Fbp::make_flow_description
+function will create a Flow description record.
+
+== Discussion
+<b>Flow Description Record</b>
+ [{:node => symbol_of_the_Fbp_node_class_name, :options =>{class_specific_hash}, :input => nil || [{index => output_channel_name_symbol}]}]
+
++:node+     This is a symbol that is the name of the Fbp node to be instantiated.
+
++:options+  This is a hash which is the IIP needed to parameterize the newly instantiated node.
+
++:input+    This is either nil or an array of hashes.  Each hash is keyed by the index in the array of nodes that are described by the Flow Description Record with the value being the symbol name of the output channel from which to get the input for the node.
+=end
+  public
+  def self.make_flow_description(node)
+    return nil if node.nil?
+    flow_desc = Array.new
+
+    # Add the nodes to flow_desc
+    add_node_to_flow_desc(node, flow_desc)
+
+    # Make a hash keyed by an object_id with the value of the index of the object in the flow_desc array
+    lookup_data = Hash.new
+    flow_desc.each_index do |idx|
+      lookup_data[flow_desc[idx][:object_id]] = idx
+    end
+
+    set_input_array(node, lookup_data, flow_desc)
+    flow_desc
+  end
+
+=begin rdoc
+Given a Flow Description Record, and a full path to a file, write out the
+Flow Description Record to the file system.
+
+See the  make_flow_description documentation for a discussion of what
+a Flow Description Record.
+=end
+  public
+  def self.write_flow_desc_to_file(flow_desc, file_path)
+    return false if flow_desc.nil? || file_path.nil?
+    
+    flow_data = Marshal.dump(flow_desc)
+
+    result = false
+    begin
+      File.open(file_path, 'w') {|f| f.write(flow_data)}
+      result = true
+    rescue
+      result = false
+    end
+
+    result
+  end
+
+=begin  rdoc
+Given a full path to a file that contains A previously written out
+Flow Description Record, this function will read in that file and
+create a Flow Description Record from the data.
+
+See the  make_flow_description documentation for a discussion of what
+a Flow Description Record.
+=end
+  public
+  def self.read_flow_desc_from_file(file_path)
+    return nil if file_path.nil?
+    encoded_data = nil
+    
+    begin
+      File.open(file_path, 'r') {|f| encoded_data = f.read()}
+    rescue
+      encoded_data = nil
+    end
+
+    return nil if encoded_data.nil?
+
+    Marshal.load(encoded_data)
+  end
+
+=begin rdoc
+== Description
+The flow node provides the ability of taking a network of Fbp nodes and wrap that network in a
+single node.  This is done by creating a Flow description record.  A Flow description record
+is a data structure that describes an entire network of Fbp noes.  Given a Flow description record
+the Flow node class will create the entire network and then wrap that network so that writing to the
+input of a Flow node instance will forward that input to the first node in the created network.
+The output of the created network will be forwarded to the output of the Flow node instance.
+
+There are module functions associated with the Flow node class. These functions provide the ability
+to take a Fbp network and create a Flow description record that describes the network. There are
+also functions that take a Flow description record and write them to the file system and read a
+file with a Flow description record and re-create the Flow description record.
+
+== Discussion
+A Flow description record is an Array of Hashes with the following keys
+
+<b>Flow Description Record</b>
+ [{:node => symbol_of_the_Fbp_node_class_name, :options =>{class_specific_hash}, :input => nil || [{index => output_channel_name_symbol}]}]
+
++:node+     This is a symbol that is the name of the Fbp node to be instantiated.
+
++:options+  This is a hash which is the IIP needed to parameterize the newly instantiated node.
+
++:input+    This is either nil or an array of hashes.  Each hash is keyed by the index in the array of nodes that are described by the Flow Description Record with the value being the symbol name of the output channel from which to get the input for the node.
+
+=== Example Flow Description Record
+The following is a simple network that will read in a file and split the file into two based upon a selector.
+The two outputs are sorted using the Sort Node and the output of the two different data streams and then writen
+out to different two files.
+
+                              ======================================================
+                              | {:selectors=>[{:data=>{:comparison=>Fbp::CONTAINS, |
+ ==============               |      {:comparison=>Fbp::CONTAINS, :value=>"Magic", |
+ | ~/input.txt |===           |        :match=>:magic, :reject=>:output}}]         |
+ ==============   |           ======================================================
+                  | (IIP)                                   | (IIP)
+                  \/                                        \/
+ ==============================                    ======================
+ |                            |  output to input   |                    |
+ | Fbp::Test_file_reader_node |==================> | Fbp::Selector_node |
+ |                            |                    |                    |
+ ==============================                    ======================
+                                                     |     |
+                     output to input                 |     |
+ =====================================================     |
+ |                     magic to input                      |
+ | =========================================================
+ | |
+ | |
+ | |
+ | |  =======================     ================================================================
+ | |  | {:sort_keys=>:data} |     | {:file_name=>"/Users/local/magic.txt", :file_open_mode=>"w"} |
+ | |  =======================     ================================================================
+ | |           | (IIP)                                  | (IIP)
+ | |           \/                                       \/
+ | |   ===================                  =============================
+ | ==> | "Fbp::Sort_node |=================>| Fbp::Text_file_writer_node |===> nil
+ |     ===================                  =============================
+ |
+ |     =======================    =================================================================
+ |     | {:sort_keys=>:data} |    | {:file_name=>"/Users/local/muggle.txt", :file_open_mode=>"w"} |
+ |     =======================    =================================================================
+ |              | (IIP)                                 | (IIP)
+ |              \/                                      \/
+ |     ===================                 ==============================
+ ====> | "Fbp::Sort_node |================>| Fbp::Text_file_writer_node | ===> nil
+       ===================                 ==============================
+
+The beginning node in this network is the Fbp::Test_file_reader_node node.  If that node is used to call
+The Fbp::make_flow_description function, the following will be returned.
+
+ [
+	{:node=>:"Fbp::Test_file_reader_node",
+	:options=>{:file_name=>"/Users/local/input.txt"},
+	:input=>nil},
+
+	{:node=>:"Fbp::Selector_node",
+	:options=>{:selectors=>[{:data=>{:comparison=>6, :value=>"Magic", :match=>:magic, :reject=>:output}}]},
+	:input=>[{0=>:output}]},
+
+	{:node=>:"Fbp::Sort_node",
+	:options=>{:sort_keys=>:data},
+	:input=>[{1=>:output}]},
+
+	{:node=>:"Fbp::Text_file_writer_node",
+	:options=>{:file_name=>"/Users/local/muggle.txt", :file_open_mode=>"w"},
+	:input=>[{2=>:output}]},
+
+	{:node=>:"Fbp::Sort_node",
+	:options=>{:sort_keys=>:data},
+	:input=>[{1=>:magic}]},
+
+	{:node=>:"Fbp::Text_file_writer_node",
+	:options=>{:file_name=>"/Users/local/magic.txt", :file_open_mode=>"w"},
+	:input=>[{4=>:output}]}
+ ]
+
+There are some important things to note about this Flow Description Record.
+Each node is represented by a Hash.  That hash has three items as noted
+previously. The order of the hashes in the array is important.  It allows
+the :input key to be based upon the index of the node from which the
+referring node will receive its input.
+
+For example the  Fbp::Selector_node which is the second node in the array
+will receive its input from the Fbp::Test_file_reader_node node which is
+the first node in the array.  This is denoted by the :input => [{0=>:output}]
+key value pair hash that defines the Fbp::Selector_node.  This reads as
+<em>The input of the Fbp::Selector_node comes form the node defined in
+the zeroth position of the Flow Description Record and takes its input
+from the output channel of that node</em>
+
+=== Future Development
+Many FBP solutions have a visual configuration tool to form networks
+from existing nodes.  The Flow node could be used in conjunction with
+a drawing tool to allow for non-programmer development of networks.
+This is an area that should be pursued.
+=end
+	class Flow_node < Node
+
+=begin rdoc
+When creating a new Flow_node instance one can optionally provide a
+flow description record.
+=end
+		def initialize(flow_desc = nil)
+			super()
+			@options[:flow_desc] = flow_desc
+			@flow_created = false
+			@node_objects = Array.new
+		end
+
+=begin rdoc
+Checks to see if this Flow_node instance has a flow description record.
+If the does have a flow description record then true will be returned
+otherwise false.  A Flow_node instance will not execute until it has a
+low description record.
+=end
+		def is_ready_to_run?
+			@options.has_key? :flow_desc
+    end
+
+=begin rdoc
+This will write to the first object in the network being serviced by this
+Flow_node instance.
+
+=== Discussion
+Should a way be provided to set which node in the network is the 'first'
+node?
+=end
+    def write_to_input(obj)
+      node =  @node_objects.first
+      node.input_queue << obj if !node.nil? && !obj.nil?
+    end
+
+=begin rdoc
+This will forward the register request to the last node in the network
+being serviced by this Flow_node instance.
+
+=== Discussion
+Should a way be provided to set which node in the network is the 'last'
+node?
+=end
+		def register_for_output_from_node(node, output_channel = :output)
+      last_name =  @node_objects.last
+      last_name.register_for_output_from_node(node, output_channel) if !last_name.nil?  &&  node.nil?
+    end
+
+=begin rdoc
+This will forward the wait_until_completed method to the last node in the network
+being serviced by this Flow_node instance.
+
+=== Discussion
+Should a way be provided to set which node in the network is the 'last'
+node?
+=end
+    def wait_until_completed
+     node =  @node_objects.last
+     node.wait_until_completed if !node.nil?
+    end
+
+=begin rdoc
+This will forward the execute method to the first node in the network
+being serviced by this Flow_node instance.
+
+=== Discussion
+Should a way be provided to set which node in the network is the 'first'
+node?
+=end
+		def execute
+      make_flow if !@flow_created
+      node =  @node_objects.first
+      node.execute if !node.nil?
+    end
+
+=begin rdoc
+This will forward the do_node_work method to the first node in the network
+being serviced by this Flow_node instance.
+
+=== Discussion
+Should a way be provided to set which node in the network is the 'first'
+node?
+=end
+    def do_node_work(args)
+      node =  @node_objects.first
+      node.do_node_work(args) if !node.nil?
+    end
+
+    private
+		def make_flow  #:nodoc:
+			return true if @flow_created
+
+			flow_desc =  @options[:flow_desc]
+			return false if !flow_desc.respond_to?(:each_index)
+
+			# This needs to be done in two passes.  One to create the
+			# Node objects and then to connect them.  The order in the
+			# flow_desc determines how the Node object are to be connected
+			# so create an array to hold the newly created object in the same
+			# order as they are defined
+
+			# Pass one
+			flow_desc.each do |node_desc|
+				node_object = Fbp.create(node_desc[:node].to_s)
+				return false if node_object.nil?
+				node_object.merge_options!(node_desc[:options]) if !node_desc[:options].nil?
+				@node_objects << node_object
+			end
+
+			# Pass two
+      flow_desc.each_index do |idx|
+        node_desc = flow_desc[idx]
+        this_node = @node_objects[idx]
+        next if node_desc.nil? ||  this_node.nil?
+        input_desc =  node_desc[:input]
+        next if input_desc.nil?
+
+        input_desc.each do |input_hash|
+          next if input_hash.nil?  || !input_hash.respond_to?(:each_pair)
+          input_hash.each_pair do |index, output_name|
+            output_node = @node_objects[index]
+            next if output_node.nil?
+            this_node.register_for_output_from_node(output_node, output_name)
+          end
+        end
+      end
+			@flow_created = true
+		end
+
+	end
+end