rbeai 0.0.1

data/lib/rbeai/RouterPipeTask.rb

@@ -0,0 +1,62 @@
+require 'thread'
+require 'thwait'
+require 'rbeai/PipeTask'
+
+module RbEAI
+  
+  
+  class RouterPipeTask < PipeTask
+                    
+     def initialize(inputQueue, task, control)
+      @inputQueue  = inputQueue
+      @task = task
+      @control = control
+      @resultQueue = task.initQueues()
+      @bufferQueue = nil    
+      @nextPipe = _getNextPipes()                
+      @size = task.size
+      @threadPool = ThreadPool.new(@size, @task.method(:doJob), inputQueue, @resultQueue, @bufferQueue, control.controlQueue)    
+    end
+    
+    def run(numItems = 1)
+      numItemsNext = @task.getItemsNext(numItems)    
+      _RunNextPipes(numItemsNext)
+      @control.start(numItems, @threadPool)
+      @threadPool.start()       
+    end
+    
+    def waitToEnd()
+      _WaitToEndNextPipes()
+      #print "JOIN-#{@task.name}\n"
+      @control.waitJoin
+      print "END-#{@task.name}\n"
+    end
+    
+    private
+    
+      def _getNextPipes()                  
+        controlQueue = Queue.new
+        nextPipeList = Hash.new(0)
+        @task.nextTask.each do | key, task |
+          inputQueue = @resultQueue[key]
+          control = Control.new(controlQueue, inputQueue)
+          nextPipeList[key] = getNextPipe(inputQueue, task, control) 
+        end
+        return nextPipeList
+      end    
+  
+      def _RunNextPipes(numItemsNext)
+        @nextPipe.each do | key, pipe |
+          pipe.run(numItemsNext)
+        end
+      end
+      
+      def _WaitToEndNextPipes()
+        @nextPipe.each do | key, pipe |
+          pipe.waitToEnd()
+        end    
+      end
+  
+  end
+
+end #module

data/lib/rbeai/TasksLogic.rb

@@ -0,0 +1,285 @@
+require 'rio'
+require 'rexml/document'
+require 'rbeai/PipeTask'
+require 'rbeai/RouterLogic'
+require 'rbeai/TransfLogic'
+require 'rbeai/GetLogic'
+require 'rbeai/PutLogic'
+
+module RbEAI
+
+  class Task
+  
+    attr_accessor  :nextTask, :status, :wflow, :size
+    attr_reader :name
+    
+    public 
+    
+      def initialize(wflow, xmlFull, xmlDoc)
+        @size = (pool_size = xmlDoc.attributes["pool"]) != nil ? pool_size.to_i : 1
+        @wflow = wflow
+        @name = xmlDoc.attributes["name"]
+        @status = 0
+        @nextTask = _getNextTask(wflow, xmlFull, xmlDoc)
+        print @name,"\n"
+      end
+  
+      def doJob(resultQueue, obj)
+        resultQueue.enq(obj)
+      end
+      
+      def getItemsNext(numItems)
+        return numItems
+      end
+   
+      def persist(object)
+        bname = File.basename("#{object}")
+        rio("tmp/#{@wflow.instid}/#{@name}").mkdir      	
+        rio("#{object}") > rio("tmp/#{@wflow.instid}/#{@name}")    		
+        return rio("tmp/#{@wflow.instid}/#{@name}/#{bname}")
+      end      
+      
+      def persistFtp(object)
+	bname = File.basename("#{object}")
+        rio("tmp/#{@wflow.instid}/#{@name}").mkdir      	
+        rio("#{object}") > rio("tmp/#{@wflow.instid}/#{@name}/#{bname}")    		
+        return rio("tmp/#{@wflow.instid}/#{@name}/#{bname}")
+      end   
+      
+      def persistStr(object, filename)
+        rio("tmp/#{@wflow.instid}/#{@name}").mkdir      
+        ario = rio("tmp/#{@wflow.instid}/#{@name}/#{filename}")    
+        ario.print(object).close
+        return ario
+      end          
+  
+    protected
+  
+      def _getNextTask(wflow, xmlFull, xmlDoc)
+        nextTaskName = xmlDoc.elements["next"].text
+        if nextTaskName != nil
+          xmlNextTask = XPath.first(xmlFull, "//task[@name='#{nextTaskName}']")
+          nextTaskType = xmlNextTask.attributes["type"]
+          return eval("#{nextTaskType}Task.new(#{:wflow},#{:xmlFull},#{:xmlNextTask})")
+        else
+          return nil
+        end
+      end
+    
+  end
+  
+  
+#    <task name="router" type="Router" pool="2">
+#      <next>
+#        <goto task="sendfile">
+#          <when filter="xpath:/test/status">91,92</when>
+#          <!--<when filter="name:test.txt"/>-->
+#        </goto>
+#        <goto task="sendftp">              
+#          <when filter="sax:status">91,92</when>
+#          <!--<when filter="name:test_2.txt"/>-->
+#        </goto>            
+#        <goto task="final"/>
+#      </next>
+#    </task>          
+  
+  class RouterTask < Task
+        
+    attr_accessor :nextTask
+        
+    public 
+    
+      def initialize(wflow, xmlFull, xmlDoc)
+        @logic = RouterLogic.new(xmlDoc)
+        super(wflow, xmlFull, xmlDoc)
+      end    
+    
+      def doJob(resultQueueList, obj)
+        task = @logic.enroute(obj)
+        print task, "=> ", obj , "\n"
+        resultQueueList[task].enq(obj) if task != nil
+      end
+  
+      def initQueues()
+        resultQueueList = Hash.new(0)
+        @logic.destinations.each do | task, logic|
+          resultQueueList[task] = Queue.new
+        end
+        return resultQueueList
+      end
+      
+    protected
+        
+      def _getNextTask(wflow, xmlFull, xmlDoc)
+        nextTask = Hash.new(0)
+        @logic.destinations.each do | dest |
+          taskName = dest[0]
+          logic = dest[1]
+          xmlNextTask = XPath.first(xmlFull, "//task[@name='#{taskName}']")
+          nextTaskType = xmlNextTask.attributes["type"]
+          nextTask[taskName] = eval("#{nextTaskType}Task.new(#{:wflow},#{:xmlFull},#{:xmlNextTask})")
+        end      
+        return nextTask
+      end
+    
+  end
+  
+
+#    <task name="transf" type="Transf" pool="3">
+#      <next>final</next>
+#      <transf ord="1" type="xsl" script="transf.xsl"/>
+#      <transf ord="2" type="awk" script="transf.awk"/>
+#      <!--<transf type="csv2xml" separator=" "/> -->
+#      <!--<transf type="xml2csv" separator="|"/> -->
+#      <!--<transf type="awk" script="transf.awk"/>-->
+#    </task>        
+
+  class TransfTask < Task
+        
+    attr_accessor :nextTask
+        
+    public 
+    
+      def initialize(wflow, xmlFull, xmlDoc)
+        @logic = TransfLogic.new(xmlDoc)
+        super(wflow, xmlFull, xmlDoc)
+      end    
+    
+      def doJob(resultQueue, source)      
+        aux = persist(source) # Create room for the transformed file
+        dest = @logic.transform(aux)
+        resultQueue.enq(dest)      
+      end
+    
+  end
+  
+  # <task name="inicio" type="Start">
+  #   <next>final</next>
+  # </task>
+  class StartTask < Task
+  end
+  
+  # <task name="inicio" type="Start">
+  #   <next/>
+  # </task>
+  class EndTask < Task
+      
+      def doJob(resultQueue, obj)
+        print "EndTask #{@name}: #{obj}\n"
+        super(resultQueue, obj)
+      end
+      
+  end
+  
+  
+  # <task name="getfile" type="Get" rate="1s" pool="2"> <!-- Valid units are: 's', 'm', 'h' -->
+  #   <next>final</next>
+  #   <protocol def="file">   <!-- Valid protocols are: 'file', 'pop3', 'http', 'ftp', 'queue' -->
+  #     <location>.</location>
+  #     <pattern>test*.txt</pattern>
+  #   </protocol>
+  # </task>
+  class IOTask < Task
+    attr_accessor  :location, :pattern, :rate, :files
+      
+    public
+      
+      def initialize(wflow, xmlFull, xmlDoc)
+        super(wflow, xmlFull, xmlDoc)      
+        @location = XPath.first(xmlDoc, "./protocol/location/text()")
+        @pattern = XPath.first(xmlDoc, "./protocol/pattern/text()")
+        @rate = _getRate(xmlDoc.attributes["rate"])    
+      end
+          
+    protected
+       
+      def _getRate(rateTx)
+        re = Regexp.new('(\d+)(\D)')
+        mrate = re.match(rateTx)
+        value = mrate[1]
+        unit  = mrate[2]
+        case unit
+          when "s"
+            return value
+          when "m"
+            return value * 60
+          when "h"
+            return value * 60 * 60
+          else
+            return nil
+        end            
+      end
+  end
+  
+
+
+#    <task name="getfile" type="Get" rate="1s" pool="2">
+#      <next>router</next>
+#      <protocol def="file">
+#        <location>.</location>
+#        <pattern>out.xml</pattern>
+#      </protocol>
+#    </task>       
+#    <task name="getfile" type="Get" rate="1s" pool="2">
+#      <next>router</next>
+#      <protocol def="email">
+#        <location address="" port=""/>
+#        <account name="" pwd=""/>
+#        <pattern>out.xml</pattern>
+#      </protocol>
+#    </task>         
+
+
+  class GetTask < IOTask
+  
+    def initialize(wflow, xmlFull, xmlDoc)    
+      super(wflow, xmlFull, xmlDoc)
+      @logic = GetLogic.new(xmlDoc, self)    
+    end    
+  
+    def doJob(resultQueue, obj)
+      files = @logic.getFiles()
+      files.each do |file|
+        resultQueue.enq("#{file}")
+        sleep(@rate.to_i)
+      end
+    end
+      
+  end
+  
+  class PutTask < IOTask
+  
+    def initialize(wflow, xmlFull, xmlDoc)    
+      super(wflow, xmlFull, xmlDoc)
+      @logic = PutLogic.new(xmlDoc, self)    
+    end    
+  
+      
+    def doJob(resultQueue, obj)    
+      if @rate.to_i != 0
+        super(resultQueue, obj)    
+      else 
+        print "Send doJob #{obj} > #{location}\n"
+        @logic.putFile(obj)
+        resultQueue.enq(obj)        
+      end
+    end
+    
+    def getItemsNext(numItems)
+      return numItems
+    end
+           
+    def doBuffered(buffer)        
+      buffer.size.times do
+        obj = buffer.deq        
+        if @rate.to_i != 0
+          print "Send doBuffered #{obj} > #{location}\n"
+          @logic.putFile(obj)
+          sleep(@rate.to_i)
+        end
+      end    
+    end
+  
+  end
+  
+end #module

data/lib/rbeai/TransfLogic.rb

@@ -0,0 +1,145 @@
+require 'rio'
+require 'rexml/document'
+
+include REXML
+
+module RbEAI
+  
+  class TransfLogic
+  
+    attr_accessor :destinations, :default
+  
+    def initialize(xmlDoc)
+      @transfs = []    
+      # Detect simple usage of one transformation not ordered
+      el = XPath.first(xmlDoc, "./transf[count(@ord) = 0]")
+      if el != nil
+        @transfs[0] = TransfScript.new(el)
+      else
+        # Case of multiple ordered transformations
+        XPath.each(xmlDoc, "./transf[count(@ord) > 0]") do |el|      
+          order = el.attributes["ord"]
+          @transfs[order.to_i] = TransfScript.new(el)      
+        end
+      end
+    end
+    
+    def transform(file)
+      aux = file
+      @transfs.each do | trfilter |      
+        aux = trfilter.transform(aux) if trfilter != nil      
+      end
+      return aux
+    end
+    
+  end
+  
+  class TransfScript
+  
+    def initialize(xmlDoc)
+      @transf = _getParser(xmlDoc)     
+    end
+    
+    def transform(file)
+      @transf.transform(file)
+    end
+    
+    private 
+    
+      def _getParser(node)
+        type = node.attributes["type"]
+        re = Regexp.new('(xsl|awk|csv2xml_nh|csv2xml_wh|xml2csv)')
+        mr = re.match(type)
+        case mr[1]
+          when 'xsl' then XslScript.new(node)
+          when 'awk' then AwkScript.new(node)
+          when 'csv2xml_nh' then Csv2XmlScript.new(node)
+          when 'csv2xml_wh' then Csv2XmlScript.new(node)
+          when 'xml2csv' then Xml2CsvScript.new(node)
+        end
+      end
+  
+  end
+  
+    class Script
+      
+      def initialize(node)
+        @script = node.attributes["script"]
+        @pos = (pos = node.attributes["ord"])!= nil ?  pos : "0"      
+      end
+      
+      def transform(file)
+        newfilename = file.to_s+"."+@pos
+        rio("#{file}") > rio("#{newfilename}")    
+        doTransform(file, newfilename)
+        return newfilename
+      end
+      
+      protected
+        
+        def doTransform(file, newfilename)
+        end
+      
+    end
+    
+    class XslScript < Script
+      
+      def doTransform(file, newfilename)
+      	print "xsltproc -o #{newfilename} #{@script} #{file} \n"
+        result = system("xsltproc -o #{newfilename} #{@script} #{file}")
+      end
+      
+    end
+  
+    class AwkScript < Script
+      
+      def initialize(node)
+        super(node)
+        @separator = (sep = node.attributes["separator"])!= nil ? "-F "+sep : ""      
+      end
+      
+      def doTransform(file, newfilename)
+        print "gawk -f #{@script} #{@separator} #{@file} > #{newfilename} \n"
+        result = system("gawk -f #{@script} #{@separator} #{file} > #{newfilename}")
+      end
+      
+    end
+    
+    
+    class Csv2XmlScript < AwkScript
+      
+      def initialize(node)
+        super(node)
+        @script = File.dirname(__FILE__)+"/templates/"+node.attributes["type"]+".awk"
+      end
+          
+    end
+    
+    class Xml2CsvScript < AwkScript
+      
+      def initialize(node)
+        super(node)   
+        @xprow = node.attributes["xprow"]
+        @xpitem = node.attributes["xpitem"]
+        @separator = (sep = node.attributes["separator"])!= nil ? sep : "\t"      
+      end
+      
+      def doTransform(file, newfilename)
+        csvarr = []
+        xmlDoc = Document.new(File.new(file))
+        XPath.each(xmlDoc, "#{@xprow}") do |el|   
+          csvline = ""
+          bsep = false        
+          XPath.each(el, "#{@xpitem}") do |el2|             
+            csvline << @separator if bsep
+            csvline << el2.to_s
+            bsep = true          
+          end
+          csvarr << csvline
+        end      
+        rio("#{newfilename}").puts!(csvarr)
+      end
+      
+    end  
+    
+end #module

data/lib/rbeai/WorkerPool.rb

@@ -0,0 +1,72 @@
+require 'thread'
+
+module RbEAI
+  
+  class WorkerPool
+  
+    def initialize(size, nmessages, inputQueue, resultQueue, method)
+      @size = size
+      @inputQueue  = inputQueue
+      @resultQueue = resultQueue
+      @controlQueue = Queue.new
+      @control = Thread.new(nmessages) do |nmess|
+          Thread.stop()
+          print "Running control thread\n"
+          count = 0
+          begin
+            aux = @controlQueue.deq
+            count = count + 1
+          end until count == nmess
+          print "Finished control thread\n"
+          stop()
+      end
+      @threadlist = []    
+      size.times do |i|
+        @threadlist << Thread.new(i) do |j|
+          Thread.stop()        
+          print "Running th ",j,"\n"
+          begin
+            obj = @inputQueue.deq
+            if obj != :FINISH_WORK
+              method.call(obj, j) 
+              @controlQueue.enq(:COUNT)
+  
+            end
+          end until obj == :FINISH_WORK
+          print "Finished th ",j,"\n"
+        end
+      end    
+    end
+    
+    def run()
+      @control.run
+      @threadlist.each { |t| t.run}    
+    end
+    
+    def stop()
+      @size.times { |i| @inputQueue.enq(:FINISH_WORK) }
+      @threadlist.each { |t| t.join }
+    end
+  
+    def stopcontrol()        
+      @control.join
+    end
+  
+  end
+  
+  class Hello
+  
+    def sayHello(obj, i)    
+      print obj.object_id,"-#{obj}-#{i}\n"
+    end
+  end
+  
+  #h = Hello.new
+  #inputQ = Queue.new
+  #wp = WorkerPool.new(5, 1000, inputQ, Queue.new, h.method(:sayHello))
+  #wp.run()
+  #1000.times { |i| inputQ.enq(i) }
+  #wp.stopcontrol()
+
+
+end #module

data/lib/rbeai/Workflow.rb

@@ -0,0 +1,79 @@
+require 'rio'
+require 'rexml/document'
+require 'rbeai/PipeTask'
+require 'rbeai/TasksLogic'
+
+include REXML
+
+module RbEAI
+
+  class Workflow
+  
+    attr_accessor :instid
+  
+    def initialize(xmlStr)
+      @xmlDoc = Document.new(xmlStr)
+      aux = XPath.first(@xmlDoc, "./workflow")
+      @instid = ""
+      rate    = _getRate(aux.attributes["rate"])    
+      @name   = aux.attributes["name"]
+      @inputQueue = Queue.new
+      @startPipe = _initPipeTask()    
+      @thControl = _initProcess(rate.to_i)
+    end
+    
+    def run()
+      @startPipe.run()    
+      @thControl.run()
+    end
+    
+    def finish()
+      @thControl.kill()
+      @startPipe.finish()
+      @startPipe.waitToEnd()      
+    end
+    
+    private 
+    
+      def _initInstance()
+        t = Time.new
+        ts = "%10.5f" % t.to_f
+        @instid = @name+"/"+ts
+        rio("tmp/#{@instid}").mkpath
+      end
+    
+      def _initPipeTask()
+        startTask = StartTask.new(self, @xmlDoc, XPath.first(@xmlDoc, "//task[@type='Start']"))  
+        return PipeTask.new(@inputQueue, startTask, Control.new(Queue.new, @inputQueue))
+      end
+      
+      def _initProcess(timer)
+        Thread.new(timer) do |timer|      
+          Thread.stop()
+          loop do
+            _initInstance()
+            @inputQueue.enq(:start)
+            sleep(timer)
+          end
+        end
+      end
+      
+      def _getRate(rateTx)
+        re = Regexp.new('(\d+)(\D)')
+        mrate = re.match(rateTx)
+        value = mrate[1]
+        unit  = mrate[2]
+        case unit
+          when "s"
+            return value
+          when "m"
+            return value * 60
+          when "h"
+            return value * 60 * 60
+          else
+            return nil
+        end            
+      end    
+  end #class
+
+end #module

data/lib/rbeai/templates/csv2xml_nh.awk

@@ -0,0 +1,14 @@
+BEGIN{
+        print "<?xml version=\"1.0\" encoding=\"ISO-8859-1\"?>"
+	print "<all>"
+}
+
+{
+        print "\t<doc>"
+        for(i=1;i<=NF;i++){printf ("\t\t<data>%s</data>\n",$i)}
+        print "\t</doc>"
+}
+
+END{
+        print "</all>"
+}

data/lib/rbeai/templates/csv2xml_wh.awk

@@ -0,0 +1,16 @@
+BEGIN{
+        print "<?xml version=\"1.0\" encoding=\"ISO-8859-1\"?>"
+	print "<all>"
+        getline
+        for(i=1;i<=NF;i++){elem[i]=$i}
+}
+
+{
+        print "\t<doc>"
+        for(i=1;i<=NF;i++){printf ("\t\t<%s>%s</%s>\n",elem[i],$i,elem[i])}
+        print "\t</doc>"
+}
+
+END{
+        print "</all>"
+}

data/lib/rbeai.rb

@@ -0,0 +1,13 @@
+module RbEAI
+end
+
+require 'rbeai/GetLogic'
+require 'rbeai/Main'
+require 'rbeai/PipeTask'
+require 'rbeai/PutLogic'
+require 'rbeai/RouterLogic'
+require 'rbeai/RouterPipeTask'
+require 'rbeai/TasksLogic'
+require 'rbeai/TransfLogic'
+require 'rbeai/WorkerPool'
+require 'rbeai/Workflow'