pione 0.2.1 → 0.2.2

data/example/PegasusWMS/Merge/Merge.pione

@@ -0,0 +1,21 @@
+Rule Main
+  output 'binaries.txt'
+Flow
+  rule List
+  rule Join
+End
+
+Rule List
+  output 'bin_{$INDEX}.txt'
+  param $INDEX := 0.upto(2)
+  param $DIRS := ("/bin" | "/usr/bin" | "/usr/local/bin").all
+Action
+  ls -l {$DIRS[$INDEX+1]} > {$O[1]}
+End
+
+Rule Join
+  input 'bin_*.txt'.all
+  output 'binaries.txt'
+Action
+  cat {$I[1].as_string.join(" ")} > {$O[1]}
+End

data/example/PegasusWMS/Merge/package.yml

@@ -0,0 +1,2 @@
+PackageName: PegasusWMSMerge
+Version: 0.1.0

data/example/PegasusWMS/Pipeline/Pipeline.pione

@@ -0,0 +1,20 @@
+Rule Main
+  output 'count.txt'
+Flow
+  rule GetHTML
+  rule Count
+End
+
+Rule GetHTML
+  output 'pegasus.html'
+Action
+  curl -o {$O[1]} "http://pegasus.isi.edu"
+End
+
+Rule Count
+  input 'pegasus.html'
+  output 'count.txt'
+Action
+  wc {$I[1]} > {$O[1]}
+End
+

data/example/PegasusWMS/Pipeline/package.yml

@@ -0,0 +1,2 @@
+PackageName: PegasusWMSPipeline
+Version: 0.1.0

data/example/PegasusWMS/Split/Split.pione

@@ -0,0 +1,27 @@
+Rule Main
+  output 'count.txt.*'.all
+Flow
+  rule GetHTML
+  rule Split
+  rule Count
+End
+
+Rule GetHTML
+  output 'pegasus.html'
+Action
+  curl -o {$O[1]} "http://pegasus.isi.edu"
+End
+
+Rule Split
+  input 'pegasus.html'
+  output 'part.*'.all
+Action
+  split -l 100 -a 1 "{$I[1]}" part.
+End
+
+Rule Count
+  input 'part.*'
+  output 'count.txt.{$*}'
+Action
+  wc {$I[1]} > {$O[1]}
+End

data/example/PegasusWMS/Split/package.yml

@@ -0,0 +1,2 @@
+PackageName: PegasusWMSSplit
+Version: 0.1.0

data/example/ScoreAggregation/package.yml

@@ -1 +1,5 @@
 PackageName: ScoreAggregation
+Documents:
+  - "ScoreAggregation.pione"
+Scenarios:
+  - "scenario/case1"

data/example/ScoreAggregation/scenario/case1/scenario.yml

@@ -1,3 +1,29 @@
 ScenarioName: case1
-
+Inputs:
+  - "A.score"
+  - "B.score"
+  - "C.score"
+  - "D.score"
+  - "E.score"
+  - "F.score"
+  - "G.score"
+  - "H.score"
+  - "I.score"
+  - "J.score"
+  - "K.score"
+  - "L.score"
+  - "M.score"
+  - "N.score"
+  - "O.score"
+  - "P.score"
+  - "Q.score"
+  - "R.score"
+  - "S.score"
+  - "T.score"
+  - "U.score"
+  - "V.score"
+  - "W.score"
+  - "X.score"
+  - "Y.score"
+  - "Z.score"
 

data/example/SerialProcessing/package.yml

@@ -1 +1,5 @@
-PackageName: SerialProcessing
+PackageName: "SerialProcessing"
+Documents:
+  - "SerialProcessing.pione"
+Scenarios:
+  - "scenario"

data/example/SerialProcessing/scenario/scenario.yml

@@ -0,0 +1,10 @@
+ScenarioName: SerialProcessing Test
+Outputs:
+  - "1.a"
+  - "2.a"
+  - "3.a"
+  - "4.a"
+  - "1.b"
+  - "2.b"
+  - "3.b"
+  - "4.b"

data/lib/pione.rb

@@ -1,7 +1,11 @@
 #
 # load libraries
 #
+
+# bundler
 require 'bundler/setup'
+
+# standard
 require 'set'
 require 'socket'
 require 'digest'
@@ -24,11 +28,13 @@ require 'etc'
 require 'json'
 require 'rexml/document'
 require 'logger'
+require 'ostruct'
+require 'net/ftp'
+require 'net/http'
 
+# gems
 require 'uuidtools'
 require 'parslet'
-require 'ostruct'
-require 'net/ftp'
 require 'highline'
 require 'dropbox_sdk'
 require 'hamster'
@@ -39,10 +45,15 @@ require 'sys/uname'
 require 'simple-identity'
 require 'rainbow'
 require 'em-ftpd'
-require 'pione/patch/em-ftpd-patch'
+require 'pione/patch/em-ftpd-patch' # patch
 require 'sys/cpu'
 require 'structx'
 require 'syslog-logger'
+require 'zipruby'
+require 'retriable'
+require 'childprocess'
+
+ChildProcess.posix_spawn = true
 
 #
 # load pione
@@ -50,14 +61,7 @@ require 'syslog-logger'
 
 require 'pione/version'
 require 'pione/util'
-
-# patch
-require 'pione/patch/array-patch'
-require 'pione/patch/drb-patch'
-require 'pione/patch/rinda-patch'
-require 'pione/patch/uri-patch'
-require 'pione/patch/monitor-patch'
-
+require 'pione/patch'
 require 'pione/uri-scheme'
 require 'pione/location'
 require 'pione/log'
@@ -66,71 +70,16 @@ require 'pione/system'
 Pione.module_exec {const_set(:PioneObject, Pione::System::PioneObject)}
 Pione.module_exec {const_set(:Global, Pione::System::Global)}
 
-# relay
-require 'pione/relay/transmitter-socket'
-require 'pione/relay/trampoline'
-require 'pione/relay/receiver-socket'
-require 'pione/relay/relay-socket'
-require 'pione/relay/relay-client-db'
-require 'pione/relay/relay-account-db'
-
-# tuple-space
-require 'pione/tuple-space/tuple-space-server-interface'
-require 'pione/tuple-space/presence-notifier'
-require 'pione/tuple-space/tuple-space-server'
-require 'pione/tuple-space/tuple-space-receiver'
-require 'pione/tuple-space/tuple-space-provider'
-require 'pione/tuple-space/data-finder'
-
-# rule-handler
+require 'pione/relay'
+require 'pione/tuple-space'
 require 'pione/rule-handler.rb'
-
 require 'pione/model'
 require 'pione/component'
-
-# tuple
 require 'pione/tuple'
-
-# parser
-require 'pione/parser/parslet-extension'
-require 'pione/parser/common-parser'
-require 'pione/parser/literal-parser'
-require 'pione/parser/feature-expr-parser'
-require 'pione/parser/expr-parser'
-require 'pione/parser/flow-element-parser'
-require 'pione/parser/block-parser'
-require 'pione/parser/rule-definition-parser'
-require 'pione/parser/document-parser'
-
-# transformer
+require 'pione/parser'
 require 'pione/transformer'
-
-# agent
-require 'pione/agent/basic-agent'
-require 'pione/agent/tuple-space-client'
-require 'pione/agent/command-listener'
-require 'pione/agent/task-worker'
-require 'pione/agent/input-generator'
-require 'pione/agent/rule-provider'
-require 'pione/agent/logger'
-require 'pione/agent/broker'
-require 'pione/agent/process-manager'
-require 'pione/agent/trivial-routine-worker'
-require 'pione/agent/tuple-space-server-client-life-checker'
-require 'pione/agent/messenger'
-
-# front
-require 'pione/front/basic-front'
-require 'pione/front/task-worker-owner'
-require 'pione/front/tuple-space-provider-owner'
-require 'pione/front/client-front'
-require 'pione/front/broker-front'
-require 'pione/front/task-worker-front'
-require 'pione/front/tuple-space-provider-front'
-require 'pione/front/tuple-space-receiver-front'
-require 'pione/front/relay-front'
-
-# command
+require 'pione/agent'
+require 'pione/front'
 require 'pione/command'
 
 #

data/lib/pione/agent.rb

@@ -0,0 +1,17 @@
+module Pione
+  # Agent is a namespace for agents.
+  module Agent; end
+end
+
+require 'pione/agent/basic-agent'
+require 'pione/agent/tuple-space-client'
+require 'pione/agent/command-listener'
+require 'pione/agent/task-worker'
+require 'pione/agent/input-generator'
+require 'pione/agent/rule-provider'
+require 'pione/agent/logger'
+require 'pione/agent/broker'
+require 'pione/agent/process-manager'
+require 'pione/agent/trivial-routine-worker'
+require 'pione/agent/tuple-space-server-client-life-checker'
+require 'pione/agent/messenger'

data/lib/pione/agent/basic-agent.rb

@@ -216,12 +216,10 @@ module Pione
 
       # Terminate to transit.
       def terminate
-        # abort the agent when called by other thread
-        abort unless @running_thread == Thread.current
-        # transit to terminated
         begin
           res = call_transition_method(:terminated)
-        rescue DRb::DRbConnError, DRb::ReplyReaderThreadError
+        rescue DRb::DRbConnError, DRb::ReplyReaderThreadError => e
+          Util::ErrorReport.warn("raised a connection error when we terminated", self, e, __FILE__, __LINE__)
         end
         # set agent state
         set_current_state(:terminated)

data/lib/pione/command.rb

@@ -21,4 +21,5 @@ require 'pione/command/pione-clean'
 require 'pione/command/pione-syntax-checker'
 require 'pione/command/pione-log'
 require 'pione/command/pione-val'
+require 'pione/command/pione-package'
 

data/lib/pione/command/option.rb

@@ -140,8 +140,11 @@ module Pione
       end
 
       def setup_item_value(opt, item, defs)
-        if item.value.kind_of?(Proc)
+        case item.value
+        when Proc
           opt.on(*defs, Proc.new{|*args| @option[item.name] = self.instance_exec(*args, &item.value)})
+        when :as_is
+          opt.on(*defs, Proc.new{|*args| @option[item.name] = args.first})
         end
       end
 

data/lib/pione/command/pione-package.rb

@@ -0,0 +1,97 @@
+module Pione
+  module Command
+    class PionePackage < BasicCommand
+      define_info do
+        set_name "pione-package"
+        set_banner "PIONE package utility."
+      end
+
+      define_option do
+        use :color
+        use :debug
+
+        default :output, Location["./"]
+
+        define(:build) do |item|
+          item.long = '--build'
+          item.desc = 'build PIONE archive file(*.ppg)'
+          item.action = lambda do |option, location|
+            option[:action] = :build
+          end
+        end
+
+        define(:output) do |item|
+          item.short = "-o"
+          item.long = "--output=LOCATION"
+          item.desc = "output file or directory location"
+          item.value = lambda {|val| Location[val]}
+        end
+
+        define(:tag) do |item|
+          item.long = "--tag=NAME"
+          item.desc = "specify tag name"
+          item.value = :as_is
+        end
+
+        define(:tag) do |item|
+          item.long = "--branch=NAME"
+          item.desc = "specify branch name"
+          item.value = :as_is
+        end
+
+        define(:hash_id) do |item|
+          item.long = "--hash-id=HASH"
+          item.desc = "specify git hash id"
+          item.value = :as_is
+        end
+
+        validate do |option|
+          unless option[:output].directory?
+            abort("output location should be a directory: %s" % option[:output])
+          end
+        end
+      end
+
+      start do
+        if option[:action] == :build
+          # package is not found
+          if @argv.first.nil?
+            abort("There are no PIONE documents or packages.")
+          end
+
+          # archive
+          if ppg = try_to_archive(Location[git: @argv.first], Location[data: @argv.first])
+            puts "suceeded: %s" % ppg.address
+          else
+            abort("Faild to archive.")
+          end
+        end
+      end
+
+      private
+
+      def archive_from_directory_package
+        archiver = Component::PackageArchiver.new(Location[@argv.first])
+      end
+
+      def try_to_archive(*locations)
+        locations.each do |location|
+          begin
+            # make archiver
+            archiver_option = {tag: option[:tag], branch: option[:branch], hash_id: option[:hash_id]}
+            if location.location_type == :data
+              archiver_option[:tag] = Time.now.strftime("%Y%m%d%H%M")
+            end
+            archiver = Component::PackageArchiver.new(location, archiver_option)
+
+            # archive
+            return archiver.archive(option[:output])
+          rescue => e
+            Util::ErrorReport.warn("archiver faild: %s" % location, self, e, __FILE__, __LINE__)
+          end
+        end
+        return false
+      end
+    end
+  end
+end

data/lib/pione/command/pione-task-worker.rb

@@ -115,28 +115,32 @@ BANNER
       terminate do
         Global.monitor.synchronize do
           begin
+            # flag
             return if @terminated
+            @terminated = true
 
             # terminate the agent
             if @agent
               @agent.terminate
 
-              while true
-                break if @agent.terminated? and @agent.running_thread and @agent.running_thread.stop?
-                sleep 1
+              Timeout.timeout(5) do
+                while true
+                  break if @agent.terminated? and @agent.running_thread and @agent.running_thread.stop?
+                  sleep 1
+                end
               end
             end
 
             # disconnect parent front
             option[:parent_front].remove_task_worker_front(self, option[:connection_id])
 
-            # flag
-            @terminated = true
           rescue DRb::DRbConnError, DRb::ReplyReaderThreadError => e
-            ErrorReport.warn("Disconnected in termination process of task worker agent.", self, e, __FILE__, __LINE__)
+            Util::ErrorReport.warn("Disconnected in termination process of task worker agent.", self, e, __FILE__, __LINE__)
           rescue ThreadError => e
             # tuple space may be closed
-            ErrorReport.warn("Failed in termination process of task worker agent.", self, e, __FILE__, __LINE__)
+            Util::ErrorReport.warn("Failed in termination process of task worker agent.", self, e, __FILE__, __LINE__)
+          rescue Timeout::Error => e
+            Util::ErrorReport.warn("Timeouted in termination of pione-task-worker.", self, e, __FILE__, __LINE__)
           end
         end
       end