pione 0.2.0 → 0.2.1

data/lib/pione/command/pione-broker.rb

@@ -10,11 +10,16 @@ module Pione
       end
 
       define_option do
-        use Option::TupleSpaceReceiverOption
-        use Option::TaskWorkerOwnerOption
-
-        validate do |data|
-          unless data[:task_worker] > 0
+        use :color
+        use :daemon
+        use :debug
+        use :features
+        use :my_ip_address
+        use :show_communication
+        use :task_worker
+
+        validate do |option|
+          unless option[:task_worker] > 0
             abort("error: no task worker resources")
           end
         end

data/lib/pione/command/pione-clean.rb

@@ -7,6 +7,10 @@ module Pione
         set_banner "Clean working directories and file cache directories."
       end
 
+      define_option do
+        use :debug
+      end
+
       start do
         FileUtils.remove_entry_secure(Global.working_directory_root)
         FileUtils.remove_entry_secure(Global.file_cache_directory_root)

data/lib/pione/command/pione-client.rb

@@ -2,114 +2,137 @@ module Pione
   module Command
     # PioneClient is a command to request processing.
     class PioneClient < FrontOwnerCommand
+      include TupleSpaceServerInterface
+
       define_info do
         set_name "pione-client"
-        set_tail {|cmd|
-          args = [cmd.option[:filename], cmd.option[:output_uri], cmd.option[:stream]]
-          "{Document: %s, OutputURI: %s, Stream: %s}" % args
-        }
+        set_tail {|cmd| Global.front.uri}
         set_banner "Requests to process PIONE document."
       end
 
       define_option do
-        use Option::TaskWorkerOwnerOption
-        use Option::TupleSpaceProviderOwnerOption
-
-        default :output_location, Location["local:./output/"]
-        default :stream, false
-        default :params, Model::Parameters.empty
-        default :dry_run, false
-        default :task_worker, Agent::TaskWorker.default_number
-        default :request_task_worker, 1
-        default :stand_alone, false
-        default :relay, nil
-        default :filename, "-"
-        default :without_tuple_space_provider, false
-        default :features, "^Interactive"
-        default :list_params, false
-
-        # --input
-        option('-i LOCATION', '--input=LOCATION', 'set input directory') do |data, uri|
-          begin
-            data[:input_location] = Location[uri]
-          rescue ArgumentError
-            abort("opiton error: bad location '%s'" % uri)
+        use :debug
+        use :color
+        use :show_communication
+        use :my_ip_address
+        use :presence_notification_address
+        use :show_presence_notifier
+        use :task_worker
+        use :features
+
+        define(:input_location) do |item|
+          item.short = '-i LOCATION'
+          item.long = '--input=LOCATION'
+          item.desc = 'set input directory'
+          item.value = proc do |uri|
+            begin
+              Location[uri]
+            rescue ArgumentError
+              abort("ERROR: bad location '%s'" % uri)
+            end
           end
         end
 
-        # --output
-        option('-o LOCATION', '--output=LOCATION', 'set output directory') do |data, uri|
-          begin
-            data[:output_location] = Location[uri]
-            if URI.parse(uri).scheme == "myftp"
-              data[:myftp] = URI.parse(uri).normalize
+        define(:output_location) do |item|
+          item.short = '-o LOCATION'
+          item.long = '--output=LOCATION'
+          item.desc = 'set output directory'
+          item.default = Location["local:./output/"]
+          item.action = proc do |option, uri|
+            begin
+              option[:output_location] = Location[uri]
+              if URI.parse(uri).scheme == "myftp"
+                option[:myftp] = URI.parse(uri).normalize
+              end
+            rescue ArgumentError
+              abort("ERROR: bad location '%s'" % uri)
             end
-          rescue ArgumentError
-            abort("opiton error: bad location '%s'" % uri)
           end
         end
 
-        # --stream
-        option('--stream', 'turn on stream mode') do |data|
-          data[:stream] = true
+        define(:stream) do |item|
+          item.long = '--stream'
+          item.desc = 'turn on stream mode'
+          item.default = false
+          item.value = true
         end
 
-        # --request-task-worker
-        option('--request-task-worker=N', 'set request number of task workers') do |data, n|
-          data[:request_task_worker] = n.to_i
+        define(:request_task_worker) do |item|
+          item.long = '--request-task-worker=N'
+          item.desc = 'set request number of task workers'
+          item.default = 1
+          item.value = proc {|n| n.to_i}
         end
 
-        # --params
-        option('--params="{Var:1,...}"', "set &main:Main rule's parameters") do |data, str|
-          begin
-            params = DocumentTransformer.new.apply(
-              DocumentParser.new.parameters.parse(str)
-            )
-            data[:params].merge!(params)
-          rescue Parslet::ParseFailed => e
-            puts "invalid parameters: " + str
-            Util::ErrorReport.print(e)
-            abort
+        define(:params) do |item|
+          item.long = '--params="{Var:1,...}"'
+          item.desc = "set user parameters"
+          item.default = Model::Parameters.empty
+          item.action = proc do |option, str|
+            begin
+              params = DocumentTransformer.new.apply(
+                DocumentParser.new.parameters.parse(str)
+              )
+              option[:params].merge!(params)
+            rescue Parslet::ParseFailed => e
+              $stderr.puts "invalid parameters: " + str
+              Util::ErrorReport.print(e)
+              abort
+            end
           end
         end
 
-        # --stand-alone
-        option('--stand-alone', 'turn on stand alone mode') do |data|
-          data[:stand_alone] = true
-          data[:without_tuple_space_provider] = true
+        define(:stand_alone) do |item|
+          item.long = '--stand-alone'
+          item.desc = 'turn on stand alone mode'
+          item.default = false
+          item.action = proc do |option|
+            option[:stand_alone] = true
+            option[:without_tuple_space_provider] = true
+          end
         end
 
-        # --dry-run
-        option('--dry-run', 'turn on dry run mode') do |data, b|
-          data[:dry_run] = true
+        define(:dry_run) do |item|
+          item.long = '--dry-run'
+          item.desc = 'turn on dry run mode'
+          item.default = false
+          item.value = true
         end
 
-        # --relay
-        option('--relay=URI', 'turn on relay mode and set relay address') do |data, uri|
-          data[:relay] = uri
+        item(:features).default = "^Interactive"
+
+        define(:relay) do |item|
+          item.long = '--relay=URI'
+          item.desc = 'turn on relay mode and set relay address'
+          item.default = nil
+          item.value = proc {|uri| uri}
         end
 
-        # --name
-        option('--name=NAME') do |data, name|
-          data[:name] = name
+        define(:list_params) do |item|
+          item.long = '--list-params'
+          item.desc = 'show user parameter list in the document'
+          item.value = true
         end
 
-        # --list-parameters
-        option('--list-params', 'show user parameter list in the document') do |data|
-          data[:list_params] = true
+        define(:rehearse) do |item|
+          item.long = '--rehearse[=SCENARIO]'
+          item.desc = 'rehearse the scenario'
+          item.value = proc {|scenario_name| scenario_name || :anything}
         end
 
-        option('--rehearse[=SCENARIO]', 'rehearse the scenario') do |data, scenario_name|
-          data[:rehearse] = scenario_name || :anything
+        define(:without_tuple_space_provider) do |item|
+          item.long = '--without-tuple-space-provider'
+          item.desc = 'process without tuple space provider'
+          item.value = true
         end
 
-        validate do |data|
-          unless data[:task_worker] > 0 or
-              (not(data[:stand_alone]) and data[:task_worker] == 0)
-            abort("option error: invalid resource size '%s'" % data[:task_worker])
+        validate do |option|
+          unless option[:task_worker] > 0 or
+              (not(option[:stand_alone]) and option[:task_worker] == 0)
+            abort("option error: invalid resource size '%s'" % option[:task_worker])
           end
 
-          if data[:stream] and data[:input_location].nil?
+          if option[:stream] and option[:input_location].nil?
             abort("option error: no input URI on stream mode")
           end
         end
@@ -118,12 +141,12 @@ module Pione
       attr_reader :task_worker
       attr_reader :features
       attr_reader :tuple_space_server
-      attr_reader :name
 
-      def initialize
-        super()
+      def initialize(*options)
+        super(*options)
         @worker_threads = []
         @tuple_space_server = nil
+        @child_process_infos = []
       end
 
       private
@@ -133,93 +156,71 @@ module Pione
       end
 
       prepare do
-        @filename = ARGF.filename
-
-        # ftp server
-        if myftp = option[:myftp]
-          location = Location[myftp.path]
-          location.path.mkdir unless location.exist?
-          if myftp.userinfo
-            Util::FTPServer.auth_info = Util::FTPAuthInfo.new(myftp.user, myftp.password)
-          end
-          if myftp.port
-            Util::FTPServer.port = myftp.port
-          end
-          Util::FTPServer.start(Util::FTPLocalFS.new(location))
-        end
+        # FTP server
+        setup_ftp_server(myftp) if myftp = option[:myftp]
 
-        @tuple_space_server = TupleSpaceServer.new(
-          task_worker_resource: option[:request_task_worker]
-        )
+        # run tuple space server
+        @tuple_space_server = TupleSpaceServer.new(task_worker_resource: option[:request_task_worker])
+        set_tuple_space_server(@tuple_space_server)
 
-        # setup base uri
+        # setup output location
         case option[:output_location]
         when Location::LocalLocation
           option[:output_location] = Location[option[:output_location].path.expand_path]
           option[:output_location].path.mkpath
         when Location::DropboxLocation
-          # start session
-          session = nil
-          consumer_key = nil
-          consumer_secret = nil
-
-          cache = Pathname.new("~/.pione/dropbox_api.cache").expand_path
-          if cache.exist?
-            session = DropboxSession.deserialize(cache.read)
-            Location::Dropbox.set_session(session)
-            consumer_key = session.instance_variable_get(:@consumer_key)
-            consumer_secret = session.instance_variable_get(:@consumer_secret)
-          else
-            api = YAML.load(Pathname.new("~/.pione/dropbox_api.yml").expand_path.read)
-            consumer_key = api["key"]
-            consumer_secret = api["secret"]
-            session = DropboxSession.new(consumer_key, consumer_secret)
-            Location::Dropbox.set_session(session)
-            authorize_url = session.get_authorize_url
-            puts "AUTHORIZING", authorize_url
-            puts "Please visit that web page and hit 'Allow', then hit Enter here."
-            STDIN.gets
-            session.get_access_token
-
-            # cache session
-            cache.open("w+") {|c| c.write session.serialize}
-          end
-
-          # check session state
-          unless session.authorized?
-            abort("We cannot authorize dropbox access to PIONE.")
-          end
-
-          # share access token in tuple space
-          Location::Dropbox.share_access_token(tuple_space_server, consumer_key, consumer_secret)
+          setup_dropbox
         end
 
         @tuple_space_server.set_base_location(option[:output_location])
       end
 
-      start do
-        read_process_document
+      start(:pre) {read_package}
 
+      # Print list of user parameters.
+      start do
         if option[:list_params]
-          print_parameter_list
-          exit!
-        end
-
-        write_tuples
-        connect_relay if option[:relay]
-        start_agents
+          unless @package.params.empty?
+            # print basic parameters
+            basic_user_params = @package.params.basic
+            unless basic_user_params.empty?
+              puts "Basic Paramters:"
+              basic_user_params.data.each do |var, val|
+                puts "  %s := %s" % [var.name, val.textize]
+              end
+            end
 
-        # start tuple space provider with thread
-        # the thread is terminated when the client terminated
-        unless option[:without_tuple_space_provider]
-          @start_tuple_space_provider_thread = Thread.new do
-            start_tuple_space_provider
+            advanced_user_params = @package.params.advanced
+            unless advanced_user_params.empty?
+              puts "Advanced Paramters:"
+              advanced_user_params.data.each do |var, val|
+                puts "  %s := %s" % [var.name, val.textize]
+              end
+            end
+          else
+            puts "  there are no user parameters in %s" % ARGF.path
           end
+          exit
         end
+      end
 
-        start_workers
-        @agent = Agent[:process_manager].start(@tuple_space_server, @document, option[:params], option[:stream])
-        @agent.running_thread.join
+      # Run processing.
+      start do
+        # write tuples
+        write(Tuple[:process_info].new('standalone', 'Standalone'))
+        write(Tuple[:dry_run].new(option[:dry_run]))
+
+        # start
+        start_relay_connection if option[:relay]
+        start_precedent_agents
+        start_task_workers
+        start_process_manager
+
+        # check result
+        check_rehearsal_result if option[:rehearse]
+
+        @child_process_infos.each {|info| info.kill}
+        @child_process_infos.each {|info| info.wait}
       end
 
       terminate do
@@ -245,59 +246,88 @@ module Pione
 
       private
 
-      # Read PIONE process document.
-      #
-      # @return [void]
-      def read_process_document
-        # process definition document is not found.
-        if ARGF.filename == "-"
-          abort("There are no process definition documents.")
+      # Setup FTP server with the URI.
+      def setup_ftp_server(uri)
+        location = Location[uri.path]
+        location.path.mkdir unless location.exist?
+        if uri.userinfo
+          Util::FTPServer.auth_info = Util::FTPAuthInfo.new(uri.user, uri.password)
         end
+        if uri.port
+          Util::FTPServer.port = myftp.port
+        end
+        Util::FTPServer.start(Util::FTPLocalFS.new(location))
+      end
 
-        # get script dirname
-        @dir = File.dirname(File.expand_path(__FILE__))
-
-        location = Location[ARGF.path]
-
-        # read process document
-        begin
-          if location.directory?
-            # package
-            package = Component::PackageReader.new(location).read
-            package.upload(option[:output_location] + "package")
-            if option[:rehearse]
-              unless package.scenarios.empty?
-                if scenario = package.find_scenario(option[:rehearse])
-                  option[:input_location] = scenario.input
-                else
-                  abort "the scenario not found: %s" % option[:rehearse]
-                end
-              end
-            end
-            @document = package
-          else
-            @document = Component::Document.parse(location.read)
-          end
-        rescue Pione::Parser::ParserError => e
-          abort("Pione syntax error: " + e.message)
-        rescue Pione::Model::PioneModelTypeError, Pione::Model::VariableBindingError => e
-          abort("Pione model error: " + e.message)
+      # Setup dropbox.
+      def setup_dropbox
+        # start session
+        session = nil
+        consumer_key = nil
+        consumer_secret = nil
+
+        cache = Pathname.new("~/.pione/dropbox_api.cache").expand_path
+        if cache.exist?
+          session = DropboxSession.deserialize(cache.read)
+          Location::Dropbox.set_session(session)
+          consumer_key = session.instance_variable_get(:@consumer_key)
+          consumer_secret = session.instance_variable_get(:@consumer_secret)
+        else
+          api = YAML.load(Pathname.new("~/.pione/dropbox_api.yml").expand_path.read)
+          consumer_key = api["key"]
+          consumer_secret = api["secret"]
+          session = DropboxSession.new(consumer_key, consumer_secret)
+          Location::Dropbox.set_session(session)
+          authorize_url = session.get_authorize_url
+          puts "AUTHORIZING", authorize_url
+          puts "Please visit that web page and hit 'Allow', then hit Enter here."
+          STDIN.gets
+          session.get_access_token
+
+          # cache session
+          cache.open("w+") {|c| c.write session.serialize}
+        end
+
+        # check session state
+        unless session.authorized?
+          abort("We cannot authorize dropbox access to PIONE.")
         end
+
+        # share access token in tuple space
+        Location::Dropbox.share_access_token(tuple_space_server, consumer_key, consumer_secret)
       end
 
-      # Write initial tuples.
+      # Read a package.
       #
       # @return [void]
-      def write_tuples
-        [ Tuple[:process_info].new('standalone', 'Standalone'),
-          Tuple[:dry_run].new(option[:dry_run])
-        ].each {|tuple| @tuple_space_server.write(tuple) }
+      def read_package
+        # package is not found
+        if @argv.first.nil?
+          abort("There are no PIONE documents or packages.")
+        end
+
+        # read package
+        @package = Component::PackageReader.read(Location[@argv.first])
+        @package.upload(option[:output_location] + "package")
+
+        # check rehearse scenario
+        if option[:rehearse] and not(@package.scenarios.empty?)
+          if scenario = @package.find_scenario(option[:rehearse])
+            option[:input_location] = scenario.input
+          else
+            abort "the scenario not found: %s" % option[:rehearse]
+          end
+        end
+      rescue Pione::Parser::ParserError => e
+        abort("Pione syntax error: " + e.message)
+      rescue Pione::Model::PioneModelTypeError, Pione::Model::VariableBindingError => e
+        abort("Pione model error: " + e.message)
       end
 
       # Start agent activities.
       #
       # @return [void]
-      def start_agents
+      def start_precedent_agents
         # messenger
         @messenger = Agent[:messenger].start(@tuple_space_server)
 
@@ -306,7 +336,7 @@ module Pione
 
         # rule provider
         @rule_loader = Agent[:rule_provider].start(@tuple_space_server)
-        @rule_loader.read_document(@document)
+        @rule_loader.read_rules(@package)
         @rule_loader.wait_till(:request_waiting)
 
         # input generators
@@ -317,30 +347,42 @@ module Pione
 
         # command listener
         @command_listener = Agent[:command_listener].start(@tuple_space_server, self)
-      end
 
-      # Wake up tuple space provider process and connect my tuple space server
-      # to it.
-      def start_tuple_space_provider
-        @tuple_space_provider = Pione::TupleSpaceProvider.instance
-        @tuple_space_provider.add_tuple_space_server(@tuple_space_server)
+        # start tuple space provider and connect tuple space server
+        unless option[:without_tuple_space_provider]
+          @start_tuple_space_provider_thread = Thread.new do
+            @tuple_space_provider = Pione::TupleSpaceProvider.instance(@child_process_infos)
+            @tuple_space_provider.add_tuple_space_server(@tuple_space_server)
+          end
+        end
       end
 
       # Start task workers.
-      #
-      # @return [void]
-      def start_workers
+      def start_task_workers
+        features = DocumentTransformer.new.apply(
+          DocumentParser.new.feature_expr.parse(option[:features])
+        )
         option[:task_worker].times do
-          Thread.new {
-            Agent[:task_worker].spawn(Global.front, Util::UUID.generate, option[:features])
-          }
+          if option[:stand_alone]
+            Thread.new do
+              Agent[:task_worker].start(@tuple_space_server, features)
+            end
+          else
+            Thread.new do
+              @child_process_infos << Agent[:task_worker].spawn(Global.front, Util::UUID.generate, option[:features])
+            end
+          end
         end
       end
 
+      # Start process manager agent.
+      def start_process_manager
+        @agent = Agent[:process_manager].start(@tuple_space_server, @package, option[:params], option[:stream])
+        @agent.running_thread.join
+      end
+
       # Connect relay server.
-      #
-      # @return [void]
-      def connect_relay
+      def start_relay_connection
         Global.relay_tuple_space_server = @tuple_space_server
         @relay_ref = DRbObject.new_with_uri(option[:relay])
         @relay_ref.__connect
@@ -361,13 +403,18 @@ module Pione
         abort("You failed authentication to connect the relay server: %s" % @relay_ref.__drburi)
       end
 
-      # Print parameter list of the document.
-      #
-      # @return [void]
-      def print_parameter_list
-        puts "Parameters:"
-        @document.params.data.select{|var, val| var.user_param}.each do |var, val|
-          puts "  %s := %s" % [var.name, val.textize]
+      # Check rehearsal result.
+      def check_rehearsal_result
+        return unless option[:rehearse] and not(@package.scenarios.empty?)
+        return unless scenario = @package.find_scenario(option[:rehearse])
+
+        errors = scenario.validate(option[:output_location])
+        if errors.empty?
+          puts "Rehearsal Result: Succeeded"
+        else
+          puts "Rehearsal Result: Failed"
+          errors.each {|error| puts "- %s" % error.to_s}
+          Global.exit_status = false
         end
       end
     end