pione 0.3.1 → 0.3.2

data/lib/pione/command.rb

@@ -17,10 +17,12 @@ require 'pione/command/pione-tuple-space-viewer'
 require 'pione/command/pione-relay'
 require 'pione/command/pione-relay-client-db'
 require 'pione/command/pione-relay-account-db'
-require 'pione/command/pione-clean'
+require 'pione/command/pione-clean'               # pione clean
 require 'pione/command/pione-syntax-checker'
-require 'pione/command/pione-log'
-require 'pione/command/pione-val'
+require 'pione/command/pione-log'                 # pione log
+require 'pione/command/pione-val'                 # pione-val
 require 'pione/command/pione-package'
 require 'pione/command/pione-compiler'
-
+require 'pione/command/pione-update-package-info' # pione update-package-info
+require 'pione/command/pione-action'              # pione action
+require 'pione/command/pione-action-list'         # pione action:list

data/lib/pione/global/tuple-space-notifier-variable.rb

@@ -51,7 +51,7 @@ module Pione
       item.desc = "presence notification addresses"
       item.init = ["255.255.255.255:56000"]
       item.define_updater do |vals|
-        vals.map {|val| URI.parse("broadcast://%s" % val)}
+        vals.map {|val| val.is_a?(String) ? URI.parse("broadcast://%s" % val) : val}
       end
     end
 

data/lib/pione/lang/environment.rb

@@ -9,7 +9,7 @@ module Pione
 
       # Return true if the name with the package id is bound.
       def bound?(package_id, name)
-        @table[package_id][name] || (@parent ? @parent.bound?(package_id, name) : false)
+        (@table.has_key?(package_id) and @table[package_id][name]) || (@parent ? @parent.bound?(package_id, name) : false)
       end
 
       # Find value of the reference recursively. We will raise
@@ -55,12 +55,10 @@ module Pione
           # get value from parent table
           return @parent.get_value(env, ref) if @parent
         else
-          # or find by parent package id
-          if parent_ids = env.find_parent_ids(ref.package_id)
-            parent_ids.each do |parent_id|
-              if val = get_value(env, ref.set(package_id: parent_id))
-                return val
-              end
+          # otherwise, find by parent package id
+          env.find_ancestor_ids(ref.package_id).each do |ancestor_id|
+            if val = get_value(env, ref.set(package_id: ancestor_id))
+              return val
             end
           end
         end
@@ -94,6 +92,22 @@ module Pione
         end
       end
 
+      # Return all names that related to the package ID and ancestors.
+      #
+      # @return [Array<String>]
+      #   all names in the table
+      def select_names_by(env, package_id)
+        names = @parent ? @parent.select_names_by(package_id) : []
+        target_ids = [package_id, *env.find_ancestor_ids(package_id)]
+        target_ids.each_with_object(names) do |target_id, names|
+          @table[target_id].keys.each do |name|
+            if not(names.include?(name))
+              names << name
+            end
+          end
+        end
+      end
+
       def inspect
         if @parent
           "#%s(%s,%s)" % [self.class.name.split("::").last, @table, @parent.inspect]
@@ -256,10 +270,20 @@ module Pione
         package_id_table[package_name]
       end
 
-      def find_parent_ids(package_id)
+      # Find ancestor's IDs of the package ID. The way of ancestors search is depth-first.
+      def find_ancestor_ids(package_id)
+        ancestor_ids = []
         if package = package_get(PackageExpr.new(package_id: package_id))
-          package.parent_ids
+          ancestor_ids += package.parent_ids
+          package.parent_ids.each do |parent_id|
+            find_ancestor_ids(parent_id).each do |ancestor_id|
+              if not(ancestor_ids.include?(ancestor_id))
+                ancestor_ids << ancestor_id
+              end
+            end
+          end
         end
+        return ancestor_ids
       end
 
       # Create a new environment which tables are overlayed current tables.
@@ -274,11 +298,17 @@ module Pione
         )
       end
 
-      def merge(param_set)
+      # Merge the parameter set as variables into this environment. Rebinding
+      # variables raise error, but it is ignored when force flag is true.
+      #
+      # @param param_set [Lang::ParameterSet]
+      #   parameter set to be merged
+      # @option option
+      def merge_param_set(param_set, option={})
         param_set.keys.each do |key|
           var = Variable.new(name: key, package_id: current_package_id)
           val = param_set[key]
-          variable_set(var, val)
+          option[:force] ? variable_set!(var, val) : variable_set(var, val)
         end
         return self
       end
@@ -305,16 +335,17 @@ module Pione
         set(current_package_id: package_id, current_definition: definition)
       end
 
-      def make_root_rule(param_set)
-        # set $ROOT_PRARAM_SET
-        variable_set(Variable.new("ROOT_PARAM_SET"), param_set)
+      def make_root_rule(main_param_set)
+        # put variable of parameter set for main rule
+        variable_set(Variable.new("MAIN_PARAM_SET"), ParameterSetSequence.of(main_param_set))
+
         # make root rule
-        Package::Document.parse(<<-PIONE, current_package_id, nil, nil, "*system*").eval(self)
+        Package::Document.parse(<<-PIONE, current_package_id, nil, nil, "*System*").eval(self)
            Rule Root
              input '*'.all or null
              output '*'.all
            Flow
-             rule Main.param($ROOT_PARAM_SET)
+             rule Main.param($MAIN_PARAM_SET)
            End
         PIONE
         rule_get(RuleExpr.new("Root"))

data/lib/pione/lang/parameters.rb

@@ -17,7 +17,13 @@ module Pione
         set(table: table.merge(other.table))
       end
 
+      # Return a parameter set that match the names.
       def filter(names)
+        set(table: table.select{|key, _| names.include?(key)})
+      end
+
+      # Return a parameter set excluding the names.
+      def delete_all(names)
         set(table: table.reject{|key, _| names.include?(key)})
       end
 

data/lib/pione/literate-action/document.rb

@@ -0,0 +1,25 @@
+module Pione
+  module LiterateAction
+    class Document
+      def self.load(location)
+        new(location.read)
+      end
+
+      def initialize(src)
+        @action = Parser.parse(src)
+      end
+
+      # Return action names in the document.
+      def action_names
+        @action.keys
+      end
+
+      # Find target action fromt the name.
+      def find(name)
+        if action = @action[name]
+          Handler.new(action)
+        end
+      end
+    end
+  end
+end

data/lib/pione/literate-action/handler.rb

@@ -0,0 +1,24 @@
+module Pione
+  module LiterateAction
+    class Handler
+      def initialize(action)
+        @lang = action[:lang]
+        @content = action[:content]
+      end
+
+      # Convert the action to a string.
+      def textize(domain_info)
+        @content.to_s
+      end
+
+      # Execute the action.
+      def execute(domain_info, dir=Location[Global.pwd])
+        text = textize(domain_info)
+        location = Location[Temppath.create]
+        location.write(("#!/usr/bin/env %s\n" % @lang) + text)
+        location.path.chmod(0700)
+        `cd #{dir.path}; #{location.path}`
+      end
+    end
+  end
+end

data/lib/pione/literate-action/parser.rb

@@ -0,0 +1,54 @@
+module Pione
+  module LiterateAction
+    class Parser
+      # Parse the source string and return the result.
+      def self.parse(src)
+        new(src).parse
+      end
+
+      def initialize(src)
+        @src = src
+      end
+
+      # Parse the source string.
+      def parse
+        @parsed = Kramdown::Parser::GFM.parse(@src)
+        current_name = nil
+        root = @parsed.first
+        root.children.each_with_object(Hash.new) do |elt, action|
+          if name = find_rule_name(elt)
+            current_name = name
+            next
+          end
+
+          if current_name
+            lang, content = find_action(elt)
+            if content
+              action[current_name] ||= {content: ""}
+              action[current_name][:lang] = lang
+              action[current_name][:content] << content
+            end
+          end
+        end
+      end
+
+      private
+
+      # Find a rule name from the element.
+      def find_rule_name(elt)
+        if elt.type == :header and elt.options[:level] == 2
+          elt.options[:raw_text]
+        end
+      end
+
+      # Find an action from the element.
+      def find_action(elt)
+        if elt.type == :codeblock
+          if elt.attr["class"] and elt.attr["class"].start_with?("language-")
+            return [elt.attr["class"].sub("language-", ""), elt.value]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/pione/literate-action.rb

@@ -0,0 +1,10 @@
+module Pione
+  module LiterateAction
+
+  end
+end
+
+require 'pione/literate-action/parser'   # literate action parser
+require 'pione/literate-action/document' # literate action document
+require 'pione/literate-action/handler'  # action handler
+

data/lib/pione/location/data-location.rb

@@ -191,15 +191,23 @@ module Pione
         raise NotImplementedError
       end
 
-      # Return last modification time of the location.
+      # Return ctime of the location.
       #
       # @return [Time]
-      #   last modification time
+      #   ctime
+      def ctime
+        raise NotImplementedError
+      end
+
+      # Return mtime of the location.
+      #
+      # @return [Time]
+      #   mtime
       def mtime
         raise NotImplementedError
       end
 
-      # Set last modification time of the location.
+      # Set mtime of the location.
       #
       # @return [void]
       def mtime=(time)

data/lib/pione/location/dropbox-location.rb

@@ -90,6 +90,8 @@ module Pione
         @client.file_delete(@path)
       end
 
+      # dropbox have "modified" time only, therefore ctime is not implemented
+
       def mtime
         metadata = @client.metadata(@path)
         Time.parse(metadata["modified"])

data/lib/pione/location/local-location.rb

@@ -60,6 +60,10 @@ module Pione
         @path.mkpath unless exist?
       end
 
+      def ctime
+        @path.exist? ? @path.ctime : (raise NotFound.new(self))
+      end
+
       def mtime
         @path.exist? ? @path.mtime : (raise NotFound.new(self))
       end

data/lib/pione/package/package-exception.rb

@@ -31,7 +31,23 @@ module Pione
     class InvalidPackage < PackageError; end
 
     # InvalidScenario raises when the package is something bad.
-    class InvalidScenario < PackageError; end
+    class InvalidScenario < PackageError
+      def initialize(location, reason)
+        @location = location
+        @reason = reason
+      end
+
+      def message
+        "%<path>s is an invalid scenario document: %<reason>s" % args
+      end
+
+      def args
+        {
+          :path => @location.path,
+          :reason => @reason
+        }
+      end
+    end
 
     class DatabaseError < PackageError; end
   end

data/lib/pione/package/package-handler.rb

@@ -4,13 +4,22 @@ module Pione
     class PackageHandler
       class << self
         # Update package info file and scenario info files.
-        def write_info_files(location)
+        def write_info_files(location, option={})
+          # scan the package directory
           package_info = PackageScanner.new(location).scan
-          (location + "pione-package.json").write(JSON.pretty_generate(package_info))
-          Log::SystemLog.info("write %s" % (location + "pione-package.json").address)
+          last_time = Util::LastTime.get(package_info.filepaths.map{|path| location + path})
+
+          # update the package info file
+          info_location = location + "pione-package.json"
+          if option[:force] or not(info_location.exist?) or last_time > info_location.mtime
+            info_location.write(JSON.pretty_generate(package_info))
+            Log::SystemLog.info("update the package info file: %s" % info_location.address)
+          end
+
+          # write scenario info files
           package_info.scenarios.each do |scenario|
             scenario_info = ScenarioScanner.new(location + scenario).scan
-            ScenarioHandler.new(location + scenario, scenario_info).write_info_file
+            ScenarioHandler.new(location + scenario, scenario_info).write_info_file(option)
           end
         end
       end

data/lib/pione/package/scenario-handler.rb

@@ -17,9 +17,15 @@ module Pione
       end
 
       # Write an information file for the scenario.
-      def write_info_file
-        (@location + "pione-scenario.json").write(JSON.pretty_generate(@info))
-        Log::SystemLog.info("write %s" % (@location + "pione-scenario.json").address)
+      def write_info_file(option={})
+        last_time = Util::LastTime.get(@info.filepaths.map{|path| @location + path})
+
+        # update the scenario info file
+        location = @location + "pione-scenario.json"
+        if option[:force] or not(location.exist?) or last_time > location.mtime
+          location.write(JSON.pretty_generate(@info))
+          Log::SystemLog.info("update the scenario info file: %s" % location.address)
+        end
       end
 
       # Return input location of the scenario. If the scenario doesn't have

data/lib/pione/package/scenario-info.rb

@@ -2,7 +2,7 @@ module Pione
   module Package
     class ScenarioInfo < StructX
       member :name
-      member :param_set
+      member :textual_param_sets
       member :inputs, default: lambda {Array.new}
       member :outputs, default: lambda {Array.new}
 
@@ -10,12 +10,13 @@ module Pione
       # location of the file.
       def self.read(src)
         data = JSON.load(src.is_a?(Location::DataLocation) ? src.read : src)
-        new(name: data["ScenarioName"], param_set: data["ParamSet"], inputs: data["Inputs"], outputs: data["Outputs"])
+        new(name: data["ScenarioName"], textual_param_sets: data["ParamSet"], inputs: data["Inputs"], outputs: data["Outputs"])
       end
 
       # Return file paths of the scenario.
       def filepaths
         list = []
+        list << "Scenario.pione"
         list += inputs
         list += outputs
         return list
@@ -24,7 +25,7 @@ module Pione
       def to_json(*args)
         data = Hash.new
         data["ScenarioName"] = name
-        data["ParamSet"] = param_set
+        data["ParamSet"] = textual_param_sets
         data["Inputs"] = inputs
         data["Outputs"] = outputs
         data.to_json(*args)

data/lib/pione/package/scenario-scanner.rb

@@ -29,20 +29,20 @@ module Pione
           name, param_set = scan_annotations
           inputs = scan_data_dir("input")
           outputs = scan_data_dir("output")
-          ScenarioInfo.new(name: name, param_set: param_set, inputs: inputs, outputs: outputs)
+          ScenarioInfo.new(name: name, textual_param_sets: param_set, inputs: inputs, outputs: outputs)
         end
-      rescue Parslet::ParseFailed => e
-        raise InvalidScenario.new("invalid scenario document because of parser failed: %s" % e.message)
       end
 
       # Scan annotations from scenario file and return scenario name and
       # parameter set.
       def scan_annotations
+        location = @location + "Scenario.pione"
+
         # setup fake language environment
         env = Lang::Environment.new.setup_new_package("ScenarioScanner")
 
         # parse the scenario document
-        Document.load(env, @location + "Scenario.pione", nil, nil, nil, @location + "Scenario.pione")
+        Document.load(env, location, nil, nil, nil, @location + "Scenario.pione")
 
         # get name and parameter set from fake package's annotations
         annotations = env.package_get(Lang::PackageExpr.new(package_id: env.current_package_id)).annotations
@@ -50,6 +50,8 @@ module Pione
         param_set = find_param_set(annotations)
 
         return name, param_set
+      rescue Parslet::ParseFailed => e
+        raise InvalidScenario.new(location, "Parser failed: " + e.message)
       end
 
       # Scan data files.
@@ -67,15 +69,16 @@ module Pione
 
       # Find "ScenarioName" annotaion.
       def find_name(annotations)
+        location = @location + "Scenario.pione"
         names = annotations.select {|annotation| annotation.annotation_type == "ScenarioName"}
         case names.size
         when 0
-          raise InvalidScenario.new("No scenario name in %s" % (@location + "Scenario.pione"))
+          raise InvalidScenario.new(location, "No scenario name exists.")
         when 1
           names.first.value
         else
           name_list = names.map {|name| name.value}.join(", ")
-          raise InvalidScenario.new("Multiple scenario names found: %s" % name_list)
+          raise InvalidScenario.new(location, "Duplicated scenario name found." % name_list)
         end
       end
 
@@ -88,7 +91,8 @@ module Pione
         when 1
           param_sets.first.value
         else
-          raise InvalidScenario.new("Multiple parameter set found in %s" % @location.address)
+          location = @location + "Scenario.pione"
+          raise InvalidScenario.new(location, "Duplicated parameter set found.")
         end
       end
     end

data/lib/pione/rule-engine/basic-handler.rb

@@ -132,7 +132,7 @@ module Pione
         # set current package id
         _env.set(current_package_id: package_id)
         # merge parameter set
-        _env.merge(param_set)
+        _env.merge_param_set(param_set)
 
         ### system environment
         # ENV.each do |key, value|

data/lib/pione/rule-engine/flow-handler.rb

@@ -129,7 +129,7 @@ module Pione
       # Make a task tuple from the application.
       def make_tuple(caller_id)
         features = rule_condition.features.inject(Lang::FeatureSequence.new) do |f, expr|
-          f + f.eval(env)
+          f + expr.eval(env)
         end
         TupleSpace::TaskTuple.new(
           digest,
@@ -229,7 +229,7 @@ module Pione
             pieces.each do |param_set|
               ### merge default parameter values ####
               # setup task's environment by parameter set
-              _env = plain_env.layer.merge(param_set)
+              _env = plain_env.layer.merge_param_set(param_set)
               _env.set(current_package_id: rule.package_id || env.current_package_id)
 
               # get task's condition
@@ -239,9 +239,9 @@ module Pione
               _param_set = param_set.merge_default_values(rule_condition)
 
               # handle parameter distribution
-              _param_set.expand do |expanded_param_set|
+              _param_set.eval(_env).expand do |expanded_param_set|
                 # rebuild environment by expanded param set
-                _env = plain_env.layer.merge(expanded_param_set)
+                _env = plain_env.layer.merge_param_set(expanded_param_set)
                 _env.set(current_package_id: rule.package_id || env.current_package_id)
 
                 # get task's condition
@@ -437,7 +437,8 @@ module Pione
       # Wait until tasks completed.
       def wait_task_completion(tasks)
         tasks.each do |task|
-          # wait to finish the work
+          # wait to finish the distributed task, note that finished tuple is in
+          # the task domain
           finished = read(TupleSpace::FinishedTuple.new(domain: task.domain_id))
 
           ### task completion processing ###

data/lib/pione/test-helper/command-helper.rb

@@ -54,7 +54,7 @@ module Pione
         private
 
         # Run the command execution action.
-        def execute(options, &b)
+        def execute(*options, &b)
           # initialize exit status
           Global.exit_status = true
 
@@ -88,5 +88,48 @@ module Pione
         end
       end
     end
+
+    class PioneClientRunner < StructX
+      member :title
+      member :template, default: "%s"
+      member :args
+      member :base, default: lambda {Pione::Location[Temppath.mkdir]}
+      member :default_arguments
+      member :context
+
+      def self.test(context, &b)
+        # with client mode
+        new(context: context).tap do |runner|
+          runner.default_arguments = ["-o", runner.base.path.to_s]
+          b.call(runner)
+        end
+
+        # with stand alone mode
+        new(context: context, template: "%s with stand alone mode").tap do |runner|
+          runner.default_arguments = ["-o", runner.base.path.to_s, "--stand-alone"]
+          b.call(runner)
+        end
+      end
+
+      def run(&b)
+        _args = args
+        _base = base
+        context.it(template % title) do
+          TestHelper::Command.succeed do
+            Pione::Command::PioneClient.run(_args)
+          end
+          b.call(_base)
+        end
+      end
+
+      def timeout(sec)
+        _args = args + ["--timeout", sec.to_s]
+        context.it(template % title) do
+          TestHelper::Command.fail do
+            Pione::Command::PioneClient.run(_args)
+          end
+        end
+      end
+    end
   end
 end

data/lib/pione/test-helper/internet-connectivity.rb

@@ -0,0 +1,10 @@
+module Pione
+  module TestHelper
+    module InternetConnectivity
+      def self.ok?
+        `ping -c 3 -W 1 8.8.8.8 >> /dev/null`
+        $? == 0
+      end
+    end
+  end
+end

data/lib/pione/test-helper.rb

@@ -28,16 +28,17 @@ module Pione
 end
 
 # load test helpers
-require "pione/test-helper/extension"           # extensions for test
-require "pione/test-helper/webserver"           # fake HTTP server
-require "pione/test-helper/location-helper"     # location test helper
-require "pione/test-helper/command-helper"      # command test helper
-require "pione/test-helper/parser-helper"       # parser test utility
-require "pione/test-helper/transformer-helper"  # transformer test utility
-require "pione/test-helper/lang-helper"         # language test method
-require "pione/test-helper/package-helper"      # pakcage test
-require "pione/test-helper/tuple-helper"        # tuple test
-require "pione/test-helper/tuple-space-helper"  # tuple space
+require "pione/test-helper/extension"             # extensions for test
+require "pione/test-helper/webserver"             # fake HTTP server
+require "pione/test-helper/location-helper"       # location test helper
+require "pione/test-helper/command-helper"        # command test helper
+require "pione/test-helper/parser-helper"         # parser test utility
+require "pione/test-helper/transformer-helper"    # transformer test utility
+require "pione/test-helper/lang-helper"           # language test method
+require "pione/test-helper/package-helper"        # pakcage test
+require "pione/test-helper/tuple-helper"          # tuple test
+require "pione/test-helper/tuple-space-helper"    # tuple space
+require "pione/test-helper/internet-connectivity" # check internet connection
 
 # extend bacon's context
 class Bacon::Context

data/lib/pione/util/digest.rb

@@ -10,7 +10,7 @@ module Pione
         else
           _inputs = "%s,..." % inputs.flatten[0..2].map{|i| i.name}.join(",")
         end
-        _param_set = param_set.filter(["I", "INPUT", "O", "OUTPUT", "*"])
+        _param_set = param_set.delete_all(["I", "INPUT", "O", "OUTPUT", "*"])
         _param_set = _param_set.map{|k,v| "%s:%s" % [k, v.textize]}.join(",")
         "&%s:%s([%s],{%s})" % [package_id, rule_name, _inputs, _param_set]
       end

data/lib/pione/util/id.rb

@@ -5,7 +5,7 @@ module Pione
       # Make a task id by input data names and parameters.
       def generate(inputs, params)
         # NOTE: auto variables are ignored
-        param_set = params.filter(["I", "INPUT", "O", "OUTPUT", "*"])
+        param_set = params.delete_all(["I", "INPUT", "O", "OUTPUT", "*"])
         inputs = inputs.map {|t| t.is_a?(TupleSpace::DataTuple) ? t.name : t}
         Digest::MD5.hexdigest("%s::%s" % [inputs.join(":"), param_set.textize])
       end