pione 0.3.2 → 0.4.0

data/.gitignore

@@ -9,6 +9,8 @@ Gemfile.lock
 gems
 log.txt
 nohup.out
+test-report.txt
+*.log
 
 /input
 /output

data/.s3cfg

@@ -0,0 +1,42 @@
+[default]
+bucket_location = US
+cloudfront_host = cloudfront.amazonaws.com
+default_mime_type = binary/octet-stream
+delete_removed = False
+dry_run = False
+enable_multipart = True
+encoding = UTF-8
+encrypt = False
+follow_symlinks = False
+force = False
+get_continue = False
+gpg_command = /usr/bin/gpg
+gpg_decrypt = %(gpg_command)s -d --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s
+gpg_encrypt = %(gpg_command)s -c --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s
+gpg_passphrase = 
+guess_mime_type = True
+host_base = s3.amazonaws.com
+host_bucket = %(bucket)s.s3.amazonaws.com
+human_readable_sizes = False
+invalidate_on_cf = False
+list_md5 = False
+log_target_prefix = 
+mime_type = 
+multipart_chunk_size_mb = 15
+preserve_attrs = True
+progress_meter = True
+proxy_host = 
+proxy_port = 0
+recursive = False
+recv_chunk = 4096
+reduced_redundancy = False
+send_chunk = 4096
+simpledb_host = sdb.amazonaws.com
+skip_existing = False
+socket_timeout = 300
+urlencoding_mode = normal
+use_https = False
+verbosity = WARNING
+website_endpoint = http://%(bucket)s.s3-website-%(location)s.amazonaws.com/
+website_error = 
+website_index = index.html

data/.travis.yml

@@ -1,11 +1,25 @@
-before_script:
-  - sudo apt-get install bc # for processing Fib.pione
-
 language: ruby
 rvm:
-#  - "1.9.2" # PIONE doesn't support 1.9.2
   - "1.9.3"
 #  - "2.0.0" # pione doesn't work with 2.0
 #  - jruby-19mode
 #  - rbx-19mode # pione doesn't work with 2.0
 script: rake test
+
+env:
+  global:
+    - secure: "RBvMvbfZYegOQWQ9AoWD1S6MejhyEx61ufLIBBFS4LgGV1cZLvlbEoPLLT3FJsKOtel1UJ/WFLD1CSelzoIaLHCWtDwg58HvKImRTGI/rQRFKpY82P+gW5E9gB80Z74d1bwhKMvC51Unj/EmPU5SinFDaGCwJD871Eedhs1oKMc="
+    - secure: "FvqAAf9yqE5se0xavXIn5huGAee9H//7yUHpGFhF6xJN9HlFJ4ciEu5kDzQIAzFh24qgLZtA+s3gY8Us5h+uW2ztEo/VHsohVK6KvZkT9ucouOepa2ubxbhacTbKJADYng8f1wlaIBt208AD22gkZbAR+/T2CwZMqiskgh5mmg8="
+
+before_script:
+  - LANG=C date >> test-report.txt
+  - sh misc/machine-info.sh >> test-report.txt
+  - sudo apt-get install -y bc      # for processing Fib.pione
+  - sudo apt-get install -y gnuplot # for processing ScoreAggregation.pione
+  - sudo apt-get install -y s3cmd
+
+after_script:
+  - echo "access_key = $TRAVIS_AWS_ACCESS_KEY_ID" >> .s3cfg
+  - echo "secret_key = $TRAVIS_AWS_SECRET_KEY" >> .s3cfg
+  - s3cmd put --rr --config=.s3cfg --acl-public test-report.txt s3://pione-travis-result/pione/test-report-$TRAVIS_JOB_NUMBER.txt
+

data/.yardopts

@@ -0,0 +1 @@
+--markup markdown

data/Gemfile

@@ -6,3 +6,6 @@ gemspec
 
 gem 'simplecov', :require => false, :group => :test
 gem 'coveralls', :require => false, :group => :test
+gem 'clockwork'
+gem 'foreman'
+gem 'god'

data/History.txt

@@ -1,5 +1,19 @@
 # History
 
+## 0.4.0(2014/4/17)
+
+* Added new PNML compiler(`pione compile`).
+* Added experimental `pione-interactive` command.
+* Modified continuation condition for rule application.
+* Changed to stop processing when action rules cause errors.
+* Refined notification system, and added `pione diagnosis notification`.
+* Renamed broker as `pione-task-worker-broker`.
+* Added a new option `--file-sliding`.
+* Added a new option `--file-cache-method`.
+* Introduced `pione-notification-listener`.
+* Fixed SpawnRetry problem with many task workers.
+* Fixed data cache problem.
+
 ## 0.3.2(2013/11/05)
 
 * Added literate action document.

data/Procfile

@@ -0,0 +1,7 @@
+# If you want to make upstart script, try this:
+#
+#     sudo foreman export upstart /etc/init --app pione -d `pwd` -u `whoami` -l `pwd`
+#
+
+notification-listener: pione-notification-listener
+task-worker-broker: pione-task-worker-broker

data/Rakefile

@@ -85,4 +85,29 @@ task "man" do
   generate_man("doc/man/pione-update-package-info.md", "man/pione-update-package-info.1")
   generate_man("doc/man/pione-action.md", "man/pione-action.1")
   generate_man("doc/man/pione-action-list.md", "man/pione-action-list.1")
+  generate_man("doc/man/pione-list-param.md", "man/pione-list-param.1")
+end
+
+#
+# Completion
+#
+
+desc "generate bash completion file"
+task "completion:bash" do
+  Pione::Command.load_all
+  require 'pione/util/completion'
+  misc = Pione::Location[File.dirname(__FILE__)] + "misc"
+  source = misc + "pione-completion.erb"
+  target = misc + "pione-completion.bash"
+  Pione::Util::BashCompletion.compile(source, target)
+end
+
+desc "generate zsh completion file"
+task "completion:zsh" do
+  Pione::Command.load_all
+  require 'pione/util/completion'
+  misc = Pione::Location[File.dirname(__FILE__)] + "misc"
+  source = misc + "pione-completion.erb"
+  target = misc + "pione-completion.zsh"
+  Pione::Util::ZshCompletion.compile(source, target)
 end

data/bin/pione-interactive

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+# -*- ruby -*-
+
+require 'pione'
+require 'pione/command/pione-interactive'
+Pione::Command::PioneInteractive.run(ARGV)

data/bin/{pione-compiler → pione-notification-listener}

@@ -2,4 +2,4 @@
 # -*- ruby -*-
 
 require 'pione'
-Pione::Command::PioneCompiler.run(ARGV)
+Pione::Command::PioneNotificationListener.run(ARGV)

data/bin/{pione-broker → pione-task-worker-broker}

@@ -2,4 +2,4 @@
 # -*- ruby -*-
 
 require 'pione'
-Pione::Command::PioneBroker.run(ARGV)
+Pione::Command::PioneTaskWorkerBroker.run(ARGV)

data/doc/man/pione-list-param.md

@@ -0,0 +1,36 @@
+% pione-list-param(1) PIONE User Manual
+% Keita Yamaguchi
+
+# NAME
+
+pione-list-param - show a list of parameters in the document or package.
+
+# SYNOPSIS
+
+**pione list-param** [options] location
+
+# DESCRIPTION
+
+**list-param** command shows a list of parameters in the document or
+package. The list includes basic parameters only by default, set `--advanced` option
+if you want to show advanced parameters.
+
+# OPTIONS
+
+-a, --advanced
+:   Show advanced parameters.
+
+# EXAMPLES
+
+pione list-param example/HelloWorld/HelloWorld.pione
+:    Show basic parameters in the document `example/HelloWorld/HelloWorld.pione`.
+
+pione list-param example/HelloWorld
+:    Show basic parameters in the package `example/HelloWorld`.
+
+pione list-param --advanced example/HelloWorld
+:    Show basic and advanced parameters in the package `example/HelloWorld`.
+
+# REPORTING BUGS
+
+Report bugs or feature requests to PIONE's issue tracker(https://github.com/pione/pione/issues).

data/example/ActionError/ActionError.pione

@@ -0,0 +1,9 @@
+.@ PackageName :: "ActionError"
+
+Rule Main
+  input null
+  output 'o1'
+Action
+  sleep 3
+  hoge.sh > o1
+End

data/example/ActionError/pione-package.json

@@ -0,0 +1,15 @@
+{
+  "PackageName": "ActionError",
+  "Parents": [
+
+  ],
+  "Documents": [
+    "ActionError.pione"
+  ],
+  "Scenarios": [
+
+  ],
+  "Bins": [
+
+  ]
+}

data/example/CountChar/CountChar.pione

@@ -46,7 +46,7 @@ Action
 #!/usr/bin/env ruby
 # coding: utf-8
   table = {}
-  "{$I[1]}".split(":").each do |path|
+  "{$I[1]}".split(" ").each do |path|
     File.read(path).split("\n").map do |line|
       c, number = line.split(":")
       table[c] = (table.has_key?(c) ? table[c] : 0) + number.to_i

data/example/ScoreAggregation/ScoreAggregation.pione

@@ -111,7 +111,7 @@ Rule MD2HTML
   input '*.stat.md'
   output '{$*}.html.part'
 Action
-  redcarpet --parse-tables {$I[1]} > {$O[1]}
+  kramdown {$I[1]} > {$O[1]}
 End
 
 Rule ApplyTemplate

data/example/Touch/pione-package.json

@@ -0,0 +1,15 @@
+{
+  "PackageName": "Touch",
+  "Parents": [
+
+  ],
+  "Documents": [
+    "Touch.pione"
+  ],
+  "Scenarios": [
+
+  ],
+  "Bins": [
+
+  ]
+}

data/lib/pione.rb

@@ -26,6 +26,7 @@ require 'socket'
 require 'digest'
 require 'forwardablex'
 require 'socket'
+require 'ipaddr'
 require 'drb/drb'
 require 'drb/ssl'
 require 'rinda/rinda'
@@ -35,6 +36,7 @@ require 'yaml'
 require 'singleton'
 require 'timeout'
 require 'thread'
+require 'thwait'
 require 'monitor'
 require 'uri'
 require 'pathname'
@@ -71,10 +73,14 @@ require 'retriable'
 require 'childprocess'
 require 'lettercase/core_ext'
 require 'kramdown'
+require 'os'
+require 'sys/proctable'
 
 # configuration for childprcess
 ChildProcess.posix_spawn = true
 
+require 'rootage'
+
 #
 # load pione
 #
@@ -82,6 +88,7 @@ ChildProcess.posix_spawn = true
 require 'pione/version'         # PIONE version information
 require 'pione/util'            # various helper functions
 require 'pione/patch'           # patches for libraries
+require 'pione/notification'    # PIONE notification system
 require 'pione/location'        # location system for data and package
 require 'pione/log'             # log and format
 require 'pione/global'          # global variable manager
@@ -92,14 +99,19 @@ require 'pione/lang'            # PIONE languge
 require 'pione/tuple-space'     # tuple space functions
 require 'pione/rule-engine'     # rule processing behaviors
 require 'pione/agent'           # agent system
+require 'pione/model'           # command model
 require 'pione/front'           # command front interface
 require 'pione/command'         # command definitions
 require 'pione/literate-action' # literate action
+require 'pione/task-worker-broker' # task worker broker
+require 'pione/pnml'            # PNML compiler
 
 #
 # other settings
 #
 
+Version = Pione::VERSION
+
 module Pione
   # expand name spaces
   include Relay

data/lib/pione/agent.rb

@@ -10,8 +10,8 @@ require 'pione/agent/job-terminator'
 require 'pione/agent/task-worker'
 require 'pione/agent/input-generator'
 require 'pione/agent/logger'
-require 'pione/agent/broker'
-require 'pione/agent/process-manager'
+require 'pione/agent/task-worker-broker'
+require 'pione/agent/job-manager'
 require 'pione/agent/messenger'
 require 'pione/agent/tuple-space-provider'
-require 'pione/agent/tuple-space-receiver'
+require 'pione/agent/notification-listener'

data/lib/pione/agent/basic-agent.rb

@@ -124,7 +124,7 @@ module Pione
         @__wait_until_after_cv__ = Hash.new {|h, k| h[k] = ConditionVariable.new}
       end
 
-      # Start agent activity.
+      # Start the agent activity.
       def start
         unless @chain_threads.list.empty?
           raise TerminationError.new(self, states)
@@ -140,6 +140,12 @@ module Pione
         return self
       end
 
+      # Start the agent activity and wait the termination.
+      def start!
+        start
+        wait_until_terminated(nil)
+      end
+
       # Fire the transtion with inputs.
       def transit(transition, transition_inputs)
         # wake up threads that wait by wait_until_before method
@@ -278,9 +284,9 @@ module Pione
               end
             end
           rescue Exception => e
-            # throw the exception to command's runnning thread
-            if Global.command and Global.command.running_thread and Global.command.running_thread.alive?
-              Global.command.running_thread.raise e
+            # throw the exception to owner thread
+            if @__owner_thread__ and @__owner_thread__.alive?
+              @__owner_thread__.raise e
             else
               raise e
             end

data/lib/pione/agent/input-generator.rb

@@ -35,13 +35,13 @@ module Pione
       forward :@generator, :stream?
 
       # Initialize the agent.
-      def initialize(space, generator_name, input_location, stream)
-        super(space)
+      def initialize(tuple_space, generator_name, input_location, stream)
+        super(tuple_space)
         @base_location = base_location
 
         # generator method
         if generator_method = InputGenerator.generator_method[generator_name]
-          @generator = generator_method.new(space, input_location, stream)
+          @generator = generator_method.new(tuple_space, input_location, stream)
         else
           raise UnknownInputGeneratorMethod.new(generator_name)
         end

data/lib/pione/agent/{process-manager.rb → job-manager.rb}

@@ -1,8 +1,8 @@
 module Pione
   module Agent
-    class ProcessManager < TupleSpaceClient
+    class JobManager < TupleSpaceClient
       include Log::MessageLog
-      set_agent_type :process_manager, self
+      set_agent_type :job_manager, self
 
       #
       # instance method
@@ -10,13 +10,13 @@ module Pione
 
       attr_reader :package
 
-      def initialize(space, env, package, param_set, stream)
+      def initialize(tuple_space, env, package, param_set, stream)
         unless env.rule_get!(Lang::RuleExpr.new("Main"))
           raise JobError.new("Rule `Main` not found in the package.")
         end
 
-        super(space)
-        @space = space
+        super(tuple_space)
+        @tuple_space = tuple_space
         @env = env
         @package = package
         @param_set = param_set
@@ -61,7 +61,7 @@ module Pione
       end
 
       def transit_to_run
-        finder = RuleEngine::DataFinder.new(@space, 'root')
+        finder = RuleEngine::DataFinder.new(@tuple_space, 'root')
         list = Enumerator.new(finder, :find, :input, @rule_condition.inputs, @env).to_a
         if list.empty?
           user_message "error: no inputs"
@@ -70,11 +70,14 @@ module Pione
           # call root rule of the current package
           list.each do |env, inputs|
             package_id = @env.current_package_id
-            handler = RuleEngine.make(@space, @env, package_id, "Root", inputs, Lang::ParameterSet.new, 'root', nil)
+            handler = RuleEngine.make(@tuple_space, @env, package_id, "Root", inputs, Lang::ParameterSet.new, 'root', nil)
             handler.handle
           end
         end
 
+        # terminate if the agent is not stream mode
+        terminate unless @stream
+
         return
       end
     end