pione 0.1.2 → 0.1.3

data/.gitignore

@@ -11,4 +11,5 @@ vendor/bundle
 pkg
 \#*
 nohup.out
-Gemfile.lock
+Gemfile.lock
+gems

data/History.txt

@@ -1,5 +1,16 @@
 # History
 
+## 0.1.3(2013/04/XX)
+
+* Added new data expression literal "null" for accepting data nonexistence
+* Added new data expression property keyword "neglect" and "care" for the update criterion about data mtime
+* Changed log location to be in output location
+* Changed log format to JSON
+* Added pione-log command that generates agent activity log as XES format
+* Use locations in tuples
+* Enabled to test FTP location handling without setting up FTP server
+* Added the feature to create domain information files(".domain_info")
+
 ## 0.1.2(2013/04/01)
 
 * Added shorthand notation for setting parameters

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2012 NaU Data Institute Inc.
+Copyright (c) 2012-2013 NaU Data Institute Inc and contributors.
 
 MIT License
 

data/README.md

@@ -84,7 +84,7 @@ because these are patches including original codes:
 
 ## Links
 
-* [PIONE project homepage](http://pione.github.com/)
+* [PIONE project homepage](http://pione.github.io/)
     * [repository on github](https://github.com/pione/pione)
 * [Yasunaga Laboratory](http://www.yasunaga-lab.bio.kyutech.ac.jp/)
     * [EOS](http://www.yasunaga-lab.bio.kyutech.ac.jp/Eos/)([ja](http://www.yasunaga-lab.bio.kyutech.ac.jp/EosJ/))

data/bin/pione-log

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+# -*- ruby -*-
+
+require 'pione'
+Pione::Command::PioneLog.run

data/example/CountChar/CountChar.pione

@@ -1,5 +1,5 @@
 Rule Main
-  input '*.txt:terminate.cmd'.except('summary.txt').all
+  input ('*.txt' or 'terminate.cmd').except('summary.txt').all
   output '*.count'.all
   output 'summary.txt'
 Flow

data/example/SieveOfEratosthenes/SieveOfEratosthenes.pione

@@ -1,61 +1,62 @@
+param $MAX := 20
+
 Rule Main
-  output '*.prime'
-  param $MAX := 20
+  output '*.prime'.all
 Flow
-  rule CurrentNumber.params({NUM: 2})
-  rule Sieve.params({MAX: $MAX, NUM: 2})
+  rule Sieve
 End
 
 Rule Sieve
-  input '{$NUM}.current'
-  input '*.not_prime'.all or null
-  output '*.prime'
-  param $MAX
-  param $NUM
+  input '*-*.not_prime'.all or null
+  output '*.current'.all or '*.prime'.all
+  param $NUM := 2
 Flow
-  if $NUM <= $MAX
-    if $*.include?($NUM.as_string).not
-      rule CreateNotPrimeAll.params({MAX: $MAX, NUM: $NUM})
-      rule CreatePrime.params({NUM: $NUM})
+  $NEXT_NUM := $NUM + 1
+  $NEXT_RULE := Sieve {NUM: $NEXT_NUM}
+  rule Current {N: $NUM}
+  if not :: $*.as_data_expr.match?($NUM.as_string)
+    rule CreatePrime {N: $NUM}
+    if ($NUM * $NUM) <= $MAX
+      rule CreateNotPrimeAll {P: $NUM} >>> $NEXT_RULE
     else
-      rule CurrentNumber.params({NUM: $NUM + 1})
+      rule $NEXT_RULE
+    end
+  else
+    if $NEXT_NUM <= $MAX
+      rule $NEXT_RULE
     end
-    rule Sieve.params({MAX: $MAX, NUM: $NUM + 1})
   end
 End
 
-Rule CurrentNumber
-  output '{$NUM}.current'
-  param $NUM
-Action
-  touch {$NUM}.current
-End
-
 Rule CreateNotPrimeAll
-  output '*.not_prime'
-  output '{$NUM + 1}.current'
-  param $MAX
-  param $NUM
-  param $i := 2
+  output '*-{$P}.not_prime'.all
+  param $P
+  param $INDEX := 2
 Flow
-  if $NUM * ($i + 1) <= $MAX
-    rule CreateNotPrimeAll.params({MAX: $MAX, NUM: $NUM, i: $i + 1})
+  if ($P * ($INDEX + 1)) <= $MAX
+    rule CreateNotPrimeAll {P: $P, INDEX: $INDEX + 1}
   end
-  rule CreateNotPrime.params({NUM: $NUM * $i})
-  rule CurrentNumber.params({NUM: $NUM + 1})
+  rule CreateNotPrime {P: $P, N: $P * $INDEX}
 End
 
 Rule CreateNotPrime
-  output '{$NUM}.not_prime'
-  param $NUM
+  output '{$N}-{$P}.not_prime'
+  param $P
+  param $N
 Action
-  touch {$NUM}.not_prime
-end
+  touch {$N}-{$P}.not_prime
+End
 
 Rule CreatePrime
-  output '{$NUM}.prime'
-  param $NUM
+  output '{$N}.prime'
+  param $N
 Action
-  touch {$NUM}.prime
+  touch {$N}.prime
 End
 
+Rule Current
+  output '{$N}.current'
+  param $N
+Action
+  touch {$N}.current
+End

data/lib/pione.rb

@@ -5,7 +5,7 @@ require 'bundler/setup'
 require 'set'
 require 'socket'
 require 'digest'
-require 'forwardable'
+require 'forwardablex'
 require 'socket'
 require 'drb/drb'
 require 'drb/ssl'
@@ -22,6 +22,7 @@ require 'pathname'
 require 'time'
 require 'etc'
 require 'json'
+require 'rexml/document'
 
 require 'uuidtools'
 require 'parslet'
@@ -31,6 +32,8 @@ require 'highline'
 require 'dropbox_sdk'
 require 'hamster'
 require 'naming'
+require 'temppath'
+require 'xes'
 
 #
 # load pione
@@ -43,10 +46,12 @@ require 'pione/version'
 require 'pione/util/misc'
 require 'pione/util/terminal'
 require 'pione/util/console-message'
-require 'pione/util/log'
 require 'pione/util/waiter-table'
 require 'pione/util/error-report'
 
+# log
+require 'pione/log'
+
 # patch
 require 'pione/patch/array-patch'
 require 'pione/patch/drb-patch'
@@ -73,6 +78,9 @@ require 'pione/uri-scheme/local-scheme'
 require 'pione/uri-scheme/dropbox-scheme'
 require 'pione/uri-scheme/broadcast-scheme'
 
+# location
+require 'pione/location'
+
 # relay
 require 'pione/relay/transmitter-socket'
 require 'pione/relay/trampoline'
@@ -111,7 +119,7 @@ require 'pione/tuple/agent-tuple'
 require 'pione/tuple/data-tuple'
 require 'pione/tuple/finished-tuple'
 require 'pione/tuple/process-info-tuple'
-require 'pione/tuple/shift-tuple'
+require 'pione/tuple/lift-tuple'
 require 'pione/tuple/working-tuple'
 require 'pione/tuple/attribute-tuple'
 require 'pione/tuple/bye-tuple'
@@ -119,7 +127,7 @@ require 'pione/tuple/dry-run-tuple'
 require 'pione/tuple/foreground-tuple'
 require 'pione/tuple/request-rule-tuple'
 require 'pione/tuple/task-tuple'
-require 'pione/tuple/base-uri-tuple'
+require 'pione/tuple/base-location-tuple'
 require 'pione/tuple/command-tuple'
 require 'pione/tuple/exception-tuple'
 require 'pione/tuple/log-tuple'
@@ -150,12 +158,6 @@ require 'pione/parser/document-parser'
 # transformer
 require 'pione/transformer'
 
-# resource
-require 'pione/resource/basic-resource'
-require 'pione/resource/local'
-require 'pione/resource/ftp'
-require 'pione/resource/dropbox-resource'
-
 # rule-handler
 require 'pione/rule-handler/basic-handler'
 require 'pione/rule-handler/flow-handler'
@@ -188,33 +190,10 @@ require 'pione/front/tuple-space-receiver-front'
 require 'pione/front/relay-front'
 
 # command-option
-require 'pione/command-option/basic-option'
-require 'pione/command-option/common-option'
-require 'pione/command-option/daemon-option'
-require 'pione/command-option/child-process-option'
-require 'pione/command-option/presence-notifier-option'
-require 'pione/command-option/tuple-space-provider-option'
-require 'pione/command-option/tuple-space-provider-owner-option'
-require 'pione/command-option/tuple-space-receiver-option'
-require 'pione/command-option/task-worker-owner-option'
+require 'pione/option'
 
 # command
-require 'pione/command/basic-command'
-require 'pione/command/front-owner-command'
-require 'pione/command/daemon-process'
-require 'pione/command/child-process'
-require 'pione/command/pione-client'
-require 'pione/command/pione-task-worker'
-require 'pione/command/pione-broker'
-require 'pione/command/pione-tuple-space-provider'
-require 'pione/command/pione-tuple-space-receiver'
-require 'pione/command/pione-tuple-space-viewer'
-require 'pione/command/pione-relay'
-require 'pione/command/pione-relay-client-db'
-require 'pione/command/pione-relay-account-db'
-require 'pione/command/pione-clean'
-require 'pione/command/pione-syntax-checker'
-
+require 'pione/command'
 
 #
 # other settings

data/lib/pione/agent/input-generator.rb

@@ -16,7 +16,7 @@ module Pione
           @tuple_space_server.now
         end
 
-        # Generates an input.
+        # Generate an input.
         def generate
           raise RuntimeError
         end
@@ -29,22 +29,18 @@ module Pione
         end
       end
 
-      # Directory based generator.
+      # DirGeneratorMethod is a directory based generator.
       class DirGeneratorMethod < GeneratorMethod
         # Create a generator.
-        # @param [TupleSpaceServer] ts_server
+        #
+        # @param tuple_space_server [TupleSpaceServer]
         #   tuple space server
-        # @param [URI] dir_path
-        #   directory URI for loading target
-        def initialize(ts_server, dir_path)
-          raise TypeError.new(dir_path) unless dir_path.kind_of?(URI) or dir_path.nil?
-          super(ts_server)
-          @dir_path = dir_path
-          if dir_path
-            @gen = Resource[@dir_path].entries.to_enum
-          else
-            @gen = [].each
-          end
+        # @param dir [BasicLocation]
+        #   input directory location
+        def initialize(tuple_space_server, dir)
+          raise Argument.new(dir) unless dir.kind_of?(Location::BasicLocation) or dir.nil?
+          super(tuple_space_server)
+          @gen = dir ? dir.entries.to_enum : [].each
         end
 
         def generate
@@ -55,10 +51,10 @@ module Pione
 
       # StreamGeneratorMethod handles stream inputs.
       class StreamGeneratorMethod < GeneratorMethod
-        def initialize(ts_server, dir_path)
-          raise TypeError.new(dir_path) unless dir_path.kind_of?(URI) or dir_path.nil?
+        def initialize(ts_server, dir)
+          raise TypeError.new(dir) unless dir.kind_of?(Location) or dir.nil?
           super(ts_server)
-          @dir_path = dir_path
+          @dir = dir
           @table = Hash.new
           init
         end
@@ -78,11 +74,7 @@ module Pione
         #
         # @return [void]
         def init
-          if @dir_path
-            @gen = Resource[@dir_path].entries.to_enum
-          else
-            @gen = [].each
-          end
+          @gen = dir ? @dir.entries.to_enum : [].each
         end
 
         # @api private
@@ -102,11 +94,6 @@ module Pione
         end
       end
 
-      # Create a input generator agent by simple method.
-      def self.start_by_simple(ts_server, *args)
-        start(ts_server, SimpleGeneratorMethod.new(ts_server, ts_server.base_uri, *args))
-      end
-
       # Create a input generator agent by directory method.
       def self.start_by_dir(ts_server, *args)
         start(ts_server, DirGeneratorMethod.new(ts_server, *args))
@@ -121,8 +108,7 @@ module Pione
       define_state :sleeping
       define_state :stop_iteration
 
-      define_state_transition :initialized => :reading_base_uri
-      define_state_transition :reading_base_uri => :generating
+      define_state_transition :initialized => :generating
       define_state_transition :generating => :generating
       define_state_transition :stop_iteration => lambda{|agent, res|
         agent.stream? ? :sleeping : :terminated
@@ -148,26 +134,38 @@ module Pione
 
       private
 
-      def transit_to_reading_base_uri
-        @base_uri = read(Tuple[:base_uri].any).uri
+      def transit_to_initialized
+        @base_location = base_location
       end
 
       # State generating generates a data from generator and puts it into tuple
       # space.
       def transit_to_generating
         if input = @generator.generate
+          # put into history
           @inputs << input
-          # log
-          log do |msg|
-            msg.add_record(agent_type, "action", "generate_input_data")
-            msg.add_record(agent_type, "uuid", uuid)
-            msg.add_record(agent_type, "object", input.name)
+
+          # build original location
+          orig_location = Location[input.uri]
+
+          # build input location
+          input_location = @base_location + File.join("input", input.name)
+
+          # make process log record
+          record = Log::PutDataProcessRecord.new.tap do |record|
+            record.agent_type = agent_type
+            record.agent_uuid = uuid
+            record.location = input_location
+            record.size = orig_location.size
           end
+
           # upload the file
-          input_uri = @base_uri + File.join("input", input.name)
-          Resource[input_uri].create(Resource[input.uri].read)
-          # make the tuple
-          write(Tuple[:data].new(DOMAIN, input.name, input_uri, input.time))
+          with_process_log(record) do
+            orig_location.copy(input_location)
+          end
+
+          # put data tuple into tuple space
+          write(Tuple[:data].new(DOMAIN, input.name, input_location, input.time))
         end
       end
 

data/lib/pione/agent/logger.rb

@@ -1,17 +1,36 @@
 module Pione
   module Agent
-    # Logger is an agent for logging in tuple space.
+    # Logger is an agent for logging processings in tuple space.
     class Logger < TupleSpaceClient
       set_agent_type :logger
 
-      def initialize(tuple_space_server, out=$stdout)
+      # @return [BasicLocation]
+      attr_reader :location
+
+      # @return [Pathname]
+      attr_reader :out
+
+      # @return [Array<Log::ProcessRecord>]
+      attr_reader :records
+
+      # Create a logger agent.
+      #
+      # @param tuple_space_server [TupleSpaceServer]
+      #   tuple space server
+      # @param location [BasicLocation]
+      #   the path to store log records
+      def initialize(tuple_space_server, location)
         super(tuple_space_server)
-        @out = out
-        @logs = []
+        @location = location
+        @temporary = Location[Pione.temporary_path(@location.basename)]
+        @out = @temporary.path.open("w+")
+        @records = []
       end
 
+      define_state :initialized
       define_state :take
       define_state :store
+      define_state :terminated
 
       define_state_transition :initialized => :take
       define_state_transition :take => :store
@@ -20,43 +39,57 @@ module Pione
       define_exception_handler Exception => :terminated
 
       # Sleeps till the logger clears logs.
+      #
       # @param [Float]
       #   timespan for clearing logs
+      # @return [void]
       def wait_to_clear_logs(timespan=0.1)
-        while count_tuple(Tuple[:log].any) > 0 || @logs.size > 0
+        while count_tuple(Tuple[:log].any) > 0 || @records.size > 0
           sleep timespan
         end
       end
 
+      def store_records
+        unless @records.empty?
+          @records.sort{|a,b| a.timestamp <=> b.timestamp}.each do |record|
+            @out.puts record.format
+          end
+          @out.flush
+          @out.fsync
+          @records = []
+        end
+      end
+
       private
 
-      # Transits to the state +take_log+.
+      # Transits to the state +take+.
       def transit_to_take
         timeout(2) do
-          @logs << take(Tuple[:log].any)
+          loop do
+            if @current_tuple = take(Tuple[:log].any)
+              @records << @current_tuple.message
+              @current_tuple = nil
+            end
+          end
         end
       rescue TimeoutError
-        # ignore
+        if @current_tuple
+          @records << @current_tuple.message
+          @current_tuple = nil
+        end
       end
 
       # Transits to the state +store+.
       def transit_to_store
-        unless @logs.empty?
-          @logs.sort{|a,b| a.timestamp <=> b.timestamp}.each do |log|
-            @out.puts log.message.format
-            @out.flush
-            @out.sync
-          end
-          @logs = []
-        end
+        store_records
       end
 
       # State terminated.
       def transit_to_terminated
+        store_records
+        Util.ignore_exception {@out.close}
+        @temporary.copy(@location)
         super
-        unless @out == STDOUT
-          Util.ignore_exception { @out.close }
-        end
       end
     end