xpflow 0.1b → 0.1c

data/bin/xpflow

@@ -47,12 +47,26 @@ def usage
 end
 
 def use(*args)
-    
-    # TODO: this should be more sophisticated
-    # for now it's so so
+    # obsolete, but backwards compatible 'use'
+    return import(*args)
+end
+
+def import_via_name(name)
+    # TODO: it should be more sophisticated
+    name = name.to_s
+    begin
+        Kernel.load("./#{name}")
+    rescue LoadError
+        Kernel.require("xpflow/exts/#{name}_use")
+    end
+end
 
-    args.each do |lib|
-        require "xpflow/exts/#{lib.to_s}_use"
+def import(*args)
+    if args.length == 0
+        raise "Import requires a file or a library"
+    end
+    args.each do |it|
+        import_via_name(it)
     end
 end
 

data/lib/xpflow/cmdline.rb

@@ -131,12 +131,12 @@ module XPFlow
                     @config[:labels] += labels.split(',').map { |x| x.downcase.to_sym }
                 end
                 
-                opts.on("-I", "--ignore-checkpoints", "Ignore automatically saved checkpoints") do
+                opts.on("-i", "--ignore-checkpoints", "Ignore automatically saved checkpoints") do
                     @config[:ignore_checkpoints] = true
                 end
-                # opts.on("-C", "--checkpoint NAME", "Jump to checkpoint NAME (if exists)") do |name|
-                #     @config[:checkpoint] = name
-                # end
+                opts.on("-c", "--checkpoint NAME", "Jump to checkpoint NAME (if exists)") do |name|
+                    @config[:checkpoint] = name
+                end
                 # opts.on("-c", "--list-checkpoints", "List available checkpoints") do
                 #    @config[:instead] += [ RunInfo.new(:list_checkpoints) ]
                 # end

data/lib/xpflow/data.rb

@@ -48,58 +48,87 @@ module XPFlow
             return average_stddev().last
         end
 
-        #Credit for cdf_inverse : http://home.online.no/~pjacklam/notes/invnorm/
+        # START OF DISTRIBUTIONS
+
+        # Credit for cdf_inverse : http://home.online.no/~pjacklam/notes/invnorm/
         # inverse standard normal cumulative distribution function
-        def cdf_inverse(p)
-          a = [0, -3.969683028665376e+01, 2.209460984245205e+02, -2.759285104469687e+02, 1.383577518672690e+02, -3.066479806614716e+01, 2.506628277459239e+00]
-          b = [0, -5.447609879822406e+01, 1.615858368580409e+02, -1.556989798598866e+02, 6.680131188771972e+01, -1.328068155288572e+01]
-          c = [0, -7.784894002430293e-03, -3.223964580411365e-01, -2.400758277161838e+00, -2.549732539343734e+00, 4.374664141464968e+00, 2.938163982698783e+00]
-          d = [0, 7.784695709041462e-03, 3.224671290700398e-01, 2.445134137142996e+00, 3.754408661907416e+00]
-          #Define break-points.
-          p_low  = 0.02425
-          p_high = 1.0 - p_low
-
-          x = 0.0
-          q = 0.0
-          #Rational approximation for lower region.
-          if 0.0 < p && p < p_low
-            q = Math.sqrt(-2.0*Math.log(p))
-            x = (((((c[1]*q+c[2])*q+c[3])*q+c[4])*q+c[5])*q+c[6]) / ((((d[1]*q+d[2])*q+d[3])*q+d[4])*q+1.0)
-
-            #Rational approximation for central region.
-          elsif p_low <= p && p <= p_high
-            q = p - 0.5
-            r = q*q
-            x = (((((a[1]*r+a[2])*r+a[3])*r+a[4])*r+a[5])*r+a[6])*q / (((((b[1]*r+b[2])*r+b[3])*r+b[4])*r+b[5])*r+1.0)
-
-            #Rational approximation for upper region.
-          elsif p_high < p && p < 1.0
-            q = Math.sqrt(-2.0*Math.log(1.0-p))
-            x = -(((((c[1]*q+c[2])*q+c[3])*q+c[4])*q+c[5])*q+c[6]) / ((((d[1]*q+d[2])*q+d[3])*q+d[4])*q+1.0)
-          end
-
-          #The relative error of the approximation has
-          #absolute value less than 1.15 × 10−9.  One iteration of
-          #Halley’s rational method (third order) gives full machine precision.
-          if 0 < p && p < 1
-            e = 0.5 * Math.erfc(-x/Math.sqrt(2.0)) - p
-            u = e * Math.sqrt(2.0*Math::PI) * Math.exp((x**2.0)/2.0)
-            x = x - u/(1.0 + x*u/2.0)
-          end
-          x
-        end
-        def minimal_sample_prel(prec,confidance)
-          minimal_sample_both(prec,nil,confidance)
-        end
-        def minimal_sample_pabs(prec,confidance)
-          minimal_sample_both(nil,prec,condidance)
-        end
-        #vectors is values and prec and confidence is in percentage
-        def minimal_sample_both(prec_rel,prec,confidence)
-          avg = average
-          prec = prec_rel.nil? ? prec : avg*prec_rel
-          critical_value=cdf_inverse((1-confidence)/2)
-          ((critical_value*stddev() / prec) ** 2).to_i + 1
+        def self.cdf_inverse(p)
+            a = [0, -3.969683028665376e+01, 2.209460984245205e+02, -2.759285104469687e+02, 1.383577518672690e+02, -3.066479806614716e+01, 2.506628277459239e+00]
+            b = [0, -5.447609879822406e+01, 1.615858368580409e+02, -1.556989798598866e+02, 6.680131188771972e+01, -1.328068155288572e+01]
+            c = [0, -7.784894002430293e-03, -3.223964580411365e-01, -2.400758277161838e+00, -2.549732539343734e+00, 4.374664141464968e+00, 2.938163982698783e+00]
+            d = [0, 7.784695709041462e-03, 3.224671290700398e-01, 2.445134137142996e+00, 3.754408661907416e+00]
+            #Define break-points.
+            p_low  = 0.02425
+            p_high = 1.0 - p_low
+
+            x = 0.0
+            q = 0.0
+            #Rational approximation for lower region.
+            if 0.0 < p && p < p_low
+                q = Math.sqrt(-2.0*Math.log(p))
+                x = (((((c[1]*q+c[2])*q+c[3])*q+c[4])*q+c[5])*q+c[6]) / ((((d[1]*q+d[2])*q+d[3])*q+d[4])*q+1.0)
+
+                #Rational approximation for central region.
+            elsif p_low <= p && p <= p_high
+                q = p - 0.5
+                r = q*q
+                x = (((((a[1]*r+a[2])*r+a[3])*r+a[4])*r+a[5])*r+a[6])*q / (((((b[1]*r+b[2])*r+b[3])*r+b[4])*r+b[5])*r+1.0)
+
+                #Rational approximation for upper region.
+            elsif p_high < p && p < 1.0
+                q = Math.sqrt(-2.0*Math.log(1.0-p))
+                x = -(((((c[1]*q+c[2])*q+c[3])*q+c[4])*q+c[5])*q+c[6]) / ((((d[1]*q+d[2])*q+d[3])*q+d[4])*q+1.0)
+            end
+
+            #The relative error of the approximation has
+            #absolute value less than 1.15 × 10−9.  One iteration of
+            #Halley’s rational method (third order) gives full machine precision.
+            if 0 < p && p < 1
+                e = 0.5 * Math.erfc(-x/Math.sqrt(2.0)) - p
+                u = e * Math.sqrt(2.0*Math::PI) * Math.exp((x**2.0)/2.0)
+                x = x - u/(1.0 + x*u/2.0)
+            end
+            return x
+        end
+
+        # computes confidence interval, assuming that
+        # the number of measures is large enough
+        # to be approximated with CLT
+        # prec is ABSOLUTE (here and below)
+
+        def _compute_dist(prec, conf, cvalue)
+            m, s = average_stddev()
+            d = (cvalue * s).to_f / (@values.length ** 0.5)
+            sample = ((cvalue * s / prec) ** 2).to_i + 1
+            return {
+                :interval => [ m - d, m + d ],
+                :d => d,
+                :sample => sample
+            }
+        end
+
+        def _dist(name, prec, conf)
+            r = case name
+                when :n then _dist_n(prec, conf)
+                when :t then _dist_t(prec, conf)
+                else
+                    raise "Unknown distribution: #{name}"
+                end
+            return r
+        end
+
+        def _dist_n(prec, conf)
+            cvalue = ValueData.cdf_inverse((1 + conf) * 0.5)
+            return _compute_dist(prec, conf, cvalue)
+        end
+
+        def confidence_interval_n(conf)
+            return _dist_n(1.0, conf)[:interval]
+        end
+
+        # computes the minimal sample size
+        def minimal_sample_n(prec, conf)
+            return _dist_n(prec, conf)[:sample]
         end
 
         TSTUDENT = [
@@ -108,33 +137,31 @@ module XPFlow
             2.074, 2.069, 2.064, 2.060, 2.056, 2.052, 2.048, 2.045, 2.042, 2.021, 2.009,
             2.000, 1.990, 1.984, 1.980, 1.960 ]
 
-        def confidence_interval
-            if size() >= TSTUDENT.length
-                factor = TSTUDENT[-1]
-            else
-                factor = TSTUDENT[size()]
-            end
-            m, s = average_stddev()
-            d = (factor * s).to_f / (@values.length ** 0.5)
-            return [m - d, m + d]
+        def _get_tstudent_factor
+            n = size()
+            raise "size of sample must be positive" if n <= 0
+            return (n < TSTUDENT.length) ? TSTUDENT[n] : TSTUDENT[-1]
         end
 
-        def confidence_precision
-            a, b = confidence_interval()
-            return (b - a) * 0.5
+        def _dist_t(prec, conf)
+            cvalue = _get_tstudent_factor()
+            return _compute_dist(prec, conf, cvalue)
         end
 
-        def confidence_ratio
-            # provides conf. as a percentage around the estimated mean value
-            prec = confidence_precision()
-            mean = average().abs
-            return (prec / mean)
+        # computes confidence interval, assuming that
+        # each measure is normal; therefore their
+        # average is t-student
+        def confidence_interval_t(conf)
+            # TODO: confidence is ignored
+            return _dist_t(1.0, conf)[:interval]
         end
 
-        def conf_ratio
-            return confidence_ratio
+        def minimal_sample_t(prec, conf)
+            return _dist_t(prec, conf)[:sample]
         end
 
+        # END OF DISTRIBUTIONS
+
         def map(&block)
             arr = @values.map(&block)
             return ValueData.new(arr)

data/lib/xpflow/exts/g5k.rb

@@ -204,8 +204,6 @@ module XPFlow; module G5K
 
         def get_sites
             sites = get_items('sites').list
-            # TODO: temporary fix for two sites
-            sites = sites.select { |x| ! [ "Bordeaux", "Reims" ].include?(x['name']) }
             return sites
         end
 

data/lib/xpflow/nodes.rb

@@ -458,7 +458,8 @@ module XPFlow
             :bootstrap_taktuk, :simple_node, :node_range, :taktuk_raw, :test_connectivity,
             :nodes_from_file, :nodes_from_result, :distribute, :chain_copy, :ssh_key,
             :nodes_from_machinefile, :g5k_deploy_keys, :localhost, :file_consistency,
-            :ping_localhost, :ping_node, :g5k_reserve_nodes
+            :ping_localhost, :ping_node, :g5k_reserve_nodes,
+            :g5k_sites
 
         def setup
             nil
@@ -975,21 +976,30 @@ module XPFlow
             return true
         end
 
-        def g5k_reserve_nodes(*args)
-            p = proxy()
+        def with_g5k_lib(p)
             lib = G5K::Library.new
             lib.logging = proc { |x| p.engine.log(x) }
-            lib.proxy = proxy
-            return lib.reserve_nodes(*args)
+            lib.proxy = p
+            return yield(lib)
+
+        end
+
+        def g5k_reserve_nodes(*args)
+            with_g5k_lib(proxy) do |lib|
+                lib.reserve_nodes(*args)
+            end
+        end
+
+        def g5k_sites(*args)
+            with_g5k_lib(proxy) do |lib|
+                lib.sites()
+            end
         end
 
         def g5k_get_avail(opts = {})
-            p = proxy()
-            lib = G5K::Library.new
-            lib.logging = proc { |x| p.engine.log(x) }
-            lib.proxy = proxy
-            job = lib.pick_reservation(opts)
-            return job
+            with_g5k_lib(proxy) do |lib|
+                lib.pick_reservation(opts)
+            end
         end
 
         def nodes_file(user, filepath)

data/lib/xpflow/stdlib.rb

@@ -84,8 +84,9 @@ module XPFlow
             return result
         end
 
-        def value(v)
-            return v
+        def value(*args)
+            return args.first if args.length == 1
+            return args
         end
 
         def code(*args)
@@ -249,9 +250,9 @@ module XPFlow
     class DataLibrary < SyncedActivityLibrary
 
         activities :store, :data, :avg, :sum, :stddev, :gauss,
-            :confidence_precision, :confidence_interval, :save_yaml,
-            :data_vector, :conf_precision, :data_append, :data_push,
-            :run_later, :get_of, :minimal_sample, :minimal_sample_enough
+            :conf_interval, :save_yaml,
+            :data_vector, :data_push,
+            :run_later, :get_of, :minimal_sample, :sample_enough
 
         def data_vector(values = nil)
             return ValueData.new(values)
@@ -318,10 +319,6 @@ module XPFlow
             return m + s * Math.cos(2*Math::PI*x) * p 
         end
 
-        def data_append(data, x)
-            return data.append(x)
-        end
-
         def data_push(data, x)
             data.push(x)
         end
@@ -353,13 +350,54 @@ module XPFlow
         def save_yaml(filename, obj)
             IO.write(filename, obj.to_yaml)
         end
-        def minimal_sample_enough(name,opts={})
-          minimal_sample(name,opts)<name.size
+
+        # opts:
+        #  :dist => distribution type (default: :normal, also: :n, :t, :tstudent)
+        #  :conf => confidence (default: 0.95)
+        #  :rel => relative precision in percents (default: 0.1)
+        #  :abs => absolute precision (e.g., 5)
+
+        def _parse_sample_opts(v, _opts)
+            opts = { :conf => 0.95, :rel => 0.1, :abs => nil, :dist => :n }.merge(_opts)
+            dists = { :n => :n, :normal => :n, :t => :t, :tstudent => :t }
+            opts[:dist] = dists[opts[:dist]]
+            raise "Wrong confidence" if (opts[:conf] <= 0.0 or opts[:conf] >= 1.0)
+            if opts[:abs].nil?
+                opts[:abs] = v.average() * opts[:rel]
+            end
+            opts[:info] = v._dist(opts[:dist], opts[:abs], opts[:conf])
+            return opts
         end
-        def minimal_sample(name,opts={})
-          opts.merge({:conf=>0.95,:abs=>1})
-          name.minimal_sample_both(opts[:rel],opts[:abs],opts[:conf])
+
+        def flatten_data(x)
+            if x.is_a?(Array)
+                return ValueData.new(x)
+            elsif x.is_a?(ValueData)
+                return x
+            else
+                raise "Wrong data type: #{x.class}"
+            end
         end
+
+        def sample_enough(v, opts = {})
+            v = flatten_data(v)
+            opts = _parse_sample_opts(v, opts)
+            info = opts[:info]
+            return info[:d] <= opts[:abs]
+        end
+
+        def minimal_sample(v, opts = {})
+            v = flatten_data(v)
+            opts = _parse_sample_opts(v, opts)
+            return opts[:info][:sample]
+        end
+
+        def conf_interval(v, opts = {})
+            v = flatten_data(v)
+            opts = _parse_sample_opts(v, opts)
+            return opts[:info][:interval]
+        end
+
     end
 
     class TestLibrary < SyncedActivityLibrary

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: xpflow
 version: !ruby/object:Gem::Version
-  version: 0.1b
+  version: 0.1c
   prerelease: 3
 platform: ruby
 authors: