datafarming 1.2.0 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a228e8b48eaa2cd95669a37b0954e660e806df0e49bb9050a91ee46273f0cbbb
-  data.tar.gz: 7303e31f70cad28645b92d2196a69770f22d02b7181c0e34ce502dd8c9e75115
+  metadata.gz: 9356e4448e1a6aa818a5361cc04a9149d8e3d78c74ca20aaf2bb1c353e0cbd87
+  data.tar.gz: 0e18063cfe4c92c66ed822debc6a20ef7c9a5b063e3c5769f0647d9109f98034
 SHA512:
-  metadata.gz: 4d214779a72e0c9afdbc97abd54f6f8e027a069f5216189f3d49a1fe108f3e02f880095afd80e3a67c54636309a98b0dc13911fc4da5c2c66addcc90883ae762
-  data.tar.gz: 5565f12ac35ed08841eecb2e775b2ec2021ac9d95c497b6f63a5505586305148672c06ed7a8633b363d6fbeda9d4e559493ebb7ac36969c0dcc9781bb4ebff7a
+  metadata.gz: 82b4d9c0ae4d331b5612bbaa930aabe57fd67b4fdb8b1c3e19f7d2df3904289c6d7adb209fcccef8186ec979d3a876dcca8b093229a69570d5b7f291abb26c61
+  data.tar.gz: b9972a61a075af31a8b59957fac9f308ddea8647ff2035d442c8285c609c01c3ef2a83c65857c11a95f568cbedf3fdee311cd6f8cc1dc7e712d94c103ba36931

data/README.md

@@ -50,4 +50,8 @@ generates star points to augment a fractional factorial
   - `cross.rb` —
 creates a combinatorial design by crossing all combinations of any # of individual smaller designs
   - `mser.rb` —
- uses MSER truncation to remove initial transient effects for time-series output, reports truncated average and number of observations for each run to facilitate construction of a properly weighted confidence interval.
+uses MSER truncation to remove initial transient effects for time-series output, reports truncated average and number of observations for each run to facilitate construction of a properly weighted confidence interval.
+  - `mser_nolbm.rb` —
+uses MSER truncation to remove initial transient effects for time-series output, then calculates a 95% confidence interval on the remaining data using non-overlapping batch means.
+  - `mser_olbm.rb` —
+uses MSER truncation to remove initial transient effects for time-series output, then calculates a 95% confidence interval on the remaining data using overlapping batch means.

data/datafarming.gemspec

@@ -1,10 +1,10 @@
 # -*- ruby -*-
-_VERSION = "1.2.0"
+_VERSION = "1.3.0"
 
 Gem::Specification.new do |s|
   s.name = "datafarming"
   s.version = _VERSION
-  s.date = "2018-07-27"
+  s.date = "2018-08-17"
   s.summary = "Useful scripts for data farming."
   s.homepage = "https://gitlab.nps.edu/pjsanche/datafarmingrubyscripts.git"
   s.email = "pjs@alum.mit.edu"

data/exe/blank2csv.rb

@@ -36,5 +36,5 @@ end.parse!
 
 ErrorHandling.clean_abort help_msg if ARGV[0] == '?'
 
-$-i = '.orig'
+$-i = '.orig' unless ARGV.empty? # specify backup suffix unless STDIN
 ARGF.each { |line| puts line.strip.gsub(/\s+/, ',') }

data/exe/convert_line_endings.rb

@@ -34,5 +34,5 @@ end.parse!
 
 ErrorHandling.clean_abort help_msg if ARGV[0] == '?'
 
-$-i = '.orig'
+$-i = '.orig' unless ARGV.empty? # specify backup suffix unless STDIN
 ARGF.each { |line| puts line.split(/\r\n|\r/) }

data/exe/csv2blank.rb

@@ -35,5 +35,5 @@ end.parse!
 
 ErrorHandling.clean_abort help_msg if ARGV[0] == '?'
 
-$-i = '.orig'
+$-i = '.orig' unless ARGV.empty? # specify backup suffix unless STDIN
 ARGF.each { |line| puts line.strip.tr(',', ' ') }

data/exe/moving_average.rb

@@ -0,0 +1,51 @@
+#!/usr/bin/env ruby -w
+
+# Efficiently calculate a moving average
+
+require 'colorize'
+
+String.disable_colorization false
+
+require 'optparse'
+require 'datafarming/error_handling'
+require 'datafarming/moving_average'
+
+help_msg = [
+  'Calculate moving averages for one or more input files, or for ' +
+    'stdin'.green + '.', 'Results are sent to ' + 'stdout'.green + '.', '',
+  'Syntax:' + "\t#{ErrorHandling.prog_name} [OPTIONS] [FILENAMES...]".yellow, '',
+  "Prefix the command with '" + 'ruby'.yellow +
+    "' if it is not on your PATH.", '',
+  'Options:', ''
+]
+
+opts_help = nil
+options = { column: 0 }
+OptionParser.new do |opts|
+  opts.banner = help_msg.join("\n")
+  opts.on('-h', '-?', '--help', 'Print this help') do
+    puts
+    puts opts
+    puts
+    exit
+  end
+  opts.on('-w LEN',
+          '--window LEN',
+          'Window size to use for moving average',
+          'average ' + '[REQUIRED]'.red) { |win| options[:window] = win.to_i }
+  opts.on('-c COL',
+          '--column COL',
+          'Which column to average.',
+          'Defaults to column 1.') { |col| options[:column] = col.to_i - 1 }
+  opts_help = opts
+end.parse!
+
+ErrorHandling.clean_abort [opts_help] if ARGV[0] == '?' || options[:window].nil?
+
+column = options[:column]
+ma = MovingAverage.new(options[:window])
+ARGF.each do |line|
+  value = line.strip.split(/[,:;]\s*|\s+/).map(&:to_f)
+  result = ma.new_obs(value[column]) unless value[column].nil?
+  puts result unless result.nil?
+end

data/exe/mser_nolbm.rb

@@ -0,0 +1,122 @@
+#!/usr/bin/env ruby -w
+
+require 'rubygems' if RUBY_VERSION =~ /^1\.8/
+require 'colorize'
+
+String.disable_colorization false
+
+require 'optparse'
+require 'datafarming/error_handling'
+
+begin
+  require 'quickstats'
+rescue LoadError
+  ErrorHandling.clean_abort [
+    "\n\tALERT: quickstats gem is not installed!".red,
+    "\tIf you have network connectivity, type:",
+    "\n\t\tgem install quickstats\n".yellow,
+    "\t(Admin privileges may be required.)\n\n"
+  ]
+end
+
+help_msg = [
+  'Calculate confidence intervals using batch means after MSER truncation.',
+  'Results are written to ' + 'stdout'.blue + ' in CSV format, with headers.',
+  '', 'Syntax:',
+  "\n\t#{ErrorHandling.prog_name} [--help] [--column COL] [filenames...]".yellow, '',
+  "Arguments in square brackets are optional.  A vertical bar '|'",
+  'indicates valid alternatives for invoking the option.  Prefix',
+  'the command with "' + 'ruby'.yellow +
+  '" if it is not on your PATH.', '',
+  '  --help | -h | -? | ?'.green,
+  "\tProduce this help message.",
+  '  [--column COL | -c COL]'.green,
+  "\tSpecify column to analyze. (Optional - default is column 1)",
+  '  [filenames...]'.green,
+  "\tThe names of one or more files containing data to be analyzed.",
+  "\t(Optional - uses " + "stdin".blue + " if no files are specified.)"
+]
+
+OPTIONS = {:column => 0}
+
+OptionParser.new do |opts|
+  opts.banner = "Usage: #{$PROGRAM_NAME} [-h|--help] [filenames...]"
+  opts.on('-h', '-?', '--help') { ErrorHandling.clean_abort help_msg }
+  opts.on('-c COL',
+          '--column COL',
+          'Which column to average.',
+          'Defaults to column 1.') { |col| OPTIONS[:column] = col.to_i - 1 }
+end.parse!
+
+ErrorHandling.clean_abort help_msg if ARGV[0] == '?'
+
+def square(x)
+  x * x
+end
+
+T_VALUE = [
+  Float::INFINITY, 12.706, 4.303, 3.182, 2.776,
+  2.571, 2.447, 2.365, 2.306, 2.262,
+  2.228, 2.201, 2.179, 2.160, 2.145,
+  2.131, 2.120, 2.110, 2.101, 2.093,
+  2.086, 2.080, 2.074, 2.069, 2.064,
+  2.060, 2.056, 2.052, 2.048, 2.045, 2.042
+]
+
+def nolbm(data)
+  data.shift if data[0] =~ /[A-Za-z]/ # strip header if one present
+  data.map! { |line| line.strip.split(/[,:;]\s*|\s+/)[OPTIONS[:column]].to_f }
+  m_stats = QuickStats.new
+  warmup = [2 * data.length / 3, data.length - 10].min
+  index = data.length - 1
+  while index > (data.length - warmup) && index > 1
+    m_stats.new_obs(data[index])
+    index -= 1
+  end
+  best = [m_stats.std_err, m_stats.avg, index]
+
+  while index > -1
+    m_stats.new_obs(data[index])
+    best = [m_stats.std_err, m_stats.avg, index] if m_stats.std_err <= best[0]
+    index -= 1
+  end
+
+  avg = best[1]
+  index = best[2]
+  length = data.length - index
+  m = [length / 31, index].max
+  b = length / m
+  m = length / b
+  index += length - m * b
+
+  mean_stats = QuickStats.new
+  batch_means = Array.new(b) do
+    mean_stats.reset
+    m.times do
+      mean_stats.new_obs(data[index])
+      index += 1
+    end
+    mean_stats.avg
+  end
+  sum_squared_deviations = batch_means.
+    map { |y| square(y - avg)}.
+    inject(&:+)
+  var_hat = sum_squared_deviations / (b - 1)
+  df = (b - 1)
+  std_err = Math.sqrt(var_hat / b)
+  half_width = T_VALUE[df] * std_err
+  lower = avg - half_width
+  upper = avg + half_width
+  printf "%f,%f,%d,%f,%f\n", avg, std_err, df, lower, upper
+end
+
+puts "sample_mean,std_err,df,lower95_bound,upper95_bound"
+if ARGF.filename == "-"
+  data = STDIN.readlines
+  nolbm(data)
+else
+  ARGV.each do |fname|
+    data = File.readlines(fname)
+    nolbm(data)
+  end
+end

data/exe/mser_olbm.rb

@@ -0,0 +1,119 @@
+#!/usr/bin/env ruby -w
+
+require 'rubygems' if RUBY_VERSION =~ /^1\.8/
+require 'colorize'
+
+String.disable_colorization false
+
+require 'optparse'
+require 'datafarming/error_handling'
+require 'datafarming/moving_average'
+
+begin
+  require 'quickstats'
+rescue LoadError
+  ErrorHandling.clean_abort [
+    "\n\tALERT: quickstats gem is not installed!".red,
+    "\tIf you have network connectivity, type:",
+    "\n\t\tgem install quickstats\n".yellow,
+    "\t(Admin privileges may be required.)\n\n"
+  ]
+end
+
+help_msg = [
+  'Calculate confidence intervals using overlapping batch means after MSER',
+  'truncation.  Results are written to ' + 'stdout'.blue + ' in CSV format, with headers.',
+  '', 'Syntax:',
+  "\n\t#{ErrorHandling.prog_name} [--help] [--column COL] [filenames...]".yellow, '',
+  "Arguments in square brackets are optional.  A vertical bar '|'",
+  'indicates valid alternatives for invoking the option.  Prefix',
+  'the command with "' + 'ruby'.yellow +
+  '" if it is not on your PATH.', '',
+  '  --help | -h | -? | ?'.green,
+  "\tProduce this help message.",
+  '  [--column COL | -c COL]'.green,
+  "\tSpecify column to analyze. (Optional - default is column 1)",
+  '  [filenames...]'.green,
+  "\tThe names of one or more files containing data to be analyzed.",
+  "\t(Optional - uses " + "stdin".blue + " if no files are specified.)"
+]
+
+OPTIONS = {:column => 0}
+
+OptionParser.new do |opts|
+  opts.banner = "Usage: #{$PROGRAM_NAME} [-h|--help] [filenames...]"
+  opts.on('-h', '-?', '--help') { ErrorHandling.clean_abort help_msg }
+  opts.on('-c COL',
+          '--column COL',
+          'Which column to average.',
+          'Defaults to column 1.') { |col| OPTIONS[:column] = col.to_i - 1 }
+end.parse!
+
+ErrorHandling.clean_abort help_msg if ARGV[0] == '?'
+
+def square(x)
+  x * x
+end
+
+T_VALUE = [
+  Float::INFINITY, 12.706, 4.303, 3.182, 2.776,
+  2.571, 2.447, 2.365, 2.306, 2.262,
+  2.228, 2.201, 2.179, 2.160, 2.145,
+  2.131, 2.120, 2.110, 2.101, 2.093,
+  2.086, 2.080, 2.074, 2.069, 2.064,
+  2.060, 2.056, 2.052, 2.048, 2.045,
+  2.042, 2.040, 2.037, 2.035, 2.032,
+  2.030, 2.028, 2.026, 2.024, 2.023,
+  2.021, 2.020, 2.018, 2.017, 2.015, 2.014
+]
+
+def olbm(data)
+  data.shift if data[0] =~ /[A-Za-z]/ # strip header if one present
+  data.map! { |line| line.strip.split(/[,:;]\s*|\s+/)[OPTIONS[:column]].to_f }
+  mser = QuickStats.new
+  warmup = [2 * data.length / 3, data.length - 10].min
+  index = data.length - 1
+  while index > (data.length - warmup) && index > 1
+    mser.new_obs(data[index])
+    index -= 1
+  end
+  best = [mser.std_err, mser.avg, index]
+
+  while index > -1
+    mser.new_obs(data[index])
+    best = [mser.std_err, mser.avg, index] if mser.std_err <= best[0]
+    index -= 1
+  end
+
+  avg = best[1]
+  start_index = best[2]
+  length = data.length - start_index
+  m = [[length / 21, start_index].max, length / 3].min
+  b = length / m
+  m = length / b
+  start_index += length - m * b
+  ma = MovingAverage.new(m)
+
+  sum_squared_deviations = data[start_index..-1].
+    map { |y| ma.new_obs(y) }.compact.
+    map { |y| square(y - avg)}.
+    inject(&:+)
+
+  se_sqr = (m.to_f / ((length - m) * (length - m + 1))) * sum_squared_deviations
+  df = (3 * (b - 1) * (1 + (b - 1.0)**(-0.5 - 0.6 * b))).to_i / 2
+  half_width = T_VALUE[df] * Math.sqrt(se_sqr)
+  lower = avg - half_width
+  upper = avg + half_width
+  printf "%f,%f,%d,%f,%f\n", avg, Math.sqrt(se_sqr), df, lower, upper
+end
+
+puts "sample_mean,std_err,df,lower95_bound,upper95_bound"
+if ARGF.filename == "-"
+  data = STDIN.readlines
+  olbm(data)
+else
+  ARGV.each do |fname|
+    data = File.readlines(fname)
+    olbm(data)
+  end
+end

data/exe/stripheaderdups.rb

@@ -37,7 +37,7 @@ end.parse!
 
 ErrorHandling.clean_abort help_msg if ARGV[0] == '?'
 
-$-i = '.orig' # specify backup suffix
+$-i = '.orig' unless ARGV.empty? # specify backup suffix unless STDIN
 
 oldfilename = ''
 header = ''

data/exe/stripheaders.rb

@@ -36,7 +36,7 @@ end.parse!
 
 ErrorHandling.clean_abort help_msg if ARGV[0] == '?'
 
-$-i = '.orig' # specify backup suffix
+$-i = '.orig' unless ARGV.empty? # specify backup suffix unless STDIN
 
 oldfilename = ''
 

data/lib/datafarming/factorial_generator.rb

@@ -13,8 +13,8 @@ require 'fwt'
 #     corresponds to a factor and each row is a design point.
 #
 # Author:: Paul J Sanchez (mailto:pjs@alum.mit.edu)
-# Copyright:: Copyright (c) Paul J Sanchez
-# License:: LGPL
+# Copyright:: Copyright (c) 2018 Paul J Sanchez
+# License:: MIT
 #
 def make_design(number_of_factors)
   index = [1, 2, 4, 8, 15, 16, 32, 51, 64, 85, 106, 128, 150, 171,

data/lib/datafarming/moving_average.rb

@@ -0,0 +1,63 @@
+#!/usr/bin/env ruby -w
+
+# A class to efficiently perform Moving Average calculations.
+# Calculating moving averages of length m on a set of data of length n
+# requires Θ(m) storage and Θ(n) work.
+#
+# Author:: Paul J Sanchez (mailto:pjs at alum.mit.edu)
+# Copyright:: Copyright (c) 2018 Paul J Sanchez
+# License:: MIT
+#
+class MovingAverage
+  # Number of elements in the moving average
+  attr_reader :m
+
+  # Initialize the MovingAverage object.
+  #
+  # *Arguments*::
+  #   - +m+ -> the number of elements to be averaged
+  #
+  # *Raises*::
+  #   - RuntimeError if +m+ < 1
+  #
+  def initialize(m)
+    fail 'Number of terms to avg (m) must be strictly positive' if m < 1
+    @m = m
+    @current_set = Array.new(@m)
+    @current_avg = 0.0
+    @current_count = 0
+  end
+
+  # Add a new observation, get the resulting moving average.
+  #
+  # *Arguments*::
+  #   - +x+ -> the number of elements to be averaged
+  #
+  # *Raises*::
+  #   - RuntimeError if +x+ is non-numeric
+  #
+  # *Returns*::
+  #   - Average of the last +m+ observations, or +nil+ if fewer than +m+ values have been processed.
+  #
+  def new_obs(x)
+    x = x.to_f
+    if @current_count < @m
+      @current_set[@current_count] = x
+      @current_count += 1
+      @current_avg += (x - @current_avg) / @current_count
+      @current_count == @m ? @current_avg : nil
+    else
+      @current_set << x
+      @current_avg += (x - @current_set.shift) / @m
+    end
+  end
+end
+
+# Simple but effective test case
+if __FILE__ == $PROGRAM_NAME
+  puts "Ruby v" + RUBY_VERSION
+  ma = MovingAverage.new(3)
+  results = []
+  20.times { |i| avg = ma.new_obs(i); results << avg if avg }
+  puts results.join ', '
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: datafarming
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.3.0
 platform: ruby
 authors:
 - Paul J Sanchez
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-07-27 00:00:00.000000000 Z
+date: 2018-08-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fwt
@@ -63,7 +63,10 @@ executables:
 - convert_line_endings.rb
 - cross.rb
 - csv2blank.rb
+- moving_average.rb
 - mser.rb
+- mser_nolbm.rb
+- mser_olbm.rb
 - pool_files.rb
 - rundesign_general.rb
 - scaled_fde.rb
@@ -84,7 +87,10 @@ files:
 - exe/convert_line_endings.rb
 - exe/cross.rb
 - exe/csv2blank.rb
+- exe/moving_average.rb
 - exe/mser.rb
+- exe/mser_nolbm.rb
+- exe/mser_olbm.rb
 - exe/pool_files.rb
 - exe/rundesign_general.rb
 - exe/scaled_fde.rb
@@ -96,6 +102,7 @@ files:
 - lib/datafarming/error_handling.rb
 - lib/datafarming/factorial_generator.rb
 - lib/datafarming/freq_sets.rb
+- lib/datafarming/moving_average.rb
 - lib/datafarming/nolh_designs.rb
 homepage: https://gitlab.nps.edu/pjsanche/datafarmingrubyscripts.git
 licenses: