datafarming 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 5aa6e5a57e29e45261e3322740ae3694c0c5eb7e32f841b71a15fb4da631d2d1
+  data.tar.gz: c7d299e10278175d1a87020f9e84bc91d697754ab981c1af07174d25c8467e9a
+SHA512:
+  metadata.gz: fffc9ed4734f4d1a8ee9ebd18bacbe7f9f679ade49b8910e61ac751de2adb8f7ec5bd2475b7572597d31307f8ab08d425fb2db176438527a4c68264d0e8fdbd6
+  data.tar.gz: 8bed8adb0b6d0e1f1ad19ad5b3e63517d590ce07e8128e18c4eb24d9545b3c03ac68fcd5bd26ec497f643995c4b31563ed310162c5df38da22669ea1405376d8

data/README.md

@@ -0,0 +1,49 @@
+### SETUP
+
+This gem requires Ruby v2.3 or later to be installed on your system.
+  - On MacOS, Ruby comes pre-installed.  Note: You can use the homebrew package manager from [brew.sh](https://brew.sh) if you want to install a newer version of Ruby which is noticeably faster than the one that currently ships with MacOS.
+  - Windows users should use one of the installer packages available from [rubyinstaller.org](http://rubyinstaller.org/).  You will need to determine whether your have a 32 or 64 bit processor, and download the appropriate installer.  We recommend Ruby v2.5.x or later, as it is substantially faster than prior versions.  When running the installer, check the checkboxes to add the Ruby installation to your PATH and to associate `.rb` and `.rbw` files with Ruby. A terminal window will pop up during the installation—press "enter" to automatically install all of the required tools, and when prompted again (several minutes later) press "enter" again to end the installation.
+
+You can check that Ruby is properly installed by opening `Terminal.app` (MacOS) or `CMD.EXE` (Windows) and typing `ruby --version` (note: two dashes).  You should see a response telling you which version of Ruby you are running.
+
+After your Ruby installation is confirmed you can install the data farming Ruby scripts by running the following command.
+
+    gem install datafarming
+
+This assumes that you have a network connection. Additional dependencies will be installed automatically.  On MacOS or Linux systems, you may be required to use the `sudo` command to to authenticate the installation.  In that case, type `sudo gem install datafarming` and enter your password when prompted.
+
+Gem installation only needs to be done once.  If you explicitly downloaded the `.gem` file rather than using a network installation, you can delete it after performing the installation.
+
+### USAGE
+
+Ruby is a powerful and concise  object-oriented scripting language.  Scripts are normally run from a command-line or terminal environment by typing the `ruby` command followed by a script name, often followed by one or more command-line arguments.  However, scripts installed as gems do not need the `ruby` command to be typed explicitly.  For example, typing `stripheaderdups.rb my_file.txt` will invoke the `stripheaderdups.rb` script and apply it to file `my_file.txt` in your current working directory.  Note that on Windows the `.rb` suffix is not needed to run your scripts, e.g., `stripheaderdups my_file.txt` will suffice.
+
+All scripts in this distribution are self documenting if run with a `--help`, `-h`, or `-?` option.  Quick descriptions follow.
+
+### GENERAL FILE MANIPULATION SCRIPTS:
+  - `blank2csv.rb` —
+converts whitespace-delimited files to Comma Separated Values (CSV) files
+  - `cat.rb` —
+Unix “cat” program work-alike for Windows—for viewing or creating text files or concatenating multiple files, based on stdio
+  - `convert_line_endings.rb` —
+converts text files (including CSV files) to your current operating system environment (e.g., DOS to Unix or Unix to DOS)
+  - `csv2blank.rb` —
+converts CSV files to whitespace-delimited files
+
+### DATA FARMING RUBY SCRIPTS:
+  - `batchrunner.rb` —
+run a model interactively with replication
+  - `rundesign_general.rb` —
+run control to replicate a designed experiment with a model
+  - `pool_files.rb` —
+pool columns from separate files into a new file, useful for combining design file & output files or multiple output files
+  - `stack_nolhs.rb` —
+generate designs by reassigning columns from built-in NOLHs
+  - `stripheaders.rb`, `stripheaderdups.rb` —
+used to remove headers from a file, or duplicate headers within a file, respectively
+  - `augment_design.rb` —
+generates star points to augment a fractional factorial
+  - `cross.rb` —
+creates a combinatorial design by crossing all combinations of any # of individual smaller designs
+  - `mser.rb` —
+ uses MSER truncation to remove initial transient effects for time-series output, reports truncated average and number of observations for each run to facilitate construction of a properly weighted confidence interval.

data/datafarming.gemspec

@@ -0,0 +1,24 @@
+# -*- ruby -*-
+_VERSION = "1.0.0"
+
+Gem::Specification.new do |s|
+  s.name = "datafarming"
+  s.version = _VERSION
+  s.date = "2018-06-25"
+  s.summary = "Useful scripts for data farming."
+  s.homepage = "https://gitlab.nps.edu/pjsanche/datafarmingrubyscripts.git"
+  s.email = "pjs@alum.mit.edu"
+  s.description = "Ruby scripts for data farming, including pre- and post-processing, design generation and scaling, and run control."
+  s.author = "Paul J Sanchez"
+  s.files = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test/|features/|.gitignore)})
+  end
+  s.bindir = "exe"
+  s.executables = s.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  s.require_paths = ["lib"]
+  s.add_runtime_dependency 'fwt', '~> 1.2'
+  s.add_runtime_dependency 'colorize', '~> 0.8'
+  s.add_runtime_dependency 'quickstats', '~> 2'
+  s.required_ruby_version = '>= 2.3'
+  s.license = 'MIT'
+end

data/exe/augment_design.rb

@@ -0,0 +1,66 @@
+#!/usr/bin/env ruby -w
+
+# This script generates star points to augment a fractional factorial
+# so you can check for quadratic effects.
+
+require 'colorize'
+
+String.disable_colorization false
+
+require 'optparse'
+require 'datafarming/error_handling'
+
+help_msg = [
+  'Generate star points to augment a fractional factorial ' \
+  'with quadratic effects.',
+  'Results are white-space delimited data written to ' +
+    'stdout'.light_blue + ', and can be redirected', 'as desired.', '',
+  'Syntax:',
+  "\n\t#{ErrorHandling.prog_name} [--help] FACTOR_INFO".yellow, '',
+  "Arguments in square brackets are optional.  A vertical bar '|'",
+  'indicates valid alternatives for invoking the option.  Prefix',
+  'the command with "' + 'ruby'.yellow +
+  '" if it is not on your PATH.', '',
+  '  --help | -h | -? | ?'.green,
+  "\tProduce this help message.",
+  '  FACTOR_INFO'.green,
+  "\tEITHER the number of factors (produces standardized +/-1 design),",
+  "\tOR pairs of values " + 'low1 hi1 low2 hi2...lowN hiN'.green +
+    ' for each of the',
+  "\tN factors.", ''
+]
+
+OptionParser.new do |opts|
+  opts.banner = "Usage: #{$PROGRAM_NAME} [-h|--help] [number of factors]"
+  opts.on('-h', '-?', '--help') { ErrorHandling.clean_abort help_msg }
+end.parse!
+
+if ARGV.length == 0 || (ARGV[0] == '?') || (ARGV.length > 1 && ARGV.length.odd?)
+  ErrorHandling.clean_abort help_msg
+else
+  num_factors = 0
+  if ARGV.length == 1
+    num_factors = ARGV.shift.to_i
+    x = Array.new(num_factors, ' 0')
+    ranges = Array.new(2 * num_factors)
+    ranges.each_index { |i| ranges[i] = (i.even? ? '-1' : ' 1') }
+  else
+    num_factors = ARGV.length / 2
+    x = Array.new(num_factors)
+    ranges = ARGV
+    num_factors.times do |i|
+      x[i] = 0.5 * (ranges[2 * i].to_f + ranges[2 * i + 1].to_f)
+    end
+  end
+  # create and print an array at the global center point (0,0,...,0)
+  puts x.join(' ')
+
+  # now generate and print the star points at +/-1 along each factor axis
+  num_factors.times do |i|
+    y = x.clone
+    y[i] = ranges[2 * i]
+    puts y.join(' ')
+    y[i] = ranges[2 * i + 1]
+    puts y.join(' ')
+  end
+end

data/exe/batchrunner.rb

@@ -0,0 +1,46 @@
+#!/usr/bin/env ruby -w
+
+require 'colorize'
+
+String.disable_colorization false
+
+require 'optparse'
+require 'datafarming/error_handling'
+
+help_msg = [
+  'Run a model interactively with replication.', '',
+  'Prompts the user interactively for a model/program to be run,',
+  'parameter values to use on the model command-line, and desired',
+  'number of replications.  The model is run the specified number',
+  'of times with the given arguments.  Separate output files are',
+  'created for each run of the model.', '',
+  'Syntax:',
+  "\n\t#{ErrorHandling.prog_name} [--help] ".yellow +
+    '[--outfile fname]'.yellow, '',
+  "Arguments in square brackets are optional.  A vertical bar '|'",
+  'indicates valid alternatives for invoking the option.  Prefix',
+  'the command with "' + 'ruby'.yellow +
+  '" if it is not on your PATH.', '',
+  '  --help | -h | -? | ?'.green,
+  "\tProduce this help message.",
+  '  --outfile fname | -o fname'.green,
+  "\tSpecify a specific prefix for output file names, defaults to 'outfile'."
+]
+
+outfile_name = 'outfile'
+OptionParser.new do |opts|
+  opts.banner = "Usage: #{$PROGRAM_NAME} [OPTIONS]"
+  opts.on('-h', '-?', '--help') { ErrorHandling.clean_abort help_msg }
+  opts.on('-o', '--outfile fname') { |fname| outfile_name = fname }
+end.parse!
+
+ErrorHandling.clean_abort help_msg if ARGV[0] == '?'
+
+cmd = (STDERR.print 'Enter command: '; gets.strip)
+params = (STDERR.print 'Enter parameters: '; gets.strip)
+num_runs = (STDERR.print 'Enter # runs: '; gets).to_i
+
+(1..num_runs).each do |run|
+  STDERR.print "run #{run}:",
+               `#{cmd} #{params} > #{outfile_name}-#{'%05d' % run}.csv`, "\n"
+end

data/exe/blank2csv.rb

@@ -0,0 +1,40 @@
+#!/usr/bin/env ruby -W0
+# converts blank separated files to csv
+
+require 'colorize'
+
+String.disable_colorization false
+
+require 'optparse'
+require 'datafarming/error_handling'
+
+help_msg = [
+  'Convert whitespace delimited data to comma separated values.', '',
+  'If filenames are specified, a backup is made for each file with',
+  "suffix '.orig' appended to the original filename and changes will",
+  'be made in-place in the original file.  If no filenames are given,',
+  'the script reads from ' + 'stdin'.blue + ' and writes to ' +
+    'stdout'.blue + '.  In either case,',
+  'all occurrences of one or more whitespace characters are replaced',
+  'by commas.', '',
+  'Syntax:',
+  "\n\t#{ErrorHandling.prog_name} [--help] [filenames...]".yellow, '',
+  "Arguments in square brackets are optional.  A vertical bar '|'",
+  'indicates valid alternatives for invoking the option.  Prefix',
+  'the command with "' + 'ruby'.yellow +
+  '" if it is not on your PATH.', '',
+  '  --help | -h | -? | ?'.green,
+  "\tProduce this help message.",
+  '  filenames...'.green,
+  "\tThe name[s] of the file[s] to be converted."
+]
+
+OptionParser.new do |opts|
+  opts.banner = "Usage: #{$PROGRAM_NAME} [-h|--help] [filenames...[]"
+  opts.on('-h', '-?', '--help') { ErrorHandling.clean_abort help_msg }
+end.parse!
+
+ErrorHandling.clean_abort help_msg if ARGV[0] == '?'
+
+$-i = '.orig'
+ARGF.each { |line| puts line.strip.gsub(/\s+/, ',') }

data/exe/cat.rb

@@ -0,0 +1,36 @@
+#!/usr/bin/env ruby -w
+# concatenate one or more inputs to stdout
+
+require 'colorize'
+
+String.disable_colorization false
+
+require 'optparse'
+require 'datafarming/error_handling'
+
+help_msg = [
+  'Concatenate one or more input files, or ' + 'stdin'.blue +
+    ', to ' + 'stdout'.blue + '.', '',
+  'Syntax:',
+  "\n\t#{ErrorHandling.prog_name} [--help] [filenames...]".yellow, '',
+  "Arguments in square brackets are optional.  A vertical bar '|'",
+  'indicates valid alternatives for invoking the option.  Prefix',
+  'the command with "' + 'ruby'.yellow +
+  '" if it is not on your PATH.', '',
+  '  --help | -h | -? | ?'.green,
+  "\tProduce this help message.",
+  '  filenames...'.green,
+  "\tThe name[s] of the file[s] to be concatenated.",
+  "\tRead from " + 'stdin'.blue + ' if no files are specified.',
+  "\tTo terminate interactive input enter " + 'ctrl-d'.cyan,
+  "\t(Mac/Unix/Linux) or " + 'ctrl-z'.cyan + ' (Windows).'
+]
+
+OptionParser.new do |opts|
+  opts.banner = "Usage: #{$PROGRAM_NAME} [-h|--help] [filenames...[]"
+  opts.on('-h', '-?', '--help') { ErrorHandling.clean_abort help_msg }
+end.parse!
+
+ErrorHandling.clean_abort help_msg if ARGV[0] == '?'
+
+ARGF.each { |line| print line }

data/exe/convert_line_endings.rb

@@ -0,0 +1,38 @@
+#!/usr/bin/env ruby -w
+# Ruby script to convert end of line to current system default
+
+require 'colorize'
+
+String.disable_colorization false
+
+require 'optparse'
+require 'datafarming/error_handling'
+
+help_msg = [
+  'Convert DOS line endings to Unix/Mac OS X line endings or vice versa.', '',
+  'If filenames are specified, a backup is made for each file with',
+  "suffix '.orig' appended to the original filename and changes will",
+  'be made in-place in the original file.  If no filenames are given,',
+  'the script reads from ' + 'stdin'.blue + ' and writes to ' +
+    'stdout'.blue + '.', '',
+  'Syntax:',
+  "\n\t#{ErrorHandling.prog_name} [--help] [filenames...]".yellow, '',
+  "Arguments in square brackets are optional.  A vertical bar '|'",
+  'indicates valid alternatives for invoking the option.  Prefix',
+  'the command with "' + 'ruby'.yellow +
+  '" if it is not on your PATH.', '',
+  '  --help | -h | -? | ?'.green,
+  "\tProduce this help message.",
+  '  filenames...'.green,
+  "\tThe name[s] of the file[s] to be converted."
+]
+
+OptionParser.new do |opts|
+  opts.banner = "Usage: #{$PROGRAM_NAME} [-h|--help] [filenames...[]"
+  opts.on('-h', '-?', '--help') { ErrorHandling.clean_abort help_msg }
+end.parse!
+
+ErrorHandling.clean_abort help_msg if ARGV[0] == '?'
+
+$-i = '.orig'
+ARGF.each { |line| puts line.split(/\r\n|\r/) }

data/exe/cross.rb

@@ -0,0 +1,44 @@
+#!/usr/bin/env ruby -w
+
+require 'colorize'
+
+String.disable_colorization false
+
+require 'optparse'
+require 'datafarming/error_handling'
+require 'datafarming/cross'
+
+help_msg = [
+  'Create a crossed design from two or more input design files',
+  'where each line is a design point.  The crossed design is',
+  'written to ' + 'stdout'.blue + ' in CSV format.', '',
+  'Syntax:',
+  "\n\t#{ErrorHandling.prog_name} [--help] filenames...".yellow, '',
+  "Arguments in square brackets are optional.  A vertical bar '|'",
+  'indicates valid alternatives for invoking the option.  Prefix',
+  'the command with "' + 'ruby'.yellow +
+    '" if it is not on your PATH.', '',
+  '  --help | -h | -? | ?'.green,
+  "\tProduce this help message.",
+  '  filenames...'.green,
+  "\tThe names of the files containing designs to be crossed.",
+  "\tInput file data can be delimited by commas, semicolons,",
+  "\tcolons, or whitespace."
+]
+
+OptionParser.new do |opts|
+  opts.banner = "Usage: #{$PROGRAM_NAME} [-h|--help] [filenames...[]"
+  opts.on('-h', '-?', '--help') { ErrorHandling.clean_abort help_msg }
+end.parse!
+
+ErrorHandling.clean_abort help_msg if ARGV[0] == '?' || ARGV.length < 2
+
+input_array = []
+ARGV.each do |filename| # for each file given as a command-line arg...
+  # open the file, read all the lines, and then for each line use
+  # spaces, commas, colons, or semicolons to tokenize.
+  input_array << File.open(filename).readlines.map do |line|
+    line.strip.split(/[,:;]|\s+/)
+  end
+end
+CrossedDesigns.cross(input_array).each { |line| puts line.join(',') }

data/exe/csv2blank.rb

@@ -0,0 +1,39 @@
+#!/usr/bin/env ruby -W0
+# converts csv separated files to blank separated
+
+require 'colorize'
+
+String.disable_colorization false
+
+require 'optparse'
+require 'datafarming/error_handling'
+
+help_msg = [
+  'Convert comma separated values data to whitespace delimited.', '',
+  'If filenames are specified, a backup is made for each file with',
+  "suffix '.orig' appended to the original filename and changes will",
+  'be made in-place in the original file.  If no filenames are given,',
+  'the script reads from ' + 'stdin'.blue + ' and writes to ' +
+    'stdout'.blue + '.  In either case,',
+  'all occurrences of commas are replaced by blanks.', '',
+  'Syntax:',
+  "\n\t#{ErrorHandling.prog_name} [--help] [filenames...]".yellow, '',
+  "Arguments in square brackets are optional.  A vertical bar '|'",
+  'indicates valid alternatives for invoking the option.  Prefix',
+  'the command with "' + 'ruby'.yellow +
+  '" if it is not on your PATH.', '',
+  '  --help | -h | -? | ?'.green,
+  "\tProduce this help message.",
+  '  filenames...'.green,
+  "\tThe name[s] of the file[s] to be converted."
+]
+
+OptionParser.new do |opts|
+  opts.banner = "Usage: #{$PROGRAM_NAME} [-h|--help] [filenames...[]"
+  opts.on('-h', '-?', '--help') { ErrorHandling.clean_abort help_msg }
+end.parse!
+
+ErrorHandling.clean_abort help_msg if ARGV[0] == '?'
+
+$-i = '.orig'
+ARGF.each { |line| puts line.strip.tr(',', ' ') }

data/exe/mser.rb

@@ -0,0 +1,71 @@
+#!/usr/bin/env ruby -w
+
+require 'rubygems' if RUBY_VERSION =~ /^1\.8/
+require 'colorize'
+
+String.disable_colorization false
+
+require 'optparse'
+require 'datafarming/error_handling'
+
+begin
+  require 'quickstats'
+rescue LoadError
+  ErrorHandling.clean_abort [
+    "\n\tALERT: quickstats gem is not installed!".red,
+    "\tIf you have network connectivity, type:",
+    "\n\t\tgem install quickstats\n".yellow,
+    "\t(Admin privileges may be required.)\n\n"
+  ]
+end
+
+help_msg = [
+  'Calculate MSER truncation statistics for one or more input files.', '',
+  'Input files should consist of one column of data per file, with or',
+  'without headers.  The output consists of one line per input file,',
+  'comprised of the MSER-based average of the data, the number of',
+  'observations used to calculate that average, and the number of',
+  'observations truncated, separated by commas.  Output is written',
+  'to ' + 'stdout'.blue + ' in CSV format, with headers.', '',
+  'Syntax:',
+  "\n\t#{ErrorHandling.prog_name} [--help] filenames...".yellow, '',
+  "Arguments in square brackets are optional.  A vertical bar '|'",
+  'indicates valid alternatives for invoking the option.  Prefix',
+  'the command with "' + 'ruby'.yellow +
+  '" if it is not on your PATH.', '',
+  '  --help | -h | -? | ?'.green,
+  "\tProduce this help message.",
+  '  filenames...'.green,
+  "\tThe names of two or more files containing data to be analyzed."
+]
+
+OptionParser.new do |opts|
+  opts.banner = "Usage: #{$PROGRAM_NAME} [-h|--help] filenames..."
+  opts.on('-h', '-?', '--help') { ErrorHandling.clean_abort help_msg }
+end.parse!
+
+ErrorHandling.clean_abort help_msg if ARGV.empty? || ARGV[0] == '?'
+
+puts 'x-bar,n,trunc'
+
+ARGV.each do |fname|
+  data = File.readlines(fname)
+  data.shift if data[0] =~ /[A-Za-z]/ # strip header if one present
+  data.map! { |line| line.chomp.strip.to_f }
+  m_stats = QuickStats.new
+  warmup = [(data.length * 0.5).to_i, data.length - 10].min
+  index = data.length - 1
+  while index > (data.length - warmup) && index > 1
+    m_stats.new_obs(data[index])
+    index -= 1
+  end
+  best = [m_stats.std_err, m_stats.avg, warmup]
+
+  while index > -1
+    m_stats.new_obs(data[index])
+    best = [m_stats.std_err, m_stats.avg, index] if m_stats.std_err <= best[0]
+    index -= 1
+  end
+
+  printf "%f,%d,%d\n", best[1], data.length - best[2], best[2]
+end