ix-cli 0.0.1

data/bin/ix-histogram

@@ -0,0 +1,80 @@
+#!/usr/bin/env ruby
+#
+
+require 'tmpdir'
+require 'optparse'
+
+options = {}
+
+OptionParser.new do |opts|
+
+  opts.banner = "
+    histogram - Creates png charts using date-time and value data from text.
+
+    Reads two numeric columns from stdin (x, y)
+    and creates a png graphical chart.
+
+    Usage: cat data | #{File.basename($0)} [options]
+  "
+
+  opts.on("-tTITLE", "--title=TITLE", "title of chart") do |v|
+    options[:title] = v
+  end
+
+  opts.on("-xTEXT", "--x-label=TEXT", "label for x axis") do |v|
+    options[:x_label] = v
+  end
+
+  opts.on("-yTEXT", "--y-label=TEXT", "label for y axis") do |v|
+    options[:y_label] = v
+  end
+
+  opts.on("-oOUTPUT_NAME", "--output-name=OUTPUT_NAME", "name of output file (example: speed_vs_time") do |v|
+    options[:output] = v
+  end
+
+end.parse!
+
+unless options[:title]
+  options[:title] = 'title'
+end
+
+unless options[:x_label]
+  options[:x_label] = 'x'
+end
+
+unless options[:y_label]
+  options[:y_label] = 'y'
+end
+
+unless options[:output]
+  options[:output] = 'output'
+end
+
+Dir.mktmpdir do |dir|
+
+  data_path   = "#{dir}/histogram.data"
+  script_path = "#{dir}/script.gnuplot"
+
+  File.open(data_path, 'w+') do |file|
+    STDIN.each_line do |line|
+      file.puts line
+    end
+  end
+
+  File.open(script_path, 'w+') do |script|
+    script.puts "set term png size 800,600"
+    script.puts "set style histogram clustered"
+    script.puts "set grid"
+    script.puts "set timestamp"
+    script.puts "set output \"#{options[:output]}.png\""
+    script.puts "plot for [COL=2:4] '#{data_path}' using COL:xticlabels(1) title columnheader"
+    script.puts "plot for [COL=2:4] '#{data_path}' using COL:xticlabels(1) title columnheader(2), for [i=3:22] '' using i title columnheader(i)"
+    script.puts "quit"
+  end
+
+  system "gnuplot #{script_path}"
+  puts "open #{options[:output]}.png"
+  
+end
+

data/bin/ix-hls

@@ -0,0 +1,21 @@
+#!/usr/bin/env ruby
+
+#
+# options
+#
+#    line      ix-highlight-line highlight a specific line
+#    vertical  ix-highlight-vertical highlight a particular column
+#    highlight ix-highlight-match highlight whatever matches a regex
+#
+
+
+abort 'No pattern given' if ARGV[0].nil?
+
+trap('SIGINT') { }
+
+$stdout.sync = true
+
+STDIN.each_line do |line|
+  puts line.gsub(/(#{ARGV[0]})/i) { |x| "#{$1}" }
+end
+

data/bin/ix-html

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+
+ix bcat | ix prefix '<pre>' | ix suffix '</pre>'

data/bin/ix-human-time

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+
+puts Time.now.strftime("%Y%m%d%H%M%S")

data/bin/ix-hyphen

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+
+STDIN.each_line do |line|
+  line = line.chomp
+  line.gsub!(/\W+/, ' ')
+  line.gsub!(/\s+/, '-')
+  line.gsub!(/^[-]+|[-]+$/, '')
+  puts line.downcase
+end

data/bin/ix-hypothesis

@@ -0,0 +1,546 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+
+require 'fileutils'
+require 'optparse'
+require 'ostruct'
+require 'isna'
+
+# Percentage bar.
+#
+# items  = (1..100).to_a
+# size   = (10..100).to_a
+# length = size.shuffle.first
+# 
+# items.each do |item|
+#   puts Percentage.new(100, item, length).to_bar
+# end
+#
+class Percentage
+  def initialize(product, divisor, scale = 100)
+    @product        = product
+    @scale          = scale - 5
+    @pre_percentage = (divisor * @scale / product).to_f
+    @percentage     = (@pre_percentage * 100 / @scale)
+  end
+  def to_i
+    @percentage.to_i
+  end
+  def to_f
+    @percentage.to_f
+  end
+
+  # 0   ~  50   green
+  # 50  ~  80   yellow
+  # 80  ~  100  red
+  def to_bar(color = true, bar = '|')
+    filling = (bar * @pre_percentage.to_i)
+    padding = 0
+    if color
+      if to_i >= 0 and to_i < 50
+        filling = "" + (bar * @pre_percentage.to_i) + ""
+      end
+      if to_i >= 50 and to_i < 80
+        filling = "" + (bar * @pre_percentage.to_i) + ""
+      end
+      if to_i >= 80
+        filling = "" + (bar * @pre_percentage.to_i) + ""
+      end
+      if filling.size > 0
+        padding = 13
+      end
+    end
+    bindings = [filling, to_f, '%']
+    "[%-#{@scale + padding}s%5.1f%s]" % bindings
+  end
+end
+
+# Usage:
+#
+# sample = [2, 4, 4, 4, 5, 5, 7, 9]
+# puts sample.to_stat
+#
+# #=> mean: 5, standard deviation: 2.0
+#
+# http://easycalculation.com/statistics/standard-deviation.php
+# http://en.wikipedia.org/wiki/Standard_deviation
+#
+class Array
+
+  def mean
+    product / size
+  end
+
+  def product
+    sum
+  end
+
+  def normalize(scale_min, scale_max)
+    map do |number|
+      divisor    = (number - min) * (scale_max - scale_min)
+      dividend   = max - min
+      divisor / dividend
+    end
+  end
+
+  def median
+    return size[0] if size == 1
+    self[(size + 1) / 2]
+  end
+
+  def sum_of_squares(avg = nil)
+    avg = mean unless avg
+    sqares = map do |number|
+      result = number - avg
+      result ** 2
+    end
+    sqares.sum
+  end
+
+  def variance(minus = 0)
+    sum_of_squares / (size - minus)
+  end
+
+  def standard_deviation
+    Math.sqrt(variance(1.5))
+  end
+
+  def population_standard_deviation
+    Math.sqrt(variance(0))
+  end
+
+  def sample_standard_deviation
+    med = median
+    sqr = map do |number|
+      result = number - med
+      result ** 2
+    end
+    Math.sqrt(sqr.mean)
+  end
+
+  def sum
+    inject do |cumulative, value|
+      cumulative += value
+    end
+  end
+
+  def sigmas(length, sigma)
+    avg = mean
+    list = []
+    length.times { |n| list << avg - ((n + 1) * sigma) }
+    list << avg
+    length.times { |n| list << avg + ((n + 1) * sigma) }
+    list.sort
+  end
+
+  def to_bell(text, standard_deviations_to_show = 3, base = :population, kazu_dev = 1, percentage_scale, hide)
+
+    if base == :population
+      sigma = population_standard_deviation
+    end
+
+    if base == :sample
+      sigma = sample_standard_deviation
+    end
+
+    if base == :normal
+      sigma = standard_deviation
+    end
+
+    total = size
+    avg   = mean
+
+    sigmas_copy = sigmas(standard_deviations_to_show * kazu_dev, sigma / kazu_dev)
+
+    puts ''
+    puts "#{text} Normal distribution for a #{base} standard distribution."
+    puts "  (#{sigma})"
+    puts '-' * 80
+    groups = group_by do |number|
+      return_value = nil
+      sigmas_copy.each_with_index do |sigma, index|
+        if number < sigma
+          if number > sigmas_copy[index -1]
+            return_value = sigma
+          end
+        end
+        unless return_value
+          if number == sigma
+            return_value = sigma
+          end
+        end
+      end
+      return_value
+    end
+
+    template = "%6s %15s   %6s      %s"
+    headers = ['sigma', 'deviation', 'samples', 'percentage']
+    puts template % headers 
+    template = "%6s %s   %s %s %s"
+
+    the_min = sigmas_copy.map do |sigma|
+      ((groups[sigma] ? groups[sigma].size : 0.0) / total.to_f) * 100
+	end.min
+
+    the_max = sigmas_copy.map do |sigma|
+      ((groups[sigma] ? groups[sigma].size : 0.0) / total.to_f) * 100
+	end.max
+
+    puts [the_min, the_max].inspect
+
+    sigmas_copy.each_with_index do |sigma, index|
+
+      size    = groups[sigma] ? groups[sigma].size : 0
+
+      if hide
+        next if size == 0
+      end
+
+      bell_line = []
+      std_dev_ruler = "%3.3f" % ((index - sigmas_copy.size / 2) / kazu_dev.to_f)
+      if std_dev_ruler == '0.000'
+		bell_line << ' ' + std_dev_ruler.to_ansi.green.to_s
+      else
+		bell_line << std_dev_ruler
+      end
+
+      if sigma == avg
+        bell_line << ("%15.4f" % [sigma.to_s]).to_ansi.green.to_s
+      else
+        bell_line << ("%15.4f" % [sigma.to_s])
+      end
+
+      if size > 0
+        bell_line << (size.to_s.ljust(10, ' ').to_ansi.green.to_s)
+      else
+        bell_line << '-'.ljust(10, ' ')
+      end
+
+      bell_line << ("%6.2f%% " % [(size.to_f / total.to_f) * 100]).to_s
+
+      # puts groups[sigma].inspect
+
+	  bn = [
+        the_min,
+        (size.to_f / total.to_f) * 100,
+        the_max
+	  ]
+
+      an = bn.normalize(
+		0,
+        percentage_scale
+	  )
+
+      # bell_line << Percentage.new(total, size, percentage_scale).to_bar
+
+      case (std_dev_ruler).to_f
+      when 0.000
+		bell_line << '|' + ('|' * an[1]).to_s.to_ansi.pink.to_s
+      when 0..1
+		bell_line << '|' + ('|' * an[1]).to_s.to_ansi.green.to_s
+      when -1..0
+		bell_line << '|' + ('|' * an[1]).to_s.to_ansi.green.to_s
+      when 1..2
+		bell_line << '|' + ('|' * an[1]).to_s.to_ansi.yellow.to_s
+      when -2..1
+		bell_line << '|' + ('|' * an[1]).to_s.to_ansi.yellow.to_s
+      when 2..3
+		bell_line << '|' + ('|' * an[1]).to_s.to_ansi.red.to_s
+      when -3..2
+		bell_line << '|' + ('|' * an[1]).to_s.to_ansi.red.to_s
+      when 3..100
+		bell_line << '|' + ('|' * an[1]).to_s.to_ansi.cyan.blink.to_s
+      when -3..-100
+		bell_line << '|' + ('|' * an[1]).to_s.to_ansi.cyan.blink.to_s
+      else
+		bell_line << '|' + ('|' * an[1]).to_s.to_ansi.yellow.to_s
+      end
+
+      puts template % bell_line
+
+    end
+    if groups[nil]
+      puts "#{groups[nil].size} outliers: => #{groups[nil].sort.inspect}"
+    end
+  end
+
+  def to_stat(padding = 15)
+    symbols  = ['x', 'µ', 'σ', 'P', 'S', 'VP', 'VS', 'N', '∑x']
+    template = symbols.map { |symbol| "#{symbol}: %-#{padding}.2f" }
+    template = template.join(" ")
+    bindings = [
+      median,
+      mean,
+      standard_deviation,
+      population_standard_deviation,
+      sample_standard_deviation,
+      variance(1),
+      variance(0),
+      size,
+      sum
+    ]
+    template % bindings
+  end
+
+end
+
+# =============================================================================
+
+def print_header(title)
+  puts ''
+  puts title
+  puts '-' * 80
+end
+
+# =============================================================================
+
+def print_help
+  puts 'Hypothesis crunches data and performs statistical analysis.'
+  puts ''
+  puts 'USAGE'
+  puts ''
+  puts '  $ cat some_file | ix hypothesis'
+  puts ''
+  puts 'INPUT'
+  puts ''
+  puts '  Send data to STDIN in a key value format.'
+  puts '  KEYS must be strings no bigger than ten chars'
+  puts '  VALUES must be numeric floats or integers'
+  puts '  each line must be KEY VALUE separated by a space in between'
+  puts '  then terminate with an EOL'
+  puts ''
+  puts 'EXAMPLE'
+  puts ''
+  puts '  Key value format is explained below'
+  puts '  "|" char means begining of line in a file.'
+  puts ''
+  puts '  -----------------------------------'
+  puts '  |group-a 1'
+  puts '  |group-a 3'
+  puts '  |group-b 2'
+  puts '  |group-b 1'
+  puts '  |group-c 3'
+  puts '  |group-c 5'
+  puts ''
+  puts '   Heva fun!'
+  puts ''
+end
+
+# =============================================================================
+
+@configuration = {}
+
+@configuration[:kazus_deviation] = 1
+@configuration[:sigma] = 3
+@configuration[:distribution] = :normal
+@configuration[:scale] = 34
+@configuration[:hide_points] = false
+@configuration[:print_statistics] = false
+
+OptionParser.new do |opts|
+
+  opts.banner = "USAGE: #{File.basename($0)} [OPTIONS]"
+
+  help_string = 'Expand the bell to expose kazu\'s deviation.'
+  opts.on('-k', '--kazus-deviation [N]', OptionParser::DecimalInteger, help_string) do |value|
+    @configuration[:kazus_deviation] = value
+  end
+
+  help_string = 'How many standard deviations (sigmas) to show'
+  opts.on('-s', '--sigma [N]', OptionParser::DecimalInteger, help_string) do |value|
+    @configuration[:sigma] = value
+  end
+
+  help_string = 'Render analysis as a normal standard deviation'
+  opts.on('-n', '--normal', help_string) do |value|
+    @configuration[:distribution] = :normal
+  end
+
+  help_string = 'Render analysis as a population standard deviation'
+  opts.on('-p', '--population', help_string) do |value|
+    @configuration[:distribution] = :population
+  end
+
+  help_string = 'Render analysis as a sample standard deviation'
+  opts.on('-a', '--sample', help_string) do |value|
+    @configuration[:distribution] = :sample
+  end
+
+  help_string = 'Scale, expand and show more detail on probabilistic bar data.'
+  opts.on('-b', '--bars [N]', OptionParser::DecimalInteger, help_string) do |value|
+    @configuration[:scale] = value.to_i
+  end
+
+  help_string = 'Hide away points that dont have data.'
+  opts.on('-i', '--hide-points', help_string) do |value|
+    @configuration[:hide_points] = true
+  end
+
+  help_string = 'Print Statistical Breakdown.'
+  opts.on('-r', '--print-statistics', help_string) do |value|
+    @configuration[:print_statistics] = true
+  end
+
+  opts.on('-e', '--example', 'Give me more iformation about this program') do |value|
+    print_help
+    exit 0
+  end
+
+end.parse!
+
+# =============================================================================
+
+$stdin.sync = true
+
+sample = { }
+
+STDIN.each do |line|
+  key, value = line.split(/\s+/)
+  sample[key] ||= {} 
+  sample[key][:data] ||= []
+  if sample[key][:total]
+    sample[key][:total] += value.to_f
+  else
+    sample[key][:total] = value.to_f
+  end
+  sample[key][:data] << value.to_f
+end
+
+print_header "Report Generated on: #{Time.now}"
+
+grand_total      = 0
+number_of_groups = 0
+categories       = sample.keys.sort
+
+categories.each do |category|
+  grand_total      += sample[category][:total]
+  number_of_groups += 1
+end
+
+if @configuration[:print_statistics]
+
+  print_header 'Median'.to_ansi.green.to_s
+  categories.each do |category|
+    puts "%-60s %19.6f" % [category, sample[category][:data].median]
+  end
+  
+  print_header 'Mean'.to_ansi.green.to_s
+  categories.each do |category|
+    puts "%-60s %19.6f" % [category, sample[category][:data].mean]
+  end
+  
+  print_header 'Standard Deviation'.to_ansi.green.to_s
+  categories.each do |category|
+    puts "%-60s %19.6f" % [category, sample[category][:data].standard_deviation]
+  end
+  
+  print_header 'Population Standard Deviation'.to_ansi.green.to_s
+  categories.each do |category|
+    puts "%-60s %19.6f" % [category, sample[category][:data].population_standard_deviation]
+  end
+  
+  print_header 'Sample Standard Deviation'.to_ansi.green.to_s
+  categories.each do |category|
+    puts "%-60s %19.6f" % [category, sample[category][:data].sample_standard_deviation]
+  end
+  
+  print_header 'Variance of a Population'.to_ansi.green.to_s
+  categories.each do |category|
+    puts "%-60s %19.6f" % [category, sample[category][:data].variance(1)]
+  end
+  
+  print_header 'Variance of a Sample'.to_ansi.green.to_s
+  categories.each do |category|
+    puts "%-60s %19.6f" % [category, sample[category][:data].variance(0)]
+  end
+  
+  print_header 'Count of samples'.to_ansi.green.to_s
+  categories.each do |category|
+    puts "%-60s %19.6f" % [category, sample[category][:data].size]
+  end
+  
+  print_header 'Product'.to_ansi.green.to_s
+  categories.each do |category|
+    puts "%-60s %19.6f" % [category, sample[category][:data].sum]
+  end
+
+  print_header 'Percentage breakdown per category'
+  categories.each do |category|
+    percentage = Percentage.new(grand_total, sample[category][:data].sum, 66)
+    # puts "%-10s %-s" % [category, percentage.to_bar]
+    puts "%-s %s" % [percentage.to_bar, category]
+  end
+end
+
+categories.each do |category|
+  sample[category][:data].to_bell(
+    category,
+    @configuration[:sigma],
+    @configuration[:distribution],
+    @configuration[:kazus_deviation],
+    @configuration[:scale],
+    @configuration[:hide_points],
+  )
+end
+
+# All ANOVA goes below.
+# ==============================================================================
+
+if sample.size < 2
+  print_header "You must provide at least two groups for an anova calculation."
+  exit 0
+end
+
+if sample.map { |k, v| v[:data].size }.uniq.size > 1
+  print_header "All groups must have the same size in order for a reliable anova calculation."
+  exit 0
+end
+
+if sample.map { |k, v| v[:data].size }.all? { |data_size| data_size == 1 }
+  print_header "You must provide more samples for each category."
+  exit 0
+end
+
+# In anova we first grab the grand mean.
+# eaither by individually or altogether.
+
+sum_of_sizes = 0
+sample.each do |name, data|
+    sum_of_sizes += data[:data].size
+end
+grand_mean = grand_total / sum_of_sizes
+
+# Next we want to extract the sum of squares using the grand mean.
+sum_of_squares_total = 0
+sum_of_squares_within = 0
+sum_of_squares_between = 0
+
+sample.each do |name, data|
+  sum_of_squares_total += data[:data].sum_of_squares(grand_mean)
+  sum_of_squares_within += data[:data].sum_of_squares
+  sum_of_squares_between += (data[:data].size * ((data[:data].mean - grand_mean) ** 2))
+end
+
+degrees_of_freedom_between = number_of_groups - 1;
+degrees_of_freedom_within = number_of_groups * (sample.first[1][:data].size - 1);
+
+f_test = (sum_of_squares_between / degrees_of_freedom_between) / (sum_of_squares_within / degrees_of_freedom_within)
+
+output = { 
+  :number_of_groups           => number_of_groups,
+  :grand_total                => grand_total,
+  :grand_mean                 => grand_mean,
+  :sum_of_squares_total       => sum_of_squares_total,
+  :sum_of_squares_within      => sum_of_squares_within,
+  :sum_of_squares_between     => sum_of_squares_between,
+  :degrees_of_freedom_within  => degrees_of_freedom_within,
+  :degrees_of_freedom_between => degrees_of_freedom_between,
+  :f_test                     => f_test,
+# :f_bars                     => '|' * f_test,
+}
+
+print_header 'ANOVA Breakdown.'
+output.keys.map { |key| key.to_s }.sort.each do |key|
+  puts "#{key.to_s.ljust(40, '.')} #{output[key.to_sym].to_s}"
+end
+

data/bin/ix-ids

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+
+if [ "$1" == "" ]; then
+    regex='\d+'
+else
+    regex="$1"
+fi
+
+ix scan "$regex" | ix array | ix prefix 'ids = ' | par

data/bin/ix-image-generate

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+echo "https://placeholdit.imgix.net/~text?txtsize=33&txt=$1%C3%97$2&w=$1&h=$2"

data/bin/ix-image-to-base64

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+
+require 'base64'
+
+template = '<img src=\"data:image/png;base64,/%s\" />'
+
+base64 = Base64.encode64(STDIN.read)
+
+puts template % [base64]

data/bin/ix-increments

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+
+NUMBER_REGEX = /\d+/
+
+previous_number = nil
+
+STDIN.each_line do |line|
+  next unless NUMBER_REGEX =~ line
+  number = line.scan(NUMBER_REGEX).first.to_f
+  if previous_number == nil
+    previous_number = number
+  end
+  puts "#{number} #{number - previous_number}"
+  previous_number = number 
+end
+