mddb 0.0.3 → 0.0.4

data/bin/aggregater

@@ -0,0 +1,71 @@
+#!/usr/bin/env ruby
+require "#{__FILE__}/../stat"
+require 'micro-optparse'
+
+@options = Parser.new do |p|
+  p.banner = "Binning, see below for options"
+  p.version = "0.01"
+  p.option :live, "print the live status", :default => false
+  p.option :header, "use if the stream contains a header", :default => false
+  p.option :delimiter, "specify the delimiting character", :default => "tsv", :value_in_set => ['tsv', 'csv', 'ssv']
+  p.option :drop, "Drop lines that are poorly formatted, instead of failing", :default => false
+  p.option :keys, "Number of columns to use as keys", :default => 1, :value_satisfies => lambda {|x| x > 0 }
+end.process!
+
+def header?
+  @options[:header] && ARGF.lineno == 1
+end
+
+catch :shit_stream do
+  input = 0
+  output = 0
+  stats = Stats.new "Aggregate", :input, :output, :ratio, :elapsed
+  last = @options[:keys] - 1
+  old_key = nil
+  totals = []
+  ARGF.each do |line|
+    stats.clock if ARGF.lineno ==  1
+    input += 1
+    line.chomp!
+    if header?
+      puts line
+      output += 1
+    else
+      cols = line.split("\t")
+      cols = cols[0..(last)] +cols[(last+1)..(cols.count-1)].map {|d| d.to_f}
+
+      # Set the key
+      key = cols[0..last]
+
+      # First time round we set the old key to equal the new one
+      if old_key.nil?
+        totals = Array.new((cols.count-1-last),0) 
+        old_key = key
+      end
+      
+      # If the key is new, print the old key and its aggregates,
+      # otherwise aggregate the total for this key
+      if key == old_key
+        (cols.count - last - 1).times do |i|
+          totals[i] += cols[i+last+1]
+        end
+      end
+      
+      if key != old_key
+        puts old_key.map {|k| k.to_s}.join("\t") + "\t" +  totals.map {|d| d.to_s}.join("\t")
+        old_key = key
+        # Reset the total
+        totals = Array.new((cols.count-1-last),0) 
+        (cols.count - last - 1).times do |i|
+          totals[i] += cols[i+last+1]
+        end
+        output += 1
+      end
+
+      stats.show(input,output) if @options[:live]
+    end
+  end
+  output += 1
+  puts old_key.map {|k| k.to_s}.join("\t") + "\t" +  totals.map {|d| d.to_s}.join("\t")
+  stats.finish(input,output) 
+end

data/bin/average

@@ -0,0 +1,59 @@
+#!/usr/bin/env ruby
+require "#{__FILE__}/../stat"
+require 'micro-optparse'
+
+valid_columns = lambda {|x| 
+  valid = true
+  x.each {|d|
+    valid = false unless d.to_i > 0
+  }
+  valid
+}
+    
+@options = Parser.new do |p|
+  p.banner = "Average, see below for options"
+  p.version = "0.01"
+  p.option :live, "print the live status", :default => false
+  p.option :header, "use if the stream contains a header", :default => false
+  p.option :delimiter, "specify the delimiting character", :default => "tsv", :value_in_set => ['tsv', 'csv', 'ssv']
+  p.option :drop, "Drop lines that are poorly formatted, instead of failing", :default => false
+  p.option :columns, "The columns whose average you wish to obtain, columns are integers starting at 1", :default => [], :value_satisfies => valid_columns
+end.process!
+
+def header?
+  @options[:header] && ARGF.lineno == 1
+end
+
+columns = @options[:columns].map {|d| d.to_i - 1}
+max_col = columns.max
+input = 0.0
+output = 0.0
+stat = Stats.new("Average", :input, :output, :elapsed)
+catch :shit_happened do
+  options = []
+  
+  ARGF.each do |line|
+    stat.clock if ARGF.lineno == 1
+    input += 1
+    if header?
+      puts line
+      output += 1
+    else
+      cols = line.chomp.split("\t").map {|d| d.to_f}
+
+      if (max_col + 1)  > cols.length
+        puts "There are only #{cols.length} columns, but you said there was a column #{max_col + 1}"
+        throw :shit_happened
+      end
+
+      columns.each do |option|
+        cols[option] = cols[option]/cols.last
+      end
+      
+      puts cols.map {|c| c.to_s}.join("\t")
+      output += 1
+      stat.show(input,output) if @options[:live]
+    end
+  end
+end
+stat.finish(input,output)

data/bin/binner

@@ -0,0 +1,95 @@
+#!/usr/bin/env ruby
+require 'micro-optparse'
+require "#{__FILE__}/../stat.rb"
+
+class String
+  def is_numeric?
+    true if Float(self) rescue false
+  end
+end
+
+all_are_numeric = lambda {|x| 
+  valid = true
+  x.each {|x| valid = false unless x.is_numeric?}
+  x.each {|x| valid = false unless x.to_f >= 0 }
+  valid
+} 
+
+@options = Parser.new do |p|
+  p.banner = "Binning, see below for options"
+  p.version = "0.01"
+  p.option :live, "print the live status", :default => false
+  p.option :header, "use if the stream contains a header", :default => false
+  p.option :delimiter, "specify the delimiting character", :default => "tsv", :value_in_set => ['tsv', 'csv', 'ssv']
+  p.option :emit, "Emit 1 for map-reduce", :default => false
+  p.option :drop, "Drop lines that are poorly formatted, instead of failing", :default => false
+  p.option :bins, "bin width for each column, a width of 0 leaves a column unchanged", :default => [],
+    :value_satisfies => all_are_numeric
+end.process!
+
+@options[:bins].map! {|d| d.to_f}
+
+@delimiters = {'tsv' => "\t", 'csv' => ",", 'ssv' => " "}
+
+@delimiter = @delimiters[@options[:delimiter]]
+
+def header?
+  @options[:header] && ARGF.lineno == 1
+end
+
+def bin value, width
+  if width == 0.0
+    value
+  else
+    ((value/width).floor * width).round(6)
+  end
+end
+
+def emit
+  @options[:emit]? @delimiter + "1" : ""
+end
+
+def emit_header
+  @options[:emit]? @delimiter + "count" : ""
+end
+
+def columns line
+  line.split(@delimiter)
+end
+
+def drop? line
+  if line.length != @options[:bins].length
+    if @options[:drop]
+      true
+    else
+      $stderr.puts "There were #{line.length} columns on line #{ARGF.lineno}, but you gave #{@options[:bins].length} bins"
+      $stderr.puts "Perhaps this wasn't a #{@options[:delimiter]}. Try the -d --help for a list of filetypes"
+      $stderr.puts "If you think that some lines might be an issue, use the -r flag to drop bad lines"
+      throw :dropped
+    end
+  else
+    false
+  end
+end
+
+catch :dropped do
+  input = 0
+  output = 0
+  stats = Stats.new "Binning", :input, :output, :dropped, :elapsed
+  ARGF.each do |line|
+    input += 1
+    row = line.chomp.split(@delimiter)
+    unless drop? row
+      if header?
+        puts "#" + line.chomp + emit_header
+      else
+        row = row.map {|d| d.to_f}
+        binned = row.each_with_index.map {|d,i| bin(d,@options[:bins][i])}
+        puts binned.join(@delimiter) + emit
+      end
+      output += 1
+    end
+    stats.show(input,output) if @options[:live]
+  end
+  stats.finish(input,output)
+end

data/lib/mddb/version.rb

@@ -1,3 +1,3 @@
 module Mddb
-  VERSION = "0.0.3"
+  VERSION = "0.0.4"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mddb
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-02-15 00:00:00.000000000Z
+date: 2012-02-16 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70212474145840 !ruby/object:Gem::Requirement
+  requirement: &70129250480800 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70212474145840
+  version_requirements: *70129250480800
 - !ruby/object:Gem::Dependency
   name: mongo_mapper
-  requirement: &70212474145420 !ruby/object:Gem::Requirement
+  requirement: &70129250480340 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70212474145420
+  version_requirements: *70129250480340
 - !ruby/object:Gem::Dependency
   name: hirb
-  requirement: &70212474145000 !ruby/object:Gem::Requirement
+  requirement: &70129250479880 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70212474145000
+  version_requirements: *70129250479880
 - !ruby/object:Gem::Dependency
   name: thor
-  requirement: &70212474144540 !ruby/object:Gem::Requirement
+  requirement: &70129250479340 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,11 +54,14 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70212474144540
+  version_requirements: *70129250479340
 description: MDDB makes analysing molecular dynamics simulations easy
 email:
 - thom.mulvaney@gmail.com
 executables:
+- aggregater
+- average
+- binner
 - mddb
 extensions: []
 extra_rdoc_files: []
@@ -66,6 +69,9 @@ files:
 - .gitignore
 - Gemfile
 - Rakefile
+- bin/aggregater
+- bin/average
+- bin/binner
 - bin/mddb
 - config.yml
 - lib/mddb.rb