sciruby 0.1.0 → 0.1.3

data/lib/ext/string.rb

@@ -0,0 +1,39 @@
+class String
+  unless method_defined?(:constantize)
+    # Based on constantize from ActiveSupport::Inflector
+    def constantize
+      names = self.split('::')
+      names.shift if names.empty? || names.first.empty?
+
+      constant = Object
+      names.each do |name|
+        constant = constant.const_defined?(name, false) ? constant.const_get(name) : constant.const_missing(name)
+      end
+      constant
+    end
+  end
+
+  unless method_defined?(:camelize)
+    # Adapted from camelize from ActiveSupport::Inflector
+    def camelize first_letter_in_uppercase = true
+      if first_letter_in_uppercase
+        self.to_s.gsub(/\/(.?)/) { "::#{$1.upcase}" }.gsub(/(?:^|_)(.)/) { $1.upcase }
+      else
+        self.to_s[0].chr.downcase + self[1..-1].camelize
+      end
+    end
+  end
+
+  unless method_defined?(:underscore)
+    # Adapted from underscore from ActiveSupport::Inflector
+    def underscore
+      word = self.dup
+      word.gsub!(/::/, '/')
+      word.gsub!(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
+      word.gsub!(/([a-z\d])([A-Z])/,'\1_\2')
+      word.tr!("-", "_")
+      word.downcase!
+      word
+    end
+  end
+end

data/lib/sciruby.rb

@@ -1,4 +1,4 @@
-# = sci_ruby.rb -
+# = sciruby.rb -
 # SciRuby - Ruby scientific visualization and computation.
 #
 # Copyright (C) 2011  SciRuby Development Team
@@ -9,7 +9,7 @@
 #
 # This program is free software; you can redistribute it and/or
 # modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
+# as published by the Free Software Foundation; either version 3
 # of the License, or (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
@@ -24,8 +24,54 @@
 # Specific notices will be placed where they are appropriate.
 #
 
+require "rubygems"
+require "bundler/setup"
+
 module SciRuby
-  VERSION = '0.1.0'
+  VERSION = '0.1.3'
+  DIR     = Pathname.new(__FILE__).realpath.dirname.to_s
+
+  require File.join(::SciRuby::DIR, 'ext', 'string.rb')
+  require File.join(::SciRuby::DIR, 'ext', 'csv.rb')
+
+  class << self
+    def plot script
+      SciRuby::Plotter.new script
+    end
+
+    def integrate *args, &block
+      require "integration"
+      ::Integration.integrate(*args, &block)
+    end
 
-  autoload(:Recommend, 'sciruby/recommend')
+    # Produce a list of datasets that can be loaded using the +dataset+ method
+    def dataset_search database, args = {}
+      "SciRuby::Data::#{database.to_s.camelize}".constantize.new(args).datasets.keys
+    end
+
+    # Load a dataset from a specific database. For a list of datasets, use `dataset_search(:guardian)`, for example.
+    def dataset database, source_id
+      begin
+        "SciRuby::Data::#{database.to_s.camelize}".constantize.new.dataset(source_id)
+      rescue DatabaseUnavailableError => e
+        warn "Database appears to be unavailable. Attempting to use cached version."
+        SciRuby::Data::Cacher.new.dataset(source_id, database)
+      end
+    end
+
+    # Shorthand for SciRuby::Analysis.store(*args, &block)
+    def analyze *args, &block
+      SciRuby::Analysis.store(*args, &block)
+    end
+  end
+
+  autoload(:Analysis, File.join(DIR, 'sciruby', 'analysis'))
+  autoload(:Config, File.join(DIR, 'sciruby', 'config'))
+  autoload(:Editor, File.join(DIR, 'sciruby', 'editor'))
+  autoload(:Plotter, File.join(DIR, 'sciruby', 'plotter'))
+  autoload(:Recommend, File.join(DIR, 'sciruby', 'recommend'))
+  autoload(:Validation, File.join(DIR, 'sciruby', 'validation'))
+  autoload(:Data, File.join(DIR, 'sciruby', 'data'))
 end
+
+autoload(:Shoes, File.join(SciRuby::DIR, 'ext', 'shoes'))

data/lib/sciruby/analysis.rb

@@ -0,0 +1,98 @@
+require 'sciruby/analysis/suite'
+require 'sciruby/analysis/suite_report_builder'
+
+module SciRuby
+  # DSL to run a statistical analysis without hassle.
+  # * Shortcut methods to avoid having to use complete namespaces, many based on R.
+  # * Attach/detach vectors to workspace, as with R
+  # == Example
+  #  an1 = Statsample::Analysis.store(:first) do
+  #    # Load excel file with x,y,z vectors
+  #    ds = excel('data.xls')
+  #    # See variables on ds dataset
+  #    names(ds)
+  #    # Attach the vectors to workspace, like R
+  #    attach(ds)
+  #    # vector 'x' is attached to workspace like a method,
+  #    # so you can use like any variable
+  #    mean,sd = x.mean, x.sd
+  #    # Shameless R robbery
+  #    a = c( 1:10)
+  #    b = c(21:30)
+  #    summary(cor(ds)) # Call summary method on correlation matrix
+  #  end
+  #  # You can run the analysis by its name
+  #  Statsample::Analysis.run(:first)
+  #  # or using the returned variables
+  #  an1.run
+  #  # You can also generate a report using ReportBuilder.
+  #  # .summary() method call 'report_building' on the object,
+  #  # instead of calling text summary
+  #  an1.generate("report.html")
+  module Analysis
+    @@stored_analyses={}
+    @@last_analysis=nil
+    def self.clear_analysis
+      @@stored_analyses.clear
+    end
+    def self.stored_analyses
+      @@stored_analyses
+    end
+    def self.last
+      @@stored_analyses[@@last_analysis]
+    end
+    def self.store(name, opts=Hash.new,&block)
+      raise "You should provide a block" if !block
+      @@last_analysis=name
+      opts={:name=>name}.merge(opts)
+      @@stored_analyses[name]=Suite.new(opts,&block)
+    end
+    # Run analysis +*args+
+    # Without arguments, run all stored analyses
+    # Only 'echo' will be printed to screen.
+    def self.run(*args)
+      args=stored_analyses.keys if args.size==0
+      raise "Analysis #{args} doesn't exists" if (args - stored_analyses.keys).size>0
+      args.each do |name|
+        stored_analyses[name].run
+      end
+    end
+
+    # Add analysis +*args+ to a ReportBuilder object.
+    # Without arguments, add all stored analyses.
+    # Each analysis is wrapped inside a ReportBuilder::Section object.
+    # This is the method used by +save+ and +to_text+.
+    def self.add_to_reportbuilder(rb, *args)
+      args=stored_analyses.keys if args.size==0
+      raise "Analysis #{name} doesn't exists" if (args - stored_analyses.keys).size>0
+      args.each do |name|
+        section=ReportBuilder::Section.new(:name=>stored_analyses[name].name)
+        rb_an=stored_analyses[name].add_to_reportbuilder(section)
+        rb.add(section)
+        rb_an.run
+      end
+    end
+
+    # Save the analysis to a file.
+    # Without arguments, adds all stored analyses.
+    def self.save(filename, *args)
+      rb=ReportBuilder.new(:name=>filename)
+      add_to_reportbuilder(rb, *args)
+      rb.save(filename)
+    end
+
+    # Run analysis and return as string.
+    # Only 'echo' will be printed to screen.
+    # Without arguments, add all stored analyses.
+    def self.to_text(*args)
+      rb=ReportBuilder.new(:name=>"Analysis #{Time.now}")
+      add_to_reportbuilder(rb, *args)
+      rb.to_text
+    end
+
+    # Run analysis and print to screen all echo and summary callings
+    def self.run_batch(*args)
+      puts to_text(*args)
+    end
+  end
+end

data/lib/sciruby/analysis/suite.rb

@@ -0,0 +1,87 @@
+require "statsample"
+
+module SciRuby
+  module Analysis
+    class Suite
+      include ::Statsample::Shorthand
+      attr_accessor :output
+      attr_accessor :name
+      attr_reader :block
+      def initialize(opts=Hash.new(), &block)
+        if !opts.is_a? Hash
+          opts={:name=>opts}
+        end
+
+        @block=block
+        @name=opts[:name] || "Analysis #{Time.now}"
+        @attached=[]
+        @output=opts[:output] || ::STDOUT
+      end
+      # Run the analysis, putting output on
+      def run
+         @block.arity<1 ? instance_eval(&@block) : @block.call(self)
+      end
+      # Provides a description of the procedure. Only appears as a commentary on
+      # SuiteReportBuilder outputs
+      def desc(d)
+        @output.puts("Description:")
+        @output.puts("  #{d}")
+      end
+      def echo(*args)
+        @output.puts(*args)
+      end
+      def summary(obj)
+        obj.summary
+      end
+      def add_to_reportbuilder(rb)
+        SuiteReportBuilder.new({:name=>name, :rb=>rb}, &block)
+      end
+
+      def generate(filename)
+        ar=SuiteReportBuilder.new({:name=>name}, &block)
+        ar.generate(filename)
+      end
+      def to_text
+        ar=SuiteReportBuilder.new({:name=>name}, &block)
+        ar.to_text
+      end
+
+      def attach(ds)
+        @attached.push(ds)
+      end
+      def detach(ds=nil)
+        if ds.nil?
+          @attached.pop
+        else
+          @attached.delete(ds)
+        end
+      end
+      alias :old_boxplot :boxplot
+      alias :old_histogram :histogram
+      alias :old_scatterplot :scatterplot
+
+      def show_svg(svg)
+        require 'tmpdir'
+        fn=Dir.tmpdir+"/image_#{Time.now.to_f}.svg"
+        File.open(fn,"w") {|fp| fp.write svg}
+        `xdg-open '#{fn}'`
+      end
+      def boxplot(*args)
+        show_svg(old_boxplot(*args).to_svg)
+      end
+      def histogram(*args)
+        show_svg(old_histogram(*args).to_svg)
+      end
+      def scatterplot(*args)
+        show_svg(old_scatterplot(*args).to_svg)
+      end
+
+      def method_missing(name, *args,&block)
+        @attached.reverse.each do |ds|
+          return ds[name.to_s] if ds.fields.include? (name.to_s)
+        end
+        raise "Method #{name} doesn't exists"
+      end
+    end
+  end
+end

data/lib/sciruby/analysis/suite_report_builder.rb

@@ -0,0 +1,44 @@
+module SciRuby
+  module Analysis
+    class SuiteReportBuilder < Suite
+      attr_accessor :rb
+      def initialize(opts=Hash.new,&block)
+        if !opts.is_a? Hash
+          opts={:name=>opts}
+        end
+        super(opts,&block)
+        @rb=opts[:rb] || ReportBuilder.new(:name=>name)
+      end
+      def generate(filename)
+        run if @block
+        @rb.save(filename)
+      end
+      def to_text
+        run if @block
+        @rb.to_text
+      end
+      def summary(o)
+        @rb.add(o)
+      end
+      def desc(d)
+        @rb.add(d)
+      end
+      def echo(*args)
+        args.each do |a|
+          @rb.add(a)
+        end
+      end
+
+      def boxplot(*args)
+        @rb.add(old_boxplot(*args))
+      end
+      def histogram(*args)
+        @rb.add(old_histogram(*args))
+      end
+      def boxplot(*args)
+        @rb.add(old_boxplot(*args))
+      end
+
+    end
+  end
+end

data/lib/sciruby/config.rb

@@ -0,0 +1,93 @@
+module SciRuby
+  module Config
+    class << self
+
+      # Create a .sciruby directory if it doesn't exist (.sciruby) and chdir to it.
+      def dir
+        Dir.chdir(Dir.home) do
+          FileUtils.mkdir('.sciruby') unless Dir.exists?('.sciruby')
+          Dir.chdir '.sciruby' do
+            yield if block_given?
+          end
+        end
+      end
+
+      # Create a data dir in the .sciruby directory if it doesn't exist (data/) and chdir to it.
+      def data_dir
+        dir do
+          FileUtils.mkdir('data') unless Dir.exists?('data')
+          Dir.chdir 'data' do
+            yield
+          end
+        end
+      end
+
+      # Create a data source directory within the .sciruby dir for a given module, e.g., ./sciruby/data/guardian for Guardian.
+      def data_source_dir module_name, create=true
+        dir_name = module_name.to_s if module_name.is_a?(Symbol)
+        dir_name ||= module_name.split('::').tap{ |m| 2.times { m.shift } }.join('::').underscore
+        data_dir do
+          FileUtils.mkdir(dir_name) if !Dir.exists?(dir_name) && create
+          Dir.chdir dir_name do
+            yield if block_given?
+          end
+        end
+      end
+
+
+      # Add an extension to the basename for a dataset based on the format.
+      def filename_for_dataset id, format=nil
+        basename = basename_for_dataset(id)
+        format.nil? ? basename : [basename, format.to_s].join('.')
+      end
+
+      # Generate a unique and safe filename for a dataset. This may need to be improved to incorporate some kind of hash.
+      # Hopefully there will be no collisions.
+      def basename_for_dataset id
+        return id.gsub(/[^a-zA-Z0-9\_]/, '_')
+      end
+
+      # Determines whether the basename for a cached dataset exists in some format or another.
+      def basename_exists? id
+        matches = Dir.glob("#{basename_for_dataset(id)}.*")
+        return matches.first if matches.size >= 1
+        return nil
+      end
+
+      # Store a given dataset in the .sciruby/data directory.
+      def cache_dataset module_name, dataset_id, file_contents, format
+        for_dataset_filename(module_name, dataset_id, format) do |dataset_filename|
+          unless File.exists?(dataset_filename) || basename_exists?(dataset_id)
+            File.open(dataset_filename, 'w') do |file|
+              file.write file_contents
+            end
+          end
+        end
+      end
+
+      # In the data source directory, do something with the dataset cache file. e.g.,
+      #     for_dataset('Guardian', '963', :cvs) do |dataset_filename|
+      #       File.open(dataset_filename, 'w') do |f|
+      #         f.write "Hello, world!"
+      #       end
+      #     end
+      #
+      # It computes the block arg (here, +dataset_filename+) for you using Config::filename_for_dataset. It also puts
+      # you in the correct directory.
+      #
+      # This function is used by Config::cache_dataset.
+      def for_dataset_filename module_name, dataset_id, format, &block
+        data_source_dir module_name do
+          yield filename_for_dataset(dataset_id, format)
+        end
+      end
+
+      def for_dataset_basename module_name, dataset_id, &block
+        data_source_dir module_name do
+          yield basename_for_dataset(dataset_id)
+        end
+      end
+
+    end
+  end
+end

data/lib/sciruby/data.rb

@@ -0,0 +1,168 @@
+require "json"
+require "net/http"
+require "uri"
+require "cgi"
+require "ostruct"
+
+
+module SciRuby
+  class DatabaseUnavailableError < IOError
+    def initialize domain, path, http_get_result=nil
+      @domain = domain
+      @path = path
+      @http_get_result = http_get_result
+    end
+    attr_reader :domain, :path, :http_get_result
+
+    def to_s
+      "Database at domain '#{@domain}', path '#{@path}' appears to be unavailable."
+    end
+  end
+
+  class DatasetNotFoundError < TypeError
+    def initialize e
+      @exp=e
+    end
+
+    def to_s
+      "Dataset does not exist. It may have moved, is not available in a format SciRuby can interpret." + @exp.message + "\n" + @exp.backtrace.join("\n")
+    end
+  end
+
+
+  module Data
+    DIR = File.join(SciRuby::DIR, 'sciruby', 'data')
+
+    def self.in_dir &block
+      Dir.chdir(File.join(SciRuby::DIR, '..', 'data')) do
+        yield
+      end
+    end
+
+    # Really just a placeholder.
+    class Base #:nodoc:
+    end
+
+    # Basic dataset type -- handles caching of datasets, that's about it.
+    class Cacher < Base
+
+      # Attempt to load a dataset. This is overridden for publicly-searchable datasets.
+      # Basically it works as a fallback if a publicly-searchable database is unavailable for some reason, but we
+      # may already have the data in the cache.
+      def dataset source_id, module_name=nil
+        module_name ||= self.class.to_s
+        raw = cached_dataset(source_id, module_name)
+        if raw.nil?
+          raise(ArgumentError, "Dataset is not cached.")
+        else
+          match  = SciRuby::Config.data_source_dir(module_name, false) { SciRuby::Config.basename_exists?(source_id) }
+          format = match.split('.').last.to_sym
+          title  = SciRuby::Config.basename_for_dataset(source_id)
+          parse_dataset(format, raw, title)
+        end
+      end
+      
+    protected
+
+      # Attempt to get the dataset from the cache. This function is a little bit fragile for the following reason:
+      # The +dataset+ function [eventually] allows for different +download_links+ of a dataset, which may be in different
+      # formats. +cached_dataset+, however, guesses the format based on the format indicated for the first download link.
+      #
+      # TODO: Consider gzipping cached datasets.
+      def cached_dataset source_id, module_name=nil
+        module_name ||= self.class.to_s
+        SciRuby::Config.for_dataset_basename(module_name, source_id) do |basename|
+          filename = SciRuby::Config.basename_exists?(source_id)
+          return nil unless filename
+          File.read(filename)
+        end
+      end
+
+      # Store a dataset locally. Use cached_dataset to retrieve.
+      def cache_dataset source_id, raw_data, format
+        SciRuby::Config.cache_dataset self.class.to_s, source_id, raw_data, format
+      end
+
+      # Parse and cache a dataset, using the appropriate interpreter.
+      def parse_dataset format, raw, name
+        begin
+          case format
+            when :csv
+              CSV.parse(raw, :headers => true, :converters => :all).to_dataset.tap { |da| da.name = name }
+            when :excel
+              Statsample::Excel.parse(raw, :name => name)
+          end
+        rescue NameError => e
+          STDERR.puts "Unable to load statsample"
+          raise e
+        rescue => e
+          STDERR.puts e.inspect
+          raise TypeError, "Format was not as expected; dataset may have moved"
+        end
+      end
+    end
+
+
+    # Base class for searching datasets. R dataset interpreter and PublicSearcherBase (and thus Guardian) are all derived
+    # from this type.
+    class Searcher < Cacher
+      def initialize args={}
+        @search_result = search(args)
+      end
+    end
+
+
+    # Handles searching public datasets. Doesn't actually do it itself, but you can derive searchers from this -- e.g.,
+    # Guardian.
+    class PublicSearcher < Searcher
+      FOUR_OH_FOUR_MESSAGE = '404'
+      attr_reader :search_result
+
+      # Search the site or database using some set of parameters.
+      #
+      # This function is the one that you should redefine if you want to require certain parameters, or if there are
+      # parameter co-dependencies. Ultimately, you call `search_internal(params)`.
+      #
+      # == Example Arguments
+      # * q: keywords
+      # * facet_country: country code abbreviation to search
+      # * facet_source_title: e.g., data from Australian government would be data.nsw.org.au
+      def search args={}
+        JSON.parse(search_internal(args))
+      end
+
+      # Download a dataset from a given link.
+      def download_dataset link
+        url = URI.parse link
+        http_get(url.host, url.path)
+      end
+
+    protected
+      # Like http_get, but gets the domain and path from the child searcher class.
+      def search_internal params={} #:nodoc:
+        domain = self.class.const_get(:QUERY_DOMAIN, true)
+        path   = self.class.const_get(:QUERY_PATH, true)
+
+        result = http_get(domain, path, params)
+
+        if result.include?(self.class.const_get(:FOUR_OH_FOUR_MESSAGE, true))
+          raise(DatabaseUnavailableError.new(domain, path, result))
+        end
+        
+        result
+      end
+
+      # Execute an HTTP get request with or without parameters.
+      #
+      # Adapted from: http://stackoverflow.com/questions/1252210/parametrized-get-request-in-ruby/1252305#1252305
+      def http_get domain, path, params = {} #:nodoc:
+        path_with_params = "#{path}?".concat(params.collect { |k,v| "#{k.to_s}=#{CGI::escape(v.to_s)}"}.join('&'))
+        return Net::HTTP.get(domain, path_with_params) unless params.empty?
+        Net::HTTP.get(domain, path)
+      end
+    end
+
+    autoload(:R, File.join(DIR, 'r'))
+    autoload(:Guardian, File.join(DIR, 'guardian'))
+  end
+end