sciruby 0.1.0 → 0.1.3

data/lib/sciruby/data/guardian.rb

@@ -0,0 +1,96 @@
+module SciRuby
+  module Data
+    
+    # World Government Data from the Guardian.
+    class Guardian < PublicSearcher
+      QUERY_DOMAIN = %q{www.guardian.co.uk}
+      QUERY_PATH   = %q{/world-government-data/search.json}
+      FOUR_OH_FOUR_MESSAGE = '404 Page not found'
+      ALLOWED_FORMATS = [:csv, :excel]
+
+      class DatasetInfo < ::OpenStruct
+        def initialize h
+          super h
+          self.download_links.each_index do |i|
+            self.download_links[i] = ::OpenStruct.new(self.download_links[i])
+          end
+        end
+      end
+
+
+      # Search the site or database using some set of parameters.
+      #
+      # This function is the one that you should redefine if you want to require certain parameters, or if there are
+      # parameter co-dependencies. Ultimately, you call `search_internal(params)`.
+      #
+      # == Arguments
+      # * q: keywords (default: '', if no other parameters are supplied)
+      # * facet_country: country code abbreviation to search
+      # * facet_source_title: e.g., data from Australian government would be data.nsw.org.au
+      # * facet_format: e.g., csv, excel, xml, shapefile, kml
+      def initialize args={}
+        #args[:facet_format] ||= :csv
+        #@require_format ||= args[:facet_format] # This should be removed when we can interpret other formats.
+
+        @search_result = search(args)
+      end
+
+      # Return dataset meta-data found in the search, hashed by source_id. So, do datasets.keys if you want a list of
+      # source_ids.
+      def datasets
+        @datasets ||= begin
+          h = {}
+          search_result["results"].each do |res|
+            h[res['source_id']] = DatasetInfo.new(res)
+          end
+          h
+        end
+      end
+
+      # Download a specific dataset by +source_id+ and cache it in the searcher. Returns a Statsample::Dataset.
+      #
+      # If this raises an exception, you can try this:
+      #
+      #     links = raw_dataset_links_cached(source_id)
+      #
+      # And then for each of +links+, do `raw_dataset(source_id, link)` to see what the actual downloaded data was.
+      # This is good for debugging -- e.g., did the page move? or is there something wrong with Ruby's CSV interpreter?
+      # Or is it in some other format altogether?
+      #
+      # Right now, this function only handles CSV. TODO: Add more format handlers!
+      def dataset source_id
+        @dataset ||= {}
+        @dataset[source_id] ||= begin # Datasets are stored by source ID
+          pos = 0
+          datasets[source_id].download_links.each do |link_info|
+
+            unless ALLOWED_FORMATS.include?(link_info.format)
+              pos += 1
+              next # Format is incorrect.
+            end
+
+            # Format appears to be correct, prior to actually downloading. Proceed.
+
+            # Attempt to read the cached one first, and if that fails, try downloading.
+            raw = cached_dataset(source_id) || download_dataset(link_info.link)
+            
+            begin
+              ds  = parse_dataset link_info.format, raw, datasets[source_id].title
+              cache_dataset(source_id, raw, link_info.format)
+            rescue TypeError => e
+              if pos == datasets[source_id].download_links.size - 1
+                raise DatasetNotFoundError.new(e)
+              end
+            ensure
+              pos += 1
+            end
+
+            return ds unless ds.nil?
+
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/sciruby/data/r.rb

@@ -0,0 +1,155 @@
+module SciRuby
+  module Data
+    # R data module.
+    class R < Base
+      DIR = Pathname.new(__FILE__).realpath.dirname.to_s
+
+      require "simpler"
+
+      # Attempt to parse an R dataset through simpler. Works with most datasets (but not for table, dist, or array).
+      #
+      # Note that not all of these datasets have functions for converting directly to Statsample or SciRuby types. In
+      # other words, parsing works, but it may not be as simple as calling to_dataset or to_h (yet).
+      #
+      # TODO: Add basic conversion functions like to_h, to_a, etc.
+      #
+      # == R datasets that don't work
+      # * crimtab (table)
+      # * eurodist (dist)
+      # * HairEyeColor (table)
+      # * iris3 (array)
+      # * occupationalStatus (table)
+      # * Titanic (table)
+      # * UCBAdmissions (table)
+      # * volcano (matrix): TODO: Handle non-named rows and columns in matrix
+      #
+      # TODO: rownames that are just counters need to be ignored in some cases, e.g., Puromycin
+      #
+      # == R datasets that work partially
+      # * chickwts: doesn't know how to handle levels, but still loads them as strings.
+      def dataset id
+        begin
+          r(id)
+        rescue Simpler::RError => e
+          raise DatasetNotFoundError.new(e)
+        end
+      end
+
+      # TODO: Fix so that aggregate datasets, like state, are listed properly in search results.
+      def search args={}
+        parse_datasets_index(r.eval! { %q{library(help="datasets")} })
+      end
+      alias_method :datasets, :search
+
+      # Alias for self.r.
+      def r obj=nil; SciRuby::Data::R.r(obj); end
+
+      
+      class << self
+        def in_dir &block
+          SciRuby::Data.in_dir {  Dir.chdir('r') { yield } }
+        end
+
+        def in_man_dir &block
+          in_dir {   Dir.chdir('man') { yield } }
+        end
+
+        # With an argument, this function attempts to read from R some variable (probably a built-in dataset).
+        # Without an argument, this function gives access to the R console. See also: simpler by jtprince on github.
+        def r obj=nil
+          require "simpler"
+          @@r ||= ::Simpler.new
+          unless obj.nil?
+            r_class = Base.class(obj)
+            if r_class == 'numeric' || r_class == 'integer' || r_class == 'ordered' || r_class == 'factor' || r_class == 'character'
+              return Vector.new(obj)
+            elsif r_class == 'data.frame'
+              return DataFrame.new(obj)
+            elsif r_class == 'nfnGroupedData'
+              return GroupedData.new(obj)
+            elsif r_class == 'matrix'
+              return RMatrix.new(obj)
+            elsif r_class == 'ts'
+              return TimeSeries.new(obj)
+            elsif r_class == 'mts'
+              return MultiTimeSeries.new(obj)
+            elsif r_class == 'list'
+              return List.new(obj).to_h
+            else
+              raise(NotImplementedError, "Don't know how to recognize class #{r_class} yet.")
+            end
+          end
+          return @@r
+        end
+      end
+
+
+
+      # Hacked together tex parser to extract useful information from .Rd R manual files. Unlikely to work on any other
+      # TeX or LaTeX files.
+      class Man < OpenStruct
+        class << self
+          def in_dir &block
+            SciRuby::Data::R.in_man_dir { yield }
+          end
+        end
+
+        def in_dir &block
+          SciRuby::Data::R::Man.in_dir { yield }
+        end
+
+
+        def initialize dataset_id
+          h = {}
+          in_dir do
+            raw = File.read("#{dataset_id}.Rd")
+            entries = raw.split("\n\\") # this is a total hack
+            entries.each do |entry|
+              next if entry =~ /^%/
+              command, content = entry.split('{', 2)
+              h[command.underscore] = content.strip.gsub(/}$/, '').gsub(/\n$/, '')
+            end
+          end
+          super(h)
+        end
+      end
+
+    protected
+
+      # Listing of datasets read directly from R.
+      def parse_datasets_index raw
+        h = {}
+        mode = :pre
+
+        last_key = nil
+
+        raw.split("\n").each do |line|
+          next if mode == :pre && line !~ /^Index\:/
+          mode = :index
+          next if line =~ /^Index\:/
+          next if line.strip.empty?
+
+          if line =~ /^ /
+            h[last_key] = [h[last_key], line.strip].join(' ')
+          else
+            k, v        = line.split(' ', 2)
+            last_key    = k.strip
+            h[last_key] = v.strip
+          end
+        end
+
+        h
+      end
+    end
+  end
+end
+
+require File.join(SciRuby::Data::R::DIR, 'r', 'base.rb')
+require File.join(SciRuby::Data::R::DIR, 'r', 'data_frame.rb')
+require File.join(SciRuby::Data::R::DIR, 'r', 'time_series_base.rb')
+require File.join(SciRuby::Data::R::DIR, 'r', 'time_series.rb')
+require File.join(SciRuby::Data::R::DIR, 'r', 'multi_time_series.rb')
+require File.join(SciRuby::Data::R::DIR, 'r', 'vector.rb')
+require File.join(SciRuby::Data::R::DIR, 'r', 'r_matrix.rb')
+require File.join(SciRuby::Data::R::DIR, 'r', 'grouped_data.rb')
+require File.join(SciRuby::Data::R::DIR, 'r', 'list.rb')

data/lib/sciruby/data/r/base.rb

@@ -0,0 +1,110 @@
+module SciRuby::Data
+  class R
+    # Parses datasets from R directly.
+    class Base
+      FLOAT_RE = /([.eE])/
+
+      require "simpler"
+
+      def initialize id
+        @rob  = id # R object name
+
+        assign_properties # Read as many properties as possible from R
+      end
+
+      def self.class obj
+        #STDERR.puts "obj=#{obj}"
+        Base.new(obj).send :read_class
+      end
+
+      attr_reader :rob
+      alias_method :rname, :rob
+
+    protected
+      def assign_properties; end
+
+      def r obj=nil
+        SciRuby::Data::R.r(obj)
+      end
+
+      def float_re
+        SciRuby::Data::R::Base::FLOAT_RE
+      end
+
+      def call_function fn=nil
+        #STDERR.puts "Call function: #{fn.to_s}\t#{rob}"
+        fn.nil? ? r.eval! { rob } : r.eval! { "#{fn.to_s}(#{rob})" }
+      end
+
+      def call_property prop
+        r.eval! { "#{rob}$'#{prop.to_s}'"}
+      end
+
+      def read_class fn=:class
+        read_single_line(fn).first
+      end
+
+      def read_single_line fn=nil
+        line = call_function fn
+        #STDERR.puts "rsl Got back: #{line}"
+        CSV::parse_line(line.split(' ', 2).tap{ |s| s.shift }.first, :col_sep => ' ')
+      end
+
+      def read_single_token fn=nil
+        line = call_function fn
+        #STDERR.puts "rst Got back: #{line}"
+        line.split.tap{ |s| s.shift }.first
+      end
+
+      # Read multiple lines from a function call. You can also pass in a block if you want to ask for a property instead
+      # of a function call, e.g.,
+      #     read_multiple_lines { call_property('height') }
+      def read_multiple_lines fn=nil
+        lines = block_given? ? yield : call_function(fn)
+        #STDERR.puts "rml Got back:\n#{lines}"
+
+        lines = lines.split("\n")
+
+        return nil if lines.first =~ /^NULL/
+        if lines.first =~ /^ *\[/
+          return lines.map do |line|
+            remaining_line = CSV::parse_line(line.split(' ', 2).tap { |s| s.shift }.first, :col_sep => ' ')
+            remaining_line = remaining_line.tap { |l| l.pop } if remaining_line.last.nil?
+            remaining_line
+          end.flatten
+        end
+
+        raise "Unrecognized R output"
+      end
+
+      def read_row_names fn='rownames'
+        attempt = read_multiple_lines(fn) # may return nil if no rownames found.
+        return [] if attempt.nil?
+        attempt
+      end
+
+      def read_col_names fn='colnames'
+        read_row_names fn
+      end
+
+      def read_names fn='names'
+        read_row_names fn
+      end
+
+      def read_levels fn='levels'
+        read_row_names fn
+      end
+
+      def read_columns fields
+        columns = {}
+        fields.each do |field|
+          raise(ArgumentError, "nil field") if field.nil?
+          columns_for_field = SciRuby::Data::R.r("#{rob}[,'#{field.to_s}']")
+          columns[field] = (columns_for_field.is_a?(Vector) && columns_for_field.has_levels?) || columns_for_field.is_a?(TimeSeries) ? columns_for_field : columns_for_field.to_a
+        end
+        columns
+      end
+
+    end
+  end
+end

data/lib/sciruby/data/r/data_frame.rb

@@ -0,0 +1,24 @@
+module SciRuby::Data
+  class R
+    class DataFrame < Base
+      attr_reader :row_names, :columns
+
+      def col_names
+        columns.keys
+      end
+
+      def levels col_name
+        columns[col_name].levels
+      end
+
+    protected
+
+      def assign_properties
+        @row_names = read_row_names
+        col_names  = read_col_names
+        @columns   = read_columns(col_names)
+      end
+
+    end
+  end
+end

data/lib/sciruby/data/r/grouped_data.rb

@@ -0,0 +1,7 @@
+module SciRuby::Data
+  class R
+    class GroupedData < DataFrame
+
+    end
+  end
+end

data/lib/sciruby/data/r/list.rb

@@ -0,0 +1,20 @@
+module SciRuby::Data
+  class R
+    # An intermediate object that doesn't really get used -- immediately gets converted to a Ruby Hash of other R objects.
+    class List < Base
+      def to_h
+        @data
+      end
+    protected
+      def assign_properties
+        @names = read_names
+        @names = nil if @names.nil? || (@names.is_a?(Array) && @names.empty?)
+
+        @data = {}
+        @names.each do |list_item|
+          @data[list_item] = r("#{rob}[['#{list_item}']]")
+        end
+      end
+    end
+  end
+end

data/lib/sciruby/data/r/multi_time_series.rb

@@ -0,0 +1,24 @@
+module SciRuby::Data
+  class R
+    # class 'mts' in R
+    class MultiTimeSeries < TimeSeriesBase
+      attr_reader :row_names, :columns
+
+      def col_names
+        columns.keys
+      end
+
+      def levels col_name
+        columns[col_name].levels
+      end
+
+    protected
+      def assign_properties
+        @row_names = read_row_names
+        col_names  = read_col_names
+        @columns   = read_columns(col_names)
+        super
+      end
+    end
+  end
+end

data/lib/sciruby/data/r/r_matrix.rb

@@ -0,0 +1,7 @@
+module SciRuby::Data
+  class R
+    class RMatrix < DataFrame
+
+    end
+  end
+end