red-datasets 0.0.8 → 0.1.3

data/lib/datasets/error.rb

@@ -0,0 +1,4 @@
+module Datasets
+  class Error < StandardError
+  end
+end

data/lib/datasets/hepatitis.rb

@@ -0,0 +1,207 @@
+require "csv"
+
+require_relative "dataset"
+
+module Datasets
+  class Hepatitis < Dataset
+    class Record < Struct.new(:label,
+                              :age,
+                              :sex,
+                              :steroid,
+                              :antivirals,
+                              :fatigue,
+                              :malaise,
+                              :anorexia,
+                              :liver_big,
+                              :liver_firm,
+                              :spleen_palpable,
+                              :spiders,
+                              :ascites,
+                              :varices,
+                              :bilirubin,
+                              :alkaline_phosphate,
+                              :sgot,
+                              :albumin,
+                              :protime,
+                              :histology)
+      def initialize(*values)
+        super()
+        members.zip(values) do |member, value|
+          __send__("#{member}=", value)
+        end
+      end
+
+      def label=(label)
+        case label
+        when "1"
+          super(:die)
+        when "2"
+          super(:live)
+        else
+          super(label)
+        end
+      end
+
+      def age=(age)
+        super(normalize_integer(age))
+      end
+
+      def sex=(sex)
+        case sex
+        when "1"
+          super(:male)
+        when "2"
+          super(:female)
+        else
+          super(sex)
+        end
+      end
+
+      def steroid=(steroid)
+        super(normalize_boolean(steroid))
+      end
+
+      def antivirals=(antivirals)
+        super(normalize_boolean(antivirals))
+      end
+
+      def fatigue=(fatigue)
+        super(normalize_boolean(fatigue))
+      end
+
+      def malaise=(malaise)
+        super(normalize_boolean(malaise))
+      end
+
+      def anorexia=(anorexia)
+        super(normalize_boolean(anorexia))
+      end
+
+      def liver_big=(liver_big)
+        super(normalize_boolean(liver_big))
+      end
+
+      def liver_firm=(liver_firm)
+        super(normalize_boolean(liver_firm))
+      end
+
+      def spleen_palpable=(spleen_palpable)
+        super(normalize_boolean(spleen_palpable))
+      end
+
+      def spiders=(spiders)
+        super(normalize_boolean(spiders))
+      end
+
+      def ascites=(ascites)
+        super(normalize_boolean(ascites))
+      end
+
+      def varices=(varices)
+        super(normalize_boolean(varices))
+      end
+
+      def bilirubin=(bilirubin)
+        super(normalize_float(bilirubin))
+      end
+
+      def alkaline_phosphate=(alkaline_phosphate)
+        super(normalize_integer(alkaline_phosphate))
+      end
+
+      def sgot=(sgot)
+        super(normalize_integer(sgot))
+      end
+
+      def albumin=(albumin)
+        super(normalize_float(albumin))
+      end
+
+      def protime=(protime)
+        super(normalize_integer(protime))
+      end
+
+      def histology=(histology)
+        super(normalize_boolean(histology))
+      end
+
+      private
+      def normalize_boolean(value)
+        case value
+        when "?"
+          nil
+        when "1"
+          false
+        when "2"
+          true
+        else
+          value
+        end
+      end
+
+      def normalize_float(value)
+        case value
+        when "?"
+          nil
+        else
+          Float(value)
+        end
+      end
+
+      def normalize_integer(value)
+        case value
+        when "?"
+          nil
+        else
+          Integer(value, 10)
+        end
+      end
+    end
+
+    def initialize
+      super()
+      @metadata.id = "hepatitis"
+      @metadata.name = "Hepatitis"
+      @metadata.url = "https://archive.ics.uci.edu/ml/datasets/hepatitis"
+      @metadata.description = lambda do
+        read_names
+      end
+    end
+
+    def each
+      return to_enum(__method__) unless block_given?
+
+      open_data do |csv|
+        csv.each do |row|
+          record = Record.new(*row)
+          yield(record)
+        end
+      end
+    end
+
+    private
+    def base_url
+      "https://archive.ics.uci.edu/ml/machine-learning-databases/hepatitis"
+    end
+
+    def open_data
+      data_path = cache_dir_path + "hepatitis.csv"
+      unless data_path.exist?
+        data_url = "#{base_url}/hepatitis.data"
+        download(data_path, data_url)
+      end
+      CSV.open(data_path) do |csv|
+        yield(csv)
+      end
+    end
+
+    def read_names
+      names_path = cache_dir_path + "hepatitis.names"
+      unless names_path.exist?
+        names_url = "#{base_url}/hepatitis.names"
+        download(names_path, names_url)
+      end
+      names_path.read
+    end
+  end
+end

data/lib/datasets/libsvm-dataset-list.rb

@@ -1,5 +1,6 @@
-require "English"
-require "rexml/document"
+require "rexml/streamlistener"
+require "rexml/parsers/baseparser"
+require "rexml/parsers/streamparser"
 
 require_relative "dataset"
 
@@ -32,26 +33,17 @@ module Datasets
       end
     end
 
-    def each
+    def each(&block)
       return to_enum(__method__) unless block_given?
 
       open_data do |input|
-        # TODO: Improve performance
-        document = REXML::Document.new(input)
-        is_header = true
-        document.each_element("//tr") do |tr|
-          if is_header
-            is_header = false
-            next
+        catch do |abort_tag|
+          listener = IndexListener.new(abort_tag) do |href, record|
+            parse_detail(href, record)
+            yield(record)
           end
-          name = tr.elements.first
-          a = name.elements.first
-          href = a.attributes["href"]
-          record = Record.new
-          record.name = a.text
-          record.files = []
-          parse_detail(href, record)
-          yield(record)
+          parser = REXML::Parsers::StreamParser.new(input, listener)
+          parser.parse
         end
       end
     end
@@ -69,17 +61,11 @@ module Datasets
 
     def extract_description
       open_data do |input|
-        document = REXML::Document.new(input)
         description = []
-        in_content = false
-        document.each_element("//body/*") do |element|
-          unless in_content
-            in_content = (element.name == "h1")
-            next
-          end
-          break if element.name == "hr"
-          content = extract_text(element)
-          description << content unless content.empty?
+        catch do |abort_tag|
+          listener = DescriptionListener.new(abort_tag, description)
+          parser = REXML::Parsers::StreamParser.new(input, listener)
+          parser.parse
         end
         description.join("\n\n")
       end
@@ -102,36 +88,190 @@ module Datasets
 
     def parse_detail(href, record)
       path, id = href.split("#")
-      open_detail(path) do |detail|
-        detail_document = REXML::Document.new(detail)
-        anchor = REXML::XPath.match(detail_document, "//*[@name='#{id}']")[0]
-        ul = anchor.next_sibling
-        ul.each_element do |li|
-          text = extract_text(li)
-          case text
-          when /\ASource: /
-            record.source = $POSTMATCH
-          when /\APreprocessing: /
-            record.preprocessing = $POSTMATCH
-          when /\A\# of classes: (\d+)/
-            record.n_classes = Integer($1, 10)
-          when /\A\# of data: ([\d,]+)/
-            record.n_data = Integer($1.gsub(/,/, ""), 10)
-          when /\A\# of features: ([\d,]+)/
-            record.n_features = Integer($1.gsub(/,/, ""), 10)
-          when /\AFiles:/
-            li.elements.first.each_element do |file_li|
-              file_a = file_li.elements.first
-              file = File.new
-              file.name = file_a.text
-              file.url = @metadata.url + file_a.attributes["href"]
-              file_note = file_li.text
-              file.note = file_note.strip.gsub(/[()]/, "") if file_note
-              record.files << file
+      open_detail(path) do |input|
+        catch do |abort_tag|
+          listener = DetailListener.new(abort_tag, id, @metadata.url, record)
+          parser = REXML::Parsers::StreamParser.new(input, listener)
+          parser.parse
+        end
+      end
+    end
+
+    class IndexListener
+      include REXML::StreamListener
+
+      def initialize(abort_tag, &block)
+        @abort_tag = abort_tag
+        @block = block
+        @row = nil
+        @in_td = false
+      end
+
+      def tag_start(name, attributes)
+        case name
+        when "tr"
+          @row = []
+        when "td"
+          @in_td = true
+          @row << {:text => ""}
+        when "a"
+          @row.last[:href] = attributes["href"] if @in_td
+        end
+      end
+
+      def tag_end(name)
+        case name
+        when "table"
+          throw(@abort_tag)
+        when "tr"
+          name_column = @row[0]
+          return unless name_column
+          record = Record.new
+          record.name = name_column[:text]
+          record.files = []
+          @block.call(name_column[:href], record)
+        when "td"
+          @in_td = false
+        end
+      end
+
+      def text(data)
+        @row.last[:text] << data if @in_td
+      end
+    end
+
+    class DetailListener
+      include REXML::StreamListener
+
+      def initialize(abort_tag, id, base_url, record)
+        @abort_tag = abort_tag
+        @id = id
+        @base_url = base_url
+        @record = record
+        @in_target = false
+        @target_li_level = nil
+        @key = nil
+        @data = nil
+        @file = nil
+      end
+
+      def tag_start(name, attributes)
+        if @in_target
+          case name
+          when "li"
+            @target_li_level += 1
+            case @target_li_level
+            when 0
+              @key = nil
+              @data = nil
+              @file = nil
+            when 1
+              @file = File.new
             end
+          when "a"
+            @file.url = @base_url + attributes["href"] if @file
+          end
+        else
+          if attributes["name"] == @id
+            @in_target = true
+            @target_li_level = -1
+          end
+        end
+      end
+
+      def tag_end(name)
+        if @in_target
+          case name
+          when "ul"
+            throw(@abort_tag) if @target_li_level == -1
+          when "li"
+            case @target_li_level
+            when 0
+              if @key
+                data = @data
+                data = data.gsub(/[ \t\n]+/, " ").strip if data.is_a?(String)
+                @record[@key] = data
+              end
+            when 1
+              @data << @file if @data and @file
+            end
+            @target_li_level -= 1
+          end
+        end
+      end
+
+      def text(data)
+        case @target_li_level
+        when 0
+          if @key
+            @data << data
+          else
+            case data.gsub(/[ \t\n]+/, " ")
+            when /\ASource: /
+              @key = :source
+              @data = $POSTMATCH
+            when /\APreprocessing: /
+              @key = :preprocessing
+              @data = $POSTMATCH
+            when /\A\# of classes: (\d+)/
+              @key = :n_classes
+              @data = Integer($1, 10)
+            when /\A\# of data: ([\d,]+)/
+              @key = :n_data
+              @data = Integer($1.gsub(/,/, ""), 10)
+            when /\A\# of features: ([\d,]+)/
+              @key = :n_features
+              @data = Integer($1.gsub(/,/, ""), 10)
+            when /\AFiles:/
+              @key = :files
+              @data = []
+            end
+          end
+        when 1
+          if @file.name.nil?
+            @file.name = data
+          else
+            @file.note = data.strip.gsub(/[()]/, "")
           end
         end
       end
     end
+
+    class DescriptionListener
+      include REXML::StreamListener
+
+      def initialize(abort_tag, description)
+        @abort_tag = abort_tag
+        @description = description
+        @in_content = false
+        @p = nil
+      end
+
+      def tag_start(name, attributes)
+        case name
+        when "p"
+          @in_content = true
+          @p = []
+        when "br"
+          @description << @p.join(" ")
+          @p = []
+        when "hr"
+          throw(@abort_tag)
+        end
+      end
+
+      def tag_end(name)
+        case name
+        when "p"
+          @description << @p.join(" ")
+        end
+      end
+
+      def text(data)
+        return unless @in_content
+        content = data.gsub(/[ \t\n]+/, " ").strip
+        @p << content unless content.empty?
+      end
+    end
   end
 end