red-datasets 0.0.8 → 0.1.3

data/lib/datasets/libsvm.rb

@@ -103,15 +103,7 @@ module Datasets
         download(data_path, @file.url)
       end
       if data_path.extname == ".bz2"
-        input, output = IO.pipe
-        pid = spawn("bzcat", data_path.to_s, {:out => output})
-        begin
-          output.close
-          yield(input)
-        ensure
-          input.close
-          Process.waitpid(pid)
-        end
+        extract_bz2(data_path, &block)
       else
         File.open(data_path, &block)
       end

data/lib/datasets/mnist.rb

@@ -2,8 +2,6 @@ require 'zlib'
 
 require_relative "dataset"
 
-class SetTypeError < StandardError; end
-
 module Datasets
   class MNIST < Dataset
     BASE_URL = "http://yann.lecun.com/exdb/mnist/"
@@ -67,7 +65,9 @@ module Datasets
         n_bytes = n_uint32s * 4
         mnist_magic_number = 2051
         magic, n_images, n_rows, n_cols = f.read(n_bytes).unpack("N*")
-        raise "This is not #{dataset_name} image file" if magic != mnist_magic_number
+        if magic != mnist_magic_number
+          raise Error, "This is not #{dataset_name} image file"
+        end
         n_images.times do |i|
           data = f.read(n_rows * n_cols)
           label = labels[i]
@@ -101,7 +101,9 @@ module Datasets
         n_bytes = n_uint32s * 2
         mnist_magic_number = 2049
         magic, n_labels = f.read(n_bytes).unpack('N2')
-        raise "This is not #{dataset_name} label file" if magic != mnist_magic_number
+        if magic != mnist_magic_number
+          raise Error, "This is not #{dataset_name} label file"
+        end
         f.read(n_labels).unpack('C*')
       end
     end

data/lib/datasets/mushroom.rb

@@ -0,0 +1,256 @@
+require "csv"
+
+require_relative "dataset"
+
+module Datasets
+  class Mushroom < Dataset
+    Record = Struct.new(
+      :label,
+      :cap_shape,
+      :cap_surface,
+      :cap_color,
+      :bruises,
+      :odor,
+      :gill_attachment,
+      :gill_spacing,
+      :gill_size,
+      :gill_color,
+      :stalk_shape,
+      :stalk_root,
+      :stalk_surface_above_ring,
+      :stalk_surface_below_ring,
+      :stalk_color_above_ring,
+      :stalk_color_below_ring,
+      :veil_type,
+      :veil_color,
+      :n_rings,
+      :ring_type,
+      :spore_print_color,
+      :population,
+      :habitat,
+    )
+
+    def initialize
+      super()
+      @metadata.id = "mushroom"
+      @metadata.name = "Mushroom"
+      @metadata.url = "https://archive.ics.uci.edu/ml/datasets/mushroom"
+      @metadata.description = lambda do
+        read_names
+      end
+    end
+
+    def each
+      return to_enum(__method__) unless block_given?
+
+      open_data do |csv|
+        csv.each do |row|
+          next if row[0].nil?
+          record = Record.new(*row)
+          record.members.each do |member|
+            record[member] = CONVERTERS[member][record[member]]
+          end
+          yield(record)
+        end
+      end
+    end
+
+    private
+    def open_data
+      data_path = cache_dir_path + "agaricus-lepiota.data"
+      unless data_path.exist?
+        data_url = "http://archive.ics.uci.edu/ml/machine-learning-databases/mushroom/agaricus-lepiota.data"
+        download(data_path, data_url)
+      end
+      CSV.open(data_path) do |csv|
+        yield(csv)
+      end
+    end
+
+    def read_names
+      names_path = cache_dir_path + "agaricus-lepiota.names"
+      unless names_path.exist?
+        names_url = "https://archive.ics.uci.edu/ml/machine-learning-databases//mushroom/agaricus-lepiota.names"
+        download(names_path, names_url)
+      end
+      names_path.read
+    end
+
+    CONVERTERS = {
+      label: {
+        "p" => "poisonous",
+        "e" => "edible",
+      },
+      cap_shape: {
+        "b" => "bell",
+        "c" => "conical",
+        "x" => "convex",
+        "f" => "flat",
+        "k" => "knobbed",
+        "s" => "sunken",
+      },
+      cap_surface: {
+        "f" => "fibrous",
+        "g" => "grooves",
+        "y" => "scaly",
+        "s" => "smooth",
+      },
+      cap_color: {
+        "n" => "brown",
+        "b" => "buff",
+        "c" => "cinnamon",
+        "g" => "gray",
+        "r" => "green",
+        "p" => "pink",
+        "u" => "purple",
+        "e" => "red",
+        "w" => "white",
+        "y" => "yellow",
+      },
+      bruises: {
+        "t" => "bruises",
+        "f" => "no",
+      },
+      odor: {
+        "a" => "almond",
+        "l" => "anise",
+        "c" => "creosote",
+        "y" => "fishy",
+        "f" => "foul",
+        "m" => "musty",
+        "n" => "none",
+        "p" => "pungent",
+        "s" => "spicy",
+      },
+      gill_attachment: {
+        "a" => "attached",
+        "d" => "descending",
+        "f" => "free",
+        "n" => "notched",
+      },
+      gill_spacing: {
+        "c" => "close",
+        "w" => "crowded",
+        "d" => "distant",
+      },
+      gill_size: {
+        "b" => "broad",
+        "n" => "narrow",
+      },
+      gill_color: {
+        "k" => "black",
+        "n" => "brown",
+        "b" => "buff",
+        "h" => "chocolate",
+        "g" => "gray",
+        "r" => "green",
+        "o" => "orange",
+        "p" => "pink",
+        "u" => "purple",
+        "e" => "red",
+        "w" => "white",
+        "y" => "yellow",
+      },
+      stalk_shape: {
+        "e" => "enlarging",
+        "t" => "tapering",
+      },
+      stalk_root: {
+        "b" => "bulbous",
+        "c" => "club",
+        "u" => "cup",
+        "e" => "equal",
+        "z" => "rhizomorphs",
+        "r" => "rooted",
+        "?" => "missing",
+      },
+      stalk_surface_above_ring: {
+        "f" => "fibrous",
+        "y" => "scaly",
+        "k" => "silky",
+        "s" => "smooth",
+      },
+      stalk_surface_below_ring: {
+        "f" => "fibrous",
+        "y" => "scaly",
+        "k" => "silky",
+        "s" => "smooth",
+      },
+      stalk_color_above_ring: {
+        "n" => "brown",
+        "b" => "buff",
+        "c" => "cinnamon",
+        "g" => "gray",
+        "o" => "orange",
+        "p" => "pink",
+        "e" => "red",
+        "w" => "white",
+        "y" => "yellow",
+      },
+      stalk_color_below_ring: {
+        "n" => "brown",
+        "b" => "buff",
+        "c" => "cinnamon",
+        "g" => "gray",
+        "o" => "orange",
+        "p" => "pink",
+        "e" => "red",
+        "w" => "white",
+        "y" => "yellow",
+      },
+      veil_type: {
+        "p" => "partial",
+        "u" => "universal",
+      },
+      veil_color: {
+        "n" => "brown",
+        "o" => "orange",
+        "w" => "white",
+        "y" => "yellow",
+      },
+      n_rings: {
+        "n" => 0,
+        "o" => 1,
+        "t" => 2,
+      },
+      ring_type: {
+        "c" => "cobwebby",
+        "e" => "evanescent",
+        "f" => "flaring",
+        "l" => "large",
+        "n" => "none",
+        "p" => "pendant",
+        "s" => "sheathing",
+        "z" => "zone",
+      },
+      spore_print_color: {
+        "k" => "black",
+        "n" => "brown",
+        "b" => "buff",
+        "h" => "chocolate",
+        "r" => "green",
+        "o" => "orange",
+        "u" => "purple",
+        "w" => "white",
+        "y" => "yellow",
+      },
+      population: {
+        "a" => "abundant",
+        "c" => "clustered",
+        "n" => "numerous",
+        "s" => "scattered",
+        "v" => "several",
+        "y" => "solitary",
+      },
+      habitat: {
+        "g" => "grasses",
+        "l" => "leaves",
+        "m" => "meadows",
+        "p" => "paths",
+        "u" => "urban",
+        "w" => "waste",
+        "d" => "woods",
+      }
+    }
+  end
+end

data/lib/datasets/penguins.rb

@@ -0,0 +1,146 @@
+require_relative "dataset"
+
+module Datasets
+  module PenguinsRawData
+    Record = Struct.new(:study_name,
+                        :sample_number,
+                        :species,
+                        :region,
+                        :island,
+                        :stage,
+                        :individual_id,
+                        :clutch_completion,
+                        :date_egg,
+                        :culmen_length_mm,
+                        :culmen_depth_mm,
+                        :flipper_length_mm,
+                        :body_mass_g,
+                        :sex,
+                        :delta_15_n_permil,
+                        :delta_13_c_permil,
+                        :comments)
+    class SpeciesBase < Dataset
+      def initialize
+        super
+        species = self.class.name.split("::").last.downcase
+        @metadata.id = "palmerpenguins-raw-#{species}"
+        @metadata.url = self.class::URL
+        @metadata.licenses = ["CC0"]
+        @data_path = cache_dir_path + "penguins" + (species + ".csv")
+      end
+
+      attr_reader :data_path
+
+      def each
+        return to_enum(__method__) unless block_given?
+
+        open_data do |csv|
+          csv.each do |row|
+            next if row[0].nil?
+            record = Record.new(*row.fields)
+            yield record
+          end
+        end
+      end
+
+      private def open_data
+        download unless data_path.exist?
+        CSV.open(data_path, headers: :first_row, converters: :all) do |csv|
+          yield csv
+        end
+      end
+
+      private def download
+        super(data_path, metadata.url)
+      end
+    end
+
+    # Adelie penguin data from: https://doi.org/10.6073/pasta/abc50eed9138b75f54eaada0841b9b86
+    class Adelie < SpeciesBase
+      DOI = "doi.org/10.6073/pasta/abc50eed9138b75f54eaada0841b9b86".freeze
+      URL = "https://portal.edirepository.org/nis/dataviewer?packageid=knb-lter-pal.219.3&entityid=002f3893385f710df69eeebe893144ff".freeze
+    end
+
+    # Chinstrap penguin data from: https://doi.org/10.6073/pasta/409c808f8fc9899d02401bdb04580af7
+    class Chinstrap < SpeciesBase
+      DOI = "doi.org/10.6073/pasta/409c808f8fc9899d02401bdb04580af7".freeze
+      URL = "https://portal.edirepository.org/nis/dataviewer?packageid=knb-lter-pal.221.2&entityid=fe853aa8f7a59aa84cdd3197619ef462".freeze
+    end
+
+    # Gentoo penguin data from: https://doi.org/10.6073/pasta/2b1cff60f81640f182433d23e68541ce
+    class Gentoo < SpeciesBase
+      DOI = "doi.org/10.6073/pasta/2b1cff60f81640f182433d23e68541ce".freeze
+      URL = "https://portal.edirepository.org/nis/dataviewer?packageid=knb-lter-pal.220.3&entityid=e03b43c924f226486f2f0ab6709d2381".freeze
+    end
+  end
+
+  # This dataset provides the same dataset as https://github.com/allisonhorst/palmerpenguins
+  class Penguins < Dataset
+    Record = Struct.new(:species,
+                        :island,
+                        :bill_length_mm,
+                        :bill_depth_mm,
+                        :flipper_length_mm,
+                        :body_mass_g,
+                        :sex,
+                        :year)
+
+    def initialize
+      super
+      @metadata.id = "palmerpenguins"
+      @metadata.name = "palmerpenguins"
+      @metadata.url = "https://allisonhorst.github.io/palmerpenguins/"
+      @metadata.licenses = ["CC0"]
+      @metadata.description = "A great dataset for data exploration & visualization, as an alternative to iris"
+    end
+
+    def each(&block)
+      return to_enum(__method__) unless block_given?
+
+      species_classes = [
+        PenguinsRawData::Adelie,
+        PenguinsRawData::Chinstrap,
+        PenguinsRawData::Gentoo,
+      ]
+
+      species_classes.each do |species_class|
+        species_class.new.each do |raw_record|
+          yield convert_record(raw_record)
+        end
+      end
+    end
+
+    private def convert_record(raw_record)
+      Record.new(*cleanse_fields(raw_record))
+    end
+
+    private def cleanse_fields(raw_record)
+      species = raw_record.species.split(' ')[0]
+      flipper_length_mm = raw_record.flipper_length_mm&.to_i
+      body_mass_g = raw_record.body_mass_g&.to_i
+      sex = normalize_sex(raw_record.sex)
+      year = raw_record.date_egg&.year
+
+      [
+        species,
+        raw_record.island,
+        raw_record.culmen_length_mm,
+        raw_record.culmen_depth_mm,
+        flipper_length_mm,
+        body_mass_g,
+        sex,
+        year
+      ]
+    end
+
+    private def normalize_sex(val)
+      val = val&.downcase
+      case val
+      when "female", "male", nil
+        val
+      else
+        nil
+      end
+    end
+  end
+end