red-datasets 0.0.6 → 0.1.1

data/lib/datasets/libsvm.rb

@@ -0,0 +1,135 @@
+require "csv"
+
+require_relative "dataset"
+
+module Datasets
+  class LIBSVM < Dataset
+    class Record
+      attr_reader :label
+      attr_reader :features
+      def initialize(label, features)
+        @label = label
+        @features = features
+      end
+
+      def [](index)
+        @features[index]
+      end
+
+      def to_h
+        hash = {
+          label: @label,
+        }
+        @features.each_with_index do |feature, i|
+          hash[i] = feature
+        end
+        hash
+      end
+
+      def values
+        [@label] + @features
+      end
+    end
+
+    def initialize(name,
+                   note: nil,
+                   default_feature_value: 0)
+      super()
+      @libsvm_dataset_metadata = fetch_dataset_info(name)
+      @file = choose_file(note)
+      @default_feature_value = default_feature_value
+      @metadata.id = "libsvm-#{normalize_name(name)}"
+      @metadata.name = "LIBSVM dataset: #{name}"
+      @metadata.url = "https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/"
+    end
+
+    def each
+      return to_enum(__method__) unless block_given?
+
+      open_data do |input|
+        n_features = @libsvm_dataset_metadata.n_features
+        csv = CSV.new(input, col_sep: " ")
+        csv.each do |row|
+          label = parse_label(row.shift)
+          features = [@default_feature_value] * n_features
+          row.each do |column|
+            next if column.nil?
+            index, value = column.split(":", 2)
+            features[Integer(index, 10) - 1] = parse_value(value)
+          end
+          yield(Record.new(label, features))
+        end
+      end
+    end
+
+    private
+    def fetch_dataset_info(name)
+      list = LIBSVMDatasetList.new
+      available_datasets = []
+      list.each do |record|
+        available_datasets << record.name
+        if record.name == name
+          return record
+        end
+      end
+      message = "unavailable LIBSVM dataset: #{name.inspect}: "
+      message << "available datasets: ["
+      message << available_datasets.collect(&:inspect).join(", ")
+      message << "]"
+      raise ArgumentError, message
+    end
+
+    def choose_file(note)
+      files = @libsvm_dataset_metadata.files
+      return files.first if note.nil?
+
+      available_notes = []
+      @libsvm_dataset_metadata.files.find do |file|
+        return file if file.note == note
+        available_notes << file.note if file.note
+      end
+
+      name = @libsvm_dataset_metadata.name
+      message = "unavailable note: #{name}: #{note.inspect}: "
+      message << "available notes: ["
+      message << available_notes.collect(&:inspect).join(", ")
+      message << "]"
+      raise ArgumentError, message
+    end
+
+    def open_data(&block)
+      data_path = cache_dir_path + @file.name
+      unless data_path.exist?
+        download(data_path, @file.url)
+      end
+      if data_path.extname == ".bz2"
+        extract_bz2(data_path, &block)
+      else
+        File.open(data_path, &block)
+      end
+    end
+
+    def normalize_name(name)
+      name.gsub(/[()]/, "").gsub(/[ _;]+/, "-").downcase
+    end
+
+    def parse_label(label)
+      labels = label.split(",").collect do |value|
+        parse_value(value)
+      end
+      if labels.size == 1
+        labels[0]
+      else
+        labels
+      end
+    end
+
+    def parse_value(value)
+      if value.include?(".")
+        Float(value)
+      else
+        Integer(value, 10)
+      end
+    end
+  end
+end

data/lib/datasets/mnist.rb

@@ -2,10 +2,9 @@ require 'zlib'
 
 require_relative "dataset"
 
-class SetTypeError < StandardError; end
-
 module Datasets
   class MNIST < Dataset
+    BASE_URL = "http://yann.lecun.com/exdb/mnist/"
 
     class Record < Struct.new(:data, :label)
       def pixels
@@ -26,9 +25,9 @@ module Datasets
 
       super()
 
-      @metadata.id = "mnist-#{type}"
-      @metadata.name = "MNIST: #{type}"
-      @metadata.url = "http://yann.lecun.com/exdb/mnist/"
+      @metadata.id = "#{dataset_name.downcase}-#{type}"
+      @metadata.name = "#{dataset_name}: #{type}"
+      @metadata.url = self.class::BASE_URL
       @type = type
 
       case type
@@ -44,7 +43,7 @@ module Datasets
 
       image_path = cache_dir_path + target_file(:image)
       label_path = cache_dir_path + target_file(:label)
-      base_url = "http://yann.lecun.com/exdb/mnist/"
+      base_url = self.class::BASE_URL
 
       unless image_path.exist?
         download(image_path, base_url + target_file(:image))
@@ -66,7 +65,7 @@ module Datasets
         n_bytes = n_uint32s * 4
         mnist_magic_number = 2051
         magic, n_images, n_rows, n_cols = f.read(n_bytes).unpack("N*")
-        raise 'This is not MNIST image file' if magic != mnist_magic_number
+        raise "This is not #{dataset_name} image file" if magic != mnist_magic_number
         n_images.times do |i|
           data = f.read(n_rows * n_cols)
           label = labels[i]
@@ -100,9 +99,13 @@ module Datasets
         n_bytes = n_uint32s * 2
         mnist_magic_number = 2049
         magic, n_labels = f.read(n_bytes).unpack('N2')
-        raise 'This is not MNIST label file' if magic != mnist_magic_number
+        raise "This is not #{dataset_name} label file" if magic != mnist_magic_number
         f.read(n_labels).unpack('C*')
       end
     end
+
+    def dataset_name
+      "MNIST"
+    end
   end
 end

data/lib/datasets/mushroom.rb

@@ -0,0 +1,256 @@
+require "csv"
+
+require_relative "dataset"
+
+module Datasets
+  class Mushroom < Dataset
+    Record = Struct.new(
+      :label,
+      :cap_shape,
+      :cap_surface,
+      :cap_color,
+      :bruises,
+      :odor,
+      :gill_attachment,
+      :gill_spacing,
+      :gill_size,
+      :gill_color,
+      :stalk_shape,
+      :stalk_root,
+      :stalk_surface_above_ring,
+      :stalk_surface_below_ring,
+      :stalk_color_above_ring,
+      :stalk_color_below_ring,
+      :veil_type,
+      :veil_color,
+      :n_rings,
+      :ring_type,
+      :spore_print_color,
+      :population,
+      :habitat,
+    )
+
+    def initialize
+      super()
+      @metadata.id = "mushroom"
+      @metadata.name = "Mushroom"
+      @metadata.url = "https://archive.ics.uci.edu/ml/datasets/mushroom"
+      @metadata.description = lambda do
+        read_names
+      end
+    end
+
+    def each
+      return to_enum(__method__) unless block_given?
+
+      open_data do |csv|
+        csv.each do |row|
+          next if row[0].nil?
+          record = Record.new(*row)
+          record.members.each do |member|
+            record[member] = CONVERTERS[member][record[member]]
+          end
+          yield(record)
+        end
+      end
+    end
+
+    private
+    def open_data
+      data_path = cache_dir_path + "agaricus-lepiota.data"
+      unless data_path.exist?
+        data_url = "http://archive.ics.uci.edu/ml/machine-learning-databases/mushroom/agaricus-lepiota.data"
+        download(data_path, data_url)
+      end
+      CSV.open(data_path) do |csv|
+        yield(csv)
+      end
+    end
+
+    def read_names
+      names_path = cache_dir_path + "agaricus-lepiota.names"
+      unless names_path.exist?
+        names_url = "https://archive.ics.uci.edu/ml/machine-learning-databases//mushroom/agaricus-lepiota.names"
+        download(names_path, names_url)
+      end
+      names_path.read
+    end
+
+    CONVERTERS = {
+      label: {
+        "p" => "poisonous",
+        "e" => "edible",
+      },
+      cap_shape: {
+        "b" => "bell",
+        "c" => "conical",
+        "x" => "convex",
+        "f" => "flat",
+        "k" => "knobbed",
+        "s" => "sunken",
+      },
+      cap_surface: {
+        "f" => "fibrous",
+        "g" => "grooves",
+        "y" => "scaly",
+        "s" => "smooth",
+      },
+      cap_color: {
+        "n" => "brown",
+        "b" => "buff",
+        "c" => "cinnamon",
+        "g" => "gray",
+        "r" => "green",
+        "p" => "pink",
+        "u" => "purple",
+        "e" => "red",
+        "w" => "white",
+        "y" => "yellow",
+      },
+      bruises: {
+        "t" => "bruises",
+        "f" => "no",
+      },
+      odor: {
+        "a" => "almond",
+        "l" => "anise",
+        "c" => "creosote",
+        "y" => "fishy",
+        "f" => "foul",
+        "m" => "musty",
+        "n" => "none",
+        "p" => "pungent",
+        "s" => "spicy",
+      },
+      gill_attachment: {
+        "a" => "attached",
+        "d" => "descending",
+        "f" => "free",
+        "n" => "notched",
+      },
+      gill_spacing: {
+        "c" => "close",
+        "w" => "crowded",
+        "d" => "distant",
+      },
+      gill_size: {
+        "b" => "broad",
+        "n" => "narrow",
+      },
+      gill_color: {
+        "k" => "black",
+        "n" => "brown",
+        "b" => "buff",
+        "h" => "chocolate",
+        "g" => "gray",
+        "r" => "green",
+        "o" => "orange",
+        "p" => "pink",
+        "u" => "purple",
+        "e" => "red",
+        "w" => "white",
+        "y" => "yellow",
+      },
+      stalk_shape: {
+        "e" => "enlarging",
+        "t" => "tapering",
+      },
+      stalk_root: {
+        "b" => "bulbous",
+        "c" => "club",
+        "u" => "cup",
+        "e" => "equal",
+        "z" => "rhizomorphs",
+        "r" => "rooted",
+        "?" => "missing",
+      },
+      stalk_surface_above_ring: {
+        "f" => "fibrous",
+        "y" => "scaly",
+        "k" => "silky",
+        "s" => "smooth",
+      },
+      stalk_surface_below_ring: {
+        "f" => "fibrous",
+        "y" => "scaly",
+        "k" => "silky",
+        "s" => "smooth",
+      },
+      stalk_color_above_ring: {
+        "n" => "brown",
+        "b" => "buff",
+        "c" => "cinnamon",
+        "g" => "gray",
+        "o" => "orange",
+        "p" => "pink",
+        "e" => "red",
+        "w" => "white",
+        "y" => "yellow",
+      },
+      stalk_color_below_ring: {
+        "n" => "brown",
+        "b" => "buff",
+        "c" => "cinnamon",
+        "g" => "gray",
+        "o" => "orange",
+        "p" => "pink",
+        "e" => "red",
+        "w" => "white",
+        "y" => "yellow",
+      },
+      veil_type: {
+        "p" => "partial",
+        "u" => "universal",
+      },
+      veil_color: {
+        "n" => "brown",
+        "o" => "orange",
+        "w" => "white",
+        "y" => "yellow",
+      },
+      n_rings: {
+        "n" => 0,
+        "o" => 1,
+        "t" => 2,
+      },
+      ring_type: {
+        "c" => "cobwebby",
+        "e" => "evanescent",
+        "f" => "flaring",
+        "l" => "large",
+        "n" => "none",
+        "p" => "pendant",
+        "s" => "sheathing",
+        "z" => "zone",
+      },
+      spore_print_color: {
+        "k" => "black",
+        "n" => "brown",
+        "b" => "buff",
+        "h" => "chocolate",
+        "r" => "green",
+        "o" => "orange",
+        "u" => "purple",
+        "w" => "white",
+        "y" => "yellow",
+      },
+      population: {
+        "a" => "abundant",
+        "c" => "clustered",
+        "n" => "numerous",
+        "s" => "scattered",
+        "v" => "several",
+        "y" => "solitary",
+      },
+      habitat: {
+        "g" => "grasses",
+        "l" => "leaves",
+        "m" => "meadows",
+        "p" => "paths",
+        "u" => "urban",
+        "w" => "waste",
+        "d" => "woods",
+      }
+    }
+  end
+end

data/lib/datasets/penguins.rb

@@ -0,0 +1,125 @@
+require_relative "dataset"
+
+module Datasets
+  module PenguinsRawData
+    Record = Struct.new(:study_name,
+                        :sample_number,
+                        :species,
+                        :region,
+                        :island,
+                        :stage,
+                        :individual_id,
+                        :clutch_completion,
+                        :date_egg,
+                        :culmen_length_mm,
+                        :culmen_depth_mm,
+                        :flipper_length_mm,
+                        :body_mass_g,
+                        :sex,
+                        :delta_15_n_permil,
+                        :delta_13_c_permil,
+                        :comments)
+
+    class SpeciesBase < Dataset
+      def initialize
+        super
+        species = self.class.name.split("::").last.downcase
+        @metadata.id = "palmerpenguins-raw-#{species}"
+        @metadata.url = self.class::URL
+        @metadata.licenses = ["CC0"]
+        @data_path = cache_dir_path + "penguins" + (species + ".csv")
+      end
+
+      attr_reader :data_path
+
+      def each
+        return to_enum(__method__) unless block_given?
+
+        open_data do |csv|
+          csv.each do |row|
+            next if row[0].nil?
+            record = Record.new(*row.fields)
+            yield record
+          end
+        end
+      end
+
+      private def open_data
+        download unless data_path.exist?
+        CSV.open(data_path, headers: :first_row, converters: :all) do |csv|
+          yield csv
+        end
+      end
+
+      private def download
+        super(data_path, metadata.url)
+      end
+    end
+
+    # Adelie penguin data from: https://doi.org/10.6073/pasta/abc50eed9138b75f54eaada0841b9b86
+    class Adelie < SpeciesBase
+      DOI = "doi.org/10.6073/pasta/abc50eed9138b75f54eaada0841b9b86".freeze
+      URL = "https://portal.edirepository.org/nis/dataviewer?packageid=knb-lter-pal.219.3&entityid=002f3893385f710df69eeebe893144ff".freeze
+    end
+
+    # Gentoo penguin data from: https://doi.org/10.6073/pasta/2b1cff60f81640f182433d23e68541ce
+    class Gentoo < SpeciesBase
+      DOI = "doi.org/10.6073/pasta/2b1cff60f81640f182433d23e68541ce".freeze
+      URL = "https://portal.edirepository.org/nis/dataviewer?packageid=knb-lter-pal.220.3&entityid=e03b43c924f226486f2f0ab6709d2381".freeze
+    end
+
+    # Chinstrap penguin data from: https://doi.org/10.6073/pasta/409c808f8fc9899d02401bdb04580af7
+    class Chinstrap < SpeciesBase
+      DOI = "doi.org/10.6073/pasta/409c808f8fc9899d02401bdb04580af7".freeze
+      URL = "https://portal.edirepository.org/nis/dataviewer?packageid=knb-lter-pal.221.2&entityid=fe853aa8f7a59aa84cdd3197619ef462".freeze
+    end
+  end
+
+  # This dataset provides the same dataset as https://github.com/allisonhorst/palmerpenguins
+  class Penguins < Dataset
+    Record = Struct.new(:species,
+                        :island,
+                        :bill_length_mm,
+                        :bill_depth_mm,
+                        :flipper_length_mm,
+                        :body_mass_g,
+                        :sex,
+                        :year)
+
+    def initialize
+      super
+      @metadata.id = "palmerpenguins"
+      @metadata.name = "palmerpenguins"
+      @metadata.url = "https://allisonhorst.github.io/palmerpenguins/"
+      @metadata.licenses = ["CC0"]
+      @metadata.description = "A great dataset for data exploration & visualization, as an alternative to iris"
+    end
+
+    def each(&block)
+      return to_enum(__method__) unless block_given?
+
+      species_classes = [
+        PenguinsRawData::Adelie,
+        PenguinsRawData::Gentoo,
+        PenguinsRawData::Chinstrap
+      ]
+
+      species_classes.each do |species_class|
+        species_class.new.each do |raw_record|
+          yield convert_record(raw_record)
+        end
+      end
+    end
+
+    private def convert_record(raw_record)
+      Record.new(raw_record.species.split(' ')[0],
+                 raw_record.island,
+                 raw_record.culmen_length_mm,
+                 raw_record.culmen_depth_mm,
+                 raw_record.flipper_length_mm&.to_i,
+                 raw_record.body_mass_g&.to_i,
+                 raw_record.sex&.downcase,
+                 raw_record.date_egg&.year)
+    end
+  end
+end