red-datasets 0.1.0 → 0.1.4

data/lib/datasets/error.rb

@@ -0,0 +1,4 @@
+module Datasets
+  class Error < StandardError
+  end
+end

data/lib/datasets/mnist.rb

@@ -2,8 +2,6 @@ require 'zlib'
 
 require_relative "dataset"
 
-class SetTypeError < StandardError; end
-
 module Datasets
   class MNIST < Dataset
     BASE_URL = "http://yann.lecun.com/exdb/mnist/"
@@ -67,7 +65,9 @@ module Datasets
         n_bytes = n_uint32s * 4
         mnist_magic_number = 2051
         magic, n_images, n_rows, n_cols = f.read(n_bytes).unpack("N*")
-        raise "This is not #{dataset_name} image file" if magic != mnist_magic_number
+        if magic != mnist_magic_number
+          raise Error, "This is not #{dataset_name} image file"
+        end
         n_images.times do |i|
           data = f.read(n_rows * n_cols)
           label = labels[i]
@@ -101,7 +101,9 @@ module Datasets
         n_bytes = n_uint32s * 2
         mnist_magic_number = 2049
         magic, n_labels = f.read(n_bytes).unpack('N2')
-        raise "This is not #{dataset_name} label file" if magic != mnist_magic_number
+        if magic != mnist_magic_number
+          raise Error, "This is not #{dataset_name} label file"
+        end
         f.read(n_labels).unpack('C*')
       end
     end

data/lib/datasets/penguins.rb

@@ -0,0 +1,146 @@
+require_relative "dataset"
+
+module Datasets
+  module PenguinsRawData
+    Record = Struct.new(:study_name,
+                        :sample_number,
+                        :species,
+                        :region,
+                        :island,
+                        :stage,
+                        :individual_id,
+                        :clutch_completion,
+                        :date_egg,
+                        :culmen_length_mm,
+                        :culmen_depth_mm,
+                        :flipper_length_mm,
+                        :body_mass_g,
+                        :sex,
+                        :delta_15_n_permil,
+                        :delta_13_c_permil,
+                        :comments)
+    class SpeciesBase < Dataset
+      def initialize
+        super
+        species = self.class.name.split("::").last.downcase
+        @metadata.id = "palmerpenguins-raw-#{species}"
+        @metadata.url = self.class::URL
+        @metadata.licenses = ["CC0"]
+        @data_path = cache_dir_path + "penguins" + (species + ".csv")
+      end
+
+      attr_reader :data_path
+
+      def each
+        return to_enum(__method__) unless block_given?
+
+        open_data do |csv|
+          csv.each do |row|
+            next if row[0].nil?
+            record = Record.new(*row.fields)
+            yield record
+          end
+        end
+      end
+
+      private def open_data
+        download unless data_path.exist?
+        CSV.open(data_path, headers: :first_row, converters: :all) do |csv|
+          yield csv
+        end
+      end
+
+      private def download
+        super(data_path, metadata.url)
+      end
+    end
+
+    # Adelie penguin data from: https://doi.org/10.6073/pasta/abc50eed9138b75f54eaada0841b9b86
+    class Adelie < SpeciesBase
+      DOI = "doi.org/10.6073/pasta/abc50eed9138b75f54eaada0841b9b86".freeze
+      URL = "https://portal.edirepository.org/nis/dataviewer?packageid=knb-lter-pal.219.3&entityid=002f3893385f710df69eeebe893144ff".freeze
+    end
+
+    # Chinstrap penguin data from: https://doi.org/10.6073/pasta/409c808f8fc9899d02401bdb04580af7
+    class Chinstrap < SpeciesBase
+      DOI = "doi.org/10.6073/pasta/409c808f8fc9899d02401bdb04580af7".freeze
+      URL = "https://portal.edirepository.org/nis/dataviewer?packageid=knb-lter-pal.221.2&entityid=fe853aa8f7a59aa84cdd3197619ef462".freeze
+    end
+
+    # Gentoo penguin data from: https://doi.org/10.6073/pasta/2b1cff60f81640f182433d23e68541ce
+    class Gentoo < SpeciesBase
+      DOI = "doi.org/10.6073/pasta/2b1cff60f81640f182433d23e68541ce".freeze
+      URL = "https://portal.edirepository.org/nis/dataviewer?packageid=knb-lter-pal.220.3&entityid=e03b43c924f226486f2f0ab6709d2381".freeze
+    end
+  end
+
+  # This dataset provides the same dataset as https://github.com/allisonhorst/palmerpenguins
+  class Penguins < Dataset
+    Record = Struct.new(:species,
+                        :island,
+                        :bill_length_mm,
+                        :bill_depth_mm,
+                        :flipper_length_mm,
+                        :body_mass_g,
+                        :sex,
+                        :year)
+
+    def initialize
+      super
+      @metadata.id = "palmerpenguins"
+      @metadata.name = "palmerpenguins"
+      @metadata.url = "https://allisonhorst.github.io/palmerpenguins/"
+      @metadata.licenses = ["CC0"]
+      @metadata.description = "A great dataset for data exploration & visualization, as an alternative to iris"
+    end
+
+    def each(&block)
+      return to_enum(__method__) unless block_given?
+
+      species_classes = [
+        PenguinsRawData::Adelie,
+        PenguinsRawData::Chinstrap,
+        PenguinsRawData::Gentoo,
+      ]
+
+      species_classes.each do |species_class|
+        species_class.new.each do |raw_record|
+          yield convert_record(raw_record)
+        end
+      end
+    end
+
+    private def convert_record(raw_record)
+      Record.new(*cleanse_fields(raw_record))
+    end
+
+    private def cleanse_fields(raw_record)
+      species = raw_record.species.split(' ')[0]
+      flipper_length_mm = raw_record.flipper_length_mm&.to_i
+      body_mass_g = raw_record.body_mass_g&.to_i
+      sex = normalize_sex(raw_record.sex)
+      year = raw_record.date_egg&.year
+
+      [
+        species,
+        raw_record.island,
+        raw_record.culmen_length_mm,
+        raw_record.culmen_depth_mm,
+        flipper_length_mm,
+        body_mass_g,
+        sex,
+        year
+      ]
+    end
+
+    private def normalize_sex(val)
+      val = val&.downcase
+      case val
+      when "female", "male", nil
+        val
+      else
+        nil
+      end
+    end
+  end
+end

data/lib/datasets/rdatasets.rb

@@ -0,0 +1,95 @@
+require_relative "dataset"
+require_relative "tar-gz-readable"
+
+module Datasets
+  class RdatasetsList < Dataset
+    Record = Struct.new(:package,
+                        :dataset,
+                        :title,
+                        :rows,
+                        :cols,
+                        :n_binary,
+                        :n_character,
+                        :n_factor,
+                        :n_logical,
+                        :n_numeric,
+                        :csv,
+                        :doc)
+
+    def initialize
+      super
+      @metadata.id = "rdatasets"
+      @metadata.name = "Rdatasets"
+      @metadata.url = "https://vincentarelbundock.github.io/Rdatasets/"
+      @metadata.licenses = ["GPL-3"]
+      @data_url = "https://raw.githubusercontent.com/vincentarelbundock/Rdatasets/master/datasets.csv"
+      @data_path = cache_dir_path + "datasets.csv"
+    end
+
+    def filter(package: nil, dataset: nil)
+      return to_enum(__method__, package: package, dataset: dataset) unless block_given?
+
+      conds = {}
+      conds["Package"] = package if package
+      conds["Item"]    = dataset if dataset
+      if conds.empty?
+        each_row {|row| yield Record.new(*row.fields) }
+      else
+        each_row do |row|
+          if conds.all? {|k, v| row[k] == v }
+            yield Record.new(*row.fields)
+          end
+        end
+      end
+    end
+
+    def each(&block)
+      filter(&block)
+    end
+
+    private def each_row(&block)
+      download(@data_path, @data_url) unless @data_path.exist?
+      CSV.open(@data_path, headers: :first_row, converters: :all) do |csv|
+        csv.each(&block)
+      end
+    end
+  end
+
+  class Rdatasets < Dataset
+    def initialize(package_name, dataset_name)
+      list = RdatasetsList.new
+
+      info = list.filter(package: package_name, dataset: dataset_name).first
+      unless info
+        raise ArgumentError, "Unable to locate dataset #{package_name}/#{dataset_name}"
+      end
+
+      super()
+      @metadata.id = "rdatasets-#{package_name}-#{dataset_name}"
+      @metadata.name = "Rdatasets: #{package_name}: #{dataset_name}"
+      @metadata.url = info.csv
+      @metadata.licenses = ["GPL-3"]
+      @metadata.description = info.title
+
+      # Follow the original directory structure in the cache directory
+      @data_path = cache_dir_path + (dataset_name + ".csv")
+
+      @package_name = package_name
+      @dataset_name = dataset_name
+    end
+
+    def each(&block)
+      return to_enum(__method__) unless block_given?
+
+      download(@data_path, @metadata.url) unless @data_path.exist?
+      CSV.open(@data_path, headers: :first_row, converters: :all) do |csv|
+        csv.each do |row|
+          record = row.to_h
+          record.delete("")
+          record.transform_keys!(&:to_sym)
+          yield record
+        end
+      end
+    end
+  end
+end

data/lib/datasets/seaborn-data.rb

@@ -0,0 +1,49 @@
+module Datasets
+  class SeabornData < Dataset
+    URL_FORMAT = "https://raw.githubusercontent.com/mwaskom/seaborn-data/master/%{name}.csv".freeze
+
+    def initialize(name)
+      super()
+      @metadata.id = "seaborn-data-#{name}"
+      @metadata.name = "SeabornData: #{name}"
+      @metadata.url = URL_FORMAT % {name: name}
+
+      @data_path = cache_dir_path + (name + ".csv")
+      @name = name
+    end
+
+    def each(&block)
+      return to_enum(__method__) unless block_given?
+
+      download(@data_path, @metadata.url) unless @data_path.exist?
+      CSV.open(@data_path, headers: :first_row, converters: :all) do |csv|
+        csv.each do |row|
+          record = prepare_record(row)
+          yield record
+        end
+      end
+    end
+
+    private
+    def prepare_record(csv_row)
+      record = csv_row.to_h
+      record.transform_keys!(&:to_sym)
+
+      # Perform the same preprocessing as seaborn's load_dataset function
+      preprocessor = :"preprocess_#{@name}_record"
+      __send__(preprocessor, record) if respond_to?(preprocessor, true)
+
+      record
+    end
+
+    # The same preprocessing as seaborn.load_dataset
+    def preprocess_flights_record(record)
+      record[:month] &&= record[:month][0,3]
+    end
+
+    # The same preprocessing as seaborn.load_dataset
+    def preprocess_penguins_record(record)
+      record[:sex] &&= record[:sex].capitalize
+    end
+  end
+end

data/lib/datasets/sudachi-synonym-dictionary.rb

@@ -0,0 +1,171 @@
+require "csv"
+
+require_relative "dataset"
+
+module Datasets
+  class SudachiSynonymDictionary < Dataset
+    class Synonym < Struct.new(:group_id,
+                               :is_noun,
+                               :expansion_type,
+                               :lexeme_id,
+                               :form_type,
+                               :acronym_type,
+                               :variant_type,
+                               :categories,
+                               :notation)
+      alias_method :noun?, :is_noun
+    end
+
+    def initialize
+      super()
+      @metadata.id = "sudachi-synonym-dictionary"
+      @metadata.name = "Sudachi synonym dictionary"
+      @metadata.url = "https://github.com/WorksApplications/SudachiDict/blob/develop/docs/synonyms.md"
+      @metadata.licenses = [
+        "Apache-2.0",
+      ]
+      @metadata.description = lambda do
+        download_description
+      end
+    end
+
+    def each
+      return to_enum(__method__) unless block_given?
+
+      lexeme_id_context = {}
+      open_data do |csv|
+        csv.each do |row|
+          group_id = row[0]
+          if group_id != lexeme_id_context[:group_id]
+            lexeme_id_context[:group_id] = group_id
+            lexeme_id_context[:counter] = 0
+          end
+          is_noun = (row[1] == "1")
+          expansion_type = normalize_expansion_type(row[2])
+          lexeme_id = normalize_lexeme_id(row[3], lexeme_id_context)
+          form_type = normalize_form_type(row[4])
+          acronym_type = normalize_acronym_type(row[5])
+          variant_type = normalize_variant_type(row[6])
+          categories = normalize_categories(row[7])
+          notation = row[8]
+          synonym = Synonym.new(group_id,
+                                is_noun,
+                                expansion_type,
+                                lexeme_id,
+                                form_type,
+                                acronym_type,
+                                variant_type,
+                                categories,
+                                notation)
+          yield(synonym)
+        end
+      end
+    end
+
+    private
+    def open_data
+      data_path = cache_dir_path + "synonyms.txt"
+      unless data_path.exist?
+        data_url = "https://raw.githubusercontent.com/WorksApplications/SudachiDict/develop/src/main/text/synonyms.txt"
+        download(data_path, data_url)
+      end
+      CSV.open(data_path,
+               encoding: "UTF-8",
+               skip_blanks: true) do |csv|
+        yield(csv)
+      end
+    end
+
+    def download_description
+      description_path = cache_dir_path + "synonyms.md"
+      unless description_path.exist?
+        description_url = "https://raw.githubusercontent.com/WorksApplications/SudachiDict/develop/docs/synonyms.md"
+        download(description_path, description_url)
+      end
+      description_path.read
+    end
+
+    def normalize_expansion_type(type)
+      case type
+      when "0", ""
+        :always
+      when "1"
+        :expanded
+      when "2"
+        :never
+      else
+        raise Error, "unknown expansion type: #{type.inspect}"
+      end
+    end
+
+    def normalize_lexeme_id(id, context)
+      case id
+      when ""
+        lexeme_id_context[:counter] += 1
+        lexeme_id_context[:counter]
+      else
+        # Use only the first lexeme ID.
+        # Example:
+        #   000116,1,0,1/2,0,2,0,(IT/娯楽),ネットゲー,,
+        #   000116,1,0,1/2,0,2,0,(IT/娯楽),ネトゲ,,
+        Integer(id.split("/").first, 10)
+      end
+    end
+
+    def normalize_form_type(type)
+      case type
+      when "0", ""
+        :typical
+      when "1"
+        :translation
+      when "2"
+        :alias
+      when "3"
+        :old_name
+      when "4"
+        :misnomer
+      else
+        raise Error, "unknown form type: #{type.inspect}"
+      end
+    end
+
+    def normalize_acronym_type(type)
+      case type
+      when "0", ""
+        :typical
+      when "1"
+        :alphabet
+      when "2"
+        :others
+      else
+        raise Error, "unknown acronym type: #{type.inspect}"
+      end
+    end
+
+    def normalize_variant_type(type)
+      case type
+      when "0", ""
+        :typical
+      when "1"
+        :alphabet
+      when "2"
+        :general
+      when "3"
+        :misspelled
+      else
+        raise Error, "unknown variant type: #{type.inspect}"
+      end
+    end
+
+    def normalize_categories(categories)
+      case categories
+      when ""
+        nil
+      when /\A\((.*)\)\z/
+        $1.split("/")
+      else
+        raise Error, "invalid categories: #{categories.inspect}"
+      end
+    end
+  end
+end