red-datasets 0.1.4 → 0.1.5

data/lib/datasets/fuel-economy.rb

@@ -0,0 +1,35 @@
+require_relative "ggplot2-dataset"
+
+module Datasets
+  class FuelEconomy < Ggplot2Dataset
+    Record = Struct.new(:manufacturer,
+                        :model,
+                        :displacement,
+                        :year,
+                        :n_cylinders,
+                        :transmission,
+                        :drive_train,
+                        :city_mpg,
+                        :highway_mpg,
+                        :fuel,
+                        :type)
+
+    def initialize
+      super("mpg")
+      @metadata.id = "fuel-economy"
+      @metadata.name = "Fuel economy"
+      @metadata.licenses = ["CC0-1.0"]
+    end
+
+    COLUMN_NAME_MAPPING = {
+      "displ" => "displacement",
+      "cyl" => "n_cylinders",
+      "trans" => "transmissions",
+      "drv" => "drive_train",
+      "cty" => "city_mpg",
+      "hwy" => "highway_mpg",
+      "fl" => "fuel",
+      "class" => "type",
+    }
+  end
+end

data/lib/datasets/geolonia.rb

@@ -0,0 +1,67 @@
+require 'csv'
+
+require_relative 'dataset'
+
+module Datasets
+  class Geolonia < Dataset
+    Record = Struct.new(:prefecture_code,
+                        :prefecture_name,
+                        :prefecture_kana,
+                        :prefecture_romaji,
+                        :municipality_code,
+                        :municipality_name,
+                        :municipality_kana,
+                        :municipality_romaji,
+                        :street_name,
+                        :street_kana,
+                        :street_romaji,
+                        :alias,
+                        :latitude,
+                        :longitude)
+
+    def initialize
+      super
+      @metadata.id = 'geolonia'
+      @metadata.name = 'Geolonia'
+      @metadata.url = 'https://github.com/geolonia/japanese-addresses'
+      @metadata.licenses = ["CC-BY-4.0"]
+      @metadata.description = lambda do
+        fetch_readme
+      end
+    end
+
+    def each
+      return to_enum(__method__) unless block_given?
+
+      open_data do |csv|
+        csv.readline
+        csv.each do |row|
+          record = Record.new(*row)
+          yield(record)
+        end
+      end
+    end
+
+    private
+    def download_base_url
+      "https://raw.githubusercontent.com/geolonia/japanese-addresses/master"
+    end
+
+    def open_data
+      data_path = cache_dir_path + 'latest.csv'
+      data_url = "#{download_base_url}/data/latest.csv"
+      download(data_path, data_url)
+      CSV.open(data_path) do |csv|
+        yield(csv)
+      end
+    end
+
+    def fetch_readme
+      readme_base_name = "README.md"
+      readme_path = cache_dir_path + readme_base_name
+      readme_url = "#{download_base_url}/#{readme_base_name}"
+      download(readme_path, readme_url)
+      readme_path.read.split(/^## API/, 2)[0].strip
+    end
+  end
+end

data/lib/datasets/ggplot2-dataset.rb

@@ -0,0 +1,79 @@
+module Datasets
+  class Ggplot2Dataset < Dataset
+    def initialize(ggplot2_dataset_name)
+      super()
+      @ggplot2_dataset_name = ggplot2_dataset_name
+      @metadata.url =
+        "https://ggplot2.tidyverse.org/reference/#{@ggplot2_dataset_name}.html"
+      @metadata.description = lambda do
+        fetch_description
+      end
+    end
+
+    def each
+      return to_enum(__method__) unless block_given?
+
+      data_base_name = "#{@ggplot2_dataset_name}.csv"
+      data_path = cache_dir_path + data_base_name
+      data_url = "#{download_base_url}/data-raw/#{data_base_name}"
+      download(data_path, data_url)
+      CSV.open(data_path, headers: :first_row, converters: :all) do |csv|
+        record_class = self.class::Record
+        csv.each do |row|
+          record = record_class.new(*row.fields)
+          yield record
+        end
+      end
+    end
+
+    private
+    def download_base_url
+      "https://raw.githubusercontent.com/tidyverse/ggplot2/main"
+    end
+
+    def fetch_description
+      data_r_base_name = "data.R"
+      data_r_path = cache_dir_path + data_r_base_name
+      data_r_url = "#{download_base_url}/R/#{data_r_base_name}"
+      download(data_r_path, data_r_url)
+      descriptions = {}
+      comment = ""
+      File.open(data_r_path) do |data_r|
+        data_r.each_line do |line|
+          case line.chomp
+          when /\A#'/
+            comment_content = Regexp.last_match.post_match
+            unless comment_content.empty?
+              comment_content = comment_content[1..-1]
+            end
+            comment << comment_content
+            comment << "\n"
+          when /\A"(.+)"\z/
+            name = Regexp.last_match[1]
+            descriptions[name] = parse_roxygen(comment.rstrip)
+            comment = ""
+          end
+        end
+        descriptions[@ggplot2_dataset_name]
+      end
+    end
+
+    def parse_roxygen(roxygen)
+      column_name_mapping = self.class::COLUMN_NAME_MAPPING
+      roxygen
+        .gsub(/\\url\{(.*?)\}/, "\\1")
+        .gsub(/^@format /, "")
+        .gsub(/\\describe\{(.*)\}/m) do
+        content = $1
+        content.gsub(/\\item\{(.*?)\}\{(.*?)\}/m) do
+          column_name = $1
+          description = $2
+          column_name = column_name_mapping[column_name] || column_name
+          description = description
+                          .gsub(/\\\$/, "$")
+          "* #{column_name}: #{description}"
+        end
+      end
+    end
+  end
+end

data/lib/datasets/hepatitis.rb

@@ -163,6 +163,7 @@ module Datasets
       @metadata.id = "hepatitis"
       @metadata.name = "Hepatitis"
       @metadata.url = "https://archive.ics.uci.edu/ml/datasets/hepatitis"
+      @metadata.licenses = ["CC-BY-4.0"]
       @metadata.description = lambda do
         read_names
       end
@@ -186,10 +187,8 @@ module Datasets
 
     def open_data
       data_path = cache_dir_path + "hepatitis.csv"
-      unless data_path.exist?
-        data_url = "#{base_url}/hepatitis.data"
-        download(data_path, data_url)
-      end
+      data_url = "#{base_url}/hepatitis.data"
+      download(data_path, data_url)
       CSV.open(data_path) do |csv|
         yield(csv)
       end
@@ -197,10 +196,8 @@ module Datasets
 
     def read_names
       names_path = cache_dir_path + "hepatitis.names"
-      unless names_path.exist?
-        names_url = "#{base_url}/hepatitis.names"
-        download(names_path, names_url)
-      end
+      names_url = "#{base_url}/hepatitis.names"
+      download(names_path, names_url)
       names_path.read
     end
   end

data/lib/datasets/iris.rb

@@ -15,6 +15,7 @@ module Datasets
       @metadata.id = "iris"
       @metadata.name = "Iris"
       @metadata.url = "https://archive.ics.uci.edu/ml/datasets/Iris"
+      @metadata.licenses = ["CC-BY-4.0"]
       @metadata.description = lambda do
         read_names
       end
@@ -35,10 +36,8 @@ module Datasets
     private
     def open_data
       data_path = cache_dir_path + "iris.csv"
-      unless data_path.exist?
-        data_url = "https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data"
-        download(data_path, data_url)
-      end
+      data_url = "https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data"
+      download(data_path, data_url)
       CSV.open(data_path, converters: [:numeric]) do |csv|
         yield(csv)
       end
@@ -46,10 +45,8 @@ module Datasets
 
     def read_names
       names_path = cache_dir_path + "iris.names"
-      unless names_path.exist?
-        names_url = "https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.names"
-        download(names_path, names_url)
-      end
+      names_url = "https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.names"
+      download(names_path, names_url)
       names_path.read
     end
   end

data/lib/datasets/ita-corpus.rb

@@ -0,0 +1,57 @@
+require_relative 'dataset'
+
+module Datasets
+  class ITACorpus < Dataset
+    Record = Struct.new(:id,
+                        :sentence)
+
+    def initialize(type: :emotion)
+      unless [:emotion, :recitation].include?(type)
+        raise ArgumentError, "Please set type :emotion or :recitation: #{type.inspect}"
+      end
+
+      super()
+      @type = type
+      @metadata.id = 'ita-corpus'
+      @metadata.name = 'ITA-corpus'
+      @metadata.url = 'https://github.com/mmorise/ita-corpus'
+      @metadata.licenses = ['Unlicense']
+      @metadata.description = lambda do
+        fetch_readme
+      end
+    end
+
+    def each(&block)
+      return to_enum(__method__) unless block_given?
+
+      data_path = cache_dir_path + "#{@type}_transcript_utf8.txt"
+      data_url = "#{download_base_url}/#{@type}_transcript_utf8.txt"
+      download(data_path, data_url)
+
+      parse_data(data_path, &block)
+    end
+
+    private
+    def fetch_readme
+      readme_base_name = "README.md"
+      readme_path = cache_dir_path + readme_base_name
+      readme_url = "#{download_base_url}/#{readme_base_name}"
+      download(readme_path, readme_url)
+      readme_path.read.split(/^## ファイル構成/, 2)[0].strip
+    end
+
+    def download_base_url
+      "https://raw.githubusercontent.com/mmorise/ita-corpus/main"
+    end
+
+    def parse_data(data_path)
+      File.open(data_path) do |f|
+        f.each_line(chomp: true) do |line|
+          id, sentence = line.split(':', 2)
+          record = Record.new(id , sentence)
+          yield(record)
+        end
+      end
+    end
+  end
+end

data/lib/datasets/kuzushiji-mnist.rb

@@ -0,0 +1,16 @@
+require_relative 'mnist'
+
+module Datasets
+  class KuzushijiMNIST < MNIST
+    BASE_URL = "http://codh.rois.ac.jp/kmnist/dataset/kmnist/"
+
+    private
+    def dataset_name
+      "Kuzushiji-MNIST"
+    end
+
+    def licenses
+      ["CC-BY-SA-4.0"]
+    end
+  end
+end

data/lib/datasets/libsvm-dataset-list.rb

@@ -28,6 +28,7 @@ module Datasets
       @metadata.id = "libsvm-dataset-list"
       @metadata.name = "LIBSVM dataset list"
       @metadata.url = "https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/"
+      @metadata.licenses = ["BSD-3-Clause"]
       @metadata.description = lambda do
         extract_description
       end
@@ -51,10 +52,8 @@ module Datasets
     private
     def open_data
       data_path = cache_dir_path + "index.html"
-      unless data_path.exist?
-        download(data_path, @metadata.url)
-      end
-      ::File.open(data_path) do |input|
+      download(data_path, @metadata.url)
+      data_path.open do |input|
         yield(input)
       end
     end
@@ -78,10 +77,8 @@ module Datasets
 
     def open_detail(detail)
       data_path = cache_dir_path + detail
-      unless data_path.exist?
-        download(data_path, @metadata.url + detail)
-      end
-      ::File.open(data_path) do |input|
+      download(data_path, @metadata.url + detail)
+      data_path.open do |input|
         yield(input)
       end
     end

data/lib/datasets/libsvm.rb

@@ -41,6 +41,7 @@ module Datasets
       @metadata.id = "libsvm-#{normalize_name(name)}"
       @metadata.name = "LIBSVM dataset: #{name}"
       @metadata.url = "https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/"
+      @metadata.licenses = ["BSD-3-Clause"]
     end
 
     def each
@@ -99,13 +100,11 @@ module Datasets
 
     def open_data(&block)
       data_path = cache_dir_path + @file.name
-      unless data_path.exist?
-        download(data_path, @file.url)
-      end
+      download(data_path, @file.url)
       if data_path.extname == ".bz2"
         extract_bz2(data_path, &block)
       else
-        File.open(data_path, &block)
+        data_path.open(&block)
       end
     end
 

data/lib/datasets/license.rb

@@ -0,0 +1,26 @@
+module Datasets
+  class License < Struct.new(:spdx_id,
+                             :name,
+                             :url)
+    class << self
+      def try_convert(value)
+        case value
+        when self
+          value
+        when String
+          license = new
+          license.spdx_id = value
+          license
+        when Hash
+          license = new
+          license.spdx_id = value[:spdx_id]
+          license.name = value[:name]
+          license.url = value[:url]
+          license
+        else
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/datasets/livedoor-news.rb

@@ -0,0 +1,80 @@
+require_relative "dataset"
+require_relative "tar-gz-readable"
+
+module Datasets
+  class LivedoorNews < Dataset
+    include TarGzReadable
+    Record = Struct.new(:url,
+                        :timestamp,
+                        :sentence)
+
+    def initialize(type: :topic_news)
+      news_list = [
+        :topic_news,
+        :sports_watch,
+        :it_life_hack,
+        :kaden_channel,
+        :movie_enter,
+        :dokujo_tsushin,
+        :smax,
+        :livedoor_homme,
+        :peachy
+      ]
+      unless news_list.include?(type)
+        valid_type_labels = news_list.collect(&:inspect).join(", ")
+        message = ":type must be one of [#{valid_type_labels}]: #{type.inspect}"
+        raise ArgumentError, message
+      end
+
+      super()
+      @type = type
+      @metadata.id = 'livedoor-news'
+      @metadata.name = 'livedoor-news'
+      @metadata.url = 'https://www.rondhuit.com/download.html#ldcc'
+      @metadata.licenses = ['CC-BY-ND-2.1-JP']
+      @metadata.description = lambda do
+        fetch_readme
+      end
+    end
+
+    def each(&block)
+      return to_enum(__method__) unless block_given?
+
+      data_path = download_tar_gz
+      parse_data(data_path, &block)
+    end
+
+    private
+    def download_tar_gz
+      data_path = cache_dir_path + "livedoor-news.tar.gz"
+      data_url = "https://www.rondhuit.com/download/ldcc-20140209.tar.gz"
+      download(data_path, data_url)
+      data_path
+    end
+
+    def fetch_readme
+      data_path = download_tar_gz
+      target_file_name = 'text/README.txt'
+      open_tar_gz(data_path) do |tar|
+        tar.seek(target_file_name) do |entry|
+          return entry.read.force_encoding("UTF-8")
+        end
+      end
+    end
+
+    def parse_data(data_path, &block)
+      target_directory_name = "text/#{@type.to_s.gsub(/_/, '-')}"
+      open_tar_gz(data_path) do |tar|
+        tar.each do |entry|
+          next unless entry.file?
+          directory_name, base_name = File.split(entry.full_name)
+          next unless directory_name == target_directory_name
+          next if base_name == "LICENSE.txt"
+          url, timestamp, sentence = entry.read.force_encoding("UTF-8").split("\n", 3)
+          record = Record.new(url, Time.iso8601(timestamp), sentence)
+          yield(record)
+        end
+      end
+    end
+  end
+end

data/lib/datasets/metadata.rb

@@ -1,9 +1,23 @@
+require_relative "license"
+
 module Datasets
   class Metadata < Struct.new(:id,
                               :name,
                               :url,
                               :licenses,
                               :description)
+    def licenses=(licenses)
+      licenses = [licenses] unless licenses.is_a?(Array)
+      licenses = licenses.collect do |license|
+        l = License.try_convert(license)
+        if l.nil?
+          raise ArgumentError.new("invalid license: #{license.inspect}")
+        end
+        l
+      end
+      super(licenses)
+    end
+
     def description
       description_raw = super
       if description_raw.respond_to?(:call)

data/lib/datasets/mnist.rb

@@ -28,6 +28,7 @@ module Datasets
       @metadata.id = "#{dataset_name.downcase}-#{type}"
       @metadata.name = "#{dataset_name}: #{type}"
       @metadata.url = self.class::BASE_URL
+      @metadata.licenses = licenses
       @type = type
 
       case type
@@ -45,18 +46,17 @@ module Datasets
       label_path = cache_dir_path + target_file(:label)
       base_url = self.class::BASE_URL
 
-      unless image_path.exist?
-        download(image_path, base_url + target_file(:image))
-      end
-
-      unless label_path.exist?
-        download(label_path, base_url + target_file(:label))
-      end
+      download(image_path, base_url + target_file(:image))
+      download(label_path, base_url + target_file(:label))
 
       open_data(image_path, label_path, &block)
     end
 
     private
+    def licenses
+      []
+    end
+
     def open_data(image_path, label_path, &block)
       labels = parse_labels(label_path)
 

data/lib/datasets/mushroom.rb

@@ -35,6 +35,7 @@ module Datasets
       @metadata.id = "mushroom"
       @metadata.name = "Mushroom"
       @metadata.url = "https://archive.ics.uci.edu/ml/datasets/mushroom"
+      @metadata.licenses = ["CC-BY-4.0"]
       @metadata.description = lambda do
         read_names
       end
@@ -58,10 +59,8 @@ module Datasets
     private
     def open_data
       data_path = cache_dir_path + "agaricus-lepiota.data"
-      unless data_path.exist?
-        data_url = "http://archive.ics.uci.edu/ml/machine-learning-databases/mushroom/agaricus-lepiota.data"
-        download(data_path, data_url)
-      end
+      data_url = "http://archive.ics.uci.edu/ml/machine-learning-databases/mushroom/agaricus-lepiota.data"
+      download(data_path, data_url)
       CSV.open(data_path) do |csv|
         yield(csv)
       end
@@ -69,10 +68,8 @@ module Datasets
 
     def read_names
       names_path = cache_dir_path + "agaricus-lepiota.names"
-      unless names_path.exist?
-        names_url = "https://archive.ics.uci.edu/ml/machine-learning-databases//mushroom/agaricus-lepiota.names"
-        download(names_path, names_url)
-      end
+      names_url = "https://archive.ics.uci.edu/ml/machine-learning-databases//mushroom/agaricus-lepiota.names"
+      download(names_path, names_url)
       names_path.read
     end
 

data/lib/datasets/penguins.rb

@@ -23,10 +23,10 @@ module Datasets
       def initialize
         super
         species = self.class.name.split("::").last.downcase
-        @metadata.id = "palmerpenguins-raw-#{species}"
+        @metadata.id = "palmerpenguins-#{species}"
         @metadata.url = self.class::URL
-        @metadata.licenses = ["CC0"]
-        @data_path = cache_dir_path + "penguins" + (species + ".csv")
+        @metadata.licenses = ["CC0-1.0"]
+        @data_path = cache_dir_path + "#{species}.csv"
       end
 
       attr_reader :data_path
@@ -44,15 +44,11 @@ module Datasets
       end
 
       private def open_data
-        download unless data_path.exist?
+        download(data_path, metadata.url)
         CSV.open(data_path, headers: :first_row, converters: :all) do |csv|
           yield csv
         end
       end
-
-      private def download
-        super(data_path, metadata.url)
-      end
     end
 
     # Adelie penguin data from: https://doi.org/10.6073/pasta/abc50eed9138b75f54eaada0841b9b86

data/lib/datasets/penn-treebank.rb

@@ -36,10 +36,8 @@ module Datasets
 
       base_name = "ptb.#{@type}.txt"
       data_path = cache_dir_path + base_name
-      unless data_path.exist?
-        base_url = "https://raw.githubusercontent.com/wojzaremba/lstm/master/data"
-        download(data_path, "#{base_url}/#{base_name}")
-      end
+      base_url = "https://raw.githubusercontent.com/wojzaremba/lstm/master/data"
+      download(data_path, "#{base_url}/#{base_name}")
 
       parse_data(data_path, &block)
     end