red-datasets 0.1.4 → 0.1.6

data/lib/datasets/lazy.rb

@@ -0,0 +1,90 @@
+require_relative "version"
+
+module Datasets
+  class LazyLoader
+    def initialize
+      @constants = {}
+    end
+
+    def exist?(constant_name)
+      @constants.key?(constant_name)
+    end
+
+    def load(constant_name)
+      feature = @constants[constant_name]
+      raise LoadError, "unknown dataset: #{constant_name}" unless feature
+      require feature
+    end
+
+    def load_all
+      @constants.each_value do |feature|
+        require feature
+      end
+    end
+
+    def register(constant_name, feature)
+      @constants[constant_name] = feature
+    end
+
+    def constant_names
+      @constants.keys
+    end
+  end
+
+  LAZY_LOADER = LazyLoader.new
+
+  class << self
+    def const_missing(name)
+      if LAZY_LOADER.exist?(name)
+        LAZY_LOADER.load(name)
+        const_get(name)
+      else
+        super
+      end
+    end
+  end
+
+  LAZY_LOADER.register(:Adult, "datasets/adult")
+  LAZY_LOADER.register(:AFINN, "datasets/afinn")
+  LAZY_LOADER.register(:AozoraBunko, "datasets/aozora-bunko")
+  LAZY_LOADER.register(:CaliforniaHousing, "datasets/california-housing")
+  LAZY_LOADER.register(:CIFAR, "datasets/cifar")
+  LAZY_LOADER.register(:CLDRPlurals, "datasets/cldr-plurals")
+  LAZY_LOADER.register(:Communities, "datasets/communities")
+  LAZY_LOADER.register(:Diamonds, "datasets/diamonds")
+  LAZY_LOADER.register(:EStatJapan, "datasets/e-stat-japan")
+  LAZY_LOADER.register(:FashionMNIST, "datasets/fashion-mnist")
+  LAZY_LOADER.register(:FuelEconomy, "datasets/fuel-economy")
+  LAZY_LOADER.register(:Geolonia, "datasets/geolonia")
+  LAZY_LOADER.register(:Hepatitis, "datasets/hepatitis")
+  LAZY_LOADER.register(:Iris, "datasets/iris")
+  LAZY_LOADER.register(:ITACorpus, "datasets/ita-corpus")
+  LAZY_LOADER.register(:KuzushijiMNIST, "datasets/kuzushiji-mnist")
+  LAZY_LOADER.register(:LIBSVM, "datasets/libsvm")
+  LAZY_LOADER.register(:LIBSVMDatasetList, "datasets/libsvm-dataset-list")
+  LAZY_LOADER.register(:LivedoorNews, "datasets/livedoor-news")
+  LAZY_LOADER.register(:MNIST, "datasets/mnist")
+  LAZY_LOADER.register(:Mushroom, "datasets/mushroom")
+  LAZY_LOADER.register(:NagoyaUniversityConversationCorpus,
+                       "datasets/nagoya-university-conversation-corpus")
+  LAZY_LOADER.register(:Penguins, "datasets/penguins")
+  LAZY_LOADER.register(:PennTreebank, "datasets/penn-treebank")
+  LAZY_LOADER.register(:PMJTDatasetList, "datasets/pmjt-dataset-list")
+  LAZY_LOADER.register(:PostalCodeJapan, "datasets/postal-code-japan")
+  LAZY_LOADER.register(:QuoraDuplicateQuestionPair,
+                       "datasets/quora-duplicate-question-pair")
+  LAZY_LOADER.register(:RdatasetList, "datasets/rdataset")
+  # For backward compatibility
+  LAZY_LOADER.register(:RdatasetsList, "datasets/rdataset")
+  LAZY_LOADER.register(:Rdataset, "datasets/rdataset")
+  # For backward compatibility
+  LAZY_LOADER.register(:Rdatasets, "datasets/rdataset")
+  LAZY_LOADER.register(:SeabornList, "datasets/seaborn")
+  LAZY_LOADER.register(:Seaborn, "datasets/seaborn")
+  LAZY_LOADER.register(:SudachiSynonymDictionary,
+                       "datasets/sudachi-synonym-dictionary")
+  LAZY_LOADER.register(:Wikipedia, "datasets/wikipedia")
+  LAZY_LOADER.register(:WikipediaKyotoJapaneseEnglish,
+                       "datasets/wikipedia-kyoto-japanese-english")
+  LAZY_LOADER.register(:Wine, "datasets/wine")
+end

data/lib/datasets/libsvm-dataset-list.rb

@@ -28,6 +28,7 @@ module Datasets
       @metadata.id = "libsvm-dataset-list"
       @metadata.name = "LIBSVM dataset list"
       @metadata.url = "https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/"
+      @metadata.licenses = ["BSD-3-Clause"]
       @metadata.description = lambda do
         extract_description
       end
@@ -51,10 +52,8 @@ module Datasets
     private
     def open_data
       data_path = cache_dir_path + "index.html"
-      unless data_path.exist?
-        download(data_path, @metadata.url)
-      end
-      ::File.open(data_path) do |input|
+      download(data_path, @metadata.url)
+      data_path.open do |input|
         yield(input)
       end
     end
@@ -78,10 +77,8 @@ module Datasets
 
     def open_detail(detail)
       data_path = cache_dir_path + detail
-      unless data_path.exist?
-        download(data_path, @metadata.url + detail)
-      end
-      ::File.open(data_path) do |input|
+      download(data_path, @metadata.url + detail)
+      data_path.open do |input|
         yield(input)
       end
     end

data/lib/datasets/libsvm.rb

@@ -41,6 +41,7 @@ module Datasets
       @metadata.id = "libsvm-#{normalize_name(name)}"
       @metadata.name = "LIBSVM dataset: #{name}"
       @metadata.url = "https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/"
+      @metadata.licenses = ["BSD-3-Clause"]
     end
 
     def each
@@ -99,13 +100,11 @@ module Datasets
 
     def open_data(&block)
       data_path = cache_dir_path + @file.name
-      unless data_path.exist?
-        download(data_path, @file.url)
-      end
+      download(data_path, @file.url)
       if data_path.extname == ".bz2"
         extract_bz2(data_path, &block)
       else
-        File.open(data_path, &block)
+        data_path.open(&block)
       end
     end
 

data/lib/datasets/license.rb

@@ -0,0 +1,26 @@
+module Datasets
+  class License < Struct.new(:spdx_id,
+                             :name,
+                             :url)
+    class << self
+      def try_convert(value)
+        case value
+        when self
+          value
+        when String
+          license = new
+          license.spdx_id = value
+          license
+        when Hash
+          license = new
+          license.spdx_id = value[:spdx_id]
+          license.name = value[:name]
+          license.url = value[:url]
+          license
+        else
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/datasets/livedoor-news.rb

@@ -0,0 +1,80 @@
+require_relative "dataset"
+require_relative "tar-gz-readable"
+
+module Datasets
+  class LivedoorNews < Dataset
+    include TarGzReadable
+    Record = Struct.new(:url,
+                        :timestamp,
+                        :sentence)
+
+    def initialize(type: :topic_news)
+      news_list = [
+        :topic_news,
+        :sports_watch,
+        :it_life_hack,
+        :kaden_channel,
+        :movie_enter,
+        :dokujo_tsushin,
+        :smax,
+        :livedoor_homme,
+        :peachy
+      ]
+      unless news_list.include?(type)
+        valid_type_labels = news_list.collect(&:inspect).join(", ")
+        message = ":type must be one of [#{valid_type_labels}]: #{type.inspect}"
+        raise ArgumentError, message
+      end
+
+      super()
+      @type = type
+      @metadata.id = 'livedoor-news'
+      @metadata.name = 'livedoor-news'
+      @metadata.url = 'https://www.rondhuit.com/download.html#ldcc'
+      @metadata.licenses = ['CC-BY-ND-2.1-JP']
+      @metadata.description = lambda do
+        fetch_readme
+      end
+    end
+
+    def each(&block)
+      return to_enum(__method__) unless block_given?
+
+      data_path = download_tar_gz
+      parse_data(data_path, &block)
+    end
+
+    private
+    def download_tar_gz
+      data_path = cache_dir_path + "livedoor-news.tar.gz"
+      data_url = "https://www.rondhuit.com/download/ldcc-20140209.tar.gz"
+      download(data_path, data_url)
+      data_path
+    end
+
+    def fetch_readme
+      data_path = download_tar_gz
+      target_file_name = 'text/README.txt'
+      open_tar_gz(data_path) do |tar|
+        tar.seek(target_file_name) do |entry|
+          return entry.read.force_encoding("UTF-8")
+        end
+      end
+    end
+
+    def parse_data(data_path, &block)
+      target_directory_name = "text/#{@type.to_s.gsub(/_/, '-')}"
+      open_tar_gz(data_path) do |tar|
+        tar.each do |entry|
+          next unless entry.file?
+          directory_name, base_name = File.split(entry.full_name)
+          next unless directory_name == target_directory_name
+          next if base_name == "LICENSE.txt"
+          url, timestamp, sentence = entry.read.force_encoding("UTF-8").split("\n", 3)
+          record = Record.new(url, Time.iso8601(timestamp), sentence)
+          yield(record)
+        end
+      end
+    end
+  end
+end

data/lib/datasets/metadata.rb

@@ -1,9 +1,23 @@
+require_relative "license"
+
 module Datasets
   class Metadata < Struct.new(:id,
                               :name,
                               :url,
                               :licenses,
                               :description)
+    def licenses=(licenses)
+      licenses = [licenses] unless licenses.is_a?(Array)
+      licenses = licenses.collect do |license|
+        l = License.try_convert(license)
+        if l.nil?
+          raise ArgumentError.new("invalid license: #{license.inspect}")
+        end
+        l
+      end
+      super(licenses)
+    end
+
     def description
       description_raw = super
       if description_raw.respond_to?(:call)

data/lib/datasets/mnist.rb

@@ -28,6 +28,7 @@ module Datasets
       @metadata.id = "#{dataset_name.downcase}-#{type}"
       @metadata.name = "#{dataset_name}: #{type}"
       @metadata.url = self.class::BASE_URL
+      @metadata.licenses = licenses
       @type = type
 
       case type
@@ -45,18 +46,17 @@ module Datasets
       label_path = cache_dir_path + target_file(:label)
       base_url = self.class::BASE_URL
 
-      unless image_path.exist?
-        download(image_path, base_url + target_file(:image))
-      end
-
-      unless label_path.exist?
-        download(label_path, base_url + target_file(:label))
-      end
+      download(image_path, base_url + target_file(:image))
+      download(label_path, base_url + target_file(:label))
 
       open_data(image_path, label_path, &block)
     end
 
     private
+    def licenses
+      []
+    end
+
     def open_data(image_path, label_path, &block)
       labels = parse_labels(label_path)
 

data/lib/datasets/mushroom.rb

@@ -35,6 +35,7 @@ module Datasets
       @metadata.id = "mushroom"
       @metadata.name = "Mushroom"
       @metadata.url = "https://archive.ics.uci.edu/ml/datasets/mushroom"
+      @metadata.licenses = ["CC-BY-4.0"]
       @metadata.description = lambda do
         read_names
       end
@@ -58,10 +59,8 @@ module Datasets
     private
     def open_data
       data_path = cache_dir_path + "agaricus-lepiota.data"
-      unless data_path.exist?
-        data_url = "http://archive.ics.uci.edu/ml/machine-learning-databases/mushroom/agaricus-lepiota.data"
-        download(data_path, data_url)
-      end
+      data_url = "http://archive.ics.uci.edu/ml/machine-learning-databases/mushroom/agaricus-lepiota.data"
+      download(data_path, data_url)
       CSV.open(data_path) do |csv|
         yield(csv)
       end
@@ -69,10 +68,8 @@ module Datasets
 
     def read_names
       names_path = cache_dir_path + "agaricus-lepiota.names"
-      unless names_path.exist?
-        names_url = "https://archive.ics.uci.edu/ml/machine-learning-databases//mushroom/agaricus-lepiota.names"
-        download(names_path, names_url)
-      end
+      names_url = "https://archive.ics.uci.edu/ml/machine-learning-databases//mushroom/agaricus-lepiota.names"
+      download(names_path, names_url)
       names_path.read
     end
 

data/lib/datasets/nagoya-university-conversation-corpus.rb

@@ -0,0 +1,109 @@
+require_relative 'dataset'
+require_relative 'zip-extractor'
+
+module Datasets
+  class NagoyaUniversityConversationCorpus < Dataset
+    Data = Struct.new(
+      :name,
+      :date,
+      :place,
+      :participants,
+      :relationships,
+      :note,
+      :sentences
+    )
+
+    Participant = Struct.new(
+      :id,
+      :attribute,
+      :birthplace,
+      :residence
+    )
+
+    Sentence = Struct.new(:participant_id, :content) do
+      def end?
+        participant_id.nil? and content.nil?
+      end
+    end
+
+    def initialize
+      super()
+      @metadata.id = 'nagoya-university-conversation-curpus'
+      @metadata.name = 'Nagoya University Conversation Curpus'
+      @metadata.url = 'https://mmsrv.ninjal.ac.jp/nucc/'
+      @metadata.licenses = ['CC-BY-NC-ND-4.0']
+      @metadata.description = <<~DESCRIPTION
+        The "Nagoya University Conversation Corpus" is a corpus of 129 conversations,
+        total about 100 hours of chatting among native speakers of Japanese,
+        which is converted into text.
+      DESCRIPTION
+    end
+
+    def each
+      return to_enum(__method__) unless block_given?
+
+      open_data do |input_stream|
+        yield(parse_file(input_stream))
+      end
+    end
+
+    private
+
+    def open_data
+      data_path = cache_dir_path + 'nucc.zip'
+      data_url = 'https://mmsrv.ninjal.ac.jp/nucc/nucc.zip'
+      download(data_path, data_url)
+
+      extractor = ZipExtractor.new(data_path)
+      extractor.extract_files do |input_stream|
+        yield(input_stream)
+      end
+    end
+
+    def parse_file(input_stream)
+      data = Data.new
+      participants = []
+      sentences = []
+
+      input_stream.each do |input|
+        input.each_line(chomp: true) do |line|
+          line.force_encoding('utf-8')
+          if line.start_with?('＠データ')
+            data.name = line[4..]
+          elsif line.start_with?('＠収集年月日')
+            # mixed cases with and without'：'
+            data.date = line[6..].delete_prefix('：')
+          elsif line.start_with?('＠場所')
+            data.place = line[4..]
+          elsif line.start_with?('＠参加者の関係')
+            data.relationships = line.split('：', 2)[1]
+          elsif line.start_with?('＠参加者')
+            participant = Participant.new
+            participant.id, profiles = line[4..].split('：', 2)
+            participant.attribute, participant.birthplace, participant.residence = profiles.split('、', 3)
+
+            participants << participant
+          elsif line.start_with?('％ｃｏｍ')
+            data.note = line.split('：', 2)[1]
+          elsif line == '＠ＥＮＤ'
+            sentence = Sentence.new
+            sentence.participant_id = nil
+            sentence.content = nil
+
+            sentences << sentence
+          else
+            sentence = Sentence.new
+            sentence.participant_id, sentence.content = line.split('：', 2)
+
+            sentences << sentence
+          end
+        end
+      end
+
+      data.participants = participants
+      data.sentences = sentences
+
+      data
+    end
+  end
+end

data/lib/datasets/penguins.rb

@@ -1,3 +1,5 @@
+require "csv"
+
 require_relative "dataset"
 
 module Datasets
@@ -23,10 +25,10 @@ module Datasets
       def initialize
         super
         species = self.class.name.split("::").last.downcase
-        @metadata.id = "palmerpenguins-raw-#{species}"
+        @metadata.id = "palmerpenguins-#{species}"
         @metadata.url = self.class::URL
-        @metadata.licenses = ["CC0"]
-        @data_path = cache_dir_path + "penguins" + (species + ".csv")
+        @metadata.licenses = ["CC0-1.0"]
+        @data_path = cache_dir_path + "#{species}.csv"
       end
 
       attr_reader :data_path
@@ -44,15 +46,11 @@ module Datasets
       end
 
       private def open_data
-        download unless data_path.exist?
+        download(data_path, metadata.url)
         CSV.open(data_path, headers: :first_row, converters: :all) do |csv|
           yield csv
         end
       end
-
-      private def download
-        super(data_path, metadata.url)
-      end
     end
 
     # Adelie penguin data from: https://doi.org/10.6073/pasta/abc50eed9138b75f54eaada0841b9b86

data/lib/datasets/penn-treebank.rb

@@ -36,10 +36,8 @@ module Datasets
 
       base_name = "ptb.#{@type}.txt"
       data_path = cache_dir_path + base_name
-      unless data_path.exist?
-        base_url = "https://raw.githubusercontent.com/wojzaremba/lstm/master/data"
-        download(data_path, "#{base_url}/#{base_name}")
-      end
+      base_url = "https://raw.githubusercontent.com/wojzaremba/lstm/master/data"
+      download(data_path, "#{base_url}/#{base_name}")
 
       parse_data(data_path, &block)
     end

data/lib/datasets/pmjt-dataset-list.rb

@@ -0,0 +1,67 @@
+require_relative "dataset"
+
+module Datasets
+  class PMJTDatasetList < Dataset
+    Record = Struct.new(:unit,
+                        :open_data_category,
+                        :tag,
+                        :release_time,
+                        :n_volumes,
+                        :type,
+                        :publication_year,
+                        :original_request_code,
+                        :id,
+                        :title,
+                        :text,
+                        :bibliographical_introduction,
+                        :year)
+
+    def initialize
+      super()
+      @metadata.id = "pmjt-dataset-list"
+      @metadata.name = "List of pre-modern Japanese text dataset"
+      @metadata.url = "http://codh.rois.ac.jp/pmjt/"
+      @metadata.licenses = ["CC-BY-SA-4.0"]
+      @metadata.description = <<~DESCRIPTION
+        Pre-Modern Japanese Text, owned by National Institute of Japanese Literature, is released image and text data as open data.
+        In addition, some text has description, transcription, and tagging data.
+      DESCRIPTION
+
+      @data_path = cache_dir_path + (@metadata.id + ".csv")
+    end
+
+    def each(&block)
+      return to_enum(__method__) unless block_given?
+
+      latest_version = "201901"
+      url = "http://codh.rois.ac.jp/pmjt/list/pmjt-dataset-list-#{latest_version}.csv"
+      download(@data_path, url)
+      CSV.open(@data_path, headers: :first_row, encoding: "Windows-31J:UTF-8") do |csv|
+        csv.each do |row|
+          record = create_record(row)
+          yield record
+        end
+      end
+    end
+
+    private
+    def create_record(csv_row)
+      record = Record.new
+      record.unit = csv_row["(単位)"]
+      record.open_data_category = csv_row["オープンデータ分類"]
+      record.tag = csv_row["タグ"]
+      record.release_time = csv_row["公開時期"]
+      record.n_volumes = csv_row["冊数等"]
+      record.type = csv_row["刊・写"]
+      record.publication_year = csv_row["刊年・書写年"]
+      record.original_request_code = csv_row["原本請求記号"]
+      record.id = csv_row["国文研書誌ID"]
+      record.title = csv_row["書名（統一書名）"]
+      record.text = csv_row["本文"]
+      record.bibliographical_introduction = csv_row["解題"]
+      record.year = csv_row["（西暦）"]
+
+      record
+    end
+  end
+end

data/lib/datasets/postal-code-japan.rb

@@ -49,9 +49,7 @@ module Datasets
       @metadata.id = "postal-code-japan-#{@reading}"
       @metadata.name = "Postal code in Japan (#{@reading})"
       @metadata.url = "https://www.post.japanpost.jp/zipcode/download.html"
-      @metadata.licenses = [
-        "CC0-1.0",
-      ]
+      @metadata.licenses = ["CC0-1.0"]
       @metadata.description = "Postal code in Japan (reading: #{@reading})"
     end
 
@@ -116,9 +114,7 @@ module Datasets
         data_url << "/roman/ken_all_rome.zip"
       end
       data_path = cache_dir_path + "#{@reading}-ken-all.zip"
-      unless data_path.exist?
-        download(data_path, data_url)
-      end
+      download(data_path, data_url)
 
       Zip::File.open(data_path.to_s) do |zip_file|
         zip_file.each do |entry|

data/lib/datasets/quora-duplicate-question-pair.rb

@@ -0,0 +1,51 @@
+require "csv"
+
+require_relative "dataset"
+
+module Datasets
+  class QuoraDuplicateQuestionPair < Dataset
+    class Record < Struct.new(:id,
+                              :first_question_id,
+                              :second_question_id,
+                              :first_question,
+                              :second_question,
+                              :duplicated)
+      alias_method :duplicated?, :duplicated
+    end
+
+    def initialize
+      super()
+      @metadata.id = "quora-duplicate-question-pair"
+      @metadata.name = "Quora's duplicated question pair dataset"
+      @metadata.url = "https://quoradata.quora.com/First-Quora-Dataset-Release-Question-Pairs"
+      @metadata.licenses = [
+        {
+          name: "Quora's Terms of Service",
+          url: "https://www.quora.com/about/tos",
+        }
+      ]
+    end
+
+    def each
+      return to_enum(__method__) unless block_given?
+
+      open_data do |csv|
+        csv.each do |row|
+          row["is_duplicate"] = (row["is_duplicate"] == 1)
+          record = Record.new(*row.fields)
+          yield(record)
+        end
+      end
+    end
+
+    private
+    def open_data
+      data_path = cache_dir_path + "quora_duplicate_questions.tsv"
+      data_url = "https://qim.fs.quoracdn.net/quora_duplicate_questions.tsv"
+      download(data_path, data_url)
+      CSV.open(data_path, col_sep: "\t", headers: true, converters: :all) do |csv|
+        yield(csv)
+      end
+    end
+  end
+end