red-datasets 0.1.4 → 0.1.6

data/lib/datasets/{rdatasets.rb → rdataset.rb}

@@ -2,7 +2,7 @@ require_relative "dataset"
 require_relative "tar-gz-readable"
 
 module Datasets
-  class RdatasetsList < Dataset
+  class RdatasetList < Dataset
     Record = Struct.new(:package,
                         :dataset,
                         :title,
@@ -18,8 +18,8 @@ module Datasets
 
     def initialize
       super
-      @metadata.id = "rdatasets"
-      @metadata.name = "Rdatasets"
+      @metadata.id = "rdataset-list"
+      @metadata.name = "Rdataset"
       @metadata.url = "https://vincentarelbundock.github.io/Rdatasets/"
       @metadata.licenses = ["GPL-3"]
       @data_url = "https://raw.githubusercontent.com/vincentarelbundock/Rdatasets/master/datasets.csv"
@@ -48,16 +48,19 @@ module Datasets
     end
 
     private def each_row(&block)
-      download(@data_path, @data_url) unless @data_path.exist?
+      download(@data_path, @data_url)
       CSV.open(@data_path, headers: :first_row, converters: :all) do |csv|
         csv.each(&block)
       end
     end
   end
 
-  class Rdatasets < Dataset
+  # For backward compatibility
+  RdatasetsList = RdatasetList
+
+  class Rdataset < Dataset
     def initialize(package_name, dataset_name)
-      list = RdatasetsList.new
+      list = RdatasetList.new
 
       info = list.filter(package: package_name, dataset: dataset_name).first
       unless info
@@ -65,8 +68,8 @@ module Datasets
       end
 
       super()
-      @metadata.id = "rdatasets-#{package_name}-#{dataset_name}"
-      @metadata.name = "Rdatasets: #{package_name}: #{dataset_name}"
+      @metadata.id = "rdataset-#{package_name}-#{dataset_name}"
+      @metadata.name = "Rdataset: #{package_name}: #{dataset_name}"
       @metadata.url = info.csv
       @metadata.licenses = ["GPL-3"]
       @metadata.description = info.title
@@ -81,15 +84,63 @@ module Datasets
     def each(&block)
       return to_enum(__method__) unless block_given?
 
-      download(@data_path, @metadata.url) unless @data_path.exist?
-      CSV.open(@data_path, headers: :first_row, converters: :all) do |csv|
-        csv.each do |row|
-          record = row.to_h
-          record.delete("")
-          record.transform_keys!(&:to_sym)
-          yield record
+      download(@data_path, @metadata.url)
+
+      na_converter = lambda do |field|
+        begin
+          if field.encode(CSV::ConverterEncoding) == "NA"
+            nil
+          else
+            field
+          end
+        rescue
+          field
         end
       end
+
+      inf_converter = lambda do |field|
+        begin
+          if field.encode(CSV::ConverterEncoding) == "Inf"
+            Float::INFINITY
+          else
+            field
+          end
+        rescue
+          field
+        end
+      end
+
+      quote_preserving_converter = lambda do |field, info|
+        f = field.encode(CSV::ConverterEncoding)
+        return f if info.quoted?
+
+        begin
+          begin
+            begin
+              return DateTime.parse(f) if f.match?(DateTimeMatcher)
+            rescue
+              return Integer(f)
+            end
+          rescue
+            return Float(f)
+          end
+        rescue
+          field
+        end
+      end
+
+      table = CSV.table(@data_path,
+                        header_converters: [:symbol_raw],
+                        # quote_preserving_converter should be the last
+                        converters: [na_converter, inf_converter, quote_preserving_converter])
+      table.delete(:"") # delete 1st column for indices.
+
+      table.each do |row|
+        yield row.to_h
+      end
     end
   end
+
+  # For backward compatibility
+  Rdatasets = Rdataset
 end

data/lib/datasets/seaborn.rb

@@ -0,0 +1,90 @@
+require "json"
+
+module Datasets
+  class SeabornList < Dataset
+    def initialize
+      super
+      @metadata.id = "seaborn-data-list"
+      @metadata.name = "seaborn: data list"
+      @metadata.url = "https://github.com/mwaskom/seaborn-data"
+      # Treat as the same license as seaborn
+      @metadata.licenses = ["BSD-3-Clause"]
+      @metadata.description = "Datasets for seaborn examples."
+    end
+
+    def each(&block)
+      return to_enum(__method__) unless block_given?
+
+      data_path = cache_dir_path + "trees.json"
+      url = "https://api.github.com/repos/mwaskom/seaborn-data/git/trees/master"
+      download(data_path, url)
+
+      tree = JSON.parse(File.read(data_path))["tree"]
+      tree.each do |content|
+        path = content["path"]
+        next unless path.end_with?(".csv")
+        dataset = File.basename(path, ".csv")
+        record = {dataset: dataset}
+        yield record
+      end
+    end
+  end
+
+  class Seaborn < Dataset
+    URL_FORMAT = "https://raw.githubusercontent.com/mwaskom/seaborn-data/master/%{name}.csv".freeze
+
+    def initialize(name)
+      super()
+      @metadata.id = "seaborn-#{name}"
+      @metadata.name = "seaborn: #{name}"
+      @metadata.url = URL_FORMAT % {name: name}
+      # @metadata.licenses = TODO
+
+      @name = name
+    end
+
+    def each(&block)
+      return to_enum(__method__) unless block_given?
+
+      data_path = cache_dir_path + "#{@name}.csv"
+      download(data_path, @metadata.url)
+      CSV.open(data_path, headers: :first_row, converters: :all) do |csv|
+        csv.each do |row|
+          record = prepare_record(row)
+          yield record
+        end
+      end
+    end
+
+    private
+    def prepare_record(csv_row)
+      record = csv_row.to_h
+      record.transform_keys! do |key|
+        if key.nil?
+          :index
+        else
+          key.to_sym
+        end
+      end
+
+      # Perform the same preprocessing as seaborn's load_dataset function
+      preprocessor = :"preprocess_#{@name}_record"
+      __send__(preprocessor, record) if respond_to?(preprocessor, true)
+
+      record
+    end
+
+    # The same preprocessing as seaborn.load_dataset
+    def preprocess_flights_record(record)
+      record[:month] &&= record[:month][0,3]
+    end
+
+    # The same preprocessing as seaborn.load_dataset
+    def preprocess_penguins_record(record)
+      record[:sex] &&= record[:sex].capitalize
+    end
+  end
+
+  # For backward compatibility
+  SeabornData = Seaborn
+end

data/lib/datasets/sudachi-synonym-dictionary.rb

@@ -21,9 +21,7 @@ module Datasets
       @metadata.id = "sudachi-synonym-dictionary"
       @metadata.name = "Sudachi synonym dictionary"
       @metadata.url = "https://github.com/WorksApplications/SudachiDict/blob/develop/docs/synonyms.md"
-      @metadata.licenses = [
-        "Apache-2.0",
-      ]
+      @metadata.licenses = ["Apache-2.0"]
       @metadata.description = lambda do
         download_description
       end
@@ -65,10 +63,8 @@ module Datasets
     private
     def open_data
       data_path = cache_dir_path + "synonyms.txt"
-      unless data_path.exist?
-        data_url = "https://raw.githubusercontent.com/WorksApplications/SudachiDict/develop/src/main/text/synonyms.txt"
-        download(data_path, data_url)
-      end
+      data_url = "https://raw.githubusercontent.com/WorksApplications/SudachiDict/develop/src/main/text/synonyms.txt"
+      download(data_path, data_url)
       CSV.open(data_path,
                encoding: "UTF-8",
                skip_blanks: true) do |csv|
@@ -78,10 +74,8 @@ module Datasets
 
     def download_description
       description_path = cache_dir_path + "synonyms.md"
-      unless description_path.exist?
-        description_url = "https://raw.githubusercontent.com/WorksApplications/SudachiDict/develop/docs/synonyms.md"
-        download(description_path, description_url)
-      end
+      description_url = "https://raw.githubusercontent.com/WorksApplications/SudachiDict/develop/docs/synonyms.md"
+      download(description_path, description_url)
       description_path.read
     end
 

data/lib/datasets/version.rb

@@ -1,3 +1,3 @@
 module Datasets
-  VERSION = "0.1.4"
+  VERSION = "0.1.6"
 end

data/lib/datasets/wikipedia-kyoto-japanese-english.rb

@@ -0,0 +1,219 @@
+require "csv"
+require "rexml/streamlistener"
+require "rexml/parsers/baseparser"
+require "rexml/parsers/streamparser"
+require "time"
+
+require_relative "dataset"
+require_relative "tar-gz-readable"
+
+module Datasets
+  class WikipediaKyotoJapaneseEnglish < Dataset
+    include TarGzReadable
+
+    Article = Struct.new(:source,
+                         :copyright,
+                         :contents,
+                         :sections)
+
+    Section = Struct.new(:id,
+                         :title,
+                         :contents)
+
+    class Title < Struct.new(:section,
+                             :japanese,
+                             :english)
+      def title?
+        true
+      end
+
+      def sentence?
+        false
+      end
+    end
+
+    Paragraph = Struct.new(:id,
+                           :sentences)
+
+    class Sentence < Struct.new(:id,
+                                :section,
+                                :paragraph,
+                                :japanese,
+                                :english)
+      def title?
+        false
+      end
+
+      def sentence?
+        true
+      end
+    end
+
+    Entry = Struct.new(:japanese,
+                       :english)
+
+    def initialize(type: :article)
+      unless [:article, :lexicon].include?(type)
+        raise ArgumentError, "Please set type :article or :lexicon: #{type.inspect}"
+      end
+
+      super()
+      @type = type
+      @metadata.id = "wikipedia-kyoto-japanese-english"
+      @metadata.name =
+        "The Japanese-English Bilingual Corpus of Wikipedia's Kyoto Articles"
+      @metadata.url = "https://alaginrc.nict.go.jp/WikiCorpus/index_E.html"
+      @metadata.licenses = ["CC-BY-SA-3.0"]
+      @metadata.description = <<-DESCRIPTION
+"The Japanese-English Bilingual Corpus of Wikipedia's Kyoto Articles"
+aims mainly at supporting research and development relevant to
+high-performance multilingual machine translation, information
+extraction, and other language processing technologies. The National
+Institute of Information and Communications Technology (NICT) has
+created this corpus by manually translating Japanese Wikipedia
+articles (related to Kyoto) into English.
+      DESCRIPTION
+    end
+
+    def each(&block)
+      return to_enum(__method__) unless block_given?
+
+      data_path = download_tar_gz
+
+      open_tar_gz(data_path) do |tar|
+        tar.each do |entry|
+          next unless entry.file?
+          base_name = File.basename(entry.full_name)
+          case @type
+          when :article
+            next unless base_name.end_with?(".xml")
+            listener = ArticleListener.new(block)
+            parser = REXML::Parsers::StreamParser.new(entry.read, listener)
+            parser.parse
+          when :lexicon
+            next unless base_name == "kyoto_lexicon.csv"
+            is_header = true
+            CSV.parse(entry.read.force_encoding("UTF-8")) do |row|
+              if is_header
+                is_header = false
+                next
+              end
+              yield(Entry.new(row[0], row[1]))
+            end
+          end
+        end
+      end
+    end
+
+    private
+    def download_tar_gz
+      base_name = "wiki_corpus_2.01.tar.gz"
+      data_path = cache_dir_path + base_name
+      data_url = "https://alaginrc.nict.go.jp/WikiCorpus/src/#{base_name}"
+      download(data_path, data_url)
+      data_path
+    end
+
+    class ArticleListener
+      include REXML::StreamListener
+
+      def initialize(block)
+        @block = block
+        @article = nil
+        @title = nil
+        @section = nil
+        @page = nil
+        @sentence = nil
+        @text_container_stack = []
+        @element_stack = []
+        @text_stack = [""]
+      end
+
+      def tag_start(name, attributes)
+        push_stacks(name, attributes)
+        case name
+        when "art"
+          @article = Article.new
+          @article.contents = []
+          @article.sections = []
+        when "tit"
+          @title = Title.new
+          @title.section = @section
+          @text_container_stack.push(@title)
+        when "sec"
+          @section = Section.new
+          @section.id = attributes["id"]
+          @section.contents = []
+          @text_container_stack.push(@section)
+        when "par"
+          @paragraph = Paragraph.new
+          @paragraph.id = attributes["id"]
+          @paragraph.sentences = []
+          @text_container_stack.push(@paragraph)
+        when "sen"
+          @sentence = Sentence.new
+          @sentence.id = attributes["id"]
+          @text_container_stack.push(@sentence)
+        end
+      end
+
+      def tag_end(name)
+        case name
+        when "art"
+          @block.call(@article)
+          @article = nil
+        when "inf"
+          @article.source = @text_stack.last
+        when "copyright"
+          @article.copyright = @text_stack.last
+        when "tit"
+          @article.contents << @title
+          if @section
+            @section.title = @title
+            @section.contents << @title
+          end
+          @title = nil
+          @text_container_stack.pop
+        when "sec"
+          @article.sections << @section
+          @section = nil
+          @text_container_stack.pop
+        when "par"
+          @paragraph = nil
+          @text_container_stack.pop
+        when "sen"
+          @article.contents << @sentence
+          @sentence.section = @section
+          @section.contents << @sentence if @section
+          @sentence.paragraph = @paragraph
+          @paragraph.sentences << @sentence if @paragraph
+          @sentence = nil
+          @text_container_stack.pop
+        when "j"
+          @text_container_stack.last.japanese = @text_stack.last
+        when "e"
+          attributes = @element_stack.last[:attributes]
+          if attributes["type"] == "check"
+            @text_container_stack.last.english = @text_stack.last
+          end
+        end
+        pop_stacks
+      end
+
+      def text(data)
+        @text_stack.last << data
+      end
+
+      private
+      def push_stacks(name, attributes)
+        @element_stack.push({name: name, attributes: attributes})
+        @text_stack.push("")
+      end
+
+      def pop_stacks
+        @text_stack.pop
+        @element_stack.pop
+      end
+    end
+  end
+end

data/lib/datasets/wikipedia.rb

@@ -1,6 +1,7 @@
 require "rexml/streamlistener"
 require "rexml/parsers/baseparser"
 require "rexml/parsers/streamparser"
+require "time"
 
 require_relative "dataset"
 
@@ -52,15 +53,22 @@ module Datasets
     end
 
     private
+    def base_name
+      "#{@language}wiki-latest-#{type_in_path}.xml.bz2"
+    end
+
+    def data_path
+      cache_dir_path + base_name
+    end
+
     def open_data(&block)
-      base_name = "#{@language}wiki-latest-#{type_in_path}.xml.bz2"
-      data_path = cache_dir_path + base_name
-      unless data_path.exist?
-        data_url = "https://dumps.wikimedia.org/#{@language}wiki/latest/#{base_name}"
-        download(data_path, data_url)
+      data_url = "https://dumps.wikimedia.org/#{@language}wiki/latest/#{base_name}"
+      bz2 = Enumerator.new do |yielder|
+        download(data_path, data_url) do |bz2_chunk|
+          yielder << bz2_chunk
+        end
       end
-
-      extract_bz2(data_path, &block)
+      extract_bz2(bz2, &block)
     end
 
     def type_in_path
@@ -153,7 +161,7 @@ module Datasets
         @text_stack.last << data
       end
 
-      def cdata(contnet)
+      def cdata(content)
         @text_stack.last << content
       end
 

data/lib/datasets/wine.rb

@@ -23,7 +23,8 @@ module Datasets
       super
       @metadata.id = 'wine'
       @metadata.name = 'Wine'
-      @metadata.url = 'http://archive.ics.uci.edu/ml/datasets/wine'
+      @metadata.url = 'https://archive.ics.uci.edu/ml/datasets/wine'
+      @metadata.licenses = ["CC-BY-4.0"]
       @metadata.description = -> { read_names }
     end
 
@@ -43,19 +44,15 @@ module Datasets
 
     def read_names
       names_path = cache_dir_path + 'wine.names'
-      unless names_path.exist?
-        names_url = 'http://archive.ics.uci.edu/ml/machine-learning-databases/wine/wine.names'
-        download(names_path, names_url)
-      end
+      names_url = 'https://archive.ics.uci.edu/ml/machine-learning-databases/wine/wine.names'
+      download(names_path, names_url)
       names_path.read
     end
 
     def open_data
       data_path = cache_dir_path + 'wine.data'
-      unless data_path.exist?
-        data_url = 'http://archive.ics.uci.edu/ml/machine-learning-databases/wine/wine.data'
-        download(data_path, data_url)
-      end
+      data_url = 'https://archive.ics.uci.edu/ml/machine-learning-databases/wine/wine.data'
+      download(data_path, data_url)
       CSV.open(data_path, converters: %i[numeric]) do |csv|
         yield(csv)
       end

data/lib/datasets/zip-extractor.rb

@@ -0,0 +1,48 @@
+require 'zip'
+
+module Datasets
+  class ZipExtractor
+    def initialize(path)
+      @path = path
+    end
+
+    def extract_first_file
+      Zip::File.open(@path) do |zip_file|
+        zip_file.each do |entry|
+          next unless entry.file?
+
+          entry.get_input_stream do |input|
+            return yield(input)
+          end
+        end
+      end
+      nil
+    end
+
+    def extract_file(file_path)
+      Zip::File.open(@path) do |zip_file|
+        zip_file.each do |entry|
+          next unless entry.file?
+          next unless entry.name == file_path
+
+          entry.get_input_stream do |input|
+            return yield(input)
+          end
+        end
+      end
+      nil
+    end
+
+    def extract_files
+      Zip::File.open(@path) do |zip_file|
+        zip_file.each do |entry|
+          next unless entry.file?
+
+          entry.get_input_stream do |input|
+            yield(input)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datasets.rb

@@ -1,22 +1,2 @@
-require_relative "datasets/version"
-
-require_relative "datasets/adult"
-require_relative "datasets/cifar"
-require_relative "datasets/cldr-plurals"
-require_relative "datasets/communities"
-require_relative "datasets/e-stat-japan"
-require_relative "datasets/fashion-mnist"
-require_relative "datasets/hepatitis"
-require_relative "datasets/iris"
-require_relative "datasets/libsvm"
-require_relative "datasets/libsvm-dataset-list"
-require_relative "datasets/mnist"
-require_relative "datasets/mushroom"
-require_relative "datasets/penguins"
-require_relative "datasets/penn-treebank"
-require_relative "datasets/postal-code-japan"
-require_relative "datasets/rdatasets"
-require_relative "datasets/seaborn-data"
-require_relative "datasets/sudachi-synonym-dictionary"
-require_relative "datasets/wikipedia"
-require_relative "datasets/wine"
+require_relative "datasets/lazy"
+Datasets::LAZY_LOADER.load_all

data/red-datasets.gemspec

@@ -34,7 +34,7 @@ Gem::Specification.new do |spec|
   spec.files += Dir.glob("doc/text/*")
   spec.test_files += Dir.glob("test/**/*")
 
-  spec.add_runtime_dependency("csv", ">= 3.0.5")
+  spec.add_runtime_dependency("csv", ">= 3.2.4")
   spec.add_runtime_dependency("rexml")
   spec.add_runtime_dependency("rubyzip")
 

data/test/helper.rb

@@ -1,6 +1,7 @@
 require "fileutils"
 require "pathname"
 require "time"
+require "tmpdir"
 
 require "datasets"
 
@@ -18,4 +19,24 @@ module Helper
       FileUtils.rm_rf(@tmp_dir)
     end
   end
+
+  module PathRestorable
+    def restore_path(path)
+      unless path.exist?
+        return yield
+      end
+
+      Dir.mktmpdir do |dir|
+        FileUtils.cp_r(path, dir, preserve: true)
+        begin
+          yield
+        ensure
+          FileUtils.rmtree(path, secure: true) if path.exist?
+          FileUtils.cp_r(Pathname(dir) + path.basename,
+                         path,
+                         preserve: true)
+        end
+      end
+    end
+  end
 end