red-datasets 0.1.4 → 0.1.5

data/lib/datasets/pmjt-dataset-list.rb

@@ -0,0 +1,67 @@
+require_relative "dataset"
+
+module Datasets
+  class PMJTDatasetList < Dataset
+    Record = Struct.new(:unit,
+                        :open_data_category,
+                        :tag,
+                        :release_time,
+                        :n_volumes,
+                        :type,
+                        :publication_year,
+                        :original_request_code,
+                        :id,
+                        :title,
+                        :text,
+                        :bibliographical_introduction,
+                        :year)
+
+    def initialize
+      super()
+      @metadata.id = "pmjt-dataset-list"
+      @metadata.name = "List of pre-modern Japanese text dataset"
+      @metadata.url = "http://codh.rois.ac.jp/pmjt/"
+      @metadata.licenses = ["CC-BY-SA-4.0"]
+      @metadata.description = <<~DESCRIPTION
+        Pre-Modern Japanese Text, owned by National Institute of Japanese Literature, is released image and text data as open data.
+        In addition, some text has description, transcription, and tagging data.
+      DESCRIPTION
+
+      @data_path = cache_dir_path + (@metadata.id + ".csv")
+    end
+
+    def each(&block)
+      return to_enum(__method__) unless block_given?
+
+      latest_version = "201901"
+      url = "http://codh.rois.ac.jp/pmjt/list/pmjt-dataset-list-#{latest_version}.csv"
+      download(@data_path, url)
+      CSV.open(@data_path, headers: :first_row, encoding: "Windows-31J:UTF-8") do |csv|
+        csv.each do |row|
+          record = create_record(row)
+          yield record
+        end
+      end
+    end
+
+    private
+    def create_record(csv_row)
+      record = Record.new
+      record.unit = csv_row["(単位)"]
+      record.open_data_category = csv_row["オープンデータ分類"]
+      record.tag = csv_row["タグ"]
+      record.release_time = csv_row["公開時期"]
+      record.n_volumes = csv_row["冊数等"]
+      record.type = csv_row["刊・写"]
+      record.publication_year = csv_row["刊年・書写年"]
+      record.original_request_code = csv_row["原本請求記号"]
+      record.id = csv_row["国文研書誌ID"]
+      record.title = csv_row["書名（統一書名）"]
+      record.text = csv_row["本文"]
+      record.bibliographical_introduction = csv_row["解題"]
+      record.year = csv_row["（西暦）"]
+
+      record
+    end
+  end
+end

data/lib/datasets/postal-code-japan.rb

@@ -49,9 +49,7 @@ module Datasets
       @metadata.id = "postal-code-japan-#{@reading}"
       @metadata.name = "Postal code in Japan (#{@reading})"
       @metadata.url = "https://www.post.japanpost.jp/zipcode/download.html"
-      @metadata.licenses = [
-        "CC0-1.0",
-      ]
+      @metadata.licenses = ["CC0-1.0"]
       @metadata.description = "Postal code in Japan (reading: #{@reading})"
     end
 
@@ -116,9 +114,7 @@ module Datasets
         data_url << "/roman/ken_all_rome.zip"
       end
       data_path = cache_dir_path + "#{@reading}-ken-all.zip"
-      unless data_path.exist?
-        download(data_path, data_url)
-      end
+      download(data_path, data_url)
 
       Zip::File.open(data_path.to_s) do |zip_file|
         zip_file.each do |entry|

data/lib/datasets/quora-duplicate-question-pair.rb

@@ -0,0 +1,51 @@
+require "csv"
+
+require_relative "dataset"
+
+module Datasets
+  class QuoraDuplicateQuestionPair < Dataset
+    class Record < Struct.new(:id,
+                              :first_question_id,
+                              :second_question_id,
+                              :first_question,
+                              :second_question,
+                              :duplicated)
+      alias_method :duplicated?, :duplicated
+    end
+
+    def initialize
+      super()
+      @metadata.id = "quora-duplicate-question-pair"
+      @metadata.name = "Quora's duplicated question pair dataset"
+      @metadata.url = "https://quoradata.quora.com/First-Quora-Dataset-Release-Question-Pairs"
+      @metadata.licenses = [
+        {
+          name: "Quora's Terms of Service",
+          url: "https://www.quora.com/about/tos",
+        }
+      ]
+    end
+
+    def each
+      return to_enum(__method__) unless block_given?
+
+      open_data do |csv|
+        csv.each do |row|
+          row["is_duplicate"] = (row["is_duplicate"] == 1)
+          record = Record.new(*row.fields)
+          yield(record)
+        end
+      end
+    end
+
+    private
+    def open_data
+      data_path = cache_dir_path + "quora_duplicate_questions.tsv"
+      data_url = "https://qim.fs.quoracdn.net/quora_duplicate_questions.tsv"
+      download(data_path, data_url)
+      CSV.open(data_path, col_sep: "\t", headers: true, converters: :all) do |csv|
+        yield(csv)
+      end
+    end
+  end
+end

data/lib/datasets/{rdatasets.rb → rdataset.rb}

@@ -2,7 +2,7 @@ require_relative "dataset"
 require_relative "tar-gz-readable"
 
 module Datasets
-  class RdatasetsList < Dataset
+  class RdatasetList < Dataset
     Record = Struct.new(:package,
                         :dataset,
                         :title,
@@ -18,8 +18,8 @@ module Datasets
 
     def initialize
       super
-      @metadata.id = "rdatasets"
-      @metadata.name = "Rdatasets"
+      @metadata.id = "rdataset-list"
+      @metadata.name = "Rdataset"
       @metadata.url = "https://vincentarelbundock.github.io/Rdatasets/"
       @metadata.licenses = ["GPL-3"]
       @data_url = "https://raw.githubusercontent.com/vincentarelbundock/Rdatasets/master/datasets.csv"
@@ -48,16 +48,19 @@ module Datasets
     end
 
     private def each_row(&block)
-      download(@data_path, @data_url) unless @data_path.exist?
+      download(@data_path, @data_url)
       CSV.open(@data_path, headers: :first_row, converters: :all) do |csv|
         csv.each(&block)
       end
     end
   end
 
-  class Rdatasets < Dataset
+  # For backward compatibility
+  RdatasetsList = RdatasetList
+
+  class Rdataset < Dataset
     def initialize(package_name, dataset_name)
-      list = RdatasetsList.new
+      list = RdatasetList.new
 
       info = list.filter(package: package_name, dataset: dataset_name).first
       unless info
@@ -65,8 +68,8 @@ module Datasets
       end
 
       super()
-      @metadata.id = "rdatasets-#{package_name}-#{dataset_name}"
-      @metadata.name = "Rdatasets: #{package_name}: #{dataset_name}"
+      @metadata.id = "rdataset-#{package_name}-#{dataset_name}"
+      @metadata.name = "Rdataset: #{package_name}: #{dataset_name}"
       @metadata.url = info.csv
       @metadata.licenses = ["GPL-3"]
       @metadata.description = info.title
@@ -81,15 +84,63 @@ module Datasets
     def each(&block)
       return to_enum(__method__) unless block_given?
 
-      download(@data_path, @metadata.url) unless @data_path.exist?
-      CSV.open(@data_path, headers: :first_row, converters: :all) do |csv|
-        csv.each do |row|
-          record = row.to_h
-          record.delete("")
-          record.transform_keys!(&:to_sym)
-          yield record
+      download(@data_path, @metadata.url)
+
+      na_converter = lambda do |field|
+        begin
+          if field.encode(CSV::ConverterEncoding) == "NA"
+            nil
+          else
+            field
+          end
+        rescue
+          field
         end
       end
+
+      inf_converter = lambda do |field|
+        begin
+          if field.encode(CSV::ConverterEncoding) == "Inf"
+            Float::INFINITY
+          else
+            field
+          end
+        rescue
+          field
+        end
+      end
+
+      quote_preserving_converter = lambda do |field, info|
+        f = field.encode(CSV::ConverterEncoding)
+        return f if info.quoted?
+
+        begin
+          begin
+            begin
+              return DateTime.parse(f) if f.match?(DateTimeMatcher)
+            rescue
+              return Integer(f)
+            end
+          rescue
+            return Float(f)
+          end
+        rescue
+          field
+        end
+      end
+
+      table = CSV.table(@data_path,
+                        header_converters: [:symbol_raw],
+                        # quote_preserving_converter should be the last
+                        converters: [na_converter, inf_converter, quote_preserving_converter])
+      table.delete(:"") # delete 1st column for indices.
+
+      table.each do |row|
+        yield row.to_h
+      end
     end
   end
+
+  # For backward compatibility
+  Rdatasets = Rdataset
 end

data/lib/datasets/seaborn.rb

@@ -0,0 +1,90 @@
+require "json"
+
+module Datasets
+  class SeabornList < Dataset
+    def initialize
+      super
+      @metadata.id = "seaborn-data-list"
+      @metadata.name = "seaborn: data list"
+      @metadata.url = "https://github.com/mwaskom/seaborn-data"
+      # Treat as the same license as seaborn
+      @metadata.licenses = ["BSD-3-Clause"]
+      @metadata.description = "Datasets for seaborn examples."
+    end
+
+    def each(&block)
+      return to_enum(__method__) unless block_given?
+
+      data_path = cache_dir_path + "trees.json"
+      url = "https://api.github.com/repos/mwaskom/seaborn-data/git/trees/master"
+      download(data_path, url)
+
+      tree = JSON.parse(File.read(data_path))["tree"]
+      tree.each do |content|
+        path = content["path"]
+        next unless path.end_with?(".csv")
+        dataset = File.basename(path, ".csv")
+        record = {dataset: dataset}
+        yield record
+      end
+    end
+  end
+
+  class Seaborn < Dataset
+    URL_FORMAT = "https://raw.githubusercontent.com/mwaskom/seaborn-data/master/%{name}.csv".freeze
+
+    def initialize(name)
+      super()
+      @metadata.id = "seaborn-#{name}"
+      @metadata.name = "seaborn: #{name}"
+      @metadata.url = URL_FORMAT % {name: name}
+      # @metadata.licenses = TODO
+
+      @name = name
+    end
+
+    def each(&block)
+      return to_enum(__method__) unless block_given?
+
+      data_path = cache_dir_path + "#{@name}.csv"
+      download(data_path, @metadata.url)
+      CSV.open(data_path, headers: :first_row, converters: :all) do |csv|
+        csv.each do |row|
+          record = prepare_record(row)
+          yield record
+        end
+      end
+    end
+
+    private
+    def prepare_record(csv_row)
+      record = csv_row.to_h
+      record.transform_keys! do |key|
+        if key.nil?
+          :index
+        else
+          key.to_sym
+        end
+      end
+
+      # Perform the same preprocessing as seaborn's load_dataset function
+      preprocessor = :"preprocess_#{@name}_record"
+      __send__(preprocessor, record) if respond_to?(preprocessor, true)
+
+      record
+    end
+
+    # The same preprocessing as seaborn.load_dataset
+    def preprocess_flights_record(record)
+      record[:month] &&= record[:month][0,3]
+    end
+
+    # The same preprocessing as seaborn.load_dataset
+    def preprocess_penguins_record(record)
+      record[:sex] &&= record[:sex].capitalize
+    end
+  end
+
+  # For backward compatibility
+  SeabornData = Seaborn
+end

data/lib/datasets/sudachi-synonym-dictionary.rb

@@ -21,9 +21,7 @@ module Datasets
       @metadata.id = "sudachi-synonym-dictionary"
       @metadata.name = "Sudachi synonym dictionary"
       @metadata.url = "https://github.com/WorksApplications/SudachiDict/blob/develop/docs/synonyms.md"
-      @metadata.licenses = [
-        "Apache-2.0",
-      ]
+      @metadata.licenses = ["Apache-2.0"]
       @metadata.description = lambda do
         download_description
       end
@@ -65,10 +63,8 @@ module Datasets
     private
     def open_data
       data_path = cache_dir_path + "synonyms.txt"
-      unless data_path.exist?
-        data_url = "https://raw.githubusercontent.com/WorksApplications/SudachiDict/develop/src/main/text/synonyms.txt"
-        download(data_path, data_url)
-      end
+      data_url = "https://raw.githubusercontent.com/WorksApplications/SudachiDict/develop/src/main/text/synonyms.txt"
+      download(data_path, data_url)
       CSV.open(data_path,
                encoding: "UTF-8",
                skip_blanks: true) do |csv|
@@ -78,10 +74,8 @@ module Datasets
 
     def download_description
       description_path = cache_dir_path + "synonyms.md"
-      unless description_path.exist?
-        description_url = "https://raw.githubusercontent.com/WorksApplications/SudachiDict/develop/docs/synonyms.md"
-        download(description_path, description_url)
-      end
+      description_url = "https://raw.githubusercontent.com/WorksApplications/SudachiDict/develop/docs/synonyms.md"
+      download(description_path, description_url)
       description_path.read
     end
 

data/lib/datasets/version.rb

@@ -1,3 +1,3 @@
 module Datasets
-  VERSION = "0.1.4"
+  VERSION = "0.1.5"
 end

data/lib/datasets/wikipedia-kyoto-japanese-english.rb

@@ -0,0 +1,219 @@
+require "csv"
+require "rexml/streamlistener"
+require "rexml/parsers/baseparser"
+require "rexml/parsers/streamparser"
+require "time"
+
+require_relative "dataset"
+require_relative "tar-gz-readable"
+
+module Datasets
+  class WikipediaKyotoJapaneseEnglish < Dataset
+    include TarGzReadable
+
+    Article = Struct.new(:source,
+                         :copyright,
+                         :contents,
+                         :sections)
+
+    Section = Struct.new(:id,
+                         :title,
+                         :contents)
+
+    class Title < Struct.new(:section,
+                             :japanese,
+                             :english)
+      def title?
+        true
+      end
+
+      def sentence?
+        false
+      end
+    end
+
+    Paragraph = Struct.new(:id,
+                           :sentences)
+
+    class Sentence < Struct.new(:id,
+                                :section,
+                                :paragraph,
+                                :japanese,
+                                :english)
+      def title?
+        false
+      end
+
+      def sentence?
+        true
+      end
+    end
+
+    Entry = Struct.new(:japanese,
+                       :english)
+
+    def initialize(type: :article)
+      unless [:article, :lexicon].include?(type)
+        raise ArgumentError, "Please set type :article or :lexicon: #{type.inspect}"
+      end
+
+      super()
+      @type = type
+      @metadata.id = "wikipedia-kyoto-japanese-english"
+      @metadata.name =
+        "The Japanese-English Bilingual Corpus of Wikipedia's Kyoto Articles"
+      @metadata.url = "https://alaginrc.nict.go.jp/WikiCorpus/index_E.html"
+      @metadata.licenses = ["CC-BY-SA-3.0"]
+      @metadata.description = <<-DESCRIPTION
+"The Japanese-English Bilingual Corpus of Wikipedia's Kyoto Articles"
+aims mainly at supporting research and development relevant to
+high-performance multilingual machine translation, information
+extraction, and other language processing technologies. The National
+Institute of Information and Communications Technology (NICT) has
+created this corpus by manually translating Japanese Wikipedia
+articles (related to Kyoto) into English.
+      DESCRIPTION
+    end
+
+    def each(&block)
+      return to_enum(__method__) unless block_given?
+
+      data_path = download_tar_gz
+
+      open_tar_gz(data_path) do |tar|
+        tar.each do |entry|
+          next unless entry.file?
+          base_name = File.basename(entry.full_name)
+          case @type
+          when :article
+            next unless base_name.end_with?(".xml")
+            listener = ArticleListener.new(block)
+            parser = REXML::Parsers::StreamParser.new(entry.read, listener)
+            parser.parse
+          when :lexicon
+            next unless base_name == "kyoto_lexicon.csv"
+            is_header = true
+            CSV.parse(entry.read.force_encoding("UTF-8")) do |row|
+              if is_header
+                is_header = false
+                next
+              end
+              yield(Entry.new(row[0], row[1]))
+            end
+          end
+        end
+      end
+    end
+
+    private
+    def download_tar_gz
+      base_name = "wiki_corpus_2.01.tar.gz"
+      data_path = cache_dir_path + base_name
+      data_url = "https://alaginrc.nict.go.jp/WikiCorpus/src/#{base_name}"
+      download(data_path, data_url)
+      data_path
+    end
+
+    class ArticleListener
+      include REXML::StreamListener
+
+      def initialize(block)
+        @block = block
+        @article = nil
+        @title = nil
+        @section = nil
+        @page = nil
+        @sentence = nil
+        @text_container_stack = []
+        @element_stack = []
+        @text_stack = [""]
+      end
+
+      def tag_start(name, attributes)
+        push_stacks(name, attributes)
+        case name
+        when "art"
+          @article = Article.new
+          @article.contents = []
+          @article.sections = []
+        when "tit"
+          @title = Title.new
+          @title.section = @section
+          @text_container_stack.push(@title)
+        when "sec"
+          @section = Section.new
+          @section.id = attributes["id"]
+          @section.contents = []
+          @text_container_stack.push(@section)
+        when "par"
+          @paragraph = Paragraph.new
+          @paragraph.id = attributes["id"]
+          @paragraph.sentences = []
+          @text_container_stack.push(@paragraph)
+        when "sen"
+          @sentence = Sentence.new
+          @sentence.id = attributes["id"]
+          @text_container_stack.push(@sentence)
+        end
+      end
+
+      def tag_end(name)
+        case name
+        when "art"
+          @block.call(@article)
+          @article = nil
+        when "inf"
+          @article.source = @text_stack.last
+        when "copyright"
+          @article.copyright = @text_stack.last
+        when "tit"
+          @article.contents << @title
+          if @section
+            @section.title = @title
+            @section.contents << @title
+          end
+          @title = nil
+          @text_container_stack.pop
+        when "sec"
+          @article.sections << @section
+          @section = nil
+          @text_container_stack.pop
+        when "par"
+          @paragraph = nil
+          @text_container_stack.pop
+        when "sen"
+          @article.contents << @sentence
+          @sentence.section = @section
+          @section.contents << @sentence if @section
+          @sentence.paragraph = @paragraph
+          @paragraph.sentences << @sentence if @paragraph
+          @sentence = nil
+          @text_container_stack.pop
+        when "j"
+          @text_container_stack.last.japanese = @text_stack.last
+        when "e"
+          attributes = @element_stack.last[:attributes]
+          if attributes["type"] == "check"
+            @text_container_stack.last.english = @text_stack.last
+          end
+        end
+        pop_stacks
+      end
+
+      def text(data)
+        @text_stack.last << data
+      end
+
+      private
+      def push_stacks(name, attributes)
+        @element_stack.push({name: name, attributes: attributes})
+        @text_stack.push("")
+      end
+
+      def pop_stacks
+        @text_stack.pop
+        @element_stack.pop
+      end
+    end
+  end
+end

data/lib/datasets/wikipedia.rb

@@ -1,6 +1,7 @@
 require "rexml/streamlistener"
 require "rexml/parsers/baseparser"
 require "rexml/parsers/streamparser"
+require "time"
 
 require_relative "dataset"
 
@@ -55,10 +56,8 @@ module Datasets
     def open_data(&block)
       base_name = "#{@language}wiki-latest-#{type_in_path}.xml.bz2"
       data_path = cache_dir_path + base_name
-      unless data_path.exist?
-        data_url = "https://dumps.wikimedia.org/#{@language}wiki/latest/#{base_name}"
-        download(data_path, data_url)
-      end
+      data_url = "https://dumps.wikimedia.org/#{@language}wiki/latest/#{base_name}"
+      download(data_path, data_url)
 
       extract_bz2(data_path, &block)
     end
@@ -153,7 +152,7 @@ module Datasets
         @text_stack.last << data
       end
 
-      def cdata(contnet)
+      def cdata(content)
         @text_stack.last << content
       end
 

data/lib/datasets/wine.rb

@@ -23,7 +23,8 @@ module Datasets
       super
       @metadata.id = 'wine'
       @metadata.name = 'Wine'
-      @metadata.url = 'http://archive.ics.uci.edu/ml/datasets/wine'
+      @metadata.url = 'https://archive.ics.uci.edu/ml/datasets/wine'
+      @metadata.licenses = ["CC-BY-4.0"]
       @metadata.description = -> { read_names }
     end
 
@@ -43,19 +44,15 @@ module Datasets
 
     def read_names
       names_path = cache_dir_path + 'wine.names'
-      unless names_path.exist?
-        names_url = 'http://archive.ics.uci.edu/ml/machine-learning-databases/wine/wine.names'
-        download(names_path, names_url)
-      end
+      names_url = 'https://archive.ics.uci.edu/ml/machine-learning-databases/wine/wine.names'
+      download(names_path, names_url)
       names_path.read
     end
 
     def open_data
       data_path = cache_dir_path + 'wine.data'
-      unless data_path.exist?
-        data_url = 'http://archive.ics.uci.edu/ml/machine-learning-databases/wine/wine.data'
-        download(data_path, data_url)
-      end
+      data_url = 'https://archive.ics.uci.edu/ml/machine-learning-databases/wine/wine.data'
+      download(data_path, data_url)
       CSV.open(data_path, converters: %i[numeric]) do |csv|
         yield(csv)
       end