red-datasets 0.1.4 → 0.1.6

data/lib/datasets/dataset.rb

@@ -1,5 +1,6 @@
 require "pathname"
 
+require_relative "cache-path"
 require_relative "downloader"
 require_relative "error"
 require_relative "metadata"
@@ -19,38 +20,72 @@ module Datasets
     end
 
     def clear_cache!
-      if cache_dir_path.exist?
-        FileUtils.rmtree(cache_dir_path.to_s, secure: true)
-      end
+      cache_path.remove
     end
 
     private
+
     def cache_dir_path
-      case RUBY_PLATFORM
-      when /mswin/, /mingw/
-        base_dir = ENV["LOCALAPPDATA"] || "~/AppData/Local"
-      when /darwin/
-        base_dir = "~/Library/Caches"
-      else
-        base_dir = ENV["XDG_CACHE_HOME"] || "~/.cache"
-      end
-      Pathname(base_dir).expand_path + "red-datasets" + metadata.id
+      cache_path.base_dir
     end
 
-    def download(output_path, url)
+    def cache_path
+      @cache_path ||= CachePath.new(@metadata.id)
+    end
+
+    def download(output_path, url, &block)
       downloader = Downloader.new(url)
-      downloader.download(output_path)
+      downloader.download(output_path, &block)
     end
 
-    def extract_bz2(path)
-      input, output = IO.pipe
-      pid = spawn("bzcat", path.to_s, {:out => output})
-      begin
-        output.close
-        yield(input)
-      ensure
-        input.close
-        Process.waitpid(pid)
+    def extract_bz2(bz2)
+      case bz2
+      when Pathname, String
+        IO.pipe do |input, output|
+          pid = spawn("bzcat", bz2.to_s, {out: output})
+          begin
+            output.close
+            yield(input)
+          ensure
+            input.close
+            Process.waitpid(pid)
+          end
+        end
+      else
+        IO.pipe do |bz2_input, bz2_output|
+          IO.pipe do |plain_input, plain_output|
+            bz2_stop = false
+            bz2_thread = Thread.new do
+              begin
+                bz2.each do |chunk|
+                  bz2_output.write(chunk)
+                  bz2_output.flush
+                  break if bz2_stop
+                end
+              rescue => error
+                message = "Failed to read bzcat input: " +
+                          "#{error.class}: #{error.message}"
+                $stderr.puts(message)
+              ensure
+                bz2_output.close
+              end
+            end
+            begin
+              pid = spawn("bzcat", {in: bz2_input, out: plain_output})
+              begin
+                bz2_input.close
+                plain_output.close
+                yield(plain_input)
+              ensure
+                plain_input.close
+                Process.waitpid(pid)
+              end
+            ensure
+              bz2_stop = true
+              bz2_thread.join
+            end
+          end
+        end
       end
     end
   end

data/lib/datasets/diamonds.rb

@@ -0,0 +1,26 @@
+require_relative "ggplot2-dataset"
+
+module Datasets
+  class Diamonds < Ggplot2Dataset
+    Record = Struct.new(:carat,
+                        :cut,
+                        :color,
+                        :clarity,
+                        :depth,
+                        :table,
+                        :price,
+                        :x,
+                        :y,
+                        :z)
+
+    def initialize()
+      super("diamonds")
+      @metadata.id = "diamonds"
+      @metadata.name = "Diamonds"
+      @metadata.licenses = ["CC0-1.0"]
+    end
+
+    COLUMN_NAME_MAPPING = {
+    }
+  end
+end

data/lib/datasets/downloader.rb

@@ -22,45 +22,115 @@ module Datasets
       end
     end
 
-    def download(output_path)
-      output_path.parent.mkpath
+    def download(output_path, &block)
+      if output_path.exist?
+        yield_chunks(output_path, &block) if block_given?
+        return
+      end
 
-      headers = {"User-Agent" => "Red Datasets/#{VERSION}"}
-      start = nil
       partial_output_path = Pathname.new("#{output_path}.partial")
-      if partial_output_path.exist?
-        start = partial_output_path.size
-        headers["Range"] = "bytes=#{start}-"
-      end
+      synchronize(output_path, partial_output_path) do
+        output_path.parent.mkpath
 
-      start_http(@url, headers) do |response|
-        if response.is_a?(Net::HTTPPartialContent)
-          mode = "ab"
-        else
+        n_retries = 0
+        n_max_retries = 5
+        begin
+          headers = {
+            "Accept-Encoding" => "identity",
+            "User-Agent" => "Red Datasets/#{VERSION}",
+          }
           start = nil
-          mode = "wb"
-        end
+          if partial_output_path.exist?
+            start = partial_output_path.size
+            headers["Range"] = "bytes=#{start}-"
+          end
+
+          start_http(@url, headers) do |response|
+            if response.is_a?(Net::HTTPPartialContent)
+              mode = "ab"
+            else
+              start = nil
+              mode = "wb"
+            end
 
-        base_name = @url.path.split("/").last
-        size_current = 0
-        size_max = response.content_length
-        if start
-          size_current += start
-          size_max += start
+            base_name = @url.path.split("/").last
+            size_current = 0
+            size_max = response.content_length
+            if start
+              size_current += start
+              size_max += start
+              if block_given? and n_retries.zero?
+                yield_chunks(partial_output_path, &block)
+              end
+            end
+            progress_reporter = ProgressReporter.new(base_name, size_max)
+            partial_output_path.open(mode) do |output|
+              response.read_body do |chunk|
+                size_current += chunk.bytesize
+                progress_reporter.report(size_current)
+                output.write(chunk)
+                yield(chunk) if block_given?
+              end
+            end
+          end
+          FileUtils.mv(partial_output_path, output_path)
+        rescue Net::ReadTimeout => error
+          n_retries += 1
+          retry if n_retries < n_max_retries
+          raise
+        rescue TooManyRedirects => error
+          last_url = error.message[/\Atoo many redirections: (.+)\z/, 1]
+          raise TooManyRedirects, "too many redirections: #{@url} .. #{last_url}"
         end
-        progress_reporter = ProgressReporter.new(base_name, size_max)
-        partial_output_path.open(mode) do |output|
-          response.read_body do |chunk|
-            size_current += chunk.bytesize
-            progress_reporter.report(size_current)
-            output.write(chunk)
+      end
+    end
+
+    private def synchronize(output_path, partial_output_path)
+      begin
+        Process.getpgid(Process.pid)
+      rescue NotImplementedError
+        return yield
+      end
+
+      lock_path = Pathname("#{output_path}.lock")
+      loop do
+        lock_path.parent.mkpath
+        begin
+          lock = lock_path.open(File::RDWR | File::CREAT | File::EXCL)
+        rescue SystemCallError
+          valid_lock_path = true
+          begin
+            pid = Integer(lock_path.read.chomp, 10)
+          rescue ArgumentError
+            # The process that acquired the lock will be exited before
+            # it stores its process ID.
+            valid_lock_path = (lock_path.mtime > 10)
+          else
+            begin
+              Process.getpgid(pid)
+            rescue SystemCallError
+              # Process that acquired the lock doesn't exist
+              valid_lock_path = false
+            end
+          end
+          if valid_lock_path
+            sleep(1 + rand(10))
+          else
+            lock_path.delete
           end
+          retry
+        else
+          begin
+            lock.puts(Process.pid.to_s)
+            lock.flush
+            yield
+          ensure
+            lock.close
+            lock_path.delete
+          end
+          break
         end
       end
-      FileUtils.mv(partial_output_path, output_path)
-    rescue TooManyRedirects => error
-      last_url = error.message[/\Atoo many redirections: (.+)\z/, 1]
-      raise TooManyRedirects, "too many redirections: #{@url} .. #{last_url}"
     end
 
     private def start_http(url, headers, limit = 10, &block)
@@ -94,6 +164,16 @@ module Datasets
       end
     end
 
+    private def yield_chunks(path)
+      path.open("rb") do |output|
+        chunk_size = 1024 * 1024
+        chunk = ""
+        while output.read(chunk_size, chunk)
+          yield(chunk)
+        end
+      end
+    end
+
     class ProgressReporter
       def initialize(base_name, size_max)
         @base_name = base_name

data/lib/datasets/e-stat-japan.rb

@@ -74,6 +74,7 @@ module Datasets
         @metadata.id = "e-stat-japan-#{@api_version}"
         @metadata.name = "e-Stat API #{@api_version}"
         @metadata.url = @base_url
+        @metadata.licenses = ["CC-BY-4.0"]
         @metadata.description = "e-Stat API #{@api_version}"
 
         @id = id
@@ -214,7 +215,7 @@ module Datasets
         # even if error happens dispite of its error mapping.
         # So we can't avoid caching retrieved response from the api.
         # ref: https://www.e-stat.go.jp/api/api-info/e-stat-manual3-0
-        download(@data_path, @url.to_s) unless @data_path.exist?
+        download(@data_path, @url.to_s)
       end
 
       def index_data

data/lib/datasets/fashion-mnist.rb

@@ -8,5 +8,9 @@ module Datasets
     def dataset_name
       "Fashion-MNIST"
     end
+
+    def licenses
+      ["MIT"]
+    end
   end
 end

data/lib/datasets/fuel-economy.rb

@@ -0,0 +1,35 @@
+require_relative "ggplot2-dataset"
+
+module Datasets
+  class FuelEconomy < Ggplot2Dataset
+    Record = Struct.new(:manufacturer,
+                        :model,
+                        :displacement,
+                        :year,
+                        :n_cylinders,
+                        :transmission,
+                        :drive_train,
+                        :city_mpg,
+                        :highway_mpg,
+                        :fuel,
+                        :type)
+
+    def initialize
+      super("mpg")
+      @metadata.id = "fuel-economy"
+      @metadata.name = "Fuel economy"
+      @metadata.licenses = ["CC0-1.0"]
+    end
+
+    COLUMN_NAME_MAPPING = {
+      "displ" => "displacement",
+      "cyl" => "n_cylinders",
+      "trans" => "transmissions",
+      "drv" => "drive_train",
+      "cty" => "city_mpg",
+      "hwy" => "highway_mpg",
+      "fl" => "fuel",
+      "class" => "type",
+    }
+  end
+end

data/lib/datasets/geolonia.rb

@@ -0,0 +1,67 @@
+require 'csv'
+
+require_relative 'dataset'
+
+module Datasets
+  class Geolonia < Dataset
+    Record = Struct.new(:prefecture_code,
+                        :prefecture_name,
+                        :prefecture_kana,
+                        :prefecture_romaji,
+                        :municipality_code,
+                        :municipality_name,
+                        :municipality_kana,
+                        :municipality_romaji,
+                        :street_name,
+                        :street_kana,
+                        :street_romaji,
+                        :alias,
+                        :latitude,
+                        :longitude)
+
+    def initialize
+      super
+      @metadata.id = 'geolonia'
+      @metadata.name = 'Geolonia'
+      @metadata.url = 'https://github.com/geolonia/japanese-addresses'
+      @metadata.licenses = ["CC-BY-4.0"]
+      @metadata.description = lambda do
+        fetch_readme
+      end
+    end
+
+    def each
+      return to_enum(__method__) unless block_given?
+
+      open_data do |csv|
+        csv.readline
+        csv.each do |row|
+          record = Record.new(*row)
+          yield(record)
+        end
+      end
+    end
+
+    private
+    def download_base_url
+      "https://raw.githubusercontent.com/geolonia/japanese-addresses/master"
+    end
+
+    def open_data
+      data_path = cache_dir_path + 'latest.csv'
+      data_url = "#{download_base_url}/data/latest.csv"
+      download(data_path, data_url)
+      CSV.open(data_path) do |csv|
+        yield(csv)
+      end
+    end
+
+    def fetch_readme
+      readme_base_name = "README.md"
+      readme_path = cache_dir_path + readme_base_name
+      readme_url = "#{download_base_url}/#{readme_base_name}"
+      download(readme_path, readme_url)
+      readme_path.read.split(/^## API/, 2)[0].strip
+    end
+  end
+end

data/lib/datasets/ggplot2-dataset.rb

@@ -0,0 +1,79 @@
+module Datasets
+  class Ggplot2Dataset < Dataset
+    def initialize(ggplot2_dataset_name)
+      super()
+      @ggplot2_dataset_name = ggplot2_dataset_name
+      @metadata.url =
+        "https://ggplot2.tidyverse.org/reference/#{@ggplot2_dataset_name}.html"
+      @metadata.description = lambda do
+        fetch_description
+      end
+    end
+
+    def each
+      return to_enum(__method__) unless block_given?
+
+      data_base_name = "#{@ggplot2_dataset_name}.csv"
+      data_path = cache_dir_path + data_base_name
+      data_url = "#{download_base_url}/data-raw/#{data_base_name}"
+      download(data_path, data_url)
+      CSV.open(data_path, headers: :first_row, converters: :all) do |csv|
+        record_class = self.class::Record
+        csv.each do |row|
+          record = record_class.new(*row.fields)
+          yield record
+        end
+      end
+    end
+
+    private
+    def download_base_url
+      "https://raw.githubusercontent.com/tidyverse/ggplot2/main"
+    end
+
+    def fetch_description
+      data_r_base_name = "data.R"
+      data_r_path = cache_dir_path + data_r_base_name
+      data_r_url = "#{download_base_url}/R/#{data_r_base_name}"
+      download(data_r_path, data_r_url)
+      descriptions = {}
+      comment = ""
+      File.open(data_r_path) do |data_r|
+        data_r.each_line do |line|
+          case line.chomp
+          when /\A#'/
+            comment_content = Regexp.last_match.post_match
+            unless comment_content.empty?
+              comment_content = comment_content[1..-1]
+            end
+            comment << comment_content
+            comment << "\n"
+          when /\A"(.+)"\z/
+            name = Regexp.last_match[1]
+            descriptions[name] = parse_roxygen(comment.rstrip)
+            comment = ""
+          end
+        end
+        descriptions[@ggplot2_dataset_name]
+      end
+    end
+
+    def parse_roxygen(roxygen)
+      column_name_mapping = self.class::COLUMN_NAME_MAPPING
+      roxygen
+        .gsub(/\\url\{(.*?)\}/, "\\1")
+        .gsub(/^@format /, "")
+        .gsub(/\\describe\{(.*)\}/m) do
+        content = $1
+        content.gsub(/\\item\{(.*?)\}\{(.*?)\}/m) do
+          column_name = $1
+          description = $2
+          column_name = column_name_mapping[column_name] || column_name
+          description = description
+                          .gsub(/\\\$/, "$")
+          "* #{column_name}: #{description}"
+        end
+      end
+    end
+  end
+end

data/lib/datasets/hepatitis.rb

@@ -163,6 +163,7 @@ module Datasets
       @metadata.id = "hepatitis"
       @metadata.name = "Hepatitis"
       @metadata.url = "https://archive.ics.uci.edu/ml/datasets/hepatitis"
+      @metadata.licenses = ["CC-BY-4.0"]
       @metadata.description = lambda do
         read_names
       end
@@ -186,10 +187,8 @@ module Datasets
 
     def open_data
       data_path = cache_dir_path + "hepatitis.csv"
-      unless data_path.exist?
-        data_url = "#{base_url}/hepatitis.data"
-        download(data_path, data_url)
-      end
+      data_url = "#{base_url}/hepatitis.data"
+      download(data_path, data_url)
       CSV.open(data_path) do |csv|
         yield(csv)
       end
@@ -197,10 +196,8 @@ module Datasets
 
     def read_names
       names_path = cache_dir_path + "hepatitis.names"
-      unless names_path.exist?
-        names_url = "#{base_url}/hepatitis.names"
-        download(names_path, names_url)
-      end
+      names_url = "#{base_url}/hepatitis.names"
+      download(names_path, names_url)
       names_path.read
     end
   end

data/lib/datasets/iris.rb

@@ -15,6 +15,7 @@ module Datasets
       @metadata.id = "iris"
       @metadata.name = "Iris"
       @metadata.url = "https://archive.ics.uci.edu/ml/datasets/Iris"
+      @metadata.licenses = ["CC-BY-4.0"]
       @metadata.description = lambda do
         read_names
       end
@@ -35,10 +36,8 @@ module Datasets
     private
     def open_data
       data_path = cache_dir_path + "iris.csv"
-      unless data_path.exist?
-        data_url = "https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data"
-        download(data_path, data_url)
-      end
+      data_url = "https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data"
+      download(data_path, data_url)
       CSV.open(data_path, converters: [:numeric]) do |csv|
         yield(csv)
       end
@@ -46,10 +45,8 @@ module Datasets
 
     def read_names
       names_path = cache_dir_path + "iris.names"
-      unless names_path.exist?
-        names_url = "https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.names"
-        download(names_path, names_url)
-      end
+      names_url = "https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.names"
+      download(names_path, names_url)
       names_path.read
     end
   end

data/lib/datasets/ita-corpus.rb

@@ -0,0 +1,57 @@
+require_relative 'dataset'
+
+module Datasets
+  class ITACorpus < Dataset
+    Record = Struct.new(:id,
+                        :sentence)
+
+    def initialize(type: :emotion)
+      unless [:emotion, :recitation].include?(type)
+        raise ArgumentError, "Please set type :emotion or :recitation: #{type.inspect}"
+      end
+
+      super()
+      @type = type
+      @metadata.id = 'ita-corpus'
+      @metadata.name = 'ITA-corpus'
+      @metadata.url = 'https://github.com/mmorise/ita-corpus'
+      @metadata.licenses = ['Unlicense']
+      @metadata.description = lambda do
+        fetch_readme
+      end
+    end
+
+    def each(&block)
+      return to_enum(__method__) unless block_given?
+
+      data_path = cache_dir_path + "#{@type}_transcript_utf8.txt"
+      data_url = "#{download_base_url}/#{@type}_transcript_utf8.txt"
+      download(data_path, data_url)
+
+      parse_data(data_path, &block)
+    end
+
+    private
+    def fetch_readme
+      readme_base_name = "README.md"
+      readme_path = cache_dir_path + readme_base_name
+      readme_url = "#{download_base_url}/#{readme_base_name}"
+      download(readme_path, readme_url)
+      readme_path.read.split(/^## ファイル構成/, 2)[0].strip
+    end
+
+    def download_base_url
+      "https://raw.githubusercontent.com/mmorise/ita-corpus/main"
+    end
+
+    def parse_data(data_path)
+      File.open(data_path) do |f|
+        f.each_line(chomp: true) do |line|
+          id, sentence = line.split(':', 2)
+          record = Record.new(id , sentence)
+          yield(record)
+        end
+      end
+    end
+  end
+end

data/lib/datasets/kuzushiji-mnist.rb

@@ -0,0 +1,16 @@
+require_relative 'mnist'
+
+module Datasets
+  class KuzushijiMNIST < MNIST
+    BASE_URL = "http://codh.rois.ac.jp/kmnist/dataset/kmnist/"
+
+    private
+    def dataset_name
+      "Kuzushiji-MNIST"
+    end
+
+    def licenses
+      ["CC-BY-SA-4.0"]
+    end
+  end
+end