red-datasets 0.0.6 → 0.1.1

data/lib/datasets/penn-treebank.rb

@@ -2,7 +2,7 @@ require_relative "dataset"
 
 module Datasets
   class PennTreebank < Dataset
-    Record = Struct.new(:word, :id)
+    Record = Struct.new(:word)
 
     DESCRIPTION = <<~DESC
       `Penn Tree Bank <https://www.cis.upenn.edu/~treebank/>`_ is originally a
@@ -46,17 +46,10 @@ module Datasets
 
     private
     def parse_data(data_path)
-      index = 0
-      vocabulary = {}
       File.open(data_path) do |f|
         f.each_line do |line|
           line.split.each do |word|
-            word = word.strip
-            unless vocabulary.key?(word)
-              vocabulary[word] = index
-              index += 1
-            end
-            yield(Record.new(word, vocabulary[word]))
+            yield(Record.new(word.strip))
           end
         end
       end

data/lib/datasets/postal-code-japan.rb

@@ -0,0 +1,154 @@
+require "csv"
+require "zip"
+
+require_relative "dataset"
+
+module Datasets
+  class PostalCodeJapan < Dataset
+    class Record < Struct.new(:organization_code,
+                              :old_postal_code,
+                              :postal_code,
+                              :prefecture_reading,
+                              :city_reading,
+                              :address_reading,
+                              :prefecture,
+                              :city,
+                              :address,
+                              :have_multiple_postal_codes,
+                              :have_address_number_per_koaza,
+                              :have_chome,
+                              :postal_code_is_shared,
+                              :changed,
+                              :change_reason)
+      alias_method :have_multiple_postal_codes?,
+                   :have_multiple_postal_codes
+      alias_method :have_address_number_per_koaza?,
+                   :have_address_number_per_koaza
+      alias_method :have_chome?,
+                   :have_chome
+      alias_method :postal_code_is_shared?,
+                   :postal_code_is_shared
+      alias_method :changed?,
+                   :changed
+    end
+
+    VALID_READINGS = [
+      :lowercase,
+      :uppercase,
+      :romaji,
+    ]
+    def initialize(reading: :lowercase)
+      super()
+      @reading = reading
+      unless VALID_READINGS.include?(@reading)
+        message = ":reading must be one of ["
+        message << VALID_READINGS.collect(&:inspect).join(", ")
+        message << "]: #{@reading.inspect}"
+        raise ArgumentError, message
+      end
+      @metadata.id = "postal-code-japan-#{@reading}"
+      @metadata.name = "Postal code in Japan (#{@reading})"
+      @metadata.url = "https://www.post.japanpost.jp/zipcode/download.html"
+      @metadata.licenses = [
+        "CC0-1.0",
+      ]
+      @metadata.description = "Postal code in Japan (reading: #{@reading})"
+    end
+
+    def each(&block)
+      return to_enum(__method__) unless block_given?
+
+      open_data do |input|
+        utf8_data = input.read.encode(Encoding::UTF_8, Encoding::CP932)
+        options = {
+          quote_char: nil,
+          strip: %Q["],
+        }
+        if @reading == :romaji
+          CSV.parse(utf8_data, **options) do |row|
+            yield(Record.new(nil,
+                             nil,
+                             row[0],
+                             row[4],
+                             row[5],
+                             row[6],
+                             row[1],
+                             row[2],
+                             row[3],
+                             false,
+                             false,
+                             false,
+                             false,
+                             false,
+                             nil))
+          end
+        else
+          CSV.parse(utf8_data, **options) do |row|
+            yield(Record.new(row[0],
+                             row[1].rstrip,
+                             row[2],
+                             row[3],
+                             row[4],
+                             row[5],
+                             row[6],
+                             row[7],
+                             row[8],
+                             (row[9] == "1"),
+                             (row[10] == "1"),
+                             (row[11] == "1"),
+                             (row[12] == "1"),
+                             (row[13] != "0"),
+                             convert_change_reason(row[14])))
+          end
+        end
+      end
+    end
+
+    private
+    def open_data
+      data_url = "https://www.post.japanpost.jp/zipcode/dl"
+      case @reading
+      when :lowercase
+        data_url << "/kogaki/zip/ken_all.zip"
+      when :uppercase
+        data_url << "/oogaki/zip/ken_all.zip"
+      when :romaji
+        data_url << "/roman/ken_all_rome.zip"
+      end
+      data_path = cache_dir_path + "#{@reading}-ken-all.zip"
+      unless data_path.exist?
+        download(data_path, data_url)
+      end
+
+      Zip::File.open(data_path.to_s) do |zip_file|
+        zip_file.each do |entry|
+          next unless entry.file?
+          entry.get_input_stream do |input|
+            yield(input)
+          end
+        end
+      end
+    end
+
+    def convert_change_reason(reason)
+      case reason
+      when "0"
+        nil
+      when "1"
+        :new
+      when "2"
+        :japanese_addressing_system
+      when "3"
+        :land_readjustment
+      when "4"
+        :postal_district_adjustment
+      when "5"
+        :correction
+      when "6"
+        :deletion
+      else
+        :unknown
+      end
+    end
+  end
+end

data/lib/datasets/table.rb

@@ -1,17 +1,109 @@
+require "datasets/dictionary"
+
 module Datasets
   class Table
+    class Record
+      include Enumerable
+
+      def initialize(table, index)
+        @table = table
+        @index = index
+      end
+
+      def [](column_name_or_column_index)
+        @table[column_name_or_column_index][@index]
+      end
+
+      def each
+        return to_enum(__method__) unless block_given?
+        @table.each_column.each do |column_name, column_values|
+          yield(column_name, column_values[@index])
+        end
+      end
+
+      def values
+        @table.each_column.collect do |_column_name, column_values|
+          column_values[@index]
+        end
+      end
+
+      def to_h
+        hash = {}
+        each do |column_name, column_value|
+          hash[column_name] = column_value
+        end
+        hash
+      end
+
+      def inspect
+        "#<#{self.class.name} #{@table.dataset.metadata.name}[#{@index}] #{to_h.inspect}>"
+      end
+    end
+
     include Enumerable
 
+    attr_reader :dataset
     def initialize(dataset)
       @dataset = dataset
+      @dictionaries = {}
     end
 
-    def each(&block)
+    def n_columns
+      columner_data.size
+    end
+    alias_method :size, :n_columns
+    alias_method :length, :n_columns
+
+    def n_rows
+      first_column = columner_data.first
+      return 0 if first_column.nil?
+      first_column[1].size
+    end
+
+    def column_names
+      columner_data.keys
+    end
+
+    def each_column(&block)
       columner_data.each(&block)
     end
+    alias_method :each, :each_column
+
+    def each_record
+      return to_enum(__method__) unless block_given?
+      n_rows.times do |i|
+        yield(Record.new(self, i))
+      end
+    end
+
+    def find_record(row)
+      row += n_rows if row < 0
+      return nil if row < 0
+      return nil if row >= n_rows
+      Record.new(self, row)
+    end
+
+    def [](name_or_index)
+      case name_or_index
+      when Integer
+        index = name_or_index
+        columner_data.each_with_index do |(_name, values), i|
+          return values if i == index
+        end
+        nil
+      else
+        name = name_or_index
+        columner_data[normalize_name(name)]
+      end
+    end
 
-    def [](name)
-      columner_data[name.to_sym]
+    def dictionary_encode(name)
+      @dictionaries[normalize_name(name)] ||= Dictionary.new(self[name])
+    end
+
+    def label_encode(name)
+      dictionary = dictionary_encode(name)
+      dictionary.encode(self[name])
     end
 
     def fetch_values(*keys)
@@ -55,5 +147,9 @@ module Datasets
     def columner_data
       @columns ||= to_h
     end
+
+    def normalize_name(name)
+      name.to_sym
+    end
   end
 end

data/lib/datasets/version.rb

@@ -1,3 +1,3 @@
 module Datasets
-  VERSION = "0.0.6"
+  VERSION = "0.1.1"
 end

data/lib/datasets/wikipedia.rb

@@ -52,7 +52,7 @@ module Datasets
     end
 
     private
-    def open_data
+    def open_data(&block)
       base_name = "#{@language}wiki-latest-#{type_in_path}.xml.bz2"
       data_path = cache_dir_path + base_name
       unless data_path.exist?
@@ -60,15 +60,7 @@ module Datasets
         download(data_path, data_url)
       end
 
-      input, output = IO.pipe
-      pid = spawn("bzcat", data_path.to_s, {:out => output})
-      begin
-        output.close
-        yield(input)
-      ensure
-        input.close
-        Process.waitpid(pid)
-      end
+      extract_bz2(data_path, &block)
     end
 
     def type_in_path

data/lib/datasets/wine.rb

@@ -0,0 +1,64 @@
+require 'csv'
+
+require_relative 'dataset'
+
+module Datasets
+  class Wine < Dataset
+    Record = Struct.new(:label,
+                        :alcohol,
+                        :malic_acid,
+                        :ash,
+                        :alcalinity_of_ash,
+                        :n_magnesiums,
+                        :total_phenols,
+                        :total_flavonoids,
+                        :total_nonflavanoid_phenols,
+                        :total_proanthocyanins,
+                        :color_intensity,
+                        :hue,
+                        :optical_nucleic_acid_concentration,
+                        :n_prolines)
+
+    def initialize
+      super
+      @metadata.id = 'wine'
+      @metadata.name = 'Wine'
+      @metadata.url = 'http://archive.ics.uci.edu/ml/datasets/wine'
+      @metadata.description = -> { read_names }
+    end
+
+    def each
+      return to_enum(__method__) unless block_given?
+
+      open_data do |csv|
+        csv.each do |row|
+          next if row[0].nil?
+          record = Record.new(*row)
+          yield(record)
+        end
+      end
+    end
+
+    private
+
+    def read_names
+      names_path = cache_dir_path + 'wine.names'
+      unless names_path.exist?
+        names_url = 'http://archive.ics.uci.edu/ml/machine-learning-databases/wine/wine.names'
+        download(names_path, names_url)
+      end
+      names_path.read
+    end
+
+    def open_data
+      data_path = cache_dir_path + 'wine.data'
+      unless data_path.exist?
+        data_url = 'http://archive.ics.uci.edu/ml/machine-learning-databases/wine/wine.data'
+        download(data_path, data_url)
+      end
+      CSV.open(data_path, converters: %i[numeric]) do |csv|
+        yield(csv)
+      end
+    end
+  end
+end

data/red-datasets.gemspec

@@ -34,6 +34,10 @@ Gem::Specification.new do |spec|
   spec.files += Dir.glob("doc/text/*")
   spec.test_files += Dir.glob("test/**/*")
 
+  spec.add_runtime_dependency("csv", ">= 3.0.5")
+  spec.add_runtime_dependency("rexml")
+  spec.add_runtime_dependency("rubyzip")
+
   spec.add_development_dependency("bundler")
   spec.add_development_dependency("rake")
   spec.add_development_dependency("test-unit")

data/test/helper.rb

@@ -1,5 +1,6 @@
 require "fileutils"
 require "pathname"
+require "time"
 
 require "datasets"
 

data/test/run-test.rb

@@ -13,4 +13,6 @@ $LOAD_PATH.unshift(lib_dir.to_s)
 
 require_relative "helper"
 
+ARGV.unshift("--max-diff-target-string-size=#{10 * 1024}")
+
 exit(Test::Unit::AutoRunner.run(true, test_dir.to_s))

data/test/test-adult.rb

@@ -0,0 +1,126 @@
+class AdultTest < Test::Unit::TestCase
+  sub_test_case("train") do
+    def setup
+      @dataset = Datasets::Adult.new(type: :train)
+    end
+
+    def record(*args)
+      Datasets::Adult::Record.new(*args)
+    end
+
+    test("#each") do
+      records = @dataset.each.to_a
+      assert_equal([
+                     32561,
+                     {
+                       :age => 39,
+                       :work_class => "State-gov",
+                       :final_weight => 77516,
+                       :education => "Bachelors",
+                       :n_education_years => 13,
+                       :marital_status => "Never-married",
+                       :occupation => "Adm-clerical",
+                       :relationship => "Not-in-family",
+                       :race => "White",
+                       :sex => "Male",
+                       :capital_gain => 2174,
+                       :capital_loss => 0,
+                       :hours_per_week => 40,
+                       :native_country => "United-States",
+                       :label => "<=50K"
+                     },
+                     {
+                       :age => 52,
+                       :work_class => "Self-emp-inc",
+                       :final_weight => 287927,
+                       :education => "HS-grad",
+                       :n_education_years => 9,
+                       :marital_status => "Married-civ-spouse",
+                       :occupation => "Exec-managerial",
+                       :relationship => "Wife",
+                       :race => "White",
+                       :sex => "Female",
+                       :capital_gain => 15024,
+                       :capital_loss => 0,
+                       :hours_per_week => 40,
+                       :native_country => "United-States",
+                       :label => ">50K"
+                     }
+                   ],
+                   [
+                     records.size,
+                     records[0].to_h,
+                     records[-1].to_h
+                   ])
+    end
+  end
+
+  sub_test_case("test") do
+    def setup
+      @dataset = Datasets::Adult.new(type: :test)
+    end
+
+    def record(*args)
+      Datasets::Adult::Record.new(*args)
+    end
+
+    test("#each") do
+      records = @dataset.each.to_a
+      assert_equal([
+                     16281,
+                     {
+                       :age => 25,
+                       :work_class => "Private",
+                       :final_weight => 226802,
+                       :education => "11th",
+                       :n_education_years => 7,
+                       :marital_status => "Never-married",
+                       :occupation => "Machine-op-inspct",
+                       :relationship => "Own-child",
+                       :race => "Black",
+                       :sex => "Male",
+                       :capital_gain => 0,
+                       :capital_loss => 0,
+                       :hours_per_week => 40,
+                       :native_country => "United-States",
+                       :label => "<=50K."
+                     },
+                     {
+                       :age => 35,
+                       :work_class => "Self-emp-inc",
+                       :final_weight => 182148,
+                       :education => "Bachelors",
+                       :n_education_years => 13,
+                       :marital_status => "Married-civ-spouse",
+                       :occupation => "Exec-managerial",
+                       :relationship => "Husband",
+                       :race => "White",
+                       :sex => "Male",
+                       :capital_gain => 0,
+                       :capital_loss => 0,
+                       :hours_per_week => 60,
+                       :native_country => "United-States",
+                       :label => ">50K."
+                     }
+                   ],
+                   [
+                     records.size,
+                     records[0].to_h,
+                     records[-1].to_h
+                   ])
+    end
+  end
+
+  sub_test_case("#metadata") do
+    def setup
+      @dataset = Datasets::Adult.new(type: :train)
+    end
+
+    test("#description") do
+      description = @dataset.metadata.description
+      assert do
+        description.start_with?("| This data was extracted from the census bureau database found at")
+      end
+    end
+  end
+end