red-datasets 0.0.7 → 0.1.2

data/lib/datasets/postal-code-japan.rb

@@ -0,0 +1,154 @@
+require "csv"
+require "zip"
+
+require_relative "dataset"
+
+module Datasets
+  class PostalCodeJapan < Dataset
+    class Record < Struct.new(:organization_code,
+                              :old_postal_code,
+                              :postal_code,
+                              :prefecture_reading,
+                              :city_reading,
+                              :address_reading,
+                              :prefecture,
+                              :city,
+                              :address,
+                              :have_multiple_postal_codes,
+                              :have_address_number_per_koaza,
+                              :have_chome,
+                              :postal_code_is_shared,
+                              :changed,
+                              :change_reason)
+      alias_method :have_multiple_postal_codes?,
+                   :have_multiple_postal_codes
+      alias_method :have_address_number_per_koaza?,
+                   :have_address_number_per_koaza
+      alias_method :have_chome?,
+                   :have_chome
+      alias_method :postal_code_is_shared?,
+                   :postal_code_is_shared
+      alias_method :changed?,
+                   :changed
+    end
+
+    VALID_READINGS = [
+      :lowercase,
+      :uppercase,
+      :romaji,
+    ]
+    def initialize(reading: :lowercase)
+      super()
+      @reading = reading
+      unless VALID_READINGS.include?(@reading)
+        message = ":reading must be one of ["
+        message << VALID_READINGS.collect(&:inspect).join(", ")
+        message << "]: #{@reading.inspect}"
+        raise ArgumentError, message
+      end
+      @metadata.id = "postal-code-japan-#{@reading}"
+      @metadata.name = "Postal code in Japan (#{@reading})"
+      @metadata.url = "https://www.post.japanpost.jp/zipcode/download.html"
+      @metadata.licenses = [
+        "CC0-1.0",
+      ]
+      @metadata.description = "Postal code in Japan (reading: #{@reading})"
+    end
+
+    def each(&block)
+      return to_enum(__method__) unless block_given?
+
+      open_data do |input|
+        utf8_data = input.read.encode(Encoding::UTF_8, Encoding::CP932)
+        options = {
+          quote_char: nil,
+          strip: %Q["],
+        }
+        if @reading == :romaji
+          CSV.parse(utf8_data, **options) do |row|
+            yield(Record.new(nil,
+                             nil,
+                             row[0],
+                             row[4],
+                             row[5],
+                             row[6],
+                             row[1],
+                             row[2],
+                             row[3],
+                             false,
+                             false,
+                             false,
+                             false,
+                             false,
+                             nil))
+          end
+        else
+          CSV.parse(utf8_data, **options) do |row|
+            yield(Record.new(row[0],
+                             row[1].rstrip,
+                             row[2],
+                             row[3],
+                             row[4],
+                             row[5],
+                             row[6],
+                             row[7],
+                             row[8],
+                             (row[9] == "1"),
+                             (row[10] == "1"),
+                             (row[11] == "1"),
+                             (row[12] == "1"),
+                             (row[13] != "0"),
+                             convert_change_reason(row[14])))
+          end
+        end
+      end
+    end
+
+    private
+    def open_data
+      data_url = "https://www.post.japanpost.jp/zipcode/dl"
+      case @reading
+      when :lowercase
+        data_url << "/kogaki/zip/ken_all.zip"
+      when :uppercase
+        data_url << "/oogaki/zip/ken_all.zip"
+      when :romaji
+        data_url << "/roman/ken_all_rome.zip"
+      end
+      data_path = cache_dir_path + "#{@reading}-ken-all.zip"
+      unless data_path.exist?
+        download(data_path, data_url)
+      end
+
+      Zip::File.open(data_path.to_s) do |zip_file|
+        zip_file.each do |entry|
+          next unless entry.file?
+          entry.get_input_stream do |input|
+            yield(input)
+          end
+        end
+      end
+    end
+
+    def convert_change_reason(reason)
+      case reason
+      when "0"
+        nil
+      when "1"
+        :new
+      when "2"
+        :japanese_addressing_system
+      when "3"
+        :land_readjustment
+      when "4"
+        :postal_district_adjustment
+      when "5"
+        :correction
+      when "6"
+        :deletion
+      else
+        :unknown
+      end
+    end
+  end
+end

data/lib/datasets/rdatasets.rb

@@ -0,0 +1,95 @@
+require_relative "dataset"
+require_relative "tar_gz_readable"
+
+module Datasets
+  class RdatasetsList < Dataset
+    Record = Struct.new(:package,
+                        :dataset,
+                        :title,
+                        :rows,
+                        :cols,
+                        :n_binary,
+                        :n_character,
+                        :n_factor,
+                        :n_logical,
+                        :n_numeric,
+                        :csv,
+                        :doc)
+
+    def initialize
+      super
+      @metadata.id = "rdatasets"
+      @metadata.name = "Rdatasets"
+      @metadata.url = "https://vincentarelbundock.github.io/Rdatasets/"
+      @metadata.licenses = ["GPL-3"]
+      @data_url = "https://raw.githubusercontent.com/vincentarelbundock/Rdatasets/master/datasets.csv"
+      @data_path = cache_dir_path + "datasets.csv"
+    end
+
+    def filter(package: nil, dataset: nil)
+      return to_enum(__method__, package: package, dataset: dataset) unless block_given?
+
+      conds = {}
+      conds["Package"] = package if package
+      conds["Item"]    = dataset if dataset
+      if conds.empty?
+        each_row {|row| yield Record.new(*row.fields) }
+      else
+        each_row do |row|
+          if conds.all? {|k, v| row[k] == v }
+            yield Record.new(*row.fields)
+          end
+        end
+      end
+    end
+
+    def each(&block)
+      filter(&block)
+    end
+
+    private def each_row(&block)
+      download(@data_path, @data_url) unless @data_path.exist?
+      CSV.open(@data_path, headers: :first_row, converters: :all) do |csv|
+        csv.each(&block)
+      end
+    end
+  end
+
+  class Rdatasets < Dataset
+    def initialize(package_name, dataset_name)
+      list = RdatasetsList.new
+
+      info = list.filter(package: package_name, dataset: dataset_name).first
+      unless info
+        raise ArgumentError, "Unable to locate dataset #{package_name}/#{dataset_name}"
+      end
+
+      super()
+      @metadata.id = "rdatasets-#{package_name}-#{dataset_name}"
+      @metadata.name = "Rdatasets: #{package_name}: #{dataset_name}"
+      @metadata.url = info.csv
+      @metadata.licenses = ["GPL-3"]
+      @metadata.description = info.title
+
+      # Follow the original directory structure in the cache directory
+      @data_path = cache_dir_path + (dataset_name + ".csv")
+
+      @package_name = package_name
+      @dataset_name = dataset_name
+    end
+
+    def each(&block)
+      return to_enum(__method__) unless block_given?
+
+      download(@data_path, @metadata.url) unless @data_path.exist?
+      CSV.open(@data_path, headers: :first_row, converters: :all) do |csv|
+        csv.each do |row|
+          record = row.to_h
+          record.delete("")
+          record.transform_keys!(&:to_sym)
+          yield record
+        end
+      end
+    end
+  end
+end

data/lib/datasets/table.rb

@@ -2,19 +2,99 @@ require "datasets/dictionary"
 
 module Datasets
   class Table
+    class Record
+      include Enumerable
+
+      def initialize(table, index)
+        @table = table
+        @index = index
+      end
+
+      def [](column_name_or_column_index)
+        @table[column_name_or_column_index][@index]
+      end
+
+      def each
+        return to_enum(__method__) unless block_given?
+        @table.each_column.each do |column_name, column_values|
+          yield(column_name, column_values[@index])
+        end
+      end
+
+      def values
+        @table.each_column.collect do |_column_name, column_values|
+          column_values[@index]
+        end
+      end
+
+      def to_h
+        hash = {}
+        each do |column_name, column_value|
+          hash[column_name] = column_value
+        end
+        hash
+      end
+
+      def inspect
+        "#<#{self.class.name} #{@table.dataset.metadata.name}[#{@index}] #{to_h.inspect}>"
+      end
+    end
+
     include Enumerable
 
+    attr_reader :dataset
     def initialize(dataset)
       @dataset = dataset
       @dictionaries = {}
     end
 
-    def each(&block)
+    def n_columns
+      columner_data.size
+    end
+    alias_method :size, :n_columns
+    alias_method :length, :n_columns
+
+    def n_rows
+      first_column = columner_data.first
+      return 0 if first_column.nil?
+      first_column[1].size
+    end
+
+    def column_names
+      columner_data.keys
+    end
+
+    def each_column(&block)
       columner_data.each(&block)
     end
+    alias_method :each, :each_column
 
-    def [](name)
-      columner_data[normalize_name(name)]
+    def each_record
+      return to_enum(__method__) unless block_given?
+      n_rows.times do |i|
+        yield(Record.new(self, i))
+      end
+    end
+
+    def find_record(row)
+      row += n_rows if row < 0
+      return nil if row < 0
+      return nil if row >= n_rows
+      Record.new(self, row)
+    end
+
+    def [](name_or_index)
+      case name_or_index
+      when Integer
+        index = name_or_index
+        columner_data.each_with_index do |(_name, values), i|
+          return values if i == index
+        end
+        nil
+      else
+        name = name_or_index
+        columner_data[normalize_name(name)]
+      end
     end
 
     def dictionary_encode(name)

data/lib/datasets/tar_gz_readable.rb

@@ -0,0 +1,14 @@
+require "rubygems/package"
+require "zlib"
+
+module Datasets
+  module TarGzReadable
+    def open_tar_gz(data_path)
+      Zlib::GzipReader.open(data_path) do |f|
+        Gem::Package::TarReader.new(f) do |tar|
+          yield(tar)
+        end
+      end
+    end
+  end
+end

data/lib/datasets/version.rb

@@ -1,3 +1,3 @@
 module Datasets
-  VERSION = "0.0.7"
+  VERSION = "0.1.2"
 end

data/lib/datasets/wikipedia.rb

@@ -52,7 +52,7 @@ module Datasets
     end
 
     private
-    def open_data
+    def open_data(&block)
       base_name = "#{@language}wiki-latest-#{type_in_path}.xml.bz2"
       data_path = cache_dir_path + base_name
       unless data_path.exist?
@@ -60,15 +60,7 @@ module Datasets
         download(data_path, data_url)
       end
 
-      input, output = IO.pipe
-      pid = spawn("bzcat", data_path.to_s, {:out => output})
-      begin
-        output.close
-        yield(input)
-      ensure
-        input.close
-        Process.waitpid(pid)
-      end
+      extract_bz2(data_path, &block)
     end
 
     def type_in_path

data/red-datasets.gemspec

@@ -34,6 +34,10 @@ Gem::Specification.new do |spec|
   spec.files += Dir.glob("doc/text/*")
   spec.test_files += Dir.glob("test/**/*")
 
+  spec.add_runtime_dependency("csv", ">= 3.0.5")
+  spec.add_runtime_dependency("rexml")
+  spec.add_runtime_dependency("rubyzip")
+
   spec.add_development_dependency("bundler")
   spec.add_development_dependency("rake")
   spec.add_development_dependency("test-unit")

data/test/run-test.rb

@@ -13,4 +13,6 @@ $LOAD_PATH.unshift(lib_dir.to_s)
 
 require_relative "helper"
 
+ARGV.unshift("--max-diff-target-string-size=#{10 * 1024}")
+
 exit(Test::Unit::AutoRunner.run(true, test_dir.to_s))

data/test/test-cldr-plurals.rb

@@ -0,0 +1,180 @@
+class CLDRPluralsTest < Test::Unit::TestCase
+  def setup
+    @dataset = Datasets::CLDRPlurals.new
+  end
+
+  def locale(*args)
+    Datasets::CLDRPlurals::Locale.new(*args)
+  end
+
+  def rule(*args)
+    Datasets::CLDRPlurals::Rule.new(*args)
+  end
+
+  test("#each") do
+    locales = @dataset.each.to_a
+    assert_equal([
+                   215,
+                   locale("bm",
+                          [
+                            rule("other",
+                                 nil,
+                                 [
+                                   0..15,
+                                   100,
+                                   1000,
+                                   10000,
+                                   100000,
+                                   1000000,
+                                   :elipsis,
+                                 ],
+                                 [
+                                   0.0..1.5,
+                                   10.0,
+                                   100.0,
+                                   1000.0,
+                                   10000.0,
+                                   100000.0,
+                                   1000000.0,
+                                   :elipsis,
+                                 ])
+                          ]),
+                   locale("kw",
+                          [
+                            rule("zero",
+                                 [:equal, "n", [0]],
+                                 [0],
+                                 [0.0, 0.00, 0.000, 0.0000]),
+                            rule("one",
+                                 [:equal, "n", [1]],
+                                 [1],
+                                 [1.0, 1.00, 1.000, 1.0000]),
+                            rule("two",
+                                 [:or,
+                                  [:equal,
+                                   [:mod, "n", 100],
+                                   [2, 22, 42, 62, 82]],
+                                  [:and,
+                                   [:equal, [:mod, "n", 1000], [0]],
+                                   [:equal,
+                                    [:mod, "n", 100000],
+                                    [1000..20000, 40000, 60000, 80000]]],
+                                  [:and,
+                                   [:not_equal, "n", [0]],
+                                   [:equal, [:mod, "n", 1000000], [100000]]]],
+                                 [
+                                   2,
+                                   22,
+                                   42,
+                                   62,
+                                   82,
+                                   102,
+                                   122,
+                                   142,
+                                   1000,
+                                   10000,
+                                   100000,
+                                   :elipsis,
+                                 ],
+                                 [
+                                   2.0,
+                                   22.0,
+                                   42.0,
+                                   62.0,
+                                   82.0,
+                                   102.0,
+                                   122.0,
+                                   142.0,
+                                   1000.0,
+                                   10000.0,
+                                   100000.0,
+                                   :elipsis,
+                                 ]),
+                            rule("few",
+                                 [:equal,
+                                  [:mod, "n", 100],
+                                  [3, 23, 43, 63, 83]],
+                                 [
+                                   3,
+                                   23,
+                                   43,
+                                   63,
+                                   83,
+                                   103,
+                                   123,
+                                   143,
+                                   1003,
+                                   :elipsis,
+                                 ],
+                                 [
+                                   3.0,
+                                   23.0,
+                                   43.0,
+                                   63.0,
+                                   83.0,
+                                   103.0,
+                                   123.0,
+                                   143.0,
+                                   1003.0,
+                                   :elipsis,
+                                 ]),
+                            rule("many",
+                                 [:and,
+                                  [:not_equal, "n", [1]],
+                                  [:equal,
+                                   [:mod, "n", 100],
+                                   [1, 21, 41, 61, 81]]],
+                                 [
+                                   21,
+                                   41,
+                                   61,
+                                   81,
+                                   101,
+                                   121,
+                                   141,
+                                   161,
+                                   1001,
+                                   :elipsis,
+                                 ],
+                                 [
+                                   21.0,
+                                   41.0,
+                                   61.0,
+                                   81.0,
+                                   101.0,
+                                   121.0,
+                                   141.0,
+                                   161.0,
+                                   1001.0,
+                                   :elipsis,
+                                 ]),
+                            rule("other",
+                                 nil,
+                                 [4..19, 100, 1004, 1000000, :elipsis],
+                                 [
+                                   0.1..0.9,
+                                   1.1..1.7,
+                                   10.0,
+                                   100.0,
+                                   1000.1,
+                                   1000000.0,
+                                   :elipsis,
+                                 ]),
+                          ]),
+                 ],
+                 [
+                   locales.size,
+                   locales[0],
+                   locales[-4],
+                 ])
+  end
+
+  sub_test_case("#metadata") do
+    test("#description") do
+      description = @dataset.metadata.description
+      assert do
+        description.start_with?("Language plural rules in Unicode Common Locale Data Repository.")
+      end
+    end
+  end
+end