red-datasets 0.0.7 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 222271b814e3a5ce23b5e0dd1d2578bffb84afdab10110b0869985c6056bfd3b
-  data.tar.gz: ac30931b3317ab04afd394b28a45a9206c784d78b3bcaf98fc3a2a48227c7930
+  metadata.gz: 6fbd4d11063f89ba2e09250b751886086c953ec8bc92c75a6a351c31a36da0c4
+  data.tar.gz: acc6ff31f0f4ae3a6c6565fe569233c01615718c01300b0838ff744571edc34d
 SHA512:
-  metadata.gz: 8a94a3d66baaed4948904e97dc53100d73ae96c528c09b02252caabd05b8545587abf6fbcba3a578725812327a9a2c8827bbb7e283ccd3d7e66753bf30035e2e
-  data.tar.gz: 2ab44b5aa3ee5da0ac8e8307546c71942938de4497bfec05fc929715a4e5ef6df1cb091bce0d5f12978582d2c9fa7eaffff9edd54be0d845627dccfce42a63dd
+  metadata.gz: 26361511155b447ffed56a79b2336a9a1db96494bf856b23e7b39cc6a8b6a2039e7ed27564140761bdb2daaae7ee563b3695c464a7a7b21ff93b0636f6b8338d
+  data.tar.gz: 40446f90e410e0d86abeec186a1d7adcc5375e29c19dc934f823befb26a87d904458ef5ea18c9d64055493d29ed305dba53d6e4d86bd7d84488baf3745ebd792

data/README.md

@@ -1,8 +1,7 @@
-# README
+# Red Datasets
 
-## Name
-
-Red Datasets
+[![Build Status](https://travis-ci.org/red-data-tools/red-datasets.svg?branch=master)](https://travis-ci.org/red-data-tools/red-datasets)
+[![Gem Version](https://badge.fury.io/rb/red-datasets.svg)](https://badge.fury.io/rb/red-datasets)
 
 ## Description
 
@@ -16,6 +15,20 @@ You can use datasets easily because you can access each dataset with multiple wa
 % gem install red-datasets
 ```
 
+## Available datasets
+
+TODO: Document them in source code to list in document: https://www.rubydoc.info/gems/red-datasets
+
+* Adult Dataset
+* CIFAR-10 Dataset
+* CIFAR-100 Dataset
+* Fashion-MNIST
+* Iris Dataset
+* MNIST database
+* The Penn Treebank Project
+* Wikipedia
+* Wine Dataset
+
 ## Usage
 
 Here is an example to access [Iris Data Set](https://archive.ics.uci.edu/ml/datasets/iris) by `#each`  or `Table#to_h` or `Table#fetch_values`.
@@ -118,6 +131,9 @@ mnist.each do |record|
 end
 ```
 
+## NArray compatibility
+
+* [red-datasets-numo-narray](https://github.com/red-data-tools/red-datasets-numo-narray)
 
 ## License
 

data/doc/text/news.md

@@ -1,5 +1,107 @@
 # News
 
+## 0.1.2 - 2021-06-03
+
+### Improvements
+
+  * `Datasets::Rdatasets` and `Datasets::RdatasetsList`: Added.
+
+  * `Datasets::Penguins`: Changed for compatibility with seaborn's
+    penguins dataset.
+
+## 0.1.1 - 2021-04-11
+
+### Improvements
+
+  * Added support for Ruby 3.0.
+
+  * `Datasets::Communities`: Added.
+    [GitHub#64][Patch by Yasuo Honda]
+
+  * `Datasets::EStatJapan`: Added.
+    [GitHub#90][Patch by Kunihiko Miyoshi]
+
+  * `Datasets::Penguins`: Added.
+    [GitHub#100][Patch by Kenta Murata]
+
+  * `Datasets::CLDRPlurals`: Added.
+
+### Thanks
+
+  * Yasuo Honda
+
+  * Kunihiko Miyoshi
+
+  * Kenta Murata
+
+## 0.1.0 - 2020-02-04
+
+### Improvements
+
+  * Added support for Ruby 2.7.
+    [GitHub#82][GitHub#83][Patch by Yasuo Honda]
+
+  * `Datasets::Hepatitis`: Added.
+    [GitHub#70][Patch by KazuhiroYoshimoto]
+
+  * `Datasets::Downloader`: Added support for query.
+
+### Thanks
+
+  * Yasuo Honda
+
+  * KazuhiroYoshimoto
+
+## 0.0.9 - 2019-09-09
+
+### Improvements
+
+  * `Datasets::LIBSVMDatasetList`: Improved performance.
+
+  * `Datasets::Mushroom`: Added.
+    [GitHub#33][Patch by Yasuo Honda]
+
+  * `Datasets::Table#n_columns`: Added.
+
+  * `Datasets::Table#n_rows`: Added.
+
+  * `Datasets::Table#[]`: Added support for index access.
+
+  * `Datasets::Table#coolumn_names`: Added.
+
+  * `Datasets::Table#size`: Added.
+
+  * `Datasets::Table#length`: Added.
+
+  * `Datasets::Table#each_column`: Added.
+
+  * `Datasets::Table#each_record`: Added.
+
+  * `Datasets::Table#find_record`: Added.
+
+### Thanks
+
+  * Yasuo Honda
+
+### Improvements
+
+## 0.0.8 - 2019-03-24
+
+### Improvements
+
+  * Improved README.
+    [GitHub#40][Patch by kojix2]
+
+  * `Datasets::PostalCodeJapan`: Added.
+
+  * `Datasets::LIBSVMDatasetList`: Added.
+
+  * `Datasets::LIBSVM`: Added.
+
+### Thanks
+
+  * kojix2
+
 ## 0.0.7 - 2018-11-21
 
 ### Improvements

data/lib/datasets.rb

@@ -1,10 +1,20 @@
-require "datasets/version"
+require_relative "datasets/version"
 
-require "datasets/adult"
-require "datasets/cifar"
-require "datasets/fashion-mnist"
-require "datasets/iris"
-require "datasets/mnist"
-require "datasets/penn-treebank"
-require "datasets/wikipedia"
-require "datasets/wine"
+require_relative "datasets/adult"
+require_relative "datasets/cifar"
+require_relative "datasets/cldr-plurals"
+require_relative "datasets/communities"
+require_relative "datasets/e-stat-japan"
+require_relative "datasets/fashion-mnist"
+require_relative "datasets/hepatitis"
+require_relative "datasets/iris"
+require_relative "datasets/libsvm"
+require_relative "datasets/libsvm-dataset-list"
+require_relative "datasets/mnist"
+require_relative "datasets/mushroom"
+require_relative "datasets/penguins"
+require_relative "datasets/penn-treebank"
+require_relative "datasets/postal-code-japan"
+require_relative "datasets/rdatasets"
+require_relative "datasets/wikipedia"
+require_relative "datasets/wine"

data/lib/datasets/adult.rb

@@ -62,11 +62,12 @@ module Datasets
         data_url = "http://archive.ics.uci.edu/ml/machine-learning-databases/adult/adult.#{ext}"
         download(data_path, data_url)
       end
-      CSV.open(data_path,
-               {
+
+      options = {
                  converters: [:numeric, lambda {|f| f.strip}],
                  skip_lines: /\A\|/,
-               }) do |csv|
+      }
+      CSV.open(data_path, **options) do |csv|
         yield(csv)
       end
     end

data/lib/datasets/cifar.rb

@@ -1,10 +1,10 @@
-require "rubygems/package"
-require "zlib"
-
+require_relative "tar_gz_readable"
 require_relative "dataset"
 
 module Datasets
   class CIFAR < Dataset
+    include TarGzReadable
+
     module Pixelable
       def pixels
         data.unpack("C*")
@@ -61,7 +61,7 @@ module Datasets
     private
 
     def parse_data(data_path, &block)
-      open_tar(data_path) do |tar|
+      open_tar_gz(data_path) do |tar|
         target_file_names.each do |target_file_name|
           tar.seek(target_file_name) do |entry|
             parse_entry(entry, &block)
@@ -124,14 +124,6 @@ module Datasets
         end
       end
     end
-
-    def open_tar(data_path)
-      Zlib::GzipReader.open(data_path) do |f|
-        Gem::Package::TarReader.new(f) do |tar|
-          yield(tar)
-        end
-      end
-    end
   end
 end
 

data/lib/datasets/cldr-plurals.rb

@@ -0,0 +1,385 @@
+require "rexml/streamlistener"
+require "rexml/parsers/baseparser"
+require "rexml/parsers/streamparser"
+require "strscan"
+
+require_relative "dataset"
+
+module Datasets
+  class CLDRPlurals < Dataset
+    Locale = Struct.new(:name,
+                        :rules)
+
+    Rule = Struct.new(:count,
+                      :condition,
+                      :integer_samples,
+                      :decimal_samples)
+
+    def initialize
+      super()
+      @metadata.id = "cldr-plurals"
+      @metadata.name = "CLDR language plural rules"
+      @metadata.url = "https://raw.githubusercontent.com/unicode-org/cldr/master/common/supplemental/plurals.xml"
+      @metadata.licenses = ["Unicode-DFS-2016"]
+      @metadata.description = <<~DESCRIPTION
+        Language plural rules in Unicode Common Locale Data Repository.
+        See also: https://unicode-org.github.io/cldr-staging/charts/latest/supplemental/language_plural_rules.html
+      DESCRIPTION
+    end
+
+    def each(&block)
+      return to_enum(__method__) unless block_given?
+
+      open_data do |input|
+        catch do |abort_tag|
+          listener = Listener.new(abort_tag, &block)
+          parser = REXML::Parsers::StreamParser.new(input, listener)
+          parser.parse
+        end
+      end
+    end
+
+    private
+    def open_data
+      data_path = cache_dir_path + "plurals.xml"
+      unless data_path.exist?
+        download(data_path, @metadata.url)
+      end
+      ::File.open(data_path) do |input|
+        yield(input)
+      end
+    end
+
+    # Spec: https://unicode.org/reports/tr35/tr35-numbers.html#Language_Plural_Rules
+    class Listener
+      include REXML::StreamListener
+
+      def initialize(abort_tag, &block)
+        @abort_tag = abort_tag
+        @block = block
+        @tag_name_stack = []
+      end
+
+      def tag_start(name, attributes)
+        @tag_name_stack.push(name)
+        case name
+        when "pluralRules"
+          @locales = attributes["locales"].split
+          @rules = []
+        when "pluralRule"
+          @rule = Rule.new(attributes["count"])
+        end
+      end
+
+      def tag_end(name)
+        case name
+        when "pluralRules"
+          @locales.each do |locale_name|
+            @block.call(Locale.new(locale_name, @rules))
+          end
+        when "pluralRule"
+          @rules << @rule
+        end
+        @tag_name_stack.pop
+      end
+
+      def text(data)
+        case @tag_name_stack.last
+        when "pluralRule"
+          parse_plural_rule(data)
+        end
+      end
+
+      private
+      def parse_plural_rule(data)
+        parser = RuleParser.new(@rule, data)
+        parser.parse
+      end
+    end
+    private_constant :Listener
+
+    # Syntax: http://unicode.org/reports/tr35/tr35-numbers.html#Plural_rules_syntax
+    class RuleParser
+      def initialize(rule, data)
+        @rule = rule
+        @data = data
+        @scanner = StringScanner.new(@data)
+      end
+
+      def parse
+        @rule.condition = parse_condition
+        skip_whitespaces
+        if @scanner.scan(/@integer/)
+          @rule.integer_samples = parse_sample_list
+        end
+        skip_whitespaces
+        if @scanner.scan(/@decimal/)
+          @rule.decimal_samples = parse_sample_list
+        end
+      end
+
+      private
+      def skip_whitespaces
+        @scanner.skip(/\p{Pattern_White_Space}+/)
+      end
+
+      def parse_condition
+        and_condition = parse_and_condition
+        return nil if and_condition.nil?
+        and_conditions = [and_condition]
+        while parse_or
+          and_conditions << parse_and_condition
+        end
+        if and_conditions.size == 1
+          and_condition
+        else
+          [:or, *and_conditions]
+        end
+      end
+
+      def parse_or
+        skip_whitespaces
+        @scanner.scan(/or/)
+      end
+
+      def parse_and_condition
+        skip_whitespaces
+        relation = parse_relation
+        return nil if relation.nil?
+        relations = [relation]
+        while parse_and
+          relations << parse_relation
+        end
+        if relations.size == 1
+          relation
+        else
+          [:and, *relations]
+        end
+      end
+
+      def parse_and
+        skip_whitespaces
+        @scanner.scan(/and/)
+      end
+
+      def parse_relation
+        parse_is_relation or
+          parse_in_relation or
+          parse_within_relation
+      end
+
+      def parse_is_relation
+        position = @scanner.pos
+        skip_whitespaces
+        expr = parse_expr
+        unless parse_is
+          @scanner.pos = position
+          return nil
+        end
+        if parse_not
+          operator = :is_not
+        else
+          operator = :is
+        end
+        value = parse_value
+        if value.nil?
+          raise Error.new("no value for #{operator}: #{@scanner.inspect}")
+        end
+        [operator, expr, value]
+      end
+
+      def parse_is
+        skip_whitespaces
+        @scanner.scan(/is/)
+      end
+
+      def parse_not
+        skip_whitespaces
+        @scanner.scan(/not/)
+      end
+
+      def parse_in_relation
+        position = @scanner.pos
+        skip_whitespaces
+        expr = parse_expr
+        if parse_not
+          if parse_in
+            operator = :not_in
+          else
+            @scanner.ops = position
+            return nil
+          end
+        elsif parse_in
+          operator = :in
+        elsif parse_equal
+          operator = :equal
+        elsif parse_not_equal
+          operator = :not_equal
+        else
+          @scanner.pos = position
+          return nil
+        end
+        range_list = parse_range_list
+        [operator, expr, range_list]
+      end
+
+      def parse_in
+        skip_whitespaces
+        @scanner.scan(/in/)
+      end
+
+      def parse_equal
+        skip_whitespaces
+        @scanner.scan(/=/)
+      end
+
+      def parse_not_equal
+        skip_whitespaces
+        @scanner.scan(/!=/)
+      end
+
+      def parse_within_relation
+        position = @scanner.pos
+        skip_whitespaces
+        expr = parse_expr
+        have_not = parse_not
+        unless parse_within
+          @scanner.pos = position
+          return nil
+        end
+        if have_not
+          operator = :not_within
+        else
+          operator = :within
+        end
+        range_list = parse_range_list
+        [operator, expr, range_list]
+      end
+
+      def parse_within
+        skip_whitespaces
+        @scanner.scan(/within/)
+      end
+
+      def parse_expr
+        operand = parse_operand
+        operator = parse_expr_operator
+        if operator
+          value = parse_value
+          if value.nil?
+            raise Error.new("no value for #{operator}: #{@scanner.inspect}")
+          end
+          [operator, operand, value]
+        else
+          operand
+        end
+      end
+
+      def parse_operand
+        skip_whitespaces
+        @scanner.scan(/[niftvwce]/)
+      end
+
+      def parse_expr_operator
+        skip_whitespaces
+        if @scanner.scan(/(?:mod|%)/)
+          :mod
+        else
+          nil
+        end
+      end
+
+      def parse_range_list
+        ranges = [parse_range || parse_value]
+        loop do
+          skip_whitespaces
+          break unless @scanner.scan(/,/)
+          ranges << (parse_range || parse_value)
+        end
+        ranges
+      end
+
+      def parse_range
+        position = @scanner.pos
+        range_start = parse_value
+        skip_whitespaces
+        unless @scanner.scan(/\.\./)
+          @scanner.pos = position
+          return nil
+        end
+        range_end = parse_value
+        range_start..range_end
+      end
+
+      def parse_value
+        skip_whitespaces
+        value = @scanner.scan(/\d+/)
+        return nil if value.nil?
+        Integer(value, 10)
+      end
+
+      def parse_sample_list
+        samples = [parse_sample_range]
+        loop do
+          position = @scanner.pos
+          skip_whitespaces
+          break unless @scanner.scan(/,/)
+          sample_range = parse_sample_range
+          unless sample_range
+            @scanner.pos = position
+            break
+          end
+          samples << sample_range
+        end
+        skip_whitespaces
+        if @scanner.scan(/,/)
+          skip_whitespaces
+          # U+2026 HORIZONTAL ELLIPSIS
+          unless @scanner.scan(/\u2026|\.\.\./)
+            raise "no ellipsis: #{@scanner.inspect}"
+          end
+          samples << :elipsis
+        end
+        samples
+      end
+
+      def parse_sample_range
+        value = parse_sample_value
+        return nil if value.nil?
+        skip_whitespaces
+        if @scanner.scan(/~/)
+          range_end = parse_sample_value
+          value..range_end
+        else
+          value
+        end
+      end
+
+      def parse_sample_value
+        value = parse_value
+        return nil if value.nil?
+        if @scanner.scan(/\./)
+          skip_whitespaces
+          decimal = @scanner.scan(/[0-9]+/)
+          if decimal.nil?
+            raise "no decimal: #{@scanner.inspect}"
+          end
+          value += Float("0.#{decimal}")
+          skip_whitespaces
+        end
+        if @scanner.scan(/[ce]/)
+          # Workardoun for a spec bug. "e1" should be accepted.
+          #
+          # Spec:
+          #   sampleValue     = value ('.' digit+)? ([ce] digitPos digit+)?
+          #   digit           = [0-9]
+          #   digitPos        = [1-9]
+          e = @scanner.scan(/[1-9][0-9]*/)
+          value *= 10 * Integer(e, 10)
+        end
+        value
+      end
+    end
+    private_constant :RuleParser
+  end
+end