red-datasets 0.0.8 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c7a9199546e7a001c97e45c6fa28db15c0d96b748e527d9705dfee4e4b1db6fd
-  data.tar.gz: c659f6ae1e658ad91210e4427be063463124d89ef90388d34ebfb73ceb49068a
+  metadata.gz: 07c55b47d31b30ceaf4cdd3ea22da5c737d81884a494c6a11abc6fda6fbea22b
+  data.tar.gz: a28d34b5d28cb57349a81112ffc2db8fe9f94939beb21477af4d9d0c9d5b59ab
 SHA512:
-  metadata.gz: d8a23c4a165a596df22ce5bbe1f8f0cd5c0f002deecafbb26cd5e5f75abb3c0224c1013898162a67787159258d1b801395fc4d949c17939d95940664cffd5600
-  data.tar.gz: f2fd4eb733e6205f138c4005627e815e3787040a8a4b6cce7eca9fd5d4adaa12263e17e8f5bd9394a851e5210f28736ee3c682c81e110da304ae17fb3f0bedba
+  metadata.gz: 4871ac4ec167cb78b3fce8f9c5de9f6cff6a4089b8e4fd87fe7bb3265865cfcbd86935e8f2fa0bc5e40fde8a471e1655390fcdf8dcc0a5197342143e0cb855e5
+  data.tar.gz: 66d31943cb857632518a90166972bfa9ebe4b8ec355eca8291da40183c260e3c175d5866220efc0e4174f780c8449b45004db425a8fc7453268236f9b7dcbc1d

data/README.md

@@ -1,5 +1,8 @@
 # Red Datasets
 
+[![Build Status](https://travis-ci.org/red-data-tools/red-datasets.svg?branch=master)](https://travis-ci.org/red-data-tools/red-datasets)
+[![Gem Version](https://badge.fury.io/rb/red-datasets.svg)](https://badge.fury.io/rb/red-datasets)
+
 ## Description
 
 Red Datasets provides classes that provide common datasets such as iris dataset.
@@ -128,6 +131,9 @@ mnist.each do |record|
 end
 ```
 
+## NArray compatibility
+
+* [red-datasets-numo-narray](https://github.com/red-data-tools/red-datasets-numo-narray)
 
 ## License
 

data/doc/text/news.md

@@ -1,5 +1,98 @@
 # News
 
+## 0.1.3 - 2021-07-09
+
+### Improvements
+
+  * `Datasets::SeabornData`: Added.
+
+  * `Datasets::SudachiSynonymDictionary`: Added.
+
+## 0.1.2 - 2021-06-03
+
+### Improvements
+
+  * `Datasets::Rdatasets` and `Datasets::RdatasetsList`: Added.
+
+  * `Datasets::Penguins`: Changed for compatibility with seaborn's
+    penguins dataset.
+
+## 0.1.1 - 2021-04-11
+
+### Improvements
+
+  * Added support for Ruby 3.0.
+
+  * `Datasets::Communities`: Added.
+    [GitHub#64][Patch by Yasuo Honda]
+
+  * `Datasets::EStatJapan`: Added.
+    [GitHub#90][Patch by Kunihiko Miyoshi]
+
+  * `Datasets::Penguins`: Added.
+    [GitHub#100][Patch by Kenta Murata]
+
+  * `Datasets::CLDRPlurals`: Added.
+
+### Thanks
+
+  * Yasuo Honda
+
+  * Kunihiko Miyoshi
+
+  * Kenta Murata
+
+## 0.1.0 - 2020-02-04
+
+### Improvements
+
+  * Added support for Ruby 2.7.
+    [GitHub#82][GitHub#83][Patch by Yasuo Honda]
+
+  * `Datasets::Hepatitis`: Added.
+    [GitHub#70][Patch by KazuhiroYoshimoto]
+
+  * `Datasets::Downloader`: Added support for query.
+
+### Thanks
+
+  * Yasuo Honda
+
+  * KazuhiroYoshimoto
+
+## 0.0.9 - 2019-09-09
+
+### Improvements
+
+  * `Datasets::LIBSVMDatasetList`: Improved performance.
+
+  * `Datasets::Mushroom`: Added.
+    [GitHub#33][Patch by Yasuo Honda]
+
+  * `Datasets::Table#n_columns`: Added.
+
+  * `Datasets::Table#n_rows`: Added.
+
+  * `Datasets::Table#[]`: Added support for index access.
+
+  * `Datasets::Table#coolumn_names`: Added.
+
+  * `Datasets::Table#size`: Added.
+
+  * `Datasets::Table#length`: Added.
+
+  * `Datasets::Table#each_column`: Added.
+
+  * `Datasets::Table#each_record`: Added.
+
+  * `Datasets::Table#find_record`: Added.
+
+### Thanks
+
+  * Yasuo Honda
+
+### Improvements
+
 ## 0.0.8 - 2019-03-24
 
 ### Improvements

data/lib/datasets.rb

@@ -2,12 +2,21 @@ require_relative "datasets/version"
 
 require_relative "datasets/adult"
 require_relative "datasets/cifar"
+require_relative "datasets/cldr-plurals"
+require_relative "datasets/communities"
+require_relative "datasets/e-stat-japan"
 require_relative "datasets/fashion-mnist"
+require_relative "datasets/hepatitis"
 require_relative "datasets/iris"
 require_relative "datasets/libsvm"
 require_relative "datasets/libsvm-dataset-list"
 require_relative "datasets/mnist"
+require_relative "datasets/mushroom"
+require_relative "datasets/penguins"
 require_relative "datasets/penn-treebank"
 require_relative "datasets/postal-code-japan"
+require_relative "datasets/rdatasets"
+require_relative "datasets/seaborn-data"
+require_relative "datasets/sudachi-synonym-dictionary"
 require_relative "datasets/wikipedia"
 require_relative "datasets/wine"

data/lib/datasets/adult.rb

@@ -62,11 +62,12 @@ module Datasets
         data_url = "http://archive.ics.uci.edu/ml/machine-learning-databases/adult/adult.#{ext}"
         download(data_path, data_url)
       end
-      CSV.open(data_path,
-               {
+
+      options = {
                  converters: [:numeric, lambda {|f| f.strip}],
                  skip_lines: /\A\|/,
-               }) do |csv|
+      }
+      CSV.open(data_path, **options) do |csv|
         yield(csv)
       end
     end

data/lib/datasets/cifar.rb

@@ -1,10 +1,10 @@
-require "rubygems/package"
-require "zlib"
-
+require_relative "tar-gz-readable"
 require_relative "dataset"
 
 module Datasets
   class CIFAR < Dataset
+    include TarGzReadable
+
     module Pixelable
       def pixels
         data.unpack("C*")
@@ -61,7 +61,7 @@ module Datasets
     private
 
     def parse_data(data_path, &block)
-      open_tar(data_path) do |tar|
+      open_tar_gz(data_path) do |tar|
         target_file_names.each do |target_file_name|
           tar.seek(target_file_name) do |entry|
             parse_entry(entry, &block)
@@ -124,14 +124,6 @@ module Datasets
         end
       end
     end
-
-    def open_tar(data_path)
-      Zlib::GzipReader.open(data_path) do |f|
-        Gem::Package::TarReader.new(f) do |tar|
-          yield(tar)
-        end
-      end
-    end
   end
 end
 

data/lib/datasets/cldr-plurals.rb

@@ -0,0 +1,385 @@
+require "rexml/streamlistener"
+require "rexml/parsers/baseparser"
+require "rexml/parsers/streamparser"
+require "strscan"
+
+require_relative "dataset"
+
+module Datasets
+  class CLDRPlurals < Dataset
+    Locale = Struct.new(:name,
+                        :rules)
+
+    Rule = Struct.new(:count,
+                      :condition,
+                      :integer_samples,
+                      :decimal_samples)
+
+    def initialize
+      super()
+      @metadata.id = "cldr-plurals"
+      @metadata.name = "CLDR language plural rules"
+      @metadata.url = "https://raw.githubusercontent.com/unicode-org/cldr/master/common/supplemental/plurals.xml"
+      @metadata.licenses = ["Unicode-DFS-2016"]
+      @metadata.description = <<~DESCRIPTION
+        Language plural rules in Unicode Common Locale Data Repository.
+        See also: https://unicode-org.github.io/cldr-staging/charts/latest/supplemental/language_plural_rules.html
+      DESCRIPTION
+    end
+
+    def each(&block)
+      return to_enum(__method__) unless block_given?
+
+      open_data do |input|
+        catch do |abort_tag|
+          listener = Listener.new(abort_tag, &block)
+          parser = REXML::Parsers::StreamParser.new(input, listener)
+          parser.parse
+        end
+      end
+    end
+
+    private
+    def open_data
+      data_path = cache_dir_path + "plurals.xml"
+      unless data_path.exist?
+        download(data_path, @metadata.url)
+      end
+      ::File.open(data_path) do |input|
+        yield(input)
+      end
+    end
+
+    # Spec: https://unicode.org/reports/tr35/tr35-numbers.html#Language_Plural_Rules
+    class Listener
+      include REXML::StreamListener
+
+      def initialize(abort_tag, &block)
+        @abort_tag = abort_tag
+        @block = block
+        @tag_name_stack = []
+      end
+
+      def tag_start(name, attributes)
+        @tag_name_stack.push(name)
+        case name
+        when "pluralRules"
+          @locales = attributes["locales"].split
+          @rules = []
+        when "pluralRule"
+          @rule = Rule.new(attributes["count"])
+        end
+      end
+
+      def tag_end(name)
+        case name
+        when "pluralRules"
+          @locales.each do |locale_name|
+            @block.call(Locale.new(locale_name, @rules))
+          end
+        when "pluralRule"
+          @rules << @rule
+        end
+        @tag_name_stack.pop
+      end
+
+      def text(data)
+        case @tag_name_stack.last
+        when "pluralRule"
+          parse_plural_rule(data)
+        end
+      end
+
+      private
+      def parse_plural_rule(data)
+        parser = RuleParser.new(@rule, data)
+        parser.parse
+      end
+    end
+    private_constant :Listener
+
+    # Syntax: http://unicode.org/reports/tr35/tr35-numbers.html#Plural_rules_syntax
+    class RuleParser
+      def initialize(rule, data)
+        @rule = rule
+        @data = data
+        @scanner = StringScanner.new(@data)
+      end
+
+      def parse
+        @rule.condition = parse_condition
+        skip_whitespaces
+        if @scanner.scan(/@integer/)
+          @rule.integer_samples = parse_sample_list
+        end
+        skip_whitespaces
+        if @scanner.scan(/@decimal/)
+          @rule.decimal_samples = parse_sample_list
+        end
+      end
+
+      private
+      def skip_whitespaces
+        @scanner.skip(/\p{Pattern_White_Space}+/)
+      end
+
+      def parse_condition
+        and_condition = parse_and_condition
+        return nil if and_condition.nil?
+        and_conditions = [and_condition]
+        while parse_or
+          and_conditions << parse_and_condition
+        end
+        if and_conditions.size == 1
+          and_condition
+        else
+          [:or, *and_conditions]
+        end
+      end
+
+      def parse_or
+        skip_whitespaces
+        @scanner.scan(/or/)
+      end
+
+      def parse_and_condition
+        skip_whitespaces
+        relation = parse_relation
+        return nil if relation.nil?
+        relations = [relation]
+        while parse_and
+          relations << parse_relation
+        end
+        if relations.size == 1
+          relation
+        else
+          [:and, *relations]
+        end
+      end
+
+      def parse_and
+        skip_whitespaces
+        @scanner.scan(/and/)
+      end
+
+      def parse_relation
+        parse_is_relation or
+          parse_in_relation or
+          parse_within_relation
+      end
+
+      def parse_is_relation
+        position = @scanner.pos
+        skip_whitespaces
+        expr = parse_expr
+        unless parse_is
+          @scanner.pos = position
+          return nil
+        end
+        if parse_not
+          operator = :is_not
+        else
+          operator = :is
+        end
+        value = parse_value
+        if value.nil?
+          raise Error, "no value for #{operator}: #{@scanner.inspect}"
+        end
+        [operator, expr, value]
+      end
+
+      def parse_is
+        skip_whitespaces
+        @scanner.scan(/is/)
+      end
+
+      def parse_not
+        skip_whitespaces
+        @scanner.scan(/not/)
+      end
+
+      def parse_in_relation
+        position = @scanner.pos
+        skip_whitespaces
+        expr = parse_expr
+        if parse_not
+          if parse_in
+            operator = :not_in
+          else
+            @scanner.ops = position
+            return nil
+          end
+        elsif parse_in
+          operator = :in
+        elsif parse_equal
+          operator = :equal
+        elsif parse_not_equal
+          operator = :not_equal
+        else
+          @scanner.pos = position
+          return nil
+        end
+        range_list = parse_range_list
+        [operator, expr, range_list]
+      end
+
+      def parse_in
+        skip_whitespaces
+        @scanner.scan(/in/)
+      end
+
+      def parse_equal
+        skip_whitespaces
+        @scanner.scan(/=/)
+      end
+
+      def parse_not_equal
+        skip_whitespaces
+        @scanner.scan(/!=/)
+      end
+
+      def parse_within_relation
+        position = @scanner.pos
+        skip_whitespaces
+        expr = parse_expr
+        have_not = parse_not
+        unless parse_within
+          @scanner.pos = position
+          return nil
+        end
+        if have_not
+          operator = :not_within
+        else
+          operator = :within
+        end
+        range_list = parse_range_list
+        [operator, expr, range_list]
+      end
+
+      def parse_within
+        skip_whitespaces
+        @scanner.scan(/within/)
+      end
+
+      def parse_expr
+        operand = parse_operand
+        operator = parse_expr_operator
+        if operator
+          value = parse_value
+          if value.nil?
+            raise Error, "no value for #{operator}: #{@scanner.inspect}"
+          end
+          [operator, operand, value]
+        else
+          operand
+        end
+      end
+
+      def parse_operand
+        skip_whitespaces
+        @scanner.scan(/[niftvwce]/)
+      end
+
+      def parse_expr_operator
+        skip_whitespaces
+        if @scanner.scan(/(?:mod|%)/)
+          :mod
+        else
+          nil
+        end
+      end
+
+      def parse_range_list
+        ranges = [parse_range || parse_value]
+        loop do
+          skip_whitespaces
+          break unless @scanner.scan(/,/)
+          ranges << (parse_range || parse_value)
+        end
+        ranges
+      end
+
+      def parse_range
+        position = @scanner.pos
+        range_start = parse_value
+        skip_whitespaces
+        unless @scanner.scan(/\.\./)
+          @scanner.pos = position
+          return nil
+        end
+        range_end = parse_value
+        range_start..range_end
+      end
+
+      def parse_value
+        skip_whitespaces
+        value = @scanner.scan(/\d+/)
+        return nil if value.nil?
+        Integer(value, 10)
+      end
+
+      def parse_sample_list
+        samples = [parse_sample_range]
+        loop do
+          position = @scanner.pos
+          skip_whitespaces
+          break unless @scanner.scan(/,/)
+          sample_range = parse_sample_range
+          unless sample_range
+            @scanner.pos = position
+            break
+          end
+          samples << sample_range
+        end
+        skip_whitespaces
+        if @scanner.scan(/,/)
+          skip_whitespaces
+          # U+2026 HORIZONTAL ELLIPSIS
+          unless @scanner.scan(/\u2026|\.\.\./)
+            raise Error, "no ellipsis: #{@scanner.inspect}"
+          end
+          samples << :elipsis
+        end
+        samples
+      end
+
+      def parse_sample_range
+        value = parse_sample_value
+        return nil if value.nil?
+        skip_whitespaces
+        if @scanner.scan(/~/)
+          range_end = parse_sample_value
+          value..range_end
+        else
+          value
+        end
+      end
+
+      def parse_sample_value
+        value = parse_value
+        return nil if value.nil?
+        if @scanner.scan(/\./)
+          skip_whitespaces
+          decimal = @scanner.scan(/[0-9]+/)
+          if decimal.nil?
+            raise Error, "no decimal: #{@scanner.inspect}"
+          end
+          value += Float("0.#{decimal}")
+          skip_whitespaces
+        end
+        if @scanner.scan(/[ce]/)
+          # Workardoun for a spec bug. "e1" should be accepted.
+          #
+          # Spec:
+          #   sampleValue     = value ('.' digit+)? ([ce] digitPos digit+)?
+          #   digit           = [0-9]
+          #   digitPos        = [1-9]
+          e = @scanner.scan(/[1-9][0-9]*/)
+          value *= 10 * Integer(e, 10)
+        end
+        value
+      end
+    end
+    private_constant :RuleParser
+  end
+end