red-datasets 0.1.0 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 120492172aae9cec1c4fc4f3b73575cb5349caf2f0b67d70676c8896324e1491
-  data.tar.gz: e46eb3f2875cb407e86cc0976eff7d612beb62ca6b421a51435b5d5e1bfa6e03
+  metadata.gz: 8d18fa976f1b368a6a3f9cc85dc7a58a1785fd02901157672484f2a7d8b1fa88
+  data.tar.gz: c91d651a0d8de6722ee759ce29545f5f382d1e9f060c7e4ee5a0fcd557be4d21
 SHA512:
-  metadata.gz: 360bbf78c131f20a67359ddc2055cd58502da1f4e95adf30475cd405d5eb50be6ba4fd9aa0a0857226dc803e14282cc4231de113843e96657a65e287c7500137
-  data.tar.gz: f88ed1ae8c8f0dad9f4d8904a265c833ceee723ba92860c0e3bed4c193d56a901c31184abd4290058de47fbc089b12b4d3b1da064f138214e2954d45eee928da
+  metadata.gz: 0ff7694dd27e4293206de81fc2a7b5ccccb886579ed73eb7f97d390472692ce310993e2ece741cf85f5fbe265f1deb2a7ea326590383b4bdf0d3f77f10b1bbc1
+  data.tar.gz: 38ac6aa12d3e33ab0c26c0750273b60386d90fd4d916776a0d561c3f25a79fa2d7d216ac465842207cd65f62e2fcbd348389e65f905583187fe23c30908d92dc

data/README.md

@@ -1,5 +1,8 @@
 # Red Datasets
 
+[![Build Status](https://travis-ci.org/red-data-tools/red-datasets.svg?branch=master)](https://travis-ci.org/red-data-tools/red-datasets)
+[![Gem Version](https://badge.fury.io/rb/red-datasets.svg)](https://badge.fury.io/rb/red-datasets)
+
 ## Description
 
 Red Datasets provides classes that provide common datasets such as iris dataset.
@@ -128,6 +131,9 @@ mnist.each do |record|
 end
 ```
 
+## NArray compatibility
+
+* [red-datasets-numo-narray](https://github.com/red-data-tools/red-datasets-numo-narray)
 
 ## License
 

data/doc/text/news.md

@@ -1,5 +1,53 @@
 # News
 
+## 0.1.4 - 2021-07-13
+
+### Improvements
+
+  * `Datasets::SudachiSynonymDictionary`: Stopped depending on `LANG`.
+
+## 0.1.3 - 2021-07-09
+
+### Improvements
+
+  * `Datasets::SeabornData`: Added.
+
+  * `Datasets::SudachiSynonymDictionary`: Added.
+
+## 0.1.2 - 2021-06-03
+
+### Improvements
+
+  * `Datasets::Rdatasets` and `Datasets::RdatasetsList`: Added.
+
+  * `Datasets::Penguins`: Changed for compatibility with seaborn's
+    penguins dataset.
+
+## 0.1.1 - 2021-04-11
+
+### Improvements
+
+  * Added support for Ruby 3.0.
+
+  * `Datasets::Communities`: Added.
+    [GitHub#64][Patch by Yasuo Honda]
+
+  * `Datasets::EStatJapan`: Added.
+    [GitHub#90][Patch by Kunihiko Miyoshi]
+
+  * `Datasets::Penguins`: Added.
+    [GitHub#100][Patch by Kenta Murata]
+
+  * `Datasets::CLDRPlurals`: Added.
+
+### Thanks
+
+  * Yasuo Honda
+
+  * Kunihiko Miyoshi
+
+  * Kenta Murata
+
 ## 0.1.0 - 2020-02-04
 
 ### Improvements

data/lib/datasets.rb

@@ -2,6 +2,9 @@ require_relative "datasets/version"
 
 require_relative "datasets/adult"
 require_relative "datasets/cifar"
+require_relative "datasets/cldr-plurals"
+require_relative "datasets/communities"
+require_relative "datasets/e-stat-japan"
 require_relative "datasets/fashion-mnist"
 require_relative "datasets/hepatitis"
 require_relative "datasets/iris"
@@ -9,7 +12,11 @@ require_relative "datasets/libsvm"
 require_relative "datasets/libsvm-dataset-list"
 require_relative "datasets/mnist"
 require_relative "datasets/mushroom"
+require_relative "datasets/penguins"
 require_relative "datasets/penn-treebank"
 require_relative "datasets/postal-code-japan"
+require_relative "datasets/rdatasets"
+require_relative "datasets/seaborn-data"
+require_relative "datasets/sudachi-synonym-dictionary"
 require_relative "datasets/wikipedia"
 require_relative "datasets/wine"

data/lib/datasets/cifar.rb

@@ -1,10 +1,10 @@
-require "rubygems/package"
-require "zlib"
-
+require_relative "tar-gz-readable"
 require_relative "dataset"
 
 module Datasets
   class CIFAR < Dataset
+    include TarGzReadable
+
     module Pixelable
       def pixels
         data.unpack("C*")
@@ -61,7 +61,7 @@ module Datasets
     private
 
     def parse_data(data_path, &block)
-      open_tar(data_path) do |tar|
+      open_tar_gz(data_path) do |tar|
         target_file_names.each do |target_file_name|
           tar.seek(target_file_name) do |entry|
             parse_entry(entry, &block)
@@ -124,14 +124,6 @@ module Datasets
         end
       end
     end
-
-    def open_tar(data_path)
-      Zlib::GzipReader.open(data_path) do |f|
-        Gem::Package::TarReader.new(f) do |tar|
-          yield(tar)
-        end
-      end
-    end
   end
 end
 

data/lib/datasets/cldr-plurals.rb

@@ -0,0 +1,385 @@
+require "rexml/streamlistener"
+require "rexml/parsers/baseparser"
+require "rexml/parsers/streamparser"
+require "strscan"
+
+require_relative "dataset"
+
+module Datasets
+  class CLDRPlurals < Dataset
+    Locale = Struct.new(:name,
+                        :rules)
+
+    Rule = Struct.new(:count,
+                      :condition,
+                      :integer_samples,
+                      :decimal_samples)
+
+    def initialize
+      super()
+      @metadata.id = "cldr-plurals"
+      @metadata.name = "CLDR language plural rules"
+      @metadata.url = "https://raw.githubusercontent.com/unicode-org/cldr/master/common/supplemental/plurals.xml"
+      @metadata.licenses = ["Unicode-DFS-2016"]
+      @metadata.description = <<~DESCRIPTION
+        Language plural rules in Unicode Common Locale Data Repository.
+        See also: https://unicode-org.github.io/cldr-staging/charts/latest/supplemental/language_plural_rules.html
+      DESCRIPTION
+    end
+
+    def each(&block)
+      return to_enum(__method__) unless block_given?
+
+      open_data do |input|
+        catch do |abort_tag|
+          listener = Listener.new(abort_tag, &block)
+          parser = REXML::Parsers::StreamParser.new(input, listener)
+          parser.parse
+        end
+      end
+    end
+
+    private
+    def open_data
+      data_path = cache_dir_path + "plurals.xml"
+      unless data_path.exist?
+        download(data_path, @metadata.url)
+      end
+      ::File.open(data_path) do |input|
+        yield(input)
+      end
+    end
+
+    # Spec: https://unicode.org/reports/tr35/tr35-numbers.html#Language_Plural_Rules
+    class Listener
+      include REXML::StreamListener
+
+      def initialize(abort_tag, &block)
+        @abort_tag = abort_tag
+        @block = block
+        @tag_name_stack = []
+      end
+
+      def tag_start(name, attributes)
+        @tag_name_stack.push(name)
+        case name
+        when "pluralRules"
+          @locales = attributes["locales"].split
+          @rules = []
+        when "pluralRule"
+          @rule = Rule.new(attributes["count"])
+        end
+      end
+
+      def tag_end(name)
+        case name
+        when "pluralRules"
+          @locales.each do |locale_name|
+            @block.call(Locale.new(locale_name, @rules))
+          end
+        when "pluralRule"
+          @rules << @rule
+        end
+        @tag_name_stack.pop
+      end
+
+      def text(data)
+        case @tag_name_stack.last
+        when "pluralRule"
+          parse_plural_rule(data)
+        end
+      end
+
+      private
+      def parse_plural_rule(data)
+        parser = RuleParser.new(@rule, data)
+        parser.parse
+      end
+    end
+    private_constant :Listener
+
+    # Syntax: http://unicode.org/reports/tr35/tr35-numbers.html#Plural_rules_syntax
+    class RuleParser
+      def initialize(rule, data)
+        @rule = rule
+        @data = data
+        @scanner = StringScanner.new(@data)
+      end
+
+      def parse
+        @rule.condition = parse_condition
+        skip_whitespaces
+        if @scanner.scan(/@integer/)
+          @rule.integer_samples = parse_sample_list
+        end
+        skip_whitespaces
+        if @scanner.scan(/@decimal/)
+          @rule.decimal_samples = parse_sample_list
+        end
+      end
+
+      private
+      def skip_whitespaces
+        @scanner.skip(/\p{Pattern_White_Space}+/)
+      end
+
+      def parse_condition
+        and_condition = parse_and_condition
+        return nil if and_condition.nil?
+        and_conditions = [and_condition]
+        while parse_or
+          and_conditions << parse_and_condition
+        end
+        if and_conditions.size == 1
+          and_condition
+        else
+          [:or, *and_conditions]
+        end
+      end
+
+      def parse_or
+        skip_whitespaces
+        @scanner.scan(/or/)
+      end
+
+      def parse_and_condition
+        skip_whitespaces
+        relation = parse_relation
+        return nil if relation.nil?
+        relations = [relation]
+        while parse_and
+          relations << parse_relation
+        end
+        if relations.size == 1
+          relation
+        else
+          [:and, *relations]
+        end
+      end
+
+      def parse_and
+        skip_whitespaces
+        @scanner.scan(/and/)
+      end
+
+      def parse_relation
+        parse_is_relation or
+          parse_in_relation or
+          parse_within_relation
+      end
+
+      def parse_is_relation
+        position = @scanner.pos
+        skip_whitespaces
+        expr = parse_expr
+        unless parse_is
+          @scanner.pos = position
+          return nil
+        end
+        if parse_not
+          operator = :is_not
+        else
+          operator = :is
+        end
+        value = parse_value
+        if value.nil?
+          raise Error, "no value for #{operator}: #{@scanner.inspect}"
+        end
+        [operator, expr, value]
+      end
+
+      def parse_is
+        skip_whitespaces
+        @scanner.scan(/is/)
+      end
+
+      def parse_not
+        skip_whitespaces
+        @scanner.scan(/not/)
+      end
+
+      def parse_in_relation
+        position = @scanner.pos
+        skip_whitespaces
+        expr = parse_expr
+        if parse_not
+          if parse_in
+            operator = :not_in
+          else
+            @scanner.ops = position
+            return nil
+          end
+        elsif parse_in
+          operator = :in
+        elsif parse_equal
+          operator = :equal
+        elsif parse_not_equal
+          operator = :not_equal
+        else
+          @scanner.pos = position
+          return nil
+        end
+        range_list = parse_range_list
+        [operator, expr, range_list]
+      end
+
+      def parse_in
+        skip_whitespaces
+        @scanner.scan(/in/)
+      end
+
+      def parse_equal
+        skip_whitespaces
+        @scanner.scan(/=/)
+      end
+
+      def parse_not_equal
+        skip_whitespaces
+        @scanner.scan(/!=/)
+      end
+
+      def parse_within_relation
+        position = @scanner.pos
+        skip_whitespaces
+        expr = parse_expr
+        have_not = parse_not
+        unless parse_within
+          @scanner.pos = position
+          return nil
+        end
+        if have_not
+          operator = :not_within
+        else
+          operator = :within
+        end
+        range_list = parse_range_list
+        [operator, expr, range_list]
+      end
+
+      def parse_within
+        skip_whitespaces
+        @scanner.scan(/within/)
+      end
+
+      def parse_expr
+        operand = parse_operand
+        operator = parse_expr_operator
+        if operator
+          value = parse_value
+          if value.nil?
+            raise Error, "no value for #{operator}: #{@scanner.inspect}"
+          end
+          [operator, operand, value]
+        else
+          operand
+        end
+      end
+
+      def parse_operand
+        skip_whitespaces
+        @scanner.scan(/[niftvwce]/)
+      end
+
+      def parse_expr_operator
+        skip_whitespaces
+        if @scanner.scan(/(?:mod|%)/)
+          :mod
+        else
+          nil
+        end
+      end
+
+      def parse_range_list
+        ranges = [parse_range || parse_value]
+        loop do
+          skip_whitespaces
+          break unless @scanner.scan(/,/)
+          ranges << (parse_range || parse_value)
+        end
+        ranges
+      end
+
+      def parse_range
+        position = @scanner.pos
+        range_start = parse_value
+        skip_whitespaces
+        unless @scanner.scan(/\.\./)
+          @scanner.pos = position
+          return nil
+        end
+        range_end = parse_value
+        range_start..range_end
+      end
+
+      def parse_value
+        skip_whitespaces
+        value = @scanner.scan(/\d+/)
+        return nil if value.nil?
+        Integer(value, 10)
+      end
+
+      def parse_sample_list
+        samples = [parse_sample_range]
+        loop do
+          position = @scanner.pos
+          skip_whitespaces
+          break unless @scanner.scan(/,/)
+          sample_range = parse_sample_range
+          unless sample_range
+            @scanner.pos = position
+            break
+          end
+          samples << sample_range
+        end
+        skip_whitespaces
+        if @scanner.scan(/,/)
+          skip_whitespaces
+          # U+2026 HORIZONTAL ELLIPSIS
+          unless @scanner.scan(/\u2026|\.\.\./)
+            raise Error, "no ellipsis: #{@scanner.inspect}"
+          end
+          samples << :elipsis
+        end
+        samples
+      end
+
+      def parse_sample_range
+        value = parse_sample_value
+        return nil if value.nil?
+        skip_whitespaces
+        if @scanner.scan(/~/)
+          range_end = parse_sample_value
+          value..range_end
+        else
+          value
+        end
+      end
+
+      def parse_sample_value
+        value = parse_value
+        return nil if value.nil?
+        if @scanner.scan(/\./)
+          skip_whitespaces
+          decimal = @scanner.scan(/[0-9]+/)
+          if decimal.nil?
+            raise Error, "no decimal: #{@scanner.inspect}"
+          end
+          value += Float("0.#{decimal}")
+          skip_whitespaces
+        end
+        if @scanner.scan(/[ce]/)
+          # Workardoun for a spec bug. "e1" should be accepted.
+          #
+          # Spec:
+          #   sampleValue     = value ('.' digit+)? ([ce] digitPos digit+)?
+          #   digit           = [0-9]
+          #   digitPos        = [1-9]
+          e = @scanner.scan(/[1-9][0-9]*/)
+          value *= 10 * Integer(e, 10)
+        end
+        value
+      end
+    end
+    private_constant :RuleParser
+  end
+end