red-datasets 0.0.6 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3b96f5bf8fb7d8d7280451086dda394b65c42023b15ae077167e2d320c4361c1
-  data.tar.gz: 96f7936d62d70749f92d3bdd1d7ef2d79cfff3091e7dae8221d6a0537dbd6d7b
+  metadata.gz: '0239c4ab86dd9f589b1f67b9d6c381570e25a29289c261470943ed48f7dfc3d0'
+  data.tar.gz: 2f3f3af1f17a1bd1e7aa307e2b182108790549754d907262105e18479997cde6
 SHA512:
-  metadata.gz: 859196aa39020d924fa7af4df6d96c110f41ac2b90a39dc89ed6935fc64e857b2bffb5776a366660ab61c55a96dd35b9bd6663ec23c7ee4249cae3103bc0a2aa
-  data.tar.gz: b07ec53917af58e737058c504685d283850e072f0794c457bd961d39b9815c85b2fc2a9bed4de2a643675dc0e0f7bb2077b4c41b2c28c9c94f948a532baae6bb
+  metadata.gz: 04b3dbc23dc8679855a6104a9f3da39871594979f149295ef13b3be864a3dbbdb6bec3fb59153db9b5be4fade6819686e13b60a38f1d1721bf7e1163d4bb49b8
+  data.tar.gz: 476a9081fe0db32aad8a4e00c7e08f77002e58a2f2c68eb37aecf2a70054d43877707ddda29137b361a5a37ff979f3c28c5f2a03c2d1e96cbc7f7289f659ba9f

data/README.md

@@ -1,8 +1,7 @@
-# README
+# Red Datasets
 
-## Name
-
-Red Datasets
+[![Build Status](https://travis-ci.org/red-data-tools/red-datasets.svg?branch=master)](https://travis-ci.org/red-data-tools/red-datasets)
+[![Gem Version](https://badge.fury.io/rb/red-datasets.svg)](https://badge.fury.io/rb/red-datasets)
 
 ## Description
 
@@ -16,6 +15,20 @@ You can use datasets easily because you can access each dataset with multiple wa
 % gem install red-datasets
 ```
 
+## Available datasets
+
+TODO: Document them in source code to list in document: https://www.rubydoc.info/gems/red-datasets
+
+* Adult Dataset
+* CIFAR-10 Dataset
+* CIFAR-100 Dataset
+* Fashion-MNIST
+* Iris Dataset
+* MNIST database
+* The Penn Treebank Project
+* Wikipedia
+* Wine Dataset
+
 ## Usage
 
 Here is an example to access [Iris Data Set](https://archive.ics.uci.edu/ml/datasets/iris) by `#each`  or `Table#to_h` or `Table#fetch_values`.
@@ -30,7 +43,7 @@ iris.each do |record|
      record.sepal_width,
      record.petal_length,
      record.petal_width,
-     record.class,
+     record.label,
   ]
 end
 # => [5.1, 3.5, 1.4, 0.2, "Iris-setosa"]
@@ -48,7 +61,7 @@ p iris_hash[:petal_length]
 # => [1.4, 1.4, .. , 4.7, ..
 p iris_hash[:petal_width]
 # => [0.2, 0.2, .. , 1.4, ..
-p iris_hash[:class]
+p iris_hash[:label]
 # => ["Iris-setosa", "Iris-setosa", .. , "Iris-versicolor", ..
 
 
@@ -60,7 +73,7 @@ p iris_table.fetch_values(:sepal_length, :sepal_width, :petal_length, :petal_wid
       [7.0, 3.2, 4.7, 1.4],
       :
 
-p iris_table[:class]
+p iris_table[:label]
 # => ["Iris-setosa", "Iris-setosa", .. , "Iris-versicolor", ..
 ```
 
@@ -118,6 +131,9 @@ mnist.each do |record|
 end
 ```
 
+## NArray compatibility
+
+* [red-datasets-numo-narray](https://github.com/red-data-tools/red-datasets-numo-narray)
 
 ## License
 

data/doc/text/news.md

@@ -1,5 +1,129 @@
 # News
 
+## 0.1.1 - 2021-04-11
+
+### Improvements
+
+  * Added support for Ruby 3.0.
+
+  * `Datasets::Communities`: Added.
+    [GitHub#64][Patch by Yasuo Honda]
+
+  * `Datasets::EStatJapan`: Added.
+    [GitHub#90][Patch by Kunihiko Miyoshi]
+
+  * `Datasets::Penguins`: Added.
+    [GitHub#100][Patch by Kenta Murata]
+
+  * `Datasets::CLDRPlurals`: Added.
+
+### Thanks
+
+  * Yasuo Honda
+
+  * Kunihiko Miyoshi
+
+  * Kenta Murata
+
+## 0.1.0 - 2020-02-04
+
+### Improvements
+
+  * Added support for Ruby 2.7.
+    [GitHub#82][GitHub#83][Patch by Yasuo Honda]
+
+  * `Datasets::Hepatitis`: Added.
+    [GitHub#70][Patch by KazuhiroYoshimoto]
+
+  * `Datasets::Downloader`: Added support for query.
+
+### Thanks
+
+  * Yasuo Honda
+
+  * KazuhiroYoshimoto
+
+## 0.0.9 - 2019-09-09
+
+### Improvements
+
+  * `Datasets::LIBSVMDatasetList`: Improved performance.
+
+  * `Datasets::Mushroom`: Added.
+    [GitHub#33][Patch by Yasuo Honda]
+
+  * `Datasets::Table#n_columns`: Added.
+
+  * `Datasets::Table#n_rows`: Added.
+
+  * `Datasets::Table#[]`: Added support for index access.
+
+  * `Datasets::Table#coolumn_names`: Added.
+
+  * `Datasets::Table#size`: Added.
+
+  * `Datasets::Table#length`: Added.
+
+  * `Datasets::Table#each_column`: Added.
+
+  * `Datasets::Table#each_record`: Added.
+
+  * `Datasets::Table#find_record`: Added.
+
+### Thanks
+
+  * Yasuo Honda
+
+### Improvements
+
+## 0.0.8 - 2019-03-24
+
+### Improvements
+
+  * Improved README.
+    [GitHub#40][Patch by kojix2]
+
+  * `Datasets::PostalCodeJapan`: Added.
+
+  * `Datasets::LIBSVMDatasetList`: Added.
+
+  * `Datasets::LIBSVM`: Added.
+
+### Thanks
+
+  * kojix2
+
+## 0.0.7 - 2018-11-21
+
+### Improvements
+
+  * `Datasets::Table#dictionary_encode`: Added.
+    [GitHub#22]
+
+  * `Datasets::Table#label_encode`: Added.
+
+  * `Datasets::Dictionary`: Added.
+
+  * `Datasets::Wine`: Added.
+    [GitHub#26][Patch by Ryuta Suzuki]
+
+  * `Datasets::FashionMNIST`: Added.
+    [GitHub#27][Patch by chimame]
+
+  * `Datasets::Iris::Record#label`: Renamed from `#class`. This is an
+    incompatible change.
+
+  * `Datasets::Adult`: Added.
+    [GitHub#30][Patch by Yasuo Honda]
+
+### Thanks
+
+  * Ryuta Suzuki
+
+  * chimame
+
+  * Yasuo Honda
+
 ## 0.0.6 - 2018-07-25
 
 ### Improvements

data/lib/datasets.rb

@@ -1,7 +1,19 @@
-require "datasets/version"
+require_relative "datasets/version"
 
-require "datasets/cifar"
-require "datasets/iris"
-require "datasets/mnist"
-require "datasets/penn-treebank"
-require "datasets/wikipedia"
+require_relative "datasets/adult"
+require_relative "datasets/cifar"
+require_relative "datasets/cldr-plurals"
+require_relative "datasets/communities"
+require_relative "datasets/e-stat-japan"
+require_relative "datasets/fashion-mnist"
+require_relative "datasets/hepatitis"
+require_relative "datasets/iris"
+require_relative "datasets/libsvm"
+require_relative "datasets/libsvm-dataset-list"
+require_relative "datasets/mnist"
+require_relative "datasets/mushroom"
+require_relative "datasets/penguins"
+require_relative "datasets/penn-treebank"
+require_relative "datasets/postal-code-japan"
+require_relative "datasets/wikipedia"
+require_relative "datasets/wine"

data/lib/datasets/adult.rb

@@ -0,0 +1,84 @@
+require "csv"
+
+require_relative "dataset"
+
+module Datasets
+  class Adult < Dataset
+    Record = Struct.new(
+      :age,
+      :work_class,
+      :final_weight,
+      :education,
+      :n_education_years,
+      :marital_status,
+      :occupation,
+      :relationship,
+      :race,
+      :sex,
+      :capital_gain,
+      :capital_loss,
+      :hours_per_week,
+      :native_country,
+      :label
+    )
+
+    def initialize(type: :train)
+      unless [:train, :test].include?(type)
+        raise ArgumentError, 'Please set type :train or :test'
+      end
+
+      super()
+      @type = type
+      @metadata.id = "adult-#{@type}"
+      @metadata.name = "Adult: #{@type}"
+      @metadata.url = "http://archive.ics.uci.edu/ml/datasets/adult"
+      @metadata.description = lambda do
+        read_names
+      end
+    end
+
+    def each
+      return to_enum(__method__) unless block_given?
+
+      open_data do |csv|
+        csv.each do |row|
+          next if row[0].nil?
+          record = Record.new(*row)
+          yield(record)
+        end
+      end
+    end
+
+    private
+    def open_data
+      case @type
+      when :train
+        ext = "data"
+      when :test
+        ext = "test"
+      end
+      data_path = cache_dir_path + "adult-#{ext}.csv"
+      unless data_path.exist?
+        data_url = "http://archive.ics.uci.edu/ml/machine-learning-databases/adult/adult.#{ext}"
+        download(data_path, data_url)
+      end
+
+      options = {
+                 converters: [:numeric, lambda {|f| f.strip}],
+                 skip_lines: /\A\|/,
+      }
+      CSV.open(data_path, **options) do |csv|
+        yield(csv)
+      end
+    end
+
+    def read_names
+      names_path = cache_dir_path + "adult.names"
+      unless names_path.exist?
+        names_url = "https://archive.ics.uci.edu/ml/machine-learning-databases/adult/adult.names"
+        download(names_path, names_url)
+      end
+      names_path.read
+    end
+  end
+end

data/lib/datasets/cldr-plurals.rb

@@ -0,0 +1,385 @@
+require "rexml/streamlistener"
+require "rexml/parsers/baseparser"
+require "rexml/parsers/streamparser"
+require "strscan"
+
+require_relative "dataset"
+
+module Datasets
+  class CLDRPlurals < Dataset
+    Locale = Struct.new(:name,
+                        :rules)
+
+    Rule = Struct.new(:count,
+                      :condition,
+                      :integer_samples,
+                      :decimal_samples)
+
+    def initialize
+      super()
+      @metadata.id = "cldr-plurals"
+      @metadata.name = "CLDR language plural rules"
+      @metadata.url = "https://raw.githubusercontent.com/unicode-org/cldr/master/common/supplemental/plurals.xml"
+      @metadata.licenses = ["Unicode-DFS-2016"]
+      @metadata.description = <<~DESCRIPTION
+        Language plural rules in Unicode Common Locale Data Repository.
+        See also: https://unicode-org.github.io/cldr-staging/charts/latest/supplemental/language_plural_rules.html
+      DESCRIPTION
+    end
+
+    def each(&block)
+      return to_enum(__method__) unless block_given?
+
+      open_data do |input|
+        catch do |abort_tag|
+          listener = Listener.new(abort_tag, &block)
+          parser = REXML::Parsers::StreamParser.new(input, listener)
+          parser.parse
+        end
+      end
+    end
+
+    private
+    def open_data
+      data_path = cache_dir_path + "plurals.xml"
+      unless data_path.exist?
+        download(data_path, @metadata.url)
+      end
+      ::File.open(data_path) do |input|
+        yield(input)
+      end
+    end
+
+    # Spec: https://unicode.org/reports/tr35/tr35-numbers.html#Language_Plural_Rules
+    class Listener
+      include REXML::StreamListener
+
+      def initialize(abort_tag, &block)
+        @abort_tag = abort_tag
+        @block = block
+        @tag_name_stack = []
+      end
+
+      def tag_start(name, attributes)
+        @tag_name_stack.push(name)
+        case name
+        when "pluralRules"
+          @locales = attributes["locales"].split
+          @rules = []
+        when "pluralRule"
+          @rule = Rule.new(attributes["count"])
+        end
+      end
+
+      def tag_end(name)
+        case name
+        when "pluralRules"
+          @locales.each do |locale_name|
+            @block.call(Locale.new(locale_name, @rules))
+          end
+        when "pluralRule"
+          @rules << @rule
+        end
+        @tag_name_stack.pop
+      end
+
+      def text(data)
+        case @tag_name_stack.last
+        when "pluralRule"
+          parse_plural_rule(data)
+        end
+      end
+
+      private
+      def parse_plural_rule(data)
+        parser = RuleParser.new(@rule, data)
+        parser.parse
+      end
+    end
+    private_constant :Listener
+
+    # Syntax: http://unicode.org/reports/tr35/tr35-numbers.html#Plural_rules_syntax
+    class RuleParser
+      def initialize(rule, data)
+        @rule = rule
+        @data = data
+        @scanner = StringScanner.new(@data)
+      end
+
+      def parse
+        @rule.condition = parse_condition
+        skip_whitespaces
+        if @scanner.scan(/@integer/)
+          @rule.integer_samples = parse_sample_list
+        end
+        skip_whitespaces
+        if @scanner.scan(/@decimal/)
+          @rule.decimal_samples = parse_sample_list
+        end
+      end
+
+      private
+      def skip_whitespaces
+        @scanner.skip(/\p{Pattern_White_Space}+/)
+      end
+
+      def parse_condition
+        and_condition = parse_and_condition
+        return nil if and_condition.nil?
+        and_conditions = [and_condition]
+        while parse_or
+          and_conditions << parse_and_condition
+        end
+        if and_conditions.size == 1
+          and_condition
+        else
+          [:or, *and_conditions]
+        end
+      end
+
+      def parse_or
+        skip_whitespaces
+        @scanner.scan(/or/)
+      end
+
+      def parse_and_condition
+        skip_whitespaces
+        relation = parse_relation
+        return nil if relation.nil?
+        relations = [relation]
+        while parse_and
+          relations << parse_relation
+        end
+        if relations.size == 1
+          relation
+        else
+          [:and, *relations]
+        end
+      end
+
+      def parse_and
+        skip_whitespaces
+        @scanner.scan(/and/)
+      end
+
+      def parse_relation
+        parse_is_relation or
+          parse_in_relation or
+          parse_within_relation
+      end
+
+      def parse_is_relation
+        position = @scanner.pos
+        skip_whitespaces
+        expr = parse_expr
+        unless parse_is
+          @scanner.pos = position
+          return nil
+        end
+        if parse_not
+          operator = :is_not
+        else
+          operator = :is
+        end
+        value = parse_value
+        if value.nil?
+          raise Error.new("no value for #{operator}: #{@scanner.inspect}")
+        end
+        [operator, expr, value]
+      end
+
+      def parse_is
+        skip_whitespaces
+        @scanner.scan(/is/)
+      end
+
+      def parse_not
+        skip_whitespaces
+        @scanner.scan(/not/)
+      end
+
+      def parse_in_relation
+        position = @scanner.pos
+        skip_whitespaces
+        expr = parse_expr
+        if parse_not
+          if parse_in
+            operator = :not_in
+          else
+            @scanner.ops = position
+            return nil
+          end
+        elsif parse_in
+          operator = :in
+        elsif parse_equal
+          operator = :equal
+        elsif parse_not_equal
+          operator = :not_equal
+        else
+          @scanner.pos = position
+          return nil
+        end
+        range_list = parse_range_list
+        [operator, expr, range_list]
+      end
+
+      def parse_in
+        skip_whitespaces
+        @scanner.scan(/in/)
+      end
+
+      def parse_equal
+        skip_whitespaces
+        @scanner.scan(/=/)
+      end
+
+      def parse_not_equal
+        skip_whitespaces
+        @scanner.scan(/!=/)
+      end
+
+      def parse_within_relation
+        position = @scanner.pos
+        skip_whitespaces
+        expr = parse_expr
+        have_not = parse_not
+        unless parse_within
+          @scanner.pos = position
+          return nil
+        end
+        if have_not
+          operator = :not_within
+        else
+          operator = :within
+        end
+        range_list = parse_range_list
+        [operator, expr, range_list]
+      end
+
+      def parse_within
+        skip_whitespaces
+        @scanner.scan(/within/)
+      end
+
+      def parse_expr
+        operand = parse_operand
+        operator = parse_expr_operator
+        if operator
+          value = parse_value
+          if value.nil?
+            raise Error.new("no value for #{operator}: #{@scanner.inspect}")
+          end
+          [operator, operand, value]
+        else
+          operand
+        end
+      end
+
+      def parse_operand
+        skip_whitespaces
+        @scanner.scan(/[niftvwce]/)
+      end
+
+      def parse_expr_operator
+        skip_whitespaces
+        if @scanner.scan(/(?:mod|%)/)
+          :mod
+        else
+          nil
+        end
+      end
+
+      def parse_range_list
+        ranges = [parse_range || parse_value]
+        loop do
+          skip_whitespaces
+          break unless @scanner.scan(/,/)
+          ranges << (parse_range || parse_value)
+        end
+        ranges
+      end
+
+      def parse_range
+        position = @scanner.pos
+        range_start = parse_value
+        skip_whitespaces
+        unless @scanner.scan(/\.\./)
+          @scanner.pos = position
+          return nil
+        end
+        range_end = parse_value
+        range_start..range_end
+      end
+
+      def parse_value
+        skip_whitespaces
+        value = @scanner.scan(/\d+/)
+        return nil if value.nil?
+        Integer(value, 10)
+      end
+
+      def parse_sample_list
+        samples = [parse_sample_range]
+        loop do
+          position = @scanner.pos
+          skip_whitespaces
+          break unless @scanner.scan(/,/)
+          sample_range = parse_sample_range
+          unless sample_range
+            @scanner.pos = position
+            break
+          end
+          samples << sample_range
+        end
+        skip_whitespaces
+        if @scanner.scan(/,/)
+          skip_whitespaces
+          # U+2026 HORIZONTAL ELLIPSIS
+          unless @scanner.scan(/\u2026|\.\.\./)
+            raise "no ellipsis: #{@scanner.inspect}"
+          end
+          samples << :elipsis
+        end
+        samples
+      end
+
+      def parse_sample_range
+        value = parse_sample_value
+        return nil if value.nil?
+        skip_whitespaces
+        if @scanner.scan(/~/)
+          range_end = parse_sample_value
+          value..range_end
+        else
+          value
+        end
+      end
+
+      def parse_sample_value
+        value = parse_value
+        return nil if value.nil?
+        if @scanner.scan(/\./)
+          skip_whitespaces
+          decimal = @scanner.scan(/[0-9]+/)
+          if decimal.nil?
+            raise "no decimal: #{@scanner.inspect}"
+          end
+          value += Float("0.#{decimal}")
+          skip_whitespaces
+        end
+        if @scanner.scan(/[ce]/)
+          # Workardoun for a spec bug. "e1" should be accepted.
+          #
+          # Spec:
+          #   sampleValue     = value ('.' digit+)? ([ce] digitPos digit+)?
+          #   digit           = [0-9]
+          #   digitPos        = [1-9]
+          e = @scanner.scan(/[1-9][0-9]*/)
+          value *= 10 * Integer(e, 10)
+        end
+        value
+      end
+    end
+    private_constant :RuleParser
+  end
+end