rblearn 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 98bd238b96729a72b7823d341a8f59ed084f7787
-  data.tar.gz: 8d800e60ab8af1f69367e010b744c715f37b2826
+  metadata.gz: e421c233da7861bb3062a93b964b9cc4a3b23f1c
+  data.tar.gz: b2be5eb9d5f61bb7a70bda13be2d876afd509dbc
 SHA512:
-  metadata.gz: 63751de5c1a69ca150c8e0abfad8b54e27f5757c9897736090df3e1b3b7a3a67aad6db3087afbb1e033d7221012a78e44f0d271be713fcbcf0fd5cc24d863a4f
-  data.tar.gz: b4d033b2798aa332fa5ac8501b7ccf6bd14cd32caf4631d077f7f1a2c0d172a376358375392c493edaf061203b47c19fc937a44f734bf3c4918a08d210dfd510
+  metadata.gz: 6b2d6557d3a07864fe0a50a892f0ea7540fe44a585aaf451afdd220255c11fbc1bf70f9429af955b8b6b8958d4dcb157df46ae67cd0ac1d0091810489c857cc2
+  data.tar.gz: eb58039f44c2584d81b524c9dc6faaf24a32ffbf3b31d33a61817e668ede474fdd021fd315c69cfcced7fa4dc74a648b2f696ab32fed50642508e6d4f7a91de7

data/lib/rblearn/CountVectorizer.rb

@@ -0,0 +1,83 @@
+
+module Rblearn
+
+	class CountVectorizer
+
+    # TODO: consider the access controll about all variables
+		attr_accessor :feature_names, :doc_matrix, :token2index
+
+		# tokenizer: lambda function :: string -> Array<string>
+		# lowcase: whether if words are lowercases :: bool
+		# stop_words: list of stop words :: Array<string>
+		# max_features: limitation of feature size :: Float \in [0, 1]
+		# TODO: by max_features, zero vectors are sometimes created.
+		def initialize(tokenizer, lowercase=true, max_features=0.8)
+			@tokenizer = tokenizer
+			@lowercase = lowercase
+
+			stop_words = Stopwords::STOP_WORDS
+			stop_words.map! {|token| token.stem}
+			stop_words.map! {|token| token.downcase} if @lowercase
+			@stopwords = stop_words
+			@max_feature = max_features
+		end
+
+		# features: Each documents' feature :: Array<String> -> NArray::Int64
+		def fit_transform(features)
+			all_vocaburaries = []
+			word_frequency = Hash.new{|hash, key| hash[key] = 0}
+
+			features.each do |feature|
+				@tokenizer.call(feature).each do |token|
+					token.downcase! if @lowercase
+					all_vocaburaries << token
+					word_frequency[token] += 1
+				end
+			end
+
+			all_vocaburaries.uniq!
+			word_frequency =  word_frequency.sort{|(_, value1), (_, value2)| value2 <=> value1}
+			feature_names = (0...(word_frequency.size * @max_feature).to_i).map{|i| word_frequency[i][0]}
+
+			token2index = {}
+			feature_names.each_with_index do |token, i|
+				token2index[token] = i
+			end
+
+			doc_matrix = Numo::Int32.zeros([features.size, feature_names.size])
+			features.each_with_index do |feature, doc_id|
+				tokens = []
+				@tokenizer.call(feature).each do |token|
+					token.downcase! if @lowercase
+					tokens << token unless @stopwords.include?(token)
+				end
+
+				# BoW representation
+				counter = Hash.new{|hash, key| hash[key] = 0}
+				tokens.each do |token|
+					counter[token] += 1
+				end
+
+				counter.each do |token, freq|
+					doc_matrix[doc_id, token2index[token]] = freq if token2index[token]
+				end
+			end
+
+			@doc_matrix = doc_matrix
+			@feature_names = feature_names
+			@token2index = token2index
+			return @doc_matrix
+		end
+	end
+
+
+
+	if __FILE__ == $0
+		cv = CountVectorizer.new(lambda{|s| s.split.map{|token| token.stem}}, 1, 0.8)
+		features = ['I am train man which automata and philosophy', 'numerical analysis young man', 'logic programmer']
+		p cv.fit_transform(features)
+		p cv.feature_names
+		p cv.token2index
+	end
+
+end

data/lib/rblearn/CrossValidation.rb

@@ -0,0 +1,45 @@
+
+module Rblearn
+  module CrossValidation
+    # x, y: Narray object
+    # We slice a matrix by x[Array<Integer>, true]
+    def self.train_test_split(x, y, test_size=0.33)
+      doc_size = x.shape[0]
+      random_indices = (0...doc_size).to_a.shuffle
+      endpoint = (doc_size * test_size).to_i
+      train_indices = random_indices[endpoint..-1]
+      test_indices = random_indices[0...endpoint]
+
+      return [x[train_indices, true], y[train_indices, true], x[test_indices, true], y[test_indices, true]]
+    end
+
+    class KFold
+      # TODO: make indices and n_folds private
+
+      def initialize(n, n_folds, shuffle)
+        indices = (0...n).to_a
+        indices.shuffle! if shuffle
+        @indices = indices
+        @n_folds = n_folds
+      end
+
+      def create
+        groups_nfolds = @indices.each_slice((@indices.size.to_f / @n_folds).ceil).to_a
+        groups = []
+
+        @n_folds.times do |k|
+          validation_set = []
+          test_set = []
+
+          @n_folds.times do |j|
+            test_set += groups_nfolds[j] if k == j
+            validation_set += groups_nfolds[j] unless k == j
+          end
+          groups << [validation_set, test_set]
+        end
+
+        return groups
+      end
+    end
+  end
+end

data/lib/rblearn/version.rb

@@ -1,3 +1,3 @@
 module Rblearn
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/lib/rblearn.rb

@@ -1,4 +1,9 @@
-require "rblearn/version"
+require 'numo/narray'
+require 'stopwords'
+require 'stemmify'
+require 'rblearn/version'
+require 'rblearn/CountVectorizer'
+require 'rblearn/CrossValidation'
 
 module Rblearn
   # Your code goes here...

data/rblearn.gemspec

@@ -25,4 +25,8 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 1.11"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec", "~> 3.0"
+
+  spec.add_runtime_dependency "numo-narray"
+  spec.add_runtime_dependency 'stopwords'
+  spec.add_runtime_dependency 'stemmify'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rblearn
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - himkt
@@ -52,6 +52,48 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: numo-narray
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: stopwords
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: stemmify
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: rblearn (ruby-learn) provides methods for feature extracting and some
   algorithms.
 email:
@@ -71,6 +113,8 @@ files:
 - bin/console
 - bin/setup
 - lib/rblearn.rb
+- lib/rblearn/CountVectorizer.rb
+- lib/rblearn/CrossValidation.rb
 - lib/rblearn/version.rb
 - rblearn.gemspec
 homepage: https://github.com/himkt/rblearn
@@ -98,3 +142,4 @@ signing_key:
 specification_version: 4
 summary: Simple repository for machine learning
 test_files: []
+has_rdoc: