lang_libs 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: c4e9e14ff662cf98d5553724efbf699ac82186b8
+  data.tar.gz: c1efa4106f25da2fc98f8fad1dbab5bade51e8a0
+SHA512:
+  metadata.gz: c9fc4359beba39069aa8982b10c044bc5931dbe688ad5a1928472254be5049d54e68c8c9c669c24466552eba2ae5d2716b7c83e34d2d072fbc6de8856e4069ff
+  data.tar.gz: 67d944df3f7239107095b3705fcbb7d5fc7860b0fd8adc9a5cf3d837a79bf9c6b46de457841c0939113396f26baa2ce4ca3739e719be007d1aebcd6468e42804

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/.travis.yml

@@ -0,0 +1,3 @@
+language: ruby
+rvm:
+  - 2.1.5

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in lang_libs.gemspec
+gemspec

data/README.md

@@ -0,0 +1,39 @@
+# LangLibs
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/lang_libs`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+TODO: Delete this and the text above, and describe your gem
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'lang_libs'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install lang_libs
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release` to create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/lang_libs/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "lang_libs"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lang_libs.gemspec

@@ -0,0 +1,31 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'lang_libs/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "lang_libs"
+  spec.version       = LangLibs::VERSION
+  spec.authors       = ["Ryu_Mac"]
+  spec.email         = ["ts@mail.com"]
+
+  spec.summary       = %q{This is gem for language processing.}
+  spec.description   = %q{This gem can calculate tf-idf etc.}
+  #spec.homepage      = "TODO: Put your gem's website or public repo URL here."
+
+  # Prevent pushing this gem to RubyGems.org by setting 'allowed_push_host', or
+  # delete this section to allow pushing this gem to any host.
+  if spec.respond_to?(:metadata)
+    #spec.metadata['allowed_push_host'] = "TODO: Set to 'http://mygemserver.com'"
+  else
+    raise "RubyGems 2.0 or newer is required to protect against public gem pushes."
+  end
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.9"
+  spec.add_development_dependency "rake", "~> 10.0"
+end

data/lib/lang_libs.rb

@@ -0,0 +1,9 @@
+require "lang_libs/version"
+require "lang_libs/model"
+require "lang_libs/n_gram"
+require "lang_libs/tf_idf"
+require "lang_libs/file_access"
+require "lang_libs/accessor"
+require "lang_libs/index"
+require "lang_libs/getter"
+require "lang_libs/setter"

data/lib/lang_libs/accessor.rb

@@ -0,0 +1,61 @@
+module LangLibs
+  class Corpus
+
+    def dump_tf(index)
+      File.open("#{@dump_path}/tf_#{index}","wb") do |f|
+        Marshal.dump(@tf[index],f)
+      end
+    end
+    def load_tf(index)
+      result = nil
+      File.open("#{@dump_path}/tf_#{index}","rb") do |f|
+        result = Marshal.load(f)
+      end
+      set_tf(index,result)
+      result
+    end
+
+    def dump_idf
+      File.open("#{@dump_path}/idf","wb") do |f|
+        Marshal.dump(@idf,f)
+      end
+    end
+    def load_idf
+      result = nil
+      File.open("#{@dump_path}/idf","rb") do |f|
+        result = Marshal.load(f)
+      end
+      set_idf(result)
+      result
+    end
+
+    def dump_tfidf(index)
+      File.open("#{@dump_path}/tfidf_#{index}","wb") do |f|
+        Marshal.dump(@tfidf[index],f)
+      end
+    end
+    def load_tfidf(index)
+      result = nil
+      File.open("#{@dump_path}/tfidf_#{index}","rb") do |f|
+        result = Marshal.load(f)
+      end
+      set_tfidf(index,result)
+      result
+    end
+
+    def dump_count_terms
+      File.open("#{@dump_path}/count_terms","wb") do |f|
+        Marshal.dump(@count_terms,f)
+      end
+    end
+    def load_count_terms
+      result = nil
+      File.open("#{@dump_path}/count_terms","rb") do |f|
+        result = Marshal.load(f)
+      end
+      set_count_terms(result)
+      result
+    end
+
+  end
+end

data/lib/lang_libs/file_access.rb

@@ -0,0 +1,22 @@
+require 'yaml'
+
+module LangLibs
+  class Corpus
+
+    def save(file_path, *instance_names)
+      saving_hash = Hash.new
+      yield(saving_hash) if block_given?
+      instance_names.each do |name|
+        if instance_variable_defined?("@#{name}")
+          saving_hash["#{name}"] = eval("@#{name}").map{|key,val| [ key.to_s, val ] }.to_h
+        else
+          raise "instance variable defined: #{name}"
+        end
+      end
+      open(file_path,"w") do |f|
+        YAML.dump(saving_hash,f)
+      end
+    end
+
+  end
+end

data/lib/lang_libs/getter.rb

@@ -0,0 +1,33 @@
+module LangLibs
+  class Corpus
+    define_method(:get_tf) {|index| @tf[index] || load_tf(index) }
+    define_method(:get_idf) { @idf.empty? ? load_idf : @idf }
+    define_method(:get_tfidf) {|index| @tfidf[index] || load_tfidf(index) }
+    define_method(:get_count_terms) { @count_terms.empty? ? load_count_terms : @count_terms }
+
+    def each_tf(&block)
+      @index.each do |i|
+        block.call(get_tf(i))
+      end
+    end
+
+    def each_tf_with_index(&block)
+      @index.each do |i|
+        block.call(i, get_tf(i))
+      end
+    end
+
+    def each_tfidf(&block)
+      @index.each do |i|
+        block.call(get_tfidf(i))
+      end
+    end
+
+    def each_tfidf_with_index(&block)
+      @index.each do |i|
+        block.call(i, get_tfidf(i))
+      end
+    end
+
+  end
+end

data/lib/lang_libs/index.rb

@@ -0,0 +1,23 @@
+module LangLibs
+  class Index
+    def initialize
+      @index = Array.new
+    end
+
+    def regist(name)
+      @index.push(name) if !@index.include?(name)
+    end
+
+    def each(&block)
+      @index.each{|i| block.call(i)}
+    end
+
+    def size
+      @index.size
+    end
+
+    def first
+      @index.first
+    end
+  end
+end

data/lib/lang_libs/model.rb

@@ -0,0 +1,77 @@
+module LangLibs
+  class Corpus
+    # @param [Array] document(String) Array
+    def initialize(documents: nil,dump_path: "./dump",dump: false)
+      @dump_path = File.expand_path(dump_path)
+      @corpus = Hash.new
+      @tf, @idf, @tfidf, @count_terms = {},{},{},Hash.new(0)
+      @dump = dump
+      configure(dump_path)
+      documents_set(documents)
+    end
+
+    def documents_set(documents)
+      if documents.is_a?(Array)
+        self.add_documents(documents)
+      elsif documents.is_a?(Hash)
+        self.add_hashed_documents(documents)
+      elsif documents.nil?
+      else
+        raise "LangLibs::Corpus#documents_set : augument is not Array or Hash"
+      end
+    end
+
+    def configure(dump_path)
+      config = nil
+      config_file_path = File.join(dump_path,"configure")
+      if File.exist?(config_file_path)
+        File.open(config_file_path, "rb") do |f|
+          config = Marshal.load(f)
+        end
+        @index = config["index"]
+      else
+        @index = Index.new
+      end
+    end
+
+    def store_configure
+      config = Hash.new
+      config["index"] = @index
+      File.open(File.join(@dump_path,"configure"), "wb") do |f|
+        Marshal.dump(config,f)
+      end
+    end
+
+    def all_document
+      @corpus
+    end
+
+    def [](index)
+      @corpus[index]
+    end
+
+    def add_document(document)
+      @corpus[@index.size] = document
+      @index.regist(@index.size)
+    end
+
+    def add_documents(documents)
+      documents.each{|doc| add_document(doc) }
+    end
+
+    def add_hashed_documents(hashed_documents)
+      hashed_documents.each do |label,doc|
+        @corpus[label] = doc
+        @index.regist(label)
+      end
+    end
+
+    def each(&block)
+      @corpus.each{|_, val| block.call(val)}
+    end
+
+    def each_with_key(&block)
+      @corpus.each{|key, val| block.call(key,val)}
+    end
+  end
+end

data/lib/lang_libs/n_gram.rb

@@ -0,0 +1,18 @@
+module NGram
+  def ngram(doc,n,sep: "\s")
+    ngram_list = Hash.new(0)
+    grams = split_for_ngram(doc,n,sep: sep)
+    grams.each do |gram|
+      ngram_list[gram] += 1
+    end
+    ngram_list
+  end
+
+  private
+
+  def split_for_ngram(doc,n,sep: "\s")
+    characters = doc.split(sep)
+    return [ doc ] if characters.size <= n
+    return characters.each_cons(n).map{|chars| chars.join(sep)}
+  end
+end

data/lib/lang_libs/setter.rb

@@ -0,0 +1,8 @@
+module LangLibs
+  class Corpus
+    define_method(:set_tf) {|index,value| @tf[index] = value}
+    define_method(:set_idf) {|value| @idf = value }
+    define_method(:set_tfidf) {|index,value| @tfidf[index] = value}
+    define_method(:set_count_terms) {|value| @count_terms = value}
+  end
+end

data/lib/lang_libs/tf_idf.rb

@@ -0,0 +1,71 @@
+require "lang_libs/n_gram"
+
+module LangLibs
+  class Corpus
+    include NGram
+    def calc_tf(n, sep: "\s")
+      term_list = Hash.new
+      ngrams = Hash.new
+      count_terms = Hash.new(0)
+      each_with_key do |i,doc|
+        ngrams[i] = ngram(doc, n, sep: sep)
+        ngrams[i].each do |term,num|
+          term_list[term] = 0
+          count_terms[term] += num
+        end
+      end
+      set_count_terms(count_terms)
+      ngrams.each do |i,terms|
+        temp_term_list = term_list.dup
+        ngrams_sum = 0
+        terms.each do |term,num|
+          temp_term_list[term] = num
+          ngrams_sum += num
+        end
+        set_tf(i, temp_term_list.map{|term,freq| [term, freq.to_f/ngrams_sum]}.to_h )
+      end
+      if @dump
+        @index.each{|i| dump_tf(i) }
+        dump_count_terms
+      end
+    end
+
+    def calc_idf
+      term_freq = Hash.new(0)
+      get_tf(@index.first).each do |term,_|
+        each_tf do |tfs|
+          term_freq[term] += 1 if tfs[term] > 0
+        end
+      end
+      set_idf( term_freq.map{|term,freq| [ term, Math.log(@index.size.to_f / freq) ] }.to_h )
+      dump_idf if @dump
+    end
+
+    def calc_tfidf
+      each_tf_with_index do |i,tfs|
+        temp_tfidf = get_idf.dup
+        tfs.each do |term, tf|
+          temp_tfidf[term] = tf * get_idf[term]
+        end
+        set_tfidf(i, temp_tfidf)
+      end
+      @index.each{|i| dump_tfidf(i)} if @dump
+    end
+
+    def top_term_tfidf(n)
+      result = Hash.new
+      set_count_terms(get_count_terms.sort_by{|term,num| -num}.to_h)
+      each_tfidf_with_index do |i,tfidfs|
+        result[i] = Hash.new
+        count = 0
+        get_count_terms.each do |term,_|
+          result[i][term] = tfidfs[term]
+          count += 1
+          break if count >= n
+        end
+      end
+      result
+    end
+
+  end
+end

data/lib/lang_libs/version.rb

@@ -0,0 +1,3 @@
+module LangLibs
+  VERSION = "0.1.0"
+end

metadata

@@ -0,0 +1,91 @@
+--- !ruby/object:Gem::Specification
+name: lang_libs
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Ryu_Mac
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2016-07-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: This gem can calculate tf-idf etc.
+email:
+- ts@mail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lang_libs.gemspec
+- lib/lang_libs.rb
+- lib/lang_libs/accessor.rb
+- lib/lang_libs/file_access.rb
+- lib/lang_libs/getter.rb
+- lib/lang_libs/index.rb
+- lib/lang_libs/model.rb
+- lib/lang_libs/n_gram.rb
+- lib/lang_libs/setter.rb
+- lib/lang_libs/tf_idf.rb
+- lib/lang_libs/version.rb
+homepage: 
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: This is gem for language processing.
+test_files: []
+has_rdoc: