analy_z 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 758343df058bfed451689c1c991e1354fb92d289
-  data.tar.gz: e150373446af0e1430e78a8db54f8abbd5f0db93
+  metadata.gz: 257d11a71d341c246d265954ebdd1f557332867d
+  data.tar.gz: 0ad929efdd028daa2509005473322951f9d37ba1
 SHA512:
-  metadata.gz: 8a7858dab8bf27d15663fbba157b11de25fbc6b955b675f812ecc422d0c90ab5f885be5cd6e0956dc4c4310a8465e9e425c5e1eaf674857aeb01b655ca9882c2
-  data.tar.gz: 106f76e8109d8762fdcf49d3fdfe8853d1a6aff6028bc10bf1613d1f0bfc616354bbe4da0987b4bc3190122582f579b807d93b2545acf75335b24b7f33981ec0
+  metadata.gz: 6da8e91ad8f952e7f2f0c9ee02c79557fda628f5b2995c60a119d0de91d71fd7937965c77d9ee2e7277fbaa24b1b83c82c20d95e4c1bb336a9b745da1d91e891
+  data.tar.gz: 448bbe2ae21c0bde8d4964e966cd6909de326cc63696b74a631d8a41b5923d292af8d0e6e6706fce47301cf9f73f3d1ae2f8f3435e601e9fad596e7d49a10a55

data/lib/analy_z.rb

@@ -5,6 +5,7 @@ module AnalyZ
   class Analyzer
 
     require 'pp'
+    require 'date'
     require 'natto'
     require 'nokogiri'
 
@@ -19,26 +20,42 @@ module AnalyZ
     def initialize html_path, selector = 'body', type_ary = ['名詞']
       @sentences = {}
       Dir.glob(html_path).each do |f|
+        print '.'
         @sentences[f] = parse_html(Nokogiri::HTML.parse(File.read(f), nil, nil).css(selector).to_html)
       end
-      analyze_words(@sentences)
-    end
 
-    def analyze_words sentences, type_ary = ['名詞']
+      puts "\n=== creating sentences file ==="
+      txt = ""
+      @sentences.each do |k, sentences|
+        print '.'
+        txt << sentences.map{|s| s[0] }.join + '/=== EOS ===/'
+      end
+
+      text_file_path = "tmp/#{DateTime.now}.txt"
+      File.write(text_file_path, txt)
 
-      @texts, @words, @tf, @idf, @hse = {}, {}, {}, {}, {}
+      puts "\n=== analyzing... ==="
+      analyze_words(@sentences, text_file_path)
+    end
 
-      sentences.each{|k, sentence| @texts[k] = sentence.map {|s| s[0]}.join }
+    def analyze_words sentences, text_file_path, type_ary = ['名詞']
 
+      @words, @tf, @idf, @hse = {}, {}, {}, {}
+
+      puts "=== calculating tf and idf and hse ==="
       sentences.each do |key, sentence_ary|
+        print '.'
         text = sentence_ary.map {|s| s[0] }.join
         @words[key] = parse_by_natto(text, type_ary)
         @tf[key] = calc_tf(@words[key])
-        @idf[key] = calc_idf(@texts, @words[key])
+        @idf[key] = calc_idf(@words[key], text_file_path)
         @hse[key] = calc_hse(@words[key], sentence_ary)
       end
 
+      puts "\n=== calculating tf idf ==="
       @tf_idf = calc_tf_idf(@tf, @idf)
+
+      puts "=== calculating hse tf idf ==="
       @hse_tf_idf = calc_hse_tf_idf(@tf_idf, @hse)
 
     end
@@ -103,10 +120,13 @@ module AnalyZ
       end
     end
 
-    def calc_idf sentences, words
+    def calc_idf words, text_file_path
+      texts = File.read(text_file_path).split('/=== EOS ===/')
       words.map do |word|
         cnt = 0
-        sentences.each {|k, v| cnt += 1 if v.include?(word) }
+        texts.each do |text|
+          cnt += 1 if text.include?(word)
+        end
         [word, Math.log(sentences.length / cnt.to_f)]
       end
     end

data/lib/analy_z/version.rb

@@ -1,3 +1,3 @@
 module AnalyZ
-  VERSION = "0.1.2"
+  VERSION = "0.1.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: analy_z
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - nao215
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-04-29 00:00:00.000000000 Z
+date: 2016-04-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: natto