sosowa 0.1 → 0.2

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.2
+* Sosowa::Novel.titleを追加。むしろどうして今まで無かった
+* Sosowa::Author, Sosowa::Commentが取得出来ないバグを修正
+
 ## 0.1
 * Sosowa#searchを追加
 * Sosowa::Novel#plainを使って<br>タグや改行コードが取り除かれたテキストを得ることが出来ます。

data/README.md

@@ -1,6 +1,7 @@
 # Sosowa
 
-Sosowa Parser for Ruby 1.9.x
+創想話パーサー for Ruby 1.9.x
+samples/に各種サンプルが入っています。
 
 ## Requirements
 

data/lib/sosowa/scheme.rb

@@ -2,6 +2,10 @@ module Sosowa
   class Scheme
     protected
     
+    def initialize(element)
+      @element = element
+    end
+    
     def method_missing(action, *args)
       return @element[action.to_s.to_sym] rescue nil
     end
@@ -24,6 +28,7 @@ module Sosowa
     def fetch(log, key)
       params = Sosowa.serialize_parameter({:mode => :read, :log => log, :key => key})
       @page = @agent.get(URI.join(Sosowa::BASE_URL, params))
+      title = (@page/%{div[@class="header"] > h1})[0].inner_html.to_s.toutf8.strip
       tags = (@page/%{dl[@class="info"][1] > dd > a}).map{|t| t.inner_html.to_s.toutf8 }
       text = (@page/%{div[@class="contents ss"]})[0].inner_html.to_s.toutf8
       ps = (@page/%{div[@class="aft"]})[0].inner_html.to_s.toutf8
@@ -48,6 +53,7 @@ module Sosowa
         comments << comment
       end
       novel = {
+        :title => title,
         :text => text,
         :ps => ps,
         :author => author,
@@ -93,12 +99,7 @@ module Sosowa
     
   end
   
-  class Index < Scheme
-    def initialize(element)
-      super(@element)
-      @element = element
-    end
-    
+  class Index < Scheme    
     def fetch
       Novel.new(:log => @element[:log], :key => @element[:key])
     end

data/lib/sosowa/version.rb

@@ -1,3 +1,3 @@
 module Sosowa
-  VERSION = "0.1"
+  VERSION = "0.2"
 end

data/lib/sosowa.rb

@@ -10,6 +10,8 @@ require "sosowa/parser"
 
 module Sosowa
   BASE_URL = "http://coolier.sytes.net:8080/sosowa/ssw_l/"
+
+  protected
   
   # @param [Hash] parameter
   # @return [String] URL Serialized parameters
@@ -22,6 +24,8 @@ module Sosowa
     param = ant.inject(""){|k,v|k+"&#{v[0]}=#{URI.escape(v[1])}"}.sub!(/^&/,"?")
     return param ? param : ""
   end
+
+  public
   
   def self.get(args={})
     args[:log] ||= 0

data/samples/tf-idf.rb

@@ -0,0 +1,83 @@
+#!/usr/bin/env ruby
+# coding: utf-8
+# 創想話の最新版から適当なSSのテキストを取得してMeCab(+ 東方MeCab辞書)を用いて代表キーワード候補を名詞限定で選出し、TF-IDF法による特徴語抽出を行います。
+# 注意: ugigi gemが必要です
+
+require "MeCab"
+require "kconv"
+require "sosowa"
+require "ugigi"
+
+module MeCab
+  class Tagger
+    alias_method :parseToNode_org, :parseToNode
+    private :parseToNode_org
+
+    def parseToNode(*args)
+      node = parseToNode_org(*args)
+      nodes = []
+      while node
+        nodes.push(node)
+        node = node.next
+      end
+      return nodes[1, nodes.size - 2]
+    end
+  end
+  
+  class Node
+    alias_method :feature_org, :feature
+    alias_method :surface_org, :surface
+    private :feature_org
+    private :surface_org
+    
+    def feature ; feature_org.toutf8 end
+    def surface ; surface_org.toutf8 end
+  end
+end
+
+puts "東方MeCab辞書をダウンロード中..."
+system("curl -L https://github.com/oame/thdic-mecab/raw/master/pkg/thdic-mecab.dic > thdic-mecab.dic")
+
+puts "完了. MeCab::Taggerを初期化します"
+mecab = MeCab::Tagger.new("-u thdic-mecab.dic")
+
+novel = Sosowa.get.sample.fetch
+puts "-"*30
+puts novel.title
+puts "作者: #{novel.author.name}"
+puts "-"*30
+text = novel.plain
+tf = {}
+n = 15646.0
+puts "代表キーワード候補を抽出中..."
+tokens = mecab.parseToNode(text)
+tokens.each do |token|
+  next unless token.feature =~ /名詞/
+  tf[token.surface] ||= 0
+  tf[token.surface] += 1
+end
+
+puts "代表キーワード候補数: #{tf.size}"
+
+tfidf_list = []
+tf.each do |e|
+  print "TF: #{e[0]} ... \t"
+  df = Ugigi.total_count(:free => e[0], :sswp => 0, :compe => 0)
+  if df == 0
+    print "N/A\n"
+    tfidf_list << [e[0], 0]
+    next
+  end
+  print "DF: #{df} \t"
+  tfidf = e[1] * Math.log(n/df)
+  print "TF-IDF: #{tfidf}\n"
+  tfidf_list << [e[0], tfidf]
+end
+tfidf_list = tfidf_list.sort{|a, b| b[1] <=> a[1]}
+
+puts "集計終わり！"
+
+10.times do |n|
+  l = tfidf_list[n]
+  puts "#{n+1}. #{l[0]} \tTF-IDF: #{l[1]}"
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sosowa
 version: !ruby/object:Gem::Version
-  version: '0.1'
+  version: '0.2'
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-07-10 00:00:00.000000000Z
+date: 2012-07-11 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
-  requirement: &70307433904340 !ruby/object:Gem::Requirement
+  requirement: &70146581283880 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,7 +21,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70307433904340
+  version_requirements: *70146581283880
 description: Sosowa Parser for Ruby
 email:
 - oame@oameya.com
@@ -39,6 +39,7 @@ files:
 - lib/sosowa/parser.rb
 - lib/sosowa/scheme.rb
 - lib/sosowa/version.rb
+- samples/tf-idf.rb
 - samples/token_segment.rb
 - sosowa.gemspec
 homepage: ''