linkterm 0.0.1

data/History.txt

@@ -0,0 +1,4 @@
+== 0.0.1 2007-07-08
+
+* 1 major enhancement:
+  * Initial release

data/License.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2007 Linkterm
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Manifest.txt

@@ -0,0 +1,16 @@
+History.txt
+License.txt
+Manifest.txt
+README.txt
+Rakefile
+lib/linkterm.rb
+lib/linkterm/version.rb
+scripts/txt2html
+setup.rb
+test/test_helper.rb
+test/test_linkterm.rb
+website/index.html
+website/index.txt
+website/javascripts/rounded_corners_lite.inc.js
+website/stylesheets/screen.css
+website/template.rhtml

data/README.txt

@@ -0,0 +1,24 @@
+README for linkterm
+===================
+
+== INSTALL
+ruby-mecabを使っているため、以下のものを事前にインストールする必要があります。
+
+* mecab
+* rb-mecab
+
+これらのインストールが完了すると、Rubyで
+require 'MeCab'できるようになります。
+
+OS Xでのインストール方法は
+http://ujihisa.nowa.jp/entry/0da2db2747
+にあります。
+
+== How to use
+コマンドラインインタフェースを用意しています。
+example/example.rbを実行して、適当なキーワードを入力してください。
+
+example.rbはexample/docs以下のファイルから知識を得、
+その知識に基づいて入力されたキーワードに対応する
+ルールを出力します。
+

data/Rakefile

@@ -0,0 +1,123 @@
+require 'rubygems'
+require 'rake'
+require 'rake/clean'
+require 'rake/testtask'
+require 'rake/packagetask'
+require 'rake/gempackagetask'
+require 'rake/rdoctask'
+require 'rake/contrib/rubyforgepublisher'
+require 'fileutils'
+require 'hoe'
+
+include FileUtils
+require File.join(File.dirname(__FILE__), 'lib', 'linkterm', 'version')
+
+AUTHOR = 'Linkterm'  # can also be an array of Authors
+EMAIL = "ujihisa gmail com"
+DESCRIPTION = "description of gem"
+GEM_NAME = 'linkterm' # what ppl will type to install your gem
+
+@config_file = "~/.rubyforge/user-config.yml"
+@config = nil
+def rubyforge_username
+  unless @config
+    begin
+      @config = YAML.load(File.read(File.expand_path(@config_file)))
+    rescue
+      puts <<-EOS
+ERROR: No rubyforge config file found: #{@config_file}"
+Run 'rubyforge setup' to prepare your env for access to Rubyforge
+ - See http://newgem.rubyforge.org/rubyforge.html for more details
+      EOS
+      exit
+    end
+  end
+  @rubyforge_username ||= @config["username"]
+end
+
+RUBYFORGE_PROJECT = 'linkterm' # The unix name for your project
+HOMEPATH = "http://#{RUBYFORGE_PROJECT}.rubyforge.org"
+DOWNLOAD_PATH = "http://rubyforge.org/projects/#{RUBYFORGE_PROJECT}"
+
+NAME = "linkterm"
+REV = nil 
+# UNCOMMENT IF REQUIRED: 
+# REV = `svn info`.each {|line| if line =~ /^Revision:/ then k,v = line.split(': '); break v.chomp; else next; end} rescue nil
+VERS = Linkterm::VERSION::STRING + (REV ? ".#{REV}" : "")
+CLEAN.include ['**/.*.sw?', '*.gem', '.config', '**/.DS_Store']
+RDOC_OPTS = ['--quiet', '--title', 'linkterm documentation',
+    "--opname", "index.html",
+    "--line-numbers", 
+    "--main", "README",
+    "--inline-source"]
+
+class Hoe
+  def extra_deps 
+    @extra_deps.reject { |x| Array(x).first == 'hoe' } 
+  end 
+end
+
+# Generate all the Rake tasks
+# Run 'rake -T' to see list of generated tasks (from gem root directory)
+hoe = Hoe.new(GEM_NAME, VERS) do |p|
+  p.author = AUTHOR 
+  p.description = DESCRIPTION
+  p.email = EMAIL
+  p.summary = DESCRIPTION
+  p.url = HOMEPATH
+  p.rubyforge_name = RUBYFORGE_PROJECT if RUBYFORGE_PROJECT
+  p.test_globs = ["test/**/test_*.rb"]
+  p.clean_globs |= CLEAN  #An array of file patterns to delete on clean.
+  
+  # == Optional
+  p.changes = p.paragraphs_of("History.txt", 0..1).join("\n\n")
+  #p.extra_deps = []     # An array of rubygem dependencies [name, version], e.g. [ ['active_support', '>= 1.3.1'] ]
+  #p.spec_extras = {}    # A hash of extra values to set in the gemspec.
+end
+
+CHANGES = hoe.paragraphs_of('History.txt', 0..1).join("\n\n")
+PATH    = (RUBYFORGE_PROJECT == GEM_NAME) ? RUBYFORGE_PROJECT : "#{RUBYFORGE_PROJECT}/#{GEM_NAME}"
+hoe.remote_rdoc_dir = File.join(PATH.gsub(/^#{RUBYFORGE_PROJECT}\/?/,''), 'rdoc')
+
+desc 'Generate website files'
+task :website_generate do
+  Dir['website/**/*.txt'].each do |txt|
+    sh %{ ruby scripts/txt2html #{txt} > #{txt.gsub(/txt$/,'html')} }
+  end
+end
+
+desc 'Upload website files to rubyforge'
+task :website_upload do
+  host = "#{rubyforge_username}@rubyforge.org"
+  remote_dir = "/var/www/gforge-projects/#{PATH}/"
+  local_dir = 'website'
+  sh %{rsync -aCv #{local_dir}/ #{host}:#{remote_dir}}
+end
+
+desc 'Generate and upload website files'
+task :website => [:website_generate, :website_upload, :publish_docs]
+
+desc 'Release the website and new gem version'
+task :deploy => [:check_version, :website, :release] do
+  puts "Remember to create SVN tag:"
+  puts "svn copy svn+ssh://#{rubyforge_username}@rubyforge.org/var/svn/#{PATH}/trunk " +
+    "svn+ssh://#{rubyforge_username}@rubyforge.org/var/svn/#{PATH}/tags/REL-#{VERS} "
+  puts "Suggested comment:"
+  puts "Tagging release #{CHANGES}"
+end
+
+desc 'Runs tasks website_generate and install_gem as a local deployment of the gem'
+task :local_deploy => [:website_generate, :install_gem]
+
+task :check_version do
+  unless ENV['VERSION']
+    puts 'Must pass a VERSION=x.y.z release version'
+    exit
+  end
+  unless ENV['VERSION'] == VERS
+    puts "Please update your version.rb to match the release version, currently #{VERS}"
+    exit
+  end
+end
+
+

data/lib/linkterm/version.rb

@@ -0,0 +1,9 @@
+module Linkterm #:nodoc:
+  module VERSION #:nodoc:
+    MAJOR = 0
+    MINOR = 0
+    TINY  = 1
+
+    STRING = [MAJOR, MINOR, TINY].join('.')
+  end
+end

data/lib/linkterm.rb

@@ -0,0 +1,212 @@
+$KCODE = 'u'
+require 'MeCab'
+require 'pp'
+require 'set'
+require 'mathn'
+require 'linktermlisc'
+require 'nkf'
+
+class Array
+  def to_set
+    Set.new self
+  end
+
+  def parm
+    '[ [x,y] | x <- self, y <- self, x != y]'.lisc_do binding
+  end
+
+  def to_hash
+    Hash[*self.flatten]
+  end
+end
+
+module Enumerable
+  def partitions
+    self.inject({}) {|hash, i| (hash[yield(i)] ||= []) << i; hash }.
+      map {|key, value| value }
+  end
+end
+
+module MeCab
+  class Node
+    include Enumerable
+
+    def category
+      return self.feature.split(/,/)[0]
+    end
+
+    def each(&b)
+      b[self]
+      self.next.each(&b) if self.next
+    end
+  end
+end
+
+# 進捗経過を出力するためのもの
+# example:
+#   cs = %w(the quick silver)
+#   counter = Counter.new(a.size)
+#   cs.each do |c|
+#     counter.display!
+#     puts c
+#   end
+class Counter
+  def initialize(max, message = nil, output = STDOUT)
+    @i   = 0
+    @max = max
+    @message = message.nil? ? '' : "[#{message}]: "
+  end
+
+  def display!
+    output.puts "#{@message} #{@i += 1}/#{@max}"
+  end
+end
+
+#######################################################################
+module Linkterm
+  class Base
+    def initialize(options = {})
+      @doc_dir = options[:doc_dir] || 'docs'
+
+      @documents = Documents.new Dir.open(@doc_dir).reject {|f| /^\.+$/ =~ f }.map {|f| "#{@doc_dir}/" + f }
+    end
+
+    def idf(word)
+      unless @documents_count
+        all_count = Hash.new(0)
+        counter = Counter.new(@documents.filenames.length, 'idf')
+        @documents.each do |document|
+          counter.display!
+          words = Set.new []
+          document.each {|paragraph|
+            paragraph.each {|node|
+              words << node.surface.downcase if node.category == '名詞' and
+                node.surface.length > 1 and
+                not /^\W+$/ =~ node.surface and
+                not /^\d+$/ =~ node.surface
+            }
+          }
+          words.each {|word| all_count[word] += 1 }
+        end
+        @documents_count = all_count
+      end
+
+      Math.log( @documents.filenames.length / @documents_count[word] )
+    end
+
+    def rule_table
+      table = []
+      counter = Counter.new(@documents.filenames.size, 'trans to table')
+      #@documents.each(0..3) do |document|
+      @documents.each() do |document|
+        counter.display!
+        document.each {|paragraph|
+          hash = Hash.new 0
+          paragraph.each {|node|
+            hash[node.surface.downcase] += 1 if node.category == '名詞' and
+              node.surface.length > 2 and
+              not /^\W+$/ =~ node.surface and
+              not /^\d+$/ =~ node.surface
+          }
+          word_count = hash
+          # TF-IDF値の高いものみ取り出す
+          word_count = word_count.sort_by {|word, count| count * idf(word) }.reverse[0...3].to_hash
+          #word_count = word_count.select {|word, count| count * idf(word) > 1.5 }.to_hash
+
+          word_count = word_count.map {|word, _| word }.sort
+          table << word_count unless word_count.empty?
+        }
+      end
+
+      # rule induction
+      table = table.map {|line|
+        arr = line.partitions {|term| (/^[a-z_]+$/ =~ term) ? :eng : (term.length > 10) ? :long : :short }
+        arr << [] until arr.size >= 3
+        arr
+      }
+
+      array = (0...table.first.length).to_a.parm
+      counter = Counter.new(array.length, 'induce rule')
+      rules = array.map {|c1, c2|
+        counter.display!
+        pattern = '[ [x,y] | x <- table.map{|u| u[c1] }.to_set, y <- table.map{|u| u[c2] }.to_set, !x.empty?, !y.empty? ]'.lisc_do binding
+        pattern.map {|a1, a2|
+          spt = table.select {|u| u[c1] == a1 and u[c2] == a2 }.length / table.select {|u| u[c1] == a1 }.length
+          {:a1 => a1, :a2 => a2, :spt => spt}
+        }
+      }.flatten.select {|r| r[:spt] >= 1/2 }
+
+      rules
+    end
+  end
+
+  class Documents
+    attr_reader :filenames
+    include Enumerable
+
+    def initialize(filenames)
+      @filenames = filenames
+    end
+
+    def each(range = 0...@filenames.size)
+      filenames[range].each {|filename|
+        document = Document.new(File.open(filename, 'r') {|io| io.read })
+        yield document
+      }
+    end
+  end
+
+  class Document
+    include Enumerable
+
+    def initialize(text)
+      @text = text
+    end
+
+    def each
+      ps = @text.gsub(/<.*?>|nbsp/, ' ').split(/(\r?\n\s*){2}/)
+      # 短すぎるパラグラフは連結する
+      ps2 = ps.inject(['']) {|result, item|
+        (item.size > 150 ? result : result.last << ' ') << item
+        result
+      }
+
+      ps2.each {|paragraph|
+        yield Paragraph.new(paragraph)
+      }
+    end
+  end
+
+  class Paragraph
+    include Enumerable
+
+    def initialize(text)
+      @text = text
+    end
+
+    def each
+      mecab = MeCab::Tagger.new("-Ochasen")
+      mecab.parseToNode(NKF.nkf('-w', @text)).each {|node|
+        yield node
+      }
+    end
+  end
+end
+
+
+# CUI
+linkterm = Linkterm::Base.new
+rule_table = linkterm.rule_table
+pp rule_table
+puts "input?"
+loop do
+  str = NKF.nkf('-w', gets.chop)
+  rule_table.select {|record|
+    record[:a1].any? {|r| /^#{str}/ =~ r } ||
+    record[:a2].any? {|r| /^#{str}/ =~ r }
+  }.each {|record|
+    puts "[#{record[:a1].join(', ')}]\t\t-> [#{record[:a2].join(', ')}]"
+  }
+end
+
+

data/scripts/txt2html

@@ -0,0 +1,67 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'redcloth'
+require 'syntax/convertors/html'
+require 'erb'
+require File.dirname(__FILE__) + '/../lib/linkterm/version.rb'
+
+version  = Linkterm::VERSION::STRING
+download = 'http://rubyforge.org/projects/linkterm'
+
+class Fixnum
+  def ordinal
+    # teens
+    return 'th' if (10..19).include?(self % 100)
+    # others
+    case self % 10
+    when 1: return 'st'
+    when 2: return 'nd'
+    when 3: return 'rd'
+    else    return 'th'
+    end
+  end
+end
+
+class Time
+  def pretty
+    return "#{mday}#{mday.ordinal} #{strftime('%B')} #{year}"
+  end
+end
+
+def convert_syntax(syntax, source)
+  return Syntax::Convertors::HTML.for_syntax(syntax).convert(source).gsub(%r!^<pre>|</pre>$!,'')
+end
+
+if ARGV.length >= 1
+  src, template = ARGV
+  template ||= File.dirname(__FILE__) + '/../website/template.rhtml'
+  
+else
+  puts("Usage: #{File.split($0).last} source.txt [template.rhtml] > output.html")
+  exit!
+end
+
+template = ERB.new(File.open(template).read)
+
+title = nil
+body = nil
+File.open(src) do |fsrc|
+  title_text = fsrc.readline
+  body_text = fsrc.read
+  syntax_items = []
+  body_text.gsub!(%r!<(pre|code)[^>]*?syntax=['"]([^'"]+)[^>]*>(.*?)</
>!m){
+    ident = syntax_items.length
+    element, syntax, source = $1, $2, $3
+    syntax_items << "<#{element} class='syntax'>#{convert_syntax(syntax, source)}</#{element}>"
+    "syntax-temp-#{ident}"
+  }
+  title = RedCloth.new(title_text).to_html.gsub(%r!<.*?>!,'').strip
+  body = RedCloth.new(body_text).to_html
+  body.gsub!(%r!(?:<pre><code>)?syntax-temp-(d+)(?:</code></pre>)?!){ syntax_items[$1.to_i] }
+end
+stat = File.stat(src)
+created = stat.ctime
+modified = stat.mtime
+
+$stdout << template.result(binding)