middleman-blog-similar 1.0.0 → 1.1.0

data/lib/middleman-blog-similar/algorithm/word_frequency.rb

@@ -0,0 +1,69 @@
+# encoding: utf-8
+
+require 'fast-stemmer'
+require 'csv'
+
+# logic ported from https://plugins.trac.wordpress.org/browser/wordpress-23-related-posts-plugin/trunk/recommendations.php
+
+class Middleman::Blog::Similar::Algorithm::WordFrequency < ::Middleman::Blog::Similar::Algorithm
+  @@unigrams = nil
+  class << self
+    def unigrams_path
+      File.join File.dirname(__FILE__), 'unigrams.csv'
+    end
+    def unigrams
+      if @@unigrams.nil?
+        @@unigrams = {}
+        CSV.foreach(unigrams_path, { :col_sep => "\t" }) do|row|
+          @@unigrams[row[0]] = row[4].to_f if row.length == 5
+        end
+      end
+      @@unigrams
+    end
+  end
+  def distance a
+    d = 0xffffff
+    wf = a.similarity_algorithm.word_freq
+    word_freq.each do|word, freq|
+      if wf.has_key? word
+        d -= wf[word] * freq
+      end
+    end
+    d
+  end
+  def words
+    re = /[\t\s\n,\.、。　]/
+    article.untagged_body.split(re) + article.title.split(re)
+  end
+  def generate_word_freq
+    suitable_words = unigrams.dup
+    word_freq= {}
+    words.each do|word|
+      word.downcase!
+      word = word.stem if word =~ /^\w+$/
+      word_freq[word] ||= 0
+      word_freq[word] += 1
+    end
+    selected_words = {}
+    word_freq.each do|word, freq|
+      selected_words[word] = unigrams[word] * Math.sqrt(freq) if unigrams[word]
+    end
+    article.tags.each do|tag|
+      tag = tag.downcase.stem
+      word_freq[tag] ||= 0
+      word_freq[tag] += tag_weight
+    end
+    Hash[ word_freq.sort_by{|k, v| k }.sort_by{|k, v| v } ]
+  end
+  def word_freq
+    @word_freq ||= generate_word_freq
+  end
+  def generate_tags
+    generate_word_freq.keys.reverse
+  end
+  def tags
+    @tags ||= generate_tags
+  end
+  def tag_weight   ; 5                      ; end
+  def unigrams     ; self.class.unigrams    ; end
+end

data/lib/middleman-blog-similar/algorithm/word_frequency/mecab.rb

@@ -0,0 +1,20 @@
+# -*- coding: UTF-8 -*-
+
+require 'middleman-blog-similar/algorithm/word_frequency'
+
+class Middleman::Blog::Similar::Algorithm::WordFrequency::Mecab < ::Middleman::Blog::Similar::Algorithm::WordFrequency
+  class CommandNotFound < StandardError; end
+  def words
+    res = []
+    IO.popen("mecab 2>/dev/null", 'r+') {|f|
+      f.puts article.untagged_body
+      f.puts article.title
+      f.close_write
+      while line = f.gets
+        word, pos = line.split(/[\t\s]+/)
+        res << word if pos && pos.start_with?('名詞')
+      end
+    }
+    res
+  end
+end

data/lib/middleman-blog-similar/algorithm/word_frequency/tree_tagger.rb

@@ -0,0 +1,20 @@
+require 'middleman-blog-similar/algorithm/word_frequency'
+
+class Middleman::Blog::Similar::Algorithm::WordFrequency::TreeTagger < ::Middleman::Blog::Similar::Algorithm::WordFrequency
+  class CommandNotFound < StandardError; end
+  def words
+    raise CommandNotFound.new "You need to tree tagger command with ENV['TREETAGGER_COMMAND']" unless ENV['TREETAGGER_COMMAND']
+    res = []
+    IO.popen("#{ ENV['TREETAGGER_COMMAND'] } 2>/dev/null", 'r+') {|f|
+      f.puts article.untagged_body
+      f.puts article.title
+      f.close_write
+      while line = f.gets
+        word, pos = line.split(/\s+/)
+        # http://courses.washington.edu/hypertxt/csar-v02/penntable.html
+        res << word if %w{NN JJ NP}.include? pos[0..2]
+      end
+    }
+    res
+  end
+end

data/lib/middleman-blog-similar/blog_article_extensions.rb

@@ -1,14 +1,23 @@
-module Middleman
-  module Blog
-    module Similar
-      module BlogArticleExtensions
-        def similar_articles
-          if !@similar_articles && (engine = app.similarity_engine)
-            @similar_articles = engine.new(self).similar_articles
-          end
-          @similar_articles || []
-        end
-      end
+module Middleman::Blog::Similar::BlogArticleExtensions
+  def similar_articles
+    if !@similar_articles && similarity_algorithm
+      @similar_articles = similarity_algorithm.similar_articles
     end
+    @similar_articles || []
+  end
+  def words
+    unless @words && similarity_algorithm
+      @words = similarity_algorithm.words
+    end
+    @words
+  end
+  def similarity_algorithm
+    if !@similarity_algorithm && (algorithm = app.similarity_algorithm)
+      @similarity_algorithm = algorithm.new self
+    end
+    @similarity_algorithm
+  end
+  def untagged_body
+    body.gsub(/<[^>]*>/ui,'')
   end
 end

data/lib/middleman-blog-similar/extension.rb

@@ -1,25 +1,28 @@
 require 'middleman-blog-similar/blog_article_extensions'
 require 'middleman-blog-similar/helpers'
-require 'middleman-blog-similar/engines/base'
+require 'middleman-blog-similar/algorithm'
 
 module Middleman
   module Blog
     class SimilarExtension < ::Middleman::Extension
 
-      option :engine, :levenshtein, 'Similar lookup engine'
+      option :algorithm, :word_frequency, 'Similar lookup algorithm'
 
       self.defined_helpers = [ Middleman::Blog::Similar::Helpers ]
 
       def after_configuration
         require 'middleman-blog/blog_article'
         ::Middleman::Sitemap::Resource.send :include, Middleman::Blog::Similar::BlogArticleExtensions
-        engine = options[:engine].to_s
+        algorithm = options[:algorithm].to_s
         begin
-          require "middleman-blog-similar/engines/#{engine}"
-          engine = ::Middleman::Blog::Similar::Engines.const_get engine.camelize
-          app.set :similarity_engine, engine
+          require "middleman-blog-similar/algorithm/#{algorithm}"
+          ns = ::Middleman::Blog::Similar::Algorithm
+          algorithm.split('/').each do|n|
+            ns = ns.const_get n.camelize
+          end
+          app.set :similarity_algorithm, ns
         rescue LoadError => e
-          app.logger.error "Requested similar engine '#{engine}' not found."
+          app.logger.error "Requested similar algorithm '#{algorithm}' not found."
           raise e
         end
       end

data/lib/middleman-blog-similar/helpers.rb

@@ -1,15 +1,9 @@
-module Middleman
-  module Blog
-    module Similar
-      module Helpers
-        def similar_articles
-          if is_blog_article?
-            current_article.similar_articles
-          else
-            []
-          end
-        end
-      end
+module Middleman::Blog::Similar::Helpers
+  def similar_articles
+    if is_blog_article?
+      current_article.similar_articles
+    else
+      []
     end
   end
 end

data/lib/middleman-blog-similar/version.rb

@@ -1,7 +1,7 @@
 module Middleman
   module Blog
     module Similar
-      VERSION = "1.0.0"
+      VERSION = "1.1.0"
     end
   end
 end

data/middleman-blog-similar.gemspec

@@ -15,6 +15,7 @@ Gem::Specification.new do |s|
   s.files = `git ls-files -z`.split("\0")
   s.test_files = `git ls-files -z -- {fixtures,features,spec}/*`.split("\0")
   s.require_paths = ["lib"]
-  s.add_runtime_dependency("middleman-core", ["~> 3.2"])
-  s.add_runtime_dependency("middleman-blog", ["~> 3.5"])
+  s.add_runtime_dependency("middleman-core", ["~> 3.2"  ])
+  s.add_runtime_dependency("middleman-blog", ["~> 3.5"  ])
+  s.add_runtime_dependency("fast-stemmer",   ["~> 1.0.2"])
 end

data/spec/middleman-blog-similar/algorithm/damerau_levenshtein_spec.rb

@@ -0,0 +1,42 @@
+require 'spec_helper'
+require 'middleman-blog-similar/algorithm/damerau_levenshtein'
+
+describe 'Middleman::Blog::Similar::Algorithm::DamerauLevenshtein' do
+  let(:app)     {
+    middleman_app('test-app') {
+      activate :similar, :algorithm => :damerau_levenshtein
+    }
+  }
+  let(:article) { app.sitemap.find_resource_by_destination_path '/2014/05/08/article0.html' }
+  let(:algorithm)  { article.similarity_algorithm }
+  describe(:app) {
+    describe(:similarity_algorithm) {
+      subject { app.similarity_algorithm }
+      it { should be ::Middleman::Blog::Similar::Algorithm::DamerauLevenshtein  }
+    }
+  }
+  describe(:similarity_algorithm) {
+    subject { algorithm }
+    it { should be_a_kind_of ::Middleman::Blog::Similar::Algorithm::DamerauLevenshtein }
+    describe(:similar_articles) {
+      subject { algorithm.similar_articles.map(&:url) }
+      it {
+        should eq [
+          "/2014/05/13/article5.html",
+          "/2014/05/09/article1.html",
+          "/2014/05/12/article4.html",
+          "/2014/05/14/article6.html",
+          "/2014/05/10/article2.html",
+          "/2014/05/11/article3.html"
+        ]
+      }
+    }
+  }
+  describe(:article) {
+    describe(:similarity_algorithm) {
+      subject { algorithm }
+      it { should be_a_kind_of ::Middleman::Blog::Similar::Algorithm::DamerauLevenshtein }
+    }
+  }
+
+end

data/spec/middleman-blog-similar/algorithm/levenshtein_spec.rb

@@ -0,0 +1,42 @@
+require 'spec_helper'
+require 'middleman-blog-similar/algorithm/levenshtein'
+
+describe 'Middleman::Blog::Similar::Algorithm::Levenshtein' do
+  let(:app)     {
+    middleman_app('test-app') {
+      activate :similar, :algorithm => :levenshtein
+    }
+  }
+  let(:article) { app.sitemap.find_resource_by_destination_path '/2014/05/08/article0.html' }
+  let(:algorithm)  { article.similarity_algorithm }
+  describe(:app) {
+    describe(:similarity_algorithm) {
+      subject { app.similarity_algorithm }
+      it { should be ::Middleman::Blog::Similar::Algorithm::Levenshtein  }
+    }
+  }
+  describe(:similarity_algorithm) {
+    subject { algorithm }
+    it { should be_a_kind_of ::Middleman::Blog::Similar::Algorithm::Levenshtein }
+    describe(:similar_articles) {
+      subject { algorithm.similar_articles.map(&:url) }
+      it {
+        should eq [
+          "/2014/05/13/article5.html",
+          "/2014/05/09/article1.html",
+          "/2014/05/12/article4.html",
+          "/2014/05/14/article6.html",
+          "/2014/05/10/article2.html",
+          "/2014/05/11/article3.html"
+        ]
+      }
+    }
+  }
+  describe(:article) {
+    describe(:similarity_algorithm) {
+      subject { algorithm }
+      it { should be_a_kind_of ::Middleman::Blog::Similar::Algorithm::Levenshtein }
+    }
+  }
+
+end

data/spec/middleman-blog-similar/algorithm/word_frequency/mecab_spec.rb

@@ -0,0 +1,41 @@
+# -*- coding: UTF-8 -*-
+
+require 'spec_helper'
+require 'middleman-blog-similar/algorithm/word_frequency/mecab'
+
+describe 'Middleman::Blog::Similar::Algorithm::WordFrequency::Mecab' do
+  let(:app)     {
+    middleman_app('test-app') {
+      activate :similar, :algorithm => :'word_frequency/mecab'
+    }
+  }
+  let(:article) { app.sitemap.find_resource_by_destination_path '/2014/05/11/article3.html' }
+  let(:algorithm)  { article.similarity_algorithm }
+  describe(:app) {
+    describe(:similarity_algorithm) {
+      subject { app.similarity_algorithm }
+      it { should be ::Middleman::Blog::Similar::Algorithm::WordFrequency::Mecab  }
+    }
+  }
+  describe(:similarity_algorithm) {
+    subject { algorithm }
+    it { should be_a_kind_of ::Middleman::Blog::Similar::Algorithm::WordFrequency::Mecab }
+  }
+  describe(:tags) {
+    describe(:output) {
+      if %x{which mecab}
+        subject { algorithm.tags }
+        it { should eq ["fox","の","国家","さん","方","誰","私","坊ちゃん","立脚","西洋","矛盾","相違","発会","時分","昨日","講演","教師","拡張","悪口","尻","変","結果","開始","周旋","向","叫び","反駁","反抗","前","人間","ネルソン","よう","関係","なん","ため","それ","そう","院","この世","お話","隙","expect","articl","3"] }
+      else
+        pending "mecab is not installed."
+      end
+    }
+  }
+  describe(:article) {
+    describe(:similarity_algorithm) {
+      subject { algorithm }
+      it { should be_a_kind_of ::Middleman::Blog::Similar::Algorithm::WordFrequency::Mecab }
+    }
+  }
+
+end

data/spec/middleman-blog-similar/algorithm/word_frequency/tree_tagger_spec.rb

@@ -0,0 +1,52 @@
+require 'spec_helper'
+require 'middleman-blog-similar/algorithm/word_frequency/tree_tagger'
+
+describe 'Middleman::Blog::Similar::Algorithm::WordFrequency::TreeTagger' do
+  let(:app)     {
+    middleman_app('test-app') {
+      activate :similar, :algorithm => :'word_frequency/tree_tagger'
+    }
+  }
+  let(:article) { app.sitemap.find_resource_by_destination_path '/2014/05/10/article2.html' }
+  let(:algorithm)  { article.similarity_algorithm }
+  describe(:app) {
+    describe(:similarity_algorithm) {
+      subject { app.similarity_algorithm }
+      it { should be ::Middleman::Blog::Similar::Algorithm::WordFrequency::TreeTagger  }
+    }
+  }
+  describe(:similarity_algorithm) {
+    subject { algorithm }
+    it { should be_a_kind_of ::Middleman::Blog::Similar::Algorithm::WordFrequency::TreeTagger }
+  }
+  describe(:tags) {
+    describe(:output) {
+      if ENV['TREETAGGER_COMMAND']
+        subject { algorithm.tags }
+        it { should eq ["quick", "fox", "dog", "brown", "lazi", "articl"] }
+      else
+        pending "ENV['TREETAGGER_COMMAND'] not set."
+      end
+    }
+    context('if command path is not set') {
+      subject { -> { algorithm.tags } }
+      before {
+        @cmd = ENV['TREETAGGER_COMMAND']
+        ENV['TREETAGGER_COMMAND'] = nil
+      }
+      after {
+        ENV['TREETAGGER_COMMAND'] = @cmd if @cmd
+      }
+      describe('raises error') {
+        it { should raise_error Middleman::Blog::Similar::Algorithm::WordFrequency::TreeTagger::CommandNotFound }
+      }
+    }
+  }
+  describe(:article) {
+    describe(:similarity_algorithm) {
+      subject { algorithm }
+      it { should be_a_kind_of ::Middleman::Blog::Similar::Algorithm::WordFrequency::TreeTagger }
+    }
+  }
+
+end

data/spec/middleman-blog-similar/algorithm/word_frequency_spec.rb

@@ -0,0 +1,73 @@
+require 'spec_helper'
+require 'middleman-blog-similar/algorithm/word_frequency'
+
+describe 'Middleman::Blog::Similar::Algorithm::WordFrequency' do
+  let(:app)     {
+    middleman_app('test-app') {
+      activate :similar, :algorithm => :word_frequency
+    }
+  }
+  let(:article) { app.sitemap.find_resource_by_destination_path '/2014/05/10/article2.html' }
+  let(:algorithm)  { article.similarity_algorithm }
+  describe(:app) {
+    describe(:similarity_algorithm) {
+      subject { app.similarity_algorithm }
+      it { should be ::Middleman::Blog::Similar::Algorithm::WordFrequency  }
+    }
+  }
+  describe(:algorithm) {
+    subject { algorithm }
+    it { should be_a_kind_of ::Middleman::Blog::Similar::Algorithm::WordFrequency }
+  }
+  describe(:unigrams) {
+    describe('length of keys') {
+      subject { algorithm.unigrams.keys.length }
+      it { should be 21089 }
+    }
+    describe('class') {
+      subject { algorithm.unigrams }
+      it { should be_a_kind_of Hash }
+    }
+  }
+  describe(:similar_articles) {
+    subject { algorithm.similar_articles.map(&:url) }
+    it {
+      should eq [
+        "/2014/05/11/article3.html",
+        "/2014/05/08/article0.html",
+        "/2014/05/12/article4.html",
+        "/2014/05/13/article5.html",
+        "/2014/05/09/article1.html",
+        "/2014/05/14/article6.html"
+      ]
+    }
+  }
+  describe(:tags) {
+    subject { algorithm.tags }
+    it { should eq ["fox", "quick", "dog", "brown", "the", "jump", "lazi", "over", "articl", "2"] }
+  }
+  describe(:word_freq) {
+    subject { algorithm.word_freq }
+    it {
+      should eq({
+       "brown" => 2,
+       "dog" => 3,
+       "fox" => 6,
+       "jump" => 1,
+       "lazi" => 1,
+       "over" => 1,
+       "quick" => 6,
+       "the" => 2,
+       "2" => 1,
+       "articl" => 1
+      })
+    }
+  }
+  describe(:article) {
+    describe(:similarity_algorithm) {
+      subject { algorithm }
+      it { should be_a_kind_of ::Middleman::Blog::Similar::Algorithm::WordFrequency }
+    }
+  }
+
+end