middleman-blog-similar 1.1.1 → 2.0.0

data/lib/middleman-blog-similar/algorithm/word_frequency.rb

@@ -1,69 +0,0 @@
-# encoding: utf-8
-
-require 'fast-stemmer'
-require 'csv'
-
-# logic ported from https://plugins.trac.wordpress.org/browser/wordpress-23-related-posts-plugin/trunk/recommendations.php
-
-class Middleman::Blog::Similar::Algorithm::WordFrequency < ::Middleman::Blog::Similar::Algorithm
-  @@unigrams = nil
-  class << self
-    def unigrams_path
-      File.join File.dirname(__FILE__), 'unigrams.csv'
-    end
-    def unigrams
-      if @@unigrams.nil?
-        @@unigrams = {}
-        CSV.foreach(unigrams_path, { :col_sep => "\t" }) do|row|
-          @@unigrams[row[0]] = row[4].to_f if row.length == 5
-        end
-      end
-      @@unigrams
-    end
-  end
-  def distance a
-    d = 0xffffff
-    wf = a.similarity_algorithm.word_freq
-    word_freq.each do|word, freq|
-      if wf.has_key? word
-        d -= wf[word] * freq
-      end
-    end
-    d
-  end
-  def words
-    re = /[\t\s\n,\.、。　]/
-    article.untagged_body.split(re) + article.title.split(re)
-  end
-  def generate_word_freq
-    suitable_words = unigrams.dup
-    word_freq= {}
-    words.each do|word|
-      word.downcase!
-      word = word.stem if word =~ /^\w+$/
-      word_freq[word] ||= 0
-      word_freq[word] += 1
-    end
-    selected_words = {}
-    word_freq.each do|word, freq|
-      selected_words[word] = unigrams[word] * Math.sqrt(freq) if unigrams[word]
-    end
-    article.tags.each do|tag|
-      tag = tag.downcase.stem
-      word_freq[tag] ||= 0
-      word_freq[tag] += tag_weight
-    end
-    Hash[ word_freq.sort_by{|k, v| k }.sort_by{|k, v| v } ]
-  end
-  def word_freq
-    @word_freq ||= generate_word_freq
-  end
-  def generate_tags
-    generate_word_freq.keys.reverse
-  end
-  def tags
-    @tags ||= generate_tags
-  end
-  def tag_weight   ; 5                      ; end
-  def unigrams     ; self.class.unigrams    ; end
-end

data/lib/middleman-blog-similar/algorithm/word_frequency/mecab.rb

@@ -1,22 +0,0 @@
-# -*- coding: UTF-8 -*-
-
-require 'middleman-blog-similar/algorithm/word_frequency'
-
-class Middleman::Blog::Similar::Algorithm::WordFrequency::Mecab < ::Middleman::Blog::Similar::Algorithm::WordFrequency
-  class CommandNotFound < StandardError; end
-  def words
-    res = []
-    IO.popen("mecab 2>/dev/null", 'r+') {|f|
-      f.puts article.untagged_body
-      f.puts article.title
-      f.close_write
-      while line = f.gets
-        word, pos = line.split(/[\t\s]+/)
-        next unless pos
-        pos = pos.split(',')
-        res << word if pos[0] == '名詞' && %w{一般 固有名詞}.include?(pos[1])
-      end
-    }
-    res
-  end
-end

data/lib/middleman-blog-similar/algorithm/word_frequency/tree_tagger.rb

@@ -1,20 +0,0 @@
-require 'middleman-blog-similar/algorithm/word_frequency'
-
-class Middleman::Blog::Similar::Algorithm::WordFrequency::TreeTagger < ::Middleman::Blog::Similar::Algorithm::WordFrequency
-  class CommandNotFound < StandardError; end
-  def words
-    raise CommandNotFound.new "You need to tree tagger command with ENV['TREETAGGER_COMMAND']" unless ENV['TREETAGGER_COMMAND']
-    res = []
-    IO.popen("#{ ENV['TREETAGGER_COMMAND'] } 2>/dev/null", 'r+') {|f|
-      f.puts article.untagged_body
-      f.puts article.title
-      f.close_write
-      while line = f.gets
-        word, pos = line.split(/\s+/)
-        # http://courses.washington.edu/hypertxt/csar-v02/penntable.html
-        res << word if %w{NN JJ NP}.include? pos[0..2]
-      end
-    }
-    res
-  end
-end

data/spec/helper_spec.rb

@@ -1,4 +0,0 @@
-require 'spec_helper'
-
-describe "Middleman::Blog::BlogArticle" do
-end

data/spec/middleman-blog-similar/algorithm/damerau_levenshtein_spec.rb

@@ -1,42 +0,0 @@
-require 'spec_helper'
-require 'middleman-blog-similar/algorithm/damerau_levenshtein'
-
-describe 'Middleman::Blog::Similar::Algorithm::DamerauLevenshtein' do
-  let(:app)     {
-    middleman_app('test-app') {
-      activate :similar, :algorithm => :damerau_levenshtein
-    }
-  }
-  let(:article) { app.sitemap.find_resource_by_destination_path '/2014/05/08/article0.html' }
-  let(:algorithm)  { article.similarity_algorithm }
-  describe(:app) {
-    describe(:similarity_algorithm) {
-      subject { app.similarity_algorithm }
-      it { should be ::Middleman::Blog::Similar::Algorithm::DamerauLevenshtein  }
-    }
-  }
-  describe(:similarity_algorithm) {
-    subject { algorithm }
-    it { should be_a_kind_of ::Middleman::Blog::Similar::Algorithm::DamerauLevenshtein }
-    describe(:similar_articles) {
-      subject { algorithm.similar_articles.map(&:url) }
-      it {
-        should eq [
-          "/2014/05/13/article5.html",
-          "/2014/05/09/article1.html",
-          "/2014/05/12/article4.html",
-          "/2014/05/14/article6.html",
-          "/2014/05/10/article2.html",
-          "/2014/05/11/article3.html"
-        ]
-      }
-    }
-  }
-  describe(:article) {
-    describe(:similarity_algorithm) {
-      subject { algorithm }
-      it { should be_a_kind_of ::Middleman::Blog::Similar::Algorithm::DamerauLevenshtein }
-    }
-  }
-
-end

data/spec/middleman-blog-similar/algorithm/levenshtein_spec.rb

@@ -1,42 +0,0 @@
-require 'spec_helper'
-require 'middleman-blog-similar/algorithm/levenshtein'
-
-describe 'Middleman::Blog::Similar::Algorithm::Levenshtein' do
-  let(:app)     {
-    middleman_app('test-app') {
-      activate :similar, :algorithm => :levenshtein
-    }
-  }
-  let(:article) { app.sitemap.find_resource_by_destination_path '/2014/05/08/article0.html' }
-  let(:algorithm)  { article.similarity_algorithm }
-  describe(:app) {
-    describe(:similarity_algorithm) {
-      subject { app.similarity_algorithm }
-      it { should be ::Middleman::Blog::Similar::Algorithm::Levenshtein  }
-    }
-  }
-  describe(:similarity_algorithm) {
-    subject { algorithm }
-    it { should be_a_kind_of ::Middleman::Blog::Similar::Algorithm::Levenshtein }
-    describe(:similar_articles) {
-      subject { algorithm.similar_articles.map(&:url) }
-      it {
-        should eq [
-          "/2014/05/13/article5.html",
-          "/2014/05/09/article1.html",
-          "/2014/05/12/article4.html",
-          "/2014/05/14/article6.html",
-          "/2014/05/10/article2.html",
-          "/2014/05/11/article3.html"
-        ]
-      }
-    }
-  }
-  describe(:article) {
-    describe(:similarity_algorithm) {
-      subject { algorithm }
-      it { should be_a_kind_of ::Middleman::Blog::Similar::Algorithm::Levenshtein }
-    }
-  }
-
-end

data/spec/middleman-blog-similar/algorithm/word_frequency/mecab_spec.rb

@@ -1,41 +0,0 @@
-# -*- coding: UTF-8 -*-
-
-require 'spec_helper'
-require 'middleman-blog-similar/algorithm/word_frequency/mecab'
-
-describe 'Middleman::Blog::Similar::Algorithm::WordFrequency::Mecab' do
-  let(:app)     {
-    middleman_app('test-app') {
-      activate :similar, :algorithm => :'word_frequency/mecab'
-    }
-  }
-  let(:article) { app.sitemap.find_resource_by_destination_path '/2014/05/11/article3.html' }
-  let(:algorithm)  { article.similarity_algorithm }
-  describe(:app) {
-    describe(:similarity_algorithm) {
-      subject { app.similarity_algorithm }
-      it { should be ::Middleman::Blog::Similar::Algorithm::WordFrequency::Mecab  }
-    }
-  }
-  describe(:similarity_algorithm) {
-    subject { algorithm }
-    it { should be_a_kind_of ::Middleman::Blog::Similar::Algorithm::WordFrequency::Mecab }
-  }
-  describe(:tags) {
-    describe(:output) {
-      if %x{which mecab}
-        subject { algorithm.tags }
-        it { should eq ["fox", "国家", "隙", "教師", "悪口", "尻", "坊ちゃん", "時分", "向", "叫び", "人間", "ネルソン", "この世", "西洋", "expect", "articl"] }
-      else
-        pending "mecab is not installed."
-      end
-    }
-  }
-  describe(:article) {
-    describe(:similarity_algorithm) {
-      subject { algorithm }
-      it { should be_a_kind_of ::Middleman::Blog::Similar::Algorithm::WordFrequency::Mecab }
-    }
-  }
-
-end

data/spec/middleman-blog-similar/algorithm/word_frequency/tree_tagger_spec.rb

@@ -1,52 +0,0 @@
-require 'spec_helper'
-require 'middleman-blog-similar/algorithm/word_frequency/tree_tagger'
-
-describe 'Middleman::Blog::Similar::Algorithm::WordFrequency::TreeTagger' do
-  let(:app)     {
-    middleman_app('test-app') {
-      activate :similar, :algorithm => :'word_frequency/tree_tagger'
-    }
-  }
-  let(:article) { app.sitemap.find_resource_by_destination_path '/2014/05/10/article2.html' }
-  let(:algorithm)  { article.similarity_algorithm }
-  describe(:app) {
-    describe(:similarity_algorithm) {
-      subject { app.similarity_algorithm }
-      it { should be ::Middleman::Blog::Similar::Algorithm::WordFrequency::TreeTagger  }
-    }
-  }
-  describe(:similarity_algorithm) {
-    subject { algorithm }
-    it { should be_a_kind_of ::Middleman::Blog::Similar::Algorithm::WordFrequency::TreeTagger }
-  }
-  describe(:tags) {
-    describe(:output) {
-      if ENV['TREETAGGER_COMMAND']
-        subject { algorithm.tags }
-        it { should eq ["quick", "fox", "dog", "brown", "lazi", "articl"] }
-      else
-        pending "ENV['TREETAGGER_COMMAND'] not set."
-      end
-    }
-    context('if command path is not set') {
-      subject { -> { algorithm.tags } }
-      before {
-        @cmd = ENV['TREETAGGER_COMMAND']
-        ENV['TREETAGGER_COMMAND'] = nil
-      }
-      after {
-        ENV['TREETAGGER_COMMAND'] = @cmd if @cmd
-      }
-      describe('raises error') {
-        it { should raise_error Middleman::Blog::Similar::Algorithm::WordFrequency::TreeTagger::CommandNotFound }
-      }
-    }
-  }
-  describe(:article) {
-    describe(:similarity_algorithm) {
-      subject { algorithm }
-      it { should be_a_kind_of ::Middleman::Blog::Similar::Algorithm::WordFrequency::TreeTagger }
-    }
-  }
-
-end

data/spec/middleman-blog-similar/algorithm/word_frequency_spec.rb

@@ -1,73 +0,0 @@
-require 'spec_helper'
-require 'middleman-blog-similar/algorithm/word_frequency'
-
-describe 'Middleman::Blog::Similar::Algorithm::WordFrequency' do
-  let(:app)     {
-    middleman_app('test-app') {
-      activate :similar, :algorithm => :word_frequency
-    }
-  }
-  let(:article) { app.sitemap.find_resource_by_destination_path '/2014/05/10/article2.html' }
-  let(:algorithm)  { article.similarity_algorithm }
-  describe(:app) {
-    describe(:similarity_algorithm) {
-      subject { app.similarity_algorithm }
-      it { should be ::Middleman::Blog::Similar::Algorithm::WordFrequency  }
-    }
-  }
-  describe(:algorithm) {
-    subject { algorithm }
-    it { should be_a_kind_of ::Middleman::Blog::Similar::Algorithm::WordFrequency }
-  }
-  describe(:unigrams) {
-    describe('length of keys') {
-      subject { algorithm.unigrams.keys.length }
-      it { should be 21089 }
-    }
-    describe('class') {
-      subject { algorithm.unigrams }
-      it { should be_a_kind_of Hash }
-    }
-  }
-  describe(:similar_articles) {
-    subject { algorithm.similar_articles.map(&:url) }
-    it {
-      should eq [
-        "/2014/05/11/article3.html",
-        "/2014/05/08/article0.html",
-        "/2014/05/12/article4.html",
-        "/2014/05/13/article5.html",
-        "/2014/05/09/article1.html",
-        "/2014/05/14/article6.html"
-      ]
-    }
-  }
-  describe(:tags) {
-    subject { algorithm.tags }
-    it { should eq ["fox", "quick", "dog", "brown", "the", "jump", "lazi", "over", "articl", "2"] }
-  }
-  describe(:word_freq) {
-    subject { algorithm.word_freq }
-    it {
-      should eq({
-       "brown" => 2,
-       "dog" => 3,
-       "fox" => 6,
-       "jump" => 1,
-       "lazi" => 1,
-       "over" => 1,
-       "quick" => 6,
-       "the" => 2,
-       "2" => 1,
-       "articl" => 1
-      })
-    }
-  }
-  describe(:article) {
-    describe(:similarity_algorithm) {
-      subject { algorithm }
-      it { should be_a_kind_of ::Middleman::Blog::Similar::Algorithm::WordFrequency }
-    }
-  }
-
-end

data/spec/middleman-blog-similar/algorithm_spec.rb

@@ -1,40 +0,0 @@
-require 'spec_helper'
-require 'middleman-blog-similar/algorithm/word_frequency'
-
-describe 'Middleman::Blog::Similar::Algorithm' do
-  let(:app)     {
-    middleman_app('test-app') {
-      activate :similar
-    }
-  }
-  let(:article) { app.sitemap.find_resource_by_destination_path '/2014/05/08/article0.html' }
-  let(:algorithm)  { article.similarity_algorithm }
-  describe(:app) {
-    describe(:similarity_algorithm) {
-      subject { app.similarity_algorithm }
-      it { should be ::Middleman::Blog::Similar::Algorithm::WordFrequency  }
-    }
-  }
-  describe(:similarity_algorithm) {
-    describe(:articles) {
-      subject { algorithm.articles.map(&:url) }
-      it {
-        should eq [
-          "/2014/05/14/article6.html",
-          "/2014/05/13/article5.html",
-          "/2014/05/12/article4.html",
-          "/2014/05/11/article3.html",
-          "/2014/05/10/article2.html",
-          "/2014/05/09/article1.html",
-          "/2014/05/08/article0.html"
-        ]
-      }
-    }
-  }
-  describe(:article) {
-    describe(:similarity_algorithm) {
-      subject { algorithm }
-      it { should be_a_kind_of ::Middleman::Blog::Similar::Algorithm::WordFrequency  }
-    }
-  }
-end