jekyll-related-blog-posts 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0c8751f960fdd690fb4a0c4c9899f9b425b9108cbf9cf5580f01e4afad33580b
-  data.tar.gz: ffcffde09d1e6c7ce0830295429b663c9577ac5b723c5039857ed3e23a7238b1
+  metadata.gz: faa98f92d223adef6735d0c20410214d48c6b08fd7ef76acd7797eccb8200588
+  data.tar.gz: 6a59b5733d1e62387510cdf3ea96c353a522d3ff231d6c8de059b454ca236f0e
 SHA512:
-  metadata.gz: 003dd525f02a5fd168430eb04d553070b7d77fce4433fc941b555402f95a65c9851a82abd1c190fee064d5b722d72ed668b28b81ce86f7d5e5851447fc6dfedb
-  data.tar.gz: 8986f4ec51761d582bcbb06e5bbb651e04a6f87f04dc7cb8e10e110cea9ab0c0dd38fbdac4edaa74741b1b894a83291c5e122c12c2c06c6e3adc6f4caf96a1d2
+  metadata.gz: bdd965321a679ec0d709d0424f5c70baa58541831479b4b983a3539af980a4b127314592c37d3877090eacef7137e567b613ffc504399f5e9ec7eb64100940aa
+  data.tar.gz: 03d96f7be1c06510090c6b085f3975195d0f0a3857d17050ba080cb977732a334b93b99aff9d3dfc3cc0c5eb336ff4a6f5fb94ada21ee3a7f09822312e78712c

data/lib/jekyll-related-blog-posts.rb

@@ -1,62 +1,96 @@
 require 'rubygems'
 require 'jekyll'
+require 'singleton'
+require 'tokenizer'
+require 'yaml'
+require 'liquid'
 require 'fast_stemmer'
 require 'stopwords'
 require 'pqueue'
 require 'nmatrix'
+require 'nmatrix/lapacke'
 
-module SangsooNam
+module Amadeusz
 module Jekyll
-  class TFIDFRelatedPosts
+  class RelatedPosts
+    include Singleton
+
     def initialize
-      @docs = Array.new
+      @posts = Array.new
       @keywords = Array.new
-      @tags_and_categories = Array.new
+      @tokenizer = Tokenizer::WhitespaceTokenizer.new(:en)
       @stopwords_filter = Stopwords::Snowball::Filter.new('en')
     end
 
     def add_post(post)
-      tags = post.data['tags'].map { |e| "@tag:" + e }.map(&:to_sym)
-      categories = post.data['categories'].map { |e| "@category:" + e }.map(&:to_sym)
-      doc = {
-        post: post,
-        content: (stem(post.content) + stem(post.data['title']) + tags + categories)
+      post = {
+        url: post.url,
+        title: post.data['title'].dup,
+        content: (stem(post.content) + stem(post.data['title']))
       }
-      @docs << doc
-      @keywords += doc[:content]
-      @tags_and_categories += tags + categories
-    end
 
-    def build(site)
+      @posts << post
+      @keywords += post[:content]
       @keywords.uniq!
-      @tags_and_categories.uniq!
-      @weights = custom_weights(@tags_and_categories)
-      related = build_related_docs_with_score(site.config['related_posts_count'] || 4)
+    end
+
+    def build!(site)
+      conf = config(site)
+      @weights = keywords_weights(conf['weights'])
+      related = find_releated(conf['max_count'], conf['min_score'], conf['accuracy'])
+      template = Liquid::Template.parse(File.read(template_path(site)))
 
-      @docs.each do |doc|
-        doc[:post].instance_variable_set(:@related_posts,related[doc].map { |x| x[:post] })
+      @posts.each do |post|
+        filename = File.join(site.config['destination'], post[:url])
+        filename = File.join(filename, 'index.html') if File.directory? filename
+        rendered = File.read(filename)
+
+        output = template.render('related_posts' => related[post])
+
+        rendered.gsub! '<related-posts />', output
+        File.write(filename, rendered)
       end
     end
 
     private
 
-    def build_related_docs_with_score(count = 8)
-      dc = document_correleation
+    def config(site)
+      builtin_file = File.join(File.absolute_path(File.dirname(__FILE__)), '_config.yml')
+      defaults = YAML.load_file(builtin_file)
+
+      defaults['related'].merge(site.config['related'] || {})
+    end
+
+    def template_path(site)
+      site_file = File.join(site.config['source'], site.config['layouts_dir'], 'related.html')
+      builtin_file = File.join(File.absolute_path(File.dirname(__FILE__)), 'related.html')
+
+      if File.exist? site_file
+        site_file
+      else
+        builtin_file
+      end
+    end
+
+    def find_releated(count = 5, min_score = -10.0, accuracy = 1.0)
+      dc = document_correleation(accuracy)
       result = Hash.new
-      count = [count, @docs.size].min
+      count = [count, @posts.size].min
 
-      @docs.each_with_index do |doc, index|
-        queue = PQueue.new(dc.row(index).each_with_index) do |a, b|
+      @posts.each_with_index do |post, index|
+        queue = PQueue.new(dc.row(index).each_with_index.select{|s,_| s>=min_score}) do |a, b|
           a[0] > b[0]
         end
 
-        result[doc] = []
+        result[post] = []
         count.times do
           score, id = queue.pop
+          break unless score
           begin
-            result[doc] << {
-              score: score,
-              post: @docs[id][:post]
+            result[post] << {
+              'score' => score,
+              'url' => @posts[id][:url],
+              'title' => @posts[id][:title]
             }
           rescue
             break
@@ -67,8 +101,27 @@ module Jekyll
       return result
     end
 
-    def document_correleation()
-      scores = tfidf
+    def lsi(matrix, accuracy)
+      degree = (@keywords.size * accuracy - 1).floor
+      u, sigma, vt = matrix.transpose.gesdd
+
+      u2 = u.slice(0..degree, 0..degree)
+      sigma_d = NMatrix.zeros([degree+1, @posts.size])
+      sigma.each_with_indices do |v, i, j|
+        break if i > degree
+        sigma_d[i, i] = v
+      end
+
+      return u2.dot(sigma_d).dot(vt).transpose
+    end
+
+    def document_correleation(accuracy = 1.0)
+      if accuracy == 1.0
+        scores = tfidf
+      else
+        scores = lsi(tfidf, accuracy)
+      end
+
       result = scores.dot(scores.transpose)
 
       result.each_with_indices do |_, u, v|
@@ -83,11 +136,11 @@ module Jekyll
     end
 
     def bag_of_words
-      result = NMatrix.new([@docs.size, @keywords.size], 0.0)
-      @max = NMatrix.new([@docs.size], 0.0)
+      result = NMatrix.new([@posts.size, @keywords.size], 0.0)
+      @max = NMatrix.new([@posts.size], 0.0)
 
       result.each_with_indices do |_, pi, ki|
-        result[pi, ki] = @docs[pi][:content].count(@keywords[ki])
+        result[pi, ki] = @posts[pi][:content].count(@keywords[ki])
 
         if result[pi, ki] > @max[pi]
           @max[pi] = result[pi, ki]
@@ -109,11 +162,15 @@ module Jekyll
       return result
     end
 
-    def custom_weights(terms, weight = 8.0)
+    def keywords_weights(weights)
       result = NMatrix.new([1, @keywords.size], 1.0)
 
-      terms.each do |term|
-        result[0, @keywords.index(term)] = weight
+      weights.each do |word, weight|
+        keyword = word.to_s.stem.to_sym
+
+        next unless @keywords.include? keyword
+
+        result[0, @keywords.index(keyword)] = weight
       end
 
       return result
@@ -145,23 +202,20 @@ module Jekyll
     end
 
     def stem(data)
-      data = data.gsub(/{%.+%}/, ' ') # Replace liquid templates
-      tokenized = data.scan(/\w+/).map(&:downcase)
+      tokenized = @tokenizer.tokenize(data.gsub(/[^a-z \t'_\-\n.,+]/i, '')).map(&:downcase)
       filtered = @stopwords_filter.filter(tokenized)
-      stemmed = filtered.map(&:stem).select{|s| s.length > 1}.map(&:to_sym)
+      stemmed = filtered.map(&:stem).select{|s| not s.empty?}.map(&:to_sym)
+
       return stemmed
     end
   end
 end
 end
 
-Jekyll::Hooks.register :site, :pre_render do |site|
-  Jekyll.logger.info("Building TFIDF index...")
-  tfidf = SangsooNam::Jekyll::TFIDFRelatedPosts.new
-  site.posts.docs.each do |x|
-      tfidf.add_post(x)
-  end
+Jekyll::Hooks.register :posts, :pre_render do |post|
+  Amadeusz::Jekyll::RelatedPosts.instance.add_post(post)
+end
 
-  Jekyll.logger.info("Replaceing Related Posts...")
-  tfidf.build(site)
+Jekyll::Hooks.register :site, :post_write do |site|
+  Amadeusz::Jekyll::RelatedPosts.instance.build! site
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: jekyll-related-blog-posts
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Manpreet singh