jekyll-related-blog-posts 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: faa98f92d223adef6735d0c20410214d48c6b08fd7ef76acd7797eccb8200588
-  data.tar.gz: 6a59b5733d1e62387510cdf3ea96c353a522d3ff231d6c8de059b454ca236f0e
+  metadata.gz: 3ec8e865c5b63044c782f0be8b21b91fcd97ba7ad4f49bcd76c77d4e7b560ccf
+  data.tar.gz: ab1be4898ff69303b4620901782851c76934eaeebefbbc1e8766b4cade444ba2
 SHA512:
-  metadata.gz: bdd965321a679ec0d709d0424f5c70baa58541831479b4b983a3539af980a4b127314592c37d3877090eacef7137e567b613ffc504399f5e9ec7eb64100940aa
-  data.tar.gz: 03d96f7be1c06510090c6b085f3975195d0f0a3857d17050ba080cb977732a334b93b99aff9d3dfc3cc0c5eb336ff4a6f5fb94ada21ee3a7f09822312e78712c
+  metadata.gz: cf21deb16f506691cbc3733a3dbd40e332f31b1d6f7365cd3349c844df9a454c07386d0b8adc6506c7361e320cb2dd87a3908fd94a57feda86cc1b75ccb501d1
+  data.tar.gz: e464db2de9d96407d0e7d9888db372d76427005cbeb24a3f15261c31121f802bae0a6659a742f8e776179b4daab477b06cb82e2c26347add28d77224ea31dabe

data/lib/jekyll-related-blog-posts.rb

@@ -1,96 +1,62 @@
 require 'rubygems'
 require 'jekyll'
-require 'singleton'
-require 'tokenizer'
-require 'yaml'
-require 'liquid'
 require 'fast_stemmer'
 require 'stopwords'
 require 'pqueue'
 require 'nmatrix'
-require 'nmatrix/lapacke'
 
-module Amadeusz
+module SangsooNam
 module Jekyll
-  class RelatedPosts
-    include Singleton
-
+  class TFIDFRelatedPosts
     def initialize
-      @posts = Array.new
+      @docs = Array.new
       @keywords = Array.new
-      @tokenizer = Tokenizer::WhitespaceTokenizer.new(:en)
+      @tags_and_categories = Array.new
       @stopwords_filter = Stopwords::Snowball::Filter.new('en')
     end
 
     def add_post(post)
-      post = {
-        url: post.url,
-        title: post.data['title'].dup,
-        content: (stem(post.content) + stem(post.data['title']))
+      tags = post.data['tags'].map { |e| "@tag:" + e }.map(&:to_sym)
+      categories = post.data['categories'].map { |e| "@category:" + e }.map(&:to_sym)
+      doc = {
+        post: post,
+        content: (stem(post.data['title']) + tags + categories)
       }
-
-      @posts << post
-      @keywords += post[:content]
-      @keywords.uniq!
+      @docs << doc
+      @keywords += doc[:content]
+      @tags_and_categories += tags + categories
     end
 
-    def build!(site)
-      conf = config(site)
-      @weights = keywords_weights(conf['weights'])
-      related = find_releated(conf['max_count'], conf['min_score'], conf['accuracy'])
-      template = Liquid::Template.parse(File.read(template_path(site)))
-
-      @posts.each do |post|
-        filename = File.join(site.config['destination'], post[:url])
-        filename = File.join(filename, 'index.html') if File.directory? filename
-        rendered = File.read(filename)
-
-        output = template.render('related_posts' => related[post])
+    def build(site)
+      @keywords.uniq!
+      @tags_and_categories.uniq!
+      @weights = custom_weights(@tags_and_categories)
+      related = build_related_docs_with_score(site.config['related_posts_count'] || 4)
 
-        rendered.gsub! '<related-posts />', output
-        File.write(filename, rendered)
+      @docs.each do |doc|
+        doc[:post].instance_variable_set(:@related_posts,related[doc].map { |x| x[:post] })
       end
     end
 
     private
 
-    def config(site)
-      builtin_file = File.join(File.absolute_path(File.dirname(__FILE__)), '_config.yml')
-      defaults = YAML.load_file(builtin_file)
-
-      defaults['related'].merge(site.config['related'] || {})
-    end
-
-    def template_path(site)
-      site_file = File.join(site.config['source'], site.config['layouts_dir'], 'related.html')
-      builtin_file = File.join(File.absolute_path(File.dirname(__FILE__)), 'related.html')
-
-      if File.exist? site_file
-        site_file
-      else
-        builtin_file
-      end
-    end
-
-    def find_releated(count = 5, min_score = -10.0, accuracy = 1.0)
-      dc = document_correleation(accuracy)
+    def build_related_docs_with_score(count = 8)
+      dc = document_correleation
       result = Hash.new
-      count = [count, @posts.size].min
+      count = [count, @docs.size].min
 
-      @posts.each_with_index do |post, index|
-        queue = PQueue.new(dc.row(index).each_with_index.select{|s,_| s>=min_score}) do |a, b|
+      @docs.each_with_index do |doc, index|
+        queue = PQueue.new(dc.row(index).each_with_index) do |a, b|
           a[0] > b[0]
         end
 
-        result[post] = []
+        result[doc] = []
         count.times do
           score, id = queue.pop
-          break unless score
           begin
-            result[post] << {
-              'score' => score,
-              'url' => @posts[id][:url],
-              'title' => @posts[id][:title]
+            result[doc] << {
+              score: score,
+              post: @docs[id][:post]
             }
           rescue
             break
@@ -101,27 +67,8 @@ module Jekyll
       return result
     end
 
-    def lsi(matrix, accuracy)
-      degree = (@keywords.size * accuracy - 1).floor
-      u, sigma, vt = matrix.transpose.gesdd
-
-      u2 = u.slice(0..degree, 0..degree)
-      sigma_d = NMatrix.zeros([degree+1, @posts.size])
-      sigma.each_with_indices do |v, i, j|
-        break if i > degree
-        sigma_d[i, i] = v
-      end
-
-      return u2.dot(sigma_d).dot(vt).transpose
-    end
-
-    def document_correleation(accuracy = 1.0)
-      if accuracy == 1.0
-        scores = tfidf
-      else
-        scores = lsi(tfidf, accuracy)
-      end
-
+    def document_correleation()
+      scores = tfidf
       result = scores.dot(scores.transpose)
 
       result.each_with_indices do |_, u, v|
@@ -136,11 +83,11 @@ module Jekyll
     end
 
     def bag_of_words
-      result = NMatrix.new([@posts.size, @keywords.size], 0.0)
-      @max = NMatrix.new([@posts.size], 0.0)
+      result = NMatrix.new([@docs.size, @keywords.size], 0.0)
+      @max = NMatrix.new([@docs.size], 0.0)
 
       result.each_with_indices do |_, pi, ki|
-        result[pi, ki] = @posts[pi][:content].count(@keywords[ki])
+        result[pi, ki] = @docs[pi][:content].count(@keywords[ki])
 
         if result[pi, ki] > @max[pi]
           @max[pi] = result[pi, ki]
@@ -162,15 +109,11 @@ module Jekyll
       return result
     end
 
-    def keywords_weights(weights)
+    def custom_weights(terms, weight = 8.0)
       result = NMatrix.new([1, @keywords.size], 1.0)
 
-      weights.each do |word, weight|
-        keyword = word.to_s.stem.to_sym
-
-        next unless @keywords.include? keyword
-
-        result[0, @keywords.index(keyword)] = weight
+      terms.each do |term|
+        result[0, @keywords.index(term)] = weight
       end
 
       return result
@@ -202,20 +145,23 @@ module Jekyll
     end
 
     def stem(data)
-      tokenized = @tokenizer.tokenize(data.gsub(/[^a-z \t'_\-\n.,+]/i, '')).map(&:downcase)
+      data = data.gsub(/{%.+%}/, ' ') # Replace liquid templates
+      tokenized = data.scan(/\w+/).map(&:downcase)
       filtered = @stopwords_filter.filter(tokenized)
-      stemmed = filtered.map(&:stem).select{|s| not s.empty?}.map(&:to_sym)
-
+      stemmed = filtered.map(&:stem).select{|s| s.length > 1}.map(&:to_sym)
       return stemmed
     end
   end
 end
 end
 
-Jekyll::Hooks.register :posts, :pre_render do |post|
-  Amadeusz::Jekyll::RelatedPosts.instance.add_post(post)
-end
+Jekyll::Hooks.register :site, :pre_render do |site|
+  Jekyll.logger.info("Building TFIDF index...")
+  tfidf = SangsooNam::Jekyll::TFIDFRelatedPosts.new
+  site.posts.docs.each do |x|
+      tfidf.add_post(x)
+  end
 
-Jekyll::Hooks.register :site, :post_write do |site|
-  Amadeusz::Jekyll::RelatedPosts.instance.build! site
+  Jekyll.logger.info("Replaceing Related Posts...")
+  tfidf.build(site)
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: jekyll-related-blog-posts
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Manpreet singh