jekyll-tfidf-related-posts 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: be9718e2724638708bdd9f40b2c9a6a8b9eb37a8
+  data.tar.gz: c26b700ff4ec87fbb228a835bd487d33c0754191
+SHA512:
+  metadata.gz: 9d5e168c31cc4c799174bb099006ac00321d130a09ed5445dbad5f56d6fd35c056e7e579f10ae191c523cf11678a03e8b4905268e24c491178a89a23aec25b42
+  data.tar.gz: 9f3d690f6be343d64a15793ec1edc521bbb83f828f4beff3d85d69b976e96691661b2285658d59d7c6694be2c3e652c33af669473aa30a84f305d6877b811631

data/.gitignore

@@ -0,0 +1,2 @@
+Gemfile.lock
+*.gem

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in jekyll-related-posts.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2019 Sangsoo Nam
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,13 @@
+# jekyll-tdidf-related-posts
+
+[![DUB](https://img.shields.io/dub/l/vibe-d.svg)]()
+
+[Jekyll](http://jekyllrb.com) plugin to show related posts based on the content, tags, and categories. The similarity is calculated using TF-IDF(term frequency-inverted document frequency). Since tags and categories are use-defined values, those are considered with higher weights than a content while calculating.
+
+### Configuration
+
+By default, there are 4 related posts. You can configure it in the `_config.yml`
+
+```
+related_posts_count: 8
+```

data/jekyll-tfidf-related-posts.gemspec

@@ -0,0 +1,23 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "jekyll-tfidf-related-posts"
+  spec.version       = "0.1.0"
+  spec.authors       = ["Sangsoo Nam"]
+  spec.email         = ["sangsoo.ted@gmail.com"]
+  spec.summary       = %q{Jekyll plugin to show related posts based on the content, tags, and categories.}
+  spec.description   = %q{[Jekyll](http://jekyllrb.com) plugin to show related posts based on the content, tags, and categories.  The similarity is calculated using TF-IDF(term frequency-inverted document frequency). Since tags and categories are use-defined values, those are considered with higher weights than a content while calculating.}
+  spec.homepage      = "https://github.com/SangsooNam/jekyll-tfidf-related-posts"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.require_paths = ["lib"]
+
+	spec.add_dependency "jekyll", "~> 3.0"
+  spec.add_dependency "stopwords-filter", "~> 0.4"
+  spec.add_dependency "fast-stemmer", "~> 1.0"
+  spec.add_dependency "pqueue", "~> 2.1"
+  spec.add_dependency "nmatrix", "~> 0.2"
+end

data/lib/jekyll-tfidf-related-posts.rb

@@ -0,0 +1,167 @@
+require 'rubygems'
+require 'jekyll'
+require 'fast_stemmer'
+require 'stopwords'
+require 'pqueue'
+require 'nmatrix'
+
+module SangsooNam
+module Jekyll
+  class TFIDFRelatedPosts
+    def initialize
+      @docs = Array.new
+      @keywords = Array.new
+      @tags_and_categories = Array.new
+      @stopwords_filter = Stopwords::Snowball::Filter.new('en')
+    end
+
+    def add_post(post)
+      tags = post.data['tags'].map { |e| "@tag:" + e }.map(&:to_sym)
+      categories = post.data['categories'].map { |e| "@category:" + e }.map(&:to_sym)
+      doc = {
+        post: post,
+        content: (stem(post.content) + stem(post.data['title']) + tags + categories)
+      }
+      @docs << doc
+      @keywords += doc[:content]
+      @tags_and_categories += tags + categories
+    end
+
+    def build(site)
+      @keywords.uniq!
+      @tags_and_categories.uniq!
+      @weights = custom_weights(@tags_and_categories)
+      related = build_related_docs_with_score(site.config['related_posts_count'] || 4)
+
+      @docs.each do |doc|
+        doc[:post].instance_variable_set(:@related_posts,related[doc].map { |x| x[:post] })
+      end
+    end
+
+    private
+
+    def build_related_docs_with_score(count = 8)
+      dc = document_correleation
+      result = Hash.new
+      count = [count, @docs.size].min
+
+      @docs.each_with_index do |doc, index|
+        queue = PQueue.new(dc.row(index).each_with_index) do |a, b|
+          a[0] > b[0]
+        end
+
+        result[doc] = []
+        count.times do
+          score, id = queue.pop
+          begin
+            result[doc] << {
+              score: score,
+              post: @docs[id][:post]
+            }
+          rescue
+            break
+          end
+        end
+      end
+
+      return result
+    end
+
+    def document_correleation()
+      scores = tfidf
+      result = scores.dot(scores.transpose)
+
+      result.each_with_indices do |_, u, v|
+        if u != v
+          result[u, v] /= (result[u, u] + result[v, v] - result[u, v])
+        else
+          result[u, v] = 0.0
+        end
+      end
+
+      return result
+    end
+
+    def bag_of_words
+      result = NMatrix.new([@docs.size, @keywords.size], 0.0)
+      @max = NMatrix.new([@docs.size], 0.0)
+
+      result.each_with_indices do |_, pi, ki|
+        result[pi, ki] = @docs[pi][:content].count(@keywords[ki])
+
+        if result[pi, ki] > @max[pi]
+          @max[pi] = result[pi, ki]
+        end
+      end
+
+      @bag_of_words = result.dup
+      return result
+    end
+
+    def term_frequency
+      result = bag_of_words
+
+      result.rows.times do |r|
+        result[r, 0..-1] *= @weights
+        result[r, 0..-1] /= @max[r]
+      end
+
+      return result
+    end
+
+    def custom_weights(terms, weight = 8.0)
+      result = NMatrix.new([1, @keywords.size], 1.0)
+
+      terms.each do |term|
+        result[0, @keywords.index(term)] = weight
+      end
+
+      return result
+    end
+
+    def inverse_document_frequency
+      result = NMatrix.new([1, @keywords.size], 0.0)
+
+      @bag_of_words.each_column do |column|
+        occurences = column.reduce do |m, c|
+          m + (c > 0 ? 1.0 : 0.0)
+        end
+
+        result[0, column.offset[1]] = Math.log(column.size / occurences) if occurences > 0
+      end
+
+      return result
+    end
+
+    def tfidf
+      result = term_frequency
+      idf = inverse_document_frequency
+
+      result.rows.times do |r|
+        result[r, 0..-1] *= idf
+      end
+
+      return result
+    end
+
+    def stem(data)
+      data = data.gsub(/{%.+%}/, ' ') # Replace liquid templates
+      tokenized = data.scan(/\w+/).map(&:downcase)
+      filtered = @stopwords_filter.filter(tokenized)
+      stemmed = filtered.map(&:stem).select{|s| s.length > 1}.map(&:to_sym)
+      return stemmed
+    end
+  end
+end
+end
+
+Jekyll::Hooks.register :site, :pre_render do |site|
+  Jekyll.logger.info("Building TFIDF index...")
+  tfidf = SangsooNam::Jekyll::TFIDFRelatedPosts.new
+  site.posts.docs.each do |x|
+      tfidf.add_post(x)
+  end
+
+  Jekyll.logger.info("Replaceing Related Posts...")
+  tfidf.build(site)
+end

metadata

@@ -0,0 +1,123 @@
+--- !ruby/object:Gem::Specification
+name: jekyll-tfidf-related-posts
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Sangsoo Nam
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-01-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: jekyll
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: stopwords-filter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+- !ruby/object:Gem::Dependency
+  name: fast-stemmer
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: pqueue
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+- !ruby/object:Gem::Dependency
+  name: nmatrix
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+description: "[Jekyll](http://jekyllrb.com) plugin to show related posts based on
+  the content, tags, and categories.  The similarity is calculated using TF-IDF(term
+  frequency-inverted document frequency). Since tags and categories are use-defined
+  values, those are considered with higher weights than a content while calculating."
+email:
+- sangsoo.ted@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- jekyll-tfidf-related-posts.gemspec
+- lib/jekyll-tfidf-related-posts.rb
+homepage: https://github.com/SangsooNam/jekyll-tfidf-related-posts
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.11
+signing_key: 
+specification_version: 4
+summary: Jekyll plugin to show related posts based on the content, tags, and categories.
+test_files: []