middleman-blog-similar 1.1.1 → 2.0.0

data/fixtures/test-app/source/{2014-05-08-article0.md → 2014-05-08-article0.html.md}

@@ -1,7 +1,8 @@
 ---
 title: Article 0
 date: 2014-05-08 07:00
-tags: dog
+tags: dog, cat, brown
+category: test
 ---
 
 content

data/fixtures/test-app/source/{2014-05-09-article1.md → 2014-05-09-article1.html.md}

@@ -1,6 +1,8 @@
 ---
 title: Article 1
 date: 2014-05-09 07:00
+tags: dog, cat
+category: test
 ---
 
 content 1

data/fixtures/test-app/source/{2014-05-10-article2.md → 2014-05-10-article2.html.md}

@@ -1,7 +1,7 @@
 ---
 title: Article 2
 date: 2014-05-10 07:00
-tags: Quick, Fox
+tags: quick, Fox
 ---
 
 The quick brown fox jumps over the lazy dog dog dog brown

data/fixtures/test-app/source/{2014-05-12-article4.md → 2014-05-12-article4.html.md}

@@ -1,6 +1,7 @@
 ---
 title: Article 4
 date: 2014-05-12 07:00
+tags: dog, cat, fox
 ---
 
 content test 4

data/fixtures/test-app/source/{2014-05-13-article5.md → 2014-05-13-article5.html.md}

@@ -1,6 +1,7 @@
 ---
 title: Article 5
 date: 2014-05-13 07:00
+tags: dog
 ---
 
 content!

data/fixtures/test-app/source/{2014-05-14-article6.md → 2014-05-14-article6.html.md}

@@ -1,6 +1,7 @@
 ---
 title: Article 6
 date: 2014-05-14 07:00
+tags: dog, Brown, cat
 ---
 
 contents tests 6

data/fixtures/test-app/source/index.html.slim

@@ -1,6 +1,7 @@
 ---
 description: My Description
 title: My Title
+tags: dog, cat, quick, brown
 ---
 
 h1 Hello index

data/fixtures/test-app/source/layout.slim

@@ -2,6 +2,6 @@ html
   head
     meta charset="utf-8"
     title= current_resource.data.title
-  body
+  body data-similar-article-count=similar_articles.count
     .container
       = yield

data/fixtures/test-app/source/layouts/article.slim

@@ -2,16 +2,16 @@ html
   head
     meta charset="utf-8"
     title= current_article.title
-  body
+  body data-similar-article-count=similar_articles.count
     .container
       h1= current_article.title
+      p.tags= current_article.tags.join ', '
       = yield
 
       h2 Similar Entries
       ul
-        - similar_articles.first(5).each_with_index do|article, index|
+        - similar_articles.each_with_index do|article, index|
           li class="a#{index}"
-            = link_to article.title, article.url
-
-      blockquote.algorithm
-        = current_article.app.similarity_algorithm
+            = link_to article.url do
+              span.title= article.title
+              span.tags= article.tags.join ', '

data/lib/middleman-blog-similar.rb

@@ -1,7 +1,7 @@
-require "middleman-core"
-require "middleman-blog-similar/version"
+require 'middleman-core'
+require 'middleman-blog-similar/version'
 
 ::Middleman::Extensions.register(:similar) do
-  require "middleman-blog-similar/extension"
+  require 'middleman-blog-similar/extension'
   ::Middleman::Blog::SimilarExtension
 end

data/lib/middleman-blog-similar/blog_article_extensions.rb

@@ -1,23 +1,11 @@
-module Middleman::Blog::Similar::BlogArticleExtensions
-  def similar_articles
-    if !@similar_articles && similarity_algorithm
-      @similar_articles = similarity_algorithm.similar_articles
+module Middleman
+  module Blog
+    module Similar
+      module BlogArticleExtensions
+        def similar_articles
+          locals[:similar_db].find_similar(self)
+        end
+      end
     end
-    @similar_articles || []
-  end
-  def words
-    unless @words && similarity_algorithm
-      @words = similarity_algorithm.words
-    end
-    @words
-  end
-  def similarity_algorithm
-    if !@similarity_algorithm && (algorithm = app.similarity_algorithm)
-      @similarity_algorithm = algorithm.new self
-    end
-    @similarity_algorithm
-  end
-  def untagged_body
-    body.gsub(/<[^>]*>/ui,'')
   end
 end

data/lib/middleman-blog-similar/database.rb

@@ -0,0 +1,70 @@
+require 'sqlite3'
+require 'active_record'
+require 'middleman-blog-similar/models/article'
+require 'middleman-blog-similar/models/tag'
+require 'middleman-blog-similar/models/tagging'
+require 'middleman-blog-similar/models/migration'
+
+module Middleman
+  module Blog
+    module Similar
+      class Database
+        attr_reader :taggers
+        def initialize(path, taggers)
+          ActiveRecord::Base.establish_connection(
+            adapter: 'sqlite3',
+            database: path
+          )
+          Migration.apply
+          @taggers = taggers
+          @id_map = {}
+        end
+
+        def store_articles(resources)
+          @id_map = {}
+          ActiveRecord::Base.transaction do
+            ids = []
+            resources.each do |res|
+              next unless res.is_a?(Middleman::Blog::BlogArticle)
+              execute_article res
+              ids << res.page_id
+              @id_map[res.page_id.to_s] = res
+            end
+            Article.where.not(page_id: ids).delete_all unless ids.empty?
+          end
+        end
+
+        def execute_article(resource)
+          source_file = resource.source_file
+          page_id = resource.page_id
+          digest = ::Digest::SHA1.file(source_file).hexdigest
+          return page_id if Article.exists?(digest: digest, page_id: page_id)
+          article = Article.find_or_create_by(page_id: page_id)
+          new_tagging_ids = []
+          @taggers.each do |tagger|
+            tagger[1].call(resource).map(&:downcase).each do |tag_name|
+              tag = Tag.find_or_create_by name: tag_name
+              tagging = Tagging.find_or_create_by tag_id: tag.id, article_id: article.id
+              tagging.weight = tagger[0]
+              tagging.save!
+              new_tagging_ids << tagging.id
+            end
+          end
+          if new_tagging_ids.any?
+            article.taggings.where.not(id: new_tagging_ids).delete_all
+          end
+          article.update! digest: digest
+          page_id
+        end
+
+        def find_similar(article)
+          article = Article.find_by(page_id: article.page_id)
+          return [] unless article
+          article.similar_article_page_ids.map do |page_id|
+            @id_map[page_id]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/middleman-blog-similar/extension.rb

@@ -1,32 +1,54 @@
 require 'middleman-blog-similar/blog_article_extensions'
 require 'middleman-blog-similar/helpers'
-require 'middleman-blog-similar/algorithm'
+require 'middleman-blog-similar/resource_list_manipulator'
+require 'middleman-blog-similar/database'
 
 module Middleman
   module Blog
     class SimilarExtension < ::Middleman::Extension
+      option :tagger, :tags, 'Article tagger'
+      option :db, '.similar.db', 'SQLite3 Database'
 
-      option :algorithm, :word_frequency, 'Similar lookup algorithm'
-
-      self.defined_helpers = [ Middleman::Blog::Similar::Helpers ]
+      self.defined_helpers = [Middleman::Blog::Similar::Helpers]
 
       def after_configuration
         require 'middleman-blog/blog_article'
         ::Middleman::Sitemap::Resource.send :include, Middleman::Blog::Similar::BlogArticleExtensions
-        algorithm = options[:algorithm].to_s
-        begin
-          require "middleman-blog-similar/algorithm/#{algorithm}"
-          ns = ::Middleman::Blog::Similar::Algorithm
-          algorithm.split('/').each do|n|
-            ns = ns.const_get n.camelize
-          end
-          app.set :similarity_algorithm, ns
-        rescue LoadError => e
-          app.logger.error "Requested similar algorithm '#{algorithm}' not found."
-          raise e
+
+        @taggers = []
+        case options.tagger
+          when String, Symbol
+            @taggers << [1, load_tagger(options.tagger)]
+          when Hash
+            options.tagger.each do |k, v|
+              if v.is_a?(Array)
+                k = v[1]
+                v = v[0]
+              end
+              @taggers << [v, load_tagger(k)]
+            end
+          else
+            raise "Invalid type for tagger option: #{options.tagger.class}"
         end
+        db_path = options.db
+        db_path = File.expand_path(options.db, app.root) if db_path != ':memory:'
+        @db = Middleman::Blog::Similar::Database.new db_path, @taggers
+        @resource_list_manipulator = Middleman::Blog::Similar::ResourceListManipulator.new app, @db
+        @app.sitemap.register_resource_list_manipulator :blog_similar, @resource_list_manipulator
       end
 
+      def load_tagger(tagger)
+        return tagger unless tagger.is_a?(String) || tagger.is_a?(Symbol)
+        require "middleman-blog-similar/tagger/#{tagger}"
+        ns = ::Middleman::Blog::Similar::Tagger
+        tagger.to_s.split('/').each do |n|
+          ns = ns.const_get n.camelize
+        end
+        ns.new
+      rescue LoadError => e
+        app.logger.error "Requested similar tagger '#{tagger}' not found."
+        raise e
+      end
     end
   end
 end

data/lib/middleman-blog-similar/helpers.rb

@@ -1,9 +1,15 @@
-module Middleman::Blog::Similar::Helpers
-  def similar_articles
-    if is_blog_article?
-      current_article.similar_articles
-    else
-      []
+module Middleman
+  module Blog
+    module Similar
+      module Helpers
+        def similar_articles
+          if is_blog_article?
+            current_article.similar_articles
+          else
+            []
+          end
+        end
+      end
     end
   end
 end

data/lib/middleman-blog-similar/models/article.rb

@@ -0,0 +1,33 @@
+module Middleman
+  module Blog
+    module Similar
+      class Article < ActiveRecord::Base
+        has_many :taggings
+        has_many :tags, through: :taggings
+        def similar_article_page_ids
+          return self.class.none if tags.empty?
+          # http://stackoverflow.com/a/22472153
+          res = ActiveRecord::Base.connection.select_all "
+            SELECT rtr.article_id FROM taggings AS rtr
+              INNER JOIN taggings rtr2
+                ON (rtr2.tag_id = rtr.tag_id AND rtr2.article_id = #{id})
+              LEFT JOIN
+                (SELECT * FROM taggings WHERE article_id = #{id}) AS r
+                ON rtr.tag_id = r.tag_id
+              LEFT JOIN articles a ON a.id = rtr.article_id
+              WHERE rtr.article_id != #{id}
+              GROUP BY rtr.article_id
+              HAVING COUNT(*) > 0
+              ORDER BY COUNT(*) * rtr.weight DESC, a.page_id DESC"
+          ids = res.to_hash.map { |h| h['article_id'] }
+          page_id_map = {}
+          articles = self.class.where(id: ids).select(:id, :page_id)
+          articles.each do |a|
+            page_id_map[a.id] = a.page_id
+          end
+          ids.map { |id| page_id_map[id] }
+        end
+      end
+    end
+  end
+end

data/lib/middleman-blog-similar/models/migration.rb

@@ -0,0 +1,32 @@
+module Middleman
+  module Blog
+    module Similar
+      module Migration
+        def self.apply
+          ActiveRecord::Schema.define(version: 201703240752) do # rubocop:disable Style/NumericLiterals
+            unless ActiveRecord::Base.connection.data_source_exists? 'articles'
+              create_table :articles do |table|
+                table.column :page_id, :string, index: true, unique: true
+                table.column :digest, :string, index: true
+              end
+            end
+
+            unless ActiveRecord::Base.connection.data_source_exists? 'tags'
+              create_table :tags do |table|
+                table.column :name, :string, index: true, unique: true
+              end
+            end
+
+            unless ActiveRecord::Base.connection.data_source_exists? 'taggings'
+              create_table :taggings do |table|
+                table.references :article, foreign_key: true
+                table.references :tag, foreign_key: true
+                table.column :weight, :integer, default: 1, null: false
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/middleman-blog-similar/models/tag.rb

@@ -0,0 +1,10 @@
+module Middleman
+  module Blog
+    module Similar
+      class Tag < ActiveRecord::Base
+        has_many :taggings
+        has_many :articles, through: :taggings
+      end
+    end
+  end
+end

data/lib/middleman-blog-similar/models/tagging.rb

@@ -0,0 +1,10 @@
+module Middleman
+  module Blog
+    module Similar
+      class Tagging < ActiveRecord::Base
+        belongs_to :article
+        belongs_to :tag
+      end
+    end
+  end
+end

data/lib/middleman-blog-similar/resource_list_manipulator.rb

@@ -0,0 +1,19 @@
+module Middleman
+  module Blog
+    module Similar
+      class ResourceListManipulator
+        attr_reader :article, :app, :db
+        def initialize(app, db)
+          @app = app
+          @db = db
+        end
+
+        def manipulate_resource_list(resources)
+          resources.each { |res| res.add_metadata locals: { similar_db: @db } }
+          @db.store_articles resources
+          resources
+        end
+      end
+    end
+  end
+end

data/lib/middleman-blog-similar/tagger/entagger.rb

@@ -0,0 +1,17 @@
+require 'engtagger'
+
+module Middleman
+  module Blog
+    module Similar
+      module Tagger
+        class Entagger
+          def call(article)
+            tgr = EngTagger.new
+            tagged = tgr.add_tags article.body.gsub(%r{</?[^>]+>}, '')
+            tgr.get_nouns(tagged).keys
+          end
+        end
+      end
+    end
+  end
+end