perron 0.16.0 → 0.18.0

data/lib/perron/resource/related.rb

@@ -5,93 +5,114 @@ require "perron/resource/related/stop_words"
 module Perron
   module Site
     class Resource
+      # Finds related resources using TF-IDF cosine similarity.
+      #
+      # Pre-normalizes vectors so cosine similarity reduces to a dot product,
+      # then builds a symmetric similarity matrix once per collection.
+      # Results are cached at the class level so the O(n²) comparison
+      # is paid once, not once per resource.
       class Related
+        Cache = Struct.new(:resources, :similarity_matrix, :fingerprint)
+
+        @collection_caches = {}
+
+        def self.cache_for(collection_name)
+          clear_cache!(collection_name) if stale?(collection_name)
+          @collection_caches[collection_name] ||= Cache.new(nil, nil, content_fingerprint(collection_name))
+        end
+
+        def self.clear_cache!(collection_name)
+          @collection_caches.delete(collection_name)
+        end
+
+        def self.stale?(collection_name)
+          @collection_caches[collection_name]&.fingerprint != content_fingerprint(collection_name)
+        end
+
+        def self.content_fingerprint(collection_name)
+          path = File.join(Perron.configuration.input, collection_name)
+          files = Dir.glob(File.join(path, "**", "*.*"))
+          [files.size, files.map { File.mtime(it) }.max]
+        end
+
         def initialize(resource)
           @resource = resource
           @collection = resource.collection
+          @cache = self.class.cache_for(@collection.name)
         end
 
         def find(limit: 5)
-          @collection.resources
+          scores = similarity_matrix[@resource.slug] || {}
+
+          resources
             .reject { it.slug == @resource.slug }
-            .map { [it, cosine_similarities_for(@resource, it)] }
-            .sort_by { |_, score| -score }
-            .map(&:first)
+            .sort_by { -(scores[it.slug] || 0.0) }
             .first(limit)
         end
 
         private
 
-        def cosine_similarities_for(resource_one, resource_two)
-          first_vector = tfidf_vector_for(resource_one)
-          second_vector = tfidf_vector_for(resource_two)
+        def resources = @cache.resources ||= @collection.resources
 
-          return 0.0 if first_vector.empty? || second_vector.empty?
+        def similarity_matrix = @cache.similarity_matrix ||= build_similarity_matrix
 
-          dot_product = 0.0
+        def build_similarity_matrix
+          vectors = resources.to_h { [it.slug, normalize(tfidf_vector_for(it))] }
+          matrix = Hash.new { |h, k| h[k] = {} }
 
-          first_vector.each_key { dot_product += first_vector[it] * second_vector[it] if second_vector.key?(it) }
+          slugs = vectors.keys
+          slugs.each_with_index do |slug_a, i|
+            next if vectors[slug_a].empty?
 
-          first_magnitude = Math.sqrt(first_vector.values.sum { it**2 })
-          second_magnitude = Math.sqrt(second_vector.values.sum { it**2 })
-          denominator = first_magnitude * second_magnitude
+            slugs[(i + 1)..].each do |slug_b|
+              next if vectors[slug_b].empty?
 
-          return 0.0 if denominator.zero?
+              score = dot_product(vectors[slug_a], vectors[slug_b])
+              matrix[slug_a][slug_b] = score
+              matrix[slug_b][slug_a] = score
+            end
+          end
 
-          dot_product / denominator
+          matrix
         end
 
-        def tfidf_vector_for(target_resource)
-          @tfidf_vectors ||= {}
-
-          return @tfidf_vectors[target_resource] if @tfidf_vectors.key?(target_resource)
-
-          tokens = tokenize_content(target_resource)
-          token_count = tokens.size
-
-          return {} if token_count.zero?
+        def dot_product(vec_a, vec_b)
+          score = 0.0
+          vec_a.each_key { score += vec_a[it] * vec_b[it] if vec_b.key?(it) }
+          score
+        end
 
-          term_count = Hash.new(0)
+        def normalize(vector)
+          return {} if vector.empty?
 
-          tokens.each { |token| term_count[token] += 1 }
+          magnitude = Math.sqrt(vector.values.sum { it**2 })
+          return {} if magnitude.zero?
 
-          tfidf_vector = {}
+          vector.transform_values { it / magnitude }
+        end
 
-          term_count.each do |term, count|
-            terms = count.to_f / token_count
+        def tfidf_vector_for(resource)
+          tokens = tokenize(resource)
+          return {} if tokens.empty?
 
-            tfidf_vector[term] = terms * inverse_document_frequency[term]
-          end
+          token_count = tokens.size.to_f
 
-          @tfidf_vectors[target_resource] = tfidf_vector
+          tokens.tally.to_h { |term, count| [term, (count / token_count) * inverse_document_frequency[term]] }
         end
 
-        def tokenize_content(target_resource)
-          @tokenized_content ||= {}
+        def tokenize(resource)
+          return [] if resource.content.blank?
 
-          return @tokenized_content[target_resource] if @tokenized_content.key?(target_resource)
-          return [] if target_resource.content.blank?
-
-          content = target_resource.content.gsub(/<[^>]*>/, " ")
-          tokens = content.downcase.scan(/\w+/).reject { StopWords.all.include?(it) || it.length < 3 }
-
-          @tokenized_content[target_resource] = tokens
+          resource.content.gsub(/<[^>]*>/, " ").downcase.scan(/\w+/).reject { StopWords.all.include?(it) || it.length < 3 }
         end
 
         def inverse_document_frequency
           @inverse_document_frequency ||= begin
-            resource_frequency = Hash.new(0)
-
-            @collection.resources.each { tokenize_content(it).uniq.each { resource_frequency[it] += 1 } }
-
-            frequencies = {}
-            total_resources = @collection.resources.size
-
-            resource_frequency.each do |term, frequency|
-              frequencies[term] = Math.log(total_resources.to_f / (1 + frequency))
-            end
+            doc_frequency = Hash.new(0)
+            resources.each { tokenize(it).uniq.each { doc_frequency[it] += 1 } }
 
-            frequencies
+            total = resources.size.to_f
+            doc_frequency.transform_values { Math.log(total / (1 + it)) }
           end
         end
       end

data/lib/perron/resource/scopes.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Perron
+  class Resource
+    module Scopes
+      extend ActiveSupport::Concern
+
+      class_methods do
+        def scope(name, body)
+          unless body.respond_to?(:call)
+            raise ArgumentError, "The scope body needs to be callable."
+          end
+
+          if respond_to?(name, true)
+            raise ArgumentError, "Cannot define scope :#{name} because it already exists."
+          end
+
+          singleton_class.define_method(name) do |*arguments|
+            instance_exec(*arguments, &body)
+          end
+
+          Perron::Relation.define_method(name) do |*arguments|
+            instance_exec(*arguments, &body)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/perron/resource/searchable.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Perron
+  class Resource
+    module Searchable
+      extend ActiveSupport::Concern
+
+      included do
+        class_attribute :search_fields_list, default: []
+      end
+
+      class_methods do
+        def search_fields(*fields)
+          self.search_fields_list = fields
+        end
+      end
+    end
+  end
+end

data/lib/perron/resource/sourceable.rb

@@ -40,7 +40,7 @@ module Perron
         end
 
         def combinations
-          datasets = source_names.map { Perron::Data.new(it.to_s) }
+          datasets = source_names.map { Perron::DataSource.new(it.to_s) }
 
           datasets.first.product(*datasets[1..])
         end
@@ -72,7 +72,7 @@ module Perron
             singular_name = name.to_s.singularize
             identifier = frontmatter["#{singular_name}_#{primary_key}"]
 
-            hash[name] = Perron::Data.new(name.to_s).find { it.public_send(primary_key).to_s == identifier.to_s }
+            hash[name] = Perron::DataSource.new(name.to_s).find { it.public_send(primary_key).to_s == identifier.to_s }
           end
 
           Source.new(data)

data/lib/perron/resource/sweeper.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Perron
+  class Resource
+    module Sweeper
+      extend ActiveSupport::Concern
+
+      def extracted_headings
+        extract_heading_texts(from: rendered_document)
+      end
+
+      def sweeped_content
+        ActionView::Base.full_sanitizer.sanitize(
+          rendered_document.text.gsub(/\s+/, " ").strip
+        )
+      end
+
+      private
+
+      def rendered_document
+        @rendered_document ||= Nokogiri::HTML::DocumentFragment.parse(Markdown.render(content))
+      end
+
+      def extract_heading_texts(from:, levels: "h1, h2, h3, h4, h5, h6")
+        from.css(levels).map { it.text.strip }.compact_blank
+      end
+
+      def extract_headings(from:, levels:)
+        from.css(levels).each_with_object([]) do |heading, headings|
+          heading_text = heading.text.strip
+          id = heading["id"] || heading.at("a")&.[]("id")
+
+          next if heading_text.empty? || id.blank?
+
+          headings << TableOfContent::Item.new(
+            id: id,
+            text: heading_text,
+            level: heading.name[1..].to_i,
+            children: []
+          )
+        end
+      end
+    end
+  end
+end

data/lib/perron/resource/table_of_content.rb

@@ -20,24 +20,6 @@ module Perron
 
       Item = ::Data.define(:id, :text, :level, :children)
 
-      def extract_headings(from:, levels:)
-        from.css(levels).each_with_object([]) do |heading, headings|
-          heading.tap do |node|
-            heading_text = node.text.strip
-            id = node["id"] || node.at("a")&.[]("id")
-
-            next if heading_text.empty? || id.blank?
-
-            headings << Item.new(
-              id: id,
-              text: heading_text,
-              level: node.name[1..].to_i,
-              children: []
-            )
-          end
-        end
-      end
-
       class Builder
         def build(headings)
           parents = {0 => {children: []}}

data/lib/perron/resource.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "perron/relation"
 require "perron/resource/configuration"
 require "perron/resource/core"
 require "perron/resource/class_methods"
@@ -10,26 +11,30 @@ require "perron/resource/publishable"
 require "perron/resource/reading_time"
 require "perron/resource/related"
 require "perron/resource/renderer"
+require "perron/resource/scopes"
 require "perron/resource/slug"
+require "perron/resource/searchable"
 require "perron/resource/separator"
 require "perron/resource/sourceable"
+require "perron/resource/sweeper"
 require "perron/resource/table_of_content"
 
 module Perron
   class Resource
-    ID_LENGTH = 8
-
     include ActiveModel::Validations
 
-    include Perron::Resource::Configuration
-    include Perron::Resource::Core
-    include Perron::Resource::ClassMethods
-    include Perron::Resource::Associations
-    include Perron::Resource::ReadingTime
-    include Perron::Resource::Sourceable
-    include Perron::Resource::Publishable
-    include Perron::Resource::Previewable
-    include Perron::Resource::TableOfContent
+    include Configuration
+    include Core
+    include ClassMethods
+    include Associations
+    include ReadingTime
+    include Searchable
+    include Sourceable
+    include Publishable
+    include Previewable
+    include Scopes
+    include Sweeper
+    include TableOfContent
 
     attr_reader :file_path, :id
 
@@ -41,6 +46,16 @@ module Perron
       raise Errors::FileNotFoundError, "No such file: #{file_path}" unless File.exist?(file_path)
     end
 
+    def pluck(*attributes)
+      raise ArgumentError, "wrong number of arguments (given 0, expected 1+)" if attributes.empty?
+
+      if attributes.size == 1
+        public_send(attributes.first)
+      else
+        attributes.map { |attr| public_send(attr) }
+      end
+    end
+
     def filename = File.basename(@file_path)
 
     def slug = Perron::Resource::Slug.new(self, frontmatter).create
@@ -66,15 +81,8 @@ module Perron
       render_inline_erb using: page_content
     end
 
-    def association_value(key) = metadata[key]
-
-    def to_partial_path
-      @to_partial_path ||= begin
-        element = ActiveSupport::Inflector.underscore(ActiveSupport::Inflector.demodulize(self.class.model_name))
-        collection = ActiveSupport::Inflector.tableize(self.class.model_name)
-
-        File.join("content", collection, element)
-      end
+    def inline(layout: "application", **options)
+      {html: content, layout: layout}.merge(options)
     end
 
     def collection = Collection.new(self.class.model_name.collection)
@@ -82,8 +90,14 @@ module Perron
     def related_resources(limit: 5) = Perron::Site::Resource::Related.new(self).find(limit:)
     alias_method :related, :related_resources
 
+    def root?
+      slug == "/"
+    end
+
     private
 
+    ID_LENGTH = 8
+
     def frontmatter
       @frontmatter ||= Perron::Resource::Separator.new(raw_content).frontmatter
     end
@@ -103,9 +117,5 @@ module Perron
     def erb_processing?
       @file_path.ends_with?(".erb") || metadata.erb == true
     end
-
-    def root?
-      collection.name.inquiry.pages? && File.basename(filename) == "root"
-    end
   end
 end

data/lib/perron/site/builder/additional_routes.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Perron
+  module Site
+    class Builder
+      class AdditionalRoutes
+        def initialize(paths)
+          @paths = paths
+        end
+
+        def get
+          Perron.configuration.additional_routes.each do |route_name|
+            @paths << routes.public_send(route_name) if routes.respond_to?(route_name)
+          end
+        end
+
+        private
+
+        def routes = Rails.application.routes.url_helpers
+      end
+    end
+  end
+end

data/lib/perron/site/builder/feeds/author.rb

@@ -8,7 +8,8 @@ module Perron
           private
 
           def author(resource)
-            author = resource.author || feed_configuration.author
+            author = (resource&.respond_to?(:author) && resource.author) ||
+              feed_configuration.author
 
             Author.new(author) if author
           end

data/lib/perron/site/builder/paths.rb

@@ -13,11 +13,9 @@ module Perron
 
           if routes.respond_to?(show_path)
             @collection.send(:load_resources).select(&:buildable?).each do |resource|
-              root = resource.slug == "/"
+              next if resource.root?
 
-              next if skip? root
-
-              @paths << (root ? routes.root_path : routes.public_send(show_path, resource))
+              @paths << routes.public_send(show_path, resource)
 
               (resource.class.try(:nested_routes) || []).each do |nested|
                 @paths << routes.polymorphic_path([resource, nested])
@@ -28,11 +26,6 @@ module Perron
 
         private
 
-        def skip?(root)
-          root &&
-            Perron.configuration.mode.integrated? && Perron.configuration.exclude_root?
-        end
-
         def routes = Rails.application.routes.url_helpers
 
         def index_path = "#{@collection.name}_path"

data/lib/perron/site/builder/sitemap.rb

@@ -13,6 +13,8 @@ module Perron
 
           xml = Nokogiri::XML::Builder.new(encoding: "UTF-8") do |builder|
             builder.urlset(xmlns: "http://www.sitemaps.org/schemas/sitemap/0.9") do
+              add_additional_routes(with: builder)
+
               Perron::Site.collections.each do |collection|
                 add_urls_for(collection, with: builder)
               end
@@ -24,6 +26,21 @@ module Perron
 
         private
 
+        def add_additional_routes(with:)
+          (Perron.configuration.additional_routes || []).each do |route_name|
+            next unless routes.respond_to?(route_name)
+
+            routes.with_options(Perron.configuration.default_url_options) do |url|
+              with.url do
+                with.loc url.public_send(route_name.to_s.gsub("_path", "_url"))
+                with.priority Perron.configuration.sitemap.priority
+                with.changefreq Perron.configuration.sitemap.change_frequency
+                with.lastmod Time.current.iso8601
+              end
+            end
+          end
+        end
+
         def add_urls_for(collection, with:)
           return if collection.configuration.blank?
           return if collection.configuration.sitemap.exclude == true
@@ -31,13 +48,12 @@ module Perron
           collection.resources.each do |resource|
             next if resource.metadata.sitemap == false
 
-            root = resource.slug == "/"
             priority = resource.metadata.sitemap_priority || collection.configuration.sitemap.priority || Perron.configuration.sitemap.priority
             change_frequency = resource.metadata.sitemap_change_frequency || collection.configuration.sitemap.change_frequency || Perron.configuration.sitemap.change_frequency
 
             Rails.application.routes.url_helpers.with_options(Perron.configuration.default_url_options) do |url|
               with.url do
-                with.loc root ? url.root_url : url.polymorphic_url(resource)
+                with.loc resource.root? ? url.root_url : url.polymorphic_url(resource)
                 with.priority priority
                 with.changefreq change_frequency
                 begin
@@ -49,6 +65,8 @@ module Perron
             end
           end
         end
+
+        def routes = Rails.application.routes.url_helpers
       end
     end
   end

data/lib/perron/site/builder.rb

@@ -5,6 +5,7 @@ require "perron/site/builder/sitemap"
 require "perron/site/builder/feeds"
 require "perron/site/builder/public_files"
 require "perron/site/builder/paths"
+require "perron/site/builder/additional_routes"
 require "perron/site/builder/page"
 
 module Perron
@@ -40,6 +41,7 @@ module Perron
 
       def paths
         Set.new.tap do |paths|
+          Perron::Site::Builder::AdditionalRoutes.new(paths).get
           Perron::Site.collections.each { Perron::Site::Builder::Paths.new(it, paths).get }
         end
       end

data/lib/perron/site.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
 require "perron/collection"
-require "perron/data"
-require "perron/data/proxy"
+require "perron/data_source"
+require "perron/data_source/proxy"
 require "perron/site/builder"
 require "perron/site/validate"
 
@@ -26,7 +26,7 @@ module Perron
     def data(name = nil)
       Perron.deprecator.deprecation_warning(:data, "Use Content::Data::ClassName instead, e.g. `Content::Data::Users.all`")
 
-      (name && Perron::Data.new(name)) || Perron::Data::Proxy.new
+      (name && Perron::DataSource.new(name)) || Perron::DataSource::Proxy.new
     end
   end
 end

data/lib/perron/tasks/build.rake

@@ -1,6 +1,13 @@
 namespace :perron do
+  task :set_production_env do
+    unless ENV["RAILS_ENV"]
+      ENV["RAILS_ENV"] = "production"
+      puts "RAILS_ENV not set, defaulting to production"
+    end
+  end
+
   desc "Generate static HTML files from Perron collections"
-  task build: :environment do
+  task build: [:set_production_env, :environment] do
     Perron::Site.build
   end
 end

data/lib/perron/version.rb

@@ -1,3 +1,3 @@
 module Perron
-  VERSION = "0.16.0"
+  VERSION = "0.18.0"
 end

data/lib/perron.rb

@@ -4,8 +4,8 @@ require "perron/version"
 require "perron/configuration"
 require "perron/deprecator"
 require "perron/errors"
-require "perron/root"
 require "perron/site"
+require "perron/relation"
 require "perron/content/data"
 require "perron/resource"
 require "perron/markdown"

data/perron.gemspec

@@ -19,6 +19,7 @@ Gem::Specification.new do |spec|
   spec.required_ruby_version = ">= 3.4.0"
 
   spec.add_dependency "rails", ">= 7.2.0"
+  spec.add_dependency "mata", "~> 0.8.0"
 
   spec.add_runtime_dependency "csv"
   spec.add_runtime_dependency "json"