jekyll-client-search 0.1.0

data/lib/jekyll/client_search/index_cache.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+require "json"
+require "digest"
+require "fileutils"
+
+module Jekyll
+  module ClientSearch
+    # Persists content hashes and cached embeddings across Jekyll builds so
+    # that unchanged documents are not re-embedded. The cache file lives in
+    # the site source directory as +.jekyll-client-search-cache.json+ and
+    # should be git-ignored.
+    class IndexCache
+      CACHE_FILE = ".jekyll-client-search-cache.json"
+
+      attr_reader :path
+
+      def initialize(site_source, embedding_identity: nil)
+        @path = File.join(site_source, CACHE_FILE)
+        @embedding_identity = embedding_identity
+        @entries = load
+      end
+
+      # Returns the cached entry for +id+ if the content hash and embedding
+      # identity match, nil otherwise (or if not cached).
+      def lookup(id, content_hash)
+        entry = @entries[id]
+        return nil unless entry
+        return nil unless entry["content_hash"] == content_hash
+        return nil if @embedding_identity && entry["embedding_identity"] != @embedding_identity
+
+        entry
+      end
+
+      # Stores or updates a cache entry for +id+.
+      def store(id, content_hash, embedding = nil)
+        @entries[id] = {
+          "content_hash" => content_hash,
+          "embedding" => embedding,
+          "embedding_identity" => @embedding_identity
+        }.compact
+      end
+
+      # Removes entries for IDs that are no longer present.
+      def prune(known_ids)
+        @entries.select! { |id, _| known_ids.include?(id) }
+      end
+
+      # Writes the cache to disk if any entries have changed.
+      def save
+        return unless dirty?
+
+        temporary_path = "#{@path}.tmp.#{Process.pid}.#{Thread.current.object_id}"
+        File.write(temporary_path, JSON.pretty_generate(@entries))
+        File.rename(temporary_path, @path)
+      ensure
+        FileUtils.rm_f(temporary_path) if temporary_path && File.exist?(temporary_path)
+      end
+
+      def self.content_hash(document)
+        Digest::SHA256.hexdigest(document.to_json)
+      end
+
+      private
+
+      def load
+        return {} unless File.exist?(@path)
+
+        data = JSON.parse(File.read(@path))
+        data.is_a?(Hash) ? data : {}
+      rescue JSON::ParserError
+        {}
+      end
+
+      def dirty?
+        return true unless File.exist?(@path)
+
+        File.read(@path) != JSON.pretty_generate(@entries)
+      end
+    end
+  end
+end

data/lib/jekyll/client_search/live_search_configuration.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Jekyll
+  module ClientSearch
+    # Normalizes optional browser live-search behavior.
+    class LiveSearchConfiguration
+      DEFAULTS = {
+        "min_chars" => 2,
+        "debounce_ms" => 150,
+        "semantic_debounce_ms" => 500,
+        "update_url" => true
+      }.freeze
+
+      def initialize(config, engine: "minisearch")
+        unless config.is_a?(Hash)
+          raise Jekyll::Errors::FatalException,
+                "client_search live_search configuration must be a mapping"
+        end
+
+        @engine = engine
+        @values = DEFAULTS.merge(config)
+        @values["enabled"] = default_enabled?(engine) unless config.key?("enabled")
+        validate!
+      end
+
+      def to_h(engine:)
+        {
+          "enabled" => @values.fetch("enabled"),
+          "minChars" => @values.fetch("min_chars"),
+          "debounceMs" => debounce_ms(engine),
+          "updateUrl" => @values.fetch("update_url")
+        }
+      end
+
+      private
+
+      def default_enabled?(engine)
+        engine != "semantic"
+      end
+
+      def debounce_ms(engine)
+        key = engine == "semantic" ? "semantic_debounce_ms" : "debounce_ms"
+        @values.fetch(key)
+      end
+
+      def validate!
+        validate_boolean!("enabled")
+        validate_boolean!("update_url")
+        validate_integer!("min_chars", minimum: 0)
+        validate_integer!("debounce_ms", minimum: 0)
+        validate_integer!("semantic_debounce_ms", minimum: 0)
+      end
+
+      def validate_boolean!(key)
+        return if [true, false].include?(@values[key])
+
+        raise Jekyll::Errors::FatalException,
+              "client_search live_search.#{key} must be true or false"
+      end
+
+      def validate_integer!(key, minimum:)
+        value = @values.fetch(key)
+        return if value.is_a?(Integer) && value >= minimum
+
+        raise Jekyll::Errors::FatalException,
+              "client_search live_search.#{key} must be an integer greater than or equal to #{minimum}"
+      end
+    end
+  end
+end

data/lib/jekyll/client_search/ollama_embedding_adapter.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Jekyll
+  module ClientSearch
+    # Generates embeddings via a local Ollama server using the +ollama-ruby+
+    # gem. The gem is required lazily so that users who do not enable
+    # embeddings never need to install it.
+    #
+    # Configuration:
+    #   embedding:
+    #     enabled: true
+    #     model: all-minilm          # or nomic-embed-text, bge-m3, etc.
+    #     base_url: http://localhost:11434
+    class OllamaEmbeddingAdapter
+      attr_reader :model, :base_url, :connect_timeout, :read_timeout
+
+      def initialize(model:, base_url: "http://localhost:11434", connect_timeout: 5, read_timeout: 120)
+        @model = model
+        @base_url = base_url
+        @connect_timeout = connect_timeout
+        @read_timeout = read_timeout
+      end
+
+      # Returns a float vector for the given text. Raises a clear error if
+      # the ollama-ruby gem is not installed or the server is unreachable.
+      def embed(text)
+        embedding = client.embed(model: @model, input: text).embeddings&.first
+        return embedding if valid_embedding?(embedding)
+
+        Jekyll.logger.warn "ClientSearch:", "embedding response was empty or invalid"
+        nil
+      rescue LoadError, NameError
+        raise Jekyll::Errors::FatalException,
+              "Add gem \"ollama-ruby\" to your Gemfile to use embedding features"
+      rescue StandardError => e
+        Jekyll.logger.warn "ClientSearch:", "embedding failed for text: #{e.message}"
+        nil
+      end
+
+      private
+
+      def valid_embedding?(embedding)
+        embedding.is_a?(Array) && !embedding.empty? &&
+          embedding.all? { |value| value.is_a?(Numeric) && value.finite? }
+      end
+
+      def client
+        @client ||= begin
+          require "ollama"
+          Ollama::Client.new(
+            base_url: @base_url,
+            connect_timeout: @connect_timeout,
+            read_timeout: @read_timeout
+          )
+        end
+      end
+    end
+  end
+end

data/lib/jekyll/client_search/query_embedder_configuration.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+module Jekyll
+  module ClientSearch
+    # Normalizes browser query-embedder settings.
+    class QueryEmbedderConfiguration
+      TYPES = %w[transformers ollama_api none].freeze
+
+      MODEL_MAP = {
+        "embeddinggemma:300m" => "onnx-community/embeddinggemma-300m-ONNX",
+        "all-minilm" => "Xenova/all-MiniLM-L6-v2"
+      }.freeze
+
+      DEFAULT_LIBRARY_URL = "https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.8.1"
+
+      DEFAULTS = {
+        "type" => "transformers",
+        "library_url" => DEFAULT_LIBRARY_URL,
+        "dtype" => "q8",
+        "worker" => true,
+        "timeout_ms" => 300_000,
+        "retry_attempts" => 1,
+        "max_tokens" => 512
+      }.freeze
+
+      JSON_KEYS = {
+        "library_url" => "libraryUrl",
+        "model_base_url" => "modelBaseUrl",
+        "wasm_base_url" => "wasmBaseUrl",
+        "worker_url" => "workerUrl",
+        "device" => "device",
+        "dtype" => "dtype",
+        "worker" => "worker",
+        "timeout_ms" => "timeoutMs",
+        "retry_attempts" => "retryAttempts",
+        "max_tokens" => "maxTokens"
+      }.freeze
+
+      def initialize(config, build_model:, build_base_url:, query_prefix:)
+        unless config.is_a?(Hash)
+          raise Jekyll::Errors::FatalException,
+                "client_search embedding.query_embedder configuration must be a mapping"
+        end
+
+        @values = DEFAULTS.merge(config)
+        @values["timeout_ms"] = 30_000 if @values["type"] == "ollama_api" && !config.key?("timeout_ms")
+        @build_model = build_model
+        @build_base_url = build_base_url
+        @query_prefix = query_prefix
+        validate!
+      end
+
+      def type
+        @values.fetch("type")
+      end
+
+      def model
+        return @values.fetch("model") if @values.key?("model")
+        return @build_model unless type == "transformers"
+        return MODEL_MAP.fetch(@build_model) if MODEL_MAP.key?(@build_model)
+
+        raise Jekyll::Errors::FatalException,
+              "embedding query_embedder.model is required for #{@build_model.inspect}"
+      end
+
+      def api_url
+        @values.fetch("api_url", "#{@build_base_url}/api/embed")
+      end
+
+      def assets
+        case type
+        when "transformers"
+          files = ["assets/query-embedders/transformers.js"]
+          files << "assets/query-embedders/transformers-worker.js" if @values["worker"]
+          files
+        when "ollama_api"
+          ["assets/query-embedders/ollama-api.js"]
+        else
+          []
+        end
+      end
+
+      def asset
+        assets.first
+      end
+
+      def to_json(*)
+        config = {
+          "type" => type,
+          "model" => model,
+          "apiUrl" => api_url,
+          "buildModel" => @build_model,
+          "queryPrefix" => @query_prefix
+        }
+        JSON_KEYS.each do |source, target|
+          config[target] = @values[source] if @values.key?(source)
+        end
+        config.to_json
+      end
+
+      private
+
+      def validate!
+        unless TYPES.include?(type)
+          valid = TYPES.join(", ")
+          raise Jekyll::Errors::FatalException,
+                "embedding query_embedder type must be one of #{valid} (got #{type.inspect})"
+        end
+        validate_integer!("timeout_ms", minimum: 1)
+        validate_integer!("retry_attempts", minimum: 0)
+        validate_integer!("max_tokens", minimum: 1)
+        return if [true, false].include?(@values["worker"])
+
+        raise Jekyll::Errors::FatalException,
+              "embedding query_embedder.worker must be true or false"
+      end
+
+      def validate_integer!(key, minimum:)
+        value = @values.fetch(key)
+        return if value.is_a?(Integer) && value >= minimum
+
+        raise Jekyll::Errors::FatalException,
+              "embedding query_embedder.#{key} must be an integer greater than or equal to #{minimum}"
+      end
+    end
+  end
+end

data/lib/jekyll/client_search/related_analyzer.rb

@@ -0,0 +1,152 @@
+# frozen_string_literal: true
+
+module Jekyll
+  module ClientSearch
+    # Builds deterministic related-article records from metadata and vectors.
+    class RelatedAnalyzer
+      def initialize(configuration)
+        @configuration = configuration
+      end
+
+      def analyze(documents)
+        relations = documents.each_with_index.to_h do |source, index|
+          candidates = documents.each_with_index.filter_map do |target, target_index|
+            next if index == target_index
+
+            build_relation(source, target)
+          end
+          [source.fetch("id"), sort_relations(candidates)]
+        end
+
+        {
+          "version" => 1,
+          "minimum_similarity" => @configuration.minimum_similarity,
+          "relations" => relations
+        }
+      end
+
+      private
+
+      def build_relation(source, target)
+        metadata = metadata_relation(source, target)
+        semantic_similarity = similarity(source["embedding"], target["embedding"])
+        return unless metadata || semantic_match?(semantic_similarity)
+
+        relation = base_relation(target, metadata, semantic_similarity)
+        add_semantic_relation(relation, semantic_similarity) if semantic_match?(semantic_similarity)
+        add_metadata_relation(relation, metadata)
+        relation
+      end
+
+      def semantic_match?(similarity)
+        @configuration.semantic? && similarity && similarity >= @configuration.minimum_similarity
+      end
+
+      def base_relation(target, metadata, semantic_similarity)
+        metadata_score = metadata ? metadata.fetch(:score) : 0.0
+        score = [metadata_score, semantic_similarity || 0.0].max
+        reasons = metadata ? metadata.fetch(:reasons).dup : []
+        reasons << "semantic-similarity" if semantic_match?(semantic_similarity)
+        {
+          "id" => target.fetch("id"),
+          "title" => target.fetch("title"),
+          "url" => target.fetch("url"),
+          "date" => target["date"],
+          "date_timestamp" => target["date_timestamp"],
+          "score" => score.round(6),
+          "reasons" => reasons.uniq.sort
+        }.compact
+      end
+
+      def add_semantic_relation(relation, similarity)
+        relation["semantic_similarity"] = similarity.round(6)
+      end
+
+      def add_metadata_relation(relation, metadata)
+        return unless metadata
+
+        relation["shared_tags"] = metadata.fetch(:tags) if metadata.fetch(:tags).any?
+        relation["shared_categories"] = metadata.fetch(:categories) if metadata.fetch(:categories).any?
+        relation["shared_domains"] = metadata.fetch(:domains) if metadata.fetch(:domains).any?
+      end
+
+      def metadata_relation(source, target)
+        shared_tags = configured_intersection(source["tags"], target["tags"], @configuration.shared_tags?)
+        shared_categories = configured_intersection(
+          source["categories"], target["categories"], @configuration.same_category?
+        )
+        shared_domains = configured_intersection(
+          domain_paths(source), domain_paths(target), @configuration.include_parent_domains?
+        ) - shared_categories
+        return if shared_tags.empty? && shared_categories.empty? && shared_domains.empty?
+
+        {
+          score: metadata_score(source, target, shared_tags, shared_categories, shared_domains),
+          tags: shared_tags,
+          categories: shared_categories,
+          domains: shared_domains,
+          reasons: metadata_reasons(shared_tags, shared_categories, shared_domains)
+        }
+      end
+
+      def configured_intersection(first, second, enabled)
+        enabled ? intersection(first, second) : []
+      end
+
+      def metadata_score(source, target, tags, categories, domains)
+        tag_score = tags.empty? ? 0.0 : 0.25 * tags.length.to_f / union(source["tags"], target["tags"]).length
+        [tag_score + (categories.any? ? 0.5 : 0.0) + (domains.any? ? 0.25 : 0.0), 0.01].max
+      end
+
+      def metadata_reasons(tags, categories, domains)
+        tags.map { |tag| "shared-tag: #{tag}" } +
+          categories.map { |category| "shared-category: #{category}" } +
+          domains.map { |domain| "shared-domain: #{domain}" }
+      end
+
+      def sort_relations(relations)
+        sorted = relations.sort_by do |relation|
+          [-relation.fetch("score"), relation.fetch("title"), relation.fetch("id")]
+        end
+        maximum = @configuration.max_items
+        maximum ? sorted.first(maximum) : sorted
+      end
+
+      def similarity(first, second)
+        return unless valid_vector?(first)
+        return unless valid_vector?(second)
+        return unless first.length == second.length
+
+        dot = first.zip(second).sum { |a, b| a * b }
+        denominator = vector_norm(first) * vector_norm(second)
+        denominator.zero? ? 0.0 : dot / denominator
+      end
+
+      def vector_norm(vector)
+        Math.sqrt(vector.sum { |value| value * value })
+      end
+
+      def valid_vector?(vector)
+        return false unless vector.is_a?(Array)
+        return false if vector.empty?
+
+        vector.all? { |value| value.is_a?(Numeric) && value.finite? }
+      end
+
+      def intersection(first, second)
+        (Array(first).map(&:to_s) & Array(second).map(&:to_s)).sort
+      end
+
+      def union(first, second)
+        (Array(first).map(&:to_s) | Array(second).map(&:to_s))
+      end
+
+      def domain_paths(document)
+        Array(document["categories"]).flat_map do |category|
+          parts = category.to_s.split("/").reject(&:empty?)
+          parts.each_index.map { |index| parts.first(index + 1).join("/") }
+        end.uniq
+      end
+    end
+  end
+end

data/lib/jekyll/client_search/related_configuration.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module Jekyll
+  module ClientSearch
+    # Validates related-article output and matching rules.
+    class RelatedConfiguration
+      DEFAULTS = {
+        "enabled" => false,
+        "output" => "search-relations.json",
+        "same_category" => true,
+        "shared_tags" => true,
+        "include_parent_domains" => true,
+        "semantic" => true,
+        "minimum_similarity" => 0.55,
+        "max_items" => nil
+      }.freeze
+
+      def initialize(config)
+        unless config.is_a?(Hash)
+          raise Jekyll::Errors::FatalException,
+                "client_search related configuration must be a mapping"
+        end
+
+        @values = DEFAULTS.merge(config)
+        validate!
+      end
+
+      def enabled?
+        @values.fetch("enabled") == true
+      end
+
+      def output
+        value = @values.fetch("output").to_s.sub(%r{\A/+}, "")
+        normalized = Pathname.new(value).cleanpath.to_s
+        return normalized unless value.empty? || normalized == "." || normalized.start_with?("../")
+
+        raise Jekyll::Errors::FatalException,
+              "client_search related output must be a relative file path inside the destination"
+      end
+
+      def same_category?
+        @values.fetch("same_category") == true
+      end
+
+      def shared_tags?
+        @values.fetch("shared_tags") == true
+      end
+
+      def include_parent_domains?
+        @values.fetch("include_parent_domains") == true
+      end
+
+      def semantic?
+        @values.fetch("semantic") == true
+      end
+
+      def minimum_similarity
+        @values.fetch("minimum_similarity").to_f
+      end
+
+      def max_items
+        @values.fetch("max_items")
+      end
+
+      private
+
+      def validate!
+        validate_booleans!
+        validate_similarity!
+        validate_max_items!
+      end
+
+      def validate_booleans!
+        %w[enabled same_category shared_tags include_parent_domains semantic].each do |key|
+          validate_boolean!(key)
+        end
+      end
+
+      def validate_similarity!
+        similarity = @values.fetch("minimum_similarity")
+        return if similarity.is_a?(Numeric) && similarity.finite? && similarity.between?(-1, 1)
+
+        raise Jekyll::Errors::FatalException,
+              "client_search related minimum_similarity must be finite and between -1 and 1"
+      end
+
+      def validate_max_items!
+        maximum = max_items
+        return if maximum.nil? || (maximum.is_a?(Integer) && maximum.positive?)
+
+        raise Jekyll::Errors::FatalException,
+              "client_search related max_items must be a positive integer or null"
+      end
+
+      def validate_boolean!(key)
+        return if [true, false].include?(@values[key])
+
+        raise Jekyll::Errors::FatalException,
+              "client_search related #{key} must be true or false"
+      end
+    end
+  end
+end

data/lib/jekyll/client_search/related_page.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Jekyll
+  module ClientSearch
+    # Writes build-time related-article data as a standalone JSON page.
+    class RelatedPage < Jekyll::PageWithoutAFile
+      def initialize(site, output, relation_data)
+        super(site, site.source, File.dirname(output), File.basename(output))
+        self.data = { "layout" => nil, "sitemap" => false }
+        self.content = JSON.generate(relation_data)
+      end
+    end
+  end
+end

data/lib/jekyll/client_search/related_tag.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module Jekyll
+  module ClientSearch
+    # Liquid tag that renders the related-articles container, sort control,
+    # and runtime scripts in one line. Drop into any post layout:
+    #
+    #   {% related_articles %}
+    #   {% related_articles sort:date %}
+    #   {% related_articles no_scripts %}
+    #
+    # When +related.enabled+ is false the tag renders nothing, so it is safe
+    # to leave in a layout even when the feature is off.
+    class RelatedTag < Liquid::Tag
+      SYNTAX = /\A\s*(sort:(\w+))?\s*(no_scripts)?\s*\z/
+
+      def initialize(tag_name, markup, tokens)
+        super
+        @markup = markup.to_s
+        unless (match = @markup.match(SYNTAX))
+          raise Liquid::SyntaxError,
+                "related_articles: invalid syntax. Use {% related_articles %}, " \
+                "{% related_articles sort:date %}, or {% related_articles no_scripts %}"
+        end
+
+        @sort = match[2] if match[2]
+        @include_scripts = match[3].nil?
+      end
+
+      def render(context)
+        site = context.registers[:site]
+        config = site&.config&.fetch("client_search", {})
+        return "" unless related_enabled?(config)
+
+        asset_prefix = asset_prefix(site)
+        sort_attr = @sort ? " data-related-sort=\"#{@sort}\"" : ""
+        scripts = build_scripts(asset_prefix)
+        build_html(sort_attr, scripts)
+      end
+
+      private
+
+      def related_enabled?(config)
+        related = config.fetch("related", {})
+        related["enabled"] == true
+      end
+
+      def asset_prefix(site)
+        baseurl = site.config["baseurl"].to_s.gsub(%r{\A/+|/+$}, "")
+        baseurl.empty? ? "" : "/#{baseurl}"
+      end
+
+      def build_scripts(prefix)
+        return "" unless @include_scripts
+
+        "\n<script src=\"#{prefix}/assets/search-runtime-config.js\"></script>\n" \
+          "<script src=\"#{prefix}/assets/client-search-related.js\"></script>"
+      end
+
+      def build_html(sort_attr, scripts)
+        <<~HTML
+          <section class="related-articles-section">
+            <label for="related-sort">Sort related articles</label>
+            <select id="related-sort">
+              <option value="relevance">Most related</option>
+              <option value="date">Newest</option>
+            </select>
+            <div id="related-articles"#{sort_attr}></div>
+          </section>#{scripts}
+        HTML
+      end
+    end
+  end
+end
+
+Liquid::Template.register_tag("related_articles", Jekyll::ClientSearch::RelatedTag)