jekyll-lunr 0.3.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 12b342989b9d4a31417fec52ca2af903d336abba1f540e18c96943c8c2f4e685
-  data.tar.gz: df578d7126aa4f04a93565e9acf96b6cb7f8ed84fd6411f8ae8ce15898c94ce9
+  metadata.gz: c6f2a0690b0ffa2d4427d9c37bc3ea7faa782daee6321854cb5e2c2eeba0db1c
+  data.tar.gz: 24f4998cb969809aca6f063571f30e19932658f9a320334985574b3c72624e8c
 SHA512:
-  metadata.gz: f05b4cdb24b2e4e6974de66670f17e99a876c9a0ad0493b42af8a7140f5559d15aa70766d47e8892a6f475111d01656edcea6b216d4f39968ea5a94e4d2adefd
-  data.tar.gz: c806c1dea9e7f497f2a754b92a962599c9021e9d1d6a3656a051000405b8bb9089ed6da02cac39994e8bcf9b502495d0d90634af2252018edc565327a280c684
+  metadata.gz: 2b44afdd44ec6d47e7fffc9284135846d96347d46e38ab416a53f093bf24a26a92e44fc79d830618fe9cfe06bbc5deb217a98bfe5c97663e34b8b0a0be99dc15
+  data.tar.gz: 2452c12b8d19cf43817c40474b8262679114800252a5a388eff83253bfbc9eaef0256f13bb5bf4fe7a6d29d7b8b3d317b944326853c4c78b369442198259eb8b

data/lib/jekyll/lunr.rb

@@ -3,12 +3,14 @@
 require 'json'
 require 'open3'
 require 'loofah'
+require 'stopwords'
 
 module Jekyll
   module Lunr
     # Error
     class Error < StandardError; end
 
+    # Add the indexer to the site
     module IndexableSite
       def self.included(base)
         base.class_eval do
@@ -19,6 +21,7 @@ module Jekyll
       end
     end
 
+    # Index a Jekyll::Document
     module IndexableDocument
       def self.included(base)
         base.class_eval do
@@ -26,8 +29,7 @@ module Jekyll
             data
               .slice(*site.indexer.fields)
               .transform_values { |value| extract_data_recursively(value) }
-              .merge('content' => Loofah.fragment(content).to_text,
-                     'url' => url,
+              .merge('content' => Loofah.fragment(content).to_text.strip,
                      'id' => url,
                      'year' => date&.year)
           end
@@ -38,6 +40,12 @@ module Jekyll
             when Hash then value.transform_values { |v| extract_data(v) }
             when Jekyll::Document then extract_data(value)
             when Jekyll::Page then extract_data(value)
+            when String
+              if value.include? '<'
+                Loofah.fragment(value).to_text.strip
+              else
+                value.strip
+              end
             else value
             end
           end
@@ -76,7 +84,10 @@ module Jekyll
         @indexable_data ||= data.map do |d|
           d.transform_values do |v|
             case v
-            when Array then v.map { |vv| vv.is_a?(Hash) ? vv['title'] : vv }.compact.join(', ')
+            when Array
+              v.map do |vv|
+                vv.is_a?(Hash) ? vv['title'] : vv
+              end.compact.join(', ')
             when Hash then v['title'] || v.values.map(&:to_s).join(', ')
             else v.to_s
             end
@@ -84,6 +95,19 @@ module Jekyll
         end
       end
 
+      def cleanup(data)
+        cleaned_data = data.dup
+
+        if (lang = lang&.to_sym)
+          sieve = Stopwords::Snowball::WordSieve.new
+          words = cleaned_data.split(' ').map(&:strip)
+
+          cleaned_data = site.filter(lang: lang, words: words).join(' ')
+        end
+
+        cleaned_data
+      end
+
       def write
         File.open(data_file.path, 'w') do |df|
           df.write data.to_json
@@ -111,7 +135,12 @@ module Jekyll
       def index
         Open3.popen2(env, *indexer) do |stdin, stdout, wait|
           indexable_data.each do |data|
-            stdin.puts data.to_json
+            stdin.puts(data.transform_values do |val|
+              case val
+              when String then cleanup val
+              else val
+              end
+            end.to_json)
           end
           stdin.close
 
@@ -127,7 +156,7 @@ module Jekyll
 
       # Site lang
       def lang
-        @lang ||= site.config.dig('lang').freeze
+        @lang ||= site.config['lang'].freeze
       end
 
       # Indexable fields

data/lib/jekyll-lunr.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'jekyll/lunr'
 
 Jekyll::Hooks.register :site, :pre_render do |site|

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: jekyll-lunr
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.1
 platform: ruby
 authors:
 - f
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-07-07 00:00:00.000000000 Z
+date: 2023-05-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: loofah
@@ -24,6 +24,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.4'
+- !ruby/object:Gem::Dependency
+  name: stopwords-filter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.6'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -97,7 +111,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.3.26
 signing_key:
 specification_version: 4
 summary: Lunr indexer for Jekyll