jekyll-lunr 0.2.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e18aceb9b02aa376f8c9006f399c0135362db2cdf2c429876e7e4a5856b560d8
-  data.tar.gz: e3fed8cb38dd1610db828a54ae1b3a9361b795e9b816f2df4c8826f72d566bd6
+  metadata.gz: 6d9231b2c5cdb6d759407c816f1b4bd7261d645d618fdc6e4a8e4820c30a7f31
+  data.tar.gz: 7bfcba35fffba4f8320d1f71d30fd3a3b049bac0f676051949c9a7092a983a6b
 SHA512:
-  metadata.gz: 530ec545d0ce6d20ed562534d2e77f0d6abdac0510e816f525fd12196d05d675fd0286190944aaebed329399af744451115324a43aa285062b8c92e04f4409d3
-  data.tar.gz: 88a4ef07c75174d03aaa89ebf0834b1665b53c3219c0c89a3d72e36915149270142b76d7ee9aa630c947dcb2b68a0cc4ad65195fee35761345c707513f11540f
+  metadata.gz: 02d33e0d128d47d83ee4222262a52918052a193ebcba6abe0b14bd3a0a82e019686ff8c4fd227f41881ff7ef70cc72e816e439f7c587ce33e401861d03b2e7af
+  data.tar.gz: daf4707ffa247fd056c3c50eb187f8bc5606702b50c03376aff61f03b643f5683e8dbf681c84d0804e3db7b243aa6510de24835844312b48f32218b1a84b79f1

data/README.md

@@ -55,6 +55,11 @@ jekyll-lunr:
 Two files will be generated, `data.json` and `idx.json`.  You can
 download them from your `search.js`.
 
+## Skip indexing
+
+Following jekyll-sitemap convention, posts with `sitemap: false` in
+their front matter are skipped from indexing.
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies.

data/lib/jekyll/lunr.rb

@@ -3,12 +3,14 @@
 require 'json'
 require 'open3'
 require 'loofah'
+require 'stopwords'
 
 module Jekyll
   module Lunr
     # Error
     class Error < StandardError; end
 
+    # Add the indexer to the site
     module IndexableSite
       def self.included(base)
         base.class_eval do
@@ -19,6 +21,7 @@ module Jekyll
       end
     end
 
+    # Index a Jekyll::Document
     module IndexableDocument
       def self.included(base)
         base.class_eval do
@@ -26,8 +29,7 @@ module Jekyll
             data
               .slice(*site.indexer.fields)
               .transform_values { |value| extract_data_recursively(value) }
-              .merge('content' => Loofah.fragment(content).to_text,
-                     'url' => url,
+              .merge('content' => Loofah.fragment(content).to_text.strip,
                      'id' => url,
                      'year' => date&.year)
           end
@@ -38,6 +40,12 @@ module Jekyll
             when Hash then value.transform_values { |v| extract_data(v) }
             when Jekyll::Document then extract_data(value)
             when Jekyll::Page then extract_data(value)
+            when String
+              if value.include? '<'
+                Loofah.fragment(value).to_text.strip
+              else
+                value.strip
+              end
             else value
             end
           end
@@ -60,7 +68,11 @@ module Jekyll
       # The data is the register where Lunr looks for results.
       # TODO: Write a single data file per doc?
       def data
-        @data ||= site.documents.map(&:to_data)
+        @data ||= site.documents.select do |doc|
+          doc.respond_to? :to_data
+        end.reject do |doc|
+          doc.data['sitemap'] == false
+        end.map(&:to_data)
       end
 
       def data_file
@@ -72,14 +84,30 @@ module Jekyll
         @indexable_data ||= data.map do |d|
           d.transform_values do |v|
             case v
-            when Array then v.join(', ')
-            when Hash then v.values.join(', ')
+            when Array
+              v.map do |vv|
+                vv.is_a?(Hash) ? vv['title'] : vv
+              end.compact.join(', ')
+            when Hash then v['title'] || v.values.map(&:to_s).join(', ')
             else v.to_s
             end
           end
         end
       end
 
+      def cleanup(data)
+        cleaned_data = data.dup
+
+        if (lang = lang&.to_sym)
+          sieve = Stopwords::Snowball::WordSieve.new
+          words = cleaned_data.split(' ').map(&:strip)
+
+          cleaned_data = site.filter(lang: lang, words: words).join(' ')
+        end
+
+        cleaned_data
+      end
+
       def write
         File.open(data_file.path, 'w') do |df|
           df.write data.to_json
@@ -107,7 +135,12 @@ module Jekyll
       def index
         Open3.popen2(env, *indexer) do |stdin, stdout, wait|
           indexable_data.each do |data|
-            stdin.puts data.to_json
+            stdin.puts(data.transform_values do |val|
+              case val
+              when String then cleanup val
+              else val
+              end
+            end.to_json)
           end
           stdin.close
 
@@ -123,7 +156,7 @@ module Jekyll
 
       # Site lang
       def lang
-        @lang ||= site.config.dig('lang').freeze
+        @lang ||= site.config['lang'].freeze
       end
 
       # Indexable fields

data/lib/jekyll-lunr.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'jekyll/lunr'
 
 Jekyll::Hooks.register :site, :pre_render do |site|

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: jekyll-lunr
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.4.0
 platform: ruby
 authors:
 - f
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-10-15 00:00:00.000000000 Z
+date: 2023-05-04 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: jekyll
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.2'
 - !ruby/object:Gem::Dependency
   name: loofah
   requirement: !ruby/object:Gem::Requirement
@@ -24,6 +38,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.4'
+- !ruby/object:Gem::Dependency
+  name: stopwords-filter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.6'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -97,7 +125,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.3.26
 signing_key:
 specification_version: 4
 summary: Lunr indexer for Jekyll