jekyll_ranked_search 0.0.5 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 285d83435623362b1b2a64895f39e9083a23e994f44772b8d7d0ee119bb0a2a8
-  data.tar.gz: 3f87f6708fb4bb8070ffbf22a388345ec0ead46c251aa94b7828d3cb8e0653d6
+  metadata.gz: de4d3c57baffb900a2d63699eb24a6f625ad620d0a5b11601bae89c38259ac77
+  data.tar.gz: 8673a43723f4a2a3c26e3f9b7e0bc3ad8c468775ab2a619761fdac66d3de8223
 SHA512:
-  metadata.gz: c5b400d6ad614276e3f325923c7a0ba56a78fc216e4eeaf26acb5fd070a89e90b41e995c16760f337c983b1902d892d8f96bfdba1312000e8f9bfff71104cb00
-  data.tar.gz: 132c37490020d2205f526ae84c0f190236e265ac19f33f12abb4654d0350547055e0549517e3df554b5a46d9271f43e80556dafbeadf5fd98f79c4992a126d70
+  metadata.gz: 763bda3f5f7be27375fd589a68a1fae591d189aebf7618965eeb3a46ca40538e81495b04621821f8a71a37fdbfac152bbac3f6e2b4232c1b3bf335b389c8788c
+  data.tar.gz: a6ee82171745c1d46e1ea3dba7f74985f0ff00d4ff84f569e41a888f7b95f920e74195bad8a5fb253312b583b4ef4df3c349e58cbf1eae4302ff327bce2fe1e0

data/lib/jekyll_ranked_search.rb

@@ -122,32 +122,40 @@ class TfidfConverter < Jekyll::Generator
     site.data['tfidf'] = tfidf.to_json
   end
 
+  # Tokenize document by removing special characters and splitting
+  # the document into tokens.
+  # @param [String] doc The document to tokenize
+  # @return [Array<String>] individual tokens/words
   def tokenize_words(doc)
     # Remove stopwords from document
     @stopwords ||= self.load_stopwords
 
+    # TODO: Remove Liquid tags via regex
+
     # Split document into tokens
     splitted_doc = doc.strip.downcase.split
 
-    # Remove stopwords in place
-    splitted_doc.delete_if { |word| @stopwords.include?(word) }
-
-    # Remove special characters (only at beginning and end)
+    # Remove special characters
     splitted_doc.map! { |word| word.gsub(/[^a-z0-9_\/\-\s]/i, '') }
 
+    # Remove stopwords in place
+    splitted_doc.delete_if { |t| @stopwords.include? t }
+
     splitted_doc
   end
 
-  # Load stopwords from file
+  # Load english stopwords from file
+  # @return [Set<String>] the stopwords
   def load_stopwords
-    Jekyll.logger.info "Loading stopwords"
+    filename = File.join(File.dirname(__FILE__), "stopwords/en.txt")
+    Jekyll.logger.info "Loading stopwords: ", filename
     stopwords = Set.new
-    File.open(File.join(File.dirname(__FILE__), "stopwords.txt"), "r") do |f|
+    File.open(filename, "r") do |f|
       f.each_line do |line|
         stopwords.add line.strip
       end
     end
-    Jekyll.logger.info "Done loading #{stopwords.length} stopwords"
+    Jekyll.logger.info "Loaded #{stopwords.length} stopwords"
     stopwords
   end
 

data/lib/{stopwords.txt → stopwords/en.txt}

@@ -2,8 +2,6 @@
 'tis
 'twas
 've
-10
-39
 a
 a's
 able

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: jekyll_ranked_search
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.7
 platform: ruby
 authors:
 - Friedrich Ewald
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-07-28 00:00:00.000000000 Z
+date: 2024-01-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redcarpet
@@ -24,7 +24,10 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.6'
-description: Offline search plugin for Jekyll posts using TF-IDF
+description: |
+  A webcomponent based search box that provides search functionality for your Jekyll blog.
+
+  If you have any feedback or suggestions for improvement, please open an issue on Github.
 email: freddiemailster@gmail.com
 executables: []
 extensions: []
@@ -33,7 +36,7 @@ files:
 - lib/jekyll_ranked_search.rb
 - lib/search.js
 - lib/search.json
-- lib/stopwords.txt
+- lib/stopwords/en.txt
 homepage: https://github.com/f-ewald/jekyll_ranked_search
 licenses:
 - MIT