jekyll-documents 0.6.0 → 0.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 989b1372181a7ca00444d25b5fab29751a077d9818c41cca1fe8c14208050214
-  data.tar.gz: 8215c9234df6eba090b83a296acd0c058a73ae3bd16b0d11fd2ce69e539d788e
+  metadata.gz: be2955da22172cc24f7aed4e3dca0ac968bf131040a36db5fe2ba048a64bca90
+  data.tar.gz: 2b3d0544fa7c86739e2d83cd9228ee24c64f6bc4ff7691211aefbc7f3533f837
 SHA512:
-  metadata.gz: ba352213f38733179f053f989cfe5f363dee453176f4d301629fa0d5ab34b5d4f3aad443e8f46bc9b4ee9106626d09cbc255544862b12ba05850259a5b8879e9
-  data.tar.gz: 911cb909b005316a55d1978127935cf1bbf6da093c5dd0a9b6ec9e66112a9aa88d094e82c318f8c7312450ec970592caf506a62b392b47e3f0f35380be5be9d5
+  metadata.gz: 1f2c7c21a5cdd8c5e996379467b2ceb38d69e368aa9f8bdae882a657366911fb1262f66a7c8ff549d848b1304fd90a8e56e9f37efe1b856f572fab550899d3d5
+  data.tar.gz: fce8b514c14788e75405a026dcd6e7eabf25955a12ac16445380f995728cf6f2c65099ab42af18c51c8fb2c1361f3f3bb9cdfa1845a3642e4495e532640a7d16

data/CHANGELOG.md

@@ -2,6 +2,28 @@
 
 ## [Unreleased]
 
+## [0.6.1] - 2026-09-03
+
+### Changed
+- Text extraction cache keys now include the source digest, extraction configuration, Plaintext version, and cache schema version so configuration or extractor changes invalidate stale results
+- Extracted document content now retains searchable metadata for title, category, file type, and date
+- Text cache cleanup now preserves shared digest files and removes orphaned files safely
+- Text extraction now enforces `text_max_bytes` as a UTF-8 byte limit and closes source files after reading
+- PDF extraction now discovers `pdftotext` from `PATH`; added macOS and Debian/Ubuntu setup instructions and CI coverage with `poppler-utils`
+- Configured text cache directories are excluded from Jekyll output regardless of the `extract_text` setting
+- Rake packaging and test tasks no longer perform duplicate work and install only the current package artifact
+- Removed redundant package metadata and duplicate development dependency declarations
+
+### Fixed
+- Replaced use of the private ActiveSupport deprecation API with the public `ActiveSupport.deprecator` API
+- Empty extraction results now fall back to metadata-only searchable content
+- Manifest loading now validates entries, logs filesystem read failures, and resets dirty state after successful saves
+- Updated stale CI, development, cache-cleanup, and package-lock documentation
+
+### Tests
+- Added real PDF extraction coverage and stronger cache-hit/persistence assertions
+- Added tests for cache configuration invalidation, UTF-8 byte truncation, shared cache references, orphan cleanup, and conditional cache exclusion
+
 ## [0.6.0] - 2026-08-27
 
 ### Added

data/README.md

@@ -257,7 +257,7 @@ documents:
 - Cache invalidation uses **SHA-256 file digests** — only changed files
   are re-extracted
 - Stale cache entries for deleted files are cleaned up automatically
-  at the start of each build
+  at the end of each build
 - Falls back to metadata-only content (title, category, file type, date)
   if the `plaintext` gem is missing or extraction fails
 
@@ -287,6 +287,20 @@ tools needed). PDF extraction shells out to a system command:
 | DOCX/PPTX/XLSX | rubyzip (Ruby gem, no CLI needed) |
 | ODT/ODS/ODP | rubyzip (Ruby gem, no CLI needed) |
 
+Install `pdftotext` for PDF extraction:
+
+```bash
+# macOS
+brew install poppler
+
+# Debian/Ubuntu
+sudo apt-get install poppler-utils
+```
+
+On macOS and other non-standard installations, the executable is discovered
+from `PATH`. You can also configure a custom `pdftotext` command through the
+`plaintext` gem configuration.
+
 ## Development
 
 ```bash

data/lib/jekyll/documents/generator.rb

@@ -28,6 +28,7 @@ module Jekyll
       def generate(site)
         @site   = site
         @config = Configuration.read(site)
+        @text_manifest = nil
 
         root = File.join(site.source, @config["root"])
         unless Dir.exist?(root)
@@ -128,18 +129,19 @@ module Jekyll
         manifest = text_manifest
         rel_path = info[:rel_path]
         digest = ::Digest::SHA256.file(info[:path]).hexdigest
+        metadata = extraction_cache_metadata
 
-        cached = manifest.get(rel_path, digest)
+        cached = manifest.get(rel_path, digest, metadata)
         return cached if cached
 
-        # Suppress ActiveSupport deprecation warnings from the plaintext gem
-        # (it uses String#mb_chars, deprecated in Rails 8.2).
-        # We apply our own truncation below, so the gem's internal limit is redundant.
-        text = ActiveSupport::Deprecation._instance.silence do
-          ::Plaintext::Resolver.new(File.open(info[:path]), content_type).text
+        configure_pdf_extractor(content_type)
+        text = ActiveSupport.deprecator.silence do
+          File.open(info[:path], "rb") do |file|
+            ::Plaintext::Resolver.new(file, content_type).text
+          end
         end
-        text = text&.truncate(@config["text_max_bytes"]) if text
-        manifest.set(rel_path, digest, text) if text
+        text = truncate_bytes(text, @config["text_max_bytes"]) if text
+        manifest.set(rel_path, digest, text, metadata) if text
         text
       rescue StandardError => e
         ::Jekyll.logger.warn "jekyll-documents",
@@ -147,6 +149,19 @@ module Jekyll
         nil
       end
 
+      def configure_pdf_extractor(content_type)
+        return unless content_type == "application/pdf"
+        return unless ::Plaintext::Configuration["pdftotext"].nil?
+
+        executable = ENV.fetch("PATH", "").split(File::PATH_SEPARATOR)
+                        .map { |directory| File.join(directory, "pdftotext") }
+                        .find { |path| File.executable?(path) }
+        return unless executable
+
+        ::Plaintext::Configuration.config["pdftotext"] =
+          [executable, "-enc", "UTF-8", "__FILE__", "-"]
+      end
+
       def load_plaintext
         return true if defined?(::Plaintext)
 
@@ -163,6 +178,26 @@ module Jekyll
         @text_manifest ||= TextExtractionManifest.new(@site, @config["text_cache_dir"])
       end
 
+      def extraction_cache_metadata
+        {
+          "schema_version" => 1,
+          "text_max_bytes" => @config["text_max_bytes"],
+          "plaintext_version" => if ::Plaintext.const_defined?(:VERSION)
+                                   ::Plaintext::VERSION.to_s
+                                 else
+                                   "unknown"
+                                 end
+        }
+      end
+
+      def truncate_bytes(text, max_bytes)
+        bytes = text.to_s.encode("UTF-8", invalid: :replace, undef: :replace)
+                    .byteslice(0, max_bytes)
+        bytes = bytes.to_s.force_encoding("UTF-8")
+        bytes = bytes.byteslice(0, bytes.bytesize - 1) until bytes.valid_encoding?
+        bytes
+      end
+
       def cleanup_manifest(current_rel_paths)
         text_manifest.cleanup_deleted(current_rel_paths)
         text_manifest.save
@@ -187,11 +222,16 @@ module Jekyll
         data["permalink"]  = @config["permalink"]
                              .gsub(":category", category.to_s)
                              .gsub(":slug", info[:slug])
+        metadata_content = searchable_content(info[:title], data, info[:file_type])
         doc.content = if @config["extract_text"]
-                        extract_file_content(info) || searchable_content(info[:title], data,
-                                                                         info[:file_type])
+                        extracted_content = extract_file_content(info)
+                        if extracted_content && !extracted_content.empty?
+                          "#{metadata_content} #{extracted_content}"
+                        else
+                          metadata_content
+                        end
                       else
-                        searchable_content(info[:title], data, info[:file_type])
+                        metadata_content
                       end
       end
 

data/lib/jekyll/documents/layout_registrar.rb

@@ -60,7 +60,7 @@ Jekyll::Hooks.register :site, :after_init do |site|
 
   # Exclude the text extraction cache directory from Jekyll output
   config = Jekyll::Documents::Configuration.read(site)
-  if config["extract_text"] && config["text_cache_dir"]
+  if config["text_cache_dir"]
     site.config["exclude"] = Array(site.config["exclude"])
     unless site.config["exclude"].include?(config["text_cache_dir"])
       site.config["exclude"] << config["text_cache_dir"]

data/lib/jekyll/documents/text_extraction_manifest.rb

@@ -6,6 +6,61 @@ require "digest"
 
 module Jekyll
   module Documents
+    module TextExtractionManifestSupport
+      private
+
+      def load_manifest
+        return {} unless File.file?(@manifest_path)
+
+        data = JSON.parse(File.read(@manifest_path, encoding: "UTF-8"))
+        data.is_a?(Hash) ? data.select { |_path, entry| valid_entry?(entry) } : {}
+      rescue JSON::ParserError => e
+        ::Jekyll.logger.warn "jekyll-documents",
+                             "Corrupt text extraction manifest, starting fresh: #{e.message}"
+        {}
+      rescue StandardError => e
+        ::Jekyll.logger.warn "jekyll-documents",
+                             "Text extraction manifest read failed: #{e.message}"
+        {}
+      end
+
+      def valid_entry?(entry)
+        entry.is_a?(Hash) && entry.values_at("digest", "cache_key", "text_file").all?(String)
+      end
+
+      def cache_key(digest, metadata)
+        Digest::SHA256.hexdigest(JSON.generate("digest" => digest, "metadata" => metadata))
+      end
+
+      def text_file_path(text_file)
+        return unless text_file.is_a?(String)
+
+        path = File.expand_path(File.join(@text_dir, text_file))
+        root = File.expand_path(@text_dir)
+        path.start_with?("#{root}#{File::SEPARATOR}") ? path : nil
+      end
+
+      def write_text_file(digest, text)
+        shard = digest[0, 2]
+        subdir = File.join(@text_dir, shard)
+        FileUtils.mkdir_p(subdir)
+        filename = "#{digest}.txt"
+        File.write(File.join(subdir, filename), text, encoding: "UTF-8")
+        File.join(shard, filename)
+      end
+
+      def cleanup_unreferenced_text_files
+        referenced = @manifest.values.filter_map do |entry|
+          text_file_path(entry["text_file"]) if valid_entry?(entry)
+        end.to_set
+        Dir.glob(File.join(@text_dir, "**", "*.txt")).each do |path|
+          File.delete(path) unless referenced.include?(File.expand_path(path))
+        end
+      rescue StandardError
+        nil
+      end
+    end
+
     # Manages the text extraction manifest — a persistent JSON file that tracks
     # which documents have had their text extracted, keyed by SHA-256 digest.
     #
@@ -19,6 +74,8 @@ module Jekyll
     # - Cleanup of entries for deleted source files
     # - Survives `jekyll clean` (stored in site source, not .jekyll-cache/)
     class TextExtractionManifest
+      include TextExtractionManifestSupport
+
       MANIFEST_FILENAME = "text-extraction-manifest.json"
       TEXT_SUBDIR = "text"
 
@@ -40,13 +97,13 @@ module Jekyll
       # @param rel_path [String] relative path of the source document
       # @param digest [String] SHA-256 hex digest of the source file
       # @return [String, nil] extracted text if cache hit, nil otherwise
-      def get(rel_path, digest)
+      def get(rel_path, digest, metadata = {})
         entry = @manifest[rel_path]
-        return nil unless entry
-        return nil unless entry["digest"] == digest
+        return nil unless valid_entry?(entry)
+        return nil unless entry["cache_key"] == cache_key(digest, metadata)
 
         text_file = text_file_path(entry["text_file"])
-        return nil unless File.file?(text_file)
+        return nil unless text_file && File.file?(text_file)
 
         File.read(text_file, encoding: "UTF-8")
       rescue StandardError => e
@@ -60,13 +117,15 @@ module Jekyll
       # @param digest [String] SHA-256 hex digest of the source file
       # @param text [String] the extracted text
       # @return [void]
-      def set(rel_path, digest, text)
+      def set(rel_path, digest, text, metadata = {})
         text_file = write_text_file(digest, text)
         @manifest[rel_path] = {
           "digest" => digest,
+          "cache_key" => cache_key(digest, metadata),
           "text_file" => text_file,
           "extracted_at" => Time.now.to_i
         }
+        cleanup_unreferenced_text_files
         @dirty = true
       end
 
@@ -81,12 +140,11 @@ module Jekyll
         @manifest.each_key do |rel_path|
           next if current_set.include?(rel_path)
 
-          entry = @manifest[rel_path]
-          delete_text_file(entry["text_file"]) if entry
           @manifest.delete(rel_path)
           removed += 1
           @dirty = true
         end
+        cleanup_unreferenced_text_files
 
         removed
       end
@@ -98,7 +156,10 @@ module Jekyll
         return unless @dirty
 
         content = JSON.pretty_generate(@manifest)
-        return if File.exist?(@manifest_path) && File.binread(@manifest_path) == content
+        if File.exist?(@manifest_path) && File.binread(@manifest_path) == content
+          @dirty = false
+          return
+        end
 
         FileUtils.mkdir_p(@cache_root)
         temporary_path = "#{@manifest_path}.tmp-#{Process.pid}-#{Thread.current.object_id}"
@@ -108,6 +169,7 @@ module Jekyll
           file.fsync
         end
         File.rename(temporary_path, @manifest_path)
+        @dirty = false
       ensure
         FileUtils.rm_f(temporary_path) if defined?(temporary_path) && temporary_path
       end
@@ -116,9 +178,9 @@ module Jekyll
       # @param rel_path [String] relative path of the source document
       # @param digest [String] SHA-256 hex digest of the source file
       # @return [Boolean]
-      def cached?(rel_path, digest)
+      def cached?(rel_path, digest, metadata = {})
         entry = @manifest[rel_path]
-        !!(entry && entry["digest"] == digest)
+        !!(valid_entry?(entry) && entry["cache_key"] == cache_key(digest, metadata))
       end
 
       # Number of entries in the manifest.
@@ -126,44 +188,6 @@ module Jekyll
       def size
         @manifest.size
       end
-
-      private
-
-      def load_manifest
-        return {} unless File.file?(@manifest_path)
-
-        data = JSON.parse(File.read(@manifest_path, encoding: "UTF-8"))
-        data.is_a?(Hash) ? data : {}
-      rescue JSON::ParserError => e
-        ::Jekyll.logger.warn "jekyll-documents",
-                             "Corrupt text extraction manifest, starting fresh: #{e.message}"
-        {}
-      rescue StandardError
-        {}
-      end
-
-      def text_file_path(text_file)
-        File.join(@text_dir, text_file)
-      end
-
-      def write_text_file(digest, text)
-        shard = digest[0, 2]
-        subdir = File.join(@text_dir, shard)
-        FileUtils.mkdir_p(subdir)
-        filename = "#{digest}.txt"
-        path = File.join(subdir, filename)
-        File.write(path, text, encoding: "UTF-8")
-        File.join(shard, filename)
-      end
-
-      def delete_text_file(text_file)
-        return unless text_file
-
-        path = text_file_path(text_file)
-        File.delete(path) if File.file?(path)
-      rescue StandardError
-        nil
-      end
     end
   end
 end

data/lib/jekyll/documents/version.rb

@@ -2,6 +2,6 @@
 
 module Jekyll
   module Documents
-    VERSION = "0.6.0"
+    VERSION = "0.6.1"
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: jekyll-documents
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.6.1
 platform: ruby
 authors:
 - Svend Gundestrup