llm-docs-builder 0.9.2 → 0.9.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e2b8530a9f890cb3deed824d3520f0a267a10b1e60a38e6bc8948b6447f50200
-  data.tar.gz: 7fdca5e5403050d9f4845bf7deb29fd64fc7df8758004a8f97d5e5ff4c519d27
+  metadata.gz: 66983a07a7271c966999350d03fbde1b1080ef0ac05a7209452cbb8720074e1b
+  data.tar.gz: 5d7cb81a700db6a43c17145af56ffcd2f6cea4a9a5182d4a2b14fd7772c8ee07
 SHA512:
-  metadata.gz: 0bd796067af5a3246ff99989d0190c33882218a5f5ea0a6f8dfc9a6291ab5fba122269fc1b522acebd2782d07f1ff99a38557efcf616e5e88bfb5ad1c34ec75e
-  data.tar.gz: 9b2e456821ccc73790fb7feb8f1046c89d4111c47be4f6755ba2059ea6ab2b5433229e5053a883b0b976b73875889af085f266089d57468084550aa6deb248d3
+  metadata.gz: 416b6f94c7e7dbac3bf3e6ae8793adcf9893d685aec29cc83665c2f9a6ab312bd422542e0074bbeec961f23740698aac6e517ebb7c87f3cb0fef3c8c6067c662
+  data.tar.gz: 2fba65092d82dbbeea60ce05317a781ed82f20367313d4eab9a69425e6e51f70ec106ca5f56218bbe255d1485752732de2a876359fde7018e706b948d51053fb

data/CHANGELOG.md

@@ -1,5 +1,40 @@
 # Changelog
 
+## 0.9.4 (2025-10-27)
+- [Feature] **Auto-Exclude Hidden Directories** - Hidden directories (starting with `.`) are now automatically excluded by default to prevent noise from `.git`, `.lint`, `.github`, etc.
+  - Adds `include_hidden: false` as default behavior
+  - Set `include_hidden: true` in config to include hidden directories if needed
+  - Uses `Find.prune` for efficient directory tree traversal
+  - Prevents scanning of common directories like `.lint`, `.gh`, `.git`, `node_modules` (if hidden)
+  - Fixed bug where root directory `.` was being pruned when used as docs_path
+- [Fix] **Excludes Pattern Matching** - Fixed fnmatch pattern handling for better glob pattern support.
+  - Fixed `**/.dir/**` patterns now correctly match root-level directories
+  - Normalized patterns ending with `/**` to `/**/*` for proper fnmatch behavior
+  - Handles `**/` prefix matching for zero-directory cases
+  - Fixed relative path calculation to avoid "different prefix" errors
+- [Test] Added unit tests for hidden directory exclusion feature (5 tests)
+- [Test] Added integration tests for hidden directory behavior (3 tests)
+
+## 0.9.3 (2025-10-27)
+- [Fix] **Generate Command Excludes Support** - The `generate` command now properly respects the `excludes` configuration option to filter out files from llms.txt generation.
+  - Added `should_exclude?` method to Generator class that matches files against glob patterns
+  - Supports both simple patterns (e.g., `draft.md`) and glob patterns (e.g., `**/private/**`, `draft-*.md`)
+  - Uses `File.fnmatch` with `FNM_PATHNAME` and `FNM_DOTMATCH` flags for proper pattern matching
+  - Checks patterns against both absolute and relative paths from docs_path
+  - Excludes configuration works consistently with bulk-transform command
+- [Fix] **Token Count from Transformed Content** - Token counts in metadata now accurately reflect the actual content after applying transformations.
+  - Token count is now calculated from transformed content when any transformation options are enabled
+  - Adds `has_transformations?` helper method to detect if transformations are active
+  - Ensures token metadata represents the actual size of processed content, not raw files
+  - Falls back to raw content token count when no transformations are enabled
+- [Fix] **Boolean Config Options** - Fixed config merging bug where explicitly setting transformation options to `false` in YAML was being overridden to `true`.
+  - Updated `Config#merge_with_options` to properly handle `false` values for boolean options
+  - Fixed the `|| true` pattern that was incorrectly treating `false` config values as falsy
+  - Now correctly uses `!self['option'].nil?` check before falling back to defaults
+  - Applies to all boolean transformation options: `remove_comments`, `normalize_whitespace`, `remove_badges`, `remove_frontmatter`
+- [Test] Added comprehensive unit tests for excludes functionality in Generator
+- [Test] Added integration tests for generate command with excludes and token counting
+
 ## 0.9.2 (2025-10-17)
 - [Fix] Tackle one more block boundaries tracking edge-case.
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    llm-docs-builder (0.9.2)
+    llm-docs-builder (0.9.4)
       zeitwerk (~> 2.6)
 
 GEM

data/lib/llm_docs_builder/config.rb

@@ -69,29 +69,38 @@ module LlmDocsBuilder
                       end,
         remove_comments: if options.key?(:remove_comments)
                            options[:remove_comments]
+                         elsif !self['remove_comments'].nil?
+                           self['remove_comments']
                          else
-                           self['remove_comments'] || true
+                           true
                          end,
         normalize_whitespace: if options.key?(:normalize_whitespace)
                                 options[:normalize_whitespace]
+                              elsif !self['normalize_whitespace'].nil?
+                                self['normalize_whitespace']
                               else
-                                self['normalize_whitespace'] || true
+                                true
                               end,
         remove_badges: if options.key?(:remove_badges)
                          options[:remove_badges]
+                       elsif !self['remove_badges'].nil?
+                         self['remove_badges']
                        else
-                         self['remove_badges'] || true
+                         true
                        end,
         remove_frontmatter: if options.key?(:remove_frontmatter)
                               options[:remove_frontmatter]
+                            elsif !self['remove_frontmatter'].nil?
+                              self['remove_frontmatter']
                             else
-                              self['remove_frontmatter'] || true
+                              true
                             end,
         verbose: options.key?(:verbose) ? options[:verbose] : (self['verbose'] || false),
         # Bulk transformation options
         suffix: options[:suffix] || self['suffix'] || '.llm',
         excludes: options[:excludes] || self['excludes'] || [],
         bulk: options.key?(:bulk) ? options[:bulk] : (self['bulk'] || false),
+        include_hidden: options.key?(:include_hidden) ? options[:include_hidden] : (self['include_hidden'] || false),
         # New compression options
         remove_code_examples: if options.key?(:remove_code_examples)
                                 options[:remove_code_examples]

data/lib/llm_docs_builder/generator.rb

@@ -76,9 +76,17 @@ module LlmDocsBuilder
       files = []
 
       Find.find(docs_path) do |path|
+        # Skip hidden directories unless explicitly enabled
+        # Don't prune the root docs_path itself (even if it's ".")
+        if File.directory?(path) && path != docs_path && File.basename(path).start_with?('.') && !options[:include_hidden]
+          Find.prune
+          next
+        end
+
         next unless File.file?(path)
         next unless path.match?(/\.md$/i)
         next if File.basename(path).start_with?('.')
+        next if should_exclude?(path)
 
         files << analyze_file(path)
       end
@@ -111,7 +119,12 @@ module LlmDocsBuilder
 
       # Add optional enhanced metadata
       if options[:include_metadata]
-        metadata[:tokens] = TokenEstimator.estimate(content) if options[:include_tokens]
+        # Calculate token count from transformed content if any transformations are enabled
+        if options[:include_tokens]
+          token_content = has_transformations? ? apply_transformations(content, file_path) : content
+          metadata[:tokens] = TokenEstimator.estimate(token_content)
+        end
+
         metadata[:updated] = File.mtime(file_path).strftime('%Y-%m-%d') if options[:include_timestamps]
 
         # Calculate compression ratio if transformation is enabled
@@ -289,5 +302,70 @@ module LlmDocsBuilder
         'priority:low'
       end
     end
+
+    # Tests if file matches any exclusion pattern from options
+    #
+    # Uses File.fnmatch with pathname and dotmatch flags.
+    # Checks against both absolute path and relative path from docs_path.
+    #
+    # @param file_path [String] path to check
+    # @return [Boolean] true if file should be excluded
+    def should_exclude?(file_path)
+      excludes = Array(options[:excludes])
+      return false if excludes.empty?
+
+      # Get relative path from docs_path for matching
+      relative_path = begin
+        if File.directory?(docs_path)
+          # Convert both to absolute paths first to avoid "different prefix" error
+          abs_file = File.expand_path(file_path)
+          abs_docs = File.expand_path(docs_path)
+          Pathname.new(abs_file).relative_path_from(Pathname.new(abs_docs)).to_s
+        else
+          File.basename(file_path)
+        end
+      rescue ArgumentError
+        # If paths can't be made relative (different roots), use basename
+        File.basename(file_path)
+      end
+
+      excludes.any? do |pattern|
+        # Normalize pattern: ensure /** is followed by something
+        # fnmatch requires /** to be followed by at least one component
+        normalized_pattern = pattern.end_with?('/**') ? "#{pattern}/*" : pattern
+
+        # Check both absolute and relative paths
+        matches = File.fnmatch(normalized_pattern, file_path, File::FNM_PATHNAME | File::FNM_DOTMATCH) ||
+                  File.fnmatch(normalized_pattern, relative_path, File::FNM_PATHNAME | File::FNM_DOTMATCH)
+
+        # If pattern starts with **/, also try without it (for root-level matches)
+        # Since **/ in fnmatch doesn't match zero directories
+        if !matches && normalized_pattern.start_with?('**/')
+          pattern_without_prefix = normalized_pattern.sub(%r{^\*\*/}, '')
+          matches = File.fnmatch(pattern_without_prefix, file_path, File::FNM_PATHNAME | File::FNM_DOTMATCH) ||
+                    File.fnmatch(pattern_without_prefix, relative_path, File::FNM_PATHNAME | File::FNM_DOTMATCH)
+        end
+
+        matches
+      end
+    end
+
+    # Checks if any transformation options are enabled
+    #
+    # @return [Boolean] true if any transformation option is enabled
+    def has_transformations?
+      [
+        :remove_comments,
+        :normalize_whitespace,
+        :remove_badges,
+        :remove_frontmatter,
+        :remove_code_examples,
+        :remove_images,
+        :simplify_links,
+        :remove_blockquotes,
+        :remove_stopwords,
+        :remove_duplicates
+      ].any? { |opt| options[opt] }
+    end
   end
 end

data/lib/llm_docs_builder/version.rb

@@ -2,5 +2,5 @@
 
 module LlmDocsBuilder
   # Current version of the LlmDocsBuilder gem
-  VERSION = '0.9.2'
+  VERSION = '0.9.4'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: llm-docs-builder
 version: !ruby/object:Gem::Version
-  version: 0.9.2
+  version: 0.9.4
 platform: ruby
 authors:
 - Maciej Mensfeld