llm-docs-builder 0.9.1 → 0.9.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6ddb04b5a0e30d913d2043a79a3d7e14d35bd0166cf7104a300457887b1019cf
-  data.tar.gz: 5301d904225d0d139a2c2dd8184695eb66e2d858b8f3b5a86026b16a1ccb8c44
+  metadata.gz: 75429a83cfd019e7059f76a5fe7df200bff3bc14066c5f83e0739c85a5a68b63
+  data.tar.gz: 850d0568023dd1602cad0b48af5e6437b61f34f760c77b99adb565056313d241
 SHA512:
-  metadata.gz: 3daacfdde22c93023677e7e0e7487158b3e1f30a9c4e7ec2bf015bc220ff32296334411c74523aaa411d1ef21b1aa47a2e8cb2c7cada797922d89d5690f7ab0a
-  data.tar.gz: 50a8f8d29f9e79e6f5ab2774111385f6098378491b2732bb84a9d3eee0b2f5be4b6beae196ef76739a6ac1c81562da9ec6f3e66da562e657828874c55ec06ce1
+  metadata.gz: f0384e50c10837ec00e4d195115882d4fdaade5e522bba02929b028ac315d686324baf189bfc16803120b77718b28c10f2026e10f2f2bd3dd1ea7aab09f36549
+  data.tar.gz: 184e427780364d704067b1f84962fa7c658782ff2e048c991b34c06ca2643029d107872df1a0beac92cfb07076ae72cf9289527d534cad6274a99fdeaef22ac0

data/CHANGELOG.md

@@ -1,8 +1,35 @@
 # Changelog
 
+## 0.9.3 (2025-10-27)
+- [Fix] **Generate Command Excludes Support** - The `generate` command now properly respects the `excludes` configuration option to filter out files from llms.txt generation.
+  - Added `should_exclude?` method to Generator class that matches files against glob patterns
+  - Supports both simple patterns (e.g., `draft.md`) and glob patterns (e.g., `**/private/**`, `draft-*.md`)
+  - Uses `File.fnmatch` with `FNM_PATHNAME` and `FNM_DOTMATCH` flags for proper pattern matching
+  - Checks patterns against both absolute and relative paths from docs_path
+  - Excludes configuration works consistently with bulk-transform command
+- [Fix] **Token Count from Transformed Content** - Token counts in metadata now accurately reflect the actual content after applying transformations.
+  - Token count is now calculated from transformed content when any transformation options are enabled
+  - Adds `has_transformations?` helper method to detect if transformations are active
+  - Ensures token metadata represents the actual size of processed content, not raw files
+  - Falls back to raw content token count when no transformations are enabled
+- [Fix] **Boolean Config Options** - Fixed config merging bug where explicitly setting transformation options to `false` in YAML was being overridden to `true`.
+  - Updated `Config#merge_with_options` to properly handle `false` values for boolean options
+  - Fixed the `|| true` pattern that was incorrectly treating `false` config values as falsy
+  - Now correctly uses `!self['option'].nil?` check before falling back to defaults
+  - Applies to all boolean transformation options: `remove_comments`, `normalize_whitespace`, `remove_badges`, `remove_frontmatter`
+- [Test] Added comprehensive unit tests for excludes functionality in Generator
+- [Test] Added integration tests for generate command with excludes and token counting
+
+## 0.9.2 (2025-10-17)
+- [Fix] Tackle one more block boundaries tracking edge-case.
+
 ## 0.9.1 (2025-10-17)
 - [Fix] Fixed HeadingTransformer incorrectly treating hash symbols in code blocks as headings.
   - Now properly tracks code block boundaries (fenced with ``` or ~~~)
+  - Fixed regex pattern from `/^```|^~~~/` to `/^(```|~~~)/` for correct operator precedence
+  - Skips heading processing for lines inside code blocks
+  - Prevents Ruby/Python/Shell comments from being interpreted as markdown headings
+  - Added 5 comprehensive test cases covering multiple scenarios to prevent regression
   - Skips heading processing for lines inside code blocks
   - Prevents Ruby/Python/Shell comments from being interpreted as markdown headings
   - Added comprehensive test coverage for code block handling

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    llm-docs-builder (0.9.1)
+    llm-docs-builder (0.9.3)
       zeitwerk (~> 2.6)
 
 GEM

data/lib/llm_docs_builder/config.rb

@@ -69,23 +69,31 @@ module LlmDocsBuilder
                       end,
         remove_comments: if options.key?(:remove_comments)
                            options[:remove_comments]
+                         elsif !self['remove_comments'].nil?
+                           self['remove_comments']
                          else
-                           self['remove_comments'] || true
+                           true
                          end,
         normalize_whitespace: if options.key?(:normalize_whitespace)
                                 options[:normalize_whitespace]
+                              elsif !self['normalize_whitespace'].nil?
+                                self['normalize_whitespace']
                               else
-                                self['normalize_whitespace'] || true
+                                true
                               end,
         remove_badges: if options.key?(:remove_badges)
                          options[:remove_badges]
+                       elsif !self['remove_badges'].nil?
+                         self['remove_badges']
                        else
-                         self['remove_badges'] || true
+                         true
                        end,
         remove_frontmatter: if options.key?(:remove_frontmatter)
                               options[:remove_frontmatter]
+                            elsif !self['remove_frontmatter'].nil?
+                              self['remove_frontmatter']
                             else
-                              self['remove_frontmatter'] || true
+                              true
                             end,
         verbose: options.key?(:verbose) ? options[:verbose] : (self['verbose'] || false),
         # Bulk transformation options

data/lib/llm_docs_builder/generator.rb

@@ -79,6 +79,7 @@ module LlmDocsBuilder
         next unless File.file?(path)
         next unless path.match?(/\.md$/i)
         next if File.basename(path).start_with?('.')
+        next if should_exclude?(path)
 
         files << analyze_file(path)
       end
@@ -111,7 +112,12 @@ module LlmDocsBuilder
 
       # Add optional enhanced metadata
       if options[:include_metadata]
-        metadata[:tokens] = TokenEstimator.estimate(content) if options[:include_tokens]
+        # Calculate token count from transformed content if any transformations are enabled
+        if options[:include_tokens]
+          token_content = has_transformations? ? apply_transformations(content, file_path) : content
+          metadata[:tokens] = TokenEstimator.estimate(token_content)
+        end
+
         metadata[:updated] = File.mtime(file_path).strftime('%Y-%m-%d') if options[:include_timestamps]
 
         # Calculate compression ratio if transformation is enabled
@@ -289,5 +295,48 @@ module LlmDocsBuilder
         'priority:low'
       end
     end
+
+    # Tests if file matches any exclusion pattern from options
+    #
+    # Uses File.fnmatch with pathname and dotmatch flags.
+    # Checks against both absolute path and relative path from docs_path.
+    #
+    # @param file_path [String] path to check
+    # @return [Boolean] true if file should be excluded
+    def should_exclude?(file_path)
+      excludes = Array(options[:excludes])
+      return false if excludes.empty?
+
+      # Get relative path from docs_path for matching
+      relative_path = if File.directory?(docs_path)
+                        Pathname.new(file_path).relative_path_from(Pathname.new(docs_path)).to_s
+                      else
+                        File.basename(file_path)
+                      end
+
+      excludes.any? do |pattern|
+        # Check both absolute and relative paths
+        File.fnmatch(pattern, file_path, File::FNM_PATHNAME | File::FNM_DOTMATCH) ||
+          File.fnmatch(pattern, relative_path, File::FNM_PATHNAME | File::FNM_DOTMATCH)
+      end
+    end
+
+    # Checks if any transformation options are enabled
+    #
+    # @return [Boolean] true if any transformation option is enabled
+    def has_transformations?
+      [
+        :remove_comments,
+        :normalize_whitespace,
+        :remove_badges,
+        :remove_frontmatter,
+        :remove_code_examples,
+        :remove_images,
+        :simplify_links,
+        :remove_blockquotes,
+        :remove_stopwords,
+        :remove_duplicates
+      ].any? { |opt| options[opt] }
+    end
   end
 end

data/lib/llm_docs_builder/transformers/heading_transformer.rb

@@ -42,7 +42,7 @@ module LlmDocsBuilder
 
         transformed_lines = lines.map do |line|
           # Track code block boundaries (fenced code blocks with ``` or ~~~)
-          if line.match?(/^```|^~~~/)
+          if line.match?(/^(```|~~~)/)
             in_code_block = !in_code_block
             next line
           end

data/lib/llm_docs_builder/version.rb

@@ -2,5 +2,5 @@
 
 module LlmDocsBuilder
   # Current version of the LlmDocsBuilder gem
-  VERSION = '0.9.1'
+  VERSION = '0.9.3'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: llm-docs-builder
 version: !ruby/object:Gem::Version
-  version: 0.9.1
+  version: 0.9.3
 platform: ruby
 authors:
 - Maciej Mensfeld