llms-txt-ruby 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 59e6ef21e4d8e7cfad82d91a1d0c6eede9528850622efef8738b718bbbe0ea43
-  data.tar.gz: 0247ceaaed63bea7ba5d86753cd77e19e032ec31561ab6fbfe35874d875cbda6
+  metadata.gz: '091a1f41ec541d6f747df9f11f06f09c59b2f0c2829ae9a0a4fae5624cb551ef'
+  data.tar.gz: f93506e0ff2326c1957affb5211de3f5a6ae022da75308b2274ddebac8d3560c
 SHA512:
-  metadata.gz: 13bcd98389bbfd2193e1d21b3b5372bd1656145aec325a0f1456ce95cb699106008c35914cdab9c6ed4e7cefa4c955791347179e003def2a8569ab267c387abc
-  data.tar.gz: 4115db81602c0137224f7e5c8b9c536b5684776c3398e04893d6f1e0b2a327af71918d34c661d83eae9e773654ef282947fcd231af0a3c4cfe8263f085455381
+  metadata.gz: 0fd566dd98d2fc189de5e4553ccb613c2481569040ae3ae6468e89a95fef0c1d3a80a9bd8f81ecedcbdc306d773bbd2dde1883a0ee66878b70d82332a21aac19
+  data.tar.gz: f8e0eb6c2c46c14ab4cc5fb56a99953d0771407188deb5d2c8138992d5d729cebc3250571d10f8e694033b7b4ab4dbc82de1a09cb66f5fa8db563d960a479068

data/.github/workflows/ci.yml

@@ -52,7 +52,7 @@ jobs:
       - name: Run all tests
         env:
           GITHUB_COVERAGE: ${{ matrix.coverage }}
-        run: bundle exec rspec
+        run: bin/rspecs
 
 
   ci-success:

data/.github/workflows/push.yml

@@ -24,7 +24,7 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Ruby
-        uses: ruby/setup-ruby@0481980f17b760ef6bca5e8c55809102a0af1e5a # v1.263.0
+        uses: ruby/setup-ruby@6797dcbb9a1889fd411d07e8aba7eded53fb8b48 # v1.264.0
         with:
           bundler-cache: false
 

data/.rubocop.yml

@@ -1,6 +1,7 @@
 AllCops:
-  TargetRubyVersion: 3.1
+  TargetRubyVersion: 3.2
   NewCops: enable
+  SuggestExtensions: false
 
 Style/Documentation:
   Enabled: false
@@ -10,18 +11,64 @@ Style/StringLiterals:
 
 Layout/LineLength:
   Max: 120
+  Exclude:
+    - 'lib/llms_txt/cli.rb'
 
 Metrics/ClassLength:
-  Max: 150
+  Max: 200
+  Exclude:
+    - 'lib/llms_txt/cli.rb'
 
 Metrics/MethodLength:
-  Max: 20
+  Max: 35
+  Exclude:
+    - 'lib/llms_txt/cli.rb'
 
 Metrics/AbcSize:
-  Max: 20
+  Max: 40
+  Exclude:
+    - 'lib/llms_txt/cli.rb'
 
 Metrics/CyclomaticComplexity:
-  Max: 10
+  Max: 15
+  Exclude:
+    - 'lib/llms_txt/config.rb'
+
+Metrics/PerceivedComplexity:
+  Max: 15
+  Exclude:
+    - 'lib/llms_txt/config.rb'
+
+Metrics/BlockLength:
+  Exclude:
+    - 'spec/**/*'
+    - 'lib/llms_txt/cli.rb'
+    - '*.gemspec'
 
 Style/FrozenStringLiteralComment:
-  Enabled: true
+  Enabled: true
+
+# Specs often have multiline block chains
+Style/MultilineBlockChain:
+  Exclude:
+    - 'spec/**/*'
+
+# Disable predicate method naming rule
+Naming/PredicateMethod:
+  Enabled: false
+
+# Allow development dependencies in gemspec
+Gemspec/DevelopmentDependencies:
+  Enabled: false
+
+# Enforce first argument on new line for multiline method calls
+Layout/FirstMethodArgumentLineBreak:
+  Enabled: true
+
+# Use fixed indentation for arguments
+Layout/ArgumentAlignment:
+  EnforcedStyle: with_fixed_indentation
+
+# Ensure closing parenthesis on new line for multiline calls
+Layout/MultilineMethodCallBraceLayout:
+  EnforcedStyle: new_line

data/.ruby-version

@@ -1 +1 @@
-3.4.6
+3.4.7

data/CHANGELOG.md

@@ -1,5 +1,28 @@
 # Changelog
 
+## 0.2.0 (2025-10-07)
+- [Breaking] Removed positional argument support for all CLI commands. All file paths must now be specified using flags:
+  - `transform`: use `-d/--docs` flag instead of positional argument
+  - `parse`: use `-d/--docs` flag instead of positional argument (defaults to `llms.txt` if not specified)
+  - `validate`: use `-d/--docs` flag instead of positional argument (defaults to `llms.txt` if not specified)
+- [Enhancement] Improved CLI consistency by requiring explicit flags for all file paths.
+- [Enhancement] Added comprehensive CLI integration tests in `spec/integrations/` directory.
+  - Each command has its own dedicated integration test file
+  - Tests verify actual CLI binary execution, not just Ruby API
+  - All tests (unit and integration) run together with `bin/rspecs`
+- [Enhancement] Added convenient test runner script `bin/rspecs` for running all tests.
+- [Enhancement] Added comprehensive YARD documentation to all CLI methods.
+- [Enhancement] Resolved all RuboCop offenses (0 offenses detected).
+- [Fix] Fixed validator bug where `each_value` was incorrectly called on Array.
+
+## 0.1.3 (2025-10-07)
+- [Fix] Fixed `transform` command to accept file path from `-d/--docs` flag in addition to positional arguments.
+
+## 0.1.2 (2025-10-07)
+- [Fix] Fixed CLI error handling to use correct `LlmsTxt::Errors::BaseError` instead of non-existent `LlmsTxt::Error`.
+- [Enhancement] Extracted CLI class to `lib/llms_txt/cli.rb` for better testability.
+- [Enhancement] Added comprehensive CLI error handling specs.
+
 ## 0.1.1 (2025-10-07)
 - [Change] Updated repository metadata to use `master` branch instead of `main`.
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    llms-txt-ruby (0.1.1)
+    llms-txt-ruby (0.2.0)
       zeitwerk (~> 2.6)
 
 GEM

data/README.md

@@ -24,7 +24,7 @@ This library converts existing human-first documentation into LLM-friendly forma
    AI-optimized format by expanding relative links to absolute URLs and normalizing link
    structures
 3. **Bulk transforms** - Processes all markdown files in a directory recursively, creating
-   LLM-friendly versions alongside originals with customizable exclusion patterns
+   LLM-friendly versions alongside originals (or transforming in-place) with customizable exclusion patterns
 
 ## Installation
 
@@ -76,7 +76,7 @@ llms-txt generate
 llms-txt generate --docs ./docs
 
 # Transform a single file
-llms-txt transform README.md
+llms-txt transform --docs README.md
 
 # Transform all markdown files in directory
 llms-txt bulk-transform --docs ./docs
@@ -91,10 +91,10 @@ llms-txt generate --config my-config.yml
 
 ```bash
 llms-txt generate [options]       # Generate llms.txt from documentation (default)
-llms-txt transform [file]         # Transform a markdown file to be AI-friendly
+llms-txt transform [options]      # Transform a markdown file to be AI-friendly
 llms-txt bulk-transform [options] # Transform all markdown files in directory
-llms-txt parse [file]             # Parse existing llms.txt file
-llms-txt validate [file]          # Validate llms.txt file
+llms-txt parse [options]          # Parse existing llms.txt file
+llms-txt validate [options]       # Validate llms.txt file
 llms-txt version                  # Show version
 ```
 
@@ -108,7 +108,7 @@ llms-txt version                  # Show version
 -h, --help               Show help message
 ```
 
-*For advanced options like base_url, title, description, and convert_urls, use a config file.*
+*For advanced options like base_url, title, description, suffix, excludes, and convert_urls, use a config file.*
 
 ## Configuration File
 
@@ -137,7 +137,13 @@ output: llms.txt
 
 # Transformation options (optional)
 convert_urls: true   # Convert .html links to .md
+suffix: .llm         # Suffix for transformed files (use "" for in-place)
 verbose: false       # Enable verbose output
+
+# Exclusion patterns (optional)
+excludes:
+  - "**/private/**"
+  - "**/drafts/**"
 ```
 
 The config file will be automatically found if named:
@@ -145,28 +151,136 @@ The config file will be automatically found if named:
 - `llms-txt.yaml`
 - `.llms-txt.yml`
 
+### Configuration Options Reference
+
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `docs` | String | `./docs` | Directory containing markdown files to process |
+| `base_url` | String | - | Base URL for expanding relative links (e.g., `https://myproject.io`) |
+| `title` | String | Auto-detected | Project title for llms.txt generation |
+| `description` | String | Auto-detected | Project description for llms.txt generation |
+| `output` | String | `llms.txt` | Output filename for generated llms.txt |
+| `convert_urls` | Boolean | `false` | Convert HTML URLs to markdown format (`.html` → `.md`) |
+| `suffix` | String | `.llm` | Suffix added to transformed files. Use `""` for in-place transformation |
+| `excludes` | Array | `[]` | Glob patterns for files/directories to exclude from processing |
+| `verbose` | Boolean | `false` | Enable detailed output during processing |
+
 ## Bulk Transformation
 
 The `bulk-transform` command processes all markdown files in a directory recursively, creating
-AI-friendly versions alongside the originals. This is perfect for transforming entire
-documentation trees.
+AI-friendly versions. By default, it creates new files with a `.llm.md` suffix, but you can also transform files in-place for build pipelines.
 
 ### Key Features
 
 - **Recursive processing** - Finds and transforms all `.md` files in nested directories
 - **Preserves structure** - Maintains your existing directory layout
 - **Exclusion patterns** - Skip files/directories using glob patterns
-- **Custom suffixes** - Choose how transformed files are named
+- **Custom suffixes** - Choose how transformed files are named, or transform in-place
 - **LLM optimizations** - Expands relative links, converts HTML URLs, etc.
 
-### Usage
+### Default Behavior: Creating Separate Files
+
+By default, `bulk-transform` creates new `.llm.md` files alongside your originals:
+
+```yaml
+# llms-txt.yml
+docs: ./docs
+base_url: https://myproject.io
+suffix: .llm         # Creates .llm.md files (default if omitted)
+convert_urls: true
+```
+
+```bash
+llms-txt bulk-transform --config llms-txt.yml
+```
+
+**Result:**
+```
+docs/
+├── README.md
+├── README.llm.md          ← AI-friendly version
+├── setup.md
+└── setup.llm.md           ← AI-friendly version
+```
+
+This preserves your original files and creates LLM-optimized versions separately.
+
+### In-Place Transformation
+
+For build pipelines where you want to transform documentation directly without maintaining separate copies, use `suffix: ""`:
+
+```yaml
+# llms-txt.yml
+docs: ./docs
+base_url: https://myproject.io
+convert_urls: true
+suffix: ""  # Transform in-place, no separate files
+excludes:
+  - "**/private/**"
+  - "**/drafts/**"
+```
+
+```bash
+llms-txt bulk-transform --config llms-txt.yml
+```
+
+**Before transformation** (`docs/setup.md`):
+```markdown
+See the [configuration guide](../config.md) for details.
+Visit our [API docs](https://myproject.io/api/).
+```
+
+**After transformation** (`docs/setup.md` - same file, overwritten):
+```markdown
+See the [configuration guide](https://myproject.io/docs/config.md) for details.
+Visit our [API docs](https://myproject.io/api.md).
+```
+
+This is perfect for:
+- **Build pipelines** - Transform docs as part of your deployment process
+- **Static site generators** - Process markdown before building HTML
+- **CI/CD workflows** - Automated documentation transformation
+
+### Real-World Example: Karafka Framework
+
+The [Karafka framework](https://github.com/karafka/website) uses in-place transformation in its documentation build process. Previously, it had 140+ lines of custom Ruby code for link expansion and URL conversion. Now it uses:
+
+```yaml
+# llms-txt.yml
+docs: ./online/docs
+base_url: https://karafka.io/docs
+convert_urls: true
+suffix: ""
+excludes:
+  - "**/Enterprise-License-Setup/**"
+```
 
 ```bash
-# Transform all files with default settings
+# In their build script (sync.rb)
+system!("llms-txt bulk-transform --config llms-txt.yml")
+```
+
+This configuration:
+- Processes all markdown files recursively in `./online/docs`
+- Expands relative links to absolute URLs using the base_url
+- Converts HTML URLs to markdown format (`.html` → `.md`)
+- Transforms files in-place (no separate `.llm.md` files)
+- Excludes password-protected enterprise documentation
+- Runs as part of an automated daily deployment via GitHub Actions
+
+**Result**: Over 140 lines of custom code replaced with a 6-line configuration file.
+
+### Usage Examples
+
+```bash
+# Transform all files with default settings (creates .llm.md files)
 llms-txt bulk-transform --docs ./wiki
 
-# Using config file (recommended for complex setups)
+# Transform in-place using config file
 llms-txt bulk-transform --config karafka-config.yml
+
+# Verbose output to see processing details
+llms-txt bulk-transform --config llms-txt.yml --verbose
 ```
 
 ### Example Config for Bulk Transformation
@@ -183,7 +297,7 @@ excludes:
   - "**/old-docs/**"     # Skip legacy documentation
 ```
 
-### Example Output
+### Example Output (Default Suffix)
 
 With the config above, these files:
 ```
@@ -213,10 +327,26 @@ wiki/
     └── internal.md      ← Excluded, no .llm.md version
 ```
 
+### Example Output (In-Place Transformation)
+
+With `suffix: ""`, the original files are overwritten:
+```
+wiki/
+├── Home.md              ← Transformed in-place
+├── getting-started.md   ← Transformed in-place
+├── api/
+│   ├── consumers.md     ← Transformed in-place
+│   └── producers.md     ← Transformed in-place
+└── private/
+    └── internal.md      ← Excluded from transformation
+```
+
 ## Serving LLM-Friendly Documentation
 
 After using `bulk-transform` to create `.llm.md` versions of your documentation, you can configure your web server to automatically serve these LLM-optimized versions to AI bots while showing the original versions to human visitors.
 
+> **Note:** This section applies when using the default `suffix: .llm` behavior. If you're using `suffix: ""` for in-place transformation, the markdown files are already LLM-optimized and can be served directly.
+
 ### How It Works
 
 The strategy is simple:
@@ -341,7 +471,7 @@ export default {
 
 ### Custom Suffix
 
-If you used a different suffix with the `bulk-transform` command (e.g., `--suffix .ai`), update your web server rules accordingly.
+If you used a different suffix with the `bulk-transform` command (e.g., `suffix: .ai`), update your web server rules accordingly.
 
 **Apache:**
 ```apache
@@ -358,24 +488,6 @@ rewrite ^(.*)\.md$ $1.ai.md last;
 const llmPath = url.pathname.replace(/\.md$/, '.ai.md');
 ```
 
-### Example Setup
-
-```yaml
-# llms-txt.yml
-docs: ./docs
-base_url: https://myproject.io
-suffix: .llm
-convert_urls: true
-```
-
-```bash
-# Generate LLM-friendly versions
-llms-txt bulk-transform --config llms-txt.yml
-
-# Deploy both original and .llm.md files to your web server
-# The server will automatically serve the right version to each visitor
-```
-
 ## Ruby API
 
 ### Basic Usage
@@ -410,7 +522,7 @@ transformed = LlmsTxt.transform_markdown('README.md',
   convert_urls: true
 )
 
-# Bulk transform all files in directory
+# Bulk transform all files in directory (creates .llm.md files)
 transformed_files = LlmsTxt.bulk_transform('./wiki',
   base_url: 'https://karafka.io',
   suffix: '.llm',
@@ -418,6 +530,14 @@ transformed_files = LlmsTxt.bulk_transform('./wiki',
 )
 puts "Transformed #{transformed_files.size} files"
 
+# Bulk transform in-place (overwrites original files)
+transformed_files = LlmsTxt.bulk_transform('./wiki',
+  base_url: 'https://karafka.io',
+  suffix: '',  # Empty string for in-place transformation
+  convert_urls: true,
+  excludes: ['**/private/**']
+)
+
 # Bulk transform with config file
 transformed_files = LlmsTxt.bulk_transform('./wiki',
   config_file: 'karafka-config.yml'

data/bin/llms-txt

@@ -1,242 +1,7 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
-require 'optparse'
 require 'llms_txt'
+require 'llms_txt/cli'
 
-module LlmsTxt
-  class CLI
-    def self.run(argv = ARGV)
-      new.run(argv)
-    end
-
-    def run(argv)
-      options = parse_options(argv)
-
-      case options[:command]
-      when 'generate', nil
-        generate(options)
-      when 'transform'
-        transform(options)
-      when 'bulk-transform'
-        bulk_transform(options)
-      when 'parse'
-        parse(options)
-      when 'validate'
-        validate(options)
-      when 'version'
-        show_version
-      else
-        puts "Unknown command: #{options[:command]}"
-        puts "Run 'llms-txt --help' for usage information"
-        exit 1
-      end
-    rescue LlmsTxt::Error => e
-      puts "Error: #{e.message}"
-      exit 1
-    rescue StandardError => e
-      puts "Unexpected error: #{e.message}"
-      puts e.backtrace.join("\n") if options&.fetch(:verbose, false)
-      exit 1
-    end
-
-    private
-
-    def parse_options(argv)
-      options = {
-        command: argv.first&.match?(/^[a-z-]+$/) ? argv.shift : nil
-      }
-
-      OptionParser.new do |opts|
-        opts.banner = "llms-txt - Simple tool for generating llms.txt from markdown documentation\n\nUsage: llms-txt [command] [options]\n\nFor advanced configuration (base_url, title, description, convert_urls), use a config file."
-
-        opts.separator ''
-        opts.separator 'Commands:'
-        opts.separator '  generate       Generate llms.txt from documentation (default)'
-        opts.separator '  transform      Transform a markdown file to be AI-friendly'
-        opts.separator '  bulk-transform Transform all markdown files in directory'
-        opts.separator '  parse          Parse existing llms.txt file'
-        opts.separator '  validate       Validate llms.txt file'
-        opts.separator '  version        Show version'
-
-        opts.separator ''
-        opts.separator 'Options:'
-
-        opts.on('-c', '--config PATH', 'Configuration file path (default: llms-txt.yml)') do |path|
-          options[:config] = path
-        end
-
-        opts.on('-d', '--docs PATH', 'Path to documentation directory or file') do |path|
-          options[:docs] = path
-        end
-
-        opts.on('-o', '--output PATH', 'Output file path') do |path|
-          options[:output] = path
-        end
-
-        opts.on('-v', '--verbose', 'Verbose output') do
-          options[:verbose] = true
-        end
-
-        opts.on('-h', '--help', 'Show this message') do
-          puts opts
-          exit
-        end
-
-        opts.on('--version', 'Show version') do
-          show_version
-          exit
-        end
-      end.parse!(argv)
-
-      options[:file_path] = argv.first if argv.any?
-      options
-    end
-
-    def generate(options)
-      # Load config and merge with CLI options
-      config = LlmsTxt::Config.new(options[:config])
-      merged_options = config.merge_with_options(options)
-
-      docs_path = merged_options[:docs]
-
-      unless File.exist?(docs_path)
-        puts "Documentation path not found: #{docs_path}"
-        exit 1
-      end
-
-      puts "Generating llms.txt from #{docs_path}..." if merged_options[:verbose]
-
-      content = LlmsTxt.generate_from_docs(docs_path, merged_options)
-      output_path = merged_options[:output]
-
-      File.write(output_path, content)
-      puts "Successfully generated #{output_path}"
-
-      if merged_options[:verbose]
-        validator = LlmsTxt::Validator.new(content)
-        if validator.valid?
-          puts "Valid llms.txt format"
-        else
-          puts "Validation warnings:"
-          validator.errors.each { |error| puts "  - #{error}" }
-        end
-      end
-    end
-
-    def transform(options)
-      # Load config and merge with CLI options
-      config = LlmsTxt::Config.new(options[:config])
-      merged_options = config.merge_with_options(options)
-
-      file_path = options[:file_path]
-
-      unless file_path
-        puts "File path required for transform command"
-        exit 1
-      end
-
-      unless File.exist?(file_path)
-        puts "File not found: #{file_path}"
-        exit 1
-      end
-
-      puts "Transforming #{file_path}..." if merged_options[:verbose]
-
-      content = LlmsTxt.transform_markdown(file_path, merged_options)
-
-      if merged_options[:output] && merged_options[:output] != 'llms.txt'
-        File.write(merged_options[:output], content)
-        puts "Transformed content saved to #{merged_options[:output]}"
-      else
-        puts content
-      end
-    end
-
-    def bulk_transform(options)
-      # Load config and merge with CLI options
-      config = LlmsTxt::Config.new(options[:config])
-      merged_options = config.merge_with_options(options)
-
-      docs_path = merged_options[:docs]
-
-      unless File.exist?(docs_path)
-        puts "Documentation path not found: #{docs_path}"
-        exit 1
-      end
-
-      unless File.directory?(docs_path)
-        puts "Path must be a directory for bulk transformation: #{docs_path}"
-        exit 1
-      end
-
-      puts "Bulk transforming markdown files in #{docs_path}..." if merged_options[:verbose]
-      puts "Using suffix: #{merged_options[:suffix]}" if merged_options[:verbose]
-      puts "Excludes: #{merged_options[:excludes].join(', ')}" if merged_options[:verbose] && !merged_options[:excludes].empty?
-
-      begin
-        transformed_files = LlmsTxt.bulk_transform(docs_path, merged_options)
-
-        if transformed_files.empty?
-          puts "No markdown files found to transform"
-        else
-          puts "Successfully transformed #{transformed_files.size} files:"
-          transformed_files.each { |file| puts "  #{file}" } unless merged_options[:verbose] # verbose mode already shows progress
-        end
-      rescue LlmsTxt::Error => e
-        puts "Error during bulk transformation: #{e.message}"
-        exit 1
-      end
-    end
-
-    def parse(options)
-      file_path = options[:file_path] || 'llms.txt'
-
-      unless File.exist?(file_path)
-        puts "File not found: #{file_path}"
-        exit 1
-      end
-
-      parsed = LlmsTxt.parse(file_path)
-
-      if options[:verbose]
-        puts "Title: #{parsed.title}"
-        puts "Description: #{parsed.description}"
-        puts "Documentation Links: #{parsed.documentation_links.size}"
-        puts "Example Links: #{parsed.example_links.size}" if parsed.respond_to?(:example_links)
-        puts "Optional Links: #{parsed.optional_links.size}" if parsed.respond_to?(:optional_links)
-      else
-        puts parsed.to_xml if parsed.respond_to?(:to_xml)
-      end
-    end
-
-    def validate(options)
-      file_path = options[:file_path] || 'llms.txt'
-
-      unless File.exist?(file_path)
-        puts "File not found: #{file_path}"
-        exit 1
-      end
-
-      content = File.read(file_path)
-      valid = LlmsTxt.validate(content)
-
-      if valid
-        puts 'Valid llms.txt file'
-      else
-        puts 'Invalid llms.txt file'
-        puts "\nErrors:"
-        LlmsTxt::Validator.new(content).errors.each do |error|
-          puts "  - #{error}"
-        end
-        exit 1
-      end
-    end
-
-    def show_version
-      puts "llms-txt version #{LlmsTxt::VERSION}"
-    end
-  end
-end
-
-LlmsTxt::CLI.run # if $PROGRAM_NAME == __FILE__
+LlmsTxt::CLI.run

data/bin/rspecs

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+# Run all tests (unit and integration specs)
+
+set -e
+
+echo "Running all tests..."
+bundle exec rspec --format documentation

data/lib/llms_txt/bulk_transformer.rb

@@ -47,9 +47,7 @@ module LlmsTxt
     #
     # @return [Array<String>] paths of transformed files
     def transform_all
-      unless File.directory?(docs_path)
-        raise Errors::GenerationError, "Directory not found: #{docs_path}"
-      end
+      raise Errors::GenerationError, "Directory not found: #{docs_path}" unless File.directory?(docs_path)
 
       markdown_files = find_markdown_files
       transformed_files = []

data/lib/llms_txt/cli.rb

@@ -0,0 +1,334 @@
+# frozen_string_literal: true
+
+require 'optparse'
+
+module LlmsTxt
+  # Command-line interface for llms-txt gem
+  #
+  # Provides commands for generating, transforming, parsing, and validating llms.txt files.
+  # All file paths must be specified using flags (-d/--docs) for consistency.
+  #
+  # @example Run the CLI
+  #   LlmsTxt::CLI.run(['generate', '--docs', './docs', '--output', 'llms.txt'])
+  #
+  # @api public
+  class CLI
+    # Run the CLI with given arguments
+    #
+    # @param argv [Array<String>] command-line arguments (defaults to ARGV)
+    def self.run(argv = ARGV)
+      new.run(argv)
+    end
+
+    # Execute CLI command with error handling
+    #
+    # Parses command-line arguments and delegates to appropriate command handler.
+    # Handles all LlmsTxt errors gracefully with user-friendly messages.
+    #
+    # @param argv [Array<String>] command-line arguments
+    # @raise [SystemExit] exits with status 1 on error
+    def run(argv)
+      options = parse_options(argv)
+
+      case options[:command]
+      when 'generate', nil
+        generate(options)
+      when 'transform'
+        transform(options)
+      when 'bulk-transform'
+        bulk_transform(options)
+      when 'parse'
+        parse(options)
+      when 'validate'
+        validate(options)
+      when 'version'
+        show_version
+      else
+        puts "Unknown command: #{options[:command]}"
+        puts "Run 'llms-txt --help' for usage information"
+        exit 1
+      end
+    rescue LlmsTxt::Errors::BaseError => e
+      puts "Error: #{e.message}"
+      exit 1
+    rescue StandardError => e
+      puts "Unexpected error: #{e.message}"
+      puts e.backtrace.join("\n") if options&.fetch(:verbose, false)
+      exit 1
+    end
+
+    private
+
+    # Parse command-line options using OptionParser
+    #
+    # Extracts command and options from argv. First non-flag argument is treated as command name.
+    #
+    # @param argv [Array<String>] command-line arguments
+    # @return [Hash] parsed options including :command, :config, :docs, :output, :verbose
+    def parse_options(argv)
+      options = {
+        command: argv.first&.match?(/^[a-z-]+$/) ? argv.shift : nil
+      }
+
+      OptionParser.new do |opts|
+        opts.banner = "llms-txt - Simple tool for generating llms.txt from markdown documentation\n\nUsage: llms-txt [command] [options]\n\nFor advanced configuration (base_url, title, description, convert_urls), use a config file."
+
+        opts.separator ''
+        opts.separator 'Commands:'
+        opts.separator '  generate       Generate llms.txt from documentation (default)'
+        opts.separator '  transform      Transform a markdown file to be AI-friendly'
+        opts.separator '  bulk-transform Transform all markdown files in directory'
+        opts.separator '  parse          Parse existing llms.txt file'
+        opts.separator '  validate       Validate llms.txt file'
+        opts.separator '  version        Show version'
+
+        opts.separator ''
+        opts.separator 'Options:'
+
+        opts.on('-c', '--config PATH', 'Configuration file path (default: llms-txt.yml)') do |path|
+          options[:config] = path
+        end
+
+        opts.on('-d', '--docs PATH', 'Path to documentation directory or file') do |path|
+          options[:docs] = path
+        end
+
+        opts.on('-o', '--output PATH', 'Output file path') do |path|
+          options[:output] = path
+        end
+
+        opts.on('-v', '--verbose', 'Verbose output') do
+          options[:verbose] = true
+        end
+
+        opts.on('-h', '--help', 'Show this message') do
+          puts opts
+          exit
+        end
+
+        opts.on('--version', 'Show version') do
+          show_version
+          exit
+        end
+      end.parse!(argv)
+
+      options
+    end
+
+    # Generate llms.txt from documentation directory or file
+    #
+    # Loads configuration, merges with CLI options, generates llms.txt content,
+    # and optionally validates the output.
+    #
+    # @param options [Hash] command options from parse_options
+    # @option options [String] :config path to config file
+    # @option options [String] :docs path to documentation
+    # @option options [String] :output output file path
+    # @option options [Boolean] :verbose enable verbose output
+    # @raise [SystemExit] exits with status 1 if docs path not found
+    def generate(options)
+      # Load config and merge with CLI options
+      config = LlmsTxt::Config.new(options[:config])
+      merged_options = config.merge_with_options(options)
+
+      docs_path = merged_options[:docs]
+
+      unless File.exist?(docs_path)
+        puts "Documentation path not found: #{docs_path}"
+        exit 1
+      end
+
+      puts "Generating llms.txt from #{docs_path}..." if merged_options[:verbose]
+
+      content = LlmsTxt.generate_from_docs(docs_path, merged_options)
+      output_path = merged_options[:output]
+
+      File.write(output_path, content)
+      puts "Successfully generated #{output_path}"
+
+      return unless merged_options[:verbose]
+
+      validator = LlmsTxt::Validator.new(content)
+      if validator.valid?
+        puts 'Valid llms.txt format'
+      else
+        puts 'Validation warnings:'
+        validator.errors.each { |error| puts "  - #{error}" }
+      end
+    end
+
+    # Transform markdown file to be AI-friendly
+    #
+    # Expands relative links to absolute URLs and optionally converts HTML URLs to markdown format.
+    #
+    # @param options [Hash] command options from parse_options
+    # @option options [String] :config path to config file
+    # @option options [String] :docs path to markdown file (required)
+    # @option options [String] :output output file path
+    # @option options [String] :base_url base URL for link expansion
+    # @option options [Boolean] :convert_urls convert .html to .md
+    # @option options [Boolean] :verbose enable verbose output
+    # @raise [SystemExit] exits with status 1 if file not found or -d flag missing
+    def transform(options)
+      # Load config and merge with CLI options
+      config = LlmsTxt::Config.new(options[:config])
+      merged_options = config.merge_with_options(options)
+
+      file_path = merged_options[:docs]
+
+      unless file_path
+        puts 'File path required for transform command (use -d/--docs)'
+        exit 1
+      end
+
+      unless File.exist?(file_path)
+        puts "File not found: #{file_path}"
+        exit 1
+      end
+
+      puts "Transforming #{file_path}..." if merged_options[:verbose]
+
+      content = LlmsTxt.transform_markdown(file_path, merged_options)
+
+      if merged_options[:output] && merged_options[:output] != 'llms.txt'
+        File.write(merged_options[:output], content)
+        puts "Transformed content saved to #{merged_options[:output]}"
+      else
+        puts content
+      end
+    end
+
+    # Transform all markdown files in directory recursively
+    #
+    # Creates AI-friendly versions of all markdown files with configurable suffix and exclusions.
+    #
+    # @param options [Hash] command options from parse_options
+    # @option options [String] :config path to config file
+    # @option options [String] :docs path to documentation directory (required)
+    # @option options [String] :suffix suffix for transformed files (default: '.llm')
+    # @option options [Array<String>] :excludes glob patterns to exclude
+    # @option options [String] :base_url base URL for link expansion
+    # @option options [Boolean] :convert_urls convert .html to .md
+    # @option options [Boolean] :verbose enable verbose output
+    # @raise [SystemExit] exits with status 1 if directory not found or transformation fails
+    def bulk_transform(options)
+      # Load config and merge with CLI options
+      config = LlmsTxt::Config.new(options[:config])
+      merged_options = config.merge_with_options(options)
+
+      docs_path = merged_options[:docs]
+
+      unless File.exist?(docs_path)
+        puts "Documentation path not found: #{docs_path}"
+        exit 1
+      end
+
+      unless File.directory?(docs_path)
+        puts "Path must be a directory for bulk transformation: #{docs_path}"
+        exit 1
+      end
+
+      puts "Bulk transforming markdown files in #{docs_path}..." if merged_options[:verbose]
+      puts "Using suffix: #{merged_options[:suffix]}" if merged_options[:verbose]
+      if merged_options[:verbose] && !merged_options[:excludes].empty?
+        puts "Excludes: #{merged_options[:excludes].join(', ')}"
+      end
+
+      begin
+        transformed_files = LlmsTxt.bulk_transform(docs_path, merged_options)
+
+        if transformed_files.empty?
+          puts 'No markdown files found to transform'
+        else
+          puts "Successfully transformed #{transformed_files.size} files:"
+          # verbose mode already shows progress
+          unless merged_options[:verbose]
+            transformed_files.each do |file|
+              puts "  #{file}"
+            end
+          end
+        end
+      rescue LlmsTxt::Errors::BaseError => e
+        puts "Error during bulk transformation: #{e.message}"
+        exit 1
+      end
+    end
+
+    # Parse existing llms.txt file and display information
+    #
+    # Reads and parses llms.txt file, displaying title, description, and links.
+    # Defaults to 'llms.txt' in current directory if no file specified.
+    #
+    # @param options [Hash] command options from parse_options
+    # @option options [String] :config path to config file
+    # @option options [String] :docs path to llms.txt file (defaults to 'llms.txt')
+    # @option options [Boolean] :verbose enable verbose output with link counts
+    # @raise [SystemExit] exits with status 1 if file not found
+    def parse(options)
+      # Load config and merge with CLI options
+      config = LlmsTxt::Config.new(options[:config])
+      merged_options = config.merge_with_options(options)
+
+      file_path = merged_options[:docs] || 'llms.txt'
+
+      unless File.exist?(file_path)
+        puts "File not found: #{file_path}"
+        exit 1
+      end
+
+      parsed = LlmsTxt.parse(file_path)
+
+      if options[:verbose]
+        puts "Title: #{parsed.title}"
+        puts "Description: #{parsed.description}"
+        puts "Documentation Links: #{parsed.documentation_links.size}"
+        puts "Example Links: #{parsed.example_links.size}" if parsed.respond_to?(:example_links)
+        puts "Optional Links: #{parsed.optional_links.size}" if parsed.respond_to?(:optional_links)
+      elsif parsed.respond_to?(:to_xml)
+        puts parsed.to_xml
+      end
+    end
+
+    # Validate llms.txt file format
+    #
+    # Checks if llms.txt file follows proper format with title, description, and documentation links.
+    # Defaults to 'llms.txt' in current directory if no file specified.
+    #
+    # @param options [Hash] command options from parse_options
+    # @option options [String] :config path to config file
+    # @option options [String] :docs path to llms.txt file (defaults to 'llms.txt')
+    # @raise [SystemExit] exits with status 1 if file not found or invalid
+    def validate(options)
+      # Load config and merge with CLI options
+      config = LlmsTxt::Config.new(options[:config])
+      merged_options = config.merge_with_options(options)
+
+      file_path = merged_options[:docs] || 'llms.txt'
+
+      unless File.exist?(file_path)
+        puts "File not found: #{file_path}"
+        exit 1
+      end
+
+      content = File.read(file_path)
+      valid = LlmsTxt.validate(content)
+
+      if valid
+        puts 'Valid llms.txt file'
+      else
+        puts 'Invalid llms.txt file'
+        puts "\nErrors:"
+        LlmsTxt::Validator.new(content).errors.each do |error|
+          puts "  - #{error}"
+        end
+        exit 1
+      end
+    end
+
+    # Display version information
+    #
+    def show_version
+      puts "llms-txt version #{LlmsTxt::VERSION}"
+    end
+  end
+end

data/lib/llms_txt/config.rb

@@ -62,8 +62,11 @@ module LlmsTxt
         title: options[:title] || self['title'],
         description: options[:description] || self['description'],
         output: options[:output] || self['output'] || 'llms.txt',
-        convert_urls: options.key?(:convert_urls) ?
-          options[:convert_urls] : (self['convert_urls'] || false),
+        convert_urls: if options.key?(:convert_urls)
+                        options[:convert_urls]
+                      else
+                        self['convert_urls'] || false
+                      end,
         verbose: options.key?(:verbose) ? options[:verbose] : (self['verbose'] || false),
         # Bulk transformation options
         suffix: options[:suffix] || self['suffix'] || '.llm',

data/lib/llms_txt/generator.rb

@@ -43,7 +43,7 @@ module LlmsTxt
 
       content = build_llms_txt(docs)
 
-      if output_path = options[:output]
+      if (output_path = options[:output])
         File.write(output_path, content)
       end
 
@@ -95,10 +95,10 @@ module LlmsTxt
     def analyze_file(file_path)
       # Handle single file case differently
       relative_path = if File.file?(docs_path)
-                       File.basename(file_path)
-                     else
-                       Pathname.new(file_path).relative_path_from(Pathname.new(docs_path)).to_s
-                     end
+                        File.basename(file_path)
+                      else
+                        Pathname.new(file_path).relative_path_from(Pathname.new(docs_path)).to_s
+                      end
 
       content = File.read(file_path)
 
@@ -120,7 +120,7 @@ module LlmsTxt
     def extract_title(content, file_path)
       # Try to extract title from first # header
       if content.match(/^#\s+(.+)/)
-        $1.strip
+        ::Regexp.last_match(1).strip
       else
         # Use filename as fallback
         File.basename(file_path, '.md').gsub(/[_-]/, ' ').split.map(&:capitalize).join(' ')
@@ -176,25 +176,25 @@ module LlmsTxt
 
       content = []
       content << "# #{title}"
-      content << ""
+      content << ''
       content << "> #{description}" if description
-      content << ""
+      content << ''
 
       if docs.any?
-        content << "## Documentation"
-        content << ""
+        content << '## Documentation'
+        content << ''
 
         docs.each do |doc|
           url = build_url(doc[:path])
-          if doc[:description] && !doc[:description].empty?
-            content << "- [#{doc[:title]}](#{url}): #{doc[:description]}"
-          else
-            content << "- [#{doc[:title]}](#{url})"
-          end
+          content << if doc[:description] && !doc[:description].empty?
+                       "- [#{doc[:title]}](#{url}): #{doc[:description]}"
+                     else
+                       "- [#{doc[:title]}](#{url})"
+                     end
         end
       end
 
-      content.join("\n") + "\n"
+      "#{content.join("\n")}\n"
     end
 
     # Attempts to detect project title from README or directory name
@@ -224,7 +224,7 @@ module LlmsTxt
     # @param path [String] relative path to file
     # @return [String] full URL or relative path
     def build_url(path)
-      if base_url = options[:base_url]
+      if (base_url = options[:base_url])
         File.join(base_url, path)
       else
         path

data/lib/llms_txt/markdown_transformer.rb

@@ -61,8 +61,8 @@ module LlmsTxt
       base_url = options[:base_url]
 
       content.gsub(/\[([^\]]+)\]\(([^)]+)\)/) do |match|
-        text = $1
-        url = $2
+        text = ::Regexp.last_match(1)
+        url = ::Regexp.last_match(2)
 
         if url.start_with?('http://', 'https://', '//', '#')
           match # Already absolute or anchor

data/lib/llms_txt/validator.rb

@@ -80,9 +80,7 @@ module LlmsTxt
     def validate_required_sections
       lines = content.lines
 
-      unless lines.first&.start_with?('# ')
-        errors << 'Missing required H1 title (must start with "# ")'
-      end
+      errors << 'Missing required H1 title (must start with "# ")' unless lines.first&.start_with?('# ')
 
       return unless lines.first&.strip&.length.to_i > 80
 
@@ -153,9 +151,7 @@ module LlmsTxt
             lib/
           ).*$
         }x
-        unless url =~ url_pattern
-          errors << "Invalid URL format: #{url}"
-        end
+        errors << "Invalid URL format: #{url}" unless url =~ url_pattern
       end
     end
 
@@ -192,9 +188,9 @@ module LlmsTxt
     #
     # Warns about non-HTTPS URLs and URLs containing spaces
     def validate_links
-      links = content.scan(/\[([^\]]+)\]\(([^)]+)\)/)
+      urls = content.scan(/\[([^\]]+)\]\(([^)]+)\)/).map(&:last)
 
-      links.each do |_text, url|
+      urls.each do |url|
         if url.start_with?('http') && !url.start_with?('https')
           errors << "Non-HTTPS URL found: #{url} (consider using HTTPS)"
         end
@@ -207,9 +203,7 @@ module LlmsTxt
     #
     # Enforces 50KB file size limit and 120 character line length limit
     def validate_file_size
-      if content.bytesize > MAX_FILE_SIZE
-        errors << "File size exceeds maximum (#{MAX_FILE_SIZE} bytes)"
-      end
+      errors << "File size exceeds maximum (#{MAX_FILE_SIZE} bytes)" if content.bytesize > MAX_FILE_SIZE
 
       lines = content.lines
       lines.each_with_index do |line, index|

data/lib/llms_txt/version.rb

@@ -2,5 +2,5 @@
 
 module LlmsTxt
   # Current version of the LlmsTxt gem
-  VERSION = '0.1.1'
+  VERSION = '0.2.0'
 end

data/lib/llms_txt.rb

@@ -5,10 +5,10 @@ require 'pathname'
 require 'find'
 
 loader = Zeitwerk::Loader.for_gem
+loader.inflector.inflect('cli' => 'CLI')
 loader.setup
 
 module LlmsTxt
-
   class << self
     # Generates llms.txt from existing markdown documentation
     #

data/renovate.json

@@ -10,10 +10,13 @@
     "enabled": true,
     "pinDigests": true
   },
-  "ruby": {
-    "enabled": true
-  },
   "packageRules": [
+    {
+      "matchCategories": [
+        "ruby"
+      ],
+      "enabled": true
+    },
     {
       "matchManagers": [
         "github-actions"
@@ -27,4 +30,4 @@
       "minimumReleaseAge": "7 days"
     }
   ]
-}
+}

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: llms-txt-ruby
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -104,6 +104,7 @@ email:
 - maciej@mensfeld.pl
 executables:
 - llms-txt
+- rspecs
 extensions: []
 extra_rdoc_files: []
 files:
@@ -119,8 +120,10 @@ files:
 - README.md
 - Rakefile
 - bin/llms-txt
+- bin/rspecs
 - lib/llms_txt.rb
 - lib/llms_txt/bulk_transformer.rb
+- lib/llms_txt/cli.rb
 - lib/llms_txt/config.rb
 - lib/llms_txt/errors.rb
 - lib/llms_txt/generator.rb