llms-txt-ruby 0.0.0 → 0.1.0

data/bin/llms-txt

@@ -0,0 +1,242 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'optparse'
+require 'llms_txt'
+
+module LlmsTxt
+  class CLI
+    def self.run(argv = ARGV)
+      new.run(argv)
+    end
+
+    def run(argv)
+      options = parse_options(argv)
+
+      case options[:command]
+      when 'generate', nil
+        generate(options)
+      when 'transform'
+        transform(options)
+      when 'bulk-transform'
+        bulk_transform(options)
+      when 'parse'
+        parse(options)
+      when 'validate'
+        validate(options)
+      when 'version'
+        show_version
+      else
+        puts "Unknown command: #{options[:command]}"
+        puts "Run 'llms-txt --help' for usage information"
+        exit 1
+      end
+    rescue LlmsTxt::Error => e
+      puts "Error: #{e.message}"
+      exit 1
+    rescue StandardError => e
+      puts "Unexpected error: #{e.message}"
+      puts e.backtrace.join("\n") if options&.fetch(:verbose, false)
+      exit 1
+    end
+
+    private
+
+    def parse_options(argv)
+      options = {
+        command: argv.first&.match?(/^[a-z-]+$/) ? argv.shift : nil
+      }
+
+      OptionParser.new do |opts|
+        opts.banner = "llms-txt - Simple tool for generating llms.txt from markdown documentation\n\nUsage: llms-txt [command] [options]\n\nFor advanced configuration (base_url, title, description, convert_urls), use a config file."
+
+        opts.separator ''
+        opts.separator 'Commands:'
+        opts.separator '  generate       Generate llms.txt from documentation (default)'
+        opts.separator '  transform      Transform a markdown file to be AI-friendly'
+        opts.separator '  bulk-transform Transform all markdown files in directory'
+        opts.separator '  parse          Parse existing llms.txt file'
+        opts.separator '  validate       Validate llms.txt file'
+        opts.separator '  version        Show version'
+
+        opts.separator ''
+        opts.separator 'Options:'
+
+        opts.on('-c', '--config PATH', 'Configuration file path (default: llms-txt.yml)') do |path|
+          options[:config] = path
+        end
+
+        opts.on('-d', '--docs PATH', 'Path to documentation directory or file') do |path|
+          options[:docs] = path
+        end
+
+        opts.on('-o', '--output PATH', 'Output file path') do |path|
+          options[:output] = path
+        end
+
+        opts.on('-v', '--verbose', 'Verbose output') do
+          options[:verbose] = true
+        end
+
+        opts.on('-h', '--help', 'Show this message') do
+          puts opts
+          exit
+        end
+
+        opts.on('--version', 'Show version') do
+          show_version
+          exit
+        end
+      end.parse!(argv)
+
+      options[:file_path] = argv.first if argv.any?
+      options
+    end
+
+    def generate(options)
+      # Load config and merge with CLI options
+      config = LlmsTxt::Config.new(options[:config])
+      merged_options = config.merge_with_options(options)
+
+      docs_path = merged_options[:docs]
+
+      unless File.exist?(docs_path)
+        puts "Documentation path not found: #{docs_path}"
+        exit 1
+      end
+
+      puts "Generating llms.txt from #{docs_path}..." if merged_options[:verbose]
+
+      content = LlmsTxt.generate_from_docs(docs_path, merged_options)
+      output_path = merged_options[:output]
+
+      File.write(output_path, content)
+      puts "Successfully generated #{output_path}"
+
+      if merged_options[:verbose]
+        validator = LlmsTxt::Validator.new(content)
+        if validator.valid?
+          puts "Valid llms.txt format"
+        else
+          puts "Validation warnings:"
+          validator.errors.each { |error| puts "  - #{error}" }
+        end
+      end
+    end
+
+    def transform(options)
+      # Load config and merge with CLI options
+      config = LlmsTxt::Config.new(options[:config])
+      merged_options = config.merge_with_options(options)
+
+      file_path = options[:file_path]
+
+      unless file_path
+        puts "File path required for transform command"
+        exit 1
+      end
+
+      unless File.exist?(file_path)
+        puts "File not found: #{file_path}"
+        exit 1
+      end
+
+      puts "Transforming #{file_path}..." if merged_options[:verbose]
+
+      content = LlmsTxt.transform_markdown(file_path, merged_options)
+
+      if merged_options[:output] && merged_options[:output] != 'llms.txt'
+        File.write(merged_options[:output], content)
+        puts "Transformed content saved to #{merged_options[:output]}"
+      else
+        puts content
+      end
+    end
+
+    def bulk_transform(options)
+      # Load config and merge with CLI options
+      config = LlmsTxt::Config.new(options[:config])
+      merged_options = config.merge_with_options(options)
+
+      docs_path = merged_options[:docs]
+
+      unless File.exist?(docs_path)
+        puts "Documentation path not found: #{docs_path}"
+        exit 1
+      end
+
+      unless File.directory?(docs_path)
+        puts "Path must be a directory for bulk transformation: #{docs_path}"
+        exit 1
+      end
+
+      puts "Bulk transforming markdown files in #{docs_path}..." if merged_options[:verbose]
+      puts "Using suffix: #{merged_options[:suffix]}" if merged_options[:verbose]
+      puts "Excludes: #{merged_options[:excludes].join(', ')}" if merged_options[:verbose] && !merged_options[:excludes].empty?
+
+      begin
+        transformed_files = LlmsTxt.bulk_transform(docs_path, merged_options)
+
+        if transformed_files.empty?
+          puts "No markdown files found to transform"
+        else
+          puts "Successfully transformed #{transformed_files.size} files:"
+          transformed_files.each { |file| puts "  #{file}" } unless merged_options[:verbose] # verbose mode already shows progress
+        end
+      rescue LlmsTxt::Error => e
+        puts "Error during bulk transformation: #{e.message}"
+        exit 1
+      end
+    end
+
+    def parse(options)
+      file_path = options[:file_path] || 'llms.txt'
+
+      unless File.exist?(file_path)
+        puts "File not found: #{file_path}"
+        exit 1
+      end
+
+      parsed = LlmsTxt.parse(file_path)
+
+      if options[:verbose]
+        puts "Title: #{parsed.title}"
+        puts "Description: #{parsed.description}"
+        puts "Documentation Links: #{parsed.documentation_links.size}"
+        puts "Example Links: #{parsed.example_links.size}" if parsed.respond_to?(:example_links)
+        puts "Optional Links: #{parsed.optional_links.size}" if parsed.respond_to?(:optional_links)
+      else
+        puts parsed.to_xml if parsed.respond_to?(:to_xml)
+      end
+    end
+
+    def validate(options)
+      file_path = options[:file_path] || 'llms.txt'
+
+      unless File.exist?(file_path)
+        puts "File not found: #{file_path}"
+        exit 1
+      end
+
+      content = File.read(file_path)
+      valid = LlmsTxt.validate(content)
+
+      if valid
+        puts 'Valid llms.txt file'
+      else
+        puts 'Invalid llms.txt file'
+        puts "\nErrors:"
+        LlmsTxt::Validator.new(content).errors.each do |error|
+          puts "  - #{error}"
+        end
+        exit 1
+      end
+    end
+
+    def show_version
+      puts "llms-txt version #{LlmsTxt::VERSION}"
+    end
+  end
+end
+
+LlmsTxt::CLI.run # if $PROGRAM_NAME == __FILE__

data/lib/llms_txt/bulk_transformer.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+
+module LlmsTxt
+  # Bulk transforms multiple markdown files to be AI-friendly
+  #
+  # Processes all markdown files in a directory recursively, creating LLM-friendly versions
+  # alongside the originals. Supports exclusion patterns and maintains directory structure.
+  #
+  # @example Transform all files in a directory
+  #   transformer = LlmsTxt::BulkTransformer.new('./docs',
+  #     base_url: 'https://myproject.io',
+  #     suffix: '.llm'
+  #   )
+  #   transformer.transform_all
+  #
+  # @api public
+  class BulkTransformer
+    # @return [String] path to documentation directory
+    attr_reader :docs_path
+
+    # @return [Hash] transformation options
+    attr_reader :options
+
+    # Initialize a new bulk transformer
+    #
+    # @param docs_path [String] path to documentation directory
+    # @param options [Hash] transformation options
+    # @option options [String] :base_url base URL for expanding relative links
+    # @option options [Boolean] :convert_urls convert HTML URLs to markdown format
+    # @option options [String] :suffix suffix for transformed files (default: '.llm')
+    # @option options [Array<String>] :excludes glob patterns for files to exclude
+    # @option options [Boolean] :verbose enable verbose output
+    def initialize(docs_path, options = {})
+      @docs_path = docs_path
+      @options = {
+        suffix: '.llm',
+        excludes: []
+      }.merge(options)
+    end
+
+    # Transform all markdown files in the directory
+    #
+    # Recursively finds all markdown files, applies transformations,
+    # and saves LLM-friendly versions with the specified suffix.
+    #
+    # @return [Array<String>] paths of transformed files
+    def transform_all
+      unless File.directory?(docs_path)
+        raise Errors::GenerationError, "Directory not found: #{docs_path}"
+      end
+
+      markdown_files = find_markdown_files
+      transformed_files = []
+
+      markdown_files.each do |file_path|
+        next if should_exclude?(file_path)
+
+        puts "Transforming #{file_path}..." if options[:verbose]
+
+        transformed_content = transform_file(file_path)
+        output_path = generate_output_path(file_path)
+
+        # Ensure output directory exists
+        FileUtils.mkdir_p(File.dirname(output_path))
+
+        File.write(output_path, transformed_content)
+        transformed_files << output_path
+
+        puts "  → #{output_path}" if options[:verbose]
+      end
+
+      transformed_files
+    end
+
+    private
+
+    # Recursively scans the docs directory for markdown files
+    #
+    # Skips hidden files (starting with dot) and returns sorted array of paths
+    #
+    # @return [Array<String>] paths to markdown files
+    def find_markdown_files
+      files = []
+
+      Find.find(docs_path) do |path|
+        next unless File.file?(path)
+        next unless path.match?(/\.md$/i)
+        next if File.basename(path).start_with?('.')
+
+        files << path
+      end
+
+      files.sort
+    end
+
+    # Tests if file matches any exclusion pattern from options
+    #
+    # Uses File.fnmatch with pathname and dotmatch flags
+    #
+    # @param file_path [String] path to check
+    # @return [Boolean] true if file should be excluded
+    def should_exclude?(file_path)
+      excludes = Array(options[:excludes])
+
+      excludes.any? do |pattern|
+        File.fnmatch(pattern, file_path, File::FNM_PATHNAME | File::FNM_DOTMATCH)
+      end
+    end
+
+    # Applies markdown transformations to a single file
+    #
+    # Creates MarkdownTransformer instance and delegates transformation
+    #
+    # @param file_path [String] path to markdown file
+    # @return [String] transformed content
+    def transform_file(file_path)
+      transformer = MarkdownTransformer.new(file_path, options)
+      transformer.transform
+    end
+
+    # Constructs output path by adding suffix before .md extension
+    #
+    # For example: README.md with suffix .llm becomes README.llm.md
+    #
+    # @param input_path [String] original file path
+    # @return [String] path for transformed file
+    def generate_output_path(input_path)
+      dir = File.dirname(input_path)
+      basename = File.basename(input_path, '.md')
+      suffix = options[:suffix]
+
+      File.join(dir, "#{basename}#{suffix}.md")
+    end
+  end
+end

data/lib/llms_txt/config.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+require 'yaml'
+
+module LlmsTxt
+  # Simple configuration loader for llms-txt.yml files
+  #
+  # Loads YAML configuration files and provides a simple interface for accessing configuration
+  # values. Automatically looks for config files in the current directory if none specified.
+  #
+  # @example Load default config file
+  #   config = LlmsTxt::Config.new
+  #
+  # @example Load specific config file
+  #   config = LlmsTxt::Config.new('my-config.yml')
+  #
+  # @example Access config values
+  #   config['base_url']        # => "https://myproject.io"
+  #   config.dig('output')      # => "llms.txt"
+  #
+  # @api public
+  class Config
+    # @return [Hash] the loaded configuration data
+    attr_reader :data
+
+    # Initialize a new configuration loader
+    #
+    # @param config_file [String, nil] path to YAML config file, or nil to auto-find
+    def initialize(config_file = nil)
+      @config_file = config_file || find_config_file
+      @data = load_config
+    end
+
+    # Access configuration value by key
+    #
+    # @param key [String, Symbol] configuration key
+    # @return [Object, nil] configuration value or nil if not found
+    def [](key)
+      data[key.to_s]
+    end
+
+    # Access nested configuration values
+    #
+    # @param keys [Array<String, Symbol>] nested keys to access
+    # @return [Object, nil] configuration value or nil if not found
+    def dig(*keys)
+      data.dig(*keys.map(&:to_s))
+    end
+
+    # Merge config file values with CLI options
+    #
+    # CLI options take precedence over config file values. Config file provides
+    # defaults for any options not specified via CLI.
+    #
+    # @param options [Hash] CLI options hash
+    # @return [Hash] merged configuration with CLI overrides applied
+    def merge_with_options(options)
+      # CLI options override config file, config file provides defaults
+      {
+        docs: options[:docs] || self['docs'] || '.',
+        base_url: options[:base_url] || self['base_url'],
+        title: options[:title] || self['title'],
+        description: options[:description] || self['description'],
+        output: options[:output] || self['output'] || 'llms.txt',
+        convert_urls: options.key?(:convert_urls) ?
+          options[:convert_urls] : (self['convert_urls'] || false),
+        verbose: options.key?(:verbose) ? options[:verbose] : (self['verbose'] || false),
+        # Bulk transformation options
+        suffix: options[:suffix] || self['suffix'] || '.llm',
+        excludes: options[:excludes] || self['excludes'] || [],
+        bulk: options.key?(:bulk) ? options[:bulk] : (self['bulk'] || false)
+      }
+    end
+
+    # Check if a config file was found and exists
+    #
+    # @return [Boolean] true if config file exists, false otherwise
+    def exists?
+      @config_file && File.exist?(@config_file)
+    end
+
+    private
+
+    # Find config file in current directory
+    #
+    # Looks for config files in order of preference:
+    # 1. llms-txt.yml
+    # 2. llms-txt.yaml
+    # 3. .llms-txt.yml
+    #
+    # @return [String, nil] path to config file or nil if none found
+    def find_config_file
+      candidates = ['llms-txt.yml', 'llms-txt.yaml', '.llms-txt.yml']
+      candidates.find { |file| File.exist?(file) }
+    end
+
+    # Load and parse YAML config file
+    #
+    # @return [Hash] parsed config data, empty hash if no file
+    # @raise [Errors::GenerationError] if YAML is invalid or file cannot be read
+    def load_config
+      return {} unless @config_file && File.exist?(@config_file)
+
+      begin
+        YAML.load_file(@config_file) || {}
+      rescue Psych::SyntaxError => e
+        raise Errors::GenerationError, "Invalid YAML in config file #{@config_file}: #{e.message}"
+      rescue StandardError => e
+        raise Errors::GenerationError, "Failed to load config file #{@config_file}: #{e.message}"
+      end
+    end
+  end
+end

data/lib/llms_txt/errors.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module LlmsTxt
+  # Namespace used to encapsulate all the internal errors of LlmsTxt
+  module Errors
+    # Base class for all the LlmsTxt internal errors
+    BaseError = Class.new(StandardError)
+
+    # Raised when llms.txt generation fails due to configuration issues,
+    # missing directories, invalid YAML, or file access problems
+    #
+    # @example When directory doesn't exist
+    #   LlmsTxt.bulk_transform('/nonexistent/path')
+    #   # => raises GenerationError: "Directory not found: /nonexistent/path"
+    #
+    # @example When config YAML is invalid
+    #   LlmsTxt.generate_from_docs(config_file: 'invalid.yml')
+    #   # => raises GenerationError: "Invalid YAML in config file..."
+    GenerationError = Class.new(BaseError)
+
+    # Raised when llms.txt content validation fails
+    #
+    # This error is intended for validation failures but currently not used.
+    # The Validator class returns boolean results instead of raising errors.
+    #
+    # @example Future usage (when validation raises)
+    #   LlmsTxt.validate!(invalid_content)
+    #   # => raises ValidationError: "Missing required H1 title"
+    ValidationError = Class.new(BaseError)
+  end
+end