llms-txt-ruby 0.0.0 → 0.1.1

data/lib/llms_txt/validator.rb

@@ -0,0 +1,222 @@
+# frozen_string_literal: true
+
+module LlmsTxt
+  # Validates llms.txt content against the llms.txt specification
+  #
+  # Ensures that llms.txt content follows proper formatting rules including:
+  # - Required H1 title header
+  # - Optional description blockquote
+  # - Proper section ordering (Documentation, Examples, Optional)
+  # - Valid markdown syntax and link formats
+  # - File size and line length limits
+  #
+  # @example Validate llms.txt content
+  #   validator = LlmsTxt::Validator.new(content)
+  #   validator.valid? # => true or false
+  #   validator.errors # => Array of error messages
+  #
+  # @api public
+  class Validator
+    # @return [String] the llms.txt content being validated
+    attr_reader :content
+
+    # @return [Array<String>] array of validation error messages
+    attr_reader :errors
+
+    # Required sections that must appear in llms.txt
+    REQUIRED_SECTIONS = ['# '].freeze
+
+    # Optional sections that may appear in llms.txt
+    OPTIONAL_SECTIONS = ['> ', '## Documentation', '## Examples', '## Optional'].freeze
+
+    # Maximum length for a single line in characters
+    MAX_LINE_LENGTH = 120
+
+    # Maximum file size in bytes
+    MAX_FILE_SIZE = 50_000
+
+    # Initialize a new validator
+    #
+    # @param content [String] the llms.txt content to validate
+    def initialize(content)
+      @content = content
+      @errors = []
+    end
+
+    # Check if content is valid
+    #
+    # Runs all validation checks and returns whether the content is valid.
+    # Use {#errors} to access validation error messages.
+    #
+    # @return [Boolean] true if content is valid, false otherwise
+    def valid?
+      validate!
+      errors.empty?
+    end
+
+    # Validate content and return result
+    #
+    # Runs all validation checks, populates {#errors} array, and returns whether
+    # the content is valid.
+    #
+    # @return [Boolean] true if content is valid, false otherwise
+    def validate!
+      @errors = []
+
+      validate_required_sections
+      validate_structure
+      validate_markdown_syntax
+      validate_links
+      validate_file_size
+
+      errors.empty?
+    end
+
+    private
+
+    # Checks for required H1 title header and validates title length
+    #
+    # Adds errors if title is missing or exceeds 80 characters
+    def validate_required_sections
+      lines = content.lines
+
+      unless lines.first&.start_with?('# ')
+        errors << 'Missing required H1 title (must start with "# ")'
+      end
+
+      return unless lines.first&.strip&.length.to_i > 80
+
+      errors << 'Title is too long (max 80 characters)'
+    end
+
+    # Validates H1 uniqueness, description length, and section ordering
+    #
+    # Ensures only one H1, description under 200 chars, and proper section order
+    def validate_structure
+      lines = content.lines
+      h1_count = lines.count { |line| line.start_with?('# ') }
+
+      errors << 'Multiple H1 headers found (only one allowed)' if h1_count > 1
+
+      if lines[1]&.start_with?('> ') && lines[1].strip.length > 200
+        errors << 'Description blockquote is too long (max 200 characters)'
+      end
+
+      validate_section_order
+    end
+
+    # Verifies sections appear in correct order: Documentation, Examples, Optional
+    #
+    # Detects out-of-order sections and adds validation errors
+    def validate_section_order
+      sections = content.scan(/^## (.+)$/).flatten
+      expected_order = %w[Documentation Examples Optional]
+
+      current_index = -1
+      sections.each do |section|
+        index = expected_order.index(section)
+        next unless index
+
+        errors << "Section '#{section}' is out of order" if index < current_index
+        current_index = index
+      end
+    end
+
+    # Validates markdown syntax including links, lists, and headers
+    #
+    # Delegates to specialized validators for different markdown elements
+    def validate_markdown_syntax
+      validate_link_format
+      validate_list_format
+      validate_headers
+    end
+
+    # Checks markdown links for empty text/URLs and valid URL formats
+    #
+    # Validates URLs follow expected patterns for relative/absolute paths
+    def validate_link_format
+      content.scan(/\[([^\]]*)\]\(([^)]*)\)/) do |text, url|
+        errors << 'Empty link text found' if text.empty?
+
+        errors << 'Empty link URL found' if url.empty?
+
+        # Allow relative paths, absolute paths, HTTP(S) URLs, and common file extensions
+        url_pattern = %r{
+          ^(?:
+            https?://|
+            /|
+            \.\.?/|
+            [a-zA-Z0-9_.-]+(?:/|\.md|\.txt|\.rb|\.html)?|
+            [A-Z]+[a-zA-Z]*|
+            docs/|
+            examples/|
+            lib/
+          ).*$
+        }x
+        unless url =~ url_pattern
+          errors << "Invalid URL format: #{url}"
+        end
+      end
+    end
+
+    # Validates list items match expected markdown link format
+    #
+    # Ensures list items with links have proper syntax with optional descriptions
+    def validate_list_format
+      content.lines.each_with_index do |line, index|
+        next unless line =~ /^[-*]\s+\[/
+
+        # Allow both with and without descriptions
+        next if line =~ /^[-*]\s+\[.+\]\(.+\)(?::\s*.+)?$/
+
+        errors << "Invalid list item format at line #{index + 1}"
+      end
+    end
+
+    # Validates header levels and content
+    #
+    # Checks for empty H1 headers and warns about headers deeper than H2
+    def validate_headers
+      content.scan(/^(#+)\s+(.+)$/) do |hashes, text|
+        level = hashes.length
+
+        if level == 1 && text.strip.empty?
+          errors << 'Empty H1 header text'
+        elsif level > 2
+          errors << "Headers deeper than H2 not recommended (found H#{level})"
+        end
+      end
+    end
+
+    # Validates link security and format requirements
+    #
+    # Warns about non-HTTPS URLs and URLs containing spaces
+    def validate_links
+      links = content.scan(/\[([^\]]+)\]\(([^)]+)\)/)
+
+      links.each do |_text, url|
+        if url.start_with?('http') && !url.start_with?('https')
+          errors << "Non-HTTPS URL found: #{url} (consider using HTTPS)"
+        end
+
+        errors << "URL contains spaces: #{url}" if url.include?(' ')
+      end
+    end
+
+    # Checks file size and individual line lengths against limits
+    #
+    # Enforces 50KB file size limit and 120 character line length limit
+    def validate_file_size
+      if content.bytesize > MAX_FILE_SIZE
+        errors << "File size exceeds maximum (#{MAX_FILE_SIZE} bytes)"
+      end
+
+      lines = content.lines
+      lines.each_with_index do |line, index|
+        if line.chomp.length > MAX_LINE_LENGTH
+          errors << "Line #{index + 1} exceeds maximum length (#{MAX_LINE_LENGTH} characters)"
+        end
+      end
+    end
+  end
+end

data/lib/llms_txt/version.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module LlmsTxt
+  # Current version of the LlmsTxt gem
+  VERSION = '0.1.1'
+end

data/lib/llms_txt.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+require 'zeitwerk'
+require 'pathname'
+require 'find'
+
+loader = Zeitwerk::Loader.for_gem
+loader.setup
+
+module LlmsTxt
+
+  class << self
+    # Generates llms.txt from existing markdown documentation
+    #
+    # @param docs_path [String, nil] path to documentation directory or file (optional if
+    #   config_file provided)
+    # @param options [Hash] generation options
+    # @option options [String] :config_file path to YAML config file (auto-finds llms-txt.yml)
+    # @option options [String] :base_url base URL for converting relative links (overrides config)
+    # @option options [String] :title project title (auto-detected if not provided, overrides
+    #   config)
+    # @option options [String] :description project description (auto-detected if not provided,
+    #   overrides config)
+    # @option options [String] :output output file path (default: 'llms.txt', overrides config)
+    # @option options [Boolean] :convert_urls convert HTML URLs to markdown format (overrides
+    #   config)
+    # @option options [Boolean] :verbose enable verbose output (overrides config)
+    # @return [String] generated llms.txt content
+    #
+    # @example Generate from docs directory
+    #   LlmsTxt.generate_from_docs('./docs')
+    #
+    # @example Generate using config file
+    #   LlmsTxt.generate_from_docs(config_file: 'llms-txt.yml')
+    #
+    # @example Generate with config file and overrides
+    #   LlmsTxt.generate_from_docs('./docs',
+    #     config_file: 'my-config.yml',
+    #     title: 'Override Title'
+    #   )
+    def generate_from_docs(docs_path = nil, options = {})
+      # Support config-first usage: generate_from_docs(config_file: 'path.yml')
+      if docs_path.is_a?(Hash) && docs_path.key?(:config_file)
+        options = docs_path
+        docs_path = nil
+      end
+
+      config = Config.new(options[:config_file])
+      merged_options = config.merge_with_options(options)
+
+      # Use docs_path param or config file docs setting
+      final_docs_path = docs_path || merged_options[:docs]
+
+      Generator.new(final_docs_path, merged_options).generate
+    end
+
+    # Transforms a markdown file to be AI-friendly
+    #
+    # @param file_path [String] path to markdown file
+    # @param options [Hash] transformation options
+    # @option options [String] :config_file path to YAML config file (auto-finds llms-txt.yml)
+    # @option options [String] :base_url base URL for expanding relative links (overrides config)
+    # @option options [Boolean] :convert_urls convert HTML URLs to markdown format (overrides
+    #   config)
+    # @option options [Boolean] :verbose enable verbose output (overrides config)
+    # @return [String] transformed markdown content
+    #
+    # @example Transform with direct options
+    #   LlmsTxt.transform_markdown('README.md',
+    #     base_url: 'https://myproject.io',
+    #     convert_urls: true
+    #   )
+    #
+    # @example Transform using config file
+    #   LlmsTxt.transform_markdown('README.md', config_file: 'llms-txt.yml')
+    def transform_markdown(file_path, options = {})
+      config = Config.new(options[:config_file])
+      merged_options = config.merge_with_options(options)
+
+      MarkdownTransformer.new(file_path, merged_options).transform
+    end
+
+    # Bulk transforms multiple markdown files to be AI-friendly
+    #
+    # @param docs_path [String] path to documentation directory
+    # @param options [Hash] transformation options
+    # @option options [String] :config_file path to YAML config file (auto-finds llms-txt.yml)
+    # @option options [String] :base_url base URL for expanding relative links (overrides config)
+    # @option options [Boolean] :convert_urls convert HTML URLs to markdown format (overrides
+    #   config)
+    # @option options [String] :suffix suffix for transformed files (default: '.llm', overrides
+    #   config)
+    # @option options [Array<String>] :excludes glob patterns for files to exclude (overrides
+    #   config)
+    # @option options [Boolean] :verbose enable verbose output (overrides config)
+    # @return [Array<String>] paths of transformed files
+    #
+    # @example Bulk transform with direct options
+    #   LlmsTxt.bulk_transform('./docs',
+    #     base_url: 'https://myproject.io',
+    #     suffix: '.ai',
+    #     excludes: ['**/private/**', 'draft-*.md']
+    #   )
+    #
+    # @example Bulk transform using config file
+    #   LlmsTxt.bulk_transform('./docs', config_file: 'llms-txt.yml')
+    def bulk_transform(docs_path, options = {})
+      config = Config.new(options[:config_file])
+      merged_options = config.merge_with_options(options)
+
+      BulkTransformer.new(docs_path, merged_options).transform_all
+    end
+
+    # Parses an existing llms.txt file
+    #
+    # @param file_path [String] path to the llms.txt file to parse
+    # @return [Parser] parsed llms.txt object
+    def parse(file_path)
+      Parser.new(file_path).parse
+    end
+
+    # Validates llms.txt content
+    #
+    # @param content [String] the llms.txt content to validate
+    # @return [Boolean] true if content is valid, false otherwise
+    def validate(content)
+      Validator.new(content).valid?
+    end
+  end
+end

data/llms-txt-ruby.gemspec

@@ -1,34 +1,45 @@
 # frozen_string_literal: true
 
-require_relative 'lib/llms-txt/version'
+require_relative 'lib/llms_txt/version'
 
 Gem::Specification.new do |spec|
   spec.name = 'llms-txt-ruby'
-  spec.version = LLMsTxt::VERSION
+  spec.version = LlmsTxt::VERSION
   spec.authors = ['Maciej Mensfeld']
   spec.email = %w[maciej@mensfeld.pl]
 
-  spec.summary = 'Ruby implementation of the llms.txt specification for LLM-friendly content'
+  spec.summary = 'Generate llms.txt files and transform markdown documentation to be AI-friendly'
   spec.description = <<~DESC
-    A Ruby gem that implements the llms.txt specification, providing tools to create and manage
-    llms.txt markdown files for websites. These files help Large Language Models understand
-    and navigate website content more effectively by providing curated, LLM-friendly information
-    in a standardized format.
+    A Ruby tool for transforming existing markdown documentation into AI-friendly formats
+    following the llms.txt standard. Features include: generating llms.txt files from
+    documentation directories with automatic file prioritization, transforming individual
+    markdown files by expanding relative links to absolute URLs, and bulk transforming entire
+    documentation trees with customizable exclusion patterns. Provides both CLI and Ruby API
+    with configuration file support.
   DESC
 
   spec.homepage = 'https://github.com/mensfeld/llms-txt-ruby'
   spec.license = 'MIT'
-  spec.required_ruby_version = '>= 3.1.0'
+  spec.required_ruby_version = '>= 3.2'
 
   spec.metadata['allowed_push_host'] = 'https://rubygems.org'
   spec.metadata['homepage_uri'] = spec.homepage
   spec.metadata['source_code_uri'] = 'https://github.com/mensfeld/llms-txt-ruby'
-  spec.metadata['changelog_uri'] = 'https://github.com/mensfeld/llms-txt-ruby/blob/main/CHANGELOG.md'
+  spec.metadata['changelog_uri'] = 'https://github.com/mensfeld/llms-txt-ruby/blob/master/CHANGELOG.md'
   spec.metadata['documentation_uri'] = 'https://github.com/mensfeld/llms-txt-ruby'
+  spec.metadata['rubygems_mfa_required'] = 'true'
 
-  spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec)/}) }
+  spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec|test)/}) }
 
   spec.bindir = 'bin'
-  spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.executables = spec.files.grep(%r{\Abin/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
+
+  spec.add_dependency 'zeitwerk', '~> 2.6'
+
+  spec.add_development_dependency 'bundler', '~> 2.0'
+  spec.add_development_dependency 'rake', '~> 13.0'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'rubocop', '~> 1.0'
+  spec.add_development_dependency 'simplecov', '~> 0.21'
 end

data/llms-txt.yml.example

@@ -0,0 +1,26 @@
+# Example llms-txt.yml configuration file
+# Copy to llms-txt.yml and customize for your project
+
+# Path to documentation directory or file
+docs: ./docs
+
+# Base URL for expanding relative links (optional)
+base_url: https://myproject.io
+
+# Project information (optional - will auto-detect from docs if not provided)
+title: My Awesome Project
+description: A Ruby library that helps developers build amazing applications
+
+# Output file path (optional)
+output: llms.txt
+
+# Transform options (optional)
+convert_urls: true  # Convert .html links to .md
+verbose: false      # Verbose output
+
+# Bulk transformation options (optional)
+suffix: .llm        # Suffix for transformed files (creates file.llm.md)
+excludes:           # Patterns for files to exclude from bulk transformation
+  - "**/private/**"    # Exclude private directories
+  - "**/draft-*.md"    # Exclude draft files
+  - "**/.*"            # Exclude hidden files

data/renovate.json

@@ -0,0 +1,30 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": [
+    "config:recommended"
+  ],
+  "ignorePaths": [
+    "spec/integrations"
+  ],
+  "github-actions": {
+    "enabled": true,
+    "pinDigests": true
+  },
+  "ruby": {
+    "enabled": true
+  },
+  "packageRules": [
+    {
+      "matchManagers": [
+        "github-actions"
+      ],
+      "minimumReleaseAge": "7 days"
+    },
+    {
+      "matchManagers": [
+        "bundler"
+      ],
+      "minimumReleaseAge": "7 days"
+    }
+  ]
+}

metadata

@@ -1,30 +1,136 @@
 --- !ruby/object:Gem::Specification
 name: llms-txt-ruby
 version: !ruby/object:Gem::Version
-  version: 0.0.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Maciej Mensfeld
 bindir: bin
 cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
-dependencies: []
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: zeitwerk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.6'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.21'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.21'
 description: |
-  A Ruby gem that implements the llms.txt specification, providing tools to create and manage
-  llms.txt markdown files for websites. These files help Large Language Models understand
-  and navigate website content more effectively by providing curated, LLM-friendly information
-  in a standardized format.
+  A Ruby tool for transforming existing markdown documentation into AI-friendly formats
+  following the llms.txt standard. Features include: generating llms.txt files from
+  documentation directories with automatic file prioritization, transforming individual
+  markdown files by expanding relative links to absolute URLs, and bulk transforming entire
+  documentation trees with customizable exclusion patterns. Provides both CLI and Ruby API
+  with configuration file support.
 email:
 - maciej@mensfeld.pl
-executables: []
+executables:
+- llms-txt
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/ci.yml"
+- ".github/workflows/push.yml"
 - ".gitignore"
+- ".rubocop.yml"
+- ".ruby-version"
+- CHANGELOG.md
+- Gemfile
+- Gemfile.lock
 - LICENSE
 - README.md
-- lib/llms-txt/version.rb
+- Rakefile
+- bin/llms-txt
+- lib/llms_txt.rb
+- lib/llms_txt/bulk_transformer.rb
+- lib/llms_txt/config.rb
+- lib/llms_txt/errors.rb
+- lib/llms_txt/generator.rb
+- lib/llms_txt/markdown_transformer.rb
+- lib/llms_txt/parser.rb
+- lib/llms_txt/validator.rb
+- lib/llms_txt/version.rb
 - llms-txt-ruby.gemspec
+- llms-txt.yml.example
+- renovate.json
 homepage: https://github.com/mensfeld/llms-txt-ruby
 licenses:
 - MIT
@@ -32,8 +138,9 @@ metadata:
   allowed_push_host: https://rubygems.org
   homepage_uri: https://github.com/mensfeld/llms-txt-ruby
   source_code_uri: https://github.com/mensfeld/llms-txt-ruby
-  changelog_uri: https://github.com/mensfeld/llms-txt-ruby/blob/main/CHANGELOG.md
+  changelog_uri: https://github.com/mensfeld/llms-txt-ruby/blob/master/CHANGELOG.md
   documentation_uri: https://github.com/mensfeld/llms-txt-ruby
+  rubygems_mfa_required: 'true'
 rdoc_options: []
 require_paths:
 - lib
@@ -41,14 +148,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 3.1.0
+      version: '3.2'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.7
+rubygems_version: 3.6.9
 specification_version: 4
-summary: Ruby implementation of the llms.txt specification for LLM-friendly content
+summary: Generate llms.txt files and transform markdown documentation to be AI-friendly
 test_files: []

data/lib/llms-txt/version.rb

@@ -1,3 +0,0 @@
-module LLMsTxt
-  VERSION = '0.0.0'
-end