llm-docs-builder 0.3.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 68bc5131179dcb94c393cb2973c9fa1ffbf868616effb6814a7e99d340923de2
+  data.tar.gz: c660150ac38f2687951ebd6f1e2f2014f4c5636aa4c322ef9114427b9d49a235
+SHA512:
+  metadata.gz: 45bf6443d05c9173bf308b80240595e509d596447343aca937158ebddc44d9d0d07c0b3bc0db61b743e49f9ff634f799a90e8f72c0105ed3221fc3f11bdd7e05
+  data.tar.gz: 4a2e9e04aa39dfa2954988a20e825c1d306299084e120f3332c413e008353be45336819bc0127aa03e05aaaa47db6a90b78f1c4033ea95b665262035b0998ace

data/.dockerignore

@@ -0,0 +1,44 @@
+# Git
+.git
+.github
+.gitignore
+
+# Development
+spec/
+coverage/
+.rspec
+.rubocop.yml
+
+# Documentation
+*.md
+!README.md
+CLAUDE.md
+
+# Build artifacts
+*.gem
+pkg/
+vendor/bundle
+
+# IDE
+.vscode/
+.idea/
+.claude/
+*.swp
+*.swo
+*~
+
+# Temp files
+tmp/
+*.log
+.DS_Store
+
+# Config examples
+*.example
+llms-txt.yml
+config-output.txt
+
+# CI
+.github/workflows/
+
+# Ruby
+# Keep Gemfile.lock for reproducible builds

data/.github/workflows/ci.yml

@@ -0,0 +1,71 @@
+name: CI
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+on:
+  pull_request:
+    branches: [ master ]
+  push:
+    branches: [ master ]
+  schedule:
+    - cron: '0 1 * * *'
+
+permissions:
+  contents: read
+
+jobs:
+  specs:
+    timeout-minutes: 15
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby:
+          - '3.4'
+          - '3.3'
+          - '3.2'
+        include:
+          - ruby: '3.4'
+            coverage: 'true'
+    steps:
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5
+        with:
+          fetch-depth: 0
+
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+          bundler: 'latest'
+
+      - name: Install latest bundler
+        run: |
+          gem install bundler --no-document
+          bundle config set without 'tools benchmarks docs'
+
+      - name: Bundle install
+        run: bundle install --jobs 4 --retry 3
+
+      - name: Run all tests
+        env:
+          GITHUB_COVERAGE: ${{ matrix.coverage }}
+        run: bin/rspecs
+
+
+  ci-success:
+    name: CI Success
+    runs-on: ubuntu-latest
+    if: always()
+    needs:
+      - specs
+    steps:
+      - name: Check all jobs passed
+        if: |
+          contains(needs.*.result, 'failure') ||
+          contains(needs.*.result, 'cancelled') ||
+          contains(needs.*.result, 'skipped')
+        run: exit 1
+      - run: echo "All CI checks passed!"

data/.github/workflows/docker.yml

@@ -0,0 +1,102 @@
+name: Docker
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+# Temporarily disabled - only runs on manual trigger
+on:
+  workflow_dispatch:
+
+# Automatic triggers disabled for now:
+#   push:
+#     branches:
+#       - master
+#     tags:
+#       - 'v*'
+#   pull_request:
+#     branches:
+#       - master
+#   schedule:
+#     # Rebuild weekly to get latest base image security updates
+#     - cron: '0 2 * * 0'
+
+permissions:
+  contents: read
+  packages: write
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5
+        with:
+          fetch-depth: 0
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            mensfeld/llm-docs-builder
+            ghcr.io/${{ github.repository }}
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=semver,pattern={{major}}
+            type=raw,value=latest,enable={{is_default_branch}}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Login to GitHub Container Registry
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Test Docker image
+        run: |
+          docker run --rm ${{ fromJSON(steps.meta.outputs.json).tags[0] }} version
+          docker run --rm ${{ fromJSON(steps.meta.outputs.json).tags[0] }} --help
+
+  docker-success:
+    name: Docker Success
+    runs-on: ubuntu-latest
+    if: always()
+    needs:
+      - docker
+    steps:
+      - name: Check all jobs passed
+        if: |
+          contains(needs.*.result, 'failure') ||
+          contains(needs.*.result, 'cancelled') ||
+          contains(needs.*.result, 'skipped')
+        run: exit 1
+      - run: echo "Docker workflow completed successfully!"

data/.github/workflows/push.yml

@@ -0,0 +1,35 @@
+name: Push Gem
+
+on:
+  push:
+    tags:
+      - v*
+
+permissions:
+  contents: read
+
+jobs:
+  push:
+    if: github.repository_owner == 'mensfeld'
+    runs-on: ubuntu-latest
+    environment: deployment
+
+    permissions:
+      contents: write
+      id-token: write
+
+    steps:
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+        with:
+          fetch-depth: 0
+
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@6797dcbb9a1889fd411d07e8aba7eded53fb8b48 # v1.264.0
+        with:
+          bundler-cache: false
+
+      - name: Bundle install
+        run: |
+          bundle install --jobs 4 --retry 3
+
+      - uses: rubygems/release-gem@a25424ba2ba8b387abc8ef40807c2c85b96cbe32 # v1.1.1

data/.gitignore

@@ -0,0 +1,66 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/spec/examples.txt
+/test/tmp/
+/test/version_tmp/
+/tmp/
+mise.toml
+
+# Used by dotenv library to load environment variables.
+.env
+.env.*
+
+# Ignore Byebug command history file.
+.byebug_history
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+*.bridgesupport
+build-iPhoneOS/
+build-iPhoneSimulator/
+
+## Specific to RubyMotion (use of CocoaPods):
+#
+# We recommend against adding the Pods directory to your .gitignore. However
+# you should judge for yourself, the pros and cons are mentioned at:
+# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
+#
+# vendor/Pods/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalization:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc
+
+# Used by RuboCop. Remote config files pulled in from inherit_from directive.
+# .rubocop-https?--*
+
+# Project-specific generated files
+llms.txt
+*-output.txt
+
+# Config files that might contain sensitive data
+llms-txt.yml
+.llms-txt.yml

data/.rubocop.yml

@@ -0,0 +1,74 @@
+AllCops:
+  TargetRubyVersion: 3.2
+  NewCops: enable
+  SuggestExtensions: false
+
+Style/Documentation:
+  Enabled: false
+
+Style/StringLiterals:
+  EnforcedStyle: single_quotes
+
+Layout/LineLength:
+  Max: 120
+  Exclude:
+    - 'lib/llms_txt/cli.rb'
+
+Metrics/ClassLength:
+  Max: 200
+  Exclude:
+    - 'lib/llms_txt/cli.rb'
+
+Metrics/MethodLength:
+  Max: 35
+  Exclude:
+    - 'lib/llms_txt/cli.rb'
+
+Metrics/AbcSize:
+  Max: 40
+  Exclude:
+    - 'lib/llms_txt/cli.rb'
+
+Metrics/CyclomaticComplexity:
+  Max: 15
+  Exclude:
+    - 'lib/llms_txt/config.rb'
+
+Metrics/PerceivedComplexity:
+  Max: 15
+  Exclude:
+    - 'lib/llms_txt/config.rb'
+
+Metrics/BlockLength:
+  Exclude:
+    - 'spec/**/*'
+    - 'lib/llms_txt/cli.rb'
+    - '*.gemspec'
+
+Style/FrozenStringLiteralComment:
+  Enabled: true
+
+# Specs often have multiline block chains
+Style/MultilineBlockChain:
+  Exclude:
+    - 'spec/**/*'
+
+# Disable predicate method naming rule
+Naming/PredicateMethod:
+  Enabled: false
+
+# Allow development dependencies in gemspec
+Gemspec/DevelopmentDependencies:
+  Enabled: false
+
+# Enforce first argument on new line for multiline method calls
+Layout/FirstMethodArgumentLineBreak:
+  Enabled: true
+
+# Use fixed indentation for arguments
+Layout/ArgumentAlignment:
+  EnforcedStyle: with_fixed_indentation
+
+# Ensure closing parenthesis on new line for multiline calls
+Layout/MultilineMethodCallBraceLayout:
+  EnforcedStyle: new_line

data/.ruby-version

@@ -0,0 +1 @@
+3.4.7

data/CHANGELOG.md

@@ -0,0 +1,66 @@
+# Changelog
+
+## Unreleased
+- [Breaking] **Project renamed from `llms-txt-ruby` to `llm-docs-builder`** to better reflect expanded functionality beyond just llms.txt generation.
+  - Gem name: `llms-txt-ruby` → `llm-docs-builder`
+  - Module name: `LlmsTxt` → `LlmDocsBuilder`
+  - CLI command: `llms-txt` → `llm-docs-builder`
+  - Config file: `llms-txt.yml` → `llm-docs-builder.yml`
+  - Docker images: `mensfeld/llms-txt-ruby` → `mensfeld/llm-docs-builder`
+  - Repository: `llms-txt-ruby` → `llm-docs-builder`
+  - Updated all documentation, examples, and tests
+- [Feature] Added Docker support for easy CLI usage without Ruby installation.
+  - Multi-stage Dockerfile for minimal image size (~78MB)
+  - Multi-architecture support (linux/amd64, linux/arm64)
+  - Published to Docker Hub (`mensfeld/llm-docs-builder`) and GitHub Container Registry
+  - GitHub Actions workflow for automated Docker builds and publishing
+  - Comprehensive Docker usage documentation with examples for all commands
+  - CI/CD integration examples (GitHub Actions, GitLab CI, Jenkins)
+- [Feature] Added `compare` command to measure context window savings by comparing content sizes between human and AI versions.
+  - Compare remote URL with different User-Agents (human browser vs AI bot)
+  - Compare remote URL with local markdown file
+  - Display reduction percentage, bytes saved, and compression factor
+  - Support for custom User-Agents and verbose output
+- [Enhancement] Added `Comparator` class with comprehensive specs for HTTP fetching and size comparison.
+- [Enhancement] Added `-u/--url` and `-f/--file` CLI flags for compare command.
+- [Security] Added redirect depth limiting (MAX_REDIRECTS = 10) to prevent infinite redirect loops.
+- [Security] Added URL validation to reject non-HTTP/HTTPS schemes (prevents file://, javascript:, ftp://, etc.).
+- [Security] Added URL format validation to ensure proper host and scheme presence.
+- [Enhancement] Added verbose redirect logging to show redirect chains when --verbose flag is used.
+
+## 0.2.0 (2025-10-07)
+- [Breaking] Removed positional argument support for all CLI commands. All file paths must now be specified using flags:
+  - `transform`: use `-d/--docs` flag instead of positional argument
+  - `parse`: use `-d/--docs` flag instead of positional argument (defaults to `llms.txt` if not specified)
+  - `validate`: use `-d/--docs` flag instead of positional argument (defaults to `llms.txt` if not specified)
+- [Enhancement] Improved CLI consistency by requiring explicit flags for all file paths.
+- [Enhancement] Added comprehensive CLI integration tests in `spec/integrations/` directory.
+  - Each command has its own dedicated integration test file
+  - Tests verify actual CLI binary execution, not just Ruby API
+  - All tests (unit and integration) run together with `bin/rspecs`
+- [Enhancement] Added convenient test runner script `bin/rspecs` for running all tests.
+- [Enhancement] Added comprehensive YARD documentation to all CLI methods.
+- [Enhancement] Resolved all RuboCop offenses (0 offenses detected).
+- [Fix] Fixed validator bug where `each_value` was incorrectly called on Array.
+
+## 0.1.3 (2025-10-07)
+- [Fix] Fixed `transform` command to accept file path from `-d/--docs` flag in addition to positional arguments.
+
+## 0.1.2 (2025-10-07)
+- [Fix] Fixed CLI error handling to use correct `LlmsTxt::Errors::BaseError` instead of non-existent `LlmsTxt::Error`.
+- [Enhancement] Extracted CLI class to `lib/llms_txt/cli.rb` for better testability.
+- [Enhancement] Added comprehensive CLI error handling specs.
+
+## 0.1.1 (2025-10-07)
+- [Change] Updated repository metadata to use `master` branch instead of `main`.
+
+## 0.1.0 (2025-10-07)
+- [Feature] Generate `llms.txt` files from markdown documentation.
+- [Feature] Transform individual markdown files to be AI-friendly.
+- [Feature] Bulk transformation of entire documentation directories.
+- [Feature] CLI with commands: `generate`, `transform`, `bulk-transform`, `parse`, `validate`.
+- [Feature] Configuration file support (`llms-txt.yml`).
+- [Feature] Automatic link expansion from relative to absolute URLs.
+- [Feature] File prioritization (README first, then guides, APIs, etc.).
+- [Feature] Exclusion patterns for bulk transformations.
+- [Feature] Ruby API for programmatic usage.

data/CLAUDE.md

@@ -0,0 +1,178 @@
+# CLAUDE.md
+
+llm-docs-builder is a Ruby gem that generates [llms.txt](https://llmstxt.org/) files from existing markdown documentation and transforms markdown files to be AI-friendly. It provides both a CLI tool and Ruby API.
+
+## Project Overview
+
+llm-docs-builder is a Ruby gem that generates [llms.txt](https://llmstxt.org/) files from existing markdown documentation and transforms markdown files to be AI-friendly. It provides both a CLI tool and Ruby API.
+
+**Key functionality:**
+- Generates llms.txt files from documentation directories by scanning markdown files, extracting metadata, and organizing by priority
+- Transforms individual markdown files by expanding relative links to absolute URLs
+- Bulk transforms entire documentation trees with customizable suffixes and exclusion patterns
+- Supports both config file and direct options for all operations
+
+## Development Commands
+
+### Testing
+```bash
+# Run all tests
+./bin/rspecs
+
+# Run specific test file
+bundle exec rspec spec/llm_docs_builder_spec.rb
+
+# Run specific test line
+bundle exec rspec spec/llm_docs_builder_spec.rb:42
+```
+
+### Code Quality
+```bash
+# Run RuboCop linter
+bundle exec rubocop
+
+# Auto-fix RuboCop violations
+bundle exec rubocop -a
+
+# Run all checks (tests + linting)
+bundle exec rake
+```
+
+### CLI Testing
+```bash
+# Test CLI locally
+bundle exec bin/llm-docs-builder generate --docs ./docs
+bundle exec bin/llm-docs-builder transform --docs README.md
+bundle exec bin/llm-docs-builder bulk-transform --docs ./docs
+
+# Test compare command (requires network)
+bundle exec bin/llm-docs-builder compare --url https://karafka.io/docs/Getting-Started.html
+bundle exec bin/llm-docs-builder compare --url https://example.com/page.html --file docs/local.md
+```
+
+### Building and Installing
+```bash
+# Build gem locally
+bundle exec rake build
+
+# Install locally built gem
+gem install pkg/llm-docs-builder-*.gem
+
+# Release (maintainers only)
+bundle exec rake release
+```
+
+## Architecture
+
+### Core Components
+
+**LlmDocsBuilder Module** (`lib/llm_docs_builder.rb`)
+- Main API entry point with class methods for all operations
+- Uses Zeitwerk for autoloading
+- Delegates to specialized classes for generation, transformation, and validation
+- All methods support both config file and direct options via `Config#merge_with_options`
+
+**Generator** (`lib/llm_docs_builder/generator.rb`)
+- Scans documentation directories recursively using `Find.find`
+- Extracts title from first H1 header, description from first paragraph
+- Prioritizes files: README (1), getting started (2), guides (3), tutorials (4), API (5), reference (6), others (7)
+- Builds formatted llms.txt with links and descriptions
+
+**MarkdownTransformer** (`lib/llm_docs_builder/markdown_transformer.rb`)
+- Transforms individual markdown files using regex patterns
+- `expand_relative_links`: Converts relative links to absolute URLs using base_url
+- `convert_html_urls`: Changes .html/.htm URLs to .md format
+- Leaves absolute URLs and anchor links unchanged
+
+**BulkTransformer** (`lib/llm_docs_builder/bulk_transformer.rb`)
+- Recursively processes all markdown files in a directory
+- Uses `MarkdownTransformer` for each file
+- Generates output paths with configurable suffix (default: `.llm`)
+- Empty suffix (`""`) enables in-place transformation
+- Supports glob-based exclusion patterns via `File.fnmatch`
+
+**Comparator** (`lib/llm_docs_builder/comparator.rb`)
+- Measures context window savings by comparing content sizes
+- Fetches URLs with different User-Agents (human browser vs AI bot)
+- Can compare remote URL with local markdown file
+- Uses Net::HTTP for fetching with redirect support
+- Calculates reduction percentage, bytes saved, and compression factor
+
+**Config** (`lib/llm_docs_builder/config.rb`)
+- Loads YAML config from file or auto-finds `llms-txt.yml`
+- Merges config file options with programmatic options (programmatic takes precedence)
+- Handles defaults: `suffix: '.llm'`, `output: 'llms.txt'`, `excludes: []`
+
+**CLI** (`lib/llm_docs_builder/cli.rb`)
+- Parses commands: generate, transform, bulk-transform, compare, parse, validate, version
+- Uses OptionParser for flag parsing
+- Loads config and merges with CLI options before delegating to main module
+- Handles errors gracefully with user-friendly messages
+- Compare command displays formatted output with human-readable byte sizes (bytes/KB/MB)
+
+### Configuration Precedence
+
+Options are resolved in this order (highest to lowest priority):
+1. Direct method arguments (e.g., `LlmDocsBuilder.generate_from_docs('./docs', title: 'Override')`)
+2. CLI flags (e.g., `--docs ./docs`)
+3. Config file values (e.g., `llms-txt.yml`)
+4. Defaults (e.g., `suffix: '.llm'`, `output: 'llms.txt'`)
+
+### File Priority System
+
+When generating llms.txt, files are automatically ordered by importance:
+- Priority 1: README files (always listed first)
+- Priority 2: Getting started guides
+- Priority 3: General guides
+- Priority 4: Tutorials
+- Priority 5: API documentation
+- Priority 6: Reference documentation
+- Priority 7: All other files
+
+### Link Transformation Logic
+
+**Relative Link Expansion** (when `base_url` provided):
+- Converts `[text](./path.md)` → `[text](https://base.url/path.md)`
+- Converts `[text](../other.md)` → `[text](https://base.url/other.md)`
+- Skips URLs starting with `http://`, `https://`, `//`, or `#`
+
+**URL Conversion** (when `convert_urls: true`):
+- Changes `https://example.com/page.html` → `https://example.com/page.md`
+- Changes `https://example.com/doc.htm` → `https://example.com/doc.md`
+
+### In-Place vs Separate Files
+
+**Separate Files** (`suffix: '.llm'` - default):
+- Creates new files: `README.md` → `README.llm.md`
+- Preserves originals for human-readable documentation
+- Useful for dual-serving human and AI versions
+
+**In-Place** (`suffix: ""`):
+- Overwrites originals: `README.md` → `README.md` (transformed)
+- Used in build pipelines (e.g., Karafka framework)
+- Transforms documentation before deployment
+
+## Testing Strategy
+
+- RSpec for all tests with SimpleCov coverage tracking
+- Unit tests for each component in isolation
+- Integration tests in `spec/integrations/` for end-to-end workflows
+- Example outputs saved in `spec/examples.txt` for persistence
+- CI tests against Ruby 3.2, 3.3, 3.4 via GitHub Actions
+
+## Dependencies
+
+- **zeitwerk**: Autoloading and code organization
+- **optparse**: Built-in Ruby CLI parsing (no external CLI framework)
+- **rspec**: Testing framework
+- **rubocop**: Code linting and style enforcement
+- **simplecov**: Test coverage reporting
+
+## Code Style
+
+- Ruby 3.2+ syntax and features required
+- Frozen string literals in all files
+- Explicit module nesting (no `class Foo::Bar`)
+- Comprehensive YARD documentation for public APIs
+- Private methods clearly marked and documented
+- RuboCop enforces consistent style