llm-docs-builder 1.0.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: a1d023947b940a99bfa254d72ff79e47eef15f378eeb2e46ca2f04859b5da6dd
4
- data.tar.gz: 85448c820f8fce92e9a26c094a9dd8cd485619ac45eb72ad45ed5b058ff2dc69
3
+ metadata.gz: c9ce6fbe87a426d69139719ecd76489c5aed030517167758ea0b2df452f3d636
4
+ data.tar.gz: 9ce3bfab8d25b4c3a9f11b899f20038da89463eb82ffde1b69106ce4934377c8
5
5
  SHA512:
6
- metadata.gz: 9b345c53110799db9429b5185c1f773ae7ed99bdfa4da73f038cacad8cd319aa378f483f928a76d7f73533da08201f930f74b98e28a0246042e85d5269dc0d24
7
- data.tar.gz: 6f876f2ec8b9e41a8daf1626b87e0cfb610fe164ff47231587ab3cbb78d2ea1ed699d06a24ea2927426ea7a227996828fbdc82c3a4abbc3e37883bee3277923d
6
+ metadata.gz: 9f98c868bfde76f265dd69854a43da9c67b48f5310234611105a6dec00a62236070461484b52c8e08c85cf9fd7bfcbffea0a85e92ce0943ac879b9dc532d3b77
7
+ data.tar.gz: 6a6340a373df54139338af837a7755b3148fdaef4d58810779cd88b5e2c5f1cca6edda231cee9e3cdace9cc60d465b14e0d78ecf41c3e338fb9fc40029d3f4c0
@@ -19,10 +19,12 @@ jobs:
19
19
  specs:
20
20
  timeout-minutes: 15
21
21
  runs-on: ubuntu-latest
22
+ continue-on-error: ${{ contains(matrix.ruby, 'preview') || contains(matrix.ruby, 'rc') }}
22
23
  strategy:
23
24
  fail-fast: false
24
25
  matrix:
25
26
  ruby:
27
+ - '4.1.0-preview1'
26
28
  - '4.0'
27
29
  - '3.4'
28
30
  - '3.3'
@@ -34,8 +36,12 @@ jobs:
34
36
  with:
35
37
  fetch-depth: 0
36
38
 
39
+ - name: Remove Gemfile.lock for Ruby preview versions
40
+ if: contains(matrix.ruby, 'preview') || contains(matrix.ruby, 'rc')
41
+ run: rm -f Gemfile.lock
42
+
37
43
  - name: Set up Ruby
38
- uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0
44
+ uses: ruby/setup-ruby@7f797815d8adceb1c412e7abefdd1ab4abc47a5f # v1.335.0
39
45
  with:
40
46
  ruby-version: ${{ matrix.ruby }}
41
47
  bundler-cache: true
@@ -60,9 +66,9 @@ jobs:
60
66
  steps:
61
67
  - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
62
68
  - name: Set up Ruby
63
- uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0
69
+ uses: ruby/setup-ruby@7f797815d8adceb1c412e7abefdd1ab4abc47a5f # v1.335.0
64
70
  with:
65
- ruby-version: '4.0.6'
71
+ ruby-version: '4.0.9'
66
72
  bundler-cache: true
67
73
  - name: Run yard-lint
68
74
  run: bundle exec yard-lint lib/
@@ -75,7 +81,7 @@ jobs:
75
81
  with:
76
82
  fetch-depth: 0
77
83
  - name: Set up Node.js
78
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
84
+ uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
79
85
  with:
80
86
  node-version: '20'
81
87
  cache: 'npm'
@@ -45,10 +45,10 @@ jobs:
45
45
  type=raw,value=latest,enable={{is_default_branch}}
46
46
 
47
47
  - name: Set up QEMU
48
- uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4
48
+ uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4
49
49
 
50
50
  - name: Set up Docker Buildx
51
- uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4
51
+ uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4
52
52
 
53
53
  - name: Login to Docker Hub
54
54
  if: github.event_name != 'pull_request'
@@ -66,7 +66,7 @@ jobs:
66
66
  password: ${{ secrets.GITHUB_TOKEN }}
67
67
 
68
68
  - name: Build and push
69
- uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7
69
+ uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7
70
70
  with:
71
71
  context: .
72
72
  platforms: linux/amd64,linux/arm64
@@ -24,7 +24,7 @@ jobs:
24
24
  fetch-depth: 0
25
25
 
26
26
  - name: Set up Ruby
27
- uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0
27
+ uses: ruby/setup-ruby@7f797815d8adceb1c412e7abefdd1ab4abc47a5f # v1.335.0
28
28
  with:
29
29
  bundler-cache: false
30
30
 
@@ -32,4 +32,4 @@ jobs:
32
32
  run: |
33
33
  bundle install --jobs 4 --retry 3
34
34
 
35
- - uses: rubygems/release-gem@052cc82692552de3ef2b81fd670e41d13cba8092 # v1.4.0
35
+ - uses: rubygems/release-gem@7f9650160c1a4e7989fdc9855807bdbd421d8b6b # v1.4.1
data/.ruby-version CHANGED
@@ -1 +1 @@
1
- 4.0.6
1
+ 4.0.9
data/CHANGELOG.md CHANGED
@@ -1,31 +1,55 @@
1
1
  # Changelog
2
2
 
3
- ## [Unreleased]
3
+ ## 1.1.0 (2026-11-02)
4
+ - [Enhancement] **Relative Link Resolution** - `base_url` link expansion now resolves relative links like a browser would.
5
+ - `../` and `./` segments are normalized (`../other.md` → `https://example.com/other.md` instead of `https://example.com/docs/../other.md`).
6
+ - `bulk-transform` resolves each link from its file's own directory, so `./setup.md` in `guides/intro.md` expands to `<base_url>/guides/setup.md`.
7
+ - Links starting with `/` resolve from `base_url`; anchors, query strings, absolute URLs and `mailto:`/`tel:`/`ftp:` links are unchanged.
8
+ - [Fix] **llms.txt Paths Are Percent-Encoded** - `generate` now percent-encodes each path segment in `llms.txt` entries, with and without `base_url`.
9
+ - `docs/my docs/a b.md` gives `my%20docs/a%20b.md`, so names with spaces or parentheses no longer break the Markdown link.
10
+ - Non-ASCII names are encoded as UTF-8 (`zażółć.md` → `za%C5%BC%C3%B3%C5%82%C4%87.md`).
11
+ - [Fix] **llms.txt Descriptions** - A page description is now always a single line, and the 200-character limit no longer cuts a Markdown link or image in half; a link or image that does not fit is left out.
12
+ - [Fix] **Transformed Files Picked Up as Sources** - `bulk-transform` and `generate` now skip files ending with the transform suffix (`*.llm.md` by default, or `*<suffix>.md`).
13
+ - A second `bulk-transform` run no longer writes `*.llm.llm.md` files.
14
+ - `generate` lists each page once, from its source file, instead of listing both `README.md` and `README.llm.md`.
15
+ - Source files whose names end with the suffix (e.g. `notes.llm.md`) are skipped too. An empty suffix skips nothing.
16
+ - [Fix] **Angle-Bracket Link Destinations** - `base_url` link expansion now resolves CommonMark `[text](<path with spaces>)` links like other relative links: the brackets are dropped and spaces are percent-encoded (`<../my docs/a b.md>` → `<base_url>/my%20docs/a%20b.md`) instead of producing a broken URL with a stray `>`.
17
+ - [Fix] **Query-Only Links** - `base_url` link expansion now resolves a query-only link like `[q](?tab=2)` to the current page (`guides/sub/page.md` gives `<base_url>/guides/sub/page.md?tab=2`, as RFC 3986 does) instead of the page's directory (`<base_url>/guides/sub/?tab=2`). Anchor-only links like `#self` are unchanged.
18
+ - [Fix] **`simplify_links` Strips Only One Prefix** - repeated filler prefixes are now all removed, so `Click here to see the docs` becomes `docs` (as documented) instead of `see the`.
19
+ - [Fix] **Links Inside Code Are Rewritten** - link transformations (`base_url` expansion, `convert_urls`, `simplify_links`) no longer change links in fenced code blocks, indented code blocks and inline code spans.
20
+ - [Fix] **`convert_urls` Skips Anchors, Queries and Relative Links** - `.html`/`.htm` URLs followed by an anchor or a query are now converted (`doc.html#sec` → `doc.md#sec`, `doc.html?x=1` → `doc.md?x=1`).
21
+ - Relative link targets are converted too, also without `base_url` (`[Intro](../intro.html)` → `[Intro](../intro.md)`).
22
+ - `.html` inside a query or an anchor and targets with a non-HTTP scheme (`mailto:`, `ftp:`, `sftp:`, `file:`, ...) are left unchanged.
23
+ - [Fix] **Links With Brackets in Their Text** - `base_url` link expansion now expands links whose text holds brackets, nested up to two levels deep (`[a [b] c](../intro.md)`), and linked images like `[![alt](../img/a.png)](../intro.md)`, where both the image and the outer link are now expanded. Before, such links stayed relative, and a linked image got only its image expanded.
24
+ - [Fix] **Links With Any URI Scheme** - `base_url` link expansion now leaves every link with a URI scheme unchanged, in any case (`HTTPS://`, `Mailto:`, `sftp:`, `ssh:`, `git:`, `irc:`, `urn:`, `data:`, `javascript:`, ...). Before, only lowercase `http`, `https`, `mailto`, `ftp` and `tel` links were kept, and others became broken URLs like `<base_url>/guides/sftp:/host/x`.
25
+ - [Fix] **Link Reference Definitions** - `base_url` link expansion now expands reference-style link definitions (`[id]: ../intro.md "Title"`) like inline links, so `[text][id]`, `[id][]` and `[id]` links no longer stay relative while inline links in the same file are expanded. Titles are kept; definitions with a URI scheme, anchors, footnotes (`[^1]: ...`) and definitions in code are unchanged.
26
+ - `convert_urls` converts `.html`/`.htm` definition targets too (`[id]: ../intro.html` → `[id]: ../intro.md`).
27
+ - [Fix] **`compare` Web Server Link** - The "Configure your web server" link in the `compare` output now points at the README section "Serving Optimized Docs to AI Bots" (`#serving-optimized-docs-to-ai-bots`) instead of a missing `#serving-optimized-docs` anchor.
4
28
 
5
29
  ## 1.0.0 (2026-08-24)
6
- - [Fix] **Link Expansion Corrupts Non-HTTP URI Schemes** — `LinkTransformer#expand_relative_links` no longer rewrites `mailto:`, `tel:`, and `ftp:` links into dead `https://...` URLs when `base_url` is set (#165).
30
+ - [Fix] **Link Expansion Corrupts Non-HTTP URI Schemes** - `LinkTransformer#expand_relative_links` no longer rewrites `mailto:`, `tel:`, and `ftp:` links into dead `https://...` URLs when `base_url` is set (#165).
7
31
  - Any link whose scheme is in the known absolute-scheme list (`http`, `https`, `mailto`, `ftp`, `tel`) is now left untouched, matching `HtmlToMarkdownConverter`'s `SAFE_URI_SCHEMES`.
8
32
  - Relative links (e.g. `./guide.md`) continue to expand as before.
9
33
  - [Enhancement] Add Ruby warning category opt-in to test helpers
10
34
 
11
35
  ## 0.13.0 (2026-05-11)
12
36
  - [Breaking] Dropped Ruby 3.2 support. Minimum required Ruby version is now 3.3.
13
- - [Fix] **Heading Nesting in Normalize Mode** — Fixed `normalize_headings` producing incorrect heading levels when same-level headings are nested under a parent.
37
+ - [Fix] **Heading Nesting in Normalize Mode** - Fixed `normalize_headings` producing incorrect heading levels when same-level headings are nested under a parent.
14
38
  - Consecutive same-level headings (e.g., two `##` headings) now correctly adjust the child's markdown level to be deeper than the parent's (e.g., `##` → `###`)
15
39
  - Effective level formula: `max(parent_effective_level + 1, original_level)`
16
40
  - ATX closing hashes (`## Title ##`) are now stripped from titles before building hierarchical paths
17
- - [Fix] **HTML-to-Markdown Section Nesting** — Headings inside nested `<section>` elements now receive adjusted markdown levels based on section nesting depth.
41
+ - [Fix] **HTML-to-Markdown Section Nesting** - Headings inside nested `<section>` elements now receive adjusted markdown levels based on section nesting depth.
18
42
  - `<h2>` inside a nested `<section>` under another `<section>` with its own `<h2>` is now correctly rendered as `###` instead of `##`
19
43
 
20
44
  ## 0.12.0 (2025-11-12)
21
- - [Feature] **HTML to Markdown Reverse Converter** — Added support for converting HTML content to markdown format.
45
+ - [Feature] **HTML to Markdown Reverse Converter** - Added support for converting HTML content to markdown format.
22
46
  - Enables processing of HTML documentation sources
23
47
  - Integrates seamlessly with the transformer pipeline
24
48
  - Useful for converting web-based docs to markdown for further processing
25
49
  - By @Eric-Guo in PR #32.
26
50
 
27
51
  ## 0.11.0 (2025-11-03)
28
- - [Feature] **Transform from URL** — The `transform` command now accepts a remote URL via `--url` and processes fetched content through the standard transformer pipeline.
52
+ - [Feature] **Transform from URL** - The `transform` command now accepts a remote URL via `--url` and processes fetched content through the standard transformer pipeline.
29
53
  - Example: `llm-docs-builder transform --url https://example.com/docs/page.html`
30
54
  - Applies all configured transformations and output options identically to local files
31
55
  - By @Eric-Guo and @codex in PR #28.
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- llm-docs-builder (1.0.0)
4
+ llm-docs-builder (1.1.0)
5
5
  nokogiri (~> 1.17)
6
6
  zeitwerk (~> 2.6)
7
7
 
@@ -11,8 +11,8 @@ GEM
11
11
  ast (2.4.3)
12
12
  coderay (1.1.3)
13
13
  diff-lcs (1.6.2)
14
- io-console (0.9.2)
15
- json (2.21.2)
14
+ io-console (0.9.4)
15
+ json (3.0.2)
16
16
  language_server-protocol (3.17.0.6)
17
17
  lint_roller (1.1.0)
18
18
  method_source (1.1.0)
@@ -20,19 +20,19 @@ GEM
20
20
  nokogiri (1.19.4)
21
21
  mini_portile2 (~> 2.8.2)
22
22
  racc (~> 1.4)
23
- parallel (2.1.0)
23
+ parallel (2.2.0)
24
24
  parser (3.3.12.0)
25
25
  ast (~> 2.4.1)
26
26
  racc
27
27
  prism (1.9.0)
28
- pry (0.16.0)
28
+ pry (0.16.1)
29
29
  coderay (~> 1.1)
30
30
  method_source (~> 1.0)
31
31
  reline (>= 0.6.0)
32
32
  racc (1.8.1)
33
33
  rainbow (3.1.1)
34
34
  rake (13.4.2)
35
- regexp_parser (2.12.0)
35
+ regexp_parser (2.13.1)
36
36
  reline (0.7.0)
37
37
  io-console (~> 0.5)
38
38
  rspec (3.13.2)
@@ -48,8 +48,8 @@ GEM
48
48
  diff-lcs (>= 1.2.0, < 2.0)
49
49
  rspec-support (~> 3.13.0)
50
50
  rspec-support (3.13.7)
51
- rubocop (1.89.0)
52
- json (~> 2.3)
51
+ rubocop (1.91.0)
52
+ json (>= 2.3)
53
53
  language_server-protocol (~> 3.17.0.2)
54
54
  lint_roller (~> 1.1.0)
55
55
  parallel (>= 1.10)
@@ -63,13 +63,13 @@ GEM
63
63
  parser (>= 3.3.7.2)
64
64
  prism (~> 1.7)
65
65
  ruby-progressbar (1.13.0)
66
- simplecov (1.1.1)
67
- unicode-display_width (3.2.0)
68
- unicode-emoji (~> 4.1)
69
- unicode-emoji (4.2.0)
66
+ simplecov (1.3.2)
67
+ unicode-display_width (3.3.0)
68
+ unicode-emoji (~> 4.3)
69
+ unicode-emoji (4.3.0)
70
70
  warning (1.6.0)
71
71
  yard (0.9.45)
72
- yard-lint (1.11.0)
72
+ yard-lint (1.13.0)
73
73
  yard (~> 0.9)
74
74
  zeitwerk (~> 2.6)
75
75
  zeitwerk (2.8.3)
data/README.md CHANGED
@@ -125,7 +125,7 @@ llm-docs-builder generate --config llm-docs-builder.yml
125
125
  ```yaml
126
126
  # llm-docs-builder.yml
127
127
  docs: ./docs
128
- base_url: https://myproject.io
128
+ base_url: https://myproject.io # Base URL for expanding relative links
129
129
  title: My Project
130
130
  description: Brief description
131
131
  body: Optional body content between description and sections
@@ -165,6 +165,12 @@ excludes:
165
165
  - "**/drafts/**"
166
166
  ```
167
167
 
168
+ **Link expansion (`base_url`):** relative links become absolute URLs.
169
+ - Inline links, images (also linked images, `[![alt](a.png)](../intro.md)`) and reference-style definitions (`[a]: ../intro.md`) are expanded. `<path with spaces>` destinations are resolved and their spaces encoded as `%20`.
170
+ - `bulk-transform` resolves links from each file's directory under `docs`. Single-file `transform` (and `LlmDocsBuilder.transform_markdown`) resolves them from `base_url`: with `base_url: https://myproject.io/docs`, `../intro.md` becomes `https://myproject.io/intro.md`.
171
+ - Links starting with `/` resolve from `base_url`. In `bulk-transform`, a query-only link (`[q](?tab=2)`) resolves to the current page (`.../page.md?tab=2`).
172
+ - Left unchanged: links in fenced code, indented code and inline code; anchors (`#x`); protocol-relative URLs (`//cdn.io/a.md`); URLs with any scheme, in any case (`HTTPS:`, `sftp:`, `urn:`, `mailto:`); HTML `<a href>` and `<img src>`.
173
+
168
174
  ## CLI Commands
169
175
 
170
176
  ```bash
@@ -183,31 +189,34 @@ llm-docs-builder version # Show version
183
189
  -d, --docs PATH Documentation path
184
190
  -o, --output PATH Output file
185
191
  -u, --url URL URL for comparison
192
+ -f, --file PATH Local markdown file for comparison
186
193
  -v, --verbose Detailed output
187
194
  ```
188
195
 
189
196
  ## Serving Optimized Docs to AI Bots
190
197
 
191
- After using `bulk-transform` with `suffix: .llm`, configure your web server to serve optimized versions to AI bots:
198
+ After using `bulk-transform` with `suffix: .llm`, configure your web server to serve optimized versions to AI bots. Pages without an `.llm.md` version are served as is:
192
199
 
193
200
  **Apache (.htaccess):**
194
201
  ```apache
195
202
  SetEnvIf User-Agent "(?i)(openai|anthropic|claude|gpt)" IS_LLM_BOT
203
+ RewriteEngine On
196
204
  RewriteCond %{ENV:IS_LLM_BOT} !^$
205
+ RewriteCond %{REQUEST_URI} !\.llm\.md$
206
+ RewriteCond %{REQUEST_FILENAME} ^(.+)\.md$
207
+ RewriteCond %1.llm.md -f
197
208
  RewriteRule ^(.*)\.md$ $1.llm.md [L]
198
209
  ```
199
210
 
200
211
  **Nginx:**
201
212
  ```nginx
202
- map $http_user_agent $is_llm_bot {
203
- default 0;
204
- "~*(?i)(openai|anthropic|claude|gpt)" 1;
213
+ map $http_user_agent $llm_suffix {
214
+ default "";
215
+ "~*(openai|anthropic|claude|gpt)" ".llm";
205
216
  }
206
217
 
207
- location ~ ^/docs/(.*)\.md$ {
208
- if ($is_llm_bot) {
209
- rewrite ^(.*)\.md$ $1.llm.md last;
210
- }
218
+ location ~ ^/docs/(?<doc>.*)\.md$ {
219
+ try_files /docs/$doc$llm_suffix.md $uri =404;
211
220
  }
212
221
  ```
213
222
 
@@ -258,7 +267,10 @@ api = API.new
258
267
  ```markdown
259
268
  # API Documentation
260
269
 
261
- [complete API documentation](./api.md)
270
+
271
+ > Important: This is a note
272
+
273
+ [Click here to see the complete API documentation](./api.md)
262
274
 
263
275
  api = API.new
264
276
  ```
@@ -411,7 +423,7 @@ All compression features can be used individually for fine-grained control:
411
423
  - `generate_toc: true` - Generate table of contents from headings with anchor links
412
424
  - `custom_instruction: "text"` - Inject AI context message at document top
413
425
  - `simplify_links: true` - Simplify verbose link text (e.g., "Click here to see the docs" → "docs")
414
- - `convert_urls: true` - Convert `.html`/`.htm` URLs to `.md` format
426
+ - `convert_urls: true` - Convert `.html`/`.htm` URLs to `.md` format, keeping anchors and queries (`../intro.html#x` → `../intro.md#x`): absolute `http(s)` URLs anywhere in the text, and relative link and reference-style definition targets. URLs in code are unchanged
415
427
  - `normalize_whitespace: true` - Reduce excessive blank lines and remove trailing whitespace
416
428
 
417
429
  ## License
@@ -76,16 +76,19 @@ module LlmDocsBuilder
76
76
 
77
77
  # Recursively scans the docs directory for markdown files
78
78
  #
79
- # Skips hidden files (starting with dot) and returns sorted array of paths
79
+ # Skips hidden files (starting with dot) and already transformed files (ending with
80
+ # the suffix, e.g. README.llm.md) and returns sorted array of paths
80
81
  #
81
82
  # @return [Array<String>] paths to markdown files
82
83
  def find_markdown_files
83
84
  files = []
85
+ output_ending = "#{options[:suffix]}.md".downcase
84
86
 
85
87
  Find.find(docs_path) do |path|
86
88
  next unless File.file?(path)
87
89
  next unless path.match?(/\.md$/i)
88
90
  next if File.basename(path).start_with?('.')
91
+ next if output_ending != '.md' && path.downcase.end_with?(output_ending)
89
92
 
90
93
  files << path
91
94
  end
@@ -109,12 +112,14 @@ module LlmDocsBuilder
109
112
 
110
113
  # Applies markdown transformations to a single file
111
114
  #
112
- # Creates MarkdownTransformer instance and delegates transformation
115
+ # Creates MarkdownTransformer instance and delegates transformation. Passes the file's
116
+ # path relative to the docs directory so relative links resolve from its own location.
113
117
  #
114
118
  # @param file_path [String] path to markdown file
115
119
  # @return [String] transformed content
116
120
  def transform_file(file_path)
117
- transformer = MarkdownTransformer.new(file_path, options)
121
+ source_path = Pathname.new(file_path).relative_path_from(Pathname.new(docs_path)).to_s
122
+ transformer = MarkdownTransformer.new(file_path, options.merge(source_path: source_path))
118
123
  transformer.transform
119
124
  end
120
125
 
@@ -12,6 +12,10 @@ module LlmDocsBuilder
12
12
  #
13
13
  # @api public
14
14
  class Generator
15
+ # Maximum length of a file description in llms.txt, in characters
16
+ MAX_DESCRIPTION_LENGTH = 200
17
+ private_constant :MAX_DESCRIPTION_LENGTH
18
+
15
19
  # @return [String] path to documentation directory or file
16
20
  attr_reader :docs_path
17
21
 
@@ -27,6 +31,7 @@ module LlmDocsBuilder
27
31
  # @option options [String] :description project description (overrides auto-detection)
28
32
  # @option options [String] :output output file path for saving
29
33
  # @option options [Boolean] :verbose enable verbose output
34
+ # @option options [String] :suffix suffix of bulk-transformed files to skip (default: '.llm')
30
35
  def initialize(docs_path, options = {})
31
36
  @docs_path = docs_path
32
37
  @options = options
@@ -68,11 +73,13 @@ module LlmDocsBuilder
68
73
 
69
74
  # Recursively finds and analyzes markdown files in directory
70
75
  #
71
- # Sorts by priority (README, guides, etc.) and skips hidden files
76
+ # Sorts by priority (README, guides, etc.) and skips hidden files and bulk-transformed
77
+ # files (ending with the suffix, e.g. README.llm.md)
72
78
  #
73
79
  # @return [Array<Hash>] sorted array of analyzed file metadata
74
80
  def find_markdown_files_in_directory
75
81
  files = []
82
+ output_ending = "#{options.fetch(:suffix, '.llm')}.md".downcase
76
83
 
77
84
  Find.find(docs_path) do |path|
78
85
  # Skip hidden directories unless explicitly enabled
@@ -85,6 +92,7 @@ module LlmDocsBuilder
85
92
  next unless File.file?(path)
86
93
  next unless path.match?(/\.md$/i)
87
94
  next if File.basename(path).start_with?('.')
95
+ next if output_ending != '.md' && path.downcase.end_with?(output_ending)
88
96
  next if should_exclude?(path)
89
97
 
90
98
  files << analyze_file(path)
@@ -157,7 +165,8 @@ module LlmDocsBuilder
157
165
 
158
166
  # Extracts description from file content
159
167
  #
160
- # Takes first paragraph after title, truncated to 200 characters
168
+ # Takes first paragraph after title as a single line, truncated to MAX_DESCRIPTION_LENGTH characters.
169
+ # A Markdown link or image that would be cut by the limit is dropped entirely.
161
170
  #
162
171
  # @param content [String] file content
163
172
  # @return [String] extracted description
@@ -170,7 +179,62 @@ module LlmDocsBuilder
170
179
  # Get first paragraph
171
180
  first_paragraph = description_lines.take_while { |line| !line.strip.empty? }
172
181
 
173
- first_paragraph.join(' ').strip.slice(0, 200)
182
+ description = first_paragraph.join(' ').gsub(/\s+/, ' ').strip
183
+ return description if description.length <= MAX_DESCRIPTION_LENGTH
184
+
185
+ cut_link = link_spans(description).find do |span|
186
+ span.begin < MAX_DESCRIPTION_LENGTH && span.end >= MAX_DESCRIPTION_LENGTH
187
+ end
188
+
189
+ return description.slice(0, MAX_DESCRIPTION_LENGTH) unless cut_link
190
+
191
+ description.slice(0, cut_link.begin).rstrip
192
+ end
193
+
194
+ # Finds the character ranges of Markdown links and images in text
195
+ #
196
+ # Brackets and parentheses are matched by nesting depth, so linked images, nested link text, URLs
197
+ # with parentheses and reference links are each one range. An unclosed bracket is not a link.
198
+ #
199
+ # @param text [String] single-line text
200
+ # @return [Array<Range>] inclusive index ranges
201
+ def link_spans(text)
202
+ closing = closing_indexes(text)
203
+ spans = []
204
+ index = 0
205
+
206
+ while (start = text.index('[', index))
207
+ stop = closing[start]
208
+ index = start + 1
209
+ next unless stop
210
+
211
+ stop = closing[stop + 1] while closing[stop + 1]
212
+ start -= 1 if start.positive? && text[start - 1] == '!'
213
+ spans << (start..stop)
214
+ index = stop + 1
215
+ end
216
+
217
+ spans
218
+ end
219
+
220
+ # Maps the index of each opening bracket or parenthesis to the index of the one that closes it
221
+ #
222
+ # @param text [String] text to search
223
+ # @return [Hash{Integer => Integer}] closing index by opening index, unclosed ones are absent
224
+ def closing_indexes(text)
225
+ closing = {}
226
+ open_indexes = { '[' => [], '(' => [] }
227
+ pairs = { ']' => '[', ')' => '(' }
228
+
229
+ text.each_char.with_index do |char, index|
230
+ if open_indexes.key?(char)
231
+ open_indexes[char] << index
232
+ elsif pairs.key?(char) && (opened = open_indexes[pairs[char]].pop)
233
+ closing[opened] = index
234
+ end
235
+ end
236
+
237
+ closing
174
238
  end
175
239
 
176
240
  # Assigns priority to file based on filename patterns
@@ -310,13 +374,18 @@ module LlmDocsBuilder
310
374
 
311
375
  # Constructs full URL from path using base_url option if provided
312
376
  #
377
+ # Each path segment is percent-encoded, so names with spaces, parentheses or
378
+ # non-ASCII characters give a valid Markdown link destination.
379
+ #
313
380
  # @param path [String] relative path to file
314
381
  # @return [String] full URL or relative path
315
382
  def build_url(path)
383
+ encoded_path = path.split('/').map { |segment| ERB::Util.url_encode(segment) }.join('/')
384
+
316
385
  if (base_url = options[:base_url])
317
- File.join(base_url, path)
386
+ File.join(base_url, encoded_path)
318
387
  else
319
- path
388
+ encoded_path
320
389
  end
321
390
  end
322
391
 
@@ -128,7 +128,7 @@ module LlmDocsBuilder
128
128
  puts ' llm-docs-builder bulk-transform --docs ./docs --config llm-docs-builder.yml'
129
129
  puts ''
130
130
  puts '2. Configure your web server to serve .md files to AI bots:'
131
- puts ' See: https://github.com/mensfeld/llm-docs-builder#serving-optimized-docs'
131
+ puts ' See: https://github.com/mensfeld/llm-docs-builder#serving-optimized-docs-to-ai-bots'
132
132
  puts ''
133
133
  puts '3. Measure your actual savings:'
134
134
  puts ' llm-docs-builder compare --url <your-url> --file <local-md>'
@@ -15,61 +15,349 @@ module LlmDocsBuilder
15
15
  # @param content [String] markdown content
16
16
  # @param options [Hash] transformation options
17
17
  # @option options [String] :base_url base URL for expanding relative links
18
+ # @option options [String] :source_path path of the transformed file relative to the
19
+ # docs root, used to resolve relative links against the file's own directory
18
20
  # @option options [Boolean] :convert_urls convert HTML URLs to markdown format
19
21
  # @return [String] transformed content
22
+ # @note Code (fenced and indented code blocks, inline code spans) is left unchanged
20
23
  def transform(content, options = {})
21
- result = content.dup
22
-
23
- result = expand_relative_links(result, options[:base_url]) if options[:base_url]
24
- result = convert_html_urls(result) if options[:convert_urls]
25
- result = simplify_links(result) if options[:simplify_links]
24
+ outside_code(content) do |result|
25
+ if options[:base_url]
26
+ result = expand_relative_links(
27
+ result, options[:base_url], options[:source_path], convert_urls: options[:convert_urls]
28
+ )
29
+ end
30
+ result = convert_html_urls(result, convert_definitions: !options[:base_url]) if options[:convert_urls]
31
+ result = simplify_links(result) if options[:simplify_links]
26
32
 
27
- result
33
+ result
34
+ end
28
35
  end
29
36
 
30
37
  private
31
38
 
32
- # URI schemes that must never be treated as relative paths when expanding
33
- # links with a base URL (kept in sync with the converter's safe schemes)
34
- ABSOLUTE_URL_SCHEMES = %w[http https mailto ftp tel].freeze
39
+ # A link target with a URI scheme is never a relative path
40
+ URI_SCHEME_PATTERN = /\A[a-z][a-z0-9+.-]*:/i
41
+ private_constant :URI_SCHEME_PATTERN
42
+
43
+ # Placeholder that stands in for a protected code fragment; a code block's ends with `b`
44
+ CODE_PLACEHOLDER = /\u0000(\d+)b?\u0000/
45
+
46
+ # Inline code span: a backtick run closed by a run of the same length, within one paragraph
47
+ INLINE_CODE = /(?<![`\\])(`+)(?!`)(?:(?!\n[ \t]*\n)[^\u0000])+?(?<!`)\1(?!`)/m
48
+
49
+ # Opening line of a fenced code block
50
+ FENCE_OPENING = /\A[ \t]*(`{3,}|~{3,})/
51
+
52
+ # Blockquote markers at the start of a line
53
+ BLOCKQUOTE_PREFIX = /\A(?: {0,3}> ?)*/
54
+
55
+ # Line that starts a list item
56
+ LIST_ITEM = /\A {0,3}(?:[-+*]|\d{1,9}[.)])(?:[ \t]|\s*\z)/
57
+
58
+ # Contents of a bracket pair in a link text: plain text, images and brackets that do not open a link
59
+ # (links do not nest)
60
+ BRACKET_CONTENT_PATTERN = /(?:[^\[\]]|!\[[^\[\]]*\]|\[[^\[\]]*\](?!\())*/
61
+
62
+ # Part of a link text: brackets in it may be nested up to two levels deep (e.g. a linked image)
63
+ LINK_TEXT_PART_PATTERN = /[^\[\]]|!\[#{BRACKET_CONTENT_PATTERN}\]|\[#{BRACKET_CONTENT_PATTERN}\](?!\()/
64
+
65
+ # Link destination; it never runs into protected code, only its title may contain it
66
+ LINK_DESTINATION_PATTERN = /\(([^)\u0000]+(?:\s+(?:"[^")]*"|'[^')]*'))?)\)/
67
+
68
+ # Inline link
69
+ LINK_PATTERN = /\[((?:#{LINK_TEXT_PART_PATTERN})+)\]#{LINK_DESTINATION_PATTERN}/
70
+
71
+ # Inline link or image inside a link text, where an image may have no alt text (`[![](badge.svg)](url)`)
72
+ NESTED_LINK_PATTERN = /\[((?:#{LINK_TEXT_PART_PATTERN})*)\]#{LINK_DESTINATION_PATTERN}/
73
+
74
+ # Link reference definition line (`[id]: target "title"`): its label, destination and the rest of the line.
75
+ # Footnote definitions (`[^1]: text`) are not link reference definitions.
76
+ REFERENCE_DEFINITION_PATTERN = /
77
+ ^((?:\ {0,3}>\ ?)*\ {0,3}\[(?!\^)(?:[^\[\]\\\n]|\\.)+\]:[ \t]*)
78
+ (<[^<>\n]*>|[^\s<\u0000][^\s\u0000]*)
79
+ ((?:[ \t]+(?:"[^"\n]*"|'[^'\n]*'|\([^()\n]*\)))?[ \t\r]*)$
80
+ /x
81
+
82
+ # Consecutive link reference definition lines
83
+ REFERENCE_DEFINITIONS_PATTERN = /#{REFERENCE_DEFINITION_PATTERN}(?:\n#{REFERENCE_DEFINITION_PATTERN})*/
84
+
85
+ # Line that link reference definitions may follow: a blank line, an ATX heading, a thematic break or a code
86
+ # block. A definition cannot interrupt a paragraph.
87
+ REFERENCE_DEFINITIONS_PRECEDING_LINE_PATTERN = /
88
+ (?:\ {0,3}>\ ?)*
89
+ (?:
90
+ [ \t]* |
91
+ \ {0,3}\#{1,6}(?:[ \t][^\n]*)? |
92
+ \ {0,3}(?:(?:\*[ \t]*){3,}|(?:-[ \t]*){3,}|(?:_[ \t]*){3,}) |
93
+ \u0000\d+b\u0000[ \t]*
94
+ )\r?
95
+ /x
96
+
97
+ # Definitions on the first or second line of the content
98
+ LEADING_REFERENCE_DEFINITIONS_PATTERN =
99
+ /\A(?:#{REFERENCE_DEFINITIONS_PRECEDING_LINE_PATTERN}\n)?\K#{REFERENCE_DEFINITIONS_PATTERN}/
100
+
101
+ # Definitions from the third line on, so that they never overlap the leading ones. Kept apart from the leading
102
+ # pattern: an `\A` alternative here would make the regex try every position of the content.
103
+ FOLLOWING_REFERENCE_DEFINITIONS_PATTERN =
104
+ /\n#{REFERENCE_DEFINITIONS_PRECEDING_LINE_PATTERN}\n\K#{REFERENCE_DEFINITIONS_PATTERN}/
105
+
106
+ private_constant :CODE_PLACEHOLDER, :INLINE_CODE, :FENCE_OPENING, :BLOCKQUOTE_PREFIX, :LIST_ITEM,
107
+ :BRACKET_CONTENT_PATTERN, :LINK_TEXT_PART_PATTERN, :LINK_DESTINATION_PATTERN, :LINK_PATTERN,
108
+ :NESTED_LINK_PATTERN, :REFERENCE_DEFINITION_PATTERN, :REFERENCE_DEFINITIONS_PATTERN,
109
+ :REFERENCE_DEFINITIONS_PRECEDING_LINE_PATTERN, :LEADING_REFERENCE_DEFINITIONS_PATTERN,
110
+ :FOLLOWING_REFERENCE_DEFINITIONS_PATTERN
111
+
112
+ # Run a transformation on content with code replaced by placeholders, then restore the code
113
+ #
114
+ # @param content [String] markdown content
115
+ # @yieldparam text [String] content with code fragments replaced by placeholders
116
+ # @yieldreturn [String] transformed content
117
+ # @return [String] transformed content with the original code restored
118
+ def outside_code(content)
119
+ codes = []
120
+ protect = lambda do |code, block: false|
121
+ codes << code
122
+ "\u0000#{codes.size - 1}#{'b' if block}\u0000"
123
+ end
124
+
125
+ text = protect_code_blocks(content, protect)
126
+ text = text.gsub(INLINE_CODE) { protect.call(::Regexp.last_match(0)) }
127
+
128
+ yield(text).gsub(CODE_PLACEHOLDER) { codes[::Regexp.last_match(1).to_i] }
129
+ end
130
+
131
+ # Replace fenced and indented code blocks with placeholders
132
+ #
133
+ # Indented lines count as code only after a blank line and outside lists, where they would otherwise be
134
+ # paragraph or list item continuations. Code inside blockquotes is detected after the `>` markers.
135
+ #
136
+ # @param content [String] markdown content
137
+ # @param protect [Proc] stores a code fragment and returns its placeholder
138
+ # @return [String] content with code blocks replaced
139
+ def protect_code_blocks(content, protect)
140
+ out = +''
141
+ code = +''
142
+ fence = nil
143
+ fence_depth = 0
144
+ depth = 0
145
+ in_list = false
146
+ prev_blank = true
147
+
148
+ content.each_line do |line|
149
+ quote = line[BLOCKQUOTE_PREFIX]
150
+ body = line.delete_prefix(quote)
151
+ prev_blank ||= quote.count('>') > depth
152
+ depth = quote.count('>')
153
+ fence = nil if depth < fence_depth
154
+
155
+ if fence
156
+ code << line
157
+ closing = line.sub(/\A(?: {0,3}> ?){#{fence_depth}}/, '')
158
+ fence = nil if closing.match?(/\A[ \t]*#{Regexp.escape(fence)}#{Regexp.escape(fence[0])}*\s*\z/)
159
+ prev_blank = true
160
+ next
161
+ end
162
+
163
+ if !in_list && (prev_blank || !code.empty?) && indented_code?(body)
164
+ code << line
165
+ next
166
+ end
167
+
168
+ if (fence = fence_opening(body))
169
+ fence_depth = depth
170
+ code << line
171
+ next
172
+ end
173
+
174
+ out << protect_lines(code, protect) << line
175
+ code = +''
176
+ in_list = in_list_after?(body, in_list, prev_blank)
177
+ prev_blank = body.strip.empty?
178
+ end
179
+
180
+ out << protect_lines(code, protect)
181
+ end
182
+
183
+ # Fence of a line that opens a fenced code block
184
+ #
185
+ # @param line [String] markdown line
186
+ # @return [String, nil] the opening fence, or nil when the line does not open a block
187
+ def fence_opening(line)
188
+ opening = line.match(FENCE_OPENING)
189
+ return unless opening
190
+ return if opening[1].start_with?('`') && opening.post_match.include?('`')
191
+
192
+ opening[1]
193
+ end
194
+
195
+ # Check whether a line is indented enough to be code
196
+ #
197
+ # @param line [String] markdown line
198
+ # @return [Boolean] true for a non-blank line indented by four spaces or a tab
199
+ def indented_code?(line)
200
+ line.match?(/\A(?: {4}|\t)\s*\S/)
201
+ end
202
+
203
+ # Check whether content after a line is still inside a list
204
+ #
205
+ # @param line [String] markdown line
206
+ # @param in_list [Boolean] whether the previous line was inside a list
207
+ # @param prev_blank [Boolean] whether the previous line was blank
208
+ # @return [Boolean] true when the list continues after this line
209
+ def in_list_after?(line, in_list, prev_blank)
210
+ return true if line.match?(LIST_ITEM)
211
+ return in_list if line.strip.empty? || !prev_blank || line.match?(/\A[ \t]/)
212
+
213
+ false
214
+ end
215
+
216
+ # Replace collected code lines with a placeholder, keeping the final line break
217
+ #
218
+ # @param code [String] collected code lines
219
+ # @param protect [Proc] stores a code fragment and returns its placeholder
220
+ # @return [String] placeholder, or an empty string when there is no code
221
+ def protect_lines(code, protect)
222
+ return '' if code.empty?
223
+
224
+ body = code.chomp
225
+ "#{protect.call(body, block: true)}#{code.delete_prefix(body)}"
226
+ end
35
227
 
36
- # Expand relative links to absolute URLs
228
+ # Expand relative links and link reference definitions to absolute URLs
37
229
  #
38
230
  # @param content [String] markdown content
39
231
  # @param base_url [String] base URL for expansion
232
+ # @param source_path [String, nil] path of the file relative to the docs root
233
+ # @param convert_urls [Boolean] also convert HTML targets of link reference definitions to markdown, so the
234
+ # definitions are scanned only once
40
235
  # @return [String] content with expanded links
41
- def expand_relative_links(content, base_url)
42
- content.gsub(/\[([^\]]+)\]\(([^)]+)\)/) do |match|
43
- text = ::Regexp.last_match(1)
44
- url = ::Regexp.last_match(2)
236
+ def expand_relative_links(content, base_url, source_path = nil, convert_urls: false)
237
+ result = expand_links_matching(content, LINK_PATTERN, base_url, source_path)
238
+
239
+ rewrite_reference_definitions(result) do |target|
240
+ expanded = expand_target(target, base_url, source_path)
241
+ convert_urls ? convert_html_target(expanded) : expanded
242
+ end
243
+ end
244
+
245
+ # Rewrite the destinations of link reference definitions
246
+ #
247
+ # @param content [String] markdown content
248
+ # @yieldparam target [String] destination of a definition
249
+ # @yieldreturn [String] new destination
250
+ # @return [String] content with rewritten definitions
251
+ def rewrite_reference_definitions(content)
252
+ return content unless content.include?(']:')
253
+
254
+ rewrite = lambda do |definitions|
255
+ definitions.gsub(REFERENCE_DEFINITION_PATTERN) do
256
+ match = ::Regexp.last_match
45
257
 
46
- if url.start_with?('http://', 'https://', '//', '#') || absolute_url_scheme?(url)
47
- match
48
- else
49
- clean_url = url.gsub(%r{^\./}, '')
50
- expanded_url = File.join(base_url, clean_url)
51
- "[#{text}](#{expanded_url})"
258
+ "#{match[1]}#{yield(match[2])}#{match[3]}"
52
259
  end
53
260
  end
261
+
262
+ content
263
+ .sub(LEADING_REFERENCE_DEFINITIONS_PATTERN, &rewrite)
264
+ .gsub(FOLLOWING_REFERENCE_DEFINITIONS_PATTERN, &rewrite)
54
265
  end
55
266
 
56
- # Check whether a link target uses a known absolute URI scheme
267
+ # Expand relative links that match a pattern, and the links and images in their link text
57
268
  #
58
- # @param url [String] link target
59
- # @return [Boolean] true when the target already carries its own scheme
60
- def absolute_url_scheme?(url)
61
- scheme = url.split(':', 2).first.to_s
62
- ABSOLUTE_URL_SCHEMES.include?(scheme)
269
+ # @param content [String] markdown content
270
+ # @param pattern [Regexp] pattern of the links to expand
271
+ # @param base_url [String] base URL for expansion
272
+ # @param source_path [String, nil] path of the file relative to the docs root
273
+ # @return [String] content with expanded links
274
+ def expand_links_matching(content, pattern, base_url, source_path)
275
+ content.gsub(pattern) do
276
+ url = ::Regexp.last_match(2)
277
+ text = expand_links_matching(::Regexp.last_match(1), NESTED_LINK_PATTERN, base_url, source_path)
278
+
279
+ "[#{text}](#{expand_target(url, base_url, source_path)})"
280
+ end
281
+ end
282
+
283
+ # Expand a link target unless it is absolute, protocol-relative or an anchor
284
+ #
285
+ # @param target [String] link target, optionally followed by a link title
286
+ # @param base_url [String] base URL for expansion
287
+ # @param source_path [String, nil] path of the file relative to the docs root
288
+ # @return [String] expanded or unchanged target
289
+ def expand_target(target, base_url, source_path)
290
+ destination = target.delete_prefix('<')
291
+ return target if destination.start_with?('//', '#') || destination.match?(URI_SCHEME_PATTERN)
292
+
293
+ resolve_relative_url(base_url, target, source_path)
294
+ end
295
+
296
+ # Resolve a relative link target against the base URL
297
+ #
298
+ # Targets starting with `/` are resolved from the base URL itself, other targets from the source
299
+ # file's directory. `./` and `../` segments are normalized. Titles, queries and anchors are kept.
300
+ # Angle-bracket destinations (`<path with spaces>`) are unwrapped and their spaces percent-encoded.
301
+ # A query-only target (`?tab=2`) resolves to the source file itself, as in RFC 3986, with its
302
+ # name percent-encoded.
303
+ #
304
+ # @param base_url [String] base URL for expansion
305
+ # @param target [String] relative link target, optionally followed by a link title
306
+ # @param source_path [String, nil] path of the source file relative to the docs root
307
+ # @return [String] expanded URL
308
+ def resolve_relative_url(base_url, target, source_path)
309
+ url, title = target.match(/\A(.*?)(\s+(?:"[^"]*"|'[^']*'))?\z/m).captures
310
+ url = url[/\A<(.*)>\z/m, 1]&.gsub(' ', '%20') || url
311
+ path, suffix = url.match(/\A([^?#]*)(.*)\z/m).captures
312
+ query_only = source_path && path.empty? && suffix.start_with?('?')
313
+ path = ERB::Util.url_encode(File.basename(source_path)) if query_only
314
+ source_dir = source_path ? File.dirname(source_path) : '.'
315
+ origin = base_url[%r{\A[^:/?#]+://[^/?#]*}].to_s
316
+ base_path = base_url.delete_prefix(origin)
317
+ dir = path.start_with?('/') ? base_path : File.join(base_path, source_dir)
318
+
319
+ "#{origin}#{normalize_path("#{dir}/#{path}")}#{suffix}#{title}"
320
+ end
321
+
322
+ # Normalize `.`, `..` and empty segments of a path
323
+ #
324
+ # @param path [String] path to normalize
325
+ # @return [String] normalized path, with its trailing slash kept
326
+ def normalize_path(path)
327
+ normalized = Pathname.new(path).cleanpath.to_s
328
+ normalized += '/' if path.match?(%r{/\.{0,2}\z}) && !normalized.end_with?('/')
329
+ normalized
63
330
  end
64
331
 
65
332
  # Convert HTML URLs to markdown format
66
333
  #
334
+ # Converts absolute URLs anywhere in the content and relative link and link reference definition targets,
335
+ # keeping queries and anchors.
336
+ #
67
337
  # @param content [String] markdown content
338
+ # @param convert_definitions [Boolean] whether to convert link reference definition targets
68
339
  # @return [String] content with converted URLs
69
- def convert_html_urls(content)
70
- content.gsub(%r{https?://[^\s<>]+\.html?(?=[)\s]|$)}) do |url|
71
- url.sub(/\.html?$/, '.md')
340
+ def convert_html_urls(content, convert_definitions: true)
341
+ result = content.gsub(%r{https?://[^\s<>?#]+?\.html?(?=[?#)\s]|$)}) do |url|
342
+ url.sub(/\.html?\z/, '.md')
343
+ end
344
+
345
+ result = result.gsub(/\]\(([^)\s?#]+?)\.html?(?=[?#)\s])/) do |match|
346
+ path = ::Regexp.last_match(1)
347
+ path.match?(URI_SCHEME_PATTERN) ? match : "](#{path}.md"
72
348
  end
349
+
350
+ return result unless convert_definitions
351
+
352
+ rewrite_reference_definitions(result) { |target| convert_html_target(target) }
353
+ end
354
+
355
+ # Convert a relative HTML link reference definition target to markdown
356
+ #
357
+ # @param target [String] destination of a definition
358
+ # @return [String] converted or unchanged target
359
+ def convert_html_target(target)
360
+ target.match?(URI_SCHEME_PATTERN) ? target : target.sub(/\A([^?#]+?)\.html?(?=[?#]|\z)/, '\1.md')
73
361
  end
74
362
 
75
363
  # Simplify verbose link text
@@ -77,14 +365,14 @@ module LlmDocsBuilder
77
365
  # @param content [String] markdown content
78
366
  # @return [String] content with simplified links
79
367
  def simplify_links(content)
80
- content.gsub(/\[([^\]]+)\]\(([^)]+)\)/) do
368
+ content.gsub(LINK_PATTERN) do
81
369
  text = ::Regexp.last_match(1)
82
370
  url = ::Regexp.last_match(2)
83
371
 
84
372
  simplified_text = text
85
- .gsub(/^(click here to|see|read more about|check out|visit)\s+(the\s+)?/i, '')
373
+ .gsub(/^(?:(?:click here to|see|read more about|check out|visit)\s+(?:the\s+)?)+/i, '')
86
374
  .gsub(/\s+(here|documentation|docs)$/i, '')
87
- .strip
375
+ .gsub(/\A\s+|\s+\z/, '') # String#strip also removes the NUL of code placeholders
88
376
 
89
377
  simplified_text = text if simplified_text.empty?
90
378
 
@@ -2,5 +2,5 @@
2
2
 
3
3
  module LlmDocsBuilder
4
4
  # Current version of the LlmDocsBuilder gem
5
- VERSION = '1.0.0'
5
+ VERSION = '1.1.0'
6
6
  end
@@ -6,6 +6,7 @@ require 'find'
6
6
 
7
7
  autoload(:Nokogiri, 'nokogiri')
8
8
  autoload(:CGI, 'cgi')
9
+ autoload(:ERB, 'erb')
9
10
 
10
11
  loader = Zeitwerk::Loader.for_gem
11
12
  loader.inflector.inflect('cli' => 'CLI')
@@ -1,5 +1,5 @@
1
- # Example llms-txt.yml configuration file
2
- # Copy to llms-txt.yml and customize for your project
1
+ # Example llm-docs-builder.yml configuration file
2
+ # Copy to llm-docs-builder.yml and customize for your project
3
3
 
4
4
  # Path to documentation directory or file
5
5
  docs: ./docs
data/package-lock.json CHANGED
@@ -103,9 +103,9 @@
103
103
  }
104
104
  },
105
105
  "node_modules/fastq": {
106
- "version": "1.20.1",
107
- "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.1.tgz",
108
- "integrity": "sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw==",
106
+ "version": "1.20.3",
107
+ "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.3.tgz",
108
+ "integrity": "sha512-XKv5nnLs6nLF71NgiKJLIZFLkPyIEuOselLG7ujZnGrRfQK8HpvY+WqKhAJUAdLomwVHErVS4LfxFlPq0/FTAw==",
109
109
  "dev": true,
110
110
  "license": "ISC",
111
111
  "dependencies": {
@@ -286,9 +286,9 @@
286
286
  }
287
287
  },
288
288
  "node_modules/smol-toml": {
289
- "version": "1.8.0",
290
- "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.8.0.tgz",
291
- "integrity": "sha512-kCZr2V3ch9i00x8zXRhjUNVcjG9ijES5dDudkXvUVCT5QlJNQWElSJdZqyPemffHoLNUYwOcou0Fy+ojN0uHSQ==",
289
+ "version": "1.9.0",
290
+ "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.9.0.tgz",
291
+ "integrity": "sha512-hpd+HLON7HdZXqYchMM/+LaTTbdK0AU3NngIJ4KVyWbY9bfQqdL9cD+4yf6dUoU2Ap4VsU0JkQi6FxAI1B2mXQ==",
292
292
  "dev": true,
293
293
  "license": "BSD-3-Clause",
294
294
  "engines": {
@@ -312,9 +312,9 @@
312
312
  }
313
313
  },
314
314
  "node_modules/yaml": {
315
- "version": "2.9.0",
316
- "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.0.tgz",
317
- "integrity": "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==",
315
+ "version": "2.9.1",
316
+ "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.1.tgz",
317
+ "integrity": "sha512-3NxN8+78OdzbT7C/WjGsyfPAtJaN3FNDsWxv7Y7mcDsT/oOmgW8BpyQQFFBnvZE3j9Y2Sdz1ULFLezL7Eb2yFw==",
318
318
  "dev": true,
319
319
  "license": "ISC",
320
320
  "bin": {
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: llm-docs-builder
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.0
4
+ version: 1.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Maciej Mensfeld
@@ -201,7 +201,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
201
201
  - !ruby/object:Gem::Version
202
202
  version: '0'
203
203
  requirements: []
204
- rubygems_version: 4.0.16
204
+ rubygems_version: 4.0.20
205
205
  specification_version: 4
206
206
  summary: Build and optimize documentation for LLMs - generate llms.txt, transform
207
207
  markdown, and more