llm-docs-builder 1.0.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +10 -4
- data/.github/workflows/docker.yml +3 -3
- data/.github/workflows/push.yml +2 -2
- data/.ruby-version +1 -1
- data/CHANGELOG.md +30 -6
- data/Gemfile.lock +13 -13
- data/README.md +23 -11
- data/lib/llm_docs_builder/bulk_transformer.rb +8 -3
- data/lib/llm_docs_builder/generator.rb +74 -5
- data/lib/llm_docs_builder/output_formatter.rb +1 -1
- data/lib/llm_docs_builder/transformers/link_transformer.rb +320 -32
- data/lib/llm_docs_builder/version.rb +1 -1
- data/lib/llm_docs_builder.rb +1 -0
- data/llm-docs-builder.yml.example +2 -2
- data/package-lock.json +9 -9
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: c9ce6fbe87a426d69139719ecd76489c5aed030517167758ea0b2df452f3d636
|
|
4
|
+
data.tar.gz: 9ce3bfab8d25b4c3a9f11b899f20038da89463eb82ffde1b69106ce4934377c8
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 9f98c868bfde76f265dd69854a43da9c67b48f5310234611105a6dec00a62236070461484b52c8e08c85cf9fd7bfcbffea0a85e92ce0943ac879b9dc532d3b77
|
|
7
|
+
data.tar.gz: 6a6340a373df54139338af837a7755b3148fdaef4d58810779cd88b5e2c5f1cca6edda231cee9e3cdace9cc60d465b14e0d78ecf41c3e338fb9fc40029d3f4c0
|
data/.github/workflows/ci.yml
CHANGED
|
@@ -19,10 +19,12 @@ jobs:
|
|
|
19
19
|
specs:
|
|
20
20
|
timeout-minutes: 15
|
|
21
21
|
runs-on: ubuntu-latest
|
|
22
|
+
continue-on-error: ${{ contains(matrix.ruby, 'preview') || contains(matrix.ruby, 'rc') }}
|
|
22
23
|
strategy:
|
|
23
24
|
fail-fast: false
|
|
24
25
|
matrix:
|
|
25
26
|
ruby:
|
|
27
|
+
- '4.1.0-preview1'
|
|
26
28
|
- '4.0'
|
|
27
29
|
- '3.4'
|
|
28
30
|
- '3.3'
|
|
@@ -34,8 +36,12 @@ jobs:
|
|
|
34
36
|
with:
|
|
35
37
|
fetch-depth: 0
|
|
36
38
|
|
|
39
|
+
- name: Remove Gemfile.lock for Ruby preview versions
|
|
40
|
+
if: contains(matrix.ruby, 'preview') || contains(matrix.ruby, 'rc')
|
|
41
|
+
run: rm -f Gemfile.lock
|
|
42
|
+
|
|
37
43
|
- name: Set up Ruby
|
|
38
|
-
uses: ruby/setup-ruby@
|
|
44
|
+
uses: ruby/setup-ruby@7f797815d8adceb1c412e7abefdd1ab4abc47a5f # v1.335.0
|
|
39
45
|
with:
|
|
40
46
|
ruby-version: ${{ matrix.ruby }}
|
|
41
47
|
bundler-cache: true
|
|
@@ -60,9 +66,9 @@ jobs:
|
|
|
60
66
|
steps:
|
|
61
67
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
62
68
|
- name: Set up Ruby
|
|
63
|
-
uses: ruby/setup-ruby@
|
|
69
|
+
uses: ruby/setup-ruby@7f797815d8adceb1c412e7abefdd1ab4abc47a5f # v1.335.0
|
|
64
70
|
with:
|
|
65
|
-
ruby-version: '4.0.
|
|
71
|
+
ruby-version: '4.0.9'
|
|
66
72
|
bundler-cache: true
|
|
67
73
|
- name: Run yard-lint
|
|
68
74
|
run: bundle exec yard-lint lib/
|
|
@@ -75,7 +81,7 @@ jobs:
|
|
|
75
81
|
with:
|
|
76
82
|
fetch-depth: 0
|
|
77
83
|
- name: Set up Node.js
|
|
78
|
-
uses: actions/setup-node@
|
|
84
|
+
uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
|
|
79
85
|
with:
|
|
80
86
|
node-version: '20'
|
|
81
87
|
cache: 'npm'
|
|
@@ -45,10 +45,10 @@ jobs:
|
|
|
45
45
|
type=raw,value=latest,enable={{is_default_branch}}
|
|
46
46
|
|
|
47
47
|
- name: Set up QEMU
|
|
48
|
-
uses: docker/setup-qemu-action@
|
|
48
|
+
uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4
|
|
49
49
|
|
|
50
50
|
- name: Set up Docker Buildx
|
|
51
|
-
uses: docker/setup-buildx-action@
|
|
51
|
+
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4
|
|
52
52
|
|
|
53
53
|
- name: Login to Docker Hub
|
|
54
54
|
if: github.event_name != 'pull_request'
|
|
@@ -66,7 +66,7 @@ jobs:
|
|
|
66
66
|
password: ${{ secrets.GITHUB_TOKEN }}
|
|
67
67
|
|
|
68
68
|
- name: Build and push
|
|
69
|
-
uses: docker/build-push-action@
|
|
69
|
+
uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7
|
|
70
70
|
with:
|
|
71
71
|
context: .
|
|
72
72
|
platforms: linux/amd64,linux/arm64
|
data/.github/workflows/push.yml
CHANGED
|
@@ -24,7 +24,7 @@ jobs:
|
|
|
24
24
|
fetch-depth: 0
|
|
25
25
|
|
|
26
26
|
- name: Set up Ruby
|
|
27
|
-
uses: ruby/setup-ruby@
|
|
27
|
+
uses: ruby/setup-ruby@7f797815d8adceb1c412e7abefdd1ab4abc47a5f # v1.335.0
|
|
28
28
|
with:
|
|
29
29
|
bundler-cache: false
|
|
30
30
|
|
|
@@ -32,4 +32,4 @@ jobs:
|
|
|
32
32
|
run: |
|
|
33
33
|
bundle install --jobs 4 --retry 3
|
|
34
34
|
|
|
35
|
-
- uses: rubygems/release-gem@
|
|
35
|
+
- uses: rubygems/release-gem@7f9650160c1a4e7989fdc9855807bdbd421d8b6b # v1.4.1
|
data/.ruby-version
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
4.0.
|
|
1
|
+
4.0.9
|
data/CHANGELOG.md
CHANGED
|
@@ -1,31 +1,55 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
-
##
|
|
3
|
+
## 1.1.0 (2026-11-02)
|
|
4
|
+
- [Enhancement] **Relative Link Resolution** - `base_url` link expansion now resolves relative links like a browser would.
|
|
5
|
+
- `../` and `./` segments are normalized (`../other.md` → `https://example.com/other.md` instead of `https://example.com/docs/../other.md`).
|
|
6
|
+
- `bulk-transform` resolves each link from its file's own directory, so `./setup.md` in `guides/intro.md` expands to `<base_url>/guides/setup.md`.
|
|
7
|
+
- Links starting with `/` resolve from `base_url`; anchors, query strings, absolute URLs and `mailto:`/`tel:`/`ftp:` links are unchanged.
|
|
8
|
+
- [Fix] **llms.txt Paths Are Percent-Encoded** - `generate` now percent-encodes each path segment in `llms.txt` entries, with and without `base_url`.
|
|
9
|
+
- `docs/my docs/a b.md` gives `my%20docs/a%20b.md`, so names with spaces or parentheses no longer break the Markdown link.
|
|
10
|
+
- Non-ASCII names are encoded as UTF-8 (`zażółć.md` → `za%C5%BC%C3%B3%C5%82%C4%87.md`).
|
|
11
|
+
- [Fix] **llms.txt Descriptions** - A page description is now always a single line, and the 200-character limit no longer cuts a Markdown link or image in half; a link or image that does not fit is left out.
|
|
12
|
+
- [Fix] **Transformed Files Picked Up as Sources** - `bulk-transform` and `generate` now skip files ending with the transform suffix (`*.llm.md` by default, or `*<suffix>.md`).
|
|
13
|
+
- A second `bulk-transform` run no longer writes `*.llm.llm.md` files.
|
|
14
|
+
- `generate` lists each page once, from its source file, instead of listing both `README.md` and `README.llm.md`.
|
|
15
|
+
- Source files whose names end with the suffix (e.g. `notes.llm.md`) are skipped too. An empty suffix skips nothing.
|
|
16
|
+
- [Fix] **Angle-Bracket Link Destinations** - `base_url` link expansion now resolves CommonMark `[text](<path with spaces>)` links like other relative links: the brackets are dropped and spaces are percent-encoded (`<../my docs/a b.md>` → `<base_url>/my%20docs/a%20b.md`) instead of producing a broken URL with a stray `>`.
|
|
17
|
+
- [Fix] **Query-Only Links** - `base_url` link expansion now resolves a query-only link like `[q](?tab=2)` to the current page (`guides/sub/page.md` gives `<base_url>/guides/sub/page.md?tab=2`, as RFC 3986 does) instead of the page's directory (`<base_url>/guides/sub/?tab=2`). Anchor-only links like `#self` are unchanged.
|
|
18
|
+
- [Fix] **`simplify_links` Strips Only One Prefix** - repeated filler prefixes are now all removed, so `Click here to see the docs` becomes `docs` (as documented) instead of `see the`.
|
|
19
|
+
- [Fix] **Links Inside Code Are Rewritten** - link transformations (`base_url` expansion, `convert_urls`, `simplify_links`) no longer change links in fenced code blocks, indented code blocks and inline code spans.
|
|
20
|
+
- [Fix] **`convert_urls` Skips Anchors, Queries and Relative Links** - `.html`/`.htm` URLs followed by an anchor or a query are now converted (`doc.html#sec` → `doc.md#sec`, `doc.html?x=1` → `doc.md?x=1`).
|
|
21
|
+
- Relative link targets are converted too, also without `base_url` (`[Intro](../intro.html)` → `[Intro](../intro.md)`).
|
|
22
|
+
- `.html` inside a query or an anchor and targets with a non-HTTP scheme (`mailto:`, `ftp:`, `sftp:`, `file:`, ...) are left unchanged.
|
|
23
|
+
- [Fix] **Links With Brackets in Their Text** - `base_url` link expansion now expands links whose text holds brackets, nested up to two levels deep (`[a [b] c](../intro.md)`), and linked images like `[](../intro.md)`, where both the image and the outer link are now expanded. Before, such links stayed relative, and a linked image got only its image expanded.
|
|
24
|
+
- [Fix] **Links With Any URI Scheme** - `base_url` link expansion now leaves every link with a URI scheme unchanged, in any case (`HTTPS://`, `Mailto:`, `sftp:`, `ssh:`, `git:`, `irc:`, `urn:`, `data:`, `javascript:`, ...). Before, only lowercase `http`, `https`, `mailto`, `ftp` and `tel` links were kept, and others became broken URLs like `<base_url>/guides/sftp:/host/x`.
|
|
25
|
+
- [Fix] **Link Reference Definitions** - `base_url` link expansion now expands reference-style link definitions (`[id]: ../intro.md "Title"`) like inline links, so `[text][id]`, `[id][]` and `[id]` links no longer stay relative while inline links in the same file are expanded. Titles are kept; definitions with a URI scheme, anchors, footnotes (`[^1]: ...`) and definitions in code are unchanged.
|
|
26
|
+
- `convert_urls` converts `.html`/`.htm` definition targets too (`[id]: ../intro.html` → `[id]: ../intro.md`).
|
|
27
|
+
- [Fix] **`compare` Web Server Link** - The "Configure your web server" link in the `compare` output now points at the README section "Serving Optimized Docs to AI Bots" (`#serving-optimized-docs-to-ai-bots`) instead of a missing `#serving-optimized-docs` anchor.
|
|
4
28
|
|
|
5
29
|
## 1.0.0 (2026-08-24)
|
|
6
|
-
- [Fix] **Link Expansion Corrupts Non-HTTP URI Schemes**
|
|
30
|
+
- [Fix] **Link Expansion Corrupts Non-HTTP URI Schemes** - `LinkTransformer#expand_relative_links` no longer rewrites `mailto:`, `tel:`, and `ftp:` links into dead `https://...` URLs when `base_url` is set (#165).
|
|
7
31
|
- Any link whose scheme is in the known absolute-scheme list (`http`, `https`, `mailto`, `ftp`, `tel`) is now left untouched, matching `HtmlToMarkdownConverter`'s `SAFE_URI_SCHEMES`.
|
|
8
32
|
- Relative links (e.g. `./guide.md`) continue to expand as before.
|
|
9
33
|
- [Enhancement] Add Ruby warning category opt-in to test helpers
|
|
10
34
|
|
|
11
35
|
## 0.13.0 (2026-05-11)
|
|
12
36
|
- [Breaking] Dropped Ruby 3.2 support. Minimum required Ruby version is now 3.3.
|
|
13
|
-
- [Fix] **Heading Nesting in Normalize Mode**
|
|
37
|
+
- [Fix] **Heading Nesting in Normalize Mode** - Fixed `normalize_headings` producing incorrect heading levels when same-level headings are nested under a parent.
|
|
14
38
|
- Consecutive same-level headings (e.g., two `##` headings) now correctly adjust the child's markdown level to be deeper than the parent's (e.g., `##` → `###`)
|
|
15
39
|
- Effective level formula: `max(parent_effective_level + 1, original_level)`
|
|
16
40
|
- ATX closing hashes (`## Title ##`) are now stripped from titles before building hierarchical paths
|
|
17
|
-
- [Fix] **HTML-to-Markdown Section Nesting**
|
|
41
|
+
- [Fix] **HTML-to-Markdown Section Nesting** - Headings inside nested `<section>` elements now receive adjusted markdown levels based on section nesting depth.
|
|
18
42
|
- `<h2>` inside a nested `<section>` under another `<section>` with its own `<h2>` is now correctly rendered as `###` instead of `##`
|
|
19
43
|
|
|
20
44
|
## 0.12.0 (2025-11-12)
|
|
21
|
-
- [Feature] **HTML to Markdown Reverse Converter**
|
|
45
|
+
- [Feature] **HTML to Markdown Reverse Converter** - Added support for converting HTML content to markdown format.
|
|
22
46
|
- Enables processing of HTML documentation sources
|
|
23
47
|
- Integrates seamlessly with the transformer pipeline
|
|
24
48
|
- Useful for converting web-based docs to markdown for further processing
|
|
25
49
|
- By @Eric-Guo in PR #32.
|
|
26
50
|
|
|
27
51
|
## 0.11.0 (2025-11-03)
|
|
28
|
-
- [Feature] **Transform from URL**
|
|
52
|
+
- [Feature] **Transform from URL** - The `transform` command now accepts a remote URL via `--url` and processes fetched content through the standard transformer pipeline.
|
|
29
53
|
- Example: `llm-docs-builder transform --url https://example.com/docs/page.html`
|
|
30
54
|
- Applies all configured transformations and output options identically to local files
|
|
31
55
|
- By @Eric-Guo and @codex in PR #28.
|
data/Gemfile.lock
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
PATH
|
|
2
2
|
remote: .
|
|
3
3
|
specs:
|
|
4
|
-
llm-docs-builder (1.
|
|
4
|
+
llm-docs-builder (1.1.0)
|
|
5
5
|
nokogiri (~> 1.17)
|
|
6
6
|
zeitwerk (~> 2.6)
|
|
7
7
|
|
|
@@ -11,8 +11,8 @@ GEM
|
|
|
11
11
|
ast (2.4.3)
|
|
12
12
|
coderay (1.1.3)
|
|
13
13
|
diff-lcs (1.6.2)
|
|
14
|
-
io-console (0.9.
|
|
15
|
-
json (
|
|
14
|
+
io-console (0.9.4)
|
|
15
|
+
json (3.0.2)
|
|
16
16
|
language_server-protocol (3.17.0.6)
|
|
17
17
|
lint_roller (1.1.0)
|
|
18
18
|
method_source (1.1.0)
|
|
@@ -20,19 +20,19 @@ GEM
|
|
|
20
20
|
nokogiri (1.19.4)
|
|
21
21
|
mini_portile2 (~> 2.8.2)
|
|
22
22
|
racc (~> 1.4)
|
|
23
|
-
parallel (2.
|
|
23
|
+
parallel (2.2.0)
|
|
24
24
|
parser (3.3.12.0)
|
|
25
25
|
ast (~> 2.4.1)
|
|
26
26
|
racc
|
|
27
27
|
prism (1.9.0)
|
|
28
|
-
pry (0.16.
|
|
28
|
+
pry (0.16.1)
|
|
29
29
|
coderay (~> 1.1)
|
|
30
30
|
method_source (~> 1.0)
|
|
31
31
|
reline (>= 0.6.0)
|
|
32
32
|
racc (1.8.1)
|
|
33
33
|
rainbow (3.1.1)
|
|
34
34
|
rake (13.4.2)
|
|
35
|
-
regexp_parser (2.
|
|
35
|
+
regexp_parser (2.13.1)
|
|
36
36
|
reline (0.7.0)
|
|
37
37
|
io-console (~> 0.5)
|
|
38
38
|
rspec (3.13.2)
|
|
@@ -48,8 +48,8 @@ GEM
|
|
|
48
48
|
diff-lcs (>= 1.2.0, < 2.0)
|
|
49
49
|
rspec-support (~> 3.13.0)
|
|
50
50
|
rspec-support (3.13.7)
|
|
51
|
-
rubocop (1.
|
|
52
|
-
json (
|
|
51
|
+
rubocop (1.91.0)
|
|
52
|
+
json (>= 2.3)
|
|
53
53
|
language_server-protocol (~> 3.17.0.2)
|
|
54
54
|
lint_roller (~> 1.1.0)
|
|
55
55
|
parallel (>= 1.10)
|
|
@@ -63,13 +63,13 @@ GEM
|
|
|
63
63
|
parser (>= 3.3.7.2)
|
|
64
64
|
prism (~> 1.7)
|
|
65
65
|
ruby-progressbar (1.13.0)
|
|
66
|
-
simplecov (1.
|
|
67
|
-
unicode-display_width (3.
|
|
68
|
-
unicode-emoji (~> 4.
|
|
69
|
-
unicode-emoji (4.
|
|
66
|
+
simplecov (1.3.2)
|
|
67
|
+
unicode-display_width (3.3.0)
|
|
68
|
+
unicode-emoji (~> 4.3)
|
|
69
|
+
unicode-emoji (4.3.0)
|
|
70
70
|
warning (1.6.0)
|
|
71
71
|
yard (0.9.45)
|
|
72
|
-
yard-lint (1.
|
|
72
|
+
yard-lint (1.13.0)
|
|
73
73
|
yard (~> 0.9)
|
|
74
74
|
zeitwerk (~> 2.6)
|
|
75
75
|
zeitwerk (2.8.3)
|
data/README.md
CHANGED
|
@@ -125,7 +125,7 @@ llm-docs-builder generate --config llm-docs-builder.yml
|
|
|
125
125
|
```yaml
|
|
126
126
|
# llm-docs-builder.yml
|
|
127
127
|
docs: ./docs
|
|
128
|
-
base_url: https://myproject.io
|
|
128
|
+
base_url: https://myproject.io # Base URL for expanding relative links
|
|
129
129
|
title: My Project
|
|
130
130
|
description: Brief description
|
|
131
131
|
body: Optional body content between description and sections
|
|
@@ -165,6 +165,12 @@ excludes:
|
|
|
165
165
|
- "**/drafts/**"
|
|
166
166
|
```
|
|
167
167
|
|
|
168
|
+
**Link expansion (`base_url`):** relative links become absolute URLs.
|
|
169
|
+
- Inline links, images (also linked images, `[](../intro.md)`) and reference-style definitions (`[a]: ../intro.md`) are expanded. `<path with spaces>` destinations are resolved and their spaces encoded as `%20`.
|
|
170
|
+
- `bulk-transform` resolves links from each file's directory under `docs`. Single-file `transform` (and `LlmDocsBuilder.transform_markdown`) resolves them from `base_url`: with `base_url: https://myproject.io/docs`, `../intro.md` becomes `https://myproject.io/intro.md`.
|
|
171
|
+
- Links starting with `/` resolve from `base_url`. In `bulk-transform`, a query-only link (`[q](?tab=2)`) resolves to the current page (`.../page.md?tab=2`).
|
|
172
|
+
- Left unchanged: links in fenced code, indented code and inline code; anchors (`#x`); protocol-relative URLs (`//cdn.io/a.md`); URLs with any scheme, in any case (`HTTPS:`, `sftp:`, `urn:`, `mailto:`); HTML `<a href>` and `<img src>`.
|
|
173
|
+
|
|
168
174
|
## CLI Commands
|
|
169
175
|
|
|
170
176
|
```bash
|
|
@@ -183,31 +189,34 @@ llm-docs-builder version # Show version
|
|
|
183
189
|
-d, --docs PATH Documentation path
|
|
184
190
|
-o, --output PATH Output file
|
|
185
191
|
-u, --url URL URL for comparison
|
|
192
|
+
-f, --file PATH Local markdown file for comparison
|
|
186
193
|
-v, --verbose Detailed output
|
|
187
194
|
```
|
|
188
195
|
|
|
189
196
|
## Serving Optimized Docs to AI Bots
|
|
190
197
|
|
|
191
|
-
After using `bulk-transform` with `suffix: .llm`, configure your web server to serve optimized versions to AI bots:
|
|
198
|
+
After using `bulk-transform` with `suffix: .llm`, configure your web server to serve optimized versions to AI bots. Pages without an `.llm.md` version are served as is:
|
|
192
199
|
|
|
193
200
|
**Apache (.htaccess):**
|
|
194
201
|
```apache
|
|
195
202
|
SetEnvIf User-Agent "(?i)(openai|anthropic|claude|gpt)" IS_LLM_BOT
|
|
203
|
+
RewriteEngine On
|
|
196
204
|
RewriteCond %{ENV:IS_LLM_BOT} !^$
|
|
205
|
+
RewriteCond %{REQUEST_URI} !\.llm\.md$
|
|
206
|
+
RewriteCond %{REQUEST_FILENAME} ^(.+)\.md$
|
|
207
|
+
RewriteCond %1.llm.md -f
|
|
197
208
|
RewriteRule ^(.*)\.md$ $1.llm.md [L]
|
|
198
209
|
```
|
|
199
210
|
|
|
200
211
|
**Nginx:**
|
|
201
212
|
```nginx
|
|
202
|
-
map $http_user_agent $
|
|
203
|
-
default
|
|
204
|
-
"~*(
|
|
213
|
+
map $http_user_agent $llm_suffix {
|
|
214
|
+
default "";
|
|
215
|
+
"~*(openai|anthropic|claude|gpt)" ".llm";
|
|
205
216
|
}
|
|
206
217
|
|
|
207
|
-
location ~ ^/docs/(
|
|
208
|
-
|
|
209
|
-
rewrite ^(.*)\.md$ $1.llm.md last;
|
|
210
|
-
}
|
|
218
|
+
location ~ ^/docs/(?<doc>.*)\.md$ {
|
|
219
|
+
try_files /docs/$doc$llm_suffix.md $uri =404;
|
|
211
220
|
}
|
|
212
221
|
```
|
|
213
222
|
|
|
@@ -258,7 +267,10 @@ api = API.new
|
|
|
258
267
|
```markdown
|
|
259
268
|
# API Documentation
|
|
260
269
|
|
|
261
|
-
|
|
270
|
+
|
|
271
|
+
> Important: This is a note
|
|
272
|
+
|
|
273
|
+
[Click here to see the complete API documentation](./api.md)
|
|
262
274
|
|
|
263
275
|
api = API.new
|
|
264
276
|
```
|
|
@@ -411,7 +423,7 @@ All compression features can be used individually for fine-grained control:
|
|
|
411
423
|
- `generate_toc: true` - Generate table of contents from headings with anchor links
|
|
412
424
|
- `custom_instruction: "text"` - Inject AI context message at document top
|
|
413
425
|
- `simplify_links: true` - Simplify verbose link text (e.g., "Click here to see the docs" → "docs")
|
|
414
|
-
- `convert_urls: true` - Convert `.html`/`.htm` URLs to `.md` format
|
|
426
|
+
- `convert_urls: true` - Convert `.html`/`.htm` URLs to `.md` format, keeping anchors and queries (`../intro.html#x` → `../intro.md#x`): absolute `http(s)` URLs anywhere in the text, and relative link and reference-style definition targets. URLs in code are unchanged
|
|
415
427
|
- `normalize_whitespace: true` - Reduce excessive blank lines and remove trailing whitespace
|
|
416
428
|
|
|
417
429
|
## License
|
|
@@ -76,16 +76,19 @@ module LlmDocsBuilder
|
|
|
76
76
|
|
|
77
77
|
# Recursively scans the docs directory for markdown files
|
|
78
78
|
#
|
|
79
|
-
# Skips hidden files (starting with dot) and
|
|
79
|
+
# Skips hidden files (starting with dot) and already transformed files (ending with
|
|
80
|
+
# the suffix, e.g. README.llm.md) and returns sorted array of paths
|
|
80
81
|
#
|
|
81
82
|
# @return [Array<String>] paths to markdown files
|
|
82
83
|
def find_markdown_files
|
|
83
84
|
files = []
|
|
85
|
+
output_ending = "#{options[:suffix]}.md".downcase
|
|
84
86
|
|
|
85
87
|
Find.find(docs_path) do |path|
|
|
86
88
|
next unless File.file?(path)
|
|
87
89
|
next unless path.match?(/\.md$/i)
|
|
88
90
|
next if File.basename(path).start_with?('.')
|
|
91
|
+
next if output_ending != '.md' && path.downcase.end_with?(output_ending)
|
|
89
92
|
|
|
90
93
|
files << path
|
|
91
94
|
end
|
|
@@ -109,12 +112,14 @@ module LlmDocsBuilder
|
|
|
109
112
|
|
|
110
113
|
# Applies markdown transformations to a single file
|
|
111
114
|
#
|
|
112
|
-
# Creates MarkdownTransformer instance and delegates transformation
|
|
115
|
+
# Creates MarkdownTransformer instance and delegates transformation. Passes the file's
|
|
116
|
+
# path relative to the docs directory so relative links resolve from its own location.
|
|
113
117
|
#
|
|
114
118
|
# @param file_path [String] path to markdown file
|
|
115
119
|
# @return [String] transformed content
|
|
116
120
|
def transform_file(file_path)
|
|
117
|
-
|
|
121
|
+
source_path = Pathname.new(file_path).relative_path_from(Pathname.new(docs_path)).to_s
|
|
122
|
+
transformer = MarkdownTransformer.new(file_path, options.merge(source_path: source_path))
|
|
118
123
|
transformer.transform
|
|
119
124
|
end
|
|
120
125
|
|
|
@@ -12,6 +12,10 @@ module LlmDocsBuilder
|
|
|
12
12
|
#
|
|
13
13
|
# @api public
|
|
14
14
|
class Generator
|
|
15
|
+
# Maximum length of a file description in llms.txt, in characters
|
|
16
|
+
MAX_DESCRIPTION_LENGTH = 200
|
|
17
|
+
private_constant :MAX_DESCRIPTION_LENGTH
|
|
18
|
+
|
|
15
19
|
# @return [String] path to documentation directory or file
|
|
16
20
|
attr_reader :docs_path
|
|
17
21
|
|
|
@@ -27,6 +31,7 @@ module LlmDocsBuilder
|
|
|
27
31
|
# @option options [String] :description project description (overrides auto-detection)
|
|
28
32
|
# @option options [String] :output output file path for saving
|
|
29
33
|
# @option options [Boolean] :verbose enable verbose output
|
|
34
|
+
# @option options [String] :suffix suffix of bulk-transformed files to skip (default: '.llm')
|
|
30
35
|
def initialize(docs_path, options = {})
|
|
31
36
|
@docs_path = docs_path
|
|
32
37
|
@options = options
|
|
@@ -68,11 +73,13 @@ module LlmDocsBuilder
|
|
|
68
73
|
|
|
69
74
|
# Recursively finds and analyzes markdown files in directory
|
|
70
75
|
#
|
|
71
|
-
# Sorts by priority (README, guides, etc.) and skips hidden files
|
|
76
|
+
# Sorts by priority (README, guides, etc.) and skips hidden files and bulk-transformed
|
|
77
|
+
# files (ending with the suffix, e.g. README.llm.md)
|
|
72
78
|
#
|
|
73
79
|
# @return [Array<Hash>] sorted array of analyzed file metadata
|
|
74
80
|
def find_markdown_files_in_directory
|
|
75
81
|
files = []
|
|
82
|
+
output_ending = "#{options.fetch(:suffix, '.llm')}.md".downcase
|
|
76
83
|
|
|
77
84
|
Find.find(docs_path) do |path|
|
|
78
85
|
# Skip hidden directories unless explicitly enabled
|
|
@@ -85,6 +92,7 @@ module LlmDocsBuilder
|
|
|
85
92
|
next unless File.file?(path)
|
|
86
93
|
next unless path.match?(/\.md$/i)
|
|
87
94
|
next if File.basename(path).start_with?('.')
|
|
95
|
+
next if output_ending != '.md' && path.downcase.end_with?(output_ending)
|
|
88
96
|
next if should_exclude?(path)
|
|
89
97
|
|
|
90
98
|
files << analyze_file(path)
|
|
@@ -157,7 +165,8 @@ module LlmDocsBuilder
|
|
|
157
165
|
|
|
158
166
|
# Extracts description from file content
|
|
159
167
|
#
|
|
160
|
-
# Takes first paragraph after title, truncated to
|
|
168
|
+
# Takes first paragraph after title as a single line, truncated to MAX_DESCRIPTION_LENGTH characters.
|
|
169
|
+
# A Markdown link or image that would be cut by the limit is dropped entirely.
|
|
161
170
|
#
|
|
162
171
|
# @param content [String] file content
|
|
163
172
|
# @return [String] extracted description
|
|
@@ -170,7 +179,62 @@ module LlmDocsBuilder
|
|
|
170
179
|
# Get first paragraph
|
|
171
180
|
first_paragraph = description_lines.take_while { |line| !line.strip.empty? }
|
|
172
181
|
|
|
173
|
-
first_paragraph.join(' ').
|
|
182
|
+
description = first_paragraph.join(' ').gsub(/\s+/, ' ').strip
|
|
183
|
+
return description if description.length <= MAX_DESCRIPTION_LENGTH
|
|
184
|
+
|
|
185
|
+
cut_link = link_spans(description).find do |span|
|
|
186
|
+
span.begin < MAX_DESCRIPTION_LENGTH && span.end >= MAX_DESCRIPTION_LENGTH
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
return description.slice(0, MAX_DESCRIPTION_LENGTH) unless cut_link
|
|
190
|
+
|
|
191
|
+
description.slice(0, cut_link.begin).rstrip
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
# Finds the character ranges of Markdown links and images in text
|
|
195
|
+
#
|
|
196
|
+
# Brackets and parentheses are matched by nesting depth, so linked images, nested link text, URLs
|
|
197
|
+
# with parentheses and reference links are each one range. An unclosed bracket is not a link.
|
|
198
|
+
#
|
|
199
|
+
# @param text [String] single-line text
|
|
200
|
+
# @return [Array<Range>] inclusive index ranges
|
|
201
|
+
def link_spans(text)
|
|
202
|
+
closing = closing_indexes(text)
|
|
203
|
+
spans = []
|
|
204
|
+
index = 0
|
|
205
|
+
|
|
206
|
+
while (start = text.index('[', index))
|
|
207
|
+
stop = closing[start]
|
|
208
|
+
index = start + 1
|
|
209
|
+
next unless stop
|
|
210
|
+
|
|
211
|
+
stop = closing[stop + 1] while closing[stop + 1]
|
|
212
|
+
start -= 1 if start.positive? && text[start - 1] == '!'
|
|
213
|
+
spans << (start..stop)
|
|
214
|
+
index = stop + 1
|
|
215
|
+
end
|
|
216
|
+
|
|
217
|
+
spans
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
# Maps the index of each opening bracket or parenthesis to the index of the one that closes it
|
|
221
|
+
#
|
|
222
|
+
# @param text [String] text to search
|
|
223
|
+
# @return [Hash{Integer => Integer}] closing index by opening index, unclosed ones are absent
|
|
224
|
+
def closing_indexes(text)
|
|
225
|
+
closing = {}
|
|
226
|
+
open_indexes = { '[' => [], '(' => [] }
|
|
227
|
+
pairs = { ']' => '[', ')' => '(' }
|
|
228
|
+
|
|
229
|
+
text.each_char.with_index do |char, index|
|
|
230
|
+
if open_indexes.key?(char)
|
|
231
|
+
open_indexes[char] << index
|
|
232
|
+
elsif pairs.key?(char) && (opened = open_indexes[pairs[char]].pop)
|
|
233
|
+
closing[opened] = index
|
|
234
|
+
end
|
|
235
|
+
end
|
|
236
|
+
|
|
237
|
+
closing
|
|
174
238
|
end
|
|
175
239
|
|
|
176
240
|
# Assigns priority to file based on filename patterns
|
|
@@ -310,13 +374,18 @@ module LlmDocsBuilder
|
|
|
310
374
|
|
|
311
375
|
# Constructs full URL from path using base_url option if provided
|
|
312
376
|
#
|
|
377
|
+
# Each path segment is percent-encoded, so names with spaces, parentheses or
|
|
378
|
+
# non-ASCII characters give a valid Markdown link destination.
|
|
379
|
+
#
|
|
313
380
|
# @param path [String] relative path to file
|
|
314
381
|
# @return [String] full URL or relative path
|
|
315
382
|
def build_url(path)
|
|
383
|
+
encoded_path = path.split('/').map { |segment| ERB::Util.url_encode(segment) }.join('/')
|
|
384
|
+
|
|
316
385
|
if (base_url = options[:base_url])
|
|
317
|
-
File.join(base_url,
|
|
386
|
+
File.join(base_url, encoded_path)
|
|
318
387
|
else
|
|
319
|
-
|
|
388
|
+
encoded_path
|
|
320
389
|
end
|
|
321
390
|
end
|
|
322
391
|
|
|
@@ -128,7 +128,7 @@ module LlmDocsBuilder
|
|
|
128
128
|
puts ' llm-docs-builder bulk-transform --docs ./docs --config llm-docs-builder.yml'
|
|
129
129
|
puts ''
|
|
130
130
|
puts '2. Configure your web server to serve .md files to AI bots:'
|
|
131
|
-
puts ' See: https://github.com/mensfeld/llm-docs-builder#serving-optimized-docs'
|
|
131
|
+
puts ' See: https://github.com/mensfeld/llm-docs-builder#serving-optimized-docs-to-ai-bots'
|
|
132
132
|
puts ''
|
|
133
133
|
puts '3. Measure your actual savings:'
|
|
134
134
|
puts ' llm-docs-builder compare --url <your-url> --file <local-md>'
|
|
@@ -15,61 +15,349 @@ module LlmDocsBuilder
|
|
|
15
15
|
# @param content [String] markdown content
|
|
16
16
|
# @param options [Hash] transformation options
|
|
17
17
|
# @option options [String] :base_url base URL for expanding relative links
|
|
18
|
+
# @option options [String] :source_path path of the transformed file relative to the
|
|
19
|
+
# docs root, used to resolve relative links against the file's own directory
|
|
18
20
|
# @option options [Boolean] :convert_urls convert HTML URLs to markdown format
|
|
19
21
|
# @return [String] transformed content
|
|
22
|
+
# @note Code (fenced and indented code blocks, inline code spans) is left unchanged
|
|
20
23
|
def transform(content, options = {})
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
24
|
+
outside_code(content) do |result|
|
|
25
|
+
if options[:base_url]
|
|
26
|
+
result = expand_relative_links(
|
|
27
|
+
result, options[:base_url], options[:source_path], convert_urls: options[:convert_urls]
|
|
28
|
+
)
|
|
29
|
+
end
|
|
30
|
+
result = convert_html_urls(result, convert_definitions: !options[:base_url]) if options[:convert_urls]
|
|
31
|
+
result = simplify_links(result) if options[:simplify_links]
|
|
26
32
|
|
|
27
|
-
|
|
33
|
+
result
|
|
34
|
+
end
|
|
28
35
|
end
|
|
29
36
|
|
|
30
37
|
private
|
|
31
38
|
|
|
32
|
-
#
|
|
33
|
-
|
|
34
|
-
|
|
39
|
+
# A link target with a URI scheme is never a relative path
|
|
40
|
+
URI_SCHEME_PATTERN = /\A[a-z][a-z0-9+.-]*:/i
|
|
41
|
+
private_constant :URI_SCHEME_PATTERN
|
|
42
|
+
|
|
43
|
+
# Placeholder that stands in for a protected code fragment; a code block's ends with `b`
|
|
44
|
+
CODE_PLACEHOLDER = /\u0000(\d+)b?\u0000/
|
|
45
|
+
|
|
46
|
+
# Inline code span: a backtick run closed by a run of the same length, within one paragraph
|
|
47
|
+
INLINE_CODE = /(?<![`\\])(`+)(?!`)(?:(?!\n[ \t]*\n)[^\u0000])+?(?<!`)\1(?!`)/m
|
|
48
|
+
|
|
49
|
+
# Opening line of a fenced code block
|
|
50
|
+
FENCE_OPENING = /\A[ \t]*(`{3,}|~{3,})/
|
|
51
|
+
|
|
52
|
+
# Blockquote markers at the start of a line
|
|
53
|
+
BLOCKQUOTE_PREFIX = /\A(?: {0,3}> ?)*/
|
|
54
|
+
|
|
55
|
+
# Line that starts a list item
|
|
56
|
+
LIST_ITEM = /\A {0,3}(?:[-+*]|\d{1,9}[.)])(?:[ \t]|\s*\z)/
|
|
57
|
+
|
|
58
|
+
# Contents of a bracket pair in a link text: plain text, images and brackets that do not open a link
|
|
59
|
+
# (links do not nest)
|
|
60
|
+
BRACKET_CONTENT_PATTERN = /(?:[^\[\]]|!\[[^\[\]]*\]|\[[^\[\]]*\](?!\())*/
|
|
61
|
+
|
|
62
|
+
# Part of a link text: brackets in it may be nested up to two levels deep (e.g. a linked image)
|
|
63
|
+
LINK_TEXT_PART_PATTERN = /[^\[\]]|!\[#{BRACKET_CONTENT_PATTERN}\]|\[#{BRACKET_CONTENT_PATTERN}\](?!\()/
|
|
64
|
+
|
|
65
|
+
# Link destination; it never runs into protected code, only its title may contain it
|
|
66
|
+
LINK_DESTINATION_PATTERN = /\(([^)\u0000]+(?:\s+(?:"[^")]*"|'[^')]*'))?)\)/
|
|
67
|
+
|
|
68
|
+
# Inline link
|
|
69
|
+
LINK_PATTERN = /\[((?:#{LINK_TEXT_PART_PATTERN})+)\]#{LINK_DESTINATION_PATTERN}/
|
|
70
|
+
|
|
71
|
+
# Inline link or image inside a link text, where an image may have no alt text (`[](url)`)
|
|
72
|
+
NESTED_LINK_PATTERN = /\[((?:#{LINK_TEXT_PART_PATTERN})*)\]#{LINK_DESTINATION_PATTERN}/
|
|
73
|
+
|
|
74
|
+
# Link reference definition line (`[id]: target "title"`): its label, destination and the rest of the line.
|
|
75
|
+
# Footnote definitions (`[^1]: text`) are not link reference definitions.
|
|
76
|
+
REFERENCE_DEFINITION_PATTERN = /
|
|
77
|
+
^((?:\ {0,3}>\ ?)*\ {0,3}\[(?!\^)(?:[^\[\]\\\n]|\\.)+\]:[ \t]*)
|
|
78
|
+
(<[^<>\n]*>|[^\s<\u0000][^\s\u0000]*)
|
|
79
|
+
((?:[ \t]+(?:"[^"\n]*"|'[^'\n]*'|\([^()\n]*\)))?[ \t\r]*)$
|
|
80
|
+
/x
|
|
81
|
+
|
|
82
|
+
# Consecutive link reference definition lines
|
|
83
|
+
REFERENCE_DEFINITIONS_PATTERN = /#{REFERENCE_DEFINITION_PATTERN}(?:\n#{REFERENCE_DEFINITION_PATTERN})*/
|
|
84
|
+
|
|
85
|
+
# Line that link reference definitions may follow: a blank line, an ATX heading, a thematic break or a code
|
|
86
|
+
# block. A definition cannot interrupt a paragraph.
|
|
87
|
+
REFERENCE_DEFINITIONS_PRECEDING_LINE_PATTERN = /
|
|
88
|
+
(?:\ {0,3}>\ ?)*
|
|
89
|
+
(?:
|
|
90
|
+
[ \t]* |
|
|
91
|
+
\ {0,3}\#{1,6}(?:[ \t][^\n]*)? |
|
|
92
|
+
\ {0,3}(?:(?:\*[ \t]*){3,}|(?:-[ \t]*){3,}|(?:_[ \t]*){3,}) |
|
|
93
|
+
\u0000\d+b\u0000[ \t]*
|
|
94
|
+
)\r?
|
|
95
|
+
/x
|
|
96
|
+
|
|
97
|
+
# Definitions on the first or second line of the content
|
|
98
|
+
LEADING_REFERENCE_DEFINITIONS_PATTERN =
|
|
99
|
+
/\A(?:#{REFERENCE_DEFINITIONS_PRECEDING_LINE_PATTERN}\n)?\K#{REFERENCE_DEFINITIONS_PATTERN}/
|
|
100
|
+
|
|
101
|
+
# Definitions from the third line on, so that they never overlap the leading ones. Kept apart from the leading
|
|
102
|
+
# pattern: an `\A` alternative here would make the regex try every position of the content.
|
|
103
|
+
FOLLOWING_REFERENCE_DEFINITIONS_PATTERN =
|
|
104
|
+
/\n#{REFERENCE_DEFINITIONS_PRECEDING_LINE_PATTERN}\n\K#{REFERENCE_DEFINITIONS_PATTERN}/
|
|
105
|
+
|
|
106
|
+
private_constant :CODE_PLACEHOLDER, :INLINE_CODE, :FENCE_OPENING, :BLOCKQUOTE_PREFIX, :LIST_ITEM,
|
|
107
|
+
:BRACKET_CONTENT_PATTERN, :LINK_TEXT_PART_PATTERN, :LINK_DESTINATION_PATTERN, :LINK_PATTERN,
|
|
108
|
+
:NESTED_LINK_PATTERN, :REFERENCE_DEFINITION_PATTERN, :REFERENCE_DEFINITIONS_PATTERN,
|
|
109
|
+
:REFERENCE_DEFINITIONS_PRECEDING_LINE_PATTERN, :LEADING_REFERENCE_DEFINITIONS_PATTERN,
|
|
110
|
+
:FOLLOWING_REFERENCE_DEFINITIONS_PATTERN
|
|
111
|
+
|
|
112
|
+
# Run a transformation on content with code replaced by placeholders, then restore the code
|
|
113
|
+
#
|
|
114
|
+
# @param content [String] markdown content
|
|
115
|
+
# @yieldparam text [String] content with code fragments replaced by placeholders
|
|
116
|
+
# @yieldreturn [String] transformed content
|
|
117
|
+
# @return [String] transformed content with the original code restored
|
|
118
|
+
def outside_code(content)
|
|
119
|
+
codes = []
|
|
120
|
+
protect = lambda do |code, block: false|
|
|
121
|
+
codes << code
|
|
122
|
+
"\u0000#{codes.size - 1}#{'b' if block}\u0000"
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
text = protect_code_blocks(content, protect)
|
|
126
|
+
text = text.gsub(INLINE_CODE) { protect.call(::Regexp.last_match(0)) }
|
|
127
|
+
|
|
128
|
+
yield(text).gsub(CODE_PLACEHOLDER) { codes[::Regexp.last_match(1).to_i] }
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
# Replace fenced and indented code blocks with placeholders
|
|
132
|
+
#
|
|
133
|
+
# Indented lines count as code only after a blank line and outside lists, where they would otherwise be
|
|
134
|
+
# paragraph or list item continuations. Code inside blockquotes is detected after the `>` markers.
|
|
135
|
+
#
|
|
136
|
+
# @param content [String] markdown content
|
|
137
|
+
# @param protect [Proc] stores a code fragment and returns its placeholder
|
|
138
|
+
# @return [String] content with code blocks replaced
|
|
139
|
+
def protect_code_blocks(content, protect)
|
|
140
|
+
out = +''
|
|
141
|
+
code = +''
|
|
142
|
+
fence = nil
|
|
143
|
+
fence_depth = 0
|
|
144
|
+
depth = 0
|
|
145
|
+
in_list = false
|
|
146
|
+
prev_blank = true
|
|
147
|
+
|
|
148
|
+
content.each_line do |line|
|
|
149
|
+
quote = line[BLOCKQUOTE_PREFIX]
|
|
150
|
+
body = line.delete_prefix(quote)
|
|
151
|
+
prev_blank ||= quote.count('>') > depth
|
|
152
|
+
depth = quote.count('>')
|
|
153
|
+
fence = nil if depth < fence_depth
|
|
154
|
+
|
|
155
|
+
if fence
|
|
156
|
+
code << line
|
|
157
|
+
closing = line.sub(/\A(?: {0,3}> ?){#{fence_depth}}/, '')
|
|
158
|
+
fence = nil if closing.match?(/\A[ \t]*#{Regexp.escape(fence)}#{Regexp.escape(fence[0])}*\s*\z/)
|
|
159
|
+
prev_blank = true
|
|
160
|
+
next
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
if !in_list && (prev_blank || !code.empty?) && indented_code?(body)
|
|
164
|
+
code << line
|
|
165
|
+
next
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
if (fence = fence_opening(body))
|
|
169
|
+
fence_depth = depth
|
|
170
|
+
code << line
|
|
171
|
+
next
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
out << protect_lines(code, protect) << line
|
|
175
|
+
code = +''
|
|
176
|
+
in_list = in_list_after?(body, in_list, prev_blank)
|
|
177
|
+
prev_blank = body.strip.empty?
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
out << protect_lines(code, protect)
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
# Fence of a line that opens a fenced code block
|
|
184
|
+
#
|
|
185
|
+
# @param line [String] markdown line
|
|
186
|
+
# @return [String, nil] the opening fence, or nil when the line does not open a block
|
|
187
|
+
def fence_opening(line)
|
|
188
|
+
opening = line.match(FENCE_OPENING)
|
|
189
|
+
return unless opening
|
|
190
|
+
return if opening[1].start_with?('`') && opening.post_match.include?('`')
|
|
191
|
+
|
|
192
|
+
opening[1]
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
# Check whether a line is indented enough to be code
|
|
196
|
+
#
|
|
197
|
+
# @param line [String] markdown line
|
|
198
|
+
# @return [Boolean] true for a non-blank line indented by four spaces or a tab
|
|
199
|
+
def indented_code?(line)
|
|
200
|
+
line.match?(/\A(?: {4}|\t)\s*\S/)
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
# Check whether content after a line is still inside a list
|
|
204
|
+
#
|
|
205
|
+
# @param line [String] markdown line
|
|
206
|
+
# @param in_list [Boolean] whether the previous line was inside a list
|
|
207
|
+
# @param prev_blank [Boolean] whether the previous line was blank
|
|
208
|
+
# @return [Boolean] true when the list continues after this line
|
|
209
|
+
def in_list_after?(line, in_list, prev_blank)
|
|
210
|
+
return true if line.match?(LIST_ITEM)
|
|
211
|
+
return in_list if line.strip.empty? || !prev_blank || line.match?(/\A[ \t]/)
|
|
212
|
+
|
|
213
|
+
false
|
|
214
|
+
end
|
|
215
|
+
|
|
216
|
+
# Replace collected code lines with a placeholder, keeping the final line break
|
|
217
|
+
#
|
|
218
|
+
# @param code [String] collected code lines
|
|
219
|
+
# @param protect [Proc] stores a code fragment and returns its placeholder
|
|
220
|
+
# @return [String] placeholder, or an empty string when there is no code
|
|
221
|
+
def protect_lines(code, protect)
|
|
222
|
+
return '' if code.empty?
|
|
223
|
+
|
|
224
|
+
body = code.chomp
|
|
225
|
+
"#{protect.call(body, block: true)}#{code.delete_prefix(body)}"
|
|
226
|
+
end
|
|
35
227
|
|
|
36
|
-
# Expand relative links to absolute URLs
|
|
228
|
+
# Expand relative links and link reference definitions to absolute URLs
|
|
37
229
|
#
|
|
38
230
|
# @param content [String] markdown content
|
|
39
231
|
# @param base_url [String] base URL for expansion
|
|
232
|
+
# @param source_path [String, nil] path of the file relative to the docs root
|
|
233
|
+
# @param convert_urls [Boolean] also convert HTML targets of link reference definitions to markdown, so the
|
|
234
|
+
# definitions are scanned only once
|
|
40
235
|
# @return [String] content with expanded links
|
|
41
|
-
def expand_relative_links(content, base_url)
|
|
42
|
-
content
|
|
43
|
-
|
|
44
|
-
|
|
236
|
+
def expand_relative_links(content, base_url, source_path = nil, convert_urls: false)
|
|
237
|
+
result = expand_links_matching(content, LINK_PATTERN, base_url, source_path)
|
|
238
|
+
|
|
239
|
+
rewrite_reference_definitions(result) do |target|
|
|
240
|
+
expanded = expand_target(target, base_url, source_path)
|
|
241
|
+
convert_urls ? convert_html_target(expanded) : expanded
|
|
242
|
+
end
|
|
243
|
+
end
|
|
244
|
+
|
|
245
|
+
# Rewrite the destinations of link reference definitions
|
|
246
|
+
#
|
|
247
|
+
# @param content [String] markdown content
|
|
248
|
+
# @yieldparam target [String] destination of a definition
|
|
249
|
+
# @yieldreturn [String] new destination
|
|
250
|
+
# @return [String] content with rewritten definitions
|
|
251
|
+
def rewrite_reference_definitions(content)
|
|
252
|
+
return content unless content.include?(']:')
|
|
253
|
+
|
|
254
|
+
rewrite = lambda do |definitions|
|
|
255
|
+
definitions.gsub(REFERENCE_DEFINITION_PATTERN) do
|
|
256
|
+
match = ::Regexp.last_match
|
|
45
257
|
|
|
46
|
-
|
|
47
|
-
match
|
|
48
|
-
else
|
|
49
|
-
clean_url = url.gsub(%r{^\./}, '')
|
|
50
|
-
expanded_url = File.join(base_url, clean_url)
|
|
51
|
-
"[#{text}](#{expanded_url})"
|
|
258
|
+
"#{match[1]}#{yield(match[2])}#{match[3]}"
|
|
52
259
|
end
|
|
53
260
|
end
|
|
261
|
+
|
|
262
|
+
content
|
|
263
|
+
.sub(LEADING_REFERENCE_DEFINITIONS_PATTERN, &rewrite)
|
|
264
|
+
.gsub(FOLLOWING_REFERENCE_DEFINITIONS_PATTERN, &rewrite)
|
|
54
265
|
end
|
|
55
266
|
|
|
56
|
-
#
|
|
267
|
+
# Expand relative links that match a pattern, and the links and images in their link text
|
|
57
268
|
#
|
|
58
|
-
# @param
|
|
59
|
-
# @
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
269
|
+
# @param content [String] markdown content
|
|
270
|
+
# @param pattern [Regexp] pattern of the links to expand
|
|
271
|
+
# @param base_url [String] base URL for expansion
|
|
272
|
+
# @param source_path [String, nil] path of the file relative to the docs root
|
|
273
|
+
# @return [String] content with expanded links
|
|
274
|
+
def expand_links_matching(content, pattern, base_url, source_path)
|
|
275
|
+
content.gsub(pattern) do
|
|
276
|
+
url = ::Regexp.last_match(2)
|
|
277
|
+
text = expand_links_matching(::Regexp.last_match(1), NESTED_LINK_PATTERN, base_url, source_path)
|
|
278
|
+
|
|
279
|
+
"[#{text}](#{expand_target(url, base_url, source_path)})"
|
|
280
|
+
end
|
|
281
|
+
end
|
|
282
|
+
|
|
283
|
+
# Expand a link target unless it is absolute, protocol-relative or an anchor
|
|
284
|
+
#
|
|
285
|
+
# @param target [String] link target, optionally followed by a link title
|
|
286
|
+
# @param base_url [String] base URL for expansion
|
|
287
|
+
# @param source_path [String, nil] path of the file relative to the docs root
|
|
288
|
+
# @return [String] expanded or unchanged target
|
|
289
|
+
def expand_target(target, base_url, source_path)
|
|
290
|
+
destination = target.delete_prefix('<')
|
|
291
|
+
return target if destination.start_with?('//', '#') || destination.match?(URI_SCHEME_PATTERN)
|
|
292
|
+
|
|
293
|
+
resolve_relative_url(base_url, target, source_path)
|
|
294
|
+
end
|
|
295
|
+
|
|
296
|
+
# Resolve a relative link target against the base URL
|
|
297
|
+
#
|
|
298
|
+
# Targets starting with `/` are resolved from the base URL itself, other targets from the source
|
|
299
|
+
# file's directory. `./` and `../` segments are normalized. Titles, queries and anchors are kept.
|
|
300
|
+
# Angle-bracket destinations (`<path with spaces>`) are unwrapped and their spaces percent-encoded.
|
|
301
|
+
# A query-only target (`?tab=2`) resolves to the source file itself, as in RFC 3986, with its
|
|
302
|
+
# name percent-encoded.
|
|
303
|
+
#
|
|
304
|
+
# @param base_url [String] base URL for expansion
|
|
305
|
+
# @param target [String] relative link target, optionally followed by a link title
|
|
306
|
+
# @param source_path [String, nil] path of the source file relative to the docs root
|
|
307
|
+
# @return [String] expanded URL
|
|
308
|
+
def resolve_relative_url(base_url, target, source_path)
|
|
309
|
+
url, title = target.match(/\A(.*?)(\s+(?:"[^"]*"|'[^']*'))?\z/m).captures
|
|
310
|
+
url = url[/\A<(.*)>\z/m, 1]&.gsub(' ', '%20') || url
|
|
311
|
+
path, suffix = url.match(/\A([^?#]*)(.*)\z/m).captures
|
|
312
|
+
query_only = source_path && path.empty? && suffix.start_with?('?')
|
|
313
|
+
path = ERB::Util.url_encode(File.basename(source_path)) if query_only
|
|
314
|
+
source_dir = source_path ? File.dirname(source_path) : '.'
|
|
315
|
+
origin = base_url[%r{\A[^:/?#]+://[^/?#]*}].to_s
|
|
316
|
+
base_path = base_url.delete_prefix(origin)
|
|
317
|
+
dir = path.start_with?('/') ? base_path : File.join(base_path, source_dir)
|
|
318
|
+
|
|
319
|
+
"#{origin}#{normalize_path("#{dir}/#{path}")}#{suffix}#{title}"
|
|
320
|
+
end
|
|
321
|
+
|
|
322
|
+
# Normalize `.`, `..` and empty segments of a path
|
|
323
|
+
#
|
|
324
|
+
# @param path [String] path to normalize
|
|
325
|
+
# @return [String] normalized path, with its trailing slash kept
|
|
326
|
+
def normalize_path(path)
|
|
327
|
+
normalized = Pathname.new(path).cleanpath.to_s
|
|
328
|
+
normalized += '/' if path.match?(%r{/\.{0,2}\z}) && !normalized.end_with?('/')
|
|
329
|
+
normalized
|
|
63
330
|
end
|
|
64
331
|
|
|
65
332
|
# Convert HTML URLs to markdown format
|
|
66
333
|
#
|
|
334
|
+
# Converts absolute URLs anywhere in the content and relative link and link reference definition targets,
|
|
335
|
+
# keeping queries and anchors.
|
|
336
|
+
#
|
|
67
337
|
# @param content [String] markdown content
|
|
338
|
+
# @param convert_definitions [Boolean] whether to convert link reference definition targets
|
|
68
339
|
# @return [String] content with converted URLs
|
|
69
|
-
def convert_html_urls(content)
|
|
70
|
-
content.gsub(%r{https?://[^\s
|
|
71
|
-
url.sub(/\.html
|
|
340
|
+
def convert_html_urls(content, convert_definitions: true)
|
|
341
|
+
result = content.gsub(%r{https?://[^\s<>?#]+?\.html?(?=[?#)\s]|$)}) do |url|
|
|
342
|
+
url.sub(/\.html?\z/, '.md')
|
|
343
|
+
end
|
|
344
|
+
|
|
345
|
+
result = result.gsub(/\]\(([^)\s?#]+?)\.html?(?=[?#)\s])/) do |match|
|
|
346
|
+
path = ::Regexp.last_match(1)
|
|
347
|
+
path.match?(URI_SCHEME_PATTERN) ? match : "](#{path}.md"
|
|
72
348
|
end
|
|
349
|
+
|
|
350
|
+
return result unless convert_definitions
|
|
351
|
+
|
|
352
|
+
rewrite_reference_definitions(result) { |target| convert_html_target(target) }
|
|
353
|
+
end
|
|
354
|
+
|
|
355
|
+
# Convert a relative HTML link reference definition target to markdown
|
|
356
|
+
#
|
|
357
|
+
# @param target [String] destination of a definition
|
|
358
|
+
# @return [String] converted or unchanged target
|
|
359
|
+
def convert_html_target(target)
|
|
360
|
+
target.match?(URI_SCHEME_PATTERN) ? target : target.sub(/\A([^?#]+?)\.html?(?=[?#]|\z)/, '\1.md')
|
|
73
361
|
end
|
|
74
362
|
|
|
75
363
|
# Simplify verbose link text
|
|
@@ -77,14 +365,14 @@ module LlmDocsBuilder
|
|
|
77
365
|
# @param content [String] markdown content
|
|
78
366
|
# @return [String] content with simplified links
|
|
79
367
|
def simplify_links(content)
|
|
80
|
-
content.gsub(
|
|
368
|
+
content.gsub(LINK_PATTERN) do
|
|
81
369
|
text = ::Regexp.last_match(1)
|
|
82
370
|
url = ::Regexp.last_match(2)
|
|
83
371
|
|
|
84
372
|
simplified_text = text
|
|
85
|
-
.gsub(/^(click here to|see|read more about|check out|visit)\s+(the\s+)
|
|
373
|
+
.gsub(/^(?:(?:click here to|see|read more about|check out|visit)\s+(?:the\s+)?)+/i, '')
|
|
86
374
|
.gsub(/\s+(here|documentation|docs)$/i, '')
|
|
87
|
-
.strip
|
|
375
|
+
.gsub(/\A\s+|\s+\z/, '') # String#strip also removes the NUL of code placeholders
|
|
88
376
|
|
|
89
377
|
simplified_text = text if simplified_text.empty?
|
|
90
378
|
|
data/lib/llm_docs_builder.rb
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
# Example
|
|
2
|
-
# Copy to
|
|
1
|
+
# Example llm-docs-builder.yml configuration file
|
|
2
|
+
# Copy to llm-docs-builder.yml and customize for your project
|
|
3
3
|
|
|
4
4
|
# Path to documentation directory or file
|
|
5
5
|
docs: ./docs
|
data/package-lock.json
CHANGED
|
@@ -103,9 +103,9 @@
|
|
|
103
103
|
}
|
|
104
104
|
},
|
|
105
105
|
"node_modules/fastq": {
|
|
106
|
-
"version": "1.20.
|
|
107
|
-
"resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.
|
|
108
|
-
"integrity": "sha512-
|
|
106
|
+
"version": "1.20.3",
|
|
107
|
+
"resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.3.tgz",
|
|
108
|
+
"integrity": "sha512-XKv5nnLs6nLF71NgiKJLIZFLkPyIEuOselLG7ujZnGrRfQK8HpvY+WqKhAJUAdLomwVHErVS4LfxFlPq0/FTAw==",
|
|
109
109
|
"dev": true,
|
|
110
110
|
"license": "ISC",
|
|
111
111
|
"dependencies": {
|
|
@@ -286,9 +286,9 @@
|
|
|
286
286
|
}
|
|
287
287
|
},
|
|
288
288
|
"node_modules/smol-toml": {
|
|
289
|
-
"version": "1.
|
|
290
|
-
"resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.
|
|
291
|
-
"integrity": "sha512-
|
|
289
|
+
"version": "1.9.0",
|
|
290
|
+
"resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.9.0.tgz",
|
|
291
|
+
"integrity": "sha512-hpd+HLON7HdZXqYchMM/+LaTTbdK0AU3NngIJ4KVyWbY9bfQqdL9cD+4yf6dUoU2Ap4VsU0JkQi6FxAI1B2mXQ==",
|
|
292
292
|
"dev": true,
|
|
293
293
|
"license": "BSD-3-Clause",
|
|
294
294
|
"engines": {
|
|
@@ -312,9 +312,9 @@
|
|
|
312
312
|
}
|
|
313
313
|
},
|
|
314
314
|
"node_modules/yaml": {
|
|
315
|
-
"version": "2.9.
|
|
316
|
-
"resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.
|
|
317
|
-
"integrity": "sha512-
|
|
315
|
+
"version": "2.9.1",
|
|
316
|
+
"resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.1.tgz",
|
|
317
|
+
"integrity": "sha512-3NxN8+78OdzbT7C/WjGsyfPAtJaN3FNDsWxv7Y7mcDsT/oOmgW8BpyQQFFBnvZE3j9Y2Sdz1ULFLezL7Eb2yFw==",
|
|
318
318
|
"dev": true,
|
|
319
319
|
"license": "ISC",
|
|
320
320
|
"bin": {
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: llm-docs-builder
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.
|
|
4
|
+
version: 1.1.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Maciej Mensfeld
|
|
@@ -201,7 +201,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
201
201
|
- !ruby/object:Gem::Version
|
|
202
202
|
version: '0'
|
|
203
203
|
requirements: []
|
|
204
|
-
rubygems_version: 4.0.
|
|
204
|
+
rubygems_version: 4.0.20
|
|
205
205
|
specification_version: 4
|
|
206
206
|
summary: Build and optimize documentation for LLMs - generate llms.txt, transform
|
|
207
207
|
markdown, and more
|