llm-docs-builder 0.13.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +7 -7
- data/.github/workflows/docker.yml +7 -7
- data/.github/workflows/push.yml +3 -3
- data/.ruby-version +1 -1
- data/.yard-lint.yml +35 -1
- data/CHANGELOG.md +8 -0
- data/Gemfile +4 -0
- data/Gemfile.lock +17 -21
- data/lib/llm_docs_builder/cli.rb +1 -2
- data/lib/llm_docs_builder/generator.rb +1 -2
- data/lib/llm_docs_builder/html_detector.rb +1 -2
- data/lib/llm_docs_builder/html_to_markdown/figure_code_block_renderer.rb +2 -3
- data/lib/llm_docs_builder/html_to_markdown_converter.rb +3 -5
- data/lib/llm_docs_builder/output_formatter.rb +1 -2
- data/lib/llm_docs_builder/parser.rb +3 -5
- data/lib/llm_docs_builder/text_compressor.rb +2 -3
- data/lib/llm_docs_builder/token_estimator.rb +2 -3
- data/lib/llm_docs_builder/transformers/base_transformer.rb +2 -3
- data/lib/llm_docs_builder/transformers/heading_transformer.rb +2 -3
- data/lib/llm_docs_builder/transformers/link_transformer.rb +15 -3
- data/lib/llm_docs_builder/transformers/whitespace_transformer.rb +1 -2
- data/lib/llm_docs_builder/validator.rb +1 -2
- data/lib/llm_docs_builder/version.rb +1 -1
- data/lib/llm_docs_builder.rb +2 -3
- data/llm-docs-builder.gemspec +2 -1
- data/package-lock.json +6 -6
- data/renovate.json +13 -2
- metadata +5 -4
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: a1d023947b940a99bfa254d72ff79e47eef15f378eeb2e46ca2f04859b5da6dd
|
|
4
|
+
data.tar.gz: 85448c820f8fce92e9a26c094a9dd8cd485619ac45eb72ad45ed5b058ff2dc69
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 9b345c53110799db9429b5185c1f773ae7ed99bdfa4da73f038cacad8cd319aa378f483f928a76d7f73533da08201f930f74b98e28a0246042e85d5269dc0d24
|
|
7
|
+
data.tar.gz: 6f876f2ec8b9e41a8daf1626b87e0cfb610fe164ff47231587ab3cbb78d2ea1ed699d06a24ea2927426ea7a227996828fbdc82c3a4abbc3e37883bee3277923d
|
data/.github/workflows/ci.yml
CHANGED
|
@@ -30,12 +30,12 @@ jobs:
|
|
|
30
30
|
- ruby: '4.0'
|
|
31
31
|
coverage: 'true'
|
|
32
32
|
steps:
|
|
33
|
-
- uses: actions/checkout@
|
|
33
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
34
34
|
with:
|
|
35
35
|
fetch-depth: 0
|
|
36
36
|
|
|
37
37
|
- name: Set up Ruby
|
|
38
|
-
uses: ruby/setup-ruby@
|
|
38
|
+
uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0
|
|
39
39
|
with:
|
|
40
40
|
ruby-version: ${{ matrix.ruby }}
|
|
41
41
|
bundler-cache: true
|
|
@@ -58,11 +58,11 @@ jobs:
|
|
|
58
58
|
timeout-minutes: 5
|
|
59
59
|
runs-on: ubuntu-latest
|
|
60
60
|
steps:
|
|
61
|
-
- uses: actions/checkout@
|
|
61
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
62
62
|
- name: Set up Ruby
|
|
63
|
-
uses: ruby/setup-ruby@
|
|
63
|
+
uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0
|
|
64
64
|
with:
|
|
65
|
-
ruby-version: '4.0.
|
|
65
|
+
ruby-version: '4.0.6'
|
|
66
66
|
bundler-cache: true
|
|
67
67
|
- name: Run yard-lint
|
|
68
68
|
run: bundle exec yard-lint lib/
|
|
@@ -71,11 +71,11 @@ jobs:
|
|
|
71
71
|
timeout-minutes: 5
|
|
72
72
|
runs-on: ubuntu-latest
|
|
73
73
|
steps:
|
|
74
|
-
- uses: actions/checkout@
|
|
74
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
75
75
|
with:
|
|
76
76
|
fetch-depth: 0
|
|
77
77
|
- name: Set up Node.js
|
|
78
|
-
uses: actions/setup-node@
|
|
78
|
+
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
|
79
79
|
with:
|
|
80
80
|
node-version: '20'
|
|
81
81
|
cache: 'npm'
|
|
@@ -25,13 +25,13 @@ jobs:
|
|
|
25
25
|
id-token: write
|
|
26
26
|
steps:
|
|
27
27
|
- name: Checkout
|
|
28
|
-
uses: actions/checkout@
|
|
28
|
+
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
29
29
|
with:
|
|
30
30
|
fetch-depth: 0
|
|
31
31
|
|
|
32
32
|
- name: Docker meta
|
|
33
33
|
id: meta
|
|
34
|
-
uses: docker/metadata-action@
|
|
34
|
+
uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6
|
|
35
35
|
with:
|
|
36
36
|
images: |
|
|
37
37
|
mensfeld/llm-docs-builder
|
|
@@ -45,28 +45,28 @@ jobs:
|
|
|
45
45
|
type=raw,value=latest,enable={{is_default_branch}}
|
|
46
46
|
|
|
47
47
|
- name: Set up QEMU
|
|
48
|
-
uses: docker/setup-qemu-action@
|
|
48
|
+
uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4
|
|
49
49
|
|
|
50
50
|
- name: Set up Docker Buildx
|
|
51
|
-
uses: docker/setup-buildx-action@
|
|
51
|
+
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4
|
|
52
52
|
|
|
53
53
|
- name: Login to Docker Hub
|
|
54
54
|
if: github.event_name != 'pull_request'
|
|
55
|
-
uses: docker/login-action@
|
|
55
|
+
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
|
|
56
56
|
with:
|
|
57
57
|
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
|
58
58
|
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
|
59
59
|
|
|
60
60
|
- name: Login to GitHub Container Registry
|
|
61
61
|
if: github.event_name != 'pull_request'
|
|
62
|
-
uses: docker/login-action@
|
|
62
|
+
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
|
|
63
63
|
with:
|
|
64
64
|
registry: ghcr.io
|
|
65
65
|
username: ${{ github.actor }}
|
|
66
66
|
password: ${{ secrets.GITHUB_TOKEN }}
|
|
67
67
|
|
|
68
68
|
- name: Build and push
|
|
69
|
-
uses: docker/build-push-action@
|
|
69
|
+
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7
|
|
70
70
|
with:
|
|
71
71
|
context: .
|
|
72
72
|
platforms: linux/amd64,linux/arm64
|
data/.github/workflows/push.yml
CHANGED
|
@@ -19,12 +19,12 @@ jobs:
|
|
|
19
19
|
id-token: write
|
|
20
20
|
|
|
21
21
|
steps:
|
|
22
|
-
- uses: actions/checkout@
|
|
22
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
|
23
23
|
with:
|
|
24
24
|
fetch-depth: 0
|
|
25
25
|
|
|
26
26
|
- name: Set up Ruby
|
|
27
|
-
uses: ruby/setup-ruby@
|
|
27
|
+
uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0
|
|
28
28
|
with:
|
|
29
29
|
bundler-cache: false
|
|
30
30
|
|
|
@@ -32,4 +32,4 @@ jobs:
|
|
|
32
32
|
run: |
|
|
33
33
|
bundle install --jobs 4 --retry 3
|
|
34
34
|
|
|
35
|
-
- uses: rubygems/release-gem@
|
|
35
|
+
- uses: rubygems/release-gem@052cc82692552de3ef2b81fd670e41d13cba8092 # v1.4.0
|
data/.ruby-version
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
4.0.
|
|
1
|
+
4.0.6
|
data/.yard-lint.yml
CHANGED
|
@@ -62,6 +62,34 @@ Documentation/BlankLineBeforeDefinition:
|
|
|
62
62
|
SingleBlankLine: true
|
|
63
63
|
OrphanedDocs: true
|
|
64
64
|
|
|
65
|
+
Documentation/DuplicateNamespaceComment:
|
|
66
|
+
Description: Detects namespaces documented with a YARD comment in more than one file.
|
|
67
|
+
Enabled: true
|
|
68
|
+
Severity: error
|
|
69
|
+
|
|
70
|
+
Documentation/UnderfilledLines:
|
|
71
|
+
Description: Detects documentation prose that wraps too early and wastes horizontal space.
|
|
72
|
+
Enabled: true
|
|
73
|
+
Severity: error
|
|
74
|
+
# Aligned with RuboCop's Layout/LineLength so documentation prose wraps to the same
|
|
75
|
+
# width as code.
|
|
76
|
+
MaxLength: 120
|
|
77
|
+
|
|
78
|
+
Documentation/LineLength:
|
|
79
|
+
Description: Detects documentation lines that exceed the maximum length.
|
|
80
|
+
Enabled: true
|
|
81
|
+
Severity: error
|
|
82
|
+
# Aligned with RuboCop's Layout/LineLength.
|
|
83
|
+
MaxLength: 120
|
|
84
|
+
|
|
85
|
+
Documentation/TextSubstitution:
|
|
86
|
+
Description: Detects em/en-dashes in documentation and replaces them with hyphens.
|
|
87
|
+
Enabled: true
|
|
88
|
+
Severity: error
|
|
89
|
+
Substitutions:
|
|
90
|
+
"—": "-" # em-dash (U+2014)
|
|
91
|
+
"–": "-" # en-dash (U+2013)
|
|
92
|
+
|
|
65
93
|
# Tags validators
|
|
66
94
|
Tags/Order:
|
|
67
95
|
Description: Enforces consistent ordering of YARD tags.
|
|
@@ -70,9 +98,15 @@ Tags/Order:
|
|
|
70
98
|
EnforcedOrder:
|
|
71
99
|
- param
|
|
72
100
|
- option
|
|
101
|
+
- yield
|
|
102
|
+
- yieldparam
|
|
103
|
+
- yieldreturn
|
|
73
104
|
- return
|
|
74
105
|
- raise
|
|
106
|
+
- see
|
|
75
107
|
- example
|
|
108
|
+
- note
|
|
109
|
+
- todo
|
|
76
110
|
|
|
77
111
|
Tags/InvalidTypes:
|
|
78
112
|
Description: Validates type definitions in @param, @return, @option tags.
|
|
@@ -142,7 +176,7 @@ Tags/OptionTags:
|
|
|
142
176
|
Tags/ExampleSyntax:
|
|
143
177
|
Description: Validates Ruby syntax in @example tags.
|
|
144
178
|
Enabled: true
|
|
145
|
-
Severity:
|
|
179
|
+
Severity: error
|
|
146
180
|
|
|
147
181
|
Tags/RedundantParamDescription:
|
|
148
182
|
Description: Detects meaningless parameter descriptions that add no value.
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,13 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [Unreleased]
|
|
4
|
+
|
|
5
|
+
## 1.0.0 (2026-08-24)
|
|
6
|
+
- [Fix] **Link Expansion Corrupts Non-HTTP URI Schemes** — `LinkTransformer#expand_relative_links` no longer rewrites `mailto:`, `tel:`, and `ftp:` links into dead `https://...` URLs when `base_url` is set (#165).
|
|
7
|
+
- Any link whose scheme is in the known absolute-scheme list (`http`, `https`, `mailto`, `ftp`, `tel`) is now left untouched, matching `HtmlToMarkdownConverter`'s `SAFE_URI_SCHEMES`.
|
|
8
|
+
- Relative links (e.g. `./guide.md`) continue to expand as before.
|
|
9
|
+
- [Enhancement] Add Ruby warning category opt-in to test helpers
|
|
10
|
+
|
|
3
11
|
## 0.13.0 (2026-05-11)
|
|
4
12
|
- [Breaking] Dropped Ruby 3.2 support. Minimum required Ruby version is now 3.3.
|
|
5
13
|
- [Fix] **Heading Nesting in Normalize Mode** — Fixed `normalize_headings` producing incorrect heading levels when same-level headings are nested under a parent.
|
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
PATH
|
|
2
2
|
remote: .
|
|
3
3
|
specs:
|
|
4
|
-
llm-docs-builder (0.
|
|
4
|
+
llm-docs-builder (1.0.0)
|
|
5
5
|
nokogiri (~> 1.17)
|
|
6
6
|
zeitwerk (~> 2.6)
|
|
7
7
|
|
|
@@ -11,18 +11,17 @@ GEM
|
|
|
11
11
|
ast (2.4.3)
|
|
12
12
|
coderay (1.1.3)
|
|
13
13
|
diff-lcs (1.6.2)
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
language_server-protocol (3.17.0.5)
|
|
14
|
+
io-console (0.9.2)
|
|
15
|
+
json (2.21.2)
|
|
16
|
+
language_server-protocol (3.17.0.6)
|
|
18
17
|
lint_roller (1.1.0)
|
|
19
18
|
method_source (1.1.0)
|
|
20
19
|
mini_portile2 (2.8.9)
|
|
21
|
-
nokogiri (1.19.
|
|
20
|
+
nokogiri (1.19.4)
|
|
22
21
|
mini_portile2 (~> 2.8.2)
|
|
23
22
|
racc (~> 1.4)
|
|
24
23
|
parallel (2.1.0)
|
|
25
|
-
parser (3.3.
|
|
24
|
+
parser (3.3.12.0)
|
|
26
25
|
ast (~> 2.4.1)
|
|
27
26
|
racc
|
|
28
27
|
prism (1.9.0)
|
|
@@ -34,7 +33,7 @@ GEM
|
|
|
34
33
|
rainbow (3.1.1)
|
|
35
34
|
rake (13.4.2)
|
|
36
35
|
regexp_parser (2.12.0)
|
|
37
|
-
reline (0.
|
|
36
|
+
reline (0.7.0)
|
|
38
37
|
io-console (~> 0.5)
|
|
39
38
|
rspec (3.13.2)
|
|
40
39
|
rspec-core (~> 3.13.0)
|
|
@@ -49,7 +48,7 @@ GEM
|
|
|
49
48
|
diff-lcs (>= 1.2.0, < 2.0)
|
|
50
49
|
rspec-support (~> 3.13.0)
|
|
51
50
|
rspec-support (3.13.7)
|
|
52
|
-
rubocop (1.
|
|
51
|
+
rubocop (1.89.0)
|
|
53
52
|
json (~> 2.3)
|
|
54
53
|
language_server-protocol (~> 3.17.0.2)
|
|
55
54
|
lint_roller (~> 1.1.0)
|
|
@@ -60,24 +59,20 @@ GEM
|
|
|
60
59
|
rubocop-ast (>= 1.49.0, < 2.0)
|
|
61
60
|
ruby-progressbar (~> 1.7)
|
|
62
61
|
unicode-display_width (>= 2.4.0, < 4.0)
|
|
63
|
-
rubocop-ast (1.
|
|
62
|
+
rubocop-ast (1.50.0)
|
|
64
63
|
parser (>= 3.3.7.2)
|
|
65
64
|
prism (~> 1.7)
|
|
66
65
|
ruby-progressbar (1.13.0)
|
|
67
|
-
simplecov (
|
|
68
|
-
docile (~> 1.1)
|
|
69
|
-
simplecov-html (~> 0.11)
|
|
70
|
-
simplecov_json_formatter (~> 0.1)
|
|
71
|
-
simplecov-html (0.13.2)
|
|
72
|
-
simplecov_json_formatter (0.1.4)
|
|
66
|
+
simplecov (1.1.1)
|
|
73
67
|
unicode-display_width (3.2.0)
|
|
74
68
|
unicode-emoji (~> 4.1)
|
|
75
69
|
unicode-emoji (4.2.0)
|
|
76
|
-
|
|
77
|
-
yard
|
|
70
|
+
warning (1.6.0)
|
|
71
|
+
yard (0.9.45)
|
|
72
|
+
yard-lint (1.11.0)
|
|
78
73
|
yard (~> 0.9)
|
|
79
74
|
zeitwerk (~> 2.6)
|
|
80
|
-
zeitwerk (2.
|
|
75
|
+
zeitwerk (2.8.3)
|
|
81
76
|
|
|
82
77
|
PLATFORMS
|
|
83
78
|
ruby
|
|
@@ -89,8 +84,9 @@ DEPENDENCIES
|
|
|
89
84
|
rake (~> 13.0)
|
|
90
85
|
rspec (~> 3.0)
|
|
91
86
|
rubocop (~> 1.0)
|
|
92
|
-
simplecov (~> 0
|
|
87
|
+
simplecov (~> 1.0)
|
|
88
|
+
warning
|
|
93
89
|
yard-lint
|
|
94
90
|
|
|
95
91
|
BUNDLED WITH
|
|
96
|
-
|
|
92
|
+
4.0.17
|
data/lib/llm_docs_builder/cli.rb
CHANGED
|
@@ -129,8 +129,7 @@ module LlmDocsBuilder
|
|
|
129
129
|
|
|
130
130
|
# Generate llms.txt from documentation directory or file
|
|
131
131
|
#
|
|
132
|
-
# Loads configuration, merges with CLI options, generates llms.txt content,
|
|
133
|
-
# and optionally validates the output.
|
|
132
|
+
# Loads configuration, merges with CLI options, generates llms.txt content, and optionally validates the output.
|
|
134
133
|
#
|
|
135
134
|
# @param options [Hash] command options from parse_options
|
|
136
135
|
# @option options [String] :config path to config file
|
|
@@ -34,8 +34,7 @@ module LlmDocsBuilder
|
|
|
34
34
|
|
|
35
35
|
# Generate llms.txt content from documentation
|
|
36
36
|
#
|
|
37
|
-
# Scans documentation files, extracts metadata, prioritizes them, and builds a formatted
|
|
38
|
-
# llms.txt file.
|
|
37
|
+
# Scans documentation files, extracts metadata, prioritizes them, and builds a formatted llms.txt file.
|
|
39
38
|
#
|
|
40
39
|
# @return [String] generated llms.txt content
|
|
41
40
|
def generate
|
|
@@ -14,8 +14,7 @@ module LlmDocsBuilder
|
|
|
14
14
|
full_html_document?(content)
|
|
15
15
|
end
|
|
16
16
|
|
|
17
|
-
# Prepare a snippet of content for HTML detection by removing leading whitespace
|
|
18
|
-
# and build metadata comments.
|
|
17
|
+
# Prepare a snippet of content for HTML detection by removing leading whitespace and build metadata comments.
|
|
19
18
|
#
|
|
20
19
|
# @param content [String]
|
|
21
20
|
# @return [String, nil]
|
|
@@ -3,9 +3,8 @@
|
|
|
3
3
|
module LlmDocsBuilder
|
|
4
4
|
# Provides HTML to Markdown conversion functionality
|
|
5
5
|
#
|
|
6
|
-
# This module contains specialized renderers for converting HTML elements
|
|
7
|
-
#
|
|
8
|
-
# figures, and syntax-highlighted code blocks.
|
|
6
|
+
# This module contains specialized renderers for converting HTML elements to Markdown format, with support for complex
|
|
7
|
+
# structures like tables, figures, and syntax-highlighted code blocks.
|
|
9
8
|
#
|
|
10
9
|
# @api private
|
|
11
10
|
module HtmlToMarkdown
|
|
@@ -644,11 +644,9 @@ module LlmDocsBuilder
|
|
|
644
644
|
|
|
645
645
|
# Compute effective heading level adjusted for section nesting
|
|
646
646
|
#
|
|
647
|
-
# When HTML uses nested <section> elements with same-level headings,
|
|
648
|
-
# the
|
|
649
|
-
#
|
|
650
|
-
# count and the expected count for that heading tag (h1 expects 0
|
|
651
|
-
# sections, h2 expects 1, etc.), capped at heading level 6.
|
|
647
|
+
# When HTML uses nested <section> elements with same-level headings, the inner headings should receive deeper
|
|
648
|
+
# markdown levels. The offset is calculated as the difference between the actual section ancestor count and the
|
|
649
|
+
# expected count for that heading tag (h1 expects 0 sections, h2 expects 1, etc.), capped at heading level 6.
|
|
652
650
|
#
|
|
653
651
|
# @param element [Nokogiri::XML::Element] heading element
|
|
654
652
|
# @param base_level [Integer] HTML heading level (1-6)
|
|
@@ -3,8 +3,7 @@
|
|
|
3
3
|
module LlmDocsBuilder
|
|
4
4
|
# Formats output for CLI display
|
|
5
5
|
#
|
|
6
|
-
# Provides formatting utilities for displaying comparison results,
|
|
7
|
-
# byte sizes, and numbers in a user-friendly way.
|
|
6
|
+
# Provides formatting utilities for displaying comparison results, byte sizes, and numbers in a user-friendly way.
|
|
8
7
|
#
|
|
9
8
|
# @api private
|
|
10
9
|
class OutputFormatter
|
|
@@ -3,9 +3,8 @@
|
|
|
3
3
|
module LlmDocsBuilder
|
|
4
4
|
# Parses llms.txt files into structured data
|
|
5
5
|
#
|
|
6
|
-
# Reads and parses llms.txt files according to the llms.txt specification,
|
|
7
|
-
#
|
|
8
|
-
# Examples, Optional) with their links.
|
|
6
|
+
# Reads and parses llms.txt files according to the llms.txt specification, extracting the title, description, and
|
|
7
|
+
# structured sections (Documentation, Examples, Optional) with their links.
|
|
9
8
|
#
|
|
10
9
|
# @example Parse an llms.txt file
|
|
11
10
|
# parser = LlmDocsBuilder::Parser.new('llms.txt')
|
|
@@ -109,8 +108,7 @@ module LlmDocsBuilder
|
|
|
109
108
|
|
|
110
109
|
# Represents parsed llms.txt content with structured access to sections
|
|
111
110
|
#
|
|
112
|
-
# Provides convenient access to parsed llms.txt sections including title,
|
|
113
|
-
# description, and link collections.
|
|
111
|
+
# Provides convenient access to parsed llms.txt sections including title, description, and link collections.
|
|
114
112
|
#
|
|
115
113
|
# @example Access parsed content
|
|
116
114
|
# parsed.title # => "My Project"
|
|
@@ -3,9 +3,8 @@
|
|
|
3
3
|
module LlmDocsBuilder
|
|
4
4
|
# Advanced text compression techniques for reducing token count
|
|
5
5
|
#
|
|
6
|
-
# Provides more aggressive text compression methods including stopword removal,
|
|
7
|
-
#
|
|
8
|
-
# aggressive than basic markdown cleanup and should be used carefully.
|
|
6
|
+
# Provides more aggressive text compression methods including stopword removal, duplicate content detection, and
|
|
7
|
+
# sentence deduplication. These methods are more aggressive than basic markdown cleanup and should be used carefully.
|
|
9
8
|
#
|
|
10
9
|
# @example Basic usage
|
|
11
10
|
# compressor = LlmDocsBuilder::TextCompressor.new
|
|
@@ -3,9 +3,8 @@
|
|
|
3
3
|
module LlmDocsBuilder
|
|
4
4
|
# Estimates token count for text content using character-based approximation
|
|
5
5
|
#
|
|
6
|
-
# Provides token estimation without requiring external tokenizer dependencies.
|
|
7
|
-
#
|
|
8
|
-
# which works reasonably well for documentation and markdown content.
|
|
6
|
+
# Provides token estimation without requiring external tokenizer dependencies. Uses the common heuristic that ~4
|
|
7
|
+
# characters equals 1 token for English text, which works reasonably well for documentation and markdown content.
|
|
9
8
|
#
|
|
10
9
|
# @example Basic usage
|
|
11
10
|
# estimator = LlmDocsBuilder::TokenEstimator.new
|
|
@@ -3,9 +3,8 @@
|
|
|
3
3
|
module LlmDocsBuilder
|
|
4
4
|
# Provides content transformation functionality
|
|
5
5
|
#
|
|
6
|
-
# This module contains specialized transformers for modifying markdown content,
|
|
7
|
-
#
|
|
8
|
-
# content enhancement for AI consumption.
|
|
6
|
+
# This module contains specialized transformers for modifying markdown content, including cleanup operations, link
|
|
7
|
+
# processing, heading normalization, and content enhancement for AI consumption.
|
|
9
8
|
#
|
|
10
9
|
# @api private
|
|
11
10
|
module Transformers
|
|
@@ -4,9 +4,8 @@ module LlmDocsBuilder
|
|
|
4
4
|
module Transformers
|
|
5
5
|
# Normalizes headings to include hierarchical context
|
|
6
6
|
#
|
|
7
|
-
# Transforms markdown headings to include parent context, making each section
|
|
8
|
-
#
|
|
9
|
-
# are chunked and retrieved independently.
|
|
7
|
+
# Transforms markdown headings to include parent context, making each section self-contained for RAG systems. This
|
|
8
|
+
# is particularly useful when documents are chunked and retrieved independently.
|
|
10
9
|
#
|
|
11
10
|
# @example Basic heading normalization
|
|
12
11
|
# # Configuration
|
|
@@ -4,8 +4,7 @@ module LlmDocsBuilder
|
|
|
4
4
|
module Transformers
|
|
5
5
|
# Transformer for link-related operations
|
|
6
6
|
#
|
|
7
|
-
# Handles expansion of relative links to absolute URLs and
|
|
8
|
-
# conversion of HTML URLs to markdown format.
|
|
7
|
+
# Handles expansion of relative links to absolute URLs and conversion of HTML URLs to markdown format.
|
|
9
8
|
#
|
|
10
9
|
# @api public
|
|
11
10
|
class LinkTransformer
|
|
@@ -30,6 +29,10 @@ module LlmDocsBuilder
|
|
|
30
29
|
|
|
31
30
|
private
|
|
32
31
|
|
|
32
|
+
# URI schemes that must never be treated as relative paths when expanding
|
|
33
|
+
# links with a base URL (kept in sync with the converter's safe schemes)
|
|
34
|
+
ABSOLUTE_URL_SCHEMES = %w[http https mailto ftp tel].freeze
|
|
35
|
+
|
|
33
36
|
# Expand relative links to absolute URLs
|
|
34
37
|
#
|
|
35
38
|
# @param content [String] markdown content
|
|
@@ -40,7 +43,7 @@ module LlmDocsBuilder
|
|
|
40
43
|
text = ::Regexp.last_match(1)
|
|
41
44
|
url = ::Regexp.last_match(2)
|
|
42
45
|
|
|
43
|
-
if url.start_with?('http://', 'https://', '//', '#')
|
|
46
|
+
if url.start_with?('http://', 'https://', '//', '#') || absolute_url_scheme?(url)
|
|
44
47
|
match
|
|
45
48
|
else
|
|
46
49
|
clean_url = url.gsub(%r{^\./}, '')
|
|
@@ -50,6 +53,15 @@ module LlmDocsBuilder
|
|
|
50
53
|
end
|
|
51
54
|
end
|
|
52
55
|
|
|
56
|
+
# Check whether a link target uses a known absolute URI scheme
|
|
57
|
+
#
|
|
58
|
+
# @param url [String] link target
|
|
59
|
+
# @return [Boolean] true when the target already carries its own scheme
|
|
60
|
+
def absolute_url_scheme?(url)
|
|
61
|
+
scheme = url.split(':', 2).first.to_s
|
|
62
|
+
ABSOLUTE_URL_SCHEMES.include?(scheme)
|
|
63
|
+
end
|
|
64
|
+
|
|
53
65
|
# Convert HTML URLs to markdown format
|
|
54
66
|
#
|
|
55
67
|
# @param content [String] markdown content
|
|
@@ -4,8 +4,7 @@ module LlmDocsBuilder
|
|
|
4
4
|
module Transformers
|
|
5
5
|
# Transformer for whitespace normalization
|
|
6
6
|
#
|
|
7
|
-
# Reduces excessive blank lines and trailing whitespace to make
|
|
8
|
-
# content more compact for LLM consumption.
|
|
7
|
+
# Reduces excessive blank lines and trailing whitespace to make content more compact for LLM consumption.
|
|
9
8
|
#
|
|
10
9
|
# @api public
|
|
11
10
|
class WhitespaceTransformer
|
|
@@ -56,8 +56,7 @@ module LlmDocsBuilder
|
|
|
56
56
|
|
|
57
57
|
# Validate content and return result
|
|
58
58
|
#
|
|
59
|
-
# Runs all validation checks, populates {#errors} array, and returns whether
|
|
60
|
-
# the content is valid.
|
|
59
|
+
# Runs all validation checks, populates {#errors} array, and returns whether the content is valid.
|
|
61
60
|
#
|
|
62
61
|
# @return [Boolean] true if content is valid, false otherwise
|
|
63
62
|
def validate!
|
data/lib/llm_docs_builder.rb
CHANGED
|
@@ -13,9 +13,8 @@ loader.setup
|
|
|
13
13
|
|
|
14
14
|
# Build and optimize documentation for LLMs
|
|
15
15
|
#
|
|
16
|
-
# This gem provides tools for generating llms.txt files and transforming markdown
|
|
17
|
-
#
|
|
18
|
-
# preserving essential documentation content.
|
|
16
|
+
# This gem provides tools for generating llms.txt files and transforming markdown documentation to be AI-friendly. It
|
|
17
|
+
# can reduce token consumption by 67-95% while preserving essential documentation content.
|
|
19
18
|
#
|
|
20
19
|
# @api public
|
|
21
20
|
module LlmDocsBuilder
|
data/llm-docs-builder.gemspec
CHANGED
|
@@ -27,6 +27,7 @@ Gem::Specification.new do |spec|
|
|
|
27
27
|
spec.metadata['source_code_uri'] = 'https://github.com/mensfeld/llm-docs-builder'
|
|
28
28
|
spec.metadata['changelog_uri'] = 'https://github.com/mensfeld/llm-docs-builder/blob/master/CHANGELOG.md'
|
|
29
29
|
spec.metadata['documentation_uri'] = 'https://github.com/mensfeld/llm-docs-builder'
|
|
30
|
+
spec.metadata['bug_tracker_uri'] = 'https://github.com/mensfeld/llm-docs-builder/issues'
|
|
30
31
|
spec.metadata['rubygems_mfa_required'] = 'true'
|
|
31
32
|
|
|
32
33
|
spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec|test)/}) }
|
|
@@ -42,5 +43,5 @@ Gem::Specification.new do |spec|
|
|
|
42
43
|
spec.add_development_dependency 'rake', '~> 13.0'
|
|
43
44
|
spec.add_development_dependency 'rspec', '~> 3.0'
|
|
44
45
|
spec.add_development_dependency 'rubocop', '~> 1.0'
|
|
45
|
-
spec.add_development_dependency 'simplecov', '~> 0
|
|
46
|
+
spec.add_development_dependency 'simplecov', '~> 1.0'
|
|
46
47
|
end
|
data/package-lock.json
CHANGED
|
@@ -286,9 +286,9 @@
|
|
|
286
286
|
}
|
|
287
287
|
},
|
|
288
288
|
"node_modules/smol-toml": {
|
|
289
|
-
"version": "1.
|
|
290
|
-
"resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.
|
|
291
|
-
"integrity": "sha512-
|
|
289
|
+
"version": "1.8.0",
|
|
290
|
+
"resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.8.0.tgz",
|
|
291
|
+
"integrity": "sha512-kCZr2V3ch9i00x8zXRhjUNVcjG9ijES5dDudkXvUVCT5QlJNQWElSJdZqyPemffHoLNUYwOcou0Fy+ojN0uHSQ==",
|
|
292
292
|
"dev": true,
|
|
293
293
|
"license": "BSD-3-Clause",
|
|
294
294
|
"engines": {
|
|
@@ -312,9 +312,9 @@
|
|
|
312
312
|
}
|
|
313
313
|
},
|
|
314
314
|
"node_modules/yaml": {
|
|
315
|
-
"version": "2.
|
|
316
|
-
"resolved": "https://registry.npmjs.org/yaml/-/yaml-2.
|
|
317
|
-
"integrity": "sha512-
|
|
315
|
+
"version": "2.9.0",
|
|
316
|
+
"resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.0.tgz",
|
|
317
|
+
"integrity": "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==",
|
|
318
318
|
"dev": true,
|
|
319
319
|
"license": "ISC",
|
|
320
320
|
"bin": {
|
data/renovate.json
CHANGED
|
@@ -34,13 +34,24 @@
|
|
|
34
34
|
"ruby/setup-ruby",
|
|
35
35
|
"ruby"
|
|
36
36
|
],
|
|
37
|
-
"groupName": "ruby setup"
|
|
37
|
+
"groupName": "ruby setup",
|
|
38
|
+
"internalChecksFilter": "strict"
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
"description": "Let setup-ruby pass age gate before ruby so it is ready when the group PR is created",
|
|
42
|
+
"matchPackageNames": [
|
|
43
|
+
"ruby/setup-ruby"
|
|
44
|
+
],
|
|
45
|
+
"minimumReleaseAge": "5 days"
|
|
38
46
|
}
|
|
39
47
|
],
|
|
40
48
|
"labels": [
|
|
41
49
|
"dependencies"
|
|
42
50
|
],
|
|
43
51
|
"lockFileMaintenance": {
|
|
44
|
-
"enabled": true
|
|
52
|
+
"enabled": true,
|
|
53
|
+
"schedule": [
|
|
54
|
+
"before 4am on the first day of the month"
|
|
55
|
+
]
|
|
45
56
|
}
|
|
46
57
|
}
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: llm-docs-builder
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 1.0.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Maciej Mensfeld
|
|
@@ -99,14 +99,14 @@ dependencies:
|
|
|
99
99
|
requirements:
|
|
100
100
|
- - "~>"
|
|
101
101
|
- !ruby/object:Gem::Version
|
|
102
|
-
version: '0
|
|
102
|
+
version: '1.0'
|
|
103
103
|
type: :development
|
|
104
104
|
prerelease: false
|
|
105
105
|
version_requirements: !ruby/object:Gem::Requirement
|
|
106
106
|
requirements:
|
|
107
107
|
- - "~>"
|
|
108
108
|
- !ruby/object:Gem::Version
|
|
109
|
-
version: '0
|
|
109
|
+
version: '1.0'
|
|
110
110
|
description: |
|
|
111
111
|
A comprehensive Ruby tool for building and optimizing documentation for Large Language Models.
|
|
112
112
|
Features include: generating llms.txt files from documentation directories with automatic file
|
|
@@ -185,6 +185,7 @@ metadata:
|
|
|
185
185
|
source_code_uri: https://github.com/mensfeld/llm-docs-builder
|
|
186
186
|
changelog_uri: https://github.com/mensfeld/llm-docs-builder/blob/master/CHANGELOG.md
|
|
187
187
|
documentation_uri: https://github.com/mensfeld/llm-docs-builder
|
|
188
|
+
bug_tracker_uri: https://github.com/mensfeld/llm-docs-builder/issues
|
|
188
189
|
rubygems_mfa_required: 'true'
|
|
189
190
|
rdoc_options: []
|
|
190
191
|
require_paths:
|
|
@@ -200,7 +201,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
200
201
|
- !ruby/object:Gem::Version
|
|
201
202
|
version: '0'
|
|
202
203
|
requirements: []
|
|
203
|
-
rubygems_version: 4.0.
|
|
204
|
+
rubygems_version: 4.0.16
|
|
204
205
|
specification_version: 4
|
|
205
206
|
summary: Build and optimize documentation for LLMs - generate llms.txt, transform
|
|
206
207
|
markdown, and more
|