seo_geo 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/AGENTS.md +244 -0
- data/CHANGELOG.md +40 -0
- data/LICENSE.txt +21 -0
- data/README.md +428 -0
- data/app/controllers/seo_geo/index_now_controller.rb +13 -0
- data/app/controllers/seo_geo/llms_controller.rb +29 -0
- data/app/controllers/seo_geo/og_images_controller.rb +67 -0
- data/app/controllers/seo_geo/robots_controller.rb +14 -0
- data/app/controllers/seo_geo/sitemaps_controller.rb +16 -0
- data/app/jobs/seo_geo/index_now_job.rb +21 -0
- data/app/views/seo_geo/fallback/_body.html.erb +21 -0
- data/app/views/seo_geo/fallback/_faq.html.erb +12 -0
- data/config/routes.rb +17 -0
- data/lib/generators/seo_geo/install/install_generator.rb +138 -0
- data/lib/generators/seo_geo/install/templates/home.yml.tt +14 -0
- data/lib/generators/seo_geo/install/templates/initializer.rb.tt +52 -0
- data/lib/seo_geo/audit/checks.rb +107 -0
- data/lib/seo_geo/audit/html_document.rb +132 -0
- data/lib/seo_geo/audit/report.rb +85 -0
- data/lib/seo_geo/audit/runner.rb +327 -0
- data/lib/seo_geo/audit/static_scan.rb +60 -0
- data/lib/seo_geo/audit.rb +14 -0
- data/lib/seo_geo/bot_verifier.rb +278 -0
- data/lib/seo_geo/configuration.rb +832 -0
- data/lib/seo_geo/controller.rb +72 -0
- data/lib/seo_geo/crawler_fallback.rb +45 -0
- data/lib/seo_geo/crawlers.rb +79 -0
- data/lib/seo_geo/engine.rb +123 -0
- data/lib/seo_geo/index_now.rb +187 -0
- data/lib/seo_geo/json_ld.rb +259 -0
- data/lib/seo_geo/llms_txt.rb +183 -0
- data/lib/seo_geo/markdown/document.rb +163 -0
- data/lib/seo_geo/markdown/negotiation.rb +49 -0
- data/lib/seo_geo/markdown/path_suffix.rb +72 -0
- data/lib/seo_geo/markdownable.rb +141 -0
- data/lib/seo_geo/meta.rb +382 -0
- data/lib/seo_geo/model.rb +95 -0
- data/lib/seo_geo/og_image/card.html.erb +22 -0
- data/lib/seo_geo/og_image/ferrum_backend.rb +72 -0
- data/lib/seo_geo/og_image/renderer.rb +163 -0
- data/lib/seo_geo/page.rb +201 -0
- data/lib/seo_geo/path_match.rb +36 -0
- data/lib/seo_geo/publication.rb +68 -0
- data/lib/seo_geo/robots/evaluator.rb +85 -0
- data/lib/seo_geo/robots.rb +105 -0
- data/lib/seo_geo/robots_directives.rb +170 -0
- data/lib/seo_geo/robots_path.rb +125 -0
- data/lib/seo_geo/rspec.rb +197 -0
- data/lib/seo_geo/safe_regexp.rb +50 -0
- data/lib/seo_geo/sitemap/registry.rb +239 -0
- data/lib/seo_geo/sitemap/xml.rb +138 -0
- data/lib/seo_geo/testing/environment.rb +31 -0
- data/lib/seo_geo/testing.rb +111 -0
- data/lib/seo_geo/undeclared_host.rb +40 -0
- data/lib/seo_geo/version.rb +5 -0
- data/lib/seo_geo.rb +80 -0
- data/lib/tasks/seo_geo.rake +23 -0
- data/llms.txt +27 -0
- metadata +224 -0
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "date"
|
|
5
|
+
require "time"
|
|
6
|
+
|
|
7
|
+
module SeoGeo
|
|
8
|
+
# Builds one schema.org +@graph+ document. Nodes get stable +@id+ anchors so they can reference
|
|
9
|
+
# each other (+{ "@id" => "https://www.example.com/#organization" }+) and so the same node added
|
|
10
|
+
# twice (layout + view) is emitted once — the later call wins.
|
|
11
|
+
#
|
|
12
|
+
# Anchors: site-wide nodes hang off the origin (+/#website+, +/#organization+); page nodes hang
|
|
13
|
+
# off the page URL (+<url>#faq+, +#article+, +#breadcrumb+, +#video+, +#product+, +#profile+, +#app+).
|
|
14
|
+
#
|
|
15
|
+
# Keyword arguments are snake_case and become camelCase properties (+same_as+ -> +sameAs+);
|
|
16
|
+
# nested +type:+/+id:+ become +@type+/+@id+; Date/Time values become ISO 8601; nil values are dropped;
|
|
17
|
+
# root-relative URLs (+"/logo.png"+) in URL-like properties are made absolute against the origin.
|
|
18
|
+
#
|
|
19
|
+
# Each typed method checks the properties Google requires for that type and raises {InvalidNode}
|
|
20
|
+
# listing every missing one. FAQPage is built for AI answers and Bing; Google shows FAQ rich results
|
|
21
|
+
# only for government and health sites, so no rich result is promised.
|
|
22
|
+
#
|
|
23
|
+
# @example
|
|
24
|
+
# ld = SeoGeo::JsonLd.new
|
|
25
|
+
# ld.website.organization
|
|
26
|
+
# ld.article(headline: post.title, url: post_url(post), date_published: post.published_at)
|
|
27
|
+
# ld.to_json
|
|
28
|
+
class JsonLd
|
|
29
|
+
# Raised when a node lacks a property its type requires.
|
|
30
|
+
class InvalidNode < ArgumentError; end
|
|
31
|
+
|
|
32
|
+
CONTEXT = "https://schema.org"
|
|
33
|
+
|
|
34
|
+
# @return [Hash{String => Array<String>}] required properties per +@type+ (Google structured data docs)
|
|
35
|
+
REQUIRED = {
|
|
36
|
+
"WebSite" => %w[name url],
|
|
37
|
+
"Organization" => %w[name url],
|
|
38
|
+
"SoftwareApplication" => %w[name],
|
|
39
|
+
"WebApplication" => %w[name],
|
|
40
|
+
"MobileApplication" => %w[name],
|
|
41
|
+
"Article" => %w[headline],
|
|
42
|
+
"BlogPosting" => %w[headline],
|
|
43
|
+
"NewsArticle" => %w[headline],
|
|
44
|
+
"BreadcrumbList" => %w[itemListElement],
|
|
45
|
+
"VideoObject" => %w[name thumbnailUrl uploadDate],
|
|
46
|
+
"Product" => %w[name],
|
|
47
|
+
"ProfilePage" => %w[mainEntity],
|
|
48
|
+
"FAQPage" => %w[mainEntity]
|
|
49
|
+
}.freeze
|
|
50
|
+
|
|
51
|
+
URL_KEYS = %w[url logo image thumbnailUrl contentUrl embedUrl item].freeze
|
|
52
|
+
private_constant :URL_KEYS
|
|
53
|
+
|
|
54
|
+
# @return [String] origin used for +@id+ anchors and absolute URLs
|
|
55
|
+
attr_reader :origin
|
|
56
|
+
|
|
57
|
+
# @param origin [String, nil] defaults to config.canonical_origin
|
|
58
|
+
# @param config [Configuration]
|
|
59
|
+
def initialize(origin: nil, config: SeoGeo.config)
|
|
60
|
+
@config = config
|
|
61
|
+
@origin = (origin || config.canonical_origin).to_s.chomp("/")
|
|
62
|
+
@nodes = []
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# @return [Array<Hash>] the graph nodes, in insertion order
|
|
66
|
+
def nodes
|
|
67
|
+
@nodes.dup
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# @return [Boolean]
|
|
71
|
+
def empty?
|
|
72
|
+
@nodes.empty?
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
# Adds a raw node (string or symbol keys) or every node of another {JsonLd} / +@graph+ document.
|
|
76
|
+
# Raw nodes of a known +@type+ are validated like the typed methods.
|
|
77
|
+
# @param node [Hash, JsonLd, Array]
|
|
78
|
+
# @return [self]
|
|
79
|
+
def add(node)
|
|
80
|
+
case node
|
|
81
|
+
when JsonLd then node.nodes.each { |inner| push(inner) }
|
|
82
|
+
when Array then node.each { |inner| add(inner) }
|
|
83
|
+
when Hash
|
|
84
|
+
normalized = normalize(node)
|
|
85
|
+
graph = normalized["@graph"]
|
|
86
|
+
graph ? graph.each { |inner| push(inner) } : push(normalized.except("@context"))
|
|
87
|
+
when nil then nil
|
|
88
|
+
else raise ArgumentError, "JSON-LD node must be a Hash or SeoGeo::JsonLd (got #{node.class})"
|
|
89
|
+
end
|
|
90
|
+
self
|
|
91
|
+
end
|
|
92
|
+
alias << add
|
|
93
|
+
|
|
94
|
+
# @return [self]
|
|
95
|
+
def website(name: @config.site.name, url: "#{origin}/", description: @config.site.description, **attrs)
|
|
96
|
+
node("WebSite", "#{origin}/#website", name: name, url: url, description: description, **attrs)
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
# @return [self]
|
|
100
|
+
def organization(name: @config.site.name, url: "#{origin}/", logo: @config.site.logo_url,
|
|
101
|
+
same_as: @config.site.same_as, **attrs)
|
|
102
|
+
same_as = Array(same_as)
|
|
103
|
+
node("Organization", "#{origin}/#organization", name: name, url: url, logo: logo,
|
|
104
|
+
same_as: same_as.empty? ? nil : same_as, **attrs)
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
# @param type [String] "SoftwareApplication", "WebApplication" or "MobileApplication"
|
|
108
|
+
# @return [self]
|
|
109
|
+
def software_application(name:, url: "#{origin}/", type: "SoftwareApplication", **attrs)
|
|
110
|
+
node(type, anchor(url, "app"), name: name, url: url, **attrs)
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
# @return [self]
|
|
114
|
+
def web_application(name:, **attrs)
|
|
115
|
+
software_application(name: name, type: "WebApplication", **attrs)
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
# @param type [String] "Article", "BlogPosting" or "NewsArticle"
|
|
119
|
+
# @return [self]
|
|
120
|
+
def article(headline:, url: "#{origin}/", type: "Article", **attrs)
|
|
121
|
+
node(type, anchor(url, "article"), headline: headline, url: url, main_entity_of_page: url, **attrs)
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
# @param items [Array<Array(String, String)>, Array<Hash>] +[name, url]+ pairs or +{name:, url:}+;
|
|
125
|
+
# the last item may omit its URL (the current page)
|
|
126
|
+
# @return [self]
|
|
127
|
+
def breadcrumb_list(items, url: "#{origin}/")
|
|
128
|
+
elements = Array(items).each_with_index.map do |item, index|
|
|
129
|
+
name, item_url = item.is_a?(Hash) ? [item[:name] || item["name"], item[:url] || item["url"]] : item
|
|
130
|
+
raise InvalidNode, "BreadcrumbList item #{index + 1} needs a name" if blank?(name)
|
|
131
|
+
|
|
132
|
+
{ type: "ListItem", position: index + 1, name: name, item: item_url }
|
|
133
|
+
end
|
|
134
|
+
node("BreadcrumbList", anchor(url, "breadcrumb"), item_list_element: elements.empty? ? nil : elements)
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
# @return [self]
|
|
138
|
+
def video_object(name:, thumbnail_url:, upload_date:, url: "#{origin}/", **attrs)
|
|
139
|
+
node("VideoObject", anchor(url, "video"), name: name, thumbnail_url: thumbnail_url,
|
|
140
|
+
upload_date: upload_date, **attrs)
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
# Google requires one of +offers+, +review+ or +aggregate_rating+ for product snippets.
|
|
144
|
+
# @return [self]
|
|
145
|
+
def product(name:, url: "#{origin}/", **attrs)
|
|
146
|
+
unless %i[offers review aggregate_rating].any? { |key| attrs[key] }
|
|
147
|
+
raise InvalidNode, "Product #{name.inspect} needs offers, review or aggregate_rating"
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
node("Product", anchor(url, "product"), name: name, url: url, **attrs)
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
# @param main_entity [Hash] the Person or Organization the page is about (+name:+ required)
|
|
154
|
+
# @return [self]
|
|
155
|
+
def profile_page(main_entity:, url: "#{origin}/", **attrs)
|
|
156
|
+
entity = { type: "Person" }.merge(main_entity.transform_keys(&:to_sym))
|
|
157
|
+
raise InvalidNode, "ProfilePage main_entity needs a name" if blank?(entity[:name])
|
|
158
|
+
|
|
159
|
+
node("ProfilePage", anchor(url, "profile"), url: url, main_entity: entity, **attrs)
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
# @param faq [Array<Hash>] +{question:, answer:}+ items, e.g. {Page#faq}
|
|
163
|
+
# @return [self]
|
|
164
|
+
def faq_page(faq, url: "#{origin}/", **attrs)
|
|
165
|
+
questions = Array(faq).each_with_index.map do |item, index|
|
|
166
|
+
question = item[:question] || item["question"]
|
|
167
|
+
answer = item[:answer] || item["answer"]
|
|
168
|
+
raise InvalidNode, "FAQPage item #{index + 1} needs a question and an answer" if blank?(question) || blank?(answer)
|
|
169
|
+
|
|
170
|
+
{ type: "Question", name: question, accepted_answer: { type: "Answer", text: answer } }
|
|
171
|
+
end
|
|
172
|
+
node("FAQPage", anchor(url, "faq"), main_entity: questions.empty? ? nil : questions, **attrs)
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
# @return [Hash{String => Object}] +{"@context" => ..., "@graph" => [...]}+
|
|
176
|
+
def to_h
|
|
177
|
+
{ "@context" => CONTEXT, "@graph" => nodes }
|
|
178
|
+
end
|
|
179
|
+
alias as_json to_h
|
|
180
|
+
|
|
181
|
+
# Plain JSON. Use {SeoGeo::Meta::Helper#seo_geo_json_ld_tag} (or +ERB::Util.json_escape+) to put it in HTML.
|
|
182
|
+
# @return [String]
|
|
183
|
+
def to_json(*)
|
|
184
|
+
JSON.generate(to_h)
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
# @param type [String] schema.org type
|
|
188
|
+
# @param id [String, nil] +@id+
|
|
189
|
+
# @return [self]
|
|
190
|
+
def node(type, id, **attrs)
|
|
191
|
+
push(normalize({ type: type, id: id }.merge(attrs)))
|
|
192
|
+
self
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
private
|
|
196
|
+
|
|
197
|
+
def push(hash)
|
|
198
|
+
validate!(hash)
|
|
199
|
+
@nodes.reject! { |existing| hash["@id"] && existing["@id"] == hash["@id"] }
|
|
200
|
+
@nodes << hash
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
def validate!(hash)
|
|
204
|
+
missing = REQUIRED.fetch(hash["@type"].to_s, []).select { |key| blank?(hash[key]) }
|
|
205
|
+
return if missing.empty?
|
|
206
|
+
|
|
207
|
+
raise InvalidNode, "#{hash["@type"]} #{hash["@id"]} is missing #{missing.join(", ")}"
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
def anchor(url, name)
|
|
211
|
+
"#{absolute(url.to_s).sub(/#.*\z/, "")}##{name}"
|
|
212
|
+
end
|
|
213
|
+
|
|
214
|
+
def normalize(value, key = nil)
|
|
215
|
+
case value
|
|
216
|
+
when Hash
|
|
217
|
+
value.each_with_object({}) do |(name, inner), out|
|
|
218
|
+
property = property_name(name)
|
|
219
|
+
normalized = normalize(inner, property)
|
|
220
|
+
out[property] = normalized unless normalized.nil?
|
|
221
|
+
end
|
|
222
|
+
when Array then value.map { |inner| normalize(inner, key) }
|
|
223
|
+
when Time, DateTime then value.iso8601
|
|
224
|
+
when Date then value.iso8601
|
|
225
|
+
when Symbol then value.to_s
|
|
226
|
+
when String then URL_KEYS.include?(key) ? absolute(value) : value
|
|
227
|
+
else value.respond_to?(:iso8601) ? value.iso8601 : value
|
|
228
|
+
end
|
|
229
|
+
end
|
|
230
|
+
|
|
231
|
+
def property_name(name)
|
|
232
|
+
return name if name.is_a?(String)
|
|
233
|
+
return "@#{name}" if %i[type id context graph].include?(name)
|
|
234
|
+
|
|
235
|
+
name.to_s.gsub(/_([a-z\d])/) { Regexp.last_match(1).upcase }
|
|
236
|
+
end
|
|
237
|
+
|
|
238
|
+
def absolute(value)
|
|
239
|
+
value.start_with?("/") && !value.start_with?("//") ? "#{origin}#{value}" : value
|
|
240
|
+
end
|
|
241
|
+
|
|
242
|
+
def blank?(value)
|
|
243
|
+
value.nil? || (value.respond_to?(:empty?) && value.empty?)
|
|
244
|
+
end
|
|
245
|
+
end
|
|
246
|
+
|
|
247
|
+
# JSON-LD built from the page copy, so FAQPage always matches the visible FAQ.
|
|
248
|
+
class Page
|
|
249
|
+
# @param url [String, nil] absolute page URL used for +@id+ anchors; defaults to the canonical origin root
|
|
250
|
+
# @param config [Configuration]
|
|
251
|
+
# @return [JsonLd] a document holding the FAQPage node (empty when the page has no FAQ)
|
|
252
|
+
def json_ld(url: nil, config: SeoGeo.config)
|
|
253
|
+
document = JsonLd.new(config: config)
|
|
254
|
+
return document if faq.empty?
|
|
255
|
+
|
|
256
|
+
document.faq_page(faq, url: url || "#{document.origin}/")
|
|
257
|
+
end
|
|
258
|
+
end
|
|
259
|
+
end
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "uri"
|
|
4
|
+
require "set"
|
|
5
|
+
|
|
6
|
+
module SeoGeo
|
|
7
|
+
# Builds /llms.txt (https://llmstxt.org/) and /llms-full.txt for one request host.
|
|
8
|
+
#
|
|
9
|
+
# llms.txt: +# site.name+, +> site.description+, optional {LlmsTxtOptions#details}, then one H2 per
|
|
10
|
+
# section with +- [title](url): description+ lines and +## Optional+ last. Page links point at the
|
|
11
|
+
# page's Markdown (+.md+) when {MarkdownOptions#enabled}. Links come from {Configuration#llms_txt};
|
|
12
|
+
# when none are declared, the host's static sitemap paths are listed.
|
|
13
|
+
#
|
|
14
|
+
# llms-full.txt (community convention, not in the llmstxt.org text): the same header followed by
|
|
15
|
+
# every listed page's {Page#to_markdown}, headings demoted one level under one H1.
|
|
16
|
+
#
|
|
17
|
+
# Both are served only for hosts whose sitemap is on ({HostPolicy#sitemap}); otherwise nil (=> 404).
|
|
18
|
+
class LlmsTxt
|
|
19
|
+
OPTIONAL_SECTION = "Optional"
|
|
20
|
+
|
|
21
|
+
# @param config [Configuration]
|
|
22
|
+
# @param host [String] request host
|
|
23
|
+
# @param pages [PageRegistry]
|
|
24
|
+
# @param sitemap [Sitemap::Registry] fallback link source
|
|
25
|
+
def initialize(config, host, pages: SeoGeo.pages, sitemap: SeoGeo.sitemap)
|
|
26
|
+
@config = config
|
|
27
|
+
@host = host
|
|
28
|
+
@pages = pages
|
|
29
|
+
@sitemap = sitemap
|
|
30
|
+
@options = config.llms_txt
|
|
31
|
+
@policy = config.host_policy(host)
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# @return [String, nil] llms.txt body, nil when not served on this host
|
|
35
|
+
def to_s
|
|
36
|
+
return nil unless served?
|
|
37
|
+
|
|
38
|
+
parts = header
|
|
39
|
+
sections.each do |name, lines|
|
|
40
|
+
parts << "## #{name}\n\n#{lines.join("\n")}"
|
|
41
|
+
end
|
|
42
|
+
"#{parts.join("\n\n")}\n"
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# @return [String, nil] llms-full.txt body, nil when not served or no page is listed
|
|
46
|
+
def full
|
|
47
|
+
return nil unless served? && @options.full
|
|
48
|
+
|
|
49
|
+
page_links = links.select(&:page)
|
|
50
|
+
return nil if page_links.empty?
|
|
51
|
+
|
|
52
|
+
parts = header
|
|
53
|
+
page_links.each do |item|
|
|
54
|
+
markdown = page_for(item).to_markdown.gsub(/^(#+) /) { "##{Regexp.last_match(1)} " }
|
|
55
|
+
parts << "---\n\nSource: #{absolute(item.path)}\n\n#{markdown.strip}"
|
|
56
|
+
end
|
|
57
|
+
"#{parts.join("\n\n")}\n"
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
private
|
|
61
|
+
|
|
62
|
+
def served?
|
|
63
|
+
@options.enabled && @policy.sitemap
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def header
|
|
67
|
+
name = present(@config.site.name) || @config.canonical_host || @policy.host
|
|
68
|
+
parts = ["# #{name}"]
|
|
69
|
+
description = present(@config.site.description)
|
|
70
|
+
parts << "> #{description.gsub(/\s+/, " ")}" if description
|
|
71
|
+
details = present(@options.details)
|
|
72
|
+
parts << details_markdown(details) if details
|
|
73
|
+
parts
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# @return [Array<Array(String, Array<String>)>] section name => link lines, Optional last
|
|
77
|
+
def sections
|
|
78
|
+
grouped = links.group_by { |item| item.optional ? OPTIONAL_SECTION : (item.section || @options.default_section) }
|
|
79
|
+
optional = grouped.delete(OPTIONAL_SECTION)
|
|
80
|
+
grouped = grouped.to_a
|
|
81
|
+
grouped << [OPTIONAL_SECTION, optional] if optional
|
|
82
|
+
grouped.map { |name, items| [name, items.map { |item| line(item) }] }
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def line(item)
|
|
86
|
+
page = item.page && page_for(item)
|
|
87
|
+
title = item.title || page&.title || page&.heading || item.path
|
|
88
|
+
description = item.description || page&.description
|
|
89
|
+
url = absolute(item.path)
|
|
90
|
+
url = markdown_url(url) if item.markdown_link?(@config) && same_origin?(URI.parse(url))
|
|
91
|
+
text = "- [#{title.to_s.gsub(/[\[\]]/) { |char| "\\#{char}" }}](#{url})"
|
|
92
|
+
description ? "#{text}: #{description.to_s.gsub(/\s+/, " ").strip}" : text
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def links
|
|
96
|
+
@links ||= begin
|
|
97
|
+
declared = @options.links.select { |item| for_host?(item.hosts) }
|
|
98
|
+
(@options.links.empty? ? sitemap_links : declared).select { |item| publishable?(item.path) }
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
def sitemap_links
|
|
103
|
+
@sitemap.entries.select { |entry| entry.static? && entry.for_host?(@host, @config) && !@sitemap.excluded?(entry.source) }
|
|
104
|
+
.map do |entry|
|
|
105
|
+
LlmsTxtOptions::Link.new(page: nil, title: nil, path: entry.source, description: nil, section: nil,
|
|
106
|
+
optional: false, hosts: nil, markdown: entry.markdown)
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
# llms.txt and llms-full.txt list only what {SeoGeo::Publication} allows (the sitemap.xml and
|
|
111
|
+
# head rule: robots disallow, exclusions, noindex declarations, default_noindex). Links to other
|
|
112
|
+
# origins are not ours to judge and pass through.
|
|
113
|
+
def publishable?(path)
|
|
114
|
+
if path.match?(%r{\Ahttps?://}i)
|
|
115
|
+
return true unless same_origin?(URI.parse(path))
|
|
116
|
+
end
|
|
117
|
+
# /x.md (also /x.md?q) is the Markdown form of /x (Markdown::PathSuffix), so it is published only when /x is.
|
|
118
|
+
# The query stays for robots rules (Disallow: /x?preview=1); declarations drop it themselves.
|
|
119
|
+
page_path = RobotsPath.html_target(path, markdown: @config.markdown.suffix_urls?)
|
|
120
|
+
Publication.publishable?(@host, page_path, config: @config, sitemap: @sitemap, check_host: false,
|
|
121
|
+
declared_index: indexed_by_sitemap?(page_path))
|
|
122
|
+
rescue URI::InvalidURIError
|
|
123
|
+
false
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
def same_origin?(uri)
|
|
127
|
+
origin = URI.parse(@config.origin_for(@host))
|
|
128
|
+
uri.host && uri.host.casecmp?(origin.host) && uri.scheme.casecmp?(origin.scheme) && uri.port == origin.port
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
# Under default_noindex, records of a collection added with index: true are declared indexable too;
|
|
132
|
+
# only sitemap.xml knows their paths, so ask it (enumerated once per llms.txt render).
|
|
133
|
+
def indexed_by_sitemap?(path)
|
|
134
|
+
return false unless @config.default_noindex
|
|
135
|
+
|
|
136
|
+
@indexed_paths ||= Sitemap::Xml.new(@config, @sitemap, @host).urls
|
|
137
|
+
.to_set { |url| RobotsPath.path(url.loc) }
|
|
138
|
+
@indexed_paths.include?(RobotsPath.path(path))
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
def for_host?(hosts)
|
|
142
|
+
wanted = hosts ? hosts.map(&:downcase) : [@config.canonical_host]
|
|
143
|
+
wanted.include?(@policy.host)
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
def page_for(item)
|
|
147
|
+
@pages.fetch(item.page, locale: @config.site.default_locale)
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
def absolute(path)
|
|
151
|
+
return path if path.match?(%r{\Ahttps?://})
|
|
152
|
+
|
|
153
|
+
@config.canonical_url("#{@config.origin_for(@host)}#{path}")
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
# "/" => "/index.md", "/pricing" => "/pricing.md" (llmstxt.org convention); query dropped.
|
|
157
|
+
def markdown_url(url)
|
|
158
|
+
uri = URI.parse(url)
|
|
159
|
+
path = uri.path.to_s
|
|
160
|
+
path = path.empty? || path.end_with?("/") ? "#{path.chomp("/")}/index.md" : "#{path}.md"
|
|
161
|
+
uri.path = path
|
|
162
|
+
uri.query = nil
|
|
163
|
+
uri.fragment = nil
|
|
164
|
+
uri.to_s
|
|
165
|
+
rescue URI::InvalidURIError
|
|
166
|
+
url
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
# Details are free Markdown (paragraphs, lists, headings). In llms.txt the single H1 is the site name and H2s are
|
|
170
|
+
# link sections, so H1/H2 headings in details are demoted to H3 (outside fenced code); H3+ stay as written.
|
|
171
|
+
def details_markdown(text)
|
|
172
|
+
fenced = false
|
|
173
|
+
text.strip.lines.map do |line|
|
|
174
|
+
fenced = !fenced if line.start_with?("```", "~~~")
|
|
175
|
+
fenced ? line : line.sub(/\A\#{1,2}(?=\s)/, "###")
|
|
176
|
+
end.join
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
def present(value)
|
|
180
|
+
value.nil? || value.to_s.strip.empty? ? nil : value.to_s
|
|
181
|
+
end
|
|
182
|
+
end
|
|
183
|
+
end
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "yaml"
|
|
5
|
+
require "nokogiri"
|
|
6
|
+
|
|
7
|
+
module SeoGeo
|
|
8
|
+
module Markdown
|
|
9
|
+
# Turns a rendered HTML page into a Markdown document for agents:
|
|
10
|
+
#
|
|
11
|
+
# ---
|
|
12
|
+
# title: ... # <title>, else page copy
|
|
13
|
+
# description: ... # meta description
|
|
14
|
+
# canonical: ... # link rel=canonical, else config.canonical_url(request URL)
|
|
15
|
+
# image: ... # og:image
|
|
16
|
+
# locale: ... # <html lang>
|
|
17
|
+
# ---
|
|
18
|
+
#
|
|
19
|
+
# <main content converted to Markdown>
|
|
20
|
+
#
|
|
21
|
+
# ```json
|
|
22
|
+
# <each JSON-LD block>
|
|
23
|
+
# ```
|
|
24
|
+
#
|
|
25
|
+
# When the HTML has no main content (an Inertia/SPA mount point) and a {SeoGeo::Page} is given, the
|
|
26
|
+
# body comes from {SeoGeo::Page#to_markdown}, so agents see the same copy as the crawler fallback.
|
|
27
|
+
class Document
|
|
28
|
+
# Removed before conversion wherever they appear.
|
|
29
|
+
NOISE = %w[script style noscript template iframe svg canvas nav form button dialog].freeze
|
|
30
|
+
# Removed as well when no main element exists and the whole body is converted.
|
|
31
|
+
CHROME = %w[header footer aside].freeze
|
|
32
|
+
|
|
33
|
+
# @return [String]
|
|
34
|
+
attr_reader :html
|
|
35
|
+
|
|
36
|
+
# @param html [String] the rendered HTML response body
|
|
37
|
+
# @param config [Configuration]
|
|
38
|
+
# @param url [String, nil] request URL, used for the canonical fallback
|
|
39
|
+
# @param page [Page, nil] page copy for client-rendered pages
|
|
40
|
+
# @param selector [String, nil] main-content selector; nil uses +config.markdown.selector+
|
|
41
|
+
def initialize(html, config: SeoGeo.config, url: nil, page: nil, selector: nil)
|
|
42
|
+
@html = html.to_s
|
|
43
|
+
@config = config
|
|
44
|
+
@url = url
|
|
45
|
+
@page = page
|
|
46
|
+
@selector = selector || config.markdown.selector
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# @return [String] the full Markdown document
|
|
50
|
+
def to_s
|
|
51
|
+
body = body_markdown
|
|
52
|
+
parts = [frontmatter, body.strip]
|
|
53
|
+
json_ld.each { |block| parts << "```json\n#{block}\n```" }
|
|
54
|
+
"#{parts.reject(&:empty?).join("\n\n")}\n"
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# @return [Hash{String => String}] frontmatter fields that have a value
|
|
58
|
+
def metadata
|
|
59
|
+
{
|
|
60
|
+
"title" => text_at("head > title") || @page&.title,
|
|
61
|
+
"description" => attribute_at('meta[name="description"]', "content") || @page&.description,
|
|
62
|
+
"canonical" => attribute_at('link[rel="canonical"]', "href") || canonical_fallback,
|
|
63
|
+
"image" => attribute_at('meta[property="og:image"]', "content"),
|
|
64
|
+
"locale" => attribute_at("html", "lang") || @page&.locale&.to_s
|
|
65
|
+
}.reject { |_name, value| value.nil? || value.to_s.strip.empty? }
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# @return [RobotsDirectives] every robots and crawler-specific meta of the HTML
|
|
69
|
+
def robots
|
|
70
|
+
@robots ||= RobotsDirectives.from_html(doc)
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# Global directives (every +<meta name="robots">+), lower-cased, +none+ expanded to +noindex, nofollow+.
|
|
74
|
+
# @return [Array<String>]
|
|
75
|
+
def robots_directives
|
|
76
|
+
robots.global.to_a
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# @return [Boolean] true when the HTML asks every robot not to index the page
|
|
80
|
+
def noindex?
|
|
81
|
+
robots.noindex?
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# @return [Array<String>] pretty-printed JSON-LD blocks
|
|
85
|
+
def json_ld
|
|
86
|
+
doc.css('script[type="application/ld+json"]').filter_map do |node|
|
|
87
|
+
raw = node.text.strip
|
|
88
|
+
next if raw.empty?
|
|
89
|
+
|
|
90
|
+
begin
|
|
91
|
+
JSON.pretty_generate(JSON.parse(raw))
|
|
92
|
+
rescue JSON::ParserError
|
|
93
|
+
raw
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
private
|
|
99
|
+
|
|
100
|
+
def doc
|
|
101
|
+
@doc ||= if defined?(Nokogiri::HTML5) then Nokogiri::HTML5(@html)
|
|
102
|
+
else Nokogiri::HTML(@html)
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def frontmatter
|
|
107
|
+
data = metadata
|
|
108
|
+
return "" if data.empty?
|
|
109
|
+
|
|
110
|
+
"#{YAML.dump(data).sub(/\A---\s*\n/, "---\n")}---"
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
def body_markdown
|
|
114
|
+
from_html = convert(main_html)
|
|
115
|
+
return from_html unless from_html.strip.empty? && @page
|
|
116
|
+
|
|
117
|
+
@page.to_markdown
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
# HTML of the main content with noise removed. Works on a copy so metadata/JSON-LD stay readable.
|
|
121
|
+
def main_html
|
|
122
|
+
copy = doc.dup
|
|
123
|
+
main = copy.at_css(@selector)
|
|
124
|
+
root = main || copy.at_css("body")
|
|
125
|
+
return "" if root.nil?
|
|
126
|
+
|
|
127
|
+
removable = main ? NOISE : NOISE + CHROME
|
|
128
|
+
root.css(removable.join(",")).each(&:remove)
|
|
129
|
+
root.inner_html
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
def convert(fragment)
|
|
133
|
+
return "" if fragment.strip.empty?
|
|
134
|
+
|
|
135
|
+
converter = @config.markdown.converter
|
|
136
|
+
markdown = if converter.respond_to?(:call)
|
|
137
|
+
converter.call(fragment)
|
|
138
|
+
else
|
|
139
|
+
require "reverse_markdown"
|
|
140
|
+
ReverseMarkdown.convert(fragment, unknown_tags: :bypass, github_flavored: true)
|
|
141
|
+
end
|
|
142
|
+
markdown.to_s.gsub(/\n{3,}/, "\n\n")
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
def text_at(selector)
|
|
146
|
+
node = doc.at_css(selector)
|
|
147
|
+
value = node&.text&.strip
|
|
148
|
+
value unless value.nil? || value.empty?
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
def attribute_at(selector, name)
|
|
152
|
+
value = doc.at_css(selector)&.[](name)&.strip
|
|
153
|
+
value unless value.nil? || value.empty?
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
def canonical_fallback
|
|
157
|
+
return nil if @url.nil? || @config.canonical_origin.nil?
|
|
158
|
+
|
|
159
|
+
@config.canonical_url(@url)
|
|
160
|
+
end
|
|
161
|
+
end
|
|
162
|
+
end
|
|
163
|
+
end
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SeoGeo
|
|
4
|
+
module Markdown
|
|
5
|
+
# Decides whether an +Accept+ header asks for Markdown.
|
|
6
|
+
#
|
|
7
|
+
# Markdown wins only when +text/markdown+ is named explicitly (wildcards never select it) with a
|
|
8
|
+
# q-value above zero that is higher than the one +text/html+ gets, or equal to it and listed first.
|
|
9
|
+
# Agents send +text/markdown, text/html;q=0.9+ or +text/markdown, text/html, *\/*+; browsers never
|
|
10
|
+
# name text/markdown, so they keep getting HTML.
|
|
11
|
+
module Negotiation
|
|
12
|
+
module_function
|
|
13
|
+
|
|
14
|
+
# @param accept [String, nil] raw Accept header
|
|
15
|
+
# @return [Boolean]
|
|
16
|
+
def markdown?(accept)
|
|
17
|
+
ranges = parse(accept)
|
|
18
|
+
markdown = ranges.find { |range| range[:type] == "text/markdown" }
|
|
19
|
+
return false if markdown.nil? || markdown[:q] <= 0
|
|
20
|
+
|
|
21
|
+
html = quality_for_html(ranges)
|
|
22
|
+
return true if html.nil?
|
|
23
|
+
|
|
24
|
+
markdown[:q] > html[:q] || (markdown[:q] == html[:q] && markdown[:index] < html[:index])
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# @param accept [String, nil]
|
|
28
|
+
# @return [Array<Hash>] +{type:, q:, index:}+ in header order
|
|
29
|
+
def parse(accept)
|
|
30
|
+
accept.to_s.split(",").each_with_index.filter_map do |part, index|
|
|
31
|
+
type, *params = part.split(";").map(&:strip)
|
|
32
|
+
next if type.nil? || type.empty?
|
|
33
|
+
|
|
34
|
+
q = params.filter_map { |param| param[/\Aq\s*=\s*([0-9.]+)\z/i, 1] }.first
|
|
35
|
+
{ type: type.downcase, q: q ? q.to_f.clamp(0.0, 1.0) : 1.0, index: index }
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# The most specific range matching text/html (RFC 9110 §12.5.1).
|
|
40
|
+
def quality_for_html(ranges)
|
|
41
|
+
%w[text/html text/* */*].each do |type|
|
|
42
|
+
found = ranges.find { |range| range[:type] == type }
|
|
43
|
+
return found if found
|
|
44
|
+
end
|
|
45
|
+
nil
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
end
|