seo_geo 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. checksums.yaml +7 -0
  2. data/AGENTS.md +244 -0
  3. data/CHANGELOG.md +40 -0
  4. data/LICENSE.txt +21 -0
  5. data/README.md +428 -0
  6. data/app/controllers/seo_geo/index_now_controller.rb +13 -0
  7. data/app/controllers/seo_geo/llms_controller.rb +29 -0
  8. data/app/controllers/seo_geo/og_images_controller.rb +67 -0
  9. data/app/controllers/seo_geo/robots_controller.rb +14 -0
  10. data/app/controllers/seo_geo/sitemaps_controller.rb +16 -0
  11. data/app/jobs/seo_geo/index_now_job.rb +21 -0
  12. data/app/views/seo_geo/fallback/_body.html.erb +21 -0
  13. data/app/views/seo_geo/fallback/_faq.html.erb +12 -0
  14. data/config/routes.rb +17 -0
  15. data/lib/generators/seo_geo/install/install_generator.rb +138 -0
  16. data/lib/generators/seo_geo/install/templates/home.yml.tt +14 -0
  17. data/lib/generators/seo_geo/install/templates/initializer.rb.tt +52 -0
  18. data/lib/seo_geo/audit/checks.rb +107 -0
  19. data/lib/seo_geo/audit/html_document.rb +132 -0
  20. data/lib/seo_geo/audit/report.rb +85 -0
  21. data/lib/seo_geo/audit/runner.rb +327 -0
  22. data/lib/seo_geo/audit/static_scan.rb +60 -0
  23. data/lib/seo_geo/audit.rb +14 -0
  24. data/lib/seo_geo/bot_verifier.rb +278 -0
  25. data/lib/seo_geo/configuration.rb +832 -0
  26. data/lib/seo_geo/controller.rb +72 -0
  27. data/lib/seo_geo/crawler_fallback.rb +45 -0
  28. data/lib/seo_geo/crawlers.rb +79 -0
  29. data/lib/seo_geo/engine.rb +123 -0
  30. data/lib/seo_geo/index_now.rb +187 -0
  31. data/lib/seo_geo/json_ld.rb +259 -0
  32. data/lib/seo_geo/llms_txt.rb +183 -0
  33. data/lib/seo_geo/markdown/document.rb +163 -0
  34. data/lib/seo_geo/markdown/negotiation.rb +49 -0
  35. data/lib/seo_geo/markdown/path_suffix.rb +72 -0
  36. data/lib/seo_geo/markdownable.rb +141 -0
  37. data/lib/seo_geo/meta.rb +382 -0
  38. data/lib/seo_geo/model.rb +95 -0
  39. data/lib/seo_geo/og_image/card.html.erb +22 -0
  40. data/lib/seo_geo/og_image/ferrum_backend.rb +72 -0
  41. data/lib/seo_geo/og_image/renderer.rb +163 -0
  42. data/lib/seo_geo/page.rb +201 -0
  43. data/lib/seo_geo/path_match.rb +36 -0
  44. data/lib/seo_geo/publication.rb +68 -0
  45. data/lib/seo_geo/robots/evaluator.rb +85 -0
  46. data/lib/seo_geo/robots.rb +105 -0
  47. data/lib/seo_geo/robots_directives.rb +170 -0
  48. data/lib/seo_geo/robots_path.rb +125 -0
  49. data/lib/seo_geo/rspec.rb +197 -0
  50. data/lib/seo_geo/safe_regexp.rb +50 -0
  51. data/lib/seo_geo/sitemap/registry.rb +239 -0
  52. data/lib/seo_geo/sitemap/xml.rb +138 -0
  53. data/lib/seo_geo/testing/environment.rb +31 -0
  54. data/lib/seo_geo/testing.rb +111 -0
  55. data/lib/seo_geo/undeclared_host.rb +40 -0
  56. data/lib/seo_geo/version.rb +5 -0
  57. data/lib/seo_geo.rb +80 -0
  58. data/lib/tasks/seo_geo.rake +23 -0
  59. data/llms.txt +27 -0
  60. metadata +224 -0
@@ -0,0 +1,259 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+ require "date"
5
+ require "time"
6
+
7
+ module SeoGeo
8
+ # Builds one schema.org +@graph+ document. Nodes get stable +@id+ anchors so they can reference
9
+ # each other (+{ "@id" => "https://www.example.com/#organization" }+) and so the same node added
10
+ # twice (layout + view) is emitted once — the later call wins.
11
+ #
12
+ # Anchors: site-wide nodes hang off the origin (+/#website+, +/#organization+); page nodes hang
13
+ # off the page URL (+<url>#faq+, +#article+, +#breadcrumb+, +#video+, +#product+, +#profile+, +#app+).
14
+ #
15
+ # Keyword arguments are snake_case and become camelCase properties (+same_as+ -> +sameAs+);
16
+ # nested +type:+/+id:+ become +@type+/+@id+; Date/Time values become ISO 8601; nil values are dropped;
17
+ # root-relative URLs (+"/logo.png"+) in URL-like properties are made absolute against the origin.
18
+ #
19
+ # Each typed method checks the properties Google requires for that type and raises {InvalidNode}
20
+ # listing every missing one. FAQPage is built for AI answers and Bing; Google shows FAQ rich results
21
+ # only for government and health sites, so no rich result is promised.
22
+ #
23
+ # @example
24
+ # ld = SeoGeo::JsonLd.new
25
+ # ld.website.organization
26
+ # ld.article(headline: post.title, url: post_url(post), date_published: post.published_at)
27
+ # ld.to_json
28
+ class JsonLd
29
+ # Raised when a node lacks a property its type requires.
30
+ class InvalidNode < ArgumentError; end
31
+
32
+ CONTEXT = "https://schema.org"
33
+
34
+ # @return [Hash{String => Array<String>}] required properties per +@type+ (Google structured data docs)
35
+ REQUIRED = {
36
+ "WebSite" => %w[name url],
37
+ "Organization" => %w[name url],
38
+ "SoftwareApplication" => %w[name],
39
+ "WebApplication" => %w[name],
40
+ "MobileApplication" => %w[name],
41
+ "Article" => %w[headline],
42
+ "BlogPosting" => %w[headline],
43
+ "NewsArticle" => %w[headline],
44
+ "BreadcrumbList" => %w[itemListElement],
45
+ "VideoObject" => %w[name thumbnailUrl uploadDate],
46
+ "Product" => %w[name],
47
+ "ProfilePage" => %w[mainEntity],
48
+ "FAQPage" => %w[mainEntity]
49
+ }.freeze
50
+
51
+ URL_KEYS = %w[url logo image thumbnailUrl contentUrl embedUrl item].freeze
52
+ private_constant :URL_KEYS
53
+
54
+ # @return [String] origin used for +@id+ anchors and absolute URLs
55
+ attr_reader :origin
56
+
57
+ # @param origin [String, nil] defaults to config.canonical_origin
58
+ # @param config [Configuration]
59
+ def initialize(origin: nil, config: SeoGeo.config)
60
+ @config = config
61
+ @origin = (origin || config.canonical_origin).to_s.chomp("/")
62
+ @nodes = []
63
+ end
64
+
65
+ # @return [Array<Hash>] the graph nodes, in insertion order
66
+ def nodes
67
+ @nodes.dup
68
+ end
69
+
70
+ # @return [Boolean]
71
+ def empty?
72
+ @nodes.empty?
73
+ end
74
+
75
+ # Adds a raw node (string or symbol keys) or every node of another {JsonLd} / +@graph+ document.
76
+ # Raw nodes of a known +@type+ are validated like the typed methods.
77
+ # @param node [Hash, JsonLd, Array]
78
+ # @return [self]
79
+ def add(node)
80
+ case node
81
+ when JsonLd then node.nodes.each { |inner| push(inner) }
82
+ when Array then node.each { |inner| add(inner) }
83
+ when Hash
84
+ normalized = normalize(node)
85
+ graph = normalized["@graph"]
86
+ graph ? graph.each { |inner| push(inner) } : push(normalized.except("@context"))
87
+ when nil then nil
88
+ else raise ArgumentError, "JSON-LD node must be a Hash or SeoGeo::JsonLd (got #{node.class})"
89
+ end
90
+ self
91
+ end
92
+ alias << add
93
+
94
+ # @return [self]
95
+ def website(name: @config.site.name, url: "#{origin}/", description: @config.site.description, **attrs)
96
+ node("WebSite", "#{origin}/#website", name: name, url: url, description: description, **attrs)
97
+ end
98
+
99
+ # @return [self]
100
+ def organization(name: @config.site.name, url: "#{origin}/", logo: @config.site.logo_url,
101
+ same_as: @config.site.same_as, **attrs)
102
+ same_as = Array(same_as)
103
+ node("Organization", "#{origin}/#organization", name: name, url: url, logo: logo,
104
+ same_as: same_as.empty? ? nil : same_as, **attrs)
105
+ end
106
+
107
+ # @param type [String] "SoftwareApplication", "WebApplication" or "MobileApplication"
108
+ # @return [self]
109
+ def software_application(name:, url: "#{origin}/", type: "SoftwareApplication", **attrs)
110
+ node(type, anchor(url, "app"), name: name, url: url, **attrs)
111
+ end
112
+
113
+ # @return [self]
114
+ def web_application(name:, **attrs)
115
+ software_application(name: name, type: "WebApplication", **attrs)
116
+ end
117
+
118
+ # @param type [String] "Article", "BlogPosting" or "NewsArticle"
119
+ # @return [self]
120
+ def article(headline:, url: "#{origin}/", type: "Article", **attrs)
121
+ node(type, anchor(url, "article"), headline: headline, url: url, main_entity_of_page: url, **attrs)
122
+ end
123
+
124
+ # @param items [Array<Array(String, String)>, Array<Hash>] +[name, url]+ pairs or +{name:, url:}+;
125
+ # the last item may omit its URL (the current page)
126
+ # @return [self]
127
+ def breadcrumb_list(items, url: "#{origin}/")
128
+ elements = Array(items).each_with_index.map do |item, index|
129
+ name, item_url = item.is_a?(Hash) ? [item[:name] || item["name"], item[:url] || item["url"]] : item
130
+ raise InvalidNode, "BreadcrumbList item #{index + 1} needs a name" if blank?(name)
131
+
132
+ { type: "ListItem", position: index + 1, name: name, item: item_url }
133
+ end
134
+ node("BreadcrumbList", anchor(url, "breadcrumb"), item_list_element: elements.empty? ? nil : elements)
135
+ end
136
+
137
+ # @return [self]
138
+ def video_object(name:, thumbnail_url:, upload_date:, url: "#{origin}/", **attrs)
139
+ node("VideoObject", anchor(url, "video"), name: name, thumbnail_url: thumbnail_url,
140
+ upload_date: upload_date, **attrs)
141
+ end
142
+
143
+ # Google requires one of +offers+, +review+ or +aggregate_rating+ for product snippets.
144
+ # @return [self]
145
+ def product(name:, url: "#{origin}/", **attrs)
146
+ unless %i[offers review aggregate_rating].any? { |key| attrs[key] }
147
+ raise InvalidNode, "Product #{name.inspect} needs offers, review or aggregate_rating"
148
+ end
149
+
150
+ node("Product", anchor(url, "product"), name: name, url: url, **attrs)
151
+ end
152
+
153
+ # @param main_entity [Hash] the Person or Organization the page is about (+name:+ required)
154
+ # @return [self]
155
+ def profile_page(main_entity:, url: "#{origin}/", **attrs)
156
+ entity = { type: "Person" }.merge(main_entity.transform_keys(&:to_sym))
157
+ raise InvalidNode, "ProfilePage main_entity needs a name" if blank?(entity[:name])
158
+
159
+ node("ProfilePage", anchor(url, "profile"), url: url, main_entity: entity, **attrs)
160
+ end
161
+
162
+ # @param faq [Array<Hash>] +{question:, answer:}+ items, e.g. {Page#faq}
163
+ # @return [self]
164
+ def faq_page(faq, url: "#{origin}/", **attrs)
165
+ questions = Array(faq).each_with_index.map do |item, index|
166
+ question = item[:question] || item["question"]
167
+ answer = item[:answer] || item["answer"]
168
+ raise InvalidNode, "FAQPage item #{index + 1} needs a question and an answer" if blank?(question) || blank?(answer)
169
+
170
+ { type: "Question", name: question, accepted_answer: { type: "Answer", text: answer } }
171
+ end
172
+ node("FAQPage", anchor(url, "faq"), main_entity: questions.empty? ? nil : questions, **attrs)
173
+ end
174
+
175
+ # @return [Hash{String => Object}] +{"@context" => ..., "@graph" => [...]}+
176
+ def to_h
177
+ { "@context" => CONTEXT, "@graph" => nodes }
178
+ end
179
+ alias as_json to_h
180
+
181
+ # Plain JSON. Use {SeoGeo::Meta::Helper#seo_geo_json_ld_tag} (or +ERB::Util.json_escape+) to put it in HTML.
182
+ # @return [String]
183
+ def to_json(*)
184
+ JSON.generate(to_h)
185
+ end
186
+
187
+ # @param type [String] schema.org type
188
+ # @param id [String, nil] +@id+
189
+ # @return [self]
190
+ def node(type, id, **attrs)
191
+ push(normalize({ type: type, id: id }.merge(attrs)))
192
+ self
193
+ end
194
+
195
+ private
196
+
197
+ def push(hash)
198
+ validate!(hash)
199
+ @nodes.reject! { |existing| hash["@id"] && existing["@id"] == hash["@id"] }
200
+ @nodes << hash
201
+ end
202
+
203
+ def validate!(hash)
204
+ missing = REQUIRED.fetch(hash["@type"].to_s, []).select { |key| blank?(hash[key]) }
205
+ return if missing.empty?
206
+
207
+ raise InvalidNode, "#{hash["@type"]} #{hash["@id"]} is missing #{missing.join(", ")}"
208
+ end
209
+
210
+ def anchor(url, name)
211
+ "#{absolute(url.to_s).sub(/#.*\z/, "")}##{name}"
212
+ end
213
+
214
+ def normalize(value, key = nil)
215
+ case value
216
+ when Hash
217
+ value.each_with_object({}) do |(name, inner), out|
218
+ property = property_name(name)
219
+ normalized = normalize(inner, property)
220
+ out[property] = normalized unless normalized.nil?
221
+ end
222
+ when Array then value.map { |inner| normalize(inner, key) }
223
+ when Time, DateTime then value.iso8601
224
+ when Date then value.iso8601
225
+ when Symbol then value.to_s
226
+ when String then URL_KEYS.include?(key) ? absolute(value) : value
227
+ else value.respond_to?(:iso8601) ? value.iso8601 : value
228
+ end
229
+ end
230
+
231
+ def property_name(name)
232
+ return name if name.is_a?(String)
233
+ return "@#{name}" if %i[type id context graph].include?(name)
234
+
235
+ name.to_s.gsub(/_([a-z\d])/) { Regexp.last_match(1).upcase }
236
+ end
237
+
238
+ def absolute(value)
239
+ value.start_with?("/") && !value.start_with?("//") ? "#{origin}#{value}" : value
240
+ end
241
+
242
+ def blank?(value)
243
+ value.nil? || (value.respond_to?(:empty?) && value.empty?)
244
+ end
245
+ end
246
+
247
+ # JSON-LD built from the page copy, so FAQPage always matches the visible FAQ.
248
+ class Page
249
+ # @param url [String, nil] absolute page URL used for +@id+ anchors; defaults to the canonical origin root
250
+ # @param config [Configuration]
251
+ # @return [JsonLd] a document holding the FAQPage node (empty when the page has no FAQ)
252
+ def json_ld(url: nil, config: SeoGeo.config)
253
+ document = JsonLd.new(config: config)
254
+ return document if faq.empty?
255
+
256
+ document.faq_page(faq, url: url || "#{document.origin}/")
257
+ end
258
+ end
259
+ end
@@ -0,0 +1,183 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "uri"
4
+ require "set"
5
+
6
+ module SeoGeo
7
+ # Builds /llms.txt (https://llmstxt.org/) and /llms-full.txt for one request host.
8
+ #
9
+ # llms.txt: +# site.name+, +> site.description+, optional {LlmsTxtOptions#details}, then one H2 per
10
+ # section with +- [title](url): description+ lines and +## Optional+ last. Page links point at the
11
+ # page's Markdown (+.md+) when {MarkdownOptions#enabled}. Links come from {Configuration#llms_txt};
12
+ # when none are declared, the host's static sitemap paths are listed.
13
+ #
14
+ # llms-full.txt (community convention, not in the llmstxt.org text): the same header followed by
15
+ # every listed page's {Page#to_markdown}, headings demoted one level under one H1.
16
+ #
17
+ # Both are served only for hosts whose sitemap is on ({HostPolicy#sitemap}); otherwise nil (=> 404).
18
+ class LlmsTxt
19
+ OPTIONAL_SECTION = "Optional"
20
+
21
+ # @param config [Configuration]
22
+ # @param host [String] request host
23
+ # @param pages [PageRegistry]
24
+ # @param sitemap [Sitemap::Registry] fallback link source
25
+ def initialize(config, host, pages: SeoGeo.pages, sitemap: SeoGeo.sitemap)
26
+ @config = config
27
+ @host = host
28
+ @pages = pages
29
+ @sitemap = sitemap
30
+ @options = config.llms_txt
31
+ @policy = config.host_policy(host)
32
+ end
33
+
34
+ # @return [String, nil] llms.txt body, nil when not served on this host
35
+ def to_s
36
+ return nil unless served?
37
+
38
+ parts = header
39
+ sections.each do |name, lines|
40
+ parts << "## #{name}\n\n#{lines.join("\n")}"
41
+ end
42
+ "#{parts.join("\n\n")}\n"
43
+ end
44
+
45
+ # @return [String, nil] llms-full.txt body, nil when not served or no page is listed
46
+ def full
47
+ return nil unless served? && @options.full
48
+
49
+ page_links = links.select(&:page)
50
+ return nil if page_links.empty?
51
+
52
+ parts = header
53
+ page_links.each do |item|
54
+ markdown = page_for(item).to_markdown.gsub(/^(#+) /) { "##{Regexp.last_match(1)} " }
55
+ parts << "---\n\nSource: #{absolute(item.path)}\n\n#{markdown.strip}"
56
+ end
57
+ "#{parts.join("\n\n")}\n"
58
+ end
59
+
60
+ private
61
+
62
+ def served?
63
+ @options.enabled && @policy.sitemap
64
+ end
65
+
66
+ def header
67
+ name = present(@config.site.name) || @config.canonical_host || @policy.host
68
+ parts = ["# #{name}"]
69
+ description = present(@config.site.description)
70
+ parts << "> #{description.gsub(/\s+/, " ")}" if description
71
+ details = present(@options.details)
72
+ parts << details_markdown(details) if details
73
+ parts
74
+ end
75
+
76
+ # @return [Array<Array(String, Array<String>)>] section name => link lines, Optional last
77
+ def sections
78
+ grouped = links.group_by { |item| item.optional ? OPTIONAL_SECTION : (item.section || @options.default_section) }
79
+ optional = grouped.delete(OPTIONAL_SECTION)
80
+ grouped = grouped.to_a
81
+ grouped << [OPTIONAL_SECTION, optional] if optional
82
+ grouped.map { |name, items| [name, items.map { |item| line(item) }] }
83
+ end
84
+
85
+ def line(item)
86
+ page = item.page && page_for(item)
87
+ title = item.title || page&.title || page&.heading || item.path
88
+ description = item.description || page&.description
89
+ url = absolute(item.path)
90
+ url = markdown_url(url) if item.markdown_link?(@config) && same_origin?(URI.parse(url))
91
+ text = "- [#{title.to_s.gsub(/[\[\]]/) { |char| "\\#{char}" }}](#{url})"
92
+ description ? "#{text}: #{description.to_s.gsub(/\s+/, " ").strip}" : text
93
+ end
94
+
95
+ def links
96
+ @links ||= begin
97
+ declared = @options.links.select { |item| for_host?(item.hosts) }
98
+ (@options.links.empty? ? sitemap_links : declared).select { |item| publishable?(item.path) }
99
+ end
100
+ end
101
+
102
+ def sitemap_links
103
+ @sitemap.entries.select { |entry| entry.static? && entry.for_host?(@host, @config) && !@sitemap.excluded?(entry.source) }
104
+ .map do |entry|
105
+ LlmsTxtOptions::Link.new(page: nil, title: nil, path: entry.source, description: nil, section: nil,
106
+ optional: false, hosts: nil, markdown: entry.markdown)
107
+ end
108
+ end
109
+
110
+ # llms.txt and llms-full.txt list only what {SeoGeo::Publication} allows (the sitemap.xml and
111
+ # head rule: robots disallow, exclusions, noindex declarations, default_noindex). Links to other
112
+ # origins are not ours to judge and pass through.
113
+ def publishable?(path)
114
+ if path.match?(%r{\Ahttps?://}i)
115
+ return true unless same_origin?(URI.parse(path))
116
+ end
117
+ # /x.md (also /x.md?q) is the Markdown form of /x (Markdown::PathSuffix), so it is published only when /x is.
118
+ # The query stays for robots rules (Disallow: /x?preview=1); declarations drop it themselves.
119
+ page_path = RobotsPath.html_target(path, markdown: @config.markdown.suffix_urls?)
120
+ Publication.publishable?(@host, page_path, config: @config, sitemap: @sitemap, check_host: false,
121
+ declared_index: indexed_by_sitemap?(page_path))
122
+ rescue URI::InvalidURIError
123
+ false
124
+ end
125
+
126
+ def same_origin?(uri)
127
+ origin = URI.parse(@config.origin_for(@host))
128
+ uri.host && uri.host.casecmp?(origin.host) && uri.scheme.casecmp?(origin.scheme) && uri.port == origin.port
129
+ end
130
+
131
+ # Under default_noindex, records of a collection added with index: true are declared indexable too;
132
+ # only sitemap.xml knows their paths, so ask it (enumerated once per llms.txt render).
133
+ def indexed_by_sitemap?(path)
134
+ return false unless @config.default_noindex
135
+
136
+ @indexed_paths ||= Sitemap::Xml.new(@config, @sitemap, @host).urls
137
+ .to_set { |url| RobotsPath.path(url.loc) }
138
+ @indexed_paths.include?(RobotsPath.path(path))
139
+ end
140
+
141
+ def for_host?(hosts)
142
+ wanted = hosts ? hosts.map(&:downcase) : [@config.canonical_host]
143
+ wanted.include?(@policy.host)
144
+ end
145
+
146
+ def page_for(item)
147
+ @pages.fetch(item.page, locale: @config.site.default_locale)
148
+ end
149
+
150
+ def absolute(path)
151
+ return path if path.match?(%r{\Ahttps?://})
152
+
153
+ @config.canonical_url("#{@config.origin_for(@host)}#{path}")
154
+ end
155
+
156
+ # "/" => "/index.md", "/pricing" => "/pricing.md" (llmstxt.org convention); query dropped.
157
+ def markdown_url(url)
158
+ uri = URI.parse(url)
159
+ path = uri.path.to_s
160
+ path = path.empty? || path.end_with?("/") ? "#{path.chomp("/")}/index.md" : "#{path}.md"
161
+ uri.path = path
162
+ uri.query = nil
163
+ uri.fragment = nil
164
+ uri.to_s
165
+ rescue URI::InvalidURIError
166
+ url
167
+ end
168
+
169
+ # Details are free Markdown (paragraphs, lists, headings). In llms.txt the single H1 is the site name and H2s are
170
+ # link sections, so H1/H2 headings in details are demoted to H3 (outside fenced code); H3+ stay as written.
171
+ def details_markdown(text)
172
+ fenced = false
173
+ text.strip.lines.map do |line|
174
+ fenced = !fenced if line.start_with?("```", "~~~")
175
+ fenced ? line : line.sub(/\A\#{1,2}(?=\s)/, "###")
176
+ end.join
177
+ end
178
+
179
+ def present(value)
180
+ value.nil? || value.to_s.strip.empty? ? nil : value.to_s
181
+ end
182
+ end
183
+ end
@@ -0,0 +1,163 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+ require "yaml"
5
+ require "nokogiri"
6
+
7
+ module SeoGeo
8
+ module Markdown
9
+ # Turns a rendered HTML page into a Markdown document for agents:
10
+ #
11
+ # ---
12
+ # title: ... # <title>, else page copy
13
+ # description: ... # meta description
14
+ # canonical: ... # link rel=canonical, else config.canonical_url(request URL)
15
+ # image: ... # og:image
16
+ # locale: ... # <html lang>
17
+ # ---
18
+ #
19
+ # <main content converted to Markdown>
20
+ #
21
+ # ```json
22
+ # <each JSON-LD block>
23
+ # ```
24
+ #
25
+ # When the HTML has no main content (an Inertia/SPA mount point) and a {SeoGeo::Page} is given, the
26
+ # body comes from {SeoGeo::Page#to_markdown}, so agents see the same copy as the crawler fallback.
27
+ class Document
28
+ # Removed before conversion wherever they appear.
29
+ NOISE = %w[script style noscript template iframe svg canvas nav form button dialog].freeze
30
+ # Removed as well when no main element exists and the whole body is converted.
31
+ CHROME = %w[header footer aside].freeze
32
+
33
+ # @return [String]
34
+ attr_reader :html
35
+
36
+ # @param html [String] the rendered HTML response body
37
+ # @param config [Configuration]
38
+ # @param url [String, nil] request URL, used for the canonical fallback
39
+ # @param page [Page, nil] page copy for client-rendered pages
40
+ # @param selector [String, nil] main-content selector; nil uses +config.markdown.selector+
41
+ def initialize(html, config: SeoGeo.config, url: nil, page: nil, selector: nil)
42
+ @html = html.to_s
43
+ @config = config
44
+ @url = url
45
+ @page = page
46
+ @selector = selector || config.markdown.selector
47
+ end
48
+
49
+ # @return [String] the full Markdown document
50
+ def to_s
51
+ body = body_markdown
52
+ parts = [frontmatter, body.strip]
53
+ json_ld.each { |block| parts << "```json\n#{block}\n```" }
54
+ "#{parts.reject(&:empty?).join("\n\n")}\n"
55
+ end
56
+
57
+ # @return [Hash{String => String}] frontmatter fields that have a value
58
+ def metadata
59
+ {
60
+ "title" => text_at("head > title") || @page&.title,
61
+ "description" => attribute_at('meta[name="description"]', "content") || @page&.description,
62
+ "canonical" => attribute_at('link[rel="canonical"]', "href") || canonical_fallback,
63
+ "image" => attribute_at('meta[property="og:image"]', "content"),
64
+ "locale" => attribute_at("html", "lang") || @page&.locale&.to_s
65
+ }.reject { |_name, value| value.nil? || value.to_s.strip.empty? }
66
+ end
67
+
68
+ # @return [RobotsDirectives] every robots and crawler-specific meta of the HTML
69
+ def robots
70
+ @robots ||= RobotsDirectives.from_html(doc)
71
+ end
72
+
73
+ # Global directives (every +<meta name="robots">+), lower-cased, +none+ expanded to +noindex, nofollow+.
74
+ # @return [Array<String>]
75
+ def robots_directives
76
+ robots.global.to_a
77
+ end
78
+
79
+ # @return [Boolean] true when the HTML asks every robot not to index the page
80
+ def noindex?
81
+ robots.noindex?
82
+ end
83
+
84
+ # @return [Array<String>] pretty-printed JSON-LD blocks
85
+ def json_ld
86
+ doc.css('script[type="application/ld+json"]').filter_map do |node|
87
+ raw = node.text.strip
88
+ next if raw.empty?
89
+
90
+ begin
91
+ JSON.pretty_generate(JSON.parse(raw))
92
+ rescue JSON::ParserError
93
+ raw
94
+ end
95
+ end
96
+ end
97
+
98
+ private
99
+
100
+ def doc
101
+ @doc ||= if defined?(Nokogiri::HTML5) then Nokogiri::HTML5(@html)
102
+ else Nokogiri::HTML(@html)
103
+ end
104
+ end
105
+
106
+ def frontmatter
107
+ data = metadata
108
+ return "" if data.empty?
109
+
110
+ "#{YAML.dump(data).sub(/\A---\s*\n/, "---\n")}---"
111
+ end
112
+
113
+ def body_markdown
114
+ from_html = convert(main_html)
115
+ return from_html unless from_html.strip.empty? && @page
116
+
117
+ @page.to_markdown
118
+ end
119
+
120
+ # HTML of the main content with noise removed. Works on a copy so metadata/JSON-LD stay readable.
121
+ def main_html
122
+ copy = doc.dup
123
+ main = copy.at_css(@selector)
124
+ root = main || copy.at_css("body")
125
+ return "" if root.nil?
126
+
127
+ removable = main ? NOISE : NOISE + CHROME
128
+ root.css(removable.join(",")).each(&:remove)
129
+ root.inner_html
130
+ end
131
+
132
+ def convert(fragment)
133
+ return "" if fragment.strip.empty?
134
+
135
+ converter = @config.markdown.converter
136
+ markdown = if converter.respond_to?(:call)
137
+ converter.call(fragment)
138
+ else
139
+ require "reverse_markdown"
140
+ ReverseMarkdown.convert(fragment, unknown_tags: :bypass, github_flavored: true)
141
+ end
142
+ markdown.to_s.gsub(/\n{3,}/, "\n\n")
143
+ end
144
+
145
+ def text_at(selector)
146
+ node = doc.at_css(selector)
147
+ value = node&.text&.strip
148
+ value unless value.nil? || value.empty?
149
+ end
150
+
151
+ def attribute_at(selector, name)
152
+ value = doc.at_css(selector)&.[](name)&.strip
153
+ value unless value.nil? || value.empty?
154
+ end
155
+
156
+ def canonical_fallback
157
+ return nil if @url.nil? || @config.canonical_origin.nil?
158
+
159
+ @config.canonical_url(@url)
160
+ end
161
+ end
162
+ end
163
+ end
@@ -0,0 +1,49 @@
1
+ # frozen_string_literal: true
2
+
3
+ module SeoGeo
4
+ module Markdown
5
+ # Decides whether an +Accept+ header asks for Markdown.
6
+ #
7
+ # Markdown wins only when +text/markdown+ is named explicitly (wildcards never select it) with a
8
+ # q-value above zero that is higher than the one +text/html+ gets, or equal to it and listed first.
9
+ # Agents send +text/markdown, text/html;q=0.9+ or +text/markdown, text/html, *\/*+; browsers never
10
+ # name text/markdown, so they keep getting HTML.
11
+ module Negotiation
12
+ module_function
13
+
14
+ # @param accept [String, nil] raw Accept header
15
+ # @return [Boolean]
16
+ def markdown?(accept)
17
+ ranges = parse(accept)
18
+ markdown = ranges.find { |range| range[:type] == "text/markdown" }
19
+ return false if markdown.nil? || markdown[:q] <= 0
20
+
21
+ html = quality_for_html(ranges)
22
+ return true if html.nil?
23
+
24
+ markdown[:q] > html[:q] || (markdown[:q] == html[:q] && markdown[:index] < html[:index])
25
+ end
26
+
27
+ # @param accept [String, nil]
28
+ # @return [Array<Hash>] +{type:, q:, index:}+ in header order
29
+ def parse(accept)
30
+ accept.to_s.split(",").each_with_index.filter_map do |part, index|
31
+ type, *params = part.split(";").map(&:strip)
32
+ next if type.nil? || type.empty?
33
+
34
+ q = params.filter_map { |param| param[/\Aq\s*=\s*([0-9.]+)\z/i, 1] }.first
35
+ { type: type.downcase, q: q ? q.to_f.clamp(0.0, 1.0) : 1.0, index: index }
36
+ end
37
+ end
38
+
39
+ # The most specific range matching text/html (RFC 9110 §12.5.1).
40
+ def quality_for_html(ranges)
41
+ %w[text/html text/* */*].each do |type|
42
+ found = ranges.find { |range| range[:type] == type }
43
+ return found if found
44
+ end
45
+ nil
46
+ end
47
+ end
48
+ end
49
+ end