abqari 1.1.0

data/lib/abqari/robots.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+module Abqari
+  class Robots
+    # AI training crawlers that explicitly identify themselves and
+    # respect robots.txt. Blocked when `block_ai_scraping: true` is set
+    # in config (the default). Bots that ignore robots.txt won't be
+    # stopped by anything in this file — but the major AI training
+    # crawlers (OpenAI, Anthropic, Google's AI carve-out, Meta, Apple,
+    # Perplexity, Common Crawl, ByteDance, Cohere) all honour it.
+    #
+    # Notes:
+    #   - `Google-Extended` is Google's separate user-agent for AI
+    #     training. Blocking it does NOT affect regular Googlebot
+    #     search indexing.
+    #   - `Applebot-Extended` is the same idea for Apple.
+    #   - `CCBot` is Common Crawl, used by many third-party trainers.
+    #   - User-facing fetchers (ChatGPT-User, Perplexity's on-demand
+    #     fetcher) ARE included — block-all-AI is the safer default;
+    #     remove specific bots from this list if you want their search
+    #     citations.
+    AI_BOTS = %w[
+      GPTBot
+      ChatGPT-User
+      OAI-SearchBot
+      ClaudeBot
+      Claude-Web
+      anthropic-ai
+      Google-Extended
+      Applebot-Extended
+      Meta-ExternalAgent
+      FacebookBot
+      PerplexityBot
+      Perplexity-User
+      YouBot
+      cohere-ai
+      Bytespider
+      MistralAI-User
+      CCBot
+      Diffbot
+      omgili
+      omgilibot
+    ].freeze
+
+    def initialize(site)
+      @site = site
+    end
+
+    def write
+      File.write(File.join(@site.output_dir, 'robots.txt'), build)
+    end
+
+    private
+
+    def build
+      indexable = @site.config['indexable']
+      base = @site.config['url']&.sub(%r{/\z}, '')
+
+      if indexable == false
+        <<~ROBOTS
+          # Indexing disabled in #{@site.env}.
+          User-agent: *
+          Disallow: /
+        ROBOTS
+      else
+        sections = []
+        sections << ai_block if block_ai_scraping?
+        sections << build_default(base)
+        sections.join("\n")
+      end
+    end
+
+    def build_default(base)
+      lines = ['User-agent: *', 'Allow: /']
+      Array(@site.config['robots_disallow']).each { |path| lines << "Disallow: #{path}" }
+      lines << ''
+      lines << "Sitemap: #{base}/sitemap.xml" if base
+      "#{lines.join("\n")}\n"
+    end
+
+    def ai_block
+      lines = ['# Block AI training crawlers (config: block_ai_scraping)']
+      AI_BOTS.each { |bot| lines << "User-agent: #{bot}" }
+      lines << 'Disallow: /'
+      lines << ''
+      "#{lines.join("\n")}\n"
+    end
+
+    def block_ai_scraping?
+      @site.config['block_ai_scraping']
+    end
+  end
+end

data/lib/abqari/search.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+
+module Abqari
+  # Static search via Pagefind. After bin/build emits HTML, Pagefind walks
+  # _site/, builds a chunked search index under _site/pagefind/, and ships
+  # a tiny JS runtime that runs entirely in the visitor's browser.
+  #
+  # Privacy-friendly (no server, no API), free, and the index is just static
+  # files served alongside your site.
+  #
+  # Pagefind is a single Rust binary. Install it once:
+  #   macOS:    brew install pagefind
+  #   General:  curl -LSf https://pagefind.app/install.sh | bash
+  #   Cargo:    cargo install pagefind
+  #
+  # Enable in config/site.yml:
+  #   search: true
+  #
+  # Then render the search widget anywhere in your templates:
+  #   <%= render 'partials/search' %>
+  class Search
+    BINARY = 'pagefind'
+
+    def initialize(site)
+      @site = site
+    end
+
+    def enabled?
+      @site.config['search'] == true
+    end
+
+    # Called from Site#build — only runs when `search: true` is set so a
+    # default site doesn't pay the cost.
+    def run
+      return unless enabled?
+
+      run!
+    end
+
+    # Always runs pagefind, regardless of the config flag. Used by bin/index
+    # so an explicit invocation works even when search isn't enabled in config.
+    # Skips with a helpful warning if the binary isn't on PATH; never raises
+    # so a fresh checkout never fails the build.
+    def run!
+      unless command_available?
+        Log.warn "Abqari: `#{BINARY}` is not on PATH."
+        Log.warn '       Install: https://pagefind.app/docs/installation/'
+        Log.warn '       Skipping search index.'
+        return
+      end
+
+      success = system(BINARY, '--site', @site.output_dir, out: File::NULL, err: File::NULL)
+      if success
+        Log.info 'Indexed for search → _site/pagefind/'
+      else
+        Log.warn "Abqari: pagefind exited with status #{$?.exitstatus}; search may be incomplete."
+      end
+    end
+
+    private
+
+    def command_available?
+      ENV['PATH'].split(File::PATH_SEPARATOR).any? { |dir| File.executable?(File.join(dir, BINARY)) }
+    end
+  end
+end

data/lib/abqari/security_txt.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require 'date'
+require 'fileutils'
+
+module Abqari
+  class SecurityTxt
+    def initialize(site)
+      @site = site
+    end
+
+    def write
+      contact = @site.config['security_contact']
+      return unless contact
+
+      dest = File.join(@site.output_dir, '.well-known', 'security.txt')
+      FileUtils.mkdir_p(File.dirname(dest))
+      File.write(dest, build(contact))
+    end
+
+    private
+
+    def build(contact)
+      expires = (Date.today + 365).strftime('%Y-%m-%dT00:00:00.000Z')
+
+      locale = @site.config['locale'] || 'en'
+      lines = ["Contact: #{contact_uri(contact)}", "Expires: #{expires}", "Preferred-Languages: #{locale}"]
+      lines << "Canonical: #{@site.config['url']&.sub(%r{/\z}, '')}/.well-known/security.txt" if @site.config['url']
+      "#{lines.join("\n")}\n"
+    end
+
+    # RFC 9116 requires the Contact value to be a URI. A bare email
+    # address (`security@example.com`) is invalid and parsers reject the
+    # whole file — normalise it to `mailto:`. Values that already carry a
+    # scheme (mailto:, https://, tel:) pass through untouched.
+    def contact_uri(contact)
+      value = contact.to_s.strip
+      return value if value.match?(%r{\A[a-z][a-z0-9+.-]*:}i) # already has a scheme
+      return "mailto:#{value}" if value.include?('@')
+
+      value
+    end
+  end
+end