unaltraweb 0.3.0

data/_plugins/content_search_index.rb

@@ -0,0 +1,210 @@
+# frozen_string_literal: true
+
+require "cgi"
+require "json"
+require "nokogiri"
+
+module Unaltraweb
+  module ContentSearchIndex
+    module_function
+
+    SUPPORTED_PROFILES = %w[unaltreselfie unaltreprojecte unaltremanual unaltredocs].freeze
+    INDEX_PATH = "/assets/js/content-search-index.json"
+    EXCLUDED_TAGS = %w[script style template noscript textarea input select button nav].freeze
+    BLOCK_TAGS = %w[address article aside blockquote div dl dt dd fieldset figcaption figure footer form h1 h2 h3 h4 h5 h6 header hr li main ol p pre section table tbody td tfoot th thead tr ul].freeze
+    DYNAMIC_SOURCE_LANGUAGES = %w[language-mermaid language-vega_lite language-plotly language-echarts language-geojson language-diff2html].freeze
+
+    def active_profile(site)
+      config = site.config["unaltraweb"] || {}
+      (config["site_profile"] || "unaltreprojecte").to_s
+    end
+
+    def add_page(site)
+      profile = active_profile(site)
+      return unless SUPPORTED_PROFILES.include?(profile)
+      if index_collision?(site)
+        raise Jekyll::Errors::FatalException, "Content search index conflicts with a consumer-managed #{INDEX_PATH}"
+      end
+
+      page = Jekyll::PageWithoutAFile.new(site, site.source, "assets/js", "content-search-index.json")
+      page.content = "[]\n"
+      page.data["layout"] = nil
+      page.data["sitemap"] = false
+      page.data["search_exclude"] = true
+      page.data["content_search_index"] = true
+      site.pages << page
+    end
+
+    def finalize_page(site)
+      page = Array(site.pages).find do |candidate|
+        candidate.url.to_s == INDEX_PATH && candidate.data["content_search_index"] == true
+      end
+      return unless page
+
+      payload = "#{JSON.pretty_generate(content_entries(site))}\n"
+      page.content = payload
+      page.output = payload
+    end
+
+    def index_collision?(site)
+      page_collision = Array(site.pages).any? { |page| page.url.to_s == INDEX_PATH }
+      static_collision = Array(site.static_files).any? { |file| file.relative_path.to_s == INDEX_PATH }
+      page_collision || static_collision
+    end
+
+    def content_entries(site)
+      profile = active_profile(site)
+      documents = site.collections.values.flat_map do |collection|
+        collection_output?(collection) ? collection.docs : []
+      end
+      items = (site.pages + documents).select { |item| searchable?(item, profile) }
+      items = items.sort_by { |item| [item.url.to_s, item.data["title"].to_s] }
+                   .uniq { |item| item.url.to_s }
+
+      items.map { |item| entry(site, item) }
+    end
+
+    def collection_output?(collection)
+      metadata = collection.respond_to?(:metadata) ? collection.metadata : {}
+      metadata.fetch("output", false) == true
+    end
+
+    def searchable?(item, profile)
+      data = item.data || {}
+      return false if data["title"].to_s.strip.empty? || item.url.to_s.strip.empty?
+      return false if data["published"] == false || data["published"].to_s == "false"
+      return false if truthy?(data["search_exclude"])
+
+      profile_match?(item, [profile])
+    end
+
+    def entry(site, item)
+      segments = rendered_segments(site, item)
+      if segments.empty? && (!item.respond_to?(:output) || item.output.to_s.empty?)
+        segments = [normalize_text(item.content)].reject(&:empty?)
+      end
+      documentation_profiles = metadata_values(item, "documentation_profiles")
+
+      {
+        title: item.data["title"].to_s,
+        description: item.data["description"].to_s,
+        lang: item.data["lang"].to_s.empty? ? (site.config["default_lang"] || site.config["lang"]).to_s : item.data["lang"].to_s,
+        url: relative_url(site, item.url),
+        section: item.data["section"].to_s,
+        subsection: item.data["subsection"].to_s,
+        keywords: Array(item.data["keywords"] || item.data["tags"]).join(" "),
+        documentation_profiles: documentation_profiles,
+        documentation_profile_slugs: documentation_profiles.map { |value| Jekyll::Utils.slugify(value) },
+        body: segments.join(" "),
+        segments: segments,
+      }
+    end
+
+    def metadata_values(item, *keys)
+      keys.flat_map { |key| Array(item.data[key]) }
+          .map { |value| value.to_s.strip }
+          .reject(&:empty?)
+          .uniq
+    end
+
+    def rendered_segments(site, item)
+      output = item.respond_to?(:output) ? item.output.to_s : ""
+      return [] if output.empty?
+
+      document = Nokogiri::HTML::Document.parse(output)
+      root = search_root_selectors(site).filter_map { |selector| document.at_css(selector) }.first
+      return [] unless root
+
+      chunks = []
+      append_searchable_text(root, chunks)
+      body = normalize_segment(chunks.join)
+      body.empty? ? [] : [body]
+    end
+
+    def search_root_selectors(site)
+      case active_profile(site)
+      when "unaltremanual"
+        [".manual-main", "[data-content-search-root]"]
+      when "unaltredocs"
+        [".documentation-main", "[data-content-search-root]"]
+      else
+        ["[data-content-search-root]"]
+      end
+    end
+
+    def append_searchable_text(node, chunks)
+      node.children.each do |child|
+        if child.text?
+          chunks << child.text
+          next
+        end
+        next unless child.element?
+        next if excluded_element?(child)
+
+        block = BLOCK_TAGS.include?(child.name.downcase) || child.name.downcase == "br"
+        chunks << " " if block
+        append_searchable_text(child, chunks) unless child.name.downcase == "br"
+        chunks << " " if block
+      end
+    end
+
+    def excluded_element?(element)
+      classes = element["class"].to_s.split
+      EXCLUDED_TAGS.include?(element.name.downcase) ||
+        element.key?("hidden") ||
+        element.key?("inert") ||
+        element["aria-hidden"] == "true" ||
+        classes.include?("content-search-navigation") ||
+        classes.include?("documentation-index") ||
+        dynamic_source?(element, classes)
+    end
+
+    def dynamic_source?(element, classes)
+      element.name.downcase == "code" && !(classes & DYNAMIC_SOURCE_LANGUAGES).empty?
+    end
+
+    def normalize_segment(value)
+      value.to_s.gsub(/[[:space:]\u00a0]+/, " ").strip
+    end
+
+    def truthy?(value)
+      %w[true yes 1 selected].include?(value.to_s.strip.downcase)
+    end
+
+    def profile_match?(item, profiles)
+      item_profiles = Array(item.data["profiles"] || item.data["site_profiles"]).map(&:to_s)
+      item_profiles.empty? || (item_profiles & profiles).any?
+    end
+
+    def normalize_text(value)
+      text = value.to_s.dup
+      text.gsub!(/\{%.*?%\}/m, " ")
+      text.gsub!(/\{\{.*?\}\}/m, " ")
+      text.gsub!(/^[ \t]*(?:`{3,}|~{3,})[^\n]*$/, " ")
+      text.gsub!(/<[^>]+>/m, " ")
+      text.gsub!(/!\[[^\]]*\]\([^)]*\)/m, " ")
+      text.gsub!(/\[([^\]]+)\]\([^)]*\)/m, "\\1")
+      text.gsub!(/[#>*_`|{}\[\]()]/, " ")
+      CGI.unescapeHTML(text).gsub(/\s+/, " ").strip
+    end
+
+    def relative_url(site, url)
+      baseurl = site.config["baseurl"].to_s.chomp("/")
+      path = url.start_with?("/") ? url : "/#{url}"
+      "#{baseurl}#{path}"
+    end
+  end
+end
+
+class UnaltrawebContentSearchIndexGenerator < Jekyll::Generator
+  safe true
+  priority :low
+
+  def generate(site)
+    Unaltraweb::ContentSearchIndex.add_page(site)
+  end
+end
+
+Jekyll::Hooks.register :site, :post_render do |site|
+  Unaltraweb::ContentSearchIndex.finalize_page(site)
+end

data/_plugins/details.rb

@@ -0,0 +1,23 @@
+# Code from http://movb.de/jekyll-details-support.html
+
+module Jekyll
+  module Tags
+    class DetailsTag < Liquid::Block
+      def initialize(tag_name, markup, tokens)
+        super
+        @caption = markup
+      end
+
+      def render(context)
+        site = context.registers[:site]
+        converter = site.find_converter_instance(::Jekyll::Converters::Markdown)
+        caption = converter.convert(@caption).gsub(/<\/?p[^>]*>/, '').chomp
+        body = converter.convert(super(context))
+        "<details><summary>#{caption}</summary>#{body}</details>"
+      end
+
+    end
+  end
+end
+
+Liquid::Template.register_tag('details', Jekyll::Tags::DetailsTag)

data/_plugins/external-posts.rb

@@ -0,0 +1,124 @@
+require 'feedjira'
+require 'httparty'
+require 'jekyll'
+require 'nokogiri'
+require 'time'
+
+module ExternalPosts
+  class ExternalPostsGenerator < Jekyll::Generator
+    safe true
+    priority :high
+
+    def generate(site)
+      if site.config['external_sources'] != nil
+        site.config['external_sources'].each do |src|
+          puts "Fetching external posts from #{src['name']}:"
+          if src['rss_url']
+            fetch_from_rss(site, src)
+          elsif src['posts']
+            fetch_from_urls(site, src)
+          end
+        end
+      end
+    end
+
+    def fetch_from_rss(site, src)
+      xml = HTTParty.get(src['rss_url']).body
+      return if xml.nil?
+      begin
+        feed = Feedjira.parse(xml)
+      rescue StandardError => e
+        puts "Error parsing RSS feed from #{src['rss_url']} - #{e.message}"
+        return
+      end
+      process_entries(site, src, feed.entries)
+    end
+
+    def process_entries(site, src, entries)
+      entries.each do |e|
+        puts "...fetching #{e.url}"
+        create_document(site, src['name'], e.url, {
+          title: e.title,
+          content: e.content,
+          summary: e.summary,
+          published: e.published
+        }, src)
+      end
+    end
+
+    def create_document(site, source_name, url, content, src = {})
+      # check if title is composed only of whitespace or foreign characters
+      if content[:title].gsub(/[^\w]/, '').strip.empty?
+        # use the source name and last url segment as fallback
+        slug = "#{source_name.downcase.strip.gsub(' ', '-').gsub(/[^\w-]/, '')}-#{url.split('/').last}"
+      else
+        # parse title from the post or use the source name and last url segment as fallback
+        slug = content[:title].downcase.strip.gsub(' ', '-').gsub(/[^\w-]/, '')
+        slug = "#{source_name.downcase.strip.gsub(' ', '-').gsub(/[^\w-]/, '')}-#{url.split('/').last}" if slug.empty?
+      end
+
+      path = site.in_source_dir("_posts/#{slug}.md")
+      doc = Jekyll::Document.new(
+        path, { :site => site, :collection => site.collections['posts'] }
+      )
+      doc.data['external_source'] = source_name
+      doc.data['title'] = content[:title]
+      doc.data['feed_content'] = content[:content]
+      doc.data['description'] = content[:summary]
+      doc.data['date'] = content[:published]
+      doc.data['redirect'] = url
+      
+      # Apply default categories and tags from source configuration
+      if src['categories'] && src['categories'].is_a?(Array) && !src['categories'].empty?
+        doc.data['categories'] = src['categories']
+      end
+      if src['tags'] && src['tags'].is_a?(Array) && !src['tags'].empty?
+        doc.data['tags'] = src['tags']
+      end
+      
+      doc.content = content[:content]
+      site.collections['posts'].docs << doc
+    end
+
+    def fetch_from_urls(site, src)
+      src['posts'].each do |post|
+        puts "...fetching #{post['url']}"
+        content = fetch_content_from_url(post['url'])
+        content[:published] = parse_published_date(post['published_date'])
+        create_document(site, src['name'], post['url'], content, src)
+      end
+    end
+
+    def parse_published_date(published_date)
+      case published_date
+      when String
+        Time.parse(published_date).utc
+      when Date
+        published_date.to_time.utc
+      else
+        raise "Invalid date format for #{published_date}"
+      end
+    end
+
+    def fetch_content_from_url(url)
+      html = HTTParty.get(url).body
+      parsed_html = Nokogiri::HTML(html)
+
+      title = parsed_html.at('head title')&.text.strip || ''
+      description = parsed_html.at('head meta[name="description"]')&.attr('content')
+      description ||= parsed_html.at('head meta[name="og:description"]')&.attr('content')
+      description ||= parsed_html.at('head meta[property="og:description"]')&.attr('content')
+
+      body_content = parsed_html.search('p').map { |e| e.text }
+      body_content = body_content.join() || ''
+
+      {
+        title: title,
+        content: body_content,
+        summary: description
+        # Note: The published date is now added in the fetch_from_urls method.
+      }
+    end
+
+  end
+end