source_monitor 0.1.1

data/lib/source_monitor/release/changelog.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require "pathname"
+
+module SourceMonitor
+  module Release
+    class Changelog
+      MissingEntryError = Class.new(StandardError)
+
+      def initialize(path: default_path)
+        @path = Pathname.new(path)
+      end
+
+      def latest_entry
+        @latest_entry ||= begin
+          sections = extract_sections
+          heading = sections.keys.find { |key| key != "## Release Checklist" }
+          raise MissingEntryError, "Unable to find changelog entry after Release Checklist" unless heading
+
+          content = ([ heading ] + sections.fetch(heading)).join
+          content.rstrip
+        end
+      end
+
+      def annotation_for(version)
+        raise ArgumentError, "version must be provided" if version.to_s.strip.empty?
+
+        [ "SourceMonitor v#{version}", latest_entry ].join("\n\n")
+      end
+
+      private
+
+      attr_reader :path
+
+      def default_path
+        Pathname.new(__dir__).join("..", "..", "..", "CHANGELOG.md").expand_path
+      end
+
+      def extract_sections
+        sections = {}
+        current_heading = nil
+
+        File.foreach(path) do |line|
+          if line.start_with?("## ")
+            current_heading = line.strip
+            sections[current_heading] ||= []
+            next
+          end
+
+          next unless current_heading
+
+          sections[current_heading] << line
+        end
+
+        sections
+      end
+    end
+  end
+end

data/lib/source_monitor/release/runner.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require "tempfile"
+require_relative "changelog"
+
+module SourceMonitor
+  module Release
+    class Runner
+      CommandFailure = Class.new(StandardError)
+
+      QUALITY_COMMANDS = [
+        [ "bin/rubocop" ],
+        [ "bin/brakeman", "--no-pager" ],
+        [ "bin/test-coverage" ],
+        [ "bin/check-diff-coverage" ]
+      ].freeze
+      GEM_BUILD_COMMAND = [ "rbenv", "exec", "gem", "build", "source_monitor.gemspec" ].freeze
+
+      def initialize(version:, executor: Executor.new, changelog: Changelog.new)
+        @version = version
+        @executor = executor
+        @changelog = changelog
+      end
+
+      def call
+        validate_version!
+        run_commands(QUALITY_COMMANDS)
+        run_command(GEM_BUILD_COMMAND)
+        create_annotated_tag
+        true
+      end
+
+      private
+
+      attr_reader :version, :executor, :changelog
+
+      def run_commands(commands)
+        commands.each do |command|
+          run_command(command)
+        end
+      end
+
+      def run_command(command, env: {})
+        success = executor.run(command, env:)
+        return if success
+
+        raise CommandFailure, "Command failed: #{command.join(' ')}"
+      end
+
+      def create_annotated_tag
+        message = changelog.annotation_for(version)
+
+        Tempfile.create([ "feed-monitor-release", ".log" ]) do |file|
+          file.write(message)
+          file.flush
+          file.rewind
+
+          run_command([ "git", "tag", "-a", "v#{version}", "-F", file.path ])
+        end
+      end
+
+      def validate_version!
+        raise ArgumentError, "version must be provided" if version.to_s.strip.empty?
+      end
+
+      class Executor
+        def run(command, env: {})
+          system(env, *command)
+        end
+      end
+    end
+  end
+end

data/lib/source_monitor/scheduler.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+require "active_support/core_ext/numeric/time"
+require "source_monitor/fetching/stalled_fetch_reconciler"
+
+module SourceMonitor
+  class Scheduler
+    DEFAULT_BATCH_SIZE = 100
+    STALE_QUEUE_TIMEOUT = 10.minutes
+    ELIGIBLE_FETCH_STATUSES = %w[idle failed].freeze
+
+    def self.run(limit: DEFAULT_BATCH_SIZE, now: Time.current)
+      new(limit:, now:).run
+    end
+
+    def initialize(limit:, now:)
+      @limit = limit
+      @now = now
+    end
+
+    def run
+      payload = { limit: limit }
+      recovery = SourceMonitor::Fetching::StalledFetchReconciler.call(now:, stale_after: STALE_QUEUE_TIMEOUT)
+      payload[:stalled_recoveries] = recovery.recovered_source_ids.size
+      payload[:stalled_jobs_removed] = recovery.jobs_removed.size
+
+      ActiveSupport::Notifications.instrument("source_monitor.scheduler.run", payload) do
+        start_monotonic = SourceMonitor::Instrumentation.monotonic_time
+        source_ids = lock_due_source_ids
+        payload[:enqueued_count] = source_ids.size
+
+        source_ids.each do |source_id|
+          SourceMonitor::Fetching::FetchRunner.enqueue(source_id)
+        end
+
+        payload[:duration_ms] = ((SourceMonitor::Instrumentation.monotonic_time - start_monotonic) * 1000.0).round(2)
+
+        source_ids.size
+      end
+    end
+
+    private
+
+    attr_reader :limit, :now
+
+    def lock_due_source_ids
+      ids = []
+
+      SourceMonitor::Source.transaction do
+        rows = due_sources_relation
+        rows = rows.limit(limit) if limit
+        ids = rows.lock("FOR UPDATE SKIP LOCKED").pluck(:id)
+      end
+
+      ids
+    end
+
+    def due_sources_relation
+      SourceMonitor::Source
+        .active
+        .where(due_for_fetch_predicate)
+        .where(fetch_status_predicate)
+        .order(Arel.sql("next_fetch_at ASC NULLS FIRST"))
+    end
+
+    def due_for_fetch_predicate
+      table = SourceMonitor::Source.arel_table
+      table[:next_fetch_at].eq(nil).or(table[:next_fetch_at].lteq(now))
+    end
+
+    def fetch_status_predicate
+      table = SourceMonitor::Source.arel_table
+
+      eligible = table[:fetch_status].in(ELIGIBLE_FETCH_STATUSES)
+      stale_cutoff = now - STALE_QUEUE_TIMEOUT
+      stale_queued = table[:fetch_status].eq("queued").and(table[:updated_at].lteq(stale_cutoff))
+      stale_fetching = table[:fetch_status].eq("fetching").and(table[:last_fetch_started_at].lteq(stale_cutoff))
+
+      eligible.or(stale_queued).or(stale_fetching)
+    end
+  end
+end

data/lib/source_monitor/scrapers/base.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+require "active_support/core_ext/hash/deep_merge"
+require "active_support/hash_with_indifferent_access"
+require "active_support/core_ext/hash/keys"
+
+module SourceMonitor
+  module Scrapers
+    # Base class for content scrapers used by the engine.
+    #
+    # == Adapter Contract
+    # Subclasses must implement #call and return a Result object describing the
+    # outcome of a scrape attempt. Implementations receive an item, the owning
+    # source, and a normalized settings hash that merges default adapter
+    # settings, source-level overrides, and per-invocation overrides. All
+    # adapters should remain stateless and thread-safe, relying on injected
+    # collaborators (e.g. HTTP clients) instead of global configuration.
+    #
+    # Adapters should:
+    # * Perform any outbound HTTP work using the provided +http+ client.
+    # * Populate the Result with :html and :content payloads when successful.
+    # * Use :status to communicate :success, :partial, or :failed outcomes.
+    # * Capture additional diagnostics (headers, timings, etc.) in :metadata.
+    class Base
+      Result = Struct.new(:status, :html, :content, :metadata, keyword_init: true)
+
+      class << self
+        def call(item:, source:, settings: nil, http: SourceMonitor::HTTP)
+          new(item: item, source: source, settings: settings, http: http).call
+        end
+
+        def adapter_name
+          name.demodulize.sub(/Scraper\z/, "").underscore
+        end
+
+        def default_settings
+          {}
+        end
+      end
+
+      def initialize(item:, source:, settings: nil, http: SourceMonitor::HTTP)
+        @item = item
+        @source = source
+        @http = http
+        @settings = build_settings(settings)
+      end
+
+      def call
+        raise NotImplementedError, "#{self.class.name} must implement #call"
+      end
+
+      protected
+
+      attr_reader :item, :source, :http, :settings
+
+      private
+
+      def build_settings(overrides)
+        combined = normalize_settings(self.class.default_settings)
+          .deep_merge(normalize_settings(source_settings))
+
+        if overrides.present? && overrides.respond_to?(:to_hash)
+          combined = combined.deep_merge(normalize_settings(overrides.to_hash))
+        end
+
+        deep_indifferent_access(combined)
+      end
+
+      def source_settings
+        value = source&.scrape_settings
+        return {} unless value.respond_to?(:to_hash)
+
+        value.to_hash
+      end
+
+      def deep_indifferent_access(value)
+        case value
+        when Hash
+          value.each_with_object(ActiveSupport::HashWithIndifferentAccess.new) do |(key, val), memo|
+            memo[key] = deep_indifferent_access(val)
+          end
+        when Array
+          value.map { |element| deep_indifferent_access(element) }
+        else
+          value
+        end
+      end
+
+      def normalize_settings(value)
+        return value if value.nil?
+
+        case value
+        when Hash
+          value.each_with_object({}) do |(key, val), memo|
+            memo[key.to_s] = normalize_settings(val)
+          end
+        when Array
+          value.map { |element| normalize_settings(element) }
+        else
+          value
+        end
+      end
+    end
+  end
+end

data/lib/source_monitor/scrapers/fetchers/http_fetcher.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+require "active_support/core_ext/object/blank"
+
+module SourceMonitor
+  module Scrapers
+    module Fetchers
+      class HttpFetcher
+        Result = Struct.new(:status, :body, :headers, :http_status, :error, :message, keyword_init: true)
+
+        def initialize(http: SourceMonitor::HTTP)
+          @http = http
+        end
+
+        def fetch(url:, settings: nil)
+          response = connection(settings).get(url)
+
+          if success_status?(response.status)
+            Result.new(status: :success, body: response.body, headers: response.headers, http_status: response.status)
+          else
+            Result.new(
+              status: :failed,
+              http_status: response.status,
+              error: "http_error",
+              message: "Non-success HTTP status"
+            )
+          end
+        rescue Faraday::ClientError => error
+          Result.new(
+            status: :failed,
+            http_status: extract_status(error),
+            error: error.class.name,
+            message: error.message
+          )
+        rescue Faraday::Error => error
+          Result.new(status: :failed, error: error.class.name, message: error.message)
+        end
+
+        private
+
+        attr_reader :http
+
+        def connection(settings)
+          normalized = normalize_settings(settings)
+          http.client(
+            proxy: normalized[:proxy],
+            headers: normalized[:headers],
+            timeout: normalized[:timeout] || SourceMonitor::HTTP::DEFAULT_TIMEOUT,
+            open_timeout: normalized[:open_timeout] || SourceMonitor::HTTP::DEFAULT_OPEN_TIMEOUT
+          )
+        end
+
+        def normalize_settings(settings)
+          return {} unless settings
+
+          settings = settings.respond_to?(:to_h) ? settings.to_h : settings
+          {
+            headers: (settings[:headers] || {}).to_h,
+            timeout: settings[:timeout],
+            open_timeout: settings[:open_timeout],
+            proxy: settings[:proxy].presence
+          }
+        end
+
+        def success_status?(status)
+          status.to_i >= 200 && status.to_i < 300
+        end
+
+        def extract_status(error)
+          candidates = []
+
+          if error.respond_to?(:response_status)
+            candidates << error.response_status
+          end
+
+          if error.respond_to?(:response)
+            response = error.response
+            if response.respond_to?(:[]) && response[:status]
+              candidates << response[:status]
+            elsif response.is_a?(Hash)
+              candidates << response["status"]
+              candidates << response[:status]
+            end
+          end
+
+          if error.respond_to?(:message) && error.message
+            error.message.scan(/\d{3}/).each do |number|
+              candidates << number.to_i
+            end
+          end
+
+          candidates.compact.first
+        end
+      end
+    end
+  end
+end

data/lib/source_monitor/scrapers/parsers/readability_parser.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+require "readability"
+require "nokolexbor"
+require "active_support/core_ext/object/blank"
+
+module SourceMonitor
+  module Scrapers
+    module Parsers
+      class ReadabilityParser
+        Result = Struct.new(:status, :content, :strategy, :title, :metadata, keyword_init: true)
+
+        def parse(html:, selectors: nil, readability: nil)
+          document = ::Nokolexbor::HTML(html)
+          selectors_hash = normalize_hash(selectors)
+          readability_options = normalize_hash(readability)
+
+          content_html = extract_with_selectors(document, selectors_hash[:content])
+          strategy = content_html.present? ? :selectors : :readability
+
+          readability_doc = build_readability_document(html, readability_options)
+          content_html = readability_doc.content&.strip if content_html.blank?
+
+          status = content_html.present? ? :success : :partial
+
+          title = extract_title(document, selectors_hash[:title], readability_doc)
+          metadata = {}
+          if readability_doc.respond_to?(:content_length)
+            metadata[:readability_text_length] = readability_doc.content_length
+          end
+
+          Result.new(
+            status: status,
+            content: content_html.presence,
+            strategy: strategy,
+            title: title,
+            metadata: metadata.compact
+          )
+        rescue StandardError => error
+          Result.new(
+            status: :failed,
+            content: nil,
+            strategy: :readability,
+            title: nil,
+            metadata: { error: error.class.name, message: error.message }
+          )
+        end
+
+        private
+
+        def normalize_hash(value)
+          return {} unless value
+
+          hash = value.respond_to?(:to_h) ? value.to_h : value
+          hash.each_with_object({}) do |(key, val), memo|
+            memo[key.to_sym] = val
+          end
+        end
+
+        def extract_with_selectors(document, selectors)
+          fragments = Array(selectors).filter_map do |selector|
+            next if selector.blank?
+
+            nodes = document.css(selector.to_s)
+            next if nodes.empty?
+
+            nodes.map(&:to_html).join("\n")
+          end
+
+          return if fragments.empty?
+
+          fragments.join("\n")
+        end
+
+        def build_readability_document(html, options)
+          symbolized = options.each_with_object({}) do |(key, value), memo|
+            memo[key.to_sym] = value
+          end
+
+          ::Readability::Document.new(html, symbolized)
+        end
+
+        def extract_title(document, selectors, readability_doc)
+          Array(selectors).each do |selector|
+            next if selector.blank?
+
+            node = document.at_css(selector.to_s)
+            return node.text.strip if node&.text.present?
+          end
+
+          if readability_doc.respond_to?(:title)
+            title = readability_doc.title&.strip
+            return title if title.present?
+          end
+
+          document.at_css("title")&.text&.strip
+        end
+      end
+    end
+  end
+end

data/lib/source_monitor/scrapers/readability.rb

@@ -0,0 +1,156 @@
+# frozen_string_literal: true
+
+require "active_support/core_ext/object/blank"
+
+require "source_monitor/scrapers/fetchers/http_fetcher"
+require "source_monitor/scrapers/parsers/readability_parser"
+
+module SourceMonitor
+  module Scrapers
+    class Readability < Base
+      DEFAULT_ACCEPT = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+      FETCHER_CLASS = SourceMonitor::Scrapers::Fetchers::HttpFetcher
+      PARSER_CLASS = SourceMonitor::Scrapers::Parsers::ReadabilityParser
+
+      def self.default_settings
+        {
+          http: {
+            headers: {
+              "Accept" => DEFAULT_ACCEPT,
+              "User-Agent" => SourceMonitor::HTTP::DEFAULT_USER_AGENT
+            },
+            timeout: SourceMonitor::HTTP::DEFAULT_TIMEOUT,
+            open_timeout: SourceMonitor::HTTP::DEFAULT_OPEN_TIMEOUT,
+            proxy: nil
+          },
+          selectors: {
+            content: nil,
+            title: nil
+          },
+          readability: {
+            remove_unlikely_candidates: true,
+            clean_conditionally: true,
+            retry_length: 250,
+            min_text_length: 25
+          }
+        }
+      end
+
+      def call
+        url = preferred_url
+        return failure_result("missing_url", "No URL available for scraping", url:) if url.blank?
+
+        fetch_result = fetcher.fetch(url:, settings: settings[:http])
+        return build_fetch_failure(fetch_result, url) if fetch_result.status == :failed
+
+        parser_result = parser.parse(
+          html: fetch_result.body.to_s,
+          selectors: settings[:selectors],
+          readability: settings[:readability]
+        )
+
+        return build_parser_failure(parser_result, fetch_result, url) if parser_result.status == :failed
+
+        Result.new(
+          status: parser_result.status,
+          html: fetch_result.body,
+          content: parser_result.content,
+          metadata: build_metadata(fetch_result:, parser_result:, url:)
+        )
+      rescue StandardError => error
+        failure_result(error.class.name, error.message, url: url)
+      end
+
+      private
+
+      def preferred_url
+        item.canonical_url.presence || item.url
+      end
+
+      def fetcher
+        @fetcher ||= FETCHER_CLASS.new(http: http)
+      end
+
+      def parser
+        @parser ||= PARSER_CLASS.new
+      end
+
+      def build_fetch_failure(fetch_result, url)
+        failure_result(
+          fetch_result.error || "fetch_error",
+          fetch_result.message || "Failed to fetch URL",
+          url: url,
+          http_status: fetch_result.http_status
+        )
+      end
+
+      def build_parser_failure(parser_result, fetch_result, url)
+        metadata = {
+          error: parser_result.metadata&.[](:error) || "parser_error",
+          message: parser_result.metadata&.[](:message) || "Failed to parse content",
+          url: url,
+          http_status: fetch_result.http_status
+        }.compact
+
+        Result.new(status: :failed, html: fetch_result.body, content: nil, metadata: metadata)
+      end
+
+      def build_metadata(fetch_result:, parser_result:, url:)
+        headers = fetch_result.headers || {}
+        content_type = headers["content-type"] || headers["Content-Type"]
+
+        metadata = {
+          url: url,
+          http_status: fetch_result.http_status,
+          content_type: content_type,
+          extraction_strategy: parser_result.strategy,
+          title: parser_result.title,
+          settings: deep_duplicate(settings)
+        }.compact
+
+        if parser_result.metadata && parser_result.metadata[:readability_text_length]
+          metadata[:readability_text_length] = parser_result.metadata[:readability_text_length]
+        end
+
+        metadata
+      end
+
+      def failure_result(error, message, url:, http_status: nil)
+        Result.new(
+          status: :failed,
+          html: nil,
+          content: nil,
+          metadata: {
+            error: error,
+            message: message,
+            url: url,
+            http_status: derive_status(message, http_status)
+          }.compact
+        )
+      end
+
+      def derive_status(message, explicit_status)
+        return explicit_status if explicit_status
+
+        return unless message
+
+        if (match = message.match(/status\s+(\d{3})/))
+          match[1].to_i
+        end
+      end
+
+      def deep_duplicate(value)
+        case value
+        when Hash
+          value.each_with_object({}) do |(key, val), memo|
+            memo[key] = deep_duplicate(val)
+          end
+        when Array
+          value.map { |element| deep_duplicate(element) }
+        else
+          value
+        end
+      end
+    end
+  end
+end