source_monitor 0.2.1 → 0.3.1

data/.claude/skills/sm-scraper-adapter/reference/example-adapter.md

@@ -0,0 +1,274 @@
+# Example Scraper Adapter
+
+A complete working example of a custom scraper adapter.
+
+## Use Case
+
+This adapter extracts content from pages that require API-based rendering (e.g., JavaScript-heavy sites that need a headless browser service).
+
+## Implementation
+
+```ruby
+# app/scrapers/my_app/scrapers/headless.rb
+module MyApp
+  module Scrapers
+    class Headless < SourceMonitor::Scrapers::Base
+      # Default settings for this adapter.
+      # Overridable per-source via source.scrape_settings JSON column,
+      # or per-invocation via the settings parameter.
+      def self.default_settings
+        {
+          render_service_url: ENV.fetch("RENDER_SERVICE_URL", "http://localhost:3001/render"),
+          wait_for_selector: "body",
+          timeout: 30,
+          selectors: {
+            content: "article, main, .content",
+            title: "h1, title"
+          }
+        }
+      end
+
+      def call
+        url = preferred_url
+        return missing_url_result unless url.present?
+
+        # Step 1: Render the page via headless service
+        render_result = render_page(url)
+        return fetch_failure(render_result) unless render_result[:success]
+
+        html = render_result[:body]
+
+        # Step 2: Extract content using CSS selectors
+        content = extract_content(html)
+        title = extract_title(html)
+
+        if content.blank?
+          return Result.new(
+            status: :partial,
+            html: html,
+            content: nil,
+            metadata: build_metadata(url: url, title: title, note: "No content extracted")
+          )
+        end
+
+        Result.new(
+          status: :success,
+          html: html,
+          content: content,
+          metadata: build_metadata(url: url, title: title)
+        )
+      rescue Faraday::TimeoutError => error
+        timeout_result(url, error)
+      rescue StandardError => error
+        error_result(url, error)
+      end
+
+      private
+
+      def preferred_url
+        item.canonical_url.presence || item.url
+      end
+
+      def render_page(url)
+        conn = http.client(timeout: settings[:timeout])
+        response = conn.post(settings[:render_service_url]) do |req|
+          req.headers["Content-Type"] = "application/json"
+          req.body = {
+            url: url,
+            wait_for: settings[:wait_for_selector],
+            timeout: (settings[:timeout].to_i * 1000)
+          }.to_json
+        end
+
+        if response.status >= 200 && response.status < 300
+          { success: true, body: response.body, status: response.status }
+        else
+          { success: false, status: response.status, error: "HTTP #{response.status}" }
+        end
+      rescue Faraday::Error => error
+        { success: false, error: error.message }
+      end
+
+      def extract_content(html)
+        return nil if html.blank?
+
+        doc = Nokogiri::HTML(html)
+        selector = settings.dig(:selectors, :content) || "body"
+
+        element = doc.at_css(selector)
+        return nil unless element
+
+        # Remove script and style tags
+        element.css("script, style, nav, footer, header").each(&:remove)
+        element.text.squeeze(" \n").strip
+      end
+
+      def extract_title(html)
+        return nil if html.blank?
+
+        doc = Nokogiri::HTML(html)
+        selector = settings.dig(:selectors, :title) || "title"
+        doc.at_css(selector)&.text&.strip
+      end
+
+      def build_metadata(url:, title: nil, note: nil)
+        meta = {
+          url: url,
+          extraction_method: "headless",
+          title: title
+        }
+        meta[:note] = note if note
+        meta.compact
+      end
+
+      def missing_url_result
+        Result.new(
+          status: :failed,
+          metadata: { error: "missing_url", message: "No URL available for scraping" }
+        )
+      end
+
+      def fetch_failure(render_result)
+        Result.new(
+          status: :failed,
+          metadata: {
+            error: "render_failed",
+            message: render_result[:error] || "Render service returned error",
+            http_status: render_result[:status]
+          }.compact
+        )
+      end
+
+      def timeout_result(url, error)
+        Result.new(
+          status: :failed,
+          metadata: {
+            error: "timeout",
+            message: error.message,
+            url: url
+          }
+        )
+      end
+
+      def error_result(url, error)
+        Result.new(
+          status: :failed,
+          metadata: {
+            error: error.class.name,
+            message: error.message,
+            url: url
+          }
+        )
+      end
+    end
+  end
+end
+```
+
+## Registration
+
+```ruby
+# config/initializers/source_monitor.rb
+SourceMonitor.configure do |config|
+  config.scrapers.register(:headless, "MyApp::Scrapers::Headless")
+end
+```
+
+## Per-Source Settings
+
+Override adapter defaults via the source's `scrape_settings` JSON column:
+
+```ruby
+source = SourceMonitor::Source.find(1)
+source.update!(scrape_settings: {
+  render_service_url: "https://render.example.com/api/render",
+  wait_for_selector: ".article-content",
+  timeout: 60,
+  selectors: {
+    content: ".article-body",
+    title: ".article-title h1"
+  }
+})
+```
+
+## Tests
+
+```ruby
+require "test_helper"
+require "webmock/minitest"
+
+class HeadlessScraperTest < ActiveSupport::TestCase
+  setup do
+    @source = create_source!
+    @item = @source.items.create!(
+      title: "Test Article",
+      url: "https://example.com/spa-article",
+      external_id: "headless-test-1"
+    )
+  end
+
+  test "successfully renders and extracts content" do
+    stub_request(:post, "http://localhost:3001/render")
+      .to_return(
+        status: 200,
+        body: <<~HTML
+          <html>
+            <head><title>Test Page</title></head>
+            <body>
+              <article>
+                <h1>Article Title</h1>
+                <p>This is the article content.</p>
+              </article>
+            </body>
+          </html>
+        HTML
+      )
+
+    result = MyApp::Scrapers::Headless.call(item: @item, source: @source)
+
+    assert_equal :success, result.status
+    assert_includes result.content, "article content"
+    assert_equal "headless", result.metadata[:extraction_method]
+  end
+
+  test "returns failed when render service is down" do
+    stub_request(:post, "http://localhost:3001/render")
+      .to_return(status: 500, body: "Internal Server Error")
+
+    result = MyApp::Scrapers::Headless.call(item: @item, source: @source)
+
+    assert_equal :failed, result.status
+    assert_equal "render_failed", result.metadata[:error]
+  end
+
+  test "returns partial when no content found" do
+    stub_request(:post, "http://localhost:3001/render")
+      .to_return(status: 200, body: "<html><body><nav>Nav only</nav></body></html>")
+
+    result = MyApp::Scrapers::Headless.call(item: @item, source: @source)
+
+    assert_equal :partial, result.status
+    assert_nil result.content
+  end
+
+  test "handles missing URL" do
+    @item.update!(url: nil)
+
+    result = MyApp::Scrapers::Headless.call(item: @item, source: @source)
+
+    assert_equal :failed, result.status
+    assert_equal "missing_url", result.metadata[:error]
+  end
+
+  test "merges source-level settings" do
+    @source.update!(scrape_settings: { timeout: 60 })
+
+    stub_request(:post, "http://localhost:3001/render")
+      .to_return(status: 200, body: "<html><body><article>Content</article></body></html>")
+
+    result = MyApp::Scrapers::Headless.call(item: @item, source: @source)
+
+    assert_equal :success, result.status
+  end
+end
+```

data/.claude/skills/solid-queue-setup/SKILL.md

@@ -0,0 +1,307 @@
+---
+name: solid-queue-setup
+description: Configures Solid Queue for background jobs in Rails 8. Use when setting up background processing, creating background jobs, configuring job queues, recurring jobs, or migrating from Sidekiq to Solid Queue.
+allowed-tools: Read, Write, Edit, Bash, Glob, Grep
+---
+
+# Solid Queue Setup for Rails 8
+
+## Overview
+
+Solid Queue is Rails 8's default Active Job backend:
+- Database-backed (no Redis required)
+- Built-in concurrency controls
+- Supports priorities and multiple queues
+- Web UI available via Mission Control
+
+## Quick Start
+
+```bash
+bundle add solid_queue
+bin/rails solid_queue:install
+bin/rails db:migrate
+```
+
+### Configuration
+
+```yaml
+# config/solid_queue.yml
+default: &default
+  dispatchers:
+    - polling_interval: 1
+      batch_size: 500
+  workers:
+    - queues: "*"
+      threads: 3
+      processes: 1
+      polling_interval: 0.1
+
+development:
+  <<: *default
+
+production:
+  <<: *default
+  workers:
+    - queues: [critical, default]
+      threads: 5
+      processes: 2
+    - queues: [low]
+      threads: 2
+      processes: 1
+```
+
+### Set as Active Job Adapter
+
+```ruby
+# config/application.rb
+config.active_job.queue_adapter = :solid_queue
+```
+
+## Naming Convention
+
+Use `_later` for async, `_now` for synchronous:
+
+```ruby
+# Async (queued via Solid Queue) - preferred
+SendWelcomeEmailJob.perform_later(user.id)
+
+# Synchronous (runs immediately, skips queue) - use sparingly
+SendWelcomeEmailJob.perform_now(user.id)
+```
+
+## Creating Jobs
+
+### Basic Job
+
+```ruby
+# app/jobs/send_welcome_email_job.rb
+class SendWelcomeEmailJob < ApplicationJob
+  queue_as :default
+
+  def perform(user_id)
+    user = User.find(user_id)
+    UserMailer.welcome(user).deliver_now
+  end
+end
+```
+
+### Job with Retries
+
+```ruby
+# app/jobs/process_payment_job.rb
+class ProcessPaymentJob < ApplicationJob
+  queue_as :critical
+
+  retry_on PaymentGatewayError, wait: :polynomially_longer, attempts: 5
+  discard_on ActiveRecord::RecordNotFound
+
+  rescue_from(StandardError) do |exception|
+    ErrorNotifier.notify(exception)
+    raise
+  end
+
+  def perform(order_id)
+    order = Order.find(order_id)
+    PaymentService.new.charge(order)
+  end
+end
+```
+
+### Job with Priority
+
+```ruby
+class UrgentNotificationJob < ApplicationJob
+  queue_as :critical
+
+  # Lower number = higher priority (default is 0)
+  def priority
+    -10
+  end
+
+  def perform(notification_id)
+    notification = Notification.find(notification_id)
+    notification.deliver!
+  end
+end
+```
+
+## Enqueueing Jobs
+
+```ruby
+# Enqueue immediately
+SendWelcomeEmailJob.perform_later(user.id)
+
+# Enqueue with delay
+SendReminderJob.set(wait: 1.hour).perform_later(user.id)
+
+# Enqueue at specific time
+SendReportJob.set(wait_until: Date.tomorrow.noon).perform_later
+
+# Enqueue on specific queue
+ProcessJob.set(queue: :low).perform_later(data)
+```
+
+## Recurring Jobs
+
+```yaml
+# config/recurring.yml
+production:
+  daily_report:
+    class: GenerateDailyReportJob
+    schedule: every day at 6am
+    queue: low
+
+  cleanup:
+    class: CleanupOldRecordsJob
+    schedule: every sunday at 2am
+
+  sync:
+    class: SyncExternalDataJob
+    schedule: every 15 minutes
+
+  session_cleanup:
+    class: SessionCleanupJob
+    schedule: every day at 3am
+```
+
+## Testing Jobs
+
+### Job Test Template
+
+```ruby
+# test/jobs/send_welcome_email_job_test.rb
+require "test_helper"
+
+class SendWelcomeEmailJobTest < ActiveJob::TestCase
+  setup do
+    @user = users(:one)
+  end
+
+  test "sends welcome email" do
+    assert_enqueued_emails 1 do
+      SendWelcomeEmailJob.perform_now(@user.id)
+    end
+  end
+
+  test "enqueues on default queue" do
+    assert_enqueued_with(job: SendWelcomeEmailJob, queue: "default") do
+      SendWelcomeEmailJob.perform_later(@user.id)
+    end
+  end
+end
+```
+
+### Testing Enqueueing
+
+```ruby
+# test/jobs/process_payment_job_test.rb
+require "test_helper"
+
+class ProcessPaymentJobTest < ActiveJob::TestCase
+  test "enqueues the job with correct arguments" do
+    order = orders(:one)
+
+    assert_enqueued_with(job: ProcessPaymentJob, args: [order.id]) do
+      ProcessPaymentJob.perform_later(order.id)
+    end
+  end
+
+  test "enqueues on critical queue" do
+    assert_enqueued_with(job: ProcessPaymentJob, queue: "critical") do
+      ProcessPaymentJob.perform_later(orders(:one).id)
+    end
+  end
+end
+```
+
+### Testing Job Side Effects
+
+```ruby
+# test/jobs/cleanup_old_records_job_test.rb
+require "test_helper"
+
+class CleanupOldRecordsJobTest < ActiveJob::TestCase
+  test "deletes old sessions" do
+    old_session = sessions(:old)
+    old_session.update!(created_at: 31.days.ago)
+    recent_session = sessions(:one)
+
+    CleanupOldRecordsJob.perform_now
+
+    assert_not Session.exists?(old_session.id)
+    assert Session.exists?(recent_session.id)
+  end
+end
+```
+
+### Testing with perform_enqueued_jobs
+
+```ruby
+# test/integration/signup_flow_test.rb
+require "test_helper"
+
+class SignupFlowTest < ActionDispatch::IntegrationTest
+  test "signup sends welcome email" do
+    perform_enqueued_jobs do
+      post signups_path, params: {
+        signup: { email: "new@example.com", name: "Test" }
+      }
+    end
+
+    assert_emails 1
+  end
+end
+```
+
+## Running Solid Queue
+
+```bash
+# Development
+bin/rails solid_queue:start
+
+# Production (Procfile)
+web: bin/rails server
+worker: bin/rails solid_queue:start
+```
+
+## Monitoring
+
+### Mission Control (Web UI)
+
+```ruby
+# Gemfile
+gem "mission_control-jobs"
+
+# config/routes.rb
+mount MissionControl::Jobs::Engine, at: "/jobs"
+```
+
+### Console Queries
+
+```ruby
+SolidQueue::Job.where(finished_at: nil).count        # Pending
+SolidQueue::FailedExecution.count                      # Failed
+SolidQueue::FailedExecution.last.retry                 # Retry
+SolidQueue::Job.where("finished_at < ?", 1.week.ago).delete_all  # Cleanup
+```
+
+## Migration from Sidekiq
+
+| Sidekiq | Solid Queue |
+|---------|-------------|
+| `perform_async(args)` | `perform_later(args)` |
+| `perform_in(5.minutes, args)` | `set(wait: 5.minutes).perform_later(args)` |
+| `sidekiq_options queue: 'critical'` | `queue_as :critical` |
+| `sidekiq_retry_in` | `retry_on` with `wait:` |
+
+## Checklist
+
+- [ ] Solid Queue gem installed
+- [ ] Migrations run
+- [ ] Queue adapter configured
+- [ ] Jobs use `perform_later` (not `perform_now`)
+- [ ] Error handling with `retry_on` / `discard_on`
+- [ ] Recurring jobs configured
+- [ ] Job tests written
+- [ ] Mission Control mounted (optional)
+- [ ] All tests GREEN