source_monitor 0.3.0 → 0.3.1

data/.claude/skills/sm-scraper-adapter/reference/example-adapter.md

@@ -0,0 +1,274 @@
+# Example Scraper Adapter
+
+A complete working example of a custom scraper adapter.
+
+## Use Case
+
+This adapter extracts content from pages that require API-based rendering (e.g., JavaScript-heavy sites that need a headless browser service).
+
+## Implementation
+
+```ruby
+# app/scrapers/my_app/scrapers/headless.rb
+module MyApp
+  module Scrapers
+    class Headless < SourceMonitor::Scrapers::Base
+      # Default settings for this adapter.
+      # Overridable per-source via source.scrape_settings JSON column,
+      # or per-invocation via the settings parameter.
+      def self.default_settings
+        {
+          render_service_url: ENV.fetch("RENDER_SERVICE_URL", "http://localhost:3001/render"),
+          wait_for_selector: "body",
+          timeout: 30,
+          selectors: {
+            content: "article, main, .content",
+            title: "h1, title"
+          }
+        }
+      end
+
+      def call
+        url = preferred_url
+        return missing_url_result unless url.present?
+
+        # Step 1: Render the page via headless service
+        render_result = render_page(url)
+        return fetch_failure(render_result) unless render_result[:success]
+
+        html = render_result[:body]
+
+        # Step 2: Extract content using CSS selectors
+        content = extract_content(html)
+        title = extract_title(html)
+
+        if content.blank?
+          return Result.new(
+            status: :partial,
+            html: html,
+            content: nil,
+            metadata: build_metadata(url: url, title: title, note: "No content extracted")
+          )
+        end
+
+        Result.new(
+          status: :success,
+          html: html,
+          content: content,
+          metadata: build_metadata(url: url, title: title)
+        )
+      rescue Faraday::TimeoutError => error
+        timeout_result(url, error)
+      rescue StandardError => error
+        error_result(url, error)
+      end
+
+      private
+
+      def preferred_url
+        item.canonical_url.presence || item.url
+      end
+
+      def render_page(url)
+        conn = http.client(timeout: settings[:timeout])
+        response = conn.post(settings[:render_service_url]) do |req|
+          req.headers["Content-Type"] = "application/json"
+          req.body = {
+            url: url,
+            wait_for: settings[:wait_for_selector],
+            timeout: (settings[:timeout].to_i * 1000)
+          }.to_json
+        end
+
+        if response.status >= 200 && response.status < 300
+          { success: true, body: response.body, status: response.status }
+        else
+          { success: false, status: response.status, error: "HTTP #{response.status}" }
+        end
+      rescue Faraday::Error => error
+        { success: false, error: error.message }
+      end
+
+      def extract_content(html)
+        return nil if html.blank?
+
+        doc = Nokogiri::HTML(html)
+        selector = settings.dig(:selectors, :content) || "body"
+
+        element = doc.at_css(selector)
+        return nil unless element
+
+        # Remove script and style tags
+        element.css("script, style, nav, footer, header").each(&:remove)
+        element.text.squeeze(" \n").strip
+      end
+
+      def extract_title(html)
+        return nil if html.blank?
+
+        doc = Nokogiri::HTML(html)
+        selector = settings.dig(:selectors, :title) || "title"
+        doc.at_css(selector)&.text&.strip
+      end
+
+      def build_metadata(url:, title: nil, note: nil)
+        meta = {
+          url: url,
+          extraction_method: "headless",
+          title: title
+        }
+        meta[:note] = note if note
+        meta.compact
+      end
+
+      def missing_url_result
+        Result.new(
+          status: :failed,
+          metadata: { error: "missing_url", message: "No URL available for scraping" }
+        )
+      end
+
+      def fetch_failure(render_result)
+        Result.new(
+          status: :failed,
+          metadata: {
+            error: "render_failed",
+            message: render_result[:error] || "Render service returned error",
+            http_status: render_result[:status]
+          }.compact
+        )
+      end
+
+      def timeout_result(url, error)
+        Result.new(
+          status: :failed,
+          metadata: {
+            error: "timeout",
+            message: error.message,
+            url: url
+          }
+        )
+      end
+
+      def error_result(url, error)
+        Result.new(
+          status: :failed,
+          metadata: {
+            error: error.class.name,
+            message: error.message,
+            url: url
+          }
+        )
+      end
+    end
+  end
+end
+```
+
+## Registration
+
+```ruby
+# config/initializers/source_monitor.rb
+SourceMonitor.configure do |config|
+  config.scrapers.register(:headless, "MyApp::Scrapers::Headless")
+end
+```
+
+## Per-Source Settings
+
+Override adapter defaults via the source's `scrape_settings` JSON column:
+
+```ruby
+source = SourceMonitor::Source.find(1)
+source.update!(scrape_settings: {
+  render_service_url: "https://render.example.com/api/render",
+  wait_for_selector: ".article-content",
+  timeout: 60,
+  selectors: {
+    content: ".article-body",
+    title: ".article-title h1"
+  }
+})
+```
+
+## Tests
+
+```ruby
+require "test_helper"
+require "webmock/minitest"
+
+class HeadlessScraperTest < ActiveSupport::TestCase
+  setup do
+    @source = create_source!
+    @item = @source.items.create!(
+      title: "Test Article",
+      url: "https://example.com/spa-article",
+      external_id: "headless-test-1"
+    )
+  end
+
+  test "successfully renders and extracts content" do
+    stub_request(:post, "http://localhost:3001/render")
+      .to_return(
+        status: 200,
+        body: <<~HTML
+          <html>
+            <head><title>Test Page</title></head>
+            <body>
+              <article>
+                <h1>Article Title</h1>
+                <p>This is the article content.</p>
+              </article>
+            </body>
+          </html>
+        HTML
+      )
+
+    result = MyApp::Scrapers::Headless.call(item: @item, source: @source)
+
+    assert_equal :success, result.status
+    assert_includes result.content, "article content"
+    assert_equal "headless", result.metadata[:extraction_method]
+  end
+
+  test "returns failed when render service is down" do
+    stub_request(:post, "http://localhost:3001/render")
+      .to_return(status: 500, body: "Internal Server Error")
+
+    result = MyApp::Scrapers::Headless.call(item: @item, source: @source)
+
+    assert_equal :failed, result.status
+    assert_equal "render_failed", result.metadata[:error]
+  end
+
+  test "returns partial when no content found" do
+    stub_request(:post, "http://localhost:3001/render")
+      .to_return(status: 200, body: "<html><body><nav>Nav only</nav></body></html>")
+
+    result = MyApp::Scrapers::Headless.call(item: @item, source: @source)
+
+    assert_equal :partial, result.status
+    assert_nil result.content
+  end
+
+  test "handles missing URL" do
+    @item.update!(url: nil)
+
+    result = MyApp::Scrapers::Headless.call(item: @item, source: @source)
+
+    assert_equal :failed, result.status
+    assert_equal "missing_url", result.metadata[:error]
+  end
+
+  test "merges source-level settings" do
+    @source.update!(scrape_settings: { timeout: 60 })
+
+    stub_request(:post, "http://localhost:3001/render")
+      .to_return(status: 200, body: "<html><body><article>Content</article></body></html>")
+
+    result = MyApp::Scrapers::Headless.call(item: @item, source: @source)
+
+    assert_equal :success, result.status
+  end
+end
+```

data/.vbw-planning/.notification-log.jsonl

@@ -190,3 +190,57 @@
   "title": "",
   "message": "Claude Code needs your attention"
 }
+{
+  "timestamp": "2026-02-11T00:46:04Z",
+  "type": "idle_prompt",
+  "title": "",
+  "message": "Claude is waiting for your input"
+}
+{
+  "timestamp": "2026-02-11T00:59:14Z",
+  "type": "idle_prompt",
+  "title": "",
+  "message": "Claude is waiting for your input"
+}
+{
+  "timestamp": "2026-02-11T03:26:32Z",
+  "type": "idle_prompt",
+  "title": "",
+  "message": "Claude is waiting for your input"
+}
+{
+  "timestamp": "2026-02-11T03:46:35Z",
+  "type": "idle_prompt",
+  "title": "",
+  "message": "Claude is waiting for your input"
+}
+{
+  "timestamp": "2026-02-11T04:21:35Z",
+  "type": "permission_prompt",
+  "title": "",
+  "message": "Claude Code needs your attention"
+}
+{
+  "timestamp": "2026-02-11T04:28:53Z",
+  "type": "permission_prompt",
+  "title": "",
+  "message": "Claude Code needs your approval for the plan"
+}
+{
+  "timestamp": "2026-02-11T04:33:09Z",
+  "type": "permission_prompt",
+  "title": "",
+  "message": "Claude Code needs your approval for the plan"
+}
+{
+  "timestamp": "2026-02-11T04:46:56Z",
+  "type": "idle_prompt",
+  "title": "",
+  "message": "Claude is waiting for your input"
+}
+{
+  "timestamp": "2026-02-11T04:55:02Z",
+  "type": "permission_prompt",
+  "title": "",
+  "message": "Claude Code needs your approval for the plan"
+}

data/.vbw-planning/.session-log.jsonl

@@ -869,3 +869,124 @@
     "other": 439
   }
 }
+{
+  "timestamp": "2026-02-11T00:45:04Z",
+  "duration_ms": 0,
+  "cost_usd": 0,
+  "tokens_in": 0,
+  "tokens_out": 0,
+  "model": "unknown",
+  "branch": "main"
+}
+{
+  "timestamp": "2026-02-11T00:45:04Z",
+  "type": "cost_summary",
+  "costs": {
+    "other": 69
+  }
+}
+{
+  "timestamp": "2026-02-11T03:41:32Z",
+  "duration_ms": 0,
+  "cost_usd": 0,
+  "tokens_in": 0,
+  "tokens_out": 0,
+  "model": "unknown",
+  "branch": "chore/dependency-updates"
+}
+{
+  "timestamp": "2026-02-11T03:41:32Z",
+  "type": "cost_summary",
+  "costs": {
+    "other": 191
+  }
+}
+{
+  "timestamp": "2026-02-11T03:41:59Z",
+  "duration_ms": 0,
+  "cost_usd": 0,
+  "tokens_in": 0,
+  "tokens_out": 0,
+  "model": "unknown",
+  "branch": "chore/dependency-updates"
+}
+{
+  "timestamp": "2026-02-11T03:41:59Z",
+  "type": "cost_summary",
+  "costs": {
+    "other": 47
+  }
+}
+{
+  "timestamp": "2026-02-11T03:42:02Z",
+  "duration_ms": 0,
+  "cost_usd": 0,
+  "tokens_in": 0,
+  "tokens_out": 0,
+  "model": "unknown",
+  "branch": "chore/dependency-updates"
+}
+{
+  "timestamp": "2026-02-11T03:42:02Z",
+  "type": "cost_summary",
+  "costs": {
+    "other": 27
+  }
+}
+{
+  "timestamp": "2026-02-11T03:45:34Z",
+  "duration_ms": 0,
+  "cost_usd": 0,
+  "tokens_in": 0,
+  "tokens_out": 0,
+  "model": "unknown",
+  "branch": "chore/dependency-updates"
+}
+{
+  "timestamp": "2026-02-11T03:45:34Z",
+  "type": "cost_summary",
+  "costs": {
+    "other": 83
+  }
+}
+{
+  "timestamp": "2026-02-11T04:45:55Z",
+  "duration_ms": 0,
+  "cost_usd": 0,
+  "tokens_in": 0,
+  "tokens_out": 0,
+  "model": "unknown",
+  "branch": "chore/dependency-updates"
+}
+{
+  "timestamp": "2026-02-11T04:45:55Z",
+  "type": "cost_summary",
+  "costs": {
+    "other": 46018
+  }
+}
+{
+  "timestamp": "2026-02-11T04:47:49Z",
+  "duration_ms": 0,
+  "cost_usd": 0,
+  "tokens_in": 0,
+  "tokens_out": 0,
+  "model": "unknown",
+  "branch": "chore/dependency-updates"
+}
+{
+  "timestamp": "2026-02-11T04:50:36Z",
+  "duration_ms": 0,
+  "cost_usd": 0,
+  "tokens_in": 0,
+  "tokens_out": 0,
+  "model": "unknown",
+  "branch": "chore/dependency-updates"
+}
+{
+  "timestamp": "2026-02-11T04:50:36Z",
+  "type": "cost_summary",
+  "costs": {
+    "other": 89
+  }
+}

data/CHANGELOG.md

@@ -15,6 +15,15 @@ All notable changes to this project are documented below. The format follows [Ke
 
 - No unreleased changes yet.
 
+## [0.3.1] - 2026-02-10
+
+### Added
+
+- 14 engine-specific Claude Code skills (`sm-*` prefix) for contributors and consumers.
+- Skills installer with consumer/contributor groups via rake tasks.
+- Skills installation integrated into guided `bin/source_monitor install` workflow.
+- Skills packaged in gem for distribution to host apps.
+
 ## [0.3.0] - 2026-02-10
 
 ### Changed

data/CLAUDE.md

@@ -177,3 +177,46 @@ These skills are available in `.claude/skills/`:
 | `performance-optimization` | Performance tuning patterns |
 | `i18n-patterns` | Internationalization patterns |
 | `active-storage-setup` | Active Storage configuration |
+
+## Source Monitor Skills
+
+Engine-specific skills (`sm-*` prefix). Consumer skills install by default; contributor skills are opt-in.
+
+### Consumer Skills (default install)
+
+| Skill | Purpose |
+|-------|---------|
+| `sm-host-setup` | Full host app setup walkthrough |
+| `sm-configure` | DSL configuration across all sub-sections |
+| `sm-scraper-adapter` | Custom scraper inheriting `Scrapers::Base` |
+| `sm-event-handler` | Lifecycle callbacks (after_item_created, etc.) |
+| `sm-model-extension` | Extend engine models from host app |
+| `sm-dashboard-widget` | Dashboard queries, presenters, Turbo broadcasts |
+
+### Contributor Skills (opt-in)
+
+| Skill | Purpose |
+|-------|---------|
+| `sm-domain-model` | Model graph, relationships, state values, scopes |
+| `sm-architecture` | Module map, autoload tree, extraction patterns |
+| `sm-engine-test` | Engine test helpers, VCR/WebMock, parallel caveats |
+| `sm-configuration-setting` | Add settings to config sub-sections |
+| `sm-pipeline-stage` | Add/modify fetch or scrape pipeline stages |
+| `sm-engine-migration` | Migrations with `sourcemon_` prefix conventions |
+| `sm-job` | Solid Queue jobs with shallow delegation |
+| `sm-health-rule` | Health status rules, circuit breaker, auto-pause |
+
+### Skills Distribution
+
+Host apps can install `sm-*` skills via rake:
+
+```bash
+bin/rails source_monitor:skills:install        # Consumer skills (default)
+bin/rails source_monitor:skills:contributor     # Contributor skills
+bin/rails source_monitor:skills:all            # All skills
+bin/rails source_monitor:skills:remove         # Remove all sm-* skills
+```
+
+## Maintenance Rules
+
+- **Skills & docs alignment**: Whenever engine code changes (models, configuration, pipeline, jobs, migrations, scrapers, events, health rules, or dashboard), the corresponding `sm-*` skill and its `reference/` files MUST be updated in the same PR to ensure skills always reflect current engine behavior.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    source_monitor (0.3.0)
+    source_monitor (0.3.1)
       cssbundling-rails (~> 1.4)
       faraday (~> 2.9)
       faraday-follow_redirects (~> 0.4)
@@ -99,7 +99,7 @@ GEM
     ast (2.4.3)
     base64 (0.3.0)
     bigdecimal (4.0.1)
-    brakeman (7.1.1)
+    brakeman (8.0.2)
       racc
     builder (3.3.0)
     capybara (3.40.0)
@@ -128,17 +128,17 @@ GEM
     erubi (1.13.1)
     et-orbi (1.4.0)
       tzinfo
-    faraday (2.14.0)
+    faraday (2.14.1)
       faraday-net_http (>= 2.0, < 3.5)
       json
       logger
     faraday-follow_redirects (0.5.0)
       faraday (>= 1, < 3)
-    faraday-gzip (3.0.4)
+    faraday-gzip (3.1.0)
       faraday (>= 2.0, < 3)
       zlib (~> 3.0)
-    faraday-net_http (3.4.1)
-      net-http (>= 0.5.0)
+    faraday-net_http (3.4.2)
+      net-http (~> 0.5)
     faraday-retry (2.4.0)
       faraday (~> 2.0)
     feedjira (4.0.1)
@@ -182,8 +182,8 @@ GEM
     minitest (6.0.1)
       prism (~> 1.5)
     minitest-mock (5.27.0)
-    net-http (0.6.0)
-      uri
+    net-http (0.9.1)
+      uri (>= 0.11.1)
     net-imap (0.6.2)
       date
       net-protocol
@@ -222,13 +222,13 @@ GEM
     parser (3.3.10.1)
       ast (~> 2.4.1)
       racc
-    pg (1.6.2)
-    pg (1.6.2-aarch64-linux)
-    pg (1.6.2-aarch64-linux-musl)
-    pg (1.6.2-arm64-darwin)
-    pg (1.6.2-x86_64-darwin)
-    pg (1.6.2-x86_64-linux)
-    pg (1.6.2-x86_64-linux-musl)
+    pg (1.6.3)
+    pg (1.6.3-aarch64-linux)
+    pg (1.6.3-aarch64-linux-musl)
+    pg (1.6.3-arm64-darwin)
+    pg (1.6.3-x86_64-darwin)
+    pg (1.6.3-x86_64-linux)
+    pg (1.6.3-x86_64-linux-musl)
     pp (0.6.3)
       prettyprint
     prettyprint (0.2.0)
@@ -326,13 +326,13 @@ GEM
       rubocop-performance (>= 1.24)
       rubocop-rails (>= 2.30)
     ruby-progressbar (1.13.0)
-    ruby-readability (0.7.2)
+    ruby-readability (0.7.3)
       guess_html_encoding (>= 0.0.4)
       nokogiri (>= 1.6.0)
     rubyzip (3.2.2)
     sax-machine (1.3.2)
     securerandom (0.4.1)
-    selenium-webdriver (4.38.0)
+    selenium-webdriver (4.40.0)
       base64 (~> 0.2)
       logger (~> 1.4)
       rexml (~> 3.2, >= 3.2.5)
@@ -358,7 +358,7 @@ GEM
       thor (>= 1.3.1)
     stackprof (0.2.27)
     stringio (3.2.0)
-    test-prof (1.4.4)
+    test-prof (1.5.2)
     thor (1.5.0)
     timeout (0.6.0)
     tsort (0.2.0)
@@ -372,8 +372,7 @@ GEM
     unicode-emoji (4.2.0)
     uri (1.1.1)
     useragent (0.16.11)
-    vcr (6.3.1)
-      base64
+    vcr (6.4.0)
     webmock (3.26.1)
       addressable (>= 2.8.0)
       crack (>= 0.3.2)
@@ -386,7 +385,7 @@ GEM
     xpath (3.2.0)
       nokogiri (~> 1.8)
     zeitwerk (2.7.4)
-    zlib (3.2.1)
+    zlib (3.2.2)
 
 PLATFORMS
   aarch64-linux-gnu

data/lib/source_monitor/setup/workflow.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "pathname"
+require_relative "skills_installer"
 
 module SourceMonitor
   module Setup
@@ -35,7 +36,8 @@ module SourceMonitor
         migration_installer: MigrationInstaller.new,
         initializer_patcher: InitializerPatcher.new,
         devise_detector: method(:default_devise_detector),
-        verifier: Verification::Runner.new
+        verifier: Verification::Runner.new,
+        skills_installer: SkillsInstaller.new
       )
         @dependency_checker = dependency_checker
         @prompter = prompter
@@ -47,6 +49,7 @@ module SourceMonitor
         @initializer_patcher = initializer_patcher
         @devise_detector = devise_detector
         @verifier = verifier
+        @skills_installer = skills_installer
       end
 
       def run
@@ -67,6 +70,13 @@ module SourceMonitor
         end
 
         verifier.call
+
+        if prompter.yes?("Install Claude Code skills for using SourceMonitor?", default: true)
+          skills_installer.install(target_dir: skills_target_dir, group: :consumer)
+          if prompter.yes?("Also install contributor skills for engine development?", default: false)
+            skills_installer.install(target_dir: skills_target_dir, group: :contributor)
+          end
+        end
       end
 
       private
@@ -80,7 +90,12 @@ module SourceMonitor
         :migration_installer,
         :initializer_patcher,
         :devise_detector,
-        :verifier
+        :verifier,
+        :skills_installer
+
+      def skills_target_dir
+        File.join(Dir.pwd, ".claude", "skills")
+      end
 
       def devise_available?
         !!devise_detector.call

data/lib/source_monitor/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module SourceMonitor
-  VERSION = "0.3.0"
+  VERSION = "0.3.1"
 end