source_monitor 0.1.1

data/package.json

@@ -0,0 +1,29 @@
+{
+  "name": "feed-monitor",
+  "private": true,
+  "scripts": {
+    "build": "npm run build:css && npm run build:js",
+    "build:css": "tailwindcss -c ./config/tailwind.config.js -i ./app/assets/stylesheets/source_monitor/application.tailwind.css -o ./app/assets/builds/source_monitor/application.css",
+    "build:css:watch": "tailwindcss -c ./config/tailwind.config.js -i ./app/assets/stylesheets/source_monitor/application.tailwind.css -o ./app/assets/builds/source_monitor/application.css --watch",
+    "build:js": "esbuild app/assets/javascripts/source_monitor/application.js --bundle --format=esm --sourcemap --outfile=app/assets/builds/source_monitor/application.js --log-level=warning --alias:stimulus-use=stimulus-use",
+    "build:js:watch": "npm run build:js -- --watch",
+    "lint:js": "eslint \"app/assets/javascripts/**/*.js\"",
+    "lint:css": "stylelint \"app/assets/**/*.css\"",
+    "lint": "npm run lint:js && npm run lint:css",
+    "check:assets": "npm run build"
+  },
+  "dependencies": {
+    "@hotwired/stimulus": "^3.2.2",
+    "stimulus-use": "^0.52.0"
+  },
+  "devDependencies": {
+    "@eslint/js": "^9.11.0",
+    "autoprefixer": "^10.4.20",
+    "esbuild": "^0.23.0",
+    "eslint": "^9.11.0",
+    "postcss": "^8.4.45",
+    "stylelint": "^16.8.0",
+    "stylelint-config-standard": "^36.0.0",
+    "tailwindcss": "^3.4.10"
+  }
+}

data/postcss.config.js

@@ -0,0 +1,6 @@
+export default {
+  plugins: {
+    tailwindcss: {},
+    autoprefixer: {}
+  }
+};

data/source_monitor.gemspec

@@ -0,0 +1,46 @@
+require_relative "lib/source_monitor/version"
+
+Gem::Specification.new do |spec|
+  spec.name        = "source_monitor"
+  spec.version     = SourceMonitor::VERSION
+  spec.authors     = [ "dchuk" ]
+  spec.email       = [ "me@dchuk.com" ]
+  spec.homepage    = "https://github.com/dchuk/source_monitor"
+  spec.summary     = "SourceMonitor engine for ingesting, scraping, and monitoring RSS/Atom/JSON feeds"
+  spec.description = "SourceMonitor is a mountable Rails 8 engine that ingests RSS, Atom, and JSON feeds, scrapes full article content, and surfaces Solid Queue powered dashboards for monitoring and remediation."
+  spec.license     = "MIT"
+
+  spec.required_ruby_version = ">= 3.4.0"
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = spec.homepage
+  spec.metadata["changelog_uri"] = "https://github.com/dchuk/source_monitor/blob/main/CHANGELOG.md"
+  spec.metadata["documentation_uri"] = "https://github.com/dchuk/source_monitor#readme"
+  spec.metadata["bug_tracker_uri"] = "https://github.com/dchuk/source_monitor/issues"
+  spec.metadata["rubygems_mfa_required"] = "true"
+
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    tracked_files = `git ls-files -z`.split("\x0")
+    tracked_files.reject do |file|
+      file.start_with?(".ai/", ".github/", "coverage/", "node_modules/", "pkg/", "spec/", "test/", "tmp/", "vendor/", "examples/", "bin/")
+    end
+  end
+  spec.files += [ "CHANGELOG.md" ].select { |path| File.exist?(File.join(__dir__, path)) }
+  spec.files.uniq!
+
+  spec.require_paths = [ "lib" ]
+
+  spec.add_dependency "rails", ">= 8.0.3", "< 9.0"
+  spec.add_dependency "cssbundling-rails", "~> 1.4"
+  spec.add_dependency "jsbundling-rails", "~> 1.3"
+  spec.add_dependency "turbo-rails", "~> 2.0"
+  spec.add_dependency "feedjira", ">= 3.2", "< 5.0"
+  spec.add_dependency "faraday", "~> 2.9"
+  spec.add_dependency "faraday-retry", "~> 2.2"
+  spec.add_dependency "faraday-follow_redirects", "~> 0.4"
+  spec.add_dependency "faraday-gzip", "~> 3.0"
+  spec.add_dependency "nokolexbor", "~> 0.5"
+  spec.add_dependency "ruby-readability", "~> 0.7"
+  spec.add_dependency "solid_queue", ">= 0.3", "< 3.0"
+  spec.add_dependency "solid_cable", ">= 3.0", "< 4.0"
+  spec.add_dependency "ransack", "~> 4.2"
+end

data/stylelint.config.js

@@ -0,0 +1,12 @@
+module.exports = {
+  extends: [ "stylelint-config-standard" ],
+  ignoreFiles: [
+    "node_modules/**",
+    "app/assets/builds/**"
+  ],
+  rules: {
+    "at-rule-no-unknown": null,
+    "import-notation": null,
+    "selector-class-pattern": null
+  }
+};

metadata

@@ -0,0 +1,469 @@
+--- !ruby/object:Gem::Specification
+name: source_monitor
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- dchuk
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 8.0.3
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '9.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 8.0.3
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '9.0'
+- !ruby/object:Gem::Dependency
+  name: cssbundling-rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+- !ruby/object:Gem::Dependency
+  name: jsbundling-rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: turbo-rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: feedjira
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.9'
+- !ruby/object:Gem::Dependency
+  name: faraday-retry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+- !ruby/object:Gem::Dependency
+  name: faraday-follow_redirects
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+- !ruby/object:Gem::Dependency
+  name: faraday-gzip
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: nokolexbor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
+- !ruby/object:Gem::Dependency
+  name: ruby-readability
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+- !ruby/object:Gem::Dependency
+  name: solid_queue
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.3'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.3'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: solid_cable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: ransack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.2'
+description: SourceMonitor is a mountable Rails 8 engine that ingests RSS, Atom, and
+  JSON feeds, scrapes full article content, and surfaces Solid Queue powered dashboards
+  for monitoring and remediation.
+email:
+- me@dchuk.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rubocop.yml"
+- ".ruby-version"
+- AGENTS.md
+- CHANGELOG.md
+- CONTRIBUTING.md
+- Gemfile
+- Gemfile.lock
+- MIT-LICENSE
+- README.md
+- Rakefile
+- app/assets/builds/.keep
+- app/assets/config/source_monitor_manifest.js
+- app/assets/images/source_monitor/.keep
+- app/assets/javascripts/source_monitor/application.js
+- app/assets/javascripts/source_monitor/controllers/async_submit_controller.js
+- app/assets/javascripts/source_monitor/controllers/dropdown_controller.js
+- app/assets/javascripts/source_monitor/controllers/modal_controller.js
+- app/assets/javascripts/source_monitor/controllers/notification_controller.js
+- app/assets/javascripts/source_monitor/turbo_actions.js
+- app/assets/stylesheets/source_monitor/application.tailwind.css
+- app/assets/svgs/source_monitor/.keep
+- app/controllers/concerns/.keep
+- app/controllers/concerns/source_monitor/sanitizes_search_params.rb
+- app/controllers/source_monitor/application_controller.rb
+- app/controllers/source_monitor/dashboard_controller.rb
+- app/controllers/source_monitor/fetch_logs_controller.rb
+- app/controllers/source_monitor/health_controller.rb
+- app/controllers/source_monitor/items_controller.rb
+- app/controllers/source_monitor/logs_controller.rb
+- app/controllers/source_monitor/scrape_logs_controller.rb
+- app/controllers/source_monitor/source_bulk_scrapes_controller.rb
+- app/controllers/source_monitor/source_fetches_controller.rb
+- app/controllers/source_monitor/source_health_checks_controller.rb
+- app/controllers/source_monitor/source_health_resets_controller.rb
+- app/controllers/source_monitor/source_retries_controller.rb
+- app/controllers/source_monitor/source_turbo_responses.rb
+- app/controllers/source_monitor/sources_controller.rb
+- app/helpers/source_monitor/application_helper.rb
+- app/jobs/source_monitor/application_job.rb
+- app/jobs/source_monitor/fetch_feed_job.rb
+- app/jobs/source_monitor/item_cleanup_job.rb
+- app/jobs/source_monitor/log_cleanup_job.rb
+- app/jobs/source_monitor/schedule_fetches_job.rb
+- app/jobs/source_monitor/scrape_item_job.rb
+- app/jobs/source_monitor/source_health_check_job.rb
+- app/mailers/source_monitor/application_mailer.rb
+- app/models/concerns/.keep
+- app/models/concerns/source_monitor/loggable.rb
+- app/models/source_monitor/application_record.rb
+- app/models/source_monitor/fetch_log.rb
+- app/models/source_monitor/health_check_log.rb
+- app/models/source_monitor/item.rb
+- app/models/source_monitor/item_content.rb
+- app/models/source_monitor/log_entry.rb
+- app/models/source_monitor/scrape_log.rb
+- app/models/source_monitor/source.rb
+- app/views/layouts/source_monitor/application.html.erb
+- app/views/source_monitor/dashboard/_fetch_schedule.html.erb
+- app/views/source_monitor/dashboard/_job_metrics.html.erb
+- app/views/source_monitor/dashboard/_recent_activity.html.erb
+- app/views/source_monitor/dashboard/_stat_card.html.erb
+- app/views/source_monitor/dashboard/_stats.html.erb
+- app/views/source_monitor/dashboard/index.html.erb
+- app/views/source_monitor/fetch_logs/show.html.erb
+- app/views/source_monitor/items/_details.html.erb
+- app/views/source_monitor/items/_details_wrapper.html.erb
+- app/views/source_monitor/items/index.html.erb
+- app/views/source_monitor/items/show.html.erb
+- app/views/source_monitor/logs/index.html.erb
+- app/views/source_monitor/scrape_logs/show.html.erb
+- app/views/source_monitor/shared/_toast.html.erb
+- app/views/source_monitor/sources/_bulk_scrape_form.html.erb
+- app/views/source_monitor/sources/_bulk_scrape_modal.html.erb
+- app/views/source_monitor/sources/_details.html.erb
+- app/views/source_monitor/sources/_details_wrapper.html.erb
+- app/views/source_monitor/sources/_empty_state_row.html.erb
+- app/views/source_monitor/sources/_fetch_interval_heatmap.html.erb
+- app/views/source_monitor/sources/_form.html.erb
+- app/views/source_monitor/sources/_health_status_badge.html.erb
+- app/views/source_monitor/sources/_row.html.erb
+- app/views/source_monitor/sources/edit.html.erb
+- app/views/source_monitor/sources/index.html.erb
+- app/views/source_monitor/sources/new.html.erb
+- app/views/source_monitor/sources/show.html.erb
+- config/coverage_baseline.json
+- config/initializers/feedjira.rb
+- config/routes.rb
+- config/tailwind.config.js
+- db/migrate/20241008120000_create_source_monitor_sources.rb
+- db/migrate/20241008121000_create_source_monitor_items.rb
+- db/migrate/20241008122000_create_source_monitor_fetch_logs.rb
+- db/migrate/20241008123000_create_source_monitor_scrape_logs.rb
+- db/migrate/20251008183000_change_fetch_interval_to_minutes.rb
+- db/migrate/20251009090000_create_source_monitor_item_contents.rb
+- db/migrate/20251009103000_add_feed_content_readability_to_sources.rb
+- db/migrate/20251010090000_add_adaptive_fetching_toggle_to_sources.rb
+- db/migrate/20251010123000_add_deleted_at_to_source_monitor_items.rb
+- db/migrate/20251010153000_add_type_to_source_monitor_sources.rb
+- db/migrate/20251010154500_add_fetch_status_to_source_monitor_sources.rb
+- db/migrate/20251010160000_create_solid_cable_messages.rb
+- db/migrate/20251011090000_add_fetch_retry_state_to_sources.rb
+- db/migrate/20251012090000_add_health_fields_to_sources.rb
+- db/migrate/20251012100000_optimize_source_monitor_database_performance.rb
+- db/migrate/20251014064947_add_not_null_constraints_to_items.rb
+- db/migrate/20251014171659_add_performance_indexes.rb
+- db/migrate/20251014172525_add_fetch_status_check_constraint.rb
+- db/migrate/20251015100000_create_source_monitor_log_entries.rb
+- db/migrate/20251022100000_create_source_monitor_health_check_logs.rb
+- db/migrate/20251108120116_refresh_fetch_status_constraint.rb
+- docs/configuration.md
+- docs/deployment.md
+- docs/gh-cli-workflow.md
+- docs/installation.md
+- docs/troubleshooting.md
+- eslint.config.mjs
+- lib/generators/source_monitor/install/install_generator.rb
+- lib/generators/source_monitor/install/templates/source_monitor.rb.tt
+- lib/source_monitor.rb
+- lib/source_monitor/analytics/source_activity_rates.rb
+- lib/source_monitor/analytics/source_fetch_interval_distribution.rb
+- lib/source_monitor/analytics/sources_index_metrics.rb
+- lib/source_monitor/assets.rb
+- lib/source_monitor/assets/bundler.rb
+- lib/source_monitor/configuration.rb
+- lib/source_monitor/dashboard/queries.rb
+- lib/source_monitor/dashboard/quick_action.rb
+- lib/source_monitor/dashboard/quick_actions_presenter.rb
+- lib/source_monitor/dashboard/recent_activity.rb
+- lib/source_monitor/dashboard/recent_activity_presenter.rb
+- lib/source_monitor/dashboard/turbo_broadcaster.rb
+- lib/source_monitor/dashboard/upcoming_fetch_schedule.rb
+- lib/source_monitor/engine.rb
+- lib/source_monitor/events.rb
+- lib/source_monitor/feedjira_extensions.rb
+- lib/source_monitor/fetching/advisory_lock.rb
+- lib/source_monitor/fetching/completion/event_publisher.rb
+- lib/source_monitor/fetching/completion/follow_up_handler.rb
+- lib/source_monitor/fetching/completion/retention_handler.rb
+- lib/source_monitor/fetching/feed_fetcher.rb
+- lib/source_monitor/fetching/fetch_error.rb
+- lib/source_monitor/fetching/fetch_runner.rb
+- lib/source_monitor/fetching/retry_policy.rb
+- lib/source_monitor/fetching/stalled_fetch_reconciler.rb
+- lib/source_monitor/health.rb
+- lib/source_monitor/health/source_health_check.rb
+- lib/source_monitor/health/source_health_monitor.rb
+- lib/source_monitor/health/source_health_reset.rb
+- lib/source_monitor/http.rb
+- lib/source_monitor/instrumentation.rb
+- lib/source_monitor/items/item_creator.rb
+- lib/source_monitor/items/retention_pruner.rb
+- lib/source_monitor/items/retention_strategies.rb
+- lib/source_monitor/items/retention_strategies/destroy.rb
+- lib/source_monitor/items/retention_strategies/soft_delete.rb
+- lib/source_monitor/jobs/cleanup_options.rb
+- lib/source_monitor/jobs/fetch_failure_subscriber.rb
+- lib/source_monitor/jobs/solid_queue_metrics.rb
+- lib/source_monitor/jobs/visibility.rb
+- lib/source_monitor/logs/entry_sync.rb
+- lib/source_monitor/logs/filter_set.rb
+- lib/source_monitor/logs/query.rb
+- lib/source_monitor/logs/table_presenter.rb
+- lib/source_monitor/metrics.rb
+- lib/source_monitor/model_extensions.rb
+- lib/source_monitor/models/sanitizable.rb
+- lib/source_monitor/models/url_normalizable.rb
+- lib/source_monitor/pagination/paginator.rb
+- lib/source_monitor/realtime.rb
+- lib/source_monitor/realtime/adapter.rb
+- lib/source_monitor/realtime/broadcaster.rb
+- lib/source_monitor/release/changelog.rb
+- lib/source_monitor/release/runner.rb
+- lib/source_monitor/scheduler.rb
+- lib/source_monitor/scrapers/base.rb
+- lib/source_monitor/scrapers/fetchers/http_fetcher.rb
+- lib/source_monitor/scrapers/parsers/readability_parser.rb
+- lib/source_monitor/scrapers/readability.rb
+- lib/source_monitor/scraping/bulk_result_presenter.rb
+- lib/source_monitor/scraping/bulk_source_scraper.rb
+- lib/source_monitor/scraping/enqueuer.rb
+- lib/source_monitor/scraping/item_scraper.rb
+- lib/source_monitor/scraping/item_scraper/adapter_resolver.rb
+- lib/source_monitor/scraping/item_scraper/persistence.rb
+- lib/source_monitor/scraping/scheduler.rb
+- lib/source_monitor/scraping/state.rb
+- lib/source_monitor/security/authentication.rb
+- lib/source_monitor/security/parameter_sanitizer.rb
+- lib/source_monitor/sources/turbo_stream_presenter.rb
+- lib/source_monitor/turbo_streams/stream_responder.rb
+- lib/source_monitor/version.rb
+- lib/tasks/recover_stalled_fetches.rake
+- lib/tasks/source_monitor_assets.rake
+- lib/tasks/source_monitor_tasks.rake
+- lib/tasks/test_smoke.rake
+- package-lock.json
+- package.json
+- postcss.config.js
+- source_monitor.gemspec
+- stylelint.config.js
+homepage: https://github.com/dchuk/source_monitor
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/dchuk/source_monitor
+  source_code_uri: https://github.com/dchuk/source_monitor
+  changelog_uri: https://github.com/dchuk/source_monitor/blob/main/CHANGELOG.md
+  documentation_uri: https://github.com/dchuk/source_monitor#readme
+  bug_tracker_uri: https://github.com/dchuk/source_monitor/issues
+  rubygems_mfa_required: 'true'
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.4.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.7.1
+specification_version: 4
+summary: SourceMonitor engine for ingesting, scraping, and monitoring RSS/Atom/JSON
+  feeds
+test_files: []