railwatch 0.1.4 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +212 -0
- data/README.md +10 -0
- data/app/channels/railwatch/environment_channel.rb +28 -0
- data/app/controllers/concerns/railwatch/telemetry_identity.rb +25 -0
- data/app/controllers/railwatch/alerts_controller.rb +48 -0
- data/app/controllers/railwatch/anomaly_rules_controller.rb +31 -0
- data/app/controllers/railwatch/attachments_controller.rb +22 -0
- data/app/controllers/railwatch/beacon_controller.rb +67 -7
- data/app/controllers/railwatch/broadcasts_controller.rb +15 -0
- data/app/controllers/railwatch/cache_events_controller.rb +30 -0
- data/app/controllers/railwatch/commands_controller.rb +16 -0
- data/app/controllers/railwatch/comments_controller.rb +11 -0
- data/app/controllers/railwatch/dashboard_controller.rb +69 -0
- data/app/controllers/railwatch/deploys_controller.rb +34 -0
- data/app/controllers/railwatch/deprecations_controller.rb +54 -0
- data/app/controllers/railwatch/environment_scoped.rb +99 -0
- data/app/controllers/railwatch/exceptions_controller.rb +53 -0
- data/app/controllers/railwatch/executions_controller.rb +13 -0
- data/app/controllers/railwatch/issues_controller.rb +258 -0
- data/app/controllers/railwatch/jobs_controller.rb +63 -0
- data/app/controllers/railwatch/llm_calls_controller.rb +124 -0
- data/app/controllers/railwatch/logs_controller.rb +40 -0
- data/app/controllers/railwatch/mails_controller.rb +19 -0
- data/app/controllers/railwatch/notifications_controller.rb +11 -0
- data/app/controllers/railwatch/outgoing_requests_controller.rb +19 -0
- data/app/controllers/railwatch/overview_controller.rb +38 -0
- data/app/controllers/railwatch/people_controller.rb +25 -0
- data/app/controllers/railwatch/processes_controller.rb +33 -0
- data/app/controllers/railwatch/profiles_controller.rb +50 -0
- data/app/controllers/railwatch/queries_controller.rb +59 -0
- data/app/controllers/railwatch/releases_controller.rb +75 -0
- data/app/controllers/railwatch/requests_controller.rb +55 -0
- data/app/controllers/railwatch/saved_views_controller.rb +53 -0
- data/app/controllers/railwatch/scheduled_tasks_controller.rb +45 -0
- data/app/controllers/railwatch/spans_controller.rb +47 -0
- data/app/controllers/railwatch/storage_ops_controller.rb +15 -0
- data/app/controllers/railwatch/tenants_controller.rb +44 -0
- data/app/controllers/railwatch/thresholds_controller.rb +40 -0
- data/app/controllers/railwatch/traces_controller.rb +72 -0
- data/app/controllers/railwatch/transactions_controller.rb +21 -0
- data/app/controllers/railwatch/view_renders_controller.rb +28 -0
- data/app/controllers/railwatch/visits_controller.rb +53 -0
- data/app/helpers/railwatch/assets_helper.rb +52 -0
- data/app/jobs/railwatch/anomaly_scan_job.rb +11 -0
- data/app/jobs/railwatch/application_job.rb +7 -0
- data/app/jobs/railwatch/auto_resolve_issues_job.rb +19 -0
- data/app/jobs/railwatch/check_scheduled_tasks_job.rb +113 -0
- data/app/jobs/railwatch/detect_anomalies_job.rb +171 -0
- data/app/jobs/railwatch/detect_performance_issues_job.rb +85 -0
- data/app/jobs/railwatch/group_exceptions_job.rb +91 -0
- data/app/jobs/railwatch/optimize_telemetry_job.rb +23 -0
- data/app/jobs/railwatch/performance_scan_job.rb +11 -0
- data/app/jobs/railwatch/prune_telemetry_job.rb +110 -0
- data/app/jobs/railwatch/release_health_rollup_job.rb +64 -0
- data/app/jobs/railwatch/rollup_catchup_job.rb +21 -0
- data/app/jobs/railwatch/rollup_job.rb +130 -0
- data/app/jobs/railwatch/scheduled_task_scan_job.rb +11 -0
- data/app/models/concerns/railwatch/detection_snapshotting.rb +20 -0
- data/app/models/railwatch/alert.rb +229 -0
- data/app/models/railwatch/alert_rule.rb +121 -0
- data/app/models/railwatch/anomaly_rule.rb +33 -0
- data/app/models/railwatch/application.rb +44 -0
- data/app/models/railwatch/application_record.rb +33 -0
- data/app/models/railwatch/comment.rb +36 -0
- data/app/models/railwatch/deploy.rb +67 -0
- data/app/models/railwatch/environment.rb +54 -0
- data/app/models/railwatch/execution_presenter.rb +185 -0
- data/app/models/railwatch/filter_query.rb +143 -0
- data/app/models/railwatch/followup_receipt.rb +27 -0
- data/app/models/railwatch/ingest/batch.rb +305 -0
- data/app/models/railwatch/ingest/mapper.rb +575 -0
- data/app/models/railwatch/ingest/payload.rb +96 -0
- data/app/models/railwatch/ingest/rollup_absorber.rb +170 -0
- data/app/models/railwatch/ingest/writer.rb +137 -0
- data/app/models/railwatch/issue.rb +277 -0
- data/app/models/railwatch/issue_activity.rb +23 -0
- data/app/models/railwatch/issue_detection_presenter.rb +309 -0
- data/app/models/railwatch/issue_detection_snapshot.rb +77 -0
- data/app/models/railwatch/maintenance_task.rb +53 -0
- data/app/models/railwatch/saved_view.rb +63 -0
- data/app/models/railwatch/telemetry/aggregations.rb +116 -0
- data/app/models/railwatch/telemetry/attachment.rb +31 -0
- data/app/models/railwatch/telemetry/bounded_gzip.rb +101 -0
- data/app/models/railwatch/telemetry/broadcast.rb +13 -0
- data/app/models/railwatch/telemetry/cache_event.rb +16 -0
- data/app/models/railwatch/telemetry/child.rb +53 -0
- data/app/models/railwatch/telemetry/cursor_page.rb +99 -0
- data/app/models/railwatch/telemetry/deprecation.rb +13 -0
- data/app/models/railwatch/telemetry/enqueued_job.rb +13 -0
- data/app/models/railwatch/telemetry/exception.rb +21 -0
- data/app/models/railwatch/telemetry/execution.rb +71 -0
- data/app/models/railwatch/telemetry/health_sample.rb +72 -0
- data/app/models/railwatch/telemetry/ingest_batch.rb +31 -0
- data/app/models/railwatch/telemetry/llm_call.rb +37 -0
- data/app/models/railwatch/telemetry/log.rb +85 -0
- data/app/models/railwatch/telemetry/mail.rb +13 -0
- data/app/models/railwatch/telemetry/n_plus_one.rb +92 -0
- data/app/models/railwatch/telemetry/notification.rb +13 -0
- data/app/models/railwatch/telemetry/outgoing_request.rb +13 -0
- data/app/models/railwatch/telemetry/person.rb +50 -0
- data/app/models/railwatch/telemetry/process.rb +10 -0
- data/app/models/railwatch/telemetry/profile.rb +37 -0
- data/app/models/railwatch/telemetry/query.rb +50 -0
- data/app/models/railwatch/telemetry/query_shape.rb +45 -0
- data/app/models/railwatch/telemetry/release_health.rb +63 -0
- data/app/models/railwatch/telemetry/rollup.rb +78 -0
- data/app/models/railwatch/telemetry/session.rb +24 -0
- data/app/models/railwatch/telemetry/span.rb +19 -0
- data/app/models/railwatch/telemetry/storage_op.rb +13 -0
- data/app/models/railwatch/telemetry/tenant.rb +221 -0
- data/app/models/railwatch/telemetry/transaction.rb +13 -0
- data/app/models/railwatch/telemetry/view_render.rb +13 -0
- data/app/models/railwatch/telemetry/visit.rb +24 -0
- data/app/models/railwatch/telemetry_record.rb +57 -0
- data/app/models/railwatch/threshold.rb +29 -0
- data/app/models/railwatch/user.rb +47 -0
- data/app/models/railwatch/viewer.rb +13 -0
- data/app/views/layouts/railwatch/dashboard.html.erb +25 -0
- data/config/routes.rb +59 -0
- data/db/railwatch_migrate/20260916000000_create_railwatch_tables.rb +151 -0
- data/db/railwatch_migrate/20260917000000_create_railwatch_maintenance_tasks.rb +18 -0
- data/db/railwatch_migrate/20260917120000_create_railwatch_followup_receipts.rb +20 -0
- data/db/railwatch_migrate/20260918120000_widen_host_user_ids.rb +71 -0
- data/db/railwatch_telemetry_migrate/20260903000001_create_telemetry.rb +481 -0
- data/db/railwatch_telemetry_migrate/20260903000002_rename_tenant_to_app_tenant.rb +14 -0
- data/db/railwatch_telemetry_migrate/20260903000003_add_statement_count_to_transactions.rb +9 -0
- data/db/railwatch_telemetry_migrate/20260903000004_add_role_and_channel.rb +10 -0
- data/db/railwatch_telemetry_migrate/20260903000005_add_locals_to_exceptions.rb +9 -0
- data/db/railwatch_telemetry_migrate/20260903000006_add_spans_health_vitals_and_fts.rb +82 -0
- data/db/railwatch_telemetry_migrate/20260903000007_rename_span_attributes_to_payload.rb +10 -0
- data/db/railwatch_telemetry_migrate/20260903000008_add_profiles_and_attachments.rb +60 -0
- data/db/railwatch_telemetry_migrate/20260903000009_add_truncated_to_attachments.rb +9 -0
- data/db/railwatch_telemetry_migrate/20260903000010_create_sessions_and_release_health.rb +51 -0
- data/db/railwatch_telemetry_migrate/20260903000011_add_fingerprint_to_exceptions.rb +11 -0
- data/db/railwatch_telemetry_migrate/20260904000012_add_failed_to_broadcasts.rb +10 -0
- data/db/railwatch_telemetry_migrate/20260904010000_add_filter_cursor_indexes.rb +37 -0
- data/db/railwatch_telemetry_migrate/20260904020000_add_n_plus_ones_execution_id_index.rb +11 -0
- data/db/railwatch_telemetry_migrate/20260904120000_create_query_shapes.rb +15 -0
- data/db/railwatch_telemetry_migrate/20260906120000_add_backpressure_factor_to_ingest_batches.rb +9 -0
- data/db/railwatch_telemetry_migrate/20260907000000_rename_lantern_version_on_processes.rb +10 -0
- data/db/railwatch_telemetry_migrate/20260913000000_rename_nightrail_version_on_processes.rb +9 -0
- data/db/railwatch_telemetry_migrate/20260914000000_drop_orphan_durable_ingest_tables.rb +18 -0
- data/db/railwatch_telemetry_migrate/20260914010000_drop_orphan_durable_ingest_columns.rb +25 -0
- data/db/railwatch_telemetry_migrate/20260915000000_create_llm_calls.rb +55 -0
- data/db/railwatch_telemetry_migrate/20260915120000_add_detail_to_llm_calls.rb +21 -0
- data/db/railwatch_telemetry_migrate/20260915200000_add_explained_index_to_queries.rb +16 -0
- data/db/railwatch_telemetry_migrate/20260915210000_add_slowest_index_to_queries.rb +18 -0
- data/db/railwatch_telemetry_migrate/20260917010000_add_batch_ledger_to_ingest_batches.rb +16 -0
- data/docs/configuration.md +26 -0
- data/docs/embedded.md +352 -0
- data/docs/getting-started.md +5 -0
- data/lib/generators/railwatch/install/install_generator.rb +222 -1
- data/lib/generators/railwatch/install/templates/{initializer.rb → initializer.rb.tt} +39 -0
- data/lib/generators/railwatch/install/templates/post-deploy +8 -0
- data/lib/puma/plugin/railwatch.rb +170 -0
- data/lib/railwatch/authentication.rb +83 -0
- data/lib/railwatch/configuration.rb +134 -3
- data/lib/railwatch/dashboard_assets.rb +45 -0
- data/lib/railwatch/embedded.rb +54 -0
- data/lib/railwatch/engine.rb +89 -1
- data/lib/railwatch/ingest_request_body_limit.rb +10 -0
- data/lib/railwatch/json_compat.rb +60 -0
- data/lib/railwatch/maintenance.rb +183 -0
- data/lib/railwatch/patches/runner_command.rb +21 -1
- data/lib/railwatch/record.rb +33 -8
- data/lib/railwatch/reporter.rb +16 -0
- data/lib/railwatch/subscribers/process_info.rb +2 -1
- data/lib/railwatch/transport/local.rb +78 -0
- data/lib/railwatch/transport/socket.rb +183 -0
- data/lib/railwatch/version.rb +1 -1
- data/lib/railwatch/writer.rb +370 -0
- data/lib/railwatch.rb +38 -1
- data/lib/tasks/railwatch_tasks.rake +128 -0
- data/public/railwatch/assets/CommitMono-Bold-D6h61ieg.woff2 +0 -0
- data/public/railwatch/assets/CommitMono-Regular-zr8w7Obm.woff2 +0 -0
- data/public/railwatch/assets/Roboto-Black-auA4GeOK.woff2 +0 -0
- data/public/railwatch/assets/Roboto-Bold-CJLnO8j1.woff2 +0 -0
- data/public/railwatch/assets/Roboto-Medium-Cm2bwKpj.woff2 +0 -0
- data/public/railwatch/assets/Roboto-Regular-Chaq1-PV.woff2 +0 -0
- data/public/railwatch/assets/app-layout-yh-sPWgK.js +1 -0
- data/public/railwatch/assets/app-wordmark-nbjkzxwQ.js +1 -0
- data/public/railwatch/assets/appearance-CDuRvQTB.js +1 -0
- data/public/railwatch/assets/application-C_kpBdnf.css +1 -0
- data/public/railwatch/assets/arrow-up-DVtOGdVA.js +1 -0
- data/public/railwatch/assets/auth-layout-TCwPpS1K.js +1 -0
- data/public/railwatch/assets/badge-Daw4Hvr8.js +1 -0
- data/public/railwatch/assets/braces-DhFbHPsz.js +1 -0
- data/public/railwatch/assets/card-BX_3HXcJ.js +1 -0
- data/public/railwatch/assets/chart-B14-N9g7.js +39 -0
- data/public/railwatch/assets/chart-hover-Vy52H4uD.js +1 -0
- data/public/railwatch/assets/chart-panel-Cb4S_ej_.js +1 -0
- data/public/railwatch/assets/checkbox-D3aRSsBj.js +1 -0
- data/public/railwatch/assets/code-KvW8k7Jr.js +1 -0
- data/public/railwatch/assets/copy-DaQMJWoT.js +1 -0
- data/public/railwatch/assets/copy-block-BKFGd11J.js +1 -0
- data/public/railwatch/assets/copy-id-Djks1fXB.js +1 -0
- data/public/railwatch/assets/cursor-load-more-BXV0f1D_.js +1 -0
- data/public/railwatch/assets/data-table-iv1bdF6u.js +1 -0
- data/public/railwatch/assets/edit-BZc_Iawe.js +1 -0
- data/public/railwatch/assets/edit-DMKUF8Zi.js +8 -0
- data/public/railwatch/assets/edit-__9yJlO3.js +1 -0
- data/public/railwatch/assets/empty-state-6j_0AaQQ.js +1 -0
- data/public/railwatch/assets/env-layout-DrT8rO6P.js +1 -0
- data/public/railwatch/assets/execution-path-CzgBUi5e.js +1 -0
- data/public/railwatch/assets/filter-bar-9SU5NrzX.js +1 -0
- data/public/railwatch/assets/flamegraph-qcekju8V.js +2 -0
- data/public/railwatch/assets/format-B9SDkrWj.js +1 -0
- data/public/railwatch/assets/frames-Cyu7KMxZ.js +1 -0
- data/public/railwatch/assets/google-sign-in-button-DsTSfmzY.js +1 -0
- data/public/railwatch/assets/index-1ol1-QWI.js +1 -0
- data/public/railwatch/assets/index-5jI4aFzC.js +1 -0
- data/public/railwatch/assets/index-9KTrVnrc.js +1 -0
- data/public/railwatch/assets/index-B0-8lcTp.js +1 -0
- data/public/railwatch/assets/index-B7jjfNfO.js +1 -0
- data/public/railwatch/assets/index-BBchRy0M.js +1 -0
- data/public/railwatch/assets/index-BRiq3SNR.js +1 -0
- data/public/railwatch/assets/index-BgKj9xhr.js +1 -0
- data/public/railwatch/assets/index-BkTZqqOu.js +1 -0
- data/public/railwatch/assets/index-BprKx8QO.js +1 -0
- data/public/railwatch/assets/index-C3jzvPs3.js +1 -0
- data/public/railwatch/assets/index-Cbs6gGyQ.js +1 -0
- data/public/railwatch/assets/index-CdRZ6AWF.js +1 -0
- data/public/railwatch/assets/index-CeYKnapu.js +1 -0
- data/public/railwatch/assets/index-Cmlwy1-V.js +1 -0
- data/public/railwatch/assets/index-Cwx6058d.js +1 -0
- data/public/railwatch/assets/index-D4CSdbHv.js +1 -0
- data/public/railwatch/assets/index-DEFMSkdG.js +1 -0
- data/public/railwatch/assets/index-DFiHEBSh.js +1 -0
- data/public/railwatch/assets/index-DL4vWdWJ.js +1 -0
- data/public/railwatch/assets/index-DU9F5b5d.js +1 -0
- data/public/railwatch/assets/index-DaXgPcGL.js +1 -0
- data/public/railwatch/assets/index-DbtaU-EE.js +1 -0
- data/public/railwatch/assets/index-DeOe83F4.js +1 -0
- data/public/railwatch/assets/index-DiucHN4B.js +1 -0
- data/public/railwatch/assets/index-DlnR_l9o.js +1 -0
- data/public/railwatch/assets/index-DlumCsWY.js +2 -0
- data/public/railwatch/assets/index-DmRd7aIG.js +1 -0
- data/public/railwatch/assets/index-DxSh2UpM.js +1 -0
- data/public/railwatch/assets/index-MIMGuFNt.js +1 -0
- data/public/railwatch/assets/index-OqI59zPb.js +1 -0
- data/public/railwatch/assets/index-P4rC7IlX.js +1 -0
- data/public/railwatch/assets/index-gpPOcFWq.js +1 -0
- data/public/railwatch/assets/index-oVkururr.js +1 -0
- data/public/railwatch/assets/index-p9puqVge.js +1 -0
- data/public/railwatch/assets/inertia-TViv6kNv.js +97 -0
- data/public/railwatch/assets/input-error-LxImUkxv.js +1 -0
- data/public/railwatch/assets/json-viewer-Ar4cjPDW.js +1 -0
- data/public/railwatch/assets/klass-CJ-J4INB.js +1 -0
- data/public/railwatch/assets/label-COUKWqE_.js +1 -0
- data/public/railwatch/assets/layout-DNSLAkw_.js +1 -0
- data/public/railwatch/assets/live-dot-ChfUtY3p.js +41 -0
- data/public/railwatch/assets/nav-CNnDqPlm.js +1 -0
- data/public/railwatch/assets/new-84S8ZJq9.js +1 -0
- data/public/railwatch/assets/new-Be55nmt9.js +1 -0
- data/public/railwatch/assets/new-Bi_xQiIb.js +1 -0
- data/public/railwatch/assets/new-BvCT8TMg.js +1 -0
- data/public/railwatch/assets/new-D05SajFR.js +1 -0
- data/public/railwatch/assets/new-D4uewYC8.js +1 -0
- data/public/railwatch/assets/onboarding-CYZi5Cqc.js +1 -0
- data/public/railwatch/assets/origin-identity-6q1-CBts.js +1 -0
- data/public/railwatch/assets/percentile-picker-gFZCXtdb.js +1 -0
- data/public/railwatch/assets/relative-time-IOOgl5n2.js +1 -0
- data/public/railwatch/assets/release-health-DC8oc7uw.js +1 -0
- data/public/railwatch/assets/route-Dv6LAWvT.js +1 -0
- data/public/railwatch/assets/segmented-h1VdDTqE.js +1 -0
- data/public/railwatch/assets/select-_AJsUa7X.js +1 -0
- data/public/railwatch/assets/separator-BwwTYtCF.js +1 -0
- data/public/railwatch/assets/series-chart-DaFPefku.js +1 -0
- data/public/railwatch/assets/show-B7NCgkEo.js +1 -0
- data/public/railwatch/assets/show-BKqyKjBK.js +1 -0
- data/public/railwatch/assets/show-BM6X2Mpo.js +1 -0
- data/public/railwatch/assets/show-BNw4tN5q.js +1 -0
- data/public/railwatch/assets/show-BO3bnG5h.js +1 -0
- data/public/railwatch/assets/show-BhrAVAEA.js +1 -0
- data/public/railwatch/assets/show-C4Ltf5i9.js +2 -0
- data/public/railwatch/assets/show-C8sHalnw.js +1 -0
- data/public/railwatch/assets/show-CeTL4B37.js +2 -0
- data/public/railwatch/assets/show-CpfgV1jP.js +1 -0
- data/public/railwatch/assets/show-DACku6AD.js +3 -0
- data/public/railwatch/assets/show-DIOSGcXV.js +6 -0
- data/public/railwatch/assets/show-DQp_1n-B.js +1 -0
- data/public/railwatch/assets/show-DVNz46RI.js +1 -0
- data/public/railwatch/assets/show-DYteoYWW.js +1 -0
- data/public/railwatch/assets/show-DgSIoRvA.js +1 -0
- data/public/railwatch/assets/show-JxFtB4eK.js +2 -0
- data/public/railwatch/assets/sort-header-DpFzXblu.js +1 -0
- data/public/railwatch/assets/source-link-B2183i2-.js +1 -0
- data/public/railwatch/assets/sparkline-cell-C3-5vFkP.js +1 -0
- data/public/railwatch/assets/stat-s4RpOS9w.js +1 -0
- data/public/railwatch/assets/status-badge-8jVV-LA4.js +1 -0
- data/public/railwatch/assets/tenant-path-G-6u9A-o.js +1 -0
- data/public/railwatch/assets/text-link-DfsiaCcP.js +1 -0
- data/public/railwatch/assets/textarea-Dye72uP7.js +1 -0
- data/public/railwatch/assets/timeline-CD7WHnbo.js +1 -0
- data/public/railwatch/assets/transition-B_AW8rMK.js +5 -0
- data/public/railwatch/assets/use-clipboard-ColgLyQ2.js +1 -0
- data/public/railwatch/icon.png +0 -0
- data/public/railwatch/icon.svg +5 -0
- data/public/railwatch/manifest.json +2171 -0
- data/public/railwatch/rails-vite.json +1 -0
- metadata +314 -4
|
@@ -0,0 +1,370 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "socket"
|
|
4
|
+
require "zlib"
|
|
5
|
+
require "json"
|
|
6
|
+
require "fileutils"
|
|
7
|
+
# The writer runs the maintenance clock. The engine requires it too, but a
|
|
8
|
+
# writer forked by the Puma plugin before the app has finished loading must
|
|
9
|
+
# not depend on that ordering.
|
|
10
|
+
require "railwatch/maintenance"
|
|
11
|
+
|
|
12
|
+
module Railwatch
|
|
13
|
+
# The embedded install's writer process. Puma workers hand their batches
|
|
14
|
+
# here over a Unix socket (Transport::Socket) and this process, and only
|
|
15
|
+
# this process, maps them, writes the two SQLite files, folds the rollups,
|
|
16
|
+
# groups exceptions, and runs the maintenance clock. Its GVL is its own, so
|
|
17
|
+
# none of that work is ever interleaved with a request.
|
|
18
|
+
#
|
|
19
|
+
# Forked from the Puma master by the gem's Puma plugin (lib/puma/plugin/
|
|
20
|
+
# railwatch.rb), the same shape as Solid Queue's `solid_queue_mode :fork`.
|
|
21
|
+
# It exits when its parent does. It can also be run on its own for a
|
|
22
|
+
# deployment that wants it supervised separately:
|
|
23
|
+
# bin/rails runner 'Railwatch::Writer.run!'
|
|
24
|
+
module Writer
|
|
25
|
+
THREADS = 2
|
|
26
|
+
MAX_REQUEST_BYTES = 64 << 20
|
|
27
|
+
# Inflated request ceiling: a batch is at most config.batch_bytes of
|
|
28
|
+
# NDJSON plus envelope, so anything past this is not a batch.
|
|
29
|
+
MAX_INFLATED_BYTES = 128 << 20
|
|
30
|
+
# Connections waiting for a writer thread. Past this the accept loop
|
|
31
|
+
# closes new ones outright; the worker sees a reset and retries on its
|
|
32
|
+
# backoff ladder, which is the right pressure to apply.
|
|
33
|
+
MAX_PENDING = 64
|
|
34
|
+
PARENT_POLL = 2
|
|
35
|
+
# How long a shutdown waits for a batch that is still being written.
|
|
36
|
+
SHUTDOWN_DRAIN = 5
|
|
37
|
+
# A batch write that has run this long is not slow, it is stuck: a
|
|
38
|
+
# SQLite lock that never clears, a t-digest on a pathological input, a
|
|
39
|
+
# connection pool that lost a connection. The supervisor cannot tell a
|
|
40
|
+
# wedged writer from a busy one (kill 0 answers for both), so the writer
|
|
41
|
+
# judges itself: past this the process exits and the plugin respawns it.
|
|
42
|
+
# The batch is retained on the worker, retried, and written by the new
|
|
43
|
+
# writer, with the ledger making that safe. Includes the time spent
|
|
44
|
+
# reading the request, so a client that connects and stops sending is a
|
|
45
|
+
# wedge too, not a permanently parked thread.
|
|
46
|
+
WEDGE_TIMEOUT = 60
|
|
47
|
+
# Linux caps a Unix socket path at 108 bytes including the terminator. A
|
|
48
|
+
# path past that cannot be bound or connected to at all, so the plugin
|
|
49
|
+
# refuses to start rather than fail on every batch, and the doctor says
|
|
50
|
+
# which path to set.
|
|
51
|
+
MAX_SOCKET_PATH = 107
|
|
52
|
+
|
|
53
|
+
@running = false
|
|
54
|
+
@expected = false
|
|
55
|
+
@server = nil
|
|
56
|
+
@stopping = false
|
|
57
|
+
@in_flight = {}
|
|
58
|
+
@in_flight_mutex = Mutex.new
|
|
59
|
+
@socket_ino = nil
|
|
60
|
+
|
|
61
|
+
module_function
|
|
62
|
+
|
|
63
|
+
# True in the writer process itself.
|
|
64
|
+
def running? = @running
|
|
65
|
+
|
|
66
|
+
# True in every process that should hand batches to a writer: set by the
|
|
67
|
+
# Puma plugin in the master before the workers fork, so they inherit it.
|
|
68
|
+
# A process without it (a runner, a Solid Queue worker) writes its own.
|
|
69
|
+
def expected? = @expected
|
|
70
|
+
|
|
71
|
+
def expected!
|
|
72
|
+
@expected = true
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
# Forks a child that is the writer from its first instruction. ForkTracker
|
|
76
|
+
# fires Railwatch.restart_after_fork! in the child before the block runs,
|
|
77
|
+
# and that reset chooses threads and transport by process role; setting
|
|
78
|
+
# the role here, before the fork, is what makes the child come up with
|
|
79
|
+
# the writer's set (reporter + Transport::Local + maintenance) rather
|
|
80
|
+
# than a web worker's followed by the writer's. The parent puts the flag
|
|
81
|
+
# back the moment fork returns, or fails.
|
|
82
|
+
def fork_writer!
|
|
83
|
+
@running = true
|
|
84
|
+
fork do
|
|
85
|
+
yield
|
|
86
|
+
exit!(0)
|
|
87
|
+
end
|
|
88
|
+
ensure
|
|
89
|
+
@running = false
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
# Whether something is listening at the configured socket right now.
|
|
93
|
+
def listening?(path = Railwatch.config.writer_socket_path)
|
|
94
|
+
return false if path.nil?
|
|
95
|
+
|
|
96
|
+
UNIXSocket.new(path).close
|
|
97
|
+
true
|
|
98
|
+
rescue SystemCallError, ArgumentError
|
|
99
|
+
# ArgumentError: the path is over the kernel's sun_path limit (108
|
|
100
|
+
# bytes on Linux), which a deep checkout can hit; see usable_path?.
|
|
101
|
+
false
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
def usable_path?(path)
|
|
105
|
+
!path.nil? && path.bytesize <= MAX_SOCKET_PATH
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# Serves until stopped or until `parent` (a pid) is gone. Never returns
|
|
109
|
+
# to a caller that expects the app to keep running; it is the process.
|
|
110
|
+
def run!(parent: nil)
|
|
111
|
+
path = Railwatch.config.writer_socket_path or raise ArgumentError, "Railwatch.config.writer_socket is unset"
|
|
112
|
+
unless usable_path?(path)
|
|
113
|
+
raise ArgumentError, "writer socket path is #{path.bytesize} bytes; Linux allows #{MAX_SOCKET_PATH}. " \
|
|
114
|
+
"Set RAILWATCH_WRITER_SOCKET to a shorter path (an absolute one under /tmp or /run works)."
|
|
115
|
+
end
|
|
116
|
+
@running = true
|
|
117
|
+
@stopping = false
|
|
118
|
+
$PROGRAM_NAME = "railwatch-writer: #{File.basename(path)}"
|
|
119
|
+
trap_signals
|
|
120
|
+
bind(path)
|
|
121
|
+
Railwatch.debug { "writer listening at #{path} (pid #{Process.pid})" }
|
|
122
|
+
# Under the Puma plugin ForkTracker has already reset and started
|
|
123
|
+
# everything for the writer role; run standalone (bin/rails runner)
|
|
124
|
+
# nothing has, so these are idempotent second calls at worst.
|
|
125
|
+
Railwatch.reporter.ensure_thread
|
|
126
|
+
Maintenance.start!
|
|
127
|
+
watch_parent(parent) if parent
|
|
128
|
+
watch_wedge
|
|
129
|
+
serve
|
|
130
|
+
ensure
|
|
131
|
+
cleanup(path)
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
# The socket's directory is created 0700 and the socket itself 0600: the
|
|
135
|
+
# writer accepts records as trusted, so only this user may reach it. A
|
|
136
|
+
# socket file already there is unlinked only when nothing answers on it;
|
|
137
|
+
# a live writer (an old container sharing a volume, a writer of a Puma
|
|
138
|
+
# that has not finished restarting) keeps its socket and this one fails
|
|
139
|
+
# to start, which the plugin retries.
|
|
140
|
+
def bind(path)
|
|
141
|
+
dir = File.dirname(path)
|
|
142
|
+
FileUtils.mkdir_p(dir, mode: 0o700)
|
|
143
|
+
if File.exist?(path)
|
|
144
|
+
raise Errno::EADDRINUSE, "#{path}: another writer is listening" if listening?(path)
|
|
145
|
+
|
|
146
|
+
File.unlink(path)
|
|
147
|
+
end
|
|
148
|
+
@server = UNIXServer.new(path)
|
|
149
|
+
File.chmod(0o600, path)
|
|
150
|
+
@socket_ino = File.stat(path).ino
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
def stop!
|
|
154
|
+
@stopping = true
|
|
155
|
+
@server&.close
|
|
156
|
+
rescue IOError
|
|
157
|
+
nil
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
# Handles one already-accepted connection: read the request, write the
|
|
161
|
+
# batch, reply. Public so a spec can drive it without a real socket. The
|
|
162
|
+
# whole thing, request read included, counts toward the wedge guard.
|
|
163
|
+
def handle(sock)
|
|
164
|
+
track_in_flight do
|
|
165
|
+
request = read_request(sock)
|
|
166
|
+
result = write_batch(request)
|
|
167
|
+
reply = JSON.generate(result.to_h)
|
|
168
|
+
write_with_deadline(sock, [ reply.bytesize ].pack("N") + reply)
|
|
169
|
+
end
|
|
170
|
+
rescue StandardError => e
|
|
171
|
+
Railwatch.debug { "writer request failed: #{e.class}: #{e.message}" }
|
|
172
|
+
reply = JSON.generate(ok: false, error: "#{e.class}: #{e.message}", retryable_error: true)
|
|
173
|
+
begin
|
|
174
|
+
write_with_deadline(sock, [ reply.bytesize ].pack("N") + reply)
|
|
175
|
+
rescue SystemCallError, IOError, Timeout::Error
|
|
176
|
+
nil
|
|
177
|
+
end
|
|
178
|
+
ensure
|
|
179
|
+
sock.close rescue nil
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
# Exactly what Transport::Local#write does; the writer is that code
|
|
183
|
+
# running in a process of its own.
|
|
184
|
+
def write_batch(request)
|
|
185
|
+
Rails.application.executor.wrap do
|
|
186
|
+
environment = Environment.current
|
|
187
|
+
batch_id = request["batch_id"]
|
|
188
|
+
if (ledger = environment.with_telemetry { Ingest::Batch.committed(batch_id) })
|
|
189
|
+
next Transport::Local::Result.new(ok: true, status: 200, accepted: ledger.accepted, rejected: ledger.rejected, rejections: [])
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
result = Ingest::Batch.new(environment, request["records"], dropped_by_client: request["dropped"].to_i,
|
|
193
|
+
backpressure_factor: request["backpressure_factor"], gem_version: Railwatch::VERSION,
|
|
194
|
+
embedded: true, batch_id: batch_id).write!
|
|
195
|
+
Transport::Local::Result.new(ok: true, status: 200, accepted: result.accepted, rejected: result.rejected,
|
|
196
|
+
rejections: result.rejections.first(10))
|
|
197
|
+
rescue StandardError => e
|
|
198
|
+
Railwatch.debug { "writer batch failed: #{e.class}: #{e.message}" }
|
|
199
|
+
Transport::Local::Result.new(ok: false, error: "#{e.class}: #{e.message}", retryable_error: true)
|
|
200
|
+
end
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
def serve
|
|
204
|
+
queue = Thread::SizedQueue.new(MAX_PENDING)
|
|
205
|
+
workers = Array.new(THREADS) do |i|
|
|
206
|
+
Thread.new do
|
|
207
|
+
Thread.current.name = "railwatch-writer-#{i}"
|
|
208
|
+
while (sock = queue.pop)
|
|
209
|
+
handle(sock)
|
|
210
|
+
end
|
|
211
|
+
end
|
|
212
|
+
end
|
|
213
|
+
loop do
|
|
214
|
+
sock = begin
|
|
215
|
+
@server.accept
|
|
216
|
+
rescue IOError, Errno::EBADF, Errno::EINVAL
|
|
217
|
+
break
|
|
218
|
+
end
|
|
219
|
+
# A full queue means both threads are busy and MAX_PENDING more are
|
|
220
|
+
# waiting; refusing here is what turns that into worker backoff
|
|
221
|
+
# instead of an unbounded pile of open descriptors.
|
|
222
|
+
sock.close unless queue.push(sock, true)
|
|
223
|
+
rescue ThreadError
|
|
224
|
+
sock&.close
|
|
225
|
+
end
|
|
226
|
+
ensure
|
|
227
|
+
# Bounded: a batch still running five seconds after the listener closed
|
|
228
|
+
# does not hold the shutdown open. Its transaction rolls back when the
|
|
229
|
+
# process exits and the client retries it by batch id against the next
|
|
230
|
+
# writer, so the batch is not lost -- it is just not finished here.
|
|
231
|
+
THREADS.times { queue&.push(nil) }
|
|
232
|
+
workers&.each { |t| t.join(SHUTDOWN_DRAIN) }
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
# Length-prefixed gzip JSON, read under the wedge deadline, inflated in
|
|
236
|
+
# bounded steps so a request that decompresses past MAX_INFLATED_BYTES
|
|
237
|
+
# is refused before it is materialised.
|
|
238
|
+
def read_request(sock)
|
|
239
|
+
deadline = Clock.monotonic + WEDGE_TIMEOUT
|
|
240
|
+
header = read_exactly(sock, 4, deadline)
|
|
241
|
+
length = header.unpack1("N")
|
|
242
|
+
raise IOError, "request too large (#{length} bytes)" if length > MAX_REQUEST_BYTES
|
|
243
|
+
|
|
244
|
+
body = read_exactly(sock, length, deadline)
|
|
245
|
+
JSON.parse(inflate_bounded(body))
|
|
246
|
+
end
|
|
247
|
+
|
|
248
|
+
INFLATE_CHUNK = 64 << 10
|
|
249
|
+
|
|
250
|
+
def inflate_bounded(body)
|
|
251
|
+
out = +""
|
|
252
|
+
inflater = Zlib::Inflate.new(Zlib::MAX_WBITS + 32)
|
|
253
|
+
# byteslice, not each_char: the body is gzip, so "characters" are a
|
|
254
|
+
# fiction its encoding may not even support, and walking a 64 MB
|
|
255
|
+
# request one character at a time to rebuild 64 KiB strings costs more
|
|
256
|
+
# than the inflate it is feeding.
|
|
257
|
+
offset = 0
|
|
258
|
+
while offset < body.bytesize
|
|
259
|
+
out << inflater.inflate(body.byteslice(offset, INFLATE_CHUNK))
|
|
260
|
+
offset += INFLATE_CHUNK
|
|
261
|
+
raise IOError, "request inflates past #{MAX_INFLATED_BYTES} bytes" if out.bytesize > MAX_INFLATED_BYTES
|
|
262
|
+
end
|
|
263
|
+
out << inflater.finish
|
|
264
|
+
raise IOError, "request inflates past #{MAX_INFLATED_BYTES} bytes" if out.bytesize > MAX_INFLATED_BYTES
|
|
265
|
+
|
|
266
|
+
out
|
|
267
|
+
ensure
|
|
268
|
+
inflater&.close
|
|
269
|
+
end
|
|
270
|
+
|
|
271
|
+
def read_exactly(sock, count, deadline)
|
|
272
|
+
buffer = +""
|
|
273
|
+
while buffer.bytesize < count
|
|
274
|
+
remaining = deadline - Clock.monotonic
|
|
275
|
+
raise Timeout::Error, "client did not finish sending within #{WEDGE_TIMEOUT}s" if remaining <= 0
|
|
276
|
+
raise IOError, "client closed the connection" unless sock.wait_readable(remaining)
|
|
277
|
+
|
|
278
|
+
chunk = sock.read_nonblock(count - buffer.bytesize, exception: false)
|
|
279
|
+
case chunk
|
|
280
|
+
when :wait_readable then next
|
|
281
|
+
when nil then raise IOError, "client closed the connection"
|
|
282
|
+
else buffer << chunk
|
|
283
|
+
end
|
|
284
|
+
end
|
|
285
|
+
buffer
|
|
286
|
+
end
|
|
287
|
+
|
|
288
|
+
def write_with_deadline(sock, data, timeout: Railwatch.config.timeout)
|
|
289
|
+
deadline = Clock.monotonic + timeout
|
|
290
|
+
offset = 0
|
|
291
|
+
while offset < data.bytesize
|
|
292
|
+
remaining = deadline - Clock.monotonic
|
|
293
|
+
raise Timeout::Error, "client did not read the reply within #{timeout}s" if remaining <= 0
|
|
294
|
+
raise IOError, "client closed the connection" unless sock.wait_writable(remaining)
|
|
295
|
+
|
|
296
|
+
written = sock.write_nonblock(data.byteslice(offset..), exception: false)
|
|
297
|
+
offset += written if written.is_a?(Integer)
|
|
298
|
+
end
|
|
299
|
+
end
|
|
300
|
+
|
|
301
|
+
def watch_parent(parent)
|
|
302
|
+
Thread.new do
|
|
303
|
+
Thread.current.name = "railwatch-writer-parent"
|
|
304
|
+
until @stopping
|
|
305
|
+
sleep PARENT_POLL
|
|
306
|
+
next if Process.ppid == parent
|
|
307
|
+
|
|
308
|
+
Railwatch.debug { "writer: parent #{parent} is gone, stopping" }
|
|
309
|
+
stop!
|
|
310
|
+
end
|
|
311
|
+
end
|
|
312
|
+
end
|
|
313
|
+
|
|
314
|
+
# Keyed by invocation, never by batch id: a worker that timed out and
|
|
315
|
+
# retried the same batch must not be able to overwrite, or delete, the
|
|
316
|
+
# entry of the original invocation still running.
|
|
317
|
+
def track_in_flight
|
|
318
|
+
token = Object.new
|
|
319
|
+
@in_flight_mutex.synchronize { @in_flight[token] = Clock.monotonic }
|
|
320
|
+
yield
|
|
321
|
+
ensure
|
|
322
|
+
@in_flight_mutex.synchronize { @in_flight.delete(token) }
|
|
323
|
+
end
|
|
324
|
+
|
|
325
|
+
# The oldest write still running, in seconds, or nil.
|
|
326
|
+
def oldest_in_flight
|
|
327
|
+
@in_flight_mutex.synchronize do
|
|
328
|
+
started = @in_flight.values.min
|
|
329
|
+
started && Clock.monotonic - started
|
|
330
|
+
end
|
|
331
|
+
end
|
|
332
|
+
|
|
333
|
+
# Exits the process, with a note, the moment any single write has run
|
|
334
|
+
# past WEDGE_TIMEOUT. Deliberately exit! and not stop!: a stuck write is
|
|
335
|
+
# holding whatever is stuck, and a graceful drain would wait on it.
|
|
336
|
+
def watch_wedge
|
|
337
|
+
Thread.new do
|
|
338
|
+
Thread.current.name = "railwatch-writer-wedge"
|
|
339
|
+
until @stopping
|
|
340
|
+
sleep PARENT_POLL
|
|
341
|
+
age = oldest_in_flight
|
|
342
|
+
next unless age && age > WEDGE_TIMEOUT
|
|
343
|
+
|
|
344
|
+
warn "[railwatch] writer: a batch write has run #{age.round}s (limit #{WEDGE_TIMEOUT}s); exiting so Puma restarts the writer"
|
|
345
|
+
Railwatch.notify_unrecoverable(WedgedError.new("writer batch write exceeded #{WEDGE_TIMEOUT}s"))
|
|
346
|
+
exit!(75)
|
|
347
|
+
end
|
|
348
|
+
end
|
|
349
|
+
end
|
|
350
|
+
|
|
351
|
+
class WedgedError < StandardError; end
|
|
352
|
+
|
|
353
|
+
def trap_signals
|
|
354
|
+
%w[TERM INT].each do |signal|
|
|
355
|
+
trap(signal) { Thread.new { stop! } }
|
|
356
|
+
end
|
|
357
|
+
end
|
|
358
|
+
|
|
359
|
+
# Unlinks the socket only if it is still the one this writer bound. A
|
|
360
|
+
# newer writer that took the path over keeps its socket.
|
|
361
|
+
def cleanup(path)
|
|
362
|
+
Maintenance.stop!
|
|
363
|
+
Railwatch.reporter.shutdown
|
|
364
|
+
if path && @socket_ino && File.exist?(path) && File.stat(path).ino == @socket_ino
|
|
365
|
+
File.unlink(path)
|
|
366
|
+
end
|
|
367
|
+
@running = false
|
|
368
|
+
end
|
|
369
|
+
end
|
|
370
|
+
end
|
data/lib/railwatch.rb
CHANGED
|
@@ -21,6 +21,13 @@ require "railwatch/current"
|
|
|
21
21
|
require "railwatch/record"
|
|
22
22
|
require "railwatch/buffer"
|
|
23
23
|
require "railwatch/transport/http"
|
|
24
|
+
require "railwatch/transport/local"
|
|
25
|
+
require "railwatch/transport/socket"
|
|
26
|
+
require "railwatch/writer"
|
|
27
|
+
require "railwatch/authentication"
|
|
28
|
+
require "railwatch/json_compat"
|
|
29
|
+
require "railwatch/embedded"
|
|
30
|
+
require "railwatch/ingest_request_body_limit"
|
|
24
31
|
require "railwatch/reporter"
|
|
25
32
|
require "railwatch/sampler"
|
|
26
33
|
require "railwatch/redactor"
|
|
@@ -56,7 +63,17 @@ module Railwatch
|
|
|
56
63
|
end
|
|
57
64
|
|
|
58
65
|
def reporter
|
|
59
|
-
@reporter ||= Reporter.new(config)
|
|
66
|
+
@reporter ||= Reporter.new(config, transport: local_transport)
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# Embedded mode: a Puma worker hands its batches to the writer process
|
|
70
|
+
# over the socket; the writer itself, and any process when no socket is
|
|
71
|
+
# configured, writes them straight into SQLite. nil means HTTP.
|
|
72
|
+
def local_transport
|
|
73
|
+
return nil unless config.local?
|
|
74
|
+
return Transport::Local.new(config) if Writer.running? || config.writer_socket_path.nil?
|
|
75
|
+
|
|
76
|
+
Transport::Socket.new(config)
|
|
60
77
|
end
|
|
61
78
|
|
|
62
79
|
def redactor
|
|
@@ -87,6 +104,7 @@ module Railwatch
|
|
|
87
104
|
Subscribers::ProcessInfo.restart_after_fork!
|
|
88
105
|
Health.restart_after_fork!
|
|
89
106
|
Sessions.restart_after_fork!
|
|
107
|
+
Maintenance.restart_after_fork!
|
|
90
108
|
end
|
|
91
109
|
|
|
92
110
|
# --- execution lifecycle -------------------------------------------------
|
|
@@ -547,4 +565,23 @@ module Railwatch
|
|
|
547
565
|
end
|
|
548
566
|
|
|
549
567
|
require "railwatch/subscribers"
|
|
568
|
+
require "railwatch/dashboard_assets"
|
|
569
|
+
|
|
570
|
+
module Railwatch
|
|
571
|
+
# Where the engine's two databases migrate from. database.yml names them
|
|
572
|
+
# (`migrations_paths: <%= Railwatch.migrations_path(:railwatch) %>`) so a
|
|
573
|
+
# host's db:prepare creates the tables on install and migrates them after
|
|
574
|
+
# every gem update, from the gem's own history, the way Active Storage
|
|
575
|
+
# migrates its tables. Nothing is copied into the app.
|
|
576
|
+
def self.migrations_path(database)
|
|
577
|
+
File.expand_path("../db/#{database}_migrate", __dir__)
|
|
578
|
+
end
|
|
579
|
+
|
|
580
|
+
# Raised when the engine's models are used in an app whose database.yml
|
|
581
|
+
# has no entry for them. They must never fall back to the host's primary
|
|
582
|
+
# connection: the telemetry tables are unprefixed (`sessions`, `visits`,
|
|
583
|
+
# `people`, `notifications`, `logs`), so a query there would read, and a
|
|
584
|
+
# write would corrupt, the application's own tables.
|
|
585
|
+
class DatabaseNotConfigured < StandardError; end
|
|
586
|
+
end
|
|
550
587
|
require "railwatch/engine" if defined?(Rails::Engine)
|
|
@@ -65,6 +65,10 @@ namespace :railwatch do
|
|
|
65
65
|
|
|
66
66
|
desc "Check that the app can reach Railwatch with the configured token"
|
|
67
67
|
task status: :environment do
|
|
68
|
+
if Railwatch.config.local?
|
|
69
|
+
puts "Railwatch OK: embedded (telemetry in this app's railwatch_telemetry database; dashboard at /railwatch)"
|
|
70
|
+
next
|
|
71
|
+
end
|
|
68
72
|
unless Railwatch.config.token.present?
|
|
69
73
|
abort "RAILWATCH_TOKEN is not set"
|
|
70
74
|
end
|
|
@@ -86,6 +90,107 @@ namespace :railwatch do
|
|
|
86
90
|
ok
|
|
87
91
|
end
|
|
88
92
|
|
|
93
|
+
if config.local?
|
|
94
|
+
# Embedded: no token and no ingest host. What can go wrong instead is
|
|
95
|
+
# the two databases the engine writes to and the jobs that derive
|
|
96
|
+
# rollups and issues from them.
|
|
97
|
+
check.call(true, "transport", "local (telemetry stays in this app; dashboard at the engine mount)")
|
|
98
|
+
%w[railwatch railwatch_telemetry].each do |name|
|
|
99
|
+
configured = ActiveRecord::Base.configurations.configs_for(env_name: Rails.env, name: name)
|
|
100
|
+
check.call(!configured.nil?, "#{name} database",
|
|
101
|
+
configured ? configured.database : "not in config/database.yml (bin/rails generate railwatch:install --local)",
|
|
102
|
+
fatal: true)
|
|
103
|
+
end
|
|
104
|
+
{ "railwatch_telemetry" => Railwatch::TelemetryRecord, "railwatch" => Railwatch::ApplicationRecord }.each do |name, base|
|
|
105
|
+
pending = begin
|
|
106
|
+
base.connection_pool.migration_context.open.pending_migrations.map(&:version)
|
|
107
|
+
rescue StandardError => e
|
|
108
|
+
e.message
|
|
109
|
+
end
|
|
110
|
+
check.call(pending == [], "#{name} migrations",
|
|
111
|
+
case pending
|
|
112
|
+
when [] then "up to date"
|
|
113
|
+
when Array then "#{pending.size} pending (bin/rails db:prepare)"
|
|
114
|
+
else "cannot check: #{pending}"
|
|
115
|
+
end, fatal: true)
|
|
116
|
+
end
|
|
117
|
+
# The maintenance clock runs in web and worker processes, not in this
|
|
118
|
+
# rake process, so what can be checked here is whether one has ticked.
|
|
119
|
+
last_tick = begin
|
|
120
|
+
Railwatch::MaintenanceTask.last_tick_at
|
|
121
|
+
rescue StandardError
|
|
122
|
+
nil
|
|
123
|
+
end
|
|
124
|
+
check.call(last_tick && last_tick > 10.minutes.ago, "maintenance",
|
|
125
|
+
if last_tick
|
|
126
|
+
"last tick #{ActiveSupport::Duration.build((Time.current - last_tick).round).inspect} ago"
|
|
127
|
+
else
|
|
128
|
+
"no tick recorded yet (runs inside the app's web and worker processes, not in rake)"
|
|
129
|
+
end)
|
|
130
|
+
socket_path = config.writer_socket_path
|
|
131
|
+
if socket_path && !Railwatch::Writer.usable_path?(socket_path)
|
|
132
|
+
check.call(false, "writer process",
|
|
133
|
+
"socket path is #{socket_path.bytesize} bytes, over Linux's #{Railwatch::Writer::MAX_SOCKET_PATH}; " \
|
|
134
|
+
"set RAILWATCH_WRITER_SOCKET to a shorter path (e.g. /tmp/#{Rails.application.class.module_parent_name.parameterize}-railwatch.sock)")
|
|
135
|
+
elsif socket_path
|
|
136
|
+
listening = Railwatch::Writer.listening?(socket_path)
|
|
137
|
+
puma_rb = Rails.root.join("config/puma.rb")
|
|
138
|
+
plugged = puma_rb.exist? && puma_rb.read.include?("plugin :railwatch")
|
|
139
|
+
# Not listening is only fine when the app is not running: this task
|
|
140
|
+
# is its own process and cannot tell, so it says which case it is
|
|
141
|
+
# looking at instead of calling a missing writer a pass.
|
|
142
|
+
check.call(listening, "writer process",
|
|
143
|
+
if listening
|
|
144
|
+
"listening at #{socket_path}"
|
|
145
|
+
elsif plugged
|
|
146
|
+
"not listening at #{socket_path} right now. Expected if the app is stopped; if it is serving, " \
|
|
147
|
+
"the writer died or `plugin :railwatch` never activated (check the Puma log)"
|
|
148
|
+
else
|
|
149
|
+
"not configured: add `plugin :railwatch` to config/puma.rb so batches are written outside the " \
|
|
150
|
+
"processes serving requests"
|
|
151
|
+
end)
|
|
152
|
+
# A socket that answers is a process that is alive; the ledger is
|
|
153
|
+
# what says it is doing its job. Nothing written in a while with the
|
|
154
|
+
# app serving traffic is a writer that is stuck or a worker that is
|
|
155
|
+
# not reaching it.
|
|
156
|
+
if listening
|
|
157
|
+
last_write = begin
|
|
158
|
+
Railwatch::Environment.current.with_telemetry { Railwatch::Telemetry::IngestBatch.maximum(:received_at) }
|
|
159
|
+
rescue StandardError
|
|
160
|
+
nil
|
|
161
|
+
end
|
|
162
|
+
check.call(last_write && last_write > 5.minutes.ago, "last write",
|
|
163
|
+
if last_write
|
|
164
|
+
"#{ActiveSupport::Duration.build((Time.current - last_write).round).inspect} ago"
|
|
165
|
+
else
|
|
166
|
+
"no batch written yet"
|
|
167
|
+
end)
|
|
168
|
+
end
|
|
169
|
+
end
|
|
170
|
+
# HTTP Basic is on and closed until credentials exist; a deploy that
|
|
171
|
+
# forgot gets a 401, not a public page, and this says so first.
|
|
172
|
+
gate = config.dashboard_gate
|
|
173
|
+
check.call(gate != :undeclared && !(gate == :basic && !config.http_basic_auth_configured?), "dashboard access",
|
|
174
|
+
case gate
|
|
175
|
+
when :basic
|
|
176
|
+
config.http_basic_auth_configured? ? "HTTP Basic, user #{config.http_basic_auth_user}" :
|
|
177
|
+
"closed: HTTP Basic is on with no credentials, so every dashboard request is 401. " \
|
|
178
|
+
"Run `bin/rails railwatch:authentication:configure`"
|
|
179
|
+
when :controller then "your own: c.base_controller_class = #{config.base_controller_class}"
|
|
180
|
+
when :resolver then "your own: c.dashboard_user decides, and live updates follow it"
|
|
181
|
+
when :open then "deliberately open: anyone who can reach the mount can read it (c.dashboard_open)"
|
|
182
|
+
else
|
|
183
|
+
"undeclared: HTTP Basic is off and no base_controller_class, dashboard_user or dashboard_open " \
|
|
184
|
+
"is set. If a routes constraint gates the mount, say so with `c.dashboard_open = true` " \
|
|
185
|
+
"(it also enables live updates); otherwise the dashboard is public"
|
|
186
|
+
end)
|
|
187
|
+
check.call(!Railwatch::JsonCompat.broken?, "json compatibility",
|
|
188
|
+
Railwatch::JsonCompat.broken? ? Railwatch::JsonCompat.advice : "json #{Railwatch::JsonCompat.json_version} decodes on Rails #{Rails.version}")
|
|
189
|
+
recurring = Rails.root.join("config/recurring.yml")
|
|
190
|
+
leftover = recurring.exist? && recurring.read.include?("Railwatch::")
|
|
191
|
+
check.call(!leftover, "recurring.yml",
|
|
192
|
+
leftover ? "Railwatch::* jobs listed in config/recurring.yml are no longer needed in embedded mode; remove them" : "no Railwatch entries (none needed)")
|
|
193
|
+
else
|
|
89
194
|
token = config.token.to_s
|
|
90
195
|
check.call(!token.empty?, "token",
|
|
91
196
|
token.empty? ? "RAILWATCH_TOKEN is not set" : Railwatch::SecretSafety.token_preview(token),
|
|
@@ -112,6 +217,7 @@ namespace :railwatch do
|
|
|
112
217
|
|
|
113
218
|
check.call(Railwatch::Transport::Http.new(config).ping, "ingest reachable",
|
|
114
219
|
"GET #{URI.join(config.ingest_url, '/ingest/ping')}", fatal: true)
|
|
220
|
+
end
|
|
115
221
|
|
|
116
222
|
middleware = Rails.application.middleware.map(&:name)
|
|
117
223
|
position = middleware.index("Railwatch::Middleware::Request")
|
|
@@ -271,9 +377,31 @@ namespace :railwatch do
|
|
|
271
377
|
TEXT
|
|
272
378
|
end
|
|
273
379
|
|
|
380
|
+
# The same shape as `mission_control:jobs:authentication:configure`: asks
|
|
381
|
+
# for a user and password and writes them to the current environment's
|
|
382
|
+
# credentials under `railwatch:`, which the engine reads at boot.
|
|
383
|
+
# RAILS_ENV=production bin/rails railwatch:authentication:configure
|
|
384
|
+
desc "Configure HTTP Basic authentication for the embedded dashboard (writes Rails credentials)"
|
|
385
|
+
task "authentication:configure" => :environment do
|
|
386
|
+
Railwatch::Authentication.configure
|
|
387
|
+
end
|
|
388
|
+
|
|
274
389
|
desc "Send deploy metadata to Railwatch: rake railwatch:deploy[ref,name,url]"
|
|
275
390
|
task :deploy, [ :ref, :name, :url ] => :environment do |_t, args|
|
|
276
391
|
deploy = Railwatch.config.deploy or abort "RAILWATCH_DEPLOY (or KAMAL_VERSION) is not set"
|
|
392
|
+
if Railwatch.config.local?
|
|
393
|
+
# Embedded: the deploy marker is a row in this app's railwatch database.
|
|
394
|
+
row = Railwatch::Environment.current.deploys.find_or_initialize_by(deploy: deploy.to_s.first(128))
|
|
395
|
+
row.assign_attributes(ref: (args[:ref] || `git rev-parse HEAD 2>/dev/null`.strip).presence&.first(128),
|
|
396
|
+
name: args[:name].presence&.first(255), url: args[:url].presence&.first(1024),
|
|
397
|
+
server: Railwatch.config.server, deployed_at: Time.current,
|
|
398
|
+
commits: Railwatch::DeployMetadata.commits,
|
|
399
|
+
detail: { performer: ENV["KAMAL_PERFORMER"], destination: ENV["KAMAL_DESTINATION"],
|
|
400
|
+
service: ENV["KAMAL_SERVICE"] }.compact)
|
|
401
|
+
row.save!
|
|
402
|
+
puts "Deploy #{deploy} recorded"
|
|
403
|
+
next
|
|
404
|
+
end
|
|
277
405
|
abort "Plain HTTP ingest is disabled; use HTTPS or set RAILWATCH_ALLOW_HTTP=true" unless Railwatch.config.ingest_url_allowed?
|
|
278
406
|
uri = URI.join(Railwatch.config.ingest_url, "/ingest/deploys")
|
|
279
407
|
req = Net::HTTP::Post.new(uri)
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|