railwatch 0.8.5 → 0.8.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ec770e9d790b50b269a6bfde1e37d453ff6deb312216ab05d8106a50bb72d9b2
-  data.tar.gz: 9b9bff09f0821fefd39ae31d7c2ac826b3a6044d899aaf53a4e8e712a093d8f2
+  metadata.gz: c483283faabf8cf2552cedbd8568e6d06f815c8e5bdf4f2cdffe1909a82fa80b
+  data.tar.gz: 2f56eb2aed77f09e55d869117c3f9bf1d6140f00fa93ebd2a5c2c89e4da62b7b
 SHA512:
-  metadata.gz: eb2fc4d6a744ca775895c0bbcb61b15321df7c3be1669c42c8e83b36563ff76fdd2c4aad2308af5eeeaa17bba68833e61a8e7c9c726adcc5b0199b502ee68c5b
-  data.tar.gz: e0685c8d6b3b9564728239e1cb6978e4eaaf888797fdf000deff1fbfcf642849424baca6e11d0d2b2d58afc40cebe23d20b14415e802835fe294dc5c2aa55e41
+  metadata.gz: a80ac73b1c0e36d6fb681da86848ac37f3bde8d6b867fbd0a5134087da9c134344595fd1407f49f72f00d8573d2304b9944b2e667ae44fa63cd9c2337527da5a
+  data.tar.gz: a3e7028f181c255b69d0831f9e9a56c163b48bdacccca8464b3d05f6233976a2d412d06d11f3ad5320430186e2c067830e96ee048f415281933da5a6bc41a22f

data/CHANGELOG.md

@@ -6,6 +6,30 @@
      filled in by the release commit, which is also the only commit that
      touches lib/railwatch/version.rb and Gemfile.lock. See CONTRIBUTING.md. -->
 
+## 0.8.6 (2026-10-06)
+
+- An install upgrading from 0.5.0 or older can migrate again. 0.5.1
+  renumbered the export queue migration (20260919000000 to
+  20260919000100), so a telemetry database that had already run it under
+  the old number saw it as pending, and its plain `create_table` raised
+  "table export_destinations already exists" -- in `db:prepare`, which a
+  deploy runs before the app boots. Every statement in it is now
+  `if_not_exists`, so it builds the queue on a new database, completes it
+  on one that ran the old number (keeping anything already queued), and
+  stays a no-op where the new number has run. Rolling it back on a
+  database that came through the old number leaves the queue in place,
+  since the old version still owns it; anywhere else it is reversed as
+  before.
+- The 0.8.0, 0.8.1 and 0.8.5 index migrations accept an index that is
+  already there. Each one reads a whole table, and on a telemetry file
+  much larger than memory that is not seconds: about 40 s per table on
+  rebulk-system's 14.8 GB file, five builds in all, against a deploy
+  that health-checks within 90 s of the container starting -- and runs
+  `db:prepare` inside that window. An operator can now build them ahead
+  of the deploy, while the old release still serves, with the
+  migrations' own `CREATE INDEX` statements; `db:prepare` then records
+  them and moves on.
+
 ## 0.8.5 (2026-09-25)
 
 - A new migration adds a partial covering index for the per-tenant sums

data/db/railwatch_telemetry_migrate/20260919000100_create_export_queue.rb

@@ -13,96 +13,134 @@
 # is stored.
 #
 # Nothing here is written unless export is explicitly enabled.
+#
+# Every statement is if_not_exists, because this migration has run before
+# under another number. Through 0.5.0 it was 20260919000000; 0.5.1 moved it
+# here so it could not collide with a host's own telemetry migration at that
+# number (Railwatch Cloud has one). An install upgrading from 0.5.0 or older
+# therefore already holds every table, index and column below, recorded
+# under the old version, and sees this one as pending: a plain create_table
+# raised "table export_destinations already exists" in db:prepare, which a
+# deploy runs before the app boots. The old run is detected by what it
+# built, not by its version, because on a host like Railwatch Cloud the old
+# number belongs to a different migration.
+#
+# Rolling back has the mirror problem: on a database that came through the
+# old number, these tables and columns -- and anything queued in them -- are
+# the old version's, which stays recorded as applied. So down leaves them
+# alone there, and reverses the migration everywhere else.
 class CreateExportQueue < ActiveRecord::Migration[8.1]
-  def change
-    # One row per destination this database has ever been pointed at.
-    create_table :export_destinations do |t|
-      t.string :url, null: false
-      t.string :url_sha256, limit: 64, null: false
-      # Permanent for this database's lineage: it is how the receiver tells
-      # our deliveries from another installation's.
-      t.string :producer_id, limit: 36, null: false
-      # A fingerprint, never the token. If the token changes, queued bytes
-      # must not follow it to whatever tenant the new one belongs to.
-      t.string :credential_sha256, limit: 64, null: false
-
-      t.string :state, limit: 16, null: false, default: "ready"
-      t.datetime :retry_at, precision: 6
-      t.string :reason, limit: 64
-
-      t.string :lease_owner, limit: 36
-      t.bigint :lease_generation, null: false, default: 0
-      t.datetime :lease_expires_at, precision: 6
-
-      # Live totals, maintained in the same transaction as the rows they
-      # describe, so admission can be decided without counting the table.
-      t.bigint :queued_bytes, null: false, default: 0
-      t.bigint :queued_deliveries, null: false, default: 0
-      # Lifetime accounting: acked, rejected, expired, discarded, shed.
-      t.json :counters, null: false, default: {}
-
-      t.timestamps
+  OLD_VERSION = 20260919000000
+
+  def up
+    build_queue
+  end
+
+  def down
+    if built_under_old_number?
+      say("left in place: #{OLD_VERSION} built this queue under its pre-0.5.1 number and is still recorded as applied")
+    else
+      revert { build_queue }
     end
+  end
 
-    add_index :export_destinations, :url_sha256, unique: true
-    add_index :export_destinations, :producer_id, unique: true
-
-    create_table :export_deliveries do |t|
-      t.references :export_destination, null: false, foreign_key: true
-      # A UUIDv7: its embedded time is how the receiver ages it out, so a
-      # delivery cannot be made young again by relabelling it.
-      t.string :delivery_id, limit: 36, null: false
-      # What this delivery is a delivery OF. One per source batch today; a
-      # future policy that selects across batches supplies its own key.
-      t.string :selection_key, limit: 160, null: false
-
-      t.binary :body
-      t.string :body_sha256, limit: 64, null: false
-      t.string :metadata_sha256, limit: 64, null: false
-      t.bigint :body_bytes, null: false
-      t.bigint :ndjson_bytes, null: false
-      t.integer :record_count, null: false
-      # Everything about the delivery that is not its body: version, drop
-      # counts, backpressure. Digested into metadata_sha256 so the same id
-      # arriving with different counts is a conflict, not an update.
-      t.json :wire_metadata, null: false, default: {}
-
-      t.string :state, limit: 8, null: false, default: "pending"
-      # Set only when done: acked, rejected, expired, discarded.
-      t.string :disposition, limit: 16
-      t.datetime :enqueued_at, null: false, precision: 6
-      t.datetime :expires_at, null: false, precision: 6
-      t.datetime :next_attempt_at, null: false, precision: 6
-      t.bigint :attempts, null: false, default: 0
-
-      t.string :claim_token, limit: 36
-      t.bigint :claim_generation
-      t.datetime :claim_expires_at, precision: 6
-
-      t.integer :last_status
-      t.string :last_reason, limit: 64
-      t.json :ack
-      t.datetime :finished_at, precision: 6
-
-      t.timestamps
+  private
+    # The old version is recorded, and no migration file this database knows
+    # of claims it: only the gem's own pre-0.5.1 CreateExportQueue did.
+    def built_under_old_number?
+      context = connection.pool.migration_context
+      context.get_all_versions.include?(OLD_VERSION) && context.migrations.none? { |m| m.version == OLD_VERSION }
     end
 
-    add_index :export_deliveries, %i[export_destination_id delivery_id], unique: true,
-      name: "index_export_deliveries_on_destination_and_delivery"
-    # One delivery per selection: a batch replayed into the same transaction
-    # cannot enqueue itself twice.
-    add_index :export_deliveries, %i[export_destination_id selection_key], unique: true,
-      name: "index_export_deliveries_on_destination_and_selection"
-    add_index :export_deliveries, %i[export_destination_id id], where: "state <> 'done'",
-      name: "index_export_deliveries_live"
-    add_index :export_deliveries, :expires_at, where: "body IS NOT NULL",
-      name: "index_export_deliveries_expiring"
-    add_index :export_deliveries, :finished_at, where: "state = 'done'",
-      name: "index_export_deliveries_finished"
-
-    # Why a batch did or did not enqueue anything. Null on every row an
-    # install without export ever writes.
-    add_column :ingest_batches, :export_disposition, :string, limit: 24
-    add_column :ingest_batches, :export_record_count, :bigint
-  end
+    def build_queue
+      # One row per destination this database has ever been pointed at.
+      create_table :export_destinations, if_not_exists: true do |t|
+        t.string :url, null: false
+        t.string :url_sha256, limit: 64, null: false
+        # Permanent for this database's lineage: it is how the receiver tells
+        # our deliveries from another installation's.
+        t.string :producer_id, limit: 36, null: false
+        # A fingerprint, never the token. If the token changes, queued bytes
+        # must not follow it to whatever tenant the new one belongs to.
+        t.string :credential_sha256, limit: 64, null: false
+
+        t.string :state, limit: 16, null: false, default: "ready"
+        t.datetime :retry_at, precision: 6
+        t.string :reason, limit: 64
+
+        t.string :lease_owner, limit: 36
+        t.bigint :lease_generation, null: false, default: 0
+        t.datetime :lease_expires_at, precision: 6
+
+        # Live totals, maintained in the same transaction as the rows they
+        # describe, so admission can be decided without counting the table.
+        t.bigint :queued_bytes, null: false, default: 0
+        t.bigint :queued_deliveries, null: false, default: 0
+        # Lifetime accounting: acked, rejected, expired, discarded, shed.
+        t.json :counters, null: false, default: {}
+
+        t.timestamps
+      end
+
+      add_index :export_destinations, :url_sha256, unique: true, if_not_exists: true
+      add_index :export_destinations, :producer_id, unique: true, if_not_exists: true
+
+      create_table :export_deliveries, if_not_exists: true do |t|
+        t.references :export_destination, null: false, foreign_key: true
+        # A UUIDv7: its embedded time is how the receiver ages it out, so a
+        # delivery cannot be made young again by relabelling it.
+        t.string :delivery_id, limit: 36, null: false
+        # What this delivery is a delivery OF. One per source batch today; a
+        # future policy that selects across batches supplies its own key.
+        t.string :selection_key, limit: 160, null: false
+
+        t.binary :body
+        t.string :body_sha256, limit: 64, null: false
+        t.string :metadata_sha256, limit: 64, null: false
+        t.bigint :body_bytes, null: false
+        t.bigint :ndjson_bytes, null: false
+        t.integer :record_count, null: false
+        # Everything about the delivery that is not its body: version, drop
+        # counts, backpressure. Digested into metadata_sha256 so the same id
+        # arriving with different counts is a conflict, not an update.
+        t.json :wire_metadata, null: false, default: {}
+
+        t.string :state, limit: 8, null: false, default: "pending"
+        # Set only when done: acked, rejected, expired, discarded.
+        t.string :disposition, limit: 16
+        t.datetime :enqueued_at, null: false, precision: 6
+        t.datetime :expires_at, null: false, precision: 6
+        t.datetime :next_attempt_at, null: false, precision: 6
+        t.bigint :attempts, null: false, default: 0
+
+        t.string :claim_token, limit: 36
+        t.bigint :claim_generation
+        t.datetime :claim_expires_at, precision: 6
+
+        t.integer :last_status
+        t.string :last_reason, limit: 64
+        t.json :ack
+        t.datetime :finished_at, precision: 6
+
+        t.timestamps
+      end
+
+      add_index :export_deliveries, %i[export_destination_id delivery_id], unique: true,
+        name: "index_export_deliveries_on_destination_and_delivery", if_not_exists: true
+      # One delivery per selection: a batch replayed into the same transaction
+      # cannot enqueue itself twice.
+      add_index :export_deliveries, %i[export_destination_id selection_key], unique: true,
+        name: "index_export_deliveries_on_destination_and_selection", if_not_exists: true
+      add_index :export_deliveries, %i[export_destination_id id], where: "state <> 'done'",
+        name: "index_export_deliveries_live", if_not_exists: true
+      add_index :export_deliveries, :expires_at, where: "body IS NOT NULL",
+        name: "index_export_deliveries_expiring", if_not_exists: true
+      add_index :export_deliveries, :finished_at, where: "state = 'done'",
+        name: "index_export_deliveries_finished", if_not_exists: true
+
+      # Why a batch did or did not enqueue anything. Null on every row an
+      # install without export ever writes.
+      add_column :ingest_batches, :export_disposition, :string, limit: 24, if_not_exists: true
+      add_column :ingest_batches, :export_record_count, :bigint, if_not_exists: true
+    end
 end

data/db/railwatch_telemetry_migrate/20260922000000_add_covering_indexes_for_dashboard_aggregates.rb

@@ -25,10 +25,10 @@
 # Execution.named_like seek them instead of reading the window.
 class AddCoveringIndexesForDashboardAggregates < ActiveRecord::Migration[8.1]
   def change
-    add_index :executions, [ :kind, :occurred_at, :queue, :outcome, :queue_latency ], name: "idx_executions_queue_stats"
+    add_index :executions, [ :kind, :occurred_at, :queue, :outcome, :queue_latency ], name: "idx_executions_queue_stats", if_not_exists: true
     add_index :health_samples, [ :sampled_at, :threads_max, :threads_busy, :backlog, :queue_depth, :queue_latency ],
-              name: "idx_health_samples_series"
-    add_index :broadcasts, :occurred_at
-    add_index :executions, [ :kind, :occurred_at ], name: "idx_executions_with_preview", where: "exception_preview IS NOT NULL"
+              name: "idx_health_samples_series", if_not_exists: true
+    add_index :broadcasts, :occurred_at, if_not_exists: true
+    add_index :executions, [ :kind, :occurred_at ], name: "idx_executions_with_preview", where: "exception_preview IS NOT NULL", if_not_exists: true
   end
 end

data/db/railwatch_telemetry_migrate/20260923000000_add_people_index.rb

@@ -10,6 +10,6 @@
 class AddPeopleIndex < ActiveRecord::Migration[8.1]
   def change
     add_index :executions, [ :user_ref, :kind, :occurred_at, :status ], name: "idx_executions_people",
-              where: "user_ref IS NOT NULL"
+              where: "user_ref IS NOT NULL", if_not_exists: true
   end
 end

data/db/railwatch_telemetry_migrate/20260925000000_add_tenant_summary_index.rb

@@ -12,6 +12,6 @@
 class AddTenantSummaryIndex < ActiveRecord::Migration[8.1]
   def change
     add_index :executions, [ :kind, :app_tenant, :occurred_at, :status, :outcome, :duration, :user_ref ],
-              name: "idx_executions_tenant_summary", where: "app_tenant IS NOT NULL"
+              name: "idx_executions_tenant_summary", where: "app_tenant IS NOT NULL", if_not_exists: true
   end
 end

data/lib/railwatch/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Railwatch
-  VERSION = "0.8.5"
+  VERSION = "0.8.6"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: railwatch
 version: !ruby/object:Gem::Version
-  version: 0.8.5
+  version: 0.8.6
 platform: ruby
 authors:
 - Cole Robertson