webhookdb 0.1.0

data/db/migrations/007_org_dns.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  change do
+    alter_table(:organizations) do
+      add_column :public_host, :text, null: false, default: ""
+      add_column :cloudflare_dns_record_json, :jsonb, null: false, default: "{}"
+      rename_column :readonly_connection_url, :readonly_connection_url_raw
+      rename_column :admin_connection_url, :admin_connection_url_raw
+    end
+  end
+end

data/db/migrations/008_webhook_subscriptions.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  change do
+    create_table(:webhook_subscriptions) do
+      primary_key :id
+      text :deliver_to_url, null: false
+      text :webhook_secret, null: false
+      text :opaque_id, null: false, unique: true
+      foreign_key :service_integration_id, :service_integrations
+      foreign_key :organization_id, :organizations
+
+      constraint(:service_integration_or_org) do
+        ((service_integration_id =~ nil) & (organization_id !~ nil)) |
+          ((service_integration_id !~ nil) & (organization_id =~ nil))
+      end
+    end
+  end
+end

data/db/migrations/009_nonunique_stripe_subscription_customer.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    alter_table(:subscriptions) do
+      drop_constraint :subscriptions_stripe_customer_id_key
+      add_index :stripe_customer_id
+    end
+  end
+  down do
+    alter_table(:subscriptions) do
+      drop_index :stripe_customer_id
+      add_unique_constraint :stripe_customer_id
+    end
+  end
+end

data/db/migrations/010_drop_integration_soft_delete.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    alter_table(:service_integrations) do
+      drop_column :soft_deleted_at
+    end
+  end
+  down do
+    alter_table(:subscriptions) do
+      add_column :soft_deleted_at, :timestamptz
+    end
+  end
+end

data/db/migrations/011_webhook_subscriptions_created_at.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  change do
+    alter_table(:webhook_subscriptions) do
+      add_column :created_at, :timestamptz, null: false, default: Sequel.function(:now)
+      add_column :updated_at, :timestamptz
+    end
+  end
+end

data/db/migrations/012_webhook_subscriptions_created_by.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  change do
+    alter_table(:webhook_subscriptions) do
+      add_foreign_key :created_by_id, :customers
+    end
+  end
+end

data/db/migrations/013_default_org_membership.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    from(:organization_memberships).where(invitation_code: nil).update(invitation_code: "")
+    alter_table(:organization_memberships) do
+      add_column :is_default, :boolean, null: false, default: false
+      set_column_default :invitation_code, ""
+      set_column_not_null :invitation_code
+      set_column_default :verified, nil
+      set_column_not_null :membership_role_id
+      drop_column :status
+      add_index(
+        [:customer_id, :organization_id],
+        name: :one_default_per_customer,
+        unique: true,
+        where: {is_default: true},
+      )
+      add_constraint(
+        :default_is_verified,
+        (Sequel[is_default: true] & Sequel[verified: true]) | Sequel[is_default: false],
+      )
+      add_constraint(
+        :invited_has_code,
+        (Sequel[verified: true] & Sequel.expr { length(invitation_code) < 1 }) |
+          (Sequel[verified: false] & Sequel.expr { length(invitation_code) > 0 }),
+      )
+    end
+  end
+end

data/db/migrations/014_webhook_subscription_deliveries.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  change do
+    alter_table(:webhook_subscriptions) do
+      add_column :deactivated_at, :timestamptz
+    end
+
+    create_table(:webhook_subscription_deliveries) do
+      primary_key :id
+      timestamptz :created_at, null: false, default: Sequel.function(:now)
+      column :attempt_timestamps, "timestamptz[]", null: false, default: []
+      column :attempt_http_response_statuses, "smallint[]", null: false, default: []
+      jsonb :payload, null: false
+
+      foreign_key :webhook_subscription_id, :webhook_subscriptions, null: false
+      index :webhook_subscription_id
+
+      constraint(
+        :balanced_attempts,
+        Sequel.function(:array_length, :attempt_timestamps,
+                        1,) =~ Sequel.function(:array_length, :attempt_http_response_statuses, 1),
+      )
+    end
+  end
+end

data/db/migrations/015_dependent_integrations.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  change do
+    alter_table(:service_integrations) do
+      add_foreign_key :depends_on_id, :service_integrations, on_delete: :restrict
+    end
+  end
+end

data/db/migrations/016_encrypted_columns.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  change do
+    alter_table(:service_integrations) do
+      add_column :data_encryption_secret, :text, null: true
+    end
+  end
+end

data/db/migrations/017_skip_verification.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  change do
+    alter_table(:service_integrations) do
+      add_column :skip_webhook_verification, :bool, null: false, default: false
+    end
+  end
+end

data/db/migrations/018_sync_targets.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  change do
+    create_table(:sync_targets) do
+      primary_key :id
+      timestamptz :created_at, null: false, default: Sequel.function(:now)
+      timestamptz :updated_at
+
+      text :opaque_id, null: false, unique: true
+      foreign_key :service_integration_id, :service_integrations, null: false, on_delete: :cascade
+      index :service_integration_id
+      foreign_key :created_by_id, :customers, null: true, on_delete: :set_null
+
+      integer :period_seconds, null: false
+      text :connection_url, null: false
+
+      text :schema, null: false, default: ""
+      text :table, null: false, default: ""
+
+      timestamptz :last_synced_at
+      index :last_synced_at
+    end
+  end
+end

data/db/migrations/019_org_schema.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  change do
+    alter_table(:organizations) do
+      add_column :replication_schema, :text, null: false, default: "public"
+    end
+  end
+end

data/db/migrations/020_org_database_migrations.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  change do
+    create_table(:organization_database_migrations) do
+      primary_key :id
+      timestamptz :created_at, null: false, default: Sequel.function(:now)
+      timestamptz :updated_at
+      timestamptz :started_at
+      timestamptz :finished_at
+
+      foreign_key :organization_id, :organizations, null: false, on_delete: :cascade
+      index :organization_id
+      index :organization_id, name: :one_inprogress_migration_per_org, unique: true, where: Sequel[finished_at: nil]
+      foreign_key :started_by_id, :customers, null: true, on_delete: :set_null
+
+      text :source_admin_connection_url
+      text :destination_admin_connection_url
+      text :organization_schema
+
+      integer :last_migrated_service_integration_id, null: false, default: 0
+      timestamptz :last_migrated_timestamp, null: true
+    end
+  end
+end

data/db/migrations/021_no_default_org_schema.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    alter_table(:organizations) do
+      set_column_default(:replication_schema, nil)
+    end
+  end
+  down do
+    alter_table(:organizations) do
+      set_column_default(:replication_schema, "public")
+    end
+  end
+end

data/db/migrations/022_database_document.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  change do
+    create_table(:database_documents) do
+      primary_key :id
+      timestamptz :created_at, null: false, default: Sequel.function(:now)
+
+      text :key, null: false, unique: true
+      bytea :content, null: false
+      text :content_type, null: false
+      text :encryption_secret, null: false
+    end
+  end
+end

data/db/migrations/023_sync_target_schema.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  change do
+    alter_table(:sync_targets) do
+      add_column :last_applied_schema, :text, null: false, default: ""
+    end
+  end
+end

data/db/migrations/024_org_semaphore_jobs.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  change do
+    alter_table(:organizations) do
+      add_column :job_semaphore_size, :int, null: false, default: 10
+    end
+  end
+end

data/db/migrations/025_integration_backfill_cursor.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  change do
+    alter_table(:service_integrations) do
+      add_column :backfill_cursor, :text, null: true
+    end
+  end
+end

data/db/migrations/026_undo_integration_backfill_cursor.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  change do
+    alter_table(:service_integrations) do
+      drop_column :backfill_cursor
+    end
+  end
+end

data/db/migrations/027_sync_target_http_sync.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  change do
+    alter_table(:sync_targets) do
+      add_column :page_size, :integer, null: false, default: 500
+    end
+    alter_table(:organizations) do
+      add_column :minimum_sync_seconds, :integer, null: false, default: 10.minutes.to_i
+    end
+  end
+end

data/db/migrations/028_logged_webhook_path.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    alter_table(:logged_webhooks) do
+      add_column :request_method, :text, null: true
+      add_column :request_path, :text, null: true
+    end
+    from(:logged_webhooks).update(
+      request_method: "POST",
+      request_path: Sequel.function(:concat, "/v1/service_integrations/", :service_integration_opaque_id),
+    )
+    alter_table(:logged_webhooks) do
+      set_column_not_null :request_method
+      set_column_not_null :request_path
+    end
+  end
+  down do
+    alter_table(:logged_webhooks) do
+      drop_column :request_method
+      drop_column :request_path
+    end
+  end
+end

data/db/migrations/029_encrypt_columns.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    testmode = false
+
+    run "LOCK TABLE organizations IN ACCESS EXCLUSIVE MODE"
+    run "LOCK TABLE service_integrations IN ACCESS EXCLUSIVE MODE"
+
+    # Migrate unencrypted columns to encrypted columns
+    # without loading production models,
+    # which we should never do for migrations.
+    #
+    # To do this, we can copy the raw value to a new column,
+    # create a shim model with the column encryption plugin,
+    # assign the encrypted column from the unencrypted version,
+    # save, then drop the unencrypted columns,
+    # leaving us with columns filled with the correct encrypted values.
+
+    Sequel::Model.plugin :column_encryption do |enc|
+      enc.key 0, Webhookdb::Postgres::Model.encryption_key_0
+    end
+
+    alter_table(:organizations) do
+      add_column :readonly_connection_url_unenc, :text
+      add_column :admin_connection_url_unenc, :text
+    end
+    from(:organizations).update(
+      readonly_connection_url_unenc: :readonly_connection_url_raw,
+      admin_connection_url_unenc: :admin_connection_url_raw,
+      readonly_connection_url_raw: nil,
+      admin_connection_url_raw: nil,
+    )
+    shim_org = Class.new(Sequel::Model(from(:organizations))) do
+      plugin :column_encryption do |enc|
+        enc.column :readonly_connection_url_raw
+        enc.column :admin_connection_url_raw
+      end
+    end
+    shim_org.each do |org|
+      org.readonly_connection_url_raw = org.readonly_connection_url_unenc
+      org.admin_connection_url_raw = org.admin_connection_url_unenc
+      org.save_changes
+    end
+    alter_table(:organizations) do
+      drop_column :readonly_connection_url_unenc
+      drop_column :admin_connection_url_unenc
+    end
+    testmode && shim_org.each do |org|
+      puts org.name
+      puts "admin #{org.admin_connection_url_raw}"
+      puts "readonly #{org.readonly_connection_url_raw}"
+    end
+
+    alter_table(:service_integrations) do
+      add_column :webhook_secret_unenc, :text
+      add_column :backfill_key_unenc, :text
+      add_column :backfill_secret_unenc, :text
+      set_column_allow_null :webhook_secret
+      set_column_allow_null :backfill_key
+      set_column_allow_null :backfill_secret
+      set_column_default :webhook_secret, nil
+      set_column_default :backfill_key, nil
+      set_column_default :backfill_secret, nil
+    end
+    from(:service_integrations).update(
+      webhook_secret_unenc: :webhook_secret,
+      backfill_key_unenc: :backfill_key,
+      backfill_secret_unenc: :backfill_secret,
+      webhook_secret: nil,
+      backfill_key: nil,
+      backfill_secret: nil,
+    )
+    shim_sint = Class.new(Sequel::Model(from(:service_integrations))) do
+      plugin :column_encryption do |enc|
+        enc.column :webhook_secret
+        enc.column :backfill_key
+        enc.column :backfill_secret
+      end
+    end
+    shim_sint.each do |sint|
+      sint.webhook_secret = sint.webhook_secret_unenc
+      sint.backfill_key = sint.backfill_key_unenc
+      sint.backfill_secret = sint.backfill_secret_unenc
+      sint.save_changes
+    end
+    alter_table(:service_integrations) do
+      drop_column :webhook_secret_unenc
+      drop_column :backfill_key_unenc
+      drop_column :backfill_secret_unenc
+    end
+    testmode && shim_sint.each do |sint|
+      puts "#{sint.opaque_id}: #{sint.webhook_secret} / #{sint.backfill_key} / #{sint.backfill_secret}"
+    end
+    raise "did not apply migration" if testmode
+  end
+end

data/db/migrations/030_org_sync_target_timeout.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  change do
+    alter_table(:organizations) do
+      add_column :sync_target_timeout, :integer, null: false, default: 30
+    end
+  end
+end

data/db/migrations/031_org_max_query_rows.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  change do
+    alter_table(:organizations) do
+      add_column :max_query_rows, :integer, null: true
+    end
+  end
+end

data/db/migrations/032_remove_db_defaults.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  change do
+    alter_table(:sync_targets) do
+      set_column_default :page_size, nil
+    end
+    alter_table(:organizations) do
+      set_column_default :minimum_sync_seconds, nil
+    end
+  end
+end

data/db/migrations/033_backfill_jobs.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  change do
+    create_table(:backfill_jobs) do
+      primary_key :id
+      timestamptz :created_at, null: false, default: Sequel.function(:now)
+      timestamptz :updated_at
+
+      timestamptz :started_at
+      timestamptz :finished_at
+
+      text :opaque_id, null: false, unique: true
+
+      foreign_key :service_integration_id, :service_integrations, null: false, on_delete: :cascade
+      index :service_integration_id
+
+      foreign_key :parent_job_id, :backfill_jobs
+      index :parent_job_id
+
+      foreign_key :created_by_id, :customers, null: true, on_delete: :set_null
+
+      boolean :incremental, null: false
+    end
+  end
+end

data/db/migrations/034_backfill_job_criteria.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  change do
+    alter_table(:backfill_jobs) do
+      add_column :criteria, :jsonb, null: false, default: "{}"
+    end
+  end
+end

data/db/migrations/035_synchronous_backfill.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  change do
+    alter_table(:organizations) do
+      add_column :priority_backfill, :boolean, default: false, null: false
+    end
+  end
+end

data/db/migrations/036_oauth.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  change do
+    create_table(:oauth_sessions) do
+      primary_key :id
+      timestamptz :created_at, null: false, default: Sequel.function(:now)
+
+      foreign_key :customer_id, :customers, null: true, on_delete: :cascade
+      index :customer_id
+
+      foreign_key :organization_id, :organizations, null: true, on_delete: :cascade
+
+      text :user_agent, null: false
+      inet :peer_ip, null: false
+      text :oauth_state, null: false
+      text :authorization_code, null: true
+    end
+
+    alter_table(:testing_pixies) do
+      # We're adding this column to the testing pixie to test how non-JSON types are handled by
+      # async events on triggered in sequel hooks
+      add_column :ip, :inet
+    end
+  end
+end

data/db/migrations/037_oauth_used.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  change do
+    alter_table(:oauth_sessions) do
+      add_column :used_at, :timestamptz
+    end
+  end
+end