webhookdb 0.1.0

data/lib/webhookdb/plivo.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require "appydays/configurable"
+require "appydays/loggable"
+
+module Webhookdb::Plivo
+  include Appydays::Configurable
+  include Appydays::Loggable
+
+  configurable(:plivo) do
+    setting :http_timeout, 25
+  end
+
+  def self.request(method, tail, auth_id:, auth_token:, body: nil, **options)
+    tail = tail.delete_suffix("/")
+    url = "https://api.plivo.com/v1/Account/#{auth_id}#{tail}/"
+    options[:basic_auth] = {username: auth_id, password: auth_token}
+    options[:logger] = self.logger
+    if body
+      options[:headers] = {"Content-Type" => "application/json"}
+      options[:body] = body.to_json
+    end
+    options[:method] = method if method != :get
+    return Webhookdb::Http.send(method, url, **options)
+  end
+
+  def self.webhook_response(request, auth_token)
+    raise Webhookdb::InvalidPrecondition, "auth_token cannot be nil/blank" if auth_token.blank?
+    # See https://www.plivo.com/docs/sms/xml/request#validation
+    # See https://www.plivo.com/docs/sms/concepts/signature-validation#code
+    (signature = request.env["HTTP_X_PLIVO_SIGNATURE_V2"]) or
+      return Webhookdb::WebhookResponse.error("missing signature")
+    (nonce = request.env["HTTP_X_PLIVO_SIGNATURE_V2_NONCE"]) or
+      return Webhookdb::WebhookResponse.error("missing nonce")
+    url = request.url
+    uri = url.split("?")[0]
+    ok = self._valid_signature?(uri, nonce, signature, auth_token)
+    return ok ? Webhookdb::WebhookResponse.ok : Webhookdb::WebhookResponse.error("invalid signature")
+  end
+
+  # Copied from https://github.com/plivo/plivo-ruby/blob/119038345475c6216bf040926747105b66fd588a/lib/plivo/utils.rb#L213C1-L220C8
+  # We do not use the Plivo gem since it is a mess.
+  def self._valid_signature?(uri, nonce, signature, auth_token)
+    parsed_uri = URI.parse(uri)
+    uri_details = {host: parsed_uri.host, path: parsed_uri.path}
+    uri_builder_module = parsed_uri.scheme == "https" ? URI::HTTPS : URI::HTTP
+    data_to_sign = uri_builder_module.build(uri_details).to_s + nonce
+    sha256_digest = OpenSSL::Digest.new("sha256")
+    encoded_digest = Base64.encode64(OpenSSL::HMAC.digest(sha256_digest, auth_token, data_to_sign)).strip
+    return ActiveSupport::SecurityUtils.secure_compare(encoded_digest, signature)
+  end
+end

data/lib/webhookdb/postgres/maintenance.rb

@@ -0,0 +1,166 @@
+# frozen_string_literal: true
+
+module Webhookdb::Postgres::Maintenance
+  include Appydays::Configurable
+
+  configurable(:pg_maintenance) do
+    setting :debug, true
+    setting :docker, true
+    # Match what's available on the server.
+    setting :pg_repack_image, "hartmutcouk/pg-repack-docker:1.4.7"
+  end
+
+  class Base
+    # @!attribute service_integration
+    #   @return [Webhookdb::ServiceIntegration]
+    attr_reader :service_integration
+
+    def debug? = Webhookdb::Postgres::Maintenance.debug
+
+    def initialize(service_integration)
+      @service_integration = service_integration
+    end
+
+    def conn_params
+      org_uri = URI(self.service_integration.organization.admin_connection_url_raw)
+      superuser_url = Webhookdb::Organization::DbBuilder.available_server_urls.find { |u| URI(u).host == org_uri.host }
+      raise Webhookdb::InvalidPrecondition, "cannot find superuser url for #{org_uri.host}" if superuser_url.nil?
+      sup_uri = URI(superuser_url)
+      return {
+        user: sup_uri.user,
+        password: sup_uri.password,
+        host: sup_uri.host,
+        port: sup_uri.port,
+        database: org_uri.path&.delete_prefix("/"),
+        table: self.service_integration.table_name,
+      }
+    end
+  end
+
+  class Query < Base
+    def connstr
+      h = self.conn_params
+      return "postgres://#{h[:user]}:#{h[:password]}@#{h[:host]}:#{h[:port]}/#{h[:database]}"
+    end
+
+    def query = raise NotImplementedError
+
+    def fetch
+      Sequel.connect(self.connstr) do |c|
+        c.fetch(self.query).all
+      end
+    end
+
+    def run = raise NotImplementedError
+    def run_fmt = raise NotImplementedError
+  end
+
+  class Command < Base
+    def docker? = Webhookdb::Postgres::Maintenance.docker
+
+    def psql_conn_params
+      h = self.conn_params
+      return [
+        "--no-password",
+        "-U", h[:user],
+        "-h", h[:host],
+        "-p", h[:port],
+        "--dbname", h[:database],
+      ]
+    end
+
+    def command_list = raise NotImplementedError
+    def docker_image = raise NotImplementedError
+    def extension = nil
+
+    def docker_preamble
+      return [
+        "docker",
+        "run",
+        "-e",
+        "PGPASSWORD=#{self.conn_params[:password]}",
+        "-it",
+        "--rm",
+        self.docker_image,
+      ]
+    end
+
+    def create_extension_command_list
+      a = ["PGPASSWORD=#{self.conn_params[:password]}", "psql"]
+      a += self.psql_conn_params
+      a << "-c"
+      a << "'CREATE EXTENSION IF NOT EXISTS #{self.extension}'"
+      return a
+    end
+
+    def command_strings
+      c = []
+      c << self.shelex(self.create_extension_command_list) if self.extension
+      c << self.shelex(self.command_list)
+      return c
+    end
+
+    def shelex(a)
+      return a.join(" ")
+    end
+  end
+
+  class Repack < Command
+    def extension = "pg_repack"
+
+    def docker_image = Webhookdb::Postgres::Maintenance.pg_repack_image
+
+    def command_list
+      c = []
+      c += self.docker_preamble if self.docker?
+      c << "pg_repack"
+      c += self.psql_conn_params
+      c += ["--table", self.conn_params[:table]]
+      c << "--no-superuser-check"
+      c << "--dry-run"
+      c += ["--echo", "--elevel=DEBUG"] if self.debug?
+      return c
+    end
+  end
+
+  class Count < Query
+    def query = "SELECT reltuples AS estimate FROM pg_class WHERE relname = '#{self.service_integration.table_name}'"
+
+    def run
+      r = self.fetch
+      return r[0][:estimate].to_i
+    end
+
+    def run_fmt = ActiveSupport::NumberHelper.number_to_delimited(self.run.to_i)
+  end
+
+  class Tables < Query
+    def query
+      return <<~SQL
+        SELECT
+            relname AS "relation",
+            reltuples as "tuples",
+            pg_size_pretty(pg_total_relation_size(C .oid)) as "size"
+        FROM pg_class C
+        LEFT JOIN pg_namespace N ON (N.oid = C.relnamespace)
+        WHERE nspname NOT IN ('pg_catalog', 'information_schema')
+        AND C.relkind = 'r'
+        AND nspname !~ '^pg_toast'
+        ORDER BY C.relname
+      SQL
+    end
+
+    def run
+      return self.fetch
+    end
+
+    def run_fmt
+      rows = self.run
+      namejust = rows.map { |r| r[:relation].size }.max
+      return rows.map do |r|
+        tuples = ActiveSupport::NumberHelper.number_to_delimited(r[:tuples].to_i).rjust(12, " ")
+        "#{r[:relation].ljust(namejust + 1, ' ')} #{r[:size].ljust(7, ' ')} #{tuples}"
+      end.join("\n")
+    end
+  end
+end

data/lib/webhookdb/postgres/model.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+require "appydays/configurable"
+require "appydays/loggable"
+require "pg"
+require "sequel"
+require "tsort"
+
+require "webhookdb"
+require "webhookdb/postgres"
+require "webhookdb/postgres/validations"
+require "webhookdb/postgres/model_utilities"
+
+# Initialize the Webhookdb::Postgres::Model class as an abstract model class (i.e.,
+# without a default dataset). This prevents it from looking for a table called
+# `models`, and makes inheriting it more straightforward.
+# Thanks to Michael Granger and Jeremy Evans for the suggestion.
+Webhookdb::Postgres::Model = Class.new(Sequel::Model)
+Webhookdb::Postgres::Model.def_Model(Webhookdb::Postgres)
+
+class Webhookdb::Postgres::Model
+  include Appydays::Configurable
+  extend Webhookdb::Postgres::ModelUtilities
+  include Appydays::Loggable
+
+  configurable(:webhookdb_db) do
+    setting :uri, "postgres:/webhookdb_test", key: "DATABASE_URL"
+
+    # rubocop:disable Naming/VariableNumber
+    setting :encryption_key_0, "Tc3X6zkxXgZfHE81MFz2EILStV++BuQY"
+    # rubocop:enable Naming/VariableNumber
+
+    setting :extension_schema, "public"
+
+    after_configured do
+      self.connect(self.uri)
+    end
+  end
+
+  # Add one or more extension +modules+ to the receiving class. This allows subsystems
+  # like Orders, etc. to decorate models outside of their purview
+  # without introducing unnecessary dependencies.
+  #
+  # Each one of the given +modules+ will be included in the receiving model class, and
+  # if it also contains a constant called ClassMethods, the model class will be
+  # also be extended with it.
+  #
+  # @example Add order methods to Webhookdb::Customer
+  #
+  #   module Webhookdb::Orders::CustomerExtensions
+  #
+  #       # Add some associations for Order models
+  #       def included( model )
+  #           super
+  #           model.one_to_many :orders, Sequel[:app][:orders]
+  #       end
+  #
+  #       def first_order
+  #           self.orders.first
+  #       end
+  #
+  #   end
+  def self.add_extensions(*modules)
+    self.logger.info "Adding extensions to %p: %p" % [self, modules]
+
+    modules.each do |mod|
+      include(mod)
+      if mod.const_defined?(:ClassMethods)
+        submod = mod.const_get(:ClassMethods)
+        self.extend(submod)
+      end
+      if mod.const_defined?(:PrependedMethods)
+        submod = mod.const_get(:PrependedMethods)
+        prepend(submod)
+      end
+    end
+  end
+
+  plugin :column_encryption do |enc|
+    enc.key 0, self.encryption_key_0
+  end
+end

data/lib/webhookdb/postgres/model_utilities.rb

@@ -0,0 +1,382 @@
+# frozen_string_literal: true
+
+require "tsort"
+require "pg"
+require "webhookdb/dbutil"
+require "webhookdb/postgres"
+
+# A collection of utilities that can be added to model superclasses.
+module Webhookdb::Postgres::ModelUtilities
+  include TSort
+
+  # Extension callback -- register the +model_class+ with Webhookdb::Postgres.
+  def self.extended(model_class)
+    super
+
+    # Sequel::Model API -- load some plugins
+    model_class.plugin(:dirty)
+    model_class.plugin(:json_serializer)
+    model_class.plugin(:many_through_many)
+    model_class.plugin(:subclasses)
+    model_class.plugin(:tactical_eager_loading)
+    model_class.plugin(:update_or_create)
+    model_class.plugin(:validation_helpers)
+
+    model_class.include(Appydays::Loggable)
+    model_class.extend(ClassMethods)
+    model_class.include(InstanceMethods)
+    model_class.dataset_module(DatasetMethods)
+    model_class.include(Webhookdb::Postgres::Validations)
+
+    Webhookdb::Postgres.register_model_superclass(model_class)
+  end
+
+  module ClassMethods
+    # The application name, set on database connections.
+    attr_reader :appname
+
+    # Connect to the given URI using the standard extensions and other options.
+    def connect(uri)
+      newdb = Sequel.connect(
+        uri,
+        logger: self.logger,
+        extensions: [
+          :is_distinct_from,
+          :pagination,
+          :pg_json,
+          :pg_inet,
+          :pg_array,
+          :pg_streaming,
+          :pg_range,
+          :pg_interval,
+          :pg_triggers,
+          :pretty_table,
+        ],
+        **Webhookdb::Dbutil.configured_connection_options,
+      )
+      self.db = newdb
+      self.descendents.each do |subclass|
+        subclass.db = newdb
+      end
+    end
+
+    # Set the PostgreSQL application name to +name+ to allow per-application connection
+    # tracking and other fun stuff.
+    def appname=(name)
+      @appname = name
+      self.update_connection_appname
+    end
+
+    # Set the connection's application name if there is one.
+    def update_connection_appname
+      return unless self.db
+      self.logger.debug "Setting application name to %p" % [self.appname]
+      self.db.synchronize do |conn|
+        escaped = conn.escape_string(self.appname)
+        conn.exec("SET application_name TO '%s'" % [escaped])
+      end
+    end
+
+    # Fetch a model class by its +classname+. This can be the fully-qualified
+    # name, or just the bit after 'Webhookdb::'.
+    def by_name(classname)
+      return self.descendents.find do |cl|
+        cl.name&.end_with?(classname)
+      end
+    end
+
+    # Return the Array of the schemas used by all descendents of the receiving
+    # model class.
+    def all_loaded_schemas
+      return self.descendents.map(&:schema_name).uniq.compact
+    end
+
+    # Create a new schema named +name+ (if it doesn't already exist).
+    def create_schema(name, &block)
+      self.db.create_schema(name, if_not_exists: true)
+      self.instance_eval(&block) if block
+    end
+
+    # Create the schema named +name+, dropping any previous schema by the same name.
+    def create_schema!(name, &)
+      self.drop_schema!(name)
+      self.create_schema(name, &)
+    end
+
+    # Drop the empty schema named +name+ (if it exists).
+    def drop_schema(name)
+      self.db.drop_schema(name, if_exists: true)
+    end
+
+    # Drop the schema named +name+ and all of its tables.
+    def drop_schema!(name)
+      self.db.drop_schema(name, if_exists: true, cascade: true)
+    end
+
+    # Returns +true+ if a schema named +name+ exists.
+    def schema_exists?(name=self.schema_name)
+      ds = self.db[Sequel[:pg_catalog][:pg_namespace]].
+        filter(nspname: name.to_s).
+        select(:nspname)
+
+      return ds.first ? true : false
+    end
+
+    # Return the name of the schema the receiving class is in.
+    def schema_name
+      schemaname, = self.db.send(:schema_and_table, self.table_name)
+      return schemaname
+    end
+
+    def now_sql
+      return Webhookdb::Postgres.now_sql
+    end
+
+    def extension_schema
+      return "public"
+    end
+
+    def install_all_extensions
+      extensions = [
+        "citext",
+        "pgcrypto",
+        "btree_gist",
+        "pg_trgm",
+      ]
+      extensions.each do |ext|
+        self.db.execute("CREATE EXTENSION IF NOT EXISTS #{ext} WITH SCHEMA #{self.extension_schema}")
+      end
+    end
+
+    # TSort API -- yield each model class.
+    def tsort_each_node(&)
+      self.descendents.select(&:name).each(&)
+    end
+
+    # TSort API -- yield each of the given +model_class+'s dependent model
+    # classes.
+    def tsort_each_child(model_class)
+      # Include (non-anonymous) parents other than Model
+      non_anon_parents = model_class.ancestors[1..].
+        select { |cl| cl < self }.
+        select(&:name)
+      # rubocop:disable Style/ExplicitBlockArgument
+      non_anon_parents.each do |parentclass|
+        yield(parentclass)
+      end
+      # rubocop:enable Style/ExplicitBlockArgument
+
+      # Include associated classes for which this model class's table has a
+      # foreign key
+      model_class.association_reflections.each do |name, config|
+        next if config[:polymorphic]
+
+        associated_class = Object.const_get(config[:class_name])
+
+        if config[:type] == :many_to_one
+          self.logger.debug "  %p#%s is dependent on %p" %
+            [model_class, name, associated_class]
+          yield(associated_class)
+        else
+          self.logger.debug "  %p#%s is *not* dependent on %p" %
+            [model_class, name, associated_class]
+        end
+      end
+    end
+  end
+
+  # Like +find_or_create+, but will +find+ again if the +create+
+  # call fails due to a +Sequel::UniqueConstraintViolation+,
+  # which is usually caused by a race condition.
+  def find_or_create_or_find(params, &)
+    # Set a savepoint, because the DB error will abort the current transaction.
+    self.db.transaction(savepoint: true) do
+      return self.find_or_create(params, &)
+    end
+  rescue Sequel::UniqueConstraintViolation
+    return self.find(params)
+  end
+
+  module InstanceMethods
+    # Return a human-readable representation of the object as a String suitable for debugging.
+    def inspect
+      values = self.values.reject do |k, v|
+        v.blank? || k.to_s.end_with?("_currency")
+      end
+      begin
+        encrypted = self.class.send(:column_encryption_metadata).to_set { |(col, _)| col.to_s }
+      rescue NoMethodError
+        encrypted = Set.new
+      end
+      values = values.map do |(k, v)|
+        k = k.to_s
+        v = if v.is_a?(Time)
+              self.inspect_time(v)
+        elsif v.respond_to?(:db_type) && v.db_type.to_s == "tstzrange"
+          "%s%s...%s%s" % [
+            v.exclude_begin? ? "(" : "[",
+            v.begin ? self.inspect_time(v.begin) : "nil",
+            v.end ? self.inspect_time(v.end) : "nil",
+            v.exclude_end? ? ")" : "]",
+          ]
+        elsif k.end_with?("_cents")
+          accessor = k.match(/^([a-z_]+)_cents/)[1]
+          k = accessor
+          self.send(accessor).format
+        elsif encrypted.include?(k)
+          # Render encrypted fields as xyz...abc, or if a URL, hide the user/password.
+          unenc = self.send(k)
+          if unenc.length < 10
+            "\"...\""
+          elsif unenc.include?("://")
+            uri = URI(unenc)
+            uri.user = "*"
+            uri.password = "*"
+            uri.to_s.inspect
+          else
+            "\"#{unenc[..2]}...#{unenc[-3..]}\""
+          end
+        else
+          v.inspect
+        end
+        "#{k}: #{v}"
+      end
+      return "#<%p %s>" % [self.class, values.join(", ")]
+    end
+
+    def inspect_time(t)
+      return t.in_time_zone(Time.zone).strftime("%Y-%m-%d %H:%M:%S")
+    end
+
+    # Return the objects validation errors as full messages joined with commas.
+    def error_messages
+      return self.errors.full_messages.join(", ")
+    end
+
+    # Return the string used as a topic for events sent from the receiving object.
+    def event_prefix
+      prefix = self.class.name or return # No events for anonymous classes
+      return prefix.gsub("::", ".").downcase
+    end
+
+    # Publish an event from the receiving object of the specified +type+ and with the given +payload+.
+    # This does *not* wait for the transaction to complete, so subscribers may not be able to observe
+    # any model changes in the database. You probably want to use published_deferred.
+    def publish_immediate(type, *payload)
+      prefix = self.event_prefix or return
+      Amigo.publish(prefix + "." + type.to_s, *payload)
+    end
+
+    # Publish an event in the current db's/transaction's +after_commit+ hook.
+    def publish_deferred(type, *payload)
+      Webhookdb::Postgres.defer_after_commit(self.db) do
+        self.publish_immediate(type, *payload)
+      end
+    end
+
+    # Take an exclusive lock on the receiver, ensuring nothing else has updated the object in the meantime.
+    # If the updated_at changed from what's on the receiver, to after it acquired the lock, raise LockFailed.
+    # Save changes and touch updated_at after calling the given block.
+    def resource_lock!
+      self.db.transaction do
+        old_updated = self.round_time(self.updated_at)
+        self.lock!
+        new_updated = self.round_time(self.updated_at)
+        raise Webhookdb::LockFailed if old_updated != new_updated
+        result = yield(self)
+        self.updated_at = Time.now
+        self.save_changes
+        return result
+      end
+    end
+
+    # Round +Time+ t to remove nanoseconds, since Postgres can only store microseconds.
+    protected def round_time(t)
+      return nil if t.nil?
+      return t.change(nsec: t.usec * 1000)
+    end
+
+    protected def now_sql
+      return Webhookdb::Postgres.now_sql
+    end
+
+    protected def prep_amigo_payload(data)
+      # Job arguments to Amigo::Router must be native JSON types, so we do some preprocessing
+      data.to_h do |key, value|
+        # Empty PG Ranges cannot be converted to ruby types for some reason, we substitute nil
+        value_is_empty_range = value.is_a?(Sequel::Postgres::PGRange) && value.empty?
+        new_value = value_is_empty_range ? nil : value.as_json
+        [key, new_value]
+      end
+    end
+
+    # Sequel hook -- send an asynchronous event after the model is saved.
+    def after_create
+      super
+      self.publish_deferred("created", self.id, prep_amigo_payload(self.values))
+    end
+
+    # Sequel hook -- send an asynchronous event after the save is committed.
+    def after_update
+      super
+      self.publish_deferred("updated", self.id, prep_amigo_payload(self.previous_changes))
+    end
+
+    # Sequel hook -- send an event after a transaction that destroys the object is committed.
+    def after_destroy
+      super
+      self.publish_deferred("destroyed", self.id, prep_amigo_payload(self.values))
+    end
+  end
+
+  module DatasetMethods
+    # Helper for applying multiple conditions for Sequel, where some can be nil.
+    def reduce_expr(op_symbol, operands, method: :where)
+      return Webhookdb::Dbutil.reduce_expr(self, op_symbol, operands, method:)
+    end
+
+    # Call a block for each row in a dataset.
+    # This is the same as paged_each or use_cursor.each, except that for each page,
+    # rows are re-fetched using self.where(primary_key => [pks]).all to enable eager loading.
+    #
+    # (Note that paged_each does not do eager loading, which makes enumerating model associations very slow)
+    def each_cursor_page(page_size: 500, order: :id, &block)
+      raise LocalJumpError unless block
+      raise "dataset requires a use_cursor method, class may need `extension(:pagination)`" unless
+        self.respond_to?(:use_cursor)
+      model = self.model
+      pk = model.primary_key
+      current_chunk_pks = []
+      order = [order] unless order.respond_to?(:to_ary)
+      self.naked.select(pk).order(*order).use_cursor(rows_per_fetch: page_size, hold: true).each do |row|
+        current_chunk_pks << row[pk]
+        next if current_chunk_pks.length < page_size
+        page = model.where(pk => current_chunk_pks).order(*order).all
+        current_chunk_pks.clear
+        page.each(&block)
+      end
+      model.where(pk => current_chunk_pks).order(*order).all.each(&block)
+    end
+
+    # See each_cursor_page, but takes an additional action on each chunk of returned rows.
+    # The action is called with pages of return values from the block when a page is is reached.
+    # Each call to action should return nil, a result, or an array of results (nil results are ignored).
+    #
+    # The most common case is for ETL: process one dataset, map it in a block to return new row values,
+    # and multi_insert it into a different table.
+    def each_cursor_page_action(action:, page_size: 500, order: :id)
+      raise LocalJumpError unless block_given?
+      returned_rows_chunk = []
+      self.each_cursor_page(page_size:, order:) do |instance|
+        new_row = yield(instance)
+        next if action.nil? || new_row.nil?
+        new_row.respond_to?(:to_ary) ? returned_rows_chunk.concat(new_row) : returned_rows_chunk.push(new_row)
+        if returned_rows_chunk.length >= page_size
+          action.call(returned_rows_chunk)
+          returned_rows_chunk.clear
+        end
+      end
+      action&.call(returned_rows_chunk)
+    end
+  end
+end

data/lib/webhookdb/postgres/testing_pixie.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require "webhookdb/postgres/model"
+require "sequel/plugins/money_fields"
+require "sequel/plugins/tstzrange_fields"
+
+# A placeholder model for the magical mixin classes (imaged, ticketable, annotated).
+# We can't easily test these things with non-Webhookdb::Postgres::Model subclasses,
+# and we can't easily define Webhookdb::Postgres::Model subclasses as part of a spec/transaction.
+# Even if Sequel can handle it, it creates some global state issues.
+# Make sure this class is only required during specs, and not normal gem usage.
+# It should never have a persistent table; only a table in the test database.
+class Webhookdb::Postgres::TestingPixie < Webhookdb::Postgres::Model(:testing_pixies)
+  plugin :money_fields, :price_per_unit
+  plugin :tstzrange_fields, :active_during
+end