webhookdb 0.1.0

data/lib/webhookdb/postgres/validations.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require "webhookdb/postgres"
+
+module Webhookdb::Postgres::Validations
+  # Ensures that only one of the passed columns is not nil.
+  def validates_mutually_exclusive(*cols, predicate: :nil?)
+    truthy_cols = cols.find_all { |col| !self[col].send(predicate) }
+    multiple_set = truthy_cols.length > 1
+    return unless multiple_set
+    self.errors.add(truthy_cols.first, "is mutually exclusive with other set columns #{truthy_cols[1..].join(', ')}")
+  end
+
+  # Ensures that either all of the passed columns are nil or none of them are.
+  def validates_all_or_none(*cols, predicate: :nil?)
+    truthy_cols = cols.find_all { |col| !self[col].send(predicate) }
+
+    return if truthy_cols.empty? || truthy_cols.length == cols.length
+
+    msg = "the columns #{cols[1..].join(', ')} must all be set or all be #{predicate.to_s.delete_suffix('?')}"
+    self.errors.add(cols.first, msg)
+  end
+
+  # Ensures that at least one of the passed columns is not nil.
+  def validates_at_least_one_of(*cols, predicate: :nil?)
+    any_truthy = cols.any? { |col| !self[col].send(predicate) }
+    return if any_truthy
+    msg = "at least one of #{cols.join(', ')} must be not #{predicate.to_s.delete_suffix('?')}"
+    self.errors.add(cols.first, msg)
+  end
+
+  # Ensures that one and only one of the passed columns is not nil.
+  def validates_exactly_one_of(*cols, predicate: :nil?)
+    self.validates_at_least_one_of(*cols, predicate:)
+    self.validates_mutually_exclusive(*cols, predicate:)
+  end
+
+  def validates_ip_address(col)
+    return if self[col].respond_to?(:ipv4?)
+    begin
+      IPAddr.new(self[col])
+    rescue IPAddr::Error
+      self.errors.add(col, "is not a valid INET address")
+    end
+  end
+end

data/lib/webhookdb/postgres.rb

@@ -0,0 +1,176 @@
+# frozen_string_literal: true
+
+require "pg"
+require "appydays/loggable"
+require "sequel"
+require "sequel/core"
+require "sequel/adapters/postgres"
+
+require "appydays/loggable/sequel_logger"
+require "webhookdb"
+
+module Webhookdb::Postgres
+  extend Webhookdb::MethodUtilities
+  include Appydays::Loggable
+
+  class InTransaction < StandardError; end
+
+  singleton_attr_accessor :unsafe_skip_transaction_check
+  @unsafe_skip_transaction_check = false
+  def self.check_transaction(db, error_msg)
+    return true if self.unsafe_skip_transaction_check
+    return true unless db.in_transaction?
+    raise InTransaction, error_msg
+  end
+
+  # Sequel API -- load some global extensions and plugins
+  Sequel.extension(
+    :core_extensions,
+    :core_refinements,
+    :is_distinct_from,
+    :pg_array,
+    :pg_array_ops,
+    :pg_inet,
+    :pg_inet_ops,
+    :pg_json,
+    :pg_json_ops,
+    :pg_range,
+    :pg_range_ops,
+    :symbol_as_refinement,
+  )
+  Sequel::Model.plugin(:force_encoding, "UTF-8")
+
+  # Require paths for model superclasses.
+  SUPERCLASSES = [
+    "webhookdb/postgres/model",
+  ].freeze
+
+  # Require paths for all Sequel models used by the app.
+  MODELS = [
+    "webhookdb/backfill_job",
+    "webhookdb/customer",
+    "webhookdb/customer/reset_code",
+    "webhookdb/database_document",
+    "webhookdb/idempotency",
+    "webhookdb/logged_webhook",
+    "webhookdb/message/body",
+    "webhookdb/message/delivery",
+    "webhookdb/oauth/session",
+    "webhookdb/organization",
+    "webhookdb/organization/database_migration",
+    "webhookdb/organization_membership",
+    "webhookdb/role",
+    "webhookdb/service_integration",
+    "webhookdb/subscription",
+    "webhookdb/sync_target",
+    "webhookdb/webhook_subscription",
+    "webhookdb/webhook_subscription/delivery",
+  ].freeze
+
+  # If true, deferred model events publish immediately.
+  # Should only be true for some types of testing.
+  singleton_predicate_accessor :do_not_defer_events
+
+  # The list of model superclasses (that get their own database connections)
+  singleton_attr_reader :model_superclasses
+  @model_superclasses = Set.new
+
+  ### Register the given +superclass+ as a base class for a set of models, for operations
+  ### which should happen on all the current database connections.
+  def self.register_model_superclass(superclass)
+    self.logger.debug "Registered model superclass: %p" % [superclass]
+    self.model_superclasses << superclass
+  end
+
+  ##
+  # The list of models that will be required once the database connection has been established.
+  singleton_attr_reader :registered_models
+  @registered_models = Set.new
+
+  ### Add a +path+ to require once the database connection is set.
+  def self.register_model(path)
+    self.logger.debug "Registered model for requiring: %s" % [path]
+
+    # If the connection's set, require the path immediately.
+    if self.model_superclasses.any?(&:db)
+      Appydays::Loggable[self].silence(:fatal) do
+        require(path)
+      end
+    end
+
+    self.registered_models << path
+  end
+
+  ### Require the model classes once the database connection has been established
+  def self.require_models
+    self.registered_models.each do |path|
+      require path
+    end
+  end
+
+  ### Call the block for each registered model superclass.
+  def self.each_model_superclass(&)
+    self.model_superclasses.each(&)
+  end
+
+  def self.each_model_class(&)
+    self.each_model_superclass do |sc|
+      sc.descendants.each(&)
+    end
+  end
+
+  def self.run_all_migrations(target: nil)
+    Sequel.extension :migration
+    Webhookdb::Postgres.each_model_superclass do |cls|
+      cls.install_all_extensions
+      Sequel::Migrator.run(cls.db, Pathname(__FILE__).dirname.parent.parent + "db/migrations", target:)
+    end
+  end
+
+  # We can always register the models right away, since it does not have a side effect.
+  MODELS.each do |m|
+    self.register_model(m)
+  end
+
+  # After configuration, load superclasses. You may need these without loading models,
+  # like if you need access to their DBs without loading them
+  # (if their tables do not yet exist)
+  def self.load_superclasses
+    SUPERCLASSES.each do |sc|
+      require(sc)
+    end
+  end
+
+  # After configuration, require in the model superclass files,
+  # to make sure their .db gets set and they're in model_superclasses.
+  def self.load_models
+    self.load_superclasses
+    Appydays::Loggable[self].silence(:fatal) do
+      self.require_models
+    end
+  end
+
+  # Return 'Time.now' as an expression suitable for Sequel/SQL.
+  # In some cases (like range @> expressions) you need to cast to a timestamptz explicitly,
+  # the implicit cast isn't enough.
+  # And because 'Time.now' is an external dependency, we should always use Sequel.delay,
+  # to avoid any internal caching it will do,
+  # like in association blocks: https://github.com/jeremyevans/sequel/blob/master/doc/association_basics.rdoc#block-
+  def self.now_sql(&block)
+    block ||= -> { Time.now }
+    return Sequel.delay { Sequel.cast(block.call, :timestamptz) }
+  end
+
+  # Call block immediately if not deferring events; otherwise call it after db commit.
+  def self.defer_after_commit(db, &block)
+    raise LocalJumpError unless block
+    return yield if self.do_not_defer_events?
+    return db.after_commit(&block)
+  end
+
+  def self.defer_after_rollback(db, &block)
+    raise LocalJumpError unless block
+    return yield if self.do_not_defer_events?
+    return db.after_rollback(&block)
+  end
+end

data/lib/webhookdb/postmark.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require "appydays/configurable"
+
+module Webhookdb::Postmark
+  include Appydays::Configurable
+
+  configurable(:postmark) do
+    # See https://postmarkapp.com/support/article/800-ips-for-firewalls#webhooks
+    setting :allowed_ips,
+            ["127.0.0.1", "3.134.147.250", "50.31.156.6", "50.31.156.77", "18.217.206.57"],
+            convert: ->(s) { s.split.map(&:strip) }
+  end
+
+  def self.webhook_response(request)
+    ip = request.ip
+    allowed = self.allowed_ips.include?(ip)
+    return allowed ? Webhookdb::WebhookResponse.ok : Webhookdb::WebhookResponse.error("invalid ip")
+  end
+end

data/lib/webhookdb/redis.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require "appydays/configurable"
+require "redis_client"
+
+module Webhookdb::Redis
+  include Appydays::Configurable
+
+  class << self
+    attr_accessor :cache
+  end
+
+  configurable(:redis) do
+    setting :cache_url, "redis://localhost:6379/0"
+    setting :cache_url_provider, "REDIS_URL"
+    setting :verify_ssl, false
+
+    after_configured do
+      url = ENV.fetch(self.cache_url_provider, self.cache_url)
+      cache_params = {url:, reconnect_attempts: 1}
+      cache_params[:ssl_params] = {verify_mode: OpenSSL::SSL::VERIFY_NONE} unless
+        self.verify_ssl
+      redis_config = RedisClient.config(**cache_params)
+      self.cache = redis_config.new_pool(
+        timeout: Webhookdb::Dbutil.pool_timeout,
+        size: Webhookdb::Dbutil.max_connections,
+      )
+    end
+  end
+
+  def self.cache_key(parts)
+    tail = parts.join("/")
+    return "cache/#{tail}"
+  end
+end

data/lib/webhookdb/replicator/atom_single_feed_v1.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+require "down"
+
+require "webhookdb/xml"
+
+class Webhookdb::Replicator::AtomSingleFeedV1 < Webhookdb::Replicator::Base
+  include Appydays::Loggable
+
+  # @return [Webhookdb::Replicator::Descriptor]
+  def self.descriptor
+    return Webhookdb::Replicator::Descriptor.new(
+      name: "atom_single_feed_v1",
+      ctor: ->(sint) { Webhookdb::Replicator::AtomSingleFeedV1.new(sint) },
+      feature_roles: [],
+      resource_name_singular: "Atom Single Feed",
+      supports_backfill: true,
+      description: "Convert any Atom XML feed into a database table for querying and persistent archiving.",
+      api_docs_url: "https://en.wikipedia.org/wiki/Atom_(web_standard)",
+    )
+  end
+
+  def _remote_key_column
+    return Webhookdb::Replicator::Column.new(:entry_id, TEXT, data_key: "id")
+  end
+
+  CONV_GEO_LAT = Webhookdb::Replicator::Column.converter_array_element(index: 0, sep: " ", cls: DECIMAL)
+  CONV_GEO_LNG = Webhookdb::Replicator::Column.converter_array_element(index: 1, sep: " ", cls: DECIMAL)
+
+  def _denormalized_columns
+    col = Webhookdb::Replicator::Column
+    return [
+      col.new(:row_created_at, TIMESTAMP, index: true, optional: true, defaulter: :now),
+      col.new(:updated, TIMESTAMP, index: true),
+      col.new(:title, TEXT),
+      col.new(:published, TIMESTAMP, index: true, optional: true),
+      col.new(:geo_lat, DECIMAL, data_key: "georss:point", optional: true, converter: CONV_GEO_LAT),
+      col.new(:geo_lng, DECIMAL, data_key: "georss:point", optional: true, converter: CONV_GEO_LNG),
+    ]
+  end
+
+  def _timestamp_column_name
+    return :updated
+  end
+
+  def _resource_and_event(request)
+    return request.body, nil
+  end
+
+  def _update_where_expr
+    return self.qualified_table_sequel_identifier[:updated] < Sequel[:excluded][:updated]
+  end
+
+  def _upsert_update_expr(inserting, **_kwargs)
+    update = super
+    # Only set created_at if it's not set so the initial insert isn't modified.
+    self._coalesce_excluded_on_update(update, [:row_created_at])
+    return update
+  end
+
+  def _fetch_backfill_page(*)
+    io = Webhookdb::Http.get(self.service_integration.api_url, logger: self.logger, timeout: 30)
+    feed_obj = Webhookdb::Xml::Atom.parse(io.body)
+    return feed_obj.fetch("entries"), nil
+  end
+
+  def _verify_backfill_err_msg
+    return "Sorry, we can't reach that URL. Please double check it and try again."
+  end
+
+  def _backfillers
+    return [Backfiller.new(self)]
+  end
+
+  class Backfiller < Webhookdb::Replicator::Base::ServiceBackfiller
+    include Webhookdb::Backfiller::Bulk
+    attr_reader :upserting_replicator
+
+    def initialize(replicator)
+      super
+      @upserting_replicator = @svc
+    end
+
+    def upsert_page_size = 500
+    def conditional_upsert? = true
+    def prepare_body(body) = body
+  end
+
+  def _webhook_response(_request)
+    return Webhookdb::WebhookResponse.ok
+  end
+
+  def calculate_backfill_state_machine
+    step = Webhookdb::Replicator::StateMachineStep.new
+    # Must set this to fake out having credentials.
+    self.service_integration.update(backfill_key: "placeholder") if self.service_integration.backfill_key.blank?
+    if self.service_integration.api_url.blank?
+      step.output = %(You're about to sync entries from an Atom URL into WebhookDB.
+This will create a row for each 'entry' in the given feed,
+and insert/update new rows periodically.
+
+Paste in the URL to sync, and press Enter.)
+      return step.prompting("URL").api_url(self.service_integration)
+    end
+    unless (result = self.verify_backfill_credentials).verified
+      self.service_integration.update(api_url: "")
+      step.output = result.message
+      return step.prompting("URL").api_url(self.service_integration)
+    end
+    step.output = %(
+All set! Your feed will be synced momentarily and then every few hours after that.
+
+#{self._query_help_output})
+    return step.completed
+  end
+end