webhookdb 0.1.0

data/lib/webhookdb/connection_cache.rb

@@ -0,0 +1,177 @@
+# frozen_string_literal: true
+
+require "appydays/configurable"
+require "appydays/loggable"
+
+require "webhookdb"
+
+# Keep a dynamic cache of open database connections.
+# Very similar in behavior to Sequel::DATABASES,
+# but we need to prune connections that have not been inactive for a while.
+#
+# When `borrow` is called, either a new connection is made,
+# or an existing one used, for that URL. The connection is yield to the block.
+#
+# Then, after the block is called,
+# if 'prune_interval' has elapsed since the last prune,
+# prune all connections with 0 current connections,
+# _other than the connection just used_.
+# Because this connection was just used,
+# we assume it will be used again soon.
+#
+# The idea here is that:
+# - We cannot connect to the DB statically; each org can have its own DB,
+#   so storing it statically would increase DB connections to the the number of orgs in the database.
+# - So we replace the organization/synchronization done in Sequel::DATABASES with ConnectionCache.
+# - Any number of worker threads need access to the same DB; rather than connecting inline,
+#   which is very slow, all DB connections for an org (or across orgs if not in database isolation)
+#   can share connections via ConnectionCache.
+# - In single-org/db environments, the active organization will always always be the same,
+#   so the connection is never returned.
+# - In multi-org/db environments, busy orgs will likely stay busy. But a reconnect isn't the end
+#   of the world.
+# - It seems more efficient to be pessimistic about future use, and prune anything with 0 connections,
+#   rather than optimistic, and use an LRU or something similar, since the connections are somewhat
+#   expensive resources to keep open for now reason. That said, we could switch this out for an LRU
+#   it the pessimistic pruning results in many reconnections. It would also be reasonable to increase
+#   the prune interval to avoid disconnecting as frequently.
+#
+# Note that, due to certain implementation details, such as setting timeouts and automatic transaction handling,
+# we implement our own threaded connection pooling, so use the SingleThreadedConnectionPool in Sequel
+# and manage multiple threads on our own.
+class Webhookdb::ConnectionCache
+  include Appydays::Configurable
+  include Appydays::Loggable
+  extend Webhookdb::MethodUtilities
+  include Webhookdb::Dbutil
+
+  class ReentranceError < StandardError; end
+
+  configurable(:connection_cache) do
+    # If this many seconds has elapsed since the last connecton was borrowed,
+    # prune connections with no pending borrows.
+    setting :prune_interval, 120
+
+    # Seconds for the :fast timeout option.
+    setting :timeout_fast, 30
+    # Seconds for the :slow timeout option.
+    setting :timeout_slow, 180
+    # Seconds for the :slow_schema timeout option.
+    setting :timeout_slow_schema, 30.minutes.to_i
+  end
+
+  singleton_attr_accessor :_instance
+
+  def self.borrow(url, **kw, &)
+    return self._instance.borrow(url, **kw, &)
+  end
+
+  def self.disconnect(url)
+    self._instance.disconnect(url)
+  end
+
+  def self.force_disconnect_all
+    self._instance.force_disconnect_all
+  end
+
+  attr_accessor :dbs_for_urls, :prune_interval, :last_pruned_at
+
+  def initialize(prune_interval:)
+    @mutex = Mutex.new
+    @dbs_for_urls = {}
+    @prune_interval = prune_interval
+    @last_pruned_at = Time.now
+  end
+
+  # Connect to the database at the given URL.
+  # borrow is not re-entrant, so if the current thread already owns a connection
+  # to the given url, raise a ReentrantError.
+  # If the url has a DB not in use by any thread,
+  # yield that.
+  # If the url has no DBs opened, or all are checked out,
+  # create and yield a new connection.
+  # See class docs for more details.
+  def borrow(url, transaction: false, timeout: nil, &block)
+    raise LocalJumpError if block.nil?
+    raise ArgumentError, "url cannot be blank" if url.blank?
+    now = Time.now
+    if timeout.is_a?(Symbol)
+      timeout_name = "timeout_#{timeout}"
+      begin
+        timeout = Webhookdb::ConnectionCache.send(timeout_name)
+      rescue NoMethodError
+        raise NoMethodError, "no timeout accessor :#{timeout_name}"
+      end
+    end
+    t = Thread.current
+    conn = nil
+    @mutex.synchronize do
+      db_loans = @dbs_for_urls[url] ||= {loaned: {}, available: []}
+      if db_loans[:loaned].key?(t)
+        raise ReentranceError,
+              "ConnectionCache#borrow is not re-entrant for the same database since the connection has stateful config"
+      end
+      conn = db_loans[:available].pop || take_conn(url, single_threaded: true, extensions: [:pg_json, :pg_streaming])
+      db_loans[:loaned][t] = conn
+    end
+    conn << "SET statement_timeout TO #{timeout * 1000}" if timeout.present?
+    conn << "BEGIN;" if transaction
+    begin
+      result = yield conn
+      conn << "COMMIT;" if transaction
+    rescue Sequel::DatabaseError
+      conn << "ROLLBACK;" if transaction
+      raise
+    ensure
+      conn << "SET statement_timeout TO 0" if timeout.present?
+      @mutex.synchronize do
+        @dbs_for_urls[url][:loaned].delete(t)
+        @dbs_for_urls[url][:available] << conn
+      end
+    end
+    self.prune(url) if now > self.next_prune_at
+    return result
+  end
+
+  def next_prune_at = self.last_pruned_at + self.prune_interval
+
+  # Disconnect the cached connection for the given url,
+  # if any. In general, this is only needed when tearing down a database.
+  def disconnect(url)
+    raise ArgumentError, "url cannot be blank" if url.blank?
+    db_loans = @dbs_for_urls[url]
+    return if db_loans.nil?
+    if db_loans[:loaned].size.positive?
+      raise Webhookdb::InvalidPrecondition,
+            "url #{displaysafe_url(url)} still has #{db_loans[:loaned].size} active connections"
+    end
+    db_loans[:available].each(&:disconnect)
+    @dbs_for_urls.delete(url)
+  end
+
+  protected def prune(skip_url)
+    @dbs_for_urls.each do |url, db_loans|
+      next false if url == skip_url
+      db_loans[:available].each(&:disconnect)
+    end
+    self.last_pruned_at = Time.now
+  end
+
+  def force_disconnect_all
+    @dbs_for_urls.each_value do |db_loans|
+      db_loans[:available].each(&:disconnect)
+      db_loans[:loaned].each_value(&:disconnect)
+    end
+    @dbs_for_urls.clear
+  end
+
+  def summarize
+    return self.dbs_for_urls.transform_values do |loans|
+      {loaned: loans[:loaned].size, available: loans[:available].size}
+    end
+  end
+end
+
+Webhookdb::ConnectionCache._instance = Webhookdb::ConnectionCache.new(
+  prune_interval: Webhookdb::ConnectionCache.prune_interval,
+)

data/lib/webhookdb/console.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require "webhookdb"
+
+module Webhookdb::Console
+  extend Webhookdb::MethodUtilities
+
+  class Error < StandardError; end
+
+  class UnsafeOperation < Error; end
+
+  class ForbiddenOperation < Error; end
+
+  singleton_attr_accessor :unsafe_mode
+  @unsafe_mode = false
+
+  singleton_attr_accessor :original_execute
+  @original_execute = nil
+
+  def self.enable_safe_mode
+    Webhookdb::Console.original_execute = Webhookdb::Postgres::Model.db.method(:execute)
+    Webhookdb::Postgres::Model.db.define_singleton_method(:execute) do |*args, &block|
+      sql = args.first
+
+      includes_where = sql.include?(" WHERE ")
+      is_update = sql.start_with?("UPDATE ")
+      is_delete = sql.start_with?("DELETE FROM")
+
+      raise ForbiddenOperation, "TRUNCATE is forbidden" if sql.start_with?("TRUNCATE TABLE")
+
+      raise ForbiddenOperation, "UPDATE without a WHERE is forbidden" if
+        is_update && !includes_where
+
+      raise ForbiddenOperation, "DELETE without a WHERE is forbidden" if
+        is_delete && !includes_where
+
+      raise UnsafeOperation, "DELETE is permitted only in an UNSAFE block" if
+        is_delete && includes_where && !Webhookdb::Console.unsafe_mode
+
+      Webhookdb::Console.original_execute.call(*args, &block)
+    end
+  end
+
+  def self.disable_safe_mode
+    Webhookdb::Postgres::Model.db.define_singleton_method(:execute, &Webhookdb::Console.original_execute)
+  end
+
+  def self.unsafe(&block)
+    raise LocalJumpError, "unsafe must be called with a block (do ... end)" unless block
+    self.unsafe_mode = true
+    begin
+      Webhookdb::Postgres::Model.db.transaction(&block)
+    ensure
+      self.unsafe_mode = false
+    end
+  end
+
+  def self.console_logger(ev)
+    Webhookdb.logger.info "ConsoleLogger: [%s] %s %p" % [ev.id, ev.name, ev.payload]
+  end
+
+  module MainMethods
+    def unsafe(&)
+      return Webhookdb::Console.unsafe(&)
+    end
+
+    def disconnect_db
+      Webhookdb::Customer.db.disconnect
+    end
+  end
+end

data/lib/webhookdb/convertkit.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+class Webhookdb::Convertkit
+  include Appydays::Configurable
+
+  configurable(:convertkit) do
+    setting :http_timeout, 30
+  end
+
+  FIND_API_SECRET_HELP = %(- Go to https://app.convertkit.com/account_settings/advanced_settings,
+- or from your ConvertKit Dashboard, go to your Advanced Account settings.
+- Under the API Header you will see your API Secret, just under your API Key.
+- Copy the API Secret.)
+end

data/lib/webhookdb/crypto.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Webhookdb::Crypto
+  def self.bin2hex(s)
+    return s.unpack1("H*")
+  end
+
+  def self.cipher
+    return OpenSSL::Cipher.new("aes-256-cbc")
+  end
+
+  # @return [Boxed]
+  def self.encryption_key
+    k = self.cipher.encrypt.random_key
+    return Boxed.from_raw(k)
+  end
+
+  # @param key [Boxed]
+  # @param value [Boxed]
+  # @return [Boxed]
+  def self.encrypt_value(key, value)
+    cipher = self.cipher.encrypt
+    cipher.key = key.raw
+    enc = cipher.update(value.raw) + cipher.final
+    return Boxed.from_raw(enc)
+  end
+
+  # @param key [Boxed]
+  # @param value [Boxed]
+  # @return [Boxed]
+  def self.decrypt_value(key, value)
+    cipher = self.cipher.decrypt
+    cipher.key = key.raw
+    dec = cipher.update(value.raw) + cipher.final
+    return Boxed.from_raw(dec)
+  end
+
+  class Boxed
+    def self.from_raw(bytestr)
+      raise ArgumentError, "bytes string cannot be nil" if bytestr.nil?
+      return self.new(bytestr, nil)
+    end
+
+    def self.from_b64(b64str)
+      raise ArgumentError, "base64 string cannot be nil" if b64str.nil?
+      return self.new(nil, b64str)
+    end
+
+    def initialize(raw, b64)
+      @raw = raw
+      @b64 = b64
+    end
+
+    # @return [String]
+    def base64
+      @b64 ||= Base64.urlsafe_encode64(@raw)
+      return @b64
+    end
+
+    # @return [String]
+    def raw
+      @raw ||= Base64.urlsafe_decode64(@b64)
+      return @raw
+    end
+  end
+end

data/lib/webhookdb/customer/reset_code.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require "securerandom"
+
+require "webhookdb/postgres"
+require "webhookdb/customer"
+
+class Webhookdb::Customer::ResetCode < Webhookdb::Postgres::Model(:customer_reset_codes)
+  class Unusable < StandardError; end
+
+  TOKEN_LENGTH = 6
+
+  plugin :timestamps
+
+  many_to_one :customer, class: Webhookdb::Customer
+
+  dataset_module do
+    def usable
+      return self.where(Sequel[used: false] & Sequel.expr { expire_at > Sequel.function(:now) })
+    end
+  end
+
+  # Invoke the given block with the reset code referred to by token.
+  # Raise Unusable if code is unusable.
+  def self.use_code_with_token(token)
+    raise LocalJumpError unless block_given?
+
+    code = self.usable[token:]
+    raise Unusable unless code&.usable?
+
+    code.db.transaction do
+      code.use!
+      yield(code)
+    end
+  end
+
+  def initialize(*)
+    super
+    self.token ||= Array.new(TOKEN_LENGTH) { rand(0..9) }.join
+    self.expire_at ||= 15.minutes.from_now
+  end
+
+  def expire!
+    self.update(expire_at: Time.now)
+    return self
+  end
+
+  def expired?
+    return self.expire_at < Time.now
+  end
+
+  def use!
+    now = Time.now
+    self.customer.reset_codes_dataset.usable.update(expire_at: now)
+    self.update(used: true, expire_at: now)
+    return self
+  end
+
+  def used?
+    return self.used
+  end
+
+  def usable?
+    return false if self.used?
+    return !self.expired?
+  end
+
+  #
+  # :section: Sequel Validation
+  #
+
+  def validate
+    super
+    self.validates_includes(["email"], :transport)
+  end
+end
+
+# Table: customer_reset_codes
+# -------------------------------------------------------------------------------------------------
+# Columns:
+#  id          | integer                  | PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY
+#  created_at  | timestamp with time zone | NOT NULL DEFAULT now()
+#  updated_at  | timestamp with time zone |
+#  transport   | text                     | NOT NULL
+#  token       | text                     | NOT NULL
+#  used        | boolean                  | NOT NULL DEFAULT false
+#  expire_at   | timestamp with time zone | NOT NULL
+#  customer_id | integer                  | NOT NULL
+# Indexes:
+#  customer_reset_codes_pkey              | PRIMARY KEY btree (id)
+#  customer_reset_codes_customer_id_index | btree (customer_id)
+# Foreign key constraints:
+#  customer_reset_codes_customer_id_fkey | (customer_id) REFERENCES customers(id) ON DELETE CASCADE
+# -------------------------------------------------------------------------------------------------