webhookdb 0.1.0

data/lib/webhookdb/db_adapter/pg.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+require "pg"
+
+require "webhookdb/db_adapter/default_sql"
+
+class Webhookdb::DBAdapter::PG < Webhookdb::DBAdapter
+  include Webhookdb::DBAdapter::ColumnTypes
+  include Webhookdb::DBAdapter::DefaultSql
+
+  VERIFY_TIMEOUT = 2
+  VERIFY_STATEMENT = "SELECT 1"
+
+  def identifier_quote_char
+    return '"'
+  end
+
+  def create_index_sql(index, concurrently:)
+    tgts = index.targets.map { |c| self.escape_identifier(c.name) }.join(", ")
+    uniq = index.unique ? " UNIQUE" : ""
+    concurrent = concurrently ? " CONCURRENTLY" : ""
+    idxname = self.escape_identifier(index.name)
+    tblname = self.qualify_table(index.table)
+    where = ""
+    where = " " + Webhookdb::Customer.where(index.where).sql.delete_prefix('SELECT * FROM "customers" ') if index.where
+    return "CREATE#{uniq} INDEX#{concurrent} IF NOT EXISTS #{idxname} ON #{tblname} (#{tgts})#{where}"
+  end
+
+  def column_create_sql(column)
+    modifiers = +""
+    coltype = COLTYPE_MAP.fetch(column.type)
+    if column.pk?
+      coltype = "bigserial" if column.type == BIGINT
+      coltype = "serial" if column.type == INTEGER
+      modifiers << " PRIMARY KEY"
+    elsif column.unique?
+      modifiers << " UNIQUE NOT NULL"
+    elsif !column.nullable?
+      modifiers << " NOT NULL"
+    end
+    colname = self.escape_identifier(column.name)
+    return "#{colname} #{coltype}#{modifiers}"
+  end
+
+  def add_column_sql(table, column, if_not_exists: false)
+    c = self.column_create_sql(column)
+    ifne = if_not_exists ? " IF NOT EXISTS" : ""
+    return "ALTER TABLE #{self.qualify_table(table)} ADD COLUMN#{ifne} #{c}"
+  end
+
+  def merge_from_csv(connection, file, table, pk_col, copy_columns)
+    qtable = self.qualify_table(table)
+    temptable = "#{self.escape_identifier(table.name)}_staging_#{SecureRandom.hex(4)}"
+    connection.using do |db|
+      db << "CREATE TEMP TABLE #{temptable} (LIKE #{qtable})"
+      db.copy_into(temptable.to_sym, options: "DELIMITER ',', HEADER true, FORMAT csv", data: file)
+      pkname = self.escape_identifier(pk_col.name)
+      col_assigns = self.assign_columns_sql("src", nil, copy_columns)
+      upsert_sql = [
+        <<~UPDATE,
+          UPDATE #{qtable} AS tgt
+          SET #{col_assigns} FROM
+          (SELECT * FROM #{temptable} WHERE #{pkname} IN (SELECT #{pkname} FROM #{qtable})) src
+          WHERE tgt.#{pkname} = src.#{pkname};
+        UPDATE
+        "INSERT INTO #{qtable} SELECT * FROM #{temptable} WHERE #{pkname} NOT IN (SELECT #{pkname} FROM #{qtable});",
+      ]
+      db << upsert_sql.join("\n")
+    end
+  end
+
+  def verify_connection(url, timeout: 2, statement: "SELECT 1")
+    conn = self.connection(url)
+    conn.using(connect_timeout: timeout) do |c|
+      c.execute("SET statement_timeout TO #{timeout * 1000}")
+      c.execute(statement)
+    end
+  end
+
+  COLTYPE_MAP = {
+    BIGINT => "bigint",
+    BIGINT_ARRAY => "bigint[]",
+    BOOLEAN => "boolean",
+    DATE => "date",
+    DECIMAL => "numeric",
+    DOUBLE => "double precision",
+    FLOAT => "float",
+    INTEGER => "integer",
+    INTEGER_ARRAY => "integer[]",
+    OBJECT => "jsonb",
+    TEXT => "text",
+    TEXT_ARRAY => "text[]",
+    TIMESTAMP => "timestamptz",
+    UUID => "uuid",
+  }.freeze
+end

data/lib/webhookdb/db_adapter/snowflake.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+require "webhookdb/db_adapter/default_sql"
+require "webhookdb/snowflake"
+
+class Webhookdb::DBAdapter::Snowflake < Webhookdb::DBAdapter
+  include Webhookdb::DBAdapter::ColumnTypes
+  include Webhookdb::DBAdapter::DefaultSql
+
+  class SnowsqlConnection < Webhookdb::DBAdapter::Connection
+    include Appydays::Loggable
+
+    def initialize(url)
+      super()
+      @url = url
+    end
+
+    def execute(sql, **)
+      self.logger.debug("snowflake_exec", statement: sql)
+      result = Webhookdb::Snowflake.run_cli(@url, sql, **)
+      self.logger.debug("snowflake_exec_result", result:)
+      return result
+    end
+  end
+
+  def connection(url)
+    return SnowsqlConnection.new(url)
+  end
+
+  def create_index_sql(*)
+    raise NotImplementedError, "Snowflake does not support indices"
+  end
+
+  def column_create_sql(column)
+    modifiers = +""
+    if column.unique?
+      modifiers << " UNIQUE NOT NULL"
+    elsif !column.nullable?
+      modifiers << " NOT NULL"
+    end
+    coltype = COLTYPE_MAP.fetch(column.type)
+    colname = self.escape_identifier(column.name)
+    return "#{colname} #{coltype}#{modifiers}"
+  end
+
+  def add_column_sql(table, column, if_not_exists: false)
+    c = self.column_create_sql(column)
+    # Snowflake has no 'ADD COLUMN IF NOT EXISTS' so we need to query the long way around
+    add_sql = "ALTER TABLE #{self.qualify_table(table)} ADD COLUMN #{c}"
+    return add_sql unless if_not_exists
+    # The 'ILIKE' is a case-insensitive string compare,
+    # which is important because snowflake uppercases values when it stores them.
+    conditional_sql = <<~SQL
+      EXECUTE IMMEDIATE $$
+      BEGIN
+        IF (NOT EXISTS(
+          SELECT * FROM INFORMATION_SCHEMA.COLUMNS
+          WHERE TABLE_SCHEMA ILIKE '#{table.schema.name}'
+            AND TABLE_NAME ILIKE '#{table.name}'
+            AND COLUMN_NAME ILIKE '#{column.name}'
+        )) THEN
+          #{add_sql};
+        END IF;
+      END;
+      $$
+    SQL
+    return conditional_sql.rstrip
+  end
+
+  def merge_from_csv(connection, file, table, pk_col, copy_columns)
+    raise Webhookdb::InvalidPrecondition, "table must have schema" if table.schema.nil?
+
+    qtable = self.qualify_table(table)
+
+    stage = self.escape_identifier("whdb_tempstage_#{SecureRandom.hex(2)}_#{table.name}")
+    stage = self.escape_identifier(table.schema.name) + "." + stage
+
+    pkname = self.escape_identifier(pk_col.name)
+    # JSON columns need to be parsed from the CSV, so object columns need parse_json calls.
+    col_assigns = self.assign_columns_sql("src", nil, copy_columns) do |c, lhs, rhs|
+      if c.type == OBJECT
+        [lhs, "parse_json(#{rhs})"]
+      else
+        [lhs, rhs]
+      end
+    end
+    col_names = copy_columns.map { |c| self.escape_identifier(c.name) }
+    col_values = col_names.each_with_index.map do |n, i|
+      if copy_columns[i].type == OBJECT
+        "parse_json(src.#{n})"
+      else
+        "src.#{n}"
+      end
+    end
+    col_placeholders = col_names.each_with_index.map { |n, i| "$#{i + 1} #{n}" }
+    # Props to https://stackoverflow.com/questions/63084511/snowflake-upsert-from-staged-files
+    # for the merge from stage code.
+    # The enclosed option is vital because otherwise it doesn't interpret JSON columns properly.
+    import_sql = <<~SQL
+      CREATE STAGE #{stage} FILE_FORMAT = (type = 'CSV' skip_header = 1 FIELD_OPTIONALLY_ENCLOSED_BY = '"');
+
+      PUT file://#{file.path} @#{stage} auto_compress=true;
+
+      MERGE INTO #{qtable} AS tgt
+        USING (
+          SELECT #{col_placeholders.join(', ')} FROM @#{stage}
+        ) src
+        ON tgt.#{pkname} = src.#{pkname}
+        WHEN MATCHED THEN UPDATE SET #{col_assigns}
+        WHEN NOT MATCHED THEN INSERT (#{col_names.join(', ')}) values (#{col_values.join(', ')});
+    SQL
+    connection.execute(import_sql)
+  end
+
+  def _verify_connection(url, timeout:, statement:)
+    _ = timeout
+    conn = self.connection(url)
+    conn.execute(statement)
+  end
+
+  def identifier_quote_char
+    return ""
+  end
+
+  COLTYPE_MAP = {
+    BIGINT => "bigint",
+    BOOLEAN => "boolean",
+    DATE => "date",
+    DECIMAL => "numeric",
+    DOUBLE => "double precision",
+    FLOAT => "float",
+    INTEGER => "integer",
+    OBJECT => "object",
+    TEXT => "text",
+    TIMESTAMP => "timestamptz",
+  }.freeze
+end

data/lib/webhookdb/db_adapter.rb

@@ -0,0 +1,208 @@
+# frozen_string_literal: true
+
+class Webhookdb::DBAdapter
+  require "webhookdb/db_adapter/column_types"
+
+  class UnsupportedAdapter < StandardError; end
+
+  VALID_IDENTIFIER = /^[a-zA-Z][a-zA-Z\d_ ]*$/
+  INVALID_IDENTIFIER_MESSAGE = "Identifiers must start with a letter and " \
+                               "contain only letters, numbers, spaces, and underscores. " \
+                               "See https://docs.webhookdb.com/concepts/valid-identifiers/ for rules " \
+                               "about identifiers like schema, table, and column names."
+
+  class Schema < Webhookdb::TypedStruct
+    attr_reader :name
+
+    def initialize(**kwargs)
+      super
+      self.typecheck!(:name, Symbol)
+    end
+  end
+
+  class Table < Webhookdb::TypedStruct
+    attr_reader :name, :schema
+
+    def initialize(**kwargs)
+      super
+      self.typecheck!(:name, Symbol)
+      self.typecheck!(:schema, Schema, nullable: true)
+    end
+  end
+
+  class Column < Webhookdb::TypedStruct
+    include ColumnTypes
+    attr_reader :name, :type, :nullable, :unique, :index, :index_where, :pk, :backfill_statement, :backfill_expr
+    alias nullable? nullable
+    alias unique? unique
+    alias index? index
+    alias pk? pk
+
+    def initialize(**kwargs)
+      super
+      self.typecheck!(:name, Symbol)
+      self.typecheck!(:type, Symbol)
+      self.typecheck!(:nullable, :boolean)
+      self.typecheck!(:unique, :boolean)
+      self.typecheck!(:index, :boolean)
+      self.typecheck!(:pk, :boolean)
+      raise ArgumentError, "type #{self.type.inspect} is not known" unless COLUMN_TYPES.include?(self.type)
+    end
+
+    def _defaults
+      return {nullable: true, unique: false, index: false, pk: false}
+    end
+  end
+
+  class Index < Webhookdb::TypedStruct
+    attr_reader :name, :table, :targets, :unique, :where
+
+    def initialize(**kwargs)
+      super
+      self.typecheck!(:name, Symbol)
+      self.typecheck!(:table, Table)
+      self.typecheck!(:targets, Array)
+      self.typecheck!(:unique, :boolean)
+    end
+
+    def _defaults
+      return {unique: false}
+    end
+
+    # @!attribute name
+    #   @return [Symbol]
+    # @!attribute table
+    #   @return [Table]
+    # @!attribute targets
+    #   @return [Array<Column>]
+    # @!attribute unique
+    #   @return [Boolean]
+  end
+
+  class TableDescriptor < Webhookdb::TypedStruct
+    attr_reader :table, :columns, :indices
+
+    def initialize(**kwargs)
+      super
+      self.typecheck!(:table, Table)
+      self.typecheck!(:columns, Array)
+      self.typecheck!(:indices, Array)
+    end
+
+    # @!attribute table
+    #   @return [Table]
+    # @!attribute columns
+    #   @return [Array<Column>]
+    # @!attribute indices
+    #   @return [Array<Index>]
+
+    def _defaults
+      return {indices: []}
+    end
+  end
+
+  # Abstract class representing a DB connection.
+  # Ususually this is a Sequel connection,
+  # but in could just be a stored URL (like for Snowflake
+  # we have to call snowsql each time).
+  class Connection
+    def execute(sql)
+      raise NotImplementedError
+    end
+  end
+
+  class SequelConnection < Connection
+    include Webhookdb::Dbutil
+
+    def initialize(url)
+      super()
+      @url = url
+    end
+
+    def using(**kw, &)
+      borrow_conn(@url, **kw, &)
+    end
+
+    def execute(sql, **kw)
+      borrow_conn(@url, **kw) do |db|
+        db << sql
+      end
+    end
+  end
+
+  # Return a new Connection for the adapter.
+  # By default, return a SequelConnection,
+  # but adapters not using Sequel will need their own type.
+  # @return [Connection]
+  def connection(url)
+    return SequelConnection.new(url)
+  end
+
+  # @param [Schema] schema
+  # @param [Boolean] if_not_exists
+  # @return [String]
+  def create_schema_sql(schema, if_not_exists: false)
+    raise NotImplementedError
+  end
+
+  # @param [Table] table
+  # @param [Array<Column>] columns
+  # @param [Schema] schema
+  # @param [Boolean] if_not_exists
+  # @return [String]
+  def create_table_sql(table, columns, schema: nil, if_not_exists: false)
+    raise NotImplementedError
+  end
+
+  # @param [Index] index
+  # @return [String]
+  def create_index_sql(index, concurrently:)
+    raise NotImplementedError
+  end
+
+  # @param [Table] table
+  # @param [Column] column
+  # @param [Boolean] if_not_exists
+  # @return [String]
+  def add_column_sql(table, column, if_not_exists: false)
+    raise NotImplementedError
+  end
+
+  # Given a table and a (temporary) file with CSV data,
+  # import it into the table. Usually this is a COPY INTO command.
+  # For PG it would read from stdin,
+  # for Snowflake it would have to stage the file.
+  # @param [Connection] connection
+  # @param [File] file
+  # @param [Table] table
+  # @param [Column] pk_col Use this to identifier the same row between source and destination.
+  # @param [Array<Column>] copy_columns All columns to copy.
+  #   NOTE: This includes the pk column, since it should be copied, as we depend on it persisting.
+  def merge_from_csv(connection, file, table, pk_col, copy_columns)
+    raise NotImplementedError
+  end
+
+  def verify_connection(url, timeout: 2, statement: "SELECT 1")
+    return self._verify_connection(url, timeout:, statement:)
+  end
+
+  # @param [String] url
+  # @return [Webhookdb::DBAdapter]
+  def self.adapter(url)
+    case url
+      when /^postgres/
+        return Webhookdb::DBAdapter::PG.new
+      when /^snowflake/
+        return Webhookdb::DBAdapter::Snowflake.new
+      else
+        raise UnsupportedAdapter, "no adapter available for #{url}"
+    end
+  end
+
+  def self.supported_adapters_message
+    return "Postgres (postgres://), SnowflakeDB (snowflake://)"
+  end
+end
+
+require "webhookdb/db_adapter/pg"
+require "webhookdb/db_adapter/snowflake"

data/lib/webhookdb/dbutil.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+require "sequel"
+
+# Mixin that provides helpers when dealing with databases and connections.
+#
+# Use borrow_conn to create a connection that is disconnected after the block runs.
+# A block must be given.
+# By default, this connection uses the Webhookdb logger,
+# and uses test: false and keep_reference: false Sequel.connect options,
+# since this is a quick-lived and self-managed connection.
+#
+# Use take_conn where you will take care of disconnecting the connection.
+# Note you MUST take care to call `disconnect` at some point
+# or connections will leak.
+module Webhookdb::Dbutil
+  include Appydays::Configurable
+
+  # See http://sequel.jeremyevans.net/rdoc/files/doc/opening_databases_rdoc.html#label-General+connection+options
+  # for Sequel option details.
+  configurable(:dbutil) do
+    # The number of (Float) seconds that should be considered "slow" for a
+    # single query; queries that take longer than this amount of time will be logged
+    # at `warn` level.
+    setting :slow_query_seconds, 0.1
+
+    # Default this to whatever concurrency is appropriate for the process type.
+    # PROC_MODE is set in the initializers in the config dir.
+    setting :max_connections,
+            (if ENV["PROC_MODE"] == "sidekiq"
+               ENV.fetch("SIDEKIQ_CONCURRENCY", "10").to_i
+            elsif ENV["PROC_MIDE"] == "puma"
+              ENV.fetch("WEB_CONCURRENCY", "4").to_i
+            else
+              4
+            end)
+    setting :pool_timeout, 10
+    # Set to 'disable' to work around segfault.
+    # See https://github.com/ged/ruby-pg/issues/538
+    setting :gssencmode, ""
+  end
+
+  # Needed when we need to work with a source.
+  MOCK_CONN = Sequel.connect("mock://")
+
+  module_function def borrow_conn(url, **opts, &block)
+    raise LocalJumpError, "borrow_conn requires a block" if block.nil?
+    opts = conn_opts(opts)
+    Sequel.connect(url, **opts, &block)
+  end
+
+  module_function def take_conn(url, **opts, &block)
+    raise LocalJumpError, "take_conn cannot use a block" unless block.nil?
+    opts = conn_opts(opts)
+    return Sequel.connect(url, **opts, &block)
+  end
+
+  private module_function def conn_opts(opts)
+    res = Webhookdb::Dbutil.configured_connection_options
+    res.merge!(opts)
+    res[:test] = false unless res.key?(:test)
+    res[:loggers] = [Webhookdb.logger] unless res.key?(:logger) || res.key?(:loggers)
+    res[:keep_reference] = false unless res.key?(:keep_reference)
+    return res
+  end
+
+  def self.configured_connection_options
+    res = {}
+    res[:sql_log_level] ||= :debug
+    res[:log_warn_duration] ||= Webhookdb::Dbutil.slow_query_seconds
+    res[:max_connections] ||= Webhookdb::Dbutil.max_connections
+    res[:pool_timeout] ||= Webhookdb::Dbutil.pool_timeout
+    res[:driver_options] = {}
+    (res[:driver_options][:gssencmode] = Webhookdb::Dbutil.gssencmode) if Webhookdb::Dbutil.gssencmode.present?
+    return res
+  end
+
+  module_function def displaysafe_url(url)
+    u = URI(url)
+    u.user = "***"
+    u.password = "***"
+    return u.to_s
+  end
+
+  module_function def reduce_expr(dataset, op_symbol, operands, method: :where)
+    return dataset if operands.blank?
+    present_ops = operands.select(&:present?)
+    return dataset if present_ops.empty?
+    full_op = present_ops.reduce(&op_symbol)
+    return dataset.send(method, full_op)
+  end
+end

data/lib/webhookdb/demo_mode.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+module Webhookdb::DemoMode
+  include Appydays::Configurable
+  include Appydays::Loggable
+
+  configurable(:demo_mode) do
+    setting :client_enabled, false
+    setting :customer_email, "demo@webhookdb.com"
+    setting :customer_org_key, "demo_org"
+    setting :demo_org_id, 0
+    setting :demo_data_api_host, "https://api.webhookdb.com"
+    setting :example_datasets_enabled, false
+  end
+
+  class << self
+    # Should requests to this server allow demo mode? Usually this is true when running
+    # in a demo context, like initial local development.
+    def client_enabled? = self.client_enabled
+    # Should requests to this server respond to API requests for demo data?
+    def server_enabled? = self.demo_org_id.positive?
+
+    # @return [Array<Webhookdb::OrganizationMembership, Webhookdb::Replicator::StateMachineStep, String>]
+    def handle_auth
+      raise Webhookdb::InvalidPrecondition unless self.client_enabled?
+      _, customer = Webhookdb::Customer.find_or_create_for_email(self.customer_email)
+      membership = self._ensure_membership(customer)
+      membership.organization.publish_deferred("syncdemodata", membership.organization_id)
+      step = Webhookdb::Replicator::StateMachineStep.new.completed
+      message = %(Hi there! This is a demo version of WebhookDB.
+
+You have been logged in automatically.
+
+Your WebhookDB organization has also been set up with replicators for some APIs, like GitHub.
+
+Run `webhookdb db connection` to get your database connection string,
+and see what data is available.
+
+To set up a new replicator, run `webhookdb services list` to see what is available.
+
+You can also head to `webhookdb.com/deploy-builder` to prepare an environment for a deployment
+into your own environment, like AWS or Heroku.
+
+Or check out https://webhookdb.com to sign up for WebhookDB Cloud so this is all managed for you.)
+      return membership, step, message
+    end
+
+    def _ensure_membership(customer)
+      org = Webhookdb::Organization.find_or_create(key: self.customer_org_key) do |o|
+        o.name = "Demo Org"
+        o.billing_email = customer.email
+      end
+      mem = customer.all_memberships_dataset[organization: org] || customer.add_membership(
+        organization: org, membership_role: Webhookdb::Role.admin_role, verified: true, is_default: true,
+      )
+      return mem
+    end
+
+    def build_demo_data
+      evar = "DEMO_MODE_DEMO_ORG_ID"
+      raise Webhookdb::InvalidPrecondition, "#{evar} not set" unless self.server_enabled?
+      org = Webhookdb::Organization[self.demo_org_id] or
+        raise Webhookdb::InvalidPrecondition, "#{evar} #{self.demo_org_id} does not exist"
+      demo_sints = org.service_integrations.select { |sint| sint.service_name.start_with?("github_") }
+      data = demo_sints.map do |sint|
+        rows_data = sint.replicator.readonly_dataset { |ds| ds.select_map(:data) }
+        {
+          service_name: sint.service_name,
+          rows_data:,
+        }
+      end
+      return {data:}
+    end
+
+    def sync_demo_data(org)
+      can_run = Webhookdb::DemoMode.client_enabled? ||
+        Webhookdb::DemoMode.example_datasets_enabled
+      return false unless can_run
+      resp = Webhookdb::Http.post("#{self.demo_data_api_host}/v1/demo/data", timeout: nil, logger: self.logger)
+      # First, create/migrate all service integrations from the demo server.
+      sints_and_datas = []
+      resp.parsed_response["data"].each do |h|
+        service_name = h.fetch("service_name")
+        table_name = "#{service_name}_demo"
+        sint = org.service_integrations.find { |si| si.table_name == table_name } ||
+          org.add_service_integration(service_name:, table_name:)
+        sints_and_datas << [sint, h.fetch("rows_data")]
+      end
+      org.migrate_replication_tables
+      # Now populate them with data.
+      sints_and_datas.each do |(sint, rows)|
+        repl = sint.replicator
+        rows.each do |row|
+          repl.upsert_webhook_body(row)
+        end
+      end
+      return true
+    end
+  end
+end

data/lib/webhookdb/developer_alert.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require "appydays/configurable"
+require "webhookdb/slack"
+
+# Decouples the need to alert from the way we want to handle alerts.
+# This is for something in between purely technical alerts (error handling in Sentry)
+# and ops/marketing alerts (which may post to Slack or whatever).
+# Instead of having to rewrite many async jobs to not use Slack,
+# we can just modiy the one job that handles developer alerts.
+class Webhookdb::DeveloperAlert
+  include Appydays::Configurable
+
+  attr_accessor :subsystem, :emoji, :fallback, :fields
+
+  def initialize(subsystem:, emoji:, fallback:, fields:)
+    @subsystem = subsystem
+    @emoji = emoji
+    @fallback = fallback
+    @fields = fields
+  end
+
+  def as_json
+    return {
+      subsystem:,
+      emoji:,
+      fallback:,
+      fields:,
+    }
+  end
+
+  def emit
+    Amigo.publish("webhookdb.developeralert.emitted", self.as_json)
+  end
+
+  def handle
+    notifier = Webhookdb::Slack.new_notifier(
+      channel: "#webhookdb-notifications",
+      username: @subsystem,
+      icon_emoji: @emoji,
+    )
+    notifier.post(
+      attachments: [
+        {
+          fallback: @fallback,
+          fields: @fields,
+        },
+      ],
+    )
+  end
+end