webhookdb 1.2.1 → 1.3.0

data/lib/webhookdb/replicator/increase_transaction_v1.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "webhookdb/increase"
 require "webhookdb/replicator/increase_v1_mixin"
 
 class Webhookdb::Replicator::IncreaseTransactionV1 < Webhookdb::Replicator::Base
@@ -11,10 +10,10 @@ class Webhookdb::Replicator::IncreaseTransactionV1 < Webhookdb::Replicator::Base
   def self.descriptor
     return Webhookdb::Replicator::Descriptor.new(
       name: "increase_transaction_v1",
-      ctor: ->(sint) { Webhookdb::Replicator::IncreaseTransactionV1.new(sint) },
+      ctor: self,
       feature_roles: [],
       resource_name_singular: "Increase Transaction",
-      supports_webhooks: true,
+      dependency_descriptor: Webhookdb::Replicator::IncreaseAppV1.descriptor,
       supports_backfill: true,
       api_docs_url: "https://increase.com/documentation/api",
     )
@@ -28,13 +27,8 @@ class Webhookdb::Replicator::IncreaseTransactionV1 < Webhookdb::Replicator::Base
     return [
       Webhookdb::Replicator::Column.new(:account_id, TEXT, index: true),
       Webhookdb::Replicator::Column.new(:amount, INTEGER, index: true),
-      Webhookdb::Replicator::Column.new(
-        :created_at,
-        TIMESTAMP,
-        data_key: "created_at",
-        optional: true,
-        index: true,
-      ),
+      Webhookdb::Replicator::Column.new(:created_at, TIMESTAMP, index: true),
+      Webhookdb::Replicator::Column.new(:updated_at, TIMESTAMP, index: true),
       # date is a legacy field that is not documented in the API,
       # but is still sent with transactions as of April 2022.
       # We need to support the v1 schema, but do not want to depend
@@ -48,27 +42,8 @@ class Webhookdb::Replicator::IncreaseTransactionV1 < Webhookdb::Replicator::Base
         converter: Webhookdb::Replicator::Column::CONV_TO_UTC_DATE,
       ),
       Webhookdb::Replicator::Column.new(:route_id, TEXT, index: true),
-      Webhookdb::Replicator::Column.new(
-        :updated_at,
-        TIMESTAMP,
-        data_key: "created_at",
-        event_key: "created_at",
-        defaulter: :now,
-        optional: true,
-        index: true,
-      ),
     ]
   end
 
-  def _update_where_expr
-    return self.qualified_table_sequel_identifier[:updated_at] < Sequel[:excluded][:updated_at]
-  end
-
-  def _resource_and_event(request)
-    return self._find_resource_and_event(request.body, "transaction")
-  end
-
-  def _mixin_backfill_url
-    return "#{self.service_integration.api_url}/transactions"
-  end
+  def _mixin_object_type = "transaction"
 end

data/lib/webhookdb/replicator/increase_v1_mixin.rb

@@ -3,119 +3,99 @@
 require "webhookdb/increase"
 
 module Webhookdb::Replicator::IncreaseV1Mixin
-  def _mixin_backfill_url
-    raise NotImplementedError
-  end
-
   def _webhook_response(request)
     return Webhookdb::Increase.webhook_response(request, self.service_integration.webhook_secret)
   end
 
+  def _resource_and_event(request) = request.body
+
   def _timestamp_column_name
+    # We derive updated_at from the event, or use 'now'
     return :updated_at
   end
 
-  def _find_resource_and_event(body, desired_object_name)
-    return nil unless Webhookdb::Increase.contains_desired_object(body, desired_object_name)
-    return body.fetch("data"), body if body.key?("event") && body.key?("event_id")
-    return body, nil
+  def _update_where_expr
+    ts = self._timestamp_column_name
+    return self.qualified_table_sequel_identifier[ts] < Sequel[:excluded][ts]
   end
 
-  def process_state_change(field, value)
-    # special handling for having a default value for api url
-    value = "https://api.increase.com" if field == "api_url" && value == ""
-    return super(field, value)
+  def on_dependency_webhook_upsert(_replicator, payload, **)
+    self.upsert_webhook_body(payload)
   end
 
-  def calculate_webhook_state_machine
-    step = Webhookdb::Replicator::StateMachineStep.new
-    # if the service integration doesn't exist, create it with some standard values
-    unless self.service_integration.webhook_secret.present?
-      step.output = %(You are about to start replicating #{self.resource_name_plural} info into WebhookDB.
-We've made an endpoint available for #{self.resource_name_singular} webhooks:
-
-#{self._webhook_endpoint}
-
-From your Increase admin dashboard, go to Applications -> Create Webhook.
-In the "Webhook endpoint URL" field you can enter the URL above.
-For the shared secret, you'll have to generate a strong password
-(you can use '#{Webhookdb::Id.rand_enc(16)}')
-and then enter it into the textbox.
-
-Copy that shared secret value.
-      )
-      return step.secret_prompt("secret").webhook_secret(self.service_integration)
-    end
-
-    step.output = %(Great! WebhookDB is now listening for #{self.resource_name_singular} webhooks.
-#{self._query_help_output}
-In order to backfill existing #{self.resource_name_plural}, run this from a shell:
-
-  #{self._backfill_command}
+  def _mixin_object_type = raise NotImplementedError
+  def _mixin_backfill_path = "/#{self._mixin_object_type}s"
+  def _mixin_backfill_url = "#{self._api_url}#{self._mixin_backfill_path}"
+  def _api_url = "https://api.increase.com"
+
+  def handle_event?(event) = event.fetch("associated_object_type") == self._mixin_object_type
+
+  def _fetch_enrichment(resource, _event, _request)
+    # If the resource type isn't what we expect, it must be an event.
+    # In that case, we need to fetch the resource from the API,
+    # and replace the event body in prepare_for_insert.
+    # The updated_at becomes the event's created_at,
+    # which should be fine- it's better than setting updated_at to 'now'
+    # since that will be confusing as it looks like a resource was recently updated.
+    rtype = resource.fetch("type")
+    return nil if rtype == self._mixin_object_type
+    raise Webhookdb::InvalidPrecondition, "unexpected resource: #{resource}" unless
+      rtype == "event" && resource.fetch("associated_object_type") == self._mixin_object_type
+    response = Webhookdb::Http.get(
+      self._mixin_backfill_url + "/#{resource.fetch('associated_object_id')}",
+      {},
+      headers: self._auth_headers,
+      logger: self.logger,
+      timeout: Webhookdb::Increase.http_timeout,
     )
-    return step.completed
+    return response.parsed_response.merge("updated_at" => resource.fetch("created_at"))
   end
 
-  def calculate_backfill_state_machine
-    step = Webhookdb::Replicator::StateMachineStep.new
-    unless self.service_integration.backfill_key.present?
-      step.output = %(In order to backfill #{self.resource_name_plural}, we need an API key.
-From your Increase admin dashboard, go to Settings -> Development -> API Keys.
-We'll need the Production key--copy that value to your clipboard.
-      )
-      return step.secret_prompt("API Key").backfill_key(self.service_integration)
-    end
-
-    unless self.service_integration.api_url.present?
-      step.output = %(Great. Now we want to make sure we're sending API requests to the right place.
-For Increase, the API url is different when you are in Sandbox mode and when you are in Production mode.
-For Sandbox mode, the API root url is:
-
-https://sandbox.increase.com
-
-For Production mode, which is our default, it is:
+  def _prepare_for_insert(resource, event, request, enrichment)
+    resource = enrichment if enrichment
+    return super(resource, event, request, nil)
+  end
 
-https://api.increase.com
+  def _app_sint = Webhookdb::Replicator.find_at_root!(self.service_integration, service_name: "increase_app_v1")
 
-Leave blank to use the default or paste the answer into this prompt.
-      )
-      return step.prompting("API url").api_url(self.service_integration)
-    end
+  def _auth_headers
+    return {"Authorization" => ("Bearer " + self._app_sint.backfill_key)}
+  end
 
-    unless (result = self.verify_backfill_credentials).verified
-      self.service_integration.replicator.clear_backfill_information
-      step.output = result.message
-      return step.secret_prompt("API Key").backfill_key(self.service_integration)
+  def calculate_backfill_state_machine
+    if (step = self.calculate_dependency_state_machine_step(dependency_help: ""))
+      step.output = %(This replicator is managed automatically using OAuth through Increase.
+Head over to #{Webhookdb::Replicator::IncreaseAppV1.descriptor.install_url} to learn more.)
+      return step
     end
-
+    step = Webhookdb::Replicator::StateMachineStep.new
     step.needs_input = false
     step.output = %(Great! We are going to start backfilling your #{self.resource_name_plural}.
-#{self._query_help_output}
-      )
+#{self._query_help_output})
     step.complete = true
     return step
   end
 
-  def _verify_backfill_401_err_msg
-    return "It looks like that API Key is invalid. Please reenter the API Key you just created:"
-  end
-
-  def _verify_backfill_err_msg
-    return "An error occurred. Please reenter the API Key you just created:"
-  end
-
   def _fetch_backfill_page(pagination_token, **_kwargs)
     query = {}
     (query[:cursor] = pagination_token) if pagination_token.present?
+    fetched_at = Time.now
     response = Webhookdb::Http.get(
       self._mixin_backfill_url,
       query,
-      headers: {"Authorization" => ("Bearer " + self.service_integration.backfill_key)},
+      headers: self._auth_headers,
       logger: self.logger,
       timeout: Webhookdb::Increase.http_timeout,
     )
     data = response.parsed_response
     next_page_param = data.dig("response_metadata", "next_cursor")
-    return data["data"], next_page_param
+    rows = data["data"]
+    # In general, we want to use webhooks/events to keep rows updated.
+    # But if we are backfilling, touch the 'updated at' timestamp to make sure
+    # these rows get inserted.
+    # It does mess up history, but we can't get that history to be accurate
+    # in the case of a backfill anyway.
+    rows.each { |r| r["updated_at"] = fetched_at }
+    return rows, next_page_param
   end
 end

data/lib/webhookdb/replicator/increase_wire_transfer_v1.rb

@@ -11,10 +11,10 @@ class Webhookdb::Replicator::IncreaseWireTransferV1 < Webhookdb::Replicator::Bas
   def self.descriptor
     return Webhookdb::Replicator::Descriptor.new(
       name: "increase_wire_transfer_v1",
-      ctor: ->(sint) { Webhookdb::Replicator::IncreaseWireTransferV1.new(sint) },
+      ctor: self,
       feature_roles: [],
       resource_name_singular: "Increase Wire Transfer",
-      supports_webhooks: true,
+      dependency_descriptor: Webhookdb::Replicator::IncreaseAppV1.descriptor,
       supports_backfill: true,
       api_docs_url: "https://increase.com/documentation/api",
     )
@@ -30,32 +30,13 @@ class Webhookdb::Replicator::IncreaseWireTransferV1 < Webhookdb::Replicator::Bas
       Webhookdb::Replicator::Column.new(:account_id, TEXT, index: true),
       Webhookdb::Replicator::Column.new(:amount, INTEGER, index: true),
       Webhookdb::Replicator::Column.new(:approved_at, TIMESTAMP, data_key: ["approval", "approved_at"]),
-      Webhookdb::Replicator::Column.new(:created_at, TIMESTAMP, optional: true, index: true),
+      Webhookdb::Replicator::Column.new(:created_at, TIMESTAMP, index: true),
       Webhookdb::Replicator::Column.new(:routing_number, TEXT, index: true),
       Webhookdb::Replicator::Column.new(:status, TEXT),
-      Webhookdb::Replicator::Column.new(:template_id, TEXT),
       Webhookdb::Replicator::Column.new(:transaction_id, TEXT, index: true),
-      Webhookdb::Replicator::Column.new(
-        :updated_at,
-        TIMESTAMP,
-        data_key: "created_at",
-        event_key: "created_at",
-        defaulter: :now,
-        optional: true,
-        index: true,
-      ),
+      Webhookdb::Replicator::Column.new(:updated_at, TIMESTAMP, index: true),
     ]
   end
 
-  def _resource_and_event(request)
-    return self._find_resource_and_event(request.body, "wire_transfer")
-  end
-
-  def _update_where_expr
-    return self.qualified_table_sequel_identifier[:updated_at] < Sequel[:excluded][:updated_at]
-  end
-
-  def _mixin_backfill_url
-    return "#{self.service_integration.api_url}/wire_transfers"
-  end
+  def _mixin_object_type = "wire_transfer"
 end

data/lib/webhookdb/replicator/intercom_contact_v1.rb

@@ -25,12 +25,59 @@ class Webhookdb::Replicator::IntercomContactV1 < Webhookdb::Replicator::Base
 
   def _denormalized_columns
     return [
-      Webhookdb::Replicator::Column.new(:external_id, TEXT),
-      Webhookdb::Replicator::Column.new(:email, TEXT),
-      Webhookdb::Replicator::Column.new(:created_at, TIMESTAMP, converter: :tsat),
-      Webhookdb::Replicator::Column.new(:updated_at, TIMESTAMP, converter: :tsat),
+      # All of these fields are missing on delete.
+      # We merge the deleted info into an existing one when handling the upsert.
+      Webhookdb::Replicator::Column.new(:external_id, TEXT, optional: true, index: true),
+      Webhookdb::Replicator::Column.new(:email, TEXT, optional: true, index: true),
+      Webhookdb::Replicator::Column.new(
+        :created_at, TIMESTAMP, converter: QUESTIONABLE_TIMESTAMP, optional: true, index: true,
+      ),
+      Webhookdb::Replicator::Column.new(
+        :updated_at, TIMESTAMP, converter: QUESTIONABLE_TIMESTAMP, optional: true, index: true,
+      ),
+      # This is set in the contact.deleted webhook
+      Webhookdb::Replicator::Column.new(:deleted_at, TIMESTAMP, optional: true),
+      # This is set in the contact.archived webhook
+      Webhookdb::Replicator::Column.new(:archived_at, TIMESTAMP, optional: true),
     ]
   end
 
   def _mixin_backfill_url = "https://api.intercom.io/contacts"
+
+  def _resource_and_event(request)
+    resource, event = super
+    return resource, nil if event.nil?
+    # noinspection RubyCaseWithoutElseBlockInspection
+    case event.fetch("topic")
+        when "contact.deleted"
+          resource["updated_at"] = Time.now
+          resource["deleted_at"] = Time.now
+        when "contact.archived"
+          resource["updated_at"] = Time.now
+          resource["archived_at"] = Time.now
+        when "contact.unsubscribed"
+          resource = resource.fetch("contact")
+      end
+    return resource, event
+  end
+
+  def _upsert_update_expr(inserting, enrichment: nil)
+    full_update = super
+    # In the case of a delete or archive, update the deleted_at/archived_at field,
+    # and merge 'deleted' or 'archived' into the :data field.
+    if inserting[:deleted_at]
+      status_key = :deleted_at
+      status_field = "deleted"
+    elsif inserting[:archived_at]
+      status_key = :archived_at
+      status_field = "archived"
+    else
+      return full_update
+    end
+    result = {updated_at: full_update.fetch(:updated_at)}
+    result[status_key] = full_update.fetch(status_key)
+    data_col = Sequel[self.service_integration.table_name.to_sym][:data]
+    result[:data] = Sequel.join([data_col, Sequel.lit("'{\"#{status_field}\":true}'::jsonb")])
+    return result
+  end
 end

data/lib/webhookdb/replicator/intercom_conversation_v1.rb

@@ -25,14 +25,50 @@ class Webhookdb::Replicator::IntercomConversationV1 < Webhookdb::Replicator::Bas
 
   def _denormalized_columns
     return [
-      Webhookdb::Replicator::Column.new(:title, TEXT),
-      Webhookdb::Replicator::Column.new(:state, TEXT),
-      Webhookdb::Replicator::Column.new(:open, BOOLEAN),
-      Webhookdb::Replicator::Column.new(:read, BOOLEAN),
-      Webhookdb::Replicator::Column.new(:created_at, TIMESTAMP, converter: :tsat),
-      Webhookdb::Replicator::Column.new(:updated_at, TIMESTAMP, converter: :tsat),
+      Webhookdb::Replicator::Column.new(:title, TEXT, optional: true),
+      Webhookdb::Replicator::Column.new(:state, TEXT, optional: true),
+      Webhookdb::Replicator::Column.new(:open, BOOLEAN, optional: true),
+      Webhookdb::Replicator::Column.new(:read, BOOLEAN, optional: true),
+      Webhookdb::Replicator::Column.new(
+        :created_at, TIMESTAMP, converter: QUESTIONABLE_TIMESTAMP, optional: true, index: true,
+      ),
+      Webhookdb::Replicator::Column.new(
+        :updated_at, TIMESTAMP, converter: QUESTIONABLE_TIMESTAMP, optional: true, index: true,
+      ),
+      Webhookdb::Replicator::Column.new(:deleted_at, TIMESTAMP, optional: true, index: true),
     ]
   end
 
   def _mixin_backfill_url = "https://api.intercom.io/conversations"
+
+  def _resource_and_event(request)
+    resource, event = super
+    return resource, nil if event.nil?
+    # noinspection RubyCaseWithoutElseBlockInspection
+    case event.fetch("topic")
+      when "conversation.deleted"
+        resource["id"] = resource.fetch("conversation_id")
+        resource["updated_at"] = Time.now
+        resource["deleted_at"] = Time.now
+      when "conversation.contact.attached", "conversation.contact.detached"
+        # The convo is in resource['conversation']['model'], and doesn't have a number of fields.
+        # This doesn't seem like an important enough event to track for now,
+        # unless we start to do it relationally.
+        return nil, nil
+    end
+    return resource, event
+  end
+
+  def _upsert_update_expr(inserting, enrichment: nil)
+    full_update = super
+    # In the case of a delete, update the deleted_at field and merge 'deleted' into the :data field.
+    return full_update unless inserting[:deleted_at]
+    data_col = Sequel[self.service_integration.table_name.to_sym][:data]
+    result = {
+      updated_at: full_update.fetch(:updated_at),
+      deleted_at: full_update.fetch(:deleted_at),
+      data: Sequel.join([data_col, Sequel.lit("'{\"deleted\":true}'::jsonb")]),
+    }
+    return result
+  end
 end

data/lib/webhookdb/replicator/intercom_marketplace_root_v1.rb

@@ -25,17 +25,13 @@ class Webhookdb::Replicator::IntercomMarketplaceRootV1 < Webhookdb::Replicator::
     return []
   end
 
-  def _upsert_webhook(**_kwargs)
-    raise NotImplementedError("This is a stub integration only for auth purposes.")
-  end
+  def _upsert_webhook(**_kwargs) = raise NotImplementedError("This is a stub integration only for auth purposes.")
 
   def _fetch_backfill_page(*)
     return [], nil
   end
 
-  def webhook_response(_request)
-    raise NotImplementedError("This is a stub integration only for auth purposes.")
-  end
+  def webhook_response(_request) = raise NotImplementedError("This is a stub integration only for auth purposes.")
 
   def calculate_backfill_state_machine
     step = Webhookdb::Replicator::StateMachineStep.new
@@ -44,13 +40,6 @@ class Webhookdb::Replicator::IntercomMarketplaceRootV1 < Webhookdb::Replicator::
     return step
   end
 
-  def get_auth_headers
-    return {
-      "Authorization" => "Bearer #{self.service_integration.backfill_key}",
-      "Accept" => "application/json",
-    }
-  end
-
   def build_dependents
     org = self.service_integration.organization
     contact_sint = Webhookdb::ServiceIntegration.create_disambiguated(

data/lib/webhookdb/replicator/intercom_v1_mixin.rb

@@ -1,7 +1,21 @@
 # frozen_string_literal: true
 
 module Webhookdb::Replicator::IntercomV1Mixin
-  # @return [Webhookdb::ServiceIntegration]
+  # Timestamps can be unix timestamps when listing a resource,
+  # or strings in other cases, like webhooks. This may have to do with API versions.
+  # Handle both.
+  QUESTIONABLE_TIMESTAMP = Webhookdb::Replicator::Column::IsomorphicProc.new(
+    ruby: lambda do |i, **_|
+      return nil if i.nil?
+      return Time.at(i)
+    rescue TypeError
+      return Time.parse(i)
+    end,
+    sql: lambda do |*|
+      # We would have to check the type of the data, which is a pain, so don't worry about this for now.
+      raise NotImplementedError
+    end,
+  )
 
   # Quick note on these Intercom integrations: although we will technically be bringing in information from webhooks,
   # all webhooks for the WebhookDB app will use a single endpoint and we use the WebhookDB app's Client Secret for
@@ -34,20 +48,12 @@ module Webhookdb::Replicator::IntercomV1Mixin
     return self.qualified_table_sequel_identifier[:updated_at] < Sequel[:excluded][:updated_at]
   end
 
-  def _timestamp_column_name
-    return :updated_at
-  end
+  def _timestamp_column_name = :updated_at
 
   def _webhook_response(request)
-    # info for debugging
-    intercom_auth = request.env["HTTP_X_HUB_SIGNATURE"]
-
-    return Webhookdb::WebhookResponse.error("missing hmac") if intercom_auth.nil?
-    request.body.rewind
-    request_data = request.body.read
-    verified = Webhookdb::Intercom.verify_webhook(request_data, intercom_auth)
-    return Webhookdb::WebhookResponse.ok if verified
-    return Webhookdb::WebhookResponse.error("invalid hmac")
+    # Intercom webhooks are done through a centralized oauth replicator,
+    # so the secret is for the app, not the individual replicator.
+    return Webhookdb::Intercom.webhook_response(request, Webhookdb::Intercom.client_secret)
   end
 
   # @return [Webhookdb::Replicator::StateMachineStep]
@@ -67,9 +73,7 @@ module Webhookdb::Replicator::IntercomV1Mixin
     return
   end
 
-  def _mixin_backfill_url
-    raise NotImplementedError
-  end
+  def _mixin_backfill_url = raise NotImplementedError
 
   def _fetch_backfill_page(pagination_token, **_kwargs)
     unless self.auth_credentials?

data/lib/webhookdb/replicator/oauth_refresh_access_token_mixin.rb

@@ -29,7 +29,7 @@ module Webhookdb::Replicator::OAuthRefreshAccessTokenMixin
       if got
         yield got
       else
-        self.logger.info "creating_access_token"
+        self.logger.debug "creating_access_token", access_token_cache_key: key
         form_body = URI.encode_www_form(
           {
             client_id: self.service_integration.backfill_key,

data/lib/webhookdb/replicator/sponsy_v1_mixin.rb

@@ -117,7 +117,7 @@ module Webhookdb::Replicator::SponsyV1Mixin
       self.find_api_key.blank?
 
     publications_svc = self.service_integration.depends_on.replicator
-    backfillers = publications_svc.readonly_dataset(timeout: :fast) do |pub_ds|
+    backfillers = publications_svc.admin_dataset(timeout: :fast) do |pub_ds|
       pub_ds = Webhookdb::Dbutil.reduce_expr(
         pub_ds,
         :|,

data/lib/webhookdb/replicator/transistor_episode_v1.rb

@@ -54,6 +54,9 @@ class Webhookdb::Replicator::TransistorEpisodeV1 < Webhookdb::Replicator::Base
         index: true,
         data_key: ["attributes", "updated_at"],
       ),
+
+      Webhookdb::Replicator::Column.new(:transcript_text, TEXT, optional: true),
+
       # Ideally these would have converters, but they'd be very confusing, and when this was built
       # we only had one transistor user, so we truncated the table instead.
       Webhookdb::Replicator::Column.new(:api_format, INTEGER, optional: true),
@@ -93,6 +96,7 @@ class Webhookdb::Replicator::TransistorEpisodeV1 < Webhookdb::Replicator::Base
       h[:logical_description] = description
       h[:api_format] = 1
     end
+    h.merge!(enrichment) if enrichment
     return h
   end
 
@@ -133,6 +137,19 @@ class Webhookdb::Replicator::TransistorEpisodeV1 < Webhookdb::Replicator::Base
 
   BLOCK_ELEMENT_TAGS = ["p", "div"].freeze
 
+  def _fetch_enrichment(resource, *)
+    transcript_url = resource.fetch("attributes").fetch("transcript_url", nil)
+    return nil if transcript_url.blank?
+    (transcript_url += ".txt") unless transcript_url.end_with?(".txt")
+    resp = Webhookdb::Http.get(
+      transcript_url,
+      logger: self.logger,
+      timeout: Webhookdb::Transistor.http_timeout,
+    )
+    transcript_text = resp.body
+    return {transcript_text:}
+  end
+
   def upsert_has_deps?
     return true
   end