webhookdb 0.1.1 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 86109800d36b097754f33ad1319331d4de0c4f90345a062c7f3222e57ce020db
-  data.tar.gz: 4ac7ca8c78b01ea0144f3c7b4ecba7f89bebd581a1eb119a580526d9c99a3c67
+  metadata.gz: '0285d7331a0f43ff14a35e0b578d011c4d330caa9a1b9acdc04d741bbd1f35ae'
+  data.tar.gz: 807c8366605a9de9ebd6f13502e88a892b077a1435793c5a3db6bc6b2c400dc2
 SHA512:
-  metadata.gz: 98e8f1f6dc67c6be2921270bf8c577c2702cf438251606bb0dbf58cb24660b078d9b2e5c54e5f4c4794aa6b2bf168abefafd5be620c56d5d8b8ca37cea9e32a3
-  data.tar.gz: 1f0800e477cb27e927debf7e7a05bb0eed867ff972f036eaaca733d8a3ca1927f7b73f1e006a49b85a79e5c781e8b8295b9c1c208773ae6df1790b08fada1076
+  metadata.gz: 89bf0867a274a37410d35de3ade1bef240898d518cf9377ff49acc006b56094221821e86c8ceda7ea2b2d6da68e004b58aff694c0ee778837f034f73cb0bbb22
+  data.tar.gz: 21cc2b77bb8779ceb7e451499a90b82cd6ac211108d52441bfc203ad1334001b4a02e0d1761506d8dee6fbd56ed4fe534eb151dda96da5aa49fddfbacfaf6464

data/integration/async_spec.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require "rspec/eventually"
+
+require "webhookdb/async"
+
+RSpec.describe "async workers", :integration do
+  it "emails the customer on reset code create" do
+    cu = with_async_publisher do
+      Webhookdb::Fixtures.reset_code.email.create
+    end
+
+    expect { cu.customer.refresh.message_deliveries }.to eventually(have_attributes(length: 1))
+  end
+end

data/integration/auth_spec.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+RSpec.describe "auth", :integration do
+  let(:password) { Webhookdb::Fixtures::Customers::PASSWORD }
+
+  it "allows me to sign up with a token, and log out" do
+    customer = Webhookdb::Fixtures.customer.create
+    code = Webhookdb::Fixtures.reset_code(customer:).create
+
+    login_resp = post(
+      "/v1/auth",
+      body: {email: customer.email, token: code.token},
+    )
+    expect(login_resp).to party_status(200)
+
+    customer_resp = get("/v1/me")
+    expect(customer_resp).to party_status(200)
+
+    logout_resp = post("/v1/auth/logout")
+    expect(logout_resp).to party_status(200)
+  end
+
+  it "allows me to login via OTP, and logout" do
+    customer = Webhookdb::Fixtures.customer.instance
+
+    login_resp = post("/v1/auth", body: {email: customer.email})
+    expect(login_resp).to party_status(202)
+
+    customer = Webhookdb::Customer[email: customer.email]
+    login_resp = post("/v1/auth/login_otp/#{customer.opaque_id}", body: {value: customer.reset_codes.last.token})
+    expect(login_resp).to party_status(200)
+
+    customer_resp = get("/v1/me")
+    expect(customer_resp).to party_status(200)
+
+    logout_resp = post("/v1/auth/logout")
+    expect(logout_resp).to party_status(200)
+  end
+
+  it "can access admin endpoints only if the customer authed as an admin and retains the role" do
+    customer = Webhookdb::Fixtures.customer.admin.instance
+    auth_customer(customer)
+
+    resp = get("/admin/v1/auth")
+    expect(resp).to party_status(200)
+    expect(resp).to party_response(match(hash_including(name: customer.name)))
+
+    customer.remove_role(Webhookdb::Role.admin_role)
+
+    resp = get("/admin/v1/auth")
+    expect(resp).to party_status(401)
+  end
+end

data/integration/database_spec.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+RSpec.describe "database", :integration do
+  def setup_integration_with_data(rows)
+    org = Webhookdb::Fixtures.organization.create
+    org.prepare_database_connections
+    sint = Webhookdb::Fixtures.service_integration(organization: org).create
+    sint.replicator.create_table
+    Array.new(rows) do |i|
+      t = (Time.now - i.days).iso8601
+      sint.replicator.upsert_webhook_body({"my_id" => i.to_s, "at" => t})
+    end
+    Webhookdb::Organization::DbBuilder.new(org).prepare_database_connections
+    return sint
+  end
+
+  it "can sync to database sync targets" do
+    sint = setup_integration_with_data(5)
+    sync_tgt = Webhookdb::Fixtures.sync_target(
+      service_integration: sint,
+      connection_url: sint.organization.admin_connection_url,
+    ).create
+    @to_destroy << sync_tgt
+    require "webhookdb/jobs/sync_target_run_sync"
+    Webhookdb::Jobs::SyncTargetRunSync.perform_async(sync_tgt.id)
+    expect { sync_tgt.refresh }.to eventually(have_attributes(last_synced_at: be_present))
+    Sequel.connect(sint.organization.readonly_connection_url) do |db|
+      expect(db[sint.table_name.to_sym].all).to have_attributes(size: 5)
+    end
+    expect(sync_tgt.advisory_lock(sync_tgt.db).dataset(this: true).all).to be_empty
+  end
+
+  it "can sync to http sync targets" do
+    sint = setup_integration_with_data(5)
+    sync_tgt = Webhookdb::Fixtures.sync_target(
+      service_integration: sint,
+      connection_url: "http://u:p@localhost:18015/mypath",
+    ).create
+    @to_destroy << sync_tgt
+
+    require "socket"
+    server = TCPServer.new "localhost", 18_015
+    received = []
+    server_thread = Thread.new do
+      # Will only have one session
+      session = server.accept
+      # Processing line by line isn't needed here, we're just testing so grab the whole body
+      received << session.recv(4096)
+      session.print "HTTP/1.1 200\r\n"
+      session.print "Content-Type: text/plain\r\n"
+      session.print "\r\n"
+      session.print "ok"
+      session.close
+    end
+
+    # We need to do this in-process so the sync can POST back to localhost;
+    # if the worker was on another machine, the tcp server wouldn't be reachable.
+    sync_tgt.run_sync(now: Time.now)
+    expect { sync_tgt.refresh }.to eventually(have_attributes(last_synced_at: be_present))
+    expect { received }.to eventually(contain_exactly(include("POST /mypath").and(include('"rows":'))))
+    Thread.kill(server_thread)
+    expect(sync_tgt.advisory_lock(sync_tgt.db).dataset(this: true).all).to be_empty
+  end
+
+  it "can run a database migration" do
+    sint = setup_integration_with_data(5)
+    org = sint.organization
+    dbmigration = with_async_publisher do
+      Webhookdb::Organization::DatabaseMigration.enqueue(
+        admin_connection_url_raw: org.admin_connection_url,
+        readonly_connection_url_raw: org.readonly_connection_url,
+        public_host: org.public_host,
+        started_by: nil,
+        organization: org,
+      )
+    end
+    expect { dbmigration.refresh }.to eventually(have_attributes(status: "finished"))
+    Sequel.connect(org.readonly_connection_url) do |db|
+      expect(db[Sequel[org.replication_schema.to_sym][sint.table_name.to_sym]].all).to have_attributes(size: 5)
+    end
+  end
+end

data/integration/helpers_spec.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+RSpec.describe "helpers", :integration do
+  it "can use the redis cache" do
+    key = "integration-test-key"
+    Webhookdb::Redis.cache.with do |r|
+      r.call("SET", key, "1")
+      expect(r.call("GET", key)).to eq("1")
+    end
+    t = Thread.start do
+      Webhookdb::Redis.cache.with do |r|
+        r.call("DEL", key)
+        expect(r.call("GET", key)).to be_nil
+      end
+    end
+    t.join
+    Webhookdb::Redis.cache.with do |r|
+      expect(r.call("GET", key)).to be_nil
+    end
+  end
+end

data/integration/service_integrations_spec.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require "rspec/eventually"
+
+RSpec.describe "service integrations", :integration do
+  def catch_missing_db(default)
+    yield
+  rescue Sequel::DatabaseError
+    return default
+  end
+
+  it "can create a full webhookdb customer integration and POST webhooks" do
+    c = auth_customer
+    expect { c.refresh.all_memberships }.to eventually(have_attributes(length: 1))
+    org = c.verified_memberships.last.organization
+    expect { org.refresh }.to eventually(have_attributes(readonly_connection_url: be_present))
+    org.add_feature_role(Webhookdb::Role.find_or_create(name: "internal"))
+
+    resp = post(
+      "/v1/organizations/#{org.id}/service_integrations/create",
+      body: {service_name: "webhookdb_customer_v1"},
+    )
+    expect(resp).to party_status(200)
+
+    expect(org.refresh.service_integrations).to have_attributes(length: 1)
+    sint = org.service_integrations.first
+
+    expect do
+      catch_missing_db(["default"]) { sint.replicator.readonly_dataset(&:all) }
+    end.to eventually(be_empty)
+
+    with_async_publisher do
+      Webhookdb::Fixtures.customer.create
+    end
+
+    expect do
+      catch_missing_db(["default"]) { sint.replicator.readonly_dataset(&:all) }
+    end.to eventually(have_attributes(length: 1))
+
+    # puts sint.opaque_id, "/v1/service_integrations/#{sint.opaque_id}"
+    resp = post(
+      "/v1/service_integrations/#{sint.opaque_id}",
+      body: c.values.as_json,
+      headers: {"Whdb-Secret" => sint.webhook_secret},
+      json: true,
+    )
+    expect(resp).to party_status(202)
+    expect(resp).to party_response(match(o: "k"))
+    logged_whs = Webhookdb::LoggedWebhook.where(service_integration_opaque_id: sint.opaque_id).all
+    expect(logged_whs).to_not be_empty
+  end
+
+  it "can upsert data synchrononously through endpoint" do
+    c = auth_customer
+    expect { c.refresh.all_memberships }.to eventually(have_attributes(length: 1))
+    org = c.verified_memberships.last.organization
+    expect { org.refresh }.to eventually(have_attributes(readonly_connection_url: be_present))
+    org.add_feature_role(Webhookdb::Role.find_or_create(name: "internal"))
+    sint = Webhookdb::Fixtures.service_integration(organization: org).create
+    sint.replicator.create_table
+
+    resp = post(
+      "/v1/organizations/#{org.id}/service_integrations/#{sint.opaque_id}/upsert",
+      body: {my_id: "id", at: Time.now},
+      json: true,
+    )
+    expect(resp).to party_status(200)
+    expect(resp).to party_response(match(hash_including(message: /You have upserted/)))
+
+    expect(sint.replicator.readonly_dataset(&:all)).to contain_exactly(include(my_id: "id"))
+  end
+end

data/integration/system_spec.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+RSpec.describe "system", :integration do
+  it "responds to a health check" do
+    response = HTTParty.get(url("/healthz"))
+    expect(response).to party_status(200)
+    expect(response).to party_response(eq(o: "k"))
+  end
+
+  it "responds to a status check" do
+    response = HTTParty.get(url("/statusz"))
+    expect(response).to party_status(200)
+    expect(response).to party_response(include(:version))
+  end
+end

data/lib/webhookdb/api/system.rb

@@ -13,7 +13,10 @@ class Webhookdb::API::System < Webhookdb::Service
   helpers Webhookdb::Service::Helpers
 
   get :healthz do
-    Webhookdb::Postgres::Model.db.execute("SELECT 1=1")
+    # Do not bother looking at dependencies like databases.
+    # If the primary is down, we can still accept webhooks
+    # if LoggedWebhook resiliency is configured,
+    # which is the primary thing about whether we're healthy or not.
     status 200
     {o: "k"}
   end
@@ -29,9 +32,11 @@ class Webhookdb::API::System < Webhookdb::Service
     }
   end
 
-  resource :debug do
-    get :echo do
-      pp params.to_h
+  if ["development", "test"].include?(Webhookdb::RACK_ENV)
+    resource :debug do
+      get :echo do
+        pp params.to_h
+      end
     end
   end
 end

data/lib/webhookdb/apps.rb

@@ -34,6 +34,32 @@ require "webhookdb/admin_api/roles"
 require "webterm/apps"
 
 module Webhookdb::Apps
+  # Call this from your rackup file, like config.ru.
+  #
+  # @example
+  # lib = File.expand_path("lib", __dir__)
+  # $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+  # require "webhookdb"
+  # Webhookdb.load_app
+  # require "webhookdb/apps"
+  # Webhookdb::Apps.rack_up(self)
+  #
+  def self.rack_up(config_ru)
+    Webhookdb::Async.setup_web
+    config_ru.instance_exec do
+      map "/admin" do
+        run Webhookdb::Apps::AdminAPI.build_app
+      end
+      map "/sidekiq" do
+        run Webhookdb::Apps::SidekiqWeb.to_app
+      end
+      map "/terminal" do
+        run Webhookdb::Apps::Webterm.to_app
+      end
+      run Webhookdb::Apps::API.build_app
+    end
+  end
+
   class API < Webhookdb::Service
     mount Webhookdb::API::Auth
     mount Webhookdb::API::Db

data/lib/webhookdb/pry.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require "webhookdb"
+
+module Webhookdb::Pry
+  # Call this from .pryrc.
+  #
+  # @example
+  # lib = File.expand_path("lib", __dir__)
+  # $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+  #
+  # require "appydays/dotenviable"
+  # Appydays::Dotenviable.load
+  #
+  # require "webhookdb/pry"
+  # Webhookdb::Pry.setup(self)
+  def self.setup(main)
+    main.instance_exec do
+      require "pry/clipboard"
+
+      Pry.config.commands.alias_command "ch", "copy-history"
+      Pry.config.commands.alias_command "cr", "copy-result"
+
+      # Decode the given cookie string. Since cookies are encrypted,
+      # this is useful for debugging what they contain.
+      def decode_cookie(s)
+        require "webhookdb/service"
+        return Webhookdb::Service.decode_cookie(s)
+      end
+
+      # Connect this session of Pry to the database.
+      # It also registers subscribers, so changes to the models are handled
+      # by their correct async jobs (since async jobs are handled in-process).
+      def connect
+        require "webhookdb"
+        Webhookdb.load_app
+        Webhookdb::Async.setup_web if Amigo.subscribers.empty?
+        return
+      end
+
+      def copt
+        rc = Appydays::Loggable[self].silence(:fatal) do
+          Webhookdb::Customer::ResetCode.order(:id).last
+        end
+        tok = rc.token
+        Clipboard.copy tok
+        puts "Copied OTP #{tok} for #{rc.customer.email} to clipboard"
+        return tok
+      end
+
+      # Load models and fixtures. Use this when riffing locally.
+      def repl
+        require "webhookdb"
+        Webhookdb.load_app
+        require "webhookdb/fixtures"
+        Webhookdb::Fixtures.load_all
+        return
+      end
+
+      def console
+        connect
+        require "webhookdb/console"
+        Webhookdb::Console.enable_safe_mode
+        self.extend Webhookdb::Console::MainMethods
+        Amigo.register_subscriber do |ev|
+          Webhookdb::Console.console_logger(ev)
+        end
+        return
+      end
+    end
+  end
+end

data/lib/webhookdb/replicator/docgen.rb

@@ -3,9 +3,10 @@
 # Write docs for docs.webhookdb.com Jekyll site.
 class Webhookdb::Replicator::Docgen
   def self.documentable_descriptors
-    return Webhookdb::Replicator.registry.values.reject do |repl|
-      repl.name.start_with?("webhookdb_", "fake_")
-    end.sort_by(&:name)
+    return Webhookdb::Replicator.registry.
+        values.
+        select(&:documentable?).
+        sort_by(&:name)
   end
 
   # @!attribute desc

data/lib/webhookdb/replicator.rb

@@ -79,6 +79,8 @@ class Webhookdb::Replicator
     # Is this an enterprise-only replicator?
     attr_reader :enterprise
 
+    def documentable? = @documentable
+
     def initialize(
       name:,
       ctor:,
@@ -91,7 +93,8 @@ class Webhookdb::Replicator
       api_docs_url: "",
       description: nil,
       enterprise: false,
-      documentation_url: nil
+      documentation_url: nil,
+      documentable: nil
     )
       raise ArgumentError, "must support one or both of webhooks and backfill" unless
         supports_webhooks || supports_backfill
@@ -109,7 +112,7 @@ class Webhookdb::Replicator
       @ctor = ctor.is_a?(Class) ? ctor.method(:new) : ctor
       @resource_name_plural = resource_name_plural || "#{self.resource_name_singular}s"
       @description = description || "Replicate #{self.resource_name_plural} into your database."
-      self.feature_roles
+      @documentable = documentable.nil? ? !self.name.start_with?("webhookdb_", "fake_", "theranest_") : documentable
     end
 
     def inspect

data/lib/webhookdb/spec_helpers/integration.rb

@@ -26,12 +26,12 @@ module Webhookdb::IntegrationSpecHelpers
         example.metadata[:integration]
 
       @to_destroy = []
-      WebMock.allow_net_connect!
+      WebMock.allow_net_connect! if defined?(WebMock)
     end
 
     context.after(:each) do
       @to_destroy.each(&:destroy)
-      WebMock.disable_net_connect!
+      WebMock.disable_net_connect! if defined?(WebMock)
     end
     super
   end

data/lib/webhookdb/tasks/specs.rb

@@ -36,7 +36,7 @@ module Webhookdb::Tasks
           require "webhookdb/heroku"
           Webhookdb::Heroku.client.dyno.create(
             Webhookdb::Heroku.app_name,
-            command: "bundle exec rake specs:integration_step3",
+            command: "bundle exec rake specs:heroku_integration_step3",
             env: {"INTEGRATION_TESTS" => "true"},
             attach: false,
             time_to_live: 10.minute.to_i,

data/lib/webhookdb/tasks.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require "webhookdb"
+
+module Webhookdb::Tasks
+  # Load all Webhookdb Rake tasks.
+  # You can also load them individually.
+  def self.load_all
+    require "webhookdb/tasks/admin"
+    Webhookdb::Tasks::Admin.new
+    require "webhookdb/tasks/annotate"
+    Webhookdb::Tasks::Annotate.new
+    require "webhookdb/tasks/db"
+    Webhookdb::Tasks::DB.new
+    require "webhookdb/tasks/docs"
+    Webhookdb::Tasks::Docs.new
+    require "webhookdb/tasks/fixture"
+    Webhookdb::Tasks::Fixture.new
+    require "webhookdb/tasks/release"
+    Webhookdb::Tasks::Release.new
+    require "webhookdb/tasks/message"
+    Webhookdb::Tasks::Message.new
+    require "webhookdb/tasks/regress"
+    Webhookdb::Tasks::Regress.new
+    require "webhookdb/tasks/sidekiq"
+    Webhookdb::Tasks::Sidekiq.new
+    require "webhookdb/tasks/specs"
+    Webhookdb::Tasks::Specs.new
+  end
+end

data/lib/webhookdb/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Webhookdb
-  VERSION = "0.1.1"
+  VERSION = "1.0.0"
 end

data/lib/webhookdb.rb

@@ -54,6 +54,7 @@ module Webhookdb
   RELEASE = ENV.fetch("HEROKU_RELEASE_VERSION", "unknown-release")
   RELEASE_CREATED_AT = ENV.fetch("HEROKU_RELEASE_CREATED_AT") { Time.at(0).utc.iso8601 }
   INTEGRATION_TESTS_ENABLED = ENV.fetch("INTEGRATION_TESTS", false)
+  require "webhookdb/version"
 
   DATA_DIR = Pathname(__FILE__).dirname.parent + "data"
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: webhookdb
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 1.0.0
 platform: ruby
 authors:
 - WebhookDB
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-01-08 00:00:00.000000000 Z
+date: 2024-01-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -108,6 +108,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.8'
+- !ruby/object:Gem::Dependency
+  name: clipboard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
   requirement: !ruby/object:Gem::Requirement
@@ -810,6 +824,12 @@ files:
 - db/migrations/035_synchronous_backfill.rb
 - db/migrations/036_oauth.rb
 - db/migrations/037_oauth_used.rb
+- integration/async_spec.rb
+- integration/auth_spec.rb
+- integration/database_spec.rb
+- integration/helpers_spec.rb
+- integration/service_integrations_spec.rb
+- integration/system_spec.rb
 - lib/amigo/durable_job.rb
 - lib/pry/clipboard.rb
 - lib/sequel/advisory_lock.rb
@@ -974,6 +994,7 @@ files:
 - lib/webhookdb/postgres/testing_pixie.rb
 - lib/webhookdb/postgres/validations.rb
 - lib/webhookdb/postmark.rb
+- lib/webhookdb/pry.rb
 - lib/webhookdb/redis.rb
 - lib/webhookdb/replicator.rb
 - lib/webhookdb/replicator/atom_single_feed_v1.rb
@@ -1081,6 +1102,7 @@ files:
 - lib/webhookdb/stripe.rb
 - lib/webhookdb/subscription.rb
 - lib/webhookdb/sync_target.rb
+- lib/webhookdb/tasks.rb
 - lib/webhookdb/tasks/admin.rb
 - lib/webhookdb/tasks/annotate.rb
 - lib/webhookdb/tasks/db.rb