webhookdb 1.1.0 → 1.2.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: d22b47783f1aaabc29dcc0fa023c0fee5bc9088d303103d1cf0cf859f807df0f
4
- data.tar.gz: 41f7fcafef5d6724060e99fd40c5cfd19e9cd8d58c5736ccdd1bbaef9ecca826
3
+ metadata.gz: d2c4da6abd3c5def6f27e180aaf435a4e9293eae4a1f926a065b87c1979372b3
4
+ data.tar.gz: 72de7406adc5049255878c5bf16fe05e92716d18c2f2492d996dbddb6b0aa307
5
5
  SHA512:
6
- metadata.gz: 5d33bedf7341af3216f623a9cc4328615d7a383251fc6d6b7b21000ce0366c99ea53d31f15f92d4a0bc754084faa063b1feb81e0174a966b27c7fda538f65a9a
7
- data.tar.gz: 57d0fe4657c4c54a2b36fc3f71f7ca14017473b06be658f87f813670d79d424d51692717e6c1e9558ce0c1314f068b1b78c967bbf49c5d3d8c43bb08c3af874f
6
+ metadata.gz: 725e0d6183cd7bfd259a4202c888e419658f77288de59adf2df1a6e48c36a06dd4f91857d43ac07f54e81aa9ee882a8e65813d3cb92d6cee0576d74a745ed28c
7
+ data.tar.gz: 6811f58d123c6693758c75c3e5ba4de991c7a5e3d621de581ac32fe76efb63ca8e910757cc3e42d18029fd10389c11d74b8255601781ed5c9267e042e0f7ca96
@@ -0,0 +1,17 @@
1
+ # frozen_string_literal: true
2
+
3
+ Sequel.migration do
4
+ change do
5
+ create_table(:saved_queries) do
6
+ primary_key :id
7
+ timestamptz :created_at, null: false, default: Sequel.function(:now)
8
+ timestamptz :updated_at
9
+ foreign_key :organization_id, :organizations, null: false, unique: true, on_delete: :cascade
10
+ foreign_key :created_by_id, :customers, on_delete: :set_null
11
+ text :opaque_id, null: false, unique: true
12
+ text :description, null: false
13
+ text :sql, null: false
14
+ boolean :public, null: false, default: false
15
+ end
16
+ end
17
+ end
@@ -124,25 +124,8 @@ class Webhookdb::API::Db < Webhookdb::API::V1
124
124
  end
125
125
  post :sql do
126
126
  org = lookup_org!(allow_connstr_auth: true)
127
- begin
128
- r = org.execute_readonly_query(params[:query])
129
- rescue Sequel::DatabaseError => e
130
- self.logger.error("db_query_database_error", error: e)
131
- # We want to handle InsufficientPrivileges and UndefinedTable explicitly
132
- # since we can hint the user at what to do.
133
- # Otherwise, we should just return the Postgres exception.
134
- case e.wrapped_exception
135
- when PG::UndefinedTable
136
- missing_table = e.wrapped_exception.message.match(/relation (.+) does not/)&.captures&.first
137
- msg = "The table #{missing_table} does not exist. Run `webhookdb db tables` to see available tables." if
138
- missing_table
139
- when PG::InsufficientPrivilege
140
- msg = "You do not have permission to perform this query. Queries must be read-only."
141
- else
142
- msg = e.wrapped_exception.message
143
- end
144
- merror!(403, msg, code: "invalid_query")
145
- end
127
+ r, msg = execute_readonly_query(org, params[:query])
128
+ merror!(403, msg, code: "invalid_query") if r.nil?
146
129
  status 200
147
130
  present({rows: r.rows, headers: r.columns, max_rows_reached: r.max_rows_reached})
148
131
  end
@@ -250,4 +250,31 @@ module Webhookdb::API::Helpers
250
250
  service_integration_opaque_id: opaque_id,
251
251
  )
252
252
  end
253
+
254
+ # Run the given SQL inside the org, and use special error handling if it fails.
255
+ # @return [Array<Webhookdb::Organization::QueryResult,String,nil>] Tuple of query result, and optional message.
256
+ # On query success, return <QueryResult, nil>.
257
+ # On DatabaseError, return <nil, message>.
258
+ # On other types of errors, raise.
259
+ def execute_readonly_query(org, sql)
260
+ result = org.execute_readonly_query(sql)
261
+ return result, nil
262
+ rescue Sequel::DatabaseError => e
263
+ self.logger.error("db_query_database_error", error: e)
264
+ # We want to handle InsufficientPrivileges and UndefinedTable explicitly
265
+ # since we can hint the user at what to do.
266
+ # Otherwise, we should just return the Postgres exception.
267
+ msg = ""
268
+ case e.wrapped_exception
269
+ when PG::UndefinedTable
270
+ missing_table = e.wrapped_exception.message.match(/relation (.+) does not/)&.captures&.first
271
+ msg = "The table #{missing_table} does not exist. Run `webhookdb db tables` to see available tables." if
272
+ missing_table
273
+ when PG::InsufficientPrivilege
274
+ msg = "You do not have permission to perform this query. Queries must be read-only."
275
+ else
276
+ msg = e.wrapped_exception.message
277
+ end
278
+ return [nil, msg]
279
+ end
253
280
  end
@@ -0,0 +1,219 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "webhookdb/api"
4
+ require "webhookdb/saved_query"
5
+
6
+ class Webhookdb::API::SavedQueries < Webhookdb::API::V1
7
+ resource :organizations do
8
+ route_param :org_identifier do
9
+ resource :saved_queries do
10
+ helpers do
11
+ def lookup!
12
+ org = lookup_org!
13
+ # We can add other identifiers in the future
14
+ cq = org.saved_queries_dataset[opaque_id: params[:query_identifier]]
15
+ merror!(403, "There is no saved query with that identifier.") if cq.nil?
16
+ return cq
17
+ end
18
+
19
+ def guard_editable!(customer, cq)
20
+ return if customer === cq.created_by
21
+ return if has_admin?(cq.organization, customer:)
22
+ permission_error!("You must be the query's creator or an org admin.")
23
+ end
24
+
25
+ def execute_readonly_query_with_suggestion(org, sql)
26
+ r, message = execute_readonly_query(org, sql)
27
+ return r, nil unless r.nil?
28
+ msg = "Something went wrong running your query. Perhaps a table it depends on was deleted. " \
29
+ "Check out #{Webhookdb::SavedQuery::DOCS_URL} for troubleshooting tips. " \
30
+ "Here's what went wrong: #{message}"
31
+ return r, msg
32
+ end
33
+ end
34
+
35
+ desc "Returns a list of all custom queries associated with the org."
36
+ get do
37
+ queries = lookup_org!.saved_queries
38
+ message = ""
39
+ if queries.empty?
40
+ message = "This organization doesn't have any saved queries yet.\n" \
41
+ "Use `webhookdb saved-query create` to set one up."
42
+ end
43
+ present_collection queries, with: SavedQueryEntity, message:
44
+ end
45
+
46
+ desc "Creates a custom query."
47
+ params do
48
+ optional :description, type: String, prompt: "What is the query used for? "
49
+ optional :sql, type: String, prompt: "Enter the SQL you would like to run: "
50
+ optional :public, type: Boolean
51
+ end
52
+ post :create do
53
+ cust = current_customer
54
+ org = lookup_org!
55
+ _, errmsg = execute_readonly_query_with_suggestion(org, params[:sql])
56
+ if errmsg
57
+ Webhookdb::API::Helpers.prompt_for_required_param!(
58
+ request,
59
+ :sql,
60
+ "Enter a new query:",
61
+ output: "That query was invalid. #{errmsg}\n" \
62
+ "You can iterate on your query by connecting to your database from any SQL editor.\n" \
63
+ "Use `webhookdb db connection` to get your query string.",
64
+ )
65
+ end
66
+ cq = Webhookdb::SavedQuery.create(
67
+ description: params[:description],
68
+ sql: params[:sql],
69
+ organization: org,
70
+ created_by: cust,
71
+ public: params[:public] || false,
72
+ )
73
+ message = "You have created a new saved query with an id of '#{cq.opaque_id}'. " \
74
+ "You can run it through the CLI, or through the API with or without authentication. " \
75
+ "See #{Webhookdb::SavedQuery::DOCS_URL} for more information."
76
+ status 200
77
+ present cq, with: SavedQueryEntity, message:
78
+ end
79
+
80
+ route_param :query_identifier, type: String do
81
+ desc "Returns the query with the given identifier."
82
+ get do
83
+ cq = lookup!
84
+ status 200
85
+ message = "See #{Webhookdb::SavedQuery::DOCS_URL} to see how to run or modify your query."
86
+ present cq, with: SavedQueryEntity, message:
87
+ end
88
+
89
+ desc "Runs the query with the given identifier."
90
+ get :run do
91
+ _customer = current_customer
92
+ org = lookup_org!
93
+ cq = lookup!
94
+ r, msg = execute_readonly_query_with_suggestion(org, cq.sql)
95
+ merror!(400, msg) if r.nil?
96
+ status 200
97
+ present({rows: r.rows, headers: r.columns, max_rows_reached: r.max_rows_reached})
98
+ end
99
+
100
+ desc "Updates the field on a custom query."
101
+ params do
102
+ optional :field, type: String, prompt: "What field would you like to update (one of: " \
103
+ "#{Webhookdb::SavedQuery::CLI_EDITABLE_FIELDS.join(', ')}): "
104
+ optional :value, type: String, prompt: "What is the new value? "
105
+ end
106
+ post :update do
107
+ customer = current_customer
108
+ cq = lookup!
109
+ guard_editable!(customer, cq)
110
+ # Instead of specifying which values are valid for the optional `field` param in the param declaration,
111
+ # we do the validation here so that we can provide a more helpful error message
112
+ unless Webhookdb::SavedQuery::CLI_EDITABLE_FIELDS.include?(params[:field])
113
+ merror!(400, "That field is not editable.")
114
+ end
115
+ value = params[:value]
116
+ case params[:field]
117
+ when "public"
118
+ begin
119
+ value = Webhookdb.parse_bool(value)
120
+ rescue ArgumentError => e
121
+ Webhookdb::API::Helpers.prompt_for_required_param!(
122
+ request,
123
+ :value,
124
+ e.message + "\nAny boolean-like string (true, false, yes, no, etc) will work:",
125
+ )
126
+ end
127
+ cq.public = value
128
+ when "sql"
129
+ r, msg = execute_readonly_query_with_suggestion(cq.organization, value)
130
+ if r.nil?
131
+ Webhookdb::API::Helpers.prompt_for_required_param!(
132
+ request,
133
+ :value,
134
+ "Enter your query:",
135
+ output: msg,
136
+ )
137
+ end
138
+ cq.sql = value
139
+ else
140
+ cq.send(:"#{params[:field]}=", value)
141
+ end
142
+ cq.save_changes
143
+ status 200
144
+ # Do not render the value here, it can be very long.
145
+ message = "You have updated '#{params[:field]}' on saved query '#{cq.opaque_id}'."
146
+ present cq, with: SavedQueryEntity, message:
147
+ end
148
+
149
+ params do
150
+ optional :field, type: String, values: Webhookdb::SavedQuery::INFO_FIELDS.keys + [""]
151
+ end
152
+ post :info do
153
+ cq = lookup!
154
+ data = Webhookdb::SavedQuery::INFO_FIELDS.
155
+ to_h { |k, v| [k.to_sym, cq.send(v)] }
156
+
157
+ field_name = params[:field]
158
+ blocks = Webhookdb::Formatting.blocks
159
+ if field_name.present?
160
+ blocks.line(data.fetch(field_name.to_sym))
161
+ else
162
+ rows = data.map do |k, v|
163
+ [k.to_s.humanize, v.to_s]
164
+ end
165
+ blocks.table(["Field", "Value"], rows)
166
+ end
167
+ r = {blocks: blocks.as_json}
168
+ status 200
169
+ present r
170
+ end
171
+
172
+ post :delete do
173
+ customer = current_customer
174
+ cq = lookup!
175
+ guard_editable!(customer, cq)
176
+ cq.destroy
177
+ status 200
178
+ present cq, with: SavedQueryEntity,
179
+ message: "You have successfully deleted the saved query '#{cq.description}'."
180
+ end
181
+ end
182
+ end
183
+ end
184
+ end
185
+
186
+ resource :saved_queries do
187
+ route_param :query_identifier, type: String do
188
+ get :run do
189
+ # This endpoint can be used publicly, so should expose as little information as possible.
190
+ # Do not expose permissions or query details.
191
+ cq = Webhookdb::SavedQuery[opaque_id: params[:query_identifier]]
192
+ forbidden! if cq.nil?
193
+ if cq.private?
194
+ authed = Webhookdb::API::ConnstrAuth.find_authed([cq.organization], request)
195
+ if !authed && (cust = current_customer?)
196
+ authed = !cust.verified_memberships_dataset.where(organization: cq.organization).empty?
197
+ end
198
+ forbidden! unless authed
199
+ end
200
+ r, _ = execute_readonly_query(cq.organization, cq.sql)
201
+ merror!(400, "Something went wrong running the query.") if r.nil?
202
+ status 200
203
+ present({rows: r.rows, headers: r.columns, max_rows_reached: r.max_rows_reached})
204
+ end
205
+ end
206
+ end
207
+
208
+ class SavedQueryEntity < Webhookdb::API::BaseEntity
209
+ expose :opaque_id, as: :id
210
+ expose :description
211
+ expose :sql
212
+ expose :public
213
+ expose :run_url
214
+
215
+ def self.display_headers
216
+ return [[:id, "Id"], [:description, "Description"], [:public, "Public"], [:run_url, "Run URL"]]
217
+ end
218
+ end
219
+ end
@@ -112,18 +112,14 @@ class Webhookdb::API::SyncTargets < Webhookdb::API::V1
112
112
  params do
113
113
  use :connection_url
114
114
  use :sync_target_params
115
- optional :service_integration_opaque_id,
116
- type: String, allow_blank: false,
117
- desc: "This is a deprecated parameter. In the future, please use `service_integration_identifier`."
118
- optional :service_integration_identifier, type: String, allow_blank: false
119
- at_least_one_of :service_integration_opaque_id, :service_integration_identifier
115
+ requires :service_integration_identifier, type: String, allow_blank: false
120
116
  end
121
117
  route_setting :target_type, target_type_resource
122
118
  post :create do
123
119
  customer = current_customer
124
120
  org = lookup_org!(customer:)
125
121
  ensure_admin!(org, customer:)
126
- identifier = params[:service_integration_identifier] || params[:service_integration_opaque_id]
122
+ identifier = params[:service_integration_identifier]
127
123
  sint = lookup_service_integration!(org, identifier)
128
124
 
129
125
  validate_period!(sint.organization, params[:period_seconds])
@@ -6,46 +6,40 @@ class Webhookdb::API::WebhookSubscriptions < Webhookdb::API::V1
6
6
  resource :organizations do
7
7
  route_param :org_identifier, type: String do
8
8
  resource :webhook_subscriptions do
9
- desc "Return all webhook subscriptions for the given org, and all integrations."
9
+ desc "Return all notifications for the given org and its integrations."
10
10
  get do
11
11
  org = lookup_org!
12
12
  subs = org.all_webhook_subscriptions
13
13
  message = ""
14
14
  if subs.empty?
15
15
  message = "Organization #{org.name} has no webhook subscriptions set up.\n" \
16
- "Use `webhookdb webhooks create` to set one up."
16
+ "Use `webhookdb notifications create` to set one up."
17
17
  end
18
18
  status 200
19
19
  present_collection subs, with: Webhookdb::API::WebhookSubscriptionEntity, message:
20
20
  end
21
21
 
22
22
  params do
23
- optional :url, prompt: "Enter the URL that WebhookDB should POST webhooks to:"
24
- optional :webhook_secret, prompt: "Enter a random secret used to sign and verify webhooks to the given url:"
25
23
  optional :service_integration_identifier,
26
24
  type: String,
27
- desc: "If provided, attach the webhook subscription to this integration rather than the org."
28
- optional :service_integration_opaque_id,
29
- type: String,
30
- desc: "This is a deprecated parameter. In the future, please use `service_integration_identifier`."
25
+ desc: "If provided, attach the webhook subscription to this integration rather than the org.",
26
+ prompt: "Which integration is this for? Use the service name, table name, or opaque id.\n" \
27
+ "See your integrations with `webhookdb integrations list`:"
28
+ optional :url, prompt: "Enter the URL that WebhookDB should POST notifications to:"
29
+ optional :webhook_secret,
30
+ prompt: "Enter a random secret used to sign and verify notifications to the given url:"
31
31
  end
32
32
  post :create do
33
33
  org = lookup_org!
34
- sint = nil
35
- identifier = params[:service_integration_identifier] || params[:service_integration_opaque_id]
36
- sint = lookup_service_integration!(org, identifier) if identifier.present?
34
+ sint = lookup_service_integration!(org, params[:service_integration_identifier])
35
+ url = params[:url]
37
36
  webhook_sub = Webhookdb::WebhookSubscription.create(
38
37
  webhook_secret: params[:webhook_secret],
39
- deliver_to_url: params[:url],
40
- organization: sint ? nil : org,
38
+ deliver_to_url: url,
41
39
  service_integration: sint,
42
40
  created_by: current_customer,
43
41
  )
44
- message = if sint
45
- "All webhooks for this #{sint.service_name} integration will be sent to #{params[:url]}"
46
- else
47
- "All webhooks for all integrations belonging to organization #{org.name} will be sent to #{params[:url]}."
48
- end
42
+ message = "All notifications for this #{sint.service_name} integration will be sent to #{url}"
49
43
  status 200
50
44
  present webhook_sub, with: Webhookdb::API::WebhookSubscriptionEntity, message:
51
45
  end
data/lib/webhookdb/api.rb CHANGED
@@ -60,13 +60,22 @@ module Webhookdb::API
60
60
  return org
61
61
  end
62
62
 
63
- def ensure_admin!(org=nil, customer: nil)
63
+ # rubocop:disable Naming/PredicateName
64
+ def has_admin?(org=nil, customer: nil)
65
+ # rubocop:enable Naming/PredicateName
64
66
  customer ||= current_customer
65
67
  org ||= lookup_org!
66
68
  has_no_admin = org.verified_memberships_dataset.
67
69
  where(customer:, membership_role: Webhookdb::Role.admin_role).
68
70
  empty?
69
- permission_error!("You don't have admin privileges with #{org.name}.") if has_no_admin
71
+ return !has_no_admin
72
+ end
73
+
74
+ def ensure_admin!(org=nil, customer: nil)
75
+ org ||= lookup_org!
76
+ admin = has_admin?(org, customer:)
77
+ # noinspection RubyNilAnalysis
78
+ permission_error!("You don't have admin privileges with #{org.name}.") unless admin
70
79
  end
71
80
  end
72
81
 
@@ -17,6 +17,7 @@ require "webhookdb/api/install"
17
17
  require "webhookdb/api/me"
18
18
  require "webhookdb/api/organizations"
19
19
  require "webhookdb/api/replay"
20
+ require "webhookdb/api/saved_queries"
20
21
  require "webhookdb/api/service_integrations"
21
22
  require "webhookdb/api/services"
22
23
  require "webhookdb/api/stripe"
@@ -68,6 +69,7 @@ module Webhookdb::Apps
68
69
  mount Webhookdb::API::Me
69
70
  mount Webhookdb::API::Organizations
70
71
  mount Webhookdb::API::Replay
72
+ mount Webhookdb::API::SavedQueries
71
73
  mount Webhookdb::API::ServiceIntegrations
72
74
  mount Webhookdb::API::Services
73
75
  mount Webhookdb::API::Stripe
@@ -2,6 +2,8 @@
2
2
 
3
3
  require "appydays/configurable"
4
4
 
5
+ require "webhookdb/crypto"
6
+
5
7
  module Webhookdb::EmailOctopus
6
8
  include Appydays::Configurable
7
9
 
@@ -0,0 +1,29 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Webhookdb
4
+ module Envfixer
5
+ # If DOCKER_DEV is set, replace 'localhost' urls with 'host.docker.internal'.
6
+ def self.replace_localhost_for_docker(env)
7
+ return unless env["DOCKER_DEV"]
8
+ env.each do |k, v|
9
+ begin
10
+ localhost = URI(v).host == "localhost"
11
+ rescue StandardError
12
+ next
13
+ end
14
+ next unless localhost
15
+ env[k] = v.gsub("localhost", "host.docker.internal")
16
+ end
17
+ end
18
+
19
+ # If MERGE_HEROKU_ENV, merge all of its environment vars into the current env
20
+ def self.merge_heroku_env(env)
21
+ return unless (heroku_app = env.fetch("MERGE_HEROKU_ENV", nil))
22
+ text = `heroku config -j --app=#{heroku_app}`
23
+ json = Oj.load(text)
24
+ json.each do |k, v|
25
+ env[k] = v
26
+ end
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,27 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "faker"
4
+
5
+ require "webhookdb"
6
+ require "webhookdb/fixtures"
7
+
8
+ module Webhookdb::Fixtures::SavedQueries
9
+ extend Webhookdb::Fixtures
10
+
11
+ fixtured_class Webhookdb::SavedQuery
12
+
13
+ base :saved_query do
14
+ self.description ||= Faker::Lorem.sentence
15
+ self.sql ||= "SELECT * FROM mytable"
16
+ end
17
+
18
+ before_saving do |instance|
19
+ instance.organization ||= Webhookdb::Fixtures.organization.create
20
+ instance
21
+ end
22
+
23
+ decorator :created_by do |c={}|
24
+ c = Webhookdb::Fixtures.customer.create(c) unless c.is_a?(Webhookdb::Customer)
25
+ self.created_by = c
26
+ end
27
+ end
@@ -9,6 +9,12 @@ module Webhookdb::GoogleCalendar
9
9
  # How many calendars/events should we fetch in a single page?
10
10
  # Higher uses slightly more memory but fewer API calls.
11
11
  # Max of 2500.
12
+ #
13
+ # **NOTE:** Changing this in production will
14
+ # INVALIDATE EXISTING RESOURCES CAUSING A FULL RESYNC.
15
+ # You should, in general, avoid modifying this number once there is
16
+ # much Google Calendar data stored. Instead, page sizes will be automatically reduced
17
+ # as requests time out.
12
18
  setting :list_page_size, 2000
13
19
  # How many rows should we upsert at a time?
14
20
  # Higher is fewer upserts, but can create very large SQL strings,
@@ -13,5 +13,11 @@ module Webhookdb::Icalendar
13
13
  # Do not store events older then this when syncing recurring events.
14
14
  # Many icalendar feeds are misconfigured and this prevents enumerating 2000+ years of recurrence.
15
15
  setting :oldest_recurring_event, "1990-01-01", convert: ->(s) { Date.parse(s) }
16
+ # Sync icalendar calendars only this often.
17
+ # Most services only update every day or so. Assume it takes 5s to sync each feed (request, parse, upsert).
18
+ # If you have 10,000 feeds, that is 50,000 seconds, or almost 14 hours of processing time,
19
+ # or two threads for 7 hours. The resyncs are spread out across the sync period
20
+ # (ie, no thundering herd every 8 hours), but it is still a good idea to sync as infrequently as possible.
21
+ setting :sync_period_hours, 6
16
22
  end
17
23
  end
@@ -3,6 +3,10 @@
3
3
  require "webhookdb/async/scheduled_job"
4
4
  require "webhookdb/jobs"
5
5
 
6
+ # For every IcalendarCalendar row needing a sync (across all service integrations),
7
+ # enqueue a +Webhookdb::Jobs::IcalendarSync+ job.
8
+ # Jobs are 'splayed' over 1/4 of the configured calendar sync period (see +Webhookdb::Icalendar+)
9
+ # to avoid a thundering herd.
6
10
  class Webhookdb::Jobs::IcalendarEnqueueSyncs
7
11
  extend Webhookdb::Async::ScheduledJob
8
12
 
@@ -10,12 +14,14 @@ class Webhookdb::Jobs::IcalendarEnqueueSyncs
10
14
  splay 30
11
15
 
12
16
  def _perform
17
+ max_splay = Webhookdb::Icalendar.sync_period_hours.hours.to_i / 4
13
18
  Webhookdb::ServiceIntegration.dataset.where_each(service_name: "icalendar_calendar_v1") do |sint|
14
19
  sint.replicator.admin_dataset do |ds|
15
20
  sint.replicator.rows_needing_sync(ds).each do |row|
16
21
  calendar_external_id = row.fetch(:external_id)
17
22
  self.with_log_tags(sint.log_tags) do
18
- enqueued_job_id = Webhookdb::Jobs::IcalendarSync.perform_async(sint.id, calendar_external_id)
23
+ splay = rand(1..max_splay)
24
+ enqueued_job_id = Webhookdb::Jobs::IcalendarSync.perform_in(splay, sint.id, calendar_external_id)
19
25
  self.logger.info("enqueued_icalendar_sync", calendar_external_id:, enqueued_job_id:)
20
26
  end
21
27
  end
@@ -15,8 +15,8 @@ class Webhookdb::Jobs::PrepareDatabaseConnections
15
15
  org.db.transaction do
16
16
  # If creating the public host fails, we end up with an orphaned database,
17
17
  # but that's not a big deal- we can eventually see it's empty/unlinked and drop it.
18
- org.prepare_database_connections
19
- org.create_public_host_cname
18
+ org.prepare_database_connections(safe: true)
19
+ org.create_public_host_cname(safe: true)
20
20
  end
21
21
  end
22
22
  end
@@ -9,7 +9,7 @@ module Webhookdb::Jobs
9
9
  # Create a single way to do the common task of automatic scheduled backfills.
10
10
  # Each integration that needs automated backfills can add a specification here.
11
11
  module ScheduledBackfills
12
- Spec = Struct.new(:klass, :service_name, :cron_expr, :splay, :incremental)
12
+ Spec = Struct.new(:klass, :service_name, :cron_expr, :splay, :incremental, :recursive)
13
13
 
14
14
  # @param spec [Spec]
15
15
  def self.install(spec)
@@ -21,7 +21,8 @@ module Webhookdb::Jobs
21
21
 
22
22
  define_method(:_perform) do
23
23
  Webhookdb::ServiceIntegration.dataset.where_each(service_name: spec.service_name) do |sint|
24
- Webhookdb::BackfillJob.create(service_integration: sint, incremental: spec.incremental).enqueue
24
+ m = spec.recursive ? :create_recursive : :create
25
+ Webhookdb::BackfillJob.send(m, service_integration: sint, incremental: spec.incremental).enqueue
25
26
  end
26
27
  end
27
28
  end
@@ -31,19 +32,19 @@ module Webhookdb::Jobs
31
32
  [
32
33
  Spec.new(
33
34
  "ConvertkitBroadcastBackfill", "convertkit_broadcast_v1",
34
- "10 * * * *", 2.minutes, false,
35
+ "10 * * * *", 2.minutes, false, false,
35
36
  ),
36
37
  Spec.new(
37
38
  "ConvertkitSubscriberBackfill", "convertkit_subscriber_v1",
38
- "20 * * * *", 2.minutes, true,
39
+ "20 * * * *", 2.minutes, true, false,
39
40
  ),
40
41
  Spec.new(
41
42
  "ConvertkitTagBackfill", "convertkit_tag_v1",
42
- "30 * * * *", 2.minutes, false,
43
+ "30 * * * *", 2.minutes, false, false,
43
44
  ),
44
45
  Spec.new(
45
46
  "EmailOctopusScheduledBackfill", "email_octopus_list_v1",
46
- Webhookdb::EmailOctopus.cron_expression, 2.minutes, false,
47
+ Webhookdb::EmailOctopus.cron_expression, 2.minutes, false, true,
47
48
  ),
48
49
  Spec.new(
49
50
  "GithubRepoActivityScheduledBackfill", "github_repository_event_v1",
@@ -51,31 +52,31 @@ module Webhookdb::Jobs
51
52
  ),
52
53
  Spec.new(
53
54
  "IntercomScheduledBackfill", "intercom_marketplace_root_v1",
54
- "*/1 * * * *", 0, false,
55
+ "*/1 * * * *", 0, false, true,
55
56
  ),
56
57
  Spec.new(
57
58
  "AtomSingleFeedPoller", "atom_single_feed_v1",
58
- "11 * * * *", 10.seconds, true,
59
+ "11 * * * *", 10.seconds, true, false,
59
60
  ),
60
61
  Spec.new(
61
62
  "SponsyScheduledBackfill", "sponsy_publication_v1",
62
- Webhookdb::Sponsy.cron_expression, 30.seconds, true,
63
+ Webhookdb::Sponsy.cron_expression, 30.seconds, true, true,
63
64
  ),
64
65
  Spec.new(
65
66
  "TransistorEpisodeBackfill", "transistor_episode_v1",
66
- Webhookdb::Transistor.episode_cron_expression, 2.minutes, true,
67
+ Webhookdb::Transistor.episode_cron_expression, 2.minutes, true, true,
67
68
  ),
68
69
  Spec.new(
69
70
  "TransistorShowBackfill", "transistor_show_v1",
70
- Webhookdb::Transistor.show_cron_expression, 2.minutes, true,
71
+ Webhookdb::Transistor.show_cron_expression, 2.minutes, true, false,
71
72
  ),
72
73
  Spec.new(
73
74
  "TwilioSmsBackfill", "twilio_sms_v1",
74
- "*/1 * * * *", 0, true,
75
+ "*/1 * * * *", 0, true, false,
75
76
  ),
76
77
  Spec.new(
77
78
  "SignalwireMessageBackfill", "signalwire_message_v1",
78
- "*/1 * * * *", 0, true,
79
+ "*/1 * * * *", 0, true, false,
79
80
  ),
80
81
  ].each { |sp| self.install(sp) }
81
82
  end
@@ -34,6 +34,7 @@ class Webhookdb::Organization < Webhookdb::Postgres::Model(:organizations)
34
34
  adder: ->(om) { om.update(organization_id: id, verified: false) },
35
35
  order: :id
36
36
  one_to_many :service_integrations, class: "Webhookdb::ServiceIntegration", order: :id
37
+ one_to_many :saved_queries, class: "Webhookdb::SavedQuery", order: :id
37
38
  one_to_many :webhook_subscriptions, class: "Webhookdb::WebhookSubscription", order: :id
38
39
  many_to_many :feature_roles, class: "Webhookdb::Role", join_table: :feature_roles_organizations, right_key: :role_id
39
40
  one_to_many :all_webhook_subscriptions,
@@ -204,6 +205,7 @@ class Webhookdb::Organization < Webhookdb::Postgres::Model(:organizations)
204
205
  end
205
206
 
206
207
  # Build the org-specific users, database, and set our connection URLs to it.
208
+ # @param safe [*] If true, noop if connection urls are set.
207
209
  def prepare_database_connections(safe: false)
208
210
  self.db.transaction do
209
211
  self.lock!
@@ -220,13 +222,17 @@ class Webhookdb::Organization < Webhookdb::Postgres::Model(:organizations)
220
222
  end
221
223
 
222
224
  # Create a CNAME in Cloudflare for the currently configured connection urls.
223
- def create_public_host_cname
225
+ # @param safe [*] If true, noop if the public host is set.
226
+ def create_public_host_cname(safe: false)
224
227
  self.db.transaction do
225
228
  self.lock!
226
229
  # We must have a host to create a CNAME to.
227
230
  raise Webhookdb::InvalidPrecondition, "connection urls must be set" if self.readonly_connection_url_raw.blank?
228
231
  # Should only be used once when creating the org DBs.
229
- raise Webhookdb::InvalidPrecondition, "public_host must not be set" if self.public_host.present?
232
+ if self.public_host.present?
233
+ return if safe
234
+ raise Webhookdb::InvalidPrecondition, "public_host must not be set"
235
+ end
230
236
  # Use the raw URL, even though we know at this point
231
237
  # public_host is empty so raw and public host urls are the same.
232
238
  Webhookdb::Organization::DbBuilder.new(self).create_public_host_cname(self.readonly_connection_url_raw)
@@ -60,6 +60,7 @@ module Webhookdb::Postgres
60
60
  "webhookdb/organization/database_migration",
61
61
  "webhookdb/organization_membership",
62
62
  "webhookdb/role",
63
+ "webhookdb/saved_query",
63
64
  "webhookdb/service_integration",
64
65
  "webhookdb/subscription",
65
66
  "webhookdb/sync_target",
@@ -120,11 +121,13 @@ module Webhookdb::Postgres
120
121
  end
121
122
 
122
123
  def self.run_all_migrations(target: nil)
124
+ # :nocov:
123
125
  Sequel.extension :migration
124
126
  Webhookdb::Postgres.each_model_superclass do |cls|
125
127
  cls.install_all_extensions
126
128
  Sequel::Migrator.run(cls.db, Pathname(__FILE__).dirname.parent.parent + "db/migrations", target:)
127
129
  end
130
+ # :nocov:
128
131
  end
129
132
 
130
133
  # We can always register the models right away, since it does not have a side effect.
@@ -263,7 +263,7 @@ All of this information can be found in the WebhookDB docs, at https://docs.webh
263
263
  item[:front_message_id] = front_response_body.fetch("message_uid")
264
264
  end
265
265
  else
266
- messaged_at = Time.at(item.fetch(:data).fetch("created_at"))
266
+ messaged_at = Time.at(item.fetch(:data).fetch("payload").fetch("created_at"))
267
267
  if messaged_at < Webhookdb::Front.channel_sync_refreshness_cutoff.seconds.ago
268
268
  # Do not sync old rows, just mark them synced
269
269
  item[:signalwire_sid] = "skipped_due_to_age"
@@ -125,10 +125,9 @@ The secret to use for signing is:
125
125
  end
126
126
 
127
127
  CLEANUP_SERVICE_NAMES = ["icalendar_event_v1"].freeze
128
- SYNC_PERIOD = 4.hours
129
128
 
130
129
  def rows_needing_sync(dataset, now: Time.now)
131
- cutoff = now - SYNC_PERIOD
130
+ cutoff = now - Webhookdb::Icalendar.sync_period_hours.hours
132
131
  return dataset.where(Sequel[last_synced_at: nil] | Sequel.expr { last_synced_at < cutoff })
133
132
  end
134
133
 
@@ -215,7 +214,11 @@ The secret to use for signing is:
215
214
  expected_errors = [
216
215
  417, # If someone uses an Outlook HTML calendar, fetch gives us a 417
217
216
  ]
218
- is_problem_error = (response_status > 404 || response_status < 400) &&
217
+ # For most client errors, we can't do anything about it. For example,
218
+ # and 'unshared' URL could result in a 401, 403, 404, or even a 405.
219
+ # For now, other client errors, we can raise on,
220
+ # in case it's something we can fix/work around.
221
+ is_problem_error = (response_status > 405 || response_status < 400) &&
219
222
  !expected_errors.include?(response_status)
220
223
  raise e if is_problem_error
221
224
  response_body = e.response.body.to_s
@@ -155,7 +155,9 @@ Press 'Show' next to the newly-created API token, and copy it.)
155
155
  def _fetch_backfill_page(pagination_token, last_backfilled:)
156
156
  urltail = pagination_token
157
157
  if pagination_token.blank?
158
- date_send_max = Date.tomorrow
158
+ # We need to handle positive and negative UTC offset running locally (non-UTC).
159
+ # Using UTC + 1 day would give 'today' in some cases, we always want 'tomorrow the day after'.
160
+ date_send_max = (Time.now.utc + 2.days).to_date
159
161
  urltail = "/2010-04-01/Accounts/#{self.service_integration.backfill_key}/Messages.json" \
160
162
  "?PageSize=100&DateSend%3C=#{date_send_max}"
161
163
  end
@@ -128,7 +128,9 @@ Both of these values should be visible from the homepage of your Twilio admin Da
128
128
  def _fetch_backfill_page(pagination_token, last_backfilled:)
129
129
  url = "https://api.twilio.com"
130
130
  if pagination_token.blank?
131
- date_send_max = Date.tomorrow
131
+ # We need to handle positive and negative UTC offset running locally (non-UTC).
132
+ # Using UTC + 1 day would give 'today' in some cases, we always want 'tomorrow the day after'.
133
+ date_send_max = (Time.now.utc + 2.days).to_date
132
134
  url += "/2010-04-01/Accounts/#{self.service_integration.backfill_key}/Messages.json" \
133
135
  "?PageSize=100&DateSend%3C=#{date_send_max}"
134
136
  else
@@ -0,0 +1,28 @@
1
+ # frozen_string_literal: true
2
+
3
+ class Webhookdb::SavedQuery < Webhookdb::Postgres::Model(:saved_queries)
4
+ plugin :timestamps
5
+
6
+ CLI_EDITABLE_FIELDS = ["description", "sql", "public"].freeze
7
+ INFO_FIELDS = {
8
+ "id" => :opaque_id,
9
+ "description" => :description,
10
+ "public" => :public,
11
+ "run_url" => :run_url,
12
+ "sql" => :sql,
13
+ }.freeze
14
+ DOCS_URL = "https://docs.webhookdb.com/docs/integrating/saved-queries.html"
15
+
16
+ many_to_one :organization, class: "Webhookdb::Organization"
17
+ many_to_one :created_by, class: "Webhookdb::Customer"
18
+
19
+ alias public? public
20
+ def private? = !self.public?
21
+
22
+ def before_create
23
+ self[:opaque_id] ||= Webhookdb::Id.new_opaque_id("svq")
24
+ super
25
+ end
26
+
27
+ def run_url = "#{Webhookdb.api_url}/v1/saved_queries/#{self.opaque_id}/run"
28
+ end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Webhookdb
4
- VERSION = "1.1.0"
4
+ VERSION = "1.2.0"
5
5
  end
data/lib/webhookdb.rb CHANGED
@@ -8,29 +8,12 @@ require "money"
8
8
  require "pathname"
9
9
  require "phony"
10
10
 
11
+ require "webhookdb/envfixer"
11
12
  require "webhookdb/json"
13
+ require "webhookdb/method_utilities"
12
14
 
13
- if ENV["DOCKER_DEV"]
14
- # If DOCKER_DEV is set, replace 'localhost' urls with 'host.docker.internal'.
15
- ENV.each do |k, v|
16
- begin
17
- localhost = URI(v).host == "localhost"
18
- rescue StandardError
19
- next
20
- end
21
- next unless localhost
22
- ENV[k] = v.gsub("localhost", "host.docker.internal")
23
- end
24
- end
25
-
26
- if (heroku_app = ENV.fetch("MERGE_HEROKU_ENV", nil))
27
- # If MERGE_HEROKU_ENV, merge all of its environment vars into the current env
28
- text = `heroku config -j --app=#{heroku_app}`
29
- json = Oj.load(text)
30
- json.each do |k, v|
31
- ENV[k] = v
32
- end
33
- end
15
+ Webhookdb::Envfixer.replace_localhost_for_docker(ENV)
16
+ Webhookdb::Envfixer.merge_heroku_env(ENV)
34
17
 
35
18
  Money.locale_backend = :i18n
36
19
  Money.default_currency = "USD"
@@ -50,6 +33,7 @@ end
50
33
  module Webhookdb
51
34
  include Appydays::Loggable
52
35
  include Appydays::Configurable
36
+ extend Webhookdb::MethodUtilities
53
37
 
54
38
  # Error raised when we cannot take an action
55
39
  # because some condition has not been set up right.
@@ -86,6 +70,9 @@ module Webhookdb
86
70
 
87
71
  DATA_DIR = Pathname(__FILE__).dirname.parent + "data"
88
72
 
73
+ singleton_attr_reader :globals_cache
74
+ @globals_cache = {}
75
+
89
76
  configurable(:webhookdb) do
90
77
  setting :log_level_override,
91
78
  nil,
@@ -100,6 +87,10 @@ module Webhookdb
100
87
  setting :support_email, "hello@webhookdb.com"
101
88
  setting :use_globals_cache, false
102
89
  setting :regression_mode, false
90
+
91
+ after_configured do
92
+ globals_cache.clear
93
+ end
103
94
  end
104
95
 
105
96
  # Regression mode is true when we re replaying webhooks locally,
@@ -110,9 +101,6 @@ module Webhookdb
110
101
  return self.regression_mode
111
102
  end
112
103
 
113
- require "webhookdb/method_utilities"
114
- extend Webhookdb::MethodUtilities
115
-
116
104
  require "webhookdb/sentry"
117
105
 
118
106
  def self.load_app
@@ -129,9 +117,6 @@ module Webhookdb
129
117
  # :section: Globals cache
130
118
  #
131
119
 
132
- singleton_attr_reader :globals_cache
133
- @globals_cache = {}
134
-
135
120
  # If globals caching is enabled, see if there is a cached value under +key+
136
121
  # and return it if so. If there is not, evaluate the given block and store that value.
137
122
  # Generally used for looking up well-known database objects like certain roles.
@@ -141,7 +126,7 @@ module Webhookdb
141
126
  return result if result
142
127
  end
143
128
  result = yield()
144
- self.globals_cache[key] = result
129
+ (self.globals_cache[key] = result) if self.use_globals_cache
145
130
  return result
146
131
  end
147
132
 
@@ -167,18 +152,6 @@ module Webhookdb
167
152
  return key
168
153
  end
169
154
 
170
- #
171
- # :section: Unambiguous/promo code chars
172
- #
173
-
174
- # Remove ambiguous characters (L, I, 1 or 0, O) and vowels from possible codes
175
- # to avoid creating ambiguous codes or real words.
176
- UNAMBIGUOUS_CHARS = "CDFGHJKMNPQRTVWXYZ23469".chars.freeze
177
-
178
- def self.take_unambiguous_chars(n)
179
- return Array.new(n) { UNAMBIGUOUS_CHARS.sample }.join
180
- end
181
-
182
155
  # Convert a string into something we consistently use for slugs:
183
156
  # a-z, 0-9, and underscores only. Leading numbers are converted to words.
184
157
  #
@@ -206,6 +179,17 @@ module Webhookdb
206
179
  "9" => "nine",
207
180
  }.freeze
208
181
 
182
+ def self.parse_bool(s)
183
+ # rubocop:disable Style/NumericPredicate
184
+ return false if s == nil? || s.blank? || s == 0
185
+ # rubocop:enable Style/NumericPredicate
186
+ return true if s.is_a?(Integer)
187
+ sb = s.to_s.downcase
188
+ return true if ["true", "t", "yes", "y", "on", "1"].include?(sb)
189
+ return false if ["false", "f", "no", "n", "off", "0"].include?(sb)
190
+ raise ArgumentError, "unparseable bool: #{s.inspect}"
191
+ end
192
+
209
193
  # Return the request user and admin stored in TLS. See service.rb for implementation.
210
194
  #
211
195
  # Note that the second return value (the admin) will be nil if not authed as an admin,
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: webhookdb
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.1.0
4
+ version: 1.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - WebhookDB
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2024-01-19 00:00:00.000000000 Z
11
+ date: 2024-02-05 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activesupport
@@ -853,6 +853,7 @@ files:
853
853
  - db/migrations/036_oauth.rb
854
854
  - db/migrations/037_oauth_used.rb
855
855
  - db/migrations/038_webhookdb_api_key.rb
856
+ - db/migrations/039_saved_query.rb
856
857
  - integration/async_spec.rb
857
858
  - integration/auth_spec.rb
858
859
  - integration/database_spec.rb
@@ -883,6 +884,7 @@ files:
883
884
  - lib/webhookdb/api/me.rb
884
885
  - lib/webhookdb/api/organizations.rb
885
886
  - lib/webhookdb/api/replay.rb
887
+ - lib/webhookdb/api/saved_queries.rb
886
888
  - lib/webhookdb/api/service_integrations.rb
887
889
  - lib/webhookdb/api/services.rb
888
890
  - lib/webhookdb/api/stripe.rb
@@ -918,6 +920,7 @@ files:
918
920
  - lib/webhookdb/developer_alert.rb
919
921
  - lib/webhookdb/email_octopus.rb
920
922
  - lib/webhookdb/enumerable.rb
923
+ - lib/webhookdb/envfixer.rb
921
924
  - lib/webhookdb/fixtures.rb
922
925
  - lib/webhookdb/fixtures/backfill_jobs.rb
923
926
  - lib/webhookdb/fixtures/customers.rb
@@ -930,6 +933,7 @@ files:
930
933
  - lib/webhookdb/fixtures/organization_memberships.rb
931
934
  - lib/webhookdb/fixtures/organizations.rb
932
935
  - lib/webhookdb/fixtures/reset_codes.rb
936
+ - lib/webhookdb/fixtures/saved_queries.rb
933
937
  - lib/webhookdb/fixtures/service_integrations.rb
934
938
  - lib/webhookdb/fixtures/subscriptions.rb
935
939
  - lib/webhookdb/fixtures/sync_targets.rb
@@ -1103,6 +1107,7 @@ files:
1103
1107
  - lib/webhookdb/replicator/webhook_request.rb
1104
1108
  - lib/webhookdb/replicator/webhookdb_customer_v1.rb
1105
1109
  - lib/webhookdb/role.rb
1110
+ - lib/webhookdb/saved_query.rb
1106
1111
  - lib/webhookdb/sentry.rb
1107
1112
  - lib/webhookdb/service.rb
1108
1113
  - lib/webhookdb/service/auth.rb