sendproof-core 0.1.0.alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (229) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +21 -0
  3. data/README.md +55 -0
  4. data/lib/sendproof/assets.rb +70 -0
  5. data/lib/sendproof/attribution.rb +98 -0
  6. data/lib/sendproof/capture.rb +66 -0
  7. data/lib/sendproof/clock.rb +54 -0
  8. data/lib/sendproof/configuration.rb +241 -0
  9. data/lib/sendproof/data_store.rb +34 -0
  10. data/lib/sendproof/dynamodb/store.rb +183 -0
  11. data/lib/sendproof/dynamodb.rb +17 -0
  12. data/lib/sendproof/elasticsearch/store.rb +24 -0
  13. data/lib/sendproof/elasticsearch.rb +17 -0
  14. data/lib/sendproof/firestore/interceptor.rb +38 -0
  15. data/lib/sendproof/firestore/store.rb +150 -0
  16. data/lib/sendproof/firestore.rb +99 -0
  17. data/lib/sendproof/handler.rb +23 -0
  18. data/lib/sendproof/identity.rb +37 -0
  19. data/lib/sendproof/limits.rb +113 -0
  20. data/lib/sendproof/middleware.rb +68 -0
  21. data/lib/sendproof/mongo/engine.rb +692 -0
  22. data/lib/sendproof/mongo/guarded.rb +100 -0
  23. data/lib/sendproof/mongo/plain.rb +45 -0
  24. data/lib/sendproof/mongo/store.rb +54 -0
  25. data/lib/sendproof/mongo/subscriber.rb +21 -0
  26. data/lib/sendproof/mongo.rb +41 -0
  27. data/lib/sendproof/mongoid/store.rb +58 -0
  28. data/lib/sendproof/mongoid.rb +65 -0
  29. data/lib/sendproof/observed_request.rb +108 -0
  30. data/lib/sendproof/opensearch/store.rb +21 -0
  31. data/lib/sendproof/opensearch.rb +17 -0
  32. data/lib/sendproof/operation_matcher.rb +28 -0
  33. data/lib/sendproof/pool_hooks.rb +80 -0
  34. data/lib/sendproof/preview_request.rb +44 -0
  35. data/lib/sendproof/preview_transport.rb +331 -0
  36. data/lib/sendproof/protocol.rb +81 -0
  37. data/lib/sendproof/provider.rb +83 -0
  38. data/lib/sendproof/recorder.rb +298 -0
  39. data/lib/sendproof/redaction.rb +98 -0
  40. data/lib/sendproof/redis/commands.rb +76 -0
  41. data/lib/sendproof/redis/store.rb +205 -0
  42. data/lib/sendproof/redis.rb +18 -0
  43. data/lib/sendproof/search/middleware.rb +39 -0
  44. data/lib/sendproof/search/store.rb +213 -0
  45. data/lib/sendproof/search.rb +82 -0
  46. data/lib/sendproof/stores/classify.rb +346 -0
  47. data/lib/sendproof/stores/guard.rb +127 -0
  48. data/lib/sendproof/stores/tables.rb +1328 -0
  49. data/lib/sendproof/stores.rb +12 -0
  50. data/lib/sendproof/trace_store.rb +110 -0
  51. data/lib/sendproof/transaction_control.rb +187 -0
  52. data/lib/sendproof/ui/action-bar.js +91 -0
  53. data/lib/sendproof/ui/api.js +239 -0
  54. data/lib/sendproof/ui/app.css +848 -0
  55. data/lib/sendproof/ui/app.js +742 -0
  56. data/lib/sendproof/ui/autofill.js +137 -0
  57. data/lib/sendproof/ui/code-copy.js +71 -0
  58. data/lib/sendproof/ui/code-picker.js +149 -0
  59. data/lib/sendproof/ui/commands.js +197 -0
  60. data/lib/sendproof/ui/components.js +77 -0
  61. data/lib/sendproof/ui/context-bar.js +82 -0
  62. data/lib/sendproof/ui/copy.js +92 -0
  63. data/lib/sendproof/ui/coverage-popover.js +35 -0
  64. data/lib/sendproof/ui/crumb-fit.js +80 -0
  65. data/lib/sendproof/ui/deprecation.js +164 -0
  66. data/lib/sendproof/ui/display-settings.js +175 -0
  67. data/lib/sendproof/ui/doc-diff.js +209 -0
  68. data/lib/sendproof/ui/docs-view.js +347 -0
  69. data/lib/sendproof/ui/favicon.png +0 -0
  70. data/lib/sendproof/ui/favicon.svg +1 -0
  71. data/lib/sendproof/ui/field-help.js +52 -0
  72. data/lib/sendproof/ui/field-modules.js +23 -0
  73. data/lib/sendproof/ui/fields/controls.js +179 -0
  74. data/lib/sendproof/ui/fields/dom.js +36 -0
  75. data/lib/sendproof/ui/fields/infer.js +71 -0
  76. data/lib/sendproof/ui/fields/lookup-source.js +63 -0
  77. data/lib/sendproof/ui/fields/lookup.js +285 -0
  78. data/lib/sendproof/ui/fields/render.js +496 -0
  79. data/lib/sendproof/ui/fields/select.js +175 -0
  80. data/lib/sendproof/ui/fields/validate.js +363 -0
  81. data/lib/sendproof/ui/fields/validation-formats.js +164 -0
  82. data/lib/sendproof/ui/fields/validation-references.js +68 -0
  83. data/lib/sendproof/ui/fields/validation-values.js +137 -0
  84. data/lib/sendproof/ui/generate-menu.js +44 -0
  85. data/lib/sendproof/ui/generate.js +184 -0
  86. data/lib/sendproof/ui/header-editor.js +777 -0
  87. data/lib/sendproof/ui/headers.js +188 -0
  88. data/lib/sendproof/ui/highlight.js +257 -0
  89. data/lib/sendproof/ui/history-popover.js +24 -0
  90. data/lib/sendproof/ui/i18n.js +253 -0
  91. data/lib/sendproof/ui/icons.js +87 -0
  92. data/lib/sendproof/ui/index.html +70 -0
  93. data/lib/sendproof/ui/issues.js +187 -0
  94. data/lib/sendproof/ui/json-view.js +166 -0
  95. data/lib/sendproof/ui/lang-icons.js +62 -0
  96. data/lib/sendproof/ui/layout.js +262 -0
  97. data/lib/sendproof/ui/listbox.js +334 -0
  98. data/lib/sendproof/ui/locales/ar-capture.js +366 -0
  99. data/lib/sendproof/ui/locales/ar-stores.js +116 -0
  100. data/lib/sendproof/ui/locales/ar.js +561 -0
  101. data/lib/sendproof/ui/locales/cs-capture.js +442 -0
  102. data/lib/sendproof/ui/locales/cs-stores.js +116 -0
  103. data/lib/sendproof/ui/locales/cs.js +689 -0
  104. data/lib/sendproof/ui/locales/da-capture.js +347 -0
  105. data/lib/sendproof/ui/locales/da-stores.js +116 -0
  106. data/lib/sendproof/ui/locales/da.js +522 -0
  107. data/lib/sendproof/ui/locales/de-capture.js +347 -0
  108. data/lib/sendproof/ui/locales/de-stores.js +116 -0
  109. data/lib/sendproof/ui/locales/de.js +519 -0
  110. data/lib/sendproof/ui/locales/el-capture.js +347 -0
  111. data/lib/sendproof/ui/locales/el-stores.js +116 -0
  112. data/lib/sendproof/ui/locales/el.js +520 -0
  113. data/lib/sendproof/ui/locales/en-capture.js +349 -0
  114. data/lib/sendproof/ui/locales/en-stores.js +95 -0
  115. data/lib/sendproof/ui/locales/en.js +522 -0
  116. data/lib/sendproof/ui/locales/es-capture.js +353 -0
  117. data/lib/sendproof/ui/locales/es-stores.js +116 -0
  118. data/lib/sendproof/ui/locales/es.js +525 -0
  119. data/lib/sendproof/ui/locales/fa-capture.js +348 -0
  120. data/lib/sendproof/ui/locales/fa-stores.js +116 -0
  121. data/lib/sendproof/ui/locales/fa.js +521 -0
  122. data/lib/sendproof/ui/locales/fi-capture.js +347 -0
  123. data/lib/sendproof/ui/locales/fi-stores.js +116 -0
  124. data/lib/sendproof/ui/locales/fi.js +523 -0
  125. data/lib/sendproof/ui/locales/fr-capture.js +353 -0
  126. data/lib/sendproof/ui/locales/fr-stores.js +116 -0
  127. data/lib/sendproof/ui/locales/fr.js +525 -0
  128. data/lib/sendproof/ui/locales/he-capture.js +358 -0
  129. data/lib/sendproof/ui/locales/he-stores.js +116 -0
  130. data/lib/sendproof/ui/locales/he.js +529 -0
  131. data/lib/sendproof/ui/locales/hi-capture.js +348 -0
  132. data/lib/sendproof/ui/locales/hi-stores.js +116 -0
  133. data/lib/sendproof/ui/locales/hi.js +523 -0
  134. data/lib/sendproof/ui/locales/id-capture.js +347 -0
  135. data/lib/sendproof/ui/locales/id-stores.js +116 -0
  136. data/lib/sendproof/ui/locales/id.js +521 -0
  137. data/lib/sendproof/ui/locales/it-capture.js +361 -0
  138. data/lib/sendproof/ui/locales/it-stores.js +116 -0
  139. data/lib/sendproof/ui/locales/it.js +523 -0
  140. data/lib/sendproof/ui/locales/ja-capture.js +346 -0
  141. data/lib/sendproof/ui/locales/ja-stores.js +116 -0
  142. data/lib/sendproof/ui/locales/ja.js +521 -0
  143. data/lib/sendproof/ui/locales/ko-capture.js +346 -0
  144. data/lib/sendproof/ui/locales/ko-stores.js +116 -0
  145. data/lib/sendproof/ui/locales/ko.js +522 -0
  146. data/lib/sendproof/ui/locales/nb-capture.js +349 -0
  147. data/lib/sendproof/ui/locales/nb-stores.js +116 -0
  148. data/lib/sendproof/ui/locales/nb.js +521 -0
  149. data/lib/sendproof/ui/locales/nl-capture.js +355 -0
  150. data/lib/sendproof/ui/locales/nl-stores.js +116 -0
  151. data/lib/sendproof/ui/locales/nl.js +520 -0
  152. data/lib/sendproof/ui/locales/pl-capture.js +432 -0
  153. data/lib/sendproof/ui/locales/pl-stores.js +116 -0
  154. data/lib/sendproof/ui/locales/pl.js +528 -0
  155. data/lib/sendproof/ui/locales/pt-BR-capture.js +350 -0
  156. data/lib/sendproof/ui/locales/pt-BR-stores.js +116 -0
  157. data/lib/sendproof/ui/locales/pt-BR.js +523 -0
  158. data/lib/sendproof/ui/locales/pt-PT-capture.js +350 -0
  159. data/lib/sendproof/ui/locales/pt-PT-stores.js +116 -0
  160. data/lib/sendproof/ui/locales/pt-PT.js +523 -0
  161. data/lib/sendproof/ui/locales/ru-capture.js +437 -0
  162. data/lib/sendproof/ui/locales/ru-stores.js +116 -0
  163. data/lib/sendproof/ui/locales/ru.js +624 -0
  164. data/lib/sendproof/ui/locales/sv-capture.js +349 -0
  165. data/lib/sendproof/ui/locales/sv-stores.js +116 -0
  166. data/lib/sendproof/ui/locales/sv.js +518 -0
  167. data/lib/sendproof/ui/locales/th-capture.js +347 -0
  168. data/lib/sendproof/ui/locales/th-stores.js +116 -0
  169. data/lib/sendproof/ui/locales/th.js +521 -0
  170. data/lib/sendproof/ui/locales/tr-capture.js +348 -0
  171. data/lib/sendproof/ui/locales/tr-stores.js +116 -0
  172. data/lib/sendproof/ui/locales/tr.js +521 -0
  173. data/lib/sendproof/ui/locales/uk-capture.js +442 -0
  174. data/lib/sendproof/ui/locales/uk-stores.js +116 -0
  175. data/lib/sendproof/ui/locales/uk.js +630 -0
  176. data/lib/sendproof/ui/locales/vi-capture.js +346 -0
  177. data/lib/sendproof/ui/locales/vi-stores.js +116 -0
  178. data/lib/sendproof/ui/locales/vi.js +523 -0
  179. data/lib/sendproof/ui/locales/zh-Hans-capture.js +347 -0
  180. data/lib/sendproof/ui/locales/zh-Hans-stores.js +116 -0
  181. data/lib/sendproof/ui/locales/zh-Hans.js +523 -0
  182. data/lib/sendproof/ui/locales/zh-Hant-capture.js +346 -0
  183. data/lib/sendproof/ui/locales/zh-Hant-stores.js +116 -0
  184. data/lib/sendproof/ui/locales/zh-Hant.js +522 -0
  185. data/lib/sendproof/ui/manifest.json +866 -0
  186. data/lib/sendproof/ui/markdown.js +39 -0
  187. data/lib/sendproof/ui/model-view.js +182 -0
  188. data/lib/sendproof/ui/navigation.js +233 -0
  189. data/lib/sendproof/ui/operation-header.js +76 -0
  190. data/lib/sendproof/ui/operations.js +7 -0
  191. data/lib/sendproof/ui/overview-view.js +63 -0
  192. data/lib/sendproof/ui/personas.js +59 -0
  193. data/lib/sendproof/ui/popover.js +71 -0
  194. data/lib/sendproof/ui/preview-false.js +108 -0
  195. data/lib/sendproof/ui/request-editor.js +496 -0
  196. data/lib/sendproof/ui/response-metadata.js +354 -0
  197. data/lib/sendproof/ui/response-viewer.js +618 -0
  198. data/lib/sendproof/ui/result-blocked.js +155 -0
  199. data/lib/sendproof/ui/result-capture.js +25 -0
  200. data/lib/sendproof/ui/result-database.js +185 -0
  201. data/lib/sendproof/ui/result-effects.js +176 -0
  202. data/lib/sendproof/ui/result-headers.js +106 -0
  203. data/lib/sendproof/ui/result-logs.js +408 -0
  204. data/lib/sendproof/ui/result-states.js +213 -0
  205. data/lib/sendproof/ui/results.js +371 -0
  206. data/lib/sendproof/ui/router.js +118 -0
  207. data/lib/sendproof/ui/schema-model.js +336 -0
  208. data/lib/sendproof/ui/schema-tree.js +344 -0
  209. data/lib/sendproof/ui/schema.js +397 -0
  210. data/lib/sendproof/ui/security.js +31 -0
  211. data/lib/sendproof/ui/seeded-menu.js +69 -0
  212. data/lib/sendproof/ui/settings.js +93 -0
  213. data/lib/sendproof/ui/shortcuts-sheet.js +73 -0
  214. data/lib/sendproof/ui/shortcuts.js +190 -0
  215. data/lib/sendproof/ui/sidebar.js +289 -0
  216. data/lib/sendproof/ui/signin.js +259 -0
  217. data/lib/sendproof/ui/snippets.js +530 -0
  218. data/lib/sendproof/ui/store-marks.js +47 -0
  219. data/lib/sendproof/ui/stores.js +131 -0
  220. data/lib/sendproof/ui/styles.css +971 -0
  221. data/lib/sendproof/ui/toggletip.js +53 -0
  222. data/lib/sendproof/ui/uuid.js +10 -0
  223. data/lib/sendproof/ui/view.js +122 -0
  224. data/lib/sendproof/ui/views.css +653 -0
  225. data/lib/sendproof/version.rb +9 -0
  226. data/lib/sendproof/workbench.rb +102 -0
  227. data/lib/sendproof-core.rb +3 -0
  228. data/lib/sendproof.rb +121 -0
  229. metadata +292 -0
@@ -0,0 +1,39 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Sendproof
4
+ module Search
5
+ # Faraday middleware that hands each request to its store before the adapter sends it. It sits first
6
+ # in the stack, so a request the guard refuses raises {Sendproof::PreviewWriteBlocked} and never
7
+ # reaches the adapter, and no byte of it leaves the process.
8
+ #
9
+ # The middleware holds only the store's system id. It looks the store up on each request, so it does
10
+ # nothing once its store was unhooked, and guards for the new one when another is registered.
11
+ class Middleware < Faraday::Middleware
12
+ # @param app [#call] the next middleware or the adapter
13
+ # @param system [String] +"elasticsearch"+ or +"opensearch"+
14
+ # @param prefix [String] the connection's path prefix, removed before classifying
15
+ def initialize(app, system, prefix = "/")
16
+ super(app)
17
+ @system = system
18
+ @prefix = prefix.to_s.chomp("/")
19
+ end
20
+
21
+ def call(env)
22
+ store = Sendproof::Search.store(@system)
23
+ return @app.call(env) unless store&.active?
24
+
25
+ store.around(env.method, request_path(env), env.body) { @app.call(env) }
26
+ end
27
+
28
+ private
29
+
30
+ # The request path without the connection's own prefix (for example +/search+ behind a proxy).
31
+ def request_path(env)
32
+ path = env.url.path
33
+ return path if @prefix.empty?
34
+
35
+ path == @prefix || path.start_with?("#{@prefix}/") ? path.delete_prefix(@prefix) : path
36
+ end
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,213 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Sendproof
4
+ module Search
5
+ # The write guard shared by {Sendproof::Elasticsearch::Store} and {Sendproof::OpenSearch::Store}.
6
+ #
7
+ # {Middleware} calls {#around} with the method, path and body of each request before the Faraday
8
+ # adapter sends it. Reads run and are logged as +Db+ entries. In a Preview the first write raises
9
+ # {Sendproof::PreviewWriteBlocked} and the Preview answers 409; nothing of the request is sent. Neither
10
+ # engine can roll back, so the store always runs in +guard+ mode.
11
+ #
12
+ # *Coverage.* A store is +global+ by default: every client of its library, built before or after the
13
+ # store, is guarded. Live clients are hooked in place when the store is created, and a Faraday
14
+ # connection built later, including one rebuilt by sniffing, gets the middleware as it is built. A
15
+ # client that uses another HTTP transport (Curb, Manticore, a custom +transport:+) can't be guarded,
16
+ # so the store refuses to be created while one exists and fails any client built later. Pass
17
+ # +global: false+ to guard only the clients you hook with {#hook} or build with {#install}; every
18
+ # other client is then not covered.
19
+ #
20
+ # Subclasses name their system and the library classes they guard.
21
+ class Store < Sendproof::DataStore
22
+ # @param client [Object, nil] a client to hook in place (implies +global: false+ unless given)
23
+ # @param name [String, nil] display label for discovery, such as +search+; never a URL or host
24
+ # @param treat_as_read [Array<String>, nil] operations you state never write, matched exactly: the
25
+ # operation name a +Db+ entry shows, such as +POST /{}/_my_read+. A wrongly listed write is not blocked.
26
+ # @param global [Boolean, nil] guard every client of the library, before and after this store
27
+ # @raise [Sendproof::ConfigurationError] when a live client can't be guarded, or a store of this system exists
28
+ def initialize(client: nil, name: nil, treat_as_read: nil, global: nil)
29
+ super()
30
+ @name = Stores::Guard.store_label(name)
31
+ @reads = Stores::Guard.read_set(treat_as_read, spaces: true)
32
+ @global = global.nil? ? client.nil? : global
33
+ @active = true
34
+ @covered = ObjectSpace::WeakMap.new
35
+ Search.register(self)
36
+ begin
37
+ install_hooks
38
+ ObjectSpace.each_object(client_class).to_a.each { |live| hook(live) if live.transport } if @global
39
+ hook(client) if client
40
+ rescue StandardError
41
+ unhook!
42
+ raise
43
+ end
44
+ end
45
+
46
+ # @return [Boolean] whether every client of the library is guarded
47
+ def global? = @global
48
+
49
+ def describe
50
+ { system: system, name: @name, mode: :guard }
51
+ end
52
+
53
+ # Turns the guard off. Connections keep their middleware, which then does nothing.
54
+ def unhook!
55
+ @active = false
56
+ Search.unregister(self)
57
+ end
58
+
59
+ # @return [Boolean] whether the guard is still on
60
+ def active? = @active
61
+
62
+ # Adds the guard to a Faraday connection, as a first (outermost) middleware, once. Use it to build
63
+ # a client of a non-global store: +Elasticsearch::Client.new(url: u) { |f| store.install(f) }+.
64
+ #
65
+ # @param faraday [Faraday::Connection]
66
+ # @return [Faraday::Connection]
67
+ def install(faraday)
68
+ return faraday if faraday.builder.handlers.any? { |handler| handler.klass == Middleware }
69
+
70
+ faraday.builder.insert(0, Middleware, system, faraday.url_prefix.path)
71
+ faraday
72
+ end
73
+
74
+ # Called as the transport builds a connection: guards it when this store covers the transport.
75
+ def cover(transport, faraday)
76
+ install(faraday) if @global || @covered.key?(transport)
77
+ end
78
+
79
+ # Hooks a client in place: its connections are rebuilt with the middleware, and so is every
80
+ # connection it rebuilds later. Hooking a client twice does nothing.
81
+ #
82
+ # @param client [Object] an +Elasticsearch::Client+ or +OpenSearch::Client+, or the transport client inside it
83
+ # @raise [Sendproof::ConfigurationError] when it doesn't use the Faraday HTTP transport
84
+ def hook(client)
85
+ transport_client = transport_client_of(client)
86
+ raise ArgumentError, "#{self.class} takes a #{wrapper_name} client." unless transport_client
87
+
88
+ transport = verify(transport_client)
89
+ @covered[transport] = true
90
+ # The transport's own constructor builds its collection this way, through the prepended
91
+ # __build_connection. Its reload helper keeps connections to hosts it already has, bare ones included.
92
+ transport.instance_variable_set(:@connections, transport.__build_connections)
93
+ unguarded = transport.connections.map(&:connection).reject { |faraday| faraday.builder.handlers.any? { |h| h.klass == Middleware } }
94
+ raise ConfigurationError, "A #{display} client's connections could not be guarded." unless unguarded.empty?
95
+
96
+ client
97
+ end
98
+
99
+ # Checks that a transport client uses the Faraday HTTP transport, the one the guard covers.
100
+ # @return [Object] the HTTP transport
101
+ def verify(transport_client)
102
+ transport = transport_client.transport
103
+ return transport if transport_class === transport
104
+
105
+ raise ConfigurationError, "A #{display} client uses #{transport.class}, which Sendproof can't guard. " \
106
+ "Use the default Faraday transport, or create the store with global: false and don't use that client in a Preview."
107
+ end
108
+
109
+ # Guards and logs one request around +block+, which sends it.
110
+ #
111
+ # @param method [Symbol, String] the HTTP method
112
+ # @param path [String] the request path without the connection's prefix
113
+ # @param body [String, Hash, nil] the request body as handed to the adapter
114
+ # @yieldreturn [Faraday::Response]
115
+ def around(method, path, body)
116
+ return yield unless @active
117
+
118
+ result = Stores::Classify.elasticsearch(method, path, body, @reads)
119
+ return yield if result.kind == :session
120
+
121
+ Stores::Guard.guard(system, result.operation, result.kind)
122
+ recorder = Capture.current
123
+ return yield unless recorder
124
+
125
+ started = Clock.now
126
+ entry = { system: system, operation: result.operation, target: target_of(path), statement: self.class.statement_of(body) }
127
+ begin
128
+ response = yield
129
+ rescue StandardError => error
130
+ recorder.db(**entry, error: error.class.name.to_s.split("::").last, duration_ms: Clock.elapsed_ms(started))
131
+ raise
132
+ end
133
+ parsed = parse(response.body)
134
+ error = response.status.to_i >= 400 ? error_type(parsed) || response.status.to_s : nil
135
+ recorder.db(**entry, rows_affected: error ? nil : rows_of(parsed), error: error, duration_ms: Clock.elapsed_ms(started))
136
+ response
137
+ end
138
+
139
+ # A request body as a +Db+ statement: every literal replaced by +?+, one line per NDJSON line.
140
+ # A body that isn't JSON (compressed, for example) gives +nil+.
141
+ def self.statement_of(body)
142
+ case body
143
+ when nil then nil
144
+ when Hash, Array then lines_of([body])
145
+ when String then lines_of(body.each_line.map(&:strip).reject(&:empty?).map { |line| JSON.parse(line) })
146
+ end
147
+ rescue JSON::ParserError, EncodingError, ArgumentError # ArgumentError: a String that isn't valid UTF-8
148
+ nil
149
+ end
150
+
151
+ # @api private
152
+ def self.lines_of(documents)
153
+ documents.flat_map { |document| document.is_a?(Array) ? document : [document] }
154
+ .map { |document| JSON.generate(Stores::Guard.sanitize_document(document)) }.join("\n")
155
+ end
156
+ private_class_method :lines_of
157
+
158
+ private
159
+
160
+ def display = Stores::Guard.store_name(system)
161
+
162
+ def wrapper_name = display
163
+
164
+ # The +Elastic::Transport::Client+ (or +OpenSearch::Transport::Client+) of a client or its wrapper.
165
+ def transport_client_of(client)
166
+ return client if client_class === client
167
+
168
+ inner = client.instance_variable_get(:@transport)
169
+ inner if client_class === inner
170
+ end
171
+
172
+ # Prepends the hooks that cover connections and clients built later, once per library class.
173
+ def install_hooks
174
+ transport_class.prepend(BuildHook) unless transport_class < BuildHook
175
+ client_class.prepend(ClientHook) if @global && !(client_class < ClientHook)
176
+ end
177
+
178
+ # The index a request names, for +db.collection.name+: the first path segment unless it is an endpoint.
179
+ def target_of(path)
180
+ first = Stores::Classify.search_segments(path).first
181
+ first unless first.nil? || first.start_with?("_")
182
+ end
183
+
184
+ def parse(body)
185
+ return nil unless body.is_a?(String) && body.bytesize <= 1_048_576 && body.lstrip.start_with?("{")
186
+
187
+ JSON.parse(body)
188
+ rescue JSON::ParserError, EncodingError, ArgumentError
189
+ nil
190
+ end
191
+
192
+ # The engine's error type: +error.type+ of an error object. Some replies (a 405, say) carry a string.
193
+ def error_type(parsed)
194
+ error = parsed["error"] if parsed.is_a?(Hash)
195
+ type = error["type"] if error.is_a?(Hash)
196
+ type if type.is_a?(String)
197
+ end
198
+
199
+ # Documents returned, updated or deleted, when the reply says so.
200
+ def rows_of(parsed)
201
+ return nil unless parsed.is_a?(Hash)
202
+
203
+ hits = parsed.dig("hits", "hits")
204
+ return hits.length if hits.is_a?(Array)
205
+
206
+ count = parsed["updated"] || parsed["deleted"]
207
+ return count if count.is_a?(Integer)
208
+
209
+ 1 if parsed["result"]
210
+ end
211
+ end
212
+ end
213
+ end
@@ -0,0 +1,82 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "faraday"
4
+ require "sendproof"
5
+
6
+ module Sendproof
7
+ # Shared by the Elasticsearch and OpenSearch write guards (spec/data-stores.md, +guard+ mode). Both
8
+ # clients send every request through a Faraday connection, so one middleware, {Middleware}, sees the
9
+ # method, path and body of every call before the adapter sends it.
10
+ #
11
+ # Require {Sendproof::Elasticsearch} or {Sendproof::OpenSearch}, not this file.
12
+ module Search
13
+ autoload :Middleware, "sendproof/search/middleware"
14
+ autoload :Store, "sendproof/search/store"
15
+
16
+ @stores = {}
17
+ @mutex = Mutex.new
18
+
19
+ class << self
20
+ # The store guarding +system+ (+"elasticsearch"+ or +"opensearch"+), or +nil+. The middleware asks
21
+ # on every request, so replacing a store never leaves connections pointing at an old one.
22
+ def store(system) = @mutex.synchronize { @stores[system] }
23
+
24
+ # Registers +store+ as the one guarding its system.
25
+ # @raise [ConfigurationError] when another store is already guarding that system
26
+ def register(store)
27
+ @mutex.synchronize do
28
+ current = @stores[store.system]
29
+ if current&.active?
30
+ raise ConfigurationError, "A #{Stores::Guard.store_name(store.system)} store is already registered; call unhook! on it first."
31
+ end
32
+
33
+ @stores[store.system] = store
34
+ end
35
+ end
36
+
37
+ # Removes +store+ from the registry, if it is the registered one.
38
+ def unregister(store) = @mutex.synchronize { @stores.delete(store.system) if @stores[store.system].equal?(store) }
39
+
40
+ # The active store whose transport class is the one of +transport+.
41
+ def store_for_transport(transport)
42
+ @mutex.synchronize { @stores.values.find { |store| store.active? && store.transport_class === transport } }
43
+ end
44
+
45
+ # Called by {BuildHook} after the HTTP transport built a Faraday connection: adds the guard to it
46
+ # when a store covers that transport. Runs for the first build and for every rebuild (sniffing,
47
+ # reloading), so a rebuilt connection is never left bare.
48
+ def guard_connection(transport, faraday)
49
+ store_for_transport(transport)&.cover(transport, faraday)
50
+ end
51
+
52
+ # Called by {ClientHook} after a transport client was built: a client the guard can't cover
53
+ # (another HTTP transport than Faraday) fails its construction when a global store is active.
54
+ def verify_client(client)
55
+ store = @mutex.synchronize { @stores.values.find { |s| s.active? && s.global? && s.client_class === client } }
56
+ store&.verify(client)
57
+ rescue ConfigurationError
58
+ client.transport = nil # a client that failed to build is unusable; later checks skip it
59
+ raise
60
+ end
61
+ end
62
+
63
+ # Prepended to an HTTP transport class: every Faraday connection it builds gets the guard, whether
64
+ # built at startup or later by sniffing or reloading.
65
+ module BuildHook
66
+ def __build_connection(host, options = {}, block = nil)
67
+ connection = super
68
+ Sendproof::Search.guard_connection(self, connection.connection)
69
+ connection
70
+ end
71
+ end
72
+
73
+ # Prepended to a transport client class: a client built after a global store was registered is
74
+ # checked for a transport the guard can cover.
75
+ module ClientHook
76
+ ruby2_keywords def initialize(*args, &block)
77
+ super
78
+ Sendproof::Search.verify_client(self)
79
+ end
80
+ end
81
+ end
82
+ end
@@ -0,0 +1,346 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+ require "set"
5
+ require "uri"
6
+
7
+ module Sendproof
8
+ module Stores
9
+ # Write classification for the batch-one data stores, driven by the shared tables in
10
+ # spec/data-stores/ (embedded in {Tables}). Each table lists reads and connection-level commands;
11
+ # everything else, including a command the table has never heard of, is a write, so the guard
12
+ # fails closed when a store adds a command.
13
+ #
14
+ # Every method is pure. The shared vectors in spec/data-stores/classification-cases.json run
15
+ # through them in test/stores/classify_test.rb.
16
+ module Classify
17
+ # The result of classifying a MongoDB command document.
18
+ # @!attribute kind [Symbol] +:read+, +:write+ or +:session+
19
+ # @!attribute refused_in_transaction [String, nil] the command or aggregation stage MongoDB
20
+ # refuses inside a transaction, such as +$out+
21
+ MongoResult = Struct.new(:kind, :refused_in_transaction)
22
+
23
+ REDIS = {
24
+ read: (Tables::REDIS["read"] + Tables::REDIS["moduleRead"]).to_set,
25
+ session: Tables::REDIS["session"].to_set,
26
+ transaction: Tables::REDIS["transaction"].to_set
27
+ }.freeze
28
+ MONGO = {
29
+ read: Tables::MONGODB["read"].to_set(&:downcase),
30
+ session: Tables::MONGODB["session"].to_set(&:downcase),
31
+ not_in_transaction: Tables::MONGODB["notInTransaction"]["commands"].to_set(&:downcase),
32
+ stages: Tables::MONGODB["notInTransaction"]["stages"].freeze
33
+ }.freeze
34
+ DYNAMO = {
35
+ read: Tables::DYNAMODB["read"].to_set,
36
+ partiql: Tables::DYNAMODB["partiql"]["operations"].to_set
37
+ }.freeze
38
+ WRITING_STAGES = %w[$out $merge].to_set.freeze
39
+
40
+ # The result of classifying an Elasticsearch, OpenSearch or Firestore request.
41
+ # @!attribute kind [Symbol] +:read+, +:write+ or +:session+
42
+ # @!attribute operation [String] the +db.operation.name+: the table's endpoint id or RPC name
43
+ Result = Struct.new(:kind, :operation)
44
+
45
+ # One endpoint of elasticsearch.json with its paths split into segments for matching.
46
+ SearchEndpoint = Struct.new(:kind, :name, :methods, :paths)
47
+
48
+ SEARCH = {
49
+ endpoints: Tables::ELASTICSEARCH["endpoints"].map do |entry|
50
+ SearchEndpoint.new(entry["kind"].to_sym, entry["name"], entry["methods"].to_set.freeze,
51
+ entry["paths"].map { |path| path.split("/").reject(&:empty?).freeze }.freeze).freeze
52
+ end.freeze,
53
+ method_default: Tables::ELASTICSEARCH["methodDefault"].freeze,
54
+ sql_words: Tables::ELASTICSEARCH["sql"]["readWords"].to_set.freeze
55
+ }.freeze
56
+ FIRESTORE = {
57
+ services: Tables::FIRESTORE["services"].freeze,
58
+ verbs: Tables::FIRESTORE["fromRest"]["verbs"].freeze
59
+ }.freeze
60
+
61
+ module_function
62
+
63
+ # Classifies a Redis command.
64
+ #
65
+ # @param name [String, Symbol] the command name in any case
66
+ # @param args [Array] its arguments; only the first is read, to form +CONTAINER|SUBCOMMAND+ keys
67
+ # such as +OBJECT|ENCODING+
68
+ # @param treat_as_read [Set<String>, nil] host-declared reads, compared exactly with the
69
+ # upper-case key
70
+ # @return [Symbol] +:read+, +:write+, +:session+ or +:transaction+
71
+ def redis(name, args = [], treat_as_read = nil)
72
+ command = name.to_s.upcase
73
+ first = args.first
74
+ with_sub = first.nil? ? command : "#{command}|#{first.to_s.upcase}"
75
+ [with_sub, command].each do |key|
76
+ return :session if REDIS[:session].include?(key)
77
+ return :transaction if REDIS[:transaction].include?(key)
78
+ return :read if REDIS[:read].include?(key) || treat_as_read&.include?(key)
79
+ end
80
+ :write
81
+ end
82
+
83
+ # Classifies a MongoDB command document by its first key, case-insensitively. +aggregate+ is a
84
+ # write when any stage at any depth is +$out+ or +$merge+; +explain+ takes the classification
85
+ # of the command inside it.
86
+ #
87
+ # @param command [Hash] the command document, string or symbol keys
88
+ # @param treat_as_read [Set<String>, nil] host-declared reads in lower case
89
+ # @return [MongoResult] the kind, and what a +preview+-mode Preview must block before sending
90
+ def mongo(command, treat_as_read = nil)
91
+ command = command.to_h
92
+ name = command.keys.first.to_s
93
+ lower = name.downcase
94
+ inner = command[command.keys.first]
95
+ return mongo(inner, treat_as_read) if lower == "explain" && inner.is_a?(Hash)
96
+
97
+ refused = name if MONGO[:not_in_transaction].include?(lower)
98
+ pipeline = fetch(command, "pipeline")
99
+ refused = MONGO[:stages].find { |stage| pipeline_uses?(pipeline, [stage]) } || refused if lower == "aggregate"
100
+ return MongoResult.new(:session, nil) if MONGO[:session].include?(lower)
101
+ return MongoResult.new(:write, refused) if lower == "aggregate" && pipeline_uses?(pipeline, WRITING_STAGES)
102
+ return MongoResult.new(:read, refused) if MONGO[:read].include?(lower) || treat_as_read&.include?(lower)
103
+
104
+ MongoResult.new(:write, refused)
105
+ end
106
+
107
+ # Skips whitespace and SQL-style comments, then returns the first word of a PartiQL statement
108
+ # in upper case. An unterminated block comment gives an empty string.
109
+ def partiql_verb(statement)
110
+ text = statement.to_s
111
+ loop do
112
+ trimmed = text.sub(/\A\s+/, "")
113
+ if trimmed.start_with?("--")
114
+ newline = trimmed.index("\n")
115
+ text = newline ? trimmed[(newline + 1)..] : ""
116
+ elsif trimmed.start_with?("/*")
117
+ close = trimmed.index("*/")
118
+ return "" unless close
119
+
120
+ text = trimmed[(close + 2)..]
121
+ else
122
+ return trimmed[/\A[A-Za-z]+/].to_s.upcase
123
+ end
124
+ end
125
+ end
126
+
127
+ # Classifies a DynamoDB action and its input as +:read+ or +:write+. A PartiQL request is a read
128
+ # only when every statement in it is a +SELECT+.
129
+ #
130
+ # @param operation [String] the API action name, such as +UpdateItem+
131
+ # @param input [Hash] the request parameters
132
+ # @param treat_as_read [Set<String>, nil] host-declared action names
133
+ def dynamodb(operation, input = {}, treat_as_read = nil)
134
+ operation = operation.to_s
135
+ return :read if DYNAMO[:read].include?(operation) || treat_as_read&.include?(operation)
136
+ return :write unless DYNAMO[:partiql].include?(operation)
137
+
138
+ statements = if operation == "ExecuteStatement"
139
+ [fetch(input, "Statement")]
140
+ else
141
+ list = fetch(input, "Statements") || fetch(input, "TransactStatements")
142
+ list.is_a?(Array) ? list.map { |item| fetch(item, "Statement") } : []
143
+ end
144
+ statements.any? && statements.all? { |statement| partiql_verb(statement) == "SELECT" } ? :read : :write
145
+ end
146
+
147
+ # Splits a request path into decoded segments: no query string, no repeated or trailing slashes.
148
+ # A segment that isn't valid percent-encoding is kept as written.
149
+ def search_segments(path)
150
+ path.to_s.split("?", 2).first.to_s.split("/").reject(&:empty?).map do |segment|
151
+ URI.decode_uri_component(segment)
152
+ rescue ArgumentError
153
+ segment
154
+ end
155
+ end
156
+
157
+ # Classifies an Elasticsearch or OpenSearch request by HTTP method and path, the unit
158
+ # elasticsearch.json defines, and for the SQL endpoints by the body too. The caller removes any
159
+ # path prefix the client is configured with. +GET+ and +HEAD+ read on every path; anything else
160
+ # the table doesn't list is a write.
161
+ #
162
+ # @param method [String, Symbol] the HTTP method in any case
163
+ # @param path [String] the request path, with or without a query string
164
+ # @param body [String, Hash, nil] the request body, read only for the SQL endpoints
165
+ # @param treat_as_read [Set<String>, nil] host-declared reads, compared exactly with the operation name
166
+ # @return [Result] +:read+, +:write+ or +:session+, with the endpoint id as the operation
167
+ def elasticsearch(method, path, body = nil, treat_as_read = nil)
168
+ verb = method.to_s.upcase
169
+ segments = search_segments(path)
170
+ SEARCH[:endpoints].each do |entry|
171
+ next unless entry.methods.include?(verb)
172
+ next unless entry.paths.any? { |pattern| search_path_matches?(pattern, segments) }
173
+
174
+ kind = entry.kind == :sql ? (search_sql_reads?(body) ? :read : :write) : entry.kind
175
+ return Result.new(kind, entry.name)
176
+ end
177
+ operation = "#{verb} /#{segments.map { |segment| segment.start_with?('_') ? segment : '{}' }.join('/')}"
178
+ default = SEARCH[:method_default][verb] || Tables::ELASTICSEARCH["default"]
179
+ default = "read" if treat_as_read&.include?(operation)
180
+ Result.new(default.to_sym, operation)
181
+ end
182
+
183
+ # Whether a request path's segments match an endpoint pattern: +{index}+ is any segment that
184
+ # doesn't start with an underscore, +{id}+ any non-empty segment, anything else must be equal.
185
+ def search_path_matches?(pattern, segments)
186
+ return false unless pattern.length == segments.length
187
+
188
+ pattern.each_with_index.all? do |part, index|
189
+ segment = segments[index]
190
+ case part
191
+ when "{index}" then !segment.empty? && !segment.start_with?("_")
192
+ when "{id}" then !segment.empty?
193
+ else part == segment
194
+ end
195
+ end
196
+ end
197
+
198
+ # Whether an SQL request is a read: one statement starting with SELECT, SHOW, DESCRIBE or DESC with
199
+ # nothing after it but whitespace, comments and one semicolon, or a cursor page with no query.
200
+ def search_sql_reads?(body)
201
+ parsed = body
202
+ if body.is_a?(String)
203
+ begin
204
+ parsed = JSON.parse(body)
205
+ rescue JSON::ParserError
206
+ return false
207
+ end
208
+ end
209
+ return false unless parsed.is_a?(Hash)
210
+
211
+ query = fetch(parsed, "query")
212
+ if query.is_a?(String)
213
+ shape = statement_shape(query, backticks: true)
214
+ return !shape[:broken] && !shape[:multiple] && SEARCH[:sql_words].include?(shape[:first])
215
+ end
216
+ query.nil? && fetch(parsed, "cursor").is_a?(String)
217
+ end
218
+
219
+ # Reads the shape of one SQL-like statement: its first word in upper case, and whether anything but
220
+ # whitespace, comments and one semicolon follows it outside quotes. +broken+ is true for an
221
+ # unterminated comment or quote. Quotes are +'+, +"+ and, with +backticks+, a backtick.
222
+ #
223
+ # @return [Hash] +:first+, +:multiple+ and +:broken+
224
+ def statement_shape(text, backticks: false)
225
+ source = text.to_s
226
+ index = 0
227
+ skip = lambda do
228
+ loop do
229
+ index += 1 while index < source.length && source[index].match?(/\s/)
230
+ if source[index, 2] == "--"
231
+ newline = source.index("\n", index)
232
+ index = newline ? newline + 1 : source.length
233
+ elsif source[index, 2] == "/*"
234
+ close = source.index("*/", index + 2)
235
+ raise ArgumentError, "unterminated comment" unless close
236
+
237
+ index = close + 2
238
+ else
239
+ break
240
+ end
241
+ end
242
+ end
243
+ skip.call
244
+ first = source[index..][/\A[A-Za-z_]+/].to_s
245
+ index += first.length
246
+ multiple = false
247
+ while index < source.length
248
+ if source[index, 2] == "--" || source[index, 2] == "/*"
249
+ skip.call
250
+ next
251
+ end
252
+ char = source[index]
253
+ if char == "'" || char == '"' || (backticks && char == "`")
254
+ close = index + 1
255
+ loop do
256
+ close = source.index(char, close) or raise ArgumentError, "unterminated quote"
257
+ if source[close + 1] == char
258
+ close += 2
259
+ next
260
+ end
261
+ break
262
+ end
263
+ index = close + 1
264
+ next
265
+ end
266
+ index += 1
267
+ next unless char == ";"
268
+
269
+ skip.call
270
+ multiple = index < source.length
271
+ break
272
+ end
273
+ { first: first.upcase, multiple: multiple, broken: false }
274
+ rescue ArgumentError
275
+ { first: "", multiple: false, broken: true }
276
+ end
277
+
278
+ # Classifies a Firestore gRPC call by RPC name, and for +Commit+ by whether it carries writes.
279
+ #
280
+ # @param rpc [String] the gRPC path (+/google.firestore.v1.Firestore/Commit+) or a bare v1 method name
281
+ # @param message [Hash, #writes, nil] the request message, when the hook can read it; a +Commit+
282
+ # whose message can't be read is a write
283
+ # @param treat_as_read [Set<String>, nil] host-declared reads, compared exactly with the RPC name
284
+ # @return [Result] +:read+, +:session+ or +:write+, with the RPC name as the operation
285
+ def firestore(rpc, message = nil, treat_as_read = nil)
286
+ service, name = rpc.to_s.include?("/") ? rpc.to_s.split("/").reject(&:empty?) : ["google.firestore.v1.Firestore", rpc.to_s]
287
+ operation = name.to_s
288
+ known = FIRESTORE[:services][service]
289
+ return Result.new(:read, operation) if treat_as_read&.include?(operation)
290
+ return Result.new(:write, operation) unless known
291
+ return Result.new(known["readPrefixes"].any? { |prefix| operation.start_with?(prefix) } ? :read : :write, operation) if known["readPrefixes"]
292
+ return Result.new(firestore_commit(message), operation) if operation == "Commit"
293
+ return Result.new(:read, operation) if known["read"].include?(operation)
294
+ return Result.new(:session, operation) if known["session"].include?(operation)
295
+
296
+ Result.new(:write, operation)
297
+ end
298
+
299
+ # A +Commit+ is +:session+ only when its message is readable and its +writes+ list is empty.
300
+ def firestore_commit(message)
301
+ return :write if message.nil?
302
+
303
+ writes = message.respond_to?(:writes) ? message.writes : fetch(message, "writes")
304
+ return :session if writes.nil? && message.is_a?(Hash)
305
+ return :write unless writes.respond_to?(:size)
306
+
307
+ writes.size.zero? ? :session : :write
308
+ end
309
+
310
+ # Derives the Firestore RPC name from a REST request, as firestore.json's +fromRest+ describes.
311
+ # The binding hooks gRPC only; this keeps the shared REST vectors honest for a future REST hook.
312
+ def firestore_rest(method, path)
313
+ segments = search_segments(path)
314
+ last = segments.last.to_s
315
+ colon = last.rindex(":")
316
+ return FIRESTORE[:verbs].fetch(last[(colon + 1)..], last[(colon + 1)..]) if colon
317
+
318
+ after = segments.length - ((segments.index("documents") || -1) + 1)
319
+ document = segments.include?("documents") && after.positive? && after.even?
320
+ case method.to_s.upcase
321
+ when "GET" then document ? "GetDocument" : "ListDocuments"
322
+ when "PATCH" then "UpdateDocument"
323
+ when "DELETE" then "DeleteDocument"
324
+ when "POST" then document ? "POST" : "CreateDocument"
325
+ else method.to_s.upcase
326
+ end
327
+ end
328
+
329
+ # Whether any key of an aggregation pipeline, at any depth (sub-pipelines included), is one of +stages+.
330
+ def pipeline_uses?(value, stages)
331
+ case value
332
+ when Array then value.any? { |item| pipeline_uses?(item, stages) }
333
+ when Hash then value.any? { |key, inner| stages.include?(key.to_s) || pipeline_uses?(inner, stages) }
334
+ else false
335
+ end
336
+ end
337
+
338
+ # Reads +key+ from a hash with string or symbol keys.
339
+ def fetch(hash, key)
340
+ return nil unless hash.respond_to?(:key?)
341
+
342
+ hash.key?(key) ? hash[key] : hash[key.to_sym]
343
+ end
344
+ end
345
+ end
346
+ end