sendproof-core 0.1.0.alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (229) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +21 -0
  3. data/README.md +55 -0
  4. data/lib/sendproof/assets.rb +70 -0
  5. data/lib/sendproof/attribution.rb +98 -0
  6. data/lib/sendproof/capture.rb +66 -0
  7. data/lib/sendproof/clock.rb +54 -0
  8. data/lib/sendproof/configuration.rb +241 -0
  9. data/lib/sendproof/data_store.rb +34 -0
  10. data/lib/sendproof/dynamodb/store.rb +183 -0
  11. data/lib/sendproof/dynamodb.rb +17 -0
  12. data/lib/sendproof/elasticsearch/store.rb +24 -0
  13. data/lib/sendproof/elasticsearch.rb +17 -0
  14. data/lib/sendproof/firestore/interceptor.rb +38 -0
  15. data/lib/sendproof/firestore/store.rb +150 -0
  16. data/lib/sendproof/firestore.rb +99 -0
  17. data/lib/sendproof/handler.rb +23 -0
  18. data/lib/sendproof/identity.rb +37 -0
  19. data/lib/sendproof/limits.rb +113 -0
  20. data/lib/sendproof/middleware.rb +68 -0
  21. data/lib/sendproof/mongo/engine.rb +692 -0
  22. data/lib/sendproof/mongo/guarded.rb +100 -0
  23. data/lib/sendproof/mongo/plain.rb +45 -0
  24. data/lib/sendproof/mongo/store.rb +54 -0
  25. data/lib/sendproof/mongo/subscriber.rb +21 -0
  26. data/lib/sendproof/mongo.rb +41 -0
  27. data/lib/sendproof/mongoid/store.rb +58 -0
  28. data/lib/sendproof/mongoid.rb +65 -0
  29. data/lib/sendproof/observed_request.rb +108 -0
  30. data/lib/sendproof/opensearch/store.rb +21 -0
  31. data/lib/sendproof/opensearch.rb +17 -0
  32. data/lib/sendproof/operation_matcher.rb +28 -0
  33. data/lib/sendproof/pool_hooks.rb +80 -0
  34. data/lib/sendproof/preview_request.rb +44 -0
  35. data/lib/sendproof/preview_transport.rb +331 -0
  36. data/lib/sendproof/protocol.rb +81 -0
  37. data/lib/sendproof/provider.rb +83 -0
  38. data/lib/sendproof/recorder.rb +298 -0
  39. data/lib/sendproof/redaction.rb +98 -0
  40. data/lib/sendproof/redis/commands.rb +76 -0
  41. data/lib/sendproof/redis/store.rb +205 -0
  42. data/lib/sendproof/redis.rb +18 -0
  43. data/lib/sendproof/search/middleware.rb +39 -0
  44. data/lib/sendproof/search/store.rb +213 -0
  45. data/lib/sendproof/search.rb +82 -0
  46. data/lib/sendproof/stores/classify.rb +346 -0
  47. data/lib/sendproof/stores/guard.rb +127 -0
  48. data/lib/sendproof/stores/tables.rb +1328 -0
  49. data/lib/sendproof/stores.rb +12 -0
  50. data/lib/sendproof/trace_store.rb +110 -0
  51. data/lib/sendproof/transaction_control.rb +187 -0
  52. data/lib/sendproof/ui/action-bar.js +91 -0
  53. data/lib/sendproof/ui/api.js +239 -0
  54. data/lib/sendproof/ui/app.css +848 -0
  55. data/lib/sendproof/ui/app.js +742 -0
  56. data/lib/sendproof/ui/autofill.js +137 -0
  57. data/lib/sendproof/ui/code-copy.js +71 -0
  58. data/lib/sendproof/ui/code-picker.js +149 -0
  59. data/lib/sendproof/ui/commands.js +197 -0
  60. data/lib/sendproof/ui/components.js +77 -0
  61. data/lib/sendproof/ui/context-bar.js +82 -0
  62. data/lib/sendproof/ui/copy.js +92 -0
  63. data/lib/sendproof/ui/coverage-popover.js +35 -0
  64. data/lib/sendproof/ui/crumb-fit.js +80 -0
  65. data/lib/sendproof/ui/deprecation.js +164 -0
  66. data/lib/sendproof/ui/display-settings.js +175 -0
  67. data/lib/sendproof/ui/doc-diff.js +209 -0
  68. data/lib/sendproof/ui/docs-view.js +347 -0
  69. data/lib/sendproof/ui/favicon.png +0 -0
  70. data/lib/sendproof/ui/favicon.svg +1 -0
  71. data/lib/sendproof/ui/field-help.js +52 -0
  72. data/lib/sendproof/ui/field-modules.js +23 -0
  73. data/lib/sendproof/ui/fields/controls.js +179 -0
  74. data/lib/sendproof/ui/fields/dom.js +36 -0
  75. data/lib/sendproof/ui/fields/infer.js +71 -0
  76. data/lib/sendproof/ui/fields/lookup-source.js +63 -0
  77. data/lib/sendproof/ui/fields/lookup.js +285 -0
  78. data/lib/sendproof/ui/fields/render.js +496 -0
  79. data/lib/sendproof/ui/fields/select.js +175 -0
  80. data/lib/sendproof/ui/fields/validate.js +363 -0
  81. data/lib/sendproof/ui/fields/validation-formats.js +164 -0
  82. data/lib/sendproof/ui/fields/validation-references.js +68 -0
  83. data/lib/sendproof/ui/fields/validation-values.js +137 -0
  84. data/lib/sendproof/ui/generate-menu.js +44 -0
  85. data/lib/sendproof/ui/generate.js +184 -0
  86. data/lib/sendproof/ui/header-editor.js +777 -0
  87. data/lib/sendproof/ui/headers.js +188 -0
  88. data/lib/sendproof/ui/highlight.js +257 -0
  89. data/lib/sendproof/ui/history-popover.js +24 -0
  90. data/lib/sendproof/ui/i18n.js +253 -0
  91. data/lib/sendproof/ui/icons.js +87 -0
  92. data/lib/sendproof/ui/index.html +70 -0
  93. data/lib/sendproof/ui/issues.js +187 -0
  94. data/lib/sendproof/ui/json-view.js +166 -0
  95. data/lib/sendproof/ui/lang-icons.js +62 -0
  96. data/lib/sendproof/ui/layout.js +262 -0
  97. data/lib/sendproof/ui/listbox.js +334 -0
  98. data/lib/sendproof/ui/locales/ar-capture.js +366 -0
  99. data/lib/sendproof/ui/locales/ar-stores.js +116 -0
  100. data/lib/sendproof/ui/locales/ar.js +561 -0
  101. data/lib/sendproof/ui/locales/cs-capture.js +442 -0
  102. data/lib/sendproof/ui/locales/cs-stores.js +116 -0
  103. data/lib/sendproof/ui/locales/cs.js +689 -0
  104. data/lib/sendproof/ui/locales/da-capture.js +347 -0
  105. data/lib/sendproof/ui/locales/da-stores.js +116 -0
  106. data/lib/sendproof/ui/locales/da.js +522 -0
  107. data/lib/sendproof/ui/locales/de-capture.js +347 -0
  108. data/lib/sendproof/ui/locales/de-stores.js +116 -0
  109. data/lib/sendproof/ui/locales/de.js +519 -0
  110. data/lib/sendproof/ui/locales/el-capture.js +347 -0
  111. data/lib/sendproof/ui/locales/el-stores.js +116 -0
  112. data/lib/sendproof/ui/locales/el.js +520 -0
  113. data/lib/sendproof/ui/locales/en-capture.js +349 -0
  114. data/lib/sendproof/ui/locales/en-stores.js +95 -0
  115. data/lib/sendproof/ui/locales/en.js +522 -0
  116. data/lib/sendproof/ui/locales/es-capture.js +353 -0
  117. data/lib/sendproof/ui/locales/es-stores.js +116 -0
  118. data/lib/sendproof/ui/locales/es.js +525 -0
  119. data/lib/sendproof/ui/locales/fa-capture.js +348 -0
  120. data/lib/sendproof/ui/locales/fa-stores.js +116 -0
  121. data/lib/sendproof/ui/locales/fa.js +521 -0
  122. data/lib/sendproof/ui/locales/fi-capture.js +347 -0
  123. data/lib/sendproof/ui/locales/fi-stores.js +116 -0
  124. data/lib/sendproof/ui/locales/fi.js +523 -0
  125. data/lib/sendproof/ui/locales/fr-capture.js +353 -0
  126. data/lib/sendproof/ui/locales/fr-stores.js +116 -0
  127. data/lib/sendproof/ui/locales/fr.js +525 -0
  128. data/lib/sendproof/ui/locales/he-capture.js +358 -0
  129. data/lib/sendproof/ui/locales/he-stores.js +116 -0
  130. data/lib/sendproof/ui/locales/he.js +529 -0
  131. data/lib/sendproof/ui/locales/hi-capture.js +348 -0
  132. data/lib/sendproof/ui/locales/hi-stores.js +116 -0
  133. data/lib/sendproof/ui/locales/hi.js +523 -0
  134. data/lib/sendproof/ui/locales/id-capture.js +347 -0
  135. data/lib/sendproof/ui/locales/id-stores.js +116 -0
  136. data/lib/sendproof/ui/locales/id.js +521 -0
  137. data/lib/sendproof/ui/locales/it-capture.js +361 -0
  138. data/lib/sendproof/ui/locales/it-stores.js +116 -0
  139. data/lib/sendproof/ui/locales/it.js +523 -0
  140. data/lib/sendproof/ui/locales/ja-capture.js +346 -0
  141. data/lib/sendproof/ui/locales/ja-stores.js +116 -0
  142. data/lib/sendproof/ui/locales/ja.js +521 -0
  143. data/lib/sendproof/ui/locales/ko-capture.js +346 -0
  144. data/lib/sendproof/ui/locales/ko-stores.js +116 -0
  145. data/lib/sendproof/ui/locales/ko.js +522 -0
  146. data/lib/sendproof/ui/locales/nb-capture.js +349 -0
  147. data/lib/sendproof/ui/locales/nb-stores.js +116 -0
  148. data/lib/sendproof/ui/locales/nb.js +521 -0
  149. data/lib/sendproof/ui/locales/nl-capture.js +355 -0
  150. data/lib/sendproof/ui/locales/nl-stores.js +116 -0
  151. data/lib/sendproof/ui/locales/nl.js +520 -0
  152. data/lib/sendproof/ui/locales/pl-capture.js +432 -0
  153. data/lib/sendproof/ui/locales/pl-stores.js +116 -0
  154. data/lib/sendproof/ui/locales/pl.js +528 -0
  155. data/lib/sendproof/ui/locales/pt-BR-capture.js +350 -0
  156. data/lib/sendproof/ui/locales/pt-BR-stores.js +116 -0
  157. data/lib/sendproof/ui/locales/pt-BR.js +523 -0
  158. data/lib/sendproof/ui/locales/pt-PT-capture.js +350 -0
  159. data/lib/sendproof/ui/locales/pt-PT-stores.js +116 -0
  160. data/lib/sendproof/ui/locales/pt-PT.js +523 -0
  161. data/lib/sendproof/ui/locales/ru-capture.js +437 -0
  162. data/lib/sendproof/ui/locales/ru-stores.js +116 -0
  163. data/lib/sendproof/ui/locales/ru.js +624 -0
  164. data/lib/sendproof/ui/locales/sv-capture.js +349 -0
  165. data/lib/sendproof/ui/locales/sv-stores.js +116 -0
  166. data/lib/sendproof/ui/locales/sv.js +518 -0
  167. data/lib/sendproof/ui/locales/th-capture.js +347 -0
  168. data/lib/sendproof/ui/locales/th-stores.js +116 -0
  169. data/lib/sendproof/ui/locales/th.js +521 -0
  170. data/lib/sendproof/ui/locales/tr-capture.js +348 -0
  171. data/lib/sendproof/ui/locales/tr-stores.js +116 -0
  172. data/lib/sendproof/ui/locales/tr.js +521 -0
  173. data/lib/sendproof/ui/locales/uk-capture.js +442 -0
  174. data/lib/sendproof/ui/locales/uk-stores.js +116 -0
  175. data/lib/sendproof/ui/locales/uk.js +630 -0
  176. data/lib/sendproof/ui/locales/vi-capture.js +346 -0
  177. data/lib/sendproof/ui/locales/vi-stores.js +116 -0
  178. data/lib/sendproof/ui/locales/vi.js +523 -0
  179. data/lib/sendproof/ui/locales/zh-Hans-capture.js +347 -0
  180. data/lib/sendproof/ui/locales/zh-Hans-stores.js +116 -0
  181. data/lib/sendproof/ui/locales/zh-Hans.js +523 -0
  182. data/lib/sendproof/ui/locales/zh-Hant-capture.js +346 -0
  183. data/lib/sendproof/ui/locales/zh-Hant-stores.js +116 -0
  184. data/lib/sendproof/ui/locales/zh-Hant.js +522 -0
  185. data/lib/sendproof/ui/manifest.json +866 -0
  186. data/lib/sendproof/ui/markdown.js +39 -0
  187. data/lib/sendproof/ui/model-view.js +182 -0
  188. data/lib/sendproof/ui/navigation.js +233 -0
  189. data/lib/sendproof/ui/operation-header.js +76 -0
  190. data/lib/sendproof/ui/operations.js +7 -0
  191. data/lib/sendproof/ui/overview-view.js +63 -0
  192. data/lib/sendproof/ui/personas.js +59 -0
  193. data/lib/sendproof/ui/popover.js +71 -0
  194. data/lib/sendproof/ui/preview-false.js +108 -0
  195. data/lib/sendproof/ui/request-editor.js +496 -0
  196. data/lib/sendproof/ui/response-metadata.js +354 -0
  197. data/lib/sendproof/ui/response-viewer.js +618 -0
  198. data/lib/sendproof/ui/result-blocked.js +155 -0
  199. data/lib/sendproof/ui/result-capture.js +25 -0
  200. data/lib/sendproof/ui/result-database.js +185 -0
  201. data/lib/sendproof/ui/result-effects.js +176 -0
  202. data/lib/sendproof/ui/result-headers.js +106 -0
  203. data/lib/sendproof/ui/result-logs.js +408 -0
  204. data/lib/sendproof/ui/result-states.js +213 -0
  205. data/lib/sendproof/ui/results.js +371 -0
  206. data/lib/sendproof/ui/router.js +118 -0
  207. data/lib/sendproof/ui/schema-model.js +336 -0
  208. data/lib/sendproof/ui/schema-tree.js +344 -0
  209. data/lib/sendproof/ui/schema.js +397 -0
  210. data/lib/sendproof/ui/security.js +31 -0
  211. data/lib/sendproof/ui/seeded-menu.js +69 -0
  212. data/lib/sendproof/ui/settings.js +93 -0
  213. data/lib/sendproof/ui/shortcuts-sheet.js +73 -0
  214. data/lib/sendproof/ui/shortcuts.js +190 -0
  215. data/lib/sendproof/ui/sidebar.js +289 -0
  216. data/lib/sendproof/ui/signin.js +259 -0
  217. data/lib/sendproof/ui/snippets.js +530 -0
  218. data/lib/sendproof/ui/store-marks.js +47 -0
  219. data/lib/sendproof/ui/stores.js +131 -0
  220. data/lib/sendproof/ui/styles.css +971 -0
  221. data/lib/sendproof/ui/toggletip.js +53 -0
  222. data/lib/sendproof/ui/uuid.js +10 -0
  223. data/lib/sendproof/ui/view.js +122 -0
  224. data/lib/sendproof/ui/views.css +653 -0
  225. data/lib/sendproof/version.rb +9 -0
  226. data/lib/sendproof/workbench.rb +102 -0
  227. data/lib/sendproof-core.rb +3 -0
  228. data/lib/sendproof.rb +121 -0
  229. metadata +292 -0
@@ -0,0 +1,298 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Sendproof
4
+ # Collects effects, logs and SQL timing for one preview or observed send. Thread safe.
5
+ class Recorder
6
+ Snapshot = Struct.new(:effects, :logs, :logs_truncated, :effects_truncated, :changes, :changes_truncated,
7
+ keyword_init: true)
8
+
9
+ # The first write the guard refused during a preview: what the 409 answer names.
10
+ BlockedWrite = Struct.new(:store, :operation, :detail)
11
+
12
+ attr_reader :started
13
+ # @return [BlockedWrite, nil] the first refused write; it decides a preview's 409, whatever the handler returns
14
+ attr_reader :blocked
15
+ # @return [Exception, nil] the first transaction-control refusal; it withholds confirmation
16
+ attr_reader :fault
17
+
18
+ # @param rollback [Boolean] true for a preview (effects are suppressed), false for an observed send
19
+ # @param include_sql_values [Boolean] whether non-secret SQL parameter values are recorded
20
+ def initialize(rollback:, started: Clock.now, include_sql_values: false)
21
+ @rollback = rollback
22
+ @started = started
23
+ @include_sql_values = include_sql_values
24
+ @mutex = Mutex.new
25
+ @effects = []
26
+ @logs = []
27
+ @active = true
28
+ @logs_truncated = false
29
+ @effects_truncated = false
30
+ @log_size = 0
31
+ @effect_size = 0
32
+ @db_ms = 0.0
33
+ @queries = 0
34
+ @serialize_ms = 0.0
35
+ @blocked = nil
36
+ @fault = nil
37
+ @changes = []
38
+ @changes_truncated = false
39
+ @store_state = {}.compare_by_identity
40
+ end
41
+
42
+ # Per-preview state a data-store hook keeps on the recorder, keyed by the hook itself, so it lives
43
+ # exactly as long as the preview and a late write can still find it.
44
+ # @return [Object, nil]
45
+ def store_state(owner) = @mutex.synchronize { @store_state[owner] }
46
+
47
+ # @see #store_state
48
+ def set_store_state(owner, value)
49
+ @mutex.synchronize { @store_state[owner] = value }
50
+ end
51
+
52
+ def rollback? = @rollback
53
+
54
+ def include_sql_values? = @include_sql_values
55
+
56
+ def active?
57
+ @mutex.synchronize { @active }
58
+ end
59
+
60
+ # Records an effect. Returns true when the host must suppress delivery (preview only).
61
+ def record_effect(kind, description, data = nil)
62
+ @mutex.synchronize do
63
+ return false unless @active
64
+
65
+ if @effects.length >= Limits::EFFECTS || @effect_size >= Limits::LOG_BUDGET
66
+ @effects_truncated = true
67
+ return @rollback
68
+ end
69
+
70
+ started = Clock.now
71
+ captured = data.nil? ? nil : Limits.snapshot(data)
72
+ @serialize_ms += Clock.elapsed_ms(started)
73
+ kind_text = Limits.clip(kind, Limits::NAME)
74
+ description_text = Limits.clip(description)
75
+ @effect_size += kind_text.length + description_text.length + Limits.size(captured)
76
+ @effects_truncated ||= kind_text != kind.to_s || description_text != description.to_s ||
77
+ (captured.nil? && !data.nil?)
78
+ if @effect_size <= Limits::LOG_BUDGET
79
+ effect = { kind: kind_text, description: description_text, data: captured }
80
+ effect[:delivered] = true unless @rollback
81
+ @effects << effect
82
+ else
83
+ @effects_truncated = true
84
+ end
85
+ @rollback
86
+ end
87
+ end
88
+
89
+ # Keeps the first refused write. Only a preview records one; the first block wins.
90
+ def block(store:, operation:, detail:)
91
+ return unless @rollback
92
+
93
+ @mutex.synchronize do
94
+ @blocked ||= BlockedWrite.new(Limits.clip(store, Limits::NAME), Limits.clip(operation, Limits::NAME),
95
+ Limits.clip(detail))
96
+ end
97
+ end
98
+
99
+ # Keeps the first fault: a refused transaction-control command, an escaped write or a timeout.
100
+ # Only a preview records one; it answers 500 without confirmation, whatever the handler returns.
101
+ def refuse(error)
102
+ return unless @rollback
103
+
104
+ @mutex.synchronize { @fault ||= error }
105
+ end
106
+
107
+ # Records a row, document, item or key change made through a data-store hook. Secret-named
108
+ # fields never leave the process, at any depth of a nested document.
109
+ #
110
+ # @param change [Hash] +:operation+, +:table+, +:key+, +:before+, +:after+ and optionally
111
+ # +:store+, +:namespace+ and +:via+
112
+ def change(change)
113
+ @mutex.synchronize do
114
+ return unless @active
115
+
116
+ if @changes.length > Limits::CHANGES
117
+ @changes_truncated = true
118
+ else
119
+ @changes << change.merge(key: Redaction.deep(change[:key]), before: Redaction.deep(change[:before]),
120
+ after: Redaction.deep(change[:after]))
121
+ end
122
+ end
123
+ end
124
+
125
+ # Marks the changes as cut short, for example when a hook stopped reading candidates.
126
+ def truncate_changes
127
+ @mutex.synchronize { @changes_truncated = true }
128
+ end
129
+
130
+ # Records one data-store command as a +Db+ log entry. The message is the OpenTelemetry span name,
131
+ # +{operation} {target}+. Hooks pass sanitized statements only; values never reach this method.
132
+ #
133
+ # @param system [String] the store id
134
+ # @param duration_ms [Float, nil]
135
+ # @param blocked [Boolean] true for the entry of a command the guard refused
136
+ def db(system:, operation:, target: nil, namespace: nil, statement: nil, rows_affected: nil, blocked: false,
137
+ error: nil, duration_ms: nil)
138
+ @mutex.synchronize do
139
+ return unless @active
140
+
141
+ unless blocked
142
+ @queries += 1
143
+ @db_ms += duration_ms.to_f
144
+ end
145
+ return unless log_room?
146
+
147
+ db = { system: clip(system, Limits::NAME), operation: clip(operation, Limits::NAME) }
148
+ db[:target] = clip(target, Limits::NAME) if target
149
+ db[:namespace] = clip(namespace, Limits::NAME) if namespace
150
+ db[:statement] = clip(statement) if statement
151
+ db[:rowsAffected] = rows_affected if rows_affected.is_a?(Integer) && rows_affected >= 0
152
+ db[:blocked] = true if blocked
153
+ db[:error] = clip(error, Limits::NAME) if error
154
+ entry = { level: "Db", message: db[:target] ? "#{db[:operation]} #{db[:target]}" : db[:operation], db: db }
155
+ if duration_ms
156
+ entry[:elapsedMs] = [0.0, duration_ms].max
157
+ entry[:offsetMs] = [0.0, offset_ms - duration_ms].max
158
+ else
159
+ entry[:offsetMs] = offset_ms
160
+ end
161
+ add_log(entry)
162
+ end
163
+ end
164
+
165
+ # Records a plain application log message. Returns false when the recorder is finished.
166
+ def record_log(level, message)
167
+ log(level: level, message: message)
168
+ end
169
+
170
+ # Records a structured log entry.
171
+ # @param properties [Hash, nil] string-valued properties
172
+ # @param exception [Exception, nil]
173
+ # @param event_id [Hash, nil] +{id:, name:}+
174
+ def log(level:, message:, category: nil, event_id: nil, properties: nil, exception: nil)
175
+ @mutex.synchronize do
176
+ return false unless @active
177
+ return true unless log_room?
178
+
179
+ entry = { level: clip(level, Limits::NAME), message: clip(message) }
180
+ entry[:category] = clip(category, Limits::NAME) if category
181
+ entry[:eventId] = event_id if event_id
182
+ entry[:offsetMs] = offset_ms
183
+ if properties && !properties.empty?
184
+ entry[:properties] = capture_properties(properties)
185
+ end
186
+ if exception
187
+ entry[:exception] = { type: clip(exception.class.name || "Exception", Limits::NAME),
188
+ message: clip(exception.message) }
189
+ end
190
+ add_log(entry)
191
+ true
192
+ end
193
+ rescue StandardError
194
+ @mutex.synchronize { @logs_truncated = true }
195
+ true
196
+ end
197
+
198
+ # Records one SQL command. The block returns +[sql_hash, truncated]+ and runs only when there is log room.
199
+ # @param duration_ms [Float]
200
+ def record_sql(duration_ms:, category: nil, event_id: nil)
201
+ @mutex.synchronize do
202
+ return unless @active
203
+
204
+ @db_ms += duration_ms
205
+ @queries += 1
206
+ return unless log_room?
207
+
208
+ begin
209
+ sql, truncated = yield
210
+ @logs_truncated ||= truncated
211
+ entry = { level: "Sql", message: sql[:text] }
212
+ entry[:category] = category if category
213
+ entry[:eventId] = event_id if event_id
214
+ entry[:offsetMs] = [0.0, offset_ms - duration_ms].max
215
+ entry[:elapsedMs] = duration_ms
216
+ entry[:sql] = sql
217
+ add_log(entry)
218
+ rescue StandardError
219
+ @logs_truncated = true
220
+ end
221
+ end
222
+ end
223
+
224
+ # @return [Hash] auth, handler, db, queryCount, serialize
225
+ def timing(clock, extra_serialize_ms = 0.0)
226
+ @mutex.synchronize { clock.read(@db_ms, @queries, @serialize_ms, extra_serialize_ms) }
227
+ end
228
+
229
+ # Stops recording and returns what was captured.
230
+ # @return [Snapshot]
231
+ def finish
232
+ @mutex.synchronize do
233
+ @active = false
234
+ snapshot = Snapshot.new(effects: @effects.dup, logs: @logs.dup,
235
+ logs_truncated: @logs_truncated, effects_truncated: @effects_truncated,
236
+ changes: @changes.dup, changes_truncated: @changes_truncated)
237
+ @effects.clear
238
+ @logs.clear
239
+ @changes.clear
240
+ snapshot
241
+ end
242
+ end
243
+
244
+ private
245
+
246
+ def offset_ms = Clock.elapsed_ms(@started)
247
+
248
+ def clip(value, maximum = Limits::TEXT)
249
+ text = value.to_s
250
+ result = Limits.clip(text, maximum)
251
+ @logs_truncated ||= result.length != text.length
252
+ result
253
+ end
254
+
255
+ def log_room?
256
+ room = @logs.length < Limits::LOGS && @log_size < Limits::LOG_BUDGET
257
+ @logs_truncated ||= !room
258
+ room
259
+ end
260
+
261
+ def add_log(entry)
262
+ size = entry_size(entry)
263
+ if @log_size + size <= Limits::LOG_BUDGET
264
+ @logs << entry
265
+ @log_size += size
266
+ else
267
+ @logs_truncated = true
268
+ end
269
+ end
270
+
271
+ def entry_size(entry)
272
+ size = entry[:level].length + entry[:message].length + entry[:category].to_s.length
273
+ size += entry[:eventId][:name].to_s.length if entry[:eventId]
274
+ entry[:properties]&.each { |name, value| size += name.length + value.length }
275
+ size += entry[:exception][:type].length + entry[:exception][:message].length if entry[:exception]
276
+ if (db = entry[:db])
277
+ size += %i[system operation target namespace statement error].sum { |name| db[name].to_s.length }
278
+ end
279
+ if (sql = entry[:sql])
280
+ size += sql[:text].length
281
+ sql[:parameters].each { |p| size += p[:name].length + p[:value].to_s.length + p[:dbType].length }
282
+ end
283
+ size
284
+ end
285
+
286
+ def capture_properties(properties)
287
+ result = {}
288
+ properties.each do |name, value|
289
+ if result.length >= Limits::PROPERTIES
290
+ @logs_truncated = true
291
+ break
292
+ end
293
+ result[clip(name, Limits::NAME)] = clip(value)
294
+ end
295
+ result
296
+ end
297
+ end
298
+ end
@@ -0,0 +1,98 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Sendproof
4
+ # Decides whether SQL text or a parameter name refers to a secret.
5
+ #
6
+ # Matching uses whole words, so +IdempotencyKey+ or +monkey+ stay visible while
7
+ # +password_hash+, +api_key+ and +RefreshToken+ are hidden.
8
+ module Redaction
9
+ REDACTED = "[REDACTED]"
10
+
11
+ SECRET_WORDS = %w[password passwd pwd secret token hash salt credential credentials apikey].freeze
12
+ # "key" alone is ordinary (primary key, idempotency key); it is secret only after these words.
13
+ KEY_QUALIFIERS = %w[api private secret access signing encryption client].freeze
14
+
15
+ module_function
16
+
17
+ # @return [Boolean] whether any word in +text+ names a secret
18
+ def looks_secret?(text)
19
+ previous = nil
20
+ words(text.to_s).any? do |word|
21
+ secret = SECRET_WORDS.include?(word) || (word == "key" && KEY_QUALIFIERS.include?(previous))
22
+ previous = word
23
+ secret
24
+ end
25
+ end
26
+
27
+ # Replaces the value of every secret-named field with {REDACTED}, at any depth of a nested
28
+ # document or array. A secret-named field hides its whole value, nested or not.
29
+ #
30
+ # @return [Object] a copy; scalars come back as given
31
+ def deep(value)
32
+ case value
33
+ when Hash then value.to_h { |name, inner| [name, looks_secret?(name) ? REDACTED : deep(inner)] }
34
+ when Array then value.map { |item| deep(item) }
35
+ else value
36
+ end
37
+ end
38
+
39
+ # Replaces the literals in SQL text with +?+: single-quoted strings (with +''+ and backslash escapes),
40
+ # hexadecimal and numeric literals. Quoted identifiers (backticks and double quotes) and names with
41
+ # digits in them are kept. A database that doesn't use prepared statements (MySQL through Trilogy or
42
+ # mysql2) inlines parameter values into the statement text, so hiding values means masking them here too.
43
+ #
44
+ # @return [String]
45
+ def mask_literals(sql)
46
+ text = sql.to_s
47
+ output = +""
48
+ index = 0
49
+ while index < text.length
50
+ char = text[index]
51
+ if char == "'" || (char.match?(/[xX]/) && text[index + 1] == "'" && !word_before?(text, index))
52
+ stop = closing_quote(text, index + (char == "'" ? 0 : 1))
53
+ output << "?"
54
+ index = stop
55
+ elsif char == "`" || char == '"'
56
+ close = text.index(char, index + 1)
57
+ stop = close ? close + 1 : text.length
58
+ output << text[index...stop]
59
+ index = stop
60
+ elsif char.match?(/[0-9]/) && !word_before?(text, index)
61
+ number = text[index..][/\A(?:0[xX]\h+|[0-9]+(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?)/]
62
+ output << "?"
63
+ index += number.length
64
+ else
65
+ output << char
66
+ index += 1
67
+ end
68
+ end
69
+ output
70
+ end
71
+
72
+ # @return [Integer] the index after the single-quoted string that starts at +start+
73
+ def closing_quote(text, start)
74
+ index = start + 1
75
+ while index < text.length
76
+ if text[index] == "\\"
77
+ index += 2
78
+ elsif text[index] == "'" && text[index + 1] == "'"
79
+ index += 2
80
+ elsif text[index] == "'"
81
+ return index + 1
82
+ else
83
+ index += 1
84
+ end
85
+ end
86
+ text.length
87
+ end
88
+
89
+ def word_before?(text, index) = index.positive? && text[index - 1].match?(/[[:alnum:]_$.@]/)
90
+
91
+ # Splits at punctuation, underscores, digits-to-letters and camelCase boundaries; lowercases.
92
+ def words(text)
93
+ text.scan(/[[:alnum:]]+/).flat_map do |chunk|
94
+ chunk.gsub(/(?<=[[:lower:]])(?=[[:upper:]])/, " ").split
95
+ end.map(&:downcase)
96
+ end
97
+ end
98
+ end
@@ -0,0 +1,76 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "set"
4
+
5
+ module Sendproof
6
+ module Redis
7
+ # What a +Db+ log entry shows of a Redis command: its name, its key arguments as written, and
8
+ # every other argument as +?+ (+SET session:42 ?+). Values never appear. A command this table
9
+ # doesn't know shows no keys, so an unknown command can't leak an argument.
10
+ module Commands
11
+ # Commands whose first argument is the key (or the index or channel it works on).
12
+ FIRST = %w[
13
+ GET SET SETEX PSETEX SETNX GETSET GETDEL GETEX APPEND STRLEN INCR DECR INCRBY DECRBY INCRBYFLOAT SETRANGE GETRANGE
14
+ SUBSTR SETBIT GETBIT BITCOUNT BITPOS BITFIELD BITFIELD_RO LCS EXPIRE PEXPIRE EXPIREAT PEXPIREAT EXPIRETIME PEXPIRETIME
15
+ TTL PTTL PERSIST TYPE DUMP RESTORE SORT SORT_RO
16
+ HGET HSET HMGET HMSET HGETALL HDEL HEXISTS HINCRBY HINCRBYFLOAT HKEYS HVALS HLEN HSETNX HSTRLEN HRANDFIELD HSCAN
17
+ HEXPIRE HPEXPIRE HTTL HPTTL HPERSIST
18
+ LPUSH RPUSH LPOP RPOP LLEN LRANGE LINDEX LSET LTRIM LREM LINSERT LPUSHX RPUSHX LPOS
19
+ SADD SREM SMEMBERS SISMEMBER SMISMEMBER SCARD SPOP SRANDMEMBER SSCAN
20
+ ZADD ZREM ZRANGE ZRANGEBYSCORE ZREVRANGE ZREVRANGEBYSCORE ZCARD ZSCORE ZMSCORE ZRANK ZREVRANK ZINCRBY ZCOUNT ZLEXCOUNT
21
+ ZRANGEBYLEX ZREVRANGEBYLEX ZREMRANGEBYRANK ZREMRANGEBYSCORE ZREMRANGEBYLEX ZPOPMIN ZPOPMAX ZSCAN ZRANDMEMBER
22
+ XADD XLEN XRANGE XREVRANGE XDEL XTRIM XACK XAUTOCLAIM XCLAIM XPENDING XGROUP
23
+ PFADD GEOADD GEOPOS GEODIST GEOHASH GEOSEARCH GEORADIUS GEORADIUS_RO GEORADIUSBYMEMBER GEORADIUSBYMEMBER_RO
24
+ PUBLISH SPUBLISH
25
+ JSON.GET JSON.SET JSON.DEL JSON.MGET JSON.TYPE JSON.NUMINCRBY JSON.STRAPPEND JSON.STRLEN JSON.ARRAPPEND JSON.ARRLEN
26
+ JSON.OBJKEYS JSON.OBJLEN JSON.ARRINDEX JSON.RESP JSON.TOGGLE JSON.CLEAR
27
+ TS.ADD TS.GET TS.RANGE TS.REVRANGE TS.CREATE TS.INCRBY TS.DECRBY TS.INFO
28
+ FT.SEARCH FT.AGGREGATE FT.INFO FT.EXPLAIN
29
+ ].to_set.freeze
30
+
31
+ # Commands where every argument is a key.
32
+ ALL = %w[DEL EXISTS UNLINK TOUCH MGET WATCH PFCOUNT SINTER SUNION SDIFF SINTERSTORE SUNIONSTORE SDIFFSTORE PFMERGE
33
+ RENAME RENAMENX COPY LMOVE SMOVE RPOPLPUSH SUNIONSTORE].to_set.freeze
34
+
35
+ # Commands that alternate key and value.
36
+ ALTERNATE = %w[MSET MSETNX].to_set.freeze
37
+
38
+ # Commands whose every argument but the last (a timeout) is a key.
39
+ ALL_BUT_LAST = %w[BLPOP BRPOP BLMOVE BRPOPLPUSH].to_set.freeze
40
+
41
+ # Container commands whose key follows the subcommand: +OBJECT ENCODING key+.
42
+ AFTER_SUBCOMMAND = %w[OBJECT XINFO MEMORY].to_set.freeze
43
+
44
+ # Script commands: +EVAL script numkeys key ... arg ...+.
45
+ SCRIPTS = %w[EVAL EVALSHA EVAL_RO EVALSHA_RO FCALL FCALL_RO].to_set.freeze
46
+
47
+ module_function
48
+
49
+ # @param name [String] the command name
50
+ # @param args [Array<String>] its arguments
51
+ # @return [Array<Integer>] the indexes of +args+ that are keys
52
+ def key_indexes(name, args)
53
+ command = name.to_s.upcase
54
+ if FIRST.include?(command) then args.empty? ? [] : [0]
55
+ elsif ALL.include?(command) then (0...args.length).to_a
56
+ elsif ALTERNATE.include?(command) then (0...args.length).step(2).to_a
57
+ elsif ALL_BUT_LAST.include?(command) then (0...[args.length - 1, 0].max).to_a
58
+ elsif AFTER_SUBCOMMAND.include?(command) then args.length > 1 ? [1] : []
59
+ elsif SCRIPTS.include?(command)
60
+ count = args[1].to_s.match?(/\A\d+\z/) ? args[1].to_i : 0
61
+ (2...[2 + count, args.length].min).to_a
62
+ else []
63
+ end
64
+ end
65
+
66
+ # @return [Hash] +:statement+ and, when the command has a key, +:target+
67
+ def describe(name, args)
68
+ keys = key_indexes(name, args)
69
+ shown = args.each_with_index.map { |arg, index| keys.include?(index) ? arg.to_s : "?" }
70
+ description = { statement: [name.to_s.upcase, *shown].join(" ") }
71
+ description[:target] = args[keys.first].to_s unless keys.empty?
72
+ description
73
+ end
74
+ end
75
+ end
76
+ end
@@ -0,0 +1,205 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Sendproof
4
+ module Redis
5
+ # The Redis data store: a +redis-client+ middleware that classifies every command, pipeline and
6
+ # +MULTI+ before any byte of it is written, refuses writes during a Preview, and logs each command as
7
+ # a +Db+ entry.
8
+ #
9
+ # Reads run normally. In a Preview the first write raises {Sendproof::PreviewWriteBlocked} and the
10
+ # Preview answers 409. Session commands (+AUTH+, +HELLO+, +SELECT+, +PING+ ...) are never guarded or
11
+ # logged, so the password never reaches a capture.
12
+ #
13
+ # *Coverage.* +RedisClient.register+ is process-wide, so by default every +redis-client+ (and +redis+
14
+ # 5) client in the process is guarded, including ones created later. Pass +global: false+ and add
15
+ # {#middleware} to a single client's +middlewares:+ to cover only that client. A +pubsub+ connection
16
+ # writes straight to its socket, past the middleware, so its +call+ and +call_v+ are guarded too, for
17
+ # the clients the store covers. Not covered: clients of other libraries and raw sockets. Sidekiq
18
+ # enqueues are writes, so a Preview that enqueues a job is blocked: record the job as an effect
19
+ # with +Sendproof.record_job+ instead.
20
+ #
21
+ # @example
22
+ # config.sendproof.data_store Sendproof::Redis::Store.new(name: "cache")
23
+ class Store < Sendproof::DataStore
24
+ # @param name [String, nil] display label for discovery, such as +cache+; never a URL or host
25
+ # @param treat_as_read [Array<String>, nil] commands you state never write, such as a module read
26
+ # +MYMODULE.GET+: exact upper-case names (or +CONTAINER|SUB+), no wildcards. A wrongly listed
27
+ # write is not blocked.
28
+ # @param global [Boolean] register the middleware for every client in the process (default)
29
+ def initialize(name: nil, treat_as_read: nil, global: true)
30
+ super()
31
+ @name = Stores::Guard.store_label(name)
32
+ @reads = Stores::Guard.read_set(treat_as_read) { |command| command.to_s.upcase }
33
+ @active = true
34
+ @middleware = build_middleware
35
+ PubSubGuard.install!
36
+ self.class.stores << self
37
+ return unless global
38
+
39
+ # A second hook would log every command twice and guard it twice.
40
+ raise ConfigurationError, "A Sendproof::Redis::Store is already registered; call unhook! on it first." if self.class.registered
41
+
42
+ ::RedisClient.register(@middleware)
43
+ self.class.registered = self
44
+ end
45
+
46
+ class << self
47
+ # @return [Store, nil] the store whose middleware is registered process-wide
48
+ attr_accessor :registered
49
+
50
+ # @return [Array<Store>] every store still guarding, global or per client
51
+ def stores = @stores ||= []
52
+
53
+ # Guards one command a +pubsub+ connection is about to write, with the store covering the client
54
+ # the connection came from (+middlewares+ is that client's middleware stack).
55
+ def guard_pubsub(middlewares, command, &)
56
+ store = middlewares && stores.find { |candidate| candidate.active? && middlewares.is_a?(candidate.middleware) }
57
+ store ? store.around([command], &) : yield
58
+ end
59
+ end
60
+
61
+ # Guards the commands of +pubsub+ connections, which redis-client writes to the socket directly.
62
+ module PubSubGuard
63
+ # Remembers which client's middleware stack a +pubsub+ connection came from.
64
+ module Client
65
+ def pubsub
66
+ middlewares = @middlewares
67
+ super.tap { |connection| connection.instance_variable_set(:@sendproof_middlewares, middlewares) }
68
+ end
69
+ end
70
+
71
+ # Classifies each command before it is written.
72
+ module Connection
73
+ def call(*command, **kwargs)
74
+ Store.guard_pubsub(@sendproof_middlewares, @command_builder.generate(command, kwargs)) { super }
75
+ end
76
+
77
+ def call_v(command)
78
+ Store.guard_pubsub(@sendproof_middlewares, @command_builder.generate(command)) { super }
79
+ end
80
+ end
81
+
82
+ # Prepends both modules once.
83
+ def self.install!
84
+ return if @installed
85
+
86
+ @installed = true
87
+ ::RedisClient.prepend(Client)
88
+ ::RedisClient::PubSub.prepend(Connection)
89
+ end
90
+ end
91
+
92
+ # @return [Module] the +redis-client+ middleware, for +RedisClient.config(middlewares: [...])+
93
+ attr_reader :middleware
94
+
95
+ def system = "redis"
96
+
97
+ def describe
98
+ { system: "redis", name: @name, mode: :guard }
99
+ end
100
+
101
+ # Turns the hook off. The middleware stays registered in +redis-client+ (a module can't be
102
+ # removed from a class) but does nothing, and another store may then register.
103
+ def unhook!
104
+ @active = false
105
+ self.class.stores.delete(self)
106
+ self.class.registered = nil if self.class.registered.equal?(self)
107
+ end
108
+
109
+ # @return [Boolean] whether the hook is still guarding
110
+ def active? = @active
111
+
112
+ # Classifies +commands+ (arrays of strings), guards them, runs the block and logs each command.
113
+ # The middleware calls this; it is public so a batch can be checked without a server.
114
+ #
115
+ # @param commands [Array<Array<String>>]
116
+ # @return the block's result
117
+ def around(commands)
118
+ return yield unless @active
119
+
120
+ entries = commands.map { |command| [command.first.to_s.upcase, command.drop(1)] }
121
+ kinds = entries.map { |name, args| Stores::Classify.redis(name, args, @reads) }
122
+ # Classify the whole batch before the first command is written: refusing one command mid-pipeline
123
+ # corrupts reply matching, and a queued write must never be followed by EXEC.
124
+ entries.zip(kinds) { |(name, args), kind| Stores::Guard.guard("redis", operation(name, args), kind) }
125
+ recorder = Capture.current
126
+ # Session commands (AUTH, SELECT ...) and MULTI/EXEC are never logged.
127
+ indexes = recorder ? entries.each_index.reject { |index| %i[session transaction].include?(kinds[index]) } : []
128
+ return yield if indexes.empty?
129
+
130
+ started = Clock.now
131
+ begin
132
+ result = yield
133
+ rescue StandardError => error
134
+ log(recorder, entries.values_at(*indexes), started, [error] * indexes.length)
135
+ raise
136
+ end
137
+ log(recorder, entries.values_at(*indexes), started, errors_in(result, entries.length, indexes))
138
+ result
139
+ end
140
+
141
+ private
142
+
143
+ # The operation name the guard reports: the command, or +CONTAINER|SUB+ for a container command.
144
+ def operation(name, _args) = name
145
+
146
+ def build_middleware
147
+ store = self
148
+ Module.new do
149
+ define_method(:call) do |command, config, &block|
150
+ store.send(:single, self, command) { super(command, config, &block) }
151
+ end
152
+ define_method(:call_pipelined) do |commands, config, &block|
153
+ store.around(commands) { super(commands, config, &block) }
154
+ end
155
+ end
156
+ end
157
+
158
+ # One command sent on its own. A write refused after +MULTI+ discards the transaction, so the
159
+ # connection is never left with a queue that a later +EXEC+ could run.
160
+ def single(middlewares, command)
161
+ name = command.first.to_s.upcase
162
+ begin
163
+ result = around([command]) { yield }
164
+ rescue Sendproof::PreviewWriteBlocked
165
+ discard(middlewares) if middlewares.instance_variable_get(:@sendproof_multi)
166
+ raise
167
+ end
168
+ if @active
169
+ multi = middlewares.instance_variable_get(:@sendproof_multi)
170
+ middlewares.instance_variable_set(:@sendproof_multi, true) if name == "MULTI"
171
+ middlewares.instance_variable_set(:@sendproof_multi, false) if multi && %w[EXEC DISCARD].include?(name)
172
+ end
173
+ result
174
+ end
175
+
176
+ def discard(middlewares)
177
+ middlewares.instance_variable_set(:@sendproof_multi, false)
178
+ middlewares.client.call("DISCARD")
179
+ rescue StandardError
180
+ middlewares.client.close
181
+ end
182
+
183
+ # The command errors a pipeline's replies carry (it is read with +exception: false+ inside multi).
184
+ def errors_in(result, count, indexes)
185
+ return Array.new(indexes.length) unless result.is_a?(Array) && result.length == count
186
+
187
+ indexes.map { |index| result[index].is_a?(StandardError) ? result[index] : nil }
188
+ end
189
+
190
+ def log(recorder, loggable, started, errors)
191
+ elapsed = Clock.elapsed_ms(started) / loggable.length
192
+ loggable.each_with_index do |(name, args), index|
193
+ description = Commands.describe(name, args)
194
+ error = errors[index]
195
+ recorder.db(system: "redis", operation: name, target: description[:target], statement: description[:statement],
196
+ error: error && error_code(error), duration_ms: elapsed)
197
+ end
198
+ end
199
+
200
+ def error_code(error)
201
+ error.message.to_s.split.first.to_s[/\A[A-Z_]+\z/] || error.class.name.to_s.split("::").last
202
+ end
203
+ end
204
+ end
205
+ end
@@ -0,0 +1,18 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "redis-client"
4
+ require "sendproof"
5
+
6
+ module Sendproof
7
+ # Sendproof write guard for Redis (spec/data-stores.md, +guard+ mode), for +redis-client+ and the
8
+ # +redis+ 5 gem built on it, including Rails' +RedisCacheStore+.
9
+ #
10
+ # store = Sendproof::Redis::Store.new(name: "cache") # registers redis-client middleware
11
+ # config.sendproof.data_store store
12
+ #
13
+ # Require it explicitly: +require "sendproof/redis"+. It needs no Redis server.
14
+ module Redis
15
+ autoload :Commands, "sendproof/redis/commands"
16
+ autoload :Store, "sendproof/redis/store"
17
+ end
18
+ end