moderate 0.1.0 → 1.0.0.beta1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +8 -0
  3. data/.simplecov +62 -0
  4. data/AGENTS.md +7 -0
  5. data/Appraisals +16 -0
  6. data/CHANGELOG.md +71 -1
  7. data/CLAUDE.md +7 -0
  8. data/README.md +376 -29
  9. data/Rakefile +28 -2
  10. data/app/controllers/concerns/moderate/moderation.rb +161 -0
  11. data/app/controllers/moderate/appeals_controller.rb +190 -0
  12. data/app/controllers/moderate/application_controller.rb +45 -0
  13. data/app/controllers/moderate/notices_controller.rb +382 -0
  14. data/app/controllers/moderate/transparency_reports_controller.rb +30 -0
  15. data/app/helpers/moderate/engine_helper.rb +151 -0
  16. data/app/views/moderate/appeals/new.html.erb +78 -0
  17. data/app/views/moderate/notices/new.html.erb +255 -0
  18. data/app/views/moderate/transparency_reports/_summary_card.html.erb +20 -0
  19. data/app/views/moderate/transparency_reports/show.html.erb +52 -0
  20. data/config/moderate/blocklists/en.yml +81 -0
  21. data/config/moderate/blocklists/es.yml +40 -0
  22. data/config/routes.rb +36 -0
  23. data/docs/compliance.md +178 -0
  24. data/docs/configuration.md +326 -0
  25. data/docs/dsa-notice-form.md +371 -0
  26. data/docs/madmin.md +490 -0
  27. data/docs/notifications.md +363 -0
  28. data/examples/aws_rekognition_adapter.rb +140 -0
  29. data/examples/openai_moderation_adapter.rb +111 -0
  30. data/gemfiles/rails_7.1.gemfile +36 -0
  31. data/gemfiles/rails_7.2.gemfile +36 -0
  32. data/gemfiles/rails_8.1.gemfile +36 -0
  33. data/lib/generators/moderate/install_generator.rb +56 -0
  34. data/lib/generators/moderate/templates/create_moderate_tables.rb.erb +237 -0
  35. data/lib/generators/moderate/templates/initializer.rb +198 -0
  36. data/lib/generators/moderate/views_generator.rb +63 -0
  37. data/lib/moderate/configuration.rb +341 -0
  38. data/lib/moderate/engine.rb +138 -0
  39. data/lib/moderate/errors.rb +26 -0
  40. data/lib/moderate/event.rb +75 -0
  41. data/lib/moderate/filters/base.rb +126 -0
  42. data/lib/moderate/filters/wordlist.rb +255 -0
  43. data/lib/moderate/jobs/classify_job.rb +158 -0
  44. data/lib/moderate/label.rb +111 -0
  45. data/lib/moderate/macros.rb +90 -0
  46. data/lib/moderate/models/appeal.rb +154 -0
  47. data/lib/moderate/models/application_record.rb +31 -0
  48. data/lib/moderate/models/block.rb +203 -0
  49. data/lib/moderate/models/concerns/actor.rb +174 -0
  50. data/lib/moderate/models/concerns/content_filterable.rb +155 -0
  51. data/lib/moderate/models/concerns/reportable.rb +282 -0
  52. data/lib/moderate/models/flag.rb +136 -0
  53. data/lib/moderate/models/report.rb +620 -0
  54. data/lib/moderate/result.rb +176 -0
  55. data/lib/moderate/services/intake_appeal.rb +89 -0
  56. data/lib/moderate/services/intake_notice.rb +132 -0
  57. data/lib/moderate/services/intake_report.rb +132 -0
  58. data/lib/moderate/services/resolve_appeal.rb +134 -0
  59. data/lib/moderate/services/resolve_flag.rb +101 -0
  60. data/lib/moderate/services/resolve_report.rb +291 -0
  61. data/lib/moderate/version.rb +1 -1
  62. data/lib/moderate.rb +365 -18
  63. data/log/development.log +0 -0
  64. data/log/test.log +0 -0
  65. metadata +154 -15
@@ -0,0 +1,101 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Moderate
4
+ module Services
5
+ # ResolveFlag — the decision on an auto-filter `Moderate::Flag`, behind the
6
+ # flag's `action!` / `dismiss!`.
7
+ #
8
+ # A Flag is a SYSTEM-raised queue item (the wordlist/image/external classifier
9
+ # tripped on a `:flag`-mode field, or a human filed one manually) — it's the
10
+ # "ongoing moderation" surface Apple Guideline 1.2 and Google Play UGC expect:
11
+ # - https://developer.apple.com/app-store/review/guidelines/#user-generated-content
12
+ # - https://support.google.com/googleplay/android-developer/answer/9876937
13
+ #
14
+ # A flag is lighter than a report: there's no reporter to inform and no appeal
15
+ # window to stamp (an appeal attaches to a *report* decision, not to a raw
16
+ # queue item). So this service is the simplest of the resolvers — atomic
17
+ # transition, mandatory note, audit — but it follows the same discipline:
18
+ # - `with_lock` + re-check `pending?` inside the lock for idempotency under
19
+ # concurrent review;
20
+ # - a NOTE IS MANDATORY (the moderator's rationale is the audit trail);
21
+ # - the decision is recorded via `Moderate.audit`, never a host-specific log.
22
+ #
23
+ # NOTE: unlike ResolveReport, resolving a flag does NOT itself run content
24
+ # removal/bans. A flag marks "this needs a look"; the enforcement decision is a
25
+ # *report* concern. If a moderator wants to remove the flagged content, they file
26
+ # /resolve a report on it. This keeps the flag a pure triage record and avoids
27
+ # two divergent enforcement paths. (Documented so a maintainer doesn't "helpfully"
28
+ # add removal here and create that divergence.)
29
+ class ResolveFlag
30
+ def initialize(flag, by:)
31
+ @flag = flag
32
+ @moderator = by
33
+ end
34
+
35
+ # The flagged content was indeed objectionable — mark the flag actioned. (Any
36
+ # actual takedown is done by acting on a report; see the class note.)
37
+ def action!(note:)
38
+ transition!("actioned", note)
39
+ end
40
+
41
+ # False positive / acceptable — dismiss the flag.
42
+ def dismiss!(note:)
43
+ transition!("dismissed", note)
44
+ end
45
+
46
+ private
47
+
48
+ attr_reader :flag, :moderator
49
+
50
+ def transition!(status, note)
51
+ note = require_note!(note)
52
+
53
+ flag.with_lock do
54
+ flag.reload
55
+
56
+ # Re-check inside the lock. A flag already reviewed is returned as-is (NOT
57
+ # an error): unlike a report, double-reviewing a flag is harmless and a
58
+ # benign "already done" is friendlier for a fast triage queue.
59
+ return flag unless flag.pending?
60
+
61
+ flag.update!(
62
+ status: status,
63
+ resolution_note: note,
64
+ reviewed_by: moderator,
65
+ reviewed_at: Time.now
66
+ )
67
+
68
+ audit_decision(status, note)
69
+ end
70
+
71
+ flag
72
+ end
73
+
74
+ def audit_decision(status, note)
75
+ Moderate.audit(
76
+ :flag_decision,
77
+ subject: flag,
78
+ actor: moderator,
79
+ payload: {
80
+ flag_id: flag.id,
81
+ flaggable_type: flag.flaggable_type,
82
+ flaggable_id: flag.flaggable_id,
83
+ field: flag.field,
84
+ source: flag.source,
85
+ categories: flag.categories,
86
+ note: note,
87
+ summary: "Flag ##{flag.id} #{status} by moderator"
88
+ }.compact
89
+ )
90
+ end
91
+
92
+ def require_note!(note)
93
+ normalized = note.to_s.strip
94
+ return normalized unless normalized.empty?
95
+
96
+ flag.errors.add(:resolution_note, :blank)
97
+ raise ActiveRecord::RecordInvalid, flag
98
+ end
99
+ end
100
+ end
101
+ end
@@ -0,0 +1,291 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Moderate
4
+ module Services
5
+ # ResolveReport — the audited decision engine behind `report.resolve!` /
6
+ # `report.dismiss!`.
7
+ #
8
+ # This is where a moderator's decision actually happens, and it's the most
9
+ # legally-loaded path in the gem, so it's built defensively:
10
+ #
11
+ # 1. ATOMIC + IDEMPOTENT. The whole transition runs inside `report.with_lock`
12
+ # (a SELECT ... FOR UPDATE row lock). Two moderators clicking "resolve" at
13
+ # once must not both run enforcement (double-ban, double-remove) — the lock
14
+ # serializes them, and we RE-CHECK `open?` *inside* the lock after reload so
15
+ # the second one sees the closed record and bails with a clean error rather
16
+ # than re-applying actions.
17
+ #
18
+ # 2. A NOTE IS MANDATORY. DSA Art. 17 requires a "clear and specific statement
19
+ # of reasons"; the moderator's note is the human-readable ground. No note,
20
+ # no decision — we raise RecordInvalid with the note error attached.
21
+ # See: https://eur-lex.europa.eu/eli/reg/2022/2065/oj (Article 17).
22
+ #
23
+ # 3. ENFORCEMENT IS HOST-AGNOSTIC. Content removal calls the reportable's own
24
+ # `remove_reported_field!` (the host decides what "remove" means for its
25
+ # content); a ban calls `Moderate.apply_ban` → the host's `ban_handler`
26
+ # (the host decides what "banned" means). The gem never reaches into a
27
+ # host's domain — it only invokes the contracts.
28
+ #
29
+ # 4. TWO DECISION EVENTS, ON PURPOSE. `report_decision` tells the *reporter*
30
+ # "we handled it" (Art. 16(5)); `affected_user_decision` gives the *content
31
+ # owner* the Art. 17 statement of reasons (action taken, ground, automated-
32
+ # means disclosure, appeal path). Different people, different rights — see
33
+ # docs/notifications.md ("Why two decision events").
34
+ class ResolveReport
35
+ def initialize(report, by:)
36
+ @report = report
37
+ @moderator = by
38
+ end
39
+
40
+ # Resolve WITH action (the report was valid; we acted on the content/account).
41
+ # `resolution_basis` records the legal/contractual ground bucket; it's
42
+ # validated against the migration's check-constraint list by the model.
43
+ def resolve!(note:, remove_content: false, ban_user: false, resolution_basis: "terms")
44
+ note = require_note!(note)
45
+ actions = {
46
+ remove_content: truthy?(remove_content),
47
+ ban_user: truthy?(ban_user),
48
+ resolution_basis: resolution_basis.to_s.strip.presence || "terms"
49
+ }
50
+
51
+ transition!("actioned", note: note, actions: actions) do
52
+ remove_reported_content! if actions[:remove_content]
53
+ ban_reported_user!(note: note) if actions[:ban_user]
54
+ end
55
+ end
56
+
57
+ # Dismiss (no violation found). Still requires a note (Art. 17 applies to the
58
+ # reporter's right to know the outcome too) and still opens an appeal window —
59
+ # the *reporter* can appeal a dismissal.
60
+ def dismiss!(note:)
61
+ note = require_note!(note)
62
+ transition!("dismissed", note: note, actions: { resolution_basis: "no_violation" })
63
+ end
64
+
65
+ private
66
+
67
+ attr_reader :report, :moderator
68
+
69
+ # The atomic core. Everything that mutates state happens under the row lock;
70
+ # the (slow, fallible) notifications happen AFTER the lock is released, so a
71
+ # broken mailer can't hold a database lock or roll back the decision.
72
+ def transition!(status, note:, actions:)
73
+ report.with_lock do
74
+ report.reload
75
+
76
+ # Re-check inside the lock — see class doc point (1). A closed report is a
77
+ # no-op error, never a silent re-apply of enforcement.
78
+ unless report.open?
79
+ report.errors.add(:base, already_closed_message)
80
+ raise ActiveRecord::RecordInvalid, report
81
+ end
82
+
83
+ # Enforcement runs INSIDE the transaction so that if removal/ban raises,
84
+ # the whole decision rolls back — we never record "actioned" while the
85
+ # content is still up.
86
+ yield if block_given?
87
+
88
+ report.update!(
89
+ status: status,
90
+ resolution_note: note,
91
+ resolution_actions: actions.transform_keys(&:to_s),
92
+ resolution_basis: actions.fetch(:resolution_basis, "no_violation"),
93
+ decision_visibility: decision_visibility_for(status, actions),
94
+ # Stamp the appeal window NOW (Art. 20: open ≥ 6 months from the
95
+ # decision). The model owns APPEAL_WINDOW so the duration is configured
96
+ # in one place.
97
+ appeal_deadline_at: Time.now + Moderate::Report::APPEAL_WINDOW,
98
+ resolved_by: moderator,
99
+ resolved_at: Time.now
100
+ )
101
+
102
+ audit_decision(status, actions, note)
103
+ end
104
+
105
+ deliver_decision_notices
106
+ report
107
+ end
108
+
109
+ # --- Enforcement (host contracts only) ----------------------------------
110
+
111
+ def remove_reported_content!
112
+ return if report.reportable.blank?
113
+ return unless report.reportable.respond_to?(:remove_reported_field!)
114
+
115
+ report.reportable.remove_reported_field!(report.reported_field)
116
+ end
117
+
118
+ def ban_reported_user!(note:)
119
+ user = report.reported_user
120
+ return if user.blank?
121
+
122
+ # The gem never bans directly — it asks the host's ban_handler what "banned"
123
+ # means (suspend, soft-delete, flip a flag…). No-op by default, so the
124
+ # decision still completes and audits even if no ban is wired.
125
+ Moderate.apply_ban(user: user, by: moderator, reason: note)
126
+ end
127
+
128
+ # --- Notifications (after the lock) -------------------------------------
129
+
130
+ def deliver_decision_notices
131
+ notify_reporter
132
+ notify_affected_user
133
+ end
134
+
135
+ # report_decision → the reporter (Art. 16(5): inform the notice provider of the
136
+ # decision + redress). We stamp `decision_notified_at` ONLY when the hook
137
+ # reported a delivery — that timestamp is the record that the legal
138
+ # communication actually went out, so it must reflect reality, not intent.
139
+ # `Moderate.notify` returns the delivered boolean precisely for this gate.
140
+ def notify_reporter
141
+ return if report.notifier_email.blank?
142
+
143
+ delivered = Moderate.notify(
144
+ :report_decision,
145
+ subject: report,
146
+ actor: moderator,
147
+ recipients: [report.reporter].compact,
148
+ payload: decision_payload.merge(
149
+ summary: "Decision on Report ##{report.id}: #{report.status}"
150
+ )
151
+ )
152
+
153
+ report.update_column(:decision_notified_at, Time.now) if delivered
154
+ end
155
+
156
+ # affected_user_decision → the content owner: the DSA Art. 17 statement of
157
+ # reasons. Only fires when we actually restricted something (an "actioned"
158
+ # decision) and there's an identifiable owner to tell. Carries the action, the
159
+ # ground, the automated-means disclosure, and the appeal entry point.
160
+ def notify_affected_user
161
+ return unless report.actioned?
162
+
163
+ affected = report.reported_user
164
+ return if affected.blank?
165
+ return unless affected.respond_to?(:email) && affected.email.present?
166
+
167
+ delivered = Moderate.notify(
168
+ :affected_user_decision,
169
+ subject: report,
170
+ actor: moderator,
171
+ recipients: [affected],
172
+ payload: decision_payload.merge(
173
+ summary: "Statement of reasons for Report ##{report.id}"
174
+ )
175
+ )
176
+
177
+ report.update_column(:affected_user_notified_at, Time.now) if delivered
178
+ end
179
+
180
+ # The shared statement-of-reasons payload (Art. 17(3)). Every field the law
181
+ # wants the affected user to receive is carried here so the host's mailer can
182
+ # render it without recomputing anything:
183
+ # - action: what was done (the restriction imposed + its scope)
184
+ # - ground: the legal (DSA legal_reason) or contractual (community category)
185
+ # basis — `resolution_basis` is the bucket, the moderator note is the prose
186
+ # - automated: whether automated means participated in detection/decision
187
+ # (Art. 17(3)(c)) — read off the report's recorded automated_processing
188
+ # - reason: the moderator's human-readable note
189
+ # The HOST renders the redress/appeal copy (it names the jurisdiction); the gem
190
+ # supplies the data + the report so the host can mint the signed appeal link.
191
+ def decision_payload
192
+ {
193
+ report_id: report.id,
194
+ status: report.status,
195
+ action: action_label,
196
+ resolution_basis: report.resolution_basis,
197
+ # The contractual ground (in-app reports) vs. the legal ground (DSA notices).
198
+ category: report.category,
199
+ legal_reason: report.legal_reason,
200
+ # Art. 17(3)(c) automated-means disclosure. The model captured this at
201
+ # intake (which filter/flag, if any, surfaced the content); we just relay
202
+ # the boolean so a decision email can never claim "No" after a classifier
203
+ # already participated.
204
+ automated: automated_processing_used?,
205
+ reason: report.resolution_note,
206
+ appeal_deadline_at: report.appeal_deadline_at
207
+ }.compact
208
+ end
209
+
210
+ def audit_decision(status, actions, note)
211
+ Moderate.audit(
212
+ :report_decision,
213
+ subject: report,
214
+ actor: moderator,
215
+ payload: {
216
+ report_id: report.id,
217
+ status: status,
218
+ reported_user_id: report.reported_user_id,
219
+ reportable_type: report.reportable_type,
220
+ reportable_id: report.reportable_id,
221
+ actions: actions,
222
+ resolution_basis: report.resolution_basis,
223
+ automated: automated_processing_used?,
224
+ appeal_deadline_at: report.appeal_deadline_at,
225
+ note: note,
226
+ summary: "Report ##{report.id} #{status} by moderator"
227
+ }.compact
228
+ )
229
+ end
230
+
231
+ # --- Helpers ------------------------------------------------------------
232
+
233
+ # A neutral, host-agnostic description of the restriction (Art. 17 "specific
234
+ # restriction imposed"). Deliberately NOT host vocabulary — "content removed"
235
+ # / "account suspended" are domain-neutral.
236
+ def action_label
237
+ actions = report.resolution_actions.to_h
238
+ return "account_suspended" if truthy?(actions["ban_user"])
239
+ return "content_removed" if truthy?(actions["remove_content"])
240
+ return "no_action" if report.dismissed?
241
+
242
+ "other_restriction"
243
+ end
244
+
245
+ # Records the *scope* of the restriction in the report's own column for the
246
+ # statement of reasons. Same neutral vocabulary as action_label.
247
+ def decision_visibility_for(status, actions)
248
+ return "no_restriction" unless status == "actioned"
249
+ return "account_suspended" if actions[:ban_user]
250
+ return "content_removed" if actions[:remove_content]
251
+
252
+ "other_restriction"
253
+ end
254
+
255
+ def automated_processing_used?
256
+ report.respond_to?(:automated_processing_used?) && report.automated_processing_used?
257
+ end
258
+
259
+ # Mandatory-note guard. We mutate the record's errors (not raise a bare string)
260
+ # so a controller's `rescue ActiveRecord::RecordInvalid` / `report.errors`
261
+ # flow works exactly like any failed save.
262
+ def require_note!(note)
263
+ normalized = note.to_s.strip
264
+ return normalized unless normalized.empty?
265
+
266
+ report.errors.add(:resolution_note, :blank)
267
+ raise ActiveRecord::RecordInvalid, report
268
+ end
269
+
270
+ def already_closed_message
271
+ # Fall back to a plain string if I18n isn't available (plain-Ruby contexts);
272
+ # the host can override the key in its locale files.
273
+ if defined?(I18n)
274
+ I18n.t("moderate.errors.report_already_closed", default: "This report has already been resolved.")
275
+ else
276
+ "This report has already been resolved."
277
+ end
278
+ end
279
+
280
+ # Boolean cast that works with form params ("1"/"true"/"0") and real booleans,
281
+ # without depending on ActiveModel being loaded in a plain-Ruby context.
282
+ def truthy?(value)
283
+ if defined?(ActiveModel::Type::Boolean)
284
+ ActiveModel::Type::Boolean.new.cast(value) || false
285
+ else
286
+ [true, "1", "true", "t", "yes", "y", 1].include?(value)
287
+ end
288
+ end
289
+ end
290
+ end
291
+ end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Moderate
4
- VERSION = "0.1.0"
4
+ VERSION = "1.0.0.beta1"
5
5
  end