moderate 1.0.0.beta1 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +96 -1
  3. data/.simplecov +2 -10
  4. data/AGENTS.md +1 -1
  5. data/Appraisals +10 -8
  6. data/CHANGELOG.md +660 -20
  7. data/CLAUDE.md +1 -1
  8. data/README.md +149 -7
  9. data/Rakefile +2 -0
  10. data/app/controllers/concerns/moderate/moderation.rb +23 -11
  11. data/app/controllers/moderate/appeals_controller.rb +9 -5
  12. data/app/controllers/moderate/application_controller.rb +3 -3
  13. data/app/controllers/moderate/notices_controller.rb +3 -6
  14. data/app/helpers/moderate/engine_helper.rb +32 -11
  15. data/app/views/moderate/appeals/new.html.erb +6 -0
  16. data/app/views/moderate/transparency_reports/_summary_card.html.erb +2 -2
  17. data/app/views/moderate/transparency_reports/show.html.erb +5 -5
  18. data/config/locales/en.yml +200 -0
  19. data/config/locales/es.yml +177 -0
  20. data/context7.json +4 -0
  21. data/db/migrate/20261018200000_add_pending_key_to_moderate_flags.rb +139 -0
  22. data/db/migrate/20261018210000_add_retention_and_classifications_to_moderate_tables.rb +103 -0
  23. data/docs/chats.md +140 -0
  24. data/docs/compliance.md +92 -3
  25. data/docs/configuration.md +30 -2
  26. data/docs/dsa-notice-form.md +1 -1
  27. data/docs/images/moderate-user-report-block-actions.webp +0 -0
  28. data/docs/madmin.md +44 -14
  29. data/docs/notifications.md +17 -4
  30. data/examples/aws_rekognition_adapter.rb +6 -6
  31. data/examples/openai_moderation_adapter.rb +1 -1
  32. data/examples/regexp_pattern_adapter.rb +68 -0
  33. data/gemfiles/{rails_7.1.gemfile → rails_8.0.gemfile} +8 -6
  34. data/gemfiles/rails_8.1.gemfile +6 -5
  35. data/lib/generators/moderate/install_generator.rb +25 -0
  36. data/lib/generators/moderate/templates/create_moderate_tables.rb.erb +17 -2
  37. data/lib/generators/moderate/templates/initializer.rb +9 -15
  38. data/lib/generators/moderate/views_generator.rb +3 -3
  39. data/lib/moderate/attachments.rb +137 -0
  40. data/lib/moderate/configuration.rb +101 -12
  41. data/lib/moderate/engine.rb +86 -25
  42. data/lib/moderate/event.rb +21 -0
  43. data/lib/moderate/filters/base.rb +1 -0
  44. data/lib/moderate/filters/wordlist.rb +5 -2
  45. data/lib/moderate/jobs/classify_job.rb +100 -16
  46. data/lib/moderate/label.rb +2 -2
  47. data/lib/moderate/macros.rb +8 -3
  48. data/lib/moderate/models/appeal.rb +10 -1
  49. data/lib/moderate/models/block.rb +23 -19
  50. data/lib/moderate/models/classification.rb +24 -0
  51. data/lib/moderate/models/concerns/actor.rb +17 -2
  52. data/lib/moderate/models/concerns/content_filterable.rb +241 -56
  53. data/lib/moderate/models/concerns/deferred_notices.rb +90 -0
  54. data/lib/moderate/models/concerns/reportable.rb +84 -4
  55. data/lib/moderate/models/flag.rb +199 -3
  56. data/lib/moderate/models/report.rb +180 -16
  57. data/lib/moderate/result.rb +3 -10
  58. data/lib/moderate/retention.rb +241 -0
  59. data/lib/moderate/services/intake_notice.rb +10 -10
  60. data/lib/moderate/services/intake_report.rb +42 -13
  61. data/lib/moderate/services/resolve_appeal.rb +8 -2
  62. data/lib/moderate/services/resolve_flag.rb +183 -35
  63. data/lib/moderate/services/resolve_report.rb +288 -35
  64. data/lib/moderate/text_validator.rb +15 -14
  65. data/lib/moderate/version.rb +1 -1
  66. data/lib/moderate.rb +337 -17
  67. data/lib/tasks/moderate.rake +23 -0
  68. metadata +24 -14
  69. data/gemfiles/rails_7.2.gemfile +0 -36
  70. data/lib/moderate/text.rb +0 -38
  71. data/lib/moderate/word_list.rb +0 -90
@@ -0,0 +1,200 @@
1
+ # moderate's default English copy. Override any key in your app's own
2
+ # config/locales: Rails loads an engine's locale files BEFORE the host's, so
3
+ # your translation always wins.
4
+ en:
5
+ # The error `moderates :field, mode: :block` (and `validates :field, moderate: true`)
6
+ # adds to a rejected field. Override it per model and attribute the usual
7
+ # Active Record way (activerecord.errors.models.<model>.attributes.<field>.objectionable_content).
8
+ errors:
9
+ messages:
10
+ objectionable_content: "contains language that isn't allowed"
11
+ # A :block attachment whose direct upload hasn't finished: nothing enters unverified.
12
+ upload_not_finished: "hasn't finished uploading yet"
13
+ moderate:
14
+ # Flash messages of the Moderate::Moderation controller concern.
15
+ moderation:
16
+ resolved: "Report resolved."
17
+ dismissed: "Report dismissed."
18
+ flag_resolved: "Flag actioned."
19
+ flag_dismissed: "Flag dismissed."
20
+ upheld: "Appeal upheld."
21
+ rejected: "Appeal rejected."
22
+ # Refusals and validation messages (the inline defaults in the code are English).
23
+ errors:
24
+ anonymous_notice_child_safety_only: "anonymous notices are only allowed for offences against minors"
25
+ appeal_already_closed: "This appeal has already been decided."
26
+ appeal_report_must_be_closed: "decision can only be appealed after it is made"
27
+ appeal_window_expired: "the appeal window for this decision has closed"
28
+ cannot_block_self: "You can't block yourself"
29
+ invalid_reported_field: "is not a reportable field"
30
+ invalid_subject_url: "must be a valid http(s) URL"
31
+ missing_subject_url: "is required"
32
+ not_reportable_by_reporter: "isn't something you can report"
33
+ nothing_removed: "Nothing was removed: %{content} has nothing removable there (already removed, deleted, or not removable). Resolve without removing content, or dismiss."
34
+ report_already_closed: "This report has already been resolved."
35
+ reporter_cannot_report_self: "You can't report yourself"
36
+ resolve_needs_action: "Resolving a report restricts something: remove the content, ban the account, or both. To close it without a restriction, dismiss it."
37
+ appeals:
38
+ reported_field: "Content"
39
+
40
+ # Human labels for reported / flagged fields — what the moderation queue,
41
+ # the statement of reasons and the appeal page print instead of the column
42
+ # name. Resolution (Moderate.reportable_field_label):
43
+ #
44
+ # 1. `has_reportable_content :body, labels: { body: "…" }` on the model
45
+ # 2. moderate.reportable_fields.<model i18n_key>.<field> e.g. chats/message.body
46
+ # 3. moderate.reportable_fields.<field> the generic ones below
47
+ # 4. field.humanize
48
+ #
49
+ # The generic set is deliberately small and avoids names that are also common
50
+ # MODEL names (message, comment, photo…): model-scoped and generic keys share
51
+ # this namespace, so a host's `comment: { body: … }` block would otherwise
52
+ # replace a generic `comment:` label.
53
+ reportable_fields:
54
+ body: "Text"
55
+ title: "Title"
56
+ description: "Description"
57
+ name: "Name"
58
+ display_name: "Display name"
59
+ username: "Username"
60
+ bio: "Bio"
61
+ avatar: "Profile photo"
62
+ photos: "Photos"
63
+ images: "Images"
64
+ attachments: "Attachments"
65
+ files: "Files"
66
+
67
+ # Labels for the gem's own value lists, read by Moderate.taxonomy_label(kind,
68
+ # value) and Moderate::Report.category_options (and, for legal_reasons,
69
+ # content_types and member_states, by the public notice form). The stored values
70
+ # never change; only what people read does.
71
+ categories:
72
+ harassment: "Harassment"
73
+ hate: "Hate"
74
+ threats: "Threats"
75
+ sexual_content: "Sexual content"
76
+ spam: "Spam"
77
+ fraud: "Fraud"
78
+ unsafe_behavior: "Unsafe behavior"
79
+ illegal_content: "Illegal content"
80
+ privacy: "Privacy"
81
+ child_safety: "Child safety"
82
+ other: "Other"
83
+ hate_abuse_harassment: "Hate, abuse or harassment"
84
+ violent_speech: "Violent speech"
85
+ graphic_violent_media: "Graphic or violent media"
86
+ illegal_regulated_behaviors: "Illegal or regulated behavior"
87
+ impersonation: "Impersonation"
88
+ adult_sexual_content: "Adult sexual content"
89
+ private_non_consensual_content: "Private or non-consensual content"
90
+ suicide_self_harm: "Suicide or self-harm"
91
+ terrorism_violent_extremism: "Terrorism or violent extremism"
92
+ scam_fraud: "Scam or fraud"
93
+
94
+ # The EU statement-of-reasons categories (DSA Art. 17 / the Transparency Database).
95
+ legal_reasons:
96
+ animal_welfare: "Animal welfare"
97
+ consumer_information: "Consumer information infringements"
98
+ cyber_violence: "Cyber violence"
99
+ data_protection_privacy: "Data protection and privacy violations"
100
+ illegal_or_harmful_speech: "Illegal or harmful speech"
101
+ civic_elections: "Negative effects on civic discourse or elections"
102
+ non_consensual_behavior: "Non-consensual behavior"
103
+ pornography_sexualized_content: "Pornography or sexualized content"
104
+ protection_of_minors: "Protection of minors"
105
+ public_security: "Risk for public security"
106
+ scams_fraud: "Scams and fraud"
107
+ scope_of_platform_service: "Scope of platform service"
108
+ self_harm: "Self-harm"
109
+ unsafe_illegal_products: "Unsafe or illegal products"
110
+ violence: "Violence"
111
+ intellectual_property: "Intellectual property infringements"
112
+ other: "Other"
113
+
114
+ member_states:
115
+ AT: "Austria"
116
+ BE: "Belgium"
117
+ BG: "Bulgaria"
118
+ CY: "Cyprus"
119
+ CZ: "Czechia"
120
+ DE: "Germany"
121
+ DK: "Denmark"
122
+ EE: "Estonia"
123
+ ES: "Spain"
124
+ FI: "Finland"
125
+ FR: "France"
126
+ GR: "Greece"
127
+ HR: "Croatia"
128
+ HU: "Hungary"
129
+ IE: "Ireland"
130
+ IT: "Italy"
131
+ LT: "Lithuania"
132
+ LU: "Luxembourg"
133
+ LV: "Latvia"
134
+ MT: "Malta"
135
+ NL: "Netherlands"
136
+ PL: "Poland"
137
+ PT: "Portugal"
138
+ RO: "Romania"
139
+ SE: "Sweden"
140
+ SI: "Slovenia"
141
+ SK: "Slovakia"
142
+ EU: "The whole EU"
143
+
144
+ content_types:
145
+ user_profile: "User profile"
146
+ profile_avatar: "Profile photo"
147
+ listing: "Listing"
148
+ message: "Message"
149
+ conversation: "Conversation"
150
+ group: "Group"
151
+ other: "Other"
152
+
153
+ resolution_bases:
154
+ terms: "Terms of service"
155
+ law: "The law"
156
+ law_and_terms: "The law and the terms of service"
157
+ insufficient_information: "Insufficient information"
158
+ no_violation: "No violation"
159
+
160
+ decisions:
161
+ no_restriction: "No restriction"
162
+ no_action: "No action"
163
+ content_removed: "Content removed"
164
+ account_suspended: "Account suspended"
165
+ other_restriction: "Other restriction"
166
+
167
+ intake_kinds:
168
+ community: "In-app report"
169
+ dsa: "Legal notice"
170
+
171
+ statuses:
172
+ report:
173
+ open: "Open"
174
+ actioned: "Actioned"
175
+ dismissed: "Dismissed"
176
+ flag:
177
+ pending: "Awaiting review"
178
+ actioned: "Actioned"
179
+ dismissed: "Dismissed"
180
+ appeal:
181
+ open: "Open"
182
+ upheld: "Upheld"
183
+ rejected: "Rejected"
184
+
185
+ flag_sources:
186
+ text_filter: "Text filter"
187
+ image_filter: "Image filter"
188
+ external_classifier: "External classifier"
189
+ manual: "Filed by hand"
190
+ wordlist: "Word list"
191
+
192
+ flag_modes:
193
+ flag: "Published, queued for review"
194
+ block: "Blocked"
195
+
196
+ appeal_sources:
197
+ notifier: "The notifier"
198
+ affected_user: "The affected person"
199
+ admin: "An administrator"
200
+ other: "Other"
@@ -0,0 +1,177 @@
1
+ # moderate's default Spanish copy. See en.yml for how these keys resolve and
2
+ # how to override them from your app.
3
+ es:
4
+ errors:
5
+ messages:
6
+ objectionable_content: "contiene lenguaje no permitido"
7
+ upload_not_finished: "aún no ha terminado de subirse"
8
+ moderate:
9
+ # Flash messages of the Moderate::Moderation controller concern.
10
+ moderation:
11
+ resolved: "Denuncia gestionada."
12
+ dismissed: "Denuncia descartada."
13
+ flag_resolved: "Marca gestionada."
14
+ flag_dismissed: "Marca descartada."
15
+ upheld: "Recurso estimado."
16
+ rejected: "Recurso desestimado."
17
+ # Refusals and validation messages (the inline defaults in the code are English).
18
+ errors:
19
+ anonymous_notice_child_safety_only: "solo se admiten avisos anónimos sobre delitos contra menores"
20
+ appeal_already_closed: "Este recurso ya está decidido."
21
+ appeal_report_must_be_closed: "una decisión solo puede recurrirse una vez tomada"
22
+ appeal_window_expired: "el plazo para recurrir esta decisión ha terminado"
23
+ cannot_block_self: "No puedes bloquearte a ti mismo"
24
+ invalid_reported_field: "no es un campo que se pueda denunciar"
25
+ invalid_subject_url: "debe ser una URL http(s) válida"
26
+ missing_subject_url: "es obligatoria"
27
+ not_reportable_by_reporter: "no es algo que puedas denunciar"
28
+ nothing_removed: "No se ha retirado nada: %{content} no tiene nada que retirar (ya retirado, eliminado o no retirable). Resuelve sin retirar contenido, o descarta la denuncia."
29
+ report_already_closed: "Esta denuncia ya está resuelta."
30
+ reporter_cannot_report_self: "No puedes denunciarte a ti mismo"
31
+ resolve_needs_action: "Resolver una denuncia impone una restricción: retira el contenido, suspende la cuenta o ambas. Para cerrarla sin restricción, descártala."
32
+ appeals:
33
+ reported_field: "Contenido"
34
+
35
+ reportable_fields:
36
+ body: "Texto"
37
+ title: "Título"
38
+ description: "Descripción"
39
+ name: "Nombre"
40
+ display_name: "Nombre visible"
41
+ username: "Nombre de usuario"
42
+ bio: "Biografía"
43
+ avatar: "Foto de perfil"
44
+ photos: "Fotos"
45
+ images: "Imágenes"
46
+ attachments: "Archivos adjuntos"
47
+ files: "Archivos"
48
+
49
+ categories:
50
+ harassment: "Acoso"
51
+ hate: "Odio"
52
+ threats: "Amenazas"
53
+ sexual_content: "Contenido sexual"
54
+ spam: "Spam"
55
+ fraud: "Fraude"
56
+ unsafe_behavior: "Conducta insegura"
57
+ illegal_content: "Contenido ilegal"
58
+ privacy: "Privacidad"
59
+ child_safety: "Seguridad infantil"
60
+ other: "Otro"
61
+ hate_abuse_harassment: "Odio, abuso o acoso"
62
+ violent_speech: "Discurso violento"
63
+ graphic_violent_media: "Contenido gráfico o violento"
64
+ illegal_regulated_behaviors: "Conductas ilegales o reguladas"
65
+ impersonation: "Suplantación"
66
+ adult_sexual_content: "Contenido sexual adulto"
67
+ private_non_consensual_content: "Contenido privado o no consentido"
68
+ suicide_self_harm: "Suicidio o autolesiones"
69
+ terrorism_violent_extremism: "Terrorismo o extremismo violento"
70
+ scam_fraud: "Estafa o fraude"
71
+
72
+ legal_reasons:
73
+ animal_welfare: "Bienestar animal"
74
+ consumer_information: "Infracciones de información al consumidor"
75
+ cyber_violence: "Ciberviolencia"
76
+ data_protection_privacy: "Protección de datos y privacidad"
77
+ illegal_or_harmful_speech: "Discurso ilegal o dañino"
78
+ civic_elections: "Efectos negativos en el discurso cívico o las elecciones"
79
+ non_consensual_behavior: "Conducta no consentida"
80
+ pornography_sexualized_content: "Pornografía o contenido sexualizado"
81
+ protection_of_minors: "Protección de menores"
82
+ public_security: "Riesgo para la seguridad pública"
83
+ scams_fraud: "Estafas y fraude"
84
+ scope_of_platform_service: "Ámbito del servicio de plataforma"
85
+ self_harm: "Autolesiones"
86
+ unsafe_illegal_products: "Productos inseguros o ilegales"
87
+ violence: "Violencia"
88
+ intellectual_property: "Propiedad intelectual"
89
+ other: "Otro"
90
+
91
+ member_states:
92
+ AT: "Austria"
93
+ BE: "Bélgica"
94
+ BG: "Bulgaria"
95
+ CY: "Chipre"
96
+ CZ: "Chequia"
97
+ DE: "Alemania"
98
+ DK: "Dinamarca"
99
+ EE: "Estonia"
100
+ ES: "España"
101
+ FI: "Finlandia"
102
+ FR: "Francia"
103
+ GR: "Grecia"
104
+ HR: "Croacia"
105
+ HU: "Hungría"
106
+ IE: "Irlanda"
107
+ IT: "Italia"
108
+ LT: "Lituania"
109
+ LU: "Luxemburgo"
110
+ LV: "Letonia"
111
+ MT: "Malta"
112
+ NL: "Países Bajos"
113
+ PL: "Polonia"
114
+ PT: "Portugal"
115
+ RO: "Rumanía"
116
+ SE: "Suecia"
117
+ SI: "Eslovenia"
118
+ SK: "Eslovaquia"
119
+ EU: "Toda la UE"
120
+
121
+ content_types:
122
+ user_profile: "Perfil de usuario"
123
+ profile_avatar: "Foto de perfil"
124
+ listing: "Anuncio"
125
+ message: "Mensaje"
126
+ conversation: "Conversación"
127
+ group: "Grupo"
128
+ other: "Otro"
129
+
130
+ resolution_bases:
131
+ terms: "Condiciones del servicio"
132
+ law: "La ley"
133
+ law_and_terms: "La ley y las condiciones del servicio"
134
+ insufficient_information: "Información insuficiente"
135
+ no_violation: "Sin infracción"
136
+
137
+ decisions:
138
+ no_restriction: "Sin restricción"
139
+ no_action: "Sin medidas"
140
+ content_removed: "Contenido retirado"
141
+ account_suspended: "Cuenta suspendida"
142
+ other_restriction: "Otra restricción"
143
+
144
+ intake_kinds:
145
+ community: "Denuncia en la app"
146
+ dsa: "Aviso legal"
147
+
148
+ statuses:
149
+ report:
150
+ open: "Abierta"
151
+ actioned: "Gestionada"
152
+ dismissed: "Descartada"
153
+ flag:
154
+ pending: "Pendiente de revisión"
155
+ actioned: "Gestionada"
156
+ dismissed: "Descartada"
157
+ appeal:
158
+ open: "Abierto"
159
+ upheld: "Estimado"
160
+ rejected: "Desestimado"
161
+
162
+ flag_sources:
163
+ text_filter: "Filtro de texto"
164
+ image_filter: "Filtro de imagen"
165
+ external_classifier: "Clasificador externo"
166
+ manual: "Marcado a mano"
167
+ wordlist: "Lista de palabras"
168
+
169
+ flag_modes:
170
+ flag: "Publicado, en revisión"
171
+ block: "Bloqueado"
172
+
173
+ appeal_sources:
174
+ notifier: "Quien lo notificó"
175
+ affected_user: "La persona afectada"
176
+ admin: "Un administrador"
177
+ other: "Otro"
data/context7.json ADDED
@@ -0,0 +1,4 @@
1
+ {
2
+ "url": "https://context7.com/rameerez/moderate",
3
+ "public_key": "pk_HibNJE5rTFvy1txHHXUot"
4
+ }
@@ -0,0 +1,139 @@
1
+ # frozen_string_literal: true
2
+
3
+ # 1.0.0.beta4: at most one PENDING flag per content and field, enforced by the
4
+ # database. `pending_key` ("<flaggable_type>/<flaggable_id>/<field>") is set while a
5
+ # flag is pending and NULL once it is closed (Moderate::Flag keeps it in sync on every
6
+ # save); a UNIQUE index on it lets only one pending flag per content + field exist,
7
+ # and NULLs never collide (PostgreSQL, MySQL and SQLite alike). Flag.flag! and
8
+ # Flag.flag_for_review! insert and, on the collision, return the flag already waiting.
9
+ #
10
+ # THE ONE MIGRATION THAT CREATES pending_key, in every app. A fresh install gets it
11
+ # from `bin/rails generate moderate:install`, copied right after create_moderate_tables
12
+ # (as "<number>_add_pending_key_to_moderate_flags.moderate.rb"); an app upgrading from
13
+ # beta3 gets it through the engine, which adds the gem's db/migrate to the app's
14
+ # migration paths unless the app holds copies. Safe to run twice.
15
+ #
16
+ # The key compares as bytes: on MySQL the column is utf8mb4_bin, because the default
17
+ # collation ignores case and would make "Post/1/body" and "POST/1/body" one key
18
+ # (PostgreSQL and SQLite compare bytes already). utf8mb4_bin still ignores TRAILING
19
+ # spaces (PAD SPACE); Moderate::Flag strips `field`, so keys it builds never differ by
20
+ # them.
21
+ #
22
+ # Backfill: each content + field's NEWEST pending flag takes the key; older duplicate
23
+ # pending flags (possible before this release) stay pending without one. The keys are
24
+ # checked by the database before any is written, grouped under the column's own
25
+ # collation: keys it would treat as one though they differ as strings (legacy rows
26
+ # whose field has trailing spaces, on MySQL) stop the migration, naming the flags.
27
+ #
28
+ # `down` drops the index and the column: this migration is the only thing that creates
29
+ # them, so rolling it back leaves create_moderate_tables' schema exactly as it was.
30
+ class AddPendingKeyToModerateFlags < ActiveRecord::Migration[7.1]
31
+ INDEX = "index_moderate_flags_on_pending_key"
32
+ BINARY_COLLATION = "utf8mb4_bin"
33
+ CHECK_TABLE = "moderate_pending_key_check"
34
+
35
+ # The tables as they are, not the app's model: a migration must not depend on code.
36
+ class Flag < ActiveRecord::Base
37
+ self.table_name = "moderate_flags"
38
+ end
39
+
40
+ def up
41
+ return unless table_exists?(:moderate_flags)
42
+
43
+ if column_exists?(:moderate_flags, :pending_key)
44
+ require_binary_comparison
45
+ else
46
+ add_column :moderate_flags, :pending_key, :string, **pending_key_options
47
+ end
48
+ backfill_pending_keys
49
+ return if index_exists?(:moderate_flags, :pending_key, name: INDEX)
50
+
51
+ add_index :moderate_flags, :pending_key, unique: true, name: INDEX
52
+ end
53
+
54
+ def down
55
+ return unless table_exists?(:moderate_flags)
56
+
57
+ remove_index :moderate_flags, name: INDEX if index_exists?(:moderate_flags, :pending_key, name: INDEX)
58
+ remove_column :moderate_flags, :pending_key if column_exists?(:moderate_flags, :pending_key)
59
+ end
60
+
61
+ private
62
+
63
+ # MySQL under either of Rails' adapters: mysql2, or Trilogy.
64
+ def mysql?
65
+ connection.adapter_name.downcase.match?(/mysql|trilogy/)
66
+ end
67
+
68
+ def pending_key_options
69
+ mysql? ? { collation: BINARY_COLLATION } : {}
70
+ end
71
+
72
+ # Only this migration creates the column, binary. One that isn't was made some other
73
+ # way: say so, rather than altering it here.
74
+ def require_binary_comparison
75
+ return unless mysql?
76
+
77
+ collation = connection.columns(:moderate_flags).find { |c| c.name == "pending_key" }.collation
78
+ return if collation == BINARY_COLLATION
79
+
80
+ raise ActiveRecord::MigrationError, <<~MESSAGE.squish
81
+ moderate_flags.pending_key compares with #{collation.inspect}, which ignores case, so
82
+ flags on content types that differ only by case would share one key. Make it
83
+ binary, then run the migration again: ALTER TABLE moderate_flags MODIFY pending_key
84
+ VARCHAR(255) COLLATE #{BINARY_COLLATION} NULL
85
+ MESSAGE
86
+ end
87
+
88
+ def backfill_pending_keys
89
+ Flag.reset_column_information
90
+ chosen = newest_pending_without_key
91
+ refuse_keys_the_database_would_merge(chosen)
92
+ chosen.each { |id, key| Flag.where(id: id).update_all(pending_key: key) }
93
+ end
94
+
95
+ # { flag id => key } for each content + field's newest pending flag without a key,
96
+ # unless a flag already holds that exact key.
97
+ def newest_pending_without_key
98
+ newest = {}
99
+ Flag.where(status: "pending", pending_key: nil).select(:id, :flaggable_type, :flaggable_id, :field, :created_at)
100
+ .find_each do |flag|
101
+ key = "#{flag.flaggable_type}/#{flag.flaggable_id}/#{flag.field}"
102
+ newest[key] = flag if newest[key].nil? || flag.created_at > newest[key].created_at
103
+ end
104
+ taken = Flag.where.not(pending_key: nil).pluck(:pending_key).to_set
105
+ newest.except(*taken).to_h { |key, flag| [flag.id, key] }
106
+ end
107
+
108
+ # The keys to write and the keys already written, in a temporary table whose key
109
+ # column compares exactly as pending_key does; any key the database groups twice
110
+ # would make the index (or a backfill UPDATE) fail, so stop first and name them.
111
+ # Plain SQL: Rails' schema lookups don't all see temporary tables (PostgreSQL's pg_temp).
112
+ def refuse_keys_the_database_would_merge(chosen)
113
+ create_table CHECK_TABLE, temporary: true, id: false do |t|
114
+ t.bigint :flag_id, null: false
115
+ t.string :pending_key, null: false, **pending_key_options
116
+ end
117
+ table = quote_table_name(CHECK_TABLE)
118
+ rows = chosen.to_a + Flag.where.not(pending_key: nil).pluck(:id, :pending_key)
119
+ rows.each_slice(500) do |slice|
120
+ values = slice.map { |id, key| "(#{Integer(id)}, #{quote(key)})" }.join(", ")
121
+ execute "INSERT INTO #{table} (flag_id, pending_key) VALUES #{values}"
122
+ end
123
+
124
+ merged = select_values("SELECT pending_key FROM #{table} GROUP BY pending_key HAVING COUNT(*) > 1")
125
+ return if merged.empty?
126
+
127
+ listed = merged.map do |key|
128
+ ids = select_values("SELECT flag_id FROM #{table} WHERE pending_key = #{quote(key)} ORDER BY flag_id")
129
+ "#{key.inspect} (flags #{ids.map { |id| Integer(id) }.join(", ")})"
130
+ end
131
+ raise ActiveRecord::MigrationError, <<~MESSAGE.squish
132
+ moderate_flags has pending flags whose pending_key the unique index would treat as
133
+ one: #{listed.join("; ")}. Close all but one of each (dismiss them, or fix a field
134
+ with stray spaces), then run the migration again.
135
+ MESSAGE
136
+ ensure
137
+ drop_table CHECK_TABLE, temporary: true, if_exists: true
138
+ end
139
+ end
@@ -0,0 +1,103 @@
1
+ # frozen_string_literal: true
2
+
3
+ # 1.0.0: what retention, the appeal window and attachment classification need.
4
+ #
5
+ # moderate_reports.redacted_at when Moderate.purge removed the report's personal
6
+ # moderate_flags.redacted_at data (nil: never). The purge selects on this, never
7
+ # moderate_appeals.redacted_at on the content, so a message that literally reads
8
+ # "[redacted]" is still purged.
9
+ # moderate_reports.held_until a legal hold (Moderate.hold): until then the purge
10
+ # leaves the report, its appeals and the flags on its
11
+ # content alone.
12
+ # moderate_reports.covered_by_report_id
13
+ # the decision that removed this report's content while
14
+ # it was pending (stamped by that decision, under the
15
+ # content's lock); only such a report is closed as
16
+ # "already removed by" it.
17
+ # moderate_classifications one row per record, field and set of attached
18
+ # files: who is classifying it (a claim) and when it
19
+ # was classified (Moderate::Attachments). Ids, a
20
+ # digest of blob ids and timestamps; no content.
21
+ #
22
+ # The one migration that creates all of this, in a fresh app as in an upgraded one:
23
+ # `rails generate moderate:install` copies it after create_moderate_tables (which
24
+ # leaves it out), an app holding copies gets it with `bin/rails
25
+ # moderate:install:migrations`, and an app installed without copies gets it through
26
+ # the engine, which adds the gem's db/migrate to its paths. Its `down` drops only what
27
+ # it created. Guarded, so a second run is a no-op. No backfill: nothing was redacted,
28
+ # held or claimed before 1.0.
29
+ class AddRetentionAndClassificationsToModerateTables < ActiveRecord::Migration[8.0]
30
+ COLUMNS = {
31
+ moderate_reports: %i[redacted_at held_until],
32
+ moderate_flags: %i[redacted_at],
33
+ moderate_appeals: %i[redacted_at]
34
+ }.freeze
35
+ HOLD_INDEX = "index_moderate_reports_on_held_until"
36
+ COVERED_INDEX = "index_moderate_reports_on_covered_by_report_id"
37
+ CLASSIFICATION_INDEX = "index_moderate_classifications_on_record_field_digest"
38
+
39
+ def up
40
+ COLUMNS.each do |table, columns|
41
+ next unless table_exists?(table)
42
+
43
+ columns.each { |column| add_column table, column, :datetime unless column_exists?(table, column) }
44
+ end
45
+ if table_exists?(:moderate_reports) && !index_exists?(:moderate_reports, :held_until, name: HOLD_INDEX)
46
+ add_index :moderate_reports, :held_until, name: HOLD_INDEX
47
+ end
48
+ add_covered_by if table_exists?(:moderate_reports)
49
+
50
+ create_classifications unless table_exists?(:moderate_classifications)
51
+ end
52
+
53
+ def down
54
+ drop_table :moderate_classifications if table_exists?(:moderate_classifications)
55
+ if table_exists?(:moderate_reports) && column_exists?(:moderate_reports, :covered_by_report_id)
56
+ remove_index :moderate_reports, name: COVERED_INDEX if index_exists?(:moderate_reports, :covered_by_report_id, name: COVERED_INDEX)
57
+ remove_column :moderate_reports, :covered_by_report_id
58
+ end
59
+ if table_exists?(:moderate_reports) && index_exists?(:moderate_reports, :held_until, name: HOLD_INDEX)
60
+ remove_index :moderate_reports, name: HOLD_INDEX
61
+ end
62
+
63
+ COLUMNS.each do |table, columns|
64
+ next unless table_exists?(table)
65
+
66
+ columns.each { |column| remove_column table, column if column_exists?(table, column) }
67
+ end
68
+ end
69
+
70
+ private
71
+
72
+ def add_covered_by
73
+ _primary_key_type, foreign_key_type = key_types
74
+ add_column :moderate_reports, :covered_by_report_id, foreign_key_type unless column_exists?(:moderate_reports, :covered_by_report_id)
75
+ return if index_exists?(:moderate_reports, :covered_by_report_id, name: COVERED_INDEX)
76
+
77
+ add_index :moderate_reports, :covered_by_report_id, name: COVERED_INDEX
78
+ end
79
+
80
+ def create_classifications
81
+ primary_key_type, foreign_key_type = key_types
82
+ create_table :moderate_classifications, id: primary_key_type do |t|
83
+ t.references :record, polymorphic: true, type: foreign_key_type, null: false, index: false
84
+ t.string :field, null: false
85
+ t.string :digest, null: false
86
+ t.string :claimed_by
87
+ t.datetime :claimed_at
88
+ t.datetime :classified_at
89
+ t.string :adapter
90
+ t.boolean :flagged
91
+ t.timestamps
92
+ end
93
+ add_index :moderate_classifications, %i[record_type record_id field digest], unique: true, name: CLASSIFICATION_INDEX
94
+ add_index :moderate_classifications, :classified_at, name: "index_moderate_classifications_on_classified_at"
95
+ end
96
+
97
+ # The app's primary-key type, as the install generator reads it (uuid or bigint).
98
+ def key_types
99
+ config = Rails.configuration.generators
100
+ setting = config.options[config.orm][:primary_key_type]
101
+ [setting || :primary_key, setting || :bigint]
102
+ end
103
+ end