moderate 1.0.0.beta1 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +96 -1
- data/.simplecov +2 -10
- data/AGENTS.md +1 -1
- data/Appraisals +10 -8
- data/CHANGELOG.md +660 -20
- data/CLAUDE.md +1 -1
- data/README.md +149 -7
- data/Rakefile +2 -0
- data/app/controllers/concerns/moderate/moderation.rb +23 -11
- data/app/controllers/moderate/appeals_controller.rb +9 -5
- data/app/controllers/moderate/application_controller.rb +3 -3
- data/app/controllers/moderate/notices_controller.rb +3 -6
- data/app/helpers/moderate/engine_helper.rb +32 -11
- data/app/views/moderate/appeals/new.html.erb +6 -0
- data/app/views/moderate/transparency_reports/_summary_card.html.erb +2 -2
- data/app/views/moderate/transparency_reports/show.html.erb +5 -5
- data/config/locales/en.yml +200 -0
- data/config/locales/es.yml +177 -0
- data/context7.json +4 -0
- data/db/migrate/20261018200000_add_pending_key_to_moderate_flags.rb +139 -0
- data/db/migrate/20261018210000_add_retention_and_classifications_to_moderate_tables.rb +103 -0
- data/docs/chats.md +140 -0
- data/docs/compliance.md +92 -3
- data/docs/configuration.md +30 -2
- data/docs/dsa-notice-form.md +1 -1
- data/docs/images/moderate-user-report-block-actions.webp +0 -0
- data/docs/madmin.md +44 -14
- data/docs/notifications.md +17 -4
- data/examples/aws_rekognition_adapter.rb +6 -6
- data/examples/openai_moderation_adapter.rb +1 -1
- data/examples/regexp_pattern_adapter.rb +68 -0
- data/gemfiles/{rails_7.1.gemfile → rails_8.0.gemfile} +8 -6
- data/gemfiles/rails_8.1.gemfile +6 -5
- data/lib/generators/moderate/install_generator.rb +25 -0
- data/lib/generators/moderate/templates/create_moderate_tables.rb.erb +17 -2
- data/lib/generators/moderate/templates/initializer.rb +9 -15
- data/lib/generators/moderate/views_generator.rb +3 -3
- data/lib/moderate/attachments.rb +137 -0
- data/lib/moderate/configuration.rb +101 -12
- data/lib/moderate/engine.rb +86 -25
- data/lib/moderate/event.rb +21 -0
- data/lib/moderate/filters/base.rb +1 -0
- data/lib/moderate/filters/wordlist.rb +5 -2
- data/lib/moderate/jobs/classify_job.rb +100 -16
- data/lib/moderate/label.rb +2 -2
- data/lib/moderate/macros.rb +8 -3
- data/lib/moderate/models/appeal.rb +10 -1
- data/lib/moderate/models/block.rb +23 -19
- data/lib/moderate/models/classification.rb +24 -0
- data/lib/moderate/models/concerns/actor.rb +17 -2
- data/lib/moderate/models/concerns/content_filterable.rb +241 -56
- data/lib/moderate/models/concerns/deferred_notices.rb +90 -0
- data/lib/moderate/models/concerns/reportable.rb +84 -4
- data/lib/moderate/models/flag.rb +199 -3
- data/lib/moderate/models/report.rb +180 -16
- data/lib/moderate/result.rb +3 -10
- data/lib/moderate/retention.rb +241 -0
- data/lib/moderate/services/intake_notice.rb +10 -10
- data/lib/moderate/services/intake_report.rb +42 -13
- data/lib/moderate/services/resolve_appeal.rb +8 -2
- data/lib/moderate/services/resolve_flag.rb +183 -35
- data/lib/moderate/services/resolve_report.rb +288 -35
- data/lib/moderate/text_validator.rb +15 -14
- data/lib/moderate/version.rb +1 -1
- data/lib/moderate.rb +337 -17
- data/lib/tasks/moderate.rake +23 -0
- metadata +24 -14
- data/gemfiles/rails_7.2.gemfile +0 -36
- data/lib/moderate/text.rb +0 -38
- data/lib/moderate/word_list.rb +0 -90
data/lib/moderate/models/flag.rb
CHANGED
|
@@ -16,6 +16,8 @@ module Moderate
|
|
|
16
16
|
# consumer (e.g. a job that auto-actions high-confidence flags), so the gem
|
|
17
17
|
# doesn't presume a human is the only reviewer.
|
|
18
18
|
class Flag < ApplicationRecord
|
|
19
|
+
include Moderate::DeferredNotices
|
|
20
|
+
|
|
19
21
|
self.table_name = "moderate_flags"
|
|
20
22
|
|
|
21
23
|
STATUSES = %w[pending actioned dismissed].freeze
|
|
@@ -48,6 +50,7 @@ module Moderate
|
|
|
48
50
|
# NotNullViolation on MySQL. (SQLite/PostgreSQL get the defaults from the
|
|
49
51
|
# migration; coalescing is harmless there.)
|
|
50
52
|
before_save :default_json_columns
|
|
53
|
+
before_save :sync_pending_key
|
|
51
54
|
|
|
52
55
|
# Announce a new flag through the host's notify hook so admins get pinged (e.g.
|
|
53
56
|
# a Telegram alert) the moment content is flagged. after_create_COMMIT, not
|
|
@@ -60,6 +63,10 @@ module Moderate
|
|
|
60
63
|
scope :dismissed, -> { where(status: "dismissed") }
|
|
61
64
|
scope :recent_first, -> { order(created_at: :desc) }
|
|
62
65
|
|
|
66
|
+
# Stripped wherever it is assigned (and in `where(field: …)`), so a field never
|
|
67
|
+
# differs by surrounding spaces: neither in what a moderator reads nor in the
|
|
68
|
+
# pending_key, which MySQL's utf8mb4_bin compares ignoring trailing spaces (PAD SPACE).
|
|
69
|
+
normalizes :field, with: :strip.to_proc
|
|
63
70
|
validates :field, presence: true
|
|
64
71
|
validates :status, inclusion: { in: STATUSES }
|
|
65
72
|
validates :source, inclusion: { in: ->(_flag) { sources } }, on: :create
|
|
@@ -71,20 +78,136 @@ module Moderate
|
|
|
71
78
|
# whatever shape a Moderate::Result exposed without each one re-normalizing.
|
|
72
79
|
# `context` is freeform JSON for audit (which policy fired, the raw classifier
|
|
73
80
|
# payload, etc.) — never relied on by the gem's own logic.
|
|
81
|
+
#
|
|
82
|
+
# `owner` is kept only if it is a person (the configured user class): hosts
|
|
83
|
+
# commonly pass `record.reported_owner`, which can be a desk or a bot.
|
|
84
|
+
#
|
|
85
|
+
# AT MOST ONE PENDING FLAG PER CONTENT + FIELD, enforced by the database (the
|
|
86
|
+
# unique `pending_key`, see #sync_pending_key). If a flag already waits on this
|
|
87
|
+
# field, the insert collides and that waiting flag is returned instead: the
|
|
88
|
+
# content is already in the queue (`flag.previously_new_record?` is false). This
|
|
89
|
+
# holds whatever the isolation level or snapshot, because uniqueness is checked
|
|
90
|
+
# against what is committed, and the waiting flag is then read with a LOCKING
|
|
91
|
+
# read (the latest committed row, also under MySQL's REPEATABLE READ).
|
|
92
|
+
#
|
|
93
|
+
# On MySQL an insert colliding with a flag that is being decided can be chosen as
|
|
94
|
+
# a deadlock victim: InnoDB queues the duplicate check as a lock on the gap the
|
|
95
|
+
# decided flag's index entry moves into. When this call is the whole transaction
|
|
96
|
+
# (the filters' after_commit, ClassifyJob, the admin button) it is simply tried
|
|
97
|
+
# again; inside a caller's transaction InnoDB has rolled that transaction back
|
|
98
|
+
# too, so ActiveRecord::Deadlocked is raised for the caller to retry.
|
|
99
|
+
#
|
|
100
|
+
# Up to ATTEMPTS tries, then the last failure is raised as it was:
|
|
101
|
+
# ActiveRecord::Deadlocked if the database kept choosing this insert as a deadlock
|
|
102
|
+
# victim, ActiveRecord::RecordNotUnique only if, after a collision, no pending
|
|
103
|
+
# flag could be found to join (each one was decided before it could be read).
|
|
74
104
|
def self.flag!(flaggable:, field:, owner:, source:, mode:, excerpt:, categories:, scores:, context:)
|
|
75
|
-
|
|
105
|
+
attributes = {
|
|
76
106
|
flaggable: flaggable,
|
|
77
107
|
field: field,
|
|
78
|
-
owner: owner,
|
|
108
|
+
owner: (owner if owner.is_a?(Moderate.user_class)),
|
|
79
109
|
source: source,
|
|
80
110
|
mode: mode,
|
|
81
111
|
excerpt: excerpt,
|
|
82
112
|
categories: Array(categories),
|
|
83
113
|
scores: scores.to_h,
|
|
84
114
|
context: context.to_h
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
attempts = 0
|
|
118
|
+
begin
|
|
119
|
+
attempts += 1
|
|
120
|
+
# A savepoint, so a collision doesn't poison a surrounding transaction.
|
|
121
|
+
transaction(requires_new: true) { create!(attributes) }
|
|
122
|
+
rescue ActiveRecord::RecordNotUnique
|
|
123
|
+
waiting = waiting_flag(pending_key_for(flaggable.class.polymorphic_name, flaggable.id, field.to_s.strip))
|
|
124
|
+
return waiting if waiting&.pending?
|
|
125
|
+
raise if attempts >= ATTEMPTS
|
|
126
|
+
|
|
127
|
+
retry # the waiting flag was decided while we looked: its key is free
|
|
128
|
+
rescue ActiveRecord::Deadlocked
|
|
129
|
+
# The caller's transaction died with ours: theirs to retry, as a whole.
|
|
130
|
+
raise if connection.transaction_open? || attempts >= ATTEMPTS
|
|
131
|
+
|
|
132
|
+
retry
|
|
133
|
+
end
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
ATTEMPTS = 3
|
|
137
|
+
|
|
138
|
+
# The pending_key a pending flag on that content and field carries.
|
|
139
|
+
def self.pending_key_for(flaggable_type, flaggable_id, field)
|
|
140
|
+
"#{flaggable_type}/#{flaggable_id}/#{field}"
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
# The flag waiting under `key`, by a locking read: on MySQL a plain read could
|
|
144
|
+
# return the transaction's older snapshot, without the flag the insert collided with.
|
|
145
|
+
def self.waiting_flag(key)
|
|
146
|
+
where(pending_key: key).lock.first
|
|
147
|
+
end
|
|
148
|
+
private_class_method :waiting_flag
|
|
149
|
+
|
|
150
|
+
# An administrator filing `record`'s `field` for review by hand — the "Flag for
|
|
151
|
+
# review" button on a transcript line or a profile page. It lands in the same
|
|
152
|
+
# `pending` queue the filters feed, to be decided like any other flag. Nothing is
|
|
153
|
+
# removed or sanctioned here.
|
|
154
|
+
#
|
|
155
|
+
# flag = Moderate::Flag.flag_for_review!(message, field: :body, by: current_user)
|
|
156
|
+
# flag.previously_new_record? # => true if filed now, false if one already waited
|
|
157
|
+
#
|
|
158
|
+
# ONCE per field: if a pending flag on that field already waits, it is returned
|
|
159
|
+
# instead of a second one. The check and the insert run under the RECORD's row
|
|
160
|
+
# lock, so two administrators pressing it together file one flag (a check-then-
|
|
161
|
+
# insert without the lock files two). The record must be persisted.
|
|
162
|
+
#
|
|
163
|
+
# The flag is `source: "manual"`, owned by the person responsible for the record
|
|
164
|
+
# (Moderate.responsible_user), with an excerpt from the record's evidence snapshot
|
|
165
|
+
# (or the field's text), and `context` naming who filed it and their note. The
|
|
166
|
+
# filing is audited as `:flag_filed` with the administrator as actor.
|
|
167
|
+
def self.flag_for_review!(record, field:, by:, note: nil)
|
|
168
|
+
field_s = field.to_s.strip
|
|
169
|
+
raise ArgumentError, "field is required" if field_s.empty?
|
|
170
|
+
unless record.respond_to?(:persisted?) && record.persisted?
|
|
171
|
+
raise ArgumentError, "#{record.class.name} must be persisted to be flagged"
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
note = note.to_s.strip.presence
|
|
175
|
+
# No lock taken first and no plain read deciding anything: the insert either
|
|
176
|
+
# files the flag or collides with the one already waiting (the unique
|
|
177
|
+
# pending_key), which Flag.flag! then returns. Only on that collision does it
|
|
178
|
+
# lock a row, the waiting flag's, so the lock order (Moderate.with_row_lock)
|
|
179
|
+
# holds: it never holds content while waiting on a flag.
|
|
180
|
+
flag = file_for_review!(record, field_s, by: by, note: note)
|
|
181
|
+
audit_filed_for_review(flag, by: by, note: note) if flag.previously_new_record?
|
|
182
|
+
flag
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
def self.file_for_review!(record, field, by:, note:)
|
|
186
|
+
flag!(
|
|
187
|
+
flaggable: record, field: field, owner: Moderate.responsible_user(record),
|
|
188
|
+
source: "manual", mode: "flag", excerpt: review_excerpt(record, field),
|
|
189
|
+
categories: [], scores: {}, context: { "flagged_by_id" => by&.id, "note" => note }.compact
|
|
85
190
|
)
|
|
86
191
|
end
|
|
87
192
|
|
|
193
|
+
def self.audit_filed_for_review(flag, by:, note:)
|
|
194
|
+
payload = {
|
|
195
|
+
flag_id: flag.id, flaggable_type: flag.flaggable_type, flaggable_id: flag.flaggable_id,
|
|
196
|
+
field: flag.field, note: note,
|
|
197
|
+
summary: "Flag ##{flag.id} filed by hand on #{flag.flaggable_label} · #{flag.field_label}"
|
|
198
|
+
}
|
|
199
|
+
Moderate.audit(:flag_filed, subject: flag, actor: by, payload: payload.compact)
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
# What the reviewer reads in the queue: the record's evidence snapshot for the
|
|
203
|
+
# field (the same text a report would freeze), else the field's own text.
|
|
204
|
+
def self.review_excerpt(record, field)
|
|
205
|
+
text = record.moderation_snapshot(field) if record.respond_to?(:moderation_snapshot)
|
|
206
|
+
text = record.public_send(field) if text.nil? && record.respond_to?(field)
|
|
207
|
+
text.to_s.truncate(500) if text.is_a?(String) && text.present?
|
|
208
|
+
end
|
|
209
|
+
private_class_method :file_for_review!, :audit_filed_for_review, :review_excerpt
|
|
210
|
+
|
|
88
211
|
def self.sources
|
|
89
212
|
(SOURCES + Moderate.config.adapters.keys.map(&:to_s)).uniq
|
|
90
213
|
end
|
|
@@ -97,6 +220,48 @@ module Moderate
|
|
|
97
220
|
status.in?(%w[actioned dismissed])
|
|
98
221
|
end
|
|
99
222
|
|
|
223
|
+
# Close this flag as ACTIONED — someone (or an automated policy) took
|
|
224
|
+
# action on the flagged content. Sugar over Moderate::Services::ResolveFlag
|
|
225
|
+
# (row lock, re-check, audit), mirroring Report#resolve!/#dismiss!.
|
|
226
|
+
# `by:` is the reviewing user when a human decided; leave it nil for
|
|
227
|
+
# automated closes — recording a human who never looked would poison the
|
|
228
|
+
# audit trail (DSA statements of reasons read these rows).
|
|
229
|
+
#
|
|
230
|
+
# Enforce from the flag itself — the bridge every host used to hand-write:
|
|
231
|
+
#
|
|
232
|
+
# flag.action!(by: admin, note: "Slur", remove_content: true)
|
|
233
|
+
# flag.action!(by: admin, note: "Repeat offender", ban_user: true)
|
|
234
|
+
#
|
|
235
|
+
# Either option escalates to a report (the open one on the same content and
|
|
236
|
+
# field, or a system one) and resolves it, so the content is removed through
|
|
237
|
+
# the reportable's `remove_reported_field!`, the ban through your `ban_handler`,
|
|
238
|
+
# and the affected user gets the statement of reasons and the appeal window a
|
|
239
|
+
# restriction requires. `flag.report` is that report afterwards.
|
|
240
|
+
def action!(note:, by: nil, **enforcement)
|
|
241
|
+
Moderate::Services::ResolveFlag.new(self, by: by).action!(note: note, **enforcement)
|
|
242
|
+
end
|
|
243
|
+
|
|
244
|
+
# The queue-wide verb: Report#resolve! and Flag#resolve! take the same
|
|
245
|
+
# keywords, so one admin action (Moderate::Moderation#resolve) decides both.
|
|
246
|
+
alias resolve! action!
|
|
247
|
+
|
|
248
|
+
# Close this flag with NO action taken. The canonical automated use is
|
|
249
|
+
# SUPERSEDED content: when the flagged field changes (text edited, photo
|
|
250
|
+
# replaced/reverted), a still-pending flag is about content that is no
|
|
251
|
+
# longer live — leaving it pending keeps `flagged?` true and mislabels
|
|
252
|
+
# the NEW content in any host UI keyed on it. Hosts should dismiss with a
|
|
253
|
+
# note saying what superseded it.
|
|
254
|
+
def dismiss!(note:, by: nil)
|
|
255
|
+
Moderate::Services::ResolveFlag.new(self, by: by).dismiss!(note: note)
|
|
256
|
+
end
|
|
257
|
+
|
|
258
|
+
# The report an enforcing `action!` went through, or nil (a triage-only close,
|
|
259
|
+
# or a flag still pending).
|
|
260
|
+
def report
|
|
261
|
+
report_id = context.to_h["report_id"]
|
|
262
|
+
Moderate::Report.find_by(id: report_id) if report_id.present?
|
|
263
|
+
end
|
|
264
|
+
|
|
100
265
|
# A label for the flagged thing in the queue. Asks the flaggable for its own
|
|
101
266
|
# `moderation_label` (Moderate::Reportable interface); falls back to a generic
|
|
102
267
|
# "Type id" string for content that doesn't implement it.
|
|
@@ -106,8 +271,38 @@ module Moderate
|
|
|
106
271
|
"#{flaggable_type} #{flaggable_id}"
|
|
107
272
|
end
|
|
108
273
|
|
|
274
|
+
# What humans call the flagged field ("Chat message", not "body"). Same
|
|
275
|
+
# resolution as Report#reported_field_label, so a queue that lists reports and
|
|
276
|
+
# flags side by side names a field the same way in both. Works after the
|
|
277
|
+
# flagged record is gone (falls back to `flaggable_type`).
|
|
278
|
+
def field_label
|
|
279
|
+
Moderate.reportable_field_label(flaggable || flaggable_type, field)
|
|
280
|
+
end
|
|
281
|
+
|
|
282
|
+
# Lock the row (`with_lock`, `lock!`) — and, for a pending flag, its entry in the
|
|
283
|
+
# unique pending_key index FIRST. InnoDB keeps a row in two places, and a flag's
|
|
284
|
+
# two are taken in opposite orders otherwise: an insert colliding on the key
|
|
285
|
+
# (Flag.flag!) holds the index entry and then waits for the row, while closing the
|
|
286
|
+
# flag holds the row and then, freeing the key, needs the entry — a deadlock. One
|
|
287
|
+
# order, entry then row, everywhere. On PostgreSQL and SQLite it re-locks the
|
|
288
|
+
# same row (or nothing): harmless.
|
|
289
|
+
def lock!(lock = true) # rubocop:disable Style/OptionalBooleanParameter -- Rails' signature
|
|
290
|
+
key = pending_key if persisted? && !has_changes_to_save?
|
|
291
|
+
self.class.unscoped.where(pending_key: key).lock(lock).ids if key
|
|
292
|
+
super
|
|
293
|
+
end
|
|
294
|
+
|
|
109
295
|
private
|
|
110
296
|
|
|
297
|
+
# A pending flag carries its content + field's pending_key; any other status
|
|
298
|
+
# carries NULL, so closing a flag (ResolveFlag, or any save that changes its
|
|
299
|
+
# status) frees the key in the same transaction, and reopening one takes it back
|
|
300
|
+
# (raising RecordNotUnique if another flag is pending there by then). Writes that
|
|
301
|
+
# skip callbacks (update_column/update_all on status) must set it themselves.
|
|
302
|
+
def sync_pending_key
|
|
303
|
+
self.pending_key = (Flag.pending_key_for(flaggable_type, flaggable_id, field) if pending?)
|
|
304
|
+
end
|
|
305
|
+
|
|
111
306
|
# See the before_save comment: keep the NOT-NULL JSON columns non-null on MySQL.
|
|
112
307
|
def default_json_columns
|
|
113
308
|
self.categories ||= []
|
|
@@ -126,9 +321,10 @@ module Moderate
|
|
|
126
321
|
flaggable_type: flaggable_type,
|
|
127
322
|
flaggable_id: flaggable_id,
|
|
128
323
|
field: field,
|
|
324
|
+
field_label: field_label,
|
|
129
325
|
source: source,
|
|
130
326
|
categories: Array(categories),
|
|
131
|
-
summary: "content flagged (#{source}) on #{flaggable_label}
|
|
327
|
+
summary: "content flagged (#{source}) on #{flaggable_label} · #{field_label}"
|
|
132
328
|
}
|
|
133
329
|
)
|
|
134
330
|
end
|
|
@@ -19,6 +19,8 @@ module Moderate
|
|
|
19
19
|
# what fields may be reported are all answered by the polymorphic `has_reportable_content`
|
|
20
20
|
# (through the `Moderate::Reportable` interface), never by anything domain-specific.
|
|
21
21
|
class Report < ApplicationRecord
|
|
22
|
+
include Moderate::DeferredNotices
|
|
23
|
+
|
|
22
24
|
self.table_name = "moderate_reports"
|
|
23
25
|
|
|
24
26
|
STATUSES = %w[open actioned dismissed].freeze
|
|
@@ -86,6 +88,11 @@ module Moderate
|
|
|
86
88
|
# locate the *report itself* (and vice versa) — GlobalID verifies the purpose on
|
|
87
89
|
# the way back out. See https://github.com/rails/globalid#signed-global-ids.
|
|
88
90
|
# The strings are gem-stable ("moderate_*") and host-agnostic.
|
|
91
|
+
#
|
|
92
|
+
# @api private: build a report target with `record.to_moderation_sgid` (or the
|
|
93
|
+
# `moderate_report_target_path` helper) and read one back with
|
|
94
|
+
# `Report.locate_signed_reportable`; appeal links with `report.signed_appeal_gid`.
|
|
95
|
+
# The strings never change (tokens already sent out carry them).
|
|
89
96
|
SIGNED_GLOBAL_ID_PURPOSE = "moderate_report"
|
|
90
97
|
APPEAL_SIGNED_GLOBAL_ID_PURPOSE = "moderate_appeal"
|
|
91
98
|
|
|
@@ -98,6 +105,9 @@ module Moderate
|
|
|
98
105
|
# receipt for this seeded record", "also block the reported user"). They never
|
|
99
106
|
# persist; the services read them. Kept here so both the model and its services
|
|
100
107
|
# share one definition.
|
|
108
|
+
#
|
|
109
|
+
# `skip_received_notice` is public (seed and import scripts set it).
|
|
110
|
+
# `block_reported_user` is @api private: say `report!(…, block: true)`.
|
|
101
111
|
attr_accessor :skip_received_notice, :block_reported_user
|
|
102
112
|
|
|
103
113
|
# All actor associations resolve to the HOST's configured user class, read
|
|
@@ -113,6 +123,10 @@ module Moderate
|
|
|
113
123
|
|
|
114
124
|
has_many :appeals, class_name: "Moderate::Appeal", dependent: :destroy
|
|
115
125
|
|
|
126
|
+
# The decision that removed this report's content while this report was pending
|
|
127
|
+
# (stamped by that decision); see ResolveReport's covering removal.
|
|
128
|
+
belongs_to :covered_by, class_name: "Moderate::Report", foreign_key: :covered_by_report_id, optional: true
|
|
129
|
+
|
|
116
130
|
# The auto-filter flags that touched the SAME (record, field) this report is
|
|
117
131
|
# about. Used to disclose automated means at intake (Art. 17(3)(c)) — if the
|
|
118
132
|
# wordlist/image adapter already flagged this exact content, the decision email
|
|
@@ -126,8 +140,10 @@ module Moderate
|
|
|
126
140
|
foreign_key: :flaggable_id
|
|
127
141
|
|
|
128
142
|
before_validation :normalize_strings
|
|
143
|
+
before_validation :default_community_message, on: :create
|
|
129
144
|
before_validation :hydrate_reporter_contact, if: :reporter
|
|
130
145
|
before_validation :infer_reported_user, on: :create
|
|
146
|
+
before_validation :infer_content_type, on: :create
|
|
131
147
|
before_validation :capture_snapshot, on: :create
|
|
132
148
|
before_validation :capture_automated_processing, on: :create
|
|
133
149
|
# Default the JSON columns to their empty shape before save. WHY: the migration
|
|
@@ -192,8 +208,11 @@ module Moderate
|
|
|
192
208
|
validates :resolution_basis, inclusion: { in: RESOLUTION_BASES }, allow_blank: true
|
|
193
209
|
validate :reporter_cannot_report_self
|
|
194
210
|
validate :reportable_field_must_be_allowed
|
|
211
|
+
validate :reporter_must_be_allowed_to_report, on: :create
|
|
195
212
|
validate :subject_url_must_be_http_url
|
|
196
|
-
|
|
213
|
+
# At intake only: a report whose content is deleted later has its pointer
|
|
214
|
+
# nullified, and must still be decidable.
|
|
215
|
+
validate :public_notice_requires_subject_url, on: :create
|
|
197
216
|
validate :dsa_notice_must_be_substantiated
|
|
198
217
|
validate :anonymous_notice_only_for_child_safety
|
|
199
218
|
|
|
@@ -207,6 +226,17 @@ module Moderate
|
|
|
207
226
|
Array(Moderate.config.report_categories).map(&:to_s).presence || DEFAULT_CATEGORIES
|
|
208
227
|
end
|
|
209
228
|
|
|
229
|
+
# The community categories in effect as `[label, value]` pairs, in the current
|
|
230
|
+
# locale, ready for a report form's select:
|
|
231
|
+
#
|
|
232
|
+
# f.select :category, Moderate::Report.category_options
|
|
233
|
+
#
|
|
234
|
+
# Honors `config.report_categories` (a host list's own values are labeled from
|
|
235
|
+
# `moderate.categories.<value>`, else humanized). See Moderate.taxonomy_label.
|
|
236
|
+
def self.category_options
|
|
237
|
+
report_categories.map { |category| [Moderate.taxonomy_label(:category, category), category] }
|
|
238
|
+
end
|
|
239
|
+
|
|
210
240
|
# --- Signed-GlobalID locators --------------------------------------------
|
|
211
241
|
|
|
212
242
|
# Resolve a signed token back into the reported content. Prefer the
|
|
@@ -225,6 +255,8 @@ module Moderate
|
|
|
225
255
|
# Resolve a signed token back into the Report it was minted for (used by the
|
|
226
256
|
# public appeal flow, where the appellant arrives via an emailed signed link).
|
|
227
257
|
# Locked to `self` so the token can only ever name a Report.
|
|
258
|
+
#
|
|
259
|
+
# @api private: the public appeal form's lookup.
|
|
228
260
|
def self.locate_signed_appeal_report(token)
|
|
229
261
|
return if token.blank?
|
|
230
262
|
|
|
@@ -237,6 +269,10 @@ module Moderate
|
|
|
237
269
|
status == "open"
|
|
238
270
|
end
|
|
239
271
|
|
|
272
|
+
# Awaiting a decision — the record-level twin of the `pending` scope, so one
|
|
273
|
+
# predicate reads the same across Report, Flag and Appeal queues.
|
|
274
|
+
alias pending? open?
|
|
275
|
+
|
|
240
276
|
def actioned?
|
|
241
277
|
status == "actioned"
|
|
242
278
|
end
|
|
@@ -282,16 +318,27 @@ module Moderate
|
|
|
282
318
|
subject_url.presence || I18n.t("moderate.reports.legal_notice_label", default: "Legal notice")
|
|
283
319
|
end
|
|
284
320
|
|
|
285
|
-
#
|
|
286
|
-
#
|
|
287
|
-
|
|
288
|
-
|
|
321
|
+
# What humans call the reported field ("Chat message", not "body"), or nil when
|
|
322
|
+
# the report targets the whole record. Resolved from the reportable's CLASS, so
|
|
323
|
+
# the label survives the content being deleted: deleting a reportable nullifies
|
|
324
|
+
# BOTH polymorphic columns (`dependent: :nullify`), so we fall back to the type
|
|
325
|
+
# frozen into the evidence snapshot. See Moderate.reportable_field_label.
|
|
326
|
+
def reported_field_label
|
|
327
|
+
type = reportable_type.presence || snapshot.to_h.with_indifferent_access[:reportable_type]
|
|
328
|
+
Moderate.reportable_field_label(reportable || type, reported_field)
|
|
329
|
+
end
|
|
289
330
|
|
|
290
|
-
|
|
331
|
+
# The reported field's text as it was when the report was filed: the evidence
|
|
332
|
+
# snapshot taken at intake, so it survives the content being edited or deleted.
|
|
333
|
+
# nil when the reportable exposed no text, or once a purge redacted the report.
|
|
334
|
+
def reported_content_text
|
|
335
|
+
snapshot.to_h.with_indifferent_access[:content_text]
|
|
291
336
|
end
|
|
292
337
|
|
|
293
338
|
# --- Signed GIDs for emailed links ---------------------------------------
|
|
294
339
|
|
|
340
|
+
# @api private: an expiring token for the reported content. Use the reportable's
|
|
341
|
+
# `to_moderation_sgid`, which doesn't expire.
|
|
295
342
|
def signed_reportable_gid
|
|
296
343
|
reportable&.to_sgid_param(for: SIGNED_GLOBAL_ID_PURPOSE)
|
|
297
344
|
end
|
|
@@ -326,6 +373,8 @@ module Moderate
|
|
|
326
373
|
# `update_column` skips validations/callbacks on purpose — this is a timestamp
|
|
327
374
|
# stamp, not a content change, and must succeed even on an already-validated row.
|
|
328
375
|
# Idempotent: only sets the first acknowledgement.
|
|
376
|
+
#
|
|
377
|
+
# @api private: stamped by the intake services.
|
|
329
378
|
def acknowledge!(at: Time.current)
|
|
330
379
|
update_column(:acknowledged_at, at) if acknowledged_at.blank?
|
|
331
380
|
end
|
|
@@ -333,6 +382,8 @@ module Moderate
|
|
|
333
382
|
# DSA Art. 20(1): open the redress (appeal) window for at least six months after
|
|
334
383
|
# the decision. Stamped once when the report is decided; idempotent so re-running
|
|
335
384
|
# a decision flow doesn't slide the deadline.
|
|
385
|
+
#
|
|
386
|
+
# @api private: stamped by the decision services.
|
|
336
387
|
def close_redress_window!(at: resolved_at || Time.current)
|
|
337
388
|
update_column(:appeal_deadline_at, at + APPEAL_WINDOW) if appeal_deadline_at.blank?
|
|
338
389
|
end
|
|
@@ -344,8 +395,8 @@ module Moderate
|
|
|
344
395
|
# out-of-transaction notify, statement-of-reasons + appeal-window stamping); this
|
|
345
396
|
# only forwards. `by:` is the moderator; the rest (`note:`, `remove_content:`,
|
|
346
397
|
# `ban_user:`, `resolution_basis:`) pass straight through.
|
|
347
|
-
def resolve!(by:, **
|
|
348
|
-
Moderate::Services::ResolveReport.new(self, by: by).resolve!(**
|
|
398
|
+
def resolve!(by:, **)
|
|
399
|
+
Moderate::Services::ResolveReport.new(self, by: by).resolve!(**)
|
|
349
400
|
end
|
|
350
401
|
|
|
351
402
|
# Dismiss this report (no action taken) — sugar over
|
|
@@ -354,6 +405,39 @@ module Moderate
|
|
|
354
405
|
Moderate::Services::ResolveReport.new(self, by: by).dismiss!(note: note)
|
|
355
406
|
end
|
|
356
407
|
|
|
408
|
+
# Which delivery timestamp each decision notice confirms (Moderate.delivered!).
|
|
409
|
+
DELIVERY_COLUMNS = {
|
|
410
|
+
decision: :decision_notified_at, # report_decision, to the reporter / notifier
|
|
411
|
+
affected_user: :affected_user_notified_at # affected_user_decision, the statement of reasons
|
|
412
|
+
}.freeze
|
|
413
|
+
|
|
414
|
+
# DSA Art. 20(2): the (at least) six-month appeal window starts when the person
|
|
415
|
+
# is INFORMED of the decision, not when it was taken. The decision stamps
|
|
416
|
+
# `appeal_deadline_at` six months from itself, a minimum. When the host confirms
|
|
417
|
+
# that a decision notice actually went out (Moderate.delivered!, called from the
|
|
418
|
+
# mailer's delivery, not when it was queued), its timestamp is stamped, once, and
|
|
419
|
+
# the deadline moves to six months after it, never earlier. Later confirmations
|
|
420
|
+
# change nothing.
|
|
421
|
+
#
|
|
422
|
+
# @api private: call Moderate.delivered!(report, :decision | :affected_user).
|
|
423
|
+
def record_delivery!(recipient, at: Time.current)
|
|
424
|
+
column = DELIVERY_COLUMNS.fetch(recipient.to_sym) do
|
|
425
|
+
raise ArgumentError, "unknown decision notice #{recipient.inspect}; one of: #{DELIVERY_COLUMNS.keys.join(", ")}"
|
|
426
|
+
end
|
|
427
|
+
|
|
428
|
+
with_lock do
|
|
429
|
+
next if self[column].present? || resolved_at.blank?
|
|
430
|
+
|
|
431
|
+
update_columns(column => at, appeal_deadline_at: [appeal_deadline_at, at + APPEAL_WINDOW].compact.max)
|
|
432
|
+
end
|
|
433
|
+
self
|
|
434
|
+
end
|
|
435
|
+
|
|
436
|
+
# Under a legal hold right now (Moderate.hold)? The purge leaves it alone.
|
|
437
|
+
def held?(at: Time.current)
|
|
438
|
+
held_until.present? && held_until > at
|
|
439
|
+
end
|
|
440
|
+
|
|
357
441
|
# DSA Art. 17(3)(c): did automated means (a wordlist/image/remote classifier)
|
|
358
442
|
# participate in surfacing or deciding this report? The decision email reads this
|
|
359
443
|
# to truthfully state whether automation was used. We treat the disclosure as
|
|
@@ -365,7 +449,7 @@ module Moderate
|
|
|
365
449
|
|
|
366
450
|
return true if boolean.cast(automation["used"])
|
|
367
451
|
|
|
368
|
-
automation.except("used").values.any? { |value|
|
|
452
|
+
automation.except("used").values.any? { |value| [true, "true"].include?(value) }
|
|
369
453
|
end
|
|
370
454
|
|
|
371
455
|
private
|
|
@@ -432,6 +516,18 @@ module Moderate
|
|
|
432
516
|
self.resolution_note = resolution_note.to_s.strip.presence
|
|
433
517
|
end
|
|
434
518
|
|
|
519
|
+
# An in-app reporter's free text is optional: the category they picked IS the
|
|
520
|
+
# reason (a person who was just insulted shouldn't have to write an essay). The
|
|
521
|
+
# column is NOT NULL and the queue needs something to read, so a blank message
|
|
522
|
+
# becomes the category's label, in the reporter's language. A DSA notice must
|
|
523
|
+
# still be substantiated in the notifier's own words (Art. 16(2)(a)), so this
|
|
524
|
+
# never applies to one.
|
|
525
|
+
def default_community_message
|
|
526
|
+
return unless community? && message.blank? && category.present?
|
|
527
|
+
|
|
528
|
+
self.message = Moderate.taxonomy_label(:category, category)
|
|
529
|
+
end
|
|
530
|
+
|
|
435
531
|
# When an authenticated user files the report, copy their contact onto the
|
|
436
532
|
# notifier fields so DSA notices and community reports carry the same identity
|
|
437
533
|
# shape downstream. We read `email`/`display_name` via `try` so the host's user
|
|
@@ -445,11 +541,31 @@ module Moderate
|
|
|
445
541
|
# (its `reported_owner`, per the Moderate::Reportable interface). This is how a
|
|
446
542
|
# report against a piece of content becomes a report "about" the user who posted
|
|
447
543
|
# it — without this model knowing anything about the host's ownership model.
|
|
544
|
+
# Only a person is stored: an owner that isn't the configured user class (a
|
|
545
|
+
# support desk, a bot) means "nobody to reach" — see Moderate.responsible_user.
|
|
448
546
|
def infer_reported_user
|
|
449
547
|
return if reported_user.present?
|
|
450
|
-
return unless reportable.respond_to?(:reported_owner)
|
|
451
548
|
|
|
452
|
-
self.reported_user = reportable
|
|
549
|
+
self.reported_user = Moderate.responsible_user(reportable)
|
|
550
|
+
end
|
|
551
|
+
|
|
552
|
+
# The generic content bucket, from the reportable when the caller didn't name
|
|
553
|
+
# one: `moderation_content_type` (with or without the field — chats' takes none),
|
|
554
|
+
# kept if it is one of CONTENT_TYPES and filed as "other" if it isn't, so a
|
|
555
|
+
# host's own word ("ride") can never fail an intake. A caller's explicit value
|
|
556
|
+
# (the DSA notice form's select) is left alone.
|
|
557
|
+
def infer_content_type
|
|
558
|
+
return if content_type.present?
|
|
559
|
+
|
|
560
|
+
value = reportable_content_type
|
|
561
|
+
self.content_type = CONTENT_TYPES.include?(value) ? value : "other" if value
|
|
562
|
+
end
|
|
563
|
+
|
|
564
|
+
def reportable_content_type
|
|
565
|
+
return unless reportable.respond_to?(:moderation_content_type)
|
|
566
|
+
|
|
567
|
+
reader = reportable.method(:moderation_content_type)
|
|
568
|
+
(reader.arity.zero? ? reader.call : reader.call(reported_field)).to_s.squish.presence
|
|
453
569
|
end
|
|
454
570
|
|
|
455
571
|
# Capture an IMMUTABLE evidence snapshot at create time. This is the heart of a
|
|
@@ -472,11 +588,18 @@ module Moderate
|
|
|
472
588
|
legal_country_code: legal_country_code,
|
|
473
589
|
content_type: content_type,
|
|
474
590
|
reported_account_identifier: reported_account_identifier,
|
|
475
|
-
content_text:
|
|
591
|
+
content_text: live_content_text,
|
|
476
592
|
captured_at: Time.current.iso8601
|
|
477
593
|
}.compact
|
|
478
594
|
end
|
|
479
595
|
|
|
596
|
+
# The reported field's text now, asked of the reportable (what the snapshot keeps).
|
|
597
|
+
def live_content_text
|
|
598
|
+
return unless reportable.respond_to?(:moderation_snapshot)
|
|
599
|
+
|
|
600
|
+
reportable.moderation_snapshot(reported_field)
|
|
601
|
+
end
|
|
602
|
+
|
|
480
603
|
def capture_automated_processing
|
|
481
604
|
evidence = automated_processing_evidence
|
|
482
605
|
return if evidence.blank?
|
|
@@ -504,12 +627,50 @@ module Moderate
|
|
|
504
627
|
# isn't a managed reportable), we don't constrain the field.
|
|
505
628
|
def reportable_field_must_be_allowed
|
|
506
629
|
return if reportable.blank?
|
|
630
|
+
return if escalated_from_this_flag?
|
|
507
631
|
return unless reportable.respond_to?(:reportable_field_allowed?)
|
|
508
632
|
return if reportable.reportable_field_allowed?(reported_field)
|
|
509
633
|
|
|
510
634
|
errors.add(:reported_field, I18n.t("moderate.errors.invalid_reported_field", default: "is not a reportable field"))
|
|
511
635
|
end
|
|
512
636
|
|
|
637
|
+
# An in-app (community) report from a signed-in person must be about something
|
|
638
|
+
# that person may report: the reportable's own `report_visible_to?` (never your
|
|
639
|
+
# own content; for chats, only a conversation you're in). The report affordance
|
|
640
|
+
# hides itself on that rule, but a hidden link is not a check: a signed target
|
|
641
|
+
# pasted or replayed from elsewhere reached the model unchecked until now. DSA
|
|
642
|
+
# notices (anyone may notify) and system reports (no reporter) are not affected.
|
|
643
|
+
# The self-report and field rules above already explain their own refusals.
|
|
644
|
+
def reporter_must_be_allowed_to_report
|
|
645
|
+
return unless visibility_checked?
|
|
646
|
+
return if reportable.report_visible_to?(reporter, field: reported_field)
|
|
647
|
+
|
|
648
|
+
message = I18n.t("moderate.errors.not_reportable_by_reporter", default: "isn't something you can report")
|
|
649
|
+
errors.add(:reportable, message)
|
|
650
|
+
end
|
|
651
|
+
|
|
652
|
+
# The Moderate::Flag a system report was opened from (Flag#action! enforcing).
|
|
653
|
+
# Such a report may name the flag's field even if people can't report that field.
|
|
654
|
+
# Private: only Services::ResolveFlag sets it, so it isn't mass-assignable or a
|
|
655
|
+
# general way around the field whitelist.
|
|
656
|
+
attr_accessor :escalated_from_flag
|
|
657
|
+
|
|
658
|
+
# The report was opened from a flag on THIS content and field (Flag#action!) —
|
|
659
|
+
# a system report: no reporter, community intake, never a DSA notice.
|
|
660
|
+
def escalated_from_this_flag?
|
|
661
|
+
flag = escalated_from_flag
|
|
662
|
+
return false unless flag.is_a?(Moderate::Flag) && community? && reporter.nil?
|
|
663
|
+
|
|
664
|
+
flag.field.to_s == reported_field.to_s &&
|
|
665
|
+
flag.flaggable_type == reportable_type && flag.flaggable_id.to_s == reportable_id.to_s
|
|
666
|
+
end
|
|
667
|
+
|
|
668
|
+
def visibility_checked?
|
|
669
|
+
return false unless community? && reporter.present? && reportable.respond_to?(:report_visible_to?)
|
|
670
|
+
|
|
671
|
+
errors.exclude?(:reported_user) && errors.exclude?(:reported_field)
|
|
672
|
+
end
|
|
673
|
+
|
|
513
674
|
def subject_url_must_be_http_url
|
|
514
675
|
invalid_urls = subject_url_list.reject { |url| parsed_subject_uri(url).present? }
|
|
515
676
|
return if invalid_urls.empty?
|
|
@@ -545,7 +706,9 @@ module Moderate
|
|
|
545
706
|
return unless anonymous?
|
|
546
707
|
return if legal_reason == "protection_of_minors"
|
|
547
708
|
|
|
548
|
-
errors.add(:anonymous,
|
|
709
|
+
errors.add(:anonymous,
|
|
710
|
+
I18n.t("moderate.errors.anonymous_notice_child_safety_only",
|
|
711
|
+
default: "anonymous notices are only allowed for offences against minors"))
|
|
549
712
|
end
|
|
550
713
|
|
|
551
714
|
# Accept either a multi-URL list or newline-separated text, fold in the single
|
|
@@ -579,7 +742,8 @@ module Moderate
|
|
|
579
742
|
def automated_processing_evidence
|
|
580
743
|
evidence = {}
|
|
581
744
|
|
|
582
|
-
|
|
745
|
+
# A manual flag is an administrator's own judgment, not automated means.
|
|
746
|
+
matching_flags = flags.reject { |flag| flag.source == "manual" }
|
|
583
747
|
if matching_flags.any?
|
|
584
748
|
evidence["used"] = true
|
|
585
749
|
evidence["flag_ids"] = matching_flags.map(&:id)
|
|
@@ -591,7 +755,7 @@ module Moderate
|
|
|
591
755
|
if result&.flagged?
|
|
592
756
|
evidence["used"] = true
|
|
593
757
|
evidence["field"] = reported_field
|
|
594
|
-
evidence["sources"] = Array(evidence["sources"]).
|
|
758
|
+
evidence["sources"] = Array(evidence["sources"]).push(result.source).compact.uniq
|
|
595
759
|
evidence["categories"] = Array(evidence["categories"]).concat(result.categories).compact.uniq
|
|
596
760
|
evidence["scores"] = result.scores
|
|
597
761
|
# `raw` is the untouched provider payload (Moderate::Result#raw) — kept for
|
|
@@ -611,7 +775,7 @@ module Moderate
|
|
|
611
775
|
policy = Moderate.filter_policy_for(reportable, reported_field)
|
|
612
776
|
return unless policy.block?
|
|
613
777
|
|
|
614
|
-
value =
|
|
778
|
+
value = live_content_text
|
|
615
779
|
return if value.blank?
|
|
616
780
|
|
|
617
781
|
Moderate.classify(value, policy: policy)
|
data/lib/moderate/result.rb
CHANGED
|
@@ -109,15 +109,8 @@ module Moderate
|
|
|
109
109
|
# Reconcile the two accepted shapes (`labels:` and `categories:`+`scores:`)
|
|
110
110
|
# into one frozen array of `Moderate::Label`.
|
|
111
111
|
def build_labels(labels, categories, scores_hash)
|
|
112
|
-
out =
|
|
113
|
-
|
|
114
|
-
Array(labels).each do |label|
|
|
115
|
-
out << coerce_label(label, scores_hash)
|
|
116
|
-
end
|
|
117
|
-
|
|
118
|
-
Array(categories).each do |category|
|
|
119
|
-
out << label_from_slug(category.to_s, scores_hash)
|
|
120
|
-
end
|
|
112
|
+
out = Array(labels).map { |label| coerce_label(label, scores_hash) }
|
|
113
|
+
out += Array(categories).map { |category| label_from_slug(category.to_s, scores_hash) }
|
|
121
114
|
|
|
122
115
|
out
|
|
123
116
|
end
|
|
@@ -145,7 +138,7 @@ module Moderate
|
|
|
145
138
|
end
|
|
146
139
|
|
|
147
140
|
def label_from_hash(hash, scores_hash)
|
|
148
|
-
hash = hash.transform_keys
|
|
141
|
+
hash = hash.transform_keys(&:to_s)
|
|
149
142
|
category = hash["category"]
|
|
150
143
|
subcategory = hash["subcategory"]
|
|
151
144
|
slug = subcategory ? "#{category}/#{subcategory}" : category.to_s
|