moderate 1.0.0.beta2 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +96 -1
- data/.simplecov +2 -10
- data/AGENTS.md +1 -1
- data/Appraisals +10 -8
- data/CHANGELOG.md +654 -52
- data/CLAUDE.md +1 -1
- data/README.md +147 -7
- data/Rakefile +2 -0
- data/app/controllers/concerns/moderate/moderation.rb +23 -11
- data/app/controllers/moderate/appeals_controller.rb +9 -5
- data/app/controllers/moderate/application_controller.rb +3 -3
- data/app/controllers/moderate/notices_controller.rb +3 -6
- data/app/helpers/moderate/engine_helper.rb +32 -11
- data/app/views/moderate/appeals/new.html.erb +6 -0
- data/app/views/moderate/transparency_reports/_summary_card.html.erb +2 -2
- data/app/views/moderate/transparency_reports/show.html.erb +5 -5
- data/config/locales/en.yml +200 -0
- data/config/locales/es.yml +177 -0
- data/db/migrate/20261018200000_add_pending_key_to_moderate_flags.rb +139 -0
- data/db/migrate/20261018210000_add_retention_and_classifications_to_moderate_tables.rb +103 -0
- data/docs/chats.md +140 -0
- data/docs/compliance.md +92 -3
- data/docs/configuration.md +30 -2
- data/docs/dsa-notice-form.md +1 -1
- data/docs/madmin.md +44 -14
- data/docs/notifications.md +17 -4
- data/examples/aws_rekognition_adapter.rb +6 -6
- data/examples/openai_moderation_adapter.rb +1 -1
- data/examples/regexp_pattern_adapter.rb +68 -0
- data/gemfiles/{rails_7.1.gemfile → rails_8.0.gemfile} +8 -6
- data/gemfiles/rails_8.1.gemfile +6 -5
- data/lib/generators/moderate/install_generator.rb +25 -0
- data/lib/generators/moderate/templates/create_moderate_tables.rb.erb +17 -2
- data/lib/generators/moderate/templates/initializer.rb +9 -15
- data/lib/generators/moderate/views_generator.rb +3 -3
- data/lib/moderate/attachments.rb +137 -0
- data/lib/moderate/configuration.rb +87 -12
- data/lib/moderate/engine.rb +86 -25
- data/lib/moderate/event.rb +21 -0
- data/lib/moderate/filters/base.rb +1 -0
- data/lib/moderate/filters/wordlist.rb +5 -2
- data/lib/moderate/jobs/classify_job.rb +96 -16
- data/lib/moderate/label.rb +2 -2
- data/lib/moderate/macros.rb +8 -3
- data/lib/moderate/models/appeal.rb +10 -1
- data/lib/moderate/models/block.rb +23 -19
- data/lib/moderate/models/classification.rb +24 -0
- data/lib/moderate/models/concerns/actor.rb +17 -2
- data/lib/moderate/models/concerns/content_filterable.rb +181 -55
- data/lib/moderate/models/concerns/deferred_notices.rb +90 -0
- data/lib/moderate/models/concerns/reportable.rb +84 -4
- data/lib/moderate/models/flag.rb +182 -19
- data/lib/moderate/models/report.rb +180 -16
- data/lib/moderate/result.rb +3 -10
- data/lib/moderate/retention.rb +241 -0
- data/lib/moderate/services/intake_notice.rb +10 -10
- data/lib/moderate/services/intake_report.rb +42 -13
- data/lib/moderate/services/resolve_appeal.rb +8 -2
- data/lib/moderate/services/resolve_flag.rb +183 -35
- data/lib/moderate/services/resolve_report.rb +288 -35
- data/lib/moderate/text_validator.rb +15 -14
- data/lib/moderate/version.rb +1 -1
- data/lib/moderate.rb +337 -17
- data/lib/tasks/moderate.rake +23 -0
- metadata +22 -14
- data/gemfiles/rails_7.2.gemfile +0 -36
- data/lib/moderate/text.rb +0 -38
- data/lib/moderate/word_list.rb +0 -90
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Moderate
|
|
4
|
+
# Opt-in retention: strip the personal data out of moderation records nobody can
|
|
5
|
+
# act on any more, and keep the decisions.
|
|
6
|
+
#
|
|
7
|
+
# Moderate.purge(older_than: 1.year) # what a nightly job runs
|
|
8
|
+
# Moderate.purge(older_than: 18.months, dry_run: true) # => counts, nothing changed
|
|
9
|
+
# bin/rails moderate:purge OLDER_THAN_DAYS=365 DRY_RUN=1
|
|
10
|
+
#
|
|
11
|
+
# Nothing in the gem calls this: until a host schedules it, every report, flag,
|
|
12
|
+
# appeal and block is kept as it was written. docs/compliance.md ("Retention")
|
|
13
|
+
# says what is stored, what this removes, and why the rest stays.
|
|
14
|
+
#
|
|
15
|
+
# What it touches — only records whose decision is final:
|
|
16
|
+
#
|
|
17
|
+
# * a REPORT (in-app report or DSA notice) that is closed, was decided more than
|
|
18
|
+
# `older_than` ago, whose appeal window has closed, and that has no open
|
|
19
|
+
# appeal: its message, evidence snapshot, reported URLs, reported account
|
|
20
|
+
# identifier, notifier name and email, and reporter are removed;
|
|
21
|
+
# * a FLAG that was decided more than `older_than` ago: its excerpt and its
|
|
22
|
+
# context (the classifier's raw response, a reviewer's note) are removed;
|
|
23
|
+
# * an APPEAL that was decided more than `older_than` ago: its reason and the
|
|
24
|
+
# appellant's name and email are removed.
|
|
25
|
+
#
|
|
26
|
+
# What it keeps: every row, with its type, category, legal ground, member state,
|
|
27
|
+
# content type, status, timestamps, the decision (resolution basis, actions,
|
|
28
|
+
# note, decision visibility, automated-means disclosure) and who it was about
|
|
29
|
+
# (reported user, flag owner). Those are the statement of reasons, the record a
|
|
30
|
+
# repeat-misuse policy (DSA Art. 23) and a legal claim rely on, and what
|
|
31
|
+
# `Moderate.transparency` counts, so a published Art. 15/24 report stays
|
|
32
|
+
# reproducible. Blocks are never touched: they are live safety state, not a
|
|
33
|
+
# record of the past, and go when either person unblocks or is deleted.
|
|
34
|
+
#
|
|
35
|
+
# Pending reports, flags and appeals are never touched, whatever their age.
|
|
36
|
+
module Retention
|
|
37
|
+
# What a removed text column reads afterwards (message, excerpt and reason are
|
|
38
|
+
# NOT NULL). Only a filler: whether a row is done is its `redacted_at`.
|
|
39
|
+
REDACTED = "[redacted]"
|
|
40
|
+
|
|
41
|
+
# The default age, counted from the decision. Six months of appeal window
|
|
42
|
+
# (Art. 20), the time an out-of-court dispute settlement body may take (Art. 21),
|
|
43
|
+
# and a margin for a complaint or legal claim that arrives late.
|
|
44
|
+
DEFAULT_AGE = 1.year
|
|
45
|
+
|
|
46
|
+
# `older_than` may not be shorter than the appeal window: a decision that can
|
|
47
|
+
# still be appealed must keep its evidence.
|
|
48
|
+
def self.minimum_age
|
|
49
|
+
Moderate::Report::APPEAL_WINDOW
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# Redact what is old enough, and return how many rows of each kind changed (or
|
|
53
|
+
# would change, with `dry_run: true`), plus, under `:held`, how many were old
|
|
54
|
+
# enough but kept for a legal hold. Audited as `:retention_purged` when anything
|
|
55
|
+
# changed. Idempotent: a row is selected while its `redacted_at` is nil, whatever
|
|
56
|
+
# its content says, and stamped when it is redacted.
|
|
57
|
+
#
|
|
58
|
+
# Moderate.purge # => { reports: 3, flags: 9, appeals: 1, classifications: 40,
|
|
59
|
+
# # held: { reports: 1, flags: 2, appeals: 0 } }
|
|
60
|
+
def self.purge(older_than: DEFAULT_AGE, now: Time.current, dry_run: false)
|
|
61
|
+
age = older_than.to_i
|
|
62
|
+
if older_than.nil? || age < minimum_age.to_i
|
|
63
|
+
raise ArgumentError,
|
|
64
|
+
"older_than must be at least the #{minimum_age.inspect} appeal window (got #{older_than.inspect}): " \
|
|
65
|
+
"a decision that can still be appealed keeps its evidence"
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
cutoff = now - older_than
|
|
69
|
+
due = { reports: reports_due(cutoff, now), flags: flags_due(cutoff), appeals: appeals_due(cutoff) }
|
|
70
|
+
held = due.to_h { |kind, scope| [kind, scope.where(hold_condition(kind, now, held: true)).count] }
|
|
71
|
+
free = due.to_h { |kind, scope| [kind, scope.where(hold_condition(kind, now, held: false))] }
|
|
72
|
+
classifications = classifications_due(cutoff, now)
|
|
73
|
+
kept = { claims_kept: live_claims(cutoff, now).count, held: held }
|
|
74
|
+
return free.transform_values(&:count).merge(classifications: classifications.count, **kept) if dry_run
|
|
75
|
+
|
|
76
|
+
counts = {
|
|
77
|
+
reports: redact(free[:reports], report_redaction(now)),
|
|
78
|
+
flags: redact(free[:flags], flag_redaction(now)),
|
|
79
|
+
appeals: redact(free[:appeals], appeal_redaction(now)),
|
|
80
|
+
classifications: delete(classifications)
|
|
81
|
+
}
|
|
82
|
+
audit(counts, held, cutoff: cutoff, older_than: age) if counts.values.sum.positive?
|
|
83
|
+
counts.merge(kept)
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
# Put `report` under a legal hold until `until:` (a future time): the purge leaves
|
|
87
|
+
# the report, its appeals and the flags on its content alone, however old they
|
|
88
|
+
# are — an out-of-court dispute (DSA Art. 21), a court case, an authority's
|
|
89
|
+
# request. Placing it again moves the date (later or earlier). Audited as
|
|
90
|
+
# `:legal_hold_placed`.
|
|
91
|
+
def self.hold(report, until:, by: nil, reason: nil)
|
|
92
|
+
held_until = binding.local_variable_get(:until)
|
|
93
|
+
unless held_until.respond_to?(:to_time) && held_until.to_time > Time.current
|
|
94
|
+
raise ArgumentError, "a legal hold needs a time in the future (until: #{held_until.inspect})"
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
report.update_columns(held_until: held_until, updated_at: Time.current)
|
|
98
|
+
Moderate.audit(:legal_hold_placed, **hold_event(report, by: by, reason: reason, held_until: held_until))
|
|
99
|
+
report
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# Lift `report`'s legal hold: the next purge treats it like any other report.
|
|
103
|
+
# Audited as `:legal_hold_released`. A report with no hold is returned unchanged.
|
|
104
|
+
def self.release_hold(report, by: nil, reason: nil)
|
|
105
|
+
return report if report.held_until.nil?
|
|
106
|
+
|
|
107
|
+
report.update_columns(held_until: nil, updated_at: Time.current)
|
|
108
|
+
Moderate.audit(:legal_hold_released, **hold_event(report, by: by, reason: reason, held_until: nil))
|
|
109
|
+
report
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
# Closed reports decided before `cutoff`, out of their appeal window and with
|
|
113
|
+
# no appeal still open, not redacted yet.
|
|
114
|
+
def self.reports_due(cutoff, now)
|
|
115
|
+
reports = Moderate::Report
|
|
116
|
+
open_appeals = Moderate::Appeal.pending.select(:report_id)
|
|
117
|
+
|
|
118
|
+
reports.where(status: reports::STATUSES - ["open"], redacted_at: nil)
|
|
119
|
+
.where(resolved_at: ..cutoff)
|
|
120
|
+
.where(reports.arel_table[:appeal_deadline_at].eq(nil).or(reports.arel_table[:appeal_deadline_at].lteq(now)))
|
|
121
|
+
.where.not(id: open_appeals)
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
def self.flags_due(cutoff)
|
|
125
|
+
flags = Moderate::Flag
|
|
126
|
+
flags.where(status: flags::STATUSES - ["pending"], redacted_at: nil).where(reviewed_at: ..cutoff)
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
# Attachment classification bookkeeping (Moderate::Classification: ids, a digest,
|
|
130
|
+
# timestamps) last touched before `cutoff`: classified then, or a claim nobody
|
|
131
|
+
# completed. Deleted, not redacted: nothing in them is evidence. Never a row with
|
|
132
|
+
# a LIVE claim (younger than config.classification_claim_ttl), whatever its age:
|
|
133
|
+
# a worker is classifying those files right now, and would discard its result.
|
|
134
|
+
def self.classifications_due(cutoff, now)
|
|
135
|
+
rows = Moderate::Classification.arel_table
|
|
136
|
+
Moderate::Classification.where(updated_at: ..cutoff)
|
|
137
|
+
.where(rows[:claimed_at].eq(nil).or(rows[:claimed_at].lt(now - Moderate::Attachments.claim_ttl)))
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
# Rows the age cutoff would take but a live claim keeps (`:claims_kept`).
|
|
141
|
+
def self.live_claims(cutoff, now)
|
|
142
|
+
Moderate::Classification.where(updated_at: ..cutoff).where(claimed_at: (now - Moderate::Attachments.claim_ttl)..)
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
def self.delete(scope)
|
|
146
|
+
deleted = 0
|
|
147
|
+
scope.in_batches(of: 500) { |batch| deleted += batch.delete_all }
|
|
148
|
+
deleted
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
def self.appeals_due(cutoff)
|
|
152
|
+
appeals = Moderate::Appeal
|
|
153
|
+
appeals.where(status: appeals::STATUSES - ["open"], redacted_at: nil).where(resolved_at: ..cutoff)
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
# Arel for "kept by a legal hold" (held: true) or "not" (held: false): the
|
|
157
|
+
# report's own hold; an appeal's report's; for a flag, a held report on the same
|
|
158
|
+
# content and field (or on the whole record). Written both ways rather than
|
|
159
|
+
# negated, because NOT (held_until > now) is NULL, not true, for a report that
|
|
160
|
+
# was never held.
|
|
161
|
+
def self.hold_condition(kind, now, held:)
|
|
162
|
+
reports = Moderate::Report.arel_table
|
|
163
|
+
held_reports = Moderate::Report.where(reports[:held_until].gt(now))
|
|
164
|
+
case kind
|
|
165
|
+
when :reports
|
|
166
|
+
held ? reports[:held_until].gt(now) : reports[:held_until].eq(nil).or(reports[:held_until].lteq(now))
|
|
167
|
+
when :appeals
|
|
168
|
+
ids = held_reports.select(:id).arel
|
|
169
|
+
held ? Moderate::Appeal.arel_table[:report_id].in(ids) : Moderate::Appeal.arel_table[:report_id].not_in(ids)
|
|
170
|
+
when :flags
|
|
171
|
+
flags = Moderate::Flag.arel_table
|
|
172
|
+
same_content = held_reports
|
|
173
|
+
.where(reports[:reportable_type].eq(flags[:flaggable_type]))
|
|
174
|
+
.where(reports[:reportable_id].eq(flags[:flaggable_id]))
|
|
175
|
+
.where(reports[:reported_field].eq(flags[:field]).or(reports[:reported_field].eq(nil)))
|
|
176
|
+
.arel.exists
|
|
177
|
+
held ? same_content : same_content.not
|
|
178
|
+
end
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
def self.report_redaction(now)
|
|
182
|
+
{
|
|
183
|
+
message: REDACTED,
|
|
184
|
+
snapshot: { "redacted_at" => now.iso8601 },
|
|
185
|
+
subject_url: nil,
|
|
186
|
+
subject_urls: [],
|
|
187
|
+
reported_account_identifier: nil,
|
|
188
|
+
notifier_name: nil,
|
|
189
|
+
notifier_email: nil,
|
|
190
|
+
reporter_id: nil,
|
|
191
|
+
redacted_at: now,
|
|
192
|
+
updated_at: now
|
|
193
|
+
}
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
def self.flag_redaction(now)
|
|
197
|
+
{ excerpt: REDACTED, context: { "redacted_at" => now.iso8601 }, redacted_at: now, updated_at: now }
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
def self.appeal_redaction(now)
|
|
201
|
+
{ reason: REDACTED, appellant_name: nil, appellant_email: nil, redacted_at: now, updated_at: now }
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
# One UPDATE per batch, no callbacks or validations: a redacted DSA notice no
|
|
205
|
+
# longer carries the notifier details its validations ask for, and nothing
|
|
206
|
+
# should re-save a final record anyway.
|
|
207
|
+
def self.redact(scope, attributes)
|
|
208
|
+
changed = 0
|
|
209
|
+
scope.in_batches(of: 500) { |batch| changed += batch.update_all(attributes) }
|
|
210
|
+
changed
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
def self.audit(counts, held, cutoff:, older_than:)
|
|
214
|
+
Moderate.audit(
|
|
215
|
+
:retention_purged,
|
|
216
|
+
payload: counts.merge(
|
|
217
|
+
held: held,
|
|
218
|
+
cutoff: cutoff.iso8601,
|
|
219
|
+
older_than_seconds: older_than,
|
|
220
|
+
summary: "retention: redacted #{counts[:reports]} reports, #{counts[:flags]} flags, " \
|
|
221
|
+
"#{counts[:appeals]} appeals decided before #{cutoff.to_date}; deleted " \
|
|
222
|
+
"#{counts[:classifications]} classification rows"
|
|
223
|
+
)
|
|
224
|
+
)
|
|
225
|
+
end
|
|
226
|
+
|
|
227
|
+
def self.hold_event(report, by:, reason:, held_until:)
|
|
228
|
+
{
|
|
229
|
+
subject: report,
|
|
230
|
+
actor: by,
|
|
231
|
+
payload: {
|
|
232
|
+
report_id: report.id, held_until: held_until&.iso8601, reason: reason,
|
|
233
|
+
summary: held_until ? "Report ##{report.id} held until #{held_until.to_date}" : "Report ##{report.id} hold released"
|
|
234
|
+
}.compact
|
|
235
|
+
}
|
|
236
|
+
end
|
|
237
|
+
|
|
238
|
+
private_class_method :reports_due, :flags_due, :appeals_due, :classifications_due, :live_claims, :delete,
|
|
239
|
+
:hold_condition, :report_redaction, :flag_redaction, :appeal_redaction, :redact, :audit, :hold_event
|
|
240
|
+
end
|
|
241
|
+
end
|
|
@@ -69,6 +69,15 @@ module Moderate
|
|
|
69
69
|
true
|
|
70
70
|
end
|
|
71
71
|
|
|
72
|
+
# The anonymous-notifier recipient. Intentionally tiny and NOT a User: the
|
|
73
|
+
# host's notify hook reaches `recipient.email` / `recipient.name` and nothing
|
|
74
|
+
# else. Frozen value object.
|
|
75
|
+
NotifierRecipient = Data.define(:email, :name) do
|
|
76
|
+
# Mirror the User-ish reader some host mailers reach for, so a notifier and a
|
|
77
|
+
# User are interchangeable at the `recipient.email` call site.
|
|
78
|
+
def display_name = name.to_s.empty? ? email : name
|
|
79
|
+
end
|
|
80
|
+
|
|
72
81
|
private
|
|
73
82
|
|
|
74
83
|
attr_reader :reporter
|
|
@@ -103,7 +112,7 @@ module Moderate
|
|
|
103
112
|
subject_url: report.subject_url,
|
|
104
113
|
# Redaction-safe admin one-liner. We deliberately keep host content out
|
|
105
114
|
# of it — just the legal ground and an opaque pointer to the record.
|
|
106
|
-
summary: "New DSA notice (#{report.legal_reason ||
|
|
115
|
+
summary: "New DSA notice (#{report.legal_reason || "illegal_content"}) — Report ##{report.id}"
|
|
107
116
|
}
|
|
108
117
|
)
|
|
109
118
|
end
|
|
@@ -118,15 +127,6 @@ module Moderate
|
|
|
118
127
|
|
|
119
128
|
NotifierRecipient.new(report.notifier_email, report.notifier_name)
|
|
120
129
|
end
|
|
121
|
-
|
|
122
|
-
# The anonymous-notifier recipient. Intentionally tiny and NOT a User: the
|
|
123
|
-
# host's notify hook reaches `recipient.email` / `recipient.name` and nothing
|
|
124
|
-
# else. Frozen value object.
|
|
125
|
-
NotifierRecipient = Data.define(:email, :name) do
|
|
126
|
-
# Mirror the User-ish reader some host mailers reach for, so a notifier and a
|
|
127
|
-
# User are interchangeable at the `recipient.email` call site.
|
|
128
|
-
def display_name = name.to_s.empty? ? email : name
|
|
129
|
-
end
|
|
130
130
|
end
|
|
131
131
|
end
|
|
132
132
|
end
|
|
@@ -51,18 +51,27 @@ module Moderate
|
|
|
51
51
|
# failed (the report carries its errors, Rails-conventionally) so a controller
|
|
52
52
|
# can `if intake.save ... else render :new` exactly like a bare model save.
|
|
53
53
|
def save
|
|
54
|
-
|
|
54
|
+
# The report and, for "report and block", the block are ONE act: both commit
|
|
55
|
+
# or neither does (a host on_block hook that raises rolls the report back
|
|
56
|
+
# too, and the error reaches the caller), and the receipt goes out only after.
|
|
57
|
+
Moderate::Report.transaction do
|
|
58
|
+
# Captured before the save (Moderate::DeferredNotices).
|
|
59
|
+
connection = Moderate::DeferredNotices.connection_for(Moderate::Report)
|
|
60
|
+
next false unless report.save
|
|
55
61
|
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
62
|
+
# DSA Art. 16(4): the provider must confirm receipt "without undue delay."
|
|
63
|
+
# `acknowledge!` stamps `acknowledged_at` — the durable, on-record proof of
|
|
64
|
+
# receipt — BEFORE we attempt the (best-effort, possibly-undelivered) email,
|
|
65
|
+
# so the legal obligation is met by the database fact, not by a mailer that
|
|
66
|
+
# might not be wired. See: https://eur-lex.europa.eu/eli/reg/2022/2065/oj
|
|
67
|
+
report.acknowledge!
|
|
68
|
+
block_reported_user
|
|
69
|
+
# The receipt waits for the report to be durable (Moderate::DeferredNotices):
|
|
70
|
+
# never a "we got your report" for a report a surrounding transaction undid.
|
|
71
|
+
report.after_commit_notice(on: connection) { deliver_receipt }
|
|
72
|
+
audit_intake
|
|
73
|
+
true
|
|
74
|
+
end
|
|
66
75
|
end
|
|
67
76
|
|
|
68
77
|
private
|
|
@@ -87,10 +96,13 @@ module Moderate
|
|
|
87
96
|
payload: {
|
|
88
97
|
category: report.category,
|
|
89
98
|
intake_kind: report.intake_kind,
|
|
99
|
+
reported_field: report.reported_field,
|
|
100
|
+
reported_field_label: field_label,
|
|
90
101
|
# `:summary` is the contract every event carries — a redaction-safe,
|
|
91
102
|
# ready-to-send one-liner for the admin Telegram ping (docs/notifications.md).
|
|
92
|
-
|
|
93
|
-
|
|
103
|
+
# The field label is host-declared copy, never user content, so it's safe here.
|
|
104
|
+
summary: ["New #{report.category} report on #{reportable_label}", field_label].compact.join(" · ")
|
|
105
|
+
}.compact
|
|
94
106
|
)
|
|
95
107
|
end
|
|
96
108
|
|
|
@@ -114,10 +126,27 @@ module Moderate
|
|
|
114
126
|
)
|
|
115
127
|
end
|
|
116
128
|
|
|
129
|
+
# "Report and block": the reporter asked to block the person responsible too.
|
|
130
|
+
# After the report is safely filed, through the one block path (Block.block!:
|
|
131
|
+
# idempotent, audited, on_block, user_blocked). A self-block is a no-op there.
|
|
132
|
+
def block_reported_user
|
|
133
|
+
return unless ActiveModel::Type::Boolean.new.cast(report.block_reported_user)
|
|
134
|
+
|
|
135
|
+
reporter = report.reporter
|
|
136
|
+
blocked = report.reported_user
|
|
137
|
+
return if reporter.blank? || blocked.blank? || !reporter.respond_to?(:block!)
|
|
138
|
+
|
|
139
|
+
reporter.block!(blocked)
|
|
140
|
+
end
|
|
141
|
+
|
|
117
142
|
def recipient_email
|
|
118
143
|
report.notifier_email
|
|
119
144
|
end
|
|
120
145
|
|
|
146
|
+
def field_label
|
|
147
|
+
report.reported_field_label
|
|
148
|
+
end
|
|
149
|
+
|
|
121
150
|
# The reportable's own human label, or a neutral fallback — NEVER a host-
|
|
122
151
|
# specific string. The Reportable concern supplies `moderation_label`.
|
|
123
152
|
def reportable_label
|
|
@@ -45,8 +45,13 @@ module Moderate
|
|
|
45
45
|
def transition!(status, note)
|
|
46
46
|
note = require_note!(note)
|
|
47
47
|
|
|
48
|
+
# Captured before any save (Moderate::DeferredNotices).
|
|
49
|
+
connection = Moderate::DeferredNotices.connection_for(appeal.class)
|
|
50
|
+
|
|
48
51
|
appeal.with_lock do
|
|
49
|
-
|
|
52
|
+
# `with_lock` already re-read the row under the lock (SELECT … FOR UPDATE):
|
|
53
|
+
# no plain reload here — under MySQL's REPEATABLE READ a plain read pins the
|
|
54
|
+
# transaction's snapshot, and every later check must see what committed.
|
|
50
55
|
|
|
51
56
|
# A decided appeal is immutable — re-check inside the lock so a concurrent
|
|
52
57
|
# second decision bails cleanly instead of overwriting the first.
|
|
@@ -63,9 +68,10 @@ module Moderate
|
|
|
63
68
|
)
|
|
64
69
|
|
|
65
70
|
audit_decision(status, note)
|
|
71
|
+
# Sent once the decision is durable (Moderate::DeferredNotices).
|
|
72
|
+
appeal.after_commit_notice(on: connection) { deliver_decision_notice(status) }
|
|
66
73
|
end
|
|
67
74
|
|
|
68
|
-
deliver_decision_notice(status)
|
|
69
75
|
appeal
|
|
70
76
|
end
|
|
71
77
|
|
|
@@ -2,40 +2,48 @@
|
|
|
2
2
|
|
|
3
3
|
module Moderate
|
|
4
4
|
module Services
|
|
5
|
-
# ResolveFlag — the decision on
|
|
6
|
-
#
|
|
5
|
+
# ResolveFlag — the decision on a `Moderate::Flag`, behind the flag's
|
|
6
|
+
# `action!` / `dismiss!` (and `resolve!`, the queue-wide verb).
|
|
7
7
|
#
|
|
8
8
|
# A Flag is a SYSTEM-raised queue item (the wordlist/image/external classifier
|
|
9
|
-
# tripped on a `:flag`-mode field, or
|
|
10
|
-
# "ongoing moderation" surface Apple Guideline 1.2 and Google Play UGC expect:
|
|
9
|
+
# tripped on a `:flag`-mode field, or an administrator filed one by hand) — it's
|
|
10
|
+
# the "ongoing moderation" surface Apple Guideline 1.2 and Google Play UGC expect:
|
|
11
11
|
# - https://developer.apple.com/app-store/review/guidelines/#user-generated-content
|
|
12
12
|
# - https://support.google.com/googleplay/android-developer/answer/9876937
|
|
13
13
|
#
|
|
14
|
-
#
|
|
15
|
-
# window to stamp (an appeal attaches to a *report* decision, not to a raw
|
|
16
|
-
# queue item). So this service is the simplest of the resolvers — atomic
|
|
17
|
-
# transition, mandatory note, audit — but it follows the same discipline:
|
|
14
|
+
# Same discipline as every resolver:
|
|
18
15
|
# - `with_lock` + re-check `pending?` inside the lock for idempotency under
|
|
19
|
-
# concurrent review
|
|
16
|
+
# concurrent review (a flag already reviewed is returned as-is: a benign
|
|
17
|
+
# "already done" suits a fast triage queue);
|
|
20
18
|
# - a NOTE IS MANDATORY (the moderator's rationale is the audit trail);
|
|
21
|
-
# - the decision is recorded via `Moderate.audit
|
|
19
|
+
# - the decision is recorded via `Moderate.audit` (`:flag_decision`).
|
|
22
20
|
#
|
|
23
|
-
#
|
|
24
|
-
#
|
|
25
|
-
#
|
|
26
|
-
#
|
|
27
|
-
#
|
|
28
|
-
#
|
|
21
|
+
# ENFORCEMENT GOES THROUGH A REPORT. A flag carries no reporter, no statement of
|
|
22
|
+
# reasons and no appeal window — but removing content or suspending an account is
|
|
23
|
+
# a restriction, and DSA Art. 17 (statement of reasons) and Art. 20 (appeal) attach
|
|
24
|
+
# to restrictions, not to queue items. So `action!(remove_content: or ban_user:)`
|
|
25
|
+
# ESCALATES: it takes the open report on the same content and field, or opens a
|
|
26
|
+
# system one (no reporter, no receipt), and resolves it through ResolveReport —
|
|
27
|
+
# the one enforcement path in the gem (remove via `remove_reported_field!`, ban via
|
|
28
|
+
# `ban_handler`, the decision notices, the appeal window). All of it happens under
|
|
29
|
+
# the FLAG's row lock, so two moderators pressing "remove" at once enforce once;
|
|
30
|
+
# the decision notices go out after the lock is released, like every decision.
|
|
31
|
+
#
|
|
32
|
+
# Without enforcement options, `action!` is a pure triage close, as before.
|
|
29
33
|
class ResolveFlag
|
|
30
34
|
def initialize(flag, by:)
|
|
31
35
|
@flag = flag
|
|
32
36
|
@moderator = by
|
|
33
37
|
end
|
|
34
38
|
|
|
35
|
-
# The flagged content was indeed objectionable — mark the flag actioned
|
|
36
|
-
#
|
|
37
|
-
|
|
38
|
-
|
|
39
|
+
# The flagged content was indeed objectionable — mark the flag actioned, and,
|
|
40
|
+
# when asked, enforce through a report (see the class doc). `resolution_basis`
|
|
41
|
+
# is the report's legal/contractual ground (Report::RESOLUTION_BASES).
|
|
42
|
+
def action!(note:, remove_content: false, ban_user: false, resolution_basis: "terms")
|
|
43
|
+
enforcement = { remove_content: truthy?(remove_content), ban_user: truthy?(ban_user) }
|
|
44
|
+
enforcement = nil unless enforcement.values.any?
|
|
45
|
+
|
|
46
|
+
transition!("actioned", note, enforcement: enforcement, resolution_basis: resolution_basis)
|
|
39
47
|
end
|
|
40
48
|
|
|
41
49
|
# False positive / acceptable — dismiss the flag.
|
|
@@ -47,31 +55,165 @@ module Moderate
|
|
|
47
55
|
|
|
48
56
|
attr_reader :flag, :moderator
|
|
49
57
|
|
|
50
|
-
def transition!(status, note)
|
|
58
|
+
def transition!(status, note, enforcement: nil, resolution_basis: nil)
|
|
51
59
|
note = require_note!(note)
|
|
60
|
+
decision = nil
|
|
61
|
+
# Captured before any save: the decision's notices join this connection's
|
|
62
|
+
# transaction, whatever a host callback switches to (Moderate::DeferredNotices).
|
|
63
|
+
connection = Moderate::DeferredNotices.connection_for(flag.class)
|
|
52
64
|
|
|
53
65
|
flag.with_lock do
|
|
54
|
-
|
|
66
|
+
# `with_lock` already re-read the row under the lock (SELECT … FOR UPDATE):
|
|
67
|
+
# no plain reload here — under MySQL's REPEATABLE READ a plain read pins the
|
|
68
|
+
# transaction's snapshot, and every later check must see what committed.
|
|
55
69
|
|
|
56
70
|
# Re-check inside the lock. A flag already reviewed is returned as-is (NOT
|
|
57
|
-
# an error)
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
)
|
|
67
|
-
|
|
68
|
-
audit_decision(status, note)
|
|
71
|
+
# an error) — and nothing is enforced a second time.
|
|
72
|
+
next unless flag.pending?
|
|
73
|
+
|
|
74
|
+
decision, report = enforce!(note, enforcement, resolution_basis) if enforcement
|
|
75
|
+
close!(status, note, report)
|
|
76
|
+
|
|
77
|
+
# The escalated decision's notices, queued on the flag: sent when the
|
|
78
|
+
# outermost transaction commits, never on a rollback (DeferredNotices).
|
|
79
|
+
flag.after_commit_notice(on: connection) { decision.deliver_decision_notices } if decision
|
|
69
80
|
end
|
|
70
81
|
|
|
71
82
|
flag
|
|
72
83
|
end
|
|
73
84
|
|
|
74
|
-
|
|
85
|
+
# Enforce under the CONTENT's row lock and, when the decision can ban, the
|
|
86
|
+
# PERSON's (after the flag's, before any report's: the lock order in
|
|
87
|
+
# Moderate.with_row_lock), so two flags on the same content can't both enforce,
|
|
88
|
+
# nor two bans of the same person: the second waits, then finds the decision the
|
|
89
|
+
# first took. Removal-only enforcement never touches the person's row.
|
|
90
|
+
#
|
|
91
|
+
# Both locks come BEFORE the first read of moderate_reports, and that matters on
|
|
92
|
+
# MySQL: its locking reads of the reports table (the checks below) take gap locks
|
|
93
|
+
# under REPEATABLE READ, and two decisions holding gap locks that each then insert
|
|
94
|
+
# a report deadlock. Waiting on the content or the person first means the second
|
|
95
|
+
# decision touches the reports table only after the first one has committed.
|
|
96
|
+
# Returns [the ResolveReport that decided, or nil, the report the flag answers to].
|
|
97
|
+
def enforce!(note, enforcement, resolution_basis)
|
|
98
|
+
Moderate.with_row_lock(flag.flaggable) do
|
|
99
|
+
Moderate.with_row_lock(enforcement[:ban_user] ? flag.owner : nil) do
|
|
100
|
+
prior = covering_decision(enforcement)
|
|
101
|
+
next [nil, prior] if prior
|
|
102
|
+
|
|
103
|
+
report = escalation_report!(note)
|
|
104
|
+
decision = ResolveReport.new(report, by: moderator, deliver: false, conduct_since: flag.created_at)
|
|
105
|
+
decision.resolve!(note: note, resolution_basis: resolution_basis, **enforcement)
|
|
106
|
+
[decision, report]
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
# A report on the same content and field, decided since this flag was filed,
|
|
112
|
+
# that already did everything asked now (removed it, banned the person). The
|
|
113
|
+
# flag closes against it instead of restricting a second time. A recorded
|
|
114
|
+
# removal only counts if the content is actually gone now: an older decision
|
|
115
|
+
# may say "removed" without having removed anything.
|
|
116
|
+
def covering_decision(enforcement)
|
|
117
|
+
asked = enforcement.select { |_action, wanted| wanted }.keys.map(&:to_s)
|
|
118
|
+
return if asked.include?("remove_content") && !content_gone?
|
|
119
|
+
|
|
120
|
+
# A locking read (see Moderate.with_row_lock): sees the decision a concurrent
|
|
121
|
+
# moderator committed while we waited, whatever the isolation level.
|
|
122
|
+
decided = Moderate::Report.actioned.where(same_content).where(resolved_at: flag.created_at..).recent_first.lock
|
|
123
|
+
decided.find { |report| asked.all? { |action| truthy?(report.resolution_actions.to_h[action]) } }
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
# Is the flagged field's content gone? The record deleted, the record saying it
|
|
127
|
+
# is (`deleted?`, a soft-delete tombstone), or the field reading as empty. Read
|
|
128
|
+
# fresh, under the content's lock: a decision that just committed removed it.
|
|
129
|
+
def content_gone?
|
|
130
|
+
content = flag.flaggable&.reload(lock: true) # a locking read: what committed, not our snapshot
|
|
131
|
+
return true if content.nil?
|
|
132
|
+
return true if content.respond_to?(:deleted?) && content.deleted?
|
|
133
|
+
|
|
134
|
+
value = flagged_value(content)
|
|
135
|
+
value.respond_to?(:attached?) ? !value.attached? : value.blank?
|
|
136
|
+
rescue ActiveRecord::RecordNotFound
|
|
137
|
+
true
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
def flagged_value(content)
|
|
141
|
+
return content.send(:moderation_field_value, flag.field) if content.respond_to?(:moderation_field_value, true)
|
|
142
|
+
|
|
143
|
+
content.public_send(flag.field) if content.respond_to?(flag.field)
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
def close!(status, note, report)
|
|
147
|
+
context = report ? flag.context.to_h.merge("report_id" => report.id) : flag.context
|
|
148
|
+
flag.update!(status: status, resolution_note: note, reviewed_by: moderator, reviewed_at: Time.current,
|
|
149
|
+
context: context)
|
|
150
|
+
audit_decision(status, note, report)
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
# The report enforcement runs through: the newest OPEN report on the same
|
|
154
|
+
# content and field (a person may already have reported what the filter
|
|
155
|
+
# caught), else a system report opened for this decision. A system report has
|
|
156
|
+
# no reporter, so it sends no receipt and no reporter-facing decision; the
|
|
157
|
+
# affected user still gets the statement of reasons and the appeal window.
|
|
158
|
+
def escalation_report!(note)
|
|
159
|
+
existing = Moderate::Report.open.where(same_content).recent_first.lock.first
|
|
160
|
+
return disclose_automation!(existing) if existing
|
|
161
|
+
|
|
162
|
+
system_report!(note)
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
def same_content
|
|
166
|
+
{ reportable_type: flag.flaggable_type, reportable_id: flag.flaggable_id, reported_field: flag.field }
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
# A system report opened for this decision. It may name any field the flag is
|
|
170
|
+
# on (`escalated_from_flag`): a filtered field (an image, say) needn't be one a
|
|
171
|
+
# PERSON may report for the gem to enforce on it.
|
|
172
|
+
def system_report!(note)
|
|
173
|
+
message = "Escalated from flag ##{flag.id} (#{flag.source}): #{note}"
|
|
174
|
+
report = Moderate::Report.new(
|
|
175
|
+
reportable: flag.flaggable, reported_field: flag.field, reported_user: flag.owner,
|
|
176
|
+
intake_kind: "community", category: escalation_category,
|
|
177
|
+
message: message.truncate(Moderate::Report::MESSAGE_MAX_LENGTH),
|
|
178
|
+
good_faith_confirmed: true, skip_received_notice: true
|
|
179
|
+
)
|
|
180
|
+
# Private on Report, set only here: not a host-assignable way around the
|
|
181
|
+
# field whitelist.
|
|
182
|
+
report.send(:escalated_from_flag=, flag)
|
|
183
|
+
report.save!
|
|
184
|
+
report
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
# A person's report filed BEFORE a classifier flagged the same content never
|
|
188
|
+
# saw that flag, so its automated-means disclosure (DSA Art. 17(3)(c)) says
|
|
189
|
+
# "no". When the flag is what's being enforced, the decision must say "yes":
|
|
190
|
+
# merge the flag's evidence in first. A manual flag is a person, not automation.
|
|
191
|
+
def disclose_automation!(report)
|
|
192
|
+
return report if flag.source == "manual"
|
|
193
|
+
|
|
194
|
+
evidence = report.automated_processing.to_h
|
|
195
|
+
disclosed = evidence.merge(
|
|
196
|
+
"used" => true,
|
|
197
|
+
"flag_ids" => Array(evidence["flag_ids"]) | [flag.id],
|
|
198
|
+
"sources" => Array(evidence["sources"]) | [flag.source],
|
|
199
|
+
"categories" => Array(evidence["categories"]) | Array(flag.categories)
|
|
200
|
+
)
|
|
201
|
+
report.update_column(:automated_processing, disclosed)
|
|
202
|
+
report
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
# The flag's own categories speak the classifier taxonomy ("hate",
|
|
206
|
+
# "hate/threatening", "sexual/minors"); keep the first one the host's community
|
|
207
|
+
# list also has, else "other" (or the list's first entry, for a host whose list
|
|
208
|
+
# has no "other").
|
|
209
|
+
def escalation_category
|
|
210
|
+
allowed = Moderate::Report.report_categories
|
|
211
|
+
top_level = Array(flag.categories).map { |category| category.to_s.split("/").first }
|
|
212
|
+
found = top_level.find { |category| allowed.include?(category) }
|
|
213
|
+
found || (allowed.include?("other") ? "other" : allowed.first)
|
|
214
|
+
end
|
|
215
|
+
|
|
216
|
+
def audit_decision(status, note, report)
|
|
75
217
|
Moderate.audit(
|
|
76
218
|
:flag_decision,
|
|
77
219
|
subject: flag,
|
|
@@ -81,8 +223,10 @@ module Moderate
|
|
|
81
223
|
flaggable_type: flag.flaggable_type,
|
|
82
224
|
flaggable_id: flag.flaggable_id,
|
|
83
225
|
field: flag.field,
|
|
226
|
+
field_label: flag.field_label,
|
|
84
227
|
source: flag.source,
|
|
85
228
|
categories: flag.categories,
|
|
229
|
+
report_id: report&.id,
|
|
86
230
|
note: note,
|
|
87
231
|
summary: "Flag ##{flag.id} #{status} by moderator"
|
|
88
232
|
}.compact
|
|
@@ -96,6 +240,10 @@ module Moderate
|
|
|
96
240
|
flag.errors.add(:resolution_note, :blank)
|
|
97
241
|
raise ActiveRecord::RecordInvalid, flag
|
|
98
242
|
end
|
|
243
|
+
|
|
244
|
+
def truthy?(value)
|
|
245
|
+
ActiveModel::Type::Boolean.new.cast(value) || false
|
|
246
|
+
end
|
|
99
247
|
end
|
|
100
248
|
end
|
|
101
249
|
end
|