moderate 1.0.0.beta2 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +96 -1
- data/.simplecov +2 -10
- data/AGENTS.md +1 -1
- data/Appraisals +10 -8
- data/CHANGELOG.md +654 -52
- data/CLAUDE.md +1 -1
- data/README.md +147 -7
- data/Rakefile +2 -0
- data/app/controllers/concerns/moderate/moderation.rb +23 -11
- data/app/controllers/moderate/appeals_controller.rb +9 -5
- data/app/controllers/moderate/application_controller.rb +3 -3
- data/app/controllers/moderate/notices_controller.rb +3 -6
- data/app/helpers/moderate/engine_helper.rb +32 -11
- data/app/views/moderate/appeals/new.html.erb +6 -0
- data/app/views/moderate/transparency_reports/_summary_card.html.erb +2 -2
- data/app/views/moderate/transparency_reports/show.html.erb +5 -5
- data/config/locales/en.yml +200 -0
- data/config/locales/es.yml +177 -0
- data/db/migrate/20261018200000_add_pending_key_to_moderate_flags.rb +139 -0
- data/db/migrate/20261018210000_add_retention_and_classifications_to_moderate_tables.rb +103 -0
- data/docs/chats.md +140 -0
- data/docs/compliance.md +92 -3
- data/docs/configuration.md +30 -2
- data/docs/dsa-notice-form.md +1 -1
- data/docs/madmin.md +44 -14
- data/docs/notifications.md +17 -4
- data/examples/aws_rekognition_adapter.rb +6 -6
- data/examples/openai_moderation_adapter.rb +1 -1
- data/examples/regexp_pattern_adapter.rb +68 -0
- data/gemfiles/{rails_7.1.gemfile → rails_8.0.gemfile} +8 -6
- data/gemfiles/rails_8.1.gemfile +6 -5
- data/lib/generators/moderate/install_generator.rb +25 -0
- data/lib/generators/moderate/templates/create_moderate_tables.rb.erb +17 -2
- data/lib/generators/moderate/templates/initializer.rb +9 -15
- data/lib/generators/moderate/views_generator.rb +3 -3
- data/lib/moderate/attachments.rb +137 -0
- data/lib/moderate/configuration.rb +87 -12
- data/lib/moderate/engine.rb +86 -25
- data/lib/moderate/event.rb +21 -0
- data/lib/moderate/filters/base.rb +1 -0
- data/lib/moderate/filters/wordlist.rb +5 -2
- data/lib/moderate/jobs/classify_job.rb +96 -16
- data/lib/moderate/label.rb +2 -2
- data/lib/moderate/macros.rb +8 -3
- data/lib/moderate/models/appeal.rb +10 -1
- data/lib/moderate/models/block.rb +23 -19
- data/lib/moderate/models/classification.rb +24 -0
- data/lib/moderate/models/concerns/actor.rb +17 -2
- data/lib/moderate/models/concerns/content_filterable.rb +181 -55
- data/lib/moderate/models/concerns/deferred_notices.rb +90 -0
- data/lib/moderate/models/concerns/reportable.rb +84 -4
- data/lib/moderate/models/flag.rb +182 -19
- data/lib/moderate/models/report.rb +180 -16
- data/lib/moderate/result.rb +3 -10
- data/lib/moderate/retention.rb +241 -0
- data/lib/moderate/services/intake_notice.rb +10 -10
- data/lib/moderate/services/intake_report.rb +42 -13
- data/lib/moderate/services/resolve_appeal.rb +8 -2
- data/lib/moderate/services/resolve_flag.rb +183 -35
- data/lib/moderate/services/resolve_report.rb +288 -35
- data/lib/moderate/text_validator.rb +15 -14
- data/lib/moderate/version.rb +1 -1
- data/lib/moderate.rb +337 -17
- data/lib/tasks/moderate.rake +23 -0
- metadata +22 -14
- data/gemfiles/rails_7.2.gemfile +0 -36
- data/lib/moderate/text.rb +0 -38
- data/lib/moderate/word_list.rb +0 -90
data/lib/moderate/engine.rb
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "rails/engine"
|
|
4
|
+
require_relative "text_validator" # `validates :field, moderate: true` (0.x)
|
|
4
5
|
|
|
5
6
|
module Moderate
|
|
6
7
|
# The Rails engine that wires `moderate` into a host app.
|
|
@@ -48,11 +49,9 @@ module Moderate
|
|
|
48
49
|
# `event`, `configuration`, `engine`, `macros`) is `require_relative`'d from
|
|
49
50
|
# `lib/moderate.rb` so the value objects work in a plain-Ruby context with
|
|
50
51
|
# no Rails — those files MUST NOT also be managed by Zeitwerk (a constant
|
|
51
|
-
# can't be both manually required and autoloaded). The 0.x
|
|
52
|
-
#
|
|
53
|
-
#
|
|
54
|
-
# `ActiveModel::Validations`, not a `Moderate::*` constant) or are legacy and
|
|
55
|
-
# loaded only on demand.
|
|
52
|
+
# can't be both manually required and autoloaded). The 0.x validator
|
|
53
|
+
# (`text_validator`) is ignored too: it reopens `ActiveModel::Validations`,
|
|
54
|
+
# not a `Moderate::*` constant, and this file requires it.
|
|
56
55
|
#
|
|
57
56
|
# We do this on the host's MAIN autoloader (`Rails.autoloaders.main`) inside an
|
|
58
57
|
# initializer scheduled BEFORE `:set_autoload_paths`, the same point Rails wires
|
|
@@ -61,13 +60,13 @@ module Moderate
|
|
|
61
60
|
LIB_ROOT = File.expand_path("..", __dir__) # => .../lib
|
|
62
61
|
MODERATE_LIB = File.expand_path("moderate", LIB_ROOT) # => .../lib/moderate
|
|
63
62
|
|
|
64
|
-
# Spine
|
|
65
|
-
#
|
|
63
|
+
# Spine files and the 0.x validator: Zeitwerk must NOT manage them (they're
|
|
64
|
+
# require_relative'd, and the validator defines a non-Moderate constant).
|
|
66
65
|
ZEITWERK_IGNORED = %w[
|
|
67
66
|
version.rb errors.rb label.rb result.rb event.rb
|
|
68
|
-
configuration.rb engine.rb macros.rb
|
|
69
|
-
text.rb text_validator.rb word_list.rb
|
|
67
|
+
configuration.rb engine.rb macros.rb text_validator.rb
|
|
70
68
|
].freeze
|
|
69
|
+
private_constant :LIB_ROOT, :MODERATE_LIB, :ZEITWERK_IGNORED
|
|
71
70
|
|
|
72
71
|
initializer "moderate.autoload", before: :set_autoload_paths do
|
|
73
72
|
loader = Rails.autoloaders.main
|
|
@@ -98,19 +97,78 @@ module Moderate
|
|
|
98
97
|
# autoloadable; adding it to eager_load_paths makes the boot-time pass cover it.
|
|
99
98
|
config.eager_load_paths << MODERATE_LIB
|
|
100
99
|
|
|
101
|
-
#
|
|
102
|
-
#
|
|
103
|
-
#
|
|
104
|
-
#
|
|
105
|
-
#
|
|
100
|
+
# The gem's own migrations (db/migrate, e.g. beta4's pending_key) and how an app
|
|
101
|
+
# gets them. The tables come from `rails generate moderate:install`, which writes
|
|
102
|
+
# the adaptive create migration and then COPIES the gem's migrations after it
|
|
103
|
+
# (*.moderate.rb), so a fresh app runs both, in order. An app installed before it
|
|
104
|
+
# copied them (beta3) holds no copies: this appends the gem's db/migrate to its
|
|
105
|
+
# migration paths, so it runs only what it lacks.
|
|
106
|
+
#
|
|
107
|
+
# Not appended when the same migrations would then be found twice (Active Record
|
|
108
|
+
# raises DuplicateMigrationNameError):
|
|
109
|
+
# - under the gem's own Rakefile (ENGINE_ROOT is this engine), where Rails' engine
|
|
110
|
+
# tasks already add the engine's db/migrate;
|
|
111
|
+
# - when the app holds copies (*.moderate.rb, from moderate:install or
|
|
112
|
+
# `bin/rails moderate:install:migrations`): the copies run instead, and a later
|
|
113
|
+
# release's migrations arrive with `bin/rails moderate:install:migrations`.
|
|
106
114
|
initializer "moderate.migrations" do |app|
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
115
|
+
next if app.root.to_s == root.to_s
|
|
116
|
+
next if defined?(ENGINE_ROOT) && ENGINE_ROOT.to_s == root.to_s
|
|
117
|
+
|
|
118
|
+
Engine.migration_paths_for(app.config.paths["db/migrate"].expanded).each do |path|
|
|
119
|
+
app.config.paths["db/migrate"] << path
|
|
111
120
|
end
|
|
112
121
|
end
|
|
113
122
|
|
|
123
|
+
# The gem's migration directories an app with `host_dirs` as its own should add.
|
|
124
|
+
def self.migration_paths_for(host_dirs)
|
|
125
|
+
return [] if host_dirs.any? { |dir| Dir.glob(File.join(dir, "*.moderate.rb")).any? }
|
|
126
|
+
|
|
127
|
+
config.paths["db/migrate"].expanded - host_dirs
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
# An app that holds copies of the gem's migrations runs only those (see above), so
|
|
131
|
+
# a copy missed after an upgrade would never run, silently. At boot, compare the
|
|
132
|
+
# copies with the gem's migrations by name; when one is missing, raise (or log,
|
|
133
|
+
# per `config.migration_copies_check`), naming it and the command that copies it.
|
|
134
|
+
config.after_initialize do |app|
|
|
135
|
+
next if app.root.to_s == Engine.root.to_s
|
|
136
|
+
|
|
137
|
+
Engine.check_migration_copies!(app.config.paths["db/migrate"].expanded)
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
# The gem's migrations (by name) an app holding copies has no copy of; [] for an
|
|
141
|
+
# app without copies, which gets the gem's own path instead.
|
|
142
|
+
def self.missing_migration_copies(host_dirs)
|
|
143
|
+
copies = host_dirs.flat_map { |dir| Dir.glob(File.join(dir, "*.moderate.rb")) }
|
|
144
|
+
return [] if copies.empty?
|
|
145
|
+
|
|
146
|
+
copied = copies.map { |file| File.basename(file, ".moderate.rb").split("_", 2).last }
|
|
147
|
+
gem_migrations = Dir[root.join("db/migrate/[0-9]*_*.rb").to_s].map { |file| File.basename(file, ".rb").split("_", 2).last }
|
|
148
|
+
gem_migrations - copied
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
def self.check_migration_copies!(host_dirs, mode: Moderate.config.migration_copies_check)
|
|
152
|
+
return if mode == :off
|
|
153
|
+
|
|
154
|
+
missing = missing_migration_copies(host_dirs)
|
|
155
|
+
return if missing.empty?
|
|
156
|
+
|
|
157
|
+
message = "moderate: your app holds copies of moderate's migrations (*.moderate.rb) but none of " \
|
|
158
|
+
"#{missing.join(", ")}, which this version of moderate needs. Run `bin/rails " \
|
|
159
|
+
"moderate:install:migrations`, then `bin/rails db:migrate`."
|
|
160
|
+
# Under a rake task (the very command that copies them, or db:migrate) log
|
|
161
|
+
# instead: raising there would make the fix impossible to run.
|
|
162
|
+
raise Moderate::ConfigurationError, message if mode == :raise && !running_rake_task?
|
|
163
|
+
|
|
164
|
+
Rails.logger&.error(message)
|
|
165
|
+
warn(message)
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
def self.running_rake_task?
|
|
169
|
+
defined?(::Rake.application) && ::Rake.application.top_level_tasks.any?
|
|
170
|
+
end
|
|
171
|
+
|
|
114
172
|
# Register the model macros on ActiveRecord, the canonical Rails way.
|
|
115
173
|
#
|
|
116
174
|
# `ActiveSupport.on_load(:active_record)` defers until ActiveRecord::Base is
|
|
@@ -127,12 +185,15 @@ module Moderate
|
|
|
127
185
|
end
|
|
128
186
|
end
|
|
129
187
|
|
|
130
|
-
#
|
|
131
|
-
#
|
|
132
|
-
#
|
|
133
|
-
#
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
188
|
+
# The gem's I18n files (config/locales/*.yml — default copy and the generic
|
|
189
|
+
# reportable-field labels) need NO initializer: every Rails::Engine registers
|
|
190
|
+
# its `config/locales/**/*.{rb,yml}` in `config.i18n.railties_load_path`, which
|
|
191
|
+
# Rails puts AHEAD of the host app's own locale files — so a host translation
|
|
192
|
+
# of any `moderate.*` key overrides ours, as it must.
|
|
193
|
+
#
|
|
194
|
+
# Do NOT also append them to `app.config.i18n.load_path` (an earlier version
|
|
195
|
+
# did, while the directory was still empty): that list is loaded AFTER the
|
|
196
|
+
# host's files, so the gem's defaults would silently clobber the host's
|
|
197
|
+
# translations. test/integration/locale_precedence_test.rb pins this.
|
|
137
198
|
end
|
|
138
199
|
end
|
data/lib/moderate/event.rb
CHANGED
|
@@ -72,4 +72,25 @@ module Moderate
|
|
|
72
72
|
payload.to_h.transform_keys(&:to_sym)
|
|
73
73
|
end
|
|
74
74
|
end
|
|
75
|
+
|
|
76
|
+
# The event vocabulary, frozen for 1.0. Defined by reopening the class (a
|
|
77
|
+
# constant inside the `Data.define` block would land on the enclosing scope).
|
|
78
|
+
# test/public_api_test.rb checks that every name the gem emits is listed here and
|
|
79
|
+
# that the README lists exactly these.
|
|
80
|
+
class Event
|
|
81
|
+
# What `config.notify` receives: moments a person (or an admin channel) is told about.
|
|
82
|
+
NOTIFY_NAMES = %i[
|
|
83
|
+
report_received notice_received report_decision affected_user_decision
|
|
84
|
+
appeal_received appeal_decision user_blocked user_unblocked user_banned
|
|
85
|
+
content_flagged content_removed
|
|
86
|
+
].freeze
|
|
87
|
+
|
|
88
|
+
# What `config.audit` receives: the record of every intake, decision, block,
|
|
89
|
+
# ban and removal, plus the gem's own housekeeping.
|
|
90
|
+
AUDIT_NAMES = %i[
|
|
91
|
+
report_received appeal_received report_decision flag_decision appeal_decision
|
|
92
|
+
flag_filed user_blocked user_unblocked user_banned content_removed
|
|
93
|
+
notify_failed classification_skipped retention_purged legal_hold_placed legal_hold_released
|
|
94
|
+
].freeze
|
|
95
|
+
end
|
|
75
96
|
end
|
|
@@ -47,6 +47,7 @@ module Moderate
|
|
|
47
47
|
# async via its own `synchronous? == false`, which the spine honors regardless of
|
|
48
48
|
# whether the adapter inherits from `Base`.
|
|
49
49
|
module Filters
|
|
50
|
+
# The optional base class for adapters (see the comment above).
|
|
50
51
|
class Base
|
|
51
52
|
class << self
|
|
52
53
|
# The class-level entry point the registry resolution path calls. Spins up
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "yaml"
|
|
4
|
-
require "set"
|
|
5
4
|
|
|
6
5
|
module Moderate
|
|
7
6
|
module Filters
|
|
@@ -77,6 +76,10 @@ module Moderate
|
|
|
77
76
|
# at runtime, so compile them ONCE per process and memoize on the class.
|
|
78
77
|
# (config.additional_words / excluded_words are layered in per-call, since the
|
|
79
78
|
# host can in principle reconfigure between calls — and they're cheap.)
|
|
79
|
+
#
|
|
80
|
+
# @api private (as are reset!, compile_bundled_patterns, compile_pair,
|
|
81
|
+
# compact_source and the normalization constants): the public surface of an
|
|
82
|
+
# adapter is `classify` and `synchronous?`.
|
|
80
83
|
def self.patterns
|
|
81
84
|
@patterns ||= compile_bundled_patterns
|
|
82
85
|
end
|
|
@@ -139,7 +142,7 @@ module Moderate
|
|
|
139
142
|
.sub(/\\b\z/, "") # drop trailing word boundary
|
|
140
143
|
.gsub(/\\s[*+]?/, "") # drop \s, \s*, \s+ whitespace matchers
|
|
141
144
|
.gsub(/\[ ([^\]]*)\]/, '[\1]') # drop a literal space inside a char class
|
|
142
|
-
.delete(" ")
|
|
145
|
+
.delete(" ") # drop any remaining literal spaces
|
|
143
146
|
end
|
|
144
147
|
|
|
145
148
|
def classify(value)
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module Moderate
|
|
4
|
-
# The background worker for ASYNCHRONOUS filter adapters (
|
|
5
|
-
#
|
|
4
|
+
# The background worker for ASYNCHRONOUS filter adapters (a remote moderation API
|
|
5
|
+
# such as the OpenAI or Rekognition reference adapters in examples/, or a host's
|
|
6
|
+
# own classifier) running in :flag mode.
|
|
6
7
|
#
|
|
7
8
|
# ── Why a job exists at all ──────────────────────────────────────────────────
|
|
8
9
|
# An async adapter does blocking I/O (a moderation API call), which must never run
|
|
@@ -23,11 +24,27 @@ module Moderate
|
|
|
23
24
|
# value. If the record was edited or deleted between enqueue and run, we classify
|
|
24
25
|
# what's actually there now (or skip a vanished record), never a stale snapshot.
|
|
25
26
|
#
|
|
26
|
-
# ──
|
|
27
|
+
# ── One flag per run ─────────────────────────────────────────────────────────
|
|
27
28
|
# Flag creation goes through `Moderate::Flag.flag!`, the single builder shared by
|
|
28
|
-
# the synchronous and asynchronous paths
|
|
29
|
-
#
|
|
30
|
-
#
|
|
29
|
+
# the synchronous and asynchronous paths, and it is the job's last step, so a job
|
|
30
|
+
# that fails earlier (the adapter's network call) and is retried files one flag.
|
|
31
|
+
# `flag!` always creates: two runs for the same content (two saves, or a host that
|
|
32
|
+
# also enqueues the job itself) file two flags, both in the queue.
|
|
33
|
+
#
|
|
34
|
+
# ── An attachment whose file isn't on the service yet ────────────────────────
|
|
35
|
+
# For an attachment the filter enqueues this job before Active Storage uploads
|
|
36
|
+
# the file (the guard: it runs even if the upload fails) and, once the upload
|
|
37
|
+
# returns, classifies right away (see Moderate::ContentFilterable). When the job
|
|
38
|
+
# runs, a blob can be missing: the upload failed or is still running, a direct
|
|
39
|
+
# upload hasn't finished, a replica lags. The job then re-enqueues itself,
|
|
40
|
+
# waiting UPLOAD_WAIT × attempt seconds, up to UPLOAD_ATTEMPTS runs in all, and
|
|
41
|
+
# gives up with a `classification_skipped` audit event. It never hands an adapter
|
|
42
|
+
# a file that isn't there.
|
|
43
|
+
#
|
|
44
|
+
# `blob_ids:` names the files the job was enqueued for: if the field holds other
|
|
45
|
+
# files by the time it runs, the newer change has its own job and this one stops.
|
|
46
|
+
# A field already classified for exactly these files (Moderate::Attachments marks
|
|
47
|
+
# the blobs) is not classified again.
|
|
31
48
|
#
|
|
32
49
|
# ── Base class ───────────────────────────────────────────────────────────────
|
|
33
50
|
# We subclass `ActiveJob::Base` directly (not the host's ApplicationJob) so the gem
|
|
@@ -35,16 +52,27 @@ module Moderate
|
|
|
35
52
|
# rest of the gem stays host-agnostic. The host configures the queue adapter,
|
|
36
53
|
# retries, and queue name globally as usual.
|
|
37
54
|
class ClassifyJob < ActiveJob::Base
|
|
38
|
-
#
|
|
39
|
-
#
|
|
40
|
-
#
|
|
41
|
-
queue_as
|
|
55
|
+
# The :default queue. Moderation flagging is important but not latency-critical
|
|
56
|
+
# (the content is already published in :flag mode); a host that wants it elsewhere
|
|
57
|
+
# says so in an initializer: `Moderate::ClassifyJob.queue_as :low`.
|
|
58
|
+
queue_as :default
|
|
59
|
+
|
|
60
|
+
# Seconds to wait before the next look at a blob that isn't uploaded yet,
|
|
61
|
+
# multiplied by the attempt number (5s, 10s, 15s, 20s: under a minute in all).
|
|
62
|
+
UPLOAD_WAIT = 5
|
|
63
|
+
# Runs of this job, the first included, before it gives up on such a blob.
|
|
64
|
+
UPLOAD_ATTEMPTS = 5
|
|
42
65
|
|
|
43
66
|
# @param record [ActiveRecord::Base] the flaggable record (any Moderate::Reportable).
|
|
44
67
|
# @param field [String, Symbol] the field whose value to classify.
|
|
45
68
|
# @param adapter [String, Symbol, nil] optional explicit adapter name; when nil,
|
|
46
69
|
# the field's resolved FilterPolicy (or the global default) decides.
|
|
47
|
-
|
|
70
|
+
# @param upload_attempt [Integer] which look at a not-yet-uploaded attachment
|
|
71
|
+
# this is (the job sets it when it re-enqueues itself; callers leave it).
|
|
72
|
+
# @param blob_ids [Array<Integer>, nil] the attachment's blobs when enqueued.
|
|
73
|
+
# @param claim_recheck [Boolean] this run is the one look the job takes after
|
|
74
|
+
# another worker's claim on the same files expires (set by the job itself).
|
|
75
|
+
def perform(record, field, adapter: nil, upload_attempt: 1, blob_ids: nil, claim_recheck: false)
|
|
48
76
|
# The record may have been destroyed between enqueue and execution — nothing
|
|
49
77
|
# to classify, nothing to flag. (ActiveJob raises DeserializationError for a
|
|
50
78
|
# GlobalID that no longer resolves; that's rescued at the framework level, but
|
|
@@ -60,11 +88,13 @@ module Moderate
|
|
|
60
88
|
|
|
61
89
|
value = field_value(record, field)
|
|
62
90
|
return if blank?(value)
|
|
91
|
+
if Moderate::Attachments.attachment?(value)
|
|
92
|
+
return classify_attachment(record, field, policy, adapter: adapter, attempt: upload_attempt, blob_ids: blob_ids,
|
|
93
|
+
claim_recheck: claim_recheck)
|
|
94
|
+
end
|
|
63
95
|
|
|
64
96
|
result = Moderate.classify(value, policy: policy)
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
file_flag(record, field, policy, result, value)
|
|
97
|
+
file_flag(record, field, policy, result, value) if result.flagged?
|
|
68
98
|
end
|
|
69
99
|
|
|
70
100
|
private
|
|
@@ -86,7 +116,7 @@ module Moderate
|
|
|
86
116
|
)
|
|
87
117
|
end
|
|
88
118
|
|
|
89
|
-
# File
|
|
119
|
+
# File the Moderate::Flag via the shared builder. We pass exactly the
|
|
90
120
|
# columns the install migration defines for moderate_flags. `Flag.flag!` owns the
|
|
91
121
|
# `content_flagged` notify event so there's a single emission site across the
|
|
92
122
|
# sync and async paths — the job doesn't fire it itself, to avoid double-notifying.
|
|
@@ -115,7 +145,7 @@ module Moderate
|
|
|
115
145
|
# otherwise (the owner column is nullable — a flag with no resolvable owner is
|
|
116
146
|
# still a valid queue item).
|
|
117
147
|
def content_owner(record)
|
|
118
|
-
|
|
148
|
+
Moderate.responsible_user(record)
|
|
119
149
|
end
|
|
120
150
|
|
|
121
151
|
# Diagnostic context persisted on the flag (the `context` jsonb column): the raw
|
|
@@ -138,6 +168,56 @@ module Moderate
|
|
|
138
168
|
value.to_s[0, 500]
|
|
139
169
|
end
|
|
140
170
|
|
|
171
|
+
# Claim, classify outside any lock, complete (Moderate::Attachments): this job
|
|
172
|
+
# and the inline path or the guard never classify the same files at once, and a
|
|
173
|
+
# slow adapter never holds the content's row. When another worker holds a live
|
|
174
|
+
# claim, the job looks once more when that claim expires, so a worker that died
|
|
175
|
+
# mid-call doesn't leave the files unclassified.
|
|
176
|
+
def classify_attachment(record, field, policy, adapter:, attempt:, blob_ids:, claim_recheck:)
|
|
177
|
+
value = field_value(record, field)
|
|
178
|
+
return if blob_ids && Moderate::Attachments.blob_ids(value) != Array(blob_ids).sort
|
|
179
|
+
return wait_for_upload(record, field, adapter, attempt, blob_ids) unless Moderate::Attachments.uploaded?(value)
|
|
180
|
+
|
|
181
|
+
outcome = Moderate::Attachments.classify_once(
|
|
182
|
+
record, field,
|
|
183
|
+
owner: job_id,
|
|
184
|
+
read: -> { fresh_value(record, field) },
|
|
185
|
+
classify: ->(current) { Moderate.classify(current, policy: policy) },
|
|
186
|
+
file: ->(current, result) { file_flag(record, field, policy, result, current) }
|
|
187
|
+
)
|
|
188
|
+
return unless outcome == :claimed && !claim_recheck
|
|
189
|
+
|
|
190
|
+
expires_at = Moderate::Attachments.claim_expires_at(record, field, value) || Time.current
|
|
191
|
+
self.class.set(wait_until: expires_at + 1)
|
|
192
|
+
.perform_later(record, field, adapter: adapter, blob_ids: blob_ids, claim_recheck: true)
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
# The field as committed now, read through a fresh instance (the job's own copy
|
|
196
|
+
# caches the attachment it was enqueued with).
|
|
197
|
+
def fresh_value(record, field)
|
|
198
|
+
fresh = record.class.unscoped.find_by(id: record.id)
|
|
199
|
+
fresh && field_value(fresh, field)
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
def wait_for_upload(record, field, adapter, attempt, blob_ids)
|
|
203
|
+
if attempt < UPLOAD_ATTEMPTS
|
|
204
|
+
self.class.set(wait: UPLOAD_WAIT * attempt)
|
|
205
|
+
.perform_later(record, field, adapter: adapter, upload_attempt: attempt + 1, blob_ids: blob_ids)
|
|
206
|
+
return
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
Moderate.audit(
|
|
210
|
+
:classification_skipped,
|
|
211
|
+
subject: record,
|
|
212
|
+
payload: {
|
|
213
|
+
field: field,
|
|
214
|
+
reason: "attachment_not_uploaded",
|
|
215
|
+
attempts: attempt,
|
|
216
|
+
summary: "#{record.class.name}##{field} not classified: its file never reached the storage service"
|
|
217
|
+
}
|
|
218
|
+
)
|
|
219
|
+
end
|
|
220
|
+
|
|
141
221
|
def field_value(record, field)
|
|
142
222
|
record.public_send(field)
|
|
143
223
|
rescue NoMethodError
|
data/lib/moderate/label.rb
CHANGED
|
@@ -16,7 +16,7 @@ module Moderate
|
|
|
16
16
|
# reasons, and the Art. 24 transparency counters all aggregate over a single set.
|
|
17
17
|
# See: https://developers.openai.com/api/docs/guides/moderation
|
|
18
18
|
#
|
|
19
|
-
# Implemented with Ruby's `Data.define` (Ruby 3.2
|
|
19
|
+
# Implemented with Ruby's `Data.define` (Ruby 3.2+; the gemspec requires 3.3+):
|
|
20
20
|
# an immutable, frozen-by-construction value object — exactly what a label
|
|
21
21
|
# should be (you never mutate a verdict after the fact).
|
|
22
22
|
Label = Data.define(:category, :subcategory, :score, :flagged, :input) do
|
|
@@ -36,7 +36,7 @@ module Moderate
|
|
|
36
36
|
super(
|
|
37
37
|
category: normalize_symbol(category),
|
|
38
38
|
subcategory: subcategory.nil? ? nil : normalize_symbol(subcategory),
|
|
39
|
-
score: score
|
|
39
|
+
score: score&.to_f,
|
|
40
40
|
flagged: flagged ? true : false,
|
|
41
41
|
input: normalize_symbol(input || :unknown)
|
|
42
42
|
)
|
data/lib/moderate/macros.rb
CHANGED
|
@@ -51,10 +51,15 @@ module Moderate
|
|
|
51
51
|
# empty, and a blank reported_field is then allowed — see
|
|
52
52
|
# Reportable#reportable_field_allowed?).
|
|
53
53
|
#
|
|
54
|
-
#
|
|
55
|
-
|
|
54
|
+
# `labels:` gives each field the human name moderators and affected users see
|
|
55
|
+
# ("Chat message" instead of "Body"); see Reportable.reportable_fields.
|
|
56
|
+
#
|
|
57
|
+
# has_reportable_content :body, :files, labels: { body: "Chat message", files: "Chat photo" }
|
|
58
|
+
#
|
|
59
|
+
# Equivalent to `include Moderate::Reportable` + `reportable_fields(*fields, labels:)`.
|
|
60
|
+
def has_reportable_content(*fields, labels: nil)
|
|
56
61
|
include Moderate::Reportable unless include?(Moderate::Reportable)
|
|
57
|
-
reportable_fields(*fields) if fields.any?
|
|
62
|
+
reportable_fields(*fields, labels: labels) if fields.any? || labels
|
|
58
63
|
self
|
|
59
64
|
end
|
|
60
65
|
|
|
@@ -16,6 +16,8 @@ module Moderate
|
|
|
16
16
|
# report (`report_must_be_closed`).
|
|
17
17
|
# Source: https://eur-lex.europa.eu/eli/reg/2022/2065/oj (Art. 20)
|
|
18
18
|
class Appeal < ApplicationRecord
|
|
19
|
+
include Moderate::DeferredNotices
|
|
20
|
+
|
|
19
21
|
self.table_name = "moderate_appeals"
|
|
20
22
|
|
|
21
23
|
# Validated by the `validates :status, inclusion` below — NOT a DB constraint, so
|
|
@@ -63,12 +65,18 @@ module Moderate
|
|
|
63
65
|
validates :appellant_email, presence: true, format: { with: URI::MailTo::EMAIL_REGEXP }
|
|
64
66
|
validates :resolution_note, presence: true, if: :closed?
|
|
65
67
|
validate :report_must_be_closed
|
|
66
|
-
|
|
68
|
+
# Only when the complaint is FILED: Art. 20 bounds when one may be lodged, not
|
|
69
|
+
# when it must be decided, so a complaint filed in time is decided (and saved)
|
|
70
|
+
# whenever the review ends, even after the window has closed.
|
|
71
|
+
validate :report_must_still_be_appealable, on: :create
|
|
67
72
|
|
|
68
73
|
def open?
|
|
69
74
|
status == "open"
|
|
70
75
|
end
|
|
71
76
|
|
|
77
|
+
# Awaiting a decision — the twin of the `pending` scope (see Report#pending?).
|
|
78
|
+
alias pending? open?
|
|
79
|
+
|
|
72
80
|
def closed?
|
|
73
81
|
status.in?(%w[upheld rejected])
|
|
74
82
|
end
|
|
@@ -144,6 +152,7 @@ module Moderate
|
|
|
144
152
|
# Enforce the DSA Art. 20 six-month window: refuse appeals filed after the
|
|
145
153
|
# report's `appeal_deadline_at` (stamped to decision-time + APPEAL_WINDOW). If no
|
|
146
154
|
# deadline was stamped yet, we don't block — the window simply hasn't been opened.
|
|
155
|
+
# Checked on create only (see the declaration above).
|
|
147
156
|
def report_must_still_be_appealable
|
|
148
157
|
return if report.blank? || report.appeal_deadline_at.blank?
|
|
149
158
|
return if Time.current <= report.appeal_deadline_at
|
|
@@ -16,6 +16,8 @@ module Moderate
|
|
|
16
16
|
# Apple: https://developer.apple.com/app-store/review/guidelines/#user-generated-content
|
|
17
17
|
# Google: https://support.google.com/googleplay/android-developer/answer/9876937
|
|
18
18
|
class Block < ApplicationRecord
|
|
19
|
+
include Moderate::DeferredNotices
|
|
20
|
+
|
|
19
21
|
self.table_name = "moderate_blocks"
|
|
20
22
|
|
|
21
23
|
# Both ends resolve to the host's configured user class (read lazily as a String
|
|
@@ -35,7 +37,7 @@ module Moderate
|
|
|
35
37
|
|
|
36
38
|
# Find the edge between two users in EITHER direction (used to answer
|
|
37
39
|
# "blocked_with?" — is there a block between us, regardless of who blocked whom).
|
|
38
|
-
scope :between,
|
|
40
|
+
scope :between, lambda { |user_a, user_b|
|
|
39
41
|
return none if user_a.blank? || user_b.blank?
|
|
40
42
|
|
|
41
43
|
where(blocker_id: user_a.id, blocked_id: user_b.id)
|
|
@@ -75,6 +77,7 @@ module Moderate
|
|
|
75
77
|
created = false
|
|
76
78
|
|
|
77
79
|
transaction do
|
|
80
|
+
connection = Moderate::DeferredNotices.connection_for(self) # before any save
|
|
78
81
|
block = find_or_initialize_by(blocker: blocker, blocked: blocked)
|
|
79
82
|
created = block.new_record?
|
|
80
83
|
block.save! if created
|
|
@@ -97,18 +100,12 @@ module Moderate
|
|
|
97
100
|
summary: "user #{blocker.id} blocked user #{blocked.id}"
|
|
98
101
|
)
|
|
99
102
|
)
|
|
100
|
-
end
|
|
101
|
-
end
|
|
102
103
|
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
subject: block,
|
|
109
|
-
actor: blocker,
|
|
110
|
-
payload: { blocker_id: blocker.id, blocked_id: blocked.id, summary: "user #{blocker.id} blocked user #{blocked.id}" }
|
|
111
|
-
)
|
|
104
|
+
# Announced once the block is durable (Moderate::DeferredNotices): after the
|
|
105
|
+
# outermost commit, never if a surrounding transaction rolls back, and
|
|
106
|
+
# outside any transaction, so notify can't roll back what it announces.
|
|
107
|
+
block.after_commit_notice(on: connection) { block.send(:announce, :user_blocked, blocker, blocked) }
|
|
108
|
+
end
|
|
112
109
|
end
|
|
113
110
|
|
|
114
111
|
block
|
|
@@ -121,7 +118,9 @@ module Moderate
|
|
|
121
118
|
return false if block.blank?
|
|
122
119
|
|
|
123
120
|
transaction do
|
|
121
|
+
connection = Moderate::DeferredNotices.connection_for(self) # before the destroy
|
|
124
122
|
block.destroy!
|
|
123
|
+
block.after_commit_notice(on: connection) { block.send(:announce, :user_unblocked, blocker, blocked) }
|
|
125
124
|
Moderate.audit(
|
|
126
125
|
name: :user_unblocked,
|
|
127
126
|
subject: block,
|
|
@@ -134,13 +133,6 @@ module Moderate
|
|
|
134
133
|
)
|
|
135
134
|
end
|
|
136
135
|
|
|
137
|
-
Moderate.notify(
|
|
138
|
-
:user_unblocked,
|
|
139
|
-
subject: block,
|
|
140
|
-
actor: blocker,
|
|
141
|
-
payload: { blocker_id: blocker.id, blocked_id: blocked.id, summary: "user #{blocker.id} unblocked user #{blocked.id}" }
|
|
142
|
-
)
|
|
143
|
-
|
|
144
136
|
true
|
|
145
137
|
end
|
|
146
138
|
|
|
@@ -192,6 +184,18 @@ module Moderate
|
|
|
192
184
|
end
|
|
193
185
|
private_class_method :audit_payload_from_on_block
|
|
194
186
|
|
|
187
|
+
def announce(name, blocker, blocked)
|
|
188
|
+
verb = name == :user_blocked ? "blocked" : "unblocked"
|
|
189
|
+
Moderate.notify(
|
|
190
|
+
name,
|
|
191
|
+
subject: self,
|
|
192
|
+
actor: blocker,
|
|
193
|
+
payload: {
|
|
194
|
+
blocker_id: blocker.id, blocked_id: blocked.id, summary: "user #{blocker.id} #{verb} user #{blocked.id}"
|
|
195
|
+
}
|
|
196
|
+
)
|
|
197
|
+
end
|
|
198
|
+
|
|
195
199
|
# The DB has a CHECK constraint (`moderate_blocks_no_self_block`) too; this gives
|
|
196
200
|
# the friendly validation error before the row ever reaches the database.
|
|
197
201
|
def cannot_block_self
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Moderate
|
|
4
|
+
# One record's field, for one exact set of attached files (`digest`, of the ordered
|
|
5
|
+
# blob ids): who is classifying it right now (`claimed_by`, `claimed_at`) and when
|
|
6
|
+
# it was classified (`classified_at`, with the adapter and whether it flagged).
|
|
7
|
+
# The claim protocol that writes it lives in Moderate::Attachments; rows hold ids,
|
|
8
|
+
# a digest and timestamps, no content, and Moderate.purge deletes old ones.
|
|
9
|
+
#
|
|
10
|
+
# @api private
|
|
11
|
+
class Classification < ApplicationRecord
|
|
12
|
+
self.table_name = "moderate_classifications"
|
|
13
|
+
|
|
14
|
+
belongs_to :record, polymorphic: true, optional: true
|
|
15
|
+
|
|
16
|
+
scope :for_set, lambda { |record, field, digest|
|
|
17
|
+
where(record_type: record.class.polymorphic_name, record_id: record.id, field: field.to_s, digest: digest)
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
def classified?
|
|
21
|
+
classified_at.present?
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
@@ -72,10 +72,19 @@ module Moderate
|
|
|
72
72
|
# model (not here) means the public DSA notice intake and this in-app path share
|
|
73
73
|
# one source of truth.
|
|
74
74
|
#
|
|
75
|
-
# `details:` is the
|
|
75
|
+
# `details:` is optional (blank, the report's message is the category's label);
|
|
76
|
+
# it's the README's name for the reporter's free-text reason, and it maps
|
|
76
77
|
# onto the Report's `message`. Extra keyword args (e.g. `field:`) pass straight
|
|
77
78
|
# through, so this stays forward-compatible with the Report model's attributes.
|
|
78
|
-
|
|
79
|
+
#
|
|
80
|
+
# `block: true` also blocks the person responsible for the content once the
|
|
81
|
+
# report is filed — the "Report and block" checkbox. (Nothing is blocked if the
|
|
82
|
+
# content has no person behind it, or if it's your own.)
|
|
83
|
+
#
|
|
84
|
+
# The report is refused (RecordInvalid, an error on :reportable) when this actor
|
|
85
|
+
# may not report it — the reportable's own `report_visible_to?`, the same rule
|
|
86
|
+
# that hides the report link.
|
|
87
|
+
def report!(reportable, category:, details: nil, block: false, **attributes)
|
|
79
88
|
attributes[:message] = details if details && !attributes.key?(:message)
|
|
80
89
|
reported_field = attributes.delete(:reported_field)
|
|
81
90
|
field = attributes.delete(:field)
|
|
@@ -96,6 +105,7 @@ module Moderate
|
|
|
96
105
|
intake_kind: "community",
|
|
97
106
|
**attributes
|
|
98
107
|
)
|
|
108
|
+
report.block_reported_user = block
|
|
99
109
|
|
|
100
110
|
intake = Moderate::Services::IntakeReport.new(
|
|
101
111
|
report: report,
|
|
@@ -164,6 +174,11 @@ module Moderate
|
|
|
164
174
|
self
|
|
165
175
|
end
|
|
166
176
|
|
|
177
|
+
# A report about a person is about their profile.
|
|
178
|
+
def moderation_content_type(_field = nil)
|
|
179
|
+
"user_profile"
|
|
180
|
+
end
|
|
181
|
+
|
|
167
182
|
# You can never report yourself, and a field still has to be reportable. This
|
|
168
183
|
# tightens Reportable's default with the self-report guard, so the
|
|
169
184
|
# `moderate_report_link` helper hides the control on your own profile.
|