moderate 0.1.0 → 1.0.0.beta1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +8 -0
- data/.simplecov +62 -0
- data/AGENTS.md +7 -0
- data/Appraisals +16 -0
- data/CHANGELOG.md +71 -1
- data/CLAUDE.md +7 -0
- data/README.md +376 -29
- data/Rakefile +28 -2
- data/app/controllers/concerns/moderate/moderation.rb +161 -0
- data/app/controllers/moderate/appeals_controller.rb +190 -0
- data/app/controllers/moderate/application_controller.rb +45 -0
- data/app/controllers/moderate/notices_controller.rb +382 -0
- data/app/controllers/moderate/transparency_reports_controller.rb +30 -0
- data/app/helpers/moderate/engine_helper.rb +151 -0
- data/app/views/moderate/appeals/new.html.erb +78 -0
- data/app/views/moderate/notices/new.html.erb +255 -0
- data/app/views/moderate/transparency_reports/_summary_card.html.erb +20 -0
- data/app/views/moderate/transparency_reports/show.html.erb +52 -0
- data/config/moderate/blocklists/en.yml +81 -0
- data/config/moderate/blocklists/es.yml +40 -0
- data/config/routes.rb +36 -0
- data/docs/compliance.md +178 -0
- data/docs/configuration.md +326 -0
- data/docs/dsa-notice-form.md +371 -0
- data/docs/madmin.md +490 -0
- data/docs/notifications.md +363 -0
- data/examples/aws_rekognition_adapter.rb +140 -0
- data/examples/openai_moderation_adapter.rb +111 -0
- data/gemfiles/rails_7.1.gemfile +36 -0
- data/gemfiles/rails_7.2.gemfile +36 -0
- data/gemfiles/rails_8.1.gemfile +36 -0
- data/lib/generators/moderate/install_generator.rb +56 -0
- data/lib/generators/moderate/templates/create_moderate_tables.rb.erb +237 -0
- data/lib/generators/moderate/templates/initializer.rb +198 -0
- data/lib/generators/moderate/views_generator.rb +63 -0
- data/lib/moderate/configuration.rb +341 -0
- data/lib/moderate/engine.rb +138 -0
- data/lib/moderate/errors.rb +26 -0
- data/lib/moderate/event.rb +75 -0
- data/lib/moderate/filters/base.rb +126 -0
- data/lib/moderate/filters/wordlist.rb +255 -0
- data/lib/moderate/jobs/classify_job.rb +158 -0
- data/lib/moderate/label.rb +111 -0
- data/lib/moderate/macros.rb +90 -0
- data/lib/moderate/models/appeal.rb +154 -0
- data/lib/moderate/models/application_record.rb +31 -0
- data/lib/moderate/models/block.rb +203 -0
- data/lib/moderate/models/concerns/actor.rb +174 -0
- data/lib/moderate/models/concerns/content_filterable.rb +155 -0
- data/lib/moderate/models/concerns/reportable.rb +282 -0
- data/lib/moderate/models/flag.rb +136 -0
- data/lib/moderate/models/report.rb +620 -0
- data/lib/moderate/result.rb +176 -0
- data/lib/moderate/services/intake_appeal.rb +89 -0
- data/lib/moderate/services/intake_notice.rb +132 -0
- data/lib/moderate/services/intake_report.rb +132 -0
- data/lib/moderate/services/resolve_appeal.rb +134 -0
- data/lib/moderate/services/resolve_flag.rb +101 -0
- data/lib/moderate/services/resolve_report.rb +291 -0
- data/lib/moderate/version.rb +1 -1
- data/lib/moderate.rb +365 -18
- data/log/development.log +0 -0
- data/log/test.log +0 -0
- metadata +154 -15
data/lib/moderate.rb
CHANGED
|
@@ -1,34 +1,381 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require "set"
|
|
4
|
+
|
|
5
|
+
# We lean on a couple of ActiveSupport core extensions (notably String#constantize
|
|
6
|
+
# for the lazy class-name resolution that lets `config.user_class = "User"` work
|
|
7
|
+
# before the User model is loaded). In a Rails host these are always present; we
|
|
8
|
+
# require the specific extension here so the gem also works in a plain-Ruby
|
|
9
|
+
# context (a console, a non-Rails test) without booting all of Rails. ActiveSupport
|
|
10
|
+
# is a declared runtime dependency in the gemspec.
|
|
11
|
+
require "active_support/core_ext/string/inflections"
|
|
12
|
+
|
|
3
13
|
require_relative "moderate/version"
|
|
4
|
-
require_relative "moderate/
|
|
5
|
-
require_relative "moderate/
|
|
6
|
-
require_relative "moderate/
|
|
14
|
+
require_relative "moderate/errors"
|
|
15
|
+
require_relative "moderate/label"
|
|
16
|
+
require_relative "moderate/result"
|
|
17
|
+
require_relative "moderate/event"
|
|
18
|
+
require_relative "moderate/configuration"
|
|
7
19
|
|
|
8
|
-
|
|
9
|
-
|
|
20
|
+
# The class-level DSL (has_reporting_and_blocking / has_reportable_content / moderates). Required here,
|
|
21
|
+
# eagerly, because the engine's `moderate.active_record` initializer does
|
|
22
|
+
# `extend Moderate::Macros` inside an `on_load(:active_record)` block — the constant
|
|
23
|
+
# must already be defined by the time that hook fires. It's a plain module that only
|
|
24
|
+
# references the (autoloaded) concern constants inside its method BODIES, so loading
|
|
25
|
+
# it at gem-require time is cheap and order-independent. It is also in the engine's
|
|
26
|
+
# Zeitwerk ignore-list precisely so this manual require is the single load path.
|
|
27
|
+
require_relative "moderate/macros"
|
|
28
|
+
|
|
29
|
+
# The engine wires the gem into a Rails host: it teaches Zeitwerk to autoload the
|
|
30
|
+
# AR models / concerns / services / jobs / adapters under `lib/moderate/*`, mounts
|
|
31
|
+
# the public DSA notice form, and registers the macros on ActiveRecord. Required
|
|
32
|
+
# only when Rails is present so the value objects + macros above can still be used
|
|
33
|
+
# in a plain-Ruby context (a console, a non-Rails script) without booting an engine.
|
|
34
|
+
require_relative "moderate/engine" if defined?(::Rails::Engine)
|
|
10
35
|
|
|
36
|
+
# `moderate` — a complete Trust & Safety layer for Rails apps with user-generated
|
|
37
|
+
# content: report, block, filter, moderate, appeal, comply (EU DSA / Apple App
|
|
38
|
+
# Store Guideline 1.2 / Google Play UGC).
|
|
39
|
+
#
|
|
40
|
+
# This file is the SPINE: the `Moderate` module and its facade. Everything else in
|
|
41
|
+
# the gem imports from here. The facade is deliberately small — it owns
|
|
42
|
+
# configuration, lazy identity resolution, the notify/audit hook dispatch, the
|
|
43
|
+
# blocked-ids source-of-truth delegate, and the content-classification entry point.
|
|
44
|
+
# The heavy lifting (the AR models, the decision services, the controllers) lives
|
|
45
|
+
# in the autoloaded `app/` tree and calls back into these facade methods.
|
|
46
|
+
module Moderate
|
|
11
47
|
class << self
|
|
12
|
-
|
|
13
|
-
@configuration ||= Configuration.new
|
|
14
|
-
end
|
|
48
|
+
# --- Configuration --------------------------------------------------------
|
|
15
49
|
|
|
16
|
-
|
|
17
|
-
|
|
50
|
+
# The singleton Configuration. Lazily built so merely requiring the gem (before
|
|
51
|
+
# any initializer runs) yields a fully-defaulted, usable config.
|
|
52
|
+
def config
|
|
53
|
+
@config ||= Configuration.new
|
|
18
54
|
end
|
|
19
55
|
|
|
56
|
+
# Read alias. Some ecosystem gems expose `.configuration`; we keep `.config`
|
|
57
|
+
# as the canonical name (matches the README's `Moderate.config`) and provide
|
|
58
|
+
# `.configuration` only as a courtesy alias for muscle memory.
|
|
59
|
+
alias_method :configuration, :config
|
|
60
|
+
|
|
61
|
+
# The host's entry point: `Moderate.configure do |config| ... end`.
|
|
62
|
+
#
|
|
63
|
+
# We `yield` the live config object (so every assignment lands on the singleton)
|
|
64
|
+
# and then run `validate!` ONCE at the end — this is the documented behavior in
|
|
65
|
+
# docs/configuration.md: "The block is validated at the end of `configure`, so a
|
|
66
|
+
# typo'd mode or unknown adapter raises a plain-English ArgumentError
|
|
67
|
+
# immediately instead of failing mysteriously later." Per-setter checks already
|
|
68
|
+
# fired on assignment; this final pass catches cross-field problems (e.g. a
|
|
69
|
+
# :block filter on an async adapter).
|
|
20
70
|
def configure
|
|
21
|
-
yield
|
|
71
|
+
yield config if block_given?
|
|
72
|
+
config.validate!
|
|
73
|
+
config
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# Reset to a pristine, fully-defaulted Configuration. The primary consumer is
|
|
77
|
+
# the test suite (`Moderate.reset!` between cases); it's documented as part of
|
|
78
|
+
# the public API. We also drop the cached user-class constant so a test that
|
|
79
|
+
# swaps `config.user_class` doesn't see a stale lazily-memoized class.
|
|
80
|
+
#
|
|
81
|
+
# IMPORTANT: we do NOT clear the reportable REGISTRY here. Reportable classes are
|
|
82
|
+
# discovered once, at MODEL LOAD time (the `has_reportable_content` macro / `include
|
|
83
|
+
# Moderate::Reportable` runs `Moderate.register_reportable(self)` on inclusion).
|
|
84
|
+
# In a booted app (and the eager-loaded test suite) the models load exactly once,
|
|
85
|
+
# so wiping the registry on every `reset!` would leave `Moderate.reportable_classes`
|
|
86
|
+
# permanently empty after the first reset — the macros would never re-run to
|
|
87
|
+
# repopulate it. The registry is a load-time FACT, not configuration, so it
|
|
88
|
+
# correctly survives a config reset.
|
|
89
|
+
def reset!
|
|
90
|
+
@config = Configuration.new
|
|
91
|
+
@user_class = nil
|
|
92
|
+
self
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
# --- Identity -------------------------------------------------------------
|
|
96
|
+
|
|
97
|
+
# The actor model (who reports/blocks/gets reported/gets banned), resolved by
|
|
98
|
+
# constantizing `config.user_class` LAZILY on first use. Lazy on purpose: the
|
|
99
|
+
# initializer that sets `config.user_class = "User"` runs before the User model
|
|
100
|
+
# is necessarily loaded, so we must not constantize at configure time.
|
|
101
|
+
#
|
|
102
|
+
# Memoized, but cleared by `reset!` and re-derived if the configured name
|
|
103
|
+
# changes (guards against a stale constant in long-lived processes/tests).
|
|
104
|
+
def user_class
|
|
105
|
+
name = config.user_class
|
|
106
|
+
if @user_class.nil? || @user_class_name != name
|
|
107
|
+
@user_class = name.constantize
|
|
108
|
+
@user_class_name = name
|
|
109
|
+
end
|
|
110
|
+
@user_class
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
# --- Reportable registry --------------------------------------------------
|
|
114
|
+
|
|
115
|
+
# Auto-discovered set of classes that declared themselves reportable (via the
|
|
116
|
+
# `has_reportable_content` macro or `include Moderate::Reportable`). The Reportable concern
|
|
117
|
+
# calls `Moderate.register_reportable(self)` on inclusion, so there's NO manual
|
|
118
|
+
# registry to maintain — exactly what the README promises ("Reportable classes
|
|
119
|
+
# are auto-discovered from the `has_reportable_content` macro — no manual registry.").
|
|
120
|
+
#
|
|
121
|
+
# Stored as a Set of STRING class names (not Class objects) so we never pin a
|
|
122
|
+
# class in memory across a Zeitwerk reload in development; we constantize on read.
|
|
123
|
+
def register_reportable(klass)
|
|
124
|
+
name = klass.is_a?(Class) ? klass.name : klass.to_s
|
|
125
|
+
return if name.nil? || name.empty?
|
|
126
|
+
|
|
127
|
+
reportable_registry << name
|
|
128
|
+
name
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
# The reportable classes, constantized on demand. We rescue a NameError per
|
|
132
|
+
# entry so a class that was registered then removed (a dev-time edit) doesn't
|
|
133
|
+
# break the whole list.
|
|
134
|
+
def reportable_classes
|
|
135
|
+
reportable_registry.filter_map do |name|
|
|
136
|
+
name.constantize
|
|
137
|
+
rescue NameError
|
|
138
|
+
nil
|
|
139
|
+
end
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
# --- Notify / audit hooks -------------------------------------------------
|
|
143
|
+
|
|
144
|
+
# Dispatch a notifiable moment to the host's `config.notify` hook.
|
|
145
|
+
#
|
|
146
|
+
# Accepts either a ready-made Moderate::Event or an event NAME plus payload —
|
|
147
|
+
# the gem's services mostly call `Moderate.notify(:report_received, subject:,
|
|
148
|
+
# recipients:, ...)`, but a pre-built Event is accepted too. We always hand the
|
|
149
|
+
# hook a Moderate::Event so the host's single `case event.name` works uniformly.
|
|
150
|
+
#
|
|
151
|
+
# RETURNS a "delivered" boolean. This exists specifically for legal-email
|
|
152
|
+
# gating: DSA Art. 16(4) requires a confirmation of receipt for a notice, and
|
|
153
|
+
# the notice flow needs to know whether the confirmation actually went out so
|
|
154
|
+
# it can fall back (e.g. show an on-screen receipt) if the host hasn't wired a
|
|
155
|
+
# mailer. "Delivered" means the hook ran without raising and returned a truthy
|
|
156
|
+
# value — the default no-op hook returns nil ⇒ false, which correctly signals
|
|
157
|
+
# "nothing was sent."
|
|
158
|
+
#
|
|
159
|
+
# We never let a host hook's exception bubble into a moderation action (a slow
|
|
160
|
+
# or broken mailer must not roll back a decision). On error we audit the failure
|
|
161
|
+
# and return false.
|
|
162
|
+
def notify(event_or_name, **payload)
|
|
163
|
+
event = event_or_name.is_a?(Event) ? event_or_name : Event.new(name: event_or_name, **payload)
|
|
164
|
+
|
|
165
|
+
begin
|
|
166
|
+
result = config.notify.call(event)
|
|
167
|
+
# A lambda may legitimately return a delivery handle, a job, true, etc.
|
|
168
|
+
# Anything truthy counts as delivered; nil/false counts as not-delivered.
|
|
169
|
+
result ? true : false
|
|
170
|
+
rescue => error
|
|
171
|
+
audit(
|
|
172
|
+
name: :notify_failed,
|
|
173
|
+
subject: event.subject,
|
|
174
|
+
payload: {
|
|
175
|
+
event: event.name,
|
|
176
|
+
error_class: error.class.name,
|
|
177
|
+
error_message: error.message,
|
|
178
|
+
summary: "notify hook failed for #{event.name}: #{error.class}"
|
|
179
|
+
}
|
|
180
|
+
)
|
|
181
|
+
false
|
|
182
|
+
end
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
# Dispatch an auditable moment to the host's `config.audit` hook (no-op by
|
|
186
|
+
# default). Same Event envelope as notify, so a host can point both hooks at the
|
|
187
|
+
# same `case`. Like notify, an audit hook exception is swallowed (turned into a
|
|
188
|
+
# logged warning) so it can never roll back the action it's recording — audit is
|
|
189
|
+
# observational, never load-bearing.
|
|
190
|
+
def audit(event_or_name = nil, **payload)
|
|
191
|
+
event = event_or_name.is_a?(Event) ? event_or_name : Event.new(name: event_or_name, **payload)
|
|
192
|
+
config.audit.call(event)
|
|
193
|
+
true
|
|
194
|
+
rescue => error
|
|
195
|
+
logger&.warn("[moderate] audit hook failed for #{event&.name}: #{error.class}: #{error.message}")
|
|
196
|
+
false
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
# Run the optional `on_block` side-effect hook (cancel a pending invite, leave a
|
|
200
|
+
# shared room, …). Keyword-arg signature per docs/configuration.md. `at:` is the
|
|
201
|
+
# block row's creation time so hosts can apply time-aware teardown without
|
|
202
|
+
# reaching back into the database. Kept as a facade method so Moderate::Block has
|
|
203
|
+
# one call site and doesn't reach into config internals. No-op by default.
|
|
204
|
+
def run_on_block(blocker:, blocked:, at:)
|
|
205
|
+
config.on_block.call(blocker: blocker, blocked: blocked, at: at)
|
|
206
|
+
end
|
|
207
|
+
|
|
208
|
+
# Apply a ban via the host's `ban_handler` (suspend!, soft-delete, flip a flag,
|
|
209
|
+
# whatever "banned" means in the host's domain). Keyword-arg signature. No-op by
|
|
210
|
+
# default — the surrounding decision still audits and notifies even if no ban is
|
|
211
|
+
# wired, so the action is never silently dropped (docs/configuration.md).
|
|
212
|
+
def apply_ban(user:, by:, reason:)
|
|
213
|
+
result = config.ban_handler.call(user: user, by: by, reason: reason)
|
|
214
|
+
payload = {
|
|
215
|
+
user_id: user&.id,
|
|
216
|
+
reason: reason,
|
|
217
|
+
summary: "user #{user&.id || '(unknown)'} banned"
|
|
218
|
+
}.compact
|
|
219
|
+
|
|
220
|
+
audit(:user_banned, subject: user, actor: by, recipients: [user].compact, payload: payload)
|
|
221
|
+
notify(:user_banned, subject: user, actor: by, recipients: [user].compact, payload: payload)
|
|
222
|
+
result
|
|
223
|
+
end
|
|
224
|
+
|
|
225
|
+
# --- Blocking SSOT --------------------------------------------------------
|
|
226
|
+
|
|
227
|
+
# The single source-of-truth list of user ids "related to" `user` via a block
|
|
228
|
+
# edge — i.e. everyone this user has blocked AND everyone who has blocked them
|
|
229
|
+
# (the edge is bidirectional; once either side blocks, neither should see the
|
|
230
|
+
# other). The host enforces blocking everywhere with one query:
|
|
231
|
+
#
|
|
232
|
+
# Post.where.not(user_id: Moderate.blocked_ids_for(current_user))
|
|
233
|
+
#
|
|
234
|
+
# Delegates to Moderate::Block (the model that owns the block SQL) so the join
|
|
235
|
+
# logic lives in exactly one place. Returns an empty array for a blank user.
|
|
236
|
+
def blocked_ids_for(user)
|
|
237
|
+
return [] if user.nil?
|
|
238
|
+
|
|
239
|
+
Block.related_user_ids(user)
|
|
240
|
+
end
|
|
241
|
+
|
|
242
|
+
# --- Content classification ----------------------------------------------
|
|
243
|
+
|
|
244
|
+
# Classify a value (text or image) and return a Moderate::Result.
|
|
245
|
+
#
|
|
246
|
+
# Moderate.classify("some sketchy text") # uses the default adapter
|
|
247
|
+
# Moderate.classify(value, policy: some_policy) # uses the policy's adapter
|
|
248
|
+
#
|
|
249
|
+
# Adapter selection precedence:
|
|
250
|
+
# 1. the adapter named on the passed `policy` (per-field config / `moderates`)
|
|
251
|
+
# 2. the global `config.filter_adapter` default
|
|
252
|
+
#
|
|
253
|
+
# The adapter contract is the whole point of the gem's filtering design: ANY
|
|
254
|
+
# object responding to `classify(value) → Moderate::Result` is a valid adapter,
|
|
255
|
+
# so the built-in wordlist/image backends and a host's registered remote
|
|
256
|
+
# classifier are perfectly interchangeable. We tolerate an adapter that returns
|
|
257
|
+
# a plain Hash (a common simpler shape) by funneling it through Result.new, so
|
|
258
|
+
# older/simpler adapters keep working.
|
|
259
|
+
def classify(value, policy: nil)
|
|
260
|
+
adapter_name = policy&.adapter || config.filter_adapter
|
|
261
|
+
adapter = config.adapter_for(adapter_name)
|
|
262
|
+
|
|
263
|
+
raise ConfigurationError, "no filter adapter registered for #{adapter_name.inspect}" if adapter.nil?
|
|
264
|
+
|
|
265
|
+
raw = adapter.classify(value)
|
|
266
|
+
coerce_result(raw, source: adapter_name)
|
|
267
|
+
end
|
|
268
|
+
|
|
269
|
+
# Resolve the FilterPolicy for a given record/class + field, walking the
|
|
270
|
+
# ancestor chain so a policy declared on a base/STI parent applies to subclasses
|
|
271
|
+
# (this is why a marketplace's `Listing` policy covers `Listing::Featured`, etc.
|
|
272
|
+
# — host-agnostically). Falls back to an `:off` policy when nothing is declared,
|
|
273
|
+
# so callers can treat "no policy" and ":off" identically.
|
|
274
|
+
def filter_policy_for(record_or_class, field)
|
|
275
|
+
klass = record_or_class.is_a?(Class) ? record_or_class : record_or_class.class
|
|
276
|
+
field_s = field.to_s
|
|
277
|
+
|
|
278
|
+
policy = klass.ancestors.filter_map do |ancestor|
|
|
279
|
+
next unless ancestor.respond_to?(:name) && ancestor.name
|
|
280
|
+
|
|
281
|
+
config.filters[[ancestor.name, field_s]]
|
|
282
|
+
end.first
|
|
283
|
+
|
|
284
|
+
policy || Configuration::FilterPolicy.new(
|
|
285
|
+
class_name: klass.name, field: field_s, adapter: config.filter_adapter, mode: :off
|
|
286
|
+
)
|
|
287
|
+
end
|
|
288
|
+
|
|
289
|
+
# Register a filter adapter at runtime (the facade twin of
|
|
290
|
+
# `config.register_adapter`, so a host can call either
|
|
291
|
+
# `Moderate.register_adapter(...)` or `config.register_adapter(...)`).
|
|
292
|
+
def register_adapter(name, adapter)
|
|
293
|
+
config.register_adapter(name, adapter)
|
|
294
|
+
end
|
|
295
|
+
|
|
296
|
+
# The locale for copy the gem generates itself, falling back to the app's
|
|
297
|
+
# default. Read lazily so a host setting I18n.default_locale after our boot
|
|
298
|
+
# still wins.
|
|
299
|
+
def locale
|
|
300
|
+
config.locale || (defined?(I18n) ? I18n.default_locale : :en)
|
|
301
|
+
end
|
|
302
|
+
|
|
303
|
+
# DSA Art. 24 transparency aggregation for a period — the numbers a host
|
|
304
|
+
# publishes (notices received by intake/ground, actions taken, automated-means
|
|
305
|
+
# usage, appeal outcomes, median handling times). This is the queryable building
|
|
306
|
+
# block: the public `/transparency` page (off by default — see
|
|
307
|
+
# `config.transparency_report_enabled`) renders this, and a host that keeps the
|
|
308
|
+
# page off can still call this to publish its own report in its own format.
|
|
309
|
+
def transparency(from: nil, to: nil)
|
|
310
|
+
to ||= Time.respond_to?(:current) ? Time.current : Time.now
|
|
311
|
+
from ||= to - (365 * 24 * 60 * 60)
|
|
312
|
+
reports = Moderate::Report.where(created_at: from..to)
|
|
313
|
+
appeals = Moderate::Appeal.where(created_at: from..to)
|
|
314
|
+
flags = Moderate::Flag.where(created_at: from..to)
|
|
315
|
+
|
|
316
|
+
{
|
|
317
|
+
period: { from: from, to: to },
|
|
318
|
+
notices_by_intake: reports.group(:intake_kind).count,
|
|
319
|
+
dsa_notices_by_legal_reason: reports.where(intake_kind: "dsa").group(:legal_reason).count,
|
|
320
|
+
actions_by_basis: reports.where.not(resolved_at: nil).group(:resolution_basis).count,
|
|
321
|
+
automated_flags_by_source: flags.group(:source).count,
|
|
322
|
+
appeals_by_status: appeals.group(:status).count,
|
|
323
|
+
median_notice_action_seconds: transparency_median(reports.where.not(resolved_at: nil).pluck(:created_at, :resolved_at)),
|
|
324
|
+
median_appeal_action_seconds: transparency_median(appeals.where.not(resolved_at: nil).pluck(:created_at, :resolved_at))
|
|
325
|
+
}
|
|
326
|
+
end
|
|
327
|
+
|
|
328
|
+
private
|
|
329
|
+
|
|
330
|
+
# Median seconds between paired (created_at, resolved_at) timestamps; 0 when empty.
|
|
331
|
+
def transparency_median(pairs)
|
|
332
|
+
values = pairs.filter_map { |created_at, resolved_at| resolved_at && created_at ? (resolved_at - created_at).to_i : nil }.sort
|
|
333
|
+
return 0 if values.empty?
|
|
334
|
+
|
|
335
|
+
values[values.length / 2]
|
|
336
|
+
end
|
|
337
|
+
|
|
338
|
+
# The internal reportable-name set. Set (not Array) so re-including the concern
|
|
339
|
+
# is idempotent.
|
|
340
|
+
def reportable_registry
|
|
341
|
+
@reportable_classes ||= Set.new
|
|
342
|
+
end
|
|
343
|
+
|
|
344
|
+
# Coerce whatever an adapter returned into a Moderate::Result, stamping the
|
|
345
|
+
# adapter NAME as the result's `source` when the adapter didn't set one — this
|
|
346
|
+
# is what makes `Moderate::Flag#source` show which backend flagged an item, so
|
|
347
|
+
# the moderation queue is legible. An adapter that DID set an explicit source
|
|
348
|
+
# keeps it (we only backfill the "unknown" default). A Hash is funneled through
|
|
349
|
+
# Result.new (back-compat with the reference `{ allowed:, categories:, scores:,
|
|
350
|
+
# source:, raw: }` shape).
|
|
351
|
+
def coerce_result(raw, source:)
|
|
352
|
+
if raw.is_a?(Result)
|
|
353
|
+
return raw unless raw.source == "unknown"
|
|
354
|
+
|
|
355
|
+
return Result.new(allowed: raw.allowed?, labels: raw.labels, source: source.to_s, raw: raw.raw)
|
|
356
|
+
end
|
|
357
|
+
|
|
358
|
+
if raw.respond_to?(:to_h)
|
|
359
|
+
hash = raw.to_h.transform_keys(&:to_sym)
|
|
360
|
+
return Result.new(
|
|
361
|
+
allowed: hash[:allowed],
|
|
362
|
+
labels: hash[:labels],
|
|
363
|
+
categories: hash[:categories],
|
|
364
|
+
scores: hash[:scores],
|
|
365
|
+
source: hash[:source] || source,
|
|
366
|
+
# Tolerate the reference adapters' `:metadata` key as `raw` for audit.
|
|
367
|
+
raw: hash[:raw] || hash[:metadata]
|
|
368
|
+
)
|
|
369
|
+
end
|
|
370
|
+
|
|
371
|
+
# An adapter returning something opaque (truthy ⇒ allowed) — defensive only.
|
|
372
|
+
Result.new(allowed: raw ? true : false, source: source, raw: raw)
|
|
22
373
|
end
|
|
23
|
-
end
|
|
24
374
|
|
|
25
|
-
|
|
26
|
-
|
|
375
|
+
def logger
|
|
376
|
+
return Rails.logger if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger
|
|
27
377
|
|
|
28
|
-
|
|
29
|
-
@error_message = "contains moderatable content (bad words)"
|
|
30
|
-
@additional_words = []
|
|
31
|
-
@excluded_words = []
|
|
378
|
+
nil
|
|
32
379
|
end
|
|
33
380
|
end
|
|
34
381
|
end
|
data/log/development.log
ADDED
|
File without changes
|
data/log/test.log
ADDED
|
File without changes
|
metadata
CHANGED
|
@@ -1,57 +1,197 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: moderate
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 1.0.0.beta1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- rameerez
|
|
8
|
-
autorequire:
|
|
9
8
|
bindir: exe
|
|
10
9
|
cert_chain: []
|
|
11
|
-
date:
|
|
10
|
+
date: 2026-06-03 00:00:00.000000000 Z
|
|
12
11
|
dependencies:
|
|
13
12
|
- !ruby/object:Gem::Dependency
|
|
14
|
-
name:
|
|
13
|
+
name: activerecord
|
|
15
14
|
requirement: !ruby/object:Gem::Requirement
|
|
16
15
|
requirements:
|
|
17
16
|
- - ">="
|
|
18
17
|
- !ruby/object:Gem::Version
|
|
19
|
-
version: 7.
|
|
18
|
+
version: 7.1.0
|
|
19
|
+
- - "<"
|
|
20
|
+
- !ruby/object:Gem::Version
|
|
21
|
+
version: '9.0'
|
|
20
22
|
type: :runtime
|
|
21
23
|
prerelease: false
|
|
22
24
|
version_requirements: !ruby/object:Gem::Requirement
|
|
23
25
|
requirements:
|
|
24
26
|
- - ">="
|
|
25
27
|
- !ruby/object:Gem::Version
|
|
26
|
-
version: 7.
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
28
|
+
version: 7.1.0
|
|
29
|
+
- - "<"
|
|
30
|
+
- !ruby/object:Gem::Version
|
|
31
|
+
version: '9.0'
|
|
32
|
+
- !ruby/object:Gem::Dependency
|
|
33
|
+
name: activesupport
|
|
34
|
+
requirement: !ruby/object:Gem::Requirement
|
|
35
|
+
requirements:
|
|
36
|
+
- - ">="
|
|
37
|
+
- !ruby/object:Gem::Version
|
|
38
|
+
version: 7.1.0
|
|
39
|
+
- - "<"
|
|
40
|
+
- !ruby/object:Gem::Version
|
|
41
|
+
version: '9.0'
|
|
42
|
+
type: :runtime
|
|
43
|
+
prerelease: false
|
|
44
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
45
|
+
requirements:
|
|
46
|
+
- - ">="
|
|
47
|
+
- !ruby/object:Gem::Version
|
|
48
|
+
version: 7.1.0
|
|
49
|
+
- - "<"
|
|
50
|
+
- !ruby/object:Gem::Version
|
|
51
|
+
version: '9.0'
|
|
52
|
+
- !ruby/object:Gem::Dependency
|
|
53
|
+
name: globalid
|
|
54
|
+
requirement: !ruby/object:Gem::Requirement
|
|
55
|
+
requirements:
|
|
56
|
+
- - "~>"
|
|
57
|
+
- !ruby/object:Gem::Version
|
|
58
|
+
version: '1.0'
|
|
59
|
+
type: :runtime
|
|
60
|
+
prerelease: false
|
|
61
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
62
|
+
requirements:
|
|
63
|
+
- - "~>"
|
|
64
|
+
- !ruby/object:Gem::Version
|
|
65
|
+
version: '1.0'
|
|
66
|
+
- !ruby/object:Gem::Dependency
|
|
67
|
+
name: railties
|
|
68
|
+
requirement: !ruby/object:Gem::Requirement
|
|
69
|
+
requirements:
|
|
70
|
+
- - ">="
|
|
71
|
+
- !ruby/object:Gem::Version
|
|
72
|
+
version: 7.1.0
|
|
73
|
+
- - "<"
|
|
74
|
+
- !ruby/object:Gem::Version
|
|
75
|
+
version: '9.0'
|
|
76
|
+
type: :runtime
|
|
77
|
+
prerelease: false
|
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
79
|
+
requirements:
|
|
80
|
+
- - ">="
|
|
81
|
+
- !ruby/object:Gem::Version
|
|
82
|
+
version: 7.1.0
|
|
83
|
+
- - "<"
|
|
84
|
+
- !ruby/object:Gem::Version
|
|
85
|
+
version: '9.0'
|
|
86
|
+
description: 'moderate is a complete Trust & Safety system and content moderation
|
|
87
|
+
engine for Ruby on Rails apps. Trust & Safety (T&S) is the system within an app
|
|
88
|
+
that lets users report abusive content, block each other, filter objectionable text
|
|
89
|
+
and images before they''re posted (profanity, bad words, NSFW/nudity, etc.), and
|
|
90
|
+
run a moderation queue your admins actually use. It also allows you to easily plug
|
|
91
|
+
in automated AI moderation systems like OpenAI Moderation or AWS Rekognition to
|
|
92
|
+
quickly filter, flag and/or automatically block harmful content (text or image).
|
|
93
|
+
Any app with user-generated content (UGC) needs this: social apps, marketplaces,
|
|
94
|
+
dating, communities, forums, comments, reviews, chat, etc. The moderate gem bundles
|
|
95
|
+
the four things every UGC app needs behind one data model and one set of hooks:
|
|
96
|
+
abuse reporting (report posts, comments, profiles, listings, messages, and other
|
|
97
|
+
users), bidirectional user blocking behind a single enforced source of truth, pre-publication
|
|
98
|
+
content filtering for profanity, slurs, hate, spam, harassment, and objectionable
|
|
99
|
+
or NSFW text and images in off/block/flag modes, and an audited moderation queue
|
|
100
|
+
with locked resolve/dismiss/remove-content/ban decisions, internal appeals, and
|
|
101
|
+
statement-of-reasons notifications. Filtering uses a tiny classify(value) => Result
|
|
102
|
+
adapter contract: a fast, offline, multilingual wordlist/profanity/bad-word blocklist
|
|
103
|
+
ships as the zero-dependency default, and you bring your own classifier (OpenAI
|
|
104
|
+
omni-moderation, AWS Rekognition image/NSFW detection, Google Perspective, or self-hosted)
|
|
105
|
+
as an optional reference adapter, not a forced dependency. moderate also ships primitives
|
|
106
|
+
aligned with the EU Digital Services Act (DSA notice-and-action, statement of reasons,
|
|
107
|
+
appeals, transparency) and with the Apple App Store (Guideline 1.2) and Google Play
|
|
108
|
+
user-generated-content review rules that get apps rejected without report/block/filter.
|
|
109
|
+
It is a mountable, UI-agnostic Rails engine with no hard dependencies beyond normal
|
|
110
|
+
Rails stuff.'
|
|
30
111
|
email:
|
|
31
112
|
- rubygems@rameerez.com
|
|
32
113
|
executables: []
|
|
33
114
|
extensions: []
|
|
34
115
|
extra_rdoc_files: []
|
|
35
116
|
files:
|
|
117
|
+
- ".rubocop.yml"
|
|
118
|
+
- ".simplecov"
|
|
119
|
+
- AGENTS.md
|
|
120
|
+
- Appraisals
|
|
36
121
|
- CHANGELOG.md
|
|
122
|
+
- CLAUDE.md
|
|
37
123
|
- LICENSE.txt
|
|
38
124
|
- README.md
|
|
39
125
|
- Rakefile
|
|
126
|
+
- app/controllers/concerns/moderate/moderation.rb
|
|
127
|
+
- app/controllers/moderate/appeals_controller.rb
|
|
128
|
+
- app/controllers/moderate/application_controller.rb
|
|
129
|
+
- app/controllers/moderate/notices_controller.rb
|
|
130
|
+
- app/controllers/moderate/transparency_reports_controller.rb
|
|
131
|
+
- app/helpers/moderate/engine_helper.rb
|
|
132
|
+
- app/views/moderate/appeals/new.html.erb
|
|
133
|
+
- app/views/moderate/notices/new.html.erb
|
|
134
|
+
- app/views/moderate/transparency_reports/_summary_card.html.erb
|
|
135
|
+
- app/views/moderate/transparency_reports/show.html.erb
|
|
136
|
+
- config/moderate/blocklists/en.yml
|
|
137
|
+
- config/moderate/blocklists/es.yml
|
|
138
|
+
- config/routes.rb
|
|
139
|
+
- docs/compliance.md
|
|
140
|
+
- docs/configuration.md
|
|
141
|
+
- docs/dsa-notice-form.md
|
|
142
|
+
- docs/madmin.md
|
|
143
|
+
- docs/notifications.md
|
|
144
|
+
- examples/aws_rekognition_adapter.rb
|
|
145
|
+
- examples/openai_moderation_adapter.rb
|
|
146
|
+
- gemfiles/rails_7.1.gemfile
|
|
147
|
+
- gemfiles/rails_7.2.gemfile
|
|
148
|
+
- gemfiles/rails_8.1.gemfile
|
|
149
|
+
- lib/generators/moderate/install_generator.rb
|
|
150
|
+
- lib/generators/moderate/templates/create_moderate_tables.rb.erb
|
|
151
|
+
- lib/generators/moderate/templates/initializer.rb
|
|
152
|
+
- lib/generators/moderate/views_generator.rb
|
|
40
153
|
- lib/moderate.rb
|
|
154
|
+
- lib/moderate/configuration.rb
|
|
155
|
+
- lib/moderate/engine.rb
|
|
156
|
+
- lib/moderate/errors.rb
|
|
157
|
+
- lib/moderate/event.rb
|
|
158
|
+
- lib/moderate/filters/base.rb
|
|
159
|
+
- lib/moderate/filters/wordlist.rb
|
|
160
|
+
- lib/moderate/jobs/classify_job.rb
|
|
161
|
+
- lib/moderate/label.rb
|
|
162
|
+
- lib/moderate/macros.rb
|
|
163
|
+
- lib/moderate/models/appeal.rb
|
|
164
|
+
- lib/moderate/models/application_record.rb
|
|
165
|
+
- lib/moderate/models/block.rb
|
|
166
|
+
- lib/moderate/models/concerns/actor.rb
|
|
167
|
+
- lib/moderate/models/concerns/content_filterable.rb
|
|
168
|
+
- lib/moderate/models/concerns/reportable.rb
|
|
169
|
+
- lib/moderate/models/flag.rb
|
|
170
|
+
- lib/moderate/models/report.rb
|
|
171
|
+
- lib/moderate/result.rb
|
|
172
|
+
- lib/moderate/services/intake_appeal.rb
|
|
173
|
+
- lib/moderate/services/intake_notice.rb
|
|
174
|
+
- lib/moderate/services/intake_report.rb
|
|
175
|
+
- lib/moderate/services/resolve_appeal.rb
|
|
176
|
+
- lib/moderate/services/resolve_flag.rb
|
|
177
|
+
- lib/moderate/services/resolve_report.rb
|
|
41
178
|
- lib/moderate/text.rb
|
|
42
179
|
- lib/moderate/text_validator.rb
|
|
43
180
|
- lib/moderate/version.rb
|
|
44
181
|
- lib/moderate/word_list.rb
|
|
182
|
+
- log/development.log
|
|
183
|
+
- log/test.log
|
|
45
184
|
- sig/moderate.rbs
|
|
46
185
|
homepage: https://github.com/rameerez/moderate
|
|
47
186
|
licenses:
|
|
48
187
|
- MIT
|
|
49
188
|
metadata:
|
|
50
189
|
allowed_push_host: https://rubygems.org
|
|
51
|
-
homepage_uri: https://github.com/rameerez/moderate
|
|
52
190
|
source_code_uri: https://github.com/rameerez/moderate
|
|
53
191
|
changelog_uri: https://github.com/rameerez/moderate/blob/main/CHANGELOG.md
|
|
54
|
-
|
|
192
|
+
bug_tracker_uri: https://github.com/rameerez/moderate/issues
|
|
193
|
+
documentation_uri: https://github.com/rameerez/moderate#readme
|
|
194
|
+
rubygems_mfa_required: 'true'
|
|
55
195
|
rdoc_options: []
|
|
56
196
|
require_paths:
|
|
57
197
|
- lib
|
|
@@ -59,15 +199,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
59
199
|
requirements:
|
|
60
200
|
- - ">="
|
|
61
201
|
- !ruby/object:Gem::Version
|
|
62
|
-
version: 3.
|
|
202
|
+
version: 3.2.0
|
|
63
203
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
64
204
|
requirements:
|
|
65
205
|
- - ">="
|
|
66
206
|
- !ruby/object:Gem::Version
|
|
67
207
|
version: '0'
|
|
68
208
|
requirements: []
|
|
69
|
-
rubygems_version: 3.
|
|
70
|
-
signing_key:
|
|
209
|
+
rubygems_version: 3.6.2
|
|
71
210
|
specification_version: 4
|
|
72
|
-
summary:
|
|
211
|
+
summary: Let your Rails users report content and block each other (Trust & Safety)
|
|
73
212
|
test_files: []
|