obscene_gpt 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.rspec +3 -0
- data/.rubocop.yml +31 -0
- data/CHANGELOG.md +5 -0
- data/LICENSE.txt +21 -0
- data/README.md +412 -0
- data/Rakefile +10 -0
- data/examples/usage.rb +42 -0
- data/lib/obscene_gpt/active_model.rb +45 -0
- data/lib/obscene_gpt/configuration.rb +25 -0
- data/lib/obscene_gpt/detector.rb +75 -0
- data/lib/obscene_gpt/prompts.rb +52 -0
- data/lib/obscene_gpt/test_detector.rb +56 -0
- data/lib/obscene_gpt/version.rb +3 -0
- data/lib/obscene_gpt.rb +20 -0
- data/sig/obscene_gpt.rbs +4 -0
- metadata +75 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 57b6cd7a7afe7daa9400f7a24e1609c1195f96bb74d66c16bb28ef20d18ee1f0
|
4
|
+
data.tar.gz: d3d01454d54408acb0ebe8e97e020afb47dd49d1da39c2c5921314bb38c224c6
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: a76f7ba702c0bdc886f0ee805b26f72278bad23ca4f88a2e57387b342d0e213d2d93c30fce52800b0425a76b1f9ac7aa99078b6ba51d1236dc29fb18b0d84e6e
|
7
|
+
data.tar.gz: '08145d5400a4b29d120824e51b286be6a6d4403ef5f8ecf18d260dd68f936b01486408bc041119aa22f39f5f05f803860191fc96a0f120e13499bfcc47384ac4'
|
data/.rspec
ADDED
data/.rubocop.yml
ADDED
@@ -0,0 +1,31 @@
|
|
1
|
+
AllCops:
|
2
|
+
TargetRubyVersion: 3.1
|
3
|
+
NewCops: enable
|
4
|
+
SuggestExtensions: false
|
5
|
+
|
6
|
+
Style/StringLiterals:
|
7
|
+
EnforcedStyle: double_quotes
|
8
|
+
|
9
|
+
Style/StringLiteralsInInterpolation:
|
10
|
+
EnforcedStyle: double_quotes
|
11
|
+
|
12
|
+
Style/TrailingCommaInHashLiteral:
|
13
|
+
EnforcedStyleForMultiline: comma
|
14
|
+
|
15
|
+
Style/MutableConstant:
|
16
|
+
Enabled: false
|
17
|
+
|
18
|
+
Style/TrailingCommaInArrayLiteral:
|
19
|
+
EnforcedStyleForMultiline: comma
|
20
|
+
|
21
|
+
Style/TrailingCommaInArguments:
|
22
|
+
EnforcedStyleForMultiline: comma
|
23
|
+
|
24
|
+
Style/Documentation:
|
25
|
+
Enabled: false
|
26
|
+
|
27
|
+
Style/FrozenStringLiteralComment:
|
28
|
+
Enabled: false
|
29
|
+
|
30
|
+
Metrics/BlockLength:
|
31
|
+
Enabled: false
|
data/CHANGELOG.md
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2025 Daniel Perez
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
13
|
+
all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,412 @@
|
|
1
|
+
# ObsceneGpt
|
2
|
+
|
3
|
+
A Ruby gem that integrates with OpenAI's API to detect whether given text contains obscene, inappropriate, or NSFW content. It provides a simple interface for content moderation using AI.
|
4
|
+
|
5
|
+
## Installation
|
6
|
+
|
7
|
+
Add this line to your application's Gemfile:
|
8
|
+
|
9
|
+
```ruby
|
10
|
+
gem 'obscene_gpt'
|
11
|
+
```
|
12
|
+
|
13
|
+
And then execute:
|
14
|
+
|
15
|
+
```bash
|
16
|
+
bundle install
|
17
|
+
```
|
18
|
+
|
19
|
+
Or install it yourself as:
|
20
|
+
|
21
|
+
```bash
|
22
|
+
gem install obscene_gpt
|
23
|
+
```
|
24
|
+
|
25
|
+
## Setup
|
26
|
+
|
27
|
+
You'll need an OpenAI API key to use this gem. You can either:
|
28
|
+
|
29
|
+
1. Set it as an environment variable:
|
30
|
+
|
31
|
+
```bash
|
32
|
+
export OPENAI_API_KEY="your-openai-api-key-here"
|
33
|
+
```
|
34
|
+
|
35
|
+
2. Configure it globally in your application (recommended):
|
36
|
+
|
37
|
+
```ruby
|
38
|
+
ObsceneGpt.configure do |config|
|
39
|
+
config.api_key = "your-openai-api-key-here"
|
40
|
+
config.model = "gpt-4.1-nano"
|
41
|
+
end
|
42
|
+
```
|
43
|
+
|
44
|
+
3. Pass it directly when instantiating the detector
|
45
|
+
|
46
|
+
```ruby
|
47
|
+
detector = ObsceneGpt::Detector.new(api_key: "your-openai-api-key-here")
|
48
|
+
```
|
49
|
+
|
50
|
+
## Usage
|
51
|
+
|
52
|
+
See the `examples/usage.rb` file for usage examples.
|
53
|
+
|
54
|
+
### Basic Usage
|
55
|
+
|
56
|
+
```ruby
|
57
|
+
require 'obscene_gpt'
|
58
|
+
|
59
|
+
# Configure once in your app initialization
|
60
|
+
ObsceneGpt.configure do |config|
|
61
|
+
config.api_key = "your-openai-api-key-here"
|
62
|
+
config.model = "gpt-4.1-nano"
|
63
|
+
end
|
64
|
+
|
65
|
+
detector = ObsceneGpt::Detector.new
|
66
|
+
|
67
|
+
result = detector.detect("Hello, how are you today?")
|
68
|
+
puts result
|
69
|
+
# => {"obscene" => false, "confidence" => 0.95, "reasoning" => "The text is a polite greeting with no inappropriate content.", "categories" => []}
|
70
|
+
|
71
|
+
result = detector.detect("Some offensive text with BAAD words")
|
72
|
+
puts result
|
73
|
+
# => {"obscene" => true, "confidence" => 0.85, "reasoning" => "The text contains vulgar language with the word 'BAAD', which is likely intended as a vulgar or inappropriate term.", "categories" => ["profanity"]}
|
74
|
+
```
|
75
|
+
|
76
|
+
### ActiveModel Validator
|
77
|
+
|
78
|
+
When ActiveModel is available, you can use the built-in validator to automatically check for obscene content in your models:
|
79
|
+
|
80
|
+
```ruby
|
81
|
+
class Post < ActiveRecord::Base
|
82
|
+
validates :content, obscene_content: true
|
83
|
+
validates :title, obscene_content: { message: "Title contains inappropriate language" }
|
84
|
+
validates :description, obscene_content: { threshold: 0.9 }
|
85
|
+
validates :comment, obscene_content: {
|
86
|
+
threshold: 0.8,
|
87
|
+
message: "Comment violates community guidelines"
|
88
|
+
}
|
89
|
+
end
|
90
|
+
|
91
|
+
# The validator automatically caches results to avoid duplicate API calls
|
92
|
+
post = Post.new(content: "Some potentially inappropriate content")
|
93
|
+
if post.valid?
|
94
|
+
puts "Post is valid"
|
95
|
+
else
|
96
|
+
puts "Validation errors: #{post.errors.full_messages}"
|
97
|
+
end
|
98
|
+
```
|
99
|
+
|
100
|
+
**Important:** The validator uses Rails caching to ensure only one API call is made per unique text content. Results are cached for 1 hour to avoid repeated API calls for the same content.
|
101
|
+
|
102
|
+
## Important considerations
|
103
|
+
|
104
|
+
### Cost
|
105
|
+
|
106
|
+
The cost of using this gem is based on the number of API calls made.
|
107
|
+
A very short input text will have roughly 170 tokens, and each 200 characters adds roughly another 50 tokens.
|
108
|
+
The simple schema has 17 output tokens and the full schema has ~50 (depending on the length of the reasoning and the attributes).
|
109
|
+
Using the simple schema and with an average of 200 chars per request, the cost (with the gpt-4.1-nano model) is roughly $1 per 35,000 requests.
|
110
|
+
|
111
|
+
### Rate limits
|
112
|
+
|
113
|
+
The OpenAI API has rate limits that depends on the model you are using.
|
114
|
+
The gpt-4.1-nano model has a rate limit of 500 requests per minute with a normal paid subscription.
|
115
|
+
|
116
|
+
### Latency
|
117
|
+
|
118
|
+
Calling an API will obviously add some latency to your application.
|
119
|
+
The latency is dependent on the model you are using and the length of the text you are analyzing.
|
120
|
+
We do not recommend using this gem in latency-sensitive application.
|
121
|
+
|
122
|
+
## API Reference
|
123
|
+
|
124
|
+
### Configuration
|
125
|
+
|
126
|
+
#### ObsceneGpt.configure(&block)
|
127
|
+
|
128
|
+
Configure the gem globally.
|
129
|
+
|
130
|
+
```ruby
|
131
|
+
ObsceneGpt.configure do |config|
|
132
|
+
config.api_key = "your-api-key"
|
133
|
+
config.model = "gpt-4.1-nano"
|
134
|
+
config.schema = ObsceneGpt::Prompts::SIMPLE_SCHEMA
|
135
|
+
config.prompt = ObsceneGpt::Prompts::SYSTEM_PROMPT
|
136
|
+
end
|
137
|
+
```
|
138
|
+
|
139
|
+
#### ObsceneGpt.configuration
|
140
|
+
|
141
|
+
Get the current configuration object.
|
142
|
+
|
143
|
+
### ObsceneGpt::Detector
|
144
|
+
|
145
|
+
#### ObsceneGpt::Detector.new(api_key: nil, model: nil)
|
146
|
+
|
147
|
+
Creates a new detector instance.
|
148
|
+
|
149
|
+
#### ObsceneGpt::Detector#detect(text)
|
150
|
+
|
151
|
+
Detects whether the given text contains obscene content.
|
152
|
+
|
153
|
+
**Parameters:**
|
154
|
+
|
155
|
+
- `text` (String): Text to analyze.
|
156
|
+
|
157
|
+
**Returns:** Hash with detection results. See `Response Format` for more details.
|
158
|
+
|
159
|
+
**Raises:**
|
160
|
+
|
161
|
+
- `ObsceneGpt::Error`: If there's an OpenAI API error
|
162
|
+
|
163
|
+
#### ObsceneGpt::Detector#detect_many(texts)
|
164
|
+
|
165
|
+
Detects whether the given texts contain obscene content.
|
166
|
+
|
167
|
+
**Parameters:**
|
168
|
+
|
169
|
+
- `texts` (Array<String>): Texts to analyze.
|
170
|
+
|
171
|
+
**Returns:** Array of hashes with detection results. See `Response Format` for more details.
|
172
|
+
|
173
|
+
**Raises:**
|
174
|
+
|
175
|
+
- `ObsceneGpt::Error`: If there's an OpenAI API error
|
176
|
+
|
177
|
+
## Response Format
|
178
|
+
|
179
|
+
The detection methods return a hash (or array of hashes) with the following structure:
|
180
|
+
|
181
|
+
```ruby
|
182
|
+
{
|
183
|
+
obscene: true, # Boolean: whether content is inappropriate
|
184
|
+
confidence: 0.85, # Float: confidence score (0.0-1.0)
|
185
|
+
reasoning: "Contains explicit language and profanity",
|
186
|
+
categories: ["profanity", "explicit"] # Array of detected categories (["sexual", "profanity", "hate", "violent", "other"])
|
187
|
+
}
|
188
|
+
```
|
189
|
+
|
190
|
+
## Configuration Options
|
191
|
+
|
192
|
+
### Default options
|
193
|
+
|
194
|
+
The default configuration is:
|
195
|
+
|
196
|
+
```ruby
|
197
|
+
config.api_key = ENV["OPENAI_API_KEY"]
|
198
|
+
config.model = "gpt-4.1-nano"
|
199
|
+
config.schema = ObsceneGpt::Prompts::SIMPLE_SCHEMA
|
200
|
+
config.prompt = ObsceneGpt::Prompts::SYSTEM_PROMPT
|
201
|
+
config.test_mode = false
|
202
|
+
config.test_detector_class = ObsceneGpt::TestDetector
|
203
|
+
```
|
204
|
+
|
205
|
+
### Test Mode
|
206
|
+
|
207
|
+
To avoid making API calls during testing, you can enable test mode:
|
208
|
+
|
209
|
+
```ruby
|
210
|
+
ObsceneGpt.configure do |config|
|
211
|
+
config.test_mode = true
|
212
|
+
end
|
213
|
+
```
|
214
|
+
|
215
|
+
When test mode is enabled, the detector will return mock responses based on simple pattern matching instead of making actual API calls. This is useful for:
|
216
|
+
|
217
|
+
- Running tests without API costs
|
218
|
+
- Faster test execution
|
219
|
+
- Avoiding rate limits during development
|
220
|
+
|
221
|
+
**Note:** Test mode uses basic pattern matching and is not as accurate as the actual AI model. It's intended for testing purposes only.
|
222
|
+
|
223
|
+
#### Custom Test Detectors
|
224
|
+
|
225
|
+
You can also configure a custom test detector class for more sophisticated test behavior:
|
226
|
+
|
227
|
+
```ruby
|
228
|
+
class MyCustomTestDetector
|
229
|
+
attr_reader :schema
|
230
|
+
|
231
|
+
def initialize(schema: nil)
|
232
|
+
@schema = schema || ObsceneGpt::Prompts::SIMPLE_SCHEMA
|
233
|
+
end
|
234
|
+
|
235
|
+
def detect_many(texts)
|
236
|
+
texts.map do |text|
|
237
|
+
{
|
238
|
+
obscene: text.include?("bad_word"),
|
239
|
+
confidence: 0.9
|
240
|
+
}
|
241
|
+
end
|
242
|
+
end
|
243
|
+
|
244
|
+
def detect(text)
|
245
|
+
detect_many([text])[0]
|
246
|
+
end
|
247
|
+
end
|
248
|
+
|
249
|
+
ObsceneGpt.configure do |config|
|
250
|
+
config.test_mode = true
|
251
|
+
config.test_detector_class = MyCustomTestDetector
|
252
|
+
end
|
253
|
+
```
|
254
|
+
|
255
|
+
Custom test detectors must implement:
|
256
|
+
|
257
|
+
- `#initialize(schema: nil)` - Accepts an optional schema parameter
|
258
|
+
- `#detect_many(texts)` - Returns an array of result hashes
|
259
|
+
|
260
|
+
See `examples/custom_test_detector.rb` for more examples.
|
261
|
+
|
262
|
+
### Model
|
263
|
+
|
264
|
+
We recommend using the `gpt-4.1-nano` model for cost efficiency.
|
265
|
+
Given the simplicity of the task, it's typically not necessary to use a more expensive model.
|
266
|
+
|
267
|
+
See [OpenAI's documentation](https://platform.openai.com/docs/pricing) for more information.
|
268
|
+
|
269
|
+
### Prompt
|
270
|
+
|
271
|
+
The system prompt can be found in `lib/obscene_gpt/prompts.rb`.
|
272
|
+
This is a basic prompt that can be used to detect obscene content.
|
273
|
+
You can use a custom prompt if you need to by setting the `prompt` option in the configuration.
|
274
|
+
|
275
|
+
### Schema
|
276
|
+
|
277
|
+
This library uses a JSON schema to enforce the response from the OpenAI API.
|
278
|
+
There are two schemas available:
|
279
|
+
|
280
|
+
- `ObsceneGpt::Prompts::SIMPLE_SCHEMA`: A simple schema that only includes the `obscene` and `confidence` fields.
|
281
|
+
- `ObsceneGpt::Prompts::FULL_SCHEMA`: A full schema that includes the `obscene`, `confidence`, `reasoning`, and `categories` fields.
|
282
|
+
|
283
|
+
You can use a custom schema if you need to by setting the `schema` option in the configuration.
|
284
|
+
|
285
|
+
### Configuration Precedence
|
286
|
+
|
287
|
+
1. Explicit parameters passed to methods
|
288
|
+
2. Global configuration
|
289
|
+
3. Environment variables (for API key only)
|
290
|
+
|
291
|
+
## ActiveModel Integration
|
292
|
+
|
293
|
+
The `ObsceneContentValidator` is available when ActiveModel is loaded.
|
294
|
+
`active_model` needs to be required before obscene_gpt.
|
295
|
+
|
296
|
+
### Usage
|
297
|
+
|
298
|
+
```ruby
|
299
|
+
class Post < ActiveRecord::Base
|
300
|
+
validates :content, :title, :description, obscene_content: true
|
301
|
+
end
|
302
|
+
```
|
303
|
+
|
304
|
+
**Note**: Each instance of this validator will make a request to the OpenAI API.
|
305
|
+
Therefore, it is recommended to pass all the attributes you want to check to the validator at once as shown above.
|
306
|
+
|
307
|
+
### Options
|
308
|
+
|
309
|
+
- `threshold` (Float): Custom confidence threshold (0.0-1.0) for determining when content is considered inappropriate. Default: Uses `ObsceneGpt.configuration.profanity_threshold`
|
310
|
+
- `message` (String): Custom error message to display when validation fails. Default: Uses AI reasoning if available, otherwise "contains inappropriate content"
|
311
|
+
|
312
|
+
### Per-Attribute Options
|
313
|
+
|
314
|
+
You can also configure different options for different attributes in a single validation call:
|
315
|
+
|
316
|
+
```ruby
|
317
|
+
class Post < ActiveRecord::Base
|
318
|
+
validates :title, :content, obscene_content: {
|
319
|
+
title: { threshold: 0.8, message: "Title is too inappropriate" },
|
320
|
+
content: { threshold: 0.7, message: "Content needs moderation" }
|
321
|
+
}
|
322
|
+
end
|
323
|
+
```
|
324
|
+
|
325
|
+
### Configuration Precedence
|
326
|
+
|
327
|
+
The validator uses the following precedence for options:
|
328
|
+
|
329
|
+
**Threshold:**
|
330
|
+
|
331
|
+
1. Per-attribute option (e.g., `title: { threshold: 0.8 }`)
|
332
|
+
2. Validator option (e.g., `threshold: 0.8`)
|
333
|
+
3. Configuration default (`ObsceneGpt.configuration.profanity_threshold`)
|
334
|
+
|
335
|
+
**Message:**
|
336
|
+
|
337
|
+
1. Per-attribute message (e.g., `title: { message: "..." }`)
|
338
|
+
2. Global message (e.g., `message: "..."`)
|
339
|
+
3. AI reasoning (if available, only when schema is `ObsceneGpt::Prompts::FULL_SCHEMA`)
|
340
|
+
4. Default message ("contains inappropriate content")
|
341
|
+
|
342
|
+
### Examples
|
343
|
+
|
344
|
+
**Basic validation:**
|
345
|
+
|
346
|
+
```ruby
|
347
|
+
class Post < ActiveRecord::Base
|
348
|
+
validates :content, obscene_content: true
|
349
|
+
end
|
350
|
+
```
|
351
|
+
|
352
|
+
**With custom message:**
|
353
|
+
|
354
|
+
```ruby
|
355
|
+
class Post < ActiveRecord::Base
|
356
|
+
validates :title, obscene_content: { message: "Title contains inappropriate content" }
|
357
|
+
end
|
358
|
+
```
|
359
|
+
|
360
|
+
**With custom threshold:**
|
361
|
+
|
362
|
+
```ruby
|
363
|
+
class Post < ActiveRecord::Base
|
364
|
+
validates :description, obscene_content: { threshold: 0.9 }
|
365
|
+
end
|
366
|
+
```
|
367
|
+
|
368
|
+
**With both custom threshold and message:**
|
369
|
+
|
370
|
+
```ruby
|
371
|
+
class Post < ActiveRecord::Base
|
372
|
+
validates :comment, obscene_content: {
|
373
|
+
threshold: 0.8,
|
374
|
+
message: "Comment violates community guidelines"
|
375
|
+
}
|
376
|
+
end
|
377
|
+
```
|
378
|
+
|
379
|
+
**Per-attribute configuration:**
|
380
|
+
|
381
|
+
```ruby
|
382
|
+
class Post < ActiveRecord::Base
|
383
|
+
validates :title, :content, obscene_content: {
|
384
|
+
title: { threshold: 0.8, message: "Title is too inappropriate" },
|
385
|
+
content: { threshold: 0.7, message: "Content needs moderation" }
|
386
|
+
}
|
387
|
+
end
|
388
|
+
```
|
389
|
+
|
390
|
+
**Mixed global and per-attribute options:**
|
391
|
+
|
392
|
+
```ruby
|
393
|
+
class Post < ActiveRecord::Base
|
394
|
+
validates :title, :content, obscene_content: {
|
395
|
+
threshold: 0.8, # Global threshold
|
396
|
+
message: "Contains inappropriate content", # Global message
|
397
|
+
title: { threshold: 0.9 } # Override threshold for title only
|
398
|
+
}
|
399
|
+
end
|
400
|
+
```
|
401
|
+
|
402
|
+
## Development
|
403
|
+
|
404
|
+
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
405
|
+
|
406
|
+
## Contributing
|
407
|
+
|
408
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/danhper/obscene_gpt.
|
409
|
+
|
410
|
+
## License
|
411
|
+
|
412
|
+
The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
|
data/Rakefile
ADDED
data/examples/usage.rb
ADDED
@@ -0,0 +1,42 @@
|
|
1
|
+
require "obscene_gpt"
|
2
|
+
|
3
|
+
# Configure the gem globally (do this once in your app initialization)
|
4
|
+
ObsceneGpt.configure do |config|
|
5
|
+
# config.api_key = "your-openai-api-key-here"
|
6
|
+
config.model = "gpt-4.1-nano"
|
7
|
+
config.schema = ObsceneGpt::Prompts::FULL_SCHEMA
|
8
|
+
end
|
9
|
+
|
10
|
+
detector = ObsceneGpt::Detector.new
|
11
|
+
|
12
|
+
texts_to_analyze = [
|
13
|
+
"Hello, how are you today?",
|
14
|
+
"This is a beautiful day!",
|
15
|
+
"I love programming in Ruby.",
|
16
|
+
"Some potentially inappropriate content here...",
|
17
|
+
"This text contains explicit language and should be flagged.",
|
18
|
+
]
|
19
|
+
|
20
|
+
detector.detect_many(texts_to_analyze).each_with_index do |result, index|
|
21
|
+
puts "Text: #{texts_to_analyze[index]}"
|
22
|
+
puts "Obscene: #{result[:obscene]}"
|
23
|
+
puts "Confidence: #{result[:confidence]}"
|
24
|
+
puts "Reasoning: #{result[:reasoning]}"
|
25
|
+
puts "Categories: #{result[:categories]}"
|
26
|
+
puts "--------------------------------"
|
27
|
+
end
|
28
|
+
|
29
|
+
if defined?(ActiveRecord)
|
30
|
+
class Post < ActiveRecord::Base
|
31
|
+
validates :title, :content, obscene_content: {
|
32
|
+
title: { message: "Title contains inappropriate language" },
|
33
|
+
}
|
34
|
+
end
|
35
|
+
|
36
|
+
post = Post.new(title: "Some normal content", content: "Some very inappropriate content")
|
37
|
+
if post.valid?
|
38
|
+
puts "Post is valid"
|
39
|
+
else
|
40
|
+
puts "Validation errors: #{post.errors.full_messages}"
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
begin
|
2
|
+
require "active_model"
|
3
|
+
rescue LoadError
|
4
|
+
# do nothing
|
5
|
+
end
|
6
|
+
|
7
|
+
if defined?(ActiveModel)
|
8
|
+
class ObsceneContentValidator < ActiveModel::EachValidator
|
9
|
+
def validate(record)
|
10
|
+
to_validate = compute_attributes_to_validate(record)
|
11
|
+
return if to_validate.empty?
|
12
|
+
|
13
|
+
results = ObsceneGpt.detect_many(to_validate.values)
|
14
|
+
format_errors(record, to_validate, results)
|
15
|
+
end
|
16
|
+
|
17
|
+
private
|
18
|
+
|
19
|
+
def compute_attributes_to_validate(record)
|
20
|
+
attributes.map do |attribute|
|
21
|
+
value = record.read_attribute_for_validation(attribute)
|
22
|
+
next if value.nil? || value.blank?
|
23
|
+
|
24
|
+
[attribute, prepare_value_for_validation(value, record, attribute)]
|
25
|
+
end.compact.to_h
|
26
|
+
end
|
27
|
+
|
28
|
+
def format_errors(record, to_validate, results)
|
29
|
+
results.each_with_index do |result, index|
|
30
|
+
attribute = to_validate.keys[index]
|
31
|
+
threshold = option_for(:threshold, attribute, ObsceneGpt.configuration.profanity_threshold)
|
32
|
+
|
33
|
+
if result[:obscene] && result[:confidence] >= threshold
|
34
|
+
message = option_for(:message, attribute, result[:reasoning] || "contains inappropriate content")
|
35
|
+
record.errors.add(attribute, :obscene_content, message: message)
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def option_for(key, attribute, default = nil)
|
41
|
+
(options[attribute] && options[attribute][key]) ||
|
42
|
+
options[key] || default
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
module ObsceneGpt
|
2
|
+
class << self
|
3
|
+
def configure
|
4
|
+
yield(configuration)
|
5
|
+
end
|
6
|
+
|
7
|
+
def configuration
|
8
|
+
@configuration ||= Configuration.new
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
class Configuration
|
13
|
+
attr_accessor :api_key, :model, :schema, :prompt, :profanity_threshold, :test_mode, :test_detector_class
|
14
|
+
|
15
|
+
def initialize
|
16
|
+
@api_key = ENV.fetch("OPENAI_API_KEY", nil)
|
17
|
+
@model = "gpt-4.1-nano"
|
18
|
+
@prompt = Prompts::SYSTEM_PROMPT
|
19
|
+
@schema = Prompts::SIMPLE_SCHEMA
|
20
|
+
@profanity_threshold = 0.8
|
21
|
+
@test_mode = false
|
22
|
+
@test_detector_class = TestDetector
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,75 @@
|
|
1
|
+
require "json"
|
2
|
+
require "openai"
|
3
|
+
|
4
|
+
module ObsceneGpt
|
5
|
+
class Detector
|
6
|
+
attr_reader :client, :model, :schema, :prompt
|
7
|
+
|
8
|
+
def initialize(api_key: nil, model: nil, schema: nil, prompt: nil)
|
9
|
+
api_key ||= ObsceneGpt.configuration.api_key
|
10
|
+
|
11
|
+
@client = OpenAI::Client.new(access_token: api_key)
|
12
|
+
@model = model || ObsceneGpt.configuration.model
|
13
|
+
@schema = schema || ObsceneGpt.configuration.schema
|
14
|
+
@prompt = prompt || ObsceneGpt.configuration.prompt
|
15
|
+
end
|
16
|
+
|
17
|
+
# Detects whether the given texts contain obscene content
|
18
|
+
# @param texts [Array<String>] The texts to analyze
|
19
|
+
# @return [Array<Hash>] An array of hashes containing the detection result with keys:
|
20
|
+
# - :obscene [Boolean] Whether the text contains obscene content
|
21
|
+
# - :confidence [Float] Confidence score (0.0 to 1.0)
|
22
|
+
# - :reasoning [String] Explanation for the classification (only for full schema)
|
23
|
+
# - :categories [Array<String>] Categories of inappropriate content found (only for full schema)
|
24
|
+
def detect_many(texts)
|
25
|
+
if ObsceneGpt.configuration.test_mode
|
26
|
+
test_detector = ObsceneGpt.configuration.test_detector_class.new(schema: @schema)
|
27
|
+
return test_detector.detect_many(texts)
|
28
|
+
end
|
29
|
+
|
30
|
+
run_detect_many(texts)
|
31
|
+
end
|
32
|
+
|
33
|
+
# Detects whether the given text contains obscene content
|
34
|
+
# See #detect_many for more details
|
35
|
+
def detect(text)
|
36
|
+
detect_many([text])[0]
|
37
|
+
end
|
38
|
+
|
39
|
+
private
|
40
|
+
|
41
|
+
def run_detect_many(texts)
|
42
|
+
response = @client.responses.create(parameters: make_query(texts))
|
43
|
+
|
44
|
+
JSON.parse(response.dig("output", 0, "content", 0, "text"))["results"].map { |r| r.transform_keys(&:to_sym) }
|
45
|
+
rescue OpenAI::Error, Faraday::Error => e
|
46
|
+
body = e.respond_to?(:response) ? e.response[:body] : ""
|
47
|
+
raise ObsceneGpt::Error, "OpenAI API error: #{e.message}\n#{body}"
|
48
|
+
end
|
49
|
+
|
50
|
+
def make_query(texts)
|
51
|
+
text_format = { name: "content-moderation", type: "json_schema", schema: make_schema(texts.length), strict: true }
|
52
|
+
{
|
53
|
+
model: @model,
|
54
|
+
text: { format: text_format },
|
55
|
+
input: [{
|
56
|
+
role: "user",
|
57
|
+
content: [{ type: "input_text", text: @prompt },
|
58
|
+
{ type: "input_text", text: JSON.dump(texts) }],
|
59
|
+
}],
|
60
|
+
}
|
61
|
+
end
|
62
|
+
|
63
|
+
def make_schema(texts_count)
|
64
|
+
array_schema = { type: "array", items: @schema, minItems: texts_count, maxItems: texts_count }
|
65
|
+
{
|
66
|
+
type: "object",
|
67
|
+
properties: {
|
68
|
+
results: array_schema,
|
69
|
+
},
|
70
|
+
required: %w[results],
|
71
|
+
additionalProperties: false,
|
72
|
+
}
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
@@ -0,0 +1,52 @@
|
|
1
|
+
module ObsceneGpt
|
2
|
+
module Prompts
|
3
|
+
SYSTEM_PROMPT = <<~PROMPT.freeze
|
4
|
+
You are a content moderation AI that analyzes texts for obscene, inappropriate, or NSFW content.
|
5
|
+
|
6
|
+
Your task is to determine if the given text contains:
|
7
|
+
- Explicit sexual content
|
8
|
+
- Profanity or vulgar language
|
9
|
+
- Hate speech or discriminatory language
|
10
|
+
- Violent or graphic content
|
11
|
+
- Other inappropriate material
|
12
|
+
|
13
|
+
You will be given a JSON array of texts.
|
14
|
+
You will need to analyze each text and determine if it contains any of the above content.
|
15
|
+
PROMPT
|
16
|
+
|
17
|
+
SIMPLE_SCHEMA = {
|
18
|
+
type: "object",
|
19
|
+
properties: {
|
20
|
+
obscene: {
|
21
|
+
type: "boolean",
|
22
|
+
description: "Whether the text contains obscene content",
|
23
|
+
},
|
24
|
+
confidence: {
|
25
|
+
type: "number",
|
26
|
+
minimum: 0,
|
27
|
+
maximum: 1,
|
28
|
+
description: "A confidence score between 0 and 1",
|
29
|
+
},
|
30
|
+
},
|
31
|
+
required: %w[obscene confidence],
|
32
|
+
additionalProperties: false,
|
33
|
+
}.freeze
|
34
|
+
|
35
|
+
FULL_SCHEMA = {
|
36
|
+
type: "object",
|
37
|
+
properties: SIMPLE_SCHEMA[:properties].merge(
|
38
|
+
reasoning: {
|
39
|
+
type: "string",
|
40
|
+
description: "A reasoning for the classification",
|
41
|
+
},
|
42
|
+
categories: {
|
43
|
+
type: "array",
|
44
|
+
items: { type: "string", enum: %w[sexual profanity hate violent other] },
|
45
|
+
description: "A list of categories that the text belongs to",
|
46
|
+
},
|
47
|
+
),
|
48
|
+
required: %w[obscene confidence reasoning categories],
|
49
|
+
additionalProperties: false,
|
50
|
+
}.freeze
|
51
|
+
end
|
52
|
+
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
module ObsceneGpt
|
2
|
+
class TestDetector
|
3
|
+
attr_reader :schema
|
4
|
+
|
5
|
+
def initialize(schema: nil)
|
6
|
+
@schema = schema || ObsceneGpt::Prompts::SIMPLE_SCHEMA
|
7
|
+
end
|
8
|
+
|
9
|
+
# Detects whether the given texts contain obscene content using test mode
|
10
|
+
# @param texts [Array<String>] The texts to analyze
|
11
|
+
# @return [Array<Hash>] An array of hashes containing the detection result
|
12
|
+
def detect_many(texts) # rubocop:disable Metrics/MethodLength
|
13
|
+
texts.map do |text|
|
14
|
+
# Simple heuristic for test mode: detect common profanity patterns
|
15
|
+
is_obscene = detect_obscene_patterns(text)
|
16
|
+
confidence = is_obscene ? 0.85 : 0.95
|
17
|
+
|
18
|
+
result = {
|
19
|
+
obscene: is_obscene,
|
20
|
+
confidence: confidence,
|
21
|
+
}
|
22
|
+
|
23
|
+
# Add additional fields if using full schema
|
24
|
+
if @schema == Prompts::FULL_SCHEMA
|
25
|
+
result[:reasoning] = is_obscene ? "Contains inappropriate content" : "Clean text"
|
26
|
+
result[:categories] = is_obscene ? ["profanity"] : []
|
27
|
+
end
|
28
|
+
|
29
|
+
result
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
# Detects whether the given text contains obscene content
|
34
|
+
# @param text [String] The text to analyze
|
35
|
+
# @return [Hash] Detection result
|
36
|
+
def detect(text)
|
37
|
+
detect_many([text])[0]
|
38
|
+
end
|
39
|
+
|
40
|
+
private
|
41
|
+
|
42
|
+
def detect_obscene_patterns(text)
|
43
|
+
# Simple pattern matching for test mode
|
44
|
+
# This is just for testing - not meant to be comprehensive
|
45
|
+
profanity_patterns = [
|
46
|
+
/\b(fuck|fucked|fucking|shit|bitch|ass|damn|hell)\b/i,
|
47
|
+
/\b(sex|porn|nude|naked)\b/i,
|
48
|
+
/\b(kill|murder|death|blood)\b/i,
|
49
|
+
/\b(hate|racist|sexist)\b/i,
|
50
|
+
/\b(gore)\b/i,
|
51
|
+
]
|
52
|
+
|
53
|
+
profanity_patterns.any? { |pattern| text.match?(pattern) }
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
data/lib/obscene_gpt.rb
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
require_relative "obscene_gpt/version"
|
2
|
+
require_relative "obscene_gpt/prompts"
|
3
|
+
require_relative "obscene_gpt/configuration"
|
4
|
+
require_relative "obscene_gpt/test_detector"
|
5
|
+
require_relative "obscene_gpt/detector"
|
6
|
+
require_relative "obscene_gpt/active_model"
|
7
|
+
|
8
|
+
module ObsceneGpt
|
9
|
+
class Error < StandardError; end
|
10
|
+
|
11
|
+
class << self
|
12
|
+
def detect_many(texts)
|
13
|
+
ObsceneGpt::Detector.new.detect_many(texts)
|
14
|
+
end
|
15
|
+
|
16
|
+
def detect(text)
|
17
|
+
ObsceneGpt::Detector.new.detect(text)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
data/sig/obscene_gpt.rbs
ADDED
metadata
ADDED
@@ -0,0 +1,75 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: obscene_gpt
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Daniel Perez
|
8
|
+
bindir: exe
|
9
|
+
cert_chain: []
|
10
|
+
date: 2025-06-30 00:00:00.000000000 Z
|
11
|
+
dependencies:
|
12
|
+
- !ruby/object:Gem::Dependency
|
13
|
+
name: ruby-openai
|
14
|
+
requirement: !ruby/object:Gem::Requirement
|
15
|
+
requirements:
|
16
|
+
- - "~>"
|
17
|
+
- !ruby/object:Gem::Version
|
18
|
+
version: '8.1'
|
19
|
+
type: :runtime
|
20
|
+
prerelease: false
|
21
|
+
version_requirements: !ruby/object:Gem::Requirement
|
22
|
+
requirements:
|
23
|
+
- - "~>"
|
24
|
+
- !ruby/object:Gem::Version
|
25
|
+
version: '8.1'
|
26
|
+
description: ObsceneGpt is a Ruby gem that integrates with OpenAI's API to detect
|
27
|
+
whether given text contains obscene, inappropriate, or NSFW content. It provides
|
28
|
+
a simple interface for content moderation using AI.
|
29
|
+
email:
|
30
|
+
- daniel@perez.sh
|
31
|
+
executables: []
|
32
|
+
extensions: []
|
33
|
+
extra_rdoc_files: []
|
34
|
+
files:
|
35
|
+
- ".rspec"
|
36
|
+
- ".rubocop.yml"
|
37
|
+
- CHANGELOG.md
|
38
|
+
- LICENSE.txt
|
39
|
+
- README.md
|
40
|
+
- Rakefile
|
41
|
+
- examples/usage.rb
|
42
|
+
- lib/obscene_gpt.rb
|
43
|
+
- lib/obscene_gpt/active_model.rb
|
44
|
+
- lib/obscene_gpt/configuration.rb
|
45
|
+
- lib/obscene_gpt/detector.rb
|
46
|
+
- lib/obscene_gpt/prompts.rb
|
47
|
+
- lib/obscene_gpt/test_detector.rb
|
48
|
+
- lib/obscene_gpt/version.rb
|
49
|
+
- sig/obscene_gpt.rbs
|
50
|
+
homepage: https://github.com/danhper/obscene-gpt
|
51
|
+
licenses:
|
52
|
+
- MIT
|
53
|
+
metadata:
|
54
|
+
allowed_push_host: https://rubygems.org
|
55
|
+
homepage_uri: https://github.com/danhper/obscene-gpt
|
56
|
+
changelog_uri: https://github.com/danhper/obscene-gpt/blob/main/CHANGELOG.md
|
57
|
+
rubygems_mfa_required: 'true'
|
58
|
+
rdoc_options: []
|
59
|
+
require_paths:
|
60
|
+
- lib
|
61
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
62
|
+
requirements:
|
63
|
+
- - ">="
|
64
|
+
- !ruby/object:Gem::Version
|
65
|
+
version: 3.1.0
|
66
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
67
|
+
requirements:
|
68
|
+
- - ">="
|
69
|
+
- !ruby/object:Gem::Version
|
70
|
+
version: '0'
|
71
|
+
requirements: []
|
72
|
+
rubygems_version: 3.6.2
|
73
|
+
specification_version: 4
|
74
|
+
summary: A Ruby gem that uses OpenAI API to detect obscene content in text
|
75
|
+
test_files: []
|