obscene_gpt 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 57b6cd7a7afe7daa9400f7a24e1609c1195f96bb74d66c16bb28ef20d18ee1f0
4
+ data.tar.gz: d3d01454d54408acb0ebe8e97e020afb47dd49d1da39c2c5921314bb38c224c6
5
+ SHA512:
6
+ metadata.gz: a76f7ba702c0bdc886f0ee805b26f72278bad23ca4f88a2e57387b342d0e213d2d93c30fce52800b0425a76b1f9ac7aa99078b6ba51d1236dc29fb18b0d84e6e
7
+ data.tar.gz: '08145d5400a4b29d120824e51b286be6a6d4403ef5f8ecf18d260dd68f936b01486408bc041119aa22f39f5f05f803860191fc96a0f120e13499bfcc47384ac4'
data/.rspec ADDED
@@ -0,0 +1,3 @@
1
+ --format documentation
2
+ --color
3
+ --require spec_helper
data/.rubocop.yml ADDED
@@ -0,0 +1,31 @@
1
+ AllCops:
2
+ TargetRubyVersion: 3.1
3
+ NewCops: enable
4
+ SuggestExtensions: false
5
+
6
+ Style/StringLiterals:
7
+ EnforcedStyle: double_quotes
8
+
9
+ Style/StringLiteralsInInterpolation:
10
+ EnforcedStyle: double_quotes
11
+
12
+ Style/TrailingCommaInHashLiteral:
13
+ EnforcedStyleForMultiline: comma
14
+
15
+ Style/MutableConstant:
16
+ Enabled: false
17
+
18
+ Style/TrailingCommaInArrayLiteral:
19
+ EnforcedStyleForMultiline: comma
20
+
21
+ Style/TrailingCommaInArguments:
22
+ EnforcedStyleForMultiline: comma
23
+
24
+ Style/Documentation:
25
+ Enabled: false
26
+
27
+ Style/FrozenStringLiteralComment:
28
+ Enabled: false
29
+
30
+ Metrics/BlockLength:
31
+ Enabled: false
data/CHANGELOG.md ADDED
@@ -0,0 +1,5 @@
1
+ ## [Unreleased]
2
+
3
+ ## [0.1.0] - 2025-06-30
4
+
5
+ - Initial release
data/LICENSE.txt ADDED
@@ -0,0 +1,21 @@
1
+ The MIT License (MIT)
2
+
3
+ Copyright (c) 2025 Daniel Perez
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in
13
+ all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,412 @@
1
+ # ObsceneGpt
2
+
3
+ A Ruby gem that integrates with OpenAI's API to detect whether given text contains obscene, inappropriate, or NSFW content. It provides a simple interface for content moderation using AI.
4
+
5
+ ## Installation
6
+
7
+ Add this line to your application's Gemfile:
8
+
9
+ ```ruby
10
+ gem 'obscene_gpt'
11
+ ```
12
+
13
+ And then execute:
14
+
15
+ ```bash
16
+ bundle install
17
+ ```
18
+
19
+ Or install it yourself as:
20
+
21
+ ```bash
22
+ gem install obscene_gpt
23
+ ```
24
+
25
+ ## Setup
26
+
27
+ You'll need an OpenAI API key to use this gem. You can either:
28
+
29
+ 1. Set it as an environment variable:
30
+
31
+ ```bash
32
+ export OPENAI_API_KEY="your-openai-api-key-here"
33
+ ```
34
+
35
+ 2. Configure it globally in your application (recommended):
36
+
37
+ ```ruby
38
+ ObsceneGpt.configure do |config|
39
+ config.api_key = "your-openai-api-key-here"
40
+ config.model = "gpt-4.1-nano"
41
+ end
42
+ ```
43
+
44
+ 3. Pass it directly when instantiating the detector
45
+
46
+ ```ruby
47
+ detector = ObsceneGpt::Detector.new(api_key: "your-openai-api-key-here")
48
+ ```
49
+
50
+ ## Usage
51
+
52
+ See the `examples/usage.rb` file for usage examples.
53
+
54
+ ### Basic Usage
55
+
56
+ ```ruby
57
+ require 'obscene_gpt'
58
+
59
+ # Configure once in your app initialization
60
+ ObsceneGpt.configure do |config|
61
+ config.api_key = "your-openai-api-key-here"
62
+ config.model = "gpt-4.1-nano"
63
+ end
64
+
65
+ detector = ObsceneGpt::Detector.new
66
+
67
+ result = detector.detect("Hello, how are you today?")
68
+ puts result
69
+ # => {"obscene" => false, "confidence" => 0.95, "reasoning" => "The text is a polite greeting with no inappropriate content.", "categories" => []}
70
+
71
+ result = detector.detect("Some offensive text with BAAD words")
72
+ puts result
73
+ # => {"obscene" => true, "confidence" => 0.85, "reasoning" => "The text contains vulgar language with the word 'BAAD', which is likely intended as a vulgar or inappropriate term.", "categories" => ["profanity"]}
74
+ ```
75
+
76
+ ### ActiveModel Validator
77
+
78
+ When ActiveModel is available, you can use the built-in validator to automatically check for obscene content in your models:
79
+
80
+ ```ruby
81
+ class Post < ActiveRecord::Base
82
+ validates :content, obscene_content: true
83
+ validates :title, obscene_content: { message: "Title contains inappropriate language" }
84
+ validates :description, obscene_content: { threshold: 0.9 }
85
+ validates :comment, obscene_content: {
86
+ threshold: 0.8,
87
+ message: "Comment violates community guidelines"
88
+ }
89
+ end
90
+
91
+ # The validator automatically caches results to avoid duplicate API calls
92
+ post = Post.new(content: "Some potentially inappropriate content")
93
+ if post.valid?
94
+ puts "Post is valid"
95
+ else
96
+ puts "Validation errors: #{post.errors.full_messages}"
97
+ end
98
+ ```
99
+
100
+ **Important:** The validator uses Rails caching to ensure only one API call is made per unique text content. Results are cached for 1 hour to avoid repeated API calls for the same content.
101
+
102
+ ## Important considerations
103
+
104
+ ### Cost
105
+
106
+ The cost of using this gem is based on the number of API calls made.
107
+ A very short input text will have roughly 170 tokens, and each 200 characters adds roughly another 50 tokens.
108
+ The simple schema has 17 output tokens and the full schema has ~50 (depending on the length of the reasoning and the attributes).
109
+ Using the simple schema and with an average of 200 chars per request, the cost (with the gpt-4.1-nano model) is roughly $1 per 35,000 requests.
110
+
111
+ ### Rate limits
112
+
113
+ The OpenAI API has rate limits that depends on the model you are using.
114
+ The gpt-4.1-nano model has a rate limit of 500 requests per minute with a normal paid subscription.
115
+
116
+ ### Latency
117
+
118
+ Calling an API will obviously add some latency to your application.
119
+ The latency is dependent on the model you are using and the length of the text you are analyzing.
120
+ We do not recommend using this gem in latency-sensitive application.
121
+
122
+ ## API Reference
123
+
124
+ ### Configuration
125
+
126
+ #### ObsceneGpt.configure(&block)
127
+
128
+ Configure the gem globally.
129
+
130
+ ```ruby
131
+ ObsceneGpt.configure do |config|
132
+ config.api_key = "your-api-key"
133
+ config.model = "gpt-4.1-nano"
134
+ config.schema = ObsceneGpt::Prompts::SIMPLE_SCHEMA
135
+ config.prompt = ObsceneGpt::Prompts::SYSTEM_PROMPT
136
+ end
137
+ ```
138
+
139
+ #### ObsceneGpt.configuration
140
+
141
+ Get the current configuration object.
142
+
143
+ ### ObsceneGpt::Detector
144
+
145
+ #### ObsceneGpt::Detector.new(api_key: nil, model: nil)
146
+
147
+ Creates a new detector instance.
148
+
149
+ #### ObsceneGpt::Detector#detect(text)
150
+
151
+ Detects whether the given text contains obscene content.
152
+
153
+ **Parameters:**
154
+
155
+ - `text` (String): Text to analyze.
156
+
157
+ **Returns:** Hash with detection results. See `Response Format` for more details.
158
+
159
+ **Raises:**
160
+
161
+ - `ObsceneGpt::Error`: If there's an OpenAI API error
162
+
163
+ #### ObsceneGpt::Detector#detect_many(texts)
164
+
165
+ Detects whether the given texts contain obscene content.
166
+
167
+ **Parameters:**
168
+
169
+ - `texts` (Array<String>): Texts to analyze.
170
+
171
+ **Returns:** Array of hashes with detection results. See `Response Format` for more details.
172
+
173
+ **Raises:**
174
+
175
+ - `ObsceneGpt::Error`: If there's an OpenAI API error
176
+
177
+ ## Response Format
178
+
179
+ The detection methods return a hash (or array of hashes) with the following structure:
180
+
181
+ ```ruby
182
+ {
183
+ obscene: true, # Boolean: whether content is inappropriate
184
+ confidence: 0.85, # Float: confidence score (0.0-1.0)
185
+ reasoning: "Contains explicit language and profanity",
186
+ categories: ["profanity", "explicit"] # Array of detected categories (["sexual", "profanity", "hate", "violent", "other"])
187
+ }
188
+ ```
189
+
190
+ ## Configuration Options
191
+
192
+ ### Default options
193
+
194
+ The default configuration is:
195
+
196
+ ```ruby
197
+ config.api_key = ENV["OPENAI_API_KEY"]
198
+ config.model = "gpt-4.1-nano"
199
+ config.schema = ObsceneGpt::Prompts::SIMPLE_SCHEMA
200
+ config.prompt = ObsceneGpt::Prompts::SYSTEM_PROMPT
201
+ config.test_mode = false
202
+ config.test_detector_class = ObsceneGpt::TestDetector
203
+ ```
204
+
205
+ ### Test Mode
206
+
207
+ To avoid making API calls during testing, you can enable test mode:
208
+
209
+ ```ruby
210
+ ObsceneGpt.configure do |config|
211
+ config.test_mode = true
212
+ end
213
+ ```
214
+
215
+ When test mode is enabled, the detector will return mock responses based on simple pattern matching instead of making actual API calls. This is useful for:
216
+
217
+ - Running tests without API costs
218
+ - Faster test execution
219
+ - Avoiding rate limits during development
220
+
221
+ **Note:** Test mode uses basic pattern matching and is not as accurate as the actual AI model. It's intended for testing purposes only.
222
+
223
+ #### Custom Test Detectors
224
+
225
+ You can also configure a custom test detector class for more sophisticated test behavior:
226
+
227
+ ```ruby
228
+ class MyCustomTestDetector
229
+ attr_reader :schema
230
+
231
+ def initialize(schema: nil)
232
+ @schema = schema || ObsceneGpt::Prompts::SIMPLE_SCHEMA
233
+ end
234
+
235
+ def detect_many(texts)
236
+ texts.map do |text|
237
+ {
238
+ obscene: text.include?("bad_word"),
239
+ confidence: 0.9
240
+ }
241
+ end
242
+ end
243
+
244
+ def detect(text)
245
+ detect_many([text])[0]
246
+ end
247
+ end
248
+
249
+ ObsceneGpt.configure do |config|
250
+ config.test_mode = true
251
+ config.test_detector_class = MyCustomTestDetector
252
+ end
253
+ ```
254
+
255
+ Custom test detectors must implement:
256
+
257
+ - `#initialize(schema: nil)` - Accepts an optional schema parameter
258
+ - `#detect_many(texts)` - Returns an array of result hashes
259
+
260
+ See `examples/custom_test_detector.rb` for more examples.
261
+
262
+ ### Model
263
+
264
+ We recommend using the `gpt-4.1-nano` model for cost efficiency.
265
+ Given the simplicity of the task, it's typically not necessary to use a more expensive model.
266
+
267
+ See [OpenAI's documentation](https://platform.openai.com/docs/pricing) for more information.
268
+
269
+ ### Prompt
270
+
271
+ The system prompt can be found in `lib/obscene_gpt/prompts.rb`.
272
+ This is a basic prompt that can be used to detect obscene content.
273
+ You can use a custom prompt if you need to by setting the `prompt` option in the configuration.
274
+
275
+ ### Schema
276
+
277
+ This library uses a JSON schema to enforce the response from the OpenAI API.
278
+ There are two schemas available:
279
+
280
+ - `ObsceneGpt::Prompts::SIMPLE_SCHEMA`: A simple schema that only includes the `obscene` and `confidence` fields.
281
+ - `ObsceneGpt::Prompts::FULL_SCHEMA`: A full schema that includes the `obscene`, `confidence`, `reasoning`, and `categories` fields.
282
+
283
+ You can use a custom schema if you need to by setting the `schema` option in the configuration.
284
+
285
+ ### Configuration Precedence
286
+
287
+ 1. Explicit parameters passed to methods
288
+ 2. Global configuration
289
+ 3. Environment variables (for API key only)
290
+
291
+ ## ActiveModel Integration
292
+
293
+ The `ObsceneContentValidator` is available when ActiveModel is loaded.
294
+ `active_model` needs to be required before obscene_gpt.
295
+
296
+ ### Usage
297
+
298
+ ```ruby
299
+ class Post < ActiveRecord::Base
300
+ validates :content, :title, :description, obscene_content: true
301
+ end
302
+ ```
303
+
304
+ **Note**: Each instance of this validator will make a request to the OpenAI API.
305
+ Therefore, it is recommended to pass all the attributes you want to check to the validator at once as shown above.
306
+
307
+ ### Options
308
+
309
+ - `threshold` (Float): Custom confidence threshold (0.0-1.0) for determining when content is considered inappropriate. Default: Uses `ObsceneGpt.configuration.profanity_threshold`
310
+ - `message` (String): Custom error message to display when validation fails. Default: Uses AI reasoning if available, otherwise "contains inappropriate content"
311
+
312
+ ### Per-Attribute Options
313
+
314
+ You can also configure different options for different attributes in a single validation call:
315
+
316
+ ```ruby
317
+ class Post < ActiveRecord::Base
318
+ validates :title, :content, obscene_content: {
319
+ title: { threshold: 0.8, message: "Title is too inappropriate" },
320
+ content: { threshold: 0.7, message: "Content needs moderation" }
321
+ }
322
+ end
323
+ ```
324
+
325
+ ### Configuration Precedence
326
+
327
+ The validator uses the following precedence for options:
328
+
329
+ **Threshold:**
330
+
331
+ 1. Per-attribute option (e.g., `title: { threshold: 0.8 }`)
332
+ 2. Validator option (e.g., `threshold: 0.8`)
333
+ 3. Configuration default (`ObsceneGpt.configuration.profanity_threshold`)
334
+
335
+ **Message:**
336
+
337
+ 1. Per-attribute message (e.g., `title: { message: "..." }`)
338
+ 2. Global message (e.g., `message: "..."`)
339
+ 3. AI reasoning (if available, only when schema is `ObsceneGpt::Prompts::FULL_SCHEMA`)
340
+ 4. Default message ("contains inappropriate content")
341
+
342
+ ### Examples
343
+
344
+ **Basic validation:**
345
+
346
+ ```ruby
347
+ class Post < ActiveRecord::Base
348
+ validates :content, obscene_content: true
349
+ end
350
+ ```
351
+
352
+ **With custom message:**
353
+
354
+ ```ruby
355
+ class Post < ActiveRecord::Base
356
+ validates :title, obscene_content: { message: "Title contains inappropriate content" }
357
+ end
358
+ ```
359
+
360
+ **With custom threshold:**
361
+
362
+ ```ruby
363
+ class Post < ActiveRecord::Base
364
+ validates :description, obscene_content: { threshold: 0.9 }
365
+ end
366
+ ```
367
+
368
+ **With both custom threshold and message:**
369
+
370
+ ```ruby
371
+ class Post < ActiveRecord::Base
372
+ validates :comment, obscene_content: {
373
+ threshold: 0.8,
374
+ message: "Comment violates community guidelines"
375
+ }
376
+ end
377
+ ```
378
+
379
+ **Per-attribute configuration:**
380
+
381
+ ```ruby
382
+ class Post < ActiveRecord::Base
383
+ validates :title, :content, obscene_content: {
384
+ title: { threshold: 0.8, message: "Title is too inappropriate" },
385
+ content: { threshold: 0.7, message: "Content needs moderation" }
386
+ }
387
+ end
388
+ ```
389
+
390
+ **Mixed global and per-attribute options:**
391
+
392
+ ```ruby
393
+ class Post < ActiveRecord::Base
394
+ validates :title, :content, obscene_content: {
395
+ threshold: 0.8, # Global threshold
396
+ message: "Contains inappropriate content", # Global message
397
+ title: { threshold: 0.9 } # Override threshold for title only
398
+ }
399
+ end
400
+ ```
401
+
402
+ ## Development
403
+
404
+ After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
405
+
406
+ ## Contributing
407
+
408
+ Bug reports and pull requests are welcome on GitHub at https://github.com/danhper/obscene_gpt.
409
+
410
+ ## License
411
+
412
+ The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
data/Rakefile ADDED
@@ -0,0 +1,10 @@
1
+ require "bundler/gem_tasks"
2
+ require "rspec/core/rake_task"
3
+
4
+ RSpec::Core::RakeTask.new(:spec)
5
+
6
+ require "rubocop/rake_task"
7
+
8
+ RuboCop::RakeTask.new
9
+
10
+ task default: %i[spec rubocop]
data/examples/usage.rb ADDED
@@ -0,0 +1,42 @@
1
+ require "obscene_gpt"
2
+
3
+ # Configure the gem globally (do this once in your app initialization)
4
+ ObsceneGpt.configure do |config|
5
+ # config.api_key = "your-openai-api-key-here"
6
+ config.model = "gpt-4.1-nano"
7
+ config.schema = ObsceneGpt::Prompts::FULL_SCHEMA
8
+ end
9
+
10
+ detector = ObsceneGpt::Detector.new
11
+
12
+ texts_to_analyze = [
13
+ "Hello, how are you today?",
14
+ "This is a beautiful day!",
15
+ "I love programming in Ruby.",
16
+ "Some potentially inappropriate content here...",
17
+ "This text contains explicit language and should be flagged.",
18
+ ]
19
+
20
+ detector.detect_many(texts_to_analyze).each_with_index do |result, index|
21
+ puts "Text: #{texts_to_analyze[index]}"
22
+ puts "Obscene: #{result[:obscene]}"
23
+ puts "Confidence: #{result[:confidence]}"
24
+ puts "Reasoning: #{result[:reasoning]}"
25
+ puts "Categories: #{result[:categories]}"
26
+ puts "--------------------------------"
27
+ end
28
+
29
+ if defined?(ActiveRecord)
30
+ class Post < ActiveRecord::Base
31
+ validates :title, :content, obscene_content: {
32
+ title: { message: "Title contains inappropriate language" },
33
+ }
34
+ end
35
+
36
+ post = Post.new(title: "Some normal content", content: "Some very inappropriate content")
37
+ if post.valid?
38
+ puts "Post is valid"
39
+ else
40
+ puts "Validation errors: #{post.errors.full_messages}"
41
+ end
42
+ end
@@ -0,0 +1,45 @@
1
+ begin
2
+ require "active_model"
3
+ rescue LoadError
4
+ # do nothing
5
+ end
6
+
7
+ if defined?(ActiveModel)
8
+ class ObsceneContentValidator < ActiveModel::EachValidator
9
+ def validate(record)
10
+ to_validate = compute_attributes_to_validate(record)
11
+ return if to_validate.empty?
12
+
13
+ results = ObsceneGpt.detect_many(to_validate.values)
14
+ format_errors(record, to_validate, results)
15
+ end
16
+
17
+ private
18
+
19
+ def compute_attributes_to_validate(record)
20
+ attributes.map do |attribute|
21
+ value = record.read_attribute_for_validation(attribute)
22
+ next if value.nil? || value.blank?
23
+
24
+ [attribute, prepare_value_for_validation(value, record, attribute)]
25
+ end.compact.to_h
26
+ end
27
+
28
+ def format_errors(record, to_validate, results)
29
+ results.each_with_index do |result, index|
30
+ attribute = to_validate.keys[index]
31
+ threshold = option_for(:threshold, attribute, ObsceneGpt.configuration.profanity_threshold)
32
+
33
+ if result[:obscene] && result[:confidence] >= threshold
34
+ message = option_for(:message, attribute, result[:reasoning] || "contains inappropriate content")
35
+ record.errors.add(attribute, :obscene_content, message: message)
36
+ end
37
+ end
38
+ end
39
+
40
+ def option_for(key, attribute, default = nil)
41
+ (options[attribute] && options[attribute][key]) ||
42
+ options[key] || default
43
+ end
44
+ end
45
+ end
@@ -0,0 +1,25 @@
1
+ module ObsceneGpt
2
+ class << self
3
+ def configure
4
+ yield(configuration)
5
+ end
6
+
7
+ def configuration
8
+ @configuration ||= Configuration.new
9
+ end
10
+ end
11
+
12
+ class Configuration
13
+ attr_accessor :api_key, :model, :schema, :prompt, :profanity_threshold, :test_mode, :test_detector_class
14
+
15
+ def initialize
16
+ @api_key = ENV.fetch("OPENAI_API_KEY", nil)
17
+ @model = "gpt-4.1-nano"
18
+ @prompt = Prompts::SYSTEM_PROMPT
19
+ @schema = Prompts::SIMPLE_SCHEMA
20
+ @profanity_threshold = 0.8
21
+ @test_mode = false
22
+ @test_detector_class = TestDetector
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,75 @@
1
+ require "json"
2
+ require "openai"
3
+
4
+ module ObsceneGpt
5
+ class Detector
6
+ attr_reader :client, :model, :schema, :prompt
7
+
8
+ def initialize(api_key: nil, model: nil, schema: nil, prompt: nil)
9
+ api_key ||= ObsceneGpt.configuration.api_key
10
+
11
+ @client = OpenAI::Client.new(access_token: api_key)
12
+ @model = model || ObsceneGpt.configuration.model
13
+ @schema = schema || ObsceneGpt.configuration.schema
14
+ @prompt = prompt || ObsceneGpt.configuration.prompt
15
+ end
16
+
17
+ # Detects whether the given texts contain obscene content
18
+ # @param texts [Array<String>] The texts to analyze
19
+ # @return [Array<Hash>] An array of hashes containing the detection result with keys:
20
+ # - :obscene [Boolean] Whether the text contains obscene content
21
+ # - :confidence [Float] Confidence score (0.0 to 1.0)
22
+ # - :reasoning [String] Explanation for the classification (only for full schema)
23
+ # - :categories [Array<String>] Categories of inappropriate content found (only for full schema)
24
+ def detect_many(texts)
25
+ if ObsceneGpt.configuration.test_mode
26
+ test_detector = ObsceneGpt.configuration.test_detector_class.new(schema: @schema)
27
+ return test_detector.detect_many(texts)
28
+ end
29
+
30
+ run_detect_many(texts)
31
+ end
32
+
33
+ # Detects whether the given text contains obscene content
34
+ # See #detect_many for more details
35
+ def detect(text)
36
+ detect_many([text])[0]
37
+ end
38
+
39
+ private
40
+
41
+ def run_detect_many(texts)
42
+ response = @client.responses.create(parameters: make_query(texts))
43
+
44
+ JSON.parse(response.dig("output", 0, "content", 0, "text"))["results"].map { |r| r.transform_keys(&:to_sym) }
45
+ rescue OpenAI::Error, Faraday::Error => e
46
+ body = e.respond_to?(:response) ? e.response[:body] : ""
47
+ raise ObsceneGpt::Error, "OpenAI API error: #{e.message}\n#{body}"
48
+ end
49
+
50
+ def make_query(texts)
51
+ text_format = { name: "content-moderation", type: "json_schema", schema: make_schema(texts.length), strict: true }
52
+ {
53
+ model: @model,
54
+ text: { format: text_format },
55
+ input: [{
56
+ role: "user",
57
+ content: [{ type: "input_text", text: @prompt },
58
+ { type: "input_text", text: JSON.dump(texts) }],
59
+ }],
60
+ }
61
+ end
62
+
63
+ def make_schema(texts_count)
64
+ array_schema = { type: "array", items: @schema, minItems: texts_count, maxItems: texts_count }
65
+ {
66
+ type: "object",
67
+ properties: {
68
+ results: array_schema,
69
+ },
70
+ required: %w[results],
71
+ additionalProperties: false,
72
+ }
73
+ end
74
+ end
75
+ end
@@ -0,0 +1,52 @@
1
+ module ObsceneGpt
2
+ module Prompts
3
+ SYSTEM_PROMPT = <<~PROMPT.freeze
4
+ You are a content moderation AI that analyzes texts for obscene, inappropriate, or NSFW content.
5
+
6
+ Your task is to determine if the given text contains:
7
+ - Explicit sexual content
8
+ - Profanity or vulgar language
9
+ - Hate speech or discriminatory language
10
+ - Violent or graphic content
11
+ - Other inappropriate material
12
+
13
+ You will be given a JSON array of texts.
14
+ You will need to analyze each text and determine if it contains any of the above content.
15
+ PROMPT
16
+
17
+ SIMPLE_SCHEMA = {
18
+ type: "object",
19
+ properties: {
20
+ obscene: {
21
+ type: "boolean",
22
+ description: "Whether the text contains obscene content",
23
+ },
24
+ confidence: {
25
+ type: "number",
26
+ minimum: 0,
27
+ maximum: 1,
28
+ description: "A confidence score between 0 and 1",
29
+ },
30
+ },
31
+ required: %w[obscene confidence],
32
+ additionalProperties: false,
33
+ }.freeze
34
+
35
+ FULL_SCHEMA = {
36
+ type: "object",
37
+ properties: SIMPLE_SCHEMA[:properties].merge(
38
+ reasoning: {
39
+ type: "string",
40
+ description: "A reasoning for the classification",
41
+ },
42
+ categories: {
43
+ type: "array",
44
+ items: { type: "string", enum: %w[sexual profanity hate violent other] },
45
+ description: "A list of categories that the text belongs to",
46
+ },
47
+ ),
48
+ required: %w[obscene confidence reasoning categories],
49
+ additionalProperties: false,
50
+ }.freeze
51
+ end
52
+ end
@@ -0,0 +1,56 @@
1
+ module ObsceneGpt
2
+ class TestDetector
3
+ attr_reader :schema
4
+
5
+ def initialize(schema: nil)
6
+ @schema = schema || ObsceneGpt::Prompts::SIMPLE_SCHEMA
7
+ end
8
+
9
+ # Detects whether the given texts contain obscene content using test mode
10
+ # @param texts [Array<String>] The texts to analyze
11
+ # @return [Array<Hash>] An array of hashes containing the detection result
12
+ def detect_many(texts) # rubocop:disable Metrics/MethodLength
13
+ texts.map do |text|
14
+ # Simple heuristic for test mode: detect common profanity patterns
15
+ is_obscene = detect_obscene_patterns(text)
16
+ confidence = is_obscene ? 0.85 : 0.95
17
+
18
+ result = {
19
+ obscene: is_obscene,
20
+ confidence: confidence,
21
+ }
22
+
23
+ # Add additional fields if using full schema
24
+ if @schema == Prompts::FULL_SCHEMA
25
+ result[:reasoning] = is_obscene ? "Contains inappropriate content" : "Clean text"
26
+ result[:categories] = is_obscene ? ["profanity"] : []
27
+ end
28
+
29
+ result
30
+ end
31
+ end
32
+
33
+ # Detects whether the given text contains obscene content
34
+ # @param text [String] The text to analyze
35
+ # @return [Hash] Detection result
36
+ def detect(text)
37
+ detect_many([text])[0]
38
+ end
39
+
40
+ private
41
+
42
+ def detect_obscene_patterns(text)
43
+ # Simple pattern matching for test mode
44
+ # This is just for testing - not meant to be comprehensive
45
+ profanity_patterns = [
46
+ /\b(fuck|fucked|fucking|shit|bitch|ass|damn|hell)\b/i,
47
+ /\b(sex|porn|nude|naked)\b/i,
48
+ /\b(kill|murder|death|blood)\b/i,
49
+ /\b(hate|racist|sexist)\b/i,
50
+ /\b(gore)\b/i,
51
+ ]
52
+
53
+ profanity_patterns.any? { |pattern| text.match?(pattern) }
54
+ end
55
+ end
56
+ end
@@ -0,0 +1,3 @@
1
+ module ObsceneGpt
2
+ VERSION = "0.1.0"
3
+ end
@@ -0,0 +1,20 @@
1
+ require_relative "obscene_gpt/version"
2
+ require_relative "obscene_gpt/prompts"
3
+ require_relative "obscene_gpt/configuration"
4
+ require_relative "obscene_gpt/test_detector"
5
+ require_relative "obscene_gpt/detector"
6
+ require_relative "obscene_gpt/active_model"
7
+
8
+ module ObsceneGpt
9
+ class Error < StandardError; end
10
+
11
+ class << self
12
+ def detect_many(texts)
13
+ ObsceneGpt::Detector.new.detect_many(texts)
14
+ end
15
+
16
+ def detect(text)
17
+ ObsceneGpt::Detector.new.detect(text)
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,4 @@
1
+ module ObsceneGpt
2
+ VERSION: String
3
+ # See the writing guide of rbs: https://github.com/ruby/rbs#guides
4
+ end
metadata ADDED
@@ -0,0 +1,75 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: obscene_gpt
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - Daniel Perez
8
+ bindir: exe
9
+ cert_chain: []
10
+ date: 2025-06-30 00:00:00.000000000 Z
11
+ dependencies:
12
+ - !ruby/object:Gem::Dependency
13
+ name: ruby-openai
14
+ requirement: !ruby/object:Gem::Requirement
15
+ requirements:
16
+ - - "~>"
17
+ - !ruby/object:Gem::Version
18
+ version: '8.1'
19
+ type: :runtime
20
+ prerelease: false
21
+ version_requirements: !ruby/object:Gem::Requirement
22
+ requirements:
23
+ - - "~>"
24
+ - !ruby/object:Gem::Version
25
+ version: '8.1'
26
+ description: ObsceneGpt is a Ruby gem that integrates with OpenAI's API to detect
27
+ whether given text contains obscene, inappropriate, or NSFW content. It provides
28
+ a simple interface for content moderation using AI.
29
+ email:
30
+ - daniel@perez.sh
31
+ executables: []
32
+ extensions: []
33
+ extra_rdoc_files: []
34
+ files:
35
+ - ".rspec"
36
+ - ".rubocop.yml"
37
+ - CHANGELOG.md
38
+ - LICENSE.txt
39
+ - README.md
40
+ - Rakefile
41
+ - examples/usage.rb
42
+ - lib/obscene_gpt.rb
43
+ - lib/obscene_gpt/active_model.rb
44
+ - lib/obscene_gpt/configuration.rb
45
+ - lib/obscene_gpt/detector.rb
46
+ - lib/obscene_gpt/prompts.rb
47
+ - lib/obscene_gpt/test_detector.rb
48
+ - lib/obscene_gpt/version.rb
49
+ - sig/obscene_gpt.rbs
50
+ homepage: https://github.com/danhper/obscene-gpt
51
+ licenses:
52
+ - MIT
53
+ metadata:
54
+ allowed_push_host: https://rubygems.org
55
+ homepage_uri: https://github.com/danhper/obscene-gpt
56
+ changelog_uri: https://github.com/danhper/obscene-gpt/blob/main/CHANGELOG.md
57
+ rubygems_mfa_required: 'true'
58
+ rdoc_options: []
59
+ require_paths:
60
+ - lib
61
+ required_ruby_version: !ruby/object:Gem::Requirement
62
+ requirements:
63
+ - - ">="
64
+ - !ruby/object:Gem::Version
65
+ version: 3.1.0
66
+ required_rubygems_version: !ruby/object:Gem::Requirement
67
+ requirements:
68
+ - - ">="
69
+ - !ruby/object:Gem::Version
70
+ version: '0'
71
+ requirements: []
72
+ rubygems_version: 3.6.2
73
+ specification_version: 4
74
+ summary: A Ruby gem that uses OpenAI API to detect obscene content in text
75
+ test_files: []