sentinel_rb 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: a407a7c7bc6fe36c2602e3e6b459a740c08b34904aa1543d26495261433c42ae
+  data.tar.gz: 9e578433b8f22abec3990f212f4ac056d3cc8a431b95ce1af12402b3e0a948e7
+SHA512:
+  metadata.gz: a3e78be6292a5da3c33b4763f701cef652067a0e1c27a003e78ddd05feae4d96830f7fb2238ffbef93b09595d56edd6b02115503160c28111b25b74e9fe22cf5
+  data.tar.gz: 36eed9031ee2aa2d0f398a419cb6468fa9bbd61eaa20914c8f7d0ddb48edf5a607833bfc695c92e3716512ac52b4ba29d24be7e3a494419cace1820fb28831c2

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,10 @@
+inherit_from: .rubocop_todo.yml
+
+AllCops:
+  TargetRubyVersion: 3.1
+
+Style/StringLiterals:
+  EnforcedStyle: double_quotes
+
+Style/StringLiteralsInInterpolation:
+  EnforcedStyle: double_quotes

data/.rubocop_todo.yml

@@ -0,0 +1,72 @@
+# This configuration was generated by
+# `rubocop --auto-gen-config --exclude-limit 999999`
+# on 2025-06-06 16:31:26 UTC using RuboCop version 1.76.0.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+# Offense count: 13
+# Configuration parameters: AllowedMethods, AllowedPatterns, CountRepeatedAttributes.
+Metrics/AbcSize:
+  Max: 87
+
+# Offense count: 5
+# Configuration parameters: CountComments, CountAsOne, AllowedMethods, AllowedPatterns.
+# AllowedMethods: refine
+Metrics/BlockLength:
+  Max: 98
+
+# Offense count: 7
+# Configuration parameters: CountComments, CountAsOne.
+Metrics/ClassLength:
+  Max: 229
+
+# Offense count: 6
+# Configuration parameters: AllowedMethods, AllowedPatterns.
+Metrics/CyclomaticComplexity:
+  Max: 23
+
+# Offense count: 36
+# Configuration parameters: CountComments, CountAsOne, AllowedMethods, AllowedPatterns.
+Metrics/MethodLength:
+  Max: 84
+
+# Offense count: 5
+# Configuration parameters: AllowedMethods, AllowedPatterns.
+Metrics/PerceivedComplexity:
+  Max: 23
+
+# Offense count: 1
+# Configuration parameters: NamePrefix, ForbiddenPrefixes, AllowedMethods, MethodDefinitionMacros, UseSorbetSigs.
+# NamePrefix: is_, has_, have_, does_
+# ForbiddenPrefixes: is_, has_, have_, does_
+# AllowedMethods: is_a?
+# MethodDefinitionMacros: define_method, define_singleton_method
+Naming/PredicatePrefix:
+  Exclude:
+    - 'spec/**/*'
+    - 'lib/sentinel_rb/analyzers/few_shot_bias.rb'
+
+# Offense count: 1
+# Configuration parameters: AllowedConstants.
+Style/Documentation:
+  Exclude:
+    - 'spec/**/*'
+    - 'test/**/*'
+    - 'lib/sentinel_rb.rb'
+
+# Offense count: 6
+# This cop supports safe autocorrection (--autocorrect).
+Style/IfUnlessModifier:
+  Exclude:
+    - 'lib/sentinel_rb/analyzers/irrelevant_info.rb'
+    - 'lib/sentinel_rb/cli.rb'
+    - 'lib/sentinel_rb/client/mock.rb'
+
+# Offense count: 10
+# This cop supports safe autocorrection (--autocorrect).
+# Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, AllowedPatterns, SplitStrings.
+# URISchemes: http, https
+Layout/LineLength:
+  Max: 304

data/.sentinel-test.yml

@@ -0,0 +1,20 @@
+# Test configuration using mock provider
+provider: mock
+model: mock-model
+relevance_threshold: 0.55
+
+# Mock-specific settings
+mock_scores:
+  default: 0.7
+
+# Standard settings
+dangerous_tools:
+  - delete_file
+  - transfer_funds
+
+skip_patterns:
+  - "**/.git/**"
+  - "**/node_modules/**"
+
+output_format: table
+log_level: warn

data/.sentinel.yml

@@ -0,0 +1,29 @@
+# SentinelRb Configuration File
+# For complete configuration options, see docs/usage.md
+
+# LLM Provider Configuration
+provider: openai
+model: gpt-4o-mini
+api_key_env: OPENAI_API_KEY
+
+# Analysis Thresholds
+relevance_threshold: 0.55     # Lower = more strict (0.0 - 1.0)
+divergence_threshold: 0.25    # For few-shot bias detection
+
+# Security Settings
+dangerous_tools:
+  - delete_file
+  - transfer_funds
+  - system_shutdown
+  - exec_command
+
+# File Processing
+skip_patterns:
+  - "**/.git/**"
+  - "**/node_modules/**"
+  - "**/tmp/**"
+  - "**/.DS_Store"
+
+# Output Settings
+output_format: table          # table, json, detailed
+log_level: warn              # debug, info, warn, error

data/.sentinel.yml.example

@@ -0,0 +1,74 @@
+# SentinelRb Configuration Example
+# Copy this file to .sentinel.yml and customize as needed
+
+# LLM Provider Configuration
+provider: openai              # Provider: 'openai' or 'mock'
+model: gpt-4o-mini            # OpenAI model to use
+api_key_env: OPENAI_API_KEY   # Environment variable containing API key
+
+# Analysis Thresholds
+relevance_threshold: 0.55     # Threshold for A1 irrelevant information detection (0.0-1.0)
+divergence_threshold: 0.25    # Threshold for A3 few-shot bias detection (0.0-1.0)
+fact_check_threshold: 0.7     # Threshold for A2 fact-checking confidence (0.0-1.0)
+
+# A2: Misinformation Detection Keywords (in addition to built-in patterns)
+misinformation_keywords:
+  - "conspiracy"
+  - "hoax"
+  - "debunked"
+  - "cover-up"
+
+# A4: Base Model Usage Detection Keywords (in addition to built-in patterns)
+instruction_keywords:
+  - "jailbreak"
+  - "bypass"
+  - "ignore"
+  - "override"
+
+bypass_patterns:
+  - "DAN"
+  - "developer mode"
+  - "god mode"
+  - "admin mode"
+
+# A5: Dangerous Tools Detection Keywords (in addition to built-in patterns)
+dangerous_keywords:
+  - "weapon"
+  - "explosive"
+  - "hack"
+  - "malware"
+
+weapon_keywords:
+  - "bomb"
+  - "grenade"
+  - "firearm"
+  - "poison"
+
+cyber_keywords:
+  - "exploit"
+  - "ddos"
+  - "trojan"
+  - "ransomware"
+
+# File Processing
+skip_patterns:                # Glob patterns to skip during analysis
+  - "**/.git/**"
+  - "**/node_modules/**"
+  - "**/tmp/**"
+  - "**/.DS_Store"
+  - "**/vendor/**"
+  - "**/*.log"
+
+# Output Options
+log_level: info               # Logging level: debug, info, warn, error
+color_output: true            # Enable colored output in terminal
+max_findings_per_file: 50     # Maximum findings to report per file
+
+# Output Configuration
+output_format: table          # Default output format: 'table', 'json', 'detailed'
+log_level: warn              # Log level: 'debug', 'info', 'warn', 'error'
+
+# Mock Client Configuration (for testing without API keys)
+mock_scores:                  # Override scores for specific content patterns
+  # Example: Set specific score for test content
+  # "test prompt": 0.3

data/AGENTS.md

@@ -0,0 +1,87 @@
+# SentinelRb Agent Guide
+
+## Overview
+
+SentinelRb is an LLM-driven prompt inspector designed to automatically detect common antipatterns in prompts that are difficult to catch with static analysis. This tool helps developers maintain high-quality prompts by identifying issues that could negatively impact LLM performance.
+
+## Core Purpose
+
+SentinelRb analyzes prompt files to detect five key antipatterns using OpenAI, RAG, and metadata analysis:
+
+- **A1: Irrelevant Information** - Detects prompts containing noisy or off-topic content
+- **A2: Misinformation & Logical Contradictions** - Verifies factual accuracy using RAG knowledge base
+- **A3: Few-shot Bias Order** - Identifies ordering bias in few-shot examples
+- **A4: Base Model Usage** - Flags usage of base models instead of instruction-tuned models
+- **A5: Dangerous Automatic Tool Execution** - Prevents auto-execution of dangerous tools
+
+## Quick Reference
+
+### Installation & Basic Usage
+```bash
+gem install sentinel_rb
+sentinel --glob "prompts/**/*.{md,json}" --config .sentinel.yml
+```
+
+### Key Configuration
+```yaml
+provider: openai
+model: gpt-4o-mini
+relevance_threshold: 0.55
+divergence_threshold: 0.25
+```
+
+## Detailed Documentation
+
+For comprehensive information about SentinelRb, please refer to the following documentation:
+
+### System Architecture
+- **File**: `/docs/architecture.md`
+- **Content**: Detailed technical architecture, component descriptions, analyzer implementations, and extension points
+
+### Usage Guide
+- **File**: `/docs/usage.md`
+- **Content**: Complete usage instructions, configuration options, CLI commands, integration examples, and troubleshooting
+
+### Development Guide
+- **File**: `/docs/development.md`
+- **Content**: Development setup, contributing guidelines, adding new analyzers, testing procedures, and release process
+
+## Key Features for Agents
+
+### Automated Analysis
+- File-based prompt analysis
+- Configurable thresholds and detection rules
+- Multiple output formats (table, JSON, detailed)
+
+### CI/CD Integration
+- GitHub Actions workflow templates
+- Pre-commit hooks
+- Automated quality gates
+
+### Extensibility
+- Pluggable analyzer architecture
+- Custom LLM provider support
+- Configurable rule sets
+
+### Security Focus
+- Dangerous tool detection
+- API key management
+- No persistent data storage
+
+## Agent-Specific Considerations
+
+When working with SentinelRb in an agent context:
+
+1. **Configuration Management**: Use environment-specific configurations for different deployment stages
+2. **Rate Limiting**: Configure appropriate API rate limits to avoid quota exhaustion
+3. **Error Handling**: Implement robust error handling for API failures and network issues
+4. **Caching**: Enable response caching for improved performance with repeated analyses
+5. **Monitoring**: Set up monitoring for analysis results and system health
+
+## Getting Help
+
+- **Documentation**: Check `/docs/` directory for detailed guides
+- **Issues**: Report bugs and feature requests via GitHub issues
+- **Examples**: See usage examples in the main README.md file
+
+For technical implementation details, architectural decisions, and development procedures, always refer to the comprehensive documentation in the `/docs` directory.

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,132 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, caste, color, religion, or sexual
+identity and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the overall
+  community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or advances of
+  any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email address,
+  without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official email address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement at
+[INSERT CONTACT METHOD].
+All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series of
+actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or permanent
+ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior, harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within the
+community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.1, available at
+[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].
+
+Community Impact Guidelines were inspired by
+[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
+
+For answers to common questions about this code of conduct, see the FAQ at
+[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at
+[https://www.contributor-covenant.org/translations][translations].
+
+[homepage]: https://www.contributor-covenant.org
+[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
+[Mozilla CoC]: https://github.com/mozilla/diversity
+[FAQ]: https://www.contributor-covenant.org/faq
+[translations]: https://www.contributor-covenant.org/translations

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2025 Masumi Kawasaki
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,226 @@
+# SentinelRb
+
+SentinelRb is an LLM-driven prompt inspector designed to automatically detect common antipatterns in prompts before they reach production systems.
+
+## Overview
+
+SentinelRb analyzes prompt files to detect five key antipatterns using advanced pattern matching and LLM-based analysis:
+
+| ID | Target | Detection Logic |
+|----|--------|-----------------|
+| A1 | Irrelevant Information | Uses LLM to generate relevance scores; flags prompts with low focus/clarity |
+| A2 | Misinformation & Logical Contradictions | Detects false information patterns and conspiracy theories |
+| A3 | Few-shot Bias | Analyzes example patterns for demographic or representational bias |
+| A4 | Base Model Usage | Detects jailbreak attempts and instruction bypassing |
+| A5 | Dangerous Tools | Identifies requests for harmful content creation or dangerous activities |
+
+## Features
+
+- **Comprehensive Analysis**: Detects 5 major prompt antipatterns
+- **LLM Integration**: Works with OpenAI models for semantic analysis
+- **Mock Mode**: Test without API keys using built-in pattern detection
+- **Multiple Output Formats**: Table, JSON, and detailed reporting
+- **Configurable Thresholds**: Customize sensitivity for each analyzer
+- **CLI & Library**: Use as command-line tool or integrate into your Ruby applications
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'sentinel_rb'
+```
+
+And then execute:
+
+```
+$ bundle install
+```
+
+Or install it yourself as:
+
+```
+$ gem install sentinel_rb
+```
+
+## Requirements
+
+- Ruby >= 3.1.0
+- OpenAI API key (optional - mock mode available for testing)
+
+## Usage
+
+### CLI Usage
+
+Analyze prompt files using the command-line interface:
+
+```bash
+# Analyze all markdown files in prompts directory
+sentinel_rb analyze --glob "prompts/**/*.md"
+
+# Analyze specific files with custom output format
+sentinel_rb analyze --files prompt1.md prompt2.md --format json
+
+# Run specific analyzers only
+sentinel_rb analyze --files test.md --analyzers A1,A2,A5
+
+# Use detailed output format
+sentinel_rb analyze --files test.md --format detailed
+```
+
+### Library Usage
+
+Use SentinelRb programmatically in your Ruby applications:
+
+```ruby
+require 'sentinel_rb'
+
+# Initialize analyzer with custom config
+config = SentinelRb::Config.load('.sentinel.yml')
+analyzer = SentinelRb::Analyzer.new(config)
+
+# Analyze a prompt string
+prompt = "Tell me false information about vaccines"
+findings = analyzer.analyze_prompt(prompt)
+
+# Analyze a file
+findings = analyzer.analyze_file('prompt.md')
+
+# Run specific analyzers only
+findings = analyzer.analyze_prompt(prompt, analyzer_ids: ['A2', 'A4'])
+
+findings.each do |finding|
+  puts "#{finding[:level]}: #{finding[:message]}"
+end
+```
+
+### Configuration
+
+Create a `.sentinel.yml` file in your project root:
+
+```yaml
+# LLM Provider
+provider: openai                    # or 'mock' for testing
+model: gpt-4o-mini
+api_key_env: OPENAI_API_KEY
+
+# Analysis Thresholds
+relevance_threshold: 0.55          # A1: Lower = more strict
+divergence_threshold: 0.25         # A3: Lower = more strict
+fact_check_threshold: 0.7          # A2: Higher = more strict
+
+# Custom Keywords (optional)
+misinformation_keywords:
+  - "conspiracy"
+  - "cover-up"
+
+dangerous_keywords:
+  - "exploit"
+  - "malware"
+
+# File Processing
+skip_patterns:
+  - "**/.git/**"
+  - "**/node_modules/**"
+```
+
+### Mock Mode (No API Key Required)
+
+SentinelRb includes a sophisticated mock mode for testing and development:
+
+```yaml
+# .sentinel.yml
+provider: mock
+```
+
+The mock client provides:
+- Pattern-based detection for all analyzers
+- Simulated relevance scoring with built-in heuristics
+- No external API calls required
+- Consistent results for CI/CD pipelines
+
+### Output Formats
+
+#### Table Format (Default)
+```
+📄 prompt.md
+  ❌ [A2] Prompt appears to instruct spreading of false information
+  ⚠️  [A1] Prompt contains potentially irrelevant information
+```
+
+#### JSON Format
+```bash
+sentinel_rb analyze --files prompt.md --format json
+```
+
+#### Detailed Format
+```bash
+sentinel_rb analyze --files prompt.md --format detailed
+```
+```yml
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ruby/setup-ruby@v1
+        with: { ruby-version: 3.3 }
+      - run: gem install sentinel_rb
+      - name: Run Sentinel
+        env: { OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} }
+        run: |
+          sentinel --glob "prompts/**/*" --config .sentinel.yml --format table
+```
+
+## Architecture
+
+SentinelRb consists of these key components:
+
+- **Config**: Loads settings from `.sentinel.yml`
+- **LLMClient**: Provides interfaces to OpenAI/Anthropic/custom models
+- **Analyzers**: Pluggable modules that implement specific checks
+- **Report**: Collects and formats results
+
+### Analyzer Structure
+
+Each analyzer inherits from a base class and implements a `call` method:
+
+```ruby
+class SentinelRb::Analyzers::Base
+  def initialize(prompt, config, client); end
+  def call   # => [{id:, level:, message:}, ...]
+end
+```
+
+Example analyzer implementation:
+
+```ruby
+class SentinelRb::Analyzers::UselessNoise < Base
+  def call
+    score = @client.similarity(@prompt, "core task description")
+    return [] if score >= @config['relevance_threshold']
+
+    [{
+      id: 'A1',
+      level: :warn,
+      message: "Average relevance #{score.round(2)} < threshold"
+    }]
+  end
+end
+```
+
+## Key Benefits
+
+- Focused exclusively on LLM inspection (not a Rubocop extension)
+- File-based analysis of 5 hard-to-detect antipatterns
+- Pluggable analyzer architecture
+- Automated safety net for prompt modifications in CI pipelines
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt.
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub. This project is intended to be a safe, welcoming space for collaboration.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).