rails-nl2sql 0.1.7 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9723feb23a2bc5f1100279427d18b106964382689ade9c37b379cda0274e0a77
-  data.tar.gz: 9e4a429bc94b4f5d865e9efde2c19da5b6d9a775ffaf982e7ab1fe3070aa75f0
+  metadata.gz: c82ac012038c7065a0c3eeb3db0389fb64eec1fe6c95ae4c0896d6c580527dcc
+  data.tar.gz: 2a6236aad4a76959d360f6b19a7c7c38f662e66cfa7799108a4dd4e6cc546a7f
 SHA512:
-  metadata.gz: 24b563aeed1df87e3afc4683230471e18351e364794989791dd28e2057d8db42cd3a083b678640ac74dfff6bb4d4c28573ea6efa955ee1faae29d1dd2fc3984a
-  data.tar.gz: 74aa67f890208d440baaae2f863e6d92cc04d00c9ffc0012e14239e5f5bc0d2807cc170b460cba6c95c487812c9960209b6e1a7bf76ec238c8d63df342855e2b
+  metadata.gz: abe39e44ac4a2443b3cb778f7393cc012099c9266e1a35d25b3ad3bfdfae0b43b27a1206f52026f7cd31342931dc35ec0012ceb3bb8245a23fdadbaa9112da17
+  data.tar.gz: 0ab5536299aa7f4b7063c4c5264ec1b335a7e34957537c18da0bfa19d86b991efc135288d70be76938858e25ad9219d6f4f4d0b0299b72af212a7cb68ffa9418

data/README.md

@@ -42,6 +42,16 @@ To execute a natural language query, you can use the `execute` method:
 results = Rails::Nl2sql::Processor.execute("Show me all the users from California")
 ```
 
+### Using `from_nl` with ActiveRecord
+
+You can call the NL2SQL processor directly on your models. The `from_nl` method
+returns an `ActiveRecord::Relation` so you can chain scopes, pagination and
+other query modifiers as usual.
+
+```ruby
+User.from_nl("all users who signed up last week").limit(10)
+```
+
 You can also specify which tables to include or exclude:
 
 ```ruby
@@ -60,6 +70,18 @@ Rails::Nl2sql::Processor.get_tables
 Rails::Nl2sql::Processor.get_schema(include: ["users", "orders"])
 ```
 
+### Schema caching
+
+For efficiency the gem caches the full database schema on first use. The cached
+schema is reused for subsequent requests so your application doesn't need to hit
+the database every time a prompt is generated.
+
+You can clear the cached schema if your database changes:
+
+```ruby
+Rails::Nl2sql::SchemaBuilder.clear_cache!
+```
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/lib/generators/rails_nl2sql/install/install_generator.rb

@@ -0,0 +1,13 @@
+require 'rails/generators'
+
+module RailsNl2sql
+  module Generators
+    class InstallGenerator < Rails::Generators::Base
+      source_root File.expand_path('../templates', __FILE__)
+
+      def copy_initializer
+        template 'rails_nl2sql.rb', 'config/initializers/rails_nl2sql.rb'
+      end
+    end
+  end
+end

data/lib/generators/rails_nl2sql/install/templates/rails_nl2sql.rb

@@ -0,0 +1,4 @@
+Rails::Nl2sql.configure do |config|
+  config.api_key = "YOUR_API_KEY"
+  # config.model = "text-davinci-003"
+end

data/lib/rails/nl2sql/active_record_extension.rb

@@ -0,0 +1,21 @@
+require 'active_support/concern'
+require 'active_support/lazy_load_hooks'
+
+module Rails
+  module Nl2sql
+    module ActiveRecordExtension
+      extend ActiveSupport::Concern
+
+      class_methods do
+        def from_nl(prompt, options = {})
+          sql = Rails::Nl2sql::Processor.generate_query_only(prompt, options)
+          from(Arel.sql("(#{sql}) AS #{table_name}"))
+        end
+      end
+    end
+  end
+end
+
+ActiveSupport.on_load(:active_record) do
+  include Rails::Nl2sql::ActiveRecordExtension
+end

data/lib/rails/nl2sql/query_generator.rb

@@ -11,6 +11,11 @@ module Rails
       def generate_query(prompt, schema, db_server = "PostgreSQL", tables = nil)
         retrieved_context = build_context(schema, tables)
         
+        # Debug: Let's see what schema context is being sent
+        puts "=== SCHEMA CONTEXT BEING SENT TO AI ==="
+        puts retrieved_context
+        puts "=== END SCHEMA CONTEXT ==="
+        
         system_prompt = build_system_prompt(db_server, retrieved_context)
         user_prompt = build_user_prompt(prompt)
         
@@ -20,13 +25,16 @@ module Rails
           parameters: {
             model: @model,
             prompt: full_prompt,
-            max_tokens: 300,
+            max_tokens: 500,
             temperature: 0.1
           }
         )
 
         generated_query = response.dig("choices", 0, "text")&.strip
         
+        # Clean up the response to remove markdown formatting
+        generated_query = clean_sql_response(generated_query)
+        
         # Safety check
         validate_query_safety(generated_query)
         
@@ -80,9 +88,9 @@ module Rails
           ---
           **SQL GENERATION RULES:**
           1. **SQL Dialect:** All generated SQL must be valid **#{db_server} syntax**.
-             * For limiting results, use `LIMIT` (e.g., `LIMIT 10`) instead of `TOP`.
-             * Be mindful of #{db_server}'s specific function names (e.g., `COUNT(*)`, `MAX()`) and behaviors.
-             * For subqueries that return a single value to be used in a `WHERE` clause, ensure they are correctly formatted for #{db_server}.
+             * For limiting results, use LIMIT (e.g., LIMIT 10) instead of TOP.
+             * Be mindful of #{db_server}'s specific function names (e.g., COUNT(*), MAX()) and behaviors.
+             * For subqueries that return a single value to be used in a WHERE clause, ensure they are correctly formatted for #{db_server}.
           2. **Schema Adherence:** Only use table names and column names that are explicitly present in the provided context. Do not invent names.
           3. **Valid JOIN Paths:** All `JOIN` operations must be based on valid foreign key relationships. The provided schema context explicitly details many of these.
           4. **Safety First:** Absolutely **DO NOT** generate any DDL (CREATE, ALTER, DROP) or DML (INSERT, UPDATE, DELETE) statements. Only `SELECT` queries are permitted.
@@ -92,7 +100,7 @@ module Rails
              * This is essential
           6. **Ambiguity:** If a user question is ambiguous or requires more information to form a precise SQL query, clearly state that you need clarification and ask for more details. Do not guess.
           
-          **RESPOND WITH ONLY THE SQL QUERY - NO EXPLANATIONS OR ADDITIONAL TEXT.**
+          **RESPOND WITH ONLY THE SQL QUERY - NO EXPLANATIONS, NO MARKDOWN FORMATTING, NO CODE BLOCKS, NO ADDITIONAL TEXT.**
         PROMPT
       end
 
@@ -103,6 +111,45 @@ module Rails
         PROMPT
       end
 
+      def clean_sql_response(query)
+        return query unless query
+
+        # Remove markdown code blocks
+        query = query.gsub(/```sql\n?/, '')
+        query = query.gsub(/```\n?/, '')
+        
+        # Remove any leading/trailing whitespace
+        query = query.strip
+        
+        # Remove any explanatory text before or after the query
+        # Look for common patterns like "Here's the SQL query:" or "The query is:"
+        query = query.gsub(/^.*?(SELECT|WITH|INSERT|UPDATE|DELETE|CREATE|DROP|ALTER)/i, '\1')
+        
+        # Remove any trailing explanatory text after the query
+        # Split by newlines and take only the SQL part
+        lines = query.split("\n")
+        sql_lines = []
+        
+        lines.each do |line|
+          line = line.strip
+          # Skip empty lines or lines that look like explanations
+          next if line.empty?
+          next if line.match(/^(here|this|the query|explanation|note)/i)
+          
+          sql_lines << line
+        end
+        
+        # Rejoin the SQL lines
+        cleaned_query = sql_lines.join("\n").strip
+        
+        # Ensure it ends with a semicolon if it's a complete query
+        if cleaned_query.match(/^(SELECT|WITH)/i) && !cleaned_query.end_with?(';')
+          cleaned_query += ';'
+        end
+        
+        cleaned_query
+      end
+
       def validate_query_safety(query)
         return unless query
 

data/lib/rails/nl2sql/query_validator.rb

@@ -2,17 +2,43 @@ module Rails
   module Nl2sql
     class QueryValidator
       def self.validate(query)
+        return false unless query && !query.strip.empty?
+        
+        # Clean the query first
+        query = query.strip
+        
+        # Check if query is malformed (contains markdown or other formatting)
+        if query.include?('```') || query.include?('```sql')
+          raise Rails::Nl2sql::Error, "Query contains markdown formatting and could not be cleaned properly"
+        end
+        
         # Basic validation: prevent destructive commands
-        disallowed_keywords = %w(DROP DELETE UPDATE INSERT TRUNCATE ALTER CREATE)
-        if disallowed_keywords.any? { |keyword| query.upcase.include?(keyword) }
-          raise "Query contains disallowed keywords."
+        disallowed_keywords = %w(DROP DELETE UPDATE INSERT TRUNCATE ALTER CREATE EXEC EXECUTE MERGE REPLACE)
+        query_upper = query.upcase
+
+        if disallowed_keywords.any? { |keyword| query_upper.include?(keyword) }
+          raise Rails::Nl2sql::Error, "Query contains disallowed keywords."
+        end
+
+        # Ensure there is only a single statement
+        cleaned_query = query.rstrip
+        cleaned_query = cleaned_query.chomp(';')
+        if cleaned_query.include?(';')
+          raise Rails::Nl2sql::Error, "Query contains multiple statements."
+        end
+
+        # Ensure it's a SELECT query
+        unless query_upper.strip.start_with?('SELECT', 'WITH')
+          raise Rails::Nl2sql::Error, "Only SELECT queries are allowed."
         end
 
-        # Use Rails' built-in sanitization to be safe
+        # Use Rails' built-in validation with EXPLAIN
         begin
-          ActiveRecord::Base.connection.execute("EXPLAIN #{query}")
+          # Remove trailing semicolon for EXPLAIN
+          explain_query = query.gsub(/;\s*$/, '')
+          ActiveRecord::Base.connection.execute("EXPLAIN #{explain_query}")
         rescue ActiveRecord::StatementInvalid => e
-          raise "Invalid SQL query: #{e.message}"
+          raise Rails::Nl2sql::Error, "Invalid SQL query: #{e.message}"
         end
 
         true

data/lib/rails/nl2sql/schema_builder.rb

@@ -1,13 +1,26 @@
 module Rails
   module Nl2sql
     class SchemaBuilder
+      @@schema_cache = nil
+
       def self.build_schema(options = {})
+        if options.empty? && @@schema_cache
+          return @@schema_cache
+        end
+
         tables = get_filtered_tables(options)
-        
+
         schema_text = build_schema_text(tables)
+
+        @@schema_cache = schema_text if options.empty?
+
         schema_text
       end
 
+      def self.clear_cache!
+        @@schema_cache = nil
+      end
+
       def self.get_database_type
         adapter = ActiveRecord::Base.connection.adapter_name.downcase
         case adapter

data/lib/rails/nl2sql/version.rb

@@ -1,5 +1,5 @@
 module Rails
   module Nl2sql
-    VERSION = "0.1.7"
+    VERSION = "0.2.0"
   end
 end

data/lib/rails/nl2sql.rb

@@ -2,6 +2,7 @@ require "rails/nl2sql/version"
 require "rails/nl2sql/query_generator"
 require "rails/nl2sql/schema_builder"
 require "rails/nl2sql/query_validator"
+require "rails/nl2sql/active_record_extension"
 require "rails/nl2sql/railtie" if defined?(Rails)
 
 module Rails
@@ -26,6 +27,11 @@ module Rails
         # Build schema with optional table filtering
         schema = SchemaBuilder.build_schema(options)
         
+        # Debug: Show what schema is being built
+        puts "=== RAW SCHEMA FROM BUILDER ==="
+        puts schema
+        puts "=== END RAW SCHEMA ==="
+        
         # Extract tables for filtering if specified
         tables = options[:tables]
         

data/lib/rails-nl2sql.rb

@@ -0,0 +1,2 @@
+require "rails/nl2sql"
+

data/rails-nl2sql.gemspec

@@ -37,7 +37,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency "openai", "~> 0.3"
-  spec.add_development_dependency "bundler", "~> 1.17"
+  spec.add_development_dependency "bundler", ">= 2.0"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec-rails", "~> 6.0"
   spec.add_dependency "railties", ">= 6.0"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rails-nl2sql
 version: !ruby/object:Gem::Version
-  version: 0.1.7
+  version: 0.2.0
 platform: ruby
 authors:
 - Russell Van Curen
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2025-07-17 00:00:00.000000000 Z
+date: 2025-07-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: openai
@@ -28,16 +28,16 @@ dependencies:
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.17'
+        version: '2.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.17'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -89,6 +89,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".DS_Store"
 - ".gitignore"
 - Gemfile
 - Gemfile.lock
@@ -99,7 +100,11 @@ files:
 - bin/setup
 - lib/generators/rails/nl2sql/install_generator.rb
 - lib/generators/rails/nl2sql/templates/rails_nl2sql.rb
+- lib/generators/rails_nl2sql/install/install_generator.rb
+- lib/generators/rails_nl2sql/install/templates/rails_nl2sql.rb
+- lib/rails-nl2sql.rb
 - lib/rails/nl2sql.rb
+- lib/rails/nl2sql/active_record_extension.rb
 - lib/rails/nl2sql/query_generator.rb
 - lib/rails/nl2sql/query_validator.rb
 - lib/rails/nl2sql/railtie.rb