code_to_query 0.1.0

data/lib/code_to_query/validator.rb

@@ -0,0 +1,203 @@
+# frozen_string_literal: true
+
+require 'dry/schema'
+
+module CodeToQuery
+  class Validator
+    IntentSchema = Dry::Schema.Params do
+      required(:type).filled(:string)
+      required(:table).filled(:string)
+      required(:columns).array(:string)
+      optional(:filters).array(:hash) do
+        optional(:column).maybe(:string)
+        required(:op).filled(:string)
+        optional(:param).filled(:string)
+        optional(:param_start).filled(:string)
+        optional(:param_end).filled(:string)
+        # Optional fields to support correlated subqueries (NOT EXISTS)
+        optional(:related_table).filled(:string)
+        optional(:fk_column).filled(:string)
+        optional(:base_column).filled(:string)
+        optional(:related_filters).array(:hash) do
+          required(:column).filled(:string)
+          required(:op).filled(:string)
+          optional(:param).filled(:string)
+          optional(:param_start).filled(:string)
+          optional(:param_end).filled(:string)
+        end
+      end
+      optional(:order).array(:hash) do
+        required(:column).filled(:string)
+        required(:dir).filled(:string)
+      end
+      optional(:limit).filled(:integer)
+      optional(:params).hash
+      optional(:distinct).filled(:bool)
+      optional(:distinct_on).array(:string)
+      optional(:aggregations).array(:hash) do
+        required(:type).filled(:string)
+        optional(:column).filled(:string)
+      end
+      optional(:group_by).array(:string)
+    end
+
+    def validate(intent_hash, current_user: nil, allow_tables: nil)
+      preprocessed = preprocess_exists_filters(intent_hash)
+
+      if !preprocessed.key?('limit') && CodeToQuery.config.default_limit
+        preprocessed = preprocessed.merge('limit' => CodeToQuery.config.default_limit)
+      end
+
+      result = IntentSchema.call(preprocessed)
+      raise ArgumentError, "Invalid intent: #{result.errors.to_h}" unless result.success?
+
+      validated = result.to_h
+
+      original_metrics = intent_hash['_metrics'] || intent_hash[:_metrics]
+      validated['_metrics'] = original_metrics if original_metrics.is_a?(Hash)
+
+      Array(validated['filters']).each_with_index do |f, idx|
+        op = f['op'].to_s
+        if %w[exists not_exists].include?(op)
+          unless f['related_table'].to_s.strip != '' && f['fk_column'].to_s.strip != ''
+            raise ArgumentError, "Invalid intent: filters[#{idx}] requires related_table and fk_column for #{op}"
+          end
+
+          f['base_column'] ||= 'id'
+          f['column'] ||= 'id'
+        else
+          unless f['column'].to_s.strip != ''
+            raise ArgumentError, "Invalid intent: filters[#{idx}].column must be filled"
+          end
+        end
+      end
+      enforce_allowlists!(validated, current_user: current_user, allow_tables: allow_tables)
+      validated
+    end
+
+    private
+
+    def preprocess_exists_filters(intent_hash)
+      intent = intent_hash.dup
+
+      if intent['filters'].is_a?(Array)
+        intent['filters'] = intent['filters'].map do |filter|
+          if filter.is_a?(Hash) && %w[exists not_exists].include?(filter['op'].to_s) && filter['column'].nil?
+            filter.merge('column' => 'id')
+          else
+            filter
+          end
+        end
+      end
+
+      intent
+    end
+
+    def enforce_allowlists!(intent, current_user:, allow_tables:)
+      # Enforce table allowlist if provided (from user input)
+      if Array(allow_tables).any?
+        table = intent['table']
+        if (table.to_s.strip != '') && !Array(allow_tables).map { |t| t.to_s.downcase }.include?(table.to_s.downcase)
+          raise ArgumentError, "Invalid intent: table '#{table}' not allowed"
+        end
+      end
+
+      # Consult policy adapter for additional table/column allowlists
+      adapter = CodeToQuery.config.policy_adapter
+      return unless adapter.respond_to?(:call)
+
+      policy_info = safe_call_policy_adapter(adapter, current_user, table: intent['table'], intent: intent)
+      allowed_tables = Array(policy_info[:allowed_tables] || policy_info['allowed_tables']).map { |t| t.to_s.downcase }
+      if allowed_tables.any?
+        table = intent['table']
+        if (table.to_s.strip != '') && !allowed_tables.include?(table.to_s.downcase)
+          raise ArgumentError, "Invalid intent: table '#{table}' not permitted by policy"
+        end
+      end
+
+      allowed_columns = policy_info[:allowed_columns] || policy_info['allowed_columns'] || {}
+      return if allowed_columns.nil? || allowed_columns.empty?
+
+      # Normalize map keys to strings with lowercase table and column names
+      normalized = {}
+      allowed_columns.each do |tbl, cols|
+        normalized[tbl.to_s.downcase] = Array(cols).map { |c| c.to_s.downcase }
+      end
+
+      main_table = intent['table'].to_s.downcase
+
+      # Columns in SELECT
+      Array(intent['columns']).each do |col|
+        next if col == '*'
+        next unless normalized[main_table]&.any?
+        unless normalized[main_table].include?(col.to_s.downcase)
+          raise ArgumentError, "Invalid intent: selecting column '#{col}' not permitted on '#{main_table}'"
+        end
+      end
+
+      # ORDER BY columns
+      Array(intent['order']).each do |o|
+        col = o['column']
+        next if col.nil?
+        next unless normalized[main_table]&.any?
+        unless normalized[main_table].include?(col.to_s.downcase)
+          raise ArgumentError, "Invalid intent: ordering by column '#{col}' not permitted on '#{main_table}'"
+        end
+      end
+
+      # DISTINCT ON columns
+      Array(intent['distinct_on']).each do |col|
+        next unless normalized[main_table]&.any?
+        unless normalized[main_table].include?(col.to_s.downcase)
+          raise ArgumentError, "Invalid intent: distinct_on column '#{col}' not permitted on '#{main_table}'"
+        end
+      end
+
+      # GROUP BY
+      Array(intent['group_by']).each do |col|
+        next unless normalized[main_table]&.any?
+        unless normalized[main_table].include?(col.to_s.downcase)
+          raise ArgumentError, "Invalid intent: group_by column '#{col}' not permitted on '#{main_table}'"
+        end
+      end
+
+      # WHERE filters
+      Array(intent['filters']).each do |f|
+        op = f['op'].to_s
+        if %w[exists not_exists].include?(op)
+          related_table = f['related_table']
+          rel_cols = normalized[related_table.to_s.downcase]
+          next if rel_cols.nil? || rel_cols.empty?
+
+          Array(f['related_filters']).each do |rf|
+            col = rf['column']
+            next if col.nil?
+            unless rel_cols.include?(col.to_s.downcase)
+              raise ArgumentError, "Invalid intent: filter column '#{col}' not permitted on '#{related_table}'"
+            end
+          end
+        else
+          col = f['column']
+          next if col.nil?
+
+          cols = normalized[main_table]
+          next if cols.nil? || cols.empty?
+          unless cols.include?(col.to_s.downcase)
+            raise ArgumentError, "Invalid intent: filter column '#{col}' not permitted on '#{main_table}'"
+          end
+        end
+      end
+    rescue StandardError => e
+      # Re-raise as ArgumentError to keep validator contract
+      raise ArgumentError, e.message
+    end
+
+    def safe_call_policy_adapter(adapter, current_user, table:, intent:)
+      adapter.call(current_user, table: table, intent: intent)
+    rescue ArgumentError
+      adapter.call(current_user, table: table)
+    rescue StandardError
+      {}
+    end
+  end
+end

data/lib/code_to_query/version.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module CodeToQuery
+  # Gem version
+  VERSION = '0.1.0'
+end

data/lib/code_to_query.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+# Main entry point for CodeToQuery gem
+
+require 'logger'
+require 'active_support'
+require 'active_support/core_ext/object/blank'
+require 'json'
+require 'active_support/core_ext/hash/keys'
+
+require_relative 'code_to_query/version'
+require_relative 'code_to_query/configuration'
+require_relative 'code_to_query/providers/base'
+require_relative 'code_to_query/providers/openai'
+require_relative 'code_to_query/providers/local'
+require_relative 'code_to_query/planner'
+require_relative 'code_to_query/validator'
+require_relative 'code_to_query/compiler'
+require_relative 'code_to_query/runner'
+require_relative 'code_to_query/query'
+require_relative 'code_to_query/guardrails/sql_linter'
+require_relative 'code_to_query/guardrails/explain_gate'
+require_relative 'code_to_query/context/pack'
+require_relative 'code_to_query/context/builder'
+require_relative 'code_to_query/performance/cache'
+require_relative 'code_to_query/performance/optimizer'
+require_relative 'code_to_query/llm_client'
+require_relative 'code_to_query/policies/pundit_adapter'
+require_relative 'code_to_query/errors'
+require_relative 'code_to_query/railtie' if defined?(Rails)
+
+module CodeToQuery
+  class Error < StandardError; end
+
+  # Backward compatibility for new configuration accessors in older environments/tests
+  module BackCompat
+    module_function
+
+    def ensure_extended_config!(config)
+      # Logger
+      unless config.respond_to?(:logger)
+        class << config
+          attr_accessor :logger
+        end
+        config.logger = if defined?(Rails) && Rails.respond_to?(:logger)
+                          Rails.logger
+                        else
+                          Logger.new($stdout)
+                        end
+      end
+
+      # LLM knobs and prompt template
+      return if config.respond_to?(:system_prompt_template)
+
+      class << config
+        attr_accessor :system_prompt_template, :llm_api_base, :llm_timeout, :llm_temperature, :provider_options
+      end
+      config.system_prompt_template = nil
+      config.llm_api_base = ENV['CODE_TO_QUERY_LLM_API_BASE'] || 'https://api.openai.com/v1'
+      config.llm_timeout = Integer(ENV['CODE_TO_QUERY_LLM_TIMEOUT'] || 30)
+      config.llm_temperature = Float(ENV['CODE_TO_QUERY_LLM_TEMPERATURE'] || 0.1)
+      config.provider_options = {}
+    end
+  end
+
+  # Configure the gem
+  def self.configure
+    BackCompat.ensure_extended_config!(Configuration.instance)
+    yield(Configuration.instance)
+  end
+
+  # Access the current configuration
+  def self.config
+    BackCompat.ensure_extended_config!(Configuration.instance)
+    Configuration.instance
+  end
+
+  # Convert natural language to SQL query
+  # current_user is optional and only used if a policy_adapter requires it
+  def self.ask(prompt:, schema: nil, allow_tables: nil, current_user: nil)
+    intent = Planner.new(config).plan(prompt: prompt, schema: schema, allow_tables: allow_tables)
+    validated_intent = Validator.new.validate(intent, current_user: current_user, allow_tables: allow_tables).deep_stringify_keys
+    compiled = Compiler.new(config).compile(validated_intent, current_user: current_user)
+
+    Guardrails::SqlLinter.new(config, allow_tables: allow_tables).check!(compiled[:sql])
+
+    Query.new(sql: compiled[:sql], params: compiled[:params], bind_spec: compiled[:bind_spec],
+              intent: validated_intent, allow_tables: allow_tables, config: config)
+  end
+end

data/tasks/code_to_query.rake

@@ -0,0 +1,326 @@
+# frozen_string_literal: true
+
+namespace :code_to_query do
+  desc 'Bootstrap complete context pack (schema + app scan + glossary + policies)'
+  task bootstrap: :environment do
+    require 'code_to_query'
+
+    puts 'Starting CodeToQuery bootstrap process...'
+
+    begin
+      pack = CodeToQuery::Context::Builder.bootstrap!
+
+      puts 'Context pack created.'
+      puts "Location: #{CodeToQuery.config.context_pack_path}"
+      puts "Tables: #{pack.schema[:tables]&.length || 0}"
+      puts "Models: #{pack.models[:models]&.length || 0}"
+      puts "Glossary terms: #{pack.glossary&.length || 0}"
+
+      # Run verification
+      Rake::Task['code_to_query:verify'].invoke
+    rescue StandardError => e
+      puts "Bootstrap failed: #{e.message}"
+      puts 'Try running individual tasks to debug:'
+      puts '   rake code_to_query:schema'
+      puts '   rake code_to_query:scan_app'
+      puts '   rake code_to_query:glossary'
+      puts '   rake code_to_query:policies'
+      exit 1
+    end
+  end
+
+  desc 'Extract schema information from database'
+  task schema: :environment do
+    require 'code_to_query'
+
+    puts 'Extracting database schema...'
+
+    begin
+      builder = CodeToQuery::Context::Builder.new
+      schema = builder.extract_schema
+
+      pack_data = load_existing_pack
+      pack_data['schema'] = schema
+      pack_data['updated_at'] = Time.now.iso8601
+
+      write_pack_data(pack_data)
+
+      puts 'Schema extracted.'
+      puts "Tables: #{schema[:tables]&.length || 0}"
+      puts "Adapter: #{schema[:adapter]}"
+      puts "Version: #{schema[:version]}"
+    rescue StandardError => e
+      puts "Schema extraction failed: #{e.message}"
+      exit 1
+    end
+  end
+
+  desc 'Scan Rails application models and associations'
+  task scan_app: :environment do
+    require 'code_to_query'
+
+    puts 'Scanning Rails application...'
+
+    begin
+      builder = CodeToQuery::Context::Builder.new
+      app_data = builder.scan_app
+
+      pack_data = load_existing_pack
+      pack_data['models'] = app_data
+      pack_data['updated_at'] = Time.now.iso8601
+
+      write_pack_data(pack_data)
+
+      puts 'Application scan completed.'
+      puts "Models: #{app_data[:models]&.length || 0}"
+      puts "Associations: #{app_data[:associations]&.length || 0}"
+    rescue StandardError => e
+      puts "Application scan failed: #{e.message}"
+      exit 1
+    end
+  end
+
+  desc 'Generate business glossary and synonyms'
+  task glossary: :environment do
+    require 'code_to_query'
+
+    puts 'Generating business glossary...'
+
+    begin
+      builder = CodeToQuery::Context::Builder.new
+      glossary = builder.generate_glossary
+
+      pack_data = load_existing_pack
+      pack_data['glossary'] = glossary
+      pack_data['updated_at'] = Time.now.iso8601
+
+      write_pack_data(pack_data)
+
+      puts 'Glossary generated.'
+      puts "Terms: #{glossary&.length || 0}"
+    rescue StandardError => e
+      puts "Glossary generation failed: #{e.message}"
+      exit 1
+    end
+  end
+
+  desc 'Collect security policies and access rules'
+  task policies: :environment do
+    require 'code_to_query'
+
+    puts 'Collecting security policies...'
+
+    begin
+      builder = CodeToQuery::Context::Builder.new
+      policies = builder.collect_policies
+
+      pack_data = load_existing_pack
+      pack_data['policies'] = policies
+      pack_data['updated_at'] = Time.current.iso8601
+
+      write_pack_data(pack_data)
+
+      puts 'Policies collected.'
+    rescue StandardError => e
+      puts "Policy collection failed: #{e.message}"
+      exit 1
+    end
+  end
+
+  desc 'Verify context pack integrity and completeness'
+  task verify: :environment do
+    require 'code_to_query'
+
+    puts 'Verifying context pack...'
+
+    begin
+      builder = CodeToQuery::Context::Builder.new
+      builder.verify!
+
+      # Additional comprehensive verification
+      pack_data = load_existing_pack
+
+      # Check schema completeness
+      schema = pack_data['schema'] || {}
+      tables = schema['tables'] || []
+
+      puts 'Context pack verification passed.'
+      puts "Schema: #{tables.length} tables"
+
+      if pack_data['models']
+        models = pack_data['models']['models'] || {}
+        puts "Models: #{models.length} models"
+      end
+
+      puts "Glossary: #{pack_data['glossary'].length} terms" if pack_data['glossary']
+
+      # Check for potential issues
+      warnings = []
+      warnings << 'No tables found in schema' if tables.empty?
+      warnings << 'No models found' if pack_data.dig('models', 'models') && pack_data.dig('models', 'models').empty?
+      warnings << 'No glossary terms' if pack_data['glossary'] && pack_data['glossary'].empty?
+
+      if warnings.any?
+        puts "\nWarnings:"
+        warnings.each { |warning| puts "   - #{warning}" }
+      end
+
+      puts "\nContext pack location: #{CodeToQuery.config.context_pack_path}"
+    rescue StandardError => e
+      puts "Verification failed: #{e.message}"
+      exit 1
+    end
+  end
+
+  desc 'Show context pack statistics and information'
+  task info: :environment do
+    require 'code_to_query'
+
+    puts 'CodeToQuery Context Pack Information'
+    puts '=' * 50
+
+    begin
+      pack_data = load_existing_pack
+
+      puts "Location: #{CodeToQuery.config.context_pack_path}"
+      puts "Last updated: #{pack_data['updated_at'] || 'Unknown'}"
+      puts
+
+      # Schema info
+      if pack_data['schema']
+        schema = pack_data['schema']
+        puts 'Database Schema:'
+        puts "   Adapter: #{schema['adapter']}"
+        puts "   Version: #{schema['version']}"
+        puts "   Tables: #{schema['tables']&.length || 0}"
+
+        if schema['tables']&.any?
+          puts '   Sample tables:'
+          schema['tables'].first(5).each do |table|
+            column_count = table['columns']&.length || 0
+            puts "     - #{table['name']} (#{column_count} columns)"
+          end
+          puts '     ...' if schema['tables'].length > 5
+        end
+        puts
+      end
+
+      # Models info
+      if pack_data['models']
+        models_data = pack_data['models']
+        models = models_data['models'] || {}
+        associations = models_data['associations'] || {}
+
+        puts 'Rails Models:'
+        puts "   Models: #{models.length}"
+        puts "   With associations: #{associations.length}"
+
+        if models.any?
+          puts '   Sample models:'
+          models.first(5).each do |name, info|
+            puts "     - #{name} → #{info['table_name']}"
+          end
+          puts '     ...' if models.length > 5
+        end
+        puts
+      end
+
+      # Glossary info
+      if pack_data['glossary']
+        glossary = pack_data['glossary']
+        puts 'Business Glossary:'
+        puts "   Terms: #{glossary.length}"
+
+        if glossary.any?
+          puts '   Sample terms:'
+          glossary.first(5).each do |term, synonyms|
+            puts "     - #{term}: #{Array(synonyms).join(', ')}"
+          end
+          puts '     ...' if glossary.length > 5
+        end
+        puts
+      end
+
+      # Configuration info
+      config = CodeToQuery.config
+      puts 'Configuration:'
+      puts "   Database adapter: #{config.adapter}"
+      puts "   Default limit: #{config.default_limit}"
+      puts "   Max limit: #{config.max_limit}"
+      puts "   Readonly role: #{config.readonly_role || 'None'}"
+      puts "   OpenAI model: #{config.openai_model}"
+      puts "   Explain gate: #{config.enable_explain_gate ? 'Enabled' : 'Disabled'}"
+    rescue StandardError => e
+      puts "Could not load context pack: #{e.message}"
+      puts "   Run 'rake code_to_query:bootstrap' to create it."
+    end
+  end
+
+  desc 'Clean and rebuild context pack'
+  task rebuild: :environment do
+    require 'code_to_query'
+
+    puts 'Rebuilding context pack...'
+
+    # Remove existing pack
+    if File.exist?(CodeToQuery.config.context_pack_path)
+      File.delete(CodeToQuery.config.context_pack_path)
+      puts 'Removed existing context pack'
+    end
+
+    # Run bootstrap
+    Rake::Task['code_to_query:bootstrap'].invoke
+  end
+
+  desc 'Test query generation with sample prompts'
+  task test: :environment do
+    require 'code_to_query'
+
+    puts 'Testing CodeToQuery with sample prompts...'
+
+    sample_prompts = [
+      'Show me all users created this month',
+      'Top 10 orders by amount',
+      'Find invoices from July 2023',
+      'Count active customers',
+      'Recent payments over $1000'
+    ]
+
+    sample_prompts.each_with_index do |prompt, index|
+      puts "\n#{index + 1}. Testing: \"#{prompt}\""
+
+      begin
+        query = CodeToQuery.ask(prompt: prompt)
+        puts "   SQL: #{query.sql}"
+        puts "   Params: #{query.params.inspect}"
+        puts "   Safe: #{query.safe?}"
+      rescue StandardError => e
+        puts "   Error: #{e.message}"
+      end
+    end
+
+    puts "\nTest completed."
+  end
+
+  private
+
+  def load_existing_pack
+    path = CodeToQuery.config.context_pack_path
+    if File.exist?(path)
+      JSON.parse(File.read(path))
+    else
+      {}
+    end
+  rescue JSON::ParserError
+    puts 'Existing context pack has invalid JSON, starting fresh'
+    {}
+  end
+
+  def write_pack_data(data)
+    path = CodeToQuery.config.context_pack_path
+    dir = File.dirname(path)
+    FileUtils.mkdir_p(dir)
+
+    File.write(path, JSON.pretty_generate(data))
+  end
+end