glancer 1.0.0

data/lib/glancer/utils/transaction.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Glancer
+  module Utils
+    class Transaction
+      def self.make(&block)
+        original_config = ActiveRecord::Base.connection_db_config
+        connection = read_only_connection || ActiveRecord::Base.connection
+
+        Glancer::Utils::Logger.info("Utils::Transaction",
+                                    "Using \e[1;33m#{connection_config_name(connection)}\e[36m connection for query execution.")
+
+        connection.transaction { yield(connection) }
+      rescue StandardError => e
+        Glancer::Utils::Logger.error("Utils::Transaction", "An error occurred: #{e.message}")
+        raise
+      ensure
+        ActiveRecord::Base.establish_connection(original_config) if read_only_connection_used?
+
+        if defined?(connection) && connection&.transaction_open?
+          Glancer::Utils::Logger.warn("Utils::Transaction",
+                                      "Transaction was not closed properly. Please check your code.")
+        else
+          Glancer::Utils::Logger.info("Utils::Transaction", "Transaction completed successfully.")
+        end
+      end
+
+      def self.read_only_connection
+        return nil unless Glancer.configuration.read_only_db
+
+        Glancer::Utils::Logger.info("Utils::Transaction", "Establishing connection to read-only database...")
+
+        @used_read_only = true
+
+        connection = ActiveRecord::Base.establish_connection(Glancer.configuration.read_only_db).connection
+
+        Glancer::Utils::Logger.info("Utils::Transaction", "Read-only database connection established successfully.")
+        connection
+      rescue StandardError => e
+        Glancer::Utils::Logger.error("Utils::Transaction",
+                                     "Failed to connect to read-only database: #{e.class} - #{e.message}")
+        Glancer::Utils::Logger.debug("Utils::Transaction", "Backtrace:\n#{e.backtrace.join("\n")}")
+        raise Glancer::Error, "Read-only DB connection failed: #{e.message}"
+      end
+
+      def self.read_only_connection_used?
+        value = @used_read_only
+        @used_read_only = false # reset
+        value
+      end
+
+      def self.connection_config_name(connection)
+        connection.pool.db_config.name
+      rescue StandardError
+        "unknown"
+      end
+    end
+  end
+end

data/lib/glancer/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Glancer
+  VERSION = "1.0.0"
+end

data/lib/glancer/workflow/ar_executor.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+module Glancer
+  module Workflow
+    class ARExecutor
+      def self.execute(code, original_question: nil, attempt: 1, message_id: nil)
+        Glancer::Utils::Logger.info("Workflow::ARExecutor", "Executing AR expression (Attempt ##{attempt})...")
+
+        run_id = SecureRandom.uuid
+
+        begin
+          result = nil
+          Glancer::Utils::Transaction.make do |connection|
+            Glancer::Workflow::Executor.apply_statement_timeout(connection)
+            raw = evaluate(code)
+            result = normalize(raw)
+            raise ActiveRecord::Rollback
+          end
+
+          Glancer::Audit.create!(
+            question: original_question,
+            code: code,
+            code_type: "activerecord",
+            adapter: Glancer.configuration.resolved_adapter,
+            run_id: run_id,
+            executed_at: Time.current,
+            message_id: message_id
+          )
+
+          result
+        rescue StandardError => e
+          if attempt >= 3
+            Glancer::Utils::Logger.error("Workflow::ARExecutor",
+                                         "Final failure after #{attempt} attempts: #{e.message}")
+            return { error: true, message: e.message, last_code: code }
+          end
+
+          Glancer::Utils::Logger.warn("Workflow::ARExecutor",
+                                      "AR Error (Attempt ##{attempt}): #{e.message}. Requesting correction...")
+
+          fixed_code = Glancer::Workflow::Builder.fix_ar_code(code, e.message)
+          execute(fixed_code, original_question: original_question, attempt: attempt + 1, message_id: message_id)
+        end
+      end
+
+      def self.evaluate(code)
+        TOPLEVEL_BINDING.eval(code)
+      end
+
+      def self.normalize(result)
+        rows = case result
+               when ActiveRecord::Relation
+                 result.to_a.map { |r| r.respond_to?(:attributes) ? r.attributes : { "value" => r } }
+               when Array
+                 result.map do |item|
+                   if item.respond_to?(:attributes)
+                     item.attributes
+                   elsif item.is_a?(Hash)
+                     item.stringify_keys
+                   else
+                     { "value" => item }
+                   end
+                 end
+               when Hash
+                 return normalize_hash(result.stringify_keys)
+               when Numeric, String
+                 return [{ "result" => result }]
+               when NilClass
+                 return []
+               else
+                 # Single AR model object (e.g. from .first / .find)
+                 return [{ "result" => result.inspect }] unless result.respond_to?(:attributes)
+
+                 [result.attributes]
+
+               end
+
+        drop_all_nil_columns(rows)
+      end
+
+      # Removes columns where every row has a nil value (e.g. `id` when using
+      # .select("col1, col2") — AR still populates id: nil on the model objects).
+      def self.drop_all_nil_columns(rows)
+        return rows if rows.empty?
+
+        nil_cols = rows.first.keys.select { |k| rows.all? { |r| r[k].nil? } }
+        return rows if nil_cols.empty?
+
+        rows.map { |r| r.except(*nil_cols) }
+      end
+
+      # Hashes from .group().count/sum/etc. map {group_value => aggregate} and must
+      # be rendered as rows. Hashes where values are mixed types (e.g. model
+      # attributes) are kept as a single row.
+      def self.normalize_hash(hash)
+        if hash.values.all? { |val| val.is_a?(Numeric) }
+          hash.map { |key, val| { "key" => key.to_s, "value" => val } }
+        else
+          [hash]
+        end
+      end
+    end
+  end
+end

data/lib/glancer/workflow/ar_extractor.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Glancer
+  module Workflow
+    class ARExtractor
+      def self.extract(text)
+        Glancer::Utils::Logger.info("Workflow::ARExtractor", "Extracting Ruby expression from LLM response...")
+
+        code = if text =~ /```(?:ruby)?\s*\n?(.*?)\s*```/mi
+                 Glancer::Utils::Logger.debug("Workflow::ARExtractor", "Extracted from code block.")
+                 ::Regexp.last_match(1).strip
+               else
+                 Glancer::Utils::Logger.debug("Workflow::ARExtractor", "No code block found, using raw text.")
+                 text.strip
+               end
+
+        Glancer::Utils::Logger.debug("Workflow::ARExtractor", "Extracted expression:\n#{code}")
+        code
+      rescue StandardError => e
+        Glancer::Utils::Logger.error("Workflow::ARExtractor", "Extraction failed: #{e.message}")
+        raise Glancer::Error, "AR code extraction failed: #{e.message}"
+      end
+    end
+  end
+end

data/lib/glancer/workflow/ar_prompt_builder.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Glancer
+  module Workflow
+    class ARPromptBuilder
+      def self.custom_instructions_block
+        custom = Glancer::Setting.get("custom_instructions")
+        custom.present? ? "CUSTOM RULES — MUST BE FOLLOWED STRICTLY:\n#{custom}\n" : ""
+      rescue StandardError
+        ""
+      end
+
+      def self.call(question, embeddings, history: [])
+        Glancer::Utils::Logger.info("Workflow::ARPromptBuilder", "Building AR prompt for: #{question.inspect}")
+
+        now = Time.current.strftime("%Y-%m-%d %H:%M:%S")
+
+        history_context = history.map do |msg|
+          if msg.role == "assistant" && msg.code.present?
+            "ASSISTANT (Ruby expression used): #{msg.code.strip}\nASSISTANT (response): #{msg.content}"
+          else
+            "#{msg.role.upcase}: #{msg.content}"
+          end
+        end.join("\n\n")
+
+        schema_context = embeddings.map { |e| e.content.strip }.join("\n\n")
+
+        <<~PROMPT
+          Current datetime: #{now}
+
+          You are a Ruby on Rails ActiveRecord expert.
+          Your ONLY task is to generate a Ruby expression that reads data from the database using ActiveRecord.
+
+          STRICT GUIDELINES:
+          1. **Output**: Return ONLY the Ruby expression, optionally inside a ```ruby code block. No prose.
+          2. **Read-only**: Only use read methods: .where, .joins, .includes, .select, .count, .sum, .average,
+             .minimum, .maximum, .group, .order, .limit, .offset, .pluck, .first, .last, .all, .find, .find_by,
+             .having, .distinct, .reorder, .references, .eager_load, .preload.
+          3. **Forbidden**: NEVER call .destroy, .destroy_all, .delete, .delete_all, .update, .update_all,
+             .save, .save!, .create, .create!, .insert, .upsert, .touch, or any write method.
+          4. **Model names**: Use EXACT class names as they appear in the DATABASE CONTEXT (e.g., User, Order).
+          5. **Result**: The expression must return an ActiveRecord::Relation, Array, Hash, Numeric, or String.
+          6. **Aliases**: When using .select with raw columns, use AS aliases for clarity.
+          7. **Single expression**: Prefer a single-line expression. Multi-line is allowed if necessary.
+
+          DATABASE CONTEXT:
+          #{schema_context}
+
+          CONVERSATION HISTORY:
+          #{history_context.presence || "(no prior messages)"}
+
+          #{custom_instructions_block}
+          NEW QUESTION:
+          #{question}
+
+          OUTPUT RUBY EXPRESSION ONLY:
+        PROMPT
+      rescue StandardError => e
+        Glancer::Utils::Logger.error("Workflow::ARPromptBuilder", "Failed: #{e.message}")
+        raise Glancer::Error, "AR prompt construction failed: #{e.message}"
+      end
+    end
+  end
+end

data/lib/glancer/workflow/ar_sanitizer.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module Glancer
+  module Workflow
+    class ARSanitizer
+      # Destructive ActiveRecord methods
+      FORBIDDEN_AR_METHODS = %w[
+        destroy destroy_all
+        delete delete_all
+        update update! update_all update_columns update_column
+        save save!
+        create create! create_or_find_by create_or_find_by!
+        insert insert_all insert_all!
+        upsert upsert_all
+        touch toggle! increment! decrement! increment_counter decrement_counter
+      ].freeze
+
+      # Shell / OS execution
+      SHELL_PATTERNS = [
+        /\bsystem\s*[\[(]/,
+        /`[^`]+`/,
+        /\bexec\s*\(/,
+        /\bspawn\s*\(/,
+        /\bOpen3\b/,
+        /\bIO\.popen\b/,
+        /\bKernel\s*\.\s*system\b/
+      ].freeze
+
+      # Dynamic code execution
+      EVAL_PATTERNS = [
+        /\beval\s*\(/,
+        /\binstance_eval\b/,
+        /\bclass_eval\b/,
+        /\bmodule_eval\b/,
+        /\bBinding\b/
+      ].freeze
+
+      # File system writes
+      FILE_WRITE_PATTERNS = [
+        /\bFileUtils\b/,
+        /\bFile\s*\.\s*(?:write|binwrite|open)\b/,
+        /\bIO\s*\.\s*(?:write|binwrite)\b/
+      ].freeze
+
+      # Dynamic loading
+      LOAD_PATTERNS = [
+        /\brequire(?:_relative)?\s*["'(]/,
+        /\bload\s*["'(]/,
+        /\bautoload\b/
+      ].freeze
+
+      def self.ensure_safe!(code)
+        Glancer::Utils::Logger.info("Workflow::ARSanitizer", "Sanitizing ActiveRecord expression...")
+
+        FORBIDDEN_AR_METHODS.each do |method|
+          # \b after the method name ensures we don't block e.g. .creates_table or .destroy_later
+          if code.match?(/\.#{Regexp.escape(method)}\b/)
+            raise Glancer::Error,
+                  "ActiveRecord expression blocked: forbidden write method '.#{method}'"
+          end
+        end
+
+        SHELL_PATTERNS.each do |pattern|
+          raise Glancer::Error, "ActiveRecord expression blocked: shell execution detected" if code.match?(pattern)
+        end
+
+        EVAL_PATTERNS.each do |pattern|
+          raise Glancer::Error, "ActiveRecord expression blocked: dynamic eval detected" if code.match?(pattern)
+        end
+
+        FILE_WRITE_PATTERNS.each do |pattern|
+          raise Glancer::Error, "ActiveRecord expression blocked: file write detected" if code.match?(pattern)
+        end
+
+        LOAD_PATTERNS.each do |pattern|
+          raise Glancer::Error, "ActiveRecord expression blocked: dynamic load detected" if code.match?(pattern)
+        end
+
+        Glancer::Utils::Logger.info("Workflow::ARSanitizer", "Expression passed sanitization check.")
+      rescue Glancer::Error
+        raise
+      rescue StandardError => e
+        Glancer::Utils::Logger.error("Workflow::ARSanitizer", "Sanitization failed: #{e.message}")
+        raise Glancer::Error, "AR sanitization failed: #{e.message}"
+      end
+    end
+  end
+end

data/lib/glancer/workflow/builder.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+
+module Glancer
+  module Workflow
+    class Builder
+      def self.build_sql(question, embeddings, history: [])
+        Glancer::Utils::Logger.info("Workflow::Builder", "Generating SQL from question: #{question.inspect}")
+
+        prompt = Glancer::Workflow::PromptBuilder.call(
+          question, embeddings, history: history, few_shot_examples: recent_examples
+        )
+        Glancer::Utils::Logger.debug("Workflow::Builder", "Generated prompt for SQL generation:\n#{prompt}")
+
+        chat = RubyLLM.chat(
+          provider: Glancer.configuration.resolved_code_provider,
+          model: Glancer.configuration.resolved_code_model,
+          assume_model_exists: true
+        )
+
+        response = chat.ask(prompt)
+
+        Glancer::Utils::Logger.info("Workflow::Builder",
+                                    "LLM responded with SQL (length: #{response.content&.length || 0} characters)")
+
+        response.content
+      rescue StandardError => e
+        Glancer::Utils::Logger.error("Workflow::Builder", "Failed to generate SQL: #{e.class} - #{e.message}")
+        Glancer::Utils::Logger.debug("Workflow::Builder", "Backtrace:\n#{e.backtrace.join("\n")}")
+        raise Glancer::Error, "code generation failed: #{e.message}"
+      end
+
+      def self.recent_examples
+        Glancer::Audit
+          .where(adapter: Glancer.configuration.resolved_adapter.to_s)
+          .where.not(question: [nil, ""])
+          .order(executed_at: :desc)
+          .limit(3)
+          .pluck(:question, :code)
+      rescue StandardError
+        []
+      end
+
+      def self.fix_sql(failed_sql, error_message)
+        Glancer::Utils::Logger.info("Workflow::Builder", "Attempting to fix failed SQL...")
+
+        prompt = <<~PROMPT
+          The following SQL query failed to execute:
+          ```sql
+          #{failed_sql}
+          ```
+
+          The database returned the following error message:
+          "#{error_message}"
+
+          Your task is to correct the SQL query so it becomes valid for the #{Glancer.configuration.resolved_adapter.to_s.upcase} adapter.
+          - Return ONLY the corrected SQL.
+          - Do not provide explanations or comments.
+          - Ensure it remains a safe SELECT statement.
+        PROMPT
+
+        chat = RubyLLM.chat(
+          provider: Glancer.configuration.resolved_code_provider,
+          model: Glancer.configuration.resolved_code_model,
+          assume_model_exists: true
+        )
+
+        response = chat.ask(prompt)
+
+        # Clean the response to ensure we only have the raw SQL
+        Glancer::Workflow::SQLExtractor.extract(response.content)
+      rescue StandardError => e
+        Glancer::Utils::Logger.error("Workflow::Builder", "Failed to fix SQL: #{e.message}")
+        raise Glancer::Error, "code correction workflow failed: #{e.message}"
+      end
+
+      def self.build_ar_code(question, embeddings, history: [])
+        Glancer::Utils::Logger.info("Workflow::Builder", "Generating AR code from question: #{question.inspect}")
+
+        prompt = Glancer::Workflow::ARPromptBuilder.call(question, embeddings, history: history)
+
+        chat = RubyLLM.chat(
+          provider: Glancer.configuration.resolved_code_provider,
+          model: Glancer.configuration.resolved_code_model,
+          assume_model_exists: true
+        )
+
+        response = chat.ask(prompt)
+        Glancer::Utils::Logger.info("Workflow::Builder",
+                                    "LLM responded with AR code (length: #{response.content&.length || 0} chars)")
+        response.content
+      rescue StandardError => e
+        Glancer::Utils::Logger.error("Workflow::Builder", "Failed to generate AR code: #{e.class} - #{e.message}")
+        raise Glancer::Error, "AR code generation failed: #{e.message}"
+      end
+
+      def self.fix_ar_code(failed_code, error_message)
+        Glancer::Utils::Logger.info("Workflow::Builder", "Attempting to fix failed AR expression...")
+
+        prompt = <<~PROMPT
+          The following Ruby/ActiveRecord expression failed to execute:
+          ```ruby
+          #{failed_code}
+          ```
+
+          The error returned was:
+          "#{error_message}"
+
+          Your task is to correct the Ruby expression so it becomes valid and read-only.
+          - Return ONLY the corrected Ruby expression (optionally in a ```ruby block).
+          - Do not provide explanations or comments.
+          - Ensure it uses only read methods (where, joins, select, count, sum, pluck, etc.).
+          - NEVER use .destroy, .delete, .update, .save, .create, or any write method.
+        PROMPT
+
+        chat = RubyLLM.chat(
+          provider: Glancer.configuration.resolved_code_provider,
+          model: Glancer.configuration.resolved_code_model,
+          assume_model_exists: true
+        )
+
+        response = chat.ask(prompt)
+        Glancer::Workflow::ARExtractor.extract(response.content)
+      rescue StandardError => e
+        Glancer::Utils::Logger.error("Workflow::Builder", "Failed to fix AR code: #{e.message}")
+        raise Glancer::Error, "AR code correction failed: #{e.message}"
+      end
+    end
+  end
+end

data/lib/glancer/workflow/cache.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Glancer
+  module Workflow
+    class Cache
+      @@store = {} # Using a simple hash for in-memory storage (@TODO redis / DB - i don't know now)
+
+      def self.fetch(question)
+        Glancer::Utils::Logger.info("Workflow::Cache", "Attempting to fetch cache for question: #{question.inspect}")
+
+        entry = @@store[question]
+        return nil unless entry
+
+        if expired?(entry)
+          Glancer::Utils::Logger.info("Workflow::Cache", "Cache entry expired for question: #{question.inspect}")
+          @@store.delete(question)
+          return nil
+        end
+
+        Glancer::Utils::Logger.info("Workflow::Cache", "Cache hit for question: #{question.inspect}")
+        Glancer::Utils::Logger.debug("Workflow::Cache", "Cached at: #{entry[:cached_at]}")
+        entry
+      rescue StandardError => e
+        Glancer::Utils::Logger.error("Workflow::Cache", "Failed to fetch from cache: #{e.class} - #{e.message}")
+        Glancer::Utils::Logger.debug("Workflow::Cache", "Backtrace:\n#{e.backtrace.join("\n")}")
+        nil
+      end
+
+      def self.write(question, result)
+        Glancer::Utils::Logger.info("Workflow::Cache", "Writing result to cache for question: #{question.inspect}")
+        @@store[question] = result.merge(cached_at: Time.current)
+      rescue StandardError => e
+        Glancer::Utils::Logger.error("Workflow::Cache", "Failed to write to cache: #{e.class} - #{e.message}")
+        Glancer::Utils::Logger.debug("Workflow::Cache", "Backtrace:\n#{e.backtrace.join("\n")}")
+      end
+
+      def self.clear
+        Glancer::Utils::Logger.info("Workflow::Cache", "Clearing all cache entries")
+        @@store.clear
+      rescue StandardError => e
+        Glancer::Utils::Logger.error("Workflow::Cache", "Failed to clear cache: #{e.class} - #{e.message}")
+        Glancer::Utils::Logger.debug("Workflow::Cache", "Backtrace:\n#{e.backtrace.join("\n")}")
+      end
+
+      def self.expired?(entry)
+        ttl = Glancer.configuration&.workflow_cache_ttl || 5.minutes
+        age = Time.current - entry[:cached_at]
+        Glancer::Utils::Logger.debug("Workflow::Cache", "Checking expiration: age=#{age.round(2)}s, ttl=#{ttl.inspect}")
+        age > ttl
+      rescue StandardError
+        true
+      end
+    end
+  end
+end

data/lib/glancer/workflow/executor.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Glancer
+  module Workflow
+    class Executor
+      def self.execute(sql, original_question: nil, attempt: 1, message_id: nil)
+        # Security check: Ensure only read queries are executed (SELECT or CTEs starting with WITH)
+        unless sql.strip.match?(/\A\s*(select|with)\b/i)
+          Glancer::Utils::Logger.error("Workflow::Executor", "Blocked attempt to run non-SELECT SQL.")
+          raise Glancer::Error, "Only SELECT queries are allowed for execution."
+        end
+
+        Glancer::Utils::Logger.info("Workflow::Executor", "Executing SQL (Attempt ##{attempt})...")
+
+        run_id = SecureRandom.uuid
+        # Appending a comment for easier database auditing
+        sql_with_comment = "#{sql.strip} /*glancer,run_id:#{run_id}*/"
+
+        begin
+          result = nil
+          Glancer::Utils::Transaction.make do |connection|
+            apply_statement_timeout(connection)
+            result = connection.exec_query(sql_with_comment).to_a
+            raise ActiveRecord::Rollback
+          end
+
+          # Audit successful execution
+          Glancer::Audit.create!(
+            question: original_question,
+            code: sql_with_comment,
+            code_type: "sql",
+            adapter: Glancer.configuration.resolved_adapter,
+            run_id: run_id,
+            executed_at: Time.current,
+            message_id: message_id
+          )
+
+          result
+        rescue StandardError => e
+          # Stop recursion if we reached the maximum number of attempts (3)
+          if attempt >= 3
+            Glancer::Utils::Logger.error("Workflow::Executor", "Final failure after #{attempt} attempts: #{e.message}")
+            return { error: true, message: e.message, last_code: sql }
+          end
+
+          Glancer::Utils::Logger.warn("Workflow::Executor",
+                                      "SQL Error (Attempt ##{attempt}): #{e.message}. Requesting correction...")
+
+          # Invoke the Builder to analyze the error and fix the SQL
+          fixed_sql = Glancer::Workflow::Builder.fix_sql(sql, e.message)
+
+          # Retry execution with the corrected SQL
+          execute(fixed_sql, original_question: original_question, attempt: attempt + 1, message_id: message_id)
+        end
+      end
+
+      def self.apply_statement_timeout(connection)
+        timeout_ms = Glancer.configuration.statement_timeout.to_i * 1000
+        adapter = Glancer.configuration.resolved_adapter.to_s
+
+        case adapter
+        when "postgres", "postgresql"
+          connection.execute("SET statement_timeout = #{timeout_ms}")
+        when "mysql", "mysql2"
+          connection.execute("SET max_execution_time = #{timeout_ms}")
+        end
+      rescue StandardError => e
+        Glancer::Utils::Logger.warn("Workflow::Executor", "Could not set statement timeout: #{e.message}")
+      end
+    end
+  end
+end