sql-chatbot-rails 1.0.0

data/lib/sql_chatbot/services/registry_builder.rb

@@ -0,0 +1,385 @@
+# frozen_string_literal: true
+
+require "set"
+require "sql_chatbot/grammar/registry"
+
+module SqlChatbot
+  module Services
+    class RegistryBuilder
+      SOFT_DELETE_COLS = %w[deleted_at discarded_at archived_at removed_at].freeze
+
+      def initialize(default_filters: nil, custom_aliases: nil)
+        @default_filters = default_filters || {}
+        @custom_aliases = custom_aliases || {}
+      end
+
+      def build
+        entities = {}
+        discover_models.each do |model|
+          entity_name = model.name.underscore
+          entities[entity_name] = build_entity(model, entity_name)
+        end
+        registry = Grammar::Registry.new(
+          framework: "rails",
+          entities: entities,
+          aliases: build_entity_aliases(entities),
+        )
+        apply_domain_synonyms!(registry)
+        apply_custom_aliases!(registry)
+        registry
+      end
+
+      # Conservative domain-synonym aliasing. Mirror of TS applyDomainSynonyms
+      # in registry-loader.ts. For each synonym group, only register the rest
+      # of the group as aliases when exactly ONE entity claims a group member
+      # (via canonical name OR an existing alias) — avoids the "customer and
+      # user are both real entities" ambiguity.
+      SYNONYM_GROUPS = [
+        %w[ticket tickets conversation conversations chat chats],
+        %w[order orders purchase purchases transaction transactions],
+        %w[post posts article articles entry entries],
+        # Code repositories — does NOT include project/projects. Apps with
+        # both `repository` and `project` tables (Gitea) would split the
+        # group's claim and the entire group would skip via the single-
+        # claimer rule. Code review on V1.3-R caught this.
+        %w[repo repos repository repositories],
+        %w[issue issues bug bugs task tasks],
+        # Messaging / communication channels — Chatwoot uses `inbox`,
+        # Slack-style apps use `channel`. Single-claimer rule handles
+        # apps that have only one of the two as a real entity.
+        %w[channel channels inbox inboxes],
+      ].freeze
+
+      def apply_domain_synonyms!(registry)
+        SYNONYM_GROUPS.each do |group|
+          claiming = Set.new
+          group.each do |member|
+            lc = member.downcase
+            if registry.entities.key?(lc)
+              claiming << lc
+              next
+            end
+            target = registry.aliases[lc]
+            claiming << target if target
+          end
+          next unless claiming.size == 1
+          target = claiming.first
+          group.each do |member|
+            lc = member.downcase
+            next if registry.entities.key?(lc)
+            next if registry.aliases.key?(lc)
+            registry.aliases[lc] = target
+          end
+        end
+      end
+
+      # Merge developer-supplied aliases. Custom mappings always WIN — they
+      # override auto-detected aliases on conflict. Silently skips entries
+      # whose target entity doesn't exist in the registry (typo in config
+      # or stale config after schema change).
+      def apply_custom_aliases!(registry)
+        return if @custom_aliases.nil? || @custom_aliases.empty?
+        @custom_aliases.each do |raw_alias, raw_target|
+          alias_term = raw_alias.to_s.downcase.strip
+          target     = raw_target.to_s.strip
+          next if alias_term.empty? || target.empty?
+          next unless registry.entities.key?(target)
+          next if registry.entities.key?(alias_term)
+          registry.aliases[alias_term] = target
+        end
+      end
+
+      # Build aliases that map common question phrasings to canonical entity names.
+      # Rules (no clash with a real canonical entity name):
+      #   multi-word form (spaces) → snake_case entity name
+      #     "service areas" → "service_area"
+      #   plural form → singular entity name
+      #     "users" / "customers" → already handled by string-match scoring,
+      #     but we also explicitly map plural-snake → canonical
+      def build_entity_aliases(entities)
+        aliases = {}
+        # Process simpler names first so they get first dibs on auto-aliases.
+        # Without this, `account_user` (2 segments) could claim `account_users`
+        # before `user` (1 segment) had a chance to claim its own plural.
+        ordered_names = entities.keys.sort_by { |n| [n.split("_").length, n] }
+        ordered_names.each do |name|
+          # "service_area" → "service areas", "service area", "service_areas"
+          spaced     = name.tr("_", " ")
+          spaced_plu = pluralize_simple(spaced)
+          snake_plu  = pluralize_simple(name)
+          [spaced, spaced_plu, snake_plu].each do |alt|
+            next if alt == name
+            next if entities.key?(alt)
+            next if aliases.key?(alt)
+            aliases[alt] = name
+          end
+        end
+        aliases
+      end
+
+      def pluralize_simple(word)
+        return word + "es" if word.end_with?("s", "x", "ch", "sh")
+        return word[0..-2] + "ies" if word.end_with?("y") && !%w[a e i o u].include?(word[-2])
+        word + "s"
+      end
+
+      private
+
+      def discover_models
+        return [] unless defined?(ActiveRecord::Base)
+        eager_load_models!
+        ActiveRecord::Base.descendants.select do |m|
+          !m.abstract_class? && m.respond_to?(:table_name) && safe_table_exists?(m)
+        end
+      end
+
+      def eager_load_models!
+        return unless defined?(Rails) && Rails.respond_to?(:application) && Rails.application
+        return if Rails.application.config.eager_load
+        if defined?(Zeitwerk) && Rails.autoloaders.respond_to?(:main)
+          Rails.application.paths["app/models"]&.to_a&.each do |p|
+            abs = Rails.root.join(p).to_s
+            Rails.autoloaders.main.eager_load_dir(abs) if Dir.exist?(abs)
+          end
+        else
+          Rails.application.eager_load!
+        end
+      rescue => e
+        warn "[SqlChatbot] RegistryBuilder eager_load: #{e.message}"
+      end
+
+      def safe_table_exists?(model)
+        model.table_exists?
+      rescue
+        false
+      end
+
+      def build_entity(model, entity_name)
+        fields = build_fields(model)
+        timestamps = detect_timestamps(model)
+        Grammar::Entity.new(
+          name: entity_name,
+          table: model.table_name,
+          display_label: model.name,
+          row_count: safe_row_count(model),
+          primary_key: model.primary_key.to_s,
+          timestamps: timestamps,
+          fields: fields,
+          scopes: build_scopes(model),
+          associations: build_associations(model),
+          ranking_candidates: ranking_candidates_for(model),
+          implicit_filters: compile_implicit_filters(model.table_name, fields, timestamps)
+        )
+      end
+
+      # Compute the full set of implicit WHERE clauses for an entity. Two sources:
+      #   1. Schema-detected soft-delete column → `<col> IS NULL`
+      #   2. Developer-declared `default_filters` config (`*.col` or `table.col`)
+      #
+      # Returns structured `ImplicitFilter` entries — the template compiler
+      # emits SQL at compile time and uses `column` for dedup against any
+      # explicit user filter on the same column. Silently skips config entries
+      # whose column is absent from the table; a config entry on the same
+      # column as an auto-detected soft-delete entry overrides it.
+      def compile_implicit_filters(table, fields, timestamps)
+        out = []
+        deleted = timestamps[:deleted] || timestamps["deleted"]
+        if deleted
+          out << Grammar::ImplicitFilter.new(column: deleted, expr: "IS NULL", source: :soft_delete)
+        end
+        return out if @default_filters.nil? || @default_filters.empty?
+
+        @default_filters.each do |raw_key, raw_expr|
+          key = raw_key.to_s
+          expr = raw_expr.to_s.strip
+          next if key.empty? || expr.empty?
+          dot = key.index(".")
+          next if dot.nil? || dot.zero?
+          t   = key[0...dot]
+          col = key[(dot + 1)..-1].to_s
+          next if col.empty?
+          next unless t == "*" || t == table
+          next unless fields.key?(col)
+          out.reject! { |f| f.column == col }
+          out << Grammar::ImplicitFilter.new(column: col, expr: expr, source: :default_filter)
+        end
+        out
+      end
+
+      def safe_row_count(model)
+        model.count
+      rescue
+        0
+      end
+
+      def detect_timestamps(model)
+        cols = model.columns_hash.keys
+        ts = {}
+        ts[:created] = "created_at" if cols.include?("created_at")
+        ts[:updated] = "updated_at" if cols.include?("updated_at")
+        soft = SOFT_DELETE_COLS.find { |c| cols.include?(c) }
+        ts[:deleted] = soft if soft
+        ts
+      end
+
+      def build_fields(model)
+        enums = model.defined_enums
+        fields = {}
+        model.columns_hash.each do |col, info|
+          enum_vals = enums[col]
+          type = enum_vals ? :enum : map_type(info.type)
+          fields[col] = Grammar::Field.new(
+            column: col,
+            type: type,
+            nullable: info.null,
+            enum_values: enum_vals,
+            fk_to: nil,
+            user_facing_label: col.humanize,
+            searchable: type == :text
+          )
+        end
+        # Add field aliases for common human phrasings. For each real column,
+        # detect short synonyms and register alias keys pointing to the same
+        # Field struct. Only register an alias if it doesn't clash with an
+        # actual column and doesn't become ambiguous across two columns.
+        add_field_aliases!(fields)
+        fields
+      end
+
+      # Register alias keys on the fields hash so lookups like
+      # fields["rating"] work when the real column is "avg_rating".
+      # Rules:
+      #   avg_X / X_avg → X           (e.g., avg_rating → rating)
+      #   X_count       → count       (e.g., jobs_count → count)
+      #   total_X / X_total → total   (e.g., total_amount → total)
+      #   num_X         → X           (e.g., num_orders → orders)
+      # An alias is only added if (a) no real column has that name and
+      # (b) at most one existing field maps to that alias across the model.
+      def add_field_aliases!(fields)
+        alias_candidates = Hash.new { |h, k| h[k] = [] }
+        fields.each do |col, field|
+          synonyms_for(col).each { |syn| alias_candidates[syn] << field }
+        end
+        alias_candidates.each do |alias_key, matching_fields|
+          next if fields.key?(alias_key)             # don't shadow real column
+          next unless matching_fields.size == 1      # skip ambiguous aliases
+          fields[alias_key] = matching_fields.first
+        end
+      end
+
+      def synonyms_for(col)
+        syns = []
+        if col =~ /\Aavg_(.+)\z/        then syns << Regexp.last_match(1) end
+        if col =~ /\A(.+)_avg\z/        then syns << Regexp.last_match(1) end
+        if col =~ /\A(.+)_count\z/      then syns << "count"; syns << Regexp.last_match(1) end
+        if col =~ /\Atotal_(.+)\z/      then syns << "total"; syns << Regexp.last_match(1) end
+        if col =~ /\A(.+)_total\z/      then syns << "total" end
+        if col =~ /\Anum_(.+)\z/        then syns << Regexp.last_match(1) end
+        syns
+      end
+
+      def build_scopes(model)
+        scopes = {}
+        enum_generated = enum_generated_scope_names(model)
+
+        model.singleton_methods(false).each do |method_name|
+          next if enum_generated.include?(method_name)
+
+          begin
+            relation = model.send(method_name)
+            next unless relation.is_a?(ActiveRecord::Relation)
+            sql = relation.to_sql
+            where_match = sql.match(/WHERE\s+(.+?)(?:\s+ORDER\s+BY|\s+LIMIT|\s*$)/i)
+            where_clause = where_match ? where_match[1].strip : ""
+
+            scopes[method_name.to_s] = Grammar::Scope.new(
+              name: method_name.to_s,
+              where_clause: where_clause,
+              param_slots: []
+            )
+          rescue
+            # skip scopes that raise (e.g. require arguments or reference missing columns)
+          end
+        end
+
+        scopes
+      rescue => e
+        warn "[SqlChatbot] scope extraction for #{model}: #{e.message}"
+        {}
+      end
+
+      # Returns a Set of method names that AR auto-generates for enum columns
+      # (e.g. :active, :not_active, :banned, :not_banned, :statuses).
+      def enum_generated_scope_names(model)
+        generated = Set.new
+        model.defined_enums.each do |col, values|
+          values.keys.each do |v|
+            generated << v.to_sym
+            generated << :"not_#{v}"
+          end
+          # AR adds a pluralized class accessor (e.g. User.statuses)
+          generated << :"#{col}s"
+        end
+        generated
+      rescue
+        Set.new
+      end
+
+      def build_associations(model)
+        model.reflect_on_all_associations.each_with_object({}) do |refl, h|
+          begin
+            next unless refl.klass
+            target = refl.klass.name.underscore
+            h[refl.name.to_s] = Grammar::Association.new(
+              name: refl.name.to_s,
+              kind: refl.macro,
+              target_entity: target,
+              join_clause: join_clause_for(refl),
+              through_entity: refl.options[:through]&.to_s
+            )
+          rescue
+            # skip associations pointing to missing models
+          end
+        end
+      end
+
+      def join_clause_for(refl)
+        owner_table = refl.active_record.table_name
+        target_table = refl.klass.table_name
+        case refl.macro
+        when :belongs_to
+          "#{owner_table}.#{refl.foreign_key} = #{target_table}.#{refl.association_primary_key}"
+        when :has_many, :has_one
+          "#{owner_table}.#{refl.active_record.primary_key} = #{target_table}.#{refl.foreign_key}"
+        else
+          ""
+        end
+      end
+
+      def ranking_candidates_for(model)
+        all = model.columns_hash.select { |_, c| [:integer, :decimal, :float, :datetime].include?(c.type) }.keys
+        # V1.3-U: order so metric-shaped columns come first; PKs and FKs go
+        # to the end. Default rank for "biggest/smallest X" used to be `id`
+        # which produced answers like "Lowest id across Repositories: 1".
+        primary_key = model.primary_key.to_s
+        metric, poor = all.partition do |name|
+          !(name == primary_key || name == "id" || name.end_with?("_id"))
+        end
+        metric + poor
+      end
+
+      def map_type(ar_type)
+        case ar_type
+        when :integer, :bigint then :int
+        when :string, :text then :text
+        when :boolean then :bool
+        when :datetime, :date, :time then :timestamp
+        when :decimal, :float then :decimal
+        when :json, :jsonb then :jsonb
+        when :uuid then :uuid
+        else :text
+        end
+      end
+    end
+  end
+end

data/lib/sql_chatbot/services/route_introspector.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+module SqlChatbot
+  module Services
+    class RouteIntrospector
+      INTERNAL_CONTROLLERS = %w[
+        active_storage/ action_mailbox/ action_cable/ rails/
+        sql_chatbot/
+      ].freeze
+
+      def introspect
+        return [] unless defined?(Rails) && Rails.application
+
+        Rails.application.routes.routes.filter_map do |route|
+          next if route.internal
+          next if internal_controller?(route)
+
+          path = normalize_path(route)
+          next if path.nil? || path.empty?
+
+          {
+            path: path,
+            method: extract_method(route),
+            label: derive_label(route),
+            parentPath: derive_parent(path),
+          }
+        end
+      rescue => e
+        warn "[SqlChatbot] RouteIntrospector: Could not introspect routes: #{e.message}"
+        []
+      end
+
+      def format_route_list
+        routes = introspect
+        return "No application routes detected." if routes.empty?
+
+        lines = routes.select { |r| r[:method] == "GET" }.map do |r|
+          parent_note = r[:parentPath] ? " (under #{r[:parentPath]})" : ""
+          "- #{r[:path]} \u2014 #{r[:label]}#{parent_note}"
+        end
+
+        "## Available Application Pages\n#{lines.join("\n")}"
+      end
+
+      private
+
+      def internal_controller?(route)
+        controller = route.defaults[:controller].to_s
+        INTERNAL_CONTROLLERS.any? { |prefix| controller.start_with?(prefix) }
+      end
+
+      def normalize_path(route)
+        path = route.path.spec.to_s
+        path = path.sub("(.:format)", "")
+        path = path.sub(/\(\..+\)$/, "")
+        path = "/" if path.empty?
+        path
+      end
+
+      def extract_method(route)
+        verb = route.verb
+        case verb
+        when Regexp
+          match = verb.source.gsub(/[\^$]/, "")
+          match.empty? ? "GET" : match.split("|").first
+        when String
+          verb.empty? ? "GET" : verb
+        else
+          "GET"
+        end
+      end
+
+      def derive_label(route)
+        controller = route.defaults[:controller].to_s
+        action = route.defaults[:action].to_s
+        base_name = controller.split("/").last.to_s
+        humanized = base_name.split("_").map(&:capitalize).join(" ")
+        singular = singularize(humanized)
+
+        case action
+        when "index"
+          humanized
+        when "show"
+          "#{singular} Detail"
+        when "new", "create"
+          "New #{singular}"
+        when "edit", "update"
+          "Edit #{singular}"
+        when "destroy"
+          "Delete #{singular}"
+        else
+          "#{humanized} #{action.split("_").map(&:capitalize).join(" ")}"
+        end
+      end
+
+      def derive_parent(path)
+        segments = path.split("/").reject(&:empty?)
+        return nil if segments.length <= 1
+        parent_segments = segments[0...-1]
+        parent_segments.pop while parent_segments.last&.start_with?(":")
+        return nil if parent_segments.empty?
+        "/#{parent_segments.join("/")}"
+      end
+
+      def singularize(word)
+        if word.end_with?("ies")
+          word[0...-3] + "y"
+        elsif word.end_with?("ses")
+          word[0...-2]
+        elsif word.end_with?("s") && !word.end_with?("ss")
+          word[0...-1]
+        else
+          word
+        end
+      end
+    end
+  end
+end