docit 0.4.0 → 0.5.0

data/lib/docit/ai/system_insight_generator.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Docit
+  module Ai
+    class SystemInsightGenerator
+      # mode: :nodes  -> explain an arbitrary selection (diagram "AI Explain")
+      #       :section -> explain one resource and how its endpoints work together
+      def initialize(graph:, selected_node_ids: [], mode: :nodes)
+        @graph = graph
+        @selected_node_ids = selected_node_ids
+        @mode = mode
+      end
+
+      def generate
+        config = Configuration.load
+        Client.for(config).generate(prompt)
+      end
+
+      private
+
+      attr_reader :graph, :selected_node_ids, :mode
+
+      def prompt
+        mode == :section ? section_prompt : nodes_prompt
+      end
+
+      def nodes_prompt
+        <<~PROMPT
+          You are a senior engineer explaining a Rails system architecture to a developer. Keep your explanation extremely concise, professional, and clear. Avoid fluff, unnecessary details, or general tutorial information. Focus only on the provided components.
+
+          FORMAT YOUR RESPONSE EXACTLY LIKE THIS:
+
+          ## Overview
+          A 1-2 sentence plain-English summary of what this component/action does.
+
+          ## Data Flow
+          Show a simple, linear flow diagram using text and arrows (→). Keep it to 1 line if possible.
+          Example:
+            Client → GET /api/v1/users → UsersController#index → queries User model
+
+          ## Connections & Interactions
+          List only the direct, relevant relationships from the graph (max 3 bullets):
+          - **Component** (type): Action details/purpose.
+
+          Do not invent or assume anything outside of the provided graph. Keep the total response under 150 words.
+
+          ---
+
+          Selected node ids:
+          #{selected_node_ids.join("\n")}
+
+          Graph JSON:
+          #{JSON.pretty_generate(compact_graph)}
+        PROMPT
+      end
+
+      def section_prompt
+        <<~PROMPT
+          You are a senior engineer writing the introduction to a section of API documentation. The section covers ONE resource and all of its endpoints. Explain, for a developer new to this codebase, what the resource is for and how its endpoints work together as a workflow.
+
+          Use the documented summaries where available. Where an endpoint has no documentation, infer cautiously from its HTTP method and path, and do not fabricate behavior.
+
+          FORMAT YOUR RESPONSE EXACTLY LIKE THIS:
+
+          ## What this section does
+          1-2 sentences on the resource and its overall purpose.
+
+          ## How the endpoints work together
+          A short narrative (2-4 sentences) describing the typical flow across these endpoints — e.g. how a client lists, creates, then updates this resource. Reference endpoints by their HTTP method and path.
+
+          ## Notes
+          Up to 2 bullets on related models/services or anything a consumer must know. Omit this section if there is nothing concrete to say.
+
+          Do not invent endpoints, fields, or behavior not present in the graph. Keep the total response under 180 words.
+
+          ---
+
+          Endpoint node ids in this section:
+          #{selected_node_ids.join("\n")}
+
+          Graph JSON:
+          #{JSON.pretty_generate(compact_graph)}
+        PROMPT
+      end
+
+      def compact_graph
+        nodes = selected_nodes
+        node_ids = nodes.map { |node| node[:id] }
+
+        # Include edges where at least one end is in the selection
+        related_edges = graph[:edges].select do |edge|
+          node_ids.include?(edge[:source]) || node_ids.include?(edge[:target])
+        end
+
+        # Also include neighbor nodes (one hop away) for context
+        neighbor_ids = Set.new(node_ids)
+        related_edges.each do |edge|
+          neighbor_ids.add(edge[:source])
+          neighbor_ids.add(edge[:target])
+        end
+
+        neighbor_nodes = graph[:nodes].select { |node| neighbor_ids.include?(node[:id]) }
+
+        {
+          selected_nodes: nodes.map { |node| compact_hash(node, %i[id type label status file metadata]) },
+          context_nodes: (neighbor_nodes - nodes).map { |node| compact_hash(node, %i[id type label status]) },
+          edges: related_edges.map { |edge| compact_hash(edge, %i[source target type confidence evidence]) },
+          stats: graph[:stats]
+        }
+      end
+
+      def selected_nodes
+        return graph[:nodes] if selected_node_ids.empty?
+
+        graph[:nodes].select { |node| selected_node_ids.include?(node[:id]) }
+      end
+
+      def compact_hash(hash, keys)
+        keys.each_with_object({}) do |key, result|
+          result[key] = hash[key] if hash.key?(key)
+        end
+      end
+    end
+  end
+end

data/lib/docit/ai.rb

@@ -12,3 +12,4 @@ require_relative "ai/doc_writer"
 require_relative "ai/tag_injector"
 require_relative "ai/autodoc_runner"
 require_relative "ai/scaffold_generator"
+require_relative "ai/system_insight_generator"

data/lib/docit/configuration.rb

@@ -5,7 +5,8 @@ module Docit
   class Configuration
     SUPPORTED_UIS = %i[scalar swagger].freeze
 
-    attr_accessor :title, :version, :description, :base_url
+    attr_accessor :title, :version, :description, :base_url,
+                  :system_graph_enabled, :system_graph_excluded_paths
     attr_reader :default_ui
 
     def initialize
@@ -13,6 +14,8 @@ module Docit
       @version = "1.0.0"
       @description = "Welcome to the API documentation. Browse the endpoints below to get started."
       @base_url = "/"
+      @system_graph_enabled = true
+      @system_graph_excluded_paths = []
       @default_ui = :scalar
       @security_schemes = {}
       @tags = []

data/lib/docit/system_graph/edge.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Docit
+  module SystemGraph
+    class Edge
+      attr_reader :id, :source, :target, :type, :confidence, :evidence
+
+      def initialize(id:, source:, target:, type:, confidence:, evidence:)
+        @id = id.to_s
+        @source = source.to_s
+        @target = target.to_s
+        @type = type.to_s
+        @confidence = confidence.to_s
+        @evidence = evidence.to_s
+      end
+
+      def to_h
+        {
+          id: id,
+          source: source,
+          target: target,
+          type: type,
+          confidence: confidence,
+          evidence: evidence
+        }
+      end
+    end
+  end
+end

data/lib/docit/system_graph/generator.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Docit
+  module SystemGraph
+    class Generator
+      def self.generate
+        new.generate
+      end
+
+      def generate
+        RailsAnalyzer.new.analyze.to_h
+      end
+    end
+  end
+end

data/lib/docit/system_graph/graph.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require "time"
+
+module Docit
+  module SystemGraph
+    class Graph
+      VERSION = "1.0"
+
+      attr_reader :nodes, :edges, :framework
+
+      def initialize(framework: "rails")
+        @framework = framework
+        @nodes = {}
+        @edges = {}
+      end
+
+      def add_node(node)
+        nodes[node.id] ||= node
+      end
+
+      def add_edge(edge)
+        return if edge.source == edge.target
+        return unless nodes.key?(edge.source) && nodes.key?(edge.target)
+
+        edges[edge.id] ||= edge
+      end
+
+      def to_h
+        node_values = nodes.values
+        edge_values = edges.values
+
+        {
+          version: VERSION,
+          generated_at: Time.now.utc.iso8601,
+          framework: framework,
+          nodes: node_values.map(&:to_h),
+          edges: edge_values.map(&:to_h),
+          stats: stats(node_values, edge_values)
+        }
+      end
+
+      private
+
+      def stats(node_values, edge_values)
+        {
+          nodes: node_values.length,
+          edges: edge_values.length,
+          node_types: node_values.group_by(&:type).transform_values(&:length),
+          edge_types: edge_values.group_by(&:type).transform_values(&:length)
+        }
+      end
+    end
+  end
+end

data/lib/docit/system_graph/node.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Docit
+  module SystemGraph
+    class Node
+      attr_reader :id, :type, :label, :metadata, :file, :line, :status
+
+      def initialize(id:, type:, label:, metadata: {}, file: nil, line: nil, status: nil)
+        @id = id.to_s
+        @type = type.to_s
+        @label = label.to_s
+        @metadata = metadata
+        @file = file
+        @line = line
+        @status = status
+      end
+
+      def to_h
+        {
+          id: id,
+          type: type,
+          label: label,
+          metadata: metadata
+        }.tap do |hash|
+          hash[:file] = file if file
+          hash[:line] = line if line
+          hash[:status] = status if status
+        end
+      end
+    end
+  end
+end

data/lib/docit/system_graph/rails_analyzer.rb

@@ -0,0 +1,251 @@
+# frozen_string_literal: true
+
+module Docit
+  module SystemGraph
+    class RailsAnalyzer
+      VALID_METHODS = %w[get post put patch delete].freeze
+      SKIP_PREFIXES = %w[docit/ rails/ active_storage/ action_mailbox/].freeze
+
+      def initialize(graph: Graph.new, scanner: SourceScanner.new(root: Rails.root))
+        @graph = graph
+        @scanner = scanner
+      end
+
+      def analyze
+        add_routes_and_actions
+        add_schemas
+        add_models
+        add_source_nodes
+        graph
+      end
+
+      private
+
+      attr_reader :graph, :scanner
+
+      def add_routes_and_actions
+        route_infos.each do |info|
+          controller_id = node_id("controller", info[:controller])
+          action_id = "#{controller_id}##{info[:action]}"
+          route_id = "route:#{info[:method]}:#{info[:path]}"
+          operation = Registry.find(controller: info[:controller], action: info[:action])
+
+          graph.add_node(Node.new(
+                           id: controller_id,
+                           type: "controller",
+                           label: info[:controller],
+                           file: controller_file(info[:controller]),
+                           metadata: { controller_path: info[:controller_path] }
+                         ))
+          graph.add_node(Node.new(
+                           id: action_id,
+                           type: "action",
+                           label: "#{info[:controller]}##{info[:action]}",
+                           file: controller_file(info[:controller]),
+                           status: operation ? "documented" : "undocumented",
+                           metadata: { action: info[:action], http_method: info[:method], path: info[:path] }
+                         ))
+          graph.add_node(Node.new(
+                           id: route_id,
+                           type: "route",
+                           label: "#{info[:method].upcase} #{info[:path]}",
+                           metadata: { method: info[:method], path: info[:path] },
+                           status: operation ? "documented" : "undocumented"
+                         ))
+
+          graph.add_edge(edge(route_id, action_id, "routes_to", "high", "Rails route table"))
+          graph.add_edge(edge(controller_id, action_id, "contains", "high", "Rails controller action"))
+          add_doc_node(info, operation, action_id) if operation
+        end
+      end
+
+      def add_doc_node(info, operation, action_id)
+        doc_id = "doc:#{info[:controller].underscore}:#{info[:action]}"
+
+        request_body_info = nil
+        if operation._request_body
+          request_body_info = {
+            required: operation._request_body.required,
+            content_type: operation._request_body.content_type,
+            schema_ref: operation._request_body.schema_ref,
+            properties: operation._request_body.properties
+          }
+        end
+
+        responses_info = operation._responses.map do |res|
+          {
+            status: res.status,
+            description: res.description,
+            schema_ref: res.schema_ref,
+            properties: res.properties,
+            examples: res.examples
+          }
+        end
+
+        parameters_info = operation._parameters.params.map do |param|
+          {
+            name: param[:name],
+            location: param[:in],
+            type: param[:schema][:type],
+            required: param[:required],
+            description: param[:description]
+          }
+        end
+
+        graph.add_node(Node.new(
+                         id: doc_id,
+                         type: "doc",
+                         label: operation._summary || "#{info[:controller]}##{info[:action]}",
+                         metadata: {
+                           controller: info[:controller],
+                           action: info[:action],
+                           description: operation._description,
+                           tags: operation._tags,
+                           request_body: request_body_info,
+                           responses: responses_info,
+                           parameters: parameters_info
+                         },
+                         status: "documented"
+                       ))
+        graph.add_edge(edge(doc_id, action_id, "documents", "high", "Docit registry"))
+      end
+
+      def add_schemas
+        Docit.schemas.each_key do |name|
+          graph.add_node(Node.new(
+                           id: node_id("schema", name),
+                           type: "schema",
+                           label: name.to_s,
+                           metadata: { properties: Docit.schemas[name].properties.map { |prop| prop[:name].to_s } }
+                         ))
+        end
+      end
+
+      def add_models
+        return unless defined?(ActiveRecord::Base)
+
+        ActiveRecord::Base.descendants.each do |model|
+          next if model.name.nil?
+
+          model_id = node_id("model", model.name)
+          graph.add_node(Node.new(
+                           id: model_id,
+                           type: "model",
+                           label: model.name,
+                           file: model_file(model.name),
+                           metadata: { table_name: table_name(model) }
+                         ))
+          add_model_associations(model, model_id)
+        end
+      end
+
+      def add_model_associations(model, model_id)
+        return unless model.respond_to?(:reflect_on_all_associations)
+
+        model.reflect_on_all_associations.each do |association|
+          target = association.class_name
+          target_id = node_id("model", target)
+          next unless graph.nodes.key?(target_id)
+
+          graph.add_edge(edge(model_id, target_id, "association", "high", "ActiveRecord reflection: #{association.name}"))
+        end
+      end
+
+      def add_source_nodes
+        source_nodes = scanner.source_nodes
+        source_nodes.each { |node| graph.add_node(node) }
+
+        labels = graph.nodes.values.select { |node| %w[model service job mailer].include?(node.type) }.map(&:label)
+        graph.nodes.values.select { |node| %w[controller action service job mailer].include?(node.type) }.each do |source|
+          scanner.references_for(source.file, labels).each do |label|
+            target = graph.nodes.values.find { |node| node.label == label }
+            next unless target
+
+            graph.add_edge(edge(source.id, target.id, edge_type_for(target), "medium", "Constant reference in #{source.file}"))
+          end
+        end
+      end
+
+      def route_infos
+        return [] if defined?(Rails).nil? || Rails.application.routes.nil?
+
+        Rails.application.routes.routes.filter_map do |route|
+          controller_path = route.defaults[:controller]
+          action = route.defaults[:action]
+          next if controller_path.nil? || action.nil?
+          next if skip_route?(controller_path)
+
+          method = extract_verb(route)
+          next if VALID_METHODS.exclude?(method)
+
+          controller = "#{controller_path}_controller".camelize
+          next if excluded_path?(controller_file(controller))
+
+          {
+            controller: controller,
+            controller_path: controller_path,
+            action: action.to_s,
+            method: method,
+            path: normalize_path(route.path.spec.to_s)
+          }
+        end.uniq
+      end
+
+      def edge(source, target, type, confidence, evidence)
+        Edge.new(
+          id: "#{type}:#{source}->#{target}",
+          source: source,
+          target: target,
+          type: type,
+          confidence: confidence,
+          evidence: evidence
+        )
+      end
+
+      def node_id(type, label)
+        "#{type}:#{label.to_s.underscore.tr('/', ':')}"
+      end
+
+      def controller_file(controller)
+        "app/controllers/#{controller.underscore}.rb"
+      end
+
+      def model_file(model)
+        "app/models/#{model.underscore}.rb"
+      end
+
+      def edge_type_for(target)
+        target.type == "model" ? "uses_model" : "calls"
+      end
+
+      def table_name(model)
+        model.table_name if model.respond_to?(:table_name)
+      rescue ActiveRecord::StatementInvalid
+        nil
+      end
+
+      def skip_route?(controller_path)
+        SKIP_PREFIXES.any? { |prefix| controller_path.start_with?(prefix) }
+      end
+
+      def excluded_path?(path)
+        Docit.configuration.system_graph_excluded_paths.any? do |excluded|
+          path.start_with?(excluded.to_s)
+        end
+      end
+
+      def extract_verb(route)
+        verb = route.verb
+        verb = verb.source if verb.is_a?(Regexp)
+        verb.to_s.downcase.gsub(/[^a-z]/, "")
+      end
+
+      def normalize_path(path)
+        path
+          .gsub("(.:format)", "")
+          .gsub(/\(\.?:(\w+)\)/, '{\1}')
+          .gsub(/:(\w+)/, '{\1}')
+      end
+    end
+  end
+end

data/lib/docit/system_graph/source_scanner.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module Docit
+  module SystemGraph
+    class SourceScanner
+      SOURCE_TYPES = {
+        "app/services" => "service",
+        "app/jobs" => "job",
+        "app/mailers" => "mailer"
+      }.freeze
+
+      def initialize(root:, excluded_paths: Docit.configuration.system_graph_excluded_paths)
+        @root = root
+        @excluded_paths = excluded_paths.map(&:to_s)
+      end
+
+      def source_nodes
+        SOURCE_TYPES.each_with_object([]) do |(dir, type), nodes|
+          scan_dir(dir).each do |path|
+            relative = relative_path(path)
+            label = constant_name(relative, dir)
+            nodes << Node.new(
+              id: node_id(type, label),
+              type: type,
+              label: label,
+              file: relative,
+              metadata: { path: relative }
+            )
+          end
+        end
+      end
+
+      def references_for(path, candidates)
+        return [] unless path && File.exist?(full_path(path))
+
+        content = File.read(full_path(path))
+        candidates.select do |candidate|
+          content.match?(/\b#{Regexp.escape(candidate)}\b/)
+        end
+      end
+
+      private
+
+      attr_reader :root
+      attr_reader :excluded_paths
+
+      def scan_dir(dir)
+        full_dir = root.join(dir)
+        return [] unless Dir.exist?(full_dir)
+
+        Dir.glob(full_dir.join("**", "*.rb")).sort.reject do |path|
+          excluded?(relative_path(path))
+        end
+      end
+
+      def relative_path(path)
+        path.to_s.sub("#{root}/", "")
+      end
+
+      def constant_name(relative, dir)
+        relative.delete_prefix("#{dir}/").delete_suffix(".rb").camelize
+      end
+
+      def node_id(type, label)
+        "#{type}:#{label.underscore.tr('/', ':')}"
+      end
+
+      def full_path(path)
+        root.join(path)
+      end
+
+      def excluded?(path)
+        excluded_paths.any? { |excluded| path.start_with?(excluded) }
+      end
+    end
+  end
+end

data/lib/docit/system_graph.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require_relative "system_graph/node"
+require_relative "system_graph/edge"
+require_relative "system_graph/graph"
+require_relative "system_graph/source_scanner"
+require_relative "system_graph/rails_analyzer"
+require_relative "system_graph/generator"