reqcord 0.1.0 → 0.1.1

data/lib/reqcord/route_collector.rb

@@ -0,0 +1,242 @@
+# frozen_string_literal: true
+
+module Reqcord
+  # The documented surface is the route table, not the captured traffic: every
+  # matching route becomes an endpoint, and captures are attached to it. Routes
+  # no test exercised stay in the dataset as documentation gaps.
+  #
+  # Nothing is dropped silently: a route either becomes a Route, or is counted
+  # in `skipped` with the reason (redirect, Rack mount, Rails internal).
+  class RouteCollector
+    # A route that answers any verb (`via: :all`); the captured verb decides
+    # what gets documented.
+    ANY = "ANY"
+
+    Route = Struct.new(
+      :name,
+      :method,
+      :path,
+      :controller,
+      :action,
+      :resource,
+      :api_version,
+      :rails_route,
+      :mount_prefix,
+      keyword_init: true
+    ) do
+      def endpoint(method: self.method)
+        Endpoint.new(
+          method: method,
+          path: path,
+          controller: controller,
+          action: action,
+          resource: resource,
+          api_version: api_version,
+          route_name: name
+        )
+      end
+
+      def any_verb?
+        method == ANY
+      end
+
+      def matches?(request_method, request_path)
+        return false unless any_verb? || method == request_method.to_s.upcase
+
+        relative = relative_path(request_path)
+        return false if relative.nil?
+
+        !!rails_route.path.match(relative)
+      end
+
+      private
+
+      # An engine's pattern knows nothing about where it was mounted.
+      def relative_path(request_path)
+        return request_path if mount_prefix.nil? || mount_prefix.empty?
+        return nil unless request_path.start_with?(mount_prefix)
+
+        rest = request_path.delete_prefix(mount_prefix)
+        return nil unless rest.empty? || rest.start_with?("/")
+
+        rest.empty? ? "/" : rest
+      end
+    end
+
+    def self.call(
+      resources: [],
+      version: nil,
+      prefix: nil,
+      route_set: nil
+    )
+      new(
+        resources: resources,
+        version: version,
+        prefix: prefix,
+        route_set: route_set
+      ).call
+    end
+
+    # Routes that were seen but cannot be documented, by reason.
+    attr_reader :skipped
+
+    def initialize(resources:, version:, prefix:, route_set: nil)
+      @resources = Array(resources).map(&:to_s)
+      @version = version&.to_s
+      @prefix = prefix
+      @route_set = route_set
+      @skipped = Hash.new(0)
+    end
+
+    def call
+      @skipped = Hash.new(0)
+
+      collect(route_set.routes, mount_prefix: nil)
+    end
+
+    def skipped_count
+      skipped.values.sum
+    end
+
+    private
+
+    attr_reader :resources, :version, :prefix
+
+    def route_set
+      @route_set ||= Rails.application.routes
+    end
+
+    def collect(rails_routes, mount_prefix:)
+      rails_routes.flat_map { |rails_route| build_routes(rails_route, mount_prefix) }
+    end
+
+    def build_routes(rails_route, mount_prefix)
+      return [] if rails_route.internal
+
+      app = rails_route.app
+      spec = normalize_path(rails_route.path.spec.to_s)
+
+      # A mounted engine's routes live in its own table, relative to the mount.
+      if engine?(app)
+        return collect(app.rack_app.routes.routes, mount_prefix: join(mount_prefix, spec))
+      end
+
+      controller = rails_route.defaults[:controller]&.to_s
+      action = rails_route.defaults[:action]&.to_s
+
+      if controller.nil? || action.nil?
+        @skipped[skip_reason(app)] += 1
+        return []
+      end
+
+      return [] if internal?(controller)
+
+      path = join(mount_prefix, spec)
+
+      return [] unless matches_prefix?(path)
+      return [] unless matches_version?(controller, path)
+
+      resource = controller.split("/").last
+
+      return [] unless matches_resource?(resource, controller)
+
+      normalize_methods(rails_route.verb).map do |method|
+        Route.new(
+          name: rails_route.name,
+          method: method,
+          path: path,
+          controller: controller,
+          action: action,
+          resource: resource,
+          api_version: detect_version(controller, path),
+          rails_route: rails_route,
+          mount_prefix: mount_prefix
+        )
+      end
+    end
+
+    # A mounted app that carries its own route table (a Rails::Engine, or
+    # anything shaped like one). Checked by shape rather than by class: the
+    # `Rails::Engine` constant need not be loaded, and a Sinatra app — whose
+    # `routes` is a plain Hash — is a Rack mount, not an engine.
+    def engine?(app)
+      return false unless app.respond_to?(:rack_app)
+
+      rack_app = app.rack_app
+
+      rack_app.respond_to?(:routes) && rack_app.routes.respond_to?(:routes)
+    end
+
+    def skip_reason(app)
+      rack_app = app.respond_to?(:rack_app) ? app.rack_app : app
+
+      if defined?(ActionDispatch::Routing::Redirect) && rack_app.is_a?(ActionDispatch::Routing::Redirect)
+        "redirect"
+      else
+        "mount"
+      end
+    end
+
+    # Rails 8 exposes the verb as a plain string: "GET", "GET|POST" for
+    # `via: [:get, :post]`, "" for `via: :all`. Older versions used a regexp
+    # whose source carried anchors.
+    def normalize_methods(verb)
+      values = verb.to_s
+                   .gsub(/[$^]/, "")
+                   .split("|")
+                   .map(&:strip)
+                   .reject(&:empty?)
+                   .map(&:upcase)
+
+      values.empty? ? [ANY] : values
+    end
+
+    def normalize_path(path)
+      path.sub(/\(\.:format\)\z/, "")
+    end
+
+    def join(mount_prefix, path)
+      return path if mount_prefix.nil? || mount_prefix.empty?
+      return mount_prefix if path == "/"
+
+      "#{mount_prefix}#{path}"
+    end
+
+    def internal?(controller)
+      controller.start_with?(
+        "rails/",
+        "active_storage/",
+        "action_mailbox/",
+        "turbo/"
+      )
+    end
+
+    def matches_prefix?(path)
+      return true if prefix.nil? || prefix.empty?
+
+      path.start_with?(prefix)
+    end
+
+    # `RESOURCE=customers`, `RESOURCE=cart` (a singular resource is served by
+    # `carts`) and `RESOURCE=api/v2/customers` all name the same thing.
+    def matches_resource?(resource, controller)
+      return true if resources.empty?
+
+      candidates = [resource, controller, Support.singularize(resource)]
+
+      resources.any? { |wanted| candidates.include?(wanted) }
+    end
+
+    def matches_version?(controller, path)
+      return true if version.nil? || version.empty?
+
+      controller.split("/").include?(version) ||
+        path.split("/").include?(version)
+    end
+
+    def detect_version(controller, path)
+      controller.split("/").find { |segment| segment.match?(/\Av\d+\z/) } ||
+        path.split("/").find { |segment| segment.match?(/\Av\d+\z/) }
+    end
+  end
+end

data/lib/reqcord/sanitizers/sanitizer.rb

@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+module Reqcord
+  module Sanitizers
+    # Works on a raw captured exchange, before anything reaches the dataset:
+    # credentials must never be written to a generated file.
+    class Sanitizer
+      DEFAULT_SECRET_HEADERS = %w[
+        Authorization
+        Proxy-Authorization
+        Cookie
+        Set-Cookie
+        X-Api-Key
+        X-Auth-Token
+        X-Csrf-Token
+      ].freeze
+
+      SCHEME_PATTERN = /\A(Bearer|Token|Basic)\s+/i
+
+      def self.call(exchange, configuration:)
+        new(
+          exchange,
+          configuration: configuration
+        ).call
+      end
+
+      def initialize(exchange, configuration:)
+        @exchange = deep_dup(exchange)
+        @configuration = configuration
+      end
+
+      def call
+        sanitize_headers!(
+          exchange.dig("request", "headers")
+        )
+
+        sanitize_headers!(
+          exchange.dig("response", "headers"),
+          response: true
+        )
+
+        sanitize_body!("request")
+        sanitize_body!("response")
+
+        exchange
+      end
+
+      private
+
+      attr_reader :exchange, :configuration
+
+      def sanitize_headers!(headers, response: false)
+        return unless headers.is_a?(Hash)
+
+        replacements = configuration.sanitized_headers
+
+        headers.keys.each do |key|
+          if configuration.noisy_header?(key)
+            headers.delete(key)
+            next
+          end
+
+          if blank?(headers[key])
+            headers.delete(key)
+            next
+          end
+
+          replacement = find_replacement(replacements, key)
+
+          if replacement
+            headers[key] = replacement
+            next
+          end
+
+          headers[key] = redact(key, headers[key]) if secret_header?(key)
+        end
+      end
+
+      def sanitize_body!(side)
+        body = exchange.dig(side, "body")
+
+        return unless body.is_a?(Hash) || body.is_a?(Array)
+
+        exchange[side]["body"] = sanitize_value(body)
+      end
+
+      def sanitize_value(value)
+        case value
+        when Hash
+          value.each_with_object({}) do |(key, nested), result|
+            replacement = body_replacement(key)
+
+            result[key] = replacement || sanitize_value(nested)
+          end
+        when Array
+          value.map { |item| sanitize_value(item) }
+        else
+          value
+        end
+      end
+
+      def body_replacement(key)
+        configuration.sanitized_body_keys.find do |name, _value|
+          name.to_s.casecmp?(key.to_s)
+        end&.last
+      end
+
+      def find_replacement(replacements, key)
+        pair = replacements.find do |header, _value|
+          header.to_s.casecmp?(key.to_s)
+        end
+
+        pair&.last
+      end
+
+      # The scheme is kept so the generated cURL stays copy-pasteable.
+      def redact(key, value)
+        placeholder = "{{#{key.to_s.tr('-', '_').downcase}}}"
+
+        match = value.to_s.match(SCHEME_PATTERN)
+
+        match ? "#{match[1]} #{placeholder}" : placeholder
+      end
+
+      def secret_header?(key)
+        DEFAULT_SECRET_HEADERS.any? do |header|
+          header.casecmp?(key.to_s)
+        end
+      end
+
+      def blank?(value)
+        value.nil? || value.to_s.empty?
+      end
+
+      def deep_dup(value)
+        Marshal.load(Marshal.dump(value))
+      end
+    end
+  end
+end

data/lib/reqcord/schema.rb

@@ -0,0 +1,187 @@
+# frozen_string_literal: true
+
+module Reqcord
+  # Describes what an endpoint accepts, inferred from the requests that
+  # actually worked. One test sends status "active", another "passive", a third
+  # sends "inactive" and gets a 422: the documentation should say the field
+  # takes "active" | "passive", not list three requests.
+  class Schema
+    # Beyond this many distinct values a field is an open set, not a choice.
+    ENUM_LIMIT = 6
+
+    Field = Struct.new(:path, :types, :values, :present_count, :total_count, :repetition, keyword_init: true) do
+      def type
+        types.to_a.sort.join(" | ")
+      end
+
+      # A field missing from a working request cannot be required.
+      def required?
+        present_count == total_count
+      end
+
+      # Telling a closed set from free text: either a value came back in more
+      # than one request, or every value reads like a token ("active"), not
+      # like content ("Ada Lovelace", "ada@example.com").
+      def enum?
+        return false unless values.size.between?(2, ENUM_LIMIT)
+        return false unless values.all? { |value| scalar?(value) }
+        return false if identifier?
+
+        # Repetition only counts among the requests that carried the field:
+        # two requests, two different values, is not a set.
+        return true if repetition && values.size < present_count
+
+        values.all? { |value| token?(value) }
+      end
+
+      def example
+        values.first
+      end
+
+      # Test order is not stable, so a listed set must be ordered by something
+      # that is.
+      def listed_values
+        values.sort_by { |value| [value.to_s, value.class.name] }
+      end
+
+      def to_h
+        {
+          path: path,
+          type: type,
+          required: required?,
+          values: listed_values
+        }
+      end
+
+      private
+
+      def scalar?(value)
+        value.is_a?(String) || value.is_a?(Numeric) || value == true || value == false
+      end
+
+      # Every record has a different one, so listing the ones a test happened
+      # to use would read as if those were the only allowed values.
+      def identifier?
+        path.to_s.split(/[.\[\]]/).last.to_s.match?(/\A(id|uuid|.*_id|.*_uuid|.*_key|token|slug)\z/i)
+      end
+
+      # A word, not content: "active", "pending_review", "USD". Digits and
+      # dashes usually mean a generated value ("e-00056197", "safari-859f04").
+      def token?(value)
+        case value
+        when true, false then true
+        when String then value.match?(/\A[a-z][a-z_]*\z/) || value.match?(/\A[A-Z_]{2,}\z/)
+        else false
+        end
+      end
+    end
+
+    # `payloads` are request bodies, query hashes or path parameter hashes from
+    # successful captures. With `repetition: true` a value seen in more than
+    # one payload marks a closed set — right for what tests *send*, wrong for
+    # what an application *returns*, where fixtures repeat by nature; response
+    # bodies are inferred with `repetition: false`, so only token-like values
+    # ("open", "done") are listed as a set.
+    def self.infer(payloads, repetition: true)
+      new(payloads, repetition: repetition).call
+    end
+
+    # Empty payloads are kept: a request that was accepted without any
+    # parameters is the proof that every parameter is optional.
+    def initialize(payloads, repetition: true)
+      @payloads = Array(payloads)
+      @repetition = repetition
+    end
+
+    def call
+      return self.class.empty if @payloads.empty?
+
+      fields = {}
+
+      @payloads.each do |payload|
+        flatten(payload).each do |path, value|
+          field = fields[path] ||= Field.new(
+            path: path,
+            types: [],
+            values: [],
+            present_count: 0,
+            total_count: @payloads.size,
+            repetition: @repetition
+          )
+
+          field.types |= [type_of(value)]
+          field.values |= [value] unless value.nil? || value.is_a?(Hash) || value.is_a?(Array)
+          field.present_count += 1
+        end
+      end
+
+      Result.new(fields.values)
+    end
+
+    def self.empty
+      Result.new([])
+    end
+
+    # The inferred description of one payload shape.
+    class Result
+      include Enumerable
+
+      attr_reader :fields
+
+      def initialize(fields)
+        @fields = fields
+      end
+
+      def each(&block)
+        fields.each(&block)
+      end
+
+      def empty?
+        fields.empty?
+      end
+
+      def size
+        fields.size
+      end
+
+      def [](path)
+        fields.find { |field| field.path == path }
+      end
+
+      def to_a
+        fields.map(&:to_h)
+      end
+    end
+
+    private
+
+    # { "customer" => { "tags" => ["a"] } } -> { "customer.tags[]" => "a" }
+    def flatten(value, prefix = nil, result = {})
+      case value
+      when Hash
+        value.each { |key, nested| flatten(nested, prefix ? "#{prefix}.#{key}" : key.to_s, result) }
+      when Array
+        # An empty array still tells the reader the field is a list.
+        result[prefix] = [] if value.empty? && prefix
+        value.each { |item| flatten(item, "#{prefix}[]", result) }
+      else
+        result[prefix] = value if prefix
+      end
+
+      result
+    end
+
+    def type_of(value)
+      case value
+      when String then "string"
+      when Integer then "integer"
+      when Float then "number"
+      when true, false then "boolean"
+      when nil then "null"
+      when Array then "array"
+      when Hash then "object"
+      else value.class.name.downcase
+      end
+    end
+  end
+end

data/lib/reqcord/support.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Reqcord
+  # Minimal inflection helpers. ActiveSupport is used when it is available so
+  # that generated titles match the host application's inflections.
+  module Support
+    module_function
+
+    def underscore(value)
+      string = value.to_s.dup
+      return ActiveSupport::Inflector.underscore(string) if defined?(ActiveSupport::Inflector)
+
+      string.gsub("::", "/")
+            .gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
+            .gsub(/([a-z\d])([A-Z])/, '\1_\2')
+            .tr("-", "_")
+            .downcase
+    end
+
+    def humanize(value)
+      underscore(value).tr("_", " ").strip
+    end
+
+    def titleize(value)
+      humanize(value).split(/\s+/).map { |word| word.empty? ? word : word[0].upcase + word[1..] }.join(" ")
+    end
+
+    def singularize(value)
+      string = value.to_s
+      return ActiveSupport::Inflector.singularize(string) if defined?(ActiveSupport::Inflector)
+
+      case string
+      when /ies\z/i then string.sub(/ies\z/i, "y")
+      when /(ss|sh|ch|x|z)es\z/i then string.sub(/es\z/i, "")
+      when /ss\z/i then string
+      when /s\z/i then string.sub(/s\z/i, "")
+      else string
+      end
+    end
+
+    def pluralize(value)
+      string = value.to_s
+      return ActiveSupport::Inflector.pluralize(string) if defined?(ActiveSupport::Inflector)
+
+      case string
+      when /(ss|sh|ch|x|z)\z/i then "#{string}es"
+      when /[^aeiou]y\z/i then string.sub(/y\z/i, "ies")
+      # Already plural, as resource names usually are.
+      when /s\z/i then string
+      else "#{string}s"
+      end
+    end
+
+    def parameterize(value)
+      underscore(value).gsub(/[^a-z0-9]+/, "-").gsub(/\A-+|-+\z/, "")
+    end
+  end
+end

data/lib/reqcord/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Reqcord
+  VERSION = "0.1.1"
+end

data/lib/reqcord.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+require "json"
+require "yaml"
+require "time"
+require "uri"
+require "rack/utils"
+require "fileutils"
+require "pathname"
+
+require_relative "reqcord/version"
+require_relative "reqcord/errors"
+require_relative "reqcord/support"
+require_relative "reqcord/configuration"
+
+require_relative "reqcord/request_example"
+require_relative "reqcord/response_example"
+require_relative "reqcord/schema"
+require_relative "reqcord/endpoint"
+require_relative "reqcord/dataset"
+
+require_relative "reqcord/route_collector"
+
+require_relative "reqcord/capture"
+require_relative "reqcord/capture/test_context"
+require_relative "reqcord/capture/collector"
+
+require_relative "reqcord/sanitizers/sanitizer"
+require_relative "reqcord/renderers/payload"
+require_relative "reqcord/renderers/curl"
+require_relative "reqcord/exporters"
+require_relative "reqcord/exporters/curl"
+require_relative "reqcord/exporters/markdown"
+require_relative "reqcord/exporters/postman"
+require_relative "reqcord/generator"
+
+require_relative "reqcord/railtie" if defined?(Rails::Railtie)
+
+module Reqcord
+  class << self
+    attr_writer :root
+
+    def root
+      return @root if @root
+
+      if defined?(Rails) &&
+         Rails.respond_to?(:root) &&
+         Rails.root
+
+        Rails.root
+      else
+        Pathname(Dir.pwd)
+      end
+    end
+
+    def configuration
+      @configuration ||=
+        Configuration.load(
+          root: root
+        )
+    end
+
+    def reload_configuration!
+      @configuration =
+        Configuration.load(
+          root: root
+        )
+    end
+
+    def log(message)
+      $stdout.puts("[reqcord] #{message}")
+    end
+
+    def warn(message)
+      $stderr.puts("[reqcord] #{message}")
+    end
+  end
+end