full_metal_body 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 5913971dd94fbbab8523c89193d31ee59f62ea723ebdfcafd0b93d935c861a1a
+  data.tar.gz: f6ad58265ae64f3800256909269a7486b2fa3f37496e48cab82700a71c6e6e00
+SHA512:
+  metadata.gz: 4f12a070d08fe7fa391893103a6089ab60b64eb3b123c3e4dc6a1cb78d4dcef64aa705720f7e9ecc1deb17c26c33d395694d4128764a27f51d6f6ba29c6b577d
+  data.tar.gz: 8353c73e9630609a6b4c7a5200ed9133044c264b7e912ccd0c33014dbf36d11de00761c9fc2ec9672a65cee903af623b3a6559ba95ce4188cc0a075555b65f77

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright 2022 patorash
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,227 @@
+# FullMetalBody
+
+[![Test](https://github.com/hayashima/full_metal_body/actions/workflows/test.yml/badge.svg)](https://github.com/hayashima/full_metal_body/actions/workflows/test.yml)
+
+FullMetalBody is a Rails Plugin for input validation in the before_action stage.
+
+If you write a whitelist of parameters in a YAML file, only allowed keys and values will be passed through.
+
+However, in other cases, for example, if it detects a string with control characters or a large number of strings aiming for overflow, it will immediately return `400 Bad Request`.
+
+### Format of whitelist
+
+```yaml
+---
+controller_name:
+  action_name:
+    parameter_name:
+      type: (string|number|date|boolean)
+      options:
+        validator_option1: (value)
+        validator_option2: (value)
+    array_parameters:
+      type: array
+      properties:
+        parameter_name:
+          type: (string|number|date|boolean)
+          options:
+            validator_option1: (value)
+            validator_option2: (value)
+```
+
+### Sample of whitelist
+
+For example, suppose you have created a Scaffold for the `Article` model as shown below.
+
+```bash
+bin/rails g scaffold Article title:string content:text
+```
+
+The whitelist for it is as follows.
+
+```yaml
+---
+articles:
+  index:
+    p:
+      type: number
+  show:
+    id:
+      type: number
+  create:
+    article:
+      title:
+        type: string
+      content:
+        type: string
+        options:
+          max_length: 4096
+  edit:
+    id:
+      type: number
+  update:
+    id:
+      type: number
+    article:
+      title:
+        type: string
+      content:
+        type: string
+        options:
+          max_length: 4096
+  destroy:
+    id:
+      type: number
+```
+
+## Table of contents
+
+* [FullMetalBody](#fullmetalbody)
+    * [Table of contents](#tableofcontents)
+    * [Installation](#installation)
+    * [Usage](#usage)
+        * [Migration](#migration)
+        * [Modify ApplicationController](#modifyapplicationcontroller)
+        * [Creating a whitelist template](#creatingawhitelisttemplate)
+        * [If you want to allow all parameters](#ifyouwanttoallowallparameters)
+    * [Development](#development)
+        * [Preparation](#preparation)
+        * [Test](#test)
+    * [Contributing](#contributing)
+    * [License](#license)
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem "full_metal_body"
+```
+
+And then execute:
+
+```bash
+$ bundle
+```
+
+## Usage
+
+### Migration
+
+Create a migration file to store controllers, actions, and parameter keys that do not exist in the whitelist in the database.
+
+```bash
+bin/rails g full_metal_body:install
+```
+
+And then execute:
+
+```bash
+bin/rails db:migrate
+```
+
+The `blocked_actions` table and the `blocked_keys` table will be created in the database.
+
+### Modify ApplicationController
+
+Include `FullMetalBody::InputValidationAction` in ApplicationController.
+
+```ruby
+class ApplicationController < ActionController::Base
+  include FullMetalBody::InputValidationAction
+end
+```
+
+### Creating a whitelist template
+
+You can write your own whitelist, but that's a lot of work.
+
+Therefore, FullMetalBody creates a template whitelist in `tmp/whitelist/**/*.yml` by accessing each action in development mode, and then raises an exception.
+At that time, the data will be registered in the blocked_actions table and the blocked_keys table.
+Also, the contents will be output to the log.
+
+For example, if you access `GET article_path(@article)`, `tmp/whitelist/articles.yml` will be created.
+
+The contents will look like the following.
+
+```yaml
+---
+articles:
+  show:
+    id:
+      type: string
+```
+
+If you copy the contents to `config/whitelist/articles.yml` and then access it again, the exception will not occur.
+By repeating this process for each action, we can create a whitelist.
+By default, the whitelist is generated as `string`, so change the type to match the parameters (string|number|date|boolean).
+
+Parameters will be merged into the template as needed for each action, so you don't have to delete them.
+For example, if you access `DELETE article_path(@article)` after this, `tmp/whitelist/articles.yml` will look like this
+
+```yaml
+---
+articles:
+  destroy:
+    id:
+      type: string
+  show:
+    id:
+      type: string
+```
+
+### If you want to allow all parameters
+
+When using GraphQL, it is not possible to create a whitelist for the `variables` parameter, since it is defined on the client side.
+
+In such a case, you need to allow everything under `variables`. If you want to allow them all, specify `_permit_all: true`.
+
+```yaml
+---
+graphql:
+  execute:
+    operationName:
+      type: string
+    query:
+      options:
+        max_length: 1048576
+      type: string
+    variables:
+      _permit_all: true
+```
+
+However, although all keys are allowed, to prevent attacks,
+the type will be inferred from the value and the input value will be validated with the default rules for that type.
+
+## Development
+
+Please clone the repository and start development.
+
+### Preparation
+
+To develop, start PostgreSQL with docker-compose.
+
+```bash
+docker-compose up -d
+```
+
+And then execute:
+
+```bash
+bundle install
+```
+
+### Test
+
+The test uses minitest.
+
+```bash
+bin/test
+```
+
+## Contributing
+
+If you have any bug reports or pull requests, please let me know.
+
+## License
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,3 @@
+require "bundler/setup"
+
+require "bundler/gem_tasks"

data/lib/full_metal_body/controllers/concerns/input_validation_action.rb

@@ -0,0 +1,180 @@
+# frozen_string_literal: true
+
+require 'full_metal_body/input_validation'
+require 'full_metal_body/services/save_blocked_keys_service'
+require 'full_metal_body/whitelist_writer'
+
+module FullMetalBody
+  module InputValidationAction
+
+    extend ActiveSupport::Concern
+
+    RAILS_KEYS = %w(controller action format).map(&:freeze).freeze
+    MAX_BLOCKED_KEYS_COUNT = 3
+
+    included do
+      before_action :validate_params
+
+      unless respond_to?(:current_user, true)
+        class_eval do
+          def current_user
+            nil
+          end
+        end
+      end
+    end
+
+    private
+
+    def validate_params
+      blocked_keys = []
+      whitelist = get_whitelist
+
+      hash_keys(params.to_unsafe_h).each do |key|
+        next if RAILS_KEYS.include?(key.join('.'))
+
+        value = params.dig(*key)
+        if permit_all_params?(key, whitelist)
+          dynamic_whitelist_generator = DynamicWhitelistGenerator.new(key, value, whitelist)
+          whitelist = dynamic_whitelist_generator.execute!
+          # rubocop:disable Lint/UselessAssignment
+          dynamic_whitelist_generator = nil
+          # rubocop:enable Lint/UselessAssignment
+        end
+
+        valid, result = validate_each(key, value, whitelist)
+        if valid
+          if result.nil? && !permit_all_params?(key, whitelist)
+            record_blocked_key(key.join('.'))
+            blocked_keys << key
+
+            next if (ENV['USE_WHITELIST_COUNT_CHECK'] || '1') == '0'
+
+            if blocked_keys.size > MAX_BLOCKED_KEYS_COUNT
+              SaveBlockedKeysService.execute!(controller_path, action_name, blocked_keys)
+              output_error(
+                blocked_keys.to_s,
+                "Unknown parameters existed over #{MAX_BLOCKED_KEYS_COUNT}.",
+                )
+              return nil
+            end
+          end
+        else
+          output_error(key.join('.'), result.details)
+          return nil
+        end
+      end
+
+      return if blocked_keys.empty?
+
+      SaveBlockedKeysService.execute!(controller_path, action_name, blocked_keys)
+
+      return unless Rails.env.development?
+
+      WhitelistWriter.new(controller_path, action_name).write!(blocked_keys)
+      raise StandardError, "#{blocked_keys} are not included in whitelist. A template has been created in 'tmp/whitelist/#{controller_path}'"
+    end
+
+    # Validate with whitelist
+    #
+    # @param [Array<String>] key
+    # @param [Object] value
+    # @param [Hash] whitelist
+    #
+    # @return [Boolean] (true, false)
+    # @return [Hash, ActiveModel::Errors] In success: Type definition. In failure: Error infos.
+    def validate_each(key, value, whitelist)
+      key_type = nil
+      (value.nil? ? [nil] : Array(value)).flatten.each do |v|
+        validation = InputValidation.new(key.map(&:to_s), v, whitelist)
+        key_type = validation.key_type
+        return false, validation.errors unless validation.valid?
+      end
+      return true, key_type
+    end
+
+    # Get hash keys recursively.
+    #
+    # @param [Object] obj object
+    # @param [Array<String>] key
+    # @param [Array<String>] result
+    #
+    # @return [Array<String>] result
+    #
+    def hash_keys(obj, key = [], result = [])
+      case obj
+      when Hash
+        obj.each do |k, v|
+          hash_keys(v, key + [k], result)
+        end
+      when Array
+        obj.each_with_index do |v, idx|
+          hash_keys(v, key + [idx], result)
+        end
+      else
+        result << key
+      end
+      result
+    end
+
+    #
+    # Get a whitelist from config/whitelist/**/*.yml
+    #
+    # @return [Hash] Whitelist
+    #
+    def get_whitelist
+      path = Rails.root.join('config', 'whitelist', "#{controller_path}.yml")
+      return nil unless File.exist?(path)
+
+      yaml = File.open(path, "r") do |file|
+        YAML.safe_load(file)&.deep_stringify_keys
+      end
+      yaml&.dig(controller_name, action_name)
+    end
+
+    #
+    # Output validation errors
+    #
+    # @param [String] key
+    # @param [String] errors
+    #
+    def output_error(key, errors)
+      error_message = <<~ERR
+        Input validation error detected
+          Process: #{controller_path}##{action_name}
+          User: #{current_user&.id || 'unknown'}
+          IP: #{request.remote_ip}
+          Key: #{key}
+          Errors: #{errors}
+      ERR
+      # TODO: I want to be able to handle more than just Bugsnag.
+      if Rails.env.production?
+        Bugsnag.notify(error_message) if defined?(Bugsnag)
+      end
+      logger.error error_message
+      head :bad_request
+    end
+
+    # Logging key and process_name when key is not included in whitelist.
+    #
+    # @param [String] key
+    #
+    def record_blocked_key(key)
+      process_name = "#{controller_path}##{action_name}"
+      message = "Input validation warning ('#{key}' not include in whitelist for '#{process_name}')"
+      logger.warn message
+    end
+
+    # Permit all parameters if '_permit_all: true' is existed.
+    #
+    # @param [Array<String,Symbol>] keys
+    # @param [Hash] whitelist
+    # @return [Boolean] (true, false)
+    def permit_all_params?(keys, whitelist)
+      return false if whitelist.blank?
+
+      keys.size.downto(1).any? { |i| whitelist.dig(*keys.first(i))&.fetch('_permit_all', false) }
+    end
+
+  end
+end

data/lib/full_metal_body/deep_sort.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module FullMetalBody
+  module DeepSort
+
+    class Error < StandardError; end
+
+    refine Hash do
+      def deep_sort
+        keys = self.keys
+        raise DeepSort::Error, "Invalid Keys(#{keys})" unless keys.all? { |k| k.is_a?(String) || k.is_a?(Symbol) || k.is_a?(Numeric) }
+
+        sort.to_h.transform_values { |v| v.respond_to?(:deep_sort) ? v.deep_sort : v }
+      end
+
+      def deep_sort!
+        replace(deep_sort)
+      end
+    end
+
+    refine Array do
+      def deep_sort
+        case
+        when all?(Numeric) then sort
+        when all?(String), all?(Symbol) then map(&:to_s).sort
+        else
+          map { |v| v.respond_to?(:deep_sort) ? v.deep_sort : v }.sort_by(&:to_s)
+        end
+      end
+
+      def deep_sort!
+        case
+        when all?(Numeric) then sort!
+        when all?(String), all?(Symbol) then map!(&:to_s).sort!
+        else
+          map! { |v| v.respond_to?(:deep_sort!) ? v.deep_sort! : v }.sort_by!(&:to_s)
+        end
+      end
+    end
+
+  end
+
+end

data/lib/full_metal_body/dynamic_whitelist_generator.rb

@@ -0,0 +1,201 @@
+# frozen_string_literal: true
+
+module FullMetalBody
+  class DynamicWhitelistGenerator
+
+    include InputKeyUtils
+
+    # @param [Array<String,Symbol>] keys
+    # @param [Object] value
+    # @param [Hash] whitelist
+    def initialize(keys, value, whitelist = {})
+      @keys = keys
+      @value = value
+      @whitelist = whitelist
+    end
+
+    # Dynamically generate a whitelist and return a merged one.
+    # @return [ActiveSupport::HashWithIndifferentAccess]
+    def execute!
+      if keys_isnt_whitelisted?
+        if keys_include_array?(@keys)
+          generate_whitelist_for_array(@keys)
+        else
+          @whitelist.bury!((@keys + ['type']), type_definition_by_value)
+        end
+      end
+      @whitelist.with_indifferent_access
+    end
+
+    private
+
+    # Check to see if the keys are on the whitelist.
+    # @return [Boolean] (true, false)
+    def keys_isnt_whitelisted?
+      !@whitelist.dig(*@keys)
+    end
+
+    # Check to see if the keys contain the array index.
+    # @return [Boolean] (true, false)
+    def keys_include_array?(keys)
+      !!keys.find_index { |k| key_numeric?(k) }
+    end
+
+    # Return type-definition of value.
+    # @return [String] Type-definition
+    def type_definition_by_value
+      case @value
+      when Numeric
+        'number'
+      when Date
+        'date'
+      when TrueClass, FalseClass
+        'boolean'
+      else
+        'string'
+      end
+    end
+
+    # Dynamically generate an array whitelist.
+    # @param [Array<String,Symbol>] keys
+    # @option [Array<String,Symbol>] prefix_keys
+    # @raise [DynamicWhitelistGenerator::ParseArrayError]
+    def generate_whitelist_for_array(keys, prefix_keys = [])
+      parent_keys, child_keys = separate_by_array_key(keys)
+      parent_keys = prefix_keys + parent_keys
+      @whitelist.bury!((parent_keys + ['type']), 'array')
+      if child_keys.empty?
+        # keys = ["pref_ids", 0]
+        # prefix_keys # => []
+        # parent_keys # => ["pref_ids"]
+        # child_keys # => []
+        # {
+        #   "pref_ids" => {
+        #     "type" => "array",
+        #     "properties" => {
+        #        "type" => type_definition_by_value
+        #     }
+        #   }
+        # }
+        @whitelist.bury!((parent_keys + ['properties', 'type']), type_definition_by_value)
+      elsif child_keys.size == 1 && !key_numeric?(child_keys.first)
+        # keys = ["models", 0, "model_id"]
+        # prefix_keys # => []
+        # parent_keys # => ["models"]
+        # child_keys # => ["model_id"]
+        # {
+        #   "models" => {
+        #     "type" => "array",
+        #     "properties" => {
+        #       "model_id" => {
+        #         "type" => type_definition_by_value
+        #       }
+        #     }
+        #   }
+        # }
+        @whitelist.bury!((parent_keys + ['properties', child_keys.first, 'type']), type_definition_by_value)
+      elsif keys_include_array?(child_keys)
+        # case 1: 2D array
+        # Lap 1
+        # keys = ["models", 0, 0]
+        # prefix_keys # => []
+        # parent_keys # => ["models"]
+        # child_keys # => [0]
+        # generate_whitelist_for_array([0], ["models", "properties"])
+        #   Lap 2
+        #   keys = [0]
+        #   prefix_keys = ["models", "properties"]
+        #   parent_keys = ["models", "properties"]
+        #   child_keys = []
+        #   Enter `if child_keys.empty?`
+        #   {
+        #     "models" => {
+        #       "type" => "array",
+        #       "properties" => {
+        #         "type" => "array",
+        #         "properties" => {
+        #           "type" => type_definition_by_value
+        #         }
+        #       }
+        #     }
+        #   }
+        #
+        # case 2: An object in an array has a further array.
+        # Lap 1
+        # keys = ["models", 0, "model_ids", 0]
+        # prefix_keys # => []
+        # parent_keys # => ["models"]
+        # child_keys # => ["model_ids", 0]
+        # generate_whitelist_for_array(["model_ids", 0], ["models", "properties"])
+        #   Lap 2
+        #   keys = ["model_ids", 0]
+        #   prefix_keys = ["models", "properties"]
+        #   parent_keys = ["models", "properties", "model_ids"]
+        #   child_keys = []
+        #   Enter `if child_keys.empty?`
+        #   {
+        #     "models" => {
+        #       "type" => "array",
+        #       "properties" => {
+        #         "model_ids" => {
+        #           "type" => "array",
+        #           "properties" => {
+        #             "type" => type_definition_by_value
+        #           }
+        #         }
+        #       }
+        #     }
+        #   }
+        #
+        # case 3: The object in the array has a further array, and the object in the array
+        # Lap 1
+        # keys = ["articles", 0, "comments", 0, "content"]
+        # prefix_keys # => []
+        # parent_keys # => ["articles"]
+        # child_keys # => ["comments", 0, "content"]
+        # generate_whitelist_for_array(["comments", 0, "content"], ["articles", "properties"])
+        #   Lap 2
+        #   keys = ["comments", 0, "content"]
+        #   prefix_keys = ["articles", "properties"]
+        #   parent_keys = ["articles", "properties", "comments"]
+        #   child_keys = ["content"]
+        #   Enter `elsif child_keys.size == 1 && !key_numeric?(child_keys.first)`
+        #   {
+        #     "articles" => {
+        #       "type" => "array",
+        #       "properties" => {
+        #         "comments" => {
+        #           "type" => "array",
+        #           "properties" => {
+        #             "content" => {
+        #               "type" => type_definition_by_value
+        #             }
+        #           }
+        #         }
+        #       }
+        #     }
+        #   }
+        generate_whitelist_for_array(child_keys, parent_keys + ['properties'])
+      else
+        # Normally, there are no cases that come here.
+        # But if it should come, raise Exception.
+        invalid_keys = prefix_keys + keys
+        invalid_keys.delete('properties')
+        raise DynamicWhitelistGenerator::ParseArrayError.new("Invalid keys", invalid_keys)
+      end
+    end
+
+    class ParseArrayError < StandardError
+
+      # @param [String] message
+      # @param [Array<String,Symbol>] keys
+      def initialize(message, keys)
+        @keys = keys
+        super("#{message}: #{@keys}")
+      end
+
+    end
+
+  end
+
+end