firetail 0.0.0 → 1.0.1

data/lib/firetail.rb

@@ -1,6 +1,269 @@
 require "firetail/version"
+require "rack"
+require 'objspace'
+require 'yaml'
+require 'json'
+require 'net/http'
+require 'case_sensitive_headers' # a hack because firetail API headers is case-sensitive
+require "async"
+require 'digest/sha1'
+require 'jwt'
+require 'logger'
+require 'background_tasks'
+require 'committee'
+
+# If the library detects rails, it will load rail's methods
+if defined?(Rails)
+  require 'action_dispatch'
+  require 'action_pack'
+  require 'railtie'
+end
 
 module Firetail
   class Error < StandardError; end
-  # Your code goes here...
+
+  class Run
+    MAX_BULK_SIZE_IN_BYTES = 1 * 1024 * 1024  # 1 MB
+
+    def initialize app
+      @app = app
+      @reqres ||= [] # request data in stored in array memory
+      @init_time ||= Time.now # initialize time
+    end
+ 
+    def call(env)
+      # This block initialises the configuration and checks
+      # sets the values for certain necessary configuration
+      # If it is Rails
+      if defined?(Rails)
+        begin
+          default_location = File.join(Rails.root, "config/firetail.yml")
+          config = YAML.safe_load(ERB.new(File.read(default_location)).result)
+        rescue Errno::ENOENT
+          # error message if firetail is not installed
+          puts ""
+          puts "Please run 'rails generate firetail:install' first"
+          puts ""
+        end
+      else # other frameworks
+        config = YAML.load_file("firetail.yml")
+      end
+
+      raise Error.new "Please run 'rails generate firetail:install' first" if config.nil?
+      raise Error.new "API Key is missing from firetail.yml configuration"  if config['api_key'].nil?
+
+      @api_key            = config['api_key']
+      @url                = config['url'] ? config['url'] : "https://api.logging.eu-west-1.prod.firetail.app/logs/bulk" # default goes to europe
+      @log_drains_timeout = config['log_drains_timeout'] ? config['log_drains_timeout'] : 5
+      @network_timeout    = config['network_timeout'] ? config['network_timeout'] : 10
+      @number_of_retries  = config['number_of_retries'] ? config['number_of_retries'] : 4
+      @retry_timeout      = config['retry_timeout'] ? config['retry_timeout'] : 2
+      # End of configuration initialization
+
+      # Gets the rack middleware requests
+      @request = Rack::Request.new(env)
+      started_on = Time.now
+      begin
+        status, client_headers, body = response = @app.call(env)
+	log(env, status, body, started_on, Time.now)
+      rescue Exception => exception
+	log(env, status, body, started_on, Time.now, exception)
+        raise exception
+      end
+
+      response
+    end
+
+    def log(env,
+	    status,
+	    body,
+            started_on,
+	    ended_on,
+            exception = nil)
+
+      # request values
+      time_spent                = ended_on - started_on
+      request_ip                = defined?(Rails) ? env['action_dispatch.remote_ip'].calculate_ip : env['REMOTE_ADDR']
+      request_method            = env['REQUEST_METHOD']
+      request_path              = env['REQUEST_PATH']
+      request_http_version      = env['HTTP_VERSION']
+       
+      # get the resource parameters if it is rails
+      if defined?(Rails)
+        resource = Rails.application.routes.recognize_path(request_path)
+	#Firetail.logger.debug "res: #{resource}"
+	# sample hash of the above resource:
+	# example url: /posts/1/comments/2/options/3
+	# hash = {:controller=>"options", :action=>"show", :comment_id => 3, :post_id=>"1", :id=>"1"}
+	# take the resource hash above, get keys, conver to string, split "_" to get name at first index, together
+	# with the key, to string and camelcase route id name and keys that only include "id", compact (remove nil) and add "s" to the key
+	rmap = resource.map {|k,v| [k.to_s.split("_")[0], "{#{k.to_s.camelize(:lower)}}"] if k.to_s.include? "id" }
+	.compact.map {|k,v| [k.to_s + "s", v] if k != "id" }
+
+	if resource.key? :id 
+	    # It will appear like: [["comments", "commentId"], ["posts", "postId"], ["id", "id"]], 
+	    # but we want post to be first in order, so we reverse sort, and drop "id", which will be first in array
+	    # after being sorted
+	    reverse_resource = rmap.reverse.drop(1)
+            resource_path = "/" + reverse_resource * "/" + "/" + resource[:controller] + "/" + "{id}"
+	    # rebuild the resource path
+	    # reverse_resource * "/" will loop the array and add "/"
+	    #resource_path = "/" + reverse_resource * "/" + "/" + resource[:controller] + "/" + "{id}"
+	    # end result is /posts/{postId}/comments/{commentId}/options/{id}
+	else
+	  if rmap.empty?
+	    # if resoruce is empty, means we are at the first level of the url path, so no need extra paths
+	    resource_path = "/" + rmap * "/" + resource[:controller]
+	  else
+	    # resource path from rmap above without the [:id] key (which is the last parameter in URL)
+            # only used for index, create which does not have id
+  	    resource_path = "/" + rmap * "/" + "/" + resource[:controller]
+	  end
+	end
+      else
+	resource_path = nil
+      end
+
+      #Firetail.logger.debug("resource path: #{resource_path}")
+      # select those with "HTTP_" prefix, these are request headers
+      request_headers = env.select {|key,val| key.start_with? 'HTTP_' } # find HTTP_ prefixes, these are requests only
+	                .collect {|key, val| { "#{key.sub(/^HTTP_/, '')}": [val] }} # remove HTTP_ prefix
+	                .reduce({}, :merge)  # reduce from [{key:val},{key2: val2}] to {key: val, key2: val2}
+
+      # do the inverse of the above and get rack specific keys
+      response_headers = env.select {|key,val| !key.start_with? 'HTTP_' } # only keys with no HTTP_ prefix
+	                 .select {|key, val| key =~ /^[A-Z._]*$/} # select keys with uppercase and underline
+                         .map {|key, val| { "#{key}": [val] }} # map to firetail api format
+                         .reduce({}, :merge) # reduce from [{key:val},{key2: val2}] to {key: val, key2: val2}
+
+      # get the jwt "sub" information
+      if request_headers[:AUTHORIZATION]
+        subject = self.jwt_decoder(request_headers[:AUTHORIZATION])
+      else
+	subject = nil
+      end
+
+      # default time spent in ruby is in seconds, so multiple by 1000 to ms
+      time_spent_in_ms = time_spent * 1000
+      #Firetail.logger.debug "request params: #{@request.params.inspect}"
+      # add the request and response data 
+      # to array of data for batching up
+      @request.body.rewind
+      if body.is_a? Array
+        body = body[0]
+      else
+        body = body.body
+      end
+      @reqres.push({
+	version: "1.0.0-alpha",
+	dateCreated: Time.now.utc.to_i,
+	executionTime: time_spent_in_ms,
+        request: {
+ 	  httpProtocol: request_http_version,
+	  headers: request_headers, # headers must be in: headers: {"key": ["value"]}, array in object
+	  method: request_method,
+	  body: @request.body.read,
+	  ip: request_ip,
+	  resource: resource_path,
+	  uri: @request.url
+	},
+	response: {
+  	  statusCode: status,
+	  body: body,
+          headers: response_headers,
+	}
+      })
+      @request.body.rewind
+      #Firetail.logger.debug "Request: #{body}"
+
+      # the time we calculate if request that is
+      # buffered max is 120 seconds
+      current_time = Time.now
+      # duration in millseconds
+      duration = (current_time - @init_time)
+
+      #Firetail.logger.debug "size in bytes #{ObjectSpace.memsize_of(@request_data.to_s)}"
+      #request data size in bytes
+      request_data_size = ObjectSpace.memsize_of(@request_data)
+      # It is difficult to calculate the object size in bytes, 
+      # seems to not return the accurate values
+
+      # This will send the data we need in batches of 5 requests or when it is more than 120 seconds
+      # if there are more than 5 requests or is more than
+      # 2 minutes, then send to backend - this is for testing
+      if @reqres.length >= 5 || duration > 120
+        #Firetail.logger.debug "request data #{@reqres}"
+	# we parse the data hash into json-nl (json-newlines)
+	payload = @reqres.map { |data| JSON.generate(data) }.join("\n")
+
+	# send the data to backend API
+	# This is an async task
+	BackgroundTasks.http_task(@url,
+			@network_timeout,
+			@api_key,
+			@number_of_tries,
+		        payload)
+
+	# reset back to the initial conditions
+	payload = nil
+	@reqres = []
+	@init_time = Time.now
+      end
+    rescue Exception => exception
+      Firetail.logger.error(exception.message)
+    end
+
+    def sha1_hash(value)
+      encode_utf8 = value.encode(Encoding::UTF_8)
+      hash = Digest::SHA1.hexdigest(encode_utf8)
+      sha1 = "sha1: #{hash}"
+    end
+
+    def jwt_decoder(value)
+      bearer_string = value
+      # if authorization exists, get the value at index
+      # split the values which has a space (example: "Bearer 123") and 
+      # get the value at index 1
+      token = bearer_string.split(" ")[1]
+      # decode the token
+      jwt_value = JWT.decode token, nil, false
+      # get the subject value
+      subject = jwt_value[0]["sub"]
+      #Firetail.logger.debug("subject value: #{subject}")
+    end
+  end
+
+  def self.logger
+    @@logger ||= defined?(Rails) ? Rails.logger : Logger.new(STDOUT)
+  end
+
+  def self.logger=(logger)
+    @@logger = logger
+  end
+
+  # custom error message
+  # https://blog.frankel.ch/structured-errors-http-apis/
+  # https://www.rfc-editor.org/rfc/rfc7807
+  class Committee::ValidationError
+    def error_body
+      {
+        errors: [
+          {
+            type: "#{request.env['rack.url_scheme']}://#{request.env['HTTP_HOST']}#{request.env['REQUEST_URI']}",
+            title: id,
+            detail: message,
+            status: status
+          }
+        ]
+      }
+    end
+
+    def render
+      [
+        status,
+        { "Content-Type" => "application/json" },
+        [JSON.generate(error_body)]
+      ]
+    end
+  end
 end

data/lib/generators/firetail/install/templates/README

@@ -0,0 +1,15 @@
+====================================================================================
+
+Thank you for installing Firetail!
+
+Before you start using firetail, please edit the file in "config/firetail.yml"
+with your API Key, Token and URL first before running your Rails app.
+
+The middleware is already configured in "config/application.rb" (you can check that)
+
+Also we have installed a sample copy of schema.json in "config/schema.json" (your API schema) 
+for you to better understand and create your schema based on your needs.
+
+Happy coding!
+
+- From the team at Firetail

data/lib/generators/firetail/install/templates/firetail.yml

@@ -0,0 +1,2 @@
+api_key: <%= ENV.fetch("FIRETAIL_API_KEY", nil) %>
+url: <%= ENV.fetch("FIRETAIL_URL", nil) %>

data/lib/generators/firetail/install/templates/schema.json

@@ -0,0 +1,108 @@
+{
+  "openapi": "3.0.0",
+  "info": {
+    "title": "api-test",
+    "version": "1.0"
+  },
+  "servers": [
+    {
+      "url": "http://localhost:3000",
+      "description": "dev"
+    }
+  ],
+  "paths": {
+    "/posts": {
+      "post": {
+        "parameters": [
+          {
+            "in": "query",
+            "name": "page",
+            "required": true,
+            "schema": {
+              "type": "integer"
+            }
+          }
+        ],
+        "requestBody": {
+          "content": {
+            "application/json": {
+              "schema": {
+                "type": "object",
+                "required": [
+                  "year"
+                ],
+                "properties": {
+                  "year": {
+                    "type": "integer"
+                  }
+                }
+              }
+            }
+          }
+        },
+        "responses": {
+          "200": {
+            "description": "correct response",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "type": "object",
+                  "required": [
+                    "title",
+                    "page",
+                    "year"
+                  ],
+                  "properties": {
+                    "title": {
+                      "type": "string"
+                    },
+                    "page": {
+                      "type": "integer"
+                    },
+                    "year": {
+                      "type": "integer"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    },
+    "/dont_allow_additional_parameter": {
+      "post": {
+        "requestBody": {
+          "content": {
+            "application/json": {
+              "schema": {
+                "type": "object",
+                "additionalProperties": false,
+                "required": [
+                  "only_param"
+                ],
+                "properties": {
+                  "only_param": {
+                    "type": "string"
+                  }
+                }
+              }
+            }
+          }
+        },
+        "responses": {
+          "200": {
+            "description": "No Content",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "type": "object"
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

data/lib/generators/firetail/install_generator.rb

@@ -0,0 +1,26 @@
+module Firetail
+  module Generators
+    class InstallGenerator < Rails::Generators::Base
+      source_root File.expand_path("../install/templates", __FILE__)
+
+      desc "create firetail configuration template to config/firetail.yml and add middleware"
+      def add_firetail_configuration
+        template "firetail.yml", File.join("config", "firetail.yml")
+      end
+
+      desc "add firetail middleware to rails application.rb"
+      def add_firetail_middleware
+        application "config.middleware.use Firetail::Run"
+      end
+
+      desc "add firetail sample json-schema template"
+      def add_firetail_sample_schema
+        template "schema.json", File.join("config", "schema.json")
+      end
+
+      def show_readme
+        readme "README" if behavior == :invoke
+      end
+    end
+  end
+end

data/lib/railtie.rb

@@ -0,0 +1,29 @@
+require 'committee'
+require 'rails'
+
+class Error < StandardError; end
+
+class Railtie < Rails::Railtie
+
+    initializer "commitee.insert_middleware" do |app|
+      begin
+        schema_path = File.join(Rails.root, "config/schema.json")
+
+	# check if schema file exists in config/
+	if File.exist?(schema_path)
+          app.config.middleware.use Committee::Middleware::RequestValidation,
+            schema_path: schema_path,
+            coerce_date_times: true,
+            params_key: 'action_dispatch.request.request_parameters',
+            query_hash_key: 'action_dispatch.request.query_parameters'
+
+          app.config.middleware.use Committee::Middleware::ResponseValidation, schema_path: schema_path
+	else 
+          puts "Need schema.json in \"config/\" directory"
+          puts "Try re-running \"rails g firetail:install\" again"
+        end
+      rescue Error => e
+        puts e
+      end
+    end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: firetail
 version: !ruby/object:Gem::Version
-  version: 0.0.0
+  version: 1.0.1
 platform: ruby
 authors:
 - Muhammad Nuzaihan
-autorequire:
+autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-09-08 00:00:00.000000000 Z
+date: 2024-04-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -52,6 +52,76 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.18.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.18.1
+- !ruby/object:Gem::Dependency
+  name: async
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.30.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.30.3
+- !ruby/object:Gem::Dependency
+  name: jwt
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.5'
+- !ruby/object:Gem::Dependency
+  name: json-schema
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+- !ruby/object:Gem::Dependency
+  name: committee_firetail
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.0.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.0.1
 description: API security library that is designed for ruby
 email:
 - zaihan@flitnetics.com
@@ -59,19 +129,110 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/pull_request_template.md"
+- ".github/workflows/main.yml"
 - ".gitignore"
 - ".rspec"
 - ".travis.yml"
 - CODE_OF_CONDUCT.md
 - Gemfile
+- Gemfile.lock
+- LICENSE
 - LICENSE.txt
 - README.md
 - Rakefile
 - bin/console
 - bin/setup
+- examples/rails/.gitattributes
+- examples/rails/.gitignore
+- examples/rails/.rspec
+- examples/rails/.ruby-version
+- examples/rails/Gemfile
+- examples/rails/Gemfile.lock
+- examples/rails/README.md
+- examples/rails/Rakefile
+- examples/rails/app/channels/application_cable/channel.rb
+- examples/rails/app/channels/application_cable/connection.rb
+- examples/rails/app/controllers/application_controller.rb
+- examples/rails/app/controllers/comments_controller.rb
+- examples/rails/app/controllers/concerns/.keep
+- examples/rails/app/controllers/posts_controller.rb
+- examples/rails/app/jobs/application_job.rb
+- examples/rails/app/mailers/application_mailer.rb
+- examples/rails/app/models/application_record.rb
+- examples/rails/app/models/comment.rb
+- examples/rails/app/models/concerns/.keep
+- examples/rails/app/models/post.rb
+- examples/rails/app/views/layouts/mailer.html.erb
+- examples/rails/app/views/layouts/mailer.text.erb
+- examples/rails/bin/bundle
+- examples/rails/bin/rails
+- examples/rails/bin/rake
+- examples/rails/bin/setup
+- examples/rails/bin/spring
+- examples/rails/config.ru
+- examples/rails/config/application.rb
+- examples/rails/config/boot.rb
+- examples/rails/config/cable.yml
+- examples/rails/config/credentials.yml.enc
+- examples/rails/config/database.yml
+- examples/rails/config/environment.rb
+- examples/rails/config/environments/development.rb
+- examples/rails/config/environments/production.rb
+- examples/rails/config/environments/test.rb
+- examples/rails/config/firetail.yml
+- examples/rails/config/initializers/application_controller_renderer.rb
+- examples/rails/config/initializers/backtrace_silencers.rb
+- examples/rails/config/initializers/cors.rb
+- examples/rails/config/initializers/filter_parameter_logging.rb
+- examples/rails/config/initializers/inflections.rb
+- examples/rails/config/initializers/mime_types.rb
+- examples/rails/config/initializers/wrap_parameters.rb
+- examples/rails/config/locales/en.yml
+- examples/rails/config/puma.rb
+- examples/rails/config/routes.rb
+- examples/rails/config/schema.json
+- examples/rails/config/spring.rb
+- examples/rails/config/storage.yml
+- examples/rails/db/migrate/20230730163722_create_posts.rb
+- examples/rails/db/migrate/20230730163741_create_comments.rb
+- examples/rails/db/migrate/20230730164121_add_fields_to_post.rb
+- examples/rails/db/migrate/20230730164214_add_fields_to_comments.rb
+- examples/rails/db/schema.rb
+- examples/rails/db/seeds.rb
+- examples/rails/lib/tasks/.keep
+- examples/rails/log/.keep
+- examples/rails/public/robots.txt
+- examples/rails/spec/models/comment_spec.rb
+- examples/rails/spec/models/post_spec.rb
+- examples/rails/spec/rails_helper.rb
+- examples/rails/spec/requests/comments_spec.rb
+- examples/rails/spec/requests/posts_spec.rb
+- examples/rails/spec/routing/comments_routing_spec.rb
+- examples/rails/spec/routing/posts_routing_spec.rb
+- examples/rails/spec/spec_helper.rb
+- examples/rails/storage/.keep
+- examples/rails/test/channels/application_cable/connection_test.rb
+- examples/rails/test/controllers/.keep
+- examples/rails/test/fixtures/files/.keep
+- examples/rails/test/integration/.keep
+- examples/rails/test/mailers/.keep
+- examples/rails/test/models/.keep
+- examples/rails/test/test_helper.rb
+- examples/rails/tmp/.keep
+- examples/rails/tmp/pids/.keep
+- examples/rails/vendor/.keep
 - firetail.gemspec
+- lib/backend.rb
+- lib/background_tasks.rb
+- lib/case_sensitive_headers.rb
 - lib/firetail.rb
 - lib/firetail/version.rb
+- lib/generators/firetail/install/templates/README
+- lib/generators/firetail/install/templates/firetail.yml
+- lib/generators/firetail/install/templates/schema.json
+- lib/generators/firetail/install_generator.rb
+- lib/railtie.rb
 homepage: https://www.firetail.io
 licenses:
 - MIT
@@ -80,7 +241,7 @@ metadata:
   homepage_uri: https://www.firetail.io
   source_code_uri: https://github.com/firetail-io/ruby
   changelog_uri: https://github.com/firetail-io/ruby/CHANGELOG.md
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -95,8 +256,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.8
-signing_key:
+rubygems_version: 3.5.9
+signing_key: 
 specification_version: 4
 summary: Ruby library for firetail
 test_files: []