speakeasy_ruby_sdk 0.0.2

data/lib/speakeasy_ruby_sdk.rb

@@ -0,0 +1,112 @@
+require_relative 'speakeasy_ruby_sdk/version'
+require_relative 'speakeasy_ruby_sdk/config'
+require_relative 'speakeasy_ruby_sdk/url_utils'
+require_relative 'speakeasy_ruby_sdk/masker'
+require_relative 'speakeasy_ruby_sdk/http_transaction'
+require_relative 'speakeasy_ruby_sdk/har_builder'
+require_relative 'speakeasy_ruby_sdk/time'
+
+require "delegate"
+
+require 'speakeasy_pb'
+include Ingest
+include Embedaccesstoken
+
+module SpeakeasyRubySdk
+  class RouteWrapper < SimpleDelegator
+    def path
+      super.spec.to_s
+    end
+  end
+
+  class Middleware
+    attr_accessor :ingest_client
+    attr_reader :config
+
+    def initialize(app, config=nil)
+      @config = Config.default.merge config
+      validation_errors = @config.validate
+      if validation_errors.length > 0
+        raise Exception.new validation_errors
+      end
+      @app = app
+      @masker = SpeakeasyRubySdk::Masker.new @config
+      credentials = GRPC::Core::ChannelCredentials.new()
+      @ingest_client = Ingest::IngestService::Stub.new(@config.ingestion_server_url, credentials)
+
+    end
+   
+    def call(env)
+      if env.include? 'time_utils'
+        time_utils = env['time_utils']
+      else
+        time_utils = TimeUtils.new
+      end
+
+      status, response_headers, response_body = @app.call(env)
+
+      ## If we are not in test, record the end time after calling the app
+      if !env.include? 'time_utils'
+        time_utils.set_end_time
+      end
+
+      http_transaction = HttpTransaction.new time_utils, env, status, response_headers, response_body, @masker
+
+      path_hint = ''
+      # todo - support other frameworks than rails
+      if @config.routes 
+        req = ActionDispatch::Request.new(env)
+        found_route = nil
+        @config.routes.router.recognize(req) do |route, params|
+          found_route = route
+        end
+        if ! found_route.nil?
+          path_hint = RouteWrapper.new(found_route).path
+        end
+      end
+      if ! env[:path_hint].nil?
+        path_hint = env[:path_hint]
+      end
+ 
+      har_builder = HarBuilder.new @config
+      har = har_builder.construct_har http_transaction
+
+      request = Ingest::IngestRequest.new
+      request.api_id = @config.api_id
+      request.path_hint = path_hint
+      request.version_id = @config.version_id
+      if env.include? :customer_id
+        request.customer_id = env[:customer_id].to_s
+      end
+      request.har = har
+
+      metadata = {"x-api-key": @config.api_key}
+      response = @ingest_client.ingest(request, metadata: metadata)
+
+      [status, response_headers, response_body]
+    end
+  end
+
+  def self.get_embedded_access_token key, operator, value, config=nil
+    if config.nil?
+      working_config = Middleware::Config.default
+    else
+      working_config = Middleware::Config.default.merge config
+    end
+
+    credentials = GRPC::Core::ChannelCredentials.new()
+    embed_client = Embedaccesstoken::EmbedAccessTokenService::Stub.new(working_config.ingestion_server_url, credentials)
+    request = Embedaccesstoken::EmbedAccessTokenRequest.new
+
+    filter = Embedaccesstoken::EmbedAccessTokenRequest::Filter.new
+    filter.key = key
+    filter.operator = operator
+    filter.value = value
+
+    request.filters.push filter
+
+    metadata = {"x-api-key": working_config.api_key}
+    response = embed_client.get(request, metadata: metadata)
+
+  end
+end

data/test/bulk_test.rb

@@ -0,0 +1,176 @@
+require 'test/unit'
+require 'rack/test'
+require 'json'
+require 'ostruct'
+require 'spy'
+require 'timecop'
+require_relative '../lib/speakeasy_ruby_sdk'
+
+require 'speakeasy_pb'
+include Ingest
+
+def add_http_prefix header_dict
+  modified_headers = {}
+  for k, v in header_dict
+    modified_headers["HTTP_#{k}"] = v
+  end
+  modified_headers
+end
+
+def header_to_dict headers
+  modified_headers = {}
+  for d in headers
+    key = d['key']
+    if d['values'].length > 1
+      modified_headers[key] = d['values']
+    else
+      modified_headers[key] = d['values'][0]
+    end
+  end
+  modified_headers
+end
+
+class BulkTest < Test::Unit::TestCase
+  include Rack::Test::Methods
+
+  def test_bulk
+    file_pairs = []
+    ## Find all test files, and pair input and outputs 
+    Dir.foreach("./test/testdata") do |file_name|
+      if file_name == "." || file_name == ".."
+        next
+      end
+      infile = nil
+      outfile = nil
+      if file_name.end_with?("input.json")
+        prefix_name = file_name.slice(0..-11)
+        output_filename = "#{prefix_name}output.json"
+        pair = OpenStruct.new
+        pair.infile = "./test/testdata/#{file_name}"
+        pair.outfile = "./test/testdata/#{output_filename}"
+        file_pairs << pair
+      end
+    end
+    for pair in file_pairs
+      input = JSON.parse(File.read(pair.infile))
+      puts "Testing #{pair.infile}, #{pair.outfile}"
+      config = {api_id: '123', version_id: '1.0'}
+
+      ## Parse Masking Inputs
+      masking = []
+      if input['args'].include? 'query_string_masks'
+        for k, v in input['args']['query_string_masks']
+          masking << SpeakeasyRubySdk::MaskConfig.new(:query_params, [k], [v])
+        end
+      end
+      if input['args'].include? 'request_header_masks'
+        for k, v in input['args']['request_header_masks']
+          masking << SpeakeasyRubySdk::MaskConfig.new(:request_headers, [k], [v])
+        end
+      end
+      if input['args'].include? 'request_cookie_masks'
+        for k, v in input['args']['request_cookie_masks']
+          masking << SpeakeasyRubySdk::MaskConfig.new(:request_cookies, [k], [v])
+        end
+      end
+      if input['args'].include? 'request_field_masks_string'
+        for k, v in input['args']['request_field_masks_string']
+          masking << SpeakeasyRubySdk::MaskConfig.new(:request_body_string, [k], [v])
+        end
+      end
+      if input['args'].include? 'request_field_masks_number'
+        for k, v in input['args']['request_field_masks_number']
+          masking << SpeakeasyRubySdk::MaskConfig.new(:request_body_number, [k], [v])
+        end
+      end
+      if input['args'].include? 'response_header_masks'
+        for k, v in input['args']['response_header_masks']
+          masking << SpeakeasyRubySdk::MaskConfig.new(:response_headers, [k], [v])
+        end
+      end
+      if input['args'].include? 'response_cookie_masks'
+        for k, v in input['args']['response_cookie_masks']
+          masking << SpeakeasyRubySdk::MaskConfig.new(:response_cookies, [k], [v])
+        end
+      end
+      if input['args'].include? 'response_field_masks_string'
+        for k, v in input['args']['response_field_masks_string']
+          masking << SpeakeasyRubySdk::MaskConfig.new(:response_body_string, [k], [v])
+        end
+      end
+      if input['args'].include? 'response_field_masks_number'
+        for k, v in input['args']['response_field_masks_number']
+          masking << SpeakeasyRubySdk::MaskConfig.new(:response_body_number, [k], [v])
+        end
+      end
+
+      config[:masking] = masking
+
+
+      if input['fields'].include? 'max_capture_size'
+        config[:max_capture_size] = input['fields']['max_capture_size']
+      end
+
+      response_status = input['args']['response_status']
+
+      if input['args'].include? 'response_headers'
+        input['args']['response_headers'] = header_to_dict(input['args']['response_headers'])
+      else
+        input['args']['response_headers'] = {}
+      end
+
+      def simple_app input
+        ->(_env) { [input['args']['response_status'], input['args']['response_headers'], input['args']['response_body']]}
+      end
+
+      opts = {
+        "SERVER_PROTOCOL" => 'HTTP/1.1'
+      }
+      if input['args'].include? 'method'
+        opts[:method] = input['args']['method']
+      end
+      if input['args'].include? 'headers'
+        opts = opts.merge add_http_prefix(header_to_dict(input['args']['headers']))
+      end
+
+      if input['args'].include? 'request_start_time'
+        start_time = Time.parse(input['args']['request_start_time'])
+      else
+        start_time = Time.utc(2020, 1, 1, 0, 0, 0)
+      end
+
+      if input['args'].include? 'elapsed_time'
+        elapsed_time = input['args']['elapsed_time']
+      else
+        elapsed_time = 1
+      end
+
+      opts['time_utils'] = SpeakeasyRubySdk::TimeUtils.new(start_time, elapsed_time)
+
+      ## override Time.now for the cookie maxTime handling
+      Timecop.freeze(2020, 1, 1, 0, 0, 0)
+      
+      if input['args'].include? 'body'
+        rack_input = StringIO.open(input['args']['body'])
+        opts[:input] = rack_input
+      end
+      request = Rack::MockRequest.env_for(input['args']['url'], opts)
+      
+      subject = SpeakeasyRubySdk::Middleware.new(simple_app(input), config)
+      subject.ingest_client = Spy.mock(Ingest::IngestService::Stub)
+      
+      spy = Spy.on(subject.ingest_client, :ingest).and_return(nil)
+
+      status, _headers, _response = subject.call(request)
+
+      har = JSON.parse(spy.calls.first.args[0].har)
+      output = JSON.parse(File.read(pair.outfile))
+
+      assert_equal(har, output)
+
+      ## reset time mock
+      Timecop.return
+    end
+  end
+
+end

data/test/http_test.rb

@@ -0,0 +1,32 @@
+require 'test/unit'
+require 'rack/test'
+require 'json'
+require_relative '../lib/speakeasy_ruby_sdk'
+
+class BasicHttpTest < Test::Unit::TestCase
+  include Rack::Test::Methods
+
+  def simple_app
+    ->(_env) { [200, { 'content-type' => 'text/javascript' }, ActionDispatch::Response::RackBody.new(['All responses are OK'])] }
+  end
+
+  def fuller_app
+    ->(_env) { [200, {"X-Frame-Options"=>"SAMEORIGIN", "X-XSS-Protection"=>"0", "X-Content-Type-Options"=>"nosniff", "X-Download-Options"=>"noopen", "X-Permitted-Cross-Domain-Policies"=>"none", "Referrer-Policy"=>"strict-origin-when-cross-origin", "Content-Type"=>"application/json; charset=utf-8"}, ActionDispatch::Response::RackBody.new]}
+  end
+
+  def test_response_is_unchanged
+    request = Rack::MockRequest.env_for('/', method: :get)
+
+    subject = SpeakeasyRubySdk::Middleware.new(simple_app)
+    status, _headers, _response = subject.call(request)
+    assert_equal status, 200
+  end
+
+  def test_full_request
+    request = Rack::MockRequest.env_for('/', method: :get)
+    subject = SpeakeasyRubySdk::Middleware.new(fuller_app)
+    status, _headers, _response = subject.call(request)
+    assert_equal status, 200
+  end
+
+end

data/test/masker_test.rb

@@ -0,0 +1,124 @@
+require 'test/unit'
+require 'rack/test'
+require 'json'
+require_relative '../lib/speakeasy_ruby_sdk'
+
+class MaskerTest < Test::Unit::TestCase
+  include Rack::Test::Methods
+
+  def self.init_masker config
+    config = SpeakeasyRubySdk::Middleware::Config.default.merge config
+    SpeakeasyRubySdk::Masker.new config
+  end
+
+  def test_mask_query_param_simple
+    masker_config = {
+      routes: nil,
+      masking: [SpeakeasyRubySdk::MaskConfig.new(:query_params, 'carrot')]
+    }
+    masker = MaskerTest::init_masker(masker_config)
+
+    query_params = {'carrot': 'stick', 'parsnip': 'triangular'}
+
+    masked_params = masker.mask_query_params '', query_params
+
+    assert_equal(SpeakeasyRubySdk::Masker::SIMPLE_MASK, masked_params[:carrot])
+    assert_equal(query_params[:parsnip], masked_params[:parsnip]) # Should be unchanged
+  end
+
+  def test_mask_query_param_multiple_attrs
+    masker_config = {
+      routes: nil,
+      masking: [SpeakeasyRubySdk::MaskConfig.new(:query_params, ['carrot', 'parsnip', 'unknown'])]
+    }
+    masker = MaskerTest::init_masker(masker_config)
+
+    query_params = {'carrot': 'stick', 'parsnip': 'triangular'}
+
+    masked_params = masker.mask_query_params '', query_params
+
+    assert_equal(SpeakeasyRubySdk::Masker::SIMPLE_MASK, masked_params[:carrot])
+    assert_equal(SpeakeasyRubySdk::Masker::SIMPLE_MASK, masked_params[:parsnip])
+  end
+
+  def test_mask_query_param_custom_mask
+    masker_config = {
+      routes: nil,
+      masking: [SpeakeasyRubySdk::MaskConfig.new(:query_params, ['carrot'], ['__MASK__'])]
+    }
+    masker = MaskerTest::init_masker(masker_config)
+
+    query_params = {'carrot': 'stick', 'parsnip': 'triangular'}
+
+    masked_params = masker.mask_query_params '', query_params
+
+    assert_equal("__MASK__", masked_params[:carrot])
+    assert_equal(query_params[:parsnip], masked_params[:parsnip]) # Should be unchanged
+  end
+
+
+  def test_mask_response_cookies_simple
+    masker_config = {
+      masking: [SpeakeasyRubySdk::MaskConfig.new(:response_cookies, ['magic'])]
+    }
+    masker = MaskerTest::init_masker(masker_config)
+
+    response_cookies = HTTP::Cookie.parse("magic=itsnotmagic; path=/; max-age=86400; SameSite=Lax", 'http://localhost:8000')
+    
+    masked_cookies = masker.mask_response_cookies '', response_cookies
+    
+    first_cookie = masked_cookies[0]
+
+    assert_equal(SpeakeasyRubySdk::Masker::SIMPLE_MASK, first_cookie.value)
+
+  end
+
+  def test_mask_body_string_simple
+    masker_config = {
+      masking: [SpeakeasyRubySdk::MaskConfig.new(:request_body_string, ['sign_in'])]
+    }
+    masker = MaskerTest::init_masker(masker_config)
+
+    body = {'sign_in': 'wiley', 'nobody': 'somebody'}.to_json
+      
+    masked_body = masker.mask_request_body('', body)
+    assert !masked_body.include?('wiley')
+    assert masked_body.include?('somebody')
+  end
+
+  def test_mask_body_number_simple
+    masker_config = {
+      masking: [SpeakeasyRubySdk::MaskConfig.new(:request_body_number, ['sign_in'])]
+    }
+    masker = MaskerTest::init_masker(masker_config)
+
+    body = {
+      'sign_in': 5476,
+      'nobody': 'somebody'
+    }.to_json
+    masked_body = masker.mask_request_body('', body)
+    assert ! masked_body.include?('5476')
+    assert masked_body.include?('somebody')
+  end
+
+  def test_mask_body_number_and_string
+    masker_config = {
+      masking: [
+        SpeakeasyRubySdk::MaskConfig.new(:request_body_number, ['attr1']),
+        SpeakeasyRubySdk::MaskConfig.new(:request_body_string, ['attr2']),
+      ]
+    }
+    masker = MaskerTest::init_masker(masker_config)
+
+    body = {
+      'attr1': 5476,
+      'attr2': 'somebody',
+      'attr3': 'keep it'
+    }.to_json
+
+    masked_body = masker.mask_request_body('', body)
+    assert !masked_body.include?('5476')
+    assert !masked_body.include?('somebody')
+    assert masked_body.include?('keep it')
+  end
+end

data/test/testdata/captures_basic_request_and_no_response_body_input.json

@@ -0,0 +1,15 @@
+{
+  "name": "captures basic request and no response body",
+  "fields": {
+    "max_capture_size": 9437184
+  },
+  "args": {
+    "method": "GET",
+    "url": "http://test.com:8080/test",
+    "headers": [
+      { "key": "Host", "values": ["test.com"] },
+      { "key": "Accept-Encoding", "values": ["gzip, deflate"] },
+      { "key": "Connection", "values": ["close"] }
+    ]
+  }
+}

data/test/testdata/captures_basic_request_and_no_response_body_output.json

@@ -0,0 +1,61 @@
+{
+  "log": {
+    "version": "1.2",
+    "creator": {
+      "name": "SpeakeasyRubySdk",
+      "version": "0.0.1"
+    },
+    "entries": [
+      {
+        "startedDateTime": "2020-01-01T00:00:00.000000000Z",
+        "time": 1,
+        "request": {
+          "method": "GET",
+          "url": "http://test.com:8080/test",
+          "httpVersion": "HTTP/1.1",
+          "cookies": [],
+          "headers": [
+            {
+              "name": "accept-encoding",
+              "value": "gzip, deflate"
+            },
+            {
+              "name": "connection",
+              "value": "close"
+            },
+            {
+              "name": "host",
+              "value": "test.com"
+            }
+          ],
+          "queryString": [],
+          "headersSize": 67,
+          "bodySize": -1
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "HTTP/1.1",
+          "cookies": [],
+          "headers": [],
+          "content": {
+            "size": -1,
+            "mimeType": "application/octet-stream"
+          },
+          "redirectURL": "",
+          "headersSize": 0,
+          "bodySize": -1
+        },
+        "cache": {},
+        "timings": {
+          "send": -1,
+          "wait": -1,
+          "receive": -1
+        },
+        "serverIPAddress": "test.com",
+        "connection": "8080"
+      }
+    ],
+    "comment": "request capture for http://test.com:8080/test"
+  }
+}

data/test/testdata/captures_basic_request_and_response_input.json

@@ -0,0 +1,21 @@
+{
+  "name": "captures basic request and response",
+  "fields": {
+    "max_capture_size": 9437184
+  },
+  "args": {
+    "method": "GET",
+    "url": "http://test.com/test",
+    "response_status": 200,
+    "response_body": "test",
+    "headers": [
+      { "key": "Host", "values": ["test.com"] },
+      { "key": "Accept-Encoding", "values": ["gzip, deflate"] },
+      { "key": "Connection", "values": ["close"] }
+    ],
+    "response_headers": [
+      { "key": "Content-Type", "values": ["text/html; charset=utf-8"] },
+      { "key": "Content-Length", "values": ["4"] }
+    ]
+  }
+}

data/test/testdata/captures_basic_request_and_response_output.json

@@ -0,0 +1,70 @@
+{
+  "log": {
+    "version": "1.2",
+    "creator": {
+      "name": "SpeakeasyRubySdk",
+      "version": "0.0.1"
+    },
+    "entries": [
+      {
+        "startedDateTime": "2020-01-01T00:00:00.000000000Z",
+        "time": 1,
+        "request": {
+          "method": "GET",
+          "url": "http://test.com/test",
+          "httpVersion": "HTTP/1.1",
+          "cookies": [],
+          "headers": [
+            {
+              "name": "accept-encoding",
+              "value": "gzip, deflate"
+            },
+            {
+              "name": "connection",
+              "value": "close"
+            },
+            {
+              "name": "host",
+              "value": "test.com"
+            }
+          ],
+          "queryString": [],
+          "headersSize": 67,
+          "bodySize": -1
+        },
+        "response": {
+          "status": 200,
+          "statusText": "OK",
+          "httpVersion": "HTTP/1.1",
+          "cookies": [],
+          "headers": [
+            {
+              "name": "content-length",
+              "value": "4"
+            },
+            {
+              "name": "content-type",
+              "value": "text/html; charset=utf-8"
+            }
+          ],
+          "content": {
+            "size": 4,
+            "mimeType": "text/html; charset=utf-8",
+            "text": "test"
+          },
+          "redirectURL": "",
+          "headersSize": 59,
+          "bodySize": 4
+        },
+        "cache": {},
+        "timings": {
+          "send": -1,
+          "wait": -1,
+          "receive": -1
+        },
+        "serverIPAddress": "test.com"
+      }
+    ],
+    "comment": "request capture for http://test.com/test"
+  }
+}

data/test/testdata/captures_basic_request_and_response_with_different_content_types_input.json

@@ -0,0 +1,22 @@
+{
+  "name": "captures basic request and response with different content types",
+  "fields": {
+    "max_capture_size": 9437184
+  },
+  "args": {
+    "method": "GET",
+    "url": "http://test.com/test",
+    "headers": [
+      { "key": "Content-Type", "values": ["application/json"] },
+      { "key": "Host", "values": ["test.com"] },
+      { "key": "Accept-Encoding", "values": ["gzip, deflate"] },
+      { "key": "Connection", "values": ["close"] }
+    ],
+    "response_status": -1,
+    "response_body": "test",
+    "response_headers": [
+      { "key": "Content-Type", "values": ["text/plain; charset=utf-8"] },
+      { "key": "Content-Length", "values": ["4"] }
+    ]
+  }
+}