opener-webservice 2.1.0 → 2.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 32302b9a2aac215c69670cd55e4c28d22901df9c
-  data.tar.gz: d24fb9f1fc3113ff70685bd1c2ce440eced844c6
+SHA256:
+  metadata.gz: e9d2c615347f8c18229b705527f0256122040331a5db76517edb5e7da0bb0ee7
+  data.tar.gz: 53b8a41ca1dc581f00296272c05479c62a3e671cfb416b1b4bb43fbab660e7dd
 SHA512:
-  metadata.gz: 5c96ce12b9449ee865ee208180fed338a7bdb9077741cecc23a618cd31fe499570a0fa750f905ca1f17ebebb09294a9a431fa83d7c43910a6890a83779c68285
-  data.tar.gz: 602204c799cca06cf486f1fb44c1f6d2ea2a8f53417916313a08e7e80a035ef2e6f4c2eb75f784ce0f3ac7389c121851b041ef8b7a628493f2e1aaec2334663a
+  metadata.gz: 1f8d549b4dc5535635dca23ed3950eefbe39c7ea3c26a5572aecee715724cdc1a7cca36cb8547951bf95e8ad034d38c518a3e94a489834025884abd8d4596dd5
+  data.tar.gz: f4760946b5cdf462346bcbf5271b428a4e56489839052af0377d9dff3190b5d4465a1ee17a7b55c3c97759c534fb45ef4b317fd85efef6498ff9741eeffd1ea7

data/README.md

@@ -93,6 +93,8 @@ receives the following parameters:
 
 * A supported Ruby version (see below)
 * Amazon S3 (only when one wants to store ouput in S3)
+* libarchive (for running the tests and such), on Debian/Ubuntu based systems
+  this can be installed using `sudo apt-get install libarchive-dev`
 
 The following Ruby versions are supported:
 

data/config/newrelic.yml

@@ -0,0 +1,206 @@
+# Here are the settings that are common to all environments
+common: &default_settings
+  # ============================== LICENSE KEY ===============================
+
+  # You must specify the license key associated with your New Relic
+  # account.  This key binds your Agent's data to your account in the
+  # New Relic service.
+  license_key: <%= ENV['NEWRELIC_TOKEN'] %>
+
+  # Agent Enabled (Ruby/Rails Only)
+  # Use this setting to force the agent to run or not run.
+  # Default is 'auto' which means the agent will install and run only
+  # if a valid dispatcher such as Mongrel is running.  This prevents
+  # it from running with Rake or the console.  Set to false to
+  # completely turn the agent off regardless of the other settings.
+  # Valid values are true, false and auto.
+  #
+  agent_enabled: true
+
+  # Application Name Set this to be the name of your application as
+  # you'd like it show up in New Relic. The service will then auto-map
+  # instances of your application into an "application" on your
+  # dashboard page. If you want to map this instance into multiple
+  # apps, like "AJAX Requests" and "All UI" then specify a semicolon
+  # separated list of up to three distinct names, or a yaml list.
+  # Defaults to the capitalized RAILS_ENV or RACK_ENV (i.e.,
+  # Production, Staging, etc)
+  #
+  # Example:
+  #
+  #   app_name:
+  #       - Ajax Service
+  #       - All Services
+  #
+  app_name: "<%= ENV['APP_NAME'] %>"
+
+  # When "true", the agent collects performance data about your
+  # application and reports this data to the New Relic service at
+  # newrelic.com. This global switch is normally overridden for each
+  # environment below. (formerly called 'enabled')
+  # monitor_mode: true
+
+  # Developer mode should be off in every environment but
+  # development as it has very high overhead in memory.
+  developer_mode: false
+
+  # The newrelic agent generates its own log file to keep its logging
+  # information separate from that of your application. Specify its
+  # log level here.
+  log_level: error
+
+  # Optionally set the path to the log file This is expanded from the
+  # root directory (may be relative or absolute, e.g. 'log/' or
+  # '/var/log/') The agent will attempt to create this directory if it
+  # does not exist.
+  log_file_path: STDOUT
+
+  # Optionally set the name of the log file, defaults to 'newrelic_agent.log'
+  #log_file_name: 'newrelic_agent.log'
+
+  # The newrelic agent communicates with the service via https by default.  This
+  # prevents eavesdropping on the performance metrics transmitted by the agent.
+  # The encryption required by SSL introduces a nominal amount of CPU overhead,
+  # which is performed asynchronously in a background thread.  If you'd prefer
+  # to send your metrics over http uncomment the following line.
+  ssl: true
+
+  #============================== Browser Monitoring ===============================
+  # New Relic Real User Monitoring gives you insight into the performance real users are
+  # experiencing with your website. This is accomplished by measuring the time it takes for
+  # your users' browsers to download and render your web pages by injecting a small amount
+  # of JavaScript code into the header and footer of each page.
+  browser_monitoring:
+      # By default the agent automatically injects the monitoring JavaScript
+      # into web pages. Set this attribute to false to turn off this behavior.
+      auto_instrument: false
+
+  # Proxy settings for connecting to the New Relic server.
+  #
+  # If a proxy is used, the host setting is required.  Other settings
+  # are optional. Default port is 8080.
+  #
+  # proxy_host: hostname
+  # proxy_port: 8080
+  # proxy_user:
+  # proxy_pass:
+
+  # The agent can optionally log all data it sends to New Relic servers to a
+  # separate log file for human inspection and auditing purposes. To enable this
+  # feature, change 'enabled' below to true.
+  # See: https://newrelic.com/docs/ruby/audit-log
+  audit_log:
+    enabled: false
+
+  # Tells transaction tracer and error collector (when enabled)
+  # whether or not to capture HTTP params.  When true, frameworks can
+  # exclude HTTP parameters from being captured.
+  # Rails: the RoR filter_parameter_logging excludes parameters
+  # Java: create a config setting called "ignored_params" and set it to
+  #     a comma separated list of HTTP parameter names.
+  #     ex: ignored_params: credit_card, ssn, password
+  capture_params: false
+
+  # Transaction tracer captures deep information about slow
+  # transactions and sends this to the New Relic service once a
+  # minute. Included in the transaction is the exact call sequence of
+  # the transactions including any SQL statements issued.
+  transaction_tracer:
+
+    # Transaction tracer is enabled by default. Set this to false to
+    # turn it off. This feature is only available at the Professional
+    # and above product levels.
+    enabled: true
+
+    # Threshold in seconds for when to collect a transaction
+    # trace. When the response time of a controller action exceeds
+    # this threshold, a transaction trace will be recorded and sent to
+    # New Relic. Valid values are any float value, or (default) "apdex_f",
+    # which will use the threshold for an dissatisfying Apdex
+    # controller action - four times the Apdex T value.
+    transaction_threshold: 0
+
+    # When transaction tracer is on, SQL statements can optionally be
+    # recorded. The recorder has three modes, "off" which sends no
+    # SQL, "raw" which sends the SQL statement in its original form,
+    # and "obfuscated", which strips out numeric and string literals.
+    record_sql: obfuscated
+
+    # Threshold in seconds for when to collect stack trace for a SQL
+    # call. In other words, when SQL statements exceed this threshold,
+    # then capture and send to New Relic the current stack trace. This is
+    # helpful for pinpointing where long SQL calls originate from.
+    stack_trace_threshold: 0.500
+
+    # Determines whether the agent will capture query plans for slow
+    # SQL queries.  Only supported in mysql and postgres.  Should be
+    # set to false when using other adapters.
+    # explain_enabled: true
+
+    # Threshold for query execution time below which query plans will
+    # not be captured.  Relevant only when `explain_enabled` is true.
+    # explain_threshold: 0.5
+
+  # Error collector captures information about uncaught exceptions and
+  # sends them to New Relic for viewing
+  error_collector:
+
+    # Error collector is enabled by default. Set this to false to turn
+    # it off. This feature is only available at the Professional and above
+    # product levels.
+    enabled: false
+
+    # To stop specific errors from reporting to New Relic, set this property
+    # to comma-separated values.  Default is to ignore routing errors,
+    # which are how 404's get triggered.
+    ignore_errors: "ActionController::RoutingError,Sinatra::NotFound"
+
+  # If you're interested in capturing memcache keys as though they
+  # were SQL uncomment this flag. Note that this does increase
+  # overhead slightly on every memcached call, and can have security
+  # implications if your memcached keys are sensitive
+  # capture_memcache_keys: true
+
+# Application Environments
+# ------------------------------------------
+# Environment-specific settings are in this section.
+# For Rails applications, RAILS_ENV is used to determine the environment.
+# For Java applications, pass -Dnewrelic.environment <environment> to set
+# the environment.
+
+# NOTE if your application has other named environments, you should
+# provide newrelic configuration settings for these environments here.
+
+development:
+  <<: *default_settings
+  # Turn on communication to New Relic service in development mode
+  monitor_mode: false
+  app_name: "<%= ENV['APP_NAME'] %> (Development)"
+
+test:
+  <<: *default_settings
+  # It almost never makes sense to turn on the agent when running
+  # unit, functional or integration tests or the like.
+  monitor_mode: false
+
+# Turn on the agent in production for 24x7 monitoring. NewRelic
+# testing shows an average performance impact of < 5 ms per
+# transaction, you can leave this on all the time without
+# incurring any user-visible performance degradation.
+production:
+  <<: *default_settings
+  monitor_mode: true
+
+  error_collector:
+    enabled: true
+
+# Many applications have a staging environment which behaves
+# identically to production. Support for that environment is provided
+# here.  By default, the staging environment has the agent turned on.
+staging:
+  <<: *default_settings
+  monitor_mode: true
+  app_name: "<%= ENV['APP_NAME'] %> (Staging)"
+
+  error_collector:
+    enabled: true

data/lib/opener/webservice.rb

@@ -1,6 +1,7 @@
 require 'securerandom'
 require 'json'
 
+require 'aws-sdk'
 require 'puma/cli'
 require 'slop'
 require 'sinatra/base'
@@ -17,3 +18,4 @@ require_relative 'webservice/error_handler'
 require_relative 'webservice/input_extractor'
 require_relative 'webservice/input_sanitizer'
 require_relative 'webservice/option_parser'
+require_relative 'webservice/transaction'

data/lib/opener/webservice/configuration.rb

@@ -68,6 +68,15 @@ module Opener
         return !!ENV['ROLLBAR_TOKEN']
       end
 
+      ##
+      # Returns `true` if New Relic monitoring should be enabled.
+      #
+      # @return [TrueClass|FalseClass]
+      #
+      def self.newrelic?
+        return !!ENV['NEWRELIC_TOKEN']
+      end
+
       ##
       # Configures Rollbar.
       #

data/lib/opener/webservice/option_parser.rb

@@ -114,8 +114,7 @@ Puma Options:
             abort
           end
 
-          on :b=,
-            :bucket=,
+          on :bucket=,
             'The S3 bucket to store output in',
             :as => String
 
@@ -159,6 +158,13 @@ Puma Options:
               GC::Profiler.enable
             end
 
+            if Configuration.syslog?
+              Core::Syslog.open(
+                ENV['APP_NAME'],
+                ::Syslog::LOG_CONS | ::Syslog::LOG_PID
+              )
+            end
+
             Configuration.configure_rollbar
 
             # Puma on JRuby does some weird stuff with forking/exec. As a result

data/lib/opener/webservice/server.rb

@@ -56,7 +56,12 @@ module Opener
       end
 
       error do
-        Rollbar.report_exception(env['sinatra.error'])
+        Rollbar.error(
+          env['sinatra.error'],
+          :parameters => Transaction.current.parameters
+        )
+
+        Transaction.reset_current
 
         halt(
           500,
@@ -145,6 +150,8 @@ module Opener
 
         content_type(ctype)
 
+        Transaction.reset_current
+
         return output
       end
 
@@ -159,6 +166,11 @@ module Opener
         request_id = options['request_id'] || SecureRandom.hex
         final_url  = options['callbacks'].last
 
+        Core::Syslog.info(
+          "Processing asynchronous request with final URL #{final_url}",
+          :request_id => request_id
+        )
+
         async { analyze_async(options, request_id) }
 
         content_type :json
@@ -177,6 +189,8 @@ module Opener
       # @return [Array]
       #
       def analyze(options)
+        add_transaction_parameters(options)
+
         comp_options = InputSanitizer.new.whitelist_options(
           options,
           self.class.accepted_params
@@ -236,6 +250,11 @@ module Opener
         new_payload.delete('input')
 
         if Configuration.output_bucket
+          Core::Syslog.info(
+            "Uploading output to s3://#{Configuration.output_bucket}",
+            :request_id => request_id
+          )
+
           uploader = Uploader.new
           object   = uploader.upload(request_id, output, options['metadata'])
 
@@ -244,9 +263,30 @@ module Opener
           new_payload['input'] = output
         end
 
+        Core::Syslog.info(
+          "Submitting output to #{next_url}",
+          :request_id => request_id
+        )
+
         CallbackHandler.new.post(next_url, new_payload)
       end
 
+      ##
+      # @param [Hash] options
+      #
+      def add_transaction_parameters(options)
+        # If raw input is given we'll trim it so the payload isn't too large for
+        # Rollbar/New Relic. This uses Hash#merge so we don't modify the
+        # original options variable.
+        if options['input']
+          options = options.merge(
+            'input' => options['input'].byteslice(0, 256)
+          )
+        end
+
+        Transaction.current.add_parameters(options)
+      end
+
       ##
       # Returns a Hash containing the parameters from a JSON payload. The keys
       # of this Hash are returned as _strings_ to prevent Symbol DOS attacks.

data/lib/opener/webservice/transaction.rb

@@ -0,0 +1,56 @@
+module Opener
+  module Webservice
+    ##
+    # Class for storing information of a single transaction in a thread.
+    #
+    # @!attribute [r] parameters
+    #  @return [Hash]
+    #
+    class Transaction
+      attr_reader :parameters
+
+      ##
+      # The name of the key to store the current transaction in.
+      #
+      # @return [Symbol]
+      #
+      THREAD_KEY = :opener_webservices_transaction
+
+      ##
+      # Returns the current transaction.
+      #
+      # @return [Opener::Daemons::Transaction]
+      #
+      def self.current
+        return Thread.current[THREAD_KEY] ||= new
+      end
+
+      ##
+      # Removes the current transaction
+      #
+      def self.reset_current
+        Thread.current[THREAD_KEY] = nil
+      end
+
+      def initialize
+        @parameters = {}
+      end
+
+      ##
+      # Merges the given parameters with the existing ones.
+      #
+      # If New Relic is enabled the parameters are also added to the current
+      # New Relic transaction.
+      #
+      # @param [Hash] parameters
+      #
+      def add_parameters(parameters = {})
+        @parameters = @parameters.merge(parameters)
+
+        if Configuration.newrelic?
+          NewRelic::Agent.add_custom_parameters(parameters)
+        end
+      end
+    end # Transaction
+  end # Webservice
+end # Opener

data/lib/opener/webservice/version.rb

@@ -1,5 +1,5 @@
 module Opener
   module Webservice
-    VERSION = '2.1.0'
+    VERSION = '2.2.1'
   end # Webservice
 end # Opener

data/opener-webservice.gemspec

@@ -10,6 +10,7 @@ Gem::Specification.new do |spec|
   spec.license = 'Apache 2.0'
 
   spec.files = Dir.glob([
+    'config/**/*',
     'lib/**/*',
     '*.gemspec',
     'README.md',
@@ -17,16 +18,16 @@ Gem::Specification.new do |spec|
   ]).select { |file| File.file?(file) }
 
   spec.add_dependency 'sinatra', '~> 1.4.3'
-  spec.add_dependency 'uuidtools'
   spec.add_dependency 'json'
   spec.add_dependency 'opener-callback-handler', '~> 1.0'
   spec.add_dependency 'httpclient', ['~> 2.0', '>= 2.5.3.3']
-  spec.add_dependency 'opener-core', '~> 2.0'
+  spec.add_dependency 'opener-core', '~> 2.3'
   spec.add_dependency 'puma'
   spec.add_dependency 'slop', '~> 3.0'
+  spec.add_dependency 'aws-sdk', '~> 2.0'
 
-  spec.add_dependency 'newrelic_rpm'
-  spec.add_dependency 'rollbar', '~> 1.0'
+  spec.add_dependency 'newrelic_rpm', '~> 3.0'
+  spec.add_dependency 'rollbar', '~> 3.0'
 
   spec.add_development_dependency 'bundler', '~> 1.3'
   spec.add_development_dependency 'rake'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: opener-webservice
 version: !ruby/object:Gem::Version
-  version: 2.1.0
+  version: 2.2.1
 platform: ruby
 authors:
 - development@olery.com
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2014-11-21 00:00:00.000000000 Z
+date: 2021-03-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sinatra
@@ -24,20 +24,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.4.3
-- !ruby/object:Gem::Dependency
-  name: uuidtools
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
@@ -92,14 +78,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '2.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '2.3'
 - !ruby/object:Gem::Dependency
   name: puma
   requirement: !ruby/object:Gem::Requirement
@@ -128,34 +114,48 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: newrelic_rpm
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: rollbar
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '3.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -227,13 +227,14 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 description: Basic webservice hooks for the OpeNER toolchain
-email: 
+email:
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
 - LICENSE.txt
 - README.md
+- config/newrelic.yml
 - lib/opener/webservice.rb
 - lib/opener/webservice/configuration.rb
 - lib/opener/webservice/error_handler.rb
@@ -241,14 +242,15 @@ files:
 - lib/opener/webservice/input_sanitizer.rb
 - lib/opener/webservice/option_parser.rb
 - lib/opener/webservice/server.rb
+- lib/opener/webservice/transaction.rb
 - lib/opener/webservice/uploader.rb
 - lib/opener/webservice/version.rb
 - opener-webservice.gemspec
-homepage: 
+homepage:
 licenses:
 - Apache 2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -263,10 +265,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.2.2
-signing_key: 
+rubygems_version: 3.2.14
+signing_key:
 specification_version: 4
 summary: Basic webservice hooks for the OpeNER toolchain
 test_files: []
-has_rdoc: