svcbase 0.1.16

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: ee9043e3f2173b8d9d6adfe661dd37f7db925a00912f9c9a15e7f74dede6ea74
+  data.tar.gz: 1be47052db3c376410cee00d62d16997b46eaf31e317a9010fdc34f52eb68199
+SHA512:
+  metadata.gz: ad56907c1f5572d9677280f743b18f04f1011b4556c5b2b1f658ff975ea9006d677e20e30d72d75017373f4a41b3cb8d4c3d383a48d70e01fd44fc655f3c5a49
+  data.tar.gz: 1fce1b726d8507e00072a976cfb227321389f80fd55d5ecbe1d50d578c3f0dfc1933ca6d1904b07db2ced7ad976fb489e43e651c0347d66f455cf6bd7f038d55

data/.gitignore

@@ -0,0 +1,12 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.rubocop.yml

@@ -0,0 +1,63 @@
+AllCops:
+  TargetRubyVersion: 2.5.1
+  Exclude:
+    - 'test/*'
+    - 'vendor/**/*'
+
+inherit_from: .rubocop_todo.yml
+
+Style/AccessModifierDeclarations:
+  EnforcedStyle: inline
+
+Metrics/CyclomaticComplexity:
+  Max: 12
+
+Metrics/PerceivedComplexity:
+  Max: 12
+
+# Grape idiom for defining APIs nearly guarantees long blocks,
+# so exclude our entire api subdir from this cop.
+Metrics/BlockLength:
+  Exclude:
+    - 'spec/**/*'
+
+Metrics/AbcSize:
+  Max: 30
+  Exclude:
+    - 'spec/**/*'
+
+Metrics/ClassLength:
+  Max: 125
+  Exclude:
+    - 'spec/**/*'
+
+Metrics/ModuleLength:
+  Max: 125
+  Exclude:
+    - 'spec/**/*'
+
+Metrics/LineLength:
+  Max: 115
+
+Metrics/MethodLength:
+  Max: 20
+
+Metrics/ParameterLists:
+  CountKeywordArgs: false
+
+Layout/LeadingCommentSpace:
+  Exclude:
+    - 'config.ru'
+
+Style/MixinGrouping:
+  Exclude:
+    - 'spec/**/*'
+
+Naming/UncommunicativeMethodParamName:
+  Exclude:
+    - 'spec/**/*'
+  AllowedNames:
+    - ip
+    - id
+    - tz
+    - db

data/.rubocop_todo.yml

@@ -0,0 +1,7 @@
+# This configuration was generated by
+# `rubocop --auto-gen-config`
+# on 2017-03-31 12:03:15 -0400 using RuboCop version 0.48.0.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.

data/.ruby-version

@@ -0,0 +1 @@
+2.5.1

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.5.0
+before_install: gem install bundler -v 1.15.4

data/Gemfile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in svcbase.gemspec
+gemspec

data/Jenkinsfile

@@ -0,0 +1,137 @@
+#!/usr/bin/env groovy
+
+// load helpers
+library 'jenkins-pipeline-libs'
+version = null
+is_master = ( "${env.BRANCH_NAME}" == "master" )
+
+node('docker && !windows') {
+    String gh_cid = scm.getUserRemoteConfigs()[0].getCredentialsId()
+    String scmUrl = scm.getUserRemoteConfigs()[0].getUrl()
+    String repo_name = getRepoFromURL(scmUrl)
+
+    properties([
+
+      // dont keep builds in Jenkins.
+      buildDiscarder( logRotator(artifactDaysToKeepStr: '',
+        artifactNumToKeepStr: '3',
+        daysToKeepStr: '',
+        numToKeepStr: '30') ),
+
+      // set url for diff links to gh
+      [ $class: 'GithubProjectProperty',
+        displayName: '',
+        projectUrlStr: "${getGitHubURL(scmUrl)}" ]
+    ])
+
+    try {
+      stage('Checkout') {
+        deleteDir()
+        checkout scm
+      }
+
+      //invoke build steps
+      stage('Builds') {
+        //set display name
+        currentBuild.displayName = getVersion()
+
+        ansiColor('xterm') {
+          sh 'gem build svcbase.gemspec'
+        } // ansiColor
+      } // stage
+      //unit tests
+
+      stage('Unit Tests') {
+        String packages = "tzdata ruby-dev zlib-dev xz-dev build-base libxml2-dev libxslt-dev git"
+        String env_cmd = "apk update && apk upgrade && apk add ${packages}"
+        String test_cmd = "gem install bundler; cd /usr/src/app; bundle install --with development test; bundle exec rake"
+        String full_cmd = "${env_cmd}; ${test_cmd}"
+        String test_container = "svcbase-test-${env.BRANCH_NAME}-${env.BUILD_NUMBER}"
+        String docker_img = "ruby:2.5.1-alpine"
+        String container_dir = "/usr/src/app"
+
+        sh """
+          #!/bin/bash -le
+          docker pull ${docker_img}
+          docker run -d -it --name ${test_container} ${docker_img}
+          docker cp $WORKSPACE ${test_container}:${container_dir}
+          docker exec ${test_container} ash -c "${env_cmd}"
+          docker exec ${test_container} ash -c "${test_cmd}"
+          # copy results back
+          docker cp ${test_container}:/usr/src/app/coverage .
+          # now stop container and image
+          docker stop ${test_container}
+          docker rm ${test_container}  
+        """
+
+        // publish ut results
+        publishHTML([ allowMissing: false,
+               alwaysLinkToLastBuild: true,
+               keepAll: true,
+               reportDir: 'coverage',
+               reportFiles: 'index.html',
+               reportName: 'RCov Report'
+        ])
+      } //stage
+
+      // invoke any steps specific to the master(release) branch
+      if ( is_master ) {
+        stage('Tag & Publish') {
+          withCredentials([usernameColonPassword(credentialsId: 'bf127b02-43c5-4ca9-8523-c2f22372ea7a', variable: 'ART_KEY')]) {
+            String art_url = "https://artifactory.secureauth.com/artifactory/api/gems/rubygems-local"
+
+            // set creds and push to artifactory (is there a better way :/ )
+            sh """
+              mkdir -p ~/.gem
+              curl -s -u${ART_KEY} ${art_url}/api/v1/api_key.yaml > ~/.gem/credentials
+              chmod 600 ~/.gem/credentials
+              gem push svcbase-${getVersion()}.gem --host ${art_url}
+            """
+          }
+          applyTag(gh_cid, "${getVersion()}", scmUrl)
+        }
+      }
+    }
+    catch (e) {
+      // set status to failed
+      currentBuild.result = "FAILED"
+      throw e
+    }
+    finally {
+
+      // wrap it up
+      stage('Archive, Clean & Notify') {
+        // remove credentials
+        sh "rm -f ~/.gem/credentials"
+
+        String recipients = '#cloudteam_notify'
+        String status = currentBuild.result ?: 'SUCCESS'
+        String msg = "${env.JOB_NAME} - <${env.BUILD_URL}|${getVersion()}" +
+          "> - ${status}\n\n${getChangeString()}"
+
+        //send via library
+        notifySlack {
+          buildStatus = status
+          channel = recipients
+          message = msg
+        }
+
+        deleteDir()
+      }
+    }
+}
+
+// note - must be called after scm checkout
+String getVersion(){
+  if (!version) {
+    if ( is_master ) {
+      def matches = readFile('lib/svcbase/version.rb') =~ /VERSION *= *['"]?([0-9\.]+)['"]?/
+      version = matches ? matches[0][1] : null
+    } else {
+      version = "${env.BUILD_NUMBER}"
+    }
+  }
+
+  return version
+}
+

data/README.md

@@ -0,0 +1,61 @@
+# Svcbase
+
+This is a base class for Grape apps following our API methodology.  
+
+It includes
+
+* API Logging, including sensitive data filtering and periodic stat output
+* Configuration (file) support
+* Locale support
+* API request helpers for common data
+* Request ID tracking
+
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'svcbase'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install svcbase
+
+## Usage
+
+You can reference the entire stack by simply doing a 
+
+```ruby
+require 'svcbase'
+```
+but it is recommended that only the relevant parts are required. For example, to start the behind-the-scenes thread server, simply
+
+```ruby
+require 'svcbase/server'
+```
+and then later reference ```
+Core::Server.
+```
+
+To create a new top-level API, first
+
+```ruby
+require 'svcbase/api/base'
+```
+and then create a class that inherits from ```
+Core::APIBase
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. 
+
+

data/Rakefile

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec, %i[dir file]) do |spec, args|
+  args.with_defaults dir: '*', file: '*_spec.rb'
+  spec.pattern = "spec/#{args[:dir]}/**/#{args[:file]}"
+end
+
+require 'rubocop/rake_task'
+RuboCop::RakeTask.new(:rubocop) do |task|
+  task.options << '--display-cop-names'
+  task.options << '--parallel'
+end
+
+task default: %i[rubocop spec]

data/bin/console

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'svcbase'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require 'pry'
+# Pry.start
+
+require 'irb'
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/config/configs/base.yml

@@ -0,0 +1,5 @@
+---
+WORKER_SHUTDOWN_GRACE_PERIOD: 15                  # Seconds to wait for a worker to finish when shutting down
+STAT_LOG_SECONDS: 3600                            # How often to log endpoint statistics
+THRESHOLDER_SECONDS: 60                           # Default thresholder window
+THRESHOLDER_COUNT: 60                             # Default thresholder count

data/lib/svcbase/api/base.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require 'grape'
+require 'svcbase/exceptions'
+require 'http_accept_language'
+
+require 'i18n'
+
+require 'svcbase/api/requesthelpers'
+
+require 'svcbase/corelogger'
+require 'svcbase/middleware/dateheader'
+require 'svcbase/middleware/apilogger'
+require 'svcbase/middleware/requestid'
+
+module Core
+  # API container
+  class APIBase < Grape::API
+    format :json
+    rescue_from Grape::Exceptions::Base do |e|
+      # grape uses 'error' but we want 'message', so translate
+      error!({ status: :error, error: { code: :grape, message: e.message } }, e.status, e.headers || {})
+    end
+    rescue_from Core::Exceptions::Error do |e|
+      # our errors give us a method to call to get the response
+      error!(e.response, e.status, e.headers)
+    end
+    rescue_from :all do |e|
+      # anything that gets here is, by definition, unexpected.
+      # log the error details at FATAL, but don't return them to the caller.
+      # instead, give them a generic 500 error
+      log.fatal e
+      error!({ status: :error, error: { code: :internal_server_error } }, 500)
+    end
+
+    # Set up translations for our messages in addition to whatever else might be set up
+    I18n.enforce_available_locales ||= false
+    I18n::Backend::Simple.send(:include, I18n::Backend::Fallbacks)
+    I18n.default_locale ||= :en
+    I18n.load_path << Dir.glob(File.join(File.dirname(__FILE__), '../../../locale', '*.{rb,yml}'))
+
+    use Core::AddDateHeader
+    use RequestStore::Middleware
+    use Core::RequestId
+    use Core::ApiLogger, logger: Core::Logger.instance, filter: Core::ParamFilter.new
+    use HttpAcceptLanguage::Middleware
+    do_not_route_head!
+    do_not_route_options!
+
+    helpers RequestHelpers
+
+    # restrict payload content-type
+    # Although grape provides "format :json", that doesn't stop rack from handling
+    # certain types before grape even gets involved (x-www-form-urlencoded, etc.)
+    # This will enforce the rule, regardless.
+    before do
+      # set the language for this Thread based on request headers
+      I18n.locale = accept_language unless accept_language.nil?
+
+      # authenticate!
+      next unless env && env['CONTENT_LENGTH'] && env['CONTENT_LENGTH'] != '0'
+      # NB. would be nice if we could figure out how to translate the declared format
+      # which lives in env[Grape::Env::API_FORMAT] into a content-type so we don't
+      # have to hardcode application/json and we don't have to add stuff to the
+      # endpoint description.
+      allowed_content_types = Array(route_desc[:allowed_content_types])
+      allowed_content_types = ['application/json'] if allowed_content_types.empty?
+      unless allowed_content_types.include?(env['CONTENT_TYPE'])
+        raise Core::Exceptions::UnsupportedMediaType, msg: :invalid, msgobj: :content_type
+      end
+    end
+  end
+end

data/lib/svcbase/api/etag.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+# compute etag hashes based on ID and updated_at of all items passed
+module Etag
+  def self.weak_hash(*args)
+    val = args.flatten.compact.map do |item|
+      item.id.to_s + item.updated_at.to_i.to_s
+    end.join
+    "W/\"#{val.to_i.to_s(32)}\""
+  end
+end

data/lib/svcbase/api/requesthelpers.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require 'request_store'
+
+# request helpers
+module RequestHelpers
+  extend Grape::API::Helpers
+
+  def http_request_id
+    RequestStore.store[:http_request_id]
+  end
+
+  def input_data
+    env[Grape::Env::API_REQUEST_BODY]
+  end
+
+  # Get accept-language
+  def accept_language
+    available = ::LOCALE_LIST if defined? ::LOCALE_LIST
+    env.http_accept_language.compatible_language_from(available)
+  end
+
+  def client_ip
+    request.env['HTTP_X_FORWARDED_FOR'] || request.env['REMOTE_ADDR'] || '0.0.0.0'
+  end
+
+  def user_agent
+    request.env['HTTP_USER_AGENT'] || 'unknown'
+  end
+
+  def route_desc
+    route_setting(:description) || {}
+  end
+end

data/lib/svcbase/appversion.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Core
+  # version info of application (from metadata)
+  class Version
+    private_class_method def self.safe_set(name, file)
+      const_set name, File.read(file).strip
+    rescue StandardError
+      const_set name, "no #{name}"
+    end
+
+    safe_set :ID, './git-commit'
+    safe_set :TAG, './git-tag'
+  end
+end

data/lib/svcbase/config/config_helper.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+# helpers module to DRY out config fetches with optional default
+#
+# - iterates through args looking for first symbol that is present.
+# - non-symbols are treated as literals instead of lookups (so only one matters)
+# - raises an error if value is present but not valid for type (Integer or Float)
+# - raises an error if value is missing and no default specified
+# - if supplied, default must either be valid for type or nil
+module ConfigHelper
+  def get_x!(*args)
+    while args.length.positive?
+      k = args.shift
+      if k.is_a? Symbol
+        k = k.to_s
+        return yield(self[k]) if key? k
+      else
+        raise 'default not last argument' unless args.empty?
+        return nil if k.nil?
+        return yield(k)
+      end
+    end
+    raise 'value not found'
+  end
+
+  def get_i!(*args)
+    get_x!(*args) { |v| Integer(v) }
+  end
+
+  def get_f!(*args)
+    get_x!(*args) { |v| Float(v) }
+  end
+
+  def get_s!(*args)
+    get_x!(*args, &:to_s)
+  end
+
+  def get_b!(*args)
+    get_x!(*args) do |v|
+      return v if [true, false].include?(v)
+      raise ArgumentError, 'not a bool or string' unless v.is_a? String
+      case v.downcase
+      when 'true'  then true
+      when 'false' then false
+      else raise ArgumentError, 'not true or false'
+      end
+    end
+  end
+end

data/lib/svcbase/config.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require 'active_support/core_ext/hash/deep_merge'
+require 'forwardable'
+require 'pathname'
+require 'singleton'
+require 'yaml'
+
+require_relative 'config/config_helper'
+
+# Global config wrapper
+module Core
+  # configuration wrapper
+  class Config
+    include Singleton
+    include ConfigHelper
+
+    attr_reader :config_name
+
+    def initialize
+      @config_name = ENV['SA_ENV'] ||= 'local'
+      @config = {}
+      ['base', @config_name].each do |name|
+        f = Pathname.new(CONFIG_DIR).join("#{name}.yml")
+        c = YAML.load_file(f)
+        next unless c
+        @config.deep_merge!(c)
+      rescue StandardError => e
+        STDERR.puts "Error reading config file #{e}"
+        raise
+      end
+    end
+
+    def key?(key)
+      ENV.key?(key) || @config.key?(key)
+    end
+
+    def [](key)
+      ENV.key?(key) ? ENV[key] : @config[key]
+    end
+
+    # forward class method calls to the instance so we can do Config['VARNAME'] or Config.get_i!(:FOO)
+    class << self
+      extend Forwardable
+      def_delegators :instance, *Config.instance_methods(false), *ConfigHelper.instance_methods(false)
+    end
+  end
+end
+
+Core::Config.instance

data/lib/svcbase/corelogger.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require 'logger'
+require 'singleton'
+require 'svcbase/formatter'
+
+# define our own custom logger
+module Core
+  # our logger
+  class Logger < ::Logger
+    include Singleton
+    def initialize
+      super($stdout)
+      self.formatter = Core::Formatters::Log.new
+    end
+
+    def none(*_unused)
+      # intentional no-op.  Sequel will log things at a defined log level
+      # but we want to be able to log long-running queries without logging
+      # ALL of them.  this gives us a method to /dev/zero messages for fast
+      # queries
+    end
+  end
+
+  # truncate long parameters, mask password
+  class ParamFilter
+    def filter(paramhash, maskkeys)
+      paramhash.each do |key, value|
+        next paramhash[key] = '*' if maskkeys&.include?(key.to_sym)
+        next
+
+        case value
+        when String
+          paramhash[key] = "#{value[1..128]}... (#{value.length})" if value.length > 128
+        when Hash, Array
+          value = value.to_json.to_s
+          paramhash[key] = "#{value[1..128]}... (#{value.length})" if value.length > 128
+        end
+      end
+      paramhash
+    end
+  end
+end
+
+# add a logger method to all objects so that we can access our logger anywhere
+class Object
+  def log
+    Core::Logger.instance
+  end
+
+  alias logger log
+end

data/lib/svcbase/dumpstats.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require 'singleton'
+require 'svcbase/stats'
+require 'svcbase/worker'
+
+module Core
+  module Workers
+    # Stats collector
+    class DumpStats
+      include Worker
+      include Singleton
+
+      private def run_at_startup
+        false
+      end
+
+      private def interval_seconds
+        Config.get_f!(:STAT_LOG_SECONDS, 3600)
+      end
+
+      private def do_work
+        Core::Stats.log_and_reset
+      end
+    end
+  end
+end