appoptics_apm 4.8.0 → 4.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d0ccc84ee6065cc578c0b3dd768825b56a9cd17a1661e4df703621cec10e03c7
-  data.tar.gz: f981f8b2dd4e858b52dc71ad8055c931544a3428a35ca13f30575447fb4a67d9
+  metadata.gz: c4b87b6581745b8485f7ddb38f17cb6eab85244a950e7d67cecfe0e97cdbd0bc
+  data.tar.gz: dce765afa7ceb92b523c448cd9b1b10efc1b6c5bb0f2f92842dc6af3c1934b27
 SHA512:
-  metadata.gz: 8636c892f94298fb8851641566ca7ccef1f72888c4f37242898993e68278f09904b231c75271a2448cef3530ab7477b36cab000644ab24e605eee9ba0f377c6e
-  data.tar.gz: 96b455b4744604a8d52ce794c070bca3b95543a0e820454d86e3a55ab49a9f22232b9386cecf4bdd1d97585f5e4fdb4e8d796af9ee4cf5138a67736ee0cefedd
+  metadata.gz: d9b446519730872b09136e9b80e546a68dce40abf52fbb0e5ac139244c6ed3c7fa8ef45eb855714d8e11924886d0b972aa04332cec6342339c65772f5b7aec0f
+  data.tar.gz: 80df368e2ed2c868348ba5e7e9b24dfa2498d6f0cb703d568fbf8310879d06fee5ac59187a70747f04c040d4d54df2b8925faa856dc5a01b036375f21edacfbe

data/.gitignore

@@ -1,5 +1,5 @@
 oboe*.gem
-Gemfile.lock
+*.lock
 gemfiles/*.lock
 .ruby-version
 *~
@@ -27,3 +27,7 @@ coverage
 doc
 .*byebug*
 gemfiles/vendor*
+scrap.rb
+scarp_gemfile
+.irbrc
+*.code-workspace

data/.irbrc

@@ -0,0 +1,3 @@
+require 'irb/ext/save-history'
+IRB.conf[:SAVE_HISTORY] = 2000
+IRB.conf[:HISTORY_FILE] = "#{ENV['HOME']}/.irb-history"

data/.travis.yml

@@ -11,8 +11,8 @@ env:
   - DBTYPE=mysql
 
 rvm:
-  - 2.6.2
-  - 2.5.3
+  - 2.6.4
+  - 2.5.5
   - 2.4.5
   - ruby-head
 #  - jruby-9.0.5.0
@@ -23,6 +23,7 @@ gemfile:
   - gemfiles/instrumentation_mocked.gemfile
   - gemfiles/instrumentation_mocked_oldgems.gemfile
   - gemfiles/frameworks.gemfile
+  - gemfiles/rails60.gemfile
   - gemfiles/rails52.gemfile
   - gemfiles/rails42.gemfile
   - gemfiles/delayed_job.gemfile
@@ -32,13 +33,14 @@ matrix:
   exclude:
     - rvm: ruby-head
       gemfile: gemfiles/rails42.gemfile
-    - rvm: 2.6.2
+    - rvm: 2.6.4
       gemfile: gemfiles/rails42.gemfile
+    - rvm: 2.4.5
+      gemfile: gemfiles/rails60.gemfile
 
-
-    - rmv: 2.6.2
+    - rmv: 2.6.4
       env: DBTYPE=mysql
-    - rvm: 2.5.3
+    - rvm: 2.5.5
       env: DBTYPE=mysql
     - rvm: 2.4.5
       env: DBTYPE=mysql
@@ -57,10 +59,10 @@ matrix:
       env: DBTYPE=mysql
     - gemfile: gemfiles/frameworks.gemfile
       env: DBTYPE=mysql
-    - gemfile: gemfiles/rails51.gemfile
-      env: DBTYPE=mysql
     - gemfile: gemfiles/rails52.gemfile
       env: DBTYPE=mysql
+    - gemfile: gemfiles/rails60.gemfile
+      env: DBTYPE=mysql
     - gemfile: gemfiles/delayed_job.gemfile
       env: DBTYPE=mysql
 
@@ -85,37 +87,50 @@ matrix:
 # Attempt Travis/Cassandra fix re: https://github.com/travis-ci/travis-ci/issues/1484
 # Updated Cassandra: https://github.com/travis-ci/travis-ci/issues/1650
 
-before_install:
-  - sudo service cassandra stop
-  - sudo sh -c "echo 'JVM_OPTS=\"\${JVM_OPTS} -Djava.net.preferIPv4Stack=false\"' >> /etc/cassandra/cassandra-env.sh"
-  - echo "127.0.0.1 localhost"  | sudo tee /etc/hosts
-  - echo "127.0.0.1 " `hostname` | sudo tee -a /etc/hosts
-  - sudo service cassandra start
+# the following is not working anymore on travis
+# commenting it out as per AO-13532 as we currently don't have instrumentation
+# for cassandra
+#before_install:
+#  - sudo service cassandra stop
+#  - sudo sh -c "echo 'JVM_OPTS=\"\${JVM_OPTS} -Djava.net.preferIPv4Stack=false\"' >> /etc/cassandra/cassandra-env.sh"
+#  - echo "127.0.0.1 localhost"  | sudo tee /etc/hosts
+#  - echo "127.0.0.1 " `hostname` | sudo tee -a /etc/hosts
+#  - sudo service cassandra start
 
 install:
-  - curl -LO http://kent.dl.sourceforge.net/project/swig/swig/swig-3.0.8/swig-3.0.8.tar.gz
-  - tar xzf swig-3.0.8.tar.gz
-  - pushd swig-3.0.8
+  - curl -LO http://kent.dl.sourceforge.net/project/swig/swig/swig-3.0.12/swig-3.0.12.tar.gz
+  - tar xzf swig-3.0.12.tar.gz
+  - pushd swig-3.0.12
   - ./configure && make && sudo make install
   - popd
 
 before_script:
+  - export APPOPTICS_GEM_TEST=true
+  - export APPOPTICS_REPORTER=file
+  - export APPOPTICS_COLLECTOR=/tmp/appoptics_traces.bson
+  - export APPOPTICS_REPORTER_FILE_SINGLE=false
+  - export APPOPTICS_TOKEN_BUCKET_CAPACITY=1000
+  - export APPOPTICS_TOKEN_BUCKET_RATE=1000
+
   - bundle update --jobs=3 --retry=3
-  - bundle exec rake fetch_ext_deps
-  - bundle exec rake clean
-  - bundle exec rake compile
+  - bundle exec rake clean fetch compile
   - psql -c 'create database travis_ci_test;' -U postgres
   - mysql -e 'create database travis_ci_test;'
   - redis-server --requirepass secret_pass &
   - sleep 10
-  - export APPOPTICS_TOKEN_BUCKET_CAPACITY=1000
-  - export APPOPTICS_TOKEN_BUCKET_RATE=1000
 
 script: "N=1 bundle exec rake test"
 
 services:
   - mongodb
   - memcached
-  - cassandra
-#  - redis
+  - mysql
+#  - cassandra
+  - postgresql
+#  - redis-server
   - rabbitmq
+
+addons:
+  apt:
+    packages:
+      - rabbitmq-server

data/CONFIG.md

@@ -17,7 +17,7 @@ Name | Description | Default
 
 Name | Description | Default
 ---- | ----------- | -------
-`APPOPTICS_GEM_TEST` | puts the gem in test mode.  Traces are written to /tmp/trace_output.bson. | `false`
+`APPOPTICS_GEM_TEST` | puts the gem in test mode to avoid restarting certain background services used in testing.   `false`
 `DBTYPE` | For tests on Ruby on Rails, specifies the database type to test against.  `postgres`, `mysql` and `mysql2` are valid options. | `postgres`
 `APPOPTICS_CASSANDRA_SERVER` | specifies the Cassandra server to test against. | `127.0.0.1:9160`
 `APPOPTICS_MONGO_SERVER` | specifies the Mongo server to test against. | `127.0.0.1:27017`

data/README.md

@@ -390,4 +390,4 @@ See the README in the test directory.
 
 Copyright (c) 2018 SolarWinds, LLC
 
-Released under the [Librato Open License](https://docs.appoptics.com/kb/apm_tracing/librato-open-license/)
+Released under the [Librato Open License](https://docs.appoptics.com/kb/apm_tracing/librato-open-license/)

data/Rakefile

@@ -31,6 +31,9 @@ Rake::TestTask.new do |t|
     if Rails::VERSION::MAJOR == 5
       t.test_files = FileList["test/frameworks/rails#{Rails::VERSION::MAJOR}x_test.rb"] +
                      FileList["test/frameworks/rails#{Rails::VERSION::MAJOR}x_api_test.rb"]
+    elsif Rails::VERSION::MAJOR == 6
+      t.test_files = FileList["test/frameworks/rails5x_test.rb"] +
+                     FileList["test/frameworks/rails5x_api_test.rb"]
     else
       t.test_files = FileList["test/frameworks/rails#{Rails::VERSION::MAJOR}x_test.rb"]
     end
@@ -56,74 +59,77 @@ Rake::TestTask.new do |t|
   end
 
   if defined?(JRUBY_VERSION)
-    t.ruby_opts << ['-J-javaagent:/usr/local/tracelytics/tracelyticsagent.jar']
+      t.ruby_opts << ['-J-javaagent:/usr/local/tracelytics/tracelyticsagent.jar']
+    end
   end
-end
-
-desc "Run all test suites defined by travis"
-task "docker_tests" do
-  Dir.chdir('test/run_tests')
-  exec('docker-compose run ruby_appoptics /code/ruby-appoptics/test/run_tests/ruby_setup.sh test --remove-orphans')
-end
 
-desc "Start docker container for testing and debugging"
-task "docker" do
-  Dir.chdir('test/run_tests')
-  exec('docker-compose run ruby_appoptics /code/ruby-appoptics/test/run_tests/ruby_setup.sh bash --remove-orphans')
-end
 
-desc "Stop all containers that were started for testing and debugging"
-task "docker_down" do
-  Dir.chdir('test/run_tests')
-  exec('docker-compose down')
-end
+  desc "Run all test suites defined by travis"
+  task "docker_tests" do
+    Dir.chdir('test/run_tests')
+    exec('docker-compose run ruby_appoptics /code/ruby-appoptics/test/run_tests/ruby_setup.sh test --remove-orphans')
+  end
 
-desc "Fetch extension dependency files"
-task :fetch_ext_deps do
-  swig_version = %x{swig -version} rescue ''
-  swig_version = swig_version.scan(/swig version 3.0.\d*/i)
-  if swig_version.empty?
-    $stderr.puts '== ERROR ================================================================='
-    $stderr.puts "Could not find required swig version 3.0.*, found #{swig_version.inspect}"
-    $stderr.puts 'Please install swig "~ 3.0.8" and try again.'
-    $stderr.puts '=========================================================================='
-    raise
+  desc "Start docker container for testing and debugging"
+  task "docker" do
+    Dir.chdir('test/run_tests')
+    exec('docker-compose run ruby_appoptics /code/ruby-appoptics/test/run_tests/ruby_setup.sh bash --remove-orphans')
   end
 
-  # The c-lib version is different from the gem version
-  oboe_version = ENV['OBOE_VERSION'] || 'latest'
-  oboe_s3_dir = "https://s3-us-west-2.amazonaws.com/rc-files-t2/c-lib/#{oboe_version}"
-  ext_src_dir = File.expand_path('ext/oboe_metal/src')
-
-  # VERSION is used by extconf.rb to download the correct liboboe when installing the gem
-  remote_file = File.join(oboe_s3_dir, 'VERSION')
-  local_file = File.join(ext_src_dir, 'VERSION')
-  puts "fetching #{remote_file} to #{local_file}"
-  open(remote_file, 'rb') do |rf|
-    content = rf.read
-    File.open(local_file, 'wb') { |f| f.puts content }
-    puts "!!!!!!! C-Lib VERSION: #{content.strip} !!!!!!!!"
+  desc "Stop all containers that were started for testing and debugging"
+  task "docker_down" do
+    Dir.chdir('test/run_tests')
+    exec('docker-compose down')
   end
 
-  # oboe and bson header files
-  FileUtils.mkdir_p(File.join(ext_src_dir, 'bson'))
-  %w(oboe.h oboe.hpp oboe_debug.h oboe.i bson/bson.h bson/platform_hacks.h).each do |filename|
-  # %w(oboe.h oboe_debug.h bson/bson.h bson/platform_hacks.h).each do |filename|
-    remote_file = File.join(oboe_s3_dir, 'include', filename)
-    local_file = File.join(ext_src_dir, filename)
+  desc "Fetch extension dependency files"
+  task :fetch_ext_deps do
+    swig_version = %x{swig -version} rescue ''
+    swig_version = swig_version.scan(/swig version [34].0.\d*/i)
+    if swig_version.empty?
+      $stderr.puts '== ERROR ================================================================='
+      $stderr.puts "Could not find required swig version >3.0.8, found #{swig_version.inspect}"
+      $stderr.puts 'Please install swig "~ 3.0.12" and try again.'
+      $stderr.puts '=========================================================================='
+      raise
+    end
+
+    # The c-lib version is different from the gem version
+    oboe_version = ENV['OBOE_VERSION'] || 'latest'
+    oboe_s3_dir = "https://s3-us-west-2.amazonaws.com/rc-files-t2/c-lib/#{oboe_version}"
+    ext_src_dir = File.expand_path('ext/oboe_metal/src')
 
+    # VERSION is used by extconf.rb to download the correct liboboe when installing the gem
+    remote_file = File.join(oboe_s3_dir, 'VERSION')
+    local_file = File.join(ext_src_dir, 'VERSION')
     puts "fetching #{remote_file} to #{local_file}"
     open(remote_file, 'rb') do |rf|
       content = rf.read
       File.open(local_file, 'wb') { |f| f.puts content }
+      puts "!!!!!!! C-Lib VERSION: #{content.strip} !!!!!!!!"
+    end
+
+    # oboe and bson header files
+    FileUtils.mkdir_p(File.join(ext_src_dir, 'bson'))
+    %w(oboe.h oboe.hpp oboe_debug.h oboe.i bson/bson.h bson/platform_hacks.h).each do |filename|
+    # %w(oboe.h oboe_debug.h bson/bson.h bson/platform_hacks.h).each do |filename|
+      remote_file = File.join(oboe_s3_dir, 'include', filename)
+      local_file = File.join(ext_src_dir, filename)
+
+      puts "fetching #{remote_file} to #{local_file}"
+      open(remote_file, 'rb') do |rf|
+        content = rf.read
+        File.open(local_file, 'wb') { |f| f.puts content }
+      end
     end
-  end
 
-  FileUtils.cd(ext_src_dir) do
-    system('swig -c++ -ruby -module oboe_metal oboe.i')
-    FileUtils.rm('oboe.i')
+    FileUtils.cd(ext_src_dir) do
+      system('swig -c++ -ruby -module oboe_metal oboe.i')
+      FileUtils.rm('oboe.i')
+    end
   end
-end
+
+  task :fetch => :fetch_ext_deps
 
 desc "Build the gem's c extension"
 task :compile do
@@ -136,7 +142,7 @@ task :compile do
     so_file  = File.expand_path('ext/oboe_metal/oboe_metal.so')
 
     Dir.chdir ext_dir
-    ENV['APPOPTICS_FROM_S3'] = 'true'
+    # ENV['APPOPTICS_FROM_S3'] = 'true'
     cmd = [Gem.ruby, 'extconf.rb']
     sh cmd.join(' ')
     sh '/usr/bin/env make'
@@ -174,6 +180,7 @@ task :clean do
     Dir.chdir ext_dir
     sh '/usr/bin/env make clean' if File.exist? 'Makefile'
 
+    FileUtils.rm_f "src/oboe_wrap.cxx"
     Dir.chdir pwd
   else
     puts '== Nothing to do under JRuby.'

data/appoptics_apm.gemspec

@@ -54,6 +54,6 @@ Automatic tracing and metrics for Ruby applications. Get started at appoptics.co
     s.add_development_dependency('benchmark-ips', '>= 2.7.2')
   end
 
-  s.required_ruby_version = '>= 2.0.0'
+  s.required_ruby_version = '>= 2.4.0'
   s.executables   = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
 end

data/build_gem_upload_to_packagecloud.sh

@@ -8,13 +8,8 @@ else
   ./build_gem.sh
 fi
 
-# save current rbenv setting and switch to 2.4.1 for the package_cloud commands
-current_ruby=`rbenv global`
-rbenv global 2.4.1
+gem install package_cloud --no-document
 
 # prerequisite: package_cloud token needs to be in ~/.packagecloud
 gem=`ls -dt1 appoptics_apm-[^pre]*.gem | head -1`
 package_cloud push solarwinds/appoptics-apm-ruby $gem
-
-# restore ruby version
-rbenv global $current_ruby

data/examples/SDK/01_basic_tracing.rb

@@ -63,8 +63,6 @@ def do_traced_work
   end
 end
 
-# Example with logging
 AppOpticsAPM::SDK.start_trace('outer_span_2') do
-  AppopticsAPM.logger.warn "Hello World!"
   do_traced_work
 end

data/ext/oboe_metal/README.md

@@ -0,0 +1,69 @@
+# Debug the c-code with gdb
+
+inspired by: https://dev.to/wataash/how-to-create-and-debug-ruby-gem-with-c-native-extension-3l8b
+
+
+## install ruby with sources
+
+rbenv is your friend ;) 😄 -k means keep sources
+
+```
+rbenv install -k 2.6.3
+rbenv shell 2.6.3
+
+# check that ruby is debuggable
+type ruby           # => ruby is /home/wsh/.rbenv/shims/ruby
+rbenv which ruby    # => /home/wsh/.rbenv/versions/2.6.3/bin/ruby
+```
+
+
+##
+## add debug info when compiling appoptics_apm
+add this line to extconf.rb to turn off optimization
+
+```
+CONFIG["optflags"] = "-O0"
+```
+
+
+##
+## start ruby app with gdb
+
+This will run ruby and load the app with a breakpoint in the Reporter::startThread
+c-function.
+
+`bundle exec gdb -q -ex 'set breakpoint pending on' -ex 'b Reporter::startThread' -ex run --args ruby -e 'require "./app"'`
+
+If there is a bug in the ruby code or a ruby byebug binding that halts the 
+script, the debugger will hang without showing any output. 
+So, make sure `bundle exec ruby app.rb` runs.
+
+use the gdb navigation commands to step through the code. If it says:
+
+```
+(gdb) n
+Single stepping until exit from function _ZN8Reporter11startThreadEv@plt,
+which has no line number information.
+```
+
+type `c` and it may end up stopping in the right location.
+
+##
+## make ruby .gdbinit macros available
+
+These macros are pretty elaborate. They are checked in the ruby github 
+repo: https://github.com/ruby/ruby/blob/master/.gdbinit
+The code is nicely formatted and colorized in github and easiest to read there.
+
+installation in the user's home dir:
+```
+wget https://github.com/ruby/ruby/blob/master/.gdbinit
+```
+##
+## examples
+
+Some inspiring examples here:
+
+https://jvns.ca/blog/2016/06/12/a-weird-system-call-process-vm-readv/
+
+https://medium.com/@zanker/finding-a-ruby-bug-with-gdb-56d6b321bc86

data/ext/oboe_metal/src/VERSION

@@ -1 +1 @@
-5.0.0
+6.0.0

data/ext/oboe_metal/src/oboe.h

@@ -1,12 +1,12 @@
 /**
  * @file oboe.h
- * 
+ *
  * API header for AppOptics' Oboe application tracing library for use with AppOptics.
  *
  * @package             Oboe
  * @author              AppOptics
  * @copyright           Copyright (c) 2016, SolarWinds LLC
- * @license             
+ * @license
  * @link                https://appoptics.com
  **/
 
@@ -81,6 +81,10 @@ extern "C" {
  * Default events flush batch size in KB.
  */
 #define OBOE_DEFAULT_EVENTS_FLUSH_BATCH_SIZE 2000
+/**
+ * Default EC2 metadata timeout in milliseconds
+ */
+#define OBOE_DEFAULT_EC2_METADATA_TIMEOUT 1000
 
 #define OBOE_SAMPLE_RESOLUTION 1000000
 
@@ -167,6 +171,8 @@ typedef struct oboe_init_options {
     int token_bucket_capacity;              // custom token bucket capacity
     int token_bucket_rate;                  // custom token bucket rate
     int file_single;                        // use single files in file reporter for each event
+
+    int ec2_metadata_timeout;               // EC2 metadata timeout in milliseconds
 } oboe_init_options_t;
 
 typedef struct oboe_span_params {
@@ -183,12 +189,22 @@ typedef struct oboe_span_params {
     int do_metrics;                 // boolean flag whether a (HTTP) span should be sent (1) or not (0)
 } oboe_span_params_t;
 
+//
+// oboe_get_tracing_decisions input and output structures
+//
 typedef struct oboe_tracing_decisions_in {
-    int version;
-    const char *service_name;
-    const char *in_xtrace;
-    int custom_sample_rate;
-    int custom_tracing_mode;
+    int version;                    // the version of this structure
+    const char *service_name;       // custom service name
+    const char *in_xtrace;          // existing X-Trace from passed in HTTP header
+    int custom_sample_rate;         // custom sample rate
+    int custom_tracing_mode;        // custom tracing mode
+
+    // v2
+    int custom_trigger_mode;        // custom trigger mode
+    int request_type;               // the request type: OBOE_REQUEST_TYPE_REGULAR, OBOE_REQUEST_TYPE_TRIGGER
+    const char *header_options;     // X-Trace-Options HTTP header value
+    const char *header_signature;   // X-Trace-Options-Signature HTTP header value
+    time_t header_timestamp;        // timestamp from X-Trace-Options header, converted to UNIX timestamp format
 } oboe_tracing_decisions_in_t;
 
 typedef struct oboe_tracing_decisions_out {
@@ -197,6 +213,12 @@ typedef struct oboe_tracing_decisions_out {
     int sample_source;
     int do_sample;
     int do_metrics;
+
+    // v2
+    int request_provisioned;
+    int auth_status;
+    const char *auth_message;
+    const char *status_message;
 } oboe_tracing_decisions_out_t;
 
 typedef struct oboe_internal_stats {
@@ -357,7 +379,7 @@ int oboe_init(oboe_init_options_t* options);
  * Initialize the Oboe subsytems using a specific reporter configuration.
  *
  * This should be called before any other oboe_* functions butm may also be
- * used to change or re-initialize the current reporter.  To reconnect the 
+ * used to change or re-initialize the current reporter.  To reconnect the
  * reporter use oboe_disconnect() and oboe_reconnect() instead.
  *
  * @param protocol One of  OBOE_REPORTER_PROTOCOL_FILE, OBOE_REPORTER_PROTOCOL_UDP,
@@ -367,7 +389,12 @@ int oboe_init(oboe_init_options_t* options);
  */
 int oboe_init_reporter(const char *protocol, oboe_init_options_t *options);
 
-void oboe_init_options_set_defaults(oboe_init_options_t *options);
+/**
+ * returns one of these:
+ * - OBOE_INIT_OPTIONS_SET_DEFAULTS_OK
+ * - OBOE_INIT_OPTIONS_SET_DEFAULTS_WRONG_VERSION
+ */
+int oboe_init_options_set_defaults(oboe_init_options_t *options);
 
 /**
  * Disconnect or shut down the Oboe reporter, but allow it to be reconnect()ed.
@@ -429,6 +456,7 @@ void oboe_shutdown();
 #define OBOE_SETTINGS_FLAG_SAMPLE_START   0x4
 #define OBOE_SETTINGS_FLAG_SAMPLE_THROUGH 0x8
 #define OBOE_SETTINGS_FLAG_SAMPLE_THROUGH_ALWAYS 0x10
+#define OBOE_SETTINGS_FLAG_TRIGGERED_TRACE 0x20
 #define OBOE_SETTINGS_MAX_STRLEN 256
 
 #define OBOE_SETTINGS_UNSET -1
@@ -455,9 +483,6 @@ void oboe_shutdown();
     #pragma pack(push, 1)
 #endif
 
-#define TOKEN_BUCKET_CAPACITY_DEFAULT 16               // bucket capacity (how many tokens fit into the bucket)
-#define TOKEN_BUCKET_RATE_PER_SECOND_DEFAULT 8         // rate per second (number of tokens per second)
-
 #define OBOE_SEND_EVENT 0
 #define OBOE_SEND_STATUS 1
 #define OBOE_SEND_PROFILING 2
@@ -477,7 +502,17 @@ void oboe_shutdown();
 #define OBOE_SPAN_NO_REPORTER -4
 #define OBOE_SPAN_NOT_READY -5
 
-// these codes are used by oboe_sample_layer_custom(), oboe_tracing_decisions(), oboe_reporter_is_ready()
+// these codes are returned by oboe_sample_layer_custom(), oboe_tracing_decisions(),
+// oboe_reporter_is_ready(), and referenced in settings_messages.c (to convert
+// error codes to messages.)
+//
+// they are structured such that codes that are <= 0 are successful, i.e., no
+// error status needs to be reported to the user, while codes > 0 are errors that
+// need to be reported.
+
+#define OBOE_TRACING_DECISIONS_FAILED_AUTH -5
+#define OBOE_TRACING_DECISIONS_TRIGGERED_TRACE_EXHAUSTED -4
+#define OBOE_TRACING_DECISIONS_TRIGGERED_TRACE_DISABLED -3
 #define OBOE_TRACING_DECISIONS_TRACING_DISABLED -2
 #define OBOE_TRACING_DECISIONS_XTRACE_NOT_SAMPLED -1
 #define OBOE_TRACING_DECISIONS_OK 0
@@ -486,8 +521,31 @@ void oboe_shutdown();
 #define OBOE_TRACING_DECISIONS_REPORTER_NOT_READY 3
 #define OBOE_TRACING_DECISIONS_NO_VALID_SETTINGS 4
 #define OBOE_TRACING_DECISIONS_QUEUE_FULL 5
+#define OBOE_TRACING_DECISIONS_BAD_ARG 6
+
+// convert above codes into const char* messages.
+const char* oboe_get_tracing_decisions_message(int code);
+
+#define OBOE_TRACING_DECISIONS_AUTH_NOT_CHECKED -2
+#define OBOE_TRACING_DECISIONS_AUTH_NOT_PRESENT -1
+#define OBOE_TRACING_DECISIONS_AUTH_OK 0
+#define OBOE_TRACING_DECISIONS_AUTH_NO_SIG_KEY 1
+#define OBOE_TRACING_DECISIONS_AUTH_INVALID_SIG 2
+#define OBOE_TRACING_DECISIONS_AUTH_BAD_TIMESTAMP 3
 
-// these codes are used by oboe_init(), oboe_init_reporter(), _oboe_create_reporter()
+// convert above codes into const char* messages.
+const char* oboe_get_tracing_decisions_auth_message (int code);
+
+#define OBOE_REQUEST_TYPE_NONE -1
+#define OBOE_REQUEST_TYPE_REGULAR 0
+#define OBOE_REQUEST_TYPE_TRIGGER 1
+
+#define OBOE_INIT_OPTIONS_SET_DEFAULTS_OK 0
+#define OBOE_INIT_OPTIONS_SET_DEFAULTS_WRONG_VERSION 1
+
+//
+// these codes are returned by oboe_init(), oboe_init_reporter(), _oboe_create_reporter()
+//
 #define OBOE_INIT_ALREADY_INIT -1
 #define OBOE_INIT_OK 0
 #define OBOE_INIT_WRONG_VERSION 1
@@ -501,6 +559,17 @@ void oboe_shutdown();
 #define OBOE_INIT_SSL_LOAD_CERT 9
 #define OBOE_INIT_SSL_REPORTER_CREATE 10
 
+// token buckets
+enum TOKEN_BUCKETS {
+  TOKEN_BUCKET_SAMPLING,    // for normal requests
+  TOKEN_BUCKET_TT_RELAXED,  // for triggered traces initiated by Pingdom and
+                            // other trusted sources (relaxed settings)
+  TOKEN_BUCKET_TT_STRICT,   // for triggered traces initiated by CLI and
+                            // other untrusted sources (strict settings)
+  TOKEN_BUCKET_COUNT        // IMPORTANT NOTE: this must be the last element
+                            // inside the enum
+};
+
 typedef struct {
     uint32_t magic;
     uint32_t timestamp;
@@ -510,8 +579,9 @@ typedef struct {
     uint32_t ttl;
     uint32_t _pad;
     char layer[OBOE_SETTINGS_MAX_STRLEN]; // Flawfinder: ignore
-    double bucket_capacity;
-    double bucket_rate_per_sec;
+    double bucket_capacity[TOKEN_BUCKET_COUNT];
+    double bucket_rate_per_sec[TOKEN_BUCKET_COUNT];
+    char signature_key[OBOE_SETTINGS_MAX_STRLEN]; // Flawfinder: ignore
 } oboe_settings_t;
 
 typedef struct {
@@ -531,6 +601,7 @@ typedef struct {
     volatile uint32_t through_count;            // # of through traces
     volatile uint32_t through_ignored_count;    // # of new requests, that are rejected due to start_always_flag == 0
                                                 // that have through_always_flag == 1
+    volatile uint32_t triggered_count;             // # of triggered traces
     volatile uint32_t last_used_sample_rate;
     volatile uint32_t last_used_sample_source;
 
@@ -541,12 +612,13 @@ typedef struct {
 typedef struct {
     int tracing_mode;          // pushed from server, override from config file
     int sample_rate;           // pushed from server, override from config file
+    int trigger_mode;          // pushed from server, override from config file
     oboe_settings_t *settings; // cached settings, updated by tracelyzer (init to NULL)
-    int last_auto_sample_rate; // stores last known automatic sampling rate 
-    uint16_t last_auto_flags;  // stores last known flags associated with above 
+    int last_auto_sample_rate; // stores last known automatic sampling rate
+    uint16_t last_auto_flags;  // stores last known flags associated with above
     uint32_t last_auto_timestamp; // timestamp from last *settings lookup
     uint32_t last_refresh;        // last refresh time
-    token_bucket_t bucket;
+    token_bucket_t bucket[TOKEN_BUCKET_COUNT];     // token buckets for various tasks
 } oboe_settings_cfg_t;
 
 int oboe_settings_init_local();
@@ -560,9 +632,10 @@ entry_layer_t* oboe_settings_entry_layer_get(const char* name);
 oboe_settings_cfg_t* oboe_settings_cfg_get();
 void oboe_settings_cfg_init(oboe_settings_cfg_t *cfg);
 
-void oboe_settings_set(int sample_rate, int tracing_mode);
+void oboe_settings_set(int sample_rate, int tracing_mode, int trigger_mode);
 void oboe_settings_rate_set(int sample_rate);
 void oboe_settings_mode_set(int tracing_mode);
+void oboe_settings_trigger_set(int trigger_mode);
 
 int oboe_rand_get_value();
 
@@ -588,49 +661,7 @@ int oboe_sample_is_enabled(oboe_settings_cfg_t *cfg);
  * headers, and, if appropriate, rolls the virtual dice to
  * decide if this request should be sampled.
  *
- * This is designed to be called once per layer per request.
- *
- * @param service_name Service name used for this request (may be NULL to use default settings)
- * @param xtrace X-Trace ID string from an HTTP request or higher layer (NULL or empty string if not present).
- * @param sample_rate_out The sample rate used to check if this request should be sampled
- *          (output - may be zero if not used).
- * @param sample_source_out The OBOE_SAMPLE_RATE_SOURCE used to check if this request
- *          should be sampled (output - may be zero if not used).
- * @return Non-zero if the given request should be sampled.
- */
-int oboe_sample_layer(
-    const char *service_name,
-    const char *xtrace,
-    int *sample_rate_out,
-    int *sample_source_out
-);
-
-/**
- * Same as oboe_sample_layer() but accepting custom sample rate and custom tracing mode
- *
- * @param service_name Service name used for this request (may be NULL to use default settings)
- * @param xtrace X-Trace ID string from an HTTP request or higher layer (NULL or empty string if not present).
- * @param custom_sample_rate a custom sample rate only used for this request (OBOE_SETTINGS_UNSET won't override)
- * @param custom_tracing_mode a custom tracing mode only used for this request (OBOE_SETTINGS_UNSET won't override)
- * @param sample_rate_out The sample rate used to check if this request should be sampled
- *          (output - may be zero if not used).
- * @param sample_source_out The OBOE_SAMPLE_RATE_SOURCE used to check if this request
- *          should be sampled (output - may be zero if not used).
- * @param flags_out The flags used to check if this request should be sampled
- */
-int oboe_sample_layer_custom(
-    const char *service_name,
-    const char *in_xtrace,
-    int custom_sample_rate,
-    int custom_tracing_mode,
-    int *sampling_decision_out,
-    int *sample_rate_out,
-    int *sample_source_out,
-    uint16_t *flags_out
-);
-
-/**
- * wrapper for calling oboe_sample_layer_custom() with input/output structs instead of individual params
+ * This is designed to be called once per request.
  *
  * @param in Struct containing all params to help making a tracing decision
  * @param out Struct containing all params that get set during decision making
@@ -773,21 +804,21 @@ extern int oboe_debug_log_remove(OboeDebugLoggerFcn oldLogger, void *context);
  * We use this to get a reasonable standard format between apps.
  *
  * @param module An OBOE_MODULE_* module identifier.  Use zero for undefined.
- * @param app_name Either NULL or a pointer to a string containing a name for 
- *          the application - will prefix the log entry.  Useful when multiple 
- *          apps log to the same destination. 
+ * @param app_name Either NULL or a pointer to a string containing a name for
+ *          the application - will prefix the log entry.  Useful when multiple
+ *          apps log to the same destination.
  * @param trace_mode A string identifying the configured tracing mode, one of:
  *          "enabled", "disabled", "unset", or "undef" (for invalid values)
  *          Use the oboe_tracing_mode_to_string() function to convert from
  *          numeric values.
  * @param sample_rate The configured sampling rate: -1 for unset or a
  *          integer fraction of 1000000.
- * @param reporter_type String identifying the type of reporter configured: 
+ * @param reporter_type String identifying the type of reporter configured:
  *          One of 'udp' (the default), 'ssl', or 'file'.
  * @param reporter_args The string of comma-separated key=value settings
  *          used to initialize the reporter.
  * @param extra: Either NULL or a pointer to a string to be appended to
- *          the log message and designed to include a few other 
+ *          the log message and designed to include a few other
  *          configuration parameters of interest.
  */
 #if OBOE_DEBUG_LEVEL >= OBOE_DEBUG_INFO
@@ -821,7 +852,7 @@ extern int oboe_debug_log_remove(OboeDebugLoggerFcn oldLogger, void *context);
 /**
  * Log a recoverable error.
  *
- * Each message is limited in the number of times that it will be reported at the 
+ * Each message is limited in the number of times that it will be reported at the
  * ERROR level after which it will be logged at the debug MEDIUM level.
  */
 #if OBOE_DEBUG_LEVEL >= OBOE_DEBUG_ERROR
@@ -838,7 +869,7 @@ extern int oboe_debug_log_remove(OboeDebugLoggerFcn oldLogger, void *context);
 /**
  * Log a warning.
  *
- * Each message is limited in the number of times that it will be reported at the 
+ * Each message is limited in the number of times that it will be reported at the
  * WARNING level after which it will be logged at the debug MEDIUM level.
  */
 #if OBOE_DEBUG_LEVEL >= OBOE_DEBUG_WARNING
@@ -855,7 +886,7 @@ extern int oboe_debug_log_remove(OboeDebugLoggerFcn oldLogger, void *context);
 /**
  * Log an informative message.
  *
- * Each message is limited in the number of times that it will be reported at the 
+ * Each message is limited in the number of times that it will be reported at the
  * INFO level after which it will be logged at the debug MEDIUM level.
  */
 #if OBOE_DEBUG_LEVEL >= OBOE_DEBUG_INFO