snowplow-tracker 0.7.0.pre.alpha.0 → 0.8.0

data/lib/snowplow-tracker/emitters.rb

@@ -1,4 +1,4 @@
-# Copyright (c) 2013-2014 Snowplow Analytics Ltd. All rights reserved.
+# Copyright (c) 2013-2021 Snowplow Analytics Ltd. All rights reserved.
 #
 # This program is licensed to you under the Apache License Version 2.0,
 # and you may not use this file except in compliance with the Apache License Version 2.0.
@@ -9,239 +9,418 @@
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
 
-# Author:: Alex Dean, Fred Blundun (mailto:support@snowplowanalytics.com)
-# Copyright:: Copyright (c) 2013-2014 Snowplow Analytics Ltd
+# Author:: Snowplow Analytics Ltd
+# Copyright:: Copyright (c) 2013-2021 Snowplow Analytics Ltd
 # License:: Apache License Version 2.0
 
+
 require 'net/https'
 require 'set'
 require 'logger'
-require 'contracts'
 
 module SnowplowTracker
-
+  # @see Emitter
+  # For logging Emitter activity messages
   LOGGER = Logger.new(STDERR)
   LOGGER.level = Logger::INFO
 
+  # This class sends events to the event collector. All {Tracker}s must have at
+  # least one associated Emitter or the subclass AsyncEmitter.
+  #
+  # The network settings are defined as part of the Emitter initalization. This
+  # table displays the default Emitter settings:
+  #
+  # | Property | Default setting |
+  # | --- | --- |
+  # | Protocol | HTTP |
+  # | Method | GET |
+  # | Buffer size | 1 |
+  # | Path | `/i` |
+  #
+  # The buffer size is the number of events which will be buffered before they
+  # are all sent simultaneously. The process of sending all buffered events is
+  # called "flushing". The default buffer size is 1 because GET requests can
+  # only contain one event.
+  #
+  # If you choose to use POST requests, the buffer_size defaults to 10, and the
+  # buffered events are all sent together in a single request. The default path
+  # is '/com.snowplowanalytics.snowplow/tp2' for Emitters using POST.
+  #
+  # # Logging
+  # Emitters log their activity to STDERR by default, using the Ruby standard
+  # library Logger class. A different logger can be configured during Emitter
+  # initialization. For example, to disable logging, you could provide
+  # `Logger.new(IO::NULL)` in the options hash.
+  #
+  # By default, only messages with priority "INFO" or higher will be logged.
+  # This can be changed at any time for the default logger, which is saved as a
+  # module constant (`LOGGER = Logger.new(STDERR)`). If you are not using the
+  # default logger, set the message level before initializing your Emitter.
+  #
+  # @see https://ruby-doc.org/stdlib-2.7.2/libdoc/logger/rdoc/Logger.html Logger documentation
+  #
+  # @example Changing the logger message level.
+  #   require 'logger'
+  #   SnowplowTracker::LOGGER.level = Logger::DEBUG
   class Emitter
-
-    include Contracts
-
-    @@ConfigHash = ({
-      :protocol => Maybe[Or['http', 'https']],
-      :port => Maybe[Num],
-      :method => Maybe[Or['get', 'post']],
-      :buffer_size => Maybe[Num],
-      :on_success => Maybe[Func[Num => Any]],
-      :on_failure => Maybe[Func[Num, Hash => Any]],
-      :thread_count => Maybe[Num]
-    })
-
-    @@StrictConfigHash = And[@@ConfigHash, lambda { |x|
-      x.class == Hash and Set.new(x.keys).subset? Set.new(@@ConfigHash.keys)
-    }]
-
-    @@DefaultConfig = {
-      :protocol => 'http',
-      :method => 'get'
+    # Default Emitter settings
+    DEFAULT_CONFIG = {
+      protocol: 'http',
+      method: 'get'
     }
 
-    Contract String, @@StrictConfigHash => lambda { |x| x.is_a? Emitter }
-    def initialize(endpoint, config={})
-      config = @@DefaultConfig.merge(config)
+    # @private
+    attr_reader :logger
+
+    # Create a new Emitter instance. The endpoint is required.
+    #
+    # @example Initializing an Emitter with all the possible extra configuration.
+    #   success_callback = ->(success_count) { puts "#{success_count} events sent successfully" }
+    #   failure_callback = ->(success_count, failures) do
+    #     puts "#{success_count} events sent successfully, #{failures.size} sent unsuccessfully"
+    #   end
+    #
+    #   SnowplowTracker::Emitter.new(endpoint: 'collector.example.com',
+    #               options: { path: '/my-pipeline/1',
+    #                          protocol: 'https',
+    #                          port: 443,
+    #                          method: 'post',
+    #                          buffer_size: 5,
+    #                          on_success: success_callback,
+    #                          on_failure: failure_callback,
+    #                          logger: Logger.new(STDOUT) })
+    #
+    # The options hash can have any of these optional parameters:
+    #
+    # | Parameter | Description | Type |
+    # | --- | --- | --- |
+    # | path | Override the default path for appending to the endpoint | String |
+    # | protocol | 'http' or 'https' | String |
+    # | port | The port for the connection | Integer |
+    # | method | 'get' or 'post' | String |
+    # | buffer_size | Number of events to send at once | Integer |
+    # | on_success | A method to call if events were sent successfully | Method |
+    # | on_failure | A method to call if events did not send | Method |
+    # | thread_count | Number of threads to use | Integer |
+    # | logger | Log somewhere other than STDERR | Logger |
+    #
+    # Note that `thread_count` is relevant only to the subclass {AsyncEmitter},
+    # and will be ignored if provided to an Emitter.
+    #
+    # If you choose to use HTTPS, we recommend using port 443.
+    #
+    # Only 2xx and 3xx status codes are considered successes.
+    #
+    # The `on_success` callback should accept one argument: the number of
+    # requests sent this way. The `on_failure` callback should accept two
+    # arguments: the number of successfully sent events, and an array containing
+    # the unsuccessful events.
+    #
+    # @param endpoint [String] the endpoint to send the events to
+    # @param options [Hash] allowed configuration options
+    #
+    # @see AsyncEmitter#initialize
+    # @api public
+    def initialize(endpoint:, options: {})
+      config = DEFAULT_CONFIG.merge(options)
       @lock = Monitor.new
-      @collector_uri = as_collector_uri(endpoint, config[:protocol], config[:port], config[:method])
+      path = confirm_path(config)
+      @collector_uri = create_collector_uri(endpoint, config[:protocol], config[:port], path)
       @buffer = []
-      if not config[:buffer_size].nil?
-        @buffer_size = config[:buffer_size]
-      elsif config[:method] == 'get'
-        @buffer_size = 1
-      else
-        @buffer_size = 10
-      end
+      @buffer_size = confirm_buffer_size(config)
       @method = config[:method]
       @on_success = config[:on_success]
       @on_failure = config[:on_failure]
-      LOGGER.info("#{self.class} initialized with endpoint #{@collector_uri}")
+      @logger = config[:logger] || LOGGER
+      logger.info("#{self.class} initialized with endpoint #{@collector_uri}")
+    end
+
+    # Creates the `@buffer_size` variable during initialization. Unless
+    # otherwise defined, it's 1 for Emitters using GET and 10 for Emitters using
+    # POST requests.
+    # @private
+    def confirm_buffer_size(config)
+      return config[:buffer_size] unless config[:buffer_size].nil?
 
-      self
+      config[:method] == 'get' ? 1 : 10
     end
 
-    # Build the collector URI from the configuration hash
-    #
-    Contract String, String, Maybe[Num], String => String
-    def as_collector_uri(endpoint, protocol, port, method)
-      port_string = port == nil ? '' : ":#{port.to_s}"
-      path = method == 'get' ? '/i' : '/com.snowplowanalytics.snowplow/tp2'
+    # Creates the `@path` variable during initialization. Allows a non-standard
+    # path to be provided.
+    # @private
+    def confirm_path(config)
+      return config[:path] unless config[:path].nil?
+
+      config[:method] == 'get' ? '/i' : '/com.snowplowanalytics.snowplow/tp2'
+    end
+
+    # Creates the `@collector_uri` variable during initialization.
+    # The default is "http://{endpoint}/i".
+    # @private
+    def create_collector_uri(endpoint, protocol, port, path)
+      port_string = port.nil? ? '' : ":#{port}"
 
       "#{protocol}://#{endpoint}#{port_string}#{path}"
     end
 
-    # Add an event to the buffer and flush it if maximum size has been reached
+    # Add an event to the buffer and flush it if maximum size has been reached.
+    # This method is not required for standard Ruby tracker usage. A {Tracker}
+    # privately calls this method once the event payload is ready to send.
+    #
+    # We have included it as part of the public API for its possible use in the
+    # `on_failure` callback. This is the optional method, provided in the
+    # `options` Emitter initalization hash, that is called when events fail
+    # to send. You could use {#input} as part of your callback to immediately
+    # retry the failed event.
     #
-    Contract Hash => nil
+    # The `on_failure` callback should accept two arguments: the number of
+    # successfully sent events, and an array containing the unsuccessful events.
+    #
+    # @example A possible `on_failure` method using `#input`
+    #   def retry_on_failure(failed_event_count, failed_events)
+    #     # possible backoff-and-retry timeout here
+    #     failed_events.each do |event|
+    #       my_emitter.input(event)
+    #     end
+    #   end
+    #
+    # @api public
     def input(payload)
-      payload.each { |k,v| payload[k] = v.to_s}
+      payload.each { |k, v| payload[k] = v.to_s }
       @lock.synchronize do
         @buffer.push(payload)
-        if @buffer.size >= @buffer_size
-          flush
-        end
+        flush if @buffer.size >= @buffer_size
       end
 
       nil
     end
 
-    # Flush the buffer
+    # Flush the Emitter, forcing it to send all the events in its
+    # buffer, even if the buffer is not full. {Emitter} objects, unlike
+    # {AsyncEmitter}s, can only `flush` synchronously. A {Tracker} can manually flush all
+    # its Emitters by calling {Tracker#flush}, part of the public API which
+    # calls this method.
     #
-    Contract Bool => nil
-    def flush(async=true)
+    # The unused async parameter here is to avoid ArgumentError, since
+    # {AsyncEmitter#flush} does take an argument.
+    #
+    # @see AsyncEmitter#flush
+    # @private
+    def flush(_async = true)
       @lock.synchronize do
         send_requests(@buffer)
         @buffer = []
       end
+
       nil
     end
 
     # Send all events in the buffer to the collector
-    #
-    Contract ArrayOf[Hash] => nil
-    def send_requests(evts)
-      if evts.size < 1
-        LOGGER.info("Skipping sending events since buffer is empty")
+    # @private
+    def send_requests(events)
+      if events.empty?
+        logger.info('Skipping sending events since buffer is empty')
         return
       end
-      LOGGER.info("Attempting to send #{evts.size} request#{evts.size == 1 ? '' : 's'}")
 
-      evts.each do |event|
-        event['stm'] = (Time.now.to_f * 1000).to_i.to_s # add the sent timestamp, overwrite if already exists
+      logger.info("Attempting to send #{events.size} request#{events.size == 1 ? '' : 's'}")
+
+      events.each do |event|
+        # add the sent timestamp, overwrite if already exists
+        event['stm'] = Timestamp.create.to_s
       end
 
       if @method == 'post'
-        post_succeeded = false
-        begin
-          request = http_post(SelfDescribingJson.new(
-            'iglu:com.snowplowanalytics.snowplow/payload_data/jsonschema/1-0-4',
-            evts
-          ).to_json)
-          post_succeeded = is_good_status_code(request.code)
-        rescue StandardError => se
-          LOGGER.warn(se)
-        end
-        if post_succeeded
-          unless @on_success.nil?
-            @on_success.call(evts.size)
-          end
-        else
-          unless @on_failure.nil?
-            @on_failure.call(0, evts)
-          end
-        end
-
+        send_requests_with_post(events)
       elsif @method == 'get'
-        success_count = 0
-        unsent_requests = []
-        evts.each do |evt|
-          get_succeeded = false
-          begin
-            request = http_get(evt)
-            get_succeeded = is_good_status_code(request.code)
-          rescue StandardError => se
-            LOGGER.warn(se)
-          end
-          if get_succeeded
-            success_count += 1
-          else
-            unsent_requests << evt
-          end
-        end
-        if unsent_requests.size == 0
-          unless @on_success.nil?
-            @on_success.call(success_count)
-          end
-        else
-          unless @on_failure.nil?
-            @on_failure.call(success_count, unsent_requests)
-          end
-        end
+        send_requests_with_get(events)
       end
 
       nil
     end
 
-    # Send a GET request
-    #
-    Contract Hash => lambda { |x| x.is_a? Net::HTTPResponse }
+    # Part of {#send_requests}.
+    # @private
+    def send_requests_with_post(events)
+      post_succeeded = false
+      begin
+        request = http_post(SelfDescribingJson.new(
+          'iglu:com.snowplowanalytics.snowplow/payload_data/jsonschema/1-0-4',
+          events
+        ).to_json)
+        post_succeeded = good_status_code?(request.code)
+      rescue StandardError => standard_error
+        logger.warn(standard_error)
+      end
+
+      if post_succeeded
+        @on_success.call(events.size) unless @on_success.nil?
+      else
+        @on_failure.call(0, events) unless @on_failure.nil?
+      end
+
+      nil
+    end
+
+    # Part of {#send_requests}.
+    # @private
+    def send_requests_with_get(events)
+      success_count = 0
+      unsent_requests = []
+
+      events.each do |event|
+        request = process_get_event(event)
+        request ? success_count += 1 : unsent_requests << event
+      end
+
+      if unsent_requests.size.zero?
+        @on_success.call(success_count) unless @on_success.nil?
+      else
+        @on_failure.call(success_count, unsent_requests) unless @on_failure.nil?
+      end
+
+      nil
+    end
+
+    # Part of {#send_requests_with_get}.
+    # @private
+    def process_get_event(event)
+      get_succeeded = false
+      begin
+        request = http_get(event)
+        get_succeeded = good_status_code?(request.code)
+      rescue StandardError => standard_error
+        logger.warn(standard_error)
+      end
+      get_succeeded
+    end
+
+    # Part of {#process_get_event}. This sends a GET request.
+    # @private
     def http_get(payload)
       destination = URI(@collector_uri + '?' + URI.encode_www_form(payload))
-      LOGGER.info("Sending GET request to #{@collector_uri}...")
-      LOGGER.debug("Payload: #{payload}")
+      logger.info("Sending GET request to #{@collector_uri}...")
+      logger.debug("Payload: #{payload}")
       http = Net::HTTP.new(destination.host, destination.port)
       request = Net::HTTP::Get.new(destination.request_uri)
-      if destination.scheme == 'https'
-        http.use_ssl = true
-      end
+      http.use_ssl = true if destination.scheme == 'https'
       response = http.request(request)
-      LOGGER.add(is_good_status_code(response.code) ? Logger::INFO : Logger::WARN) {
+      logger.add(good_status_code?(response.code) ? Logger::INFO : Logger::WARN) do
         "GET request to #{@collector_uri} finished with status code #{response.code}"
-      }
+      end
 
       response
     end
 
-    # Send a POST request
-    #
-    Contract Hash => lambda { |x| x.is_a? Net::HTTPResponse }
+    # Part of {#send_requests_with_post}. This sends a POST request.
+    # @private
     def http_post(payload)
-      LOGGER.info("Sending POST request to #{@collector_uri}...")
-      LOGGER.debug("Payload: #{payload}")
+      logger.info("Sending POST request to #{@collector_uri}...")
+      logger.debug("Payload: #{payload}")
       destination = URI(@collector_uri)
       http = Net::HTTP.new(destination.host, destination.port)
       request = Net::HTTP::Post.new(destination.request_uri)
-      if destination.scheme == 'https'
-        http.use_ssl = true
-      end
+      http.use_ssl = true if destination.scheme == 'https'
       request.body = payload.to_json
       request.set_content_type('application/json; charset=utf-8')
       response = http.request(request)
-      LOGGER.add(is_good_status_code(response.code) ? Logger::INFO : Logger::WARN) {
+      logger.add(good_status_code?(response.code) ? Logger::INFO : Logger::WARN) do
         "POST request to #{@collector_uri} finished with status code #{response.code}"
-      }
+      end
 
       response
     end
 
-    # Only 2xx and 3xx status codes are considered successes
-    #
-    Contract String => Bool
-    def is_good_status_code(status_code)
+    # Check if the response is good.
+    # Only 2xx and 3xx status codes are considered successes.
+    # @private
+    def good_status_code?(status_code)
       status_code.to_i >= 200 && status_code.to_i < 400
     end
 
-    private :as_collector_uri,
+    private :create_collector_uri,
             :http_get,
             :http_post
-
   end
 
-
+  # This {Emitter} subclass provides asynchronous event sending. Whenever the
+  # buffer is flushed, the AsyncEmitter places the flushed events in a work
+  # queue. The AsyncEmitter asynchronously sends events in this queue using a
+  # thread pool of a fixed size. The size of the thread pool is 1 by default,
+  # but can be configured as part of the options hash during initialization.
+  #
+  # @see Emitter
+  # @api public
   class AsyncEmitter < Emitter
-
-    Contract String, @@StrictConfigHash => lambda { |x| x.is_a? Emitter }
-    def initialize(endpoint, config={})
-      @queue = Queue.new()
+    # Create a new AsyncEmitter object. The endpoint is required.
+    #
+    # @example Initializing an AsyncEmitter with all the possible extra configuration.
+    #   success_callback = ->(success_count) { puts "#{success_count} events sent successfully" }
+    #   failure_callback = ->(success_count, failures) do
+    #     puts "#{success_count} events sent successfully, #{failures.size} sent unsuccessfully"
+    #   end
+    #
+    #   SnowplowTracker::Emitter.new(endpoint: 'collector.example.com',
+    #               options: { path: '/my-pipeline/1',
+    #                          protocol: 'https',
+    #                          port: 443,
+    #                          method: 'post',
+    #                          buffer_size: 5,
+    #                          on_success: success_callback,
+    #                          on_failure: failure_callback,
+    #                          logger: Logger.new(STDOUT),
+    #                          thread_count: 5 })
+    #
+    # The options hash can have any of these optional parameters:
+    #
+    # | Parameter | Description | Type |
+    # | --- | --- | --- |
+    # | path | Override the default path for appending to the endpoint | String |
+    # | protocol | 'http' or 'https' | String |
+    # | port | The port for the connection | Integer |
+    # | method | 'get' or 'post' | String |
+    # | buffer_size | Number of events to send at once | Integer |
+    # | on_success | A function to call if events were sent successfully | Function |
+    # | on_failure | A function to call if events did not send | Function |
+    # | thread_count | Number of threads to use | Integer |
+    # | logger | Log somewhere other than STDERR | Logger |
+    #
+    # The `thread_count` determines the number of worker threads which will be
+    # used to send events.
+    #
+    # If you choose to use HTTPS, we recommend using port 443.
+    #
+    # Only 2xx and 3xx status codes are considered successes.
+    #
+    # The `on_success` callback should accept one argument: the number of
+    # requests sent this way. The `on_failure` callback should accept two
+    # arguments: the number of successfully sent events, and an array containing
+    # the unsuccessful events.
+    #
+    # @note if you test the AsyncEmitter by using a short script to send an
+    #   event, you may find that the event fails to send. This is because the
+    #   process exits before the flushing thread is finished. You can get round
+    #   this either by adding a sleep(10) to the end of your script or by using
+    #   the synchronous flush.
+    #
+    # @param endpoint [String] the endpoint to send the events to
+    # @param options [Hash] allowed configuration options
+    #
+    # @see Emitter#initialize
+    # @api public
+    def initialize(endpoint:, options: {})
+      @queue = Queue.new
       # @all_processed_condition and @results_unprocessed are used to emulate Python's Queue.task_done()
       @queue.extend(MonitorMixin)
       @all_processed_condition = @queue.new_cond
       @results_unprocessed = 0
-      (config[:thread_count] || 1).times do
-        t = Thread.new do
-          consume
-        end
-      end
-      super(endpoint, config)
+      (options[:thread_count] || 1).times { Thread.new { consume } }
+      super(endpoint: endpoint, options: options)
     end
 
+    # AsyncEmitters use the MonitorMixin module, which provides the
+    # `synchronize` and `broadcast` methods.
+    # @private
     def consume
       loop do
         work_unit = @queue.pop
@@ -253,28 +432,34 @@ module SnowplowTracker
       end
     end
 
-    # Flush the buffer
-    #  If async is false, block until the queue is empty
+    # Flush the Emitter, forcing it to send all the events in its buffer, even
+    # if the buffer is not full.
     #
-    def flush(async=true)
+    # If `async` is true (the default), events are sent even if the queue is not
+    # empty. If `async` is false, it blocks until all queued events have been
+    # sent. Note that this method can be called by public API method
+    # {Tracker#flush}, which has a default of `async` being false.
+    #
+    # @param async [Bool] whether to flush asynchronously or not
+    #
+    # @see Emitter#flush
+    # @private
+    def flush(async = true)
       loop do
         @lock.synchronize do
-          @queue.synchronize do
-            @results_unprocessed += 1
-          end
+          @queue.synchronize { @results_unprocessed += 1 }
           @queue << @buffer
           @buffer = []
         end
-        if not async
-          LOGGER.info('Starting synchronous flush')
+        unless async
+          logger.info('Starting synchronous flush')
           @queue.synchronize do
             @all_processed_condition.wait_while { @results_unprocessed > 0 }
-            LOGGER.info('Finished synchronous flush')
+            logger.info('Finished synchronous flush')
           end
         end
-        break if @buffer.size < 1
+        break if @buffer.empty?
       end
     end
   end
-
 end

data/lib/snowplow-tracker/page.rb

@@ -0,0 +1,55 @@
+# Copyright (c) 2013-2021 Snowplow Analytics Ltd. All rights reserved.
+#
+# This program is licensed to you under the Apache License Version 2.0,
+# and you may not use this file except in compliance with the Apache License Version 2.0.
+# You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the Apache License Version 2.0 is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
+
+# Author:: Snowplow Analytics Ltd
+# Copyright:: Copyright (c) 2013-2021 Snowplow Analytics Ltd
+# License:: Apache License Version 2.0
+
+
+module SnowplowTracker
+  # If the Ruby tracker is incorporated into a website server, the events
+  # tracked will describe user activity on specific webpages. Knowing on which
+  # page an event occurred can be very valuable.
+  #
+  # Add page URL, page title and referrer URL to any event by adding a Page
+  # object to any {Tracker} `#track_x_event` method call.
+  #
+  # Page parameters are saved into the tracked event as part of the 'atomic'
+  # event properties, which have their own column in the eventual events table.
+  # For example, a Page's `page_url` parameter will be sent as `url` in the
+  # raw event payload, ending up in the `page_url` column.
+  #
+  #
+  # @note For {Tracker#track_page_view}, properties set in the Page object will
+  #   override those properties given as arguments.
+  class Page
+    # @return [Hash] the stored page properties
+    attr_reader :details
+
+    # Create a Page object for attaching page properties to events.
+    #
+    # Page properties will directly populate the event's `page_url`, `page_title` and `referrer` parameters.
+    #
+    # @example Creating a Page
+    #   SnowplowTracker::Page.new(page_url: 'http://www.example.com/second-page',
+    #            page_title: 'Example title',
+    #            referrer: 'http://www.example.com/first-page')
+    #
+    # @param page_url [String] the page URL
+    # @param page_title [String] the title of the page
+    # @param referrer [String] the URL of the previous page
+    def initialize(page_url: nil, page_title: nil, referrer: nil)
+      @details = { 'url' => page_url,
+                   'page' => page_title,
+                   'refr' => referrer }
+    end
+  end
+end