chef-telemetry 0.1.8 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7319019f20b68bb77f02c28dcdaa9dced791613f5af25c0ad43b095f55764756
-  data.tar.gz: 2fb617610995a63967cd6f00c500764aa45282518e77046d2c8fc3bd83cd8e39
+  metadata.gz: faf29c758b1dbcddd569962312d55a7f9b7648ac2fed2896a962360f62d50b93
+  data.tar.gz: '079575327ecc06c86e9fb305cd4e0b6339ea443ad53800006f74baa710d9f266'
 SHA512:
-  metadata.gz: 8914e2941c35f42086f89a0bb5da066c8c069aaee72466bfc74a55d96c0972cba8c5c275aadbef3fba54eb5eb86527655fd43c6f80e007818122cef06b9b5038
-  data.tar.gz: 06163b5ddb874c882a7c51046f4f5e34c9412e857e767fe79f2258da8ff27d93666391af58077eef2eae12b2f308cba73953fd5de56de34ea9b942c4b7b0c4f8
+  metadata.gz: 35a60bfa480f9085eac39bd3f1699ab2926f1c270e947439c13bfbc585e97849b5cb155bf8eaadef41496fc0a62c13867dc25b3380ece12d1460e84fee8794aa
+  data.tar.gz: 4bd4017b173ba7a50f5888550c16e1b0616c0a2b6176ba719513bad27ece43ce32dca9208ad7a891e519e3d957e64c26ce8ef56e20ca2eace83383c743b81592

data/lib/chef/telemeter.rb

@@ -0,0 +1,162 @@
+#
+# Copyright:: Copyright (c) 2018 Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "chef/telemetry/version"
+require "benchmark"
+require "forwardable"
+require "singleton"
+require "json"
+require "digest/sha1"
+require "securerandom"
+require "yaml"
+
+module Chef
+  # This definites the Telemeter interface. Implementation thoughts for
+  # when we unstub it:
+  # - let's track the call sequence; most of our calls will be nested inside
+  # a main 'timed_capture', and it would be good to see ordering within nested calls.
+  class Telemeter
+    include Singleton
+    DEFAULT_INSTALLATION_GUID = "00000000-0000-0000-0000-000000000000".freeze
+
+    class << self
+      extend Forwardable
+      def_delegators :instance, :setup, :timed_capture, :capture, :commit, :timed_run_capture
+      def_delegators :instance, :pending_event_count, :last_event, :enabled?
+      def_delegators :instance, :make_event_payload, :config
+    end
+
+    attr_reader :events_to_send, :run_timestamp, :config
+
+    def setup(config)
+      # TODO validate required & correct keys
+      # :payload_dir #required
+      # :session_file # required
+      # :installation_identifier_file # required
+      # :enabled  # false, not required
+      # :dev_mode # false, not required
+      config[:dev_mode] ||= false
+      config[:enabled] ||= false
+      require "chef_core/telemeter/sender"
+      @config = config
+      Sender.start_upload_thread(config)
+    end
+
+    def enabled?
+      require "chef/telemetry/decision"
+      config[:enabled] && !Telemetry::Decision.env_opt_out?
+    end
+
+    def initialize
+      @events_to_send = []
+      @run_timestamp =  Time.now.utc.strftime("%FT%TZ")
+    end
+
+    def timed_run_capture(arguments, &block)
+      timed_capture(:run, arguments: arguments, &block)
+    end
+
+    def timed_capture(name, data = {}, options = {})
+      time = Benchmark.measure { yield }
+      data[:duration] = time.real
+      capture(name, data, options)
+    end
+
+    def capture(name, data = {}, options = {})
+      # Adding it to the head of the list will ensure that the
+      # sequence of events is preserved when we send the final payload
+      payload = make_event_payload(name, data, options)
+      @events_to_send.unshift payload
+    end
+
+    def commit
+      if enabled?
+        session = convert_events_to_session
+        write_session(session)
+      end
+      @events_to_send = []
+    end
+
+    def make_event_payload(name, data, options = {})
+      payload = {
+        event: name,
+        properties: {
+          installation_id: installation_id,
+          run_timestamp: run_timestamp,
+          host_platform: host_platform,
+        },
+      }
+      if options[:flatten]
+        payload[:properties].merge! data
+      else
+        payload[:properties][:event_data] = data
+      end
+      payload
+    end
+
+    def installation_id
+      @installation_id ||=
+        begin
+          File.read(config[:installation_identifier_file]).chomp
+        rescue
+          Telemeter::Log.info "could not read #{config[:installation_identifier_file]} - using default id"
+          DEFAULT_INSTALLATION_GUID
+        end
+    end
+
+    # For testing.
+    def pending_event_count
+      @events_to_send.length
+    end
+
+    def last_event
+      @events_to_send.last
+    end
+
+    private
+
+    def host_platform
+      @host_platform ||= case RUBY_PLATFORM
+                         when /mswin|mingw|windows/
+                           "windows"
+                         else
+                           RUBY_PLATFORM.split("-")[1]
+                         end
+    end
+
+    def convert_events_to_session
+      YAML.dump("version" => Telemetry::VERSION,
+                "entries" => @events_to_send)
+    end
+
+    def write_session(session)
+      File.write(next_filename, convert_events_to_session)
+    end
+
+    def next_filename
+      id = 0
+      filename = ""
+      loop do
+        id += 1
+        filename = File.join(config[:payload_dir], "telemetry-payload-#{id}.yml")
+        break unless File.exist?(filename)
+      end
+      filename
+    end
+
+  end
+end # Chef

data/lib/chef/telemeter/sender.rb

@@ -0,0 +1,125 @@
+#
+# Copyright:: Copyright (c) 2018 Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require "chef/telemetry"
+require "chef/telemeter"
+require "logger"
+
+module Chef
+  class Telemeter
+    logger = ::Logger.new(STDERR) # TODO: maybe switch to file, maybe switch to mixlib
+    logger.level = Logger::ERROR
+    Log = logger # rubocop:disable Naming/ConstantName
+
+    class Sender
+      attr_reader :session_files, :config
+
+      def self.start_upload_thread(config)
+        # Find the files before we spawn the thread - otherwise
+        # we may accidentally pick up the current run's session file if it
+        # finishes before the thread scans for new files
+        session_files = Sender.find_session_files(config)
+        sender = Sender.new(session_files, config)
+        Thread.new { sender.run }
+      end
+
+      def self.find_session_files(config)
+        Telemeter::Log.info("Looking for telemetry data to submit")
+        session_search = File.join(config[:payload_dir], "telemetry-payload-*.yml")
+        session_files = Dir.glob(session_search)
+        Telemeter::Log.info("Found #{session_files.length} sessions to submit")
+        session_files
+      end
+
+      def initialize(session_files, config)
+        @session_files = session_files
+        @config = config
+      end
+
+      def run
+        if Telemeter.enabled?
+          Telemeter::Log.info("Telemetry enabled, beginning upload of previous session(s)")
+          # dev mode telemetry gets sent to a different location
+
+          if config[:dev_mode]
+            ENV["CHEF_TELEMETRY_ENDPOINT"] ||= "https://telemetry-acceptance.chef.io"
+          end
+          session_files.each { |path| process_session(path) }
+        else
+          # If telemetry is not enabled, just clean up and return. Even though
+          # the telemetry gem will not send if disabled, log output saying that we're submitting
+          # it when it has been disabled can be alarming.
+          Telemeter::Log.info("Telemetry disabled, clearing any existing session captures without sending them.")
+          session_files.each { |path| FileUtils.rm_rf(path) }
+        end
+        FileUtils.rm_rf(config[:session_file])
+        Telemeter::Log.info("Terminating, nothing more to do.")
+      rescue => e
+        Telemeter::Log.fatal "Sender thread aborted: '#{e}' failed at  #{e.backtrace[0]}"
+      end
+
+      def process_session(path)
+        Telemeter::Log.info("Processing telemetry entries from #{path}")
+        content = load_and_clear_session(path)
+        submit_session(content)
+      end
+
+      def submit_session(content)
+        # Each file contains the actions taken within a single run of the chef tool.
+        # Each run is one session, so we'll first remove remove the session file
+        # to force creating a new one.
+        FileUtils.rm_rf(config[:session_file])
+        # We'll use the version captured in the sesion file
+        entries = content["entries"]
+        total = entries.length
+        product_info = config[:product] || {}
+        telemetry = Telemetry.new(product: product_info[:name] || "chef-workstation",
+                                  origin: product_info[:origin] || "command-line",
+                                  product_version: product_info[:version] || content["version"],
+                                  install_context: product_info[:install_context] || "omnibus")
+        total = entries.length
+        entries.each_with_index do |entry, x|
+          submit_entry(telemetry, entry, x + 1, total)
+        end
+      end
+
+      def submit_entry(telemetry, entry, sequence, total)
+        Telemeter::Log.info("Submitting telemetry entry #{sequence}/#{total}: #{entry} ")
+        telemetry.deliver(entry)
+        Telemeter::Log.info("Entry #{sequence}/#{total} submitted.")
+      rescue => e
+        # No error handling in telemetry lib, so at least track the failrue
+        Telemeter::Log.error("Failed to send entry #{sequence}/#{total}: #{e}")
+        Telemeter::Log.error("Backtrace: #{e.backtrace} ")
+      end
+
+      private
+
+      def load_and_clear_session(path)
+        content = File.read(path)
+        # We'll remove it now instead of after we parse or submit it -
+        # if we fail to deliver, we don't want to be stuck resubmitting it if the problem
+        # was due to payload. This is a trade-off - if we get a transient error, the
+        # payload will be lost.
+        # TODO: Improve error handling so we can intelligently decide whether to
+        #       retry a failed load or failed submit.
+        FileUtils.rm_rf(path)
+        YAML.load(content)
+      end
+    end
+  end
+end # Chef

data/lib/chef/telemetry.rb

@@ -0,0 +1,45 @@
+require "chef/telemetry/client"
+require "chef/telemetry/decision"
+require "chef/telemetry/event"
+require "chef/telemetry/session"
+require "chef/telemetry/version"
+
+module Chef
+  class Telemetry
+    attr_accessor :product, :origin, :product_version, :install_context
+    def initialize(product: nil, origin: "command-line",
+                   product_version: "0.0.0",
+                   install_context: "omnibus")
+      # Reference: https://github.com/chef/es-telemetry-pipeline/blob/0730c1e2605624a50d34bab6d036b73c31e0ab0e/schema/event.schema.json#L77
+      @product = product
+      @origin = origin
+      @product_version = product_version
+      @install_context = install_context # Valid: habitat, omnibus
+    end
+
+    def deliver(data = {})
+      unless opt_out?
+        payload = event.prepare(data)
+        client.await.fire(payload)
+      end
+    end
+
+    def event
+      @event ||= Event.new(product, session, origin,
+                           install_context, product_version)
+    end
+
+    def session
+      @session ||= Session.new
+    end
+
+    def opt_out?
+      @opt_out ||= Decision.opt_out?
+    end
+
+    def client
+      endpoint = ENV.fetch("CHEF_TELEMETRY_ENDPOINT", Client::TELEMETRY_ENDPOINT)
+      @client ||= Client.new(endpoint)
+    end
+  end
+end # Chef

data/lib/chef/telemetry/client.rb

@@ -0,0 +1,26 @@
+require "http"
+require "concurrent"
+
+module Chef
+  class Telemetry
+    class Client
+      include Concurrent::Async
+
+      TELEMETRY_ENDPOINT = "https://telemetry.chef.io".freeze
+
+      attr_reader :http
+      def initialize(endpoint = TELEMETRY_ENDPOINT)
+        super()
+        @http = HTTP.persistent(endpoint)
+      end
+
+      def fire(event)
+        http.post("/events", json: event).flush
+      end
+    end
+
+    class OptOutClient
+      def fire(_); end
+    end
+  end
+end

data/lib/chef/telemetry/decision.rb

@@ -0,0 +1,67 @@
+require "chef-config/path_helper"
+require "chef-config/windows"
+
+# Decision allows us to inspect whether the user has made a decision to opt in or opt out of telemetry.
+module Chef
+  class Telemetry
+    module Decision
+      OPT_OUT_FILE = "telemetry_opt_out".freeze
+      OPT_IN_FILE = "telemetry_opt_in".freeze
+
+      class << self
+        def opt_out?
+          # We check that the user has made a decision so that we can have a default setting for robots
+          user_opted_out? || env_opt_out? || local_opt_out? || !made?
+        end
+
+        # Check whether the user has made an explicit decision on their participation.
+        def made?
+          user_opted_in? || user_opted_out?
+        end
+
+        def user_opted_out?
+          File.exist?(File.join(home, OPT_OUT_FILE))
+        end
+
+        def user_opted_in?
+          File.exist?(File.join(home, OPT_IN_FILE))
+        end
+
+        def env_opt_out?
+          ENV.key?("CHEF_TELEMETRY_OPT_OUT")
+        end
+
+        def local_opt_out?
+          found = false
+          full_path = working_directory.split(File::SEPARATOR)
+          (full_path.length - 1).downto(0) do |i|
+            candidate = File.join(full_path[0..i], ".chef", OPT_OUT_FILE)
+            if File.exist?(candidate)
+              # TODO: push up logging
+              # Log.info "Found opt out at: #{candidate}"
+              found = true
+              break
+            end
+          end
+          found
+        end
+
+        private
+
+        def working_directory
+          a = if ChefConfig.windows?
+                ENV["CD"]
+              else
+                ENV["PWD"]
+              end || Dir.pwd
+
+          a
+        end
+
+        def home
+          ChefConfig::PathHelper.home(".chef")
+        end
+      end
+    end
+  end
+end # Chef

data/lib/chef/telemetry/event.rb

@@ -0,0 +1,44 @@
+module Chef
+  class Telemetry
+    class Event
+
+      SKELETON = {
+        instance_id: "00000000-0000-0000-0000-000000000000",
+        message_version: 1.0,
+        payload_version: 1.0,
+        license_id: "00000000-0000-0000-0000-000000000000",
+        type: "track",
+      }.freeze
+
+      attr_reader :session, :product, :origin,
+        :product_version, :install_context
+      def initialize(product, session, origin = "command-line",
+                     install_context = "omnibus", product_version = "0.0.0")
+        @product = product
+        @session = session
+        @origin = origin
+        @product_version = product_version
+        @install_context = install_context
+      end
+
+      def prepare(event)
+        time = timestamp
+        event[:properties][:timestamp] = time
+        body = SKELETON.dup
+        body.tap do |b|
+          b[:session_id] = session.id
+          b[:origin] = origin
+          b[:product] = product
+          b[:product_version] = product_version
+          b[:install_context] = install_context
+          b[:timestamp] = time
+          b[:payload] = event
+        end
+      end
+
+      def timestamp
+        Time.now.utc.strftime("%FT%TZ")
+      end
+    end
+  end
+end

data/lib/chef/telemetry/session.rb

@@ -0,0 +1,39 @@
+require "securerandom"
+require "chef-config/path_helper"
+
+module Chef
+  class Telemetry
+    class Session
+      def initialize
+        @id = if live_session?
+                File.read(session_file).chomp
+              else
+                new_session
+              end
+      end
+
+      def id
+        FileUtils.touch(session_file)
+        @id
+      end
+
+      private
+
+      def live_session?
+        expiry = Time.now - 600
+        File.file?(session_file) && File.stat(session_file).mtime > expiry
+      end
+
+      def session_file
+        File.join(ChefConfig::PathHelper.home(".chef"), "TELEMETRY_SESSION_ID").freeze
+      end
+
+      def new_session
+        id = SecureRandom.uuid
+        FileUtils.mkdir_p(File.dirname(session_file))
+        File.open(session_file, "w") { |f| f.write(id) }
+        id
+      end
+    end
+  end
+end # Chef

data/lib/chef/telemetry/version.rb

@@ -0,0 +1,5 @@
+module Chef
+  class Telemetry
+    VERSION = "1.0.0".freeze
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: chef-telemetry
 version: !ruby/object:Gem::Version
-  version: 0.1.8
+  version: 1.0.0
 platform: ruby
 authors:
 - Chef Software, Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-01-16 00:00:00.000000000 Z
+date: 2019-12-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -87,12 +87,14 @@ extensions: []
 extra_rdoc_files: []
 files:
 - LICENSE
-- lib/telemetry.rb
-- lib/telemetry/client.rb
-- lib/telemetry/decision.rb
-- lib/telemetry/event.rb
-- lib/telemetry/session.rb
-- lib/telemetry/version.rb
+- lib/chef/telemeter.rb
+- lib/chef/telemeter/sender.rb
+- lib/chef/telemetry.rb
+- lib/chef/telemetry/client.rb
+- lib/chef/telemetry/decision.rb
+- lib/chef/telemetry/event.rb
+- lib/chef/telemetry/session.rb
+- lib/chef/telemetry/version.rb
 homepage: https://github.com/chef/chef-telemetry
 licenses:
 - Apache-2.0
@@ -112,8 +114,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Send user actions to the Chef telemetry system. See Chef RFC-051 for further

data/lib/telemetry.rb

@@ -1,43 +0,0 @@
-require "telemetry/client"
-require "telemetry/decision"
-require "telemetry/event"
-require "telemetry/session"
-require "telemetry/version"
-
-class Telemetry
-  attr_accessor :product, :origin, :product_version, :install_context
-  def initialize(product: nil, origin: "command-line",
-                 product_version: "0.0.0",
-                 install_context: "omnibus")
-    # Reference: https://github.com/chef/es-telemetry-pipeline/blob/0730c1e2605624a50d34bab6d036b73c31e0ab0e/schema/event.schema.json#L77
-    @product = product
-    @origin = origin
-    @product_version = product_version
-    @install_context = install_context # Valid: habitat, omnibus
-  end
-
-  def deliver(data = {})
-    unless opt_out?
-      payload = event.prepare(data)
-      client.await.fire(payload)
-    end
-  end
-
-  def event
-    @event ||= Event.new(product, session, origin,
-                         install_context, product_version)
-  end
-
-  def session
-    @session ||= Session.new
-  end
-
-  def opt_out?
-    @opt_out ||= Decision.opt_out?
-  end
-
-  def client
-    endpoint = ENV.fetch("CHEF_TELEMETRY_ENDPOINT", Client::TELEMETRY_ENDPOINT)
-    @client ||= Client.new(endpoint)
-  end
-end

data/lib/telemetry/client.rb

@@ -1,24 +0,0 @@
-require "http"
-require "concurrent"
-
-class Telemetry
-  class Client
-    include Concurrent::Async
-
-    TELEMETRY_ENDPOINT = "https://telemetry.chef.io".freeze
-
-    attr_reader :http
-    def initialize(endpoint = TELEMETRY_ENDPOINT)
-      super()
-      @http = HTTP.persistent(endpoint)
-    end
-
-    def fire(event)
-      http.post("/events", json: event).flush
-    end
-  end
-
-  class OptOutClient
-    def fire(_); end
-  end
-end

data/lib/telemetry/decision.rb

@@ -1,64 +0,0 @@
-require "chef-config/path_helper"
-require "chef-config/windows"
-
-# Decision allows us to inspect whether the user has made a decision to opt in or opt out of telemetry.
-class Telemetry
-  module Decision
-    OPT_OUT_FILE = "telemetry_opt_out".freeze
-    OPT_IN_FILE = "telemetry_opt_in".freeze
-
-    class << self
-      def opt_out?
-        # We check that the user has made a decision so that we can have a default setting for robots
-        user_opted_out? || env_opt_out? || local_opt_out? || !made?
-      end
-
-      # Check whether the user has made an explicit decision on their participation.
-      def made?
-        user_opted_in? || user_opted_out?
-      end
-
-      def user_opted_out?
-        File.exist?(File.join(home, OPT_OUT_FILE))
-      end
-
-      def user_opted_in?
-        File.exist?(File.join(home, OPT_IN_FILE))
-      end
-
-      def env_opt_out?
-        ENV.key?("CHEF_TELEMETRY_OPT_OUT")
-      end
-
-      def local_opt_out?
-        found = false
-        full_path = working_directory.split(File::SEPARATOR)
-        (full_path.length - 1).downto(0) do |i|
-          candidate = File.join(full_path[0..i], ".chef", OPT_OUT_FILE)
-          if File.exist?(candidate)
-            puts "Found opt out at: #{candidate}"
-            found = true
-            break
-          end
-        end
-        found
-      end
-
-      private
-
-      def working_directory
-        a = if ChefConfig.windows?
-              ENV["CD"]
-            else
-              ENV["PWD"]
-            end || Dir.pwd
-
-        a
-      end
-
-      def home
-        ChefConfig::PathHelper.home(".chef")
-      end
-    end
-  end
-end

data/lib/telemetry/event.rb

@@ -1,42 +0,0 @@
-class Telemetry
-  class Event
-
-    SKELETON = {
-      instance_id: "00000000-0000-0000-0000-000000000000",
-      message_version: 1.0,
-      payload_version: 1.0,
-      license_id: "00000000-0000-0000-0000-000000000000",
-      type:  "track",
-    }.freeze
-
-    attr_reader :session, :product, :origin,
-      :product_version, :install_context
-    def initialize(product, session, origin = "command-line",
-                   install_context = "omnibus", product_version = "0.0.0")
-      @product = product
-      @session = session
-      @origin = origin
-      @product_version = product_version
-      @install_context = install_context
-    end
-
-    def prepare(event)
-      time = timestamp
-      event[:properties][:timestamp] = time
-      body = SKELETON.dup
-      body.tap do |b|
-        b[:session_id] = session.id
-        b[:origin] = origin
-        b[:product] = product
-        b[:product_version] = product_version
-        b[:install_context] = install_context
-        b[:timestamp] = time
-        b[:payload] = event
-      end
-    end
-
-    def timestamp
-      Time.now.utc.strftime("%FT%TZ")
-    end
-  end
-end

data/lib/telemetry/session.rb

@@ -1,37 +0,0 @@
-require "securerandom"
-require "chef-config/path_helper"
-
-class Telemetry
-  class Session
-    def initialize
-      @id = if live_session?
-              File.read(session_file).chomp
-            else
-              new_session
-            end
-    end
-
-    def id
-      FileUtils.touch(session_file)
-      @id
-    end
-
-    private
-
-    def live_session?
-      expiry = Time.now - 600
-      File.file?(session_file) && File.stat(session_file).mtime > expiry
-    end
-
-    def session_file
-      File.join(ChefConfig::PathHelper.home(".chef"), "TELEMETRY_SESSION_ID").freeze
-    end
-
-    def new_session
-      id = SecureRandom.uuid
-      FileUtils.mkdir_p(File.dirname(session_file))
-      File.open(session_file, "w") { |f| f.write(id) }
-      id
-    end
-  end
-end

data/lib/telemetry/version.rb

@@ -1,3 +0,0 @@
-class Telemetry
-  VERSION = "0.1.8".freeze
-end