pf2 0.12.0 → 0.14.0

data/lib/pf2/reporter/pprof.rb

@@ -0,0 +1,349 @@
+# frozen_string_literal: true
+
+require 'stringio'
+require 'zlib'
+
+module Pf2
+  module Reporter
+    # A minimal Protobuf encoder.
+    class Protobuf
+      def initialize
+        @data = +"".b
+      end
+
+      def to_bytes
+        @data
+      end
+
+      # Wire types
+      # Not implemented: I64, SGROUP, EGROUP, I32
+
+      WIRETYPE_VARINT = 0
+      WIRETYPE_LEN = 2
+
+      def put_byte(b)
+        @data << (b & 0xFF).chr(Encoding::BINARY)
+      end
+
+      def put_varint(val)
+        # Emit 7-bit chunks
+        # msb=1 for continuation
+        while val > 0x7F
+          put_byte((val & 0x7F) | 0x80)
+          val >>= 7
+        end
+        put_byte(val & 0xFF)
+      end
+
+      def wire_varint(field, val)
+        put_varint((field << 3) | WIRETYPE_VARINT) # tag
+        put_varint(val)
+      end
+
+      def wire_len(field, bytes)
+        put_varint((field << 3) | WIRETYPE_LEN) # tag
+        put_varint(bytes.bytesize)
+        @data << bytes.b
+      end
+
+      # ---
+
+      # Scalar value types
+      # Not implemented:
+      #   double, float, uint32, sint32, sint64,
+      #   fixed32, fixed64, sfixed32, sfixed64, bytes
+
+      def int32(field, val)
+        twos_comp = val & 0xFFFF_FFFF
+        wire_varint(field, twos_comp)
+      end
+
+      def int64(field, val)
+        twos_comp = val & 0xFFFF_FFFF_FFFF_FFFF
+        wire_varint(field, twos_comp)
+      end
+
+      def uint64(field, val)
+        wire_varint(field, val)
+      end
+
+      def bool(field, val)
+        wire_varint(field, val ? 1 : 0)
+      end
+
+      def string(field, val)
+        wire_len(field, val)
+      end
+
+      # ---
+
+      def submessage(field, submsg)
+        wire_len(field, submsg)
+      end
+    end
+
+    PROTO_TAGS = {
+      profile: {
+        sample_type: 1,
+        sample: 2,
+        mapping: 3,
+        location: 4,
+        function: 5,
+        string_table: 6,
+        drop_frames: 7,
+        keep_frames: 8,
+        time_nanos: 9,
+        duration_nanos: 10,
+        period_type: 11,
+        period: 12,
+        comment: 13,
+        default_sample_type: 14,
+        doc_url: 15,
+      },
+      value_type: {
+        type: 1,
+        unit: 2,
+      },
+      sample: {
+        location_id: 1,
+        value: 2,
+        label: 3,
+      },
+      label: {
+        key: 1,
+        str: 2,
+        num: 3,
+        num_unit: 4,
+      },
+      mapping: {
+        id: 1,
+        memory_start: 2,
+        memory_limit: 3,
+        file_offset: 4,
+        filename: 5,
+        build_id: 6,
+        has_functions: 7,
+        has_filenames: 8,
+        has_line_numbers: 9,
+        has_inline_frames: 10,
+      },
+      location: {
+        id: 1,
+        mapping_id: 2,
+        address: 3,
+        line: 4,
+        is_folded: 5,
+      },
+      line: {
+        function_id: 1,
+        line: 2,
+        column: 3,
+      },
+      function: {
+        id: 1,
+        name: 2,
+        system_name: 3,
+        filename: 4,
+        start_line: 5,
+      },
+    }
+
+    class Pprof
+      def initialize(profile)
+        @profile = profile
+        @stack_weaver = StackWeaver.new(profile)
+
+        @strings = []
+        @string_id_table = {}
+        @function_id_map = {}
+        @location_id_map = {}
+
+        string_index("") # index 0 is empty string
+      end
+
+      def emit
+        pb = Protobuf.new
+
+        pb.int64(PROTO_TAGS[:profile][:time_nanos], @profile[:start_timestamp_ns] || 0)
+        pb.int64(PROTO_TAGS[:profile][:duration_nanos], @profile[:duration_ns] || 0)
+
+        # Sample types
+        pb.submessage(PROTO_TAGS[:profile][:sample_type], pb_value_type("samples", "count"))
+        pb.submessage(PROTO_TAGS[:profile][:sample_type], pb_value_type("cpu", "nanoseconds"))
+
+        # Period
+        pb.submessage(PROTO_TAGS[:profile][:period_type], pb_value_type("cpu", "nanoseconds"))
+        pb.int64(PROTO_TAGS[:profile][:period], inferred_period_ns)
+
+        build_functions(pb)
+        build_locations(pb)
+        build_samples(pb)
+
+        # String table
+        @strings.each do |s|
+          pb.string(PROTO_TAGS[:profile][:string_table], s)
+        end
+
+        buffer = StringIO.new
+        Zlib::GzipWriter.wrap(buffer) { |gz| gz.write(pb.to_bytes) }
+        buffer.string
+      end
+
+      private
+
+      def string_index(str)
+        if @string_id_table.key?(str)
+          return @string_id_table[str]
+        end
+
+        id = @strings.length
+        @strings << str
+        @string_id_table[str] = id
+        id
+      end
+
+      def pb_value_type(type, unit)
+        pb = Protobuf.new
+        pb.int64(PROTO_TAGS[:value_type][:type], string_index(type))
+        pb.int64(PROTO_TAGS[:value_type][:unit], string_index(unit))
+        pb.to_bytes
+      end
+
+      def pb_sample(location_ids, cpu_value_ns)
+        pb = Protobuf.new
+        # Equalivalent to:
+        # location_ids.each do |loc_id|
+        #   pb.uint64(PROTO_TAGS[:sample][:location_id], loc_id)
+        # end
+        # pb.int64(PROTO_TAGS[:sample][:value], 1) # samples
+        # pb.int64(PROTO_TAGS[:sample][:value], cpu_value_ns) # cpu nanoseconds
+        pack_repeated_uint64(pb, PROTO_TAGS[:sample][:location_id], location_ids)
+        pack_repeated_int64(pb, PROTO_TAGS[:sample][:value], [1, cpu_value_ns])
+        pb.to_bytes
+      end
+
+      def pack_repeated_uint64(pb, field_no, values)
+        packed = Protobuf.new
+        values.each { |v| packed.put_varint(v) }
+        pb.wire_len(field_no, packed.to_bytes)
+      end
+
+      def pack_repeated_int64(pb, field_no, values)
+        packed = Protobuf.new
+        values.each { |v| packed.put_varint(v & 0xFFFF_FFFF_FFFF_FFFF) }
+        pb.wire_len(field_no, packed.to_bytes)
+      end
+
+      # def pb_mapping(id, memory_start, memory_limit, file_offset, filename)
+      #   pb = Protobuf.new
+      #   pb.uint64(PROTO_TAGS[:mapping][:id], id)
+      #   pb.uint64(PROTO_TAGS[:mapping][:memory_start], memory_start)
+      #   pb.uint64(PROTO_TAGS[:mapping][:memory_limit], memory_limit)
+      #   pb.uint64(PROTO_TAGS[:mapping][:file_offset], file_offset)
+      #   pb.int64(PROTO_TAGS[:mapping][:filename], string_index(filename))
+      #   pb.int64(PROTO_TAGS[:mapping][:build_id], 123456)
+      #   pb.bool(PROTO_TAGS[:mapping][:has_functions], true)
+      #   pb.bool(PROTO_TAGS[:mapping][:has_filenames], true)
+      #   pb.bool(PROTO_TAGS[:mapping][:has_line_numbers], true)
+      #   pb.bool(PROTO_TAGS[:mapping][:has_inline_frames], false)
+      #   pb.to_bytes
+      # end
+
+      def pb_location(id, mapping_id, address, lines)
+        pb = Protobuf.new
+        pb.uint64(PROTO_TAGS[:location][:id], id)
+        pb.uint64(PROTO_TAGS[:location][:mapping_id], mapping_id)
+        pb.uint64(PROTO_TAGS[:location][:address], address)
+        lines.each do |fn_id, lineno, column|
+          pb.submessage(PROTO_TAGS[:location][:line], pb_line(fn_id, lineno, column || 0))
+        end
+        pb
+      end
+
+      def pb_line(function_id, line, column = 0)
+        pb = Protobuf.new
+        pb.uint64(PROTO_TAGS[:line][:function_id], function_id)
+        pb.int32(PROTO_TAGS[:line][:line], line)
+        pb.int32(PROTO_TAGS[:line][:column], column) if column != 0
+        pb.to_bytes
+      end
+
+      def pb_function(id, name, system_name, filename, start_line)
+        pb = Protobuf.new
+        pb.uint64(PROTO_TAGS[:function][:id], id)
+        pb.int64(PROTO_TAGS[:function][:name], string_index(name))
+        pb.int64(PROTO_TAGS[:function][:system_name], string_index(system_name))
+        pb.int64(PROTO_TAGS[:function][:filename], string_index(filename))
+        pb.int64(PROTO_TAGS[:function][:start_line], start_line)
+        pb.to_bytes
+      end
+
+      def build_functions(pb)
+        @profile[:functions].each.with_index do |fn, idx|
+          id = idx + 1
+          @function_id_map[idx] = id
+          pb.submessage(
+            PROTO_TAGS[:profile][:function],
+            pb_function(
+              id,
+              fn[:name] || "<unknown>",
+              "",
+              fn[:filename] || "<unknown>",
+              fn[:start_lineno] || 0,
+            )
+          )
+        end
+      end
+
+      def build_locations(pb)
+        @profile[:locations].each.with_index do |loc, idx|
+          id = idx + 1
+          @location_id_map[idx] = id
+          function_id = @function_id_map[loc[:function_index]] || 0
+          pb.submessage(
+            PROTO_TAGS[:profile][:location],
+            pb_location(
+              id,
+              0, # mapping_id
+              loc[:address] || 0,
+              [[function_id, loc[:lineno] || 0]],
+            ).to_bytes
+          )
+        end
+      end
+
+      def build_samples(pb)
+        cpu_value_ns = inferred_sample_value_ns
+
+        @profile[:samples].each do |sample|
+          woven_stack = @stack_weaver.weave(sample[:stack], sample[:native_stack] || [])
+          woven_stack.reverse! # leaf -> root for pprof
+          location_ids = woven_stack.map { |idx| @location_id_map[idx] }.compact
+          next if location_ids.empty?
+
+          pb.submessage(
+            PROTO_TAGS[:profile][:sample],
+            pb_sample(location_ids, cpu_value_ns)
+          )
+        end
+      end
+
+      def inferred_period_ns
+        value = inferred_sample_value_ns
+        return value if value > 0
+
+        9_000_000 # default 9 ms
+      end
+
+      def inferred_sample_value_ns
+        samples = @profile[:samples]
+        return 0 if samples.empty?
+
+        duration = @profile[:duration_ns] || 0
+        return 0 if duration == 0
+
+        (duration.to_f / samples.length).round
+      end
+    end
+  end
+end

data/lib/pf2/reporter.rb

@@ -3,3 +3,4 @@
 require_relative './reporter/annotate'
 require_relative './reporter/stack_weaver'
 require_relative './reporter/firefox_profiler_ser2'
+require_relative './reporter/pprof'

data/lib/pf2/serve.rb

@@ -27,10 +27,9 @@ module Pf2
       server = WEBrick::HTTPServer.new(CONFIG)
       server.mount_proc('/profile') do |req, res|
         profile = Pf2.stop
-        profile = JSON.parse(profile, symbolize_names: true, max_nesting: false)
         res.header['Content-Type'] = 'application/json'
         res.header['Access-Control-Allow-Origin'] = '*'
-        res.body = JSON.generate(Pf2::Reporter::FirefoxProfiler.new((profile)).emit)
+        res.body = Pf2::Reporter::FirefoxProfilerSer2.new(profile).emit
         Pf2.start
       end
 

data/lib/pf2/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Pf2
-  VERSION = '0.12.0'
+  VERSION = '0.14.0'
 end

data/lib/pf2.rb

@@ -6,29 +6,79 @@ require_relative 'pf2/version'
 module Pf2
   class Error < StandardError; end
 
+  KNOWN_FORMATS = [:pf2prof, :firefox, :pprof] # :nodoc:
+  private_constant :KNOWN_FORMATS
+
+  # Start a profiling session.
+  #
+  # Parameters:
+  # interval_ms - Sampling interval in milliseconds (default: 9)
+  # time_mode   - :cpu or :wall (default: :cpu)
   def self.start(...)
     @@session = Session.new(...)
     @@session.start
   end
 
+  # Stop the current profiling session and return the profile data.
   def self.stop
     @@session.stop
   end
 
   # Profiles the given block of code.
   #
+  # Parameters:
+  # interval_ms - Sampling interval in milliseconds (default: 9)
+  # time_mode   - :cpu or :wall (default: :cpu)
+  # out         - String or IO-like object specifying where to write profile data.
+  #                 - nil (default): do not write to file
+  #                 - String: file path to write the profile data
+  #                 - IO-like object: an object responding to #write
+  # format      - Output format. Possible values are:
+  #                 - :pf2prof: Raw profile dump loadable by 'pf2 report'
+  #                 - :firefox (default): JSON for profiler.firefox.com
+  #                 - :pprof: .pb.gz compatible with Golang's pprof tool
+  #
   # Example:
   #
   #     profile = Pf2.profile(interval_ms: 42) do
   #       your_code_here
   #     end
   #
-  def self.profile(**kwargs, &block)
+  def self.profile(interval_ms: 9, time_mode: :cpu, out: nil, format: :firefox, &block)
     raise ArgumentError, "block required" unless block_given?
-    start(**kwargs)
+    raise ArgumentError, "Unknown format: #{format}" unless KNOWN_FORMATS.include?(format)
+    if !(out.nil? || out.is_a?(String) || (out.respond_to?(:write) && out.respond_to?(:close)))
+      raise ArgumentError, "'out' must be an IO-like object"
+    end
+
+    start(interval_ms:, time_mode:)
     yield
-    result = stop
+    result = stop()
     @@session = nil # let GC clean up the session
+
+    if out
+      is_path_passed = out.is_a?(String)
+      io = is_path_passed ? File.open(out, "wb") : out
+      case format
+      in :pf2prof
+        io.write(Marshal.dump(result))
+      in :firefox
+        require 'pf2/reporter'
+        reporter = Reporter::FirefoxProfilerSer2.new(result)
+        io.write(reporter.emit)
+      in :pprof
+        require 'pf2/reporter'
+        reporter = Reporter::Pprof.new(result)
+        io.write(reporter.emit)
+      end
+      io.close if is_path_passed
+    end
+
     result
+  ensure
+    if defined?(@@session) && @@session != nil
+      stop
+      @@session = nil
+    end
   end
 end

data/vendor/profile.proto

@@ -0,0 +1,233 @@
+// Copyright 2016 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Profile is a common stacktrace profile format.
+//
+// Measurements represented with this format should follow the
+// following conventions:
+//
+// - Consumers should treat unset optional fields as if they had been
+//   set with their default value.
+//
+// - When possible, measurements should be stored in "unsampled" form
+//   that is most useful to humans.  There should be enough
+//   information present to determine the original sampled values.
+//
+// - On-disk, the serialized proto must be gzip-compressed.
+//
+// - The profile is represented as a set of samples, where each sample
+//   references a sequence of locations, and where each location belongs
+//   to a mapping.
+// - There is a N->1 relationship from sample.location_id entries to
+//   locations. For every sample.location_id entry there must be a
+//   unique Location with that id.
+// - There is an optional N->1 relationship from locations to
+//   mappings. For every nonzero Location.mapping_id there must be a
+//   unique Mapping with that id.
+
+syntax = "proto3";
+
+package perftools.profiles;
+
+option java_package = "com.google.perftools.profiles";
+option java_outer_classname = "ProfileProto";
+
+message Profile {
+  // A description of the samples associated with each Sample.value.
+  // For a cpu profile this might be:
+  //   [["cpu","nanoseconds"]] or [["wall","seconds"]] or [["syscall","count"]]
+  // For a heap profile, this might be:
+  //   [["allocations","count"], ["space","bytes"]],
+  // If one of the values represents the number of events represented
+  // by the sample, by convention it should be at index 0 and use
+  // sample_type.unit == "count".
+  repeated ValueType sample_type = 1;
+  // The set of samples recorded in this profile.
+  repeated Sample sample = 2;
+  // Mapping from address ranges to the image/binary/library mapped
+  // into that address range.  mapping[0] will be the main binary.
+  repeated Mapping mapping = 3;
+  // Locations referenced by samples.
+  repeated Location location = 4;
+  // Functions referenced by locations.
+  repeated Function function = 5;
+  // A common table for strings referenced by various messages.
+  // string_table[0] must always be "".
+  repeated string string_table = 6;
+  // frames with Function.function_name fully matching the following
+  // regexp will be dropped from the samples, along with their successors.
+  int64 drop_frames = 7;   // Index into string table.
+  // frames with Function.function_name fully matching the following
+  // regexp will be kept, even if it matches drop_frames.
+  int64 keep_frames = 8;  // Index into string table.
+
+  // The following fields are informational, do not affect
+  // interpretation of results.
+
+  // Time of collection (UTC) represented as nanoseconds past the epoch.
+  int64 time_nanos = 9;
+  // Duration of the profile, if a duration makes sense.
+  int64 duration_nanos = 10;
+  // The kind of events between sampled occurrences.
+  // e.g [ "cpu","cycles" ] or [ "heap","bytes" ]
+  ValueType period_type = 11;
+  // The number of events between sampled occurrences.
+  int64 period = 12;
+  // Free-form text associated with the profile. The text is displayed as is
+  // to the user by the tools that read profiles (e.g. by pprof). This field
+  // should not be used to store any machine-readable information, it is only
+  // for human-friendly content. The profile must stay functional if this field
+  // is cleaned.
+  repeated int64 comment = 13; // Indices into string table.
+  // Index into the string table of the type of the preferred sample
+  // value. If unset, clients should default to the last sample value.
+  int64 default_sample_type = 14;
+  // Documentation link for this profile type. The URL must be absolute,
+  // e.g., http://pprof.example.com/cpu-profile.html
+  //
+  // The URL may be missing if the profile was generated by code that did not
+  // supply a link.
+  int64 doc_url = 15;  // Index into string table.
+}
+
+// ValueType describes the semantics and measurement units of a value.
+message ValueType {
+  int64 type = 1; // Index into string table.
+  int64 unit = 2; // Index into string table.
+}
+
+// Each Sample records values encountered in some program
+// context. The program context is typically a stack trace, perhaps
+// augmented with auxiliary information like the thread-id, some
+// indicator of a higher level request being handled etc.
+message Sample {
+  // The ids recorded here correspond to a Profile.location.id.
+  // The leaf is at location_id[0].
+  repeated uint64 location_id = 1;
+  // The type and unit of each value is defined by the corresponding
+  // entry in Profile.sample_type. All samples must have the same
+  // number of values, the same as the length of Profile.sample_type.
+  // When aggregating multiple samples into a single sample, the
+  // result has a list of values that is the element-wise sum of the
+  // lists of the originals.
+  repeated int64 value = 2;
+  // label includes additional context for this sample. It can include
+  // things like a thread id, allocation size, etc.
+  //
+  // NOTE: While possible, having multiple values for the same label key is
+  // strongly discouraged and should never be used. Most tools (e.g. pprof) do
+  // not have good (or any) support for multi-value labels. And an even more
+  // discouraged case is having a string label and a numeric label of the same
+  // name on a sample.  Again, possible to express, but should not be used.
+  repeated Label label = 3;
+}
+
+message Label {
+  // Index into string table. An annotation for a sample (e.g.
+  // "allocation_size") with an associated value.
+  // Keys with "pprof::" prefix are reserved for internal use by pprof.
+  int64 key = 1;
+
+  // At most one of the following must be present
+  int64 str = 2;   // Index into string table
+  int64 num = 3;
+
+  // Should only be present when num is present.
+  // Specifies the units of num.
+  // Use arbitrary string (for example, "requests") as a custom count unit.
+  // If no unit is specified, consumer may apply heuristic to deduce the unit.
+  // Consumers may also  interpret units like "bytes" and "kilobytes" as memory
+  // units and units like "seconds" and "nanoseconds" as time units,
+  // and apply appropriate unit conversions to these.
+  int64 num_unit = 4;  // Index into string table
+}
+
+message Mapping {
+  // Unique nonzero id for the mapping.
+  uint64 id = 1;
+  // Address at which the binary (or DLL) is loaded into memory.
+  uint64 memory_start = 2;
+  // The limit of the address range occupied by this mapping.
+  uint64 memory_limit = 3;
+  // Offset in the binary that corresponds to the first mapped address.
+  uint64 file_offset = 4;
+  // The object this entry is loaded from.  This can be a filename on
+  // disk for the main binary and shared libraries, or virtual
+  // abstractions like "[vdso]".
+  int64 filename = 5;  // Index into string table
+  // A string that uniquely identifies a particular program version
+  // with high probability. E.g., for binaries generated by GNU tools,
+  // it could be the contents of the .note.gnu.build-id field.
+  int64 build_id = 6;  // Index into string table
+
+  // The following fields indicate the resolution of symbolic info.
+  bool has_functions = 7;
+  bool has_filenames = 8;
+  bool has_line_numbers = 9;
+  bool has_inline_frames = 10;
+}
+
+// Describes function and line table debug information.
+message Location {
+  // Unique nonzero id for the location.  A profile could use
+  // instruction addresses or any integer sequence as ids.
+  uint64 id = 1;
+  // The id of the corresponding profile.Mapping for this location.
+  // It can be unset if the mapping is unknown or not applicable for
+  // this profile type.
+  uint64 mapping_id = 2;
+  // The instruction address for this location, if available.  It
+  // should be within [Mapping.memory_start...Mapping.memory_limit]
+  // for the corresponding mapping. A non-leaf address may be in the
+  // middle of a call instruction. It is up to display tools to find
+  // the beginning of the instruction if necessary.
+  uint64 address = 3;
+  // Multiple line indicates this location has inlined functions,
+  // where the last entry represents the caller into which the
+  // preceding entries were inlined.
+  //
+  // E.g., if memcpy() is inlined into printf:
+  //    line[0].function_name == "memcpy"
+  //    line[1].function_name == "printf"
+  repeated Line line = 4;
+  // Provides an indication that multiple symbols map to this location's
+  // address, for example due to identical code folding by the linker. In that
+  // case the line information above represents one of the multiple
+  // symbols. This field must be recomputed when the symbolization state of the
+  // profile changes.
+  bool is_folded = 5;
+}
+
+message Line {
+  // The id of the corresponding profile.Function for this line.
+  uint64 function_id = 1;
+  // Line number in source code.
+  int64 line = 2;
+  // Column number in source code.
+  int64 column = 3;
+}
+
+message Function {
+  // Unique nonzero id for the function.
+  uint64 id = 1;
+  // Name of the function, in human-readable form if available.
+  int64 name = 2; // Index into string table
+  // Name of the function, as identified by the system.
+  // For instance, it can be a C++ mangled name.
+  int64 system_name = 3; // Index into string table
+  // Source file containing the function.
+  int64 filename = 4; // Index into string table
+  // Line number in source file.
+  int64 start_line = 5;
+}