trailblazer-developer 0.0.28 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2bc71f425eee589bda27a8b681b796650443e688c3dc46ddb7d12e6daf092f08
-  data.tar.gz: 2f128bcf1675ba3147db4737f348b52d9a1f96562e317feeddda79846c1ca0e8
+  metadata.gz: 343ca726bbb8da50f05aca2a552f018ac67683d4c0021226f828a30132ecf85c
+  data.tar.gz: f4ab036e3da016ab3283f258c749ef98cee474a9c69b8303fad36bdd9b6df945
 SHA512:
-  metadata.gz: 3b10040b636e96bc396ec89b41db298030552b0460ad7fa88b94ebf5a767e881cdf2c219ab573522f4226cb84d6e6d5baac786d384b7ef93d2788576dea4183f
-  data.tar.gz: 2eb0f50c4af74ab44ea0353e61b45a4645adb0f70132be92d329dd8123d0fd20561b58047be1bd9b2a25cb9e6bd5ccd2a6ac67089c7546e0041c78c24c455c62
+  metadata.gz: f66fa880eb4a0673fd881598fe9fa9addebadbd4989fded9366f7960b7ed9ab34794fe8d8a6c4ab3a315b13fe45b048061ac4f4330cb71942a82a3adc1d12a82
+  data.tar.gz: 7ecac119ec1d406442e1ddb5292639bac41e13df24d870f462065d90d5bf690ab4417b0736c9a134b51c2d2929a6952912803cfe2686cfca7d7b16bc14cfaa4a

data/CHANGES.md

@@ -1,3 +1,78 @@
+# 0.1.0
+
+* Improvement of tracing performance: factor ~4.
+* Consistent interface for tracing and presentation.
+
+## Adding `Debugger` layer
+
+* Introduce `Debugger::Trace` with `Trace::Node`s and `variable_versions` field
+  to maintain all data produced by tracing in one entity.
+* In `Trace::Present.call`, you need to pass a block with options for customization instead
+  of passing `:node_options`. Further on, the per-node customization is now keyed by
+  `Trace::Node` instance and not a Stack element anymore.
+* In `render_method`, remove `:tree` keyword argument, use `:debugger_trace`.
+* The `render_method` is called with keyword arguments, only. The first positional argument
+  `debugger_nodes` is now `:debugger_trace`.
+
+  ```ruby
+  output = Dev::Trace::Present.(
+    stack,
+    node_options: {
+      stack.to_a[0] => {label: "Create"}
+    }
+  )
+  ```
+  is now
+
+  ```ruby
+   Dev::Trace::Present.(stack) do |trace_nodes:, **|
+    {
+      node_options: {
+        trace_nodes[0] => {label: "Create"}
+      }
+    }
+  end
+  ```
+
+## `Trace::Node`
+
+* `Node.build_for_stack` is now called in `Trace::Present` and produces a list
+  of `Trace::Node` instances that are containers for  matching before and after snapshots.
+* Add `Node::Incomplete` nodes that have no `snapshot_after` as they represent
+  a part of the flow that was canceled.
+
+## Debugger::Normalizer
+
+* Deprecate `:captured_node` keyword argument in normalizer steps and
+  rename it to `:trace_node`.
+* Remove the concept of `:runtime_path` and `:compile_path`. You can always
+  compute paths after or during the debug rendering yourself. However, we simply
+  don't need it for IDE or wtf?.
+
+## `Trace.wtf?`
+
+* Rename `:output_data_collector` and `:input_data_collector` to `:after_snapshooter` and `:before_snapshooter`
+  as they both produce instances of `Snapshot`. The return signature has changed, the snapshooters
+  return two values now: the data object and an object representing the variable versioning.
+
+## Snapshot
+
+* Introduce the concept of `Snapshot` which can be a moment in time before or after a step.
+  The `Snapshot::Before` and `Snapshot::After` (renamed from `Captured::Input` and `Captured::Output`)
+  are created during the tracing and after tracing associated to a specific `Trace::Node`.
+* Add `Snapshot::Ctx` and `Snapshot::Versions` which is a new, faster way of capturing variables.
+  Instead of always calling `ctx.inspect` for every trace step, only variables that have changed
+  are snapshooted using the (configurable) `:value_snapshooter`. This improves `#wtf?` performance
+  up to factor 10.
+
+# 0.0.29
+
+* The `:render_method` callable can now return output along with additional returned values.
+* Pass the top-level `:activity` into the `:render_method`.
+* In `#wtf?`, only complete the stack when an exception occurred.
+* `wtf?` now returns all original return values plus the computed output from `:render_method`,
+  followed by an arbitrary object from the rendering code.
+
 # 0.0.28
 
 * Move `Introspect::Graph` over from `trailblazer-activity`. It's a data structure very specific

data/Gemfile

@@ -11,3 +11,5 @@ gemspec
 # gem "trailblazer-activity",           github: "trailblazer/trailblazer-activity"
 # gem "trailblazer-activity-dsl-linear", github: "trailblazer/trailblazer-activity-dsl-linear"
 
+
+gem "benchmark-ips"

data/lib/trailblazer/developer/debugger/normalizer.rb

@@ -0,0 +1,60 @@
+module Trailblazer
+  module Developer
+    module Debugger
+      # @private
+      # Public entry point to add Debugger::Node normalizer steps.
+      def self.add_normalizer_step!(step, id:, normalizer: Normalizer::PIPELINES.last, **options)
+        task = Normalizer.Task(step)
+
+        # We have a TaskWrap::Pipeline (a very simple style of "activity" used for normalizers) and
+        # add another step using the "friendly interface" from {Activity::Adds}.
+        options = {append: nil} unless options.any?
+
+        pipeline_extension = Activity::TaskWrap::Extension.build([task, id: id, **options])
+
+        Normalizer::PIPELINES << pipeline_extension.(normalizer)
+      end
+
+      # Run at runtime when preparing a Trace::Nodes for presentation.
+      module Normalizer
+        def self.Task(user_step) # TODO: we could keep this in the {activity} gem.
+          Activity::TaskWrap::Pipeline::TaskAdapter.for_step(user_step, option: false) # we don't need Option as we don't have ciruit_options here, and no {:exec_context}
+        end
+
+        # Default steps for the Debugger::Node options pipeline, following the step-interface.
+        module Default
+          def self.compile_id(ctx, activity:, task:, **)
+            ctx[:compile_id] = Activity::Introspect.Nodes(activity, task: task)[:id]
+          end
+
+          def self.runtime_id(ctx, compile_id:, **)
+            ctx[:runtime_id] = compile_id
+          end
+
+          def self.label(ctx, label: nil, runtime_id:, **)
+            ctx[:label] = label || runtime_id
+          end
+
+          def self.data(ctx, data: {}, **)
+            ctx[:data] = data
+          end
+
+          def self.incomplete?(ctx, trace_node:, **)
+            ctx[:incomplete?] = trace_node.is_a?(Developer::Trace::Node::Incomplete)
+          end
+        end
+
+        default_steps = {
+          compile_id:       Normalizer.Task(Default.method(:compile_id)),
+          runtime_id:       Normalizer.Task(Default.method(:runtime_id)),
+          label:            Normalizer.Task(Default.method(:label)),
+          data:             Normalizer.Task(Default.method(:data)),
+          incomplete?:      Normalizer.Task(Default.method(:incomplete?)),
+        }.
+        collect { |id, task| Activity::TaskWrap::Pipeline.Row(id, task) }
+
+        PIPELINES = [Activity::TaskWrap::Pipeline.new(default_steps)] # we do mutate this constant at compile-time. Maybe # DISCUSS and find a better way.
+      end
+    end
+  end
+end

data/lib/trailblazer/developer/debugger.rb

@@ -0,0 +1,97 @@
+module Trailblazer
+  module Developer
+    # Code in Debugger is only executed if the user wants to render the stack.
+    module Debugger
+      ATTRS = [
+        :id,
+        :trace_node,
+        :task,
+        :activity,
+        :compile_id,
+        :runtime_id,
+        :label,
+        :data,
+        :snapshot_before,
+        :snapshot_after,
+        :level,
+        :incomplete?,
+        :captured_node, # TODO: remove once macro is 2.2
+      ]
+
+      # The {Debugger::Node} is an abstraction between Trace::Node and the actual rendering layer (why?)
+      #
+      # TODO: class, "type",
+      # which track, return signal, etc
+      class Node < Struct.new(*ATTRS, keyword_init: true)
+        # we always key options for specific nodes by Stack::Captured::Input, so we don't confuse activities if they were called multiple times.
+        #
+        # @return [Debugger::Node] array of Debugger::Node
+        def self.build(trace_nodes, node_options: {}, normalizer: Debugger::Normalizer::PIPELINES.last, **options_for_nodes)
+          # DISCUSS: this might change if we introduce a new Node type for Trace.
+          _debugger_nodes = trace_nodes.collect do |trace_node|
+            # it's possible to pass per-node options, like {label: "Yo!"} via {:node_options[<snapshot_before>]}
+            options_from_user  = node_options[trace_node] || {}
+
+            options_from_trace_node = trace_node
+              .to_h # :level, :snapshot_before, :snapshot_after
+              .merge(
+                id:           trace_node.object_id,
+                trace_node:   trace_node,
+                activity:     trace_node.snapshot_before.activity,
+                task:         trace_node.task,
+                captured_node: DeprecatedCapturedNode, # TODO: remove once macro is 2.2
+              )
+
+            options_for_debugger_node, _ = normalizer.(
+              {
+                **options_from_trace_node,
+                **options_from_user
+              },
+              []
+            )
+
+            # these attributes are not changing with the presentation
+            Debugger::Node.new(**options_for_debugger_node).freeze
+          end
+        end
+
+        # TODO: remove once macro is 2.2
+        class DeprecatedCapturedNode
+          def self.method_missing(*)
+            raise "[Trailblazer] The `:captured_node` argument is deprecated, please upgrade to `trailblazer-developer-0.1.0` and use `:trace_node` if the upgrade doesn't fix it."
+          end
+        end
+      end # Node
+
+      # Interface for data (nodes, versions, etc) between tracing code and presentation layer.
+      # We have no concept of {Stack} here anymore. Nodes and arbitrary objects such as "versions".
+      # Debugger::Trace interface abstracts away the fact we have two snapshots. Here,
+      # we only have a node per task.
+      #
+      class Trace
+        # Called in {Trace::Present}.
+        # Design note: goal here is to have as little computation as possible.
+        def self.build(stack, trace_nodes, **options_for_debugger_nodes)
+          nodes = Debugger::Node.build(
+            trace_nodes,
+            **options_for_debugger_nodes,
+          )
+
+          new(nodes: nodes, variable_versions: stack.variable_versions) # after this, the concept of "Stack" doesn't exist anymore.
+        end
+
+        def initialize(nodes:, variable_versions:)
+          @options = {nodes: nodes, variable_versions: variable_versions}
+        end
+
+        def to_h
+          @options
+        end
+
+        def to_a
+          to_h[:nodes].to_a
+        end
+      end
+    end # Debugger
+  end
+end

data/lib/trailblazer/developer/trace/node.rb

@@ -0,0 +1,103 @@
+module Trailblazer
+  module Developer
+    module Trace
+      # Build array of {Trace::Node} from a snapshots stack.
+      # @private
+      def self.build_nodes(snapshots)
+        instructions = [
+          [0, snapshots]
+        ]
+
+        _nodes = Node.process_instructions(instructions)
+      end
+
+      # Datastructure representing a trace.
+      class Node < Struct.new(:level, :task, :snapshot_before, :snapshot_after)
+        class Incomplete < Node
+        end
+
+        def self.pop_from_instructions!(instructions)
+          while (level, remaining_snapshots = instructions.pop)
+            next if level.nil?
+            next if remaining_snapshots.empty?
+
+            return level, remaining_snapshots
+          end
+
+          false
+        end
+
+        # def self.BLA(instructions)
+        #   instructions.collect do |(level, remaining_snapshots)|
+        #     [
+        #       level,
+        #       remaining_snapshots.collect { |snap| [snap.class, snap.task] }
+        #     ]
+        #   end
+        # end
+
+        def self.process_instructions(instructions) # FIXME: mutating argument
+          nodes = []
+
+          while (level, remaining_snapshots = pop_from_instructions!(instructions))
+            raise unless remaining_snapshots[0].is_a?(Snapshot::Before) # DISCUSS: remove assertion?
+
+            node, new_instructions = node_and_instructions_for(remaining_snapshots[0], remaining_snapshots[1..-1], level: level)
+            # pp BLA(new_instructions)
+
+            nodes << node
+
+            instructions += new_instructions
+          end
+
+          return nodes
+        end
+
+        # Called per snapshot_before   "process_branch"
+        # 1. Find, for snapshot_before, the matching snapshot_after in the stack
+        # 2. Extract snapshots inbetween those two. These are min. 1 level deeper in!
+        # 3. Run process_siblings for 2.
+        def self.node_and_instructions_for(snapshot_before, descendants, level:)
+          # Find closing snapshot for this branch.
+          snapshot_after = descendants.find do |snapshot|
+            snapshot.is_a?(Snapshot::After) && snapshot.data[:snapshot_before] == snapshot_before
+          end
+
+          if snapshot_after
+            snapshot_after_index = descendants.index(snapshot_after)
+
+            instructions =
+              if snapshot_after_index == 0 # E.g. before/Start, after/Start
+                [
+                  [level, descendants[1..-1]]
+                ]
+              else
+                nested_instructions = [
+                  # instruction to go through the remaining, behind this tuple.
+                  [
+                    level,
+                    descendants[(snapshot_after_index + 1)..-1]
+                  ],
+                  # instruction to go through all snapshots between this current tuple.
+                  [
+                    level + 1,
+                    descendants[0..snapshot_after_index - 1], # "new descendants"
+                  ],
+                ]
+              end
+
+            node = new(level, snapshot_before.task, snapshot_before, snapshot_after)
+          else # incomplete
+            instructions = [
+              [level + 1, descendants]
+            ]
+
+            node = Incomplete.new(level, snapshot_before.task, snapshot_before, nil)
+          end
+
+          return node, instructions
+        end
+      end # Node
+    end
+  end # Developer
+end

data/lib/trailblazer/developer/trace/parent_map.rb

@@ -0,0 +1,32 @@
+module Trailblazer
+  module Developer
+    module Trace
+      # Map each {Node} instance to its parent {Node}.
+      module ParentMap # DISCUSS: where does this belong?
+        def self.build(trace_nodes)
+          levels = {}
+          trace_nodes.collect do |node|
+            level = node.level
+            levels[level] = node
+
+            [node, levels[level - 1]]
+          end.to_h
+        end
+
+        # @public
+        def self.path_for(parent_map, node)
+          path = []
+
+          while parent = parent_map[node] # DISCUSS: what if the graphs are cached and present, already?
+            node_id = Activity::Introspect.Nodes(node.snapshot_before.activity, task: node.snapshot_before.task).id
+            path << node_id
+
+            node = parent
+          end
+
+          path.reverse
+        end
+      end # ParentMap
+    end
+  end # Developer
+end

data/lib/trailblazer/developer/trace/present.rb

@@ -2,7 +2,7 @@ require "hirb"
 
 module Trailblazer::Developer
   module Trace
-    module Present
+    module Present # DISCUSS: rename to Debugger?
       module_function
 
       # @private
@@ -12,33 +12,65 @@ module Trailblazer::Developer
 
       # Returns the console output string.
       # @private
-      def render(debugger_nodes, renderer: method(:default_renderer), **options_for_renderer)
-        nodes = debugger_nodes.collect do |debugger_node|
-          renderer.(debugger_node: debugger_node, tree: debugger_nodes, **options_for_renderer)
+      def render(debugger_trace:, renderer: method(:default_renderer), **options_for_renderer)
+        nodes = debugger_trace.to_a.collect do |debugger_node|
+          renderer.(debugger_node: debugger_node, debugger_trace: debugger_trace, **options_for_renderer)
         end
 
         Hirb::Console.format_output(nodes, class: :tree, type: :directory, multi_line_nodes: true)
       end
 
-      # Entry point for rendering a Stack as a "tree branch" the way we do it in {#wtf?}.
-      def call(stack, render_method: method(:render), node_options: {}, **options)
+      # Entry point for rendering a {Trace::Stack}.
+      # Used in `#wtf?`.
+      def call(stack, render_method: method(:render), **options, &block)
+        deprecate_node_options!(**options) # TODO: remove in 0.2.0.
+
+        # Build a generic array of {Trace::Node}s.
+        trace_nodes = Trace.build_nodes(stack.to_a)
+
         # The top activity doesn't have an ID, hence we need to compute a default label.
-        # TODO: maybe we should deep-merge here.
-        captured_input_for_top_activity = stack.to_a[0]
-
-        top_activity_options = {
-          # we can pass particular label "hints".
-          captured_input_for_top_activity => {
-            # label: %{#{captured_input_for_top_activity.task.superclass} (anonymous)},
-            label: captured_input_for_top_activity.task.inspect,
-          },
+        top_activity_trace_node = trace_nodes[0]
+
+        build_options = {
+          node_options: {
+            # we can pass particular label "hints".
+            top_activity_trace_node => {
+              # label: %{#{top_activity_trace_node.task.superclass} (anonymous)},
+              label: top_activity_trace_node.task.inspect,
+            },
+          }
         }
 
-        node_options = top_activity_options.merge(node_options)
+        build_options = build_options.merge(options) # since we only have {:node_options} in {build_options}, we can safely merge here.
+
+        # specific rendering.
+        options_from_block = block_given? ? block.call(trace_nodes: trace_nodes, stack: stack, **build_options) : {}
+
+        build_options = merge_local_options(options_from_block, build_options)
+
+        # currently, we agree on using a Debugger::Node list as the presentation data structure.
+        debugger_trace = Debugger::Trace.build(stack, trace_nodes, **build_options)
 
-        debugger_nodes = Debugger::Node.build_for_stack(stack, node_options: node_options, **options) # currently, we agree on using a Debugger::Node list as the presentation data structure.
+        return render_method.(debugger_trace: debugger_trace, **build_options)
+      end
+
+      def deprecate_node_options!(node_options: nil, **) # TODO: remove in 0.2.0.
+        return unless node_options
+
+        raise "[Trailblazer] The `:node_options` option for `Trace::Present` is deprecated.
+  Please use the block style as described here: https://trailblazer.to/2.1/docs/internals.html#internals-developer-trace-present"
+      end
+
+      # @private
+      def merge_local_options(options, local_options)
+        merged_hash = options.collect do |key, value|
+          [
+            key,
+            value.is_a?(Hash) ? local_options.fetch(key, {}).merge(value) : value # options are winning over local_options[key]
+          ]
+        end.to_h
 
-        return render_method.(debugger_nodes, **options)
+        local_options.merge(merged_hash)
       end
     end # Present
   end

data/lib/trailblazer/developer/trace/snapshot/value.rb

@@ -0,0 +1,39 @@
+module Trailblazer::Developer
+  module Trace
+    class Snapshot
+      # {Value} serializes the variable value using with custom logic, e.g. {value.inspect}.
+      # A series of matchers decide which snapshooter is used.
+      class Value
+        def initialize(matchers)
+          @matchers = matchers
+        end
+
+        # DISCUSS: this could be a compiled pattern matching `case/in` block here.
+        def call(name, value, **options)
+          @matchers.each do |matcher, inspect_method|
+            if matcher.(name, value, **options)
+              return inspect_method.(name, value, **options)
+            end
+          end
+
+          raise "no matcher found for #{name.inspect}" # TODO: this should never happen.
+        end
+
+        def self.default_variable_inspect(name, value, ctx:)
+          value.inspect
+        end
+
+        def self.build
+          new(
+            [
+              [
+                ->(*) { true }, # matches everything
+                method(:default_variable_inspect)
+              ]
+            ]
+          )
+        end
+      end
+    end
+  end
+end

data/lib/trailblazer/developer/trace/snapshot/versions.rb

@@ -0,0 +1,105 @@
+module Trailblazer::Developer
+  module Trace
+    class Snapshot
+      # Snapshot::Ctx keeps an inspected version of each ctx variable.
+      # We figure out if a variable has changed by using `variable.hash` (works
+      # even with deeply nested structures).
+      #
+      # Key idea here is to have minimum work at operation-runtime. Specifics like
+      # figuring out what has changed can be done when using the debugger.
+      #
+      # By keeping "old" versions, we get three benefits.
+      # 1. We only need to call {inspect} once on a traced variable. Especially
+      #    when variables are complex structures or strings, this dramatically speeds
+      #    up tracing, from same-ish to factor 5!
+      # 2. The content sent to our debugger is much smaller which reduces network load
+      #    and storage space.
+      # 3. Presentation becomes simpler as we "know" what variable has changed.
+      #
+      # Possible problems: when {variable.hash} returns the same key even though the
+      #                    data has changed.
+      #
+      # DISCUSS: speed up by checking mutable, only?
+      # DISCUSS: we currently only use this for testing.
+      # DISCUSS: this has knowledge about {Stack} internals.
+      #
+      # This is for the "rendering" layer.
+      # @private
+      def self.snapshot_ctx_for(snapshot, variable_versions)
+        variable_versions = variable_versions.instance_variable_get(:@variables)
+
+        snapshot.data[:ctx_variable_changeset].collect do |name, hash, has_changed|
+          [
+            name,
+            {
+              value:        variable_versions[name][hash],
+              has_changed:  !!has_changed,
+            }
+          ]
+        end.to_h
+      end
+
+      # A table of all ctx variables, their hashes and serialized values.
+      #
+      #   {:current_user=>
+      #     {3298051090906520533=>"#<TraceTest::User:0x000055b2e3424460 @id=1>",
+      #      3764938782671692590=>"#<TraceTest::User:0x000055b2e33e45b8 @id=2>"},
+      #    :params=>
+      #     {2911818769466875657=>"{:name=>\"Q & I\"}",
+      #      2238394858183550663=>"{:name=>\"Q & I\", :song=>{...}}"},
+      #    :seq=>
+      #     {-105020188158523405=>"[]",
+      #      -2281497291400788995=>"[:authenticate]",
+      #      150926802063554866=>"[:authenticate, :authorize]",
+      #      3339595138798116233=>"[:authenticate, :authorize, :model]",
+      #      -3395325862879242711=>
+      #       "[:authenticate, :authorize, :model, :screw_params!]"},
+      #    :model=>{348183403054247453=>"Object"}}
+      class Versions
+        def initialize
+          @variables = {}
+        end
+
+        # DISCUSS: problem with changeset is, we have to go through variables twice.
+        def changeset_for(ctx, value_snapshooter:)
+          new_versions = []
+
+          changeset_for_snapshot = ctx.collect do |name, value|
+            # DISCUSS: do we have to call that explicitly or does Hash#[] do that for us, anyway?
+            value_hash = value.hash # DISCUSS: does this really always change when a deeply nested object changes?
+
+            if (variable_versions = @variables[name]) && variable_versions.key?(value_hash) # TODO: test {variable: nil} value
+              [name, value_hash, nil] # nil means it's an existing reference.
+            else
+              value_snapshot = value_snapshooter.(name, value, ctx: ctx)
+
+              version = [name, value_hash, value_snapshot]
+
+              new_versions << version
+              version
+            end
+          end
+
+          return changeset_for_snapshot, new_versions
+        end
+
+        def add_changes!(new_versions)
+          new_versions.each do |args|
+            add_variable_version!(*args)
+          end
+        end
+
+        # @private
+        def add_variable_version!(name, hash, value)
+          @variables[name] ||= {}
+
+          @variables[name][hash] = value # i hate mutations.
+        end
+
+        def to_h
+          @variables
+        end
+      end
+    end # Snapshot
+  end
+end