codebeacon-tracer 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: cc0306f363cf17c02cf4e0de4c37df04d52b22f7b992b047767a47e243341a9a
+  data.tar.gz: ba785f9723112678105d34030f1ea3f9960630d6701a3f14041e5d82d554c6c6
+SHA512:
+  metadata.gz: c59c1bd83110e7e4214988ca69fa6f08d9533c096b98e4e555c28d2700ef35d05e3c1fd154804d156bc9b022656dbf3b3a6c17e2592df1cc0f67abe73f3aef15
+  data.tar.gz: 7d2ae9552ea537767cb5ef93409565243674a75b65f3b9810ca5f3c894845743aa775def5dd519c4ac99687b057ab157e7b4e2e5b57ea234afb7aaaa84f9c96c

data/lib/codebeacon/tracer/src/configuration.rb

@@ -0,0 +1,199 @@
+require 'fileutils'
+require 'yaml'
+
+module Codebeacon
+  module Tracer
+    class Configuration
+      MAX_DB_FILES = 100
+      RETURN_VAL_MAX_LENGTH = 1000
+      MAX_CALL_COUNT = 100000000
+      MAX_DEPTH = 99999
+
+      def initialize()
+        @query = ""
+        @exclude_paths = []
+        ensure_db_path
+      end
+
+      def setup
+        exclude_paths << lib_root
+        reload_paths_to_record
+        load_main_config
+      end
+
+      def load_main_config
+        if File.exist?(config_path)
+          config_data = YAML.load_file(config_path)
+          load_exclude_paths(config_data['exclude'])
+        end
+      end
+
+      def load_exclude_paths(excludes)
+        return if excludes.nil?
+        (excludes['paths'] || []).each { |path| exclude_paths << path }
+        (excludes['gems'] || []).each do |gem_name|
+          Gem::Specification.find_all_by_name(gem_name).each do |gem_spec|
+            exclude_paths << gem_spec.gem_dir
+          end
+        end
+      end
+
+      def set_query_config(query)
+        @query = query || ""
+        # self.trace_enabled = @query.include?('rf__trace_enabled=true')
+        self.debug = @query.include?('rf__debug=true')
+        self.dry_run = @query.include?('rf__dry_run=true')
+        # self.local_methods_only = @query.include?('rf__local_methods_only=true')
+        # self.local_lines_only = @query.include?('rf__local_lines_only=true')
+      end
+
+      def ensure_db_path
+        FileUtils.mkpath(db_path)
+      end
+
+      # def load_ruby_flow_config
+      #   if File.exist?('.code-beacon.yml')
+      #     @config = YAML.load_file('.code-beacon.yml')
+      #   end
+      # end
+
+      def lib_root
+        File.expand_path(File.join(File.dirname(__FILE__), '..', '..', '..'))
+      end
+
+      def data_dir
+        ".code-beacon"
+      end
+
+      def db_path
+       File.join(data_dir, "db")
+      end
+
+      def tmp_dir
+        File.join(data_dir, "tmp")
+      end
+
+      def refresh_path
+        File.join(data_dir, "tmp", "refresh")
+      end
+
+      def paths_path
+        File.join(data_dir, "paths.yml")
+      end
+
+      def config_path
+        File.expand_path(File.join(lib_root, 'config.yml')) 
+      end
+
+      def read_paths
+        if File.exist?(paths_path)
+          YAML.load_file(paths_path)
+        end
+      end
+
+      def db_name
+        "codebeacon_tracer"
+      end
+
+      def max_db_files
+        MAX_DB_FILES
+      end
+
+      def gem_path
+        @gem_path ||= ENV['GEM_HOME'] || Gem.paths.home
+      end
+
+      def root_path
+        @root_path ||= defined?(Rails) ? Rails.root.to_s : Dir.pwd
+      end
+
+      def rubylib_path
+        @rubylib_path ||= RbConfig::CONFIG['rubylibdir']
+      end
+
+      def paths_to_record
+        @paths_to_record ||= [Codebeacon::Tracer.config.root_path, *Codebeacon::Tracer.config.read_paths]
+      end
+
+      def reload_paths_to_record
+        @paths_to_record = nil
+        paths_to_record
+      end
+
+      def exclude_paths(*paths)
+        @exclude_paths += paths
+      end
+
+      def trace_enabled?
+        # @trace_enabled
+        true
+      end
+
+      def trace_enabled=(value)
+        @trace_enabled = value
+      end
+
+      def debug?
+        @debug
+      end
+
+      def debug=(value)
+        Codebeacon::Tracer.logger.level = value ? ::Logger::DEBUG : ::Logger::INFO
+        @debug = value
+      end
+
+      def dry_run?
+        @dry_run
+      end
+
+      def dry_run=(value)
+        @dry_run = value
+      end
+
+      def local_methods_only?
+        # @local_methods_only
+        true
+      end
+
+      def local_methods_only=(value)
+        @local_methods_only = value
+      end
+
+      def local_lines_only?
+        # @local_lines_only
+        true
+      end
+
+      def local_lines_only=(value)
+        @local_lines_only = value
+      end
+      
+      def skip_internal?
+        true
+      end
+
+      def max_value_length
+        RETURN_VAL_MAX_LENGTH
+      end
+
+      def max_call_count
+        MAX_CALL_COUNT
+      end
+
+      def max_depth
+        MAX_DEPTH
+      end
+
+      def logger
+        @logger ||= Codebeacon::Tracer::Logger.new()
+      end
+
+      def debug_something
+        #### debug return
+        # @return_count += 1
+        # Codebeacon::Tracer.logger.info("Call count: #{@return_count}")
+        # Codebeacon::Tracer.logger.info("Return @depth: #{@depth}, method: #{tp.method_id}, line: #{tp.lineno}, path: #{tp.path}")
+      end
+    end
+  end
+end

data/lib/codebeacon/tracer/src/data/database.rb

@@ -0,0 +1,54 @@
+require 'sqlite3'
+require 'fileutils'
+require_relative 'tree_node_mapper'
+require_relative 'node_source_mapper'
+
+module Codebeacon
+  module Tracer
+    class DatabaseSchema
+      def initialize
+        @db = initialize_db
+      end
+
+      def db
+        @db
+      end
+
+      def initialize_db
+        db_path = Codebeacon::Tracer.config.db_path
+        timestamp = Time.now.strftime("%Y%m%d%H%M%S")
+        db_name = "#{Codebeacon::Tracer.config.db_name}_#{timestamp}.db"
+        db_symlink = File.join(db_path, "#{Codebeacon::Tracer.config.db_name}.db")
+
+        File.delete(db_symlink) if File.exist?(db_symlink)
+        FileUtils.ln_sf(db_name, db_symlink)
+        SQLite3::Database.new(File.join(db_path, db_name))
+      end
+
+      def create_tables
+        MetadataMapper.create_table(db)
+        TreeNodeMapper.create_table(db)
+        NodeSourceMapper.create_table(db)
+      end
+
+      def create_indexes
+        MetadataMapper.create_indexes(db)
+        TreeNodeMapper.create_indexes(db)
+        NodeSourceMapper.create_indexes(db)
+      end
+
+      def self.trim_db_files
+        db_path = Codebeacon::Tracer.config.db_path
+        db_files = Dir.glob(File.join(db_path, "*.db"))
+        db_files.reject! { |file| File.symlink?(file) }
+        db_files.sort_by! { |db_file| File.mtime(db_file) }
+        db_files.reverse!
+
+        db_files.each_with_index do |db_file, index|
+          next if index < Codebeacon::Tracer.config.max_db_files
+          File.delete(db_file)
+        end
+      end
+    end
+  end
+end

data/lib/codebeacon/tracer/src/data/metadata_mapper.rb

@@ -0,0 +1,26 @@
+module Codebeacon
+  module Tracer
+    class MetadataMapper
+      def initialize(database)
+        @db = database
+      end
+
+      def self.create_table(database)
+        database.execute <<-SQL
+          CREATE TABLE IF NOT EXISTS metadata (
+            id INTEGER PRIMARY KEY,
+            name TEXT,
+            description TEXT
+          );
+        SQL
+      end
+
+      def self.create_indexes(database)
+      end
+
+      def insert(name, description)
+        @db.execute("INSERT INTO metadata (name, description) VALUES (?, ?)", [name, description])
+      end
+    end
+  end
+end

data/lib/codebeacon/tracer/src/data/node_source_mapper.rb

@@ -0,0 +1,29 @@
+require_relative '../models/node_source'
+
+module Codebeacon
+  module Tracer
+    class NodeSourceMapper
+      def initialize(database)
+        @db = database
+      end
+
+      def self.create_table(database)
+        database.execute <<-SQL
+          CREATE TABLE IF NOT EXISTS node_sources (
+            id INTEGER PRIMARY KEY,
+            name TEXT NOT NULL,
+            root_path TEXT NOT NULL
+          );
+        SQL
+      end
+
+      def self.create_indexes(database)
+      end
+
+      def insert(name, root_path)
+        @db.execute("INSERT INTO node_sources (name, root_path) VALUES (?, CAST(? AS TEXT))", [name, root_path])
+        @db.last_insert_row_id
+      end
+    end
+  end
+end

data/lib/codebeacon/tracer/src/data/persistence_manager.rb

@@ -0,0 +1,107 @@
+require_relative 'tree_node_mapper'
+require_relative 'node_source_mapper'
+require_relative 'metadata_mapper'
+
+module Codebeacon
+  module Tracer
+    class PersistenceManager
+
+      def self.marshal(name, value, tree_node)
+        begin
+          return value.inspect[0..Codebeacon::Tracer.config.max_value_length]
+        rescue => e
+          begin
+            if Codebeacon::Tracer.config.debug?
+              Codebeacon::Tracer.logger.warn "Marshal inspect failure - attempting to_s fallback for: \"#{name}\", located at: \"#{tree_node.file}:#{tree_node.line}\"\nerror message: \"#{e.message}\", error_location: \"#{e.backtrace[0]}\""
+            end
+            return value.to_s[0..Codebeacon::Tracer.config.max_value_length]
+          rescue => e
+            Codebeacon::Tracer.logger.error "Marshal failure for: \"#{name}\", located at: \"#{tree_node.file}:#{tree_node.line}\"\nerror message: \"#{e.message}\", error_location: \"#{e.backtrace[0]}\""
+            return "--Codebeacon::Tracer ERROR-- could not marshall value. See logs."
+          end
+        end
+      end
+
+      def initialize(database)
+        @database = database
+        @tree_node_mapper = TreeNodeMapper.new(database)
+        @node_source_mapper = NodeSourceMapper.new(database)
+        @metadata_mapper = MetadataMapper.new(database)
+        @progress_logger = Codebeacon::Tracer.logger.newProgressLogger("nodes persisted")
+      end
+
+      def save_metadata(name, description)
+        @metadata_mapper.insert(name, description)
+      end
+
+      def save_node_sources(node_sources)
+        node_sources.each do |node_source|
+          next if node_source.nil?
+          node_source.id = @node_source_mapper.insert(node_source.name, node_source.root_path)
+        end
+      end
+
+      def save_trees(trees)
+        Codebeacon::Tracer.logger.info("BEGIN db persistence")
+        begin
+          trees.each do |tree|
+            save_tree(tree.root)
+          end
+        rescue => e
+          Codebeacon::Tracer.logger.error("Error during tree persistence: #{e.message}")
+          Codebeacon::Tracer.logger.error(e.backtrace.join("\n")) if Codebeacon::Tracer.config.debug?
+          # Continue execution without crashing the application
+        ensure
+          @progress_logger.finish()
+          Codebeacon::Tracer.logger.info("END db persistence")
+        end
+      end
+
+      def save_tree(tree_node, parent_id = nil)
+        _save_tree(tree_node, parent_id)
+      end
+
+      def _save_tree(tree_node, parent_id = nil)
+        @progress_logger.increment
+        return if tree_node.nil?
+
+        begin
+          node_id = @tree_node_mapper.insert(
+            tree_node.file,
+            tree_node.line,
+            tree_node.method.to_s,
+            tree_node.tp_class.to_s,
+            tree_node.tp_defined_class.to_s,
+            tree_node.tp_class_name.to_s,
+            tree_node.self_type.to_s,
+            tree_node.depth,
+            tree_node.caller,
+            tree_node.gem_entry,
+            parent_id,
+            tree_node.block,
+            tree_node.node_source&.id,
+            _return_value(tree_node)
+          )
+
+          unless tree_node.depth_truncated?
+            tree_node.children.each do |child|
+              _save_tree(child, node_id)
+            end
+          end
+        rescue => e
+          Codebeacon::Tracer.logger.error("Error saving tree node: #{e.message}")
+          Codebeacon::Tracer.logger.error("Node details: file=#{tree_node.file}, line=#{tree_node.line}, method=#{tree_node.method}") if Codebeacon::Tracer.config.debug?
+          # Continue with siblings and other nodes without crashing
+        end
+      end
+
+      def _return_value(node)
+        if node.method == :initialize
+          return nil
+        else
+          PersistenceManager.marshal(node.method, node.return_value, node)
+        end
+      end
+    end
+  end
+end

data/lib/codebeacon/tracer/src/data/tree_node_mapper.rb

@@ -0,0 +1,60 @@
+require 'sqlite3'
+require 'json'
+
+module Codebeacon
+  module Tracer
+    class TreeNodeMapper
+      def initialize(database)
+        @db = database
+      end
+
+      def insert(file, line, method, tp_class, tp_defined_class, tp_class_name, self_type, depth, caller, gem_entry, parent_id, block, node_source_id, return_value)
+        @db.execute(<<-SQL, 
+          INSERT INTO treenodes 
+          (
+              file, line, method, tp_class, tp_defined_class, tp_class_name, self_type, depth, caller, 
+              gemEntry, parent_id, block, node_source_id, return_value
+          )
+          VALUES 
+          (
+              ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
+          )
+        SQL
+        file, line, method, tp_class, tp_defined_class, tp_class_name, self_type, depth, caller, 
+        gem_entry ? 1 : 0, parent_id, block ? 1 : 0, node_source_id, return_value)
+        
+        @db.last_insert_row_id
+      end
+
+      def self.create_table(database)
+        database.execute <<-SQL
+          CREATE TABLE IF NOT EXISTS treenodes (
+            id INTEGER PRIMARY KEY,
+            file TEXT,
+            line INTEGER,
+            method TEXT,
+            tp_class TEXT,
+            tp_defined_class TEXT,
+            tp_class_name TEXT,
+            self_type TEXT,
+            depth INTEGER,
+            caller TEXT,
+            gemEntry INTEGER,
+            parent_id INTEGER,
+            block INTEGER,
+            node_source_id INTEGER,
+            return_value TEXT,
+            FOREIGN KEY (parent_id) REFERENCES treenodes(id),
+            FOREIGN KEY (node_source_id) REFERENCES node_sources(id)
+          )
+        SQL
+      end
+
+      def self.create_indexes(database)
+        database.execute("CREATE INDEX IF NOT EXISTS IDX_treenode_parent_id ON treenodes(parent_id)")
+        database.execute("CREATE INDEX IF NOT EXISTS IDX_treenode_node_source_id ON treenodes(node_source_id)")
+        database.execute("CREATE INDEX IF NOT EXISTS IDX_treenode_file ON treenodes(file)")
+      end
+    end
+  end
+end

data/lib/codebeacon/tracer/src/logger.rb

@@ -0,0 +1,58 @@
+require 'logger'
+
+module Codebeacon
+  module Tracer
+    class Logger
+      FILENAME = "codebeacon_tracer.log"
+      attr_reader :logger
+
+      def initialize(level = nil)
+        level ||= Codebeacon::Tracer.config.debug? ? ::Logger::DEBUG : ::Logger::INFO
+        @logger ||= ::Logger.new(File.join(Codebeacon::Tracer.config.data_dir, FILENAME), 3, 104857600, level: level)
+      end
+
+      def newProgressLogger(*args)
+        ProgressLogger.new(@logger, *args)
+      end
+
+      def debug(message, *args, &block)
+        return unless Codebeacon::Tracer.config.debug?
+        @logger.debug(message, *args, &block)
+      end
+
+      def method_missing(method, *args, &block)
+        if @logger.respond_to?(method)
+          @logger.send(method, *args, &block)
+        else
+          super
+        end
+      end
+
+      def respond_to_missing?(method, include_private = false)
+        @logger.respond_to?(method, include_private) || super
+      end
+    end
+
+    class ProgressLogger
+      PROGRESS_LOG_INTERVAL = 1000
+
+      def initialize(logger, msg, interval = PROGRESS_LOG_INTERVAL)
+        @logger = logger
+        @msg = msg
+        @interval = interval
+        @count = 0
+      end
+
+      def increment()
+        @count += 1
+        if @count % @interval == 0
+          @logger.info(@count.to_s + " " + @msg)
+        end
+      end
+
+      def finish()
+        @logger.info("Finished: " + @count.to_s + " " + @msg)
+      end
+    end
+  end
+end

data/lib/codebeacon/tracer/src/models/call_tree.rb

@@ -0,0 +1,54 @@
+module Codebeacon
+  module Tracer
+    class CallTree
+      @thread_id_mutex = Mutex.new
+      @thread_id = 0
+      
+      attr_reader :thread, :root, :current_node, :depth, :call_count, :block_call_count
+
+      def self.next_thread_id
+        @thread_id_mutex.synchronize do
+          @thread_id += 1
+        end
+      end
+
+      def initialize(thread)
+        @thread = thread
+        root_name = (thread.name || "thread") + " (#{CallTree.next_thread_id})"
+        @root = TreeNode.new(method: root_name)
+        @root.file, @root.line = __FILE__, __LINE__
+        @current_node = @root
+        @depth = 0
+        @call_count = 0
+        @block_call_count = 0
+      end
+
+      def total_call_count
+        @call_count + @block_call_count
+      end
+
+      def add_call()
+        @call_count += 1
+        add_node()
+      end
+
+      def add_block_call()
+        @block_call_count += 1
+        add_node()
+      end
+
+      def add_node()
+        new_node = TreeNode.new()
+        @current_node.children << new_node
+        new_node.parent = @current_node
+        @depth += 1
+        @current_node = new_node
+      end
+
+      def add_return()
+        @depth -= 1
+        @current_node = @current_node.parent if @current_node
+      end
+    end
+  end
+end

data/lib/codebeacon/tracer/src/models/node_builder.rb

@@ -0,0 +1,60 @@
+module Codebeacon
+  module Tracer
+    class NodeBuilder
+      class << self
+        def backtrace_location_eql(loc1, loc2)
+          loc1.absolute_path == loc2.absolute_path && loc1.lineno == loc2.lineno && loc1.label == loc2.label
+        end
+
+        def trace_method_call(call_tree, tp, tp_caller)
+          call_tree.add_call
+          trace_call(call_tree, tp, tp_caller, :get_method_ast)
+        end
+
+        def trace_block_call(call_tree, tp, tp_caller)
+          current_context = call_tree.add_block_call
+          current_context.block = true
+          trace_call(call_tree, tp, tp_caller, :get_block_ast)
+        end
+
+        def trace_return(call_tree, tp)
+          begin
+            current_context = call_tree.current_node
+            variable_values = {}
+            current_context.return_value = "--Codebeacon::Tracer ERROR-- could not capture return value"
+            previous_line = current_context.trace_status.previous_line
+            current_context.return_value = tp.return_value
+          ensure
+            call_tree.add_return()
+          end
+        end
+
+        private def trace_call(call_tree, tp, tp_caller, ast_get_method)
+          current_context = call_tree.current_node
+
+          current_context.file = File.absolute_path(tp.path)
+          current_context.node_source = NodeSource.find(tp.path)
+          current_context.line = tp.lineno
+          current_context.object_id = tp.self.object_id
+          current_context.method = tp.method_id
+          klass = TPKlass.new(tp)
+
+          current_context.tp_class = klass.tp_class.to_s
+          current_context.tp_defined_class = klass.defined_class
+          current_context.tp_class_name = klass.tp_class_name
+          current_context.self_type = klass.type
+          current_context.depth = call_tree.depth
+
+          gem_entry = false
+          if Codebeacon::Tracer.config.gem_path \
+            && !Codebeacon::Tracer.config.gem_path.empty? \
+            && tp.path.start_with?(Codebeacon::Tracer.config.gem_path) # && caller[1].start_with?(Codebeacon::Tracer.config.root_path)
+            gem_entry = true
+          end
+          current_context.gem_entry = gem_entry
+          current_context.caller = ""
+        end
+      end
+    end
+  end
+end

data/lib/codebeacon/tracer/src/models/node_source.rb

@@ -0,0 +1,32 @@
+require 'pathname'
+
+module Codebeacon
+  module Tracer
+    class NodeSource
+      attr_accessor :id, :name, :root_path
+      @instances = []
+
+      class << self
+        attr_accessor :instances
+      end
+
+      def initialize(name, root_path)
+        @name = name
+        @root_path = root_path
+        self.class.instances << self
+      end
+
+      def self.find(path)
+        return nil if path.nil?
+        Pathname.new(path).ascend do |dir|
+          source = self.instances.find { |ns| dir.to_s == ns.root_path }
+          return source if source
+        end
+      end
+
+      def self.clear
+        self.instances = []
+      end
+    end
+  end
+end