endoscope 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 90f6bdfe035c6da330ff7889c78313cdea6f654c
+  data.tar.gz: 866e177d694c6165eac866c813eeb962d3930151
+SHA512:
+  metadata.gz: df4505b45883aa0583255228f1f21676531a8f5e4b8b41ecc0c115e5a0c81dad1e75fe5aadcda9451afe326a23a12e22ad880538caaade657c29ae2714adc9c5
+  data.tar.gz: c074d13f39ad2a9fdf5a0873449072ca081ad2474c6e985b9fe8c4dfea829ddfc3f082610b98eab6c313062d024b14316b03f1e52ce5453fdf1a687243044276

data/.gitignore

@@ -0,0 +1,23 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log
+vendor/bundle/

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,7 @@
+language: ruby
+services:
+  - redis-server
+rvm:
+  - ruby-head
+  - 2.1.2
+  - 1.9.3

data/Gemfile

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in endoscope.gemspec
+gemspec
+
+gem "ripl"
+gem "ripl-ripper"

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Mathieu Ravaux
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,115 @@
+# Endoscope
+
+[![Code Climate](https://codeclimate.com/github/preplay/endoscope.png)](https://codeclimate.com/github/preplay/endoscope)
+[![Build Status](https://travis-ci.org/preplay/endoscope.svg?branch=master)](https://travis-ci.org/preplay/endoscope)
+
+Remote shell for live interaction with Ruby processes
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'endoscope'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install endoscope
+
+## Usage
+
+You need to start the endoscope agent when your application boots up, naming this process.
+
+For example, in a Rails app this could go in `config/initializers/endoscope.rb`
+
+```ruby
+require "endoscope"
+
+process_name = if dyno = ENV['DYNO'] || ENV['PS']
+  Process.pid == 2 ? dyno : "#{dyno}-child-#{Process.pid}"
+else
+  progname = $PROGRAM_NAME.gsub(Rails.root.to_s + '/', '')
+  "#{progname}:#{Process.pid}"
+end
+
+# using ENV['ENDOSCOPE_REDIS_URL'] and ENV['ENDOSCOPE_REDIS_NAMESPACE']
+Endoscope::Agent.new(process_name).start
+```
+
+You can also use a redis connection configuration as expected by the redis gem.
+
+You can then use the Endoscope shell to interact with your app.
+For example, on a running Heroku app comprising of web, sidekiq and
+sidekiq-scheduler dynos:
+
+```
+bundle exec endoscope
+>> 40 + 2
+40 + 2
+Sending command 40 + 2...
+
+From web.1-child-6 :
+42
+
+From web.2-child-9 :
+42
+
+From web.1-child-9 :
+42
+
+From worker.1 :
+42
+
+From web.1-child-13 :
+42
+
+From web.2-child-6 :
+42
+
+From web.2-child-13 :
+42
+
+From scheduler.1 :
+42
+
+```
+
+You can direct your commands at select process types or processes via
+the `use command`
+
+```
+>> use web
+Now adressing commands to processes listening for "web".
+>> use worker.1
+Now adressing commands to processes listening for "worker.1".
+>> use all
+Now adressing commands to processes listening for "all".
+```
+
+## Common uses
+* retrieving garbage collection stats
+* Taking thread dumps
+* Re-open classes, redefining method to test changes / add logging without needing to commit and re-deploy
+* adjust logging verbosity
+* toggle features while debugging to narrow down the possible error causes
+
+
+## Notes
+
+* The endoscope communication happens securely over Redis PubSub.
+* The commands are evaluated in the top level binding of the instrumented program.
+* stdout and stderr outputs are captured while the command is evaluated.
+* Commands evaluation exceptions are caught and reported via the endoscope.
+* Commands evaluation is protected by an execution timeout of 10 seconds.
+* The agent starts a ruby thread with no overhead while not evaluating remote commands.
+
+## Contributing
+
+1. Fork it ( https://github.com/preplay/endoscope/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,7 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec
+

data/bin/endoscope

@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+
+require_relative "../lib/endoscope/cli"
+
+if ARGV.count > 2
+  abort "endoscope [redis_url] [namespace] (or use ENDOSCOPE_REDIS_URL and/or ENDOSCOPE_REDIS_NAMESPACE ENV variables)"
+end
+
+redis_url, namespace = *ARGV
+redis_url ||= ENV['ENDOSCOPE_REDIS_URL']
+namespace ||= ENV['ENDOSCOPE_REDIS_NAMESPACE']
+Endoscope::CLI.new({url: redis_url, namespace: namespace}).start

data/bin/patient

@@ -0,0 +1,36 @@
+#!/usr/bin/env ruby
+
+require_relative "../lib/endoscope"
+
+Endoscope::Agent.new("patient.1", {url: "redis://127.0.0.1:6379"}).start
+
+module Patient
+  extend self
+
+  def live
+    say_hello
+	  loop do
+      think
+      talk
+      sleep
+	  end
+  end
+
+  def say_hello
+    puts "Hello from a sample instrumented process"
+  end
+
+  def think
+    1000.times { 42 * 42 }
+  end
+
+  def talk
+    print "."
+  end
+
+  def sleep
+    Kernel.sleep 1
+  end
+end
+
+Patient.live

data/endoscope.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'endoscope/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "endoscope"
+  spec.version       = Endoscope::VERSION
+  spec.authors       = ["Mathieu Ravaux"]
+  spec.email         = ["mathieu.ravaux@gmail.com"]
+  spec.summary       = "Remote shell for live interaction with Ruby processes"
+  spec.description   = ""
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "redis"
+
+  spec.add_development_dependency "bundler", "~> 1.6"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+end

data/lib/endoscope.rb

@@ -0,0 +1,2 @@
+require_relative "endoscope/version"
+require_relative "endoscope/agent"

data/lib/endoscope/agent.rb

@@ -0,0 +1,93 @@
+# TODO:
+#  * allow configuring the transport
+#  * allow configuring the namesapce
+
+require_relative "transport"
+
+require "timeout"
+require "stringio"
+
+module Endoscope
+  class Agent
+    ENDOSCOPE = "endoscope".freeze
+
+    attr_reader :dyno_name, :redis_options
+
+    def initialize(dyno_name, redis_options)
+      @dyno_name = dyno_name
+      @redis_options = redis_options || default_redis_options
+    end
+
+    def default_redis_options
+      {
+        url: ENV['ENDOSCOPE_REDIS_URL'] || 'redis://127.0.0.1:6379/',
+        namespace: ENV['ENDOSCOPE_REDIS_NAMESPACE']
+      }
+    end
+
+    def start
+      Thread.new(&method(:agent_listener))
+    end
+
+    def agent_listener
+      Thread.current[:name] = ENDOSCOPE
+      begin
+        wait_for_commands
+      rescue => e
+        puts e.inspect
+        puts e.backtrace.join("\n")
+      end
+    end
+
+    def wait_for_commands
+      transport = Transport.new(redis_options)
+      transport.wait_for_commands(dyno_name) do |command|
+        command_received(command)
+      end
+    rescue Transport::ConnectionError => error
+      puts "ns=endoscope at=wait_for_commands error=#{error} reconnect_in=1s"
+      sleep 1
+      retry
+    end
+
+    def command_received(command)
+      puts "ns=endoscope at=command_received"
+      to_eval = command.fetch('command')
+      result = evaluate(to_eval)
+      Transport.new(redis_options).publish_response(command, dyno_name, result)
+    end
+
+    EvalTimeout = Class.new(Timeout::Error)
+
+    def evaluate(ruby)
+      capture_streams do |out|
+        begin
+          Timeout.timeout(10, EvalTimeout) do
+            # rubocop:disable Eval
+            res = eval(ruby, TOPLEVEL_BINDING, 'remote_command')
+            # rubocop:enable Eval
+            out.puts res.inspect
+          end
+        rescue Exception => e
+          out.puts(e.inspect, *e.backtrace)
+        end
+      end
+    end
+
+    def capture_streams
+      $old_stdout = $stdout
+      $old_stderr = $stderr
+
+      out = StringIO.new
+      $stdout = out
+      $stderr = out
+      yield(out)
+
+      out.rewind
+      out.read
+    ensure
+      $stdout = $old_stdout
+      $stderr = $old_stderr
+    end
+  end
+end

data/lib/endoscope/cli.rb

@@ -0,0 +1,114 @@
+require_relative "../endoscope"
+require_relative "transport"
+
+require "set"
+require 'securerandom'
+
+module Endoscope
+  class CLI
+    attr_accessor :dyno_selector, :issued, :transport_opts
+
+    def initialize(transport_opts=nil)
+      @transport_opts = transport_opts
+    end
+
+    def start(dyno_selector = 'all')
+      @dyno_selector = dyno_selector
+      @issued = Set.new
+      start_responses_printing_thread
+      transport
+      start_shell
+    end
+
+    def start_shell
+      begin
+        require "ripl"
+        require "ripl/ripper"
+      rescue LoadError => err
+        puts err.message
+        abort("\nYou need to run:\n\ngem install ripl ripl-ripper\n\nThen launch this command again.")
+      end
+
+      cli = self
+      Ripl::Shell.send(:define_method, :loop_eval) { |str| cli.eval_(str) }
+      Ripl::Shell.send(:define_method, :print_result) { |_result| }
+      Ripl.start( argv: [], irbrc: false, riplrc: false, ripper_prompt: ' | ')
+    end
+
+    def start_responses_printing_thread
+      @responses_thread = Thread.new(&method(:responses_printer))
+    end
+
+    def responses_printer
+      Thread.current[:name] = 'endoscope-responses-printing'
+      listen_to_command_responses
+    end
+
+    def listen_to_command_responses
+      transport = Transport.new(transport_opts)
+      transport.listen_to_responses do |response|
+        handle_response(response)
+      end
+    rescue Redis::TimeoutError => _
+      retry
+    rescue => err
+      puts err.inspect
+      puts err.backtrace.join("\n")
+    end
+
+    def handle_response(res)
+      #p res
+      #p issued
+      #p res['id']
+      return unless issued.include?(res['id'])
+      puts "From #{res['dyno_name']} :\n#{res['result']}\n\n"
+      $stdout.flush
+    end
+
+    def repl
+      catch(:break) do
+        puts "\n\n ---\nRemote console ready:\n\n"
+        $stdout.flush
+        loop { re($stdout) }
+      end
+    end
+
+    def re(out = $stdout)
+      command = read
+      eval_(command, out)
+    rescue Interrupt
+      throw(:break)
+    end
+
+    def eval_(command, out = $stdout)
+      case command
+      when 'exit'
+        throw(:break)
+      when /^use /
+        @dyno_selector = command.gsub('use ', '').strip
+        puts "Now adressing commands to processes listening for #{dyno_selector.inspect}."
+      else
+        send_command(command, out) unless command.nil? || command.strip == ''
+      end
+    end
+
+    def read
+      read = gets
+      read && read.chomp
+    end
+
+    def transport
+      @transport ||= begin
+        Transport.new(transport_opts)
+      end
+    end
+
+    def send_command(command, _out = $stdout)
+      puts "Sending command #{command}..."
+      command_id = SecureRandom.uuid
+      issued << command_id
+
+      transport.send_command(command_id, command, dyno_selector)
+    end
+  end
+end

data/lib/endoscope/transport.rb

@@ -0,0 +1,80 @@
+require_relative "../endoscope"
+
+require "redis"
+require "json"
+
+module Endoscope
+  class Transport
+    ConnectionError = Class.new(RuntimeError)
+
+    attr_reader :namespace, :redis_opts
+    def initialize(opts)
+      @namespace = opts.delete(:namespace) || "endoscope"
+      @redis_opts = opts
+    end
+
+    def wait_for_commands(dyno_name)
+      channels = command_channels(dyno_name)
+      connection.subscribe(*channels) do |on|
+        on.message do |_channel, message|
+          # puts "##{channel}: #{message}"
+          command = JSON.parse(message)
+          yield(command)
+        end
+      end
+    rescue Redis::BaseConnectionError => error
+      raise ConnectionError, error.message, error
+    end
+
+    def send_command(command_id, command, dyno_selector)
+      channel = requests_channel(dyno_selector)
+      connection.publish(channel, JSON.generate(
+        id: command_id,
+        command: command,
+        channel: channel
+      ))
+    end
+
+    def publish_response(command, dyno_name, result)
+      connection.publish(responses_channel, JSON.generate(
+        id: command.fetch('id'),
+        command: command.fetch('command'),
+        dyno_name: dyno_name,
+        result: result
+      ))
+    end
+
+    def listen_to_responses
+      connection.subscribe(responses_channel) do |on|
+        on.message do |_channel_name, message|
+          response = JSON.parse(message)
+          yield(response)
+        end
+      end
+
+    end
+
+    private
+
+    def connection
+      @connection ||= Redis.connect(redis_opts)
+    end
+
+    ALL = "all".freeze
+    def command_channels(dyno)
+      type = dyno.split('.', 2).first
+      [requests_channel(type), requests_channel(dyno), requests_channel(ALL)]
+    end
+
+    def requests_channel(selector)
+      "#{namespace}:requests:#{selector}"
+    end
+
+    def responses_channel
+      "#{namespace}:responses"
+    end
+
+  end
+end
+
+

data/lib/endoscope/version.rb

@@ -0,0 +1,3 @@
+module Endoscope
+  VERSION = "0.0.1"
+end

data/spec/endoscope_spec.rb

@@ -0,0 +1,39 @@
+require 'spec_helper'
+
+require 'open3'
+
+describe Endoscope do
+  it 'has a version number' do
+    expect(Endoscope::VERSION).not_to be nil
+  end
+
+  context "with a patient process running the Endoscope agent" do
+    let(:bin) { File.expand_path("../../bin", __FILE__) }
+
+    before do
+      @patient = IO.popen(File.join(bin, 'patient'))
+    end
+
+    after do
+      Process.kill "TERM", @patient.pid
+    end
+
+    it 'allows an endoscope process to evaluate ruby code inside of a patient process and show the result' do
+      q = Queue.new
+
+      Open3.popen2(File.join(bin, 'endoscope')) do |endo_in, endo_out, _endo_wait|
+        Thread.new { endo_out.each_line { |l| q.push l.chomp } }
+
+        endo_in.puts "$0"
+
+        Timeout.timeout(5) do
+          expect(q.pop).to eql ">> $0" # user input
+          expect(q.pop).to eql ">> Sending command $0..."  # command confirmation
+          expect(q.pop).to eql "From patient.1 :"          # response banner
+          expect(q.pop).to include "endoscope/bin/patient" # response contents
+        end
+      end
+    end
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,2 @@
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+require 'endoscope'

metadata

@@ -0,0 +1,122 @@
+--- !ruby/object:Gem::Specification
+name: endoscope
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Mathieu Ravaux
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-06-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: ''
+email:
+- mathieu.ravaux@gmail.com
+executables:
+- endoscope
+- patient
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/endoscope
+- bin/patient
+- endoscope.gemspec
+- lib/endoscope.rb
+- lib/endoscope/agent.rb
+- lib/endoscope/cli.rb
+- lib/endoscope/transport.rb
+- lib/endoscope/version.rb
+- spec/endoscope_spec.rb
+- spec/spec_helper.rb
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Remote shell for live interaction with Ruby processes
+test_files:
+- spec/endoscope_spec.rb
+- spec/spec_helper.rb
+has_rdoc: