catadog 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 1316fc9c71fc1509af44246443b82ce5961031fb0d551c02b58f4b8cea6b8fb1
+  data.tar.gz: e220233c7970aed319142ab7266a6fc266862869d707e755ded7207265d7c90b
+SHA512:
+  metadata.gz: e740a5ffaf11ef2942fab450498c8ab40014c3a221df08914e0b765b9f628f83c343c4135e3f21401201bc99f345a9332c42111e3826118183ce4e6b1a5059ee
+  data.tar.gz: e4d4190457cef07377df71f971bf0d7d66cb225620beea2bbea6ce65918a74a87e7df59be5d684e35b6721120f3c90aabcd615373487dada6e0013164fb7272f

data/bin/catadog

@@ -0,0 +1,465 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "logger"
+require "ipaddr"
+require "webrick"
+require "rack"
+require "sinatra/base"
+require "net/http"
+require "uri"
+require "json"
+require "msgpack"
+require "base64"
+require "date"
+require "pathname"
+require "stringio"
+
+class WEBrick::HTTPServlet::ProcHandler
+  # rubocop:disable Style/Alias
+  alias do_PATCH do_GET
+  alias do_PUT do_GET
+  alias do_DELETE do_GET
+  # rubocop:enable Style/Alias
+end
+
+module Datadog
+  module Catadog
+    class App < Sinatra::Base
+      get "/" do
+        [200, {"Content-Type" => "text/plain"}, ["app: GET /\n"]]
+      end
+
+      post "/" do
+        [200, {"Content-Type" => "text/plain"}, ["app: POST /\n"]]
+      end
+    end
+
+    class Intercept
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        r = Rack::Request.new(env)
+
+        case r.get_header("HTTP_CONTENT_TYPE")
+        when "application/json"
+          req_d = JSON.parse(r.body.read.tap { r.body.rewind })
+        when "application/msgpack"
+          req_d = MessagePack.unpack(r.body.read.tap { r.body.rewind })
+        end
+
+        status, headers, body = @app.call(env)
+
+        # TODO: these content-type things should at best be fallbacks
+        case headers["Content-Type"]
+        when "application/json"
+          res_d = JSON.parse(body.first)
+        when "application/msgpack"
+          res_d = MessagePack.unpack(body.first)
+        else
+          case r.fullpath
+          when "/info"
+            kind = "info"
+            res_d = JSON.parse(body.first)
+          when "/v0.7/config"
+            kind = "rc"
+            res_d = JSON.parse(body.first).tap { |e| e.delete("roots") }
+            if res_d.key?("targets")
+              res_d["targets"] = Base64.strict_decode64(res_d["targets"])
+              res_d["targets"] = JSON.parse(res_d["targets"])
+            end
+            if res_d.key?("target_files")
+              res_d["target_files"].each do |f|
+                if f["raw"]
+                  f["raw"] = Base64.strict_decode64(f["raw"])
+                  begin
+                    f["parsed"] = JSON.parse(f["raw"])
+                    f.delete("raw")
+                  rescue JSON::ParseError
+                    nil
+                  end
+                end
+              end
+            end
+          when "/v0.3/traces", "/v0.4/traces", "/v0.7/traces"
+            kind = "traces"
+            res_d = JSON.parse(body.first)
+          when %r{^/telemetry/proxy/api/v2/}
+            kind = "telemetry"
+            res_d = JSON.parse(body.first)
+          end
+        end
+
+        d = {
+          kind: kind,
+          request: {
+            method: r.request_method,
+            path: r.fullpath,
+            headers: r.each_header.with_object({}) { |(k, v), h| k =~ /HTTP_(.*)/ && h[$1.tr("_", "-").downcase] = v },
+            body: req_d
+          },
+          response: {
+            status: status,
+            headers: headers,
+            body: res_d
+          }
+        }
+
+        # used for bubbling up to output middlewares
+        env["catadog.intercept"] = d
+
+        [status, headers, body]
+      end
+    end
+
+    class Print
+      def initialize(app, out: $stdout)
+        @app = app
+        @out = out
+      end
+
+      def call(env)
+        @app.call(env)
+      ensure
+        # TODO: consider exception case as well
+
+        if (d = env["catadog.intercept"])
+          @out.write(JSON.pretty_generate(d) << "\n")
+        end
+      end
+    end
+
+    # Dump from `Intercept`, one file per request
+    class Record
+      def initialize(app, dir:)
+        @app = app
+        @ts = Time.now
+        @dir = Pathname.new(dir) unless dir == :auto
+        @counter = 0
+      end
+
+      def call(env)
+        @app.call(env)
+      ensure
+        # TODO: consider exception case as well
+
+        if (d = env["catadog.intercept"])
+          write(JSON.pretty_generate(d))
+        end
+
+        @counter += 1
+      end
+
+      private
+
+      def dir
+        @dir ||= Pathname.new("records") / @ts.iso8601
+      end
+
+      def filename
+        dir / format("%05d.json", @counter)
+      end
+
+      def write(str)
+        dir.mkpath
+        File.open(filename, "wb") { |f| f << str }
+      end
+    end
+
+    class Proxy
+      def initialize(host, port)
+        @host = host
+        @port = port
+        @uuid = SecureRandom.uuid
+      end
+
+      def call(env)
+        r = Rack::Request.new(env)
+
+        catadog = if (h = r.get_header("HTTP_X_CATADOG"))
+          h.split(",")
+        else
+          []
+        end
+
+        raise "catadog loop detected!" if catadog.include?(@uuid)
+
+        host = @host
+        port = @port
+        uri = URI.join("http://#{host}:#{port}", r.fullpath)
+
+        Net::HTTP.start(uri.host, uri.port) do |http|
+          case env["REQUEST_METHOD"]
+          when "HEAD"
+            req = Net::HTTP::Head.new(uri)
+          when "GET"
+            req = Net::HTTP::Get.new(uri)
+          when "POST"
+            req = Net::HTTP::Post.new(uri)
+            req.body = env["rack.input"].read
+          when "PUT"
+            req = Net::HTTP::Put.new(uri)
+            req.body = env["rack.input"].read
+          when "PATCH"
+            req = Net::HTTP::Patch.new(uri)
+            req.body = env["rack.input"].read
+          when "DELETE"
+            req = Net::HTTP::Delete.new(uri)
+            req.body = env["rack.input"].read
+          end
+
+          r.each_header do |k, v|
+            if k == "HTTP_HOST"
+              req["HTTP_HOST"] = "#{host}:#{port}"
+            elsif k =~ /HTTP_(.*)/
+              header_name = $1.tr("_", "-").downcase
+              req[header_name] = v
+            end
+          end
+
+          req["x-catadog"] = (catadog << @uuid).join(",")
+
+          res = http.request(req)
+
+          status = Integer(res.code)
+          headers = res.each_header.with_object({}) { |(k, v), h| h[k] = v }
+          body = res.body
+
+          return [status, headers, [body]]
+        end
+      end
+    end
+
+    class NotFound
+      def call(env)
+        [404, {"Content-Type" => "application/json"}, [JSON.dump({})]]
+      end
+    end
+
+    class Server
+      def initialize(settings, logger:)
+        @logger = logger # for Rack
+
+        @server = WEBrick::HTTPServer.new(**options(settings, logger: logger))
+
+        @app = rack_app(settings)
+
+        @server.mount_proc("/", method(:handler))
+      end
+
+      def start
+        trap "INT" do
+          @server.shutdown
+        end
+
+        @server.start
+      end
+
+      private
+
+      def rack_app(settings)
+        Rack::Builder.new do
+          map "/catadog" do
+            run App.new
+          end
+
+          use Record, dir: settings.record_dir if settings.record_dir
+          use Print
+          use Intercept
+
+          settings.mocks.each do |mock|
+            mock_file, mock_classname = mock.split(":", 2)
+
+            unless mock_file.nil? || mock_file.empty?
+              mock_file = Pathname.new(mock_file)
+              mock_classname = mock_file.basename(".rb").to_s.gsub(/(^|_)\S/) { |m| m.tr("_", "").upcase } if mock_classname.nil?
+              require Pathname.pwd / mock_file
+            end
+
+            mock_class = mock_classname.split("::").reduce(Kernel) { |mod, name| mod.const_get(name) }
+
+            use mock_class
+          end
+
+          run settings.forward ? Proxy.new(settings.agent_host, settings.agent_port) : NotFound.new
+        end.to_app
+      end
+
+      def handler(req, res)
+        # https://github.com/rack/rack/blob/8f5c885f7e0427b489174a55e6d88463173f22d2/SPEC.rdoc
+
+        env = {}
+
+        env["REQUEST_METHOD"] = req.request_method
+        env["SCRIPT_NAME"] = req.script_name
+        env["PATH_INFO"] = req.path_info
+        env["QUERY_STRING"] = req.query_string || ""
+        env["SERVER_NAME"] = req.request_uri.host
+        env["SERVER_PORT"] = req.request_uri.port
+        env["SERVER_PROTOCOL"] = (/(HTTP\/\d(?:\.\d)?)/ =~ req.request_line && $1.chomp)
+        env["rack.url_scheme"] = req.request_uri.scheme
+
+        # TODO: #body_reader + IO.copy_stream
+        # TODO: body { |chunk| }
+        if req.body
+          input_stream = StringIO.new(req.body.dup.force_encoding("ASCII-8BIT"), "rb")
+          env["rack.input"] = input_stream
+        else
+          env["rack.input"] = StringIO.new
+        end
+
+        error_stream = StringIO.new(+"", "w")
+        env["rack.errors"] = error_stream
+
+        env["rack.hijack?"] = false
+
+        session = {}
+        env["rack.session"] = session
+
+        env["rack.logger"] = @logger if @logger
+
+        # env["rack.multipart.buffer_size"] = 0,
+
+        if req.header.key?("content-length")
+          env["CONTENT_LENGTH"] = Integer(req.header["content-length"].last)
+        end
+
+        # https://datatracker.ietf.org/doc/html/rfc3875#section-4.1.18
+        req.header.each { |k, v| env["HTTP_#{k.tr("-", "_").upcase}"] = v.last }
+
+        status, headers, body = @app.call(env)
+
+        # TODO: do something with rack.errors?
+
+        res.status = status
+        headers.each { |k, v| res[k] = v }
+
+        # TODO: handle callable and IO
+        res.body = body.join("")
+      end
+
+      def options(settings, logger:)
+        {
+          Logger: logger,
+          AccessLog: [[logger.instance_variable_get(:@logdev).dev, WEBrick::AccessLog::COMBINED_LOG_FORMAT]],
+          BindAddress: settings.host.to_s,
+          Port: settings.port,
+          ServerType: settings.daemon ? WEBrick::Daemon : WEBrick::SimpleServer
+          # RequestCallback: request_callback
+        }
+      end
+    end
+
+    class Settings
+      attr_accessor \
+        :debug,
+        :verbosity,
+        :host,
+        :port,
+        :forward,
+        :agent_host,
+        :agent_port,
+        :record_dir,
+        :silent,
+        :log,
+        :mocks,
+        :daemon
+
+      def initialize
+        @debug = false
+        @verbosity = 0
+        @host = IPAddr.new("127.0.0.1")
+        @port = 8128
+        @forward = true
+        @agent_host = IPAddr.new("127.0.0.1")
+        @agent_port = 8126
+        @record_dir = nil
+        @silent = false
+        @daemon = false
+        @log = $stderr
+        @mocks = []
+      end
+
+      def to_h
+        instance_variables.each_with_object({}) { |k, h| h[k] = instance_variable_get(k) }
+      end
+
+      def to_s
+        to_h.to_s
+      end
+    end
+
+    # CLI interface
+    class CLI
+      class UsageError < StandardError; end
+
+      def initialize(args)
+        settings = Settings.new
+
+        while (arg = args.shift)
+          case arg
+          when "-d", "--debug"
+            settings.debug = true
+          when "--daemon"
+            settings.daemon = true
+          when "-s", "--silent"
+            settings.silent = true
+          when "-v", "--verbose"
+            settings.verbosity += 1
+          when "-l", "--log"
+            settings.log = Pathname.new(args.shift).open("wb")
+          when "-h", "--bind"
+            settings.host = IPAddr.new(args.shift)
+          when "-p", "--port"
+            settings.port = Integer(args.shift)
+          when "-F", "--no-forward"
+            settings.forward = false
+          when "-f", "--agent-host"
+            settings.agent_host = args.shift
+          when "-g", "--agent-port"
+            settings.agent_port = Integer(args.shift)
+          when "-r", "--record"
+            settings.record_dir = (args.empty? || args.first.start_with?("-")) ? :auto : Pathname.new(args.shift)
+          when "-m", "--mock"
+            settings.mocks << String(args.shift)
+          else
+            raise UsageError, "invalid argument: #{arg}"
+          end
+        end
+
+        @settings = settings.freeze
+      end
+
+      def run
+        load_gem_mocks
+
+        logger = Logger.new(@settings.silent ? Pathname.new("/dev/null").open("wb") : @settings.log, level: log_level)
+        logger.debug { "settings: #{@settings}" }
+
+        server = Server.new(@settings, logger: logger)
+
+        server.start
+      ensure
+        @settings.log.close
+      end
+
+      def load_gem_mocks
+        gem_mocks_path.glob("*.rb").each { |f| require f.to_s }
+      end
+
+      def gem_mocks_path
+        Pathname.new(__dir__) / ".." / "mocks"
+      end
+
+      private
+
+      def log_level
+        @settings.debug ? Logger::DEBUG : Logger::INFO
+      end
+    end
+  end
+end
+
+Datadog::Catadog::CLI.new(ARGV).run

data/mocks/sink.rb

@@ -0,0 +1,46 @@
+class Sink
+  def initialize(app)
+    @app = app
+  end
+
+  def call(env)
+    r = Rack::Request.new(env)
+
+    case [r.request_method, r.path]
+    in "GET", "/info" then info
+    in "POST", %r{^/v(0\.3|0\.4|0\.7)/traces$} then traces
+    in "POST", %r{^/telemetry/proxy/} then telemetry
+    else
+      @app.call(env)
+    end
+  end
+
+  private
+
+  def traces
+    payload = {}
+
+    [200, {"Content-Type" => "application/json"}, [JSON.dump(payload)]]
+  end
+
+  def telemetry
+    payload = {}
+
+    [200, {"Content-Type" => "application/json"}, [JSON.dump(payload)]]
+  end
+
+  def info
+    payload = {
+      "catadog" => true,
+      "version" => "7.54",
+      "endpoints" => [
+        "/v0.3/traces",
+        "/v0.4/traces",
+        "/v0.7/traces",
+        "/telemetry/proxy"
+      ]
+    }
+
+    [200, {"Content-Type" => "application/json"}, [JSON.dump(payload)]]
+  end
+end

metadata

@@ -0,0 +1,119 @@
+--- !ruby/object:Gem::Specification
+name: catadog
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Datadog, Inc.
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2024-10-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: webrick
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.8.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.8.0
+- !ruby/object:Gem::Dependency
+  name: rack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+- !ruby/object:Gem::Dependency
+  name: sinatra
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: msgpack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Intercept and analyse Datadog communication
+email: dev@datadoghq.com
+executables:
+- catadog
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/catadog
+- mocks/sink.rb
+homepage: https://github.com/DataDog/catadog
+licenses:
+- BSD-3-Clause
+- Apache-2.0
+metadata:
+  rubygems_mfa_required: 'true'
+  allowed_push_host: https://rubygems.org
+  source_code_uri: https://github.com/DataDog/catadog
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.4.19
+signing_key: 
+specification_version: 4
+summary: Datadog wire introspection tool
+test_files: []