rmitm 0.0.3

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    MDdkN2Y1MDM1NzdiMjU3NmY0Nzg1ZDkzOWIwMDZkODI5MTUxMjUzYQ==
+  data.tar.gz: !binary |-
+    ZWMwNmQxM2NhYjMwYjc5M2M0MDVhMTQ4MTczMjI5NGY1ZjI2OTdjMg==
+SHA512:
+  metadata.gz: !binary |-
+    ZTI3N2UxN2E3NzMyM2NkODhkMGMwYjljNGI5NjUxNzBlMzBiZmRlNTRhYTVi
+    MGUxZWUxZDI3N2UxYzRkYzYzYmRmNjMzNjRlNTg2NmNmZjc5Mjg1NDk0MjY0
+    NjU2NTIwYmE1ZWZlNmMwYWMwYzNjNTM0MGNkYmM1MjI2NTlhOTc=
+  data.tar.gz: !binary |-
+    ODBiMjUyNDNhNGRmMDU4MDkyYzNjNWU2YWJiMzgzZTNiZWI5NzFkMjlkNzYx
+    NzU4ZTM5MmU3NTFkMDAxYmI2ZDRkNThhYjk5MjIyN2EwN2NhZTc4YjkxYjBk
+    MzU1NDFjMjA5NGE0MDlkMzIxNDA3NTVhZDhhZGU1MDdiZTU0Njk=

data/bin/readFromFlow

@@ -0,0 +1,85 @@
+#!/usr/bin/env python
+
+from libmproxy import flow
+import json, sys, getopt
+
+def parse_args(argv):
+	try:
+		opts, args = getopt.getopt(argv, "hqs", ["help", "request", "response"])
+	except getopt.GetoptError as err:
+		print "Error {0}".format(err.msg)
+		usage()
+
+	if len(args) == 0:
+		print "FILENAME not specified"
+		usage()
+	elif len(args) > 1:
+		print "Only one FILENAME can be specified"
+		usage()
+	else:
+		global f
+		f = args[0]
+
+	for opt, arg in opts:
+		global request, response
+		if opt in ('-q', '--request'):
+			response = False
+		elif opt in ('-s', '--response'):
+			request = False
+		elif opt in ('-h', '--help'):
+			usage()
+
+	if not (response or request):
+		response = True
+		request = True
+# END parse_args
+
+def usage():
+	print "--"
+	print "readFromFlow - read netlib flow from an mitmdump output file"
+	print ""
+	print "readFromFlow [OPTIONS] FILENAME"
+	print "-h, --help"
+	print "\tdisplay this help and exit"
+	print "-q, --request"
+	print "\tread requests only"
+	print "-s, --response"
+	print "\tread responses only"
+	print ""
+	print "Only one FILENAME can be specified. By default both requests and responses are read"
+	print "--"
+	sys.exit(2)
+# END usage
+
+def readFlowFile():
+	with open(f, "rb") as ff:
+		freader = flow.FlowReader(ff)
+		try:
+			for d in freader.stream():
+				print "START {0}".format(d)
+				print "{"
+				if request:
+					sys.stdout.write("\"request\" : ")
+					dump(d._get_state()['request'])
+				if response:
+					if request:
+						print ","
+					sys.stdout.write("\"response\" : ")
+					dump(d._get_state()['response'])
+				print "\n}"
+				print "END {0}".format(d)
+		except flow.FlowReadError, v:
+			print "FlowReadError reading from file {0}".format(f)
+# END readFlowFile
+
+def dump(content):
+	json.dump(content, sys.stdout, indent=4, ensure_ascii=False)
+# END dump
+
+if __name__ == '__main__':
+	global request, response
+	request = True
+	response = True
+	parse_args(sys.argv[1:])
+	readFlowFile()
+

data/bin/scripts/blacklist.py

@@ -0,0 +1,18 @@
+import sys, getopt, re
+
+def response(context, flow):
+	if path.search(flow.request.path) is not None:
+		flow.response.code = 404
+		flow.response.msg = "Not Found"
+		flow.response.content = "Oooops!!"
+
+def start(context, argv):
+	try:
+		opts, args = getopt.getopt(argv[1:], "p:", [])
+	except getopt.GetoptError as err:
+		print "Error " + err.msg + " " + err.opt
+		sys.exit(2)
+	for opt, arg in opts:
+		global path
+		if opt == '-p':
+			path = re.compile(arg)

data/bin/scripts/map_local.py

@@ -0,0 +1,31 @@
+import sys, getopt, re
+
+# #########
+# p -> escaped regular expression to match against the request path
+# f -> path to filename to replace contents with
+# #########
+
+path = ""
+filename = ""
+
+def response(context, flow):
+	if path.search(flow.request.path) is not None:
+		flow.response.content = contents()
+
+def contents():
+	with open (filename, "r") as myfile:
+		return myfile.read().replace('\n', '')
+
+def start(context, argv):
+	try:
+		opts, args = getopt.getopt(argv[1:], "p:f:", [])
+	except getopt.GetoptError as err:
+		print "Error " + err.msg + " " + err.opt
+		sys.exit(2)
+	for opt, arg in opts:
+		global path
+		global filename
+		if opt == '-p':
+			path = re.compile(arg)
+		elif opt == '-f':
+			filename = arg

data/bin/scripts/replace.py

@@ -0,0 +1,29 @@
+import sys, getopt, re
+
+regexp = ""
+replace = ""
+path = ""
+
+def response(context, flow):
+	# if flow.request.path == path:
+	if path.search(flow.request.path) is not None:
+		flow.response.content = regexp.sub(replace, flow.response.content, 1)
+		flow.response.headers["Content-Length"] =  [str(len(flow.response.content))]
+
+
+def start(context, argv):
+	try:
+		opts, args = getopt.getopt(argv[1:], "p:x:r:", [])
+	except getopt.GetoptError as err:
+		print "Error " + err.msg + " " + err.opt
+		sys.exit(2)
+	for opt, arg in opts:
+		global path
+		global regexp
+		global replace
+		if opt == '-p':
+			path = re.compile(arg)
+		elif opt == '-x':
+			regexp = re.compile(arg)
+		elif opt == '-r':
+			replace = arg

data/bin/scripts/strip_encoding.py

@@ -0,0 +1,2 @@
+def request(context, flow):
+	del flow.request.headers['Accept-Encoding']

data/lib/mitm_flow_array.rb

@@ -0,0 +1,40 @@
+=begin
+    RMITM - provides a Ruby interface to mitmdump
+    Copyright (C) 2014  Marc Bleeze (marcbleeze<at>gmail<dot>com)
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+=end
+
+require_relative './mitmdump_reader'
+require 'jsonpath'
+
+class MitmFlowArray < Array
+
+	def self.from_file(filename)
+		MitmFlowArray.new(MitmdumpReader.new(filename).get_flows_from_file)
+	end
+
+	def values_by_jpath(jpath, first=true)
+		self.map { |f| first ? JsonPath.new(jpath).first(f) : JsonPath.new(jpath).on(f) }
+	end
+
+	def filter(conditions)
+		result = self
+		conditions.each do |jpath, re|
+			result = result.select { |flow| JsonPath.new(jpath).first(flow).encode!('ISO-8859-1', 'UTF-8', :invalid => :replace, :undef => :replace, :replace => "") =~ re }
+		end
+		MitmFlowArray.new(result)
+	end
+
+end

data/lib/mitmdump.rb

@@ -0,0 +1,154 @@
+=begin
+    RMITM - provides a Ruby interface to mitmdump
+    Copyright (C) 2014  Marc Bleeze (marcbleeze<at>gmail<dot>com)
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+=end
+
+require 'fileutils'
+
+class Mitmdump
+
+	attr_reader :scripts, :params
+
+	def initialize(name, &block)
+		@name = name.to_sym
+		@port = 8080
+		@output = 'dumps/mitm.dump'
+		@scripts = []
+		@params = {}
+
+		instance_eval &block
+	end
+### DSL
+	def inherit(name)
+		intersect = @params.keys & proxy(name.to_sym).params.keys
+		intersect.empty? or
+			raise "Duplicate parameters #{intersect} when inheriting proxy '#{name.to_sym}'"
+		@scripts = @scripts | proxy(name.to_sym).scripts
+		@params.merge! proxy(name.to_sym).params
+	end
+
+	def port(port)
+		@port = port
+	end
+
+	def output(filename)
+		@output = filename
+	end
+
+	def blacklist(path)
+		script "#{script_path}blacklist.py", '-p' => path
+	end
+
+	def map_local(path, args={})
+		unless args[:file].nil?
+			script "#{script_path}map_local.py", '-p' => path, '-f' => args[:file]
+		else
+			raise ArgumentError, 'No file name provided for maplocal'
+		end
+	end
+
+	def replace(path, args={})
+		unless args[:swap].nil? | args[:with].nil?
+			script "#{script_path}replace.py", '-p' => path, '-x' => args[:swap], '-r' => args[:with]
+		else
+			raise ArgumentError, "Expecting arguments ':swap' and ':with' for replace"
+		end
+	end
+
+	def strip_encoding
+		script "#{script_path}strip_encoding.py"
+	end
+
+	def script(file, args={})
+		args.each do |k,v|
+			unless (key = v[/%[a-zA-Z_]+/]).nil?
+			 	@params.has_key?(key) or
+			 		raise "Parameter '#{key}' referenced in proxy '#{@name}' but not declared"
+			end
+		end
+		@scripts << [file, args]
+	end
+
+	def param(name)
+		!@params.has_key?("%#{name.to_s}") and @params["%#{name.to_s}"] = '' or
+			raise "Parameter name '#{name}' already declared"
+	end	
+### END DSL
+	def start(args={})
+		check_params(args)
+		if port_available?
+			manage_dumpfile
+			@pid = Process.spawn command
+			Process.detach @pid
+			connection_successful? or
+				raise "Failed to start mitmdump after 10 seconds\nCOMMAND LINE: <#{command}>"
+		else
+			raise "Cannot start mitmdump on port #{@port} - port unavailable"
+		end
+	end
+
+	def stop
+		system("kill -0 #{@pid} >& /dev/null") and (system("kill -9 #{@pid}") or
+			raise "Could not stop proxy '#{@name}' (pid: #{@pid})") if @pid
+	end
+
+	def dumpfile
+		@output
+	end
+
+	private
+
+		def port_available?
+			`nc -z 127.0.0.1 #{@port} >& /dev/null`
+			!$?.success?
+		end
+
+		def command
+			cmd = "mitmdump -q -p #{@port} -w #{@output}"
+			@scripts.each do | name, opts |
+				cmd << " -s \"#{name}"
+				opts.each { |k,v| cmd << " '#{k}' '#{interpolate(v)}'" } if opts
+				cmd << "\""
+			end
+			cmd
+		end
+
+		def connection_successful?(timeout=10)
+			timeout.times { port_available? and sleep 1 or return true }
+			false
+		end
+
+		def manage_dumpfile
+			FileUtils.rm_f @output if File.exists?(@output)
+			d = File.dirname(@output)
+			FileUtils.mkpath(d) unless File.directory?(d)
+		end
+
+		def script_path
+			"#{File.expand_path('../../bin/scripts', __FILE__)}/"
+		end
+
+		def check_params(args={})
+			@params.keys.each do |k|
+				@params[k] = args[k[1..-1]] if args.has_key? k[1..-1] or 
+					raise "Parameter '#{k}' not specified"
+			end
+		end
+
+		def interpolate(str)
+			str.gsub(/%[a-zA-Z_]+/, @params)
+		end
+end

data/lib/mitmdump_reader.rb

@@ -0,0 +1,69 @@
+=begin
+    RMITM - provides a Ruby interface to mitmdump
+    Copyright (C) 2014  Marc Bleeze (marcbleeze<at>gmail<dot>com)
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+=end
+
+require "open3"
+require "json"
+
+class MitmdumpReader
+
+	def initialize(filename)
+		@filename = filename
+	end
+
+	def get_requests_from_file
+		read_from_flow_dump("-q")
+	end
+
+	def get_responses_from_file
+		read_from_flow_dump("-s")
+	end
+
+	private
+
+		def read_from_flow_dump(options="")
+			cmd = "#{File.expand_path('../../bin', __FILE__)}/readFromFlow #{options << ' '}#{@filename}"
+			pp cmd if $DEBUG
+			Open3.popen3(cmd) do |_, o, e, w|
+				@stdout = o.read.encode!('ISO-8859-1', 'UTF-8', :invalid => :replace, :undef => :replace, :replace => "")
+				@stderr = e.read
+				@stderr == "" ? parse : error
+			end
+		end
+
+		def parse
+			json_objects = []
+			flows.each do |a|
+				json_objects << JSON.parse(a)
+				# JSONPath doesn't support symbolized names
+				# json_objects << JSON.parse(a, :symbolize_names => true)
+			end
+			json_objects
+		end
+
+		def error
+			puts "*"*60, "******* Error returned from readFromFlow python script:", "*"*60, @stderr, "*"*60
+			exit 2
+		end
+
+		def flows
+			@stdout.scan(/START <libmproxy.flow.Flow instance at [^{]+(.*?)\nEND <libmproxy.flow.Flow instance at /m).map { |e| e[0] }
+		end
+	
+		alias_method :get_flows_from_file, :read_from_flow_dump
+		public :get_flows_from_file
+end

data/lib/rmitm.rb

@@ -0,0 +1,35 @@
+=begin
+    RMITM - provides a Ruby interface to mitmdump
+    Copyright (C) 2014  Marc Bleeze (marcbleeze<at>gmail<dot>com)
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+=end
+
+require 'mitmdump'
+require 'mitmdump_reader'
+require 'mitm_flow_array'
+
+def load_proxies(glob)
+  Dir.glob(glob).each { |f| load f }
+end
+
+def proxy(name)
+  $proxies[name.to_sym] or
+    raise "Cannot find proxy '#{name}'"
+end
+
+def mitmdump(name, &block)
+  $proxies ||= {}
+  $proxies[name.to_sym] = Mitmdump.new(name, &block)
+end

metadata

@@ -0,0 +1,81 @@
+--- !ruby/object:Gem::Specification
+name: rmitm
+version: !ruby/object:Gem::Version
+  version: 0.0.3
+platform: ruby
+authors:
+- Marc Bleeze
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-07-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: jsonpath
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: ! "  rmitm provides a DSL and useful ruby classes and python scripts
+  for using mitmdump\n  for automated testing.\n"
+email: marcbleeze@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/readFromFlow
+- bin/scripts/blacklist.py
+- bin/scripts/map_local.py
+- bin/scripts/replace.py
+- bin/scripts/strip_encoding.py
+- lib/mitm_flow_array.rb
+- lib/mitmdump.rb
+- lib/mitmdump_reader.rb
+- lib/rmitm.rb
+homepage: https://github.com/marcyb/rmitm
+licenses:
+- GPL-3.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.3.0
+signing_key: 
+specification_version: 4
+summary: rmitm provides a DSL and useful ruby classes and python scripts for mitmdump
+test_files: []