stubby 0.0.1 → 0.0.2

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 James Kassemi, @Pay
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -20,17 +20,16 @@ from your project.
 
 Install the stubby gem:
 
-		> $ sudo gem install stubby
+    > $ sudo gem install stubby
 
 ## Available Options
 
-                > $ sudo stubby -h
-                > Commands:
-                >  stubby env NAME           # Switch stubby environment
-                >  stubby help [COMMAND]     # Describe available commands or one specific command
-                >  stubby search             # View all available stubs
-                >  stubby start ENVIRONMENT  # Starts stubby HTTP and DNS servers, default env ENVIRONMENT
-                >  stubby status             # View current rules
+    > $ sudo stubby -h
+    > Commands:
+    >  stubby env NAME           # Switch stubby environment
+    >  stubby help [COMMAND]     # Describe available commands or one specific command
+    >  stubby start ENVIRONMENT  # Starts stubby HTTP and DNS servers, default env ENVIRONMENT
+    >  stubby status             # View current rules
 
 ## Getting Started
 
@@ -38,72 +37,99 @@ Stubby uses `Stubfile.json` for configuration. This file includes a mapping of
 environments to a number of rules that define server configurations and stub
 usage for the project. 
 
-                > cd ~/MyProject
-                > cat Stubfile.json
-                > {
-                >  "test": {
-                >    "dependencies": {
-                >      "example": "staging"
-                >    },
-                >
-                >    "(https?:\/\/)?example.com": "http://localhost:3000"
-                >  },
-                >
-                >  "staging": {
-                >    "dependencies": {
-                >      "example": "staging"
-                >    },
-                >
-                >    "example.com": "dns-cname://aws..."
-                >  }
-                > }
-
-                > $ sudo stubby start
+    > cd ~/MyProject
+    > cat Stubfile.json
+    > {
+    >  "test": {
+    >    "dependencies": {
+    >      "https://github.com/jkassemi/example-stubby.git": "staging"
+    >    },
+    >
+    >    "example.com": "localhost:3000"
+    >  },
+    >
+    >  "staging": {
+    >    "dependencies": {
+    >      "https://github.com/jkassemi/example-stubby.git": "staging"
+    >    },
+    >
+    >    "example.com": "dns-cname://aws..."
+    >  }
+    > }
+
+    > $ sudo stubby start
 
 The 'test' and 'staging' modes for this project both include rules for the
 'example' stub, and then define a single rule of their own. Stubby starts
 by default in the 'development' environment, so with this `Stubfile.json`,
 the stubby server is not yet modifying any requests. In a new terminal:
 
-                > $ sudo stubby env test
+    > $ sudo stubby env test
 
 Switches stubby to test mode. Now the 'example' stub is activated, and
 additionally any requests to http or https versions of example.com are
 routed to http://localhost:3000. Let's take a look at the rules applied:
 
-                > $ sudo bin/stubby status
-                > {
-                >  "rules":{
-                >    "example":{
-                >      "_comment":"All SMTP traffic (NOT YET FUNCTIONAL)",
-                >      "admin.example.com":"10.0.1.1",
-                >      "admin2.example.com":"dns-cname://admin.example.com",
-                >      "(http?://)?merchant.example.com":"http://10.0.1.1",
-                >      "(https?://)?.*.example.io":"http://10.0.1.1",
-                >      "(https?://)?.*mail.*yahoo.*":"http://en.wikipedia.org/wiki/RTFM",
-                >      "(https?://)?yahoo.com":"https-redirect://duckduckgo.com",
-                >      "stubby\\..*":"file:///var/www/tmp",
-                >      "api.example.com":"file://~/.stubby/example/files",
-                >      "smtp://.*":"log:///var/log/out.txt"
-                >    },
-                >    "_":{
-                >      "dependencies":{
-                >        "example":"staging"
-                >      },
-                >      "(https?://)?example.com":"http://localhost:3000"
-                >    }
-                >  },
-                >  "environment":"test"
-                > }
+    > $ sudo bin/stubby status
+
+    > {
+    > "rules":{
+    >   "https://github.com/jkassemi/example-stubby.git":{
+    >     "dns://admin.example.com/a":"dns-a://172.16.123.1",
+    >     "http://admin.example.com":"http-redirect://blank?to=https://admin.example.com&code=302",
+    >     "https://admin.example.com":"http-proxy://10.0.1.1",
+    >     "dns://admin2.example.com/a":"dns-cname://admin.example.com",
+    >     "http://(.*)\\.?example.com":"http-proxy://10.0.1.1",
+    >     "dns://(.*)\\.?example.com/a":"dns-a://172.16.123.1",
+    >     "https://g?mail.*/.*":"http-proxy://en.wikipedia.org/wiki/RTFM",
+    >     "dns://g?mail.*/.*/a":"dns-a://172.16.123.1",
+    >     "http://yahoo.com":"http-redirect://duckduckgo.com",
+    >     "dns://yahoo.com/a":"dns-a://172.16.123.1",
+    >     "https://yahoo.com":"https-redirect://duckduckgo.com",
+    >     "dns://.*\\.stubby.dev/a":"dns-a://172.16.123.1",
+    >     "http://.*\\.stubby.dev":"file:///var/www/tmp",
+    >     "https://.*\\.stubby.dev":"file:///var/www/tmp",
+    >     "dns://api.example.com/a":"dns-a://172.16.123.1",
+    >     "http://api.example.com":"file://~/.stubby/example/files",
+    >     "https://api.example.com":"file://~/.stubby/example/files",
+    >     "dns://.*/mx":"dns-mx://172.16.123.1/?priority=10"
+    >   },
+    >   "_":{
+    >     "dependencies":{
+    >       "https://github.com/jkassemi/example-stubby.git":"staging"
+    >     },
+    >     "dns://secured.atpay.com/a":"dns-a://172.16.123.1",
+    >     "http://secured.atpay.com":"http-redirect://blank?to=https://secured.atpay.com&code=302",
+    >     "https://secured.atpay.com":"http-proxy://localhost:3000",
+    >     "dns://api.atpay.com/a":"dns-a://172.16.123.1",
+    >     "http://api.atpay.com":"http-redirect://blank?to=https://api.atpay.com&code=302",
+    >     "https://api.atpay.com":"http-proxy://localhost:4000",
+    >     "dns://.*/mx":"dns-mx://172.16.123.1/?priority=10"
+    >   },
+    >   "_smtp":{
+    >     "dns://outbox.stubby.dev/a":"dns-a://172.16.123.1",
+    >     "http://outbox.stubby.dev":"http-proxy://172.16.123.1:9001"
+    >   }
+    > },
+    > "environment":"development"
+    > }
 
 This shows us all activated rules. the "_" indicates that the rules are loaded
-from the current `Stubfile.json`. We also see that requests to yahoo.com are
-redirected to https://duckduckgo.com:
+from the current `Stubfile.json`, while "example" indicates that the rules are
+loaded from an installed stub, "example". 
 
 To revert the system back to normal, just CTRL-C from the main stubby process.
 This will revert any changes made to configure DNS servers for all network 
 interfaces and will shut down the stubby server.
 
+### Stubs
+
+- example: https://github.com/jkassemi/example-stubby.git 
+
+To contribute a stub, just add your stub to the list above and issue a pull
+request.
+
+
 ## Stubbing
 
 A stub is a folder named with the name of the stub that contains a stubby.json file. The stubby.json file contains a hash with the available
@@ -111,91 +137,94 @@ modes. Each mode contains a set of rules that define how to route DNS and how to
 
 Installed stubs are in the ~/.stubby folder:
 
-		> $ ls ~/.stubby 
-		> example		system.json
+    > $ ls ~/.stubby/jkassemi
+    > stubby-example
 
 The example folder is the `example` stub, and the system.json file contains the agent configurations. You don't need to manually edit it.
 
-		> $ find ~/.stubby/example
-		> ... example
-		> ... example/files
-		> ... example/hello.html
-		> ... example/stubby.json
+    > $ find ~/.stubby/jkassemi/example
+    > ... jkassemi/example
+    > ... jkassemi/example/files
+    > ... jkassemi/example/hello.html
+    > ... jkassemi/example/stubby.json
 		
 The example/stubby.json file has two modes, staging, and production:
 
-		> cat ~/.stubby/example/stubby.json
-		{ "staging": {...}, "production": {...} }
+    > cat ~/.stubby/jkassemi/example-stubby/stubby.json
+    { "staging": {...}, "production": {...} }
 
 Each environment contains a number of rules:
 
-		{ "staging": {
-			"MATCH_REGEXP": "INSTRUCTION"
-		} ... }		
+    { "staging": {
+            "[PROTOCOL://]MATCH_REGEXP": "INSTRUCTION"
+    } ... }		
 		
 When a request is made, either DNS or HTTP (important), the request is
-compared against the MATCH_REGEXP. If matched, the INSTRUCTION is executed. Since the same rules are consulted for DNS and HTTP, if you are
-trying to overwrite a domain, you should make sure the match won't exclude
-simply the host. For example, to proxy web traffic from test.example.com
-to a server at 10.0.1.5:
+compared against the PROTOCOL and MATCH_REGEXP (together these are 
+called the TRIGGER). If matched, INSTRUCTION is executed. 
+Excluding a protocol from the trigger causes Stubby to presuppose a few
+things about your request. It'll handle DNS, HTTP and HTTPS for definitions
+like this. 
 
-		"test.example.com": "http://10.0.1.5"
+    "test.example.com": "http://10.0.1.5"
 		
-This results in 
+Expands to:
+
+    "dns://test.example.com/a": "dns-a://10.0.1.5",
+    "http://test.example.com": "http-redirect://blank?to=https://test.example.com&code=302",
+    "https://test.example.com": "http-proxy://10.0.1.5
+
+Don't want to default over to https? You can be more explicit:
+
+    "http://test.example.com": "http://10.0.1.5"
+
+Which expands to:
 
-		> $ dig test.example.com
-		...
-		;; ANSWER SECTION:
-		test.example.com.   0       IN      A       172.16.123.1
+    "dns://test.example.com/a": "dns-a://10.0.1.5",
+    "http://test.example.com": "http-proxy://10.0.1.5"
+
+    > $ dig test.example.com
+    ...
+    ;; ANSWER SECTION:
+    test.example.com.   0       IN      A       172.16.123.1
  
-172.16.123.1 is the stubby host url (TODO: configurable). All requests
+172.16.123.1 is the stubby interface (TODO: configurable). All requests
 to http://test.example.com are routed to the stubby web server at that
 address.
  
-		> $ curl test.example.com
- 		
-Issues a request handled by the stubby web server, which proxies the request to 172.16.123.1.
+    > $ curl test.example.com
 
-## Contributing a Stub
-
-Fork this repository, update the index.json file, and submit a pull request. For
-this major version, the remote registry will just be the index.json file from
-this project's github.
+Issues a request handled by the stubby web server, which proxies the request to 172.16.123.1.
 
 ### DNS Only
 
 To simply override DNS for test.example.com, you can create an A record on lookup:
 
-		"test.example.com": "10.0.1.5"		
-		
-Or a CNAME, if no IP is given:
-
-		"test.example.com": "test.example2.com"
-		
-But you can be explicit in the INSTRUCTION:
+    "dns://test.example.com": "dns-a://10.0.1.5"		
+              
+For a CNAME:
 
-		"test.example.com": "dns-a://10.0.1.5"
-		"test.example.com": "dns-cname://test.example2.com"
+    "dns://test.example.com": "dns-cname://test.example2.com"
 		
 Using the dns-#{name} convention, you can create simple references to 
-any dns record type. TODO: need to allow mx record priority somehow.
- 
+any dns record type.
+
 ### File Server
 
 Because stubby can intercept HTTP requests, it includes a base set of functionality that allows you two serve files directly from the stub. Given a rule:
 
-		"api.example.com": "file://~/.stubby/example/files"
+    "api.example.com": "file://~/.stubby/example/files"
 		
 DNS will resolve to the stubby server:
 
-		> $ dig api.example.com
-		... 
-		api.example.com.   0       IN      A       172.16.123.1
+    > $ dig api.example.com
+    ... 
+    api.example.com.   0       IN      A       172.16.123.1
 		
 And a web request to api.example.com will serve files from the ~/.stubby/example/files directory:
 
-		> $ curl http://api.example.com/hello.html
-		> <html><head></head><body>Hello</body></html>
+    > $ curl http://api.example.com/hello.html
+    > <html><head></head><body>Hello</body></html>
 
 This is designed to allow you to create API stubs (success responses, for instance).
 
@@ -204,16 +233,48 @@ This is designed to allow you to create API stubs (success responses, for instan
 
 Given a rule:
 
-		"(https?:\/\/)?yahoo.com": "http-redirect://duckduckgo.com"
+    "http://yahoo.com": "http-redirect://blank?to=duckduckgo.com"
 		
 DNS will resolve to the stubby server, and the web request to http://yahoo.com will redirect to http://duckduckgo.com.
 
+### Capturing Outgoing Email
+
+Stubby includes an SMTP extension to capture outgoing messages. Specify the
+domain to capture mail from and forward to smtp://
+
+    "smtp://.*\.example\.com": "about://blank"
+
+will ensure any message sent by your system directly (this does not include
+messages sent from yahoo or gmail) will be captured by Stubby. Visit
+http://outbox.stubby.dev to see the captured messages. 
+
+Mail capture is currently provided by the "MailCatcher" gem. 
+
 ### Vision
 
-* protocol in instruction becomes a plugin system. dns-cname:, for instance,
-  could be handled by the dns plugin. If it didn't exist when Stubfile.json was
-  being installed, it would be installed. 
-* proxy traffic on ports and send to log systems:
-  ":25": "log-smtp://"
-  ":3306": "log-mysql://"
+* general traffic monitoring proxy traffic on ports and send to log systems:
+  ":25": "smtp://"
+  ":3306": "mysql://"
 * web app front-end: show emails sent, mysql queries made, etc.
+* github installation with no index.json
+* P2P connections allow access to dev systems running stubby (agent mode?)
+
+    "example.com" => "localhost:3000"
+    "tunnel://example.com": "tunnel://jkassemi@stubby.site"
+
+      =>
+
+    On host system:
+
+    "example.com" => "localhost:3000"
+
+    On guest system:
+
+    "dns://example.com/.*" => @
+    "http://example.com" => "tunnel://jkassemi@stubby.site/?to=http://example.com"
+    "https://example.com" => "tunnel://jkassemi@stubby.site/?to=http://example.com"
+    "smtp://.*" => "tunnel://jkassemi@stubby.site/?to=smtp://$1"
+
+
+guest opens a connection to stubby.site, requesting last broadcast of NAT address
+for host. guest attempts udp tunnel with host

data/lib/stubby/cli/application.rb

@@ -63,55 +63,6 @@ module Stubby
         puts HTTPI.post("http://#{STUBBY_MASTER}:9000/environment.json", environment: name).body
       end
     
-      desc "search", "View all available stubs"
-      long_desc <<-LONGDESC
-        View all available registered stubs. These are stubs that you can use
-        as dependencies in Stubfile.json.
-
-        > $ sudo stubby search
-        > {
-        >   "example":[
-        >     {
-        >       "name":"example",
-        >       "version":"v0.0.1",
-        >       ...
-        >    }
-        >  ],
-        >  "spreedly":[
-        >    {
-        >      "name":"spreedly",
-        >      "version":"v0.0.1",
-        >       ...
-        >    }
-        >  ]
-        > }
-
-        Wildcard supported for search:
-
-        > $ sudo stubby search ex*
-        > {
-        >   "example":[
-        >     {
-        >       "name":"example",
-        >       "version":"v0.0.1",
-        >       ...
-        >    }
-        >  ]
-        > }
-
-      LONGDESC
-      def search(name=nil)
-        if master_running?
-          available = MultiJson.load(HTTPI.get("http://#{STUBBY_MASTER}:9000/stubs/available.json").body)
-        else
-          available = Stubby::Api.registry.index
-        end
-
-        puts MultiJson.dump(available.select { |key, ri| 
-          File.fnmatch(name || "*", key)
-        }, pretty: true)
-      end
-
       desc "status", "View current rules"
       long_desc <<-LONGDESC
         > $ sudo bin/stubby status
@@ -126,7 +77,7 @@ module Stubby
         >          "example":"staging"
         >        },
         >        "(https?://)?example.com":"http://localhost:3000"
-        >      }
+        >     }
         >    },
         >    "environment":"test"
         >  }

data/lib/stubby/extensions/default.rb

@@ -0,0 +1,72 @@
+module Extensions
+  class Default
+    def initialize
+
+    end    
+
+    def run!(*args)
+
+    end
+
+    def stop!(*args)
+
+    end
+
+    def expand_rule(trigger, instruction)
+      # Default expansion:
+      #     "example.com": "localhost:3000"
+      #
+      # => 
+      #     "dns://example.com":    "@"
+      #     "http://example.com":   "http-redirect://blank?to=https://example.com&code=302"
+      #     "https://example.com":  "http-proxy://localhost:3000"
+      #    
+      #     "example.com:4000": "localhost:3000"
+      #     
+      # => 
+      #     ERROR: port in trigger unsupported
+      #
+      #     "dns://example.com":      "@"
+      #     "http://example.com:4000" 
+      #
+      #     "example.com": "http-redirect://localhost:3000"
+      #
+      # =>
+      #     "dns://example.com":    "@"
+      #     "http://example.com":   "http-redirect://?blank?to=http://localhost:3000&code=302"
+      # =====================================
+      #
+      #   ".*\\.stubby.dev": "file:///var/www/tmp
+      #
+      #     =>
+      #
+      #   "dns://.*\\.stubby.dev": "@",
+      #   "http://.*\\.stubby.dev": "file:///var/www/tmp",
+      #   "https://.*\\.stubby.dev": "file:///var/www/tmp",
+      scheme, remains = instruction.split("://")
+      scheme, remains = remains, scheme if remains.nil?
+ 
+      if scheme.nil?
+        expand_bare(trigger, instruction)
+      else
+        expand_with_protocol(trigger, instruction)
+      end
+    end
+
+    def expand_bare(trigger, instruction)
+      {
+        "dns://#{trigger}/a" => "dns-a://#{STUBBY_MASTER}",
+        "http://#{trigger}" => "http-redirect://blank?to=https://#{trigger}&code=302",
+        "https://#{trigger}" => "http-proxy://#{instruction}" 
+      }
+    end
+
+    def expand_with_protocol(trigger, instruction)
+      {
+        "dns://#{trigger}/a" => "dns-a://#{STUBBY_MASTER}",
+        "http://#{trigger}" => instruction,
+        "https://#{trigger}" => instruction
+      }
+    end
+  end
+end

data/lib/stubby/extensions/dns.rb

@@ -23,49 +23,46 @@ module Extensions
       end
 
       def process(name, resource_class, transaction)
-          body = HTTPI.post("http://#{STUBBY_MASTER}:9000/rules/search.json", trigger: name).body
+          body = HTTPI.post("http://#{STUBBY_MASTER}:9000/rules/search.json", 
+            trigger: "dns://#{name}/#{symbol_from_resource_class(resource_class)}").body
 
           instruction = MultiJson.load(body)
 
-    		if instruction.nil? or instruction == "@"
-    			transaction.passthrough!(UPSTREAM)
-    			return
-    		end
+          if instruction.nil? or instruction == "@"
+            transaction.passthrough!(UPSTREAM)
+            return
+          end
 
-    		url = URI.parse(instruction)
+          url = URI.parse(instruction)
 
-    		if url.scheme.to_s.empty?
-    			url = URI.parse("dns-a://" + instruction)
-    		elsif (url.scheme.to_s =~ /^dns-.*/).nil?
-    			url.host = STUBBY_MASTER
-    		end
+          response_resource_class = resource url.scheme.gsub('dns-', '')
 
-    		response_resource_class = resource url.scheme.gsub('dns-', '')
+          if !IPAddress.valid?(url.host) and response_resource_class == IN::A
+            response_resource_class = IN::CNAME
+          end
 
-    		if url.host.to_s.empty?
-    			url.host = STUBBY_MASTER
-    		end
+          response = url.host
 
-    		if !IPAddress.valid?(url.host) and response_resource_class == IN::A
-    			response_resource_class = IN::CNAME
-    		end
+          if [IN::CNAME, IN::MX].include? response_resource_class
+            response = Resolv::DNS::Name.create(url.host)
+          end
 
-    		response = url.host
+          puts "DNS: #{name} => #{response}-#{resource_class.name})"
 
-    		if response_resource_class == IN::CNAME
-    			response = Resolv::DNS::Name.create(url.host)
-    		end
-
-    		puts "DNS: #{name} => #{response}-#{resource_class.name})"
-
-    		transaction.respond!(response, 
-    			:resource_class => response_resource_class, 
-    			:ttl => 0)
+          if response_resource_class == IN::MX
+            transaction.respond!(10, response,
+              :resource_class => response_resource_class,
+              :ttl => 0)
+          else
+            transaction.respond!(response, 
+              :resource_class => response_resource_class, 
+              :ttl => 0)
+          end
       end
 
       def run!(session, options)
         return if options[:dns] == false
-        trap("INT"){ stop!  }
+        trap("INT"){ stop! }
 
         @session = session
         setup_references and run_dns_server
@@ -75,12 +72,32 @@ module Extensions
         teardown_references and stop_dns_server
       end
 
+      def expand_rule(trigger, instruction)
+        i = URI.parse(instruction)
+        t = URI.parse(trigger)
+
+        # If not specifying a record type, match a
+        t.path = "/a" if t.path.empty?
+    
+        if i.scheme.nil?
+          { t.to_s => "dns-a://#{instruction}" }
+        else
+          { t.to_s => instruction }
+        end
+      end
       
       private
 
       def resource(pattern)
         return IN::A unless pattern.respond_to? :to_sym
+        symbol_to_resource_class[pattern.to_sym] || IN::A
+      end
+
+      def symbol_from_resource_class(klass)
+        symbol_to_resource_class.invert[klass] || :a
+      end
 
+      def symbol_to_resource_class
         {
           a:      IN::A,
           aaaa:   IN::AAAA,
@@ -93,7 +110,7 @@ module Extensions
           soa:    IN::SOA,
           txt:    IN::TXT,
           cname:  IN::CNAME
-        }[pattern.to_sym] || IN::A
+        }
       end
 
       def run_dns_server