leggy 0.1.3 → 0.2.0

data/lib/leggy/mapping/crawl.rb

@@ -6,7 +6,7 @@ module Leggy
       kartograph do
         mapping Leggy::Crawl
 
-        property *Leggy::Crawl.attr_accessors, scopes: :read
+        property *Leggy::Crawl.attr_accessors, scopes: [ :read ]
       end
     end
   end

data/lib/leggy/mapping/crawl_options.rb

@@ -6,7 +6,7 @@ module Leggy
       kartograph do
         mapping Leggy::CrawlOptions
 
-        property *Leggy::CrawlOptions.attr_accessors, scopes: :create
+        property *Leggy::CrawlOptions.attr_accessors, scopes: [ :create ]
       end
     end
   end

data/lib/leggy/mapping/url.rb

@@ -6,7 +6,7 @@ module Leggy
       kartograph do
         mapping Leggy::Url
 
-        property *Leggy::Url.attr_accessors, scopes: :read
+        property *Leggy::Url.attr_accessors, scopes: [ :read ]
       end
     end
   end

data/lib/leggy/mapping/user.rb

@@ -6,7 +6,7 @@ module Leggy
       kartograph do
         mapping Leggy::User
 
-        property *Leggy::User.attr_accessors, scopes: :read
+        property *Leggy::User.attr_accessors, scopes: [ :read ]
       end
 
     end

data/lib/leggy/resource/app.rb

@@ -1,13 +1,10 @@
 module Leggy
   module Resource
     class App < ResourceKit::Resource
+      include Leggy::ErrorHandler
 
       resources do
 
-        default_handler do |response|
-          raise "ERROR #{response.status}: #{response.body}"
-        end
-
         # GET https://api_token:@api.80legs.com/v2/apps
         #
         action :all do
@@ -23,7 +20,7 @@ module Leggy
           verb :put
           path '/v2/apps/:name'
           body { |object| File.file?(object[:body]) ? File.read(object[:body]) : object[:body] }
-          handler(204) { |response| true }
+          handler(204) { true }
         end
 
         # GET https://api_token:@api.80legs.com/v2/apps/{APP_NAME}
@@ -39,7 +36,7 @@ module Leggy
         action :delete do
           verb :delete
           path '/v2/apps/:name'
-          handler(204) { |response| true }
+          handler(204) { true }
         end
 
       end

data/lib/leggy/resource/crawl.rb

@@ -1,11 +1,16 @@
 module Leggy
   module Resource
     class Crawl < ResourceKit::Resource
+      include Leggy::ErrorHandler
 
       resources do
 
-        default_handler do |response|
-          raise "ERROR #{response.status}: #{response.body}"
+        # GET https://api_token:@api.80legs.com/v2/crawls
+        #
+        action :all do
+          verb :get
+          path '/v2/crawls'
+          handler(200) { |response| Leggy::Mapping::Crawl.extract_collection(response.body, :read) }
         end
 
         # POST https://api_token:@api.80legs.com/v2/crawls/{CRAWL_NAME}
@@ -14,7 +19,7 @@ module Leggy
           verb :post
           path '/v2/crawls/:name'
           body { |object| object.slice(*Leggy::CrawlOptions.attr_accessors).to_json }
-          handler(204) { |response| true }
+          handler(204) { true }
         end
 
         # GET https://api_token:@api.80legs.com/v2/crawls/{CRAWL_NAME}
@@ -30,15 +35,7 @@ module Leggy
         action :cancel do
           verb :delete
           path '/v2/crawls/:name'
-          handler(204) { |response| true }
-        end
-
-        # GET https://api_token:@api.80legs.com/v2/results/{CRAWL_NAME}
-        #
-        action :results do
-          verb :get
-          path '/v2/results/:name'
-          handler(200) { |response| JSON.parse(response.body) }
+          handler(204) { true }
         end
 
       end

data/lib/leggy/resource/result.rb

@@ -0,0 +1,20 @@
+module Leggy
+  module Resource
+    class Result < ResourceKit::Resource
+      include Leggy::ErrorHandler
+
+      resources do
+
+        # GET https://api_token:@api.80legs.com/v2/results/{CRAWL_NAME}
+        #
+        action :all do
+          verb :get
+          path '/v2/results/:name'
+          handler(200) { |response| JSON.parse(response.body) }
+        end
+
+      end
+
+    end
+  end
+end

data/lib/leggy/resource/url.rb

@@ -1,13 +1,10 @@
 module Leggy
   module Resource
     class Url < ResourceKit::Resource
+      include Leggy::ErrorHandler
 
       resources do
 
-        default_handler do |response|
-          raise "ERROR #{response.status}: #{response.body}"
-        end
-
         # GET https://api_token:@api.80legs.com/v2/urllists
         #
         action :all do
@@ -23,7 +20,7 @@ module Leggy
           verb :put
           path '/v2/urllists/:name'
           body { |object| File.file?(object[:body]) ? File.read(object[:body]) : object[:body] }
-          handler(204) { |response| true }
+          handler(204) { true }
         end
 
         # GET https://api_token:@api.80legs.com/v2/urllists/{URLS_NAME}
@@ -39,7 +36,7 @@ module Leggy
         action :delete do
           verb :delete
           path '/v2/urllists/:name'
-          handler(204) { |response| true }
+          handler(204) { true }
         end
 
       end

data/lib/leggy/resource/user.rb

@@ -1,13 +1,10 @@
 module Leggy
   module Resource
     class User < ResourceKit::Resource
+      include Leggy::ErrorHandler
 
       resources do
 
-        default_handler do |response|
-          raise "ERROR #{response.status}: #{response.body}"
-        end
-
         # GET https://api_token:@api.80legs.com/v2/users/{USER_API_TOKEN}
         #
         action :find do

data/lib/leggy/url.rb

@@ -2,12 +2,11 @@ module Leggy
   class Url
     include Leggy::Helpers
 
-    attr_accessor(
+    attr_accessor \
       :name,
       :user,
       :location,
       :date_created
-    )
 
   end
 end

data/lib/leggy/user.rb

@@ -2,7 +2,7 @@ module Leggy
   class User
     include Leggy::Helpers
 
-    attr_accessor(
+    attr_accessor \
       :token,
       :organization,
       :email,
@@ -13,7 +13,6 @@ module Leggy
       :active,
       :urls_crawled,
       :date_registered
-    )
 
   end
 end

data/lib/leggy/version.rb

@@ -1,3 +1,3 @@
 module Leggy
-  VERSION = "0.1.3"
+  VERSION = "0.2.0"
 end

data/spec/cassettes/{leggy.yml → leggy_app.yml}

@@ -1,128 +1,5 @@
 ---
 http_interactions:
-- request:
-    method: put
-    uri: https://<TOKEN>:@api.80legs.com/v2/apps/sample
-    body:
-      encoding: UTF-8
-      string: |
-        // This 80app returns the header data from each URL crawled
-
-        var EightyApp = function() {
-                this.processDocument = function(html, url, headers, status, jQuery) {
-                        var app = this;
-                        var $ = jQuery;
-                        var $html = app.parseHtml(html, $);
-                        var object = {};
-
-                        if(typeof headers == 'string' || headers instanceof String) {
-                                var headersArray = headers.split("\r\n");
-                                for (var i = 0; i < headersArray.length; i++) {
-                                        var keyvalArray = headersArray[i].split(": ");
-                                        var key = keyvalArray[0];
-                                        var value = keyvalArray[1];
-                                        object[key] = value;
-                                }
-
-                                return JSON.stringify(object);
-                        }
-
-                        return JSON.stringify(headers);
-                }
-
-                this.parseLinks = function(html, url, headers, status, jQuery) {
-                        var app = this;
-                        var $ = jQuery;
-                        var $html = app.parseHtml(html, $);
-                        var links = [];
-
-                        // gets all links in the html document
-                        $html.find('a').each(function(i, obj) {
-                                var link = app.makeLink(url, $(this).attr('href'));
-                                if(link != null) {
-                                        links.push(link);
-                                }
-                        });
-
-                        return links;
-                }
-        }
-
-        try {
-                // Testing
-                module.exports = function(EightyAppBase) {
-                        EightyApp.prototype = new EightyAppBase();
-                        return new EightyApp();
-                }
-        } catch(e) {
-                // Production
-                console.log("Eighty app exists.");
-                EightyApp.prototype = new EightyAppBase();
-        }
-    headers:
-      Content-Type:
-      - application/octet-stream
-      User-Agent:
-      - Faraday v0.9.0
-      Accept-Encoding:
-      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
-      Accept:
-      - "*/*"
-  response:
-    status:
-      code: 204
-      message: No Content
-    headers:
-      Content-Length:
-      - '0'
-      Content-Type:
-      - application/octet-stream
-      Date:
-      - Tue, 14 Oct 2014 21:37:52 GMT
-      Server:
-      - Cowboy
-      Connection:
-      - keep-alive
-    body:
-      encoding: UTF-8
-      string: ''
-    http_version: 
-  recorded_at: Tue, 14 Oct 2014 21:37:54 GMT
-- request:
-    method: delete
-    uri: https://<TOKEN>:@api.80legs.com/v2/apps/sample
-    body:
-      encoding: US-ASCII
-      string: ''
-    headers:
-      Content-Type:
-      - application/json
-      User-Agent:
-      - Faraday v0.9.0
-      Accept-Encoding:
-      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
-      Accept:
-      - "*/*"
-  response:
-    status:
-      code: 204
-      message: No Content
-    headers:
-      Content-Length:
-      - '0'
-      Content-Type:
-      - application/octet-stream
-      Date:
-      - Tue, 14 Oct 2014 21:37:52 GMT
-      Server:
-      - Cowboy
-      Connection:
-      - keep-alive
-    body:
-      encoding: UTF-8
-      string: ''
-    http_version: 
-  recorded_at: Tue, 14 Oct 2014 21:37:55 GMT
 - request:
     method: put
     uri: https://<TOKEN>:@api.80legs.com/v2/apps/sample_all
@@ -186,7 +63,7 @@ http_interactions:
       Content-Type:
       - application/octet-stream
       User-Agent:
-      - Faraday v0.9.0
+      - Faraday v0.9.2
       Accept-Encoding:
       - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
       Accept:
@@ -201,7 +78,7 @@ http_interactions:
       Content-Type:
       - application/octet-stream
       Date:
-      - Tue, 14 Oct 2014 21:37:53 GMT
+      - Sat, 12 Dec 2015 06:03:54 GMT
       Server:
       - Cowboy
       Connection:
@@ -210,7 +87,7 @@ http_interactions:
       encoding: UTF-8
       string: ''
     http_version: 
-  recorded_at: Tue, 14 Oct 2014 21:37:55 GMT
+  recorded_at: Sat, 12 Dec 2015 06:03:58 GMT
 - request:
     method: get
     uri: https://<TOKEN>:@api.80legs.com/v2/apps
@@ -221,7 +98,7 @@ http_interactions:
       Content-Type:
       - application/json
       User-Agent:
-      - Faraday v0.9.0
+      - Faraday v0.9.2
       Accept-Encoding:
       - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
       Accept:
@@ -234,11 +111,11 @@ http_interactions:
       Content-Type:
       - application/json
       Date:
-      - Tue, 14 Oct 2014 21:37:53 GMT
+      - Sat, 12 Dec 2015 06:03:55 GMT
       Server:
       - Cowboy
       Content-Length:
-      - '4636'
+      - '5542'
       Connection:
       - keep-alive
     body:
@@ -251,23 +128,28 @@ http_interactions:
         the following attributes from each URL crawled: title, meta tags, links (everything
         in an ''a'' tag)","user":"bjadslid9v7uat88n70xjz0omeyun69j","date_created":"2014-7-17"},{"location":"80apps/bjadslid9v7uat88n70xjz0omeyun69j/LinkTracer.js","name":"LinkTracer.js","description":"Returns
         a timestamp for when the URL was crawled, the original URL that led to this
-        URL being crawled, and the HTML.","user":"bjadslid9v7uat88n70xjz0omeyun69j","date_created":"2014-8-25"},{"location":"80apps/bjadslid9v7uat88n70xjz0omeyun69j/LinksAndKeywords.js","name":"LinksAndKeywords.js","description":"Returns
-        all links and keywords, with their counts, found on a page","user":"bjadslid9v7uat88n70xjz0omeyun69j","date_created":"2014-8-26"},{"location":"80apps/bjadslid9v7uat88n70xjz0omeyun69j/LinkCollector.js","name":"LinkCollector.js","description":"Returns
-        all links found on a page","user":"bjadslid9v7uat88n70xjz0omeyun69j","date_created":"2014-7-17"},{"location":"80apps/bjadslid9v7uat88n70xjz0omeyun69j/KeywordCount.js","name":"KeywordCount.js","description":"Returns
-        a timestamp for when the URL was crawled, and a list of words on the page
-        with their frequencies.","user":"bjadslid9v7uat88n70xjz0omeyun69j","date_created":"2014-7-17"},{"location":"80apps/bjadslid9v7uat88n70xjz0omeyun69j/HeaderData.js","name":"HeaderData.js","description":"Returns
+        URL being crawled, and the HTML.","user":"bjadslid9v7uat88n70xjz0omeyun69j","date_created":"2014-8-25"},{"location":"80apps/bjadslid9v7uat88n70xjz0omeyun69j/LinksAndKeywords.js","name":"LinksAndKeywords.js","description":"Custom
+        80App","user":"bjadslid9v7uat88n70xjz0omeyun69j","date_created":"2015-7-9"},{"location":"80apps/bjadslid9v7uat88n70xjz0omeyun69j/LinkCollector.js","name":"LinkCollector.js","description":"Returns
+        all links found on a page, along with their anchor text.","user":"bjadslid9v7uat88n70xjz0omeyun69j","date_created":"2015-2-5"},{"location":"80apps/bjadslid9v7uat88n70xjz0omeyun69j/KeywordCount.js","name":"KeywordCount.js","description":"Custom
+        80App","user":"bjadslid9v7uat88n70xjz0omeyun69j","date_created":"2015-7-9"},{"location":"80apps/bjadslid9v7uat88n70xjz0omeyun69j/HeaderData.js","name":"HeaderData.js","description":"Returns
         the header data from each URL crawled","user":"bjadslid9v7uat88n70xjz0omeyun69j","date_created":"2014-9-8"},{"location":"80apps/bjadslid9v7uat88n70xjz0omeyun69j/FullPageContent.js","name":"FullPageContent.js","description":"Returns
         full HTML source of each URL crawled.","user":"bjadslid9v7uat88n70xjz0omeyun69j","date_created":"2014-7-17"},{"location":"80apps/bjadslid9v7uat88n70xjz0omeyun69j/FileFinder.js","name":"FileFinder.js","description":"Returns
         all links to files (e.g., .docx, .pptx, .pdf, etc.) found on a page","user":"bjadslid9v7uat88n70xjz0omeyun69j","date_created":"2014-7-17"},{"location":"80apps/bjadslid9v7uat88n70xjz0omeyun69j/ExternalLinkCollector.js","name":"ExternalLinkCollector.js","description":"Returns
         all links found on a page","user":"bjadslid9v7uat88n70xjz0omeyun69j","date_created":"2014-7-17"},{"location":"80apps/bjadslid9v7uat88n70xjz0omeyun69j/EmailsAndPageContent.js","name":"EmailsAndPageContent.js","description":"Returns
         a list of email and full page content","user":"bjadslid9v7uat88n70xjz0omeyun69j","date_created":"2014-7-17"},{"location":"80apps/bjadslid9v7uat88n70xjz0omeyun69j/EmailCollector.js","name":"EmailCollector.js","description":"Returns
-        a list of emails for every page on the domains of the URL list","user":"bjadslid9v7uat88n70xjz0omeyun69j","date_created":"2014-7-17"},{"location":"80apps/bjadslid9v7uat88n70xjz0omeyun69j/DocumentData.js","name":"DocumentData.js","description":"Returns
+        a list of emails for every page on the domains of the URL list","user":"bjadslid9v7uat88n70xjz0omeyun69j","date_created":"2014-7-17"},{"location":"80apps/bjadslid9v7uat88n70xjz0omeyun69j/DomainCollector.js","name":"DomainCollector.js","description":"This
+        80app will only crawl to links on the current domain, returning the count
+        of every domain linked from each URL crawled.","user":"bjadslid9v7uat88n70xjz0omeyun69j","date_created":"2014-10-28"},{"location":"80apps/bjadslid9v7uat88n70xjz0omeyun69j/DocumentData.js","name":"DocumentData.js","description":"Returns
         title, meta tags, and links attributes from each URL crawled","user":"bjadslid9v7uat88n70xjz0omeyun69j","date_created":"2014-7-17"},{"location":"80apps/bjadslid9v7uat88n70xjz0omeyun69j/CrawlInternalLinks.js","name":"CrawlInternalLinks.js","description":"Returns
         URLs that have the same domain as the current URL being crawled.","user":"bjadslid9v7uat88n70xjz0omeyun69j","date_created":"2014-7-17"},{"location":"80apps/bjadslid9v7uat88n70xjz0omeyun69j/CrawlImages.js","name":"CrawlImages.js","description":"Returns
-        all URLs to and base64-encoded versions of all images found on each URL crawled","user":"bjadslid9v7uat88n70xjz0omeyun69j","date_created":"2014-7-17"},{"location":"80apps/<TOKEN>/sample_all","name":"sample_all","description":"Custom
-        80App","user":"<TOKEN>","date_created":"2014-10-14"}]'
+        all URLs to and base64-encoded versions of all images found on each URL crawled","user":"bjadslid9v7uat88n70xjz0omeyun69j","date_created":"2014-7-17"},{"location":"80apps/bjadslid9v7uat88n70xjz0omeyun69j/CheckStatusOnExternalDomain.js","name":"CheckStatusOnExternalDomain.js","description":"Keeps
+        crawling until it hits an external domain and then stops. Returns status code
+        for every URL crawled.","user":"bjadslid9v7uat88n70xjz0omeyun69j","date_created":"2015-3-10"},{"location":"80apps/<TOKEN>/sample_all","name":"sample_all","description":"Custom
+        80App","user":"<TOKEN>","date_created":"2015-12-12"},{"location":"80apps/<TOKEN>/CrawlerWithoutHTML","name":"CrawlerWithoutHTML","description":"Custom
+        80App","user":"<TOKEN>","date_created":"2015-1-26"},{"location":"80apps/<TOKEN>/CrawlerWithHTML","name":"CrawlerWithHTML","description":"Custom
+        80App","user":"<TOKEN>","date_created":"2015-1-20"}]'
     http_version: 
-  recorded_at: Tue, 14 Oct 2014 21:37:55 GMT
+  recorded_at: Sat, 12 Dec 2015 06:03:59 GMT
 - request:
     method: delete
     uri: https://<TOKEN>:@api.80legs.com/v2/apps/sample_all
@@ -278,7 +160,7 @@ http_interactions:
       Content-Type:
       - application/json
       User-Agent:
-      - Faraday v0.9.0
+      - Faraday v0.9.2
       Accept-Encoding:
       - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
       Accept:
@@ -293,7 +175,7 @@ http_interactions:
       Content-Type:
       - application/octet-stream
       Date:
-      - Tue, 14 Oct 2014 21:37:53 GMT
+      - Sat, 12 Dec 2015 06:03:55 GMT
       Server:
       - Cowboy
       Connection:
@@ -302,10 +184,10 @@ http_interactions:
       encoding: UTF-8
       string: ''
     http_version: 
-  recorded_at: Tue, 14 Oct 2014 21:37:55 GMT
+  recorded_at: Sat, 12 Dec 2015 06:03:59 GMT
 - request:
     method: put
-    uri: https://<TOKEN>:@api.80legs.com/v2/apps/sample_delete
+    uri: https://<TOKEN>:@api.80legs.com/v2/apps/sample
     body:
       encoding: UTF-8
       string: |
@@ -366,7 +248,7 @@ http_interactions:
       Content-Type:
       - application/octet-stream
       User-Agent:
-      - Faraday v0.9.0
+      - Faraday v0.9.2
       Accept-Encoding:
       - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
       Accept:
@@ -381,7 +263,7 @@ http_interactions:
       Content-Type:
       - application/octet-stream
       Date:
-      - Tue, 14 Oct 2014 21:37:53 GMT
+      - Sat, 12 Dec 2015 06:03:55 GMT
       Server:
       - Cowboy
       Connection:
@@ -390,10 +272,10 @@ http_interactions:
       encoding: UTF-8
       string: ''
     http_version: 
-  recorded_at: Tue, 14 Oct 2014 21:37:55 GMT
+  recorded_at: Sat, 12 Dec 2015 06:04:00 GMT
 - request:
     method: delete
-    uri: https://<TOKEN>:@api.80legs.com/v2/apps/sample_delete
+    uri: https://<TOKEN>:@api.80legs.com/v2/apps/sample
     body:
       encoding: US-ASCII
       string: ''
@@ -401,7 +283,7 @@ http_interactions:
       Content-Type:
       - application/json
       User-Agent:
-      - Faraday v0.9.0
+      - Faraday v0.9.2
       Accept-Encoding:
       - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
       Accept:
@@ -416,7 +298,7 @@ http_interactions:
       Content-Type:
       - application/octet-stream
       Date:
-      - Tue, 14 Oct 2014 21:37:53 GMT
+      - Sat, 12 Dec 2015 06:03:56 GMT
       Server:
       - Cowboy
       Connection:
@@ -425,7 +307,7 @@ http_interactions:
       encoding: UTF-8
       string: ''
     http_version: 
-  recorded_at: Tue, 14 Oct 2014 21:37:56 GMT
+  recorded_at: Sat, 12 Dec 2015 06:04:01 GMT
 - request:
     method: put
     uri: https://<TOKEN>:@api.80legs.com/v2/apps/sample_find
@@ -489,7 +371,7 @@ http_interactions:
       Content-Type:
       - application/octet-stream
       User-Agent:
-      - Faraday v0.9.0
+      - Faraday v0.9.2
       Accept-Encoding:
       - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
       Accept:
@@ -504,7 +386,7 @@ http_interactions:
       Content-Type:
       - application/octet-stream
       Date:
-      - Tue, 14 Oct 2014 21:37:54 GMT
+      - Sat, 12 Dec 2015 06:03:57 GMT
       Server:
       - Cowboy
       Connection:
@@ -513,7 +395,7 @@ http_interactions:
       encoding: UTF-8
       string: ''
     http_version: 
-  recorded_at: Tue, 14 Oct 2014 21:37:56 GMT
+  recorded_at: Sat, 12 Dec 2015 06:04:01 GMT
 - request:
     method: get
     uri: https://<TOKEN>:@api.80legs.com/v2/apps/sample_find
@@ -524,7 +406,7 @@ http_interactions:
       Content-Type:
       - application/json
       User-Agent:
-      - Faraday v0.9.0
+      - Faraday v0.9.2
       Accept-Encoding:
       - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
       Accept:
@@ -537,7 +419,7 @@ http_interactions:
       Content-Type:
       - application/octet-stream
       Date:
-      - Tue, 14 Oct 2014 21:37:54 GMT
+      - Sat, 12 Dec 2015 06:03:57 GMT
       Server:
       - Cowboy
       Content-Length:
@@ -601,7 +483,7 @@ http_interactions:
                 EightyApp.prototype = new EightyAppBase();
         }
     http_version: 
-  recorded_at: Tue, 14 Oct 2014 21:37:56 GMT
+  recorded_at: Sat, 12 Dec 2015 06:04:02 GMT
 - request:
     method: delete
     uri: https://<TOKEN>:@api.80legs.com/v2/apps/sample_find
@@ -612,7 +494,7 @@ http_interactions:
       Content-Type:
       - application/json
       User-Agent:
-      - Faraday v0.9.0
+      - Faraday v0.9.2
       Accept-Encoding:
       - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
       Accept:
@@ -627,7 +509,7 @@ http_interactions:
       Content-Type:
       - application/octet-stream
       Date:
-      - Tue, 14 Oct 2014 21:37:54 GMT
+      - Sat, 12 Dec 2015 06:03:58 GMT
       Server:
       - Cowboy
       Connection:
@@ -636,53 +518,71 @@ http_interactions:
       encoding: UTF-8
       string: ''
     http_version: 
-  recorded_at: Tue, 14 Oct 2014 21:37:56 GMT
-- request:
-    method: get
-    uri: https://<TOKEN>:@api.80legs.com/v2/users/<TOKEN>
-    body:
-      encoding: US-ASCII
-      string: ''
-    headers:
-      Content-Type:
-      - application/json
-      User-Agent:
-      - Faraday v0.9.0
-      Accept-Encoding:
-      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
-      Accept:
-      - "*/*"
-  response:
-    status:
-      code: 200
-      message: OK
-    headers:
-      Content-Type:
-      - application/json
-      Date:
-      - Tue, 14 Oct 2014 21:37:54 GMT
-      Server:
-      - Cowboy
-      Content-Length:
-      - '294'
-      Connection:
-      - keep-alive
-    body:
-      encoding: UTF-8
-      string: '{"token":"<TOKEN>","organization":"Civvic","email":"mattsolt@gmail.com","first_name":"Matt","last_name":"Solt","phone_number":"undefined","stripe_customer_id":"cus_4lcSWy6GfXcdmE","plan_id":"plus","type":"user","active":1,"urls_crawled":0,"date_registered":"2014-4-22"}'
-    http_version: 
-  recorded_at: Tue, 14 Oct 2014 21:37:56 GMT
+  recorded_at: Sat, 12 Dec 2015 06:04:02 GMT
 - request:
-    method: post
-    uri: https://<TOKEN>:@api.80legs.com/v2/crawls/status_crawl
+    method: put
+    uri: https://<TOKEN>:@api.80legs.com/v2/apps/sample_delete
     body:
       encoding: UTF-8
-      string: '{"app":"HeaderData.js","urllist":"1","max_depth":2,"max_urls":1000}'
+      string: |
+        // This 80app returns the header data from each URL crawled
+
+        var EightyApp = function() {
+                this.processDocument = function(html, url, headers, status, jQuery) {
+                        var app = this;
+                        var $ = jQuery;
+                        var $html = app.parseHtml(html, $);
+                        var object = {};
+
+                        if(typeof headers == 'string' || headers instanceof String) {
+                                var headersArray = headers.split("\r\n");
+                                for (var i = 0; i < headersArray.length; i++) {
+                                        var keyvalArray = headersArray[i].split(": ");
+                                        var key = keyvalArray[0];
+                                        var value = keyvalArray[1];
+                                        object[key] = value;
+                                }
+
+                                return JSON.stringify(object);
+                        }
+
+                        return JSON.stringify(headers);
+                }
+
+                this.parseLinks = function(html, url, headers, status, jQuery) {
+                        var app = this;
+                        var $ = jQuery;
+                        var $html = app.parseHtml(html, $);
+                        var links = [];
+
+                        // gets all links in the html document
+                        $html.find('a').each(function(i, obj) {
+                                var link = app.makeLink(url, $(this).attr('href'));
+                                if(link != null) {
+                                        links.push(link);
+                                }
+                        });
+
+                        return links;
+                }
+        }
+
+        try {
+                // Testing
+                module.exports = function(EightyAppBase) {
+                        EightyApp.prototype = new EightyAppBase();
+                        return new EightyApp();
+                }
+        } catch(e) {
+                // Production
+                console.log("Eighty app exists.");
+                EightyApp.prototype = new EightyAppBase();
+        }
     headers:
       Content-Type:
-      - application/json
+      - application/octet-stream
       User-Agent:
-      - Faraday v0.9.0
+      - Faraday v0.9.2
       Accept-Encoding:
       - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
       Accept:
@@ -695,9 +595,9 @@ http_interactions:
       Content-Length:
       - '0'
       Content-Type:
-      - application/json
+      - application/octet-stream
       Date:
-      - Tue, 14 Oct 2014 21:37:54 GMT
+      - Sat, 12 Dec 2015 06:03:59 GMT
       Server:
       - Cowboy
       Connection:
@@ -706,46 +606,10 @@ http_interactions:
       encoding: UTF-8
       string: ''
     http_version: 
-  recorded_at: Tue, 14 Oct 2014 21:37:57 GMT
-- request:
-    method: get
-    uri: https://<TOKEN>:@api.80legs.com/v2/crawls/status_crawl
-    body:
-      encoding: US-ASCII
-      string: ''
-    headers:
-      Content-Type:
-      - application/json
-      User-Agent:
-      - Faraday v0.9.0
-      Accept-Encoding:
-      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
-      Accept:
-      - "*/*"
-  response:
-    status:
-      code: 200
-      message: OK
-    headers:
-      Content-Type:
-      - application/json
-      Date:
-      - Tue, 14 Oct 2014 21:37:55 GMT
-      Server:
-      - Cowboy
-      Content-Length:
-      - '277'
-      Connection:
-      - keep-alive
-    body:
-      encoding: UTF-8
-      string: '{"id":57303,"name":"status_crawl","user":"<TOKEN>","user_agent":"voltron","app":"HeaderData.js","urllist":"1","data":"","max_depth":2,"max_urls":1000,"status":"QUEUED","depth":0,"urls_crawled":0,"date_created":"2014-10-14
-        21:37:55","date_completed":""}'
-    http_version: 
-  recorded_at: Tue, 14 Oct 2014 21:37:57 GMT
+  recorded_at: Sat, 12 Dec 2015 06:04:03 GMT
 - request:
     method: delete
-    uri: https://<TOKEN>:@api.80legs.com/v2/crawls/status_crawl
+    uri: https://<TOKEN>:@api.80legs.com/v2/apps/sample_delete
     body:
       encoding: US-ASCII
       string: ''
@@ -753,7 +617,7 @@ http_interactions:
       Content-Type:
       - application/json
       User-Agent:
-      - Faraday v0.9.0
+      - Faraday v0.9.2
       Accept-Encoding:
       - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
       Accept:
@@ -765,590 +629,10 @@ http_interactions:
     headers:
       Content-Length:
       - '0'
-      Content-Type:
-      - application/json
-      Date:
-      - Tue, 14 Oct 2014 21:37:55 GMT
-      Server:
-      - Cowboy
-      Connection:
-      - keep-alive
-    body:
-      encoding: UTF-8
-      string: ''
-    http_version: 
-  recorded_at: Tue, 14 Oct 2014 21:37:57 GMT
-- request:
-    method: post
-    uri: https://<TOKEN>:@api.80legs.com/v2/crawls/test_crawl
-    body:
-      encoding: UTF-8
-      string: '{"app":"HeaderData.js","urllist":"1","max_depth":2,"max_urls":1000}'
-    headers:
-      Content-Type:
-      - application/json
-      User-Agent:
-      - Faraday v0.9.0
-      Accept-Encoding:
-      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
-      Accept:
-      - "*/*"
-  response:
-    status:
-      code: 422
-      message: Unprocessable Entity
-    headers:
-      Content-Type:
-      - application/json
-      Date:
-      - Tue, 14 Oct 2014 21:37:55 GMT
-      Server:
-      - Cowboy
-      Content-Length:
-      - '0'
-      Connection:
-      - keep-alive
-    body:
-      encoding: UTF-8
-      string: ''
-    http_version: 
-  recorded_at: Tue, 14 Oct 2014 21:37:57 GMT
-- request:
-    method: delete
-    uri: https://<TOKEN>:@api.80legs.com/v2/crawls/test_crawl
-    body:
-      encoding: US-ASCII
-      string: ''
-    headers:
-      Content-Type:
-      - application/json
-      User-Agent:
-      - Faraday v0.9.0
-      Accept-Encoding:
-      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
-      Accept:
-      - "*/*"
-  response:
-    status:
-      code: 500
-      message: Internal Server Error
-    headers:
-      Content-Type:
-      - application/json
-      Date:
-      - Tue, 14 Oct 2014 21:37:55 GMT
-      Server:
-      - Cowboy
-      Content-Length:
-      - '43'
-      Connection:
-      - keep-alive
-    body:
-      encoding: UTF-8
-      string: '{"error":"crawl has already been canceled"}'
-    http_version: 
-  recorded_at: Tue, 14 Oct 2014 21:37:57 GMT
-- request:
-    method: put
-    uri: https://<TOKEN>:@api.80legs.com/v2/urllists/sample
-    body:
-      encoding: UTF-8
-      string: |
-        [
-          "http://example.com",
-          "https://example.org"
-        ]
-    headers:
-      Content-Type:
-      - application/octet-stream
-      User-Agent:
-      - Faraday v0.9.0
-      Accept-Encoding:
-      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
-      Accept:
-      - "*/*"
-  response:
-    status:
-      code: 204
-      message: No Content
-    headers:
-      Content-Length:
-      - '0'
-      Content-Type:
-      - application/octet-stream
-      Date:
-      - Tue, 14 Oct 2014 21:37:55 GMT
-      Server:
-      - Cowboy
-      Connection:
-      - keep-alive
-    body:
-      encoding: UTF-8
-      string: ''
-    http_version: 
-  recorded_at: Tue, 14 Oct 2014 21:37:58 GMT
-- request:
-    method: delete
-    uri: https://<TOKEN>:@api.80legs.com/v2/urllists/sample
-    body:
-      encoding: US-ASCII
-      string: ''
-    headers:
-      Content-Type:
-      - application/json
-      User-Agent:
-      - Faraday v0.9.0
-      Accept-Encoding:
-      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
-      Accept:
-      - "*/*"
-  response:
-    status:
-      code: 204
-      message: No Content
-    headers:
-      Content-Length:
-      - '0'
-      Content-Type:
-      - application/octet-stream
-      Date:
-      - Tue, 14 Oct 2014 21:37:56 GMT
-      Server:
-      - Cowboy
-      Connection:
-      - keep-alive
-    body:
-      encoding: UTF-8
-      string: ''
-    http_version: 
-  recorded_at: Tue, 14 Oct 2014 21:37:58 GMT
-- request:
-    method: put
-    uri: https://<TOKEN>:@api.80legs.com/v2/urllists/sample_all
-    body:
-      encoding: UTF-8
-      string: |
-        [
-          "http://example.com",
-          "https://example.org"
-        ]
-    headers:
       Content-Type:
       - application/octet-stream
-      User-Agent:
-      - Faraday v0.9.0
-      Accept-Encoding:
-      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
-      Accept:
-      - "*/*"
-  response:
-    status:
-      code: 204
-      message: No Content
-    headers:
-      Content-Length:
-      - '0'
-      Content-Type:
-      - application/octet-stream
-      Date:
-      - Tue, 14 Oct 2014 21:37:56 GMT
-      Server:
-      - Cowboy
-      Connection:
-      - keep-alive
-    body:
-      encoding: UTF-8
-      string: ''
-    http_version: 
-  recorded_at: Tue, 14 Oct 2014 21:37:58 GMT
-- request:
-    method: get
-    uri: https://<TOKEN>:@api.80legs.com/v2/urllists
-    body:
-      encoding: US-ASCII
-      string: ''
-    headers:
-      Content-Type:
-      - application/json
-      User-Agent:
-      - Faraday v0.9.0
-      Accept-Encoding:
-      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
-      Accept:
-      - "*/*"
-  response:
-    status:
-      code: 200
-      message: OK
-    headers:
-      Content-Type:
-      - application/json
-      Date:
-      - Tue, 14 Oct 2014 21:37:56 GMT
-      Server:
-      - Cowboy
-      Content-Length:
-      - '299'
-      Connection:
-      - keep-alive
-    body:
-      encoding: UTF-8
-      string: '[{"location":"urllists/<TOKEN>/sample_all","name":"sample_all","user":"<TOKEN>","date_created":"2014-10-14"},{"location":"urllists/<TOKEN>/1","name":"1","user":"<TOKEN>","date_created":"2014-10-12"}]'
-    http_version: 
-  recorded_at: Tue, 14 Oct 2014 21:37:58 GMT
-- request:
-    method: delete
-    uri: https://<TOKEN>:@api.80legs.com/v2/urllists/sample_all
-    body:
-      encoding: US-ASCII
-      string: ''
-    headers:
-      Content-Type:
-      - application/json
-      User-Agent:
-      - Faraday v0.9.0
-      Accept-Encoding:
-      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
-      Accept:
-      - "*/*"
-  response:
-    status:
-      code: 204
-      message: No Content
-    headers:
-      Content-Length:
-      - '0'
-      Content-Type:
-      - application/octet-stream
-      Date:
-      - Tue, 14 Oct 2014 21:37:56 GMT
-      Server:
-      - Cowboy
-      Connection:
-      - keep-alive
-    body:
-      encoding: UTF-8
-      string: ''
-    http_version: 
-  recorded_at: Tue, 14 Oct 2014 21:37:58 GMT
-- request:
-    method: put
-    uri: https://<TOKEN>:@api.80legs.com/v2/urllists/sample_delete
-    body:
-      encoding: UTF-8
-      string: |
-        [
-          "http://example.com",
-          "https://example.org"
-        ]
-    headers:
-      Content-Type:
-      - application/octet-stream
-      User-Agent:
-      - Faraday v0.9.0
-      Accept-Encoding:
-      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
-      Accept:
-      - "*/*"
-  response:
-    status:
-      code: 204
-      message: No Content
-    headers:
-      Content-Length:
-      - '0'
-      Content-Type:
-      - application/octet-stream
-      Date:
-      - Tue, 14 Oct 2014 21:37:56 GMT
-      Server:
-      - Cowboy
-      Connection:
-      - keep-alive
-    body:
-      encoding: UTF-8
-      string: ''
-    http_version: 
-  recorded_at: Tue, 14 Oct 2014 21:37:59 GMT
-- request:
-    method: delete
-    uri: https://<TOKEN>:@api.80legs.com/v2/urllists/sample_delete
-    body:
-      encoding: US-ASCII
-      string: ''
-    headers:
-      Content-Type:
-      - application/json
-      User-Agent:
-      - Faraday v0.9.0
-      Accept-Encoding:
-      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
-      Accept:
-      - "*/*"
-  response:
-    status:
-      code: 204
-      message: No Content
-    headers:
-      Content-Length:
-      - '0'
-      Content-Type:
-      - application/octet-stream
-      Date:
-      - Tue, 14 Oct 2014 21:37:57 GMT
-      Server:
-      - Cowboy
-      Connection:
-      - keep-alive
-    body:
-      encoding: UTF-8
-      string: ''
-    http_version: 
-  recorded_at: Tue, 14 Oct 2014 21:37:59 GMT
-- request:
-    method: put
-    uri: https://<TOKEN>:@api.80legs.com/v2/urllists/sample_find
-    body:
-      encoding: UTF-8
-      string: |
-        [
-          "http://example.com",
-          "https://example.org"
-        ]
-    headers:
-      Content-Type:
-      - application/octet-stream
-      User-Agent:
-      - Faraday v0.9.0
-      Accept-Encoding:
-      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
-      Accept:
-      - "*/*"
-  response:
-    status:
-      code: 204
-      message: No Content
-    headers:
-      Content-Length:
-      - '0'
-      Content-Type:
-      - application/octet-stream
-      Date:
-      - Tue, 14 Oct 2014 21:37:57 GMT
-      Server:
-      - Cowboy
-      Connection:
-      - keep-alive
-    body:
-      encoding: UTF-8
-      string: ''
-    http_version: 
-  recorded_at: Tue, 14 Oct 2014 21:37:59 GMT
-- request:
-    method: get
-    uri: https://<TOKEN>:@api.80legs.com/v2/urllists/sample_find
-    body:
-      encoding: US-ASCII
-      string: ''
-    headers:
-      Content-Type:
-      - application/json
-      User-Agent:
-      - Faraday v0.9.0
-      Accept-Encoding:
-      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
-      Accept:
-      - "*/*"
-  response:
-    status:
-      code: 200
-      message: OK
-    headers:
-      Content-Type:
-      - application/octet-stream
-      Date:
-      - Tue, 14 Oct 2014 21:37:57 GMT
-      Server:
-      - Cowboy
-      Content-Length:
-      - '52'
-      Connection:
-      - keep-alive
-    body:
-      encoding: UTF-8
-      string: |
-        [
-          "http://example.com",
-          "https://example.org"
-        ]
-    http_version: 
-  recorded_at: Tue, 14 Oct 2014 21:37:59 GMT
-- request:
-    method: delete
-    uri: https://<TOKEN>:@api.80legs.com/v2/urllists/sample_find
-    body:
-      encoding: US-ASCII
-      string: ''
-    headers:
-      Content-Type:
-      - application/json
-      User-Agent:
-      - Faraday v0.9.0
-      Accept-Encoding:
-      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
-      Accept:
-      - "*/*"
-  response:
-    status:
-      code: 204
-      message: No Content
-    headers:
-      Content-Length:
-      - '0'
-      Content-Type:
-      - application/octet-stream
-      Date:
-      - Tue, 14 Oct 2014 21:37:57 GMT
-      Server:
-      - Cowboy
-      Connection:
-      - keep-alive
-    body:
-      encoding: UTF-8
-      string: ''
-    http_version: 
-  recorded_at: Tue, 14 Oct 2014 21:37:59 GMT
-- request:
-    method: post
-    uri: https://<TOKEN>:@api.80legs.com/v2/crawls/start_crawl
-    body:
-      encoding: UTF-8
-      string: '{"app":"HeaderData.js","urllist":"1","max_depth":2,"max_urls":1000}'
-    headers:
-      Content-Type:
-      - application/json
-      User-Agent:
-      - Faraday v0.9.0
-      Accept-Encoding:
-      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
-      Accept:
-      - "*/*"
-  response:
-    status:
-      code: 204
-      message: No Content
-    headers:
-      Content-Length:
-      - '0'
-      Content-Type:
-      - application/json
-      Date:
-      - Tue, 14 Oct 2014 21:39:05 GMT
-      Server:
-      - Cowboy
-      Connection:
-      - keep-alive
-    body:
-      encoding: UTF-8
-      string: ''
-    http_version: 
-  recorded_at: Tue, 14 Oct 2014 21:39:07 GMT
-- request:
-    method: delete
-    uri: https://<TOKEN>:@api.80legs.com/v2/crawls/start_crawl
-    body:
-      encoding: US-ASCII
-      string: ''
-    headers:
-      Content-Type:
-      - application/json
-      User-Agent:
-      - Faraday v0.9.0
-      Accept-Encoding:
-      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
-      Accept:
-      - "*/*"
-  response:
-    status:
-      code: 204
-      message: No Content
-    headers:
-      Content-Length:
-      - '0'
-      Content-Type:
-      - application/json
-      Date:
-      - Tue, 14 Oct 2014 21:39:05 GMT
-      Server:
-      - Cowboy
-      Connection:
-      - keep-alive
-    body:
-      encoding: UTF-8
-      string: ''
-    http_version: 
-  recorded_at: Tue, 14 Oct 2014 21:39:08 GMT
-- request:
-    method: post
-    uri: https://<TOKEN>:@api.80legs.com/v2/crawls/cancel_crawl
-    body:
-      encoding: UTF-8
-      string: '{"app":"HeaderData.js","urllist":"1","max_depth":2,"max_urls":1000}'
-    headers:
-      Content-Type:
-      - application/json
-      User-Agent:
-      - Faraday v0.9.0
-      Accept-Encoding:
-      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
-      Accept:
-      - "*/*"
-  response:
-    status:
-      code: 204
-      message: No Content
-    headers:
-      Content-Length:
-      - '0'
-      Content-Type:
-      - application/json
-      Date:
-      - Tue, 14 Oct 2014 21:39:48 GMT
-      Server:
-      - Cowboy
-      Connection:
-      - keep-alive
-    body:
-      encoding: UTF-8
-      string: ''
-    http_version: 
-  recorded_at: Tue, 14 Oct 2014 21:39:50 GMT
-- request:
-    method: delete
-    uri: https://<TOKEN>:@api.80legs.com/v2/crawls/cancel_crawl
-    body:
-      encoding: US-ASCII
-      string: ''
-    headers:
-      Content-Type:
-      - application/json
-      User-Agent:
-      - Faraday v0.9.0
-      Accept-Encoding:
-      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
-      Accept:
-      - "*/*"
-  response:
-    status:
-      code: 204
-      message: No Content
-    headers:
-      Content-Length:
-      - '0'
-      Content-Type:
-      - application/json
       Date:
-      - Tue, 14 Oct 2014 21:39:48 GMT
+      - Sat, 12 Dec 2015 06:03:59 GMT
       Server:
       - Cowboy
       Connection:
@@ -1357,5 +641,5 @@ http_interactions:
       encoding: UTF-8
       string: ''
     http_version: 
-  recorded_at: Tue, 14 Oct 2014 21:39:50 GMT
-recorded_with: VCR 2.9.3
+  recorded_at: Sat, 12 Dec 2015 06:04:03 GMT
+recorded_with: VCR 3.0.0