leggy 0.1.0 → 0.1.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rspec +3 -0
- data/Guardfile +33 -0
- data/leggy-0.1.0.gem +0 -0
- data/lib/leggy.rb +16 -22
- data/lib/leggy/app.rb +13 -0
- data/lib/leggy/crawl.rb +23 -0
- data/lib/leggy/crawl_options.rb +18 -0
- data/lib/leggy/helpers.rb +40 -0
- data/lib/leggy/mapping/app.rb +18 -0
- data/lib/leggy/mapping/crawl.rb +13 -0
- data/lib/leggy/mapping/crawl_options.rb +13 -0
- data/lib/leggy/mapping/url.rb +14 -0
- data/lib/leggy/mapping/user.rb +14 -0
- data/lib/leggy/resource/app.rb +55 -0
- data/lib/leggy/resource/crawl.rb +49 -0
- data/lib/leggy/resource/url.rb +55 -0
- data/lib/leggy/resource/user.rb +23 -0
- data/lib/leggy/url.rb +13 -0
- data/lib/leggy/user.rb +19 -0
- data/lib/leggy/version.rb +1 -1
- data/spec/cassettes/leggy.yml +1361 -0
- data/spec/fixtures/sample_app.js +53 -0
- data/spec/fixtures/urls.json +4 -0
- data/spec/leggy/leggy_spec.rb +177 -0
- data/spec/spec_helper.rb +45 -0
- metadata +32 -5
@@ -0,0 +1,53 @@
|
|
1
|
+
// This 80app returns the header data from each URL crawled
|
2
|
+
|
3
|
+
var EightyApp = function() {
|
4
|
+
this.processDocument = function(html, url, headers, status, jQuery) {
|
5
|
+
var app = this;
|
6
|
+
var $ = jQuery;
|
7
|
+
var $html = app.parseHtml(html, $);
|
8
|
+
var object = {};
|
9
|
+
|
10
|
+
if(typeof headers == 'string' || headers instanceof String) {
|
11
|
+
var headersArray = headers.split("\r\n");
|
12
|
+
for (var i = 0; i < headersArray.length; i++) {
|
13
|
+
var keyvalArray = headersArray[i].split(": ");
|
14
|
+
var key = keyvalArray[0];
|
15
|
+
var value = keyvalArray[1];
|
16
|
+
object[key] = value;
|
17
|
+
}
|
18
|
+
|
19
|
+
return JSON.stringify(object);
|
20
|
+
}
|
21
|
+
|
22
|
+
return JSON.stringify(headers);
|
23
|
+
}
|
24
|
+
|
25
|
+
this.parseLinks = function(html, url, headers, status, jQuery) {
|
26
|
+
var app = this;
|
27
|
+
var $ = jQuery;
|
28
|
+
var $html = app.parseHtml(html, $);
|
29
|
+
var links = [];
|
30
|
+
|
31
|
+
// gets all links in the html document
|
32
|
+
$html.find('a').each(function(i, obj) {
|
33
|
+
var link = app.makeLink(url, $(this).attr('href'));
|
34
|
+
if(link != null) {
|
35
|
+
links.push(link);
|
36
|
+
}
|
37
|
+
});
|
38
|
+
|
39
|
+
return links;
|
40
|
+
}
|
41
|
+
}
|
42
|
+
|
43
|
+
try {
|
44
|
+
// Testing
|
45
|
+
module.exports = function(EightyAppBase) {
|
46
|
+
EightyApp.prototype = new EightyAppBase();
|
47
|
+
return new EightyApp();
|
48
|
+
}
|
49
|
+
} catch(e) {
|
50
|
+
// Production
|
51
|
+
console.log("Eighty app exists.");
|
52
|
+
EightyApp.prototype = new EightyAppBase();
|
53
|
+
}
|
@@ -0,0 +1,177 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Leggy, vcr: { cassette_name: 'leggy' } do
|
4
|
+
|
5
|
+
before { Leggy.configuration.reset }
|
6
|
+
|
7
|
+
it "must be defined" do
|
8
|
+
expect(Leggy::VERSION).not_to be_nil
|
9
|
+
end
|
10
|
+
|
11
|
+
it "encodes api tokens" do
|
12
|
+
Leggy.configuration.api_token = '1234'
|
13
|
+
expect(Leggy.api_token).to eq "MTIzNDo=\n"
|
14
|
+
end
|
15
|
+
|
16
|
+
context "Users" do
|
17
|
+
|
18
|
+
context "#find" do
|
19
|
+
|
20
|
+
it "returns a single user" do
|
21
|
+
user = Leggy.users.find(api_token: Leggy.configuration.api_token)
|
22
|
+
expect(user.first_name).to eq 'Matt'
|
23
|
+
end
|
24
|
+
|
25
|
+
end
|
26
|
+
|
27
|
+
end
|
28
|
+
|
29
|
+
context "Apps" do
|
30
|
+
|
31
|
+
context "#all" do
|
32
|
+
|
33
|
+
before(:each) { Leggy.apps.create(name: 'sample_all', body: fixture('sample_app.js')) }
|
34
|
+
after(:each) { Leggy.apps.delete(name: 'sample_all') }
|
35
|
+
|
36
|
+
it "returns a list of all apps" do
|
37
|
+
apps = Leggy.apps.all
|
38
|
+
expect(apps.collect(&:name)).to include 'sample_all'
|
39
|
+
end
|
40
|
+
|
41
|
+
end
|
42
|
+
|
43
|
+
context "#create" do
|
44
|
+
|
45
|
+
after(:each) { Leggy.apps.delete(name: 'sample') }
|
46
|
+
|
47
|
+
it "uploads a new app" do
|
48
|
+
app = Leggy.apps.create(name: 'sample', body: fixture('sample_app.js'))
|
49
|
+
expect(app).to be_truthy
|
50
|
+
end
|
51
|
+
|
52
|
+
end
|
53
|
+
|
54
|
+
context "#find" do
|
55
|
+
|
56
|
+
before(:each) { Leggy.apps.create(name: 'sample_find', body: fixture('sample_app.js')) }
|
57
|
+
after(:each) { Leggy.apps.delete(name: 'sample_find') }
|
58
|
+
|
59
|
+
it "returns a single app" do
|
60
|
+
app = Leggy.apps.find(name: 'sample_find')
|
61
|
+
expect(app).to match /\A\/\//
|
62
|
+
end
|
63
|
+
|
64
|
+
end
|
65
|
+
|
66
|
+
context "#delete" do
|
67
|
+
|
68
|
+
before(:each) { Leggy.apps.create(name: 'sample_delete', body: fixture('sample_app.js')) }
|
69
|
+
|
70
|
+
it "removes an app" do
|
71
|
+
app = Leggy.apps.delete(name: 'sample_delete')
|
72
|
+
expect(app).to be_truthy
|
73
|
+
end
|
74
|
+
|
75
|
+
end
|
76
|
+
|
77
|
+
end
|
78
|
+
|
79
|
+
context "Urls" do
|
80
|
+
|
81
|
+
context "#all" do
|
82
|
+
|
83
|
+
before(:each) { Leggy.urls.create(name: 'sample_all', body: fixture('urls.json')) }
|
84
|
+
after(:each) { Leggy.urls.delete(name: 'sample_all') }
|
85
|
+
|
86
|
+
it "returns a list of all url lists" do
|
87
|
+
urls = Leggy.urls.all
|
88
|
+
expect(urls.collect(&:name)).to include 'sample_all'
|
89
|
+
end
|
90
|
+
|
91
|
+
end
|
92
|
+
|
93
|
+
context "#create" do
|
94
|
+
|
95
|
+
after(:each) { Leggy.urls.delete(name: 'sample') }
|
96
|
+
|
97
|
+
it "uploads a new url list" do
|
98
|
+
url = Leggy.urls.create(name: 'sample', body: fixture('urls.json'))
|
99
|
+
expect(url).to be_truthy
|
100
|
+
end
|
101
|
+
|
102
|
+
end
|
103
|
+
|
104
|
+
context "#find" do
|
105
|
+
|
106
|
+
before(:each) { Leggy.urls.create(name: 'sample_find', body: fixture('urls.json')) }
|
107
|
+
after(:each) { Leggy.urls.delete(name: 'sample_find') }
|
108
|
+
|
109
|
+
it "returns a single url list" do
|
110
|
+
url = Leggy.urls.find(name: 'sample_find')
|
111
|
+
expect(url).to include 'http://example.com'
|
112
|
+
end
|
113
|
+
|
114
|
+
end
|
115
|
+
|
116
|
+
context "#delete" do
|
117
|
+
|
118
|
+
before(:each) { Leggy.urls.create(name: 'sample_delete', body: fixture('urls.json')) }
|
119
|
+
|
120
|
+
it "removes an url list" do
|
121
|
+
url = Leggy.urls.delete(name: 'sample_delete')
|
122
|
+
expect(url).to be_truthy
|
123
|
+
end
|
124
|
+
|
125
|
+
end
|
126
|
+
|
127
|
+
end
|
128
|
+
|
129
|
+
context "Urls" do
|
130
|
+
|
131
|
+
let(:options) do
|
132
|
+
{
|
133
|
+
name: "start_crawl",
|
134
|
+
app: "HeaderData.js",
|
135
|
+
urllist: "1",
|
136
|
+
max_depth: 2,
|
137
|
+
max_urls: 1000
|
138
|
+
}
|
139
|
+
end
|
140
|
+
|
141
|
+
context "#start" do
|
142
|
+
|
143
|
+
after(:each) { Leggy.crawls.cancel(name: "start_crawl") }
|
144
|
+
|
145
|
+
it "starts a new crawl" do
|
146
|
+
crawl = Leggy.crawls.start(options)
|
147
|
+
expect(crawl).to be_truthy
|
148
|
+
end
|
149
|
+
|
150
|
+
end
|
151
|
+
|
152
|
+
context "#status" do
|
153
|
+
|
154
|
+
before(:each) { Leggy.crawls.start(options.merge(name: "status_crawl")) }
|
155
|
+
after(:each) { Leggy.crawls.cancel(name: "status_crawl") }
|
156
|
+
|
157
|
+
it "uploads a new url list" do
|
158
|
+
crawl = Leggy.crawls.status(name: "status_crawl")
|
159
|
+
expect(crawl.status).to eq 'QUEUED'
|
160
|
+
end
|
161
|
+
|
162
|
+
end
|
163
|
+
|
164
|
+
context "#cancel" do
|
165
|
+
|
166
|
+
before(:each) { Leggy.crawls.start(options.merge(name: "cancel_crawl")) }
|
167
|
+
|
168
|
+
it "returns a single url list" do
|
169
|
+
crawl = Leggy.crawls.cancel(name: "cancel_crawl")
|
170
|
+
expect(crawl).to be_truthy
|
171
|
+
end
|
172
|
+
|
173
|
+
end
|
174
|
+
|
175
|
+
end
|
176
|
+
|
177
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,45 @@
|
|
1
|
+
require "rspec"
|
2
|
+
require "webmock/rspec"
|
3
|
+
require "vcr"
|
4
|
+
require "faraday"
|
5
|
+
|
6
|
+
begin
|
7
|
+
require "pry"
|
8
|
+
rescue LoadError; end
|
9
|
+
|
10
|
+
require "leggy"
|
11
|
+
|
12
|
+
# This file was generated by the `rspec --init` command. Conventionally, all
|
13
|
+
# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
|
14
|
+
# Require this file using `require "spec_helper"` to ensure that it is only
|
15
|
+
# loaded once.
|
16
|
+
#
|
17
|
+
# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
|
18
|
+
RSpec.configure do |config|
|
19
|
+
config.run_all_when_everything_filtered = true
|
20
|
+
config.filter_run :focus
|
21
|
+
|
22
|
+
# Run specs in random order to surface order dependencies. If you find an
|
23
|
+
# order dependency and want to debug it, you can fix the order by providing
|
24
|
+
# the seed, which is printed after each run.
|
25
|
+
# --seed 1234
|
26
|
+
config.order = 'random'
|
27
|
+
end
|
28
|
+
|
29
|
+
VCR.configure do |c|
|
30
|
+
c.ignore_localhost = true
|
31
|
+
c.cassette_library_dir = 'spec/cassettes'
|
32
|
+
c.hook_into :webmock #:typhoeus, :faraday, :fakeweb, or :webmock
|
33
|
+
c.default_cassette_options = { :record => :new_episodes }
|
34
|
+
c.configure_rspec_metadata!
|
35
|
+
c.filter_sensitive_data('<TOKEN>') { Leggy.configuration.api_token }
|
36
|
+
c.filter_sensitive_data('<BASE64_TOKEN>') { Leggy.api_token }
|
37
|
+
end
|
38
|
+
|
39
|
+
def body(url)
|
40
|
+
Faraday.get(url).body
|
41
|
+
end
|
42
|
+
|
43
|
+
def fixture(name)
|
44
|
+
File.dirname(__FILE__) + '/fixtures/' + name
|
45
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: leggy
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Matt Solt
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-
|
11
|
+
date: 2014-12-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -214,13 +214,36 @@ extensions: []
|
|
214
214
|
extra_rdoc_files: []
|
215
215
|
files:
|
216
216
|
- ".gitignore"
|
217
|
+
- ".rspec"
|
217
218
|
- Gemfile
|
219
|
+
- Guardfile
|
218
220
|
- LICENSE.txt
|
219
221
|
- README.md
|
220
222
|
- Rakefile
|
223
|
+
- leggy-0.1.0.gem
|
221
224
|
- leggy.gemspec
|
222
225
|
- lib/leggy.rb
|
226
|
+
- lib/leggy/app.rb
|
227
|
+
- lib/leggy/crawl.rb
|
228
|
+
- lib/leggy/crawl_options.rb
|
229
|
+
- lib/leggy/helpers.rb
|
230
|
+
- lib/leggy/mapping/app.rb
|
231
|
+
- lib/leggy/mapping/crawl.rb
|
232
|
+
- lib/leggy/mapping/crawl_options.rb
|
233
|
+
- lib/leggy/mapping/url.rb
|
234
|
+
- lib/leggy/mapping/user.rb
|
235
|
+
- lib/leggy/resource/app.rb
|
236
|
+
- lib/leggy/resource/crawl.rb
|
237
|
+
- lib/leggy/resource/url.rb
|
238
|
+
- lib/leggy/resource/user.rb
|
239
|
+
- lib/leggy/url.rb
|
240
|
+
- lib/leggy/user.rb
|
223
241
|
- lib/leggy/version.rb
|
242
|
+
- spec/cassettes/leggy.yml
|
243
|
+
- spec/fixtures/sample_app.js
|
244
|
+
- spec/fixtures/urls.json
|
245
|
+
- spec/leggy/leggy_spec.rb
|
246
|
+
- spec/spec_helper.rb
|
224
247
|
homepage: https://github.com/activefx/leggy
|
225
248
|
licenses:
|
226
249
|
- MIT
|
@@ -241,9 +264,13 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
241
264
|
version: '0'
|
242
265
|
requirements: []
|
243
266
|
rubyforge_project:
|
244
|
-
rubygems_version: 2.
|
267
|
+
rubygems_version: 2.4.3
|
245
268
|
signing_key:
|
246
269
|
specification_version: 4
|
247
270
|
summary: Ruby wrapper for the 80Legs API
|
248
|
-
test_files:
|
249
|
-
|
271
|
+
test_files:
|
272
|
+
- spec/cassettes/leggy.yml
|
273
|
+
- spec/fixtures/sample_app.js
|
274
|
+
- spec/fixtures/urls.json
|
275
|
+
- spec/leggy/leggy_spec.rb
|
276
|
+
- spec/spec_helper.rb
|