leggy 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rspec +3 -0
- data/Guardfile +33 -0
- data/leggy-0.1.0.gem +0 -0
- data/lib/leggy.rb +16 -22
- data/lib/leggy/app.rb +13 -0
- data/lib/leggy/crawl.rb +23 -0
- data/lib/leggy/crawl_options.rb +18 -0
- data/lib/leggy/helpers.rb +40 -0
- data/lib/leggy/mapping/app.rb +18 -0
- data/lib/leggy/mapping/crawl.rb +13 -0
- data/lib/leggy/mapping/crawl_options.rb +13 -0
- data/lib/leggy/mapping/url.rb +14 -0
- data/lib/leggy/mapping/user.rb +14 -0
- data/lib/leggy/resource/app.rb +55 -0
- data/lib/leggy/resource/crawl.rb +49 -0
- data/lib/leggy/resource/url.rb +55 -0
- data/lib/leggy/resource/user.rb +23 -0
- data/lib/leggy/url.rb +13 -0
- data/lib/leggy/user.rb +19 -0
- data/lib/leggy/version.rb +1 -1
- data/spec/cassettes/leggy.yml +1361 -0
- data/spec/fixtures/sample_app.js +53 -0
- data/spec/fixtures/urls.json +4 -0
- data/spec/leggy/leggy_spec.rb +177 -0
- data/spec/spec_helper.rb +45 -0
- metadata +32 -5
@@ -0,0 +1,53 @@
|
|
1
|
+
// This 80app returns the header data from each URL crawled
|
2
|
+
|
3
|
+
var EightyApp = function() {
|
4
|
+
this.processDocument = function(html, url, headers, status, jQuery) {
|
5
|
+
var app = this;
|
6
|
+
var $ = jQuery;
|
7
|
+
var $html = app.parseHtml(html, $);
|
8
|
+
var object = {};
|
9
|
+
|
10
|
+
if(typeof headers == 'string' || headers instanceof String) {
|
11
|
+
var headersArray = headers.split("\r\n");
|
12
|
+
for (var i = 0; i < headersArray.length; i++) {
|
13
|
+
var keyvalArray = headersArray[i].split(": ");
|
14
|
+
var key = keyvalArray[0];
|
15
|
+
var value = keyvalArray[1];
|
16
|
+
object[key] = value;
|
17
|
+
}
|
18
|
+
|
19
|
+
return JSON.stringify(object);
|
20
|
+
}
|
21
|
+
|
22
|
+
return JSON.stringify(headers);
|
23
|
+
}
|
24
|
+
|
25
|
+
this.parseLinks = function(html, url, headers, status, jQuery) {
|
26
|
+
var app = this;
|
27
|
+
var $ = jQuery;
|
28
|
+
var $html = app.parseHtml(html, $);
|
29
|
+
var links = [];
|
30
|
+
|
31
|
+
// gets all links in the html document
|
32
|
+
$html.find('a').each(function(i, obj) {
|
33
|
+
var link = app.makeLink(url, $(this).attr('href'));
|
34
|
+
if(link != null) {
|
35
|
+
links.push(link);
|
36
|
+
}
|
37
|
+
});
|
38
|
+
|
39
|
+
return links;
|
40
|
+
}
|
41
|
+
}
|
42
|
+
|
43
|
+
try {
|
44
|
+
// Testing
|
45
|
+
module.exports = function(EightyAppBase) {
|
46
|
+
EightyApp.prototype = new EightyAppBase();
|
47
|
+
return new EightyApp();
|
48
|
+
}
|
49
|
+
} catch(e) {
|
50
|
+
// Production
|
51
|
+
console.log("Eighty app exists.");
|
52
|
+
EightyApp.prototype = new EightyAppBase();
|
53
|
+
}
|
@@ -0,0 +1,177 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Leggy, vcr: { cassette_name: 'leggy' } do
|
4
|
+
|
5
|
+
before { Leggy.configuration.reset }
|
6
|
+
|
7
|
+
it "must be defined" do
|
8
|
+
expect(Leggy::VERSION).not_to be_nil
|
9
|
+
end
|
10
|
+
|
11
|
+
it "encodes api tokens" do
|
12
|
+
Leggy.configuration.api_token = '1234'
|
13
|
+
expect(Leggy.api_token).to eq "MTIzNDo=\n"
|
14
|
+
end
|
15
|
+
|
16
|
+
context "Users" do
|
17
|
+
|
18
|
+
context "#find" do
|
19
|
+
|
20
|
+
it "returns a single user" do
|
21
|
+
user = Leggy.users.find(api_token: Leggy.configuration.api_token)
|
22
|
+
expect(user.first_name).to eq 'Matt'
|
23
|
+
end
|
24
|
+
|
25
|
+
end
|
26
|
+
|
27
|
+
end
|
28
|
+
|
29
|
+
context "Apps" do
|
30
|
+
|
31
|
+
context "#all" do
|
32
|
+
|
33
|
+
before(:each) { Leggy.apps.create(name: 'sample_all', body: fixture('sample_app.js')) }
|
34
|
+
after(:each) { Leggy.apps.delete(name: 'sample_all') }
|
35
|
+
|
36
|
+
it "returns a list of all apps" do
|
37
|
+
apps = Leggy.apps.all
|
38
|
+
expect(apps.collect(&:name)).to include 'sample_all'
|
39
|
+
end
|
40
|
+
|
41
|
+
end
|
42
|
+
|
43
|
+
context "#create" do
|
44
|
+
|
45
|
+
after(:each) { Leggy.apps.delete(name: 'sample') }
|
46
|
+
|
47
|
+
it "uploads a new app" do
|
48
|
+
app = Leggy.apps.create(name: 'sample', body: fixture('sample_app.js'))
|
49
|
+
expect(app).to be_truthy
|
50
|
+
end
|
51
|
+
|
52
|
+
end
|
53
|
+
|
54
|
+
context "#find" do
|
55
|
+
|
56
|
+
before(:each) { Leggy.apps.create(name: 'sample_find', body: fixture('sample_app.js')) }
|
57
|
+
after(:each) { Leggy.apps.delete(name: 'sample_find') }
|
58
|
+
|
59
|
+
it "returns a single app" do
|
60
|
+
app = Leggy.apps.find(name: 'sample_find')
|
61
|
+
expect(app).to match /\A\/\//
|
62
|
+
end
|
63
|
+
|
64
|
+
end
|
65
|
+
|
66
|
+
context "#delete" do
|
67
|
+
|
68
|
+
before(:each) { Leggy.apps.create(name: 'sample_delete', body: fixture('sample_app.js')) }
|
69
|
+
|
70
|
+
it "removes an app" do
|
71
|
+
app = Leggy.apps.delete(name: 'sample_delete')
|
72
|
+
expect(app).to be_truthy
|
73
|
+
end
|
74
|
+
|
75
|
+
end
|
76
|
+
|
77
|
+
end
|
78
|
+
|
79
|
+
context "Urls" do
|
80
|
+
|
81
|
+
context "#all" do
|
82
|
+
|
83
|
+
before(:each) { Leggy.urls.create(name: 'sample_all', body: fixture('urls.json')) }
|
84
|
+
after(:each) { Leggy.urls.delete(name: 'sample_all') }
|
85
|
+
|
86
|
+
it "returns a list of all url lists" do
|
87
|
+
urls = Leggy.urls.all
|
88
|
+
expect(urls.collect(&:name)).to include 'sample_all'
|
89
|
+
end
|
90
|
+
|
91
|
+
end
|
92
|
+
|
93
|
+
context "#create" do
|
94
|
+
|
95
|
+
after(:each) { Leggy.urls.delete(name: 'sample') }
|
96
|
+
|
97
|
+
it "uploads a new url list" do
|
98
|
+
url = Leggy.urls.create(name: 'sample', body: fixture('urls.json'))
|
99
|
+
expect(url).to be_truthy
|
100
|
+
end
|
101
|
+
|
102
|
+
end
|
103
|
+
|
104
|
+
context "#find" do
|
105
|
+
|
106
|
+
before(:each) { Leggy.urls.create(name: 'sample_find', body: fixture('urls.json')) }
|
107
|
+
after(:each) { Leggy.urls.delete(name: 'sample_find') }
|
108
|
+
|
109
|
+
it "returns a single url list" do
|
110
|
+
url = Leggy.urls.find(name: 'sample_find')
|
111
|
+
expect(url).to include 'http://example.com'
|
112
|
+
end
|
113
|
+
|
114
|
+
end
|
115
|
+
|
116
|
+
context "#delete" do
|
117
|
+
|
118
|
+
before(:each) { Leggy.urls.create(name: 'sample_delete', body: fixture('urls.json')) }
|
119
|
+
|
120
|
+
it "removes an url list" do
|
121
|
+
url = Leggy.urls.delete(name: 'sample_delete')
|
122
|
+
expect(url).to be_truthy
|
123
|
+
end
|
124
|
+
|
125
|
+
end
|
126
|
+
|
127
|
+
end
|
128
|
+
|
129
|
+
context "Urls" do
|
130
|
+
|
131
|
+
let(:options) do
|
132
|
+
{
|
133
|
+
name: "start_crawl",
|
134
|
+
app: "HeaderData.js",
|
135
|
+
urllist: "1",
|
136
|
+
max_depth: 2,
|
137
|
+
max_urls: 1000
|
138
|
+
}
|
139
|
+
end
|
140
|
+
|
141
|
+
context "#start" do
|
142
|
+
|
143
|
+
after(:each) { Leggy.crawls.cancel(name: "start_crawl") }
|
144
|
+
|
145
|
+
it "starts a new crawl" do
|
146
|
+
crawl = Leggy.crawls.start(options)
|
147
|
+
expect(crawl).to be_truthy
|
148
|
+
end
|
149
|
+
|
150
|
+
end
|
151
|
+
|
152
|
+
context "#status" do
|
153
|
+
|
154
|
+
before(:each) { Leggy.crawls.start(options.merge(name: "status_crawl")) }
|
155
|
+
after(:each) { Leggy.crawls.cancel(name: "status_crawl") }
|
156
|
+
|
157
|
+
it "uploads a new url list" do
|
158
|
+
crawl = Leggy.crawls.status(name: "status_crawl")
|
159
|
+
expect(crawl.status).to eq 'QUEUED'
|
160
|
+
end
|
161
|
+
|
162
|
+
end
|
163
|
+
|
164
|
+
context "#cancel" do
|
165
|
+
|
166
|
+
before(:each) { Leggy.crawls.start(options.merge(name: "cancel_crawl")) }
|
167
|
+
|
168
|
+
it "returns a single url list" do
|
169
|
+
crawl = Leggy.crawls.cancel(name: "cancel_crawl")
|
170
|
+
expect(crawl).to be_truthy
|
171
|
+
end
|
172
|
+
|
173
|
+
end
|
174
|
+
|
175
|
+
end
|
176
|
+
|
177
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,45 @@
|
|
1
|
+
require "rspec"
|
2
|
+
require "webmock/rspec"
|
3
|
+
require "vcr"
|
4
|
+
require "faraday"
|
5
|
+
|
6
|
+
begin
|
7
|
+
require "pry"
|
8
|
+
rescue LoadError; end
|
9
|
+
|
10
|
+
require "leggy"
|
11
|
+
|
12
|
+
# This file was generated by the `rspec --init` command. Conventionally, all
|
13
|
+
# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
|
14
|
+
# Require this file using `require "spec_helper"` to ensure that it is only
|
15
|
+
# loaded once.
|
16
|
+
#
|
17
|
+
# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
|
18
|
+
RSpec.configure do |config|
|
19
|
+
config.run_all_when_everything_filtered = true
|
20
|
+
config.filter_run :focus
|
21
|
+
|
22
|
+
# Run specs in random order to surface order dependencies. If you find an
|
23
|
+
# order dependency and want to debug it, you can fix the order by providing
|
24
|
+
# the seed, which is printed after each run.
|
25
|
+
# --seed 1234
|
26
|
+
config.order = 'random'
|
27
|
+
end
|
28
|
+
|
29
|
+
VCR.configure do |c|
|
30
|
+
c.ignore_localhost = true
|
31
|
+
c.cassette_library_dir = 'spec/cassettes'
|
32
|
+
c.hook_into :webmock #:typhoeus, :faraday, :fakeweb, or :webmock
|
33
|
+
c.default_cassette_options = { :record => :new_episodes }
|
34
|
+
c.configure_rspec_metadata!
|
35
|
+
c.filter_sensitive_data('<TOKEN>') { Leggy.configuration.api_token }
|
36
|
+
c.filter_sensitive_data('<BASE64_TOKEN>') { Leggy.api_token }
|
37
|
+
end
|
38
|
+
|
39
|
+
def body(url)
|
40
|
+
Faraday.get(url).body
|
41
|
+
end
|
42
|
+
|
43
|
+
def fixture(name)
|
44
|
+
File.dirname(__FILE__) + '/fixtures/' + name
|
45
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: leggy
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Matt Solt
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-
|
11
|
+
date: 2014-12-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -214,13 +214,36 @@ extensions: []
|
|
214
214
|
extra_rdoc_files: []
|
215
215
|
files:
|
216
216
|
- ".gitignore"
|
217
|
+
- ".rspec"
|
217
218
|
- Gemfile
|
219
|
+
- Guardfile
|
218
220
|
- LICENSE.txt
|
219
221
|
- README.md
|
220
222
|
- Rakefile
|
223
|
+
- leggy-0.1.0.gem
|
221
224
|
- leggy.gemspec
|
222
225
|
- lib/leggy.rb
|
226
|
+
- lib/leggy/app.rb
|
227
|
+
- lib/leggy/crawl.rb
|
228
|
+
- lib/leggy/crawl_options.rb
|
229
|
+
- lib/leggy/helpers.rb
|
230
|
+
- lib/leggy/mapping/app.rb
|
231
|
+
- lib/leggy/mapping/crawl.rb
|
232
|
+
- lib/leggy/mapping/crawl_options.rb
|
233
|
+
- lib/leggy/mapping/url.rb
|
234
|
+
- lib/leggy/mapping/user.rb
|
235
|
+
- lib/leggy/resource/app.rb
|
236
|
+
- lib/leggy/resource/crawl.rb
|
237
|
+
- lib/leggy/resource/url.rb
|
238
|
+
- lib/leggy/resource/user.rb
|
239
|
+
- lib/leggy/url.rb
|
240
|
+
- lib/leggy/user.rb
|
223
241
|
- lib/leggy/version.rb
|
242
|
+
- spec/cassettes/leggy.yml
|
243
|
+
- spec/fixtures/sample_app.js
|
244
|
+
- spec/fixtures/urls.json
|
245
|
+
- spec/leggy/leggy_spec.rb
|
246
|
+
- spec/spec_helper.rb
|
224
247
|
homepage: https://github.com/activefx/leggy
|
225
248
|
licenses:
|
226
249
|
- MIT
|
@@ -241,9 +264,13 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
241
264
|
version: '0'
|
242
265
|
requirements: []
|
243
266
|
rubyforge_project:
|
244
|
-
rubygems_version: 2.
|
267
|
+
rubygems_version: 2.4.3
|
245
268
|
signing_key:
|
246
269
|
specification_version: 4
|
247
270
|
summary: Ruby wrapper for the 80Legs API
|
248
|
-
test_files:
|
249
|
-
|
271
|
+
test_files:
|
272
|
+
- spec/cassettes/leggy.yml
|
273
|
+
- spec/fixtures/sample_app.js
|
274
|
+
- spec/fixtures/urls.json
|
275
|
+
- spec/leggy/leggy_spec.rb
|
276
|
+
- spec/spec_helper.rb
|