creepy-crawler 1.0.0 → 1.0.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +8 -8
- data/.gitignore +2 -1
- data/README.md +1 -1
- data/docker/startup.sh +1 -1
- data/lib/creepy-crawler.rb +7 -0
- metadata +3 -17
checksums.yaml
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
---
|
2
2
|
!binary "U0hBMQ==":
|
3
3
|
metadata.gz: !binary |-
|
4
|
-
|
4
|
+
YzA0Njc0ZTcyMTZjMmVmMGY0ZDljMjFmYmJiM2U1OTY1NGVmMmNkZg==
|
5
5
|
data.tar.gz: !binary |-
|
6
|
-
|
6
|
+
MmY2OTk0OWYzN2MyMGExMWZmZmRiZmY5YzNkNDNmZDQzZGE0OWUwZA==
|
7
7
|
SHA512:
|
8
8
|
metadata.gz: !binary |-
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
ZWFjMjU5NDRkZDRhODMyMTQ1OGEzZjE2N2Y2MDYxMDQ0M2EzYzc5ZWVjYTkx
|
10
|
+
MjAyZTMwZmU1Y2EyZjBlNzdjOTIzOWJmN2YyMGM0YTdiYTljOWRhMDI2NWEw
|
11
|
+
OGE1MmYwNDQ4ZDc3NDQ3ODAwNWJlOTEwM2ZmNTViYTliOGM1MDk=
|
12
12
|
data.tar.gz: !binary |-
|
13
|
-
|
14
|
-
|
15
|
-
|
13
|
+
ZWVkMTg2NDU0ZmI1N2U5NWFhYTM5NjE3Y2JmMmJlZmRhYmJiNmU1MjA5NTM4
|
14
|
+
YWJmYmI3ODU3MTViYjBhMzYwZDBlYmU2Njg3ZmVkZjU1NmU0MjYwMTI1OGI0
|
15
|
+
MzQyZTQ0NWI5MGFmYzI2MjU0ZTA4OGVhYjYzOTMyNmE2YzBlZTk=
|
data/.gitignore
CHANGED
@@ -1 +1,2 @@
|
|
1
|
-
neo4j/
|
1
|
+
neo4j/
|
2
|
+
*.gem
|
data/README.md
CHANGED
@@ -8,7 +8,7 @@ Ruby web crawler that takes a url as input and produces a sitemap using a neo4j
|
|
8
8
|
|
9
9
|
##Installation
|
10
10
|
####Clone
|
11
|
-
git clone https://github.com/udryan10/creepy-crawler.git
|
11
|
+
git clone https://github.com/udryan10/creepy-crawler.git && cd creepy-crawler
|
12
12
|
####Install Required Gems
|
13
13
|
bundle install
|
14
14
|
####Install graph database
|
data/docker/startup.sh
CHANGED
data/lib/creepy-crawler.rb
CHANGED
@@ -42,6 +42,13 @@ end
|
|
42
42
|
|
43
43
|
# allow the initiating of a crawl from command line
|
44
44
|
if __FILE__==$0
|
45
|
+
|
46
|
+
# Exit cleanly from an early interrupt
|
47
|
+
Signal.trap("INT") {
|
48
|
+
puts "Received interrupt. Stopping crawl"
|
49
|
+
exit 1
|
50
|
+
}
|
51
|
+
|
45
52
|
# setup options
|
46
53
|
opts = Trollop::options do
|
47
54
|
opt :site, "Url of site to crawl", :type => :string # flag --site
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: creepy-crawler
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ryan Grothouse
|
@@ -17,7 +17,7 @@ dependencies:
|
|
17
17
|
- - ~>
|
18
18
|
- !ruby/object:Gem::Version
|
19
19
|
version: '1.5'
|
20
|
-
type: :
|
20
|
+
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
@@ -31,21 +31,7 @@ dependencies:
|
|
31
31
|
- - ~>
|
32
32
|
- !ruby/object:Gem::Version
|
33
33
|
version: '2.14'
|
34
|
-
type: :
|
35
|
-
prerelease: false
|
36
|
-
version_requirements: !ruby/object:Gem::Requirement
|
37
|
-
requirements:
|
38
|
-
- - ~>
|
39
|
-
- !ruby/object:Gem::Version
|
40
|
-
version: '2.14'
|
41
|
-
- !ruby/object:Gem::Dependency
|
42
|
-
name: rspec-core
|
43
|
-
requirement: !ruby/object:Gem::Requirement
|
44
|
-
requirements:
|
45
|
-
- - ~>
|
46
|
-
- !ruby/object:Gem::Version
|
47
|
-
version: '2.14'
|
48
|
-
type: :development
|
34
|
+
type: :runtime
|
49
35
|
prerelease: false
|
50
36
|
version_requirements: !ruby/object:Gem::Requirement
|
51
37
|
requirements:
|