gottabeafraid 0.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (26) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +39 -0
  3. data/DEVELOPING.md +171 -0
  4. data/Gemfile +4 -0
  5. data/Gemfile.lock +109 -0
  6. data/LICENSE +22 -0
  7. data/README.md +65 -0
  8. data/Rakefile +18 -0
  9. data/bin/gonna +13 -0
  10. data/build.sh +16 -0
  11. data/db/config.yml +5 -0
  12. data/db/migrate/20140101180038_initial_migration.rb +18 -0
  13. data/db/schema.rb +26 -0
  14. data/gottabeafraid.gemspec +31 -0
  15. data/lib/gottabeafraid.rb +53 -0
  16. data/lib/gottabeafraid/Model/afraid.rb +9 -0
  17. data/lib/gottabeafraid/cli.rb +82 -0
  18. data/lib/gottabeafraid/gonna.rb +101 -0
  19. data/lib/gottabeafraid/version.rb +3 -0
  20. data/lib/mechanize/agent.rb +15 -0
  21. data/test/mockup/app.rb +6 -0
  22. data/test/mockup/config.ru +2 -0
  23. data/test/mockup/public/page-2.html +1121 -0
  24. data/test/mockup/public/page-2_files/exclaim.gif +0 -0
  25. data/test/mockup/public/page-2_files/powerani.gif +0 -0
  26. metadata +199 -0
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 7726295ad7cd64fdbeaccf582d5fd16e400959b3
4
+ data.tar.gz: d9152e691f64d472c92c17be56b16fd2b6dd6946
5
+ SHA512:
6
+ metadata.gz: eecb736dbc2bfdb7755c2d9307b227903b9c1fc53ab702d21d8eaf04a228115670333d915f881ae6bcd48289ac0394b4e102b0dde933e584a1db3310256d3d23
7
+ data.tar.gz: 19bc6d5df9b7583095514cf0ae9438ad61f2262fb5b1a7bc21d8d01a63bc4cb7d554b1c741d5c217830e2a8e04dc00b71ad909b22af97828f2921b2d21ede591
data/.gitignore ADDED
@@ -0,0 +1,39 @@
1
+ *.gem
2
+ *.rbc
3
+ /.config
4
+ /coverage/
5
+ /InstalledFiles
6
+ /pkg/
7
+ /spec/reports/
8
+ /test/tmp/
9
+ /test/version_tmp/
10
+ /tmp/
11
+ /log/
12
+
13
+ ## Specific to RubyMotion:
14
+ .dat*
15
+ .repl_history
16
+ build/
17
+
18
+ ## Documentation cache and generated files:
19
+ /.yardoc/
20
+ /_yardoc/
21
+ /doc/
22
+ /rdoc/
23
+
24
+ ## Environment normalisation:
25
+ /.bundle/
26
+ /lib/bundler/man/
27
+
28
+ # for a library or gem, you might want to ignore these files since the code is
29
+ # intended to run in multiple environments; otherwise, check them in:
30
+ # Gemfile.lock
31
+ # .ruby-version
32
+ # .ruby-gemset
33
+
34
+ # unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
35
+ .rvmrc
36
+
37
+ db/*.sqlite3
38
+ db/*.csv
39
+ notes_*.*
data/DEVELOPING.md ADDED
@@ -0,0 +1,171 @@
1
+ # Gotta Be Afraid
2
+
3
+ Basic concepts of scraping and mechanizing a website through the Tor network.
4
+
5
+
6
+ ## Login Process
7
+ * https://freedns.afraid.org/zc.php
8
+ * https://freedns.afraid.org/domain/registry/page-?.html
9
+ * https://freedns.afraid.org/logout/
10
+
11
+ ```html
12
+ <input type="text" name="username" size="16" maxlength="16">
13
+ <input type="password" name="password" size="16" maxlength="16">
14
+
15
+ <input id="oPersistInput" class="storeuserData" type="hidden" name="remote">
16
+ <input type="hidden" value="" name="from">
17
+ <input type="hidden" value="auth" name="action">
18
+
19
+ <input type="submit" onclick="fnAlertInput()" value="Login" name="submit">
20
+ ```
21
+
22
+
23
+
24
+ ## Cookie Information
25
+
26
+ * dns_cookie appears to be the primary cookie
27
+ * PHPSESSID was servered at somepoint as the cookie._
28
+
29
+
30
+ ## Scrape Target
31
+
32
+
33
+ * GET https://freedns.afraid.org/domain/registry/page-?.html
34
+
35
+ ```html
36
+ <tr class="trl">
37
+ <td>
38
+ <a href="/subdomain/edit.php?edit_domain_id=29">mooo.com</a>
39
+ <br>
40
+ <span>
41
+ (176366 hosts in use)
42
+ <a href="http://www.mooo.com/" rel="nofollow" target="_blank">website</a>
43
+ </span>
44
+ </td>
45
+ <td>public</td>
46
+ <td>
47
+ <a href="/tools/contact.php?user_id=1&subject=mooo.com">josh</a>
48
+ </td>
49
+ <td>4275 days ago (03/15/2001)</td>
50
+
51
+
52
+
53
+ <html lang="EN">
54
+ <head>
55
+ <body vlink="#808080" text="#000000" link="#000000" bgcolor="cornflowerblue" alink="#000000">
56
+ <table width="100%" height="100%" cellspacing="5" border="0">
57
+ <tbody>
58
+ <tr>
59
+ <tr>
60
+ <td width="15%" valign="top" bgcolor="white" align="center">
61
+ <td bgcolor="white">
62
+ <br>
63
+ <center>
64
+ <center>
65
+ <table width="90%" border="0">
66
+ <form action="/domain/registry/"></form>
67
+ <input type="hidden" value="5" name="sort">
68
+ <tbody>
69
+ <tr>
70
+ <tr>
71
+ <tr>
72
+ <td bgcolor="#cccccc" colspan="4">
73
+ </tr>
74
+ <tr class="trl">
75
+ </pre>
76
+ ```
77
+
78
+ ### XPaths
79
+ Simple XPath for Domain Name only
80
+
81
+ ["//tr[@class='trd']/td/span/a", "//tr[@class='trl']/td/span/a"]
82
+
83
+ - href class of a node is the domainname.
84
+
85
+ Complex XPATH for Deep Data
86
+
87
+ ["//tr[@class='trd']", "//tr[@class='trl']"]
88
+
89
+ Then dig into
90
+
91
+
92
+ td[0]/a/innerhtml = Domain Name
93
+ b.children[0].xpath('.//a')[0].children[0].to_s
94
+
95
+ td[0]/span/text = Number of Subdomains "(544 hosts in use)"
96
+ b.children[0].xpath('.//span')[0].children[0].to_s
97
+
98
+ td[1] = Public / Private
99
+ b.children[1].children[0].to_s
100
+
101
+ td[2]/a/innerhtml = Username
102
+ b.children[2].xpath('.//a')[0].children[0].to_s
103
+
104
+ td[3] = Age "1169 days ago (09/07/2009)"
105
+ b.children[3].children[0].to_s
106
+
107
+
108
+ ### XPATH Match Example
109
+
110
+ #(Element:0x3fcf5cc75298 {
111
+ name = "tr",
112
+ attributes = [ #(Attr:0x3fcf5cc75220 { name = "class", value = "trl" })],
113
+ children = [
114
+ #(Element:0x3fcf5cc78074 {
115
+ name = "td",
116
+ children = [
117
+ #(Element:0x3fcf5cc77890 {
118
+ name = "a",
119
+ attributes = [
120
+ #(Attr:0x3fcf5cc77688 {
121
+ name = "href",
122
+ value = "https://example.com/subdomain/edit.php?edit_domain_id=153370"
123
+ })],
124
+ children = [ #(Text "bestforever.com")]
125
+ }),
126
+ #(Element:0x3fcf5cc7ba58 { name = "br" }),
127
+ #(Element:0x3fcf5cc7af18 {
128
+ name = "span",
129
+ children = [
130
+ #(Text " (280 hosts in use) "),
131
+ #(Element:0x3fcf5cc7d7a4 {
132
+ name = "a",
133
+ attributes = [
134
+ #(Attr:0x3fcf5cc7d5b0 { name = "target", value = "_blank" }),
135
+ #(Attr:0x3fcf5cc7d59c { name = "rel", value = "nofollow" }),
136
+ #(Attr:0x3fcf5cc7d4fc {
137
+ name = "href",
138
+ value = "http://www.bestforever.com/"
139
+ })],
140
+ children = [ #(Text "website")]
141
+ })]
142
+ })]
143
+ }),
144
+ #(Element:0x3fcf5cc847fc { name = "td", children = [ #(Text "public")] }),
145
+ #(Element:0x3fcf5cc85da0 {
146
+ name = "td",
147
+ children = [
148
+ #(Element:0x3fcf5cc856d4 {
149
+ name = "a",
150
+ attributes = [
151
+ #(Attr:0x3fcf5cc855e4 {
152
+ name = "href",
153
+ value = "https://example.com/tools/contact.php?user_id=263456&subject=bestforever.com"
154
+ })],
155
+ children = [ #(Text "shang")]
156
+ })]
157
+ }),
158
+ #(Element:0x3fcf5cc87844 {
159
+ name = "td",
160
+ children = [ #(Text "2178 days ago (12/11/2006)")]
161
+ })]
162
+ })
163
+
164
+ ## Brandable DNS servers
165
+
166
+ afraid.org account, setup/create ns1-ns4.YOURBUSINESS.COM to point to afraid.org's IPs like so:
167
+
168
+ NS1.YOURBUSINESS.COM (50.23.197.95)
169
+ NS2.YOURBUSINESS.COM (174.37.196.55)
170
+ NS3.YOURBUSINESS.COM (72.20.15.62)
171
+ NS4.YOURBUSINESS.COM (174.128.246.102)
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in gemspec
4
+ gemspec
data/Gemfile.lock ADDED
@@ -0,0 +1,109 @@
1
+ PATH
2
+ remote: .
3
+ specs:
4
+ gottabeafraid (0.0.5)
5
+ activerecord
6
+ activesupport
7
+ chronic
8
+ highline
9
+ mechanize (~> 2.5)
10
+ nokogiri
11
+ socksify
12
+ sqlite3
13
+ standalone_migrations
14
+
15
+ GEM
16
+ remote: https://rubygems.org/
17
+ specs:
18
+ actionpack (3.2.19)
19
+ activemodel (= 3.2.19)
20
+ activesupport (= 3.2.19)
21
+ builder (~> 3.0.0)
22
+ erubis (~> 2.7.0)
23
+ journey (~> 1.0.4)
24
+ rack (~> 1.4.5)
25
+ rack-cache (~> 1.2)
26
+ rack-test (~> 0.6.1)
27
+ sprockets (~> 2.2.1)
28
+ activemodel (3.2.19)
29
+ activesupport (= 3.2.19)
30
+ builder (~> 3.0.0)
31
+ activerecord (3.2.19)
32
+ activemodel (= 3.2.19)
33
+ activesupport (= 3.2.19)
34
+ arel (~> 3.0.2)
35
+ tzinfo (~> 0.3.29)
36
+ activesupport (3.2.19)
37
+ i18n (~> 0.6, >= 0.6.4)
38
+ multi_json (~> 1.0)
39
+ arel (3.0.3)
40
+ builder (3.0.4)
41
+ chronic (0.10.2)
42
+ domain_name (0.5.21)
43
+ unf (>= 0.0.5, < 1.0.0)
44
+ erubis (2.7.0)
45
+ highline (1.6.21)
46
+ hike (1.2.3)
47
+ http-cookie (1.0.2)
48
+ domain_name (~> 0.5)
49
+ i18n (0.6.11)
50
+ journey (1.0.4)
51
+ json (1.8.1)
52
+ mechanize (2.7.3)
53
+ domain_name (~> 0.5, >= 0.5.1)
54
+ http-cookie (~> 1.0)
55
+ mime-types (~> 2.0)
56
+ net-http-digest_auth (~> 1.1, >= 1.1.1)
57
+ net-http-persistent (~> 2.5, >= 2.5.2)
58
+ nokogiri (~> 1.4)
59
+ ntlm-http (~> 0.1, >= 0.1.1)
60
+ webrobots (>= 0.0.9, < 0.2)
61
+ mime-types (2.3)
62
+ mini_portile (0.6.0)
63
+ multi_json (1.10.1)
64
+ net-http-digest_auth (1.4)
65
+ net-http-persistent (2.9.4)
66
+ nokogiri (1.6.3.1)
67
+ mini_portile (= 0.6.0)
68
+ ntlm-http (0.1.1)
69
+ rack (1.4.5)
70
+ rack-cache (1.2)
71
+ rack (>= 0.4)
72
+ rack-ssl (1.3.4)
73
+ rack
74
+ rack-test (0.6.2)
75
+ rack (>= 1.0)
76
+ railties (3.2.19)
77
+ actionpack (= 3.2.19)
78
+ activesupport (= 3.2.19)
79
+ rack-ssl (~> 1.3.2)
80
+ rake (>= 0.8.7)
81
+ rdoc (~> 3.4)
82
+ thor (>= 0.14.6, < 2.0)
83
+ rake (10.3.2)
84
+ rdoc (3.12.2)
85
+ json (~> 1.4)
86
+ socksify (1.6.0)
87
+ sprockets (2.2.2)
88
+ hike (~> 1.2)
89
+ multi_json (~> 1.0)
90
+ rack (~> 1.0)
91
+ tilt (~> 1.1, != 1.3.0)
92
+ sqlite3 (1.3.9)
93
+ standalone_migrations (2.1.5)
94
+ activerecord (~> 3.2)
95
+ railties (~> 3.2)
96
+ rake (~> 10.0)
97
+ thor (0.19.1)
98
+ tilt (1.4.1)
99
+ tzinfo (0.3.41)
100
+ unf (0.1.4)
101
+ unf_ext
102
+ unf_ext (0.0.6)
103
+ webrobots (0.1.1)
104
+
105
+ PLATFORMS
106
+ ruby
107
+
108
+ DEPENDENCIES
109
+ gottabeafraid!
data/LICENSE ADDED
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2012 Scott MacGregor
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,65 @@
1
+ # Gotta Be Afraid
2
+
3
+ Be afraid, be very afraid of afraid.org. Stare into the abyss for too long and you'll burn out your eyes!
4
+
5
+ ## Installation
6
+
7
+ Add this line to your application's Gemfile:
8
+
9
+ gem 'gottabeafraid'
10
+
11
+ And then execute:
12
+
13
+ $ bundle
14
+
15
+ Or install it yourself as:
16
+
17
+ $ gem install gottabeafraid
18
+
19
+ ## Usage
20
+
21
+ You know.. be afraid.
22
+
23
+ ```shell
24
+ $ ./bin/gonna --help
25
+ Usage: /home/user/sandbox/github/gottabeafraid/lib/gottabeafraid/cli.rb COMMAND [OPTIONS]
26
+
27
+ Onion / Tor
28
+ -o, --onion Run on onion/tor network via socks mode?
29
+ -p, --socks-port Socks 5 Proxy port for Tor
30
+ Default: 9050
31
+ Options
32
+ -t, --[no-]test Run Test suite mocks
33
+ -v, --[no-]verbose Run verbosely
34
+ -h, --help help
35
+
36
+ ```
37
+
38
+ Just run irb, silly..
39
+
40
+ ## Using Tor/Onion
41
+
42
+ When Onion flag is enabled it will attempt to use a Socks Proxy on 9050 to connect to afraid.org. Note the SSL certificate verification is disabled when doing this.
43
+
44
+ ```ruby
45
+ @agent.agent.set_socks('127.0.0.1', 9050)
46
+ @agent.agent.http.verify_mode = OpenSSL::SSL::VERIFY_NONE
47
+ ```
48
+
49
+ It will also attempt to perform a sanity check on your new Tor node IP address.
50
+
51
+ * Query to "http://checkip.dyndns.org" will be shown asking if you like this external Tor exit node.
52
+
53
+ ## Contributing
54
+
55
+ 1. Fork it
56
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
57
+ 3. Commit your changes (`git commit -am 'Added some feature'`)
58
+ 4. Push to the branch (`git push origin my-new-feature`)
59
+ 5. Create new Pull Request
60
+
61
+ ## License
62
+
63
+ MIT - See [LICENSE](./LICENSE) for details.
64
+
65
+ Copyright 2014 Shadowbq
data/Rakefile ADDED
@@ -0,0 +1,18 @@
1
+ #!/usr/bin/env rake
2
+ require "bundler/gem_tasks"
3
+ #require "rake/version_task"
4
+ require "rake/testtask"
5
+ require 'standalone_migrations'
6
+
7
+
8
+ #Rake::VersionTask.new
9
+
10
+ StandaloneMigrations::Tasks.load_tasks
11
+
12
+ task :default => [:test]
13
+
14
+ Rake::TestTask.new do |test|
15
+ test.libs << "test"
16
+ test.test_files = Dir[ "test/test_*.rb" ]
17
+ test.verbose = true
18
+ end
data/bin/gonna ADDED
@@ -0,0 +1,13 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ begin
4
+ require 'rubygems'
5
+ require 'bundler'
6
+ Bundler.setup(:default)
7
+ rescue ::Exception => e
8
+ end
9
+
10
+ # Executable with absolute path to lib for hacking and development
11
+ require File.join(File.dirname(__FILE__), '..', 'lib', 'gottabeafraid', 'cli')
12
+
13
+ Gottabeafraid::CLI.invoke