ronin-web 0.3.0.rc1 → 1.0.0.beta1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (148) hide show
  1. checksums.yaml +7 -0
  2. data/.editorconfig +11 -0
  3. data/.github/workflows/ruby.yml +31 -0
  4. data/.gitignore +13 -0
  5. data/.mailmap +1 -0
  6. data/.ruby-version +1 -0
  7. data/COPYING.txt +3 -3
  8. data/ChangeLog.md +115 -70
  9. data/Gemfile +42 -37
  10. data/README.md +159 -145
  11. data/Rakefile +12 -3
  12. data/bin/ronin-web +9 -17
  13. data/data/new/nokogiri.rb.erb +12 -0
  14. data/data/new/server.rb.erb +22 -0
  15. data/data/new/spider.rb.erb +26 -0
  16. data/data/new/webapp/.gitignore +15 -0
  17. data/data/new/webapp/.ruby-version.erb +1 -0
  18. data/data/new/webapp/Dockerfile.erb +11 -0
  19. data/data/new/webapp/Gemfile +6 -0
  20. data/data/new/webapp/app.rb.erb +15 -0
  21. data/data/new/webapp/config.ru +4 -0
  22. data/data/new/webapp/docker-compose.yml.erb +9 -0
  23. data/gemspec.yml +32 -14
  24. data/lib/ronin/web/cli/command.rb +36 -0
  25. data/lib/ronin/web/cli/commands/diff.rb +106 -0
  26. data/lib/ronin/web/cli/commands/html.rb +174 -0
  27. data/lib/ronin/web/cli/commands/irb.rb +56 -0
  28. data/lib/ronin/web/cli/commands/new/nokogiri.rb +85 -0
  29. data/lib/ronin/web/cli/commands/new/server.rb +96 -0
  30. data/lib/ronin/web/cli/commands/new/spider.rb +315 -0
  31. data/lib/ronin/web/cli/commands/new/webapp.rb +123 -0
  32. data/lib/ronin/web/cli/commands/new.rb +64 -0
  33. data/lib/ronin/web/cli/commands/reverse_proxy.rb +215 -0
  34. data/lib/ronin/web/cli/commands/server.rb +155 -0
  35. data/lib/ronin/web/cli/commands/spider.rb +822 -0
  36. data/lib/ronin/web/cli/ruby_shell.rb +50 -0
  37. data/lib/ronin/web/cli.rb +44 -0
  38. data/lib/ronin/web/html.rb +85 -0
  39. data/lib/ronin/web/mechanize.rb +34 -36
  40. data/lib/ronin/web/root.rb +27 -0
  41. data/lib/ronin/web/version.rb +7 -10
  42. data/lib/ronin/web/xml.rb +85 -0
  43. data/lib/ronin/web.rb +372 -13
  44. data/man/ronin-web-diff.1 +41 -0
  45. data/man/ronin-web-diff.1.md +30 -0
  46. data/man/ronin-web-html.1 +89 -0
  47. data/man/ronin-web-html.1.md +66 -0
  48. data/man/ronin-web-irb.1 +31 -0
  49. data/man/ronin-web-irb.1.md +22 -0
  50. data/man/ronin-web-new-nokogiri.1 +41 -0
  51. data/man/ronin-web-new-nokogiri.1.md +30 -0
  52. data/man/ronin-web-new-server.1 +45 -0
  53. data/man/ronin-web-new-server.1.md +33 -0
  54. data/man/ronin-web-new-spider.1 +173 -0
  55. data/man/ronin-web-new-spider.1.md +129 -0
  56. data/man/ronin-web-new-webapp.1 +53 -0
  57. data/man/ronin-web-new-webapp.1.md +39 -0
  58. data/man/ronin-web-new.1 +59 -0
  59. data/man/ronin-web-new.1.md +44 -0
  60. data/man/ronin-web-reverse-proxy.1 +63 -0
  61. data/man/ronin-web-reverse-proxy.1.md +47 -0
  62. data/man/ronin-web-server.1 +59 -0
  63. data/man/ronin-web-server.1.md +43 -0
  64. data/man/ronin-web-spider.1 +225 -0
  65. data/man/ronin-web-spider.1.md +168 -0
  66. data/man/ronin-web.1 +41 -0
  67. data/man/ronin-web.1.md +30 -0
  68. data/ronin-web.gemspec +39 -109
  69. data/spec/cli/ruby_shell_spec.rb +14 -0
  70. data/spec/html_spec.rb +43 -0
  71. data/spec/mechanize_spec.rb +72 -0
  72. data/spec/spec_helper.rb +5 -3
  73. data/spec/web_spec.rb +97 -0
  74. data/spec/xml_spec.rb +42 -0
  75. metadata +236 -224
  76. data/.gemtest +0 -0
  77. data/data/ronin/web/user_agents.yml +0 -247
  78. data/lib/ronin/network/mixins/web.rb +0 -258
  79. data/lib/ronin/web/config.rb +0 -34
  80. data/lib/ronin/web/extensions/nokogiri/xml/attr.rb +0 -47
  81. data/lib/ronin/web/extensions/nokogiri/xml/document.rb +0 -48
  82. data/lib/ronin/web/extensions/nokogiri/xml/element.rb +0 -57
  83. data/lib/ronin/web/extensions/nokogiri/xml/node.rb +0 -86
  84. data/lib/ronin/web/extensions/nokogiri/xml/text.rb +0 -47
  85. data/lib/ronin/web/extensions/nokogiri/xml.rb +0 -27
  86. data/lib/ronin/web/extensions/nokogiri.rb +0 -23
  87. data/lib/ronin/web/extensions.rb +0 -23
  88. data/lib/ronin/web/middleware/base.rb +0 -144
  89. data/lib/ronin/web/middleware/directories.rb +0 -179
  90. data/lib/ronin/web/middleware/files.rb +0 -144
  91. data/lib/ronin/web/middleware/filters/campaign_filter.rb +0 -77
  92. data/lib/ronin/web/middleware/filters/ip_filter.rb +0 -73
  93. data/lib/ronin/web/middleware/filters/path_filter.rb +0 -73
  94. data/lib/ronin/web/middleware/filters/referer_filter.rb +0 -71
  95. data/lib/ronin/web/middleware/filters/user_agent_filter.rb +0 -71
  96. data/lib/ronin/web/middleware/filters/vhost_filter.rb +0 -71
  97. data/lib/ronin/web/middleware/filters.rb +0 -28
  98. data/lib/ronin/web/middleware/helpers.rb +0 -145
  99. data/lib/ronin/web/middleware/proxy.rb +0 -265
  100. data/lib/ronin/web/middleware/proxy_request.rb +0 -262
  101. data/lib/ronin/web/middleware/request.rb +0 -79
  102. data/lib/ronin/web/middleware/response.rb +0 -33
  103. data/lib/ronin/web/middleware/router.rb +0 -167
  104. data/lib/ronin/web/middleware/rule.rb +0 -103
  105. data/lib/ronin/web/middleware.rb +0 -27
  106. data/lib/ronin/web/proxy/app.rb +0 -32
  107. data/lib/ronin/web/proxy/base.rb +0 -46
  108. data/lib/ronin/web/proxy/web.rb +0 -46
  109. data/lib/ronin/web/proxy.rb +0 -25
  110. data/lib/ronin/web/server/app.rb +0 -32
  111. data/lib/ronin/web/server/base.rb +0 -461
  112. data/lib/ronin/web/server/web.rb +0 -66
  113. data/lib/ronin/web/server.rb +0 -25
  114. data/lib/ronin/web/spider.rb +0 -120
  115. data/lib/ronin/web/user_agents.rb +0 -196
  116. data/lib/ronin/web/web.rb +0 -560
  117. data/spec/helpers/output.rb +0 -3
  118. data/spec/web/extensions/nokogiri_spec.rb +0 -38
  119. data/spec/web/helpers/rack_app.rb +0 -24
  120. data/spec/web/helpers/root/test1/index.html +0 -1
  121. data/spec/web/helpers/root/test1/test1.txt +0 -1
  122. data/spec/web/helpers/root/test1.txt +0 -1
  123. data/spec/web/helpers/root/test2/test2.txt +0 -1
  124. data/spec/web/helpers/root/test2.txt +0 -1
  125. data/spec/web/helpers/root/test3/test3.txt +0 -1
  126. data/spec/web/helpers/root/test3.txt +0 -1
  127. data/spec/web/helpers/root.rb +0 -15
  128. data/spec/web/mechanize_spec.rb +0 -62
  129. data/spec/web/middleware/directories_spec.rb +0 -86
  130. data/spec/web/middleware/files_spec.rb +0 -57
  131. data/spec/web/middleware/filters/campaign_filter_spec.rb +0 -30
  132. data/spec/web/middleware/filters/ip_filter_spec.rb +0 -25
  133. data/spec/web/middleware/filters/path_filter_spec.rb +0 -29
  134. data/spec/web/middleware/filters/referer_filter_spec.rb +0 -25
  135. data/spec/web/middleware/filters/user_agent_filter_spec.rb +0 -25
  136. data/spec/web/middleware/filters/vhost_filter_spec.rb +0 -23
  137. data/spec/web/middleware/proxy_spec.rb +0 -67
  138. data/spec/web/middleware/response_spec.rb +0 -20
  139. data/spec/web/middleware/router_spec.rb +0 -65
  140. data/spec/web/middleware/rule_spec.rb +0 -37
  141. data/spec/web/proxy/base_spec.rb +0 -8
  142. data/spec/web/server/base_spec.rb +0 -77
  143. data/spec/web/server/classes/public1/static1.txt +0 -1
  144. data/spec/web/server/classes/public2/static2.txt +0 -1
  145. data/spec/web/server/classes/sub_app.rb +0 -13
  146. data/spec/web/server/classes/test_app.rb +0 -20
  147. data/spec/web/user_agents_spec.rb +0 -56
  148. data/spec/web/web_spec.rb +0 -101
data/spec/web/user_agents_spec.rb DELETED
@@ -1,56 +0,0 @@
1
- require 'spec_helper'
2
- require 'ronin/web/user_agents'
3
-
4
- describe Web::UserAgents do
5
- it "should list the categories of User-Agent strings" do
6
- subject.categories.should_not be_empty
7
- end
8
-
9
- describe "#[]" do
10
- context "with Symbol" do
11
- it "should select User-Agent strings by group name" do
12
- subject[:ie].should_not be_nil
13
- end
14
-
15
- it "should return nil if the group exists" do
16
- subject[:foobarbaz].should be_nil
17
- end
18
- end
19
-
20
- context "with String" do
21
- it "should select User-Agent strings by substring" do
22
- subject['MSIE'].should_not be_nil
23
- end
24
-
25
- it "should return nil if no User-Agent matches the substring" do
26
- subject['FooBarBaz'].should be_nil
27
- end
28
- end
29
-
30
- context "with Regexp" do
31
- it "should select User-Agent strings by Regexp" do
32
- subject[/AppleWebKit/i].should_not be_nil
33
- end
34
-
35
- it "should return nil if no User-Agent matches the Regexp" do
36
- subject[/FooBarBaz/i].should be_nil
37
- end
38
- end
39
- end
40
-
41
- describe "#fetch" do
42
- it "should fetch a User-Agent string" do
43
- subject.fetch(:ie).should_not be_nil
44
- end
45
-
46
- it "should raise an ArgumentError if no match was found" do
47
- lambda {
48
- subject.fetch(:foobarbaz)
49
- }.should raise_error(ArgumentError)
50
- end
51
-
52
- it "should return the default value if no match was found" do
53
- subject.fetch(:foobarbaz,'default').should == 'default'
54
- end
55
- end
56
- end
data/spec/web/web_spec.rb DELETED
@@ -1,101 +0,0 @@
1
- require 'spec_helper'
2
- require 'ronin/web/web'
3
-
4
- describe Web do
5
- let(:url) { 'http://ronin-ruby.github.com/' }
6
- let(:title) { 'Ronin' }
7
-
8
- it "should have a VERSION constant" do
9
- Web.const_defined?('VERSION').should == true
10
- end
11
-
12
- it "should be able to parse HTML" do
13
- doc = Web.html(%{
14
- <html>
15
- <body>Hello</body>
16
- </html>
17
- })
18
-
19
- doc.at('body').inner_text.should == "Hello"
20
- end
21
-
22
- it "should be able to build HTML documents" do
23
- doc = Web.build_html do
24
- html {
25
- body {
26
- div { text("hello") }
27
- }
28
- }
29
- end
30
-
31
- doc.to_html.should include("<html><body><div>hello</div></body></html>")
32
- end
33
-
34
- it "should be able to parse XML" do
35
- doc = Web.html(%{
36
- <?xml version="1.0"?>
37
- <root>
38
- <stuff>Hello</stuff>
39
- </root>
40
- })
41
-
42
- doc.at('stuff').inner_text.should == "Hello"
43
- end
44
-
45
- it "should be able to build XML documents" do
46
- doc = Web.build_xml do
47
- root {
48
- stuff(:name => 'bla') { text("hello") }
49
- }
50
- end
51
-
52
- doc.to_xml.should include("<root>\n <stuff name=\"bla\">hello</stuff>\n</root>")
53
- end
54
-
55
- it "should have a default proxy" do
56
- Web.proxy.should_not be_nil
57
- end
58
-
59
- it "should disable the proxy by default" do
60
- Web.proxy.should_not be_enabled
61
- end
62
-
63
- it "should provide User-Agent aliases" do
64
- Web.user_agent_aliases.should_not be_empty
65
- end
66
-
67
- it "should provide a default User-Agent" do
68
- Web.user_agent.should be_nil
69
- end
70
-
71
- it "should allow setting of the User-Agent string using an alias" do
72
- Web.user_agent_alias = 'Mac FireFox'
73
-
74
- Web.user_agent.should == "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6"
75
- end
76
-
77
- it "should open URLs as temporary files" do
78
- file = Web.open(url)
79
-
80
- file.read.should include(title)
81
- end
82
-
83
- describe "agent" do
84
- it "should be persistent" do
85
- Web.agent.object_id.should == Web.agent.object_id
86
- end
87
- end
88
-
89
- it "should be able to get Mechanize pages" do
90
- page = Web.get(url)
91
-
92
- page.class.should == Mechanize::Page
93
- page.at('title').inner_text.should include(title)
94
- end
95
-
96
- it "should be able to get the bodies of Mechanize pages" do
97
- body = Web.get_body(url)
98
-
99
- body.should include(title)
100
- end
101
- end