ronin-web 0.3.0.rc1 → 1.0.0.beta2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (148) hide show
  1. checksums.yaml +7 -0
  2. data/.editorconfig +11 -0
  3. data/.github/workflows/ruby.yml +31 -0
  4. data/.gitignore +13 -0
  5. data/.mailmap +1 -0
  6. data/.ruby-version +1 -0
  7. data/COPYING.txt +3 -3
  8. data/ChangeLog.md +115 -70
  9. data/Gemfile +42 -37
  10. data/README.md +160 -145
  11. data/Rakefile +12 -3
  12. data/bin/ronin-web +9 -17
  13. data/data/new/nokogiri.rb.erb +12 -0
  14. data/data/new/server.rb.erb +22 -0
  15. data/data/new/spider.rb.erb +26 -0
  16. data/data/new/webapp/.gitignore +15 -0
  17. data/data/new/webapp/.ruby-version.erb +1 -0
  18. data/data/new/webapp/Dockerfile.erb +11 -0
  19. data/data/new/webapp/Gemfile +6 -0
  20. data/data/new/webapp/app.rb.erb +15 -0
  21. data/data/new/webapp/config.ru +4 -0
  22. data/data/new/webapp/docker-compose.yml.erb +9 -0
  23. data/gemspec.yml +38 -16
  24. data/lib/ronin/web/cli/command.rb +36 -0
  25. data/lib/ronin/web/cli/commands/diff.rb +106 -0
  26. data/lib/ronin/web/cli/commands/html.rb +174 -0
  27. data/lib/ronin/web/cli/commands/irb.rb +56 -0
  28. data/lib/ronin/web/cli/commands/new/nokogiri.rb +85 -0
  29. data/lib/ronin/web/cli/commands/new/server.rb +96 -0
  30. data/lib/ronin/web/cli/commands/new/spider.rb +315 -0
  31. data/lib/ronin/web/cli/commands/new/webapp.rb +123 -0
  32. data/lib/ronin/web/cli/commands/new.rb +64 -0
  33. data/lib/ronin/web/cli/commands/reverse_proxy.rb +215 -0
  34. data/lib/ronin/web/cli/commands/server.rb +155 -0
  35. data/lib/ronin/web/cli/commands/spider.rb +822 -0
  36. data/lib/ronin/web/cli/ruby_shell.rb +50 -0
  37. data/lib/ronin/web/cli.rb +49 -0
  38. data/lib/ronin/web/html.rb +85 -0
  39. data/lib/ronin/web/mechanize.rb +34 -36
  40. data/lib/ronin/web/root.rb +27 -0
  41. data/lib/ronin/web/version.rb +7 -10
  42. data/lib/ronin/web/xml.rb +85 -0
  43. data/lib/ronin/web.rb +372 -13
  44. data/man/ronin-web-diff.1 +41 -0
  45. data/man/ronin-web-diff.1.md +30 -0
  46. data/man/ronin-web-html.1 +89 -0
  47. data/man/ronin-web-html.1.md +66 -0
  48. data/man/ronin-web-irb.1 +31 -0
  49. data/man/ronin-web-irb.1.md +22 -0
  50. data/man/ronin-web-new-nokogiri.1 +41 -0
  51. data/man/ronin-web-new-nokogiri.1.md +30 -0
  52. data/man/ronin-web-new-server.1 +45 -0
  53. data/man/ronin-web-new-server.1.md +33 -0
  54. data/man/ronin-web-new-spider.1 +173 -0
  55. data/man/ronin-web-new-spider.1.md +129 -0
  56. data/man/ronin-web-new-webapp.1 +53 -0
  57. data/man/ronin-web-new-webapp.1.md +39 -0
  58. data/man/ronin-web-new.1 +59 -0
  59. data/man/ronin-web-new.1.md +44 -0
  60. data/man/ronin-web-reverse-proxy.1 +63 -0
  61. data/man/ronin-web-reverse-proxy.1.md +47 -0
  62. data/man/ronin-web-server.1 +59 -0
  63. data/man/ronin-web-server.1.md +43 -0
  64. data/man/ronin-web-spider.1 +225 -0
  65. data/man/ronin-web-spider.1.md +168 -0
  66. data/man/ronin-web.1 +41 -0
  67. data/man/ronin-web.1.md +30 -0
  68. data/ronin-web.gemspec +39 -109
  69. data/spec/cli/ruby_shell_spec.rb +14 -0
  70. data/spec/html_spec.rb +43 -0
  71. data/spec/mechanize_spec.rb +72 -0
  72. data/spec/spec_helper.rb +5 -3
  73. data/spec/web_spec.rb +97 -0
  74. data/spec/xml_spec.rb +42 -0
  75. metadata +236 -224
  76. data/.gemtest +0 -0
  77. data/data/ronin/web/user_agents.yml +0 -247
  78. data/lib/ronin/network/mixins/web.rb +0 -258
  79. data/lib/ronin/web/config.rb +0 -34
  80. data/lib/ronin/web/extensions/nokogiri/xml/attr.rb +0 -47
  81. data/lib/ronin/web/extensions/nokogiri/xml/document.rb +0 -48
  82. data/lib/ronin/web/extensions/nokogiri/xml/element.rb +0 -57
  83. data/lib/ronin/web/extensions/nokogiri/xml/node.rb +0 -86
  84. data/lib/ronin/web/extensions/nokogiri/xml/text.rb +0 -47
  85. data/lib/ronin/web/extensions/nokogiri/xml.rb +0 -27
  86. data/lib/ronin/web/extensions/nokogiri.rb +0 -23
  87. data/lib/ronin/web/extensions.rb +0 -23
  88. data/lib/ronin/web/middleware/base.rb +0 -144
  89. data/lib/ronin/web/middleware/directories.rb +0 -179
  90. data/lib/ronin/web/middleware/files.rb +0 -144
  91. data/lib/ronin/web/middleware/filters/campaign_filter.rb +0 -77
  92. data/lib/ronin/web/middleware/filters/ip_filter.rb +0 -73
  93. data/lib/ronin/web/middleware/filters/path_filter.rb +0 -73
  94. data/lib/ronin/web/middleware/filters/referer_filter.rb +0 -71
  95. data/lib/ronin/web/middleware/filters/user_agent_filter.rb +0 -71
  96. data/lib/ronin/web/middleware/filters/vhost_filter.rb +0 -71
  97. data/lib/ronin/web/middleware/filters.rb +0 -28
  98. data/lib/ronin/web/middleware/helpers.rb +0 -145
  99. data/lib/ronin/web/middleware/proxy.rb +0 -265
  100. data/lib/ronin/web/middleware/proxy_request.rb +0 -262
  101. data/lib/ronin/web/middleware/request.rb +0 -79
  102. data/lib/ronin/web/middleware/response.rb +0 -33
  103. data/lib/ronin/web/middleware/router.rb +0 -167
  104. data/lib/ronin/web/middleware/rule.rb +0 -103
  105. data/lib/ronin/web/middleware.rb +0 -27
  106. data/lib/ronin/web/proxy/app.rb +0 -32
  107. data/lib/ronin/web/proxy/base.rb +0 -46
  108. data/lib/ronin/web/proxy/web.rb +0 -46
  109. data/lib/ronin/web/proxy.rb +0 -25
  110. data/lib/ronin/web/server/app.rb +0 -32
  111. data/lib/ronin/web/server/base.rb +0 -461
  112. data/lib/ronin/web/server/web.rb +0 -66
  113. data/lib/ronin/web/server.rb +0 -25
  114. data/lib/ronin/web/spider.rb +0 -120
  115. data/lib/ronin/web/user_agents.rb +0 -196
  116. data/lib/ronin/web/web.rb +0 -560
  117. data/spec/helpers/output.rb +0 -3
  118. data/spec/web/extensions/nokogiri_spec.rb +0 -38
  119. data/spec/web/helpers/rack_app.rb +0 -24
  120. data/spec/web/helpers/root/test1/index.html +0 -1
  121. data/spec/web/helpers/root/test1/test1.txt +0 -1
  122. data/spec/web/helpers/root/test1.txt +0 -1
  123. data/spec/web/helpers/root/test2/test2.txt +0 -1
  124. data/spec/web/helpers/root/test2.txt +0 -1
  125. data/spec/web/helpers/root/test3/test3.txt +0 -1
  126. data/spec/web/helpers/root/test3.txt +0 -1
  127. data/spec/web/helpers/root.rb +0 -15
  128. data/spec/web/mechanize_spec.rb +0 -62
  129. data/spec/web/middleware/directories_spec.rb +0 -86
  130. data/spec/web/middleware/files_spec.rb +0 -57
  131. data/spec/web/middleware/filters/campaign_filter_spec.rb +0 -30
  132. data/spec/web/middleware/filters/ip_filter_spec.rb +0 -25
  133. data/spec/web/middleware/filters/path_filter_spec.rb +0 -29
  134. data/spec/web/middleware/filters/referer_filter_spec.rb +0 -25
  135. data/spec/web/middleware/filters/user_agent_filter_spec.rb +0 -25
  136. data/spec/web/middleware/filters/vhost_filter_spec.rb +0 -23
  137. data/spec/web/middleware/proxy_spec.rb +0 -67
  138. data/spec/web/middleware/response_spec.rb +0 -20
  139. data/spec/web/middleware/router_spec.rb +0 -65
  140. data/spec/web/middleware/rule_spec.rb +0 -37
  141. data/spec/web/proxy/base_spec.rb +0 -8
  142. data/spec/web/server/base_spec.rb +0 -77
  143. data/spec/web/server/classes/public1/static1.txt +0 -1
  144. data/spec/web/server/classes/public2/static2.txt +0 -1
  145. data/spec/web/server/classes/sub_app.rb +0 -13
  146. data/spec/web/server/classes/test_app.rb +0 -20
  147. data/spec/web/user_agents_spec.rb +0 -56
  148. data/spec/web/web_spec.rb +0 -101
data/spec/web/user_agents_spec.rb DELETED
@@ -1,56 +0,0 @@
1
- require 'spec_helper'
2
- require 'ronin/web/user_agents'
3
-
4
- describe Web::UserAgents do
5
- it "should list the categories of User-Agent strings" do
6
- subject.categories.should_not be_empty
7
- end
8
-
9
- describe "#[]" do
10
- context "with Symbol" do
11
- it "should select User-Agent strings by group name" do
12
- subject[:ie].should_not be_nil
13
- end
14
-
15
- it "should return nil if the group exists" do
16
- subject[:foobarbaz].should be_nil
17
- end
18
- end
19
-
20
- context "with String" do
21
- it "should select User-Agent strings by substring" do
22
- subject['MSIE'].should_not be_nil
23
- end
24
-
25
- it "should return nil if no User-Agent matches the substring" do
26
- subject['FooBarBaz'].should be_nil
27
- end
28
- end
29
-
30
- context "with Regexp" do
31
- it "should select User-Agent strings by Regexp" do
32
- subject[/AppleWebKit/i].should_not be_nil
33
- end
34
-
35
- it "should return nil if no User-Agent matches the Regexp" do
36
- subject[/FooBarBaz/i].should be_nil
37
- end
38
- end
39
- end
40
-
41
- describe "#fetch" do
42
- it "should fetch a User-Agent string" do
43
- subject.fetch(:ie).should_not be_nil
44
- end
45
-
46
- it "should raise an ArgumentError if no match was found" do
47
- lambda {
48
- subject.fetch(:foobarbaz)
49
- }.should raise_error(ArgumentError)
50
- end
51
-
52
- it "should return the default value if no match was found" do
53
- subject.fetch(:foobarbaz,'default').should == 'default'
54
- end
55
- end
56
- end
data/spec/web/web_spec.rb DELETED
@@ -1,101 +0,0 @@
1
- require 'spec_helper'
2
- require 'ronin/web/web'
3
-
4
- describe Web do
5
- let(:url) { 'http://ronin-ruby.github.com/' }
6
- let(:title) { 'Ronin' }
7
-
8
- it "should have a VERSION constant" do
9
- Web.const_defined?('VERSION').should == true
10
- end
11
-
12
- it "should be able to parse HTML" do
13
- doc = Web.html(%{
14
- <html>
15
- <body>Hello</body>
16
- </html>
17
- })
18
-
19
- doc.at('body').inner_text.should == "Hello"
20
- end
21
-
22
- it "should be able to build HTML documents" do
23
- doc = Web.build_html do
24
- html {
25
- body {
26
- div { text("hello") }
27
- }
28
- }
29
- end
30
-
31
- doc.to_html.should include("<html><body><div>hello</div></body></html>")
32
- end
33
-
34
- it "should be able to parse XML" do
35
- doc = Web.html(%{
36
- <?xml version="1.0"?>
37
- <root>
38
- <stuff>Hello</stuff>
39
- </root>
40
- })
41
-
42
- doc.at('stuff').inner_text.should == "Hello"
43
- end
44
-
45
- it "should be able to build XML documents" do
46
- doc = Web.build_xml do
47
- root {
48
- stuff(:name => 'bla') { text("hello") }
49
- }
50
- end
51
-
52
- doc.to_xml.should include("<root>\n <stuff name=\"bla\">hello</stuff>\n</root>")
53
- end
54
-
55
- it "should have a default proxy" do
56
- Web.proxy.should_not be_nil
57
- end
58
-
59
- it "should disable the proxy by default" do
60
- Web.proxy.should_not be_enabled
61
- end
62
-
63
- it "should provide User-Agent aliases" do
64
- Web.user_agent_aliases.should_not be_empty
65
- end
66
-
67
- it "should provide a default User-Agent" do
68
- Web.user_agent.should be_nil
69
- end
70
-
71
- it "should allow setting of the User-Agent string using an alias" do
72
- Web.user_agent_alias = 'Mac FireFox'
73
-
74
- Web.user_agent.should == "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6"
75
- end
76
-
77
- it "should open URLs as temporary files" do
78
- file = Web.open(url)
79
-
80
- file.read.should include(title)
81
- end
82
-
83
- describe "agent" do
84
- it "should be persistent" do
85
- Web.agent.object_id.should == Web.agent.object_id
86
- end
87
- end
88
-
89
- it "should be able to get Mechanize pages" do
90
- page = Web.get(url)
91
-
92
- page.class.should == Mechanize::Page
93
- page.at('title').inner_text.should include(title)
94
- end
95
-
96
- it "should be able to get the bodies of Mechanize pages" do
97
- body = Web.get_body(url)
98
-
99
- body.should include(title)
100
- end
101
- end