rails_spider 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (67) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +165 -0
  3. data/README.md +33 -0
  4. data/app/assets/config/the_spider_manifest.js +4 -0
  5. data/app/assets/javascripts/the_spider/application.js +1 -0
  6. data/app/assets/stylesheets/the_spider/application.css +4 -0
  7. data/app/controllers/the_spider/application_controller.rb +9 -0
  8. data/app/controllers/the_spider/locals_controller.rb +62 -0
  9. data/app/controllers/the_spider/works_controller.rb +60 -0
  10. data/app/helpers/the_spider/application_helper.rb +4 -0
  11. data/app/helpers/the_spider/locals_helper.rb +4 -0
  12. data/app/helpers/the_spider/works_helper.rb +4 -0
  13. data/app/jobs/the_spider/application_job.rb +4 -0
  14. data/app/jobs/the_spider/parser_job.rb +11 -0
  15. data/app/jobs/the_spider/work_job.rb +11 -0
  16. data/app/mailers/the_spider/application_mailer.rb +6 -0
  17. data/app/models/rails_spider/application_record.rb +5 -0
  18. data/app/models/rails_spider/cookie.rb +9 -0
  19. data/app/models/rails_spider/failed_url.rb +7 -0
  20. data/app/models/rails_spider/local.rb +14 -0
  21. data/app/models/rails_spider/work.rb +24 -0
  22. data/app/views/layouts/the_spider/application.html.erb +14 -0
  23. data/app/views/the_spider/locals/_form.html.erb +17 -0
  24. data/app/views/the_spider/locals/edit.html.erb +6 -0
  25. data/app/views/the_spider/locals/index.html.erb +25 -0
  26. data/app/views/the_spider/locals/new.html.erb +5 -0
  27. data/app/views/the_spider/locals/show.html.erb +4 -0
  28. data/app/views/the_spider/works/_form.html.erb +9 -0
  29. data/app/views/the_spider/works/edit.html.erb +6 -0
  30. data/app/views/the_spider/works/index.html.erb +44 -0
  31. data/app/views/the_spider/works/new.html.erb +5 -0
  32. data/app/views/the_spider/works/show.html.erb +4 -0
  33. data/config/routes.rb +8 -0
  34. data/config/schedule.rb +35 -0
  35. data/db/migrate/20170502153051_rails_spider_init.rb +38 -0
  36. data/lib/config/config.rb +27 -0
  37. data/lib/config/keywords.json +22 -0
  38. data/lib/config/proxy.json +10 -0
  39. data/lib/helper/helper.rb +6 -0
  40. data/lib/helper/location_helper.rb +46 -0
  41. data/lib/helper/price_helper.rb +23 -0
  42. data/lib/helper/tag_helper.rb +17 -0
  43. data/lib/helper/text_helper.rb +41 -0
  44. data/lib/helper/time_helper.rb +140 -0
  45. data/lib/logger.rb +146 -0
  46. data/lib/proxy/allproxylists.txt +2366 -0
  47. data/lib/proxy/proxy.rb +216 -0
  48. data/lib/proxy/proxylists.txt +625 -0
  49. data/lib/rails_spider.rb +10 -0
  50. data/lib/rails_spider/engine.rb +9 -0
  51. data/lib/rails_spider/fetchers.rb +2 -0
  52. data/lib/rails_spider/fetchers/base.rb +146 -0
  53. data/lib/rails_spider/fetchers/mechanize.rb +83 -0
  54. data/lib/rails_spider/fetchers/witar.rb +73 -0
  55. data/lib/rails_spider/parser.rb +14 -0
  56. data/lib/rails_spider/parser/szlawyers.rb +26 -0
  57. data/lib/rails_spider/resource.rb +58 -0
  58. data/lib/rails_spider/strategies.rb +6 -0
  59. data/lib/rails_spider/version.rb +3 -0
  60. data/lib/sync_qiniu.rb +35 -0
  61. data/lib/sync_qiniu/getimages.rb +98 -0
  62. data/lib/sync_qiniu/getimages_info.rb +37 -0
  63. data/lib/sync_qiniu/getlocation.rb +48 -0
  64. data/lib/sync_qiniu/getproxy.rb +95 -0
  65. data/lib/tasks/the_spider_tasks.rake +4 -0
  66. data/rakefile +284 -0
  67. metadata +165 -0
metadata ADDED
@@ -0,0 +1,165 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: rails_spider
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - qinmingyuan
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2018-10-25 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: rails
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ">="
18
+ - !ruby/object:Gem::Version
19
+ version: '5.0'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ">="
25
+ - !ruby/object:Gem::Version
26
+ version: '5.0'
27
+ - !ruby/object:Gem::Dependency
28
+ name: mechanize
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: '2.7'
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: '2.7'
41
+ - !ruby/object:Gem::Dependency
42
+ name: watir
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: 6.2.0
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: 6.2.0
55
+ - !ruby/object:Gem::Dependency
56
+ name: kimurai
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ">="
60
+ - !ruby/object:Gem::Version
61
+ version: '0'
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ">="
67
+ - !ruby/object:Gem::Version
68
+ version: '0'
69
+ description: Description of RailsSpider.
70
+ email:
71
+ - mingyuan0715@foxmail.com
72
+ executables: []
73
+ extensions: []
74
+ extra_rdoc_files: []
75
+ files:
76
+ - LICENSE
77
+ - README.md
78
+ - app/assets/config/the_spider_manifest.js
79
+ - app/assets/javascripts/the_spider/application.js
80
+ - app/assets/stylesheets/the_spider/application.css
81
+ - app/controllers/the_spider/application_controller.rb
82
+ - app/controllers/the_spider/locals_controller.rb
83
+ - app/controllers/the_spider/works_controller.rb
84
+ - app/helpers/the_spider/application_helper.rb
85
+ - app/helpers/the_spider/locals_helper.rb
86
+ - app/helpers/the_spider/works_helper.rb
87
+ - app/jobs/the_spider/application_job.rb
88
+ - app/jobs/the_spider/parser_job.rb
89
+ - app/jobs/the_spider/work_job.rb
90
+ - app/mailers/the_spider/application_mailer.rb
91
+ - app/models/rails_spider/application_record.rb
92
+ - app/models/rails_spider/cookie.rb
93
+ - app/models/rails_spider/failed_url.rb
94
+ - app/models/rails_spider/local.rb
95
+ - app/models/rails_spider/work.rb
96
+ - app/views/layouts/the_spider/application.html.erb
97
+ - app/views/the_spider/locals/_form.html.erb
98
+ - app/views/the_spider/locals/edit.html.erb
99
+ - app/views/the_spider/locals/index.html.erb
100
+ - app/views/the_spider/locals/new.html.erb
101
+ - app/views/the_spider/locals/show.html.erb
102
+ - app/views/the_spider/works/_form.html.erb
103
+ - app/views/the_spider/works/edit.html.erb
104
+ - app/views/the_spider/works/index.html.erb
105
+ - app/views/the_spider/works/new.html.erb
106
+ - app/views/the_spider/works/show.html.erb
107
+ - config/routes.rb
108
+ - config/schedule.rb
109
+ - db/migrate/20170502153051_rails_spider_init.rb
110
+ - lib/config/config.rb
111
+ - lib/config/keywords.json
112
+ - lib/config/proxy.json
113
+ - lib/helper/helper.rb
114
+ - lib/helper/location_helper.rb
115
+ - lib/helper/price_helper.rb
116
+ - lib/helper/tag_helper.rb
117
+ - lib/helper/text_helper.rb
118
+ - lib/helper/time_helper.rb
119
+ - lib/logger.rb
120
+ - lib/proxy/allproxylists.txt
121
+ - lib/proxy/proxy.rb
122
+ - lib/proxy/proxylists.txt
123
+ - lib/rails_spider.rb
124
+ - lib/rails_spider/engine.rb
125
+ - lib/rails_spider/fetchers.rb
126
+ - lib/rails_spider/fetchers/base.rb
127
+ - lib/rails_spider/fetchers/mechanize.rb
128
+ - lib/rails_spider/fetchers/witar.rb
129
+ - lib/rails_spider/parser.rb
130
+ - lib/rails_spider/parser/szlawyers.rb
131
+ - lib/rails_spider/resource.rb
132
+ - lib/rails_spider/strategies.rb
133
+ - lib/rails_spider/version.rb
134
+ - lib/sync_qiniu.rb
135
+ - lib/sync_qiniu/getimages.rb
136
+ - lib/sync_qiniu/getimages_info.rb
137
+ - lib/sync_qiniu/getlocation.rb
138
+ - lib/sync_qiniu/getproxy.rb
139
+ - lib/tasks/the_spider_tasks.rake
140
+ - rakefile
141
+ homepage: ''
142
+ licenses:
143
+ - LGPL-3.0
144
+ metadata: {}
145
+ post_install_message:
146
+ rdoc_options: []
147
+ require_paths:
148
+ - lib
149
+ required_ruby_version: !ruby/object:Gem::Requirement
150
+ requirements:
151
+ - - ">="
152
+ - !ruby/object:Gem::Version
153
+ version: '0'
154
+ required_rubygems_version: !ruby/object:Gem::Requirement
155
+ requirements:
156
+ - - ">="
157
+ - !ruby/object:Gem::Version
158
+ version: '0'
159
+ requirements: []
160
+ rubyforge_project:
161
+ rubygems_version: 2.7.6
162
+ signing_key:
163
+ specification_version: 4
164
+ summary: Summary of RailsSpider.
165
+ test_files: []