webscraper_framework 0.1.716 → 0.1.717
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/webscraper_framework/application.rb +80 -1
- data/lib/webscraper_framework/cli.rb +9 -0
- metadata +85 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9c43021752c03d1223f4e2ceb8c9d71341db09a0
|
4
|
+
data.tar.gz: a62c5fb9fd2e89984dd55a4b9ba695cb68930c3c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ebc950f510dca10b78d8ea108482d2249ceffea71cede15653114577fa2b17ff6a55e8812cc4aed9476f926db6af5ccbab22c007d839202a499e392400e2007d
|
7
|
+
data.tar.gz: a8a88c4f997f3f8c707908d5af55138f4c21e365a075974915da4fa0747b508f1a6264b3ba54da65a18db614f245c657d5105ddf46c97e4b1ac8ec2153e21397
|
@@ -1,9 +1,24 @@
|
|
1
1
|
require 'yaml'
|
2
|
+
|
3
|
+
|
4
|
+
|
5
|
+
require 'active_support/all'
|
6
|
+
|
7
|
+
|
2
8
|
module WebscraperFramework
|
3
9
|
class Application
|
10
|
+
|
11
|
+
|
12
|
+
def render_file(file, params= {})
|
13
|
+
gemdir = Gem.loaded_specs["webscraper_framework"].gem_dir
|
14
|
+
base_template = File.read(gemdir + "/views/base.html.haml")
|
15
|
+
content = Haml::Engine.new(File.read(gemdir + "/views/#{file}.html.haml")).render(Object.new, models: MODELS, scrapers: SCRAPERS, **params)
|
16
|
+
return Haml::Engine.new(base_template).render(Object.new, content: content, models: MODELS, scrapers: SCRAPERS)
|
17
|
+
end
|
18
|
+
|
4
19
|
def initialize
|
5
20
|
|
6
|
-
model_files = Dir["./
|
21
|
+
model_files = Dir["./modelodels/*"]
|
7
22
|
helper_files = Dir["./helpers/*"]
|
8
23
|
scraper_files = Dir["./scrapers/*"]
|
9
24
|
|
@@ -14,6 +29,70 @@ module WebscraperFramework
|
|
14
29
|
puts helper_files
|
15
30
|
puts scraper_files
|
16
31
|
puts "-"
|
32
|
+
|
33
|
+
|
34
|
+
|
35
|
+
|
36
|
+
data = {}
|
37
|
+
|
38
|
+
SCRAPERS = []
|
39
|
+
MODELS = []
|
40
|
+
|
41
|
+
model_files.each do |filename|
|
42
|
+
require filename
|
43
|
+
puts filename
|
44
|
+
MODELS << Object.const_get(filename.split("/").last.split(".").first.camelize)
|
45
|
+
end
|
46
|
+
|
47
|
+
helper_files.each do |filename|
|
48
|
+
require filename
|
49
|
+
puts filename
|
50
|
+
end
|
51
|
+
|
52
|
+
scraper_files.each do |filename|
|
53
|
+
require filename
|
54
|
+
puts filename
|
55
|
+
SCRAPERS << Object.const_get(filename.split("/").last.split(".").first.camelize)
|
56
|
+
Object.const_get(filename.split("/").last.split(".").first.camelize).data
|
57
|
+
end
|
58
|
+
|
59
|
+
|
60
|
+
|
61
|
+
|
62
|
+
## System routes
|
63
|
+
|
64
|
+
get '/' do
|
65
|
+
render_file("home")
|
66
|
+
end
|
67
|
+
|
68
|
+
get '/models' do
|
69
|
+
render_file("models")
|
70
|
+
end
|
71
|
+
|
72
|
+
get '/scrapers' do
|
73
|
+
render_file("scrapers")
|
74
|
+
end
|
75
|
+
|
76
|
+
|
77
|
+
get '/models/:name/csv' do
|
78
|
+
|
79
|
+
content_type 'application/csv'
|
80
|
+
attachment "myfilename.csv"
|
81
|
+
m = MODELS.select { |model| params[:name] == model.name.underscore }.first
|
82
|
+
csv_string = CSV.generate(col_sep: ";") do |csv|
|
83
|
+
header_row = ["id"] + m.fields
|
84
|
+
m.all.each do |id, obj|
|
85
|
+
csv << [id] + m.fields.map {|a| obj[a]}
|
86
|
+
end
|
87
|
+
end
|
88
|
+
# render_file("model", { })
|
89
|
+
end
|
90
|
+
get '/models/:name' do
|
91
|
+
render_file("model", {model: MODELS.select { |model| params[:name] == model.name.underscore }.first })
|
92
|
+
end
|
93
|
+
|
94
|
+
|
95
|
+
puts MODELS
|
17
96
|
end
|
18
97
|
end
|
19
98
|
end
|
@@ -46,9 +46,18 @@ module WebscraperFramework
|
|
46
46
|
`mkdir #{project_name}`
|
47
47
|
if File.directory?("./#{project_name}")
|
48
48
|
puts "#{"[Success]".green.bold} created directory!\t ./#{project_name}/"
|
49
|
+
|
49
50
|
`mkdir #{project_name}/config`
|
50
51
|
puts "#{"[Success]".green.bold} created directory!\t ./#{project_name}/config"
|
51
52
|
|
53
|
+
`mkdir #{project_name}/models`
|
54
|
+
puts "#{"[Success]".green.bold} created directory!\t ./#{project_name}/models"
|
55
|
+
|
56
|
+
`mkdir #{project_name}/scrapers`
|
57
|
+
puts "#{"[Success]".green.bold} created directory!\t ./#{project_name}/scrapers"
|
58
|
+
|
59
|
+
`mkdir #{project_name}/helpers`
|
60
|
+
puts "#{"[Success]".green.bold} created directory!\t ./#{project_name}/helpers"
|
52
61
|
else
|
53
62
|
puts "Something went wrong. Exiting...".red.bold
|
54
63
|
goodbye
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: webscraper_framework
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.717
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Rene van Pelt
|
@@ -52,6 +52,90 @@ dependencies:
|
|
52
52
|
- - ">="
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: activesupport
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '1.4'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '1.4'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: sinatra
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :runtime
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: open-uri
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
90
|
+
type: :runtime
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: uri
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
type: :runtime
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ">="
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: csv
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - ">="
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
type: :runtime
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - ">="
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0'
|
125
|
+
- !ruby/object:Gem::Dependency
|
126
|
+
name: haml
|
127
|
+
requirement: !ruby/object:Gem::Requirement
|
128
|
+
requirements:
|
129
|
+
- - ">="
|
130
|
+
- !ruby/object:Gem::Version
|
131
|
+
version: '0'
|
132
|
+
type: :runtime
|
133
|
+
prerelease: false
|
134
|
+
version_requirements: !ruby/object:Gem::Requirement
|
135
|
+
requirements:
|
136
|
+
- - ">="
|
137
|
+
- !ruby/object:Gem::Version
|
138
|
+
version: '0'
|
55
139
|
description:
|
56
140
|
email:
|
57
141
|
executables:
|