webscraper_framework 0.1.720 → 0.1.721
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/webscraper_framework/application.rb +60 -56
- data/lib/webscraper_framework/cli.rb +2 -2
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 57d72e976adf94b094b8f3bbce54059cc190b82f
|
4
|
+
data.tar.gz: 751813962105c89e61dcd2e62555aab2e203d5e7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d2a8a61dc467307fa9ff5ea13b0c9932f21f4569fc9c42ded60eb6afcc2d16b80c8613cc95d9dec73c29659bbad8afcb51173279a769f4bfb3c4095002e06424
|
7
|
+
data.tar.gz: d31a68d0401c8d3b2968b51385cbb156315ee4aa2babf823273f306b59cb8da3d3b1eb5d60771f4504dce2ec20f858f96b5aa91bc48fd420ca9d21e4372a9284
|
@@ -8,8 +8,11 @@ require 'open-uri'
|
|
8
8
|
require 'uri'
|
9
9
|
require 'csv'
|
10
10
|
|
11
|
+
require 'sinatra'
|
12
|
+
require "sinatra/base"
|
13
|
+
|
11
14
|
module WebscraperFramework
|
12
|
-
class Application
|
15
|
+
class Application < Sinatra::Base
|
13
16
|
|
14
17
|
|
15
18
|
def render_file(file, params= {})
|
@@ -19,83 +22,84 @@ module WebscraperFramework
|
|
19
22
|
return Haml::Engine.new(base_template).render(Object.new, content: content, models: @@models, scrapers: @@scrapers)
|
20
23
|
end
|
21
24
|
|
22
|
-
|
23
|
-
|
24
|
-
model_files = Dir["./modelodels/*"]
|
25
|
-
helper_files = Dir["./helpers/*"]
|
26
|
-
scraper_files = Dir["./scrapers/*"]
|
25
|
+
|
27
26
|
|
27
|
+
model_files = Dir["./modelodels/*"]
|
28
|
+
helper_files = Dir["./helpers/*"]
|
29
|
+
scraper_files = Dir["./scrapers/*"]
|
28
30
|
|
29
|
-
puts "-"
|
30
|
-
puts Dir["./"]
|
31
|
-
puts model_files
|
32
|
-
puts helper_files
|
33
|
-
puts scraper_files
|
34
|
-
puts "-"
|
35
31
|
|
32
|
+
puts "-"
|
33
|
+
puts Dir["./"]
|
34
|
+
puts model_files
|
35
|
+
puts helper_files
|
36
|
+
puts scraper_files
|
37
|
+
puts "-"
|
36
38
|
|
37
39
|
|
38
40
|
|
39
|
-
data = {}
|
40
41
|
|
41
|
-
|
42
|
-
@@models = []
|
42
|
+
data = {}
|
43
43
|
|
44
|
-
|
45
|
-
|
46
|
-
puts filename
|
47
|
-
@@models << Object.const_get(filename.split("/").last.split(".").first.camelize)
|
48
|
-
end
|
44
|
+
@@scrapers = []
|
45
|
+
@@models = []
|
49
46
|
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
47
|
+
model_files.each do |filename|
|
48
|
+
require filename
|
49
|
+
puts filename
|
50
|
+
@@models << Object.const_get(filename.split("/").last.split(".").first.camelize)
|
51
|
+
end
|
54
52
|
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
Object.const_get(filename.split("/").last.split(".").first.camelize).data
|
60
|
-
end
|
53
|
+
helper_files.each do |filename|
|
54
|
+
require filename
|
55
|
+
puts filename
|
56
|
+
end
|
61
57
|
|
58
|
+
scraper_files.each do |filename|
|
59
|
+
require filename
|
60
|
+
puts filename
|
61
|
+
@@scrapers << Object.const_get(filename.split("/").last.split(".").first.camelize)
|
62
|
+
Object.const_get(filename.split("/").last.split(".").first.camelize).data
|
63
|
+
end
|
62
64
|
|
63
65
|
|
64
66
|
|
65
|
-
## System routes
|
66
67
|
|
67
|
-
|
68
|
-
render_file("home")
|
69
|
-
end
|
68
|
+
## System routes
|
70
69
|
|
71
|
-
|
72
|
-
|
73
|
-
|
70
|
+
get '/' do
|
71
|
+
render_file("home")
|
72
|
+
end
|
74
73
|
|
75
|
-
|
76
|
-
|
77
|
-
|
74
|
+
get '/models' do
|
75
|
+
render_file("models")
|
76
|
+
end
|
78
77
|
|
78
|
+
get '/scrapers' do
|
79
|
+
render_file("scrapers")
|
80
|
+
end
|
79
81
|
|
80
|
-
get '/models/:name/csv' do
|
81
82
|
|
82
|
-
|
83
|
-
attachment "myfilename.csv"
|
84
|
-
m = @@models.select { |model| params[:name] == model.name.underscore }.first
|
85
|
-
csv_string = CSV.generate(col_sep: ";") do |csv|
|
86
|
-
header_row = ["id"] + m.fields
|
87
|
-
m.all.each do |id, obj|
|
88
|
-
csv << [id] + m.fields.map {|a| obj[a]}
|
89
|
-
end
|
90
|
-
end
|
91
|
-
# render_file("model", { })
|
92
|
-
end
|
93
|
-
get '/models/:name' do
|
94
|
-
render_file("model", {model: @@models.select { |model| params[:name] == model.name.underscore }.first })
|
95
|
-
end
|
83
|
+
get '/models/:name/csv' do
|
96
84
|
|
85
|
+
content_type 'application/csv'
|
86
|
+
attachment "myfilename.csv"
|
87
|
+
m = @@models.select { |model| params[:name] == model.name.underscore }.first
|
88
|
+
csv_string = CSV.generate(col_sep: ";") do |csv|
|
89
|
+
header_row = ["id"] + m.fields
|
90
|
+
m.all.each do |id, obj|
|
91
|
+
csv << [id] + m.fields.map {|a| obj[a]}
|
92
|
+
end
|
93
|
+
end
|
94
|
+
# render_file("model", { })
|
95
|
+
end
|
97
96
|
|
98
|
-
|
97
|
+
get '/models/:name' do
|
98
|
+
render_file("model", {model: @@models.select { |model| params[:name] == model.name.underscore }.first })
|
99
99
|
end
|
100
|
+
|
101
|
+
|
102
|
+
puts @@models
|
103
|
+
|
100
104
|
end
|
101
105
|
end
|
@@ -24,11 +24,11 @@ module WebscraperFramework
|
|
24
24
|
end
|
25
25
|
|
26
26
|
def start_server
|
27
|
-
WebscraperFramework::Application.
|
27
|
+
WebscraperFramework::Application.run!
|
28
28
|
end
|
29
29
|
|
30
30
|
def create_project
|
31
|
-
puts "Hello! Welcome Webscraper Framework (name not definitive, hahaha!)".green.bold
|
31
|
+
puts "Hello! Welcome to Webscraper Framework (name not definitive, hahaha!)".green.bold
|
32
32
|
puts "This is a work in progress... nothing works.".blue.bold
|
33
33
|
puts "Type 'exit' to quit".red.bold
|
34
34
|
project_name = nil
|