webscraper_framework 0.1.720 → 0.1.721
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/webscraper_framework/application.rb +60 -56
- data/lib/webscraper_framework/cli.rb +2 -2
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 57d72e976adf94b094b8f3bbce54059cc190b82f
|
4
|
+
data.tar.gz: 751813962105c89e61dcd2e62555aab2e203d5e7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d2a8a61dc467307fa9ff5ea13b0c9932f21f4569fc9c42ded60eb6afcc2d16b80c8613cc95d9dec73c29659bbad8afcb51173279a769f4bfb3c4095002e06424
|
7
|
+
data.tar.gz: d31a68d0401c8d3b2968b51385cbb156315ee4aa2babf823273f306b59cb8da3d3b1eb5d60771f4504dce2ec20f858f96b5aa91bc48fd420ca9d21e4372a9284
|
@@ -8,8 +8,11 @@ require 'open-uri'
|
|
8
8
|
require 'uri'
|
9
9
|
require 'csv'
|
10
10
|
|
11
|
+
require 'sinatra'
|
12
|
+
require "sinatra/base"
|
13
|
+
|
11
14
|
module WebscraperFramework
|
12
|
-
class Application
|
15
|
+
class Application < Sinatra::Base
|
13
16
|
|
14
17
|
|
15
18
|
def render_file(file, params= {})
|
@@ -19,83 +22,84 @@ module WebscraperFramework
|
|
19
22
|
return Haml::Engine.new(base_template).render(Object.new, content: content, models: @@models, scrapers: @@scrapers)
|
20
23
|
end
|
21
24
|
|
22
|
-
|
23
|
-
|
24
|
-
model_files = Dir["./modelodels/*"]
|
25
|
-
helper_files = Dir["./helpers/*"]
|
26
|
-
scraper_files = Dir["./scrapers/*"]
|
25
|
+
|
27
26
|
|
27
|
+
model_files = Dir["./modelodels/*"]
|
28
|
+
helper_files = Dir["./helpers/*"]
|
29
|
+
scraper_files = Dir["./scrapers/*"]
|
28
30
|
|
29
|
-
puts "-"
|
30
|
-
puts Dir["./"]
|
31
|
-
puts model_files
|
32
|
-
puts helper_files
|
33
|
-
puts scraper_files
|
34
|
-
puts "-"
|
35
31
|
|
32
|
+
puts "-"
|
33
|
+
puts Dir["./"]
|
34
|
+
puts model_files
|
35
|
+
puts helper_files
|
36
|
+
puts scraper_files
|
37
|
+
puts "-"
|
36
38
|
|
37
39
|
|
38
40
|
|
39
|
-
data = {}
|
40
41
|
|
41
|
-
|
42
|
-
@@models = []
|
42
|
+
data = {}
|
43
43
|
|
44
|
-
|
45
|
-
|
46
|
-
puts filename
|
47
|
-
@@models << Object.const_get(filename.split("/").last.split(".").first.camelize)
|
48
|
-
end
|
44
|
+
@@scrapers = []
|
45
|
+
@@models = []
|
49
46
|
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
47
|
+
model_files.each do |filename|
|
48
|
+
require filename
|
49
|
+
puts filename
|
50
|
+
@@models << Object.const_get(filename.split("/").last.split(".").first.camelize)
|
51
|
+
end
|
54
52
|
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
Object.const_get(filename.split("/").last.split(".").first.camelize).data
|
60
|
-
end
|
53
|
+
helper_files.each do |filename|
|
54
|
+
require filename
|
55
|
+
puts filename
|
56
|
+
end
|
61
57
|
|
58
|
+
scraper_files.each do |filename|
|
59
|
+
require filename
|
60
|
+
puts filename
|
61
|
+
@@scrapers << Object.const_get(filename.split("/").last.split(".").first.camelize)
|
62
|
+
Object.const_get(filename.split("/").last.split(".").first.camelize).data
|
63
|
+
end
|
62
64
|
|
63
65
|
|
64
66
|
|
65
|
-
## System routes
|
66
67
|
|
67
|
-
|
68
|
-
render_file("home")
|
69
|
-
end
|
68
|
+
## System routes
|
70
69
|
|
71
|
-
|
72
|
-
|
73
|
-
|
70
|
+
get '/' do
|
71
|
+
render_file("home")
|
72
|
+
end
|
74
73
|
|
75
|
-
|
76
|
-
|
77
|
-
|
74
|
+
get '/models' do
|
75
|
+
render_file("models")
|
76
|
+
end
|
78
77
|
|
78
|
+
get '/scrapers' do
|
79
|
+
render_file("scrapers")
|
80
|
+
end
|
79
81
|
|
80
|
-
get '/models/:name/csv' do
|
81
82
|
|
82
|
-
|
83
|
-
attachment "myfilename.csv"
|
84
|
-
m = @@models.select { |model| params[:name] == model.name.underscore }.first
|
85
|
-
csv_string = CSV.generate(col_sep: ";") do |csv|
|
86
|
-
header_row = ["id"] + m.fields
|
87
|
-
m.all.each do |id, obj|
|
88
|
-
csv << [id] + m.fields.map {|a| obj[a]}
|
89
|
-
end
|
90
|
-
end
|
91
|
-
# render_file("model", { })
|
92
|
-
end
|
93
|
-
get '/models/:name' do
|
94
|
-
render_file("model", {model: @@models.select { |model| params[:name] == model.name.underscore }.first })
|
95
|
-
end
|
83
|
+
get '/models/:name/csv' do
|
96
84
|
|
85
|
+
content_type 'application/csv'
|
86
|
+
attachment "myfilename.csv"
|
87
|
+
m = @@models.select { |model| params[:name] == model.name.underscore }.first
|
88
|
+
csv_string = CSV.generate(col_sep: ";") do |csv|
|
89
|
+
header_row = ["id"] + m.fields
|
90
|
+
m.all.each do |id, obj|
|
91
|
+
csv << [id] + m.fields.map {|a| obj[a]}
|
92
|
+
end
|
93
|
+
end
|
94
|
+
# render_file("model", { })
|
95
|
+
end
|
97
96
|
|
98
|
-
|
97
|
+
get '/models/:name' do
|
98
|
+
render_file("model", {model: @@models.select { |model| params[:name] == model.name.underscore }.first })
|
99
99
|
end
|
100
|
+
|
101
|
+
|
102
|
+
puts @@models
|
103
|
+
|
100
104
|
end
|
101
105
|
end
|
@@ -24,11 +24,11 @@ module WebscraperFramework
|
|
24
24
|
end
|
25
25
|
|
26
26
|
def start_server
|
27
|
-
WebscraperFramework::Application.
|
27
|
+
WebscraperFramework::Application.run!
|
28
28
|
end
|
29
29
|
|
30
30
|
def create_project
|
31
|
-
puts "Hello! Welcome Webscraper Framework (name not definitive, hahaha!)".green.bold
|
31
|
+
puts "Hello! Welcome to Webscraper Framework (name not definitive, hahaha!)".green.bold
|
32
32
|
puts "This is a work in progress... nothing works.".blue.bold
|
33
33
|
puts "Type 'exit' to quit".red.bold
|
34
34
|
project_name = nil
|