zerofetcher 0.0.5 → 0.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/zerofetcher.rb +243 -206
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 07f69907316db9502c48570a9af8709c04bc8cbf
|
|
4
|
+
data.tar.gz: d11a4225908538c7c4d1fe04ef2a7b9debdcde21
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: e07822f820e27c378574936981c630539a718aea5b4f137f2acfe226003ea8513344aeeba115f6a41c906bf7c6b4f1c00ce8cba9efbb6d224637d343e94bc848
|
|
7
|
+
data.tar.gz: 6cbc24f801d986b209f5d59000d8598196965bc28ddfafe26cc240adc5a640dc4ffecfc0ef32a0c51a09b93541dbbacf0734ac4003f12a4f1c738fa118e6afb1
|
data/lib/zerofetcher.rb
CHANGED
|
@@ -5,217 +5,249 @@ require 'json'
|
|
|
5
5
|
require 'fileutils'
|
|
6
6
|
require 'pathname'
|
|
7
7
|
require 'open-uri'
|
|
8
|
+
require 'logger'
|
|
8
9
|
|
|
9
|
-
#app_path = File.dirname(__FILE__)
|
|
10
|
-
#require './code/functions.rb'
|
|
11
10
|
require 'code/JekyllFile.rb'
|
|
12
11
|
require 'code/JekyllPost.rb'
|
|
13
12
|
|
|
14
13
|
class ZeroFetcher
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
14
|
+
@@logger
|
|
15
|
+
@@jekyll_path
|
|
16
|
+
|
|
17
|
+
def self.run
|
|
18
|
+
# Paths
|
|
19
|
+
app_path = File.dirname(__FILE__)
|
|
20
|
+
jekyll_path = Dir.pwd
|
|
21
|
+
@@jekyll_path = jekyll_path
|
|
22
|
+
|
|
23
|
+
# App Requires
|
|
24
|
+
FileUtils::mkdir_p jekyll_path+'/_logs'
|
|
25
|
+
FileUtils::mkdir_p jekyll_path+'/_data'
|
|
26
|
+
FileUtils::mkdir_p jekyll_path+'/_posts'
|
|
27
|
+
FileUtils::mkdir_p jekyll_path+'/assets'
|
|
28
|
+
FileUtils::mkdir_p jekyll_path+'/assets/images'
|
|
29
|
+
FileUtils::mkdir_p jekyll_path+'/assets/files'
|
|
30
|
+
FileUtils::mkdir_p jekyll_path+'/_includes'
|
|
31
|
+
FileUtils::mkdir_p jekyll_path+'/_includes/pages'
|
|
32
|
+
FileUtils::mkdir_p jekyll_path+'/_includes/content_blocks'
|
|
33
|
+
|
|
34
|
+
# Setup Logger
|
|
35
|
+
@@logger = Logger.new(jekyll_path + '/_logs/logfile.log', 10, 1024000)
|
|
36
|
+
|
|
37
|
+
if !Pathname.new(jekyll_path+"/_config.yml").file?
|
|
38
|
+
self.log('_config.yml not found')
|
|
39
|
+
abort('_config.yml not found')
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
config = YAML.load_file(jekyll_path+"/_config.yml")
|
|
43
|
+
|
|
44
|
+
if !config.has_key?('fetcher')
|
|
45
|
+
self.log('No Fetcher Info Found')
|
|
46
|
+
abort('No Fetcher Info Found')
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
end_point = config['fetcher']['end_point']
|
|
50
|
+
|
|
51
|
+
data = self.loadDataFromApi(end_point, config['fetcher']['api_key'], config['fetcher']['site_id'])
|
|
52
|
+
|
|
53
|
+
# Site Info
|
|
54
|
+
if data.key?("site")
|
|
55
|
+
File.write(jekyll_path+'/_data/site.json', data['site'].to_json)
|
|
56
|
+
self.log('Writing File /_data/site.json')
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# Pages
|
|
60
|
+
if data.key?("pages")
|
|
61
|
+
pages_files_saved = Array.new
|
|
62
|
+
|
|
63
|
+
pages_folder = jekyll_path+'/_includes/pages'
|
|
64
|
+
|
|
65
|
+
puts 'Pages - ' + data['pages'].length.to_s
|
|
66
|
+
self.log('Pages - ' + data['pages'].length.to_s)
|
|
67
|
+
|
|
68
|
+
# array to store hash info on all pages
|
|
69
|
+
pages = Array.new
|
|
70
|
+
|
|
71
|
+
# Loop through each page
|
|
72
|
+
data['pages'].each do |page|
|
|
73
|
+
puts page['name']
|
|
74
|
+
self.log(' - ' + page['name'])
|
|
75
|
+
|
|
76
|
+
jpage = JekyllFile.new(jekyll_path, page, 'page')
|
|
77
|
+
|
|
78
|
+
jpage.saveContentFile(pages_folder+'/'+page['url']+'.md', page['contents'])
|
|
79
|
+
jpage.savePageFile
|
|
80
|
+
#pages_files_saved.push(jpage.getFileName)
|
|
81
|
+
pages_files_saved.push( Pathname.new( jpage.getFileName ).basename.to_s )
|
|
82
|
+
self.log(" - Saved:"+jpage.getFileName)
|
|
83
|
+
|
|
84
|
+
# Add to pages array
|
|
85
|
+
pages.push({
|
|
86
|
+
'name' => page['name'],
|
|
87
|
+
'slug' => page['slug'],
|
|
88
|
+
'url' => page['url'],
|
|
89
|
+
'meta_title' => (page['meta_title']) ? page['meta_title'] : page['name'],
|
|
90
|
+
'meta_description' => page['meta_description'],
|
|
91
|
+
'short_description' => page['short_description'],
|
|
92
|
+
})
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
File.write(jekyll_path+'/_data/pages.json', pages.to_json)
|
|
96
|
+
self.log("Writing File /_data/pages.json")
|
|
97
|
+
|
|
98
|
+
# Clean Unused Pages
|
|
99
|
+
existing_pages_files = Dir.glob(File.join(jekyll_path, "*.md"))
|
|
100
|
+
|
|
101
|
+
existing_pages_files.each do |file|
|
|
102
|
+
file_basename = Pathname.new(file).basename.to_s
|
|
103
|
+
if 'README.md' != file_basename
|
|
104
|
+
if !pages_files_saved.include?( file_basename )
|
|
105
|
+
self.log(" - Deleting:"+jfile)
|
|
106
|
+
FileUtils.rm(file)
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
# Variables
|
|
113
|
+
if data.key?("variables")
|
|
114
|
+
File.write(jekyll_path+'/_data/vars.json', data['variables'].to_json)
|
|
115
|
+
self.log("Writing File /_data/vars.json")
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
# Content Blocks
|
|
119
|
+
if data.key?("content_blocks")
|
|
120
|
+
puts 'Content Blocks - ' + data['content_blocks'].length.to_s
|
|
121
|
+
self.log('Content Blocks - ' + data['content_blocks'].length.to_s)
|
|
122
|
+
cb_folder = jekyll_path+'/_includes/content_blocks'
|
|
123
|
+
|
|
124
|
+
data['content_blocks'].each do |key, content_block|
|
|
125
|
+
self.log('writing File /_includes/content_blocks/'+key+'.md')
|
|
126
|
+
File.write(cb_folder+'/'+key+'.md' , content_block.to_s)
|
|
127
|
+
end
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
# Posts
|
|
132
|
+
existing_post_files = Dir.glob(File.join(jekyll_path+'/_posts', "*"))
|
|
133
|
+
|
|
134
|
+
# Remove all post files
|
|
135
|
+
existing_post_files.each do |file|
|
|
136
|
+
self.log('Deleting Post File ' + file)
|
|
137
|
+
FileUtils.rm(file)
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
# Post files
|
|
141
|
+
if data.key?("posts")
|
|
142
|
+
puts "Posts - " + data['posts'].length.to_s
|
|
143
|
+
data['posts'].each do |post|
|
|
144
|
+
jpost = JekyllPost.new(jekyll_path+'/_posts', post)
|
|
145
|
+
jpost.savePageFile
|
|
146
|
+
self.log('Saving Post File ' + post['date']+'-'+post['slug']+'.md')
|
|
147
|
+
end
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
# Galleries
|
|
152
|
+
if data.key?("galleries")
|
|
153
|
+
FileUtils::mkdir_p jekyll_path+'/assets/images/galleries'
|
|
154
|
+
|
|
155
|
+
puts "Galleries - " + data['galleries'].length.to_s
|
|
156
|
+
self.log("Galleries - " + data['galleries'].length.to_s)
|
|
157
|
+
|
|
158
|
+
# Save Json Data
|
|
159
|
+
File.write(jekyll_path+'/_data/galleries.json', data['galleries'].to_json)
|
|
160
|
+
self.log('Writing file /_data/galleries.json')
|
|
161
|
+
|
|
162
|
+
# Save Photos
|
|
163
|
+
data['galleries'].each do |gallery|
|
|
164
|
+
FileUtils::mkdir_p jekyll_path+'/assets/images/galleries/' + gallery['id']
|
|
165
|
+
|
|
166
|
+
puts " - " + gallery['name'] + ' has ' + gallery['photos'].length.to_s + ' photos'
|
|
167
|
+
self.log(" - " + gallery['name'] + ' has ' + gallery['photos'].length.to_s + ' photos')
|
|
168
|
+
|
|
169
|
+
gallery['photos'].each do |photo|
|
|
170
|
+
self.getAndSaveFile(end_point + '/media/images/galleries/photos/' + gallery['id'] + '/' + photo['image'], jekyll_path+'/assets/images/galleries/' + gallery['id'] + '/' + photo['image'])
|
|
171
|
+
end
|
|
172
|
+
end
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
# Calendar
|
|
176
|
+
if data.key?("calendar")
|
|
177
|
+
FileUtils::mkdir_p jekyll_path+'/assets/images/calendar'
|
|
178
|
+
FileUtils::mkdir_p jekyll_path+'/assets/files/calendar'
|
|
179
|
+
|
|
180
|
+
puts "Calendar - " + data['calendar'].length.to_s
|
|
181
|
+
self.log("Calendar - " + data['calendar'].length.to_s)
|
|
182
|
+
|
|
183
|
+
# Save Json Data
|
|
184
|
+
File.write(jekyll_path+'/_data/calendar.json', data['calendar'].to_json)
|
|
185
|
+
self.log('Writing File /_data/calendar.json')
|
|
186
|
+
|
|
187
|
+
# Save Photos / Files
|
|
188
|
+
if data['calendar'].key?("events")
|
|
189
|
+
data['calendar']['events'].each do |event|
|
|
190
|
+
if event['image']
|
|
191
|
+
self.getAndSaveFile(end_point + '/media/images/calendar/' + event['image'], jekyll_path+'/assets/images/calendar/' + event['image'])
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
if event['file']
|
|
195
|
+
self.getAndSaveFile(end_point + '/media/files/calendar/' + event['file'], jekyll_path+'/assets/files/calendar/' + event['file'])
|
|
196
|
+
end
|
|
197
|
+
end
|
|
198
|
+
end
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
# Custom Collections
|
|
203
|
+
if data.key?("collections")
|
|
204
|
+
# Create empty collection hash
|
|
205
|
+
collection_config = {};
|
|
206
|
+
|
|
207
|
+
data['collections'].each do |collection_type,collection_data|
|
|
208
|
+
puts "Collection - " + collection_type + ' - ' + collection_data.length.to_s
|
|
209
|
+
self.log("Collection - " + collection_type + ' - ' + collection_data.length.to_s)
|
|
210
|
+
|
|
211
|
+
# Add to collection hash
|
|
212
|
+
collection_config[ collection_type ] = ['output' => true];
|
|
213
|
+
|
|
214
|
+
# Save Json Data
|
|
215
|
+
File.write(jekyll_path+'/_data/'+collection_type+'.json', collection_data.to_json)
|
|
216
|
+
self.log('Writing File /_data/'+collection_type+'.json')
|
|
217
|
+
|
|
218
|
+
# Directory Check
|
|
219
|
+
collection_path = jekyll_path+'/_' + collection_type
|
|
220
|
+
FileUtils::mkdir_p collection_path
|
|
221
|
+
|
|
222
|
+
# Create files saved array
|
|
223
|
+
files_saved = Array.new
|
|
224
|
+
|
|
225
|
+
# Loop through the collection data
|
|
226
|
+
collection_data.each do |row|
|
|
227
|
+
jfile = JekyllFile.new(collection_path, row, 'collection')
|
|
228
|
+
jfile.savePageFile
|
|
229
|
+
|
|
230
|
+
files_saved.push( Pathname.new( jfile.getFileName ).basename.to_s )
|
|
231
|
+
|
|
232
|
+
self.log('Saving Collection File ' + file.getFileName)
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
# Clean up unused collection files
|
|
236
|
+
self.cleanUpFolder(collection_path, files_saved)
|
|
237
|
+
end
|
|
238
|
+
|
|
239
|
+
# Add Collections to _config.tml
|
|
240
|
+
config['collections'] = collection_config
|
|
241
|
+
File.write(jekyll_path+"/_config.yml", config.to_yaml)
|
|
242
|
+
self.log('Writing File /_config.yml')
|
|
243
|
+
end
|
|
244
|
+
|
|
245
|
+
@@logger.close
|
|
246
|
+
end
|
|
247
|
+
|
|
248
|
+
def self.log(data)
|
|
249
|
+
@@logger.debug { data }
|
|
217
250
|
end
|
|
218
|
-
|
|
219
251
|
|
|
220
252
|
def self.readFile(file)
|
|
221
253
|
file = File.open(file, "r")
|
|
@@ -236,6 +268,7 @@ class ZeroFetcher
|
|
|
236
268
|
|
|
237
269
|
def self.getAndSaveFile(source, dest)
|
|
238
270
|
if !Pathname.new(dest).file?
|
|
271
|
+
self.log('Get/Save ' + dest)
|
|
239
272
|
uri = URI.encode(source)
|
|
240
273
|
|
|
241
274
|
open(dest, 'wb') do |file|
|
|
@@ -245,10 +278,14 @@ class ZeroFetcher
|
|
|
245
278
|
end
|
|
246
279
|
|
|
247
280
|
def self.cleanUpFolder(path, files)
|
|
281
|
+
self.log('Clean up Path ' + path)
|
|
248
282
|
existing_files = Dir.glob(File.join(path, "*"))
|
|
249
283
|
|
|
250
284
|
existing_files.each do |file|
|
|
251
|
-
|
|
285
|
+
file_basename = Pathname.new(file).basename.to_s
|
|
286
|
+
|
|
287
|
+
if !files.include?( file_basename )
|
|
288
|
+
self.log(' - ' + file)
|
|
252
289
|
FileUtils.rm(file)
|
|
253
290
|
end
|
|
254
291
|
end
|