adva-static 0.0.3 → 0.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/adva-static.rb +1 -0
- data/lib/adva/static.rb +13 -0
- data/lib/adva/static/export.rb +104 -0
- data/lib/adva/static/export/page.rb +45 -0
- data/lib/adva/static/export/path.rb +49 -0
- data/lib/adva/static/export/queue.rb +27 -0
- data/lib/adva/static/export/store.rb +30 -0
- data/lib/adva/static/export/templates/config.ru +14 -0
- data/lib/adva/static/import.rb +42 -0
- data/lib/adva/static/import/format.rb +58 -0
- data/lib/adva/static/import/model.rb +21 -0
- data/lib/adva/static/import/model/base.rb +78 -0
- data/lib/adva/static/import/model/blog.rb +33 -0
- data/lib/adva/static/import/model/page.rb +28 -0
- data/lib/adva/static/import/model/post.rb +78 -0
- data/lib/adva/static/import/model/section.rb +51 -0
- data/lib/adva/static/import/model/site.rb +59 -0
- data/lib/adva/static/import/request.rb +92 -0
- data/lib/adva/static/import/source.rb +82 -0
- data/lib/adva/static/rack.rb +15 -0
- data/lib/adva/static/rack/export.rb +59 -0
- data/lib/adva/static/rack/request.rb +39 -0
- data/lib/adva/static/rack/static.rb +40 -0
- data/lib/adva/static/rack/watch.rb +88 -0
- data/lib/adva/static/setup.rb +68 -0
- data/lib/adva/static/watch.rb +7 -0
- data/lib/adva/static/watch/handler.rb +57 -0
- data/lib/adva/tasks/static.rb +73 -0
- data/lib/adva_static/version.rb +3 -0
- data/lib/testing/step_definitions.rb +85 -0
- data/lib/testing/test_helper.rb +133 -0
- metadata +35 -5
- data/lib/bundler/repository.rb +0 -118
data/lib/adva-static.rb
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require 'adva/static'
|
data/lib/adva/static.rb
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
require 'adva/core'
|
2
|
+
|
3
|
+
module Adva
|
4
|
+
class Static < ::Rails::Engine
|
5
|
+
autoload :Export, 'adva/static/export'
|
6
|
+
autoload :Import, 'adva/static/import'
|
7
|
+
autoload :Watch, 'adva/static/watch'
|
8
|
+
autoload :Rack, 'adva/static/rack'
|
9
|
+
autoload :Setup, 'adva/static/setup'
|
10
|
+
|
11
|
+
include Adva::Engine
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,104 @@
|
|
1
|
+
require 'nokogiri'
|
2
|
+
require 'uri'
|
3
|
+
require 'benchmark'
|
4
|
+
|
5
|
+
module Adva
|
6
|
+
class Static
|
7
|
+
class Export
|
8
|
+
autoload :Page, 'adva/static/export/page'
|
9
|
+
autoload :Path, 'adva/static/export/path'
|
10
|
+
autoload :Queue, 'adva/static/export/queue'
|
11
|
+
autoload :Store, 'adva/static/export/store'
|
12
|
+
|
13
|
+
attr_reader :app, :queue, :store, :options
|
14
|
+
|
15
|
+
DEFAULT_OPTIONS = {
|
16
|
+
:source => "#{Dir.pwd}/public",
|
17
|
+
:target => "#{Dir.pwd}/export"
|
18
|
+
}
|
19
|
+
|
20
|
+
def initialize(app, options = {})
|
21
|
+
@options = options.reverse_merge!(DEFAULT_OPTIONS)
|
22
|
+
|
23
|
+
@app = app
|
24
|
+
@store = Store.new(target)
|
25
|
+
@queue = Queue.new
|
26
|
+
|
27
|
+
queue.push(options[:queue] || Path.new('/'))
|
28
|
+
|
29
|
+
FileUtils.rm_r(Dir[target.join('*')])
|
30
|
+
end
|
31
|
+
|
32
|
+
def run
|
33
|
+
configure
|
34
|
+
copy_assets
|
35
|
+
process(queue.shift) until queue.empty?
|
36
|
+
end
|
37
|
+
|
38
|
+
protected
|
39
|
+
|
40
|
+
def source
|
41
|
+
@source ||= Pathname.new(options[:source])
|
42
|
+
end
|
43
|
+
|
44
|
+
def target
|
45
|
+
@target ||= Pathname.new(options[:target])
|
46
|
+
end
|
47
|
+
|
48
|
+
def copy_assets
|
49
|
+
%w(images javascripts stylesheets).each do |dir|
|
50
|
+
FileUtils.cp_r(source.join(dir), target.join(dir)) if source.join(dir).exist?
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
def process(path)
|
55
|
+
if page = get(path)
|
56
|
+
store.write(path, page.body)
|
57
|
+
enqueue_urls(page) if path.html?
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
def get(path)
|
62
|
+
result = nil
|
63
|
+
bench = Benchmark.measure do
|
64
|
+
result = app.call(env_for(path))
|
65
|
+
result = follow_redirects(result)
|
66
|
+
end
|
67
|
+
|
68
|
+
status, headers, response = result
|
69
|
+
if status == 200
|
70
|
+
Adva.out.puts "#{bench.total.to_s[0..3]}s: exporting #{path}"
|
71
|
+
Page.new(path, headers['X-Sendfile'] ? File.read(headers['X-Sendfile']) : response)
|
72
|
+
else
|
73
|
+
Adva.out.puts "can not export #{path} (status: #{status})"
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
def follow_redirects(response)
|
78
|
+
response = app.call(env_for(response[1]['Location'])) while redirect?(response[0])
|
79
|
+
response
|
80
|
+
end
|
81
|
+
|
82
|
+
def redirect?(status)
|
83
|
+
status == 301
|
84
|
+
end
|
85
|
+
|
86
|
+
def env_for(path)
|
87
|
+
site = Site.first || raise('could not find any site')
|
88
|
+
name, port = site.host.split(':')
|
89
|
+
::Rack::MockRequest.env_for(path).merge('SERVER_NAME' => name,'SERVER_PORT' => port || '80')
|
90
|
+
end
|
91
|
+
|
92
|
+
def enqueue_urls(page)
|
93
|
+
queue.push(page.urls.reject { |path| path.remote? || store.exists?(path) }.uniq)
|
94
|
+
end
|
95
|
+
|
96
|
+
def configure
|
97
|
+
config = Path.new('config.ru')
|
98
|
+
unless store.exists?(config)
|
99
|
+
store.write(config, File.read(File.expand_path('../export/templates/config.ru', __FILE__)))
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
require 'rack'
|
2
|
+
|
3
|
+
module Adva
|
4
|
+
class Static
|
5
|
+
class Export
|
6
|
+
class Page
|
7
|
+
URL_ATTRIBUTES = {
|
8
|
+
'//a[@href]' => 'href',
|
9
|
+
'//script[@src]' => 'src',
|
10
|
+
'//link[@rel="stylesheet"]' => 'href'
|
11
|
+
}
|
12
|
+
|
13
|
+
attr_reader :url, :response
|
14
|
+
|
15
|
+
def initialize(url, response)
|
16
|
+
@url = Path.new(url)
|
17
|
+
@response = response
|
18
|
+
end
|
19
|
+
|
20
|
+
def urls
|
21
|
+
URL_ATTRIBUTES.inject([]) do |urls, (xpath, name)|
|
22
|
+
urls += dom.xpath(xpath).map { |node| Path.new(node.attributes[name]) }
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def body
|
27
|
+
@body ||= case response
|
28
|
+
when ActionDispatch::Response
|
29
|
+
response.body
|
30
|
+
when ::Rack::File
|
31
|
+
File.read(response.path)
|
32
|
+
else
|
33
|
+
response.to_s
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
protected
|
38
|
+
|
39
|
+
def dom
|
40
|
+
@dom ||= Nokogiri::HTML(body)
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
require 'uri'
|
2
|
+
|
3
|
+
module Adva
|
4
|
+
class Static
|
5
|
+
class Export
|
6
|
+
class Path < String
|
7
|
+
attr_reader :host
|
8
|
+
|
9
|
+
def initialize(path)
|
10
|
+
@host = URI.parse(path.to_s).host rescue 'invalid.host'
|
11
|
+
path = normalize_path(path)
|
12
|
+
super
|
13
|
+
end
|
14
|
+
|
15
|
+
def filename
|
16
|
+
@filename ||= normalize_filename(self)
|
17
|
+
end
|
18
|
+
|
19
|
+
def extname
|
20
|
+
@extname ||= File.extname(self)
|
21
|
+
end
|
22
|
+
|
23
|
+
def html?
|
24
|
+
extname.blank? || extname == '.html'
|
25
|
+
end
|
26
|
+
|
27
|
+
def remote?
|
28
|
+
host.present?
|
29
|
+
end
|
30
|
+
|
31
|
+
protected
|
32
|
+
|
33
|
+
def normalize_path(path)
|
34
|
+
path = URI.parse(path.to_s).path || '/' rescue '/' # extract path
|
35
|
+
path = path[0..-2] if path[-1, 1] == '/' # remove trailing slash
|
36
|
+
path = "/#{path}" unless path[0, 1] == '/' # add leading slash
|
37
|
+
path
|
38
|
+
end
|
39
|
+
|
40
|
+
def normalize_filename(path)
|
41
|
+
path = path[1..-1] if path[0, 1] == '/' # remove leading slash
|
42
|
+
path = 'index' if path.empty? # use 'index' instead of empty paths
|
43
|
+
path = (html? ? "#{path.gsub(extname, '')}.html" : path) # add .html extension if necessary
|
44
|
+
path
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
module Adva
|
2
|
+
class Static
|
3
|
+
class Export
|
4
|
+
class Queue < Array
|
5
|
+
def push(*elements)
|
6
|
+
elements = Array(elements).flatten.uniq
|
7
|
+
elements.reject! { |element| seen?(element) }
|
8
|
+
seen(elements)
|
9
|
+
super
|
10
|
+
end
|
11
|
+
|
12
|
+
def seen?(element)
|
13
|
+
log.include?(element)
|
14
|
+
end
|
15
|
+
|
16
|
+
def seen(elements)
|
17
|
+
@log = log.concat(elements)
|
18
|
+
log.uniq!
|
19
|
+
end
|
20
|
+
|
21
|
+
def log
|
22
|
+
@log ||= []
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
require 'fileutils'
|
2
|
+
|
3
|
+
module Adva
|
4
|
+
class Static
|
5
|
+
class Export
|
6
|
+
class Store
|
7
|
+
attr_reader :dir
|
8
|
+
|
9
|
+
def initialize(dir)
|
10
|
+
@dir = Pathname.new(dir.to_s)
|
11
|
+
FileUtils.mkdir_p(dir)
|
12
|
+
end
|
13
|
+
|
14
|
+
def exists?(path)
|
15
|
+
File.exists?(dir.join(path.filename))
|
16
|
+
end
|
17
|
+
|
18
|
+
def write(path, body)
|
19
|
+
path = dir.join(path.filename)
|
20
|
+
FileUtils.mkdir_p(File.dirname(path))
|
21
|
+
File.open(path, 'w+') { |f| f.write(body) }
|
22
|
+
end
|
23
|
+
|
24
|
+
def purge(path)
|
25
|
+
dir.join(path.filename).delete rescue Errno::ENOENT
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
Dir.chdir('..') until File.exists?('config/environment.rb')
|
2
|
+
|
3
|
+
require 'config/environment.rb'
|
4
|
+
|
5
|
+
Rails::Application.configure do
|
6
|
+
ActionController::Base.allow_forgery_protection = false
|
7
|
+
end
|
8
|
+
|
9
|
+
use Adva::Static::Rack::Watch
|
10
|
+
use Adva::Static::Rack::Export
|
11
|
+
use Adva::Static::Rack::Static, ::File.expand_path('../export', __FILE__)
|
12
|
+
|
13
|
+
puts 'listening.'
|
14
|
+
run Rails.application
|
@@ -0,0 +1,42 @@
|
|
1
|
+
module Adva
|
2
|
+
class Static
|
3
|
+
class Import
|
4
|
+
autoload :Format, 'adva/static/import/format'
|
5
|
+
autoload :Model, 'adva/static/import/model'
|
6
|
+
autoload :Request, 'adva/static/import/request'
|
7
|
+
autoload :Source, 'adva/static/import/source'
|
8
|
+
|
9
|
+
attr_reader :root
|
10
|
+
|
11
|
+
def initialize(options = {})
|
12
|
+
@root = Pathname.new(File.expand_path(options[:source] || 'import'))
|
13
|
+
end
|
14
|
+
|
15
|
+
def run
|
16
|
+
Adva.out.puts "importing from #{root}"
|
17
|
+
Account.all.each(&:destroy)
|
18
|
+
Model::Site.new(root).updated_record.save!
|
19
|
+
end
|
20
|
+
|
21
|
+
def import(path)
|
22
|
+
model = recognize(path).first
|
23
|
+
model.updated_record.save! if model
|
24
|
+
end
|
25
|
+
|
26
|
+
def request_for(path)
|
27
|
+
model = recognize(path).first
|
28
|
+
Request.new(model.source, model.record, model.attributes)
|
29
|
+
end
|
30
|
+
|
31
|
+
protected
|
32
|
+
|
33
|
+
def source(path)
|
34
|
+
Source.new(path, root)
|
35
|
+
end
|
36
|
+
|
37
|
+
def recognize(path)
|
38
|
+
Model.recognize([source(path)])
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,58 @@
|
|
1
|
+
module Adva
|
2
|
+
class Static
|
3
|
+
class Import
|
4
|
+
module Format
|
5
|
+
def self.for(path)
|
6
|
+
name = File.extname(path).gsub('.', '').camelize
|
7
|
+
const_get(name).new(path) if name.present?
|
8
|
+
end
|
9
|
+
|
10
|
+
class Base
|
11
|
+
attr_reader :path
|
12
|
+
|
13
|
+
def initialize(path)
|
14
|
+
@path = path
|
15
|
+
end
|
16
|
+
|
17
|
+
def load(target)
|
18
|
+
data.each do |name, value|
|
19
|
+
define_attribute(target, name) if define_attribute?(target, name)
|
20
|
+
target.instance_variable_set(:"@#{name}", value)
|
21
|
+
end if data.is_a?(Hash)
|
22
|
+
end
|
23
|
+
|
24
|
+
def define_attribute?(target, name)
|
25
|
+
!target.attribute_name?(name) && target.column_name?(name)
|
26
|
+
end
|
27
|
+
|
28
|
+
def define_attribute(target, name)
|
29
|
+
target.attribute_names << name
|
30
|
+
target.attribute_names.uniq!
|
31
|
+
target.class.send(:attr_reader, name) unless target.respond_to?(name)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
class Yml < Base
|
36
|
+
def data
|
37
|
+
@data ||= YAML.load_file(path)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
class Jekyll < Base
|
42
|
+
def data
|
43
|
+
@data ||= begin
|
44
|
+
file =~ /^(---\s*\n.*?\n?)^---\s*$\n?(.*)/m
|
45
|
+
data = YAML.load($1) rescue {}
|
46
|
+
data.merge!(:body => $2) if $2
|
47
|
+
data
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
def file
|
52
|
+
@file ||= File.read(path)
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
module Adva
|
2
|
+
class Static
|
3
|
+
class Import
|
4
|
+
module Model
|
5
|
+
autoload :Base, 'adva/static/import/model/base'
|
6
|
+
autoload :Blog, 'adva/static/import/model/blog'
|
7
|
+
autoload :Page, 'adva/static/import/model/page'
|
8
|
+
autoload :Post, 'adva/static/import/model/post'
|
9
|
+
autoload :Section, 'adva/static/import/model/section'
|
10
|
+
autoload :Site, 'adva/static/import/model/site'
|
11
|
+
|
12
|
+
class << self
|
13
|
+
def recognize(sources)
|
14
|
+
types = [Site, Post, Section]
|
15
|
+
types.map { |type| type.recognize(sources) }.flatten.compact.sort
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,78 @@
|
|
1
|
+
require 'core_ext/ruby/array/flatten_once'
|
2
|
+
|
3
|
+
module Adva
|
4
|
+
class Static
|
5
|
+
class Import
|
6
|
+
module Model
|
7
|
+
class Base
|
8
|
+
attr_reader :source, :attribute_names
|
9
|
+
|
10
|
+
def initialize(source)
|
11
|
+
@source = source
|
12
|
+
load
|
13
|
+
end
|
14
|
+
|
15
|
+
def attributes
|
16
|
+
attributes = attribute_names.map { |name| [name, self.send(name)] unless self.send(name).nil? }
|
17
|
+
attributes = Hash[*attributes.compact.flatten_once]
|
18
|
+
record && record.id ? attributes.merge(:id => record.id.to_s) : attributes
|
19
|
+
end
|
20
|
+
|
21
|
+
def attribute_name?(name)
|
22
|
+
attribute_names.include?(name.to_sym)
|
23
|
+
end
|
24
|
+
|
25
|
+
def column_name?(name)
|
26
|
+
model.column_names.include?(name.to_s)
|
27
|
+
end
|
28
|
+
|
29
|
+
def updated_record
|
30
|
+
record.tap { |record| record.attributes = attributes }
|
31
|
+
end
|
32
|
+
|
33
|
+
def model
|
34
|
+
self.class.name.demodulize.constantize
|
35
|
+
end
|
36
|
+
|
37
|
+
def site_id
|
38
|
+
site.id.to_s
|
39
|
+
end
|
40
|
+
|
41
|
+
def slug
|
42
|
+
source.basename
|
43
|
+
end
|
44
|
+
|
45
|
+
def path
|
46
|
+
source.path
|
47
|
+
end
|
48
|
+
|
49
|
+
def body
|
50
|
+
@body || ''
|
51
|
+
end
|
52
|
+
|
53
|
+
def updated_at
|
54
|
+
source.mtime
|
55
|
+
end
|
56
|
+
|
57
|
+
def loadable
|
58
|
+
@loadable ||= source.full_path
|
59
|
+
end
|
60
|
+
|
61
|
+
def load
|
62
|
+
if loadable.exist?
|
63
|
+
format = Format.for(loadable) and format.load(self)
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
def ==(other)
|
68
|
+
source == other
|
69
|
+
end
|
70
|
+
|
71
|
+
def <=>(other)
|
72
|
+
source <=> other.source
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|