staticd 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/bin/staticd +7 -0
- data/lib/rack/auth/hmac.rb +81 -0
- data/lib/rack/request_time.rb +19 -0
- data/lib/staticd.rb +7 -0
- data/lib/staticd/api.rb +356 -0
- data/lib/staticd/app.rb +130 -0
- data/lib/staticd/cache_engine.rb +45 -0
- data/lib/staticd/cli.rb +70 -0
- data/lib/staticd/config.rb +115 -0
- data/lib/staticd/database.rb +41 -0
- data/lib/staticd/datastore.rb +64 -0
- data/lib/staticd/datastores/local.rb +48 -0
- data/lib/staticd/datastores/s3.rb +63 -0
- data/lib/staticd/domain_generator.rb +42 -0
- data/lib/staticd/http_cache.rb +65 -0
- data/lib/staticd/http_server.rb +197 -0
- data/lib/staticd/json_request.rb +15 -0
- data/lib/staticd/json_response.rb +29 -0
- data/lib/staticd/models/base.rb +43 -0
- data/lib/staticd/models/domain_name.rb +17 -0
- data/lib/staticd/models/release.rb +20 -0
- data/lib/staticd/models/resource.rb +19 -0
- data/lib/staticd/models/route.rb +19 -0
- data/lib/staticd/models/site.rb +36 -0
- data/lib/staticd/models/staticd_config.rb +71 -0
- data/lib/staticd/public/jquery-1.11.1.min.js +4 -0
- data/lib/staticd/public/main.css +61 -0
- data/lib/staticd/public/main.js +15 -0
- data/lib/staticd/version.rb +3 -0
- data/lib/staticd/views/main.haml +9 -0
- data/lib/staticd/views/setup.haml +47 -0
- data/lib/staticd/views/welcome.haml +92 -0
- data/lib/staticd_utils/archive.rb +80 -0
- data/lib/staticd_utils/file_size.rb +26 -0
- data/lib/staticd_utils/gli_object.rb +14 -0
- data/lib/staticd_utils/memory_file.rb +36 -0
- data/lib/staticd_utils/sitemap.rb +100 -0
- data/lib/staticd_utils/tar.rb +43 -0
- metadata +300 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: a3f8e905eb11b074a1b854a2405f944a7d87b324
|
4
|
+
data.tar.gz: e747882c6c79a1bf71873d649e7e66d1e3c68f2d
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 03a74ca0b48fe5ec8826f3063dcf9ef8c58850dfe79b7afb54d997ef13c690e5498396a36479a831519ca997f745406408646d60403d8125388ccd797df20f9c
|
7
|
+
data.tar.gz: 89b7000862534763252743bacbe9dddec3c92dddd2a48cf183af01f8fafce8eac80227b8f62b6864660c1442cf0bb62c8cdbe4835f30adbce08142ecd9b3f1aa
|
data/bin/staticd
ADDED
@@ -0,0 +1,81 @@
|
|
1
|
+
require "api_auth"
|
2
|
+
require "base64"
|
3
|
+
|
4
|
+
# Rack middleware to authenticate requests using the HMAC-SHA1 protocol.
|
5
|
+
#
|
6
|
+
# It take a Proc as argument to find the entity secret key using the access id
|
7
|
+
# provided in the request. Once returned, the entity secret key is compared
|
8
|
+
# with secret key provided by the the request.
|
9
|
+
# Unless the two keys match, a 401 Unauthorized HTTP Error page is sent.
|
10
|
+
#
|
11
|
+
# Options:
|
12
|
+
# * environment: bypass authentication if set to "test"
|
13
|
+
# * except: list of HTTP paths with no authentication required
|
14
|
+
#
|
15
|
+
# Example:
|
16
|
+
# use Rack::Auth::HMAC do |request_access_id|
|
17
|
+
# return entity_secret_key if request_access_id == entity_access_id
|
18
|
+
# end
|
19
|
+
class Rack::Auth::HMAC
|
20
|
+
|
21
|
+
def initialize(app, options={}, &block)
|
22
|
+
@app = app
|
23
|
+
options[:except] ||= []
|
24
|
+
@options = options
|
25
|
+
@block = block
|
26
|
+
end
|
27
|
+
|
28
|
+
def call(env)
|
29
|
+
dup._call(env)
|
30
|
+
end
|
31
|
+
|
32
|
+
def _call(env)
|
33
|
+
return @app.call(env) if @options[:except].include?(env["PATH_INFO"])
|
34
|
+
|
35
|
+
env = fix_content_type(env)
|
36
|
+
|
37
|
+
request = Rack::Request.new(env)
|
38
|
+
access_id = ApiAuth.access_id(request)
|
39
|
+
secret_key = @block.call(access_id)
|
40
|
+
|
41
|
+
if ApiAuth.authentic?(request, secret_key)
|
42
|
+
status, headers, response = @app.call(env)
|
43
|
+
else
|
44
|
+
send_401(content_type: env["HTTP_ACCEPT"], ip: request.ip)
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
private
|
49
|
+
|
50
|
+
# Fix an issue with the HMAC canonical string calculation.
|
51
|
+
#
|
52
|
+
# Ensure the request Content-Type is not set to anything when a GET or
|
53
|
+
# DELETE method is used. Sinatra (or Rack) seems to set it to 'plain/text'
|
54
|
+
# when not specified.
|
55
|
+
def fix_content_type(env)
|
56
|
+
if ["GET", "DELETE"].include?(env["REQUEST_METHOD"])
|
57
|
+
env["CONTENT_TYPE"] = ""
|
58
|
+
end
|
59
|
+
env
|
60
|
+
end
|
61
|
+
|
62
|
+
def send_401(content_type: "text/plain", ip: nil)
|
63
|
+
message = "Valid access ID and secret key are required."
|
64
|
+
snonce = Base64.encode64(Time.now.to_s + ip)
|
65
|
+
|
66
|
+
body =
|
67
|
+
case content_type
|
68
|
+
when "application/json" then %({"error": "#{message}"})
|
69
|
+
when "text/html" then "<h1>#{message}</h1>"
|
70
|
+
else
|
71
|
+
message
|
72
|
+
end
|
73
|
+
|
74
|
+
headers = {
|
75
|
+
"Content-Type" => content_type,
|
76
|
+
"WWW-Authenticate" => %(HMACDigest realm="#{message}" snonce="#{snonce}")
|
77
|
+
}
|
78
|
+
|
79
|
+
Rack::Response.new(body, 401, headers)
|
80
|
+
end
|
81
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
|
2
|
+
# Rack middleware to log request time.
|
3
|
+
#
|
4
|
+
# Add the request time to the request environment using request.time key.
|
5
|
+
class Rack::RequestTime
|
6
|
+
|
7
|
+
def initialize(app)
|
8
|
+
@app = app
|
9
|
+
end
|
10
|
+
|
11
|
+
def call(env)
|
12
|
+
dup._call(env)
|
13
|
+
end
|
14
|
+
|
15
|
+
def _call(env)
|
16
|
+
env["request.time"] = Time.now
|
17
|
+
@app.call(env)
|
18
|
+
end
|
19
|
+
end
|
data/lib/staticd.rb
ADDED
data/lib/staticd/api.rb
ADDED
@@ -0,0 +1,356 @@
|
|
1
|
+
require "sinatra/base"
|
2
|
+
require "rack/auth/hmac"
|
3
|
+
require "digest/sha1"
|
4
|
+
require "haml"
|
5
|
+
require "open-uri"
|
6
|
+
|
7
|
+
require "staticd/json_response"
|
8
|
+
require "staticd/json_request"
|
9
|
+
require "staticd/domain_generator"
|
10
|
+
require "staticd/datastore"
|
11
|
+
require "staticd/database"
|
12
|
+
|
13
|
+
require "staticd_utils/archive"
|
14
|
+
require "staticd_utils/sitemap"
|
15
|
+
|
16
|
+
module Staticd
|
17
|
+
class APIError < StandardError; end
|
18
|
+
|
19
|
+
class API < Sinatra::Base
|
20
|
+
include Staticd::Models
|
21
|
+
|
22
|
+
VERSION = "v1"
|
23
|
+
|
24
|
+
PUBLIC_URI = %w(
|
25
|
+
/welcome /ping /main.css /main.js /jquery-1.11.1.min.js
|
26
|
+
)
|
27
|
+
|
28
|
+
configure do
|
29
|
+
set :app_file, __FILE__
|
30
|
+
set :show_exceptions, false
|
31
|
+
set :raise_errors, false
|
32
|
+
end
|
33
|
+
|
34
|
+
# Manage API errors.
|
35
|
+
error { raise env['sinatra.error'] }
|
36
|
+
error(APIError) { JSONResponse.send(:error, env['sinatra.error'].message) }
|
37
|
+
|
38
|
+
def initialize(params={})
|
39
|
+
@config = params
|
40
|
+
|
41
|
+
raise "Missing :domain parameter" unless @config[:domain]
|
42
|
+
raise "Missing :port parameter" unless @config[:port]
|
43
|
+
|
44
|
+
super
|
45
|
+
end
|
46
|
+
|
47
|
+
# Auto parse the request body when JSON and present.
|
48
|
+
before { load_json_body }
|
49
|
+
|
50
|
+
# Getting the Welcome Page.
|
51
|
+
#
|
52
|
+
# Display a welcome page with instructions to finish setup and configure
|
53
|
+
# the Staticd toolbelt.
|
54
|
+
get "/welcome" do
|
55
|
+
@staticd_host = @config[:domain]
|
56
|
+
if @config[:public_port] && @config[:public_port] != "80"
|
57
|
+
@staticd_host += ":#{@config[:public_port]}"
|
58
|
+
end
|
59
|
+
@staticd_url = "http://#{@staticd_host}/api/#{VERSION}"
|
60
|
+
if StaticdConfig.ask_value?(:disable_setup_page)
|
61
|
+
haml :welcome, layout: :main
|
62
|
+
else
|
63
|
+
@domain_resolve = ping?(@config[:domain], @config[:public_port])
|
64
|
+
@wildcard_resolve =
|
65
|
+
ping?("ping.#{@config[:domain]}", @config[:public_port])
|
66
|
+
haml :setup, layout: :main
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
# Hide the Welcome Page.
|
71
|
+
#
|
72
|
+
# After initial setup, you want to hide the welcome page displaying
|
73
|
+
# sensitive data.
|
74
|
+
delete "/welcome" do
|
75
|
+
StaticdConfig.set_value(:disable_setup_page, true)
|
76
|
+
end
|
77
|
+
|
78
|
+
# Ping page.
|
79
|
+
#
|
80
|
+
# Used by the ping command to verify a specified domain resolve to this app.
|
81
|
+
get "/ping" do
|
82
|
+
@config[:domain]
|
83
|
+
end
|
84
|
+
|
85
|
+
# Create a new site.
|
86
|
+
#
|
87
|
+
# Responds with the new site attributes.
|
88
|
+
# Parameters:
|
89
|
+
# * name: the name of the new site
|
90
|
+
#
|
91
|
+
# Example:
|
92
|
+
# $> curl --data '{"name": "my_app"}' http://localhost/api/sites
|
93
|
+
# {"name":"my_app"}
|
94
|
+
post "/sites" do
|
95
|
+
site = Site.new(name: @json["name"])
|
96
|
+
domain_suffix = ".#{@config[:domain]}"
|
97
|
+
domain = DomainGenerator.new(suffix: domain_suffix) do |generated_domain|
|
98
|
+
!DomainName.get(generated_domain)
|
99
|
+
end
|
100
|
+
site.domain_names << DomainName.new(name: domain)
|
101
|
+
|
102
|
+
if site.save
|
103
|
+
JSONResponse.send(:success, site.to_h(:full))
|
104
|
+
else
|
105
|
+
raise APIError, "Cannot create the new site (#{site.error})"
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
# Get a list of all sites.
|
110
|
+
#
|
111
|
+
# Responds with a list of sites with all attributes, releases and domain
|
112
|
+
# names.
|
113
|
+
#
|
114
|
+
# Example:
|
115
|
+
# $> curl localhost/api/sites
|
116
|
+
# [{
|
117
|
+
# "name":"my_app",
|
118
|
+
# "releases":[],
|
119
|
+
# "domain_names":[{"name":"my_app.com","site_name":"my_app"}]
|
120
|
+
# }]
|
121
|
+
get "/sites" do
|
122
|
+
sites = Site.map{ |site| site.to_h(:full) }
|
123
|
+
JSONResponse.send(:success, sites)
|
124
|
+
end
|
125
|
+
|
126
|
+
# Delete a site and all its associated resources (releases, domains,
|
127
|
+
# etc...).
|
128
|
+
#
|
129
|
+
# If the site is successfully deleted, it does not responds with any
|
130
|
+
# content.
|
131
|
+
# Parameters:
|
132
|
+
# * site_name: the name of the site (url)
|
133
|
+
#
|
134
|
+
# Example:
|
135
|
+
# $> curl --request DELETE localhost/api/sites/my_app
|
136
|
+
delete "/sites/:site_name" do
|
137
|
+
if current_site.destroy
|
138
|
+
JSONResponse.send(:success_no_content)
|
139
|
+
else
|
140
|
+
raise APIError, "Cannot remove the site '#{current_site}'"
|
141
|
+
end
|
142
|
+
end
|
143
|
+
|
144
|
+
# Create a new site release.
|
145
|
+
#
|
146
|
+
# Responds with the new releases attributes.
|
147
|
+
# Parameters:
|
148
|
+
# * site_name: the name of the site (url)
|
149
|
+
# * file: the archive containing the site resources
|
150
|
+
# * sitemap: a sitemap file indexing the site resources
|
151
|
+
#
|
152
|
+
# Example:
|
153
|
+
# $> curl --form "file=@archive.tar.gz;sitemap=@sitemap.yml" \
|
154
|
+
# localhost/api/sites/my_app/releases
|
155
|
+
# {
|
156
|
+
# "id":1,
|
157
|
+
# "tag":"v1",
|
158
|
+
# "site_name":"my_app"
|
159
|
+
# }
|
160
|
+
post "/sites/:site_name/releases" do
|
161
|
+
archive_path = load_file_attachment(:file)
|
162
|
+
sitemap_path = load_file_attachment(:sitemap)
|
163
|
+
|
164
|
+
# Create a new release.
|
165
|
+
release = Release.new(
|
166
|
+
site: current_site,
|
167
|
+
tag: "v#{current_site.releases.count + 1}"
|
168
|
+
)
|
169
|
+
|
170
|
+
# Open the archive and sitemap file.
|
171
|
+
archive = StaticdUtils::Archive.open_file(archive_path)
|
172
|
+
sitemap = StaticdUtils::Sitemap.open_file(sitemap_path)
|
173
|
+
|
174
|
+
archive.open do
|
175
|
+
|
176
|
+
# Store each new resources and build routes for known resources.
|
177
|
+
sitemap.each_resources do |sha1, path|
|
178
|
+
|
179
|
+
# Store or retrieve each resources.
|
180
|
+
resource =
|
181
|
+
if File.exist?(sha1)
|
182
|
+
|
183
|
+
# Verify file integrity.
|
184
|
+
calc_sha1 = Digest::SHA1.hexdigest(File.read(sha1))
|
185
|
+
unless sha1 == calc_sha1
|
186
|
+
raise APIError, "The file #{path} digest recorded inside " +
|
187
|
+
"the sitemap file is not correct"
|
188
|
+
end
|
189
|
+
|
190
|
+
# Store the file.
|
191
|
+
resource_url = Datastore.put(sha1)
|
192
|
+
|
193
|
+
# Create the resource.
|
194
|
+
Resource.new(sha1: sha1, url: resource_url)
|
195
|
+
else
|
196
|
+
|
197
|
+
# Get the resource from the database.
|
198
|
+
cached = Resource.get(sha1)
|
199
|
+
unless cached
|
200
|
+
raise APIError, "A resource is missing (missing file and " +
|
201
|
+
"database record)"
|
202
|
+
end
|
203
|
+
cached
|
204
|
+
end
|
205
|
+
|
206
|
+
# Create the release route.
|
207
|
+
release.routes.new(resource: resource, path: path)
|
208
|
+
end
|
209
|
+
end
|
210
|
+
|
211
|
+
if release.save
|
212
|
+
JSONResponse.send(:success, release.to_h(:full))
|
213
|
+
else
|
214
|
+
raise APIError, "Cannot create the new release (#{release.error})"
|
215
|
+
end
|
216
|
+
end
|
217
|
+
|
218
|
+
# Get all releases of a site.
|
219
|
+
#
|
220
|
+
# Respond with a list of releases and their attributes.
|
221
|
+
# Parameters:
|
222
|
+
# * site_name: the name of the site (url)
|
223
|
+
#
|
224
|
+
# Example:
|
225
|
+
# $> curl localhost/api/sites/my_app/releases
|
226
|
+
# [{
|
227
|
+
# "id": "1",
|
228
|
+
# "tag": "v1",
|
229
|
+
# "url": "/tmp/store/20ebde5306c481363297008c70bd45e2.tar.gz",
|
230
|
+
# "site_name": "my_app"
|
231
|
+
# }]
|
232
|
+
get "/sites/:site_name/releases" do
|
233
|
+
releases = current_site.releases.map{ |release| release.to_h }
|
234
|
+
JSONResponse.send(:success, releases)
|
235
|
+
end
|
236
|
+
|
237
|
+
# Attach a new domain name to a site.
|
238
|
+
#
|
239
|
+
# Respond with the domain list attributes.
|
240
|
+
# Parameters:
|
241
|
+
# * site_name: the name of the site (url)
|
242
|
+
# * name: the domain name to attach
|
243
|
+
#
|
244
|
+
# Example:
|
245
|
+
# $> curl --data '{"name": "hello.io"}' \
|
246
|
+
# localhost/api/sites/my_app/domain_names
|
247
|
+
# {"name":"hello.io","site_name":"my_app"}
|
248
|
+
post "/sites/:site_name/domain_names" do
|
249
|
+
domain_name = DomainName.new(site: current_site, name: @json["name"])
|
250
|
+
|
251
|
+
if domain_name.save
|
252
|
+
JSONResponse.send(:success, domain_name.to_h)
|
253
|
+
else
|
254
|
+
raise APIError, "Cannot create the new domain name " +
|
255
|
+
"(#{domain_name.error})"
|
256
|
+
end
|
257
|
+
end
|
258
|
+
|
259
|
+
# Detach a domain name from a site.
|
260
|
+
#
|
261
|
+
# If successfully deleted, it does not respond with any content.
|
262
|
+
# Parameters:
|
263
|
+
# * site_name: the name of the site (url)
|
264
|
+
# * domain_name: the domain name (url)
|
265
|
+
#
|
266
|
+
# Example:
|
267
|
+
# $> curl --request DELETE \
|
268
|
+
# localhost/api/sites/my_app/domain_names/domain.tld
|
269
|
+
delete "/sites/:site_name/domain_names/:domain_name" do
|
270
|
+
if current_domain.destroy
|
271
|
+
JSONResponse.send(:success_no_content)
|
272
|
+
else
|
273
|
+
raise APIError, "Cannot detach the #{params[:domain_name]} domain " +
|
274
|
+
"name from the #{site} site"
|
275
|
+
end
|
276
|
+
end
|
277
|
+
|
278
|
+
# Get all domain names attched to a site.
|
279
|
+
#
|
280
|
+
# Respond with a list of all domain names and their attributes attached to
|
281
|
+
# the site.
|
282
|
+
# Parameters:
|
283
|
+
# * site_name: the name of the site (url)
|
284
|
+
#
|
285
|
+
# Example:
|
286
|
+
# $> curl localhost/api/sites/my_app/domain_names
|
287
|
+
# [{"name": "hello.io"}]
|
288
|
+
get "/sites/:site_name/domain_names" do
|
289
|
+
domains = current_site.domain_names.map{ |domain| domain.to_h }
|
290
|
+
JSONResponse.send(:success, domains)
|
291
|
+
end
|
292
|
+
|
293
|
+
# Get all already known resources included in a sitemap.
|
294
|
+
#
|
295
|
+
# Get a list of resources in a custom sitemap format (translated into json)
|
296
|
+
# and respond with the list of new resources (not already cached by any
|
297
|
+
# release) in the same format.
|
298
|
+
#
|
299
|
+
# Example:
|
300
|
+
# $> curl --data \
|
301
|
+
# '{ \
|
302
|
+
# "92136ff551f50188f46486ab80db269eda4dfd4e":"/hi.html", \
|
303
|
+
# "56897ff551f50188f46486ab80db269eda4dfd4e":"/ho.html" \
|
304
|
+
# }' localhost/api/resources/get_cached
|
305
|
+
# {"92136ff551f50188f46486ab80db269eda4dfd4e":"/hi.html"}
|
306
|
+
post "/resources/get_cached" do
|
307
|
+
unknow_resources_map = map = @json
|
308
|
+
sitemap = StaticdUtils::Sitemap.new(map)
|
309
|
+
known_resources = Resource.all(sha1: sitemap.digests)
|
310
|
+
known_resources.each do |resource|
|
311
|
+
unknow_resources_map.delete(resource.sha1)
|
312
|
+
end
|
313
|
+
JSONResponse.send(:success, unknow_resources_map)
|
314
|
+
end
|
315
|
+
|
316
|
+
private
|
317
|
+
|
318
|
+
def ping?(domain, port=80)
|
319
|
+
open("http://#{domain}:#{port}/api/#{VERSION}/ping", read_timeout: 1) do |response|
|
320
|
+
response.read == @config[:domain]
|
321
|
+
end
|
322
|
+
rescue
|
323
|
+
false
|
324
|
+
end
|
325
|
+
|
326
|
+
def load_json_body
|
327
|
+
if request.content_type == "application/json"
|
328
|
+
request.body.rewind
|
329
|
+
@json = JSONRequest.parse(request.body.read)
|
330
|
+
end
|
331
|
+
end
|
332
|
+
|
333
|
+
def load_file_attachment(field)
|
334
|
+
unless params.has_key?(field.to_s) && params[field].has_key?(:tempfile)
|
335
|
+
raise APIError, "No valid #{field} file submitted"
|
336
|
+
end
|
337
|
+
params[field][:tempfile].path
|
338
|
+
end
|
339
|
+
|
340
|
+
def current_site
|
341
|
+
unless (@current_site ||= Site.get(params[:site_name]))
|
342
|
+
raise APIError, "This site (#{params[:site_name]}) does not exist"
|
343
|
+
end
|
344
|
+
@current_site
|
345
|
+
end
|
346
|
+
|
347
|
+
def current_domain
|
348
|
+
@current_domain ||= current_site.domain_names.get(params[:domain_name])
|
349
|
+
unless @current_domain
|
350
|
+
raise APIError, "This domain name (#{params[:domain_name]}) is not " +
|
351
|
+
"attached to the #{current_site} site"
|
352
|
+
end
|
353
|
+
@current_domain
|
354
|
+
end
|
355
|
+
end
|
356
|
+
end
|