staticd 0.0.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (40) hide show
  1. checksums.yaml +7 -0
  2. data/bin/staticd +7 -0
  3. data/lib/rack/auth/hmac.rb +81 -0
  4. data/lib/rack/request_time.rb +19 -0
  5. data/lib/staticd.rb +7 -0
  6. data/lib/staticd/api.rb +356 -0
  7. data/lib/staticd/app.rb +130 -0
  8. data/lib/staticd/cache_engine.rb +45 -0
  9. data/lib/staticd/cli.rb +70 -0
  10. data/lib/staticd/config.rb +115 -0
  11. data/lib/staticd/database.rb +41 -0
  12. data/lib/staticd/datastore.rb +64 -0
  13. data/lib/staticd/datastores/local.rb +48 -0
  14. data/lib/staticd/datastores/s3.rb +63 -0
  15. data/lib/staticd/domain_generator.rb +42 -0
  16. data/lib/staticd/http_cache.rb +65 -0
  17. data/lib/staticd/http_server.rb +197 -0
  18. data/lib/staticd/json_request.rb +15 -0
  19. data/lib/staticd/json_response.rb +29 -0
  20. data/lib/staticd/models/base.rb +43 -0
  21. data/lib/staticd/models/domain_name.rb +17 -0
  22. data/lib/staticd/models/release.rb +20 -0
  23. data/lib/staticd/models/resource.rb +19 -0
  24. data/lib/staticd/models/route.rb +19 -0
  25. data/lib/staticd/models/site.rb +36 -0
  26. data/lib/staticd/models/staticd_config.rb +71 -0
  27. data/lib/staticd/public/jquery-1.11.1.min.js +4 -0
  28. data/lib/staticd/public/main.css +61 -0
  29. data/lib/staticd/public/main.js +15 -0
  30. data/lib/staticd/version.rb +3 -0
  31. data/lib/staticd/views/main.haml +9 -0
  32. data/lib/staticd/views/setup.haml +47 -0
  33. data/lib/staticd/views/welcome.haml +92 -0
  34. data/lib/staticd_utils/archive.rb +80 -0
  35. data/lib/staticd_utils/file_size.rb +26 -0
  36. data/lib/staticd_utils/gli_object.rb +14 -0
  37. data/lib/staticd_utils/memory_file.rb +36 -0
  38. data/lib/staticd_utils/sitemap.rb +100 -0
  39. data/lib/staticd_utils/tar.rb +43 -0
  40. metadata +300 -0
@@ -0,0 +1,48 @@
1
+ require "digest/sha1"
2
+
3
+ module Staticd
4
+ module Datastores
5
+
6
+ # Datastore storing files on local directory.
7
+ #
8
+ # It use the file SHA1 digest as a filename so two identical files are not
9
+ # stored twice.
10
+ #
11
+ # Example:
12
+ # datastore = Staticd::Datastores::Local.new(path: "/tmp/datastore")
13
+ # datastore.put(file_path) unless datastore.exist?(file_path)
14
+ # # => "/tmp/datastore/sha1_digest"
15
+ class Local
16
+
17
+ def initialize(params)
18
+ @path = params[:path]
19
+ check_store_directory
20
+ end
21
+
22
+ def put(file_path)
23
+ datastore_file = stored_file_path(file_path)
24
+ FileUtils.copy_file(file_path, datastore_file) unless exist?(file_path)
25
+ datastore_file
26
+ end
27
+
28
+ def exist?(file_path)
29
+ datastore_file = stored_file_path(file_path)
30
+ File.exist?(datastore_file) ? datastore_file : false
31
+ end
32
+
33
+ private
34
+
35
+ def check_store_directory
36
+ FileUtils.mkdir_p(@path) unless File.directory?(@path)
37
+ end
38
+
39
+ def sha1(file_path)
40
+ Digest::SHA1.hexdigest(File.read(file_path))
41
+ end
42
+
43
+ def stored_file_path(file_path)
44
+ "#{@path}/#{sha1(file_path)}"
45
+ end
46
+ end
47
+ end
48
+ end
@@ -0,0 +1,63 @@
1
+ require "digest/sha1"
2
+ require "aws-sdk"
3
+
4
+ module Staticd
5
+ module Datastores
6
+
7
+ # Datastore storing files on Amazon S3.
8
+ #
9
+ # It use the file SHA1 digest as a filename so two identical files are not
10
+ # stored twice.
11
+ # Each files can be accessed afterwards using a public HTTP(S) address.
12
+ #
13
+ # Example:
14
+ # datastore = Staticd::Datastores::S3.new(
15
+ # host: bucket_name,
16
+ # username: access_key_id,
17
+ # password: secret_access_key
18
+ # )
19
+ # datastore.put(file_path) unless datastore.exist?(file_path)
20
+ # # => http://bucket_name.hostname/sha1_digest
21
+ class S3
22
+
23
+ def initialize(params)
24
+ @bucket_name = params[:host]
25
+ @access_key = ENV["AWS_ACCESS_KEY_ID"]
26
+ @secret_key = ENV["AWS_SECRET_ACCESS_KEY"]
27
+ end
28
+
29
+ def put(file_path)
30
+ s3_object = object(file_path)
31
+ s3_object.write(file: file_path)
32
+ s3_object.acl = :public_read
33
+ s3_object.public_url(secure: false).to_s
34
+ end
35
+
36
+ def exist?(file_path)
37
+ s3_object = object(file_path)
38
+ s3_object.exists? ? s3_object.public_url(secure: false) : false
39
+ end
40
+
41
+ private
42
+
43
+ def s3
44
+ @s3 ||= AWS::S3.new(
45
+ access_key_id: @access_key,
46
+ secret_access_key: @secret_key
47
+ )
48
+ end
49
+
50
+ def bucket
51
+ @bucket ||= s3.buckets[@bucket_name]
52
+ end
53
+
54
+ def object(file_path)
55
+ bucket.objects[sha1(file_path)]
56
+ end
57
+
58
+ def sha1(file_path)
59
+ Digest::SHA1.hexdigest(File.read(file_path))
60
+ end
61
+ end
62
+ end
63
+ end
@@ -0,0 +1,42 @@
1
+ module Staticd
2
+
3
+ # Domain name generator.
4
+ #
5
+ # This class can be used to generate random words of various length.
6
+ # Options:
7
+ # * length: the length of the random word
8
+ # * suffix: a suffix to append to the random world (default is none)
9
+ #
10
+ # Example:
11
+ # DomainGenerator.new(length: 2, suffix: ".domain.tld")
12
+ # # => rb.domain.tld
13
+ #
14
+ # A block can be used to validate the generated domain. It must return true
15
+ # to validate the domain, otherwise a new one is proposed. This feature can
16
+ # be used to validate the domain against certain rules.
17
+ #
18
+ # Example:
19
+ # DomainGenerator.new(suffix: ".domain.tld") do |generated_domain|
20
+ # ["admin", "www"].include?(generated_domain)
21
+ # end
22
+ class DomainGenerator
23
+
24
+ def self.new(options={})
25
+ if block_given?
26
+ until domain = generate(options)
27
+ yield domain
28
+ end
29
+ domain
30
+ else
31
+ generate(options)
32
+ end
33
+ end
34
+
35
+ def self.generate(options={})
36
+ length = options[:length] || 6
37
+ suffix = options[:suffix] || ""
38
+ random = ("a".."z").to_a.shuffle[0, length].join
39
+ random + suffix
40
+ end
41
+ end
42
+ end
@@ -0,0 +1,65 @@
1
+ require "staticd/database"
2
+ require "staticd/cache_engine"
3
+
4
+ module Staticd
5
+
6
+ # Rack middleware to manage remote HTTP resources caching.
7
+ #
8
+ # For each request, it will find the last release of the site associed with
9
+ # the requested domain name and cache the corresponding resource if not
10
+ # already cached.
11
+ class HTTPCache
12
+ include Staticd::Models
13
+
14
+ def initialize(http_root, app)
15
+ @app = app
16
+ @http_root = http_root
17
+
18
+ raise "No HTTP root folder provided" unless @http_root
19
+ raise "No rack app provided" unless @app
20
+ end
21
+
22
+ def call(env)
23
+ dup._call(env)
24
+ end
25
+
26
+ def _call(env)
27
+ @env = env
28
+
29
+ # Change the Request Path to '/index.html' if root path is asked.
30
+ @env["PATH_INFO"] = "/index.html" if @env["PATH_INFO"] == '/'
31
+
32
+ # Get the release from the request Host header.
33
+ release = Release.last(
34
+ Release.site.domain_names.name => Rack::Request.new(@env).host
35
+ )
36
+ return next_middleware unless release
37
+
38
+ # Change the script name to include the site name and release version.
39
+ @env["SCRIPT_NAME"] = "/#{release.site_name}/#{release}"
40
+
41
+ req = Rack::Request.new(@env)
42
+ cache_engine = CacheEngine.new(@http_root)
43
+
44
+ # Do nothing else if the resource is already cached.
45
+ return next_middleware if cache_engine.cached?(req.path)
46
+
47
+ # Get the resource to cache.
48
+ resource = Resource.first(
49
+ Resource.routes.release_id => release.id,
50
+ Resource.routes.path => req.path_info
51
+ )
52
+ return next_middleware unless resource
53
+
54
+ # Cache the resource.
55
+ cache_engine.cache(req.path, resource.url)
56
+ next_middleware
57
+ end
58
+
59
+ private
60
+
61
+ def next_middleware
62
+ @app.call(@env)
63
+ end
64
+ end
65
+ end
@@ -0,0 +1,197 @@
1
+ require "sendfile"
2
+
3
+ module Staticd
4
+
5
+ # Simple HTTP server Rack app.
6
+ #
7
+ # If the resource is readable from the root folder at the path given by the
8
+ # request, the resource is sent to the client. Otherwise a 404 Not Found HTTP
9
+ # error is sent.
10
+ class HTTPServer
11
+
12
+ # Mime types served by the webserver (from NGiNX mime.types file).
13
+ EXT_MIME_TYPE = {
14
+ ".html" => "text/html",
15
+ ".htm" => "text/html",
16
+ ".shtml" => "text/html",
17
+ ".css" => "text/css",
18
+ ".xml" => "text/xml",
19
+ ".rss" => "text/xml",
20
+ ".gif" => "image/gif",
21
+ ".jpeg" => "image/jpeg",
22
+ ".jpg" => "image/jpeg",
23
+ ".js" => "application/x-javascript",
24
+ ".txt" => "text/plain",
25
+ ".htc" => "text/x-component",
26
+ ".mml" => "text/mathml",
27
+ ".png" => "image/png",
28
+ ".ico" => "image/x-icon",
29
+ ".jng" => "image/x-jng",
30
+ ".wbmp" => "image/vnd.wap.wbmp",
31
+ ".jar" => "application/java-archive",
32
+ ".war" => "application/java-archive",
33
+ ".ear" => "application/java-archive",
34
+ ".hqx" => "application/mac-binhex40",
35
+ ".pdf" => "application/pdf",
36
+ ".cco" => "application/x-cocoa",
37
+ ".jardiff" => "application/x-java-archive-diff",
38
+ ".jnlp" => "application/x-java-jnlp-file",
39
+ ".run" => "application/x-makeself",
40
+ ".pm" => "application/x-perl",
41
+ ".pl" => "application/x-perl",
42
+ ".prc" => "application/x-pilot",
43
+ ".pdb" => "application/x-pilot",
44
+ ".rar" => "application/x-rar-compressed",
45
+ ".rpm" => "application/x-redhat-package-manager",
46
+ ".sea" => "application/x-sea",
47
+ ".swf" => "application/x-shockwave-flash",
48
+ ".sit" => "application/x-stuffit",
49
+ ".tcl" => "application/x-tcl",
50
+ ".tk" => "application/x-tcl",
51
+ ".der" => "application/x-x509-ca-cert",
52
+ ".pem" => "application/x-x509-ca-cert",
53
+ ".crt" => "application/x-x509-ca-cert",
54
+ ".xpi" => "application/x-xpinstall",
55
+ ".zip" => "application/zip",
56
+ ".deb" => "application/octet-stream",
57
+ ".bin" => "application/octet-stream",
58
+ ".exe" => "application/octet-stream",
59
+ ".dll" => "application/octet-stream",
60
+ ".dmg" => "application/octet-stream",
61
+ ".eot" => "application/octet-stream",
62
+ ".iso" => "application/octet-stream",
63
+ ".img" => "application/octet-stream",
64
+ ".msi" => "application/octet-stream",
65
+ ".msp" => "application/octet-stream",
66
+ ".msm" => "application/octet-stream",
67
+ ".mp3" => "audio/mpeg",
68
+ ".ra" => "audio/x-realaudio",
69
+ ".mpeg" => "video/mpeg",
70
+ ".mpg" => "video/mpeg",
71
+ ".mov" => "video/quicktime",
72
+ ".flv" => "video/x-flv",
73
+ ".avi" => "video/x-msvideo",
74
+ ".wmv" => "video/x-ms-wmv",
75
+ ".asx" => "video/x-ms-asf",
76
+ ".asf" => "video/x-ms-asf",
77
+ ".mng" => "video/x-mng"
78
+ }
79
+
80
+ # Mime type used when no type has been identified.
81
+ DEFAULT_MIME_TYPE = "application/octet-stream"
82
+
83
+ def initialize(http_root, access_logger=nil)
84
+ @http_root = http_root
85
+ unless (@access_logger = access_logger)
86
+ @access_logger = Logger.new(STDOUT)
87
+ @access_logger.formatter = proc { |_, _, _, msg| "#{msg}\n"}
88
+ end
89
+
90
+ raise "No HTTP root folder provided" unless @http_root
91
+ end
92
+
93
+ def call(env)
94
+ dup._call(env)
95
+ end
96
+
97
+ def _call(env)
98
+ @env = env
99
+ req = Rack::Request.new(@env)
100
+ file_path = @http_root + req.path
101
+ res = File.readable?(file_path) ? serve(file_path) : send_404
102
+ log(req, res)
103
+ end
104
+
105
+ private
106
+
107
+ # Log a request using the "Extended Log File Format".
108
+ # See: http://www.w3.org/TR/WD-logfile.html
109
+ #
110
+ # Use the request.time key setup by the Rack::RequestTime middleware.
111
+ #
112
+ # Version: 1.0
113
+ # Fields: time cs-dns cs-ip date cs-method cs-uri sc-status sc-byte sc-time-taken
114
+ def log(req, res)
115
+ request_stop_time = Time.now
116
+ request_start_time =
117
+ req.env.key?("request.time") ? req.env["request.time"] : nil
118
+ request_completed_time =
119
+ if request_start_time
120
+ (request_stop_time - request_start_time).round(4)
121
+ else
122
+ "-"
123
+ end
124
+ content_length =
125
+ res[1].key?("Content-Length") ? " #{res[1]["Content-Length"]}" : "-"
126
+
127
+ log_string = "#{request_stop_time.strftime("%Y-%m-%d %H:%M:%S")}"
128
+ log_string << " #{req.host}"
129
+ log_string << " #{req.env["REMOTE_ADDR"]}"
130
+ log_string << " #{req.env["REQUEST_METHOD"]} #{req.path_info}"
131
+ log_string << " #{res[0]}"
132
+ log_string << content_length
133
+ log_string << " #{request_completed_time}"
134
+ @access_logger.info(log_string)
135
+
136
+ res
137
+ end
138
+
139
+ # Serve a file.
140
+ #
141
+ # This method will return a Rack compatible response ready to be served to
142
+ # the client. It will use the appropriate method (loading file into memory
143
+ # vs serving file using the sendfile system call) based on availability.
144
+ def serve(file_path)
145
+ @env['rack.hijack?'] ? sendfile(file_path) : send(file_path)
146
+ end
147
+
148
+ # Send a file loading it in memory.
149
+ #
150
+ # Method used in the first implementation.
151
+ # Keep it for compatibility purpose when the Rack hijack API is not
152
+ # supported.
153
+ def send(file_path)
154
+ response = Rack::Response.new
155
+ response["Content-Type"] = mime(file_path)
156
+ File.foreach(file_path) { |chunk| response.write(chunk) }
157
+ response.finish
158
+ end
159
+
160
+ # Use sendfile system call to send file without loading it into memory.
161
+ #
162
+ # It use the sendfile gem and the rack hijacking api.
163
+ # See: https://github.com/codeslinger/sendfile
164
+ # See: http://blog.phusion.nl/2013/01/23/the-new-rack-socket-hijacking-api/
165
+ def sendfile(file_path)
166
+
167
+ response_header = {
168
+ "Content-Type" => mime(file_path),
169
+ "Content-Length" => size(file_path),
170
+ "Connection" => "close"
171
+ }
172
+ response_header["rack.hijack"] = lambda do |io|
173
+ begin
174
+ File.open(file_path) { |file| io.sendfile(file) }
175
+ io.flush
176
+ ensure
177
+ io.close
178
+ end
179
+ end
180
+ [200, response_header]
181
+ end
182
+
183
+ def send_404
184
+ res = Rack::Response.new(["Not Found"], 404, {})
185
+ res.finish
186
+ end
187
+
188
+ def mime(file_path)
189
+ ext = File.extname(file_path).downcase
190
+ EXT_MIME_TYPE.key?(ext) ? EXT_MIME_TYPE[ext] : DEFAULT_MIME_TYPE
191
+ end
192
+
193
+ def size(file_path)
194
+ File.size(file_path).to_s
195
+ end
196
+ end
197
+ end
@@ -0,0 +1,15 @@
1
+ require "json"
2
+
3
+ module Staticd
4
+
5
+ # Simple JSON body parser for HTTP request.
6
+ #
7
+ # Example:
8
+ # hash = JSONRequest.parse(request_body)
9
+ class JSONRequest
10
+
11
+ def self.parse(body)
12
+ body.empty? ? {} : JSON.parse(body)
13
+ end
14
+ end
15
+ end
@@ -0,0 +1,29 @@
1
+ require "json"
2
+
3
+ module Staticd
4
+
5
+ # Simple HTTP response constructor for JSON content.
6
+ #
7
+ # Example:
8
+ # response = JSONResponse.send(:success, {foo: :bar})
9
+ class JSONResponse
10
+
11
+ def self.send(type, content=nil)
12
+ case type
13
+ when :success
14
+ @status = 200
15
+ @body = content
16
+ when :success_no_content
17
+ @status = 204
18
+ when :error
19
+ @status = 403
20
+ @body = {error: content}
21
+ else
22
+ @status = 500
23
+ @body = {error: "Something went wrong on our side."}
24
+ end
25
+ json_body = @body ? JSON.generate(@body) : nil
26
+ [@status, json_body]
27
+ end
28
+ end
29
+ end