coldwire-rails 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +53 -0
- data/LICENSE +21 -0
- data/README.md +69 -0
- data/VERSION +1 -0
- data/app/assets/javascripts/coldwire/archives.js +52 -0
- data/app/assets/javascripts/coldwire/cache_controller.js +1058 -0
- data/app/assets/javascripts/coldwire/entries.js +55 -0
- data/app/assets/javascripts/coldwire/format.js +60 -0
- data/app/assets/javascripts/coldwire/worker.js +35 -0
- data/app/controllers/coldwire/application_controller.rb +21 -0
- data/app/controllers/coldwire/caches_controller.rb +33 -0
- data/app/controllers/coldwire/service_worker_controller.rb +26 -0
- data/app/helpers/coldwire/service_worker_helper.rb +59 -0
- data/app/views/coldwire/caches/show.html.erb +304 -0
- data/app/views/coldwire/service_worker/offline_frame.html.erb +16 -0
- data/app/views/coldwire/service_worker/offline_page.html.erb +112 -0
- data/app/views/coldwire/service_worker/show.js.erb +66 -0
- data/config/importmap.rb +7 -0
- data/config/routes.rb +8 -0
- data/docs/README.md +19 -0
- data/docs/configuration.md +539 -0
- data/docs/how-it-works.md +64 -0
- data/docs/images/offline-fallback.png +0 -0
- data/docs/images/offline-settings-cached.png +0 -0
- data/docs/images/offline-settings.png +0 -0
- data/docs/setup.md +218 -0
- data/lib/coldwire/client/api.js +44 -0
- data/lib/coldwire/client/cookie.js +13 -0
- data/lib/coldwire/client/forced.js +17 -0
- data/lib/coldwire/client/identity.js +28 -0
- data/lib/coldwire/client/marker.js +35 -0
- data/lib/coldwire/client/register.js +19 -0
- data/lib/coldwire/client/store.js +69 -0
- data/lib/coldwire/client/sync.js +122 -0
- data/lib/coldwire/client_user_agent.rb +48 -0
- data/lib/coldwire/configuration.rb +319 -0
- data/lib/coldwire/debug.css +197 -0
- data/lib/coldwire/engine.rb +34 -0
- data/lib/coldwire/source.rb +50 -0
- data/lib/coldwire/version.rb +11 -0
- data/lib/coldwire/worker/archives.js +158 -0
- data/lib/coldwire/worker/events.js +79 -0
- data/lib/coldwire/worker/inspect.js +81 -0
- data/lib/coldwire/worker/ranges.js +136 -0
- data/lib/coldwire/worker/rules.js +100 -0
- data/lib/coldwire/worker/serve.js +226 -0
- data/lib/coldwire/worker/sync.js +234 -0
- data/lib/coldwire-rails.rb +5 -0
- data/lib/coldwire.rb +55 -0
- data/lib/generators/coldwire/install/install_generator.rb +110 -0
- data/lib/generators/coldwire/install/templates/coldwire.rb +53 -0
- data/lib/tasks/coldwire.rake +8 -0
- metadata +113 -0
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
// One entry per URL. cache.put() replaces an entry only where the two agree about Vary, and
|
|
2
|
+
// Rails answers HTML with `Vary: Accept` — so precaching (`*/*`) and a Turbo visit
|
|
3
|
+
// (`text/html`) are two records of one page, which WebKit keeps and Chrome collapses.
|
|
4
|
+
//
|
|
5
|
+
// Not ignoreSearch: range and chunk entries share the path with their own query, and a page
|
|
6
|
+
// write must never take a downloaded archive with it.
|
|
7
|
+
async function putFresh(cache, key, response) {
|
|
8
|
+
await cache.delete(key, { ignoreVary: true })
|
|
9
|
+
await cache.put(key, response)
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
async function fetchAndCache(cache, href, { managed = false } = {}) {
|
|
13
|
+
const request = new Request(href, { credentials: "same-origin" })
|
|
14
|
+
// Skipped rather than failed: the app asked for this URL never to be stored, and a manifest
|
|
15
|
+
// that also names it is a contradiction to resolve quietly in favour of not storing.
|
|
16
|
+
if (isNeverCached(new URL(request.url))) return []
|
|
17
|
+
|
|
18
|
+
const response = await fetch(request)
|
|
19
|
+
if (!isCacheable(request, response)) {
|
|
20
|
+
throw new Error(response.redirected ? `Redirected to ${response.url}` : `HTTP ${response.status}`)
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
await putFresh(cache, cacheKey(request, { managed }), response.clone())
|
|
24
|
+
|
|
25
|
+
const contentType = response.headers.get("Content-Type") || ""
|
|
26
|
+
if (!contentType.includes("text/html")) return []
|
|
27
|
+
|
|
28
|
+
return urlsFromHtml(await response.text(), href)
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
// Subresources only — do not follow <a href> or this becomes a site crawler.
|
|
32
|
+
function urlsFromHtml(html, pageUrl) {
|
|
33
|
+
const urls = new Set()
|
|
34
|
+
const base = new URL(pageUrl)
|
|
35
|
+
|
|
36
|
+
const add = (raw) => {
|
|
37
|
+
if (!raw) return
|
|
38
|
+
raw.split(",").forEach((part) => {
|
|
39
|
+
const token = part.trim().split(/\s+/)[0]
|
|
40
|
+
if (!token || token.startsWith("data:")) return
|
|
41
|
+
try {
|
|
42
|
+
const url = new URL(token, base)
|
|
43
|
+
// Any origin we are allowed to cache, not just our own. A page whose map library
|
|
44
|
+
// comes off a CDN is not offline-ready without it: precaching the page and skipping
|
|
45
|
+
// the script it cannot run without leaves a blank screen and a full cache.
|
|
46
|
+
if (!cacheableOrigin(url)) return
|
|
47
|
+
if (matchesPath(url, NEVER_INTERCEPT)) return
|
|
48
|
+
urls.add(url.href)
|
|
49
|
+
} catch {}
|
|
50
|
+
})
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
for (const match of html.matchAll(/<link\b[^>]*\bhref=["']([^"']+)["']/gi)) add(match[1])
|
|
54
|
+
for (const match of html.matchAll(/<(?:script|img|source)\b[^>]*\bsrc=["']([^"']+)["']/gi)) add(match[1])
|
|
55
|
+
for (const match of html.matchAll(/\bsrcset=["']([^"']+)["']/gi)) add(match[1])
|
|
56
|
+
|
|
57
|
+
const importmap = html.match(/<script[^>]*type=["']importmap["'][^>]*>([\s\S]*?)<\/script>/i)
|
|
58
|
+
if (importmap) {
|
|
59
|
+
try {
|
|
60
|
+
const json = JSON.parse(importmap[1])
|
|
61
|
+
Object.values(json.imports || {}).forEach(add)
|
|
62
|
+
Object.values(json.scopes || {}).forEach((map) => Object.values(map).forEach(add))
|
|
63
|
+
} catch {}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
return [...urls]
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// The network answers whenever it can, and the cache is what is left when it cannot. Nothing
|
|
70
|
+
// here second-guesses the browser about what it already holds: an asset is served from the
|
|
71
|
+
// browser's own cache without a request either way, and a worker sitting in front of that can
|
|
72
|
+
// only get it wrong.
|
|
73
|
+
//
|
|
74
|
+
// So the cache is not even consulted until the network has failed. There is no lookup on the
|
|
75
|
+
// path where everything is working.
|
|
76
|
+
async function handleFetch(request, event) {
|
|
77
|
+
const cache = await caches.open(CACHE_NAME)
|
|
78
|
+
|
|
79
|
+
if (forcedOffline) return offlineFallback(cache, request)
|
|
80
|
+
|
|
81
|
+
try {
|
|
82
|
+
const response = await fetch(request)
|
|
83
|
+
if (isCacheable(request, response) && isAutoCacheable(request)) {
|
|
84
|
+
// waitUntil rather than a promise left running: a worker can be stopped the moment it
|
|
85
|
+
// has answered, and a page stored without the stylesheet it names is worse offline than
|
|
86
|
+
// a page not stored at all.
|
|
87
|
+
const stored = storeResponse(cache, request, response.clone())
|
|
88
|
+
if (event) event.waitUntil(stored)
|
|
89
|
+
}
|
|
90
|
+
return response
|
|
91
|
+
} catch {
|
|
92
|
+
return offlineFallback(cache, request)
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
async function offlineFallback(cache, request) {
|
|
97
|
+
const cached = await cache.match(request, MATCH_OPTIONS)
|
|
98
|
+
|
|
99
|
+
return (await cachedPageResponse(cache, request, cached)) || offlineResponse(request)
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// A page is stored with whatever it asks for. The lists say which pages are worth keeping as
|
|
103
|
+
// you browse; what one needs in order to render is not a second question, and a page held
|
|
104
|
+
// without its stylesheet is the offline equivalent of not holding it at all.
|
|
105
|
+
async function storeResponse(cache, request, response) {
|
|
106
|
+
await putFresh(cache, cacheKey(request), response.clone())
|
|
107
|
+
|
|
108
|
+
const type = response.headers.get("Content-Type") || ""
|
|
109
|
+
if (!type.includes("text/html")) return
|
|
110
|
+
|
|
111
|
+
const urls = urlsFromHtml(await response.text(), request.url)
|
|
112
|
+
await Promise.all(urls.map((href) => storeSubresource(cache, href)))
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
// Only what is missing. A digested address is stored once and then named by every page that
|
|
116
|
+
// uses it, so after the first visit this settles down to a lookup and nothing else.
|
|
117
|
+
async function storeSubresource(cache, href) {
|
|
118
|
+
if (isNeverCached(new URL(href))) return
|
|
119
|
+
if (await cache.match(href, MATCH_OPTIONS)) return
|
|
120
|
+
|
|
121
|
+
try {
|
|
122
|
+
await fetchAndCache(cache, href)
|
|
123
|
+
} catch {
|
|
124
|
+
// One subresource that will not fetch is no reason to lose the page.
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
// Turbo will not render a page whose data-turbo-track="reload" elements differ from the
|
|
129
|
+
// current page's; it reloads instead, to pick up the new assets. Offline there are none to
|
|
130
|
+
// pick up and the reload is answered from this same cache, so it costs a document load that
|
|
131
|
+
// Hotwire Native can hang on.
|
|
132
|
+
//
|
|
133
|
+
// Asset digests change with every deploy, so any page cached before the current one was built
|
|
134
|
+
// disagrees with it — nothing served from this cache is tracked. A live page still is, so the
|
|
135
|
+
// first fresh page after the connection returns still reloads, which is what was wanted.
|
|
136
|
+
function untrack(html) {
|
|
137
|
+
return html.replace(/\sdata-turbo-track\s*=\s*(?:"reload"|'reload'|reload)(?=[\s>/])/gi, "")
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
// Everything answered from the cache: untracked always, marked when mark_cached_pages is on.
|
|
141
|
+
//
|
|
142
|
+
// Two markers, read at different moments. The <html> attributes are for the first paint of a
|
|
143
|
+
// cold boot, before any JS runs. The <meta> is for Turbo visits, which merge the head but
|
|
144
|
+
// never copy <html> attributes.
|
|
145
|
+
async function cachedPageResponse(cache, request, cached) {
|
|
146
|
+
if (!cached || !wantsHtml(request)) return cached
|
|
147
|
+
|
|
148
|
+
const type = cached.headers.get("Content-Type") || ""
|
|
149
|
+
if (!type.includes("text/html")) return cached
|
|
150
|
+
|
|
151
|
+
const html = await cached.clone().text()
|
|
152
|
+
if (!/<html\b/i.test(html)) return cached
|
|
153
|
+
|
|
154
|
+
let body = untrack(html)
|
|
155
|
+
|
|
156
|
+
if (MARK_CACHED_PAGES) {
|
|
157
|
+
// The timestamp rides on the stored key, so ask the cache for the key that matched.
|
|
158
|
+
const [ key ] = await cache.keys(request, MATCH_OPTIONS)
|
|
159
|
+
const cachedAt = key ? key.headers.get(TIMESTAMP_HEADER) : null
|
|
160
|
+
const stamp = cachedAt ? ` ${CACHED_AT_ATTRIBUTE}="${escapeHtml(cachedAt)}"` : ""
|
|
161
|
+
|
|
162
|
+
body = body
|
|
163
|
+
.replace(/<html\b([^>]*)>/i, `<html$1 ${OFFLINE_ATTRIBUTE}${stamp}>`)
|
|
164
|
+
.replace(/<head\b([^>]*)>/i, `<head$1><meta name="coldwire-offline" content="${escapeHtml(cachedAt || "")}">`)
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
// Rewriting changed the length, so the stored Content-Length no longer describes the
|
|
168
|
+
// body — carrying it over invites the consumer to truncate the page. Drop it and let the
|
|
169
|
+
// response report its own length.
|
|
170
|
+
const headers = new Headers(cached.headers)
|
|
171
|
+
headers.delete("Content-Length")
|
|
172
|
+
|
|
173
|
+
return new Response(body, {
|
|
174
|
+
status: cached.status,
|
|
175
|
+
statusText: cached.statusText,
|
|
176
|
+
headers
|
|
177
|
+
})
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
// Turbo Drive visits and frame loads ask for HTML; assets and JSON do not. Handing an HTML
|
|
181
|
+
// body to a stylesheet or an <img> just produces a broken asset, so those fail instead.
|
|
182
|
+
function wantsHtml(request) {
|
|
183
|
+
if (request.mode === "navigate" || request.destination === "document") return true
|
|
184
|
+
|
|
185
|
+
return (request.headers.get("Accept") || "").includes("text/html")
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
function offlineResponse(request) {
|
|
189
|
+
if (!wantsHtml(request)) {
|
|
190
|
+
return new Response("", { status: 504, statusText: "Offline" })
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
// A frame request only ever renders a matching <turbo-frame>; the full page would be
|
|
194
|
+
// discarded and the frame would sit on its loading state forever.
|
|
195
|
+
const frame = request.headers.get("Turbo-Frame")
|
|
196
|
+
|
|
197
|
+
return new Response(frame ? offlineFrame(frame, request.url) : OFFLINE_PAGE, {
|
|
198
|
+
// 200 on purpose. Hotwire Native treats a non-2xx visit as a failed request and shows
|
|
199
|
+
// its own native error screen, so an error-status body is never rendered.
|
|
200
|
+
status: 200,
|
|
201
|
+
headers: {
|
|
202
|
+
"Content-Type": "text/html; charset=utf-8",
|
|
203
|
+
"Cache-Control": "no-store"
|
|
204
|
+
}
|
|
205
|
+
})
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
// The frame template is baked when the worker is built, so it cannot know which URL it will
|
|
209
|
+
// stand in for. Substitute it here: a retry link inside a frame has to point at the frame's
|
|
210
|
+
// own URL, since an empty href would resolve to the page and load the whole document into
|
|
211
|
+
// the card.
|
|
212
|
+
function offlineFrame(id, url) {
|
|
213
|
+
const content = OFFLINE_FRAME_CONTENT.split(RETRY_URL_TOKEN).join(escapeHtml(url))
|
|
214
|
+
|
|
215
|
+
return `<turbo-frame id="${escapeHtml(id)}">${content}</turbo-frame>`
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
function escapeHtml(value) {
|
|
219
|
+
return String(value).replace(/[&<>"']/g, (character) => ({
|
|
220
|
+
"&": "&",
|
|
221
|
+
"<": "<",
|
|
222
|
+
">": ">",
|
|
223
|
+
'"': """,
|
|
224
|
+
"'": "'"
|
|
225
|
+
})[character])
|
|
226
|
+
}
|
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
// Bringing the cache in line with the manifest, without refetching what is already good.
|
|
2
|
+
//
|
|
3
|
+
// Only one runs at a time: the page triggers this on load, and a slow sync must not have a
|
|
4
|
+
// second one pile in behind it.
|
|
5
|
+
let syncing = null
|
|
6
|
+
|
|
7
|
+
function syncManifest() {
|
|
8
|
+
if (!syncing) {
|
|
9
|
+
syncing = runSync().finally(() => { syncing = null })
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
return syncing
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
async function runSync() {
|
|
16
|
+
// Force offline is a request to use no network at all. A sync is nothing but network, and
|
|
17
|
+
// the worker's own fetches do not pass through its fetch handler — so without this it would
|
|
18
|
+
// quietly go to the network anyway, which is the one thing the switch exists to prevent.
|
|
19
|
+
if (forcedOffline) {
|
|
20
|
+
const paused = { ok: false, offline: true, reason: "forced", complete: false,
|
|
21
|
+
cached: 0, retired: 0, remaining: 0, failed: [], finishedAt: Date.now() }
|
|
22
|
+
await notifyClients({ type: SYNC_MESSAGE, state: "finished", ...paused })
|
|
23
|
+
|
|
24
|
+
return paused
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
if (!cachingEnabled) {
|
|
28
|
+
const paused = { ok: false, offline: true, reason: "disabled", complete: false,
|
|
29
|
+
cached: 0, retired: 0, remaining: 0, failed: [], finishedAt: Date.now() }
|
|
30
|
+
await notifyClients({ type: SYNC_MESSAGE, state: "finished", ...paused })
|
|
31
|
+
|
|
32
|
+
return paused
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
const cache = await caches.open(CACHE_NAME)
|
|
36
|
+
|
|
37
|
+
let manifest
|
|
38
|
+
try {
|
|
39
|
+
manifest = await fetchManifest()
|
|
40
|
+
} catch (error) {
|
|
41
|
+
// No connection is not the manifest misbehaving, and retrying sooner cannot help. Said
|
|
42
|
+
// plainly so a page waits for the network instead of counting it as a failed attempt.
|
|
43
|
+
const failure = { ok: false, error: error.message, offline: Boolean(error.offline),
|
|
44
|
+
complete: false, cached: 0, retired: 0, remaining: 0, failed: [],
|
|
45
|
+
finishedAt: Date.now() }
|
|
46
|
+
await notifyClients({ type: SYNC_MESSAGE, state: "finished", ...failure })
|
|
47
|
+
|
|
48
|
+
return failure
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const wanted = new Set(manifest.map((value) => cacheUrl(new URL(value, self.location.origin).href)))
|
|
52
|
+
|
|
53
|
+
// Retire what the manifest dropped — an unpublished site, say. Only entries the manifest
|
|
54
|
+
// owns, so assets and ordinary browsing are left alone.
|
|
55
|
+
const retired = await retireUnlisted(cache, wanted)
|
|
56
|
+
|
|
57
|
+
// Fetch what is missing (a new site) or past its age (a stale one). Everything else is
|
|
58
|
+
// already good and costs nothing.
|
|
59
|
+
const now = Date.now() / 1000
|
|
60
|
+
const pending = []
|
|
61
|
+
for (const href of wanted) {
|
|
62
|
+
const [ key ] = await cache.keys(new Request(href), MATCH_OPTIONS)
|
|
63
|
+
const at = key ? unixTimestamp(key.headers.get(TIMESTAMP_HEADER)) : null
|
|
64
|
+
|
|
65
|
+
if (!key || !at || (REFETCH_AFTER !== null && now - at > REFETCH_AFTER)) pending.push(href)
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
// Every open page hears about this, not just the one that asked. A sync outlives the page
|
|
69
|
+
// that triggered it, so by the time it lands the user is usually somewhere else — and the
|
|
70
|
+
// page that started it can no longer record that it finished, or show it happening.
|
|
71
|
+
notifyClients({ type: SYNC_MESSAGE, state: "started", pending: pending.length, retired })
|
|
72
|
+
|
|
73
|
+
const result = await precacheUrls(pending, (progress) => {
|
|
74
|
+
notifyClients({ type: SYNC_MESSAGE, state: "progress", ...progress })
|
|
75
|
+
})
|
|
76
|
+
|
|
77
|
+
// The whole manifest was attempted, so what is left is what would not fetch. Those stay
|
|
78
|
+
// missing from the cache, so the next sync finds them pending again and tries once more.
|
|
79
|
+
const remaining = result.pageFailures
|
|
80
|
+
const complete = remaining === 0
|
|
81
|
+
// When the run actually ended, decided here rather than by whoever hears about it. A page
|
|
82
|
+
// that opens later and asks what it missed must be able to say "synced four minutes ago",
|
|
83
|
+
// not re-date the run to the moment it happened to look.
|
|
84
|
+
const finished = { ...result, retired, synced: pending.length - remaining, remaining, complete,
|
|
85
|
+
finishedAt: Date.now() }
|
|
86
|
+
|
|
87
|
+
await notifyClients({ type: SYNC_MESSAGE, state: "finished", ...finished })
|
|
88
|
+
|
|
89
|
+
return finished
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
// Kept so a page can ask what it missed. The head snippet starts a sync while the page is
|
|
93
|
+
// still parsing, so a short run can begin and end before any Stimulus controller has
|
|
94
|
+
// connected to hear it — without this the offline settings page would sit on "Idle" through a sync it
|
|
95
|
+
// caused itself.
|
|
96
|
+
let lastSyncMessage = null
|
|
97
|
+
|
|
98
|
+
async function notifyClients(message) {
|
|
99
|
+
lastSyncMessage = message
|
|
100
|
+
|
|
101
|
+
const clients = await self.clients.matchAll({ includeUncontrolled: true, type: "window" })
|
|
102
|
+
clients.forEach((client) => client.postMessage(message))
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
async function fetchManifest() {
|
|
106
|
+
let response
|
|
107
|
+
|
|
108
|
+
try {
|
|
109
|
+
response = await fetch(PACK_PATH, {
|
|
110
|
+
credentials: "same-origin",
|
|
111
|
+
// The manifest has to describe the server as it is now. Left to the HTTP cache a
|
|
112
|
+
// browser may serve a heuristically-fresh copy without asking, and a sync would then
|
|
113
|
+
// faithfully fetch an old list, missing the pages it exists to pick up.
|
|
114
|
+
cache: "no-store",
|
|
115
|
+
headers: { Accept: "application/json" }
|
|
116
|
+
})
|
|
117
|
+
} catch (error) {
|
|
118
|
+
// The fetch never completed at all: no connection, rather than a server saying no.
|
|
119
|
+
const offline = new Error("No connection")
|
|
120
|
+
offline.offline = true
|
|
121
|
+
throw offline
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
// Signed out, this follows a redirect to the login page and arrives as a perfectly ok 200
|
|
125
|
+
// of HTML, which would fail as an opaque JSON parse error.
|
|
126
|
+
if (response.redirected) throw new Error("Signed out")
|
|
127
|
+
if (!response.ok) throw new Error(`HTTP ${response.status}`)
|
|
128
|
+
|
|
129
|
+
const pack = await response.json()
|
|
130
|
+
|
|
131
|
+
return Array.isArray(pack.urls) ? pack.urls : []
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
async function retireUnlisted(cache, wanted) {
|
|
135
|
+
const keys = await cache.keys()
|
|
136
|
+
let retired = 0
|
|
137
|
+
|
|
138
|
+
for (const key of keys) {
|
|
139
|
+
if (key.headers.get(MANAGED_HEADER) !== "1") continue
|
|
140
|
+
if (wanted.has(cacheUrl(key.url))) continue
|
|
141
|
+
|
|
142
|
+
await cache.delete(key)
|
|
143
|
+
retired++
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
return retired
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
// Compare URLs the same way they are stored, or every entry looks unlisted the moment a
|
|
150
|
+
// query string is involved.
|
|
151
|
+
function cacheUrl(href) {
|
|
152
|
+
const url = new URL(href)
|
|
153
|
+
if (IGNORE_SEARCH) url.search = ""
|
|
154
|
+
|
|
155
|
+
return url.href
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
// A fixed number of lanes pulling from one queue.
|
|
159
|
+
//
|
|
160
|
+
// One at a time takes as many round trips as there are URLs — minutes for a real manifest.
|
|
161
|
+
// Promise.all over the lot opens a connection per URL and leaves the app's own requests
|
|
162
|
+
// queued behind its own precaching, which is worse than slow.
|
|
163
|
+
async function runPool(items, concurrency, handler) {
|
|
164
|
+
const queue = items.slice()
|
|
165
|
+
const lanes = Math.max(1, Math.min(concurrency, queue.length))
|
|
166
|
+
|
|
167
|
+
await Promise.all(Array.from({ length: lanes }, async () => {
|
|
168
|
+
while (queue.length) {
|
|
169
|
+
try {
|
|
170
|
+
await handler(queue.shift())
|
|
171
|
+
} catch {
|
|
172
|
+
// A lane has to outlive its work. Callers record their own failures, so this is a
|
|
173
|
+
// backstop — but without it one unexpected throw kills that lane, rejects the
|
|
174
|
+
// Promise.all, and abandons every URL still queued behind it.
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
}))
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
// One retry, because a sync runs for a while on a phone and a single dropped request should
|
|
181
|
+
// not leave a page missing until the next interval comes round.
|
|
182
|
+
async function fetchWithRetry(cache, href, options) {
|
|
183
|
+
try {
|
|
184
|
+
return await fetchAndCache(cache, href, options)
|
|
185
|
+
} catch {
|
|
186
|
+
await new Promise((resolve) => setTimeout(resolve, 500))
|
|
187
|
+
|
|
188
|
+
return fetchAndCache(cache, href, options)
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
async function precacheUrls(inputUrls, onProgress = () => {}) {
|
|
193
|
+
const cache = await caches.open(CACHE_NAME)
|
|
194
|
+
// A Set, not an array: lanes finish out of order and an array turns the "already have it"
|
|
195
|
+
// check into a scan per asset.
|
|
196
|
+
const cached = new Set()
|
|
197
|
+
const failed = []
|
|
198
|
+
const pageFailures = []
|
|
199
|
+
const assets = new Set()
|
|
200
|
+
|
|
201
|
+
let done = 0
|
|
202
|
+
onProgress({ phase: "pages", done, total: inputUrls.length })
|
|
203
|
+
|
|
204
|
+
await runPool(inputUrls, SYNC_CONCURRENCY, async (value) => {
|
|
205
|
+
try {
|
|
206
|
+
const href = new URL(value, self.location.origin).href
|
|
207
|
+
const extra = await fetchWithRetry(cache, href, { managed: true })
|
|
208
|
+
cached.add(href)
|
|
209
|
+
extra.forEach((url) => assets.add(url))
|
|
210
|
+
} catch {
|
|
211
|
+
failed.push(String(value))
|
|
212
|
+
pageFailures.push(String(value))
|
|
213
|
+
}
|
|
214
|
+
onProgress({ phase: "pages", done: ++done, total: inputUrls.length })
|
|
215
|
+
})
|
|
216
|
+
|
|
217
|
+
// Assets are only known once the pages are parsed, so they get their own count.
|
|
218
|
+
const pending = [...assets].filter((href) => !cached.has(href))
|
|
219
|
+
done = 0
|
|
220
|
+
onProgress({ phase: "assets", done, total: pending.length })
|
|
221
|
+
|
|
222
|
+
await runPool(pending, SYNC_CONCURRENCY, async (href) => {
|
|
223
|
+
try {
|
|
224
|
+
await fetchWithRetry(cache, href)
|
|
225
|
+
cached.add(href)
|
|
226
|
+
} catch {
|
|
227
|
+
failed.push(href)
|
|
228
|
+
}
|
|
229
|
+
onProgress({ phase: "assets", done: ++done, total: pending.length })
|
|
230
|
+
})
|
|
231
|
+
|
|
232
|
+
return { ok: failed.length === 0, cached: cached.size, failed, pageFailures: pageFailures.length }
|
|
233
|
+
}
|
|
234
|
+
|
data/lib/coldwire.rb
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "coldwire/version"
|
|
4
|
+
require "coldwire/configuration"
|
|
5
|
+
require "coldwire/source"
|
|
6
|
+
require "coldwire/client_user_agent"
|
|
7
|
+
require "coldwire/engine"
|
|
8
|
+
|
|
9
|
+
module Coldwire
|
|
10
|
+
class << self
|
|
11
|
+
def config
|
|
12
|
+
@config ||= Configuration.new
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def configure
|
|
16
|
+
yield config
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
# Paths the worker never intercepts: whatever the host excluded, plus the engine endpoints
|
|
20
|
+
# passed in.
|
|
21
|
+
#
|
|
22
|
+
# Only two of Coldwire's own endpoints qualify — the worker script and the manifest —
|
|
23
|
+
# because caching either would strand the app on a stale copy of the thing meant to
|
|
24
|
+
# refresh it. The offline settings page is ordinary HTML and is left interceptable, so a host that
|
|
25
|
+
# wants to reach it offline can list it in `cache_as_you_go` like any other page.
|
|
26
|
+
def never_intercept(*engine_paths)
|
|
27
|
+
normalize(config.never_intercept + engine_paths)
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# Serializes a mixed list of strings and Regexps into something the worker can rebuild.
|
|
31
|
+
# Ruby's `i` is the only flag with a safe JavaScript equivalent; the others are rejected
|
|
32
|
+
# when the list is assigned.
|
|
33
|
+
def cache_rules(patterns)
|
|
34
|
+
Array(patterns).filter_map do |pattern|
|
|
35
|
+
if pattern.is_a?(Regexp)
|
|
36
|
+
{ type: "regexp", source: pattern.source,
|
|
37
|
+
flags: pattern.options.anybits?(Regexp::IGNORECASE) ? "i" : "" }
|
|
38
|
+
else
|
|
39
|
+
# Trim a trailing slash so "/sites/" and "/sites" behave alike, but not from "/"
|
|
40
|
+
# itself — chomping that leaves an empty string and the rule vanishes, which is a
|
|
41
|
+
# silent way to lose the root path.
|
|
42
|
+
path = pattern.to_s
|
|
43
|
+
path = path.chomp("/") unless path == "/"
|
|
44
|
+
{ type: "path", value: path } unless path.empty?
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
private
|
|
50
|
+
|
|
51
|
+
def normalize(paths)
|
|
52
|
+
paths.map { |path| path.to_s.chomp("/") }.reject(&:empty?).uniq
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "rails/generators"
|
|
4
|
+
|
|
5
|
+
module Coldwire
|
|
6
|
+
module Generators
|
|
7
|
+
# `bin/rails coldwire:install` / `bin/rails generate coldwire:install`
|
|
8
|
+
#
|
|
9
|
+
# Mounts the engine, writes the initializer, registers the Stimulus controller, and
|
|
10
|
+
# drops the service worker tag into the layout. Each step is skipped if it is already
|
|
11
|
+
# done, so running it twice is safe.
|
|
12
|
+
class InstallGenerator < Rails::Generators::Base
|
|
13
|
+
source_root File.expand_path("templates", __dir__)
|
|
14
|
+
|
|
15
|
+
desc "Install Coldwire into this application"
|
|
16
|
+
|
|
17
|
+
LAYOUTS = [
|
|
18
|
+
"app/views/layouts/application.html.erb"
|
|
19
|
+
].freeze
|
|
20
|
+
|
|
21
|
+
STIMULUS_INDEXES = [
|
|
22
|
+
"app/javascript/controllers/index.js",
|
|
23
|
+
"app/javascript/controllers/index.ts"
|
|
24
|
+
].freeze
|
|
25
|
+
|
|
26
|
+
STIMULUS_REGISTRATION = <<~JS
|
|
27
|
+
|
|
28
|
+
import ColdwireCacheController from "coldwire"
|
|
29
|
+
application.register("coldwire-cache", ColdwireCacheController)
|
|
30
|
+
JS
|
|
31
|
+
|
|
32
|
+
def add_route
|
|
33
|
+
if file_contains?("config/routes.rb", "Coldwire::Engine")
|
|
34
|
+
say "Coldwire is already mounted in config/routes.rb, skipping"
|
|
35
|
+
return
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
route 'mount Coldwire::Engine => "/offline"'
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def copy_initializer
|
|
42
|
+
if file_exists?("config/initializers/coldwire.rb")
|
|
43
|
+
say "config/initializers/coldwire.rb already exists, skipping"
|
|
44
|
+
return
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
copy_file "coldwire.rb", "config/initializers/coldwire.rb"
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def add_layout_tag
|
|
51
|
+
layout = first_existing(LAYOUTS)
|
|
52
|
+
unless layout
|
|
53
|
+
say "Could not find app/views/layouts/application.html.erb. " \
|
|
54
|
+
"Add <%= coldwire_service_worker_tag %> inside <head>.", :yellow
|
|
55
|
+
return
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
if file_contains?(layout, "coldwire_service_worker_tag")
|
|
59
|
+
say "#{layout} already includes coldwire_service_worker_tag, skipping"
|
|
60
|
+
return
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
unless file_contains?(layout, "</head>")
|
|
64
|
+
say "Could not find </head> in #{layout}. " \
|
|
65
|
+
"Add <%= coldwire_service_worker_tag %> inside <head>.", :yellow
|
|
66
|
+
return
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
insert_into_file layout, " <%= coldwire_service_worker_tag %>\n", before: "</head>"
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def register_stimulus
|
|
73
|
+
index = first_existing(STIMULUS_INDEXES)
|
|
74
|
+
unless index
|
|
75
|
+
say "Could not find app/javascript/controllers/index.js. Register the controller with:", :yellow
|
|
76
|
+
say STIMULUS_REGISTRATION
|
|
77
|
+
return
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
if file_contains?(index, "coldwire-cache") || file_contains?(index, 'from "coldwire"')
|
|
81
|
+
say "#{index} already registers Coldwire, skipping"
|
|
82
|
+
return
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
append_to_file index, STIMULUS_REGISTRATION
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def next_steps
|
|
89
|
+
say ""
|
|
90
|
+
say "Coldwire is mounted at /offline.", :green
|
|
91
|
+
say "Set config.cache_identity if anyone signs in."
|
|
92
|
+
say "Turn on config.auto_sync to precache pages."
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
private
|
|
96
|
+
|
|
97
|
+
def first_existing(paths)
|
|
98
|
+
paths.find { |path| file_exists?(path) }
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def file_exists?(relative)
|
|
102
|
+
File.exist?(File.join(destination_root, relative))
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def file_contains?(relative, snippet)
|
|
106
|
+
file_exists?(relative) && File.read(File.join(destination_root, relative)).include?(snippet)
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
end
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Offline caching. Only `auto_sync` really needs your attention; the rest has a working
|
|
4
|
+
# default. See docs/configuration.md in the coldwire-rails gem for every option.
|
|
5
|
+
Coldwire.configure do |config|
|
|
6
|
+
# What to precache and how often. Evaluated against your app's URL helpers.
|
|
7
|
+
config.auto_sync do |sync|
|
|
8
|
+
sync.enabled = false # off by default: background fetching is somebody's data plan
|
|
9
|
+
sync.precache_urls = -> { [] }
|
|
10
|
+
# sync.precache_urls = -> { Article.published.map { |a| article_path(a) } }
|
|
11
|
+
sync.interval = 6.hours
|
|
12
|
+
sync.max_age = 7.days
|
|
13
|
+
sync.concurrency = 4
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
# Who the cache belongs to. Evaluated in the view. When it changes, the cache is dropped —
|
|
17
|
+
# which is what makes signing out, and switching accounts, safe.
|
|
18
|
+
config.cache_identity = -> { nil }
|
|
19
|
+
# config.cache_identity = -> { current_user&.id }
|
|
20
|
+
|
|
21
|
+
# Where the worker registers at all. Evaluated in the view, so `request` and `current_user`
|
|
22
|
+
# are both in scope. A page that does not register does not cache or sync.
|
|
23
|
+
config.register_if = -> { true }
|
|
24
|
+
|
|
25
|
+
# Default for the Offline support switch on the offline settings page. People can turn
|
|
26
|
+
# it off there, which deletes what is stored. A fresh device follows this.
|
|
27
|
+
config.caching_enabled_by_default = true
|
|
28
|
+
|
|
29
|
+
# The importmap module the offline page loads to boot Turbo. nil if you are not on
|
|
30
|
+
# importmap-rails; load Turbo your own way in the template instead.
|
|
31
|
+
config.offline_import = "@hotwired/turbo-rails"
|
|
32
|
+
|
|
33
|
+
# Which pages are kept as somebody browses. What one needs to render comes with it.
|
|
34
|
+
# "/*" is everything; `never_cache` always wins. An empty list stores nothing by browsing.
|
|
35
|
+
config.cache_as_you_go = [ "/*" ]
|
|
36
|
+
config.never_cache = []
|
|
37
|
+
|
|
38
|
+
# Never intercepted, so these fail outright offline. Coldwire's own routes are added for you.
|
|
39
|
+
config.never_intercept = [ "/up" ] # probe_path is added for you
|
|
40
|
+
|
|
41
|
+
# Origins besides your own the worker may cache, and URLs whose Range requests it caches.
|
|
42
|
+
config.cache_origins = []
|
|
43
|
+
config.cache_ranges = []
|
|
44
|
+
|
|
45
|
+
# Large files somebody can download for offline use. Nothing downloads on its own.
|
|
46
|
+
config.cache_archives = []
|
|
47
|
+
|
|
48
|
+
config.probe_path = "/up" # pinged to tell online from offline
|
|
49
|
+
config.mark_cached_pages = true # stamp HTML served from cache
|
|
50
|
+
config.ignore_query_params = true # treat "/map" and "/map?zoom=9" as one page
|
|
51
|
+
config.cache_name = "coldwire" # bump to invalidate every entry at once
|
|
52
|
+
config.worker_scope = "/"
|
|
53
|
+
end
|