coldwire-rails 0.3.0 → 0.5.0

data/lib/coldwire/worker/serve.js

@@ -9,8 +9,14 @@ async function putFresh(cache, key, response) {
   await cache.put(key, response)
 }
 
-async function fetchAndCache(cache, href, { managed = false } = {}) {
-  const request = new Request(href, { credentials: "same-origin" })
+async function fetchAndCache(cache, href, { managed = false, frame = null, accept = null } = {}) {
+  const headers = {}
+  // A manifest listing that names a frame or a format is asking for the body a request like
+  // that would get, so the request has to look like one.
+  if (frame) headers["Turbo-Frame"] = frame
+  if (accept) headers.Accept = accept
+
+  const request = new Request(href, { credentials: "same-origin", headers })
   // Skipped rather than failed: the app asked for this URL never to be stored, and a manifest
   // that also names it is a contradiction to resolve quietly in favour of not storing.
   if (isNeverCached(new URL(request.url))) return []
@@ -20,7 +26,10 @@ async function fetchAndCache(cache, href, { managed = false } = {}) {
     throw new Error(response.redirected ? `Redirected to ${response.url}` : `HTTP ${response.status}`)
   }
 
-  await putFresh(cache, cacheKey(request, { managed }), response.clone())
+  const key = managed
+    ? cacheKey(request, { managed, frame })
+    : await bodyKey(cache, request, response)
+  await putFresh(cache, key, response.clone())
 
   const contentType = response.headers.get("Content-Type") || ""
   if (!contentType.includes("text/html")) return []
@@ -94,21 +103,124 @@ async function handleFetch(request, event) {
 }
 
 async function offlineFallback(cache, request) {
-  const cached = await cache.match(request, MATCH_OPTIONS)
+  const key = await matchStored(cache, request)
+  const cached = key ? await cache.match(key) : undefined
 
-  return (await cachedPageResponse(cache, request, cached)) || offlineResponse(request)
+  return (await cachedPageResponse(request, key, cached)) || offlineResponse(request)
+}
+
+// Which stored entry answers this request. One URL can hold the page and a frame of the same
+// name, so this asks for the kind of body the request wants rather than taking whatever the
+// URL turns up — and with ignoreSearch on, what it turns up might be a frame, a byte range, or
+// a chunk of an archive.
+//
+// A frame takes its own entry first and a whole page second: an app that answers frame
+// requests with the full document stores one of those, and Turbo pulls the frame out of it
+// exactly as it does online. The trade never runs the other way. A fragment served to an
+// ordinary visit is a document with no <html>, which is a blank screen, and in Hotwire Native
+// a page where window.Turbo never appears.
+async function matchStored(cache, request) {
+  const frame = request.headers.get("Turbo-Frame")
+
+  if (frame) {
+    // Asked for by name rather than found by scanning. With ignore_query_params off the search
+    // has to match exactly, and a request carrying no query would never turn up the key that
+    // carries the frame in its own.
+    const [ own ] = await cache.keys(variantUrl(request.url, { frame }), { ignoreVary: true })
+    if (own) return own
+  }
+
+  // Asked for by name, for the same reason the frame is.
+  const format = formatOf(request)
+  if (format !== "page") {
+    const [ named ] = await cache.keys(variantUrl(request.url, { format }), { ignoreVary: true })
+    if (named) return named
+  }
+
+  const keys = await cache.keys(request, MATCH_OPTIONS)
+  const page = keys.find((candidate) => variantOf(candidate) === "page")
+  if (!page) return undefined
+
+  // The unnamed entry is the right answer for anything cached before formats were named, and
+  // for every asset a precache stored without an Accept to go on. It is the wrong answer for a
+  // fetch that asked for data and would be handed a page, which is the mistake that gives a
+  // stylesheet an HTML body. So ask what it holds before handing it over.
+  if (format === "page") return page
+
+  const held = await cache.match(page)
+
+  return (held && !(held.headers.get("Content-Type") || "").includes("text/html")) ? page : undefined
+}
+
+// Where a body that is not a page goes: named by the format the request asked for, unless this
+// URL already holds something that is not a page.
+//
+// That exception is the whole reason this is a function. A precache carries no Accept and lands
+// on the unnamed key; the browser then asks for the same stylesheet as `text/css` and would
+// write a second copy beside it. Two of every stylesheet and every image is not a rounding
+// error on a phone.
+//
+// The lookup is only paid where a name would be written at all, so `*/*` requests — scripts,
+// fonts, and precaching itself — cost nothing extra.
+async function bodyKey(cache, request, response) {
+  if (formatOf(request) !== "page") {
+    const [ plain ] = await cache.keys(variantUrl(request.url, {}), { ignoreVary: true })
+    const held = plain ? await cache.match(plain) : null
+    if (held && !(held.headers.get("Content-Type") || "").includes("text/html")) {
+      return cacheKey(request, { frame: null, format: "page" })
+    }
+
+    return cacheKey(request, { frame: null })
+  }
+
+  // A URL that names its own format is the same key from every direction, so there is nothing
+  // another request could have named differently and nothing to look for.
+  if (extensionFormat(request.url) !== "page") return cacheKey(request, { frame: null })
+
+  // And the other way round: this request named nothing, but the browser may already have
+  // stored the same file under the format it asked for. Work that name out from what came
+  // back, rather than writing a second copy beside it.
+  const guess = formatFromType(response)
+  if (guess !== "page") {
+    const [ named ] = await cache.keys(variantUrl(request.url, { format: guess }), { ignoreVary: true })
+    if (named) return cacheKey(request, { frame: null, format: guess })
+  }
+
+  return cacheKey(request, { frame: null })
+}
+
+// What a browser's Accept would have produced for a body like this. Only the two families a
+// browser asks for by type: a script or a font arrives on `*/*` and is unnamed on both sides,
+// so neither pays for this lookup.
+function formatFromType(response) {
+  const type = (response.headers.get("Content-Type") || "").split(";")[0].trim().toLowerCase()
+  if (type.startsWith("text/css")) return "css"
+  if (type.startsWith("image/")) return "image"
+
+  return "page"
 }
 
 // A page is stored with whatever it asks for. The lists say which pages are worth keeping as
 // you browse; what one needs in order to render is not a second question, and a page held
 // without its stylesheet is the offline equivalent of not holding it at all.
 async function storeResponse(cache, request, response) {
-  await putFresh(cache, cacheKey(request), response.clone())
-
   const type = response.headers.get("Content-Type") || ""
-  if (!type.includes("text/html")) return
+  if (!type.includes("text/html")) {
+    await putFresh(cache, await bodyKey(cache, request, response), response.clone())
+    return
+  }
+
+  // Read before writing, because what this is decides where it goes. Cloning twice is free:
+  // each clone tees the stream, so consuming one leaves the other whole.
+  const body = await response.clone().text()
+  // A frame request the app answered with the entire document is a page like any other, and
+  // Turbo will find the frame inside it. Storing that under a frame key would hold the same
+  // bytes twice and leave the next ordinary visit unable to find them.
+  const frame = /<html\b/i.test(body) ? null : request.headers.get("Turbo-Frame")
 
-  const urls = urlsFromHtml(await response.text(), request.url)
+  await putFresh(cache, cacheKey(request, { frame }), response.clone())
+
+  const urls = urlsFromHtml(body, request.url)
   await Promise.all(urls.map((href) => storeSubresource(cache, href)))
 }
 
@@ -118,7 +230,14 @@ async function storeResponse(cache, request, response) {
 async function storeSubresource(cache, href) {
   if (isNeverCached(new URL(href))) return
 
-  const [ key ] = await cache.keys(href, MATCH_OPTIONS)
+  // Any body for this URL that is not a frame of it and not a piece of a download. Asking only
+  // for the unnamed one would refetch a stylesheet the browser has already stored under its
+  // own format, and land a second copy of it here.
+  const key = (await cache.keys(href, MATCH_OPTIONS)).find((held) => {
+    const variant = variantOf(held)
+
+    return variant !== "download" && !variant.startsWith("frame:")
+  })
   if (key) return renew(cache, key)
 
   try {
@@ -145,7 +264,7 @@ function untrack(html) {
 // Two markers, read at different moments. The <html> attributes are for the first paint of a
 // cold boot, before any JS runs. The <meta> is for Turbo visits, which merge the head but
 // never copy <html> attributes.
-async function cachedPageResponse(cache, request, cached) {
+async function cachedPageResponse(request, key, cached) {
   if (!cached || !wantsHtml(request)) return cached
 
   const type = cached.headers.get("Content-Type") || ""
@@ -157,8 +276,8 @@ async function cachedPageResponse(cache, request, cached) {
   let body = untrack(html)
 
   if (MARK_CACHED_PAGES) {
-    // The timestamp rides on the stored key, so ask the cache for the key that matched.
-    const [ key ] = await cache.keys(request, MATCH_OPTIONS)
+    // The timestamp rides on the stored key, which the caller has already had to find: asking
+    // the cache again could pick a different entry for the same URL than the one being served.
     const cachedAt = key ? key.headers.get(TIMESTAMP_HEADER) : null
     const stamp = cachedAt ? ` ${CACHED_AT_ATTRIBUTE}="${escapeHtml(cachedAt)}"` : ""
 

data/lib/coldwire/worker/sync.js

@@ -48,7 +48,8 @@ async function runSync() {
     return failure
   }
 
-  const wanted = new Set(manifest.map((value) => cacheUrl(new URL(value, self.location.origin).href)))
+  const entries = manifest.map(manifestEntry)
+  const wanted = new Set(entries.map((entry) => entry.key))
 
   // Retire what the manifest dropped — an unpublished site, say. Only entries the manifest
   // owns, so assets and ordinary browsing are left alone.
@@ -58,11 +59,13 @@ async function runSync() {
   // already good and costs nothing.
   const now = Date.now() / 1000
   const pending = []
-  for (const href of wanted) {
-    const [ key ] = await cache.keys(new Request(href), MATCH_OPTIONS)
+  for (const entry of entries) {
+    // Asked for by the exact key this entry will be stored under. A page and a frame of it are
+    // two listings now, and either one being held says nothing about the other.
+    const [ key ] = await cache.keys(entry.key, { ignoreVary: true })
     const at = key ? unixTimestamp(key.headers.get(TIMESTAMP_HEADER)) : null
 
-    if (!key || !at || (REFETCH_AFTER !== null && now - at > REFETCH_AFTER)) pending.push(href)
+    if (!key || !at || (REFETCH_AFTER !== null && now - at > REFETCH_AFTER)) pending.push(entry)
   }
 
   // Every open page hears about this, not just the one that asked. A sync outlives the page
@@ -137,7 +140,10 @@ async function retireUnlisted(cache, wanted) {
 
   for (const key of keys) {
     if (key.headers.get(MANAGED_HEADER) !== "1") continue
-    if (wanted.has(cacheUrl(key.url))) continue
+    // Compared whole, params and all. Stripping the search here would take the frame or the
+    // format off a managed entry, leave it looking like a page nobody listed, and retire it on
+    // the first sync after it was fetched.
+    if (wanted.has(key.url)) continue
 
     await cache.delete(key)
     retired++
@@ -146,13 +152,26 @@ async function retireUnlisted(cache, wanted) {
   return retired
 }
 
-// Compare URLs the same way they are stored, or every entry looks unlisted the moment a
-// query string is involved.
-function cacheUrl(href) {
-  const url = new URL(href)
-  if (IGNORE_SEARCH) url.search = ""
+// One manifest listing, worked out once: the request that fetches it, and the key it will be
+// stored under. A bare URL is a page, as it always was; a listing that names a frame or an
+// Accept is asking for the body a request like that would get, and has to be stored where a
+// request like that will look for it.
+function manifestEntry(value) {
+  const listing = typeof value === "string" ? { url: value } : (value || {})
+  const href = new URL(listing.url, self.location.origin).href
+  const headers = {}
+
+  if (listing.frame) headers["Turbo-Frame"] = listing.frame
+  // What Turbo sends alongside the frame header. Without it the manifest asks with `*/*`, and
+  // a `respond_to` block answers with whatever it happens to list first.
+  if (listing.accept || listing.frame) {
+    headers.Accept = listing.accept || "text/html, application/xhtml+xml"
+  }
+
+  const request = new Request(href, { credentials: "same-origin", headers })
+  const key = variantUrl(href, { frame: listing.frame, format: formatOf(request) })
 
-  return url.href
+  return { href, frame: listing.frame, accept: headers.Accept, key, request }
 }
 
 // A fixed number of lanes pulling from one queue.
@@ -201,15 +220,16 @@ async function precacheUrls(inputUrls, onProgress = () => {}) {
   let done = 0
   onProgress({ phase: "pages", done, total: inputUrls.length })
 
-  await runPool(inputUrls, SYNC_CONCURRENCY, async (value) => {
+  await runPool(inputUrls, SYNC_CONCURRENCY, async (entry) => {
     try {
-      const href = new URL(value, self.location.origin).href
-      const extra = await fetchWithRetry(cache, href, { managed: true })
-      cached.add(href)
+      const extra = await fetchWithRetry(cache, entry.href, {
+        managed: true, frame: entry.frame, accept: entry.accept
+      })
+      cached.add(entry.href)
       extra.forEach((url) => assets.add(url))
     } catch {
-      failed.push(String(value))
-      pageFailures.push(String(value))
+      failed.push(entry.href)
+      pageFailures.push(entry.href)
     }
     onProgress({ phase: "pages", done: ++done, total: inputUrls.length })
   })

data/lib/coldwire.rb

@@ -2,6 +2,7 @@
 
 require "coldwire/version"
 require "coldwire/configuration"
+require "coldwire/precache"
 require "coldwire/source"
 require "coldwire/client_user_agent"
 require "coldwire/engine"

data/lib/generators/coldwire/install/templates/coldwire.rb

@@ -8,6 +8,8 @@ Coldwire.configure do |config|
     sync.enabled = false          # off by default: background fetching is somebody's data plan
     sync.precache_urls = -> { [] }
     # sync.precache_urls = -> { Article.published.map { |a| article_path(a) } }
+    # A Hash names a Turbo Frame or a format, for URLs that answer more than one way:
+    # sync.precache_urls = -> { Article.published.map { |a| { url: article_path(a), frame: "preview" } } }
     sync.interval = 1.day
     sync.max_age = 30.days
     sync.concurrency = 4

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: coldwire-rails
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Noreaster Group, Stuart Yamartino
@@ -80,6 +80,7 @@ files:
 - lib/coldwire/configuration.rb
 - lib/coldwire/debug.css
 - lib/coldwire/engine.rb
+- lib/coldwire/precache.rb
 - lib/coldwire/source.rb
 - lib/coldwire/version.rb
 - lib/coldwire/worker/archives.js