coldwire-rails 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b5e35d2c0930a9eac9b39b725205a467bda9a982b1fb543f127e3f2a0cd3adbe
-  data.tar.gz: 2bb1ad7cb8ded065ad6820885ca8429c582f0bc688a9c2eb288328e85e02591d
+  metadata.gz: 7b29177b5dd7843a32faa8d813d3e69cbac5e17c080740944203d0b441fd9fd2
+  data.tar.gz: 5a0623dd26cce626b03e899aacba3cf4514423dd204609d7a6be214b619bc78c
 SHA512:
-  metadata.gz: d15283073b65d9100749f0247c5ca4abebcdcfbbe429218499212ce67898b078a8222e6ebc533b8c1051607051e113ecf7ea12bfcc3273c16698859ce6c77e11
-  data.tar.gz: 9573b8cc389f6519222d606cdb1feb0a6019dc76024902d041f18697bb9d70fc149ddb88ce2c6ffe44da8267d25cbc232fe66758b4b8179c0eebc63fa63986ae
+  metadata.gz: c97aaf5882d7203931d463a8f9f0bb825bea50c5b8ddd5871db33411587359d1caf943c51d19f8a7629ac152b5310d466dc9504e078e9e7d47212202dbe796b7
+  data.tar.gz: 5be03d9e50e9cb0f85e7ae4a4233a65a249b709f4c9eb994a2c2b699fdb682a8d4350e087d1674be978b6ec22ecbd500c2e3a634b88eda91ddcc9c49168c011b

data/CHANGELOG.md

@@ -2,6 +2,22 @@
 
 ## [Unreleased]
 
+## [0.5.0]
+
+- **A Turbo Frame is cached apart from its page.** A frame navigation used to overwrite the
+  page at the same URL, so a later visit got a fragment as a whole document — a blank screen,
+  and in Hotwire Native no `window.Turbo`. Frame requests now take their own entry first and
+  the page second, the way Turbo does online.
+- **Every `respond_to` format is cached apart from the page.** One URL answering HTML, JSON,
+  CSV, or RSS no longer shares a single slot. A path that already names its format
+  (`/report.json`, `/app.css`) is left alone. A request asking for data gets data or nothing,
+  never the page.
+- **The precache manifest can name a frame or a format.** A listing may be a Hash:
+  `{ url: feature_path(f), frame: "map_feature_popup" }` or `{ url: report_path(r), format: :json }`.
+  `accept:` takes a media type outright. A bare URL is still the page.
+- **A Turbo Stream is never stored.** It is a list of mutations, not a page, and replaying a
+  stale one would apply yesterday's changes to today's DOM.
+
 ## [0.4.0]
 
 - Updated the offline page to restructure storage and downloads.
@@ -30,17 +46,8 @@ First release. The API may still change before 1.0.
   with every option and its default, registers the Stimulus controller, and tags the
   layout. Safe to run twice.
 - **Garbage collection.** `config.garbage_collection` sweeps entries nothing has used in
-  `max_age` (60 days by default) and, once the cache is over `max_size` (250 MB by default),
-  the least recently read of what is left until it fits — so a cache that fills as people
-  browse does not fill forever, on a device that revisits nothing as much as on one that
-  revisits everything. The ceiling is offered as a ladder of sizes on the offline settings
-  page and remembered per device, since how much of a phone to spend is not something an app
-  can know. It measures only what a sweep may take: downloaded archives are an opt-in spend
-  of somebody's data plan, so they are neither counted nor evicted. Choosing a size applies it
-  at once rather than at the next sweep, and that pass does not wait for a connection: it is a
-  deliberate instruction about somebody's own storage, which is how Clear cache has always
-  behaved.
-  On by default, unlike syncing: it spends no data. A sweep runs only with a
+  `max_age` (30 days by default), so a cache that fills as people browse does not fill
+  forever. On by default, unlike syncing: it spends no data. A sweep runs only with a
   connection it has confirmed by pinging `probe_path`, because deleting is the one cache
   operation with no way back. Age is measured from last use, not from when an entry was
   fetched — storing a page renews everything it names, so the stylesheet every page loads

data/VERSION

@@ -1 +1 @@
-0.4.0
+0.5.0

data/app/controllers/coldwire/caches_controller.rb

@@ -12,7 +12,7 @@ module Coldwire
     def pack
       response.headers["Cache-Control"] = "no-store, private"
 
-      render json: { urls: Array(precache_urls) }
+      render json: { urls: Coldwire.precache_entries(precache_urls) }
     end
 
     private

data/app/views/coldwire/service_worker/show.js.erb

@@ -26,6 +26,17 @@ const CACHE_RANGES = compileRules(<%= raw Coldwire.cache_rules(Coldwire.config.c
 // A range entry is stored under its own key, with the range in the query and the archive's
 // total size in a header — the two things needed to rebuild a 206 that was never storable.
 const RANGE_PARAM = "__coldwire_range"
+// Turbo sends `Turbo-Frame` on a frame navigation, and an app that answers it with just the
+// frame returns something that is not the page at that URL. Keyed apart, the same way, or one
+// would answer for the other: a fragment served to a cold visit is a document with no <html>.
+const FRAME_PARAM = "__coldwire_frame"
+// And the same URL answers a `respond_to` block in as many formats as the app defines. A page
+// is the default and carries no param, so every entry stored before this existed keeps the key
+// it already had; anything else names what it is.
+const FORMAT_PARAM = "__coldwire_format"
+// A stream is a list of changes to make to a page, not a page. Replaying a stale one later
+// would apply yesterday's mutations to today's DOM, so it is never stored at all.
+const STREAM_TYPE = "text/vnd.turbo-stream.html"
 // Archives offered for download in full, and the size of the pieces they arrive in. 8 MB is
 // small enough that a dropped connection loses seconds rather than minutes, and large enough
 // that a 300 MB archive is tens of requests rather than thousands.

data/docs/configuration.md

@@ -154,6 +154,36 @@ Listing a URL here is an explicit instruction: `cache_as_you_go` does not filter
 
 A stored page's stylesheets, scripts, and images are fetched with it, whatever the lists say.
 
+**Naming a frame or a format.** A URL is one field short of naming a body: the same path
+answers a visit, a Turbo Frame and a `respond_to` format with three different things, and a
+bare listing asks for the page. A Hash says which one you mean:
+
+```ruby
+sync.precache_urls = -> {
+  Feature.published.map { |f| { url: feature_path(f), frame: "map_feature_popup" } } +
+  Report.all.map { |r| { url: report_path(r), format: :json } } +
+  [ root_path ]
+}
+```
+
+`frame:` is the frame tag's own id, which is what Turbo puts in the `Turbo-Frame` header. The
+worker fetches with that header, so an app branching on `turbo_frame_request?` answers with the
+frame, and the body is stored where a frame request will look for it.
+
+`format:` is any registered Rails format, including one your app registered itself. It is
+resolved to that format's media type here and sent as `Accept`, so the worker never has to know
+what `:json` means. `accept:` takes a media type directly, for anything with no registered name.
+
+Without this, a frame URL is precached as its whole page. That still works offline, since a
+frame request falls back to the page and Turbo pulls the frame out of it, but only while the
+page actually contains that frame, and it stores the whole document where a fragment would do.
+
+List a URL twice to precache both:
+
+```ruby
+[ feature_path(f), { url: feature_path(f), frame: "map_feature_popup" } ]
+```
+
 ### `auto_sync.interval`
 
 **Default:** `1.day`

data/docs/how-it-works.md

@@ -23,6 +23,27 @@ Turbo — rather than a native error screen:
 3. **Turbo Frames need a frame.** A frame request discards any response without a matching
    `<turbo-frame>`, leaving the frame loading forever. Coldwire reads the `Turbo-Frame`
    header and answers with one.
+
+   **And a frame is not the page it came from.** Turbo sends that header on every frame
+   navigation, and an app answering it with `turbo_frame_request?` returns just the frame.
+   Keyed on the URL alone, that body lands in the slot the page occupies: a later cold visit
+   is then served a fragment as a whole document, which is a blank screen, and in Hotwire
+   Native a page where `window.Turbo` never appears. Whichever was cached last wins, so it
+   also happens in reverse. `Vary` cannot fix this, because matching is URL-only by design
+   (see 1), so Coldwire puts the frame in the key instead. A frame takes its own entry first
+   and the page second, since Turbo pulls a frame out of a document exactly as it does
+   online. An ordinary visit never takes the reverse trade.
+
+   **And neither is any other format.** The same URL answers a `respond_to` block in as many
+   formats as the app defines: `/report` is a page to Turbo, JSON to a `fetch`, a CSV to an
+   export link and an RSS feed to a reader. Coldwire names the format in the key too, with the
+   page left unnamed so anything cached before this keeps the key it had. The name is worked
+   out from the request's `Accept`, not the response's type, because the same name has to be
+   produced again when the entry is looked for, and there is no response to read then. A
+   request asking for data gets data or nothing, since handing it a page is the mistake in 6
+   wearing a different hat. A path that names its own format is left alone: `/report.json` is
+   JSON and nothing else, so only `/report` needs telling apart. A Turbo Stream is never stored at all: it is a list of changes to
+   make to a page, and replaying a stale one applies yesterday's mutations to today's DOM.
 4. **A followed redirect poisons the cache.** A signed-out request to `/` gets a `302` that
    `fetch` follows; the result looks fine and `cache.put()` stores it without complaint.
    Now `/` holds the sign-in page and keeps `redirected: true` — and serving a redirected

data/lib/coldwire/precache.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require "active_support/core_ext/object/blank"
+require "active_support/core_ext/array/wrap"
+
+module Coldwire
+  class << self
+    # The precache manifest, as the worker needs it. A bare URL is a page; a Hash can also name
+    # the Turbo frame it is loaded into, or the format it is fetched as, which are the two
+    # things a URL alone cannot say:
+    #
+    #   sync.precache_urls = -> {
+    #     Feature.published.map { |f| { url: feature_path(f), frame: "map_feature_popup" } } +
+    #     Report.all.map { |r| { url: report_path(r), format: :json } }
+    #   }
+    #
+    # Listing one URL twice is how you ask for both the page and the frame, so nothing here
+    # collapses them.
+    def precache_entries(list)
+      Array.wrap(list).map { |entry| precache_entry(entry) }
+    end
+
+    private
+
+    def precache_entry(entry)
+      return { url: entry.to_s } unless entry.is_a?(Hash)
+
+      entry = entry.transform_keys(&:to_sym)
+      url = entry[:url].to_s
+      raise ArgumentError, "Coldwire precache_urls entries need a url: #{entry.inspect}" if url.empty?
+
+      { url: url, frame: entry[:frame].presence&.to_s, accept: precache_accept(entry) }.compact
+    end
+
+    # A format becomes the Accept header the worker asks with, resolved here rather than in the
+    # worker because Rails already knows what every registered format means — including any the
+    # app registered itself.
+    #
+    # `format: :json` is the friendly way to say it and `accept:` is the way out when a media
+    # type has no registered name. Naming a format that is not registered is a typo worth
+    # refusing: it would otherwise be fetched as HTML and cached as the page.
+    def precache_accept(entry)
+      return entry[:accept].to_s if entry[:accept].present?
+      return nil if entry[:format].blank?
+
+      mime = Mime[entry[:format]]
+      unless mime
+        raise ArgumentError,
+              "Coldwire precache_urls format #{entry[:format].inspect} is not a registered Mime " \
+              "type. Register it with `Mime::Type.register`, or give `accept:` instead."
+      end
+
+      mime.to_s
+    end
+  end
+end

data/lib/coldwire/worker/collect.js

@@ -210,5 +210,10 @@ async function renew(cache, key) {
   if (!response) return
 
   const managed = key.headers.get(MANAGED_HEADER) === "1"
-  await putFresh(cache, cacheKey(new Request(key.url, { method: "GET" }), { managed }), response)
+  // Put it back where it was. Rebuilt from the URL alone, a frame entry would lose the frame
+  // it names and land on its page's key, which is the collision this all exists to avoid.
+  const params = new URL(key.url).searchParams
+  const frame = params.get(FRAME_PARAM)
+  const format = params.get(FORMAT_PARAM)
+  await putFresh(cache, cacheKey(new Request(key.url, { method: "GET" }), { managed, frame, format }), response)
 }

data/lib/coldwire/worker/inspect.js

@@ -67,18 +67,111 @@ async function entrySize(response) {
 // would find the entry either way, but every distinct query string would still write its own
 // copy — a map that rewrites lat/lng/zoom on each pan would bury the cache in near-duplicates
 // of one page.
-function cacheKey(request, { managed = false } = {}) {
+// `frame` names the Turbo frame this response is a body for, and is what keeps a frame out of
+// the slot its page occupies. Defaults to whatever the request asked for; passed explicitly
+// where the caller knows better — a frame request answered with a whole document is a page,
+// and renewing an entry has to put it back where it already was.
+function cacheKey(request, { managed = false, frame = request.headers.get("Turbo-Frame"), format = formatOf(request) } = {}) {
   const headers = new Headers(request.headers)
   headers.set(TIMESTAMP_HEADER, String(Math.floor(Date.now() / 1000)))
   if (managed) headers.set(MANAGED_HEADER, "1")
 
   // `new Request(request, init)` downgrades a navigation request's mode for us; rebuilding
-  // from a URL string needs the method stated explicitly.
-  if (!IGNORE_SEARCH) return new Request(request, { headers })
+  // from a URL string needs the method stated explicitly. Only available while the URL is
+  // being kept as it is, which naming a variant is not: taking this path with a format to
+  // write stored a JSON body under the page's own key, which is the whole bug in miniature.
+  const named = Boolean(frame) || (format && format !== "page")
+  if (!IGNORE_SEARCH && !named) return new Request(request, { headers })
 
-  const url = new URL(request.url)
-  url.search = ""
-  return new Request(url.href, { method: "GET", headers })
+  return new Request(variantUrl(request.url, { frame, format }), { method: "GET", headers })
+}
+
+// Where a body of this kind lives. Built in one place because it is written by cacheKey and
+// looked for by matchStored, and a frame stored under a URL nothing asks for is a frame that
+// never answers.
+function variantUrl(url, { frame = null, format = null } = {}) {
+  const target = new URL(url)
+  if (IGNORE_SEARCH) target.search = ""
+  if (frame) target.searchParams.set(FRAME_PARAM, frame)
+  // A page is the default and says nothing, which is what leaves every entry stored before
+  // formats existed exactly where it was.
+  if (format && format !== "page") target.searchParams.set(FORMAT_PARAM, format)
+
+  return target.href
+}
+
+// The format a request negotiated for, as a short token.
+//
+// Read from the request rather than from the response, which is the opposite of what it should
+// be and is forced: the same derivation has to run when the entry is looked for again, and at
+// that moment there is no response to read. With ignore_query_params off a named key can only
+// be found by building its name, so a name the request cannot produce is a name nothing ever
+// finds.
+//
+// A page is the default and is left unnamed, which is what leaves every entry cached before
+// formats existed exactly where it was.
+function formatOf(request) {
+  const token = negotiatedFormat(request)
+  if (token === "page") return "page"
+
+  // A URL that already names its format has nothing to disambiguate. "/report.json" is JSON and
+  // nothing else, where "/report" is a page, a JSON body and a CSV depending on who asks, so the
+  // param there would be a second way of saying what the path already said.
+  //
+  // Only where the extension agrees with what was asked for. "/sites/acme.com" is a page whose
+  // last segment happens to contain a dot, and ".com" says nothing about a format — so that one
+  // keeps its param and stays apart from the JSON at the same URL.
+  return extensionFormat(request.url) === token ? "page" : token
+}
+
+// What the path itself declares, if anything. Unknown extensions say nothing and keep their
+// param, which is noisier than it needs to be and never wrong.
+const EXTENSION_FORMATS = {
+  json: "json", geojson: "json", css: "css", xml: "xml", rss: "xml", atom: "xml",
+  csv: "csv", ics: "calendar", pdf: "pdf", txt: "plain", md: "markdown",
+  png: "image", jpg: "image", jpeg: "image", gif: "image", webp: "image", avif: "image",
+  svg: "image", ico: "image"
+}
+
+function extensionFormat(url) {
+  const name = new URL(url).pathname.split("/").pop() || ""
+  const dot = name.lastIndexOf(".")
+  if (dot < 1) return "page"
+
+  return EXTENSION_FORMATS[name.slice(dot + 1).toLowerCase()] || "page"
+}
+
+function negotiatedFormat(request) {
+  const accept = (request.headers.get("Accept") || "").split(",")[0].split(";")[0].trim().toLowerCase()
+
+  // Nothing definite asked for. A precache, a fetch that set no Accept, a browser asking for a
+  // script or a font: all of them get the unnamed key, as they always have.
+  if (!accept || accept === "*/*" || accept.includes("html")) return "page"
+
+  const [ top, sub = "" ] = accept.split("/")
+  // image/avif and image/webp are one question asked two ways, and which one a browser puts
+  // first is not a distinction worth a second copy. The family is the answer.
+  if (top && top !== "text" && top !== "application") return top
+
+  // "application/vnd.api+json" is JSON. "text/csv" is csv.
+  const parts = sub.split("+")
+  const token = (parts.length > 1 ? parts[parts.length - 1] : parts[0]) || ""
+
+  return token.replace(/[^a-z0-9.-]/g, "") || "page"
+}
+
+// What kind of body an entry holds, read back off its key. Documents have none of these
+// params and answer to null, which is what an ordinary visit asks for.
+function variantOf(key) {
+  const params = new URL(key.url).searchParams
+  // Downloads are their own thing entirely and must never answer a page request, which
+  // ignoreSearch would otherwise let them do.
+  if (params.has(CHUNK_PARAM) || params.has(RANGE_PARAM)) return "download"
+
+  const frame = params.get(FRAME_PARAM)
+  if (frame) return `frame:${frame}`
+
+  return params.get(FORMAT_PARAM) || "page"
 }
 
 function unixTimestamp(value) {

data/lib/coldwire/worker/rules.js

@@ -96,5 +96,9 @@ function isAutoCacheable(request) {
 // network error by spec, so the app fails to launch offline rather than showing the
 // cached page. Never store one.
 function isCacheable(request, response) {
-  return request.method === "GET" && response.ok && !response.redirected
+  if (request.method !== "GET" || !response.ok || response.redirected) return false
+
+  // A stream is a list of changes to make to a page, not a page. Stored, it would sit in the
+  // slot the page occupies and be replayed later against a DOM it was never written for.
+  return !(response.headers.get("Content-Type") || "").includes(STREAM_TYPE)
 }

data/lib/coldwire/worker/serve.js

@@ -9,8 +9,14 @@ async function putFresh(cache, key, response) {
   await cache.put(key, response)
 }
 
-async function fetchAndCache(cache, href, { managed = false } = {}) {
-  const request = new Request(href, { credentials: "same-origin" })
+async function fetchAndCache(cache, href, { managed = false, frame = null, accept = null } = {}) {
+  const headers = {}
+  // A manifest listing that names a frame or a format is asking for the body a request like
+  // that would get, so the request has to look like one.
+  if (frame) headers["Turbo-Frame"] = frame
+  if (accept) headers.Accept = accept
+
+  const request = new Request(href, { credentials: "same-origin", headers })
   // Skipped rather than failed: the app asked for this URL never to be stored, and a manifest
   // that also names it is a contradiction to resolve quietly in favour of not storing.
   if (isNeverCached(new URL(request.url))) return []
@@ -20,7 +26,10 @@ async function fetchAndCache(cache, href, { managed = false } = {}) {
     throw new Error(response.redirected ? `Redirected to ${response.url}` : `HTTP ${response.status}`)
   }
 
-  await putFresh(cache, cacheKey(request, { managed }), response.clone())
+  const key = managed
+    ? cacheKey(request, { managed, frame })
+    : await bodyKey(cache, request, response)
+  await putFresh(cache, key, response.clone())
 
   const contentType = response.headers.get("Content-Type") || ""
   if (!contentType.includes("text/html")) return []
@@ -94,21 +103,124 @@ async function handleFetch(request, event) {
 }
 
 async function offlineFallback(cache, request) {
-  const cached = await cache.match(request, MATCH_OPTIONS)
+  const key = await matchStored(cache, request)
+  const cached = key ? await cache.match(key) : undefined
 
-  return (await cachedPageResponse(cache, request, cached)) || offlineResponse(request)
+  return (await cachedPageResponse(request, key, cached)) || offlineResponse(request)
+}
+
+// Which stored entry answers this request. One URL can hold the page and a frame of the same
+// name, so this asks for the kind of body the request wants rather than taking whatever the
+// URL turns up — and with ignoreSearch on, what it turns up might be a frame, a byte range, or
+// a chunk of an archive.
+//
+// A frame takes its own entry first and a whole page second: an app that answers frame
+// requests with the full document stores one of those, and Turbo pulls the frame out of it
+// exactly as it does online. The trade never runs the other way. A fragment served to an
+// ordinary visit is a document with no <html>, which is a blank screen, and in Hotwire Native
+// a page where window.Turbo never appears.
+async function matchStored(cache, request) {
+  const frame = request.headers.get("Turbo-Frame")
+
+  if (frame) {
+    // Asked for by name rather than found by scanning. With ignore_query_params off the search
+    // has to match exactly, and a request carrying no query would never turn up the key that
+    // carries the frame in its own.
+    const [ own ] = await cache.keys(variantUrl(request.url, { frame }), { ignoreVary: true })
+    if (own) return own
+  }
+
+  // Asked for by name, for the same reason the frame is.
+  const format = formatOf(request)
+  if (format !== "page") {
+    const [ named ] = await cache.keys(variantUrl(request.url, { format }), { ignoreVary: true })
+    if (named) return named
+  }
+
+  const keys = await cache.keys(request, MATCH_OPTIONS)
+  const page = keys.find((candidate) => variantOf(candidate) === "page")
+  if (!page) return undefined
+
+  // The unnamed entry is the right answer for anything cached before formats were named, and
+  // for every asset a precache stored without an Accept to go on. It is the wrong answer for a
+  // fetch that asked for data and would be handed a page, which is the mistake that gives a
+  // stylesheet an HTML body. So ask what it holds before handing it over.
+  if (format === "page") return page
+
+  const held = await cache.match(page)
+
+  return (held && !(held.headers.get("Content-Type") || "").includes("text/html")) ? page : undefined
+}
+
+// Where a body that is not a page goes: named by the format the request asked for, unless this
+// URL already holds something that is not a page.
+//
+// That exception is the whole reason this is a function. A precache carries no Accept and lands
+// on the unnamed key; the browser then asks for the same stylesheet as `text/css` and would
+// write a second copy beside it. Two of every stylesheet and every image is not a rounding
+// error on a phone.
+//
+// The lookup is only paid where a name would be written at all, so `*/*` requests — scripts,
+// fonts, and precaching itself — cost nothing extra.
+async function bodyKey(cache, request, response) {
+  if (formatOf(request) !== "page") {
+    const [ plain ] = await cache.keys(variantUrl(request.url, {}), { ignoreVary: true })
+    const held = plain ? await cache.match(plain) : null
+    if (held && !(held.headers.get("Content-Type") || "").includes("text/html")) {
+      return cacheKey(request, { frame: null, format: "page" })
+    }
+
+    return cacheKey(request, { frame: null })
+  }
+
+  // A URL that names its own format is the same key from every direction, so there is nothing
+  // another request could have named differently and nothing to look for.
+  if (extensionFormat(request.url) !== "page") return cacheKey(request, { frame: null })
+
+  // And the other way round: this request named nothing, but the browser may already have
+  // stored the same file under the format it asked for. Work that name out from what came
+  // back, rather than writing a second copy beside it.
+  const guess = formatFromType(response)
+  if (guess !== "page") {
+    const [ named ] = await cache.keys(variantUrl(request.url, { format: guess }), { ignoreVary: true })
+    if (named) return cacheKey(request, { frame: null, format: guess })
+  }
+
+  return cacheKey(request, { frame: null })
+}
+
+// What a browser's Accept would have produced for a body like this. Only the two families a
+// browser asks for by type: a script or a font arrives on `*/*` and is unnamed on both sides,
+// so neither pays for this lookup.
+function formatFromType(response) {
+  const type = (response.headers.get("Content-Type") || "").split(";")[0].trim().toLowerCase()
+  if (type.startsWith("text/css")) return "css"
+  if (type.startsWith("image/")) return "image"
+
+  return "page"
 }
 
 // A page is stored with whatever it asks for. The lists say which pages are worth keeping as
 // you browse; what one needs in order to render is not a second question, and a page held
 // without its stylesheet is the offline equivalent of not holding it at all.
 async function storeResponse(cache, request, response) {
-  await putFresh(cache, cacheKey(request), response.clone())
-
   const type = response.headers.get("Content-Type") || ""
-  if (!type.includes("text/html")) return
+  if (!type.includes("text/html")) {
+    await putFresh(cache, await bodyKey(cache, request, response), response.clone())
+    return
+  }
+
+  // Read before writing, because what this is decides where it goes. Cloning twice is free:
+  // each clone tees the stream, so consuming one leaves the other whole.
+  const body = await response.clone().text()
+  // A frame request the app answered with the entire document is a page like any other, and
+  // Turbo will find the frame inside it. Storing that under a frame key would hold the same
+  // bytes twice and leave the next ordinary visit unable to find them.
+  const frame = /<html\b/i.test(body) ? null : request.headers.get("Turbo-Frame")
 
-  const urls = urlsFromHtml(await response.text(), request.url)
+  await putFresh(cache, cacheKey(request, { frame }), response.clone())
+
+  const urls = urlsFromHtml(body, request.url)
   await Promise.all(urls.map((href) => storeSubresource(cache, href)))
 }
 
@@ -118,7 +230,14 @@ async function storeResponse(cache, request, response) {
 async function storeSubresource(cache, href) {
   if (isNeverCached(new URL(href))) return
 
-  const [ key ] = await cache.keys(href, MATCH_OPTIONS)
+  // Any body for this URL that is not a frame of it and not a piece of a download. Asking only
+  // for the unnamed one would refetch a stylesheet the browser has already stored under its
+  // own format, and land a second copy of it here.
+  const key = (await cache.keys(href, MATCH_OPTIONS)).find((held) => {
+    const variant = variantOf(held)
+
+    return variant !== "download" && !variant.startsWith("frame:")
+  })
   if (key) return renew(cache, key)
 
   try {
@@ -145,7 +264,7 @@ function untrack(html) {
 // Two markers, read at different moments. The <html> attributes are for the first paint of a
 // cold boot, before any JS runs. The <meta> is for Turbo visits, which merge the head but
 // never copy <html> attributes.
-async function cachedPageResponse(cache, request, cached) {
+async function cachedPageResponse(request, key, cached) {
   if (!cached || !wantsHtml(request)) return cached
 
   const type = cached.headers.get("Content-Type") || ""
@@ -157,8 +276,8 @@ async function cachedPageResponse(cache, request, cached) {
   let body = untrack(html)
 
   if (MARK_CACHED_PAGES) {
-    // The timestamp rides on the stored key, so ask the cache for the key that matched.
-    const [ key ] = await cache.keys(request, MATCH_OPTIONS)
+    // The timestamp rides on the stored key, which the caller has already had to find: asking
+    // the cache again could pick a different entry for the same URL than the one being served.
     const cachedAt = key ? key.headers.get(TIMESTAMP_HEADER) : null
     const stamp = cachedAt ? ` ${CACHED_AT_ATTRIBUTE}="${escapeHtml(cachedAt)}"` : ""
 

data/lib/coldwire/worker/sync.js

@@ -48,7 +48,8 @@ async function runSync() {
     return failure
   }
 
-  const wanted = new Set(manifest.map((value) => cacheUrl(new URL(value, self.location.origin).href)))
+  const entries = manifest.map(manifestEntry)
+  const wanted = new Set(entries.map((entry) => entry.key))
 
   // Retire what the manifest dropped — an unpublished site, say. Only entries the manifest
   // owns, so assets and ordinary browsing are left alone.
@@ -58,11 +59,13 @@ async function runSync() {
   // already good and costs nothing.
   const now = Date.now() / 1000
   const pending = []
-  for (const href of wanted) {
-    const [ key ] = await cache.keys(new Request(href), MATCH_OPTIONS)
+  for (const entry of entries) {
+    // Asked for by the exact key this entry will be stored under. A page and a frame of it are
+    // two listings now, and either one being held says nothing about the other.
+    const [ key ] = await cache.keys(entry.key, { ignoreVary: true })
     const at = key ? unixTimestamp(key.headers.get(TIMESTAMP_HEADER)) : null
 
-    if (!key || !at || (REFETCH_AFTER !== null && now - at > REFETCH_AFTER)) pending.push(href)
+    if (!key || !at || (REFETCH_AFTER !== null && now - at > REFETCH_AFTER)) pending.push(entry)
   }
 
   // Every open page hears about this, not just the one that asked. A sync outlives the page
@@ -137,7 +140,10 @@ async function retireUnlisted(cache, wanted) {
 
   for (const key of keys) {
     if (key.headers.get(MANAGED_HEADER) !== "1") continue
-    if (wanted.has(cacheUrl(key.url))) continue
+    // Compared whole, params and all. Stripping the search here would take the frame or the
+    // format off a managed entry, leave it looking like a page nobody listed, and retire it on
+    // the first sync after it was fetched.
+    if (wanted.has(key.url)) continue
 
     await cache.delete(key)
     retired++
@@ -146,13 +152,26 @@ async function retireUnlisted(cache, wanted) {
   return retired
 }
 
-// Compare URLs the same way they are stored, or every entry looks unlisted the moment a
-// query string is involved.
-function cacheUrl(href) {
-  const url = new URL(href)
-  if (IGNORE_SEARCH) url.search = ""
+// One manifest listing, worked out once: the request that fetches it, and the key it will be
+// stored under. A bare URL is a page, as it always was; a listing that names a frame or an
+// Accept is asking for the body a request like that would get, and has to be stored where a
+// request like that will look for it.
+function manifestEntry(value) {
+  const listing = typeof value === "string" ? { url: value } : (value || {})
+  const href = new URL(listing.url, self.location.origin).href
+  const headers = {}
+
+  if (listing.frame) headers["Turbo-Frame"] = listing.frame
+  // What Turbo sends alongside the frame header. Without it the manifest asks with `*/*`, and
+  // a `respond_to` block answers with whatever it happens to list first.
+  if (listing.accept || listing.frame) {
+    headers.Accept = listing.accept || "text/html, application/xhtml+xml"
+  }
+
+  const request = new Request(href, { credentials: "same-origin", headers })
+  const key = variantUrl(href, { frame: listing.frame, format: formatOf(request) })
 
-  return url.href
+  return { href, frame: listing.frame, accept: headers.Accept, key, request }
 }
 
 // A fixed number of lanes pulling from one queue.
@@ -201,15 +220,16 @@ async function precacheUrls(inputUrls, onProgress = () => {}) {
   let done = 0
   onProgress({ phase: "pages", done, total: inputUrls.length })
 
-  await runPool(inputUrls, SYNC_CONCURRENCY, async (value) => {
+  await runPool(inputUrls, SYNC_CONCURRENCY, async (entry) => {
     try {
-      const href = new URL(value, self.location.origin).href
-      const extra = await fetchWithRetry(cache, href, { managed: true })
-      cached.add(href)
+      const extra = await fetchWithRetry(cache, entry.href, {
+        managed: true, frame: entry.frame, accept: entry.accept
+      })
+      cached.add(entry.href)
       extra.forEach((url) => assets.add(url))
     } catch {
-      failed.push(String(value))
-      pageFailures.push(String(value))
+      failed.push(entry.href)
+      pageFailures.push(entry.href)
     }
     onProgress({ phase: "pages", done: ++done, total: inputUrls.length })
   })

data/lib/coldwire.rb

@@ -2,6 +2,7 @@
 
 require "coldwire/version"
 require "coldwire/configuration"
+require "coldwire/precache"
 require "coldwire/source"
 require "coldwire/client_user_agent"
 require "coldwire/engine"

data/lib/generators/coldwire/install/templates/coldwire.rb

@@ -8,6 +8,8 @@ Coldwire.configure do |config|
     sync.enabled = false          # off by default: background fetching is somebody's data plan
     sync.precache_urls = -> { [] }
     # sync.precache_urls = -> { Article.published.map { |a| article_path(a) } }
+    # A Hash names a Turbo Frame or a format, for URLs that answer more than one way:
+    # sync.precache_urls = -> { Article.published.map { |a| { url: article_path(a), frame: "preview" } } }
     sync.interval = 1.day
     sync.max_age = 30.days
     sync.concurrency = 4

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: coldwire-rails
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Noreaster Group, Stuart Yamartino
@@ -80,6 +80,7 @@ files:
 - lib/coldwire/configuration.rb
 - lib/coldwire/debug.css
 - lib/coldwire/engine.rb
+- lib/coldwire/precache.rb
 - lib/coldwire/source.rb
 - lib/coldwire/version.rb
 - lib/coldwire/worker/archives.js