coldwire-rails 0.1.0

data/lib/coldwire/worker/archives.js

@@ -0,0 +1,158 @@
+// The facts about a chunk go on the key, not only on the response. cache.keys() hands back
+// Requests, so anything read while counting up what is stored has to live there — the same
+// reason the timestamp does. Matching ignores headers, so a lookup built without them still
+// finds the entry.
+function chunkKey(url, index, facts = null) {
+  const target = new URL(url)
+  target.searchParams.set(CHUNK_PARAM, String(index))
+  if (!facts) return new Request(target.href, { method: "GET" })
+
+  const headers = new Headers()
+  headers.set("coldwire-archive-total", String(facts.total))
+  headers.set("coldwire-chunk-size", String(facts.size))
+  headers.set(TIMESTAMP_HEADER, String(Math.floor(Date.now() / 1000)))
+
+  return new Request(target.href, { method: "GET", headers })
+}
+
+function chunkCount(total) {
+  return Math.ceil(total / ARCHIVE_CHUNK)
+}
+
+// How big the thing is, asked for in the cheapest way there is: one byte, and read the total
+// off the Content-Range that comes back.
+async function archiveTotal(url) {
+  const response = await fetch(url, { headers: { Range: "bytes=0-0" } })
+  if (response.status !== 206) throw new Error(`HTTP ${response.status}`)
+
+  const total = rangeTotal(response.headers.get("Content-Range"))
+  if (!total) throw new Error("No Content-Range")
+
+  return total
+}
+
+// Counted from what is actually stored rather than from a note written when the download ran.
+// A cache can be evicted piecemeal, and a status that disagrees with the cache is worse than
+// no status at all.
+async function archiveStatus(url) {
+  if (!CACHE_ARCHIVES.includes(url)) return { ok: false, error: "Not a listed archive" }
+
+  const cache = await caches.open(CACHE_NAME)
+  const stored = await cache.keys()
+  const prefix = new URL(url)
+  prefix.search = ""
+
+  let bytes = 0
+  let chunks = 0
+  let total = null
+  // The newest chunk: when this last got any of itself, which for a finished download is
+  // when it finished. Chunks already held are skipped on a later pass, so an older one
+  // would date the archive from an attempt that may have stopped in the first megabyte.
+  let cachedAt = null
+
+  for (const key of stored) {
+    const keyUrl = new URL(key.url)
+    keyUrl.search = ""
+    if (keyUrl.href !== prefix.href) continue
+    if (!new URL(key.url).searchParams.has(CHUNK_PARAM)) continue
+
+    chunks += 1
+    const size = Number(key.headers.get("coldwire-chunk-size"))
+    if (Number.isFinite(size)) bytes += size
+    const declared = Number(key.headers.get("coldwire-archive-total"))
+    if (Number.isFinite(declared) && declared > 0) total = declared
+    const stamp = unixTimestamp(key.headers.get(TIMESTAMP_HEADER))
+    if (stamp && (!cachedAt || stamp > cachedAt)) cachedAt = stamp
+  }
+
+  return {
+    ok: true,
+    url,
+    chunks,
+    bytes,
+    total,
+    cachedAt,
+    expected: total ? chunkCount(total) : null,
+    complete: Boolean(total) && chunks === chunkCount(total)
+  }
+}
+
+// Resumable by construction: every chunk already stored is skipped, so an interrupted download
+// picks up where it stopped rather than starting again. Sequential on purpose — this is a
+// large download over somebody's connection, and running it in parallel lanes would take the
+// bandwidth the app itself is using.
+async function downloadArchive(url) {
+  if (!CACHE_ARCHIVES.includes(url)) return { ok: false, error: "Not a listed archive" }
+  if (forcedOffline) return { ok: false, offline: true, reason: "forced" }
+
+  const cache = await caches.open(CACHE_NAME)
+
+  let total
+  try {
+    total = await archiveTotal(url)
+  } catch (error) {
+    return { ok: false, offline: true, error: error.message }
+  }
+
+  const count = chunkCount(total)
+  let stored = 0
+
+  for (let index = 0; index < count; index++) {
+    const key = chunkKey(url, index)
+
+    if (await cache.match(key)) {
+      stored += 1
+      notifyClients({ type: ARCHIVE_MESSAGE, url, state: "progress", done: stored, total: count })
+      continue
+    }
+
+    const start = index * ARCHIVE_CHUNK
+    const end = Math.min(start + ARCHIVE_CHUNK, total) - 1
+
+    try {
+      const response = await fetch(url, { headers: { Range: `bytes=${start}-${end}` } })
+      if (response.status !== 206) throw new Error(`HTTP ${response.status}`)
+
+      const body = await response.arrayBuffer()
+      const headers = new Headers()
+      headers.set("Content-Type", response.headers.get("Content-Type") || "application/octet-stream")
+      // On the response as well, because serving a range reads the total from here.
+      headers.set("coldwire-archive-total", String(total))
+
+      await cache.put(chunkKey(url, index, { total, size: body.byteLength }),
+                      new Response(body, { status: 200, headers }))
+    } catch (error) {
+      // Everything already stored stays stored, so asking again resumes from here.
+      const failure = { ok: false, url, error: error.message, done: stored, total: count }
+      await notifyClients({ type: ARCHIVE_MESSAGE, state: "finished", ...failure })
+
+      return failure
+    }
+
+    stored += 1
+    notifyClients({ type: ARCHIVE_MESSAGE, url, state: "progress", done: stored, total: count })
+  }
+
+  const finished = { ok: true, url, done: stored, total: count, bytes: total, complete: true }
+  await notifyClients({ type: ARCHIVE_MESSAGE, state: "finished", ...finished })
+
+  return finished
+}
+
+async function removeArchive(url) {
+  const cache = await caches.open(CACHE_NAME)
+  const prefix = new URL(url)
+  prefix.search = ""
+
+  let removed = 0
+  for (const key of await cache.keys()) {
+    const keyUrl = new URL(key.url)
+    const bare = new URL(key.url)
+    bare.search = ""
+    if (bare.href !== prefix.href) continue
+    if (!keyUrl.searchParams.has(CHUNK_PARAM)) continue
+    if (await cache.delete(key)) removed += 1
+  }
+
+  return { ok: true, url, removed }
+}

data/lib/coldwire/worker/events.js

@@ -0,0 +1,79 @@
+self.addEventListener("install", (event) => {
+  event.waitUntil(Promise.all([ self.skipWaiting(), cacheOfflinePage() ]))
+})
+
+// The offline page is the one page that has to render when nothing else can, and Hotwire
+// Native rejects any page where window.Turbo never appears — so it carries an importmap and
+// imports Turbo. Those assets are digested, which means a deploy changes their URLs and the
+// cache filled by the previous deploy does not have the new ones.
+//
+// Left to a sync, the fallback is broken for exactly as long as nobody has been online since
+// the deploy — which is to say, broken precisely when it is needed. Fetching them as the
+// worker installs is the moment the new URLs first become known.
+async function cacheOfflinePage() {
+  try {
+    const cache = await caches.open(CACHE_NAME)
+    const urls = urlsFromHtml(OFFLINE_PAGE, self.location.origin + "/")
+
+    await runPool(urls, SYNC_CONCURRENCY, async (href) => {
+      // One at a time and forgiving: a worker that refuses to install because a stylesheet
+      // was briefly unavailable is worse than one whose fallback is missing a stylesheet.
+      try {
+        await fetchAndCache(cache, href)
+      } catch {}
+    })
+  } catch {
+    // Installing must not fail over this. A worker with an imperfect fallback still caches
+    // pages, still syncs, and still beats no worker at all.
+  }
+}
+
+self.addEventListener("activate", (event) => {
+  event.waitUntil(self.clients.claim())
+})
+
+self.addEventListener("fetch", (event) => {
+  if (!shouldHandle(event.request)) return
+
+  if (event.request.headers.has("Range")) {
+    event.respondWith(handleRange(event.request))
+    return
+  }
+
+  event.respondWith(handleFetch(event.request, event))
+})
+
+self.addEventListener("message", (event) => {
+  const type = event.data?.type
+  const reply = (promise) => {
+    event.waitUntil(
+      Promise.resolve(promise)
+        .then((result) => event.ports[0]?.postMessage(result))
+        .catch((error) => event.ports[0]?.postMessage({ ok: false, error: error.message }))
+    )
+  }
+
+  if (type === "clearCache") return reply(clearCaches())
+  if (type === "forget") return reply(forgetUrl(event.data.url, event.data.cache))
+  if (type === "listCache") return reply(listCaches())
+  if (type === "getForcedOffline") return reply({ ok: true, forcedOffline })
+  if (type === "setForcedOffline") {
+    forcedOffline = Boolean(event.data.value)
+    return reply({ ok: true, forcedOffline })
+  }
+  if (type === "setCachingEnabled") {
+    cachingEnabled = Boolean(event.data.value)
+    return reply({ ok: true, cachingEnabled })
+  }
+  // One way to fill the cache, whether a page load asked for it on a timer or somebody
+  // pressed the button. Progress goes out on the broadcast, not this port, so every open
+  // page sees the run and not just whoever started it.
+  if (type === "sync") return reply(syncManifest())
+
+  // What a page missed by not listening yet, and whether it is still going on.
+  if (type === "syncState") return reply({ ok: true, running: Boolean(syncing), last: lastSyncMessage })
+
+  if (type === "archiveStatus") return reply(archiveStatus(event.data.url))
+  if (type === "archiveDownload") return reply(downloadArchive(event.data.url))
+  if (type === "archiveRemove") return reply(removeArchive(event.data.url))
+})

data/lib/coldwire/worker/inspect.js

@@ -0,0 +1,81 @@
+// Drop one entry. Matched with the same options everything else uses, or a URL that was
+// stored with a query string would refuse to be found by the one shown in the list.
+async function forgetUrl(url, name) {
+  if (!url) return { ok: false, error: "No URL given" }
+
+  const cache = await caches.open(name || CACHE_NAME)
+  const deleted = await cache.delete(new Request(url), MATCH_OPTIONS)
+
+  return { ok: deleted, deleted }
+}
+
+async function clearCaches() {
+  const names = await caches.keys()
+  await Promise.all(names.map((name) => caches.delete(name)))
+  return { ok: true, cleared: names.length }
+}
+
+async function listCaches() {
+  const names = await caches.keys()
+  const result = []
+
+  for (const name of names) {
+    const cache = await caches.open(name)
+    const requests = await cache.keys()
+    const entries = []
+    for (const request of requests) {
+      entries.push(await describeCached(request, await cache.match(request, MATCH_OPTIONS)))
+    }
+    result.push({ name, entries })
+  }
+
+  return { ok: true, caches: result }
+}
+
+async function describeCached(request, response) {
+  return {
+    url: request.url,
+    size: await entrySize(response),
+    timestamp: unixTimestamp(request.headers.get(TIMESTAMP_HEADER))
+  }
+}
+
+// Ask the headers before reading the body. Listing a real cache means hundreds of entries and
+// tens of megabytes, and blob() on every one of them is a multi-second job for a number that
+// Content-Length already carries.
+async function entrySize(response) {
+  if (!response) return 0
+
+  const declared = Number(response.headers.get("Content-Length"))
+  if (Number.isFinite(declared) && declared >= 0) return declared
+
+  return (await response.clone().blob()).size
+}
+
+// The key an entry is stored under.
+//
+// Cache does not honor HTTP freshness headers, and WebKit drops Date on match(). Stamp unix
+// seconds on the request key — keys() returns it, and URL matching still finds the entry.
+//
+// When ignoring query params, drop the search here too, not just in MATCH_OPTIONS. Matching
+// would find the entry either way, but every distinct query string would still write its own
+// copy — a map that rewrites lat/lng/zoom on each pan would bury the cache in near-duplicates
+// of one page.
+function cacheKey(request, { managed = false } = {}) {
+  const headers = new Headers(request.headers)
+  headers.set(TIMESTAMP_HEADER, String(Math.floor(Date.now() / 1000)))
+  if (managed) headers.set(MANAGED_HEADER, "1")
+
+  // `new Request(request, init)` downgrades a navigation request's mode for us; rebuilding
+  // from a URL string needs the method stated explicitly.
+  if (!IGNORE_SEARCH) return new Request(request, { headers })
+
+  const url = new URL(request.url)
+  url.search = ""
+  return new Request(url.href, { method: "GET", headers })
+}
+
+function unixTimestamp(value) {
+  const seconds = Number(value)
+  return Number.isFinite(seconds) && seconds > 0 ? seconds : null
+}

data/lib/coldwire/worker/ranges.js

@@ -0,0 +1,136 @@
+// Stitched out of whatever chunks are stored. Blob slices reference bytes rather than copying
+// them, so answering a tile request out of a 300 MB archive costs about as much as answering
+// it out of a single stored range.
+async function rangeFromChunks(cache, request, spec) {
+  const bare = new URL(request.url)
+  bare.search = ""
+  if (!CACHE_ARCHIVES.includes(bare.href)) return null
+
+  const first = Math.floor(spec.start / ARCHIVE_CHUNK)
+  const parts = []
+  let total = null
+  let index = first
+  let wanted = spec.end === null ? Infinity : spec.end - spec.start + 1
+  let taken = 0
+
+  while (taken < wanted) {
+    const stored = await cache.match(chunkKey(bare.href, index))
+    // A gap: fall back rather than answer with a hole in the middle.
+    if (!stored) return null
+
+    total = total || Number(stored.headers.get("coldwire-archive-total")) || null
+    const blob = await stored.blob()
+    const offset = index * ARCHIVE_CHUNK
+    const from = Math.max(spec.start - offset, 0)
+    const to = wanted === Infinity ? blob.size : Math.min(from + (wanted - taken), blob.size)
+
+    parts.push(blob.slice(from, to))
+    taken += to - from
+    index += 1
+
+    // Ran off the end of the archive, which is a legitimate end to an open-ended range.
+    if (total && offset + blob.size >= total) break
+    if (to < blob.size) break
+  }
+
+  if (!parts.length) return null
+
+  const body = new Blob(parts)
+  const headers = new Headers()
+  const type = "application/octet-stream"
+  headers.set("Content-Type", type)
+  headers.set("Content-Length", String(body.size))
+  headers.set("Content-Range", `bytes ${spec.start}-${spec.start + body.size - 1}/${total || "*"}`)
+  headers.set("Accept-Ranges", "bytes")
+
+  return new Response(body, { status: 206, statusText: "Partial Content", headers })
+}
+
+// Cache first, always. A byte range of an archive is immutable for as long as the archive is,
+// and the entire point of caching one is to stop asking for it.
+async function handleRange(request) {
+  const spec = parseRange(request.headers.get("Range"))
+  // A single closed range is what a tile archive asks for. Multipart, or anything unparseable,
+  // is not something to take apart and reassemble — send it and store nothing.
+  if (!spec) return fetch(request)
+
+  const cache = await caches.open(CACHE_NAME)
+
+  // A fully downloaded archive answers everything, so try it before anything else.
+  const fromChunks = await rangeFromChunks(cache, request, spec)
+  if (fromChunks) return fromChunks
+
+  const key = rangeKey(request, spec)
+  // Deliberately not MATCH_OPTIONS: `ignore_query_params` would collapse every range of a
+  // file onto one entry, since the range lives in the query.
+  const stored = await cache.match(key, { ignoreVary: true })
+
+  if (stored) return partialResponse(stored, spec)
+  if (forcedOffline) return rangeUnavailable()
+
+  try {
+    const response = await fetch(request)
+    if (response.status !== 206) return response
+
+    // Clone before reading: the body is needed twice, once to store and once to return.
+    const body = await response.clone().arrayBuffer()
+    const headers = new Headers()
+    const type = response.headers.get("Content-Type")
+    if (type) headers.set("Content-Type", type)
+    headers.set(RANGE_TOTAL_HEADER, String(rangeTotal(response.headers.get("Content-Range")) ?? ""))
+    headers.set(TIMESTAMP_HEADER, String(Math.floor(Date.now() / 1000)))
+
+    cache.put(key, new Response(body, { status: 200, headers }))
+
+    return response
+  } catch {
+    return rangeUnavailable()
+  }
+}
+
+// The stored body *is* the range that was asked for, so this is only about the headers a
+// partial response has to carry.
+async function partialResponse(stored, spec) {
+  const body = await stored.arrayBuffer()
+  const total = stored.headers.get(RANGE_TOTAL_HEADER) || "*"
+  const headers = new Headers()
+  const type = stored.headers.get("Content-Type")
+  if (type) headers.set("Content-Type", type)
+  headers.set("Content-Length", String(body.byteLength))
+  headers.set("Content-Range", `bytes ${spec.start}-${spec.start + body.byteLength - 1}/${total}`)
+  headers.set("Accept-Ranges", "bytes")
+
+  return new Response(body, { status: 206, statusText: "Partial Content", headers })
+}
+
+// 504 rather than the offline page: whatever asked for a byte range wants bytes, and handing
+// it HTML would be answered with a parse error instead of a failure it can report.
+function rangeUnavailable() {
+  return new Response(null, { status: 504, statusText: "Offline" })
+}
+
+function parseRange(value) {
+  const match = /^bytes=(\d+)-(\d*)$/.exec(String(value || "").trim())
+  if (!match) return null
+
+  const start = Number(match[1])
+  const end = match[2] === "" ? null : Number(match[2])
+  if (end !== null && end < start) return null
+
+  return { start, end }
+}
+
+function rangeTotal(contentRange) {
+  const match = /\/(\d+)\s*$/.exec(String(contentRange || ""))
+
+  return match ? Number(match[1]) : null
+}
+
+// The range goes in the query so it is part of the cache key, and the request is never sent
+// anywhere, so the extra parameter cannot confuse a server.
+function rangeKey(request, spec) {
+  const url = new URL(request.url)
+  url.searchParams.set(RANGE_PARAM, `${spec.start}-${spec.end ?? ""}`)
+
+  return new Request(url.href, { method: "GET" })
+}

data/lib/coldwire/worker/rules.js

@@ -0,0 +1,100 @@
+function shouldHandle(request) {
+  if (!cachingEnabled) return false
+  if (request.method !== "GET") return false
+
+  const url = new URL(request.url)
+  if (!cacheableOrigin(url)) return false
+
+  // A Range request cannot be stored as it arrives — cache.put refuses a 206 — so it is
+  // stored as a 200 under a key naming the range, and answered with a 206 built here. Only
+  // for URLs nominated for it: everything else streams straight to the network, which is what
+  // you want for media, and leaves ordinary requests exactly as they were.
+  if (request.headers.has("Range")) return matchesRules(url, CACHE_RANGES)
+
+  return !matchesPath(url, NEVER_INTERCEPT)
+}
+
+// Our own origin, plus any the host app has nominated. A worker sees every request a page
+// makes, and caching other people's responses uninvited is not its business.
+function cacheableOrigin(url) {
+  return url.origin === self.location.origin || CACHE_ORIGINS.includes(url.origin)
+}
+
+function matchesPath(url, paths) {
+  return paths.some((path) => url.pathname === path || url.pathname.startsWith(`${path}/`))
+}
+
+// Rules arrive as plain objects so a Regexp survives the trip through JSON.
+function compileRules(rules) {
+  return rules.map((rule) =>
+    rule.type === "regexp" ? new RegExp(rule.source, rule.flags) : segments(rule.value))
+}
+
+function segments(value) {
+  return value.split("/").filter(Boolean)
+}
+
+// Strings are route-shaped and match nothing beyond their own shape: "/sites" is /sites and
+// not /sites/1, ":id" is exactly one segment, and a trailing "*" takes the rest. A Regexp is
+// tested against the whole path.
+//
+// Deliberately strict. A prefix rule reads as "this section of the app", but it quietly takes
+// everything underneath — search results, new/edit forms, nested collections — and with
+// `ignore_query_params` a single "/sites/search" entry ends up answering every search.
+function matchesRules(url, rules) {
+  return rules.some((rule) =>
+    rule instanceof RegExp
+      ? rule.test(url.pathname)
+      : matchesPattern(segments(url.pathname), rule)
+  )
+}
+
+function matchesPattern(path, pattern) {
+  for (let index = 0; index < pattern.length; index++) {
+    const part = pattern[index]
+
+    // "*" is only ever the last segment — the Ruby side refuses it anywhere else. A lone
+    // "/*" is every path, including "/". Anywhere else it takes everything remaining, so
+    // there has to be something remaining: "/sites/*" is not "/sites".
+    if (part === "*") return (index === 0 && pattern.length === 1) || path.length > index
+    if (index >= path.length) return false
+    if (part.charAt(0) === ":") continue
+    if (part !== path[index]) return false
+  }
+
+  return path.length === pattern.length
+}
+
+// Whether *automatic* caching may store this. The precache manifest deliberately skips this
+// check: listing a URL there is an explicit instruction, and quietly declining it would make
+// the manifest unpredictable.
+// The one veto. Nothing stores a response for a URL named here — not browsing, not a page
+// that references it, not the precache manifest. Meaningful only for our own paths: another
+// origin's URLs are not ours to describe.
+function isNeverCached(url) {
+  return url.origin === self.location.origin && matchesRules(url, NEVER_CACHE)
+}
+
+// Whether *browsing* to this stores it. Says nothing about the subresources of what it stores:
+// a page comes with what it needs to render, which is not a second decision. The precache
+// manifest skips this check entirely — listing a URL there is an explicit instruction.
+function isAutoCacheable(request) {
+  const url = new URL(request.url)
+
+  // A nominated origin is the opt-in; the path lists describe this app's own surfaces and say
+  // nothing useful about somebody else's.
+  if (url.origin !== self.location.origin) return CACHE_ORIGINS.includes(url.origin)
+
+  if (isNeverCached(url)) return false
+
+  return matchesRules(url, CACHE_AS_YOU_GO)
+}
+
+// A followed redirect is the trap that breaks a signed-out cold launch. `cache.put()`
+// stores one happily — it does NOT reject — so "/" ends up holding the sign-in page body,
+// and the stored response keeps `redirected: true`. Serving that for a navigation is a
+// network error by spec, so the app fails to launch offline rather than showing the
+// cached page. Never store one.
+function isCacheable(request, response) {
+  return request.method === "GET" && response.ok && !response.redirected
+}