coldwire-rails 0.1.0 → 0.3.0

data/lib/coldwire/client/store.js

@@ -16,7 +16,14 @@
       // COLDWIRE.cachingEnabledByDefault, so a fresh device gets the app's default.
       caching: "coldwire-caching",
       // The URL list on the settings page. Closed unless they have opened it.
-      inspect: "coldwire-inspect"
+      inspect: "coldwire-inspect",
+      // When the cache was last swept. Only written when a sweep actually ran, so a run
+      // skipped for want of a connection leaves the next page still due.
+      collectedAt: "coldwire-collected-at",
+      // How much this device is willing to give the cache, in bytes, or "none" for no
+      // ceiling. Set only from the offline settings page; unset follows COLDWIRE.maxSize,
+      // which is what the app configured.
+      maxSize: "coldwire-max-size"
     },
 
     get: function (key) {
@@ -57,6 +64,21 @@
       this.set(key, value ? "1" : "0")
     },
 
+    // The ceiling the next sweep works to, in bytes, or null for none. Same shape as the
+    // switch above: a choice made on this device beats the app's default, and no choice yet
+    // means the default. A stored value that is not a positive number is a storage somebody
+    // has edited by hand, and "no ceiling" is not a safe reading of nonsense — fall back.
+    maxSize: function () {
+      var configured = window.COLDWIRE.maxSize
+      var stored = this.get(this.keys.maxSize)
+      if (stored === null) return configured === undefined ? null : configured
+      if (stored === "none") return null
+
+      var bytes = Number(stored)
+
+      return Number.isFinite(bytes) && bytes > 0 ? bytes : configured || null
+    },
+
     // The Offline support switch: an explicit choice beats the configured default, and no
     // choice yet means the default.
     cachingOn: function () {

data/lib/coldwire/configuration.rb

@@ -88,13 +88,19 @@ module Coldwire
     # cached pages hold whatever the previous session could see.
     attr_writer :cache_identity
 
-    # Origins besides your own that the worker may cache. Each has to send CORS headers naming
+    # The hosts besides your own that the worker may cache. Each has to send CORS headers naming
     # your app, or the response arrives opaque — status 0, no headers, no readable body — and
     # there is nothing worth storing. Ranged sources must also expose Content-Range.
-    attr_reader :cache_origins
+    #
+    #   config.cacheable_hosts = [ "tiles.example.com" ]
+    #
+    # A hostname, with a port only where it is not the default — which is what a URL's `host`
+    # reads as. No scheme: a worker runs only on a secure page, and a secure page cannot fetch
+    # http, so there was never a second scheme for one to tell apart.
+    attr_reader :cacheable_hosts
 
-    def cache_origins=(origins)
-      @cache_origins = Array(origins).map { |origin| validate_origin(origin) }
+    def cacheable_hosts=(hosts)
+      @cacheable_hosts = Array(hosts).map { |host| validate_host(host) }
     end
 
     # URLs whose Range requests are cached piece by piece, keyed by the range — for a large
@@ -142,6 +148,96 @@ module Coldwire
       @auto_sync
     end
 
+    # Clearing out what has gone unused, so a cache that fills as people browse does not fill
+    # forever. Only ever removes; never fetches.
+    #
+    #   config.garbage_collection do |gc|
+    #     gc.max_age = 60 * 60 * 24 * 60
+    #     gc.max_size = 500 * 1024 * 1024
+    #   end
+    def garbage_collection
+      @garbage_collection ||= GarbageCollection.new
+      yield(@garbage_collection) if block_given?
+
+      @garbage_collection
+    end
+
+    # Deleting is the one cache operation with no way back: whatever goes is gone until the
+    # network can be reached again. So a sweep happens only with a connection confirmed, and
+    # only for entries nothing has asked for in a long time — or, once the cache is over its
+    # ceiling, for whatever has gone longest unread.
+    class GarbageCollection
+      # On by default, unlike syncing. A sweep costs no data and takes nothing anybody has
+      # used lately — where an unbounded cache costs storage on somebody's phone forever.
+      attr_accessor :enabled
+
+      # How long an entry may go untouched before it is collected. Anything read while a page
+      # is being stored is renewed, so this measures disuse rather than age.
+      attr_accessor :max_age
+
+      # How much the cache may hold, in bytes. Past it a sweep takes the least recently used
+      # entries first, until what is left fits — so a device that browses a great deal more
+      # than it revisits has a ceiling rather than only a deadline. nil is no ceiling.
+      #
+      # Measured over what a sweep is allowed to take, which is everything but the offline
+      # page's own assets and downloaded archives: counting a 300 MB download somebody chose
+      # to keep would empty the rest of the cache to make room for it.
+      attr_reader :max_size
+
+      def max_size=(bytes)
+        @max_size = validate_size(bytes)
+      end
+
+      # How long to leave between sweeps. Also how long the cache may sit over `max_size`,
+      # since that is when the ceiling is applied.
+      attr_accessor :interval
+
+      # What the offline settings page offers, as bytes. A ladder rather than a text field:
+      # somebody adjusting this on a phone is choosing roughly how much of their device to
+      # spend, not typing a number. The configured default is always among them, or an app
+      # that set 300 MB would have no way back to it once somebody had picked something else.
+      def size_choices
+        choices = [ 50, 100, 250, 500, 1024, 2048 ].map { |mb| mb * 1024 * 1024 }
+        choices << max_size if max_size
+
+        choices.uniq.sort
+      end
+
+      def initialize
+        @enabled = true
+        @max_age = 60 * 24 * 60 * 60
+        @max_size = 250 * 1024 * 1024
+        @interval = 24 * 60 * 60
+      end
+
+      # An entry read again while a page is stored is renewed rather than refetched — but not
+      # on every navigation, or every visit would rewrite every asset the page names. A
+      # quarter of the lifetime leaves three quarters of headroom before a collection.
+      def renew_after
+        return nil unless enabled && max_age
+
+        (max_age.to_i / 4).clamp(1, max_age.to_i)
+      end
+
+      private
+
+      # Bytes, and enough of them to be a cache rather than a rounding error. A megabyte
+      # ceiling is almost always a unit mistake — someone reaching for `250.megabytes` and
+      # writing `250` — and it would sweep away all but the last page or two visited.
+      def validate_size(bytes)
+        return nil if bytes.nil?
+
+        size = bytes.to_i
+        unless size >= 1024 * 1024
+          raise ArgumentError,
+                "Coldwire garbage_collection.max_size is in bytes and has to leave room for a " \
+                "page and what it loads, so it cannot be under a megabyte: #{bytes.inspect}"
+        end
+
+        size
+      end
+    end
+
     # WebKit has no Background Sync, Periodic Background Sync or Background Fetch, so nothing
     # can wake a worker. What a page load can do is hand work to one, which then runs on
     # without it — so syncing is triggered by an open page and paced, not scheduled.
@@ -167,8 +263,8 @@ module Coldwire
       def initialize
         @enabled = false
         @precache_urls = -> { [] }
-        @interval = 6 * 60 * 60
-        @max_age = 7 * 24 * 60 * 60
+        @interval = 24 * 60 * 60
+        @max_age = 30 * 24 * 60 * 60
         @concurrency = 4
       end
     end
@@ -188,7 +284,7 @@ module Coldwire
       @register_if = -> { true }
       @caching_enabled_by_default = true
       @cache_identity = -> { nil }
-      @cache_origins = []
+      @cacheable_hosts = []
       @cache_ranges = []
       @cache_archives = []
     end
@@ -240,19 +336,27 @@ module Coldwire
 
     # An origin and nothing more: no path, no trailing slash. Anything else silently fails to
     # match a request's origin, which is the same quiet failure as a malformed path pattern.
-    def validate_origin(origin)
-      value = origin.to_s
-
-      begin
-        uri = URI.parse(value)
-      rescue URI::InvalidURIError
-        uri = nil
+    # A host, and a port only where it is not the default — which is exactly what a URL's
+    # `host` reads as, so the worker compares what you wrote against what it is handed.
+    HOST = /\A[a-z0-9](?:[a-z0-9-]*[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]*[a-z0-9])?)*(?::\d+)?\z/
+    SCHEME = %r{\A[a-z][a-z0-9+.\-]*://}
+
+    # Raised rather than quietly stripped. Everyone arriving here is renaming `cache_origins`,
+    # and a scheme silently accepted is a config that looks migrated and is not — the next
+    # person to read it learns the wrong shape.
+    def validate_host(host)
+      value = host.to_s.strip.downcase
+
+      if value.match?(SCHEME)
+        raise ArgumentError,
+              "Coldwire cacheable_hosts takes a host with no scheme — " \
+              "#{value.sub(SCHEME, '').chomp('/').inspect} rather than #{host.inspect}"
       end
 
-      unless uri&.scheme && uri.host && uri.path.to_s.empty? && uri.query.nil?
+      unless value.match?(HOST)
         raise ArgumentError,
-              "Coldwire cache_origins takes bare origins like " \
-              "\"https://tiles.example.com\": #{origin.inspect}"
+              "Coldwire cacheable_hosts takes bare hosts like " \
+              "\"tiles.example.com\": #{host.inspect}"
       end
 
       value

data/lib/coldwire/debug.css

@@ -78,6 +78,13 @@
 .coldwire-setting-title { display: block; font-weight: 600; }
 .coldwire-setting-note { display: block; margin-top: 0.15rem; font-size: 0.875rem; color: #6b7280; }
 
+/* A setting whose control is a menu rather than a switch. The words take the slack and the
+   select asks only for its own width; on a narrow screen the two wrap rather than squeezing
+   the label down to one word a line. */
+.coldwire-choice { display: flex; align-items: center; justify-content: space-between;
+  flex-wrap: wrap; gap: 0.75rem; cursor: pointer; }
+.coldwire-choice > span:first-child { flex: 1 1 12rem; min-width: 0; }
+
 .coldwire-track { height: 0.375rem; width: 100%; overflow: hidden; border-radius: 9999px; background: #e5e7eb; }
 .coldwire-bar { height: 100%; width: 100%; border-radius: 9999px; background: #374151; transition: width 200ms; }
 .coldwire-pulse { animation: coldwire-fade 1.4s ease-in-out infinite; }

data/lib/coldwire/source.rb

@@ -11,7 +11,7 @@ module Coldwire
     # The worker is served as one script but written as several. Concatenated rather than
     # imported, so the browser still fetches one file and function declarations hoist across
     # the whole of it.
-    WORKER = %w[rules serve ranges archives inspect sync events].freeze
+    WORKER = %w[rules serve ranges archives inspect sync collect events].freeze
 
     class << self
       def worker

data/lib/coldwire/worker/collect.js

@@ -0,0 +1,153 @@
+// Deleting is the one thing here with no way back. Everything else a worker gets wrong costs
+// a round trip; a collection that runs on a dead connection costs pages that cannot be
+// fetched again until one returns. So a sweep proves the network first, and only then takes
+// what nothing has asked for in a long time — and, if the cache is still over its ceiling,
+// whatever has gone longest unread until it fits.
+let collecting = null
+
+// The ceiling is a per-device choice, so it arrives with the request rather than being baked
+// into the worker. Undefined is a page that has nothing to say about it — an older client, or
+// one that never loaded the collector — and falls back to what the app configured.
+function collectGarbage({ maxSize } = {}) {
+  const limit = maxSize === undefined ? COLLECT_MAX_SIZE : maxSize
+  // A sweep already running is the answer to this one too. It may be working to a ceiling
+  // that has just changed; the next sweep uses the new one, and a sweep is cheap to be late.
+  collecting = collecting || runCollection(limit).finally(() => { collecting = null })
+
+  return collecting
+}
+
+async function runCollection(maxSize) {
+  if (COLLECT_MAX_AGE === null && maxSize === null) return { ok: true, collected: 0, skipped: "disabled" }
+  if (forcedOffline) return { ok: true, offline: true }
+  if (!(await reachable())) return { ok: true, offline: true }
+
+  const cache = await caches.open(CACHE_NAME)
+  const keys = await cache.keys()
+  const spared = offlinePageAssets()
+  const now = Date.now() / 1000
+
+  let collected = 0
+  let kept = 0
+  // What the age pass left, which is exactly what the size pass may take from.
+  const survivors = []
+
+  for (const key of keys) {
+    if (isSpared(key, spared)) { kept += 1; continue }
+
+    const at = unixTimestamp(key.headers.get(TIMESTAMP_HEADER))
+    // No stamp at all is an entry from an older worker. Age unknown is not age exceeded, so
+    // it stays; the next time its page is stored it gets one.
+    if (COLLECT_MAX_AGE === null || at === null || now - at <= COLLECT_MAX_AGE) {
+      survivors.push({ key, at })
+      continue
+    }
+
+    if (await cache.delete(key)) collected += 1
+  }
+
+  const { evicted, bytes } = await trimToSize(cache, survivors, maxSize)
+
+  return { ok: true, collected, evicted, bytes, kept: kept + survivors.length - evicted, finishedAt: Date.now() }
+}
+
+// The ceiling. Age says when something has gone stale; this says how much of somebody's phone
+// the cache may have, which on a device that browses far more than it revisits is the only one
+// of the two that ever binds.
+//
+// Oldest read out first, until what is left fits. Last used is the closest thing the cache has
+// to "least likely to be wanted back", and it is the same clock the age pass works from — an
+// entry renewed because a page still loads it is young here too, so the thing being used is
+// the last thing to go.
+async function trimToSize(cache, entries, maxSize) {
+  if (maxSize === null || entries.length === 0) return { evicted: 0, bytes: 0 }
+
+  // One match per entry, and a body read for anything without a Content-Length. Affordable
+  // because a sweep runs on `interval` rather than on navigation — but only worth paying at
+  // all when there is a ceiling to measure against, which is why it sits behind the guard.
+  const sized = []
+  let total = 0
+
+  for (const entry of entries) {
+    // ignoreVary, as the settings page does when it lists the same entries: Rails answers
+    // HTML with `Vary: Accept`, and a measurement that quietly missed would read as an entry
+    // costing nothing and so never worth evicting.
+    const size = await entrySize(await cache.match(entry.key, { ignoreVary: true }))
+    total += size
+    sized.push({ key: entry.key, at: entry.at, size })
+  }
+
+  if (total <= maxSize) return { evicted: 0, bytes: total }
+
+  // An entry with no stamp is from an older worker, and has no place in an ordering by last
+  // use. Treated as the oldest thing here: it is the one entry we can say nothing has touched
+  // since this worker started stamping them.
+  sized.sort((a, b) => (a.at || 0) - (b.at || 0))
+
+  let evicted = 0
+
+  for (const entry of sized) {
+    if (total <= maxSize) break
+    if (!(await cache.delete(entry.key))) continue
+
+    total -= entry.size
+    evicted += 1
+  }
+
+  return { evicted, bytes: total }
+}
+
+// Never collected, whatever their age — and never counted against the ceiling either, since
+// the ceiling has to measure the same thing a sweep can act on. A 300 MB archive counted in
+// would empty everything else to make room for a file nothing is allowed to take:
+//
+//   - what the offline page itself needs, which browsing never touches because nobody visits
+//     the offline page on purpose, and which is wanted precisely when there is no network
+//   - downloaded archives and the ranges of them, which somebody chose to spend a data plan
+//     on and which no amount of disuse makes safe to throw away
+function isSpared(key, spared) {
+  const url = new URL(key.url)
+  if (url.searchParams.has(CHUNK_PARAM) || url.searchParams.has(RANGE_PARAM)) return true
+
+  return spared.has(url.href)
+}
+
+function offlinePageAssets() {
+  try {
+    return new Set(urlsFromHtml(OFFLINE_PAGE, self.location.origin + "/"))
+  } catch {
+    return new Set()
+  }
+}
+
+// navigator.onLine reports whether an interface is up, which a web view answers wrongly often
+// enough to be worthless here. The probe is never intercepted, so this is a real request over
+// a real connection or it is nothing.
+async function reachable() {
+  try {
+    const response = await fetch(PROBE_PATH, { cache: "no-store", credentials: "same-origin" })
+
+    return response.ok
+  } catch {
+    return false
+  }
+}
+
+// An asset every page names is never refetched once held, so its stamp would date from the
+// first page that pulled it in — and the collector would take the stylesheet the whole app is
+// using. Renewing it on the way past is what keeps "untouched" meaning untouched.
+//
+// Rewritten from the cache rather than refetched, and only once it has aged, so an ordinary
+// navigation costs a lookup and nothing more.
+async function renew(cache, key) {
+  if (RENEW_AFTER === null) return
+
+  const at = unixTimestamp(key.headers.get(TIMESTAMP_HEADER))
+  if (at !== null && Date.now() / 1000 - at < RENEW_AFTER) return
+
+  const response = await cache.match(key)
+  if (!response) return
+
+  const managed = key.headers.get(MANAGED_HEADER) === "1"
+  await putFresh(cache, cacheKey(new Request(key.url, { method: "GET" }), { managed }), response)
+}

data/lib/coldwire/worker/events.js

@@ -70,6 +70,11 @@ self.addEventListener("message", (event) => {
   // page sees the run and not just whoever started it.
   if (type === "sync") return reply(syncManifest())
 
+  // Paced by a page the same way syncing is, and answered with whether it actually ran: a
+  // page must not record a sweep that never happened for want of a connection. The ceiling
+  // rides along, because it is the device's choice and the worker cannot read localStorage.
+  if (type === "collect") return reply(collectGarbage(event.data))
+
   // What a page missed by not listening yet, and whether it is still going on.
   if (type === "syncState") return reply({ ok: true, running: Boolean(syncing), last: lastSyncMessage })
 

data/lib/coldwire/worker/rules.js

@@ -3,7 +3,7 @@ function shouldHandle(request) {
   if (request.method !== "GET") return false
 
   const url = new URL(request.url)
-  if (!cacheableOrigin(url)) return false
+  if (!cacheableHost(url)) return false
 
   // A Range request cannot be stored as it arrives — cache.put refuses a 206 — so it is
   // stored as a 200 under a key naming the range, and answered with a 206 built here. Only
@@ -16,8 +16,8 @@ function shouldHandle(request) {
 
 // Our own origin, plus any the host app has nominated. A worker sees every request a page
 // makes, and caching other people's responses uninvited is not its business.
-function cacheableOrigin(url) {
-  return url.origin === self.location.origin || CACHE_ORIGINS.includes(url.origin)
+function cacheableHost(url) {
+  return url.origin === self.location.origin || CACHEABLE_HOSTS.includes(url.host)
 }
 
 function matchesPath(url, paths) {
@@ -83,7 +83,7 @@ function isAutoCacheable(request) {
 
   // A nominated origin is the opt-in; the path lists describe this app's own surfaces and say
   // nothing useful about somebody else's.
-  if (url.origin !== self.location.origin) return CACHE_ORIGINS.includes(url.origin)
+  if (url.origin !== self.location.origin) return CACHEABLE_HOSTS.includes(url.host)
 
   if (isNeverCached(url)) return false
 

data/lib/coldwire/worker/serve.js

@@ -43,7 +43,7 @@ function urlsFromHtml(html, pageUrl) {
         // Any origin we are allowed to cache, not just our own. A page whose map library
         // comes off a CDN is not offline-ready without it: precaching the page and skipping
         // the script it cannot run without leaves a blank screen and a full cache.
-        if (!cacheableOrigin(url)) return
+        if (!cacheableHost(url)) return
         if (matchesPath(url, NEVER_INTERCEPT)) return
         urls.add(url.href)
       } catch {}
@@ -112,11 +112,14 @@ async function storeResponse(cache, request, response) {
   await Promise.all(urls.map((href) => storeSubresource(cache, href)))
 }
 
-// Only what is missing. A digested address is stored once and then named by every page that
-// uses it, so after the first visit this settles down to a lookup and nothing else.
+// Only what is missing is fetched. What is already held is renewed instead, so an asset every
+// page names does not sit on the stamp of the first page that ever pulled it in and get
+// collected while the whole app is still using it.
 async function storeSubresource(cache, href) {
   if (isNeverCached(new URL(href))) return
-  if (await cache.match(href, MATCH_OPTIONS)) return
+
+  const [ key ] = await cache.keys(href, MATCH_OPTIONS)
+  if (key) return renew(cache, key)
 
   try {
     await fetchAndCache(cache, href)

data/lib/generators/coldwire/install/install_generator.rb

@@ -88,7 +88,7 @@ module Coldwire
       def next_steps
         say ""
         say "Coldwire is mounted at /offline.", :green
-        say "Set config.cache_identity if anyone signs in."
+        say "cache_identity uses current_user or Current.user when either is in scope."
         say "Turn on config.auto_sync to precache pages."
       end
 

data/lib/generators/coldwire/install/templates/coldwire.rb

@@ -8,15 +8,31 @@ Coldwire.configure do |config|
     sync.enabled = false          # off by default: background fetching is somebody's data plan
     sync.precache_urls = -> { [] }
     # sync.precache_urls = -> { Article.published.map { |a| article_path(a) } }
-    sync.interval = 6.hours
-    sync.max_age = 7.days
+    sync.interval = 1.day
+    sync.max_age = 30.days
     sync.concurrency = 4
   end
 
+  # Taking back what nothing has used lately, so the cache does not grow forever. Runs only
+  # with a connection, and never takes the offline page's assets or a downloaded archive.
+  # Anything a stored page still loads is renewed, so age means disuse rather than age.
+  config.garbage_collection do |gc|
+    gc.enabled = true
+    gc.max_age = 60.days          # keep comfortably longer than auto_sync.max_age
+    gc.max_size = 250.megabytes   # over this the least recently read go first; nil for no ceiling
+    gc.interval = 1.day
+  end
+
   # Who the cache belongs to. Evaluated in the view. When it changes, the cache is dropped —
-  # which is what makes signing out, and switching accounts, safe.
-  config.cache_identity = -> { nil }
-  # config.cache_identity = -> { current_user&.id }
+  # which is what makes signing out, and switching accounts, safe. Uses current_user or
+  # Current.user when either is around; otherwise nobody, and the cache stays put.
+  config.cache_identity = -> {
+    if respond_to?(:current_user)
+      current_user&.id
+    elsif defined?(Current) && Current.respond_to?(:user)
+      Current.user&.id
+    end
+  }
 
   # Where the worker registers at all. Evaluated in the view, so `request` and `current_user`
   # are both in scope. A page that does not register does not cache or sync.
@@ -38,8 +54,9 @@ Coldwire.configure do |config|
   # Never intercepted, so these fail outright offline. Coldwire's own routes are added for you.
   config.never_intercept = [ "/up" ]  # probe_path is added for you
 
-  # Origins besides your own the worker may cache, and URLs whose Range requests it caches.
-  config.cache_origins = []
+  # The hosts besides your own the worker may cache, and URLs whose Range requests it caches.
+  # No scheme — "tiles.example.com" — and a port only where it is not the default.
+  config.cacheable_hosts = []
   config.cache_ranges = []
 
   # Large files somebody can download for offline use. Nothing downloads on its own.

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: coldwire-rails
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Noreaster Group, Stuart Yamartino
@@ -26,7 +26,7 @@ dependencies:
 description: When your Hotwire wires go cold. A Rails engine that serves a Cache API
   service worker, precaches the pages you nominate, and falls back to an offline view
   Turbo will actually render. Built against Hotwire Native's rules, which are stricter
-  than a browser's, so the same cache works in a plain Hotwire app and an installed
+  than a browser's, so the same cache works in a plain Hotwire web app and an installed
   PWA.
 email:
 - stuart@noreastergroup.com
@@ -56,6 +56,11 @@ files:
 - docs/README.md
 - docs/configuration.md
 - docs/how-it-works.md
+- docs/images/coldwire-logo-dark.svg
+- docs/images/coldwire-logo.svg
+- docs/images/coldwire-mark-dark.svg
+- docs/images/coldwire-mark.svg
+- docs/images/noreaster-group.png
 - docs/images/offline-fallback.png
 - docs/images/offline-settings-cached.png
 - docs/images/offline-settings.png
@@ -63,6 +68,7 @@ files:
 - lib/coldwire-rails.rb
 - lib/coldwire.rb
 - lib/coldwire/client/api.js
+- lib/coldwire/client/collect.js
 - lib/coldwire/client/cookie.js
 - lib/coldwire/client/forced.js
 - lib/coldwire/client/identity.js
@@ -77,6 +83,7 @@ files:
 - lib/coldwire/source.rb
 - lib/coldwire/version.rb
 - lib/coldwire/worker/archives.js
+- lib/coldwire/worker/collect.js
 - lib/coldwire/worker/events.js
 - lib/coldwire/worker/inspect.js
 - lib/coldwire/worker/ranges.js