@jimhoyd/urlcode 0.3.0

package/examples/cookbook/tests/requests.json

@@ -0,0 +1,254 @@
+[
+  {
+    "path": "/hello/Ada",
+    "status": 200,
+    "expectBody": "{\"message\":\"Hello, Ada.\"}",
+    "expectHeaders": {
+      "x-middleware": "cookbook",
+      "x-app": "cookbook"
+    }
+  },
+  {
+    "path": "/hello/Ada?excited=true",
+    "status": 200,
+    "expectBody": "{\"message\":\"Hello, Ada!\"}"
+  },
+  {
+    "path": "/hello/Ada",
+    "status": 200,
+    "method": "HEAD",
+    "expectBody": "",
+    "expectHeaders": {
+      "x-middleware": "cookbook"
+    }
+  },
+  {
+    "path": "/hello/Ada?excited=maybe",
+    "status": 400
+  },
+  {
+    "path": "/echo",
+    "method": "POST",
+    "headers": {
+      "content-type": "application/json"
+    },
+    "body": "{\"ok\":true}",
+    "status": 200,
+    "expectBody": "{\"ok\":true}"
+  },
+  {
+    "path": "/echo",
+    "method": "POST",
+    "headers": {
+      "content-type": "application/json"
+    },
+    "body": "{",
+    "status": 400
+  },
+  {
+    "path": "/echo",
+    "method": "POST",
+    "headers": {
+      "content-type": "text/plain"
+    },
+    "body": "hello",
+    "status": 415
+  },
+  {
+    "path": "/text",
+    "method": "POST",
+    "headers": {
+      "content-type": "text/plain"
+    },
+    "body": "hello",
+    "status": 200,
+    "expectBody": "hello"
+  },
+  {
+    "path": "/choice",
+    "status": 302,
+    "method": "GET",
+    "expectHeaders": {
+      "location": "https://example.com/docs"
+    }
+  },
+  {
+    "path": "/product/abc?page=2&tag=red&tag=blue&utm_source=news&ignored=no",
+    "status": 302,
+    "method": "GET",
+    "expectHeaders": {
+      "location": "https://example.com/products/abc?page=2&label=red&label=blue&channel=web&utm_source=news"
+    }
+  },
+  {
+    "path": "/choice",
+    "status": 302,
+    "method": "HEAD",
+    "expectHeaders": {
+      "location": "https://example.com/docs"
+    }
+  },
+  {
+    "path": "/product/abc?page=2&tag=red&tag=blue&utm_source=news&ignored=no",
+    "status": 302,
+    "method": "HEAD",
+    "expectHeaders": {
+      "location": "https://example.com/products/abc?page=2&label=red&label=blue&channel=web&utm_source=news"
+    }
+  },
+  {
+    "path": "/product/abc?page=0",
+    "status": 400
+  },
+  {
+    "path": "/preflight",
+    "status": 204,
+    "method": "OPTIONS",
+    "expectBody": "",
+    "expectHeaders": {
+      "access-control-allow-origin": "https://app.example.com"
+    }
+  },
+  {
+    "path": "/download",
+    "status": 200,
+    "expectBody": "URLCode guide\n",
+    "expectHeaders": {
+      "content-disposition": "attachment; filename=urlcode-guide.txt"
+    }
+  },
+  {
+    "path": "/assets/site.css",
+    "status": 206,
+    "expectBody": "body",
+    "expectHeaders": {
+      "content-range": "bytes 0-3/22"
+    },
+    "headers": {
+      "range": "bytes=0-3"
+    }
+  },
+  {
+    "path": "/paused",
+    "status": 404
+  },
+  {
+    "path": "/expired",
+    "status": 410
+  },
+  {
+    "path": "/missing",
+    "status": 404
+  },
+  {
+    "path": "/secure",
+    "status": 200,
+    "expectBody": "Hardened response",
+    "expectHeaders": {
+      "referrer-policy": "no-referrer",
+      "x-frame-options": "deny",
+      "cross-origin-opener-policy": "same-origin"
+    }
+  },
+  {
+    "path": "/secure",
+    "method": "HEAD",
+    "status": 200,
+    "expectBody": "",
+    "expectHeaders": {
+      "x-frame-options": "deny"
+    }
+  },
+  {
+    "path": "/no-bots",
+    "status": 302,
+    "expectHeaders": {
+      "location": "https://example.com/article"
+    }
+  },
+  {
+    "path": "/no-bots",
+    "method": "HEAD",
+    "status": 302
+  },
+  {
+    "path": "/no-bots",
+    "headers": {
+      "user-agent": "Mozilla/5.0 (compatible; GPTBot/1.0; +https://openai.com/gptbot)"
+    },
+    "status": 403
+  },
+  {
+    "path": "/no-bots",
+    "headers": {
+      "user-agent": "curl/8.0"
+    },
+    "status": 403
+  },
+  {
+    "path": "/no-bots",
+    "headers": {
+      "user-agent": "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
+    },
+    "status": 302
+  },
+  {
+    "path": "/cached",
+    "status": 200,
+    "expectBody": "{\"cached\":true}",
+    "expectHeaders": {
+      "cache-control": "public, max-age=60"
+    }
+  },
+  {
+    "path": "/cached",
+    "method": "HEAD",
+    "status": 200,
+    "expectBody": ""
+  },
+  {
+    "path": "/budget",
+    "status": 200,
+    "expectBody": "Budgeted",
+    "expectHeaders": {
+      "ratelimit-policy": "\"default\";q=100;w=60"
+    }
+  },
+  {
+    "path": "/budget",
+    "method": "HEAD",
+    "status": 200,
+    "expectBody": ""
+  },
+  {
+    "path": "/robots.txt",
+    "status": 200,
+    "expectHeaders": {
+      "content-type": "text/plain; charset=utf-8"
+    }
+  },
+  {
+    "path": "/.well-known/security.txt",
+    "status": 200,
+    "expectBody": "Contact: mailto:security@example.com\nExpires: 2099-01-01T00:00:00Z\nPolicy: https://example.com/security-policy\nPreferred-Languages: en\n"
+  },
+  {
+    "path": "/favicon.ico",
+    "status": 200,
+    "expectHeaders": {
+      "content-type": "image/svg+xml",
+      "cache-control": "public, max-age=3600"
+    }
+  },
+  {
+    "path": "/llms.txt",
+    "status": 200,
+    "expectHeaders": {
+      "content-type": "text/plain; charset=utf-8"
+    }
+  },
+  {
+    "path": "/sitemap.xml",
+    "status": 404
+  }
+]

package/examples/cookbook/urlcode.yaml

@@ -0,0 +1,22 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/jimhoyd-com/urlcode/669632fb4f188f293a0f622847f87ee019c0627f/schemas/urlcode.schema.json
+version: "1"
+includes:
+  - routes/code.yaml
+  - routes/redirects.yaml
+  - routes/responses.yaml
+  - routes/files.yaml
+  - routes/policies.yaml
+routes: {}
+# Site conventions: each key generates one native route (see docs/SITE.md).
+# sitemap is left out here because the cookbook audit runs without --origin.
+site:
+  robots:
+    disallow: [ai-crawlers, /admin]
+    sitemap: true
+  favicon: public/favicon.svg
+  securityTxt:
+    contact: [mailto:security@example.com]
+    expires: "2099-01-01T00:00:00Z"
+    policy: [https://example.com/security-policy]
+    preferredLanguages: [en]
+  llms: public/llms.txt

package/examples/live-links/README.md

@@ -0,0 +1,11 @@
+# Live links without route reloads
+
+Follow [the dynamic-link guide](../../docs/DYNAMIC-LINKS.md) to create an external
+SQLite store and seed `demo -> https://example.com/demo`. Then start this project
+with `--link-store links=/absolute/path/links.sqlite`. Its fixtures expect that
+seed and an unused `not-created` code; use a disposable test store, not production.
+
+Create/update/delete records through the CLI or separate authenticated API while
+the public server keeps serving. YAML, code, route counts and health version do
+not change per link. This example deliberately requires explicit operator storage
+binding; the default starter still runs without a database.

package/examples/live-links/tests/requests.json

@@ -0,0 +1,6 @@
+[
+  {"path":"/r/demo","status":302,"expectHeaders":{"location":"https://example.com/demo","cache-control":"no-store"}},
+  {"path":"/r/demo","method":"HEAD","status":302,"expectBody":"","expectHeaders":{"location":"https://example.com/demo"}},
+  {"path":"/r/not-created","status":404},
+  {"path":"/r/demo","method":"POST","status":405,"expectHeaders":{"allow":"GET, HEAD"}}
+]

package/examples/live-links/urlcode.yaml

@@ -0,0 +1,16 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/jimhoyd-com/urlcode/669632fb4f188f293a0f622847f87ee019c0627f/schemas/urlcode.schema.json
+version: "1"
+dynamicLinks: true
+routes:
+  /r/{code}:
+    parameters:
+      - name: code
+        in: path
+        required: true
+        schema: {type: string, minLength: 1, maxLength: 128}
+    link:
+      collection: links
+      code: {from: path, name: code}
+  /about:
+    respond:
+      text: Links are stored separately from this YAML.

package/examples/monitoring/blackbox-jobs.yaml

@@ -0,0 +1,37 @@
+# Prometheus scrape jobs for URLCode's two probes, via blackbox_exporter.
+#
+# Keep both: /_urlcode/health answers whenever the process is alive, while
+# /_urlcode/ready reports 503 whenever a function worker or link-store reader is
+# unavailable. Alerting only on health hides a process that is up and serving
+# nothing; alerting only on ready pages for a brief, self-healing replacement.
+#
+# Both probes are unauthenticated and report the configuration digest and route
+# count. Keep them on an internal interface or restrict them at the ingress.
+scrape_configs:
+  - job_name: urlcode-ready
+    metrics_path: /probe
+    params:
+      module: [http_2xx]
+    static_configs:
+      - targets: ['http://urlcode.internal:3000/_urlcode/ready']
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: blackbox:9115
+
+  - job_name: urlcode-health
+    metrics_path: /probe
+    params:
+      module: [http_2xx]
+    static_configs:
+      - targets: ['http://urlcode.internal:3000/_urlcode/health']
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: blackbox:9115

package/examples/monitoring/prometheus-rules.yaml

@@ -0,0 +1,88 @@
+# Alert rules for a URLCode deployment.
+#
+# These rules read blackbox probes of the two health endpoints and counters
+# derived from the JSON request log by a log pipeline (see vector.toml in this
+# directory). Where the built-in endpoint (prometheus-scrape.yaml, enabled with
+# startServer({ metrics: true })) offers the same signal, a comment gives the
+# equivalent expression; it labels by status_class rather than exact status.
+# Rule names are explained in docs/MONITORING.md.
+groups:
+  - name: urlcode-availability
+    interval: 30s
+    rules:
+      - alert: UrlcodeNotReady
+        # /_urlcode/ready is 503 while a function worker or link-store reader is
+        # being replaced. Brief flaps are normal during replacement; sustained
+        # means replacement is not converging.
+        expr: probe_success{job="urlcode-ready"} == 0
+        for: 5m
+        labels:
+          severity: page
+        annotations:
+          summary: "URLCode has not been ready for 5 minutes"
+          runbook: "Check function_worker and link_store_worker records for repeated restarts."
+
+      - alert: UrlcodeDown
+        # /_urlcode/health answers whenever the process is alive and not shutting
+        # down. Failing it is a process-level problem, not a capacity one.
+        expr: probe_success{job="urlcode-health"} == 0
+        for: 2m
+        labels:
+          severity: page
+        annotations:
+          summary: "URLCode is not answering liveness probes"
+          runbook: "The process is gone, wedged or out of file descriptors. Check the supervisor."
+
+  - name: urlcode-serving
+    interval: 30s
+    rules:
+      - alert: UrlcodeServerErrors
+        # 5xx as a share of served requests, by matched route pattern.
+        # Built-in endpoint (no per-route status split):
+        #   sum(rate(urlcode_requests_total{status_class="5xx"}[5m]))
+        #     / clamp_min(sum(rate(urlcode_requests_total[5m])), 0.001) > 0.05
+        expr: |
+          sum by (route) (rate(urlcode_requests_total{status=~"5.."}[5m]))
+            / clamp_min(sum by (route) (rate(urlcode_requests_total[5m])), 0.001) > 0.05
+        for: 10m
+        labels:
+          severity: page
+        annotations:
+          summary: "Over 5% of requests to {{ $labels.route }} are failing"
+          runbook: "502 is a function error, 503 is capacity or a dead pool, 504 is a deadline."
+
+      - alert: UrlcodeCapacityShedding
+        # 503 specifically means admission or the function pool refused work.
+        # Built-in endpoint: sum(rate(urlcode_shed_total[5m])) > 1
+        expr: sum(rate(urlcode_requests_total{status="503"}[5m])) > 1
+        for: 10m
+        labels:
+          severity: ticket
+        annotations:
+          summary: "URLCode is shedding requests"
+          runbook: "Raise --max-in-flight or --workers, or find what is occupying the pool."
+
+      - alert: UrlcodeWorkerRestartLoop
+        # Replacement backs off but never stops, so a persistent cause shows up
+        # as a steady restart rate rather than a stopped pool.
+        # Built-in endpoint: urlcode_function_worker_restarts_total and
+        # urlcode_link_store_worker_restarts_total, summed.
+        expr: sum(rate(urlcode_worker_restarts_total[15m])) > 0
+        for: 15m
+        labels:
+          severity: ticket
+        annotations:
+          summary: "URLCode workers have been restarting for 15 minutes"
+          runbook: "A function is exceeding its deadline or exhausting its heap on real traffic."
+
+      - alert: UrlcodeLogsDropped
+        # The logger drops records when the collector cannot keep up, so the
+        # rules above quietly lose their input. The built-in endpoint exports
+        # the same name, so this rule works unchanged with either source.
+        expr: sum(rate(urlcode_logs_dropped_total[15m])) > 0
+        for: 15m
+        labels:
+          severity: ticket
+        annotations:
+          summary: "URLCode is dropping log records"
+          runbook: "The collector is behind or its sink failed; these alerts are blind until it recovers."

package/examples/monitoring/prometheus-scrape.yaml

@@ -0,0 +1,20 @@
+# Prometheus scrape job for URLCode's built-in metrics endpoint.
+#
+# The endpoint exists only when the operator starts the server with
+# startServer({ metrics: true }); otherwise /_urlcode/metrics answers 404. It
+# shares the probes' admission budget (--max-in-flight-health) and bind host,
+# is unauthenticated, and discloses route patterns and traffic shape: keep it
+# on an internal interface or restrict it at the ingress. Never expose it
+# publicly.
+#
+# Metric names and labels are fixed in docs/OBSERVABILITY.md. Requests are
+# labelled by status_class (2xx..5xx), not exact status, so the example rules
+# in prometheus-rules.yaml carry a status_class form of each expression next
+# to the log-derived one from vector.toml.
+scrape_configs:
+  - job_name: urlcode
+    metrics_path: /_urlcode/metrics
+    scrape_interval: 15s
+    scrape_timeout: 5s
+    static_configs:
+      - targets: ['urlcode.internal:3000']

package/examples/monitoring/vector.toml

@@ -0,0 +1,65 @@
+# Turn URLCode's JSON request log into the counters the example alert rules use.
+#
+# URLCode writes one JSON object per line to stdout. Its built-in endpoint
+# (prometheus-scrape.yaml) is opt-in and labels by status class only; this
+# pipeline keeps the exact status, at the cost of running a collector. Point your
+# supervisor's stdout at Vector (or adapt this to Fluent Bit, Promtail or
+# Alloy — the field names are what matter).
+
+[sources.urlcode]
+type = "file"
+include = ["/var/log/urlcode/*.log"]
+
+[transforms.parsed]
+type = "remap"
+inputs = ["urlcode"]
+source = '''
+. = parse_json!(string!(.message))
+# Records without an event field are not ours; drop rather than mislabel them.
+if !exists(.event) { abort }
+'''
+
+[transforms.requests]
+type = "filter"
+inputs = ["parsed"]
+condition = '.event == "request"'
+
+[transforms.workers]
+type = "filter"
+inputs = ["parsed"]
+# Both pools report replacement the same way, so one counter covers them.
+condition = '(.event == "function_worker" || .event == "link_store_worker") && .status == "restarting"'
+
+[transforms.dropped]
+type = "filter"
+inputs = ["parsed"]
+condition = '.event == "logs_dropped"'
+
+[transforms.metrics]
+type = "log_to_metric"
+inputs = ["requests", "workers", "dropped"]
+
+# route is present only with --request-log detailed; it is the configured route
+# pattern, never the requested path, so it is safe as a metric label.
+[[transforms.metrics.metrics]]
+type = "counter"
+field = "status"
+name = "urlcode_requests_total"
+namespace = ""
+tags.status = "{{status}}"
+tags.route = "{{route}}"
+
+[[transforms.metrics.metrics]]
+type = "counter"
+field = "status"
+name = "urlcode_worker_restarts_total"
+
+[[transforms.metrics.metrics]]
+type = "counter"
+field = "count"
+name = "urlcode_logs_dropped_total"
+
+[sinks.prometheus]
+type = "prometheus_exporter"
+inputs = ["metrics"]
+address = "127.0.0.1:9598"

package/examples/prerender/README.md

@@ -0,0 +1,48 @@
+# Prerender a dynamic project into a native static one
+
+Three pages rendered by a function and one shared template middleware, then
+rendered once at build time into a project that serves the same bytes with
+native `page` routes and no sandbox on the request path.
+
+From the runtime checkout:
+
+```sh
+node src/cli.ts dev --project examples/prerender        # the dynamic source, live
+node src/cli.ts test --project examples/prerender       # 7 fixtures
+node examples/prerender/prerender.mjs examples/prerender /absolute/out
+node src/cli.ts test --project /absolute/out            # fixtures the build wrote
+node src/cli.ts audit --project /absolute/out --expect-routes 3
+```
+
+The same three URLs answer identically before and after. The difference is what
+runs to serve them: the source project executes a QuickJS/WASM guest per request,
+the generated project reads a prevalidated byte buffer.
+
+| | Source project | Generated project |
+|---|---|---|
+| Handlers | 3 × `function` | 3 × `page` |
+| Middleware | shared template | none |
+| Guest execution per request | yes | none |
+| Content | reviewed literals in YAML | rendered HTML files |
+
+`prerender.mjs` is operator build tooling that runs in Node, not guest code. The
+orchestration lives in the runtime's build helper:
+
+```js
+import {prerenderPages, assertNativeProject} from '@jimhoyd/urlcode/prerender';
+```
+
+`prerenderPages` activates the source project, renders each page through its
+middleware, validates every response, enforces the budgets, derives safe
+filenames, writes the files and closes the runtime. It fails the build rather
+than publishing a bad page. What is left in this recipe is the part every site
+does differently: assembling a project from the returned `pages` and `fixtures`,
+then calling `assertNativeProject` to prove the artifact is inert. A larger site
+assembles differently — its own static and download routes, response security
+headers and a generated include — using the same helper.
+
+The recipe renders content that is already prepared. It is not a Markdown
+compiler, an HTML sanitizer, an asset pipeline or an incremental build, and it
+copies no static tree. [Prerendering](../../docs/PRERENDER.md) documents the
+helper, its options and guarantees, the limits and how a larger site generates
+its source project.

package/examples/prerender/functions/page.mjs

@@ -0,0 +1,6 @@
+// One page's content, supplied as reviewed literal arguments in urlcode.yaml.
+// This runs in the QuickJS/WASM sandbox with no filesystem, network or host
+// code, at build time exactly as it would at request time.
+export function page(request, {args}) {
+  return new Response(args.body, {headers: {'content-type': 'text/html; charset=utf-8'}});
+}

package/examples/prerender/middleware/template.mjs

@@ -0,0 +1,29 @@
+// The shared site template. It wraps a function response, whose body is
+// readable through text(); a native page/static/download body is deliberately
+// opaque, which is why the template is applied here, before prerendering, and
+// never to the generated files.
+const escape = value => String(value).replace(/[&<>"]/g, character => ({'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;'}[character]));
+export async function template(request, context, next) {
+  const response = await next();
+  // Never wrap a failed render: the build must see the original status and stop.
+  if (!response.ok) return response;
+  const content = await response.text();
+  const title = escape(context.args.title);
+  return new Response(`<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>${title}</title>
+</head>
+<body>
+<header><a href="/">URLCode prerender example</a></header>
+<main>
+<h1>${title}</h1>
+${content}
+</main>
+<footer>Prerendered at build time. No guest code runs to serve this page.</footer>
+</body>
+</html>
+`, {status: response.status, headers: {'content-type': 'text/html; charset=utf-8'}});
+}

package/examples/prerender/prerender.d.mts

@@ -0,0 +1,4 @@
+// Declarations for the recipe, so a TypeScript build script (or the test
+// suite) can import it without a cast. The runtime is prerender.mjs.
+export interface PrerenderReport { pages: number; bytes: number; output: string }
+export function prerender(project: string, output: string, options?: { log?: (event: object) => void }): Promise<PrerenderReport>;