@redirhub/cloudflare-redirects 0.1.1 → 0.2.1

package/README.md

@@ -99,10 +99,12 @@ add your own logic, pass `origin: (request, env, ctx) => …`.
 | Option | |
 | --- | --- |
 | `settings(env)` | The rules address and the key from the Worker's bindings. Empty address: off, every request goes to the origin. |
+| `report` | `false`, or `(request) => boolean`: whether to report, over the key. Default: whenever there is a key. |
 | `origin(request, env, ctx)` | Where requests no rule answers go; default `fetch(request)`. |
 | `report404` | `rules` (default): 404s of hostnames with **Report 404 pages** on; `all`: every hostname the Worker serves. |
 | `hostname(request)` | The hostname to look up in the rules; `null` leaves the request alone (straight to the origin, nothing reported). Default the `Host` header. |
-| `path(request)` | The path the visitor asked for, percent-encoded; `null` leaves the request alone. Default the request URL's path. |
+| `path(request)` | The path to look up in the rules, percent-encoded; `null` leaves the request alone. Default the request URL's path. |
+| `location(location, request)` | The `Location` to send, from the one the rule gives; also the reported destination. |
 | `clientIp(request)` | The visitor's address in reports; default `CF-Connecting-IP`. `null` writes `-`. |
 | `userAgent(request)` | The User-Agent in reports; default the `User-Agent` header. `null` writes `-`. |
 | `anonymizeIp` | Report IPv4 with the last byte zeroed, IPv6 cut to /48. |
@@ -127,7 +129,8 @@ export default createWorker<Env>({
     const { pathname } = new URL(request.url)
     return pathname.startsWith('/api/') ? null : pathname
   },
-  userAgent: (request) => request.headers.get('user-agent'),
+  // Testing on workers.dev redirects, but is not reported.
+  report: (request) => !new URL(request.url).hostname.endsWith('.workers.dev'),
   anonymizeIp: true,
 })
 ```
@@ -165,10 +168,11 @@ ignored: a Worker keeps no connection open.
 
 **Reports.** A line queued while nothing is being sent goes out at once; lines queued while a
 request is out go together in the next one (at most 100 lines or 64 KB, gzip with
-`CompressionStream`, 2 second timeout). Only the request that starts sending waits for it, in
-`ctx.waitUntil`, for at most 10 seconds; the others only add their line, so no request waits on
-another's work. A failed or rejected batch is dropped, never retried; beyond 10,000 waiting lines,
-lines are dropped.
+`CompressionStream`, 2 second timeout), and each full batch opens another request, up to four at
+once. Only the request that starts sending waits for them, in `ctx.waitUntil`, for at most 10
+seconds; the others only add their line, so no request waits on another's work. A failed or
+rejected batch is dropped, never retried; beyond 10,000 waiting lines, lines are dropped. Dropped
+lines are logged at once, then at most every ten minutes with the total by reason.
 
 ## Privacy
 

package/dist/core/redirects.d.ts

@@ -13,6 +13,11 @@ export interface RedirectsOptions {
     rulesUrl: string | null | undefined;
     /** The integration key. Without it redirects work and nothing is reported. */
     key?: string | null;
+    /**
+     * Whether to report, over the key: `false` reports nothing, a function decides per request
+     * (e.g. not for preview deployments). Default: report whenever there is a key.
+     */
+    report?: boolean | ((request: Request) => boolean);
     /**
      * Which 404 answers of the site are reported (where the site's answer is seen, `afterSite`):
      * `rules` (default) for hostnames whose `@host` line has `report404`, `all` for every hostname.
@@ -32,6 +37,11 @@ export interface RedirectsOptions {
      * your code sees it, e.g. a proxy that strips a prefix.
      */
     path?: (request: Request) => string | null;
+    /**
+     * The `Location` to send, from the one the rule gives (e.g. a same-site target in the visitor's
+     * language). It is also what the report names as the destination. Default: as the rule gives it.
+     */
+    location?: (location: string, request: Request) => string;
     /**
      * The visitor's address in reports: only a value the platform sets, never a header a client can
      * forge. Null writes `-`.

package/dist/core/redirects.js

@@ -43,7 +43,7 @@ export function createRedirects(options) {
     };
     const keepAliveOf = (context) => (context ? (work) => context.waitUntil(work) : undefined);
     const report = (request, url, host, status, size, keepAlive, redirect) => {
-        if (!reporter) {
+        if (!reporter || options.report === false || (typeof options.report === 'function' && !options.report(request))) {
             return;
         }
         const ip = options.clientIp?.(request) ?? null;
@@ -61,7 +61,8 @@ export function createRedirects(options) {
         }, now()), keepAlive);
     };
     const redirect = (request, url, rules, host, rule, keepAlive) => {
-        const location = buildLocation(rule, url);
+        const target = buildLocation(rule, url);
+        const location = options.location ? options.location(target, request) : target;
         const headers = new Headers({ Location: byteString(location) });
         if (rules.poweredBy) {
             headers.set('X-Powered-By', rules.poweredBy);

package/dist/core/report.d.ts

@@ -47,15 +47,18 @@ export interface ReporterOptions {
     maxQueue?: number;
     /** How long one invocation keeps sending before it leaves the rest to the next; default 10,000. */
     budgetMs?: number;
+    /** Requests out at the same time; default 4. */
+    maxInFlight?: number;
 }
 /**
  * Sends report lines without delaying a response and without a timer.
  *
  * A line queued while nothing is being sent goes out at once, so a site with little traffic
- * reports each visit immediately and keeps nothing waiting. Lines queued while a request is out
- * wait for it and go together in the next one (at most 100 lines, 64 KB, gzip), so a burst costs
- * one request per hundred visits. The invocation that starts sending keeps itself alive (through
- * `keepAlive`) until the queue is empty; the others only add their line, so no request waits on
+ * reports each visit immediately and keeps nothing waiting. Lines queued while requests are out
+ * go together in the next one (at most 100 lines, 64 KB, gzip), so a burst costs one request per
+ * hundred visits; up to `maxInFlight` (4) requests are out at once, so a busy instance keeps up.
+ * The invocation that starts sending keeps itself alive (through `keepAlive`) until the queue is
+ * empty and its requests are back; the others only add their line, so no request waits on
  * another request's work. A failed or rejected batch is dropped and counted, never retried.
  */
 export declare class Reporter {

package/dist/core/report.js

@@ -73,17 +73,25 @@ const DROP_LOG_EVERY_MS = 10 * 60_000;
  * Sends report lines without delaying a response and without a timer.
  *
  * A line queued while nothing is being sent goes out at once, so a site with little traffic
- * reports each visit immediately and keeps nothing waiting. Lines queued while a request is out
- * wait for it and go together in the next one (at most 100 lines, 64 KB, gzip), so a burst costs
- * one request per hundred visits. The invocation that starts sending keeps itself alive (through
- * `keepAlive`) until the queue is empty; the others only add their line, so no request waits on
+ * reports each visit immediately and keeps nothing waiting. Lines queued while requests are out
+ * go together in the next one (at most 100 lines, 64 KB, gzip), so a burst costs one request per
+ * hundred visits; up to `maxInFlight` (4) requests are out at once, so a busy instance keeps up.
+ * The invocation that starts sending keeps itself alive (through `keepAlive`) until the queue is
+ * empty and its requests are back; the others only add their line, so no request waits on
  * another request's work. A failed or rejected batch is dropped and counted, never retried.
  */
 export class Reporter {
     #options;
     #queue = [];
+    /** The sender: the invocation that started it keeps it alive until it is done. */
     #sending = null;
-    #dropped = 0;
+    /** The sender's requests out now, and when it started (for the budget); #running once it has. */
+    #out = new Set();
+    #started = 0;
+    #running = false;
+    /** Lines dropped since the last log, by reason. */
+    #dropped = new Map();
+    #droppedSince = 0;
     #lastDropLog = Number.NEGATIVE_INFINITY;
     constructor(options) {
         // `??` per field, not a spread: an option passed as undefined keeps its default.
@@ -95,6 +103,7 @@ export class Reporter {
             timeoutMs: options.timeoutMs ?? 2000,
             maxQueue: options.maxQueue ?? 10_000,
             budgetMs: options.budgetMs ?? 10_000,
+            maxInFlight: options.maxInFlight ?? 4,
         };
     }
     /** Queues one line; `keepAlive` holds the invocation open while this call has to send. */
@@ -102,6 +111,7 @@ export class Reporter {
         if (this.#options.key === '') {
             return;
         }
+        this.#logDrops();
         if (this.#queue.length >= this.#options.maxQueue) {
             this.#drop(1, 'queue full');
             return;
@@ -111,6 +121,9 @@ export class Reporter {
             this.#sending = this.#send();
             keepAlive?.(this.#sending);
         }
+        else if (this.#running) {
+            this.#fill();
+        }
     }
     /** Resolves once everything queued so far has been sent or dropped. */
     async flush() {
@@ -125,14 +138,31 @@ export class Reporter {
     async #send() {
         // Lines queued in the same turn of the event loop go in the first request.
         await Promise.resolve();
-        const started = Date.now();
+        this.#started = Date.now();
+        this.#running = true;
         for (;;) {
-            if (this.#queue.length === 0 || Date.now() - started >= this.#options.budgetMs) {
+            this.#fill();
+            if (this.#out.size === 0) {
                 // Checked and cleared in one step: a line queued after this starts a new sender.
+                this.#running = false;
                 this.#sending = null;
                 return;
             }
-            await this.#post(this.#take());
+            await Promise.race(this.#out);
+        }
+    }
+    /**
+     * Starts requests for the waiting lines, within the budget: one when none is out, and more,
+     * up to maxInFlight, only for full batches, so lines that arrive while a request is out wait
+     * for it and go together. Called by the sender and on each line queued while it runs.
+     */
+    #fill() {
+        while (this.#out.size < this.#options.maxInFlight &&
+            this.#queue.length > 0 &&
+            (this.#out.size === 0 || this.#queue.length >= MAX_BATCH_LINES) &&
+            Date.now() - this.#started < this.#options.budgetMs) {
+            const post = this.#post(this.#take()).finally(() => this.#out.delete(post));
+            this.#out.add(post);
         }
     }
     #take() {
@@ -180,15 +210,27 @@ export class Reporter {
             this.#drop(lines.length, error instanceof Error ? error.name : 'error');
         }
     }
-    /** Counts dropped lines and logs them at most once per ten minutes. */
+    /** Counts dropped lines by reason; see #logDrops. */
     #drop(lines, reason) {
-        this.#dropped += lines;
-        const now = Date.now();
-        if (now - this.#lastDropLog < DROP_LOG_EVERY_MS) {
+        if (this.#dropped.size === 0) {
+            this.#droppedSince = Date.now();
+        }
+        this.#dropped.set(reason, (this.#dropped.get(reason) ?? 0) + lines);
+        this.#logDrops();
+    }
+    /**
+     * Logs the dropped lines: the first drop at once, then at most once per ten minutes with the
+     * total since the last log, by reason. Called on every drop and every line queued, so a count
+     * is logged even after the drops stop, without a timer.
+     */
+    #logDrops() {
+        if (this.#dropped.size === 0 || Date.now() - this.#lastDropLog < DROP_LOG_EVERY_MS) {
             return;
         }
-        this.#options.logger?.warn(`${PACKAGE}: ${this.#dropped} report lines dropped (${reason})`);
-        this.#dropped = 0;
-        this.#lastDropLog = now;
+        const total = [...this.#dropped.values()].reduce((sum, n) => sum + n, 0);
+        const reasons = [...this.#dropped].map(([reason, n]) => `${reason} ${n}`).join(', ');
+        this.#options.logger?.warn(`${PACKAGE}: ${total} report lines dropped since ${new Date(this.#droppedSince).toISOString()} (${reasons})`);
+        this.#dropped.clear();
+        this.#lastDropLog = Date.now();
     }
 }

package/dist/version.d.ts

@@ -1,5 +1,5 @@
 /** This package: names its log lines and the User-Agent of rules downloads and reports. */
 export declare const PACKAGE = "cloudflare-redirects";
 /** Equal to package.json's version (a test checks it). */
-export declare const VERSION = "0.1.1";
-export declare const USER_AGENT = "cloudflare-redirects/0.1.1";
+export declare const VERSION = "0.2.1";
+export declare const USER_AGENT = "cloudflare-redirects/0.2.1";

package/dist/version.js

@@ -1,5 +1,5 @@
 /** This package: names its log lines and the User-Agent of rules downloads and reports. */
 export const PACKAGE = 'cloudflare-redirects';
 /** Equal to package.json's version (a test checks it). */
-export const VERSION = '0.1.1';
+export const VERSION = '0.2.1';
 export const USER_AGENT = `${PACKAGE}/${VERSION}`;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@redirhub/cloudflare-redirects",
-  "version": "0.1.1",
+  "version": "0.2.1",
   "description": "Bulk redirects and 404 monitoring for Cloudflare Workers from a remote rules file: thousands of 301/302 redirects, short links and website migrations in front of your origin, live within a minute without a redeploy. RedirHub Integration Mode.",
   "keywords": [
     "cloudflare",