imgtune 0.0.0-stage → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. package/CHANGELOG.md +322 -0
  2. package/LICENSE +21 -0
  3. package/NOTICE +20 -0
  4. package/README.md +350 -2
  5. package/THIRD_PARTY_LICENSES +295 -0
  6. package/dist/address.js +221 -0
  7. package/dist/animated.js +176 -0
  8. package/dist/app-routes.js +211 -0
  9. package/dist/atomic.js +63 -0
  10. package/dist/audit.js +372 -0
  11. package/dist/backgrounds.js +363 -0
  12. package/dist/broken.js +106 -0
  13. package/dist/browser.js +30 -0
  14. package/dist/calibrate.js +221 -0
  15. package/dist/candidates.js +151 -0
  16. package/dist/check-report.js +302 -0
  17. package/dist/check.js +416 -0
  18. package/dist/cli.js +531 -0
  19. package/dist/compare.js +398 -0
  20. package/dist/concurrency.js +37 -0
  21. package/dist/config.js +10 -0
  22. package/dist/crawl.js +186 -0
  23. package/dist/deep.js +76 -0
  24. package/dist/egress-proxy.js +151 -0
  25. package/dist/encode.js +101 -0
  26. package/dist/estimate-jobs.js +114 -0
  27. package/dist/fetch.js +179 -0
  28. package/dist/ffmpeg.js +495 -0
  29. package/dist/gpu.js +353 -0
  30. package/dist/group-sizes.js +120 -0
  31. package/dist/hints.js +1102 -0
  32. package/dist/image-file.js +553 -0
  33. package/dist/images.js +361 -0
  34. package/dist/layout.js +240 -0
  35. package/dist/lcp.js +66 -0
  36. package/dist/loaders.js +267 -0
  37. package/dist/loads.js +29 -0
  38. package/dist/locate-join.js +511 -0
  39. package/dist/locate-only.js +178 -0
  40. package/dist/locate.js +382 -0
  41. package/dist/lru.js +34 -0
  42. package/dist/measure.js +238 -0
  43. package/dist/media-encode.js +1050 -0
  44. package/dist/media-file.js +284 -0
  45. package/dist/media-limits.js +31 -0
  46. package/dist/media-model.js +247 -0
  47. package/dist/media-phase.js +652 -0
  48. package/dist/media-prepare.js +31 -0
  49. package/dist/metric.js +227 -0
  50. package/dist/next-config.js +87 -0
  51. package/dist/qualities-cap.js +141 -0
  52. package/dist/recheck-rows.js +40 -0
  53. package/dist/recheck.js +376 -0
  54. package/dist/remote.js +3582 -0
  55. package/dist/report.js +253 -0
  56. package/dist/result-cache.js +183 -0
  57. package/dist/routes.js +280 -0
  58. package/dist/run.js +445 -0
  59. package/dist/search.js +88 -0
  60. package/dist/site.js +31 -0
  61. package/dist/sizes-outcome.js +81 -0
  62. package/dist/sizes-recheck.js +332 -0
  63. package/dist/sizes.js +148 -0
  64. package/dist/source-root.js +77 -0
  65. package/dist/srcset-parse.js +98 -0
  66. package/dist/srcset.js +44 -0
  67. package/dist/ssim.js +196 -0
  68. package/dist/stage.js +51 -0
  69. package/dist/summary.js +495 -0
  70. package/dist/survey.js +13 -0
  71. package/dist/text.js +72 -0
  72. package/dist/third-party.js +127 -0
  73. package/dist/timing.js +60 -0
  74. package/dist/types.js +1 -0
  75. package/dist/urls.js +300 -0
  76. package/dist/version.js +3 -0
  77. package/dist/video-analysis.js +273 -0
  78. package/dist/webp-animation.js +283 -0
  79. package/dist/worst.js +42 -0
  80. package/package.json +70 -4
  81. package/skills/imgtune/AGENTS.snippet.md +47 -0
  82. package/skills/imgtune/SKILL.md +344 -0
  83. package/skills/imgtune/reference.md +339 -0
  84. package/vendor/ssimulacra2/ssim_wasm.cjs +113 -0
  85. package/vendor/ssimulacra2/ssim_wasm.d.ts +8 -0
  86. package/vendor/ssimulacra2/ssim_wasm_bg.wasm +0 -0
  87. package/vendor/ssimulacra2/ssim_wasm_bg.wasm.d.ts +6 -0
@@ -0,0 +1,221 @@
1
+ import { lookup as dnsLookup } from "node:dns/promises";
2
+ import { BlockList, isIP } from "node:net";
3
+ export const systemResolver = (hostname) => dnsLookup(hostname, { all: true });
4
+ /** Blocked ranges, from the IANA special-purpose registries. Checked with net.BlockList, never with patterns. */
5
+ const BLOCKED = new BlockList();
6
+ const BLOCKED_IPV4 = [
7
+ ["0.0.0.0", 8],
8
+ ["10.0.0.0", 8],
9
+ ["100.64.0.0", 10],
10
+ ["127.0.0.0", 8],
11
+ ["169.254.0.0", 16],
12
+ ["172.16.0.0", 12],
13
+ ["192.0.0.0", 24],
14
+ ["192.0.2.0", 24],
15
+ ["192.168.0.0", 16],
16
+ ["198.18.0.0", 15],
17
+ ["198.51.100.0", 24],
18
+ ["203.0.113.0", 24],
19
+ ["224.0.0.0", 4],
20
+ ["240.0.0.0", 4],
21
+ ["255.255.255.255", 32],
22
+ ];
23
+ const BLOCKED_IPV6 = [
24
+ ["::", 128],
25
+ ["::1", 128],
26
+ ["fc00::", 7],
27
+ ["fe80::", 10],
28
+ ["ff00::", 8],
29
+ ["2001:db8::", 32],
30
+ ["64:ff9b::", 96],
31
+ ["64:ff9b:1::", 48],
32
+ ["2001::", 32],
33
+ ["2002::", 16],
34
+ ];
35
+ for (const [net, prefix] of BLOCKED_IPV4)
36
+ BLOCKED.addSubnet(net, prefix, "ipv4");
37
+ for (const [net, prefix] of BLOCKED_IPV6)
38
+ BLOCKED.addSubnet(net, prefix, "ipv6");
39
+ /** An IPv6 address (any textual form, with an optional dotted tail and zone) as its 16 bytes; undefined if malformed. */
40
+ export function ipv6Bytes(text) {
41
+ let s = text.toLowerCase();
42
+ const zone = s.indexOf("%");
43
+ if (zone !== -1)
44
+ s = s.slice(0, zone);
45
+ const dotted = /(\d+)\.(\d+)\.(\d+)\.(\d+)$/.exec(s);
46
+ if (dotted) {
47
+ const [a, b, c, d] = dotted.slice(1).map(Number);
48
+ s = `${s.slice(0, dotted.index)}${((a << 8) | b).toString(16)}:${((c << 8) | d).toString(16)}`;
49
+ }
50
+ const halves = s.split("::");
51
+ if (halves.length > 2)
52
+ return undefined;
53
+ const parseGroups = (part) => {
54
+ if (part === "")
55
+ return [];
56
+ const out = [];
57
+ for (const group of part.split(":")) {
58
+ if (!/^[0-9a-f]{1,4}$/.test(group))
59
+ return undefined;
60
+ out.push(parseInt(group, 16));
61
+ }
62
+ return out;
63
+ };
64
+ const head = parseGroups(halves[0]);
65
+ const tail = halves.length === 2 ? parseGroups(halves[1]) : [];
66
+ if (!head || !tail)
67
+ return undefined;
68
+ let groups;
69
+ if (halves.length === 2) {
70
+ const fill = 8 - head.length - tail.length;
71
+ if (fill < 0)
72
+ return undefined;
73
+ groups = [...head, ...new Array(fill).fill(0), ...tail];
74
+ }
75
+ else {
76
+ groups = head;
77
+ }
78
+ if (groups.length !== 8)
79
+ return undefined;
80
+ const bytes = new Uint8Array(16);
81
+ groups.forEach((g, i) => {
82
+ bytes[2 * i] = g >> 8;
83
+ bytes[2 * i + 1] = g & 0xff;
84
+ });
85
+ return bytes;
86
+ }
87
+ /**
88
+ * Whether an address (IPv4 or IPv6 in any textual form) is one a scan must not reach. IPv4-mapped (::ffff:a.b.c.d) and
89
+ * IPv4-compatible (::a.b.c.d) IPv6 addresses are unwrapped and their IPv4 part is checked too. Unparsable text is blocked.
90
+ */
91
+ export function isBlockedAddress(address) {
92
+ const family = isIP(address);
93
+ if (family === 4)
94
+ return BLOCKED.check(address, "ipv4");
95
+ if (family !== 6)
96
+ return true;
97
+ const bytes = ipv6Bytes(address);
98
+ if (!bytes)
99
+ return true;
100
+ const zeroPrefix = bytes.subarray(0, 10).every((b) => b === 0);
101
+ const mapped = bytes[10] === 0xff && bytes[11] === 0xff;
102
+ const compatible = bytes[10] === 0 && bytes[11] === 0;
103
+ // NAT64 well-known prefix 64:ff9b::/96: the IPv4 address is the last 32 bits.
104
+ const nat64 = bytes[0] === 0x00 && bytes[1] === 0x64 && bytes[2] === 0xff && bytes[3] === 0x9b && bytes.subarray(4, 12).every((b) => b === 0);
105
+ // ::ffff:0:0/96 (IPv4-mapped) is blocked as a whole. It is not a BlockList entry: an IPv6 subnet there also matches
106
+ // plain IPv4 checks, which would block public IPv4 addresses.
107
+ if (mapped && zeroPrefix)
108
+ return true;
109
+ // ::/96 (IPv4-compatible, deprecated) is blocked as a whole: no routable address needs it.
110
+ if (zeroPrefix && compatible)
111
+ return true;
112
+ if (nat64) {
113
+ const embedded = `${bytes[12]}.${bytes[13]}.${bytes[14]}.${bytes[15]}`;
114
+ if (BLOCKED.check(embedded, "ipv4"))
115
+ return true;
116
+ }
117
+ return BLOCKED.check(address, "ipv6");
118
+ }
119
+ /** Loopback, RFC 1918 and unique-local addresses: where a dev server on this machine or the LAN is reached. */
120
+ const PRIVATE = new BlockList();
121
+ for (const [net, prefix] of [["127.0.0.0", 8], ["10.0.0.0", 8], ["172.16.0.0", 12], ["192.168.0.0", 16]]) {
122
+ PRIVATE.addSubnet(net, prefix, "ipv4");
123
+ }
124
+ PRIVATE.addSubnet("::1", 128, "ipv6");
125
+ PRIVATE.addSubnet("fc00::", 7, "ipv6");
126
+ /** Whether an address is a private one (IPv4-mapped IPv6 unwrapped). Unparsable text is not private. */
127
+ export function isPrivateAddress(address) {
128
+ const family = isIP(address);
129
+ if (family === 4)
130
+ return PRIVATE.check(address, "ipv4");
131
+ if (family !== 6)
132
+ return false;
133
+ const bytes = ipv6Bytes(address);
134
+ if (!bytes)
135
+ return false;
136
+ const mapped = bytes.subarray(0, 10).every((b) => b === 0) && bytes[10] === 0xff && bytes[11] === 0xff;
137
+ if (mapped)
138
+ return PRIVATE.check(`${bytes[12]}.${bytes[13]}.${bytes[14]}.${bytes[15]}`, "ipv4");
139
+ return PRIVATE.check(address, "ipv6");
140
+ }
141
+ /**
142
+ * Whether a connection to `address` is refused. A blocked address is refused, except that a private one is allowed
143
+ * when the scan's own target is local (`allowPrivate`). Link-local, metadata, unspecified and multicast addresses are
144
+ * refused in every case.
145
+ */
146
+ export function refusedAddress(address, allowPrivate) {
147
+ if (!isBlockedAddress(address))
148
+ return false;
149
+ if (!allowPrivate)
150
+ return true;
151
+ // The private allowance is narrower than the private ranges: cloud metadata endpoints sit inside them (fd00:ec2::/64
152
+ // inside fc00::/7), so they are refused even for a local target.
153
+ if (ALWAYS_REFUSED.check(address, ipFamily(address)))
154
+ return true;
155
+ const unwrapped = mappedIPv4(address);
156
+ if (unwrapped !== undefined && ALWAYS_REFUSED.check(unwrapped, "ipv4"))
157
+ return true;
158
+ return !isPrivateAddress(address);
159
+ }
160
+ /** Refused for a local target too: link-local, cloud metadata endpoints and the unspecified network. */
161
+ const ALWAYS_REFUSED = new BlockList();
162
+ for (const [net, prefix] of [["169.254.0.0", 16], ["100.100.100.200", 32], ["0.0.0.0", 8]]) {
163
+ ALWAYS_REFUSED.addSubnet(net, prefix, "ipv4");
164
+ }
165
+ for (const [net, prefix] of [["fe80::", 10], ["fd00:ec2::", 64]]) {
166
+ ALWAYS_REFUSED.addSubnet(net, prefix, "ipv6");
167
+ }
168
+ /** The IPv4 address inside an IPv4-mapped IPv6 address (::ffff:a.b.c.d), or undefined. */
169
+ function mappedIPv4(address) {
170
+ const bytes = isIP(address) === 6 ? ipv6Bytes(address) : undefined;
171
+ if (!bytes || !bytes.subarray(0, 10).every((b) => b === 0) || bytes[10] !== 0xff || bytes[11] !== 0xff)
172
+ return undefined;
173
+ return `${bytes[12]}.${bytes[13]}.${bytes[14]}.${bytes[15]}`;
174
+ }
175
+ /** Family of a parsed address, for BlockList checks. Unparsable text is treated as IPv6 (no entry matches it). */
176
+ function ipFamily(address) {
177
+ return isIP(address) === 4 ? "ipv4" : "ipv6";
178
+ }
179
+ /**
180
+ * Whether the scan's target URL is itself local: its host is localhost (or a name under it), or a literal private or
181
+ * loopback address in any spelling the URL parser accepts. Private access is granted from this alone. It never asks
182
+ * DNS, so a public name that resolves to loopback is not local. A URL that does not parse, or is not http(s), is not.
183
+ */
184
+ export function isLocalTarget(url) {
185
+ let host;
186
+ try {
187
+ const parsed = new URL(url);
188
+ if (parsed.protocol !== "http:" && parsed.protocol !== "https:")
189
+ return false;
190
+ host = parsed.hostname.replace(/^\[|\]$/g, "").replace(/\.$/, "").toLowerCase();
191
+ }
192
+ catch {
193
+ return false;
194
+ }
195
+ if (host === "")
196
+ return false;
197
+ if (host === "localhost" || host.endsWith(".localhost"))
198
+ return true;
199
+ return isIP(host) !== 0 && isPrivateAddress(host);
200
+ }
201
+ /**
202
+ * A `lookup` for a socket that resolves the host with `resolver` and refuses the connection when any address is refused
203
+ * (see refusedAddress). The socket then connects to the address that was checked, so a DNS answer that changes later
204
+ * cannot redirect it.
205
+ */
206
+ export function guardedLookup(resolver, allowPrivate = false, connectAddress) {
207
+ return ((hostname, options, callback) => {
208
+ resolver(hostname).then((addresses) => {
209
+ if (addresses.length === 0 || addresses.some((a) => refusedAddress(a.address, allowPrivate))) {
210
+ callback(new Error(`refused: ${hostname} resolves to a blocked address`));
211
+ return;
212
+ }
213
+ // The checked addresses are what the socket connects to; connectAddress maps them only in tests.
214
+ const connect = connectAddress ? addresses.map((a) => ({ ...a, address: connectAddress(a.address) })) : addresses;
215
+ if (options.all)
216
+ callback(null, connect);
217
+ else
218
+ callback(null, connect[0].address, connect[0].family);
219
+ }, (err) => callback(err));
220
+ });
221
+ }
@@ -0,0 +1,176 @@
1
+ // Animation detection by byte parsing: GIF, animated WebP and APNG. It needs no decoder and no sharp, so a scan can
2
+ // tell an animated file from a still one from the bytes it already downloaded. Only the container structure is read
3
+ // (block headers, chunk lengths and the timing fields); pixels are never decoded.
4
+ const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
5
+ /**
6
+ * Returns the animation in a file, or undefined when the file is not animated (one frame, another format, or a
7
+ * format this does not read). A truncated or corrupt file never throws: it reports what was readable before the
8
+ * damage, or undefined. Used by the scan to decide whether an image is a GIF or animated WebP/APNG that a video
9
+ * encode could replace.
10
+ */
11
+ export function animationOf(buffer) {
12
+ try {
13
+ const info = detect(buffer);
14
+ return info && info.frames > 1 ? info : undefined;
15
+ }
16
+ catch {
17
+ // Every read below is bounds-checked, so this is a last guard for a malformed buffer, not a normal path.
18
+ return undefined;
19
+ }
20
+ }
21
+ function detect(b) {
22
+ const head6 = b.toString("latin1", 0, 6);
23
+ if (head6 === "GIF87a" || head6 === "GIF89a")
24
+ return parseGif(b);
25
+ if (b.length >= 12 && b.toString("latin1", 0, 4) === "RIFF" && b.toString("latin1", 8, 12) === "WEBP") {
26
+ return parseWebp(b);
27
+ }
28
+ if (b.length >= 8 && b.subarray(0, 8).equals(PNG_SIGNATURE))
29
+ return parseApng(b);
30
+ return undefined;
31
+ }
32
+ /** Colour-table size in bytes for a packed-flags byte whose table flag is set (3 bytes per entry, 2^(n+1) entries). */
33
+ function colourTableBytes(packed) {
34
+ return 3 * 2 ** ((packed & 0x07) + 1);
35
+ }
36
+ /**
37
+ * Skips a run of GIF data sub-blocks (a size byte, then that many bytes, ending with a zero-size block) starting at
38
+ * p, and returns the offset just after the terminator. A run that runs past the end returns an offset past the end,
39
+ * which the caller's loop treats as the end of the file.
40
+ */
41
+ function skipSubBlocks(b, p) {
42
+ while (p < b.length) {
43
+ const size = b[p];
44
+ if (size === 0)
45
+ return p + 1;
46
+ p += size + 1;
47
+ }
48
+ return p;
49
+ }
50
+ /**
51
+ * Walks GIF blocks. Frames are counted as image descriptors (0x2C); each frame's delay comes from the Graphic
52
+ * Control Extension (0x21 0xF9) just before it, in hundredths of a second. The loop count comes from the NETSCAPE2.0
53
+ * application extension.
54
+ */
55
+ function parseGif(b) {
56
+ if (b.length < 13)
57
+ return undefined;
58
+ let pos = 13;
59
+ if (b[10] & 0x80)
60
+ pos += colourTableBytes(b[10]);
61
+ let frames = 0;
62
+ let durationMs = 0;
63
+ let pendingDelayCs = 0;
64
+ let loop;
65
+ while (pos < b.length) {
66
+ const introducer = b[pos];
67
+ if (introducer === 0x3b)
68
+ break; // trailer: end of the GIF
69
+ if (introducer === 0x21) {
70
+ // Extension: label, then sub-blocks. Application extensions start with an 11-byte identifier sub-block.
71
+ if (pos + 2 > b.length)
72
+ break;
73
+ const label = b[pos + 1];
74
+ if (label === 0xf9 && pos + 6 <= b.length) {
75
+ pendingDelayCs = b.readUInt16LE(pos + 4);
76
+ }
77
+ else if (label === 0xff && pos + 3 + 11 <= b.length) {
78
+ const identifier = b.toString("latin1", pos + 3, pos + 14);
79
+ if (identifier === "NETSCAPE2.0" || identifier === "ANIMEXTS1.0") {
80
+ const loopBlock = pos + 14;
81
+ if (loopBlock + 5 <= b.length && b[loopBlock] === 3 && b[loopBlock + 1] === 1) {
82
+ loop = b.readUInt16LE(loopBlock + 2);
83
+ }
84
+ }
85
+ }
86
+ pos = skipSubBlocks(b, pos + 2);
87
+ }
88
+ else if (introducer === 0x2c) {
89
+ // Image descriptor: separator, 4 x u16 position and size, packed flags, then LZW code size and sub-blocks.
90
+ if (pos + 10 > b.length)
91
+ break;
92
+ const packed = b[pos + 9];
93
+ let p = pos + 10;
94
+ if (packed & 0x80)
95
+ p += colourTableBytes(packed);
96
+ p += 1; // LZW minimum code size
97
+ frames += 1;
98
+ // A delay of 0 or 1 cs is shown as 10 cs (100 ms), as browsers show it, so the length is not understated.
99
+ durationMs += (pendingDelayCs <= 1 ? 10 : pendingDelayCs) * 10;
100
+ pendingDelayCs = 0;
101
+ pos = skipSubBlocks(b, p);
102
+ }
103
+ else {
104
+ break; // not a block introducer: the file is corrupt from here on
105
+ }
106
+ }
107
+ return { format: "gif", frames, durationMs, loop: loop ?? 1 };
108
+ }
109
+ /**
110
+ * Walks RIFF chunks of a WebP. Animation needs the VP8X animation flag (bit 1 of its first byte). ANIM holds the loop
111
+ * count (u16 at data offset 4). Each ANMF chunk is one frame, and its 24-bit duration is at data offset 12.
112
+ */
113
+ function parseWebp(b) {
114
+ let pos = 12;
115
+ let animated = false;
116
+ let loop = 0;
117
+ let frames = 0;
118
+ let durationMs = 0;
119
+ while (pos + 8 <= b.length) {
120
+ const type = b.toString("latin1", pos, pos + 4);
121
+ const size = b.readUInt32LE(pos + 4);
122
+ const data = pos + 8;
123
+ if (type === "VP8X" && data + 1 <= b.length) {
124
+ animated = (b[data] & 0x02) !== 0;
125
+ }
126
+ else if (type === "ANIM" && data + 6 <= b.length) {
127
+ loop = b.readUInt16LE(data + 4);
128
+ }
129
+ else if (type === "ANMF" && data + 15 <= b.length) {
130
+ frames += 1;
131
+ durationMs += b.readUIntLE(data + 12, 3);
132
+ }
133
+ pos = data + size + (size & 1); // chunks are padded to an even length
134
+ }
135
+ if (!animated)
136
+ return undefined;
137
+ return { format: "webp", frames, durationMs, loop };
138
+ }
139
+ /**
140
+ * Walks PNG chunks of an APNG. The acTL chunk must come before the first IDAT; it holds num_frames and num_plays (0
141
+ * means loop forever). Each fcTL chunk is one frame, and its delay is delay_num / delay_den seconds (a zero
142
+ * delay_den means 100).
143
+ */
144
+ function parseApng(b) {
145
+ let pos = 8;
146
+ let frames;
147
+ let loop = 0;
148
+ let durationMs = 0;
149
+ let sawIdat = false;
150
+ while (pos + 8 <= b.length) {
151
+ const length = b.readUInt32BE(pos);
152
+ const type = b.toString("latin1", pos + 4, pos + 8);
153
+ const data = pos + 8;
154
+ if (data + length > b.length)
155
+ break; // a chunk cut short by truncation
156
+ if (type === "IDAT") {
157
+ sawIdat = true;
158
+ }
159
+ else if (type === "acTL" && !sawIdat && length >= 8) {
160
+ frames = b.readUInt32BE(data);
161
+ loop = b.readUInt32BE(data + 4);
162
+ }
163
+ else if (type === "fcTL" && length >= 26) {
164
+ const delayNum = b.readUInt16BE(data + 20);
165
+ const delayDen = b.readUInt16BE(data + 22) || 100;
166
+ durationMs += (delayNum * 1000) / delayDen;
167
+ }
168
+ else if (type === "IEND") {
169
+ break;
170
+ }
171
+ pos = data + length + 4; // skip the CRC
172
+ }
173
+ if (frames === undefined)
174
+ return undefined;
175
+ return { format: "apng", frames, durationMs: Math.round(durationMs), loop };
176
+ }
@@ -0,0 +1,211 @@
1
+ // The route patterns of a Next.js app, read from its route files: `app/**/page.*` (the App Router) and `pages/**` (the
2
+ // Pages Router). Only the file names are read, not the source. A dynamic segment `[slug]` is `*`; a catch-all `[...slug]`
3
+ // is `**` (one segment or more), and an optional catch-all `[[...slug]]` is `**?` (none or more). Route groups `(shop)` are
4
+ // dropped; parallel slots `@modal`, intercepting routes `(.)photo` and private folders `_lib` are not routes. A URL maps to
5
+ // its most specific matching pattern, and that pattern's route file is the URL's file. Pure functions over the file list and
6
+ // the URLs: no files are read here.
7
+ import { stripControls } from "./text.js";
8
+ /** The App Router page file, under app/ or src/app/ (the segment path before `page.*`). */
9
+ const APP_PAGE = /^(?:src\/)?app\/(?:(.*)\/)?page\.(?:tsx|ts|jsx|js|mdx)$/;
10
+ /** The Pages Router file, under pages/ or src/pages/ (the path before the extension). */
11
+ const PAGES_FILE = /^(?:src\/)?pages\/(.+)\.(?:tsx|ts|jsx|js|mdx)$/;
12
+ /** Pages Router files that are not routes of their own. */
13
+ const PAGES_SPECIAL = new Set(["_app", "_document", "_error", "404", "500"]);
14
+ /** The segment list of a route folder or page path, or undefined when the path is not a route. */
15
+ function segmentsOf(folders, app) {
16
+ const out = [];
17
+ for (const seg of folders) {
18
+ if (seg === "")
19
+ continue;
20
+ if (app) {
21
+ if (seg.startsWith("@"))
22
+ return undefined; // a parallel slot
23
+ if (/^\(\.{1,3}\)/.test(seg))
24
+ return undefined; // an intercepting route: (.) (..) (...)
25
+ if (seg.startsWith("_"))
26
+ return undefined; // a private folder
27
+ if (/^\(.*\)$/.test(seg))
28
+ continue; // a route group
29
+ }
30
+ if (/^\[\[\.\.\.[^\]]+\]\]$/.test(seg)) {
31
+ out.push("**?");
32
+ }
33
+ else if (/^\[\.\.\.[^\]]+\]$/.test(seg)) {
34
+ out.push("**");
35
+ }
36
+ else if (/^\[[^\]]+\]$/.test(seg)) {
37
+ out.push("*");
38
+ }
39
+ else {
40
+ out.push(seg);
41
+ }
42
+ }
43
+ return out;
44
+ }
45
+ /** The route of one file, or undefined when the file is not a page. */
46
+ function routeOf(file) {
47
+ let segments;
48
+ const app = APP_PAGE.exec(file);
49
+ if (app) {
50
+ segments = segmentsOf((app[1] ?? "").split("/"), true);
51
+ }
52
+ else {
53
+ const pages = PAGES_FILE.exec(file);
54
+ if (pages === null)
55
+ return undefined;
56
+ const parts = pages[1].split("/");
57
+ if (parts[0] === "api" || PAGES_SPECIAL.has(parts[parts.length - 1]))
58
+ return undefined;
59
+ if (parts[parts.length - 1] === "index")
60
+ parts.pop();
61
+ segments = segmentsOf(parts, false);
62
+ }
63
+ if (segments === undefined)
64
+ return undefined;
65
+ const pattern = segments.length === 0 ? "/" : `/${segments.join("/")}`;
66
+ return { pattern, segments, file };
67
+ }
68
+ /**
69
+ * The routes of a list of files (paths relative to the app folder's root, forward slashes). A pattern that two files give
70
+ * keeps the first one in path order, so the result is the same on every run.
71
+ */
72
+ export function routesFromFiles(files) {
73
+ const seen = new Map();
74
+ for (const file of [...files].sort()) {
75
+ const route = routeOf(file);
76
+ if (route !== undefined && !seen.has(route.pattern))
77
+ seen.set(route.pattern, route);
78
+ }
79
+ return [...seen.values()];
80
+ }
81
+ /** The rank of each segment of a pattern that matches a path: static 3, one segment 2, the rest of the path 1. */
82
+ function rankOf(segments, path) {
83
+ const wild = segments.findIndex((s) => s.startsWith("**"));
84
+ const head = wild === -1 ? segments : segments.slice(0, wild);
85
+ // A required catch-all needs one segment or more; an optional one (**?) matches the head alone too.
86
+ if (wild === -1 ? segments.length !== path.length : path.length < head.length + (segments[wild] === "**" ? 1 : 0))
87
+ return undefined;
88
+ const ranks = [];
89
+ for (let i = 0; i < head.length; i++) {
90
+ if (head[i] === "*") {
91
+ if (path[i] === "")
92
+ return undefined;
93
+ ranks.push(2);
94
+ }
95
+ else if (head[i] === path[i]) {
96
+ ranks.push(3);
97
+ }
98
+ else {
99
+ return undefined;
100
+ }
101
+ }
102
+ if (wild !== -1)
103
+ ranks.push(1);
104
+ return ranks;
105
+ }
106
+ /** Negative when `a` is more specific than `b` (the first differing segment rank decides; a missing rank is 0). */
107
+ function compareRanks(a, b) {
108
+ for (let i = 0; i < Math.max(a.length, b.length); i++) {
109
+ const diff = (b[i] ?? 0) - (a[i] ?? 0);
110
+ if (diff !== 0)
111
+ return diff;
112
+ }
113
+ return 0;
114
+ }
115
+ /** The path segments of a URL (decoded), or undefined when the URL does not parse. */
116
+ export function pathSegmentsOf(url) {
117
+ try {
118
+ return new URL(url).pathname
119
+ .split("/")
120
+ .filter((s) => s !== "")
121
+ .map((s) => {
122
+ // Control characters out of decoded segments (see text.ts): a route type is printed as it is.
123
+ try {
124
+ return stripControls(decodeURIComponent(s));
125
+ }
126
+ catch {
127
+ return stripControls(s);
128
+ }
129
+ });
130
+ }
131
+ catch {
132
+ return undefined;
133
+ }
134
+ }
135
+ /** The most specific route that matches a URL's path, or undefined. Ties keep the earlier route of the list. */
136
+ export function matchRoute(url, routes) {
137
+ const path = pathSegmentsOf(url);
138
+ if (path === undefined)
139
+ return undefined;
140
+ let best;
141
+ for (const route of routes) {
142
+ const ranks = rankOf(route.segments, path);
143
+ if (ranks === undefined)
144
+ continue;
145
+ if (best === undefined || compareRanks(ranks, best.ranks) < 0)
146
+ best = { route, ranks };
147
+ }
148
+ return best?.route;
149
+ }
150
+ /**
151
+ * The route file of each URL of a set: the file of its most specific matching route, or undefined when no route matches.
152
+ * The route type of a URL is its URL template (routes.ts); the file is an attribute of the URL, for the report and the
153
+ * locate step.
154
+ */
155
+ export function routeFileOf(url, routes) {
156
+ return matchRoute(url, routes)?.file;
157
+ }
158
+ /** A segment of a pattern that takes any value (`*`, `**`, `**?`). */
159
+ const isDynamic = (segment) => segment.startsWith("*");
160
+ /** The static routes (no parameters) as URLs of an origin: they are seeds of a crawl even when no link reaches them. */
161
+ export function staticRouteUrls(origin, routes) {
162
+ return seedUrlsOf(origin, routes, []);
163
+ }
164
+ /**
165
+ * The seed URLs of an origin from the app's routes: every static route, and every route under a dynamic first segment
166
+ * (`/[store]/about-us`) once for each learned value of that segment (see learnedValues). A route with a dynamic segment
167
+ * after its first one is not a seed: a crawl finds those pages from links.
168
+ */
169
+ export function seedUrlsOf(origin, routes, values) {
170
+ const out = [];
171
+ for (const route of routes) {
172
+ if (!route.segments.some(isDynamic)) {
173
+ out.push(route.pattern === "/" ? `${origin}/` : `${origin}${route.pattern}`);
174
+ }
175
+ else if (route.segments[0] === "*" && !route.segments.slice(1).some(isDynamic)) {
176
+ const rest = route.segments.slice(1);
177
+ for (const value of values)
178
+ out.push([origin, value, ...rest].join("/"));
179
+ }
180
+ }
181
+ return [...new Set(out)];
182
+ }
183
+ /** The first path segment of a URL as written (not decoded), or undefined for the home page or a URL that does not parse. */
184
+ function firstSegmentOf(url) {
185
+ try {
186
+ return new URL(url).pathname.split("/").filter((s) => s !== "")[0];
187
+ }
188
+ catch {
189
+ return undefined;
190
+ }
191
+ }
192
+ /**
193
+ * The values a dynamic first segment (`[store]`, `[locale]`) takes on a site, learned from the URLs the home page led to.
194
+ * `seen` is the home page's final URL first, then its links. The first segment of the final URL (where the home page
195
+ * redirects) comes first; the other values follow, most frequent first. A segment that names a static route of the app
196
+ * (`about-us`, `api`) is not a value, and nor is one with a dot. At most `max` values.
197
+ */
198
+ export function learnedValues(seen, routes, max) {
199
+ const reserved = new Set(routes.filter((r) => r.segments[0] !== undefined && !isDynamic(r.segments[0])).map((r) => r.segments[0]));
200
+ const valid = (segment) => segment !== undefined && segment !== "" && !segment.includes(".") && !segment.startsWith("_") && !reserved.has(segment) && segment !== "api";
201
+ const counts = new Map();
202
+ for (const url of seen.slice(1)) {
203
+ const segment = firstSegmentOf(url);
204
+ if (valid(segment))
205
+ counts.set(segment, (counts.get(segment) ?? 0) + 1);
206
+ }
207
+ const first = firstSegmentOf(seen[0] ?? "");
208
+ const ranked = [...counts.entries()].sort((a, b) => b[1] - a[1]).map(([segment]) => segment);
209
+ const values = valid(first) ? [first, ...ranked.filter((segment) => segment !== first)] : ranked;
210
+ return values.slice(0, Math.max(0, max));
211
+ }
package/dist/atomic.js ADDED
@@ -0,0 +1,63 @@
1
+ import { createHash, randomBytes } from "node:crypto";
2
+ import { basename, dirname, isAbsolute, join, relative, sep } from "node:path";
3
+ import { lstat, open, realpath, rename, stat, unlink } from "node:fs/promises";
4
+ export function sha1(data) {
5
+ return createHash("sha1").update(data).digest("hex");
6
+ }
7
+ const isNotFound = (err) => err.code === "ENOENT";
8
+ /** True when `path` is `root` (if allowed) or lies below it. Both paths must already be resolved. */
9
+ function inside(root, path, allowRoot) {
10
+ const rel = relative(root, path);
11
+ if (rel === "")
12
+ return allowRoot;
13
+ return rel !== ".." && !rel.startsWith(`..${sep}`) && !isAbsolute(rel);
14
+ }
15
+ /**
16
+ * The file a write lands in. The parent directory must resolve inside the root. A symlink at `path` is refused.
17
+ */
18
+ async function resolveWriteTarget(path, opts) {
19
+ const rootReal = await realpath(opts.root);
20
+ const parentReal = await realpath(dirname(path));
21
+ if (!inside(rootReal, parentReal, true))
22
+ throw new Error(`refusing to write outside the project: ${path}`);
23
+ const plain = join(parentReal, basename(path));
24
+ const info = await lstat(path).catch((err) => {
25
+ if (isNotFound(err))
26
+ return undefined;
27
+ throw err;
28
+ });
29
+ if (info?.isSymbolicLink())
30
+ throw new Error(`refusing to write through symlink: ${path}`);
31
+ return plain;
32
+ }
33
+ /**
34
+ * Writes `data` to a sibling temp file, then renames it over the target, so a crash never leaves a half-written file.
35
+ * The temp name is random and created exclusively, so concurrent writers never share one. An existing file keeps its
36
+ * permission bits. Symlinks are refused.
37
+ */
38
+ export async function writeFileAtomic(path, data, opts) {
39
+ const target = await resolveWriteTarget(path, opts);
40
+ const mode = await stat(target).then((s) => s.mode & 0o777, () => opts.mode);
41
+ const tmp = `${target}.${randomBytes(6).toString("hex")}.tmp`;
42
+ let handle;
43
+ let created = false;
44
+ try {
45
+ handle = await open(tmp, "wx");
46
+ // Only a temp file this call created may be removed on failure.
47
+ created = true;
48
+ if (mode !== undefined)
49
+ await handle.chmod(mode);
50
+ await handle.writeFile(typeof data === "string" ? Buffer.from(data, "utf8") : data);
51
+ if (opts.sync)
52
+ await handle.sync();
53
+ await handle.close();
54
+ handle = undefined;
55
+ await rename(tmp, target);
56
+ }
57
+ catch (err) {
58
+ await handle?.close().catch(() => undefined);
59
+ if (created)
60
+ await unlink(tmp).catch(() => undefined);
61
+ throw err;
62
+ }
63
+ }