imgtune 0.0.0-stage → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. package/CHANGELOG.md +322 -0
  2. package/LICENSE +21 -0
  3. package/NOTICE +20 -0
  4. package/README.md +350 -2
  5. package/THIRD_PARTY_LICENSES +295 -0
  6. package/dist/address.js +221 -0
  7. package/dist/animated.js +176 -0
  8. package/dist/app-routes.js +211 -0
  9. package/dist/atomic.js +63 -0
  10. package/dist/audit.js +372 -0
  11. package/dist/backgrounds.js +363 -0
  12. package/dist/broken.js +106 -0
  13. package/dist/browser.js +30 -0
  14. package/dist/calibrate.js +221 -0
  15. package/dist/candidates.js +151 -0
  16. package/dist/check-report.js +302 -0
  17. package/dist/check.js +416 -0
  18. package/dist/cli.js +531 -0
  19. package/dist/compare.js +398 -0
  20. package/dist/concurrency.js +37 -0
  21. package/dist/config.js +10 -0
  22. package/dist/crawl.js +186 -0
  23. package/dist/deep.js +76 -0
  24. package/dist/egress-proxy.js +151 -0
  25. package/dist/encode.js +101 -0
  26. package/dist/estimate-jobs.js +114 -0
  27. package/dist/fetch.js +179 -0
  28. package/dist/ffmpeg.js +495 -0
  29. package/dist/gpu.js +353 -0
  30. package/dist/group-sizes.js +120 -0
  31. package/dist/hints.js +1102 -0
  32. package/dist/image-file.js +553 -0
  33. package/dist/images.js +361 -0
  34. package/dist/layout.js +240 -0
  35. package/dist/lcp.js +66 -0
  36. package/dist/loaders.js +267 -0
  37. package/dist/loads.js +29 -0
  38. package/dist/locate-join.js +511 -0
  39. package/dist/locate-only.js +178 -0
  40. package/dist/locate.js +382 -0
  41. package/dist/lru.js +34 -0
  42. package/dist/measure.js +238 -0
  43. package/dist/media-encode.js +1050 -0
  44. package/dist/media-file.js +284 -0
  45. package/dist/media-limits.js +31 -0
  46. package/dist/media-model.js +247 -0
  47. package/dist/media-phase.js +652 -0
  48. package/dist/media-prepare.js +31 -0
  49. package/dist/metric.js +227 -0
  50. package/dist/next-config.js +87 -0
  51. package/dist/qualities-cap.js +141 -0
  52. package/dist/recheck-rows.js +40 -0
  53. package/dist/recheck.js +376 -0
  54. package/dist/remote.js +3582 -0
  55. package/dist/report.js +253 -0
  56. package/dist/result-cache.js +183 -0
  57. package/dist/routes.js +280 -0
  58. package/dist/run.js +445 -0
  59. package/dist/search.js +88 -0
  60. package/dist/site.js +31 -0
  61. package/dist/sizes-outcome.js +81 -0
  62. package/dist/sizes-recheck.js +332 -0
  63. package/dist/sizes.js +148 -0
  64. package/dist/source-root.js +77 -0
  65. package/dist/srcset-parse.js +98 -0
  66. package/dist/srcset.js +44 -0
  67. package/dist/ssim.js +196 -0
  68. package/dist/stage.js +51 -0
  69. package/dist/summary.js +495 -0
  70. package/dist/survey.js +13 -0
  71. package/dist/text.js +72 -0
  72. package/dist/third-party.js +127 -0
  73. package/dist/timing.js +60 -0
  74. package/dist/types.js +1 -0
  75. package/dist/urls.js +300 -0
  76. package/dist/version.js +3 -0
  77. package/dist/video-analysis.js +273 -0
  78. package/dist/webp-animation.js +283 -0
  79. package/dist/worst.js +42 -0
  80. package/package.json +70 -4
  81. package/skills/imgtune/AGENTS.snippet.md +47 -0
  82. package/skills/imgtune/SKILL.md +344 -0
  83. package/skills/imgtune/reference.md +339 -0
  84. package/vendor/ssimulacra2/ssim_wasm.cjs +113 -0
  85. package/vendor/ssimulacra2/ssim_wasm.d.ts +8 -0
  86. package/vendor/ssimulacra2/ssim_wasm_bg.wasm +0 -0
  87. package/vendor/ssimulacra2/ssim_wasm_bg.wasm.d.ts +6 -0
package/dist/audit.js ADDED
@@ -0,0 +1,372 @@
1
+ // `imgtune audit`: the deep mode for your own Next.js app. It reads the app's route files (app/ and pages/, with src/) and its
2
+ // next.config (the allowed qualities), measures the production origin (--prod) the way `check` does, and, with --dev,
3
+ // locates the deep pages' images in the sources through the dev server. The report (report.md) names the file:line of every
4
+ // located fix and the route file of every route type. Preflight: the production home must load its stylesheets, or the
5
+ // audit stops with the likely cause (a CSS host the machine cannot reach: the CDN variable must be empty for next build and
6
+ // next start).
7
+ import { mkdir, readdir, readFile } from "node:fs/promises";
8
+ import { join, resolve } from "node:path";
9
+ import { DEFAULT_VIEWPORTS } from "./config.js";
10
+ import { BROKEN_EXIT, brokenOf } from "./broken.js";
11
+ import { locateCommand } from "./locate-only.js";
12
+ import { isLocalTarget } from "./address.js";
13
+ import { learnedValues, matchRoute, routeFileOf, routesFromFiles, seedUrlsOf, staticRouteUrls } from "./app-routes.js";
14
+ import { CRAWL_DEPTH, crawlSite } from "./crawl.js";
15
+ import { nextQualities } from "./next-config.js";
16
+ import { DEFAULT_CHECK_SAMPLE, finishCheck, parseCheckFlags, routeMapOf, sampleUrls, scanCheckPages } from "./check.js";
17
+ import { groupRoutes, routeGrouping } from "./routes.js";
18
+ import { urlsFromSource } from "./urls.js";
19
+ import { devUrlOf } from "./run.js";
20
+ import { pageFileName, parseLimit, readStorageState, runScan, scanPlan } from "./remote.js";
21
+ import { safeText, shellWord } from "./text.js";
22
+ /** Where an audit's directory goes when --out is not given: a timestamped folder under this. */
23
+ export const DEFAULT_AUDIT_DIR = ".imgtune/audit";
24
+ /** The folders under the app root that hold route files. */
25
+ const SOURCE_DIRS = ["app", "src/app", "pages", "src/pages"];
26
+ const AUDIT_USAGE = "usage: imgtune audit --app <dir> --prod <origin> [--dev <origin>] [--urls-from <file|sitemap-url>] [--max-locales N] [--sample-per-pattern N] [--deep N] [--route-map file] [--out dir] [--qualities list] [--gpu] [--gpu-device n] [--page-concurrency N] [--idle-timeout ms] [--no-cache] [--ignore-robots] [--storage-state file] [--block-third-party] [--allow-host h1,h2]";
27
+ /** The values of a dynamic first segment (`[store]`, `[locale]`) the crawl expands, when --max-locales is not given. */
28
+ export const DEFAULT_MAX_LOCALES = 3;
29
+ /** The sitemap URLs an audit takes, at most: the rest are not listed (the report says how many there were). */
30
+ export const SITEMAP_URL_CAP = 5000;
31
+ function originOf(text, name) {
32
+ let url;
33
+ try {
34
+ url = new URL(text);
35
+ }
36
+ catch {
37
+ throw new Error(`${name} needs an http or https origin, not ${safeText(text)} (${AUDIT_USAGE})`);
38
+ }
39
+ if (url.protocol !== "http:" && url.protocol !== "https:")
40
+ throw new Error(`${name} needs an http or https origin (${AUDIT_USAGE})`);
41
+ return url.origin;
42
+ }
43
+ /** The arguments after `audit`. Throws with the message the CLI prints. */
44
+ export function parseAuditArgs(argv) {
45
+ let app;
46
+ let prod;
47
+ let dev;
48
+ let maxLocales = DEFAULT_MAX_LOCALES;
49
+ const rest = [];
50
+ for (let i = 0; i < argv.length; i++) {
51
+ const arg = argv[i];
52
+ if (arg === "--max-locales") {
53
+ const value = argv[i + 1];
54
+ if (value === undefined || value.startsWith("--"))
55
+ throw new Error(`${arg} needs a value (${AUDIT_USAGE})`);
56
+ maxLocales = parseLimit(arg, value);
57
+ i++;
58
+ }
59
+ else if (arg === "--app" || arg === "--prod" || arg === "--dev") {
60
+ const value = argv[i + 1];
61
+ if (value === undefined || value.startsWith("--"))
62
+ throw new Error(`${arg} needs a value (${AUDIT_USAGE})`);
63
+ i++;
64
+ if (arg === "--app")
65
+ app = value;
66
+ else if (arg === "--prod")
67
+ prod = originOf(value, "--prod");
68
+ else
69
+ dev = originOf(value, "--dev");
70
+ }
71
+ else {
72
+ rest.push(arg);
73
+ }
74
+ }
75
+ if (app === undefined)
76
+ throw new Error(`audit needs --app <dir> (${AUDIT_USAGE})`);
77
+ if (prod === undefined)
78
+ throw new Error(`audit needs --prod <origin> (${AUDIT_USAGE})`);
79
+ const flags = parseCheckFlags(rest);
80
+ if (flags.target !== undefined)
81
+ throw new Error(`audit takes --prod for the site, not the URL ${safeText(flags.target)} (${AUDIT_USAGE})`);
82
+ return { app, prod, dev, maxLocales, flags };
83
+ }
84
+ const stamp = (date) => date.toISOString().replace(/[:.]/g, "-");
85
+ const secondsSince = (startedAt) => Math.round((performance.now() - startedAt) / 100) / 10;
86
+ const viewportsOf = (names) => DEFAULT_VIEWPORTS.filter((v) => names === undefined || names.includes(v.name)).map((v) => ({ ...v }));
87
+ /** The route files of an app: the files under app/, src/app/, pages/ and src/pages/ (paths relative to the app root). */
88
+ export async function sourceFilesOf(appRoot) {
89
+ const files = [];
90
+ const walk = async (absolute, relative) => {
91
+ let entries;
92
+ try {
93
+ entries = await readdir(absolute, { withFileTypes: true });
94
+ }
95
+ catch {
96
+ return; // no such folder
97
+ }
98
+ for (const entry of entries) {
99
+ if (entry.name === "node_modules" || entry.name.startsWith("."))
100
+ continue;
101
+ const path = `${relative}/${entry.name}`;
102
+ if (entry.isDirectory())
103
+ await walk(join(absolute, entry.name), path);
104
+ else if (entry.isFile())
105
+ files.push(path);
106
+ }
107
+ };
108
+ for (const dir of SOURCE_DIRS)
109
+ await walk(join(appRoot, dir), dir);
110
+ return files;
111
+ }
112
+ /**
113
+ * The URLs of an audit. --urls-from replaces the discovery: its URLs as given, and no seeds are added. Otherwise the site is
114
+ * crawled from its home page and from the app's seeds (its static routes, and its dynamic first segments expanded for the
115
+ * values the home page leads to; see crawl.ts). The sitemap of --prod adds its URLs (capped, see SITEMAP_URL_CAP). A sitemap
116
+ * that lists another host, when --prod is local, is read on --prod: its URLs are moved onto that origin, and the audit says so.
117
+ */
118
+ export async function auditUrls(args, appRoutes, cwd, log) {
119
+ if (args.flags.urlsFrom !== undefined) {
120
+ const source = await urlsFromSource(args.flags.urlsFrom, cwd);
121
+ log(`--urls-from: ${source.urls.length} URL(s) from ${safeText(args.flags.urlsFrom)}`);
122
+ return [...new Set(source.urls)];
123
+ }
124
+ const prod = args.prod;
125
+ const sitemap = `${prod}/sitemap.xml`;
126
+ let sitemapUrls = [];
127
+ try {
128
+ const source = await urlsFromSource(sitemap, cwd, { rewriteOriginTo: isLocalTarget(prod) ? prod : undefined });
129
+ if (source.urls.length > 0) {
130
+ sitemapUrls = source.urls.slice(0, SITEMAP_URL_CAP);
131
+ log(`sitemap ${sitemap}: ${source.urls.length} URL(s)${source.urls.length > sitemapUrls.length ? `, the first ${sitemapUrls.length} are used` : ""}`);
132
+ }
133
+ if (source.rewritten > 0) {
134
+ log(`sitemap ${sitemap} lists ${source.rewritten} URL(s) on another host: they are read on ${safeText(prod)} instead (the other host is not fetched)`);
135
+ }
136
+ }
137
+ catch (err) {
138
+ log(`no sitemap at ${sitemap} (${safeText(err.message, 200)})`);
139
+ }
140
+ const seeds = staticRouteUrls(prod, appRoutes);
141
+ const dynamicFirst = appRoutes.some((r) => r.segments[0] === "*");
142
+ const crawled = await crawlSite(prod, {
143
+ depth: CRAWL_DEPTH,
144
+ seeds,
145
+ seedsAfterHome: (seen) => {
146
+ if (!dynamicFirst)
147
+ return [];
148
+ const values = learnedValues(seen, appRoutes, args.maxLocales);
149
+ log(`dynamic first segment: ${values.length} value(s) learned from the home page: ${values.map((v) => safeText(v, 80)).join(", ") || "none"}`);
150
+ return seedUrlsOf(prod, appRoutes, values);
151
+ },
152
+ });
153
+ log(`crawl from ${safeText(prod)}: ${crawled.length} page(s) from ${seeds.length} static seed(s) and the home page, up to ${CRAWL_DEPTH} links deep`);
154
+ return [...new Set([...crawled, ...sitemapUrls])];
155
+ }
156
+ /**
157
+ * The route types of an audit: groupRoutes (routes.ts) over the URLs, where the first segment of a URL that the app takes as a
158
+ * dynamic segment (`[store]`, `[locale]`) is stripped first. That segment is a dimension of the site, not part of the page's
159
+ * template, whatever its shape: groupRoutes strips only a locale-shaped one (`ae-en`), and a store named `el-seed` would
160
+ * otherwise be a route type of its own. The device variant after it (`desktop`, `mobile`) is stripped the same way, as groupRoutes
161
+ * strips it after a locale. The stripped values are kept as the URL's locale and variant. --route-map patterns match the path
162
+ * after those segments.
163
+ */
164
+ export function groupAuditRoutes(urls, routes, routeMap) {
165
+ const distinct = [...new Set(urls)];
166
+ const keyOf = new Map();
167
+ const dimensionOf = new Map();
168
+ for (const url of distinct) {
169
+ const route = matchRoute(url, routes);
170
+ let parsed;
171
+ try {
172
+ parsed = new URL(url);
173
+ }
174
+ catch {
175
+ parsed = undefined;
176
+ }
177
+ if (route === undefined || route.segments[0] !== "*" || parsed === undefined) {
178
+ keyOf.set(url, url);
179
+ continue;
180
+ }
181
+ const segments = parsed.pathname.split("/").filter((s) => s !== "");
182
+ let rest = segments.slice(1);
183
+ const variant = rest[0] === "desktop" || rest[0] === "mobile" ? rest[0] : undefined;
184
+ if (variant !== undefined)
185
+ rest = rest.slice(1);
186
+ dimensionOf.set(url, variant === undefined ? { locale: segments[0] } : { locale: segments[0], variant });
187
+ const stripped = new URL(url);
188
+ stripped.pathname = `/${rest.join("/")}`;
189
+ keyOf.set(url, stripped.href);
190
+ }
191
+ const grouped = groupRoutes([...new Set(keyOf.values())], { routeMap });
192
+ const byKey = new Map(grouped.routes.map((r) => [r.url, r]));
193
+ const out = distinct.map((url) => {
194
+ const key = keyOf.get(url);
195
+ const found = byKey.get(key);
196
+ const route = { url, type: found.type, mapped: found.mapped };
197
+ const dimension = dimensionOf.get(url);
198
+ if (dimension !== undefined) {
199
+ route.locale = dimension.locale;
200
+ if (dimension.variant !== undefined)
201
+ route.variant = dimension.variant;
202
+ }
203
+ return route;
204
+ });
205
+ return routeGrouping(out);
206
+ }
207
+ /** The route files of each route type: the files of its URLs' most specific routes, in the order the types are listed. */
208
+ export function routeFilesByType(grouping, routes) {
209
+ const out = new Map();
210
+ for (const type of grouping.types) {
211
+ const files = new Set();
212
+ for (const url of type.urls) {
213
+ const file = routeFileOf(url, routes);
214
+ if (file !== undefined)
215
+ files.add(file);
216
+ }
217
+ out.set(type.type, [...files]);
218
+ }
219
+ return out;
220
+ }
221
+ /**
222
+ * The preflight: the production home is loaded once (survey viewports). A home page whose stylesheets did not load is
223
+ * returned as the broken info (the audit stops); any other outcome lets the audit go on.
224
+ */
225
+ async function preflight(prod, settings, outRoot, cwd, log, storageState) {
226
+ const plan = scanPlan({ urls: [], out: outRoot, aggressive: false, quality: true, thirdParty: settings.thirdParty, survey: true });
227
+ const pages = await runScan([`${prod}/`], {
228
+ cwd,
229
+ log,
230
+ storageState,
231
+ cache: !settings.noCache,
232
+ ignoreRobots: settings.ignoreRobots,
233
+ aggressive: false,
234
+ quality: false,
235
+ thirdParty: settings.thirdParty,
236
+ outDir: join(outRoot, "preflight"),
237
+ viewports: viewportsOf(plan.viewports),
238
+ survey: true,
239
+ pageConcurrency: plan.pageConcurrency,
240
+ idleTimeoutMs: plan.idleTimeoutMs,
241
+ });
242
+ const home = pages[0];
243
+ if (home === undefined)
244
+ return undefined;
245
+ const info = home.broken ?? brokenOf(home);
246
+ return info?.kind === "stylesheet" ? info : undefined;
247
+ }
248
+ /** The terminal text of a failed preflight: the cause, and the fix for it. */
249
+ function preflightText(info, prod) {
250
+ return [
251
+ `imgtune audit: the home page of ${safeText(prod)} did not load its stylesheets, so its images would be measured on an unstyled page.`,
252
+ "likely cause: the CSS is served from a CDN host this machine cannot reach. Rebuild and restart the app with the CDN variable",
253
+ "empty for BOTH `next build` and `next start` (for example NEXT_PUBLIC_CDN_URL= ), then run the audit again.",
254
+ `reason: ${safeText(info.reason, 300)}`,
255
+ `fix: ${safeText(info.fix, 300)}`,
256
+ "",
257
+ ].join("\n");
258
+ }
259
+ /**
260
+ * The locate step: each measured deep page is located at the same path on the dev server, with the app as the source root.
261
+ * A page that does not locate keeps its measured record; the warnings name it.
262
+ */
263
+ async function locateDeep(input) {
264
+ const started = performance.now();
265
+ const deepDir = join(input.outRoot, "deep");
266
+ const locatedDir = join(input.outRoot, "located");
267
+ const out = [];
268
+ let attempted = 0;
269
+ let located = 0;
270
+ for (const page of input.deep) {
271
+ if (page.status !== "ok" || page.page === undefined) {
272
+ out.push(page);
273
+ continue;
274
+ }
275
+ attempted++;
276
+ const name = pageFileName(new URL(page.url));
277
+ const from = join(deepDir, name);
278
+ const to = join(locatedDir, name.replace(/\.json$/, ".located.json"));
279
+ // The dev server is asked for the page the measure ended on: a page that redirects (the home page to its locale) would be refused.
280
+ const dev = devUrlOf(input.dev, page.finalUrl ?? page.url);
281
+ const args = [dev, "--locate-only", "--from", from, "--locate-out", to, "--root", input.appRoot];
282
+ if (input.ignoreRobots)
283
+ args.push("--ignore-robots");
284
+ input.commands.push({
285
+ phase: "locate",
286
+ command: ["imgtune scan", shellWord(dev), "--locate-only", "--from", shellWord(from), "--locate-out", shellWord(to), "--root", shellWord(input.appRoot)].join(" "),
287
+ });
288
+ try {
289
+ const outcome = await locateCommand(args, input.cwd, input.log);
290
+ if (outcome.exitCode !== 0)
291
+ input.warnings.push(`located ${safeText(page.url, 200)}: nothing located at the dev server`);
292
+ out.push(JSON.parse(await readFile(to, "utf8")));
293
+ located++;
294
+ }
295
+ catch (err) {
296
+ input.warnings.push(`located ${safeText(page.url, 200)}: ${safeText(err.message, 300)}`);
297
+ out.push(page);
298
+ }
299
+ }
300
+ return { deep: out, phase: { name: "locate", seconds: secondsSince(started), pages: attempted, ok: located } };
301
+ }
302
+ /** `imgtune audit`: the arguments after `audit`. Writes the audit directory and returns the terminal text and the exit code. */
303
+ export async function auditCommand(argv, cwd, log) {
304
+ const args = parseAuditArgs(argv);
305
+ const started = new Date();
306
+ const appRoot = resolve(cwd, args.app);
307
+ const outRoot = resolve(cwd, args.flags.out ?? join(DEFAULT_AUDIT_DIR, stamp(started)));
308
+ // 1. The source: the route files, and the allowed qualities (--qualities overrides next.config).
309
+ const appRoutes = routesFromFiles(await sourceFilesOf(appRoot));
310
+ if (appRoutes.length === 0)
311
+ throw new Error(`no route files under ${safeText(args.app)}: expected app/**/page.* (or src/app) or pages/**`);
312
+ const quality = args.flags.qualities !== undefined ? { qualities: args.flags.qualities, note: "allowed qualities from --qualities" } : await nextQualities(appRoot);
313
+ log(`app: ${appRoutes.length} route(s); ${safeText(quality.note)}`);
314
+ const settings = { ...args.flags, qualities: quality.qualities };
315
+ // 2. Preflight: the production home must load its stylesheets.
316
+ await mkdir(outRoot, { recursive: true });
317
+ const storageState = args.flags.storageState !== undefined ? await readStorageState(resolve(cwd, args.flags.storageState)) : undefined;
318
+ const broken = await preflight(args.prod, settings, outRoot, cwd, log, storageState);
319
+ if (broken !== undefined)
320
+ return { output: preflightText(broken, args.prod), exitCode: BROKEN_EXIT };
321
+ // 3. URLs, route types (the route file of each URL), the sample, and the survey and deep scan.
322
+ const listed = await auditUrls(args, appRoutes, cwd, log);
323
+ if (listed.length === 0)
324
+ throw new Error("no URL to audit");
325
+ const routeMap = await routeMapOf(args.flags.routeMap, cwd);
326
+ const fullRoutes = groupAuditRoutes(listed, appRoutes, routeMap);
327
+ const routeFiles = routeFilesByType(fullRoutes, appRoutes);
328
+ const sample = args.flags.samplePerPattern ?? DEFAULT_CHECK_SAMPLE;
329
+ const picked = sampleUrls(listed, fullRoutes, sample, log);
330
+ const scans = await scanCheckPages({ cwd, log, outRoot, urls: picked.urls, routes: picked.routes, settings, storageState });
331
+ // 4. Locate the deep pages on the dev server, when --dev is given.
332
+ const warnings = [];
333
+ const extraCommands = [];
334
+ const extraPhases = [];
335
+ if (args.dev !== undefined) {
336
+ const located = await locateDeep({
337
+ dev: args.dev,
338
+ appRoot,
339
+ cwd,
340
+ log,
341
+ outRoot,
342
+ deep: scans.deep,
343
+ ignoreRobots: settings.ignoreRobots,
344
+ warnings,
345
+ commands: extraCommands,
346
+ });
347
+ scans.deep = located.deep;
348
+ extraPhases.push(located.phase);
349
+ }
350
+ // 5. The report and the manifest.
351
+ const finished = await finishCheck({
352
+ mode: "audit",
353
+ title: "imgtune audit",
354
+ cwd,
355
+ outRoot,
356
+ started,
357
+ site: args.prod,
358
+ listed: listed.length,
359
+ routes: picked.routes,
360
+ settings,
361
+ sample,
362
+ scans,
363
+ storageState: args.flags.storageState !== undefined,
364
+ extraPhases,
365
+ extraCommands,
366
+ warnings,
367
+ qualitiesNote: quality.note,
368
+ dev: args.dev,
369
+ routeFiles,
370
+ });
371
+ return { output: finished.output, exitCode: finished.exitCode };
372
+ }